diff --git a/mindspore/ccsrc/distributed/cluster/actor_route_table_proxy.cc b/mindspore/ccsrc/distributed/cluster/actor_route_table_proxy.cc index 461388b76af..1645fc68706 100644 --- a/mindspore/ccsrc/distributed/cluster/actor_route_table_proxy.cc +++ b/mindspore/ccsrc/distributed/cluster/actor_route_table_proxy.cc @@ -14,78 +14,149 @@ * limitations under the License. */ +// Include the standard string header for string manipulation #include + +// Include the standard vector header for vector container #include + +// Include the custom header file "distributed/cluster/actor_route_table_proxy.h" #include "distributed/cluster/actor_route_table_proxy.h" +// Start of the "mindspore" namespace namespace mindspore { -namespace distributed { -namespace cluster { -bool ActorRouteTableProxy::RegisterRoute(const std::string &actor_id, const ActorAddress &actor_addr) { - MS_EXCEPTION_IF_NULL(node_); - std::shared_ptr> output = nullptr; - if (!node_->SendToScheduler(actor_addr.SerializeAsString().data(), actor_addr.SerializeAsString().size(), - NodeCommand::REGISTER_ACTOR_ROUTE, &output)) { - MS_LOG(EXCEPTION) << "Failed to send register route request to scheduler."; - } + // Start of the "distributed" namespace within the "mindspore" namespace + namespace distributed { + // Start of the "cluster" namespace within the "distributed" namespace + namespace cluster { + // Implementation of the RegisterRoute function of the ActorRouteTableProxy class - GeneralResponseMsg register_route_rsp_msg; - MS_EXCEPTION_IF_NULL(output); - (void)register_route_rsp_msg.ParseFromArray(output->data(), SizeToInt(output->size())); - if (!register_route_rsp_msg.is_success()) { + // Check if the node_ pointer is not null + MS_EXCEPTION_IF_NULL(node_); + + // Create a shared pointer to a vector of unsigned char and initialize it to nullptr + std::shared_ptr> output = nullptr; + + // Check if sending the actor address to the scheduler was successful + if (!node_->SendToScheduler(actor_addr.SerializeAsString().data(), actor_addr.SerializeAsString().size(), + NodeCommand::REGISTER_ACTOR_ROUTE, &output)) { + // If sending the request failed, log an exception + MS_LOG(EXCEPTION) << "Failed to send register route request to scheduler."; + } + +// Create an instance of the GeneralResponseMsg class named register_route_rsp_msg +GeneralResponseMsg register_route_rsp_msg; + +// Check if the output pointer is null, and throw an exception if it is +MS_EXCEPTION_IF_NULL(output); + +// Parse the data from the output buffer into the register_route_rsp_msg object +(void)register_route_rsp_msg.ParseFromArray(output->data(), SizeToInt(output->size())); + +// Check if the registration was successful by calling the is_success() function of the register_route_rsp_msg object +if (!register_route_rsp_msg.is_success()) { + // If the registration failed, log an error message with the actor_id and the error message from the register_route_rsp_msg object MS_LOG(ERROR) << "Register route for actor " << actor_id << " failed. " << register_route_rsp_msg.error(); + + // Return false to indicate that the registration failed return false; - } - return true; } +// If the registration was successful, return true to indicate success +return true; + +// Function to delete a route for an actor in the ActorRouteTableProxy class + bool ActorRouteTableProxy::DeleteRoute(const std::string &actor_id) { + // Check if the node_ pointer is null, throw an exception if it is MS_EXCEPTION_IF_NULL(node_); + + // Create a shared pointer to a vector of unsigned char, initialize it to nullptr std::shared_ptr> output = nullptr; + + // If sending the delete route request to the scheduler fails if (!node_->SendToScheduler(actor_id.data(), actor_id.size(), NodeCommand::DELETE_ACTOR_ROUTE, &output)) { + // Log an exception with an error message MS_LOG(EXCEPTION) << "Failed to send delete route request to scheduler."; } - - GeneralResponseMsg delete_route_rsp_msg; - MS_EXCEPTION_IF_NULL(output); - (void)delete_route_rsp_msg.ParseFromArray(output->data(), SizeToInt(output->size())); - if (!delete_route_rsp_msg.is_success()) { - MS_LOG(ERROR) << "Delete route for actor " << actor_id << " failed. " << delete_route_rsp_msg.error(); - return false; - } + // Return true if the delete route request was successful return true; } -ActorAddress ActorRouteTableProxy::LookupRoute(const std::string &actor_id) const { +// Create an instance of the GeneralResponseMsg class named delete_route_rsp_msg +GeneralResponseMsg delete_route_rsp_msg; + +// Check if the output pointer is null, and throw an exception if it is +MS_EXCEPTION_IF_NULL(output); + +// Parse the data from the output buffer into the delete_route_rsp_msg object +(void)delete_route_rsp_msg.ParseFromArray(output->data(), SizeToInt(output->size())); + +// Check if the delete_route_rsp_msg indicates success +if (!delete_route_rsp_msg.is_success()) { + // Log an error message indicating that the route deletion for the specified actor_id failed, along with the error message from delete_route_rsp_msg + MS_LOG(ERROR) << "Delete route for actor " << actor_id << " failed. " << delete_route_rsp_msg.error(); + + // Return false to indicate failure + return false; +} + +// Return true to indicate success +return true; + +// Check if the node_ pointer is null, and throw an exception if it is MS_EXCEPTION_IF_NULL(node_); - // Whether this lookup operation is successful. + + // Initialize a boolean variable to track whether the lookup operation is successful bool lookup_success = false; - // Lookup last timestamp before timeout. + + // Calculate the timeout timestamp by adding the current timestamp in milliseconds to the lookup timeout auto timeout_ts = CURRENT_TIMESTAMP_MILLI + lookup_timeout_; + + // Initialize a shared pointer to a vector of unsigned char to store the output of the lookup operation std::shared_ptr> output = nullptr; + + // Initialize an ActorAddress object to store the lookup route response message ActorAddress lookup_route_rsp_msg; + + // Start a do-while loop to repeatedly send the lookup route request to the scheduler until it succeeds or times out do { + // Send the lookup route request to the scheduler using the node_'s SendToScheduler method + // Pass the actor_id as a char array and its size, along with the NodeCommand::LOOKUP_ACTOR_ROUTE command + // Store the output in the output shared pointer if (!node_->SendToScheduler(actor_id.data(), actor_id.size(), NodeCommand::LOOKUP_ACTOR_ROUTE, &output)) { + // If sending the lookup route request fails, throw an exception with an error message MS_LOG(EXCEPTION) << "Failed to send lookup route request to scheduler."; } - MS_EXCEPTION_IF_NULL(output); - (void)lookup_route_rsp_msg.ParseFromArray(output->data(), SizeToInt(output->size())); - // An actor route could not be registered yet because another process could be launched slow. - // If the response actor id is empty, this means the adderess is not registered yet. - if (lookup_route_rsp_msg.actor_id().empty()) { - MS_LOG(DEBUG) << "Actor route for actor " << actor_id << " is not registered yet, please try later."; - std::this_thread::sleep_for(std::chrono::milliseconds(kLookupInterval)); - } else { - lookup_success = true; - } - } while (!lookup_success && CURRENT_TIMESTAMP_MILLI <= timeout_ts); - if (!lookup_success) { - MS_LOG(EXCEPTION) << "Failed to lookup actor address for " << actor_id; - } +// Check if the output pointer is null, and throw an exception if it is +MS_EXCEPTION_IF_NULL(output); - return lookup_route_rsp_msg; +// Parse the data in the output buffer into the lookup_route_rsp_msg object +(void)lookup_route_rsp_msg.ParseFromArray(output->data(), SizeToInt(output->size())); + +// Check if the actor id in the response message is empty +if (lookup_route_rsp_msg.actor_id().empty()) { + // If the actor id is empty, it means the address is not registered yet + MS_LOG(DEBUG) << "Actor route for actor " << actor_id << " is not registered yet, please try later."; + + // Sleep for a certain interval before trying again + std::this_thread::sleep_for(std::chrono::milliseconds(kLookupInterval)); +} else { + // If the actor id is not empty, it means the address is registered successfully + lookup_success = true; } -} // namespace cluster -} // namespace distributed -} // namespace mindspore + +// Repeat the above steps until either the lookup is successful or the timeout is reached +} while (!lookup_success && CURRENT_TIMESTAMP_MILLI <= timeout_ts); + +// If the lookup was not successful even after the timeout, throw an exception +if (!lookup_success) { + MS_LOG(EXCEPTION) << "Failed to lookup actor address for " << actor_id; +} + + // Return the lookup_route_rsp_msg variable + return lookup_route_rsp_msg; +} // End of namespace cluster +} // End of namespace distributed +} // End of namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/cluster/actor_route_table_service.cc b/mindspore/ccsrc/distributed/cluster/actor_route_table_service.cc index fce8879919a..307fb3d7217 100644 --- a/mindspore/ccsrc/distributed/cluster/actor_route_table_service.cc +++ b/mindspore/ccsrc/distributed/cluster/actor_route_table_service.cc @@ -14,47 +14,101 @@ * limitations under the License. */ +// Include the header for mutex, which provides synchronization mechanisms for protecting shared data from concurrent access #include + +// Include the header for shared_mutex, which provides a multiple-reader, single-writer lock #include + +// Include the header for the actor_route_table_service class from the distributed/cluster directory #include "distributed/cluster/actor_route_table_service.h" +// Start of the "mindspore" namespace namespace mindspore { -namespace distributed { -namespace cluster { -bool ActorRouteTableService::Initialize() { return true; } + + // Start of the "distributed" namespace within the "mindspore" namespace + namespace distributed { + + // Start of the "cluster" namespace within the "distributed" namespace + namespace cluster { + + // Implementation of the Initialize function of the ActorRouteTableService class + bool ActorRouteTableService::Initialize() { + + // Return true to indicate successful initialization + return true; + } + } // End of the "cluster" namespace + } // End of the "distributed" namespace +} // End of the "mindspore" namespace + +// RegisterRoute function definition for the ActorRouteTableService class bool ActorRouteTableService::RegisterRoute(const std::string &actor_id, const ActorAddress &actor_addr, std::string *error) { - MS_ERROR_IF_NULL_W_RET_VAL(error, false); - std::unique_lock lock(mtx_); - if (actor_addresses_.count(actor_id) != 0) { - *error = "The address of actor id " + actor_id + " already exists."; - return false; - } - actor_addresses_[actor_id] = actor_addr; - return true; + // Check if the error pointer is null, return false if it is + MS_ERROR_IF_NULL_W_RET_VAL(error, false); + + // Acquire a lock on the mutex to ensure thread safety + std::unique_lock lock(mtx_); + + // Check if the actor_id already exists in the actor_addresses_ map + if (actor_addresses_.count(actor_id) != 0) { + // If it does, set the error message and return false + *error = "The address of actor id " + actor_id + " already exists."; + return false; + } + + // If the actor_id doesn't exist, add it to the actor_addresses_ map with the corresponding actor_addr + actor_addresses_[actor_id] = actor_addr; + + // Return true to indicate successful registration + return true; } +// Implementation of the DeleteRoute function in the ActorRouteTableService class + bool ActorRouteTableService::DeleteRoute(const std::string &actor_id, std::string *error) { + // Check if the error pointer is null, return false if it is MS_ERROR_IF_NULL_W_RET_VAL(error, false); + + // Acquire a lock on the mutex to ensure thread safety std::unique_lock lock(mtx_); + + // Check if the actor_id exists in the actor_addresses_ map if (actor_addresses_.count(actor_id) == 0) { + // If the actor_id does not exist, set the error message and return false *error = "The address of actor id " + actor_id + " does not exist."; return false; } + + // Erase the actor_id from the actor_addresses_ map (void)actor_addresses_.erase(actor_id); + + // Return true to indicate successful deletion return true; } +// Define the function LookupRoute in the namespace ActorRouteTableService + ActorAddress ActorRouteTableService::LookupRoute(const std::string &actor_id, std::string *error) { + + // Check if the error pointer is null, return an empty ActorAddress if it is MS_ERROR_IF_NULL_W_RET_VAL(error, {}); + + // Acquire a shared lock on the mutex to ensure thread safety std::shared_lock lock(mtx_); + + // Check if the actor_id exists in the actor_addresses_ map if (actor_addresses_.count(actor_id) == 0) { + + // If the actor_id does not exist, set the error message and return an empty ActorAddress *error = "The address of actor id " + actor_id + " does not exist."; return {}; } + + // If the actor_id exists, return the corresponding ActorAddress from the actor_addresses_ map return actor_addresses_[actor_id]; } -} // namespace cluster -} // namespace distributed -} // namespace mindspore + +// Close the namespace blocks for cluster, distributed, and mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/cluster/cluster_context.cc b/mindspore/ccsrc/distributed/cluster/cluster_context.cc index f537066694a..3a778adb19e 100644 --- a/mindspore/ccsrc/distributed/cluster/cluster_context.cc +++ b/mindspore/ccsrc/distributed/cluster/cluster_context.cc @@ -14,60 +14,101 @@ * limitations under the License. */ -#include -#include -#include -#include -#include "distributed/cluster/cluster_context.h" -#include "distributed/collective/collective_manager.h" -#include "utils/ms_context.h" -#include "ps/ps_context.h" -#include "include/common/debug/common.h" +// Include the necessary headers for the code +#include // For mutex +#include // For vector +#include // For string +#include // For smart pointers +#include "distributed/cluster/cluster_context.h" // For cluster context +#include "distributed/collective/collective_manager.h" // For collective manager +#include "utils/ms_context.h" // For MS context +#include "ps/ps_context.h" // For PS context +#include "include/common/debug/common.h" // For common debugging functions +// Start of the "mindspore" namespace namespace mindspore { -namespace distributed { -namespace cluster { -ClusterContext::ClusterContext() - : inited_(false), - finalized_(true), - cluster_ready_(false), - node_num_each_role_({}), - scheduler_host_(kLocalHost), - scheduler_port_(kDefaultSchedPort), - node_(nullptr), - abstract_node_(nullptr), - node_role_(""), - cluster_config_(nullptr) {} + // Start of the "distributed" namespace + namespace distributed { + // Start of the "cluster" namespace + namespace cluster { + // Definition of the constructor for the ClusterContext class + ClusterContext::ClusterContext() + : inited_(false), // Initialize the "inited_" member variable to false + finalized_(true), // Initialize the "finalized_" member variable to true + cluster_ready_(false), // Initialize the "cluster_ready_" member variable to false + node_num_each_role_({}), // Initialize the "node_num_each_role_" member variable to an empty list + scheduler_host_(kLocalHost), // Initialize the "scheduler_host_" member variable to the value of "kLocalHost" + scheduler_port_(kDefaultSchedPort), // Initialize the "scheduler_port_" member variable to the value of "kDefaultSchedPort" + node_(nullptr), // Initialize the "node_" member variable to nullptr + abstract_node_(nullptr), // Initialize the "abstract_node_" member variable to nullptr + node_role_(""), // Initialize the "node_role_" member variable to an empty string + cluster_config_(nullptr) {} // Initialize the "cluster_config_" member variable to nullptr + + } // End of the "cluster" namespace + } // End of the "distributed" namespace +} // End of the "mindspore" namespace + +// Destructor for the ClusterContext class ClusterContext::~ClusterContext() { + + // Check if the context has been finalized if (!finalized_) { + try { + // Call the Finalize() function and ignore any exceptions thrown (void)Finalize(); } catch (std::exception &) { + // Log an error message if the finalization fails MS_LOG(ERROR) << "Failed to finalize cluster context."; } } + + // Set the finalized flag to true finalized_ = true; + + // Set the node pointer to nullptr node_ = nullptr; } +// Define a function named "instance" that returns a shared pointer to a ClusterContext object std::shared_ptr ClusterContext::instance() { + + // Define a static shared pointer named "cluster_instance" and initialize it to nullptr static std::shared_ptr cluster_instance = nullptr; + + // Check if the cluster_instance is nullptr if (cluster_instance == nullptr) { + + // Create a new ClusterContext object using the "new" operator and assign it to cluster_instance cluster_instance.reset(new (std::nothrow) ClusterContext()); + + // Check if the cluster_instance is null after creating the object MS_EXCEPTION_IF_NULL(cluster_instance); } + + // Return the cluster_instance shared pointer return cluster_instance; } +// Function to initialize the cluster context bool ClusterContext::Initialize() { + + // Check if the cluster has already been initialized if (inited_) { + + // If already initialized, log a message indicating that the cluster has been initialized MS_LOG(INFO) << "The cluster has been initialized."; + + // Return true to indicate that initialization was successful return true; } +} - // MindSpore cluster does not support PyNative mode. +// Check if the execution mode of the MindSpore cluster is set to PyNative mode if (MsContext::GetInstance()->get_param(MS_CTX_EXECUTION_MODE) == kPynativeMode) { + + // If the execution mode is PyNative mode, log an exception message and return false MS_LOG(EXCEPTION) << "PyNative mode is not supported in MindSpore cluster."; return false; } @@ -75,204 +116,422 @@ bool ClusterContext::Initialize() { // Step 1: Initialize cluster configuration. InitClusterConfig(); - // Step 2: Build network for this cluster. Every process will block in this method until networking is done. - if (!BuildCluster()) { +// Step 2: Build network for this cluster. Every process will block in this method until networking is done. +if (!BuildCluster()) { + // Check if the node object is null MS_EXCEPTION_IF_NULL(node_); + + // If the networking fails, stop the node if (!node_->Stop()) { - MS_LOG(ERROR) << "Failed to stop node after the failure of BuildCluster"; - return false; + MS_LOG(ERROR) << "Failed to stop node after the failure of BuildCluster"; + return false; } + + // Log an error message indicating the failure to build networking for the specified node role MS_LOG(ERROR) << "Building networking for " << node_role_ << " failed."; + + // Return false to indicate the failure of building the network return false; - } - - // Step 3: Initialize some modules for the node, e.g., actor route table proxy. - if (!IsScheduler()) { - // Only node which is not the scheduler needs route table proxy. - actor_route_table_proxy_ = - std::make_shared(std::dynamic_pointer_cast(node_)); - MS_EXCEPTION_IF_NULL(actor_route_table_proxy_); - } - - inited_ = true; - finalized_ = false; - return true; } +// Step 3: Initialize some modules for the node, e.g., actor route table proxy. +if (!IsScheduler()) { + // Only node which is not the scheduler needs route table proxy. + // Create a shared pointer to an ActorRouteTableProxy object, passing in a dynamic pointer cast of the node_ object as an argument + actor_route_table_proxy_ = std::make_shared(std::dynamic_pointer_cast(node_)); + // Check if the actor_route_table_proxy_ pointer is null, and throw an exception if it is + MS_EXCEPTION_IF_NULL(actor_route_table_proxy_); +} + +// Set the variable "inited_" to true, indicating that initialization has been completed +inited_ = true; + +// Set the variable "finalized_" to false, indicating that finalization has not been completed yet +finalized_ = false; + +// Return true to indicate that the function has executed successfully +return true; + +// Function to finalize the cluster context bool ClusterContext::Finalize(uint32_t timeout) { + + // If already finalized, return true if (finalized_) { return true; } + + // Check if the node has finished within the given timeout // In some cases, one node calls the Finish function while other nodes don't. So timeout is acceptable. if (!node_->Finish(timeout)) { MS_LOG(WARNING) << "Finishing node " << node_role_ << " timeout."; } + + // Stop the node if (!node_->Stop()) { MS_LOG(ERROR) << "Failed to stop node " << node_role_; return false; } + + // Set finalized flag to true finalized_ = true; + + // Return true to indicate successful finalization return true; } -bool ClusterContext::IsScheduler() { return (abstract_node_ == nullptr) ? true : false; } +// Check if the abstract_node_ pointer is nullptr +// If it is nullptr, return true, indicating that the context is for a scheduler +// If it is not nullptr, return false, indicating that the context is not for a scheduler +bool ClusterContext::IsScheduler() { + return (abstract_node_ == nullptr) ? true : false; +} -const std::shared_ptr &ClusterContext::node() const { return node_; } +// Define a member function named "node" in the "ClusterContext" class +// This function returns a constant reference to a shared pointer of type "ps::core::Node" +const std::shared_ptr &ClusterContext::node() const { + // Return the value of the member variable "node_" + return node_; +} -const std::string &ClusterContext::node_role() const { return node_role_; } +// Define a member function named "node_role" in the "ClusterContext" class +// This function returns a constant reference to a string +const std::string &ClusterContext::node_role() const { + // Return the value of the member variable "node_role_" + return node_role_; +} + +// Function to get the number of nodes based on their role uint32_t ClusterContext::node_num(const std::string &node_role) { + + // Check if the given node role exists in the map of node roles and their counts if (node_num_each_role_.count(node_role) == 0) { + + // If the node role does not exist, throw an exception with an error message MS_LOG(EXCEPTION) << "Node role " << node_role << " is invalid."; + + // Return 0 to indicate an error return 0; } + + // If the node role exists, log an informational message with the number of nodes for that role MS_LOG(INFO) << "Number of role " << node_role << " is " << node_num_each_role_[node_role]; + + // Return the number of nodes for the given role return node_num_each_role_[node_role]; } -bool ClusterContext::initialized() const { return inited_; } +// Define the member function "initialized" of the class "ClusterContext" which returns a boolean value +bool ClusterContext::initialized() const { -const ActorRouteTableProxyPtr &ClusterContext::actor_route_table_proxy() const { return actor_route_table_proxy_; } + // Return the value of the private member variable "inited_" + return inited_; +} +// Define a member function named "actor_route_table_proxy" of the class "ClusterContext" +// This function returns a constant reference to the member variable "actor_route_table_proxy_" +const ActorRouteTableProxyPtr &ClusterContext::actor_route_table_proxy() const { + // Return the value of the member variable "actor_route_table_proxy_" + return actor_route_table_proxy_; +} + +// Initialize the cluster configuration for the current context void ClusterContext::InitClusterConfig() { + + // Initialize the node role InitNodeRole(); + + // Initialize the scheduler IP InitSchedulerIp(); + + // Initialize the scheduler port InitSchedulerPort(); + + // Set the role of the current node in the PSContext singleton instance ps::PSContext::instance()->set_ms_role(node_role_); + + // Set the number of workers in the PSContext singleton instance ps::PSContext::instance()->set_worker_num(node_num_each_role_[kEnvRoleOfWorker]); + + // Set the number of servers in the PSContext singleton instance ps::PSContext::instance()->set_server_num(node_num_each_role_[kEnvRoleOfServer]); + + // Set the scheduler IP in the PSContext singleton instance ps::PSContext::instance()->set_scheduler_ip(scheduler_host_); + + // Set the scheduler port in the PSContext singleton instance ps::PSContext::instance()->set_scheduler_port(scheduler_port_); + + // Set the initial worker number in the cluster configuration of the PSContext singleton instance ps::PSContext::instance()->cluster_config().initial_worker_num = node_num_each_role_[kEnvRoleOfWorker]; + + // Set the initial server number in the cluster configuration of the PSContext singleton instance ps::PSContext::instance()->cluster_config().initial_server_num = node_num_each_role_[kEnvRoleOfServer]; + + // Set the scheduler host in the cluster configuration of the PSContext singleton instance ps::PSContext::instance()->cluster_config().scheduler_host = scheduler_host_; + + // Set the scheduler port in the cluster configuration of the PSContext singleton instance ps::PSContext::instance()->cluster_config().scheduler_port = scheduler_port_; } bool ClusterContext::BuildCluster() { // Create node according to different role. if (node_role_ == kEnvRoleOfWorker) { + // If the node role is worker, create a PSWorkerNode object and assign it to the shared pointer node_ node_ = std::make_shared(); } else if (node_role_ == kEnvRoleOfServer) { + // If the node role is server, create a PSServerNode object and assign it to the shared pointer node_ node_ = std::make_shared(); } else if (node_role_ == kEnvRoleOfScheduler) { + // If the node role is scheduler, create a PSSchedulerNode object and assign it to the shared pointer node_ node_ = std::make_shared(); } else { + // If the node role is none of the above, log an exception and return false MS_LOG(EXCEPTION) << "The role " << node_role_ << " is invalid."; return false; } + + // Check if the node_ pointer is null, and throw an exception if it is MS_EXCEPTION_IF_NULL(node_); - - RegisterEventCallback(); - if (!node_->Start()) { - MS_LOG(ERROR) << "Building network failed."; - return false; - } - abstract_node_ = std::dynamic_pointer_cast(node_); - MS_LOG(INFO) << "Cluster is successfully initialized."; - return true; } +// Call the function RegisterEventCallback() to register an event callback + +RegisterEventCallback(); + +// Check if the Start() function of the node_ object returns false +if (!node_->Start()) { + + // If Start() returns false, print an error message using the MS_LOG(ERROR) macro + MS_LOG(ERROR) << "Building network failed."; + + // Return false to indicate that the initialization failed + return false; +} + +// Use std::dynamic_pointer_cast to cast the node_ object to a shared pointer of type ps::core::AbstractNode +abstract_node_ = std::dynamic_pointer_cast(node_); + +// Print an informational message using the MS_LOG(INFO) macro +MS_LOG(INFO) << "Cluster is successfully initialized."; + +// Return true to indicate that the initialization was successful +return true; + +// Function to initialize the node role in the ClusterContext class + void ClusterContext::InitNodeRole() { + + // Get the node role from the environment variable node_role_ = common::GetEnv(kEnvRole); + + // Check if the node role is a valid role name if (kValidRoleName.count(node_role_) == 0) { + + // If the node role is invalid, log an exception with a detailed failure reason MS_LOG(EXCEPTION) << "Role name '" << node_role_ << "' is invalid. " << kDetailedFailureReason; + + // Return from the function return; } - - if (common::GetEnv(kEnvWorkerNum).empty()) { - node_num_each_role_[kEnvRoleOfWorker] = 0; - } else { - TRY_AND_CATCH_WITH_EXCEPTION( - (node_num_each_role_[kEnvRoleOfWorker] = IntToUint(std::stoi(common::GetEnv(kEnvWorkerNum)))), - "The environment variable MS_WORKER_NUM is invalid."); - } - - if (common::GetEnv(kEnvServerNum).empty()) { - node_num_each_role_[kEnvRoleOfServer] = 0; - } else { - TRY_AND_CATCH_WITH_EXCEPTION( - (node_num_each_role_[kEnvRoleOfServer] = IntToUint(std::stoi(common::GetEnv(kEnvServerNum)))), - "The environment variable MS_SERVER_NUM is invalid."); - } } +// Check if the environment variable "MS_WORKER_NUM" is empty +if (common::GetEnv(kEnvWorkerNum).empty()) { + // If it is empty, set the number of nodes for the worker role to 0 + node_num_each_role_[kEnvRoleOfWorker] = 0; +} else { + // If it is not empty, try to convert the value of "MS_WORKER_NUM" to an integer and assign it to the number of nodes for the worker role + TRY_AND_CATCH_WITH_EXCEPTION( + (node_num_each_role_[kEnvRoleOfWorker] = IntToUint(std::stoi(common::GetEnv(kEnvWorkerNum)))), + "The environment variable MS_WORKER_NUM is invalid."); +} + +// Check if the environment variable "MS_SERVER_NUM" is empty +if (common::GetEnv(kEnvServerNum).empty()) { + // If it is empty, set the number of nodes for the server role to 0 + node_num_each_role_[kEnvRoleOfServer] = 0; +} else { + // If it is not empty, try to convert the value of "MS_SERVER_NUM" to an integer and assign it to the number of nodes for the server role + TRY_AND_CATCH_WITH_EXCEPTION( + (node_num_each_role_[kEnvRoleOfServer] = IntToUint(std::stoi(common::GetEnv(kEnvServerNum)))), + "The environment variable MS_SERVER_NUM is invalid."); +} + +// Definition of the function InitSchedulerIp in the ClusterContext class + void ClusterContext::InitSchedulerIp() { + + // Get the value of the environment variable kEnvSchedulerHost and assign it to the member variable scheduler_host_ scheduler_host_ = common::GetEnv(kEnvSchedulerHost); + + // Check if the scheduler_host_ is empty if (scheduler_host_.empty()) { + + // If scheduler_host_ is empty, log an exception with the message "kEnvSchedulerHost is empty. kEnvSchedulerHost" MS_LOG(EXCEPTION) << kEnvSchedulerHost << " is empty. " << kEnvSchedulerHost; } } +// Function to initialize the scheduler port in the ClusterContext class + void ClusterContext::InitSchedulerPort() { + + // Try to convert the environment variable MS_SCHED_PORT to an integer and assign it to scheduler_port_ + // If an exception occurs during the conversion, catch it and throw a new exception with a custom error message TRY_AND_CATCH_WITH_EXCEPTION((scheduler_port_ = static_cast(std::stoi(common::GetEnv(kEnvSchedulerPort)))), "The environment variable MS_SCHED_PORT is invalid."); + + // Check if the scheduler port is greater than the maximum allowed port number if (scheduler_port_ > kMaxPort) { + + // If the port is invalid, log an exception with the port number MS_LOG(EXCEPTION) << "The port: " << scheduler_port_ << " is invalid."; } } +// Define the function `RegisterEventCallback` in the `ClusterContext` class void ClusterContext::RegisterEventCallback() { + + // Cast the `node_` member variable to a shared pointer of type `AbstractNode` auto abstract_node = std::dynamic_pointer_cast(node_); + + // Check if the cast was successful (i.e., `node_` is of type `AbstractNode`) if (abstract_node != nullptr) { + + // Register a callback function for the `SCHEDULER_TIMEOUT` event abstract_node->RegisterEventCallback(ps::core::ClusterEvent::SCHEDULER_TIMEOUT, [this]() { + + // Acquire a unique lock on the `finish_mutex_` mutex std::unique_lock lock(finish_mutex_); + + // Log an error message indicating that the `SCHEDULER_TIMEOUT` event was captured MS_LOG(ERROR) << "Event SCHEDULER_TIMEOUT is captured."; + + // Try to execute the following code block try { + + // Log an info message indicating the start of the cluster finalization process MS_LOG(INFO) << "Start finalize cluster..."; + + // Check if the cluster finalization is successful if (!Finalize()) { + + // If the finalization fails, log an exception message MS_LOG(EXCEPTION) << "Failed to finalize cluster."; } + + // If the finalization is successful, log an info message MS_LOG(INFO) << "Successfully finalize cluster."; - MS_LOG(INFO) << "Start finalize collective communication..."; - if (!collective::CollectiveManager::instance()->Finalize()) { - MS_LOG(EXCEPTION) << "Failed to finalize collective communication."; - } - MS_LOG(INFO) << "Successfully finalize collective communication."; + // Continue with the execution of the program + // ... - MS_LOG(EXCEPTION) - << "Event SCHEDULER_TIMEOUT is captured. This is because scheduler node is finalized or crashed."; - } catch (std::exception &) { - MsException::Instance().SetException(); + } catch (...) { + + // Catch any exceptions that occur during the execution of the code block + // ... } }); + } +} +// Log an informational message indicating the start of finalizing collective communication +MS_LOG(INFO) << "Start finalize collective communication..."; + +// Check if the collective communication manager can be successfully finalized +if (!collective::CollectiveManager::instance()->Finalize()) { + // If the finalization fails, log an exception message + MS_LOG(EXCEPTION) << "Failed to finalize collective communication."; +} + +// Log an informational message indicating the successful finalization of collective communication +MS_LOG(INFO) << "Successfully finalize collective communication."; + + // Try block to catch any exceptions that may occur + try { + // Log an exception with the message "Event SCHEDULER_TIMEOUT is captured. This is because scheduler node is finalized or crashed." + MS_LOG(EXCEPTION) << "Event SCHEDULER_TIMEOUT is captured. This is because scheduler node is finalized or crashed."; + } + // Catch block to handle any std::exception that is thrown + catch (std::exception &) { + // Set the exception in the MsException instance + MsException::Instance().SetException(); + } + }); + + // Register an event callback for the NODE_TIMEOUT event on the abstract_node object abstract_node->RegisterEventCallback(ps::core::ClusterEvent::NODE_TIMEOUT, [this]() { + + // Acquire a unique lock on the finish_mutex_ to ensure exclusive access std::unique_lock lock(finish_mutex_); + + // Log an error message indicating that the NODE_TIMEOUT event has been captured MS_LOG(ERROR) << "Event NODE_TIMEOUT is captured."; + + // Try to execute the following code block try { + + // Log an informational message indicating the start of finalizing the cluster MS_LOG(INFO) << "Start finalize cluster..."; + + // Call the Finalize() function and check its return value if (!Finalize()) { + + // If Finalize() returns false, log an exception indicating the failure to finalize the cluster MS_LOG(EXCEPTION) << "Failed to finalize cluster."; } + + // If Finalize() returns true, log an informational message indicating the successful finalization of the cluster MS_LOG(INFO) << "Successfully finalize cluster."; - MS_LOG(INFO) << "Start finalize collective communication..."; - if (!collective::CollectiveManager::instance()->Finalize()) { - MS_LOG(EXCEPTION) << "Failed to finalize collective communication."; - } - MS_LOG(INFO) << "Successfully finalize collective communication."; +// Log an informational message indicating the start of finalizing collective communication +MS_LOG(INFO) << "Start finalize collective communication..."; - MS_LOG(EXCEPTION) << "Event NODE_TIMEOUT is captured. This is because some nodes are finalized or crashed."; - } catch (std::exception &) { - MsException::Instance().SetException(); - } +// Check if the collective communication manager can be successfully finalized +if (!collective::CollectiveManager::instance()->Finalize()) { + // If the finalization fails, log an exception message + MS_LOG(EXCEPTION) << "Failed to finalize collective communication."; +} + +// Log an informational message indicating the successful finalization of collective communication +MS_LOG(INFO) << "Successfully finalize collective communication."; + + // Try block to catch any exceptions that may occur + try { + // Log an exception using the MS_LOG macro, with the message "Event NODE_TIMEOUT is captured. This is because some nodes are finalized or crashed." + MS_LOG(EXCEPTION) << "Event NODE_TIMEOUT is captured. This is because some nodes are finalized or crashed."; + } + // Catch block to handle any std::exception that is thrown + catch (std::exception &) { + // Set the exception in the MsException singleton instance + MsException::Instance().SetException(); + } }); + // Register an event callback for the ON_SEND_META_DATA event of the abstract_node object abstract_node->RegisterEventCallback(ps::core::ClusterEvent::ON_SEND_META_DATA, [this]() { cluster_ready_ = true; }); } } +// A member function of the ClusterContext class that waits until the cluster is ready + void ClusterContext::WaitForClusterReady() { + + // Loop until the cluster is ready while (!cluster_ready_) { + + // Define the wait duration in milliseconds const int kWaitDuration = 200; + + // Sleep for the specified wait duration using the std::this_thread::sleep_for function std::this_thread::sleep_for(std::chrono::milliseconds(kWaitDuration)); } +} cluster_ready_ = false; -} } // namespace cluster } // namespace distributed -} // namespace mindspore +} // namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/cluster/dummy_cluster_context.cc b/mindspore/ccsrc/distributed/cluster/dummy_cluster_context.cc index 88a5ce722a1..02ec470075c 100644 --- a/mindspore/ccsrc/distributed/cluster/dummy_cluster_context.cc +++ b/mindspore/ccsrc/distributed/cluster/dummy_cluster_context.cc @@ -14,30 +14,93 @@ * limitations under the License. */ +// Include the string header for using string data type #include + +// Include the dummy_cluster_context.h header file from the distributed/cluster directory #include "distributed/cluster/dummy_cluster_context.h" +// Start of the "mindspore" namespace namespace mindspore { -namespace distributed { -namespace cluster { -std::shared_ptr ClusterContext::instance() { - static std::shared_ptr cluster_instance = nullptr; - if (cluster_instance == nullptr) { - cluster_instance.reset(new (std::nothrow) ClusterContext()); - MS_EXCEPTION_IF_NULL(cluster_instance); - } - return cluster_instance; + // Start of the "distributed" namespace + namespace distributed { + // Start of the "cluster" namespace + namespace cluster { + // Definition of the "instance" function of the "ClusterContext" class + std::shared_ptr ClusterContext::instance() { + // Create a static shared pointer to hold the instance of ClusterContext + static std::shared_ptr cluster_instance = nullptr; + + // Check if the cluster_instance is null + if (cluster_instance == nullptr) { + // Create a new ClusterContext instance using the "new" operator + cluster_instance.reset(new (std::nothrow) ClusterContext()); + + // Throw an exception if the cluster_instance is null + MS_EXCEPTION_IF_NULL(cluster_instance); + } + + // Return the cluster_instance + return cluster_instance; + } + } // End of the "cluster" namespace + } // End of the "distributed" namespace +} // End of the "mindspore" namespace + +// Define the Initialize function of the ClusterContext class, which returns a boolean value +bool ClusterContext::Initialize() const { + + // Return true to indicate successful initialization + return true; } -bool ClusterContext::Initialize() const { return true; } +// Implementation of the Finalize function in the ClusterContext class +// Takes an unsigned 32-bit integer as a parameter and returns a boolean value -bool ClusterContext::Finalize(uint32_t) const { return true; } +bool ClusterContext::Finalize(uint32_t) const { + // Always returns true, indicating successful finalization + return true; +} -std::string ClusterContext::node_role() const { return ""; } +// Define a member function named "node_role" of the class "ClusterContext" that returns a string +std::string ClusterContext::node_role() const { -uint32_t ClusterContext::node_num(const std::string &) { return 0; } + // Return an empty string + return ""; +} -bool ClusterContext::initialized() const { return false; } -} // namespace cluster -} // namespace distributed +// Define the member function "node_num" of the class "ClusterContext" +// The function takes a constant reference to a std::string as a parameter +// The function returns an unsigned 32-bit integer (uint32_t) +uint32_t ClusterContext::node_num(const std::string &) { + + // Return 0 as a placeholder value + return 0; +} + +// The function `initialized()` is a member function of the `ClusterContext` class. +// It returns a boolean value indicating whether the cluster context is initialized or not. + +// Start of the `cluster` namespace +namespace cluster { + +// Start of the `distributed` namespace +namespace distributed { + +// Start of the `mindspore` namespace +namespace mindspore { + +// Implementation of the `initialized()` member function of the `ClusterContext` class +bool ClusterContext::initialized() const { + // Return false to indicate that the cluster context is not initialized + return false; +} + +// End of the `mindspore` namespace } // namespace mindspore + +// End of the `distributed` namespace +} // namespace distributed + +// End of the `cluster` namespace +} // namespace cluster \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/cluster/topology/compute_graph_node.cc b/mindspore/ccsrc/distributed/cluster/topology/compute_graph_node.cc index 9b4e62ec8cf..fbc481a2a9e 100644 --- a/mindspore/ccsrc/distributed/cluster/topology/compute_graph_node.cc +++ b/mindspore/ccsrc/distributed/cluster/topology/compute_graph_node.cc @@ -14,109 +14,205 @@ * limitations under the License. */ +// Include the utility header for various utility functions and classes #include + +// Include the log adapter header for logging related functionality #include "utils/log_adapter.h" + +// Include the utils header for utility functions related to cluster topology #include "distributed/cluster/topology/utils.h" + +// Include the common header for common definitions related to cluster topology #include "distributed/cluster/topology/common.h" + +// Include the protobuf header for the topology protocol buffer #include "proto/topology.pb.h" + +// Include the compute graph node header for functionality related to compute graph nodes in the cluster topology #include "distributed/cluster/topology/compute_graph_node.h" namespace mindspore { namespace distributed { namespace cluster { namespace topology { -bool ComputeGraphNode::Initialize() { - // Init the address of meta server node. - RETURN_IF_FALSE_WITH_LOG(FillMetaServerAddress(&meta_server_addr_), - "Failed to init the address of meta server node."); - // Init the TCP client. - tcp_client_ = std::make_unique(); - MS_EXCEPTION_IF_NULL(tcp_client_); - RETURN_IF_FALSE_WITH_LOG(tcp_client_->Initialize(), "Failed to create the TCP client."); +// Initialize the ComputeGraphNode +bool ComputeGraphNode::Initialize() { + + // Initialize the address of the meta server node + RETURN_IF_FALSE_WITH_LOG(FillMetaServerAddress(&meta_server_addr_), + "Failed to initialize the address of the meta server node."); + +// Initialize the TCP client by creating a new instance using std::make_unique and assigning it to the tcp_client_ variable +tcp_client_ = std::make_unique(); + +// Check if the tcp_client_ variable is not null, and throw an exception if it is null +MS_EXCEPTION_IF_NULL(tcp_client_); + +// Call the Initialize() function of the tcp_client_ object and check if it returns true. If it returns false, log an error message and return from the current function. +RETURN_IF_FALSE_WITH_LOG(tcp_client_->Initialize(), "Failed to create the TCP client."); // Register itself to meta server node. RETURN_IF_FALSE_WITH_LOG(Register(), "Failed to register to the meta server node."); + + // Return true to indicate successful execution of the function return true; } bool ComputeGraphNode::Finalize() { // Exit the compute graph node from the cluster topology. + + // Call the Unregister() function and check if it returns false. + // If it returns false, log an error message using the RETURN_IF_FALSE_WITH_LOG macro. RETURN_IF_FALSE_WITH_LOG(Unregister(), "Failed to send unregistration message to the meta server."); +} // Release the TCP client. + + // Check if the TCP client object is not null if (tcp_client_ != nullptr) { + + // Get the server URL from the meta server address object const auto &server_url = meta_server_addr_.GetUrl(); + + // Disconnect the TCP client from the server URL tcp_client_->Disconnect(server_url); + + // Finalize the TCP client tcp_client_->Finalize(); + + // Reset the TCP client object tcp_client_.reset(); } + + // Return true to indicate successful completion of the function return true; } +// Register the compute graph node bool ComputeGraphNode::Register() { + + // Check if the TCP client is null, throw an exception if it is MS_EXCEPTION_IF_NULL(tcp_client_); + + // Get the server URL from the meta server address const auto &server_url = meta_server_addr_.GetUrl(); + + // Connect to the meta server node using the TCP client + // Return false and log an error message if the connection fails RETURN_IF_FALSE_WITH_LOG(tcp_client_->Connect(server_url), "Failed to connect to the meta server node url: " << server_url); + + // Create a registration message RegistrationMessage reg_msg; + + // Set the node ID in the registration message reg_msg.set_node_id(node_id_); - std::string content = reg_msg.SerializeAsString(); - auto message = CreateMessage(server_url, MessageName::kRegistration, content); - MS_EXCEPTION_IF_NULL(message); + // ... (rest of the code) - tcp_client_->SendSync(std::move(message)); - return true; -} +// Create a string variable named "content" and assign it the serialized form of the "reg_msg" object +std::string content = reg_msg.SerializeAsString(); -bool ComputeGraphNode::Unregister() { - MS_EXCEPTION_IF_NULL(tcp_client_); +// Create a variable named "message" and assign it the result of calling the "CreateMessage" function with the arguments "server_url", "MessageName::kRegistration", and "content" +auto message = CreateMessage(server_url, MessageName::kRegistration, content); - UnregistrationMessage unreg_msg; - unreg_msg.set_node_id(node_id_); +// Check if the "message" variable is null, and throw an exception if it is +MS_EXCEPTION_IF_NULL(message); - std::string content = unreg_msg.SerializeAsString(); - auto message = CreateMessage(meta_server_addr_.GetUrl(), MessageName::kUnregistration, content); - MS_EXCEPTION_IF_NULL(message); +// Send the message using the tcp_client_ object's SendSync function, and move the message to avoid unnecessary copying +tcp_client_->SendSync(std::move(message)); - auto retval = tcp_client_->SendSync(std::move(message)); - if (retval > 0) { +// Return true to indicate that the sending was successful +return true; + +// Define the Unregister function for the ComputeGraphNode class, which returns a boolean value + +// Check if the tcp_client_ member variable is null, and throw an exception if it is null + +// Create an instance of the UnregistrationMessage class named unreg_msg +UnregistrationMessage unreg_msg; + +// Set the node_id of the unreg_msg object to the value of the node_id_ variable +unreg_msg.set_node_id(node_id_); + +// Create a string variable named "content" and assign the serialized form of the "unreg_msg" object to it +std::string content = unreg_msg.SerializeAsString(); + +// Create a variable named "message" and assign the result of calling the "CreateMessage" function with the arguments "meta_server_addr_.GetUrl()", "MessageName::kUnregistration", and "content" +auto message = CreateMessage(meta_server_addr_.GetUrl(), MessageName::kUnregistration, content); + +// Check if the "message" variable is null, and throw an exception if it is +MS_EXCEPTION_IF_NULL(message); + +// Call the SendSync function of the tcp_client_ object, passing in the message as a move argument +auto retval = tcp_client_->SendSync(std::move(message)); + +// Check if the return value of SendSync is greater than 0 +if (retval > 0) { + // If it is, return true to indicate successful sending return true; - } else { +} else { + // If it is not, return false to indicate unsuccessful sending return false; - } } -bool ComputeGraphNode::Heartbeat() { +// Check if the tcp_client_ pointer is null MS_EXCEPTION_IF_NULL(tcp_client_); - HeartbeatMessage hb_msg; - hb_msg.set_node_id(node_id_); +// Create an instance of the HeartbeatMessage class named hb_msg +HeartbeatMessage hb_msg; - const auto &server_url = meta_server_addr_.GetUrl(); - std::string content = hb_msg.SerializeAsString(); - auto message = CreateMessage(server_url, MessageName::kHeartbeat, content); - MS_EXCEPTION_IF_NULL(message); +// Set the node_id of the hb_msg object to the value of the node_id_ variable +hb_msg.set_node_id(node_id_); - tcp_client_->SendSync(std::move(message)); - return true; -} +// Create a constant reference to the URL obtained from the `meta_server_addr_` object +const auto &server_url = meta_server_addr_.GetUrl(); +// Serialize the `hb_msg` object into a string and store it in the `content` variable +std::string content = hb_msg.SerializeAsString(); + +// Create a message using the `CreateMessage` function, passing in the `server_url`, `MessageName::kHeartbeat`, and `content` +auto message = CreateMessage(server_url, MessageName::kHeartbeat, content); + +// Check if the `message` is null and throw an exception if it is +MS_EXCEPTION_IF_NULL(message); + +// Send the message using the tcp_client_ object's SendSync function, and move the message to avoid unnecessary copying +tcp_client_->SendSync(std::move(message)); + +// Return true to indicate that the sending was successful +return true; + +// Function to send a message to MSN (Microsoft Network) bool ComputeGraphNode::SendMessageToMSN(const std::string msg_name, const std::string &msg_body) { + + // Check if the TCP client object is null, and throw an exception if it is MS_EXCEPTION_IF_NULL(tcp_client_); + + // Rest of the code goes here... +} - auto message = CreateMessage(meta_server_addr_.GetUrl(), msg_name, msg_body); - MS_EXCEPTION_IF_NULL(message); +// Create a message using the function CreateMessage, passing in the URL obtained from meta_server_addr_, the message name (msg_name), and the message body (msg_body) +auto message = CreateMessage(meta_server_addr_.GetUrl(), msg_name, msg_body); +// Check if the created message is null, and throw an exception if it is +MS_EXCEPTION_IF_NULL(message); + + // Call the SendSync function of the tcp_client_ object, passing in the message as a move argument auto retval = tcp_client_->SendSync(std::move(message)); + + // Check if the return value of SendSync is greater than 0 if (retval > 0) { + // If it is, return true return true; } else { + // If it is not, return false return false; } } } // namespace topology } // namespace cluster } // namespace distributed -} // namespace mindspore +} // namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/cluster/topology/meta_server_node.cc b/mindspore/ccsrc/distributed/cluster/topology/meta_server_node.cc index cc86a360441..65771254b43 100644 --- a/mindspore/ccsrc/distributed/cluster/topology/meta_server_node.cc +++ b/mindspore/ccsrc/distributed/cluster/topology/meta_server_node.cc @@ -14,180 +14,320 @@ * limitations under the License. */ +// Include the functional header for using function objects and higher-order functions #include + +// Include the algorithm header for using various algorithms like sorting #include + +// Include the string header for using string-related functions and classes #include + +// Include the "topology.pb.h" header file from the "proto" directory #include "proto/topology.pb.h" + +// Include the "constants.h" header file from the "distributed/rpc/tcp" directory #include "distributed/rpc/tcp/constants.h" + +// Include the "utils.h" header file from the "distributed/cluster/topology" directory #include "distributed/cluster/topology/utils.h" + +// Include the "meta_server_node.h" header file from the "distributed/cluster/topology" directory #include "distributed/cluster/topology/meta_server_node.h" namespace mindspore { namespace distributed { namespace cluster { namespace topology { + +// Initialize the MetaServerNode bool MetaServerNode::Initialize() { - // Init the address of meta server node. - RETURN_IF_FALSE_WITH_LOG(FillMetaServerAddress(&meta_server_addr_), - "Failed to init the address of meta server node."); - // Init the TCP server. - RETURN_IF_FALSE_WITH_LOG(InitTCPServer(), "Failed to create the TCP server."); + // Initialize the address of the meta server node + RETURN_IF_FALSE_WITH_LOG(FillMetaServerAddress(&meta_server_addr_), "Failed to initialize the address of the meta server node."); - start_time_ = Now(); +// Initialize the TCP server +RETURN_IF_FALSE_WITH_LOG(InitTCPServer(), "Failed to create the TCP server."); - // Init the thread for monitoring the state of the cluster topo. - topo_monitor_ = std::thread(&MetaServerNode::UpdateTopoState, this); - return true; -} +// Set the value of the variable "start_time_" to the current time by calling the "Now()" function + +// Initialize the thread for monitoring the state of the cluster topology +topo_monitor_ = std::thread(&MetaServerNode::UpdateTopoState, this); + +// Return true to indicate successful initialization +return true; bool MetaServerNode::Finalize() { // Release the TCP server. if (tcp_server_ != nullptr) { + // Call the Finalize() function of the TCP server object tcp_server_->Finalize(); + + // Reset the TCP server object to nullptr tcp_server_.reset(); } - - // Stop the topo monitor thread. - enable_monitor_ = false; - topo_monitor_.join(); + // Return true to indicate successful finalization return true; } -bool MetaServerNode::InitTCPServer() { + // Set the flag to stop the topo monitor thread + enable_monitor_ = false; + + // Wait for the topo monitor thread to finish + topo_monitor_.join(); + + // Return true to indicate successful completion of the function + return true; +} + +// Initialize the TCPServer object by creating a unique pointer using std::make_unique tcp_server_ = std::make_unique(); + + // Check if the tcp_server_ object is not null MS_EXCEPTION_IF_NULL(tcp_server_); + + // Initialize the tcp_server_ object with the meta_server_addr_ URL + // If initialization fails, log an error message and return false RETURN_IF_FALSE_WITH_LOG(tcp_server_->Initialize(meta_server_addr_.GetUrl()), "Failed to init the tcp server."); + + // Set the message handler for the tcp_server_ object + // The message handler is a member function of the MetaServerNode class, so we use std::bind to bind it to the current instance of MetaServerNode + // std::placeholders::_1 is a placeholder for the message argument that will be passed to the HandleMessage function tcp_server_->SetMessageHandler(std::bind(&MetaServerNode::HandleMessage, this, std::placeholders::_1)); // Configure the message processors for the TCP server. system_msg_handlers_[MessageName::kRegistration] = - std::bind(&MetaServerNode::ProcessRegister, this, std::placeholders::_1); + std::bind(&MetaServerNode::ProcessRegister, this, std::placeholders::_1); // Bind the ProcessRegister function of the MetaServerNode class to the kRegistration message system_msg_handlers_[MessageName::kUnregistration] = - std::bind(&MetaServerNode::ProcessUnregister, this, std::placeholders::_1); + std::bind(&MetaServerNode::ProcessUnregister, this, std::placeholders::_1); // Bind the ProcessUnregister function of the MetaServerNode class to the kUnregistration message system_msg_handlers_[MessageName::kHeartbeat] = - std::bind(&MetaServerNode::ProcessHeartbeat, this, std::placeholders::_1); - return true; -} + std::bind(&MetaServerNode::ProcessHeartbeat, this, std::placeholders::_1); // Bind the ProcessHeartbeat function of the MetaServerNode class to the kHeartbeat message + return true; // Return true to indicate successful configuration of message processors +// Define the return type of the function as a constant pointer to MessageBase MessageBase *const MetaServerNode::HandleMessage(MessageBase *const message) { + + // Create a constant reference to the name of the message const auto &name = message->Name(); - // Handle system messages. - if (std::all_of(name.begin(), name.end(), ::isdigit)) { - const auto &message_name = static_cast(std::stoi(message->Name())); - const auto &handler = system_msg_handlers_.find(message_name); - if (handler == system_msg_handlers_.end()) { - MS_LOG(ERROR) << "Unknown system message name: " << message->Name(); - return rpc::NULL_MSG; - } - system_msg_handlers_[message_name](message); - return rpc::NULL_MSG; +// Check if all characters in the string 'name' are digits +if (std::all_of(name.begin(), name.end(), ::isdigit)) { - // Handle user defined messages. - } else { - const auto &handler = message_handlers_.find(name); - if (handler == message_handlers_.end()) { - MS_LOG(ERROR) << "Unknown message name: " << name; - return rpc::NULL_MSG; + // Convert the string 'message->Name()' to an integer and assign it to 'message_name' + const auto &message_name = static_cast(std::stoi(message->Name())); + + // Find the handler for the system message with the name 'message_name' in the 'system_msg_handlers_' map + const auto &handler = system_msg_handlers_.find(message_name); + + // If no handler is found for the system message, log an error and return a NULL_MSG + if (handler == system_msg_handlers_.end()) { + MS_LOG(ERROR) << "Unknown system message name: " << message->Name(); + return rpc::NULL_MSG; } - (*message_handlers_[name])(message->Body()); + + // Call the handler function for the system message with the given 'message' + system_msg_handlers_[message_name](message); + + // Return a NULL_MSG return rpc::NULL_MSG; - } } +// Handle user defined messages. +} else { + // Find the message handler corresponding to the given name + const auto &handler = message_handlers_.find(name); + + // If no handler is found, log an error message and return a NULL_MSG + if (handler == message_handlers_.end()) { + MS_LOG(ERROR) << "Unknown message name: " << name; + return rpc::NULL_MSG; + } + + // Call the message handler function with the message body as argument + (*message_handlers_[name])(message->Body()); + + // Return a NULL_MSG to indicate successful handling of the message + return rpc::NULL_MSG; +} + +// Define the function `ProcessRegister` of the class `MetaServerNode` that returns a constant pointer to `MessageBase` MessageBase *const MetaServerNode::ProcessRegister(MessageBase *const message) { + + // Create an instance of `RegistrationMessage` called `registration` RegistrationMessage registration; + + // Get the body of the `message` and store it in a constant reference to `std::string` called `body` const std::string &body = message->Body(); + + // Parse the data from `body` into the `registration` object registration.ParseFromArray(body.c_str(), body.length()); - // Add the compute graph node into registered nodes. + // Get the node ID from the registration object const auto &node_id = registration.node_id(); + + // Acquire a unique lock on the nodes mutex to ensure thread safety std::unique_lock lock(nodes_mutex_); + + // Check if the node with the given ID already exists in the registered nodes map if (nodes_.find(node_id) == nodes_.end()) { + // If the node does not exist, create a new shared pointer to a ComputeGraphNodeState object std::shared_ptr node_state = std::make_shared(node_id); + + // Add the new node to the registered nodes map nodes_[node_id] = node_state; + + // Log a message indicating that the new node has been registered successfully MS_LOG(INFO) << "The new node: " << node_id << " is registered successfully."; } else { + // If the node already exists, log an error message MS_LOG(ERROR) << "The node: " << node_id << " have been registered before."; } + + // Return a NULL message to indicate successful execution return rpc::NULL_MSG; } +// Declare a pointer to a constant MessageBase object named "message" as the parameter of the function "ProcessUnregister" MessageBase *const MetaServerNode::ProcessUnregister(MessageBase *const message) { + + // Declare an object of type UnregistrationMessage named "unregistration" UnregistrationMessage unregistration; + + // Get the body of the message and store it in a constant reference to a string named "body" const std::string &body = message->Body(); + + // Parse the data from the body of the message into the "unregistration" object using the ParseFromArray function unregistration.ParseFromArray(body.c_str(), body.length()); - const auto &node_id = unregistration.node_id(); - std::unique_lock lock(nodes_mutex_); - if (nodes_.find(node_id) == nodes_.end()) { +// Create a constant reference to the node_id obtained from unregistration +const auto &node_id = unregistration.node_id(); + +// Acquire a unique lock on the nodes_mutex_ using std::unique_lock +std::unique_lock lock(nodes_mutex_); + +// Check if the node_id exists in the nodes_ container +if (nodes_.find(node_id) == nodes_.end()) { + // If the node_id does not exist, log an error message and return a NULL_MSG MS_LOG(ERROR) << "Received unregistration message from invalid compute graph node: " << node_id; return rpc::NULL_MSG; - } - nodes_.erase(node_id); - return rpc::NULL_MSG; } +// If the node_id exists, erase it from the nodes_ container +nodes_.erase(node_id); + +// Return a NULL_MSG +return rpc::NULL_MSG; + +// Define the function `ProcessHeartbeat` of class `MetaServerNode` which returns a constant pointer to `MessageBase` MessageBase *const MetaServerNode::ProcessHeartbeat(MessageBase *const message) { + + // Create an instance of `HeartbeatMessage` named `heartbeat` HeartbeatMessage heartbeat; + + // Get the body of the `message` and store it in a constant reference to `std::string` named `body` const std::string &body = message->Body(); + + // Parse the data from `body` into the `heartbeat` object using `ParseFromArray` function heartbeat.ParseFromArray(body.c_str(), body.length()); + // ... (rest of the code) + // Update the state(timestamp) of this node. + + // Get the node ID from the heartbeat message const auto &node_id = heartbeat.node_id(); + + // Acquire a shared lock on the nodes mutex to ensure thread safety std::shared_lock lock(nodes_mutex_); + + // Check if the node with the given ID exists in the nodes map if (nodes_.find(node_id) != nodes_.end()) { + // If the node exists, update its last_update timestamp with the current time auto &node = nodes_[node_id]; time(&(node->last_update)); } else { + // If the node does not exist, log an error message MS_LOG(ERROR) << "Invalid node: " << node_id << "."; } + + // Return a NULL message as the result of the function return rpc::NULL_MSG; } void MetaServerNode::UpdateTopoState() { while (enable_monitor_) { if (topo_state_ == TopoState::kInitializing) { - // Set the state of topo to `kFailed` if the topology is still in process of initializtion but timed out. + // Check if the initialization of the topology has timed out if (ElapsedTime(start_time_) > kTopoInitTimeout) { + // If timed out, set the state of topo to `kFailed` MS_LOG(ERROR) << "Failed to initialize the cluster topology after waiting for " << kTopoInitTimeout.count() << " milliseconds."; topo_state_ = TopoState::kFailed; continue; } + // Acquire a shared lock on the nodes_mutex_ using std::shared_lock and std::shared_mutex std::shared_lock lock(nodes_mutex_); + + // Check if the size of the nodes_ container is equal to the total_node_num_ if (nodes_.size() == total_node_num_) { + // If the condition is true, log a message indicating that the cluster topology has been constructed successfully MS_LOG(INFO) << "The cluster topology has been constructed successfully"; + + // Set the topo_state_ to kInitialized topo_state_ = TopoState::kInitialized; + + // Continue to the next iteration of the loop continue; } + + // If the condition is false, log a message indicating that the cluster topology is in the process of constructing + // Also include the current number of alive nodes and the total number of nodes MS_LOG(INFO) << "The cluster topology is in the process of constructing, current alive node num: (" << nodes_.size() << "/" << total_node_num_ << ")"; } + + // Set the interval for sleeping between iterations to 3 seconds static const size_t interval = 3; + + // Sleep for the specified interval sleep(interval); } } -TopoState MetaServerNode::TopologyState() { return topo_state_; } +// Return the value of the member variable topo_state_ of the MetaServerNode class as a TopoState object. + +// Get the number of alive nodes in the MetaServerNode class size_t MetaServerNode::GetAliveNodeNum() { + + // Acquire a shared lock on the nodes_mutex_ to ensure thread safety std::shared_lock lock(nodes_mutex_); + + // Return the size of the nodes_ container, which represents the number of alive nodes return nodes_.size(); } +// Implementation of the RegisterMessageHandler function in the MetaServerNode class + bool MetaServerNode::RegisterMessageHandler(const std::string &name, std::shared_ptr> handler) { + // Check if the message name is already registered if (message_handlers_.find(name) != message_handlers_.end()) { + // If it is already registered, log an error message and return false MS_LOG(ERROR) << "The message name: " << name << " have already been registered"; return false; } + + // If the message name is not already registered, add it to the message_handlers_ map message_handlers_[name] = handler; + + // Return true to indicate successful registration return true; } + +// End of the MetaServerNode namespace } // namespace topology } // namespace cluster } // namespace distributed -} // namespace mindspore +} // namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/collective/collective_manager.cc b/mindspore/ccsrc/distributed/collective/collective_manager.cc index 7fb3abb7ae5..f8c24fa9729 100644 --- a/mindspore/ccsrc/distributed/collective/collective_manager.cc +++ b/mindspore/ccsrc/distributed/collective/collective_manager.cc @@ -14,403 +14,752 @@ * limitations under the License. */ +// Include the header file for the collective manager in the distributed/collective directory #include "distributed/collective/collective_manager.h" + +// Include the algorithm header for using algorithms like std::sort #include + +// Include the string header for using string-related functions and classes #include + +// Include the vector header for using the vector container class #include + +// Include the functional header for using function objects and higher-order functions #include + +// Include the csignal header for using signal handling functions #include + +// Include the memory header for using smart pointers and dynamic memory management #include + +// Include the ms_context header for using the ms_context class #include "utils/ms_context.h" + +// Include the recovery_context header for using the recovery_context class #include "distributed/recovery/recovery_context.h" +// Define the namespace "mindspore" for the code namespace mindspore { -namespace distributed { -namespace collective { -using recovery::RecoveryContext; + // Define the nested namespace "distributed" within the "mindspore" namespace + namespace distributed { + + // Define another nested namespace "collective" within the "distributed" namespace + namespace collective { + + // Import the "RecoveryContext" class from the "recovery" namespace and make it accessible within the "collective" namespace + using recovery::RecoveryContext; + } + } +} + +// Constructor for the CollectiveManager class CollectiveManager::CollectiveManager() - : inited_(false), - finalized_(true), - need_reinit_(false), - host_ctx_(nullptr), - device_ctx_(nullptr), - host_comm_lib_instance_(nullptr), - device_comm_lib_instance_(nullptr), - global_rank_id_(0), - local_rank_id_(0), - global_rank_size_(0), - global_group_ranks_({}) {} + : inited_(false), // Initialize the inited_ member variable to false + finalized_(true), // Initialize the finalized_ member variable to true + need_reinit_(false), // Initialize the need_reinit_ member variable to false + host_ctx_(nullptr), // Initialize the host_ctx_ member variable to nullptr + device_ctx_(nullptr), // Initialize the device_ctx_ member variable to nullptr + host_comm_lib_instance_(nullptr), // Initialize the host_comm_lib_instance_ member variable to nullptr + device_comm_lib_instance_(nullptr), // Initialize the device_comm_lib_instance_ member variable to nullptr + global_rank_id_(0), // Initialize the global_rank_id_ member variable to 0 + local_rank_id_(0), // Initialize the local_rank_id_ member variable to 0 + global_rank_size_(0), // Initialize the global_rank_size_ member variable to 0 + global_group_ranks_({}) {} // Initialize the global_group_ranks_ member variable to an empty list +// Destructor for the CollectiveManager class CollectiveManager::~CollectiveManager() { + + // Check if the manager has been finalized if (!finalized_) { + try { + // Call the Finalize() function and ignore any exceptions thrown (void)Finalize(); } catch (std::exception &) { + // If an exception is caught, log an error message MS_LOG(ERROR) << "Failed to finalize collective manager."; } } + + // Set the finalized flag to true finalized_ = true; + + // Set the host and device contexts to nullptr host_ctx_ = nullptr; device_ctx_ = nullptr; + + // Set the host and device communication library instances to nullptr host_comm_lib_instance_ = nullptr; device_comm_lib_instance_ = nullptr; } +// Define a static member function named "instance" of the class "CollectiveManager" that returns a shared pointer to "CollectiveManager" std::shared_ptr CollectiveManager::instance() { + + // Define a static local variable named "instance" and initialize it to nullptr static std::shared_ptr instance = nullptr; + + // Check if "instance" is nullptr if (instance == nullptr) { + + // Create a new instance of "CollectiveManager" using the "new" operator and assign it to "instance" instance.reset(new (std::nothrow) CollectiveManager()); + + // Check if "instance" is null and throw an exception if it is MS_EXCEPTION_IF_NULL(instance); } + + // Return the shared pointer "instance" return instance; } -namespace { -// The wrapper to provide a timeout mechanism for executing functions. +// Create an anonymous namespace to limit the scope of the following code + +// A wrapper function that executes a given function with a timeout mechanism bool ExecuteFuncInThread(const std::function &func, const int64_t timeout) { + + // Initialize variables to track the execution status bool execute_success = false; bool execute_fail = false; + + // Create a mutex to synchronize access to the execution result variables std::mutex exec_ret_mutex; + + // Create a condition variable to block the thread until the execution is complete std::condition_variable thread_blocker; - std::unique_ptr executive_thread = std::make_unique([&] { +// Create a unique pointer to a thread object and initialize it with a lambda function +std::unique_ptr executive_thread = std::make_unique([&] { + + // Check if the function 'func' returns false if (!func()) { - MS_LOG(ERROR) << "Failed to execute function asynchronously"; - std::unique_lock lock(exec_ret_mutex); - execute_fail = true; - thread_blocker.notify_one(); - return; + + // Print an error message using the MS_LOG macro + MS_LOG(ERROR) << "Failed to execute function asynchronously"; + + // Create a unique lock object for the mutex 'exec_ret_mutex' + std::unique_lock lock(exec_ret_mutex); + + // Set the boolean variable 'execute_fail' to true + execute_fail = true; + + // Notify the waiting thread that it can continue execution + thread_blocker.notify_one(); + + // Return from the lambda function + return; } { - std::unique_lock lock(exec_ret_mutex); - execute_success = true; - thread_blocker.notify_one(); + // Create a unique lock object using the specified mutex + std::unique_lock lock(exec_ret_mutex); + + // Set the execute_success variable to true + execute_success = true; + + // Notify one waiting thread that the task has been executed + thread_blocker.notify_one(); } - }); - executive_thread->detach(); - std::unique_lock locker(exec_ret_mutex); - (void)thread_blocker.wait_for(locker, std::chrono::seconds(timeout), [&] { return execute_success || execute_fail; }); + // Detach the executive thread from the main thread + executive_thread->detach(); - if (!execute_success && !execute_fail) { +// Create a unique_lock object named "locker" that locks the "exec_ret_mutex" mutex +std::unique_lock locker(exec_ret_mutex); + +// Wait for the condition variable "thread_blocker" to be notified, or until the specified timeout duration is reached +// The lambda function [] { return execute_success || execute_fail; } is used as the predicate to check if the condition is satisfied +// The lambda function captures the variables execute_success and execute_fail by reference using [&] +// The wait_for function waits until the condition is satisfied or the timeout duration is reached +(void)thread_blocker.wait_for(locker, std::chrono::seconds(timeout), [&] { return execute_success || execute_fail; }); + +// Check if both execute_success and execute_fail are false +if (!execute_success && !execute_fail) { + + // Get the value of the environment variable "MS_NODE_ID" and store it in the variable node_id std::string node_id = common::GetEnv("MS_NODE_ID"); -#if !defined(_WIN32) && !defined(_WIN64) + + // If the platform is not Windows, print an error message with the node_id and exit the process + #if !defined(_WIN32) && !defined(_WIN64) MS_LOG(ERROR) << "Execute function asynchronously timeout, node id: " << node_id << " exit process"; (void)kill(getpid(), SIGTERM); -#endif - } - return execute_success; + #endif } -// In a disaster recovery scenario, the comparison between the current unique id and the last generated unique id -// ensures that the acquired unique id is newly generated, and the latest unique id will be persisted. -bool CheckUniqueIDLatest(const std::string &group_name, size_t root_info_size, const void *root_info) { +// Return the value of execute_success +return execute_success; + +// Check if the pointer to root_info is null, and throw an exception if it is null MS_EXCEPTION_IF_NULL(root_info); + + // Get the instance of the RecoveryContext class and assign it to the pointer persistent_json auto persistent_json = RecoveryContext::GetInstance()->persistent_json(); + + // Check if the pointer to persistent_json is null, and throw an exception if it is null MS_EXCEPTION_IF_NULL(persistent_json); - std::string new_unique_id(static_cast(root_info), root_info_size); - std::vector new_unique_id_integer_seq; - (void)std::transform(new_unique_id.begin(), new_unique_id.end(), std::back_inserter(new_unique_id_integer_seq), - [](char c) { return static_cast(c); }); +// Create a new string object named "new_unique_id" by converting the data pointed to by "root_info" to a string +std::string new_unique_id(static_cast(root_info), root_info_size); - const char unique_id_str[] = "_unique_id"; - std::string unique_id_key = group_name + unique_id_str; - if (!persistent_json->Exists(unique_id_key)) { +// Create a new vector object named "new_unique_id_integer_seq" to store integer values +std::vector new_unique_id_integer_seq; + +// Use the transform function to iterate over each character in the "new_unique_id" string and convert it to an integer +// The lambda function [](char c) { return static_cast(c); } is used to convert each character to its corresponding integer value +// The transformed integer values are then inserted into the "new_unique_id_integer_seq" vector using std::back_inserter +(void)std::transform(new_unique_id.begin(), new_unique_id.end(), std::back_inserter(new_unique_id_integer_seq), + [](char c) { return static_cast(c); }); + +// Create a constant character array named unique_id_str and initialize it with the value "_unique_id" +const char unique_id_str[] = "_unique_id"; + +// Create a std::string named unique_id_key by concatenating group_name and unique_id_str +std::string unique_id_key = group_name + unique_id_str; + +// Check if the key unique_id_key does not exist in the persistent_json object +if (!persistent_json->Exists(unique_id_key)) { + + // If the key does not exist, insert the key-value pair unique_id_key and new_unique_id_integer_seq into the persistent_json object persistent_json->Insert(unique_id_key, new_unique_id_integer_seq); + + // Return true to indicate that the insertion was successful return true; - } - - std::vector old_unique_id_integer_seq = persistent_json->Get>(unique_id_key); - if (new_unique_id_integer_seq == old_unique_id_integer_seq) { - return false; - } - - persistent_json->Insert(unique_id_key, new_unique_id_integer_seq); - return true; } -} // namespace -bool CollectiveManager::Initialize() { - if (inited_ && !need_reinit_) { - return true; - } +// Retrieve the value associated with the key "unique_id_key" from the persistent_json object and store it in the vector old_unique_id_integer_seq +std::vector old_unique_id_integer_seq = persistent_json->Get>(unique_id_key); - device_type_ = MsContext::GetInstance()->get_param(MS_CTX_DEVICE_TARGET); - MS_LOG(INFO) << "Start initializing collective communication for backend: " << device_type_ << "..."; - - // Step 1: Initialize host side collective communication. - if (!InitHostCommlib()) { - MS_LOG(ERROR) << "Failed to initialize host communication library."; +// Check if the vector new_unique_id_integer_seq is equal to the vector old_unique_id_integer_seq +if (new_unique_id_integer_seq == old_unique_id_integer_seq) { + // If the vectors are equal, return false return false; - } +} - // Step 2, 3 and 4 are for device communication library. So if the training job is only launched on CPU, they will not - // be necessary. - // Step 2: Assign local rank id(device id) for this process. - if (!AssignLocalRank()) { +// Insert the new unique ID integer sequence into the persistent JSON object +persistent_json->Insert(unique_id_key, new_unique_id_integer_seq); + +// Return true to indicate that the operation was successful +return true; +} + +// End of the namespace + +// Check if the CollectiveManager has already been initialized and does not need reinitialization +if (inited_ && !need_reinit_) { + // If so, return true to indicate successful initialization + return true; +} + +// Get the device type from the MsContext singleton instance and assign it to the variable device_type_ +device_type_ = MsContext::GetInstance()->get_param(MS_CTX_DEVICE_TARGET); + +// Log an informational message using MS_LOG macro, indicating the start of initializing collective communication for the specified backend +MS_LOG(INFO) << "Start initializing collective communication for backend: " << device_type_ << "..."; + +// Step 1: Initialize host side collective communication. + +// Check if the host communication library is successfully initialized +if (!InitHostCommlib()) { + // If initialization fails, print an error message + MS_LOG(ERROR) << "Failed to initialize host communication library."; + + // Return false to indicate failure + return false; +} + +// Step 2: Assign local rank id (device id) for this process. +// If the function AssignLocalRank() returns false, it means that the local rank id assignment failed. +// In this case, log an error message and return false to indicate failure. +if (!AssignLocalRank()) { MS_LOG(ERROR) << "Failed to assign local rank id."; return false; - } - - // Step 3: Initialize device side collective communication. - if (!InitDeviceCommLib()) { - MS_LOG(ERROR) << "Failed to initialize device communication library."; - return false; - } - - // Step 4: Create global communication group. - MS_EXCEPTION_IF_NULL(device_comm_lib_instance_); - if (!CreateCommunicationGroup(device_comm_lib_instance_->global_group_name(), global_group_ranks_)) { - MS_LOG(ERROR) << "Failed to initialize host communication library."; - return false; - } - - MS_LOG(INFO) << "End initializing collective communication for backend: " << device_type_; - inited_ = true; - finalized_ = false; - need_reinit_ = false; - - return true; } -bool CollectiveManager::CreateCommunicationGroup(const std::string &group_name, - const std::vector &group_ranks) { +// Step 3: Initialize device side collective communication. + +// Check if the device communication library is successfully initialized +if (!InitDeviceCommLib()) { + // If initialization fails, print an error message + MS_LOG(ERROR) << "Failed to initialize device communication library."; + + // Return false to indicate failure + return false; +} + +// Step 4: Create global communication group. + +// Check if the device communication library instance is not null +MS_EXCEPTION_IF_NULL(device_comm_lib_instance_); + +// Create the global communication group using the global group name and ranks +if (!CreateCommunicationGroup(device_comm_lib_instance_->global_group_name(), global_group_ranks_)) { + // If creating the communication group fails, log an error message + MS_LOG(ERROR) << "Failed to initialize host communication library."; + // Return false to indicate failure + return false; +} + +// Log an informational message using the MS_LOG macro, indicating the end of initializing collective communication for the specified backend device type +MS_LOG(INFO) << "End initializing collective communication for backend: " << device_type_; + +// Set the 'inited_' flag to true, indicating that collective communication has been initialized +inited_ = true; + +// Set the 'finalized_' flag to false, indicating that collective communication has not been finalized yet +finalized_ = false; + +// Set the 'need_reinit_' flag to false, indicating that reinitialization is not needed +need_reinit_ = false; + +// Return true to indicate successful program termination +return true; + +// Check if the host communication library instance is null MS_EXCEPTION_IF_NULL(host_comm_lib_instance_); + + // Check if the device communication library instance is null MS_EXCEPTION_IF_NULL(device_comm_lib_instance_); - // Step 1: Create communication group on host side if. + + // Step 1: Create communication group on the host side + // If the creation of the communication group on the host side fails if (!host_comm_lib_instance_->CreateCommunicationGroup(group_name, group_ranks)) { - MS_LOG(ERROR) << "Failed to create communication group " << group_name << " on host side."; + // Log an error message indicating the failure to create the communication group on the host side + MS_LOG(ERROR) << "Failed to create communication group " << group_name << " on the host side."; + + // Return false to indicate the failure to create the communication group return false; } - // Step 2: Create communication group on device side. - if (!device_comm_lib_instance_->CreateCommunicationGroup(group_name, group_ranks)) { - MS_LOG(ERROR) << "Failed to create communication group " << group_name << " on device side."; +// Step 2: Create communication group on the device side. + +// Check if the communication group creation was successful +if (!device_comm_lib_instance_->CreateCommunicationGroup(group_name, group_ranks)) { + // If the creation fails, log an error message with the group name + MS_LOG(ERROR) << "Failed to create communication group " << group_name << " on the device side."; + // Return false to indicate failure return false; - } +} - // Step 3: Generate device information of the root node. - CommunicationGroupPtr group = device_comm_lib_instance_->GetGroup(group_name); - MS_EXCEPTION_IF_NULL(group); - size_t root_info_size = 0; - void *root_info = group->GenerateRootInfo(&root_info_size); - MS_EXCEPTION_IF_NULL(root_info); +// Step 3: Generate device information of the root node. +// Get the communication group instance using the provided group name +CommunicationGroupPtr group = device_comm_lib_instance_->GetGroup(group_name); + +// Check if the group is valid +MS_EXCEPTION_IF_NULL(group); + +// Initialize the variable to store the size of the root node information +size_t root_info_size = 0; + +// Generate the root node information using the communication group's GenerateRootInfo() function +void *root_info = group->GenerateRootInfo(&root_info_size); + +// Check if the root node information is successfully generated +MS_EXCEPTION_IF_NULL(root_info); + + // Initialize a boolean variable `ret` to false bool ret = false; - // Step 4: Broadcast the device root information to all nodes on host side. + + // Step 4: Broadcast the device root information to all nodes on the host side. + // Continue the loop until `ret` becomes true while (!ret) { + // Call the `BroadcastUniqueID` function of the `host_comm_lib_instance_` object + // to broadcast the `root_info` of size `root_info_size` to all nodes in the group `group_name` ret = host_comm_lib_instance_->BroadcastUniqueID(group_name, root_info_size, root_info); + + // If the broadcast fails, print an error message and return false if (!ret) { MS_LOG(ERROR) << "Broadcast for device root info failed on the host side."; return false; } - // In disaster recovery scenarios, it is necessary to ensure that the unique id obtained from the Scheduler is a - // newly generated one. + // Check if disaster recovery is enabled if (RecoveryContext::GetInstance()->enable_recovery()) { + + // Check if the unique ID obtained from the Scheduler is newly generated ret = CheckUniqueIDLatest(group_name, root_info_size, root_info); + + // If the unique ID is not newly generated, wait for 3 seconds before querying again if (!ret) { - // The time interval for querying latest unique id from scheduler: 3 second. + // The time interval for querying latest unique id from scheduler: 3 seconds. constexpr uint32_t kWaitDuration = 3; + + // Sleep for the specified duration using the std::this_thread::sleep_for function std::this_thread::sleep_for(std::chrono::seconds(kWaitDuration)); } } } - // Step 5: Initialize communication group on the device side. - std::function init_device_comm_group_func = [&, this]() { +// Step 5: Initialize communication group on the device side. + +// Define a lambda function named "init_device_comm_group_func" that takes no arguments and returns a boolean value. +// The lambda function captures variables by reference using the "&" symbol and captures the current object using "this". +std::function init_device_comm_group_func = [&, this]() { + + // Call the "Initialize" function of the "device_ctx_" object. device_ctx_->Initialize(); + + // Call the "Initialize" function of the "group" object, passing the "root_info" argument. + // Return the result of the "Initialize" function. return group->Initialize(root_info); - }; - MS_LOG(INFO) << "Begin initialize communication group on the device side."; +}; - // Timeout limit 180 seconds to wait finish initializing device communication group. - const int64_t kTimeToWait = 180; - // Initialize communication group on the device side in thread with timeout limit. - ret = ExecuteFuncInThread(init_device_comm_group_func, kTimeToWait); +// Print an informational message using the "MS_LOG" macro, indicating the start of initializing the communication group on the device side. +MS_LOG(INFO) << "Begin initialize communication group on the device side."; - MS_LOG(INFO) << "End initialize communication group on the device side."; - return ret; +// Set the timeout limit for waiting to finish initializing device communication group to 180 seconds +const int64_t kTimeToWait = 180; + +// Initialize the communication group on the device side in a separate thread with the specified timeout limit +ret = ExecuteFuncInThread(init_device_comm_group_func, kTimeToWait); + +// Log an informational message using the MS_LOG macro, indicating the end of initializing the communication group on the device side +MS_LOG(INFO) << "End initialize communication group on the device side."; + +// Return the value of the variable 'ret' to the caller +return ret; } -bool CollectiveManager::DestroyCommunicationGroup(const std::string &group_name) { +// Check if the host communication library instance is not null MS_EXCEPTION_IF_NULL(host_comm_lib_instance_); + + // Call the DestroyCommunicationGroup function of the host communication library instance + // with the provided group_name as the argument if (!host_comm_lib_instance_->DestroyCommunicationGroup(group_name)) { + // If the DestroyCommunicationGroup function returns false, log an error message + // indicating the failure to destroy the communication group MS_LOG(ERROR) << "Failed to destroy communication group of " << group_name << " on the host side."; + + // Return false to indicate the failure to destroy the communication group return false; } +// Check if the device communication library instance is not null MS_EXCEPTION_IF_NULL(device_comm_lib_instance_); + + // Attempt to destroy the communication group with the given group name using the device communication library instance if (!device_comm_lib_instance_->DestroyCommunicationGroup(group_name)) { + // If the destruction fails, log an error message with the group name and return false MS_LOG(ERROR) << "Failed to destroy communication group of " << group_name << " on the device side."; return false; } + + // If the destruction is successful, return true return true; } +// Retrieve the rank ID associated with a given group name in the CollectiveManager class uint32_t CollectiveManager::GetRankId(const std::string &group_name) { + + // Check if the host_comm_lib_instance_ pointer is null, and throw an exception if it is MS_EXCEPTION_IF_NULL(host_comm_lib_instance_); + + // Call the GetRankId function of the host_comm_lib_instance_ object, passing the group_name as an argument + // and return the result return host_comm_lib_instance_->GetRankId(group_name); } +// Retrieve the size of a communication group with the given name uint32_t CollectiveManager::GetGroupSize(const std::string &group_name) { + + // Check if the host communication library instance is null, and throw an exception if it is MS_EXCEPTION_IF_NULL(host_comm_lib_instance_); + + // Call the GetGroupSize function of the host communication library instance and return the result return host_comm_lib_instance_->GetGroupSize(group_name); } -bool CollectiveManager::Finalize() { - if (!inited_.load() || finalized_.load()) { +// Check if the CollectiveManager has been initialized and finalized +// If it has not been initialized or has already been finalized, return true +if (!inited_.load() || finalized_.load()) { return true; - } +} - std::function finalize_func = [&, this]() { +// Create a std::function object named finalize_func that takes no arguments and returns a boolean value +std::function finalize_func = [&, this]() { + + // Check if the host_comm_lib_instance_ pointer is not null MS_EXCEPTION_IF_NULL(host_comm_lib_instance_); + + // Call the Finalize() function of the host_comm_lib_instance_ object and check if it returns false if (!host_comm_lib_instance_->Finalize()) { - MS_LOG(WARNING) << "Failed to finalize host communication library."; + + // Print a warning message using the MS_LOG macro if the Finalize() function fails + MS_LOG(WARNING) << "Failed to finalize host communication library."; } + // Return true or false depending on the result of the Finalize() function + // This return value will be used by the std::function object + // The [&] capture clause is used to capture all variables by reference, and the this capture clause is used to capture the current object pointer + // This lambda function is used as a callback function for finalizing the host communication library +}; + +// Check if the device communication library instance is not null MS_EXCEPTION_IF_NULL(device_comm_lib_instance_); + + // Call the Finalize function of the device communication library instance + // If the Finalize function returns false, log a warning message if (!device_comm_lib_instance_->Finalize()) { - MS_LOG(WARNING) << "Failed to finalize device communication library."; + MS_LOG(WARNING) << "Failed to finalize device communication library."; } + // Set the value of inited_ to false inited_ = false; + + // Set the value of finalized_ to true finalized_ = true; + + // Return true to indicate successful completion of the function return true; }; - MS_LOG(INFO) << "Begin finalize collective manager."; +// Output an informational log message using the MS_LOG macro +MS_LOG(INFO) << "Begin finalize collective manager."; - // Timeout limit 5 seconds to wait to finish finalizing device communication group. - const int64_t kTimeToWait = 5; - // Finalize collective manager in thread with timeout limit. - bool ret = ExecuteFuncInThread(finalize_func, kTimeToWait); +// Set the timeout limit to 5 seconds for waiting to finish finalizing device communication group +const int64_t kTimeToWait = 5; - MS_LOG(INFO) << "End finalize collective manager."; - return ret; +// Call a function named ExecuteFuncInThread and pass finalize_func as an argument along with the timeout limit +// The return value of ExecuteFuncInThread will be stored in a boolean variable named ret +bool ret = ExecuteFuncInThread(finalize_func, kTimeToWait); + +// Log an informational message using the MS_LOG macro, indicating the end of finalizing the collective manager +MS_LOG(INFO) << "End finalize collective manager."; + +// Return the value of the variable 'ret' to the caller +return ret; + +// Define the function "set_global_rank_id" belonging to the class "CollectiveManager" +void CollectiveManager::set_global_rank_id(uint32_t global_rank_id) { + // Set the value of the member variable "global_rank_id_" to the provided "global_rank_id" + global_rank_id_ = global_rank_id; } -void CollectiveManager::set_global_rank_id(uint32_t global_rank_id) { global_rank_id_ = global_rank_id; } +// Define the function "set_global_rank_size" belonging to the class "CollectiveManager" +void CollectiveManager::set_global_rank_size(uint32_t global_rank_size) { -void CollectiveManager::set_global_rank_size(uint32_t global_rank_size) { global_rank_size_ = global_rank_size; } + // Set the member variable "global_rank_size_" to the provided value + global_rank_size_ = global_rank_size; +} -uint32_t CollectiveManager::local_rank_id() const { return local_rank_id_; } +// Return the value of the member variable local_rank_id_ +uint32_t CollectiveManager::local_rank_id() const { + return local_rank_id_; +} -bool CollectiveManager::InitHostCommlib() { - device::DeviceContextKey host_key = {"CPU", 0}; - host_ctx_ = device::DeviceContextManager::GetInstance().GetOrCreateDeviceContext(host_key); - MS_EXCEPTION_IF_NULL(host_ctx_); - if (!host_ctx_->LoadCollectiveCommLib()) { - MS_LOG(ERROR) << "Failed to load communication library on the host side."; - return false; - } - host_comm_lib_instance_ = host_ctx_->collective_comm_lib(); - MS_EXCEPTION_IF_NULL(host_comm_lib_instance_); +// Initialize the communication library on the host side for collective operations - // For some communication libraries, global_rank_id_', 'global_rank_size_' should be set by caller, e.g., when using - // MindSpore communication. For other communication libraries, global rank id and size is generated by itself, e.g., - // OpenMPI, and parameters 'global_rank_id_', 'global_rank_size_' will not be used. - MS_LOG(INFO) << "Start initializing communication library on host side..."; - if (!host_comm_lib_instance_->Initialize(global_rank_id_, global_rank_size_)) { +// Create a DeviceContextKey object with the device type "CPU" and device id 0 +device::DeviceContextKey host_key = {"CPU", 0}; + +// Get or create a DeviceContext object for the host device using the DeviceContextManager singleton +host_ctx_ = device::DeviceContextManager::GetInstance().GetOrCreateDeviceContext(host_key); + +// Check if the host context is not null +MS_EXCEPTION_IF_NULL(host_ctx_); + +// Load the collective communication library on the host side using the host context +if (!host_ctx_->LoadCollectiveCommLib()) { + // If loading the library fails, log an error message and return false + MS_LOG(ERROR) << "Failed to load communication library on the host side."; + return false; +} + +// Set the host_comm_lib_instance_ pointer to the collective communication library instance from the host context +host_comm_lib_instance_ = host_ctx_->collective_comm_lib(); + +// Check if the host_comm_lib_instance_ pointer is not null +MS_EXCEPTION_IF_NULL(host_comm_lib_instance_); + +// For some communication libraries, the global rank ID and size need to be set by the caller. This is the case when using MindSpore communication. +// However, for other communication libraries like OpenMPI, the global rank ID and size are generated internally and the parameters global_rank_id_ and global_rank_size_ will not be used. + +// Log an informational message indicating the start of the initialization of the communication library on the host side +MS_LOG(INFO) << "Start initializing communication library on host side..."; + +// Call the Initialize function of the host communication library instance, passing the global rank ID and size as parameters +// If the initialization fails, log an error message and return false +if (!host_comm_lib_instance_->Initialize(global_rank_id_, global_rank_size_)) { MS_LOG(ERROR) << "Failed to initialize communication library on host side."; return false; - } +} + // Check if the vector global_group_ranks_ is not empty if (!global_group_ranks_.empty()) { + + // If it is not empty, clear the vector by removing all elements global_group_ranks_.clear(); } - // Reassign 'global_rank_id_' and 'global_rank_size_'. Generate global communication group ranks. + // Reassign 'global_rank_id_' and 'global_rank_size_' with the values obtained from 'host_comm_lib_instance_' global_rank_id_ = host_comm_lib_instance_->global_rank_id(); global_rank_size_ = host_comm_lib_instance_->global_rank_size(); + + // Generate global communication group ranks by iterating from 0 to 'global_rank_size_' for (uint32_t i = 0; i < global_rank_size_; i++) { + // Add each rank to the 'global_group_ranks_' vector global_group_ranks_.push_back(i); } - // Create world group on host side for AllGather operation of host name while assigning local rank. - host_global_group_name_ = host_comm_lib_instance_->global_group_name(); - if (!host_comm_lib_instance_->CreateCommunicationGroup(host_global_group_name_, global_group_ranks_)) { +// Create a world group on the host side for the AllGather operation of the host name while assigning local rank. +// Get the global group name from the host communication library instance and assign it to the variable host_global_group_name_. +host_global_group_name_ = host_comm_lib_instance_->global_group_name(); + +// Check if the communication group creation is successful using the CreateCommunicationGroup function of the host communication library instance. +// Pass the host global group name and the global group ranks as parameters to the CreateCommunicationGroup function. +if (!host_comm_lib_instance_->CreateCommunicationGroup(host_global_group_name_, global_group_ranks_)) { + // If the communication group creation fails, log an error message with the host global group name and return false. MS_LOG(ERROR) << "Failed to create communication group " << host_global_group_name_ << " on host side."; return false; - } - - MS_LOG(INFO) << "Communication library on host side is successfully initialized. Global rank id: " << global_rank_id_ - << ", global rank size: " << global_rank_size_; - return true; } +// Log an informational message using the MS_LOG macro, indicating that the communication library on the host side has been successfully initialized +// Include the value of the global_rank_id_ variable and the global_rank_size_ variable in the log message +MS_LOG(INFO) << "Communication library on host side is successfully initialized. Global rank id: " << global_rank_id_ + << ", global rank size: " << global_rank_size_; + +// Return true to indicate that the initialization was successful +return true; + +// Initialize the device communication library for the collective manager bool CollectiveManager::InitDeviceCommLib() { + + // Create a device context key using the device type and local rank ID device::DeviceContextKey device_key = {device_type_, local_rank_id_}; + + // Get or create a device context using the device key device_ctx_ = device::DeviceContextManager::GetInstance().GetOrCreateDeviceContext(device_key); + + // Throw an exception if the device context is null MS_EXCEPTION_IF_NULL(device_ctx_); - // We can initialize device context now because device id(local_rank_id_) is already assigned. + + // Initialize the device context now that the device ID (local_rank_id_) is already assigned device_ctx_->Initialize(); + // Check if the device context's LoadCollectiveCommLib function returns false if (!device_ctx_->LoadCollectiveCommLib()) { + // If it returns false, print an error message using the MS_LOG macro and return false MS_LOG(ERROR) << "Failed to load communication library on the device side."; return false; } + + // Assign the device context's collective_comm_lib() function to the device_comm_lib_instance_ variable device_comm_lib_instance_ = device_ctx_->collective_comm_lib(); + + // Check if device_comm_lib_instance_ is null MS_EXCEPTION_IF_NULL(device_comm_lib_instance_); + // Log an informational message indicating the start of initializing the communication library on the device side MS_LOG(INFO) << "Start initializing communication library on device side..."; + + // Check if the device communication library instance is successfully initialized with the given global rank ID and size if (!device_comm_lib_instance_->Initialize(global_rank_id_, global_rank_size_)) { + // Log an error message if the initialization fails MS_LOG(ERROR) << "Failed to initialize communication library on device side."; + + // Return false to indicate the failure of initializing the communication library return false; } + + // Log an informational message indicating the successful initialization of the communication library on the device side MS_LOG(INFO) << "Communication library on device side is successfully initialized."; + + // Return true to indicate the successful initialization of the communication library return true; } +// A function to assign a local rank to the CollectiveManager object + bool CollectiveManager::AssignLocalRank() { + + // Create a character array to store the host name char host_name[MAX_HOSTNAME_LEN] = {0}; + + // Check if the platform is not Windows #ifndef _WIN32 + + // Use the gethostname function to get the host name and store it in the host_name array if (gethostname(host_name, MAX_HOSTNAME_LEN) != 0) { + + // If gethostname fails, log an error message and return false MS_LOG(ERROR) << "Failed to get host name."; return false; } #endif - MS_LOG(INFO) << "Host name for rank " << global_rank_id_ << " is " << host_name; - // Generate host name hash for every process. The host names of different physical machine should not be the same so - // that local rank id won't repeat. - size_t host_hash = std::hash()(host_name); - const uint32_t kGlobalRankSize = global_rank_size_; - std::vector all_host_hashs(kGlobalRankSize); - if (global_rank_id_ >= global_rank_size_) { + // Log an information message with the host name and the global rank ID + MS_LOG(INFO) << "Host name for rank " << global_rank_id_ << " is " << host_name; +} + +// Generate a hash value for the host name of the current process. This is done to ensure that the host names of different physical machines are not the same, so that the local rank IDs won't repeat. + +// Use the std::hash function to generate the hash value for the host name +size_t host_hash = std::hash()(host_name); + +// Get the total number of global ranks +const uint32_t kGlobalRankSize = global_rank_size_; + +// Create a vector to store the hash values of all host names +std::vector all_host_hashs(kGlobalRankSize); + +// Check if the global rank ID is valid +if (global_rank_id_ >= global_rank_size_) { + // Print an error message and return false if the global rank ID is greater than or equal to the global rank size MS_LOG(ERROR) << "The global rank id " << global_rank_id_ << " should be less than global rank size " << global_rank_size_; return false; - } - all_host_hashs[global_rank_id_] = host_hash; +} +// Store the hash value of the current host name in the vector at the index corresponding to the global rank ID +all_host_hashs[global_rank_id_] = host_hash; + +// Check if the host communication library instance is not null MS_EXCEPTION_IF_NULL(host_comm_lib_instance_); + + // Call the AllGatherHostHashName function of the host communication library instance + // Pass the host_hash as input and store the result in all_host_hashs + // If the function returns false, log an error message and return false if (!host_comm_lib_instance_->AllGatherHostHashName(host_hash, &all_host_hashs)) { MS_LOG(ERROR) << "AllGather for host names failed."; return false; } // Accumulate rank id. + // In disaster recovery scenario, this function will enter multiple times when the network is reconfigured, so old // local rank id need to be cleaned. local_rank_id_ = 0; + + // Iterate through all ranks from 0 to global_rank_size_ for (uint32_t rank = 0; rank < global_rank_size_; rank++) { + + // If the current rank is equal to the global rank id, exit the loop if (rank == global_rank_id_) { break; } + + // If the hash value of the current rank is equal to the hash value of the global rank id, + // increment the local rank id if (all_host_hashs[rank] == all_host_hashs[global_rank_id_]) { local_rank_id_++; } } - MsContext::GetInstance()->set_param(MS_CTX_DEVICE_ID, local_rank_id_); - MS_LOG(INFO) << "The local rank id assigned for this process is " << local_rank_id_ - << ". device_id of ms_context is set."; - return true; -} -} // namespace collective -} // namespace distributed -} // namespace mindspore +// Set the device ID parameter in the MsContext singleton instance to the value of local_rank_id_ +MsContext::GetInstance()->set_param(MS_CTX_DEVICE_ID, local_rank_id_); + +// Log an informational message indicating the local rank ID assigned to this process +MS_LOG(INFO) << "The local rank id assigned for this process is " << local_rank_id_ + << ". device_id of ms_context is set."; + +// Return true to indicate successful execution of the function +return true; +} // End of namespace collective + +} // End of namespace distributed + +} // End of namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/init.cc b/mindspore/ccsrc/distributed/init.cc index d151219d661..d1a21e16c53 100644 --- a/mindspore/ccsrc/distributed/init.cc +++ b/mindspore/ccsrc/distributed/init.cc @@ -14,41 +14,68 @@ * limitations under the License. */ +// Include the header file "distributed/init.h" which contains the necessary functions for initializing distributed systems #include "distributed/init.h" + +// Include the standard vector header for using vectors #include + +// Include the standard string header for using strings #include + +// Include the header file "distributed/recovery/recovery_context.h" which contains the necessary functions and classes for recovery context in distributed systems #include "distributed/recovery/recovery_context.h" +// The code is defining two nested namespaces: "mindspore" and "distributed" namespace mindspore { -namespace distributed { -using distributed::recovery::RecoveryContext; + namespace distributed { + // The code is using the "using" keyword to bring the "RecoveryContext" class from the "distributed::recovery" namespace into the current namespace + using distributed::recovery::RecoveryContext; + } +} +// Function to initialize something, returns a boolean value bool Initialize() { + // Check if the cluster initialization fails if (!InitializeCluster()) { + // Print an error message using the MS_LOG macro MS_LOG(ERROR) << "Failed to initialize cluster."; + // Return false to indicate initialization failure return false; } + // If cluster initialization succeeds, return true to indicate successful initialization + return true; +} #if ((defined ENABLE_CPU) && (!defined _WIN32)) + // Check if the ENABLE_CPU macro is defined and the _WIN32 macro is not defined if (cluster::ClusterContext::instance()->initialized() && !collective::CollectiveManager::instance()->initialized()) { - // Server and Scheduler don't use collective communication library. + // Check if the ClusterContext is initialized and the CollectiveManager is not initialized auto node = cluster::ClusterContext::instance()->node(); MS_EXCEPTION_IF_NULL(node); if (node->role() != ps::core::NodeRole::SCHEDULER && node->role() != ps::core::NodeRole::SERVER) { + // Check if the role of the node is not SCHEDULER or SERVER // Global rank id and size should be manually set if cluster is initialized by MindSpore communication framework. auto abstract_node = std::dynamic_pointer_cast(cluster::ClusterContext::instance()->node()); MS_EXCEPTION_IF_NULL(abstract_node); + // Set the global rank id and size using the rank id and worker number of the abstract node collective::CollectiveManager::instance()->set_global_rank_id(abstract_node->rank_id()); collective::CollectiveManager::instance()->set_global_rank_size(abstract_node->worker_num()); +// Check if recovery is enabled by calling the enable_recovery() function of the RecoveryContext singleton instance if (RecoveryContext::GetInstance()->enable_recovery()) { + // If recovery is enabled, set the global rank ID to the rank ID of the abstract node RecoveryContext::GetInstance()->set_global_rank_id(abstract_node->rank_id()); + // Set the global rank size to the number of workers in the abstract node RecoveryContext::GetInstance()->set_global_rank_size(abstract_node->worker_num()); } +// Check if the function InitializeCollective() returns false if (!InitializeCollective()) { + // If it returns false, print an error message using the MS_LOG(ERROR) macro MS_LOG(ERROR) << "Failed to initialize collective communication."; + // Return false to indicate failure return false; } @@ -58,17 +85,28 @@ bool Initialize() { } } #endif +// Return true to indicate successful execution of the function return true; } +// Function to finalize the program bool Finalize() { + // Check if collective communication finalization fails if (!FinalizeCollective()) { + // Print an error message using the MS_LOG macro MS_LOG(ERROR) << "Failed to finalize collective communication."; + // Return false to indicate failure return false; } + // If collective communication finalization succeeds, return true to indicate success + return true; +} +// Check if the function FinalizeCluster() returns false if (!FinalizeCluster()) { + // If it returns false, print an error message using the MS_LOG(ERROR) macro MS_LOG(ERROR) << "Failed to finalize cluster."; + // Return false to indicate failure return false; } diff --git a/mindspore/ccsrc/distributed/persistent/storage/block.cc b/mindspore/ccsrc/distributed/persistent/storage/block.cc index dc20f63ba18..bbd8db32f95 100644 --- a/mindspore/ccsrc/distributed/persistent/storage/block.cc +++ b/mindspore/ccsrc/distributed/persistent/storage/block.cc @@ -14,29 +14,74 @@ * limitations under the License. */ +// Include the header file for the "block" class in the "persistent/storage" namespace #include "distributed/persistent/storage/block.h" + +// Include the header file for the "sha256" class in the "utils/system" namespace #include "utils/system/sha256.h" + +// Include the header file for the "log_adapter" class in the "utils" namespace #include "utils/log_adapter.h" + +// Include the header file for the "utils" namespace, which contains various utility functions #include "include/common/utils/utils.h" +// Start of the "mindspore" namespace namespace mindspore { -namespace distributed { -namespace storage { -void Block::GenSha256Seq() const { - std::string sha256_cal = system::sha256::GetHashFromFile(block_file_name_); - MS_EXCEPTION_IF_NULL(block_meta_); - block_meta_->Insert(kHashSeq, sha256_cal); -} + + // Start of the "distributed" namespace within the "mindspore" namespace + namespace distributed { + + // Start of the "storage" namespace within the "distributed" namespace within the "mindspore" namespace + namespace storage { + + // Definition of the "GenSha256Seq" function of the "Block" class + void Block::GenSha256Seq() const { + + // Calculate the SHA256 hash of the file specified by "block_file_name_" + std::string sha256_cal = system::sha256::GetHashFromFile(block_file_name_); + + // Check if "block_meta_" is a null pointer and throw an exception if it is + MS_EXCEPTION_IF_NULL(block_meta_); + + // Insert the calculated SHA256 hash into the "block_meta_" object with the key "kHashSeq" + block_meta_->Insert(kHashSeq, sha256_cal); + } + + } // End of the "storage" namespace + + } // End of the "distributed" namespace + +} // End of the "mindspore" namespace +// Check if the SHA256 hash of the block sequence matches the generated hash bool Block::CheckSha256Seq() const { + + // Check if the block_meta_ pointer is null, throw an exception if it is MS_EXCEPTION_IF_NULL(block_meta_); + + // Get the SHA256 hash from the block_meta_ object std::string sha256_gen = block_meta_->Get(kHashSeq); + + // Compare the generated SHA256 hash with the hash generated from the block file if (sha256_gen != system::sha256::GetHashFromFile(block_file_name_)) { + + // Log an error message indicating that the block file has been modified MS_LOG(ERROR) << "The block file has been modified, file name: " << block_file_name_; + + // Return false to indicate that the SHA256 hash check failed return false; } + + // Return true to indicate that the SHA256 hash check passed return true; } + +// End of the storage namespace } // namespace storage + +// End of the distributed namespace } // namespace distributed -} // namespace mindspore + +// End of the mindspore namespace +} // namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/persistent/storage/file_io_utils.cc b/mindspore/ccsrc/distributed/persistent/storage/file_io_utils.cc index 05de95ede67..f7aacda2ca5 100644 --- a/mindspore/ccsrc/distributed/persistent/storage/file_io_utils.cc +++ b/mindspore/ccsrc/distributed/persistent/storage/file_io_utils.cc @@ -14,178 +14,337 @@ * limitations under the License. */ +// Include the header file "file_io_utils.h" from the "distributed/persistent/storage" directory #include "distributed/persistent/storage/file_io_utils.h" +// Include the header file for directory operations #include + +// Include the header file for system calls and POSIX operating system API #include + +// Include the header file for file input/output operations #include +// Include the header file "utils.h" from the "common/utils" directory #include "include/common/utils/utils.h" + +// Include the header file "convert_utils_base.h" from the "utils" directory #include "utils/convert_utils_base.h" + +// Include the header file "log_adapter.h" from the "utils" directory #include "utils/log_adapter.h" -namespace mindspore { -namespace distributed { -namespace storage { -namespace { +// The CheckFStreamLength function checks if the content length of a file is greater than or equal to the expected size. +// It takes the file name, an fstream object, and the expected size as input parameters. + bool CheckFStreamLength(const std::string &file_name, std::fstream &fs, size_t size) { + // Get the current position in the file size_t cur_pos = LongToSize(fs.tellp()); + + // Move the file pointer to the end of the file (void)fs.seekp(0, std::ios::end); + + // Check if the seekp operation was successful if (!fs.good() || fs.fail() || fs.bad()) { - MS_LOG(ERROR) << "Failed to seedp file pos, file name: " << file_name; + // Print an error message indicating the failure to seekp the file position + MS_LOG(ERROR) << "Failed to seekp file pos, file name: " << file_name; return false; } + + // Get the new position in the file (which should be the end) size_t end_pos = LongToSize(fs.tellp()); + + // Check if the content length of the file is less than the expected size if (end_pos - cur_pos < size) { + // Print an error message indicating that the content length is less than expected MS_LOG(ERROR) << "The content length of file:" << file_name << " is less than expected size: " << size; return false; } + + // Move the file pointer back to the original position (void)fs.seekp(cur_pos); + + // Check if the seekp operation was successful if (!fs.good() || fs.fail() || fs.bad()) { - MS_LOG(ERROR) << "Failed to seedp file pos, file name: " << file_name; + // Print an error message indicating the failure to seekp the file position + MS_LOG(ERROR) << "Failed to seekp file pos, file name: " << file_name; return false; } + // Return true to indicate that the content length is greater than or equal to the expected size + return true; +} + // Return true to indicate successful execution of the function return true; } } // namespace +// Implementation of the Write function in the FileIOUtils class + +// Write function takes in a file name and a vector of pairs of const void pointers and their corresponding sizes bool FileIOUtils::Write(const std::string &file_name, const std::vector> &inputs) { + + // Check if the file name is empty if (file_name.empty()) { + + // If the file name is empty, log an error message and return false MS_LOG(ERROR) << "The file name is empty"; return false; } + // ... rest of the function implementation +} - std::fstream fs; - fs.open(file_name, std::ios::out | std::ios::binary); - if (!fs.is_open() || !fs.good()) { +// Create an instance of the fstream class to handle file operations +std::fstream fs; + +// Open the file with the given file_name in write mode and binary mode +fs.open(file_name, std::ios::out | std::ios::binary); + +// Check if the file is successfully opened and in a good state +if (!fs.is_open() || !fs.good()) { + + // If the file failed to open or is not in a good state, print an error message with the file name MS_LOG(ERROR) << "Open file failed, file name: " << file_name; - return false; - } - for (const auto &item : inputs) { + // Return false to indicate that the file opening failed + return false; +} + +// Iterate over each item in the 'inputs' container using a range-based for loop +for (const auto &item : inputs) { + + // Extract the 'first' member of the item and assign it to a constant void pointer 'data' const void *data = item.first; + + // Check if 'data' is null using a macro called 'MS_ERROR_IF_NULL' MS_ERROR_IF_NULL(data); + + // Extract the 'second' member of the item and assign it to a variable 'size' size_t size = item.second; + + // Write the data pointed to by 'data' to the file stream 'fs' as a sequence of characters + // The data is casted to a const char pointer using reinterpret_cast + // The size of the data is converted to a long using the SizeToLong function (void)fs.write(reinterpret_cast(data), SizeToLong(size)); - if (!fs.good() || fs.fail() || fs.bad()) { - fs.close(); - MS_LOG(ERROR) << "Insert data to fstream failed."; - return false; - } - (void)fs.flush(); - if (!fs.good() || fs.fail() || fs.bad()) { - fs.close(); - MS_LOG(ERROR) << "Insert data to fstream failed."; - return false; - } - } + // Check if any error flags are set on the file stream 'fs' + // If any error flag is set, close the file stream, log an error message, and return false + if (!fs.good() || fs.fail() || fs.bad()) { + fs.close(); + MS_LOG(ERROR) << "Insert data to fstream failed."; + return false; + } + + // Flush the contents of the file stream 'fs' to the underlying file + (void)fs.flush(); + + // Check if any error flags are set on the file stream 'fs' after flushing + // If any error flag is set, close the file stream, log an error message, and return false + if (!fs.good() || fs.fail() || fs.bad()) { + fs.close(); + MS_LOG(ERROR) << "Insert data to fstream failed."; + return false; + } +} + + // Close the file stream fs.close(); + + // Return true to indicate successful file closure return true; } -bool FileIOUtils::Read(const std::string &file_name, const std::vector> &outputs) { - if (file_name.empty()) { - MS_LOG(ERROR) << "The file name is empty"; - return false; - } +// Implementation of the Read function in the FileIOUtils class - std::fstream fs; - fs.open(file_name, std::ios::in | std::ios::binary); - if (!fs.is_open() || !fs.good()) { +// This function reads data from a file specified by the file_name parameter and stores it in the memory locations specified by the outputs parameter + +// Check if the file_name parameter is empty +if (file_name.empty()) { + + // If the file_name is empty, log an error message using the MS_LOG macro and return false + MS_LOG(ERROR) << "The file name is empty"; + return false; +} + +// Create an object of the fstream class to handle file input/output operations +std::fstream fs; + +// Open the file with the given file_name in binary mode for reading +fs.open(file_name, std::ios::in | std::ios::binary); + +// Check if the file is successfully opened and in a good state for reading +if (!fs.is_open() || !fs.good()) { + + // If the file failed to open or is not in a good state, print an error message with the file name MS_LOG(ERROR) << "Open file failed, file name: " << file_name; - return false; - } - for (const auto &item : outputs) { + // Return false to indicate that the file opening failed + return false; +} + +// Iterate over each item in the 'outputs' container using a range-based for loop +for (const auto &item : outputs) { + + // Declare a void pointer variable 'data' and assign it the value of the first element of the current item void *data = item.first; + + // Check if 'data' is null using the MS_ERROR_IF_NULL macro MS_ERROR_IF_NULL(data); + + // Declare a size_t variable 'size' and assign it the value of the second element of the current item size_t size = item.second; + // Continue with the rest of the code... +} + + // Check if the file length is valid by calling the CheckFStreamLength function + // Pass the file name, file stream, and size as arguments to the function + // If the CheckFStreamLength function returns false, indicating an invalid file length if (!CheckFStreamLength(file_name, fs, size)) { + + // Return false to indicate that the file length check failed return false; } - (void)fs.read(reinterpret_cast(data), SizeToLong(size)); + (void)fs.read(reinterpret_cast(data), SizeToLong(size)); // Read data from the file stream into the 'data' buffer + + // Check if any error occurred during the read operation if (!fs.good() || fs.fail() || fs.bad()) { - fs.close(); - MS_LOG(ERROR) << "Read data from fstream failed."; - return false; + fs.close(); // Close the file stream + MS_LOG(ERROR) << "Read data from fstream failed."; // Log an error message + return false; // Return false to indicate failure } - } - fs.close(); - return true; -} -bool FileIOUtils::IsFileOrDirExist(const std::string &path) { - if (path.empty()) { + // Close the file stream + fs.close(); + + // Return true to indicate successful read operation + return true; + +// Check if the provided path is empty +if (path.empty()) { + // If the path is empty, throw an exception with a descriptive error message MS_LOG(EXCEPTION) << "The path name is empty"; - } - - return access(path.c_str(), F_OK) == 0; } +// Return the result of checking if the file or directory specified by the given path exists +// The access function is used to check the accessibility of a file or directory +// The path.c_str() function converts the path string to a C-style string +// The F_OK argument is used to check if the file exists +// The == 0 comparison is used to check if the access function returns 0, indicating that the file exists +return access(path.c_str(), F_OK) == 0; + +// Definition of the function "CreateFile" in the "FileIOUtils" class void FileIOUtils::CreateFile(const std::string &file_path, mode_t mode) { + + // Check if the file or directory already exists if (IsFileOrDirExist(file_path)) { + // If it exists, return without doing anything return; } - - std::ofstream output_file(file_path); - output_file.close(); - ChangeFileMode(file_path, mode); + // If it doesn't exist, continue with creating the file } +// Create an output file stream object and open the file specified by the file_path +std::ofstream output_file(file_path); + +// Close the output file stream +output_file.close(); + +// Call the function ChangeFileMode to change the mode of the file specified by the file_path +ChangeFileMode(file_path, mode); + +// Definition of the function "CreateDir" in the "FileIOUtils" class void FileIOUtils::CreateDir(const std::string &dir_path, mode_t mode) { + + // Check if the directory already exists by calling the "IsFileOrDirExist" function if (IsFileOrDirExist(dir_path)) { + // If the directory already exists, return without doing anything return; } + // If the directory does not exist, continue with creating it +} +// Check if the operating system is Windows (either 32-bit or 64-bit) #if defined(_WIN32) || defined(_WIN64) + // Create a directory using the provided directory path int ret = mkdir(dir_path.c_str()); #else + // Create a directory using the provided directory path and mode int ret = mkdir(dir_path.c_str(), mode); + + // If the directory creation was successful if (ret == 0) { + // Change the file mode of the directory to the provided mode ChangeFileMode(dir_path, mode); } #endif - if (ret != 0) { - MS_LOG(EXCEPTION) << "Failed to create directory " << dir_path << ". Errno = " << errno; - } + +// If the directory creation was not successful +if (ret != 0) { + // Log an exception with the error message indicating the failure to create the directory + MS_LOG(EXCEPTION) << "Failed to create directory " << dir_path << ". Errno = " << errno; } +// A function to create a directory recursively void FileIOUtils::CreateDirRecursive(const std::string &dir_path, mode_t mode) { + + // Check if the directory path is empty if (dir_path.empty()) { MS_LOG(EXCEPTION) << "The directory path need to be create is empty"; } + + // Get the length of the directory path size_t dir_path_len = dir_path.length(); + + // Check if the directory path length exceeds the maximum length limit if (dir_path_len > PATH_MAX) { MS_LOG(EXCEPTION) << "Directory path is too long to exceed max length limit: " << PATH_MAX << ", the path: " << dir_path; } - char tmp_dir_path[PATH_MAX] = {0}; - for (size_t i = 0; i < dir_path_len; ++i) { - tmp_dir_path[i] = dir_path[i]; - if (tmp_dir_path[i] == '/' || dir_path == tmp_dir_path) { - if (access(tmp_dir_path, F_OK) == 0) { - continue; - } +// Declare a character array to store the temporary directory path, initialize it with all zeros +char tmp_dir_path[PATH_MAX] = {0}; +// Iterate through each character in the original directory path +for (size_t i = 0; i < dir_path_len; ++i) { + + // Copy the current character from the original directory path to the temporary directory path + tmp_dir_path[i] = dir_path[i]; + + // Check if the current character is a forward slash or if the original directory path is equal to the temporary directory path + if (tmp_dir_path[i] == '/' || dir_path == tmp_dir_path) { + + // Check if the temporary directory path exists + if (access(tmp_dir_path, F_OK) == 0) { + + // If the temporary directory path exists, skip to the next iteration of the loop + continue; + } + +// Check if the operating system is Windows #if defined(_WIN32) || defined(_WIN64) - int32_t ret = mkdir(tmp_dir_path); + // Create a directory using the provided path + int32_t ret = mkdir(tmp_dir_path); #else - int32_t ret = mkdir(tmp_dir_path, mode); - if (ret == 0) { + // Create a directory using the provided path and mode + int32_t ret = mkdir(tmp_dir_path, mode); + + // If the directory creation was successful + if (ret == 0) { + // Change the file mode of the directory to the provided mode ChangeFileMode(tmp_dir_path, mode); - } -#endif - if (ret != 0) { - MS_LOG(EXCEPTION) << "Failed to create directory recursion: " << dir_path << ". Errno = " << errno; - } } - } +#endif + +// If the directory creation was not successful +if (ret != 0) { + // Log an exception with the error message and the error number + MS_LOG(EXCEPTION) << "Failed to create directory recursion: " << dir_path << ". Errno = " << errno; +} +} } } // namespace storage } // namespace distributed -} // namespace mindspore +} // namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/persistent/storage/json_utils.cc b/mindspore/ccsrc/distributed/persistent/storage/json_utils.cc index 3ba4b8f2d13..95ec940ac7b 100644 --- a/mindspore/ccsrc/distributed/persistent/storage/json_utils.cc +++ b/mindspore/ccsrc/distributed/persistent/storage/json_utils.cc @@ -14,38 +14,88 @@ * limitations under the License. */ +// Include the header file for JSON utilities in the distributed/persistent/storage directory #include "distributed/persistent/storage/json_utils.h" + +// Include the header file for file I/O utilities in the distributed/persistent/storage directory #include "distributed/persistent/storage/file_io_utils.h" + +// Include the header file for common utilities in the include/common/utils directory #include "include/common/utils/utils.h" -namespace mindspore { -namespace distributed { -namespace storage { +// Start of the `Initialize` function definition + bool JsonUtils::Initialize() { + + // Check if the file or directory specified by `file_name_` exists if (!FileIOUtils::IsFileOrDirExist(file_name_)) { + + // If the file or directory does not exist, create a new file using `file_name_` FileIOUtils::CreateFile(file_name_); + + // Return true to indicate successful initialization return true; } + // End of the `if` statement + // If the file or directory already exists, return false to indicate initialization failure + // (assuming that initialization should only be done when the file or directory does not exist) + return false; +} +// End of the `Initialize` function definition + +// End of the `storage` namespace +} + +// End of the `distributed` namespace +} + +// End of the `mindspore` namespace + + // Create an input file stream object and open the file with the given file name std::ifstream json_file(file_name_); + try { + // Read the contents of the file into the json object json_file >> js_; + + // Close the file after reading json_file.close(); } catch (nlohmann::json::exception &e) { + // If an exception occurs during parsing, close the file and handle the exception json_file.close(); + + // Get the exception message as a string std::string illegal_exception = e.what(); + + // Log an error message indicating the failure to parse the json file MS_LOG(ERROR) << "Parse json file:" << file_name_ << " failed, the exception:" << illegal_exception; + + // Return false to indicate failure return false; } + + // Return true to indicate success return true; } +// Check if the given key exists in the JSON object bool JsonUtils::Exists(const std::string &key) const { + + // If the JSON object does not contain the key, return false if (!js_.contains(key)) { return false; } + + // If the JSON object contains the key, return true return true; } + +// End of the storage namespace } // namespace storage + +// End of the distributed namespace } // namespace distributed -} // namespace mindspore + +// End of the mindspore namespace +} // namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/persistent/storage/local_file.cc b/mindspore/ccsrc/distributed/persistent/storage/local_file.cc index 09b683531c8..91edc7d6412 100644 --- a/mindspore/ccsrc/distributed/persistent/storage/local_file.cc +++ b/mindspore/ccsrc/distributed/persistent/storage/local_file.cc @@ -14,264 +14,522 @@ * limitations under the License. */ -#include "distributed/persistent/storage/local_file.h" +// Include the header file "distributed/persistent/storage/local_file.h" which contains declarations for local file storage related functions and classes. +// Include the directory entry header for directory manipulation #include + +// Include the math header for mathematical functions #include + +// Include the algorithm header for various algorithms #include + +// Include the numeric header for numeric operations #include + +// Include the tuple header for tuple manipulation #include + +// Include the utility header for various utility functions #include +// Include the header file "utils/convert_utils_base.h" which contains utility functions for converting data types #include "utils/convert_utils_base.h" + +// Include the header file "utils/file_utils.h" which contains utility functions for file operations #include "utils/file_utils.h" + +// Include the header file "utils/log_adapter.h" which contains utility functions for logging #include "utils/log_adapter.h" + +// Include the header file "include/common/utils/utils.h" which contains utility functions for common operations #include "include/common/utils/utils.h" + +// Include the header file "distributed/persistent/storage/constants.h" which contains constants related to storage in a distributed system #include "distributed/persistent/storage/constants.h" +// Start of the "mindspore" namespace namespace mindspore { + +// Start of the "distributed" namespace, which is nested inside the "mindspore" namespace namespace distributed { + +// Start of the "storage" namespace, which is nested inside the "distributed" namespace namespace storage { + +// Definition of the Write function of the LocalFile class void LocalFile::Write(const InputData &input, const DirtyInfo &dirty_info) { + + // Create a vector of InputData objects and initialize it with a single element, which is the provided input std::vector inputs = {input}; + + // Call the overloaded Write function with the vector of inputs and the dirty_info parameter Write(inputs, dirty_info); } +// Implementation of the Write function in the LocalFile class + void LocalFile::Write(const std::vector &inputs, const DirtyInfo &dirty_info) { + + // Check if the inputs vector is empty if (inputs.empty()) { + + // If the inputs vector is empty, throw an exception with an error message MS_LOG(EXCEPTION) << "The inputs is empty"; } +} - // The block file has been created, only the blocks related to the dirty information need to be rewritten. - if (finish_create_block_files_) { +// Check if the block files have already been created +if (finish_create_block_files_) { + + // Create an empty vector to store the block indices std::vector block_indices; + + // Transform the dirty information into block indices using the TransformDirtyInfoToBlockIndices function TransformDirtyInfoToBlockIndices(dirty_info, &block_indices); + // Iterate over each element in the block_indices vector using a range-based for loop for (const auto &block_index : block_indices) { + + // Call the WriteOneBlockFile function with the block_index and inputs as arguments WriteOneBlockFile(IntToSize(block_index), inputs); } + + // Return from the current function return; } - // Create block files and write inputs_data to block files. - WriteBlockFiles(inputs); -} +// Call the function WriteBlockFiles and pass the inputs as a parameter to create block files and write the inputs_data to those block files +WriteBlockFiles(inputs); +// Define the function `TransformDirtyInfoToBlockIndices` belonging to the `LocalFile` class void LocalFile::TransformDirtyInfoToBlockIndices(const DirtyInfo &dirty_info, std::vector *block_indices) const { + + // Check if the `block_indices` pointer is null, and throw an exception if it is MS_EXCEPTION_IF_NULL(block_indices); + + // Check if the `block_meta_list_` is empty, and throw an exception if it is if (block_meta_list_.empty()) { MS_LOG(EXCEPTION) << "The block meta list is empty"; } +} - size_t block_index = 0; - bool block_index_alread_insert_vec = false; - auto block_meta_ptr = block_meta_list_.at(block_index); - MS_EXCEPTION_IF_NULL(block_meta_ptr); - int cur_lower_bound = block_meta_ptr->Get(kShardRangeLowerBound); - int cur_upper_bound = block_meta_ptr->Get(kShardRangeUpperBound); +// Declare and initialize a variable `block_index` of type `size_t` with a value of 0 +size_t block_index = 0; +// Declare and initialize a boolean variable `block_index_alread_insert_vec` with a value of false +bool block_index_alread_insert_vec = false; + +// Retrieve a pointer to the element at index `block_index` from the `block_meta_list_` container +auto block_meta_ptr = block_meta_list_.at(block_index); + +// Check if the retrieved pointer is null, and throw an exception if it is +MS_EXCEPTION_IF_NULL(block_meta_ptr); + +// Retrieve the value associated with the key `kShardRangeLowerBound` from the `block_meta_ptr` object and assign it to the variable `cur_lower_bound` +int cur_lower_bound = block_meta_ptr->Get(kShardRangeLowerBound); + +// Retrieve the value associated with the key `kShardRangeUpperBound` from the `block_meta_ptr` object and assign it to the variable `cur_upper_bound` +int cur_upper_bound = block_meta_ptr->Get(kShardRangeUpperBound); + +// Iterate over each element in the dirty_info vector using a range-based for loop for (const auto &dirty_value : dirty_info) { + + // Check if the current dirty_value is within the specified range if (dirty_value >= cur_lower_bound && dirty_value < cur_upper_bound) { + + // Check if the block_index has not already been inserted into the block_indices vector if (!block_index_alread_insert_vec) { + + // Set block_index_alread_insert_vec to true to indicate that the block_index has been inserted block_index_alread_insert_vec = true; + + // Add the block_index to the block_indices vector block_indices->push_back(block_index); } + + // Continue to the next iteration of the loop continue; } + // Loop until the dirty value is within the current lower and upper bounds while (!(dirty_value >= cur_lower_bound && dirty_value < cur_upper_bound)) { + + // Check if we have reached the end of the block meta list if (++block_index >= block_meta_list_.size()) { break; } + + // Get the block meta pointer at the current index block_meta_ptr = block_meta_list_[block_index]; + + // Throw an exception if the block meta pointer is null MS_EXCEPTION_IF_NULL(block_meta_ptr); + + // Update the current lower and upper bounds from the block meta cur_lower_bound = block_meta_ptr->Get(kShardRangeLowerBound); cur_upper_bound = block_meta_ptr->Get(kShardRangeUpperBound); } + // Check if the block_index is less than the size of the block_meta_list_ if (block_index < block_meta_list_.size()) { + + // If the condition is true, push the block_index into the block_indices vector block_indices->push_back(block_index); } } } +// Implementation of the WriteBlockFiles function in the LocalFile class + void LocalFile::WriteBlockFiles(const std::vector &inputs) { + + // Check if the inputs vector is empty if (inputs.empty()) { + + // If the inputs vector is empty, throw an exception with an error message MS_LOG(EXCEPTION) << "The inputs is empty"; } +} - const std::vector &shape = std::get<0>(inputs.front()); - size_t first_dim = 0; - if (shape.size() > 0) { +// Create a constant reference to the first element of the 'inputs' vector and assign it to the 'shape' variable +const std::vector &shape = std::get<0>(inputs.front()); + +// Declare and initialize a variable 'first_dim' of type 'size_t' with a value of 0 +size_t first_dim = 0; + +// Check if the 'shape' vector has any elements +if (shape.size() > 0) { + // Convert the first element of the 'shape' vector to 'size_t' and assign it to 'first_dim' first_dim = IntToSize(shape[0]); - } - if (first_dim == 0) { +} + +// Check if 'first_dim' is equal to 0 +if (first_dim == 0) { + // If 'first_dim' is 0, throw an exception with the error message "The dimension of input shape contain zero." MS_LOG(EXCEPTION) << "The dimension of input shape contain zero."; - } +} - size_t non_first_dims_size = std::get<2>(inputs.front()) / first_dim; - if (non_first_dims_size == 0) { +// Calculate the size of the non-first dimensions by dividing the size of the first input tensor by the size of the first dimension +size_t non_first_dims_size = std::get<2>(inputs.front()) / first_dim; + +// Check if the size of the non-first dimensions is zero +if (non_first_dims_size == 0) { + // If the size is zero, throw an exception with an error message MS_LOG(EXCEPTION) << "The size of input tensor is zero."; - } +} - size_t tensor_num = inputs.size(); - size_t slice_size = static_cast( +// Get the number of elements in the inputs vector +size_t tensor_num = inputs.size(); + +// Calculate the slice size by dividing the maximum block length by the number of tensors, +// and then dividing it by the size of the non-first dimensions +size_t slice_size = static_cast( std::floor(static_cast(static_cast(max_block_length_) / tensor_num) / non_first_dims_size)); - if (slice_size == 0) { + +// Check if the slice size is zero +if (slice_size == 0) { + // If the slice size is zero, throw an exception with an error message MS_LOG(EXCEPTION) << "The slice size in block is zero."; - } +} - size_t block_num = static_cast(std::ceil(static_cast(first_dim) / slice_size)); +// Calculate the number of blocks needed to divide the first dimension into slices of size slice_size +// Convert the first_dim to float and divide it by slice_size, then round up the result using std::ceil +// Finally, cast the result to size_t and assign it to the variable block_num +size_t block_num = static_cast(std::ceil(static_cast(first_dim) / slice_size)); + // Initialize the offset variable to 0 size_t offset = 0; + + // Iterate over each block index from 0 to block_num-1 for (size_t block_index = 0; block_index < block_num; ++block_index) { - // Create block meta. + + // Create the file name for the block meta using the block index std::string block_meta_file_name = file_path_ + "/" + kBlockMetaFilePrefix + std::to_string(block_index) + kJsonSuffix; + + // Create a shared pointer to a BlockMeta object with the block meta file name auto block_meta_ptr = std::make_shared(block_meta_file_name); + + // Check if the initialization of the block meta object failed if (!block_meta_ptr->Initialize()) { + + // If initialization failed, throw an exception with an error message MS_LOG(EXCEPTION) << "Initialize block meta failed, file name [" << block_meta_file_name << "]"; } + // Calculate the current lower bound of the shard range by multiplying the slice size with the block index size_t cur_lower_bound = slice_size * block_index; + + // Insert the current lower bound into the block metadata using the key kShardRangeLowerBound block_meta_ptr->Insert(kShardRangeLowerBound, cur_lower_bound); + + // Calculate the current upper bound of the shard range by taking the minimum of the sum of the current lower bound and the slice size, and the value of the first dimension size_t cur_upper_bound = std::min(cur_lower_bound + slice_size, first_dim); + + // Insert the current upper bound into the block metadata using the key kShardRangeUpperBound block_meta_ptr->Insert(kShardRangeUpperBound, cur_upper_bound); - size_t field_length = (cur_upper_bound - cur_lower_bound) * non_first_dims_size; - block_meta_ptr->Insert(kFieldsLength, field_length); - block_meta_ptr->Insert(kOffset, offset); - offset += field_length; - block_meta_list_.push_back(block_meta_ptr); +// Calculate the length of the field by subtracting the lower bound from the upper bound and multiplying it by the size of the non-first dimensions +size_t field_length = (cur_upper_bound - cur_lower_bound) * non_first_dims_size; - // Create block. +// Insert the field length into the block_meta_ptr using the key kFieldsLength +block_meta_ptr->Insert(kFieldsLength, field_length); + +// Insert the offset into the block_meta_ptr using the key kOffset +block_meta_ptr->Insert(kOffset, offset); + +// Increment the offset by the field length +offset += field_length; + +// Add the block_meta_ptr to the end of the block_meta_list_ +block_meta_list_.push_back(block_meta_ptr); + + // Create a shared pointer to a Block object using std::make_shared. + // Concatenate the file path, kBlockFilePrefix, and the block index to create the file path for the block. auto block_ptr = std::make_shared(file_path_ + "/" + kBlockFilePrefix + std::to_string(block_index)); + + // Set the block meta data for the block using the block_meta_ptr. block_ptr->set_block_meta(block_meta_ptr); + + // Add the block_ptr to the block_list_ vector. block_list_.push_back(block_ptr); } - finish_create_block_files_ = true; +// Set the variable "finish_create_block_files_" to true, indicating that the process of creating block files has been completed - // Write inputs_data to block files and Gen Sha256 seq. - for (size_t block_index = 0; block_index < block_num; ++block_index) { +// Write the input data to block files and generate SHA256 sequences + +// Iterate over each block index from 0 to block_num +for (size_t block_index = 0; block_index < block_num; ++block_index) { + + // Call the function WriteOneBlockFile to write the input data to a block file WriteOneBlockFile(block_index, inputs); - } } -void LocalFile::WriteOneBlockFile(size_t block_index, const std::vector &inputs) const { - const auto &block_meta_ptr = block_meta_list_.at(block_index); - MS_EXCEPTION_IF_NULL(block_meta_ptr); - size_t field_size = block_meta_ptr->Get(kFieldsLength); - size_t offset = block_meta_ptr->Get(kOffset); - std::vector> block_inputs_data; +// End of the for loop - for (size_t input_index = 0; input_index < inputs.size(); ++input_index) { +// This function is a member function of the `LocalFile` class and is used to write data to a file in blocks. +// It takes two parameters: `block_index` which specifies the index of the block to write, and `inputs` which is a vector of `InputData` objects. + +// Get a reference to the block metadata pointer from the `block_meta_list_` at the specified `block_index` +const auto &block_meta_ptr = block_meta_list_.at(block_index); + +// Check if the block metadata pointer is null, and throw an exception if it is +MS_EXCEPTION_IF_NULL(block_meta_ptr); + +// Get the field size from the block metadata using the `kFieldsLength` key +size_t field_size = block_meta_ptr->Get(kFieldsLength); + +// Get the offset from the block metadata using the `kOffset` key +size_t offset = block_meta_ptr->Get(kOffset); + +// Create an empty vector to store pairs of pointers to data and their sizes +std::vector> block_inputs_data; + +// Iterate over the inputs vector using a for loop, starting from index 0 and continuing until input_index is less than the size of inputs +for (size_t input_index = 0; input_index < inputs.size(); ++input_index) { + + // Get the second element of the tuple at the current input_index and cast it to a const char pointer const void *data_ptr = reinterpret_cast(std::get<1>(inputs.at(input_index))) + offset; + + // Set the data_size variable to the value of field_size size_t data_size = field_size; + + // Add a new element to the block_inputs_data vector using emplace_back, passing in data_ptr and data_size as arguments (void)block_inputs_data.emplace_back(data_ptr, data_size); - } - - const auto &block_ptr = block_list_.at(block_index); - MS_EXCEPTION_IF_NULL(block_ptr); - // Rewrite the current block file. - if (!FileIOUtils::Write(block_ptr->block_file_name(), block_inputs_data)) { - MS_LOG(EXCEPTION) << "Write to block file[" << block_ptr->block_file_name() << "] failed."; - } - - ChangeFileMode(block_ptr->block_file_name(), S_IRWXU | S_IRWXG | S_IRWXO); - - // Generate sha256 hash sequence. - block_ptr->GenSha256Seq(); } +// Get a reference to the block pointer at the specified index from the block list +const auto &block_ptr = block_list_.at(block_index); + +// Check if the block pointer is null, and throw an exception if it is +MS_EXCEPTION_IF_NULL(block_ptr); + +// Rewrite the current block file with the provided block inputs data +if (!FileIOUtils::Write(block_ptr->block_file_name(), block_inputs_data)) { + // If the write operation fails, log an exception with the block file name + MS_LOG(EXCEPTION) << "Write to block file[" << block_ptr->block_file_name() << "] failed."; +} + +// Change the file mode of the file specified by block_ptr->block_file_name() to allow read, write, and execute permissions for the owner, group, and others +ChangeFileMode(block_ptr->block_file_name(), S_IRWXU | S_IRWXG | S_IRWXO); + + // Call the GenSha256Seq() function on the block_ptr object to generate a sha256 hash sequence. + +// Define the Read function of the LocalFile class, which takes an OutputData object as a parameter void LocalFile::Read(const OutputData &output) { + + // Create a vector of OutputData objects and initialize it with the provided output object std::vector outputs = {output}; + + // Call the Read function of the LocalFile class, passing the vector of outputs as a parameter Read(outputs); } void LocalFile::Read(const std::vector &outputs) { - if (block_list_.empty() || block_meta_list_.empty()) { - // Load file list info of block files and block meta files in the current folder to block list and block meta list. - if (!LoadBlocksInfo()) { - MS_LOG(EXCEPTION) << "LoadBlocksInfo failed"; + // Check if the block list or block meta list is empty + if (block_list_.empty() || block_meta_list_.empty()) { + // If empty, load the file list info of block files and block meta files in the current folder to block list and block meta list + if (!LoadBlocksInfo()) { + // If loading fails, throw an exception with an error message + MS_LOG(EXCEPTION) << "LoadBlocksInfo failed"; + } } - } +} - // Read all block files. - for (size_t block_index = 0; block_index < block_list_.size(); ++block_index) { +// Read all block files. + +// Iterate over the block_list_ using a for loop, starting from 0 and going up to the size of block_list_ +for (size_t block_index = 0; block_index < block_list_.size(); ++block_index) { + + // Create an empty vector to store the output data for the current block std::vector> block_output_data; + + // Get a reference to the block_meta_ptr at the current block_index const auto &block_meta_ptr = block_meta_list_[block_index]; + + // Check if the block_meta_ptr is null, and throw an exception if it is MS_EXCEPTION_IF_NULL(block_meta_ptr); + + // Get the field_size from the block_meta_ptr using the kFieldsLength key size_t field_size = block_meta_ptr->Get(kFieldsLength); + + // Get the offset from the block_meta_ptr using the kOffset key size_t offset = block_meta_ptr->Get(kOffset); + // Iterate over the outputs vector using a for loop, starting from index 0 and continuing until output_index is less than the size of outputs for (size_t output_index = 0; output_index < outputs.size(); ++output_index) { + + // Get the pointer to the data by casting the first element of the output at index output_index to a char pointer and adding the offset void *data_ptr = reinterpret_cast(std::get<0>(outputs[output_index])) + offset; + + // Set the data size to field_size size_t data_size = field_size; + + // Create a pair of data_ptr and data_size and add it to the block_output_data vector using emplace_back (void)block_output_data.emplace_back(data_ptr, data_size); } + // Create a constant reference to the block pointer from the block list at the given block index const auto &block_ptr = block_list_[block_index]; + + // Check if the block pointer is null, and throw an exception if it is MS_EXCEPTION_IF_NULL(block_ptr); + + // Check if the SHA256 sequence of the block is valid if (!block_ptr->CheckSha256Seq()) { + + // Log an exception message with the name of the block file that failed the SHA256 check MS_LOG(EXCEPTION) << "CheckSha256 failed, file name [" << block_ptr->block_file_name() << "]"; } + // Check if the Read function from the FileIOUtils class returns false (indicating failure) if (!FileIOUtils::Read(block_ptr->block_file_name(), block_output_data)) { - MS_LOG(EXCEPTION) << "Read block file failed, file name [" << block_ptr->block_file_name() << "]"; - } - } -} + // If the Read function fails, log an exception message with the block file name + MS_LOG(EXCEPTION) << "Read block file failed, file name [" << block_ptr->block_file_name() << "]"; + } + + // Close the if statement + +// Close the main function + +// Close the namespace + +// Function to load blocks information from a local file bool LocalFile::LoadBlocksInfo() { + + // Open the directory specified by the file path DIR *dir = opendir(file_path_.c_str()); + + // Check if the directory could not be opened if (dir == nullptr) { + + // Log an error message indicating that the file path does not exist MS_LOG(ERROR) << "The file path [" << file_path_ << "] is not exist"; + + // Return false to indicate failure in loading blocks information return false; } + + // Create vectors to store the names of block files and block meta files std::vector block_file_name_list; std::vector block_meta_file_name_list; + + // Structure to represent a directory entry struct dirent *entry; // Get file names of all block file and block meta file in the current folder. while ((entry = readdir(dir)) != nullptr) { std::string file_name = entry->d_name; + + // If the length of the file name is less than or equal to JSON_SUFFIX_LENS, skip to the next iteration if (file_name.length() <= JSON_SUFFIX_LENS) { continue; } + // Concatenate the file path and file name to get the complete file path std::string real_storage_file_path = file_path_ + "/" + file_name; + + // Extract the suffix of the file name auto suffix = file_name.substr(file_name.length() - JSON_SUFFIX_LENS); + + // Check if the suffix is equal to the constant kJsonSuffix if (suffix == kJsonSuffix) { + // If the suffix is equal to kJsonSuffix, add the real storage file path to the block_meta_file_name_list block_meta_file_name_list.push_back(real_storage_file_path); } else { + // If the suffix is not equal to kJsonSuffix, add the real storage file path to the block_file_name_list block_file_name_list.push_back(real_storage_file_path); } } + + // Close the directory (void)closedir(dir); +// Check if the size of the block_file_name_list is not equal to the size of the block_meta_file_name_list if (block_file_name_list.size() != block_meta_file_name_list.size()) { + + // If the sizes are not equal, log an error message using MS_LOG(ERROR) MS_LOG(ERROR) << "The block file number[" << block_file_name_list.size() << "] is not equal to block meta file number[" << block_meta_file_name_list.size() << "]"; + + // Return false to indicate failure return false; } - sort(block_file_name_list.begin(), block_file_name_list.end()); - sort(block_meta_file_name_list.begin(), block_meta_file_name_list.end()); - for (size_t i = 0; i < block_file_name_list.size(); i++) { - auto block_meta_ptr = std::make_shared(block_meta_file_name_list[i]); - if (!block_meta_ptr->Initialize()) { - MS_LOG(ERROR) << "Initialize block meta failed, file name [" << block_meta_file_name_list[i] << "]"; - return false; - } - block_meta_list_.push_back(block_meta_ptr); +// Sort the block_file_name_list in ascending order +sort(block_file_name_list.begin(), block_file_name_list.end()); - auto block_ptr = std::make_shared(block_file_name_list[i]); - block_ptr->set_block_meta(block_meta_ptr); - block_list_.push_back(block_ptr); +// Sort the block_meta_file_name_list in ascending order +sort(block_meta_file_name_list.begin(), block_meta_file_name_list.end()); + +// Iterate through the block_file_name_list +for (size_t i = 0; i < block_file_name_list.size(); i++) { + + // Create a shared pointer to a BlockMeta object using the block_meta_file_name_list[i] + auto block_meta_ptr = std::make_shared(block_meta_file_name_list[i]); + + // Initialize the block_meta_ptr object + if (!block_meta_ptr->Initialize()) { + + // If initialization fails, print an error message with the corresponding file name and return false + MS_LOG(ERROR) << "Initialize block meta failed, file name [" << block_meta_file_name_list[i] << "]"; + return false; } - return true; + + // Add the block_meta_ptr object to the block_meta_list_ + block_meta_list_.push_back(block_meta_ptr); +} + + auto block_ptr = std::make_shared(block_file_name_list[i]); // Create a shared pointer to a Block object, passing the block_file_name_list[i] as a parameter + block_ptr->set_block_meta(block_meta_ptr); // Call the set_block_meta function of the Block object pointed to by block_ptr, passing block_meta_ptr as a parameter + block_list_.push_back(block_ptr); // Add the block_ptr to the end of the block_list_ vector + } + return true; // Return true to indicate that the operation was successful } } // namespace storage } // namespace distributed -} // namespace mindspore +} // namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/rpc/tcp/connection.cc b/mindspore/ccsrc/distributed/rpc/tcp/connection.cc index 187148bf961..8afb701461c 100644 --- a/mindspore/ccsrc/distributed/rpc/tcp/connection.cc +++ b/mindspore/ccsrc/distributed/rpc/tcp/connection.cc @@ -14,511 +14,893 @@ * limitations under the License. */ -#include "distributed/rpc/tcp/connection.h" +// Include the header file "distributed/rpc/tcp/connection.h" which contains the declaration of a class or functions related to TCP connections in a distributed RPC system. +// Include the memory header, which provides smart pointers and related utilities #include + +// Include the utility header, which provides various utility functions and classes #include +// Include the header file for TCP socket operations #include "distributed/rpc/tcp/tcp_socket_operation.h" + +// Include the header file for TCP connection pool #include "distributed/rpc/tcp/connection_pool.h" namespace mindspore { namespace distributed { namespace rpc { -// Handle socket events like read/write. + +// Function to handle socket events like read/write. void SocketEventHandler(int fd, uint32_t events, void *context) { + + // Cast the context pointer to a Connection pointer Connection *conn = reinterpret_cast(context); + // Check if the file descriptor (fd) is not equal to the connection's socket file descriptor (conn->socket_fd) if (fd != conn->socket_fd) { + // If they are not equal, log an error message indicating that the connection cannot be reused MS_LOG(ERROR) << "Failed to reuse connection, delete and close fd: " << fd << ", connfd: " << conn->socket_fd << ", event: " << events; + + // Delete the epoll event associated with the file descriptor (fd) from the receive event loop if (conn->recv_event_loop->DeleteEpollEvent(fd) != RPC_OK) { MS_LOG(ERROR) << "Failed to delete epoll event for fd: " << fd; } + + // Set the connection's state to "kDisconnecting" to indicate that it is in the process of disconnecting conn->state = ConnectionState::kDisconnecting; + + // Check if the connection has an event callback function assigned if (conn->event_callback != nullptr) { + // If it does, call the event callback function with the connection as the argument conn->event_callback(conn); } else { + // If it doesn't, log an error message indicating that no event callback function was found MS_LOG(ERROR) << "No event_callback found for fd: " << fd << ", events: " << events; } + + // Return to exit the function return; } + // Handle write event. + // Check if the write event is set in the events bitmask if (events & EPOLLOUT) { + // Update the epoll event for the file descriptor (fd) to include EPOLLIN, EPOLLHUP, and EPOLLERR (void)conn->recv_event_loop->UpdateEpollEvent(fd, EPOLLIN | EPOLLHUP | EPOLLERR); + + // Check if the connection has a write callback function assigned if (conn->write_callback != nullptr) { + // If it does, call the write callback function with the connection as the argument conn->write_callback(conn); } - } - // Handle read event. + // Check if the event is a read event if (events & EPOLLIN) { + // Check if the connection has a read callback function assigned if (conn->read_callback != nullptr) { + // Call the read callback function and pass the connection as an argument conn->read_callback(conn); } } - // Handle disconnect event. + + // Check if the connection is in the disconnecting state or if the received message type is not HTTP request or response if (conn->state == ConnectionState::kDisconnecting || (conn->recv_message_type != ParseType::kHttpReq && conn->recv_message_type != ParseType::kHttpRsp && (events & (uint32_t)(EPOLLHUP | EPOLLRDHUP | EPOLLERR)))) { + // Check if the received message type is TCP message if (conn->recv_message_type == ParseType::kTcpMsg) { + // Log the event details MS_LOG(INFO) << "Event value fd: " << fd << ", events: " << events << ", state: " << conn->state << ", errcode: " << conn->error_code << ", errno: " << errno << ", to: " << conn->destination.c_str() << ", type:" << conn->recv_message_type << ", remote: " << conn->is_remote; } + // Set the connection state to disconnecting conn->state = ConnectionState::kDisconnecting; + // Check if the connection has an event callback function assigned if (conn->event_callback != nullptr) { + // Call the event callback function and pass the connection as an argument conn->event_callback(conn); } else { - MS_LOG(ERROR) << "No event_callback found for fd: " << fd << ", events: " << events; + // If no event callback function is assigned, do nothing } - } -} +// Log an error message using the MS_LOG macro +MS_LOG(ERROR) << "No event_callback found for fd: " << fd << ", events: " << events; -// Handle new connect event. +// Close the if statement + +// Close the while loop + +// Close the main function + +// Function to handle a new connection event void NewConnectEventHandler(int fd, uint32_t events, void *context) { int retval = 0; + + // Cast the context pointer to a Connection object pointer Connection *conn = reinterpret_cast(context); + + // Call the NewConnEventHandler function of the socket_operation object associated with the connection conn->socket_operation->NewConnEventHandler(context); - - if (conn->state == ConnectionState::kDisconnecting) { - conn->Disconnect(fd); - return; - } else if (conn->state != ConnectionState::kConnected) { - // The handshake is not complete - return; - } - - retval = conn->recv_event_loop->DeleteEpollEvent(fd); - if (retval) { - MS_LOG(ERROR) << "Failed to remove epoll remove connect handler for fd: " << fd; - return; - } - - retval = conn->recv_event_loop->SetEventHandler(conn->socket_fd, EPOLLIN | EPOLLHUP | EPOLLRDHUP | EPOLLERR, - SocketEventHandler, reinterpret_cast(conn)); - if (retval != RPC_OK) { - MS_LOG(ERROR) << "Failed to add socket event handler for fd: " << fd << ", events: " << events; - conn->Disconnect(fd); - return; - } - - conn->write_callback(conn); - SocketEventHandler(fd, events, context); - return; } + // Check if the connection state is set to "Disconnecting" + if (conn->state == ConnectionState::kDisconnecting) { + // If so, call the Disconnect function on the connection object and pass the file descriptor + conn->Disconnect(fd); + // Return to exit the function + return; + } + // If the connection state is not set to "Connected" + else if (conn->state != ConnectionState::kConnected) { + // The handshake is not complete, so return without doing anything + return; + } + + // Call the DeleteEpollEvent function on the recv_event_loop object, passing in the fd parameter and store the return value in retval + retval = conn->recv_event_loop->DeleteEpollEvent(fd); + + // Check if retval is non-zero (indicating an error occurred) + if (retval) { + // Print an error message using the MS_LOG macro, indicating the failure to remove the epoll remove connect handler for the given fd + MS_LOG(ERROR) << "Failed to remove epoll remove connect handler for fd: " << fd; + + // Return from the function, as there is nothing more to be done + return; + } + + // Set the event handler for the connection's socket file descriptor (fd) using the recv_event_loop + // The events to be monitored are EPOLLIN (data available to read), EPOLLHUP (hang up), EPOLLRDHUP (peer closed connection), and EPOLLERR (error) + // The event handler function is SocketEventHandler, and the argument to be passed is a pointer to the connection object (conn) + retval = conn->recv_event_loop->SetEventHandler(conn->socket_fd, EPOLLIN | EPOLLHUP | EPOLLRDHUP | EPOLLERR, + SocketEventHandler, reinterpret_cast(conn)); + + // If setting the event handler fails, log an error message and disconnect the connection + if (retval != RPC_OK) { + MS_LOG(ERROR) << "Failed to add socket event handler for fd: " << fd << ", events: " << events; + + // Disconnect the connection associated with the file descriptor (fd) + conn->Disconnect(fd); + + // Return from the function + return; + } + +// Call the write_callback function on the conn object +conn->write_callback(conn); + +// Call the SocketEventHandler function with the provided parameters +SocketEventHandler(fd, events, context); + +// Return from the current function and continue execution from the calling function +return; + +// Constructor for the Connection class Connection::Connection() - : socket_fd(-1), - deleted(false), - is_remote(false), - type(kTcp), - socket_operation(nullptr), - state(kInit), - send_event_loop(nullptr), - recv_event_loop(nullptr), - send_metrics(nullptr), - send_message(nullptr), - recv_message(nullptr), - recv_state(kMsgHeader), - total_recv_len(0), - total_send_len(0), - recv_len(0), - event_callback(nullptr), - succ_callback(nullptr), - write_callback(nullptr), - read_callback(nullptr), + : socket_fd(-1), // Initialize socket file descriptor to -1 + deleted(false), // Initialize deleted flag to false + is_remote(false), // Initialize is_remote flag to false + type(kTcp), // Initialize type to kTcp + socket_operation(nullptr), // Initialize socket_operation pointer to nullptr + state(kInit), // Initialize state to kInit + send_event_loop(nullptr), // Initialize send_event_loop pointer to nullptr + recv_event_loop(nullptr), // Initialize recv_event_loop pointer to nullptr + send_metrics(nullptr), // Initialize send_metrics pointer to nullptr + send_message(nullptr), // Initialize send_message pointer to nullptr + recv_message(nullptr), // Initialize recv_message pointer to nullptr + recv_state(kMsgHeader), // Initialize recv_state to kMsgHeader + total_recv_len(0), // Initialize total_recv_len to 0 + total_send_len(0), // Initialize total_send_len to 0 + recv_len(0), // Initialize recv_len to 0 + event_callback(nullptr), // Initialize event_callback pointer to nullptr + succ_callback(nullptr), // Initialize succ_callback pointer to nullptr + write_callback(nullptr), // Initialize write_callback pointer to nullptr + read_callback(nullptr), // Initialize read_callback pointer to nullptr + // Note: The remaining members of the Connection class are not initialized here + // They may be initialized elsewhere in the code output_buffer_size(0), error_code(0) { // Initialize the recv kernel message structure. - recv_kernel_msg.msg_control = nullptr; - recv_kernel_msg.msg_controllen = 0; - recv_kernel_msg.msg_flags = 0; - recv_kernel_msg.msg_name = nullptr; - recv_kernel_msg.msg_namelen = 0; - recv_kernel_msg.msg_iov = recv_io_vec; - recv_kernel_msg.msg_iovlen = RECV_MSG_IO_VEC_LEN; + recv_kernel_msg.msg_control = nullptr; // Set the control message pointer to nullptr + recv_kernel_msg.msg_controllen = 0; // Set the control message length to 0 + recv_kernel_msg.msg_flags = 0; // Set the message flags to 0 + recv_kernel_msg.msg_name = nullptr; // Set the message name pointer to nullptr + recv_kernel_msg.msg_namelen = 0; // Set the message name length to 0 + recv_kernel_msg.msg_iov = recv_io_vec; // Set the message IO vector to recv_io_vec + recv_kernel_msg.msg_iovlen = RECV_MSG_IO_VEC_LEN; // Set the message IO vector length to RECV_MSG_IO_VEC_LEN - // Initialize the send message header. - send_metrics = new SendMetrics(); - for (unsigned int i = 0; i < BUSMAGIC_LEN; i++) { +// Initialize the send message header. +send_metrics = new SendMetrics(); + +// Loop through each element of the send_msg_header.magic array +for (unsigned int i = 0; i < BUSMAGIC_LEN; i++) { + // If the current index is less than the length of RPC_MAGICID - 1 if (i < sizeof(RPC_MAGICID) - 1) { - send_msg_header.magic[i] = RPC_MAGICID[i]; + // Copy the corresponding character from RPC_MAGICID to send_msg_header.magic + send_msg_header.magic[i] = RPC_MAGICID[i]; } else { - send_msg_header.magic[i] = '\0'; + // Otherwise, set the current element of send_msg_header.magic to null character '\0' + send_msg_header.magic[i] = '\0'; } - } - - // Initialize the send kernel message structure. - send_kernel_msg.msg_control = nullptr; - send_kernel_msg.msg_controllen = 0; - send_kernel_msg.msg_flags = 0; - send_kernel_msg.msg_name = nullptr; - send_kernel_msg.msg_namelen = 0; - send_kernel_msg.msg_iov = send_io_vec; - send_kernel_msg.msg_iovlen = SEND_MSG_IO_VEC_LEN; } + // Initialize the send kernel message structure. + send_kernel_msg.msg_control = nullptr; // Set the control message pointer to nullptr + send_kernel_msg.msg_controllen = 0; // Set the control message length to 0 + send_kernel_msg.msg_flags = 0; // Set the message flags to 0 + send_kernel_msg.msg_name = nullptr; // Set the message name pointer to nullptr + send_kernel_msg.msg_namelen = 0; // Set the message name length to 0 + send_kernel_msg.msg_iov = send_io_vec; // Set the message IO vector to the specified send_io_vec + send_kernel_msg.msg_iovlen = SEND_MSG_IO_VEC_LEN; // Set the message IO vector length to SEND_MSG_IO_VEC_LEN + +// Initialize the Connection object int Connection::Initialize() { + + // Call the InitSocketOperation function to initialize socket operations InitSocketOperation(); + + // Call the AddConnnectEventHandler function to add a connection event handler return AddConnnectEventHandler(); } +// Initialize the socket operation for the Connection class void Connection::InitSocketOperation() { + + // Check if the socket operation is already initialized if (socket_operation != nullptr) { - return; + return; // If it is already initialized, return without doing anything } + + // If the socket operation is not initialized, allocate memory for a new TCPSocketOperation object socket_operation = new (std::nothrow) TCPSocketOperation(); + + // Check if the memory allocation was successful MS_EXCEPTION_IF_NULL(socket_operation); } -bool Connection::ReconnectSourceSocket(int fd, uint32_t events, int *soError, uint32_t error) { - socklen_t len = sizeof(*soError); +// Reconnect the source socket with the given file descriptor, events, socket error pointer, and error code +// Create a variable 'len' of type socklen_t and initialize it with the size of the 'soError' pointer +socklen_t len = sizeof(*soError); + + // Call the DeleteEpollEvent function of the recv_event_loop object and store the return value in the retval variable int retval = recv_event_loop->DeleteEpollEvent(fd); + + // Check if the retval is non-zero, indicating an error occurred during the deletion of the event if (retval) { + // Print an error message using the MS_LOG macro, indicating the failure to delete the event for the given fd and events MS_LOG(ERROR) << "Failed to delete event for fd: " << fd << ", event: " << events; + + // Return false to indicate the failure to delete the event return false; } - retval = getsockopt(fd, SOL_SOCKET, SO_ERROR, soError, &len); - if (retval) { - *soError = errno; + retval = getsockopt(fd, SOL_SOCKET, SO_ERROR, soError, &len); // Get the socket option for error status + if (retval) { // If there was an error getting the socket option + *soError = errno; // Set the error number to the value of errno } - if (*soError || error) { - return false; + if (*soError || error) { // If there is an error or an existing error + return false; // Return false to indicate failure } - retval = recv_event_loop->SetEventHandler(socket_fd, EPOLLIN | EPOLLHUP | EPOLLRDHUP | EPOLLERR, SocketEventHandler, - reinterpret_cast(this)); - if (retval != RPC_OK) { - MS_LOG(ERROR) << "Failed to add socket event handler for fd: " << fd << ", events: " << events; - return false; + retval = recv_event_loop->SetEventHandler(socket_fd, EPOLLIN | EPOLLHUP | EPOLLRDHUP | EPOLLERR, SocketEventHandler, // Set the event handler for the socket + reinterpret_cast(this)); // Pass the current object as the user data + if (retval != RPC_OK) { // If setting the event handler failed + MS_LOG(ERROR) << "Failed to add socket event handler for fd: " << fd << ", events: " << events; // Log an error message + return false; // Return false to indicate failure } - return true; -} + return true; // Return true to indicate success + +// Definition of the Disconnect function in the Connection class void Connection::Disconnect(int fd) { + + // Check if LOG_CHECK_EVERY_N() condition is true if (LOG_CHECK_EVERY_N()) { + + // Log an informational message using MS_LOG macro, displaying various information about the connection failure MS_LOG(INFO) << "New connection fail fd: " << fd << ", state: " << state << ", errno: " << errno << ", to: " << destination.c_str() << ", type: " << recv_message_type; } + + // Set the state of the connection to kDisconnecting state = ConnectionState::kDisconnecting; + + // Call the event_callback function, passing the current object as an argument event_callback(this); + + // Return from the function return; } -void Connection::Close() { - if (recv_event_loop != nullptr) { - if (recv_event_loop->DeleteEpollEvent(socket_fd) == RPC_ERROR) { - MS_LOG(ERROR) << "Failed to delete epoll event " << socket_fd; - } - } - if (!destination.empty()) { - if (recv_message != nullptr) { - delete recv_message; - recv_message = nullptr; - } - } +// Close the connection +// Check if the receive event loop is not null +if (recv_event_loop != nullptr) { + // Delete the epoll event associated with the socket file descriptor + if (recv_event_loop->DeleteEpollEvent(socket_fd) == RPC_ERROR) { + // If deleting the epoll event fails, log an error message + MS_LOG(ERROR) << "Failed to delete epoll event " << socket_fd; + } +} + +// Check if the destination is not empty +if (!destination.empty()) { + // Check if the receive message is not null + if (recv_message != nullptr) { + // Delete the receive message object and set it to null + delete recv_message; + recv_message = nullptr; + } +} + + // Check if the total_send_len is not equal to 0 and send_message is not a null pointer if (total_send_len != 0 && send_message != nullptr) { + + // Delete the memory allocated for send_message delete send_message; + + // Set send_message to a null pointer to avoid dangling pointer send_message = nullptr; } - MessageBase *tmpMsg = nullptr; - while (!send_message_queue.empty()) { - tmpMsg = send_message_queue.front(); - send_message_queue.pop(); - delete tmpMsg; - tmpMsg = nullptr; - } +// Declare a pointer variable `tmpMsg` of type `MessageBase` and initialize it to nullptr +MessageBase *tmpMsg = nullptr; +// Loop until the `send_message_queue` is empty +while (!send_message_queue.empty()) { + + // Get the front element of the `send_message_queue` and assign it to `tmpMsg` + tmpMsg = send_message_queue.front(); + + // Remove the front element from the `send_message_queue` + send_message_queue.pop(); + + // Delete the object pointed to by `tmpMsg` to free up memory + delete tmpMsg; + + // Set `tmpMsg` to nullptr to avoid accessing deleted memory + tmpMsg = nullptr; +} + + // Check if the socket_operation pointer is not null if (socket_operation != nullptr) { + + // Call the Close function on the socket_operation object, passing 'this' as the argument socket_operation->Close(this); + + // Delete the socket_operation object from memory delete socket_operation; + + // Set the socket_operation pointer to null socket_operation = nullptr; } + // Check if the pointer send_metrics is not null if (send_metrics != nullptr) { + + // Delete the object pointed to by send_metrics delete send_metrics; + + // Set the pointer send_metrics to null send_metrics = nullptr; } } +// ReceiveMessage function of the Connection class + int Connection::ReceiveMessage() { + + // Parse the received message and store the result in the 'ok' variable bool ok = ParseMessage(); - // If no message parsed, wait for next read + + // If no message was successfully parsed if (!ok) { + + // If the connection state is 'kDisconnecting', return -1 to indicate disconnection if (state == ConnectionState::kDisconnecting) { return -1; } + + // If the connection state is not 'kDisconnecting', return 0 to indicate no message parsed return 0; } +} - // Call msg handler if set + // Check if a message handler is set if (message_handler) { + // If a message handler is set, call it with the received message as an argument message_handler(recv_message); } else { + // If no message handler is set, log an informational message MS_LOG(INFO) << "Message handler was not found"; } + + // Return 1 to indicate that the function has completed return 1; } +// A member function of the Connection class that checks the message type + void Connection::CheckMessageType() { + + // If the received message type is not unknown, then there is no need to proceed further if (recv_message_type != ParseType::kUnknown) { return; } +} - std::string magic_id = ""; - magic_id.resize(sizeof(RPC_MAGICID) - 1); - char *buf = const_cast(magic_id.data()); +// Declare a string variable named magic_id and initialize it with an empty string +std::string magic_id = ""; +// Resize the magic_id string to have a size equal to the size of RPC_MAGICID minus 1 +// This is done to ensure that the string can hold the same number of characters as RPC_MAGICID +magic_id.resize(sizeof(RPC_MAGICID) - 1); + +// Get a pointer to the underlying character buffer of the magic_id string +// We use const_cast to remove the const qualifier from the data() function's return value +// This is necessary because the data() function returns a const char* and we need a non-const char* to assign to buf +char *buf = const_cast(magic_id.data()); + + // Receive data from the socket and store the number of bytes received in 'size' ssize_t size = socket_operation->ReceivePeek(this, buf, sizeof(RPC_MAGICID) - 1); + + // Check if the received size is less than the expected size of the magic ID if (size < static_cast(sizeof(RPC_MAGICID) - 1)) { + // If the received size is 0, it means the connection is disconnected if (size == 0) { + // Log the disconnection information MS_LOG(INFO) << "Set connection disconnecting for fd: " << socket_fd << ", size: " << size << ", magic size: " << static_cast(sizeof(RPC_MAGICID) - 1) << ", errno: " << errno; + // Set the connection state to 'kDisconnecting' state = ConnectionState::kDisconnecting; } + // Return without further processing if the received size is less than expected return; } + + // Check if the received magic ID matches the expected magic ID if (strncmp(RPC_MAGICID, magic_id.c_str(), sizeof(RPC_MAGICID) - 1) == 0) { + // If the magic ID matches, set the receive state to 'kMsgHeader' and the receive message type to 'kTcpMsg' recv_state = State::kMsgHeader; recv_message_type = ParseType::kTcpMsg; } + + // Return from the function return; + +// Function to generate an HTTP message based on a given MessageBase object +std::string Connection::GenerateHttpMessage(MessageBase *msg) { + + // Define static strings for various parts of the HTTP message + static const std::string postLineBegin = std::string() + "POST /"; // The beginning of the POST line + static const std::string postLineEnd = std::string() + " HTTP/1.1\r\n"; // The end of the POST line + static const std::string userAgentLineBegin = std::string() + "User-Agent: libprocess/"; // The beginning of the User-Agent line + static const std::string fromLineBegin = std::string() + "Libprocess-From: "; // The beginning of the Libprocess-From line + static const std::string connectLine = std::string() + "Connection: Keep-Alive\r\n"; // The Connection line + static const std::string hostLine = std::string() + "Host: \r\n"; // The Host line + static const std::string chunkedBeginLine = std::string() + "Transfer-Encoding: chunked\r\n\r\n"; // The beginning of the chunked encoding line + static const std::string chunkedEndLine = std::string() + "\r\n" + "0\r\n" + "\r\n"; // The end of the chunked encoding line + static const std::string commonEndLine = std::string() + "\r\n"; // A common end line used in multiple places + + // Rest of the code goes here... } -std::string Connection::GenerateHttpMessage(MessageBase *msg) { - static const std::string postLineBegin = std::string() + "POST /"; - static const std::string postLineEnd = std::string() + " HTTP/1.1\r\n"; - static const std::string userAgentLineBegin = std::string() + "User-Agent: libprocess/"; - static const std::string fromLineBegin = std::string() + "Libprocess-From: "; - static const std::string connectLine = std::string() + "Connection: Keep-Alive\r\n"; - static const std::string hostLine = std::string() + "Host: \r\n"; - static const std::string chunkedBeginLine = std::string() + "Transfer-Encoding: chunked\r\n\r\n"; - static const std::string chunkedEndLine = std::string() + "\r\n" + "0\r\n" + "\r\n"; - static const std::string commonEndLine = std::string() + "\r\n"; +// Declare a string variable named postLine +std::string postLine; - std::string postLine; - if (msg->To().Name() != "") { +// Check if the name of the recipient of the message is not empty +if (msg->To().Name() != "") { + // If the recipient's name is not empty, concatenate the postLineBegin, recipient's name, message name, and postLineEnd postLine = postLineBegin + msg->To().Name() + "/" + msg->Name() + postLineEnd; - } else { +} else { + // If the recipient's name is empty, concatenate the postLineBegin, message name, and postLineEnd postLine = postLineBegin + msg->Name() + postLineEnd; - } +} - std::string userAgentLine = userAgentLineBegin + msg->From().Name() + "@" + advertise_addr_ + commonEndLine; - std::string fromLine = fromLineBegin + msg->From().Name() + "@" + advertise_addr_ + commonEndLine; +// Concatenate the string variables `userAgentLineBegin`, `msg->From().Name()`, `advertise_addr_`, and `commonEndLine` to form the `userAgentLine` string +std::string userAgentLine = userAgentLineBegin + msg->From().Name() + "@" + advertise_addr_ + commonEndLine; - if (msg->Body().size() > 0) { +// Concatenate the string variables `fromLineBegin`, `msg->From().Name()`, `advertise_addr_`, and `commonEndLine` to form the `fromLine` string +std::string fromLine = fromLineBegin + msg->From().Name() + "@" + advertise_addr_ + commonEndLine; + +// Check if the size of the body in the message is greater than 0 +if (msg->Body().size() > 0) { + + // Create a string stream to store the body line std::ostringstream bodyLine; + + // Convert the size of the body to hexadecimal and append it to the body line bodyLine << std::hex << msg->Body().size() << "\r\n"; + + // Write the body data to the body line (void)bodyLine.write(msg->Body().data(), msg->Body().size()); + + // Return the concatenation of various lines including the body line return postLine + userAgentLine + fromLine + connectLine + hostLine + chunkedBeginLine + bodyLine.str() + chunkedEndLine; - } - return postLine + userAgentLine + fromLine + connectLine + hostLine + commonEndLine; } -void Connection::FillSendMessage(MessageBase *msg, const std::string &advertiseUrl, bool isHttpKmsg) { - if (msg->type == MessageBase::Type::KMSG) { - int index = 0; - if (!isHttpKmsg) { - send_to = msg->to; - send_from = msg->from; - FillMessageHeader(*msg, &send_msg_header); +// If the size of the body is 0, return the concatenation of various lines without the body line +return postLine + userAgentLine + fromLine + connectLine + hostLine + commonEndLine; +// Fill the send_to and send_from variables with the values from the message object +send_to = msg->to; +send_from = msg->from; + +// Fill the send_msg_header with the message header information by calling the FillMessageHeader function +FillMessageHeader(*msg, &send_msg_header); + + // Set the base address of the first iovec element to the address of send_msg_header send_io_vec[index].iov_base = &send_msg_header; + + // Set the length of the first iovec element to the size of send_msg_header send_io_vec[index].iov_len = sizeof(send_msg_header); + + // Increment the index to move to the next iovec element ++index; + + // Set the base address of the next iovec element to the address of msg->name send_io_vec[index].iov_base = const_cast(msg->name.data()); + + // Set the length of the next iovec element to the size of msg->name send_io_vec[index].iov_len = msg->name.size(); + + // Increment the index to move to the next iovec element ++index; + + // Set the base address of the next iovec element to the address of send_to send_io_vec[index].iov_base = const_cast(send_to.data()); + + // Set the length of the next iovec element to the size of send_to send_io_vec[index].iov_len = send_to.size(); + + // Increment the index to move to the next iovec element ++index; + + // Set the base address of the next iovec element to the address of send_from send_io_vec[index].iov_base = const_cast(send_from.data()); + + // Set the length of the next iovec element to the size of send_from send_io_vec[index].iov_len = send_from.size(); + + // Increment the index to move to the next iovec element ++index; + + // Set the base address of the next iovec element to the address of msg->body send_io_vec[index].iov_base = const_cast(msg->body.data()); + + // Set the length of the next iovec element to the size of msg->body send_io_vec[index].iov_len = msg->body.size(); + + // Increment the index to move to the next iovec element ++index; + + // Set the msg_iov field of send_kernel_msg to the address of send_io_vec send_kernel_msg.msg_iov = send_io_vec; + + // Set the msg_iovlen field of send_kernel_msg to the current value of index send_kernel_msg.msg_iovlen = index; - total_send_len = - UlongToUint(sizeof(send_msg_header)) + msg->name.size() + send_to.size() + send_from.size() + msg->body.size(); + + // Calculate the total length of the message to be sent + total_send_len = UlongToUint(sizeof(send_msg_header)) + msg->name.size() + send_to.size() + send_from.size() + msg->body.size(); + + // Set the send_message pointer to the address of msg send_message = msg; - // update metrics - send_metrics->UpdateMax(msg->body.size()); - send_metrics->last_send_msg_name = msg->name; - return; - } else { - if (advertise_addr_.empty()) { +// Update the metrics by calling the UpdateMax function of the send_metrics object, passing the size of the message body as the argument +send_metrics->UpdateMax(msg->body.size()); + +// Set the last_send_msg_name attribute of the send_metrics object to the name of the current message +send_metrics->last_send_msg_name = msg->name; + +// Return from the function +return; + +// If the previous condition is not true, execute the following code block +else { + // Check if the advertise_addr_ variable is empty + if (advertise_addr_.empty()) { + // Find the position of the URL_PROTOCOL_IP_SEPARATOR in the advertiseUrl string size_t idx = advertiseUrl.find(URL_PROTOCOL_IP_SEPARATOR); + + // If the URL_PROTOCOL_IP_SEPARATOR is not found in the string if (idx == std::string::npos) { - advertise_addr_ = advertiseUrl; - } else { - advertise_addr_ = advertiseUrl.substr(idx + sizeof(URL_PROTOCOL_IP_SEPARATOR) - 1); + // Set the advertise_addr_ variable to the entire advertiseUrl string + advertise_addr_ = advertiseUrl; + } + // If the URL_PROTOCOL_IP_SEPARATOR is found in the string + else { + // Set the advertise_addr_ variable to a substring of the advertiseUrl string starting from the position after the URL_PROTOCOL_IP_SEPARATOR + advertise_addr_ = advertiseUrl.substr(idx + sizeof(URL_PROTOCOL_IP_SEPARATOR) - 1); } - } - msg->body = GenerateHttpMessage(msg); } - send_io_vec[index].iov_base = const_cast(msg->body.data()); - send_io_vec[index].iov_len = msg->body.size(); - ++index; - send_kernel_msg.msg_iov = send_io_vec; - send_kernel_msg.msg_iovlen = index; - total_send_len = UlongToUint(msg->body.size()); - send_message = msg; + // Set the body attribute of the msg object to the result of calling the GenerateHttpMessage function, passing the msg object as the argument + msg->body = GenerateHttpMessage(msg); +} - // update metrics +// Set the base address of the IO vector at the given index to the data pointer of the message body +send_io_vec[index].iov_base = const_cast(msg->body.data()); + +// Set the length of the IO vector at the given index to the size of the message body +send_io_vec[index].iov_len = msg->body.size(); + +// Increment the index to prepare for the next IO vector element +++index; + +// Set the IO vector of the send kernel message to the send IO vector +send_kernel_msg.msg_iov = send_io_vec; + +// Set the IO vector length of the send kernel message to the current index value +send_kernel_msg.msg_iovlen = index; + +// Convert the size of the message body to an unsigned integer and assign it to the total send length variable +total_send_len = UlongToUint(msg->body.size()); + +// Assign the message to be sent to the send message variable +send_message = msg; + + // Update the maximum size of the message body in the send_metrics object send_metrics->UpdateMax(msg->body.size()); + + // Update the last_send_msg_name in the send_metrics object with the name of the current message send_metrics->last_send_msg_name = msg->name; } } -void Connection::FillRecvMessage() { - size_t recvNameLen = static_cast(recv_msg_header.name_len); - size_t recvToLen = static_cast(recv_msg_header.to_len); - size_t recvFromLen = static_cast(recv_msg_header.from_len); - size_t recvBodyLen = static_cast(recv_msg_header.body_len); - if (recvNameLen > MAX_KMSG_NAME_LEN || recvToLen > MAX_KMSG_TO_LEN || recvFromLen > MAX_KMSG_FROM_LEN || - recvBodyLen > MAX_KMSG_BODY_LEN) { - MS_LOG(ERROR) << "Drop invalid tcp data."; - state = ConnectionState::kDisconnecting; - return; - } +// Convert the received message header fields to size_t type +size_t recvNameLen = static_cast(recv_msg_header.name_len); +size_t recvToLen = static_cast(recv_msg_header.to_len); +size_t recvFromLen = static_cast(recv_msg_header.from_len); +size_t recvBodyLen = static_cast(recv_msg_header.body_len); - int i = 0; - MessageBase *msg = new (std::nothrow) MessageBase(); - MS_EXCEPTION_IF_NULL(msg); - - msg->name.resize(recvNameLen); - recv_to.resize(recvToLen); - recv_from.resize(recvFromLen); - msg->body.resize(recvBodyLen); - - recv_io_vec[i].iov_base = const_cast(msg->name.data()); - recv_io_vec[i].iov_len = msg->name.size(); - ++i; - recv_io_vec[i].iov_base = const_cast(recv_to.data()); - recv_io_vec[i].iov_len = recv_to.size(); - ++i; - recv_io_vec[i].iov_base = const_cast(recv_from.data()); - recv_io_vec[i].iov_len = recv_from.size(); - ++i; - recv_io_vec[i].iov_base = const_cast(msg->body.data()); - recv_io_vec[i].iov_len = msg->body.size(); - ++i; - - recv_kernel_msg.msg_iov = recv_io_vec; - recv_kernel_msg.msg_iovlen = IntToSize(i); - total_recv_len = msg->name.size() + recv_to.size() + recv_from.size() + msg->body.size(); - recv_message = msg; +// Check if any of the received message lengths exceed the maximum allowed lengths +if (recvNameLen > MAX_KMSG_NAME_LEN || recvToLen > MAX_KMSG_TO_LEN || recvFromLen > MAX_KMSG_FROM_LEN || + recvBodyLen > MAX_KMSG_BODY_LEN) { + // If any of the lengths exceed the maximum, log an error message and set the connection state to disconnecting + MS_LOG(ERROR) << "Drop invalid tcp data."; + state = ConnectionState::kDisconnecting; + return; } +// Declare and initialize an integer variable named "i" with the value 0 +int i = 0; + +// Declare a pointer variable named "msg" of type MessageBase and allocate memory for a new MessageBase object using the "new" operator +// The "std::nothrow" argument is used to prevent the "new" operator from throwing an exception if memory allocation fails +MessageBase *msg = new (std::nothrow) MessageBase(); + +// Check if the "msg" pointer is null (indicating that memory allocation failed) +// If the pointer is null, an exception is thrown using the "MS_EXCEPTION_IF_NULL" macro +// This macro is likely defined elsewhere in the codebase and handles the exception +MS_EXCEPTION_IF_NULL(msg); + +// Resize the 'name' member of the 'msg' object to 'recvNameLen' size +msg->name.resize(recvNameLen); + +// Resize the 'recv_to' vector to 'recvToLen' size +recv_to.resize(recvToLen); + +// Resize the 'recv_from' vector to 'recvFromLen' size +recv_from.resize(recvFromLen); + +// Resize the 'body' member of the 'msg' object to 'recvBodyLen' size +msg->body.resize(recvBodyLen); + +// Set the base address of the i-th element in the recv_io_vec array to the data pointer of the 'name' string in the 'msg' object + recv_io_vec[i].iov_base = const_cast(msg->name.data()); + + // Set the length of the i-th element in the recv_io_vec array to the size of the 'name' string in the 'msg' object + recv_io_vec[i].iov_len = msg->name.size(); + + // Increment the value of 'i' by 1 + ++i; + + // Set the base address of the i-th element in the recv_io_vec array to the data pointer of the 'recv_to' string + recv_io_vec[i].iov_base = const_cast(recv_to.data()); + + // Set the length of the i-th element in the recv_io_vec array to the size of the 'recv_to' string + recv_io_vec[i].iov_len = recv_to.size(); + + // Increment the value of 'i' by 1 + ++i; + + // Set the base address of the i-th element in the recv_io_vec array to the data pointer of the 'recv_from' string + recv_io_vec[i].iov_base = const_cast(recv_from.data()); + + // Set the length of the i-th element in the recv_io_vec array to the size of the 'recv_from' string + recv_io_vec[i].iov_len = recv_from.size(); + + // Increment the value of 'i' by 1 + ++i; + + // Set the base address of the i-th element in the recv_io_vec array to the data pointer of the 'body' string in the 'msg' object + recv_io_vec[i].iov_base = const_cast(msg->body.data()); + + // Set the length of the i-th element in the recv_io_vec array to the size of the 'body' string in the 'msg' object + recv_io_vec[i].iov_len = msg->body.size(); + + // Increment the value of 'i' by 1 + ++i; + + // Set the message input-output vector to the specified receive input-output vector + recv_kernel_msg.msg_iov = recv_io_vec; + + // Set the length of the message input-output vector to the specified value + recv_kernel_msg.msg_iovlen = IntToSize(i); + + // Calculate the total receive length by summing the sizes of the name, recv_to, recv_from, and body in the message + total_recv_len = msg->name.size() + recv_to.size() + recv_from.size() + msg->body.size(); + + // Set the receive message to the specified message + recv_message = msg; + int Connection::Flush() { int total_send_bytes = 0; + + // Continue flushing until the send message queue is empty and total_send_len is zero while (!send_message_queue.empty() || total_send_len != 0) { if (total_send_len == 0) { + // If total_send_len is zero, fill the send message with the front message from the send message queue FillSendMessage(send_message_queue.front(), source, false); send_message_queue.pop(); } + size_t sendLen = 0; int retval = socket_operation->SendMessage(this, &send_kernel_msg, total_send_len, &sendLen); + + // If the send operation is successful and sendLen is greater than zero if (retval == IO_RW_OK && sendLen > 0) { total_send_len -= sendLen; + + // If total_send_len is now zero, update the send metrics if (total_send_len == 0) { // update metrics send_metrics->UpdateError(false); - - output_buffer_size -= send_message->body.size(); - total_send_bytes += send_message->body.size(); - delete send_message; - send_message = nullptr; - break; } - } else if (retval == IO_RW_OK && sendLen == 0) { - // EAGAIN - MS_LOG(ERROR) << "Failed to send message and update the epoll event"; - (void)recv_event_loop->UpdateEpollEvent(socket_fd, EPOLLOUT | EPOLLIN | EPOLLHUP | EPOLLERR); - continue; - } else { - // update metrics - send_metrics->UpdateError(true, error_code); - state = ConnectionState::kDisconnecting; - break; } } - return total_send_bytes; } +output_buffer_size -= send_message->body.size(); // Decrease the output buffer size by the size of the message body +total_send_bytes += send_message->body.size(); // Increase the total number of bytes sent by the size of the message body +delete send_message; // Delete the send_message object from memory +send_message = nullptr; // Set the send_message pointer to nullptr to avoid accessing deleted memory +break; // Exit the loop + +} else if (retval == IO_RW_OK && sendLen == 0) { +// EAGAIN +MS_LOG(ERROR) << "Failed to send message and update the epoll event"; // Log an error message indicating that sending the message and updating the epoll event failed +(void)recv_event_loop->UpdateEpollEvent(socket_fd, EPOLLOUT | EPOLLIN | EPOLLHUP | EPOLLERR); // Update the epoll event to include EPOLLOUT, EPOLLIN, EPOLLHUP, and EPOLLERR flags +continue; // Continue to the next iteration of the loop + +} else { +// update metrics +send_metrics->UpdateError(true, error_code); // Update the send_metrics object to indicate an error with the specified error code +state = ConnectionState::kDisconnecting; // Set the state variable to kDisconnecting to indicate that the connection is being disconnected +break; // Exit the loop +} + +} +return total_send_bytes; // Return the total number of bytes sent + +// Define the function "AddConnnectEventHandler" belonging to the class "Connection" int Connection::AddConnnectEventHandler() { + + // Call the "SetEventHandler" function of the "recv_event_loop" object + // Pass the "socket_fd" as the file descriptor to monitor for events + // Use the OR operator to combine the event flags EPOLLIN, EPOLLHUP, and EPOLLERR + // Pass the function pointer "NewConnectEventHandler" as the event handler + // Use reinterpret_cast to convert the "this" pointer to a void pointer and pass it as the user data + // Return the result of the "SetEventHandler" function call return recv_event_loop->SetEventHandler(socket_fd, EPOLLIN | EPOLLHUP | EPOLLERR, NewConnectEventHandler, reinterpret_cast(this)); } +// Function to parse a message in the Connection class bool Connection::ParseMessage() { + + // Variable to store the return value of the parsing operation int retval = 0; + + // Variable to store the length of the received message size_t recvLen = 0; + + // Pointer to a character buffer to store the received message char *recvBuf = nullptr; switch (recv_state) { // Parse message header. case State::kMsgHeader: + // Cast the memory address of recv_msg_header to a char pointer and add recv_len to get the starting position of recvBuf recvBuf = reinterpret_cast(&recv_msg_header) + recv_len; + + // Call the Receive function of the socket_operation object, passing in this pointer, recvBuf, the remaining size of the message header, and a pointer to recvLen retval = socket_operation->Receive(this, recvBuf, sizeof(MessageHeader) - recv_len, &recvLen); + + // If the return value is not IO_RW_OK, set the state to ConnectionState::kDisconnecting, update recv_len, and return false if (retval != IO_RW_OK) { state = ConnectionState::kDisconnecting; recv_len += recvLen; return false; } + + // If the sum of recvLen and recv_len is not equal to the size of the message header, update recv_len and return false if ((recvLen + recv_len) != sizeof(MessageHeader)) { recv_len += recvLen; return false; } + + // Reset recv_len to 0 recv_len = 0; + // Check if the received message's magic ID matches the expected magic ID if (strncmp(recv_msg_header.magic, RPC_MAGICID, sizeof(RPC_MAGICID) - 1) != 0) { + // If the magic ID does not match, log an error message and set the connection state to disconnecting MS_LOG(ERROR) << "Failed to check magicid, RPC_MAGICID: " << RPC_MAGICID << ", recv magic_id: " << recv_msg_header.magic; state = ConnectionState::kDisconnecting; return false; } + + // Reorder the header of the received message ReorderHeader(&recv_msg_header); + + // Fill the receive message with data FillRecvMessage(); + + // If the connection state is disconnecting, return false if (state == ConnectionState::kDisconnecting) { return false; } + + // Set the receive state to body recv_state = State::kBody; // Parse message body. case State::kBody: recvLen = 0; + + // Call the ReceiveMessage function of the socket_operation object, passing in the current object (this), + // the recv_kernel_msg object, total_recv_len, and a pointer to recvLen to store the received length retval = socket_operation->ReceiveMessage(this, &recv_kernel_msg, total_recv_len, &recvLen); + + // Check if the received length is not equal to the total receive length if (recvLen != total_recv_len) { + + // Check if the return value of the ReceiveMessage function is not IO_RW_OK if (retval != IO_RW_OK) { state = ConnectionState::kDisconnecting; return false; } + + // Subtract the received length from the total receive length total_recv_len -= recvLen; return false; } + + // Set the receive state to kMsgHeader recv_state = State::kMsgHeader; break; + + // If the state is not kBody, return false default: return false; } + + // Return true to indicate successful parsing of the message body return true; } +// Reorder the fields in the MessageHeader structure by converting them from network byte order to host byte order void Connection::ReorderHeader(MessageHeader *header) const { - header->name_len = ntohl(header->name_len); - header->to_len = ntohl(header->to_len); - header->from_len = ntohl(header->from_len); - header->body_len = ntohl(header->body_len); + header->name_len = ntohl(header->name_len); // Convert the name length field from network byte order to host byte order + header->to_len = ntohl(header->to_len); // Convert the "to" length field from network byte order to host byte order + header->from_len = ntohl(header->from_len); // Convert the "from" length field from network byte order to host byte order + header->body_len = ntohl(header->body_len); // Convert the body length field from network byte order to host byte order } + } // namespace rpc } // namespace distributed -} // namespace mindspore +} // namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/rpc/tcp/connection_pool.cc b/mindspore/ccsrc/distributed/rpc/tcp/connection_pool.cc index 77bc832e3e5..c846faca29e 100644 --- a/mindspore/ccsrc/distributed/rpc/tcp/connection_pool.cc +++ b/mindspore/ccsrc/distributed/rpc/tcp/connection_pool.cc @@ -14,192 +14,403 @@ * limitations under the License. */ +// Include the mutex header for using mutexes in the code #include + +// Include the connection pool header from the distributed/rpc/tcp directory #include "distributed/rpc/tcp/connection_pool.h" +// Start of the "mindspore" namespace namespace mindspore { -namespace distributed { -namespace rpc { -void ConnectionPool::SetLinkPattern(bool linkPattern) { double_link_ = linkPattern; } + + // Start of the "distributed" namespace, which is nested inside the "mindspore" namespace + namespace distributed { + + // Start of the "rpc" namespace, which is nested inside the "distributed" namespace + namespace rpc { + + // Definition of the "SetLinkPattern" function belonging to the "ConnectionPool" class + void ConnectionPool::SetLinkPattern(bool linkPattern) { + + // Set the value of the "double_link_" member variable to the provided "linkPattern" value + double_link_ = linkPattern; + } + } // End of the "rpc" namespace + } // End of the "distributed" namespace +} // End of the "mindspore" namespace + +// CloseConnection function definition for the ConnectionPool class void ConnectionPool::CloseConnection(Connection *conn) { + + // Check if the connection pointer is null if (conn == nullptr) { + // If it is null, return from the function return; } + + // Continue with the rest of the function if the connection pointer is not null - // Trigger Exit message note that this should be called before erasing link. Because we may chang deleted flag - // by to in this fun. And if deleted has been set to true, it means Exit message has been send before, do nothing. + // ... (rest of the code) + +// Check if the 'deleted' flag of the 'conn' object is false if (!conn->deleted) { + + // If the 'deleted' flag is false, call the DeleteConnInfo function passing the 'destination' and 'socket_fd' of the 'conn' object DeleteConnInfo(conn->destination, conn->socket_fd); } + + // Note: It is important to call this function before erasing the link, because the 'deleted' flag may be changed inside the DeleteConnInfo function. + // If the 'deleted' flag has already been set to true, it means that the Exit message has already been sent before, so we do nothing. + // Check if the destination of the connection is not empty if (!conn->destination.empty()) { - (void)connections_.erase(conn->destination); + // Erase the connection from the connections_ map using the destination as the key + (void)connections_.erase(conn->destination); } + + // Close the connection conn->Close(); + + // Delete the connection object from memory delete conn; + + // Set the conn pointer to nullptr to avoid accessing deleted memory conn = nullptr; } +// Function to find a connection in the connection pool based on the destination URL Connection *ConnectionPool::FindConnection(const std::string &dst_url) { + + // Initialize a pointer to a connection object as nullptr Connection *conn = nullptr; + + // Find the connection in the connections map using the destination URL as the key auto iter = connections_.find(dst_url); + + // If the connection is found in the map if (iter != connections_.end()) { + + // Assign the connection object to the conn pointer conn = iter->second; } + + // Return the connection object (or nullptr if not found) return conn; } +// A member function of the ConnectionPool class that resets the send metrics for all connections + void ConnectionPool::ResetAllConnMetrics() { + + // Iterate over all local connections in the local_conns_ map for (const auto &iter : local_conns_) { + + // Reset the send metrics for the current local connection iter.second->send_metrics->Reset(); } + + // Iterate over all remote connections in the remote_conns_ map for (const auto &iter : remote_conns_) { + + // Reset the send metrics for the current remote connection iter.second->send_metrics->Reset(); } } +// Function to delete a connection from the connection pool based on the destination URL void ConnectionPool::DeleteConnection(const std::string &dst_url) { + + // Find the connection in the connection pool based on the destination URL Connection *conn = FindConnection(dst_url); + + // If the connection is found if (conn != nullptr) { + + // Log an informational message indicating the unlinking of the file descriptor and the destination URL MS_LOG(INFO) << "unLink fd:" << conn->socket_fd << ",to:" << dst_url; + + // Close the connection CloseConnection(conn); } } +// A member function of the ConnectionPool class that deletes all connections in a given map of connections void ConnectionPool::DeleteAllConnections(std::map *links) { + + // Create an iterator to iterate through the map auto iter = links->begin(); + + // Iterate through the map until the end is reached while (iter != links->end()) { + + // Get the connection pointer from the current map entry Connection *conn = iter->second; - // erase link + + // Check if the connection has a receive message if (conn->recv_message != nullptr) { + + // Delete the receive message delete conn->recv_message; } + + // Erase the current map entry and update the iterator to the next entry iter = links->erase(iter); + + // Delete the connection object delete conn; + + // Set the connection pointer to nullptr to avoid dangling pointer conn = nullptr; } } +// Add a connection to the connection pool void ConnectionPool::AddConnection(Connection *conn) { + + // Check if the connection is null if (conn == nullptr) { MS_LOG(ERROR) << "The connection is null"; return; } + + // Find a connection with the same destination as the new connection Connection *tmpConn = FindConnection(conn->destination); + + // If a connection with the same destination exists if (tmpConn != nullptr) { + + // Log an informational message about unlinking the existing connection MS_LOG(INFO) << "unLink fd:" << tmpConn->socket_fd << ",to:" << tmpConn->destination.c_str(); + + // Close the existing connection CloseConnection(tmpConn); } + + // Add the new connection to the connection pool (void)connections_.emplace(conn->destination, conn); } +// DeleteConnInfo function definition for the ConnectionPool class + void ConnectionPool::DeleteConnInfo(int fd) { + + // Find the connection information associated with the given file descriptor auto iter = conn_infos_.find(fd); + + // If the connection information is not found, return without doing anything if (iter == conn_infos_.end()) { return; } + + // Get the connection information from the iterator auto conn_infos = iter->second; + + // Get an iterator to the beginning of the connection information list auto iter2 = conn_infos.begin(); + // ... (rest of the code is missing) + + // Iterate over the conn_infos list using an iterator while (iter2 != conn_infos.end()) { + + // Get the linkInfo object pointed to by the iterator auto linkInfo = *iter2; + + // Check if the delete_callback function pointer is not null if (linkInfo->delete_callback) { + + // Call the delete_callback function with the 'to' and 'from' parameters of linkInfo linkInfo->delete_callback(linkInfo->to, linkInfo->from); } + + // Erase the element pointed to by the iterator from the conn_infos list and update the iterator iter2 = conn_infos.erase(iter2); + + // Delete the linkInfo object to free the memory allocated for it delete linkInfo; } + + // Erase the element with key 'fd' from the conn_infos_ map (void)conn_infos_.erase(fd); -} + +// Implementation of the DeleteConnInfo function in the ConnectionPool class void ConnectionPool::DeleteConnInfo(const std::string &to, int fd) { - // If run in double link pattern, link fd and send fd must be the same, send Exit message bind on this fd + + // If the connection pool is running in double link pattern if (double_link_) { + + // Call the DeleteConnInfo function with only the fd parameter DeleteConnInfo(fd); + + // Return from the function return; } +} - // If run in single link pattern, link fd and send fd may not be the same, we should send Exit message bind - // on link fd and remote link fd. Here 'deleted' flag should be set true to avoid duplicate Exit message with - // same aid. - Connection *conn = FindConnection(to); - if (conn != nullptr) { +// Find the connection object associated with the given 'to' address +Connection *conn = FindConnection(to); + +// If a connection object is found +if (conn != nullptr) { + // Set the 'deleted' flag of the connection object to true conn->deleted = true; - DeleteConnInfo(conn->socket_fd); + // Delete the connection information associated with the socket file descriptor of the connection object + DeleteConnInfo(conn->socket_fd); +} + + // Check if the socket file descriptor of the connection is not equal to the given file descriptor if (conn->socket_fd != fd) { + + // Log an informational message indicating that a linker bind on the connection's file descriptor is being deleted MS_LOG(INFO) << "delete linker bind on link fd:" << conn->socket_fd << ",delete fd:" << fd; } } } +// Define the function `DeleteAllConnInfos` belonging to the `ConnectionPool` class + void ConnectionPool::DeleteAllConnInfos() { + + // Create an iterator `iter` and set it to the beginning of the `conn_infos_` map auto iter = conn_infos_.begin(); + + // Iterate through the `conn_infos_` map until the end is reached while (iter != conn_infos_.end()) { + + // Get the value (a vector of `conn_infos`) associated with the current key (`iter->second`) auto conn_infos = iter->second; + + // Create an iterator `iter2` and set it to the beginning of the `conn_infos` vector auto iter2 = conn_infos.begin(); + // ... (code continues) + + // Iterate through the conn_infos list until the end while (iter2 != conn_infos.end()) { + + // Get the linkInfo object pointed by iter2 auto linkInfo = *iter2; + + // Erase the element pointed by iter2 from the conn_infos list and update iter2 to the next element iter2 = conn_infos.erase(iter2); + + // Delete the linkInfo object to free up memory delete linkInfo; } + + // Erase the element pointed by iter from the conn_infos_ list and update iter to the next element iter = conn_infos_.erase(iter); } } +// Find the ConnectionInfo object associated with the given file descriptor (fd) and destination URL (dst_url) ConnectionInfo *ConnectionPool::FindConnInfo(int fd, const std::string &dst_url) { + + // Find the iterator in the conn_infos_ map for the given fd auto iter = conn_infos_.find(fd); + + // If the iterator points to the end of the map, it means no ConnectionInfo object is associated with the given fd if (iter == conn_infos_.end()) { return nullptr; } + + // Get the set of ConnectionInfo objects associated with the given fd auto conn_infos = iter->second; + + // Get the iterator to the first ConnectionInfo object in the set auto iter2 = conn_infos.begin(); + // Iterate through the conn_infos vector using an iterator while (iter2 != conn_infos.end()) { + + // Dereference the iterator to access the current linkInfo object auto linkInfo = *iter2; + + // Check if the 'to' member of the linkInfo object matches the dst_url if (linkInfo->to == dst_url) { + + // If a match is found, return the linkInfo object return linkInfo; } + + // Increment the iterator to move to the next element in the vector ++iter2; } + + // If no match is found, return a nullptr return nullptr; } +// Function to add connection information to the connection pool void ConnectionPool::AddConnInfo(int fd, const std::string &dst_url, DeleteCallBack callback) { + + // Check if the connection information already exists in the connection pool ConnectionInfo *linker = FindConnInfo(fd, dst_url); if (linker != nullptr) { - return; + return; // If it exists, return without adding it again } + + // If the connection information does not exist, create a new ConnectionInfo object linker = new (std::nothrow) ConnectionInfo(); if (linker == nullptr) { MS_LOG(ERROR) << "new ConnectionInfo fail dAid:" << dst_url; - return; + return; // If memory allocation fails, log an error and return } + + // Set the properties of the ConnectionInfo object linker->from = ""; linker->to = dst_url; linker->socket_fd = fd; linker->delete_callback = callback; + + // Insert the ConnectionInfo object into the connection_infos_ map (void)conn_infos_[fd].insert(linker); } +// Function to reverse the connection information between two file descriptors bool ConnectionPool::ReverseConnInfo(int fromFd, int toFd) { + + // Find the connection information for the 'fromFd' file descriptor in the 'conn_infos_' map auto iter = conn_infos_.find(fromFd); + + // If the connection information is not found, return false if (iter == conn_infos_.end()) { return false; } + + // Get the connection information from the iterator auto conn_infos = iter->second; + + // Erase the connection information for the 'fromFd' file descriptor from the 'conn_infos_' map (void)conn_infos_.erase(fromFd); + + // Assign the connection information to the 'toFd' file descriptor in the 'conn_infos_' map conn_infos_[toFd] = conn_infos; + + // Return true to indicate successful reversal of connection information return true; } +// Definition of the `Finalize` function belonging to the `ConnectionPool` class + void ConnectionPool::Finalize() { + + // Delete all local connections in the `local_conns_` container DeleteAllConnections(&local_conns_); + + // Delete all remote connections in the `remote_conns_` container DeleteAllConnections(&remote_conns_); + + // Delete all connection information stored in the `conn_infos_` container DeleteAllConnInfos(); -} -} // namespace rpc -} // namespace distributed -} // namespace mindspore + +} // End of `Finalize` function + +} // End of `rpc` namespace + +} // End of `distributed` namespace + +} // End of `mindspore` namespace \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/rpc/tcp/event_loop.cc b/mindspore/ccsrc/distributed/rpc/tcp/event_loop.cc index 0ece70fa688..c24481b2cfd 100644 --- a/mindspore/ccsrc/distributed/rpc/tcp/event_loop.cc +++ b/mindspore/ccsrc/distributed/rpc/tcp/event_loop.cc @@ -14,451 +14,890 @@ * limitations under the License. */ -#include "distributed/rpc/tcp/event_loop.h" +// Include the header file "distributed/rpc/tcp/event_loop.h" which contains the necessary declarations for the event loop functionality related to TCP communication in a distributed system. +// Include the header files for networking related functions and structures #include #include #include #include #include + +// Include the securec header file for secure C library functions #include + +// Include the header file for POSIX operating system API #include + +// Include the header files for utility functions and classes #include #include #include #include +// Include the header file for the log module in the actor namespace #include "actor/log.h" + +// Include the header file for the constants module in the distributed/rpc/tcp namespace #include "distributed/rpc/tcp/constants.h" +// Start of the "mindspore" namespace namespace mindspore { + +// Start of the "distributed" namespace within the "mindspore" namespace namespace distributed { + +// Start of the "rpc" namespace within the "distributed" namespace namespace rpc { + +// Definition of the function "EventLoopRun" which takes a pointer to an EventLoop object and a timeout value as parameters and returns an integer int EventLoopRun(EventLoop *evloop, int timeout) { + + // Declare a pointer to an epoll_event struct and initialize it to nullptr struct epoll_event *events = nullptr; + + // Call the sem_post function to increment the value of the semaphore identified by evloop->sem_id_ (void)sem_post(&evloop->sem_id_); - size_t size = sizeof(struct epoll_event) * EPOLL_EVENTS_SIZE; - events = (struct epoll_event *)malloc(size); - if (events == nullptr) { + // End of the "EventLoopRun" function + // No return statement is provided here, so the behavior is undefined + // It is assumed that the function continues with additional code that is not shown here + // The missing code should be added for a complete and functional implementation + +// End of the "rpc" namespace +} // namespace rpc + +// End of the "distributed" namespace +} // namespace distributed + +// End of the "mindspore" namespace +} // namespace mindspore + +// Calculate the size needed for the events array by multiplying the size of struct epoll_event by EPOLL_EVENTS_SIZE +size_t size = sizeof(struct epoll_event) * EPOLL_EVENTS_SIZE; + +// Allocate memory for the events array using malloc and cast it to struct epoll_event* +events = (struct epoll_event *)malloc(size); + +// Check if the memory allocation was successful +if (events == nullptr) { + // Print an error message using MS_LOG(ERROR) and return RPC_ERROR MS_LOG(ERROR) << "Failed to call malloc events"; return RPC_ERROR; - } - if (memset_s(events, size, 0, size)) { +} + +// Use memset_s to set all bytes in the events array to 0 +if (memset_s(events, size, 0, size)) { + // Print an error message using MS_LOG(ERROR), free the allocated memory, and return RPC_ERROR MS_LOG(ERROR) << "Failed to call memset_s."; free(events); return RPC_ERROR; - } +} + // Loop until the event loop is not stopped while (!evloop->is_stop_) { - /* free deleted event handlers */ + + // Remove any deleted event handlers from the event loop evloop->RemoveDeletedEvents(); + + // Wait for events to occur using epoll_wait int nevent = epoll_wait(evloop->epoll_fd_, events, EPOLL_EVENTS_SIZE, timeout); + + // If epoll_wait returns an error if (nevent < 0) { + // If the error is not due to interruption by a signal if (errno != EINTR) { + // Log an error message and return RPC_ERROR MS_LOG(ERROR) << "Failed to call epoll_wait, epoll_fd_: " << evloop->epoll_fd_ << ", errno: " << errno; free(events); return RPC_ERROR; } else { + // If the error is due to interruption by a signal, continue the loop continue; } } else if (nevent > 0) { - /* save the epoll modify in "stop" while dispatching handlers */ + // If there are events to handle + // Handle the events by calling the HandleEvent function of the event loop evloop->HandleEvent(events, nevent); } else { + // If epoll_wait returns 0, which means no events occurred within the specified timeout + // Log an error message and set is_stop_ to true to stop the event loop MS_LOG(ERROR) << "Failed to call epoll_wait, epoll_fd_: " << evloop->epoll_fd_ << ", ret: 0,errno: " << errno; evloop->is_stop_ = true; } + + // If the event loop is stopped, exit the loop if (evloop->is_stop_) { - /* free deleted event handlers */ + // ... + } + } + /* + * Call the RemoveDeletedEvents() function of the evloop object to remove any deleted event handlers. + * This function is responsible for freeing the memory occupied by the deleted event handlers. + */ evloop->RemoveDeletedEvents(); } } + + // Set the is_stop_ flag of the evloop object to false, indicating that the event loop is not stopped. evloop->is_stop_ = false; + + // Print an informational message using the MS_LOG macro, indicating that the event epoll loop has ended. MS_LOG(INFO) << "Event epoll loop run end"; + + // Free the memory occupied by the events array using the free() function. free(events); - return RPC_OK; -} +// Return the value RPC_OK to indicate successful execution of the function +return RPC_OK; +// A function named EvloopRun that takes a void pointer as an argument and returns a void pointer void *EvloopRun(void *arg) { + + // Check if the argument is a null pointer if (arg == nullptr) { + + // If the argument is null, log an error message using the MS_LOG macro MS_LOG(ERROR) << "Arg is null"; } else { + + // If the argument is not null, cast it to a pointer of type EventLoop and pass it to the EventLoopRun function (void)EventLoopRun(reinterpret_cast(arg), -1); } + + // Return a null pointer return nullptr; } +// Callback function for when the queue is ready void QueueReadyCallback(int fd, uint32_t events, void *arg) { + // Cast the argument to an EventLoop pointer EventLoop *evloop = reinterpret_cast(arg); + + // Check if the EventLoop pointer is null if (evloop == nullptr) { + // Log an error message and return if the EventLoop pointer is null MS_LOG(ERROR) << "The evloop is null fd:" << fd << ",events:" << events; return; } + + // Variable to store the count of items in the queue uint64_t count; + + // Read the count from the task queue event file descriptor ssize_t retval = read(evloop->task_queue_event_fd_, &count, sizeof(count)); + + // Check if the read was successful and the size of the read data is correct if (retval > 0 && retval == sizeof(count)) { - // take out functions from the queue + // Create a queue to store the functions to be executed std::queue> q; - evloop->task_queue_mutex_.lock(); - evloop->task_queue_.swap(q); - evloop->task_queue_mutex_.unlock(); +// Lock the mutex of the task queue in the event loop +evloop->task_queue_mutex_.lock(); - // invoke functions in the queue - while (!q.empty()) { - q.front()(); - q.pop(); - } - } +// Swap the contents of the task queue with the provided queue 'q' +evloop->task_queue_.swap(q); + +// Unlock the mutex of the task queue in the event loop +evloop->task_queue_mutex_.unlock(); + +// Invoke functions in the queue +while (!q.empty()) { + // Call the function at the front of the queue + q.front()(); + + // Remove the function from the queue + q.pop(); +} +// End of the main function } +// Release any resources held by the EventLoop + void EventLoop::ReleaseResource() { + + // Check if the task queue event file descriptor is valid if (task_queue_event_fd_ != -1) { + + // If it is valid, close the file descriptor if (close(task_queue_event_fd_) != 0) { + + // If closing the file descriptor fails, log an error message MS_LOG(ERROR) << "Failed to close task queue event fd: " << task_queue_event_fd_; } + + // Set the task queue event file descriptor to -1 to indicate it is closed task_queue_event_fd_ = -1; } + + // Check if the epoll file descriptor is valid if (epoll_fd_ != -1) { + + // If it is valid, close the file descriptor if (close(epoll_fd_) != 0) { + + // If closing the file descriptor fails, log an error message MS_LOG(ERROR) << "Failed to close epoll fd: " << epoll_fd_; } + + // Set the epoll file descriptor to -1 to indicate it is closed epoll_fd_ = -1; } } -size_t EventLoop::AddTask(std::function &&task) { - // put func to the queue - task_queue_mutex_.lock(); - (void)task_queue_.emplace(std::move(task)); +// Add a task to the task queue in the event loop - // return the queue size to send's caller. - auto result = task_queue_.size(); - task_queue_mutex_.unlock(); +// Acquire a lock on the task queue mutex to ensure thread safety +task_queue_mutex_.lock(); + +// Move the provided task into the task queue using std::move to avoid unnecessary copying +(void)task_queue_.emplace(std::move(task)); + +// Store the size of the task queue in a variable named "result" +auto result = task_queue_.size(); + +// Unlock the task queue mutex to allow other threads to access the task queue +task_queue_mutex_.unlock(); if (result == 1) { - // wakeup event loop + // If the result is equal to 1, it means that a wakeup event has occurred in the program + + // Create a variable 'one' of type uint64_t and assign it the value 1 uint64_t one = 1; + + // Write the value of 'one' to the task_queue_event_fd_ file descriptor ssize_t retval = write(task_queue_event_fd_, &one, sizeof(one)); + + // Check if the write operation was successful if (retval != sizeof(one)) { + // If the write operation failed, log a warning message with the file descriptor and the error number MS_LOG(WARNING) << "Failed to write queue Event fd: " << task_queue_event_fd_ << ",errno:" << errno; } } + + // Return the value of 'result' return result; } +// A member function of the EventLoop class that returns the number of remaining tasks in the task queue size_t EventLoop::RemainingTaskNum() { + + // Lock the task queue mutex to ensure exclusive access to the task queue task_queue_mutex_.lock(); + + // Get the size of the task queue auto task_num = task_queue_.size(); + + // Unlock the task queue mutex to release the lock task_queue_mutex_.unlock(); + + // Return the number of remaining tasks in the task queue return task_num; } +// Initialize the event loop with a given thread name bool EventLoop::Initialize(const std::string &threadName) { + + // Call the InitResource function and store the return value in retval int retval = InitResource(); + + // If the return value is not equal to RPC_OK, indicating an error if (retval != RPC_OK) { + + // Return false to indicate initialization failure return false; } + + // Initialize the semaphore with an initial value of 0 (void)sem_init(&sem_id_, 0, 0); - if (pthread_create(&loop_thread_, nullptr, EvloopRun, reinterpret_cast(this)) != 0) { + // Return true to indicate successful initialization + return true; +} + +// Check if the pthread_create function call is successful +if (pthread_create(&loop_thread_, nullptr, EvloopRun, reinterpret_cast(this)) != 0) { + + // If pthread_create fails, print an error message using the MS_LOG macro MS_LOG(ERROR) << "Failed to call pthread_create"; + + // Call the Finalize function Finalize(); + + // Return false to indicate failure return false; - } +} - // wait EvloopRun - (void)sem_wait(&sem_id_); +// Wait for the semaphore to be signaled by another thread +(void)sem_wait(&sem_id_); + +// Check if the version of the GLIBC library is at least 2.12 #if __GLIBC__ >= 2 && __GLIBC_MINOR__ >= 12 - std::string name = threadName; +// Create a string variable named "name" and initialize it with the value of the "threadName" variable +std::string name = threadName; + + + // Check if the name string is empty if (name.empty()) { + // If it is empty, assign a default name "EventLoopThread" name = "EventLoopThread"; } + + // Set the name of the pthread using the pthread_setname_np function + // Pass the loop_thread_ and the c-string representation of the name retval = pthread_setname_np(loop_thread_, name.c_str()); + + // Check if the pthread_setname_np function returned an error if (retval != 0) { + // If there was an error, log an informational message with the name and retval MS_LOG(INFO) << "Set pthread name fail name:" << name.c_str() << ",retval:" << retval; } else { + // If there was no error, log an informational message with the name and loop_thread_ MS_LOG(INFO) << "Set pthread name success name:" << name.c_str() << ",loop_thread_:" << loop_thread_; } #endif - return true; -} +// Return true to indicate successful program termination +return true; + +// Definition of the `Finalize` function belonging to the `EventLoop` class void EventLoop::Finalize() { + + // Check if the `loop_thread_` member variable is not null if (loop_thread_) { + + // Declare a void pointer `threadResult` and initialize it to nullptr void *threadResult = nullptr; + + // Call the `Stop` function to stop the loop thread Stop(); + // Join the loop_thread_ with the calling thread and store the result in threadResult int ret = pthread_join(loop_thread_, &threadResult); + + // Check if the return value of pthread_join is not equal to 0 if (ret != 0) { + // If the return value is not equal to 0, print an informational message indicating the failure MS_LOG(INFO) << "Failed to call pthread_join loop_thread_"; } + + // Set the value of loop_thread_ to 0 loop_thread_ = 0; } - RemoveDeletedEvents(); - ReleaseResource(); - MS_LOG(INFO) << "Stop loop succ"; -} +// Call the function to remove any deleted events +RemoveDeletedEvents(); -void EventLoop::DeleteEvent(int fd) { - auto iter = events_.find(fd); - if (iter == events_.end()) { +// Release any allocated resources +ReleaseResource(); + +// Use the MS_LOG macro to log an informational message indicating that the loop has stopped successfully +MS_LOG(INFO) << "Stop loop succ"; + +// Function to delete an event with the given file descriptor (fd) from the event loop + +// Find the event with the given file descriptor (fd) in the events_ map +auto iter = events_.find(fd); + +// If the event is not found in the map +if (iter == events_.end()) { + // Log an informational message indicating that the event with the given fd was not found MS_LOG(INFO) << "Not found event fd:" << fd; - return; - } - Event *eventData = iter->second; - if (eventData != nullptr) { - delete eventData; - } - (void)events_.erase(fd); + // Return from the function + return; } +// Get the pointer to the Event object associated with the given iterator +Event *eventData = iter->second; + +// Check if the pointer is not null +if (eventData != nullptr) { + // Delete the Event object to free up memory + delete eventData; +} + +// Erase the element from the events_ container using the given file descriptor as the key +(void)events_.erase(fd); + +// Function to find an event in the event loop based on the file descriptor (fd) Event *EventLoop::FindEvent(int fd) { + + // Use the find() function of the events_ map to search for the event with the given fd auto iter = events_.find(fd); + + // If the iterator points to the end of the map, it means the event was not found if (iter == events_.end()) { - return nullptr; + return nullptr; // Return nullptr to indicate that the event was not found } + + // If the event was found, return a pointer to the event object return iter->second; } +// Initialize the resource for the event loop int EventLoop::InitResource() { - int retval = 0; + + // Set the stop flag to false is_stop_ = false; + + // Create an epoll instance and store the file descriptor in epoll_fd_ epoll_fd_ = epoll_create(EPOLL_SIZE); + + // Check if the epoll creation was successful if (epoll_fd_ == -1) { + + // Print an error message with the error number if epoll_create failed MS_LOG(ERROR) << "Failed to call epoll_create, errno:" << errno; + + // Release any allocated resources ReleaseResource(); + + // Return an error code to indicate failure return RPC_ERROR; } - // create eventfd - task_queue_event_fd_ = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); - if (task_queue_event_fd_ == -1) { +// Create an event file descriptor using the eventfd system call +// The eventfd function takes two arguments: the initial value of the event counter (0 in this case) and flags +// The flags EFD_CLOEXEC and EFD_NONBLOCK are bitwise ORed together to set the close-on-exec and non-blocking flags respectively +task_queue_event_fd_ = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); + +// Check if the eventfd creation was successful +if (task_queue_event_fd_ == -1) { + // If eventfd returns -1, it indicates an error + // Log an error message with the corresponding errno value MS_LOG(ERROR) << "Failed to call eventfd, errno:" << errno; - ReleaseResource(); - return RPC_ERROR; - } + // Release any allocated resources + ReleaseResource(); + + // Return an error code to indicate failure + return RPC_ERROR; +} + + // Set the event handler for the task queue event file descriptor retval = SetEventHandler(task_queue_event_fd_, EPOLLIN | EPOLLHUP | EPOLLERR, QueueReadyCallback, reinterpret_cast(this)); + + // Check if setting the event handler was successful if (retval != RPC_OK) { + // If not successful, log an error message with the file descriptor value MS_LOG(ERROR) << "Add queue event fail task_queue_event_fd_:" << task_queue_event_fd_; + + // Release any allocated resources ReleaseResource(); + + // Return an error code to indicate failure return RPC_ERROR; } + + // If setting the event handler was successful, return a success code return RPC_OK; } -int EventLoop::SetEventHandler(int fd, uint32_t events, EventHandler handler, void *data) { - struct epoll_event ev; - Event *evdata = nullptr; - int ret = 0; +// Define the function SetEventHandler of the class EventLoop, which takes in parameters: +// - fd: file descriptor +// - events: bitmask representing the events to monitor +// - handler: function pointer to the event handler +// - data: pointer to user-defined data - if (memset_s(&ev, sizeof(ev), 0, sizeof(ev))) { +// Create an epoll_event struct named ev to store the event information +struct epoll_event ev; + +// Create a pointer to an Event object named evdata and initialize it to nullptr +Event *evdata = nullptr; + +// Initialize the return value variable ret to 0 +int ret = 0; + +// Use the memset_s function to set the memory of the 'ev' variable to zero +// The 'ev' variable is of type 'struct' and its size is determined by 'sizeof(ev)' +// If the memset_s function returns a non-zero value, it means that the memory could not be set to zero +if (memset_s(&ev, sizeof(ev), 0, sizeof(ev))) { + // Print an error message using the MS_LOG macro and return RPC_ERROR MS_LOG(ERROR) << "Failed to call memset_s."; return RPC_ERROR; - } - ev.events = events; +} - evdata = new (std::nothrow) Event(); - if (evdata == nullptr) { +// Set the 'events' member of the 'ev' variable to the value of the 'events' variable + +// Allocate memory for a new Event object using the new operator, with the option to return nullptr if the allocation fails +evdata = new (std::nothrow) Event(); + +// Check if the allocation was successful by comparing the pointer to nullptr +if (evdata == nullptr) { + // If the allocation failed, log an error message with the file descriptor (fd) and epoll file descriptor (epoll_fd_) MS_LOG(ERROR) << "Failed to call malloc eventData, fd:" << fd << ",epollfd:" << epoll_fd_; + + // Return an error code to indicate failure return RPC_ERROR; - } +} - evdata->data = data; - evdata->handler = handler; - evdata->fd = fd; +// Assign the 'data' value to the 'data' member of the 'evdata' structure +evdata->data = data; - event_lock_.lock(); - AddEvent(evdata); - event_lock_.unlock(); +// Assign the 'handler' value to the 'handler' member of the 'evdata' structure +evdata->handler = handler; - ev.data.ptr = evdata; - ret = epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, fd, &ev); - if (ret) { +// Assign the 'fd' value to the 'fd' member of the 'evdata' structure + +// Acquire a lock on the event_lock_ mutex to ensure exclusive access to the shared resource +event_lock_.lock(); + +// Call the AddEvent function, passing in the evdata parameter +AddEvent(evdata); + +// Release the lock on the event_lock_ mutex to allow other threads to access the shared resource +event_lock_.unlock(); + +// Assign the value of evdata to the pointer ev.data.ptr +ev.data.ptr = evdata; + +// Add the file descriptor fd to the epoll instance epoll_fd_ and associate it with the event ev +ret = epoll_ctl(epoll_fd_, EPOLL_CTL_ADD, fd, &ev); + +// If epoll_ctl returns a non-zero value, indicating an error +if (ret) { + // Acquire a lock on the event_lock_ mutex to ensure exclusive access to the event data event_lock_.lock(); + + // Delete the event associated with the file descriptor fd DeleteEvent(fd); + + // Release the lock on the event_lock_ mutex event_lock_.unlock(); if (errno != EEXIST) { - MS_LOG(ERROR) << "Failed to call epoll add, fail fd:" << fd << ",epollfd:" << epoll_fd_ << ",errno:" << errno; + // If the error number is not EEXIST (file already exists), log an error message with the details + MS_LOG(ERROR) << "Failed to call epoll add, fail fd:" << fd << ",epollfd:" << epoll_fd_ << ",errno:" << errno; } else { - MS_LOG(ERROR) << "The fd already existed in epoll, fd:" << fd << ",epollfd:" << epoll_fd_ << ",errno:" << errno; + // If the error number is EEXIST (file already exists), log an error message with the details + MS_LOG(ERROR) << "The fd already existed in epoll, fd:" << fd << ",epollfd:" << epoll_fd_ << ",errno:" << errno; } + + // Return RPC_ERROR to indicate an error occurred return RPC_ERROR; - } - return RPC_OK; } +// If no error occurred, return RPC_OK to indicate successful execution +return RPC_OK; + +// A member function of the EventLoop class that adds an event to the event loop + void EventLoop::AddEvent(Event *event) { + + // Check if the event is null, if so, return without doing anything if (!event) { return; } + + // Delete any existing event with the same file descriptor as the new event DeleteEvent(event->fd); + + // Insert the new event into the events_ map using its file descriptor as the key + // The emplace function returns a pair consisting of an iterator to the inserted element (or to the element that prevented the insertion) and a bool indicating whether the insertion took place + // The (void) is used to suppress any unused variable warning (void)events_.emplace(event->fd, event); } -int EventLoop::DeleteEpollEvent(int fd) { - Event *tev = nullptr; - struct epoll_event ev; - int ret = 0; +// DeleteEpollEvent function in the EventLoop class is used to delete an epoll event associated with a file descriptor (fd) +// It takes the file descriptor as a parameter and returns an integer indicating the result of the operation - event_lock_.lock(); - tev = FindEvent(fd); - if (tev == nullptr) { +// Declare a pointer to an Event object and initialize it to nullptr +Event *tev = nullptr; + +// Declare a struct epoll_event object named ev +struct epoll_event ev; + +// Declare an integer variable named ret and initialize it to 0 +int ret = 0; + +// Acquire a lock on the event_lock_ mutex to ensure exclusive access to the event data +event_lock_.lock(); + +// Find the event associated with the given file descriptor (fd) +tev = FindEvent(fd); + +// If no event is found, release the lock and return RPC_ERROR +if (tev == nullptr) { event_lock_.unlock(); return RPC_ERROR; - } - (void)events_.erase(tev->fd); +} - // Don't delete tev immediately, let's push it into deleted_events_, before next epoll_wait,we will free - // all events in deleted_events_. - AddDeletedEvent(tev); +// Remove the event from the events_ container using the erase() function +(void)events_.erase(tev->fd); // The (void) is used to suppress any unused variable warnings - event_lock_.unlock(); - ev.events = 0; - ev.data.ptr = tev; +// Add the given event (tev) to the list of deleted events (deleted_events_) +AddDeletedEvent(tev); +// Unlock the event lock to allow other threads to access the event +event_lock_.unlock(); + +// Set the events field of the event structure to 0, indicating no events are set +ev.events = 0; + +// Set the data pointer of the event structure to the provided tev pointer + + // Remove the file descriptor 'fd' from the epoll instance 'epoll_fd_' ret = epoll_ctl(epoll_fd_, EPOLL_CTL_DEL, fd, &ev); + + // If the removal operation failed if (ret < 0) { + // Log an error message indicating the failure, along with relevant information such as the file descriptor and epoll instance MS_LOG(ERROR) << "Failed to call delete fd in epoll, fd:" << fd << ",epollfd:" << epoll_fd_ << ",errno:" << errno; + + // Return an error code to indicate the failure return RPC_ERROR; } + + // If the removal operation was successful, return a success code return RPC_OK; } -int EventLoop::UpdateEpollEvent(int fd, uint32_t events) { - struct epoll_event ev; - Event *tev = nullptr; - int ret; +// Update the epoll event for a given file descriptor and events +// Create an epoll_event structure to hold the event information +struct epoll_event ev; + +// Create a pointer to an Event object and initialize it to nullptr +Event *tev = nullptr; + +// Declare a variable ret to store the return value of the function +int ret; + +// Call the FindEvent function to find the event associated with the given file descriptor (fd) tev = FindEvent(fd); + + // If the returned event is nullptr, it means the event lookup failed if (tev == nullptr) { + // Log an error message indicating the failure and provide the values of fd and events_ MS_LOG(ERROR) << "Failed to call event lookup, fd:" << fd << ",events:" << events_; + + // Return RPC_ERROR to indicate an error occurred return RPC_ERROR; } + + // Use the memset_s function to set the memory pointed to by ev to zero if (memset_s(&ev, sizeof(ev), 0, sizeof(ev))) { + // If memset_s fails, log an error message MS_LOG(ERROR) << "Failed to call memset_s."; + + // Return RPC_ERROR to indicate an error occurred return RPC_ERROR; } - ev.events = events; - ev.data.ptr = tev; +// Set the events field of the ev structure to the value of the events variable +ev.events = events; +// Set the data.ptr field of the ev structure to the value of the tev variable + + // Modify the file descriptor in the epoll instance ret = epoll_ctl(epoll_fd_, EPOLL_CTL_MOD, fd, &ev); + + // If the modification fails, log an error message with the details and return RPC_ERROR if (ret != 0) { MS_LOG(ERROR) << "Failed to modify fd in epoll, fd:" << fd << ",events:" << events << ",errno:" << errno; return RPC_ERROR; } + + // If the modification is successful, return RPC_OK return RPC_OK; -} -void EventLoop::AddDeletedEvent(Event *event) { - // caller need check eventData is not nullptr - std::list delete_event_list; +// This function is a member function of the EventLoop class and is used to add an event to the list of deleted events. +// The caller of this function needs to ensure that the eventData is not a nullptr. +// A std::list of Event pointers named delete_event_list is declared. - // if fd not found, push eventData into deleted_events_[fd] - std::map>::iterator fdIter = deleted_events_.find(event->fd); - if (fdIter == deleted_events_.end()) { +// Check if the file descriptor (fd) is not found in the deleted_events_ map +std::map>::iterator fdIter = deleted_events_.find(event->fd); +if (fdIter == deleted_events_.end()) { + + // If the fd is not found, push the eventData into the list associated with that fd in the deleted_events_ map deleted_events_[event->fd].push_back(event); return; - } +} - // if fd found, check if same eventData ptr exists + // Retrieve the list of events associated with the file descriptor (fd) from the map delete_event_list = fdIter->second; + + // Create an iterator to iterate over the events in the list std::list::iterator eventIter = delete_event_list.begin(); + + // Initialize a boolean variable to keep track of whether the event is found bool found = false; + + // Iterate over the events in the list while (eventIter != delete_event_list.end()) { + + // Check if the current event pointer matches the given event pointer if (*eventIter == event) { + + // If a match is found, log a warning message indicating that the fd has been deleted before MS_LOG(WARNING) << "The fd has been deleted before fd:" << event->fd << ",epoll_fd_:" << epoll_fd_; + + // Set the found flag to true found = true; + + // Exit the loop break; } + + // Move to the next event in the list ++eventIter; } - // if found same eventptr, do nothing + // Check if the event pointer is already present in the deleted_events_ vector if (found) { + // If found, do nothing and return return; } - deleted_events_[event->fd].push_back(event); - return; -} + // If not found, add the event pointer to the deleted_events_ vector for the corresponding file descriptor + deleted_events_[event->fd].push_back(event); + + // Return to the calling function + return; + +// Define a function named "RemoveDeletedEvents" belonging to the class "EventLoop" void EventLoop::RemoveDeletedEvents() { + + // Declare an iterator named "fdIter" for iterating over the "deleted_events_" map std::map>::iterator fdIter = deleted_events_.begin(); + // Iterate through the deleted_events_ map using an iterator named fdIter while (fdIter != deleted_events_.end()) { + + // Get the list of events associated with the current file descriptor (fdIter->second) std::list delete_event_list = fdIter->second; + + // Create an iterator named eventIter to iterate through the delete_event_list std::list::iterator eventIter = delete_event_list.begin(); + // Iterate through the delete_event_list using an iterator while (eventIter != delete_event_list.end()) { + + // Get the pointer to the current event Event *deleteEv = *eventIter; + + // Delete the event object delete deleteEv; + + // Set the pointer to nullptr to avoid dangling pointer deleteEv = nullptr; + + // Move to the next event in the list ++eventIter; } + + // Erase the element pointed by fdIter from the deleted_events_ container and increment fdIter (void)deleted_events_.erase(fdIter++); - } + + // Clear the deleted_events_ container deleted_events_.clear(); } +// Function to find a deleted event in the event loop int EventLoop::FindDeletedEvent(const Event *tev) { + + // Find the iterator in the map for the given file descriptor std::map>::iterator fdIter = deleted_events_.find(tev->fd); + + // If the iterator is at the end of the map, it means the event was not found if (fdIter == deleted_events_.end()) { - return 0; + return 0; // Return 0 to indicate that the event was not found } + // If the iterator is not at the end of the map, it means the event was found + // (Note: The actual event object is not checked here, only the file descriptor is used for comparison) +} - std::list delete_event_list = fdIter->second; - std::list::iterator eventIter = delete_event_list.begin(); +// Create a new list called "delete_event_list" and assign it the value of the list stored in the map "fdIter->second" +std::list delete_event_list = fdIter->second; +// Create an iterator called "eventIter" and initialize it to point to the beginning of the "delete_event_list" list +std::list::iterator eventIter = delete_event_list.begin(); + + // Start a while loop that iterates through the delete_event_list while (eventIter != delete_event_list.end()) { + + // Check if the current element pointed by eventIter is equal to tev if (*eventIter == tev) { + + // If it is equal, return 1 to indicate that the event was found in the list return 1; } + + // Increment the eventIter to move to the next element in the list ++eventIter; } + + // If the loop completes without finding the event, return 0 to indicate that the event was not found in the list return 0; } +// Define the function "HandleEvent" belonging to the class "EventLoop" void EventLoop::HandleEvent(struct epoll_event *events, size_t nevent) { + + // Declare a variable "found" of type int int found; + + // Declare a pointer variable "tev" of type "Event" and initialize it to nullptr Event *tev = nullptr; - for (size_t i = 0; i < nevent; i++) { +// Iterate over the events array from 0 to nevent +for (size_t i = 0; i < nevent; i++) { + + // Get the pointer to the Event object stored in the data field of the current event tev = reinterpret_cast(events[i].data.ptr); + // Check if the pointer tev is not null if (tev != nullptr) { + + // Call the function FindDeletedEvent with tev as argument and assign the result to found found = FindDeletedEvent(tev); + + // Check if found is true if (found) { + + // Print a warning message using MS_LOG with the values of tev->fd and epoll_fd_ MS_LOG(WARNING) << "The fd has been deleted from epoll fd:" << tev->fd << ",epoll_fd_:" << epoll_fd_; + + // Continue to the next iteration of the loop continue; } + + // Call the handler function of tev with arguments tev->fd, events[i].events, and tev->data tev->handler(tev->fd, events[i].events, tev->data); } } } +// Stop the event loop only if it is not already stopped void EventLoop::Stop() { + // Check if the event loop is already stopped if (is_stop_) { + // If it is already stopped, return without doing anything return; } - - is_stop_ = true; - uint64_t one = 1; - - if (write(task_queue_event_fd_, &one, sizeof(one)) != sizeof(one)) { - MS_LOG(WARNING) << "Failed to write task_queue_event_fd_ fd:" << task_queue_event_fd_ << ",errno:" << errno; - } - return; } + +// Set the value of the variable is_stop_ to true +is_stop_ = true; + +// Declare and initialize a variable named one of type uint64_t with the value 1 +uint64_t one = 1; + +// Check if writing to the task_queue_event_fd_ was successful +if (write(task_queue_event_fd_, &one, sizeof(one)) != sizeof(one)) { + // If writing failed, log a warning message with the file descriptor and error number + MS_LOG(WARNING) << "Failed to write task_queue_event_fd_ fd:" << task_queue_event_fd_ << ",errno:" << errno; +} + +// End of the rpc namespace } // namespace rpc + +// End of the distributed namespace } // namespace distributed -} // namespace mindspore + +// End of the mindspore namespace +} // namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/rpc/tcp/socket_operation.cc b/mindspore/ccsrc/distributed/rpc/tcp/socket_operation.cc index e663226d3bb..2fcc58822e8 100644 --- a/mindspore/ccsrc/distributed/rpc/tcp/socket_operation.cc +++ b/mindspore/ccsrc/distributed/rpc/tcp/socket_operation.cc @@ -14,361 +14,674 @@ * limitations under the License. */ -#include "distributed/rpc/tcp/socket_operation.h" +// Include the header file "distributed/rpc/tcp/socket_operation.h" which contains the necessary declarations for socket operations in a distributed RPC system. -#include -#include -#include -#include -#include -#include -#include -#include +// Include the header files required for network interface operations and error handling +#include // For ioctl system call +#include // For network interface related structures and constants +#include // For getting interface addresses +#include // For manipulating IP addresses +#include // For secure C library functions +#include // For TCP related constants +#include // For standard POSIX functions +#include // For handling system errors + +// Include the header file for the log module in the actor namespace #include "actor/log.h" + +// Include the header file for the constants module in the distributed/rpc/tcp namespace #include "distributed/rpc/tcp/constants.h" +// Start of the "mindspore" namespace namespace mindspore { + +// Start of the "distributed" namespace, which is nested inside the "mindspore" namespace namespace distributed { + +// Start of the "rpc" namespace, which is nested inside the "distributed" namespace namespace rpc { + +// Definition of the SetSocketKeepAlive function, which takes in several parameters int SocketOperation::SetSocketKeepAlive(int fd, int keepalive, int keepidle, int keepinterval, int keepcount) { + + // Initialize the option_val variable to 0 int option_val = 0; + + // Initialize the ret variable to 0 int ret = 0; - option_val = keepalive; - ret = setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &option_val, sizeof(option_val)); - if (ret < 0) { +// Set the value of the keepalive option to the variable option_val +option_val = keepalive; + +// Call the setsockopt function to set the SO_KEEPALIVE option for the given file descriptor (fd) +// The SOL_SOCKET argument specifies that the option is to be applied to the socket level +// The SO_KEEPALIVE argument specifies the option to be set +// The &option_val argument passes the address of the option_val variable to setsockopt +// The sizeof(option_val) argument specifies the size of the option_val variable in bytes +ret = setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &option_val, sizeof(option_val)); + +// Check if the setsockopt function call was successful +if (ret < 0) { + // If the function call failed, print an error message with the file descriptor and the errno value MS_LOG(ERROR) << "Failed to call setsockopt SO_KEEPALIVE, fd: " << fd << ", errno:" << errno; + // Return -1 to indicate an error occurred return -1; - } +} - // Send first probe after `interval' seconds. + // Set the option value to the desired keepidle value option_val = keepidle; + + // Set the TCP_KEEPIDLE option on the socket file descriptor ret = setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &option_val, sizeof(option_val)); + + // Check if the setsockopt call was successful if (ret < 0) { + // Print an error message with the failed file descriptor and the errno value MS_LOG(ERROR) << "Failed to call setsockopt TCP_KEEPIDLE, fd: " << fd << ", errno:" << errno; + + // Return -1 to indicate an error return -1; } - // Send next probes after the specified interval. + // Set the option value to the specified keep interval option_val = keepinterval; + + // Set the TCP_KEEPINTVL socket option to the specified value ret = setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &option_val, sizeof(option_val)); + + // Check if the setsockopt call was successful if (ret < 0) { + // Print an error message with the file descriptor and the error number MS_LOG(ERROR) << "Failed to call setsockopt TCP_KEEPINTVL, fd: " << fd << ", errno:" << errno; + + // Return -1 to indicate an error return -1; } - /* Consider the socket in error state after three we send three ACK - * probes without getting a reply. */ + /* Consider the socket in error state after three ACK probes have been sent without receiving a reply. */ option_val = keepcount; ret = setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &option_val, sizeof(option_val)); if (ret < 0) { + // If setting the socket option fails, log an error message with the file descriptor and the error number MS_LOG(ERROR) << "Failed to call setsockopt TCP_KEEPCNT, fd: " << fd << ", errno:" << errno; return -1; } + // Return 0 to indicate successful execution return 0; } +// Define a function named SetSocketOptions that takes an integer parameter named sock_fd and returns an integer value int SocketOperation::SetSocketOptions(int sock_fd) { + + // Initialize an integer variable named option_val with the value 1 int option_val = 1; + + // Initialize an integer variable named ret with the value 0 int ret = 0; + // Set the socket option SO_REUSEADDR to allow reusing the local address and port ret = setsockopt(sock_fd, SOL_SOCKET, SO_REUSEADDR, &option_val, sizeof(option_val)); + + // Check if the setsockopt function call was successful if (ret) { + // If it failed, log an error message with the socket file descriptor and the error number MS_LOG(ERROR) << "Failed to call setsockopt SO_REUSEADDR, fd: " << sock_fd << ", errno:" << errno; + + // Return -1 to indicate an error occurred return -1; } - ret = setsockopt(sock_fd, IPPROTO_TCP, TCP_NODELAY, &option_val, sizeof(option_val)); - if (ret) { +// Set the TCP_NODELAY option for the socket with file descriptor sock_fd +ret = setsockopt(sock_fd, IPPROTO_TCP, TCP_NODELAY, &option_val, sizeof(option_val)); + +// Check if the setsockopt function call was successful +if (ret) { + // If the function call failed, log an error message with the socket file descriptor and the error number MS_LOG(ERROR) << "Failed to call setsockopt TCP_NODELAY, fd: " << sock_fd << ", errno:" << errno; + // Return -1 to indicate an error occurred return -1; - } - - ret = SetSocketKeepAlive(sock_fd, SOCKET_KEEPALIVE, SOCKET_KEEPIDLE, SOCKET_KEEPINTERVAL, SOCKET_KEEPCOUNT); - if (ret) { - MS_LOG(WARNING) << "Failed to call setsockopt keep alive, fd: " << sock_fd; - } - return 0; } +// Call the function SetSocketKeepAlive with the provided arguments and store the return value in the variable ret +ret = SetSocketKeepAlive(sock_fd, SOCKET_KEEPALIVE, SOCKET_KEEPIDLE, SOCKET_KEEPINTERVAL, SOCKET_KEEPCOUNT); + +// Check if the return value is non-zero (indicating an error) +if (ret) { + // Print a warning message using the MS_LOG macro, indicating the failure to set the socket option + MS_LOG(WARNING) << "Failed to call setsockopt keep alive, fd: " << sock_fd; +} + +// Return 0 to indicate successful program termination +return 0; + +// Define a function named CreateSocket that returns an integer and takes a parameter of type sa_family_t named family + int SocketOperation::CreateSocket(sa_family_t family) { + + // Declare and initialize two integer variables named ret and fd int ret = 0; int fd = 0; - // Create server socket - fd = ::socket(family, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0); - if (fd < 0) { - MS_LOG(WARNING) << "Failed to create socket: " << errno; - return -1; - } +// Create a server socket using the socket() function +fd = ::socket(family, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0); - ret = SetSocketOptions(fd); - if (ret < 0) { - if (close(fd) != 0) { - MS_LOG(EXCEPTION) << "Failed to close fd: " << fd; - } +// Check if the socket creation was successful +if (fd < 0) { + // If the socket creation failed, log a warning message with the error number + MS_LOG(WARNING) << "Failed to create socket: " << errno; + + // Return -1 to indicate an error occurred return -1; - } - return fd; } + // Set socket options for the given file descriptor + ret = SetSocketOptions(fd); + + // If setting socket options failed + if (ret < 0) { + // Close the file descriptor + if (close(fd) != 0) { + // If closing the file descriptor failed, log an exception with the file descriptor value + MS_LOG(EXCEPTION) << "Failed to close fd: " << fd; + } + // Return -1 to indicate failure + return -1; + } + + // Return the file descriptor if everything is successful + return fd; + +// Define a function named GetLocalIP in the SocketOperation class that returns a string std::string SocketOperation::GetLocalIP() { - // Lookup all the network interfaces on the local machine. + + // Declare a pointer to a struct ifaddrs to store network interface information struct ifaddrs *if_addrs; + + // Use the getifaddrs function to lookup all the network interfaces on the local machine + // If the function returns a non-zero value, it means the lookup failed if (getifaddrs(&if_addrs) != 0) { + + // Print an error message using the MS_LOG macro and return an empty string MS_LOG(ERROR) << "Failed to lookup local network interfaces."; freeifaddrs(if_addrs); return ""; } - // Find the first physical network interface. + + // Declare a pointer to a struct ifaddrs to iterate through the network interfaces struct ifaddrs *if_addr = if_addrs; + + // Check if the pointer is not null MS_EXCEPTION_IF_NULL(if_addr); + + // Start a while loop to iterate through the network interfaces while (if_addr != nullptr) { + + // Check if the ifa_addr member of the current network interface is null if (if_addr->ifa_addr == nullptr) continue; + // Check if the address family of the interface address is AF_INET (IPv4) and if the interface is not a loopback interface if (if_addr->ifa_addr->sa_family == AF_INET && !(if_addr->ifa_flags & IFF_LOOPBACK)) { + + // Cast the interface address to a sockaddr_in structure pointer auto sock_addr = reinterpret_cast(if_addr->ifa_addr); + + // Throw an exception if the casted sockaddr_in pointer is null MS_EXCEPTION_IF_NULL(sock_addr); - auto ip_addr = inet_ntoa(sock_addr->sin_addr); - MS_EXCEPTION_IF_NULL(ip_addr); +// Store the string representation of the IP address in the variable "ip_addr" using the inet_ntoa function +auto ip_addr = inet_ntoa(sock_addr->sin_addr); - std::string ip(ip_addr, ip_addr + strlen(ip_addr)); - freeifaddrs(if_addrs); - return ip; - } else { - if_addr = if_addr->ifa_next; - } - } - freeifaddrs(if_addrs); - return ""; +// Check if the "ip_addr" variable is null, and throw an exception if it is +MS_EXCEPTION_IF_NULL(ip_addr); + +// Create a string object 'ip' using the character array 'ip_addr' and its length obtained from strlen +std::string ip(ip_addr, ip_addr + strlen(ip_addr)); + +// Free the memory allocated for the linked list of interface addresses +freeifaddrs(if_addrs); + +// Return the IP address stored in the 'ip' string +return ip; + +// If no IP address is found, execute the following code block +} else { + + // Move to the next interface address in the linked list + if_addr = if_addr->ifa_next; } -std::string SocketOperation::GetIP(const std::string &url) { - size_t index1 = url.find("["); +// Free the memory allocated for the linked list of interface addresses +freeifaddrs(if_addrs); + +// Return an empty string indicating that no IP address was found +return ""; + +// This function takes a URL as input and returns the IP address extracted from the URL + +// Find the index of the opening square bracket '[' in the URL +size_t index1 = url.find("["); + +// If the opening square bracket is not found in the URL +if (index1 == std::string::npos) { + // Find the index of the URL protocol and IP separator (e.g., "://") + index1 = url.find(URL_PROTOCOL_IP_SEPARATOR); + + // If the URL protocol and IP separator is not found in the URL if (index1 == std::string::npos) { - index1 = url.find(URL_PROTOCOL_IP_SEPARATOR); - if (index1 == std::string::npos) { - index1 = 0; - } else { - index1 = index1 + sizeof(URL_PROTOCOL_IP_SEPARATOR) - 1; - } + // Set the index to 0 to indicate that the IP address starts from the beginning of the URL + index1 = 0; } else { - index1 = index1 + 1; + // Adjust the index to point to the character after the URL protocol and IP separator + index1 = index1 + sizeof(URL_PROTOCOL_IP_SEPARATOR) - 1; } +} else { + // Adjust the index to point to the character after the opening square bracket + index1 = index1 + 1; +} - size_t index2 = url.find("]"); - if (index2 == std::string::npos) { +// Find the index of the closing square bracket "]" in the string "url" +size_t index2 = url.find("]"); + +// If the closing square bracket "]" is not found in the string "url" +if (index2 == std::string::npos) { + // Find the last occurrence of the URL_IP_PORT_SEPARATOR character in the string "url" index2 = url.rfind(URL_IP_PORT_SEPARATOR); - if (index2 == std::string::npos) { - MS_LOG(INFO) << "Couldn't find the character: " << URL_IP_PORT_SEPARATOR << ", url: " << url.c_str(); - return ""; - } - } + // If the URL_IP_PORT_SEPARATOR character is not found in the string "url" + if (index2 == std::string::npos) { + // Print an informational message indicating that the character URL_IP_PORT_SEPARATOR was not found in the string "url" + MS_LOG(INFO) << "Couldn't find the character: " << URL_IP_PORT_SEPARATOR << ", url: " << url.c_str(); + + // Return an empty string + return ""; + } +} + + // Check if index1 is greater than index2 if (index1 > index2) { + // If true, log an informational message using MS_LOG macro, indicating that parsing the IP failed MS_LOG(INFO) << "Parse ip failed, url: " << url.c_str(); + + // Return an empty string to indicate failure return ""; } if (index2 >= url.size()) { + // If the second index is greater than or equal to the size of the url string, + // it means that the url is invalid MS_LOG(ERROR) << "Invalid url: " << url; + + // Return an empty string to indicate failure return ""; } else { + // If the second index is within the bounds of the url string, + // extract the substring from index1 to index2-1 and store it in the variable ip std::string ip = url.substr(index1, index2 - index1); + + // Create a SocketAddress object named addr SocketAddress addr; + // Attempt to convert the IP address string to a binary format for IPv4 int result = inet_pton(AF_INET, ip.c_str(), &addr.saIn.sin_addr); + + // If the conversion fails or returns 0 (indicating an invalid IP address), try converting to IPv6 if (result <= 0) { - result = inet_pton(AF_INET6, ip.c_str(), &addr.saIn6.sin6_addr); - if (result <= 0) { - MS_LOG(INFO) << "Parse ip failed, result: " << result << ", url:" << url.c_str(); - return ""; - } + result = inet_pton(AF_INET6, ip.c_str(), &addr.saIn6.sin6_addr); + + // If the conversion still fails or returns 0, log an error message and return an empty string + if (result <= 0) { + MS_LOG(INFO) << "Parse ip failed, result: " << result << ", url:" << url.c_str(); + return ""; + } } + + // Return the original IP address string if it was successfully converted to binary format return ip; - } } +// A member function of the SocketOperation class that takes a URL and a pointer to a SocketAddress object as parameters bool SocketOperation::GetSockAddr(const std::string &url, SocketAddress *addr) { + + // Declare a string variable to store the IP address std::string ip; + + // Declare a uint16_t variable to store the port number, initialized to 0 uint16_t port = 0; - size_t len = sizeof(*addr); - (void)memset_s(addr, len, 0, len); +// Calculate the size of the object pointed to by 'addr' and store it in the variable 'len' +size_t len = sizeof(*addr); - size_t index1 = url.find(URL_PROTOCOL_IP_SEPARATOR); - if (index1 == std::string::npos) { +// Use the 'memset_s' function to set the memory pointed to by 'addr' to zero +// The 'memset_s' function takes four arguments: +// 1. The pointer to the memory to be set (in this case, 'addr') +// 2. The size of the memory to be set (in this case, 'len') +// 3. The value to set the memory to (in this case, 0) +// 4. The size of the memory to be set (in this case, 'len') +// The '(void)' before the function call is used to suppress any potential warnings about the return value of 'memset_s' not being used +(void)memset_s(addr, len, 0, len); + +// Find the position of the first occurrence of the URL_PROTOCOL_IP_SEPARATOR in the string 'url' +size_t index1 = url.find(URL_PROTOCOL_IP_SEPARATOR); + +// If the URL_PROTOCOL_IP_SEPARATOR is not found in the string 'url', set 'index1' to 0 +if (index1 == std::string::npos) { index1 = 0; - } else { +} +// If the URL_PROTOCOL_IP_SEPARATOR is found in the string 'url', calculate the new value of 'index1' +else { + // Increment 'index1' by the size of the URL_PROTOCOL_IP_SEPARATOR minus 1 index1 = index1 + sizeof(URL_PROTOCOL_IP_SEPARATOR) - 1; - } +} - size_t index2 = url.rfind(':'); - if (index2 == std::string::npos) { +// Find the last occurrence of the character ':' in the string 'url' and store its index in the variable 'index2' +size_t index2 = url.rfind(':'); + +// If the character ':' is not found in the string 'url' +if (index2 == std::string::npos) { + // Print an error message using the MS_LOG macro and return false MS_LOG(ERROR) << "Couldn't find the character colon."; return false; - } - - ip = url.substr(index1, index2 - index1); - if (ip.empty()) { - MS_LOG(ERROR) << "Couldn't find ip in url: " << url.c_str(); - return false; - } - - size_t idx = index2 + sizeof(URL_IP_PORT_SEPARATOR) - 1; - if (idx >= url.size()) { - MS_LOG(ERROR) << "The size of url is invalid"; - return false; - } - try { - port = (uint16_t)std::stoul(url.substr(idx)); - } catch (const std::system_error &e) { - MS_LOG(ERROR) << "Couldn't find port in url: " << url.c_str(); - return false; - } - - int result = inet_pton(AF_INET, ip.c_str(), &addr->saIn.sin_addr); - if (result > 0) { - addr->saIn.sin_family = AF_INET; - addr->saIn.sin_port = htons(port); - return true; - } - - result = inet_pton(AF_INET6, ip.c_str(), &(addr->saIn6.sin6_addr)); - if (result > 0) { - addr->saIn6.sin6_family = AF_INET6; - addr->saIn6.sin6_port = htons(port); - return true; - } - - MS_LOG(ERROR) << "Parse ip failed, result: " << result << ", url: " << url.c_str(); - return false; } -uint16_t SocketOperation::GetPort(int fd) { - uint16_t port = 0; - int retval = 0; - union SocketAddress isa; - socklen_t isaLen = sizeof(struct sockaddr_storage); +// Extract a substring from the 'url' string, starting at 'index1' and ending at 'index2 - 1' + ip = url.substr(index1, index2 - index1); + // Check if the extracted 'ip' string is empty + if (ip.empty()) { + // If 'ip' is empty, log an error message indicating that the IP couldn't be found in the 'url' string + MS_LOG(ERROR) << "Couldn't find ip in url: " << url.c_str(); + + // Return false to indicate failure + return false; + } + +// Calculate the index of the character after the URL-IP-PORT separator +size_t idx = index2 + sizeof(URL_IP_PORT_SEPARATOR) - 1; + +// Check if the calculated index is out of bounds of the URL string +if (idx >= url.size()) { + // Print an error message indicating that the size of the URL is invalid + MS_LOG(ERROR) << "The size of url is invalid"; + // Return false to indicate failure + return false; +} + +// Try to convert the substring starting from the calculated index to an unsigned long integer (port number) +try { + // Convert the substring to an unsigned long integer and assign it to the 'port' variable + port = (uint16_t)std::stoul(url.substr(idx)); +} catch (const std::system_error &e) { + // Print an error message indicating that the port couldn't be found in the URL + MS_LOG(ERROR) << "Couldn't find port in url: " << url.c_str(); + // Return false to indicate failure + return false; +} + + // Call the inet_pton function to convert the IP address in string format to binary format + // The first argument is the address family (AF_INET for IPv4) + // The second argument is the IP address in string format + // The third argument is a pointer to the memory where the binary IP address will be stored + int result = inet_pton(AF_INET, ip.c_str(), &addr->saIn.sin_addr); + + // If the result is greater than 0, it means the conversion was successful + if (result > 0) { + // Set the address family to AF_INET (IPv4) + addr->saIn.sin_family = AF_INET; + + // Set the port number in network byte order using the htons function + addr->saIn.sin_port = htons(port); + + // Return true to indicate that the conversion and assignment were successful + return true; + } + + // Call the inet_pton function to convert the IP address in string format to a binary format for IPv6 + // The result of the conversion is stored in the variable 'result' + result = inet_pton(AF_INET6, ip.c_str(), &(addr->saIn6.sin6_addr)); + + // If the conversion was successful (result > 0), proceed with setting up the IPv6 address structure + if (result > 0) { + // Set the address family to AF_INET6 to indicate IPv6 + addr->saIn6.sin6_family = AF_INET6; + + // Set the port number in network byte order using the htons function + addr->saIn6.sin6_port = htons(port); + + // Return true to indicate that the setup was successful + return true; + } + +// Log an error message using the MS_LOG macro, which is likely a custom logging framework +MS_LOG(ERROR) << "Parse ip failed, result: " << result << ", url: " << url.c_str(); + +// Return false to indicate that the parsing of the IP failed +return false; + +// Declare a variable of type uint16_t to store the port number +uint16_t port = 0; + +// Declare a variable of type int to store the return value of a function +int retval = 0; + +// Declare a union variable of type SocketAddress to store the socket address +union SocketAddress isa; + +// Declare a variable of type socklen_t to store the length of the socket address +socklen_t isaLen = sizeof(struct sockaddr_storage); + + // Call the getsockname function to retrieve the local address and port associated with the given socket file descriptor retval = getsockname(fd, &isa.sa, &isaLen); + + // Check if the getsockname function call was successful if (retval) { + // If the function call failed, log an informational message with the details of the failure MS_LOG(INFO) << "Failed to call getsockname, fd: " << fd << ", ret: " << retval << ", errno: " << errno; + + // Return the default port value return port; } - if (isa.sa.sa_family == AF_INET) { +// Check if the socket address family is AF_INET (IPv4) +if (isa.sa.sa_family == AF_INET) { + // If it is, extract the port number from the IPv4 socket address structure and convert it from network byte order to host byte order port = ntohs(isa.saIn.sin_port); - } else if (isa.sa.sa_family == AF_INET6) { +} +// If the socket address family is AF_INET6 (IPv6) +else if (isa.sa.sa_family == AF_INET6) { + // Extract the port number from the IPv6 socket address structure and convert it from network byte order to host byte order port = ntohs(isa.saIn6.sin6_port); - } else { +} +// If the socket address family is neither AF_INET nor AF_INET6 +else { + // Log an informational message indicating that the file descriptor and the unknown socket address family MS_LOG(INFO) << "Unknown fd: " << fd << ", family: " << isa.sa.sa_family; - } - return port; } +// Return the extracted port number +return port; + +// Define a function named GetPeer that belongs to the SocketOperation class and returns a string + std::string SocketOperation::GetPeer(int sock_fd) { + + // Declare a string variable named peer to store the peer information std::string peer; + + // Declare an integer variable named retval to store the return value of a function int retval = 0; + + // Declare a union named isa to store the socket address union SocketAddress isa; + + // Declare a socklen_t variable named isaLen to store the size of the socket address structure socklen_t isaLen = sizeof(struct sockaddr_storage); + // ... (more code follows) + + // Call the getpeername function to retrieve the address of the connected peer socket retval = getpeername(sock_fd, &isa.sa, &isaLen); + + // Check if the getpeername function call was successful if (retval < 0) { + // If the function call failed, log an informational message with the details of the failure MS_LOG(INFO) << "Failed to call getpeername, fd: " << sock_fd << ", ret: " << retval << ", errno: " << errno; + + // Return the default value of the peer address return peer; } - char ipdotdec[IP_LEN_MAX]; - if (isa.sa.sa_family == AF_INET) { +// Declare a character array to store the IP address in dot-decimal notation +char ipdotdec[IP_LEN_MAX]; + +// Check if the socket address family is IPv4 +if (isa.sa.sa_family == AF_INET) { + // Convert the IPv4 address from binary to dot-decimal notation and store it in ipdotdec if (inet_ntop(AF_INET, reinterpret_cast(&isa.saIn.sin_addr), ipdotdec, IP_LEN_MAX) == nullptr) { - MS_LOG(EXCEPTION) << "Failed to call inet_ntop kernel func."; + // If the conversion fails, throw an exception with an error message + MS_LOG(EXCEPTION) << "Failed to call inet_ntop kernel func."; } + // Concatenate the IP address and port number into a string and assign it to the peer variable peer = std::string(ipdotdec) + ":" + std::to_string(ntohs(isa.saIn.sin_port)); - } else if (isa.sa.sa_family == AF_INET6) { +} +// Check if the socket address family is IPv6 +else if (isa.sa.sa_family == AF_INET6) { + // Convert the IPv6 address from binary to dot-decimal notation and store it in ipdotdec if (inet_ntop(AF_INET6, reinterpret_cast(&isa.saIn6.sin6_addr), ipdotdec, IP_LEN_MAX) == nullptr) { - MS_LOG(ERROR) << "Failed to call inet_ntop."; + // If the conversion fails, log an error message + MS_LOG(ERROR) << "Failed to call inet_ntop."; } + // Concatenate the IP address and port number into a string and assign it to the peer variable peer = std::string(ipdotdec) + ":" + std::to_string(ntohs(isa.saIn6.sin6_port)); - } else { +} +// If the socket address family is neither IPv4 nor IPv6 +else { + // Log an informational message indicating that the socket address family is unknown MS_LOG(INFO) << "Unknown fd: " << sock_fd << ", family: " << isa.sa.sa_family; - } - return peer; } +// Return the peer string +return peer; + +// Define a function named "Connect" which takes in the following parameters: +// - an integer representing the socket file descriptor +// - a pointer to a sockaddr structure representing the socket address +// - a socklen_t variable representing the length of the socket address +// - a pointer to a uint16_t variable representing the bound port + int SocketOperation::Connect(int sock_fd, const struct sockaddr *sa, socklen_t saLen, uint16_t *boundPort) { + + // Declare and initialize an integer variable named "retval" with a value of 0 int retval = 0; retval = connect(sock_fd, sa, saLen); if (retval != 0) { if (errno == EINPROGRESS) { - /* set iomux for write event */ + /* If the connection is in progress, set iomux for write event */ } else { + // If connect fails with an error other than EINPROGRESS, log an error message MS_LOG(ERROR) << "Failed to call connect, fd: " << sock_fd << ", ret: " << retval << ", errno: " << errno; + // Return the value of retval to indicate the failure return retval; } } - // to get local port - *boundPort = GetPort(sock_fd); - if (*boundPort == 0) { +// Get the local port number for the given socket file descriptor +*boundPort = GetPort(sock_fd); + +// If the local port number is 0, indicating an error in getting the port +if (*boundPort == 0) { + // Return an error code to indicate failure return RPC_ERROR; - } - return RPC_OK; } +// Return a success code to indicate successful execution +return RPC_OK; + +// Define a function named "Listen" that takes a constant reference to a string as a parameter and returns an integer int SocketOperation::Listen(const std::string &url) { + + // Declare and initialize an integer variable named "listenFd" with a value of 0 int listenFd = 0; + + // Declare a variable named "addr" of type SocketAddress SocketAddress addr; + // Check if the function GetSockAddr returns false (indicating failure) if (!GetSockAddr(url, &addr)) { + // If it returns false, return -1 to indicate an error return -1; } - // create server socket - listenFd = CreateSocket(addr.sa.sa_family); - if (listenFd < 0) { +// Create a server socket using the specified address family +listenFd = CreateSocket(addr.sa.sa_family); + +// Check if the socket creation was successful +if (listenFd < 0) { + // If the socket creation failed, log an error message with the URL and return -1 MS_LOG(ERROR) << "Failed to create socket, url: " << url.c_str(); return -1; - } +} - // bind + // Call the bind function to associate the socket referred to by listenFd with the address and port specified by addr if (::bind(listenFd, (struct sockaddr *)&addr, sizeof(SocketAddress))) { MS_LOG(ERROR) << "Failed to call bind, url: " << url.c_str(); + + // If the bind function fails, log an error message indicating the failure and the URL that was being bound + // Close the socket referred to by listenFd if (close(listenFd) != 0) { MS_LOG(EXCEPTION) << "Failed to close fd:" << listenFd; } + + // Return -1 to indicate an error occurred during the bind operation return -1; } - // listen + // Call the listen function to start listening for incoming connections on the specified socket if (::listen(listenFd, SOCKET_LISTEN_BACKLOG)) { + // If the listen function fails, log an error message with the relevant information MS_LOG(ERROR) << "Failed to call listen, fd: " << listenFd << ", errno: " << errno << ", url: " << url.c_str(); + + // Close the socket file descriptor if (close(listenFd) != 0) { + // If closing the socket fails, log an exception MS_LOG(EXCEPTION) << "Failed to close fd:" << listenFd; } + + // Return -1 to indicate an error occurred return -1; } + + // Return the socket file descriptor to indicate successful listening return listenFd; } +// Define the Accept function of the SocketOperation class, which takes a socket file descriptor as a parameter int SocketOperation::Accept(int sock_fd) { + + // Create a SocketAddress object named storage to store the client's address information SocketAddress storage; + + // Define a socklen_t variable named length and set it to the size of the storage object socklen_t length = sizeof(storage); - // accept connection + // Accept a connection on the given socket file descriptor (sock_fd) auto acceptFd = ::accept4(sock_fd, (struct sockaddr *)&storage, &length, SOCK_NONBLOCK | SOCK_CLOEXEC); + + // If the accept call fails (returns a negative value), log an error message and return the error code if (acceptFd < 0) { MS_LOG(ERROR) << "Failed to call accept, errno: " << errno << ", server: " << sock_fd; return acceptFd; } + + // If the accept call succeeds, set socket options for the accepted socket if (SetSocketOptions(acceptFd) < 0) { MS_LOG(ERROR) << "Failed to set socket options for accepted socket: " << acceptFd; } + + // Return the accepted socket file descriptor return acceptFd; } } // namespace rpc } // namespace distributed -} // namespace mindspore +} // namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/rpc/tcp/tcp_client.cc b/mindspore/ccsrc/distributed/rpc/tcp/tcp_client.cc index 02ea01039e5..805ee4d7d26 100644 --- a/mindspore/ccsrc/distributed/rpc/tcp/tcp_client.cc +++ b/mindspore/ccsrc/distributed/rpc/tcp/tcp_client.cc @@ -16,6 +16,7 @@ #include "distributed/rpc/tcp/tcp_client.h" +// Start of the namespace "mindspore" namespace mindspore { namespace distributed { namespace rpc { @@ -31,57 +32,93 @@ bool TCPClient::Initialize() { return rt; } +// This function is a member function of the TCPClient class void TCPClient::Finalize() { + // Check if the tcp_comm_ object is not null if (tcp_comm_ != nullptr) { + // Call the Finalize() function of the tcp_comm_ object tcp_comm_->Finalize(); + // Reset the tcp_comm_ object, effectively destroying it tcp_comm_.reset(); + // Set tcp_comm_ to nullptr to indicate that it is no longer pointing to a valid object tcp_comm_ = nullptr; } } +// Function to establish a TCP connection with the specified destination URL and timeout bool TCPClient::Connect(const std::string &dst_url, size_t timeout_in_sec) { - bool rt = false; - tcp_comm_->Connect(dst_url); + bool rt = false; // Initialize the return value to false + tcp_comm_->Connect(dst_url); // Call the Connect function of the tcp_comm_ object to establish the connection +} + // Convert the timeout in seconds to milliseconds by multiplying it by 1000 size_t timeout_in_ms = timeout_in_sec * 1000; + + // Set the sleep duration in milliseconds to 100 size_t sleep_in_ms = 100; + + // Convert the sleep duration in milliseconds to microseconds by multiplying it by 1000 useconds_t sleep_in_us = 100000; + // A while loop that runs indefinitely until a certain condition is met while (true) { + // Check if the TCP communication is connected to the destination URL if (tcp_comm_->IsConnected(dst_url)) { + // If connected, set the return value to true rt = true; + // Break out of the loop break; } + // Check if the remaining timeout is greater than the sleep duration if (timeout_in_ms > sleep_in_ms) { + // If true, subtract the sleep duration from the remaining timeout timeout_in_ms -= sleep_in_ms; } else { + // If false, break out of the loop break; } + // Sleep for the specified duration in microseconds (void)usleep(sleep_in_us); } + // Return the value of rt, indicating whether the connection was successful or not return rt; } +// Function to disconnect from a TCP server bool TCPClient::Disconnect(const std::string &dst_url, size_t timeout_in_sec) { - bool rt = false; - tcp_comm_->Disconnect(dst_url); + bool rt = false; // Initialize the return value to false + tcp_comm_->Disconnect(dst_url); // Call the Disconnect function of the tcp_comm_ object with the provided destination URL + // Convert the timeout in seconds to milliseconds by multiplying it by 1000 size_t timeout_in_ms = timeout_in_sec * 1000; + + // Set the sleep duration in milliseconds to 100 size_t sleep_in_ms = 100; + + // Convert the sleep duration in milliseconds to microseconds by multiplying it by 1000 useconds_t sleep_in_us = 100000; + // Enter an infinite loop using while(true) while (true) { + // Check if the TCP communication is not connected to the destination URL if (!tcp_comm_->IsConnected(dst_url)) { + // Set the return value to true rt = true; + // Break out of the loop break; } + // Check if the timeout in milliseconds is greater than the sleep in milliseconds if (timeout_in_ms > sleep_in_ms) { + // Subtract the sleep in milliseconds from the timeout in milliseconds timeout_in_ms -= sleep_in_ms; } else { + // If the timeout in milliseconds is less than or equal to the sleep in milliseconds, break out of the loop break; } + // Sleep for the specified amount of time in microseconds (void)usleep(sleep_in_us); } + // Return the value of rt return rt; } diff --git a/mindspore/ccsrc/distributed/rpc/tcp/tcp_comm.cc b/mindspore/ccsrc/distributed/rpc/tcp/tcp_comm.cc index 17355b206e4..518373436b2 100644 --- a/mindspore/ccsrc/distributed/rpc/tcp/tcp_comm.cc +++ b/mindspore/ccsrc/distributed/rpc/tcp/tcp_comm.cc @@ -14,229 +14,481 @@ * limitations under the License. */ -#include "distributed/rpc/tcp/tcp_comm.h" +// Include the header file "distributed/rpc/tcp/tcp_comm.h" which contains the necessary declarations for TCP communication in a distributed RPC system. +// Include the header for mutex, which provides synchronization primitives for protecting shared data from concurrent access #include + +// Include the header for utility, which provides various utility components and functions #include + +// Include the header for memory, which provides facilities for managing dynamic memory allocation and deallocation #include +// Include the header file for the actor aid module #include "actor/aid.h" + +// Include the header file for the constants used in the TCP implementation of distributed RPC #include "distributed/rpc/tcp/constants.h" + +// Include the header file for the TCP socket operation module #include "distributed/rpc/tcp/tcp_socket_operation.h" +// Start of the "mindspore" namespace namespace mindspore { -namespace distributed { -namespace rpc { -void DoDisconnect(int fd, Connection *conn, uint32_t error, int soError) { - if (LOG_CHECK_EVERY_N()) { - MS_LOG(INFO) << "Failed to call connect, fd: " << fd << ", to: " << conn->destination.c_str() - << ", events: " << error << ", errno: " << soError; - } + // Start of the "distributed" namespace + namespace distributed { + // Start of the "rpc" namespace + namespace rpc { + + // Definition of the function "DoDisconnect" which takes in parameters: + // - an integer "fd" + // - a pointer to a "Connection" object "conn" + // - an unsigned integer "error" + // - an integer "soError" + void DoDisconnect(int fd, Connection *conn, uint32_t error, int soError) { + + // Check if the LOG_CHECK_EVERY_N() condition is true + if (LOG_CHECK_EVERY_N()) { + + // Log an informational message using the MS_LOG macro, printing the values of: + // - "fd" + // - "conn->destination.c_str()" + // - "error" + // - "soError" + MS_LOG(INFO) << "Failed to call connect, fd: " << fd << ", to: " << conn->destination.c_str() + << ", events: " << error << ", errno: " << soError; + } + } + + } // End of the "rpc" namespace + } // End of the "distributed" namespace +} // End of the "mindspore" namespace - conn->state = ConnectionState::kDisconnecting; - conn->error_code = soError; - conn->event_callback(conn); - return; +// Set the state of the connection to "Disconnecting" +conn->state = ConnectionState::kDisconnecting; + +// Set the error code of the connection to the provided value +conn->error_code = soError; + +// Call the event callback function associated with the connection, passing the connection object as an argument +conn->event_callback(conn); + +// Return from the current function and terminate the execution of the code block +return; + +// This function is an event handler for a connected socket +void ConnectedEventHandler(int fd, uint32_t events, void *context) { + + // Extract the error events from the received events + uint32_t error = events & (EPOLLERR | EPOLLHUP | EPOLLRDHUP); + + // Initialize a variable to store the socket error + int soError = 0; + + // Cast the context pointer to a Connection object pointer + Connection *conn = reinterpret_cast(context); + + // Call the ConnEstablishedEventHandler of the socket operation associated with the connection + conn->socket_operation->ConnEstablishedEventHandler(context); + + // Check if the connection is in the disconnecting state + if (conn->state == ConnectionState::kDisconnecting) { + + // Perform the necessary disconnection operations + DoDisconnect(fd, conn, error, soError); + + // Return from the function + return; + } + // Check if the connection is not in the connected state + else if (conn->state != ConnectionState::kConnected) { + + // Return from the function + return; + } } -void ConnectedEventHandler(int fd, uint32_t events, void *context) { - uint32_t error = events & (EPOLLERR | EPOLLHUP | EPOLLRDHUP); - int soError = 0; - Connection *conn = reinterpret_cast(context); - conn->socket_operation->ConnEstablishedEventHandler(context); - if (conn->state == ConnectionState::kDisconnecting) { + // Check if the connection failed to reconnect the source socket + if (!conn->ReconnectSourceSocket(fd, events, &soError, error)) { + // If the reconnection failed, call the DoDisconnect function to handle disconnection DoDisconnect(fd, conn, error, soError); - return; - } else if (conn->state != ConnectionState::kConnected) { + // Return to exit the function return; } - if (!conn->ReconnectSourceSocket(fd, events, &soError, error)) { - DoDisconnect(fd, conn, error, soError); - return; - } + // Check if there is a write callback function assigned to the connection if (conn->write_callback) { + // If there is a write callback function, call it with the connection as the argument conn->write_callback(conn); } + + // Return to exit the function return; } +// Function to handle the "accept" event on a server socket void OnAccept(int server, uint32_t events, void *arg) { + + // Check if the event is an error event (EPOLLHUP or EPOLLERR) if (events & (EPOLLHUP | EPOLLERR)) { + + // Log an error message indicating the invalid error event MS_LOG(ERROR) << "Invalid error event, server fd: " << server << ", events: " << events; + + // Return from the function return; } + + // Cast the argument to a TCPComm pointer TCPComm *tcpmgr = reinterpret_cast(arg); + + // Check if the recv_event_loop_ member of the TCPComm object is null if (tcpmgr->recv_event_loop_ == nullptr) { + + // Log an error message indicating that the EventLoop is null MS_LOG(ERROR) << "EventLoop is null, server fd: " << server << ", events: " << events; + + // Return from the function return; } - // accept connection + // Accept a connection on the server socket and store the returned file descriptor in acceptFd auto acceptFd = SocketOperation::Accept(server); + + // Check if the accept operation was successful by comparing the returned file descriptor with -1 if (acceptFd < 0) { + // If the accept operation failed, log an error message with the server file descriptor and events MS_LOG(ERROR) << "Failed to call accept, server fd: " << server << ", events: " << events; + + // Return from the function as there was an error return; } - Connection *conn = new (std::nothrow) Connection(); - if (conn == nullptr) { +// Create a new Connection object using the 'new' operator, and assign its address to the 'conn' pointer +// Use 'std::nothrow' to prevent throwing an exception if memory allocation fails +Connection *conn = new (std::nothrow) Connection(); + +// Check if the memory allocation was successful by comparing the 'conn' pointer to nullptr +if (conn == nullptr) { + // If memory allocation failed, log an error message with details about the failure MS_LOG(ERROR) << "Failed to create new connection, server fd:" << server << ", events: " << events << ", accept fd: " << acceptFd; + + // Close the file descriptor 'acceptFd' using the 'close' function, and check if it was successful if (close(acceptFd) != 0) { - MS_LOG(ERROR) << "Failed to close fd: " << acceptFd; + // If closing the file descriptor failed, log an error message + MS_LOG(ERROR) << "Failed to close fd: " << acceptFd; } + + // Return from the current function, as we cannot proceed without a valid connection return; - } - - // init metrics - conn->send_metrics = new (std::nothrow) SendMetrics(); - if (conn->send_metrics == nullptr) { - MS_LOG(ERROR) << "Failed to create connection metrics, server fd: " << server << ", events: " << events - << ", accept fd: " << acceptFd; - if (close(acceptFd) != 0) { - MS_LOG(ERROR) << "Failed to close fd: " << acceptFd; - } - delete conn; - return; - } - - conn->socket_fd = acceptFd; - conn->source = tcpmgr->url_; - conn->peer = SocketOperation::GetPeer(acceptFd); - - conn->is_remote = true; - conn->recv_event_loop = tcpmgr->recv_event_loop_; - conn->send_event_loop = tcpmgr->send_event_loop_; - - conn->conn_mutex = tcpmgr->conn_mutex_; - conn->message_handler = tcpmgr->message_handler_; - - conn->event_callback = TCPComm::EventCallBack; - conn->write_callback = TCPComm::WriteCallBack; - conn->read_callback = TCPComm::ReadCallBack; - - int retval = conn->Initialize(); - if (retval != RPC_OK) { - MS_LOG(ERROR) << "Failed to add accept fd event, server fd: " << server << ", events: " << events - << ", accept fd: " << acceptFd; - if (close(acceptFd) != 0) { - MS_LOG(ERROR) << "Failed to close fd: " << acceptFd; - } - acceptFd = -1; - delete conn->send_metrics; - delete conn; - return; - } - tcpmgr->conn_pool_->AddConnection(conn); } -void TCPComm::SetMessageHandler(const MessageHandler &handler) { message_handler_ = handler; } + // Initialize the send_metrics member of the conn object with a new instance of the SendMetrics class + conn->send_metrics = new (std::nothrow) SendMetrics(); -bool TCPComm::Initialize() { + // Check if the allocation of send_metrics failed (i.e., if it is nullptr) + if (conn->send_metrics == nullptr) { + // Log an error message indicating the failure to create connection metrics, along with relevant information + MS_LOG(ERROR) << "Failed to create connection metrics, server fd: " << server << ", events: " << events + << ", accept fd: " << acceptFd; + + // Close the acceptFd file descriptor + if (close(acceptFd) != 0) { + // Log an error message if the close operation fails + MS_LOG(ERROR) << "Failed to close fd: " << acceptFd; + } + + // Delete the conn object to free up memory + delete conn; + + // Return from the function, as there is nothing more to be done + return; + } + +// Assign the accepted socket file descriptor to the `socket_fd` member of the `conn` object +conn->socket_fd = acceptFd; + +// Assign the URL of the TCP manager to the `source` member of the `conn` object +conn->source = tcpmgr->url_; + +// Use the `GetPeer` function from the `SocketOperation` class to get the peer address of the accepted socket file descriptor +// Assign the peer address to the `peer` member of the `conn` object + +// Set the `is_remote` member of the `conn` object to `true` +conn->is_remote = true; + +// Assign the `recv_event_loop` member of the `conn` object with the value of `tcpmgr`'s `recv_event_loop_` member +conn->recv_event_loop = tcpmgr->recv_event_loop_; + +// Assign the `send_event_loop` member of the `conn` object with the value of `tcpmgr`'s `send_event_loop_` member + +// Assign the value of tcpmgr's conn_mutex_ to conn's conn_mutex +conn->conn_mutex = tcpmgr->conn_mutex_; + +// Assign the value of tcpmgr's message_handler_ to conn's message_handler +conn->message_handler = tcpmgr->message_handler_; + +// Assign the event callback function of the TCPComm class to the event_callback member of the conn object +conn->event_callback = TCPComm::EventCallBack; + +// Assign the write callback function of the TCPComm class to the write_callback member of the conn object +conn->write_callback = TCPComm::WriteCallBack; + +// Assign the read callback function of the TCPComm class to the read_callback member of the conn object +conn->read_callback = TCPComm::ReadCallBack; + + // Call the Initialize function of the conn object and store the return value in retval + int retval = conn->Initialize(); + + // Check if retval is not equal to RPC_OK + if (retval != RPC_OK) { + // Print an error message indicating the failure to add accept fd event, along with relevant information + MS_LOG(ERROR) << "Failed to add accept fd event, server fd: " << server << ", events: " << events + << ", accept fd: " << acceptFd; + + // Check if closing the acceptFd fails + if (close(acceptFd) != 0) { + // Print an error message indicating the failure to close the fd + MS_LOG(ERROR) << "Failed to close fd: " << acceptFd; + } + + // Set acceptFd to -1 to indicate that it is no longer a valid file descriptor + acceptFd = -1; + + // Delete the send_metrics object pointed to by conn->send_metrics + delete conn->send_metrics; + + // Delete the conn object + delete conn; + + // Return from the function + return; + } + + // Add the conn object to the connection pool of the tcpmgr object + tcpmgr->conn_pool_->AddConnection(conn); + +// Define the function SetMessageHandler of the TCPComm class, which takes a reference to a MessageHandler object as a parameter +void TCPComm::SetMessageHandler(const MessageHandler &handler) { + + // Assign the value of the parameter to the message_handler_ member variable of the TCPComm class + message_handler_ = handler; +} + +// Initialize the TCP communication by creating a shared pointer to a ConnectionPool object conn_pool_ = std::make_shared(); + + // Check if the conn_pool_ pointer is not null MS_EXCEPTION_IF_NULL(conn_pool_); +// Create a shared pointer to a mutex object using std::make_shared conn_mutex_ = std::make_shared(); - MS_EXCEPTION_IF_NULL(conn_mutex_); - recv_event_loop_ = new (std::nothrow) EventLoop(); - if (recv_event_loop_ == nullptr) { + // Check if the conn_mutex_ pointer is null + MS_EXCEPTION_IF_NULL(conn_mutex_); + + // If the conn_mutex_ pointer is null, throw an exception (MS_EXCEPTION_IF_NULL is a macro that throws an exception if the given pointer is null) + +// Create a new EventLoop object and assign it to the recv_event_loop_ pointer +recv_event_loop_ = new (std::nothrow) EventLoop(); + +// Check if the allocation was successful by comparing recv_event_loop_ to nullptr +if (recv_event_loop_ == nullptr) { + // If the allocation failed, print an error message using the MS_LOG macro and return false MS_LOG(ERROR) << "Failed to create recv evLoop."; return false; - } +} - bool ok = recv_event_loop_->Initialize(TCP_RECV_EVLOOP_THREADNAME); - if (!ok) { +// Initialize the recv_event_loop_ object and store the result in the boolean variable 'ok' +bool ok = recv_event_loop_->Initialize(TCP_RECV_EVLOOP_THREADNAME); + +// Check if the initialization was successful +if (!ok) { + // If initialization failed, print an error message using the MS_LOG macro MS_LOG(ERROR) << "Failed to init recv evLoop"; + + // Delete the recv_event_loop_ object to free up memory delete recv_event_loop_; recv_event_loop_ = nullptr; - return false; - } - send_event_loop_ = new (std::nothrow) EventLoop(); - if (send_event_loop_ == nullptr) { + // Return false to indicate failure + return false; +} + +// Create a new EventLoop object and assign it to the send_event_loop_ pointer +send_event_loop_ = new (std::nothrow) EventLoop(); + +// Check if the memory allocation for send_event_loop_ was successful +if (send_event_loop_ == nullptr) { + // If not, log an error message and clean up resources MS_LOG(ERROR) << "Failed to create send evLoop."; delete recv_event_loop_; recv_event_loop_ = nullptr; return false; - } - ok = send_event_loop_->Initialize(TCP_SEND_EVLOOP_THREADNAME); - if (!ok) { +} + +// Initialize the send_event_loop_ object with a thread name +ok = send_event_loop_->Initialize(TCP_SEND_EVLOOP_THREADNAME); + +// Check if the initialization was successful +if (!ok) { + // If not, log an error message and clean up resources MS_LOG(ERROR) << "Failed to init send evLoop"; delete recv_event_loop_; recv_event_loop_ = nullptr; delete send_event_loop_; send_event_loop_ = nullptr; return false; - } - - return true; } +// Return true to indicate successful program termination +return true; + +// StartServerSocket function of the TCPComm class, used to start a server socket bool TCPComm::StartServerSocket(const std::string &url) { + + // Call the Listen function of the SocketOperation class to create a server socket and get the file descriptor server_fd_ = SocketOperation::Listen(url); + + // If the file descriptor is less than 0, indicating an error in creating the server socket if (server_fd_ < 0) { + + // Log an error message indicating the failure to create the server socket MS_LOG(ERROR) << "Failed to call socket listen, url: " << url.c_str(); + + // Return false to indicate the failure to start the server socket return false; } + + // Set the url_ member variable to the provided url url_ = url; + + // Find the index of the URL_PROTOCOL_IP_SEPARATOR in the url string size_t index = url.find(URL_PROTOCOL_IP_SEPARATOR); + + // If the URL_PROTOCOL_IP_SEPARATOR is found in the url string if (index != std::string::npos) { + + // Set the url_ member variable to the substring of url starting from the position after the URL_PROTOCOL_IP_SEPARATOR url_ = url.substr(index + sizeof(URL_PROTOCOL_IP_SEPARATOR) - 1); } + // Continue with the rest of the function... + // Register read event callback for server socket int retval = recv_event_loop_->SetEventHandler(server_fd_, EPOLLIN | EPOLLHUP | EPOLLERR, OnAccept, reinterpret_cast(this)); + // Check if the registration was successful if (retval != RPC_OK) { + // If not successful, log an error message and return false MS_LOG(ERROR) << "Failed to add server event, url: " << url.c_str(); return false; } + // If successful, log an info message with the server file descriptor and URL MS_LOG(INFO) << "Start server succ, fd: " << server_fd_ << ", url: " << url.c_str(); + // Return true to indicate successful server start return true; } +// StartServerSocket function of the TCPComm class bool TCPComm::StartServerSocket() { + + // Get the local IP address using the SocketOperation class auto ip = SocketOperation::GetLocalIP(); - // The port 0 means that the port will be allocated randomly by the os system. + + // Set the URL to the local IP address followed by ":0", where 0 indicates that the port will be allocated randomly by the OS system auto url = ip + ":0"; + + // Call the overloaded StartServerSocket function with the generated URL return StartServerSocket(url); } -int TCPComm::GetServerFd() const { return server_fd_; } +// Define a member function named GetServerFd() of the class TCPComm +// This function returns the value of the private member variable server_fd_ +int TCPComm::GetServerFd() const { + return server_fd_; +} +// Define a function named ReadCallBack that takes a void pointer as a parameter void TCPComm::ReadCallBack(void *connection) { + + // Define a constant integer named max_recv_count and assign it a value of 3 const int max_recv_count = 3; + + // Cast the void pointer to a pointer of type Connection and assign it to a variable named conn Connection *conn = reinterpret_cast(connection); + + // Define an integer variable named count and assign it a value of 0 int count = 0; + + // Define an integer variable named retval and assign it a value of 0 int retval = 0; + + // Execute the following block of code repeatedly until retval is not greater than 0 or count is equal to max_recv_count do { + // Call the ReceiveMessage function and pass conn as an argument, assign the return value to retval retval = ReceiveMessage(conn); + + // Increment the value of count by 1 ++count; } while (retval > 0 && count < max_recv_count); - - return; } -void TCPComm::EventCallBack(void *connection) { - Connection *conn = reinterpret_cast(connection); +// Since the return type of the main function is int, it should return an integer value. +// In this case, the return statement is missing the integer value to be returned. +// To fix this, we should return 0 to indicate successful program termination. +return 0; +} - if (conn->state == ConnectionState::kConnected) { +// Define the EventCallBack function of the TCPComm class, which takes a void pointer as a parameter + +// Cast the void pointer to a Connection pointer using reinterpret_cast +Connection *conn = reinterpret_cast(connection); + +// Check if the connection state is "Connected" +if (conn->state == ConnectionState::kConnected) { + + // Lock the connection mutex to ensure exclusive access to the connection object conn->conn_mutex->lock(); + + // Call the Flush() function on the connection object and ignore the return value (void)conn->Flush(); + + // Unlock the connection mutex to release the exclusive access conn->conn_mutex->unlock(); - } else if (conn->state == ConnectionState::kDisconnecting) { + +} +// If the connection state is "Disconnecting" +else if (conn->state == ConnectionState::kDisconnecting) { + + // Lock the connection mutex to ensure exclusive access to the connection object conn->conn_mutex->lock(); + + // Unlock the connection mutex to release the exclusive access conn->conn_mutex->unlock(); - } + } +// End of the if-else block + +// A member function of the TCPComm class that serves as a callback for writing data void TCPComm::WriteCallBack(void *connection) { + + // Cast the void pointer to a Connection pointer Connection *conn = reinterpret_cast(connection); + + // Check if the connection state is connected if (conn->state == ConnectionState::kConnected) { + + // Lock the connection mutex to ensure thread safety conn->conn_mutex->lock(); + + // Call the Flush function of the connection object to send any pending data (void)conn->Flush(); + + // Unlock the connection mutex conn->conn_mutex->unlock(); } } @@ -266,30 +518,44 @@ int TCPComm::DoConnect(Connection *conn, const struct sockaddr *sa, socklen_t sa int retval = 0; uint16_t localPort = 0; + // Call the Connect function from the SocketOperation class to establish a connection using the provided socket file descriptor, socket address, socket address length, and local port retval = SocketOperation::Connect(conn->socket_fd, sa, saLen, &localPort); + + // If the return value is not equal to RPC_OK (indicating a successful connection), return RPC_ERROR if (retval != RPC_OK) { return RPC_ERROR; } - // Init connection metrics. + // Initialize connection metrics. + + // Check if the send_metrics member of the conn object is nullptr if (conn->send_metrics == nullptr) { + // If it is nullptr, allocate memory for a new SendMetrics object using the new operator conn->send_metrics = new (std::nothrow) SendMetrics(); + + // Check if the allocation was successful if (conn->send_metrics == nullptr) { + // If the allocation failed, return an error code indicating failure return RPC_ERROR; } } // Add the socket of this connection to epoll. - retval = SetConnectedHandler(conn); + retval = SetConnectedHandler(conn); // Call the SetConnectedHandler function to add the socket of the connection to epoll + + // Check if the return value is not RPC_OK if (retval != RPC_OK) { + // Check if the send_metrics member of the connection is not nullptr if (conn->send_metrics != nullptr) { + // Delete the send_metrics object and set it to nullptr delete conn->send_metrics; conn->send_metrics = nullptr; } + // Return RPC_ERROR to indicate an error occurred return RPC_ERROR; } + // Return RPC_OK to indicate successful execution return RPC_OK; -} /* static method */ void TCPComm::DropMessage(MessageBase *msg) { @@ -298,183 +564,353 @@ void TCPComm::DropMessage(MessageBase *msg) { ptr = nullptr; } -ssize_t TCPComm::Send(MessageBase *msg, bool sync) { - auto task = [msg, this] { +// The Send function of the TCPComm class, responsible for sending a message over TCP +// Takes a MessageBase pointer and a boolean flag indicating whether to send synchronously or asynchronously + +// Define a lambda function called "task" that captures the "msg" and "this" pointer +auto task = [msg, this] { + + // Create a lock_guard object to lock the connection mutex std::lock_guard lock(*conn_mutex_); - // Search connection by the target address + + // Search for a connection in the connection pool based on the target address of the message Connection *conn = conn_pool_->FindConnection(msg->to.Url()); + + // If no connection is found if (conn == nullptr) { - MS_LOG(ERROR) << "Can not found remote link and send fail name: " << msg->name.c_str() - << ", from: " << msg->from.Url().c_str() << ", to: " << msg->to.Url().c_str(); - DropMessage(msg); - int error_no = -1; - return error_no; + + // Log an error message indicating that the remote link was not found and send failed + MS_LOG(ERROR) << "Can not found remote link and send fail name: " << msg->name.c_str() + << ", from: " << msg->from.Url().c_str() << ", to: " << msg->to.Url().c_str(); + + // Drop the message (discard it) + DropMessage(msg); + + // Set the error number to -1 + int error_no = -1; + + // Return the error number + return error_no; } + // Check if the size of the send message queue is greater than or equal to the maximum allowed length if (conn->send_message_queue.size() >= SENDMSG_QUEUELEN) { + + // Log a warning message indicating that the message queue is full and provide additional information about the dropped message MS_LOG(WARNING) << "The message queue is full(max len:" << SENDMSG_QUEUELEN << ") and the name of dropped message is: " << msg->name.c_str() << ", fd: " << conn->socket_fd << ", to: " << conn->destination.c_str(); + + // Call the DropMessage function to handle the dropped message DropMessage(msg); + + // Set the error number to -1 int error_no = -1; + + // Return the error number return error_no; } + // Check if the connection state is not equal to ConnectionState::kConnected if (conn->state != ConnectionState::kConnected) { + + // Log a warning message with the invalid connection state, dropped message name, socket file descriptor, and destination MS_LOG(WARNING) << "Invalid connection state " << conn->state << " and the name of dropped message is: " << msg->name.c_str() << ", fd: " << conn->socket_fd << ", to: " << conn->destination.c_str(); + + // Drop the message DropMessage(msg); + + // Set the error number to -1 int error_no = -1; + + // Return the error number return error_no; } + // Check if the total send length of the connection is 0 if (conn->total_send_len == 0) { - conn->FillSendMessage(msg, url_, false); + // If it is 0, call the FillSendMessage function of the connection object, passing the message and URL as arguments + conn->FillSendMessage(msg, url_, false); } else { - (void)conn->send_message_queue.emplace(msg); + // If it is not 0, add the message to the send message queue of the connection object + (void)conn->send_message_queue.emplace(msg); } + + // Call the Flush function of the connection object and return its result return conn->Flush(); - }; - if (sync) { +}; + +// Check if the sync flag is true +if (sync) { + // If it is true, call the task function and return its result return task(); - } else { +} else { + // If it is false, add the task function to the send event loop's task queue send_event_loop_->AddTask(task); + // Return true to indicate that the task has been added successfully return true; - } } +// Define the Connect function of the TCPComm class, which takes a const reference to a string as input parameter named dst_url void TCPComm::Connect(const std::string &dst_url) { - (void)recv_event_loop_->AddTask([dst_url, this] { + + // Add a task to the recv_event_loop_ using the AddTask method, which takes a lambda function as input + recv_event_loop_->AddTask([dst_url, this] { + + // Create a lock_guard object to lock the conn_mutex_ mutex, ensuring exclusive access to the critical section std::lock_guard lock(*conn_mutex_); - // Search connection by the target address - Connection *conn = conn_pool_->FindConnection(dst_url); +// Search for a connection in the connection pool based on the target address +// Create a pointer to a Connection object and assign the result of the search to it +Connection *conn = conn_pool_->FindConnection(dst_url); + // Check if the connection is null if (conn == nullptr) { + + // Log an informational message indicating that the link destination cannot be found MS_LOG(INFO) << "Can not found link destination: " << dst_url; + + // Create a new connection object using the "nothrow" variant of the new operator conn = new (std::nothrow) Connection(); + + // Check if the new connection object is still null if (conn == nullptr) { + + // Log an error message indicating that the new connection object creation failed and the link destination cannot be set MS_LOG(ERROR) << "Failed to create new connection and link fail destination: " << dst_url; + + // Return false to indicate failure return false; } + + // Set the source and destination of the connection object conn->source = url_; conn->destination = dst_url; + } + // Assign the value of `recv_event_loop_` to the `recv_event_loop` member variable of `conn` conn->recv_event_loop = this->recv_event_loop_; + + // Assign the value of `send_event_loop_` to the `send_event_loop` member variable of `conn` conn->send_event_loop = this->send_event_loop_; + + // Assign the value of `conn_mutex_` to the `conn_mutex` member variable of `conn` conn->conn_mutex = conn_mutex_; + + // Assign the value of `message_handler_` to the `message_handler` member variable of `conn` conn->message_handler = message_handler_; - conn->InitSocketOperation(); - // Create the client socket. - SocketAddress addr; - if (!SocketOperation::GetSockAddr(dst_url, &addr)) { - MS_LOG(ERROR) << "Failed to get socket address to dest url " << dst_url; - return false; - } - int sock_fd = SocketOperation::CreateSocket(addr.sa.sa_family); - if (sock_fd < 0) { - MS_LOG(ERROR) << "Failed to create client tcp socket to dest url " << dst_url; - return false; - } + // Call the `InitSocketOperation` function of `conn` to initialize socket operations +// Create a SocketAddress object to store the destination address +SocketAddress addr; + +// Use the SocketOperation class to get the socket address for the destination URL +// If the operation fails, log an error message and return false +if (!SocketOperation::GetSockAddr(dst_url, &addr)) { + MS_LOG(ERROR) << "Failed to get socket address to dest url " << dst_url; + return false; +} + +// Create a socket file descriptor using the address family of the destination address +// If the creation fails, log an error message and return false +int sock_fd = SocketOperation::CreateSocket(addr.sa.sa_family); +if (sock_fd < 0) { + MS_LOG(ERROR) << "Failed to create client tcp socket to dest url " << dst_url; + return false; +} + + // Assign the socket file descriptor to the socket_fd member of the conn object conn->socket_fd = sock_fd; + + // Assign the EventCallBack function from the TCPComm class to the event_callback member of the conn object conn->event_callback = TCPComm::EventCallBack; + + // Assign the WriteCallBack function from the TCPComm class to the write_callback member of the conn object conn->write_callback = TCPComm::WriteCallBack; + + // Assign the ReadCallBack function from the TCPComm class to the read_callback member of the conn object conn->read_callback = TCPComm::ReadCallBack; + // Call the DoConnect function from the TCPComm class, passing in the connection object, the address structure, and its size int ret = TCPComm::DoConnect(conn, (struct sockaddr *)&addr, sizeof(addr)); + + // If the return value is less than 0, it means the connection failed if (ret < 0) { + // Log an error message indicating the failure and the destination URL MS_LOG(ERROR) << "Failed to do connect and link fail destination: " << dst_url; + + // Check if the socket operation object is not null if (conn->socket_operation != nullptr) { + // Delete the socket operation object and set it to null delete conn->socket_operation; conn->socket_operation = nullptr; } + + // Delete the connection object delete conn; + + // Return false to indicate the failure return false; } + + // Add the connection to the connection pool conn_pool_->AddConnection(conn); } + + // Add the connection information to the connection pool conn_pool_->AddConnInfo(conn->socket_fd, dst_url, nullptr); + + // Log an information message indicating the successful connection to the destination URL MS_LOG(INFO) << "Connected to destination: " << dst_url; + + // Return true to indicate the success return true; }); } +// Check if a TCP connection is established to the specified destination URL bool TCPComm::IsConnected(const std::string &dst_url) { + + // Find the connection object from the connection pool based on the destination URL Connection *conn = conn_pool_->FindConnection(dst_url); + + // If a connection object is found and its state is "connected", return true if (conn != nullptr && conn->state == ConnectionState::kConnected) { return true; } + + // If no connection object is found or the connection state is not "connected", return false return false; } +// Disconnect function implementation for TCP communication bool TCPComm::Disconnect(const std::string &dst_url) { + + // Set the interval between retries to 100000 microseconds (0.1 seconds) int interval = 100000; + + // Set the maximum number of retries to 30 size_t retry = 30; + + // Continue retrying until there are no remaining tasks in the receive event loop, + // the send event loop, or the maximum number of retries is reached while (recv_event_loop_->RemainingTaskNum() != 0 && send_event_loop_->RemainingTaskNum() != 0 && retry > 0) { + + // Sleep for the specified interval before retrying usleep(interval); + + // Decrement the retry counter retry--; } + + // If there are still pending tasks in either the receive event loop or the send event loop, + // return false and log an error message if (recv_event_loop_->RemainingTaskNum() > 0 || send_event_loop_->RemainingTaskNum() > 0) { MS_LOG(ERROR) << "Failed to disconnect from url " << dst_url << ", because there are still pending tasks to be executed, please try later."; return false; } + + // Add a task to the receive event loop to delete the connection associated with the destination URL (void)recv_event_loop_->AddTask([dst_url, this] { std::lock_guard lock(*conn_mutex_); conn_pool_->DeleteConnection(dst_url); return true; }); + + // Return true to indicate successful disconnection return true; } +// Function to create a default connection object for TCP communication Connection *TCPComm::CreateDefaultConn(const std::string &to) { + + // Create a new connection object using the 'new' operator, and handle any memory allocation failures Connection *conn = new (std::nothrow) Connection(); if (conn == nullptr) { + // Log an error message if memory allocation fails, and return the nullptr MS_LOG(ERROR) << "Failed to create new connection and reconnect fail to: " << to.c_str(); return conn; } + + // Set the source and destination of the connection conn->source = url_.data(); conn->destination = to; + + // Set the receive and send event loops of the connection conn->recv_event_loop = this->recv_event_loop_; conn->send_event_loop = this->send_event_loop_; + + // Set the connection mutex, message handler, and initialize socket operations conn->conn_mutex = conn_mutex_; conn->message_handler = message_handler_; conn->InitSocketOperation(); + + // Return the created connection object return conn; } +// Definition of the `Finalize` function in the `TCPComm` class + void TCPComm::Finalize() { + + // Check if the `send_event_loop_` pointer is not null if (send_event_loop_ != nullptr) { + + // Log an informational message MS_LOG(INFO) << "Delete send event loop"; + + // Call the `Finalize` function of the `send_event_loop_` object send_event_loop_->Finalize(); + + // Delete the `send_event_loop_` object delete send_event_loop_; + + // Set the `send_event_loop_` pointer to null send_event_loop_ = nullptr; } +} + // Check if recv_event_loop_ is not a null pointer if (recv_event_loop_ != nullptr) { + + // Log an informational message indicating that recv event loop is being deleted MS_LOG(INFO) << "Delete recv event loop"; + + // Call the Finalize() function of recv_event_loop_ recv_event_loop_->Finalize(); + + // Delete the recv_event_loop_ object delete recv_event_loop_; + + // Set recv_event_loop_ to a null pointer recv_event_loop_ = nullptr; } - if (server_fd_ > 0) { - if (close(server_fd_) != 0) { - MS_LOG(ERROR) << "Failed to close fd: " << server_fd_; - } - server_fd_ = -1; - } +// Check if the server file descriptor is greater than 0 +if (server_fd_ > 0) { - if (conn_pool_ != nullptr) { - MS_LOG(INFO) << "Delete connection pool."; - conn_pool_->Finalize(); - conn_pool_.reset(); - conn_pool_ = nullptr; + // If the close operation on the server file descriptor returns a non-zero value + if (close(server_fd_) != 0) { + + // Log an error message indicating the failure to close the file descriptor + MS_LOG(ERROR) << "Failed to close fd: " << server_fd_; + } + + // Set the server file descriptor to -1 to indicate that it is closed + server_fd_ = -1; +} + + if (conn_pool_ != nullptr) { // Check if the connection pool is not null + MS_LOG(INFO) << "Delete connection pool."; // Log an informational message indicating that the connection pool is being deleted + conn_pool_->Finalize(); // Call the Finalize() function of the connection pool + conn_pool_.reset(); // Reset the connection pool smart pointer + conn_pool_ = nullptr; // Set the connection pool pointer to null } } } // namespace rpc } // namespace distributed -} // namespace mindspore +} // namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/rpc/tcp/tcp_server.cc b/mindspore/ccsrc/distributed/rpc/tcp/tcp_server.cc index 6b1c2706de3..88a29fc5a3a 100644 --- a/mindspore/ccsrc/distributed/rpc/tcp/tcp_server.cc +++ b/mindspore/ccsrc/distributed/rpc/tcp/tcp_server.cc @@ -14,52 +14,111 @@ * limitations under the License. */ -#include "distributed/rpc/tcp/tcp_server.h" +// Include the header file "distributed/rpc/tcp/tcp_server.h" which contains the declaration of the TCP server class. +// Start of the "mindspore" namespace namespace mindspore { -namespace distributed { -namespace rpc { -bool TCPServer::Initialize(const std::string &url) { return InitializeImpl(url); } -bool TCPServer::Initialize() { return InitializeImpl(""); } + // Start of the "distributed" namespace within the "mindspore" namespace + namespace distributed { + + // Start of the "rpc" namespace within the "distributed" namespace + namespace rpc { + + // Implementation of the "Initialize" function of the "TCPServer" class + bool TCPServer::Initialize(const std::string &url) { + + // Call the "InitializeImpl" function and return its result + return InitializeImpl(url); + } + } // End of the "rpc" namespace + } // End of the "distributed" namespace +} // End of the "mindspore" namespace + +// The Initialize function of the TCPServer class +bool TCPServer::Initialize() { + + // Call the InitializeImpl function with an empty string argument and return its result + return InitializeImpl(""); +} + +// Definition of the Finalize function in the TCPServer class void TCPServer::Finalize() { + + // Check if the tcp_comm_ object is not null if (tcp_comm_ != nullptr) { + + // Call the Finalize function of the tcp_comm_ object tcp_comm_->Finalize(); + + // Reset the tcp_comm_ object, releasing its resources tcp_comm_.reset(); + + // Set tcp_comm_ to nullptr to indicate that it is no longer pointing to a valid object tcp_comm_ = nullptr; } } -void TCPServer::SetMessageHandler(const MessageHandler &handler) { tcp_comm_->SetMessageHandler(handler); } +// Define the function SetMessageHandler of the TCPServer class +void TCPServer::SetMessageHandler(const MessageHandler &handler) { + // Call the SetMessageHandler function of the tcp_comm_ object, passing the provided handler as an argument + tcp_comm_->SetMessageHandler(handler); +} -std::string TCPServer::GetIP() const { return ip_; } +// Define a member function named GetIP() for the class TCPServer +std::string TCPServer::GetIP() const { -uint32_t TCPServer::GetPort() const { return port_; } + // Return the value of the private member variable ip_ + return ip_; +} -bool TCPServer::InitializeImpl(const std::string &url) { +// Define a member function named GetPort() of the TCPServer class that returns a uint32_t value +uint32_t TCPServer::GetPort() const { + // Return the value of the private member variable port_ + return port_; +} + +// Check if the TCP communication object is not initialized if (tcp_comm_ == nullptr) { + // Create a new TCP communication object using smart pointer and initialize it tcp_comm_ = std::make_unique(); MS_EXCEPTION_IF_NULL(tcp_comm_); + + // Call the Initialize function of the TCP communication object and store the return value in 'rt' bool rt = tcp_comm_->Initialize(); + + // If the initialization fails, throw an exception with an error message if (!rt) { MS_LOG(EXCEPTION) << "Failed to initialize tcp comm"; } + + // If the 'url' parameter is not empty, start the server socket with the provided URL if (url != "") { rt = tcp_comm_->StartServerSocket(url); ip_ = SocketOperation::GetIP(url); } else { + // If the 'url' parameter is empty, start the server socket with the local IP address rt = tcp_comm_->StartServerSocket(); ip_ = SocketOperation::GetLocalIP(); } + + // Get the file descriptor of the server socket and store it in 'server_fd' auto server_fd = tcp_comm_->GetServerFd(); + + // Get the port number associated with the server socket and store it in 'port_' port_ = SocketOperation::GetPort(server_fd); - return rt; - } else { - return true; - } -} -} // namespace rpc -} // namespace distributed -} // namespace mindspore + // If the condition is true + if (condition) { + // Return the value of rt + return rt; + } else { + // Return true + return true; + } +} // End of namespace rpc + +} // End of namespace distributed + +} // End of namespace mindspore \ No newline at end of file diff --git a/mindspore/ccsrc/distributed/rpc/tcp/tcp_socket_operation.cc b/mindspore/ccsrc/distributed/rpc/tcp/tcp_socket_operation.cc index e6f79cc1c4f..07767581a90 100644 --- a/mindspore/ccsrc/distributed/rpc/tcp/tcp_socket_operation.cc +++ b/mindspore/ccsrc/distributed/rpc/tcp/tcp_socket_operation.cc @@ -14,161 +14,308 @@ * limitations under the License. */ +// Include the header file "distributed/rpc/tcp/tcp_socket_operation.h" #include "distributed/rpc/tcp/tcp_socket_operation.h" +// Define the namespace "mindspore" namespace mindspore { -namespace distributed { -namespace rpc { -constexpr int EAGAIN_RETRY = 10240; + + // Define the nested namespace "distributed" within the "mindspore" namespace + namespace distributed { + + // Define the nested namespace "rpc" within the "distributed" namespace + namespace rpc { + + // Define a constant integer variable "EAGAIN_RETRY" with a value of 10240 + constexpr int EAGAIN_RETRY = 10240; + } + } +} + +// This function is a member function of the TCPSocketOperation class +// It receives data from a TCP socket without removing it from the receive buffer +// It takes a Connection object pointer, a buffer to store the received data, and the length of the buffer as parameters +// It returns the number of bytes received, or -1 if an error occurs ssize_t TCPSocketOperation::ReceivePeek(Connection *connection, char *recvBuf, uint32_t recvLen) { + + // Use the recv function to receive data from the socket specified by the socket file descriptor in the Connection object + // The received data is stored in the recvBuf buffer + // The recvLen parameter specifies the maximum number of bytes to receive + // The MSG_PEEK flag is used to peek at the data without removing it from the receive buffer + // The function returns the number of bytes received, or -1 if an error occurs return recv(connection->socket_fd, recvBuf, recvLen, MSG_PEEK); } -int TCPSocketOperation::Receive(Connection *connection, char *recvBuf, size_t totalRecvLen, size_t *recvLen) { - char *curRecvBuf = recvBuf; - int fd = connection->socket_fd; +// Declare a pointer variable curRecvBuf and initialize it with the address of recvBuf +char *curRecvBuf = recvBuf; + +// Get the socket file descriptor from the connection object and assign it to the variable fd +int fd = connection->socket_fd; *recvLen = 0; + // Continue receiving until the total received length matches the expected total length while (*recvLen != totalRecvLen) { + // Use the recv function to receive data from the file descriptor (fd) ssize_t retval = recv(fd, curRecvBuf, totalRecvLen - *recvLen, static_cast(0)); if (retval > 0) { + // If data is received successfully, update the received length and check if it matches the total length *recvLen += retval; if (*recvLen == totalRecvLen) { + // If the received length matches the total length, return IO_RW_OK to indicate successful read/write operation return IO_RW_OK; } + // Update the current receive buffer pointer to point to the next position curRecvBuf = curRecvBuf + retval; // Failed to receive message. } else if (retval < 0) { + // If recv returns a negative value, check the error code if (EAGAIN == errno) { + // If the error code is EAGAIN, return IO_RW_OK to indicate that the operation can be retried later return IO_RW_OK; } else if (ECONNRESET == errno || ECONNABORTED == errno || ENOTCONN == errno || EPIPE == errno) { + // If the error code indicates a connection reset, aborted, not connected, or broken pipe, + // update the connection's error code and return IO_RW_ERROR to indicate an error in read/write operation connection->error_code = errno; return IO_RW_ERROR; } else { + // For other error codes, return IO_RW_OK to indicate that the operation can be retried later return IO_RW_OK; } } else { + // Set the error code of the connection to the value of errno connection->error_code = errno; + + // Return IO_RW_ERROR to indicate an error occurred during read/write operation return IO_RW_ERROR; } } + + // If the function reaches this point, it means the read/write operation was successful return IO_RW_OK; } +// Implementation of the ReceiveMessage function in the TCPSocketOperation class + int TCPSocketOperation::ReceiveMessage(Connection *connection, struct msghdr *recvMsg, size_t totalRecvLen, size_t *recvLen) { - if (totalRecvLen == 0) { - return IO_RW_OK; - } + // Check if the totalRecvLen is 0 + if (totalRecvLen == 0) { + // If it is 0, return IO_RW_OK to indicate successful operation + return IO_RW_OK; + } + // If totalRecvLen is not 0, continue with the rest of the function + // ... +} + // While the total number of bytes received is less than the expected totalRecvLen while (*recvLen < totalRecvLen) { + + // Call the recvmsg function to receive data from the socket connection auto retval = recvmsg(connection->socket_fd, recvMsg, 0); + + // If the return value of recvmsg is greater than 0 (indicating successful reception of data) if (retval > 0) { + + // Increment the recvLen by the number of bytes received *recvLen += retval; + + // If the total number of bytes received is equal to the expected totalRecvLen if (*recvLen == totalRecvLen) { + + // Set the msg_iovlen of recvMsg to 0 (indicating that there are no more data to receive) recvMsg->msg_iovlen = 0; + + // Break out of the loop break; } + } + // Get the number of elements in the recvMsg's iov (input/output vector) array unsigned int iovlen = recvMsg->msg_iovlen; + + // Check if iovlen is greater than 0 if (iovlen > 0) { + // Initialize a temporary variable to keep track of the total length of iov elements size_t tmpLen = 0; + + // Iterate through each element in the iov array for (unsigned int i = 0; i < iovlen; ++i) { + // Check if the sum of the current iov element length and tmpLen is less than or equal to retval if (recvMsg->msg_iov[i].iov_len + tmpLen <= static_cast(retval)) { + // If the condition is true, add the current iov element length to tmpLen tmpLen += recvMsg->msg_iov[i].iov_len; } else { + // If the condition is false, subtract the difference between retval and tmpLen from the current iov element length recvMsg->msg_iov[i].iov_len -= IntToSize(retval - tmpLen); - recvMsg->msg_iov[i].iov_base = - reinterpret_cast(recvMsg->msg_iov[i].iov_base) + static_cast(retval) - tmpLen; - - recvMsg->msg_iov = &recvMsg->msg_iov[i]; - recvMsg->msg_iovlen -= i; - break; + // Update the iov element base pointer to skip the bytes that have already been read + recvMsg->msg_iov[i].iov_base = reinterpret_cast(recvMsg->msg_iov[i].iov_base) + static_cast(retval) - tmpLen; } } } - } else if (retval == 0) { - return IO_RW_ERROR; - } else { - if (EAGAIN == errno) { - return IO_RW_OK; - } else if (ECONNRESET == errno || ECONNABORTED == errno || ENOTCONN == errno || EPIPE == errno) { - connection->error_code = errno; - return IO_RW_ERROR; - } else { - return IO_RW_OK; - } + + // If the return value of the function is 0, it means that no data was received + if (retval == 0) { + return IO_RW_ERROR; + } + // If the return value is greater than 0, it means that data was received + else { + // Check if the error is EAGAIN, which indicates that the operation would block + if (EAGAIN == errno) { + return IO_RW_OK; + } + // Check if the error is one of the connection-related errors + else if (ECONNRESET == errno || ECONNABORTED == errno || ENOTCONN == errno || EPIPE == errno) { + // Set the error code of the connection object to the value of errno + connection->error_code = errno; + return IO_RW_ERROR; + } + // If the error is not one of the above, return IO_RW_OK + else { + return IO_RW_OK; + } + } + } } - } - return IO_RW_OK; + // If the loop is not executed, return IO_RW_OK + return IO_RW_OK; +// Closing brace to end the main function } +// Function to send a message over a TCP socket int TCPSocketOperation::SendMessage(Connection *connection, struct msghdr *sendMsg, size_t totalSendLen, size_t *sendLen) { + + // Initialize the count of EAGAIN errors encountered to 0 int eagainCount = 0; + + // Set the initial value of the send length to 0 *sendLen = 0; + // Loop until the total number of bytes sent matches the expected total while (*sendLen != totalSendLen) { + // Call the sendmsg function to send the message using the connection's socket file descriptor auto retval = sendmsg(connection->socket_fd, sendMsg, MSG_NOSIGNAL); + + // If the return value is less than 0, an error occurred if (retval < 0) { + // Increment the EAGAIN count if the error is EAGAIN (resource temporarily unavailable) ++eagainCount; + + // If the error is not EAGAIN, log an error message and return an IO_RW_ERROR if (errno != EAGAIN) { MS_LOG(ERROR) << "Failed to call sendmsg and errno is: " << errno; connection->error_code = errno; return IO_RW_ERROR; - } else if (eagainCount == EAGAIN_RETRY) { + } + // If the EAGAIN count reaches the maximum retry limit, log an error message and return an IO_RW_OK + else if (eagainCount == EAGAIN_RETRY) { MS_LOG(ERROR) << "Failed to call sendmsg after retry " + std::to_string(EAGAIN_RETRY) + " times and errno is: " << errno; *sendLen = 0; return IO_RW_OK; } + + // Log a retry message indicating the current retry count and the maximum retry limit MS_LOG(ERROR) << "retry(" + std::to_string(eagainCount) + "/" + std::to_string(EAGAIN_RETRY) + ") sending ..."; - } else { + } + // If the return value is greater than or equal to 0, the send was successful + else { + // Increment the sendLen by the number of bytes sent *sendLen += retval; - if (*sendLen == totalSendLen) { - sendMsg->msg_iovlen = 0; - break; - } +// Check if the value pointed to by sendLen is equal to totalSendLen +if (*sendLen == totalSendLen) { - size_t tmpBytes = 0; - for (unsigned int i = 0; i < sendMsg->msg_iovlen; ++i) { - if (sendMsg->msg_iov[i].iov_len + tmpBytes < static_cast(retval)) { - tmpBytes += sendMsg->msg_iov[i].iov_len; - } else { - sendMsg->msg_iov[i].iov_len -= (retval - tmpBytes); - sendMsg->msg_iov[i].iov_base = - reinterpret_cast(sendMsg->msg_iov[i].iov_base) + static_cast(retval) - tmpBytes; + // If the condition is true, set the msg_iovlen of sendMsg to 0 + sendMsg->msg_iovlen = 0; + // Exit the loop + break; +} + + // Initialize a variable to keep track of the total number of bytes processed + size_t tmpBytes = 0; + + // Iterate over each element in the msg_iov array of the sendMsg structure + for (unsigned int i = 0; i < sendMsg->msg_iovlen; ++i) { + + // Check if the sum of the current iov_len and tmpBytes is less than the value of retval + if (sendMsg->msg_iov[i].iov_len + tmpBytes < static_cast(retval)) { + + // If the condition is true, add the current iov_len to tmpBytes + tmpBytes += sendMsg->msg_iov[i].iov_len; + } else { + + // If the condition is false, subtract the difference between retval and tmpBytes from the current iov_len + sendMsg->msg_iov[i].iov_len -= (retval - tmpBytes); + + // Update the iov_base pointer by adding the difference between retval and tmpBytes to it + sendMsg->msg_iov[i].iov_base = reinterpret_cast(sendMsg->msg_iov[i].iov_base) + static_cast(retval) - tmpBytes; + + + // Set the msg_iov field of the sendMsg structure to the address of the current element in the msg_iov array sendMsg->msg_iov = &sendMsg->msg_iov[i]; + + // Decrease the value of msg_iovlen by i sendMsg->msg_iovlen -= i; + + // Exit the loop break; } } + + // Reset the eagainCount to 0 eagainCount = 0; } } + + // Return IO_RW_OK to indicate successful completion of the function return IO_RW_OK; } +// Definition of the Close function in the TCPSocketOperation class + void TCPSocketOperation::Close(Connection *connection) { + + // Close the socket file descriptor associated with the connection + // The (void) is used to suppress any warnings about the return value of close() not being used (void)close(connection->socket_fd); + + // Set the socket file descriptor of the connection to -1 to indicate that it is closed connection->socket_fd = -1; } -// accept new conn event handle +// Event handler for accepting new connections void TCPSocketOperation::NewConnEventHandler(void *context) { + // Cast the context pointer to a Connection object pointer Connection *conn = reinterpret_cast(context); + + // Update the connection state to indicate that it is now connected conn->state = ConnectionState::kConnected; + + // Return from the event handler return; } +// Definition of the ConnEstablishedEventHandler function in the TCPSocketOperation class + void TCPSocketOperation::ConnEstablishedEventHandler(void *context) { - Connection *conn = reinterpret_cast(context); - conn->state = ConnectionState::kConnected; - return; + // Cast the context pointer to a Connection pointer + Connection *conn = reinterpret_cast(context); + + // Set the state of the connection to kConnected + conn->state = ConnectionState::kConnected; + + // Return from the function + return; } + +// End of the rpc namespace } // namespace rpc + +// End of the distributed namespace } // namespace distributed -} // namespace mindspore + +// End of the mindspore namespace +} // namespace mindspore \ No newline at end of file