diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/fl/fused_pull_weight_kernel.h b/mindspore/ccsrc/backend/kernel_compiler/cpu/fl/fused_pull_weight_kernel.h index a40dea4b92b..641fa10ff2b 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/fl/fused_pull_weight_kernel.h +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/fl/fused_pull_weight_kernel.h @@ -47,13 +47,13 @@ class FusedPullWeightKernel : public CPUKernel { return false; } - std::shared_ptr fbb = std::make_shared(); + std::shared_ptr fbb = std::make_shared(); MS_EXCEPTION_IF_NULL(fbb); total_iteration_++; // The worker has to train kWorkerTrainStepNum standalone iterations before it communicates with server. - if (total_iteration_ % ps::worker::FLWorker::GetInstance().worker_step_num_per_iteration() != - ps::kTrainBeginStepNum) { + if (total_iteration_ % fl::worker::FLWorker::GetInstance().worker_step_num_per_iteration() != + fl::kTrainBeginStepNum) { return true; } @@ -72,10 +72,10 @@ class FusedPullWeightKernel : public CPUKernel { const schema::ResponsePullWeight *pull_weight_rsp = nullptr; int retcode = schema::ResponseCode_SucNotReady; while (retcode == schema::ResponseCode_SucNotReady) { - if (!ps::worker::FLWorker::GetInstance().SendToServer( + if (!fl::worker::FLWorker::GetInstance().SendToServer( 0, fbb->GetBufferPointer(), fbb->GetSize(), ps::core::TcpUserCommand::kPullWeight, &pull_weight_rsp_msg)) { MS_LOG(WARNING) << "Sending request for FusedPullWeight to server 0 failed. This iteration is dropped."; - ps::worker::FLWorker::GetInstance().SetIterationRunning(); + fl::worker::FLWorker::GetInstance().SetIterationRunning(); return true; } MS_EXCEPTION_IF_NULL(pull_weight_rsp_msg); @@ -116,7 +116,7 @@ class FusedPullWeightKernel : public CPUKernel { } } MS_LOG(INFO) << "Pull weights for " << weight_full_names_ << " succeed. Iteration: " << fl_iteration_; - ps::worker::FLWorker::GetInstance().SetIterationRunning(); + fl::worker::FLWorker::GetInstance().SetIterationRunning(); return true; } @@ -154,7 +154,7 @@ class FusedPullWeightKernel : public CPUKernel { void InitSizeLists() { return; } private: - bool BuildPullWeightReq(std::shared_ptr fbb) { + bool BuildPullWeightReq(std::shared_ptr fbb) { MS_EXCEPTION_IF_NULL(fbb); std::vector> fbs_weight_names; for (const std::string &weight_name : weight_full_names_) { diff --git a/mindspore/ccsrc/backend/kernel_compiler/cpu/fl/fused_push_weight_kernel.h b/mindspore/ccsrc/backend/kernel_compiler/cpu/fl/fused_push_weight_kernel.h index 5ed47a73f0e..1caf3489499 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/cpu/fl/fused_push_weight_kernel.h +++ b/mindspore/ccsrc/backend/kernel_compiler/cpu/fl/fused_push_weight_kernel.h @@ -45,13 +45,13 @@ class FusedPushWeightKernel : public CPUKernel { return false; } - std::shared_ptr fbb = std::make_shared(); + std::shared_ptr fbb = std::make_shared(); MS_EXCEPTION_IF_NULL(fbb); total_iteration_++; // The worker has to train kWorkerTrainStepNum standalone iterations before it communicates with server. - if (total_iteration_ % ps::worker::FLWorker::GetInstance().worker_step_num_per_iteration() != - ps::kTrainBeginStepNum) { + if (total_iteration_ % fl::worker::FLWorker::GetInstance().worker_step_num_per_iteration() != + fl::kTrainBeginStepNum) { return true; } @@ -67,17 +67,17 @@ class FusedPushWeightKernel : public CPUKernel { } // The server number may change after scaling in/out. - for (uint32_t i = 0; i < ps::worker::FLWorker::GetInstance().server_num(); i++) { + for (uint32_t i = 0; i < fl::worker::FLWorker::GetInstance().server_num(); i++) { std::shared_ptr> push_weight_rsp_msg = nullptr; const schema::ResponsePushWeight *push_weight_rsp = nullptr; int retcode = schema::ResponseCode_SucNotReady; while (retcode == schema::ResponseCode_SucNotReady) { - if (!ps::worker::FLWorker::GetInstance().SendToServer(i, fbb->GetBufferPointer(), fbb->GetSize(), + if (!fl::worker::FLWorker::GetInstance().SendToServer(i, fbb->GetBufferPointer(), fbb->GetSize(), ps::core::TcpUserCommand::kPushWeight, &push_weight_rsp_msg)) { MS_LOG(WARNING) << "Sending request for FusedPushWeight to server " << i << " failed. This iteration is dropped."; - ps::worker::FLWorker::GetInstance().SetIterationCompleted(); + fl::worker::FLWorker::GetInstance().SetIterationCompleted(); return true; } MS_EXCEPTION_IF_NULL(push_weight_rsp_msg); @@ -105,7 +105,7 @@ class FusedPushWeightKernel : public CPUKernel { } MS_LOG(INFO) << "Push weights for " << weight_full_names_ << " succeed. Iteration: " << fl_iteration_; - ps::worker::FLWorker::GetInstance().SetIterationCompleted(); + fl::worker::FLWorker::GetInstance().SetIterationCompleted(); return true; } @@ -143,7 +143,7 @@ class FusedPushWeightKernel : public CPUKernel { void InitSizeLists() { return; } private: - bool BuildPushWeightReq(std::shared_ptr fbb, const std::vector &weights) { + bool BuildPushWeightReq(std::shared_ptr fbb, const std::vector &weights) { std::vector> fbs_feature_maps; for (size_t i = 0; i < weight_full_names_.size(); i++) { const std::string &weight_name = weight_full_names_[i]; diff --git a/mindspore/ccsrc/fl/armour/cipher/cipher_init.cc b/mindspore/ccsrc/fl/armour/cipher/cipher_init.cc index bf058f53748..d2dd96c7738 100644 --- a/mindspore/ccsrc/fl/armour/cipher/cipher_init.cc +++ b/mindspore/ccsrc/fl/armour/cipher/cipher_init.cc @@ -31,8 +31,8 @@ bool CipherInit::Init(const CipherPublicPara ¶m, size_t time_out_mutex, size int return_num = 0; cipher_meta_storage_.RegisterClass(); const std::string new_prime(reinterpret_cast(param.prime), PRIME_MAX_LEN); - cipher_meta_storage_.RegisterPrime(ps::server::kCtxCipherPrimer, new_prime); - if (!cipher_meta_storage_.GetPrimeFromServer(ps::server::kCtxCipherPrimer, publicparam_.prime)) { + cipher_meta_storage_.RegisterPrime(fl::server::kCtxCipherPrimer, new_prime); + if (!cipher_meta_storage_.GetPrimeFromServer(fl::server::kCtxCipherPrimer, publicparam_.prime)) { MS_LOG(ERROR) << "Cipher Param Update is invalid."; return false; } @@ -45,7 +45,7 @@ bool CipherInit::Init(const CipherPublicPara ¶m, size_t time_out_mutex, size publicparam_.t = param.t; secrets_minnums_ = param.t; client_num_need_ = cipher_initial_client_cnt; - featuremap_ = ps::server::ModelStore::GetInstance().model_size() / sizeof(float); + featuremap_ = fl::server::ModelStore::GetInstance().model_size() / sizeof(float); share_clients_num_need_ = cipher_share_secrets_cnt; reconstruct_clients_num_need_ = cipher_reconstruct_secrets_down_cnt + 1; get_model_num_need_ = cipher_get_clientlist_cnt; diff --git a/mindspore/ccsrc/fl/armour/cipher/cipher_keys.cc b/mindspore/ccsrc/fl/armour/cipher/cipher_keys.cc index a0b2e9bf1e6..160f42c756d 100644 --- a/mindspore/ccsrc/fl/armour/cipher/cipher_keys.cc +++ b/mindspore/ccsrc/fl/armour/cipher/cipher_keys.cc @@ -21,7 +21,7 @@ namespace mindspore { namespace armour { bool CipherKeys::GetKeys(const int cur_iterator, const std::string &next_req_time, const schema::GetExchangeKeys *get_exchange_keys_req, - std::shared_ptr get_exchange_keys_resp_builder) { + std::shared_ptr get_exchange_keys_resp_builder) { MS_LOG(INFO) << "CipherMgr::GetKeys START"; if (get_exchange_keys_req == nullptr || get_exchange_keys_resp_builder == nullptr) { MS_LOG(ERROR) << "Request is nullptr or Response builder is nullptr."; @@ -32,7 +32,7 @@ bool CipherKeys::GetKeys(const int cur_iterator, const std::string &next_req_tim // get clientlist from memory server. std::vector clients; - cipher_init_->cipher_meta_storage_.GetClientListFromServer(ps::server::kCtxExChangeKeysClientList, &clients); + cipher_init_->cipher_meta_storage_.GetClientListFromServer(fl::server::kCtxExChangeKeysClientList, &clients); size_t cur_clients_num = clients.size(); std::string fl_id = get_exchange_keys_req->fl_id()->str(); @@ -61,7 +61,7 @@ bool CipherKeys::GetKeys(const int cur_iterator, const std::string &next_req_tim bool CipherKeys::ExchangeKeys(const int cur_iterator, const std::string &next_req_time, const schema::RequestExchangeKeys *exchange_keys_req, - std::shared_ptr exchange_keys_resp_builder) { + std::shared_ptr exchange_keys_resp_builder) { MS_LOG(INFO) << "CipherMgr::ExchangeKeys START"; // step 0: judge if the input param is legal. if (exchange_keys_req == nullptr || exchange_keys_resp_builder == nullptr) { @@ -75,8 +75,8 @@ bool CipherKeys::ExchangeKeys(const int cur_iterator, const std::string &next_re // step 1: get clientlist and client keys from memory server. std::map>> record_public_keys; std::vector client_list; - cipher_init_->cipher_meta_storage_.GetClientListFromServer(ps::server::kCtxExChangeKeysClientList, &client_list); - cipher_init_->cipher_meta_storage_.GetClientKeysFromServer(ps::server::kCtxClientsKeys, &record_public_keys); + cipher_init_->cipher_meta_storage_.GetClientListFromServer(fl::server::kCtxExChangeKeysClientList, &client_list); + cipher_init_->cipher_meta_storage_.GetClientKeysFromServer(fl::server::kCtxClientsKeys, &record_public_keys); // step2: process new item data. and update new item data to memory server. size_t cur_clients_num = client_list.size(); @@ -131,9 +131,9 @@ bool CipherKeys::ExchangeKeys(const int cur_iterator, const std::string &next_re cur_public_key.push_back(spk); bool retcode_key = - cipher_init_->cipher_meta_storage_.UpdateClientKeyToServer(ps::server::kCtxClientsKeys, fl_id, cur_public_key); + cipher_init_->cipher_meta_storage_.UpdateClientKeyToServer(fl::server::kCtxClientsKeys, fl_id, cur_public_key); bool retcode_client = - cipher_init_->cipher_meta_storage_.UpdateClientToServer(ps::server::kCtxExChangeKeysClientList, fl_id); + cipher_init_->cipher_meta_storage_.UpdateClientToServer(fl::server::kCtxExChangeKeysClientList, fl_id); if (retcode_key && retcode_client) { MS_LOG(INFO) << "The client " << fl_id << " CipherMgr::ExchangeKeys Success"; BuildExchangeKeysRsp(exchange_keys_resp_builder, schema::ResponseCode_SUCCEED, @@ -147,7 +147,7 @@ bool CipherKeys::ExchangeKeys(const int cur_iterator, const std::string &next_re } } -void CipherKeys::BuildExchangeKeysRsp(std::shared_ptr exchange_keys_resp_builder, +void CipherKeys::BuildExchangeKeysRsp(std::shared_ptr exchange_keys_resp_builder, const schema::ResponseCode retcode, const std::string &reason, const std::string &next_req_time, const int iteration) { auto rsp_reason = exchange_keys_resp_builder->CreateString(reason); @@ -162,7 +162,7 @@ void CipherKeys::BuildExchangeKeysRsp(std::shared_ptr exc return; } -bool CipherKeys::BuildGetKeys(std::shared_ptr fbb, const schema::ResponseCode retcode, +bool CipherKeys::BuildGetKeys(std::shared_ptr fbb, const schema::ResponseCode retcode, const int iteration, const std::string &next_req_time, bool is_good) { bool flag = true; if (is_good) { @@ -170,7 +170,7 @@ bool CipherKeys::BuildGetKeys(std::shared_ptr fbb, const std::vector> public_keys_list; MS_LOG(INFO) << "Get Keys: "; std::map>> record_public_keys; - cipher_init_->cipher_meta_storage_.GetClientKeysFromServer(ps::server::kCtxClientsKeys, &record_public_keys); + cipher_init_->cipher_meta_storage_.GetClientKeysFromServer(fl::server::kCtxClientsKeys, &record_public_keys); if (record_public_keys.size() < cipher_init_->client_num_need_) { MS_LOG(INFO) << "NOT READY. keys num: " << record_public_keys.size() << "clients num: " << cipher_init_->client_num_need_; @@ -221,8 +221,8 @@ bool CipherKeys::BuildGetKeys(std::shared_ptr fbb, const } void CipherKeys::ClearKeys() { - ps::server::DistributedMetadataStore::GetInstance().ResetMetadata(ps::server::kCtxExChangeKeysClientList); - ps::server::DistributedMetadataStore::GetInstance().ResetMetadata(ps::server::kCtxClientsKeys); + fl::server::DistributedMetadataStore::GetInstance().ResetMetadata(fl::server::kCtxExChangeKeysClientList); + fl::server::DistributedMetadataStore::GetInstance().ResetMetadata(fl::server::kCtxClientsKeys); } } // namespace armour diff --git a/mindspore/ccsrc/fl/armour/cipher/cipher_keys.h b/mindspore/ccsrc/fl/armour/cipher/cipher_keys.h index 1b915f537cb..205dea247da 100644 --- a/mindspore/ccsrc/fl/armour/cipher/cipher_keys.h +++ b/mindspore/ccsrc/fl/armour/cipher/cipher_keys.h @@ -45,18 +45,18 @@ class CipherKeys { // handle the client's request of get keys. bool GetKeys(const int cur_iterator, const std::string &next_req_time, const schema::GetExchangeKeys *get_exchange_keys_req, - std::shared_ptr get_exchange_keys_resp_builder); + std::shared_ptr get_exchange_keys_resp_builder); // handle the client's request of exchange keys. bool ExchangeKeys(const int cur_iterator, const std::string &next_req_time, const schema::RequestExchangeKeys *exchange_keys_req, - std::shared_ptr exchange_keys_resp_builder); + std::shared_ptr exchange_keys_resp_builder); // build response code of get keys. - bool BuildGetKeys(std::shared_ptr fbb, const schema::ResponseCode retcode, const int iteration, + bool BuildGetKeys(std::shared_ptr fbb, const schema::ResponseCode retcode, const int iteration, const std::string &next_req_time, bool is_good); // build response code of exchange keys. - void BuildExchangeKeysRsp(std::shared_ptr exchange_keys_resp_builder, + void BuildExchangeKeysRsp(std::shared_ptr exchange_keys_resp_builder, const schema::ResponseCode retcode, const std::string &reason, const std::string &next_req_time, const int iteration); // clear the shared memory. diff --git a/mindspore/ccsrc/fl/armour/cipher/cipher_meta_storage.cc b/mindspore/ccsrc/fl/armour/cipher/cipher_meta_storage.cc index 37ab1e554ac..14ea1b35a7a 100644 --- a/mindspore/ccsrc/fl/armour/cipher/cipher_meta_storage.cc +++ b/mindspore/ccsrc/fl/armour/cipher/cipher_meta_storage.cc @@ -21,16 +21,16 @@ namespace armour { void CipherMetaStorage::GetClientSharesFromServer( const char *list_name, std::map> *clients_shares_list) { - const ps::PBMetadata &clients_shares_pb_out = - ps::server::DistributedMetadataStore::GetInstance().GetMetadata(list_name); - const ps::ClientShares &clients_shares_pb = clients_shares_pb_out.client_shares(); + const fl::PBMetadata &clients_shares_pb_out = + fl::server::DistributedMetadataStore::GetInstance().GetMetadata(list_name); + const fl::ClientShares &clients_shares_pb = clients_shares_pb_out.client_shares(); auto iter = clients_shares_pb.client_secret_shares().begin(); for (; iter != clients_shares_pb.client_secret_shares().end(); ++iter) { std::string fl_id = iter->first; - const ps::SharesPb &shares_pb = iter->second; + const fl::SharesPb &shares_pb = iter->second; std::vector encrpted_shares_new; for (int index_shares = 0; index_shares < shares_pb.clientsharestrs_size(); ++index_shares) { - const ps::ClientShareStr &client_share_str_pb = shares_pb.clientsharestrs(index_shares); + const fl::ClientShareStr &client_share_str_pb = shares_pb.clientsharestrs(index_shares); clientshare_str new_clientshare; new_clientshare.fl_id = client_share_str_pb.fl_id(); new_clientshare.index = client_share_str_pb.index(); @@ -42,8 +42,8 @@ void CipherMetaStorage::GetClientSharesFromServer( } void CipherMetaStorage::GetClientListFromServer(const char *list_name, std::vector *clients_list) { - const ps::PBMetadata &client_list_pb_out = ps::server::DistributedMetadataStore::GetInstance().GetMetadata(list_name); - const ps::UpdateModelClientList &client_list_pb = client_list_pb_out.client_list(); + const fl::PBMetadata &client_list_pb_out = fl::server::DistributedMetadataStore::GetInstance().GetMetadata(list_name); + const fl::UpdateModelClientList &client_list_pb = client_list_pb_out.client_list(); for (int i = 0; i < client_list_pb.fl_id_size(); ++i) { std::string fl_id = client_list_pb.fl_id(i); clients_list->push_back(fl_id); @@ -52,14 +52,14 @@ void CipherMetaStorage::GetClientListFromServer(const char *list_name, std::vect void CipherMetaStorage::GetClientKeysFromServer( const char *list_name, std::map>> *clients_keys_list) { - const ps::PBMetadata &clients_keys_pb_out = - ps::server::DistributedMetadataStore::GetInstance().GetMetadata(list_name); - const ps::ClientKeys &clients_keys_pb = clients_keys_pb_out.client_keys(); + const fl::PBMetadata &clients_keys_pb_out = + fl::server::DistributedMetadataStore::GetInstance().GetMetadata(list_name); + const fl::ClientKeys &clients_keys_pb = clients_keys_pb_out.client_keys(); for (auto iter = clients_keys_pb.client_keys().begin(); iter != clients_keys_pb.client_keys().end(); ++iter) { // const PairClientKeys & pair_client_keys_pb = clients_keys_pb.client_keys(i); std::string fl_id = iter->first; - ps::KeysPb keys_pb = iter->second; + fl::KeysPb keys_pb = iter->second; std::vector cpk(keys_pb.key(0).begin(), keys_pb.key(0).end()); std::vector spk(keys_pb.key(1).begin(), keys_pb.key(1).end()); std::vector> cur_keys; @@ -70,9 +70,9 @@ void CipherMetaStorage::GetClientKeysFromServer( } bool CipherMetaStorage::GetClientNoisesFromServer(const char *list_name, std::vector *cur_public_noise) { - const ps::PBMetadata &clients_noises_pb_out = - ps::server::DistributedMetadataStore::GetInstance().GetMetadata(list_name); - const ps::ClientNoises &clients_noises_pb = clients_noises_pb_out.client_noises(); + const fl::PBMetadata &clients_noises_pb_out = + fl::server::DistributedMetadataStore::GetInstance().GetMetadata(list_name); + const fl::ClientNoises &clients_noises_pb = clients_noises_pb_out.client_noises(); while (clients_noises_pb.has_one_client_noises() == false) { MS_LOG(INFO) << "GetClientNoisesFromServer NULL."; std::this_thread::sleep_for(std::chrono::milliseconds(50)); @@ -83,8 +83,8 @@ bool CipherMetaStorage::GetClientNoisesFromServer(const char *list_name, std::ve } bool CipherMetaStorage::GetPrimeFromServer(const char *prime_name, unsigned char *prime) { - const ps::PBMetadata &prime_pb_out = ps::server::DistributedMetadataStore::GetInstance().GetMetadata(prime_name); - ps::Prime prime_pb(prime_pb_out.prime()); + const fl::PBMetadata &prime_pb_out = fl::server::DistributedMetadataStore::GetInstance().GetMetadata(prime_name); + fl::Prime prime_pb(prime_pb_out.prime()); std::string str = *(prime_pb.mutable_prime()); MS_LOG(INFO) << "get prime from metastorage :" << str; @@ -99,20 +99,20 @@ bool CipherMetaStorage::GetPrimeFromServer(const char *prime_name, unsigned char bool CipherMetaStorage::UpdateClientToServer(const char *list_name, const std::string &fl_id) { bool retcode = true; - ps::FLId fl_id_pb; + fl::FLId fl_id_pb; fl_id_pb.set_fl_id(fl_id); - ps::PBMetadata client_pb; + fl::PBMetadata client_pb; client_pb.mutable_fl_id()->MergeFrom(fl_id_pb); - retcode = ps::server::DistributedMetadataStore::GetInstance().UpdateMetadata(list_name, client_pb); + retcode = fl::server::DistributedMetadataStore::GetInstance().UpdateMetadata(list_name, client_pb); return retcode; } void CipherMetaStorage::RegisterPrime(const char *list_name, const std::string &prime) { MS_LOG(INFO) << "register prime: " << prime; - ps::Prime prime_id_pb; + fl::Prime prime_id_pb; prime_id_pb.set_prime(prime); - ps::PBMetadata prime_pb; + fl::PBMetadata prime_pb; prime_pb.mutable_prime()->MergeFrom(prime_id_pb); - ps::server::DistributedMetadataStore::GetInstance().RegisterMetadata(list_name, prime_pb); + fl::server::DistributedMetadataStore::GetInstance().RegisterMetadata(list_name, prime_pb); } bool CipherMetaStorage::UpdateClientKeyToServer(const char *list_name, const std::string &fl_id, @@ -123,25 +123,25 @@ bool CipherMetaStorage::UpdateClientKeyToServer(const char *list_name, const std return false; } // update new item to memory server. - ps::KeysPb keys; + fl::KeysPb keys; keys.add_key()->assign(cur_public_key[0].begin(), cur_public_key[0].end()); keys.add_key()->assign(cur_public_key[1].begin(), cur_public_key[1].end()); - ps::PairClientKeys pair_client_keys_pb; + fl::PairClientKeys pair_client_keys_pb; pair_client_keys_pb.set_fl_id(fl_id); pair_client_keys_pb.mutable_client_keys()->MergeFrom(keys); - ps::PBMetadata client_and_keys_pb; + fl::PBMetadata client_and_keys_pb; client_and_keys_pb.mutable_pair_client_keys()->MergeFrom(pair_client_keys_pb); - retcode = ps::server::DistributedMetadataStore::GetInstance().UpdateMetadata(list_name, client_and_keys_pb); + retcode = fl::server::DistributedMetadataStore::GetInstance().UpdateMetadata(list_name, client_and_keys_pb); return retcode; } bool CipherMetaStorage::UpdateClientNoiseToServer(const char *list_name, const std::vector &cur_public_noise) { // update new item to memory server. - ps::OneClientNoises noises_pb; + fl::OneClientNoises noises_pb; *noises_pb.mutable_noise() = {cur_public_noise.begin(), cur_public_noise.end()}; - ps::PBMetadata client_noises_pb; + fl::PBMetadata client_noises_pb; client_noises_pb.mutable_one_client_noises()->MergeFrom(noises_pb); - return ps::server::DistributedMetadataStore::GetInstance().UpdateMetadata(list_name, client_noises_pb); + return fl::server::DistributedMetadataStore::GetInstance().UpdateMetadata(list_name, client_noises_pb); } bool CipherMetaStorage::UpdateClientShareToServer( @@ -149,10 +149,10 @@ bool CipherMetaStorage::UpdateClientShareToServer( const flatbuffers::Vector> *shares) { bool retcode = true; int size_shares = shares->size(); - ps::SharesPb shares_pb; + fl::SharesPb shares_pb; for (int index = 0; index < size_shares; ++index) { // new item - ps::ClientShareStr *client_share_str_new_p = shares_pb.add_clientsharestrs(); + fl::ClientShareStr *client_share_str_new_p = shares_pb.add_clientsharestrs(); std::string fl_id_new = (*shares)[index]->fl_id()->str(); int index_new = (*shares)[index]->index(); auto share = (*shares)[index]->share(); @@ -160,32 +160,32 @@ bool CipherMetaStorage::UpdateClientShareToServer( client_share_str_new_p->set_fl_id(fl_id_new); client_share_str_new_p->set_index(index_new); } - ps::PairClientShares pair_client_shares_pb; + fl::PairClientShares pair_client_shares_pb; pair_client_shares_pb.set_fl_id(fl_id); pair_client_shares_pb.mutable_client_shares()->MergeFrom(shares_pb); - ps::PBMetadata client_and_shares_pb; + fl::PBMetadata client_and_shares_pb; client_and_shares_pb.mutable_pair_client_shares()->MergeFrom(pair_client_shares_pb); - retcode = ps::server::DistributedMetadataStore::GetInstance().UpdateMetadata(list_name, client_and_shares_pb); + retcode = fl::server::DistributedMetadataStore::GetInstance().UpdateMetadata(list_name, client_and_shares_pb); return retcode; } void CipherMetaStorage::RegisterClass() { - ps::PBMetadata exchange_kyes_client_list; - ps::server::DistributedMetadataStore::GetInstance().RegisterMetadata(ps::server::kCtxExChangeKeysClientList, + fl::PBMetadata exchange_kyes_client_list; + fl::server::DistributedMetadataStore::GetInstance().RegisterMetadata(fl::server::kCtxExChangeKeysClientList, exchange_kyes_client_list); - ps::PBMetadata clients_keys; - ps::server::DistributedMetadataStore::GetInstance().RegisterMetadata(ps::server::kCtxClientsKeys, clients_keys); - ps::PBMetadata reconstruct_client_list; - ps::server::DistributedMetadataStore::GetInstance().RegisterMetadata(ps::server::kCtxReconstructClientList, + fl::PBMetadata clients_keys; + fl::server::DistributedMetadataStore::GetInstance().RegisterMetadata(fl::server::kCtxClientsKeys, clients_keys); + fl::PBMetadata reconstruct_client_list; + fl::server::DistributedMetadataStore::GetInstance().RegisterMetadata(fl::server::kCtxReconstructClientList, reconstruct_client_list); - ps::PBMetadata clients_reconstruct_shares; - ps::server::DistributedMetadataStore::GetInstance().RegisterMetadata(ps::server::kCtxClientsReconstructShares, + fl::PBMetadata clients_reconstruct_shares; + fl::server::DistributedMetadataStore::GetInstance().RegisterMetadata(fl::server::kCtxClientsReconstructShares, clients_reconstruct_shares); - ps::PBMetadata share_secretes_client_list; - ps::server::DistributedMetadataStore::GetInstance().RegisterMetadata(ps::server::kCtxShareSecretsClientList, + fl::PBMetadata share_secretes_client_list; + fl::server::DistributedMetadataStore::GetInstance().RegisterMetadata(fl::server::kCtxShareSecretsClientList, share_secretes_client_list); - ps::PBMetadata clients_encrypt_shares; - ps::server::DistributedMetadataStore::GetInstance().RegisterMetadata(ps::server::kCtxClientsEncryptedShares, + fl::PBMetadata clients_encrypt_shares; + fl::server::DistributedMetadataStore::GetInstance().RegisterMetadata(fl::server::kCtxClientsEncryptedShares, clients_encrypt_shares); } } // namespace armour diff --git a/mindspore/ccsrc/fl/armour/cipher/cipher_reconstruct.cc b/mindspore/ccsrc/fl/armour/cipher/cipher_reconstruct.cc index 91571e306ae..52daf7e421b 100644 --- a/mindspore/ccsrc/fl/armour/cipher/cipher_reconstruct.cc +++ b/mindspore/ccsrc/fl/armour/cipher/cipher_reconstruct.cc @@ -101,15 +101,15 @@ bool CipherReconStruct::ReconstructSecretsGenNoise(const std::vector &cl MS_LOG(INFO) << "CipherReconStruct::ReconstructSecretsGenNoise START"; bool retcode = true; std::map> reconstruct_secret_list_ori; - cipher_init_->cipher_meta_storage_.GetClientSharesFromServer(ps::server::kCtxClientsReconstructShares, + cipher_init_->cipher_meta_storage_.GetClientSharesFromServer(fl::server::kCtxClientsReconstructShares, &reconstruct_secret_list_ori); std::map>> record_public_keys; - cipher_init_->cipher_meta_storage_.GetClientKeysFromServer(ps::server::kCtxClientsKeys, &record_public_keys); + cipher_init_->cipher_meta_storage_.GetClientKeysFromServer(fl::server::kCtxClientsKeys, &record_public_keys); std::vector clients_reconstruct_list; - cipher_init_->cipher_meta_storage_.GetClientListFromServer(ps::server::kCtxReconstructClientList, + cipher_init_->cipher_meta_storage_.GetClientListFromServer(fl::server::kCtxReconstructClientList, &clients_reconstruct_list); std::vector clients_share_list; - cipher_init_->cipher_meta_storage_.GetClientListFromServer(ps::server::kCtxShareSecretsClientList, + cipher_init_->cipher_meta_storage_.GetClientListFromServer(fl::server::kCtxShareSecretsClientList, &clients_share_list); if (reconstruct_secret_list_ori.size() != clients_reconstruct_list.size() || record_public_keys.size() < cipher_init_->client_num_need_ || @@ -146,7 +146,7 @@ bool CipherReconStruct::ReconstructSecretsGenNoise(const std::vector &cl client_keys.clear(); MS_LOG(INFO) << " ReconstructSecretsGenNoise updata noise to server"; - if (cipher_init_->cipher_meta_storage_.UpdateClientNoiseToServer(ps::server::kCtxClientNoises, noise) == false) + if (cipher_init_->cipher_meta_storage_.UpdateClientNoiseToServer(fl::server::kCtxClientNoises, noise) == false) return false; MS_LOG(INFO) << " ReconstructSecretsGenNoise Success"; } else { @@ -159,7 +159,7 @@ bool CipherReconStruct::ReconstructSecretsGenNoise(const std::vector &cl // reconstruct secrets bool CipherReconStruct::ReconstructSecrets(const int cur_iterator, const std::string &next_req_time, const schema::SendReconstructSecret *reconstruct_secret_req, - std::shared_ptr reconstruct_secret_resp_builder, + std::shared_ptr reconstruct_secret_resp_builder, const std::vector &client_list) { MS_LOG(INFO) << "CipherReconStruct::ReconstructSecrets START"; clock_t start_time = clock(); @@ -178,10 +178,10 @@ bool CipherReconStruct::ReconstructSecrets(const int cur_iterator, const std::st return false; } std::vector clients_reconstruct_list; - cipher_init_->cipher_meta_storage_.GetClientListFromServer(ps::server::kCtxReconstructClientList, + cipher_init_->cipher_meta_storage_.GetClientListFromServer(fl::server::kCtxReconstructClientList, &clients_reconstruct_list); std::map> clients_shares_all; - cipher_init_->cipher_meta_storage_.GetClientSharesFromServer(ps::server::kCtxClientsReconstructShares, + cipher_init_->cipher_meta_storage_.GetClientSharesFromServer(fl::server::kCtxClientsReconstructShares, &clients_shares_all); size_t count_client_num = clients_shares_all.size(); @@ -215,9 +215,9 @@ bool CipherReconStruct::ReconstructSecrets(const int cur_iterator, const std::st } auto reconstruct_secret_shares = reconstruct_secret_req->reconstruct_secret_shares(); bool retcode_client = - cipher_init_->cipher_meta_storage_.UpdateClientToServer(ps::server::kCtxReconstructClientList, fl_id); + cipher_init_->cipher_meta_storage_.UpdateClientToServer(fl::server::kCtxReconstructClientList, fl_id); bool retcode_share = cipher_init_->cipher_meta_storage_.UpdateClientShareToServer( - ps::server::kCtxClientsReconstructShares, fl_id, reconstruct_secret_shares); + fl::server::kCtxClientsReconstructShares, fl_id, reconstruct_secret_shares); if (!(retcode_client && retcode_share)) { BuildReconstructSecretsRsp(reconstruct_secret_resp_builder, schema::ResponseCode_OutOfTime, "reconstruct update shares or client failed.", cur_iterator, next_req_time); @@ -233,9 +233,9 @@ bool CipherReconStruct::ReconstructSecrets(const int cur_iterator, const std::st return true; } else { bool retcode_result = true; - const ps::PBMetadata &clients_noises_pb_out = - ps::server::DistributedMetadataStore::GetInstance().GetMetadata(ps::server::kCtxClientNoises); - const ps::ClientNoises &clients_noises_pb = clients_noises_pb_out.client_noises(); + const fl::PBMetadata &clients_noises_pb_out = + fl::server::DistributedMetadataStore::GetInstance().GetMetadata(fl::server::kCtxClientNoises); + const fl::ClientNoises &clients_noises_pb = clients_noises_pb_out.client_noises(); if (clients_noises_pb.has_one_client_noises() == false) { MS_LOG(INFO) << "Success,the secret will be reconstructed."; retcode_result = ReconstructSecretsGenNoise(client_list); @@ -279,13 +279,13 @@ bool CipherReconStruct::GetNoiseMasksSum(std::vector *result, void CipherReconStruct::ClearReconstructSecrets() { MS_LOG(INFO) << "CipherReconStruct::ClearReconstructSecrets START"; - ps::server::DistributedMetadataStore::GetInstance().ResetMetadata(ps::server::kCtxReconstructClientList); - ps::server::DistributedMetadataStore::GetInstance().ResetMetadata(ps::server::kCtxClientsReconstructShares); - ps::server::DistributedMetadataStore::GetInstance().ResetMetadata(ps::server::kCtxClientNoises); + fl::server::DistributedMetadataStore::GetInstance().ResetMetadata(fl::server::kCtxReconstructClientList); + fl::server::DistributedMetadataStore::GetInstance().ResetMetadata(fl::server::kCtxClientsReconstructShares); + fl::server::DistributedMetadataStore::GetInstance().ResetMetadata(fl::server::kCtxClientNoises); MS_LOG(INFO) << "CipherReconStruct::ClearReconstructSecrets Success"; } -void CipherReconStruct::BuildReconstructSecretsRsp(std::shared_ptr fbb, +void CipherReconStruct::BuildReconstructSecretsRsp(std::shared_ptr fbb, const schema::ResponseCode retcode, const std::string &reason, const int iteration, const std::string &next_req_time) { auto fbs_reason = fbb->CreateString(reason); diff --git a/mindspore/ccsrc/fl/armour/cipher/cipher_reconstruct.h b/mindspore/ccsrc/fl/armour/cipher/cipher_reconstruct.h index c78018a0179..d43fad277e9 100644 --- a/mindspore/ccsrc/fl/armour/cipher/cipher_reconstruct.h +++ b/mindspore/ccsrc/fl/armour/cipher/cipher_reconstruct.h @@ -44,11 +44,11 @@ class CipherReconStruct { // reconstruct secret mask bool ReconstructSecrets(const int cur_iterator, const std::string &next_req_time, const schema::SendReconstructSecret *reconstruct_secret_req, - std::shared_ptr reconstruct_secret_resp_builder, + std::shared_ptr reconstruct_secret_resp_builder, const std::vector &client_list); // build response code of reconstruct secret. - void BuildReconstructSecretsRsp(std::shared_ptr fbb, const schema::ResponseCode retcode, + void BuildReconstructSecretsRsp(std::shared_ptr fbb, const schema::ResponseCode retcode, const std::string &reason, const int iteration, const std::string &next_req_time); // clear the shared memory. diff --git a/mindspore/ccsrc/fl/armour/cipher/cipher_shares.cc b/mindspore/ccsrc/fl/armour/cipher/cipher_shares.cc index 88f0812c1b4..6de5d973f97 100644 --- a/mindspore/ccsrc/fl/armour/cipher/cipher_shares.cc +++ b/mindspore/ccsrc/fl/armour/cipher/cipher_shares.cc @@ -21,7 +21,7 @@ namespace mindspore { namespace armour { bool CipherShares::ShareSecrets(const int cur_iterator, const schema::RequestShareSecrets *share_secrets_req, - std::shared_ptr share_secrets_resp_builder, + std::shared_ptr share_secrets_resp_builder, const string next_req_time) { MS_LOG(INFO) << "CipherShares::ShareSecrets START"; if (share_secrets_req == nullptr) { @@ -35,13 +35,13 @@ bool CipherShares::ShareSecrets(const int cur_iterator, const schema::RequestSha // step 1: get client list and share secrets from memory server. clock_t start_time = clock(); std::vector clients_share_list; - cipher_init_->cipher_meta_storage_.GetClientListFromServer(ps::server::kCtxShareSecretsClientList, + cipher_init_->cipher_meta_storage_.GetClientListFromServer(fl::server::kCtxShareSecretsClientList, &clients_share_list); std::vector clients_exchange_list; - cipher_init_->cipher_meta_storage_.GetClientListFromServer(ps::server::kCtxExChangeKeysClientList, + cipher_init_->cipher_meta_storage_.GetClientListFromServer(fl::server::kCtxExChangeKeysClientList, &clients_exchange_list); std::map> encrypted_shares_all; - cipher_init_->cipher_meta_storage_.GetClientSharesFromServer(ps::server::kCtxClientsEncryptedShares, + cipher_init_->cipher_meta_storage_.GetClientSharesFromServer(fl::server::kCtxClientsEncryptedShares, &encrypted_shares_all); MS_LOG(INFO) << "Client of keys size : " << clients_exchange_list.size() @@ -75,9 +75,9 @@ bool CipherShares::ShareSecrets(const int cur_iterator, const schema::RequestSha const flatbuffers::Vector> *encrypted_shares = (share_secrets_req->encrypted_shares()); bool retcode_share = cipher_init_->cipher_meta_storage_.UpdateClientShareToServer( - ps::server::kCtxClientsEncryptedShares, fl_id_src, encrypted_shares); + fl::server::kCtxClientsEncryptedShares, fl_id_src, encrypted_shares); bool retcode_client = - cipher_init_->cipher_meta_storage_.UpdateClientToServer(ps::server::kCtxShareSecretsClientList, fl_id_src); + cipher_init_->cipher_meta_storage_.UpdateClientToServer(fl::server::kCtxShareSecretsClientList, fl_id_src); bool retcode = retcode_share && retcode_client; if (retcode) { BuildShareSecretsRsp(share_secrets_resp_builder, schema::ResponseCode_SUCCEED, "OK", next_req_time, iteration); @@ -95,7 +95,7 @@ bool CipherShares::ShareSecrets(const int cur_iterator, const schema::RequestSha } bool CipherShares::GetSecrets(const schema::GetShareSecrets *get_secrets_req, - std::shared_ptr get_secrets_resp_builder, + std::shared_ptr get_secrets_resp_builder, const std::string &next_req_time) { MS_LOG(INFO) << "CipherShares::GetSecrets START"; clock_t start_time = clock(); @@ -108,10 +108,10 @@ bool CipherShares::GetSecrets(const schema::GetShareSecrets *get_secrets_req, // step 1: get client list and client shares list from memory server. std::vector clients_share_list; - cipher_init_->cipher_meta_storage_.GetClientListFromServer(ps::server::kCtxShareSecretsClientList, + cipher_init_->cipher_meta_storage_.GetClientListFromServer(fl::server::kCtxShareSecretsClientList, &clients_share_list); std::map> encrypted_shares_all; - cipher_init_->cipher_meta_storage_.GetClientSharesFromServer(ps::server::kCtxClientsEncryptedShares, + cipher_init_->cipher_meta_storage_.GetClientSharesFromServer(fl::server::kCtxClientsEncryptedShares, &encrypted_shares_all); int iteration = get_secrets_req->iteration(); size_t share_clients_num = clients_share_list.size(); @@ -180,7 +180,7 @@ bool CipherShares::GetSecrets(const schema::GetShareSecrets *get_secrets_req, } void CipherShares::BuildGetSecretsRsp( - std::shared_ptr get_secrets_resp_builder, schema::ResponseCode retcode, int iteration, + std::shared_ptr get_secrets_resp_builder, schema::ResponseCode retcode, int iteration, std::string next_req_time, std::vector> *encrypted_shares) { int rsp_retcode = retcode; int rsp_iteration = iteration; @@ -199,7 +199,7 @@ void CipherShares::BuildGetSecretsRsp( return; } -void CipherShares::BuildShareSecretsRsp(std::shared_ptr share_secrets_resp_builder, +void CipherShares::BuildShareSecretsRsp(std::shared_ptr share_secrets_resp_builder, const schema::ResponseCode retcode, const string &reason, const string &next_req_time, const int iteration) { auto rsp_reason = share_secrets_resp_builder->CreateString(reason); @@ -211,8 +211,8 @@ void CipherShares::BuildShareSecretsRsp(std::shared_ptr s } void CipherShares::ClearShareSecrets() { - ps::server::DistributedMetadataStore::GetInstance().ResetMetadata(ps::server::kCtxShareSecretsClientList); - ps::server::DistributedMetadataStore::GetInstance().ResetMetadata(ps::server::kCtxClientsEncryptedShares); + fl::server::DistributedMetadataStore::GetInstance().ResetMetadata(fl::server::kCtxShareSecretsClientList); + fl::server::DistributedMetadataStore::GetInstance().ResetMetadata(fl::server::kCtxClientsEncryptedShares); } } // namespace armour diff --git a/mindspore/ccsrc/fl/armour/cipher/cipher_shares.h b/mindspore/ccsrc/fl/armour/cipher/cipher_shares.h index a21ec96658e..1915db0dfc9 100644 --- a/mindspore/ccsrc/fl/armour/cipher/cipher_shares.h +++ b/mindspore/ccsrc/fl/armour/cipher/cipher_shares.h @@ -43,17 +43,17 @@ class CipherShares { // handle the client's request of share secrets. bool ShareSecrets(const int cur_iterator, const schema::RequestShareSecrets *share_secrets_req, - std::shared_ptr share_secrets_resp_builder, const string next_req_time); + std::shared_ptr share_secrets_resp_builder, const string next_req_time); // handle the client's request of get secrets. bool GetSecrets(const schema::GetShareSecrets *get_secrets_req, - std::shared_ptr get_secrets_resp_builder, const std::string &next_req_time); + std::shared_ptr get_secrets_resp_builder, const std::string &next_req_time); // build response code of share secrets. - void BuildShareSecretsRsp(std::shared_ptr share_secrets_resp_builder, + void BuildShareSecretsRsp(std::shared_ptr share_secrets_resp_builder, const schema::ResponseCode retcode, const string &reason, const string &next_req_time, const int iteration); // build response code of get secrets. - void BuildGetSecretsRsp(std::shared_ptr get_secrets_resp_builder, + void BuildGetSecretsRsp(std::shared_ptr get_secrets_resp_builder, const schema::ResponseCode retcode, const int iteration, std::string next_req_time, std::vector> *encrypted_shares); // clear the shared memory. diff --git a/mindspore/ccsrc/fl/armour/cipher/cipher_unmask.cc b/mindspore/ccsrc/fl/armour/cipher/cipher_unmask.cc index 2b585742458..b19279973e7 100644 --- a/mindspore/ccsrc/fl/armour/cipher/cipher_unmask.cc +++ b/mindspore/ccsrc/fl/armour/cipher/cipher_unmask.cc @@ -26,13 +26,13 @@ bool CipherUnmask::UnMask(const std::map &data) { clock_t start_time = clock(); std::vector noise; - cipher_init_->cipher_meta_storage_.GetClientNoisesFromServer(ps::server::kCtxClientNoises, &noise); + cipher_init_->cipher_meta_storage_.GetClientNoisesFromServer(fl::server::kCtxClientNoises, &noise); if (noise.size() != cipher_init_->featuremap_) { MS_LOG(ERROR) << " CipherMgr UnMask ERROR"; return false; } - size_t data_size = ps::server::LocalMetaStore::GetInstance().value(ps::server::kCtxFedAvgTotalDataSize); + size_t data_size = fl::server::LocalMetaStore::GetInstance().value(fl::server::kCtxFedAvgTotalDataSize); int sum_size = 0; for (auto iter = data.begin(); iter != data.end(); ++iter) { int size_data = iter->second->size / sizeof(float); diff --git a/mindspore/ccsrc/fl/server/collective_ops_impl.cc b/mindspore/ccsrc/fl/server/collective_ops_impl.cc index d9ab5cdce60..cbd9e507a1e 100644 --- a/mindspore/ccsrc/fl/server/collective_ops_impl.cc +++ b/mindspore/ccsrc/fl/server/collective_ops_impl.cc @@ -17,13 +17,13 @@ #include "fl/server/collective_ops_impl.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { -void CollectiveOpsImpl::Initialize(const std::shared_ptr &server_node) { +void CollectiveOpsImpl::Initialize(const std::shared_ptr &server_node) { MS_EXCEPTION_IF_NULL(server_node); server_node_ = server_node; local_rank_ = server_node_->rank_id(); - server_num_ = PSContext::instance()->initial_server_num(); + server_num_ = ps::PSContext::instance()->initial_server_num(); return; } @@ -66,7 +66,7 @@ bool CollectiveOpsImpl::RingAllReduce(const void *sendbuff, void *recvbuff, size // Step 1: Async send data to next rank. size_t send_chunk_index = (local_rank_ - i + rank_size) % rank_size; T *send_chunk = output_buff + chunk_offset[send_chunk_index]; - auto send_req_id = server_node_->CollectiveSendAsync(core::NodeRole::SERVER, send_to_rank, send_chunk, + auto send_req_id = server_node_->CollectiveSendAsync(ps::core::NodeRole::SERVER, send_to_rank, send_chunk, chunk_sizes[send_chunk_index] * sizeof(T)); // Step 2: Async receive data to next rank and wait until it's done. size_t recv_chunk_index = (local_rank_ - i - 1 + rank_size) % rank_size; @@ -76,12 +76,16 @@ bool CollectiveOpsImpl::RingAllReduce(const void *sendbuff, void *recvbuff, size << ", recv count:" << chunk_sizes[recv_chunk_index] << ", iteration:" << i; std::shared_ptr> recv_str; - auto recv_req_id = server_node_->CollectiveReceiveAsync(core::NodeRole::SERVER, recv_from_rank, &recv_str); + auto recv_req_id = server_node_->CollectiveReceiveAsync(ps::core::NodeRole::SERVER, recv_from_rank, &recv_str); if (!server_node_->CollectiveWait(recv_req_id)) { MS_LOG(ERROR) << "CollectiveWait " << recv_req_id << " failed."; return false; } - memcpy_s(tmp_recv_chunk.get(), chunk_sizes[recv_chunk_index] * sizeof(T), recv_str->data(), recv_str->size()); + ret = memcpy_s(tmp_recv_chunk.get(), chunk_sizes[recv_chunk_index] * sizeof(T), recv_str->data(), recv_str->size()); + if (ret != 0) { + MS_LOG(ERROR) << "memcpy_s error, errorno(" << ret << ")"; + return false; + } // Step 3: Reduce the data so we can overlap the time cost of send. for (size_t j = 0; j < chunk_sizes[recv_chunk_index]; j++) { @@ -100,7 +104,7 @@ bool CollectiveOpsImpl::RingAllReduce(const void *sendbuff, void *recvbuff, size for (size_t i = 0; i < rank_size - 1; i++) { size_t send_chunk_index = (local_rank_ - i + 1 + rank_size) % rank_size; T *send_chunk = output_buff + chunk_offset[send_chunk_index]; - auto send_req_id = server_node_->CollectiveSendAsync(core::NodeRole::SERVER, send_to_rank, send_chunk, + auto send_req_id = server_node_->CollectiveSendAsync(ps::core::NodeRole::SERVER, send_to_rank, send_chunk, chunk_sizes[send_chunk_index] * sizeof(T)); size_t recv_chunk_index = (local_rank_ - i + rank_size) % rank_size; T *recv_chunk = output_buff + chunk_offset[recv_chunk_index]; @@ -109,13 +113,17 @@ bool CollectiveOpsImpl::RingAllReduce(const void *sendbuff, void *recvbuff, size << ", recv count:" << chunk_sizes[recv_chunk_index] << ", iteration:" << i; std::shared_ptr> recv_str; - auto recv_req_id = server_node_->CollectiveReceiveAsync(core::NodeRole::SERVER, recv_from_rank, &recv_str); + auto recv_req_id = server_node_->CollectiveReceiveAsync(ps::core::NodeRole::SERVER, recv_from_rank, &recv_str); if (!server_node_->CollectiveWait(recv_req_id)) { MS_LOG(ERROR) << "CollectiveWait " << recv_req_id << " failed."; return false; } - memcpy_s(recv_chunk, chunk_sizes[recv_chunk_index] * sizeof(T), recv_str->data(), recv_str->size()); + ret = memcpy_s(recv_chunk, chunk_sizes[recv_chunk_index] * sizeof(T), recv_str->data(), recv_str->size()); + if (ret != 0) { + MS_LOG(ERROR) << "memcpy_s error, errorno(" << ret << ")"; + return false; + } if (!server_node_->Wait(send_req_id, 1)) { MS_LOG(ERROR) << "CollectiveWait " << send_req_id << " failed."; return false; @@ -143,19 +151,23 @@ bool CollectiveOpsImpl::ReduceBroadcastAllReduce(const void *sendbuff, void *rec for (uint32_t i = 1; i < rank_size; i++) { std::shared_ptr> recv_str; MS_LOG(DEBUG) << "Reduce rank 0 receive from rank " << i; - auto recv_req_id = server_node_->CollectiveReceiveAsync(core::NodeRole::SERVER, i, &recv_str); + auto recv_req_id = server_node_->CollectiveReceiveAsync(ps::core::NodeRole::SERVER, i, &recv_str); if (!server_node_->CollectiveWait(recv_req_id)) { MS_LOG(ERROR) << "CollectiveWait " << recv_req_id << " failed."; return false; } - memcpy_s(tmp_recv_buff.get(), count * sizeof(T), recv_str->data(), recv_str->size()); + ret = memcpy_s(tmp_recv_buff.get(), count * sizeof(T), recv_str->data(), recv_str->size()); + if (ret != 0) { + MS_LOG(ERROR) << "memcpy_s error, errorno(" << ret << ")"; + return false; + } for (size_t j = 0; j < count; j++) { output_buff[j] += tmp_recv_buff[j]; } } } else { MS_LOG(DEBUG) << "Reduce send data to rank 0 process."; - auto send_req_id = server_node_->CollectiveSendAsync(core::NodeRole::SERVER, 0, sendbuff, count * sizeof(T)); + auto send_req_id = server_node_->CollectiveSendAsync(ps::core::NodeRole::SERVER, 0, sendbuff, count * sizeof(T)); if (!server_node_->Wait(send_req_id)) { MS_LOG(ERROR) << "CollectiveWait " << send_req_id << " failed."; return false; @@ -168,7 +180,8 @@ bool CollectiveOpsImpl::ReduceBroadcastAllReduce(const void *sendbuff, void *rec if (local_rank_ == 0) { for (uint32_t i = 1; i < rank_size; i++) { MS_LOG(DEBUG) << "Broadcast data to process " << i; - auto send_req_id = server_node_->CollectiveSendAsync(core::NodeRole::SERVER, i, output_buff, count * sizeof(T)); + auto send_req_id = + server_node_->CollectiveSendAsync(ps::core::NodeRole::SERVER, i, output_buff, count * sizeof(T)); if (!server_node_->Wait(send_req_id)) { MS_LOG(ERROR) << "CollectiveWait " << send_req_id << " failed."; return false; @@ -177,12 +190,16 @@ bool CollectiveOpsImpl::ReduceBroadcastAllReduce(const void *sendbuff, void *rec } else { MS_LOG(DEBUG) << "Broadcast receive from rank 0."; std::shared_ptr> recv_str; - auto recv_req_id = server_node_->CollectiveReceiveAsync(core::NodeRole::SERVER, 0, &recv_str); + auto recv_req_id = server_node_->CollectiveReceiveAsync(ps::core::NodeRole::SERVER, 0, &recv_str); if (!server_node_->CollectiveWait(recv_req_id)) { MS_LOG(ERROR) << "CollectiveWait " << recv_req_id << " failed."; return false; } - memcpy_s(output_buff, count * sizeof(T), recv_str->data(), recv_str->size()); + ret = memcpy_s(output_buff, count * sizeof(T), recv_str->data(), recv_str->size()); + if (ret != 0) { + MS_LOG(ERROR) << "memcpy_s error, errorno(" << ret << ")"; + return false; + } } MS_LOG(DEBUG) << "End broadcast."; return true; @@ -231,5 +248,5 @@ template bool CollectiveOpsImpl::AllReduce(const void *sendbuff, void *re template bool CollectiveOpsImpl::AllReduce(const void *sendbuff, void *recvbuff, size_t count); template bool CollectiveOpsImpl::AllReduce(const void *sendbuff, void *recvbuff, size_t count); } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/collective_ops_impl.h b/mindspore/ccsrc/fl/server/collective_ops_impl.h index 5bd15b8d735..1e77d9da018 100644 --- a/mindspore/ccsrc/fl/server/collective_ops_impl.h +++ b/mindspore/ccsrc/fl/server/collective_ops_impl.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_COLLECTIVE_OPS_IMPL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_COLLECTIVE_OPS_IMPL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_COLLECTIVE_OPS_IMPL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_COLLECTIVE_OPS_IMPL_H_ #include #include @@ -27,7 +27,7 @@ #include "fl/server/common.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { // CollectiveOpsImpl is the collective communication API of the server. // For now, it implements two AllReduce algorithms: RingAllReduce and BroadcastAllReduce. Elastic AllReduce is also @@ -39,7 +39,7 @@ class CollectiveOpsImpl { return instance; } - void Initialize(const std::shared_ptr &server_node); + void Initialize(const std::shared_ptr &server_node); template bool AllReduce(const void *sendbuff, void *recvbuff, size_t count); @@ -48,7 +48,7 @@ class CollectiveOpsImpl { bool ReInitForScaling(); private: - CollectiveOpsImpl() = default; + CollectiveOpsImpl() : server_node_(nullptr), local_rank_(0), server_num_(0) {} ~CollectiveOpsImpl() = default; CollectiveOpsImpl(const CollectiveOpsImpl &) = delete; CollectiveOpsImpl &operator=(const CollectiveOpsImpl &) = delete; @@ -61,7 +61,7 @@ class CollectiveOpsImpl { template bool ReduceBroadcastAllReduce(const void *sendbuff, void *recvbuff, size_t count); - std::shared_ptr server_node_; + std::shared_ptr server_node_; uint32_t local_rank_; uint32_t server_num_; @@ -69,6 +69,6 @@ class CollectiveOpsImpl { std::mutex mtx_; }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_COLLECTIVE_OPS_IMPL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_COLLECTIVE_OPS_IMPL_H_ diff --git a/mindspore/ccsrc/fl/server/common.h b/mindspore/ccsrc/fl/server/common.h index 83b28bf9f82..c958808ccf3 100644 --- a/mindspore/ccsrc/fl/server/common.h +++ b/mindspore/ccsrc/fl/server/common.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_COMMON_H_ -#define MINDSPORE_CCSRC_PS_SERVER_COMMON_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_COMMON_H_ +#define MINDSPORE_CCSRC_FL_SERVER_COMMON_H_ #include #include @@ -37,7 +37,7 @@ #include "ps/core/communicator/message_handler.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { // Definitions for the server framework. enum ServerMode { PARAMETER_SERVER = 0, FL_SERVER }; @@ -73,7 +73,7 @@ using TimeOutCb = std::function; using StopTimerCb = std::function; using FinishIterCb = std::function; using FinalizeCb = std::function; -using MessageCallback = std::function &)>; +using MessageCallback = std::function &)>; // Information about whether server kernel will reuse kernel node memory from the front end. // Key refers to the server kernel's parameter name, like "weights", "grad", "learning_rate". @@ -237,6 +237,6 @@ inline AddressPtr GenerateParameterNodeAddrPtr(const CNodePtr &kernel_node, size // Definitions for Parameter Server. } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_COMMON_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_COMMON_H_ diff --git a/mindspore/ccsrc/fl/server/consistent_hash_ring.cc b/mindspore/ccsrc/fl/server/consistent_hash_ring.cc index 2b477885678..ef70a009ae3 100644 --- a/mindspore/ccsrc/fl/server/consistent_hash_ring.cc +++ b/mindspore/ccsrc/fl/server/consistent_hash_ring.cc @@ -17,7 +17,7 @@ #include "fl/server/consistent_hash_ring.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { bool ConsistentHashRing::Insert(uint32_t rank) { for (uint32_t i = 0; i < virtual_node_num_; i++) { @@ -53,5 +53,5 @@ uint32_t ConsistentHashRing::Find(const std::string &key) { return iterator->second; } } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/consistent_hash_ring.h b/mindspore/ccsrc/fl/server/consistent_hash_ring.h index c3e6d5223bb..ddf19e66620 100644 --- a/mindspore/ccsrc/fl/server/consistent_hash_ring.h +++ b/mindspore/ccsrc/fl/server/consistent_hash_ring.h @@ -14,15 +14,15 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_CONSISTENT_HASH_RING_H_ -#define MINDSPORE_CCSRC_PS_SERVER_CONSISTENT_HASH_RING_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_CONSISTENT_HASH_RING_H_ +#define MINDSPORE_CCSRC_FL_SERVER_CONSISTENT_HASH_RING_H_ #include #include #include "utils/log_adapter.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { // To support distributed storage and make servers easy to scale-out and scale-in for a large load of metadata in // server, we use class ConsistentHashRing to help servers find out which metadata is stored in which server node. @@ -59,6 +59,6 @@ class ConsistentHashRing { std::map ring_; }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_CONSISTENT_HASH_RING_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_CONSISTENT_HASH_RING_H_ diff --git a/mindspore/ccsrc/fl/server/distributed_count_service.cc b/mindspore/ccsrc/fl/server/distributed_count_service.cc index 644a728be5d..7a6ee018da2 100644 --- a/mindspore/ccsrc/fl/server/distributed_count_service.cc +++ b/mindspore/ccsrc/fl/server/distributed_count_service.cc @@ -20,19 +20,19 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { -void DistributedCountService::Initialize(const std::shared_ptr &server_node, +void DistributedCountService::Initialize(const std::shared_ptr &server_node, uint32_t counting_server_rank) { server_node_ = server_node; MS_EXCEPTION_IF_NULL(server_node_); local_rank_ = server_node_->rank_id(); - server_num_ = PSContext::instance()->initial_server_num(); + server_num_ = ps::PSContext::instance()->initial_server_num(); counting_server_rank_ = counting_server_rank; return; } -void DistributedCountService::RegisterMessageCallback(const std::shared_ptr &communicator) { +void DistributedCountService::RegisterMessageCallback(const std::shared_ptr &communicator) { communicator_ = communicator; MS_EXCEPTION_IF_NULL(communicator_); communicator_->RegisterMsgCallBack( @@ -94,14 +94,14 @@ bool DistributedCountService::Count(const std::string &name, const std::string & report_count_req.set_id(id); std::shared_ptr> report_cnt_rsp_msg = nullptr; - if (!communicator_->SendPbRequest(report_count_req, counting_server_rank_, core::TcpUserCommand::kCount, + if (!communicator_->SendPbRequest(report_count_req, counting_server_rank_, ps::core::TcpUserCommand::kCount, &report_cnt_rsp_msg)) { MS_LOG(ERROR) << "Sending reporting count message to leader server failed for " << name; return false; } CountResponse count_rsp; - count_rsp.ParseFromArray(report_cnt_rsp_msg->data(), report_cnt_rsp_msg->size()); + count_rsp.ParseFromArray(report_cnt_rsp_msg->data(), SizeToInt(report_cnt_rsp_msg->size())); if (!count_rsp.result()) { MS_LOG(ERROR) << "Reporting count failed:" << count_rsp.reason(); return false; @@ -126,13 +126,14 @@ bool DistributedCountService::CountReachThreshold(const std::string &name) { std::shared_ptr> query_cnt_enough_rsp_msg = nullptr; if (!communicator_->SendPbRequest(count_reach_threshold_req, counting_server_rank_, - core::TcpUserCommand::kReachThreshold, &query_cnt_enough_rsp_msg)) { + ps::core::TcpUserCommand::kReachThreshold, &query_cnt_enough_rsp_msg)) { MS_LOG(ERROR) << "Sending querying whether count reaches threshold message to leader server failed for " << name; return false; } CountReachThresholdResponse count_reach_threshold_rsp; - count_reach_threshold_rsp.ParseFromArray(query_cnt_enough_rsp_msg->data(), query_cnt_enough_rsp_msg->size()); + count_reach_threshold_rsp.ParseFromArray(query_cnt_enough_rsp_msg->data(), + SizeToInt(query_cnt_enough_rsp_msg->size())); return count_reach_threshold_rsp.is_enough(); } } @@ -164,14 +165,14 @@ bool DistributedCountService::ReInitForScaling() { return true; } -void DistributedCountService::HandleCountRequest(const std::shared_ptr &message) { +void DistributedCountService::HandleCountRequest(const std::shared_ptr &message) { if (message == nullptr) { MS_LOG(ERROR) << "Message is nullptr."; return; } CountRequest report_count_req; - report_count_req.ParseFromArray(message->data(), message->len()); + report_count_req.ParseFromArray(message->data(), SizeToInt(message->len())); const std::string &name = report_count_req.name(); const std::string &id = report_count_req.id(); @@ -213,14 +214,15 @@ void DistributedCountService::HandleCountRequest(const std::shared_ptr &message) { +void DistributedCountService::HandleCountReachThresholdRequest( + const std::shared_ptr &message) { if (message == nullptr) { MS_LOG(ERROR) << "Message is nullptr."; return; } CountReachThresholdRequest count_reach_threshold_req; - count_reach_threshold_req.ParseFromArray(message->data(), message->len()); + count_reach_threshold_req.ParseFromArray(message->data(), SizeToInt(message->len())); const std::string &name = count_reach_threshold_req.name(); std::unique_lock lock(mutex_[name]); @@ -236,7 +238,7 @@ void DistributedCountService::HandleCountReachThresholdRequest(const std::shared return; } -void DistributedCountService::HandleCounterEvent(const std::shared_ptr &message) { +void DistributedCountService::HandleCounterEvent(const std::shared_ptr &message) { if (message == nullptr) { MS_LOG(ERROR) << "Message is nullptr."; return; @@ -248,7 +250,7 @@ void DistributedCountService::HandleCounterEvent(const std::shared_ptrSendResponse(couter_event_rsp_msg.data(), couter_event_rsp_msg.size(), message); CounterEvent counter_event; - counter_event.ParseFromArray(message->data(), message->len()); + counter_event.ParseFromArray(message->data(), SizeToInt(message->len())); const auto &type = counter_event.type(); const auto &name = counter_event.name(); @@ -289,7 +291,7 @@ bool DistributedCountService::TriggerFirstCountEvent(const std::string &name) { // Broadcast to all follower servers. for (uint32_t i = 1; i < server_num_; i++) { - if (!communicator_->SendPbRequest(first_count_event, i, core::TcpUserCommand::kCounterEvent)) { + if (!communicator_->SendPbRequest(first_count_event, i, ps::core::TcpUserCommand::kCounterEvent)) { MS_LOG(ERROR) << "Activating first count event to server " << i << " failed."; return false; } @@ -307,7 +309,7 @@ bool DistributedCountService::TriggerLastCountEvent(const std::string &name) { // Broadcast to all follower servers. for (uint32_t i = 1; i < server_num_; i++) { - if (!communicator_->SendPbRequest(last_count_event, i, core::TcpUserCommand::kCounterEvent)) { + if (!communicator_->SendPbRequest(last_count_event, i, ps::core::TcpUserCommand::kCounterEvent)) { MS_LOG(ERROR) << "Activating last count event to server " << i << " failed."; return false; } @@ -317,5 +319,5 @@ bool DistributedCountService::TriggerLastCountEvent(const std::string &name) { return true; } } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/distributed_count_service.h b/mindspore/ccsrc/fl/server/distributed_count_service.h index 4df8af528a7..2e37bbd9b8f 100644 --- a/mindspore/ccsrc/fl/server/distributed_count_service.h +++ b/mindspore/ccsrc/fl/server/distributed_count_service.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_DISTRIBUTED_COUNT_SERVICE_H_ -#define MINDSPORE_CCSRC_PS_SERVER_DISTRIBUTED_COUNT_SERVICE_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_DISTRIBUTED_COUNT_SERVICE_H_ +#define MINDSPORE_CCSRC_FL_SERVER_DISTRIBUTED_COUNT_SERVICE_H_ #include #include @@ -27,7 +27,7 @@ #include "ps/core/communicator/tcp_communicator.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { constexpr uint32_t kDefaultCountingServerRank = 0; constexpr auto kModuleDistributedCountService = "DistributedCountService"; @@ -54,10 +54,10 @@ class DistributedCountService { } // Initialize counter service with the server node because communication is needed. - void Initialize(const std::shared_ptr &server_node, uint32_t counting_server_rank); + void Initialize(const std::shared_ptr &server_node, uint32_t counting_server_rank); // Register message callbacks of the counting server to handle messages sent by the other servers. - void RegisterMessageCallback(const std::shared_ptr &communicator); + void RegisterMessageCallback(const std::shared_ptr &communicator); // Register counter to the counting server for the name with its threshold count in server cluster dimension and // first/last count event callbacks. @@ -87,15 +87,15 @@ class DistributedCountService { DistributedCountService &operator=(const DistributedCountService &) = delete; // Callback for the reporting count message from other servers. Only counting server will call this method. - void HandleCountRequest(const std::shared_ptr &message); + void HandleCountRequest(const std::shared_ptr &message); // Callback for the querying whether threshold count is reached message from other servers. Only counting // server will call this method. - void HandleCountReachThresholdRequest(const std::shared_ptr &message); + void HandleCountReachThresholdRequest(const std::shared_ptr &message); // Callback for the first/last event message from the counting server. Only other servers will call this // method. - void HandleCounterEvent(const std::shared_ptr &message); + void HandleCounterEvent(const std::shared_ptr &message); // Call the callbacks when the first/last count event is triggered. bool TriggerCounterEvent(const std::string &name); @@ -103,8 +103,8 @@ class DistributedCountService { bool TriggerLastCountEvent(const std::string &name); // Members for the communication between counting server and other servers. - std::shared_ptr server_node_; - std::shared_ptr communicator_; + std::shared_ptr server_node_; + std::shared_ptr communicator_; uint32_t local_rank_; uint32_t server_num_; @@ -126,6 +126,6 @@ class DistributedCountService { std::unordered_map mutex_; }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_DISTRIBUTED_COUNT_SERVICE_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_DISTRIBUTED_COUNT_SERVICE_H_ diff --git a/mindspore/ccsrc/fl/server/distributed_metadata_store.cc b/mindspore/ccsrc/fl/server/distributed_metadata_store.cc index 09d1d6da024..b15af8ec604 100644 --- a/mindspore/ccsrc/fl/server/distributed_metadata_store.cc +++ b/mindspore/ccsrc/fl/server/distributed_metadata_store.cc @@ -20,18 +20,18 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { -void DistributedMetadataStore::Initialize(const std::shared_ptr &server_node) { +void DistributedMetadataStore::Initialize(const std::shared_ptr &server_node) { server_node_ = server_node; MS_EXCEPTION_IF_NULL(server_node); local_rank_ = server_node_->rank_id(); - server_num_ = PSContext::instance()->initial_server_num(); + server_num_ = ps::PSContext::instance()->initial_server_num(); InitHashRing(); return; } -void DistributedMetadataStore::RegisterMessageCallback(const std::shared_ptr &communicator) { +void DistributedMetadataStore::RegisterMessageCallback(const std::shared_ptr &communicator) { communicator_ = communicator; MS_EXCEPTION_IF_NULL(communicator_); communicator_->RegisterMsgCallBack( @@ -100,7 +100,7 @@ bool DistributedMetadataStore::UpdateMetadata(const std::string &name, const PBM metadata_with_name.set_name(name); *metadata_with_name.mutable_metadata() = meta; std::shared_ptr> update_meta_rsp_msg = nullptr; - if (!communicator_->SendPbRequest(metadata_with_name, stored_rank, core::TcpUserCommand::kUpdateMetadata, + if (!communicator_->SendPbRequest(metadata_with_name, stored_rank, ps::core::TcpUserCommand::kUpdateMetadata, &update_meta_rsp_msg)) { MS_LOG(ERROR) << "Sending updating metadata message to server " << stored_rank << " failed."; return false; @@ -133,12 +133,12 @@ PBMetadata DistributedMetadataStore::GetMetadata(const std::string &name) { PBMetadata get_metadata_rsp; std::shared_ptr> get_meta_rsp_msg = nullptr; - if (!communicator_->SendPbRequest(get_metadata_req, stored_rank, core::TcpUserCommand::kGetMetadata, + if (!communicator_->SendPbRequest(get_metadata_req, stored_rank, ps::core::TcpUserCommand::kGetMetadata, &get_meta_rsp_msg)) { MS_LOG(ERROR) << "Sending getting metadata message to server " << stored_rank << " failed."; return get_metadata_rsp; } - get_metadata_rsp.ParseFromArray(get_meta_rsp_msg->data(), get_meta_rsp_msg->size()); + get_metadata_rsp.ParseFromArray(get_meta_rsp_msg->data(), SizeToInt(get_meta_rsp_msg->size())); return get_metadata_rsp; } } @@ -174,14 +174,14 @@ void DistributedMetadataStore::InitHashRing() { return; } -void DistributedMetadataStore::HandleUpdateMetadataRequest(const std::shared_ptr &message) { +void DistributedMetadataStore::HandleUpdateMetadataRequest(const std::shared_ptr &message) { if (message == nullptr) { MS_LOG(ERROR) << "Message is nullptr."; return; } PBMetadataWithName meta_with_name; - meta_with_name.ParseFromArray(message->data(), message->len()); + meta_with_name.ParseFromArray(message->data(), SizeToInt(message->len())); const std::string &name = meta_with_name.name(); MS_LOG(INFO) << "Update metadata for " << name; @@ -196,7 +196,7 @@ void DistributedMetadataStore::HandleUpdateMetadataRequest(const std::shared_ptr return; } -void DistributedMetadataStore::HandleGetMetadataRequest(const std::shared_ptr &message) { +void DistributedMetadataStore::HandleGetMetadataRequest(const std::shared_ptr &message) { if (message == nullptr) { MS_LOG(ERROR) << "Message is nullptr."; return; @@ -267,7 +267,7 @@ bool DistributedMetadataStore::DoUpdateMetadata(const std::string &name, const P auto &client_shares_map = *metadata_[name].mutable_client_shares()->mutable_client_secret_shares(); auto &fl_id = meta.pair_client_shares().fl_id(); auto &client_shares = meta.pair_client_shares().client_shares(); - // google::protobuf::Map< std::string, mindspore::ps::core::SharesPb >::const_iterator iter; + // google::protobuf::Map< std::string, mindspore::fl::ps::core::SharesPb >::const_iterator iter; // Check whether the new item already exists. bool add_flag = true; for (auto iter = client_shares_map.begin(); iter != client_shares_map.end(); iter++) { @@ -299,5 +299,5 @@ bool DistributedMetadataStore::DoUpdateMetadata(const std::string &name, const P return true; } } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/distributed_metadata_store.h b/mindspore/ccsrc/fl/server/distributed_metadata_store.h index 0559401fc44..bf13c13ea19 100644 --- a/mindspore/ccsrc/fl/server/distributed_metadata_store.h +++ b/mindspore/ccsrc/fl/server/distributed_metadata_store.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_DISTRIBUTED_META_STORE_H_ -#define MINDSPORE_CCSRC_PS_SERVER_DISTRIBUTED_META_STORE_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_DISTRIBUTED_META_STORE_H_ +#define MINDSPORE_CCSRC_FL_SERVER_DISTRIBUTED_META_STORE_H_ #include #include @@ -27,7 +27,7 @@ #include "fl/server/consistent_hash_ring.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { constexpr auto kModuleDistributedMetadataStore = "DistributedMetadataStore"; // This class is used for distributed metadata storage using consistent hash. All metadata is distributedly @@ -44,10 +44,10 @@ class DistributedMetadataStore { } // Initialize metadata storage with the server node because communication is needed. - void Initialize(const std::shared_ptr &server_node); + void Initialize(const std::shared_ptr &server_node); // Register callbacks for the server to handle update/get metadata messages from other servers. - void RegisterMessageCallback(const std::shared_ptr &communicator); + void RegisterMessageCallback(const std::shared_ptr &communicator); // Register metadata for the name with the initial value. This method should be only called once for each name. void RegisterMetadata(const std::string &name, const PBMetadata &meta); @@ -65,7 +65,13 @@ class DistributedMetadataStore { bool ReInitForScaling(); private: - DistributedMetadataStore() = default; + DistributedMetadataStore() + : server_node_(nullptr), + communicator_(nullptr), + local_rank_(0), + server_num_(0), + router_(nullptr), + metadata_({}) {} ~DistributedMetadataStore() = default; DistributedMetadataStore(const DistributedMetadataStore &) = delete; DistributedMetadataStore &operator=(const DistributedMetadataStore &) = delete; @@ -74,17 +80,17 @@ class DistributedMetadataStore { void InitHashRing(); // Callback for updating metadata request sent to the server. - void HandleUpdateMetadataRequest(const std::shared_ptr &message); + void HandleUpdateMetadataRequest(const std::shared_ptr &message); // Callback for getting metadata request sent to the server. - void HandleGetMetadataRequest(const std::shared_ptr &message); + void HandleGetMetadataRequest(const std::shared_ptr &message); // Do updating metadata in the server where the metadata for the name is stored. bool DoUpdateMetadata(const std::string &name, const PBMetadata &meta); // Members for the communication between servers. - std::shared_ptr server_node_; - std::shared_ptr communicator_; + std::shared_ptr server_node_; + std::shared_ptr communicator_; uint32_t local_rank_; uint32_t server_num_; @@ -100,6 +106,6 @@ class DistributedMetadataStore { std::unordered_map mutex_; }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_DISTRIBUTED_META_STORE_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_DISTRIBUTED_META_STORE_H_ diff --git a/mindspore/ccsrc/fl/server/executor.cc b/mindspore/ccsrc/fl/server/executor.cc index fb5db0559c4..151a48014d6 100644 --- a/mindspore/ccsrc/fl/server/executor.cc +++ b/mindspore/ccsrc/fl/server/executor.cc @@ -21,7 +21,7 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { void Executor::Initialize(const FuncGraphPtr &func_graph, size_t aggregation_count) { MS_EXCEPTION_IF_NULL(func_graph); @@ -320,5 +320,5 @@ bool Executor::InitParamAggregator(const FuncGraphPtr &func_graph) { return true; } } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/executor.h b/mindspore/ccsrc/fl/server/executor.h index f2bd643a835..462f967aab4 100644 --- a/mindspore/ccsrc/fl/server/executor.h +++ b/mindspore/ccsrc/fl/server/executor.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_EXECUTOR_H_ -#define MINDSPORE_CCSRC_PS_SERVER_EXECUTOR_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_EXECUTOR_H_ +#define MINDSPORE_CCSRC_FL_SERVER_EXECUTOR_H_ #include #include @@ -31,7 +31,7 @@ #endif namespace mindspore { -namespace ps { +namespace fl { namespace server { // Executor is the entrance for server to handle aggregation, optimizing, model querying, etc. It handles // logics relevant to kernel launching. @@ -94,7 +94,7 @@ class Executor { bool Unmask(); private: - Executor() {} + Executor() : initialized_(false), aggregation_count_(0), param_names_({}), param_aggrs_({}) {} ~Executor() = default; Executor(const Executor &) = delete; Executor &operator=(const Executor &) = delete; @@ -126,6 +126,6 @@ class Executor { #endif }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_EXECUTOR_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_EXECUTOR_H_ diff --git a/mindspore/ccsrc/fl/server/iteration.cc b/mindspore/ccsrc/fl/server/iteration.cc index fb3aea1f4bc..ad394ed723f 100644 --- a/mindspore/ccsrc/fl/server/iteration.cc +++ b/mindspore/ccsrc/fl/server/iteration.cc @@ -23,10 +23,10 @@ #include "fl/server/server.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { class Server; -void Iteration::RegisterMessageCallback(const std::shared_ptr &communicator) { +void Iteration::RegisterMessageCallback(const std::shared_ptr &communicator) { MS_EXCEPTION_IF_NULL(communicator); communicator_ = communicator; communicator_->RegisterMsgCallBack("syncIteration", @@ -42,12 +42,12 @@ void Iteration::RegisterMessageCallback(const std::shared_ptr &server_node) { +void Iteration::RegisterEventCallback(const std::shared_ptr &server_node) { MS_EXCEPTION_IF_NULL(server_node); server_node_ = server_node; - server_node->RegisterCustomEventCallback(static_cast(CustomEvent::kIterationRunning), + server_node->RegisterCustomEventCallback(static_cast(ps::CustomEvent::kIterationRunning), std::bind(&Iteration::HandleIterationRunningEvent, this)); - server_node->RegisterCustomEventCallback(static_cast(CustomEvent::kIterationCompleted), + server_node->RegisterCustomEventCallback(static_cast(ps::CustomEvent::kIterationCompleted), std::bind(&Iteration::HandleIterationCompletedEvent, this)); } @@ -56,7 +56,7 @@ void Iteration::AddRound(const std::shared_ptr &round) { rounds_.push_back(round); } -void Iteration::InitRounds(const std::vector> &communicators, +void Iteration::InitRounds(const std::vector> &communicators, const TimeOutCb &timeout_cb, const FinishIterCb &finish_iteration_cb) { if (communicators.empty()) { MS_LOG(EXCEPTION) << "Communicators for rounds is empty."; @@ -64,7 +64,7 @@ void Iteration::InitRounds(const std::vector &communicator) { + [&](const std::shared_ptr &communicator) { for (auto &round : rounds_) { if (round == nullptr) { continue; @@ -120,7 +120,7 @@ void Iteration::SetIterationRunning() { } if (server_node_->rank_id() == kLeaderServerRank) { // This event helps worker/server to be consistent in iteration state. - server_node_->BroadcastEvent(static_cast(CustomEvent::kIterationRunning)); + server_node_->BroadcastEvent(static_cast(ps::CustomEvent::kIterationRunning)); } iteration_state_ = IterationState::kRunning; } @@ -133,7 +133,7 @@ void Iteration::SetIterationCompleted() { } if (server_node_->rank_id() == kLeaderServerRank) { // This event helps worker/server to be consistent in iteration state. - server_node_->BroadcastEvent(static_cast(CustomEvent::kIterationCompleted)); + server_node_->BroadcastEvent(static_cast(ps::CustomEvent::kIterationCompleted)); } iteration_state_ = IterationState::kCompleted; } @@ -171,7 +171,7 @@ bool Iteration::SyncIteration(uint32_t rank) { sync_iter_req.set_rank(rank); std::shared_ptr> sync_iter_rsp_msg = nullptr; - if (!communicator_->SendPbRequest(sync_iter_req, kLeaderServerRank, core::TcpUserCommand::kSyncIteration, + if (!communicator_->SendPbRequest(sync_iter_req, kLeaderServerRank, ps::core::TcpUserCommand::kSyncIteration, &sync_iter_rsp_msg)) { MS_LOG(ERROR) << "Sending synchronizing iteration message to leader server failed."; return false; @@ -189,7 +189,7 @@ bool Iteration::SyncIteration(uint32_t rank) { return true; } -void Iteration::HandleSyncIterationRequest(const std::shared_ptr &message) { +void Iteration::HandleSyncIterationRequest(const std::shared_ptr &message) { if (message == nullptr) { MS_LOG(ERROR) << "Message is nullptr."; return; @@ -224,14 +224,14 @@ bool Iteration::NotifyLeaderMoveToNextIteration(bool is_last_iter_valid, const s notify_leader_to_next_iter_req.set_iter_num(iteration_num_); notify_leader_to_next_iter_req.set_reason(reason); if (!communicator_->SendPbRequest(notify_leader_to_next_iter_req, kLeaderServerRank, - core::TcpUserCommand::kNotifyLeaderToNextIter)) { + ps::core::TcpUserCommand::kNotifyLeaderToNextIter)) { MS_LOG(WARNING) << "Sending notify leader server to proceed next iteration request to leader server 0 failed."; return false; } return true; } -void Iteration::HandleNotifyLeaderMoveToNextIterRequest(const std::shared_ptr &message) { +void Iteration::HandleNotifyLeaderMoveToNextIterRequest(const std::shared_ptr &message) { if (message == nullptr) { return; } @@ -278,7 +278,7 @@ bool Iteration::BroadcastPrepareForNextIterRequest(bool is_last_iter_valid, cons std::vector offline_servers = {}; for (uint32_t i = 1; i < IntToUint(server_node_->server_num()); i++) { - if (!communicator_->SendPbRequest(prepare_next_iter_req, i, core::TcpUserCommand::kPrepareForNextIter)) { + if (!communicator_->SendPbRequest(prepare_next_iter_req, i, ps::core::TcpUserCommand::kPrepareForNextIter)) { MS_LOG(WARNING) << "Sending prepare for next iteration request to server " << i << " failed. Retry later."; offline_servers.push_back(i); continue; @@ -289,17 +289,18 @@ bool Iteration::BroadcastPrepareForNextIterRequest(bool is_last_iter_valid, cons std::for_each(offline_servers.begin(), offline_servers.end(), [&](uint32_t rank) { // Should avoid endless loop if the server communicator is stopped. while (communicator_->running() && - !communicator_->SendPbRequest(prepare_next_iter_req, rank, core::TcpUserCommand::kPrepareForNextIter)) { + !communicator_->SendPbRequest(prepare_next_iter_req, rank, ps::core::TcpUserCommand::kPrepareForNextIter)) { MS_LOG(WARNING) << "Retry sending prepare for next iteration request to server " << rank << " failed. The server has not recovered yet."; std::this_thread::sleep_for(std::chrono::milliseconds(kRetryDurationForPrepareForNextIter)); } MS_LOG(INFO) << "Offline server " << rank << " preparing for next iteration success."; }); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); return true; } -void Iteration::HandlePrepareForNextIterRequest(const std::shared_ptr &message) { +void Iteration::HandlePrepareForNextIterRequest(const std::shared_ptr &message) { if (message == nullptr) { return; } @@ -329,7 +330,7 @@ bool Iteration::BroadcastMoveToNextIterRequest(bool is_last_iter_valid, const st proceed_to_next_iter_req.set_last_iter_num(iteration_num_); proceed_to_next_iter_req.set_reason(reason); for (uint32_t i = 1; i < IntToUint(server_node_->server_num()); i++) { - if (!communicator_->SendPbRequest(proceed_to_next_iter_req, i, core::TcpUserCommand::kProceedToNextIter)) { + if (!communicator_->SendPbRequest(proceed_to_next_iter_req, i, ps::core::TcpUserCommand::kProceedToNextIter)) { MS_LOG(WARNING) << "Sending proceed to next iteration request to server " << i << " failed."; continue; } @@ -339,7 +340,7 @@ bool Iteration::BroadcastMoveToNextIterRequest(bool is_last_iter_valid, const st return true; } -void Iteration::HandleMoveToNextIterRequest(const std::shared_ptr &message) { +void Iteration::HandleMoveToNextIterRequest(const std::shared_ptr &message) { if (message == nullptr) { return; } @@ -388,7 +389,7 @@ bool Iteration::BroadcastEndLastIterRequest(uint64_t last_iter_num) { EndLastIterRequest end_last_iter_req; end_last_iter_req.set_last_iter_num(last_iter_num); for (uint32_t i = 1; i < IntToUint(server_node_->server_num()); i++) { - if (!communicator_->SendPbRequest(end_last_iter_req, i, core::TcpUserCommand::kEndLastIter)) { + if (!communicator_->SendPbRequest(end_last_iter_req, i, ps::core::TcpUserCommand::kEndLastIter)) { MS_LOG(WARNING) << "Sending proceed to next iteration request to server " << i << " failed."; continue; } @@ -398,7 +399,7 @@ bool Iteration::BroadcastEndLastIterRequest(uint64_t last_iter_num) { return true; } -void Iteration::HandleEndLastIterRequest(const std::shared_ptr &message) { +void Iteration::HandleEndLastIterRequest(const std::shared_ptr &message) { if (message == nullptr) { return; } @@ -429,9 +430,9 @@ void Iteration::EndLastIter() { MS_LOG(INFO) << "End the last iteration " << iteration_num_; iteration_num_++; // After the job is done, reset the iteration to the initial number and reset ModelStore. - if (iteration_num_ > PSContext::instance()->fl_iteration_num()) { + if (iteration_num_ > ps::PSContext::instance()->fl_iteration_num()) { MS_LOG(INFO) << "Iteration loop " << iteration_loop_count_ - << " is completed. Iteration number: " << PSContext::instance()->fl_iteration_num(); + << " is completed. Iteration number: " << ps::PSContext::instance()->fl_iteration_num(); iteration_num_ = 1; iteration_loop_count_++; ModelStore::GetInstance().Reset(); @@ -444,5 +445,5 @@ void Iteration::EndLastIter() { MS_LOG(INFO) << "Move to next iteration:" << iteration_num_ << "\n"; } } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/iteration.h b/mindspore/ccsrc/fl/server/iteration.h index 3f30a51d84e..6e8f89de713 100644 --- a/mindspore/ccsrc/fl/server/iteration.h +++ b/mindspore/ccsrc/fl/server/iteration.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_ITERATION_H_ -#define MINDSPORE_CCSRC_PS_SERVER_ITERATION_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_ITERATION_H_ +#define MINDSPORE_CCSRC_FL_SERVER_ITERATION_H_ #include #include @@ -26,7 +26,7 @@ #include "fl/server/local_meta_store.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { enum class IterationState { // This iteration is still in process. @@ -48,16 +48,16 @@ class Iteration { } // Register callbacks for other servers to synchronize iteration information from leader server. - void RegisterMessageCallback(const std::shared_ptr &communicator); + void RegisterMessageCallback(const std::shared_ptr &communicator); // Register event callbacks for iteration state synchronization. - void RegisterEventCallback(const std::shared_ptr &server_node); + void RegisterEventCallback(const std::shared_ptr &server_node); // Add a round for the iteration. This method will be called multiple times for each round. void AddRound(const std::shared_ptr &round); // Initialize all the rounds in the iteration. - void InitRounds(const std::vector> &communicators, + void InitRounds(const std::vector> &communicators, const TimeOutCb &timeout_cb, const FinishIterCb &finish_iteration_cb); // This method will control servers to proceed to next iteration. @@ -104,7 +104,7 @@ class Iteration { // Synchronize iteration form the leader server(Rank 0). bool SyncIteration(uint32_t rank); - void HandleSyncIterationRequest(const std::shared_ptr &message); + void HandleSyncIterationRequest(const std::shared_ptr &message); // The request for moving to next iteration is not reentrant. bool IsMoveToNextIterRequestReentrant(uint64_t iteration_num); @@ -112,28 +112,28 @@ class Iteration { // The methods for moving to next iteration for all the servers. // Step 1: follower servers notify leader server that they need to move to next iteration. bool NotifyLeaderMoveToNextIteration(bool is_last_iter_valid, const std::string &reason); - void HandleNotifyLeaderMoveToNextIterRequest(const std::shared_ptr &message); + void HandleNotifyLeaderMoveToNextIterRequest(const std::shared_ptr &message); // Step 2: leader server broadcast to all follower servers to prepare for next iteration and switch to safemode. bool BroadcastPrepareForNextIterRequest(bool is_last_iter_valid, const std::string &reason); - void HandlePrepareForNextIterRequest(const std::shared_ptr &message); + void HandlePrepareForNextIterRequest(const std::shared_ptr &message); // The server prepare for the next iteration. This method will switch the server to safemode. void PrepareForNextIter(); // Step 3: leader server broadcast to all follower servers to move to next iteration. bool BroadcastMoveToNextIterRequest(bool is_last_iter_valid, const std::string &reason); - void HandleMoveToNextIterRequest(const std::shared_ptr &message); + void HandleMoveToNextIterRequest(const std::shared_ptr &message); // Move to next iteration. Store last iterations model and reset all the rounds. void Next(bool is_iteration_valid, const std::string &reason); // Step 4: leader server broadcasts to all follower servers to end last iteration and cancel the safemode. bool BroadcastEndLastIterRequest(uint64_t iteration_num); - void HandleEndLastIterRequest(const std::shared_ptr &message); + void HandleEndLastIterRequest(const std::shared_ptr &message); // The server end the last iteration. This method will increase the iteration number and cancel the safemode. void EndLastIter(); - std::shared_ptr server_node_; - std::shared_ptr communicator_; + std::shared_ptr server_node_; + std::shared_ptr communicator_; // All the rounds in the server. std::vector> rounds_; @@ -155,6 +155,6 @@ class Iteration { std::mutex pinned_mtx_; }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_ITERATION_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_ITERATION_H_ diff --git a/mindspore/ccsrc/fl/server/iteration_timer.cc b/mindspore/ccsrc/fl/server/iteration_timer.cc index bcbf77f2ca2..f805aefafaf 100644 --- a/mindspore/ccsrc/fl/server/iteration_timer.cc +++ b/mindspore/ccsrc/fl/server/iteration_timer.cc @@ -17,7 +17,7 @@ #include "fl/server/iteration_timer.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { void IterationTimer::Start(const std::chrono::milliseconds &duration) { if (running_.load()) { @@ -46,11 +46,11 @@ void IterationTimer::SetTimeOutCallBack(const TimeOutCb &timeout_cb) { return; } -bool IterationTimer::IsTimeOut(const std::chrono::milliseconds ×tamp) { +bool IterationTimer::IsTimeOut(const std::chrono::milliseconds ×tamp) const { return timestamp > end_time_ ? true : false; } -bool IterationTimer::IsRunning() { return running_; } +bool IterationTimer::IsRunning() const { return running_; } } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/iteration_timer.h b/mindspore/ccsrc/fl/server/iteration_timer.h index 6b44af629cc..752e392f8d8 100644 --- a/mindspore/ccsrc/fl/server/iteration_timer.h +++ b/mindspore/ccsrc/fl/server/iteration_timer.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_ITERATION_TIMER_H_ -#define MINDSPORE_CCSRC_PS_SERVER_ITERATION_TIMER_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_ITERATION_TIMER_H_ +#define MINDSPORE_CCSRC_FL_SERVER_ITERATION_TIMER_H_ #include #include @@ -24,7 +24,7 @@ #include "fl/server/common.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { // IterationTimer controls the time window for the purpose of eliminating trailing time of each iteration. class IterationTimer { @@ -42,10 +42,10 @@ class IterationTimer { void SetTimeOutCallBack(const TimeOutCb &timeout_cb); // Judge whether current timestamp is out of time window's range since the Start function is called. - bool IsTimeOut(const std::chrono::milliseconds ×tamp); + bool IsTimeOut(const std::chrono::milliseconds ×tamp) const; // Judge whether the timer is keeping timing. - bool IsRunning(); + bool IsRunning() const; private: // The running state for the timer. @@ -59,6 +59,6 @@ class IterationTimer { TimeOutCb timeout_callback_; }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_ITERATION_TIMER_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_ITERATION_TIMER_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/aggregation_kernel.h b/mindspore/ccsrc/fl/server/kernel/aggregation_kernel.h index 937f5be91df..a0c41771163 100644 --- a/mindspore/ccsrc/fl/server/kernel/aggregation_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/aggregation_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_AGGREGATION_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_AGGREGATION_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_AGGREGATION_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_AGGREGATION_KERNEL_H_ #include #include @@ -26,7 +26,7 @@ #include "fl/server/kernel/params_info.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { // AggregationKernel is the kernel for weight, grad or other kinds of parameters' aggregation. @@ -99,6 +99,6 @@ class AggregationKernel : public CPUKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_AGGREGATION_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_AGGREGATION_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/aggregation_kernel_factory.cc b/mindspore/ccsrc/fl/server/kernel/aggregation_kernel_factory.cc index f648eb1d056..d09b9de4e17 100644 --- a/mindspore/ccsrc/fl/server/kernel/aggregation_kernel_factory.cc +++ b/mindspore/ccsrc/fl/server/kernel/aggregation_kernel_factory.cc @@ -18,7 +18,7 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { bool AggregationKernelFactory::Matched(const ParamsInfo ¶ms_info, const CNodePtr &kernel_node) { @@ -67,5 +67,5 @@ bool AggregationKernelFactory::Matched(const ParamsInfo ¶ms_info, const CNod } } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/aggregation_kernel_factory.h b/mindspore/ccsrc/fl/server/kernel/aggregation_kernel_factory.h index b1768791b12..a3a92df2174 100644 --- a/mindspore/ccsrc/fl/server/kernel/aggregation_kernel_factory.h +++ b/mindspore/ccsrc/fl/server/kernel/aggregation_kernel_factory.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_AGGREGATION_KERNEL_FACTORY_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_AGGREGATION_KERNEL_FACTORY_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_AGGREGATION_KERNEL_FACTORY_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_AGGREGATION_KERNEL_FACTORY_H_ #include #include @@ -24,7 +24,7 @@ #include "fl/server/kernel/aggregation_kernel.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { using AggregationKernelCreator = std::function()>; @@ -51,6 +51,7 @@ class AggregationKernelRegister { AggregationKernelCreator &&creator) { AggregationKernelFactory::GetInstance().Register(name, params_info, std::move(creator)); } + ~AggregationKernelRegister() = default; }; // Register aggregation kernel with one template type T. @@ -66,6 +67,6 @@ class AggregationKernelRegister { #NAME, PARAMS_INFO, []() { return std::make_shared>(); }); } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_AGGREGATION_KERNEL_FACTORY_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_AGGREGATION_KERNEL_FACTORY_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/apply_momentum_kernel.cc b/mindspore/ccsrc/fl/server/kernel/apply_momentum_kernel.cc index 82c65c72f73..0b16aee5b65 100644 --- a/mindspore/ccsrc/fl/server/kernel/apply_momentum_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/apply_momentum_kernel.cc @@ -17,7 +17,7 @@ #include "fl/server/kernel/apply_momentum_kernel.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { REG_OPTIMIZER_KERNEL(ApplyMomentum, @@ -30,5 +30,5 @@ REG_OPTIMIZER_KERNEL(ApplyMomentum, ApplyMomentumKernel, float) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/apply_momentum_kernel.h b/mindspore/ccsrc/fl/server/kernel/apply_momentum_kernel.h index c9a1c6e4034..6ac0ebccc05 100644 --- a/mindspore/ccsrc/fl/server/kernel/apply_momentum_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/apply_momentum_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_APPLY_MOMENTUM_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_APPLY_MOMENTUM_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_APPLY_MOMENTUM_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_APPLY_MOMENTUM_KERNEL_H_ #include #include @@ -25,7 +25,7 @@ #include "fl/server/kernel/optimizer_kernel_factory.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { using mindspore::kernel::ApplyMomentumCPUKernel; @@ -57,6 +57,6 @@ class ApplyMomentumKernel : public ApplyMomentumCPUKernel, public OptimizerKerne }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_APPLY_MOMENTUM_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_APPLY_MOMENTUM_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/dense_grad_accum_kernel.cc b/mindspore/ccsrc/fl/server/kernel/dense_grad_accum_kernel.cc index 7a74ed29ff5..17099c70dc6 100644 --- a/mindspore/ccsrc/fl/server/kernel/dense_grad_accum_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/dense_grad_accum_kernel.cc @@ -17,7 +17,7 @@ #include "fl/server/kernel/dense_grad_accum_kernel.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { REG_AGGREGATION_KERNEL( @@ -26,5 +26,5 @@ REG_AGGREGATION_KERNEL( DenseGradAccumKernel, float) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/dense_grad_accum_kernel.h b/mindspore/ccsrc/fl/server/kernel/dense_grad_accum_kernel.h index 2ce89f99f43..3b11184f8f2 100644 --- a/mindspore/ccsrc/fl/server/kernel/dense_grad_accum_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/dense_grad_accum_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_DENSE_GRAD_ACCUM_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_DENSE_GRAD_ACCUM_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_DENSE_GRAD_ACCUM_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_DENSE_GRAD_ACCUM_KERNEL_H_ #include #include @@ -26,7 +26,7 @@ #include "fl/server/kernel/aggregation_kernel_factory.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { template @@ -90,6 +90,6 @@ class DenseGradAccumKernel : public AggregationKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_DENSE_GRAD_ACCUM_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_DENSE_GRAD_ACCUM_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/fed_avg_kernel.cc b/mindspore/ccsrc/fl/server/kernel/fed_avg_kernel.cc index 2b379217036..7f3935d9926 100644 --- a/mindspore/ccsrc/fl/server/kernel/fed_avg_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/fed_avg_kernel.cc @@ -17,7 +17,7 @@ #include "fl/server/kernel/fed_avg_kernel.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { REG_AGGREGATION_KERNEL_TWO(FedAvg, @@ -29,5 +29,5 @@ REG_AGGREGATION_KERNEL_TWO(FedAvg, FedAvgKernel, float, size_t) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/fed_avg_kernel.h b/mindspore/ccsrc/fl/server/kernel/fed_avg_kernel.h index edbe187e347..69e04e27ec8 100644 --- a/mindspore/ccsrc/fl/server/kernel/fed_avg_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/fed_avg_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_FED_AVG_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_FED_AVG_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_FED_AVG_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_FED_AVG_KERNEL_H_ #include #include @@ -31,7 +31,7 @@ #include "fl/server/kernel/aggregation_kernel_factory.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { // The implementation for the federated average. We do weighted average for the weights. The uploaded weights from @@ -42,7 +42,13 @@ namespace kernel { template class FedAvgKernel : public AggregationKernel { public: - FedAvgKernel() : participated_(false) {} + FedAvgKernel() + : cnode_weight_idx_(0), + weight_addr_(nullptr), + data_size_addr_(nullptr), + new_weight_addr_(nullptr), + new_data_size_addr_(nullptr), + participated_(false) {} ~FedAvgKernel() override = default; void InitKernel(const CNodePtr &kernel_node) override { @@ -68,13 +74,13 @@ class FedAvgKernel : public AggregationKernel { AnfAlgo::VisitKernelWithReturnType(AnfAlgo::GetInputNode(kernel_node, cnode_weight_idx_), 0).first; MS_EXCEPTION_IF_NULL(weight_node); name_ = cnode_name + "." + weight_node->fullname_with_scope(); - first_cnt_handler_ = [&](std::shared_ptr) { + first_cnt_handler_ = [&](std::shared_ptr) { std::unique_lock lock(weight_mutex_); if (!participated_) { ClearWeightAndDataSize(); } }; - last_cnt_handler_ = [&](std::shared_ptr) { + last_cnt_handler_ = [&](std::shared_ptr) { T *weight_addr = reinterpret_cast(weight_addr_->addr); size_t weight_size = weight_addr_->size; S *data_size_addr = reinterpret_cast(data_size_addr_->addr); @@ -193,7 +199,7 @@ class FedAvgKernel : public AggregationKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_FED_AVG_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_FED_AVG_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/kernel_factory.h b/mindspore/ccsrc/fl/server/kernel/kernel_factory.h index 4d7d3768e5b..7a3e6afbfa9 100644 --- a/mindspore/ccsrc/fl/server/kernel/kernel_factory.h +++ b/mindspore/ccsrc/fl/server/kernel/kernel_factory.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_KERNEL_FACTORY_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_KERNEL_FACTORY_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_KERNEL_FACTORY_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_KERNEL_FACTORY_H_ #include #include @@ -26,7 +26,7 @@ #include "fl/server/kernel/params_info.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { // KernelFactory is used to select and build kernels in server. It's the base class of OptimizerKernelFactory @@ -87,6 +87,6 @@ class KernelFactory { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_KERNEL_FACTORY_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_KERNEL_FACTORY_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/optimizer_kernel.h b/mindspore/ccsrc/fl/server/kernel/optimizer_kernel.h index 3d4bb362ebc..f744df961f9 100644 --- a/mindspore/ccsrc/fl/server/kernel/optimizer_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/optimizer_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_OPTIMIZER_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_OPTIMIZER_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_OPTIMIZER_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_OPTIMIZER_KERNEL_H_ #include #include @@ -28,7 +28,7 @@ #include "fl/server/kernel/params_info.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { using mindspore::kernel::IsSameShape; @@ -92,6 +92,6 @@ class OptimizerKernel : public CPUKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_OPTIMIZER_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_OPTIMIZER_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/optimizer_kernel_factory.cc b/mindspore/ccsrc/fl/server/kernel/optimizer_kernel_factory.cc index f06ecc2c606..50b7accab0e 100644 --- a/mindspore/ccsrc/fl/server/kernel/optimizer_kernel_factory.cc +++ b/mindspore/ccsrc/fl/server/kernel/optimizer_kernel_factory.cc @@ -18,7 +18,7 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { bool OptimizerKernelFactory::Matched(const ParamsInfo ¶ms_info, const CNodePtr &kernel_node) { @@ -66,5 +66,5 @@ bool OptimizerKernelFactory::Matched(const ParamsInfo ¶ms_info, const CNodeP } } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/optimizer_kernel_factory.h b/mindspore/ccsrc/fl/server/kernel/optimizer_kernel_factory.h index b486bafd9a3..4822f680078 100644 --- a/mindspore/ccsrc/fl/server/kernel/optimizer_kernel_factory.h +++ b/mindspore/ccsrc/fl/server/kernel/optimizer_kernel_factory.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_OPTIMIZER_KERNEL_FACTORY_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_OPTIMIZER_KERNEL_FACTORY_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_OPTIMIZER_KERNEL_FACTORY_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_OPTIMIZER_KERNEL_FACTORY_H_ #include #include @@ -24,7 +24,7 @@ #include "fl/server/kernel/optimizer_kernel.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { using OptimizerKernelCreator = std::function()>; @@ -50,6 +50,7 @@ class OptimizerKernelRegister { OptimizerKernelRegister(const std::string &name, const ParamsInfo ¶ms_info, OptimizerKernelCreator &&creator) { OptimizerKernelFactory::GetInstance().Register(name, params_info, std::move(creator)); } + ~OptimizerKernelRegister() = default; }; // Register optimizer kernel with one template type T. @@ -59,6 +60,6 @@ class OptimizerKernelRegister { #NAME, PARAMS_INFO, []() { return std::make_shared>(); }); } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_OPTIMIZER_KERNEL_FACTORY_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_OPTIMIZER_KERNEL_FACTORY_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/params_info.cc b/mindspore/ccsrc/fl/server/kernel/params_info.cc index 693754b19f2..bf6b4e929f2 100644 --- a/mindspore/ccsrc/fl/server/kernel/params_info.cc +++ b/mindspore/ccsrc/fl/server/kernel/params_info.cc @@ -18,7 +18,7 @@ #include "utils/log_adapter.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { ParamsInfo &ParamsInfo::AddInputNameType(const std::string &name, TypeId type) { @@ -64,5 +64,5 @@ const std::vector &ParamsInfo::workspace_names() const { return wor const std::vector &ParamsInfo::outputs_names() const { return outputs_names_; } } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/params_info.h b/mindspore/ccsrc/fl/server/kernel/params_info.h index 522a46cb41b..725a09d4cd2 100644 --- a/mindspore/ccsrc/fl/server/kernel/params_info.h +++ b/mindspore/ccsrc/fl/server/kernel/params_info.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_PARAMS_INFO_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_PARAMS_INFO_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_PARAMS_INFO_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_PARAMS_INFO_H_ #include #include @@ -23,7 +23,7 @@ #include "ir/dtype/type_id.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { // ParamsInfo is used for server computation kernel's register, e.g, ApplyMomentumKernel, FedAvgKernel, etc. @@ -65,6 +65,6 @@ class ParamsInfo { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_PARAMS_INFO_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_PARAMS_INFO_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/round/client_list_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/client_list_kernel.cc index 80884e76e16..98c6a756fda 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/client_list_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/client_list_kernel.cc @@ -22,7 +22,7 @@ #include "schema/cipher_generated.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { void ClientListKernel::InitKernel(size_t) { @@ -150,7 +150,7 @@ bool ClientListKernel::Launch(const std::vector &inputs, const std:: double duration = static_cast((end_time - start_time) * 1.0 / CLOCKS_PER_SEC); MS_LOG(INFO) << "client_list_kernel success time is : " << duration; return true; -} // namespace ps +} // namespace fl bool ClientListKernel::Reset() { MS_LOG(INFO) << "ITERATION NUMBER IS : " << LocalMetaStore::GetInstance().curr_iter_num(); @@ -196,5 +196,5 @@ void ClientListKernel::BuildClientListRsp(std::shared_ptr cli REG_ROUND_KERNEL(getClientList, ClientListKernel) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/client_list_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/client_list_kernel.h index 3100fc22790..96a08cf20b1 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/client_list_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/client_list_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_CLIENT_LIST_KERNEL_H -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_CLIENT_LIST_KERNEL_H +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_CLIENT_LIST_KERNEL_H +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_CLIENT_LIST_KERNEL_H #include #include #include @@ -26,7 +26,7 @@ #include "fl/server/executor.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { class ClientListKernel : public RoundKernel { @@ -50,6 +50,6 @@ class ClientListKernel : public RoundKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_CLIENT_LIST_KERNEL_H +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_CLIENT_LIST_KERNEL_H diff --git a/mindspore/ccsrc/fl/server/kernel/round/exchange_keys_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/exchange_keys_kernel.cc index 623a4ab90cb..f2c41d16d20 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/exchange_keys_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/exchange_keys_kernel.cc @@ -20,7 +20,7 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { void ExchangeKeysKernel::InitKernel(size_t) { @@ -100,5 +100,5 @@ bool ExchangeKeysKernel::Reset() { REG_ROUND_KERNEL(exchangeKeys, ExchangeKeysKernel) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/exchange_keys_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/exchange_keys_kernel.h index 7ae4cca1944..09ca502b127 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/exchange_keys_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/exchange_keys_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_EXCHANGE_KEYS_KERNEL_H -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_EXCHANGE_KEYS_KERNEL_H +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_EXCHANGE_KEYS_KERNEL_H +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_EXCHANGE_KEYS_KERNEL_H #include #include "fl/server/common.h" @@ -25,7 +25,7 @@ #include "fl/armour/cipher/cipher_keys.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { class ExchangeKeysKernel : public RoundKernel { @@ -44,7 +44,7 @@ class ExchangeKeysKernel : public RoundKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_EXCHANGE_KEYS_KERNEL_H +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_EXCHANGE_KEYS_KERNEL_H diff --git a/mindspore/ccsrc/fl/server/kernel/round/get_keys_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/get_keys_kernel.cc index 6fc0422632c..1aafa8c16af 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/get_keys_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/get_keys_kernel.cc @@ -19,7 +19,7 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { void GetKeysKernel::InitKernel(size_t) { @@ -99,5 +99,5 @@ bool GetKeysKernel::Reset() { REG_ROUND_KERNEL(getKeys, GetKeysKernel) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/get_keys_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/get_keys_kernel.h index 5b603d67a3d..4a86568ec8c 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/get_keys_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/get_keys_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_GET_KEYS_KERNEL_H -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_GET_KEYS_KERNEL_H +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_GET_KEYS_KERNEL_H +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_GET_KEYS_KERNEL_H #include #include "fl/server/common.h" @@ -25,7 +25,7 @@ #include "fl/armour/cipher/cipher_keys.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { class GetKeysKernel : public RoundKernel { @@ -44,7 +44,7 @@ class GetKeysKernel : public RoundKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_GET_KEYS_KERNEL_H +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_GET_KEYS_KERNEL_H diff --git a/mindspore/ccsrc/fl/server/kernel/round/get_model_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/get_model_kernel.cc index c0a362000d6..046390f40ae 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/get_model_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/get_model_kernel.cc @@ -23,7 +23,7 @@ #include "fl/server/model_store.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { void GetModelKernel::InitKernel(size_t) { @@ -39,9 +39,8 @@ void GetModelKernel::InitKernel(size_t) { } } -bool GetModelKernel::Launch(const std::vector &inputs, const std::vector &workspace, +bool GetModelKernel::Launch(const std::vector &inputs, const std::vector &, const std::vector &outputs) { - MS_LOG(INFO) << "Launching GetModelKernel kernel."; void *req_data = inputs[0]->addr; std::shared_ptr fbb = std::make_shared(); if (fbb == nullptr || req_data == nullptr) { @@ -49,6 +48,11 @@ bool GetModelKernel::Launch(const std::vector &inputs, const std::ve return false; } + ++retry_count_; + if (retry_count_.load() % kPrintGetModelForEveryRetryTime == 1) { + MS_LOG(INFO) << "Launching GetModelKernel retry count is " << retry_count_.load(); + } + const schema::RequestGetModel *get_model_req = flatbuffers::GetRoot(req_data); GetModel(get_model_req, fbb); GenerateOutput(outputs, fbb->GetBufferPointer(), fbb->GetSize()); @@ -58,6 +62,7 @@ bool GetModelKernel::Launch(const std::vector &inputs, const std::ve bool GetModelKernel::Reset() { MS_LOG(INFO) << "Get model kernel reset!"; StopTimer(); + retry_count_ = 0; return true; } @@ -76,7 +81,9 @@ void GetModelKernel::GetModel(const schema::RequestGetModel *get_model_req, cons "2. Worker has not push all the weights to servers."; BuildGetModelRsp(fbb, schema::ResponseCode_SucNotReady, reason, current_iter, feature_maps, std::to_string(next_req_time)); - MS_LOG(WARNING) << reason; + if (retry_count_.load() % kPrintGetModelForEveryRetryTime == 1) { + MS_LOG(WARNING) << reason; + } return; } @@ -126,5 +133,5 @@ void GetModelKernel::BuildGetModelRsp(const std::shared_ptr &fbb, con REG_ROUND_KERNEL(getModel, GetModelKernel) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/get_model_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/get_model_kernel.h index 98ca3ca1d2e..d8b0947552f 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/get_model_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/get_model_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_GET_MODEL_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_GET_MODEL_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_GET_MODEL_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_GET_MODEL_KERNEL_H_ #include #include @@ -27,12 +27,13 @@ #include "fl/server/kernel/round/round_kernel_factory.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { +constexpr uint32_t kPrintGetModelForEveryRetryTime = 50; class GetModelKernel : public RoundKernel { public: - GetModelKernel() = default; + GetModelKernel() : executor_(nullptr), iteration_time_window_(0), retry_count_(0) {} ~GetModelKernel() override = default; void InitKernel(size_t) override; @@ -51,9 +52,12 @@ class GetModelKernel : public RoundKernel { // The time window of one iteration. size_t iteration_time_window_; + + // The count of retrying because the iteration is not finished. + std::atomic retry_count_; }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_UPDATE_MODEL_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_UPDATE_MODEL_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/round/get_secrets_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/get_secrets_kernel.cc index 4b1f5d5a4b3..0cedb02612e 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/get_secrets_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/get_secrets_kernel.cc @@ -21,7 +21,7 @@ #include "fl/armour/cipher/cipher_shares.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { void GetSecretsKernel::InitKernel(size_t) { @@ -102,5 +102,5 @@ bool GetSecretsKernel::Reset() { REG_ROUND_KERNEL(getSecrets, GetSecretsKernel) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/get_secrets_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/get_secrets_kernel.h index 0422478a5de..eb8f0023cb4 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/get_secrets_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/get_secrets_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_GET_SECRETS_KERNEL_H -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_GET_SECRETS_KERNEL_H +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_GET_SECRETS_KERNEL_H +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_GET_SECRETS_KERNEL_H #include #include "fl/server/common.h" @@ -25,7 +25,7 @@ #include "fl/server/executor.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { class GetSecretsKernel : public RoundKernel { @@ -44,7 +44,7 @@ class GetSecretsKernel : public RoundKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_GET_SECRETS_KERNEL_H +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_GET_SECRETS_KERNEL_H diff --git a/mindspore/ccsrc/fl/server/kernel/round/pull_weight_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/pull_weight_kernel.cc index 847d3830bc5..53bf7041bf1 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/pull_weight_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/pull_weight_kernel.cc @@ -22,10 +22,9 @@ #include "fl/server/model_store.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { -uint64_t PullWeightKernel::retry_count_ = 0; void PullWeightKernel::InitKernel(size_t) { executor_ = &Executor::GetInstance(); MS_EXCEPTION_IF_NULL(executor_); @@ -35,7 +34,7 @@ void PullWeightKernel::InitKernel(size_t) { } } -bool PullWeightKernel::Launch(const std::vector &inputs, const std::vector &workspace, +bool PullWeightKernel::Launch(const std::vector &inputs, const std::vector &, const std::vector &outputs) { MS_LOG(DEBUG) << "Launching PullWeightKernel kernel."; void *req_data = inputs[0]->addr; @@ -59,13 +58,16 @@ bool PullWeightKernel::Launch(const std::vector &inputs, const std:: return true; } -bool PullWeightKernel::Reset() { return true; } +bool PullWeightKernel::Reset() { + retry_count_ = 0; + return true; +} void PullWeightKernel::PullWeight(std::shared_ptr fbb, const schema::RequestPullWeight *pull_weight_req) { std::map feature_maps = {}; size_t current_iter = LocalMetaStore::GetInstance().curr_iter_num(); size_t pull_weight_iter = static_cast(pull_weight_req->iteration()); - // The PullWeight round should be in the same iteration as other rounds. + // The iteration from worker should be the same as server's, otherwise return SucNotReady so that worker could retry. if (pull_weight_iter != current_iter) { std::string reason = "PullWeight iteration " + std::to_string(pull_weight_iter) + " is invalid. Server current iteration: " + std::to_string(current_iter); @@ -76,15 +78,19 @@ void PullWeightKernel::PullWeight(std::shared_ptr fbb, const schema:: std::vector weight_names = {}; auto weights_names_fbs = pull_weight_req->weight_names(); + if (weights_names_fbs == nullptr) { + MS_LOG(ERROR) << "weights_names_fbs is nullptr."; + return; + } for (size_t i = 0; i < weights_names_fbs->size(); i++) { weight_names.push_back(weights_names_fbs->Get(i)->str()); } if (!executor_->IsWeightAggrDone(weight_names)) { - retry_count_++; + ++retry_count_; std::string reason = "The aggregation for the weights is not done yet."; BuildPullWeightRsp(fbb, schema::ResponseCode_SucNotReady, reason, current_iter, feature_maps); - if (retry_count_ % kPrintPullWeightForEveryRetryTime == 1) { - MS_LOG(WARNING) << reason << " Retry count is " << retry_count_; + if (retry_count_.load() % kPrintPullWeightForEveryRetryTime == 1) { + MS_LOG(WARNING) << reason << " Retry count is " << retry_count_.load(); } return; } @@ -131,5 +137,5 @@ void PullWeightKernel::BuildPullWeightRsp(std::shared_ptr fbb, const REG_ROUND_KERNEL(pullWeight, PullWeightKernel) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/pull_weight_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/pull_weight_kernel.h index 62cca7f3f99..9f26a747fe6 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/pull_weight_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/pull_weight_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_PULL_WEIGHT_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_PULL_WEIGHT_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_PULL_WEIGHT_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_PULL_WEIGHT_KERNEL_H_ #include #include @@ -27,13 +27,13 @@ #include "fl/server/executor.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { constexpr uint32_t kPrintPullWeightForEveryRetryTime = 500; class PullWeightKernel : public RoundKernel { public: - PullWeightKernel() = default; + PullWeightKernel() : executor_(nullptr), retry_count_(0) {} ~PullWeightKernel() override = default; void InitKernel(size_t required_cnt) override; @@ -49,10 +49,10 @@ class PullWeightKernel : public RoundKernel { Executor *executor_; // The count of retrying because the aggregation of the weights is not done. - static uint64_t retry_count_; + std::atomic retry_count_; }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_PULL_WEIGHT_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_PULL_WEIGHT_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/round/push_weight_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/push_weight_kernel.cc index fb472c8548e..4aed671d753 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/push_weight_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/push_weight_kernel.cc @@ -17,7 +17,7 @@ #include "fl/server/kernel/round/push_weight_kernel.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { void PushWeightKernel::InitKernel(size_t) { @@ -30,7 +30,7 @@ void PushWeightKernel::InitKernel(size_t) { local_rank_ = DistributedCountService::GetInstance().local_rank(); } -bool PushWeightKernel::Launch(const std::vector &inputs, const std::vector &workspace, +bool PushWeightKernel::Launch(const std::vector &inputs, const std::vector &, const std::vector &outputs) { MS_LOG(INFO) << "Launching PushWeightKernel kernel."; void *req_data = inputs[0]->addr; @@ -60,8 +60,8 @@ bool PushWeightKernel::Reset() { return true; } -void PushWeightKernel::OnLastCountEvent(const std::shared_ptr &) { - if (PSContext::instance()->resetter_round() == ResetterRound::kPushWeight) { +void PushWeightKernel::OnLastCountEvent(const std::shared_ptr &) { + if (ps::PSContext::instance()->resetter_round() == ps::ResetterRound::kPushWeight) { FinishIteration(); } return; @@ -111,6 +111,7 @@ std::map PushWeightKernel::ParseFeatureMap(const schema::R RETURN_IF_NULL(push_weight_req, {}); std::map upload_feature_map; auto fbs_feature_map = push_weight_req->feature_map(); + RETURN_IF_NULL(push_weight_req, upload_feature_map); for (size_t i = 0; i < fbs_feature_map->size(); i++) { std::string weight_full_name = fbs_feature_map->Get(i)->weight_fullname()->str(); float *weight_data = const_cast(fbs_feature_map->Get(i)->data()->data()); @@ -135,5 +136,5 @@ void PushWeightKernel::BuildPushWeightRsp(std::shared_ptr fbb, const REG_ROUND_KERNEL(pushWeight, PushWeightKernel) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/push_weight_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/push_weight_kernel.h index 4bea60c5ced..17d26e99d56 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/push_weight_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/push_weight_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_PUSH_WEIGHT_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_PUSH_WEIGHT_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_PUSH_WEIGHT_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_PUSH_WEIGHT_KERNEL_H_ #include #include @@ -27,19 +27,19 @@ #include "fl/server/executor.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { class PushWeightKernel : public RoundKernel { public: - PushWeightKernel() = default; + PushWeightKernel() : executor_(nullptr), local_rank_(0) {} ~PushWeightKernel() override = default; void InitKernel(size_t threshold_count) override; bool Launch(const std::vector &inputs, const std::vector &workspace, const std::vector &outputs); bool Reset() override; - void OnLastCountEvent(const std::shared_ptr &message) override; + void OnLastCountEvent(const std::shared_ptr &message) override; private: bool PushWeight(std::shared_ptr fbb, const schema::RequestPushWeight *push_weight_req); @@ -52,6 +52,6 @@ class PushWeightKernel : public RoundKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_PUSH_WEIGHT_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_PUSH_WEIGHT_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/round/reconstruct_secrets_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/reconstruct_secrets_kernel.cc index d36d7b4be28..da1d4dc1f08 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/reconstruct_secrets_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/reconstruct_secrets_kernel.cc @@ -20,7 +20,7 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { void ReconstructSecretsKernel::InitKernel(size_t required_cnt) { @@ -34,17 +34,17 @@ void ReconstructSecretsKernel::InitKernel(size_t required_cnt) { MS_LOG(EXCEPTION) << "Executor must be initialized in server pipeline."; return; } - auto last_cnt_handler = [&](std::shared_ptr) { + auto last_cnt_handler = [&](std::shared_ptr) { MS_LOG(INFO) << "start FinishIteration"; FinishIteration(); MS_LOG(INFO) << "end FinishIteration"; return; }; - auto first_cnt_handler = [&](std::shared_ptr) { return; }; + auto first_cnt_handler = [&](std::shared_ptr) { return; }; name_unmask_ = "UnMaskKernel"; MS_LOG(INFO) << "ReconstructSecretsKernel Init, ITERATION NUMBER IS : " << LocalMetaStore::GetInstance().curr_iter_num(); - DistributedCountService::GetInstance().RegisterCounter(name_unmask_, PSContext::instance()->initial_server_num(), + DistributedCountService::GetInstance().RegisterCounter(name_unmask_, ps::PSContext::instance()->initial_server_num(), {first_cnt_handler, last_cnt_handler}); } @@ -134,9 +134,9 @@ bool ReconstructSecretsKernel::Launch(const std::vector &inputs, con return true; } -void ReconstructSecretsKernel::OnLastCountEvent(const std::shared_ptr &message) { +void ReconstructSecretsKernel::OnLastCountEvent(const std::shared_ptr &message) { MS_LOG(INFO) << "ITERATION NUMBER IS : " << LocalMetaStore::GetInstance().curr_iter_num(); - if (PSContext::instance()->encrypt_type() == kPWEncryptType) { + if (ps::PSContext::instance()->encrypt_type() == ps::kPWEncryptType) { while (!Executor::GetInstance().IsAllWeightAggregationDone()) { std::this_thread::sleep_for(std::chrono::milliseconds(5)); } @@ -164,5 +164,5 @@ bool ReconstructSecretsKernel::Reset() { REG_ROUND_KERNEL(reconstructSecrets, ReconstructSecretsKernel) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/reconstruct_secrets_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/reconstruct_secrets_kernel.h index a5dce9e20c1..7bf9dd26b04 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/reconstruct_secrets_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/reconstruct_secrets_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_RECONSTRUCT_SECRETS_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_RECONSTRUCT_SECRETS_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_RECONSTRUCT_SECRETS_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_RECONSTRUCT_SECRETS_KERNEL_H_ #include #include @@ -27,7 +27,7 @@ #include "fl/server/executor.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { class ReconstructSecretsKernel : public RoundKernel { @@ -39,7 +39,7 @@ class ReconstructSecretsKernel : public RoundKernel { bool Launch(const std::vector &inputs, const std::vector &workspace, const std::vector &outputs) override; bool Reset() override; - void OnLastCountEvent(const std::shared_ptr &message) override; + void OnLastCountEvent(const std::shared_ptr &message) override; private: std::string name_unmask_; @@ -49,6 +49,6 @@ class ReconstructSecretsKernel : public RoundKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_RECONSTRUCT_SECRETS_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_RECONSTRUCT_SECRETS_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/round/round_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/round_kernel.cc index 1127b3f0015..1b7dd685467 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/round_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/round_kernel.cc @@ -24,7 +24,7 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { RoundKernel::RoundKernel() : name_(""), current_count_(0), required_count_(0), error_reason_(""), running_(true) { @@ -61,25 +61,25 @@ RoundKernel::~RoundKernel() { } } -void RoundKernel::OnFirstCountEvent(const std::shared_ptr &message) { return; } +void RoundKernel::OnFirstCountEvent(const std::shared_ptr &message) { return; } -void RoundKernel::OnLastCountEvent(const std::shared_ptr &message) { return; } +void RoundKernel::OnLastCountEvent(const std::shared_ptr &message) { return; } -void RoundKernel::StopTimer() { +void RoundKernel::StopTimer() const { if (stop_timer_cb_) { stop_timer_cb_(); } return; } -void RoundKernel::FinishIteration() { +void RoundKernel::FinishIteration() const { if (finish_iteration_cb_) { finish_iteration_cb_(true, ""); } return; } -void RoundKernel::Release(AddressPtr addr_ptr) { +void RoundKernel::Release(const AddressPtr &addr_ptr) { if (addr_ptr == nullptr) { MS_LOG(ERROR) << "Data to be released is empty."; return; @@ -91,13 +91,13 @@ void RoundKernel::Release(AddressPtr addr_ptr) { void RoundKernel::set_name(const std::string &name) { name_ = name; } -void RoundKernel::set_stop_timer_cb(StopTimerCb timer_stopper) { stop_timer_cb_ = timer_stopper; } +void RoundKernel::set_stop_timer_cb(const StopTimerCb &timer_stopper) { stop_timer_cb_ = timer_stopper; } -void RoundKernel::set_finish_iteration_cb(FinishIterCb finish_iteration_cb) { +void RoundKernel::set_finish_iteration_cb(const FinishIterCb &finish_iteration_cb) { finish_iteration_cb_ = finish_iteration_cb; } -void RoundKernel::GenerateOutput(const std::vector &outputs, void *data, size_t len) { +void RoundKernel::GenerateOutput(const std::vector &outputs, const void *data, size_t len) { if (data == nullptr) { MS_LOG(ERROR) << "The data is nullptr."; return; @@ -129,5 +129,5 @@ void RoundKernel::GenerateOutput(const std::vector &outputs, void *d } } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/round_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/round_kernel.h index 511f97930c3..4f5cab58666 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/round_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/round_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_ROUND_ROUND_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_ROUND_ROUND_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_ROUND_ROUND_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_ROUND_ROUND_KERNEL_H_ #include #include @@ -35,7 +35,7 @@ #include "fl/server/distributed_metadata_store.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { // RoundKernel contains the main logic of server handling messages from workers. One iteration has multiple round @@ -67,31 +67,31 @@ class RoundKernel : virtual public CPUKernel { // The counter event handlers for DistributedCountService. // The callbacks when first message and last message for this round kernel is received. // These methods is called by class DistributedCountService and triggered by counting server. - virtual void OnFirstCountEvent(const std::shared_ptr &message); - virtual void OnLastCountEvent(const std::shared_ptr &message); + virtual void OnFirstCountEvent(const std::shared_ptr &message); + virtual void OnLastCountEvent(const std::shared_ptr &message); // Called when this round is finished. This round timer's Stop method will be called. - void StopTimer(); + void StopTimer() const; // Called after this iteration(including all rounds) is finished. All rounds' Reset method will // be called. - void FinishIteration(); + void FinishIteration() const; // Release the response data allocated inside the round kernel. // Server framework must call this after the response data is sent back. - void Release(AddressPtr addr_ptr); + void Release(const AddressPtr &addr_ptr); // Set round kernel name, which could be used in round kernel's methods. void set_name(const std::string &name); // Set callbacks to be called under certain triggered conditions. - void set_stop_timer_cb(StopTimerCb timer_stopper); - void set_finish_iteration_cb(FinishIterCb finish_iteration_cb); + void set_stop_timer_cb(const StopTimerCb &timer_stopper); + void set_finish_iteration_cb(const FinishIterCb &finish_iteration_cb); protected: // Generating response data of this round. The data is allocated on the heap to ensure it's not released before sent // back to worker. - void GenerateOutput(const std::vector &outputs, void *data, size_t len); + void GenerateOutput(const std::vector &outputs, const void *data, size_t len); // Round kernel's name. std::string name_; @@ -123,6 +123,6 @@ class RoundKernel : virtual public CPUKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_ROUND_ROUND_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_ROUND_ROUND_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/round/round_kernel_factory.cc b/mindspore/ccsrc/fl/server/kernel/round/round_kernel_factory.cc index efdebbf7554..6619bf439f0 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/round_kernel_factory.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/round_kernel_factory.cc @@ -17,7 +17,7 @@ #include "fl/server/kernel/round/round_kernel_factory.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { RoundKernelFactory &RoundKernelFactory::GetInstance() { @@ -40,5 +40,5 @@ std::shared_ptr RoundKernelFactory::Create(const std::string &name) } } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/round_kernel_factory.h b/mindspore/ccsrc/fl/server/kernel/round/round_kernel_factory.h index 37412c4f994..0f799ea7944 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/round_kernel_factory.h +++ b/mindspore/ccsrc/fl/server/kernel/round/round_kernel_factory.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_ROUND_ROUND_KERNEL_FACTORY_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_ROUND_ROUND_KERNEL_FACTORY_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_ROUND_ROUND_KERNEL_FACTORY_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_ROUND_ROUND_KERNEL_FACTORY_H_ #include #include @@ -25,7 +25,7 @@ #include "fl/server/kernel/round/round_kernel.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { using RoundKernelCreator = std::function()>; @@ -50,6 +50,7 @@ class RoundKernelRegister { RoundKernelRegister(const std::string &name, RoundKernelCreator &&creator) { RoundKernelFactory::GetInstance().Register(name, std::move(creator)); } + ~RoundKernelRegister() = default; }; #define REG_ROUND_KERNEL(NAME, CLASS) \ @@ -57,6 +58,6 @@ class RoundKernelRegister { static const RoundKernelRegister g_##NAME##_round_kernel_reg(#NAME, []() { return std::make_shared(); }); } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_ROUND_ROUND_KERNEL_FACTORY_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_ROUND_ROUND_KERNEL_FACTORY_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/round/share_secrets_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/share_secrets_kernel.cc index 9df01405783..1786e98b060 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/share_secrets_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/share_secrets_kernel.cc @@ -19,7 +19,7 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { void ShareSecretsKernel::InitKernel(size_t) { @@ -101,5 +101,5 @@ bool ShareSecretsKernel::Reset() { REG_ROUND_KERNEL(shareSecrets, ShareSecretsKernel) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/share_secrets_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/share_secrets_kernel.h index dc2d8023778..892b6012d99 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/share_secrets_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/share_secrets_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_SHARE_SECRETS_KERNEL_H -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_SHARE_SECRETS_KERNEL_H +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_SHARE_SECRETS_KERNEL_H +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_SHARE_SECRETS_KERNEL_H #include #include "fl/server/common.h" @@ -25,7 +25,7 @@ #include "fl/armour/cipher/cipher_shares.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { class ShareSecretsKernel : public RoundKernel { @@ -44,7 +44,7 @@ class ShareSecretsKernel : public RoundKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_SHARE_SECRETS_KERNEL_H +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_SHARE_SECRETS_KERNEL_H diff --git a/mindspore/ccsrc/fl/server/kernel/round/start_fl_job_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/start_fl_job_kernel.cc index c0e29b49468..55d78471a80 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/start_fl_job_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/start_fl_job_kernel.cc @@ -26,7 +26,7 @@ #endif namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { void StartFLJobKernel::InitKernel(size_t) { @@ -34,7 +34,7 @@ void StartFLJobKernel::InitKernel(size_t) { if (LocalMetaStore::GetInstance().has_value(kCtxTotalTimeoutDuration)) { iteration_time_window_ = LocalMetaStore::GetInstance().value(kCtxTotalTimeoutDuration); } - iter_next_req_timestamp_ = CURRENT_TIME_MILLI.count() + iteration_time_window_; + iter_next_req_timestamp_ = LongToUlong(CURRENT_TIME_MILLI.count()) + iteration_time_window_; LocalMetaStore::GetInstance().put_value(kCtxIterationNextRequestTimestamp, iter_next_req_timestamp_); executor_ = &Executor::GetInstance(); @@ -49,7 +49,7 @@ void StartFLJobKernel::InitKernel(size_t) { return; } -bool StartFLJobKernel::Launch(const std::vector &inputs, const std::vector &workspace, +bool StartFLJobKernel::Launch(const std::vector &inputs, const std::vector &, const std::vector &outputs) { MS_LOG(INFO) << "Launching StartFLJobKernel kernel."; if (inputs.size() != 1 || outputs.size() != 1) { @@ -113,7 +113,7 @@ bool StartFLJobKernel::Reset() { return true; } -void StartFLJobKernel::OnFirstCountEvent(const std::shared_ptr &) { +void StartFLJobKernel::OnFirstCountEvent(const std::shared_ptr &) { iter_next_req_timestamp_ = CURRENT_TIME_MILLI.count() + iteration_time_window_; LocalMetaStore::GetInstance().put_value(kCtxIterationNextRequestTimestamp, iter_next_req_timestamp_); // The first startFLJob request means a new iteration starts running. @@ -133,6 +133,7 @@ bool StartFLJobKernel::ReachThresholdForStartFLJob(const std::shared_ptrfl_name()->str(); std::string fl_id = start_fl_job_req->fl_id()->str(); int data_size = start_fl_job_req->data_size(); @@ -141,7 +142,7 @@ DeviceMeta StartFLJobKernel::CreateDeviceMetadata(const schema::RequestFLJob *st DeviceMeta device_meta; device_meta.set_fl_name(fl_name); device_meta.set_fl_id(fl_id); - device_meta.set_data_size(data_size); + device_meta.set_data_size(IntToSize(data_size)); return device_meta; } @@ -154,7 +155,7 @@ bool StartFLJobKernel::ReadyForStartFLJob(const std::shared_ptr &fbb, } if (!ret) { BuildStartFLJobRsp( - fbb, schema::ResponseCode_NotSelected, reason, false, + fbb, schema::ResponseCode_RequestError, reason, false, std::to_string(LocalMetaStore::GetInstance().value(kCtxIterationNextRequestTimestamp))); MS_LOG(ERROR) << reason; } @@ -163,6 +164,7 @@ bool StartFLJobKernel::ReadyForStartFLJob(const std::shared_ptr &fbb, bool StartFLJobKernel::CountForStartFLJob(const std::shared_ptr &fbb, const schema::RequestFLJob *start_fl_job_req) { + RETURN_IF_NULL(start_fl_job_req, false); if (!DistributedCountService::GetInstance().Count(name_, start_fl_job_req->fl_id()->str())) { std::string reason = "Counting start fl job request failed. Please retry later."; BuildStartFLJobRsp( @@ -192,8 +194,8 @@ void StartFLJobKernel::BuildStartFLJobRsp(const std::shared_ptr &fbb, std::map feature_maps) { auto fbs_reason = fbb->CreateString(reason); auto fbs_next_req_time = fbb->CreateString(next_req_time); - auto fbs_server_mode = fbb->CreateString(PSContext::instance()->server_mode()); - auto fbs_fl_name = fbb->CreateString(PSContext::instance()->fl_name()); + auto fbs_server_mode = fbb->CreateString(ps::PSContext::instance()->server_mode()); + auto fbs_fl_name = fbb->CreateString(ps::PSContext::instance()->fl_name()); #ifdef ENABLE_ARMOUR auto *param = armour::CipherInit::GetInstance().GetPublicParams(); @@ -204,7 +206,7 @@ void StartFLJobKernel::BuildStartFLJobRsp(const std::shared_ptr &fbb, float dp_eps = param->dp_eps; float dp_delta = param->dp_delta; float dp_norm_clip = param->dp_norm_clip; - auto encrypt_type = fbb->CreateString(PSContext::instance()->encrypt_type()); + auto encrypt_type = fbb->CreateString(ps::PSContext::instance()->encrypt_type()); auto cipher_public_params = schema::CreateCipherPublicParams(*fbb.get(), t, p, g, prime, dp_eps, dp_delta, dp_norm_clip, encrypt_type); @@ -213,10 +215,10 @@ void StartFLJobKernel::BuildStartFLJobRsp(const std::shared_ptr &fbb, schema::FLPlanBuilder fl_plan_builder(*(fbb.get())); fl_plan_builder.add_fl_name(fbs_fl_name); fl_plan_builder.add_server_mode(fbs_server_mode); - fl_plan_builder.add_iterations(PSContext::instance()->fl_iteration_num()); - fl_plan_builder.add_epochs(PSContext::instance()->client_epoch_num()); - fl_plan_builder.add_mini_batch(PSContext::instance()->client_batch_size()); - fl_plan_builder.add_lr(PSContext::instance()->client_learning_rate()); + fl_plan_builder.add_iterations(ps::PSContext::instance()->fl_iteration_num()); + fl_plan_builder.add_epochs(ps::PSContext::instance()->client_epoch_num()); + fl_plan_builder.add_mini_batch(ps::PSContext::instance()->client_batch_size()); + fl_plan_builder.add_lr(ps::PSContext::instance()->client_learning_rate()); #ifdef ENABLE_ARMOUR fl_plan_builder.add_cipher(cipher_public_params); #endif @@ -235,7 +237,7 @@ void StartFLJobKernel::BuildStartFLJobRsp(const std::shared_ptr &fbb, schema::ResponseFLJobBuilder rsp_fl_job_builder(*(fbb.get())); rsp_fl_job_builder.add_retcode(retcode); rsp_fl_job_builder.add_reason(fbs_reason); - rsp_fl_job_builder.add_iteration(LocalMetaStore::GetInstance().curr_iter_num()); + rsp_fl_job_builder.add_iteration(SizeToInt(LocalMetaStore::GetInstance().curr_iter_num())); rsp_fl_job_builder.add_is_selected(is_selected); rsp_fl_job_builder.add_next_req_time(fbs_next_req_time); rsp_fl_job_builder.add_fl_plan_config(fbs_fl_plan); @@ -248,5 +250,5 @@ void StartFLJobKernel::BuildStartFLJobRsp(const std::shared_ptr &fbb, REG_ROUND_KERNEL(startFLJob, StartFLJobKernel) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/start_fl_job_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/start_fl_job_kernel.h index 2355113c27a..6af55537d2d 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/start_fl_job_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/start_fl_job_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_START_FL_JOB_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_START_FL_JOB_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_START_FL_JOB_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_START_FL_JOB_KERNEL_H_ #include #include @@ -27,7 +27,7 @@ #include "fl/server/kernel/round/round_kernel_factory.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { class StartFLJobKernel : public RoundKernel { @@ -40,7 +40,7 @@ class StartFLJobKernel : public RoundKernel { const std::vector &outputs) override; bool Reset() override; - void OnFirstCountEvent(const std::shared_ptr &message) override; + void OnFirstCountEvent(const std::shared_ptr &message) override; private: // Returns whether the startFLJob count of this iteration has reached the threshold. @@ -74,6 +74,6 @@ class StartFLJobKernel : public RoundKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_START_FL_JOB_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_START_FL_JOB_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/kernel/round/update_model_kernel.cc b/mindspore/ccsrc/fl/server/kernel/round/update_model_kernel.cc index e4bcd0604f7..af2aacbe04f 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/update_model_kernel.cc +++ b/mindspore/ccsrc/fl/server/kernel/round/update_model_kernel.cc @@ -21,7 +21,7 @@ #include "fl/server/kernel/round/update_model_kernel.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { void UpdateModelKernel::InitKernel(size_t threshold_count) { @@ -42,7 +42,7 @@ void UpdateModelKernel::InitKernel(size_t threshold_count) { LocalMetaStore::GetInstance().put_value(kCtxFedAvgTotalDataSize, kInitialDataSizeSum); } -bool UpdateModelKernel::Launch(const std::vector &inputs, const std::vector &workspace, +bool UpdateModelKernel::Launch(const std::vector &inputs, const std::vector &, const std::vector &outputs) { if (inputs.size() != 1 || outputs.size() != 1) { MS_LOG(ERROR) << "inputs or outputs size is invalid."; @@ -87,8 +87,8 @@ bool UpdateModelKernel::Reset() { return true; } -void UpdateModelKernel::OnLastCountEvent(const std::shared_ptr &message) { - if (PSContext::instance()->resetter_round() == ResetterRound::kUpdateModel) { +void UpdateModelKernel::OnLastCountEvent(const std::shared_ptr &message) { + if (ps::PSContext::instance()->resetter_round() == ps::ResetterRound::kUpdateModel) { while (!executor_->IsAllWeightAggregationDone()) { std::this_thread::sleep_for(std::chrono::milliseconds(5)); } @@ -96,7 +96,7 @@ void UpdateModelKernel::OnLastCountEvent(const std::shared_ptr(kCtxFedAvgTotalDataSize); MS_LOG(INFO) << "Total data size for iteration " << LocalMetaStore::GetInstance().curr_iter_num() << " is " << total_data_size; - if (PSContext::instance()->encrypt_type() != kPWEncryptType) { + if (ps::PSContext::instance()->encrypt_type() != ps::kPWEncryptType) { FinishIteration(); } } @@ -116,6 +116,7 @@ bool UpdateModelKernel::ReachThresholdForUpdateModel(const std::shared_ptr &fbb) { + RETURN_IF_NULL(update_model_req, false); size_t iteration = static_cast(update_model_req->iteration()); if (iteration != LocalMetaStore::GetInstance().curr_iter_num()) { std::string reason = "UpdateModel iteration number is invalid:" + std::to_string(iteration) + @@ -131,6 +132,7 @@ bool UpdateModelKernel::UpdateModel(const schema::RequestUpdateModel *update_mod PBMetadata device_metas = DistributedMetadataStore::GetInstance().GetMetadata(kCtxDeviceMetas); FLIdToDeviceMeta fl_id_to_meta = device_metas.device_metas(); std::string update_model_fl_id = update_model_req->fl_id()->str(); + MS_LOG(INFO) << "Update model for fl id " << update_model_fl_id; if (fl_id_to_meta.fl_id_to_meta().count(update_model_fl_id) == 0) { std::string reason = "devices_meta for " + update_model_fl_id + " is not set. Please retry later."; BuildUpdateModelRsp( @@ -180,6 +182,7 @@ std::map UpdateModelKernel::ParseFeatureMap( RETURN_IF_NULL(update_model_req, {}); std::map feature_map; auto fbs_feature_map = update_model_req->feature_map(); + RETURN_IF_NULL(fbs_feature_map, feature_map); for (size_t i = 0; i < fbs_feature_map->size(); i++) { std::string weight_full_name = fbs_feature_map->Get(i)->weight_fullname()->str(); float *weight_data = const_cast(fbs_feature_map->Get(i)->data()->data()); @@ -194,6 +197,7 @@ std::map UpdateModelKernel::ParseFeatureMap( bool UpdateModelKernel::CountForUpdateModel(const std::shared_ptr &fbb, const schema::RequestUpdateModel *update_model_req) { + RETURN_IF_NULL(update_model_req, false); if (!DistributedCountService::GetInstance().Count(name_, update_model_req->fl_id()->str())) { std::string reason = "Counting for update model request failed. Please retry later."; BuildUpdateModelRsp( @@ -222,5 +226,5 @@ void UpdateModelKernel::BuildUpdateModelRsp(const std::shared_ptr &fb REG_ROUND_KERNEL(updateModel, UpdateModelKernel) } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/kernel/round/update_model_kernel.h b/mindspore/ccsrc/fl/server/kernel/round/update_model_kernel.h index 62e8c89a9fe..258d70ebda7 100644 --- a/mindspore/ccsrc/fl/server/kernel/round/update_model_kernel.h +++ b/mindspore/ccsrc/fl/server/kernel/round/update_model_kernel.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_KERNEL_UPDATE_MODEL_KERNEL_H_ -#define MINDSPORE_CCSRC_PS_SERVER_KERNEL_UPDATE_MODEL_KERNEL_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_KERNEL_UPDATE_MODEL_KERNEL_H_ +#define MINDSPORE_CCSRC_FL_SERVER_KERNEL_UPDATE_MODEL_KERNEL_H_ #include #include @@ -27,7 +27,7 @@ #include "fl/server/executor.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { namespace kernel { // The initial data size sum of federated learning is 0, which will be accumulated in updateModel round. @@ -35,7 +35,7 @@ constexpr uint64_t kInitialDataSizeSum = 0; class UpdateModelKernel : public RoundKernel { public: - UpdateModelKernel() = default; + UpdateModelKernel() : executor_(nullptr), iteration_time_window_(0) {} ~UpdateModelKernel() override = default; void InitKernel(size_t threshold_count) override; @@ -44,7 +44,7 @@ class UpdateModelKernel : public RoundKernel { bool Reset() override; // In some cases, the last updateModel message means this server iteration is finished. - void OnLastCountEvent(const std::shared_ptr &message) override; + void OnLastCountEvent(const std::shared_ptr &message) override; private: bool ReachThresholdForUpdateModel(const std::shared_ptr &fbb); @@ -62,6 +62,6 @@ class UpdateModelKernel : public RoundKernel { }; } // namespace kernel } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_KERNEL_UPDATE_MODEL_KERNEL_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_KERNEL_UPDATE_MODEL_KERNEL_H_ diff --git a/mindspore/ccsrc/fl/server/local_meta_store.cc b/mindspore/ccsrc/fl/server/local_meta_store.cc index 32c02e873c4..a79e4caac2c 100644 --- a/mindspore/ccsrc/fl/server/local_meta_store.cc +++ b/mindspore/ccsrc/fl/server/local_meta_store.cc @@ -17,7 +17,7 @@ #include "fl/server/local_meta_store.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { void LocalMetaStore::remove_value(const std::string &name) { std::unique_lock lock(mtx_); @@ -41,5 +41,5 @@ const size_t LocalMetaStore::curr_iter_num() { return curr_iter_num_; } } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/local_meta_store.h b/mindspore/ccsrc/fl/server/local_meta_store.h index 1d3ab1208eb..a9ced6e8af3 100644 --- a/mindspore/ccsrc/fl/server/local_meta_store.h +++ b/mindspore/ccsrc/fl/server/local_meta_store.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_LOCAL_META_STORE_H_ -#define MINDSPORE_CCSRC_PS_SERVER_LOCAL_META_STORE_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_LOCAL_META_STORE_H_ +#define MINDSPORE_CCSRC_FL_SERVER_LOCAL_META_STORE_H_ #include #include @@ -24,7 +24,7 @@ #include "fl/server/common.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { // LocalMetaStore class is used for metadata storage of this server process. // For example, the current iteration number, time windows for round kernels, etc. @@ -71,7 +71,7 @@ class LocalMetaStore { const size_t curr_iter_num(); private: - LocalMetaStore() = default; + LocalMetaStore() : key_to_meta_({}), curr_iter_num_(0) {} ~LocalMetaStore() = default; LocalMetaStore(const LocalMetaStore &) = delete; LocalMetaStore &operator=(const LocalMetaStore &) = delete; @@ -83,6 +83,6 @@ class LocalMetaStore { size_t curr_iter_num_; }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_LOCAL_META_STORE_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_LOCAL_META_STORE_H_ diff --git a/mindspore/ccsrc/fl/server/memory_register.cc b/mindspore/ccsrc/fl/server/memory_register.cc index 91c00ab9fa4..e89cdfea0ca 100644 --- a/mindspore/ccsrc/fl/server/memory_register.cc +++ b/mindspore/ccsrc/fl/server/memory_register.cc @@ -18,7 +18,7 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { void MemoryRegister::RegisterAddressPtr(const std::string &name, const AddressPtr &address) { addresses_.try_emplace(name, address); @@ -32,5 +32,5 @@ void MemoryRegister::StoreUint64Array(std::unique_ptr *array) { uint64 void MemoryRegister::StoreCharArray(std::unique_ptr *array) { char_arrays_.push_back(std::move(*array)); } } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/memory_register.h b/mindspore/ccsrc/fl/server/memory_register.h index 1eea82424a9..fa359d026d2 100644 --- a/mindspore/ccsrc/fl/server/memory_register.h +++ b/mindspore/ccsrc/fl/server/memory_register.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_MEMORY_REGISTER_H_ -#define MINDSPORE_CCSRC_PS_SERVER_MEMORY_REGISTER_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_MEMORY_REGISTER_H_ +#define MINDSPORE_CCSRC_FL_SERVER_MEMORY_REGISTER_H_ #include #include @@ -26,7 +26,7 @@ #include "fl/server/common.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { // Memory allocated in server is normally trainable parameters, hyperparameters, gradients, etc. // MemoryRegister registers the Memory with key-value format where key refers to address's name("grad", "weights", @@ -88,6 +88,6 @@ class MemoryRegister { } }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_MEMORY_REGISTER_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_MEMORY_REGISTER_H_ diff --git a/mindspore/ccsrc/fl/server/model_store.cc b/mindspore/ccsrc/fl/server/model_store.cc index ba61f8c56d6..b762fa728dd 100644 --- a/mindspore/ccsrc/fl/server/model_store.cc +++ b/mindspore/ccsrc/fl/server/model_store.cc @@ -21,7 +21,7 @@ #include "fl/server/executor.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { void ModelStore::Initialize(uint32_t max_count) { if (!Executor::GetInstance().initialized()) { @@ -155,5 +155,5 @@ size_t ModelStore::ComputeModelSize() { return model_size; } } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/model_store.h b/mindspore/ccsrc/fl/server/model_store.h index a5fd9d8e5d7..97f26ada0e0 100644 --- a/mindspore/ccsrc/fl/server/model_store.h +++ b/mindspore/ccsrc/fl/server/model_store.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_MODEL_STORE_H_ -#define MINDSPORE_CCSRC_PS_SERVER_MODEL_STORE_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_MODEL_STORE_H_ +#define MINDSPORE_CCSRC_FL_SERVER_MODEL_STORE_H_ #include #include @@ -25,7 +25,7 @@ #include "fl/server/executor.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { // The initial iteration number is 0 in server. constexpr size_t kInitIterationNum = 0; @@ -84,6 +84,6 @@ class ModelStore { std::map> iteration_to_model_; }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_MODEL_STORE_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_MODEL_STORE_H_ diff --git a/mindspore/ccsrc/fl/server/parameter_aggregator.cc b/mindspore/ccsrc/fl/server/parameter_aggregator.cc index 820b73b7969..72be46aaeee 100644 --- a/mindspore/ccsrc/fl/server/parameter_aggregator.cc +++ b/mindspore/ccsrc/fl/server/parameter_aggregator.cc @@ -23,7 +23,7 @@ #include namespace mindspore { -namespace ps { +namespace fl { namespace server { bool ParameterAggregator::Init(const CNodePtr &cnode, size_t threshold_count) { MS_EXCEPTION_IF_NULL(cnode); @@ -199,8 +199,8 @@ bool ParameterAggregator::InitAggregationKernels(const CNodePtr &cnode) { } bool ParameterAggregator::InitOptimizerKernels(const CNodePtr &cnode) { - if (PSContext::instance()->server_mode() == kServerModeFL || - PSContext::instance()->server_mode() == kServerModeHybrid) { + if (ps::PSContext::instance()->server_mode() == ps::kServerModeFL || + ps::PSContext::instance()->server_mode() == ps::kServerModeHybrid) { MS_LOG(DEBUG) << "Federated learning mode doesn't need optimizer kernel."; return true; } @@ -319,15 +319,15 @@ bool ParameterAggregator::GenerateOptimizerKernelParams(const std::shared_ptr ParameterAggregator::SelectAggregationAlgorithm(const CNodePtr &cnode) { +std::vector ParameterAggregator::SelectAggregationAlgorithm(const CNodePtr &) { std::vector aggregation_algorithm = {}; - if (PSContext::instance()->server_mode() == kServerModeFL || - PSContext::instance()->server_mode() == kServerModeHybrid) { + if (ps::PSContext::instance()->server_mode() == ps::kServerModeFL || + ps::PSContext::instance()->server_mode() == ps::kServerModeHybrid) { aggregation_algorithm.push_back("FedAvg"); - } else if (PSContext::instance()->server_mode() == kServerModePS) { + } else if (ps::PSContext::instance()->server_mode() == ps::kServerModePS) { aggregation_algorithm.push_back("DenseGradAccum"); } else { - MS_LOG(ERROR) << "Server doesn't support mode " << PSContext::instance()->server_mode(); + MS_LOG(ERROR) << "Server doesn't support mode " << ps::PSContext::instance()->server_mode(); } MS_LOG(INFO) << "Aggregation algorithm selection result: " << aggregation_algorithm; @@ -344,5 +344,5 @@ template bool ParameterAggregator::AssignMemory(std::shared_ptr memory_register); } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/parameter_aggregator.h b/mindspore/ccsrc/fl/server/parameter_aggregator.h index d595e51acac..0121f0d97bc 100644 --- a/mindspore/ccsrc/fl/server/parameter_aggregator.h +++ b/mindspore/ccsrc/fl/server/parameter_aggregator.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_PARAMETER_AGGREGATOR_H_ -#define MINDSPORE_CCSRC_PS_SERVER_PARAMETER_AGGREGATOR_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_PARAMETER_AGGREGATOR_H_ +#define MINDSPORE_CCSRC_FL_SERVER_PARAMETER_AGGREGATOR_H_ #include #include @@ -28,7 +28,7 @@ #include "fl/server/kernel/optimizer_kernel_factory.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { // Encapsulate the parameters for a kernel into a struct to make it convenient for ParameterAggregator to launch server // kernels. @@ -137,6 +137,6 @@ class ParameterAggregator { std::vector, KernelParams>> optimizer_kernel_parameters_; }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_PARAMETER_AGGREGATOR_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_PARAMETER_AGGREGATOR_H_ diff --git a/mindspore/ccsrc/fl/server/round.cc b/mindspore/ccsrc/fl/server/round.cc index 88a51db54b3..4d25693fd25 100644 --- a/mindspore/ccsrc/fl/server/round.cc +++ b/mindspore/ccsrc/fl/server/round.cc @@ -21,7 +21,7 @@ #include "fl/server/iteration.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { class Server; class Iteration; @@ -34,14 +34,14 @@ Round::Round(const std::string &name, bool check_timeout, size_t time_window, bo threshold_count_(threshold_count), server_num_as_threshold_(server_num_as_threshold) {} -void Round::Initialize(const std::shared_ptr &communicator, TimeOutCb timeout_cb, +void Round::Initialize(const std::shared_ptr &communicator, TimeOutCb timeout_cb, FinishIterCb finish_iteration_cb) { MS_EXCEPTION_IF_NULL(communicator); communicator_ = communicator; // Register callback for round kernel. communicator_->RegisterMsgCallBack( - name_, [&](std::shared_ptr message) { LaunchRoundKernel(message); }); + name_, [&](std::shared_ptr message) { LaunchRoundKernel(message); }); // Callback when the iteration is finished. finish_iteration_cb_ = [this, finish_iteration_cb](bool is_iteration_valid, const std::string &) -> void { @@ -106,7 +106,7 @@ void Round::BindRoundKernel(const std::shared_ptr &kernel) return; } -void Round::LaunchRoundKernel(const std::shared_ptr &message) { +void Round::LaunchRoundKernel(const std::shared_ptr &message) { if (message == nullptr) { MS_LOG(ERROR) << "Message is nullptr."; return; @@ -152,7 +152,7 @@ bool Round::check_timeout() const { return check_timeout_; } size_t Round::time_window() const { return time_window_; } -void Round::OnFirstCountEvent(const std::shared_ptr &message) { +void Round::OnFirstCountEvent(const std::shared_ptr &message) { MS_LOG(INFO) << "Round " << name_ << " first count event is triggered."; // The timer starts only after the first count event is triggered by DistributedCountService. if (check_timeout_) { @@ -164,7 +164,7 @@ void Round::OnFirstCountEvent(const std::shared_ptr &messa return; } -void Round::OnLastCountEvent(const std::shared_ptr &message) { +void Round::OnLastCountEvent(const std::shared_ptr &message) { MS_LOG(INFO) << "Round " << name_ << " last count event is triggered."; // Same as the first count event, the timer must be stopped by DistributedCountService. if (check_timeout_) { @@ -176,5 +176,5 @@ void Round::OnLastCountEvent(const std::shared_ptr &messag return; } } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/round.h b/mindspore/ccsrc/fl/server/round.h index 8b1e60f4198..e2ba6586904 100644 --- a/mindspore/ccsrc/fl/server/round.h +++ b/mindspore/ccsrc/fl/server/round.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_ROUND_H_ -#define MINDSPORE_CCSRC_PS_SERVER_ROUND_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_ROUND_H_ +#define MINDSPORE_CCSRC_FL_SERVER_ROUND_H_ #include #include @@ -26,7 +26,7 @@ #include "fl/server/kernel/round/round_kernel.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { // Round helps server to handle network round messages and launch round kernels. One iteration in server consists of // multiple rounds like startFLJob, updateModel, Push, Pull, etc. Some round kernels may be stateful because of counting @@ -37,7 +37,7 @@ class Round { bool check_count = false, size_t threshold_count = 8, bool server_num_as_threshold = false); ~Round() = default; - void Initialize(const std::shared_ptr &communicator, TimeOutCb timeout_cb, + void Initialize(const std::shared_ptr &communicator, TimeOutCb timeout_cb, FinishIterCb finish_iteration_cb); // Reinitialize count service and round kernel of this round after scaling operations are done. @@ -48,7 +48,7 @@ class Round { // This method is the callback which will be set to the communicator and called after the corresponding round message // is sent to the server. - void LaunchRoundKernel(const std::shared_ptr &message); + void LaunchRoundKernel(const std::shared_ptr &message); // Round needs to be reset after each iteration is finished or its timer expires. void Reset(); @@ -60,8 +60,8 @@ class Round { private: // The callbacks which will be set to DistributedCounterService. - void OnFirstCountEvent(const std::shared_ptr &message); - void OnLastCountEvent(const std::shared_ptr &message); + void OnFirstCountEvent(const std::shared_ptr &message); + void OnLastCountEvent(const std::shared_ptr &message); std::string name_; @@ -83,7 +83,7 @@ class Round { // Whether this round uses the server number as its threshold count. bool server_num_as_threshold_; - std::shared_ptr communicator_; + std::shared_ptr communicator_; // The round kernel for this Round. std::shared_ptr kernel_; @@ -97,6 +97,6 @@ class Round { FinalizeCb finalize_cb_; }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_ROUND_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_ROUND_H_ diff --git a/mindspore/ccsrc/fl/server/server.cc b/mindspore/ccsrc/fl/server/server.cc index ca4d9f984a1..0ea8924cd72 100644 --- a/mindspore/ccsrc/fl/server/server.cc +++ b/mindspore/ccsrc/fl/server/server.cc @@ -30,17 +30,8 @@ #include "fl/server/kernel/round/round_kernel_factory.h" namespace mindspore { -namespace ps { +namespace fl { namespace server { -static std::vector> global_worker_server_comms = {}; -// This function is for the exit of server process when an interrupt signal is captured. -void SignalHandler(int signal) { - MS_LOG(INFO) << "Interrupt signal captured: " << signal; - std::for_each(global_worker_server_comms.begin(), global_worker_server_comms.end(), - [](const std::shared_ptr &communicator) { communicator->Stop(); }); - return; -} - void Server::Initialize(bool use_tcp, bool use_http, uint16_t http_port, const std::vector &rounds_config, const CipherConfig &cipher_config, const FuncGraphPtr &func_graph, size_t executor_threshold) { MS_EXCEPTION_IF_NULL(func_graph); @@ -76,7 +67,6 @@ void Server::Initialize(bool use_tcp, bool use_http, uint16_t http_port, const s // Getting Model Size relies on ModelStorage Initialization which relies on Executor Initialization: // InitCipher---->InitExecutor void Server::Run() { - signal(SIGINT, SignalHandler); std::unique_lock lock(scaling_mtx_); InitServerContext(); InitCluster(); @@ -84,8 +74,8 @@ void Server::Run() { RegisterCommCallbacks(); StartCommunicator(); InitExecutor(); - std::string encrypt_type = PSContext::instance()->encrypt_type(); - if (encrypt_type != kNotEncryptType) { + std::string encrypt_type = ps::PSContext::instance()->encrypt_type(); + if (encrypt_type != ps::kNotEncryptType) { InitCipher(); MS_LOG(INFO) << "Parameters for secure aggregation have been initiated."; } @@ -96,7 +86,7 @@ void Server::Run() { // Wait communicators to stop so the main thread is blocked. std::for_each(communicators_with_worker_.begin(), communicators_with_worker_.end(), - [](const std::shared_ptr &communicator) { communicator->Join(); }); + [](const std::shared_ptr &communicator) { communicator->Join(); }); communicator_with_server_->Join(); MsException::Instance().CheckException(); return; @@ -115,18 +105,18 @@ void Server::CancelSafeMode() { bool Server::IsSafeMode() { return safemode_.load(); } void Server::InitServerContext() { - PSContext::instance()->GenerateResetterRound(); - scheduler_ip_ = PSContext::instance()->scheduler_host(); - scheduler_port_ = PSContext::instance()->scheduler_port(); - worker_num_ = PSContext::instance()->initial_worker_num(); - server_num_ = PSContext::instance()->initial_server_num(); + ps::PSContext::instance()->GenerateResetterRound(); + scheduler_ip_ = ps::PSContext::instance()->scheduler_host(); + scheduler_port_ = ps::PSContext::instance()->scheduler_port(); + worker_num_ = ps::PSContext::instance()->initial_worker_num(); + server_num_ = ps::PSContext::instance()->initial_server_num(); return; } void Server::InitCluster() { - server_node_ = std::make_shared(); + server_node_ = std::make_shared(); MS_EXCEPTION_IF_NULL(server_node_); - task_executor_ = std::make_shared(32); + task_executor_ = std::make_shared(32); MS_EXCEPTION_IF_NULL(task_executor_); if (!InitCommunicatorWithServer()) { MS_LOG(EXCEPTION) << "Initializing cross-server communicator failed."; @@ -136,7 +126,6 @@ void Server::InitCluster() { MS_LOG(EXCEPTION) << "Initializing worker-server communicator failed."; return; } - global_worker_server_comms = communicators_with_worker_; return; } @@ -187,8 +176,8 @@ void Server::InitIteration() { } #ifdef ENABLE_ARMOUR - std::string encrypt_type = PSContext::instance()->encrypt_type(); - if (encrypt_type == kPWEncryptType) { + std::string encrypt_type = ps::PSContext::instance()->encrypt_type(); + if (encrypt_type == ps::kPWEncryptType) { cipher_initial_client_cnt_ = rounds_config_[0].threshold_count; cipher_exchange_secrets_cnt_ = cipher_initial_client_cnt_ * 1.0; cipher_share_secrets_cnt_ = cipher_initial_client_cnt_ * cipher_config_.share_secrets_ratio; @@ -245,10 +234,10 @@ void Server::InitCipher() { unsigned char cipher_p[SECRET_MAX_LEN] = {0}; int cipher_g = 1; unsigned char cipher_prime[PRIME_MAX_LEN] = {0}; - float dp_eps = PSContext::instance()->dp_eps(); - float dp_delta = PSContext::instance()->dp_delta(); - float dp_norm_clip = PSContext::instance()->dp_norm_clip(); - std::string encrypt_type = PSContext::instance()->encrypt_type(); + float dp_eps = ps::PSContext::instance()->dp_eps(); + float dp_delta = ps::PSContext::instance()->dp_delta(); + float dp_norm_clip = ps::PSContext::instance()->dp_norm_clip(); + std::string encrypt_type = ps::PSContext::instance()->encrypt_type(); mpz_t prim; mpz_init(prim); @@ -276,7 +265,7 @@ void Server::RegisterCommCallbacks() { // The message callbacks of round kernels are already set in method InitIteration, so here we don't need to register // rounds' callbacks. - auto tcp_comm = std::dynamic_pointer_cast(communicator_with_server_); + auto tcp_comm = std::dynamic_pointer_cast(communicator_with_server_); MS_EXCEPTION_IF_NULL(tcp_comm); // Set message callbacks for server-to-server communication. @@ -304,23 +293,23 @@ void Server::RegisterCommCallbacks() { std::bind(&Server::ProcessAfterScalingIn, this)); } -void Server::RegisterExceptionEventCallback(const std::shared_ptr &communicator) { +void Server::RegisterExceptionEventCallback(const std::shared_ptr &communicator) { MS_EXCEPTION_IF_NULL(communicator); - communicator->RegisterEventCallback(core::ClusterEvent::SCHEDULER_TIMEOUT, [&]() { + communicator->RegisterEventCallback(ps::core::ClusterEvent::SCHEDULER_TIMEOUT, [&]() { MS_LOG(ERROR) << "Event SCHEDULER_TIMEOUT is captured. This is because scheduler node is finalized or crashed."; safemode_ = true; std::for_each(communicators_with_worker_.begin(), communicators_with_worker_.end(), - [](const std::shared_ptr &communicator) { communicator->Stop(); }); + [](const std::shared_ptr &communicator) { communicator->Stop(); }); communicator_with_server_->Stop(); }); - communicator->RegisterEventCallback(core::ClusterEvent::NODE_TIMEOUT, [&]() { + communicator->RegisterEventCallback(ps::core::ClusterEvent::NODE_TIMEOUT, [&]() { MS_LOG(ERROR) << "Event NODE_TIMEOUT is captured. This is because some server nodes are finalized or crashed after the " "network building phase."; safemode_ = true; std::for_each(communicators_with_worker_.begin(), communicators_with_worker_.end(), - [](const std::shared_ptr &communicator) { communicator->Stop(); }); + [](const std::shared_ptr &communicator) { communicator->Stop(); }); communicator_with_server_->Stop(); }); } @@ -377,7 +366,7 @@ void Server::StartCommunicator() { MS_LOG(INFO) << "Start communicator with worker."; std::for_each(communicators_with_worker_.begin(), communicators_with_worker_.end(), - [](const std::shared_ptr &communicator) { communicator->Start(); }); + [](const std::shared_ptr &communicator) { communicator->Start(); }); } void Server::ProcessBeforeScalingOut() { @@ -398,23 +387,18 @@ void Server::ProcessAfterScalingOut() { if (!DistributedMetadataStore::GetInstance().ReInitForScaling()) { MS_LOG(WARNING) << "DistributedMetadataStore reinitializing failed."; - return; } if (!CollectiveOpsImpl::GetInstance().ReInitForScaling()) { MS_LOG(WARNING) << "DistributedMetadataStore reinitializing failed."; - return; } if (!DistributedCountService::GetInstance().ReInitForScaling()) { MS_LOG(WARNING) << "DistributedCountService reinitializing failed."; - return; } if (!iteration_->ReInitForScaling(IntToUint(server_node_->server_num()), server_node_->rank_id())) { MS_LOG(WARNING) << "Iteration reinitializing failed."; - return; } if (!Executor::GetInstance().ReInitForScaling()) { MS_LOG(WARNING) << "Executor reinitializing failed."; - return; } std::this_thread::sleep_for(std::chrono::milliseconds(1000)); safemode_ = false; @@ -429,7 +413,7 @@ void Server::ProcessAfterScalingIn() { if (server_node_->rank_id() == UINT32_MAX) { MS_LOG(WARNING) << "This server the one to be scaled in. Server exiting."; std::for_each(communicators_with_worker_.begin(), communicators_with_worker_.end(), - [](const std::shared_ptr &communicator) { communicator->Stop(); }); + [](const std::shared_ptr &communicator) { communicator->Stop(); }); communicator_with_server_->Stop(); return; } @@ -437,27 +421,22 @@ void Server::ProcessAfterScalingIn() { // If the server is not the one to be scaled in, reintialize modules and recover service. if (!DistributedMetadataStore::GetInstance().ReInitForScaling()) { MS_LOG(WARNING) << "DistributedMetadataStore reinitializing failed."; - return; } if (!CollectiveOpsImpl::GetInstance().ReInitForScaling()) { MS_LOG(WARNING) << "DistributedMetadataStore reinitializing failed."; - return; } if (!DistributedCountService::GetInstance().ReInitForScaling()) { MS_LOG(WARNING) << "DistributedCountService reinitializing failed."; - return; } if (!iteration_->ReInitForScaling(IntToUint(server_node_->server_num()), server_node_->rank_id())) { MS_LOG(WARNING) << "Iteration reinitializing failed."; - return; } if (!Executor::GetInstance().ReInitForScaling()) { MS_LOG(WARNING) << "Executor reinitializing failed."; - return; } std::this_thread::sleep_for(std::chrono::milliseconds(1000)); safemode_ = false; } } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/server/server.h b/mindspore/ccsrc/fl/server/server.h index cc8da047efc..30c2491f5e9 100644 --- a/mindspore/ccsrc/fl/server/server.h +++ b/mindspore/ccsrc/fl/server/server.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_SERVER_SERVER_H_ -#define MINDSPORE_CCSRC_PS_SERVER_SERVER_H_ +#ifndef MINDSPORE_CCSRC_FL_SERVER_SERVER_H_ +#define MINDSPORE_CCSRC_FL_SERVER_SERVER_H_ #include #include @@ -31,7 +31,7 @@ #endif namespace mindspore { -namespace ps { +namespace fl { namespace server { // Class Server is the entrance of MindSpore's parameter server training mode and federated learning. class Server { @@ -90,7 +90,7 @@ class Server { void RegisterCommCallbacks(); // Register cluster exception callbacks. This method is called in RegisterCommCallbacks. - void RegisterExceptionEventCallback(const std::shared_ptr &communicator); + void RegisterExceptionEventCallback(const std::shared_ptr &communicator); // Initialize executor according to the server mode. void InitExecutor(); @@ -113,11 +113,11 @@ class Server { void ProcessAfterScalingIn(); // The server node is initialized in Server. - std::shared_ptr server_node_; + std::shared_ptr server_node_; // The task executor of the communicators. This helps server to handle network message concurrently. The tasks // submitted to this task executor is asynchronous. - std::shared_ptr task_executor_; + std::shared_ptr task_executor_; // Which protocol should communicators use. bool use_tcp_; @@ -136,12 +136,12 @@ class Server { // Server need a tcp communicator to communicate with other servers for counting, metadata storing, collective // operations, etc. - std::shared_ptr communicator_with_server_; + std::shared_ptr communicator_with_server_; // The communication with workers(including mobile devices), has multiple protocol types: HTTP and TCP. // In some cases, both types should be supported in one distributed training job. So here we may have multiple // communicators. - std::vector> communicators_with_worker_; + std::vector> communicators_with_worker_; // Mutex for scaling operations. We must wait server's initialization done before handle scaling events. std::mutex scaling_mtx_; @@ -176,6 +176,6 @@ class Server { float percent_for_get_model_; }; } // namespace server -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_SERVER_SERVER_H_ +#endif // MINDSPORE_CCSRC_FL_SERVER_SERVER_H_ diff --git a/mindspore/ccsrc/fl/worker/fl_worker.cc b/mindspore/ccsrc/fl/worker/fl_worker.cc index b0de5a8b490..a004f53facf 100644 --- a/mindspore/ccsrc/fl/worker/fl_worker.cc +++ b/mindspore/ccsrc/fl/worker/fl_worker.cc @@ -22,26 +22,27 @@ #include "utils/ms_exception.h" namespace mindspore { -namespace ps { +namespace fl { namespace worker { void FLWorker::Run() { - worker_num_ = PSContext::instance()->worker_num(); - server_num_ = PSContext::instance()->server_num(); - scheduler_ip_ = PSContext::instance()->scheduler_ip(); - scheduler_port_ = PSContext::instance()->scheduler_port(); - worker_step_num_per_iteration_ = PSContext::instance()->worker_step_num_per_iteration(); - PSContext::instance()->cluster_config().scheduler_host = scheduler_ip_; - PSContext::instance()->cluster_config().scheduler_port = scheduler_port_; - PSContext::instance()->cluster_config().initial_worker_num = worker_num_; - PSContext::instance()->cluster_config().initial_server_num = server_num_; + worker_num_ = ps::PSContext::instance()->worker_num(); + server_num_ = ps::PSContext::instance()->server_num(); + scheduler_ip_ = ps::PSContext::instance()->scheduler_ip(); + scheduler_port_ = ps::PSContext::instance()->scheduler_port(); + worker_step_num_per_iteration_ = ps::PSContext::instance()->worker_step_num_per_iteration(); + ps::PSContext::instance()->cluster_config().scheduler_host = scheduler_ip_; + ps::PSContext::instance()->cluster_config().scheduler_port = scheduler_port_; + ps::PSContext::instance()->cluster_config().initial_worker_num = worker_num_; + ps::PSContext::instance()->cluster_config().initial_server_num = server_num_; MS_LOG(INFO) << "Initialize cluster config for worker. Worker number:" << worker_num_ << ", Server number:" << server_num_ << ", Scheduler ip:" << scheduler_ip_ - << ", Scheduler port:" << scheduler_port_; + << ", Scheduler port:" << scheduler_port_ + << ", Worker training step per iteration:" << worker_step_num_per_iteration_; - worker_node_ = std::make_shared(); + worker_node_ = std::make_shared(); MS_EXCEPTION_IF_NULL(worker_node_); - worker_node_->RegisterEventCallback(core::ClusterEvent::SCHEDULER_TIMEOUT, [this]() { + worker_node_->RegisterEventCallback(ps::core::ClusterEvent::SCHEDULER_TIMEOUT, [this]() { Finalize(); try { MS_LOG(EXCEPTION) @@ -50,7 +51,7 @@ void FLWorker::Run() { MsException::Instance().SetException(); } }); - worker_node_->RegisterEventCallback(core::ClusterEvent::NODE_TIMEOUT, [this]() { + worker_node_->RegisterEventCallback(ps::core::ClusterEvent::NODE_TIMEOUT, [this]() { Finalize(); try { MS_LOG(EXCEPTION) @@ -73,7 +74,7 @@ void FLWorker::Finalize() { worker_node_->Stop(); } -bool FLWorker::SendToServer(uint32_t server_rank, void *data, size_t size, core::TcpUserCommand command, +bool FLWorker::SendToServer(uint32_t server_rank, const void *data, size_t size, ps::core::TcpUserCommand command, std::shared_ptr> *output) { // If the worker is in safemode, do not communicate with server. while (safemode_.load()) { @@ -96,7 +97,8 @@ bool FLWorker::SendToServer(uint32_t server_rank, void *data, size_t size, core: if (output != nullptr) { while (true) { - if (!worker_node_->Send(core::NodeRole::SERVER, server_rank, message, size, static_cast(command), output)) { + if (!worker_node_->Send(ps::core::NodeRole::SERVER, server_rank, message, size, static_cast(command), + output)) { MS_LOG(ERROR) << "Sending message to server " << server_rank << " failed."; return false; } @@ -105,7 +107,7 @@ bool FLWorker::SendToServer(uint32_t server_rank, void *data, size_t size, core: return false; } - if (std::string(reinterpret_cast((*output)->data()), (*output)->size()) == kClusterSafeMode) { + if (std::string(reinterpret_cast((*output)->data()), (*output)->size()) == ps::kClusterSafeMode) { MS_LOG(INFO) << "The server " << server_rank << " is in safemode."; std::this_thread::sleep_for(std::chrono::milliseconds(kWorkerRetryDurationForSafeMode)); } else { @@ -113,7 +115,7 @@ bool FLWorker::SendToServer(uint32_t server_rank, void *data, size_t size, core: } } } else { - if (!worker_node_->Send(core::NodeRole::SERVER, server_rank, message, size, static_cast(command))) { + if (!worker_node_->Send(ps::core::NodeRole::SERVER, server_rank, message, size, static_cast(command))) { MS_LOG(ERROR) << "Sending message to server " << server_rank << " failed."; return false; } @@ -147,16 +149,16 @@ void FLWorker::InitializeFollowerScaler() { worker_node_->RegisterFollowerScalerBarrierBeforeScaleOut("WorkerPipeline", std::bind(&FLWorker::ProcessBeforeScalingOut, this)); worker_node_->RegisterFollowerScalerBarrierBeforeScaleIn("WorkerPipeline", - std::bind(&FLWorker::ProcessBeforeScalingOut, this)); + std::bind(&FLWorker::ProcessBeforeScalingIn, this)); // Set handlers after scheduler scaling operations are done. worker_node_->RegisterFollowerScalerHandlerAfterScaleOut("WorkerPipeline", std::bind(&FLWorker::ProcessAfterScalingOut, this)); worker_node_->RegisterFollowerScalerHandlerAfterScaleIn("WorkerPipeline", std::bind(&FLWorker::ProcessAfterScalingIn, this)); - worker_node_->RegisterCustomEventCallback(static_cast(CustomEvent::kIterationRunning), + worker_node_->RegisterCustomEventCallback(static_cast(ps::CustomEvent::kIterationRunning), std::bind(&FLWorker::HandleIterationRunningEvent, this)); - worker_node_->RegisterCustomEventCallback(static_cast(CustomEvent::kIterationCompleted), + worker_node_->RegisterCustomEventCallback(static_cast(ps::CustomEvent::kIterationCompleted), std::bind(&FLWorker::HandleIterationCompletedEvent, this)); } @@ -221,5 +223,5 @@ void FLWorker::ProcessAfterScalingIn() { safemode_ = false; } } // namespace worker -} // namespace ps +} // namespace fl } // namespace mindspore diff --git a/mindspore/ccsrc/fl/worker/fl_worker.h b/mindspore/ccsrc/fl/worker/fl_worker.h index 4eb5c963330..a1511c24398 100644 --- a/mindspore/ccsrc/fl/worker/fl_worker.h +++ b/mindspore/ccsrc/fl/worker/fl_worker.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef MINDSPORE_CCSRC_PS_WORKER_FL_WORKER_H_ -#define MINDSPORE_CCSRC_PS_WORKER_FL_WORKER_H_ +#ifndef MINDSPORE_CCSRC_FL_WORKER_FL_WORKER_H_ +#define MINDSPORE_CCSRC_FL_WORKER_FL_WORKER_H_ #include #include @@ -28,14 +28,14 @@ #include "ps/core/communicator/tcp_communicator.h" namespace mindspore { -namespace ps { +namespace fl { using FBBuilder = flatbuffers::FlatBufferBuilder; // The step number for worker to judge whether to communicate with server. constexpr uint32_t kTrainBeginStepNum = 1; constexpr uint32_t kTrainEndStepNum = 0; -// The worker has to sleep for a while before the networking is completed. +// The sleeping time of the worker thread before the networking is completed. constexpr uint32_t kWorkerSleepTimeForNetworking = 1000; // The time duration between retrying when server is in safemode. @@ -59,7 +59,7 @@ class FLWorker { } void Run(); void Finalize(); - bool SendToServer(uint32_t server_rank, void *data, size_t size, core::TcpUserCommand command, + bool SendToServer(uint32_t server_rank, const void *data, size_t size, ps::core::TcpUserCommand command, std::shared_ptr> *output = nullptr); uint32_t server_num() const; @@ -104,10 +104,9 @@ class FLWorker { uint32_t worker_num_; std::string scheduler_ip_; uint16_t scheduler_port_; - std::shared_ptr worker_node_; + std::shared_ptr worker_node_; - // The worker standalone training step number before communicating with server. This used in hybrid training mode for - // now. + // The worker standalone training step number before communicating with server. This used in hybrid training mode. uint64_t worker_step_num_per_iteration_; // The iteration state is either running or completed. @@ -115,13 +114,13 @@ class FLWorker { // kIterationRunning/kIterationCompleted. triggered by server. std::atomic server_iteration_state_; - // The variable represents the worker iteration state and should be changed by worker training process. + // This variable represents the worker iteration state and should be changed by worker training process. std::atomic worker_iteration_state_; // The flag that represents whether worker is in safemode, which is decided by both worker and server iteration state. std::atomic_bool safemode_; }; } // namespace worker -} // namespace ps +} // namespace fl } // namespace mindspore -#endif // MINDSPORE_CCSRC_PS_WORKER_FL_WORKER_H_ +#endif // MINDSPORE_CCSRC_FL_WORKER_FL_WORKER_H_ diff --git a/mindspore/ccsrc/pipeline/jit/action.cc b/mindspore/ccsrc/pipeline/jit/action.cc index 3c818d10aac..b10f954222c 100644 --- a/mindspore/ccsrc/pipeline/jit/action.cc +++ b/mindspore/ccsrc/pipeline/jit/action.cc @@ -639,7 +639,7 @@ bool StartPSWorkerAction(const ResourcePtr &res) { return true; } bool StartFLWorkerAction(const ResourcePtr &) { - ps::worker::FLWorker::GetInstance().Run(); + fl::worker::FLWorker::GetInstance().Run(); return true; } @@ -665,7 +665,7 @@ bool StartServerAction(const ResourcePtr &res) { uint64_t start_fl_job_time_window = ps::PSContext::instance()->start_fl_job_time_window(); uint64_t update_model_time_window = ps::PSContext::instance()->update_model_time_window(); - std::vector rounds_config = { + std::vector rounds_config = { {"startFLJob", true, start_fl_job_time_window, true, start_fl_job_threshold}, {"updateModel", true, update_model_time_window, true, update_model_threshold}, {"getModel"}, @@ -676,22 +676,22 @@ bool StartServerAction(const ResourcePtr &res) { uint64_t cipher_time_window = ps::PSContext::instance()->cipher_time_window(); size_t reconstruct_secrets_threshhold = ps::PSContext::instance()->reconstruct_secrets_threshhold(); - ps::server::CipherConfig cipher_config = {share_secrets_ratio, cipher_time_window, reconstruct_secrets_threshhold}; + fl::server::CipherConfig cipher_config = {share_secrets_ratio, cipher_time_window, reconstruct_secrets_threshhold}; size_t executor_threshold = 0; if (server_mode_ == ps::kServerModeFL || server_mode_ == ps::kServerModeHybrid) { executor_threshold = update_model_threshold; - ps::server::Server::GetInstance().Initialize(true, true, fl_server_port, rounds_config, cipher_config, func_graph, + fl::server::Server::GetInstance().Initialize(true, true, fl_server_port, rounds_config, cipher_config, func_graph, executor_threshold); } else if (server_mode_ == ps::kServerModePS) { executor_threshold = worker_num; - ps::server::Server::GetInstance().Initialize(true, false, 0, rounds_config, cipher_config, func_graph, + fl::server::Server::GetInstance().Initialize(true, false, 0, rounds_config, cipher_config, func_graph, executor_threshold); } else { MS_LOG(EXCEPTION) << "Server mode " << server_mode_ << " is not supported."; return false; } - ps::server::Server::GetInstance().Run(); + fl::server::Server::GetInstance().Run(); return true; } diff --git a/mindspore/ccsrc/pipeline/jit/pipeline.cc b/mindspore/ccsrc/pipeline/jit/pipeline.cc index 772062add84..5e2674a0177 100644 --- a/mindspore/ccsrc/pipeline/jit/pipeline.cc +++ b/mindspore/ccsrc/pipeline/jit/pipeline.cc @@ -1293,7 +1293,7 @@ void ClearResAtexit() { MS_LOG(INFO) << "Start finalizing worker."; const std::string &server_mode = ps::PSContext::instance()->server_mode(); if ((server_mode == ps::kServerModeFL || server_mode == ps::kServerModeHybrid)) { - ps::worker::FLWorker::GetInstance().Finalize(); + fl::worker::FLWorker::GetInstance().Finalize(); } else { ps::Worker::GetInstance().Finalize(); } diff --git a/mindspore/ccsrc/ps/core/protos/fl.proto b/mindspore/ccsrc/ps/core/protos/fl.proto index 70dbe6efc54..a478d92a33f 100644 --- a/mindspore/ccsrc/ps/core/protos/fl.proto +++ b/mindspore/ccsrc/ps/core/protos/fl.proto @@ -14,7 +14,7 @@ * limitations under the License. */ syntax = "proto3"; -package mindspore.ps; +package mindspore.fl; message CollectiveData { bytes data = 1; diff --git a/mindspore/ccsrc/ps/ps_context.cc b/mindspore/ccsrc/ps/ps_context.cc index 18b81df612c..9f05f2443a8 100644 --- a/mindspore/ccsrc/ps/ps_context.cc +++ b/mindspore/ccsrc/ps/ps_context.cc @@ -286,8 +286,9 @@ void PSContext::GenerateResetterRound() { return; } - binary_server_context = (is_parameter_server_mode << 0) | (is_federated_learning_mode << 1) | - (is_mixed_training_mode << 2) | (secure_aggregation_ << 3); + binary_server_context = ((unsigned int)is_parameter_server_mode << 0) | + ((unsigned int)is_federated_learning_mode << 1) | + ((unsigned int)is_mixed_training_mode << 2) | ((unsigned int)secure_aggregation_ << 3); if (kServerContextToResetRoundMap.count(binary_server_context) == 0) { resetter_round_ = ResetterRound::kNoNeedToReset; } else {