diff --git a/mindspore/ccsrc/runtime/device/CMakeLists.txt b/mindspore/ccsrc/runtime/device/CMakeLists.txt index 25f1a43edd4..d45b0ea50e7 100644 --- a/mindspore/ccsrc/runtime/device/CMakeLists.txt +++ b/mindspore/ccsrc/runtime/device/CMakeLists.txt @@ -36,7 +36,7 @@ if(ENABLE_MPI) set_property(SOURCE ${MPI_SRC_LIST} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_DEVICE) add_library(mpi_adapter SHARED ${MPI_SRC_LIST}) - target_link_libraries(mpi_adapter PRIVATE mindspore::ompi) + target_link_libraries(mpi_adapter PRIVATE mindspore::ompi mindspore::pybind11_module -ldl ${SECUREC_LIBRARY}) endif() if(ENABLE_GPU) diff --git a/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_adapter.cc b/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_adapter.cc index 2ee6266ae61..ce34eacd860 100644 --- a/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_adapter.cc +++ b/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_adapter.cc @@ -27,7 +27,6 @@ namespace cpu { std::shared_ptr MPIAdapter::instance_ = nullptr; std::shared_ptr MPIAdapter::Instance() { if (instance_ == nullptr) { - MS_LOG(DEBUG) << "Create new mpi adapter instance."; instance_.reset(new (std::nothrow) MPIAdapter()); } return instance_; diff --git a/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_export.cc b/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_export.cc index 50eeec92b5b..71ef5c70c64 100644 --- a/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_export.cc +++ b/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_export.cc @@ -15,9 +15,9 @@ */ #include "runtime/device/cpu/mpi/mpi_export.h" #include -#include #include "runtime/device/cpu/mpi/mpi_adapter.h" +extern "C" { int GetMPIRankId() { auto inst = mindspore::device::cpu::MPIAdapter::Instance(); if (inst == nullptr) { @@ -59,3 +59,4 @@ bool MPIAllGather(const float *input, float *output, const std::vector &ran } return inst->AllGather(input, output, ranks_group, data_num); } +} diff --git a/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_export.h b/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_export.h index ca7ea098d43..afc953a9473 100644 --- a/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_export.h +++ b/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_export.h @@ -22,14 +22,13 @@ #define FUNC_EXPORT __attribute__((visibility("default"))) #endif -extern "C" FUNC_EXPORT FUNC_EXPORT int GetMPIRankId(); -extern "C" FUNC_EXPORT FUNC_EXPORT int GetMPIRankSize(); -extern "C" FUNC_EXPORT bool MPIReduceScatter(const float *input, float *output, const std::vector &ranks_group, - size_t data_num, const std::string &op_type); -extern "C" FUNC_EXPORT bool MPIReduceScatterOverwriteInput(float *input, const std::vector &ranks_group, - size_t in_data_num, size_t output_size, - const std::string &op_type, float *output); -extern "C" FUNC_EXPORT bool MPIAllGather(const float *input, float *output, const std::vector &ranks_group, - size_t data_num); - +extern "C" { +FUNC_EXPORT int GetMPIRankId(); +FUNC_EXPORT int GetMPIRankSize(); +FUNC_EXPORT bool MPIReduceScatter(const float *input, float *output, const std::vector &ranks_group, + size_t data_num, const std::string &op_type); +FUNC_EXPORT bool MPIReduceScatterOverwriteInput(float *input, const std::vector &ranks_group, size_t in_data_num, + size_t output_size, const std::string &op_type, float *output); +FUNC_EXPORT bool MPIAllGather(const float *input, float *output, const std::vector &ranks_group, size_t data_num); +} #endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_CPU_MPI_MPI_EXPORT_H_ diff --git a/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_interface.cc b/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_interface.cc index fcc9d441d0a..20713146068 100644 --- a/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_interface.cc +++ b/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_interface.cc @@ -19,6 +19,7 @@ #include #include #include "utils/log_adapter.h" +#include "utils/dlopen_macro.h" inline void *LoadLibrary(const char *name) { auto handle = dlopen(name, RTLD_LAZY | RTLD_LOCAL); @@ -29,16 +30,17 @@ inline void *LoadLibrary(const char *name) { } inline void *GetMPIAdapterHandle() { - static void *handle = LoadLibrary("mpi_adapter.so"); + static void *handle = LoadLibrary("libmpi_adapter.so"); return handle; } -void *GetMPIAdapterFunc(const char *name) { - static void *handle = GetMPIAdapterHandle(); +template +static T GetMPIAdapterFunc(const char *name) { + void *handle = GetMPIAdapterHandle(); if (handle == nullptr) { MS_LOG(EXCEPTION) << "Load lib " << name << " failed, make sure you have installed it!"; } - void *func = dlsym(handle, name); + auto func = reinterpret_cast(dlsym(handle, name)); if (func == nullptr) { MS_LOG(EXCEPTION) << "Load func " << name << " failed, make sure you have implied it!"; } @@ -56,30 +58,29 @@ typedef bool (*MPIAllGatherFunc)(const float *input, float *output, const std::v size_t data_num); int GetMPIRankId() { - static GetMPIRankIdFunc func = reinterpret_cast(GetMPIAdapterFunc("GetMPIRankId")); + auto func = GetMPIAdapterFunc("GetMPIRankId"); return func(); } int GetMPIRankSize() { - static GetMPIRankIdFunc func = reinterpret_cast(GetMPIAdapterFunc("GetMPIRankSize")); + auto func = GetMPIAdapterFunc("GetMPIRankSize"); return func(); } bool MPIReduceScatter(const float *input, float *output, const std::vector &ranks_group, size_t data_num, const std::string &op_type) { - static MPIReduceScatterFunc func = reinterpret_cast(GetMPIAdapterFunc("MPIReduceScatter")); + auto func = GetMPIAdapterFunc("MPIReduceScatter"); return func(input, output, ranks_group, data_num, op_type); } bool MPIReduceScatterOverwriteInput(float *input, const std::vector &ranks_group, size_t in_data_num, size_t output_size, const std::string &op_type, float *output) { - static MPIReduceScatterOverwriteInputFunc func = - reinterpret_cast(GetMPIAdapterFunc("MPIReduceScatterOverwriteInput")); + auto func = GetMPIAdapterFunc("MPIReduceScatterOverwriteInput"); return func(input, ranks_group, in_data_num, output_size, op_type, output); } bool MPIAllGather(const float *input, float *output, const std::vector &ranks_group, size_t data_num) { - static MPIAllGatherFunc func = reinterpret_cast(GetMPIAdapterFunc("MPIAllGather")); + auto func = GetMPIAdapterFunc("MPIAllGather"); return func(input, output, ranks_group, data_num); } #endif // ENABLE_MPI diff --git a/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_interface.h b/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_interface.h index cc0ef6e08b4..878f5456820 100644 --- a/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_interface.h +++ b/mindspore/ccsrc/runtime/device/cpu/mpi/mpi_interface.h @@ -17,11 +17,8 @@ #define MINDSPORE_CCSRC_RUNTIME_DEVICE_CPU_MPI_MPI_INTERFACE_H_ #include #include -#ifndef FUNC_EXPORT -#define FUNC_EXPORT __attribute__((visibility("default"))) -#endif -constexpr auto kMPIOpTypeSum = "sum"; #ifdef ENABLE_MPI +constexpr auto kMPIOpTypeSum = "sum"; int GetMPIRankId(); int GetMPIRankSize(); bool MPIReduceScatter(const float *input, float *output, const std::vector &ranks_group, size_t data_num, diff --git a/mindspore/ccsrc/runtime/device/memory_scheduler.cc b/mindspore/ccsrc/runtime/device/memory_scheduler.cc index 1649713fc66..22357cea09b 100644 --- a/mindspore/ccsrc/runtime/device/memory_scheduler.cc +++ b/mindspore/ccsrc/runtime/device/memory_scheduler.cc @@ -47,7 +47,7 @@ void MemScheduler::Record(const void *key, const EventType &event_type, size_t m auto event = std::make_shared(event_type, compute_index_); event->mem_size = mem_size; event->key = key; - mem_events_[key].emplace_back(event); + (void)mem_events_[key].emplace_back(event); } void MemScheduler::Init(const void *key, void *host_ptr, size_t mem_size, MemPriority priority) { @@ -136,7 +136,7 @@ bool MemScheduler::PreCompute(void *stream) { mem_result_[event->key] = device_ptr; if (!from_init) { mem_handler_->FreeHost(host_ptr); - swap_host_ptr_.erase(event->key); + (void)swap_host_ptr_.erase(event->key); } } } @@ -177,7 +177,7 @@ bool MemScheduler::PostCompute(void *stream) { MS_EXCEPTION_IF_NULL(host_ptr); mem_handler_->SwapOut(device_ptr, host_ptr, event->mem_size, stream); mem_handler_->FreeDevice(device_ptr); - mem_result_.erase(device_ptr); + (void)mem_result_.erase(device_ptr); } } ++compute_index_; @@ -303,7 +303,7 @@ void MemScheduler::GenNoSwapEventSet() { cur_mem_used[i] -= event->mem_size; } } else { - no_swap_events_.emplace(event); + (void)no_swap_events_.emplace(event); } } } @@ -329,7 +329,7 @@ void MemScheduler::GenEvents() { } if ((first_event->type == kInit || first_event->type == kMalloc) && first_event->index < pre_compute_events_.size()) { - pre_compute_events_[first_event->index].emplace_back(first_event); + (void)pre_compute_events_[first_event->index].emplace_back(first_event); } else { MS_LOG_EXCEPTION << "First event should be init or malloc!"; } @@ -347,14 +347,14 @@ void MemScheduler::GenEvents() { auto swap_out_event = std::make_shared(kSwapOut, pre_index); swap_out_event->key = item.first; swap_out_event->mem_size = first_event->mem_size; - post_compute_events_[pre_index].emplace_back(swap_out_event); + (void)post_compute_events_[pre_index].emplace_back(swap_out_event); auto swap_in_event = std::make_shared(kSwapIn, event->index); swap_in_event->key = item.first; swap_in_event->mem_size = first_event->mem_size; - pre_compute_events_[event->index].emplace_back(swap_in_event); + (void)pre_compute_events_[event->index].emplace_back(swap_in_event); } if (event->index < pre_compute_events_.size()) { - pre_compute_events_[event->index].emplace_back(event); + (void)pre_compute_events_[event->index].emplace_back(event); } pre_index = event->index; } @@ -366,7 +366,7 @@ void MemScheduler::GenEvents() { auto free_event = std::make_shared(kFree, last_event->index); free_event->key = item.first; if (last_event->index < post_compute_events_.size()) { - post_compute_events_[last_event->index].emplace_back(free_event); + (void)post_compute_events_[last_event->index].emplace_back(free_event); } } }