From abd4239da9ae554a0d536ec5f281209bbde48446 Mon Sep 17 00:00:00 2001 From: xiefangqi Date: Wed, 22 Apr 2020 20:50:37 +0800 Subject: [PATCH] fix random stuck problem --- mindspore/ccsrc/dataset/CMakeLists.txt | 3 +++ .../ccsrc/dataset/engine/datasetops/map_op.cc | 22 ------------------- .../ccsrc/dataset/engine/datasetops/map_op.h | 4 ---- .../dataset/engine/datasetops/shuffle_op.cc | 7 +++++- mindspore/ccsrc/dataset/util/random.cc | 7 +++++- mindspore/ccsrc/dataset/util/services.cc | 6 ++++- 6 files changed, 20 insertions(+), 29 deletions(-) diff --git a/mindspore/ccsrc/dataset/CMakeLists.txt b/mindspore/ccsrc/dataset/CMakeLists.txt index 879a9346bc..8e9b2664dc 100644 --- a/mindspore/ccsrc/dataset/CMakeLists.txt +++ b/mindspore/ccsrc/dataset/CMakeLists.txt @@ -13,6 +13,9 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-attributes") ############################# Options ################################ +if (${CMAKE_SYSTEM_NAME} MATCHES "Windows") + add_definitions(-D _CRT_RAND_S) +endif () if (ENABLE_GPUQUE) add_definitions(-D ENABLE_GPUQUE) message(STATUS "GPU queue is enabled") diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc index 3f8d70b606..b6d603bac9 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.cc @@ -65,9 +65,6 @@ MapOp::MapOp(const std::vector &in_col_names, const std::vectorGetNextBuffer(&buff, 0)); is_eof = buff->eof(); RETURN_IF_NOT_OK(local_queues_[que_id]->Add(std::move(buff))); -#if defined(_WIN32) || defined(_WIN64) - if (is_eof) { - eof_worker_id_ = que_id; - for (int32_t id = 0; id < num_workers_; id++) { - if (id != eof_worker_id_) { - auto eof_buffer = std::make_unique(0, DataBuffer::kDeBFlagEOF); - RETURN_IF_NOT_OK(local_queues_[id]->Add(std::move(eof_buffer))); - } - } - } -#endif que_id = (que_id + 1) % num_workers_; } } @@ -173,14 +159,6 @@ Status MapOp::WorkerEntry(int32_t worker_id) { continue; } else if (in_buffer->eof()) { // Calling base class EofReceived to forward eof buffer. -#if defined(_WIN32) || defined(_Win64) - if (perf_mode_) { - if (eof_worker_id_ == worker_id) { - RETURN_IF_NOT_OK(EofReceived(worker_id)); - } - break; - } -#endif RETURN_IF_NOT_OK(EofReceived(worker_id)); break; } diff --git a/mindspore/ccsrc/dataset/engine/datasetops/map_op.h b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h index 5e16bc3fed..4c9d27f9c7 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/map_op.h +++ b/mindspore/ccsrc/dataset/engine/datasetops/map_op.h @@ -193,10 +193,6 @@ class MapOp : public ParallelOp { // cause additional blocking because pop calls to Connector from the threads are synchronized to enforce the order. bool perf_mode_; -#if defined(_WIN32) || defined(_WIN64) - // EOF worker id is only work on Performance mode, to record the worker id of queue which gets EOF - int32_t eof_worker_id_; -#endif // Private function for worker/thread to loop continuously. It comprises the main // logic of MapOp: getting the data from previous Op, validating user specified column names, // applying a list of TensorOps to each of the data, process the results and then diff --git a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc index bdf39b6a39..422c38f2f2 100644 --- a/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc +++ b/mindspore/ccsrc/dataset/engine/datasetops/shuffle_op.cc @@ -13,6 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#if defined(_WIN32) || defined(_WIN64) +#include +#endif #include #include #include @@ -86,7 +89,9 @@ Status ShuffleOp::SelfReset() { rng_ = std::mt19937_64(shuffle_seed_); } else { #if defined(_WIN32) || defined(_WIN64) - std::random_device random_device; + unsigned int number; + rand_s(&number); + std::mt19937 random_device{static_cast(number)}; #else std::random_device random_device("/dev/urandom"); #endif diff --git a/mindspore/ccsrc/dataset/util/random.cc b/mindspore/ccsrc/dataset/util/random.cc index 2a0762c920..43b3ee4afd 100644 --- a/mindspore/ccsrc/dataset/util/random.cc +++ b/mindspore/ccsrc/dataset/util/random.cc @@ -18,6 +18,9 @@ #include "dataset/util/random.h" +#if defined(_WIN32) || defined(_WIn64) +#include +#endif #include #include #include @@ -33,7 +36,9 @@ uint32_t GetSeed() { uint32_t seed = GlobalContext::config_manager()->seed(); if (seed == std::mt19937::default_seed) { #if defined(_WIN32) || defined(_WIN64) - std::random_device random_device; + unsigned int number; + rand_s(&number); + std::mt19937 random_device{static_cast(number)}; #else std::random_device random_device("/dev/urandom"); #endif diff --git a/mindspore/ccsrc/dataset/util/services.cc b/mindspore/ccsrc/dataset/util/services.cc index ea7b11014c..a2b3f734c2 100644 --- a/mindspore/ccsrc/dataset/util/services.cc +++ b/mindspore/ccsrc/dataset/util/services.cc @@ -18,6 +18,8 @@ #include #if !defined(_WIN32) && !defined(_WIN64) #include +#else +#include #endif #include #include @@ -49,7 +51,9 @@ int Services::GetLWP() { return syscall(SYS_gettid); } std::string Services::GetUniqueID() { const std::string kStr = "abcdefghijklmnopqrstuvwxyz0123456789"; #if defined(_WIN32) || defined(_WIN64) - std::mt19937 gen{std::random_device{}()}; + unsigned int number; + rand_s(&number); + std::mt19937 gen{static_cast(number)}; #else std::mt19937 gen{std::random_device{"/dev/urandom"}()}; #endif