From a064677be90db6ac75cba05fd93a42cc88ad877c Mon Sep 17 00:00:00 2001 From: xiefangqi Date: Tue, 24 Aug 2021 20:30:43 +0800 Subject: [PATCH] add shared memory note --- mindspore/dataset/core/config.py | 3 +++ mindspore/dataset/engine/datasets.py | 1 + 2 files changed, 4 insertions(+) diff --git a/mindspore/dataset/core/config.py b/mindspore/dataset/core/config.py index 02137f128f6..603d537acbc 100644 --- a/mindspore/dataset/core/config.py +++ b/mindspore/dataset/core/config.py @@ -448,6 +448,9 @@ def set_enable_shared_mem(enable): """ if not isinstance(enable, bool): raise TypeError("enable must be of type bool.") + logger.warning("The shared memory is on, multiprocessing performance will be improved. " + "Note: the required shared memory can't exceeds 80% of the available shared memory. " + "You can reduce max_rowsize or reduce num_parallel_workers to reduce shared memory usage.") _config.set_enable_shared_mem(enable) diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index 34d21d570ff..ed83b706079 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -3582,6 +3582,7 @@ def _check_shm_usage(num_worker, queue_size, max_rowsize, num_queues=1): if shm_estimate_usage >= threshold_ratio * shm_available: raise RuntimeError( "Insufficient shared memory available. Required: {}, Available: {}. " + "The required memory can't exceed 80% of the available shared memory. " "Recommend to set_enable_shared_mem to False, reduce max_rowsize or reduce num_parallel_workers." .format(shm_estimate_usage, shm_available)) except FileNotFoundError: