diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc index d8e9514c31..a50986d76c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc @@ -129,7 +129,7 @@ Status CacheAdminArgHandler::AssignArg(const std::string &option, std::vector> value_as_uint; if (arg_stream->fail()) { arg_stream->clear(); - arg_stream->seekg(pos, arg_stream->beg); + (void)arg_stream->seekg(pos, std::ios::beg); break; } else { out_arg->push_back(value_as_uint); diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/storage_manager.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/storage_manager.cc index 7f38d4f5d9..4c088a3328 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/storage_manager.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/storage_manager.cc @@ -59,6 +59,7 @@ Status StorageManager::DoServiceStart() { writable_containers_pool_.reserve(pool_size_); if (root_.IsDirectory()) { // create multiple containers and store their index in a pool + CHECK_FAIL_RETURN_UNEXPECTED(pool_size_ > 0, "Expect positive pool_size_, but got:" + std::to_string(pool_size_)); for (int i = 0; i < pool_size_; i++) { RETURN_IF_NOT_OK(AddOneContainer()); } @@ -172,10 +173,10 @@ Status StorageManager::DoServiceStop() noexcept { return rc1; } -StorageManager::StorageManager(const Path &root) : root_(root), pool_size_(1), file_id_(0), index_() {} +StorageManager::StorageManager(const Path &root) : root_(root), file_id_(0), index_(), pool_size_(1) {} StorageManager::StorageManager(const Path &root, int pool_size) - : root_(root), pool_size_(pool_size), file_id_(0), index_() {} + : root_(root), file_id_(0), index_(), pool_size_(pool_size) {} StorageManager::~StorageManager() { (void)StorageManager::DoServiceStop(); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc index d68f0b5e0e..64cdb151a7 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc @@ -248,7 +248,6 @@ Status RandomDataOp::WorkerEntry(int32_t worker_id) { // Send new_row out RETURN_IF_NOT_OK(out_connector_->Add(std::move(new_row), worker_id)); - } else { // Now, let's enter the epoch sync RETURN_IF_NOT_OK(EpochSync(worker_id, &quitting)); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc index e28ea44ef3..a0e9076eac 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc @@ -24,7 +24,7 @@ namespace mindspore { namespace dataset { MindRecordSamplerRT::MindRecordSamplerRT(mindrecord::ShardReader *shard_reader, int64_t samples_per_tensor) - : SamplerRT(0, samples_per_tensor), next_id_(0), shard_reader_(shard_reader) {} + : SamplerRT(0, samples_per_tensor), shard_reader_(shard_reader), sample_ids_(nullptr), next_id_(0) {} Status MindRecordSamplerRT::GetNextSample(TensorRow *out) { if (next_id_ > num_samples_) { @@ -50,7 +50,6 @@ Status MindRecordSamplerRT::GetNextSample(TensorRow *out) { Status MindRecordSamplerRT::InitSampler() { sample_ids_ = shard_reader_->GetSampleIds(); - if (!sample_ids_) { // Note, sample_ids_.empty() is okay and will just give no sample ids. RETURN_STATUS_UNEXPECTED( diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/samplers/mindrecord_sampler_ir.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/samplers/mindrecord_sampler_ir.cc index fd07a2f3d1..d2babdf39a 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/samplers/mindrecord_sampler_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/samplers/mindrecord_sampler_ir.cc @@ -26,7 +26,6 @@ namespace mindspore { namespace dataset { - #ifndef ENABLE_ANDROID // This function not only creates a runtime sampler object, but also creates a ShardReader, // which will also be needed to build a runtime MindRecordOp @@ -51,6 +50,5 @@ Status MindRecordSamplerObj::GetShardReader(std::unique_ptr(task); diff --git a/mindspore/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc b/mindspore/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc index 5cd3f754ac..fe472a9cd8 100644 --- a/mindspore/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc +++ b/mindspore/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc @@ -43,8 +43,7 @@ int64_t ShardShuffle::GetNumSamples(int64_t dataset_size, int64_t num_classes) { } MSRStatus ShardShuffle::CategoryShuffle(ShardTaskList &tasks) { - uint32_t individual_size; - individual_size = tasks.sample_ids_.size() / tasks.categories; + uint32_t individual_size = tasks.sample_ids_.size() / tasks.categories; std::vector> new_permutations(tasks.categories, std::vector(individual_size)); for (uint32_t i = 0; i < tasks.categories; i++) { for (uint32_t j = 0; j < individual_size; j++) new_permutations[i][j] = static_cast(j); diff --git a/mindspore/ccsrc/minddata/mindrecord/meta/shard_task_list.cc b/mindspore/ccsrc/minddata/mindrecord/meta/shard_task_list.cc index a4d74ad0fd..5f52630ccf 100644 --- a/mindspore/ccsrc/minddata/mindrecord/meta/shard_task_list.cc +++ b/mindspore/ccsrc/minddata/mindrecord/meta/shard_task_list.cc @@ -79,7 +79,8 @@ uint32_t ShardTaskList::SizeOfRows() const { if (task_list_.size() == 0) return static_cast(0); // 1 task is 1 page - auto sum_num_rows = [](int x, ShardTask y) { return x + std::get<2>(y)[0]; }; + const size_t kBlobInfoIndex = 2; + auto sum_num_rows = [](int x, ShardTask y) { return x + std::get(y)[0]; }; uint32_t nRows = std::accumulate(task_list_.begin(), task_list_.end(), 0, sum_num_rows); return nRows; } diff --git a/mindspore/dataset/core/validator_helpers.py b/mindspore/dataset/core/validator_helpers.py index b9efb14882..60436eadaf 100644 --- a/mindspore/dataset/core/validator_helpers.py +++ b/mindspore/dataset/core/validator_helpers.py @@ -229,6 +229,13 @@ def check_uint32(value, arg_name=""): def check_pos_uint32(value, arg_name=""): + """ + Validates the value of a variable is within the range of positive uint32. + + :param value: the value of the variable + :param arg_name: name of the variable to be validated + :return: Exception: when the validation fails, nothing otherwise. + """ type_check(value, (int,), arg_name) check_value(value, [POS_INT_MIN, UINT32_MAX]) diff --git a/mindspore/dataset/transforms/py_transforms.py b/mindspore/dataset/transforms/py_transforms.py index bfc83b4a52..a804bcc719 100644 --- a/mindspore/dataset/transforms/py_transforms.py +++ b/mindspore/dataset/transforms/py_transforms.py @@ -21,6 +21,7 @@ from .validators import check_one_hot_op, check_compose_list, check_random_apply from . import py_transforms_util as util from .c_transforms import TensorOperation + def not_random(function): """ Specify the function as "not random", i.e., it produces deterministic result. diff --git a/mindspore/dataset/transforms/py_transforms_util.py b/mindspore/dataset/transforms/py_transforms_util.py index 20230902e4..05d5b2a5f7 100644 --- a/mindspore/dataset/transforms/py_transforms_util.py +++ b/mindspore/dataset/transforms/py_transforms_util.py @@ -148,7 +148,7 @@ class FuncWrapper: try: if hasattr(self.transform, "random") and not self.transform.random: self.random = False - except Exception: + except KeyError: self.random = True def __call__(self, *args): diff --git a/model_zoo/official/cv/mobilenetv2/scripts/run_train_nfs_cache.sh b/model_zoo/official/cv/mobilenetv2/scripts/run_train_nfs_cache.sh index 6c6b872bcd..cc9ee9b98e 100644 --- a/model_zoo/official/cv/mobilenetv2/scripts/run_train_nfs_cache.sh +++ b/model_zoo/official/cv/mobilenetv2/scripts/run_train_nfs_cache.sh @@ -15,6 +15,7 @@ # ============================================================================ CURPATH="$(dirname "$0")" +# shellcheck source=/dev/null . ${CURPATH}/cache_util.sh run_ascend() diff --git a/model_zoo/official/cv/resnet/scripts/run_distribute_train.sh b/model_zoo/official/cv/resnet/scripts/run_distribute_train.sh index 017c3cffcd..f85fbbdb81 100755 --- a/model_zoo/official/cv/resnet/scripts/run_distribute_train.sh +++ b/model_zoo/official/cv/resnet/scripts/run_distribute_train.sh @@ -15,6 +15,7 @@ # ============================================================================ CURPATH="$(dirname "$0")" +# shellcheck source=/dev/null . ${CURPATH}/cache_util.sh if [ $# != 3 ] && [ $# != 4 ] && [ $# != 5 ] diff --git a/model_zoo/official/cv/resnet/scripts/run_distribute_train_gpu.sh b/model_zoo/official/cv/resnet/scripts/run_distribute_train_gpu.sh index ebd382a6a5..b44116f992 100755 --- a/model_zoo/official/cv/resnet/scripts/run_distribute_train_gpu.sh +++ b/model_zoo/official/cv/resnet/scripts/run_distribute_train_gpu.sh @@ -15,6 +15,7 @@ # ============================================================================ CURPATH="$(dirname "$0")" +# shellcheck source=/dev/null . ${CURPATH}/cache_util.sh if [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ] diff --git a/model_zoo/official/cv/resnet/scripts/run_standalone_train.sh b/model_zoo/official/cv/resnet/scripts/run_standalone_train.sh index 7e8e24f157..402e01a686 100755 --- a/model_zoo/official/cv/resnet/scripts/run_standalone_train.sh +++ b/model_zoo/official/cv/resnet/scripts/run_standalone_train.sh @@ -15,6 +15,7 @@ # ============================================================================ CURPATH="$(dirname "$0")" +# shellcheck source=/dev/null . ${CURPATH}/cache_util.sh if [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ] diff --git a/model_zoo/official/cv/resnet/scripts/run_standalone_train_gpu.sh b/model_zoo/official/cv/resnet/scripts/run_standalone_train_gpu.sh index 1a38f7ba45..edb85580ac 100755 --- a/model_zoo/official/cv/resnet/scripts/run_standalone_train_gpu.sh +++ b/model_zoo/official/cv/resnet/scripts/run_standalone_train_gpu.sh @@ -15,6 +15,7 @@ # ============================================================================ CURPATH="$(dirname "$0")" +# shellcheck source=/dev/null . ${CURPATH}/cache_util.sh if [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ]