From be65ebcb28fda69e298ac82bb7049a5fb64e1778 Mon Sep 17 00:00:00 2001 From: ms_yan Date: Mon, 19 Apr 2021 21:37:07 +0800 Subject: [PATCH] complete dataset log --- .../minddata/dataset/engine/data_schema.cc | 2 +- .../dataset/engine/datasetops/batch_op.cc | 47 +++++++++------- .../datasetops/bucket_batch_by_length_op.cc | 6 ++- .../build_sentence_piece_vocab_op.cc | 9 ++-- .../engine/datasetops/build_vocab_op.cc | 6 +-- .../engine/datasetops/cache_base_op.cc | 4 +- .../engine/datasetops/cache_merge_op.cc | 11 ++-- .../dataset/engine/datasetops/cache_op.cc | 4 +- .../dataset/engine/datasetops/dataset_op.cc | 18 ++++--- .../engine/datasetops/device_queue_op.cc | 23 ++++---- .../engine/datasetops/epoch_ctrl_op.cc | 2 +- .../dataset/engine/datasetops/filter_op.cc | 3 +- .../dataset/engine/datasetops/parallel_op.cc | 2 +- .../dataset/engine/datasetops/project_op.cc | 2 +- .../dataset/engine/datasetops/shuffle_op.cc | 6 +-- .../engine/datasetops/source/album_op.cc | 9 ++-- .../engine/datasetops/source/celeba_op.cc | 19 ++++--- .../engine/datasetops/source/cifar_op.cc | 37 ++++++++----- .../engine/datasetops/source/clue_op.cc | 11 ++-- .../engine/datasetops/source/coco_op.cc | 54 ++++++++++--------- .../engine/datasetops/source/csv_op.cc | 21 +++++--- .../engine/datasetops/source/generator_op.cc | 20 +++++-- .../datasetops/source/image_folder_op.cc | 6 +-- .../engine/datasetops/source/io_block.cc | 8 +-- .../engine/datasetops/source/manifest_op.cc | 14 ++--- .../datasetops/source/mappable_leaf_op.cc | 2 +- .../engine/datasetops/source/mindrecord_op.cc | 11 ++-- .../engine/datasetops/source/mnist_op.cc | 16 +++--- .../datasetops/source/random_data_op.cc | 4 +- .../source/sampler/distributed_sampler.cc | 9 ++-- .../source/sampler/mind_record_sampler.cc | 8 ++- .../datasetops/source/sampler/pk_sampler.cc | 10 ++-- .../source/sampler/python_sampler.cc | 2 +- .../source/sampler/random_sampler.cc | 7 +-- .../datasetops/source/sampler/sampler.cc | 12 ++--- .../source/sampler/sequential_sampler.cc | 12 +++-- .../source/sampler/weighted_random_sampler.cc | 2 +- .../engine/datasetops/source/text_file_op.cc | 7 ++- .../engine/datasetops/source/tf_reader_op.cc | 19 ++++--- .../engine/datasetops/source/voc_op.cc | 3 +- .../dataset/engine/datasetops/zip_op.cc | 4 +- mindspore/dataset/engine/validators.py | 4 +- .../python/dataset/test_datasets_cifarop.py | 4 +- tests/ut/python/dataset/test_datasets_coco.py | 4 +- .../python/dataset/test_datasets_generator.py | 2 +- .../dataset/test_datasets_manifestop.py | 4 +- .../ut/python/dataset/test_datasets_mnist.py | 2 +- tests/ut/python/dataset/test_var_batch_map.py | 2 +- 48 files changed, 290 insertions(+), 204 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc b/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc index f3931e3fb99..bab9a57a8c4 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/data_schema.cc @@ -358,7 +358,7 @@ Status DataSchema::LoadSchemaFile(const std::string &schema_file_path, } } catch (const std::exception &err) { // Catch any exception and convert to Status return code - RETURN_STATUS_UNEXPECTED("Schema file failed to load"); + RETURN_STATUS_UNEXPECTED("Schema file failed to load with JSON tools. File is: " + schema_file_path); } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc index 6bed7eee60b..2e3beaba4d8 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc @@ -169,7 +169,7 @@ void BatchOp::Print(std::ostream &out, bool show_all) const { Status BatchOp::BatchRows(const std::unique_ptr *src, TensorRow *dest, dsize_t batch_size) { if ((*src)->size() != batch_size) { - RETURN_STATUS_UNEXPECTED("[Internal Batch ERROR] Source table size does not match the batch_size"); + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Source table size does not match the batch_size."); } if (batch_size == 1) { @@ -201,9 +201,12 @@ Status BatchOp::BatchRows(const std::unique_ptr *src, TensorRow *d } // Don't do anything if the tensor has no data } else { + std::stringstream shape1, shape2; + first_shape.Print(shape1); + old_tensor->shape().Print(shape2); RETURN_STATUS_UNEXPECTED( "Invalid data, expect same shape for each data row, but got inconsistent data shapes in column " + - std::to_string(i)); + std::to_string(i) + " expected shape for this column is:" + shape1.str() + ", got shape:" + shape2.str()); } } } else { // handle string column differently @@ -253,7 +256,8 @@ Status BatchOp::MakeBatchedRow(std::pair, CBatchIn Status BatchOp::LaunchThreadsAndInitOp() { if (tree_ == nullptr) { - return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "Pipeline init failed, Execution tree not set."); + return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, + "[Internal ERROR] Pipeline init failed, Execution tree not set."); } RETURN_IF_NOT_OK(worker_queues_.Register(tree_->AllTasks())); RETURN_IF_NOT_OK( @@ -330,23 +334,24 @@ Status BatchOp::InvokeBatchSizeFunc(int32_t *batch_size, CBatchInfo info) { // Acquire Python GIL py::gil_scoped_acquire gil_acquire; if (Py_IsInitialized() == 0) { - return Status(StatusCode::kMDPythonInterpreterFailure, "Python Interpreter is finalized."); + return Status(StatusCode::kMDPythonInterpreterFailure, "[Internal ERROR] Python Interpreter is finalized."); } try { py::object size = batch_size_func_(info); *batch_size = size.cast(); if (*batch_size <= 0) { return Status(StatusCode::kMDPyFuncException, - "Invalid parameter, batch size function should return an integer greater than 0."); + "Invalid parameter, batch_size function should return an integer greater than 0, but got: " + + std::to_string(*batch_size)); } } catch (const py::error_already_set &e) { return Status(StatusCode::kMDPyFuncException, e.what()); } catch (const py::cast_error &e) { return Status(StatusCode::kMDPyFuncException, - "Invalid parameter, batch size function should return an integer greater than 0."); + "Invalid parameter, batch_size function should return an integer greater than 0."); } } - return Status(StatusCode::kSuccess, "Batch size func call succeed."); + return Status(StatusCode::kSuccess, "batch_size function call succeedeed."); } Status BatchOp::InvokeBatchMapFunc(TensorTable *input, TensorTable *output, CBatchInfo info) { @@ -354,7 +359,7 @@ Status BatchOp::InvokeBatchMapFunc(TensorTable *input, TensorTable *output, CBat // Acquire Python GIL py::gil_scoped_acquire gil_acquire; if (Py_IsInitialized() == 0) { - return Status(StatusCode::kMDPythonInterpreterFailure, "Python Interpreter is finalized."); + return Status(StatusCode::kMDPythonInterpreterFailure, "[Internal ERROR] Python Interpreter is finalized."); } try { // Prepare batch map call back parameters @@ -373,11 +378,12 @@ Status BatchOp::InvokeBatchMapFunc(TensorTable *input, TensorTable *output, CBat py::object ret_py_obj = batch_map_func_(*input_args); // Parse batch map return value py::tuple ret_tuple = py::cast(ret_py_obj); - CHECK_FAIL_RETURN_UNEXPECTED(py::isinstance(ret_tuple), "Batch map function should return a tuple."); - CHECK_FAIL_RETURN_UNEXPECTED( - ret_tuple.size() == out_col_names_.size(), - "Incorrect number of columns returned. expects: " + std::to_string(out_col_names_.size()) + - " gets: " + std::to_string(ret_tuple.size())); + CHECK_FAIL_RETURN_UNEXPECTED(py::isinstance(ret_tuple), + "per_batch_map function should return a tuple."); + CHECK_FAIL_RETURN_UNEXPECTED(ret_tuple.size() == out_col_names_.size(), + "Incorrect number of columns returned in per_batch_map function. Expects: " + + std::to_string(out_col_names_.size()) + + " got: " + std::to_string(ret_tuple.size())); for (size_t i = 0; i < ret_tuple.size(); i++) { TensorRow output_batch; // If user returns a type that is neither a list nor an array, issue a error msg. @@ -399,7 +405,7 @@ Status BatchOp::InvokeBatchMapFunc(TensorTable *input, TensorTable *output, CBat return Status(StatusCode::kMDPyFuncException, e.what()); } catch (const py::cast_error &e) { return Status(StatusCode::kMDPyFuncException, - "Invalid parameter, batch map function should return a tuple of list of numpy array."); + "Invalid parameter, per_batch_map function of batch should return a tuple of list of numpy array."); } } return Status::OK(); @@ -488,9 +494,12 @@ Status BatchOp::UnpackPadInfo(const PadInfo &pad_info, } Status BatchOp::ComputeColMap() { - CHECK_FAIL_RETURN_UNEXPECTED(child_.size() == 1, - "Batch has " + std::to_string(child_.size()) + " child/children, expects only 1 child."); - CHECK_FAIL_RETURN_UNEXPECTED(!(child_[0]->column_name_id_map().empty()), "BatchOp child map is empty."); + CHECK_FAIL_RETURN_UNEXPECTED( + child_.size() == 1, + "Batch operator expects only 1 child node, but number of children nodes is: " + std::to_string(child_.size()) + + ". Check your script how many node composed into Batch node."); + CHECK_FAIL_RETURN_UNEXPECTED(!(child_[0]->column_name_id_map().empty()), + "Column of Batch operator's child is empty."); if (in_col_names_.empty()) { // if per_batch_map is not set, do not need to deal with out_col_names column_name_id_map_ = child_[0]->column_name_id_map(); @@ -502,7 +511,7 @@ Status BatchOp::ComputeColMap() { // check all input columns exist for (const auto &col : in_col_names_) { - CHECK_FAIL_RETURN_UNEXPECTED(child_map_.find(col) != child_map_.end(), "col:" + col + " doesn't exist."); + CHECK_FAIL_RETURN_UNEXPECTED(child_map_.find(col) != child_map_.end(), "col:" + col + " doesn't exist in dataset."); } // following logic deals with per_batch_map @@ -538,7 +547,7 @@ Status BatchOp::ComputeColMap() { } CHECK_FAIL_RETURN_UNEXPECTED(column_name_id_map_.size() == (child_map_no_in_col.size() + out_col_names_.size()), - "Key error in column_name_id_map_. output_columns is NOT set correctly!"); + "Key error in column_name_id_map_. output_columns in batch is not set correctly!"); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/bucket_batch_by_length_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/bucket_batch_by_length_op.cc index e53de99c0ab..e9b61aeb8ec 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/bucket_batch_by_length_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/bucket_batch_by_length_op.cc @@ -107,7 +107,8 @@ Status BucketBatchByLengthOp::ObtainElementLength(int32_t *out_element_length, T for (size_t i = 0; i < number_of_arguments; i++) { auto map_item = column_name_id_map_.find(length_dependent_columns_[i]); if (map_item == column_name_id_map_.end()) { - RETURN_STATUS_UNEXPECTED("BucketBatchByLength: Couldn't find the specified column in the dataset"); + RETURN_STATUS_UNEXPECTED("BucketBatchByLength: Couldn't find the specified column(" + + length_dependent_columns_[i] + ") in the dataset."); } int32_t column_index = map_item->second; input.push_back(element[column_index]); @@ -116,7 +117,8 @@ Status BucketBatchByLengthOp::ObtainElementLength(int32_t *out_element_length, T RETURN_IF_NOT_OK(output.at(0)->GetItemAt(out_element_length, {0})); if (*out_element_length < 0) { RETURN_STATUS_UNEXPECTED( - "Invalid parameter, element_length_function must return an integer greater than or equal to 0."); + "Invalid parameter, element_length_function must return an integer greater than or equal to 0, but got" + + std::to_string(*out_element_length)); } } else { *out_element_length = element[0]->shape()[0]; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.cc index eba40384184..67c280b3eaa 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.cc @@ -58,7 +58,7 @@ Status BuildSentencePieceVocabOp::operator()() { RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); } RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); - CHECK_FAIL_RETURN_UNEXPECTED(!eoe_warning, "no op should be after from_dataset (repeat detected)"); + CHECK_FAIL_RETURN_UNEXPECTED(!eoe_warning, "no operator should be after from_dataset (repeat detected)"); eoe_warning = true; } // add empty tensorRow for quit @@ -71,12 +71,13 @@ Status BuildSentencePieceVocabOp::SentenceThread() { TaskManager::FindMe()->Post(); if (col_names_.empty() == true) { auto itr = column_name_id_map_.find("text"); - CHECK_FAIL_RETURN_UNEXPECTED(itr != column_name_id_map_.end(), "Invalid data, 'text' column does not exist."); + CHECK_FAIL_RETURN_UNEXPECTED(itr != column_name_id_map_.end(), + "Invalid data, 'text' column does not exist in dataset."); col_id_ = itr->second; } else { auto itr = column_name_id_map_.find(col_names_[0]); CHECK_FAIL_RETURN_UNEXPECTED(itr != column_name_id_map_.end(), - "Invalid parameter, column name: " + col_names_[0] + " does not exist."); + "Invalid parameter, column name: " + col_names_[0] + " does not exist in dataset."); col_id_ = itr->second; } std::unique_ptr sentence_iter = std::make_unique(this); @@ -88,7 +89,7 @@ Status BuildSentencePieceVocabOp::SentenceThread() { } else { if (vocab_ == nullptr) { return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, - "Invalid parameter, sentencepiece vocab not set."); + "Invalid parameter, SentencePiece vocab not set."); } vocab_->set_model_proto(model_proto); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.cc index d988d2eb7f2..be363ade17a 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.cc @@ -97,7 +97,7 @@ Status BuildVocabOp::operator()() { for (std::string col : col_names_) { auto itr = column_name_id_map_.find(col); CHECK_FAIL_RETURN_UNEXPECTED(itr != column_name_id_map_.end(), - "Invalid parameter, column name: " + col + " does not exist."); + "Invalid parameter, column name: " + col + " does not exist in dataset."); col_ids_.push_back(itr->second); } } else { @@ -113,7 +113,7 @@ Status BuildVocabOp::operator()() { RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); } RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row)); - CHECK_FAIL_RETURN_UNEXPECTED(!eoe_warning, "no op should be after from_dataset (repeat detected)"); + CHECK_FAIL_RETURN_UNEXPECTED(!eoe_warning, "no operator should be after from_dataset (repeat detected)"); eoe_warning = true; } @@ -137,7 +137,7 @@ Status BuildVocabOp::CollectorThread() { ++num_quited_worker; } } // all frequencies are obtained - CHECK_FAIL_RETURN_UNEXPECTED(!word_cnt_.empty(), "Invalid data, no words in the dataset."); + CHECK_FAIL_RETURN_UNEXPECTED(!word_cnt_.empty(), "Invalid data, there are no words in the dataset."); std::vector words; // make sure enough is reserved, this will become a partially sorted list eventually words.reserve(wrkr_map->size()); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc index 83065591f22..089426ecbfa 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc @@ -227,7 +227,7 @@ Status CacheBase::UpdateColumnMapFromCache() { Status CacheBase::GetPrefetchRow(row_id_type row_id, TensorRow *out) { RETURN_UNEXPECTED_IF_NULL(out); - CHECK_FAIL_RETURN_UNEXPECTED(row_id >= 0, "Expect positive row id"); + CHECK_FAIL_RETURN_UNEXPECTED(row_id >= 0, "Expect positive row id, but got:" + std::to_string(row_id)); RETURN_IF_NOT_OK(prefetch_.PopFront(row_id, out)); return Status::OK(); } @@ -280,7 +280,7 @@ Status CacheBase::Prefetcher(int32_t worker_id) { cache_miss.clear(); std::unique_ptr blk; RETURN_IF_NOT_OK(prefetch_queues_[worker_id]->PopFront(&blk)); - CHECK_FAIL_RETURN_UNEXPECTED(!blk->eof(), "Expect eoe or a regular io block"); + CHECK_FAIL_RETURN_UNEXPECTED(!blk->eof(), "Expect eoe or a regular io block."); if (!blk->eoe()) { RETURN_IF_NOT_OK(blk->GetKeys(&prefetch_keys)); Status rc; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc index 45b061c320a..46a9699f47f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc @@ -127,7 +127,7 @@ Status CacheMergeOp::CacheMissWorkerEntry(int32_t workerId) { } else { row_id_type row_id = new_row.getId(); if (row_id < 0) { - std::string errMsg = "Expect positive row id: " + std::to_string(row_id); + std::string errMsg = "Expect positive row id, but got: " + std::to_string(row_id); RETURN_STATUS_UNEXPECTED(errMsg); } if (cache_missing_rows_) { @@ -191,7 +191,9 @@ Status CacheMergeOp::Cleaner() { Status CacheMergeOp::PrepareOperator() { // Run any common code from super class first before adding our own // specific logic - CHECK_FAIL_RETURN_UNEXPECTED(child_.size() == kNumChildren, "Incorrect number of children"); + CHECK_FAIL_RETURN_UNEXPECTED( + child_.size() == kNumChildren, + "Incorrect number of children of CacheMergeOp, required num is 2, but got:" + std::to_string(child_.size())); RETURN_IF_NOT_OK(DatasetOp::PrepareOperator()); // Get the computed check sum from all ops in the cache miss class uint32_t cache_crc = DatasetOp::GenerateCRC(child_[kCacheMissChildIdx]); @@ -209,11 +211,12 @@ Status CacheMergeOp::PrepareOperator() { // Run any common code from super clas } Status CacheMergeOp::ComputeColMap() { - CHECK_FAIL_RETURN_UNEXPECTED(child_[kCacheMissChildIdx] != nullptr, "Invalid data, cache miss stream empty."); + CHECK_FAIL_RETURN_UNEXPECTED(child_[kCacheMissChildIdx] != nullptr, "Invalid data, cache miss stream is empty."); if (column_name_id_map().empty()) { column_name_id_map_ = child_[kCacheMissChildIdx]->column_name_id_map(); } - CHECK_FAIL_RETURN_UNEXPECTED(!column_name_id_map().empty(), "Invalid data, column_name_id_map is empty."); + CHECK_FAIL_RETURN_UNEXPECTED(!column_name_id_map().empty(), + "Invalid data, column_name_id_map of CacheMergeOp is empty."); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc index 97220e8b84e..a6fb74646d0 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc @@ -123,7 +123,7 @@ Status CacheOp::CacheAllRows(int32_t worker_id) { // from again. RETURN_IF_NOT_OK(child_iterator->FetchNextTensorRow(&row)); if (!row.eof()) { - RETURN_STATUS_UNEXPECTED("Cache op expects to get an eof after eoe from child."); + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Cache op expects to get an eof after eoe from child."); } break; } @@ -173,7 +173,7 @@ Status CacheOp::WaitForCachingAllRows() { case CacheServiceState::kNone: case CacheServiceState::kError: default: - RETURN_STATUS_UNEXPECTED("Unexpected state: " + std::to_string(out)); + RETURN_STATUS_UNEXPECTED("Unexpected Cache server state: " + std::to_string(out)); } } while (!BuildPhaseDone); // Get statistics from the server, and if we are not the one to create the cache, diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc index 22b148a690a..4da9d776d31 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc @@ -57,12 +57,14 @@ DatasetOp::DatasetOp(int32_t op_connector_size, std::shared_ptr sampl // Adds a operator to become our child. Status DatasetOp::AddChild(std::shared_ptr child) { if (std::dynamic_pointer_cast(child) != nullptr) { - std::string err_msg("DeviceQueueOp cannot be added as a child, DeviceQueueOp must be a root node"); + std::string err_msg( + "DeviceQueueOp cannot be added as a child. DeviceQueueOp must be a root node, which means no operator should be " + "after device_queue operation."); RETURN_STATUS_UNEXPECTED(err_msg); } if (operator_id_ == kInvalidOperatorId) { std::string err_msg( - "Cannot add child node. Tree node connections can only" + "Cannot add child node. Tree node connections can only " "be made if the node belongs to a tree."); RETURN_STATUS_UNEXPECTED(err_msg); } @@ -70,7 +72,7 @@ Status DatasetOp::AddChild(std::shared_ptr child) { // disallow relationships with other trees if (tree_ != child->tree_) { std::string err_msg( - "Cannot add child node. Tree node connections can only be made if both nodes belong to the same tree."); + "Cannot add child node. Tree node connections can only be made if both nodes belong to the same tree."); RETURN_STATUS_UNEXPECTED(err_msg); } child_.push_back(child); @@ -81,7 +83,7 @@ Status DatasetOp::AddChild(std::shared_ptr child) { Status DatasetOp::RemoveChild(std::shared_ptr child) { if (operator_id_ == kInvalidOperatorId) { std::string err_msg( - "Cannot remove child node. Tree node connections can only" + "Cannot remove child node. Tree node connections can only " "be made if the node belongs to a tree."); RETURN_STATUS_UNEXPECTED(err_msg); } @@ -89,7 +91,7 @@ Status DatasetOp::RemoveChild(std::shared_ptr child) { // disallow relationships with other trees if (tree_ != child->tree_) { std::string err_msg( - "Cannot remove child node. Tree node connections can only be made if both nodes belong to the same tree."); + "Cannot remove child node. Tree node connections can only be made if both nodes belong to the same tree."); RETURN_STATUS_UNEXPECTED(err_msg); } @@ -130,11 +132,11 @@ void DatasetOp::RemoveParent(const DatasetOp *parent) { // Removes this node from the tree and connects it's parent/child together Status DatasetOp::Remove() { if (parent_.size() > 1) { - std::string err_msg("No support for op removal if the operator has more than one parent"); + std::string err_msg("No support for op removal if the operator has more than one parent."); RETURN_STATUS_UNEXPECTED(err_msg); } if (child_.size() > 1) { - std::string err_msg("No support for op removal if the operator has more than one child"); + std::string err_msg("No support for op removal if the operator has more than one child."); RETURN_STATUS_UNEXPECTED(err_msg); } @@ -290,7 +292,7 @@ Status DatasetOp::GetClassIndexing(std::vectorGetClassIndexing(output_class_indexing); } else { *output_class_indexing = {}; - RETURN_STATUS_UNEXPECTED("Trying to get class index from leaf node, missing override"); + RETURN_STATUS_UNEXPECTED("Trying to get class index from leaf node, missing override."); } } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc index bf6ef7e552b..3e95710ba87 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc @@ -130,7 +130,7 @@ Status DeviceQueueOp::operator()() { #ifdef ENABLE_DUMP_IR if (md_channel_info_ == nullptr) { - return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "RDR module init failed."); + return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "[Internal ERROR] RDR module init failed."); } #endif if (device_type_ == DeviceType::Ascend) { @@ -236,9 +236,9 @@ Status DeviceQueueOp::SendDataToAscend() { return Status::OK(); } return Status(StatusCode::kMDTDTPushFailure, - "TDT Push data into device Failed, please check the first error or TraceBack first, following are" + "TDT Push data into device Failed, check the first error or TraceBack first, following are" " several possible checking way: 1) if training is not ready, still in network graph compiling" - " stage, please check error raised by Network used operator or environment configuration. 2) if" + " stage, check error raised by Network used operator or environment configuration. 2) if" " interrupt in middle process of training, may check whether dataset sending num and network" " training num mismatch. 3) if this error raised in end of training, ignore this. 4) other cases," " try find ascend host log or checking info log etc."); @@ -289,12 +289,12 @@ Status DeviceQueueOp::SendRowToTdt(TensorRow currRow, bool isProfilingEnable, in return Status::OK(); } return Status(StatusCode::kMDTDTPushFailure, - "TDT Push data into device Failed, please check the first error or TraceBack first, following are" + "TDT Push data into device Failed, check the first error or TraceBack first, following are" " several possible checking way: 1) if training is not ready, still in network graph compiling" - " stage, please check error raised by Network used operator or environment configuration. 2) if" + " stage, check error raised by Network used operator or environment configuration. 2) if" " interrupt in middle process of training, may check whether dataset sending num and network" " training num mismatch. 3) if this error raised in end of training, ignore this. 4) other cases," - " try find ascend host log or checking info log ects."); + " try find ascend host log or checking info log ects or search this in mindspore's FAQ."); } if (create_data_info_queue_) { DATA_INFO data_info; @@ -405,7 +405,8 @@ Status DeviceQueueOp::PushDataToGPU() { } handle = GpuBufferMgr::GetInstance().Open(0, channel_name_, data_size, release_function); if (handle == INVALID_HANDLE) { - return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "Failed to open channel for sending data."); + return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, + "[Internal ERROR] Failed to open channel for sending data."); } is_open = true; } @@ -413,7 +414,8 @@ Status DeviceQueueOp::PushDataToGPU() { // Data prefetch only when PS mode enables cache. if (!ps::PsDataPrefetch::GetInstance().PrefetchData(channel_name_, items[0].data_ptr_, items[0].data_len_, items[0].data_type_)) { - return Status(StatusCode::kMDTimeOut, __LINE__, __FILE__, "Failed to prefetch data."); + return Status(StatusCode::kMDTimeOut, __LINE__, __FILE__, + "Failed to prefetch data in current PS mode(cache data when sending)."); } RETURN_IF_NOT_OK(RetryPushData(handle, items)); send_batch++; @@ -471,7 +473,8 @@ Status DeviceQueueOp::RetryPushData(unsigned int handle, const std::vector *items, if (memcpy_s(sub_item.data_ptr_, sub_item.data_len_, column_data, static_cast(curr_row[i++]->SizeInBytes())) != 0) { MS_LOG(ERROR) << "memcpy_s failed!"; - return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "memcpy_s failed."); + return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "memcpy failed when using memcpy_s do copy."); } } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc index 2e3c7ecac05..a044758fd7b 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc @@ -42,7 +42,7 @@ void EpochCtrlOp::Print(std::ostream &out, bool show_all) const { Status EpochCtrlOp::GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) { if (child_.empty()) { - RETURN_STATUS_UNEXPECTED("EpochCtrlOp can't be the leaf node."); + RETURN_STATUS_UNEXPECTED("EpochCtrlOp can't be the leaf node(first operator) of pipeline."); } // `retry_if_eoe` is false because EpochCtrlOp does not eat EOE. diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc index b1ec6ddbb62..83d764da17f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc @@ -37,7 +37,8 @@ FilterOp::FilterOp(const std::vector &in_col_names, int32_t num_wor Status FilterOp::LaunchThreadsAndInitOp() { // The operator class just starts off threads by calling the tree_ function. if (tree_ == nullptr) { - return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "Pipeline init failed, Execution tree not set."); + return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, + "[Internal ERROR] Pipeline init failed, Execution tree not set."); } filter_queues_.Init(num_workers_, oc_queue_size_); RETURN_IF_NOT_OK(filter_queues_.Register(tree_->AllTasks())); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.cc index 76aeda6c284..d637e4bb6ec 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.cc @@ -44,7 +44,7 @@ ParallelOp::ParallelOp(int32_t num_workers, int32_t op_connector_size, std::shar // Creates the internal worker connector for the parallel op if the derived class wants to use it Status ParallelOp::CreateWorkerConnector(int32_t worker_connector_size) { if (worker_connector_size == 0) { - RETURN_STATUS_UNEXPECTED("Worker connector size 0 is invalid."); + RETURN_STATUS_UNEXPECTED("Create Worker Connector failed, as given connector size 0 is invalid."); } num_producers_ = 1; worker_connector_size_ = worker_connector_size; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc index 9b5ca7ae84c..1a47056c058 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc @@ -113,7 +113,7 @@ Status ProjectOp::ComputeColMap() { for (size_t i = 0; i < columns_to_project_.size(); i++) { std::string ¤t_column = columns_to_project_[i]; if (child_column_name_mapping.find(current_column) == child_column_name_mapping.end()) { - std::string err_msg = "Invalid parameter, column name: " + current_column + " does not exist."; + std::string err_msg = "Invalid parameter, column name: " + current_column + " does not exist in dataset."; RETURN_STATUS_UNEXPECTED(err_msg); } // Setup the new column name mapping for ourself (base class field) diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc index a653e91f8f8..2d4643eb95e 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc @@ -96,7 +96,7 @@ Status ShuffleOp::AddRowToShuffleBuffer(TensorRow new_shuffle_row) { } else { if (!(*shuffle_buffer_)[shuffle_last_row_idx_].empty()) { return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, - "Last row of shuffle buffer should not be occupied!"); + "[Internal ERROR] Last row of shuffle buffer should not be occupied!"); } (*shuffle_buffer_)[shuffle_last_row_idx_] = std::move(new_shuffle_row); } @@ -206,7 +206,7 @@ Status ShuffleOp::InitShuffleBuffer() { // rows. if (shuffle_buffer_state_ != kShuffleStateInit) { return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, - "Invalid shuffle buffer state (SHUFFLE_STATE_INIT expected)"); + "Invalid shuffle buffer state, shuffle buffer should be init first or reset after each epoch."); } // Before we drop into the fetching loop, call the fetch once for the first time @@ -221,7 +221,7 @@ Status ShuffleOp::InitShuffleBuffer() { } if (new_row.empty()) { - RETURN_STATUS_UNEXPECTED("Unable to fetch a single row for shuffle buffer."); + RETURN_STATUS_UNEXPECTED("Invalid data, unable to fetch a single row for shuffle buffer."); } // Now fill the rest of the shuffle buffer until we are unable to get the next row or we reached diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc index ad441b4dbdb..4eaeb936f86 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc @@ -66,7 +66,7 @@ Status AlbumOp::PrescanEntry() { dirname_offset_ = folder_path_.length(); std::shared_ptr dirItr = Path::DirIterator::OpenDirectory(&folder); if (!folder.Exists() || dirItr == nullptr) { - RETURN_STATUS_UNEXPECTED("Invalid file, failed to open folder: " + folder_path_); + RETURN_STATUS_UNEXPECTED("Invalid file, failed to open folder: " + folder_path_ + "."); } MS_LOG(INFO) << "Album folder Path found: " << folder_path_ << "."; @@ -84,7 +84,8 @@ Status AlbumOp::PrescanEntry() { num_rows_ = image_rows_.size(); if (num_rows_ == 0) { RETURN_STATUS_UNEXPECTED( - "Invalid data, no valid data matching the dataset API AlbumDataset. Please check file path or dataset API."); + "Invalid data, data file may not be suitable to read with AlbumDataset API. Check file path:" + folder_path_ + + "."); } return Status::OK(); } @@ -210,7 +211,7 @@ Status AlbumOp::LoadIntArrayTensor(const nlohmann::json &json_obj, uint32_t col_ RETURN_IF_NOT_OK(Tensor::CreateFromVector(data, &label)); } else { - RETURN_STATUS_UNEXPECTED("Invalid data, column type is neither int32 nor int64, it is " + + RETURN_STATUS_UNEXPECTED("Invalid data, column type in data_schema is neither int32 nor int64, it is " + data_schema_->column(col_num).type().ToString()); } row->push_back(std::move(label)); @@ -239,7 +240,7 @@ Status AlbumOp::LoadFloatArrayTensor(const nlohmann::json &json_obj, uint32_t co RETURN_IF_NOT_OK(Tensor::CreateFromVector(data, &float_array)); } else { - RETURN_STATUS_UNEXPECTED("Invalid data, column type is neither float32 nor float64, it is " + + RETURN_STATUS_UNEXPECTED("Invalid data, column type in data_schema is neither float32 nor float64, it is " + data_schema_->column(col_num).type().ToString()); } row->push_back(std::move(float_array)); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc index 18b0a7d44eb..16cbddb4cda 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc @@ -94,10 +94,11 @@ Status CelebAOp::ParseAttrFile() { num_rows_in_attr_file_ = static_cast(std::stoul(rows_num)); // First line is rows number in attr file } catch (std::invalid_argument &e) { RETURN_STATUS_UNEXPECTED( - "Invalid data, failed to convert rows_num from attr_file to unsigned long, invalid argument: " + rows_num); + "Invalid data, failed to convert rows_num from attr_file to unsigned long, invalid value: " + rows_num + "."); } catch (std::out_of_range &e) { RETURN_STATUS_UNEXPECTED( - "Invalid data, failed to convert rows_num from attr_file to unsigned long, out of range: " + rows_num); + "Invalid data, failed to convert rows_num from attr_file to unsigned long, value out of range: " + rows_num + + "."); } (void)getline(attr_file, attr_name); // Second line is attribute name,ignore it @@ -142,10 +143,10 @@ bool CelebAOp::CheckDatasetTypeValid() { try { type = std::stoi(vec[1]); } catch (std::invalid_argument &e) { - MS_LOG(WARNING) << "Invalid data, failed to convert to unsigned long, invalid argument: " << vec[1] << "."; + MS_LOG(WARNING) << "Invalid data, failed to convert to int, invalid value: " << vec[1] << "."; return false; } catch (std::out_of_range &e) { - MS_LOG(WARNING) << "Invalid data, failed to convert to unsigned long, out of range: " << vec[1] << "."; + MS_LOG(WARNING) << "Invalid data, failed to convert to int, value out of range: " << vec[1] << "."; return false; } // train:0, valid=1, test=2 @@ -187,9 +188,12 @@ Status CelebAOp::ParseImageAttrInfo() { try { value = std::stoi(split[label_index]); } catch (std::invalid_argument &e) { - RETURN_STATUS_UNEXPECTED("Invalid data, failed to convert to ulong, invalid argument: " + split[label_index]); + RETURN_STATUS_UNEXPECTED("Invalid data, failed to convert item from attr_file to int, corresponding value: " + + split[label_index] + "."); } catch (std::out_of_range &e) { - RETURN_STATUS_UNEXPECTED("Conversion to int failed, out of range: " + split[label_index]); + RETURN_STATUS_UNEXPECTED( + "Invalid data, failed to convert item from attr_file to int as out of range, corresponding value: " + + split[label_index] + "."); } image_labels.second.push_back(value); } @@ -203,7 +207,8 @@ Status CelebAOp::ParseImageAttrInfo() { num_rows_ = image_labels_vec_.size(); if (num_rows_ == 0) { RETURN_STATUS_UNEXPECTED( - "Invalid data, no valid data matching the dataset API CelebADataset. Please check file path or dataset API."); + "Invalid data, data file may not be suitable to read with CelebADataset API. Check attr_file in directory:" + + folder_path_ + "."); } MS_LOG(DEBUG) << "Celeba dataset rows number is " << num_rows_ << "."; return Status::OK(); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc index 560fbd9cf4f..1b7559d84e3 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc @@ -138,11 +138,13 @@ Status CifarOp::ReadCifar10BlockData() { } std::ifstream in(file, std::ios::binary); - CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar10 file: " + file); + CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar10 file: " + file + + ", make sure file not damaged or permission denied."); for (uint32_t index = 0; index < num_cifar10_records / kCifarBlockImageNum; ++index) { (void)in.read(reinterpret_cast(&(image_data[0])), block_size * sizeof(unsigned char)); - CHECK_FAIL_RETURN_UNEXPECTED(!in.fail(), "Invalid data, failed to read data from cifar10 file: " + file); + CHECK_FAIL_RETURN_UNEXPECTED(!in.fail(), "Invalid data, failed to read data from cifar10 file: " + file + + ", re-download dataset(make sure it is CIFAR-10 binary version)."); (void)cifar_raw_data_block_->EmplaceBack(image_data); // Add file path info path_record_.push_back(file); @@ -182,11 +184,13 @@ Status CifarOp::ReadCifar100BlockData() { } std::ifstream in(file, std::ios::binary); - CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar100 file: " + file); + CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar100 file: " + file + + ", make sure file not damaged or permission denied."); for (uint32_t index = 0; index < num_cifar100_records / kCifarBlockImageNum; index++) { (void)in.read(reinterpret_cast(&(image_data[0])), block_size * sizeof(unsigned char)); - CHECK_FAIL_RETURN_UNEXPECTED(!in.fail(), "Invalid data, failed to read data from cifar100 file: " + file); + CHECK_FAIL_RETURN_UNEXPECTED(!in.fail(), "Invalid data, failed to read data from cifar100 file: " + file + + ", re-download dataset(make sure it is CIFAR-100 binary version)."); (void)cifar_raw_data_block_->EmplaceBack(image_data); // Add file path info path_record_.push_back(file); @@ -209,7 +213,8 @@ Status CifarOp::GetCifarFiles() { } } } else { - RETURN_STATUS_UNEXPECTED("Invalid file, failed to open directory: " + dir_path.toString()); + RETURN_STATUS_UNEXPECTED("Invalid file, failed to open directory: " + dir_path.toString() + + ", make sure file not damaged or permission denied."); } CHECK_FAIL_RETURN_UNEXPECTED(!cifar_files_.empty(), "Invalid file, no .bin files found under " + folder_path_); std::sort(cifar_files_.begin(), cifar_files_.end()); @@ -251,8 +256,8 @@ Status CifarOp::ParseCifarData() { num_rows_ = cifar_image_label_pairs_.size(); if (num_rows_ == 0) { std::string api = cifar_type_ == kCifar10 ? "Cifar10Dataset" : "Cifar100Dataset"; - RETURN_STATUS_UNEXPECTED("Invalid data, no valid data matching the dataset API " + api + - ". Please check file path or dataset API."); + RETURN_STATUS_UNEXPECTED("Invalid data, data file may not suitable to read with " + api + + " API. Check file in directory:" + folder_path_); } cifar_raw_data_block_->Reset(); return Status::OK(); @@ -262,7 +267,7 @@ Status CifarOp::ParseCifarData() { Status CifarOp::GetClassIds(std::map> *cls_ids) const { if (cls_ids == nullptr || !cls_ids->empty()) { RETURN_STATUS_UNEXPECTED( - "Map for storaging image-index pair is nullptr or has been set in other place," + "[Internal ERROR] Map for containing image-index pair is nullptr or has been set in other place," "it must be empty before using GetClassIds."); } @@ -312,8 +317,9 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage, constexpr int64_t num_cifar10_records = 10000; for (auto &file : op->cifar_files_) { Path file_path(file); - CHECK_FAIL_RETURN_UNEXPECTED(file_path.Exists() && !file_path.IsDirectory(), - "Invalid file, failed to open cifar10 file: " + file); + CHECK_FAIL_RETURN_UNEXPECTED( + file_path.Exists() && !file_path.IsDirectory(), + "Invalid file, failed to open cifar10 file: " + file + ", make sure file not damaged or permission denied."); std::string file_name = file_path.Basename(); if (op->usage_ == "train") { @@ -326,7 +332,8 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage, std::ifstream in(file, std::ios::binary); - CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar10 file: " + file); + CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar10 file: " + file + + ", make sure file not damaged or permission denied."); *count = *count + num_cifar10_records; } return Status::OK(); @@ -336,8 +343,9 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage, Path file_path(file); std::string file_name = file_path.Basename(); - CHECK_FAIL_RETURN_UNEXPECTED(file_path.Exists() && !file_path.IsDirectory(), - "Invalid file, failed to find cifar100 file: " + file); + CHECK_FAIL_RETURN_UNEXPECTED( + file_path.Exists() && !file_path.IsDirectory(), + "Invalid file, failed to find cifar100 file: " + file + ", make sure file not damaged or permission denied."); if (op->usage_ == "train" && file_path.Basename().find("train") == std::string::npos) continue; if (op->usage_ == "test" && file_path.Basename().find("test") == std::string::npos) continue; @@ -348,7 +356,8 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage, num_cifar100_records += 50000; } std::ifstream in(file, std::ios::binary); - CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar100 file: " + file); + CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar100 file: " + file + + ", make sure file not damaged or permission denied."); } *count = num_cifar100_records; return Status::OK(); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc index b897a4243dd..77188d75c83 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc @@ -56,7 +56,7 @@ Status ClueOp::GetValue(const nlohmann::json &js, std::vector key_c if (cursor.find(key_chain[i]) != cursor.end()) { cursor = cursor[key_chain[i]]; } else { - RETURN_STATUS_UNEXPECTED("Invalid data, failed to find key: " + key_chain[i]); + RETURN_STATUS_UNEXPECTED("Invalid data, in given JSON file, failed to find key: " + key_chain[i]); } } std::string final_str = key_chain.back(); @@ -110,7 +110,7 @@ Status ClueOp::LoadFile(const std::string &file, int64_t start_offset, int64_t e js = nlohmann::json::parse(line); } catch (const std::exception &err) { // Catch any exception and convert to Status return code - RETURN_STATUS_UNEXPECTED("Invalid file, failed to parse json file: " + file); + RETURN_STATUS_UNEXPECTED("Invalid file, failed to parse JSON file: " + file); } int cols_count = cols_to_keyword_.size(); TensorRow tRow(cols_count, nullptr); @@ -208,8 +208,13 @@ Status ClueOp::CalculateNumRowsPerShard() { num_rows_ += count; } if (num_rows_ == 0) { + std::stringstream ss; + for (int i = 0; i < clue_files_list_.size(); ++i) { + ss << " " << clue_files_list_[i]; + } + std::string file_list = ss.str(); RETURN_STATUS_UNEXPECTED( - "Invalid data, no valid data matching the dataset API CLUEDataset. Please check file path or dataset API."); + "Invalid data, data file may not be suitable to read with CLUEDataset API. Check file path:" + file_list); } num_rows_per_shard_ = static_cast(std::ceil(num_rows_ * 1.0 / num_devices_)); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc index aa3dc4c50ec..fc2d8cd708f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc @@ -112,7 +112,7 @@ Status CocoOp::Builder::SanityCheck() { ? "Invalid parameter, Coco image folder path is invalid or not set, path: " + builder_dir_ + ".\n" : ""; err_msg += file.Exists() == false - ? "Invalid parameter, Coco annotation json path is invalid or not set, path: " + builder_dir_ + ".\n" + ? "Invalid parameter, Coco annotation JSON path is invalid or not set, path: " + builder_dir_ + ".\n" : ""; err_msg += builder_num_workers_ <= 0 ? "Invalid parameter, num_parallel_workers must be greater than 0, but got " + std::to_string(builder_num_workers_) + ".\n" @@ -154,7 +154,7 @@ Status CocoOp::LoadTensorRow(row_id_type row_id, TensorRow *trow) { auto itr = coordinate_map_.find(image_id); if (itr == coordinate_map_.end()) { RETURN_STATUS_UNEXPECTED("Invalid data, image_id: " + image_id + - " in annotation node is not found in image node in json file."); + " in annotation node is not found in image node in JSON file."); } std::string kImageFile = image_folder_path_ + std::string("/") + image_id; @@ -204,7 +204,7 @@ Status CocoOp::LoadDetectionTensorRow(row_id_type row_id, const std::string &ima auto itr_item = simple_item_map_.find(image_id); if (itr_item == simple_item_map_.end()) { RETURN_STATUS_UNEXPECTED("Invalid data, image_id: " + image_id + - " in annotation node is not found in image node in json file."); + " in annotation node is not found in image node in JSON file."); } std::vector annotation = itr_item->second; @@ -247,7 +247,7 @@ Status CocoOp::LoadSimpleTensorRow(row_id_type row_id, const std::string &image_ auto itr_item = simple_item_map_.find(image_id); if (itr_item == simple_item_map_.end()) { RETURN_STATUS_UNEXPECTED("Invalid data, image_id: " + image_id + - " in annotation node is not found in image node in json file."); + " in annotation node is not found in image node in JSON file."); } item_queue = itr_item->second; @@ -282,7 +282,7 @@ Status CocoOp::LoadMixTensorRow(row_id_type row_id, const std::string &image_id, auto itr_item = simple_item_map_.find(image_id); if (itr_item == simple_item_map_.end()) { RETURN_STATUS_UNEXPECTED("Invalid data, image_id: " + image_id + - " in annotation node is not found in image node in json file."); + " in annotation node is not found in image node in JSON file."); } std::vector annotation = itr_item->second; @@ -327,7 +327,7 @@ Status CocoOp::LoadMixTensorRow(row_id_type row_id, const std::string &image_id, template Status CocoOp::SearchNodeInJson(const nlohmann::json &input_tree, std::string node_name, T *output_node) { auto node = input_tree.find(node_name); - CHECK_FAIL_RETURN_UNEXPECTED(node != input_tree.end(), "Invalid data, invalid node found in json: " + node_name); + CHECK_FAIL_RETURN_UNEXPECTED(node != input_tree.end(), "Invalid data, required node not found in JSON: " + node_name); (*output_node) = *node; return Status::OK(); } @@ -338,7 +338,7 @@ Status CocoOp::ParseAnnotationIds() { std::ifstream in(annotation_path_); in >> js; } catch (const std::exception &err) { - RETURN_STATUS_UNEXPECTED("Invalid file, failed to open json file: " + annotation_path_); + RETURN_STATUS_UNEXPECTED("Invalid file, failed to open JSON file: " + annotation_path_ + "."); } std::vector image_que; @@ -359,7 +359,7 @@ Status CocoOp::ParseAnnotationIds() { auto itr_file = image_index_.find(image_id); if (itr_file == image_index_.end()) { RETURN_STATUS_UNEXPECTED("Invalid data, image_id: " + std::to_string(image_id) + - " in annotation node is not found in image node in json file."); + " in annotation node is not found in image node in JSON file."); } file_name = itr_file->second; switch (task_type_) { @@ -388,14 +388,15 @@ Status CocoOp::ParseAnnotationIds() { num_rows_ = image_ids_.size(); if (num_rows_ == 0) { RETURN_STATUS_UNEXPECTED( - "Invalid data, no valid data matching the dataset API CocoDataset. Please check file path or dataset API."); + "Invalid data, data file may not suitable to read with CocoDataset API. Check file in directory: " + + image_folder_path_ + "."); } return Status::OK(); } Status CocoOp::ImageColumnLoad(const nlohmann::json &image_tree, std::vector *image_vec) { if (image_tree.size() == 0) { - RETURN_STATUS_UNEXPECTED("Invalid data, no \"image\" node found in json file: " + annotation_path_); + RETURN_STATUS_UNEXPECTED("Invalid data, no \"image\" node found in JSON file: " + annotation_path_ + "."); } for (auto img : image_tree) { std::string file_name; @@ -418,8 +419,8 @@ Status CocoOp::DetectionColumnLoad(const nlohmann::json &annotation_tree, const RETURN_IF_NOT_OK(SearchNodeInJson(annotation_tree, std::string(kJsonAnnoCategoryId), &category_id)); auto search_category = category_set_.find(category_id); if (search_category == category_set_.end()) - RETURN_STATUS_UNEXPECTED("Invalid data, category_id can't find in categories where category_id: " + - std::to_string(category_id)); + RETURN_STATUS_UNEXPECTED( + "Invalid data, category_id can't find in categories where category_id: " + std::to_string(category_id) + "."); auto node_iscrowd = annotation_tree.find(kJsonAnnoIscrowd); if (node_iscrowd != annotation_tree.end()) iscrowd = *node_iscrowd; bbox.insert(bbox.end(), node_bbox.begin(), node_bbox.end()); @@ -456,12 +457,13 @@ Status CocoOp::KeypointColumnLoad(const nlohmann::json &annotation_tree, const s const int32_t &unique_id) { auto itr_num_keypoint = annotation_tree.find(kJsonAnnoNumKeypoints); if (itr_num_keypoint == annotation_tree.end()) - RETURN_STATUS_UNEXPECTED("Invalid data, no num_keypoint found in annotations where id: " + - std::to_string(unique_id)); + RETURN_STATUS_UNEXPECTED( + "Invalid data, no num_keypoint found in annotation file where image_id: " + std::to_string(unique_id) + "."); simple_item_map_[image_file].push_back(*itr_num_keypoint); auto itr_keypoint = annotation_tree.find(kJsonAnnoKeypoints); if (itr_keypoint == annotation_tree.end()) - RETURN_STATUS_UNEXPECTED("Invalid data, no keypoint found in annotations where id: " + std::to_string(unique_id)); + RETURN_STATUS_UNEXPECTED( + "Invalid data, no keypoint found in annotation file where image_id: " + std::to_string(unique_id) + "."); coordinate_map_[image_file].push_back(*itr_keypoint); return Status::OK(); } @@ -470,31 +472,31 @@ Status CocoOp::PanopticColumnLoad(const nlohmann::json &annotation_tree, const s const int32_t &image_id) { auto itr_segments = annotation_tree.find(kJsonAnnoSegmentsInfo); if (itr_segments == annotation_tree.end()) - RETURN_STATUS_UNEXPECTED("Invalid data, no segments_info found in annotations where image_id: " + - std::to_string(image_id)); + RETURN_STATUS_UNEXPECTED( + "Invalid data, no segments_info found in annotation file where image_id: " + std::to_string(image_id) + "."); for (auto info : *itr_segments) { std::vector bbox; uint32_t category_id = 0; auto itr_bbox = info.find(kJsonAnnoBbox); if (itr_bbox == info.end()) - RETURN_STATUS_UNEXPECTED("Invalid data, no bbox found in segments_info where image_id: " + - std::to_string(image_id)); + RETURN_STATUS_UNEXPECTED("Invalid data, no bbox found in segments_info(in annotation file) where image_id: " + + std::to_string(image_id) + "."); bbox.insert(bbox.end(), itr_bbox->begin(), itr_bbox->end()); coordinate_map_[image_file].push_back(bbox); RETURN_IF_NOT_OK(SearchNodeInJson(info, std::string(kJsonAnnoCategoryId), &category_id)); auto search_category = category_set_.find(category_id); if (search_category == category_set_.end()) - RETURN_STATUS_UNEXPECTED("Invalid data, category_id can't find in categories where category_id: " + - std::to_string(category_id)); + RETURN_STATUS_UNEXPECTED( + "Invalid data, category_id can't find in categories where category_id: " + std::to_string(category_id) + "."); auto itr_iscrowd = info.find(kJsonAnnoIscrowd); if (itr_iscrowd == info.end()) - RETURN_STATUS_UNEXPECTED("Invalid data, no iscrowd found in segments_info where image_id: " + - std::to_string(image_id)); + RETURN_STATUS_UNEXPECTED( + "Invalid data, no iscrowd found in segments_info where image_id: " + std::to_string(image_id) + "."); auto itr_area = info.find(kJsonAnnoArea); if (itr_area == info.end()) - RETURN_STATUS_UNEXPECTED("Invalid data, no area found in segments_info where image_id: " + - std::to_string(image_id)); + RETURN_STATUS_UNEXPECTED( + "Invalid data, no area found in segments_info where image_id: " + std::to_string(image_id) + "."); simple_item_map_[image_file].push_back(category_id); simple_item_map_[image_file].push_back(*itr_iscrowd); simple_item_map_[image_file].push_back(*itr_area); @@ -512,7 +514,7 @@ Status CocoOp::CategoriesColumnLoad(const nlohmann::json &categories_tree) { std::vector label_info; auto itr_id = category.find(kJsonId); if (itr_id == category.end()) { - RETURN_STATUS_UNEXPECTED("Invalid data, no json id found in categories of " + annotation_path_); + RETURN_STATUS_UNEXPECTED("Invalid data, no JSON id found in categories of " + annotation_path_); } id = *itr_id; label_info.push_back(id); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc index 0f629e11d3d..7a657873100 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc @@ -490,7 +490,7 @@ Status CsvOp::LoadFile(const std::string &file, int64_t start_offset, int64_t en std::ifstream ifs; ifs.open(file, std::ifstream::in); if (!ifs.is_open()) { - RETURN_STATUS_UNEXPECTED("Error opening file: " + file); + RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + file); } if (column_name_list_.empty()) { std::string tmp; @@ -506,17 +506,17 @@ Status CsvOp::LoadFile(const std::string &file, int64_t start_offset, int64_t en int err = csv_parser.ProcessMessage(chr); if (err != 0) { if (err == -2) return Status(kMDInterrupted); - RETURN_STATUS_UNEXPECTED("Invalid file, failed to parse file: " + file + ":" + + RETURN_STATUS_UNEXPECTED("Invalid file, failed to parse file: " + file + ": line " + std::to_string(csv_parser.GetTotalRows() + 1) + ". Error message: " + csv_parser.GetErrorMessage()); } } } catch (std::invalid_argument &ia) { std::string err_row = std::to_string(csv_parser.GetTotalRows() + 1); - RETURN_STATUS_UNEXPECTED("Invalid data, " + file + ":" + err_row + ", type does not match."); + RETURN_STATUS_UNEXPECTED("Invalid data, " + file + ": line " + err_row + ", type does not match."); } catch (std::out_of_range &oor) { std::string err_row = std::to_string(csv_parser.GetTotalRows() + 1); - RETURN_STATUS_UNEXPECTED("Invalid data, " + file + ":" + err_row + ", out of range."); + RETURN_STATUS_UNEXPECTED("Invalid data, " + file + ": line " + err_row + ", value out of range."); } return Status::OK(); } @@ -597,8 +597,13 @@ Status CsvOp::CalculateNumRowsPerShard() { num_rows_ += count; } if (num_rows_ == 0) { + std::stringstream ss; + for (int i = 0; i < csv_files_list_.size(); ++i) { + ss << " " << csv_files_list_[i]; + } + std::string file_list = ss.str(); RETURN_STATUS_UNEXPECTED( - "Invalid data, no valid data matching the dataset API CsvDataset. Please check file path or CSV format."); + "Invalid data, data file may not suitable to read with CSVDataset API. Check file path: " + file_list + "."); } num_rows_per_shard_ = static_cast(std::ceil(num_rows_ * 1.0 / num_devices_)); @@ -662,7 +667,7 @@ Status CsvOp::ComputeColMap() { // Set the column name mapping (base class field) if (column_name_id_map_.empty()) { if (!ColumnNameValidate()) { - RETURN_STATUS_UNEXPECTED("Fail to validate column name for input CSV file list"); + RETURN_STATUS_UNEXPECTED("Invalid file, failed to obtain column name from input CSV file list."); } for (auto &csv_file : csv_files_list_) { @@ -686,9 +691,9 @@ Status CsvOp::ComputeColMap() { if (column_default_list_.size() != column_name_id_map_.size()) { RETURN_STATUS_UNEXPECTED( - "Invalid parameter, the number of column names does not match the column defaults, column_default_list: " + + "Invalid parameter, the number of column names does not match the default column, size of default column_list: " + std::to_string(column_default_list_.size()) + - ", column_name_id_map: " + std::to_string(column_name_id_map_.size())); + ", size of column_name: " + std::to_string(column_name_id_map_.size())); } return Status::OK(); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc index d660cbf7728..be568a298a4 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc @@ -62,7 +62,7 @@ Status GeneratorOp::CreateGeneratorObject() { // Acquire Python GIL py::gil_scoped_acquire gil_acquire; if (Py_IsInitialized() == 0) { - return Status(StatusCode::kMDPythonInterpreterFailure, "Python Interpreter is finalized"); + return Status(StatusCode::kMDPythonInterpreterFailure, "Python Interpreter is finalized."); } try { py::array sample_ids; @@ -91,28 +91,32 @@ Status GeneratorOp::Init() { Status GeneratorOp::PyRowToTensorRow(py::object py_data, TensorRow *tensor_row) { if (!py::isinstance(py_data)) { return Status(StatusCode::kMDPyFuncException, __LINE__, __FILE__, - "Invalid parameter, Generator should return a tuple of numpy arrays."); + "Invalid data, Generator should return a tuple of NumPy arrays, currently returned is not a tuple."); } py::tuple py_row = py_data.cast(); // Check if returned number of columns matches with column names if (py_row.size() != column_names_.size()) { return Status( StatusCode::kMDPyFuncException, __LINE__, __FILE__, - "Invalid parameter, Generator should return same number of numpy arrays as specified in column names."); + "Invalid data, Generator should return same number of NumPy arrays as specified in column_names, the size of" + " column_names is:" + + std::to_string(column_names_.size()) + + "and number of returned NumPy array is:" + std::to_string(py_row.size())); } // Iterate over two containers simultaneously for memory copy for (int i = 0; i < py_row.size(); ++i) { py::object ret_py_ele = py_row[i]; if (!py::isinstance(ret_py_ele)) { return Status(StatusCode::kMDPyFuncException, __LINE__, __FILE__, - "Invalid parameter, Generator should return a tuple of numpy arrays."); + "Invalid data, Generator should return a tuple of NumPy arrays. Ensure each item in tuple that " + "returned by source function of GeneratorDataset be NumPy array."); } std::shared_ptr tensor; RETURN_IF_NOT_OK(Tensor::CreateFromNpArray(ret_py_ele.cast(), &tensor)); if ((!column_types_.empty()) && (column_types_[i] != DataType::DE_UNKNOWN) && (column_types_[i] != tensor->type())) { return Status(StatusCode::kMDPyFuncException, __LINE__, __FILE__, - "Invalid parameter, input column type is not same with output tensor type."); + "Invalid data, type of returned data in GeneratorDataset is not same with specified column_types."); } tensor_row->push_back(tensor); } @@ -181,6 +185,12 @@ Status GeneratorOp::operator()() { return Status(StatusCode::kMDPyFuncException, __LINE__, __FILE__, e.what()); } if (num_rows_sampled != -1 && num_rows_sampled != generator_counter_) { + if (generator_counter_ == 0) { + std::string msg = + "Unable to fetch data from GeneratorDataset, try iterate the source function of GeneratorDataset or check" + " value of num_epochs when create iterator."; + return Status(StatusCode::kMDPyFuncException, __LINE__, __FILE__, msg); + } std::stringstream ss; ss << "The actual amount of data read from generator " << generator_counter_ << " is different from generator.len " << num_rows_sampled diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc index 9444b650b5e..b835ff7ae1c 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc @@ -73,8 +73,8 @@ Status ImageFolderOp::PrescanMasterEntry(const std::string &filedir) { num_rows_ = image_label_pairs_.size(); if (num_rows_ == 0) { RETURN_STATUS_UNEXPECTED( - "Invalid data, no valid data matching the dataset API ImageFolderDataset. " - "Please check file path or dataset API."); + "Invalid data, data file may not be suitable to read with ImageFolderDataset API. Check file path: " + + folder_path_); } // free memory of two queues used for pre-scan folder_name_queue_->Reset(); @@ -120,7 +120,7 @@ void ImageFolderOp::Print(std::ostream &out, bool show_all) const { Status ImageFolderOp::GetClassIds(std::map> *cls_ids) const { if (cls_ids == nullptr || !cls_ids->empty() || image_label_pairs_.empty()) { if (image_label_pairs_.empty()) { - RETURN_STATUS_UNEXPECTED("No images found in dataset, please check if Op read images successfully or not."); + RETURN_STATUS_UNEXPECTED("No images found in dataset, try iterate dataset to check if read images success."); } else { RETURN_STATUS_UNEXPECTED( "Map containing image-index pair is nullptr or has been set in other place," diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/io_block.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/io_block.cc index 2b2542430bd..fc5fac10524 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/io_block.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/io_block.cc @@ -36,7 +36,7 @@ IOBlock::IOBlock(IOBlockFlags io_block_flags) : io_block_flags_(io_block_flags) // Fetches the first key from this block Status IOBlock::GetKey(int64_t *out_key) const { if (out_key == nullptr || index_keys_.empty()) { - RETURN_STATUS_UNEXPECTED("Failed to get the key from IOBlock"); + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to get the key from IOBlock."); } *out_key = index_keys_[0]; return Status::OK(); @@ -45,7 +45,7 @@ Status IOBlock::GetKey(int64_t *out_key) const { // Fetches the list of keys from this block. Status IOBlock::GetKeys(std::vector *out_keys) const { if (out_keys == nullptr) { - RETURN_STATUS_UNEXPECTED("Output arg for GetKeys is null"); + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Output arg for GetKeys is null."); } *out_keys = index_keys_; // vector copy assign return Status::OK(); @@ -64,7 +64,7 @@ FilenameBlock::FilenameBlock(IOBlockFlags io_block_flags) // Gets the filename from the block using the provided index container Status FilenameBlock::GetFilename(std::string *out_filename, const AutoIndexObj &index) const { if (out_filename == nullptr) { - RETURN_STATUS_UNEXPECTED("Failed to get filename from FilenameBlock"); + RETURN_STATUS_UNEXPECTED("Failed to get filename from FilenameBlock."); } // a FilenameBlock only has one key. Call base class method to fetch that key @@ -77,7 +77,7 @@ Status FilenameBlock::GetFilename(std::string *out_filename, const AutoIndexObj< auto &it = r.first; *out_filename = it.value(); } else { - RETURN_STATUS_UNEXPECTED("Could not find filename from index"); + RETURN_STATUS_UNEXPECTED("Could not find filename from index."); } return Status::OK(); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc index 050cee12b63..a38f816ba42 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc @@ -141,10 +141,10 @@ void ManifestOp::Print(std::ostream &out, bool show_all) const { Status ManifestOp::GetClassIds(std::map> *cls_ids) const { if (cls_ids == nullptr || !cls_ids->empty() || image_labelname_.empty()) { if (image_labelname_.empty()) { - RETURN_STATUS_UNEXPECTED("No image found in dataset, please check if Op read images successfully or not."); + RETURN_STATUS_UNEXPECTED("No image found in dataset. Try iterate dataset to check if read images success."); } else { RETURN_STATUS_UNEXPECTED( - "Map for storaging image-index pair is nullptr or has been set in other place," + "[Internal ERROR] Map for containing image-index pair is nullptr or has been set in other place," "it must be empty before using GetClassIds."); } } @@ -181,7 +181,7 @@ Status ManifestOp::ParseManifestFile() { std::string image_file_path = js.value("source", ""); if (image_file_path == "") { file_handle.close(); - RETURN_STATUS_UNEXPECTED("Invalid data, source is not found in Manifest file: " + file_ + " at line " + + RETURN_STATUS_UNEXPECTED("Invalid data, 'source' is not found in Manifest file: " + file_ + " at line " + std::to_string(line_count)); } // If image is not JPEG/PNG/GIF/BMP, drop it @@ -193,7 +193,7 @@ Status ManifestOp::ParseManifestFile() { std::string usage = js.value("usage", ""); if (usage == "") { file_handle.close(); - RETURN_STATUS_UNEXPECTED("Invalid data, usage is not found in Manifest file: " + file_ + " at line " + + RETURN_STATUS_UNEXPECTED("Invalid data, 'usage' is not found in Manifest file: " + file_ + " at line " + std::to_string(line_count)); } (void)std::transform(usage.begin(), usage.end(), usage.begin(), ::tolower); @@ -208,8 +208,8 @@ Status ManifestOp::ParseManifestFile() { classes.insert(label_name); if (label_name == "") { file_handle.close(); - RETURN_STATUS_UNEXPECTED("Invalid data, label name is not found in Manifest file: " + file_ + " at line " + - std::to_string(line_count)); + RETURN_STATUS_UNEXPECTED("Invalid data, 'name' of label is not found in Manifest file: " + file_ + + " at line " + std::to_string(line_count)); } if (class_index_.empty() || class_index_.find(label_name) != class_index_.end()) { if (label_index_.find(label_name) == label_index_.end()) { @@ -278,7 +278,7 @@ Status ManifestOp::CountDatasetInfo() { num_rows_ = static_cast(image_labelname_.size()); if (num_rows_ == 0) { RETURN_STATUS_UNEXPECTED( - "Invalid data, no valid data matching the dataset API ManifestDataset. Please check file path: " + file_); + "Invalid data, data file may not be suitable to read with ManifestDataset API. Check file path: " + file_); } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mappable_leaf_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mappable_leaf_op.cc index 0eb89a07953..a0d8a04fa18 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mappable_leaf_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mappable_leaf_op.cc @@ -115,7 +115,7 @@ Status MappableLeafOp::WorkerEntry(int32_t worker_id) { } RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block)); } - RETURN_STATUS_UNEXPECTED("Unexpected nullptr received in worker"); + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Unexpected nullptr received in worker."); } } // namespace dataset diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc index dac4a11d101..ac4a5320cb5 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc @@ -112,8 +112,9 @@ Status MindRecordOp::Init() { if (!load_all_cols) { std::unique_ptr tmp_schema = std::make_unique(); for (std::string colname : columns_to_load_) { - CHECK_FAIL_RETURN_UNEXPECTED(colname_to_ind.find(colname) != colname_to_ind.end(), - "Invalid parameter, column name: " + colname + " does not exist."); + CHECK_FAIL_RETURN_UNEXPECTED( + colname_to_ind.find(colname) != colname_to_ind.end(), + "Invalid data, specified loading column name: " + colname + " does not exist in data file."); RETURN_IF_NOT_OK(tmp_schema->AddColumn(data_schema_->column(colname_to_ind[colname]))); } data_schema_ = std::move(tmp_schema); @@ -276,7 +277,7 @@ Status MindRecordOp::LoadTensorRow(TensorRow *tensor_row, const std::vector dataset_path, std::unique_ptr shard_reader = std::make_unique(); MSRStatus rc = shard_reader->CountTotalRows(dataset_path, load_dataset, op, count, num_padded); if (rc == MSRStatus::FAILED) { - RETURN_STATUS_UNEXPECTED("Invalid data, MindRecordOp failed to count total rows."); + RETURN_STATUS_UNEXPECTED( + "Invalid data, MindRecordOp failed to count total rows. Check whether there are corresponding .db files " + "and the value of dataset_file parameter is given correctly."); } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc index 0ba04252bbd..44d5e0c8d62 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc @@ -114,10 +114,10 @@ void MnistOp::Print(std::ostream &out, bool show_all) const { Status MnistOp::GetClassIds(std::map> *cls_ids) const { if (cls_ids == nullptr || !cls_ids->empty() || image_label_pairs_.empty()) { if (image_label_pairs_.empty()) { - RETURN_STATUS_UNEXPECTED("No image found in dataset, please check if Op read images successfully or not."); + RETURN_STATUS_UNEXPECTED("No image found in dataset, try iterate dataset to check if read images success."); } else { RETURN_STATUS_UNEXPECTED( - "Map for storaging image-index pair is nullptr or has been set in other place," + "[Internal ERROR] Map for containing image-index pair is nullptr or has been set in other place," "it must be empty before using GetClassIds."); } } @@ -192,25 +192,26 @@ Status MnistOp::ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *la uint32_t num_images, num_labels; RETURN_IF_NOT_OK(CheckImage(image_names_[index], image_reader, &num_images)); RETURN_IF_NOT_OK(CheckLabel(label_names_[index], label_reader, &num_labels)); - CHECK_FAIL_RETURN_UNEXPECTED((num_images == num_labels), "Invalid data, num_images is not equal to num_labels."); + CHECK_FAIL_RETURN_UNEXPECTED((num_images == num_labels), + "Invalid data, num_images is not equal to num_labels. Ensure data file is not damaged."); // The image size of the Mnist dataset is fixed at [28,28] int64_t size = kMnistImageRows * kMnistImageCols; auto images_buf = std::make_unique(size * num_images); auto labels_buf = std::make_unique(num_images); if (images_buf == nullptr || labels_buf == nullptr) { - std::string err_msg = "Failed to allocate memory for MNIST buffer."; + std::string err_msg = "[Internal ERROR] Failed to allocate memory for MNIST buffer."; MS_LOG(ERROR) << err_msg.c_str(); RETURN_STATUS_UNEXPECTED(err_msg); } (void)image_reader->read(images_buf.get(), size * num_images); if (image_reader->fail()) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to read image: " + image_names_[index] + - ", size:" + std::to_string(size * num_images)); + ", size:" + std::to_string(size * num_images) + ". Ensure data file is not damaged."); } (void)label_reader->read(labels_buf.get(), num_images); if (label_reader->fail()) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to read label:" + label_names_[index] + - ", size: " + std::to_string(num_images)); + ", size: " + std::to_string(num_images) + ". Ensure data file is not damaged."); } TensorShape img_tensor_shape = TensorShape({kMnistImageRows, kMnistImageCols, 1}); for (int64_t j = 0; j != num_images; ++j) { @@ -247,7 +248,8 @@ Status MnistOp::ParseMnistData() { num_rows_ = image_label_pairs_.size(); if (num_rows_ == 0) { RETURN_STATUS_UNEXPECTED( - "Invalid data, no valid data matching the dataset API MnistDataset. Please check file path or dataset API."); + "Invalid data, data file may not be suitable to read with MnistDataset API. Check file in directory: " + + folder_path_); } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc index c4c2a85b5a2..286f452e717 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc @@ -112,7 +112,7 @@ void RandomDataOp::GenerateSchema() { // provide the master loop that drives the logic for performing the work. Status RandomDataOp::operator()() { CHECK_FAIL_RETURN_UNEXPECTED(total_rows_ >= num_workers_, - "RandomDataOp expects total_rows < num_workers. total_row=" + + "RandomDataOp expects total_rows < num_workers. Try adjust num_workers, total_row=" + std::to_string(total_rows_) + ", num_workers=" + std::to_string(num_workers_) + " ."); // If the amount of workers we have exceeds the number of rows to produce, then we'll have @@ -265,7 +265,7 @@ Status RandomDataOp::WorkerEntry(int32_t worker_id) { // A helper function to create random data for the row Status RandomDataOp::CreateRandomRow(int32_t worker_id, TensorRow *new_row) { if (new_row == nullptr) { - return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "Missing tensor row output"); + return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "[Internal ERROR] Missing tensor row output."); } // Create a tensor for each column, then add the tensor to the row diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.cc index 03113726e8f..8ef5d2820a4 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.cc @@ -53,7 +53,7 @@ Status DistributedSamplerRT::InitSampler() { CHECK_FAIL_RETURN_UNEXPECTED(num_samples_ > 0, "Invalid parameter, num_samples must be greater than 0, but got " + std::to_string(num_samples_) + ".\n"); CHECK_FAIL_RETURN_UNEXPECTED( - num_rows_ > 0, "Invalid parameter, num_rows must be greater than 0" + std::to_string(num_rows_) + ".\n"); + num_rows_ > 0, "Invalid parameter, num_rows must be greater than 0, but got " + std::to_string(num_rows_) + ".\n"); CHECK_FAIL_RETURN_UNEXPECTED( device_id_ < num_devices_ && device_id_ >= 0 && num_rows_ > 0 && num_samples_ > 0, "Invalid parameter, num_shard must be greater than shard_id and greater than 0, got num_shard: " + @@ -87,9 +87,8 @@ Status DistributedSamplerRT::InitSampler() { Status DistributedSamplerRT::GetNextSample(TensorRow *out) { if (cnt_ > samples_per_tensor_) { RETURN_STATUS_UNEXPECTED( - "Number of samples(cnt) that have already been filled in to Tensor should be less than or " - "equal to samples_per_tensor, but got cnt: " + - std::to_string(cnt_) + ", samples_per_tensor: " + std::to_string(samples_per_tensor_)); + "Sampler index must be less than or equal to num_samples(total rows in dataset), but got:" + + std::to_string(cnt_) + ", samples_per_tensor(num_samples): " + std::to_string(samples_per_tensor_)); } else if (cnt_ == samples_per_tensor_ && (non_empty_ || !even_dist_)) { (*out) = TensorRow(TensorRow::kFlagEOE); if (!samples_per_tensor_) { @@ -150,7 +149,7 @@ Status DistributedSamplerRT::GetNextSample(TensorRow *out) { } Status DistributedSamplerRT::ResetSampler() { - CHECK_FAIL_RETURN_UNEXPECTED(cnt_ == samples_per_tensor_, "ERROR Reset() called early/late"); + CHECK_FAIL_RETURN_UNEXPECTED(cnt_ == samples_per_tensor_, "[Internal ERROR] Reset() Sampler called early or late."); cnt_ = 0; if (shuffle_ == true) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc index fa3d65c119d..e28ea44ef35 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc @@ -28,7 +28,9 @@ MindRecordSamplerRT::MindRecordSamplerRT(mindrecord::ShardReader *shard_reader, Status MindRecordSamplerRT::GetNextSample(TensorRow *out) { if (next_id_ > num_samples_) { - RETURN_STATUS_UNEXPECTED("MindRecordSampler Internal Error"); + RETURN_STATUS_UNEXPECTED( + "Sampler index must be less than or equal to num_samples(total rows in dataset), but got: " + + std::to_string(next_id_) + ", num_samples_: " + std::to_string(num_samples_)); } else if (next_id_ == num_samples_) { (*out) = TensorRow(TensorRow::kFlagEOE); } else { @@ -51,7 +53,9 @@ Status MindRecordSamplerRT::InitSampler() { if (!sample_ids_) { // Note, sample_ids_.empty() is okay and will just give no sample ids. - RETURN_STATUS_UNEXPECTED("ShardReader did not provide a valid sample ids vector via MindRecordSamplerRT"); + RETURN_STATUS_UNEXPECTED( + "Init Sampler failed as sample_ids is empty, here ShardReader did not provide a valid sample ids vector via" + " MindRecordSamplerRT"); } // Usually, the num samples is given from the user interface. In our case, that data is in mindrecord. diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/pk_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/pk_sampler.cc index 2c5b3d3856e..27e4696a79d 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/pk_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/pk_sampler.cc @@ -59,7 +59,7 @@ Status PKSamplerRT::InitSampler() { std::sort(labels_.begin(), labels_.end()); } CHECK_FAIL_RETURN_UNEXPECTED( - num_samples_ > 0, "Invalid parameter, num_class or K (num samples per class) must be greater than 0, but got " + + num_samples_ > 0, "Invalid parameter, num_class or num samples per class must be greater than 0, but got " + std::to_string(num_samples_)); is_initialized = true; return Status::OK(); @@ -67,7 +67,9 @@ Status PKSamplerRT::InitSampler() { Status PKSamplerRT::GetNextSample(TensorRow *out) { if (next_id_ > num_samples_ || num_samples_ == 0) { - RETURN_STATUS_UNEXPECTED("Index must be less than or equal to num_samples, but got: " + std::to_string(next_id_)); + RETURN_STATUS_UNEXPECTED( + "Sampler index must be less than or equal to num_samples(total rows in dataset), but got: " + + std::to_string(next_id_) + ", num_samplers:" + std::to_string(num_samples_)); } else if (next_id_ == num_samples_) { (*out) = TensorRow(TensorRow::kFlagEOE); } else { @@ -79,7 +81,7 @@ Status PKSamplerRT::GetNextSample(TensorRow *out) { int64_t last_id = (samples_per_tensor_ + next_id_ > num_samples_) ? num_samples_ : samples_per_tensor_ + next_id_; RETURN_IF_NOT_OK(CreateSamplerTensor(&sample_ids, last_id - next_id_)); auto id_ptr = sample_ids->begin(); - CHECK_FAIL_RETURN_UNEXPECTED(samples_per_class_ != 0, "samples cannot be zero."); + CHECK_FAIL_RETURN_UNEXPECTED(samples_per_class_ != 0, "Invalid Parameter, num samples per class can't be zero."); while (next_id_ < last_id && id_ptr != sample_ids->end()) { int64_t cls_id = next_id_++ / samples_per_class_; const std::vector &samples = label_to_ids_[labels_[cls_id]]; @@ -100,7 +102,7 @@ Status PKSamplerRT::GetNextSample(TensorRow *out) { } Status PKSamplerRT::ResetSampler() { - CHECK_FAIL_RETURN_UNEXPECTED(next_id_ == num_samples_, "ERROR Reset() called early/late"); + CHECK_FAIL_RETURN_UNEXPECTED(next_id_ == num_samples_, "[Internal ERROR] Reset() Sampler called early or late."); next_id_ = 0; rnd_.seed(seed_++); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/python_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/python_sampler.cc index ff3eecf64d3..2f72d8663c1 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/python_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/python_sampler.cc @@ -90,7 +90,7 @@ Status PythonSamplerRT::InitSampler() { } Status PythonSamplerRT::ResetSampler() { - CHECK_FAIL_RETURN_UNEXPECTED(need_to_reset_, "ERROR Reset() called not at end of an epoch"); + CHECK_FAIL_RETURN_UNEXPECTED(need_to_reset_, "[Internal ERROR] Reset() Sampler called early or late."); need_to_reset_ = false; py::gil_scoped_acquire gil_acquire; if (Py_IsInitialized() == 0) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/random_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/random_sampler.cc index c2a5265ac2f..4505927b3c1 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/random_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/random_sampler.cc @@ -33,7 +33,8 @@ RandomSamplerRT::RandomSamplerRT(bool replacement, int64_t num_samples, bool res Status RandomSamplerRT::GetNextSample(TensorRow *out) { if (next_id_ > num_samples_) { - RETURN_STATUS_UNEXPECTED("RandomSampler Internal Error"); + RETURN_STATUS_UNEXPECTED("Sampler index must be less than or equal to num_samples(total rows in dataset), but got" + + std::to_string(next_id_) + ", num_samplers:" + std::to_string(num_samples_)); } else if (next_id_ == num_samples_) { (*out) = TensorRow(TensorRow::kFlagEOE); } else { @@ -77,7 +78,7 @@ Status RandomSamplerRT::InitSampler() { } CHECK_FAIL_RETURN_UNEXPECTED( num_samples_ > 0 && num_rows_ > 0, - "Invalid parameter, num_samples & num_rows must be greater than 0, but got num_samples: " + + "Invalid parameter, num_samples and num_rows must be greater than 0, but got num_samples: " + std::to_string(num_samples_) + ", num_rows: " + std::to_string(num_rows_)); samples_per_tensor_ = samples_per_tensor_ > num_samples_ ? num_samples_ : samples_per_tensor_; rnd_.seed(seed_); @@ -97,7 +98,7 @@ Status RandomSamplerRT::InitSampler() { } Status RandomSamplerRT::ResetSampler() { - CHECK_FAIL_RETURN_UNEXPECTED(next_id_ == num_samples_, "ERROR Reset() called early/late"); + CHECK_FAIL_RETURN_UNEXPECTED(next_id_ == num_samples_, "[Internal ERROR] Reset() Sampler called early or late."); next_id_ = 0; if (reshuffle_each_epoch_) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sampler.cc index 54ab63f1553..715bf993ab9 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sampler.cc @@ -27,7 +27,7 @@ Status RandomAccessOp::GetNumRowsInDataset(int64_t *num) const { // Here, it is just a getter method to return the value. However, it is invalid if there is // not a value set for this count, so generate a failure if that is the case. if (num == nullptr || num_rows_ == -1) { - RETURN_STATUS_UNEXPECTED("RandomAccessOp has not computed its num rows yet."); + RETURN_STATUS_UNEXPECTED("Get num rows in Dataset failed, num_rows has not been set yet."); } (*num) = num_rows_; return Status::OK(); @@ -45,7 +45,7 @@ Status SamplerRT::HandshakeRandomAccessOp(const RandomAccessOp *op) { if (HasChildSampler()) { child_sampler = std::dynamic_pointer_cast(child_[0]); if (!child_sampler) { - std::string err_msg("Cannot handshake, child is not a sampler object."); + std::string err_msg("[Internal ERROR] Cannot handshake, child is not a sampler object."); RETURN_STATUS_UNEXPECTED(err_msg); } @@ -53,7 +53,7 @@ Status SamplerRT::HandshakeRandomAccessOp(const RandomAccessOp *op) { RETURN_IF_NOT_OK(child_sampler->HandshakeRandomAccessOp(op)); } - CHECK_FAIL_RETURN_UNEXPECTED(op != nullptr, "RandomAccessOp is nullptr\n"); + CHECK_FAIL_RETURN_UNEXPECTED(op != nullptr, "RandomAccessOp init failed, as it is nullptr."); // If there's a child sampler, set the row count to be it's sample count if (HasChildSampler()) { @@ -99,11 +99,11 @@ Status SamplerRT::GetAllIdsThenReset(py::array *data) { sample_ids = sample_row[0]; // check this tensorRow is not a ctrl tensorRow - CHECK_FAIL_RETURN_UNEXPECTED(sample_row.Flags() == TensorRow::kFlagNone, "ERROR ctrl row received"); + CHECK_FAIL_RETURN_UNEXPECTED(sample_row.Flags() == TensorRow::kFlagNone, "[Internal ERROR] ctrl row received."); // perform error checking! Next TensorRow supposed to be EOE since last one already contains all ids for current epoch RETURN_IF_NOT_OK(GetNextSample(&sample_row)); - CHECK_FAIL_RETURN_UNEXPECTED(sample_row.eoe(), "ERROR Non EOE received"); + CHECK_FAIL_RETURN_UNEXPECTED(sample_row.eoe(), "[Internal ERROR] Non EOE received in the end of epoch."); // Reset Sampler since this is the end of the epoch RETURN_IF_NOT_OK(ResetSampler()); @@ -176,7 +176,7 @@ bool SamplerRT::HasChildSampler() const { return !child_.empty(); } Status SamplerRT::GetAssociatedChildId(int64_t *out_associated_id, int64_t id) { if (child_ids_.empty()) { - RETURN_STATUS_UNEXPECTED("Trying to get associated child id, but there are no child ids!"); + RETURN_STATUS_UNEXPECTED("[Internal ERROR] Trying to get associated child id, but there are no child ids!"); } std::shared_ptr sample_ids = child_ids_[0]; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.cc index eb717cbe987..fa3045518a6 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.cc @@ -26,7 +26,9 @@ SequentialSamplerRT::SequentialSamplerRT(int64_t start_index, int64_t num_sample Status SequentialSamplerRT::GetNextSample(TensorRow *out) { if (id_count_ > num_samples_) { - RETURN_STATUS_UNEXPECTED("SequentialSampler Internal Error"); + RETURN_STATUS_UNEXPECTED( + "Sampler index must be less than or equal to num_samples(total rows in dataset), but got:" + + std::to_string(id_count_) + ", num_samples_: " + std::to_string(num_samples_)); } else if (id_count_ == num_samples_) { (*out) = TensorRow(TensorRow::kFlagEOE); } else { @@ -80,9 +82,9 @@ Status SequentialSamplerRT::InitSampler() { if (num_samples_ == 0 || num_samples_ > available_row_count) { num_samples_ = available_row_count; } - CHECK_FAIL_RETURN_UNEXPECTED( - (num_samples_ > 0 && samples_per_tensor_ > 0) || num_samples_ == 0, - "Invalid parameter, samples_per_tensor must be greater than 0, but got " + std::to_string(samples_per_tensor_)); + CHECK_FAIL_RETURN_UNEXPECTED((num_samples_ > 0 && samples_per_tensor_ > 0) || num_samples_ == 0, + "Invalid parameter, samples_per_tensor(num_samplers) must be greater than 0, but got " + + std::to_string(samples_per_tensor_)); samples_per_tensor_ = samples_per_tensor_ > num_samples_ ? num_samples_ : samples_per_tensor_; is_initialized = true; @@ -90,7 +92,7 @@ Status SequentialSamplerRT::InitSampler() { } Status SequentialSamplerRT::ResetSampler() { - CHECK_FAIL_RETURN_UNEXPECTED(id_count_ == num_samples_, "ERROR Reset() called early/late"); + CHECK_FAIL_RETURN_UNEXPECTED(id_count_ == num_samples_, "[Internal ERROR] Reset() Sampler called early or late."); current_id_ = start_index_; id_count_ = 0; diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc index 14bc3047f56..a6ae8df8308 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc @@ -46,7 +46,7 @@ Status WeightedRandomSamplerRT::InitSampler() { "Invalid parameter, num_samples and num_rows must be greater than 0, but got num_rows: " + std::to_string(num_rows_) + ", num_samples: " + std::to_string(num_samples_)); CHECK_FAIL_RETURN_UNEXPECTED(samples_per_tensor_ > 0, - "Invalid parameter, samples_per_tensor must be greater than 0, but got " + + "Invalid parameter, samples_per_tensor(num_samples) must be greater than 0, but got " + std::to_string(samples_per_tensor_) + ".\n"); if (weights_.size() > static_cast(num_rows_)) { diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc index b0427f8d28c..c206f925c55 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc @@ -186,8 +186,13 @@ Status TextFileOp::CalculateNumRowsPerShard() { num_rows_ += count; } if (num_rows_ == 0) { + std::stringstream ss; + for (int i = 0; i < text_files_list_.size(); ++i) { + ss << " " << text_files_list_[i]; + } + std::string file_list = ss.str(); RETURN_STATUS_UNEXPECTED( - "Invalid data, no valid data matching the dataset API TextFileDataset. Please check file path or dataset API."); + "Invalid data, data file may not be suitable to read with TextFileDataset API. Check file: " + file_list); } num_rows_per_shard_ = static_cast(std::ceil(num_rows_ * 1.0 / num_devices_)); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc index d7c6cf58966..708ea548847 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc @@ -119,7 +119,9 @@ Status TFReaderOp::Init() { total_rows_ = data_schema_->num_rows(); } if (total_rows_ < 0) { - RETURN_STATUS_UNEXPECTED("Invalid parameter, num_sample or num_row for TFRecordDataset must be greater than 0."); + RETURN_STATUS_UNEXPECTED( + "Invalid parameter, num_samples or num_rows for TFRecordDataset must be greater than 0, but got: " + + std::to_string(total_rows_)); } // Build the index with our files such that each file corresponds to a key id. @@ -152,8 +154,13 @@ Status TFReaderOp::CalculateNumRowsPerShard() { } num_rows_per_shard_ = static_cast(std::ceil(num_rows_ * 1.0 / num_devices_)); if (num_rows_per_shard_ == 0) { + std::stringstream ss; + for (int i = 0; i < dataset_files_list_.size(); ++i) { + ss << " " << dataset_files_list_[i]; + } + std::string file_list = ss.str(); RETURN_STATUS_UNEXPECTED( - "Invalid data, no valid data matching the dataset API TFRecordDataset. Please check file path or dataset API."); + "Invalid data, data file may not be suitable to read with TFRecordDataset API. Check file path." + file_list); } return Status::OK(); } @@ -367,11 +374,11 @@ Status TFReaderOp::LoadFeature(TensorRow *tensor_row, const dataengine::Feature break; } case dataengine::Feature::KindCase::KIND_NOT_SET: { - std::string err_msg = "Invalid data, tf_file column type must be uint8, int64 or float32."; + std::string err_msg = "Invalid data, column type in tf record file must be uint8, int64 or float32."; RETURN_STATUS_UNEXPECTED(err_msg); } default: { - std::string err_msg = "Invalid data, tf_file column type must be uint8, int64 or float32."; + std::string err_msg = "Invalid data, column type in tf record file must be uint8, int64 or float32."; RETURN_STATUS_UNEXPECTED(err_msg); } } @@ -587,10 +594,10 @@ Status TFReaderOp::CreateSchema(const std::string tf_file, std::vector]" in test_config(["list"]) - assert "no valid data matching the dataset API Cifar10Dataset" in test_config("test") + assert "data file may not suitable to read with Cifar10Dataset API" in test_config("test") # test the usage of CIFAR10 assert test_config("test", False) == 10000 assert test_config("all", False) == 10000 - assert "no valid data matching the dataset API Cifar100Dataset" in test_config("train", False) + assert "data file may not suitable to read with Cifar100Dataset API" in test_config("train", False) assert "usage is not within the valid set of ['train', 'test', 'all']" in test_config("invalid", False) # change this directory to the folder that contains all cifar10 files diff --git a/tests/ut/python/dataset/test_datasets_coco.py b/tests/ut/python/dataset/test_datasets_coco.py index a368a78fc27..9fa4caf6a3f 100644 --- a/tests/ut/python/dataset/test_datasets_coco.py +++ b/tests/ut/python/dataset/test_datasets_coco.py @@ -300,7 +300,7 @@ def test_coco_case_exception(): pass assert False except RuntimeError as e: - assert "invalid node found in json" in str(e) + assert "required node not found in JSON" in str(e) try: data1 = ds.CocoDataset(DATA_DIR, annotation_file=INVALID_CATEGORY_ID_FILE, task="Detection") @@ -316,7 +316,7 @@ def test_coco_case_exception(): pass assert False except RuntimeError as e: - assert "failed to open json file" in str(e) + assert "failed to open JSON file" in str(e) try: sampler = ds.PKSampler(3) diff --git a/tests/ut/python/dataset/test_datasets_generator.py b/tests/ut/python/dataset/test_datasets_generator.py index 9dcf3c8c422..4e5cb48ddd7 100644 --- a/tests/ut/python/dataset/test_datasets_generator.py +++ b/tests/ut/python/dataset/test_datasets_generator.py @@ -554,7 +554,7 @@ def test_generator_error_2(): for _ in data1: pass print("========", str(info.value)) - assert "Generator should return a tuple of numpy arrays" in str(info.value) + assert "Generator should return a tuple of NumPy arrays" in str(info.value) def test_generator_error_3(): diff --git a/tests/ut/python/dataset/test_datasets_manifestop.py b/tests/ut/python/dataset/test_datasets_manifestop.py index 644c2a00ee3..2d3667b41ee 100644 --- a/tests/ut/python/dataset/test_datasets_manifestop.py +++ b/tests/ut/python/dataset/test_datasets_manifestop.py @@ -167,7 +167,7 @@ def test_manifest_dataset_exception(): pass assert False except RuntimeError as e: - assert "Invalid data, source is not found in Manifest file" in str(e) + assert "Invalid data, 'source' is not found in Manifest file" in str(e) NO_USAGE_DATA_FILE = "../data/dataset/testManifestData/invalidNoUsage.manifest" try: @@ -176,7 +176,7 @@ def test_manifest_dataset_exception(): pass assert False except RuntimeError as e: - assert "Invalid data, usage is not found in Manifest file" in str(e) + assert "Invalid data, 'usage' is not found in Manifest file" in str(e) if __name__ == '__main__': diff --git a/tests/ut/python/dataset/test_datasets_mnist.py b/tests/ut/python/dataset/test_datasets_mnist.py index 53d84338d84..40ad3cc8cf5 100644 --- a/tests/ut/python/dataset/test_datasets_mnist.py +++ b/tests/ut/python/dataset/test_datasets_mnist.py @@ -270,7 +270,7 @@ def test_mnist_usage(): assert test_config("test") == 10000 assert test_config("all") == 10000 - assert " no valid data matching the dataset API MnistDataset" in test_config("train") + assert "data file may not be suitable to read with MnistDataset API" in test_config("train") assert "usage is not within the valid set of ['train', 'test', 'all']" in test_config("invalid") assert "Argument usage with value ['list'] is not of type []" in test_config(["list"]) diff --git a/tests/ut/python/dataset/test_var_batch_map.py b/tests/ut/python/dataset/test_var_batch_map.py index 0a26b0621a7..3f6efa0d8dd 100644 --- a/tests/ut/python/dataset/test_var_batch_map.py +++ b/tests/ut/python/dataset/test_var_batch_map.py @@ -369,7 +369,7 @@ def test_multi_col_map(): # test exceptions assert "output_columns with value 233 is not of type" in batch_map_config(2, 2, split_col, ["col2"], 233) assert "column_order with value 233 is not of type" in batch_map_config(2, 2, split_col, ["col2"], ["col1"], 233) - assert "output_columns is NOT set correctly" in batch_map_config(2, 2, split_col, ["col2"], ["col1"]) + assert "output_columns in batch is not set correctly" in batch_map_config(2, 2, split_col, ["col2"], ["col1"]) assert "Incorrect number of columns" in batch_map_config(2, 2, split_col, ["col2"], ["col3", "col4", "col5"]) assert "col-1 doesn't exist" in batch_map_config(2, 2, split_col, ["col-1"], ["col_x", "col_y"])