SuLe
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

complete dataset log

This commit is contained in:
ms_yan 2021-04-19 21:37:07 +08:00
parent 850e3d4b2a
commit be65ebcb28
48 changed files with 290 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/minddata/dataset/engine/data_schema.cc
View File

@ -358,7 +358,7 @@ Status DataSchema::LoadSchemaFile(const std::string &schema_file_path,
}
} catch (const std::exception &err) {
// Catch any exception and convert to Status return code
RETURN_STATUS_UNEXPECTED("Schema file failed to load");
RETURN_STATUS_UNEXPECTED("Schema file failed to load with JSON tools. File is: " + schema_file_path);
}
return Status::OK();
}

47
mindspore/ccsrc/minddata/dataset/engine/datasetops/batch_op.cc
View File

@ -169,7 +169,7 @@ void BatchOp::Print(std::ostream &out, bool show_all) const {
Status BatchOp::BatchRows(const std::unique_ptr<TensorQTable> *src, TensorRow *dest, dsize_t batch_size) {
if ((*src)->size() != batch_size) {
RETURN_STATUS_UNEXPECTED("[Internal Batch ERROR] Source table size does not match the batch_size");
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Source table size does not match the batch_size.");
}
if (batch_size == 1) {
@ -201,9 +201,12 @@ Status BatchOp::BatchRows(const std::unique_ptr<TensorQTable> *src, TensorRow *d
}
// Don't do anything if the tensor has no data
} else {
std::stringstream shape1, shape2;
first_shape.Print(shape1);
old_tensor->shape().Print(shape2);
RETURN_STATUS_UNEXPECTED(
"Invalid data, expect same shape for each data row, but got inconsistent data shapes in column " +
std::to_string(i));
std::to_string(i) + " expected shape for this column is:" + shape1.str() + ", got shape:" + shape2.str());
}
}
} else { // handle string column differently
@ -253,7 +256,8 @@ Status BatchOp::MakeBatchedRow(std::pair<std::unique_ptr<TensorQTable>, CBatchIn
Status BatchOp::LaunchThreadsAndInitOp() {
if (tree_ == nullptr) {
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "Pipeline init failed, Execution tree not set.");
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__,
"[Internal ERROR] Pipeline init failed, Execution tree not set.");
}
RETURN_IF_NOT_OK(worker_queues_.Register(tree_->AllTasks()));
RETURN_IF_NOT_OK(
@ -330,23 +334,24 @@ Status BatchOp::InvokeBatchSizeFunc(int32_t *batch_size, CBatchInfo info) {
// Acquire Python GIL
py::gil_scoped_acquire gil_acquire;
if (Py_IsInitialized() == 0) {
return Status(StatusCode::kMDPythonInterpreterFailure, "Python Interpreter is finalized.");
return Status(StatusCode::kMDPythonInterpreterFailure, "[Internal ERROR] Python Interpreter is finalized.");
}
try {
py::object size = batch_size_func_(info);
*batch_size = size.cast<int32_t>();
if (*batch_size <= 0) {
return Status(StatusCode::kMDPyFuncException,
"Invalid parameter, batch size function should return an integer greater than 0.");
"Invalid parameter, batch_size function should return an integer greater than 0, but got: " +
std::to_string(*batch_size));
}
} catch (const py::error_already_set &e) {
return Status(StatusCode::kMDPyFuncException, e.what());
} catch (const py::cast_error &e) {
return Status(StatusCode::kMDPyFuncException,
"Invalid parameter, batch size function should return an integer greater than 0.");
"Invalid parameter, batch_size function should return an integer greater than 0.");
}
}
return Status(StatusCode::kSuccess, "Batch size func call succeed.");
return Status(StatusCode::kSuccess, "batch_size function call succeedeed.");
}
Status BatchOp::InvokeBatchMapFunc(TensorTable *input, TensorTable *output, CBatchInfo info) {
@ -354,7 +359,7 @@ Status BatchOp::InvokeBatchMapFunc(TensorTable *input, TensorTable *output, CBat
// Acquire Python GIL
py::gil_scoped_acquire gil_acquire;
if (Py_IsInitialized() == 0) {
return Status(StatusCode::kMDPythonInterpreterFailure, "Python Interpreter is finalized.");
return Status(StatusCode::kMDPythonInterpreterFailure, "[Internal ERROR] Python Interpreter is finalized.");
}
try {
// Prepare batch map call back parameters
@ -373,11 +378,12 @@ Status BatchOp::InvokeBatchMapFunc(TensorTable *input, TensorTable *output, CBat
py::object ret_py_obj = batch_map_func_(*input_args);
// Parse batch map return value
py::tuple ret_tuple = py::cast<py::tuple>(ret_py_obj);
CHECK_FAIL_RETURN_UNEXPECTED(py::isinstance<py::tuple>(ret_tuple), "Batch map function should return a tuple.");
CHECK_FAIL_RETURN_UNEXPECTED(
ret_tuple.size() == out_col_names_.size(),
"Incorrect number of columns returned. expects: " + std::to_string(out_col_names_.size()) +
" gets: " + std::to_string(ret_tuple.size()));
CHECK_FAIL_RETURN_UNEXPECTED(py::isinstance<py::tuple>(ret_tuple),
"per_batch_map function should return a tuple.");
CHECK_FAIL_RETURN_UNEXPECTED(ret_tuple.size() == out_col_names_.size(),
"Incorrect number of columns returned in per_batch_map function. Expects: " +
std::to_string(out_col_names_.size()) +
" got: " + std::to_string(ret_tuple.size()));
for (size_t i = 0; i < ret_tuple.size(); i++) {
TensorRow output_batch;
// If user returns a type that is neither a list nor an array, issue a error msg.
@ -399,7 +405,7 @@ Status BatchOp::InvokeBatchMapFunc(TensorTable *input, TensorTable *output, CBat
return Status(StatusCode::kMDPyFuncException, e.what());
} catch (const py::cast_error &e) {
return Status(StatusCode::kMDPyFuncException,
"Invalid parameter, batch map function should return a tuple of list of numpy array.");
"Invalid parameter, per_batch_map function of batch should return a tuple of list of numpy array.");
}
}
return Status::OK();
@ -488,9 +494,12 @@ Status BatchOp::UnpackPadInfo(const PadInfo &pad_info,
}
Status BatchOp::ComputeColMap() {
CHECK_FAIL_RETURN_UNEXPECTED(child_.size() == 1,
"Batch has " + std::to_string(child_.size()) + " child/children, expects only 1 child.");
CHECK_FAIL_RETURN_UNEXPECTED(!(child_[0]->column_name_id_map().empty()), "BatchOp child map is empty.");
CHECK_FAIL_RETURN_UNEXPECTED(
child_.size() == 1,
"Batch operator expects only 1 child node, but number of children nodes is: " + std::to_string(child_.size()) +
". Check your script how many node composed into Batch node.");
CHECK_FAIL_RETURN_UNEXPECTED(!(child_[0]->column_name_id_map().empty()),
"Column of Batch operator's child is empty.");
if (in_col_names_.empty()) { // if per_batch_map is not set, do not need to deal with out_col_names
column_name_id_map_ = child_[0]->column_name_id_map();
@ -502,7 +511,7 @@ Status BatchOp::ComputeColMap() {
// check all input columns exist
for (const auto &col : in_col_names_) {
CHECK_FAIL_RETURN_UNEXPECTED(child_map_.find(col) != child_map_.end(), "col:" + col + " doesn't exist.");
CHECK_FAIL_RETURN_UNEXPECTED(child_map_.find(col) != child_map_.end(), "col:" + col + " doesn't exist in dataset.");
}
// following logic deals with per_batch_map
@ -538,7 +547,7 @@ Status BatchOp::ComputeColMap() {
}
CHECK_FAIL_RETURN_UNEXPECTED(column_name_id_map_.size() == (child_map_no_in_col.size() + out_col_names_.size()),
"Key error in column_name_id_map_. output_columns is NOT set correctly!");
"Key error in column_name_id_map_. output_columns in batch is not set correctly!");
return Status::OK();
}

6
mindspore/ccsrc/minddata/dataset/engine/datasetops/bucket_batch_by_length_op.cc
View File

@ -107,7 +107,8 @@ Status BucketBatchByLengthOp::ObtainElementLength(int32_t *out_element_length, T
for (size_t i = 0; i < number_of_arguments; i++) {
auto map_item = column_name_id_map_.find(length_dependent_columns_[i]);
if (map_item == column_name_id_map_.end()) {
RETURN_STATUS_UNEXPECTED("BucketBatchByLength: Couldn't find the specified column in the dataset");
RETURN_STATUS_UNEXPECTED("BucketBatchByLength: Couldn't find the specified column(" +
length_dependent_columns_[i] + ") in the dataset.");
}
int32_t column_index = map_item->second;
input.push_back(element[column_index]);
@ -116,7 +117,8 @@ Status BucketBatchByLengthOp::ObtainElementLength(int32_t *out_element_length, T
RETURN_IF_NOT_OK(output.at(0)->GetItemAt(out_element_length, {0}));
if (*out_element_length < 0) {
RETURN_STATUS_UNEXPECTED(
"Invalid parameter, element_length_function must return an integer greater than or equal to 0.");
"Invalid parameter, element_length_function must return an integer greater than or equal to 0, but got" +
std::to_string(*out_element_length));
}
} else {
*out_element_length = element[0]->shape()[0];

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/build_sentence_piece_vocab_op.cc
View File

@ -58,7 +58,7 @@ Status BuildSentencePieceVocabOp::operator()() {
RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row));
}
RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row));
CHECK_FAIL_RETURN_UNEXPECTED(!eoe_warning, "no op should be after from_dataset (repeat detected)");
CHECK_FAIL_RETURN_UNEXPECTED(!eoe_warning, "no operator should be after from_dataset (repeat detected)");
eoe_warning = true;
}
// add empty tensorRow for quit
@ -71,12 +71,13 @@ Status BuildSentencePieceVocabOp::SentenceThread() {
TaskManager::FindMe()->Post();
if (col_names_.empty() == true) {
auto itr = column_name_id_map_.find("text");
CHECK_FAIL_RETURN_UNEXPECTED(itr != column_name_id_map_.end(), "Invalid data, 'text' column does not exist.");
CHECK_FAIL_RETURN_UNEXPECTED(itr != column_name_id_map_.end(),
"Invalid data, 'text' column does not exist in dataset.");
col_id_ = itr->second;
} else {
auto itr = column_name_id_map_.find(col_names_[0]);
CHECK_FAIL_RETURN_UNEXPECTED(itr != column_name_id_map_.end(),
"Invalid parameter, column name: " + col_names_[0] + " does not exist.");
"Invalid parameter, column name: " + col_names_[0] + " does not exist in dataset.");
col_id_ = itr->second;
}
std::unique_ptr<DatasetSentenceIterator> sentence_iter = std::make_unique<DatasetSentenceIterator>(this);
@ -88,7 +89,7 @@ Status BuildSentencePieceVocabOp::SentenceThread() {
} else {
if (vocab_ == nullptr) {
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__,
"Invalid parameter, sentencepiece vocab not set.");
"Invalid parameter, SentencePiece vocab not set.");
}
vocab_->set_model_proto(model_proto);
}

6
mindspore/ccsrc/minddata/dataset/engine/datasetops/build_vocab_op.cc
View File

@ -97,7 +97,7 @@ Status BuildVocabOp::operator()() {
for (std::string col : col_names_) {
auto itr = column_name_id_map_.find(col);
CHECK_FAIL_RETURN_UNEXPECTED(itr != column_name_id_map_.end(),
"Invalid parameter, column name: " + col + " does not exist.");
"Invalid parameter, column name: " + col + " does not exist in dataset.");
col_ids_.push_back(itr->second);
}
} else {
@ -113,7 +113,7 @@ Status BuildVocabOp::operator()() {
RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row));
}
RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row));
CHECK_FAIL_RETURN_UNEXPECTED(!eoe_warning, "no op should be after from_dataset (repeat detected)");
CHECK_FAIL_RETURN_UNEXPECTED(!eoe_warning, "no operator should be after from_dataset (repeat detected)");
eoe_warning = true;
}
@ -137,7 +137,7 @@ Status BuildVocabOp::CollectorThread() {
++num_quited_worker;
}
} // all frequencies are obtained
CHECK_FAIL_RETURN_UNEXPECTED(!word_cnt_.empty(), "Invalid data, no words in the dataset.");
CHECK_FAIL_RETURN_UNEXPECTED(!word_cnt_.empty(), "Invalid data, there are no words in the dataset.");
std::vector<std::string> words;
// make sure enough is reserved, this will become a partially sorted list eventually
words.reserve(wrkr_map->size());

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_base_op.cc
View File

@ -227,7 +227,7 @@ Status CacheBase::UpdateColumnMapFromCache() {
Status CacheBase::GetPrefetchRow(row_id_type row_id, TensorRow *out) {
RETURN_UNEXPECTED_IF_NULL(out);
CHECK_FAIL_RETURN_UNEXPECTED(row_id >= 0, "Expect positive row id");
CHECK_FAIL_RETURN_UNEXPECTED(row_id >= 0, "Expect positive row id, but got:" + std::to_string(row_id));
RETURN_IF_NOT_OK(prefetch_.PopFront(row_id, out));
return Status::OK();
}
@ -280,7 +280,7 @@ Status CacheBase::Prefetcher(int32_t worker_id) {
cache_miss.clear();
std::unique_ptr<IOBlock> blk;
RETURN_IF_NOT_OK(prefetch_queues_[worker_id]->PopFront(&blk));
CHECK_FAIL_RETURN_UNEXPECTED(!blk->eof(), "Expect eoe or a regular io block");
CHECK_FAIL_RETURN_UNEXPECTED(!blk->eof(), "Expect eoe or a regular io block.");
if (!blk->eoe()) {
RETURN_IF_NOT_OK(blk->GetKeys(&prefetch_keys));
Status rc;

11
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_merge_op.cc
View File

@ -127,7 +127,7 @@ Status CacheMergeOp::CacheMissWorkerEntry(int32_t workerId) {
} else {
row_id_type row_id = new_row.getId();
if (row_id < 0) {
std::string errMsg = "Expect positive row id: " + std::to_string(row_id);
std::string errMsg = "Expect positive row id, but got: " + std::to_string(row_id);
RETURN_STATUS_UNEXPECTED(errMsg);
}
if (cache_missing_rows_) {
@ -191,7 +191,9 @@ Status CacheMergeOp::Cleaner() {
Status CacheMergeOp::PrepareOperator() { // Run any common code from super class first before adding our own
// specific logic
CHECK_FAIL_RETURN_UNEXPECTED(child_.size() == kNumChildren, "Incorrect number of children");
CHECK_FAIL_RETURN_UNEXPECTED(
child_.size() == kNumChildren,
"Incorrect number of children of CacheMergeOp, required num is 2, but got:" + std::to_string(child_.size()));
RETURN_IF_NOT_OK(DatasetOp::PrepareOperator());
// Get the computed check sum from all ops in the cache miss class
uint32_t cache_crc = DatasetOp::GenerateCRC(child_[kCacheMissChildIdx]);
@ -209,11 +211,12 @@ Status CacheMergeOp::PrepareOperator() { // Run any common code from super clas
}
Status CacheMergeOp::ComputeColMap() {
CHECK_FAIL_RETURN_UNEXPECTED(child_[kCacheMissChildIdx] != nullptr, "Invalid data, cache miss stream empty.");
CHECK_FAIL_RETURN_UNEXPECTED(child_[kCacheMissChildIdx] != nullptr, "Invalid data, cache miss stream is empty.");
if (column_name_id_map().empty()) {
column_name_id_map_ = child_[kCacheMissChildIdx]->column_name_id_map();
}
CHECK_FAIL_RETURN_UNEXPECTED(!column_name_id_map().empty(), "Invalid data, column_name_id_map is empty.");
CHECK_FAIL_RETURN_UNEXPECTED(!column_name_id_map().empty(),
"Invalid data, column_name_id_map of CacheMergeOp is empty.");
return Status::OK();
}

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/cache_op.cc
View File

@ -123,7 +123,7 @@ Status CacheOp::CacheAllRows(int32_t worker_id) {
// from again.
RETURN_IF_NOT_OK(child_iterator->FetchNextTensorRow(&row));
if (!row.eof()) {
RETURN_STATUS_UNEXPECTED("Cache op expects to get an eof after eoe from child.");
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Cache op expects to get an eof after eoe from child.");
}
break;
}
@ -173,7 +173,7 @@ Status CacheOp::WaitForCachingAllRows() {
case CacheServiceState::kNone:
case CacheServiceState::kError:
default:
RETURN_STATUS_UNEXPECTED("Unexpected state: " + std::to_string(out));
RETURN_STATUS_UNEXPECTED("Unexpected Cache server state: " + std::to_string(out));
}
} while (!BuildPhaseDone);
// Get statistics from the server, and if we are not the one to create the cache,

18
mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc
View File

@ -57,12 +57,14 @@ DatasetOp::DatasetOp(int32_t op_connector_size, std::shared_ptr<SamplerRT> sampl
// Adds a operator to become our child.
Status DatasetOp::AddChild(std::shared_ptr<DatasetOp> child) {
if (std::dynamic_pointer_cast<DeviceQueueOp>(child) != nullptr) {
std::string err_msg("DeviceQueueOp cannot be added as a child, DeviceQueueOp must be a root node");
std::string err_msg(
"DeviceQueueOp cannot be added as a child. DeviceQueueOp must be a root node, which means no operator should be "
"after device_queue operation.");
RETURN_STATUS_UNEXPECTED(err_msg);
}
if (operator_id_ == kInvalidOperatorId) {
std::string err_msg(
"Cannot add child node. Tree node connections can only"
"Cannot add child node. Tree node connections can only "
"be made if the node belongs to a tree.");
RETURN_STATUS_UNEXPECTED(err_msg);
}
@ -70,7 +72,7 @@ Status DatasetOp::AddChild(std::shared_ptr<DatasetOp> child) {
// disallow relationships with other trees
if (tree_ != child->tree_) {
std::string err_msg(
"Cannot add child node. Tree node connections can only be made if both nodes belong to the same tree.");
"Cannot add child node. Tree node connections can only be made if both nodes belong to the same tree.");
RETURN_STATUS_UNEXPECTED(err_msg);
}
child_.push_back(child);
@ -81,7 +83,7 @@ Status DatasetOp::AddChild(std::shared_ptr<DatasetOp> child) {
Status DatasetOp::RemoveChild(std::shared_ptr<DatasetOp> child) {
if (operator_id_ == kInvalidOperatorId) {
std::string err_msg(
"Cannot remove child node. Tree node connections can only"
"Cannot remove child node. Tree node connections can only "
"be made if the node belongs to a tree.");
RETURN_STATUS_UNEXPECTED(err_msg);
}
@ -89,7 +91,7 @@ Status DatasetOp::RemoveChild(std::shared_ptr<DatasetOp> child) {
// disallow relationships with other trees
if (tree_ != child->tree_) {
std::string err_msg(
"Cannot remove child node. Tree node connections can only be made if both nodes belong to the same tree.");
"Cannot remove child node. Tree node connections can only be made if both nodes belong to the same tree.");
RETURN_STATUS_UNEXPECTED(err_msg);
}
@ -130,11 +132,11 @@ void DatasetOp::RemoveParent(const DatasetOp *parent) {
// Removes this node from the tree and connects it's parent/child together
Status DatasetOp::Remove() {
if (parent_.size() > 1) {
std::string err_msg("No support for op removal if the operator has more than one parent");
std::string err_msg("No support for op removal if the operator has more than one parent.");
RETURN_STATUS_UNEXPECTED(err_msg);
}
if (child_.size() > 1) {
std::string err_msg("No support for op removal if the operator has more than one child");
std::string err_msg("No support for op removal if the operator has more than one child.");
RETURN_STATUS_UNEXPECTED(err_msg);
}
@ -290,7 +292,7 @@ Status DatasetOp::GetClassIndexing(std::vector<std::pair<std::string, std::vecto
return child_[child_.size() - 1]->GetClassIndexing(output_class_indexing);
} else {
*output_class_indexing = {};
RETURN_STATUS_UNEXPECTED("Trying to get class index from leaf node, missing override");
RETURN_STATUS_UNEXPECTED("Trying to get class index from leaf node, missing override.");
}
}

23
mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc
View File

@ -130,7 +130,7 @@ Status DeviceQueueOp::operator()() {
#ifdef ENABLE_DUMP_IR
if (md_channel_info_ == nullptr) {
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "RDR module init failed.");
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "[Internal ERROR] RDR module init failed.");
}
#endif
if (device_type_ == DeviceType::Ascend) {
@ -236,9 +236,9 @@ Status DeviceQueueOp::SendDataToAscend() {
return Status::OK();
}
return Status(StatusCode::kMDTDTPushFailure,
"TDT Push data into device Failed, please check the first error or TraceBack first, following are"
"TDT Push data into device Failed, check the first error or TraceBack first, following are"
" several possible checking way: 1) if training is not ready, still in network graph compiling"
" stage, please check error raised by Network used operator or environment configuration. 2) if"
" stage, check error raised by Network used operator or environment configuration. 2) if"
" interrupt in middle process of training, may check whether dataset sending num and network"
" training num mismatch. 3) if this error raised in end of training, ignore this. 4) other cases,"
" try find ascend host log or checking info log etc.");
@ -289,12 +289,12 @@ Status DeviceQueueOp::SendRowToTdt(TensorRow currRow, bool isProfilingEnable, in
return Status::OK();
}
return Status(StatusCode::kMDTDTPushFailure,
"TDT Push data into device Failed, please check the first error or TraceBack first, following are"
"TDT Push data into device Failed, check the first error or TraceBack first, following are"
" several possible checking way: 1) if training is not ready, still in network graph compiling"
" stage, please check error raised by Network used operator or environment configuration. 2) if"
" stage, check error raised by Network used operator or environment configuration. 2) if"
" interrupt in middle process of training, may check whether dataset sending num and network"
" training num mismatch. 3) if this error raised in end of training, ignore this. 4) other cases,"
" try find ascend host log or checking info log ects.");
" try find ascend host log or checking info log ects or search this in mindspore's FAQ.");
}
if (create_data_info_queue_) {
DATA_INFO data_info;
@ -405,7 +405,8 @@ Status DeviceQueueOp::PushDataToGPU() {
}
handle = GpuBufferMgr::GetInstance().Open(0, channel_name_, data_size, release_function);
if (handle == INVALID_HANDLE) {
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "Failed to open channel for sending data.");
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__,
"[Internal ERROR] Failed to open channel for sending data.");
}
is_open = true;
}
@ -413,7 +414,8 @@ Status DeviceQueueOp::PushDataToGPU() {
// Data prefetch only when PS mode enables cache.
if (!ps::PsDataPrefetch::GetInstance().PrefetchData(channel_name_, items[0].data_ptr_, items[0].data_len_,
items[0].data_type_)) {
return Status(StatusCode::kMDTimeOut, __LINE__, __FILE__, "Failed to prefetch data.");
return Status(StatusCode::kMDTimeOut, __LINE__, __FILE__,
"Failed to prefetch data in current PS mode(cache data when sending).");
}
RETURN_IF_NOT_OK(RetryPushData(handle, items));
send_batch++;
@ -471,7 +473,8 @@ Status DeviceQueueOp::RetryPushData(unsigned int handle, const std::vector<DataI
BlockQueueStatus_T ret = GpuBufferMgr::GetInstance().Push(handle, items, WAIT_TIME);
if (ret) {
if (ret == BlockQueueStatus_T::ERROR_INPUT) {
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "Invalid input data, please check it.");
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__,
"Invalid data, check the output of dataset with creating iterator and print data item.");
} else {
if (!stop_send_) {
if (!flagLog) {
@ -576,7 +579,7 @@ Status DeviceQueueOp::MallocForGPUData(std::vector<device::DataItemGpu> *items,
if (memcpy_s(sub_item.data_ptr_, sub_item.data_len_, column_data,
static_cast<uint32_t>(curr_row[i++]->SizeInBytes())) != 0) {
MS_LOG(ERROR) << "memcpy_s failed!";
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "memcpy_s failed.");
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "memcpy failed when using memcpy_s do copy.");
}
}

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/epoch_ctrl_op.cc
View File

@ -42,7 +42,7 @@ void EpochCtrlOp::Print(std::ostream &out, bool show_all) const {
Status EpochCtrlOp::GetNextRow(TensorRow *row, int32_t worker_id, bool retry_if_eoe) {
if (child_.empty()) {
RETURN_STATUS_UNEXPECTED("EpochCtrlOp can't be the leaf node.");
RETURN_STATUS_UNEXPECTED("EpochCtrlOp can't be the leaf node(first operator) of pipeline.");
}
// `retry_if_eoe` is false because EpochCtrlOp does not eat EOE.

3
mindspore/ccsrc/minddata/dataset/engine/datasetops/filter_op.cc
View File

@ -37,7 +37,8 @@ FilterOp::FilterOp(const std::vector<std::string> &in_col_names, int32_t num_wor
Status FilterOp::LaunchThreadsAndInitOp() {
// The operator class just starts off threads by calling the tree_ function.
if (tree_ == nullptr) {
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "Pipeline init failed, Execution tree not set.");
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__,
"[Internal ERROR] Pipeline init failed, Execution tree not set.");
}
filter_queues_.Init(num_workers_, oc_queue_size_);
RETURN_IF_NOT_OK(filter_queues_.Register(tree_->AllTasks()));

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/parallel_op.cc
View File

@ -44,7 +44,7 @@ ParallelOp::ParallelOp(int32_t num_workers, int32_t op_connector_size, std::shar
// Creates the internal worker connector for the parallel op if the derived class wants to use it
Status ParallelOp::CreateWorkerConnector(int32_t worker_connector_size) {
if (worker_connector_size == 0) {
RETURN_STATUS_UNEXPECTED("Worker connector size 0 is invalid.");
RETURN_STATUS_UNEXPECTED("Create Worker Connector failed, as given connector size 0 is invalid.");
}
num_producers_ = 1;
worker_connector_size_ = worker_connector_size;

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/project_op.cc
View File

@ -113,7 +113,7 @@ Status ProjectOp::ComputeColMap() {
for (size_t i = 0; i < columns_to_project_.size(); i++) {
std::string &current_column = columns_to_project_[i];
if (child_column_name_mapping.find(current_column) == child_column_name_mapping.end()) {
std::string err_msg = "Invalid parameter, column name: " + current_column + " does not exist.";
std::string err_msg = "Invalid parameter, column name: " + current_column + " does not exist in dataset.";
RETURN_STATUS_UNEXPECTED(err_msg);
}
// Setup the new column name mapping for ourself (base class field)

6
mindspore/ccsrc/minddata/dataset/engine/datasetops/shuffle_op.cc
View File

@ -96,7 +96,7 @@ Status ShuffleOp::AddRowToShuffleBuffer(TensorRow new_shuffle_row) {
} else {
if (!(*shuffle_buffer_)[shuffle_last_row_idx_].empty()) {
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__,
"Last row of shuffle buffer should not be occupied!");
"[Internal ERROR] Last row of shuffle buffer should not be occupied!");
}
(*shuffle_buffer_)[shuffle_last_row_idx_] = std::move(new_shuffle_row);
}
@ -206,7 +206,7 @@ Status ShuffleOp::InitShuffleBuffer() {
// rows.
if (shuffle_buffer_state_ != kShuffleStateInit) {
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__,
"Invalid shuffle buffer state (SHUFFLE_STATE_INIT expected)");
"Invalid shuffle buffer state, shuffle buffer should be init first or reset after each epoch.");
}
// Before we drop into the fetching loop, call the fetch once for the first time
@ -221,7 +221,7 @@ Status ShuffleOp::InitShuffleBuffer() {
}
if (new_row.empty()) {
RETURN_STATUS_UNEXPECTED("Unable to fetch a single row for shuffle buffer.");
RETURN_STATUS_UNEXPECTED("Invalid data, unable to fetch a single row for shuffle buffer.");
}
// Now fill the rest of the shuffle buffer until we are unable to get the next row or we reached

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc
View File

@ -66,7 +66,7 @@ Status AlbumOp::PrescanEntry() {
dirname_offset_ = folder_path_.length();
std::shared_ptr<Path::DirIterator> dirItr = Path::DirIterator::OpenDirectory(&folder);
if (!folder.Exists() || dirItr == nullptr) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open folder: " + folder_path_);
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open folder: " + folder_path_ + ".");
}
MS_LOG(INFO) << "Album folder Path found: " << folder_path_ << ".";
@ -84,7 +84,8 @@ Status AlbumOp::PrescanEntry() {
num_rows_ = image_rows_.size();
if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API AlbumDataset. Please check file path or dataset API.");
"Invalid data, data file may not be suitable to read with AlbumDataset API. Check file path:" + folder_path_ +
".");
}
return Status::OK();
}
@ -210,7 +211,7 @@ Status AlbumOp::LoadIntArrayTensor(const nlohmann::json &json_obj, uint32_t col_
RETURN_IF_NOT_OK(Tensor::CreateFromVector(data, &label));
} else {
RETURN_STATUS_UNEXPECTED("Invalid data, column type is neither int32 nor int64, it is " +
RETURN_STATUS_UNEXPECTED("Invalid data, column type in data_schema is neither int32 nor int64, it is " +
data_schema_->column(col_num).type().ToString());
}
row->push_back(std::move(label));
@ -239,7 +240,7 @@ Status AlbumOp::LoadFloatArrayTensor(const nlohmann::json &json_obj, uint32_t co
RETURN_IF_NOT_OK(Tensor::CreateFromVector(data, &float_array));
} else {
RETURN_STATUS_UNEXPECTED("Invalid data, column type is neither float32 nor float64, it is " +
RETURN_STATUS_UNEXPECTED("Invalid data, column type in data_schema is neither float32 nor float64, it is " +
data_schema_->column(col_num).type().ToString());
}
row->push_back(std::move(float_array));

19
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc
View File

@ -94,10 +94,11 @@ Status CelebAOp::ParseAttrFile() {
num_rows_in_attr_file_ = static_cast<int64_t>(std::stoul(rows_num)); // First line is rows number in attr file
} catch (std::invalid_argument &e) {
RETURN_STATUS_UNEXPECTED(
"Invalid data, failed to convert rows_num from attr_file to unsigned long, invalid argument: " + rows_num);
"Invalid data, failed to convert rows_num from attr_file to unsigned long, invalid value: " + rows_num + ".");
} catch (std::out_of_range &e) {
RETURN_STATUS_UNEXPECTED(
"Invalid data, failed to convert rows_num from attr_file to unsigned long, out of range: " + rows_num);
"Invalid data, failed to convert rows_num from attr_file to unsigned long, value out of range: " + rows_num +
".");
}
(void)getline(attr_file, attr_name); // Second line is attribute name,ignore it
@ -142,10 +143,10 @@ bool CelebAOp::CheckDatasetTypeValid() {
try {
type = std::stoi(vec[1]);
} catch (std::invalid_argument &e) {
MS_LOG(WARNING) << "Invalid data, failed to convert to unsigned long, invalid argument: " << vec[1] << ".";
MS_LOG(WARNING) << "Invalid data, failed to convert to int, invalid value: " << vec[1] << ".";
return false;
} catch (std::out_of_range &e) {
MS_LOG(WARNING) << "Invalid data, failed to convert to unsigned long, out of range: " << vec[1] << ".";
MS_LOG(WARNING) << "Invalid data, failed to convert to int, value out of range: " << vec[1] << ".";
return false;
}
// train:0, valid=1, test=2
@ -187,9 +188,12 @@ Status CelebAOp::ParseImageAttrInfo() {
try {
value = std::stoi(split[label_index]);
} catch (std::invalid_argument &e) {
RETURN_STATUS_UNEXPECTED("Invalid data, failed to convert to ulong, invalid argument: " + split[label_index]);
RETURN_STATUS_UNEXPECTED("Invalid data, failed to convert item from attr_file to int, corresponding value: " +
split[label_index] + ".");
} catch (std::out_of_range &e) {
RETURN_STATUS_UNEXPECTED("Conversion to int failed, out of range: " + split[label_index]);
RETURN_STATUS_UNEXPECTED(
"Invalid data, failed to convert item from attr_file to int as out of range, corresponding value: " +
split[label_index] + ".");
}
image_labels.second.push_back(value);
}
@ -203,7 +207,8 @@ Status CelebAOp::ParseImageAttrInfo() {
num_rows_ = image_labels_vec_.size();
if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API CelebADataset. Please check file path or dataset API.");
"Invalid data, data file may not be suitable to read with CelebADataset API. Check attr_file in directory:" +
folder_path_ + ".");
}
MS_LOG(DEBUG) << "Celeba dataset rows number is " << num_rows_ << ".";
return Status::OK();

37
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc
View File

@ -138,11 +138,13 @@ Status CifarOp::ReadCifar10BlockData() {
}
std::ifstream in(file, std::ios::binary);
CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar10 file: " + file);
CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar10 file: " + file +
", make sure file not damaged or permission denied.");
for (uint32_t index = 0; index < num_cifar10_records / kCifarBlockImageNum; ++index) {
(void)in.read(reinterpret_cast<char *>(&(image_data[0])), block_size * sizeof(unsigned char));
CHECK_FAIL_RETURN_UNEXPECTED(!in.fail(), "Invalid data, failed to read data from cifar10 file: " + file);
CHECK_FAIL_RETURN_UNEXPECTED(!in.fail(), "Invalid data, failed to read data from cifar10 file: " + file +
", re-download dataset(make sure it is CIFAR-10 binary version).");
(void)cifar_raw_data_block_->EmplaceBack(image_data);
// Add file path info
path_record_.push_back(file);
@ -182,11 +184,13 @@ Status CifarOp::ReadCifar100BlockData() {
}
std::ifstream in(file, std::ios::binary);
CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar100 file: " + file);
CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar100 file: " + file +
", make sure file not damaged or permission denied.");
for (uint32_t index = 0; index < num_cifar100_records / kCifarBlockImageNum; index++) {
(void)in.read(reinterpret_cast<char *>(&(image_data[0])), block_size * sizeof(unsigned char));
CHECK_FAIL_RETURN_UNEXPECTED(!in.fail(), "Invalid data, failed to read data from cifar100 file: " + file);
CHECK_FAIL_RETURN_UNEXPECTED(!in.fail(), "Invalid data, failed to read data from cifar100 file: " + file +
", re-download dataset(make sure it is CIFAR-100 binary version).");
(void)cifar_raw_data_block_->EmplaceBack(image_data);
// Add file path info
path_record_.push_back(file);
@ -209,7 +213,8 @@ Status CifarOp::GetCifarFiles() {
}
}
} else {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open directory: " + dir_path.toString());
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open directory: " + dir_path.toString() +
", make sure file not damaged or permission denied.");
}
CHECK_FAIL_RETURN_UNEXPECTED(!cifar_files_.empty(), "Invalid file, no .bin files found under " + folder_path_);
std::sort(cifar_files_.begin(), cifar_files_.end());
@ -251,8 +256,8 @@ Status CifarOp::ParseCifarData() {
num_rows_ = cifar_image_label_pairs_.size();
if (num_rows_ == 0) {
std::string api = cifar_type_ == kCifar10 ? "Cifar10Dataset" : "Cifar100Dataset";
RETURN_STATUS_UNEXPECTED("Invalid data, no valid data matching the dataset API " + api +
". Please check file path or dataset API.");
RETURN_STATUS_UNEXPECTED("Invalid data, data file may not suitable to read with " + api +
" API. Check file in directory:" + folder_path_);
}
cifar_raw_data_block_->Reset();
return Status::OK();
@ -262,7 +267,7 @@ Status CifarOp::ParseCifarData() {
Status CifarOp::GetClassIds(std::map<int32_t, std::vector<int64_t>> *cls_ids) const {
if (cls_ids == nullptr || !cls_ids->empty()) {
RETURN_STATUS_UNEXPECTED(
"Map for storaging image-index pair is nullptr or has been set in other place,"
"[Internal ERROR] Map for containing image-index pair is nullptr or has been set in other place,"
"it must be empty before using GetClassIds.");
}
@ -312,8 +317,9 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage,
constexpr int64_t num_cifar10_records = 10000;
for (auto &file : op->cifar_files_) {
Path file_path(file);
CHECK_FAIL_RETURN_UNEXPECTED(file_path.Exists() && !file_path.IsDirectory(),
"Invalid file, failed to open cifar10 file: " + file);
CHECK_FAIL_RETURN_UNEXPECTED(
file_path.Exists() && !file_path.IsDirectory(),
"Invalid file, failed to open cifar10 file: " + file + ", make sure file not damaged or permission denied.");
std::string file_name = file_path.Basename();
if (op->usage_ == "train") {
@ -326,7 +332,8 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage,
std::ifstream in(file, std::ios::binary);
CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar10 file: " + file);
CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar10 file: " + file +
", make sure file not damaged or permission denied.");
*count = *count + num_cifar10_records;
}
return Status::OK();
@ -336,8 +343,9 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage,
Path file_path(file);
std::string file_name = file_path.Basename();
CHECK_FAIL_RETURN_UNEXPECTED(file_path.Exists() && !file_path.IsDirectory(),
"Invalid file, failed to find cifar100 file: " + file);
CHECK_FAIL_RETURN_UNEXPECTED(
file_path.Exists() && !file_path.IsDirectory(),
"Invalid file, failed to find cifar100 file: " + file + ", make sure file not damaged or permission denied.");
if (op->usage_ == "train" && file_path.Basename().find("train") == std::string::npos) continue;
if (op->usage_ == "test" && file_path.Basename().find("test") == std::string::npos) continue;
@ -348,7 +356,8 @@ Status CifarOp::CountTotalRows(const std::string &dir, const std::string &usage,
num_cifar100_records += 50000;
}
std::ifstream in(file, std::ios::binary);
CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar100 file: " + file);
CHECK_FAIL_RETURN_UNEXPECTED(in.is_open(), "Invalid file, failed to open cifar100 file: " + file +
", make sure file not damaged or permission denied.");
}
*count = num_cifar100_records;
return Status::OK();

11
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/clue_op.cc
View File

@ -56,7 +56,7 @@ Status ClueOp::GetValue(const nlohmann::json &js, std::vector<std::string> key_c
if (cursor.find(key_chain[i]) != cursor.end()) {
cursor = cursor[key_chain[i]];
} else {
RETURN_STATUS_UNEXPECTED("Invalid data, failed to find key: " + key_chain[i]);
RETURN_STATUS_UNEXPECTED("Invalid data, in given JSON file, failed to find key: " + key_chain[i]);
}
}
std::string final_str = key_chain.back();
@ -110,7 +110,7 @@ Status ClueOp::LoadFile(const std::string &file, int64_t start_offset, int64_t e
js = nlohmann::json::parse(line);
} catch (const std::exception &err) {
// Catch any exception and convert to Status return code
RETURN_STATUS_UNEXPECTED("Invalid file, failed to parse json file: " + file);
RETURN_STATUS_UNEXPECTED("Invalid file, failed to parse JSON file: " + file);
}
int cols_count = cols_to_keyword_.size();
TensorRow tRow(cols_count, nullptr);
@ -208,8 +208,13 @@ Status ClueOp::CalculateNumRowsPerShard() {
num_rows_ += count;
}
if (num_rows_ == 0) {
std::stringstream ss;
for (int i = 0; i < clue_files_list_.size(); ++i) {
ss << " " << clue_files_list_[i];
}
std::string file_list = ss.str();
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API CLUEDataset. Please check file path or dataset API.");
"Invalid data, data file may not be suitable to read with CLUEDataset API. Check file path:" + file_list);
}
num_rows_per_shard_ = static_cast<int64_t>(std::ceil(num_rows_ * 1.0 / num_devices_));

54
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/coco_op.cc
View File

@ -112,7 +112,7 @@ Status CocoOp::Builder::SanityCheck() {
? "Invalid parameter, Coco image folder path is invalid or not set, path: " + builder_dir_ + ".\n"
: "";
err_msg += file.Exists() == false
? "Invalid parameter, Coco annotation json path is invalid or not set, path: " + builder_dir_ + ".\n"
? "Invalid parameter, Coco annotation JSON path is invalid or not set, path: " + builder_dir_ + ".\n"
: "";
err_msg += builder_num_workers_ <= 0 ? "Invalid parameter, num_parallel_workers must be greater than 0, but got " +
std::to_string(builder_num_workers_) + ".\n"
@ -154,7 +154,7 @@ Status CocoOp::LoadTensorRow(row_id_type row_id, TensorRow *trow) {
auto itr = coordinate_map_.find(image_id);
if (itr == coordinate_map_.end()) {
RETURN_STATUS_UNEXPECTED("Invalid data, image_id: " + image_id +
" in annotation node is not found in image node in json file.");
" in annotation node is not found in image node in JSON file.");
}
std::string kImageFile = image_folder_path_ + std::string("/") + image_id;
@ -204,7 +204,7 @@ Status CocoOp::LoadDetectionTensorRow(row_id_type row_id, const std::string &ima
auto itr_item = simple_item_map_.find(image_id);
if (itr_item == simple_item_map_.end()) {
RETURN_STATUS_UNEXPECTED("Invalid data, image_id: " + image_id +
" in annotation node is not found in image node in json file.");
" in annotation node is not found in image node in JSON file.");
}
std::vector<uint32_t> annotation = itr_item->second;
@ -247,7 +247,7 @@ Status CocoOp::LoadSimpleTensorRow(row_id_type row_id, const std::string &image_
auto itr_item = simple_item_map_.find(image_id);
if (itr_item == simple_item_map_.end()) {
RETURN_STATUS_UNEXPECTED("Invalid data, image_id: " + image_id +
" in annotation node is not found in image node in json file.");
" in annotation node is not found in image node in JSON file.");
}
item_queue = itr_item->second;
@ -282,7 +282,7 @@ Status CocoOp::LoadMixTensorRow(row_id_type row_id, const std::string &image_id,
auto itr_item = simple_item_map_.find(image_id);
if (itr_item == simple_item_map_.end()) {
RETURN_STATUS_UNEXPECTED("Invalid data, image_id: " + image_id +
" in annotation node is not found in image node in json file.");
" in annotation node is not found in image node in JSON file.");
}
std::vector<uint32_t> annotation = itr_item->second;
@ -327,7 +327,7 @@ Status CocoOp::LoadMixTensorRow(row_id_type row_id, const std::string &image_id,
template <typename T>
Status CocoOp::SearchNodeInJson(const nlohmann::json &input_tree, std::string node_name, T *output_node) {
auto node = input_tree.find(node_name);
CHECK_FAIL_RETURN_UNEXPECTED(node != input_tree.end(), "Invalid data, invalid node found in json: " + node_name);
CHECK_FAIL_RETURN_UNEXPECTED(node != input_tree.end(), "Invalid data, required node not found in JSON: " + node_name);
(*output_node) = *node;
return Status::OK();
}
@ -338,7 +338,7 @@ Status CocoOp::ParseAnnotationIds() {
std::ifstream in(annotation_path_);
in >> js;
} catch (const std::exception &err) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open json file: " + annotation_path_);
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open JSON file: " + annotation_path_ + ".");
}
std::vector<std::string> image_que;
@ -359,7 +359,7 @@ Status CocoOp::ParseAnnotationIds() {
auto itr_file = image_index_.find(image_id);
if (itr_file == image_index_.end()) {
RETURN_STATUS_UNEXPECTED("Invalid data, image_id: " + std::to_string(image_id) +
" in annotation node is not found in image node in json file.");
" in annotation node is not found in image node in JSON file.");
}
file_name = itr_file->second;
switch (task_type_) {
@ -388,14 +388,15 @@ Status CocoOp::ParseAnnotationIds() {
num_rows_ = image_ids_.size();
if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API CocoDataset. Please check file path or dataset API.");
"Invalid data, data file may not suitable to read with CocoDataset API. Check file in directory: " +
image_folder_path_ + ".");
}
return Status::OK();
}
Status CocoOp::ImageColumnLoad(const nlohmann::json &image_tree, std::vector<std::string> *image_vec) {
if (image_tree.size() == 0) {
RETURN_STATUS_UNEXPECTED("Invalid data, no \"image\" node found in json file: " + annotation_path_);
RETURN_STATUS_UNEXPECTED("Invalid data, no \"image\" node found in JSON file: " + annotation_path_ + ".");
}
for (auto img : image_tree) {
std::string file_name;
@ -418,8 +419,8 @@ Status CocoOp::DetectionColumnLoad(const nlohmann::json &annotation_tree, const
RETURN_IF_NOT_OK(SearchNodeInJson(annotation_tree, std::string(kJsonAnnoCategoryId), &category_id));
auto search_category = category_set_.find(category_id);
if (search_category == category_set_.end())
RETURN_STATUS_UNEXPECTED("Invalid data, category_id can't find in categories where category_id: " +
std::to_string(category_id));
RETURN_STATUS_UNEXPECTED(
"Invalid data, category_id can't find in categories where category_id: " + std::to_string(category_id) + ".");
auto node_iscrowd = annotation_tree.find(kJsonAnnoIscrowd);
if (node_iscrowd != annotation_tree.end()) iscrowd = *node_iscrowd;
bbox.insert(bbox.end(), node_bbox.begin(), node_bbox.end());
@ -456,12 +457,13 @@ Status CocoOp::KeypointColumnLoad(const nlohmann::json &annotation_tree, const s
const int32_t &unique_id) {
auto itr_num_keypoint = annotation_tree.find(kJsonAnnoNumKeypoints);
if (itr_num_keypoint == annotation_tree.end())
RETURN_STATUS_UNEXPECTED("Invalid data, no num_keypoint found in annotations where id: " +
std::to_string(unique_id));
RETURN_STATUS_UNEXPECTED(
"Invalid data, no num_keypoint found in annotation file where image_id: " + std::to_string(unique_id) + ".");
simple_item_map_[image_file].push_back(*itr_num_keypoint);
auto itr_keypoint = annotation_tree.find(kJsonAnnoKeypoints);
if (itr_keypoint == annotation_tree.end())
RETURN_STATUS_UNEXPECTED("Invalid data, no keypoint found in annotations where id: " + std::to_string(unique_id));
RETURN_STATUS_UNEXPECTED(
"Invalid data, no keypoint found in annotation file where image_id: " + std::to_string(unique_id) + ".");
coordinate_map_[image_file].push_back(*itr_keypoint);
return Status::OK();
}
@ -470,31 +472,31 @@ Status CocoOp::PanopticColumnLoad(const nlohmann::json &annotation_tree, const s
const int32_t &image_id) {
auto itr_segments = annotation_tree.find(kJsonAnnoSegmentsInfo);
if (itr_segments == annotation_tree.end())
RETURN_STATUS_UNEXPECTED("Invalid data, no segments_info found in annotations where image_id: " +
std::to_string(image_id));
RETURN_STATUS_UNEXPECTED(
"Invalid data, no segments_info found in annotation file where image_id: " + std::to_string(image_id) + ".");
for (auto info : *itr_segments) {
std::vector<float> bbox;
uint32_t category_id = 0;
auto itr_bbox = info.find(kJsonAnnoBbox);
if (itr_bbox == info.end())
RETURN_STATUS_UNEXPECTED("Invalid data, no bbox found in segments_info where image_id: " +
std::to_string(image_id));
RETURN_STATUS_UNEXPECTED("Invalid data, no bbox found in segments_info(in annotation file) where image_id: " +
std::to_string(image_id) + ".");
bbox.insert(bbox.end(), itr_bbox->begin(), itr_bbox->end());
coordinate_map_[image_file].push_back(bbox);
RETURN_IF_NOT_OK(SearchNodeInJson(info, std::string(kJsonAnnoCategoryId), &category_id));
auto search_category = category_set_.find(category_id);
if (search_category == category_set_.end())
RETURN_STATUS_UNEXPECTED("Invalid data, category_id can't find in categories where category_id: " +
std::to_string(category_id));
RETURN_STATUS_UNEXPECTED(
"Invalid data, category_id can't find in categories where category_id: " + std::to_string(category_id) + ".");
auto itr_iscrowd = info.find(kJsonAnnoIscrowd);
if (itr_iscrowd == info.end())
RETURN_STATUS_UNEXPECTED("Invalid data, no iscrowd found in segments_info where image_id: " +
std::to_string(image_id));
RETURN_STATUS_UNEXPECTED(
"Invalid data, no iscrowd found in segments_info where image_id: " + std::to_string(image_id) + ".");
auto itr_area = info.find(kJsonAnnoArea);
if (itr_area == info.end())
RETURN_STATUS_UNEXPECTED("Invalid data, no area found in segments_info where image_id: " +
std::to_string(image_id));
RETURN_STATUS_UNEXPECTED(
"Invalid data, no area found in segments_info where image_id: " + std::to_string(image_id) + ".");
simple_item_map_[image_file].push_back(category_id);
simple_item_map_[image_file].push_back(*itr_iscrowd);
simple_item_map_[image_file].push_back(*itr_area);
@ -512,7 +514,7 @@ Status CocoOp::CategoriesColumnLoad(const nlohmann::json &categories_tree) {
std::vector<int32_t> label_info;
auto itr_id = category.find(kJsonId);
if (itr_id == category.end()) {
RETURN_STATUS_UNEXPECTED("Invalid data, no json id found in categories of " + annotation_path_);
RETURN_STATUS_UNEXPECTED("Invalid data, no JSON id found in categories of " + annotation_path_);
}
id = *itr_id;
label_info.push_back(id);

21
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/csv_op.cc
View File

@ -490,7 +490,7 @@ Status CsvOp::LoadFile(const std::string &file, int64_t start_offset, int64_t en
std::ifstream ifs;
ifs.open(file, std::ifstream::in);
if (!ifs.is_open()) {
RETURN_STATUS_UNEXPECTED("Error opening file: " + file);
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open file: " + file);
}
if (column_name_list_.empty()) {
std::string tmp;
@ -506,17 +506,17 @@ Status CsvOp::LoadFile(const std::string &file, int64_t start_offset, int64_t en
int err = csv_parser.ProcessMessage(chr);
if (err != 0) {
if (err == -2) return Status(kMDInterrupted);
RETURN_STATUS_UNEXPECTED("Invalid file, failed to parse file: " + file + ":" +
RETURN_STATUS_UNEXPECTED("Invalid file, failed to parse file: " + file + ": line " +
std::to_string(csv_parser.GetTotalRows() + 1) +
". Error message: " + csv_parser.GetErrorMessage());
}
}
} catch (std::invalid_argument &ia) {
std::string err_row = std::to_string(csv_parser.GetTotalRows() + 1);
RETURN_STATUS_UNEXPECTED("Invalid data, " + file + ":" + err_row + ", type does not match.");
RETURN_STATUS_UNEXPECTED("Invalid data, " + file + ": line " + err_row + ", type does not match.");
} catch (std::out_of_range &oor) {
std::string err_row = std::to_string(csv_parser.GetTotalRows() + 1);
RETURN_STATUS_UNEXPECTED("Invalid data, " + file + ":" + err_row + ", out of range.");
RETURN_STATUS_UNEXPECTED("Invalid data, " + file + ": line " + err_row + ", value out of range.");
}
return Status::OK();
}
@ -597,8 +597,13 @@ Status CsvOp::CalculateNumRowsPerShard() {
num_rows_ += count;
}
if (num_rows_ == 0) {
std::stringstream ss;
for (int i = 0; i < csv_files_list_.size(); ++i) {
ss << " " << csv_files_list_[i];
}
std::string file_list = ss.str();
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API CsvDataset. Please check file path or CSV format.");
"Invalid data, data file may not suitable to read with CSVDataset API. Check file path: " + file_list + ".");
}
num_rows_per_shard_ = static_cast<int64_t>(std::ceil(num_rows_ * 1.0 / num_devices_));
@ -662,7 +667,7 @@ Status CsvOp::ComputeColMap() {
// Set the column name mapping (base class field)
if (column_name_id_map_.empty()) {
if (!ColumnNameValidate()) {
RETURN_STATUS_UNEXPECTED("Fail to validate column name for input CSV file list");
RETURN_STATUS_UNEXPECTED("Invalid file, failed to obtain column name from input CSV file list.");
}
for (auto &csv_file : csv_files_list_) {
@ -686,9 +691,9 @@ Status CsvOp::ComputeColMap() {
if (column_default_list_.size() != column_name_id_map_.size()) {
RETURN_STATUS_UNEXPECTED(
"Invalid parameter, the number of column names does not match the column defaults, column_default_list: " +
"Invalid parameter, the number of column names does not match the default column, size of default column_list: " +
std::to_string(column_default_list_.size()) +
", column_name_id_map: " + std::to_string(column_name_id_map_.size()));
", size of column_name: " + std::to_string(column_name_id_map_.size()));
}
return Status::OK();

20
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/generator_op.cc
View File

@ -62,7 +62,7 @@ Status GeneratorOp::CreateGeneratorObject() {
// Acquire Python GIL
py::gil_scoped_acquire gil_acquire;
if (Py_IsInitialized() == 0) {
return Status(StatusCode::kMDPythonInterpreterFailure, "Python Interpreter is finalized");
return Status(StatusCode::kMDPythonInterpreterFailure, "Python Interpreter is finalized.");
}
try {
py::array sample_ids;
@ -91,28 +91,32 @@ Status GeneratorOp::Init() {
Status GeneratorOp::PyRowToTensorRow(py::object py_data, TensorRow *tensor_row) {
if (!py::isinstance<py::tuple>(py_data)) {
return Status(StatusCode::kMDPyFuncException, __LINE__, __FILE__,
"Invalid parameter, Generator should return a tuple of numpy arrays.");
"Invalid data, Generator should return a tuple of NumPy arrays, currently returned is not a tuple.");
}
py::tuple py_row = py_data.cast<py::tuple>();
// Check if returned number of columns matches with column names
if (py_row.size() != column_names_.size()) {
return Status(
StatusCode::kMDPyFuncException, __LINE__, __FILE__,
"Invalid parameter, Generator should return same number of numpy arrays as specified in column names.");
"Invalid data, Generator should return same number of NumPy arrays as specified in column_names, the size of"
" column_names is:" +
std::to_string(column_names_.size()) +
"and number of returned NumPy array is:" + std::to_string(py_row.size()));
}
// Iterate over two containers simultaneously for memory copy
for (int i = 0; i < py_row.size(); ++i) {
py::object ret_py_ele = py_row[i];
if (!py::isinstance<py::array>(ret_py_ele)) {
return Status(StatusCode::kMDPyFuncException, __LINE__, __FILE__,
"Invalid parameter, Generator should return a tuple of numpy arrays.");
"Invalid data, Generator should return a tuple of NumPy arrays. Ensure each item in tuple that "
"returned by source function of GeneratorDataset be NumPy array.");
}
std::shared_ptr<Tensor> tensor;
RETURN_IF_NOT_OK(Tensor::CreateFromNpArray(ret_py_ele.cast<py::array>(), &tensor));
if ((!column_types_.empty()) && (column_types_[i] != DataType::DE_UNKNOWN) &&
(column_types_[i] != tensor->type())) {
return Status(StatusCode::kMDPyFuncException, __LINE__, __FILE__,
"Invalid parameter, input column type is not same with output tensor type.");
"Invalid data, type of returned data in GeneratorDataset is not same with specified column_types.");
}
tensor_row->push_back(tensor);
}
@ -181,6 +185,12 @@ Status GeneratorOp::operator()() {
return Status(StatusCode::kMDPyFuncException, __LINE__, __FILE__, e.what());
}
if (num_rows_sampled != -1 && num_rows_sampled != generator_counter_) {
if (generator_counter_ == 0) {
std::string msg =
"Unable to fetch data from GeneratorDataset, try iterate the source function of GeneratorDataset or check"
" value of num_epochs when create iterator.";
return Status(StatusCode::kMDPyFuncException, __LINE__, __FILE__, msg);
}
std::stringstream ss;
ss << "The actual amount of data read from generator " << generator_counter_
<< " is different from generator.len " << num_rows_sampled

6
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc
View File

@ -73,8 +73,8 @@ Status ImageFolderOp::PrescanMasterEntry(const std::string &filedir) {
num_rows_ = image_label_pairs_.size();
if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API ImageFolderDataset. "
"Please check file path or dataset API.");
"Invalid data, data file may not be suitable to read with ImageFolderDataset API. Check file path: " +
folder_path_);
}
// free memory of two queues used for pre-scan
folder_name_queue_->Reset();
@ -120,7 +120,7 @@ void ImageFolderOp::Print(std::ostream &out, bool show_all) const {
Status ImageFolderOp::GetClassIds(std::map<int32_t, std::vector<int64_t>> *cls_ids) const {
if (cls_ids == nullptr || !cls_ids->empty() || image_label_pairs_.empty()) {
if (image_label_pairs_.empty()) {
RETURN_STATUS_UNEXPECTED("No images found in dataset, please check if Op read images successfully or not.");
RETURN_STATUS_UNEXPECTED("No images found in dataset, try iterate dataset to check if read images success.");
} else {
RETURN_STATUS_UNEXPECTED(
"Map containing image-index pair is nullptr or has been set in other place,"

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/io_block.cc
View File

@ -36,7 +36,7 @@ IOBlock::IOBlock(IOBlockFlags io_block_flags) : io_block_flags_(io_block_flags)
// Fetches the first key from this block
Status IOBlock::GetKey(int64_t *out_key) const {
if (out_key == nullptr || index_keys_.empty()) {
RETURN_STATUS_UNEXPECTED("Failed to get the key from IOBlock");
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Failed to get the key from IOBlock.");
}
*out_key = index_keys_[0];
return Status::OK();
@ -45,7 +45,7 @@ Status IOBlock::GetKey(int64_t *out_key) const {
// Fetches the list of keys from this block.
Status IOBlock::GetKeys(std::vector<int64_t> *out_keys) const {
if (out_keys == nullptr) {
RETURN_STATUS_UNEXPECTED("Output arg for GetKeys is null");
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Output arg for GetKeys is null.");
}
*out_keys = index_keys_; // vector copy assign
return Status::OK();
@ -64,7 +64,7 @@ FilenameBlock::FilenameBlock(IOBlockFlags io_block_flags)
// Gets the filename from the block using the provided index container
Status FilenameBlock::GetFilename(std::string *out_filename, const AutoIndexObj<std::string> &index) const {
if (out_filename == nullptr) {
RETURN_STATUS_UNEXPECTED("Failed to get filename from FilenameBlock");
RETURN_STATUS_UNEXPECTED("Failed to get filename from FilenameBlock.");
}
// a FilenameBlock only has one key. Call base class method to fetch that key
@ -77,7 +77,7 @@ Status FilenameBlock::GetFilename(std::string *out_filename, const AutoIndexObj<
auto &it = r.first;
*out_filename = it.value();
} else {
RETURN_STATUS_UNEXPECTED("Could not find filename from index");
RETURN_STATUS_UNEXPECTED("Could not find filename from index.");
}
return Status::OK();

14
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc
View File

@ -141,10 +141,10 @@ void ManifestOp::Print(std::ostream &out, bool show_all) const {
Status ManifestOp::GetClassIds(std::map<int32_t, std::vector<int64_t>> *cls_ids) const {
if (cls_ids == nullptr || !cls_ids->empty() || image_labelname_.empty()) {
if (image_labelname_.empty()) {
RETURN_STATUS_UNEXPECTED("No image found in dataset, please check if Op read images successfully or not.");
RETURN_STATUS_UNEXPECTED("No image found in dataset. Try iterate dataset to check if read images success.");
} else {
RETURN_STATUS_UNEXPECTED(
"Map for storaging image-index pair is nullptr or has been set in other place,"
"[Internal ERROR] Map for containing image-index pair is nullptr or has been set in other place,"
"it must be empty before using GetClassIds.");
}
}
@ -181,7 +181,7 @@ Status ManifestOp::ParseManifestFile() {
std::string image_file_path = js.value("source", "");
if (image_file_path == "") {
file_handle.close();
RETURN_STATUS_UNEXPECTED("Invalid data, source is not found in Manifest file: " + file_ + " at line " +
RETURN_STATUS_UNEXPECTED("Invalid data, 'source' is not found in Manifest file: " + file_ + " at line " +
std::to_string(line_count));
}
// If image is not JPEG/PNG/GIF/BMP, drop it
@ -193,7 +193,7 @@ Status ManifestOp::ParseManifestFile() {
std::string usage = js.value("usage", "");
if (usage == "") {
file_handle.close();
RETURN_STATUS_UNEXPECTED("Invalid data, usage is not found in Manifest file: " + file_ + " at line " +
RETURN_STATUS_UNEXPECTED("Invalid data, 'usage' is not found in Manifest file: " + file_ + " at line " +
std::to_string(line_count));
}
(void)std::transform(usage.begin(), usage.end(), usage.begin(), ::tolower);
@ -208,8 +208,8 @@ Status ManifestOp::ParseManifestFile() {
classes.insert(label_name);
if (label_name == "") {
file_handle.close();
RETURN_STATUS_UNEXPECTED("Invalid data, label name is not found in Manifest file: " + file_ + " at line " +
std::to_string(line_count));
RETURN_STATUS_UNEXPECTED("Invalid data, 'name' of label is not found in Manifest file: " + file_ +
" at line " + std::to_string(line_count));
}
if (class_index_.empty() || class_index_.find(label_name) != class_index_.end()) {
if (label_index_.find(label_name) == label_index_.end()) {
@ -278,7 +278,7 @@ Status ManifestOp::CountDatasetInfo() {
num_rows_ = static_cast<int64_t>(image_labelname_.size());
if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API ManifestDataset. Please check file path: " + file_);
"Invalid data, data file may not be suitable to read with ManifestDataset API. Check file path: " + file_);
}
return Status::OK();
}

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mappable_leaf_op.cc
View File

@ -115,7 +115,7 @@ Status MappableLeafOp::WorkerEntry(int32_t worker_id) {
}
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block));
}
RETURN_STATUS_UNEXPECTED("Unexpected nullptr received in worker");
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Unexpected nullptr received in worker.");
}
} // namespace dataset

11
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mindrecord_op.cc
View File

@ -112,8 +112,9 @@ Status MindRecordOp::Init() {
if (!load_all_cols) {
std::unique_ptr<DataSchema> tmp_schema = std::make_unique<DataSchema>();
for (std::string colname : columns_to_load_) {
CHECK_FAIL_RETURN_UNEXPECTED(colname_to_ind.find(colname) != colname_to_ind.end(),
"Invalid parameter, column name: " + colname + " does not exist.");
CHECK_FAIL_RETURN_UNEXPECTED(
colname_to_ind.find(colname) != colname_to_ind.end(),
"Invalid data, specified loading column name: " + colname + " does not exist in data file.");
RETURN_IF_NOT_OK(tmp_schema->AddColumn(data_schema_->column(colname_to_ind[colname])));
}
data_schema_ = std::move(tmp_schema);
@ -276,7 +277,7 @@ Status MindRecordOp::LoadTensorRow(TensorRow *tensor_row, const std::vector<uint
DataType type = column.type();
// Set shape
CHECK_FAIL_RETURN_UNEXPECTED(column_data_type_size != 0, "The divisor cannot be 0.");
CHECK_FAIL_RETURN_UNEXPECTED(column_data_type_size != 0, "Found memory size of column data type is 0.");
auto num_elements = n_bytes / column_data_type_size;
if (type == DataType::DE_STRING) {
std::string s{data, data + n_bytes};
@ -328,7 +329,9 @@ Status MindRecordOp::CountTotalRows(const std::vector<std::string> dataset_path,
std::unique_ptr<ShardReader> shard_reader = std::make_unique<ShardReader>();
MSRStatus rc = shard_reader->CountTotalRows(dataset_path, load_dataset, op, count, num_padded);
if (rc == MSRStatus::FAILED) {
RETURN_STATUS_UNEXPECTED("Invalid data, MindRecordOp failed to count total rows.");
RETURN_STATUS_UNEXPECTED(
"Invalid data, MindRecordOp failed to count total rows. Check whether there are corresponding .db files "
"and the value of dataset_file parameter is given correctly.");
}
return Status::OK();
}

16
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc
View File

@ -114,10 +114,10 @@ void MnistOp::Print(std::ostream &out, bool show_all) const {
Status MnistOp::GetClassIds(std::map<int32_t, std::vector<int64_t>> *cls_ids) const {
if (cls_ids == nullptr || !cls_ids->empty() || image_label_pairs_.empty()) {
if (image_label_pairs_.empty()) {
RETURN_STATUS_UNEXPECTED("No image found in dataset, please check if Op read images successfully or not.");
RETURN_STATUS_UNEXPECTED("No image found in dataset, try iterate dataset to check if read images success.");
} else {
RETURN_STATUS_UNEXPECTED(
"Map for storaging image-index pair is nullptr or has been set in other place,"
"[Internal ERROR] Map for containing image-index pair is nullptr or has been set in other place,"
"it must be empty before using GetClassIds.");
}
}
@ -192,25 +192,26 @@ Status MnistOp::ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *la
uint32_t num_images, num_labels;
RETURN_IF_NOT_OK(CheckImage(image_names_[index], image_reader, &num_images));
RETURN_IF_NOT_OK(CheckLabel(label_names_[index], label_reader, &num_labels));
CHECK_FAIL_RETURN_UNEXPECTED((num_images == num_labels), "Invalid data, num_images is not equal to num_labels.");
CHECK_FAIL_RETURN_UNEXPECTED((num_images == num_labels),
"Invalid data, num_images is not equal to num_labels. Ensure data file is not damaged.");
// The image size of the Mnist dataset is fixed at [28,28]
int64_t size = kMnistImageRows * kMnistImageCols;
auto images_buf = std::make_unique<char[]>(size * num_images);
auto labels_buf = std::make_unique<char[]>(num_images);
if (images_buf == nullptr || labels_buf == nullptr) {
std::string err_msg = "Failed to allocate memory for MNIST buffer.";
std::string err_msg = "[Internal ERROR] Failed to allocate memory for MNIST buffer.";
MS_LOG(ERROR) << err_msg.c_str();
RETURN_STATUS_UNEXPECTED(err_msg);
}
(void)image_reader->read(images_buf.get(), size * num_images);
if (image_reader->fail()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to read image: " + image_names_[index] +
", size:" + std::to_string(size * num_images));
", size:" + std::to_string(size * num_images) + ". Ensure data file is not damaged.");
}
(void)label_reader->read(labels_buf.get(), num_images);
if (label_reader->fail()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to read label:" + label_names_[index] +
", size: " + std::to_string(num_images));
", size: " + std::to_string(num_images) + ". Ensure data file is not damaged.");
}
TensorShape img_tensor_shape = TensorShape({kMnistImageRows, kMnistImageCols, 1});
for (int64_t j = 0; j != num_images; ++j) {
@ -247,7 +248,8 @@ Status MnistOp::ParseMnistData() {
num_rows_ = image_label_pairs_.size();
if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API MnistDataset. Please check file path or dataset API.");
"Invalid data, data file may not be suitable to read with MnistDataset API. Check file in directory: " +
folder_path_);
}
return Status::OK();
}

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc
View File

@ -112,7 +112,7 @@ void RandomDataOp::GenerateSchema() {
// provide the master loop that drives the logic for performing the work.
Status RandomDataOp::operator()() {
CHECK_FAIL_RETURN_UNEXPECTED(total_rows_ >= num_workers_,
"RandomDataOp expects total_rows < num_workers. total_row=" +
"RandomDataOp expects total_rows < num_workers. Try adjust num_workers, total_row=" +
std::to_string(total_rows_) + ", num_workers=" + std::to_string(num_workers_) + " .");
// If the amount of workers we have exceeds the number of rows to produce, then we'll have
@ -265,7 +265,7 @@ Status RandomDataOp::WorkerEntry(int32_t worker_id) {
// A helper function to create random data for the row
Status RandomDataOp::CreateRandomRow(int32_t worker_id, TensorRow *new_row) {
if (new_row == nullptr) {
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "Missing tensor row output");
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, "[Internal ERROR] Missing tensor row output.");
}
// Create a tensor for each column, then add the tensor to the row

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.cc
View File

@ -53,7 +53,7 @@ Status DistributedSamplerRT::InitSampler() {
CHECK_FAIL_RETURN_UNEXPECTED(num_samples_ > 0, "Invalid parameter, num_samples must be greater than 0, but got " +
std::to_string(num_samples_) + ".\n");
CHECK_FAIL_RETURN_UNEXPECTED(
num_rows_ > 0, "Invalid parameter, num_rows must be greater than 0" + std::to_string(num_rows_) + ".\n");
num_rows_ > 0, "Invalid parameter, num_rows must be greater than 0, but got " + std::to_string(num_rows_) + ".\n");
CHECK_FAIL_RETURN_UNEXPECTED(
device_id_ < num_devices_ && device_id_ >= 0 && num_rows_ > 0 && num_samples_ > 0,
"Invalid parameter, num_shard must be greater than shard_id and greater than 0, got num_shard: " +
@ -87,9 +87,8 @@ Status DistributedSamplerRT::InitSampler() {
Status DistributedSamplerRT::GetNextSample(TensorRow *out) {
if (cnt_ > samples_per_tensor_) {
RETURN_STATUS_UNEXPECTED(
"Number of samples(cnt) that have already been filled in to Tensor should be less than or "
"equal to samples_per_tensor, but got cnt: " +
std::to_string(cnt_) + ", samples_per_tensor: " + std::to_string(samples_per_tensor_));
"Sampler index must be less than or equal to num_samples(total rows in dataset), but got:" +
std::to_string(cnt_) + ", samples_per_tensor(num_samples): " + std::to_string(samples_per_tensor_));
} else if (cnt_ == samples_per_tensor_ && (non_empty_ || !even_dist_)) {
(*out) = TensorRow(TensorRow::kFlagEOE);
if (!samples_per_tensor_) {
@ -150,7 +149,7 @@ Status DistributedSamplerRT::GetNextSample(TensorRow *out) {
}
Status DistributedSamplerRT::ResetSampler() {
CHECK_FAIL_RETURN_UNEXPECTED(cnt_ == samples_per_tensor_, "ERROR Reset() called early/late");
CHECK_FAIL_RETURN_UNEXPECTED(cnt_ == samples_per_tensor_, "[Internal ERROR] Reset() Sampler called early or late.");
cnt_ = 0;
if (shuffle_ == true) {

8
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc
View File

@ -28,7 +28,9 @@ MindRecordSamplerRT::MindRecordSamplerRT(mindrecord::ShardReader *shard_reader,
Status MindRecordSamplerRT::GetNextSample(TensorRow *out) {
if (next_id_ > num_samples_) {
RETURN_STATUS_UNEXPECTED("MindRecordSampler Internal Error");
RETURN_STATUS_UNEXPECTED(
"Sampler index must be less than or equal to num_samples(total rows in dataset), but got: " +
std::to_string(next_id_) + ", num_samples_: " + std::to_string(num_samples_));
} else if (next_id_ == num_samples_) {
(*out) = TensorRow(TensorRow::kFlagEOE);
} else {
@ -51,7 +53,9 @@ Status MindRecordSamplerRT::InitSampler() {
if (!sample_ids_) {
// Note, sample_ids_.empty() is okay and will just give no sample ids.
RETURN_STATUS_UNEXPECTED("ShardReader did not provide a valid sample ids vector via MindRecordSamplerRT");
RETURN_STATUS_UNEXPECTED(
"Init Sampler failed as sample_ids is empty, here ShardReader did not provide a valid sample ids vector via"
" MindRecordSamplerRT");
}
// Usually, the num samples is given from the user interface. In our case, that data is in mindrecord.

10
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/pk_sampler.cc
View File

@ -59,7 +59,7 @@ Status PKSamplerRT::InitSampler() {
std::sort(labels_.begin(), labels_.end());
}
CHECK_FAIL_RETURN_UNEXPECTED(
num_samples_ > 0, "Invalid parameter, num_class or K (num samples per class) must be greater than 0, but got " +
num_samples_ > 0, "Invalid parameter, num_class or num samples per class must be greater than 0, but got " +
std::to_string(num_samples_));
is_initialized = true;
return Status::OK();
@ -67,7 +67,9 @@ Status PKSamplerRT::InitSampler() {
Status PKSamplerRT::GetNextSample(TensorRow *out) {
if (next_id_ > num_samples_ || num_samples_ == 0) {
RETURN_STATUS_UNEXPECTED("Index must be less than or equal to num_samples, but got: " + std::to_string(next_id_));
RETURN_STATUS_UNEXPECTED(
"Sampler index must be less than or equal to num_samples(total rows in dataset), but got: " +
std::to_string(next_id_) + ", num_samplers:" + std::to_string(num_samples_));
} else if (next_id_ == num_samples_) {
(*out) = TensorRow(TensorRow::kFlagEOE);
} else {
@ -79,7 +81,7 @@ Status PKSamplerRT::GetNextSample(TensorRow *out) {
int64_t last_id = (samples_per_tensor_ + next_id_ > num_samples_) ? num_samples_ : samples_per_tensor_ + next_id_;
RETURN_IF_NOT_OK(CreateSamplerTensor(&sample_ids, last_id - next_id_));
auto id_ptr = sample_ids->begin<int64_t>();
CHECK_FAIL_RETURN_UNEXPECTED(samples_per_class_ != 0, "samples cannot be zero.");
CHECK_FAIL_RETURN_UNEXPECTED(samples_per_class_ != 0, "Invalid Parameter, num samples per class can't be zero.");
while (next_id_ < last_id && id_ptr != sample_ids->end<int64_t>()) {
int64_t cls_id = next_id_++ / samples_per_class_;
const std::vector<int64_t> &samples = label_to_ids_[labels_[cls_id]];
@ -100,7 +102,7 @@ Status PKSamplerRT::GetNextSample(TensorRow *out) {
}
Status PKSamplerRT::ResetSampler() {
CHECK_FAIL_RETURN_UNEXPECTED(next_id_ == num_samples_, "ERROR Reset() called early/late");
CHECK_FAIL_RETURN_UNEXPECTED(next_id_ == num_samples_, "[Internal ERROR] Reset() Sampler called early or late.");
next_id_ = 0;
rnd_.seed(seed_++);

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/python_sampler.cc
View File

@ -90,7 +90,7 @@ Status PythonSamplerRT::InitSampler() {
}
Status PythonSamplerRT::ResetSampler() {
CHECK_FAIL_RETURN_UNEXPECTED(need_to_reset_, "ERROR Reset() called not at end of an epoch");
CHECK_FAIL_RETURN_UNEXPECTED(need_to_reset_, "[Internal ERROR] Reset() Sampler called early or late.");
need_to_reset_ = false;
py::gil_scoped_acquire gil_acquire;
if (Py_IsInitialized() == 0) {

7
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/random_sampler.cc
View File

@ -33,7 +33,8 @@ RandomSamplerRT::RandomSamplerRT(bool replacement, int64_t num_samples, bool res
Status RandomSamplerRT::GetNextSample(TensorRow *out) {
if (next_id_ > num_samples_) {
RETURN_STATUS_UNEXPECTED("RandomSampler Internal Error");
RETURN_STATUS_UNEXPECTED("Sampler index must be less than or equal to num_samples(total rows in dataset), but got" +
std::to_string(next_id_) + ", num_samplers:" + std::to_string(num_samples_));
} else if (next_id_ == num_samples_) {
(*out) = TensorRow(TensorRow::kFlagEOE);
} else {
@ -77,7 +78,7 @@ Status RandomSamplerRT::InitSampler() {
}
CHECK_FAIL_RETURN_UNEXPECTED(
num_samples_ > 0 && num_rows_ > 0,
"Invalid parameter, num_samples & num_rows must be greater than 0, but got num_samples: " +
"Invalid parameter, num_samples and num_rows must be greater than 0, but got num_samples: " +
std::to_string(num_samples_) + ", num_rows: " + std::to_string(num_rows_));
samples_per_tensor_ = samples_per_tensor_ > num_samples_ ? num_samples_ : samples_per_tensor_;
rnd_.seed(seed_);
@ -97,7 +98,7 @@ Status RandomSamplerRT::InitSampler() {
}
Status RandomSamplerRT::ResetSampler() {
CHECK_FAIL_RETURN_UNEXPECTED(next_id_ == num_samples_, "ERROR Reset() called early/late");
CHECK_FAIL_RETURN_UNEXPECTED(next_id_ == num_samples_, "[Internal ERROR] Reset() Sampler called early or late.");
next_id_ = 0;
if (reshuffle_each_epoch_) {

12
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sampler.cc
View File

@ -27,7 +27,7 @@ Status RandomAccessOp::GetNumRowsInDataset(int64_t *num) const {
// Here, it is just a getter method to return the value. However, it is invalid if there is
// not a value set for this count, so generate a failure if that is the case.
if (num == nullptr || num_rows_ == -1) {
RETURN_STATUS_UNEXPECTED("RandomAccessOp has not computed its num rows yet.");
RETURN_STATUS_UNEXPECTED("Get num rows in Dataset failed, num_rows has not been set yet.");
}
(*num) = num_rows_;
return Status::OK();
@ -45,7 +45,7 @@ Status SamplerRT::HandshakeRandomAccessOp(const RandomAccessOp *op) {
if (HasChildSampler()) {
child_sampler = std::dynamic_pointer_cast<SamplerRT>(child_[0]);
if (!child_sampler) {
std::string err_msg("Cannot handshake, child is not a sampler object.");
std::string err_msg("[Internal ERROR] Cannot handshake, child is not a sampler object.");
RETURN_STATUS_UNEXPECTED(err_msg);
}
@ -53,7 +53,7 @@ Status SamplerRT::HandshakeRandomAccessOp(const RandomAccessOp *op) {
RETURN_IF_NOT_OK(child_sampler->HandshakeRandomAccessOp(op));
}
CHECK_FAIL_RETURN_UNEXPECTED(op != nullptr, "RandomAccessOp is nullptr\n");
CHECK_FAIL_RETURN_UNEXPECTED(op != nullptr, "RandomAccessOp init failed, as it is nullptr.");
// If there's a child sampler, set the row count to be it's sample count
if (HasChildSampler()) {
@ -99,11 +99,11 @@ Status SamplerRT::GetAllIdsThenReset(py::array *data) {
sample_ids = sample_row[0];
// check this tensorRow is not a ctrl tensorRow
CHECK_FAIL_RETURN_UNEXPECTED(sample_row.Flags() == TensorRow::kFlagNone, "ERROR ctrl row received");
CHECK_FAIL_RETURN_UNEXPECTED(sample_row.Flags() == TensorRow::kFlagNone, "[Internal ERROR] ctrl row received.");
// perform error checking! Next TensorRow supposed to be EOE since last one already contains all ids for current epoch
RETURN_IF_NOT_OK(GetNextSample(&sample_row));
CHECK_FAIL_RETURN_UNEXPECTED(sample_row.eoe(), "ERROR Non EOE received");
CHECK_FAIL_RETURN_UNEXPECTED(sample_row.eoe(), "[Internal ERROR] Non EOE received in the end of epoch.");
// Reset Sampler since this is the end of the epoch
RETURN_IF_NOT_OK(ResetSampler());
@ -176,7 +176,7 @@ bool SamplerRT::HasChildSampler() const { return !child_.empty(); }
Status SamplerRT::GetAssociatedChildId(int64_t *out_associated_id, int64_t id) {
if (child_ids_.empty()) {
RETURN_STATUS_UNEXPECTED("Trying to get associated child id, but there are no child ids!");
RETURN_STATUS_UNEXPECTED("[Internal ERROR] Trying to get associated child id, but there are no child ids!");
}
std::shared_ptr<Tensor> sample_ids = child_ids_[0];

12
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.cc
View File

@ -26,7 +26,9 @@ SequentialSamplerRT::SequentialSamplerRT(int64_t start_index, int64_t num_sample
Status SequentialSamplerRT::GetNextSample(TensorRow *out) {
if (id_count_ > num_samples_) {
RETURN_STATUS_UNEXPECTED("SequentialSampler Internal Error");
RETURN_STATUS_UNEXPECTED(
"Sampler index must be less than or equal to num_samples(total rows in dataset), but got:" +
std::to_string(id_count_) + ", num_samples_: " + std::to_string(num_samples_));
} else if (id_count_ == num_samples_) {
(*out) = TensorRow(TensorRow::kFlagEOE);
} else {
@ -80,9 +82,9 @@ Status SequentialSamplerRT::InitSampler() {
if (num_samples_ == 0 || num_samples_ > available_row_count) {
num_samples_ = available_row_count;
}
CHECK_FAIL_RETURN_UNEXPECTED(
(num_samples_ > 0 && samples_per_tensor_ > 0) || num_samples_ == 0,
"Invalid parameter, samples_per_tensor must be greater than 0, but got " + std::to_string(samples_per_tensor_));
CHECK_FAIL_RETURN_UNEXPECTED((num_samples_ > 0 && samples_per_tensor_ > 0) || num_samples_ == 0,
"Invalid parameter, samples_per_tensor(num_samplers) must be greater than 0, but got " +
std::to_string(samples_per_tensor_));
samples_per_tensor_ = samples_per_tensor_ > num_samples_ ? num_samples_ : samples_per_tensor_;
is_initialized = true;
@ -90,7 +92,7 @@ Status SequentialSamplerRT::InitSampler() {
}
Status SequentialSamplerRT::ResetSampler() {
CHECK_FAIL_RETURN_UNEXPECTED(id_count_ == num_samples_, "ERROR Reset() called early/late");
CHECK_FAIL_RETURN_UNEXPECTED(id_count_ == num_samples_, "[Internal ERROR] Reset() Sampler called early or late.");
current_id_ = start_index_;
id_count_ = 0;

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/weighted_random_sampler.cc
View File

@ -46,7 +46,7 @@ Status WeightedRandomSamplerRT::InitSampler() {
"Invalid parameter, num_samples and num_rows must be greater than 0, but got num_rows: " +
std::to_string(num_rows_) + ", num_samples: " + std::to_string(num_samples_));
CHECK_FAIL_RETURN_UNEXPECTED(samples_per_tensor_ > 0,
"Invalid parameter, samples_per_tensor must be greater than 0, but got " +
"Invalid parameter, samples_per_tensor(num_samples) must be greater than 0, but got " +
std::to_string(samples_per_tensor_) + ".\n");
if (weights_.size() > static_cast<size_t>(num_rows_)) {

7
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/text_file_op.cc
View File

@ -186,8 +186,13 @@ Status TextFileOp::CalculateNumRowsPerShard() {
num_rows_ += count;
}
if (num_rows_ == 0) {
std::stringstream ss;
for (int i = 0; i < text_files_list_.size(); ++i) {
ss << " " << text_files_list_[i];
}
std::string file_list = ss.str();
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API TextFileDataset. Please check file path or dataset API.");
"Invalid data, data file may not be suitable to read with TextFileDataset API. Check file: " + file_list);
}
num_rows_per_shard_ = static_cast<int64_t>(std::ceil(num_rows_ * 1.0 / num_devices_));

19
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/tf_reader_op.cc
View File

@ -119,7 +119,9 @@ Status TFReaderOp::Init() {
total_rows_ = data_schema_->num_rows();
}
if (total_rows_ < 0) {
RETURN_STATUS_UNEXPECTED("Invalid parameter, num_sample or num_row for TFRecordDataset must be greater than 0.");
RETURN_STATUS_UNEXPECTED(
"Invalid parameter, num_samples or num_rows for TFRecordDataset must be greater than 0, but got: " +
std::to_string(total_rows_));
}
// Build the index with our files such that each file corresponds to a key id.
@ -152,8 +154,13 @@ Status TFReaderOp::CalculateNumRowsPerShard() {
}
num_rows_per_shard_ = static_cast<int64_t>(std::ceil(num_rows_ * 1.0 / num_devices_));
if (num_rows_per_shard_ == 0) {
std::stringstream ss;
for (int i = 0; i < dataset_files_list_.size(); ++i) {
ss << " " << dataset_files_list_[i];
}
std::string file_list = ss.str();
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API TFRecordDataset. Please check file path or dataset API.");
"Invalid data, data file may not be suitable to read with TFRecordDataset API. Check file path." + file_list);
}
return Status::OK();
}
@ -367,11 +374,11 @@ Status TFReaderOp::LoadFeature(TensorRow *tensor_row, const dataengine::Feature
break;
}
case dataengine::Feature::KindCase::KIND_NOT_SET: {
std::string err_msg = "Invalid data, tf_file column type must be uint8, int64 or float32.";
std::string err_msg = "Invalid data, column type in tf record file must be uint8, int64 or float32.";
RETURN_STATUS_UNEXPECTED(err_msg);
}
default: {
std::string err_msg = "Invalid data, tf_file column type must be uint8, int64 or float32.";
std::string err_msg = "Invalid data, column type in tf record file must be uint8, int64 or float32.";
RETURN_STATUS_UNEXPECTED(err_msg);
}
}
@ -587,10 +594,10 @@ Status TFReaderOp::CreateSchema(const std::string tf_file, std::vector<std::stri
break;
case dataengine::Feature::KindCase::KIND_NOT_SET:
RETURN_STATUS_UNEXPECTED("Invalid data, tf_file column type must be uint8, int64 or float32.");
RETURN_STATUS_UNEXPECTED("Invalid data, column type of tf record file must be uint8, int64 or float32.");
default:
RETURN_STATUS_UNEXPECTED("Invalid data, tf_file column type must be uint8, int64 or float32.");
RETURN_STATUS_UNEXPECTED("Invalid data, column type of tf record file must be uint8, int64 or float32.");
}
RETURN_IF_NOT_OK(

3
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/voc_op.cc
View File

@ -208,7 +208,8 @@ Status VOCOp::ParseAnnotationIds() {
num_rows_ = image_ids_.size();
if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED(
"Invalid data, no valid data matching the dataset API VOCDataset. Please check file path or dataset API.");
"Invalid data, data file may not be suitable to read with VOCDataset API. Check file in directory:" +
folder_path_);
}
return Status::OK();
}

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/zip_op.cc
View File

@ -107,7 +107,9 @@ Status ZipOp::ComputeColMap() {
int32_t old_id = pair.second;
// check if name already exists in column name descriptor
if (column_name_id_map_.count(name) == 1) {
RETURN_STATUS_UNEXPECTED("Invalid parameter, key: " + name + " already exists when zipping datasets.");
RETURN_STATUS_UNEXPECTED("Invalid parameter, key: " + name +
" already exists when zipping datasets. Check for duplicate key names in different "
"dataset.");
}
column_name_id_map_[name] = old_id + colsCurrent;
}

4
mindspore/dataset/engine/validators.py
View File

@ -461,11 +461,11 @@ def check_pad_info(key, val):
type_check(val, (tuple,), "value in pad_info")
if val[0] is not None:
type_check(val[0], (list,), "pad_shape")
type_check(val[0], (list,), "shape in pad_info")
for dim in val[0]:
if dim is not None:
check_pos_int32(dim, "dim in pad_shape")
check_pos_int32(dim, "dim of shape in pad_info")
if val[1] is not None:
type_check(val[1], (int, float, str, bytes), "pad_value")

4
tests/ut/python/dataset/test_datasets_cifarop.py
View File

@ -417,12 +417,12 @@ def test_cifar_usage():
assert test_config("all") == 10000
assert "usage is not within the valid set of ['train', 'test', 'all']" in test_config("invalid")
assert "Argument usage with value ['list'] is not of type [<class 'str'>]" in test_config(["list"])
assert "no valid data matching the dataset API Cifar10Dataset" in test_config("test")
assert "data file may not suitable to read with Cifar10Dataset API" in test_config("test")
# test the usage of CIFAR10
assert test_config("test", False) == 10000
assert test_config("all", False) == 10000
assert "no valid data matching the dataset API Cifar100Dataset" in test_config("train", False)
assert "data file may not suitable to read with Cifar100Dataset API" in test_config("train", False)
assert "usage is not within the valid set of ['train', 'test', 'all']" in test_config("invalid", False)
# change this directory to the folder that contains all cifar10 files

4
tests/ut/python/dataset/test_datasets_coco.py
View File

@ -300,7 +300,7 @@ def test_coco_case_exception():
pass
assert False
except RuntimeError as e:
assert "invalid node found in json" in str(e)
assert "required node not found in JSON" in str(e)
try:
data1 = ds.CocoDataset(DATA_DIR, annotation_file=INVALID_CATEGORY_ID_FILE, task="Detection")
@ -316,7 +316,7 @@ def test_coco_case_exception():
pass
assert False
except RuntimeError as e:
assert "failed to open json file" in str(e)
assert "failed to open JSON file" in str(e)
try:
sampler = ds.PKSampler(3)

2
tests/ut/python/dataset/test_datasets_generator.py
View File

@ -554,7 +554,7 @@ def test_generator_error_2():
for _ in data1:
pass
print("========", str(info.value))
assert "Generator should return a tuple of numpy arrays" in str(info.value)
assert "Generator should return a tuple of NumPy arrays" in str(info.value)
def test_generator_error_3():

4
tests/ut/python/dataset/test_datasets_manifestop.py
View File

@ -167,7 +167,7 @@ def test_manifest_dataset_exception():
pass
assert False
except RuntimeError as e:
assert "Invalid data, source is not found in Manifest file" in str(e)
assert "Invalid data, 'source' is not found in Manifest file" in str(e)
NO_USAGE_DATA_FILE = "../data/dataset/testManifestData/invalidNoUsage.manifest"
try:
@ -176,7 +176,7 @@ def test_manifest_dataset_exception():
pass
assert False
except RuntimeError as e:
assert "Invalid data, usage is not found in Manifest file" in str(e)
assert "Invalid data, 'usage' is not found in Manifest file" in str(e)
if __name__ == '__main__':

2
tests/ut/python/dataset/test_datasets_mnist.py
View File

@ -270,7 +270,7 @@ def test_mnist_usage():
assert test_config("test") == 10000
assert test_config("all") == 10000
assert " no valid data matching the dataset API MnistDataset" in test_config("train")
assert "data file may not be suitable to read with MnistDataset API" in test_config("train")
assert "usage is not within the valid set of ['train', 'test', 'all']" in test_config("invalid")
assert "Argument usage with value ['list'] is not of type [<class 'str'>]" in test_config(["list"])

2
tests/ut/python/dataset/test_var_batch_map.py
View File

@ -369,7 +369,7 @@ def test_multi_col_map():
# test exceptions
assert "output_columns with value 233 is not of type" in batch_map_config(2, 2, split_col, ["col2"], 233)
assert "column_order with value 233 is not of type" in batch_map_config(2, 2, split_col, ["col2"], ["col1"], 233)
assert "output_columns is NOT set correctly" in batch_map_config(2, 2, split_col, ["col2"], ["col1"])
assert "output_columns in batch is not set correctly" in batch_map_config(2, 2, split_col, ["col2"], ["col1"])
assert "Incorrect number of columns" in batch_map_config(2, 2, split_col, ["col2"], ["col3", "col4", "col5"])
assert "col-1 doesn't exist" in batch_map_config(2, 2, split_col, ["col-1"], ["col_x", "col_y"])