Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!22909 sync code to gitee according code review 

Merge pull request !22909 from guozhijian/sync_code_review
This commit is contained in:
i-robot 2021-09-08 08:26:58 +00:00 committed by Gitee
parent 77424eaad5 4b413922ac
commit 4cab7ffbd3
86 changed files with 544 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
mindspore/ccsrc/minddata/dataset/api/datasets.cc
View File

@ -176,7 +176,9 @@ std::shared_ptr<PullIterator> Dataset::CreatePullBasedIterator(std::vector<std::
std::shared_ptr<PullIterator> iter = std::make_shared<PullIterator>();
Status rc = iter->BuildAndLaunchTree(ds);
if (rc.IsError()) MS_LOG(ERROR) << "CreateIterator: Iterator exception caught: " << rc;
if (rc.IsError()) {
MS_LOG(ERROR) << "CreateIterator: Iterator exception caught: " << rc;
}
RETURN_SECOND_IF_ERROR(rc, nullptr);
return iter;
}
@ -203,7 +205,12 @@ bool Dataset::DeviceQueueCharIF(const std::vector<char> &queue_name, const std::
// Get ToDevice consumer
auto consumer = std::make_unique<ToDevice>(num_epochs);
ToDevice *consumer_ = consumer.get();
ToDevice *consumer_ptr = consumer.get();
if (consumer_ptr == nullptr) {
MS_LOG(ERROR) << "ToDevice: Failed to get consumer.";
return false;
}
rc = consumer->Init(ds);
if (rc.IsError()) {
MS_LOG(ERROR) << "ToDevice: Failed to init. Error status: " << rc;
@ -212,7 +219,7 @@ bool Dataset::DeviceQueueCharIF(const std::vector<char> &queue_name, const std::
runtime_context->AssignConsumer(std::move(consumer));
// Send data to device
rc = consumer_->Send();
rc = consumer_ptr->Send();
if (rc.IsError()) {
MS_LOG(ERROR) << "ToDevice: Failed to send data to device. Error status: " << rc;
return false;
@ -241,16 +248,22 @@ bool Dataset::SaveCharIF(const std::vector<char> &dataset_path, int32_t num_file
MS_LOG(ERROR) << "CreateSaver failed." << rc;
return false;
}
SaveToDisk *consumer_ = consumer.get();
SaveToDisk *consumer_ptr = consumer.get();
if (consumer_ptr == nullptr) {
MS_LOG(ERROR) << "ToDevice: Failed to get consumer.";
return false;
}
rc = consumer->Init(ds->IRNode());
if (rc.IsError()) {
MS_LOG(ERROR) << "CreateSaver failed." << rc;
return false;
}
runtime_context->AssignConsumer(std::move(consumer));
// Save data into file
rc = consumer_->Save();
rc = consumer_ptr->Save();
if (rc.IsError()) {
MS_LOG(ERROR) << "Saver: Failed to save data into file. Error status: " << rc;
return false;
@ -411,7 +424,9 @@ ConcatDataset::ConcatDataset(const std::vector<std::shared_ptr<Dataset>> &datase
FilterDataset::FilterDataset(std::shared_ptr<Dataset> input, std::function<MSTensorVec(MSTensorVec)> predicate,
const std::vector<std::vector<char>> &input_columns) {
std::shared_ptr<TensorOp> c_func = nullptr;
if (predicate) c_func = std::make_shared<CFuncOp>(std::bind(FuncPtrConverter, predicate, std::placeholders::_1));
if (predicate) {
c_func = std::make_shared<CFuncOp>(std::bind(FuncPtrConverter, predicate, std::placeholders::_1));
}
auto ds = std::make_shared<FilterNode>(input->IRNode(), c_func, VectorCharToString(input_columns));
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);

14
mindspore/ccsrc/minddata/dataset/api/iterator.cc
View File

@ -31,6 +31,7 @@ Status Iterator::GetNextRowCharIF(MSTensorMapChar *row) {
// Clean data buffer
row->clear();
std::unordered_map<std::string, std::shared_ptr<dataset::Tensor>> md_map;
CHECK_FAIL_RETURN_UNEXPECTED(consumer_ != nullptr, "consumer_ is null, pls launch iterator first.");
Status rc = consumer_->GetNextAsMap(&md_map);
if (rc.IsError()) {
MS_LOG(ERROR) << "GetNextRow: Failed to get next row. Error status: " << rc;
@ -52,6 +53,7 @@ Status Iterator::GetNextRow(MSTensorVec *row) {
row->clear();
// create a dataset tensor row and fetch. Then we convert the output to MSTensor
std::vector<std::shared_ptr<dataset::Tensor>> md_row;
CHECK_FAIL_RETURN_UNEXPECTED(consumer_ != nullptr, "consumer_ is null, pls launch iterator first.");
Status rc = consumer_->GetNextAsVector(&md_row);
if (rc.IsError()) {
row->clear();
@ -75,8 +77,10 @@ void Iterator::Stop() {
// Function to build and launch the execution tree.
Status Iterator::BuildAndLaunchTree(std::shared_ptr<Dataset> ds, int32_t num_epochs) {
runtime_context_ = std::make_unique<NativeRuntimeContext>();
CHECK_FAIL_RETURN_UNEXPECTED(runtime_context_ != nullptr, "Create runtime_context_ failed.");
RETURN_IF_NOT_OK(runtime_context_->Init());
auto consumer = std::make_unique<IteratorConsumer>(num_epochs);
CHECK_FAIL_RETURN_UNEXPECTED(consumer != nullptr, "Create consumer failed.");
consumer_ = consumer.get();
RETURN_IF_NOT_OK(consumer->Init(ds->IRNode()));
runtime_context_->AssignConsumer(std::move(consumer));
@ -84,9 +88,11 @@ Status Iterator::BuildAndLaunchTree(std::shared_ptr<Dataset> ds, int32_t num_epo
}
PullIterator::PullIterator() : pull_consumer_(nullptr) {}
// Get the next row from the data pipeline.
Status PullIterator::GetRows(int32_t num_rows, std::vector<MSTensorVec> *const row) {
RETURN_UNEXPECTED_IF_NULL(row);
CHECK_FAIL_RETURN_UNEXPECTED(pull_consumer_ != nullptr, "Consumer is nullptr. Please launch iterator fist.");
for (int i = 0; i < num_rows; i++) {
std::vector<std::shared_ptr<dataset::Tensor>> md_row;
Status rc = pull_consumer_->GetNextAsVector(&md_row);
@ -129,7 +135,10 @@ Status PullIterator::GetNextRow(MSTensorVec *const row) {
// for the tree, the reason why this is the case is due to the fact that PullBasedIterator does not need
// to instantiate threads for each op. As such, the call to the consumer will by pass the execution tree.
Status PullIterator::BuildAndLaunchTree(std::shared_ptr<Dataset> ds) {
if (pull_consumer_ == nullptr) pull_consumer_ = std::make_unique<PullBasedIteratorConsumer>();
if (pull_consumer_ == nullptr) {
pull_consumer_ = std::make_unique<PullBasedIteratorConsumer>();
}
CHECK_FAIL_RETURN_UNEXPECTED(pull_consumer_ != nullptr, "pull_consumer_ is nullptr");
RETURN_IF_NOT_OK(pull_consumer_->Init(std::move(ds->IRNode())));
return Status::OK();
}
@ -137,6 +146,9 @@ Status PullIterator::BuildAndLaunchTree(std::shared_ptr<Dataset> ds) {
Iterator::_Iterator::_Iterator(Iterator *lt) : ind_{0}, lt_{lt}, cur_row_{nullptr} {
if (lt_) {
cur_row_ = new MSTensorMap();
if (cur_row_ == nullptr) {
return;
}
Status rc = lt_->GetNextRow(cur_row_);
if (rc.IsError()) {
MS_LOG(ERROR) << "Error getting next row. Message: " << rc;

12
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/callback/bindings.cc
View File

@ -26,12 +26,12 @@ namespace dataset {
PYBIND_REGISTER(PyDSCallback, 0, ([](const py::module *m) {
(void)py::class_<PyDSCallback, std::shared_ptr<PyDSCallback>>(*m, "PyDSCallback")
.def(py::init<int32_t>())
.def("set_begin", &PyDSCallback::setBegin)
.def("set_end", &PyDSCallback::setEnd)
.def("set_epoch_begin", &PyDSCallback::setEpochBegin)
.def("set_epoch_end", &PyDSCallback::setEpochEnd)
.def("set_step_begin", &PyDSCallback::setStepBegin)
.def("set_step_end", &PyDSCallback::setStepEnd);
.def("set_begin", &PyDSCallback::SetBegin)
.def("set_end", &PyDSCallback::SetEnd)
.def("set_epoch_begin", &PyDSCallback::SetEpochBegin)
.def("set_epoch_end", &PyDSCallback::SetEpochEnd)
.def("set_step_begin", &PyDSCallback::SetStepBegin)
.def("set_step_end", &PyDSCallback::SetStepEnd);
}));
PYBIND_REGISTER(CallbackParam, 0, ([](const py::module *m) {

12
mindspore/ccsrc/minddata/dataset/api/python/pybind_conversion.cc
View File

@ -196,9 +196,15 @@ std::shared_ptr<DatasetCache> toDatasetCache(std::shared_ptr<CacheClient> cc) {
}
ShuffleMode toShuffleMode(const int32_t shuffle) {
if (shuffle == 0) return ShuffleMode::kFalse;
if (shuffle == 1) return ShuffleMode::kFiles;
if (shuffle == 2) return ShuffleMode::kGlobal;
if (shuffle == 0) {
return ShuffleMode::kFalse;
}
if (shuffle == 1) {
return ShuffleMode::kFiles;
}
if (shuffle == 2) {
return ShuffleMode::kGlobal;
}
return ShuffleMode();
}

4
mindspore/ccsrc/minddata/dataset/api/vision.cc
View File

@ -668,7 +668,7 @@ std::shared_ptr<TensorOperation> RandomResizedCropWithBBox::Parse() {
struct RandomRotation::Data {
Data(const std::vector<float> &degrees, InterpolationMode resample, bool expand, const std::vector<float> &center,
const std::vector<uint8_t> &fill_value)
: degrees_(degrees), interpolation_mode_(resample), expand_(expand), center_(center), fill_value_(fill_value) {}
: degrees_(degrees), interpolation_mode_(resample), center_(center), expand_(expand), fill_value_(fill_value) {}
std::vector<float> degrees_;
InterpolationMode interpolation_mode_;
std::vector<float> center_;
@ -852,7 +852,7 @@ std::shared_ptr<TensorOperation> ResizePreserveAR::Parse() {
struct Rotate::Data {
Data(const float &degrees, InterpolationMode resample, bool expand, const std::vector<float> &center,
const std::vector<uint8_t> &fill_value)
: degrees_(degrees), interpolation_mode_(resample), expand_(expand), center_(center), fill_value_(fill_value) {}
: degrees_(degrees), interpolation_mode_(resample), center_(center), expand_(expand), fill_value_(fill_value) {}
explicit Data(const FixRotationAngle &angle_id) : angle_id_(angle_id), lite_impl_(true) {}
FixRotationAngle angle_id_{FixRotationAngle::k0Degree};
bool lite_impl_{false};

12
mindspore/ccsrc/minddata/dataset/callback/py_ds_callback.cc
View File

@ -61,27 +61,27 @@ Status PyDSCallback::ExecutePyfunc(py::function f, const CallbackParam &cb_param
}
return Status::OK();
}
void PyDSCallback::setBegin(const py::function &f) {
void PyDSCallback::SetBegin(const py::function &f) {
begin_func_ = f;
begin_needed_ = true;
}
void PyDSCallback::setEnd(const py::function &f) {
void PyDSCallback::SetEnd(const py::function &f) {
end_func_ = f;
end_needed_ = true;
}
void PyDSCallback::setEpochBegin(const py::function &f) {
void PyDSCallback::SetEpochBegin(const py::function &f) {
epoch_begin_func_ = f;
epoch_begin_needed_ = true;
}
void PyDSCallback::setEpochEnd(const py::function &f) {
void PyDSCallback::SetEpochEnd(const py::function &f) {
epoch_end_func_ = f;
epoch_end_needed_ = true;
}
void PyDSCallback::setStepBegin(const py::function &f) {
void PyDSCallback::SetStepBegin(const py::function &f) {
step_begin_func_ = f;
step_begin_needed_ = true;
}
void PyDSCallback::setStepEnd(const py::function &f) {
void PyDSCallback::SetStepEnd(const py::function &f) {
step_end_func_ = f;
step_end_needed_ = true;
}

12
mindspore/ccsrc/minddata/dataset/callback/py_ds_callback.h
View File

@ -44,12 +44,12 @@ class PyDSCallback : public DSCallback {
~PyDSCallback() = default;
void setBegin(const py::function &f);
void setEnd(const py::function &f);
void setEpochBegin(const py::function &f);
void setEpochEnd(const py::function &f);
void setStepBegin(const py::function &f);
void setStepEnd(const py::function &f);
void SetBegin(const py::function &f);
void SetEnd(const py::function &f);
void SetEpochBegin(const py::function &f);
void SetEpochEnd(const py::function &f);
void SetStepBegin(const py::function &f);
void SetStepEnd(const py::function &f);
/// \brief actual callback function for begin, needs to be overridden in the derived class
/// \param cb_param, callback parameter passed in from DatasetOp when calling the callback

2
mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin.cc vendored
View File

@ -51,7 +51,7 @@ int main(int argc, char **argv) {
if (argc == 1) {
args.Help();
return 0;
return 1;
}
// ingest all the args into a string stream for parsing

15
mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc vendored
View File

@ -374,16 +374,23 @@ Status CacheAdminArgHandler::Validate() {
// Additional checks here
auto max_num_workers = std::max<int32_t>(std::thread::hardware_concurrency(), kMaxNumWorkers);
if (used_args_[ArgValue::kArgNumWorkers] && (num_workers_ < 1 || num_workers_ > max_num_workers))
if (used_args_[ArgValue::kArgNumWorkers] && (num_workers_ < 1 || num_workers_ > max_num_workers)) {
// Check the value of num_workers only if it's provided by users.
return Status(StatusCode::kMDSyntaxError,
"Number of workers must be in range of 1 and " + std::to_string(max_num_workers) + ".");
if (log_level_ < MsLogLevel::DEBUG || log_level_ > MsLogLevel::EXCEPTION)
}
if (log_level_ < MsLogLevel::DEBUG || log_level_ > MsLogLevel::EXCEPTION) {
return Status(StatusCode::kMDSyntaxError, "Log level must be in range (0..4).");
if (memory_cap_ratio_ <= 0 || memory_cap_ratio_ > 1)
}
if (memory_cap_ratio_ <= 0 || memory_cap_ratio_ > 1) {
return Status(StatusCode::kMDSyntaxError, "Memory cap ratio should be positive and no greater than 1");
if (port_ < kMinLegalPort || port_ > kMaxLegalPort)
}
if (port_ < kMinLegalPort || port_ > kMaxLegalPort) {
return Status(StatusCode::kMDSyntaxError, "Port must be in range (1025..65535).");
}
return Status::OK();
}

4
mindspore/ccsrc/minddata/dataset/engine/cache/cache_client.cc vendored
View File

@ -56,9 +56,9 @@ Status CacheClient::Builder::SanityCheck() {
// Constructor
CacheClient::CacheClient(session_id_type session_id, uint64_t cache_mem_sz, bool spill, std::string hostname,
int32_t port, int32_t num_connections, int32_t prefetch_size)
: server_connection_id_(0),
cache_mem_sz_(cache_mem_sz),
: cache_mem_sz_(cache_mem_sz),
spill_(spill),
server_connection_id_(0),
client_id_(-1),
local_bypass_(false),
num_connections_(num_connections),

14
mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_client.cc vendored
View File

@ -73,7 +73,7 @@ Status CacheClientGreeter::DoServiceStop() {
void *tag;
while (cq_.Next(&tag, &success)) {
auto r = reinterpret_cast<CacheClientRequestTag *>(tag);
(void)req_.erase(r->seqNo_);
(void)req_.erase(r->seq_no_);
}
}
return Status::OK();
@ -82,8 +82,8 @@ Status CacheClientGreeter::DoServiceStop() {
Status CacheClientGreeter::HandleRequest(std::shared_ptr<BaseRequest> rq) {
// If there is anything extra we need to do before we send.
RETURN_IF_NOT_OK(rq->Prepare());
auto seqNo = request_cnt_.fetch_add(1);
auto tag = std::make_unique<CacheClientRequestTag>(std::move(rq), seqNo);
auto seq_no = request_cnt_.fetch_add(1);
auto tag = std::make_unique<CacheClientRequestTag>(std::move(rq), seq_no);
// One minute timeout
auto deadline = std::chrono::system_clock::now() + std::chrono::seconds(kRequestTimeoutDeadlineInSec);
tag->ctx_.set_deadline(deadline);
@ -93,7 +93,7 @@ Status CacheClientGreeter::HandleRequest(std::shared_ptr<BaseRequest> rq) {
// Insert it into the map.
{
std::unique_lock<std::mutex> lck(mux_);
auto r = req_.emplace(seqNo, std::move(tag));
auto r = req_.emplace(seq_no, std::move(tag));
if (!r.second) {
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__);
}
@ -133,9 +133,9 @@ Status CacheClientGreeter::WorkerEntry() {
{
// We can now free the memory
std::unique_lock<std::mutex> lck(mux_);
auto seqNo = rq->seqNo_;
auto n = req_.erase(seqNo);
CHECK_FAIL_RETURN_UNEXPECTED(n == 1, "Sequence " + std::to_string(seqNo) + " not found");
auto seq_no = rq->seq_no_;
auto n = req_.erase(seq_no);
CHECK_FAIL_RETURN_UNEXPECTED(n == 1, "Sequence " + std::to_string(seq_no) + " not found");
}
} else if (r == grpc_impl::CompletionQueue::NextStatus::TIMEOUT) {
// If we are interrupted, exit. Otherwise wait again.

4
mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_client.h vendored
View File

@ -39,7 +39,7 @@ class CacheClientRequestTag {
public:
friend class CacheClientGreeter;
explicit CacheClientRequestTag(std::shared_ptr<BaseRequest> rq, int64_t seqNo)
: base_rq_(std::move(rq)), seqNo_(seqNo) {}
: base_rq_(std::move(rq)), seq_no_(seqNo) {}
~CacheClientRequestTag() = default;
/// \brief Notify the client that a result has come back from the server
@ -50,7 +50,7 @@ class CacheClientRequestTag {
grpc::Status rc_;
grpc::ClientContext ctx_;
std::unique_ptr<grpc::ClientAsyncResponseReader<CacheReply>> rpc_;
int64_t seqNo_;
int64_t seq_no_;
};
/// \brief A GRPC layer to convert BaseRequest into protobuf and send to the cache server using gRPC

10
mindspore/ccsrc/minddata/dataset/engine/cache/cache_hw.cc vendored
View File

@ -123,14 +123,14 @@ Status CacheServerHW::GetNumaNodeInfo() {
numa_id_t numa_node = static_cast<numa_id_t>(strtol(node_dir.data() + strlen(kNodeName), nullptr, kDecimal));
Path f = p / kCpuList;
auto realpath = Common::GetRealPath(f.toString());
auto realpath = Common::GetRealPath(f.ToString());
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << f.toString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + f.toString());
MS_LOG(ERROR) << "Get real path failed, path=" << f.ToString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + f.ToString());
}
std::ifstream fs(realpath.value());
CHECK_FAIL_RETURN_UNEXPECTED(!fs.fail(), "Fail to open file: " + f.toString());
CHECK_FAIL_RETURN_UNEXPECTED(!fs.fail(), "Fail to open file: " + f.ToString());
std::string cpu_string;
cpu_set_t cpuset;
CPU_ZERO(&cpuset);
@ -155,7 +155,7 @@ Status CacheServerHW::GetNumaNodeInfo() {
++iter;
}
}
CHECK_FAIL_RETURN_UNEXPECTED(!fs.bad(), "Fail to read file: " + f.toString());
CHECK_FAIL_RETURN_UNEXPECTED(!fs.bad(), "Fail to read file: " + f.ToString());
fs.close();
// Remember which cpu is attached to this numa node.
numa_cpuset_.emplace(numa_node, cpuset);

6
mindspore/ccsrc/minddata/dataset/engine/cache/cache_pool.cc vendored
View File

@ -32,13 +32,13 @@ CachePool::CachePool(std::shared_ptr<NumaMemoryPool> mp, const std::string &root
Status CachePool::DoServiceStart() {
tree_ = std::make_shared<data_index>();
// If we are given a disk path, set up the StorageManager
if (!root_.toString().empty()) {
if (!root_.ToString().empty()) {
Path spill = GetSpillPath();
RETURN_IF_NOT_OK(spill.CreateDirectories());
auto &cs = CacheServer::GetInstance();
sm_ = std::make_shared<StorageManager>(spill, cs.GetNumWorkers());
RETURN_IF_NOT_OK(sm_->ServiceStart());
MS_LOG(INFO) << "CachePool will use disk folder: " << spill.toString();
MS_LOG(INFO) << "CachePool will use disk folder: " << spill.ToString();
}
return Status::OK();
}
@ -60,7 +60,7 @@ Status CachePool::DoServiceStop() {
// release each buffer in the DataLocator one by one.
tree_.reset();
if (!root_.toString().empty()) {
if (!root_.ToString().empty()) {
Path spill = GetSpillPath();
auto it = Path::DirIterator::OpenDirectory(&spill);
while (it->HasNext()) {

2
mindspore/ccsrc/minddata/dataset/engine/cache/storage_manager.cc vendored
View File

@ -41,7 +41,7 @@ Status StorageManager::AddOneContainer(int replaced_container_pos) {
const std::string kSuffix = "LB";
Path container_name = root_ / ConstructFileName(kPrefix, file_id_, kSuffix);
std::shared_ptr<StorageContainer> sc;
RETURN_IF_NOT_OK(StorageContainer::CreateStorageContainer(&sc, container_name.toString()));
RETURN_IF_NOT_OK(StorageContainer::CreateStorageContainer(&sc, container_name.ToString()));
containers_.push_back(sc);
file_id_++;
if (replaced_container_pos >= 0) {

3
mindspore/ccsrc/minddata/dataset/engine/cache/stub/cache_grpc_client.h vendored
View File

@ -37,9 +37,6 @@ class CacheClientGreeter : public Service {
Status AttachToSharedMemory(bool *local_bypass) { RETURN_STATUS_UNEXPECTED("Not supported"); }
std::string GetHostname() const { return "Not supported"; }
int32_t GetPort() const { return 0; }
protected:
private:
};
} // namespace dataset
} // namespace mindspore

5
mindspore/ccsrc/minddata/dataset/engine/consumers/pull_based_tree_consumer.h
View File

@ -42,8 +42,7 @@ class PullBasedIteratorConsumer {
/// \brief Returns the next row in a vector format
/// \note This is currently a placeholder function
/// \param[in] num_rows the number of rows that we want to get
/// \param[out] out std::vector of TensorRows
/// \return Status error code
/// \return out std::vector of TensorRows
std::vector<TensorRow> GetRows(int64_t num_rows);
/// Returns the next row in a vector format
@ -57,7 +56,7 @@ class PullBasedIteratorConsumer {
Status GetNextAsMap(std::unordered_map<std::string, TensorPtr> *out);
/// Returns the next row in as a vector
/// \param[out] out std::vector of pairs of string to Tensor
/// \param[out] vec std::vector of pairs of string to Tensor
/// \return Status error code
Status GetNextAsOrderedPair(std::vector<std::pair<std::string, std::shared_ptr<Tensor>>> *vec);

45
mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.cc
View File

@ -432,12 +432,16 @@ Status SaveToDisk::FetchFloatData(std::shared_ptr<Tensor> tensor, std::string co
std::unique_ptr<float> data, dummy;
s = TransformTensor(tensor->GetBuffer(), tensor->shape(), tensor->Size(), &data, data_ptr, &dummy);
RETURN_IF_NOT_OK(s);
if (data != nullptr) (*row_raw_data)[column_name] = std::move(*data);
if (data != nullptr) {
(*row_raw_data)[column_name] = std::move(*data);
}
} else if (column_type == DataType::DE_FLOAT64) {
std::unique_ptr<double> data, dummy;
s = TransformTensor(tensor->GetBuffer(), tensor->shape(), tensor->Size(), &data, data_ptr, &dummy);
RETURN_IF_NOT_OK(s);
if (data != nullptr) (*row_raw_data)[column_name] = std::move(*data);
if (data != nullptr) {
(*row_raw_data)[column_name] = std::move(*data);
}
}
return Status::OK();
}
@ -455,40 +459,54 @@ Status SaveToDisk::FetchItemData(std::shared_ptr<Tensor> tensor, std::string col
std::unique_ptr<int8_t> dummy;
s = TransformTensor(tensor->GetBuffer(), tensor->shape(), tensor->Size(), &data, &data_ptr, &dummy, true);
RETURN_IF_NOT_OK(s);
if (data != nullptr) (*row_raw_data)[column_name] = std::move(*data);
if (data != nullptr) {
(*row_raw_data)[column_name] = std::move(*data);
}
} else if (column_type == DataType::DE_INT16) {
std::unique_ptr<int32_t> data;
std::unique_ptr<int16_t> dummy;
s = TransformTensor(tensor->GetBuffer(), tensor->shape(), tensor->Size(), &data, &data_ptr, &dummy, true);
RETURN_IF_NOT_OK(s);
if (data != nullptr) (*row_raw_data)[column_name] = std::move(*data);
if (data != nullptr) {
(*row_raw_data)[column_name] = std::move(*data);
}
} else if (column_type == DataType::DE_UINT16) {
std::unique_ptr<int32_t> data;
std::unique_ptr<uint16_t> dummy;
s = TransformTensor(tensor->GetBuffer(), tensor->shape(), tensor->Size(), &data, &data_ptr, &dummy, true);
RETURN_IF_NOT_OK(s);
if (data != nullptr) (*row_raw_data)[column_name] = std::move(*data);
if (data != nullptr) {
(*row_raw_data)[column_name] = std::move(*data);
}
} else if (column_type == DataType::DE_UINT8) {
std::unique_ptr<uint8_t> data, dummy;
s = TransformTensor(tensor->GetBuffer(), tensor->shape(), tensor->Size(), &data, &data_ptr, &dummy);
RETURN_IF_NOT_OK(s);
if (data != nullptr) (*row_raw_data)[column_name] = std::move(*data);
if (data != nullptr) {
(*row_raw_data)[column_name] = std::move(*data);
}
} else if (column_type == DataType::DE_INT32) {
std::unique_ptr<int32_t> data, dummy;
s = TransformTensor(tensor->GetBuffer(), tensor->shape(), tensor->Size(), &data, &data_ptr, &dummy);
RETURN_IF_NOT_OK(s);
if (data != nullptr) (*row_raw_data)[column_name] = std::move(*data);
if (data != nullptr) {
(*row_raw_data)[column_name] = std::move(*data);
}
} else if (column_type == DataType::DE_UINT32) {
std::unique_ptr<int64_t> data;
std::unique_ptr<uint32_t> dummy;
s = TransformTensor(tensor->GetBuffer(), tensor->shape(), tensor->Size(), &data, &data_ptr, &dummy, true);
RETURN_IF_NOT_OK(s);
if (data != nullptr) (*row_raw_data)[column_name] = std::move(*data);
if (data != nullptr) {
(*row_raw_data)[column_name] = std::move(*data);
}
} else if (column_type == DataType::DE_INT64) {
std::unique_ptr<int64_t> data, dummy;
s = TransformTensor(tensor->GetBuffer(), tensor->shape(), tensor->Size(), &data, &data_ptr, &dummy);
RETURN_IF_NOT_OK(s);
if (data != nullptr) (*row_raw_data)[column_name] = std::move(*data);
if (data != nullptr) {
(*row_raw_data)[column_name] = std::move(*data);
}
} else if (column_type == DataType::DE_FLOAT32 || column_type == DataType::DE_FLOAT64) {
s = FetchFloatData(tensor, column_name, row_raw_data, &data_ptr);
RETURN_IF_NOT_OK(s);
@ -647,9 +665,14 @@ Status TreeGetters::GetClassIndexing(std::vector<std::pair<std::string, std::vec
}
Status TreeGetters::InternalInit() {
if (init_flag_) return Status::OK();
if (init_flag_) {
return Status::OK();
}
Status s = tree_adapter_->Compile(std::move(root_), 1);
if (s.IsOk()) init_flag_ = true;
if (s.IsOk()) {
init_flag_ = true;
}
return s;
}

9
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/album_op.cc
View File

@ -73,9 +73,9 @@ Status AlbumOp::PrescanEntry() {
while (dirItr->HasNext()) {
Path file = dirItr->Next();
if (extensions_.empty() || extensions_.find(file.Extension()) != extensions_.end()) {
(void)image_rows_.push_back(file.toString().substr(dirname_offset_));
(void)image_rows_.push_back(file.ToString().substr(dirname_offset_));
} else {
MS_LOG(INFO) << "Album operator unsupported file found: " << file.toString()
MS_LOG(INFO) << "Album operator unsupported file found: " << file.ToString()
<< ", extension: " << file.Extension() << ".";
}
}
@ -334,6 +334,7 @@ Status AlbumOp::LoadTensorRow(row_id_type row_id, TensorRow *row) {
// loop over each column descriptor, this can optimized by switch cases
for (int32_t i = 0; i < columns; i++) {
file_handle.close();
RETURN_IF_NOT_OK(loadColumnData(file, i, js, row));
}
} catch (const std::exception &err) {
@ -448,7 +449,9 @@ Status AlbumOp::ComputeColMap() {
}
Status AlbumOp::GetNextRowPullMode(TensorRow *const row) {
if (image_rows_.empty()) RETURN_IF_NOT_OK(PrescanEntry());
if (image_rows_.empty()) {
RETURN_IF_NOT_OK(PrescanEntry());
}
if (sample_ids_ == nullptr) {
RETURN_IF_NOT_OK(this->InitSampler());
TensorRow sample_row;

20
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.cc
View File

@ -71,20 +71,20 @@ Status CelebAOp::ParseAttrFile() {
TaskManager::FindMe()->Post();
Path folder_path(folder_path_);
auto realpath = Common::GetRealPath((folder_path / "list_attr_celeba.txt").toString());
auto realpath = Common::GetRealPath((folder_path / "list_attr_celeba.txt").ToString());
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_attr_celeba.txt").toString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_attr_celeba.txt").toString());
MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_attr_celeba.txt").ToString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_attr_celeba.txt").ToString());
}
std::ifstream attr_file(realpath.value());
if (!attr_file.is_open()) {
std::string attr_file_name = (folder_path / "list_attr_celeba.txt").toString();
std::string attr_file_name = (folder_path / "list_attr_celeba.txt").ToString();
return Status(StatusCode::kMDFileNotExist, __LINE__, __FILE__,
"Invalid file, failed to open Celeba attr file: " + attr_file_name);
}
attr_file_ = (folder_path / "list_attr_celeba.txt").toString();
attr_file_ = (folder_path / "list_attr_celeba.txt").ToString();
const auto PushBackToQueue = [this](std::vector<std::string> &vec, std::ifstream &attr_file,
std::ifstream &partition_file) {
Status s = attr_info_queue_->EmplaceBack(vec);
@ -135,7 +135,7 @@ Status CelebAOp::ParseAttrFile() {
bool CelebAOp::CheckDatasetTypeValid() {
if (!partition_file_.is_open()) {
Path folder_path(folder_path_);
partition_file_.open((folder_path / "list_eval_partition.txt").toString());
partition_file_.open((folder_path / "list_eval_partition.txt").ToString());
if (!partition_file_.is_open()) {
MS_LOG(ERROR) << "Celeba partition file does not exist!";
return false;
@ -186,7 +186,7 @@ Status CelebAOp::ParseImageAttrInfo() {
Path path(folder_path_);
Path file_path = path / split[0];
if (!extensions_.empty() && extensions_.find(file_path.Extension()) == extensions_.end()) {
MS_LOG(WARNING) << "Unsupported file found at " << file_path.toString().c_str() << ", its extension is "
MS_LOG(WARNING) << "Unsupported file found at " << file_path.ToString().c_str() << ", its extension is "
<< file_path.Extension().c_str() << ".";
continue;
}
@ -247,12 +247,12 @@ Status CelebAOp::LoadTensorRow(row_id_type row_id, TensorRow *row) {
Path path(folder_path_);
Path image_path = path / image_label.first;
RETURN_IF_NOT_OK(Tensor::CreateFromFile(image_path.toString(), &image));
RETURN_IF_NOT_OK(Tensor::CreateFromFile(image_path.ToString(), &image));
if (decode_ == true) {
Status rc = Decode(image, &image);
if (rc.IsError()) {
image = nullptr;
std::string err_msg = "Invalid data, failed to decode image: " + image_path.toString();
std::string err_msg = "Invalid data, failed to decode image: " + image_path.ToString();
return Status(StatusCode::kMDUnexpectedError, __LINE__, __FILE__, err_msg);
}
}
@ -271,7 +271,7 @@ Status CelebAOp::LoadTensorRow(row_id_type row_id, TensorRow *row) {
(*row) = TensorRow(row_id, {std::move(image), std::move(label)});
// Add file path info
row->setPath({image_path.toString(), attr_file_});
row->setPath({image_path.ToString(), attr_file_});
return Status::OK();
}

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/celeba_op.h
View File

@ -48,6 +48,10 @@ class CelebAOp : public MappableLeafOp {
// @param int32_t - num_workers - Num of workers reading images in parallel
// @param std::string - dir directory of celeba dataset
// @param int32_t queueSize - connector queue size
// @param bool decode - decode the images after reading
// @param std::string usage - specify the train, valid, test part or all parts of dataset
// @param std::set<std::string> exts - list of file extensions to be included in the dataset
// @param std::unique_ptr<DataSchema> schema - path to the JSON schema file or schema object
// @param std::unique_ptr<Sampler> sampler - sampler tells CelebAOp what to read
CelebAOp(int32_t num_workers, const std::string &dir, int32_t queue_size, bool decode, const std::string &usage,
const std::set<std::string> &exts, std::unique_ptr<DataSchema> schema, std::shared_ptr<SamplerRT> sampler);

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/cifar_op.cc
View File

@ -208,11 +208,11 @@ Status CifarOp::GetCifarFiles() {
while (dirIt->HasNext()) {
Path file = dirIt->Next();
if (file.Extension() == kExtension) {
cifar_files_.push_back(file.toString());
cifar_files_.push_back(file.ToString());
}
}
} else {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open directory: " + dir_path.toString() +
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open directory: " + dir_path.ToString() +
", make sure file not damaged or permission denied.");
}
CHECK_FAIL_RETURN_UNEXPECTED(!cifar_files_.empty(), "Invalid file, no .bin files found under " + folder_path_);

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/flickr_op.cc
View File

@ -126,7 +126,7 @@ Status FlickrOp::ParseFlickrData() {
"; line: " + line);
}
image_file_path = (dataset_dir / image_name).toString();
image_file_path = (dataset_dir / image_name).ToString();
std::string annotation = line.substr(flag_idx + 1);
if (annotation.empty()) {
file_handle.close();

12
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc
View File

@ -159,9 +159,9 @@ Status ImageFolderOp::PrescanWorkerEntry(int32_t worker_id) {
while (dirItr->HasNext()) {
Path file = dirItr->Next();
if (extensions_.empty() || extensions_.find(file.Extension()) != extensions_.end()) {
(void)imgs.insert(file.toString().substr(dirname_offset_));
(void)imgs.insert(file.ToString().substr(dirname_offset_));
} else {
MS_LOG(WARNING) << "Image folder operator unsupported file found: " << file.toString()
MS_LOG(WARNING) << "Image folder operator unsupported file found: " << file.ToString()
<< ", extension: " << file.Extension() << ".";
}
}
@ -186,8 +186,8 @@ Status ImageFolderOp::RecursiveWalkFolder(Path *dir) {
Path subdir = dir_itr->Next();
if (subdir.IsDirectory()) {
if (class_index_.empty() ||
class_index_.find(subdir.toString().substr(dirname_offset_ + 1)) != class_index_.end()) {
RETURN_IF_NOT_OK(folder_name_queue_->EmplaceBack(subdir.toString().substr(dirname_offset_)));
class_index_.find(subdir.ToString().substr(dirname_offset_ + 1)) != class_index_.end()) {
RETURN_IF_NOT_OK(folder_name_queue_->EmplaceBack(subdir.ToString().substr(dirname_offset_)));
}
if (recursive_ == true) {
MS_LOG(ERROR) << "RecursiveWalkFolder(&subdir) functionality is disabled permanently. No recursive walk of "
@ -259,7 +259,7 @@ Status ImageFolderOp::CountRowsAndClasses(const std::string &path, const std::se
while (dir_itr->HasNext()) {
Path subdir = dir_itr->Next();
if (subdir.IsDirectory()) {
folder_paths.push(subdir.toString());
folder_paths.push(subdir.ToString());
if (!class_index.empty()) folder_names.insert(subdir.Basename());
}
}
@ -281,7 +281,7 @@ Status ImageFolderOp::CountRowsAndClasses(const std::string &path, const std::se
Path subdir(folder_paths.front());
dir_itr = Path::DirIterator::OpenDirectory(&subdir);
if (subdir.Exists() == false || dir_itr == nullptr) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open folder: " + subdir.toString());
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open folder: " + subdir.ToString());
}
while (dir_itr->HasNext()) {
if (exts.empty() || exts.find(subdir.Extension()) != exts.end()) {

6
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/mnist_op.cc
View File

@ -229,15 +229,15 @@ Status MnistOp::WalkAllFiles() {
Path file = dir_it->Next();
std::string fname = file.Basename(); // name of the mnist file
if ((fname.find(prefix + "-images") != std::string::npos) && (fname.find(img_ext) != std::string::npos)) {
image_names_.push_back(file.toString());
image_names_.push_back(file.ToString());
MS_LOG(INFO) << "Mnist operator found image file at " << fname << ".";
} else if ((fname.find(prefix + "-labels") != std::string::npos) && (fname.find(lbl_ext) != std::string::npos)) {
label_names_.push_back(file.toString());
label_names_.push_back(file.ToString());
MS_LOG(INFO) << "Mnist Operator found label file at " << fname << ".";
}
}
} else {
MS_LOG(WARNING) << "Mnist operator unable to open directory " << dir.toString() << ".";
MS_LOG(WARNING) << "Mnist operator unable to open directory " << dir.ToString() << ".";
}
std::sort(image_names_.begin(), image_names_.end());

32
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/distributed_sampler.cc
View File

@ -61,11 +61,17 @@ Status DistributedSamplerRT::InitSampler() {
rnd_.seed(seed_++);
if (offset_ != -1 || !even_dist_) {
if (offset_ == -1) offset_ = 0;
if (offset_ == -1) {
offset_ = 0;
}
samples_per_tensor_ = (num_rows_ + offset_) / num_devices_;
int64_t remainder = (num_rows_ + offset_) % num_devices_;
if (device_id_ < remainder) samples_per_tensor_++;
if (device_id_ < offset_) samples_per_tensor_--;
if (device_id_ < remainder) {
samples_per_tensor_++;
}
if (device_id_ < offset_) {
samples_per_tensor_--;
}
} else {
offset_ = 0;
samples_per_tensor_ = (num_rows_ + num_devices_ - 1) / num_devices_; // equals to ceil(num_rows/num_devices)
@ -78,7 +84,9 @@ Status DistributedSamplerRT::InitSampler() {
}
std::shuffle(shuffle_vec_.begin(), shuffle_vec_.end(), rnd_);
}
if (!samples_per_tensor_) non_empty_ = false;
if (!samples_per_tensor_) {
non_empty_ = false;
}
is_initialized = true;
return Status::OK();
@ -174,15 +182,23 @@ int64_t DistributedSamplerRT::CalculateNumSamples(int64_t num_rows) {
int64_t remainder = (child_num_rows + offset_) % num_devices_;
int64_t shard_size = (child_num_rows + offset_) / num_devices_;
if (offset_ != -1 || !even_dist_) {
if (offset_ == -1) offset_ = 0;
if (device_id_ < remainder) shard_size++;
if (device_id_ < offset_) shard_size--;
if (offset_ == -1) {
offset_ = 0;
}
if (device_id_ < remainder) {
shard_size++;
}
if (device_id_ < offset_) {
shard_size--;
}
} else {
shard_size = (child_num_rows + num_devices_ - 1) / num_devices_;
}
// add 1 to an empty shard
// this logic is needed to follow the logic in initSampler that is written for ConcatDataset
if (shard_size == 0) shard_size++;
if (shard_size == 0) {
shard_size++;
}
return std::min(num_samples, shard_size);
}

16
mindspore/ccsrc/minddata/dataset/engine/gnn/graph_feature_parser.cc
View File

@ -40,7 +40,9 @@ Status GraphFeatureParser::LoadFeatureTensor(const std::string &key, const std::
std::vector<int64_t> column_shape;
RETURN_IF_NOT_OK(shard_column_->GetColumnValueByName(key, col_blob, {}, &data, &data_ptr, &n_bytes, &col_type,
&col_type_size, &column_shape));
if (data == nullptr) data = reinterpret_cast<const unsigned char *>(&data_ptr[0]);
if (data == nullptr) {
data = reinterpret_cast<const unsigned char *>(&data_ptr[0]);
}
RETURN_IF_NOT_OK(Tensor::CreateFromMemory(std::move(TensorShape({static_cast<dsize_t>(n_bytes / col_type_size)})),
std::move(DataType(mindrecord::ColumnDataTypeNameNormalized[col_type])),
data, tensor));
@ -58,7 +60,9 @@ Status GraphFeatureParser::LoadFeatureToSharedMemory(const std::string &key, con
std::vector<int64_t> column_shape;
RETURN_IF_NOT_OK(shard_column_->GetColumnValueByName(key, col_blob, {}, &data, &data_ptr, &n_bytes, &col_type,
&col_type_size, &column_shape));
if (data == nullptr) data = reinterpret_cast<const unsigned char *>(&data_ptr[0]);
if (data == nullptr) {
data = reinterpret_cast<const unsigned char *>(&data_ptr[0]);
}
std::shared_ptr<Tensor> tensor;
RETURN_IF_NOT_OK(Tensor::CreateEmpty(std::move(TensorShape({2})), std::move(DataType(DataType::DE_INT64)), &tensor));
auto fea_itr = tensor->begin<int64_t>();
@ -82,7 +86,9 @@ Status GraphFeatureParser::LoadFeatureIndex(const std::string &key, const std::v
RETURN_IF_NOT_OK(shard_column_->GetColumnValueByName(key, col_blob, {}, &data, &data_ptr, &n_bytes, &col_type,
&col_type_size, &column_shape));
if (data == nullptr) data = reinterpret_cast<const unsigned char *>(&data_ptr[0]);
if (data == nullptr) {
data = reinterpret_cast<const unsigned char *>(&data_ptr[0]);
}
for (int i = 0; i < n_bytes; i += col_type_size) {
int32_t feature_ind = -1;
@ -93,7 +99,9 @@ Status GraphFeatureParser::LoadFeatureIndex(const std::string &key, const std::v
} else {
RETURN_STATUS_UNEXPECTED("Feature Index needs to be int32/int64 type!");
}
if (feature_ind >= 0) indices->push_back(feature_ind);
if (feature_ind >= 0) {
indices->push_back(feature_ind);
}
}
return Status::OK();
}

2
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.cc
View File

@ -112,7 +112,7 @@ Status ValidateDatasetFilesParam(const std::string &dataset_name, const std::vec
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (access(dataset_file.toString().c_str(), R_OK) == -1) {
if (access(dataset_file.ToString().c_str(), R_OK) == -1) {
std::string err_msg = dataset_name + ": No access to specified dataset file: " + f;
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);

14
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/celeba_node.cc
View File

@ -97,15 +97,15 @@ Status CelebANode::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size
std::string line;
Path folder_path(dataset_dir_);
auto realpath = Common::GetRealPath((folder_path / "list_attr_celeba.txt").toString());
auto realpath = Common::GetRealPath((folder_path / "list_attr_celeba.txt").ToString());
if (!realpath.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_attr_celeba.txt").toString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_attr_celeba.txt").toString());
MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_attr_celeba.txt").ToString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_attr_celeba.txt").ToString());
}
std::ifstream attr_file(realpath.value());
if (!attr_file.is_open()) {
std::string attr_file_name = (folder_path / "list_attr_celeba.txt").toString();
std::string attr_file_name = (folder_path / "list_attr_celeba.txt").ToString();
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open Celeba attr file: " + attr_file_name);
}
@ -136,10 +136,10 @@ Status CelebANode::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size
}
}
if (!partition_file.is_open()) {
auto realpath_eval = Common::GetRealPath((folder_path / "list_eval_partition.txt").toString());
auto realpath_eval = Common::GetRealPath((folder_path / "list_eval_partition.txt").ToString());
if (!realpath_eval.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_eval_partition.txt").toString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_eval_partition.txt").toString());
MS_LOG(ERROR) << "Get real path failed, path=" << (folder_path / "list_eval_partition.txt").ToString();
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + (folder_path / "list_eval_partition.txt").ToString());
}
partition_file.open(realpath_eval.value());

2
mindspore/ccsrc/minddata/dataset/engine/perf/connector_size.cc
View File

@ -97,7 +97,7 @@ Status ConnectorSize::SaveToFile() {
}
Status ConnectorSize::Init(const std::string &dir_path, const std::string &device_id) {
file_path_ = (Path(dir_path) / Path("pipeline_profiling_" + device_id + ".json")).toString();
file_path_ = (Path(dir_path) / Path("pipeline_profiling_" + device_id + ".json")).ToString();
Path path = Path(file_path_);
// Remove the file if it exists (from prior profiling usage)
RETURN_IF_NOT_OK(path.Remove());

2
mindspore/ccsrc/minddata/dataset/engine/perf/connector_throughput.cc
View File

@ -130,7 +130,7 @@ Status ConnectorThroughput::SaveToFile() {
}
Status ConnectorThroughput::Init(const std::string &dir_path, const std::string &device_id) {
file_path_ = (Path(dir_path) / Path("pipeline_profiling_" + device_id + ".json")).toString();
file_path_ = (Path(dir_path) / Path("pipeline_profiling_" + device_id + ".json")).ToString();
Path path = Path(file_path_);
// Remove the file if it exists (from prior profiling usage)
RETURN_IF_NOT_OK(path.Remove());

2
mindspore/ccsrc/minddata/dataset/engine/perf/cpu_sampling.cc
View File

@ -653,7 +653,7 @@ Status CpuSampling::SaveToFile() {
}
Status CpuSampling::Init(const std::string &dir_path, const std::string &device_id) {
file_path_ = (Path(dir_path) / Path("minddata_cpu_utilization_" + device_id + ".json")).toString();
file_path_ = (Path(dir_path) / Path("minddata_cpu_utilization_" + device_id + ".json")).ToString();
std::shared_ptr<DeviceCpu> device_cpu = std::make_shared<DeviceCpu>();
std::shared_ptr<OperatorCpu> operator_cpu = std::make_shared<OperatorCpu>();
std::shared_ptr<ProcessCpu> process_cpu = std::make_shared<ProcessCpu>();

2
mindspore/ccsrc/minddata/dataset/engine/perf/dataset_iterator_tracing.cc
View File

@ -42,7 +42,7 @@ Status DatasetIteratorTracing::Record(const int32_t type, const int32_t extra_in
}
Status DatasetIteratorTracing::Init(const std::string &dir_path, const std::string &device_id) {
file_path_ = (Path(dir_path) / Path("dataset_iterator_profiling_" + device_id + ".txt")).toString();
file_path_ = (Path(dir_path) / Path("dataset_iterator_profiling_" + device_id + ".txt")).ToString();
return Status::OK();
}

2
mindspore/ccsrc/minddata/dataset/engine/perf/device_queue_tracing.cc
View File

@ -42,7 +42,7 @@ void DeviceQueueTracing::Record(const int32_t type, const int32_t extra_info, co
}
Status DeviceQueueTracing::Init(const std::string &dir_path, const std::string &device_id) {
file_path_ = (Path(dir_path) / Path("device_queue_profiling_" + device_id + ".txt")).toString();
file_path_ = (Path(dir_path) / Path("device_queue_profiling_" + device_id + ".txt")).ToString();
return Status::OK();
}

2
mindspore/ccsrc/minddata/dataset/engine/perf/monitor.cc
View File

@ -23,8 +23,6 @@ namespace dataset {
Monitor::Monitor(ExecutionTree *tree) : tree_(tree) {
std::shared_ptr<ConfigManager> cfg = GlobalContext::config_manager();
sampling_interval_ = cfg->monitor_sampling_interval();
max_samples_ = 0;
cur_row_ = 0;
}
Status Monitor::operator()() {
// Register this thread with TaskManager to receive proper interrupt signal.

5
mindspore/ccsrc/minddata/dataset/engine/perf/monitor.h
View File

@ -39,14 +39,9 @@ class Monitor {
// This function will be the entry point of mindspore::Dataset::Task
Status operator()();
int64_t GetSamplingInterval() { return sampling_interval_; }
private:
int64_t cur_row_;
int64_t max_samples_;
int64_t sampling_interval_;
ExecutionTree *tree_;
std::vector<std::shared_ptr<Sampling>> sampling_list_;
};
} // namespace dataset
} // namespace mindspore

52
mindspore/ccsrc/minddata/dataset/include/dataset/text.h
View File

@ -97,7 +97,22 @@ class BertTokenizer final : public TensorTransform {
bool with_offsets = false)
: BertTokenizer(vocab, StringToChar(suffix_indicator), max_bytes_per_token, StringToChar(unknown_token),
lower_case, keep_whitespace, normalize_form, preserve_unused_token, with_offsets) {}
/// \brief Constructor.
/// \param[in] vocab A Vocab object.
/// \param[in] suffix_indicator This parameter is used to show that the sub-word
/// is the last part of a word (default='##').
/// \param[in] max_bytes_per_token Tokens exceeding this length will not be further split (default=100).
/// \param[in] unknown_token When a token cannot be found, return the token directly if 'unknown_token' is an empty
/// string, else return the specified string (default='[UNK]').
/// \param[in] lower_case If true, apply CaseFold, NormalizeUTF8 (NFD mode) and RegexReplace operations to
/// the input text to fold the text to lower case and strip accents characters. If false, only apply
/// the NormalizeUTF8('normalization_form' mode) operation to the input text (default=false).
/// \param[in] keep_whitespace If true, the whitespace will be kept in output tokens (default=false).
/// \param[in] normalize_form This parameter is used to specify a specific normalize mode. This is only effective
/// when 'lower_case' is false. See NormalizeUTF8 for details (default=NormalizeForm::kNone).
/// \param[in] preserve_unused_token If true, do not split special tokens like '[CLS]', '[SEP]', '[UNK]', '[PAD]' and
/// '[MASK]' (default=true).
/// \param[in] with_offsets Whether to output offsets of tokens (default=false).
explicit BertTokenizer(const std::shared_ptr<Vocab> &vocab, const std::vector<char> &suffix_indicator,
int32_t max_bytes_per_token, const std::vector<char> &unknown_token, bool lower_case,
bool keep_whitespace, const NormalizeForm normalize_form, bool preserve_unused_token,
@ -151,6 +166,17 @@ class JiebaTokenizer final : public TensorTransform {
const JiebaMode &mode = JiebaMode::kMix, bool with_offsets = false)
: JiebaTokenizer(StringToChar(hmm_path), StringToChar(mp_path), mode, with_offsets) {}
/// \brief Constructor.
/// \param[in] hmm_path Dictionary file is used by the HMMSegment algorithm. The dictionary can be obtained on the
/// official website of cppjieba (https://github.com/yanyiwu/cppjieba).
/// \param[in] mp_path Dictionary file is used by the MPSegment algorithm. The dictionary can be obtained on the
/// official website of cppjieba (https://github.com/yanyiwu/cppjieba).
/// \param[in] mode Valid values can be any of JiebaMode.kMP, JiebaMode.kHMM and JiebaMode.kMIX
/// (default=JiebaMode.kMIX).
/// - JiebaMode.kMP, tokenizes with MPSegment algorithm.
/// - JiebaMode.kHMM, tokenizes with Hidden Markov Model Segment algorithm.
/// - JiebaMode.kMIX, tokenizes with a mix of MPSegment and HMMSegment algorithms.
/// \param[in] with_offsets Whether to output offsets of tokens (default=false).
explicit JiebaTokenizer(const std::vector<char> &hmm_path, const std::vector<char> &mp_path, const JiebaMode &mode,
bool with_offsets);
@ -222,6 +248,13 @@ class Lookup final : public TensorTransform {
new (this) Lookup(vocab, unknown_token_c, data_type);
}
/// \brief Constructor.
/// \param[in] vocab a Vocab object.
/// \param[in] unknown_token Word is used for lookup. In case of the word is out of vocabulary (OOV),
/// the result of lookup will be replaced to unknown_token. If the unknown_token is not specified or it is OOV,
/// runtime error will be thrown (default={}, means no unknown_token is specified).
/// \param[in] data_type mindspore::DataType of the tensor after lookup; must be numeric, including bool.
/// (default=mindspore::DataType::kNumberTypeInt32).
explicit Lookup(const std::shared_ptr<Vocab> &vocab, const std::optional<std::vector<char>> &unknown_token,
mindspore::DataType data_type = mindspore::DataType::kNumberTypeInt32);
@ -254,6 +287,15 @@ class Ngram final : public TensorTransform {
const std::pair<std::string, int32_t> &right_pad = {"", 0}, const std::string &separator = " ")
: Ngram(ngrams, PairStringToChar(left_pad), PairStringToChar(right_pad), StringToChar(separator)) {}
/// \brief Constructor.
/// \param[in] ngrams ngrams is a vector of positive integers. For example, if ngrams={4, 3}, then the result
/// would be a 4-gram followed by a 3-gram in the same tensor. If the number of words is not enough to make up
/// a n-gram, an empty string will be returned.
/// \param[in] left_pad {"pad_token", pad_width}. Padding performed on left side of the sequence. pad_width will
/// be capped at n-1. left_pad=("_",2) would pad the left side of the sequence with "__" (default={"", 0}}).
/// \param[in] right_pad {"pad_token", pad_width}. Padding performed on right side of the sequence.pad_width will
/// be capped at n-1. right_pad=("-",2) would pad the right side of the sequence with "--" (default={"", 0}}).
/// \param[in] separator Symbol used to join strings together (default=" ").
explicit Ngram(const std::vector<int32_t> &ngrams, const std::pair<std::vector<char>, int32_t> &left_pad,
const std::pair<std::vector<char>, int32_t> &right_pad, const std::vector<char> &separator);
@ -309,6 +351,11 @@ class RegexReplace final : public TensorTransform {
explicit RegexReplace(std::string pattern, std::string replace, bool replace_all = true)
: RegexReplace(StringToChar(pattern), StringToChar(replace), replace_all) {}
/// \brief Constructor.
/// \param[in] pattern The regex expression patterns. Type should be char of vector.
/// \param[in] replace The string to replace the matched element.
/// \param[in] replace_all Confirm whether to replace all. If false, only replace the first matched element;
/// if true, replace all matched elements (default=true).
explicit RegexReplace(const std::vector<char> &pattern, const std::vector<char> &replace, bool replace_all);
/// \brief Destructor
@ -368,6 +415,9 @@ class SentencePieceTokenizer final : public TensorTransform {
SentencePieceTokenizer(const std::string &vocab_path, mindspore::dataset::SPieceTokenizerOutType out_type)
: SentencePieceTokenizer(StringToChar(vocab_path), out_type) {}
/// \brief Constructor.
/// \param[in] vocab_path vocab model file path. type should be char of vector.
/// \param[in] out_type The type of the output.
SentencePieceTokenizer(const std::vector<char> &vocab_path, mindspore::dataset::SPieceTokenizerOutType out_type);
/// \brief Destructor

6
mindspore/ccsrc/minddata/dataset/kernels/data/concatenate_op.cc
View File

@ -41,10 +41,16 @@ Status ConcatenateOp::OutputShape(const std::vector<TensorShape> &inputs, std::v
output_shape = output_shape + inputs.at(0).NumOfElements();
if (prepend_ != nullptr) {
CHECK_FAIL_RETURN_UNEXPECTED(prepend_->shape().Rank() == 1, "Concatenate: only 1D prepend supported");
CHECK_FAIL_RETURN_UNEXPECTED(
(std::numeric_limits<uint64_t>::max() - output_shape) > prepend_->shape().NumOfElements(),
"Concatenate: append parameter is too large to pend.");
output_shape = output_shape + prepend_->shape().NumOfElements();
}
if (append_ != nullptr) {
CHECK_FAIL_RETURN_UNEXPECTED(append_->shape().Rank() == 1, "Concatenate: only 1D append supported");
CHECK_FAIL_RETURN_UNEXPECTED(
(std::numeric_limits<uint64_t>::max() - output_shape) > append_->shape().NumOfElements(),
"Concatenate: append parameter is too large to pend.");
output_shape = output_shape + append_->shape().NumOfElements();
}

5
mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc
View File

@ -84,7 +84,7 @@ Status OneHotEncodingSigned(const std::shared_ptr<Tensor> &input, std::shared_pt
return Status::OK();
}
Status OneHotEncoding(std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output, dsize_t num_classes) {
Status OneHotEncoding(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, dsize_t num_classes) {
input->Squeeze();
if (input->Rank() > 1) { // We expect the input to be int he first dimension
@ -117,7 +117,7 @@ Status OneHotEncoding(std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *ou
}
}
Status FillHelper(const std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *out,
Status FillHelper(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *out,
std::shared_ptr<Tensor> fill_output, std::shared_ptr<Tensor> fill_value) {
const DataType &input_type = input->type();
const TensorShape &input_shape = input->shape();
@ -592,6 +592,7 @@ Status Mask(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *outpu
Status Concatenate(const TensorRow &input, TensorRow *output, int8_t axis, std::shared_ptr<Tensor> prepend,
std::shared_ptr<Tensor> append) {
CHECK_FAIL_RETURN_UNEXPECTED(input.size() > 0, "Concatenate: input is null");
axis = Tensor::HandleNeg(axis, input[0]->shape().Rank());
CHECK_FAIL_RETURN_UNEXPECTED(axis == 0, "Concatenate: only 1D input supported");

2
mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.h
View File

@ -39,7 +39,7 @@ namespace dataset {
// @param output: Tensor. The shape of the output tensor is <input_shape, numClasses>
// and the type is same as input.
// @param num_classes: Number of classes to.
Status OneHotEncoding(std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output, dsize_t num_classes);
Status OneHotEncoding(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, dsize_t num_classes);
Status OneHotEncodingUnsigned(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output,
dsize_t num_classes, int64_t index);

1
mindspore/ccsrc/minddata/dataset/kernels/data/to_float16_op.cc
View File

@ -21,6 +21,7 @@
namespace mindspore {
namespace dataset {
Status ToFloat16Op::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
return ToFloat16(input, output);
}
Status ToFloat16Op::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {

1
mindspore/ccsrc/minddata/dataset/kernels/data/type_cast_op.cc
View File

@ -26,6 +26,7 @@ TypeCastOp::TypeCastOp(const DataType &new_type) : type_(new_type) {}
TypeCastOp::TypeCastOp(const std::string &data_type) { type_ = DataType(data_type); }
Status TypeCastOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
return TypeCast(input, output, type_);
}
Status TypeCastOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {

6
mindspore/ccsrc/minddata/dataset/kernels/image/affine_op.cc
View File

@ -47,10 +47,12 @@ AffineOp::AffineOp(float_t degrees, const std::vector<float_t> &translation, flo
Status AffineOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
CHECK_FAIL_RETURN_UNEXPECTED(translation_.size() >= 2, "AffineOp::Compute translation_ size should >= 2");
float_t translation_x = translation_[0];
float_t translation_y = translation_[1];
float_t degrees = 0.0;
RETURN_IF_NOT_OK(DegreesToRadians(degrees_, &degrees));
CHECK_FAIL_RETURN_UNEXPECTED(shear_.size() >= 2, "AffineOp::Compute shear_ size should >= 2");
float_t shear_x = shear_[0];
float_t shear_y = shear_[1];
RETURN_IF_NOT_OK(DegreesToRadians(shear_x, &shear_x));
@ -73,8 +75,12 @@ Status AffineOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<T
// Thus, the affine matrix is M = T * C * RSS * C^-1
// image is hwc, rows = shape()[0]
CHECK_FAIL_RETURN_UNEXPECTED(input->shape().Size() >= 2, "AffineOp::Compute input->shape() size should >= 2");
float_t cx = ((input->shape()[1] - 1) / 2.0);
float_t cy = ((input->shape()[0] - 1) / 2.0);
CHECK_FAIL_RETURN_UNEXPECTED(cos(shear_y) != 0.0, "AffineOp: cos(shear_y) should not be zero.");
// Calculate RSS
std::vector<float_t> matrix{
static_cast<float>(scale_ * cos(degrees + shear_y) / cos(shear_y)),

34
mindspore/ccsrc/minddata/dataset/kernels/image/bounding_box.cc
View File

@ -29,6 +29,7 @@ BoundingBox::BoundingBox(bbox_float x, bbox_float y, bbox_float width, bbox_floa
Status BoundingBox::ReadFromTensor(const TensorPtr &bbox_tensor, dsize_t index_of_bbox,
std::shared_ptr<BoundingBox> *bbox_out) {
CHECK_FAIL_RETURN_UNEXPECTED(bbox_tensor != nullptr, "BoundingBox: bbox_tensor is null.");
bbox_float x;
bbox_float y;
bbox_float width;
@ -50,16 +51,20 @@ Status BoundingBox::ValidateBoundingBoxes(const TensorRow &image_and_bbox) {
return Status(StatusCode::kMDBoundingBoxInvalidShape, __LINE__, __FILE__,
"BoundingBox: bounding boxes should have to be two-dimensional matrix at least.");
}
uint32_t num_of_features = image_and_bbox[1]->shape()[1];
int64_t num_of_features = image_and_bbox[1]->shape()[1];
if (num_of_features < kNumOfCols) {
return Status(StatusCode::kMDBoundingBoxInvalidShape, __LINE__, __FILE__,
"BoundingBox: bounding boxes should be have at least 4 features.");
}
std::vector<std::shared_ptr<BoundingBox>> bbox_list;
RETURN_IF_NOT_OK(GetListOfBoundingBoxes(image_and_bbox[1], &bbox_list));
uint32_t img_h = image_and_bbox[0]->shape()[0];
uint32_t img_w = image_and_bbox[0]->shape()[1];
int64_t img_h = image_and_bbox[0]->shape()[0];
int64_t img_w = image_and_bbox[0]->shape()[1];
for (auto &bbox : bbox_list) {
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int64_t>::max() - bbox->x()) > bbox->width(),
"BoundingBox: bbox_width is too large.");
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int64_t>::max() - bbox->y()) > bbox->height(),
"BoundingBox: bbox_height is too large.");
if ((bbox->x() + bbox->width() > img_w) || (bbox->y() + bbox->height() > img_h)) {
return Status(StatusCode::kMDBoundingBoxOutOfBounds, __LINE__, __FILE__,
"BoundingBox: bounding boxes is out of bounds of the image");
@ -73,6 +78,7 @@ Status BoundingBox::ValidateBoundingBoxes(const TensorRow &image_and_bbox) {
}
Status BoundingBox::WriteToTensor(const TensorPtr &bbox_tensor, dsize_t index_of_bbox) {
CHECK_FAIL_RETURN_UNEXPECTED(bbox_tensor != nullptr, "BoundingBox: bbox_tensor is null.");
RETURN_IF_NOT_OK(bbox_tensor->SetItemAt<bbox_float>({index_of_bbox, 0}, x_));
RETURN_IF_NOT_OK(bbox_tensor->SetItemAt<bbox_float>({index_of_bbox, 1}, y_));
RETURN_IF_NOT_OK(bbox_tensor->SetItemAt<bbox_float>({index_of_bbox, 2}, width_));
@ -82,6 +88,7 @@ Status BoundingBox::WriteToTensor(const TensorPtr &bbox_tensor, dsize_t index_of
Status BoundingBox::GetListOfBoundingBoxes(const TensorPtr &bbox_tensor,
std::vector<std::shared_ptr<BoundingBox>> *bbox_out) {
CHECK_FAIL_RETURN_UNEXPECTED(bbox_tensor != nullptr, "BoundingBox: bbox_tensor is null.");
dsize_t num_of_boxes = bbox_tensor->shape()[0];
for (dsize_t i = 0; i < num_of_boxes; i++) {
std::shared_ptr<BoundingBox> bbox;
@ -104,10 +111,15 @@ Status BoundingBox::CreateTensorFromBoundingBoxList(const std::vector<std::share
}
Status BoundingBox::PadBBoxes(const TensorPtr *bbox_list, size_t bbox_count, int32_t pad_top, int32_t pad_left) {
CHECK_FAIL_RETURN_UNEXPECTED(bbox_list != nullptr, "BoundingBox: bbox_list ptr is null.");
for (dsize_t i = 0; i < bbox_count; i++) {
std::shared_ptr<BoundingBox> bbox;
RETURN_IF_NOT_OK(ReadFromTensor(*bbox_list, i, &bbox));
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int32_t>::max() - bbox->x()) > pad_left,
"BoundingBox: pad_left is too large.");
bbox->SetX(bbox->x() + pad_left);
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int32_t>::max() - bbox->y()) > pad_top,
"BoundingBox: pad_top is too large.");
bbox->SetY(bbox->y() + pad_top);
RETURN_IF_NOT_OK(bbox->WriteToTensor(*bbox_list, i));
}
@ -116,6 +128,7 @@ Status BoundingBox::PadBBoxes(const TensorPtr *bbox_list, size_t bbox_count, int
Status BoundingBox::UpdateBBoxesForCrop(TensorPtr *bbox_list, size_t *bbox_count, int32_t CB_Xmin, int32_t CB_Ymin,
int32_t CB_Xmax, int32_t CB_Ymax) {
CHECK_FAIL_RETURN_UNEXPECTED(bbox_list != nullptr, "BoundingBox: bbox_list ptr is null.");
// PASS LIST, COUNT OF BOUNDING BOXES
// Also PAss X/Y Min/Max of image cropped region - normally obtained from 'GetCropBox' functions
std::vector<dsize_t> correct_ind;
@ -156,6 +169,7 @@ Status BoundingBox::UpdateBBoxesForCrop(TensorPtr *bbox_list, size_t *bbox_count
// create new tensor and copy over bboxes still valid to the image
// bboxes outside of new cropped region are ignored - empty tensor returned in case of none
*bbox_count = correct_ind.size();
CHECK_FAIL_RETURN_UNEXPECTED(*bbox_count >= 0, "BoundingBox: correct_ind.size() is smaller than zero.");
bbox_float temp = 0.0;
for (auto slice : correct_ind) { // for every index in the loop
for (dsize_t ix = 0; ix < bboxDim; ix++) {
@ -172,6 +186,10 @@ Status BoundingBox::UpdateBBoxesForCrop(TensorPtr *bbox_list, size_t *bbox_count
Status BoundingBox::UpdateBBoxesForResize(const TensorPtr &bbox_list, size_t bbox_count, int32_t target_width,
int32_t target_height, int32_t orig_width, int32_t orig_height) {
CHECK_FAIL_RETURN_UNEXPECTED(bbox_list != nullptr, "BoundingBox: bbox_list ptr is null.");
CHECK_FAIL_RETURN_UNEXPECTED(orig_width != 0, "BoundingBox: orig_width is zero.");
CHECK_FAIL_RETURN_UNEXPECTED(orig_height != 0, "BoundingBox: orig_height is zero.");
// cast to float to preserve fractional
bbox_float W_aspRatio = (target_width * 1.0) / (orig_width * 1.0);
bbox_float H_aspRatio = (target_height * 1.0) / (orig_height * 1.0);
@ -179,6 +197,16 @@ Status BoundingBox::UpdateBBoxesForResize(const TensorPtr &bbox_list, size_t bbo
// for each bounding box
std::shared_ptr<BoundingBox> bbox;
RETURN_IF_NOT_OK(ReadFromTensor(bbox_list, i, &bbox));
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() / bbox->x()) > W_aspRatio,
"BoundingBox: W_aspRatio is too large.");
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() / bbox->y()) > H_aspRatio,
"BoundingBox: H_aspRatio is too large.");
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() / bbox->width()) > W_aspRatio,
"BoundingBox: W_aspRatio is too large.");
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() / bbox->height()) > H_aspRatio,
"BoundingBox: H_aspRatio is too large.");
// update positions and widths
bbox->SetX(bbox->x() * W_aspRatio);
bbox->SetY(bbox->y() * H_aspRatio);

34
mindspore/ccsrc/minddata/dataset/kernels/image/cutmix_batch_op.cc
View File

@ -29,11 +29,11 @@ namespace dataset {
constexpr size_t kMinLabelShapeSize = 2;
constexpr size_t kMaxLabelShapeSize = 3;
constexpr size_t kExpectedImageShapeSize = 4;
constexpr size_t dimension_one = 1;
constexpr size_t dimension_two = 2;
constexpr size_t dimension_three = 3;
constexpr int64_t value_one = 1;
constexpr int64_t value_three = 3;
constexpr size_t kDimensionOne = 1;
constexpr size_t kDimensionTwo = 2;
constexpr size_t kDimensionThree = 3;
constexpr int64_t kValueOne = 1;
constexpr int64_t kValueThree = 3;
CutMixBatchOp::CutMixBatchOp(ImageBatchFormat image_batch_format, float alpha, float prob)
: image_batch_format_(image_batch_format), alpha_(alpha), prob_(prob) {
@ -81,11 +81,11 @@ Status CutMixBatchOp::ValidateCutMixBatch(const TensorRow &input) {
"L is the number of labels in each row, and C is the number of classes. "
"labels must be in one-hot format and in a batch.");
}
if ((image_shape[dimension_one] != value_one && image_shape[dimension_one] != value_three) &&
if ((image_shape[kDimensionOne] != kValueOne && image_shape[kDimensionOne] != kValueThree) &&
image_batch_format_ == ImageBatchFormat::kNCHW) {
RETURN_STATUS_UNEXPECTED("CutMixBatch: image doesn't match the NCHW format.");
}
if ((image_shape[dimension_three] != value_one && image_shape[dimension_three] != value_three) &&
if ((image_shape[kDimensionThree] != kValueOne && image_shape[kDimensionThree] != kValueThree) &&
image_batch_format_ == ImageBatchFormat::kNHWC) {
RETURN_STATUS_UNEXPECTED("CutMixBatch: image doesn't match the NHWC format.");
}
@ -104,22 +104,22 @@ Status CutMixBatchOp::ComputeImage(const TensorRow &input, const int64_t rand_in
RETURN_IF_NOT_OK(input.at(0)->StartAddrOfIndex({rand_indx_i, 0, 0, 0}, &start_addr_of_index, &remaining));
RETURN_IF_NOT_OK(Tensor::CreateFromMemory(
TensorShape({image_shape[dimension_one], image_shape[dimension_two], image_shape[dimension_three]}),
TensorShape({image_shape[kDimensionOne], image_shape[kDimensionTwo], image_shape[kDimensionThree]}),
input.at(0)->type(), start_addr_of_index, &rand_image));
// Compute image
if (image_batch_format_ == ImageBatchFormat::kNHWC) {
// NHWC Format
GetCropBox(static_cast<int32_t>(image_shape[dimension_one]), static_cast<int32_t>(image_shape[dimension_two]), lam,
GetCropBox(static_cast<int32_t>(image_shape[kDimensionOne]), static_cast<int32_t>(image_shape[kDimensionTwo]), lam,
&x, &y, &crop_width, &crop_height);
std::shared_ptr<Tensor> cropped;
RETURN_IF_NOT_OK(Crop(rand_image, &cropped, x, y, crop_width, crop_height));
RETURN_IF_NOT_OK(MaskWithTensor(cropped, image_i, x, y, crop_width, crop_height, ImageFormat::HWC));
*label_lam = value_one - (crop_width * crop_height /
static_cast<float>(image_shape[dimension_one] * image_shape[dimension_two]));
*label_lam = kValueOne - (crop_width * crop_height /
static_cast<float>(image_shape[kDimensionOne] * image_shape[kDimensionTwo]));
} else {
// NCHW Format
GetCropBox(static_cast<int32_t>(image_shape[dimension_two]), static_cast<int32_t>(image_shape[dimension_three]),
GetCropBox(static_cast<int32_t>(image_shape[kDimensionTwo]), static_cast<int32_t>(image_shape[kDimensionThree]),
lam, &x, &y, &crop_width, &crop_height);
std::vector<std::shared_ptr<Tensor>> channels; // A vector holding channels of the CHW image
std::vector<std::shared_ptr<Tensor>> cropped_channels; // A vector holding the channels of the cropped CHW
@ -135,8 +135,8 @@ Status CutMixBatchOp::ComputeImage(const TensorRow &input, const int64_t rand_in
RETURN_IF_NOT_OK(TensorVectorToBatchTensor(cropped_channels, &cropped));
RETURN_IF_NOT_OK(MaskWithTensor(cropped, image_i, x, y, crop_width, crop_height, ImageFormat::CHW));
*label_lam = value_one - (crop_width * crop_height /
static_cast<float>(image_shape[dimension_two] * image_shape[dimension_three]));
*label_lam = kValueOne - (crop_width * crop_height /
static_cast<float>(image_shape[kDimensionTwo] * image_shape[kDimensionThree]));
}
return Status::OK();
@ -194,8 +194,8 @@ Status CutMixBatchOp::Compute(const TensorRow &input, TensorRow *output) {
// Tensor holding the output labels
std::shared_ptr<Tensor> out_labels;
RETURN_IF_NOT_OK(TypeCast(std::move(input.at(1)), &out_labels, DataType(DataType::DE_FLOAT32)));
int64_t row_labels = label_shape.size() == value_three ? label_shape[dimension_one] : value_one;
int64_t num_classes = label_shape.size() == value_three ? label_shape[dimension_two] : label_shape[dimension_one];
int64_t row_labels = label_shape.size() == kValueThree ? label_shape[kDimensionOne] : kValueOne;
int64_t num_classes = label_shape.size() == kValueThree ? label_shape[kDimensionTwo] : label_shape[kDimensionOne];
// Compute labels and images
for (size_t i = 0; i < static_cast<size_t>(image_shape[0]); i++) {
@ -204,6 +204,8 @@ Status CutMixBatchOp::Compute(const TensorRow &input, TensorRow *output) {
// then x = x1 / (x1+x2) is a random variable from Beta(a1, a2)
float x1 = gamma_distribution(rnd_);
float x2 = gamma_distribution(rnd_);
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() - x1) > x2,
"CutMixBatchOp: gamma_distribution x1 and x2 are too large.");
float lam = x1 / (x1 + x2);
double random_number = uniform_distribution(rnd_);
if (random_number < prob_) {

3
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_crop_jpeg_op.cc
View File

@ -134,6 +134,9 @@ Status DvppCropJpegOp::OutputShape(const std::vector<TensorShape> &inputs, std::
RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs));
outputs.clear();
TensorShape out({-1, 1, 1}); // we don't know what is output image size, but we know it should be 1 channels
if (inputs.size() < 1) {
RETURN_STATUS_UNEXPECTED("DvppCropJpegOp::OutputShape inputs is null");
}
if (inputs[0].Rank() == 1) outputs.emplace_back(out);
if (!outputs.empty()) return Status::OK();
return Status(StatusCode::kMDUnexpectedError, "Input has a wrong shape");

11
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_jpeg_op.cc
View File

@ -134,8 +134,15 @@ Status DvppDecodeJpegOp::OutputShape(const std::vector<TensorShape> &inputs, std
RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs));
outputs.clear();
TensorShape out({-1, 1, 1}); // we don't know what is output image size, but we know it should be 3 channels
if (inputs[0].Rank() == 1) outputs.emplace_back(out);
if (!outputs.empty()) return Status::OK();
if (inputs.size() < 1) {
RETURN_STATUS_UNEXPECTED("DvppDecodeJpegOp::OutputShape inputs is null");
}
if (inputs[0].Rank() == 1) {
outputs.emplace_back(out);
}
if (!outputs.empty()) {
return Status::OK();
}
return Status(StatusCode::kMDUnexpectedError, "Input has a wrong shape");
}

11
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_png_op.cc
View File

@ -123,8 +123,15 @@ Status DvppDecodePngOp::OutputShape(const std::vector<TensorShape> &inputs, std:
RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs));
outputs.clear();
TensorShape out({-1, 1, 1}); // we don't know what is output image size, but we know it should be 3 channels
if (inputs[0].Rank() == 1) outputs.emplace_back(out);
if (!outputs.empty()) return Status::OK();
if (inputs.size() < 1) {
RETURN_STATUS_UNEXPECTED("DvppDecodePngOp::OutputShape inputs is null");
}
if (inputs[0].Rank() == 1) {
outputs.emplace_back(out);
}
if (!outputs.empty()) {
return Status::OK();
}
return Status(StatusCode::kMDUnexpectedError, "Input has a wrong shape");
}

3
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_crop_jpeg_op.cc
View File

@ -122,6 +122,9 @@ Status DvppDecodeResizeCropJpegOp::OutputShape(const std::vector<TensorShape> &i
RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs));
outputs.clear();
TensorShape out({-1, 1, 1}); // we don't know what is output image size, but we know it should be 3 channels
if (inputs.size() < 1) {
RETURN_STATUS_UNEXPECTED("DvppDecodeResizeCropJpegOp::OutputShape inputs is null");
}
if (inputs[0].Rank() == 1) outputs.emplace_back(out);
if (!outputs.empty()) return Status::OK();
return Status(StatusCode::kMDUnexpectedError, "Input has a wrong shape");

3
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_jpeg_op.cc
View File

@ -120,6 +120,9 @@ Status DvppDecodeResizeJpegOp::OutputShape(const std::vector<TensorShape> &input
RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs));
outputs.clear();
TensorShape out({-1, 1, 1}); // we don't know what is output image size, but we know it should be 1 channels
if (inputs.size() < 1) {
RETURN_STATUS_UNEXPECTED("DvppDecodeResizeJpegOp::OutputShape inputs is null");
}
if (inputs[0].Rank() == 1) outputs.emplace_back(out);
if (!outputs.empty()) return Status::OK();
return Status(StatusCode::kMDUnexpectedError, "Input has a wrong shape");

11
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_resize_jpeg_op.cc
View File

@ -144,8 +144,15 @@ Status DvppResizeJpegOp::OutputShape(const std::vector<TensorShape> &inputs, std
RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs));
outputs.clear();
TensorShape out({-1, 1, 1}); // we don't know what is output image size, but we know it should be 1 channels
if (inputs[0].Rank() == 1) outputs.emplace_back(out);
if (!outputs.empty()) return Status::OK();
if (inputs.size() < 1) {
RETURN_STATUS_UNEXPECTED("DvppResizeJpegOp::OutputShape inputs is null");
}
if (inputs[0].Rank() == 1) {
outputs.emplace_back(out);
}
if (!outputs.empty()) {
return Status::OK();
}
return Status(StatusCode::kMDUnexpectedError, "Input has a wrong shape");
}

12
mindspore/ccsrc/minddata/dataset/kernels/image/exif_utils.cc
View File

@ -34,7 +34,11 @@ uint8_t parse_bytes(const uint8_t *buf, bool intel_align) {
template <>
uint16_t parse_bytes(const uint8_t *buf, bool intel_align) {
uint16_t res;
if (buf == nullptr) {
return 0;
}
uint16_t res = 0;
if (intel_align) {
res = (static_cast<uint16_t>(buf[1]) << 8) | buf[0];
} else {
@ -45,7 +49,11 @@ uint16_t parse_bytes(const uint8_t *buf, bool intel_align) {
template <>
uint32_t parse_bytes(const uint8_t *buf, bool intel_align) {
uint32_t res;
if (buf == nullptr) {
return 0;
}
uint32_t res = 0;
if (intel_align) {
res = (static_cast<uint32_t>(buf[3]) << 24) | (static_cast<uint32_t>(buf[2]) << 16) |
(static_cast<uint32_t>(buf[1]) << 8) | buf[0];

1
mindspore/ccsrc/minddata/dataset/kernels/image/hwc_to_chw_op.cc
View File

@ -29,6 +29,7 @@ Status HwcToChwOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr
Status HwcToChwOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs));
outputs.clear();
CHECK_FAIL_RETURN_UNEXPECTED(inputs.size() > 0, "HwcToChwOp::OutputShape inputs size should > 0");
TensorShape in = inputs[0];
TensorShape out = TensorShape{in[2], in[0], in[1]};
if (inputs[0].Rank() == 3) {

33
mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc
View File

@ -87,7 +87,13 @@ Status GetConvertShape(ConvertMode convert_mode, const std::shared_ptr<CVTensor>
}
bool CheckTensorShape(const std::shared_ptr<Tensor> &tensor, const int &channel) {
if (tensor == nullptr) {
return false;
}
bool rc = false;
if (tensor->shape().Size() <= channel) {
return false;
}
if (tensor->Rank() != DEFAULT_IMAGE_RANK ||
(tensor->shape()[channel] != 1 && tensor->shape()[channel] != DEFAULT_IMAGE_CHANNELS)) {
rc = true;
@ -271,6 +277,8 @@ static Status JpegReadScanlines(jpeg_decompress_struct *const cinfo, int max_sca
int buffer_size, int crop_w, int crop_w_aligned, int offset, int stride) {
// scanlines will be read to this buffer first, must have the number
// of components equal to the number of components in the image
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int64_t>::max() / cinfo->output_components) > crop_w_aligned,
"JpegReadScanlines: multiplication out of bounds.");
int64_t scanline_size = crop_w_aligned * cinfo->output_components;
std::vector<JSAMPLE> scanline(scanline_size);
JSAMPLE *scanline_ptr = &scanline[0];
@ -364,6 +372,10 @@ Status JpegCropAndDecode(const std::shared_ptr<Tensor> &input, std::shared_ptr<T
} catch (std::runtime_error &e) {
return DestroyDecompressAndReturnError(e.what());
}
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int32_t>::max() - crop_w) > crop_x,
"JpegCropAndDecode: addition out of bounds.");
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int32_t>::max() - crop_h) > crop_y,
"JpegCropAndDecode: addition out of bounds.");
if (crop_x == 0 && crop_y == 0 && crop_w == 0 && crop_h == 0) {
crop_w = cinfo.output_width;
crop_h = cinfo.output_height;
@ -372,6 +384,7 @@ Status JpegCropAndDecode(const std::shared_ptr<Tensor> &input, std::shared_ptr<T
return DestroyDecompressAndReturnError("Crop: invalid crop size.");
}
const int mcu_size = cinfo.min_DCT_scaled_size;
CHECK_FAIL_RETURN_UNEXPECTED(mcu_size != 0, "JpegCropAndDecode: divisor mcu_size is zero.");
unsigned int crop_x_aligned = (crop_x / mcu_size) * mcu_size;
unsigned int crop_w_aligned = crop_w + crop_x - crop_x_aligned;
try {
@ -388,12 +401,19 @@ Status JpegCropAndDecode(const std::shared_ptr<Tensor> &input, std::shared_ptr<T
RETURN_IF_NOT_OK(Tensor::CreateEmpty(ts, DataType(DataType::DE_UINT8), &output_tensor));
const int buffer_size = output_tensor->SizeInBytes();
JSAMPLE *buffer = reinterpret_cast<JSAMPLE *>(&(*output_tensor->begin<uint8_t>()));
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() - skipped_scanlines) > crop_h,
"JpegCropAndDecode: addition out of bounds.");
const int max_scanlines_to_read = skipped_scanlines + crop_h;
// stride refers to output tensor, which has 3 components at most
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int32_t>::max() / crop_w) > kOutNumComponents,
"JpegCropAndDecode: multiplication out of bounds.");
const int stride = crop_w * kOutNumComponents;
// offset is calculated for scanlines read from the image, therefore
// has the same number of components as the image
const int offset = (crop_x - crop_x_aligned) * cinfo.output_components;
int minius_value = crop_x - crop_x_aligned;
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() / minius_value) > cinfo.output_components,
"JpegCropAndDecode: multiplication out of bounds.");
const int offset = minius_value * cinfo.output_components;
RETURN_IF_NOT_OK(
JpegReadScanlines(&cinfo, max_scanlines_to_read, buffer, buffer_size, crop_w, crop_w_aligned, offset, stride));
*output = output_tensor;
@ -426,12 +446,14 @@ Status Crop(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *outpu
if (input_cv->Rank() != DEFAULT_IMAGE_RANK && input_cv->Rank() != 2) {
RETURN_STATUS_UNEXPECTED("Crop: invalid image Shape, only support <H,W,C> or <H,W>");
}
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int32_t>::max() - y) > h, "Crop: addition out of bounds.");
// account for integer overflow
if (y < 0 || (y + h) > input_cv->shape()[0] || (y + h) < 0) {
RETURN_STATUS_UNEXPECTED(
"Crop: invalid y coordinate value for crop, "
"y coordinate value exceeds the boundary of the image.");
}
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int32_t>::max() - x) > w, "Crop: addition out of bounds.");
// account for integer overflow
if (x < 0 || (x + w) > input_cv->shape()[1] || (x + w) < 0) {
RETURN_STATUS_UNEXPECTED(
@ -495,6 +517,7 @@ Status HwcToChw(std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output)
*output = input;
return Status::OK();
}
CHECK_FAIL_RETURN_UNEXPECTED(input_cv->shape().Size() > CHANNEL_INDEX, "HWC2CHW: invalid shape.");
int num_channels = input_cv->shape()[CHANNEL_INDEX];
if (input_cv->shape().Size() < MIN_IMAGE_DIMENSION || input_cv->shape().Size() > DEFAULT_IMAGE_CHANNELS ||
(input_cv->shape().Size() == DEFAULT_IMAGE_CHANNELS && num_channels != DEFAULT_IMAGE_CHANNELS &&
@ -609,6 +632,7 @@ Status CopyTensorValue(const std::shared_ptr<Tensor> &source_tensor, std::shared
Status SwapRedAndBlue(std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output) {
try {
std::shared_ptr<CVTensor> input_cv = CVTensor::AsCVTensor(std::move(input));
CHECK_FAIL_RETURN_UNEXPECTED(input_cv->shape().Size() > CHANNEL_INDEX, "SwapRedAndBlue: shape is invalid.");
int num_channels = input_cv->shape()[CHANNEL_INDEX];
if (input_cv->shape().Size() != 3 || num_channels != DEFAULT_IMAGE_CHANNELS) {
RETURN_STATUS_UNEXPECTED("SwapRedBlue: image shape is not <H,W,C>.");
@ -882,6 +906,7 @@ Status AdjustBrightness(const std::shared_ptr<Tensor> &input, std::shared_ptr<Te
if (!input_cv->mat().data) {
RETURN_STATUS_UNEXPECTED("AdjustBrightness: load image failed.");
}
CHECK_FAIL_RETURN_UNEXPECTED(input_cv->shape().Size() > CHANNEL_INDEX, "AdjustBrightness: shape is invalid.");
int num_channels = input_cv->shape()[CHANNEL_INDEX];
// Rank of the image represents how many dimensions, image is expected to be HWC
if (input_cv->Rank() != DEFAULT_IMAGE_RANK || num_channels != DEFAULT_IMAGE_CHANNELS) {
@ -904,6 +929,7 @@ Status AdjustContrast(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tens
if (!input_cv->mat().data) {
RETURN_STATUS_UNEXPECTED("AdjustContrast: load image failed.");
}
CHECK_FAIL_RETURN_UNEXPECTED(input_cv->shape().Size() > CHANNEL_INDEX, "AdjustBrightness: shape is invalid.");
int num_channels = input_cv->shape()[CHANNEL_INDEX];
if (input_cv->Rank() != DEFAULT_IMAGE_CHANNELS || num_channels != DEFAULT_IMAGE_CHANNELS) {
RETURN_STATUS_UNEXPECTED("AdjustContrast: image shape is not <H,W,C> or channel is not 3.");
@ -990,7 +1016,7 @@ Status AutoContrast(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor
RETURN_STATUS_UNEXPECTED("AutoContrast: load image failed.");
}
if (input_cv->Rank() != DEFAULT_IMAGE_RANK && input_cv->Rank() != MIN_IMAGE_DIMENSION) {
RETURN_STATUS_UNEXPECTED("AutoContrast: image shape is not <H,W,C> or <H,W>");
RETURN_STATUS_UNEXPECTED("AutoContrast: image channel should be 1 or 3.");
}
// Reshape to extend dimension if rank is 2 for algorithm to work. then reshape output to be of rank 2 like input
if (input_cv->Rank() == MIN_IMAGE_DIMENSION) {
@ -1067,6 +1093,7 @@ Status AdjustSaturation(const std::shared_ptr<Tensor> &input, std::shared_ptr<Te
if (!input_cv->mat().data) {
RETURN_STATUS_UNEXPECTED("AdjustSaturation: load image failed.");
}
CHECK_FAIL_RETURN_UNEXPECTED(input_cv->shape().Size() > CHANNEL_INDEX, "AdjustSaturation: shape is invalid.");
int num_channels = input_cv->shape()[CHANNEL_INDEX];
if (input_cv->Rank() != DEFAULT_IMAGE_RANK || num_channels != DEFAULT_IMAGE_CHANNELS) {
RETURN_STATUS_UNEXPECTED("AdjustSaturation: image shape is not <H,W,C> or channel is not 3.");
@ -1095,6 +1122,7 @@ Status AdjustHue(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *
if (!input_cv->mat().data) {
RETURN_STATUS_UNEXPECTED("AdjustHue: load image failed.");
}
CHECK_FAIL_RETURN_UNEXPECTED(input_cv->shape().Size() > 2, "AdjustHue: shape is invalid.");
int num_channels = input_cv->shape()[2];
if (input_cv->Rank() != DEFAULT_IMAGE_RANK || num_channels != DEFAULT_IMAGE_CHANNELS) {
RETURN_STATUS_UNEXPECTED("AdjustHue: image shape is not <H,W,C> or channel is not 3.");
@ -1166,6 +1194,7 @@ Status Erase(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *outp
uint8_t fill_g, uint8_t fill_b) {
try {
std::shared_ptr<CVTensor> input_cv = CVTensor::AsCVTensor(input);
CHECK_FAIL_RETURN_UNEXPECTED(input_cv->shape().Size() > CHANNEL_INDEX, "Erase: shape is invalid.");
int num_channels = input_cv->shape()[CHANNEL_INDEX];
if (input_cv->mat().data == nullptr) {
RETURN_STATUS_UNEXPECTED("CutOut: load image failed.");

28
mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.cc
View File

@ -391,6 +391,7 @@ static bool ConvertBGR(const unsigned char *data, LDataType data_type, int w, in
if (data_type == LDataType::UINT8) {
mat.Init(w, h, 3, LDataType::UINT8);
unsigned char *dst_ptr = mat;
// mindspore lite version, there is no securec lib
(void)memcpy(dst_ptr, data, w * h * 3 * sizeof(unsigned char));
} else {
return false;
@ -637,6 +638,7 @@ static bool CropInternal(const LiteMat &src, LiteMat &dst, int x, int y, int w,
for (int i_h = 0; i_h < dst_h; i_h++) {
const T *src_index_p = src_start_p + (y + i_h) * src.width_ * dst_c + x * dst_c;
T *dst_index_p = dst_start_p + i_h * dst_w * dst_c;
// mindspore lite version, there is no securec lib
(void)memcpy(dst_index_p, src_index_p, dst_w * dst_c * sizeof(T));
}
return true;
@ -766,6 +768,7 @@ static void PadWithConstant(const LiteMat &src, LiteMat &dst, const int top, con
uint8_t *dst_ptr = reinterpret_cast<uint8_t *>(dst.data_ptr_);
uint8_t *src_ptr = reinterpret_cast<uint8_t *>(src.data_ptr_);
for (int i = 0; i < top; i++) {
// mindspore lite version, there is no securec lib
memcpy(dst_ptr + i * dst_step, const_ptr, dst_step);
}
@ -773,12 +776,14 @@ static void PadWithConstant(const LiteMat &src, LiteMat &dst, const int top, con
int right_size = right * dst.channel_ * dst.elem_size_;
uint8_t *dst_raw_data = dst_ptr + top * dst_step + left_size;
for (int i = 0; i < src.height_; i++, dst_raw_data += dst_step, src_ptr += src_step) {
// mindspore lite version, there is no securec lib
memcpy(dst_raw_data, src_ptr, src_step);
memcpy(dst_raw_data - left_size, const_ptr, left_size);
memcpy(dst_raw_data + src_step, const_ptr, right_size);
}
for (int i = dst.height_ - bottom; i < dst.height_; i++) {
// mindspore lite version, there is no securec lib
memcpy(dst_ptr + i * dst_step, const_ptr, dst_step);
}
}
@ -804,6 +809,7 @@ static void PadImplement(const LiteMat &src, LiteMat &dst, const int top, const
uint8_t *src_data_ptr = reinterpret_cast<uint8_t *>(src.data_ptr_);
uint8_t *dst_data_ptr = reinterpret_cast<uint8_t *>(dst.data_ptr_);
for (int i = 0; i < src.height_; i++) {
// mindspore lite version, there is no securec lib
memcpy(dst_data_ptr + (i + top) * dst.steps_[0] + left * dst.steps_[1], src_data_ptr + i * src.steps_[0],
src.steps_[0]);
}
@ -1749,14 +1755,18 @@ void ImageToolsConvertImage(const LiteMat &src, const LiteMat &dst, imageToolsIm
imageOut->dataType = IM_TOOL_DATA_TYPE_FLOAT;
}
void InvAffine2x3(float M[2][3], float invM[][3]) {
int InvAffine2x3(float M[2][3], float invM[][3]) {
float inv_det = M[0][0] * M[1][1] - M[1][0] * M[0][1];
if (inv_det == 0.0) {
return IM_TOOL_RETURN_STATUS_FAILED;
}
invM[1][1] = M[0][0] / inv_det;
invM[0][1] = -M[0][1] / inv_det;
invM[1][0] = -M[1][0] / inv_det;
invM[0][0] = M[1][1] / inv_det;
invM[0][2] = (M[0][1] * M[1][2] - M[1][1] * M[0][2]) / inv_det;
invM[1][2] = -(M[0][0] * M[1][2] - M[1][0] * M[0][2]) / inv_det;
return IM_TOOL_RETURN_STATUS_SUCCESS;
}
static float *CalDst(float *dst, float v1, float v2, float v3) {
@ -1901,7 +1911,9 @@ int ImageWarpAffineHWC(imageToolsImage_t image, imageToolsImage_t warped_image,
}
}
} else {
InvAffine2x3(M, invM);
if (InvAffine2x3(M, invM) != IM_TOOL_RETURN_STATUS_SUCCESS) {
return IM_TOOL_RETURN_STATUS_FAILED;
}
}
if (IM_TOOL_DATA_TYPE_FLOAT == image.dataType) {
@ -1928,6 +1940,14 @@ bool ResizePreserveARWithFiller(LiteMat &src, LiteMat &dst, int h, int w, float
const float divisor = 2.0;
int rotationDstWidth = src.width_;
int rotationDstHeight = src.height_;
if (rotationDstWidth == 0 || rotationDstHeight == 0) {
return false;
}
if (dst.height_ == 0) {
return false;
}
if (img_orientation > IM_TOOL_EXIF_ORIENTATION_0_DEG) {
UpdateOrientationAfineMat(src, &rotationDstWidth, &rotationDstHeight, &varM, img_orientation);
}
@ -1959,7 +1979,9 @@ bool ResizePreserveARWithFiller(LiteMat &src, LiteMat &dst, int h, int w, float
/* Resize and shift by affine transform */
imageToolsImage_t imageIn, imageOut;
ImageToolsConvertImage(src, dst, &imageIn, &imageOut);
InvAffine2x3(varM, *invM);
if (InvAffine2x3(varM, *invM) != IM_TOOL_RETURN_STATUS_SUCCESS) {
return false;
}
int retVal = ImageWarpAffineHWC(imageIn, imageOut, *invM, true);
if (retVal != 0) {
return false;

1
mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/image_process.h
View File

@ -41,6 +41,7 @@ namespace dataset {
#define IM_TOOL_DATA_TYPE_UINT8 (2)
#define IM_TOOL_RETURN_STATUS_SUCCESS (0)
#define IM_TOOL_RETURN_STATUS_INVALID_INPUT (1)
#define IM_TOOL_RETURN_STATUS_FAILED (2)
#define INT16_CAST(X) \
static_cast<int16_t>(::std::min(::std::max(static_cast<int>(X + (X >= 0.f ? 0.5f : -0.5f)), -32768), 32767));

6
mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/warp_affine.cc
View File

@ -205,6 +205,9 @@ static void RemapBilinearNotCurMoreC(int dx, const int16_t *HW, const uint16_t *
static void RemapBilinearCur1C(LiteMat _src, int dx, const int16_t *HW, const uint16_t *FHW, const int16_t *wblock,
size_t src_step, const uint8_t *src_ptr, uint8_t *dst_ptr, PaddBorderType borderType,
const std::vector<uint8_t> &borderValue) {
if (borderValue.size() == 0) {
return;
}
int shx = HW[dx * 2];
int shy = HW[dx * 2 + 1];
if (borderType == PADD_BORDER_CONSTANT && (shx >= _src.width_ || shx + 1 < 0 || shy >= _src.height_ || shy + 1 < 0)) {
@ -233,6 +236,9 @@ static void RemapBilinearCurMoreC(LiteMat _src, int dx, const int16_t *HW, const
PaddBorderType borderType, const std::vector<uint8_t> &borderValue) {
int shx = HW[dx * 2];
int shy = HW[dx * 2 + 1];
if (borderValue.size() < cn || borderValue.size() == 0) {
return;
}
if (borderType == PADD_BORDER_CONSTANT && (shx >= _src.width_ || shx + 1 < 0 || shy >= _src.height_ || shy + 1 < 0)) {
for (int k = 0; k < cn; k++) dst_ptr[k] = borderValue[k];
} else {

17
mindspore/ccsrc/minddata/dataset/kernels/image/lite_image_utils.cc
View File

@ -14,14 +14,17 @@
* limitations under the License.
*/
#include "minddata/dataset/kernels/image/lite_image_utils.h"
#include <algorithm>
#include <vector>
#include <limits>
#include <stdexcept>
#include "minddata/dataset/kernels/image/lite_cv/lite_mat.h"
#include "minddata/dataset/kernels/image/lite_cv/image_process.h"
#include "minddata/dataset/include/dataset/constants.h"
#include <utility>
#include <vector>
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/kernels/image/lite_cv/lite_mat.h"
#include "minddata/dataset/kernels/image/lite_cv/image_process.h"
#include "minddata/dataset/util/random.h"
#define MAX_INT_PRECISION 16777216 // float int precision is 16777216
@ -716,8 +719,8 @@ static bool IsMirror(int orientation) {
}
// rotate the image by EXIF orientation
Status Rotate(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, const uint64_t orientation) {
if (input->Rank() != 3) {
RETURN_STATUS_UNEXPECTED("Rotate: input image is not in shape of <H,W,C>");
if (input->Rank() != 2 || input->Rank() != 3) {
RETURN_STATUS_UNEXPECTED("Rotate: input image is not in shape of <H,W,C> or <H,W>");
}
if (input->type() != DataType::DE_FLOAT32 && input->type() != DataType::DE_UINT8) {

13
mindspore/ccsrc/minddata/dataset/kernels/image/lite_image_utils.h
View File

@ -13,24 +13,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_IMAGE_UTILS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_IMAGE_UTILS_H_
#include <setjmp.h>
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_LITE_IMAGE_UTILS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_LITE_IMAGE_UTILS_H_
#include <csetjmp>
#include <memory>
#include <random>
#include <string>
#include <vector>
#if defined(_WIN32) || defined(_WIN64)
#undef HAVE_STDDEF_H
#undef HAVE_STDLIB_H
#endif
#include "./jpeglib.h"
#include "./jerror.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/kernels/tensor_op.h"
#include "minddata/dataset/kernels/image/lite_cv/image_process.h"
#include "minddata/dataset/kernels/tensor_op.h"
#include "minddata/dataset/util/status.h"
#define MAX_PIXEL_VALUE 255
@ -162,4 +163,4 @@ Status GaussianBlur(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_IMAGE_UTILS_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_LITE_IMAGE_UTILS_H_

6
mindspore/ccsrc/minddata/dataset/kernels/image/math_utils.cc
View File

@ -23,6 +23,9 @@ namespace mindspore {
namespace dataset {
Status ComputeUpperAndLowerPercentiles(std::vector<int32_t> *hist, int32_t hi_p, int32_t low_p, int32_t *hi,
int32_t *lo) {
CHECK_FAIL_RETURN_UNEXPECTED(hist != nullptr, "hist is nullptr");
CHECK_FAIL_RETURN_UNEXPECTED(hi != nullptr, "hi is nullptr");
CHECK_FAIL_RETURN_UNEXPECTED(lo != nullptr, "lo is nullptr");
try {
int32_t n = std::accumulate(hist->begin(), hist->end(), 0);
constexpr float kMaxPerc = 100.0;
@ -61,11 +64,14 @@ Status ComputeUpperAndLowerPercentiles(std::vector<int32_t> *hist, int32_t hi_p,
}
Status DegreesToRadians(float_t degrees, float_t *radians_target) {
CHECK_FAIL_RETURN_UNEXPECTED(radians_target != nullptr, "radians_target is nullptr");
*radians_target = CV_PI * degrees / 180.0;
return Status::OK();
}
Status GenerateRealNumber(float_t a, float_t b, std::mt19937 *rnd, float_t *result) {
CHECK_FAIL_RETURN_UNEXPECTED(rnd != nullptr, "rnd is nullptr");
CHECK_FAIL_RETURN_UNEXPECTED(result != nullptr, "result is nullptr");
try {
std::uniform_real_distribution<float_t> distribution{a, b};
*result = distribution(*rnd);

2
mindspore/ccsrc/minddata/dataset/kernels/image/mixup_batch_op.cc
View File

@ -116,6 +116,8 @@ Status MixUpBatchOp::Compute(const TensorRow &input, TensorRow *output) {
std::gamma_distribution<float> distribution(alpha_, 1);
float x1 = distribution(rnd_);
float x2 = distribution(rnd_);
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() - x1) > x2, "multiplication out of bounds");
CHECK_FAIL_RETURN_UNEXPECTED(x1 + x2 != 0.0, "addition out of bounds");
float lam = x1 / (x1 + x2);
// Calculate random labels

8
mindspore/ccsrc/minddata/dataset/kernels/image/random_affine_op.cc
View File

@ -50,6 +50,14 @@ Status RandomAffineOp::Compute(const std::shared_ptr<Tensor> &input, std::shared
IO_CHECK(input, output);
dsize_t height = input->shape()[0];
dsize_t width = input->shape()[1];
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() / std::abs(translate_range_[0])) > width,
"RandomAffineOp: multiplication out of bounds.");
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() / std::abs(translate_range_[1])) > width,
"RandomAffineOp: multiplication out of bounds.");
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() / std::abs(translate_range_[2])) > height,
"RandomAffineOp: multiplication out of bounds.");
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<float_t>::max() / std::abs(translate_range_[3])) > height,
"RandomAffineOp: multiplication out of bounds.");
float_t min_dx = translate_range_[0] * width;
float_t max_dx = translate_range_[1] * width;
float_t min_dy = translate_range_[2] * height;

8
mindspore/ccsrc/minddata/dataset/kernels/image/random_color_op.cc
View File

@ -52,8 +52,12 @@ Status RandomColorOp::Compute(const std::shared_ptr<Tensor> &in, std::shared_ptr
*out = std::static_pointer_cast<Tensor>(cvt_out);
return Status::OK();
}
// return blended image. addWeighted takes care of overflow for uint8_t
cv::addWeighted(m1, t, cvt_out->mat(), 1 - t, 0, cvt_out->mat());
try {
// return blended image. addWeighted takes care of overflow for uint8_t
cv::addWeighted(m1, t, cvt_out->mat(), 1 - t, 0, cvt_out->mat());
} catch (const cv::Exception &e) {
RETURN_STATUS_UNEXPECTED("RandomColorOp: cv::addWeighted " + std::string(e.what()));
}
*out = std::static_pointer_cast<Tensor>(cvt_out);
return Status::OK();
}

8
mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_op.cc
View File

@ -73,6 +73,8 @@ Status RandomCropAndResizeOp::OutputShape(const std::vector<TensorShape> &inputs
return Status(StatusCode::kMDUnexpectedError, "RandomCropAndResize: invalid input shape");
}
Status RandomCropAndResizeOp::GetCropBox(int h_in, int w_in, int *x, int *y, int *crop_height, int *crop_width) {
CHECK_FAIL_RETURN_UNEXPECTED(crop_height != nullptr, "crop_height is nullptr.");
CHECK_FAIL_RETURN_UNEXPECTED(crop_width != nullptr, "crop_width is nullptr.");
*crop_width = w_in;
*crop_height = h_in;
CHECK_FAIL_RETURN_UNEXPECTED(w_in != 0, "RandomCropAndResize: Width cannot be 0.");
@ -84,6 +86,12 @@ Status RandomCropAndResizeOp::GetCropBox(int h_in, int w_in, int *x, int *y, int
// Note rnd_aspect_ is already a random distribution of the input aspect ratio in logarithmic sample_scale.
double const sample_aspect = exp(rnd_aspect_(rnd_));
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int32_t>::max() / h_in) > w_in,
"RandomCropAndResizeOp: multiplication out of bounds");
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int32_t>::max() / h_in / w_in) > sample_scale,
"RandomCropAndResizeOp: multiplication out of bounds");
CHECK_FAIL_RETURN_UNEXPECTED((std::numeric_limits<int32_t>::max() / h_in / w_in / sample_scale) > sample_aspect,
"RandomCropAndResizeOp: multiplication out of bounds");
*crop_width = static_cast<int32_t>(std::round(std::sqrt(h_in * w_in * sample_scale * sample_aspect)));
*crop_height = static_cast<int32_t>(std::round(*crop_width / sample_aspect));

12
mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_op.cc
View File

@ -121,9 +121,12 @@ Status RandomCropOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_p
// Apply padding first then crop
std::shared_ptr<Tensor> pad_image;
int32_t t_pad_top, t_pad_bottom, t_pad_left, t_pad_right;
int32_t padded_image_w;
int32_t padded_image_h;
int32_t t_pad_top = 0;
int32_t t_pad_bottom = 0;
int32_t t_pad_left = 0;
int32_t t_pad_right = 0;
int32_t padded_image_w = 0;
int32_t padded_image_h = 0;
bool crop_further = true; // whether image needs further cropping based on new size & requirements
RETURN_IF_NOT_OK( // error code sent back directly
@ -134,7 +137,8 @@ Status RandomCropOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_p
return Status::OK();
}
int x, y;
int x = 0;
int y = 0;
GenRandomXY(&x, &y, padded_image_w, padded_image_h);
return Crop(pad_image, output, x, y, crop_width_, crop_height_);
}

11
mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_with_bbox_op.cc
View File

@ -28,12 +28,15 @@ Status RandomCropWithBBoxOp::Compute(const TensorRow &input, TensorRow *output)
IO_CHECK_VECTOR(input, output);
RETURN_IF_NOT_OK(BoundingBox::ValidateBoundingBoxes(input));
std::shared_ptr<Tensor> pad_image;
int32_t t_pad_top, t_pad_bottom, t_pad_left, t_pad_right;
std::shared_ptr<Tensor> pad_image = nullptr;
int32_t t_pad_top = 0;
int32_t t_pad_bottom = 0;
int32_t t_pad_left = 0;
int32_t t_pad_right = 0;
size_t boxCount = input[1]->shape()[0]; // number of rows
int32_t padded_image_h;
int32_t padded_image_w;
int32_t padded_image_h = 0;
int32_t padded_image_w = 0;
const int output_count = 2;
output->resize(output_count);
(*output)[1] = std::move(input[1]); // since some boxes may be removed

1
mindspore/ccsrc/minddata/dataset/kernels/image/random_posterize_op.cc
View File

@ -33,6 +33,7 @@ RandomPosterizeOp::RandomPosterizeOp(const std::vector<uint8_t> &bit_range)
}
Status RandomPosterizeOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
CHECK_FAIL_RETURN_UNEXPECTED(input != nullptr, "RandomPosterizeOp: parameter input is nullptr");
bit_ = (bit_range_[0] == bit_range_[1]) ? bit_range_[0]
: std::uniform_int_distribution<uint8_t>(bit_range_[0], bit_range_[1])(rnd_);
return PosterizeOp::Compute(input, output);

2
mindspore/ccsrc/minddata/dataset/kernels/image/random_resize_with_bbox_op.h
View File

@ -39,7 +39,7 @@ class RandomResizeWithBBoxOp : public ResizeWithBBoxOp {
is_deterministic_ = false;
}
~RandomResizeWithBBoxOp() = default;
~RandomResizeWithBBoxOp() override = default;
// Description: A function that prints info about the node
void Print(std::ostream &out) const override {

2
mindspore/ccsrc/minddata/dataset/kernels/image/random_rotation_op.cc
View File

@ -67,6 +67,8 @@ Status RandomRotationOp::OutputShape(const std::vector<TensorShape> &inputs, std
int32_t outputH = -1, outputW = -1;
// if expand_, then we cannot know the shape. We need the input image to find the output shape --> set it to
// <-1,-1[,3]>
CHECK_FAIL_RETURN_UNEXPECTED(inputs.size() > 0 && inputs[0].Size() >= 2,
"RandomRotationOp::OutputShape inputs is invalid.");
if (!expand_) {
outputH = inputs[0][0];
outputW = inputs[0][1];

6
mindspore/ccsrc/minddata/dataset/kernels/image/random_select_subpolicy_op.cc
View File

@ -44,7 +44,7 @@ Status RandomSelectSubpolicyOp::Compute(const TensorRow &input, TensorRow *outpu
uint32_t RandomSelectSubpolicyOp::NumInput() {
uint32_t num_in = policy_.front().front().first->NumInput();
for (auto &sub : policy_) {
for (auto p : sub) {
for (auto &p : sub) {
if (num_in != p.first->NumInput()) {
MS_LOG(WARNING) << "Unable to determine numInput.";
return 0;
@ -57,7 +57,7 @@ uint32_t RandomSelectSubpolicyOp::NumInput() {
uint32_t RandomSelectSubpolicyOp::NumOutput() {
uint32_t num_out = policy_.front().front().first->NumOutput();
for (auto &sub : policy_) {
for (auto p : sub) {
for (auto &p : sub) {
if (num_out != p.first->NumOutput()) {
MS_LOG(WARNING) << "Unable to determine numInput.";
return 0;
@ -76,7 +76,7 @@ Status RandomSelectSubpolicyOp::OutputShape(const std::vector<TensorShape> &inpu
Status RandomSelectSubpolicyOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {
RETURN_IF_NOT_OK(policy_.front().front().first->OutputType(inputs, outputs));
for (auto &sub : policy_) {
for (auto p : sub) {
for (auto &p : sub) {
std::vector<DataType> tmp_types;
RETURN_IF_NOT_OK(p.first->OutputType(inputs, tmp_types));
if (outputs != tmp_types) {

2
mindspore/ccsrc/minddata/dataset/kernels/image/random_sharpness_op.h
View File

@ -35,7 +35,7 @@ class RandomSharpnessOp : public SharpnessOp {
/// \@param[in] start_degree A float indicating the beginning of the range.
/// \@param[in] end_degree A float indicating the end of the range.
explicit RandomSharpnessOp(float start_degree = kDefStartDegree, const float end_degree = kDefEndDegree);
explicit RandomSharpnessOp(float start_degree = kDefStartDegree, float end_degree = kDefEndDegree);
~RandomSharpnessOp() override = default;
void Print(std::ostream &out) const override { out << Name(); }

2
mindspore/ccsrc/minddata/dataset/kernels/image/random_solarize_op.h
View File

@ -37,7 +37,7 @@ class RandomSolarizeOp : public SolarizeOp {
is_deterministic_ = false;
}
~RandomSolarizeOp() = default;
~RandomSolarizeOp() override = default;
Status Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) override;

2
mindspore/ccsrc/minddata/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc
View File

@ -45,7 +45,7 @@ Status RandomVerticalFlipWithBBoxOp::Compute(const TensorRow &input, TensorRow *
}
const int output_count = 2;
output->resize(output_count);
(*output)[1] = std::move(input[1]);
(*output)[1] = input[1];
return VerticalFlip(input[0], &(*output)[0]);
}

5
mindspore/ccsrc/minddata/dataset/kernels/image/resize_op.cc
View File

@ -29,8 +29,9 @@ const InterpolationMode ResizeOp::kDefInterpolation = InterpolationMode::kLinear
Status ResizeOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
CHECK_FAIL_RETURN_UNEXPECTED(input->shape().Size() >= 2, "Resize: image shape is not <H,W,C> or <H,W>.");
int32_t output_h, output_w = 0;
CHECK_FAIL_RETURN_UNEXPECTED(input->shape().Size() >= 2, "Resize: image shape should be <H,W,C> or <H,W>.");
int32_t output_h = 0;
int32_t output_w = 0;
int32_t input_h = static_cast<int>(input->shape()[0]);
int32_t input_w = static_cast<int>(input->shape()[1]);
if (size2_ == 0) {

3
mindspore/ccsrc/minddata/dataset/kernels/image/rotate_op.cc
View File

@ -45,6 +45,9 @@ RotateOp::RotateOp(float degrees, InterpolationMode resample, bool expand, std::
Status RotateOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
CHECK_FAIL_RETURN_UNEXPECTED(
input->shape().Size() == 2 || input->shape().Size() == 3,
"Rotate: image shape " + std::to_string(input->shape().Size()) + " is not <H,W,C> or <H,W>.");
#ifndef ENABLE_ANDROID
return Rotate(input, output, center_, degrees_, interpolation_, expand_, fill_r_, fill_g_, fill_b_);
#else

10
mindspore/ccsrc/minddata/dataset/kernels/image/soft_dvpp/utils/soft_jpegd.cc
View File

@ -199,6 +199,16 @@ void DestroyLibjpegSource(struct jpeg_decompress_struct *libjpeg_handler, const
uint32_t SoftJpegd::JpegdSoftwareDecodeProcess(struct VpcInfo *vpc_input_info,
struct SoftDpProcsessInfo *soft_dp_process_info) {
if (vpc_input_info == nullptr) {
JPEGD_LOGE("vpc_input_info is nullptr");
return decodeErr;
}
if (soft_dp_process_info == nullptr) {
JPEGD_LOGE("soft_dp_process_info is nullptr");
return decodeErr;
}
int32_t width = 0;
int32_t height = 0;
int32_t sub_sample = 0;

1
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/uniform_aug_ir.cc
View File

@ -61,6 +61,7 @@ std::shared_ptr<TensorOp> UniformAugOperation::Build() {
}
Status UniformAugOperation::to_json(nlohmann::json *out_json) {
CHECK_FAIL_RETURN_UNEXPECTED(out_json != nullptr, "parameter out_json is nullptr");
nlohmann::json args;
std::vector<nlohmann::json> transforms;
for (auto op : transforms_) {

2
mindspore/ccsrc/minddata/dataset/text/ir/kernels/text_ir.cc
View File

@ -323,7 +323,7 @@ Status SentencePieceTokenizerOperation::ValidateParams() {
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (access(vocab_file.toString().c_str(), R_OK) == -1) {
if (access(vocab_file.ToString().c_str(), R_OK) == -1) {
std::string err_msg = "SentencePieceTokenizer : no access to specified dataset file: " + vocab_path_;
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);

2
mindspore/ccsrc/minddata/dataset/text/kernels/sentence_piece_tokenizer_op.cc
View File

@ -105,7 +105,7 @@ Status SentencePieceTokenizerOp::GetModelRealPath(const std::string &model_path,
}
#endif
std::string abs_path = real_path;
file_path_ = (Path(abs_path) / Path(filename)).toString();
file_path_ = (Path(abs_path) / Path(filename)).ToString();
return Status::OK();
}

2
mindspore/ccsrc/minddata/dataset/text/sentence_piece_vocab.cc
View File

@ -105,7 +105,7 @@ Status SentencePieceVocab::SaveModel(const std::shared_ptr<SentencePieceVocab> *
}
#endif
std::string abs_real_path = (Path(real_path) / Path(filename)).toString();
std::string abs_real_path = (Path(real_path) / Path(filename)).ToString();
auto realpath = Common::GetRealPath(abs_real_path);
if (!realpath.has_value()) {
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + abs_real_path);

2
mindspore/ccsrc/minddata/dataset/util/json_helper.cc
View File

@ -48,7 +48,7 @@ Status JsonHelper::CreateAlbum(const std::string &in_dir, const std::string &out
// create json file in output dir with the path
std::string out_file = out_dir + "/" + std::to_string(index) + ".json";
RETURN_IF_NOT_OK(UpdateValue(out_file, "image", v.toString(), out_file));
RETURN_IF_NOT_OK(UpdateValue(out_file, "image", v.ToString(), out_file));
index++;
}
return Status::OK();

14
mindspore/ccsrc/minddata/dataset/util/path.cc
View File

@ -56,7 +56,7 @@ Path &Path::operator=(Path &&p) noexcept {
Path::Path(Path &&p) noexcept { this->path_ = std::move(p.path_); }
Path Path::operator+(const Path &p) {
std::string q = path_ + p.toString();
std::string q = path_ + p.ToString();
return Path(q);
}
@ -71,7 +71,7 @@ Path Path::operator+(const char *p) {
}
Path &Path::operator+=(const Path &rhs) {
path_ += rhs.toString();
path_ += rhs.ToString();
return *this;
}
@ -86,7 +86,7 @@ Path &Path::operator+=(const char *p) {
}
Path Path::operator/(const Path &p) {
std::string q = path_ + separator_ + p.toString();
std::string q = path_ + separator_ + p.ToString();
return Path(q);
}
@ -170,10 +170,10 @@ std::string Path::ParentPath() {
Status Path::CreateDirectories(bool is_common_dir) {
if (IsDirectory()) {
MS_LOG(DEBUG) << "Directory " << toString() << " already exists.";
MS_LOG(DEBUG) << "Directory " << ToString() << " already exists.";
return Status::OK();
} else {
MS_LOG(DEBUG) << "Creating directory " << toString() << ".";
MS_LOG(DEBUG) << "Creating directory " << ToString() << ".";
std::string parent = ParentPath();
if (!parent.empty()) {
if (Path(parent).CreateDirectories(is_common_dir)) {
@ -319,8 +319,8 @@ Path::DirIterator::~DirIterator() {
}
Path::DirIterator::DirIterator(Path *f) : dir_(f), dp_(nullptr), entry_(nullptr) {
MS_LOG(DEBUG) << "Open directory " << f->toString() << ".";
dp_ = opendir(f->toString().c_str());
MS_LOG(DEBUG) << "Open directory " << f->ToString() << ".";
dp_ = opendir(f->ToString().c_str());
}
bool Path::DirIterator::HasNext() {

2
mindspore/ccsrc/minddata/dataset/util/path.h
View File

@ -58,7 +58,7 @@ class Path {
Path &operator=(Path &&) noexcept;
std::string toString() const { return path_; }
std::string ToString() const { return path_; }
Path operator+(const Path &);

2
mindspore/dataset/text/transforms.py
View File

@ -248,7 +248,7 @@ class JiebaTokenizer(TextTensorOperation):
def __check_path__(self, model_path):
"""check model path"""
if not os.path.exists(model_path):
if not os.path.exists(os.path.realpath(model_path)):
raise ValueError(
" jieba mode file {} is not exist.".format(model_path))

2
tests/ut/cpp/dataset/common/bboxop_common.cc
View File

@ -70,7 +70,7 @@ void BBoxOpCommon::GetInputImagesAndAnnotations(const std::string &dir, std::siz
while (image_dir_itr->HasNext()) {
Path image_path = image_dir_itr->Next();
if (image_path.Extension() == std::string(kImageExt)) {
paths_to_fetch.push_back(image_path.toString());
paths_to_fetch.push_back(image_path.ToString());
}
}
// sort fetched files

2
tests/ut/cpp/dataset/path_test.cc
View File

@ -37,7 +37,7 @@ TEST_F(MindDataTestPath, Test1) {
int i = 0;
while (dir_it->HasNext()) {
Path v = dir_it->Next();
MS_LOG(DEBUG) << v.toString() << "\n";
MS_LOG(DEBUG) << v.ToString() << "\n";
i++;
if (i == 10) {
break;