forked from huawei/mindspore2022
[MD] Decouple more Transform Ops
This commit is contained in:
luoyang
parent
e8c8b40e5a
commit
d43155b5b0
|
|
@ -17,53 +17,202 @@
|
|||
#include "minddata/dataset/include/transforms.h"
|
||||
|
||||
// Kernel data headers (in alphabetical order)
|
||||
#include "minddata/dataset/kernels/data/compose_op.h"
|
||||
#include "minddata/dataset/kernels/data/duplicate_op.h"
|
||||
#include "minddata/dataset/kernels/data/one_hot_op.h"
|
||||
#include "minddata/dataset/kernels/data/random_apply_op.h"
|
||||
#include "minddata/dataset/kernels/data/random_choice_op.h"
|
||||
#include "minddata/dataset/kernels/data/type_cast_op.h"
|
||||
#ifndef ENABLE_ANDROID
|
||||
#include "minddata/dataset/kernels/data/unique_op.h"
|
||||
#endif
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
TensorOperation::TensorOperation() {}
|
||||
|
||||
/* ####################################### Validator Functions ############################################ */
|
||||
Status ValidateVectorFillvalue(const std::string &transform_name, const std::vector<uint8_t> &fill_value) {
|
||||
if (fill_value.empty() || (fill_value.size() != 1 && fill_value.size() != 3)) {
|
||||
std::string err_msg =
|
||||
transform_name + ": fill_value vector has incorrect size: " + std::to_string(fill_value.size());
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
for (uint8_t single_fill_value : fill_value) {
|
||||
if (single_fill_value > 255) {
|
||||
std::string err_msg =
|
||||
transform_name + ": fill_value has to be between 0 and 255, got:" + std::to_string(single_fill_value);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ValidateProbability(const std::string &transform_name, const float &probability) {
|
||||
if (probability < 0.0 || probability > 1.0) {
|
||||
std::string err_msg =
|
||||
transform_name + ": probability must be between 0.0 and 1.0, got: " + std::to_string(probability);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ValidateVectorPadding(const std::string &transform_name, const std::vector<int32_t> &padding) {
|
||||
if (padding.empty() || padding.size() == 3 || padding.size() > 4) {
|
||||
std::string err_msg = transform_name + ": padding vector has incorrect size: " + std::to_string(padding.size());
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
for (int32_t i = 0; i < padding.size(); ++i) {
|
||||
if (padding[i] < 0) {
|
||||
std::string err_msg =
|
||||
transform_name +
|
||||
": invalid padding, padding value must be greater than or equal to 0, got: " + std::to_string(padding[i]);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
if (padding[i] == INT_MAX) {
|
||||
std::string err_msg =
|
||||
transform_name + ": invalid padding, padding value too large, got: " + std::to_string(padding[i]);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ValidateVectorPositive(const std::string &transform_name, const std::vector<int32_t> &size) {
|
||||
for (int32_t i = 0; i < size.size(); ++i) {
|
||||
if (size[i] <= 0) {
|
||||
std::string err_msg =
|
||||
transform_name + ": Non-positive size value: " + std::to_string(size[i]) + " at element: " + std::to_string(i);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ValidateVectorTransforms(const std::string &transform_name,
|
||||
const std::vector<std::shared_ptr<TensorOperation>> &transforms) {
|
||||
if (transforms.empty()) {
|
||||
std::string err_msg = transform_name + ": transform list must not be empty.";
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
for (int32_t i = 0; i < transforms.size(); ++i) {
|
||||
if (transforms[i] == nullptr) {
|
||||
std::string err_msg =
|
||||
transform_name + ": transform ops must not be null, got transform[" + std::to_string(i) + "] == nullptr.";
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
bool CmpFloat(const float &a, const float &b, float epsilon) { return (std::fabs(a - b) < epsilon); }
|
||||
|
||||
// Transform operations for data.
|
||||
namespace transforms {
|
||||
|
||||
// FUNCTIONS TO CREATE DATA TRANSFORM OPERATIONS
|
||||
// (In alphabetical order)
|
||||
|
||||
// Function to create ComposeOperation.
|
||||
std::shared_ptr<ComposeOperation> Compose(const std::vector<std::shared_ptr<TensorOperation>> &transforms) {
|
||||
auto op = std::make_shared<ComposeOperation>(transforms);
|
||||
// Input validation
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
|
||||
// Function to create DuplicateOperation.
|
||||
std::shared_ptr<DuplicateOperation> Duplicate() {
|
||||
auto op = std::make_shared<DuplicateOperation>();
|
||||
// Input validation
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
|
||||
// Function to create OneHotOperation.
|
||||
std::shared_ptr<OneHotOperation> OneHot(int32_t num_classes) {
|
||||
auto op = std::make_shared<OneHotOperation>(num_classes);
|
||||
// Input validation
|
||||
if (!op->ValidateParams()) {
|
||||
return nullptr;
|
||||
}
|
||||
return op;
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
|
||||
// Function to create RandomApplyOperation.
|
||||
std::shared_ptr<RandomApplyOperation> RandomApply(const std::vector<std::shared_ptr<TensorOperation>> &transforms,
|
||||
double prob) {
|
||||
auto op = std::make_shared<RandomApplyOperation>(transforms, prob);
|
||||
// Input validation
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
|
||||
// Function to create RandomChoiceOperation.
|
||||
std::shared_ptr<RandomChoiceOperation> RandomChoice(const std::vector<std::shared_ptr<TensorOperation>> &transforms) {
|
||||
auto op = std::make_shared<RandomChoiceOperation>(transforms);
|
||||
// Input validation
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
|
||||
// Function to create TypeCastOperation.
|
||||
std::shared_ptr<TypeCastOperation> TypeCast(std::string data_type) {
|
||||
auto op = std::make_shared<TypeCastOperation>(data_type);
|
||||
// Input validation
|
||||
if (!op->ValidateParams()) {
|
||||
return nullptr;
|
||||
}
|
||||
return op;
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
|
||||
#ifndef ENABLE_ANDROID
|
||||
// Function to create UniqueOperation.
|
||||
std::shared_ptr<UniqueOperation> Unique() {
|
||||
auto op = std::make_shared<UniqueOperation>();
|
||||
// Input validation
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ####################################### Validator Functions ############################################ */
|
||||
|
||||
/* ####################################### Derived TensorOperation classes ################################# */
|
||||
|
||||
// (In alphabetical order)
|
||||
|
||||
// ComposeOperation
|
||||
ComposeOperation::ComposeOperation(const std::vector<std::shared_ptr<TensorOperation>> &transforms)
|
||||
: transforms_(transforms) {}
|
||||
|
||||
Status ComposeOperation::ValidateParams() {
|
||||
RETURN_IF_NOT_OK(ValidateVectorTransforms("Compose", transforms_));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOp> ComposeOperation::Build() {
|
||||
std::vector<std::shared_ptr<TensorOp>> tensor_ops;
|
||||
(void)std::transform(transforms_.begin(), transforms_.end(), std::back_inserter(tensor_ops),
|
||||
[](std::shared_ptr<TensorOperation> op) -> std::shared_ptr<TensorOp> { return op->Build(); });
|
||||
return std::make_shared<ComposeOp>(tensor_ops);
|
||||
}
|
||||
|
||||
// DuplicateOperation
|
||||
Status DuplicateOperation::ValidateParams() { return Status::OK(); }
|
||||
|
||||
std::shared_ptr<TensorOp> DuplicateOperation::Build() { return std::make_shared<DuplicateOp>(); }
|
||||
|
||||
// OneHotOperation
|
||||
OneHotOperation::OneHotOperation(int32_t num_classes) : num_classes_(num_classes) {}
|
||||
|
||||
Status OneHotOperation::ValidateParams() {
|
||||
if (num_classes_ <= 0) {
|
||||
std::string err_msg =
|
||||
"OneHot: Number of classes must be greater than 0. num_classes: " + std::to_string(num_classes_);
|
||||
std::string err_msg = "OneHot: Number of classes must be greater than 0, but got: " + std::to_string(num_classes_);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
|
|
@ -73,6 +222,39 @@ Status OneHotOperation::ValidateParams() {
|
|||
|
||||
std::shared_ptr<TensorOp> OneHotOperation::Build() { return std::make_shared<OneHotOp>(num_classes_); }
|
||||
|
||||
// RandomApplyOperation
|
||||
RandomApplyOperation::RandomApplyOperation(const std::vector<std::shared_ptr<TensorOperation>> &transforms, double prob)
|
||||
: transforms_(transforms), prob_(prob) {}
|
||||
|
||||
Status RandomApplyOperation::ValidateParams() {
|
||||
RETURN_IF_NOT_OK(ValidateVectorTransforms("RandomApply", transforms_));
|
||||
RETURN_IF_NOT_OK(ValidateProbability("RandomApply", prob_));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOp> RandomApplyOperation::Build() {
|
||||
std::vector<std::shared_ptr<TensorOp>> tensor_ops;
|
||||
(void)std::transform(transforms_.begin(), transforms_.end(), std::back_inserter(tensor_ops),
|
||||
[](std::shared_ptr<TensorOperation> op) -> std::shared_ptr<TensorOp> { return op->Build(); });
|
||||
return std::make_shared<RandomApplyOp>(prob_, tensor_ops);
|
||||
}
|
||||
|
||||
// RandomChoiceOperation
|
||||
RandomChoiceOperation::RandomChoiceOperation(const std::vector<std::shared_ptr<TensorOperation>> &transforms)
|
||||
: transforms_(transforms) {}
|
||||
|
||||
Status RandomChoiceOperation::ValidateParams() {
|
||||
RETURN_IF_NOT_OK(ValidateVectorTransforms("RandomChoice", transforms_));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOp> RandomChoiceOperation::Build() {
|
||||
std::vector<std::shared_ptr<TensorOp>> tensor_ops;
|
||||
(void)std::transform(transforms_.begin(), transforms_.end(), std::back_inserter(tensor_ops),
|
||||
[](std::shared_ptr<TensorOperation> op) -> std::shared_ptr<TensorOp> { return op->Build(); });
|
||||
return std::make_shared<RandomChoiceOp>(tensor_ops);
|
||||
}
|
||||
|
||||
// TypeCastOperation
|
||||
TypeCastOperation::TypeCastOperation(std::string data_type) : data_type_(data_type) {}
|
||||
|
||||
|
|
@ -83,7 +265,7 @@ Status TypeCastOperation::ValidateParams() {
|
|||
if (itr == predefine_type.end()) {
|
||||
std::string err_msg = "TypeCast: Invalid data type: " + data_type_;
|
||||
MS_LOG(ERROR) << "TypeCast: Only supports data type bool, int8, uint8, int16, uint16, int32, uint32, "
|
||||
<< "int64, uint64, float16, float32, float64, string, but got " << data_type_;
|
||||
<< "int64, uint64, float16, float32, float64, string, but got: " << data_type_;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
|
||||
|
|
@ -92,6 +274,13 @@ Status TypeCastOperation::ValidateParams() {
|
|||
|
||||
std::shared_ptr<TensorOp> TypeCastOperation::Build() { return std::make_shared<TypeCastOp>(data_type_); }
|
||||
|
||||
#ifndef ENABLE_ANDROID
|
||||
// UniqueOperation
|
||||
Status UniqueOperation::ValidateParams() { return Status::OK(); }
|
||||
|
||||
std::shared_ptr<TensorOp> UniqueOperation::Build() { return std::make_shared<UniqueOp>(); }
|
||||
#endif
|
||||
|
||||
} // namespace transforms
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
|
|
|||
|
|
@ -394,77 +394,7 @@ std::shared_ptr<UniformAugOperation> UniformAugment(std::vector<std::shared_ptr<
|
|||
// Input validation
|
||||
return op->ValidateParams() ? op : nullptr;
|
||||
}
|
||||
|
||||
#endif
|
||||
/* ####################################### Validator Functions ############################################ */
|
||||
Status ValidateVectorFillvalue(const std::string &dataset_name, const std::vector<uint8_t> &fill_value) {
|
||||
if (fill_value.empty() || (fill_value.size() != 1 && fill_value.size() != 3)) {
|
||||
std::string err_msg = dataset_name + ": fill_value vector has incorrect size: " + std::to_string(fill_value.size());
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
for (uint8_t single_fill_value : fill_value) {
|
||||
if (single_fill_value > 255) {
|
||||
std::string err_msg =
|
||||
dataset_name + ": fill_value has to be between 0 and 255, got:" + std::to_string(single_fill_value);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ValidateVectorProbability(const std::string &dataset_name, const float &probability) {
|
||||
if (probability < 0.0 || probability > 1.0) {
|
||||
std::string err_msg =
|
||||
dataset_name + ": probability must be between 0.0 and 1.0, got: " + std::to_string(probability);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ValidateVectorPadding(const std::string &dataset_name, const std::vector<int32_t> &padding) {
|
||||
if (padding.empty() || padding.size() == 3 || padding.size() > 4) {
|
||||
std::string err_msg = dataset_name + ": padding vector has incorrect size: " + std::to_string(padding.size());
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
for (int32_t i = 0; i < padding.size(); ++i) {
|
||||
if (padding[i] < 0) {
|
||||
std::string err_msg =
|
||||
dataset_name +
|
||||
": invalid padding, padding value must be greater than or equal to 0, got: " + std::to_string(padding[i]);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
if (padding[i] == INT_MAX) {
|
||||
std::string err_msg =
|
||||
dataset_name + ": invalid padding, padding value too large, got: " + std::to_string(padding[i]);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ValidateVectorPositive(const std::string &dataset_name, const std::vector<int32_t> &size) {
|
||||
for (int32_t i = 0; i < size.size(); ++i) {
|
||||
if (size[i] <= 0) {
|
||||
std::string err_msg =
|
||||
dataset_name + ": Non-positive size value: " + std::to_string(size[i]) + " at element: " + std::to_string(i);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
bool CmpFloat(const float &a, const float &b, float epsilon = 0.0000000001f) { return (std::fabs(a - b) < epsilon); }
|
||||
|
||||
/* ####################################### Derived TensorOperation classes ################################# */
|
||||
|
||||
|
|
@ -503,17 +433,8 @@ BoundingBoxAugmentOperation::BoundingBoxAugmentOperation(std::shared_ptr<TensorO
|
|||
: transform_(transform), ratio_(ratio) {}
|
||||
|
||||
Status BoundingBoxAugmentOperation::ValidateParams() {
|
||||
if (transform_ == nullptr) {
|
||||
std::string err_msg = "BoundingBoxAugment: transform must not be null.";
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
|
||||
if (ratio_ < 0.0 || ratio_ > 1.0) {
|
||||
std::string err_msg = "BoundingBoxAugment: ratio has to be between 0.0 and 1.0, got: " + std::to_string(ratio_);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
RETURN_IF_NOT_OK(ValidateVectorTransforms("BoundingBoxAugment", {transform_}));
|
||||
RETURN_IF_NOT_OK(ValidateProbability("BoundingBoxAugment", ratio_));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1315,7 +1236,7 @@ std::shared_ptr<TensorOp> RandomCropWithBBoxOperation::Build() {
|
|||
RandomHorizontalFlipOperation::RandomHorizontalFlipOperation(float probability) : probability_(probability) {}
|
||||
|
||||
Status RandomHorizontalFlipOperation::ValidateParams() {
|
||||
RETURN_IF_NOT_OK(ValidateVectorProbability("RandomHorizontalFlip", probability_));
|
||||
RETURN_IF_NOT_OK(ValidateProbability("RandomHorizontalFlip", probability_));
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
|
@ -1330,7 +1251,7 @@ RandomHorizontalFlipWithBBoxOperation::RandomHorizontalFlipWithBBoxOperation(flo
|
|||
: probability_(probability) {}
|
||||
|
||||
Status RandomHorizontalFlipWithBBoxOperation::ValidateParams() {
|
||||
RETURN_IF_NOT_OK(ValidateVectorProbability("RandomHorizontalFlipWithBBox", probability_));
|
||||
RETURN_IF_NOT_OK(ValidateProbability("RandomHorizontalFlipWithBBox", probability_));
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
|
@ -1696,7 +1617,7 @@ std::shared_ptr<TensorOp> RandomSolarizeOperation::Build() {
|
|||
RandomVerticalFlipOperation::RandomVerticalFlipOperation(float probability) : probability_(probability) {}
|
||||
|
||||
Status RandomVerticalFlipOperation::ValidateParams() {
|
||||
RETURN_IF_NOT_OK(ValidateVectorProbability("RandomVerticalFlip", probability_));
|
||||
RETURN_IF_NOT_OK(ValidateProbability("RandomVerticalFlip", probability_));
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
|
@ -1711,7 +1632,7 @@ RandomVerticalFlipWithBBoxOperation::RandomVerticalFlipWithBBoxOperation(float p
|
|||
: probability_(probability) {}
|
||||
|
||||
Status RandomVerticalFlipWithBBoxOperation::ValidateParams() {
|
||||
RETURN_IF_NOT_OK(ValidateVectorProbability("RandomVerticalFlipWithBBox", probability_));
|
||||
RETURN_IF_NOT_OK(ValidateProbability("RandomVerticalFlipWithBBox", probability_));
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
|
@ -1945,21 +1866,15 @@ UniformAugOperation::UniformAugOperation(std::vector<std::shared_ptr<TensorOpera
|
|||
|
||||
Status UniformAugOperation::ValidateParams() {
|
||||
// transforms
|
||||
RETURN_IF_NOT_OK(ValidateVectorTransforms("UniformAug", transforms_));
|
||||
if (num_ops_ > transforms_.size()) {
|
||||
std::string err_msg = "UniformAug: num_ops is greater than transforms size, num_ops: " + std::to_string(num_ops_);
|
||||
std::string err_msg = "UniformAug: num_ops is greater than transforms size, but got: " + std::to_string(num_ops_);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
for (int32_t i = 0; i < transforms_.size(); ++i) {
|
||||
if (transforms_[i] == nullptr) {
|
||||
std::string err_msg = "UniformAug: transform ops must not be null.";
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
}
|
||||
// num_ops
|
||||
if (num_ops_ <= 0) {
|
||||
std::string err_msg = "UniformAug: num_ops must be greater than 0, num_ops: " + std::to_string(num_ops_);
|
||||
std::string err_msg = "UniformAug: num_ops must be greater than 0, but got: " + std::to_string(num_ops_);
|
||||
MS_LOG(ERROR) << err_msg;
|
||||
RETURN_STATUS_SYNTAX_ERROR(err_msg);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -44,12 +44,50 @@ class TensorOperation : public std::enable_shared_from_this<TensorOperation> {
|
|||
virtual Status ValidateParams() = 0;
|
||||
};
|
||||
|
||||
// Helper function to validate fill value
|
||||
Status ValidateVectorFillvalue(const std::string &transform_name, const std::vector<uint8_t> &fill_value);
|
||||
|
||||
// Helper function to validate probability
|
||||
Status ValidateProbability(const std::string &transform_name, const float &probability);
|
||||
|
||||
// Helper function to validate padding
|
||||
Status ValidateVectorPadding(const std::string &transform_name, const std::vector<int32_t> &padding);
|
||||
|
||||
// Helper function to validate size
|
||||
Status ValidateVectorPositive(const std::string &transform_name, const std::vector<int32_t> &size);
|
||||
|
||||
// Helper function to validate transforms
|
||||
Status ValidateVectorTransforms(const std::string &transform_name,
|
||||
const std::vector<std::shared_ptr<TensorOperation>> &transforms);
|
||||
|
||||
// Helper function to compare float value
|
||||
bool CmpFloat(const float &a, const float &b, float epsilon = 0.0000000001f);
|
||||
|
||||
// Transform operations for performing data transformation.
|
||||
namespace transforms {
|
||||
|
||||
// Transform Op classes (in alphabetical order)
|
||||
class ComposeOperation;
|
||||
class DuplicateOperation;
|
||||
class OneHotOperation;
|
||||
class RandomApplyOperation;
|
||||
class RandomChoiceOperation;
|
||||
class TypeCastOperation;
|
||||
#ifndef ENABLE_ANDROID
|
||||
class UniqueOperation;
|
||||
#endif
|
||||
|
||||
/// \brief Function to create a Compose TensorOperation.
|
||||
/// \notes Compose a list of transforms into a single transform.
|
||||
/// \param[in] transforms A vector of transformations to be applied.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<ComposeOperation> Compose(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
|
||||
|
||||
/// \brief Function to create a Duplicate TensorOperation.
|
||||
/// \notes Duplicate the input tensor to a new output tensor.
|
||||
/// The input tensor is carried over to the output list.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<DuplicateOperation> Duplicate();
|
||||
|
||||
/// \brief Function to create a OneHot TensorOperation.
|
||||
/// \notes Convert the labels into OneHot format.
|
||||
|
|
@ -57,14 +95,61 @@ class TypeCastOperation;
|
|||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<OneHotOperation> OneHot(int32_t num_classes);
|
||||
|
||||
/// \brief Function to create a RandomApply TensorOperation.
|
||||
/// \notes Randomly perform a series of transforms with a given probability.
|
||||
/// \param[in] transforms A vector of transformations to be applied.
|
||||
/// \param[in] prob The probability to apply the transformation list (default=0.5)
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<RandomApplyOperation> RandomApply(const std::vector<std::shared_ptr<TensorOperation>> &transforms,
|
||||
double prob = 0.5);
|
||||
|
||||
/// \brief Function to create a RandomChoice TensorOperation.
|
||||
/// \notes Randomly selects one transform from a list of transforms to perform operation.
|
||||
/// \param[in] transforms A vector of transformations to be chosen from to apply.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<RandomChoiceOperation> RandomChoice(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
|
||||
|
||||
/// \brief Function to create a TypeCast TensorOperation.
|
||||
/// \notes Tensor operation to cast to a given MindSpore data type.
|
||||
/// \param[in] data_type mindspore.dtype to be cast to.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<TypeCastOperation> TypeCast(std::string data_type);
|
||||
|
||||
#ifndef ENABLE_ANDROID
|
||||
/// \brief Function to create a Unique TensorOperation.
|
||||
/// \notes Return an output tensor containing all the unique elements of the input tensor in
|
||||
/// the same order that they occur in the input tensor.
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<UniqueOperation> Unique();
|
||||
#endif
|
||||
|
||||
/* ####################################### Derived TensorOperation classes ################################# */
|
||||
|
||||
class ComposeOperation : public TensorOperation {
|
||||
public:
|
||||
explicit ComposeOperation(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
|
||||
|
||||
~ComposeOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
};
|
||||
|
||||
class DuplicateOperation : public TensorOperation {
|
||||
public:
|
||||
DuplicateOperation() = default;
|
||||
|
||||
~DuplicateOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
};
|
||||
|
||||
class OneHotOperation : public TensorOperation {
|
||||
public:
|
||||
explicit OneHotOperation(int32_t num_classes_);
|
||||
|
|
@ -79,6 +164,35 @@ class OneHotOperation : public TensorOperation {
|
|||
float num_classes_;
|
||||
};
|
||||
|
||||
class RandomApplyOperation : public TensorOperation {
|
||||
public:
|
||||
explicit RandomApplyOperation(const std::vector<std::shared_ptr<TensorOperation>> &transforms, double prob);
|
||||
|
||||
~RandomApplyOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
double prob_;
|
||||
};
|
||||
|
||||
class RandomChoiceOperation : public TensorOperation {
|
||||
public:
|
||||
explicit RandomChoiceOperation(const std::vector<std::shared_ptr<TensorOperation>> &transforms);
|
||||
|
||||
~RandomChoiceOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
|
||||
private:
|
||||
std::vector<std::shared_ptr<TensorOperation>> transforms_;
|
||||
};
|
||||
|
||||
class TypeCastOperation : public TensorOperation {
|
||||
public:
|
||||
explicit TypeCastOperation(std::string data_type);
|
||||
|
|
@ -92,6 +206,19 @@ class TypeCastOperation : public TensorOperation {
|
|||
private:
|
||||
std::string data_type_;
|
||||
};
|
||||
|
||||
#ifndef ENABLE_ANDROID
|
||||
class UniqueOperation : public TensorOperation {
|
||||
public:
|
||||
UniqueOperation() = default;
|
||||
|
||||
~UniqueOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
Status ValidateParams() override;
|
||||
};
|
||||
#endif
|
||||
} // namespace transforms
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
|
|
|||
|
|
@ -28,6 +28,107 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
|
|||
|
||||
// Tests for data transforms ops (in alphabetical order)
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestComposeSuccess) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeSuccess.";
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, RandomSampler(false, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
std::shared_ptr<TensorOperation> compose = transforms::Compose({vision::Decode(), vision::Resize({777, 777})});
|
||||
EXPECT_NE(compose, nullptr);
|
||||
|
||||
// Create a Map operation on ds
|
||||
ds = ds->Map({compose}, {"image"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
i++;
|
||||
auto image = row["image"];
|
||||
auto label = row["label"];
|
||||
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
|
||||
MS_LOG(INFO) << "Label shape: " << label->shape();
|
||||
EXPECT_EQ(image->shape()[0], 777);
|
||||
EXPECT_EQ(image->shape()[1], 777);
|
||||
iter->GetNextRow(&row);
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 3);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestComposeFail) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail with invalid transform.";
|
||||
|
||||
// Resize: Non-positive size value: -1 at element: 0
|
||||
// Compose: transform ops must not be null
|
||||
std::shared_ptr<TensorOperation> compose1 = transforms::Compose({vision::Decode(), vision::Resize({-1})});
|
||||
EXPECT_EQ(compose1, nullptr);
|
||||
|
||||
// Compose: transform ops must not be null
|
||||
std::shared_ptr<TensorOperation> compose2 = transforms::Compose({vision::Decode(), nullptr});
|
||||
EXPECT_EQ(compose2, nullptr);
|
||||
|
||||
// Compose: transform list must not be empty
|
||||
std::shared_ptr<TensorOperation> compose3 = transforms::Compose({});
|
||||
EXPECT_EQ(compose3, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestDuplicateSuccess) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDuplicateSuccess.";
|
||||
|
||||
// Create a Cifar10 Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
std::shared_ptr<Dataset> ds = Cifar10(folder_path, "all", RandomSampler(false, 10));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
std::shared_ptr<TensorOperation> duplicate = transforms::Duplicate();
|
||||
EXPECT_NE(duplicate, nullptr);
|
||||
|
||||
// Create a Map operation on ds
|
||||
ds = ds->Map({duplicate}, {"image"}, {"image", "image_copy"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
i++;
|
||||
auto image = row["image"];
|
||||
auto image_copy = row["image_copy"];
|
||||
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
|
||||
EXPECT_EQ(*image, *image_copy);
|
||||
iter->GetNextRow(&row);
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 10);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestOneHotSuccess1) {
|
||||
// Testing CutMixBatch on a batch of CHW images
|
||||
// Create a Cifar10 Dataset
|
||||
|
|
@ -157,6 +258,127 @@ TEST_F(MindDataTestPipeline, TestOneHotFail) {
|
|||
EXPECT_EQ(one_hot_op2, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestRandomApplySuccess) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplySuccess.";
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 5));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
std::shared_ptr<TensorOperation> random_apply = transforms::RandomApply({vision::Resize({777, 777})}, 0.8);
|
||||
EXPECT_NE(random_apply, nullptr);
|
||||
|
||||
// Create a Map operation on ds
|
||||
ds = ds->Map({random_apply}, {"image"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
i++;
|
||||
auto image = row["image"];
|
||||
auto label = row["label"];
|
||||
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
|
||||
MS_LOG(INFO) << "Label shape: " << label->shape();
|
||||
iter->GetNextRow(&row);
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 5);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestRandomApplyFail) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail with invalid transform.";
|
||||
|
||||
// Resize: Non-positive size value: -1 at element: 0
|
||||
// RandomApply: transform ops must not be null
|
||||
std::shared_ptr<TensorOperation> random_apply1 = transforms::RandomApply({vision::Decode(), vision::Resize({-1})});
|
||||
EXPECT_EQ(random_apply1, nullptr);
|
||||
|
||||
// RandomApply: transform ops must not be null
|
||||
std::shared_ptr<TensorOperation> random_apply2 = transforms::RandomApply({vision::Decode(), nullptr});
|
||||
EXPECT_EQ(random_apply2, nullptr);
|
||||
|
||||
// RandomApply: transform list must not be empty
|
||||
std::shared_ptr<TensorOperation> random_apply3 = transforms::RandomApply({});
|
||||
EXPECT_EQ(random_apply3, nullptr);
|
||||
|
||||
// RandomApply: Probability has to be between 0 and 1
|
||||
std::shared_ptr<TensorOperation> random_apply4 = transforms::RandomApply({vision::Resize({100})}, -1);
|
||||
EXPECT_EQ(random_apply4, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceSuccess.";
|
||||
|
||||
// Create an ImageFolder Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data/";
|
||||
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 3));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
std::shared_ptr<TensorOperation> random_choice =
|
||||
transforms::RandomChoice({vision::Resize({777, 777}), vision::Resize({888, 888})});
|
||||
EXPECT_NE(random_choice, nullptr);
|
||||
|
||||
// Create a Map operation on ds
|
||||
ds = ds->Map({random_choice}, {"image"});
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
// Create an iterator over the result of the above dataset
|
||||
// This will trigger the creation of the Execution Tree and launch it.
|
||||
std::shared_ptr<Iterator> iter = ds->CreateIterator();
|
||||
EXPECT_NE(iter, nullptr);
|
||||
|
||||
// Iterate the dataset and get each row
|
||||
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
|
||||
iter->GetNextRow(&row);
|
||||
|
||||
uint64_t i = 0;
|
||||
while (row.size() != 0) {
|
||||
i++;
|
||||
auto image = row["image"];
|
||||
auto label = row["label"];
|
||||
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
|
||||
MS_LOG(INFO) << "Label shape: " << label->shape();
|
||||
iter->GetNextRow(&row);
|
||||
}
|
||||
|
||||
EXPECT_EQ(i, 3);
|
||||
|
||||
// Manually terminate the pipeline
|
||||
iter->Stop();
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestRandomChoiceFail) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail with invalid transform.";
|
||||
|
||||
// Resize: Non-positive size value: -1 at element: 0
|
||||
// RandomChoice: transform ops must not be null
|
||||
std::shared_ptr<TensorOperation> random_choice1 = transforms::RandomChoice({vision::Decode(), vision::Resize({-1})});
|
||||
EXPECT_EQ(random_choice1, nullptr);
|
||||
|
||||
// RandomChoice: transform ops must not be null
|
||||
std::shared_ptr<TensorOperation> random_choice2 = transforms::RandomChoice({vision::Decode(), nullptr});
|
||||
EXPECT_EQ(random_choice2, nullptr);
|
||||
|
||||
// RandomChoice: transform list must not be empty
|
||||
std::shared_ptr<TensorOperation> random_choice3 = transforms::RandomChoice({});
|
||||
EXPECT_EQ(random_choice3, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestTypeCastSuccess) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTypeCastSuccess.";
|
||||
|
||||
|
|
|
|||
|
|
@ -2799,14 +2799,8 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegFail) {
|
|||
EXPECT_EQ(soft_dvpp_decode_resize_jpeg_op4, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, DISABLED_TestUniformAugmentFail1) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUniformAugmentFail1 with invalid zero num_ops parameter.";
|
||||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 20));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestUniformAugmentFail1) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUniformAugmentFail1 with invalid num_ops parameter.";
|
||||
// Create objects for the tensor ops
|
||||
std::shared_ptr<TensorOperation> random_crop_op = vision::RandomCrop({28, 28});
|
||||
EXPECT_NE(random_crop_op, nullptr);
|
||||
|
|
@ -2814,29 +2808,33 @@ TEST_F(MindDataTestPipeline, DISABLED_TestUniformAugmentFail1) {
|
|||
std::shared_ptr<TensorOperation> center_crop_op = vision::CenterCrop({16, 16});
|
||||
EXPECT_NE(center_crop_op, nullptr);
|
||||
|
||||
// Try UniformAugment with invalid zero num_ops value
|
||||
std::shared_ptr<TensorOperation> uniform_aug_op = vision::UniformAugment({random_crop_op, center_crop_op}, 0);
|
||||
EXPECT_EQ(uniform_aug_op, nullptr);
|
||||
// UniformAug: num_ops must be greater than 0
|
||||
std::shared_ptr<TensorOperation> uniform_aug_op1 = vision::UniformAugment({random_crop_op, center_crop_op}, 0);
|
||||
EXPECT_EQ(uniform_aug_op1, nullptr);
|
||||
|
||||
// UniformAug: num_ops must be greater than 0
|
||||
std::shared_ptr<TensorOperation> uniform_aug_op2 = vision::UniformAugment({random_crop_op, center_crop_op}, -1);
|
||||
EXPECT_EQ(uniform_aug_op2, nullptr);
|
||||
|
||||
// UniformAug: num_ops is greater than transforms size
|
||||
std::shared_ptr<TensorOperation> uniform_aug_op3 = vision::UniformAugment({random_crop_op, center_crop_op}, 3);
|
||||
EXPECT_EQ(uniform_aug_op3, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, DISABLED_TestUniformAugmentFail2) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUniformAugmentFail2 with invalid negative num_ops parameter.";
|
||||
TEST_F(MindDataTestPipeline, TestUniformAugmentFail2) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestUniformAugmentFail2 with invalid transform.";
|
||||
|
||||
// Create a Mnist Dataset
|
||||
std::string folder_path = datasets_root_path_ + "/testMnistData/";
|
||||
std::shared_ptr<Dataset> ds = Mnist(folder_path, "all", RandomSampler(false, 20));
|
||||
EXPECT_NE(ds, nullptr);
|
||||
// UniformAug: transform ops must not be null
|
||||
std::shared_ptr<TensorOperation> uniform_aug_op1 = vision::UniformAugment({vision::RandomCrop({-28})}, 1);
|
||||
EXPECT_EQ(uniform_aug_op1, nullptr);
|
||||
|
||||
// Create objects for the tensor ops
|
||||
std::shared_ptr<TensorOperation> random_crop_op = vision::RandomCrop({28, 28});
|
||||
EXPECT_NE(random_crop_op, nullptr);
|
||||
// UniformAug: transform ops must not be null
|
||||
std::shared_ptr<TensorOperation> uniform_aug_op2 = vision::UniformAugment({vision::RandomCrop({28}), nullptr}, 2);
|
||||
EXPECT_EQ(uniform_aug_op2, nullptr);
|
||||
|
||||
std::shared_ptr<TensorOperation> center_crop_op = vision::CenterCrop({16, 16});
|
||||
EXPECT_NE(center_crop_op, nullptr);
|
||||
|
||||
// Try UniformAugment with invalid negative num_ops value
|
||||
std::shared_ptr<TensorOperation> uniform_aug_op = vision::UniformAugment({random_crop_op, center_crop_op}, -1);
|
||||
EXPECT_EQ(uniform_aug_op, nullptr);
|
||||
// UniformAug: transform list must not be empty
|
||||
std::shared_ptr<TensorOperation> uniform_aug_op3 = vision::UniformAugment({}, 1);
|
||||
EXPECT_EQ(uniform_aug_op3, nullptr);
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestPipeline, TestUniformAugWithOps) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue