yankf21
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

port name() to TensorOperation and its derived classes

This commit is contained in:
mohammad 2020-11-27 18:17:43 -05:00
parent 97b1c5eed5
commit 9cfeb08c4d
6 changed files with 210 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
mindspore/ccsrc/minddata/dataset/include/text.h
View File

@ -37,6 +37,14 @@ namespace dataset {
// Transform operations for text
namespace text {
// Char arrays storing name of corresponding classes (in alphabetical order)
constexpr char kJiebaTokenizerOperation[] = "JiebaTokenizer";
constexpr char kLookupOperation[] = "Lookup";
constexpr char kNgramOperation[] = "Ngram";
constexpr char kSentencepieceTokenizerOperation[] = "SentencepieceTokenizer";
constexpr char kSlidingWindowOperation[] = "SlidingWindow";
constexpr char kWhitespaceTokenizerOperation[] = "WhitespaceTokenizer";
// Text Op classes (in alphabetical order)
class JiebaTokenizerOperation;
class LookupOperation;
@ -128,6 +136,8 @@ class JiebaTokenizerOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kJiebaTokenizerOperation; }
private:
std::string hmm_path_;
std::string mp_path_;
@ -146,6 +156,8 @@ class LookupOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kLookupOperation; }
private:
std::shared_ptr<Vocab> vocab_;
std::string unknown_token_;
@ -164,6 +176,8 @@ class NgramOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kNgramOperation; }
private:
std::vector<int32_t> ngrams_;
std::pair<std::string, int32_t> left_pad_;
@ -183,6 +197,8 @@ class SentencePieceTokenizerOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kSentencepieceTokenizerOperation; }
private:
std::shared_ptr<SentencePieceVocab> vocab_;
std::string vocab_path_;
@ -200,6 +216,8 @@ class SlidingWindowOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kSlidingWindowOperation; }
private:
int32_t width_;
int32_t axis_;
@ -216,6 +234,8 @@ class WhitespaceTokenizerOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kWhitespaceTokenizerOperation; }
private:
bool with_offsets_;
};

29
mindspore/ccsrc/minddata/dataset/include/transforms.h
View File

@ -28,6 +28,17 @@ namespace dataset {
class TensorOp;
// Char arrays storing name of corresponding classes (in alphabetical order)
constexpr char kComposeOperation[] = "Compose";
constexpr char kDuplicateOperation[] = "Duplicate";
constexpr char kOneHotOperation[] = "OneHot";
constexpr char kPreBuiltOperation[] = "PreBuilt";
constexpr char kRandomApplyOperation[] = "RandomApply";
constexpr char kRandomChoiceOperation[] = "RandomChoice";
constexpr char kRandomSelectSubpolicyOperation[] = "RandomSelectSubpolicy";
constexpr char kTypeCastOperation[] = "TypeCast";
constexpr char kUniqueOperation[] = "Unique";
// Abstract class to represent a dataset in the data pipeline.
class TensorOperation : public std::enable_shared_from_this<TensorOperation> {
public:
@ -46,6 +57,8 @@ class TensorOperation : public std::enable_shared_from_this<TensorOperation> {
virtual Status ValidateParams() = 0;
virtual std::string Name() const = 0;
/// \brief Check whether the operation is deterministic.
/// \return true if this op is a random op (returns non-deterministic result e.g. RandomCrop)
bool IsRandomOp() const { return random_op_; }
@ -146,6 +159,8 @@ class ComposeOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kComposeOperation; }
private:
std::vector<std::shared_ptr<TensorOperation>> transforms_;
};
@ -159,6 +174,8 @@ class DuplicateOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
Status ValidateParams() override;
std::string Name() const override { return kDuplicateOperation; }
};
class OneHotOperation : public TensorOperation {
@ -171,6 +188,8 @@ class OneHotOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kOneHotOperation; }
private:
float num_classes_;
};
@ -185,6 +204,8 @@ class PreBuiltOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kPreBuiltOperation; }
private:
std::shared_ptr<TensorOp> op_;
};
@ -199,6 +220,8 @@ class RandomApplyOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomApplyOperation; }
private:
std::vector<std::shared_ptr<TensorOperation>> transforms_;
double prob_;
@ -214,6 +237,8 @@ class RandomChoiceOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomChoiceOperation; }
private:
std::vector<std::shared_ptr<TensorOperation>> transforms_;
};
@ -227,6 +252,8 @@ class TypeCastOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kTypeCastOperation; }
private:
std::string data_type_;
};
@ -241,6 +268,8 @@ class UniqueOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
Status ValidateParams() override;
std::string Name() const override { return kUniqueOperation; }
};
#endif
} // namespace transforms

125
mindspore/ccsrc/minddata/dataset/include/vision.h
View File

@ -19,6 +19,7 @@
#include <map>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "minddata/dataset/core/constants.h"
@ -31,6 +32,48 @@ namespace dataset {
// Transform operations for performing computer vision.
namespace vision {
// Char arrays storing name of corresponding classes (in alphabetical order)
constexpr char kAutoContrastOperation[] = "AutoContrast";
constexpr char kBoundingBoxAugmentOperation[] = "BoundingBoxAugment";
constexpr char kCenterCropOperation[] = "CenterCrop";
constexpr char kCutMixBatchOperation[] = "CutMixBatch";
constexpr char kCutOutOperation[] = "CutOut";
constexpr char kCropOperation[] = "Crop";
constexpr char kDecodeOperation[] = "Decode";
constexpr char kEqualizeOperation[] = "Equalize";
constexpr char kHwcToChwOperation[] = "HwcToChw";
constexpr char kInvertOperation[] = "Invert";
constexpr char kMixUpBatchOperation[] = "MixUpBatch";
constexpr char kNormalizeOperation[] = "Normalize";
constexpr char kPadOperation[] = "Pad";
constexpr char kRandomAffineOperation[] = "RandomAffine";
constexpr char kRandomColorAdjustOperation[] = "RandomColorAdjust";
constexpr char kRandomColorOperation[] = "RandomColor";
constexpr char kRandomCropDecodeResizeOperation[] = "RandomCropDecodeResize";
constexpr char kRandomCropOperation[] = "RandomCrop";
constexpr char kRandomCropWithBBoxOperation[] = "RandomCropWithBBox";
constexpr char kRandomHorizontalFlipWithBBoxOperation[] = "RandomHorizontalFlipWithBBox";
constexpr char kRandomHorizontalFlipOperation[] = "RandomHorizontalFlip";
constexpr char kRandomPosterizeOperation[] = "RandomPosterize";
constexpr char kRandomResizedCropOperation[] = "RandomResizedCrop";
constexpr char kRandomResizedCropWithBBoxOperation[] = "RandomResizedCropWithBBox";
constexpr char kRandomResizeOperation[] = "RandomResize";
constexpr char kRandomResizeWithBBoxOperation[] = "RandomResizeWithBBox";
constexpr char kRandomRotationOperation[] = "RandomRotation";
constexpr char kRandomSolarizeOperation[] = "RandomSolarize";
constexpr char kRandomSharpnessOperation[] = "RandomSharpness";
constexpr char kRandomVerticalFlipOperation[] = "RandomVerticalFlip";
constexpr char kRandomVerticalFlipWithBBoxOperation[] = "RandomVerticalFlipWithBBox";
constexpr char kRescaleOperation[] = "Rescale";
constexpr char kResizeOperation[] = "Resize";
constexpr char kResizeWithBBoxOperation[] = "ResizeWithBBox";
constexpr char kRgbaToBgrOperation[] = "RgbaToBgr";
constexpr char kRgbaToRgbOperation[] = "RgbaToRgb";
constexpr char kSoftDvppDecodeRandomCropResizeJpegOperation[] = "SoftDvppDecodeRandomCropResizeJpeg";
constexpr char kSoftDvppDecodeResizeJpegOperation[] = "SoftDvppDecodeResizeJpeg";
constexpr char kSwapRedBlueOperation[] = "SwapRedBlue";
constexpr char kUniformAugOperation[] = "UniformAug";
// Transform Op classes (in alphabetical order)
#ifndef ENABLE_ANDROID
class AutoContrastOperation;
@ -501,6 +544,8 @@ class AutoContrastOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kAutoContrastOperation; }
private:
float cutoff_;
std::vector<uint32_t> ignore_;
@ -516,6 +561,8 @@ class BoundingBoxAugmentOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kBoundingBoxAugmentOperation; }
private:
std::shared_ptr<TensorOperation> transform_;
float ratio_;
@ -533,6 +580,8 @@ class CenterCropOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kCenterCropOperation; }
private:
std::vector<int32_t> size_;
};
@ -547,6 +596,8 @@ class CropOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kCropOperation; }
private:
std::vector<int32_t> coordinates_;
std::vector<int32_t> size_;
@ -562,6 +613,8 @@ class CutMixBatchOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kCutMixBatchOperation; }
private:
float alpha_;
float prob_;
@ -578,6 +631,8 @@ class CutOutOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kCutOutOperation; }
private:
int32_t length_;
int32_t num_patches_;
@ -595,6 +650,8 @@ class DecodeOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kDecodeOperation; }
private:
bool rgb_;
};
@ -607,6 +664,8 @@ class EqualizeOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
Status ValidateParams() override;
std::string Name() const override { return kEqualizeOperation; }
};
class HwcToChwOperation : public TensorOperation {
@ -616,6 +675,8 @@ class HwcToChwOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
Status ValidateParams() override;
std::string Name() const override { return kHwcToChwOperation; }
};
class InvertOperation : public TensorOperation {
@ -625,6 +686,8 @@ class InvertOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
Status ValidateParams() override;
std::string Name() const override { return kInvertOperation; }
};
class MixUpBatchOperation : public TensorOperation {
@ -637,6 +700,8 @@ class MixUpBatchOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kMixUpBatchOperation; }
private:
float alpha_;
};
@ -652,6 +717,8 @@ class NormalizeOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kNormalizeOperation; }
private:
std::vector<float> mean_;
std::vector<float> std_;
@ -669,6 +736,8 @@ class PadOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kPadOperation; }
private:
std::vector<int32_t> padding_;
std::vector<uint8_t> fill_value_;
@ -689,6 +758,8 @@ class RandomAffineOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomAffineOperation; }
private:
std::vector<float_t> degrees_; // min_degree, max_degree
std::vector<float_t> translate_range_; // maximum x translation percentage, maximum y translation percentage
@ -708,6 +779,8 @@ class RandomColorOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomColorOperation; }
private:
float t_lb_;
float t_ub_;
@ -724,6 +797,8 @@ class RandomColorAdjustOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomColorAdjustOperation; }
private:
std::vector<float> brightness_;
std::vector<float> contrast_;
@ -743,6 +818,8 @@ class RandomCropOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomCropOperation; }
private:
std::vector<int32_t> size_;
std::vector<int32_t> padding_;
@ -762,6 +839,8 @@ class RandomCropDecodeResizeOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomCropDecodeResizeOperation; }
private:
std::vector<int32_t> size_;
std::vector<float> scale_;
@ -782,6 +861,8 @@ class RandomCropWithBBoxOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomCropWithBBoxOperation; }
private:
std::vector<int32_t> size_;
std::vector<int32_t> padding_;
@ -800,6 +881,8 @@ class RandomHorizontalFlipOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomHorizontalFlipOperation; }
private:
float probability_;
};
@ -814,6 +897,8 @@ class RandomHorizontalFlipWithBBoxOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomHorizontalFlipWithBBoxOperation; }
private:
float probability_;
};
@ -828,6 +913,8 @@ class RandomPosterizeOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomPosterizeOperation; }
private:
std::vector<uint8_t> bit_range_;
};
@ -842,6 +929,8 @@ class RandomResizeOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomResizeOperation; }
private:
std::vector<int32_t> size_;
};
@ -856,6 +945,8 @@ class RandomResizeWithBBoxOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomResizeWithBBoxOperation; }
private:
std::vector<int32_t> size_;
};
@ -873,6 +964,8 @@ class RandomResizedCropOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomResizedCropOperation; }
private:
std::vector<int32_t> size_;
std::vector<float> scale_;
@ -894,6 +987,8 @@ class RandomResizedCropWithBBoxOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomResizedCropWithBBoxOperation; }
private:
std::vector<int32_t> size_;
std::vector<float> scale_;
@ -913,6 +1008,8 @@ class RandomRotationOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomRotationOperation; }
private:
std::vector<float> degrees_;
InterpolationMode interpolation_mode_;
@ -932,6 +1029,8 @@ class RandomSelectSubpolicyOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomSelectSubpolicyOperation; }
private:
std::vector<std::vector<std::pair<std::shared_ptr<TensorOperation>, double>>> policy_;
};
@ -946,6 +1045,8 @@ class RandomSharpnessOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomSharpnessOperation; }
private:
std::vector<float> degrees_;
};
@ -960,6 +1061,8 @@ class RandomSolarizeOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomSolarizeOperation; }
private:
std::vector<uint8_t> threshold_;
};
@ -974,6 +1077,8 @@ class RandomVerticalFlipOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomVerticalFlipOperation; }
private:
float probability_;
};
@ -988,6 +1093,8 @@ class RandomVerticalFlipWithBBoxOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRandomVerticalFlipWithBBoxOperation; }
private:
float probability_;
};
@ -1002,6 +1109,8 @@ class RescaleOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kRescaleOperation; }
private:
float rescale_;
float shift_;
@ -1019,6 +1128,8 @@ class ResizeOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kResizeOperation; }
private:
std::vector<int32_t> size_;
InterpolationMode interpolation_;
@ -1036,6 +1147,8 @@ class ResizeWithBBoxOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kResizeWithBBoxOperation; }
private:
std::vector<int32_t> size_;
InterpolationMode interpolation_;
@ -1050,6 +1163,8 @@ class RgbaToBgrOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
Status ValidateParams() override;
std::string Name() const override { return kRgbaToBgrOperation; }
};
class RgbaToRgbOperation : public TensorOperation {
@ -1061,6 +1176,8 @@ class RgbaToRgbOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
Status ValidateParams() override;
std::string Name() const override { return kRgbaToRgbOperation; }
};
class SoftDvppDecodeRandomCropResizeJpegOperation : public TensorOperation {
@ -1074,6 +1191,8 @@ class SoftDvppDecodeRandomCropResizeJpegOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kSoftDvppDecodeRandomCropResizeJpegOperation; }
private:
std::vector<int32_t> size_;
std::vector<float> scale_;
@ -1091,6 +1210,8 @@ class SoftDvppDecodeResizeJpegOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kSoftDvppDecodeResizeJpegOperation; }
private:
std::vector<int32_t> size_;
};
@ -1104,6 +1225,8 @@ class SwapRedBlueOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
Status ValidateParams() override;
std::string Name() const override { return kSwapRedBlueOperation; }
};
class UniformAugOperation : public TensorOperation {
@ -1116,6 +1239,8 @@ class UniformAugOperation : public TensorOperation {
Status ValidateParams() override;
std::string Name() const override { return kUniformAugOperation; }
private:
std::vector<std::shared_ptr<TensorOperation>> transforms_;
int32_t num_ops_;

11
tests/ut/cpp/dataset/c_api_text_test.cc
View File

@ -461,6 +461,17 @@ TEST_F(MindDataTestPipeline, TestNgramFail) {
EXPECT_EQ(ngram_op4, nullptr);
}
TEST_F(MindDataTestPipeline, TestTextOperationName) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextOperationName.";
// Create object for the tensor op, and check the name
std::string data_file = datasets_root_path_ + "/testVocab/words.txt";
std::shared_ptr<TensorOperation> sentence_piece_tokenizer_op =
text::SentencePieceTokenizer(data_file, SPieceTokenizerOutType::kString);
std::string correct_name = "SentencepieceTokenizer";
EXPECT_EQ(correct_name, sentence_piece_tokenizer_op->Name());
}
TEST_F(MindDataTestPipeline, TestWhitespaceTokenizerSuccess) {
// Testing the parameter of WhitespaceTokenizer interface when the with_offsets is default.
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestWhitespaceTokenizerSuccess.";

9
tests/ut/cpp/dataset/c_api_transforms_test.cc
View File

@ -379,6 +379,15 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail) {
EXPECT_EQ(random_choice3, nullptr);
}
TEST_F(MindDataTestPipeline, TestTransformOperationName) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTransformOperationName.";
// Create object for the tensor op, and check the name
std::shared_ptr<TensorOperation> duplicate_op = transforms::Duplicate();
std::string correct_name = "Duplicate";
EXPECT_EQ(correct_name, duplicate_op->Name());
}
TEST_F(MindDataTestPipeline, TestTypeCastSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTypeCastSuccess.";

16
tests/ut/cpp/dataset/c_api_vision_test.cc
View File

@ -3018,3 +3018,19 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) {
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestVisionOperationName) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVisionOperationName.";
std::string correct_name;
// Create object for the tensor op, and check the name
std::shared_ptr<TensorOperation> random_vertical_flip_op = vision::RandomVerticalFlip(0.5);
correct_name = "RandomVerticalFlip";
EXPECT_EQ(correct_name, random_vertical_flip_op->Name());
// Create object for the tensor op, and check the name
std::shared_ptr<TensorOperation> softDvpp_decode_resize_jpeg_op = vision::SoftDvppDecodeResizeJpeg({1, 1});
correct_name = "SoftDvppDecodeResizeJpeg";
EXPECT_EQ(correct_name, softDvpp_decode_resize_jpeg_op->Name());
}