From aae886bcd771fd1783c3f91f83d5cb161423b549 Mon Sep 17 00:00:00 2001 From: Xiao Tianci Date: Thu, 12 Aug 2021 21:17:06 +0800 Subject: [PATCH] fix crowd contributing code --- mindspore/ccsrc/minddata/dataset/api/audio.cc | 4 +- .../dataset/audio/kernels/ir/bindings.cc | 3 +- .../audio/ir/kernels/allpass_biquad_ir.cc | 9 +- .../audio/ir/kernels/allpass_biquad_ir.h | 5 +- .../audio/ir/kernels/amplitude_to_db_ir.cc | 10 +- .../dataset/audio/ir/kernels/angle_ir.cc | 3 +- .../dataset/audio/ir/kernels/angle_ir.h | 3 +- .../audio/ir/kernels/band_biquad_ir.cc | 5 +- .../dataset/audio/ir/kernels/band_biquad_ir.h | 2 +- .../audio/ir/kernels/bandpass_biquad_ir.cc | 7 +- .../audio/ir/kernels/bandpass_biquad_ir.h | 4 +- .../audio/ir/kernels/bandreject_biquad_ir.cc | 6 +- .../audio/ir/kernels/bandreject_biquad_ir.h | 4 +- .../audio/ir/kernels/bass_biquad_ir.cc | 6 +- .../dataset/audio/ir/kernels/bass_biquad_ir.h | 1 - .../audio/ir/kernels/frequency_masking_ir.cc | 4 +- .../audio/ir/kernels/time_masking_ir.cc | 8 +- .../audio/ir/kernels/time_stretch_ir.cc | 13 +- .../minddata/dataset/audio/ir/validators.cc | 77 +------ .../minddata/dataset/audio/ir/validators.h | 34 +-- .../audio/kernels/allpass_biquad_op.cc | 18 +- .../dataset/audio/kernels/allpass_biquad_op.h | 1 + .../audio/kernels/amplitude_to_db_op.cc | 13 +- .../audio/kernels/amplitude_to_db_op.h | 1 + .../dataset/audio/kernels/angle_op.cc | 6 +- .../minddata/dataset/audio/kernels/angle_op.h | 1 + .../dataset/audio/kernels/audio_utils.cc | 79 ++----- .../dataset/audio/kernels/audio_utils.h | 115 ++++++---- .../dataset/audio/kernels/band_biquad_op.cc | 11 +- .../dataset/audio/kernels/band_biquad_op.h | 3 +- .../audio/kernels/bandpass_biquad_op.cc | 17 +- .../audio/kernels/bandpass_biquad_op.h | 3 +- .../audio/kernels/bandreject_biquad_op.cc | 19 +- .../audio/kernels/bandreject_biquad_op.h | 1 + .../dataset/audio/kernels/bass_biquad_op.cc | 11 +- .../dataset/audio/kernels/bass_biquad_op.h | 3 +- .../audio/kernels/frequency_masking_op.cc | 2 +- .../dataset/audio/kernels/time_masking_op.cc | 4 +- .../dataset/audio/kernels/time_stretch_op.cc | 13 +- .../dataset/audio/kernels/time_stretch_op.h | 4 +- .../engine/datasetops/source/flickr_op.cc | 24 ++- .../minddata/dataset/include/dataset/audio.h | 13 +- .../minddata/dataset/include/dataset/vision.h | 41 ++-- .../dataset/include/dataset/vision_lite.h | 66 +++--- .../dataset/kernels/image/adjust_gamma_op.cc | 6 +- .../dataset/kernels/image/image_utils.cc | 4 +- .../dataset/kernels/image/rgb_to_bgr_op.h | 2 +- .../kernels/ir/vision/adjust_gamma_ir.cc | 2 - .../kernels/ir/vision/rgb_to_bgr_ir.cc | 4 - mindspore/dataset/audio/__init__.py | 2 + mindspore/dataset/audio/transforms.py | 198 +++++++++--------- mindspore/dataset/audio/utils.py | 2 + mindspore/dataset/audio/validators.py | 86 ++++---- mindspore/dataset/core/validator_helpers.py | 98 ++++++--- mindspore/dataset/engine/datasets.py | 174 ++++++++------- mindspore/dataset/vision/c_transforms.py | 3 +- mindspore/dataset/vision/utils.py | 1 + mindspore/dataset/vision/validators.py | 4 +- tests/ut/cpp/dataset/CMakeLists.txt | 1 - .../ut/cpp/dataset/c_api_audio_a_to_q_test.cc | 55 +++-- .../ut/cpp/dataset/c_api_audio_r_to_z_test.cc | 11 +- .../cpp/dataset/c_api_vision_a_to_q_test.cc | 13 +- .../cpp/dataset/c_api_vision_r_to_z_test.cc | 38 ++++ tests/ut/cpp/dataset/execute_test.cc | 89 ++++---- tests/ut/cpp/dataset/rgb_to_bgr_test_op.cc | 100 --------- tests/ut/python/dataset/test_adjustgamma.py | 32 +-- .../ut/python/dataset/test_allpass_biquad.py | 48 ++--- .../ut/python/dataset/test_amplitude_to_db.py | 19 +- tests/ut/python/dataset/test_angle.py | 23 +- tests/ut/python/dataset/test_band_biquad.py | 20 +- .../ut/python/dataset/test_bandpass_biquad.py | 25 +-- .../python/dataset/test_bandreject_biquad.py | 27 ++- tests/ut/python/dataset/test_bass_biquad.py | 22 +- tests/ut/python/dataset/test_datasets_sbd.py | 2 +- .../python/dataset/test_frequency_masking.py | 34 ++- tests/ut/python/dataset/test_rgb_bgr.py | 2 - tests/ut/python/dataset/test_time_masking.py | 34 ++- tests/ut/python/dataset/test_time_stretch.py | 39 ++-- 78 files changed, 876 insertions(+), 1021 deletions(-) delete mode 100644 tests/ut/cpp/dataset/rgb_to_bgr_test_op.cc diff --git a/mindspore/ccsrc/minddata/dataset/api/audio.cc b/mindspore/ccsrc/minddata/dataset/api/audio.cc index 520e840f00a..4aad5c54d9a 100644 --- a/mindspore/ccsrc/minddata/dataset/api/audio.cc +++ b/mindspore/ccsrc/minddata/dataset/api/audio.cc @@ -47,7 +47,7 @@ std::shared_ptr AllpassBiquad::Parse() { return std::make_shared(data_->sample_rate_, data_->central_freq_, data_->Q_); } -// AmplitudeToDB Operation. +// AmplitudeToDB Transform Operation. struct AmplitudeToDB::Data { Data(ScaleType stype, float ref_value, float amin, float top_db) : stype_(stype), ref_value_(ref_value), amin_(amin), top_db_(top_db) {} @@ -175,7 +175,7 @@ std::shared_ptr TimeMasking::Parse() { data_->mask_value_); } -// TimeStretch Operation. +// TimeStretch Transform Operation. struct TimeStretch::Data { explicit Data(float hop_length, int n_freq, float fixed_rate) : hop_length_(hop_length), n_freq_(n_freq), fixed_rate_(fixed_rate) {} diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc index 44ff82d3de5..853b6dec87c 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc @@ -17,6 +17,8 @@ #include "minddata/dataset/api/python/pybind_conversion.h" #include "minddata/dataset/api/python/pybind_register.h" +#include "minddata/dataset/include/dataset/transforms.h" + #include "minddata/dataset/audio/ir/kernels/allpass_biquad_ir.h" #include "minddata/dataset/audio/ir/kernels/amplitude_to_db_ir.h" #include "minddata/dataset/audio/ir/kernels/angle_ir.h" @@ -27,7 +29,6 @@ #include "minddata/dataset/audio/ir/kernels/frequency_masking_ir.h" #include "minddata/dataset/audio/ir/kernels/time_masking_ir.h" #include "minddata/dataset/audio/ir/kernels/time_stretch_ir.h" -#include "minddata/dataset/include/dataset/transforms.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/allpass_biquad_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/allpass_biquad_ir.cc index 35cf10b83c6..b760aae4844 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/allpass_biquad_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/allpass_biquad_ir.cc @@ -16,20 +16,20 @@ #include "minddata/dataset/audio/ir/kernels/allpass_biquad_ir.h" -#include "minddata/dataset/audio/kernels/allpass_biquad_op.h" - #include "minddata/dataset/audio/ir/validators.h" +#include "minddata/dataset/audio/kernels/allpass_biquad_op.h" namespace mindspore { namespace dataset { namespace audio { + // AllpassBiquadOperation AllpassBiquadOperation::AllpassBiquadOperation(int32_t sample_rate, float central_freq, float Q) : sample_rate_(sample_rate), central_freq_(central_freq), Q_(Q) {} Status AllpassBiquadOperation::ValidateParams() { - RETURN_IF_NOT_OK(CheckScalarNotZero("AllpassBiquad", "sample_rate", sample_rate_)); - RETURN_IF_NOT_OK(CheckScalarNotZero("AllpassBiquad", "central_freq", central_freq_)); + RETURN_IF_NOT_OK(ValidateScalarNotZero("AllpassBiquad", "sample_rate", sample_rate_)); + RETURN_IF_NOT_OK(ValidateScalarNotZero("AllpassBiquad", "central_freq", central_freq_)); RETURN_IF_NOT_OK(ValidateScalar("AllpassBiquad", "Q", Q_, {0, 1.0}, true, false)); return Status::OK(); } @@ -38,6 +38,7 @@ std::shared_ptr AllpassBiquadOperation::Build() { std::shared_ptr tensor_op = std::make_shared(sample_rate_, central_freq_, Q_); return tensor_op; } + Status AllpassBiquadOperation::to_json(nlohmann::json *out_json) { nlohmann::json args; args["sample_rate"] = sample_rate_; diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/allpass_biquad_ir.h b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/allpass_biquad_ir.h index c8d2be832bf..398287db244 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/allpass_biquad_ir.h +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/allpass_biquad_ir.h @@ -20,6 +20,7 @@ #include #include #include + #include "include/api/status.h" #include "minddata/dataset/include/dataset/constants.h" #include "minddata/dataset/include/dataset/transforms.h" @@ -27,9 +28,8 @@ namespace mindspore { namespace dataset { - namespace audio { -// Char arrays storing name of corresponding classes (in alphabetical order) + constexpr char kAllpassBiquadOperation[] = "AllpassBiquad"; class AllpassBiquadOperation : public TensorOperation { @@ -52,7 +52,6 @@ class AllpassBiquadOperation : public TensorOperation { float Q_; }; } // namespace audio - } // namespace dataset } // namespace mindspore #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_ALLPASS_BIQUAD_IR_H_ diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/amplitude_to_db_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/amplitude_to_db_ir.cc index 80412b1c437..61313e7fac1 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/amplitude_to_db_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/amplitude_to_db_ir.cc @@ -15,15 +15,15 @@ */ #include "minddata/dataset/audio/ir/kernels/amplitude_to_db_ir.h" -#include "minddata/dataset/audio/kernels/amplitude_to_db_op.h" #include "minddata/dataset/audio/ir/validators.h" +#include "minddata/dataset/audio/kernels/amplitude_to_db_op.h" namespace mindspore { namespace dataset { namespace audio { -// AmplitudeToDB +// AmplitudeToDBOperation AmplitudeToDBOperation::AmplitudeToDBOperation(ScaleType stype, float ref_value, float amin, float top_db) : stype_(stype), ref_value_(ref_value), amin_(amin), top_db_(top_db) {} @@ -32,9 +32,9 @@ AmplitudeToDBOperation::~AmplitudeToDBOperation() = default; std::string AmplitudeToDBOperation::Name() const { return kAmplitudeToDBOperation; } Status AmplitudeToDBOperation::ValidateParams() { - RETURN_IF_NOT_OK(CheckFloatScalarNonNegative("AmplitudeToDB", "top_db", top_db_)); - RETURN_IF_NOT_OK(CheckFloatScalarPositive("AmplitudeToDB", "amin", amin_)); - RETURN_IF_NOT_OK(CheckFloatScalarPositive("AmplitudeToDB", "ref_value", ref_value_)); + RETURN_IF_NOT_OK(ValidateFloatScalarNonNegative("AmplitudeToDB", "top_db", top_db_)); + RETURN_IF_NOT_OK(ValidateFloatScalarPositive("AmplitudeToDB", "amin", amin_)); + RETURN_IF_NOT_OK(ValidateFloatScalarPositive("AmplitudeToDB", "ref_value", ref_value_)); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/angle_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/angle_ir.cc index 53b1850e976..131a440e279 100755 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/angle_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/angle_ir.cc @@ -16,13 +16,12 @@ #include "minddata/dataset/audio/ir/kernels/angle_ir.h" -// Kernel Audio headers #include "minddata/dataset/audio/kernels/angle_op.h" namespace mindspore { namespace dataset { - namespace audio { + // AngleOperation AngleOperation::AngleOperation() {} diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/angle_ir.h b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/angle_ir.h index e0f1ce2ff80..0c35ba075b0 100755 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/angle_ir.h +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/angle_ir.h @@ -29,9 +29,8 @@ namespace mindspore { namespace dataset { - namespace audio { -// Char arrays storing name of corresponding classes + constexpr char kAngleOperation[] = "Angle"; class AngleOperation : public TensorOperation { diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/band_biquad_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/band_biquad_ir.cc index a231ef2b943..2757a7bda1c 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/band_biquad_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/band_biquad_ir.cc @@ -16,9 +16,8 @@ #include "minddata/dataset/audio/ir/kernels/band_biquad_ir.h" -#include "minddata/dataset/audio/kernels/band_biquad_op.h" - #include "minddata/dataset/audio/ir/validators.h" +#include "minddata/dataset/audio/kernels/band_biquad_op.h" namespace mindspore { namespace dataset { @@ -30,7 +29,7 @@ BandBiquadOperation::BandBiquadOperation(int32_t sample_rate, float central_freq Status BandBiquadOperation::ValidateParams() { RETURN_IF_NOT_OK(ValidateScalar("BandBiquad", "Q", Q_, {0, 1.0}, true, false)); - RETURN_IF_NOT_OK(CheckScalarNotZero("BandBIquad", "sample_rate", sample_rate_)); + RETURN_IF_NOT_OK(ValidateScalarNotZero("BandBIquad", "sample_rate", sample_rate_)); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/band_biquad_ir.h b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/band_biquad_ir.h index 7f29f7e0aba..0d9e687302a 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/band_biquad_ir.h +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/band_biquad_ir.h @@ -21,6 +21,7 @@ #include #include #include + #include "include/api/status.h" #include "minddata/dataset/include/dataset/constants.h" #include "minddata/dataset/include/dataset/transforms.h" @@ -30,7 +31,6 @@ namespace mindspore { namespace dataset { namespace audio { -// Char arrays storing name of corresponding classes (in alphabetical order) constexpr char kBandBiquadOperation[] = "BandBiquad"; class BandBiquadOperation : public TensorOperation { diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.cc index a335f6500fd..062cfd2a43a 100755 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.cc @@ -16,13 +16,13 @@ #include "minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.h" -#include "minddata/dataset/audio/kernels/bandpass_biquad_op.h" - #include "minddata/dataset/audio/ir/validators.h" +#include "minddata/dataset/audio/kernels/bandpass_biquad_op.h" namespace mindspore { namespace dataset { namespace audio { + // BandpassBiquadOperation BandpassBiquadOperation::BandpassBiquadOperation(int32_t sample_rate, float central_freq, float Q, bool const_skirt_gain) @@ -30,9 +30,10 @@ BandpassBiquadOperation::BandpassBiquadOperation(int32_t sample_rate, float cent Status BandpassBiquadOperation::ValidateParams() { RETURN_IF_NOT_OK(ValidateScalar("BandpassBiquad", "Q", Q_, {0, 1.0}, true, false)); - RETURN_IF_NOT_OK(CheckScalarNotZero("BandpassBiquad", "sample_rate", sample_rate_)); + RETURN_IF_NOT_OK(ValidateScalarNotZero("BandpassBiquad", "sample_rate", sample_rate_)); return Status::OK(); } + std::shared_ptr BandpassBiquadOperation::Build() { std::shared_ptr tensor_op = std::make_shared(sample_rate_, central_freq_, Q_, const_skirt_gain_); diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.h b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.h index 23cb220e9f1..309d0453833 100755 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.h +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.h @@ -21,6 +21,7 @@ #include #include #include + #include "include/api/status.h" #include "minddata/dataset/include/dataset/constants.h" #include "minddata/dataset/include/dataset/transforms.h" @@ -28,9 +29,8 @@ namespace mindspore { namespace dataset { - namespace audio { -// Char arrays storing name of corresponding classes (in alphabetical order) + constexpr char kBandpassBiquadOperation[] = "BandpassBiquad"; class BandpassBiquadOperation : public TensorOperation { diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.cc index 0688cb6b4d6..f66c65030da 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.cc @@ -15,19 +15,21 @@ */ #include "minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.h" -#include "minddata/dataset/audio/kernels/bandreject_biquad_op.h" + #include "minddata/dataset/audio/ir/validators.h" +#include "minddata/dataset/audio/kernels/bandreject_biquad_op.h" namespace mindspore { namespace dataset { namespace audio { + // BandrejectBiquadOperation BandrejectBiquadOperation::BandrejectBiquadOperation(int32_t sample_rate, float central_freq, float Q) : sample_rate_(sample_rate), central_freq_(central_freq), Q_(Q) {} Status BandrejectBiquadOperation::ValidateParams() { RETURN_IF_NOT_OK(ValidateScalar("BandrejectBiquad", "Q", Q_, {0, 1.0}, true, false)); - RETURN_IF_NOT_OK(CheckScalarNotZero("BandrejectBiquad", "sample_rate", sample_rate_)); + RETURN_IF_NOT_OK(ValidateScalarNotZero("BandrejectBiquad", "sample_rate", sample_rate_)); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.h b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.h index 9a38185c4b8..28b75c60739 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.h +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.h @@ -16,10 +16,12 @@ #ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_BANDREJECT_BIQUAD_IR_H_ #define MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_BANDREJECT_BIQUAD_IR_H_ + #include #include #include #include + #include "include/api/status.h" #include "minddata/dataset/include/dataset/constants.h" #include "minddata/dataset/include/dataset/transforms.h" @@ -27,10 +29,8 @@ namespace mindspore { namespace dataset { - namespace audio { -// Char arrays storing name of corresponding classes (in alphabetical order) constexpr char kBandrejectBiquadOperation[] = "BandrejectBiquad"; class BandrejectBiquadOperation : public TensorOperation { diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bass_biquad_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bass_biquad_ir.cc index f2f22aff0be..83766e50a6a 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bass_biquad_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bass_biquad_ir.cc @@ -16,9 +16,8 @@ #include "minddata/dataset/audio/ir/kernels/bass_biquad_ir.h" -#include "minddata/dataset/audio/kernels/bass_biquad_op.h" - #include "minddata/dataset/audio/ir/validators.h" +#include "minddata/dataset/audio/kernels/bass_biquad_op.h" namespace mindspore { namespace dataset { @@ -30,7 +29,7 @@ BassBiquadOperation::BassBiquadOperation(int32_t sample_rate, float gain, float Status BassBiquadOperation::ValidateParams() { RETURN_IF_NOT_OK(ValidateScalar("BassBiquad", "Q", Q_, {0, 1.0}, true, false)); - RETURN_IF_NOT_OK(CheckScalarNotZero("BassBiquad", "sample_rate", sample_rate_)); + RETURN_IF_NOT_OK(ValidateScalarNotZero("BassBiquad", "sample_rate", sample_rate_)); return Status::OK(); } @@ -38,6 +37,7 @@ std::shared_ptr BassBiquadOperation::Build() { std::shared_ptr tensor_op = std::make_shared(sample_rate_, gain_, central_freq_, Q_); return tensor_op; } + Status BassBiquadOperation::to_json(nlohmann::json *out_json) { nlohmann::json args; args["sample_rate"] = sample_rate_; diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bass_biquad_ir.h b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bass_biquad_ir.h index 1fdd38b8a90..725000591a7 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bass_biquad_ir.h +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bass_biquad_ir.h @@ -31,7 +31,6 @@ namespace mindspore { namespace dataset { namespace audio { -// Char arrays storing name of corresponding classes (in alphabetical order) constexpr char kBassBiquadOperation[] = "BassBiquad"; class BassBiquadOperation : public TensorOperation { diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/frequency_masking_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/frequency_masking_ir.cc index 75bb3ee70ba..2ee3a8a8123 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/frequency_masking_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/frequency_masking_ir.cc @@ -33,8 +33,8 @@ FrequencyMaskingOperation::FrequencyMaskingOperation(bool iid_masks, int32_t fre FrequencyMaskingOperation::~FrequencyMaskingOperation() = default; Status FrequencyMaskingOperation::ValidateParams() { - RETURN_IF_NOT_OK(CheckIntScalarNonNegative("FrequencyMasking", "frequency_mask_param", frequency_mask_param_)); - RETURN_IF_NOT_OK(CheckIntScalarNonNegative("FrequencyMasking", "mask_start", mask_start_)); + RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("FrequencyMasking", "frequency_mask_param", frequency_mask_param_)); + RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("FrequencyMasking", "mask_start", mask_start_)); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_masking_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_masking_ir.cc index 36da3ce8698..0bdffac2590 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_masking_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_masking_ir.cc @@ -15,12 +15,12 @@ */ #include "minddata/dataset/audio/ir/kernels/time_masking_ir.h" -#include "minddata/dataset/audio/kernels/time_masking_op.h" + #include "minddata/dataset/audio/ir/validators.h" +#include "minddata/dataset/audio/kernels/time_masking_op.h" namespace mindspore { namespace dataset { - namespace audio { TimeMaskingOperation::TimeMaskingOperation(bool iid_masks, int64_t time_mask_param, int64_t mask_start, @@ -30,8 +30,8 @@ TimeMaskingOperation::TimeMaskingOperation(bool iid_masks, int64_t time_mask_par TimeMaskingOperation::~TimeMaskingOperation() = default; Status TimeMaskingOperation::ValidateParams() { - RETURN_IF_NOT_OK(CheckIntScalarNonNegative("TimeMasking", "time_mask_param", time_mask_param_)); - RETURN_IF_NOT_OK(CheckIntScalarNonNegative("TimeMasking", "mask_start", mask_start_)); + RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("TimeMasking", "time_mask_param", time_mask_param_)); + RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("TimeMasking", "mask_start", mask_start_)); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_stretch_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_stretch_ir.cc index a78c4523705..4a94c4c6693 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_stretch_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_stretch_ir.cc @@ -14,14 +14,15 @@ * limitations under the License. */ #include "minddata/dataset/audio/ir/kernels/time_stretch_ir.h" -#include "minddata/dataset/audio/kernels/time_stretch_op.h" #include "minddata/dataset/audio/ir/validators.h" +#include "minddata/dataset/audio/kernels/time_stretch_op.h" + namespace mindspore { namespace dataset { namespace audio { -// TimeStretch +// TimeStretchOperation TimeStretchOperation::TimeStretchOperation(float hop_length, int n_freq, float fixed_rate) : hop_length_(hop_length), n_freq_(n_freq), fixed_rate_(fixed_rate) {} @@ -31,10 +32,10 @@ std::string TimeStretchOperation::Name() const { return kTimeStretchOperation; } Status TimeStretchOperation::ValidateParams() { // param check - RETURN_IF_NOT_OK(CheckFloatScalarPositive("TimeStretch", "hop_length", hop_length_)); - RETURN_IF_NOT_OK(CheckIntScalarPositive("TimeStretch", "n_freq", n_freq_)); - RETURN_IF_NOT_OK(CheckFloatScalarNotNan("TimeStretch", "fixed_rate", fixed_rate_)); - RETURN_IF_NOT_OK(CheckFloatScalarPositive("TimeStretch", "fixed_rate", fixed_rate_)); + RETURN_IF_NOT_OK(ValidateFloatScalarPositive("TimeStretch", "hop_length", hop_length_)); + RETURN_IF_NOT_OK(ValidateIntScalarPositive("TimeStretch", "n_freq", n_freq_)); + RETURN_IF_NOT_OK(ValidateFloatScalarNotNan("TimeStretch", "fixed_rate", fixed_rate_)); + RETURN_IF_NOT_OK(ValidateFloatScalarPositive("TimeStretch", "fixed_rate", fixed_rate_)); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/validators.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/validators.cc index 03333c62f59..e3f8c127b54 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/validators.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/validators.cc @@ -17,87 +17,20 @@ namespace mindspore { namespace dataset { -/* ####################################### Validator Functions ############################################ */ -Status CheckFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar) { - RETURN_IF_NOT_OK(CheckScalar(op_name, scalar_name, scalar, {0}, true)); + +Status ValidateIntScalarNonNegative(const std::string &op_name, const std::string &scalar_name, int32_t scalar) { + RETURN_IF_NOT_OK(ValidateScalar(op_name, scalar_name, scalar, {0}, false)); return Status::OK(); } -Status CheckFloatScalarNotNan(const std::string &op_name, const std::string &scalar_name, float scalar) { +Status ValidateFloatScalarNotNan(const std::string &op_name, const std::string &scalar_name, float scalar) { if (std::isnan(scalar)) { - std::string err_msg = op_name + ":" + scalar_name + " should be specified, got: Nan."; + std::string err_msg = op_name + ": " + scalar_name + " should be specified, got: Nan"; MS_LOG(ERROR) << err_msg; return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); } return Status::OK(); } -Status CheckFloatScalarNonNegative(const std::string &op_name, const std::string &scalar_name, float scalar) { - RETURN_IF_NOT_OK(CheckScalar(op_name, scalar_name, scalar, {0}, false)); - return Status::OK(); -} - -Status CheckIntScalarNonNegative(const std::string &op_name, const std::string &scalar_name, int32_t scalar) { - RETURN_IF_NOT_OK(CheckScalar(op_name, scalar_name, scalar, {0}, false)); - return Status::OK(); -} - -Status CheckIntScalarPositive(const std::string &op_name, const std::string &scalar_name, int32_t scalar) { - RETURN_IF_NOT_OK(CheckScalar(op_name, scalar_name, scalar, {0}, true)); - return Status::OK(); -} - -Status CheckStringScalarInList(const std::string &op_name, const std::string &scalar_name, const std::string &scalar, - const std::vector &str_vec) { - auto ret = std::find(str_vec.begin(), str_vec.end(), scalar); - if (ret == str_vec.end()) { - std::string interval_description = "["; - for (int m = 0; m < str_vec.size(); m++) { - std::string word = str_vec[m]; - interval_description = interval_description + word; - if (m != str_vec.size() - 1) interval_description = interval_description + ", "; - } - interval_description = interval_description + "]"; - - std::string err_msg = op_name + ": " + scalar_name + " must be one of " + interval_description + ", got: " + scalar; - MS_LOG(ERROR) << err_msg; - return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); - } - return Status::OK(); -} - -template -Status CheckScalar(const std::string &op_name, const std::string &scalar_name, const T scalar, - const std::vector &range, bool left_open_interval, bool right_open_interval) { - if (range.empty() || range.size() > 2) { - std::string err_msg = "Range check expecting size 1 or 2, but got: " + std::to_string(range.size()); - MS_LOG(ERROR) << err_msg; - return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); - } - if ((left_open_interval && scalar <= range[0]) || (!left_open_interval && scalar < range[0])) { - std::string interval_description = left_open_interval ? " greater than " : " greater than or equal to "; - std::string err_msg = op_name + ":" + scalar_name + " must be" + interval_description + std::to_string(range[0]) + - ", got: " + std::to_string(scalar); - MS_LOG(ERROR) << err_msg; - return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); - } - if (range.size() == 2) { - if ((right_open_interval && scalar >= range[1]) || (!right_open_interval && scalar > range[1])) { - std::string left_bracket = left_open_interval ? "(" : "["; - std::string right_bracket = right_open_interval ? ")" : "]"; - std::string err_msg = op_name + ":" + scalar_name + " is out of range " + left_bracket + - std::to_string(range[0]) + ", " + std::to_string(range[1]) + right_bracket + - ", got: " + std::to_string(scalar); - MS_LOG(ERROR) << err_msg; - return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); - } - } - return Status::OK(); -} -template Status CheckScalar(const std::string &op_name, const std::string &scalar_name, const float scalar, - const std::vector &range, bool left_open_interval, bool right_open_interval); - -template Status CheckScalar(const std::string &op_name, const std::string &scalar_name, const int32_t scalar, - const std::vector &range, bool left_open_interval, bool right_open_interval); } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/validators.h b/mindspore/ccsrc/minddata/dataset/audio/ir/validators.h index 02d6b1abe40..2fa2978c310 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/validators.h +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/validators.h @@ -19,52 +19,32 @@ #include #include -#include "minddata/dataset/kernels/ir/validators.h" #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/kernels/ir/tensor_operation.h" +#include "minddata/dataset/kernels/ir/validators.h" #include "minddata/dataset/util/status.h" namespace mindspore { namespace dataset { -// Helper function to non-nan float scalar -Status CheckFloatScalarNotNan(const std::string &op_name, const std::string &scalar_name, float scalar); - -// Helper function to positive float scalar -Status CheckFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar); - // Helper function to positive int scalar -Status CheckIntScalarPositive(const std::string &op_name, const std::string &scalar_name, int32_t scalar); +Status ValidateIntScalarNonNegative(const std::string &op_name, const std::string &scalar_name, int32_t scalar); + +// Helper function to non-nan float scalar +Status ValidateFloatScalarNotNan(const std::string &op_name, const std::string &scalar_name, float scalar); template // Helper function to check scalar is not equal to zero -Status CheckScalarNotZero(const std::string &op_name, const std::string &scalar_name, const T scalar) { +Status ValidateScalarNotZero(const std::string &op_name, const std::string &scalar_name, const T scalar) { if (scalar == 0) { - std::string err_msg = op_name + ":" + scalar_name + " can't be 0" + ", got: " + std::to_string(scalar); + std::string err_msg = op_name + ": " + scalar_name + " can't be zero, got: " + std::to_string(scalar); MS_LOG(ERROR) << err_msg; return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg); } return Status::OK(); } -// Helper function to positive float scalar -Status CheckFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar); - -// Helper function to non-negative float scalar -Status CheckFloatScalarNonNegative(const std::string &op_name, const std::string &scalar_name, float scalar); - -// Helper function to positive int scalar -Status CheckIntScalarNonNegative(const std::string &op_name, const std::string &scalar_name, int32_t scalar); - -// Helper function to check string scalar -Status CheckStringScalarInList(const std::string &op_name, const std::string &scalar_name, const std::string &scalar, - const std::vector &str_vec); - -// Helper function to validate scalar -template -Status CheckScalar(const std::string &op_name, const std::string &scalar_name, const T scalar, - const std::vector &range, bool left_open_interval = false, bool right_open_interval = false); } // namespace dataset } // namespace mindspore #endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ADUIO_IR_VALIDATORS_H_ diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/allpass_biquad_op.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/allpass_biquad_op.cc index da2f88964af..b1b4625e066 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/allpass_biquad_op.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/allpass_biquad_op.cc @@ -20,14 +20,15 @@ namespace mindspore { namespace dataset { + Status AllpassBiquadOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { IO_CHECK(input, output); TensorShape input_shape = input->shape(); - CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "AllpassBiquad: input dimension should be greater than 0."); - CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType(DataType::DE_FLOAT32) || - input->type() == DataType(DataType::DE_FLOAT16) || - input->type() == DataType(DataType::DE_FLOAT64), - "AllpassBiquad: input type should be float, but got " + input->type().ToString()); + CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "AllpassBiquad: input tensor is not in shape of <..., time>."); + CHECK_FAIL_RETURN_UNEXPECTED( + input->type() == DataType(DataType::DE_FLOAT32) || input->type() == DataType(DataType::DE_FLOAT16) || + input->type() == DataType(DataType::DE_FLOAT64), + "AllpassBiquad: input tensor type should be float, but got: " + input->type().ToString()); double w0 = 2 * PI * central_freq_ / sample_rate_; double alpha = sin(w0) / 2 / Q_; double b0 = 1 - alpha; @@ -36,15 +37,16 @@ Status AllpassBiquadOp::Compute(const std::shared_ptr &input, std::share double a0 = b2; double a1 = -2 * cos(w0); double a2 = 1 - alpha; - if (input->type() == DataType(DataType::DE_FLOAT32)) + if (input->type() == DataType(DataType::DE_FLOAT32)) { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); - else if (input->type() == DataType(DataType::DE_FLOAT64)) + } else if (input->type() == DataType(DataType::DE_FLOAT64)) { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); - else + } else { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); + } } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/allpass_biquad_op.h b/mindspore/ccsrc/minddata/dataset/audio/kernels/allpass_biquad_op.h index d4e7e17b95a..26c7b729f0a 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/allpass_biquad_op.h +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/allpass_biquad_op.h @@ -26,6 +26,7 @@ namespace mindspore { namespace dataset { + class AllpassBiquadOp : public TensorOp { public: AllpassBiquadOp(int32_t sample_rate, float central_freq, float Q) diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/amplitude_to_db_op.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/amplitude_to_db_op.cc index dbebec42d39..8a202f497c4 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/amplitude_to_db_op.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/amplitude_to_db_op.cc @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include - #include "minddata/dataset/audio/kernels/amplitude_to_db_op.h" + #include "minddata/dataset/audio/kernels/audio_utils.h" #include "minddata/dataset/kernels/data/data_utils.h" #include "minddata/dataset/util/status.h" @@ -26,7 +25,7 @@ namespace dataset { Status AmplitudeToDBOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { IO_CHECK(input, output); if (input->shape().Rank() < 2) { - std::string err_msg = "AmplitudeToDB: input tensor shape should be <..., freq, time>"; + std::string err_msg = "AmplitudeToDB: input tensor is not in shape of <..., freq, time>."; MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); } @@ -40,12 +39,12 @@ Status AmplitudeToDBOp::Compute(const std::shared_ptr &input, std::share // typecast CHECK_FAIL_RETURN_UNEXPECTED(input->type() != DataType::DE_STRING, - "AmplitudeToDB: input type should be float, but got string."); + "AmplitudeToDB: input tensor type should be float, but got: string."); if (input->type() != DataType::DE_FLOAT64) { - CHECK_FAIL_RETURN_UNEXPECTED(TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32)), - "AmplitudeToDB: input type should be float, but got " + input->type().ToString()); + CHECK_FAIL_RETURN_UNEXPECTED( + TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32)), + "AmplitudeToDB: input tensor type should be float, but got: " + input->type().ToString()); return AmplitudeToDB(input_tensor, output, multiplier, amin, db_multiplier, top_db); - } else { input_tensor = input; return AmplitudeToDB(input_tensor, output, multiplier, amin, db_multiplier, top_db); diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/amplitude_to_db_op.h b/mindspore/ccsrc/minddata/dataset/audio/kernels/amplitude_to_db_op.h index bd84e888f9e..9aa2672878e 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/amplitude_to_db_op.h +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/amplitude_to_db_op.h @@ -29,6 +29,7 @@ namespace mindspore { namespace dataset { + class AmplitudeToDBOp : public TensorOp { public: AmplitudeToDBOp(ScaleType stype, float ref_value, float amin, float top_db) diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/angle_op.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/angle_op.cc index 54827c934ee..9dc313f606a 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/angle_op.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/angle_op.cc @@ -25,8 +25,10 @@ namespace dataset { Status AngleOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { IO_CHECK(input, output); // if If the last dimension is not 2, then it's not a complex number - CHECK_FAIL_RETURN_UNEXPECTED(input->shape()[-1] == 2, "Angle: The input is not several legal complex numbers"); - CHECK_FAIL_RETURN_UNEXPECTED(input->type().IsNumeric(), "Angle: The input type should be numbers"); + CHECK_FAIL_RETURN_UNEXPECTED(input->shape()[-1] == 2, "Angle: input tensor is not in shape of <..., complex=2>."); + CHECK_FAIL_RETURN_UNEXPECTED( + input->type().IsNumeric(), + "Angle: input tensor type should be int, float or double, but got: " + input->type().ToString()); if (input->type() == DataType(DataType::DE_FLOAT64)) { return Angle(input, output); } else { diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/angle_op.h b/mindspore/ccsrc/minddata/dataset/audio/kernels/angle_op.h index aff0ab44a4d..501981b2138 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/angle_op.h +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/angle_op.h @@ -26,6 +26,7 @@ namespace mindspore { namespace dataset { + class AngleOp : public TensorOp { public: // Convert complex numbers to angles diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc index 9de2b8baf2d..edede9c3135 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc @@ -18,58 +18,16 @@ #include "minddata/dataset/util/random.h" #include "minddata/dataset/util/status.h" + namespace mindspore { namespace dataset { -template -Status AmplitudeToDB(const std::shared_ptr &input, std::shared_ptr *output, T multiplier, T amin, - T db_multiplier, T top_db) { - TensorShape input_shape = input->shape(); - TensorShape to_shape = input_shape.Rank() == 2 - ? TensorShape({1, 1, input_shape[-2], input_shape[-1]}) - : TensorShape({input->Size() / (input_shape[-3] * input_shape[-2] * input_shape[-1]), - input_shape[-3], input_shape[-2], input_shape[-1]}); - RETURN_IF_NOT_OK(input->Reshape(to_shape)); - - std::vector max_val; - int step = to_shape[-3] * input_shape[-2] * input_shape[-1]; - int cnt = 0; - T temp_max = std::numeric_limits::lowest(); - for (auto itr = input->begin(); itr != input->end(); itr++) { - // do clamp - *itr = *itr < amin ? log10(amin) * multiplier : log10(*itr) * multiplier; - *itr -= multiplier * db_multiplier; - // calculate max by axis - cnt++; - if ((*itr) > temp_max) temp_max = *itr; - if (cnt % step == 0) { - max_val.push_back(temp_max); - temp_max = std::numeric_limits::lowest(); - } - } - - if (!std::isnan(top_db)) { - int ind = 0; - for (auto itr = input->begin(); itr != input->end(); itr++, ind++) { - float lower_bound = max_val[ind / step] - top_db; - *itr = std::max((*itr), static_cast(lower_bound)); - } - } - RETURN_IF_NOT_OK(input->Reshape(input_shape)); - *output = input; - return Status::OK(); -} -template Status AmplitudeToDB(const std::shared_ptr &input, std::shared_ptr *output, - float multiplier, float amin, float db_multiplier, float top_db); -template Status AmplitudeToDB(const std::shared_ptr &input, std::shared_ptr *output, - double multiplier, double amin, double db_multiplier, double top_db); - -/// \brief Generate linearly spaced vector +/// \brief Generate linearly spaced vector. /// \param[in] start - Value of the startpoint. /// \param[in] end - Value of the endpoint. /// \param[in] n - N points in the output tensor. /// \param[out] output - Tensor has n points with linearly space. The spacing between the points is (end-start)/(n-1). -/// \return Status return code +/// \return Status return code. template Status Linspace(std::shared_ptr *output, T start, T end, int n) { if (start > end) { @@ -91,10 +49,10 @@ Status Linspace(std::shared_ptr *output, T start, T end, int n) { return Status::OK(); } -/// \brief Calculate complex tensor angle +/// \brief Calculate complex tensor angle. /// \param[in] input - Input tensor, must be complex, . /// \param[out] output - Complex tensor angle. -/// \return Status return code +/// \return Status return code. template Status ComplexAngle(const std::shared_ptr &input, std::shared_ptr *output) { // check complex @@ -123,10 +81,10 @@ Status ComplexAngle(const std::shared_ptr &input, std::shared_ptr. /// \param[out] output - Complex tensor abs. -/// \return Status return code +/// \return Status return code. template Status ComplexAbs(const std::shared_ptr &input, std::shared_ptr *output) { // check complex @@ -152,17 +110,17 @@ Status ComplexAbs(const std::shared_ptr &input, std::shared_ptr return Status::OK(); } -/// \brief Reconstruct complex tensor from norm and angle +/// \brief Reconstruct complex tensor from norm and angle. /// \param[in] abs - The absolute value of the complex tensor. /// \param[in] angle - The angle of the complex tensor. /// \param[out] output - Complex tensor, . -/// \return Status return code +/// \return Status return code. template Status Polar(const std::shared_ptr &abs, const std::shared_ptr &angle, std::shared_ptr *output) { // check shape if (abs->shape() != angle->shape()) { - std::string err_msg = "Polar: input shape of abs and angle must be same."; + std::string err_msg = "Polar: input tensor shape of abs and angle must be the same."; MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); } @@ -185,12 +143,12 @@ Status Polar(const std::shared_ptr &abs, const std::shared_ptr & return Status::OK(); } -/// \brief Pad complex tensor +/// \brief Pad complex tensor. /// \param[in] input - The complex tensor. /// \param[in] length - The length of padding. /// \param[in] dim - The dim index for padding. /// \param[out] output - Complex tensor, . -/// \return Status return code +/// \return Status return code. template Status PadComplexTensor(const std::shared_ptr &input, std::shared_ptr *output, int length, int dim) { TensorShape input_shape = input->shape(); @@ -218,13 +176,13 @@ Status PadComplexTensor(const std::shared_ptr &input, std::shared_ptr Status Phase(const std::shared_ptr &angle_0, const std::shared_ptr &angle_1, const std::shared_ptr &phase_advance, const std::shared_ptr &phase_time0, @@ -269,12 +227,12 @@ Status Phase(const std::shared_ptr &angle_0, const std::shared_ptr Status Mag(const std::shared_ptr &abs_0, const std::shared_ptr &abs_1, std::shared_ptr *output, const std::vector &alphas) { @@ -377,9 +335,8 @@ Status TimeStretch(std::shared_ptr input, std::shared_ptr *outpu RETURN_IF_NOT_OK(TimeStretch(input, output, rate, phase_advance)); break; default: - RETURN_STATUS_UNEXPECTED( - "TimeStretch: unsupported type, currently supported types include " - "[float, double]."); + RETURN_STATUS_UNEXPECTED("TimeStretch: input tensor type should be float or double, but got: " + + input->type().ToString()); } return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.h b/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.h index 4d4be13876e..84c7ba7e787 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.h +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.h @@ -29,21 +29,58 @@ #include "minddata/dataset/util/status.h" constexpr double PI = 3.141592653589793; + namespace mindspore { namespace dataset { + /// \brief Turn a tensor from the power/amplitude scale to the decibel scale. -/// \param input/output: Tensor of shape <...,freq,time> -/// \param multiplier: power - 10, amplitude - 20 -/// \param amin: lower bound -/// \param db_multiplier: multiplier for decibels -/// \param top_db: the lower bound for decibels cut-off -/// \return Status code +/// \param input/output: Tensor of shape <..., freq, time>. +/// \param multiplier: power - 10, amplitude - 20. +/// \param amin: lower bound. +/// \param db_multiplier: multiplier for decibels. +/// \param top_db: the lower bound for decibels cut-off. +/// \return Status code. template Status AmplitudeToDB(const std::shared_ptr &input, std::shared_ptr *output, T multiplier, T amin, - T db_multiplier, T top_db); + T db_multiplier, T top_db) { + TensorShape input_shape = input->shape(); + TensorShape to_shape = input_shape.Rank() == 2 + ? TensorShape({1, 1, input_shape[-2], input_shape[-1]}) + : TensorShape({input->Size() / (input_shape[-3] * input_shape[-2] * input_shape[-1]), + input_shape[-3], input_shape[-2], input_shape[-1]}); + RETURN_IF_NOT_OK(input->Reshape(to_shape)); -/// \brief Calculate the angles of the complex numbers -/// \param input/output: Tensor of shape <...,time> + std::vector max_val; + int step = to_shape[-3] * input_shape[-2] * input_shape[-1]; + int cnt = 0; + T temp_max = std::numeric_limits::lowest(); + for (auto itr = input->begin(); itr != input->end(); itr++) { + // do clamp + *itr = *itr < amin ? log10(amin) * multiplier : log10(*itr) * multiplier; + *itr -= multiplier * db_multiplier; + // calculate max by axis + cnt++; + if ((*itr) > temp_max) temp_max = *itr; + if (cnt % step == 0) { + max_val.push_back(temp_max); + temp_max = std::numeric_limits::lowest(); + } + } + + if (!std::isnan(top_db)) { + int ind = 0; + for (auto itr = input->begin(); itr != input->end(); itr++, ind++) { + float lower_bound = max_val[ind / step] - top_db; + *itr = std::max((*itr), static_cast(lower_bound)); + } + } + RETURN_IF_NOT_OK(input->Reshape(input_shape)); + *output = input; + return Status::OK(); +} + +/// \brief Calculate the angles of the complex numbers. +/// \param input/output: Tensor of shape <..., time>. template Status Angle(const std::shared_ptr &input, std::shared_ptr *output) { TensorShape shape = input->shape(); @@ -68,14 +105,14 @@ Status Angle(const std::shared_ptr &input, std::shared_ptr *outp } /// \brief Perform a biquad filter of input tensor. -/// \param input/output: Tensor of shape <...,time> -/// \param a0: denominator coefficient of current output y[n], typically 1 -/// \param a1: denominator coefficient of current output y[n-1] -/// \param a2: denominator coefficient of current output y[n-2] -/// \param b0: numerator coefficient of current input, x[n] -/// \param b1: numerator coefficient of input one time step ago x[n-1] -/// \param b2: numerator coefficient of input two time steps ago x[n-2] -/// \return Status code +/// \param input/output: Tensor of shape <..., time>. +/// \param a0: denominator coefficient of current output y[n], typically 1. +/// \param a1: denominator coefficient of current output y[n-1]. +/// \param a2: denominator coefficient of current output y[n-2]. +/// \param b0: numerator coefficient of current input, x[n]. +/// \param b1: numerator coefficient of input one time step ago x[n-1]. +/// \param b2: numerator coefficient of input two time steps ago x[n-2]. +/// \return Status code. template Status Biquad(const std::shared_ptr &input, std::shared_ptr *output, T b0, T b1, T b2, T a0, T a1, T a2) { @@ -91,10 +128,10 @@ Status Biquad(const std::shared_ptr &input, std::shared_ptr *out } /// \brief Perform an IIR filter by evaluating difference equation. -/// \param input/output: Tensor of shape <...,time> +/// \param input/output: Tensor of shape <..., time> /// \param a_coeffs: denominator coefficients of difference equation of dimension of (n_order + 1). /// \param b_coeffs: numerator coefficients of difference equation of dimension of (n_order + 1). -/// \param clamp: If True, clamp the output signal to be in the range [-1, 1] (Default: True) +/// \param clamp: If True, clamp the output signal to be in the range [-1, 1] (Default: True). /// \return Status code template Status LFilter(const std::shared_ptr &input, std::shared_ptr *output, std::vector a_coeffs, @@ -112,7 +149,7 @@ Status LFilter(const std::shared_ptr &input, std::shared_ptr *ou size_t channel_idx = 1; size_t m_num_order = b_coeffs.size() - 1; size_t m_den_order = a_coeffs.size() - 1; - // init A_coeffs and B_coeffs by div(a0) + // init A_coeffs and B_coeffs by div(a0) for (size_t i = 1; i < a_coeffs.size(); i++) { a_coeffs[i] /= a_coeffs[0]; } @@ -172,40 +209,40 @@ Status LFilter(const std::shared_ptr &input, std::shared_ptr *ou // unpack batch Tensor::CreateFromVector(out_vect, input_shape, &out); *output = out; - delete m_px; - delete m_py; + delete[] m_px; + delete[] m_py; return Status::OK(); } /// \brief Stretch STFT in time at a given rate, without changing the pitch. -/// \param[in] input - Tensor of shape <...,freq,time>. +/// \param[in] input - Tensor of shape <..., freq, time>. /// \param[in] rate - Stretch factor. /// \param[in] phase_advance - Expected phase advance in each bin. /// \param[out] output - Tensor after stretch in time domain. -/// \return Status return code +/// \return Status return code. Status TimeStretch(std::shared_ptr input, std::shared_ptr *output, float rate, float hop_length, float n_freq); /// \brief Apply a mask along axis. -/// \param input: Tensor of shape <...,freq,time> -/// \param output: Tensor of shape <...,freq,time> -/// \param mask_param: Number of columns to be masked will be uniformly sampled from [0, mask_param] -/// \param mask_value: Value to assign to the masked columns -/// \param axis: Axis to apply masking on (1 -> frequency, 2 -> time) -/// \param rnd: Number generator -/// \return Status code +/// \param input: Tensor of shape <..., freq, time>. +/// \param output: Tensor of shape <..., freq, time>. +/// \param mask_param: Number of columns to be masked will be uniformly sampled from [0, mask_param]. +/// \param mask_value: Value to assign to the masked columns. +/// \param axis: Axis to apply masking on (1 -> frequency, 2 -> time). +/// \param rnd: Number generator. +/// \return Status code. Status RandomMaskAlongAxis(const std::shared_ptr &input, std::shared_ptr *output, int64_t mask_param, double mask_value, int axis, std::mt19937 rnd); /// \brief Apply a mask along axis. All examples will have the same mask interval. -/// \param input: Tensor of shape <...,freq,time> -/// \param output: Tensor of shape <...,freq,time> -/// \param mask_width: The width of the mask -/// \param mask_start: Starting position of the mask -/// Mask will be applied from indices [mask_start, mask_start + mask_width) -/// \param mask_value: Value to assign to the masked columns -/// \param axis: Axis to apply masking on (1 -> frequency, 2 -> time) -/// \return Status code +/// \param input: Tensor of shape <..., freq, time>. +/// \param output: Tensor of shape <..., freq, time>. +/// \param mask_width: The width of the mask. +/// \param mask_start: Starting position of the mask. +/// Mask will be applied from indices [mask_start, mask_start + mask_width). +/// \param mask_value: Value to assign to the masked columns. +/// \param axis: Axis to apply masking on (1 -> frequency, 2 -> time). +/// \return Status code. Status MaskAlongAxis(const std::shared_ptr &input, std::shared_ptr *output, int64_t mask_width, int64_t mask_start, double mask_value, int axis); } // namespace dataset diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/band_biquad_op.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/band_biquad_op.cc index a6ea14f550f..9ea9a62e3d3 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/band_biquad_op.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/band_biquad_op.cc @@ -25,12 +25,12 @@ Status BandBiquadOp::Compute(const std::shared_ptr &input, std::shared_p IO_CHECK(input, output); TensorShape input_shape = input->shape(); // check input tensor dimension, it should be greater than 0. - CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BandBiquad: input dimension should be greater than 0."); + CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BandBiquad: input tensor is not in shape of <..., time>."); // check input type, it should be DE_FLOAT32 or DE_FLOAT16 or DE_FLOAT64 CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType(DataType::DE_FLOAT32) || input->type() == DataType(DataType::DE_FLOAT16) || input->type() == DataType(DataType::DE_FLOAT64), - "BandBiquad: input type should be float, but got " + input->type().ToString()); + "BandBiquad: input tensor type should be float, but got: " + input->type().ToString()); double w0 = 2 * PI * central_freq_ / sample_rate_; double bw_Hz = central_freq_ / Q_; double a0 = 1.; @@ -45,15 +45,16 @@ Status BandBiquadOp::Compute(const std::shared_ptr &input, std::shared_p } double b1 = 0.; double b2 = 0.; - if (input->type() == DataType(DataType::DE_FLOAT32)) + if (input->type() == DataType(DataType::DE_FLOAT32)) { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); - else if (input->type() == DataType(DataType::DE_FLOAT64)) + } else if (input->type() == DataType(DataType::DE_FLOAT64)) { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); - else + } else { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); + } } } // namespace dataset diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/band_biquad_op.h b/mindspore/ccsrc/minddata/dataset/audio/kernels/band_biquad_op.h index 7ddb3ca6832..c92bda5fdd6 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/band_biquad_op.h +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/band_biquad_op.h @@ -17,8 +17,8 @@ #define MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_KERNELS_BAND_BIQUAD_OP_H_ #include -#include #include +#include #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/kernels/tensor_op.h" @@ -26,6 +26,7 @@ namespace mindspore { namespace dataset { + class BandBiquadOp : public TensorOp { public: BandBiquadOp(int32_t sample_rate, float central_freq, float Q, bool noise) diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/bandpass_biquad_op.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/bandpass_biquad_op.cc index 475485f0e1a..ab0fa546f3a 100755 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/bandpass_biquad_op.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/bandpass_biquad_op.cc @@ -24,12 +24,12 @@ namespace dataset { Status BandpassBiquadOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { IO_CHECK(input, output); TensorShape input_shape = input->shape(); - CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BandpassBiquad: inpute dimension should be greater than 0."); + CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BandpassBiquad: input tensor is not in shape of <..., time>."); // check input type, it should be DE_FLOAT32 or DE_FLOAT16 or DE_FLOAT64 - CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType(DataType::DE_FLOAT32) || - input->type() == DataType(DataType::DE_FLOAT16) || - input->type() == DataType(DataType::DE_FLOAT64), - "BandpassBiquad: input type should be float, but got " + input->type().ToString()); + CHECK_FAIL_RETURN_UNEXPECTED( + input->type() == DataType(DataType::DE_FLOAT32) || input->type() == DataType(DataType::DE_FLOAT16) || + input->type() == DataType(DataType::DE_FLOAT64), + "BandpassBiquad: input tensor type should be float, but got: " + input->type().ToString()); float w0 = 2 * PI * central_freq_ / sample_rate_; float alpha = sin(w0) / 2 / Q_; float temp; @@ -46,15 +46,16 @@ Status BandpassBiquadOp::Compute(const std::shared_ptr &input, std::shar float a1 = (-2) * cos(w0); float a2 = 1 - alpha; - if (input->type() == DataType(DataType::DE_FLOAT32)) + if (input->type() == DataType(DataType::DE_FLOAT32)) { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); - else if (input->type() == DataType(DataType::DE_FLOAT64)) + } else if (input->type() == DataType(DataType::DE_FLOAT64)) { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); - else + } else { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); + } } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/bandpass_biquad_op.h b/mindspore/ccsrc/minddata/dataset/audio/kernels/bandpass_biquad_op.h index 0fb21441425..dead035fbc4 100755 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/bandpass_biquad_op.h +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/bandpass_biquad_op.h @@ -17,8 +17,8 @@ #define MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_KERNELS_BANDPASS_BIQUAD_OP_H_ #include -#include #include +#include #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/kernels/tensor_op.h" @@ -26,6 +26,7 @@ namespace mindspore { namespace dataset { + class BandpassBiquadOp : public TensorOp { public: BandpassBiquadOp(int32_t sample_rate, float central_freq, float Q, bool const_skirt_gain) diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/bandreject_biquad_op.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/bandreject_biquad_op.cc index d321cbf6d52..0e9244af2b1 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/bandreject_biquad_op.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/bandreject_biquad_op.cc @@ -20,15 +20,17 @@ namespace mindspore { namespace dataset { + Status BandrejectBiquadOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { IO_CHECK(input, output); // check input type and input shape TensorShape input_shape = input->shape(); - CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BandrejectBiquad: input dimension should be greater than 0."); - CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType(DataType::DE_FLOAT32) || - input->type() == DataType(DataType::DE_FLOAT16) || - input->type() == DataType(DataType::DE_FLOAT64), - "BandrejectBiquad: input type should be float, but got " + input->type().ToString()); + CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, + "BandrejectBiquad: input tensor is not in shape of <..., time>."); + CHECK_FAIL_RETURN_UNEXPECTED( + input->type() == DataType(DataType::DE_FLOAT32) || input->type() == DataType(DataType::DE_FLOAT16) || + input->type() == DataType(DataType::DE_FLOAT64), + "BandrejectBiquad: input tensor type should be float, but got: " + input->type().ToString()); double w0 = 2 * PI * central_freq_ / sample_rate_; double alpha = sin(w0) / 2 / Q_; double b0 = 1; @@ -37,15 +39,16 @@ Status BandrejectBiquadOp::Compute(const std::shared_ptr &input, std::sh double a0 = 1 + alpha; double a1 = b1; double a2 = 1 - alpha; - if (input->type() == DataType(DataType::DE_FLOAT32)) + if (input->type() == DataType(DataType::DE_FLOAT32)) { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); - else if (input->type() == DataType(DataType::DE_FLOAT64)) + } else if (input->type() == DataType(DataType::DE_FLOAT64)) { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); - else + } else { return Biquad(input, output, static_cast(b0), static_cast(b1), static_cast(b2), static_cast(a0), static_cast(a1), static_cast(a2)); + } } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/bandreject_biquad_op.h b/mindspore/ccsrc/minddata/dataset/audio/kernels/bandreject_biquad_op.h index 3b42a6ccb82..e59d0cf3220 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/bandreject_biquad_op.h +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/bandreject_biquad_op.h @@ -26,6 +26,7 @@ namespace mindspore { namespace dataset { + class BandrejectBiquadOp : public TensorOp { public: BandrejectBiquadOp(int32_t sample_rate, float central_freq, float Q) diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/bass_biquad_op.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/bass_biquad_op.cc index 71799b17852..d05a7ff2471 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/bass_biquad_op.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/bass_biquad_op.cc @@ -24,12 +24,12 @@ namespace dataset { Status BassBiquadOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { IO_CHECK(input, output); TensorShape input_shape = input->shape(); - CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BassBiquad: input dimension should be greater than 0."); + CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BassBiquad: input tensor is not in shape of <..., time>."); // check input type, it should be DE_FLOAT32 or DE_FLOAT16 or DE_FLOAT64 CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType(DataType::DE_FLOAT32) || input->type() == DataType(DataType::DE_FLOAT16) || input->type() == DataType(DataType::DE_FLOAT64), - "BassBiquad: input type should be float, but got " + input->type().ToString()); + "BassBiquad: input tensor type should be float, but got: " + input->type().ToString()); double w0 = 2 * PI * central_freq_ / sample_rate_; double alpha = sin(w0) / 2 / Q_; @@ -45,17 +45,18 @@ Status BassBiquadOp::Compute(const std::shared_ptr &input, std::shared_p double a0 = (A + 1) + temp2 + temp1; double a1 = -2 * ((A - 1) + temp3); double a2 = (A + 1) + temp2 - temp1; - if (input->type() == DataType(DataType::DE_FLOAT32)) + if (input->type() == DataType(DataType::DE_FLOAT32)) { return Biquad(input, output, static_cast(b0 / a0), static_cast(b1 / a0), static_cast(b2 / a0), static_cast(1.0), static_cast(a1 / a0), static_cast(a2 / a0)); - else if (input->type() == DataType(DataType::DE_FLOAT64)) + } else if (input->type() == DataType(DataType::DE_FLOAT64)) { return Biquad(input, output, static_cast(b0 / a0), static_cast(b1 / a0), static_cast(b2 / a0), static_cast(1.0), static_cast(a1 / a0), static_cast(a2 / a0)); - else + } else { return Biquad(input, output, static_cast(b0 / a0), static_cast(b1 / a0), static_cast(b2 / a0), static_cast(1.0), static_cast(a1 / a0), static_cast(a2 / a0)); + } } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/bass_biquad_op.h b/mindspore/ccsrc/minddata/dataset/audio/kernels/bass_biquad_op.h index 2aa31f2428c..68552c1bb80 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/bass_biquad_op.h +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/bass_biquad_op.h @@ -27,6 +27,7 @@ namespace mindspore { namespace dataset { + class BassBiquadOp : public TensorOp { public: BassBiquadOp(int32_t sample_rate, float gain, float central_freq, float Q) @@ -35,7 +36,7 @@ class BassBiquadOp : public TensorOp { ~BassBiquadOp() override = default; void Print(std::ostream &out) const override { - out << Name() << ": sample_rate: " << sample_rate_ << ", gain:" << gain_ << ", central_freq: " << central_freq_ + out << Name() << ": sample_rate: " << sample_rate_ << ", gain: " << gain_ << ", central_freq: " << central_freq_ << ", Q: " << Q_ << std::endl; } diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/frequency_masking_op.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/frequency_masking_op.cc index 1b7daa5fe80..56bbfe09dfa 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/frequency_masking_op.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/frequency_masking_op.cc @@ -47,7 +47,7 @@ Status FrequencyMaskingOp::Compute(const std::shared_ptr &input, std::sh std::shared_ptr input_tensor; // typecast CHECK_FAIL_RETURN_UNEXPECTED(input->type() != DataType::DE_STRING, - "FrequencyMasking: input tensor type should be float, but got string."); + "FrequencyMasking: input tensor type should be float, but got: string."); if (input->type() != DataType::DE_FLOAT64) { RETURN_IF_NOT_OK(TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32))); } else { diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/time_masking_op.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/time_masking_op.cc index dd739ee6ab5..0bf309955ef 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/time_masking_op.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/time_masking_op.cc @@ -33,7 +33,7 @@ TimeMaskingOp::TimeMaskingOp(bool iid_masks, int64_t time_mask_param, int64_t ma Status TimeMaskingOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { IO_CHECK(input, output); // input <..., freq, time> - CHECK_FAIL_RETURN_UNEXPECTED(input->Rank() >= 2, "TimeMasking: input dimension must be greater than 2."); + CHECK_FAIL_RETURN_UNEXPECTED(input->Rank() >= 2, "TimeMasking: input tensor is not in shape of <..., freq, time>."); TensorShape input_shape = input->shape(); CHECK_FAIL_RETURN_UNEXPECTED(input_shape[-1] >= time_mask_param_, "TimeMasking: time_mask_param should be less than the length of time dimension."); @@ -41,7 +41,7 @@ Status TimeMaskingOp::Compute(const std::shared_ptr &input, std::shared_ std::shared_ptr input_tensor; // typecast CHECK_FAIL_RETURN_UNEXPECTED(input->type() != DataType::DE_STRING, - "TimeMasking: input type should be float, but got string."); + "TimeMasking: input tensor type should be float, but got: string."); if (input->type() != DataType::DE_FLOAT64) { RETURN_IF_NOT_OK(TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32))); } else { diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/time_stretch_op.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/time_stretch_op.cc index 0f990348ff7..05a14891b00 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/time_stretch_op.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/time_stretch_op.cc @@ -33,15 +33,8 @@ Status TimeStretchOp::Compute(const std::shared_ptr &input, std::shared_ IO_CHECK(input, output); // check shape - if (input->shape().Rank() < 3) { - std::string err_msg = "TimeStretch: input tensor shape is not <..., freq, num_frame, complex=2>."; - MS_LOG(ERROR) << err_msg; - RETURN_STATUS_SYNTAX_ERROR(err_msg); - } - - // check complex - if (!input->IsComplex()) { - std::string err_msg = "TimeStretch: input tensor is not in shape of <..., 2>."; + if (input->shape().Rank() < 3 || !input->IsComplex()) { + std::string err_msg = "TimeStretch: input tensor is not in shape of <..., freq, num_frame, complex=2>."; MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); } @@ -51,7 +44,7 @@ Status TimeStretchOp::Compute(const std::shared_ptr &input, std::shared_ float hop_length = std::isnan(hop_length_) ? (n_freq_ - 1) : hop_length_; // typecast CHECK_FAIL_RETURN_UNEXPECTED(input->type() != DataType::DE_STRING, - "TimeStretch: input tensor type should be [int, float, double], but got string."); + "TimeStretch: input tensor type should be int, float or double, but got: string."); if (input->type() != DataType::DE_FLOAT64) { RETURN_IF_NOT_OK(TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32))); } else { diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/time_stretch_op.h b/mindspore/ccsrc/minddata/dataset/audio/kernels/time_stretch_op.h index d1a119bf8c0..5a7b1dd9d59 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/time_stretch_op.h +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/time_stretch_op.h @@ -27,6 +27,7 @@ namespace mindspore { namespace dataset { + class TimeStretchOp : public TensorOp { public: /// Default value @@ -43,9 +44,6 @@ class TimeStretchOp : public TensorOp { std::string Name() const override { return kTimeStretchOp; } - /// \param[in] inputs - /// \param[out] outputs - /// \return Status code Status OutputShape(const std::vector &inputs, std::vector &outputs) override; private: diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/flickr_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/flickr_op.cc index 65735a488f9..e4e895a81ed 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/flickr_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/flickr_op.cc @@ -17,10 +17,10 @@ #include #include -#include #include #include +#include "debug/common.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/tensor_shape.h" #include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h" @@ -94,7 +94,13 @@ void FlickrOp::Print(std::ostream &out, bool show_all) const { } Status FlickrOp::ParseFlickrData() { - std::ifstream file_handle(file_path_); + auto real_file_path = Common::GetRealPath(file_path_); + if (!real_file_path.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file_path_; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file_path_); + } + + std::ifstream file_handle(real_file_path.value()); if (!file_handle.is_open()) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open Flickr annotation file: " + file_path_); } @@ -129,7 +135,11 @@ Status FlickrOp::ParseFlickrData() { } bool valid = false; - RETURN_IF_NOT_OK(CheckImageType(image_file_path, &valid)); + Status type_check = CheckImageType(image_file_path, &valid); + if (type_check.IsError()) { + file_handle.close(); + RETURN_IF_NOT_OK(type_check); + } if (!valid) { continue; } @@ -153,10 +163,16 @@ Status FlickrOp::ParseFlickrData() { // Optimization: Could take in a tensor // This function does not return status because we want to just skip bad input, not crash Status FlickrOp::CheckImageType(const std::string &file_name, bool *valid) { + auto real_file_name = Common::GetRealPath(file_name); + if (!real_file_name.has_value()) { + MS_LOG(ERROR) << "Get real path failed, path=" << file_name; + RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file_name); + } + std::ifstream file_handle; constexpr int read_num = 3; *valid = false; - file_handle.open(file_name, std::ios::binary | std::ios::in); + file_handle.open(real_file_name.value(), std::ios::binary | std::ios::in); if (!file_handle.is_open()) { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open image file: " + file_name); } diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/audio.h b/mindspore/ccsrc/minddata/dataset/include/dataset/audio.h index db6b6d789e5..05c73543fa3 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/audio.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/audio.h @@ -35,6 +35,7 @@ class TensorOperation; // Transform operations for performing computer audio. namespace audio { + /// \brief Compute the angle of complex tensor input. class Angle final : public TensorTransform { public: @@ -98,10 +99,10 @@ class AllpassBiquad final : public TensorTransform { class AmplitudeToDB final : public TensorTransform { public: /// \brief Constructor. - /// \param[in] stype ['kPower', 'kMagnitude'] - /// \param[in] ref_value Calculate db_multiplier - /// \param[in] amin Clamp the input waveform - /// \param[in] top_db Decibels cut-off value + /// \param[in] stype ['kPower', 'kMagnitude']. + /// \param[in] ref_value Calculate db_multiplier. + /// \param[in] amin Clamp the input waveform. + /// \param[in] top_db Decibels cut-off value. explicit AmplitudeToDB(ScaleType stype = ScaleType::kPower, float ref_value = 1.0, float amin = 1e-10, float top_db = 80.0); @@ -124,9 +125,9 @@ class BandpassBiquad final : public TensorTransform { /// \brief Constructor. /// \param[in] sample_rate Sampling rate of the waveform, e.g. 44100 (Hz). /// \param[in] central_freq Central frequency (in Hz). - /// \param[in] Q Quality factor, https://en.wikipedia.org/wiki/Q_factor (Default: 0.707). + /// \param[in] Q Quality factor, https://en.wikipedia.org/wiki/Q_factor (Default: 0.707). /// \param[in] const_skirt_gain, If ``True``, uses a constant skirt gain (peak gain = Q). If ``False``, uses a - /// constant 0dB peak gain. (Default: False). + /// constant 0dB peak gain (Default: False). explicit BandpassBiquad(int32_t sample_rate, float central_freq, float Q = 0.707, bool const_skirt_gain = false); /// \brief Destructor. diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h b/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h index c16b6e9e22b..b9775252570 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/vision.h @@ -36,26 +36,6 @@ class TensorOperation; // Transform operations for performing computer vision. namespace vision { -/// \brief Apply automatic contrast on the input image. -class AutoContrast final : public TensorTransform { - public: - /// \brief Constructor. - /// \param[in] cutoff Percent of pixels to cut off from the histogram, the valid range of cutoff value is 0 to 50. - /// \param[in] ignore Pixel values to ignore. - explicit AutoContrast(float cutoff = 0.0, std::vector ignore = {}); - - /// \brief Destructor. - ~AutoContrast() = default; - - protected: - /// \brief The function to convert a TensorTransform object into a TensorOperation object. - /// \return Shared pointer to TensorOperation object. - std::shared_ptr Parse() override; - - private: - struct Data; - std::shared_ptr data_; -}; /// \brief AdjustGamma TensorTransform. /// \notes Apply gamma correction on input image. @@ -80,6 +60,27 @@ class AdjustGamma final : public TensorTransform { std::shared_ptr data_; }; +/// \brief Apply automatic contrast on the input image. +class AutoContrast final : public TensorTransform { + public: + /// \brief Constructor. + /// \param[in] cutoff Percent of pixels to cut off from the histogram, the valid range of cutoff value is 0 to 50. + /// \param[in] ignore Pixel values to ignore. + explicit AutoContrast(float cutoff = 0.0, std::vector ignore = {}); + + /// \brief Destructor. + ~AutoContrast() = default; + + protected: + /// \brief The function to convert a TensorTransform object into a TensorOperation object. + /// \return Shared pointer to TensorOperation object. + std::shared_ptr Parse() override; + + private: + struct Data; + std::shared_ptr data_; +}; + /// \brief BoundingBoxAugment TensorTransform. /// \note Apply a given image transform on a random selection of bounding box regions of a given image. class BoundingBoxAugment final : public TensorTransform { diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h b/mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h index ff8e26bc397..e47accce1bb 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h @@ -89,39 +89,6 @@ class CenterCrop final : public TensorTransform { std::shared_ptr data_; }; -/// \brief RGB2BGR TensorTransform. -/// \notes Convert the format of input image from RGB to BGR. -class RGB2BGR final : public TensorTransform { - public: - /// \brief Constructor. - RGB2BGR() = default; - - /// \brief Destructor. - ~RGB2BGR() = default; - - protected: - /// \brief The function to convert a TensorTransform object into a TensorOperation object. - /// \return Shared pointer to TensorOperation object. - std::shared_ptr Parse() override; -}; - -/// \brief RGB2GRAY TensorTransform. -/// \note Convert RGB image or color image to grayscale image. -/// \brief Convert a RGB image or color image to a grayscale one. -class RGB2GRAY final : public TensorTransform { - public: - /// \brief Constructor. - RGB2GRAY() = default; - - /// \brief Destructor. - ~RGB2GRAY() = default; - - protected: - /// \brief The function to convert a TensorTransform object into a TensorOperation object. - /// \return Shared pointer to TensorOperation object. - std::shared_ptr Parse() override; -}; - /// \brief Crop an image based on location and crop size. class Crop final : public TensorTransform { public: @@ -308,6 +275,39 @@ class ResizePreserveAR final : public TensorTransform { std::shared_ptr data_; }; +/// \brief RGB2BGR TensorTransform. +/// \notes Convert the format of input image from RGB to BGR. +class RGB2BGR final : public TensorTransform { + public: + /// \brief Constructor. + RGB2BGR() = default; + + /// \brief Destructor. + ~RGB2BGR() = default; + + protected: + /// \brief The function to convert a TensorTransform object into a TensorOperation object. + /// \return Shared pointer to TensorOperation object. + std::shared_ptr Parse() override; +}; + +/// \brief RGB2GRAY TensorTransform. +/// \note Convert RGB image or color image to grayscale image. +/// \brief Convert a RGB image or color image to a grayscale one. +class RGB2GRAY final : public TensorTransform { + public: + /// \brief Constructor. + RGB2GRAY() = default; + + /// \brief Destructor. + ~RGB2GRAY() = default; + + protected: + /// \brief The function to convert a TensorTransform object into a TensorOperation object. + /// \return Shared pointer to TensorOperation object. + std::shared_ptr Parse() override; +}; + /// \brief Rotate the input image according to parameters. class Rotate final : public TensorTransform { public: diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/adjust_gamma_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/adjust_gamma_op.cc index 3698482fe56..338d257d547 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/adjust_gamma_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/adjust_gamma_op.cc @@ -15,21 +15,21 @@ */ #include "minddata/dataset/kernels/image/adjust_gamma_op.h" -#include + #include "minddata/dataset/kernels/data/data_utils.h" #include "minddata/dataset/kernels/image/image_utils.h" namespace mindspore { namespace dataset { -const float AdjustGammaOp::kGain = 1.0; +constexpr float AdjustGammaOp::kGain = 1.0; Status AdjustGammaOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { IO_CHECK(input, output); // typecast CHECK_FAIL_RETURN_UNEXPECTED(input->type() != DataType::DE_STRING, - "AdjustGamma: input tensor type should be [int, float, double], but got string."); + "AdjustGamma: input tensor type should be int, float or double, but got: string."); if (input->type().IsFloat()) { std::shared_ptr input_tensor; diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc index c8c263cd369..77d4931d8a3 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc @@ -877,7 +877,7 @@ Status AdjustGamma(const std::shared_ptr &input, std::shared_ptr try { int num_channels = 1; if (input->Rank() < 2) { - RETURN_STATUS_UNEXPECTED("AdjustGamma: image shape is not <...,H,W,C> or ."); + RETURN_STATUS_UNEXPECTED("AdjustGamma: input tensor is not in shape of <...,H,W,C> or ."); } if (input->Rank() > 2) { num_channels = input->shape()[-1]; @@ -1261,7 +1261,7 @@ Status RgbToBgr(const std::shared_ptr &input, std::shared_ptr *o RETURN_STATUS_UNEXPECTED("RgbToBgr: load image failed."); } if (input_cv->Rank() != 3 || input_cv->shape()[2] != 3) { - RETURN_STATUS_UNEXPECTED("RgbToBgr: image shape is not or channel is not 3."); + RETURN_STATUS_UNEXPECTED("RgbToBgr: input tensor is not in shape of or channel is not 3."); } cv::Mat image = input_cv->mat().clone(); diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/rgb_to_bgr_op.h b/mindspore/ccsrc/minddata/dataset/kernels/image/rgb_to_bgr_op.h index 031bd1982e2..b80940cab00 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/rgb_to_bgr_op.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/rgb_to_bgr_op.h @@ -17,8 +17,8 @@ #define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_RGB_TO_BGR_OP_H_ #include -#include #include +#include #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/kernels/tensor_op.h" diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/adjust_gamma_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/adjust_gamma_ir.cc index 8b81888f965..52c75289141 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/adjust_gamma_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/adjust_gamma_ir.cc @@ -13,8 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include - #include "minddata/dataset/kernels/ir/vision/adjust_gamma_ir.h" #ifndef ENABLE_ANDROID diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/rgb_to_bgr_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/rgb_to_bgr_ir.cc index 4f23dcffb07..1295acb837d 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/rgb_to_bgr_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/rgb_to_bgr_ir.cc @@ -13,14 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include - #include "minddata/dataset/kernels/ir/vision/rgb_to_bgr_ir.h" #include "minddata/dataset/kernels/image/rgb_to_bgr_op.h" -#include "minddata/dataset/kernels/ir/validators.h" - namespace mindspore { namespace dataset { diff --git a/mindspore/dataset/audio/__init__.py b/mindspore/dataset/audio/__init__.py index 2d695cb34a3..067cd2414d4 100644 --- a/mindspore/dataset/audio/__init__.py +++ b/mindspore/dataset/audio/__init__.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# ============================================================================== """ This module is to support audio augmentations. """ from . import transforms +from . import utils diff --git a/mindspore/dataset/audio/transforms.py b/mindspore/dataset/audio/transforms.py index 5bc190b00b9..e2640522bbd 100644 --- a/mindspore/dataset/audio/transforms.py +++ b/mindspore/dataset/audio/transforms.py @@ -11,14 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# ============================================================================== """ -The module audio.transforms is inherited from _c_dataengine. -and is implemented based on C++. It's a high performance module to -process audio. Users can apply suitable augmentations on audio data -to improve their training models. +The module audio.transforms is inherited from _c_dataengine and is +implemented based on C++. It's a high performance module to process +audio. Users can apply suitable augmentations on audio data to improve +their training models. """ -import mindspore._c_dataengine as cde + import numpy as np + +import mindspore._c_dataengine as cde from ..transforms.c_transforms import TensorOperation from .utils import ScaleType from .validators import check_allpass_biquad, check_amplitude_to_db, check_band_biquad, check_bandpass_biquad, \ @@ -33,38 +36,31 @@ class AudioTensorOperation(TensorOperation): def __call__(self, *input_tensor_list): for tensor in input_tensor_list: if not isinstance(tensor, (np.ndarray,)): - raise TypeError( - "Input should be NumPy audio, got {}.".format(type(tensor))) + raise TypeError("Input should be NumPy audio, got {}.".format(type(tensor))) return super().__call__(*input_tensor_list) def parse(self): - raise NotImplementedError( - "AudioTensorOperation has to implement parse() method.") + raise NotImplementedError("AudioTensorOperation has to implement parse() method.") class AllpassBiquad(AudioTensorOperation): """ - Design two-pole all-pass filter for audio waveform of dimension of `(..., time)` + Design two-pole all-pass filter for audio waveform of dimension of (..., time). - Args: - sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), - the value must be greater than 0 . - central_freq (float): central frequency (in Hz), - the value must be greater than 0 . - Q(float, optional): Quality factor,https://en.wikipedia.org/wiki/Q_factor, - Range: (0, 1] (Default=0.707). + Args: + sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), the value must be greater than 0. + central_freq (float): central frequency (in Hz), the value must be greater than 0. + Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). - Examples: - >>> import mindspore.dataset.audio.transforms as audio - >>> import numpy as np - - >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03],[9.246826171875e-03, 1.0894775390625e-02]]) - >>> allpasspass_biquad_op = audio.AllpassBiquad(44100, 200.0) - >>> waveform_filtered = allpass_biquad_op(waveform) - - References: - https://www.w3.org/2011/audio/audio-eq-cookbook.html#APF + Examples: + >>> import numpy as np + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.AllpassBiquad(44100, 200.0)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) """ + @check_allpass_biquad def __init__(self, sample_rate, central_freq, Q=0.707): self.sample_rate = sample_rate @@ -84,23 +80,22 @@ class AmplitudeToDB(AudioTensorOperation): Converts the input tensor from amplitude/power scale to decibel scale. Args: - stype (ScaleType, optional): Scale of the input tensor. (Default="ScaleType.POWER"). - It can be any of [ScaleType.MAGNITUDE, ScaleType.POWER]. + stype (ScaleType, optional): Scale of the input tensor (default=ScaleType.POWER). + It can be one of ScaleType.MAGNITUDE or ScaleType.POWER. ref_value (float, optional): Param for generate db_multiplier. amin (float, optional): Lower bound to clamp the input waveform. - top_db (float, optional): Minimum cut-off decibels. The range of values is non-negative. Commonly set at 80. - (Default=80.0) + top_db (float, optional): Minimum cut-off decibels. The range of values is non-negative. + Commonly set at 80 (default=80.0). Examples: - >>> channel = 1 - >>> n_fft = 400 - >>> n_frame = 30 - >>> specrogram = np.random.random([channel, n_fft//2+1, n_frame]) - >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=specrogram, column_names=["audio"]) + >>> import numpy as np + >>> + >>> waveform = np.random.random([1, 400//2+1, 30]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) >>> transforms = [audio.AmplitudeToDB(stype=ScaleType.POWER)] >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) """ - @ check_amplitude_to_db + @check_amplitude_to_db def __init__(self, stype=ScaleType.POWER, ref_value=1.0, amin=1e-10, top_db=80.0): self.stype = stype self.ref_value = ref_value @@ -115,15 +110,14 @@ class Angle(AudioTensorOperation): """ Calculate the angle of the complex number sequence of shape (..., 2). The first dimension represents the real part while the second represents the imaginary. - Args: Examples: - >>> import mindspore.dataset.audio.transforms as audio >>> import numpy as np - - >>> input_complex = np.array([[1.43, 5.434], [23.54, 89.38]]) - >>> angle_op = audio.Angle() - >>> angles = angle_op(input_complex) + >>> + >>> waveform = np.array([[1.43, 5.434], [23.54, 89.38]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Angle()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) """ def parse(self): @@ -132,24 +126,24 @@ class Angle(AudioTensorOperation): class BandBiquad(AudioTensorOperation): """ - Design two-pole band filter for audio waveform of dimension of `(..., time)` + Design two-pole band filter for audio waveform of dimension of (..., time). Args: - sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero. - central_freq (float): central frequency (in Hz), - Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, Range: (0, 1] (Default=0.707). - noise (bool, optional) : If ``True``, uses the alternate mode for un-pitched audio (e.g. percussion). - If ``False``, uses mode oriented to pitched audio, i.e. voice, singing, - or instrumental music (Default: ``False``). + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero. + central_freq (float): Central frequency (in Hz). + Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). + noise (bool, optional) : If True, uses the alternate mode for un-pitched audio (e.g. percussion). + If False, uses mode oriented to pitched audio, i.e. voice, singing, or instrumental music (default=False). Examples: - >>> import mindspore.dataset.audio.transforms as audio >>> import numpy as np - - >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03],[9.246826171875e-03, 1.0894775390625e-02]]) - >>> band_biquad_op = audio.BandBiquad(44100, 200.0) - >>> waveform_filtered = band_biquad_op(waveform) + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.BandBiquad(44100, 200.0)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) """ + @check_band_biquad def __init__(self, sample_rate, central_freq, Q=0.707, noise=False): self.sample_rate = sample_rate @@ -161,25 +155,26 @@ class BandBiquad(AudioTensorOperation): return cde.BandBiquadOperation(self.sample_rate, self.central_freq, self.Q, self.noise) -class BandpassBiquad(TensorOperation): +class BandpassBiquad(AudioTensorOperation): """ - Design two-pole band-pass filter. Similar to SoX implementation. + Design two-pole band-pass filter. Similar to SoX implementation. Args: - sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz) - central_freq (float): central frequency (in Hz) - Q (float, optional): https://en.wikipedia.org/wiki/Q_factor Range: (0,1] (Default=0.707). - const_skirt_gain (bool, optional) : If ``True``, uses a constant skirt gain (peak gain = Q). - If ``False``, uses a constant 0dB peak gain. (Default: ``False``) + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz). + central_freq (float): Central frequency (in Hz). + Q (float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0,1] (default=0.707). + const_skirt_gain (bool, optional) : If True, uses a constant skirt gain (peak gain = Q). + If False, uses a constant 0dB peak gain (default=False). Examples: - >>> import mindspore.dataset.audio.transforms as audio >>> import numpy as np - - >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03],[9.246826171875e-03, 1.0894775390625e-02]]) - >>> bandpass_biquad_op = audio.BandpassBiquad(44100, 200.0) - >>> waveform_filtered = bandpass_biquad_op(waveform) + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.BandpassBiquad(44100, 200.0)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) """ + @check_bandpass_biquad def __init__(self, sample_rate, central_freq, Q=0.707, const_skirt_gain=False): self.sample_rate = sample_rate @@ -193,23 +188,20 @@ class BandpassBiquad(TensorOperation): class BandrejectBiquad(AudioTensorOperation): """ - Design two-pole band filter for audio waveform of dimension of `(..., time)` + Design two-pole band filter for audio waveform of dimension of (..., time). Args: - sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), - the value must be greater than 0 . - central_freq (float): central frequency (in Hz), - the value must be greater than 0 . - Q(float, optional): Quality factor,https://en.wikipedia.org/wiki/Q_factor, - Range: (0, 1] (Default=0.707). + sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), the value must be greater than 0. + central_freq (float): central frequency (in Hz), the value must be greater than 0. + Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). Examples: - >>> import mindspore.dataset.audio.transforms as audio >>> import numpy as np - + >>> >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03],[9.246826171875e-03, 1.0894775390625e-02]]) - >>> band_biquad_op = audio.BandBiquad(44100, 200.0) - >>> waveform_filtered = band_biquad_op(waveform) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.BandrejectBiquad(44100, 200.0)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) """ @check_bandreject_biquad @@ -224,22 +216,23 @@ class BandrejectBiquad(AudioTensorOperation): class BassBiquad(AudioTensorOperation): """ - Design a bass tone-control effect for audio waveform of dimension of `(..., time)` + Design a bass tone-control effect for audio waveform of dimension of (..., time). Args: - sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz) - gain (float): desired gain at the boost (or attenuation) in dB. - central_freq (float): central frequency (in Hz)(Default=100.0). - Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, Range: (0, 1] (Default=0.707). + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz). + gain (float): Desired gain at the boost (or attenuation) in dB. + central_freq (float): Central frequency (in Hz) (default=100.0). + Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). Examples: - >>> import mindspore.dataset.audio.transforms as audio >>> import numpy as np - - >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03],[9.246826171875e-03, 1.0894775390625e-02]]) - >>> bass_biquad_op = audio.BassBiquad(44100, 100.0) - >>> waveform_filtered = bass_biquad_op(waveform) + >>> + >>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.BassBiquad(44100, 100.0)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) """ + @check_bass_biquad def __init__(self, sample_rate, gain, central_freq=100.0, Q=0.707): self.sample_rate = sample_rate @@ -263,14 +256,12 @@ class FrequencyMasking(AudioTensorOperation): mask_value (double): Mask value (default=0.0). Examples: - >>> def gen(): - ... random.seed(0) - ... data = numpy.random.random([1, 3, 2]) - ... yield (numpy.array(data, dtype=numpy.float32),) - >>> dataset = ds.GeneratorDataset(source=gen, - ... column_names=["multi_dim_data"]) - >>> dataset = dataset.map(operations=FrequencyMasking(frequency_mask_param=1), - ... input_columns=["multi_dim_data"]) + >>> import numpy as np + >>> + >>> waveform = np.random.random([1, 3, 2]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.FrequencyMasking(frequency_mask_param=1)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) """ @check_masking def __init__(self, iid_masks=False, frequency_mask_param=0, mask_start=0, mask_value=0.0): @@ -296,15 +287,14 @@ class TimeMasking(AudioTensorOperation): mask_value (double): Mask value (default=0.0). Examples: - >>> def gen(): - ... random.seed(0) - ... data = numpy.random.random([1, 3, 2]) - ... yield (numpy.array(data, dtype=numpy.float32),) - >>> dataset = ds.GeneratorDataset(source=gen, - ... column_names=["multi_dim_data"]) - >>> dataset = dataset.map(operations=TimeMasking(time_mask_param=1), - ... input_columns=["multi_dim_data"]) + >>> import numpy as np + >>> + >>> waveform = np.random.random([1, 3, 2]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.TimeMasking(time_mask_param=1)] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) """ + @check_masking def __init__(self, iid_masks=False, time_mask_param=0, mask_start=0, mask_value=0.0): self.iid_masks = iid_masks diff --git a/mindspore/dataset/audio/utils.py b/mindspore/dataset/audio/utils.py index 1bf00f2da0d..3b1f42579eb 100644 --- a/mindspore/dataset/audio/utils.py +++ b/mindspore/dataset/audio/utils.py @@ -11,9 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# ============================================================================== """ enum for audio ops """ + from enum import Enum diff --git a/mindspore/dataset/audio/validators.py b/mindspore/dataset/audio/validators.py index 6654ac35c50..f24f9ab7dbf 100644 --- a/mindspore/dataset/audio/validators.py +++ b/mindspore/dataset/audio/validators.py @@ -15,15 +15,17 @@ """ Validators for TensorOps. """ + from functools import wraps -from mindspore.dataset.core.validator_helpers import check_not_zero, check_int32, check_float32, check_value, \ - check_value_normalize_std, check_value_ratio, DOUBLE_MAX_INTEGER, FLOAT_MAX_INTEGER, INT64_MAX, parse_user_args, \ - type_check + +from mindspore.dataset.core.validator_helpers import check_float32, check_int32_not_zero, \ + check_non_negative_float32, check_non_negative_float64, check_pos_float32, check_pos_int64, check_value, \ + parse_user_args, type_check from .utils import ScaleType def check_amplitude_to_db(method): - """Wrapper method to check the parameters of amplitude_to_db.""" + """Wrapper method to check the parameters of AmplitudeToDB.""" @wraps(method) def new_method(self, *args, **kwargs): @@ -35,30 +37,30 @@ def check_amplitude_to_db(method): # type check ref_value type_check(ref_value, (int, float), "ref_value") # value check ref_value - if not ref_value is None: - check_value_ratio(ref_value, (0, FLOAT_MAX_INTEGER), "ref_value") + if ref_value is not None: + check_pos_float32(ref_value, "ref_value") # type check amin type_check(amin, (int, float), "amin") # value check amin - if not amin is None: - check_value_ratio(amin, (0, FLOAT_MAX_INTEGER), "amin") + if amin is not None: + check_pos_float32(amin, "amin") # type check top_db type_check(top_db, (int, float), "top_db") # value check top_db - if not top_db is None: - check_value_ratio(top_db, (0, FLOAT_MAX_INTEGER), "top_db") + if top_db is not None: + check_pos_float32(top_db, "top_db") return method(self, *args, **kwargs) + return new_method def check_biquad_sample_rate(sample_rate): """Wrapper method to check the parameters of sample_rate.""" type_check(sample_rate, (int,), "sample_rate") - check_int32(sample_rate, "sample_rate") - check_not_zero(sample_rate, "sample_rate") + check_int32_not_zero(sample_rate, "sample_rate") def check_biquad_central_freq(central_freq): @@ -70,7 +72,7 @@ def check_biquad_central_freq(central_freq): def check_biquad_Q(Q): """Wrapper method to check the parameters of Q.""" type_check(Q, (float, int), "Q") - check_value_normalize_std(Q, [0, 1], "Q") + check_value(Q, [0, 1], "Q", True) def check_biquad_noise(noise): @@ -106,7 +108,7 @@ def check_band_biquad(method): def check_allpass_biquad(method): - """Wrapper method to check the parameters of CutMixBatch.""" + """Wrapper method to check the parameters of AllpassBiquad.""" @wraps(method) def new_method(self, *args, **kwargs): @@ -123,7 +125,7 @@ def check_allpass_biquad(method): def check_bandpass_biquad(method): """Wrapper method to check the parameters of BandpassBiquad.""" - @ wraps(method) + @wraps(method) def new_method(self, *args, **kwargs): [sample_rate, central_freq, Q, const_skirt_gain], _ = parse_user_args( method, *args, **kwargs) @@ -152,7 +154,7 @@ def check_bandreject_biquad(method): def check_bass_biquad(method): - """Wrapper method to check the parameters of CutMixBatch.""" + """Wrapper method to check the parameters of BassBiquad.""" @wraps(method) def new_method(self, *args, **kwargs): @@ -167,8 +169,30 @@ def check_bass_biquad(method): return new_method +def check_time_stretch(method): + """Wrapper method to check the parameters of TimeStretch.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + [hop_length, n_freq, fixed_rate], _ = parse_user_args(method, *args, **kwargs) + + if hop_length is not None: + type_check(hop_length, (int,), "hop_length") + check_pos_int64(hop_length, "hop_length") + + type_check(n_freq, (int,), "n_freq") + check_pos_int64(n_freq, "n_freq") + + if fixed_rate is not None: + type_check(fixed_rate, (int, float), "fixed_rate") + check_pos_float32(fixed_rate, "fixed_rate") + return method(self, *args, **kwargs) + + return new_method + + def check_masking(method): - """Wrapper method to check the parameters of time_masking and frequency_masking""" + """Wrapper method to check the parameters of time_masking and FrequencyMasking""" @wraps(method) def new_method(self, *args, **kwargs): @@ -176,33 +200,11 @@ def check_masking(method): method, *args, **kwargs) type_check(iid_masks, (bool,), "iid_masks") type_check(mask_param, (int,), "mask_param") - check_value(mask_param, (0, FLOAT_MAX_INTEGER), "mask_param") + check_non_negative_float32(mask_param, "mask_param") type_check(mask_start, (int,), "mask_start") - check_value(mask_start, (0, FLOAT_MAX_INTEGER), "mask_start") + check_non_negative_float32(mask_start, "mask_start") type_check(mask_value, (int, float), "mask_value") - check_value(mask_value, (0, DOUBLE_MAX_INTEGER), "mask_value") - - return method(self, *args, **kwargs) - - return new_method - - -def check_time_stretch(method): - """Wrapper method to check the parameters of time_stretch.""" - @wraps(method) - def new_method(self, *args, **kwargs): - [hop_length, n_freq, fixed_rate], _ = parse_user_args(method, *args, **kwargs) - # type check - type_check(hop_length, (int, type(None)), "hop_length") - type_check(n_freq, (int,), "n_freq") - type_check(fixed_rate, (int, float, type(None)), "fixed_rate") - - # value check - if hop_length is not None: - check_value(hop_length, (1, INT64_MAX), "hop_length") - check_value(n_freq, (1, INT64_MAX), "n_freq") - if fixed_rate is not None: - check_value_ratio(fixed_rate, (0, FLOAT_MAX_INTEGER), "fixed_rate") + check_non_negative_float64(mask_value, "mask_value") return method(self, *args, **kwargs) return new_method diff --git a/mindspore/dataset/core/validator_helpers.py b/mindspore/dataset/core/validator_helpers.py index 7ca1696e857..f30454314d0 100644 --- a/mindspore/dataset/core/validator_helpers.py +++ b/mindspore/dataset/core/validator_helpers.py @@ -92,20 +92,38 @@ def pad_arg_name(arg_name): return arg_name -def check_value(value, valid_range, arg_name=""): +def check_value(value, valid_range, arg_name="", left_open_interval=False, right_open_interval=False): """ - Validates a value is within a desired range [inclusive, inclusive]. + Validates a value is within a desired range with left and right interval open or close. - :param value: the value to be validated - :param valid_range: the desired range - :param arg_name: arg_name: arg_name: name of the variable to be validated + :param value: the value to be validated. + :param valid_range: the desired range. + :param arg_name: name of the variable to be validated. + :param left_open_interval: True for left interval open and False for close. + :param right_open_interval: True for right interval open and False for close. :return: Exception: when the validation fails, nothing otherwise. """ arg_name = pad_arg_name(arg_name) - if value < valid_range[0] or value > valid_range[1]: - raise ValueError( - "Input {0}is not within the required interval of [{1}, {2}].".format(arg_name, valid_range[0], - valid_range[1])) + if not left_open_interval and not right_open_interval: + if value < valid_range[0] or value > valid_range[1]: + raise ValueError( + "Input {0}is not within the required interval of [{1}, {2}].".format(arg_name, valid_range[0], + valid_range[1])) + elif left_open_interval and not right_open_interval: + if value <= valid_range[0] or value > valid_range[1]: + raise ValueError( + "Input {0}is not within the required interval of ({1}, {2}].".format(arg_name, valid_range[0], + valid_range[1])) + elif not left_open_interval and right_open_interval: + if value < valid_range[0] or value >= valid_range[1]: + raise ValueError( + "Input {0}is not within the required interval of [{1}, {2}).".format(arg_name, valid_range[0], + valid_range[1])) + else: + if value <= valid_range[0] or value >= valid_range[1]: + raise ValueError( + "Input {0}is not within the required interval of ({1}, {2}).".format(arg_name, valid_range[0], + valid_range[1])) def check_value_cutoff(value, valid_range, arg_name=""): @@ -117,11 +135,7 @@ def check_value_cutoff(value, valid_range, arg_name=""): :param arg_name: arg_name: arg_name: name of the variable to be validated :return: Exception: when the validation fails, nothing otherwise. """ - arg_name = pad_arg_name(arg_name) - if value < valid_range[0] or value >= valid_range[1]: - raise ValueError( - "Input {0}is not within the required interval of [{1}, {2}).".format(arg_name, valid_range[0], - valid_range[1])) + check_value(value, valid_range, arg_name, False, True) def check_value_ratio(value, valid_range, arg_name=""): @@ -133,11 +147,7 @@ def check_value_ratio(value, valid_range, arg_name=""): :param arg_name: arg_name: name of the variable to be validated :return: Exception: when the validation fails, nothing otherwise. """ - arg_name = pad_arg_name(arg_name) - if value <= valid_range[0] or value > valid_range[1]: - raise ValueError( - "Input {0}is not within the required interval of ({1}, {2}].".format(arg_name, valid_range[0], - valid_range[1])) + check_value(value, valid_range, arg_name, True, False) def check_value_normalize_std(value, valid_range, arg_name=""): @@ -149,11 +159,7 @@ def check_value_normalize_std(value, valid_range, arg_name=""): :param arg_name: arg_name: name of the variable to be validated :return: Exception: when the validation fails, nothing otherwise. """ - arg_name = pad_arg_name(arg_name) - if value <= valid_range[0] or value > valid_range[1]: - raise ValueError( - "Input {0}is not within the required interval of ({1}, {2}].".format(arg_name, valid_range[0], - valid_range[1])) + check_value(value, valid_range, arg_name, True, False) def check_range(values, valid_range, arg_name=""): @@ -185,10 +191,12 @@ def check_positive(value, arg_name=""): raise ValueError("Input {0}must be greater than 0.".format(arg_name)) -def check_not_zero(value, arg_name=""): +def check_int32_not_zero(value, arg_name=""): arg_name = pad_arg_name(arg_name) - if value == 0: - raise ValueError("Input {0}can not be 0.".format(arg_name)) + type_check(value, (int,), arg_name) + if value < INT32_MIN or value > INT32_MAX or value == 0: + raise ValueError( + "Input {0}is not within the required interval of [-2147483648, 0) and (0, 2147483647].".format(arg_name)) def check_odd(value, arg_name=""): @@ -211,6 +219,13 @@ def check_2tuple(value, arg_name=""): def check_int32(value, arg_name=""): + """ + Validates the value of a variable is within the range of int32. + + :param value: the value of the variable + :param arg_name: name of the variable to be validated + :return: Exception: when the validation fails, nothing otherwise. + """ type_check(value, (int,), arg_name) check_value(value, [INT32_MIN, INT32_MAX], arg_name) @@ -284,7 +299,7 @@ def check_pos_int64(value, arg_name=""): :return: Exception: when the validation fails, nothing otherwise. """ type_check(value, (int,), arg_name) - check_value(value, [UINT64_MIN, INT64_MAX]) + check_value(value, [POS_INT_MIN, INT64_MAX]) def check_float32(value, arg_name=""): @@ -317,7 +332,7 @@ def check_pos_float32(value, arg_name=""): :param arg_name: name of the variable to be validated :return: Exception: when the validation fails, nothing otherwise. """ - check_value(value, [UINT32_MIN, FLOAT_MAX_INTEGER], arg_name) + check_value(value, [UINT32_MIN, FLOAT_MAX_INTEGER], arg_name, True) def check_pos_float64(value, arg_name=""): @@ -328,7 +343,29 @@ def check_pos_float64(value, arg_name=""): :param arg_name: name of the variable to be validated :return: Exception: when the validation fails, nothing otherwise. """ - check_value(value, [UINT64_MIN, DOUBLE_MAX_INTEGER], arg_name) + check_value(value, [UINT64_MIN, DOUBLE_MAX_INTEGER], arg_name, True) + + +def check_non_negative_float32(value, arg_name=""): + """ + Validates the value of a variable is within the range of non negative float32. + + :param value: the value of the variable + :param arg_name: name of the variable to be validated + :return: Exception: when the validation fails, nothing otherwise. + """ + check_value(value, [UINT32_MIN, FLOAT_MAX_INTEGER], arg_name) + + +def check_non_negative_float64(value, arg_name=""): + """ + Validates the value of a variable is within the range of non negative float64. + + :param value: the value of the variable + :param arg_name: name of the variable to be validated + :return: Exception: when the validation fails, nothing otherwise. + """ + check_value(value, [UINT32_MIN, DOUBLE_MAX_INTEGER], arg_name) def check_valid_detype(type_): @@ -680,4 +717,3 @@ def check_c_tensor_op(param, param_name): def replace_none(value, default): """ replaces None with a default value.""" return value if value is not None else default - \ No newline at end of file diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index 204baa0757a..1e1fb2aab35 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -5773,62 +5773,74 @@ class FlickrDataset(MappableDataset): >>> # In FLICKR dataset, each dictionary has keys "image" and "annotation" About Flickr8k dataset: - | The Flickr8k dataset consists of 8092 colour images. There are 40460 annotations in the Flickr8k.token.txt, - each image has 5 annotations. - | You can unzip the dataset files into the following directory structure and read by MindSpore's API. - | . - | └── Flickr8k - | ├── Flickr8k_Dataset - | | ├── 1000268201_693b08cb0e.jpg - | | ├── 1001773457_577c3a7d70.jpg - | | ├── ... - | └── Flickr8k.token.txt + The Flickr8k dataset consists of 8092 colour images. There are 40460 annotations in the Flickr8k.token.txt, + each image has 5 annotations. - .. code-block:: + You can unzip the dataset files into the following directory structure and read by MindSpore's API. - @article{DBLP:journals/jair/HodoshYH13, - author = {Micah Hodosh and Peter Young and Julia Hockenmaier}, - title = {Framing Image Description as a Ranking Task: Data, Models and Evaluation Metrics}, - journal = {J. Artif. Intell. Res.}, - volume = {47}, - pages = {853--899}, - year = {2013}, - url = {https://doi.org/10.1613/jair.3994}, - doi = {10.1613/jair.3994}, - timestamp = {Mon, 21 Jan 2019 15:01:17 +0100}, - biburl = {https://dblp.org/rec/journals/jair/HodoshYH13.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} - } + .. code-block:: + + . + └── Flickr8k + ├── Flickr8k_Dataset + │ ├── 1000268201_693b08cb0e.jpg + │ ├── 1001773457_577c3a7d70.jpg + │ ├── ... + └── Flickr8k.token.txt + + Citation: + + .. code-block:: + + @article{DBLP:journals/jair/HodoshYH13, + author = {Micah Hodosh and Peter Young and Julia Hockenmaier}, + title = {Framing Image Description as a Ranking Task: Data, Models and Evaluation Metrics}, + journal = {J. Artif. Intell. Res.}, + volume = {47}, + pages = {853--899}, + year = {2013}, + url = {https://doi.org/10.1613/jair.3994}, + doi = {10.1613/jair.3994}, + timestamp = {Mon, 21 Jan 2019 15:01:17 +0100}, + biburl = {https://dblp.org/rec/journals/jair/HodoshYH13.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} + } About Flickr30k dataset: - | The Flickr30k dataset consists of 31783 colour images. There are 158915 annotations in - the results_20130124.token, each image has 5 annotations. - | You can unzip the dataset files into the following directory structure and read by MindSpore's API. - | . - | └── Flickr30k - | ├── flickr30k-images - | | ├── 1000092795.jpg - | | ├── 10002456.jpg - | | ├── ... - | └── results_20130124.token + The Flickr30k dataset consists of 31783 colour images. There are 158915 annotations in + the results_20130124.token, each image has 5 annotations. - .. code-block:: + You can unzip the dataset files into the following directory structure and read by MindSpore's API. - @article{DBLP:journals/tacl/YoungLHH14, - author = {Peter Young and Alice Lai and Micah Hodosh and Julia Hockenmaier}, - title = {From image descriptions to visual denotations: New similarity metrics - for semantic inference over event descriptions}, - journal = {Trans. Assoc. Comput. Linguistics}, - volume = {2}, - pages = {67--78}, - year = {2014}, - url = {https://tacl2013.cs.columbia.edu/ojs/index.php/tacl/article/view/229}, - timestamp = {Wed, 17 Feb 2021 21:55:25 +0100}, - biburl = {https://dblp.org/rec/journals/tacl/YoungLHH14.bib}, - bibsource = {dblp computer science bibliography, https://dblp.org} - } + Citation: + + .. code-block:: + + . + └── Flickr30k + ├── flickr30k-images + │ ├── 1000092795.jpg + │ ├── 10002456.jpg + │ ├── ... + └── results_20130124.token + + .. code-block:: + + @article{DBLP:journals/tacl/YoungLHH14, + author = {Peter Young and Alice Lai and Micah Hodosh and Julia Hockenmaier}, + title = {From image descriptions to visual denotations: New similarity metrics + for semantic inference over event descriptions}, + journal = {Trans. Assoc. Comput. Linguistics}, + volume = {2}, + pages = {67--78}, + year = {2014}, + url = {https://tacl2013.cs.columbia.edu/ojs/index.php/tacl/article/view/229}, + timestamp = {Wed, 17 Feb 2021 21:55:25 +0100}, + biburl = {https://dblp.org/rec/journals/tacl/YoungLHH14.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} + } """ @check_flickr_dataset @@ -5928,39 +5940,43 @@ class SBDataset(GeneratorDataset): >>> >>> # In Semantic Boundaries Dataset, each dictionary has keys "image" and "task" - About Semantic Boundaries Dataset. - | The Semantic Boundaries Dataset consists of 11355 colour images. There are 8498 images' name in the train.txt, - 2857 images' name in the val.txt and 5623 images' name in the train_noval.txt. The category cls/ - contains the Segmentation and Boundaries results of category-level, the category inst/ catains the - Segmentation and Boundaries results of instance-level. + About Semantic Boundaries Dataset: - | You can unzip the dataset files into the following structure and read by MindSpore's API, - | . - | └── benchmark_RELEASE - | └── dataset - | ├── img - | | ├── 2008_000002.jpg - | | ├── 2008_000003.jpg - | | ├── ... - | ├── cls - | | ├── 2008_000002.mat - | | ├── 2008_000003.mat - | | ├── ... - | ├── inst - | | ├── 2008_000002.mat - | | ├── 2008_000003.mat - | | ├── ... - | ├── train.txt - | └── val.txt + The Semantic Boundaries Dataset consists of 11355 colour images. There are 8498 images' name in the train.txt, + 2857 images' name in the val.txt and 5623 images' name in the train_noval.txt. The category cls/ + contains the Segmentation and Boundaries results of category-level, the category inst/ catains the + Segmentation and Boundaries results of instance-level. - .. code-block:: + You can unzip the dataset files into the following structure and read by MindSpore's API: - @InProceedings{BharathICCV2011, - author = "Bharath Hariharan and Pablo Arbelaez and Lubomir Bourdev and - Subhransu Maji and Jitendra Malik", - title = "Semantic Contours from Inverse Detectors", - booktitle = "International Conference on Computer Vision (ICCV)", - year = "2011", + .. code-block:: + + . + └── benchmark_RELEASE + ├── dataset + ├── img + │ ├── 2008_000002.jpg + │ ├── 2008_000003.jpg + │ ├── ... + ├── cls + │ ├── 2008_000002.mat + │ ├── 2008_000003.mat + │ ├── ... + ├── inst + │ ├── 2008_000002.mat + │ ├── 2008_000003.mat + │ ├── ... + ├── train.txt + └── val.txt + + .. code-block:: + + @InProceedings{BharathICCV2011, + author = "Bharath Hariharan and Pablo Arbelaez and Lubomir Bourdev and + Subhransu Maji and Jitendra Malik", + title = "Semantic Contours from Inverse Detectors", + booktitle = "International Conference on Computer Vision (ICCV)", + year = "2011", """ @check_sb_dataset @@ -5972,7 +5988,7 @@ class SBDataset(GeneratorDataset): num_shards=num_shards, shard_id=shard_id) -class _SBDataset(): +class _SBDataset: """ Dealing with the data file with .mat extension, and return one row in tuple (image, task) each time. """ diff --git a/mindspore/dataset/vision/c_transforms.py b/mindspore/dataset/vision/c_transforms.py index 1f7fb720e82..103aafc8a64 100644 --- a/mindspore/dataset/vision/c_transforms.py +++ b/mindspore/dataset/vision/c_transforms.py @@ -109,7 +109,7 @@ def parse_padding(padding): class AdjustGamma(ImageTensorOperation): r""" - Apply gamma correction on input image. Input image is expected to be in [..., H, W, C] or [H, W, C] format. + Apply gamma correction on input image. Input image is expected to be in [..., H, W, C] or [H, W] format. .. math:: I_{\text{out}} = 255 \times \text{gain} \times \left(\frac{I_{\text{in}}}{255}\right)^{\gamma} @@ -1511,6 +1511,7 @@ class RgbToBgr(ImageTensorOperation): Examples: >>> from mindspore.dataset.vision import Inter + >>> >>> decode_op = c_vision.Decode() >>> rgb2bgr_op = c_vision.RgbToBgr() >>> transforms_list = [decode_op, rgb2bgr_op] diff --git a/mindspore/dataset/vision/utils.py b/mindspore/dataset/vision/utils.py index 2843e519dea..75ed8549707 100644 --- a/mindspore/dataset/vision/utils.py +++ b/mindspore/dataset/vision/utils.py @@ -43,6 +43,7 @@ class ImageBatchFormat(IntEnum): NHWC = 0 NCHW = 1 + class SliceMode(IntEnum): PAD = 0 DROP = 1 diff --git a/mindspore/dataset/vision/validators.py b/mindspore/dataset/vision/validators.py index d8d7b84385b..546db4a4362 100644 --- a/mindspore/dataset/vision/validators.py +++ b/mindspore/dataset/vision/validators.py @@ -22,7 +22,7 @@ from mindspore._c_dataengine import TensorOp, TensorOperation from mindspore.dataset.core.validator_helpers import check_value, check_uint8, FLOAT_MIN_INTEGER, FLOAT_MAX_INTEGER, \ check_pos_float32, check_float32, check_2tuple, check_range, check_positive, INT32_MAX, INT32_MIN, \ parse_user_args, type_check, type_check_list, check_c_tensor_op, UINT8_MAX, check_value_normalize_std, \ - check_value_cutoff, check_value_ratio, check_odd + check_value_cutoff, check_value_ratio, check_odd, check_non_negative_float32 from .utils import Inter, Border, ImageBatchFormat, SliceMode @@ -143,7 +143,7 @@ def check_degrees(degrees): """Check if the degrees is legal.""" type_check(degrees, (int, float, list, tuple), "degrees") if isinstance(degrees, (int, float)): - check_pos_float32(degrees, "degrees") + check_non_negative_float32(degrees, "degrees") elif isinstance(degrees, (list, tuple)): if len(degrees) == 2: type_check_list(degrees, (int, float), "degrees") diff --git a/tests/ut/cpp/dataset/CMakeLists.txt b/tests/ut/cpp/dataset/CMakeLists.txt index 1f33a1b4e3f..5211277faa9 100644 --- a/tests/ut/cpp/dataset/CMakeLists.txt +++ b/tests/ut/cpp/dataset/CMakeLists.txt @@ -16,7 +16,6 @@ SET(DE_UT_SRCS c_api_audio_r_to_z_test.cc c_api_cache_test.cc c_api_dataset_album_test.cc - c_api_audio_a_to_q_test.cc c_api_dataset_cifar_test.cc c_api_dataset_clue_test.cc c_api_dataset_coco_test.cc diff --git a/tests/ut/cpp/dataset/c_api_audio_a_to_q_test.cc b/tests/ut/cpp/dataset/c_api_audio_a_to_q_test.cc index 2b9d7e925eb..8755641f478 100644 --- a/tests/ut/cpp/dataset/c_api_audio_a_to_q_test.cc +++ b/tests/ut/cpp/dataset/c_api_audio_a_to_q_test.cc @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #include "common/common.h" #include "include/api/types.h" #include "utils/log_adapter.h" #include "minddata/dataset/include/dataset/audio.h" #include "minddata/dataset/include/dataset/datasets.h" -#include "minddata/dataset/include/dataset/execute.h" #include "minddata/dataset/include/dataset/transforms.h" using namespace mindspore::dataset; @@ -33,7 +33,7 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { }; TEST_F(MindDataTestPipeline, TestAmplitudeToDBPipeline) { - MS_LOG(INFO) << "Basic Function Test"; + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAmplitudeToDBPipeline."; // Original waveform std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200})); @@ -71,7 +71,7 @@ TEST_F(MindDataTestPipeline, TestAmplitudeToDBPipeline) { } TEST_F(MindDataTestPipeline, TestAmplitudeToDBWrongArgs) { - MS_LOG(INFO) << "Basic Function Test"; + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAmplitudeToDBWrongArgs."; // Original waveform std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200})); @@ -91,8 +91,8 @@ TEST_F(MindDataTestPipeline, TestAmplitudeToDBWrongArgs) { EXPECT_EQ(iter, nullptr); } -TEST_F(MindDataTestPipeline, Level0_TestBandBiquad001) { - MS_LOG(INFO) << "Basic Function Test"; +TEST_F(MindDataTestPipeline, TestBandBiquadBasic) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandBiquadBasic."; // Original waveform std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200})); @@ -130,8 +130,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandBiquad001) { iter->Stop(); } -TEST_F(MindDataTestPipeline, Level0_TestBandBiquad002) { - MS_LOG(INFO) << "Wrong Arg."; +TEST_F(MindDataTestPipeline, TestBandBiquadParamCheck) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandBiquadParamCheck."; std::shared_ptr schema = Schema(); // Original waveform ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 2})); @@ -159,8 +159,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandBiquad002) { EXPECT_EQ(iter02, nullptr); } -TEST_F(MindDataTestPipeline, Level0_TestAllpassBiquad001) { - MS_LOG(INFO) << "Basic Function Test"; +TEST_F(MindDataTestPipeline, TestAllpassBiquadBasic) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAllpassBiquadBasic."; // Original waveform std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200})); @@ -198,8 +198,8 @@ TEST_F(MindDataTestPipeline, Level0_TestAllpassBiquad001) { iter->Stop(); } -TEST_F(MindDataTestPipeline, Level0_TestAllpassBiquad002) { - MS_LOG(INFO) << "Wrong Arg."; +TEST_F(MindDataTestPipeline, TestAllpassBiquadParamCheck) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAllpassBiquadParamCheck."; std::shared_ptr schema = Schema(); // Original waveform ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 2})); @@ -227,8 +227,8 @@ TEST_F(MindDataTestPipeline, Level0_TestAllpassBiquad002) { EXPECT_EQ(iter02, nullptr); } -TEST_F(MindDataTestPipeline, Level0_TestBandpassBiquad001) { - MS_LOG(INFO) << "Basic Function Test"; +TEST_F(MindDataTestPipeline, TestBandpassBiquadBasic) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandpassBiquadBasic."; // Original waveform std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200})); @@ -266,8 +266,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandpassBiquad001) { iter->Stop(); } -TEST_F(MindDataTestPipeline, Level0_TestBandpassBiquad002) { - MS_LOG(INFO) << "Wrong Arg."; +TEST_F(MindDataTestPipeline, TestBandpassBiquadParamCheck) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandpassBiquadParamCheck."; std::shared_ptr schema = Schema(); // Original waveform ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 2})); @@ -295,8 +295,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandpassBiquad002) { EXPECT_EQ(iter02, nullptr); } -TEST_F(MindDataTestPipeline, Level0_TestBandrejectBiquad001) { - MS_LOG(INFO) << "Basic Function Test"; +TEST_F(MindDataTestPipeline, TestBandrejectBiquadBasic) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandrejectBiquadBasic."; // Original waveform std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200})); @@ -334,8 +334,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandrejectBiquad001) { iter->Stop(); } -TEST_F(MindDataTestPipeline, Level0_TestBandrejectBiquad002) { - MS_LOG(INFO) << "Wrong Arg."; +TEST_F(MindDataTestPipeline, TestBandrejectBiquadParamCheck) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandrejectBiquadParamCheck."; std::shared_ptr schema = Schema(); // Original waveform ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 2})); @@ -363,8 +363,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandrejectBiquad002) { EXPECT_EQ(iter02, nullptr); } -TEST_F(MindDataTestPipeline, Level0_TestBassBiquad001) { - MS_LOG(INFO) << "Basic Function Test"; +TEST_F(MindDataTestPipeline, TestBassBiquadBasic) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBassBiquadBasic."; // Original waveform std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200})); @@ -402,8 +402,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBassBiquad001) { iter->Stop(); } -TEST_F(MindDataTestPipeline, Level0_TestBassBiquad002) { - MS_LOG(INFO) << "Wrong Arg."; +TEST_F(MindDataTestPipeline, TestBassBiquadParamCheck) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBassBiquadParamCheck."; std::shared_ptr schema = Schema(); // Original waveform ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 2})); @@ -432,7 +432,7 @@ TEST_F(MindDataTestPipeline, Level0_TestBassBiquad002) { } TEST_F(MindDataTestPipeline, TestAnglePipeline) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAnglePipeline"; + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAnglePipeline."; std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("complex", mindspore::DataType::kNumberTypeFloat32, {2, 2})); @@ -470,7 +470,7 @@ TEST_F(MindDataTestPipeline, TestAnglePipeline) { } TEST_F(MindDataTestPipeline, TestAnglePipelineError) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAnglePipelineError"; + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAnglePipelineError."; std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("complex", mindspore::DataType::kNumberTypeFloat32, {3, 2, 1})); @@ -491,7 +491,7 @@ TEST_F(MindDataTestPipeline, TestAnglePipelineError) { } TEST_F(MindDataTestPipeline, TestFrequencyMaskingPipeline) { - MS_LOG(INFO) << "Doing TestFrequencyMasking Pipeline."; + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFrequencyMaskingPipeline."; // Original waveform std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {200, 200})); @@ -506,7 +506,6 @@ TEST_F(MindDataTestPipeline, TestFrequencyMaskingPipeline) { ds = ds->Map({frequencymasking}); EXPECT_NE(ds, nullptr); - // Filtered waveform by bandbiquad std::shared_ptr iter = ds->CreateIterator(); EXPECT_NE(ds, nullptr); @@ -530,7 +529,7 @@ TEST_F(MindDataTestPipeline, TestFrequencyMaskingPipeline) { } TEST_F(MindDataTestPipeline, TestFrequencyMaskingWrongArgs) { - MS_LOG(INFO) << "Doing TestFrequencyMasking with wrong args."; + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFrequencyMaskingWrongArgs."; // Original waveform std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {20, 20})); diff --git a/tests/ut/cpp/dataset/c_api_audio_r_to_z_test.cc b/tests/ut/cpp/dataset/c_api_audio_r_to_z_test.cc index 808e20d973b..cc833a53654 100644 --- a/tests/ut/cpp/dataset/c_api_audio_r_to_z_test.cc +++ b/tests/ut/cpp/dataset/c_api_audio_r_to_z_test.cc @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + #include "common/common.h" #include "minddata/dataset/core/tensor.h" #include "minddata/dataset/include/dataset/datasets.h" @@ -28,7 +29,7 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { }; TEST_F(MindDataTestPipeline, TestTimeMaskingPipeline) { - MS_LOG(INFO) << "Doing TestTimeMasking Pipeline."; + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTimeMaskingPipeline."; // Original waveform std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200})); @@ -67,7 +68,7 @@ TEST_F(MindDataTestPipeline, TestTimeMaskingPipeline) { } TEST_F(MindDataTestPipeline, TestTimeMaskingWrongArgs) { - MS_LOG(INFO) << "Doing TestTimeMasking with wrong args."; + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTimeMaskingWrongArgs."; // Original waveform std::shared_ptr schema = Schema(); ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 20})); @@ -89,7 +90,7 @@ TEST_F(MindDataTestPipeline, TestTimeMaskingWrongArgs) { } TEST_F(MindDataTestPipeline, TestTimeStretchPipeline) { - MS_LOG(INFO) << "Doing test TimeStretchOp with custom param value. Pipeline."; + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTimeStretchPipeline."; // op param int freq = 1025; int hop_length = 512; @@ -115,7 +116,7 @@ TEST_F(MindDataTestPipeline, TestTimeStretchPipeline) { std::unordered_map row; ASSERT_OK(iter->GetNextRow(&row)); - std::vector expected = {2, freq, int(std::ceil(400 / rate)), 2}; + std::vector expected = {2, freq, static_cast(std::ceil(400 / rate)), 2}; int i = 0; while (row.size() != 0) { @@ -131,7 +132,7 @@ TEST_F(MindDataTestPipeline, TestTimeStretchPipeline) { } TEST_F(MindDataTestPipeline, TestTimeStretchPipelineWrongArgs) { - MS_LOG(INFO) << "Doing test TimeStretchOp with wrong param value. Pipeline."; + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTimeStretchPipelineWrongArgs."; // op param int freq = 1025; int hop_length = 512; diff --git a/tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc b/tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc index 0647ae7bdd0..2d2678eb46f 100644 --- a/tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc @@ -27,9 +27,8 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { // Tests for vision C++ API A to Q TensorTransform Operations (in alphabetical order) -TEST_F(MindDataTestPipeline, TestAdjustGammaSuccess1) { - // pipeline 3-channel - MS_LOG(INFO) << "Pipeline Test."; +TEST_F(MindDataTestPipeline, TestAdjustGamma3Channel) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAdjustGamma3Channel."; std::string MindDataPath = "data/dataset"; std::string folder_path = MindDataPath + "/testImageNetData/train/"; std::shared_ptr ds1 = ImageFolder(folder_path, true, std::make_shared(false, 2)); @@ -65,9 +64,8 @@ TEST_F(MindDataTestPipeline, TestAdjustGammaSuccess1) { iter2->Stop(); } -TEST_F(MindDataTestPipeline, TestAdjustGammaSuccess2) { - // pipeline 1-channel - MS_LOG(INFO) << "Pipeline Test."; +TEST_F(MindDataTestPipeline, TestAdjustGamma1Channel) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAdjustGamma1Channel."; std::string MindDataPath = "data/dataset"; std::string folder_path = MindDataPath + "/testImageNetData/train/"; std::shared_ptr ds1 = ImageFolder(folder_path, true, std::make_shared(false, 2)); @@ -105,8 +103,7 @@ TEST_F(MindDataTestPipeline, TestAdjustGammaSuccess2) { } TEST_F(MindDataTestPipeline, TestAdjustGammaParamCheck) { - // pipeline 3-channel - MS_LOG(INFO) << "Pipeline Test."; + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAdjustGammaParamCheck."; std::string MindDataPath = "data/dataset"; std::string folder_path = MindDataPath + "/testImageNetData/train/"; std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 2)); diff --git a/tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc b/tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc index ebffd3807ee..33630b22e1b 100644 --- a/tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc @@ -312,3 +312,41 @@ TEST_F(MindDataTestPipeline, TestRotatePass) { // Manually terminate the pipeline iter->Stop(); } + +TEST_F(MindDataTestPipeline, TestRGB2BGR) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRGB2BGR."; + // create two imagenet dataset + std::string MindDataPath = "data/dataset"; + std::string folder_path = MindDataPath + "/testImageNetData/train/"; + std::shared_ptr ds1 = ImageFolder(folder_path, true, std::make_shared(false, 2)); + EXPECT_NE(ds1, nullptr); + std::shared_ptr ds2 = ImageFolder(folder_path, true, std::make_shared(false, 2)); + EXPECT_NE(ds2, nullptr); + + auto rgb2bgr_op = vision::RGB2BGR(); + + ds1 = ds1->Map({rgb2bgr_op}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter1 = ds1->CreateIterator(); + EXPECT_NE(iter1, nullptr); + std::unordered_map row1; + iter1->GetNextRow(&row1); + + std::shared_ptr iter2 = ds2->CreateIterator(); + EXPECT_NE(iter2, nullptr); + std::unordered_map row2; + iter2->GetNextRow(&row2); + + uint64_t i = 0; + while (row1.size() != 0) { + i++; + auto image =row1["image"]; + iter1->GetNextRow(&row1); + iter2->GetNextRow(&row2); + } + EXPECT_EQ(i, 2); + + iter1->Stop(); + iter2->Stop(); +} diff --git a/tests/ut/cpp/dataset/execute_test.cc b/tests/ut/cpp/dataset/execute_test.cc index 499ec1a11c8..93339693dd3 100644 --- a/tests/ut/cpp/dataset/execute_test.cc +++ b/tests/ut/cpp/dataset/execute_test.cc @@ -35,7 +35,7 @@ class MindDataTestExecute : public UT::DatasetOpTesting { }; TEST_F(MindDataTestExecute, TestAllpassBiquadWithEager) { - MS_LOG(INFO) << "Basic Function Test With Eager."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestAllpassBiquadWithEager."; // Original waveform std::vector labels = { 2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02, @@ -54,7 +54,7 @@ TEST_F(MindDataTestExecute, TestAllpassBiquadWithEager) { } TEST_F(MindDataTestExecute, TestAllpassBiquadWithWrongArg) { - MS_LOG(INFO) << "Wrong Arg."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestAllpassBiquadWithWrongArg."; std::vector labels = { 2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02, 1.138305664062500000e-02, 1.156616210937500000e-02, 1.394653320312500000e-02, 1.550292968750000000e-02, @@ -72,9 +72,8 @@ TEST_F(MindDataTestExecute, TestAllpassBiquadWithWrongArg) { EXPECT_FALSE(s01.IsOk()); } -TEST_F(MindDataTestExecute, TestAdjustGammaEager1) { - // 3-channel eager - MS_LOG(INFO) << "3-channel image test"; +TEST_F(MindDataTestExecute, TestAdjustGammaEager3Channel) { + MS_LOG(INFO) << "Doing MindDataTestExecute-TestAdjustGammaEager3Channel."; // Read images auto image = ReadFileToTensor("data/dataset/apple.jpg"); @@ -87,9 +86,8 @@ TEST_F(MindDataTestExecute, TestAdjustGammaEager1) { EXPECT_EQ(rc, Status::OK()); } -TEST_F(MindDataTestExecute, TestAdjustGammaEager2) { - // 1-channel eager - MS_LOG(INFO) << "1-channel image test"; +TEST_F(MindDataTestExecute, TestAdjustGammaEager1Channel) { + MS_LOG(INFO) << "Doing MindDataTestExecute-TestAdjustGammaEager1Channel."; auto m1 = ReadFileToTensor("data/dataset/apple.jpg"); // Transform params auto decode = vision::Decode(); @@ -102,7 +100,7 @@ TEST_F(MindDataTestExecute, TestAdjustGammaEager2) { } TEST_F(MindDataTestExecute, TestAmplitudeToDB) { - MS_LOG(INFO) << "Basic Function Test With Eager."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestAmplitudeToDB."; // Original waveform std::vector labels = { 2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02, @@ -122,7 +120,7 @@ TEST_F(MindDataTestExecute, TestAmplitudeToDB) { } TEST_F(MindDataTestExecute, TestAmplitudeToDBWrongArgs) { - MS_LOG(INFO) << "Wrong Arg."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestAmplitudeToDBWrongArgs."; // Original waveform std::vector labels = { 2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02, @@ -142,7 +140,7 @@ TEST_F(MindDataTestExecute, TestAmplitudeToDBWrongArgs) { } TEST_F(MindDataTestExecute, TestAmplitudeToDBWrongInput) { - MS_LOG(INFO) << "Wrong Input."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestAmplitudeToDBWrongInput."; // Original waveform std::vector labels = { 2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02, @@ -161,7 +159,7 @@ TEST_F(MindDataTestExecute, TestAmplitudeToDBWrongInput) { } TEST_F(MindDataTestExecute, TestComposeTransforms) { - MS_LOG(INFO) << "Doing TestComposeTransforms."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestComposeTransforms."; // Read images auto image = ReadFileToTensor("data/dataset/apple.jpg"); @@ -198,7 +196,7 @@ TEST_F(MindDataTestExecute, TestCrop) { } TEST_F(MindDataTestExecute, TestFrequencyMasking) { - MS_LOG(INFO) << "Doing TestFrequencyMasking."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestFrequencyMasking."; std::shared_ptr input_tensor_; TensorShape s = TensorShape({6, 2}); ASSERT_OK(Tensor::CreateFromVector( @@ -211,7 +209,7 @@ TEST_F(MindDataTestExecute, TestFrequencyMasking) { } TEST_F(MindDataTestExecute, TestTimeMasking) { - MS_LOG(INFO) << "Doing TestTimeMasking."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestTimeMasking."; std::shared_ptr input_tensor_; TensorShape s = TensorShape({2, 6}); ASSERT_OK(Tensor::CreateFromVector( @@ -224,7 +222,7 @@ TEST_F(MindDataTestExecute, TestTimeMasking) { } TEST_F(MindDataTestExecute, TestTimeStretchEager) { - MS_LOG(INFO) << "Doing test TimeStretchOp with custom param value. Eager."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestTimeStretchEager."; std::shared_ptr input_tensor_; // op param int freq = 4; @@ -248,8 +246,8 @@ TEST_F(MindDataTestExecute, TestTimeStretchEager) { EXPECT_TRUE(status.IsOk()); } -TEST_F(MindDataTestExecute, TestTimeStretchParamCheck1) { - MS_LOG(INFO) << "Doing MindDataTestTimeStretch-TestTimeStretchParamCheck with invalid parameters."; +TEST_F(MindDataTestExecute, TestTimeStretchParamCheck) { + MS_LOG(INFO) << "Doing MindDataTestTimeStretch-TestTimeStretchParamCheck."; // Create an input std::shared_ptr input_tensor_; std::shared_ptr output_tensor; @@ -259,26 +257,15 @@ TEST_F(MindDataTestExecute, TestTimeStretchParamCheck1) { 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f}), s, &input_tensor_)); auto input_ms = mindspore::MSTensor(std::make_shared(input_tensor_)); - std::shared_ptr timestretch = std::make_shared(4, 512, -2); - mindspore::dataset::Execute Transform({timestretch}); - Status status = Transform(input_ms, &input_ms); - EXPECT_FALSE(status.IsOk()); -} -TEST_F(MindDataTestExecute, TestTimeStretchParamCheck2) { - MS_LOG(INFO) << "Doing MindDataTestTimeStretch-TestTimeStretchParamCheck with invalid parameters."; - // Create an input - std::shared_ptr input_tensor_; - std::shared_ptr output_tensor; - TensorShape s = TensorShape({1, 4, 3, 2}); - ASSERT_OK(Tensor::CreateFromVector( - std::vector({1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f, - 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f}), - s, &input_tensor_)); - auto input_ms = mindspore::MSTensor(std::make_shared(input_tensor_)); - std::shared_ptr timestretch = std::make_shared(4, -512, 2); - mindspore::dataset::Execute Transform({timestretch}); - Status status = Transform(input_ms, &input_ms); + std::shared_ptr time_stretch1 = std::make_shared(4, 512, -2); + mindspore::dataset::Execute Transform1({time_stretch1}); + Status status = Transform1(input_ms, &input_ms); + EXPECT_FALSE(status.IsOk()); + + std::shared_ptr time_stretch2 = std::make_shared(4, -512, 2); + mindspore::dataset::Execute Transform2({time_stretch2}); + status = Transform2(input_ms, &input_ms); EXPECT_FALSE(status.IsOk()); } @@ -511,7 +498,7 @@ TEST_F(MindDataTestExecute, TestResizeWithBBox) { } TEST_F(MindDataTestExecute, TestBandBiquadWithEager) { - MS_LOG(INFO) << "Basic Function Test With Eager."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandBiquadWithEager."; // Original waveform std::vector labels = { 2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02, @@ -530,7 +517,7 @@ TEST_F(MindDataTestExecute, TestBandBiquadWithEager) { } TEST_F(MindDataTestExecute, TestBandBiquadWithWrongArg) { - MS_LOG(INFO) << "Wrong Arg."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandBiquadWithWrongArg."; std::vector labels = { 2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02, 1.138305664062500000e-02, 1.156616210937500000e-02, 1.394653320312500000e-02, 1.550292968750000000e-02, @@ -549,7 +536,7 @@ TEST_F(MindDataTestExecute, TestBandBiquadWithWrongArg) { } TEST_F(MindDataTestExecute, TestBandpassBiquadWithEager) { - MS_LOG(INFO) << "Basic Function Test With Eager."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandpassBiquadWithEager."; // Original waveform std::vector labels = { 2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02, @@ -568,7 +555,7 @@ TEST_F(MindDataTestExecute, TestBandpassBiquadWithEager) { } TEST_F(MindDataTestExecute, TestBandpassBiquadWithWrongArg) { - MS_LOG(INFO) << "Wrong Arg."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandpassBiquadWithWrongArg."; std::vector labels = { 2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02, 1.138305664062500000e-02, 1.156616210937500000e-02, 1.394653320312500000e-02, 1.550292968750000000e-02, @@ -587,7 +574,7 @@ TEST_F(MindDataTestExecute, TestBandpassBiquadWithWrongArg) { } TEST_F(MindDataTestExecute, TestBandrejectBiquadWithEager) { - MS_LOG(INFO) << "Basic Function Test With Eager."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandrejectBiquadWithEager."; // Original waveform std::vector labels = { 2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02, @@ -606,7 +593,7 @@ TEST_F(MindDataTestExecute, TestBandrejectBiquadWithEager) { } TEST_F(MindDataTestExecute, TestBandrejectBiquadWithWrongArg) { - MS_LOG(INFO) << "Wrong Arg."; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandrejectBiquadWithWrongArg."; std::vector labels = { 2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02, 1.138305664062500000e-02, 1.156616210937500000e-02, 1.394653320312500000e-02, 1.550292968750000000e-02, @@ -625,7 +612,7 @@ TEST_F(MindDataTestExecute, TestBandrejectBiquadWithWrongArg) { } TEST_F(MindDataTestExecute, TestAngleEager) { - MS_LOG(INFO) << "Doing MindDataTestExecute-TestAngleEager"; + MS_LOG(INFO) << "Doing MindDataTestExecute-TestAngleEager."; std::vector origin = {1.143, 1.3123, 2.632, 2.554, -1.213, 1.3, 0.456, 3.563}; TensorShape input_shape({4, 2}); std::shared_ptr de_tensor; @@ -638,3 +625,19 @@ TEST_F(MindDataTestExecute, TestAngleEager) { ASSERT_TRUE(s.IsOk()); } + +TEST_F(MindDataTestExecute, TestRGB2BGREager) { + MS_LOG(INFO) << "Doing MindDataTestExecute-TestRGB2BGREager."; + + // Read images + auto image = ReadFileToTensor("data/dataset/apple.jpg"); + + // Transform params + auto decode = vision::Decode(); + auto rgb2bgr_op = vision::RGB2BGR(); + + auto transform = Execute({decode, rgb2bgr_op}); + Status rc = transform(image, &image); + + EXPECT_EQ(rc, Status::OK()); +} diff --git a/tests/ut/cpp/dataset/rgb_to_bgr_test_op.cc b/tests/ut/cpp/dataset/rgb_to_bgr_test_op.cc deleted file mode 100644 index 9c93ea788b3..00000000000 --- a/tests/ut/cpp/dataset/rgb_to_bgr_test_op.cc +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright 2021 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "common/common.h" -#include "common/cvop_common.h" -#include "include/dataset/datasets.h" -#include "include/dataset/transforms.h" -#include "include/dataset/vision.h" -#include "include/dataset/execute.h" -#include "minddata/dataset/kernels/image/image_utils.h" -#include "minddata/dataset/kernels/image/rgb_to_bgr_op.h" -#include "minddata/dataset/core/cv_tensor.h" -#include "utils/log_adapter.h" - -using namespace std; -using namespace mindspore::dataset; -using mindspore::dataset::CVTensor; -using mindspore::dataset::BorderType; -using mindspore::dataset::Tensor; -using mindspore::LogStream; -using mindspore::ExceptionType::NoExceptionType; -using mindspore::MsLogLevel::INFO; - - -class MindDataTestRgbToBgrOp : public UT::DatasetOpTesting { - protected: -}; - - -TEST_F(MindDataTestRgbToBgrOp, TestOp1) { - // Eager - MS_LOG(INFO) << "Doing MindDataTestGaussianBlur-TestGaussianBlurEager."; - - // Read images - auto image = ReadFileToTensor("data/dataset/apple.jpg"); - - // Transform params - auto decode = vision::Decode(); - auto rgb2bgr_op = vision::RGB2BGR(); - - auto transform = Execute({decode, rgb2bgr_op}); - Status rc = transform(image, &image); - - EXPECT_EQ(rc, Status::OK()); -} - - -TEST_F(MindDataTestRgbToBgrOp, TestOp2) { - // pipeline - MS_LOG(INFO) << "Basic Function Test."; - // create two imagenet dataset - std::string MindDataPath = "data/dataset"; - std::string folder_path = MindDataPath + "/testImageNetData/train/"; - std::shared_ptr ds1 = ImageFolder(folder_path, true, std::make_shared(false, 2)); - EXPECT_NE(ds1, nullptr); - std::shared_ptr ds2 = ImageFolder(folder_path, true, std::make_shared(false, 2)); - EXPECT_NE(ds2, nullptr); - - auto rgb2bgr_op = vision::RGB2BGR(); - - ds1 = ds1->Map({rgb2bgr_op}); - EXPECT_NE(ds1, nullptr); - - std::shared_ptr iter1 = ds1->CreateIterator(); - EXPECT_NE(iter1, nullptr); - std::unordered_map row1; - iter1->GetNextRow(&row1); - - std::shared_ptr iter2 = ds2->CreateIterator(); - EXPECT_NE(iter2, nullptr); - std::unordered_map row2; - iter2->GetNextRow(&row2); - - uint64_t i = 0; - while (row1.size() != 0) { - i++; - auto image =row1["image"]; - iter1->GetNextRow(&row1); - iter2->GetNextRow(&row2); - } - EXPECT_EQ(i, 2); - - iter1->Stop(); - iter2->Stop(); -} diff --git a/tests/ut/python/dataset/test_adjustgamma.py b/tests/ut/python/dataset/test_adjustgamma.py index 61e91fdc5f5..32363f76b84 100644 --- a/tests/ut/python/dataset/test_adjustgamma.py +++ b/tests/ut/python/dataset/test_adjustgamma.py @@ -31,8 +31,6 @@ MNIST_DATA_DIR = "../data/dataset/testMnistData" DATA_DIR_2 = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" -GENERATE_GOLDEN = False - def generate_numpy_random_rgb(shape): """ @@ -90,26 +88,20 @@ def test_adjust_gamma_invalid_gamma_param_c(): logger.info("Test AdjustGamma C Op with invalid ignore parameter") try: data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - data_set = data_set.map(operations=[C.Decode(), - C.Resize((224, 224)), - lambda img: np.array(img[:, :, 0])], + data_set = data_set.map(operations=[C.Decode(), C.Resize((224, 224)), lambda img: np.array(img[:, :, 0])], input_columns=["image"]) # invalid gamma - data_set = data_set.map(operations=C.AdjustGamma(gamma=-10.0, - gain=1.0), + data_set = data_set.map(operations=C.AdjustGamma(gamma=-10.0, gain=1.0), input_columns="image") except ValueError as error: logger.info("Got an exception in AdjustGamma: {}".format(str(error))) assert "Input is not within the required interval of " in str(error) try: data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - data_set = data_set.map(operations=[C.Decode(), - C.Resize((224, 224)), - lambda img: np.array(img[:, :, 0])], + data_set = data_set.map(operations=[C.Decode(), C.Resize((224, 224)), lambda img: np.array(img[:, :, 0])], input_columns=["image"]) # invalid gamma - data_set = data_set.map(operations=C.AdjustGamma(gamma=[1, 2], - gain=1.0), + data_set = data_set.map(operations=C.AdjustGamma(gamma=[1, 2], gain=1.0), input_columns="image") except TypeError as error: logger.info("Got an exception in AdjustGamma: {}".format(str(error))) @@ -129,8 +121,7 @@ def test_adjust_gamma_invalid_gamma_param_py(): F.AdjustGamma(gamma=-10.0), F.ToTensor() ]) - data_set = data_set.map(operations=[trans], - input_columns=["image"]) + data_set = data_set.map(operations=[trans], input_columns=["image"]) except ValueError as error: logger.info("Got an exception in AdjustGamma: {}".format(str(error))) assert "Input is not within the required interval of " in str(error) @@ -142,8 +133,7 @@ def test_adjust_gamma_invalid_gamma_param_py(): F.AdjustGamma(gamma=[1, 2]), F.ToTensor() ]) - data_set = data_set.map(operations=[trans], - input_columns=["image"]) + data_set = data_set.map(operations=[trans], input_columns=["image"]) except TypeError as error: logger.info("Got an exception in AdjustGamma: {}".format(str(error))) assert "is not of type [, ], but got" in str(error) @@ -156,13 +146,10 @@ def test_adjust_gamma_invalid_gain_param_c(): logger.info("Test AdjustGamma C Op with invalid gain parameter") try: data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - data_set = data_set.map(operations=[C.Decode(), - C.Resize((224, 224)), - lambda img: np.array(img[:, :, 0])], + data_set = data_set.map(operations=[C.Decode(), C.Resize((224, 224)), lambda img: np.array(img[:, :, 0])], input_columns=["image"]) # invalid gain - data_set = data_set.map(operations=C.AdjustGamma(gamma=10.0, - gain=[1, 10]), + data_set = data_set.map(operations=C.AdjustGamma(gamma=10.0, gain=[1, 10]), input_columns="image") except TypeError as error: logger.info("Got an exception in AdjustGamma: {}".format(str(error))) @@ -182,8 +169,7 @@ def test_adjust_gamma_invalid_gain_param_py(): F.AdjustGamma(gamma=10.0, gain=[1, 10]), F.ToTensor() ]) - data_set = data_set.map(operations=[trans], - input_columns=["image"]) + data_set = data_set.map(operations=[trans], input_columns=["image"]) except TypeError as error: logger.info("Got an exception in AdjustGamma: {}".format(str(error))) assert "is not of type [, ], but got " in str(error) diff --git a/tests/ut/python/dataset/test_allpass_biquad.py b/tests/ut/python/dataset/test_allpass_biquad.py index e3cadece4f5..29805ab6df3 100644 --- a/tests/ut/python/dataset/test_allpass_biquad.py +++ b/tests/ut/python/dataset/test_allpass_biquad.py @@ -19,16 +19,14 @@ import mindspore.dataset.audio.transforms as audio from mindspore import log as logger -def _count_unequal_element(data_expected, data_me, rtol, atol): - +def count_unequal_element(data_expected, data_me, rtol, atol): assert data_expected.shape == data_me.shape total_count = len(data_expected.flatten()) error = np.abs(data_expected - data_me) greater = np.greater(error, atol + np.abs(data_expected) * rtol) loss_count = np.count_nonzero(greater) - assert (loss_count / total_count) < rtol, \ - "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \ - format(data_expected[greater], data_me[greater], error[greater]) + assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format( + data_expected[greater], data_me[greater], error[greater]) def test_func_allpass_biquad_eager(): @@ -37,12 +35,11 @@ def test_func_allpass_biquad_eager(): # Original waveform waveform = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64) # Expect waveform - expect_waveform = np.array([[0.96049707, 1.0, 1.0], - [1.0, 1.0, 1.0]], dtype=np.float64) + expect_waveform = np.array([[0.96049707, 1.0, 1.0], [1.0, 1.0, 1.0]], dtype=np.float64) allpass_biquad_op = audio.AllpassBiquad(44100, 200.0, 0.707) # Filtered waveform by allpassbiquad output = allpass_biquad_op(waveform) - _count_unequal_element(expect_waveform, output, 0.0001, 0.0001) + count_unequal_element(expect_waveform, output, 0.0001, 0.0001) def test_func_allpass_biquad_pipeline(): @@ -51,56 +48,57 @@ def test_func_allpass_biquad_pipeline(): # Original waveform waveform = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64) # Expect waveform - expect_waveform = np.array([[0.96049707, 1.0, 1.0], - [1.0, 1.0, 1.0]], dtype=np.float64) + expect_waveform = np.array([[0.96049707, 1.0, 1.0], [1.0, 1.0, 1.0]], dtype=np.float64) label = np.random.sample((2, 1)) data = (waveform, label) dataset = ds.NumpySlicesDataset(data, ["channel", "sample"], shuffle=False) allpass_biquad_op = audio.AllpassBiquad(44100, 200.0) # Filtered waveform by allpassbiquad - dataset = dataset.map( - input_columns=["channel"], operations=allpass_biquad_op, num_parallel_workers=8) + dataset = dataset.map(input_columns=["channel"], operations=allpass_biquad_op, num_parallel_workers=8) i = 0 - for _ in dataset.create_dict_iterator(output_numpy=True): - _count_unequal_element(expect_waveform[i, :], - _['channel'], 0.0001, 0.0001) + for item in dataset.create_dict_iterator(output_numpy=True): + count_unequal_element(expect_waveform[i, :], item['channel'], 0.0001, 0.0001) i += 1 + def test_invalid_input_all(): waveform = np.random.rand(2, 1000) + def test_invalid_input(test_name, sample_rate, central_freq, Q, error, error_msg): logger.info("Test Allpassallpassiquad with bad input: {0}".format(test_name)) with pytest.raises(error) as error_info: audio.AllpassBiquad(sample_rate, central_freq, Q)(waveform) assert error_msg in str(error_info.value) + test_invalid_input("invalid sample_rate parameter type as a float", 44100.5, 200, 0.707, TypeError, "Argument sample_rate with value 44100.5 is not of type []," - +" but got .") + + " but got .") test_invalid_input("invalid sample_rate parameter type as a String", "44100", 200, 0.707, TypeError, - "Argument sample_rate with value 44100 is not of type [],"+ + "Argument sample_rate with value 44100 is not of type []," + " but got .") test_invalid_input("invalid contral_freq parameter type as a String", 44100, "200", 0.707, TypeError, "Argument central_freq with value 200 is not of type [, ]," - +" but got .") + + " but got .") test_invalid_input("invalid Q parameter type as a String", 44100, 200, "0.707", TypeError, "Argument Q with value 0.707 is not of type [, ]," - +" but got .") + + " but got .") test_invalid_input("invalid sample_rate parameter value", 441324343243242342345300, 200, 0.707, ValueError, - "Input sample_rate is not within the required interval of [-2147483648, 2147483647].") + "Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].") test_invalid_input("invalid contral_freq parameter value", 44100, 32434324324234321, 0.707, ValueError, "Input central_freq is not within the required interval of [-16777216, 16777216].") test_invalid_input("invalid sample_rate parameter value", None, 200, 0.707, TypeError, "Argument sample_rate with value None is not of type []," - +" but got .") + + " but got .") test_invalid_input("invalid central_rate parameter value", 44100, None, 0.707, TypeError, "Argument central_freq with value None is not of type [, ]," - +" but got .") + + " but got .") test_invalid_input("invalid sample_rate parameter value", 0, 200, 0.707, ValueError, - "Input sample_rate can not be 0.") + "Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].") test_invalid_input("invalid Q parameter value", 44100, 200, 1.707, ValueError, "Input Q is not within the required interval of (0, 1].") + if __name__ == '__main__': - test_eager_allpassbiquad_mindspore_001() - test_pipeline_allpass_biquad_001() + test_func_allpass_biquad_eager() + test_func_allpass_biquad_pipeline() test_invalid_input_all() diff --git a/tests/ut/python/dataset/test_amplitude_to_db.py b/tests/ut/python/dataset/test_amplitude_to_db.py index 448b8b09ef4..9fba2ed07b9 100644 --- a/tests/ut/python/dataset/test_amplitude_to_db.py +++ b/tests/ut/python/dataset/test_amplitude_to_db.py @@ -23,7 +23,6 @@ import mindspore.dataset.audio.transforms as c_audio from mindspore import log as logger from mindspore.dataset.audio.utils import ScaleType - CHANNEL = 1 FREQ = 20 TIME = 15 @@ -32,19 +31,18 @@ TIME = 15 def gen(shape): np.random.seed(0) data = np.random.random(shape) - yield(np.array(data, dtype=np.float32),) + yield (np.array(data, dtype=np.float32),) -def _count_unequal_element(data_expected, data_me, rtol, atol): +def count_unequal_element(data_expected, data_me, rtol, atol): """ Precision calculation func """ assert data_expected.shape == data_me.shape total_count = len(data_expected.flatten()) error = np.abs(data_expected - data_me) greater = np.greater(error, atol + np.abs(data_expected) * rtol) loss_count = np.count_nonzero(greater) - assert (loss_count / total_count) < rtol, \ - "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \ - format(data_expected[greater], data_me[greater], error[greater]) + assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format( + data_expected[greater], data_me[greater], error[greater]) def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True): @@ -52,9 +50,7 @@ def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True): if np.any(np.isnan(data_expected)): assert np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan) elif not np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan): - _count_unequal_element(data_expected, data_me, rtol, atol) - else: - assert True + count_unequal_element(data_expected, data_me, rtol, atol) def test_func_amplitude_to_db_eager(): @@ -91,9 +87,7 @@ def test_func_amplitude_to_db_pipeline(): data1 = ds.GeneratorDataset(source=generator, column_names=["multi_dimensional_data"]) - transforms = [ - c_audio.AmplitudeToDB() - ] + transforms = [c_audio.AmplitudeToDB()] data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"]) for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True): @@ -102,7 +96,6 @@ def test_func_amplitude_to_db_pipeline(): def test_amplitude_to_db_invalid_input(): - def test_invalid_input(test_name, stype, ref_value, amin, top_db, error, error_msg): logger.info("Test AmplitudeToDB with bad input: {0}".format(test_name)) with pytest.raises(error) as error_info: diff --git a/tests/ut/python/dataset/test_angle.py b/tests/ut/python/dataset/test_angle.py index 6c366b6a41e..1de8e2fd0a2 100755 --- a/tests/ut/python/dataset/test_angle.py +++ b/tests/ut/python/dataset/test_angle.py @@ -19,28 +19,28 @@ import pytest import mindspore.dataset as ds import mindspore.dataset.audio.transforms as a_c_trans -def _count_unequal_element(data_expected, data_me, rtol, atol): +def count_unequal_element(data_expected, data_me, rtol, atol): assert data_expected.shape == data_me.shape total_count = len(data_expected.flatten()) error = np.abs(data_expected - data_me) greater = np.greater(error, atol + np.abs(data_expected) * rtol) loss_count = np.count_nonzero(greater) - assert (loss_count / total_count) < rtol, \ - "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \ - format(data_expected[greater], data_me[greater], error[greater]) + assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format( + data_expected[greater], data_me[greater], error[greater]) + def test_func_angle_001(): """ Eager Test """ - arr = np.array([[73.04, -13.00], [57.49, 13.20], [-57.64, 6.51], [-52.25, 30.67], [-30.11, -18.34], \ + arr = np.array([[73.04, -13.00], [57.49, 13.20], [-57.64, 6.51], [-52.25, 30.67], [-30.11, -18.34], [-63.32, 99.33], [95.82, -24.76]], dtype=np.double) - expected = np.array([-0.17614017, 0.22569334, 3.02912684, 2.6107975, -2.59450886, 2.13831337, -0.25286988], \ + expected = np.array([-0.17614017, 0.22569334, 3.02912684, 2.6107975, -2.59450886, 2.13831337, -0.25286988], dtype=np.double) angle_op = a_c_trans.Angle() output = angle_op(arr) - _count_unequal_element(expected, output, 0.0001, 0.0001) + count_unequal_element(expected, output, 0.0001, 0.0001) def test_func_angle_002(): @@ -48,9 +48,9 @@ def test_func_angle_002(): Pipeline Test """ np.random.seed(6) - arr = np.array([[[84.25, -85.92], [-92.23, 23.06], [-7.33, -44.17], [-62.95, -14.73]], \ + arr = np.array([[[84.25, -85.92], [-92.23, 23.06], [-7.33, -44.17], [-62.95, -14.73]], [[93.09, 38.18], [-81.94, 71.34], [71.33, -39.00], [95.25, -32.94]]], dtype=np.double) - expected = np.array([[-0.79521156, 2.89658848, -1.73524737, -2.91173309], \ + expected = np.array([[-0.79521156, 2.89658848, -1.73524737, -2.91173309], [0.3892177, 2.42523905, -0.50034807, -0.33295219]], dtype=np.double) label = np.random.sample((2, 4, 1)) data = (arr, label) @@ -58,7 +58,8 @@ def test_func_angle_002(): angle_op = a_c_trans.Angle() dataset = dataset.map(operations=angle_op, input_columns=["col1"]) for item1, item2 in zip(dataset.create_dict_iterator(output_numpy=True), expected): - _count_unequal_element(item2, item1['col1'], 0.0001, 0.0001) + count_unequal_element(item2, item1['col1'], 0.0001, 0.0001) + def test_func_angle_003(): """ @@ -72,7 +73,7 @@ def test_func_angle_003(): angle_op = a_c_trans.Angle() dataset = dataset.map(operations=angle_op, input_columns=["col1"]) num_itr = 0 - with pytest.raises(RuntimeError, match="The input type should be numbers"): + with pytest.raises(RuntimeError, match="input tensor type should be int, float or double"): for _ in dataset.create_dict_iterator(output_numpy=True): num_itr += 1 diff --git a/tests/ut/python/dataset/test_band_biquad.py b/tests/ut/python/dataset/test_band_biquad.py index a554a4df36c..6136159cbd7 100644 --- a/tests/ut/python/dataset/test_band_biquad.py +++ b/tests/ut/python/dataset/test_band_biquad.py @@ -19,16 +19,14 @@ import mindspore.dataset.audio.transforms as audio from mindspore import log as logger -def _count_unequal_element(data_expected, data_me, rtol, atol): - +def count_unequal_element(data_expected, data_me, rtol, atol): assert data_expected.shape == data_me.shape total_count = len(data_expected.flatten()) error = np.abs(data_expected - data_me) greater = np.greater(error, atol + np.abs(data_expected) * rtol) loss_count = np.count_nonzero(greater) - assert (loss_count / total_count) < rtol, \ - "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \ - format(data_expected[greater], data_me[greater], error[greater]) + assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format( + data_expected[greater], data_me[greater], error[greater]) def test_func_band_biquad_eager(): @@ -42,7 +40,7 @@ def test_func_band_biquad_eager(): band_biquad_op = audio.BandBiquad(44100, 200.0, 0.707, False) # Filtered waveform by bandbiquad output = band_biquad_op(waveform) - _count_unequal_element(expect_waveform, output, 0.0001, 0.0001) + count_unequal_element(expect_waveform, output, 0.0001, 0.0001) def test_func_band_biquad_pipeline(): @@ -61,9 +59,9 @@ def test_func_band_biquad_pipeline(): dataset = dataset.map( input_columns=["channel"], operations=band_biquad_op, num_parallel_workers=8) i = 0 - for _ in dataset.create_dict_iterator(output_numpy=True): - _count_unequal_element(expect_waveform[i, :], - _['channel'], 0.0001, 0.0001) + for item in dataset.create_dict_iterator(output_numpy=True): + count_unequal_element(expect_waveform[i, :], + item['channel'], 0.0001, 0.0001) i += 1 @@ -83,7 +81,7 @@ def test_band_biquad_invalid_input(): "Argument central_freq with value 200 is not of type [, ]," " but got .") test_invalid_input("invalid sample_rate parameter value", 0, 200, 0.707, True, ValueError, - "Input sample_rate can not be 0.") + "Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].") test_invalid_input("invalid contral_freq parameter value", 44100, 32434324324234321, 0.707, True, ValueError, "Input central_freq is not within the required interval of [-16777216, 16777216].") test_invalid_input("invalid Q parameter type as a String", 44100, 200, "0.707", True, TypeError, @@ -94,7 +92,7 @@ def test_band_biquad_invalid_input(): test_invalid_input("invalid Q parameter value", 44100, 200, 0, True, ValueError, "Input Q is not within the required interval of (0, 1].") test_invalid_input("invalid sample_rate parameter value", 441324343243242342345300, 200, 0.707, True, ValueError, - "Input sample_rate is not within the required interval of [-2147483648, 2147483647].") + "Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].") test_invalid_input("invalid sample_rate parameter value", None, 200, 0.707, True, TypeError, "Argument sample_rate with value None is not of type []," " but got .") diff --git a/tests/ut/python/dataset/test_bandpass_biquad.py b/tests/ut/python/dataset/test_bandpass_biquad.py index 90a8ddc78b1..caa8277dc35 100644 --- a/tests/ut/python/dataset/test_bandpass_biquad.py +++ b/tests/ut/python/dataset/test_bandpass_biquad.py @@ -19,16 +19,14 @@ import mindspore.dataset.audio.transforms as audio from mindspore import log as logger -def _count_unequal_element(data_expected, data_me, rtol, atol): - +def count_unequal_element(data_expected, data_me, rtol, atol): assert data_expected.shape == data_me.shape total_count = len(data_expected.flatten()) error = np.abs(data_expected - data_me) greater = np.greater(error, atol + np.abs(data_expected) * rtol) loss_count = np.count_nonzero(greater) - assert (loss_count / total_count) < rtol, \ - "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \ - format(data_expected[greater], data_me[greater], error[greater]) + assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format( + data_expected[greater], data_me[greater], error[greater]) def test_func_bandpass_biquad_eager(): @@ -42,7 +40,7 @@ def test_func_bandpass_biquad_eager(): bandpass_biquad_op = audio.BandpassBiquad(44000, 200.0, 0.707, False) # Filtered waveform by bandpassbiquad output = bandpass_biquad_op(waveform) - _count_unequal_element(expect_waveform, output, 0.0001, 0.0001) + count_unequal_element(expect_waveform, output, 0.0001, 0.0001) def test_func_bandpass_biquad_pipeline(): @@ -58,12 +56,10 @@ def test_func_bandpass_biquad_pipeline(): dataset = ds.NumpySlicesDataset(data, ["channel", "sample"], shuffle=False) bandpass_biquad_op = audio.BandpassBiquad(44000, 200.0) # Filtered waveform by bandpassbiquad - dataset = dataset.map( - input_columns=["channel"], operations=bandpass_biquad_op, num_parallel_workers=8) + dataset = dataset.map(input_columns=["channel"], operations=bandpass_biquad_op, num_parallel_workers=8) i = 0 - for _ in dataset.create_dict_iterator(output_numpy=True): - _count_unequal_element(expect_waveform[i, :], - _['channel'], 0.0001, 0.0001) + for item in dataset.create_dict_iterator(output_numpy=True): + count_unequal_element(expect_waveform[i, :], item['channel'], 0.0001, 0.0001) i += 1 @@ -72,8 +68,7 @@ def test_bandpass_biquad_invalid_input(): logger.info( "Test BandpassBiquad with bad input: {0}".format(test_name)) with pytest.raises(error) as error_info: - audio.BandpassBiquad( - sample_rate, central_freq, Q, const_skirt_gain) + audio.BandpassBiquad(sample_rate, central_freq, Q, const_skirt_gain) assert error_msg in str(error_info.value) test_invalid_input("invalid sample_rate parameter type as a float", 44100.5, 200, 0.707, True, TypeError, @@ -85,7 +80,7 @@ def test_bandpass_biquad_invalid_input(): "Argument central_freq with value 200 is not of type [, ]," " but got .") test_invalid_input("invalid sample_rate parameter value", 0, 200, 0.707, True, ValueError, - "Input sample_rate can not be 0.") + "Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].") test_invalid_input("invalid contral_freq parameter value", 44100, 32434324324234321, 0.707, True, ValueError, "Input central_freq is not within the required interval of [-16777216, 16777216].") test_invalid_input("invalid Q parameter type as a String", 44100, 200, "0.707", True, TypeError, @@ -96,7 +91,7 @@ def test_bandpass_biquad_invalid_input(): test_invalid_input("invalid Q parameter value", 44100, 200, 0, True, ValueError, "Input Q is not within the required interval of (0, 1].") test_invalid_input("invalid sample_rate parameter value", 441324343243242342345300, 200, 0.707, True, ValueError, - "Input sample_rate is not within the required interval of [-2147483648, 2147483647].") + "Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].") test_invalid_input("invalid sample_rate parameter value", None, 200, 0.707, True, TypeError, "Argument sample_rate with value None is not of type []," " but got .") diff --git a/tests/ut/python/dataset/test_bandreject_biquad.py b/tests/ut/python/dataset/test_bandreject_biquad.py index 3c799c6f827..af04d34de25 100644 --- a/tests/ut/python/dataset/test_bandreject_biquad.py +++ b/tests/ut/python/dataset/test_bandreject_biquad.py @@ -19,16 +19,14 @@ import mindspore.dataset.audio.transforms as audio from mindspore import log as logger -def _count_unequal_element(data_expected, data_me, rtol, atol): - +def count_unequal_element(data_expected, data_me, rtol, atol): assert data_expected.shape == data_me.shape total_count = len(data_expected.flatten()) error = np.abs(data_expected - data_me) greater = np.greater(error, atol + np.abs(data_expected) * rtol) loss_count = np.count_nonzero(greater) - assert (loss_count / total_count) < rtol, \ - "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \ - format(data_expected[greater], data_me[greater], error[greater]) + assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format( + data_expected[greater], data_me[greater], error[greater]) def test_func_bandreject_biquad_eager(): @@ -43,7 +41,7 @@ def test_func_bandreject_biquad_eager(): bandreject_biquad_op = audio.BandrejectBiquad(44100, 200.0, 0.707) # Filtered waveform by bandrejectbiquad output = bandreject_biquad_op(waveform) - _count_unequal_element(expect_waveform, output, 0.0001, 0.0001) + count_unequal_element(expect_waveform, output, 0.0001, 0.0001) def test_func_bandreject_biquad_pipeline(): @@ -63,9 +61,9 @@ def test_func_bandreject_biquad_pipeline(): dataset = dataset.map( input_columns=["channel"], operations=bandreject_biquad_op, num_parallel_workers=8) i = 0 - for _ in dataset.create_dict_iterator(output_numpy=True): - _count_unequal_element(expect_waveform[i, :], - _['channel'], 0.0001, 0.0001) + for item in dataset.create_dict_iterator(output_numpy=True): + count_unequal_element(expect_waveform[i, :], + item['channel'], 0.0001, 0.0001) i += 1 @@ -76,6 +74,7 @@ def test_bandreject_biquad_invalid_input(): with pytest.raises(error) as error_info: audio.BandrejectBiquad(sample_rate, central_freq, Q) assert error_msg in str(error_info.value) + test_invalid_input("invalid sample_rate parameter type as a float", 44100.5, 200, 0.707, TypeError, "Argument sample_rate with value 44100.5 is not of type []," " but got .") @@ -85,7 +84,7 @@ def test_bandreject_biquad_invalid_input(): "Argument central_freq with value 200 is not of type [, ]," " but got .") test_invalid_input("invalid sample_rate parameter value", 0, 200, 0.707, ValueError, - "Input sample_rate can not be 0.") + "Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].") test_invalid_input("invalid contral_freq parameter value", 44100, 32434324324234321, 0.707, ValueError, "Input central_freq is not within the required interval of [-16777216, 16777216].") test_invalid_input("invalid Q parameter type as a String", 44100, 200, "0.707", TypeError, @@ -96,7 +95,7 @@ def test_bandreject_biquad_invalid_input(): test_invalid_input("invalid Q parameter value", 44100, 200, 0, ValueError, "Input Q is not within the required interval of (0, 1].") test_invalid_input("invalid sample_rate parameter value", 441324343243242342345300, 200, 0.707, ValueError, - "Input sample_rate is not within the required interval of [-2147483648, 2147483647].") + "Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].") test_invalid_input("invalid sample_rate parameter value", None, 200, 0.707, TypeError, "Argument sample_rate with value None is not of type []," " but got .") @@ -106,6 +105,6 @@ def test_bandreject_biquad_invalid_input(): if __name__ == "__main__": - test_func_band_biquad_eager() - test_func_band_biquad_pipeline() - test_band_biquad_invalid_input() + test_func_bandreject_biquad_eager() + test_func_bandreject_biquad_pipeline() + test_bandreject_biquad_invalid_input() diff --git a/tests/ut/python/dataset/test_bass_biquad.py b/tests/ut/python/dataset/test_bass_biquad.py index c06470db271..41f1e7c87cf 100644 --- a/tests/ut/python/dataset/test_bass_biquad.py +++ b/tests/ut/python/dataset/test_bass_biquad.py @@ -19,16 +19,14 @@ import mindspore.dataset.audio.transforms as audio from mindspore import log as logger -def _count_unequal_element(data_expected, data_me, rtol, atol): - +def count_unequal_element(data_expected, data_me, rtol, atol): assert data_expected.shape == data_me.shape total_count = len(data_expected.flatten()) error = np.abs(data_expected - data_me) greater = np.greater(error, atol + np.abs(data_expected) * rtol) loss_count = np.count_nonzero(greater) - assert (loss_count / total_count) < rtol, \ - "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \ - format(data_expected[greater], data_me[greater], error[greater]) + assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format( + data_expected[greater], data_me[greater], error[greater]) def test_func_bass_biquad_eager(): @@ -42,7 +40,7 @@ def test_func_bass_biquad_eager(): bass_biquad_op = audio.BassBiquad(44100, 50.0, 100.0, 0.707) # Filtered waveform by bassbiquad output = bass_biquad_op(waveform) - _count_unequal_element(expect_waveform, output, 0.0001, 0.0001) + count_unequal_element(expect_waveform, output, 0.0001, 0.0001) def test_func_bass_biquad_pipeline(): @@ -61,9 +59,9 @@ def test_func_bass_biquad_pipeline(): dataset = dataset.map( input_columns=["channel"], operations=bass_biquad_op, num_parallel_workers=8) i = 0 - for _ in dataset.create_dict_iterator(output_numpy=True): - _count_unequal_element(expect_waveform[i, :], - _['channel'], 0.0001, 0.0001) + for item in dataset.create_dict_iterator(output_numpy=True): + count_unequal_element(expect_waveform[i, :], + item['channel'], 0.0001, 0.0001) i += 1 @@ -73,6 +71,7 @@ def test_invalid_invalid_input(): with pytest.raises(error) as error_info: audio.BassBiquad(sample_rate, gain, central_freq, Q) assert error_msg in str(error_info.value) + test_invalid_input("invalid sample_rate parameter type as a float", 44100.5, 50.0, 200, 0.707, TypeError, "Argument sample_rate with value 44100.5 is not of type []," " but got .") @@ -90,7 +89,7 @@ def test_invalid_invalid_input(): " but got .") test_invalid_input("invalid sample_rate parameter value", 441324343243242342345300, 50.0, 200, 0.707, ValueError, - "Input sample_rate is not within the required interval of [-2147483648, 2147483647].") + "Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].") test_invalid_input("invalid gain parameter value", 44100, 32434324324234321, 200, 0.707, ValueError, "Input gain is not within the required interval of [-16777216, 16777216].") test_invalid_input("invalid contral_freq parameter value", 44100, 50, 32434324324234321, 0.707, ValueError, @@ -107,10 +106,11 @@ def test_invalid_invalid_input(): " but got .") test_invalid_input("invalid sample_rate parameter value", 0, 50.0, 200, 0.707, ValueError, - "Input sample_rate can not be 0.") + "Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].") test_invalid_input("invalid Q parameter value", 44100, 50.0, 200, 1.707, ValueError, "Input Q is not within the required interval of (0, 1].") + if __name__ == '__main__': test_func_bass_biquad_eager() test_func_bass_biquad_pipeline() diff --git a/tests/ut/python/dataset/test_datasets_sbd.py b/tests/ut/python/dataset/test_datasets_sbd.py index 3801cfa669b..db7c3b9fd05 100644 --- a/tests/ut/python/dataset/test_datasets_sbd.py +++ b/tests/ut/python/dataset/test_datasets_sbd.py @@ -22,7 +22,6 @@ import mindspore.dataset as ds from mindspore import log as logger import mindspore.dataset.vision.c_transforms as c_vision - DATASET_DIR = "../data/dataset/testSBData/sbd" @@ -193,6 +192,7 @@ def test_sbd_usage(): """ Validate SBDataset image readings """ + def test_config(usage): try: data = ds.SBDataset(DATASET_DIR, task='Segmentation', usage=usage) diff --git a/tests/ut/python/dataset/test_frequency_masking.py b/tests/ut/python/dataset/test_frequency_masking.py index 3a1f03ef399..3893b2f13bb 100644 --- a/tests/ut/python/dataset/test_frequency_masking.py +++ b/tests/ut/python/dataset/test_frequency_masking.py @@ -20,10 +20,9 @@ import numpy as np import pytest import mindspore.dataset as ds -import mindspore.dataset.audio.transforms as atf +import mindspore.dataset.audio.transforms as audio from mindspore import log as logger - CHANNEL = 2 FREQ = 30 TIME = 30 @@ -32,19 +31,18 @@ TIME = 30 def gen(shape): np.random.seed(0) data = np.random.random(shape) - yield(np.array(data, dtype=np.float32),) + yield (np.array(data, dtype=np.float32),) -def _count_unequal_element(data_expected, data_me, rtol, atol): +def count_unequal_element(data_expected, data_me, rtol, atol): """ Precision calculation func """ assert data_expected.shape == data_me.shape total_count = len(data_expected.flatten()) error = np.abs(data_expected - data_me) greater = np.greater(error, atol + np.abs(data_expected) * rtol) loss_count = np.count_nonzero(greater) - assert (loss_count / total_count) < rtol, \ - "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \ - format(data_expected[greater], data_me[greater], error[greater]) + assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format( + data_expected[greater], data_me[greater], error[greater]) def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True): @@ -52,16 +50,14 @@ def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True): if np.any(np.isnan(data_expected)): assert np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan) elif not np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan): - _count_unequal_element(data_expected, data_me, rtol, atol) - else: - assert True + count_unequal_element(data_expected, data_me, rtol, atol) def test_func_frequency_masking_eager_random_input(): """ mindspore eager mode normal testcase:frequency_masking op""" logger.info("test frequency_masking op") spectrogram = next(gen((CHANNEL, FREQ, TIME)))[0] - out_put = atf.FrequencyMasking(False, 3, 1, 10)(spectrogram) + out_put = audio.FrequencyMasking(False, 3, 1, 10)(spectrogram) assert out_put.shape == (CHANNEL, FREQ, TIME) @@ -74,7 +70,7 @@ def test_func_frequency_masking_eager_precision(): [[-1.205032, 0.18922766, -0.5277673, -1.3090396], [1.8914849, -0.97001046, -0.23726775, 0.00525892], [-1.0271876, 0.33526883, 1.7413973, 0.12313101]]]).astype(np.float32) - out_ms = atf.FrequencyMasking(False, 2, 0, 0)(spectrogram) + out_ms = audio.FrequencyMasking(False, 2, 0, 0)(spectrogram) out_benchmark = np.array([[[0.0, 0.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0], [1.8175547, -0.25779432, -0.58152324, -0.00221091]], @@ -89,14 +85,10 @@ def test_func_frequency_masking_pipeline(): logger.info("test frequency_masking op, pipeline") generator = gen([CHANNEL, FREQ, TIME]) - data1 = ds.GeneratorDataset(source=generator, column_names=[ - "multi_dimensional_data"]) + data1 = ds.GeneratorDataset(source=generator, column_names=["multi_dimensional_data"]) - transforms = [ - atf.FrequencyMasking(True, 8) - ] - data1 = data1.map(operations=transforms, input_columns=[ - "multi_dimensional_data"]) + transforms = [audio.FrequencyMasking(True, 8)] + data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"]) for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True): out_put = item["multi_dimensional_data"] @@ -107,14 +99,14 @@ def test_frequency_masking_invalid_input(): def test_invalid_param(test_name, iid_masks, frequency_mask_param, mask_start, error, error_msg): logger.info("Test FrequencyMasking with wrong params: {0}".format(test_name)) with pytest.raises(error) as error_info: - atf.FrequencyMasking(iid_masks, frequency_mask_param, mask_start) + audio.FrequencyMasking(iid_masks, frequency_mask_param, mask_start) assert error_msg in str(error_info.value) def test_invalid_input(test_name, iid_masks, frequency_mask_param, mask_start, error, error_msg): logger.info("Test FrequencyMasking with wrong params: {0}".format(test_name)) with pytest.raises(error) as error_info: spectrogram = next(gen((CHANNEL, FREQ, TIME)))[0] - _ = atf.FrequencyMasking(iid_masks, frequency_mask_param, mask_start)(spectrogram) + _ = audio.FrequencyMasking(iid_masks, frequency_mask_param, mask_start)(spectrogram) assert error_msg in str(error_info.value) test_invalid_param("invalid mask_start", True, 2, -10, ValueError, diff --git a/tests/ut/python/dataset/test_rgb_bgr.py b/tests/ut/python/dataset/test_rgb_bgr.py index b6c93a64a97..6b1fd20ef59 100644 --- a/tests/ut/python/dataset/test_rgb_bgr.py +++ b/tests/ut/python/dataset/test_rgb_bgr.py @@ -24,8 +24,6 @@ import mindspore.dataset.vision.c_transforms as vision import mindspore.dataset.vision.py_transforms as py_vision import mindspore.dataset.vision.py_transforms_util as util -GENERATE_GOLDEN = False - DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" diff --git a/tests/ut/python/dataset/test_time_masking.py b/tests/ut/python/dataset/test_time_masking.py index ada96ea6d34..3aee6340590 100644 --- a/tests/ut/python/dataset/test_time_masking.py +++ b/tests/ut/python/dataset/test_time_masking.py @@ -20,10 +20,9 @@ import numpy as np import pytest import mindspore.dataset as ds -import mindspore.dataset.audio.transforms as atf +import mindspore.dataset.audio.transforms as audio from mindspore import log as logger - CHANNEL = 2 FREQ = 20 TIME = 30 @@ -32,19 +31,18 @@ TIME = 30 def gen(shape): np.random.seed(0) data = np.random.random(shape) - yield(np.array(data, dtype=np.float32),) + yield (np.array(data, dtype=np.float32),) -def _count_unequal_element(data_expected, data_me, rtol, atol): +def count_unequal_element(data_expected, data_me, rtol, atol): """ Precision calculation func """ assert data_expected.shape == data_me.shape total_count = len(data_expected.flatten()) error = np.abs(data_expected - data_me) greater = np.greater(error, atol + np.abs(data_expected) * rtol) loss_count = np.count_nonzero(greater) - assert (loss_count / total_count) < rtol, \ - "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \ - format(data_expected[greater], data_me[greater], error[greater]) + assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format( + data_expected[greater], data_me[greater], error[greater]) def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True): @@ -52,16 +50,14 @@ def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True): if np.any(np.isnan(data_expected)): assert np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan) elif not np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan): - _count_unequal_element(data_expected, data_me, rtol, atol) - else: - assert True + count_unequal_element(data_expected, data_me, rtol, atol) def test_func_time_masking_eager_random_input(): """ mindspore eager mode normal testcase:time_masking op""" logger.info("test time_masking op") spectrogram = next(gen((CHANNEL, FREQ, TIME)))[0] - out_put = atf.TimeMasking(False, 3, 1, 10)(spectrogram) + out_put = audio.TimeMasking(False, 3, 1, 10)(spectrogram) assert out_put.shape == (CHANNEL, FREQ, TIME) @@ -74,7 +70,7 @@ def test_func_time_masking_eager_precision(): [[-1.205032, 0.18922766, -0.5277673, -1.3090396], [1.8914849, -0.97001046, -0.23726775, 0.00525892], [-1.0271876, 0.33526883, 1.7413973, 0.12313101]]]).astype(np.float32) - out_ms = atf.TimeMasking(False, 2, 0, 0)(spectrogram) + out_ms = audio.TimeMasking(False, 2, 0, 0)(spectrogram) out_benchmark = np.array([[[0., 0., 0.07162686, -0.45436913], [0., 0., 0.62333095, -0.09532598], [0., 0., -0.58152324, -0.00221091]], @@ -89,14 +85,10 @@ def test_func_time_masking_pipeline(): logger.info("test time_masking op, pipeline") generator = gen([CHANNEL, FREQ, TIME]) - data1 = ds.GeneratorDataset(source=generator, column_names=[ - "multi_dimensional_data"]) + data1 = ds.GeneratorDataset(source=generator, column_names=["multi_dimensional_data"]) - transforms = [ - atf.TimeMasking(True, 8) - ] - data1 = data1.map(operations=transforms, input_columns=[ - "multi_dimensional_data"]) + transforms = [audio.TimeMasking(True, 8)] + data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"]) for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True): out_put = item["multi_dimensional_data"] @@ -107,14 +99,14 @@ def test_time_masking_invalid_input(): def test_invalid_param(test_name, iid_masks, time_mask_param, mask_start, error, error_msg): logger.info("Test TimeMasking with wrong params: {0}".format(test_name)) with pytest.raises(error) as error_info: - atf.TimeMasking(iid_masks, time_mask_param, mask_start) + audio.TimeMasking(iid_masks, time_mask_param, mask_start) assert error_msg in str(error_info.value) def test_invalid_input(test_name, iid_masks, time_mask_param, mask_start, error, error_msg): logger.info("Test TimeMasking with wrong params: {0}".format(test_name)) with pytest.raises(error) as error_info: spectrogram = next(gen((CHANNEL, FREQ, TIME)))[0] - _ = atf.TimeMasking(iid_masks, time_mask_param, mask_start)(spectrogram) + _ = audio.TimeMasking(iid_masks, time_mask_param, mask_start)(spectrogram) assert error_msg in str(error_info.value) test_invalid_param("invalid mask_start", True, 2, -10, ValueError, diff --git a/tests/ut/python/dataset/test_time_stretch.py b/tests/ut/python/dataset/test_time_stretch.py index 52a796c7ad6..577c40ebdbf 100644 --- a/tests/ut/python/dataset/test_time_stretch.py +++ b/tests/ut/python/dataset/test_time_stretch.py @@ -31,27 +31,24 @@ COMPLEX = 2 def gen(shape): np.random.seed(0) data = np.random.random(shape) - yield(np.array(data, dtype=np.float32),) + yield (np.array(data, dtype=np.float32),) -def _count_unequal_element(data_expected, data_me, rtol, atol): +def count_unequal_element(data_expected, data_me, rtol, atol): assert data_expected.shape == data_me.shape total_count = len(data_expected.flatten()) error = np.abs(data_expected - data_me) greater = np.greater(error, atol + np.abs(data_expected) * rtol) loss_count = np.count_nonzero(greater) - assert (loss_count / total_count) < rtol, \ - "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \ - format(data_expected[greater], data_me[greater], error[greater]) + assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format( + data_expected[greater], data_me[greater], error[greater]) def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True): if np.any(np.isnan(data_expected)): assert np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan) elif not np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan): - _count_unequal_element(data_expected, data_me, rtol, atol) - else: - assert True + count_unequal_element(data_expected, data_me, rtol, atol) def test_time_stretch_pipeline(): @@ -60,18 +57,14 @@ def test_time_stretch_pipeline(): """ logger.info("test TimeStretch op") generator = gen([CHANNEL_NUM, FREQ, FRAME_NUM, COMPLEX]) - data1 = ds.GeneratorDataset(source=generator, column_names=[ - "multi_dimensional_data"]) + data1 = ds.GeneratorDataset(source=generator, column_names=["multi_dimensional_data"]) - transforms = [ - c_audio.TimeStretch(512, FREQ, 1.3) - ] - data1 = data1.map(operations=transforms, input_columns=[ - "multi_dimensional_data"]) + transforms = [c_audio.TimeStretch(512, FREQ, 1.3)] + data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"]) for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True): out_put = item["multi_dimensional_data"] - assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM/1.3), COMPLEX) + assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM / 1.3), COMPLEX) def test_time_stretch_pipeline_invalid_param(): @@ -80,19 +73,15 @@ def test_time_stretch_pipeline_invalid_param(): """ logger.info("test TimeStretch op with invalid values") generator = gen([CHANNEL_NUM, FREQ, FRAME_NUM, COMPLEX]) - data1 = ds.GeneratorDataset(source=generator, column_names=[ - "multi_dimensional_data"]) + data1 = ds.GeneratorDataset(source=generator, column_names=["multi_dimensional_data"]) with pytest.raises(ValueError, match=r"Input fixed_rate is not within the required interval of \(0, 16777216\]."): - transforms = [ - c_audio.TimeStretch(512, FREQ, -1.3) - ] - data1 = data1.map(operations=transforms, input_columns=[ - "multi_dimensional_data"]) + transforms = [c_audio.TimeStretch(512, FREQ, -1.3)] + data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"]) for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True): out_put = item["multi_dimensional_data"] - assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM/1.3), COMPLEX) + assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM / 1.3), COMPLEX) def test_time_stretch_eager(): @@ -102,7 +91,7 @@ def test_time_stretch_eager(): logger.info("test TimeStretch op with customized parameter values") spectrogram = next(gen([CHANNEL_NUM, FREQ, FRAME_NUM, COMPLEX]))[0] out_put = c_audio.TimeStretch(512, FREQ, 1.3)(spectrogram) - assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM/1.3), COMPLEX) + assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM / 1.3), COMPLEX) def test_percision_time_stretch_eager():