diff --git a/mindspore/ccsrc/minddata/dataset/api/audio.cc b/mindspore/ccsrc/minddata/dataset/api/audio.cc index 55db0ea23ac..008806f91be 100644 --- a/mindspore/ccsrc/minddata/dataset/api/audio.cc +++ b/mindspore/ccsrc/minddata/dataset/api/audio.cc @@ -45,6 +45,7 @@ #include "minddata/dataset/audio/ir/kernels/phaser_ir.h" #include "minddata/dataset/audio/ir/kernels/riaa_biquad_ir.h" #include "minddata/dataset/audio/ir/kernels/sliding_window_cmn_ir.h" +#include "minddata/dataset/audio/ir/kernels/spectrogram_ir.h" #include "minddata/dataset/audio/ir/kernels/time_masking_ir.h" #include "minddata/dataset/audio/ir/kernels/time_stretch_ir.h" #include "minddata/dataset/audio/ir/kernels/treble_biquad_ir.h" @@ -530,6 +531,43 @@ std::shared_ptr SlidingWindowCmn::Parse() { data_->norm_vars_); } +// Spectrogram Transform Operation. +struct Spectrogram::Data { + Data(int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad, WindowType window, float power, + bool normalized, bool center, BorderType pad_mode, bool onesided) + : n_fft_(n_fft), + win_length_(win_length), + hop_length_(hop_length), + pad_(pad), + window_(window), + power_(power), + normalized_(normalized), + center_(center), + pad_mode_(pad_mode), + onesided_(onesided) {} + int32_t n_fft_; + int32_t win_length_; + int32_t hop_length_; + int32_t pad_; + WindowType window_; + float power_; + bool normalized_; + bool center_; + BorderType pad_mode_; + bool onesided_; +}; + +Spectrogram::Spectrogram(int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad, WindowType window, + float power, bool normalized, bool center, BorderType pad_mode, bool onesided) + : data_(std::make_shared(n_fft, win_length, hop_length, pad, window, power, normalized, center, pad_mode, + onesided)) {} + +std::shared_ptr Spectrogram::Parse() { + return std::make_shared(data_->n_fft_, data_->win_length_, data_->hop_length_, data_->pad_, + data_->window_, data_->power_, data_->normalized_, data_->center_, + data_->pad_mode_, data_->onesided_); +} + // TimeMasking Transform Operation. struct TimeMasking::Data { Data(bool iid_masks, int32_t time_mask_param, int32_t mask_start, float mask_value) diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc index 1dc12452346..2127e1714f8 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc @@ -49,6 +49,7 @@ #include "minddata/dataset/audio/ir/kernels/phaser_ir.h" #include "minddata/dataset/audio/ir/kernels/riaa_biquad_ir.h" #include "minddata/dataset/audio/ir/kernels/sliding_window_cmn_ir.h" +#include "minddata/dataset/audio/ir/kernels/spectrogram_ir.h" #include "minddata/dataset/audio/ir/kernels/time_masking_ir.h" #include "minddata/dataset/audio/ir/kernels/time_stretch_ir.h" #include "minddata/dataset/audio/ir/kernels/treble_biquad_ir.h" @@ -409,6 +410,29 @@ PYBIND_REGISTER(SlidingWindowCmnOperation, 1, ([](const py::module *m) { })); })); +PYBIND_REGISTER(WindowType, 0, ([](const py::module *m) { + (void)py::enum_(*m, "WindowType", py::arithmetic()) + .value("DE_BARTLETT", WindowType::kBartlett) + .value("DE_BLACKMAN", WindowType::kBlackman) + .value("DE_HAMMING", WindowType::kHamming) + .value("DE_HANN", WindowType::kHann) + .value("DE_KAISER", WindowType::kKaiser) + .export_values(); + })); + +PYBIND_REGISTER( + SpectrogramOperation, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "SpectrogramOperation") + .def(py::init([](int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad, WindowType window, + float power, bool normalized, bool center, BorderType pad_mode, bool onesided) { + auto spectrogram = std::make_shared(n_fft, win_length, hop_length, pad, window, + power, normalized, center, pad_mode, onesided); + THROW_IF_ERROR(spectrogram->ValidateParams()); + return spectrogram; + })); + })); + PYBIND_REGISTER( TimeMaskingOperation, 1, ([](const py::module *m) { (void)py::class_>( diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/CMakeLists.txt index b5c9ef29988..d66db1c9ab6 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/CMakeLists.txt @@ -31,6 +31,7 @@ add_library(audio-ir-kernels OBJECT phaser_ir.cc riaa_biquad_ir.cc sliding_window_cmn_ir.cc + spectrogram_ir.cc time_masking_ir.cc time_stretch_ir.cc treble_biquad_ir.cc diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/spectrogram_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/spectrogram_ir.cc new file mode 100644 index 00000000000..88117a7080e --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/spectrogram_ir.cc @@ -0,0 +1,77 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "minddata/dataset/audio/ir/kernels/spectrogram_ir.h" + +#include "minddata/dataset/audio/ir/validators.h" +#include "minddata/dataset/audio/kernels/spectrogram_op.h" + +namespace mindspore { +namespace dataset { +namespace audio { +// SpectrogramOperation +SpectrogramOperation::SpectrogramOperation(int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad, + WindowType window, float power, bool normalized, bool center, + BorderType pad_mode, bool onesided) + : n_fft_(n_fft), + win_length_(win_length), + hop_length_(hop_length), + pad_(pad), + window_(window), + power_(power), + normalized_(normalized), + center_(center), + pad_mode_(pad_mode), + onesided_(onesided) {} + +Status SpectrogramOperation::ValidateParams() { + RETURN_IF_NOT_OK(ValidateIntScalarPositive("Spectrogram", "n_fft", n_fft_)); + RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("Spectrogram", "win_length", win_length_)); + RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("Spectrogram", "hop_length", hop_length_)); + RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("Spectrogram", "pad", pad_)); + RETURN_IF_NOT_OK(ValidateFloatScalarNonNegative("Spectrogram", "power", power_)); + + CHECK_FAIL_RETURN_UNEXPECTED(win_length_ <= n_fft_, + "Spectrogram: win_length must be less than or equal to n_fft, but got win_length: " + + std::to_string(win_length_) + ", n_fft: " + std::to_string(n_fft_)); + return Status::OK(); +} + +std::shared_ptr SpectrogramOperation::Build() { + int32_t win_length = (win_length_ == 0) ? n_fft_ : win_length_; + int32_t hop_length = (hop_length_ == 0) ? win_length_ / 2 : hop_length_; + std::shared_ptr tensor_op = std::make_shared( + n_fft_, win_length, hop_length, pad_, window_, power_, normalized_, center_, pad_mode_, onesided_); + return tensor_op; +} + +Status SpectrogramOperation::to_json(nlohmann::json *out_json) { + nlohmann::json args; + args["n_fft"] = n_fft_; + args["win_length"] = win_length_; + args["hop_length"] = hop_length_; + args["pad"] = pad_; + args["window"] = window_; + args["power"] = power_; + args["normalized"] = normalized_; + args["center"] = center_; + args["pad_mode"] = pad_mode_; + args["onesided"] = onesided_; + *out_json = args; + return Status::OK(); +} +} // namespace audio +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/spectrogram_ir.h b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/spectrogram_ir.h new file mode 100644 index 00000000000..b8c17d6d685 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/spectrogram_ir.h @@ -0,0 +1,61 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_SPECTROGRAM_IR_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_SPECTROGRAM_IR_H_ + +#include +#include + +#include "include/api/status.h" +#include "minddata/dataset/kernels/ir/tensor_operation.h" + +namespace mindspore { +namespace dataset { +namespace audio { +constexpr char kSpectrogramOperation[] = "Spectrogram"; + +class SpectrogramOperation : public TensorOperation { + public: + SpectrogramOperation(int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad, WindowType window, + float power, bool normalized, bool center, BorderType pad_mode, bool onesided); + + ~SpectrogramOperation() = default; + + std::shared_ptr Build() override; + + Status ValidateParams() override; + + std::string Name() const override { return kSpectrogramOperation; } + + Status to_json(nlohmann::json *out_json) override; + + private: + int32_t n_fft_; + int32_t win_length_; + int32_t hop_length_; + int32_t pad_; + WindowType window_; + float power_; + bool normalized_; + bool center_; + BorderType pad_mode_; + bool onesided_; +}; +} // namespace audio +} // namespace dataset +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_SPECTROGRAM_IR_H_ diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/audio/kernels/CMakeLists.txt index 4ca1e0a39aa..f7699e54afd 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/CMakeLists.txt @@ -32,6 +32,7 @@ add_library(audio-kernels OBJECT phaser_op.cc riaa_biquad_op.cc sliding_window_cmn_op.cc + spectrogram_op.cc time_masking_op.cc time_stretch_op.cc treble_biquad_op.cc diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc index 697a2a228ca..e913ad95211 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc @@ -1156,6 +1156,355 @@ Status ComputeDeltasImpl(const std::shared_ptr &input, std::shared_ptr *output, int len) { + CHECK_FAIL_RETURN_UNEXPECTED(len != 0, "Bartlett: len can not be zero."); + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({len}), DataType(DataType::DE_FLOAT32), output)); + // Bartlett window function. + auto iter = (*output)->begin(); + float twice = 2.0; + for (ptrdiff_t i = 0; i < len; ++i) { + *(iter + i) = 1.0 - std::abs(twice * i / len - 1.0); + } + return Status::OK(); +} + +Status Blackman(std::shared_ptr *output, int len) { + CHECK_FAIL_RETURN_UNEXPECTED(len != 0, "Blackman: len can not be zero."); + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({len}), DataType(DataType::DE_FLOAT32), output)); + // Blackman window function. + auto iter = (*output)->begin(); + float alpha = 0.42; + float half = 0.5; + float delta = 0.08; + for (ptrdiff_t i = 0; i < len; ++i) { + *(iter + i) = alpha - half * std::cos(TWO * PI * i / len) + delta * std::cos(TWO * TWO * PI * i / len); + } + return Status::OK(); +} + +Status Hamming(std::shared_ptr *output, int len, float alpha = 0.54, float beta = 0.46) { + CHECK_FAIL_RETURN_UNEXPECTED(len != 0, "Hamming: len can not be zero."); + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({len}), DataType(DataType::DE_FLOAT32), output)); + // Hamming window function. + auto iter = (*output)->begin(); + for (ptrdiff_t i = 0; i < len; ++i) { + *(iter + i) = alpha - beta * std::cos(TWO * PI * i / len); + } + return Status::OK(); +} + +Status Hann(std::shared_ptr *output, int len) { + CHECK_FAIL_RETURN_UNEXPECTED(len != 0, "Hann: len can not be zero."); + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({len}), DataType(DataType::DE_FLOAT32), output)); + // Hann window function. + auto iter = (*output)->begin(); + float half = 0.5; + for (ptrdiff_t i = 0; i < len; ++i) { + *(iter + i) = half - half * std::cos(TWO * PI * i / len); + } + return Status::OK(); +} + +Status Kaiser(std::shared_ptr *output, int len, float beta = 12.0) { + CHECK_FAIL_RETURN_UNEXPECTED(len != 0, "Kaiser: len can not be zero."); + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({len}), DataType(DataType::DE_FLOAT32), output)); + // Kaiser window function. + auto iter = (*output)->begin(); + float twice = 2.0; + for (ptrdiff_t i = 0; i < len; ++i) { + *(iter + i) = + std::cyl_bessel_i(0, beta * std::sqrt(1 - std::pow(i * twice / (len)-1.0, TWO))) / std::cyl_bessel_i(0, beta); + } + return Status::OK(); +} + +Status Window(std::shared_ptr *output, WindowType window_type, int len) { + switch (window_type) { + case WindowType::kBartlett: + return Bartlett(output, len); + case WindowType::kBlackman: + return Blackman(output, len); + case WindowType::kHamming: + return Hamming(output, len); + case WindowType::kHann: + return Hann(output, len); + case WindowType::kKaiser: + return Kaiser(output, len); + default: + return Hann(output, len); + } +} + +// control whether return half of results after stft. +template +Status Onesided(const std::shared_ptr &input, std::shared_ptr *output, int n_fft, int n_columns) { + std::shared_ptr output_onsided; + RETURN_IF_NOT_OK( + Tensor::CreateEmpty(TensorShape({input->shape()[0], n_fft, n_columns, 2}), input->type(), &output_onsided)); + auto onside_begin = output_onsided->begin(); + auto spec_f_begin = input->begin(); + std::vector spec_f_slice = {(n_fft / 2 + 1) * n_columns * 2, n_columns * 2, 2}; + for (int r = 0; r < input->shape()[0]; r++) { + for (int i = 0; i < (n_fft / TWO + 1); i++) { + for (int j = 0; j < n_columns; j++) { + ptrdiff_t onside_offset_0 = r * n_fft * n_columns * 2 + i * spec_f_slice[1] + j * spec_f_slice[2]; + ptrdiff_t spec_f_offset_0 = r * spec_f_slice[0] + i * spec_f_slice[1] + j * spec_f_slice[2]; + ptrdiff_t onside_offset_1 = onside_offset_0 + 1; + ptrdiff_t spec_f_offset_1 = spec_f_offset_0 + 1; + *(onside_begin + onside_offset_0) = *(spec_f_begin + spec_f_offset_0); + *(onside_begin + onside_offset_1) = *(spec_f_begin + spec_f_offset_1); + } + } + for (int i = n_fft / 2 + 1; i < n_fft; i++) { + for (int j = 0; j < n_columns; j++) { + ptrdiff_t onside_offset_0 = r * n_fft * n_columns * 2 + i * spec_f_slice[1] + j * spec_f_slice[2]; + ptrdiff_t spec_f_offset_0 = r * spec_f_slice[0] + (n_fft - i) * spec_f_slice[1] + j * spec_f_slice[2]; + ptrdiff_t onside_offset_1 = onside_offset_0 + 1; + ptrdiff_t spec_f_offset_1 = spec_f_offset_0 + 1; + *(onside_begin + onside_offset_0) = *(spec_f_begin + spec_f_offset_0); + *(onside_begin + onside_offset_1) = *(spec_f_begin + spec_f_offset_1); + } + } + } + *output = output_onsided; + return Status::OK(); +} + +template +Status PowerStft(const std::shared_ptr &input, std::shared_ptr *output, float power, int n_fft, + int n_columns, int n_length) { + auto spec_f_begin = input->begin(); + std::vector spec_f_slice = {n_length * n_columns * 2, n_columns * 2, 2}; + std::vector spec_p_slice = {n_length * n_columns, n_columns}; + std::shared_ptr spec_p; + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({input->shape()[0], n_length, n_columns}), input->type(), &spec_p)); + auto spec_p_begin = spec_p->begin(); + for (int r = 0; r < input->shape()[0]; r++) { + for (int i = 0; i < n_length; i++) { + for (int j = 0; j < n_columns; j++) { + ptrdiff_t spec_f_offset_0 = r * spec_f_slice[0] + i * spec_f_slice[1] + j * spec_f_slice[2]; + ptrdiff_t spec_f_offset_1 = spec_f_offset_0 + 1; + ptrdiff_t spec_p_offset = r * spec_p_slice[0] + i * spec_p_slice[1] + j; + T spec_power_0 = *(spec_f_begin + spec_f_offset_0); + T spec_power_1 = *(spec_f_begin + spec_f_offset_1); + *(spec_p_begin + spec_p_offset) = + std::pow(std::sqrt(std::pow(spec_power_0, TWO) + std::pow(spec_power_1, TWO)), power); + } + } + } + *output = spec_p; + return Status::OK(); +} + +template +Status Stft(const std::shared_ptr &input, std::shared_ptr *output, int n_fft, + const std::shared_ptr &win, int win_length, int hop_length, int n_columns, bool normalized, + float power, bool onesided) { + CHECK_FAIL_RETURN_UNEXPECTED(win_length != 0, "Spectrogram: win_length can not be zero."); + double win_sum = 0.; + float twice = 2.0; + for (auto iter_win = win->begin(); iter_win != win->end(); iter_win++) { + win_sum += (*iter_win) * (*iter_win); + } + win_sum = std::sqrt(win_sum); + std::shared_ptr spec_f; + + RETURN_IF_NOT_OK( + Tensor::CreateEmpty(TensorShape({input->shape()[0], n_fft / 2 + 1, n_columns, 2}), input->type(), &spec_f)); + + auto spec_f_begin = spec_f->begin(); + auto input_win_begin = input->begin(); + std::vector spec_f_slice = {(n_fft / 2 + 1) * n_columns * 2, n_columns * 2, 2}; + std::vector input_win_slice = {n_columns * win_length, win_length}; + std::shared_ptr spec_p; + RETURN_IF_NOT_OK( + Tensor::CreateEmpty(TensorShape({input->shape()[0], n_fft / 2 + 1, n_columns}), input->type(), &spec_p)); + std::shared_ptr exp_complex; + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({n_fft / 2 + 1, win_length, 2}), input->type(), &exp_complex)); + auto exp_complex_begin = exp_complex->begin(); + std::vector exp_complex_slice = {win_length * 2, 2}; + for (int i = 0; i < (n_fft / TWO + 1); i++) { + for (int k = 0; k <= win_length - 1; k++) { + ptrdiff_t exp_complex_offset_0 = i * exp_complex_slice[0] + k * exp_complex_slice[1]; + ptrdiff_t exp_complex_offset_1 = exp_complex_offset_0 + 1; + *(exp_complex_begin + exp_complex_offset_0) = std::cos(twice * PI * i * k / win_length); + *(exp_complex_begin + exp_complex_offset_1) = std::sin(twice * PI * i * k / win_length); + } + } + for (int r = 0; r < input->shape()[0]; r++) { + for (int i = 0; i < (n_fft / TWO + 1); i++) { + for (int j = 0; j < n_columns; j++) { + T spec_f_0 = 0.; + T spec_f_1 = 0.; + ptrdiff_t exp_complex_offset_0 = i * exp_complex_slice[0]; + for (int k = 0; k < win_length; k++) { + ptrdiff_t exp_complex_offset_1 = exp_complex_offset_0 + 1; + T exp_complex_a = *(exp_complex_begin + exp_complex_offset_0); + T exp_complex_b = *(exp_complex_begin + exp_complex_offset_1); + ptrdiff_t input_win_offset = r * input_win_slice[0] + j * input_win_slice[1] + k; + T input_value = *(input_win_begin + input_win_offset); + spec_f_0 += input_value * exp_complex_a; + spec_f_1 += -input_value * exp_complex_b; + exp_complex_offset_0 = exp_complex_offset_1 + 1; + } + ptrdiff_t spec_f_offset_0 = r * spec_f_slice[0] + i * spec_f_slice[1] + j * spec_f_slice[2]; + ptrdiff_t spec_f_offset_1 = spec_f_offset_0 + 1; + *(spec_f_begin + spec_f_offset_0) = spec_f_0; + *(spec_f_begin + spec_f_offset_1) = spec_f_1; + } + } + } + CHECK_FAIL_RETURN_UNEXPECTED(win_sum != 0, "Window: the total value of window function can not be zero."); + if (normalized) { + for (int r = 0; r < input->shape()[0]; r++) { + for (int i = 0; i < (n_fft / TWO + 1); i++) { + for (int j = 0; j < n_columns; j++) { + ptrdiff_t spec_f_offset_0 = r * spec_f_slice[0] + i * spec_f_slice[1] + j * spec_f_slice[2]; + ptrdiff_t spec_f_offset_1 = spec_f_offset_0 + 1; + T spec_norm_a = *(spec_f_begin + spec_f_offset_0); + T spec_norm_b = *(spec_f_begin + spec_f_offset_1); + *(spec_f_begin + spec_f_offset_0) = spec_norm_a / win_sum; + *(spec_f_begin + spec_f_offset_1) = spec_norm_b / win_sum; + } + } + } + } + std::shared_ptr output_onsided; + if (!onesided) { + RETURN_IF_NOT_OK(Onesided(spec_f, &output_onsided, n_fft, n_columns)); + if (power == 0) { + *output = output_onsided; + return Status::OK(); + } + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({input->shape()[0], n_fft, n_columns}), input->type(), &spec_p)); + RETURN_IF_NOT_OK(PowerStft(output_onsided, &spec_p, power, n_fft, n_columns, n_fft)); + *output = spec_p; + return Status::OK(); + } + if (power == 0) { + *output = spec_f; + return Status::OK(); + } + RETURN_IF_NOT_OK(PowerStft(spec_f, &spec_p, power, n_fft, n_columns, n_fft / TWO + 1)); + *output = spec_p; + return Status::OK(); +} + +template +Status SpectrogramImpl(const std::shared_ptr &input, std::shared_ptr *output, int pad, + WindowType window, int n_fft, int hop_length, int win_length, float power, bool normalized, + bool center, BorderType pad_mode, bool onesided) { + std::shared_ptr fft_window_tensor; + std::shared_ptr fft_window_later; + TensorShape shape = input->shape(); + std::vector output_shape = shape.AsVector(); + output_shape.pop_back(); + int input_len = input->shape()[-1]; + + RETURN_IF_NOT_OK(input->Reshape(TensorShape({input->Size() / input_len, input_len}))); + + DataType data_type = input->type(); + // get the windows + RETURN_IF_NOT_OK(Window(&fft_window_tensor, window, win_length)); + if (win_length == 1) { + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({1}), DataType(DataType::DE_FLOAT32), &fft_window_tensor)); + auto win = fft_window_tensor->begin(); + *(win) = 1; + } + + // Pad window length + int pad_left = (n_fft - win_length) / 2; + int pad_right = n_fft - win_length - pad_left; + RETURN_IF_NOT_OK(fft_window_tensor->Reshape(TensorShape({1, win_length}))); + RETURN_IF_NOT_OK(Pad(fft_window_tensor, &fft_window_later, pad_left, pad_right, BorderType::kConstant)); + RETURN_IF_NOT_OK(fft_window_later->Reshape(TensorShape({n_fft}))); + + int length = input_len + pad * 2 + n_fft; + + std::shared_ptr input_data_tensor; + std::shared_ptr input_data_tensor_pad; + RETURN_IF_NOT_OK( + Tensor::CreateEmpty(TensorShape({input->shape()[0], input_len + pad * 2}), data_type, &input_data_tensor_pad)); + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({input->shape()[0], length}), data_type, &input_data_tensor)); + + RETURN_IF_NOT_OK(Pad(input, &input_data_tensor_pad, pad, pad, BorderType::kConstant)); + + if (center) { + RETURN_IF_NOT_OK(Pad(input_data_tensor_pad, &input_data_tensor, n_fft / TWO, n_fft / TWO, pad_mode)); + } else { + input_data_tensor = input_data_tensor_pad; + } + + CHECK_FAIL_RETURN_UNEXPECTED(n_fft <= input_data_tensor->shape()[-1], + "Spectrogram: n_fft should be more than 0 and less than " + + std::to_string(input_data_tensor->shape()[-1]) + + ", but got n_fft: " + std::to_string(n_fft) + "."); + + // calculate the sliding times of the window function + int n_columns = 0; + while ((1 + n_columns++) * hop_length + n_fft <= input_data_tensor->shape()[-1]) { + } + std::shared_ptr stft_compute; + + auto input_begin = input_data_tensor->begin(); + std::vector input_win_slice = {n_columns * n_fft, n_fft}; + auto iter_win = fft_window_later->begin(); + std::shared_ptr input_win; + RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({input_data_tensor->shape()[0], n_columns, n_fft}), + input_data_tensor->type(), &input_win)); + auto input_win_begin = input_win->begin(); + for (int r = 0; r < input_data_tensor->shape()[0]; r++) { + for (int j = 0; j < n_columns; j++) { + for (int k = 0; k < n_fft; k++) { + ptrdiff_t win_offset = k; + float win_value = *(iter_win + win_offset); + ptrdiff_t input_stft_offset = r * input_data_tensor->shape()[-1] + j * hop_length + k; + T input_value = *(input_begin + input_stft_offset); + ptrdiff_t input_win_offset = r * input_win_slice[0] + j * input_win_slice[1] + k; + *(input_win_begin + input_win_offset) = win_value * input_value; + } + } + } + RETURN_IF_NOT_OK(Stft(input_win, &stft_compute, n_fft, fft_window_later, n_fft, hop_length, n_columns, normalized, + power, onesided)); + if (onesided) { + output_shape.push_back(n_fft / TWO + 1); + } else { + output_shape.push_back(n_fft); + } + output_shape.push_back(n_columns); + if (power == 0) { + output_shape.push_back(TWO); + } + // reshape the output + RETURN_IF_NOT_OK(stft_compute->Reshape(TensorShape({output_shape}))); + *output = stft_compute; + return Status::OK(); +} + +Status Spectrogram(const std::shared_ptr &input, std::shared_ptr *output, int pad, WindowType window, + int n_fft, int hop_length, int win_length, float power, bool normalized, bool center, + BorderType pad_mode, bool onesided) { + TensorShape input_shape = input->shape(); + + CHECK_FAIL_RETURN_UNEXPECTED( + input->type().IsNumeric(), + "Spectrogram: input tensor type should be int, float or double, but got: " + input->type().ToString()); + + CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "Spectrogram: input tensor is not in shape of <..., time>."); + + std::shared_ptr input_tensor; + if (input->type() != DataType::DE_FLOAT64) { + RETURN_IF_NOT_OK(TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32))); + return SpectrogramImpl(input_tensor, output, pad, window, n_fft, hop_length, win_length, power, normalized, + center, pad_mode, onesided); + } else { + input_tensor = input; + return SpectrogramImpl(input_tensor, output, pad, window, n_fft, hop_length, win_length, power, normalized, + center, pad_mode, onesided); + } +} + Status ComputeDeltas(const std::shared_ptr &input, std::shared_ptr *output, int32_t win_length, const BorderType &mode) { RETURN_IF_NOT_OK(ValidateLowRank("ComputeDeltas", input, kDefaultAudioDim, "<..., freq, time>")); diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.h b/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.h index b76c9c72364..a1fbaf089ca 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.h +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.h @@ -34,6 +34,7 @@ constexpr double PI = 3.141592653589793; constexpr int kMinAudioDim = 1; constexpr int kDefaultAudioDim = 2; +constexpr int TWO = 2; namespace mindspore { namespace dataset { @@ -147,7 +148,7 @@ Status Contrast(const std::shared_ptr &input, std::shared_ptr *o for (auto itr_in = input->begin(); itr_in != input->end(); itr_in++) { T temp1, temp2 = 0; // PI / 2 is half of the constant PI - temp1 = static_cast(*itr_in) * (PI / 2); + temp1 = static_cast(*itr_in) * (PI / TWO); temp2 = enhancement_amount_value * std::sin(temp1 * 4); *itr_out = std::sin(temp1 + temp2); itr_out++; @@ -295,6 +296,23 @@ Status LFilter(const std::shared_ptr &input, std::shared_ptr *ou return Status::OK(); } +/// \brief Transform audio signal into spectrogram. +/// \param[in] n_fft Size of FFT, creates n_fft / 2 + 1 bins. +/// \param[in] win_length Window size. +/// \param[in] hop_length Length of hop between STFT windows. +/// \param[in] pad Two sided padding of signal. +/// \param[in] window A function to create a window tensor +/// that is applied/multiplied to each frame/window. +/// \param[in] power Exponent for the magnitude spectrogram. +/// \param[in] normalized Whether to normalize by magnitude after stft. +/// \param[in] center Whether to pad waveform on both sides. +/// \param[in] pad_mode Controls the padding method used when center is true. +/// \param[in] onesided Controls whether to return half of results to avoid redundancy. +/// \return Status code. +Status Spectrogram(const std::shared_ptr &input, std::shared_ptr *output, int pad, WindowType window, + int n_fft, int hop_length, int win_length, float power, bool normalized, bool center, + BorderType pad_mode, bool onesided); + /// \brief Stretch STFT in time at a given rate, without changing the pitch. /// \param input: Tensor of shape <..., freq, time>. /// \param rate: Stretch factor. diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/spectrogram_op.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/spectrogram_op.cc new file mode 100644 index 00000000000..05d55124b7a --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/spectrogram_op.cc @@ -0,0 +1,54 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "minddata/dataset/audio/kernels/spectrogram_op.h" + +#include "minddata/dataset/audio/kernels/audio_utils.h" +#include "minddata/dataset/util/status.h" + +namespace mindspore { +namespace dataset { +Status SpectrogramOp::Compute(const std::shared_ptr &input, std::shared_ptr *output) { + IO_CHECK(input, output); + return Spectrogram(input, output, pad_, window_, n_fft_, hop_length_, win_length_, power_, normalized_, center_, + pad_mode_, onesided_); +} + +Status SpectrogramOp::OutputShape(const std::vector &inputs, std::vector &outputs) { + RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs)); + outputs.clear(); + constexpr int two = 2; + int length_ = inputs[0][-1] + pad_ * 2 + n_fft_; + int n_columns = 0; + while ((1 + n_columns++) * hop_length_ + n_fft_ <= length_) { + } + auto vec = inputs[0].AsVector(); + vec.pop_back(); + if (onesided_) { + vec.push_back(n_fft_ / two + 1); + } else { + vec.push_back(n_fft_); + } + vec.push_back(n_columns); + if (power_ == 0) { + vec.push_back(two); + } + outputs.emplace_back(TensorShape(vec)); + + if (!outputs.empty()) return Status::OK(); + return Status(StatusCode::kMDUnexpectedError, "Spectrogram: input tensor is not in shape of <..., time>."); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/spectrogram_op.h b/mindspore/ccsrc/minddata/dataset/audio/kernels/spectrogram_op.h new file mode 100644 index 00000000000..31041154361 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/spectrogram_op.h @@ -0,0 +1,66 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_KERNELS_SPECTROGRAM_OP_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_KERNELS_SPECTROGRAM_OP_H_ + +#include +#include +#include + +#include "minddata/dataset/core/tensor.h" +#include "minddata/dataset/kernels/tensor_op.h" + +namespace mindspore { +namespace dataset { +class SpectrogramOp : public TensorOp { + public: + SpectrogramOp(int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad, WindowType window, float power, + bool normalized, bool center, BorderType pad_mode, bool onesided) + : n_fft_(n_fft), + win_length_(win_length), + hop_length_(hop_length), + pad_(pad), + window_(window), + power_(power), + normalized_(normalized), + center_(center), + pad_mode_(pad_mode), + onesided_(onesided) {} + + ~SpectrogramOp() = default; + + Status Compute(const std::shared_ptr &input, std::shared_ptr *output) override; + + std::string Name() const override { return kSpectrogramOp; }; + + Status OutputShape(const std::vector &inputs, std::vector &outputs) override; + + private: + int32_t n_fft_; + int32_t win_length_; + int32_t hop_length_; + int32_t pad_; + WindowType window_; + float power_; + bool normalized_; + bool center_; + BorderType pad_mode_; + bool onesided_; +}; +} // namespace dataset +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_KERNELS_SPECTROGRAM_OP_H_ diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/audio.h b/mindspore/ccsrc/minddata/dataset/include/dataset/audio.h index 80c26f470c6..c31b3bc3d18 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/audio.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/audio.h @@ -712,6 +712,49 @@ class MS_API SlidingWindowCmn final : public TensorTransform { std::shared_ptr data_; }; +/// \brief Create a spectrogram from an audio signal. +class MS_API Spectrogram : public TensorTransform { + public: + /// \brief Constructor. + /// \param[in] n_fft Size of FFT, creates n_fft / 2 + 1 bins (Default: 400). + /// \param[in] win_length Window size (Default: 0, will use n_fft). + /// \param[in] hop_length Length of hop between STFT windows (Default: 0, will use win_length / 2). + /// \param[in] pad Two sided padding of signal (Default: 0). + /// \param[in] window Window function that is applied/multiplied to each frame/window, + /// which can be WindowType::kBartlett, WindowType::kBlackman, WindowType::kHamming, + /// WindowType::kHann or WindowType::kKaiser (Default: WindowType::kHann). + /// \param[in] power Exponent for the magnitude spectrogram, which must be greater than or equal to 0 (Default: 2.0). + /// \param[in] normalized Whether to normalize by magnitude after stft (Default: false). + /// \param[in] center Whether to pad waveform on both sides (Default: true). + /// \param[in] pad_mode Controls the padding method used when center is true (Default: BorderType::kReflect). + /// \param[in] onesided Controls whether to return half of results to avoid redundancy (Default: true). + Spectrogram(int32_t n_fft = 400, int32_t win_length = 0, int32_t hop_length = 0, int32_t pad = 0, + WindowType window = WindowType::kHann, float power = 2.0, bool normalized = false, bool center = true, + BorderType pad_mode = BorderType::kReflect, bool onesided = true); + + /// \brief Destructor. + ~Spectrogram() = default; + + protected: + /// \brief Function to convert TensorTransform object into a TensorOperation object. + /// \return Shared pointer to TensorOperation object. + std::shared_ptr Parse() override; + + private: + int32_t n_fft_; + int32_t win_length_; + int32_t hop_length_; + int32_t pad_; + WindowType window_; + float power_; + bool normalized_; + bool center_; + BorderType pad_mode_; + bool onesided_; + struct Data; + std::shared_ptr data_; +}; + /// \brief TimeMasking TensorTransform. /// \notes Apply masking to a spectrogram in the time domain. class MS_API TimeMasking final : public TensorTransform { diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h b/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h index 736bae99a4c..9dc1a3b3de8 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/constants.h @@ -133,6 +133,15 @@ enum class MS_API FixRotationAngle { k270Degree = 8, ///< Rotate 270 degree. }; +/// \brief Possible types for windows function. +enum class MS_API WindowType { + kBartlett = 0, ///< Bartlett window function. + kBlackman = 1, ///< Blackman window function. + kHamming = 2, ///< Hamming window function. + kHann = 3, ///< Hann window function. + kKaiser = 4 ///< Kaiser window function. +}; + /// \brief Possible options for Image format types in a batch. enum class MS_API ImageBatchFormat { kNHWC = 0, ///< Indicate the input batch is of NHWC format. diff --git a/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h b/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h index 15ad88c4e49..526029bb159 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h @@ -175,6 +175,7 @@ constexpr char kOverdriveOp[] = "OverdriveOp"; constexpr char kPhaserOp[] = "PhaserOp"; constexpr char kRiaaBiquadOp[] = "RiaaBiquadOp"; constexpr char kSlidingWindowCmnOp[] = "SlidingWindowCmnOp"; +constexpr char kSpectrogramOp[] = "SpectrogramOp"; constexpr char kTimeMaskingOp[] = "TimeMaskingOp"; constexpr char kTimeStretchOp[] = "TimeStretchOp"; constexpr char kTrebleBiquadOp[] = "TrebleBiquadOp"; diff --git a/mindspore/dataset/audio/transforms.py b/mindspore/dataset/audio/transforms.py index 639f59e3578..318c25f6616 100644 --- a/mindspore/dataset/audio/transforms.py +++ b/mindspore/dataset/audio/transforms.py @@ -23,13 +23,13 @@ import numpy as np import mindspore._c_dataengine as cde from ..transforms.c_transforms import TensorOperation -from .utils import BorderType, FadeShape, GainType, Interpolation, Modulation, ScaleType +from .utils import BorderType, FadeShape, GainType, Interpolation, Modulation, ScaleType, WindowType from .validators import check_allpass_biquad, check_amplitude_to_db, check_band_biquad, check_bandpass_biquad, \ check_bandreject_biquad, check_bass_biquad, check_biquad, check_complex_norm, check_compute_deltas, \ check_contrast, check_db_to_amplitude, check_dc_shift, check_deemph_biquad, check_detect_pitch_frequency, \ check_equalizer_biquad, check_fade, check_flanger, check_highpass_biquad, check_lfilter, check_lowpass_biquad, \ check_magphase, check_masking, check_mu_law_coding, check_overdrive, check_phaser, check_riaa_biquad, \ - check_sliding_window_cmn, check_time_stretch, check_treble_biquad, check_vol + check_sliding_window_cmn, check_spectrogram, check_time_stretch, check_treble_biquad, check_vol class AudioTensorOperation(TensorOperation): @@ -947,6 +947,61 @@ class SlidingWindowCmn(AudioTensorOperation): return cde.SlidingWindowCmnOperation(self.cmn_window, self.min_cmn_window, self.center, self.norm_vars) +DE_C_WINDOW_TYPE = {WindowType.BARTLETT: cde.WindowType.DE_BARTLETT, + WindowType.BLACKMAN: cde.WindowType.DE_BLACKMAN, + WindowType.HAMMING: cde.WindowType.DE_HAMMING, + WindowType.HANN: cde.WindowType.DE_HANN, + WindowType.KAISER: cde.WindowType.DE_KAISER} + + +class Spectrogram(TensorOperation): + """ + Create a spectrogram from an audio signal. + + Args: + n_fft (int, optional): Size of FFT, creates n_fft // 2 + 1 bins (default=400). + win_length (int, optional): Window size (default=None, will use n_fft). + hop_length (int, optional): Length of hop between STFT windows (default=None, will use win_length // 2). + pad (int): Two sided padding of signal (default=0). + window (WindowType, optional): Window function that is applied/multiplied to each frame/window, + which can be WindowType.BARTLETT, WindowType.BLACKMAN, WindowType.HAMMING, WindowType.HANN + or WindowType.KAISER (default=WindowType.HANN). + power (float, optional): Exponent for the magnitude spectrogram, which must be greater + than or equal to 0, e.g., 1 for energy, 2 for power, etc. (default=2.0). + normalized (bool, optional): Whether to normalize by magnitude after stft (default=False). + center (bool, optional): Whether to pad waveform on both sides (default=True). + pad_mode (BorderType, optional): Controls the padding method used when center is True, + which can be BorderType.REFLECT, BorderType.CONSTANT, BorderType.EDGE, BorderType.SYMMETRIC + (default=BorderType.REFLECT). + onesided (bool, optional): Controls whether to return half of results to avoid redundancy (default=True). + + Examples: + >>> waveform = np.random.random([5, 10, 20]) + >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) + >>> transforms = [audio.Spectrogram()] + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) + """ + + @check_spectrogram + def __init__(self, n_fft=400, win_length=None, hop_length=None, pad=0, window=WindowType.HANN, power=2.0, + normalized=False, center=True, pad_mode=BorderType.REFLECT, onesided=True): + self.n_fft = n_fft + self.win_length = win_length if win_length else n_fft + self.hop_length = hop_length if hop_length else self.win_length // 2 + self.pad = pad + self.window = window + self.power = power + self.normalized = normalized + self.center = center + self.pad_mode = pad_mode + self.onesided = onesided + + def parse(self): + return cde.SpectrogramOperation(self.n_fft, self.win_length, self.hop_length, self.pad, + DE_C_WINDOW_TYPE[self.window], self.power, self.normalized, + self.center, DE_C_BORDER_TYPE[self.pad_mode], self.onesided) + + class TimeMasking(AudioTensorOperation): """ Apply masking to a spectrogram in the time domain. diff --git a/mindspore/dataset/audio/utils.py b/mindspore/dataset/audio/utils.py index 083a933573d..41016a753db 100644 --- a/mindspore/dataset/audio/utils.py +++ b/mindspore/dataset/audio/utils.py @@ -160,3 +160,23 @@ class BorderType(str, Enum): EDGE: str = "edge" REFLECT: str = "reflect" SYMMETRIC: str = "symmetric" + + +class WindowType(str, Enum): + """ + Window Function types, + + Possible enumeration values are: WindowType.BARTLETT, WindowType.BLACKMAN, WindowType.HAMMING, WindowType.HANN, + WindowType.KAISER. + + - WindowType.BARTLETT: means the type of window function is bartlett. + - WindowType.BLACKMAN: means the type of window function is blackman. + - WindowType.HAMMING: means the type of window function is hamming. + - WindowType.HANN: means the type of window function is hann. + - WindowType.KAISER: means the type of window function is kaiser. + """ + BARTLETT: str = "bartlett" + BLACKMAN: str = "blackman" + HAMMING: str = "hamming" + HANN: str = "hann" + KAISER: str = "kaiser" diff --git a/mindspore/dataset/audio/validators.py b/mindspore/dataset/audio/validators.py index 0773a0aa352..7180862d8b3 100644 --- a/mindspore/dataset/audio/validators.py +++ b/mindspore/dataset/audio/validators.py @@ -21,7 +21,7 @@ from functools import wraps from mindspore.dataset.core.validator_helpers import check_float32, check_float32_not_zero, check_int32, \ check_int32_not_zero, check_list_same_size, check_non_negative_float32, check_non_negative_int32, \ check_pos_float32, check_pos_int32, check_value, INT32_MAX, parse_user_args, type_check -from .utils import BorderType, FadeShape, GainType, Interpolation, Modulation, ScaleType +from .utils import BorderType, FadeShape, GainType, Interpolation, Modulation, ScaleType, WindowType def check_amplitude_to_db(method): @@ -356,6 +356,40 @@ def check_riaa_biquad(method): return new_method +def check_spectrogram(method): + """Wrapper method to check the parameters of Spectrogram.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + [n_fft, win_length, hop_length, pad, window, power, + normalized, center, pad_mode, onesided], _ = parse_user_args(method, *args, **kwargs) + type_check(n_fft, (int,), "n_fft") + check_pos_int32(n_fft, "n_fft") + if win_length is not None: + type_check(win_length, (int,), "win_length") + check_pos_int32(win_length, "win_length") + if win_length > n_fft: + raise ValueError( + "Input win_length should be no more than n_fft, but got win_length: {0} and n_fft: {1}.".format( + win_length, n_fft)) + if hop_length is not None: + type_check(hop_length, (int,), "hop_length") + check_pos_int32(hop_length, "hop_length") + + type_check(pad, (int,), "pad") + check_non_negative_int32(pad, "pad") + type_check(window, (WindowType,), "window") + type_check(power, (int, float), "power") + check_non_negative_float32(power, "power") + type_check(normalized, (bool,), "normalized") + type_check(center, (bool,), "center") + type_check(onesided, (bool,), "onesided") + type_check(pad_mode, (BorderType,), "pad_mode") + return method(self, *args, **kwargs) + + return new_method + + def check_time_stretch(method): """Wrapper method to check the parameters of TimeStretch.""" diff --git a/tests/ut/cpp/dataset/CMakeLists.txt b/tests/ut/cpp/dataset/CMakeLists.txt index 341573ceba9..e8122e96f29 100644 --- a/tests/ut/cpp/dataset/CMakeLists.txt +++ b/tests/ut/cpp/dataset/CMakeLists.txt @@ -87,6 +87,7 @@ SET(DE_UT_SRCS decode_op_test.cc distributed_sampler_test.cc equalize_op_test.cc + execute_test.cc execution_tree_test.cc fill_op_test.cc c_api_vision_gaussian_blur_test.cc diff --git a/tests/ut/cpp/dataset/c_api_audio_r_to_z_test.cc b/tests/ut/cpp/dataset/c_api_audio_r_to_z_test.cc index 0b6c1085b5a..af91c7d0521 100644 --- a/tests/ut/cpp/dataset/c_api_audio_r_to_z_test.cc +++ b/tests/ut/cpp/dataset/c_api_audio_r_to_z_test.cc @@ -255,6 +255,437 @@ TEST_F(MindDataTestPipeline, TestSlidingWindowCmnWrongArgs) { EXPECT_EQ(iter_2, nullptr); } +/// Feature: Spectrogram. +/// Description: test pipeline. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramDefault) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramDefault."; + + std::shared_ptr schema = Schema(); + ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {1, 60})); + std::shared_ptr ds = RandomData(8, schema); + EXPECT_NE(ds, nullptr); + + auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHann, 2.0, false, true, BorderType::kReflect, + true}); + + auto ds1 = ds->Map({spectrogram}, {"waveform"}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + while (row.size() != 0) { + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + EXPECT_EQ(i, 8); + iter->Stop(); +} + +/// Feature: Spectrogram. +/// Description: test parameter: onesided. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramOnesidedFalse) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramOnesidedFalse."; + + std::shared_ptr schema = Schema(); + ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {3, 50})); + std::shared_ptr ds = RandomData(8, schema); + EXPECT_NE(ds, nullptr); + + auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHann, 2.0, false, true, BorderType::kReflect, + false}); + + auto ds1 = ds->Map({spectrogram}, {"waveform"}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + while (row.size() != 0) { + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + EXPECT_EQ(i, 8); + iter->Stop(); +} + +/// Feature: Spectrogram. +/// Description: test parameter: center. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramCenterFalse) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramCenterFalse."; + + std::shared_ptr schema = Schema(); + ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeInt32, {2, 3, 50})); + std::shared_ptr ds = RandomData(8, schema); + EXPECT_NE(ds, nullptr); + + auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHann, 2.0, false, false, BorderType::kReflect, + true}); + + auto ds1 = ds->Map({spectrogram}, {"waveform"}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + while (row.size() != 0) { + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + EXPECT_EQ(i, 8); + iter->Stop(); +} + +/// Feature: Spectrogram. +/// Description: test parameter: normaliezd. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramNormalizedTrue) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramNormalizedTrue."; + + std::shared_ptr schema = Schema(); + ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeInt32, {5, 40})); + std::shared_ptr ds = RandomData(8, schema); + EXPECT_NE(ds, nullptr); + + auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHann, 2.0, true, true, BorderType::kReflect, + true}); + + auto ds1 = ds->Map({spectrogram}, {"waveform"}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + while (row.size() != 0) { + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + EXPECT_EQ(i, 8); + iter->Stop(); +} + +/// Feature: Spectrogram. +/// Description: test parameter: window. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramWindowHamming) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramWindowHamming."; + + std::shared_ptr schema = Schema(); + ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat64, {3, 50})); + std::shared_ptr ds = RandomData(8, schema); + EXPECT_NE(ds, nullptr); + + auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHamming, 2.0, false, true, + BorderType::kReflect, true}); + + auto ds1 = ds->Map({spectrogram}, {"waveform"}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + while (row.size() != 0) { + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + EXPECT_EQ(i, 8); + iter->Stop(); +} + +/// Feature: Spectrogram. +/// Description: test parameter: pad_mode. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramPadmodeEdge) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramPadmodeEdge."; + + std::shared_ptr schema = Schema(); + ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeInt32, {3, 4, 50})); + std::shared_ptr ds = RandomData(8, schema); + EXPECT_NE(ds, nullptr); + + auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHamming, 2.0, false, true, + BorderType::kEdge, true}); + + auto ds1 = ds->Map({spectrogram}, {"waveform"}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + while (row.size() != 0) { + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + EXPECT_EQ(i, 8); + iter->Stop(); +} + +/// Feature: Spectrogram. +/// Description: test parameter: power. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramPower0) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramPower0."; + + std::shared_ptr schema = Schema(); + ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeInt32, {3, 50})); + std::shared_ptr ds = RandomData(8, schema); + EXPECT_NE(ds, nullptr); + + auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHamming, 0, false, true, BorderType::kReflect, + true}); + + auto ds1 = ds->Map({spectrogram}, {"waveform"}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + while (row.size() != 0) { + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + EXPECT_EQ(i, 8); + iter->Stop(); +} + +/// Feature: Spectrogram. +/// Description: test parameter: n_fft. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramNfft50) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramNfft600."; + + std::shared_ptr schema = Schema(); + ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {1, 60})); + std::shared_ptr ds = RandomData(8, schema); + EXPECT_NE(ds, nullptr); + + auto spectrogram = audio::Spectrogram({50, 40, 20, 0, WindowType::kHann, 2.0, false, true, BorderType::kReflect, + true}); + + auto ds1 = ds->Map({spectrogram}, {"waveform"}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + while (row.size() != 0) { + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + EXPECT_EQ(i, 8); + iter->Stop(); +} + +/// Feature: Spectrogram. +/// Description: test parameter: pad. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramPad10) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramPad50."; + + std::shared_ptr schema = Schema(); + ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {3, 50})); + std::shared_ptr ds = RandomData(8, schema); + EXPECT_NE(ds, nullptr); + + auto spectrogram = audio::Spectrogram({40, 40, 20, 10, WindowType::kHann, 2.0, false, true, BorderType::kReflect, + true}); + + auto ds1 = ds->Map({spectrogram}, {"waveform"}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + while (row.size() != 0) { + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + EXPECT_EQ(i, 8); + iter->Stop(); +} + +/// Feature: Spectrogram. +/// Description: test parameter: win_length. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramWinlength30) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramWinlength300."; + + std::shared_ptr schema = Schema(); + ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {2, 2, 50})); + std::shared_ptr ds = RandomData(8, schema); + EXPECT_NE(ds, nullptr); + + auto spectrogram = audio::Spectrogram({40, 30, 20, 0, WindowType::kHann, 2.0, false, true, BorderType::kReflect, + true}); + + auto ds1 = ds->Map({spectrogram}, {"waveform"}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + while (row.size() != 0) { + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + EXPECT_EQ(i, 8); + iter->Stop(); +} + +/// Feature: Spectrogram. +/// Description: test parameters. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramHoplength30) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramHoplength300."; + + std::shared_ptr schema = Schema(); + ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {2, 50})); + std::shared_ptr ds = RandomData(8, schema); + EXPECT_NE(ds, nullptr); + + auto spectrogram = audio::Spectrogram({40, 40, 30, 0, WindowType::kHann, 2.0, false, true, BorderType::kReflect, + true}); + + auto ds1 = ds->Map({spectrogram}, {"waveform"}); + EXPECT_NE(ds1, nullptr); + + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + uint64_t i = 0; + while (row.size() != 0) { + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + EXPECT_EQ(i, 8); + iter->Stop(); +} + +/// Feature: Spectrogram. +/// Description: test some invalid parameters. +/// Expectation: success. +TEST_F(MindDataTestPipeline, TestSpectrogramWrongArgs) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramWrongArgs."; + + std::shared_ptr schema = Schema(); + // Original waveform + ASSERT_OK(schema->add_column("col", mindspore::DataType::kNumberTypeFloat32, {1, 50})); + std::shared_ptr ds = RandomData(50, schema); + std::shared_ptr ds01; + std::shared_ptr ds02; + std::shared_ptr ds03; + std::shared_ptr ds04; + std::shared_ptr ds05; + std::shared_ptr ds06; + EXPECT_NE(ds, nullptr); + + // Check n_fft + MS_LOG(INFO) << "n_fft is zero."; + auto spectrogram_op_01 = audio::Spectrogram({0, 40, 20, 0, WindowType::kHann, 2.0, false, true, + BorderType::kReflect, true}); + ds01 = ds->Map({spectrogram_op_01}); + EXPECT_NE(ds01, nullptr); + + std::shared_ptr iter01 = ds01->CreateIterator(); + EXPECT_EQ(iter01, nullptr); + + // Check win_length + MS_LOG(INFO) << "win_length is -1."; + auto spectrogram_op_02 = audio::Spectrogram({40, -1, 20, 0, WindowType::kHann, 2.0, false, true, + BorderType::kReflect, true}); + ds02 = ds->Map({spectrogram_op_02}); + EXPECT_NE(ds02, nullptr); + + std::shared_ptr iter02 = ds02->CreateIterator(); + EXPECT_EQ(iter02, nullptr); + + // Check hop_length + MS_LOG(INFO) << "hop_length is -1."; + auto spectrogram_op_03 = audio::Spectrogram({40, 40, -1, 0, WindowType::kHann, 2.0, false, true, + BorderType::kReflect, true}); + ds03 = ds->Map({spectrogram_op_03}); + EXPECT_NE(ds03, nullptr); + + std::shared_ptr iter03 = ds03->CreateIterator(); + EXPECT_EQ(iter03, nullptr); + + // Check power + MS_LOG(INFO) << "power is -1."; + auto spectrogram_op_04 = audio::Spectrogram({40, 40, 20, 0, WindowType::kHann, -1, false, true, + BorderType::kReflect, true}); + ds04 = ds->Map({spectrogram_op_04}); + EXPECT_NE(ds04, nullptr); + + std::shared_ptr iter04 = ds04->CreateIterator(); + EXPECT_EQ(iter04, nullptr); + + // Check pad + MS_LOG(INFO) << "pad is -1."; + auto spectrogram_op_05 = audio::Spectrogram({40, 40, 20, -1, WindowType::kHann, 2.0, false, true, + BorderType::kReflect, true}); + ds05 = ds->Map({spectrogram_op_05}); + EXPECT_NE(ds05, nullptr); + + std::shared_ptr iter05 = ds05->CreateIterator(); + EXPECT_EQ(iter05, nullptr); + + // Check n_fft and win)length + MS_LOG(INFO) << "n_fft is 40, win_length is 50."; + auto spectrogram_op_06 = audio::Spectrogram({40, 50, 20, -1, WindowType::kHann, 2.0, false, true, + BorderType::kReflect, true}); + ds06 = ds->Map({spectrogram_op_06}); + EXPECT_NE(ds06, nullptr); + + std::shared_ptr iter06 = ds06->CreateIterator(); + EXPECT_EQ(iter06, nullptr); +} + TEST_F(MindDataTestPipeline, TestTimeMaskingPipeline) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTimeMaskingPipeline."; // Original waveform diff --git a/tests/ut/cpp/dataset/execute_test.cc b/tests/ut/cpp/dataset/execute_test.cc index 0e3610afc82..9908b65c550 100644 --- a/tests/ut/cpp/dataset/execute_test.cc +++ b/tests/ut/cpp/dataset/execute_test.cc @@ -2175,3 +2175,20 @@ TEST_F(MindDataTestExecute, TestAutoAugmentEager) { Status rc = transform(image, &image); EXPECT_EQ(rc, Status::OK()); } + +/// Feature: Spectrogram. +/// Description: test Spectrogram in eager mode. +/// Expectation: the data is processed successfully. +TEST_F(MindDataTestExecute, TestSpectrogramEager) { + MS_LOG(INFO) << "Doing MindDataTestExecute-SpectrogramEager."; + std::shared_ptr test_input_tensor; + std::vector waveform = {1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1}; + ASSERT_OK(Tensor::CreateFromVector(waveform, TensorShape({1, (long)waveform.size()}), &test_input_tensor)); + auto input_tensor = mindspore::MSTensor(std::make_shared(test_input_tensor)); + std::shared_ptr spectrogram = std::make_shared(8, 8, 4, 0, WindowType::kHann, + 2., false, true, + BorderType::kReflect, true); + auto transform = Execute({spectrogram}); + Status rc = transform({input_tensor}, &input_tensor); + ASSERT_TRUE(rc.IsOk()); +} diff --git a/tests/ut/python/dataset/test_spectrogram.py b/tests/ut/python/dataset/test_spectrogram.py new file mode 100644 index 00000000000..f907a25b176 --- /dev/null +++ b/tests/ut/python/dataset/test_spectrogram.py @@ -0,0 +1,432 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing Spectrogram Python API +""" +import numpy as np + +import mindspore.dataset as ds +import mindspore.dataset.audio.transforms as audio +from mindspore import log as logger +from mindspore.dataset.audio.utils import WindowType, BorderType + + +def count_unequal_element(data_expected, data_me, rtol, atol): + """ Precision calculation func """ + assert data_expected.shape == data_me.shape + total_count = len(data_expected.flatten()) + error = np.abs(data_expected - data_me) + greater = np.greater(error, atol + np.abs(data_expected) * rtol) + loss_count = np.count_nonzero(greater) + assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format( + data_expected[greater], data_me[greater], error[greater]) + + +def test_spectrogram_pipeline(): + """ + Feature: mindspore pipeline mode normal testcase: spectrogram op. + Description: input audio signal to test pipeline. + Expectation: success. + """ + logger.info("test_spectrogram_pipeline") + + wav = [[[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]]] + dataset = ds.NumpySlicesDataset(wav, column_names=["audio"], shuffle=False) + out = audio.Spectrogram(n_fft=8) + dataset = dataset.map(operations=out, input_columns=["audio"], output_columns=["Spectrogram"], + column_order=['Spectrogram']) + result = np.array([[[2.8015e+01, 1.2100e+02, 3.1354e+02, 1.6900e+02, 2.5000e+01, + 1.0843e+01, 1.2100e+02, 3.3150e+02], + [3.2145e+00, 3.3914e+01, 9.4728e+01, 4.5914e+01, 9.9142e+00, + 4.5858e+00, 3.3914e+01, 9.5685e+01], + [1.0000e+00, 1.7157e-01, 1.5000e+00, 1.7157e-01, 1.7157e-01, + 5.0000e-01, 1.7157e-01, 7.5000e-01], + [4.2893e-02, 2.5736e-01, 5.8579e-01, 2.5736e-01, 2.5736e-01, + 5.8579e-01, 2.5736e-01, 1.2868e-01], + [5.0000e-01, 1.0000e+00, 8.5787e-02, 1.0000e+00, 1.0000e+00, + 5.0000e-01, 1.0000e+00, 6.2868e-01]]]) + for data1 in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): + count_unequal_element(data1["Spectrogram"], result, 0.0001, 0.0001) + + +def test_spectrogram_eager(): + """ + Feature: mindspore eager mode normal testcase: spectrogram op. + Description: input audio signal to test eager. + Expectation: success. + """ + logger.info("test_spectrogram_eager") + wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]]) + out = audio.Spectrogram(n_fft=8, win_length=8, window=WindowType.HANN, + pad_mode=BorderType.REFLECT)(np.array(wav, dtype="float")) + result = np.array([[[2.8015e+01, 1.2100e+02, 3.1354e+02, 1.6900e+02, 2.5000e+01, + 1.0843e+01, 1.2100e+02, 3.3150e+02], + [3.2145e+00, 3.3914e+01, 9.4728e+01, 4.5914e+01, 9.9142e+00, + 4.5858e+00, 3.3914e+01, 9.5685e+01], + [1.0000e+00, 1.7157e-01, 1.5000e+00, 1.7157e-01, 1.7157e-01, + 5.0000e-01, 1.7157e-01, 7.5000e-01], + [4.2893e-02, 2.5736e-01, 5.8579e-01, 2.5736e-01, 2.5736e-01, + 5.8579e-01, 2.5736e-01, 1.2868e-01], + [5.0000e-01, 1.0000e+00, 8.5787e-02, 1.0000e+00, 1.0000e+00, + 5.0000e-01, 1.0000e+00, 6.2868e-01]]]) + count_unequal_element(out, result, 0.0001, 0.0001) + + +def test_spectrogram_window_hamming_padmode_constant(): + """ + Feature: test spectrogram parameter: window, pad_mode. + Description: test parameter. + Expectation: success. + """ + logger.info("test_spectrogram_window_hamming_padmode_constant") + wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]]) + out = audio.Spectrogram(n_fft=8, window=WindowType.HAMMING, + pad_mode=BorderType.CONSTANT)(np.array(wav, dtype="float")) + result = np.array([[[1.1389e+01, 1.3736e+02, 3.5534e+02, 2.0164e+02, 3.0914e+01, + 1.3465e+01, 1.3736e+02, 2.5064e+02], + [5.6934e+00, 3.1860e+01, 8.5291e+01, 3.8484e+01, 9.1907e+00, + 4.5576e+00, 3.1860e+01, 1.0027e+02], + [1.9633e-01, 7.4475e-02, 1.2696e+00, 7.4475e-02, 7.4475e-02, + 4.2320e-01, 7.4475e-02, 1.1765e+01], + [5.0570e-01, 3.8456e-01, 6.8326e-01, 3.8456e-01, 3.8456e-01, + 6.8326e-01, 3.8456e-01, 5.4501e+00], + [6.1665e-01, 8.4640e-01, 7.2610e-02, 8.4640e-01, 8.4640e-01, + 4.2320e-01, 8.4640e-01, 1.0642e+00]]]) + count_unequal_element(out, result, 0.0001, 0.0001) + + +def test_spectrogram_nfft_10_window_bartlett_padmode_edge(): + """ + Feature: test spectrogram parameter: n_fft, window, pad_mode. + Description: test parameter. + Expectation: success. + """ + logger.info("test_spectrogram_nfft_10_window_bartlett_padmode_edge") + wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]]) + out = audio.Spectrogram(n_fft=10, window=WindowType.BARTLETT, + pad_mode=BorderType.EDGE)(np.array(wav, dtype="float")) + result = np.array([[[4.0960e+01, 2.6244e+02, 4.0000e+02, 7.7440e+01, 2.5000e+01, + 2.6244e+02, 5.9536e+02], + [4.7655e+00, 5.6721e+01, 9.4681e+01, 2.3822e+01, 5.9597e+00, + 5.6721e+01, 1.1597e+02], + [5.3889e-01, 5.8360e-03, 9.2361e-01, 5.8359e-03, 9.2361e-01, + 5.8360e-03, 2.4944e-01], + [1.1449e-01, 9.9859e-01, 3.1897e-01, 2.9828e-01, 1.0403e+00, + 9.9859e-01, 1.3072e+00], + [1.8111e-01, 2.7416e-01, 4.7639e-01, 2.7416e-01, 4.7639e-01, + 2.7416e-01, 7.0557e-02], + [6.4000e-01, 3.6000e-01, 0.0000e+00, 2.5600e+00, 1.0000e+00, + 3.6000e-01, 1.4400e+00]]]) + count_unequal_element(out, result, 0.0001, 0.0001) + + +def test_spectrogram_onsided_false(): + """ + Feature: test spectrogram parameter: onesided. + Description: test parameter. + Expectation: success. + """ + logger.info("test_spectrogram_onsided_false") + wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]]) + out = audio.Spectrogram(n_fft=10, window=WindowType.BARTLETT, + pad_mode=BorderType.EDGE, onesided=False)(np.array(wav, dtype="float")) + result = np.array([[[4.0960e+01, 2.6244e+02, 4.0000e+02, 7.7440e+01, 2.5000e+01, + 2.6244e+02, 5.9536e+02], + [4.7655e+00, 5.6721e+01, 9.4681e+01, 2.3822e+01, 5.9597e+00, + 5.6721e+01, 1.1597e+02], + [5.3889e-01, 5.8360e-03, 9.2361e-01, 5.8359e-03, 9.2361e-01, + 5.8360e-03, 2.4944e-01], + [1.1449e-01, 9.9859e-01, 3.1897e-01, 2.9828e-01, 1.0403e+00, + 9.9859e-01, 1.3072e+00], + [1.8111e-01, 2.7416e-01, 4.7639e-01, 2.7416e-01, 4.7639e-01, + 2.7416e-01, 7.0557e-02], + [6.4000e-01, 3.6000e-01, 0.0000e+00, 2.5600e+00, 1.0000e+00, + 3.6000e-01, 1.4400e+00], + [1.8111e-01, 2.7416e-01, 4.7639e-01, 2.7416e-01, 4.7639e-01, + 2.7416e-01, 7.0557e-02], + [1.1449e-01, 9.9859e-01, 3.1897e-01, 2.9828e-01, 1.0403e+00, + 9.9859e-01, 1.3072e+00], + [5.3889e-01, 5.8360e-03, 9.2361e-01, 5.8359e-03, 9.2361e-01, + 5.8360e-03, 2.4944e-01], + [4.7655e+00, 5.6721e+01, 9.4681e+01, 2.3822e+01, 5.9597e+00, + 5.6721e+01, 1.1597e+02]]]) + count_unequal_element(out, result, 0.0001, 0.0001) + + +def test_spectrogram_power_0(): + """ + Feature: test spectrogram parameter: power. + Description: test parameter. + Expectation: success. + """ + logger.info("test_spectrogram_power_0") + wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]]) + out = audio.Spectrogram(n_fft=8, window=WindowType.HANN, + pad_mode=BorderType.REFLECT, power=0)(np.array(wav, dtype="float")) + result = np.array([[[[5.2929e+00, 0.0000e+00], + [1.1000e+01, 0.0000e+00], + [1.7707e+01, 0.0000e+00], + [1.3000e+01, 0.0000e+00], + [5.0000e+00, 0.0000e+00], + [3.2929e+00, 0.0000e+00], + [1.1000e+01, 0.0000e+00], + [1.8207e+01, 0.0000e+00]], + [[-1.7929e+00, -2.5288e-07], + [-5.5000e+00, 1.9142e+00], + [-9.7071e+00, 7.0711e-01], + [-6.5000e+00, -1.9142e+00], + [-2.5000e+00, -1.9142e+00], + [-1.2929e+00, 1.7071e+00], + [-5.5000e+00, 1.9142e+00], + [-9.7071e+00, 1.2071e+00]], + [[-1.0000e+00, 0.0000e+00], + [0.0000e+00, -4.1421e-01], + [1.0000e+00, -7.0711e-01], + [0.0000e+00, 4.1421e-01], + [0.0000e+00, 4.1421e-01], + [0.0000e+00, -7.0711e-01], + [0.0000e+00, -4.1421e-01], + [5.0000e-01, -7.0711e-01]], + [[-2.0711e-01, -2.5288e-07], + [-5.0000e-01, -8.5787e-02], + [-2.9289e-01, 7.0711e-01], + [5.0000e-01, 8.5786e-02], + [5.0000e-01, 8.5786e-02], + [-7.0711e-01, -2.9289e-01], + [-5.0000e-01, -8.5787e-02], + [-2.9289e-01, 2.0711e-01]], + [[7.0711e-01, 0.0000e+00], + [1.0000e+00, 0.0000e+00], + [2.9289e-01, 0.0000e+00], + [-1.0000e+00, 0.0000e+00], + [-1.0000e+00, 0.0000e+00], + [7.0711e-01, 0.0000e+00], + [1.0000e+00, 0.0000e+00], + [7.9289e-01, 0.0000e+00]]]]) + count_unequal_element(out, result, 0.0001, 0.0001) + + +def test_spectrogram_center_false(): + """ + Feature: test spectrogram parameter: center. + Description: test parameter. + Expectation: success. + """ + logger.info("test_spectrogram_center_false") + wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]]) + out = audio.Spectrogram(n_fft=8, window=WindowType.HANN, + center=False, pad_mode=BorderType.REFLECT)(np.array(wav, dtype="float")) + result = np.array([[[1.2100e+02, 3.1354e+02, 1.6900e+02, 2.5000e+01, 1.0843e+01, + 1.2100e+02], + [3.3914e+01, 9.4728e+01, 4.5914e+01, 9.9142e+00, 4.5858e+00, + 3.3914e+01], + [1.7157e-01, 1.5000e+00, 1.7157e-01, 1.7157e-01, 5.0000e-01, + 1.7157e-01], + [2.5736e-01, 5.8579e-01, 2.5736e-01, 2.5736e-01, 5.8579e-01, + 2.5736e-01], + [1.0000e+00, 8.5787e-02, 1.0000e+00, 1.0000e+00, 5.0000e-01, + 1.0000e+00]]]) + count_unequal_element(out, result, 0.0001, 0.0001) + + +def test_spectrogram_normalized_true(): + """ + Feature: test spectrogram parameter: normalized. + Description: test parameter. + Expectation: success. + """ + logger.info("test_spectrogram_normalized_true") + wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]]) + out = audio.Spectrogram(n_fft=8, window=WindowType.HANN, + center=False, normalized=True, pad_mode=BorderType.REFLECT)(np.array(wav, dtype="float")) + result = np.array([[[4.0333e+01, 1.0451e+02, 5.6333e+01, 8.3333e+00, 3.6144e+00, + 4.0333e+01], + [1.1305e+01, 3.1576e+01, 1.5305e+01, 3.3047e+00, 1.5286e+00, + 1.1305e+01], + [5.7191e-02, 5.0000e-01, 5.7191e-02, 5.7191e-02, 1.6667e-01, + 5.7191e-02], + [8.5786e-02, 1.9526e-01, 8.5786e-02, 8.5786e-02, 1.9526e-01, + 8.5786e-02], + [3.3333e-01, 2.8596e-02, 3.3333e-01, 3.3333e-01, 1.6667e-01, + 3.3333e-01]]]) + count_unequal_element(out, result, 0.0001, 0.0001) + + +def test_spectrogram_inputrank_3(): + """ + Feature: test spectrogram parameter: input rank. + Description: test input rank. + Expectation: success. + """ + logger.info("test_spectrogram_inputrank_3") + wav = np.array([[[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1]], + [[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1]]]) + out = audio.Spectrogram(n_fft=8, window=WindowType.HANN, pad_mode=BorderType.REFLECT)(np.array(wav, dtype="float")) + result = np.array([[[[2.8015e+01, 1.2100e+02, 3.1354e+02, 1.6900e+02, 3.3558e+01], + [3.2145e+00, 3.3914e+01, 9.4728e+01, 4.5914e+01, 6.7145e+00], + [1.0000e+00, 1.7157e-01, 1.5000e+00, 1.7157e-01, 7.5000e-01], + [4.2893e-02, 2.5736e-01, 5.8579e-01, 2.5736e-01, 1.2868e-01], + [5.0000e-01, 1.0000e+00, 8.5787e-02, 1.0000e+00, 6.2868e-01]]], + [[[2.8015e+01, 1.2100e+02, 3.1354e+02, 1.6900e+02, 3.3558e+01], + [3.2145e+00, 3.3914e+01, 9.4728e+01, 4.5914e+01, 6.7145e+00], + [1.0000e+00, 1.7157e-01, 1.5000e+00, 1.7157e-01, 7.5000e-01], + [4.2893e-02, 2.5736e-01, 5.8579e-01, 2.5736e-01, 1.2868e-01], + [5.0000e-01, 1.0000e+00, 8.5787e-02, 1.0000e+00, 6.2868e-01]]]]) + count_unequal_element(out, result, 0.0001, 0.0001) + + +def test_spectrogram_winlength_7(): + """ + Feature: test spectrogram parameter: win_length. + Description: test parameter. + Expectation: success. + """ + logger.info("test_spectrogram_winlength_7") + wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]]) + out = audio.Spectrogram(n_fft=8, win_length=7, window=WindowType.HANN,)(np.array(wav, dtype="float")) + result = np.array([[[2.0140e+01, 4.9000e+01, 1.5006e+02, 2.5284e+02, 1.5006e+02, + 4.9000e+01, 4.5220e+00, 1.2250e+01, 7.6562e+01, 1.9600e+02, + 2.7265e+02], + [5.0272e+00, 1.9488e+01, 5.5103e+01, 1.0153e+02, 5.5103e+01, + 1.9488e+01, 2.0179e+00, 6.5089e+00, 2.9144e+01, 7.1406e+01, + 1.0662e+02], + [4.2200e-01, 5.4020e-01, 1.0554e+00, 4.8321e+00, 1.0554e+00, + 5.4020e-01, 9.6867e-01, 4.0345e-01, 7.8188e-01, 1.0872e+00, + 3.3055e+00], + [1.2817e-01, 3.8618e-01, 2.1917e-01, 2.6102e-01, 2.1917e-01, + 3.8618e-01, 4.3028e-01, 3.7738e-01, 2.0158e-01, 4.2135e-01, + 5.7616e-02], + [3.7364e-01, 7.1574e-01, 8.1719e-01, 9.0949e-13, 8.1720e-01, + 7.1573e-01, 2.7823e-01, 7.1573e-01, 8.1719e-01, 7.1574e-01, + 3.7364e-01]]]) + count_unequal_element(out, result, 0.0001, 0.0001) + + +def test_spectrogram_param(): + """ + Feature: test spectrogram invalid parameter. + Description: test some invalid parameters. + Expectation: success. + """ + try: + _ = audio.Spectrogram(n_fft=-1) + except ValueError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Input n_fft is not within the required interval of [1, 2147483647]." in str(error) + try: + _ = audio.Spectrogram(n_fft=0) + except ValueError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Input n_fft is not within the required interval of [1, 2147483647]." in str(error) + try: + _ = audio.Spectrogram(win_length=-1) + except ValueError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Input win_length is not within the required interval of [1, 2147483647]." in str(error) + try: + _ = audio.Spectrogram(win_length="s") + except TypeError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Argument win_length with value s is not of type [], but got ." in str(error) + try: + _ = audio.Spectrogram(hop_length=-1) + except ValueError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Input hop_length is not within the required interval of [1, 2147483647]." in str(error) + try: + _ = audio.Spectrogram(hop_length=-100) + except ValueError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Input hop_length is not within the required interval of [1, 2147483647]." in str(error) + try: + _ = audio.Spectrogram(win_length=300, n_fft=200) + except ValueError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Input win_length should be no more than n_fft, but got win_length: 300 and n_fft: 200." \ + in str(error) + try: + _ = audio.Spectrogram(pad=-1) + except ValueError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Input pad is not within the required interval of [0, 2147483647]." in str(error) + try: + _ = audio.Spectrogram(power=-1) + except ValueError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Input power is not within the required interval of [0, 16777216]." in str(error) + try: + _ = audio.Spectrogram(n_fft=False) + except TypeError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Argument n_fft with value False is not of type (,)" in str(error) + try: + _ = audio.Spectrogram(n_fft="s") + except TypeError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Argument n_fft with value s is not of type [], but got ." \ + in str(error) + try: + _ = audio.Spectrogram(window=False) + except TypeError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Argument window with value False is not of type [], but got ." \ + in str(error) + try: + _ = audio.Spectrogram(pad_mode=False) + except TypeError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Argument pad_mode with value False is not of type [], but got ." \ + in str(error) + try: + _ = audio.Spectrogram(onesided="s") + except TypeError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Argument onesided with value s is not of type [], but got ." in str(error) + try: + _ = audio.Spectrogram(center="s") + except TypeError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Argument center with value s is not of type [], but got ." in str(error) + try: + _ = audio.Spectrogram(normalized="s") + except TypeError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Argument normalized with value s is not of type [], but got ." in str(error) + try: + _ = audio.Spectrogram(normalized=1) + except TypeError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Argument normalized with value 1 is not of type [], but got ." in str(error) + try: + wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]]) + _ = audio.Spectrogram(n_fft=100, center=False)(wav) + except RuntimeError as error: + logger.info("Got an exception in Spectrogram: {}".format(str(error))) + assert "Unexpected error. Spectrogram: n_fft should be more than 0 and less than 30," \ + " but got n_fft: 100." in str(error) + + +if __name__ == "__main__": + test_spectrogram_pipeline() + test_spectrogram_eager() + test_spectrogram_window_hamming_padmode_constant() + test_spectrogram_nfft_10_window_bartlett_padmode_edge() + test_spectrogram_onsided_false() + test_spectrogram_power_0() + test_spectrogram_center_false() + test_spectrogram_normalized_true() + test_spectrogram_inputrank_3() + test_spectrogram_winlength_7() + test_spectrogram_param()