diff --git a/mindspore/ccsrc/minddata/dataset/api/audio.cc b/mindspore/ccsrc/minddata/dataset/api/audio.cc index c440946e05..812c4a58f5 100644 --- a/mindspore/ccsrc/minddata/dataset/api/audio.cc +++ b/mindspore/ccsrc/minddata/dataset/api/audio.cc @@ -37,6 +37,7 @@ #include "minddata/dataset/audio/ir/kernels/flanger_ir.h" #include "minddata/dataset/audio/ir/kernels/frequency_masking_ir.h" #include "minddata/dataset/audio/ir/kernels/gain_ir.h" +#include "minddata/dataset/audio/ir/kernels/griffin_lim_ir.h" #include "minddata/dataset/audio/ir/kernels/highpass_biquad_ir.h" #include "minddata/dataset/audio/ir/kernels/lfilter_ir.h" #include "minddata/dataset/audio/ir/kernels/lowpass_biquad_ir.h" @@ -413,6 +414,41 @@ Gain::Gain(float gain_db) : data_(std::make_shared(gain_db)) {} std::shared_ptr Gain::Parse() { return std::make_shared(data_->gain_db_); } +// GriffinLim Transform Operation. +struct GriffinLim::Data { + Data(int32_t n_fft, int32_t n_iter, int32_t win_length, int32_t hop_length, WindowType window_type, float power, + float momentum, int32_t length, bool rand_init) + : n_fft_(n_fft), + n_iter_(n_iter), + win_length_(win_length), + hop_length_(hop_length), + window_type_(window_type), + power_(power), + momentum_(momentum), + length_(length), + rand_init_(rand_init) {} + int32_t n_fft_; + int32_t n_iter_; + int32_t win_length_; + int32_t hop_length_; + WindowType window_type_; + float power_; + float momentum_; + int32_t length_; + bool rand_init_; +}; + +GriffinLim::GriffinLim(int32_t n_fft, int32_t n_iter, int32_t win_length, int32_t hop_length, WindowType window_type, + float power, float momentum, int32_t length, bool rand_init) + : data_(std::make_shared(n_fft, n_iter, win_length, hop_length, window_type, power, momentum, length, + rand_init)) {} + +std::shared_ptr GriffinLim::Parse() { + return std::make_shared(data_->n_fft_, data_->n_iter_, data_->win_length_, data_->hop_length_, + data_->window_type_, data_->power_, data_->momentum_, data_->length_, + data_->rand_init_); +} + // HighpassBiquad Transform Operation. struct HighpassBiquad::Data { Data(int32_t sample_rate, float cutoff_freq, float Q) : sample_rate_(sample_rate), cutoff_freq_(cutoff_freq), Q_(Q) {} diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc index d5e2e1043e..b5b4dbc0b2 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc @@ -41,6 +41,7 @@ #include "minddata/dataset/audio/ir/kernels/flanger_ir.h" #include "minddata/dataset/audio/ir/kernels/frequency_masking_ir.h" #include "minddata/dataset/audio/ir/kernels/gain_ir.h" +#include "minddata/dataset/audio/ir/kernels/griffin_lim_ir.h" #include "minddata/dataset/audio/ir/kernels/highpass_biquad_ir.h" #include "minddata/dataset/audio/ir/kernels/lfilter_ir.h" #include "minddata/dataset/audio/ir/kernels/lowpass_biquad_ir.h" @@ -334,6 +335,19 @@ PYBIND_REGISTER(GainOperation, 1, ([](const py::module *m) { })); })); +PYBIND_REGISTER( + GriffinLimOperation, 1, ([](const py::module *m) { + (void)py::class_>( + *m, "GriffinLimOperation") + .def(py::init([](int32_t n_fft, int32_t n_iter, int32_t win_length, int32_t hop_length, WindowType window_type, + float power, float momentum, int32_t length, bool rand_init) { + auto griffin_lim = std::make_shared( + n_fft, n_iter, win_length, hop_length, window_type, power, momentum, length, rand_init); + THROW_IF_ERROR(griffin_lim->ValidateParams()); + return griffin_lim; + })); + })); + PYBIND_REGISTER( HighpassBiquadOperation, 1, ([](const py::module *m) { (void)py::class_>( diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/CMakeLists.txt index 850b988a80..4820bda8fe 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/CMakeLists.txt @@ -23,6 +23,7 @@ add_library(audio-ir-kernels OBJECT flanger_ir.cc frequency_masking_ir.cc gain_ir.cc + griffin_lim_ir.cc highpass_biquad_ir.cc lfilter_ir.cc lowpass_biquad_ir.cc diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/griffin_lim_ir.cc b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/griffin_lim_ir.cc new file mode 100644 index 0000000000..9d054c2d8b --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/griffin_lim_ir.cc @@ -0,0 +1,92 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/audio/ir/kernels/griffin_lim_ir.h" + +#include "minddata/dataset/audio/ir/validators.h" +#include "minddata/dataset/audio/kernels/griffin_lim_op.h" + +namespace mindspore { +namespace dataset { +namespace audio { +// GriffinLim +GriffinLimOperation::GriffinLimOperation(int32_t n_fft, int32_t n_iter, int32_t win_length, int32_t hop_length, + WindowType window_type, float power, float momentum, int32_t length, + bool rand_init) + : n_fft_(n_fft), + n_iter_(n_iter), + win_length_(win_length), + hop_length_(hop_length), + window_type_(window_type), + power_(power), + momentum_(momentum), + length_(length), + rand_init_(rand_init) {} + +GriffinLimOperation::~GriffinLimOperation() = default; + +std::string GriffinLimOperation::Name() const { return kGriffinLimOperation; } + +Status GriffinLimOperation::ValidateParams() { + RETURN_IF_NOT_OK(ValidateIntScalarPositive("GriffinLim", "n_fft", n_fft_)); + RETURN_IF_NOT_OK(ValidateIntScalarPositive("GriffinLim", "n_iter", n_iter_)); + RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("GriffinLim", "win_length", win_length_)); + RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("GriffinLim", "hop_length", hop_length_)); + RETURN_IF_NOT_OK(ValidateFloatScalarPositive("GriffinLim", "power", power_)); + RETURN_IF_NOT_OK(ValidateFloatScalarNonNegative("GriffinLim", "momentum", momentum_)); + RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("GriffinLim", "length", length_)); + if (length_ != 0 && n_fft_ >= length_) { + std::string err_msg = "GriffinLim: n_fft must be less than length."; + LOG_AND_RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + + CHECK_FAIL_RETURN_SYNTAX_ERROR( + momentum_ < 1, + "GriffinLim: momentum equal to or greater than 1 can be unstable, but got: " + std::to_string(momentum_)); + CHECK_FAIL_RETURN_SYNTAX_ERROR(momentum_ >= 0, + "GriffinLim: momentum can not be less than 0, but got: " + std::to_string(momentum_)); + CHECK_FAIL_RETURN_SYNTAX_ERROR(win_length_ <= n_fft_, + "GriffinLim: win_length must be less than or equal to n_fft, but got win_length: " + + std::to_string(win_length_) + ", n_fft: " + std::to_string(n_fft_)); + return Status::OK(); +} + +std::shared_ptr GriffinLimOperation::Build() { + int32_t win_length = (win_length_ == 0) ? n_fft_ : win_length_; + int32_t hop_length = (hop_length_ == 0) ? win_length / 2 : hop_length_; + float momentum = momentum_ / (1 + momentum_); + std::shared_ptr tensor_op = std::make_shared( + n_fft_, n_iter_, win_length, hop_length, window_type_, power_, momentum, length_, rand_init_); + return tensor_op; +} + +Status GriffinLimOperation::to_json(nlohmann::json *out_json) { + nlohmann::json args; + args["n_fft"] = n_fft_; + args["n_iter"] = n_iter_; + args["win_length"] = win_length_; + args["hop_length"] = hop_length_; + args["window_type"] = window_type_; + args["power"] = power_; + args["momentum"] = momentum_; + args["length"] = length_; + args["rand_init"] = rand_init_; + *out_json = args; + return Status::OK(); +} +} // namespace audio +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/griffin_lim_ir.h b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/griffin_lim_ir.h new file mode 100644 index 0000000000..bef970bbf9 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/audio/ir/kernels/griffin_lim_ir.h @@ -0,0 +1,61 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_GRIFFIN_LIM_IR_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_GRIFFIN_LIM_IR_H_ + +#include +#include + +#include "include/api/status.h" +#include "minddata/dataset/include/dataset/constants.h" +#include "minddata/dataset/kernels/ir/tensor_operation.h" + +namespace mindspore { +namespace dataset { +namespace audio { +constexpr char kGriffinLimOperation[] = "GriffinLim"; + +class GriffinLimOperation : public TensorOperation { + public: + GriffinLimOperation(int32_t n_fft, int32_t n_iter, int32_t win_length, int32_t hop_length, WindowType window_type, + float power, float momentum, int32_t length, bool rand_init); + + ~GriffinLimOperation(); + + std::shared_ptr Build() override; + + Status ValidateParams() override; + + std::string Name() const override; + + Status to_json(nlohmann::json *out_json) override; + + private: + int32_t n_fft_; + int32_t n_iter_; + int32_t win_length_; + int32_t hop_length_; + WindowType window_type_; + float power_; + float momentum_; + int32_t length_; + bool rand_init_; +}; +} // namespace audio +} // namespace dataset +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_GRIFFIN_LIM_IR_H_ diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/audio/kernels/CMakeLists.txt index 4a2a561ac7..527a0dbb8a 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/CMakeLists.txt @@ -24,6 +24,7 @@ add_library(audio-kernels OBJECT flanger_op.cc frequency_masking_op.cc gain_op.cc + griffin_lim_op.cc highpass_biquad_op.cc lfilter_op.cc lowpass_biquad_op.cc diff --git a/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc b/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc index 7d3e33cb71..82c8653e42 100644 --- a/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc +++ b/mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc @@ -1769,5 +1769,286 @@ Status ComputeDeltas(const std::shared_ptr &input, std::shared_ptrReshape(raw_shape)); return Status::OK(); } + +/// \brief IRFFT. +Status IRFFT(const Eigen::MatrixXcd &stft_matrix, Eigen::MatrixXd *inverse) { + int32_t n = 2 * (stft_matrix.rows() - 1); + int32_t s = stft_matrix.rows() - 1; + Eigen::FFT fft; + for (int k = 0; k < stft_matrix.cols(); ++k) { + Eigen::VectorXcd output_complex(n); + // pad input + Eigen::VectorXcd input(n); + input.head(s + 1) = stft_matrix.col(k); + auto reverse_pad = stft_matrix.col(k).segment(1, s - 1).colwise().reverse().conjugate(); + input.segment(s + 1, s - 1) = reverse_pad; + fft.inv(output_complex, input); + Eigen::VectorXd output_real = output_complex.real().eval(); + inverse->col(k) = Eigen::Map(output_real.data(), n, 1); + } + return Status::OK(); +} + +/// \brief Overlap Add +Status OverlapAdd(Eigen::VectorXd *out_buf, const Eigen::MatrixXd &win_inverse_stft, int32_t hop_lengh) { + int32_t n_fft = win_inverse_stft.rows(); + for (int frame = 0; frame < win_inverse_stft.cols(); frame++) { + int32_t sample = frame * hop_lengh; + out_buf->middleRows(sample, n_fft) += win_inverse_stft.col(frame); + } + return Status::OK(); +} + +/// \brief Window Sum Square +Status WindowSumSquare(const Eigen::MatrixXf &window_matrix, Eigen::VectorXf *win_sum_square, const int32_t n_frames, + int32_t n_fft, int32_t hop_length) { + Eigen::MatrixXf win_norm = window_matrix.array().pow(2); + // window sum square fill + int32_t n = n_fft + hop_length * (n_frames - 1); + // check n_fft + CHECK_FAIL_RETURN_UNEXPECTED( + n_fft == win_norm.rows(), + "GriffinLim: n_fft must be equal to the length of the window during window sum square calculation."); + for (int ind = 0; ind < n_frames; ind++) { + int sample = ind * hop_length; + int end_ss = std::min(n, sample + n_fft); + int end_win = std::max(0, std::min(n_fft, n - sample)); + win_sum_square->segment(sample, end_ss - sample) += win_norm.col(0).head(end_win); + } + return Status::OK(); +} + +/// \brief ISTFT. +/// \param input: Complex matrix of eigen, shape of . +/// \param output: Tensor of shape