yankf21
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!26456 [assistant][ops]Add new operator Spectrogram 

Merge pull request !26456 from YJfuel123/Spectrogram
This commit is contained in:
i-robot 2021-12-07 06:39:01 +00:00 committed by Gitee
parent 6136a06a04 4ae41350a6
commit 7a8e35b4f1
20 changed files with 1736 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
mindspore/ccsrc/minddata/dataset/api/audio.cc
View File

@ -45,6 +45,7 @@
#include "minddata/dataset/audio/ir/kernels/phaser_ir.h"
#include "minddata/dataset/audio/ir/kernels/riaa_biquad_ir.h"
#include "minddata/dataset/audio/ir/kernels/sliding_window_cmn_ir.h"
#include "minddata/dataset/audio/ir/kernels/spectrogram_ir.h"
#include "minddata/dataset/audio/ir/kernels/time_masking_ir.h"
#include "minddata/dataset/audio/ir/kernels/time_stretch_ir.h"
#include "minddata/dataset/audio/ir/kernels/treble_biquad_ir.h"
@ -530,6 +531,43 @@ std::shared_ptr<TensorOperation> SlidingWindowCmn::Parse() {
data_->norm_vars_);
}
// Spectrogram Transform Operation.
struct Spectrogram::Data {
Data(int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad, WindowType window, float power,
bool normalized, bool center, BorderType pad_mode, bool onesided)
: n_fft_(n_fft),
win_length_(win_length),
hop_length_(hop_length),
pad_(pad),
window_(window),
power_(power),
normalized_(normalized),
center_(center),
pad_mode_(pad_mode),
onesided_(onesided) {}
int32_t n_fft_;
int32_t win_length_;
int32_t hop_length_;
int32_t pad_;
WindowType window_;
float power_;
bool normalized_;
bool center_;
BorderType pad_mode_;
bool onesided_;
};
Spectrogram::Spectrogram(int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad, WindowType window,
float power, bool normalized, bool center, BorderType pad_mode, bool onesided)
: data_(std::make_shared<Data>(n_fft, win_length, hop_length, pad, window, power, normalized, center, pad_mode,
onesided)) {}
std::shared_ptr<TensorOperation> Spectrogram::Parse() {
return std::make_shared<SpectrogramOperation>(data_->n_fft_, data_->win_length_, data_->hop_length_, data_->pad_,
data_->window_, data_->power_, data_->normalized_, data_->center_,
data_->pad_mode_, data_->onesided_);
}
// TimeMasking Transform Operation.
struct TimeMasking::Data {
Data(bool iid_masks, int32_t time_mask_param, int32_t mask_start, float mask_value)

24
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc
View File

@ -49,6 +49,7 @@
#include "minddata/dataset/audio/ir/kernels/phaser_ir.h"
#include "minddata/dataset/audio/ir/kernels/riaa_biquad_ir.h"
#include "minddata/dataset/audio/ir/kernels/sliding_window_cmn_ir.h"
#include "minddata/dataset/audio/ir/kernels/spectrogram_ir.h"
#include "minddata/dataset/audio/ir/kernels/time_masking_ir.h"
#include "minddata/dataset/audio/ir/kernels/time_stretch_ir.h"
#include "minddata/dataset/audio/ir/kernels/treble_biquad_ir.h"
@ -409,6 +410,29 @@ PYBIND_REGISTER(SlidingWindowCmnOperation, 1, ([](const py::module *m) {
}));
}));
PYBIND_REGISTER(WindowType, 0, ([](const py::module *m) {
(void)py::enum_<WindowType>(*m, "WindowType", py::arithmetic())
.value("DE_BARTLETT", WindowType::kBartlett)
.value("DE_BLACKMAN", WindowType::kBlackman)
.value("DE_HAMMING", WindowType::kHamming)
.value("DE_HANN", WindowType::kHann)
.value("DE_KAISER", WindowType::kKaiser)
.export_values();
}));
PYBIND_REGISTER(
SpectrogramOperation, 1, ([](const py::module *m) {
(void)py::class_<audio::SpectrogramOperation, TensorOperation, std::shared_ptr<audio::SpectrogramOperation>>(
*m, "SpectrogramOperation")
.def(py::init([](int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad, WindowType window,
float power, bool normalized, bool center, BorderType pad_mode, bool onesided) {
auto spectrogram = std::make_shared<audio::SpectrogramOperation>(n_fft, win_length, hop_length, pad, window,
power, normalized, center, pad_mode, onesided);
THROW_IF_ERROR(spectrogram->ValidateParams());
return spectrogram;
}));
}));
PYBIND_REGISTER(
TimeMaskingOperation, 1, ([](const py::module *m) {
(void)py::class_<audio::TimeMaskingOperation, TensorOperation, std::shared_ptr<audio::TimeMaskingOperation>>(

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/CMakeLists.txt
View File

@ -31,6 +31,7 @@ add_library(audio-ir-kernels OBJECT
phaser_ir.cc
riaa_biquad_ir.cc
sliding_window_cmn_ir.cc
spectrogram_ir.cc
time_masking_ir.cc
time_stretch_ir.cc
treble_biquad_ir.cc

77
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/spectrogram_ir.cc Normal file
View File

@ -0,0 +1,77 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/audio/ir/kernels/spectrogram_ir.h"
#include "minddata/dataset/audio/ir/validators.h"
#include "minddata/dataset/audio/kernels/spectrogram_op.h"
namespace mindspore {
namespace dataset {
namespace audio {
// SpectrogramOperation
SpectrogramOperation::SpectrogramOperation(int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad,
WindowType window, float power, bool normalized, bool center,
BorderType pad_mode, bool onesided)
: n_fft_(n_fft),
win_length_(win_length),
hop_length_(hop_length),
pad_(pad),
window_(window),
power_(power),
normalized_(normalized),
center_(center),
pad_mode_(pad_mode),
onesided_(onesided) {}
Status SpectrogramOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateIntScalarPositive("Spectrogram", "n_fft", n_fft_));
RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("Spectrogram", "win_length", win_length_));
RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("Spectrogram", "hop_length", hop_length_));
RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("Spectrogram", "pad", pad_));
RETURN_IF_NOT_OK(ValidateFloatScalarNonNegative("Spectrogram", "power", power_));
CHECK_FAIL_RETURN_UNEXPECTED(win_length_ <= n_fft_,
"Spectrogram: win_length must be less than or equal to n_fft, but got win_length: " +
std::to_string(win_length_) + ", n_fft: " + std::to_string(n_fft_));
return Status::OK();
}
std::shared_ptr<TensorOp> SpectrogramOperation::Build() {
int32_t win_length = (win_length_ == 0) ? n_fft_ : win_length_;
int32_t hop_length = (hop_length_ == 0) ? win_length_ / 2 : hop_length_;
std::shared_ptr<SpectrogramOp> tensor_op = std::make_shared<SpectrogramOp>(
n_fft_, win_length, hop_length, pad_, window_, power_, normalized_, center_, pad_mode_, onesided_);
return tensor_op;
}
Status SpectrogramOperation::to_json(nlohmann::json *out_json) {
nlohmann::json args;
args["n_fft"] = n_fft_;
args["win_length"] = win_length_;
args["hop_length"] = hop_length_;
args["pad"] = pad_;
args["window"] = window_;
args["power"] = power_;
args["normalized"] = normalized_;
args["center"] = center_;
args["pad_mode"] = pad_mode_;
args["onesided"] = onesided_;
*out_json = args;
return Status::OK();
}
} // namespace audio
} // namespace dataset
} // namespace mindspore

61
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/spectrogram_ir.h Normal file
View File

@ -0,0 +1,61 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_SPECTROGRAM_IR_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_SPECTROGRAM_IR_H_
#include <memory>
#include <string>
#include "include/api/status.h"
#include "minddata/dataset/kernels/ir/tensor_operation.h"
namespace mindspore {
namespace dataset {
namespace audio {
constexpr char kSpectrogramOperation[] = "Spectrogram";
class SpectrogramOperation : public TensorOperation {
public:
SpectrogramOperation(int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad, WindowType window,
float power, bool normalized, bool center, BorderType pad_mode, bool onesided);
~SpectrogramOperation() = default;
std::shared_ptr<TensorOp> Build() override;
Status ValidateParams() override;
std::string Name() const override { return kSpectrogramOperation; }
Status to_json(nlohmann::json *out_json) override;
private:
int32_t n_fft_;
int32_t win_length_;
int32_t hop_length_;
int32_t pad_;
WindowType window_;
float power_;
bool normalized_;
bool center_;
BorderType pad_mode_;
bool onesided_;
};
} // namespace audio
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_SPECTROGRAM_IR_H_

1
mindspore/ccsrc/minddata/dataset/audio/kernels/CMakeLists.txt
View File

@ -32,6 +32,7 @@ add_library(audio-kernels OBJECT
phaser_op.cc
riaa_biquad_op.cc
sliding_window_cmn_op.cc
spectrogram_op.cc
time_masking_op.cc
time_stretch_op.cc
treble_biquad_op.cc

349
mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc
View File

@ -1156,6 +1156,355 @@ Status ComputeDeltasImpl(const std::shared_ptr<Tensor> &input, std::shared_ptr<T
return Status::OK();
}
Status Bartlett(std::shared_ptr<Tensor> *output, int len) {
CHECK_FAIL_RETURN_UNEXPECTED(len != 0, "Bartlett: len can not be zero.");
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({len}), DataType(DataType::DE_FLOAT32), output));
// Bartlett window function.
auto iter = (*output)->begin<float>();
float twice = 2.0;
for (ptrdiff_t i = 0; i < len; ++i) {
*(iter + i) = 1.0 - std::abs(twice * i / len - 1.0);
}
return Status::OK();
}
Status Blackman(std::shared_ptr<Tensor> *output, int len) {
CHECK_FAIL_RETURN_UNEXPECTED(len != 0, "Blackman: len can not be zero.");
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({len}), DataType(DataType::DE_FLOAT32), output));
// Blackman window function.
auto iter = (*output)->begin<float>();
float alpha = 0.42;
float half = 0.5;
float delta = 0.08;
for (ptrdiff_t i = 0; i < len; ++i) {
*(iter + i) = alpha - half * std::cos(TWO * PI * i / len) + delta * std::cos(TWO * TWO * PI * i / len);
}
return Status::OK();
}
Status Hamming(std::shared_ptr<Tensor> *output, int len, float alpha = 0.54, float beta = 0.46) {
CHECK_FAIL_RETURN_UNEXPECTED(len != 0, "Hamming: len can not be zero.");
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({len}), DataType(DataType::DE_FLOAT32), output));
// Hamming window function.
auto iter = (*output)->begin<float>();
for (ptrdiff_t i = 0; i < len; ++i) {
*(iter + i) = alpha - beta * std::cos(TWO * PI * i / len);
}
return Status::OK();
}
Status Hann(std::shared_ptr<Tensor> *output, int len) {
CHECK_FAIL_RETURN_UNEXPECTED(len != 0, "Hann: len can not be zero.");
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({len}), DataType(DataType::DE_FLOAT32), output));
// Hann window function.
auto iter = (*output)->begin<float>();
float half = 0.5;
for (ptrdiff_t i = 0; i < len; ++i) {
*(iter + i) = half - half * std::cos(TWO * PI * i / len);
}
return Status::OK();
}
Status Kaiser(std::shared_ptr<Tensor> *output, int len, float beta = 12.0) {
CHECK_FAIL_RETURN_UNEXPECTED(len != 0, "Kaiser: len can not be zero.");
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({len}), DataType(DataType::DE_FLOAT32), output));
// Kaiser window function.
auto iter = (*output)->begin<float>();
float twice = 2.0;
for (ptrdiff_t i = 0; i < len; ++i) {
*(iter + i) =
std::cyl_bessel_i(0, beta * std::sqrt(1 - std::pow(i * twice / (len)-1.0, TWO))) / std::cyl_bessel_i(0, beta);
}
return Status::OK();
}
Status Window(std::shared_ptr<Tensor> *output, WindowType window_type, int len) {
switch (window_type) {
case WindowType::kBartlett:
return Bartlett(output, len);
case WindowType::kBlackman:
return Blackman(output, len);
case WindowType::kHamming:
return Hamming(output, len);
case WindowType::kHann:
return Hann(output, len);
case WindowType::kKaiser:
return Kaiser(output, len);
default:
return Hann(output, len);
}
}
// control whether return half of results after stft.
template <typename T>
Status Onesided(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, int n_fft, int n_columns) {
std::shared_ptr<Tensor> output_onsided;
RETURN_IF_NOT_OK(
Tensor::CreateEmpty(TensorShape({input->shape()[0], n_fft, n_columns, 2}), input->type(), &output_onsided));
auto onside_begin = output_onsided->begin<T>();
auto spec_f_begin = input->begin<T>();
std::vector<int> spec_f_slice = {(n_fft / 2 + 1) * n_columns * 2, n_columns * 2, 2};
for (int r = 0; r < input->shape()[0]; r++) {
for (int i = 0; i < (n_fft / TWO + 1); i++) {
for (int j = 0; j < n_columns; j++) {
ptrdiff_t onside_offset_0 = r * n_fft * n_columns * 2 + i * spec_f_slice[1] + j * spec_f_slice[2];
ptrdiff_t spec_f_offset_0 = r * spec_f_slice[0] + i * spec_f_slice[1] + j * spec_f_slice[2];
ptrdiff_t onside_offset_1 = onside_offset_0 + 1;
ptrdiff_t spec_f_offset_1 = spec_f_offset_0 + 1;
*(onside_begin + onside_offset_0) = *(spec_f_begin + spec_f_offset_0);
*(onside_begin + onside_offset_1) = *(spec_f_begin + spec_f_offset_1);
}
}
for (int i = n_fft / 2 + 1; i < n_fft; i++) {
for (int j = 0; j < n_columns; j++) {
ptrdiff_t onside_offset_0 = r * n_fft * n_columns * 2 + i * spec_f_slice[1] + j * spec_f_slice[2];
ptrdiff_t spec_f_offset_0 = r * spec_f_slice[0] + (n_fft - i) * spec_f_slice[1] + j * spec_f_slice[2];
ptrdiff_t onside_offset_1 = onside_offset_0 + 1;
ptrdiff_t spec_f_offset_1 = spec_f_offset_0 + 1;
*(onside_begin + onside_offset_0) = *(spec_f_begin + spec_f_offset_0);
*(onside_begin + onside_offset_1) = *(spec_f_begin + spec_f_offset_1);
}
}
}
*output = output_onsided;
return Status::OK();
}
template <typename T>
Status PowerStft(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, float power, int n_fft,
int n_columns, int n_length) {
auto spec_f_begin = input->begin<T>();
std::vector<int> spec_f_slice = {n_length * n_columns * 2, n_columns * 2, 2};
std::vector<int> spec_p_slice = {n_length * n_columns, n_columns};
std::shared_ptr<Tensor> spec_p;
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({input->shape()[0], n_length, n_columns}), input->type(), &spec_p));
auto spec_p_begin = spec_p->begin<T>();
for (int r = 0; r < input->shape()[0]; r++) {
for (int i = 0; i < n_length; i++) {
for (int j = 0; j < n_columns; j++) {
ptrdiff_t spec_f_offset_0 = r * spec_f_slice[0] + i * spec_f_slice[1] + j * spec_f_slice[2];
ptrdiff_t spec_f_offset_1 = spec_f_offset_0 + 1;
ptrdiff_t spec_p_offset = r * spec_p_slice[0] + i * spec_p_slice[1] + j;
T spec_power_0 = *(spec_f_begin + spec_f_offset_0);
T spec_power_1 = *(spec_f_begin + spec_f_offset_1);
*(spec_p_begin + spec_p_offset) =
std::pow(std::sqrt(std::pow(spec_power_0, TWO) + std::pow(spec_power_1, TWO)), power);
}
}
}
*output = spec_p;
return Status::OK();
}
template <typename T>
Status Stft(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, int n_fft,
const std::shared_ptr<Tensor> &win, int win_length, int hop_length, int n_columns, bool normalized,
float power, bool onesided) {
CHECK_FAIL_RETURN_UNEXPECTED(win_length != 0, "Spectrogram: win_length can not be zero.");
double win_sum = 0.;
float twice = 2.0;
for (auto iter_win = win->begin<float>(); iter_win != win->end<float>(); iter_win++) {
win_sum += (*iter_win) * (*iter_win);
}
win_sum = std::sqrt(win_sum);
std::shared_ptr<Tensor> spec_f;
RETURN_IF_NOT_OK(
Tensor::CreateEmpty(TensorShape({input->shape()[0], n_fft / 2 + 1, n_columns, 2}), input->type(), &spec_f));
auto spec_f_begin = spec_f->begin<T>();
auto input_win_begin = input->begin<T>();
std::vector<int> spec_f_slice = {(n_fft / 2 + 1) * n_columns * 2, n_columns * 2, 2};
std::vector<int> input_win_slice = {n_columns * win_length, win_length};
std::shared_ptr<Tensor> spec_p;
RETURN_IF_NOT_OK(
Tensor::CreateEmpty(TensorShape({input->shape()[0], n_fft / 2 + 1, n_columns}), input->type(), &spec_p));
std::shared_ptr<Tensor> exp_complex;
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({n_fft / 2 + 1, win_length, 2}), input->type(), &exp_complex));
auto exp_complex_begin = exp_complex->begin<T>();
std::vector<int> exp_complex_slice = {win_length * 2, 2};
for (int i = 0; i < (n_fft / TWO + 1); i++) {
for (int k = 0; k <= win_length - 1; k++) {
ptrdiff_t exp_complex_offset_0 = i * exp_complex_slice[0] + k * exp_complex_slice[1];
ptrdiff_t exp_complex_offset_1 = exp_complex_offset_0 + 1;
*(exp_complex_begin + exp_complex_offset_0) = std::cos(twice * PI * i * k / win_length);
*(exp_complex_begin + exp_complex_offset_1) = std::sin(twice * PI * i * k / win_length);
}
}
for (int r = 0; r < input->shape()[0]; r++) {
for (int i = 0; i < (n_fft / TWO + 1); i++) {
for (int j = 0; j < n_columns; j++) {
T spec_f_0 = 0.;
T spec_f_1 = 0.;
ptrdiff_t exp_complex_offset_0 = i * exp_complex_slice[0];
for (int k = 0; k < win_length; k++) {
ptrdiff_t exp_complex_offset_1 = exp_complex_offset_0 + 1;
T exp_complex_a = *(exp_complex_begin + exp_complex_offset_0);
T exp_complex_b = *(exp_complex_begin + exp_complex_offset_1);
ptrdiff_t input_win_offset = r * input_win_slice[0] + j * input_win_slice[1] + k;
T input_value = *(input_win_begin + input_win_offset);
spec_f_0 += input_value * exp_complex_a;
spec_f_1 += -input_value * exp_complex_b;
exp_complex_offset_0 = exp_complex_offset_1 + 1;
}
ptrdiff_t spec_f_offset_0 = r * spec_f_slice[0] + i * spec_f_slice[1] + j * spec_f_slice[2];
ptrdiff_t spec_f_offset_1 = spec_f_offset_0 + 1;
*(spec_f_begin + spec_f_offset_0) = spec_f_0;
*(spec_f_begin + spec_f_offset_1) = spec_f_1;
}
}
}
CHECK_FAIL_RETURN_UNEXPECTED(win_sum != 0, "Window: the total value of window function can not be zero.");
if (normalized) {
for (int r = 0; r < input->shape()[0]; r++) {
for (int i = 0; i < (n_fft / TWO + 1); i++) {
for (int j = 0; j < n_columns; j++) {
ptrdiff_t spec_f_offset_0 = r * spec_f_slice[0] + i * spec_f_slice[1] + j * spec_f_slice[2];
ptrdiff_t spec_f_offset_1 = spec_f_offset_0 + 1;
T spec_norm_a = *(spec_f_begin + spec_f_offset_0);
T spec_norm_b = *(spec_f_begin + spec_f_offset_1);
*(spec_f_begin + spec_f_offset_0) = spec_norm_a / win_sum;
*(spec_f_begin + spec_f_offset_1) = spec_norm_b / win_sum;
}
}
}
}
std::shared_ptr<Tensor> output_onsided;
if (!onesided) {
RETURN_IF_NOT_OK(Onesided<T>(spec_f, &output_onsided, n_fft, n_columns));
if (power == 0) {
*output = output_onsided;
return Status::OK();
}
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({input->shape()[0], n_fft, n_columns}), input->type(), &spec_p));
RETURN_IF_NOT_OK(PowerStft<T>(output_onsided, &spec_p, power, n_fft, n_columns, n_fft));
*output = spec_p;
return Status::OK();
}
if (power == 0) {
*output = spec_f;
return Status::OK();
}
RETURN_IF_NOT_OK(PowerStft<T>(spec_f, &spec_p, power, n_fft, n_columns, n_fft / TWO + 1));
*output = spec_p;
return Status::OK();
}
template <typename T>
Status SpectrogramImpl(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, int pad,
WindowType window, int n_fft, int hop_length, int win_length, float power, bool normalized,
bool center, BorderType pad_mode, bool onesided) {
std::shared_ptr<Tensor> fft_window_tensor;
std::shared_ptr<Tensor> fft_window_later;
TensorShape shape = input->shape();
std::vector output_shape = shape.AsVector();
output_shape.pop_back();
int input_len = input->shape()[-1];
RETURN_IF_NOT_OK(input->Reshape(TensorShape({input->Size() / input_len, input_len})));
DataType data_type = input->type();
// get the windows
RETURN_IF_NOT_OK(Window(&fft_window_tensor, window, win_length));
if (win_length == 1) {
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({1}), DataType(DataType::DE_FLOAT32), &fft_window_tensor));
auto win = fft_window_tensor->begin<float>();
*(win) = 1;
}
// Pad window length
int pad_left = (n_fft - win_length) / 2;
int pad_right = n_fft - win_length - pad_left;
RETURN_IF_NOT_OK(fft_window_tensor->Reshape(TensorShape({1, win_length})));
RETURN_IF_NOT_OK(Pad<float>(fft_window_tensor, &fft_window_later, pad_left, pad_right, BorderType::kConstant));
RETURN_IF_NOT_OK(fft_window_later->Reshape(TensorShape({n_fft})));
int length = input_len + pad * 2 + n_fft;
std::shared_ptr<Tensor> input_data_tensor;
std::shared_ptr<Tensor> input_data_tensor_pad;
RETURN_IF_NOT_OK(
Tensor::CreateEmpty(TensorShape({input->shape()[0], input_len + pad * 2}), data_type, &input_data_tensor_pad));
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({input->shape()[0], length}), data_type, &input_data_tensor));
RETURN_IF_NOT_OK(Pad<T>(input, &input_data_tensor_pad, pad, pad, BorderType::kConstant));
if (center) {
RETURN_IF_NOT_OK(Pad<T>(input_data_tensor_pad, &input_data_tensor, n_fft / TWO, n_fft / TWO, pad_mode));
} else {
input_data_tensor = input_data_tensor_pad;
}
CHECK_FAIL_RETURN_UNEXPECTED(n_fft <= input_data_tensor->shape()[-1],
"Spectrogram: n_fft should be more than 0 and less than " +
std::to_string(input_data_tensor->shape()[-1]) +
", but got n_fft: " + std::to_string(n_fft) + ".");
// calculate the sliding times of the window function
int n_columns = 0;
while ((1 + n_columns++) * hop_length + n_fft <= input_data_tensor->shape()[-1]) {
}
std::shared_ptr<Tensor> stft_compute;
auto input_begin = input_data_tensor->begin<T>();
std::vector<int> input_win_slice = {n_columns * n_fft, n_fft};
auto iter_win = fft_window_later->begin<float>();
std::shared_ptr<Tensor> input_win;
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({input_data_tensor->shape()[0], n_columns, n_fft}),
input_data_tensor->type(), &input_win));
auto input_win_begin = input_win->begin<T>();
for (int r = 0; r < input_data_tensor->shape()[0]; r++) {
for (int j = 0; j < n_columns; j++) {
for (int k = 0; k < n_fft; k++) {
ptrdiff_t win_offset = k;
float win_value = *(iter_win + win_offset);
ptrdiff_t input_stft_offset = r * input_data_tensor->shape()[-1] + j * hop_length + k;
T input_value = *(input_begin + input_stft_offset);
ptrdiff_t input_win_offset = r * input_win_slice[0] + j * input_win_slice[1] + k;
*(input_win_begin + input_win_offset) = win_value * input_value;
}
}
}
RETURN_IF_NOT_OK(Stft<T>(input_win, &stft_compute, n_fft, fft_window_later, n_fft, hop_length, n_columns, normalized,
power, onesided));
if (onesided) {
output_shape.push_back(n_fft / TWO + 1);
} else {
output_shape.push_back(n_fft);
}
output_shape.push_back(n_columns);
if (power == 0) {
output_shape.push_back(TWO);
}
// reshape the output
RETURN_IF_NOT_OK(stft_compute->Reshape(TensorShape({output_shape})));
*output = stft_compute;
return Status::OK();
}
Status Spectrogram(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, int pad, WindowType window,
int n_fft, int hop_length, int win_length, float power, bool normalized, bool center,
BorderType pad_mode, bool onesided) {
TensorShape input_shape = input->shape();
CHECK_FAIL_RETURN_UNEXPECTED(
input->type().IsNumeric(),
"Spectrogram: input tensor type should be int, float or double, but got: " + input->type().ToString());
CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "Spectrogram: input tensor is not in shape of <..., time>.");
std::shared_ptr<Tensor> input_tensor;
if (input->type() != DataType::DE_FLOAT64) {
RETURN_IF_NOT_OK(TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32)));
return SpectrogramImpl<float>(input_tensor, output, pad, window, n_fft, hop_length, win_length, power, normalized,
center, pad_mode, onesided);
} else {
input_tensor = input;
return SpectrogramImpl<double>(input_tensor, output, pad, window, n_fft, hop_length, win_length, power, normalized,
center, pad_mode, onesided);
}
}
Status ComputeDeltas(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, int32_t win_length,
const BorderType &mode) {
RETURN_IF_NOT_OK(ValidateLowRank("ComputeDeltas", input, kDefaultAudioDim, "<..., freq, time>"));

20
mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.h
View File

@ -34,6 +34,7 @@
constexpr double PI = 3.141592653589793;
constexpr int kMinAudioDim = 1;
constexpr int kDefaultAudioDim = 2;
constexpr int TWO = 2;
namespace mindspore {
namespace dataset {
@ -147,7 +148,7 @@ Status Contrast(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *o
for (auto itr_in = input->begin<T>(); itr_in != input->end<T>(); itr_in++) {
T temp1, temp2 = 0;
// PI / 2 is half of the constant PI
temp1 = static_cast<T>(*itr_in) * (PI / 2);
temp1 = static_cast<T>(*itr_in) * (PI / TWO);
temp2 = enhancement_amount_value * std::sin(temp1 * 4);
*itr_out = std::sin(temp1 + temp2);
itr_out++;
@ -295,6 +296,23 @@ Status LFilter(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *ou
return Status::OK();
}
/// \brief Transform audio signal into spectrogram.
/// \param[in] n_fft Size of FFT, creates n_fft / 2 + 1 bins.
/// \param[in] win_length Window size.
/// \param[in] hop_length Length of hop between STFT windows.
/// \param[in] pad Two sided padding of signal.
/// \param[in] window A function to create a window tensor
/// that is applied/multiplied to each frame/window.
/// \param[in] power Exponent for the magnitude spectrogram.
/// \param[in] normalized Whether to normalize by magnitude after stft.
/// \param[in] center Whether to pad waveform on both sides.
/// \param[in] pad_mode Controls the padding method used when center is true.
/// \param[in] onesided Controls whether to return half of results to avoid redundancy.
/// \return Status code.
Status Spectrogram(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, int pad, WindowType window,
int n_fft, int hop_length, int win_length, float power, bool normalized, bool center,
BorderType pad_mode, bool onesided);
/// \brief Stretch STFT in time at a given rate, without changing the pitch.
/// \param input: Tensor of shape <..., freq, time>.
/// \param rate: Stretch factor.

54
mindspore/ccsrc/minddata/dataset/audio/kernels/spectrogram_op.cc Normal file
View File

@ -0,0 +1,54 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/audio/kernels/spectrogram_op.h"
#include "minddata/dataset/audio/kernels/audio_utils.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {
namespace dataset {
Status SpectrogramOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
return Spectrogram(input, output, pad_, window_, n_fft_, hop_length_, win_length_, power_, normalized_, center_,
pad_mode_, onesided_);
}
Status SpectrogramOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs));
outputs.clear();
constexpr int two = 2;
int length_ = inputs[0][-1] + pad_ * 2 + n_fft_;
int n_columns = 0;
while ((1 + n_columns++) * hop_length_ + n_fft_ <= length_) {
}
auto vec = inputs[0].AsVector();
vec.pop_back();
if (onesided_) {
vec.push_back(n_fft_ / two + 1);
} else {
vec.push_back(n_fft_);
}
vec.push_back(n_columns);
if (power_ == 0) {
vec.push_back(two);
}
outputs.emplace_back(TensorShape(vec));
if (!outputs.empty()) return Status::OK();
return Status(StatusCode::kMDUnexpectedError, "Spectrogram: input tensor is not in shape of <..., time>.");
}
} // namespace dataset
} // namespace mindspore

66
mindspore/ccsrc/minddata/dataset/audio/kernels/spectrogram_op.h Normal file
View File

@ -0,0 +1,66 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_KERNELS_SPECTROGRAM_OP_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_KERNELS_SPECTROGRAM_OP_H_
#include <memory>
#include <string>
#include <vector>
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/kernels/tensor_op.h"
namespace mindspore {
namespace dataset {
class SpectrogramOp : public TensorOp {
public:
SpectrogramOp(int32_t n_fft, int32_t win_length, int32_t hop_length, int32_t pad, WindowType window, float power,
bool normalized, bool center, BorderType pad_mode, bool onesided)
: n_fft_(n_fft),
win_length_(win_length),
hop_length_(hop_length),
pad_(pad),
window_(window),
power_(power),
normalized_(normalized),
center_(center),
pad_mode_(pad_mode),
onesided_(onesided) {}
~SpectrogramOp() = default;
Status Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) override;
std::string Name() const override { return kSpectrogramOp; };
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
private:
int32_t n_fft_;
int32_t win_length_;
int32_t hop_length_;
int32_t pad_;
WindowType window_;
float power_;
bool normalized_;
bool center_;
BorderType pad_mode_;
bool onesided_;
};
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_KERNELS_SPECTROGRAM_OP_H_

43
mindspore/ccsrc/minddata/dataset/include/dataset/audio.h
View File

@ -712,6 +712,49 @@ class MS_API SlidingWindowCmn final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Create a spectrogram from an audio signal.
class MS_API Spectrogram : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] n_fft Size of FFT, creates n_fft / 2 + 1 bins (Default: 400).
/// \param[in] win_length Window size (Default: 0, will use n_fft).
/// \param[in] hop_length Length of hop between STFT windows (Default: 0, will use win_length / 2).
/// \param[in] pad Two sided padding of signal (Default: 0).
/// \param[in] window Window function that is applied/multiplied to each frame/window,
/// which can be WindowType::kBartlett, WindowType::kBlackman, WindowType::kHamming,
/// WindowType::kHann or WindowType::kKaiser (Default: WindowType::kHann).
/// \param[in] power Exponent for the magnitude spectrogram, which must be greater than or equal to 0 (Default: 2.0).
/// \param[in] normalized Whether to normalize by magnitude after stft (Default: false).
/// \param[in] center Whether to pad waveform on both sides (Default: true).
/// \param[in] pad_mode Controls the padding method used when center is true (Default: BorderType::kReflect).
/// \param[in] onesided Controls whether to return half of results to avoid redundancy (Default: true).
Spectrogram(int32_t n_fft = 400, int32_t win_length = 0, int32_t hop_length = 0, int32_t pad = 0,
WindowType window = WindowType::kHann, float power = 2.0, bool normalized = false, bool center = true,
BorderType pad_mode = BorderType::kReflect, bool onesided = true);
/// \brief Destructor.
~Spectrogram() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
int32_t n_fft_;
int32_t win_length_;
int32_t hop_length_;
int32_t pad_;
WindowType window_;
float power_;
bool normalized_;
bool center_;
BorderType pad_mode_;
bool onesided_;
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief TimeMasking TensorTransform.
/// \notes Apply masking to a spectrogram in the time domain.
class MS_API TimeMasking final : public TensorTransform {

9
mindspore/ccsrc/minddata/dataset/include/dataset/constants.h
View File

@ -133,6 +133,15 @@ enum class MS_API FixRotationAngle {
k270Degree = 8, ///< Rotate 270 degree.
};
/// \brief Possible types for windows function.
enum class MS_API WindowType {
kBartlett = 0, ///< Bartlett window function.
kBlackman = 1, ///< Blackman window function.
kHamming = 2, ///< Hamming window function.
kHann = 3, ///< Hann window function.
kKaiser = 4 ///< Kaiser window function.
};
/// \brief Possible options for Image format types in a batch.
enum class MS_API ImageBatchFormat {
kNHWC = 0, ///< Indicate the input batch is of NHWC format.

1
mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h
View File

@ -175,6 +175,7 @@ constexpr char kOverdriveOp[] = "OverdriveOp";
constexpr char kPhaserOp[] = "PhaserOp";
constexpr char kRiaaBiquadOp[] = "RiaaBiquadOp";
constexpr char kSlidingWindowCmnOp[] = "SlidingWindowCmnOp";
constexpr char kSpectrogramOp[] = "SpectrogramOp";
constexpr char kTimeMaskingOp[] = "TimeMaskingOp";
constexpr char kTimeStretchOp[] = "TimeStretchOp";
constexpr char kTrebleBiquadOp[] = "TrebleBiquadOp";

59
mindspore/dataset/audio/transforms.py
View File

@ -23,13 +23,13 @@ import numpy as np
import mindspore._c_dataengine as cde
from ..transforms.c_transforms import TensorOperation
from .utils import BorderType, FadeShape, GainType, Interpolation, Modulation, ScaleType
from .utils import BorderType, FadeShape, GainType, Interpolation, Modulation, ScaleType, WindowType
from .validators import check_allpass_biquad, check_amplitude_to_db, check_band_biquad, check_bandpass_biquad, \
check_bandreject_biquad, check_bass_biquad, check_biquad, check_complex_norm, check_compute_deltas, \
check_contrast, check_db_to_amplitude, check_dc_shift, check_deemph_biquad, check_detect_pitch_frequency, \
check_equalizer_biquad, check_fade, check_flanger, check_highpass_biquad, check_lfilter, check_lowpass_biquad, \
check_magphase, check_masking, check_mu_law_coding, check_overdrive, check_phaser, check_riaa_biquad, \
check_sliding_window_cmn, check_time_stretch, check_treble_biquad, check_vol
check_sliding_window_cmn, check_spectrogram, check_time_stretch, check_treble_biquad, check_vol
class AudioTensorOperation(TensorOperation):
@ -947,6 +947,61 @@ class SlidingWindowCmn(AudioTensorOperation):
return cde.SlidingWindowCmnOperation(self.cmn_window, self.min_cmn_window, self.center, self.norm_vars)
DE_C_WINDOW_TYPE = {WindowType.BARTLETT: cde.WindowType.DE_BARTLETT,
WindowType.BLACKMAN: cde.WindowType.DE_BLACKMAN,
WindowType.HAMMING: cde.WindowType.DE_HAMMING,
WindowType.HANN: cde.WindowType.DE_HANN,
WindowType.KAISER: cde.WindowType.DE_KAISER}
class Spectrogram(TensorOperation):
"""
Create a spectrogram from an audio signal.
Args:
n_fft (int, optional): Size of FFT, creates n_fft // 2 + 1 bins (default=400).
win_length (int, optional): Window size (default=None, will use n_fft).
hop_length (int, optional): Length of hop between STFT windows (default=None, will use win_length // 2).
pad (int): Two sided padding of signal (default=0).
window (WindowType, optional): Window function that is applied/multiplied to each frame/window,
which can be WindowType.BARTLETT, WindowType.BLACKMAN, WindowType.HAMMING, WindowType.HANN
or WindowType.KAISER (default=WindowType.HANN).
power (float, optional): Exponent for the magnitude spectrogram, which must be greater
than or equal to 0, e.g., 1 for energy, 2 for power, etc. (default=2.0).
normalized (bool, optional): Whether to normalize by magnitude after stft (default=False).
center (bool, optional): Whether to pad waveform on both sides (default=True).
pad_mode (BorderType, optional): Controls the padding method used when center is True,
which can be BorderType.REFLECT, BorderType.CONSTANT, BorderType.EDGE, BorderType.SYMMETRIC
(default=BorderType.REFLECT).
onesided (bool, optional): Controls whether to return half of results to avoid redundancy (default=True).
Examples:
>>> waveform = np.random.random([5, 10, 20])
>>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"])
>>> transforms = [audio.Spectrogram()]
>>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"])
"""
@check_spectrogram
def __init__(self, n_fft=400, win_length=None, hop_length=None, pad=0, window=WindowType.HANN, power=2.0,
normalized=False, center=True, pad_mode=BorderType.REFLECT, onesided=True):
self.n_fft = n_fft
self.win_length = win_length if win_length else n_fft
self.hop_length = hop_length if hop_length else self.win_length // 2
self.pad = pad
self.window = window
self.power = power
self.normalized = normalized
self.center = center
self.pad_mode = pad_mode
self.onesided = onesided
def parse(self):
return cde.SpectrogramOperation(self.n_fft, self.win_length, self.hop_length, self.pad,
DE_C_WINDOW_TYPE[self.window], self.power, self.normalized,
self.center, DE_C_BORDER_TYPE[self.pad_mode], self.onesided)
class TimeMasking(AudioTensorOperation):
"""
Apply masking to a spectrogram in the time domain.

20
mindspore/dataset/audio/utils.py
View File

@ -160,3 +160,23 @@ class BorderType(str, Enum):
EDGE: str = "edge"
REFLECT: str = "reflect"
SYMMETRIC: str = "symmetric"
class WindowType(str, Enum):
"""
Window Function types,
Possible enumeration values are: WindowType.BARTLETT, WindowType.BLACKMAN, WindowType.HAMMING, WindowType.HANN,
WindowType.KAISER.
- WindowType.BARTLETT: means the type of window function is bartlett.
- WindowType.BLACKMAN: means the type of window function is blackman.
- WindowType.HAMMING: means the type of window function is hamming.
- WindowType.HANN: means the type of window function is hann.
- WindowType.KAISER: means the type of window function is kaiser.
"""
BARTLETT: str = "bartlett"
BLACKMAN: str = "blackman"
HAMMING: str = "hamming"
HANN: str = "hann"
KAISER: str = "kaiser"

36
mindspore/dataset/audio/validators.py
View File

@ -21,7 +21,7 @@ from functools import wraps
from mindspore.dataset.core.validator_helpers import check_float32, check_float32_not_zero, check_int32, \
check_int32_not_zero, check_list_same_size, check_non_negative_float32, check_non_negative_int32, \
check_pos_float32, check_pos_int32, check_value, INT32_MAX, parse_user_args, type_check
from .utils import BorderType, FadeShape, GainType, Interpolation, Modulation, ScaleType
from .utils import BorderType, FadeShape, GainType, Interpolation, Modulation, ScaleType, WindowType
def check_amplitude_to_db(method):
@ -356,6 +356,40 @@ def check_riaa_biquad(method):
return new_method
def check_spectrogram(method):
"""Wrapper method to check the parameters of Spectrogram."""
@wraps(method)
def new_method(self, *args, **kwargs):
[n_fft, win_length, hop_length, pad, window, power,
normalized, center, pad_mode, onesided], _ = parse_user_args(method, *args, **kwargs)
type_check(n_fft, (int,), "n_fft")
check_pos_int32(n_fft, "n_fft")
if win_length is not None:
type_check(win_length, (int,), "win_length")
check_pos_int32(win_length, "win_length")
if win_length > n_fft:
raise ValueError(
"Input win_length should be no more than n_fft, but got win_length: {0} and n_fft: {1}.".format(
win_length, n_fft))
if hop_length is not None:
type_check(hop_length, (int,), "hop_length")
check_pos_int32(hop_length, "hop_length")
type_check(pad, (int,), "pad")
check_non_negative_int32(pad, "pad")
type_check(window, (WindowType,), "window")
type_check(power, (int, float), "power")
check_non_negative_float32(power, "power")
type_check(normalized, (bool,), "normalized")
type_check(center, (bool,), "center")
type_check(onesided, (bool,), "onesided")
type_check(pad_mode, (BorderType,), "pad_mode")
return method(self, *args, **kwargs)
return new_method
def check_time_stretch(method):
"""Wrapper method to check the parameters of TimeStretch."""

1
tests/ut/cpp/dataset/CMakeLists.txt
View File

@ -87,6 +87,7 @@ SET(DE_UT_SRCS
decode_op_test.cc
distributed_sampler_test.cc
equalize_op_test.cc
execute_test.cc
execution_tree_test.cc
fill_op_test.cc
c_api_vision_gaussian_blur_test.cc

431
tests/ut/cpp/dataset/c_api_audio_r_to_z_test.cc
View File

@ -255,6 +255,437 @@ TEST_F(MindDataTestPipeline, TestSlidingWindowCmnWrongArgs) {
EXPECT_EQ(iter_2, nullptr);
}
/// Feature: Spectrogram.
/// Description: test pipeline.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramDefault) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramDefault.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {1, 60}));
std::shared_ptr<Dataset> ds = RandomData(8, schema);
EXPECT_NE(ds, nullptr);
auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHann, 2.0, false, true, BorderType::kReflect,
true});
auto ds1 = ds->Map({spectrogram}, {"waveform"});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 8);
iter->Stop();
}
/// Feature: Spectrogram.
/// Description: test parameter: onesided.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramOnesidedFalse) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramOnesidedFalse.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {3, 50}));
std::shared_ptr<Dataset> ds = RandomData(8, schema);
EXPECT_NE(ds, nullptr);
auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHann, 2.0, false, true, BorderType::kReflect,
false});
auto ds1 = ds->Map({spectrogram}, {"waveform"});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 8);
iter->Stop();
}
/// Feature: Spectrogram.
/// Description: test parameter: center.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramCenterFalse) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramCenterFalse.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeInt32, {2, 3, 50}));
std::shared_ptr<Dataset> ds = RandomData(8, schema);
EXPECT_NE(ds, nullptr);
auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHann, 2.0, false, false, BorderType::kReflect,
true});
auto ds1 = ds->Map({spectrogram}, {"waveform"});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 8);
iter->Stop();
}
/// Feature: Spectrogram.
/// Description: test parameter: normaliezd.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramNormalizedTrue) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramNormalizedTrue.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeInt32, {5, 40}));
std::shared_ptr<Dataset> ds = RandomData(8, schema);
EXPECT_NE(ds, nullptr);
auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHann, 2.0, true, true, BorderType::kReflect,
true});
auto ds1 = ds->Map({spectrogram}, {"waveform"});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 8);
iter->Stop();
}
/// Feature: Spectrogram.
/// Description: test parameter: window.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramWindowHamming) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramWindowHamming.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat64, {3, 50}));
std::shared_ptr<Dataset> ds = RandomData(8, schema);
EXPECT_NE(ds, nullptr);
auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHamming, 2.0, false, true,
BorderType::kReflect, true});
auto ds1 = ds->Map({spectrogram}, {"waveform"});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 8);
iter->Stop();
}
/// Feature: Spectrogram.
/// Description: test parameter: pad_mode.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramPadmodeEdge) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramPadmodeEdge.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeInt32, {3, 4, 50}));
std::shared_ptr<Dataset> ds = RandomData(8, schema);
EXPECT_NE(ds, nullptr);
auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHamming, 2.0, false, true,
BorderType::kEdge, true});
auto ds1 = ds->Map({spectrogram}, {"waveform"});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 8);
iter->Stop();
}
/// Feature: Spectrogram.
/// Description: test parameter: power.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramPower0) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramPower0.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeInt32, {3, 50}));
std::shared_ptr<Dataset> ds = RandomData(8, schema);
EXPECT_NE(ds, nullptr);
auto spectrogram = audio::Spectrogram({40, 40, 20, 0, WindowType::kHamming, 0, false, true, BorderType::kReflect,
true});
auto ds1 = ds->Map({spectrogram}, {"waveform"});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 8);
iter->Stop();
}
/// Feature: Spectrogram.
/// Description: test parameter: n_fft.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramNfft50) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramNfft600.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {1, 60}));
std::shared_ptr<Dataset> ds = RandomData(8, schema);
EXPECT_NE(ds, nullptr);
auto spectrogram = audio::Spectrogram({50, 40, 20, 0, WindowType::kHann, 2.0, false, true, BorderType::kReflect,
true});
auto ds1 = ds->Map({spectrogram}, {"waveform"});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 8);
iter->Stop();
}
/// Feature: Spectrogram.
/// Description: test parameter: pad.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramPad10) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramPad50.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {3, 50}));
std::shared_ptr<Dataset> ds = RandomData(8, schema);
EXPECT_NE(ds, nullptr);
auto spectrogram = audio::Spectrogram({40, 40, 20, 10, WindowType::kHann, 2.0, false, true, BorderType::kReflect,
true});
auto ds1 = ds->Map({spectrogram}, {"waveform"});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 8);
iter->Stop();
}
/// Feature: Spectrogram.
/// Description: test parameter: win_length.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramWinlength30) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramWinlength300.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {2, 2, 50}));
std::shared_ptr<Dataset> ds = RandomData(8, schema);
EXPECT_NE(ds, nullptr);
auto spectrogram = audio::Spectrogram({40, 30, 20, 0, WindowType::kHann, 2.0, false, true, BorderType::kReflect,
true});
auto ds1 = ds->Map({spectrogram}, {"waveform"});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 8);
iter->Stop();
}
/// Feature: Spectrogram.
/// Description: test parameters.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramHoplength30) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramHoplength300.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("waveform", mindspore::DataType::kNumberTypeFloat32, {2, 50}));
std::shared_ptr<Dataset> ds = RandomData(8, schema);
EXPECT_NE(ds, nullptr);
auto spectrogram = audio::Spectrogram({40, 40, 30, 0, WindowType::kHann, 2.0, false, true, BorderType::kReflect,
true});
auto ds1 = ds->Map({spectrogram}, {"waveform"});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter = ds1->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
uint64_t i = 0;
while (row.size() != 0) {
ASSERT_OK(iter->GetNextRow(&row));
i++;
}
EXPECT_EQ(i, 8);
iter->Stop();
}
/// Feature: Spectrogram.
/// Description: test some invalid parameters.
/// Expectation: success.
TEST_F(MindDataTestPipeline, TestSpectrogramWrongArgs) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSpectrogramWrongArgs.";
std::shared_ptr<SchemaObj> schema = Schema();
// Original waveform
ASSERT_OK(schema->add_column("col", mindspore::DataType::kNumberTypeFloat32, {1, 50}));
std::shared_ptr<Dataset> ds = RandomData(50, schema);
std::shared_ptr<Dataset> ds01;
std::shared_ptr<Dataset> ds02;
std::shared_ptr<Dataset> ds03;
std::shared_ptr<Dataset> ds04;
std::shared_ptr<Dataset> ds05;
std::shared_ptr<Dataset> ds06;
EXPECT_NE(ds, nullptr);
// Check n_fft
MS_LOG(INFO) << "n_fft is zero.";
auto spectrogram_op_01 = audio::Spectrogram({0, 40, 20, 0, WindowType::kHann, 2.0, false, true,
BorderType::kReflect, true});
ds01 = ds->Map({spectrogram_op_01});
EXPECT_NE(ds01, nullptr);
std::shared_ptr<Iterator> iter01 = ds01->CreateIterator();
EXPECT_EQ(iter01, nullptr);
// Check win_length
MS_LOG(INFO) << "win_length is -1.";
auto spectrogram_op_02 = audio::Spectrogram({40, -1, 20, 0, WindowType::kHann, 2.0, false, true,
BorderType::kReflect, true});
ds02 = ds->Map({spectrogram_op_02});
EXPECT_NE(ds02, nullptr);
std::shared_ptr<Iterator> iter02 = ds02->CreateIterator();
EXPECT_EQ(iter02, nullptr);
// Check hop_length
MS_LOG(INFO) << "hop_length is -1.";
auto spectrogram_op_03 = audio::Spectrogram({40, 40, -1, 0, WindowType::kHann, 2.0, false, true,
BorderType::kReflect, true});
ds03 = ds->Map({spectrogram_op_03});
EXPECT_NE(ds03, nullptr);
std::shared_ptr<Iterator> iter03 = ds03->CreateIterator();
EXPECT_EQ(iter03, nullptr);
// Check power
MS_LOG(INFO) << "power is -1.";
auto spectrogram_op_04 = audio::Spectrogram({40, 40, 20, 0, WindowType::kHann, -1, false, true,
BorderType::kReflect, true});
ds04 = ds->Map({spectrogram_op_04});
EXPECT_NE(ds04, nullptr);
std::shared_ptr<Iterator> iter04 = ds04->CreateIterator();
EXPECT_EQ(iter04, nullptr);
// Check pad
MS_LOG(INFO) << "pad is -1.";
auto spectrogram_op_05 = audio::Spectrogram({40, 40, 20, -1, WindowType::kHann, 2.0, false, true,
BorderType::kReflect, true});
ds05 = ds->Map({spectrogram_op_05});
EXPECT_NE(ds05, nullptr);
std::shared_ptr<Iterator> iter05 = ds05->CreateIterator();
EXPECT_EQ(iter05, nullptr);
// Check n_fft and win)length
MS_LOG(INFO) << "n_fft is 40, win_length is 50.";
auto spectrogram_op_06 = audio::Spectrogram({40, 50, 20, -1, WindowType::kHann, 2.0, false, true,
BorderType::kReflect, true});
ds06 = ds->Map({spectrogram_op_06});
EXPECT_NE(ds06, nullptr);
std::shared_ptr<Iterator> iter06 = ds06->CreateIterator();
EXPECT_EQ(iter06, nullptr);
}
TEST_F(MindDataTestPipeline, TestTimeMaskingPipeline) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTimeMaskingPipeline.";
// Original waveform

17
tests/ut/cpp/dataset/execute_test.cc
View File

@ -2175,3 +2175,20 @@ TEST_F(MindDataTestExecute, TestAutoAugmentEager) {
Status rc = transform(image, &image);
EXPECT_EQ(rc, Status::OK());
}
/// Feature: Spectrogram.
/// Description: test Spectrogram in eager mode.
/// Expectation: the data is processed successfully.
TEST_F(MindDataTestExecute, TestSpectrogramEager) {
MS_LOG(INFO) << "Doing MindDataTestExecute-SpectrogramEager.";
std::shared_ptr<Tensor> test_input_tensor;
std::vector<double> waveform = {1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1};
ASSERT_OK(Tensor::CreateFromVector(waveform, TensorShape({1, (long)waveform.size()}), &test_input_tensor));
auto input_tensor = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(test_input_tensor));
std::shared_ptr<TensorTransform> spectrogram = std::make_shared<audio::Spectrogram>(8, 8, 4, 0, WindowType::kHann,
2., false, true,
BorderType::kReflect, true);
auto transform = Execute({spectrogram});
Status rc = transform({input_tensor}, &input_tensor);
ASSERT_TRUE(rc.IsOk());
}

432
tests/ut/python/dataset/test_spectrogram.py Normal file
View File

@ -0,0 +1,432 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Testing Spectrogram Python API
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.audio.transforms as audio
from mindspore import log as logger
from mindspore.dataset.audio.utils import WindowType, BorderType
def count_unequal_element(data_expected, data_me, rtol, atol):
""" Precision calculation func """
assert data_expected.shape == data_me.shape
total_count = len(data_expected.flatten())
error = np.abs(data_expected - data_me)
greater = np.greater(error, atol + np.abs(data_expected) * rtol)
loss_count = np.count_nonzero(greater)
assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format(
data_expected[greater], data_me[greater], error[greater])
def test_spectrogram_pipeline():
"""
Feature: mindspore pipeline mode normal testcase: spectrogram op.
Description: input audio signal to test pipeline.
Expectation: success.
"""
logger.info("test_spectrogram_pipeline")
wav = [[[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]]]
dataset = ds.NumpySlicesDataset(wav, column_names=["audio"], shuffle=False)
out = audio.Spectrogram(n_fft=8)
dataset = dataset.map(operations=out, input_columns=["audio"], output_columns=["Spectrogram"],
column_order=['Spectrogram'])
result = np.array([[[2.8015e+01, 1.2100e+02, 3.1354e+02, 1.6900e+02, 2.5000e+01,
1.0843e+01, 1.2100e+02, 3.3150e+02],
[3.2145e+00, 3.3914e+01, 9.4728e+01, 4.5914e+01, 9.9142e+00,
4.5858e+00, 3.3914e+01, 9.5685e+01],
[1.0000e+00, 1.7157e-01, 1.5000e+00, 1.7157e-01, 1.7157e-01,
5.0000e-01, 1.7157e-01, 7.5000e-01],
[4.2893e-02, 2.5736e-01, 5.8579e-01, 2.5736e-01, 2.5736e-01,
5.8579e-01, 2.5736e-01, 1.2868e-01],
[5.0000e-01, 1.0000e+00, 8.5787e-02, 1.0000e+00, 1.0000e+00,
5.0000e-01, 1.0000e+00, 6.2868e-01]]])
for data1 in dataset.create_dict_iterator(num_epochs=1, output_numpy=True):
count_unequal_element(data1["Spectrogram"], result, 0.0001, 0.0001)
def test_spectrogram_eager():
"""
Feature: mindspore eager mode normal testcase: spectrogram op.
Description: input audio signal to test eager.
Expectation: success.
"""
logger.info("test_spectrogram_eager")
wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]])
out = audio.Spectrogram(n_fft=8, win_length=8, window=WindowType.HANN,
pad_mode=BorderType.REFLECT)(np.array(wav, dtype="float"))
result = np.array([[[2.8015e+01, 1.2100e+02, 3.1354e+02, 1.6900e+02, 2.5000e+01,
1.0843e+01, 1.2100e+02, 3.3150e+02],
[3.2145e+00, 3.3914e+01, 9.4728e+01, 4.5914e+01, 9.9142e+00,
4.5858e+00, 3.3914e+01, 9.5685e+01],
[1.0000e+00, 1.7157e-01, 1.5000e+00, 1.7157e-01, 1.7157e-01,
5.0000e-01, 1.7157e-01, 7.5000e-01],
[4.2893e-02, 2.5736e-01, 5.8579e-01, 2.5736e-01, 2.5736e-01,
5.8579e-01, 2.5736e-01, 1.2868e-01],
[5.0000e-01, 1.0000e+00, 8.5787e-02, 1.0000e+00, 1.0000e+00,
5.0000e-01, 1.0000e+00, 6.2868e-01]]])
count_unequal_element(out, result, 0.0001, 0.0001)
def test_spectrogram_window_hamming_padmode_constant():
"""
Feature: test spectrogram parameter: window, pad_mode.
Description: test parameter.
Expectation: success.
"""
logger.info("test_spectrogram_window_hamming_padmode_constant")
wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]])
out = audio.Spectrogram(n_fft=8, window=WindowType.HAMMING,
pad_mode=BorderType.CONSTANT)(np.array(wav, dtype="float"))
result = np.array([[[1.1389e+01, 1.3736e+02, 3.5534e+02, 2.0164e+02, 3.0914e+01,
1.3465e+01, 1.3736e+02, 2.5064e+02],
[5.6934e+00, 3.1860e+01, 8.5291e+01, 3.8484e+01, 9.1907e+00,
4.5576e+00, 3.1860e+01, 1.0027e+02],
[1.9633e-01, 7.4475e-02, 1.2696e+00, 7.4475e-02, 7.4475e-02,
4.2320e-01, 7.4475e-02, 1.1765e+01],
[5.0570e-01, 3.8456e-01, 6.8326e-01, 3.8456e-01, 3.8456e-01,
6.8326e-01, 3.8456e-01, 5.4501e+00],
[6.1665e-01, 8.4640e-01, 7.2610e-02, 8.4640e-01, 8.4640e-01,
4.2320e-01, 8.4640e-01, 1.0642e+00]]])
count_unequal_element(out, result, 0.0001, 0.0001)
def test_spectrogram_nfft_10_window_bartlett_padmode_edge():
"""
Feature: test spectrogram parameter: n_fft, window, pad_mode.
Description: test parameter.
Expectation: success.
"""
logger.info("test_spectrogram_nfft_10_window_bartlett_padmode_edge")
wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]])
out = audio.Spectrogram(n_fft=10, window=WindowType.BARTLETT,
pad_mode=BorderType.EDGE)(np.array(wav, dtype="float"))
result = np.array([[[4.0960e+01, 2.6244e+02, 4.0000e+02, 7.7440e+01, 2.5000e+01,
2.6244e+02, 5.9536e+02],
[4.7655e+00, 5.6721e+01, 9.4681e+01, 2.3822e+01, 5.9597e+00,
5.6721e+01, 1.1597e+02],
[5.3889e-01, 5.8360e-03, 9.2361e-01, 5.8359e-03, 9.2361e-01,
5.8360e-03, 2.4944e-01],
[1.1449e-01, 9.9859e-01, 3.1897e-01, 2.9828e-01, 1.0403e+00,
9.9859e-01, 1.3072e+00],
[1.8111e-01, 2.7416e-01, 4.7639e-01, 2.7416e-01, 4.7639e-01,
2.7416e-01, 7.0557e-02],
[6.4000e-01, 3.6000e-01, 0.0000e+00, 2.5600e+00, 1.0000e+00,
3.6000e-01, 1.4400e+00]]])
count_unequal_element(out, result, 0.0001, 0.0001)
def test_spectrogram_onsided_false():
"""
Feature: test spectrogram parameter: onesided.
Description: test parameter.
Expectation: success.
"""
logger.info("test_spectrogram_onsided_false")
wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]])
out = audio.Spectrogram(n_fft=10, window=WindowType.BARTLETT,
pad_mode=BorderType.EDGE, onesided=False)(np.array(wav, dtype="float"))
result = np.array([[[4.0960e+01, 2.6244e+02, 4.0000e+02, 7.7440e+01, 2.5000e+01,
2.6244e+02, 5.9536e+02],
[4.7655e+00, 5.6721e+01, 9.4681e+01, 2.3822e+01, 5.9597e+00,
5.6721e+01, 1.1597e+02],
[5.3889e-01, 5.8360e-03, 9.2361e-01, 5.8359e-03, 9.2361e-01,
5.8360e-03, 2.4944e-01],
[1.1449e-01, 9.9859e-01, 3.1897e-01, 2.9828e-01, 1.0403e+00,
9.9859e-01, 1.3072e+00],
[1.8111e-01, 2.7416e-01, 4.7639e-01, 2.7416e-01, 4.7639e-01,
2.7416e-01, 7.0557e-02],
[6.4000e-01, 3.6000e-01, 0.0000e+00, 2.5600e+00, 1.0000e+00,
3.6000e-01, 1.4400e+00],
[1.8111e-01, 2.7416e-01, 4.7639e-01, 2.7416e-01, 4.7639e-01,
2.7416e-01, 7.0557e-02],
[1.1449e-01, 9.9859e-01, 3.1897e-01, 2.9828e-01, 1.0403e+00,
9.9859e-01, 1.3072e+00],
[5.3889e-01, 5.8360e-03, 9.2361e-01, 5.8359e-03, 9.2361e-01,
5.8360e-03, 2.4944e-01],
[4.7655e+00, 5.6721e+01, 9.4681e+01, 2.3822e+01, 5.9597e+00,
5.6721e+01, 1.1597e+02]]])
count_unequal_element(out, result, 0.0001, 0.0001)
def test_spectrogram_power_0():
"""
Feature: test spectrogram parameter: power.
Description: test parameter.
Expectation: success.
"""
logger.info("test_spectrogram_power_0")
wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]])
out = audio.Spectrogram(n_fft=8, window=WindowType.HANN,
pad_mode=BorderType.REFLECT, power=0)(np.array(wav, dtype="float"))
result = np.array([[[[5.2929e+00, 0.0000e+00],
[1.1000e+01, 0.0000e+00],
[1.7707e+01, 0.0000e+00],
[1.3000e+01, 0.0000e+00],
[5.0000e+00, 0.0000e+00],
[3.2929e+00, 0.0000e+00],
[1.1000e+01, 0.0000e+00],
[1.8207e+01, 0.0000e+00]],
[[-1.7929e+00, -2.5288e-07],
[-5.5000e+00, 1.9142e+00],
[-9.7071e+00, 7.0711e-01],
[-6.5000e+00, -1.9142e+00],
[-2.5000e+00, -1.9142e+00],
[-1.2929e+00, 1.7071e+00],
[-5.5000e+00, 1.9142e+00],
[-9.7071e+00, 1.2071e+00]],
[[-1.0000e+00, 0.0000e+00],
[0.0000e+00, -4.1421e-01],
[1.0000e+00, -7.0711e-01],
[0.0000e+00, 4.1421e-01],
[0.0000e+00, 4.1421e-01],
[0.0000e+00, -7.0711e-01],
[0.0000e+00, -4.1421e-01],
[5.0000e-01, -7.0711e-01]],
[[-2.0711e-01, -2.5288e-07],
[-5.0000e-01, -8.5787e-02],
[-2.9289e-01, 7.0711e-01],
[5.0000e-01, 8.5786e-02],
[5.0000e-01, 8.5786e-02],
[-7.0711e-01, -2.9289e-01],
[-5.0000e-01, -8.5787e-02],
[-2.9289e-01, 2.0711e-01]],
[[7.0711e-01, 0.0000e+00],
[1.0000e+00, 0.0000e+00],
[2.9289e-01, 0.0000e+00],
[-1.0000e+00, 0.0000e+00],
[-1.0000e+00, 0.0000e+00],
[7.0711e-01, 0.0000e+00],
[1.0000e+00, 0.0000e+00],
[7.9289e-01, 0.0000e+00]]]])
count_unequal_element(out, result, 0.0001, 0.0001)
def test_spectrogram_center_false():
"""
Feature: test spectrogram parameter: center.
Description: test parameter.
Expectation: success.
"""
logger.info("test_spectrogram_center_false")
wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]])
out = audio.Spectrogram(n_fft=8, window=WindowType.HANN,
center=False, pad_mode=BorderType.REFLECT)(np.array(wav, dtype="float"))
result = np.array([[[1.2100e+02, 3.1354e+02, 1.6900e+02, 2.5000e+01, 1.0843e+01,
1.2100e+02],
[3.3914e+01, 9.4728e+01, 4.5914e+01, 9.9142e+00, 4.5858e+00,
3.3914e+01],
[1.7157e-01, 1.5000e+00, 1.7157e-01, 1.7157e-01, 5.0000e-01,
1.7157e-01],
[2.5736e-01, 5.8579e-01, 2.5736e-01, 2.5736e-01, 5.8579e-01,
2.5736e-01],
[1.0000e+00, 8.5787e-02, 1.0000e+00, 1.0000e+00, 5.0000e-01,
1.0000e+00]]])
count_unequal_element(out, result, 0.0001, 0.0001)
def test_spectrogram_normalized_true():
"""
Feature: test spectrogram parameter: normalized.
Description: test parameter.
Expectation: success.
"""
logger.info("test_spectrogram_normalized_true")
wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]])
out = audio.Spectrogram(n_fft=8, window=WindowType.HANN,
center=False, normalized=True, pad_mode=BorderType.REFLECT)(np.array(wav, dtype="float"))
result = np.array([[[4.0333e+01, 1.0451e+02, 5.6333e+01, 8.3333e+00, 3.6144e+00,
4.0333e+01],
[1.1305e+01, 3.1576e+01, 1.5305e+01, 3.3047e+00, 1.5286e+00,
1.1305e+01],
[5.7191e-02, 5.0000e-01, 5.7191e-02, 5.7191e-02, 1.6667e-01,
5.7191e-02],
[8.5786e-02, 1.9526e-01, 8.5786e-02, 8.5786e-02, 1.9526e-01,
8.5786e-02],
[3.3333e-01, 2.8596e-02, 3.3333e-01, 3.3333e-01, 1.6667e-01,
3.3333e-01]]])
count_unequal_element(out, result, 0.0001, 0.0001)
def test_spectrogram_inputrank_3():
"""
Feature: test spectrogram parameter: input rank.
Description: test input rank.
Expectation: success.
"""
logger.info("test_spectrogram_inputrank_3")
wav = np.array([[[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1]],
[[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1]]])
out = audio.Spectrogram(n_fft=8, window=WindowType.HANN, pad_mode=BorderType.REFLECT)(np.array(wav, dtype="float"))
result = np.array([[[[2.8015e+01, 1.2100e+02, 3.1354e+02, 1.6900e+02, 3.3558e+01],
[3.2145e+00, 3.3914e+01, 9.4728e+01, 4.5914e+01, 6.7145e+00],
[1.0000e+00, 1.7157e-01, 1.5000e+00, 1.7157e-01, 7.5000e-01],
[4.2893e-02, 2.5736e-01, 5.8579e-01, 2.5736e-01, 1.2868e-01],
[5.0000e-01, 1.0000e+00, 8.5787e-02, 1.0000e+00, 6.2868e-01]]],
[[[2.8015e+01, 1.2100e+02, 3.1354e+02, 1.6900e+02, 3.3558e+01],
[3.2145e+00, 3.3914e+01, 9.4728e+01, 4.5914e+01, 6.7145e+00],
[1.0000e+00, 1.7157e-01, 1.5000e+00, 1.7157e-01, 7.5000e-01],
[4.2893e-02, 2.5736e-01, 5.8579e-01, 2.5736e-01, 1.2868e-01],
[5.0000e-01, 1.0000e+00, 8.5787e-02, 1.0000e+00, 6.2868e-01]]]])
count_unequal_element(out, result, 0.0001, 0.0001)
def test_spectrogram_winlength_7():
"""
Feature: test spectrogram parameter: win_length.
Description: test parameter.
Expectation: success.
"""
logger.info("test_spectrogram_winlength_7")
wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]])
out = audio.Spectrogram(n_fft=8, win_length=7, window=WindowType.HANN,)(np.array(wav, dtype="float"))
result = np.array([[[2.0140e+01, 4.9000e+01, 1.5006e+02, 2.5284e+02, 1.5006e+02,
4.9000e+01, 4.5220e+00, 1.2250e+01, 7.6562e+01, 1.9600e+02,
2.7265e+02],
[5.0272e+00, 1.9488e+01, 5.5103e+01, 1.0153e+02, 5.5103e+01,
1.9488e+01, 2.0179e+00, 6.5089e+00, 2.9144e+01, 7.1406e+01,
1.0662e+02],
[4.2200e-01, 5.4020e-01, 1.0554e+00, 4.8321e+00, 1.0554e+00,
5.4020e-01, 9.6867e-01, 4.0345e-01, 7.8188e-01, 1.0872e+00,
3.3055e+00],
[1.2817e-01, 3.8618e-01, 2.1917e-01, 2.6102e-01, 2.1917e-01,
3.8618e-01, 4.3028e-01, 3.7738e-01, 2.0158e-01, 4.2135e-01,
5.7616e-02],
[3.7364e-01, 7.1574e-01, 8.1719e-01, 9.0949e-13, 8.1720e-01,
7.1573e-01, 2.7823e-01, 7.1573e-01, 8.1719e-01, 7.1574e-01,
3.7364e-01]]])
count_unequal_element(out, result, 0.0001, 0.0001)
def test_spectrogram_param():
"""
Feature: test spectrogram invalid parameter.
Description: test some invalid parameters.
Expectation: success.
"""
try:
_ = audio.Spectrogram(n_fft=-1)
except ValueError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Input n_fft is not within the required interval of [1, 2147483647]." in str(error)
try:
_ = audio.Spectrogram(n_fft=0)
except ValueError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Input n_fft is not within the required interval of [1, 2147483647]." in str(error)
try:
_ = audio.Spectrogram(win_length=-1)
except ValueError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Input win_length is not within the required interval of [1, 2147483647]." in str(error)
try:
_ = audio.Spectrogram(win_length="s")
except TypeError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Argument win_length with value s is not of type [<class 'int'>], but got <class 'str'>." in str(error)
try:
_ = audio.Spectrogram(hop_length=-1)
except ValueError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Input hop_length is not within the required interval of [1, 2147483647]." in str(error)
try:
_ = audio.Spectrogram(hop_length=-100)
except ValueError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Input hop_length is not within the required interval of [1, 2147483647]." in str(error)
try:
_ = audio.Spectrogram(win_length=300, n_fft=200)
except ValueError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Input win_length should be no more than n_fft, but got win_length: 300 and n_fft: 200." \
in str(error)
try:
_ = audio.Spectrogram(pad=-1)
except ValueError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Input pad is not within the required interval of [0, 2147483647]." in str(error)
try:
_ = audio.Spectrogram(power=-1)
except ValueError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Input power is not within the required interval of [0, 16777216]." in str(error)
try:
_ = audio.Spectrogram(n_fft=False)
except TypeError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Argument n_fft with value False is not of type (<class 'int'>,)" in str(error)
try:
_ = audio.Spectrogram(n_fft="s")
except TypeError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Argument n_fft with value s is not of type [<class 'int'>], but got <class 'str'>." \
in str(error)
try:
_ = audio.Spectrogram(window=False)
except TypeError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Argument window with value False is not of type [<enum 'WindowType'>], but got <class 'bool'>." \
in str(error)
try:
_ = audio.Spectrogram(pad_mode=False)
except TypeError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Argument pad_mode with value False is not of type [<enum 'BorderType'>], but got <class 'bool'>." \
in str(error)
try:
_ = audio.Spectrogram(onesided="s")
except TypeError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Argument onesided with value s is not of type [<class 'bool'>], but got <class 'str'>." in str(error)
try:
_ = audio.Spectrogram(center="s")
except TypeError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Argument center with value s is not of type [<class 'bool'>], but got <class 'str'>." in str(error)
try:
_ = audio.Spectrogram(normalized="s")
except TypeError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Argument normalized with value s is not of type [<class 'bool'>], but got <class 'str'>." in str(error)
try:
_ = audio.Spectrogram(normalized=1)
except TypeError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Argument normalized with value 1 is not of type [<class 'bool'>], but got <class 'int'>." in str(error)
try:
wav = np.array([[1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5]])
_ = audio.Spectrogram(n_fft=100, center=False)(wav)
except RuntimeError as error:
logger.info("Got an exception in Spectrogram: {}".format(str(error)))
assert "Unexpected error. Spectrogram: n_fft should be more than 0 and less than 30," \
" but got n_fft: 100." in str(error)
if __name__ == "__main__":
test_spectrogram_pipeline()
test_spectrogram_eager()
test_spectrogram_window_hamming_padmode_constant()
test_spectrogram_nfft_10_window_bartlett_padmode_edge()
test_spectrogram_onsided_false()
test_spectrogram_power_0()
test_spectrogram_center_false()
test_spectrogram_normalized_true()
test_spectrogram_inputrank_3()
test_spectrogram_winlength_7()
test_spectrogram_param()