diff --git a/docs/api/api_python/dataset/mindspore.dataset.WaitedDSCallback.rst b/docs/api/api_python/dataset/mindspore.dataset.WaitedDSCallback.rst index 64b65fde75..9307f7ab20 100644 --- a/docs/api/api_python/dataset/mindspore.dataset.WaitedDSCallback.rst +++ b/docs/api/api_python/dataset/mindspore.dataset.WaitedDSCallback.rst @@ -3,14 +3,14 @@ mindspore.dataset.WaitedDSCallback .. py:class:: mindspore.dataset.WaitedDSCallback(step_size=1) - 数据集自定义回调类的抽象基类，用于与训练回调类(`mindspore.callback `_)的同步。 + 数据集自定义回调类的抽象基类，用于与训练回调类 `mindspore.train.callback `_ 的同步。 可用于在每个step或epoch开始前执行自定义的回调方法，注意，第二个step或epoch开始时才会触发该调用。 例如在自动数据增强中根据上一个epoch的loss值来更新增强算子参数配置。 - 用户可通过 `train_run_context` 获取模型相关信息。如 `network` 、 `train_network` 、 `epoch_num` 、 `batch_num` 、 `loss_fn` 、 `optimizer` 、 `parallel_mode` 、 `device_number` 、 `list_callback` 、 `cur_epoch_num` 、 `cur_step_num` 、 `dataset_sink_mode` 、 `net_outputs` 等，详见 `mindspore.callback `_ 。 + 用户可通过 `train_run_context` 获取网络训练相关信息，如 `network` 、 `train_network` 、 `epoch_num` 、 `batch_num` 、 `loss_fn` 、 `optimizer` 、 `parallel_mode` 、 `device_number` 、 `list_callback` 、 `cur_epoch_num` 、 `cur_step_num` 、 `dataset_sink_mode` 、 `net_outputs` 等，详见 `mindspore.train.callback `_ 。 - 用户可通过 `ds_run_context` 获取数据处理管道相关信息。包括 `cur_epoch_num` (当前epoch数)、 `cur_step_num_in_epoch` (当前epoch的step数)、 `cur_step_num` (当前step数)。 + 用户可通过 `ds_run_context` 获取数据处理管道相关信息，包括 `cur_epoch_num` (当前epoch数)、 `cur_step_num_in_epoch` (当前epoch的step数)、 `cur_step_num` (当前step数)。 **参数：** diff --git a/mindspore/python/mindspore/dataset/audio/transforms.py b/mindspore/python/mindspore/dataset/audio/transforms.py index 5c6f7f7974..fc43989bff 100644 --- a/mindspore/python/mindspore/dataset/audio/transforms.py +++ b/mindspore/python/mindspore/dataset/audio/transforms.py @@ -53,9 +53,9 @@ class AllpassBiquad(AudioTensorOperation): Design two-pole all-pass filter for audio waveform of dimension of (..., time). Args: - sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero. - central_freq (float): central frequency (in Hz). - Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero. + central_freq (float): Central frequency (in Hz). + Q (float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). Examples: >>> import numpy as np @@ -91,11 +91,12 @@ class AmplitudeToDB(AudioTensorOperation): amin (float, optional): Lower bound to clamp the input waveform. It must be greater than zero (default=1e-10). top_db (float, optional): Minimum cut-off decibels. The range of values is non-negative. Commonly set at 80 (default=80.0). + Examples: >>> import numpy as np >>> from mindspore.dataset.audio import ScaleType >>> - >>> waveform = np.random.random([1, 400//2+1, 30]) + >>> waveform = np.random.random([1, 400 // 2 + 1, 30]) >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) >>> transforms = [audio.AmplitudeToDB(stype=ScaleType.POWER)] >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) @@ -197,9 +198,9 @@ class BandrejectBiquad(AudioTensorOperation): Design two-pole band-reject filter for audio waveform of dimension of (..., time). Args: - sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero. - central_freq (float): central frequency (in Hz). - Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). + sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero. + central_freq (float): Central frequency (in Hz). + Q (float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). Examples: >>> import numpy as np @@ -227,8 +228,8 @@ class BassBiquad(AudioTensorOperation): Args: sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero. gain (float): Desired gain at the boost (or attenuation) in dB. - central_freq (float): Central frequency (in Hz) (default=100.0). - Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). + central_freq (float, optional): Central frequency (in Hz) (default=100.0). + Q (float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707). Examples: >>> import numpy as np @@ -252,7 +253,7 @@ class BassBiquad(AudioTensorOperation): class Biquad(TensorOperation): """ - Perform a biquad filter of input tensor. + Perform a biquad filter of input audio. Args: b0 (float): Numerator coefficient of current input, x[n]. @@ -359,8 +360,9 @@ class Contrast(AudioTensorOperation): Comparable with compression, this effect modifies an audio signal to make it sound louder. Args: - enhancement_amount (float): Controls the amount of the enhancement. Allowed range is [0, 100] (default=75.0). - Note that enhancement_amount equal to 0 still gives a significant contrast enhancement. + enhancement_amount (float, optional): Controls the amount of the enhancement. + Allowed range is [0, 100] (default=75.0). Note that enhancement_amount equal + to 0 still gives a significant contrast enhancement. Examples: >>> import numpy as np @@ -420,7 +422,7 @@ class DCShift(AudioTensorOperation): >>> waveform = np.array([0.60, 0.97, -1.04, -1.26, 0.97, 0.91, 0.48, 0.93]) >>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"]) >>> transforms = [audio.DCShift(0.5, 0.02)] - >>> numpy_slices_dataset = numpy_slices_dataset.map(operation=transforms, input_columns=["audio"]) + >>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"]) """ @check_dc_shift @@ -672,11 +674,11 @@ class FrequencyMasking(AudioTensorOperation): Args: iid_masks (bool, optional): Whether to apply different masks to each example (default=false). - frequency_mask_param (int): Maximum possible length of the mask, range: [0, freq_length] (default=0). + frequency_mask_param (int, optional): Maximum possible length of the mask, range: [0, freq_length] (default=0). Indices uniformly sampled from [0, frequency_mask_param]. - mask_start (int): Mask start takes effect when iid_masks=true, + mask_start (int, optional): Mask start takes effect when iid_masks=true, range: [0, freq_length-frequency_mask_param] (default=0). - mask_value (double): Mask value (default=0.0). + mask_value (float, optional): Mask value (default=0.0). Examples: >>> import numpy as np @@ -1112,11 +1114,11 @@ class TimeMasking(AudioTensorOperation): Args: iid_masks (bool, optional): Whether to apply different masks to each example (default=false). - time_mask_param (int): Maximum possible length of the mask, range: [0, time_length] (default=0). + time_mask_param (int, optional): Maximum possible length of the mask, range: [0, time_length] (default=0). Indices uniformly sampled from [0, time_mask_param]. - mask_start (int): Mask start takes effect when iid_masks=true, + mask_start (int, optional): Mask start takes effect when iid_masks=true, range: [0, time_length-time_mask_param] (default=0). - mask_value (double): Mask value (default=0.0). + mask_value (float, optional): Mask value (default=0.0). Examples: >>> import numpy as np diff --git a/mindspore/python/mindspore/dataset/audio/utils.py b/mindspore/python/mindspore/dataset/audio/utils.py index 0606a4ab05..de60002771 100644 --- a/mindspore/python/mindspore/dataset/audio/utils.py +++ b/mindspore/python/mindspore/dataset/audio/utils.py @@ -140,7 +140,9 @@ def CreateDct(n_mfcc, n_mels, norm=NormMode.NONE): numpy.ndarray, the transformation matrix, to be right-multiplied to row-wise data of size (n_mels, n_mfcc). Examples: - >>> dct = audio.CreateDct(100, 200, audio.NormMode.NONE) + >>> from mindspore.dataset.audio import CreateDct + >>> + >>> dct = CreateDct(100, 200, audio.NormMode.NONE) """ if not isinstance(n_mfcc, int): diff --git a/mindspore/python/mindspore/dataset/callback/ds_callback.py b/mindspore/python/mindspore/dataset/callback/ds_callback.py index 885be235d2..005c565b49 100644 --- a/mindspore/python/mindspore/dataset/callback/ds_callback.py +++ b/mindspore/python/mindspore/dataset/callback/ds_callback.py @@ -31,13 +31,15 @@ class DSCallback: Examples: >>> from mindspore.dataset import DSCallback + >>> import mindspore.dataset.transforms.c_transforms as c_transforms >>> >>> class PrintInfo(DSCallback): ... def ds_epoch_end(self, ds_run_context): - ... print(cb_params.cur_epoch_num) - ... print(cb_params.cur_step_num) + ... print(ds_run_context.cur_epoch_num) + ... print(ds_run_context.cur_step_num) >>> - >>> # dataset is an instance of Dataset object + >>> # dataset is an instance of Dataset object and op is a certain data processing operator + >>> op = c_transforms.Fill(3) >>> dataset = dataset.map(operations=op, callbacks=PrintInfo()) """ @@ -132,14 +134,70 @@ class WaitedDSCallback(Callback, DSCallback): will be equal to the batch size (Default=1). Examples: + >>> import mindspore.nn as nn >>> from mindspore.dataset import WaitedDSCallback + >>> from mindspore import context + >>> from mindspore.train import Model + >>> from mindspore.train.callback import Callback >>> - >>> my_cb = WaitedDSCallback(32) - >>> # dataset is an instance of Dataset object - >>> dataset = dataset.map(operations=AugOp(), callbacks=my_cb) - >>> dataset = dataset.batch(32) - >>> # define the model - >>> model.train(epochs, data, callbacks=[my_cb]) + >>> context.set_context(mode=context.GRAPH_MODE, device_target="CPU") + >>> + >>> # custom callback class for data synchronization in data pipeline + >>> class MyWaitedCallback(WaitedDSCallback): + ... def __init__(self, events, step_size=1): + ... super().__init__(step_size) + ... self.events = events + ... + ... # callback method to be executed by data pipeline before the epoch starts + ... def sync_epoch_begin(self, train_run_context, ds_run_context): + ... event = f"ds_epoch_begin_{ds_run_context.cur_epoch_num}_{ds_run_context.cur_step_num}" + ... self.events.append(event) + ... + ... # callback method to be executed by data pipeline before the step starts + ... def sync_step_begin(self, train_run_context, ds_run_context): + ... event = f"ds_step_begin_{ds_run_context.cur_epoch_num}_{ds_run_context.cur_step_num}" + ... self.events.append(event) + >>> + >>> # custom callback class for data synchronization in network training + >>> class MyMSCallback(Callback): + ... def __init__(self, events): + ... self.events = events + ... + ... # callback method to be executed by network training after the epoch ends + ... def epoch_end(self, run_context): + ... cb_params = run_context.original_args() + ... event = f"ms_epoch_end_{cb_params.cur_epoch_num}_{cb_params.cur_step_num}" + ... self.events.append(event) + ... + ... # callback method to be executed by network training after the step ends + ... def step_end(self, run_context): + ... cb_params = run_context.original_args() + ... event = f"ms_step_end_{cb_params.cur_epoch_num}_{cb_params.cur_step_num}" + ... self.events.append(event) + >>> + >>> # custom network + >>> class Net(nn.Cell): + ... def construct(self, x, y): + ... return x + >>> + >>> # define a parameter that needs to be synchronized between data pipeline and network training + >>> events = [] + >>> + >>> # define callback classes of data pipeline and netwok training + >>> my_cb1 = MyWaitedCallback(events, 1) + >>> my_cb2 = MyMSCallback(events) + >>> arr = [1, 2, 3, 4] + >>> + >>> # construct data pipeline + >>> data = ds.NumpySlicesDataset((arr, arr), column_names=["c1", "c2"], shuffle=False) + >>> # map the data callback object into the pipeline + >>> data = data.map(operations=(lambda x: x), callbacks=my_cb1) + >>> + >>> net = Net() + >>> model = Model(net) + >>> + >>> # add the data and network callback objects to the model training callback list + >>> model.train(2, data, dataset_sink_mode=False, callbacks=[my_cb2, my_cb1]) """ def __init__(self, step_size=1): diff --git a/mindspore/python/mindspore/dataset/engine/cache_client.py b/mindspore/python/mindspore/dataset/engine/cache_client.py index a0ead40dfb..d5f4900df3 100644 --- a/mindspore/python/mindspore/dataset/engine/cache_client.py +++ b/mindspore/python/mindspore/dataset/engine/cache_client.py @@ -43,8 +43,9 @@ class DatasetCache: Examples: >>> import mindspore.dataset as ds >>> - >>> # create a cache instance, in which session_id is generated from command line `cache_admin -g` - >>> some_cache = ds.DatasetCache(session_id=session_id, size=0) + >>> # Create a cache instance, in which session_id is generated from command line `cache_admin -g` + >>> # In the following code, suppose the session_id is 780643335 + >>> some_cache = ds.DatasetCache(session_id=780643335, size=0) >>> >>> dataset_dir = "path/to/imagefolder_directory" >>> ds1 = ds.ImageFolderDataset(dataset_dir, cache=some_cache) diff --git a/mindspore/python/mindspore/dataset/engine/datasets_text.py b/mindspore/python/mindspore/dataset/engine/datasets_text.py index 6508b44eac..04ebd71fae 100644 --- a/mindspore/python/mindspore/dataset/engine/datasets_text.py +++ b/mindspore/python/mindspore/dataset/engine/datasets_text.py @@ -410,7 +410,7 @@ class CoNLL2000Dataset(SourceDataset, TextBaseDataset): Examples: >>> conll2000_dataset_dir = "/path/to/conll2000_dataset_dir" - >>> dataset = ds.CoNLL2000Dataset(dataset_files=conll2000_dataset_dir, usage='all') + >>> dataset = ds.CoNLL2000Dataset(dataset_dir=conll2000_dataset_dir, usage='all') """ @check_conll2000_dataset @@ -786,7 +786,7 @@ class IWSLT2016Dataset(SourceDataset, TextBaseDataset): Examples: >>> iwslt2016_dataset_dir = "/path/to/iwslt2016_dataset_dir" - >>> dataset = ds.IWSLT2016Dataset(dataset_files=iwslt2016_dataset_dir, usage='all', + >>> dataset = ds.IWSLT2016Dataset(dataset_dir=iwslt2016_dataset_dir, usage='all', ... language_pair=('de', 'en'), valid_set='tst2013', test_set='tst2014') About IWSLT2016 dataset: @@ -907,7 +907,7 @@ class IWSLT2017Dataset(SourceDataset, TextBaseDataset): Examples: >>> iwslt2017_dataset_dir = "/path/to/iwslt207_dataset_dir" - >>> dataset = ds.IWSLT2017Dataset(dataset_files=iwslt2017_dataset_dir, usage='all', language_pair=('de', 'en')) + >>> dataset = ds.IWSLT2017Dataset(dataset_dir=iwslt2017_dataset_dir, usage='all', language_pair=('de', 'en')) About IWSLT2017 dataset: @@ -1092,7 +1092,7 @@ class SogouNewsDataset(SourceDataset, TextBaseDataset): Examples: >>> sogou_news_dataset_dir = "/path/to/sogou_news_dataset_dir" - >>> dataset = ds.SogouNewsDataset(dataset_files=sogou_news_dataset_dir, usage='all') + >>> dataset = ds.SogouNewsDataset(dataset_dir=sogou_news_dataset_dir, usage='all') About SogouNews Dataset: @@ -1234,7 +1234,7 @@ class UDPOSDataset(SourceDataset, TextBaseDataset): Examples: >>> udpos_dataset_dir = "/path/to/udpos_dataset_dir" - >>> dataset = ds.UDPOSDataset(dataset_files=udpos_dataset_dir, usage='all') + >>> dataset = ds.UDPOSDataset(dataset_dir=udpos_dataset_dir, usage='all') """ @check_udpos_dataset diff --git a/mindspore/python/mindspore/dataset/engine/serializer_deserializer.py b/mindspore/python/mindspore/dataset/engine/serializer_deserializer.py index 6bb4297d27..761a58274b 100644 --- a/mindspore/python/mindspore/dataset/engine/serializer_deserializer.py +++ b/mindspore/python/mindspore/dataset/engine/serializer_deserializer.py @@ -41,7 +41,7 @@ def serialize(dataset, json_filepath=""): OSError: Can not open a file Examples: - >>> dataset = ds.MnistDataset(mnist_dataset_dir, 100) + >>> dataset = ds.MnistDataset(mnist_dataset_dir, num_samples=100) >>> one_hot_encode = c_transforms.OneHot(10) # num_classes is input argument >>> dataset = dataset.map(operation=one_hot_encode, input_column_names="label") >>> dataset = dataset.batch(batch_size=10, drop_remainder=True) @@ -70,7 +70,7 @@ def deserialize(input_dict=None, json_filepath=None): OSError: Can not open the JSON file. Examples: - >>> dataset = ds.MnistDataset(mnist_dataset_dir, 100) + >>> dataset = ds.MnistDataset(mnist_dataset_dir, num_samples=100) >>> one_hot_encode = c_transforms.OneHot(10) # num_classes is input argument >>> dataset = dataset.map(operation=one_hot_encode, input_column_names="label") >>> dataset = dataset.batch(batch_size=10, drop_remainder=True) @@ -132,9 +132,9 @@ def compare(pipeline1, pipeline2): Whether pipeline1 is equal to pipeline2. Examples: - >>> pipeline1 = ds.MnistDataset(mnist_dataset_dir, 100) - >>> pipeline2 = ds.Cifar10Dataset(cifar_dataset_dir, 100) - >>> ds.compare(pipeline1, pipeline2) + >>> pipeline1 = ds.MnistDataset(mnist_dataset_dir, num_samples=100) + >>> pipeline2 = ds.Cifar10Dataset(cifar_dataset_dir, num_samples=100) + >>> res = ds.compare(pipeline1, pipeline2) """ return pipeline1.to_json() == pipeline2.to_json()