diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc index a82025244f9..c921b22ac40 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc @@ -512,8 +512,10 @@ Status DeviceQueueOp::RetryPushData(unsigned int handle, const std::vector &input, std::shared_ptrshape().Size() > CHANNEL_INDEX, - "AdjustBrightness: image rank should not bigger than:" + std::to_string(CHANNEL_INDEX) + ", but got" + - std::to_string(input_cv->shape().Size())); + "AdjustBrightness: image rank should not bigger than:" + std::to_string(CHANNEL_INDEX) + + ", but got: " + std::to_string(input_cv->shape().Size())); int num_channels = input_cv->shape()[CHANNEL_INDEX]; // Rank of the image represents how many dimensions, image is expected to be HWC if (input_cv->Rank() != DEFAULT_IMAGE_RANK || num_channels != DEFAULT_IMAGE_CHANNELS) { @@ -949,7 +949,7 @@ Status AdjustContrast(const std::shared_ptr &input, std::shared_ptrshape().Size() > CHANNEL_INDEX, "AdjustContrast: image rank should not bigger than:" + std::to_string(CHANNEL_INDEX) + - ", but got" + std::to_string(input_cv->shape().Size())); + ", but got: " + std::to_string(input_cv->shape().Size())); int num_channels = input_cv->shape()[CHANNEL_INDEX]; if (input_cv->Rank() != DEFAULT_IMAGE_CHANNELS || num_channels != DEFAULT_IMAGE_CHANNELS) { RETURN_STATUS_UNEXPECTED("AdjustContrast: image shape is not or channel is not 3, got image rank: " + diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/sharpness_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/sharpness_op.cc index 89800650a26..2a6a2a3e1fa 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/sharpness_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/sharpness_op.cc @@ -35,7 +35,8 @@ Status SharpnessOp::Compute(const std::shared_ptr &input, std::shared_pt } if (input_cv->Rank() == 1 || input_cv->mat().dims > 2) { - RETURN_STATUS_UNEXPECTED("Sharpness: shape of input is not or , but got rank: " + input_cv->Rank()); + RETURN_STATUS_UNEXPECTED("Sharpness: shape of input is not or , but got rank: " + + std::to_string(input_cv->Rank())); } /// creating a smoothing filter. 1, 1, 1, diff --git a/mindspore/dataset/callback/ds_callback.py b/mindspore/dataset/callback/ds_callback.py index 81be875e46a..74eb7150378 100644 --- a/mindspore/dataset/callback/ds_callback.py +++ b/mindspore/dataset/callback/ds_callback.py @@ -33,9 +33,9 @@ class DSCallback: >>> from mindspore.dataset import DSCallback >>> >>> class PrintInfo(DSCallback): - >>> def ds_epoch_end(self, ds_run_context): - >>> print(cb_params.cur_epoch_num) - >>> print(cb_params.cur_step_num) + ... def ds_epoch_end(self, ds_run_context): + ... print(cb_params.cur_epoch_num) + ... print(cb_params.cur_step_num) >>> >>> # dataset is an instance of Dataset object >>> dataset = dataset.map(operations=op, callbacks=PrintInfo()) @@ -71,7 +71,7 @@ class DSCallback: def ds_step_begin(self, ds_run_context): """ - Called before n steps are started. + Called before each step start. Args: ds_run_context (RunContext): Include some information of the pipeline. @@ -79,7 +79,7 @@ class DSCallback: def ds_step_end(self, ds_run_context): """ - Called after n steps are finished. + Called after each step finished. Args: ds_run_context (RunContext): Include some information of the pipeline. @@ -89,7 +89,8 @@ class DSCallback: """ Creates a runtime (C++) object from the callback methods defined by the user. - Returns: _c_dataengine.PyDSCallback + Returns: + _c_dataengine.PyDSCallback. """ c_cb = PyDSCallback(self.step_size) at_least_one = False @@ -225,7 +226,8 @@ class WaitedDSCallback(Callback, DSCallback): """ Creates a runtime (C++) object from the callback methods defined by the user. This method is internal. - Returns: _c_dataengine.PyDSCallback + Returns: + _c_dataengine.PyDSCallback. """ c_cb = PyDSCallback(self.step_size) at_least_one = False diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index e7e6c6cc38c..d702bd71c4c 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -401,22 +401,21 @@ class Dataset: BucketBatchByLengthDataset, dataset bucketed and batched by length. Examples: - >>> # Create a dataset where every 100 rows are combined into a batch + >>> # Create a dataset where certain counts rows are combined into a batch >>> # and drops the last incomplete batch if there is one. >>> import numpy as np >>> def generate_2_columns(n): ... for i in range(n): ... yield (np.array([i]), np.array([j for j in range(i + 1)])) + >>> >>> column_names = ["col1", "col2"] - >>> dataset = ds.GeneratorDataset(generate_2_columns(202), column_names) + >>> dataset = ds.GeneratorDataset(generate_2_columns(8), column_names) >>> bucket_boundaries = [5, 10] - >>> bucket_batch_sizes = [5, 1, 1] + >>> bucket_batch_sizes = [2, 1, 1] >>> element_length_function = (lambda col1, col2: max(len(col1), len(col2))) - >>> # Will pad col1 to shape [2, bucket_boundaries[i]] where i is the + >>> # Will pad col2 to shape [bucket_boundaries[i]] where i is the >>> # index of the bucket that is currently being batched. - >>> # Will pad col2 to a shape where each dimension is the longest in all - >>> # the elements currently being batched. - >>> pad_info = {"col1": ([2, None], -1)} + >>> pad_info = {"col2": ([None], -1)} >>> pad_to_bucket_boundary = True >>> dataset = dataset.bucket_batch_by_length(column_names, bucket_boundaries, ... bucket_batch_sizes, @@ -1129,7 +1128,7 @@ class Dataset: Returns: Vocab, vocab built from the dataset. - Example: + Examples: >>> import numpy as np >>> >>> def gen_corpus(): @@ -1197,7 +1196,7 @@ class Dataset: Returns: SentencePieceVocab, vocab built from the dataset. - Example: + Examples: >>> from mindspore.dataset.text import SentencePieceModel >>> >>> def gen_corpus(): @@ -1533,6 +1532,10 @@ class Dataset: Returns: list, list of column names in the dataset. + + Examples: + >>> # dataset is an instance object of Dataset + >>> col_names = dataset.get_col_names() """ if self._col_names is None: runtime_getter = self._init_tree_getters() @@ -1547,6 +1550,10 @@ class Dataset: Returns: list, list of shapes of each column. + + Examples: + >>> # dataset is an instance object of Dataset + >>> output_shapes = dataset.output_shapes() """ if self.saved_output_shapes is None: runtime_getter = self._init_tree_getters() @@ -1564,6 +1571,10 @@ class Dataset: Returns: list, list of data types. + + Examples: + >>> # dataset is an instance object of Dataset + >>> output_types = dataset.output_types() """ if self.saved_output_types is None: runtime_getter = self._init_tree_getters() @@ -1581,6 +1592,10 @@ class Dataset: Returns: int, number of batches. + + Examples: + >>> # dataset is an instance object of Dataset + >>> dataset_size = dataset.get_dataset_size() """ if self.dataset_size is None: runtime_getter = self.__init_size_getter() @@ -1596,6 +1611,16 @@ class Dataset: Args: columns (dict): A dict contains shape information of each column in dataset. The value of shape[i] is :py:obj:`None` indicates that the data length of shape[i] is dynamic. + + Examples: + >>> import numpy as np + >>> + >>> def generator1(): + >>> for i in range(1, 100): + >>> yield np.ones((16, i, 83)), np.array(i) + >>> + >>> dataset = ds.GeneratorDataset(generator1, ["data1", "data2"]) + >>> dataset.set_dynamic_columns(columns={"data1": [16, None, 83], "data2": []}) """ if not isinstance(columns, dict): raise TypeError("Pass a dict to set dynamic shape, example: {\"data1\": [16, None, 256]}") @@ -1608,6 +1633,17 @@ class Dataset: Returns: lists, min_shapes, max_shapes of source data. + + Examples: + >>> import numpy as np + >>> + >>> def generator1(): + >>> for i in range(1, 100): + >>> yield np.ones((16, i, 83)), np.array(i) + >>> + >>> dataset = ds.GeneratorDataset(generator1, ["data1", "data2"]) + >>> dataset.set_dynamic_columns(columns={"data1": [16, None, 83], "data2": []}) + >>> min_shapes, max_shapes = dataset.dynamic_min_max_shapes() """ if self.saved_min_shapes is None or self.saved_max_shapes is None: self.saved_output_shapes, self.saved_min_shapes, self.saved_max_shapes = self._dynamic_output_shapes() @@ -1696,6 +1732,10 @@ class Dataset: Returns: int, number of classes. + + Examples: + >>> # dataset is an instance object of Dataset + >>> num_classes = dataset.num_classes() """ if self._num_classes is None: runtime_getter = self._init_tree_getters() @@ -1755,6 +1795,10 @@ class Dataset: Returns: int, the number of data in a batch. + + Examples: + >>> # dataset is an instance object of Dataset + >>> batch_size = dataset.get_batch_size() """ if self._batch_size is None: runtime_getter = self._init_tree_getters() @@ -1769,6 +1813,10 @@ class Dataset: Returns: int, the count of repeat. + + Examples: + >>> # dataset is an instance object of Dataset + >>> repeat_count = dataset.get_repeat_count() """ if self._repeat_count is None: runtime_getter = self._init_tree_getters() @@ -1785,6 +1833,10 @@ class Dataset: dict, a str-to-int mapping from label name to index. dict, a str-to-list mapping from label name to index for Coco ONLY. The second number in the list is used to indicate the super category. + + Examples: + >>> # dataset is an instance object of Dataset + >>> class_indexing = dataset.get_class_indexing() """ if self.children: return self.children[0].get_class_indexing() @@ -1911,7 +1963,18 @@ class MappableDataset(SourceDataset): self.sampler = samplers.select_sampler(num_samples, sampler, shuffle, num_shards, shard_id) def add_sampler(self, new_sampler): - """ add a sampler """ + """ + Add a sampler for current dataset,. + + Args: + new_sampler (Sampler): The sampler to be added as the parent sampler for current dataset. + + Examples: + >>> # dataset is an instance object of Dataset + >>> # use a DistributedSampler instead + >>> new_sampler = ds.DistributedSampler(10, 2) + >>> dataset.add_sampler(new_sampler) + """ # note: By adding a sampler, the sampled IDs will flow to new_sampler # after first passing through the current samplers attached to this dataset. self.dataset_size = None @@ -4353,6 +4416,12 @@ class ManifestDataset(MappableDataset): Returns: dict, a str-to-int mapping from label name to index. + + Examples: + >>> manifest_dataset_dir = "/path/to/manifest_dataset_file" + >>> + >>> dataset = ds.ManifestDataset(dataset_file=manifest_dataset_dir) + >>> class_indexing = dataset.get_class_indexing() """ if self.class_indexing is None or not self.class_indexing: if self._class_indexing is None: @@ -4673,7 +4742,7 @@ class Schema: Raises: RuntimeError: If schema file failed to load. - Example: + Examples: >>> from mindspore import dtype as mstype >>> >>> # Create schema; specify column name, mindspore.dtype and shape of the column @@ -4726,7 +4795,7 @@ class Schema: RuntimeError: If column's name field is missing. RuntimeError: If column's type field is missing. - Example: + Examples: >>> schema = Schema() >>> columns1 = [{'name': 'image', 'type': 'int8', 'shape': [3, 3]}, >>> {'name': 'label', 'type': 'int8', 'shape': [1]}] @@ -5050,6 +5119,12 @@ class VOCDataset(MappableDataset): Returns: dict, a str-to-int mapping from label name to index. + + Examples: + >>> voc_dataset_dir = "/path/to/voc_dataset_directory" + >>> + >>> dataset = ds.VOCDataset(dataset_dir=voc_dataset_dir) + >>> class_indexing = dataset.get_class_indexing() """ if self.task != "Detection": raise NotImplementedError("Only 'Detection' support get_class_indexing.") @@ -5253,7 +5328,18 @@ class CocoDataset(MappableDataset): Get the class index. Returns: - dict, a str-to-list mapping from label name to index + dict, a str-to-list mapping from label name to index. + + Examples: + >>> coco_dataset_dir = "/path/to/coco_dataset_directory/images" + >>> coco_annotation_file = "/path/to/coco_dataset_directory/annotation_file" + >>> + >>> # Read COCO data for Detection task + >>> dataset = ds.CocoDataset(dataset_dir=coco_dataset_dir, + ... annotation_file=coco_annotation_file, + ... task='Detection') + >>> + >>> class_indexing = dataset.get_class_indexing() """ if self.task not in {"Detection", "Panoptic"}: raise NotImplementedError("Only 'Detection' and 'Panoptic' support get_class_indexing.") diff --git a/mindspore/dataset/engine/samplers.py b/mindspore/dataset/engine/samplers.py index 7181057a2af..09d1daa8f8c 100644 --- a/mindspore/dataset/engine/samplers.py +++ b/mindspore/dataset/engine/samplers.py @@ -121,7 +121,17 @@ class BuiltinSampler: self.child_sampler = sampler def get_child(self): - """ Get the child sampler. """ + """ + Get the child sampler of given sampler. + + Returns: + Sampler, The child sampler of given sampler. + + Examples: + >>> sampler = ds.SequentialSampler(start_index=0, num_samples=3) + >>> sampler.add_child(ds.RandomSampler(num_samples=2)) + >>> child_sampler = sampler.get_child() + """ return self.child_sampler def parse_child(self): @@ -189,6 +199,10 @@ class BuiltinSampler: Returns: int, the number of samples, or None. + + Examples: + >>> sampler = ds.SequentialSampler(start_index=0, num_samples=3) + >>> num_samplers = sampler.get_num_samples() """ if self.child_sampler is not None: child_samples = self.child_sampler.get_num_samples() @@ -306,7 +320,7 @@ class Sampler(BuiltinSampler): class DistributedSampler(BuiltinSampler): """ - A sampler that accesses a shard of the dataset. + A sampler that accesses a shard of the dataset, it helps divide dataset into multi-subset for distributed training. Args: num_shards (int): Number of shards to divide the dataset into. diff --git a/mindspore/dataset/text/utils.py b/mindspore/dataset/text/utils.py index 6941287e14f..7a16e8b4d20 100644 --- a/mindspore/dataset/text/utils.py +++ b/mindspore/dataset/text/utils.py @@ -120,7 +120,7 @@ class Vocab(cde.Vocab): Vocab, vocab built from the file. Examples: - >>> vocab = text.Vocab.from_file("/path/to/wordpiece/vocab/file", ",", None, ["", ""], True) + >>> vocab = text.Vocab.from_file("/path/to/simple/vocab/file", ",", None, ["", ""], True) """ if vocab_size is None: vocab_size = -1 @@ -183,6 +183,7 @@ class SentencePieceVocab(cde.SentencePieceVocab): SentencePieceVocab, vocab built from the dataset. Examples: + >>> from mindspore.dataset.text import SentencePieceModel >>> dataset = ds.TextFileDataset("/path/to/sentence/piece/vocab/file", shuffle=False) >>> vocab = text.SentencePieceVocab.from_dataset(dataset, ["text"], 5000, 0.9995, ... SentencePieceModel.UNIGRAM, {}) @@ -226,6 +227,7 @@ class SentencePieceVocab(cde.SentencePieceVocab): SentencePieceVocab, vocab built from the file. Examples: + >>> from mindspore.dataset.text import SentencePieceModel >>> vocab = text.SentencePieceVocab.from_file(["/path/to/sentence/piece/vocab/file"], 5000, 0.9995, ... SentencePieceModel.UNIGRAM, {}) """ @@ -244,6 +246,7 @@ class SentencePieceVocab(cde.SentencePieceVocab): filename(str): The name of the file. Examples: + >>> from mindspore.dataset.text import SentencePieceModel >>> vocab = text.SentencePieceVocab.from_file(["/path/to/sentence/piece/vocab/file"], 5000, 0.9995, ... SentencePieceModel.UNIGRAM, {}) >>> text.SentencePieceVocab.save_model(vocab, "./", "m.model") @@ -263,9 +266,10 @@ def to_str(array, encoding='utf8'): numpy.ndarray, NumPy array of `str`. Examples: - >>> dataset = ds.TextFileDataset("/path/to/text_file_dataset_file", shuffle=False) + >>> text_file_dataset_dir = ["/path/to/text_file_dataset_file"] + >>> dataset = ds.TextFileDataset(dataset_files=text_file_dataset_dir, shuffle=False) >>> for item in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): - >>> print(text.to_str(item["text"])) + ... print(text.to_str(item["text"])) """ if not isinstance(array, np.ndarray):