diff --git a/docs/api/api_python/dataset_transforms/mindspore.dataset.transforms.c_transforms.Slice.rst b/docs/api/api_python/dataset_transforms/mindspore.dataset.transforms.c_transforms.Slice.rst index 0746f23c5c..99fdfc3a51 100644 --- a/docs/api/api_python/dataset_transforms/mindspore.dataset.transforms.c_transforms.Slice.rst +++ b/docs/api/api_python/dataset_transforms/mindspore.dataset.transforms.c_transforms.Slice.rst @@ -12,9 +12,9 @@ mindspore.dataset.transforms.c_transforms.Slice - 1. :py:obj:`int`: 沿着第一个维度切片对索引进行切片,支持负索引。 - 2. :py:obj:`list(int)`: 沿着第一个维度切片所有索引进行切片,支持负号索引。 - 3. :py:obj:`slice`: 沿着第一个维度对 `slice `_ 对象生成的索引进行切片。 - - 4. :py:obj:`None`: 切片整个维度,类似于Python索引中的语法:py:obj:'[:]'。 + - 4. :py:obj:`None`: 切片整个维度,类似于Python索引中的语法 :py:obj:`[:]` 。 - 5. :py:obj:`Ellipsis`: 切片整个维度,效果与 `None` 相同。 **异常:** - - **TypeError** - 参数 `slices` 类型不为int、list[int]、:py:obj:`slice`、:py:obj:`None`或:py:obj:`Ellipsis`。 + - **TypeError** - 参数 `slices` 类型不为int、list[int]、:py:obj:`slice` 、:py:obj:`None` 或 :py:obj:`Ellipsis` 。 diff --git a/docs/api/api_python/mindspore.dataset.config.rst b/docs/api/api_python/mindspore.dataset.config.rst index d60010d218..95c476bec0 100644 --- a/docs/api/api_python/mindspore.dataset.config.rst +++ b/docs/api/api_python/mindspore.dataset.config.rst @@ -224,7 +224,7 @@ API示例所需模块的导入代码如下: .. py:function:: mindspore.dataset.config.set_enable_autotune(enable, json_filepath=None) - 设置是否开启自动数据加速。 + 设置是否开启自动数据加速。默认情况下不开启自动数据加速。 自动数据加速用于在训练过程中根据环境资源的负载,自动调整数据处理管道全局配置,提高数据处理的速度。 @@ -233,7 +233,7 @@ API示例所需模块的导入代码如下: **参数:** - **enable** (bool) - 是否开启自动数据加速。 - - **json_filepath** (str,可选) - 优化后的全局配置的保存路径,当路径存在同名文件时会自动覆盖。默认值:None,表示不保存配置文件。 + - **json_filepath** (str,可选) - 优化后的全局配置的保存路径,当路径存在同名文件时会自动覆盖。默认值:None,表示不保存配置文件,但可以通过INFO日志查看调优配置。 **异常:** @@ -244,7 +244,27 @@ API示例所需模块的导入代码如下: - **RuntimeError** - 当 `json_filepath` 路径不存在。 - **RuntimeError** - 当 `json_filepath` 没有写入权限。 - .. note:: 当 `enable` 为 False 时,`json_filepath` 值将会被忽略。 + .. note:: + - 当 `enable` 为 False 时,`json_filepath` 值将会被忽略。 + - 生成的JSON文件可以通过 `mindspore.dataset.deserialize` 进行加载,得到调优后的数据处理管道。 + + 生成的JSON文件内容示例如下,"remark"字段将给出结论表明数据处理管道是否进行了调整,"summary"字段将展示数据处理管道的调优配置。 + 用户可以根据调优结果修改代码脚本。 + + .. code-block:: + + { + "remark": "The following file has been auto-generated by the Dataset AutoTune.", + "summary": [ + "CifarOp(ID:5) (num_parallel_workers: 2, prefetch_size:64)", + "MapOp(ID:4) (num_parallel_workers: 2, prefetch_size:64)", + "MapOp(ID:3) (num_parallel_workers: 2, prefetch_size:64)", + "BatchOp(ID:2) (num_parallel_workers: 8, prefetch_size:64)" + ], + "tree": { + ... + } + } .. py:function:: mindspore.dataset.config.get_enable_autotune() diff --git a/mindspore/python/mindspore/dataset/core/config.py b/mindspore/python/mindspore/dataset/core/config.py index 6c9bda8808..27da5782fa 100644 --- a/mindspore/python/mindspore/dataset/core/config.py +++ b/mindspore/python/mindspore/dataset/core/config.py @@ -441,7 +441,7 @@ def load(file): def set_enable_autotune(enable, json_filepath=None): """ - Set whether to enable AutoTune. + Set whether to enable AutoTune. AutoTune is disabled by default. AutoTune is used to automatically adjust the global configuration of the data pipeline according to the workload of environmental resources during the training process to @@ -454,7 +454,7 @@ def set_enable_autotune(enable, json_filepath=None): enable (bool): Whether to enable AutoTune. json_filepath (str, optional): The filepath to save the optimized global configuration. If the file already exists, it will be automatically overwritten. Default: None, - means not to save the configuration file. + means not to save the configuration file, but the tuned result still can be checked through INFO log. Raises: TypeError: If `enable` is not of type boolean. @@ -465,7 +465,27 @@ def set_enable_autotune(enable, json_filepath=None): RuntimeError: If `json_filepath` does not have write permission. Note: - When `enable` is False, `json_filepath` will be ignored. + - When `enable` is False, `json_filepath` will be ignored. + - The JSON file can be loaded by API `mindspore.dataset.deserialize` to build a tuned pipeline. + + An example of the generated JSON file is as follow. "remark" file will conclude that if the dataset has been + tuned or not. "summary" filed will show the tuned configuration of dataset pipeline. Users can modify scripts + based on the tuned result. + + .. code-block:: + + { + "remark": "The following file has been auto-generated by the Dataset AutoTune.", + "summary": [ + "CifarOp(ID:5) (num_parallel_workers: 2, prefetch_size:64)", + "MapOp(ID:4) (num_parallel_workers: 2, prefetch_size:64)", + "MapOp(ID:3) (num_parallel_workers: 2, prefetch_size:64)", + "BatchOp(ID:2) (num_parallel_workers: 8, prefetch_size:64)" + ], + "tree": { + ... + } + } Examples: >>> # enable AutoTune and save optimized data pipeline configuration @@ -488,6 +508,9 @@ def set_enable_autotune(enable, json_filepath=None): if not enable and json_filepath is not None: logger.warning("The value of json_filepath is ignored when enable is False.") + if enable and json_filepath is None: + logger.warning("Dataset AutoTune is enabled but no json path is specified, check INFO log for tuned result.") + json_filepath = replace_none(json_filepath, "") _config.set_enable_autotune(enable, save_autoconfig, json_filepath)