diff --git a/README.md b/README.md index e3f83a8061f..492451cff6a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ ![MindSpore Logo](https://gitee.com/mindspore/mindspore/raw/master/docs/MindSpore-logo.png "MindSpore logo") - +X [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/mindspore.svg)](https://pypi.org/project/mindspore) [![PyPI](https://badge.fury.io/py/mindspore.svg)](https://badge.fury.io/py/mindspore) [![Downloads](https://pepy.tech/badge/mindspore)](https://pepy.tech/project/mindspore) diff --git a/mindspore/core/mindapi/base/base.h b/mindspore/core/mindapi/base/base.h index 4ed554bd1fd..b94f34ba010 100644 --- a/mindspore/core/mindapi/base/base.h +++ b/mindspore/core/mindapi/base/base.h @@ -29,6 +29,7 @@ class Base; } namespace mindspore::api { + /// \brief Base is the base class of many api classes, which provides basic interfaces. class MIND_API Base { public: @@ -45,9 +46,9 @@ class MIND_API Base { /// \return The id of this class. static uint32_t ClassId(); - /// \brief Get the shared_ptr to the underly implementation object. + /// \brief Get the shared_ptr to the underlying implementation object. /// - /// \return The shared_ptr to the underly implementation object. + /// \return The shared_ptr to the underlying implementation object. const std::shared_ptr &impl() const { return impl_; } /// \brief Get the string representation of this object. diff --git a/mindspore/python/mindspore/dataset/engine/datasets_user_defined.py b/mindspore/python/mindspore/dataset/engine/datasets_user_defined.py index a5cc35e5951..bd1cd2fdca2 100644 --- a/mindspore/python/mindspore/dataset/engine/datasets_user_defined.py +++ b/mindspore/python/mindspore/dataset/engine/datasets_user_defined.py @@ -632,72 +632,111 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset): def __init__(self, source, column_names=None, column_types=None, schema=None, num_samples=None, num_parallel_workers=1, shuffle=None, sampler=None, num_shards=None, shard_id=None, python_multiprocessing=True, max_rowsize=6): + #调用父类构造函数，将num_parallel_workers sampler sampler shuffle num_shards shard_id传入 super().__init__(num_parallel_workers=num_parallel_workers, sampler=sampler, num_samples=num_samples, shuffle=shuffle, num_shards=num_shards, shard_id=shard_id) + ##如果source是一个zip对象，转换为列表 if isinstance(source, builtins.zip): - # Although zip is iteratable, it does not have the feature of repeated iteration, so pass it to the array. + # Although zip is iterable, it does not have the feature of repeated iteration, so pass it to the array. + #zip是迭代器 不能重复迭代，这里把zip转换成列表再赋值给source self.source = [item for item in source] + #若source不是zip对象，输入的直接赋值给self.source else: self.source = source + #self.prepared_source赋值为none self.prepared_source = None # source to be sent to C++ + + ##若对象有operator_mixed属性，且该属性为true if hasattr(self, 'operator_mixed') and getattr(self, 'operator_mixed') is True: + #对象的num_parallel_workers赋值为1 self.num_parallel_workers = 1 + #输出警告信息：存在一些不支持多线程编译的运算符，建议将它们替换为 Python 实现的运算符 logger.warning( "Input 'source' of 'GeneratorDataset' includes network computing operators like in mindspore.nn, " "mindspore.ops, mindspore.numpy module and etc, which do not support multi-thread compiling, recommend" " to replace it with python implemented operator like numpy etc. Here decrease 'num_parallel_workers' " "into 1.") - - self.python_multiprocessing = python_multiprocessing - + #若当前平台为windows，且num_parallel_workers > 1，输出警告信息“Python 的多进程功能在 Windows 平台上不受支持” + if platform.system().lower() == 'windows' and num_parallel_workers > 1 and python_multiprocessing: + logger.warning("Python multiprocessing is not supported on Windows platform.") + #若当前操作系统是windows,把python_multiprocessing赋值给self.python_multiprocessing，若不是，后者赋值为false + self.python_multiprocessing = python_multiprocessing if platform.system().lower() != 'windows' else False + #若self.python_multiprocessing 属性的值为 True，且当前处于调试模式，警告：多进程功能在调试模式下不受支持，并把python_multiprocessing设置回false + if self.python_multiprocessing and get_debug_mode(): + logger.warning("Python multiprocessing is not supported in debug mode." + " Ignoring Python multiprocessing for GeneratorDataset.") + self.python_multiprocessing = False + #把column_names转换成列表，赋值给self.column_names self.column_names = to_list(column_names) - + #若column_types不为None，调用mstypelist_to_detypelist函数把column_types赋值给self.column_types + #mstypelist_to_detypelist函数将 MindSpore 类型列表转换为 DE 类型列表。 if column_types is not None: self.column_types = mstypelist_to_detypelist(column_types) else: self.column_types = [] - + #把self.schema赋值为schema值，仅在None值时有用 self.schema = schema + ##若schema不为None,即构造时传入了schema的值时，将传入的 schema 参数赋值给 self.schema，并保证schema是Schema 类的实例 if schema is not None: self.schema = schema + #若schema 不为 Schema 类的实例，使用 Schema 类的构造函数创建一个新的 Schema 对象，并将其赋值给 self.schema if not isinstance(schema, Schema): self.schema = Schema(schema) # Move get dataset_size by len from parse to here, because self.source will # lose attribution of '__len__' after deepcopy. - self.source_len = -1 # unknown + #self.source 在进行deepcopy后会丢失 __len__ 属性 + self.source_len = -1 # 表示未知 + + #若self.source 对象有 __len__ 属性，获取self.source长度到self.source_len里 if hasattr(self.source, "__len__"): self.source_len = len(self.source) + # if user defined sampler, update the self.source_len + #如果用户定义了采样器，要更新数据集的大小 + #若self.sampler为samplers.Sampler类的实例，或者self.sampler有__iter__属性 + #然后将 sampler 转换为列表并获取其长度，然后将结果赋值给 self.source_len + if isinstance(self.sampler, samplers.Sampler) or hasattr(self.sampler, "__iter__"): + self.source_len = len(list(sampler)) + self.max_rowsize = max_rowsize self.sample_fn = None - + #该函数进行深拷贝处理 def __deepcopy__(self, memodict): + #如果memodict 里已经存在当前对象的拷贝（已经拷贝过）（检查了 memodict 字典中是否存在键为 id(self) 的项），直接返回拷贝 if id(self) in memodict: return memodict[id(self)] + #用 __safe_deepcopy__创建新对象，排除了"source", "__transfer_dataset__" new_op = self.__safe_deepcopy__(memodict, exclude=("source", "__transfer_dataset__")) - + sample_fn = None + ## 如果new_op中的 sampler 不是空的，而且原对象的source可以操作索引（`__getitem__` 方法存在） if new_op.sampler is not None and hasattr(self.source, "__getitem__"): # The reason why there is a try catch here is because when the new op is being constructed with shared # memory enabled, there will be an exception thrown if there is not enough shared memory available - if self.source_len == -1: + #内存不够可能会错误 + if self.source_len == -1:#若长度还是未知，警告 需要“__len__”方法 raise RuntimeError("Attempt to construct a random access dataset, '__len__' method is required!") try: + #若num_parallel_workers 大于 1 if new_op.num_parallel_workers > 1: + #调用此方法 似乎是检查内存使用情况 self.__validate_memory_usage() - + #创建SamplerFn对象 sample_fn = SamplerFn(self.source, new_op.num_parallel_workers, self.python_multiprocessing, self.max_rowsize) new_op.prepared_source = (lambda sample_ids: _cpp_sampler_fn_mp(sample_ids, sample_fn)) else: new_op.prepared_source = (lambda sample_ids: _cpp_sampler_fn(sample_ids, self.source)) new_op.sample_fn = sample_fn + #遇到问题抛出异常 except RuntimeError as e: raise Exception(str(e)) else: try: + #如果新对象的sampler为空，设置sample_fn为None new_op.sampler = None new_op.sample_fn = sample_fn + #设置source_len为num_samples和source_len中的较小值 new_op.source_len = min(new_op.source_len, new_op.num_samples) if new_op.num_samples != 0 else new_op.source_len iter(self.source) @@ -710,18 +749,30 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset): new_op.prepared_source = (lambda: _iter_fn(self.source, new_op.num_samples)) return new_op - + #查询其是否已经打乱 def is_shuffled(self): - return self.sampler.is_shuffled() - + if self.sampler: + return self.sampler.is_shuffled() + return False + #查询是否sharded分片 def is_sharded(self): - return self.sampler.is_sharded() - + if self.sampler: + return self.sampler.is_sharded() + return False + #似乎用来分割元素 + def split(self, sizes, randomize=True): + if hasattr(self.source, "__getitem__"): + # If the source has __getitem__ attribute, call the split method of MappableDataset. + # Otherwise, call the split method of Dataset. + return super().split(sizes, randomize) + return super(MappableDataset, self).split(sizes, randomize) + def parse(self, children=None): if self.schema is None: return cde.GeneratorNode(self.prepared_source, self.column_names, self.column_types, self.source_len, self.sampler, self.num_parallel_workers) schema = self.schema + #如果schema是Schema对象，赋值为self.schema.cpp_schema if isinstance(schema, Schema): schema = self.schema.cpp_schema return cde.GeneratorNode(self.prepared_source, schema, self.source_len, self.sampler, @@ -734,6 +785,7 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset): if self.python_multiprocessing: # if use num_parallel_workers is to large when python_multiprocessing=True which would cause # OOM error get the num_shards + #检查内存的使用情况，在多进程模式下，当内存使用量达到 85% 时丢出警告，达到 100% 时报错。 valid_num_shards = 1 if isinstance(self.sampler, samplers.DistributedSampler): valid_num_shards = self.sampler.num_shards @@ -744,7 +796,7 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset): process = psutil.Process(os.getpid()) process_memory = process.memory_info().rss sys_memory_free = psutil.virtual_memory().free - + #内存使用量超过可用内存的85%，警告 total_memory_maybe_used = process_memory * self.num_parallel_workers * valid_num_shards if total_memory_maybe_used / sys_memory_free > 0.85: valid_num_worker = math.floor(sys_memory_free * 0.85 / valid_num_shards / process_memory) @@ -756,6 +808,7 @@ class GeneratorDataset(MappableDataset, UnionBaseDataset): logger.warning(info) + class _NumpySlicesDataset: """ Mainly for dealing with several kinds of formats of Python data, and return one row each time. diff --git a/mindspore/python/mindspore/dataset/transforms/py_transforms.py b/mindspore/python/mindspore/dataset/transforms/py_transforms.py index 85f42e6e8b7..8e4ba9e8ff6 100644 --- a/mindspore/python/mindspore/dataset/transforms/py_transforms.py +++ b/mindspore/python/mindspore/dataset/transforms/py_transforms.py @@ -28,11 +28,12 @@ from .c_transforms import TensorOperation def not_random(function): """ - Specify the function as "not random", i.e., it produces deterministic result. - A Python function can only be cached after it is specified as "not random". + 将函数标记为“非随机”，即它生成确定性结果。 + 只有在将函数标记为“非随机”后，Python 函数才能被缓存。 """ - function.random = False - return function + function.random = False #将函数的 random 属性设置为 False + return function #返回标记为“非随机”的函数 + class PyTensorOperation: @@ -40,78 +41,108 @@ class PyTensorOperation: Base Python Tensor Operations class """ - def to_json(self): - """ - Base to_json for Python tensor operations class - """ - json_obj = {} - json_trans = {} - if "transforms" in self.__dict__.keys(): - # operations which have transforms as input, need to call _to_json() for each transform to serialize - json_list = [] - for transform in self.transforms: - json_list.append(json.loads(transform.to_json())) - json_trans["transforms"] = json_list - self.__dict__.pop("transforms") - if "output_type" in self.__dict__.keys(): - json_trans["output_type"] = np.dtype( - self.__dict__["output_type"]).name - self.__dict__.pop("output_type") - json_obj["tensor_op_params"] = self.__dict__ - # append transforms to the tensor_op_params of the operation - json_obj["tensor_op_params"].update(json_trans) - json_obj["tensor_op_name"] = self.__class__.__name__ - json_obj["python_module"] = self.__class__.__module__ - return json.dumps(json_obj) +def to_json(self): + """ + 将 Python 张量操作对象序列化为 JSON 格式的字符串表示形式。 + + 返回: + str: 表示序列化对象的 JSON 格式字符串。 + + """ + json_obj = {} #创建一个空字典来存储 JSON 数据 + json_trans = {} #创建一个空字典来存储与数据 + + if "transforms" in self.__dict__.keys(): + #检查对象是否具有“transforms”属性（转换列表） + json_list = [] + + #遍历每个转换并将其序列化为 JSON + for transform in self.transforms: + json_list.append(json.loads(transform.to_json())) + + json_trans["transforms"] = json_list#将序列化的转换添加到 JSON 数据中 + self.__dict__.pop("transforms") #从对象中移除“transforms”属性 + + if "output_type" in self.__dict__.keys(): + #检查对象是否有“output_type”属性 + json_trans["output_type"] = np.dtype( + self.__dict__["output_type"]).name # 将数据类型序列化为名称形式 + self.__dict__.pop("output_type") # 从对象中移除“output_type”属性 + + json_obj["tensor_op_params"] = self.__dict__ #将对象的属性添加到 JSON 数据中 + json_obj["tensor_op_params"].update(json_trans) #合并与转换相关的数据 + json_obj["tensor_op_name"] = self.__class__.__name__ #添加类名 + json_obj["python_module"] = self.__class__.__module__ #添加模块名(?) + + return json.dumps(json_obj) #序列化整个 JSON 数据并以字符串形式返回 + + @classmethod @classmethod def from_json(cls, json_string): - """ - Base from_json for Python tensor operations class - """ + #从JSON字符串反序列化操作并构建新实例 + #传入的JSON字符串解析为 Python 字典 json_obj = json.loads(json_string) + + #创建一个新的操作实例 new_op = cls.__new__(cls) + + #将该实例的内部字典(__dict__)替换为从 JSON 中解析的字典 new_op.__dict__ = json_obj + + #检查是否存在"transforms" 键，如果存在，则需要对其中的 transforms 进行反序列化 if "transforms" in json_obj.keys(): - # operations which have transforms as input, need to call _from_json() for each transform to deseriallize transforms = [] + + #遍历transforms列表中的每个 JSON 表示的操作，并将其解析 for json_op in json_obj["transforms"]: + #获取操作所在的 Python 模块并根据模块和操作名称创建相应的操作实例 transforms.append(getattr( sys.modules[json_op["python_module"]], json_op["tensor_op_name"]).from_json( json.dumps(json_op["tensor_op_params"]))) + #将反序列化后的操作列表赋值给新操作实例的 transforms 属性 new_op.transforms = transforms + #检查是否存在"output_type"的键，如存在，转换为numpy.dtype数据类型，并赋值给新实例的output_type if "output_type" in json_obj.keys(): output_type = np.dtype(json_obj["output_type"]) new_op.output_type = output_type + #返回一个新操作实例 return new_op + class OneHotOp(PyTensorOperation): """ - Apply one hot encoding transformation to the input label, make label be more smoothing and continuous. + 对输入标签应用独热编码变换，使标签更平滑和连续化。 Args: - num_classes (int): Number of classes of objects in dataset. - It should be larger than the largest label number in the dataset. - smoothing_rate (float, optional): Adjustable hyperparameter for label smoothing level. - (Default=0.0 means no smoothing is applied.) + num_classes (int): 数据集中的对象类别数量。 + 应大于数据集中的最大标签数。 + smoothing_rate (float, 可选): 标签平滑级别的可调超参数。 + （默认值=0.0 表示不应用平滑处理。） Raises: - TypeError: `num_classes` is not of type int. - TypeError: `smoothing_rate` is not of type float. - ValueError: `smoothing_rate` is not in range [0.0, 1.0]. + TypeError: `num_classes` 不是整数类型。 + TypeError: `smoothing_rate` 不是浮点数类型。 + ValueError: `smoothing_rate` 不在范围 [0.0, 1.0] 内。 Supported Platforms: ``CPU`` Examples: - >>> # Assume that dataset has 10 classes, thus the label ranges from 0 to 9 + >>> # 假设数据集有 10 个类别，因此标签范围从 0 到 9 >>> transforms_list = [py_transforms.OneHotOp(num_classes=10, smoothing_rate=0.1)] >>> transform = py_transforms.Compose(transforms_list) >>> mnist_dataset = mnist_dataset.map(input_columns=["label"], operations=transform) """ @check_one_hot_op + #数据迁移处理 + """ + self.num_classes = num_classes + self.smoothing_rate = smoothing_rate + self.random = False + """ def __init__(self, num_classes, smoothing_rate=0.0): self.num_classes = num_classes self.smoothing_rate = smoothing_rate @@ -119,17 +150,19 @@ class OneHotOp(PyTensorOperation): def __call__(self, label): """ - Call method. + 调用方法。 Args: - label (numpy.ndarray): label to be applied label smoothing. + label (numpy.ndarray): 要应用标签平滑处理的标签。 Returns: - label (numpy.ndarray), label after being Smoothed. + label (numpy.ndarray): 经过平滑处理后的标签。 """ + #将其转换为独热编码 return util.one_hot_encoding(label, self.num_classes, self.smoothing_rate) + class Compose(PyTensorOperation): """ Compose a list of transforms. @@ -210,8 +243,8 @@ class Compose(PyTensorOperation): return util.compose(self.transforms, *args) @staticmethod - def reduce(operations): - """ +def reduce(operations): + """ Wraps adjacent Python operations in a Compose to allow mixing of Python and C++ operations. Args: @@ -219,32 +252,46 @@ class Compose(PyTensorOperation): Returns: list, the reduced list of operations. - """ - # import nn and ops locally for type check - from mindspore import nn, ops - for item in operations: - if isinstance(item, (nn.Cell, ops.Primitive)): - raise ValueError("Input operations should not contain network computing operator like in " - "mindspore.nn or mindspore.ops, got operation:", str(item)) - if len(operations) == 1: - if str(operations).find("c_transform") >= 0 or isinstance(operations[0], TensorOperation): - return operations - return [util.FuncWrapper(operations[0])] + """ + #从mindspore 模块中局部导入nn和ops + from mindspore import nn, ops + + #寻找列表中是否包含网络计算操作符 + for item in operations: + if isinstance(item, (nn.Cell, ops.Primitive)): + raise ValueError("Input operations should not contain network computing operator like in " + "mindspore.nn or mindspore.ops, got operation:", str(item)) + + # 如果操作列表中只有一个操作 + if len(operations) == 1: + # 如果该操作是 C++ 操作或 TensorOperation 的实例，返回该操作 + if str(operations).find("c_transform") >= 0 or isinstance(operations[0], TensorOperation): + return operations + # 将该操作包装在 util.FuncWrapper 中，并返回包装后的列表 + return [util.FuncWrapper(operations[0])] + + # 如果操作列表包含多个 + new_ops, start_ind, end_ind = [], 0, 0 + + # 遍历 + for i, op in enumerate(operations): + if str(op).find("c_transform") >= 0: + # 如果有 C++ 操作，将之前的所有操作打包成一个Compose操作 + if start_ind != end_ind: + new_ops.append(Compose(operations[start_ind:end_ind])) + new_ops.append(op) + start_ind, end_ind = i + 1, i + 1 + else: + end_ind += 1 + + #在循环结束后，检查并确保最后一组操作也被打包成 Compose 操作，并添加到新的操作列表中( + #(？ + if start_ind != end_ind: + new_ops.append(Compose(operations[start_ind:end_ind])) + + # 返回经过优化的操作列表 + return new_ops - new_ops, start_ind, end_ind = [], 0, 0 - for i, op in enumerate(operations): - if str(op).find("c_transform") >= 0: - # reset counts - if start_ind != end_ind: - new_ops.append(Compose(operations[start_ind:end_ind])) - new_ops.append(op) - start_ind, end_ind = i + 1, i + 1 - else: - end_ind += 1 - # do additional check in case the last operation is a Python operation - if start_ind != end_ind: - new_ops.append(Compose(operations[start_ind:end_ind])) - return new_ops class RandomApply(PyTensorOperation): @@ -365,19 +412,31 @@ class RandomOrder(PyTensorOperation): ... py_vision.ToTensor()]) >>> image_folder_dataset = image_folder_dataset.map(operations=transforms, input_columns=["image"]) """ - @check_transforms_list def __init__(self, transforms): - self.transforms = transforms + """ + 初始化 Compose 类的实例。 + + Args: + transforms (list): 一个包含要应用的转换操作的列表。 + + Raises: + TypeError: 如果 `transforms` 不是列表类型。 + ValueError: 如果 `transforms` 列表为空。 + + Returns: + None + """ + self.transforms = transforms # 将传入的转换操作列表存储在对象属性 transforms 中 def __call__(self, img): """ - Call method. - + 调用方法，用于将一组转换操作应用到输入图像上。 Args: - img (PIL image): Image to apply transformations in a random order. - + img: 需要进行转换操作的图像。 Returns: - img (PIL image), Transformed image. + img: 转换后的图像。 + Raises: + TypeError: 如果输入图像或任何转换操作无效。 """ - return util.random_order(img, self.transforms) + return util.random_order(img, self.transforms) # 调用 random_order 函数，以随机顺序应用转换操作并返回转换后的图像 \ No newline at end of file diff --git a/mindspore/python/mindspore/dataset/transforms/py_transforms_util.py b/mindspore/python/mindspore/dataset/transforms/py_transforms_util.py index 25c86d4aeff..d5317997d18 100644 --- a/mindspore/python/mindspore/dataset/transforms/py_transforms_util.py +++ b/mindspore/python/mindspore/dataset/transforms/py_transforms_util.py @@ -24,15 +24,29 @@ from ..core.py_util_helpers import is_numpy, ExceptionHandler def all_numpy(args): - """ for multi-input lambdas""" + """ + 检查给定的参数是否全部为 NumPy 数组。 + + Args: + args: 可以是单个值或一个包含多个值的元组。 + + Returns: + bool: 如果全部都是 NumPy 数组，则返回 True；否则返回 False。 + """ if isinstance(args, tuple): + #若参数是一个tuple，则迭代遍历元组中的每个值 for value in args: + #使用 is_numpy 函数检查每个值是否为 NumPy 数组 if not is_numpy(value): + #如果发现任何一个值不是 NumPy 数组，则返回 False return False + #如果所有的值都是 NumPy 数组，则返回 True return True + #如果参数不是元组，而是单个值 return is_numpy(args) + def compose(transforms, *args): """ Compose a list of transforms and apply on the image. @@ -58,35 +72,39 @@ def compose(transforms, *args): def one_hot_encoding(label, num_classes, epsilon): + #独热编码处理并进行平滑转换 """ - Apply label smoothing transformation to the input label, and make label be more smoothing and continuous. + 对输入的标签进行独热编码并应用标签平滑转换，使标签更平滑和连续。 Args: - label (numpy.ndarray): label to be applied label smoothing. - num_classes (int): Num class of object in dataset, value should over 0. - epsilon (float): The adjustable Hyper parameter. Default is 0.0. + label (numpy.ndarray): 要应用标签平滑的标签。 + num_classes (int): 数据集中的对象类别数，必须大于0。 + epsilon (float): 可调整的超参数。默认值为0.0。 Returns: - img (numpy.ndarray), label after being one hot encoded and done label smoothed. + one_hot_label (numpy.ndarray): 经过独热编码和标签平滑后的标签。 Examples: - >>> # assume num_classes = 5 - >>> # 1) input np.array(3) output [0, 0, 0, 1, 0] - >>> # 2) input np.array([4, 2, 0]) output [[0, 0, 0, 0, 1], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0]] - >>> # 3) input np.array([[4], [2], [0]]) output [[[0, 0, 0, 0, 1]], [[0, 0, 1, 0, 0][, [[1, 0, 0, 0, 0]]] + >>> # 假设 num_classes = 5 + >>> # 1) 输入 np.array(3) 输出 [0, 0, 0, 1, 0] + >>> # 2) 输入 np.array([4, 2, 0]) 输出 [[0, 0, 0, 0, 1], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0]] + >>> # 3) 输入 np.array([[4], [2], [0]]) 输出 [[[0, 0, 0, 0, 1]], [[0, 0, 1, 0, 0][, [[1, 0, 0, 0, 0]]] """ - if isinstance(label, np.ndarray): # the numpy should be () or (1, ) or shape: (n, 1) + if isinstance(label, np.ndarray): # 检查输入是否为 NumPy 数组 + #判断label的数据类型是否为指定的整数类型（np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64）之一。 if label.dtype not in [np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64]: raise ValueError('the input numpy type should be int, but the input is: ' + str(label.dtype)) - if label.ndim == 0: + if label.ndim == 0: # 处理标量标签的情况 if label >= num_classes: raise ValueError('the num_classes is smaller than the category number.') + #创建一个全零的独热编码数组，将标签对应的位置设为1 one_hot_label = np.zeros((num_classes), dtype=int) one_hot_label[label] = 1 - else: + else: # 处理多维标签的情况 + # 使用.flatten()将多维标签展平成一维数组，以便于处理 label_flatten = label.flatten() for item in label_flatten: if item >= num_classes: @@ -94,38 +112,46 @@ def one_hot_encoding(label, num_classes, epsilon): ' is smaller than the category number:' + str(item)) num_elements = label_flatten.size + #创建一个形状为(num_elements, num_classes)的使用零填充的numpy数组 one_hot_label = np.zeros((num_elements, num_classes), dtype=int) for index in range(num_elements): one_hot_label[index][label_flatten[index]] = 1 - new_shape = [] + #设置新的数组形状 + new_shape = []new_shape + #将原label.shape移到里面 for dim in label.shape: new_shape.append(dim) + #对象类别扔到维度里面 new_shape.append(num_classes) one_hot_label = one_hot_label.reshape(new_shape) else: + #报错：数据格式无效，应该是一个 NumPy 数组 raise ValueError('the input is invalid, it should be numpy.ndarray.') + # 应用标签平滑，通过将原始独热编码乘以 (1 - epsilon) 并加上 epsilon / num_classes 来平滑标签，epsilon默认为0 return (1 - epsilon) * one_hot_label + epsilon / num_classes def random_order(img, transforms): """ - Applies a list of transforms in a random order. + 将一系列变换以随机顺序应用到图像上。 Args: - img: Image to be applied transformations in a random order. - transforms (list): List of the transformations to be applied. + img: 要以随机顺序应用变换的图像。 + transforms (list): 要应用的变换的列表。 Returns: - img, Transformed image. + img: 经过变换的图像。 """ + # 随机打乱transforms元素顺序 random.shuffle(transforms) for transform in transforms: + # 对图像应用每个变换 img = transform(img) + #返回img return img - def random_apply(img, transforms, prob): """ Apply a list of transformation, randomly with a given probability. @@ -156,42 +182,51 @@ def random_choice(img, transforms): Returns: img, Transformed image. """ + #用 Python 的 random.choice 函数从 transforms 列表中随机选择一个变换方法，非常ez return random.choice(transforms)(img) class FuncWrapper: """ - Wrap function with try except logic, mainly for warping python function. + 用于包装函数并添加 try except 逻辑，主要用于包装 Python 函数。 + 包装后的函数会捕获异常并重新引发，以便处理 Python 函数的错误。 Args: - transform: Callable python function. + transform: 可调用的 Python 函数。 Returns: - result, data after apply transformation. + result, 应用变换后的数据。 """ def __init__(self, transform): + #检查输入是否是可以调用的 Python 函数 if not callable(transform): raise ValueError("Input operations should be callable python function, but got: " + str(transform)) self.transform = transform try: + #如果函数有属性"random"并且其值为 False，则将 self.random 设置为 False if hasattr(self.transform, "random") and not self.transform.random: self.random = False except KeyError: + # 如果属性 "random" 不存在，把self.random设置为 True self.random = True def __call__(self, *args): result = None try: + #调用包装的函数，并捕获可能的异常 result = self.transform(*args) except Exception: + #发生异常后，创建一个异常处理器对象，并重新引发异常 result = ExceptionHandler(where="in map(or batch) worker and execute python function") result.reraise() return result def to_json(self): + # 如果包装的函数是 FunctionType 类型，将其序列化为 JSON 对象 if isinstance(self.transform, FunctionType): json_obj = {} json_obj["tensor_op_name"] = self.transform.__name__ json_obj["python_module"] = self.__class__.__module__ return json.dumps(json_obj) + # 否则，调用包装的函数的 to_json 方法 return self.transform.to_json() diff --git a/mindspore/python/mindspore/mindrecord/filewriter.py b/mindspore/python/mindspore/mindrecord/filewriter.py index c5becd75672..0e51d917526 100644 --- a/mindspore/python/mindspore/mindrecord/filewriter.py +++ b/mindspore/python/mindspore/mindrecord/filewriter.py @@ -43,39 +43,35 @@ class FileWriter: Args: file_name (str): File name of MindRecord file. shard_num (int, optional): The Number of MindRecord files. - It should be between [1, 1000]. Default: 1. - overwrite (bool, optional): Whether to overwrite if the file already exists. Default: False. + It should be between [1, 1000]. Default: ``1`` . + overwrite (bool, optional): Whether to overwrite if the file already exists. Default: ``False`` . Raises: ParamValueError: If `file_name` or `shard_num` or `overwrite` is invalid. Examples: - >>> from mindspore.mindrecord import FileWriter - >>> schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}} - >>> indexes = ["file_name", "label"] - >>> data = [{"file_name": "1.jpg", "label": 0, - ... "data": b"\x10c\xb3w\xa8\xee$o&\xd4\x00\xf8\x129\x15\xd9\xf2q\xc0\xa2\x91YFUO\x1dsE1"}, - ... {"file_name": "3.jpg", "label": 99, - ... "data": b"\xaf\xafU<\xb8|6\xbd}\xc1\x99[\xeaj+\x8f\x84\xd3\xcc\xa0,i\xbb\xb9-\xcdz\xecp{T\xb1"}] - >>> writer = FileWriter(file_name="test.mindrecord", shard_num=1, overwrite=True) - >>> writer.add_schema(schema_json, "test_schema") - 0 - >>> writer.add_index(indexes) - MSRStatus.SUCCESS - >>> writer.write_raw_data(data) - MSRStatus.SUCCESS - >>> writer.commit() - MSRStatus.SUCCESS + from mindspore.mindrecord import FileWriter + + writer = FileWriter(file_name="test.mindrecord", shard_num=1, overwrite=True) + schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}} + schema_id = writer.add_schema(schema_json, "test_schema") + indexes = ["file_name", "label"] + status = writer.add_index(indexes) + for i in range(10): + ... data = [{"file_name": str(i) + ".jpg", "label": i, + ... "data": b"\x10c\xb3w\xa8\xee$o& MAX_SHARD_COUNT: @@ -85,7 +81,7 @@ class FileWriter: raise ParamValueError("Parameter shard_num's type is not int.") else: raise ParamValueError("Parameter shard_num is None.") - + #若overwrite类型不是bool，也抛出异常 if not isinstance(overwrite, bool): raise ParamValueError("Parameter overwrite's type is not bool.") @@ -99,7 +95,7 @@ class FileWriter: self._paths = ["{}{}".format(self._file_name, str(x).rjust(suffix_shard_size, '0')) for x in range(self._shard_num)] - + self._overwrite = overwrite self._append = False self._flush = False @@ -107,6 +103,13 @@ class FileWriter: self._writer = ShardWriter() self._generator = None + # parallel write mode + self._parallel_writer = None + self._writers = None + self._queue = None + self._workers = None + self._index_workers = None + @classmethod def open_for_append(cls, file_name): r""" @@ -125,22 +128,22 @@ class FileWriter: MRMOpenForAppendError: If failed to open file for appending data. Examples: - >>> from mindspore.mindrecord import FileWriter - >>> schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}} - >>> data = [{"file_name": "1.jpg", "label": 0, + from mindspore.mindrecord import FileWriter + + data = [{"file_name": "0.jpg", "label": 0, ... "data": b"\x10c\xb3w\xa8\xee$o&>> writer = FileWriter(file_name="test.mindrecord", shard_num=1, overwrite=True) - >>> writer.add_schema(schema_json, "test_schema") - 0 - >>> writer.write_raw_data(data) - MSRStatus.SUCCESS - >>> writer.commit() - MSRStatus.SUCCESS - >>> write_append = FileWriter.open_for_append("test.mindrecord") - >>> write_append.write_raw_data(data) - MSRStatus.SUCCESS - >>> write_append.commit() - MSRStatus.SUCCESS + writer = FileWriter(file_name="test.mindrecord", shard_num=1, overwrite=True) + schema_json = {"file_name": {"type": "string"}, "label": {"type": "int32"}, "data": {"type": "bytes"}} + schema_id = writer.add_schema(schema_json, "test_schema") + status = writer.write_raw_data(data) + status = writer.commit() + + write_append = FileWriter.open_for_append("test.mindrecord") + for i in range(9): + ... data = [{"file_name": str(i+1) + ".jpg", "label": i, + ... "data": b"\x10c\xb3w\xa8\xee$o&`_. - - Args: - source (str): The cifar10 directory to be transformed. - destination (str): MindRecord file path to transform into, ensure that no file with the same name - exists in the directory. - - Raises: - ValueError: If source or destination is invalid. - """ - def __init__(self, source, destination): check_filename(source) self.source = source @@ -59,10 +63,12 @@ class Cifar10ToMR: train_data_flag = False test_data_flag = False for file in files: + #检查当前文件名是否以data_batch_或者test_batch开头 if file.startswith("data_batch_"): train_data_flag = True if file.startswith("test_batch"): test_data_flag = True + #若没有（train_data_flag或者test_data_flag为False），报错 if not train_data_flag: raise PathNotExistsError("data_batch_*") @@ -73,36 +79,30 @@ class Cifar10ToMR: self.destination = destination self.writer = None + # pylint: disable=missing-docstring def run(self, fields=None): - """ - Execute transformation from cifar10 to MindRecord. - - Args: - fields (list[str], optional): A list of index fields. Default: None. - - Returns: - MSRStatus, SUCCESS or FAILED. - """ - + #若fields不是列表，报错The parameter fields should be None or list if fields and not isinstance(fields, list): raise ValueError("The parameter fields should be None or list") - + # 创建一个Cifar10类的实例 cifar10_data = Cifar10(self.source, False) cifar10_data.load_data() - + images = cifar10_data.images logger.info("train images: {}".format(images.shape)) + #加载并记录数据集图像的形状 labels = cifar10_data.labels logger.info("train images label: {}".format(labels.shape)) - + test_images = cifar10_data.Test.images logger.info("test images: {}".format(test_images.shape)) + #记录测试标签的形状 test_labels = cifar10_data.Test.labels logger.info("test images label: {}".format(test_labels.shape)) data_list = _construct_raw_data(images, labels) test_data_list = _construct_raw_data(test_images, test_labels) - + # 若有转换MindRecord文件的失败则返回FAILED if _generate_mindrecord(self.destination, data_list, fields, "img_train") != SUCCESS: return FAILED if _generate_mindrecord(self.destination + "_test", test_data_list, fields, "img_test") != SUCCESS: @@ -111,15 +111,36 @@ class Cifar10ToMR: def transform(self, fields=None): """ - Encapsulate the run function to exit normally + Execute transformation from cifar10 to MindRecord. + + Note: + Please refer to the Examples of :class:`mindspore.mindrecord.Cifar10ToMR` . Args: - fields (list[str], optional): A list of index fields. Default: None. + fields (list[str], optional): A list of index fields. Default: ``None`` . For index field settings, + please refer to :func:`mindspore.mindrecord.FileWriter.add_index` . Returns: MSRStatus, SUCCESS or FAILED. - """ + Raises: + ParamTypeError: If index field is invalid. + MRMOpenError: If failed to open MindRecord file. + MRMValidateDataError: If data does not match blob fields. + MRMSetHeaderError: If failed to set header. + MRMWriteDatasetError: If failed to write dataset. + TypeError: If `parallel_writer` is not bool. + ValueError: If parameter `fields` is invalid. + + 异常： ParamTypeError: 如果索引字段无效。 + MRMOpenError: 如果无法打开 MindRecord 文件。 + MRMValidateDataError: 如果数据与 blob 字段不匹配。 + MRMSetHeaderError: 如果无法设置标题。 + MRMWriteDatasetError: 如果无法写入数据集。 + TypeError: 如果 parallel_writer 不是布尔值。 + ValueError: 如果参数 fields 无效。 + """ + #一个ExceptionThread线程类对象（？ 目标为 self.run t = ExceptionThread(target=self.run, kwargs={'fields': fields}) t.daemon = True t.start() @@ -129,55 +150,66 @@ class Cifar10ToMR: return t.res -def _construct_raw_data(images, labels): - """ - Construct raw data from cifar10 data. + def _construct_raw_data(images, labels): + #从cifar10中构建原始数据 + """ + Construct raw data from cifar10 data. - Args: - images (list): image list from cifar10. - labels (list): label list from cifar10. + Args: + images (list): image list from cifar10. + labels (list): label list from cifar10. - Returns: - list[dict], data dictionary constructed from cifar10. - """ + Returns: + list[dict], data dictionary constructed from cifar10. + """ + #images，labels表示来自cifar10的图像和标签 - if not cv2: - raise ModuleNotFoundError("opencv-python module not found, please use pip install it.") + #若没有导入opencv-python模块，报错 + if not cv_import: + raise ModuleNotFoundError("opencv-python module not found, please use pip install it.") - raw_data = [] - for i, img in enumerate(images): - label = np.int(labels[i][0]) - _, img = cv2.imencode(".jpeg", img[..., [2, 1, 0]]) - row_data = {"id": int(i), - "data": img.tobytes(), - "label": int(label)} - raw_data.append(row_data) - return raw_data + raw_data = [] + #遍历图像数据和标签数据 + for i, img in enumerate(images): + label = np.int(labels[i][0]) + _, img = cv_import.imencode(".jpeg", img[..., [2, 1, 0]]) + #创建一个整合信息用的字典 + row_data = {"id": int(i), + "data": img.tobytes(),#将数组的内容转换为一个字节字符串对象 + "label": int(label)}#将标签转换为整数 + raw_data.append(row_data) + return raw_data -def _generate_mindrecord(file_name, raw_data, fields, schema_desc): - """ - Generate MindRecord file from raw data. + def _generate_mindrecord(file_name, raw_data, fields, schema_desc): + """ + Generate MindRecord file from raw data. - Args: - file_name (str): File name of MindRecord File. - fields (list[str]): Fields would be set as index which - could not belong to blob fields and type could not be 'array' or 'bytes'. - raw_data (dict): dict of raw data. - schema_desc (str): String of schema description. + Args: + file_name (str): File name of MindRecord File. + fields (list[str]): Fields would be set as index which + could not belong to blob fields and type could not be 'array' or 'bytes'. + raw_data (dict): dict of raw data. + schema_desc (str): String of schema description. + # 参数: + # file_name (str): MindRecord文件的文件名。 + # fields (list[str]): 将被设置为索引的字段，不能属于blob字段且类型不能为 'array' 或 'bytes'。 + # raw_data (dict): 原始数据的字典。 + # schema_desc (str): schema 描述的字符串。 + Returns: + MSRStatus, SUCCESS or FAILED. + """ - Returns: - MSRStatus, SUCCESS or FAILED. - """ + schema = {"id": {"type": "int64"}, "label": {"type": "int64"}, + "data": {"type": "bytes"}} + #记录MindRecord schema + logger.info("transformed MindRecord schema is: {}".format(schema)) - schema = {"id": {"type": "int64"}, "label": {"type": "int64"}, - "data": {"type": "bytes"}} - - logger.info("transformed MindRecord schema is: {}".format(schema)) - - writer = FileWriter(file_name, 1) - writer.add_schema(schema, schema_desc) - if fields and isinstance(fields, list): - writer.add_index(fields) - writer.write_raw_data(raw_data) - return writer.commit() + writer = FileWriter(file_name, 1) + writer.add_schema(schema, schema_desc) + # 若有fields参数且它是一个列表，为它添加索引字段 + if fields and isinstance(fields, list): + writer.add_index(fields) + writer.write_raw_data(raw_data) + #返回结果 + return writer.commit() \ No newline at end of file diff --git a/mindspore/python/mindspore/nn/layer/math.py b/mindspore/python/mindspore/nn/layer/math.py index 672a297c020..f614dd01050 100644 --- a/mindspore/python/mindspore/nn/layer/math.py +++ b/mindspore/python/mindspore/nn/layer/math.py @@ -99,20 +99,22 @@ class ReduceLogSumExp(Cell): def __init__(self, axis, keep_dims=False): """Initialize ReduceLogSumExp.""" super(ReduceLogSumExp, self).__init__() + # Checking and assigning the axis and keep_dims values validator.check_value_type('axis', axis, [int, list, tuple], self.cls_name) validator.check_value_type('keep_dims', keep_dims, [bool], self.cls_name) self.axis = axis + # Creating necessary operators for calculation self.exp = P.Exp() self.sum = P.ReduceSum(keep_dims) self.log = P.Log() def construct(self, x): + # Performing the ReduceLogSumExp operation exp = self.exp(x) sumexp = self.sum(exp, self.axis) logsumexp = self.log(sumexp) return logsumexp - class Range(Cell): r""" Creates a sequence of numbers in range [start, limit) with step size delta. @@ -132,7 +134,7 @@ class Range(Cell): delta (Union[int, float]): Increment of the range. It can not be equal to zero. Default: 1. Outputs: - Tensor, the dtype is int if the dtype of `start`, `limit` and `delta` all are int. Otherwise, dtype is float. + Tensor, the dtype is int if the dtype of `start`, `limit`, and `delta` all are int. Otherwise, dtype is float. Supported Platforms: ``Ascend`` ``GPU`` ``CPU`` @@ -147,16 +149,24 @@ class Range(Cell): def __init__(self, start, limit=None, delta=1): """Initialize Range.""" super(Range, self).__init__() + # Checking if delta is zero, which is not allowed if delta == 0: raise ValueError(f"For '{self.cls_name}', the 'delta' can not be zero.") + + # Creating a numpy array with the specified range data = np.arange(start, limit, delta) + + # Determining the dtype of the tensor based on the data type of start, limit, and delta if data.dtype == np.float: self.ms_dtype = mstype.float32 else: self.ms_dtype = mstype.int32 + + # Creating a MindSpore tensor with the specified data and dtype self.result_tensor = Tensor(data, dtype=self.ms_dtype) def construct(self): + # Returning the result tensor return self.result_tensor @@ -246,10 +256,13 @@ class LGamma(Cell): self.isfinite = P.IsFinite() def construct(self, x): + # Checking the dtype of the input x input_dtype = self.dtype(x) _check_input_dtype("x", input_dtype, [mstype.float16, mstype.float32], self.cls_name) + # Defining some constant values infinity = self.fill(input_dtype, self.shape(x), self.inf) + # Using Euler's reflection formula if x is less than 0.5 need_to_reflect = self.less(x, 0.5) neg_input = -x z = self.select(need_to_reflect, neg_input, x - 1) @@ -350,8 +363,10 @@ class DiGamma(Cell): self.logicaland = P.LogicalAnd() def construct(self, x): + # Checking the dtype of the input x input_dtype = self.dtype(x) _check_input_dtype("x", input_dtype, [mstype.float16, mstype.float32], self.cls_name) + # Applying Euler's reflection formula if x is less than 0.5 need_to_reflect = self.less(x, 0.5) neg_input = -x z = self.select(need_to_reflect, neg_input, x - 1) @@ -380,6 +395,7 @@ class DiGamma(Cell): nan, real_result) + eps_fp32 = Tensor(np.finfo(np.float32).eps, mstype.float32) diff --git a/mindspore/python/mindspore/nn/loss/loss.py b/mindspore/python/mindspore/nn/loss/loss.py index 714f947b790..f92e116dfc7 100644 --- a/mindspore/python/mindspore/nn/loss/loss.py +++ b/mindspore/python/mindspore/nn/loss/loss.py @@ -318,7 +318,7 @@ class RMSELoss(LossBase): r""" RMSELoss creates a criterion to measure the root mean square error between :math:`x` and :math:`y` element-wise, where :math:`x` is the input and :math:`y` is the labels. - + For simplicity, let :math:`x` and :math:`y` be 1-dimensional Tensor with length :math:`N`, the loss of :math:`x` and :math:`y` is given as: @@ -354,29 +354,30 @@ class RMSELoss(LossBase): >>> print(output) 1.0 """ - + def __init__(self): """Initialize RMSELoss.""" - super(RMSELoss, self).__init__() - self.MSELoss = MSELoss() + super(RMSELoss, self).__init__() # 调用父类的初始化方法 + self.MSELoss = MSELoss() # 创建一个MSELoss的实例，用于后续的计算 def construct(self, logits, label): + # 计算均方误差，并取平方根，得到RMSE rmse_loss = F.sqrt(self.MSELoss(logits, label)) - return rmse_loss + return rmse_loss # 返回RMSE作为损失值 class MAELoss(LossBase): r""" MAELoss creates a criterion to measure the average absolute error between :math:`x` and :math:`y` element-wise, where :math:`x` is the input and :math:`y` is the labels. - + For simplicity, let :math:`x` and :math:`y` be 1-dimensional Tensor with length :math:`N`, the unreduced loss (i.e. with argument reduction set to 'none') of :math:`x` and :math:`y` is given as: .. math:: \ell(x, y) = L = \{l_1,\dots,l_N\}^\top, \quad \text{with } l_n = \left| x_n - y_n \right|, - + where :math:`N` is the batch size. If `reduction` is not 'none', then: .. math:: @@ -427,14 +428,14 @@ class MAELoss(LossBase): def __init__(self, reduction='mean'): """Initialize MAELoss.""" - super(MAELoss, self).__init__(reduction) - self.abs = P.Abs() + super(MAELoss, self).__init__(reduction) # 调用父类的初始化方法，同时传入 reduction 参数 + self.abs = P.Abs() # 创建一个 P.Abs() 的实例并存储在 self.abs 中 def construct(self, logits, label): - _check_is_tensor('logits', logits, self.cls_name) - _check_is_tensor('labels', label, self.cls_name) - x = self.abs(logits - label) - return self.get_loss(x) + _check_is_tensor('logits', logits, self.cls_name) # 检查 logits 是否是 Tensor 类型 + _check_is_tensor('labels', label, self.cls_name) # 检查 labels 是否是 Tensor 类型 + x = self.abs(logits - label) # 计算 |logits - label| + return self.get_loss(x) # 调用 get_loss 方法计算损失值，并返回 class SmoothL1Loss(LossBase): diff --git a/mindspore/python/mindspore/nn/transformer/loss.py b/mindspore/python/mindspore/nn/transformer/loss.py index 5b1d361eb8a..810dcdb3fed 100644 --- a/mindspore/python/mindspore/nn/transformer/loss.py +++ b/mindspore/python/mindspore/nn/transformer/loss.py @@ -68,6 +68,7 @@ class CrossEntropyLoss(Cell): if not isinstance(parallel_config, OpParallelConfig) and not isinstance(parallel_config): raise TypeError("For 'CrossEntropyLoss', the class variable 'parallel_config' must be OpParallelConfig" ", but got the type: {}.".format(type(parallel_config))) + #确保 parallel_config 的类型是 OpParallelConfig 或者其子类，如果不是则提供一个错误信息 dp = parallel_config.data_parallel mp = parallel_config.model_parallel self.sum = P.ReduceSum().shard(((dp, mp),)) @@ -93,38 +94,38 @@ class CrossEntropyLoss(Cell): self.div2 = P.RealDiv() def construct(self, logits, label, input_mask): - self._check_input(logits, label, input_mask) + self._check_input(logits, label, input_mask)#检查数据是否符合要求 # the shape is [bs*seq_length, vocab_size] - logits = F.cast(logits, mstype.float32) + logits = F.cast(logits, mstype.float32)#将logits强制转换为32位float的形式 # LogSoftmax for logits over last dimension - _, logit_max = self.max(logits) - logit_sub = self.sub(logits, logit_max) - logit_exp = self.exp(logit_sub) + _, logit_max = self.max(logits)#计算logits样本中的最大值 + logit_sub = self.sub(logits, logit_max)#令logits减去最大值 + logit_exp = self.exp(logit_sub)#计算logit_sub的指数值 exp_sum = self.sum(logit_exp, -1) - exp_sum = P.Reshape()(exp_sum, (F.shape(exp_sum)[0], 1)) - softmax_result = self.div(logit_exp, exp_sum) - log_softmax_result = self.log(self.add(softmax_result, self.eps_const)) + exp_sum = P.Reshape()(exp_sum, (F.shape(exp_sum)[0], 1))#对指数值进行求和 + softmax_result = self.div(logit_exp, exp_sum)#计算softmax + log_softmax_result = self.log(self.add(softmax_result, self.eps_const))#计算对数softmax # Flatten label to [bs*seq_length] - label = P.Reshape()(label, (-1,)) + label = P.Reshape()(label, (-1,))#将label变为一维向量 # Get onehot label [bs*seq_length, vocab_size] one_hot_label = self.onehot(label, F.shape(logits)[-1], self.on_value, - self.off_value) + self.off_value)#将label转换为独热编码格式 # Cross-Entropy loss - loss = self.mul(log_softmax_result, one_hot_label) - loss_unsum = self.neg(loss) - loss_reduce = self.sum(loss_unsum, -1) + loss = self.mul(log_softmax_result, one_hot_label)#将softmax的对数值与标签的独热编码相乘 + loss_unsum = self.neg(loss)#将计算的loss取负数 + loss_reduce = self.sum(loss_unsum, -1)#对损失进行求和 # input_mask indicates whether there is padded inputs and for padded inputs it will not be counted into loss - input_mask = P.Reshape()(input_mask, (-1,)) - numerator = self.sum2(self.mul2(loss_reduce, input_mask)) + input_mask = P.Reshape()(input_mask, (-1,))#将input_mask变为一维向量 + numerator = self.sum2(self.mul2(loss_reduce, input_mask))#计算分子，将损失值与input_mask相乘并将结果相加 denominator = self.add2( self.sum2(input_mask), - P.Cast()(F.tuple_to_array((1e-5,)), mstype.float32)) - loss = self.div2(numerator, denominator) - return loss - + P.Cast()(F.tuple_to_array((1e-5,)), mstype.float32))#计算分母部分，将input_mask进行求和，并且加上一个1e-5避免分母为0 + loss = self.div2(numerator, denominator)#计算最终的损失值，将分子除以分母 + return loss#返回损失值 + def _check_input(self, logits, label, input_mask): r"""Check the input tensor shape and type""" _check_is_tensor('logits', logits, self.cls_name) diff --git a/mindspore/python/mindspore/numpy/__init__.py b/mindspore/python/mindspore/numpy/__init__.py index 87fe1603522..03eabacd65f 100644 --- a/mindspore/python/mindspore/numpy/__init__.py +++ b/mindspore/python/mindspore/numpy/__init__.py @@ -26,6 +26,7 @@ Note: - dtypes.py defines all the mindspore.numpy dtypes (mainly redirected from mindspore) """ +# Importing necessary modules and functions from other files from .array_ops import (transpose, expand_dims, squeeze, rollaxis, swapaxes, reshape, ravel, concatenate, where, atleast_1d, atleast_2d, atleast_3d, column_stack, hstack, dstack, vstack, stack, unique, moveaxis, @@ -33,6 +34,7 @@ from .array_ops import (transpose, expand_dims, squeeze, rollaxis, swapaxes, res flip, flipud, fliplr, hsplit, dsplit, take_along_axis, take, repeat, rot90, select, array_split, choose, size, array_str, apply_along_axis, piecewise, unravel_index, apply_over_axes) + from .array_creations import copy_ as copy from .array_creations import (array, asarray, asfarray, ones, zeros, full, randn, rand, randint, arange, linspace, logspace, eye, identity, empty, empty_like, @@ -41,9 +43,11 @@ from .array_creations import (array, asarray, asfarray, ones, zeros, full, randn diag, diag_indices, ix_, indices, geomspace, vander, hamming, hanning, bartlett, blackman, triu_indices, tril_indices, triu_indices_from, tril_indices_from, histogram_bin_edges, pad) + from .dtypes import (int_, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float_, float16, float32, float64, bool_, inf, nan, pi, numeric_types, PINF, NINF) + from .math_ops import (mean, inner, add, subtract, multiply, divide, true_divide, power, dot, outer, tensordot, absolute, std, var, average, minimum, matmul, square, sqrt, reciprocal, log, maximum, heaviside, amax, amin, @@ -58,11 +62,13 @@ from .math_ops import (mean, inner, add, subtract, multiply, divide, true_divide histogramdd, histogram2d, matrix_power, around, polyadd, polysub, polyval, polyder, polymul, polyint, result_type, unwrap, cumprod, ravel_multi_index, norm, bitwise_and, bitwise_or, bitwise_xor, invert, rint, correlate, radians) + from .logic_ops import (not_equal, less_equal, less, greater_equal, greater, equal, isfinite, isnan, isinf, isposinf, isneginf, isscalar, logical_and, logical_not, logical_or, logical_xor, in1d, isin, isclose, signbit, sometrue, array_equal, array_equiv) +# Aliasing some functions for convenience mod = remainder fabs = absolute round = around # pylint: disable=redefined-builtin @@ -75,6 +81,7 @@ sum = sum_ # pylint: disable=redefined-builtin del sum_ bitwise_not = invert +# Creating module-specific lists for easy management array_ops_module = ['transpose', 'expand_dims', 'squeeze', 'rollaxis', 'swapaxes', 'reshape', 'ravel', 'concatenate', 'where', 'atleast_1d', 'atleast_2d', 'atleast_3d', 'column_stack', 'hstack', 'dstack', 'vstack', 'stack', 'unique', 'moveaxis', @@ -113,6 +120,8 @@ logic_module = ['not_equal', 'less_equal', 'less', 'greater_equal', 'greater', ' 'logical_or', 'logical_xor', 'in1d', 'isin', 'isclose', 'signbit', 'sometrue', 'array_equal', 'array_equiv'] +# Combining all the modules and functions into a single list __all__ = array_ops_module + array_creations_module + math_module + logic_module + numeric_types +# Sorting the list alphabetically __all__.sort() diff --git a/mindspore/python/mindspore/run_check/run_check.py b/mindspore/python/mindspore/run_check/run_check.py index 225cd92a36b..675f44d5953 100644 --- a/mindspore/python/mindspore/run_check/run_check.py +++ b/mindspore/python/mindspore/run_check/run_check.py @@ -34,8 +34,9 @@ def _check_mul(): finally: pass - print(f"MindSpore version: ", ms.__version__) + print(f"MindSpore version: ", ms.__version__) # Print MindSpore version + # Create tensors and perform multiplication input_x = ms.Tensor(np.array([1.0, 2.0, 3.0]), ms.float32) input_y = ms.Tensor(np.array([4.0, 5.0, 6.0]), ms.float32) mul = ms.ops.Mul() @@ -55,10 +56,10 @@ def run_check(): The result of multiplication calculation is correct, MindSpore has been installed successfully! """ try: - _check_mul() + _check_mul() # Call _check_mul function to perform the check # pylint: disable=broad-except except Exception as e: - print("MindSpore running check failed.") - print(str(e)) + print("MindSpore running check failed.") # Print error message if check fails + print(str(e)) # Print specific error message finally: - pass + pass # Cleanup code if needed