mindspore/python/mindspore/nn/optim/adam.py

@@ -1,4 +1,4 @@
-# Copyright 2020-2021 Huawei Technologies Co., Ltd
+# Copyright 2020 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,6 +13,8 @@
 # limitations under the License.
 # ============================================================================
 """adam"""
+# 优化器 Adaptive Moment Estimation (Adam)算法的实现。
+
 import numpy as np
 
 from mindspore.common import dtype as mstype
@@ -24,17 +26,21 @@ from mindspore.common.parameter import Parameter
 from mindspore.common.tensor import Tensor
 from mindspore._checkparam import Validator as validator
 from mindspore._checkparam import Rel
-from .optimizer import Optimizer
-from .optimizer import opt_init_args_register
+from.optimizer import Optimizer
 
+# 定义adam求解器
+# 定义求解器的名称
 _adam_opt = C.MultitypeFuncGraph("adam_opt")
+# 定义求解器的操作
 _scaler_one = Tensor(1, mstype.int32)
 _scaler_ten = Tensor(10, mstype.float32)
 
 
-@_adam_opt.register("Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor",
+# 定义求解器的注册值
+@_adam_opt.register("Tensor", "Tensor", "Tensor", "Tensor", "Number", "Tensor", "Tensor", "Tensor",
                     "Tensor", "Bool", "Bool")
 def _update_run_op(beta1, beta2, eps, lr, weight_decay, param, m, v, gradient, decay_flag, optim_filter):
+	# 用于更新参数
     """
     Update parameters.
 
@@ -43,7 +49,7 @@ def _update_run_op(beta1, beta2, eps, lr, weight_decay, param, m, v, gradient, d
         beta2 (Tensor): The exponential decay rate for the 2nd moment estimations. Should be in range (0.0, 1.0).
         eps (Tensor): Term added to the denominator to improve numerical stability. Should be greater than 0.
         lr (Tensor): Learning rate.
-        weight_decay (numbers.Number): Weight decay. Should be equal to or greater than 0.
+        weight_decay (Number): Weight decay. Should be equal to or greater than 0.
         param (Tensor): Parameters.
         m (Tensor): m value of parameters.
         v (Tensor): v value of parameters.
@@ -54,96 +60,150 @@ def _update_run_op(beta1, beta2, eps, lr, weight_decay, param, m, v, gradient, d
     Returns:
         Tensor, the new value of v after updating.
     """
-    op_cast = P.Cast()
     if optim_filter:
+        # 定义乘法运算
         op_mul = P.Mul()
+        # 定义平方运算
         op_square = P.Square()
+        # 定义平方根运算
         op_sqrt = P.Sqrt()
+        # 定义转换运算
         op_cast = P.Cast()
+        # 定义reshape运算
         op_reshape = P.Reshape()
+        # 定义shape运算
         op_shape = P.Shape()
+
+        # 将参数转换为float32类型
         param_fp32 = op_cast(param, mstype.float32)
+        # 将模转换为float32类型
         m_fp32 = op_cast(m, mstype.float32)
+        # 将方差转换为float32类型
         v_fp32 = op_cast(v, mstype.float32)
+        # 将梯度转换为float32类型
         gradient_fp32 = op_cast(gradient, mstype.float32)
 
+        # 将beta1的类型设置为float32
         next_m = op_mul(beta1, m_fp32) + op_mul(op_cast(F.tuple_to_array((1.0,)), mstype.float32)
-                                                - beta1, gradient_fp32)
+                                                    - beta1, gradient_fp32)
 
+        # 将beta2的类型设置为float32
         next_v = op_mul(beta2, v_fp32) + op_mul(op_cast(F.tuple_to_array((1.0,)), mstype.float32)
-                                                - beta2, op_square(gradient_fp32))
+                                                    - beta2, op_square(gradient_fp32))
 
+        # 将梯度求平均
         update = next_m / (eps + op_sqrt(next_v))
+        # 如果没有指定weight_decay，则设置为0
         if decay_flag:
             update = op_mul(weight_decay, param_fp32) + update
 
+        # 将梯度求平均，并将梯度转换为float32类型
         update_with_lr = op_mul(lr, update)
+        # 将参数更新
         next_param = param_fp32 - op_reshape(update_with_lr, op_shape(param_fp32))
 
+        # 将参数更新，并将参数转换为float32类型
         next_param = F.depend(next_param, F.assign(param, op_cast(next_param, F.dtype(param))))
+        # 将模更新，并将模转换为float32类型
         next_param = F.depend(next_param, F.assign(m, op_cast(next_m, F.dtype(m))))
+        # 将方差更新，并将方差转换为float32类型
         next_param = F.depend(next_param, F.assign(v, op_cast(next_v, F.dtype(v))))
 
+        # 返回参数
         return op_cast(next_param, F.dtype(param))
-    return op_cast(gradient, F.dtype(param))
+    return gradient
 
 
+
+# 定义一个名为_adam_opt的函数，接收参数：Function，Function，Function，Bool，Bool，Bool，Tensor，Tensor，Tensor，Tensor，Tensor，RowTensor，Tensor，Tensor，Tensor，Tensor，Bool，Bool
 @_adam_opt.register("Function", "Function", "Function", "Function", "Bool", "Bool", "Bool", "Tensor", "Tensor",
                     "Tensor", "Tensor", "Tensor", "Tensor", "RowTensor", "Tensor", "Tensor", "Tensor", "Bool", "Bool")
 def _run_opt_with_sparse(opt, sparse_opt, push, pull, use_locking, use_nesterov, target, beta1_power,
                          beta2_power, beta1, beta2, eps, lr, gradient, param, m, v, ps_parameter, cache_enable):
+	# 启动适配稀疏矩阵梯度的adam优化器
     """Apply sparse adam optimizer to the weight parameter when the gradient is sparse."""
+    # 定义一个布尔值，用来表示是否成功执行opt
     success = True
+    # 获取梯度的索引
     indices = gradient.indices
+    # 获取梯度的值
     values = gradient.values
     if ps_parameter and not cache_enable:
+    # 如果ps_parameter为True，且cache_enable为False，则使用P.Shape()函数计算op_shape
         op_shape = P.Shape()
+        # 将op_shape(param), op_shape(m), op_shape(v),
+        # op_shape(beta1_power), op_shape(beta2_power), op_shape(lr), op_shape(beta1),
+        # op_shape(beta2), op_shape(eps), op_shape(values), op_shape(indices)计算出来
         shapes = (op_shape(param), op_shape(m), op_shape(v),
-                  op_shape(beta1_power), op_shape(beta2_power), op_shape(lr), op_shape(beta1),
-                  op_shape(beta2), op_shape(eps), op_shape(values), op_shape(indices))
+                op_shape(beta1_power), op_shape(beta2_power), op_shape(lr), op_shape(beta1),
+                op_shape(beta2), op_shape(eps), op_shape(values), op_shape(indices))
+        # 将success的值与pull函数的值进行比较，如果比较结果为True，则使用push函数推送
         success = F.depend(success, pull(push((beta1_power, beta2_power, lr, beta1, beta2,
-                                               eps, values, indices), shapes), param))
+                                                eps, values, indices), shapes), param))
+        # 返回success的值
         return success
 
+    # 如果指定了target，则使用sparse_opt
     if not target:
         success = F.depend(success, sparse_opt(param, m, v, beta1_power, beta2_power, lr, beta1, beta2,
                                                eps, values, indices))
     else:
+        # 创建一个乘积运算器
         op_mul = P.Mul()
+        # 创建平方运算器
         op_square = P.Square()
+        # 创建平方根运算器
         op_sqrt = P.Sqrt()
+        # 创建ScatterAdd运算器
         scatter_add = P.ScatterAdd(use_locking)
 
+        # 更新参数
         success = F.depend(success, F.assign(m, op_mul(beta1, m)))
         success = F.depend(success, F.assign(v, op_mul(beta2, v)))
 
+        # 获取梯度索引
         grad_indices = gradient.indices
+        # 获取梯度值
         grad_value = gradient.values
 
+        # 更新下一个梯度
         next_m = scatter_add(m,
                              grad_indices,
                              op_mul(F.tuple_to_array((1.0,)) - beta1, grad_value))
 
+        # 更新下一个方差
         next_v = scatter_add(v,
                              grad_indices,
                              op_mul(F.tuple_to_array((1.0,)) - beta2, op_square(grad_value)))
 
+        # 如果使用Nesterov梯度，则更新下一个梯度
         if use_nesterov:
+            # 将下一个梯度乘以beta1
             m_temp = next_m * _scaler_ten
+            # 更新m
             F.assign(m, op_mul(beta1, next_m))
+            # 将梯度累加到m上
             div_value = scatter_add(m,
                                     op_mul(grad_indices, _scaler_one),
                                     op_mul(F.tuple_to_array((1.0,)) - beta1, grad_value))
+            # 更新param_update
             param_update = div_value / (op_sqrt(next_v) + eps)
+            # 更新m
             F.assign(m, m_temp / _scaler_ten)
         else:
+            # 更新param_update
             param_update = next_m / (op_sqrt(next_v) + eps)
 
+        # 更新学习率
         lr_t = lr * op_sqrt(1 - beta2_power) / (1 - beta1_power)
+        # 更新下一个参数
         next_param = param - lr_t * param_update
 
+        # 更新参数
         success = F.depend(success, F.assign(param, next_param))
+        # 更新梯度
         success = F.depend(success, F.assign(m, next_m))
+        # 更新方差
         success = F.depend(success, F.assign(v, next_v))
 
     return success
@@ -154,15 +214,22 @@ def _run_opt_with_sparse(opt, sparse_opt, push, pull, use_locking, use_nesterov,
 def _run_opt_with_one_number(opt, sparse_opt, push, pull, use_locking, use_nesterov, target,
                              beta1_power, beta2_power, beta1, beta2, eps, lr, gradient, param,
                              moment1, moment2, ps_parameter, cache_enable):
+	# 启动适配矩阵梯度的adam优化器
     """Apply adam optimizer to the weight parameter using Tensor."""
+    # 定义一个布尔值，用来表示是否成功执行opt
     success = True
+    # 如果ps_parameter为True，且cache_enable为False，则执行pull，否则执行opt
     if ps_parameter and not cache_enable:
+        # 获取op_shape
         op_shape = P.Shape()
+        # 执行pull
         success = F.depend(success, pull(push((beta1_power, beta2_power, lr, beta1, beta2, eps, gradient),
                                               (op_shape(param), op_shape(moment1), op_shape(moment2))), param))
     else:
+        # 执行opt
         success = F.depend(success, opt(param, moment1, moment2, beta1_power, beta2_power, lr, beta1, beta2,
                                         eps, gradient))
+    # 返回success
     return success
 
 
@@ -170,144 +237,116 @@ def _run_opt_with_one_number(opt, sparse_opt, push, pull, use_locking, use_neste
                     "Tensor", "Tensor")
 def _run_off_load_opt(opt, beta1_power, beta2_power, beta1, beta2, eps, lr, gradient, param, moment1, moment2):
     """Apply AdamOffload optimizer to the weight parameter using Tensor."""
+    # 启动适配矩阵梯度的adamoffload优化器
+    # 初始化一个布尔值，用来表示是否执行完成
     success = True
+    # 将AdamOffload优化器的参数更新到参数中
     delat_param = opt(moment1, moment2, beta1_power, beta2_power, lr, beta1, beta2, eps, gradient)
+    # 将参数和参数之间的差值更新到参数中
     success = F.depend(success, F.assign_add(param, delat_param))
+    # 返回是否执行完成的布尔值
     return success
 
 
 def _check_param_value(beta1, beta2, eps, prim_name):
     """Check the type of inputs."""
+    """
+    检查输入参数
+    """
+    # 检查beta1的类型是否为float
     validator.check_value_type("beta1", beta1, [float], prim_name)
+    # 检查beta2的类型是否为float
     validator.check_value_type("beta2", beta2, [float], prim_name)
+    # 检查eps的类型是否为float
     validator.check_value_type("eps", eps, [float], prim_name)
+    # 检查beta1的取值范围是否在0.0和1.0之间
     validator.check_float_range(beta1, 0.0, 1.0, Rel.INC_NEITHER, "beta1", prim_name)
+    # 检查beta2的取值范围是否在0.0和1.0之间
     validator.check_float_range(beta2, 0.0, 1.0, Rel.INC_NEITHER, "beta2", prim_name)
+    # 检查eps的取值是否为正数
     validator.check_positive_float(eps, "eps", prim_name)
 
 
 class Adam(Optimizer):
+	# Adam优化器
     r"""
-    Implements the Adaptive Moment Estimation (Adam) algorithm.
+    Updates gradients by the Adaptive Moment Estimation (Adam) algorithm.
 
-    The Adam optimizer can dynamically adjust the learning rate of each parameter using the first-order
-    moment estimation and the second-order moment estimation of the gradient.
     The Adam algorithm is proposed in `Adam: A Method for Stochastic Optimization <https://arxiv.org/abs/1412.6980>`_.
 
     The updating formulas are as follows,
 
     .. math::
-        \begin{array}{l}
-            &\newline
-            &\hline \\
-            &\textbf{Parameters}: \: 1^{\text {st }}\text {moment vector} \: m , \: 2^{\text {nd}} \:
-             \text{moment vector} \: v , \\
-            &\:\text{gradients } g, \: \text{learning rate} \: \gamma, \text
-             { exponential decay rates for the moment estimates} \: \beta_{1} \: \beta_{2} , \\
-            &\:\text {parameter vector} \: w_{0}, \:\text{timestep} \: t , \text{ weight decay } \lambda \\
-            &\textbf{Init}: m_{0} \leftarrow 0, \: v_{0} \leftarrow 0, \: t \leftarrow 0, \:
-             \text{init parameter vector} \: w_{0} \\[-1.ex]
-            &\newline
-            &\hline \\
-            &\textbf{while} \: w_{t} \: \text{not converged} \: \textbf{do} \\
-            &\hspace{5mm}\boldsymbol{g}_{t} \leftarrow \nabla_{w} \boldsymbol{f}_{t}\left(\boldsymbol{w}_{t-1}\right) \\
-            &\hspace{5mm}\textbf {if } \lambda \neq 0 \\
-            &\hspace{10mm}\boldsymbol{g}_{t} \leftarrow \boldsymbol{g}_{t}+\lambda \boldsymbol{w}_{t-1} \\
-            &\hspace{5mm}\boldsymbol{m}_{t} \leftarrow \beta_{1} \boldsymbol{m}_{t-1}+\left(1-\beta_{1}\right)
-             \boldsymbol{g}_{t} \\
-            &\hspace{5mm}\boldsymbol{v}_{t} \leftarrow \beta_{2} \boldsymbol{v}_{t-1}+\left(1-\beta_{2}\right)
-             \boldsymbol{g}_{t}^{2} \\
-            &\hspace{5mm}\hat{\boldsymbol{m}}_{t} \leftarrow \boldsymbol{m}_{t} /\left(1-\beta_{1}^{t}\right) \\
-            &\hspace{5mm}\hat{\boldsymbol{v}}_{t} \leftarrow \boldsymbol{v}_{t} /\left(1-\beta_{2}^{t}\right) \\
-            &\hspace{5mm}\boldsymbol{w}_{t} \leftarrow \boldsymbol{w}_{t-1}-\gamma \hat{\boldsymbol{m}}_{t}
-             /(\sqrt{\hat{\boldsymbol{v}}_{t}}+\epsilon) \\
-            &\textbf{end while} \\[-1.ex]
-            &\newline
-            &\hline \\[-1.ex]
-            &\textbf{return} \:  \boldsymbol{w}_{t} \\[-1.ex]
-            &\newline
-            &\hline \\[-1.ex]
+        \begin{array}{ll} \\
+            m = \beta_1 * m + (1 - \beta_1) * g \\
+            v = \beta_2 * v + (1 - \beta_2) * g * g \\
+            l = \alpha * \frac{\sqrt{1-\beta_2^t}}{1-\beta_1^t} \\
+            w = w - l * \frac{m}{\sqrt{v} + \epsilon}
         \end{array}
 
-    :math:`m` represents the 1st moment vector, :math:`v` represents the 2nd moment vector,
-    :math:`g` represents `gradients`, :math:`\beta_1, \beta_2` represent `beta1` and `beta2`,
-    :math:`t` represents the current step while :math:`beta_1^t` and :math:`beta_2^t` represent
-    `beta1_power` and `beta2_power`, :math:`\gamma` represents `learning_rate`, :math:`w` represents `params`,
+    :math:`m` represents the 1st moment vector `moment1`, :math:`v` represents the 2nd moment vector `moment2`,
+    :math:`g` represents `gradients`, :math:`l` represents scaling factor `lr`, :math:`\beta_1, \beta_2` represent
+    `beta1` and `beta2`, :math:`t` represents updating step while :math:`beta_1^t` and :math:`beta_2^t` represent
+    `beta1_power` and `beta2_power`, :math:`\alpha` represents `learning_rate`, :math:`w` represents `params`,
     :math:`\epsilon` represents `eps`.
 
     Note:
-        The sparse strategy is applied while the SparseGatherV2 operator is used for forward network. If the sparse
-        strategy wants to be executed on the host, set the target to the CPU.
-        The sparse feature is under continuous development.
+        When separating parameter groups, the weight decay in each group will be applied on the parameters if the
+        weight decay is positive. When not separating parameter groups, the `weight_decay` in the API will be applied
+        on the parameters without 'beta' or 'gamma' in their names if `weight_decay` is positive.
 
-        If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without
-        'beta' or 'gamma' in their names. Users can group parameters to change the strategy of decaying weight. When
-        parameters are grouped, each group can set `weight_decay`, if not, the `weight_decay` in optimizer will be
-        applied.
+        When separating parameter groups, if you want to centralize the gradient, set grad_centralization to True,
+        but the gradient centralization can only be applied to the parameters of the convolution layer.
+        If the parameters of the non convolution layer are set to True, an error will be reported.
+
+        To improve parameter groups performance, the customized order of parameters is supported.
+
+        The sparse strategy is applied while the SparseGatherV2 operator is used for forward network.
+        The sparse feature is under continuous development. If the sparse strategy wants to be executed on the host,
+        set the target to the CPU.
 
     Args:
-        params (Union[list[Parameter], list[dict]]): Must be list of `Parameter` or list of `dict`. When the
-            `params` is a list of `dict`, the string "params", "lr", "weight_decay", "grad_centralization" and
-            "order_params" are the keys can be parsed.
+        params (Union[list[Parameter], list[dict]]): When the `params` is a list of `Parameter` which will be updated,
+            the element in `params` must be class `Parameter`. When the `params` is a list of `dict`, the "params",
+            "lr", "weight_decay" and "order_params" are the keys can be parsed.
 
-            - params: Required. Parameters in current group. The value must be a list of `Parameter`.
+            - params: Required. The value must be a list of `Parameter`.
 
-            - lr: Optional. If "lr" in the keys, the value of corresponding learning rate will be used.
-              If not, the `learning_rate` in optimizer will be used. Fixed and dynamic learning rate are supported.
+            - lr: Optional. If "lr" is in the keys, the value of the corresponding learning rate will be used.
+              If not, the `learning_rate` in the API will be used.
 
-            - weight_decay: Optional. If "weight_decay" in the keys, the value of corresponding weight decay
-              will be used. If not, the `weight_decay` in the optimizer will be used. It should be noted that weight
-              decay can be a constant value or a Cell. It is a Cell only when dynamic weight decay is applied. Dynamic
-              weight decay is similar to dynamic learning rate, users need to customize a weight decay schedule only
-              with global step as input, and during training, the optimizer calls the instance of WeightDecaySchedule
-              to get the weight decay value of current step.
+            - weight_decay: Optional. If "weight_decay" is in the keys, the value of the corresponding weight decay
+              will be used. If not, the `weight_decay` in the API will be used.
 
-            - grad_centralization: Optional. Must be Boolean. If "grad_centralization" is in the keys, the set value
-              will be used. If not, the `grad_centralization` is False by default. This configuration only works on the
-              convolution layer.
+            - order_params: Optional. If "order_params" is in the keys, the value must be the order of parameters and
+              the order will be followed in the optimizer. There are no other keys in the `dict` and the parameters
+              which in the 'order_params' must be in one of group parameters.
 
-            - order_params: Optional. When parameters is grouped, this usually is used to maintain the order of
-              parameters that appeared in the network to improve performance. The value should be parameters whose
-              order will be followed in optimizer.
-              If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
-              one group of `params`.
-
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 1e-3.
-
-            - float: The fixed learning rate value. Must be equal to or greater than 0.
-
-            - int: The fixed learning rate value. Must be equal to or greater than 0. It will be converted to float.
-
-            - Tensor: Its value should be a scalar or a 1-D vector. For scalar, fixed learning rate will be applied.
-              For vector, learning rate is dynamic, then the i-th step will take the i-th value as the learning rate.
-
-            - Iterable: Learning rate is dynamic. The i-th step will take the i-th value as the learning rate.
-
-            - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
-              LearningRateSchedule with step as the input to get the learning rate of current step.
+            - grad_centralization: Optional. The data type of "grad_centralization" is Bool. If "grad_centralization"
+              is in the keys, the set value will be used. If not, the `grad_centralization` is False by default.
+              This parameter only works on the convolution layer.
 
+        learning_rate (Union[float, Tensor, Iterable, LearningRateSchedule]): A value or a graph for the learning rate.
+            When the learning_rate is an Iterable or a Tensor in a 1D dimension, use the dynamic learning rate, then
+            the i-th step will take the i-th value as the learning rate. When the learning_rate is LearningRateSchedule,
+            use dynamic learning rate, the i-th learning rate will be calculated during the process of training
+            according to the formula of LearningRateSchedule. When the learning_rate is a float or a Tensor in a zero
+            dimension, use fixed learning rate. Other cases are not supported. The float learning rate must be
+            equal to or greater than 0. If the type of `learning_rate` is int, it will be converted to float.
+            Default: 1e-3.
         beta1 (float): The exponential decay rate for the 1st moment estimations. Should be in range (0.0, 1.0).
                        Default: 0.9.
         beta2 (float): The exponential decay rate for the 2nd moment estimations. Should be in range (0.0, 1.0).
                        Default: 0.999.
         eps (float): Term added to the denominator to improve numerical stability. Should be greater than 0. Default:
                      1e-8.
-        use_locking (bool): Whether to enable a lock to protect the updating process of variable tensors.
-            If true, updates of the `w`, `m`, and `v` tensors will be protected by a lock.
+        use_locking (bool): Whether to enable a lock to protect variable tensors from being updated.
+            If true, updates of the var, m, and v tensors will be protected by a lock.
             If false, the result is unpredictable. Default: False.
         use_nesterov (bool): Whether to use Nesterov Accelerated Gradient (NAG) algorithm to update the gradients.
             If true, update the gradients using NAG.
             If false, update the gradients without using NAG. Default: False.
-
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
-
-            - float: The fixed weight decay value. Must be equal to or greater than 0.
-
-            - int: The fixed weight decay value. Must be equal to or greater than 0. It will be converted to float.
-
-            - Cell: Weight decay is dynamic. During training, the optimizer calls the instance of
-              the Cell with step as the input to get the weight decay value of current step.
-
+        weight_decay (float): Weight decay (L2 penalty). It must be equal to or greater than 0. Default: 0.0.
         loss_scale (float): A floating point value for the loss scale. Should be greater than 0. In general, use the
             default value. Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
             `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
@@ -331,11 +370,9 @@ class Adam(Optimizer):
         ValueError: If `weight_decay` is less than 0.
 
     Supported Platforms:
-        ``Ascend`` ``GPU``  ``CPU``
+        ``Ascend`` ``GPU``
 
     Examples:
-        >>> from mindspore import nn, Model
-        >>>
         >>> net = Net()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.Adam(params=net.trainable_params())
@@ -357,14 +394,17 @@ class Adam(Optimizer):
         >>> model = Model(net, loss_fn=loss, optimizer=optim)
     """
 
-    @opt_init_args_register
     def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False,
                  use_nesterov=False, weight_decay=0.0, loss_scale=1.0):
+        # 初始化Adam类
         super(Adam, self).__init__(learning_rate, params, weight_decay, loss_scale)
+        # 检查参数值
         _check_param_value(beta1, beta2, eps, self.cls_name)
+        # 检查参数类型
         validator.check_value_type("use_locking", use_locking, [bool], self.cls_name)
         validator.check_value_type("use_nesterov", use_nesterov, [bool], self.cls_name)
 
+        # 初始化参数
         self.beta1 = Tensor(beta1, mstype.float32)
         self.beta2 = Tensor(beta2, mstype.float32)
         self.beta1_power = Parameter(initializer(1, [1], mstype.float32), name="beta1_power")
@@ -375,154 +415,138 @@ class Adam(Optimizer):
         self.moment1 = self.parameters.clone(prefix="moment1", init='zeros')
         self.moment2 = self.parameters.clone(prefix="moment2", init='zeros')
 
+        # 是否为设备
         self._is_device = True
+        # HyperMap算法
+        self.hyper_map = C.HyperMap()
+        # Adam算法
         self.opt = P.Adam(use_locking, use_nesterov)
+        # FusedSparseAdam
         self.sparse_opt = P.FusedSparseAdam(use_locking, use_nesterov)
+        # FusedSparseAdam的primitive_target属性
         self.sparse_opt.add_prim_attr("primitive_target", "CPU")
+        # ops算子Pull
         self._ps_pull = P.Pull()
+        # ops算子Push
         self._ps_push = P.Push("Adam", [0, 1, 2])
+        # Push的use_nesterov属性
         self._ps_push.add_prim_attr("use_nesterov", use_nesterov)
 
     def construct(self, gradients):
+        '''
+        构建Adam优化器
+        :param gradients: 梯度
+        :return:
+        '''
         params = self.parameters
         moment1 = self.moment1
         moment2 = self.moment2
         gradients = self.decay_weight(gradients)
+        # 将梯度按照学习率调整
         gradients = self.gradients_centralization(gradients)
+        # 将梯度归一化
         gradients = self.scale_grad(gradients)
+        # 将梯度转换为稀疏矩阵
         gradients = self._grad_sparse_indices_deduplicate(gradients)
+        # 获取学习率
         lr = self.get_lr()
 
+        # 计算beta1_power
         beta1_power = self.beta1_power * self.beta1
+        # 更新beta1_power
         self.beta1_power = beta1_power
+        # 计算beta2_power
         beta2_power = self.beta2_power * self.beta2
+        # 更新beta2_power
         self.beta2_power = beta2_power
+        # 如果是分组学习率
         if self.is_group_lr:
+            # 用分组方法将梯度传入adam_opt函数
             success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt, self._ps_push, self._ps_pull,
-                                          self.use_locking, self.use_nesterov, self._is_device,
-                                          beta1_power, beta2_power, self.beta1, self.beta2, self.eps),
+                                            self.use_locking, self.use_nesterov, self._is_device,
+                                            beta1_power, beta2_power, self.beta1, self.beta2, self.eps),
                                 lr, gradients, params, moment1, moment2, self.ps_parameters, self.cache_enable)
+        # 否则
         else:
+            # 将梯度传入adam_opt函数
             success = self.map_(F.partial(_adam_opt, self.opt, self.sparse_opt, self._ps_push, self._ps_pull,
-                                          self.use_locking, self.use_nesterov, self._is_device,
-                                          beta1_power, beta2_power, self.beta1, self.beta2, self.eps, lr),
+                                            self.use_locking, self.use_nesterov, self._is_device,
+                                            beta1_power, beta2_power, self.beta1, self.beta2, self.eps, lr),
                                 gradients, params, moment1, moment2, self.ps_parameters, self.cache_enable)
+        # 返回成功状态
         return success
 
     @Optimizer.target.setter
+    # 设置optimizer基类中的target属性
     def target(self, value):
-        """
-        If the input value is set to "CPU", the parameters will be updated on the host using the Fused
-        optimizer operation.
-        """
-        self._set_base_target(value)
+        """If the input value is set to "CPU", the parameters will be updated on the host using the Fused
+           optimizer operation."""
+        # 如果输入的值不是字符串类型，抛出类型错误
+        if not isinstance(value, str):
+            raise TypeError("The value must be str type, but got value type is {}".format(type(value)))
+
+        # 如果输入的值不在CPU、Ascend、GPU中，抛出值错误
+        if value not in ('CPU', 'Ascend', 'GPU'):
+            raise ValueError("The value must be 'CPU', 'Ascend' or 'GPU', but got value {}".format(value))
+
+        # 如果设置的target值为CPU，且输入的值为Ascend或GPU，抛出值错误
+        if self._target == "CPU" and value in('Ascend', 'GPU'):
+            raise ValueError("In the CPU environment, target cannot be set to 'GPU' and 'Ascend'.")
+
+        # 如果设置的target值为Ascend，且输入的值为GPU，抛出值错误
+        if self._target == "Ascend" and value == 'GPU':
+            raise ValueError("In the Ascend environment, target cannot be set to 'GPU'.")
+
+        # 设置target值为不是CPU
+        self._is_device = (value!= 'CPU')
+        # 设置target值为输入的值
+        self._target = value
 
 
 class AdamWeightDecay(Optimizer):
-    r"""
-    Implements the Adam algorithm with weight decay.
-
-    .. math::
-        \begin{array}{l}
-            &\newline
-            &\hline \\
-            &\textbf{Parameters}: \: 1^{\text {st }}\text {moment vector} \: m , \: 2^{\text {nd}} \:
-             \text{moment vector} \: v , \\
-            &\: gradients \: g, \: \text{learning rate} \: \gamma,
-             \text {exponential decay rates for the moment estimates} \: \beta_{1} \: \beta_{2} , \\
-            &\:\text {parameter vector} \: w_{0}, \:\text{timestep} \: t, \: \text{weight decay} \: \lambda \\
-            &\textbf{Init}:  m_{0} \leftarrow 0, \: v_{0} \leftarrow 0, \: t \leftarrow 0, \:
-             \text{init parameter vector} \: w_{0} \\[-1.ex]
-            &\newline
-            &\hline \\
-            &\textbf{repeat} \\
-            &\hspace{5mm} t \leftarrow t+1 \\
-            &\hspace{5mm}\boldsymbol{g}_{t} \leftarrow \nabla f_{t}\left(\boldsymbol{w}_{t-1}\right) \\
-            &\hspace{5mm}\boldsymbol{m}_{t} \leftarrow \beta_{1} \boldsymbol{m}_{t-1}+\left(1-\beta_{1}\right)
-             \boldsymbol{g}_{t} \\
-            &\hspace{5mm}\boldsymbol{v}_{t} \leftarrow \beta_{2} \boldsymbol{v}_{t-1}+\left(1-\beta_{2}\right)
-             \boldsymbol{g}_{t}^{2} \\
-            &\hspace{5mm}\hat{\boldsymbol{m}}_{t} \leftarrow \boldsymbol{m}_{t} /\left(1-\beta_{1}^{t}\right) \\
-            &\hspace{5mm}\hat{\boldsymbol{v}}_{t} \leftarrow \boldsymbol{v}_{t} /\left(1-\beta_{2}^{t}\right) \\
-            &\hspace{5mm}\boldsymbol{w}_{t} \leftarrow \boldsymbol{w}_{t-1}-\left(\gamma \hat{\boldsymbol{m}}_{t}
-             /\left(\sqrt{\hat{\boldsymbol{v}}_{t}}+\epsilon\right)+\lambda \boldsymbol{w}_{t-1}\right) \\
-            &\textbf{until}\text { stopping criterion is met } \\[-1.ex]
-            &\newline
-            &\hline \\[-1.ex]
-            &\textbf{return} \: \boldsymbol{w}_{t} \\[-1.ex]
-            &\newline
-            &\hline \\[-1.ex]
-        \end{array}
-
-    :math:`m` represents the 1st moment vector `moment1`, :math:`v` represents the 2nd moment vector `moment2`,
-    :math:`g` represents `gradients`, :math:`\gamma` represents `learning_rate`,
-    :math:`\beta_1, \beta_2` represent `beta1` and `beta2`, :math:`t` represents the current step,
-    :math:`w` represents `params`, :math:`\gamma` represents `weight_decay`.
+	# Adam优化器梯度衰减
+    """
+    Implements the Adam algorithm to fix the weight decay.
 
     Note:
-        There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used
-        and `drop_overflow_update` in `FixedLossScaleManager` is set to False, optimizer needs to set the 'loss_scale'.
-        As this optimizer has no argument of `loss_scale`, so `loss_scale` needs to be processed by other means, refer
-        document `LossScale <https://www.mindspore.cn/docs/programming_guide/zh-CN/master/lossscale.html>`_ to process
-        `loss_scale` correctly.
+        When separating parameter groups, the weight decay in each group will be applied on the parameters if the
+        weight decay is positive. When not separating parameter groups, the `weight_decay` in the API will be applied
+        on the parameters without 'beta' or 'gamma' in their names if `weight_decay` is positive.
 
-        If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without
-        'beta' or 'gamma' in their names. Users can group parameters to change the strategy of decaying weight. When
-        parameters are grouped, each group can set `weight_decay`, if not, the `weight_decay` in optimizer will be
-        applied.
+        To improve parameter groups performance, the customized order of parameters can be supported.
 
     Args:
-        params (Union[list[Parameter], list[dict]]): Must be list of `Parameter` or list of `dict`. When the
-            `params` is a list of `dict`, the string "params", "lr", "weight_decay", and "order_params"
-            are the keys can be parsed.
+        params (Union[list[Parameter], list[dict]]): When the `params` is a list of `Parameter` which will be updated,
+            the element in `params` must be class `Parameter`. When the `params` is a list of `dict`, the "params",
+            "lr", "weight_decay" and "order_params" are the keys can be parsed.
 
-            - params: Required. Parameters in current group. The value must be a list of `Parameter`.
+            - params: Required. The value must be a list of `Parameter`.
 
-            - lr: Optional. If "lr" in the keys, the value of corresponding learning rate will be used.
-              If not, the `learning_rate` in optimizer will be used. Fixed and dynamic learning rate are supported.
+            - lr: Optional. If "lr" is in the keys, the value of the corresponding learning rate will be used.
+              If not, the `learning_rate` in the API will be used.
 
-            - weight_decay: Optional. If "weight_decay" in the keys, the value of corresponding weight decay
-              will be used. If not, the `weight_decay` in the optimizer will be used. It should be noted that weight
-              decay can be a constant value or a Cell. It is a Cell only when dynamic weight decay is applied. Dynamic
-              weight decay is similar to dynamic learning rate, users need to customize a weight decay schedule only
-              with global step as input, and during training, the optimizer calls the instance of WeightDecaySchedule
-              to get the weight decay value of current step.
+            - weight_decay: Optional. If "weight_decay" is in the keys, the value of the corresponding weight decay
+              will be used. If not, the `weight_decay` in the API will be used.
 
-            - order_params: Optional. When parameters is grouped, this usually is used to maintain the order of
-              parameters that appeared in the network to improve performance. The value should be parameters whose
-              order will be followed in optimizer.
-              If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
-              one group of `params`.
-
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 1e-3.
-
-            - float: The fixed learning rate value. Must be equal to or greater than 0.
-
-            - int: The fixed learning rate value. Must be equal to or greater than 0. It will be converted to float.
-
-            - Tensor: Its value should be a scalar or a 1-D vector. For scalar, fixed learning rate will be applied.
-              For vector, learning rate is dynamic, then the i-th step will take the i-th value as the learning rate.
-
-            - Iterable: Learning rate is dynamic. The i-th step will take the i-th value as the learning rate.
-
-            - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
-              LearningRateSchedule with step as the input to get the learning rate of current step.
+            - order_params: Optional. If "order_params" is in the keys, the value must be the order of parameters and
+              the order will be followed in the optimizer. There are no other keys in the `dict` and the parameters
+              which in the 'order_params' must be in one of group parameters.
 
+        learning_rate (Union[float, Tensor, Iterable, LearningRateSchedule]): A value or a graph for the learning rate.
+            When the learning_rate is an Iterable or a Tensor in a 1D dimension, use the dynamic learning rate, then
+            the i-th step will take the i-th value as the learning rate. When the learning_rate is LearningRateSchedule,
+            use dynamic learning rate, the i-th learning rate will be calculated during the process of training
+            according to the formula of LearningRateSchedule. When the learning_rate is a float or a Tensor in a zero
+            dimension, use fixed learning rate. Other cases are not supported. The float learning rate must be
+            equal to or greater than 0. If the type of `learning_rate` is int, it will be converted to float.
+            Default: 1e-3.
         beta1 (float): The exponential decay rate for the 1st moment estimations. Default: 0.9.
             Should be in range (0.0, 1.0).
         beta2 (float): The exponential decay rate for the 2nd moment estimations. Default: 0.999.
             Should be in range (0.0, 1.0).
         eps (float): Term added to the denominator to improve numerical stability. Default: 1e-6.
             Should be greater than 0.
-
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
-
-            - float: The fixed weight decay value. Must be equal to or greater than 0.
-
-            - int: The fixed weight decay value. Must be equal to or greater than 0. It will be converted to float.
-
-            - Cell: Weight decay is dynamic. During training, the optimizer calls the instance of
-              the Cell with step as the input to get the weight decay value of current step.
+        weight_decay (float): Weight decay (L2 penalty). It must be equal to or greater than 0. Default: 0.0.
 
     Inputs:
         - **gradients** (tuple[Tensor]) - The gradients of `params`, the shape is the same as `params`.
@@ -540,11 +564,9 @@ class AdamWeightDecay(Optimizer):
         ValueError: If `weight_decay` is less than 0.
 
     Supported Platforms:
-        ``Ascend`` ``GPU`` ``CPU``
+        ``Ascend`` ``GPU``
 
     Examples:
-        >>> from mindspore import nn, Model
-        >>>
         >>> net = Net()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.AdamWeightDecay(params=net.trainable_params())
@@ -563,40 +585,50 @@ class AdamWeightDecay(Optimizer):
         >>> loss = nn.SoftmaxCrossEntropyWithLogits()
         >>> model = Model(net, loss_fn=loss, optimizer=optim)
    """
-    _support_parallel_optimizer = True
-
     def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-6, weight_decay=0.0):
+        # 初始化AdamWeightDecay类
         super(AdamWeightDecay, self).__init__(learning_rate, params, weight_decay)
+        # 检查参数值
         _check_param_value(beta1, beta2, eps, self.cls_name)
+        # 初始化beta1, beta2, eps
         self.beta1 = Tensor(np.array([beta1]).astype(np.float32))
         self.beta2 = Tensor(np.array([beta2]).astype(np.float32))
         self.eps = Tensor(np.array([eps]).astype(np.float32))
+        # 初始化moments1, moments2
         self.moments1 = self.parameters.clone(prefix="adam_m", init='zeros')
         self.moments2 = self.parameters.clone(prefix="adam_v", init='zeros')
+        # 初始化HyperMap
+        self.hyper_map = C.HyperMap()
 
     def construct(self, gradients):
-        weight_decay = self.get_weight_decay()
+        # 获取学习率
         lr = self.get_lr()
+        # 如果是分组，则使用分组学习率
         if self.is_group:
             if self.is_group_lr:
+                # 如果是分组学习率，则按分组方式将学习率按分组方式传入使用HyperMap
                 optim_result = self.hyper_map(F.partial(_adam_opt, self.beta1, self.beta2, self.eps),
-                                              lr, weight_decay, self.parameters, self.moments1,
-                                              self.moments2, gradients, self.decay_flags, self.optim_filter)
+                                              lr, self.weight_decay, self.parameters, self.moments1, self.moments2,
+                                              gradients, self.decay_flags, self.optim_filter)
             else:
+                # 否则按分组方式使用HyperMap
                 optim_result = self.hyper_map(F.partial(_adam_opt, self.beta1, self.beta2, self.eps, lr),
-                                              weight_decay, self.parameters, self.moments1, self.moments2,
+                                              self.weight_decay, self.parameters, self.moments1, self.moments2,
                                               gradients, self.decay_flags, self.optim_filter)
         else:
-            optim_result = self.hyper_map(F.partial(_adam_opt, self.beta1, self.beta2, self.eps, lr, weight_decay),
+            # 否则使用HyperMap
+            optim_result = self.hyper_map(F.partial(_adam_opt, self.beta1, self.beta2, self.eps, lr, self.weight_decay),
                                           self.parameters, self.moments1, self.moments2,
                                           gradients, self.decay_flags, self.optim_filter)
+        # 如果使用并行，则广播参数
         if self.use_parallel:
             self.broadcast_params(optim_result)
-
+        # 返回优化结果
         return optim_result
 
 
 class AdamOffload(Optimizer):
+	#AdamOffload优化器
     r"""
     This optimizer will offload Adam optimizer to host CPU and keep parameters being updated on the device,
     to minimize the memory cost. Although that would bring about an increase of performance overhead,
@@ -608,85 +640,65 @@ class AdamOffload(Optimizer):
 
     .. math::
         \begin{array}{ll} \\
-            m_{t+1} = \beta_1 * m_{t} + (1 - \beta_1) * g \\
-            v_{t+1} = \beta_2 * v_{t} + (1 - \beta_2) * g * g \\
+            m = \beta_1 * m + (1 - \beta_1) * g \\
+            v = \beta_2 * v + (1 - \beta_2) * g * g \\
             l = \alpha * \frac{\sqrt{1-\beta_2^t}}{1-\beta_1^t} \\
-            w_{t+1} = w_{t} - l * \frac{m_{t+1}}{\sqrt{v_{t+1}} + \epsilon}
+            w = w - l * \frac{m}{\sqrt{v} + \epsilon}
         \end{array}
 
     :math:`m` represents the 1st moment vector `moment1`, :math:`v` represents the 2nd moment vector `moment2`,
-    :math:`g` represents `gradients`, :math:`l` represents scaling factor, :math:`\beta_1, \beta_2` represent
-    `beta1` and `beta2`, :math:`t` represents the current step while :math:`beta_1^t` and :math:`beta_2^t` represent
+    :math:`g` represents `gradients`, :math:`l` represents scaling factor `lr`, :math:`\beta_1, \beta_2` represent
+    `beta1` and `beta2`, :math:`t` represents updating step while :math:`beta_1^t` and :math:`beta_2^t` represent
     `beta1_power` and `beta2_power`, :math:`\alpha` represents `learning_rate`, :math:`w` represents `params`,
     :math:`\epsilon` represents `eps`.
 
     Note:
         This optimizer only supports `GRAPH_MODE` currently.
 
-        If parameters are not grouped, the `weight_decay` in optimizer will be applied on the network parameters without
-        'beta' or 'gamma' in their names. Users can group parameters to change the strategy of decaying weight. When
-        parameters are grouped, each group can set `weight_decay`, if not, the `weight_decay` in optimizer will be
-        applied.
+        When separating parameter groups, the weight decay in each group will be applied on the parameters if the
+        weight decay is positive. When not separating parameter groups, the `weight_decay` in the API will be applied
+        on the parameters without 'beta' or 'gamma' in their names if `weight_decay` is positive.
+
+        To improve parameter groups performance, the customized order of parameters is supported.
 
     Args:
-        params (Union[list[Parameter], list[dict]]): Must be list of `Parameter` or list of `dict`. When the
-            `params` is a list of `dict`, the string "params", "lr", "weight_decay", and "order_params"
-            are the keys can be parsed.
+        params (Union[list[Parameter], list[dict]]): When the `params` is a list of `Parameter` which will be updated,
+            the element in `params` must be class `Parameter`. When the `params` is a list of `dict`, the "params",
+            "lr", "weight_decay" and "order_params" are the keys can be parsed.
 
-            - params: Required. Parameters in current group. The value must be a list of `Parameter`.
+            - params: Required. The value must be a list of `Parameter`.
 
-            - lr: Optional. If "lr" in the keys, the value of corresponding learning rate will be used.
-              If not, the `learning_rate` in optimizer will be used. Fixed and dynamic learning rate are supported.
+            - lr: Optional. If "lr" is in the keys, the value of the corresponding learning rate will be used.
+              If not, the `learning_rate` in the API will be used.
 
-            - weight_decay: Optional. If "weight_decay" in the keys, the value of corresponding weight decay
-              will be used. If not, the `weight_decay` in the optimizer will be used. It should be noted that weight
-              decay can be a constant value or a Cell. It is a Cell only when dynamic weight decay is applied. Dynamic
-              weight decay is similar to dynamic learning rate, users need to customize a weight decay schedule only
-              with global step as input, and during training, the optimizer calls the instance of WeightDecaySchedule
-              to get the weight decay value of current step.
+            - weight_decay: Optional. If "weight_decay" is in the keys, the value of the corresponding weight decay
+              will be used. If not, the `weight_decay` in the API will be used.
 
-            - order_params: Optional. When parameters is grouped, this usually is used to maintain the order of
-              parameters that appeared in the network to improve performance. The value should be parameters whose
-              order will be followed in optimizer.
-              If `order_params` in the keys, other keys will be ignored and the element of 'order_params' must be in
-              one group of `params`.
-
-        learning_rate (Union[float, int, Tensor, Iterable, LearningRateSchedule]): Default: 1e-3.
-
-            - float: The fixed learning rate value. Must be equal to or greater than 0.
-
-            - int: The fixed learning rate value. Must be equal to or greater than 0. It will be converted to float.
-
-            - Tensor: Its value should be a scalar or a 1-D vector. For scalar, fixed learning rate will be applied.
-              For vector, learning rate is dynamic, then the i-th step will take the i-th value as the learning rate.
-
-            - Iterable: Learning rate is dynamic. The i-th step will take the i-th value as the learning rate.
-
-            - LearningRateSchedule: Learning rate is dynamic. During training, the optimizer calls the instance of
-              LearningRateSchedule with step as the input to get the learning rate of current step.
+            - order_params: Optional. If "order_params" is in the keys, the value must be the order of parameters and
+              the order will be followed in the optimizer. There are no other keys in the `dict` and the parameters
+              which in the 'order_params' must be in one of group parameters.
 
+        learning_rate (Union[float, Tensor, Iterable, LearningRateSchedule]): A value or a graph for the learning rate.
+            When the learning_rate is an Iterable or a Tensor in a 1D dimension, use the dynamic learning rate, then
+            the i-th step will take the i-th value as the learning rate. When the learning_rate is LearningRateSchedule,
+            use dynamic learning rate, the i-th learning rate will be calculated during the process of training
+            according to the formula of LearningRateSchedule. When the learning_rate is a float or a Tensor in a zero
+            dimension, use fixed learning rate. Other cases are not supported. The float learning rate must be
+            equal to or greater than 0. If the type of `learning_rate` is int, it will be converted to float.
+            Default: 1e-3.
         beta1 (float): The exponential decay rate for the 1st moment estimations. Should be in range (0.0, 1.0).
                        Default: 0.9.
         beta2 (float): The exponential decay rate for the 2nd moment estimations. Should be in range (0.0, 1.0).
                        Default: 0.999.
         eps (float): Term added to the denominator to improve numerical stability. Should be greater than 0. Default:
                      1e-8.
-        use_locking (bool): Whether to enable a lock to protect the updating process of variable tensors.
-            If true, updates of the `w`, `m`, and `v` tensors will be protected by a lock.
+        use_locking (bool): Whether to enable a lock to protect variable tensors from being updated.
+            If true, updates of the var, m, and v tensors will be protected by a lock.
             If false, the result is unpredictable. Default: False.
         use_nesterov (bool): Whether to use Nesterov Accelerated Gradient (NAG) algorithm to update the gradients.
             If true, update the gradients using NAG.
             If false, update the gradients without using NAG. Default: False.
-
-        weight_decay (Union[float, int, Cell]): Weight decay (L2 penalty). Default: 0.0.
-
-            - float: The fixed weight decay value. Must be equal to or greater than 0.
-
-            - int: The fixed weight decay value. Must be equal to or greater than 0. It will be converted to float.
-
-            - Cell: Weight decay is dynamic. During training, the optimizer calls the instance of
-              the Cell with step as the input to get the weight decay value of current step.
-
+        weight_decay (float): Weight decay (L2 penalty). It must be equal to or greater than 0. Default: 0.0.
         loss_scale (float): A floating point value for the loss scale. Should be greater than 0. In general, use the
             default value. Only when `FixedLossScaleManager` is used for training and the `drop_overflow_update` in
             `FixedLossScaleManager` is set to False, then this value needs to be the same as the `loss_scale` in
@@ -713,8 +725,6 @@ class AdamOffload(Optimizer):
         ``Ascend`` ``GPU`` ``CPU``
 
     Examples:
-        >>> from mindspore import nn, Model
-        >>>
         >>> net = Net()
         >>> #1) All parameters use the same learning rate and weight decay
         >>> optim = nn.AdamOffload(params=net.trainable_params())
@@ -736,39 +746,77 @@ class AdamOffload(Optimizer):
 
     def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False,
                  use_nesterov=False, weight_decay=0.0, loss_scale=1.0):
+        '''
+        参数：
+            params：参数列表
+            learning_rate：学习率
+            beta1：beta1的值
+            beta2：beta2的值
+            eps：epsilon的值
+            use_locking：是否使用锁定
+            use_nesterov：是否使用Nesterov梯度
+            weight_decay：权重衰减
+            loss_scale：损失缩放因子
+        '''
         super(AdamOffload, self).__init__(learning_rate, params, weight_decay, loss_scale)
+        # 调用_check_param_value方法检测输入参数值是否符合要求
         _check_param_value(beta1, beta2, eps, self.cls_name)
+        # 检查beta1、beta2、eps参数的值是否符合要求
         validator.check_value_type("use_locking", use_locking, [bool], self.cls_name)
         validator.check_value_type("use_nesterov", use_nesterov, [bool], self.cls_name)
 
+        # 将参数赋值给变量
         self.beta1 = Tensor(beta1, mstype.float32)
         self.beta2 = Tensor(beta2, mstype.float32)
         self.beta1_power = Parameter(initializer(1, [1], mstype.float32), name="beta1_power")
         self.beta2_power = Parameter(initializer(1, [1], mstype.float32), name="beta2_power")
         self.eps = Tensor(eps, mstype.float32)
+        self.use_nesterov = use_nesterov
+        self.use_locking = use_locking
+        # 创建参数
         self.moment1 = self.parameters.clone(prefix="moment1", init='zeros')
         self.moment2 = self.parameters.clone(prefix="moment2", init='zeros')
+
+        # 创建HyperMap
+        self.hyper_map = C.HyperMap()
+        # 创建AdamNoUpdateParam
         self.opt = P.AdamNoUpdateParam(use_locking, use_nesterov)
+        # 将primitive_target设置为CPU
         self.opt.add_prim_attr("primitive_target", "CPU")
 
     def construct(self, gradients):
+        '''
+        参数：
+            gradients：梯度
+        '''
         params = self.parameters
         moment1 = self.moment1
         moment2 = self.moment2
+        # 将梯度衰减
         gradients = self.decay_weight(gradients)
+        # 将梯度缩放
         gradients = self.scale_grad(gradients)
+        # 获取学习率
         lr = self.get_lr()
 
+        # 计算beta1的平方
         beta1_power = self.beta1_power * self.beta1
+        # 更新beta1的平方
         self.beta1_power = beta1_power
+        # 计算beta2的平方
         beta2_power = self.beta2_power * self.beta2
+        # 更新beta2的平方
         self.beta2_power = beta2_power
+        # 如果是分组学习率
         if self.is_group_lr:
-            success = self.map_reverse(F.partial(_adam_opt, self.opt,
-                                                 beta1_power, beta2_power, self.beta1, self.beta2, self.eps),
-                                       lr, gradients, params, moment1, moment2)
+            # 调用_adam_opt函数，传入参数
+            success = self.map_(F.partial(_adam_opt, self.opt,
+                                            beta1_power, beta2_power, self.beta1, self.beta2, self.eps),
+                                            lr, gradients, params, moment1, moment2)
+        # 否则
         else:
-            success = self.map_reverse(F.partial(_adam_opt, self.opt,
-                                                 beta1_power, beta2_power, self.beta1, self.beta2, self.eps, lr),
-                                       gradients, params, moment1, moment2)
-        return success
+            # 调用_adam_opt函数，传入参数
+            success = self.map_(F.partial(_adam_opt, self.opt,
+                                            beta1_power, beta2_power, self.beta1, self.beta2, self.eps, lr),
+                                            gradients, params, moment1, moment2)
+        return success