From 3297a89dc72813715a63c08268163c4634adf8b4 Mon Sep 17 00:00:00 2001 From: muchenjin Date: Thu, 17 Mar 2022 20:38:19 +0800 Subject: [PATCH] Unified formula expression --- .../api_python/nn/mindspore.nn.Adagrad.rst | 27 ++++-- docs/api/api_python/nn/mindspore.nn.Adam.rst | 37 ++++++-- .../nn/mindspore.nn.AdamWeightDecay.rst | 43 ++++++--- docs/api/api_python/nn/mindspore.nn.LARS.rst | 32 ++++--- docs/api/api_python/nn/mindspore.nn.Lamb.rst | 46 +++++++--- .../api_python/nn/mindspore.nn.LazyAdam.rst | 8 +- .../nn/mindspore.nn.ProximalAdagrad.rst | 8 +- docs/api/api_python/nn/mindspore.nn.thor.rst | 21 ++--- .../python/mindspore/nn/optim/ada_grad.py | 28 +++--- mindspore/python/mindspore/nn/optim/adam.py | 88 ++++++++++++++----- mindspore/python/mindspore/nn/optim/lamb.py | 49 ++++++++--- mindspore/python/mindspore/nn/optim/lars.py | 36 +++++--- .../mindspore/nn/optim/proximal_ada_grad.py | 10 +-- mindspore/python/mindspore/nn/optim/thor.py | 21 ++--- 14 files changed, 314 insertions(+), 140 deletions(-) diff --git a/docs/api/api_python/nn/mindspore.nn.Adagrad.rst b/docs/api/api_python/nn/mindspore.nn.Adagrad.rst index a05a140aa97..e438d72d5a6 100644 --- a/docs/api/api_python/nn/mindspore.nn.Adagrad.rst +++ b/docs/api/api_python/nn/mindspore.nn.Adagrad.rst @@ -12,12 +12,29 @@ mindspore.nn.Adagrad .. math:: \begin{array}{ll} \\ - h_{t+1} = h_{t} + g*g\\ - w_{t+1} = w_{t} - lr*\frac{1}{\sqrt{h_{t+1}}}*g - \end{array} + &\newline + &\hline \\ + &\textbf{Parameters}: \text{learning rate } \gamma, \: \text{ params } w_0, \: + \: \text{ weight decay } \lambda, \\ + &\hspace{12mm} \text{ initial accumulator value } state\_sum\\ + &\textbf{Init}: state\_sum_0 \leftarrow 0 \\[-1.ex] + &\newline + &\hline \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{w} f_t (w_{t-1}) \\ + &\hspace{5mm} \textbf{if} \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda w_{t-1} \\ + &\hspace{5mm}state\_sum_t \leftarrow state\_sum_{t-1} + g^2_t \\ + &\hspace{5mm}w_t \leftarrow w_{t-1}- \gamma*\frac{g_t}{\sqrt{state\_sum_t} + \epsilon} \\ + &\newline + &\hline \\ + &\bf{return} \: w_t \\[-1.ex] + &\newline + &\hline \\ + \end{array} - :math:`h` 表示梯度平方的累积和，:math:`g` 表示 `grads` 。 - :math:`lr` 代表 `learning_rate`，:math:`w` 代表 `params` 。 + :math:`state\_sum` 表示梯度平方的累积和 :math:`accum` ，:math:`g` 表示 `grads` ，:math:`\lambda` 代表 `weight_decay` 。 + :math:`\gamma` 代表 `learning_rate`，:math:`w` 代表 `params` 。 .. note:: .. include:: mindspore.nn.optim_note_weight_decay.rst diff --git a/docs/api/api_python/nn/mindspore.nn.Adam.rst b/docs/api/api_python/nn/mindspore.nn.Adam.rst index fc61578d4b0..2689400bf74 100644 --- a/docs/api/api_python/nn/mindspore.nn.Adam.rst +++ b/docs/api/api_python/nn/mindspore.nn.Adam.rst @@ -10,14 +10,39 @@ mindspore.nn.Adam 公式如下： .. math:: - \begin{array}{ll} \\ - m_{t+1} = \beta_1 * m_{t} + (1 - \beta_1) * g \\ - v_{t+1} = \beta_2 * v_{t} + (1 - \beta_2) * g * g \\ - l = \alpha * \frac{\sqrt{1-\beta_2^t}}{1-\beta_1^t} \\ - w_{t+1} = w_{t} - l * \frac{m_{t+1}}{\sqrt{v_{t+1}} + \epsilon} + \begin{array}{l} + &\newline + &\hline \\ + &\textbf{Parameters}: \: 1^{\text {st }}\text {moment vector} \: m , \: 2^{\text {nd}} \: + \text{moment vector} \: v , \\ + &\:\text{gradients } g, \: \text{learning rate} \: \gamma, \text + { exponential decay rates for the moment estimates} \: \beta_{1} \: \beta_{2} , \\ + &\:\text {parameter vector} \: w_{0}, \:\text{timestep} \: t , \text{ weight decay } \lambda \\ + &\textbf{Init}: m_{0} \leftarrow 0, \: v_{0} \leftarrow 0, \: t \leftarrow 0, \: + \text{init parameter vector} \: w_{0} \\[-1.ex] + &\newline + &\hline \\ + &\textbf{while} \: w_{t} \: \text{not converged} \: \textbf{do} \\ + &\hspace{5mm}\boldsymbol{g}_{t} \leftarrow \nabla_{w} \boldsymbol{f}_{t}\left(\boldsymbol{w}_{t-1}\right) \\ + &\hspace{5mm}\textbf {if } \lambda \neq 0 \\ + &\hspace{10mm}\boldsymbol{g}_{t} \leftarrow \boldsymbol{g}_{t}+\lambda \boldsymbol{w}_{t-1} \\ + &\hspace{5mm}\boldsymbol{m}_{t} \leftarrow \beta_{1} \boldsymbol{m}_{t-1}+\left(1-\beta_{1}\right) + \boldsymbol{g}_{t} \\ + &\hspace{5mm}\boldsymbol{v}_{t} \leftarrow \beta_{2} \boldsymbol{v}_{t-1}+\left(1-\beta_{2}\right) + \boldsymbol{g}_{t}^{2} \\ + &\hspace{5mm}\hat{\boldsymbol{m}}_{t} \leftarrow \boldsymbol{m}_{t} /\left(1-\beta_{1}^{t}\right) \\ + &\hspace{5mm}\hat{\boldsymbol{v}}_{t} \leftarrow \boldsymbol{v}_{t} /\left(1-\beta_{2}^{t}\right) \\ + &\hspace{5mm}\boldsymbol{w}_{t} \leftarrow \boldsymbol{w}_{t-1}-\gamma \hat{\boldsymbol{m}}_{t} + /(\sqrt{\hat{\boldsymbol{v}}_{t}}+\epsilon) \\ + &\textbf{end while} \\[-1.ex] + &\newline + &\hline \\[-1.ex] + &\textbf{return} \: \boldsymbol{w}_{t} \\[-1.ex] + &\newline + &\hline \\[-1.ex] \end{array} - :math:`m` 代表第一个动量矩阵 `moment1` ，:math:`v` 代表第二个动量矩阵 `moment2` ，:math:`g` 代表 `gradients` ，:math:`l` 代表缩放因子，:math:`\beta_1,\beta_2` 代表 `beta1` 和 `beta2` ，:math:`t` 代表当前step，:math:`beta_1^t` 和 :math:`beta_2^t` 代表 `beta1_power` 和 `beta2_power` ，:math:`\alpha` 代表 `learning_rate` ，:math:`w` 代表 `params` ，:math:`\epsilon` 代表 `eps` 。 + :math:`m` 代表第一个动量矩阵 `moment1` ，:math:`v` 代表第二个动量矩阵 `moment2` ，:math:`g` 代表梯度 `gradients` ，:math:`\gamma` 代表学习率 `learning_rate` ，:math:`\beta_1, \beta_2` 代表衰减速率 `beta1` 和 `beta2` ，:math:`t` 代表当前step，:math:`beta_1^t` 和 :math:`beta_2^t` 代表 `beta1` 和 `beta2` 的t次方 ， :math:`w` 代表 `params` ， :math:`\epsilon` 代表 `eps` 。 .. note:: .. include:: mindspore.nn.optim_note_sparse.rst diff --git a/docs/api/api_python/nn/mindspore.nn.AdamWeightDecay.rst b/docs/api/api_python/nn/mindspore.nn.AdamWeightDecay.rst index e230105f766..4438e22cecb 100644 --- a/docs/api/api_python/nn/mindspore.nn.AdamWeightDecay.rst +++ b/docs/api/api_python/nn/mindspore.nn.AdamWeightDecay.rst @@ -6,21 +6,38 @@ mindspore.nn.AdamWeightDecay 权重衰减Adam算法的实现。 .. math:: - \begin{array}{ll} \\ - m_{t+1} = \beta_1 * m_{t} + (1 - \beta_1) * g \\ - v_{t+1} = \beta_2 * v_{t} + (1 - \beta_2) * g * g \\ - update = \frac{m_{t+1}}{\sqrt{v_{t+1}} + eps} \\ - update = - \begin{cases} - update + weight\_decay * w_{t} - & \text{ if } weight\_decay > 0 \\ - update - & \text{ otherwise } - \end{cases} \\ - w_{t+1} = w_{t} - lr * update + \begin{array}{l} + &\newline + &\hline \\ + &\textbf{Parameters}: \: 1^{\text {st }}\text {moment vector} \: m , \: 2^{\text {nd}} \: + \text{moment vector} \: v , \\ + &\: gradients \: g, \: \text{learning rate} \: \gamma, + \text {exponential decay rates for the moment estimates} \: \beta_{1} \: \beta_{2} , \\ + &\:\text {parameter vector} \: w_{0}, \:\text{timestep} \: t, \: \text{weight decay} \: \lambda \\ + &\textbf{Init}: m_{0} \leftarrow 0, \: v_{0} \leftarrow 0, \: t \leftarrow 0, \: + \text{init parameter vector} \: w_{0} \\[-1.ex] + &\newline + &\hline \\ + &\textbf{repeat} \\ + &\hspace{5mm} t \leftarrow t+1 \\ + &\hspace{5mm}\boldsymbol{g}_{t} \leftarrow \nabla f_{t}\left(\boldsymbol{w}_{t-1}\right) \\ + &\hspace{5mm}\boldsymbol{m}_{t} \leftarrow \beta_{1} \boldsymbol{m}_{t-1}+\left(1-\beta_{1}\right) + \boldsymbol{g}_{t} \\ + &\hspace{5mm}\boldsymbol{v}_{t} \leftarrow \beta_{2} \boldsymbol{v}_{t-1}+\left(1-\beta_{2}\right) + \boldsymbol{g}_{t}^{2} \\ + &\hspace{5mm}\hat{\boldsymbol{m}}_{t} \leftarrow \boldsymbol{m}_{t} /\left(1-\beta_{1}^{t}\right) \\ + &\hspace{5mm}\hat{\boldsymbol{v}}_{t} \leftarrow \boldsymbol{v}_{t} /\left(1-\beta_{2}^{t}\right) \\ + &\hspace{5mm}\boldsymbol{w}_{t} \leftarrow \boldsymbol{w}_{t-1}-\left(\gamma \hat{\boldsymbol{m}}_{t} + /\left(\sqrt{\hat{\boldsymbol{v}}_{t}}+\epsilon\right)+\lambda \boldsymbol{w}_{t-1}\right) \\ + &\textbf{until}\text { stopping criterion is met } \\[-1.ex] + &\newline + &\hline \\[-1.ex] + &\textbf{return} \: \boldsymbol{w}_{t} \\[-1.ex] + &\newline + &\hline \\[-1.ex] \end{array} - :math:`m` 表示第1矩向量 `moment1` , :math:`v` 表示第2矩向量 `moment2`， :math:`g` 表示 `gradients` ，:math:`lr` 表示 `learning_rate` ，:math:`\beta_1, \beta_2` 表示 `beta1` 和 `beta2` , :math:`t` 表示当前step，:math:`w` 表示 `params`。 + :math:`m` 代表第一个动量矩阵 `moment1` ，:math:`v` 代表第二个动量矩阵 `moment2` ，:math:`g` 代表 `gradients` ，:math:`\gamma` 代表 `learning_rate` ，:math:`\beta_1, \beta_2` 代表 `beta1` 和 `beta2` ， :math:`t` 代表当前step，:math:`w` 代表 `params` ，:math:`\gamma` 代表 `weight_decay` 。 .. note:: .. include:: mindspore.nn.optim_note_loss_scale.rst diff --git a/docs/api/api_python/nn/mindspore.nn.LARS.rst b/docs/api/api_python/nn/mindspore.nn.LARS.rst index 54c6d060cc6..bfdd1524822 100644 --- a/docs/api/api_python/nn/mindspore.nn.LARS.rst +++ b/docs/api/api_python/nn/mindspore.nn.LARS.rst @@ -10,21 +10,29 @@ mindspore.nn.LARS 更新公式如下： .. math:: - \begin{array}{ll} \\ - \lambda = \frac{\theta \text{ * } || \omega || } \\ - {|| g_{t} || \text{ + } \delta \text{ * } || \omega || } \\ - \lambda = - \begin{cases} - \min(\frac{\lambda}{\alpha }, 1) - & \text{ if } clip = True \\ - \lambda - & \text{ otherwise } - \end{cases}\\ - g_{t+1} = \lambda * (g_{t} + \delta * \omega) + &\newline + &\hline \\ + &\textbf{Parameters}: \text{base learning rate } \gamma_{0} , \text{ momentum m}, \text{ weight decay } + \lambda , \\ + &\hspace{5mm}\text{ LARS coefficient } \eta , \text{ number of steps } T \\ + &\textbf{Init}: \text{ t=0, v=0, init weight } w_{0}^{l} \text{ for each layer } l \\[-1.ex] + &\newline + &\hline \\ + &\textbf{while} \text{ t`_。 .. math:: - accum_{t+1} = accum_{t} + grad * grad + accum_{t+1} = accum_{t} + g * g .. math:: - \text{prox_v} = var_{t} - lr * grad * \frac{1}{\sqrt{accum_{t+1}}} + \text{prox_v} = w_{t} - \gamma * g * \frac{1}{\sqrt{accum_{t+1}}} .. math:: - var_{t+1} = \frac{sign(\text{prox_v})}{1 + lr * l2} * \max(\left| \text{prox_v} \right| - lr * l1, 0) + w_{t+1} = \frac{sign(\text{prox_v})}{1 + \gamma * l2} * \max(\left| \text{prox_v} \right| - \gamma * l1, 0) - 其中，grad、lr、var、accum和t分别表示 `grads`, `learning_rate`, `params` 、累加器和当前step。 + 其中， :math:`g` 、 :math:`\gamma` 、 :math:`w` 、 :math:`accum` 和 :math:`t` 分别表示 `grads` 、 `learning_rate` 、 `params` 、累加器和当前step。 .. note:: .. include:: mindspore.nn.optim_note_sparse.rst diff --git a/docs/api/api_python/nn/mindspore.nn.thor.rst b/docs/api/api_python/nn/mindspore.nn.thor.rst index 183d5589580..a20208b0442 100644 --- a/docs/api/api_python/nn/mindspore.nn.thor.rst +++ b/docs/api/api_python/nn/mindspore.nn.thor.rst @@ -12,19 +12,20 @@ mindspore.nn.thor 更新公式如下： .. math:: - \begin{array}{ll} \\ - A_i = a_i{a_i}^T \\ - G_i = D_{s_i}{ D_{s_i}}^T \\ - m_i = \beta * m_i + ({G_i^{(k)}}+\lambda I)^{-1}) g_i ({\overline A_{i-1}^{(k)}}+\lambda I)^{-1} \\ - w_i = w_i - \alpha * m_i \\ + \begin{array}{ll} + & \textbf{Parameter:} \: \text{the learning rate } \gamma\text{, the damping parameter }\lambda \\ + & \textbf{Init:} \: \lambda \leftarrow 0 \\ + & A_{i-1}=\mathbb{E}\left[a_{i-1} a_{i-1}^{T}\right] \\ + & G_{i}=\mathbb{E}\left[D_{s_i} D_{s_i}^{T}\right] \\ + & w_{i}^{(k+1)} \leftarrow w_{i}^{(k)}-\gamma\left(\left(A_{i-1}^{(k)}+\lambda I\right)^{-1} + \otimes\left(G_{i}^{(k)}+\lambda I\right)^{-1}\right) \nabla_{w_{i}} J^{(k)} \end{array} - :math:`D_{s_i}` 表示第i层输出的loss函数的导数。 :math:`a_{i-1}` 表示第i层的输入，它是上一层的激活。 - :math:`\beta` 表示动量， :math:`I` 代表单位矩阵。 - :math:`\overline A` 表示矩阵A的转置。 - :math:`\lambda` 表示'damping'， :math:`g_i` 表示第i层的梯度。 - :math:`\otimes` 表示克罗内克尔积， :math:`\alpha` 表示学习率。 + :math:`D_{s_i}` 表示第i层输出的loss函数的导数。 + :math:`I` 代表单位矩阵。 + :math:`\lambda` 表示 :math:`damping` 参数， :math:`g_i` 表示第i层的梯度。 + :math:`\otimes` 表示克罗内克尔积， :math:`\gamma` 表示学习率。 .. note:: 在分离参数组时，如果权重衰减为正，则每个组的权重衰减将应用于参数。当不分离参数组时，如果 `weight_decay` 为正数，则API中的 `weight_decay` 将应用于名称中没有'beta'或 'gamma'的参数。 diff --git a/mindspore/python/mindspore/nn/optim/ada_grad.py b/mindspore/python/mindspore/nn/optim/ada_grad.py index 8ddd9a5e10e..e93dd06f5f9 100644 --- a/mindspore/python/mindspore/nn/optim/ada_grad.py +++ b/mindspore/python/mindspore/nn/optim/ada_grad.py @@ -48,24 +48,24 @@ class Adagrad(Optimizer): The updating Pseudo codes are as follows, .. math:: - \begin{aligned} - &\hline \\ - &\textbf{Input} : lr \text{ (learning rate)}, \: w_0 \text{ (params)}, \: f(w) - \text{ (objective)}, \: \lambda \text{ (weight decay)}, \\ - &\hspace{12mm} accum \text{ (initial accumulator value)} \\ - &\textbf{Initialize} : state\_sum_0 \leftarrow 0.1 \\[-1.ex] + \begin{aligned} \\ &\newline &\hline \\ - &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ - &\hspace{5mm}g_t \leftarrow \nabla_{w} f_t (w_{t-1}) \\ - &\hspace{5mm} \textbf{if} \: \lambda \neq 0 \\ - &\hspace{10mm} g_t \leftarrow g_t + \lambda w_{t-1} \\ - &\hspace{5mm}state\_sum_t \leftarrow state\_sum_{t-1} + g^2_t \\ - &\hspace{5mm}w_t \leftarrow - w_{t-1}- lr \frac{g_t}{\sqrt{state\_sum_t}} \\ + &\textbf{Parameters}: \text{learning rate } \gamma, \: \text{ params } w_0, \: + \: \text{ weight decay } \lambda, \\ + &\hspace{12mm} \text{ initial accumulator value } state\_sum\\ + &\textbf{Init}: state\_sum_0 \leftarrow 0 \\[-1.ex] &\newline &\hline \\ - &\bf{return} \: w_t \\[-1.ex] + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{w} f_t (w_{t-1}) \\ + &\hspace{5mm} \textbf{if} \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda w_{t-1} \\ + &\hspace{5mm}state\_sum_t \leftarrow state\_sum_{t-1} + g^2_t \\ + &\hspace{5mm}w_t \leftarrow w_{t-1}- \gamma*\frac{g_t}{\sqrt{state\_sum_t} + \epsilon} \\ + &\newline + &\hline \\ + &\bf{return} \: w_t \\[-1.ex] &\newline &\hline \\ \end{aligned} diff --git a/mindspore/python/mindspore/nn/optim/adam.py b/mindspore/python/mindspore/nn/optim/adam.py index 20b48d67735..97213a84e11 100755 --- a/mindspore/python/mindspore/nn/optim/adam.py +++ b/mindspore/python/mindspore/nn/optim/adam.py @@ -197,17 +197,42 @@ class Adam(Optimizer): The updating formulas are as follows, .. math:: - \begin{gather*} - m_{t+1} = \beta_1 * m_{t} + (1 - \beta_1) * g \\ - v_{t+1} = \beta_2 * v_{t} + (1 - \beta_2) * g * g \\ - l = \alpha * \frac{\sqrt{1-\beta_2^t}}{1-\beta_1^t} \\ - w_{t+1} = w_{t} - l * \frac{m_{t+1}}{\sqrt{v_{t+1}} + \epsilon} - \end{gather*} + \begin{array}{l} + &\newline + &\hline \\ + &\textbf{Parameters}: \: 1^{\text {st }}\text {moment vector} \: m , \: 2^{\text {nd}} \: + \text{moment vector} \: v , \\ + &\:\text{gradients } g, \: \text{learning rate} \: \gamma, \text + { exponential decay rates for the moment estimates} \: \beta_{1} \: \beta_{2} , \\ + &\:\text {parameter vector} \: w_{0}, \:\text{timestep} \: t , \text{ weight decay } \lambda \\ + &\textbf{Init}: m_{0} \leftarrow 0, \: v_{0} \leftarrow 0, \: t \leftarrow 0, \: + \text{init parameter vector} \: w_{0} \\[-1.ex] + &\newline + &\hline \\ + &\textbf{while} \: w_{t} \: \text{not converged} \: \textbf{do} \\ + &\hspace{5mm}\boldsymbol{g}_{t} \leftarrow \nabla_{w} \boldsymbol{f}_{t}\left(\boldsymbol{w}_{t-1}\right) \\ + &\hspace{5mm}\textbf {if } \lambda \neq 0 \\ + &\hspace{10mm}\boldsymbol{g}_{t} \leftarrow \boldsymbol{g}_{t}+\lambda \boldsymbol{w}_{t-1} \\ + &\hspace{5mm}\boldsymbol{m}_{t} \leftarrow \beta_{1} \boldsymbol{m}_{t-1}+\left(1-\beta_{1}\right) + \boldsymbol{g}_{t} \\ + &\hspace{5mm}\boldsymbol{v}_{t} \leftarrow \beta_{2} \boldsymbol{v}_{t-1}+\left(1-\beta_{2}\right) + \boldsymbol{g}_{t}^{2} \\ + &\hspace{5mm}\hat{\boldsymbol{m}}_{t} \leftarrow \boldsymbol{m}_{t} /\left(1-\beta_{1}^{t}\right) \\ + &\hspace{5mm}\hat{\boldsymbol{v}}_{t} \leftarrow \boldsymbol{v}_{t} /\left(1-\beta_{2}^{t}\right) \\ + &\hspace{5mm}\boldsymbol{w}_{t} \leftarrow \boldsymbol{w}_{t-1}-\gamma \hat{\boldsymbol{m}}_{t} + /(\sqrt{\hat{\boldsymbol{v}}_{t}}+\epsilon) \\ + &\textbf{end while} \\[-1.ex] + &\newline + &\hline \\[-1.ex] + &\textbf{return} \: \boldsymbol{w}_{t} \\[-1.ex] + &\newline + &\hline \\[-1.ex] + \end{array} :math:`m` represents the 1st moment vector `moment1`, :math:`v` represents the 2nd moment vector `moment2`, - :math:`g` represents `gradients`, :math:`l` represents scaling factor, :math:`\beta_1, \beta_2` represent - `beta1` and `beta2`, :math:`t` represents the current step while :math:`beta_1^t` and :math:`beta_2^t` represent - `beta1_power` and `beta2_power`, :math:`\alpha` represents `learning_rate`, :math:`w` represents `params`, + :math:`g` represents `gradients`, :math:`\beta_1, \beta_2` represent `beta1` and `beta2`, + :math:`t` represents the current step while :math:`beta_1^t` and :math:`beta_2^t` represent + `beta1_power` and `beta2_power`, :math:`\gamma` represents `learning_rate`, :math:`w` represents `params`, :math:`\epsilon` represents `eps`. Note: @@ -398,24 +423,41 @@ class AdamWeightDecay(Optimizer): Implements the Adam algorithm with weight decay. .. math:: - \begin{array}{ll} \\ - m_{t+1} = \beta_1 * m_{t} + (1 - \beta_1) * g \\ - v_{t+1} = \beta_2 * v_{t} + (1 - \beta_2) * g * g \\ - update = \frac{m_{t+1}}{\sqrt{v_{t+1}} + eps} \\ - update = - \begin{cases} - update + weight\_decay * w_{t} - & \text{ if } weight\_decay > 0 \\ - update - & \text{ otherwise } - \end{cases} \\ - w_{t+1} = w_{t} - lr * update + \begin{array}{l} + &\newline + &\hline \\ + &\textbf{Parameters}: \: 1^{\text {st }}\text {moment vector} \: m , \: 2^{\text {nd}} \: + \text{moment vector} \: v , \\ + &\: gradients \: g, \: \text{learning rate} \: \gamma, + \text {exponential decay rates for the moment estimates} \: \beta_{1} \: \beta_{2} , \\ + &\:\text {parameter vector} \: w_{0}, \:\text{timestep} \: t, \: \text{weight decay} \: \lambda \\ + &\textbf{Init}: m_{0} \leftarrow 0, \: v_{0} \leftarrow 0, \: t \leftarrow 0, \: + \text{init parameter vector} \: w_{0} \\[-1.ex] + &\newline + &\hline \\ + &\textbf{repeat} \\ + &\hspace{5mm} t \leftarrow t+1 \\ + &\hspace{5mm}\boldsymbol{g}_{t} \leftarrow \nabla f_{t}\left(\boldsymbol{w}_{t-1}\right) \\ + &\hspace{5mm}\boldsymbol{m}_{t} \leftarrow \beta_{1} \boldsymbol{m}_{t-1}+\left(1-\beta_{1}\right) + \boldsymbol{g}_{t} \\ + &\hspace{5mm}\boldsymbol{v}_{t} \leftarrow \beta_{2} \boldsymbol{v}_{t-1}+\left(1-\beta_{2}\right) + \boldsymbol{g}_{t}^{2} \\ + &\hspace{5mm}\hat{\boldsymbol{m}}_{t} \leftarrow \boldsymbol{m}_{t} /\left(1-\beta_{1}^{t}\right) \\ + &\hspace{5mm}\hat{\boldsymbol{v}}_{t} \leftarrow \boldsymbol{v}_{t} /\left(1-\beta_{2}^{t}\right) \\ + &\hspace{5mm}\boldsymbol{w}_{t} \leftarrow \boldsymbol{w}_{t-1}-\left(\gamma \hat{\boldsymbol{m}}_{t} + /\left(\sqrt{\hat{\boldsymbol{v}}_{t}}+\epsilon\right)+\lambda \boldsymbol{w}_{t-1}\right) \\ + &\textbf{until}\text { stopping criterion is met } \\[-1.ex] + &\newline + &\hline \\[-1.ex] + &\textbf{return} \: \boldsymbol{w}_{t} \\[-1.ex] + &\newline + &\hline \\[-1.ex] \end{array} :math:`m` represents the 1st moment vector `moment1`, :math:`v` represents the 2nd moment vector `moment2`, - :math:`g` represents `gradients`, :math:`lr` represents `learning_rate`, + :math:`g` represents `gradients`, :math:`\gamma` represents `learning_rate`, :math:`\beta_1, \beta_2` represent `beta1` and `beta2`, :math:`t` represents the current step, - :math:`w` represents `params`. + :math:`w` represents `params`, :math:`\gamma` represents `weight_decay`. Note: There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used diff --git a/mindspore/python/mindspore/nn/optim/lamb.py b/mindspore/python/mindspore/nn/optim/lamb.py index ae44e6fc7cc..3ce5300aa63 100755 --- a/mindspore/python/mindspore/nn/optim/lamb.py +++ b/mindspore/python/mindspore/nn/optim/lamb.py @@ -182,17 +182,46 @@ class Lamb(Optimizer): The updating of parameters follows: .. math:: - \begin{gather*} - m_t = \beta_1 m_{t - 1}+ (1 - \beta_1)g_t\\ - v_t = \beta_2 v_{t - 1} + (1 - \beta_2)g_t^2\\ - m_t = \frac{m_t}{\beta_1^t}\\ - v_t = \frac{v_t}{\beta_2^t}\\ - r_t = \frac{m_t}{\sqrt{v_t}+\epsilon}\\ - w_t = w_{t-1} -\eta_t \frac{\| w_{t-1} \|}{\| r_t + \lambda w_{t-1} \|} (r_t + \lambda w_{t-1}) - \end{gather*} + \begin{array}{l} + &\newline + &\hline \\ + &\textbf{Parameters}: \: 1^{\text {st }}\text {moment vector} \: m , \: 2^{\text {nd}} \: + \text{moment vector} \: v , \\ + &\hspace{5mm}\text{learning rate } \left\{ \gamma_{t}\right\}_{t=1}^{T} , \: \text + {exponential decay rates for the moment estimates} \: \beta_{1} \: \beta_{2} , \\ + &\hspace{5mm}\text{scaling function } \phi \\ + &\textbf{Init}: \boldsymbol{m}_{0} \leftarrow 0, \: \boldsymbol{v}_{0} \leftarrow 0 \\[-1.ex] + &\newline + &\hline \\ + &\textbf{for} \text { t=1 to T } \textbf{do} \\ + &\hspace{5mm}\text{Draw b samples } S_{t} \text{ from } \mathbb{P} \text{ . } \\ + &\hspace{5mm}\text{Compute } g_{t}=\frac{1}{\left|\mathcal{S}_{t}\right|} \sum_{s_{t} \in \mathcal{S}_{t}} + \nabla \ell\left(x_{t}, s_{t}\right) . \\ + &\hspace{5mm}\boldsymbol{m}_{t} \leftarrow \beta_{1} \boldsymbol{m}_{t-1}+\left(1-\beta_{1}\right) + \boldsymbol{g}_{t} \\ + &\hspace{5mm}\boldsymbol{v}_{t} \leftarrow \beta_{2} \boldsymbol{v}_{t-1}+\left(1-\beta_{2}\right) + \boldsymbol{g}_{t}^{2} \\ + &\hspace{5mm}\hat{\boldsymbol{m}}_{t} \leftarrow \boldsymbol{m}_{t} /\left(1-\beta_{1}^{t}\right) \\ + &\hspace{5mm}\hat{\boldsymbol{v}}_{t} \leftarrow \boldsymbol{v}_{t} /\left(1-\beta_{2}^{t}\right) \\ + &\hspace{5mm}\text{Compute ratio } \boldsymbol{r}_{t}=\hat{\boldsymbol{m}}_{t} + /(\sqrt{\hat{\boldsymbol{v}}_{t}}+\epsilon) \\ + &\hspace{5mm}\boldsymbol{w}_{t+1}^{(i)}=\boldsymbol{w}_{t}^{(i)}- \gamma_{t} + \frac{\boldsymbol{\phi}\left(\left\|\boldsymbol{w}_{t}^{(i)}\right\|\right)} + {\left\|\boldsymbol{w}_{t}^{(i)}+\lambda \boldsymbol{w}_{t}^{(i)}\right\|}\left(\boldsymbol{r}_{t}^{(i)}+ + \lambda \boldsymbol{w}_{t}^{(i)}\right) \\ + &\textbf{end for} \\[-1.ex] + &\newline + &\hline \\[-1.ex] + &\textbf{return} \: \boldsymbol{w}_{t+1}\\[-1.ex] + &\newline + &\hline \\[-1.ex] + \end{array} - where :math:`m` is the 1st moment, and :math:`v` the 2nd moment, :math:`\eta` the - learning rate, :math:`\lambda` the LAMB weight decay rate. + :math:`m` represents the 1st moment vector `moment1`, :math:`v` represents the 2nd moment vector `moment2`, + :math:`g` represents `gradients`, :math:`\beta_1, \beta_2` represent `beta1` and `beta2`, + :math:`t` represents the current step while :math:`beta_1^t` and :math:`beta_2^t` represent + `beta1_power` and `beta2_power`, :math:`\gamma` represents `learning_rate`, :math:`w` represents `params`, + :math:`\epsilon` represents `eps`, :math:`\lambda` represents `weight_decay`. Note: There is usually no connection between a optimizer and mixed precision. But when `FixedLossScaleManager` is used diff --git a/mindspore/python/mindspore/nn/optim/lars.py b/mindspore/python/mindspore/nn/optim/lars.py index 3bd29c4eb77..75df81409f2 100755 --- a/mindspore/python/mindspore/nn/optim/lars.py +++ b/mindspore/python/mindspore/nn/optim/lars.py @@ -57,23 +57,31 @@ class LARS(Optimizer): The updating formulas are as follows, .. math:: - \begin{array}{ll} \\ - \lambda = \frac{\theta \text{ * } || \omega || } \\ - {|| g_{t} || \text{ + } \delta \text{ * } || \omega || } \\ - \lambda = - \begin{cases} - \min(\frac{\lambda}{\alpha }, 1) - & \text{ if } clip = True \\ - \lambda - & \text{ otherwise } - \end{cases}\\ - g_{t+1} = \lambda * (g_{t} + \delta * \omega) + &\newline + &\hline \\ + &\textbf{Parameters}: \text{base learning rate } \gamma_{0} , \text{ momentum m}, \text{ weight decay } + \lambda , \\ + &\hspace{5mm}\text{ LARS coefficient } \eta , \text{ number of steps } T \\ + &\textbf{Init}: \text{ t=0, v=0, init weight } w_{0}^{l} \text{ for each layer } l \\[-1.ex] + &\newline + &\hline \\ + &\textbf{while} \text{ t`_. .. math:: - accum_{t+1} = accum_{t} + grad * grad + accum_{t+1} = accum_{t} + g * g .. math:: - \text{prox_v} = var_{t} - lr * grad * \frac{1}{\sqrt{accum_{t+1}}} + \text{prox_v} = w_{t} - \gamma * g * \frac{1}{\sqrt{accum_{t+1}}} .. math:: - var_{t+1} = \frac{sign(\text{prox_v})}{1 + lr * l2} * \max(\left| \text{prox_v} \right| - lr * l1, 0) + w_{t+1} = \frac{sign(\text{prox_v})}{1 + \gamma * l2} * \max(\left| \text{prox_v} \right| - \gamma * l1, 0) - Here : where grad, lr, var, accum and t denote the `grads`, `learning_rate`, `params`, accumulation and current - step respectively. + Here : where :math:`g` , :math:`\gamma`, :math:`w` , :math:`accum` and :math:`t` denote the `grads`, + `learning_rate`, `params`, accumulation and current step respectively. Note: The sparse strategy is applied while the SparseGatherV2 operator is used for forward network. If the sparse diff --git a/mindspore/python/mindspore/nn/optim/thor.py b/mindspore/python/mindspore/nn/optim/thor.py index 6c7a6bcbbb0..4a63c0ea247 100644 --- a/mindspore/python/mindspore/nn/optim/thor.py +++ b/mindspore/python/mindspore/nn/optim/thor.py @@ -257,19 +257,20 @@ def thor(net, learning_rate, damping, momentum, weight_decay=0.0, loss_scale=1.0 The updating formulas are as follows, .. math:: - \begin{array}{ll} \\ - A_i = a_i{a_i}^T \\ - G_i = D_{s_i}{ D_{s_i}}^T \\ - m_i = \beta * m_i + ({G_i^{(k)}}+\lambda I)^{-1}) g_i ({\overline A_{i-1}^{(k)}}+\lambda I)^{-1} \\ - w_i = w_i - \alpha * m_i \\ + \begin{array}{ll} + & \textbf{Parameter:} \: \text{the learning rate } \gamma\text{, the damping parameter }\lambda \\ + & \textbf{Init:} \: \lambda \leftarrow 0 \\ + & A_{i-1}=\mathbb{E}\left[a_{i-1} a_{i-1}^{T}\right] \\ + & G_{i}=\mathbb{E}\left[D_{s_i} D_{s_i}^{T}\right] \\ + & w_{i}^{(k+1)} \leftarrow w_{i}^{(k)}-\gamma\left(\left(A_{i-1}^{(k)}+\lambda I\right)^{-1} + \otimes\left(G_{i}^{(k)}+\lambda I\right)^{-1}\right) \nabla_{w_{i}} J^{(k)} \end{array} - :math:`D_{s_i}` represents the derivative of the loss function of the output of the i-th layer, :math:`a_{i-1}` represents the input of i-th layer,and which is the activations of previous layer, - :math:`\beta` represents momentum, :math:`I` represents the identity matrix, - :math:`\overline A` represents the transpose of matrix A, - :math:`\lambda` represents 'damping', :math:`g_i` represents gradients of the i-th layer, - :math:`\otimes` represents Kronecker product, :math:`\alpha` represents 'learning rate' + :math:`D_{s_i}` represents the derivative of the loss function of the output of the i-th layer, + :math:`I` represents the identity matrix, + :math:`\lambda` represents :math:`damping`, :math:`g_i` represents gradients of the i-th layer, + :math:`\otimes` represents Kronecker product, :math:`\gamma` represents 'learning rate' Args: net (Cell): The training network.