From accb87606a245299bcbcfaa91689cf6347baadf4 Mon Sep 17 00:00:00 2001 From: danansheng Date: Thu, 8 Jul 2021 11:10:42 +0800 Subject: [PATCH 1/3] [feat][assistant][I3PYD4] add new data operator HShrink --- mindspore/core/base/core_ops.h | 1 + mindspore/core/ops/hshrink.cc | 58 ++++++++++++++++++++++++++ mindspore/core/ops/hshrink.h | 44 +++++++++++++++++++ mindspore/nn/layer/activation.py | 45 ++++++++++++++++++++ mindspore/ops/_op_impl/tbe/__init__.py | 1 + mindspore/ops/_op_impl/tbe/hshrink.py | 33 +++++++++++++++ mindspore/ops/operations/__init__.py | 4 +- mindspore/ops/operations/nn_ops.py | 43 ++++++++++++++++++- tests/ut/python/ops/test_ops.py | 5 +++ 9 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 mindspore/core/ops/hshrink.cc create mode 100644 mindspore/core/ops/hshrink.h create mode 100644 mindspore/ops/_op_impl/tbe/hshrink.py diff --git a/mindspore/core/base/core_ops.h b/mindspore/core/base/core_ops.h index 5657d8f2df9..74e1c1b08a7 100644 --- a/mindspore/core/base/core_ops.h +++ b/mindspore/core/base/core_ops.h @@ -375,6 +375,7 @@ inline const PrimitivePtr kSquareSumV1 = std::make_shared("SquareSumV inline const PrimitivePtr kFusedMulAdd = std::make_shared("FusedMulAdd"); inline const PrimitivePtr kPrimSoftShrink = std::make_shared("SoftShrink"); inline const PrimitivePtr kPrimSoftShrinkGrad = std::make_shared("SoftShrinkGrad"); +inline const PrimitivePtr kPrimHShrink = std::make_shared("HShrink"); // Comm ops inline const PrimitivePtr kPrimMirror = std::make_shared("_MirrorOperator"); diff --git a/mindspore/core/ops/hshrink.cc b/mindspore/core/ops/hshrink.cc new file mode 100644 index 00000000000..34bf26893ca --- /dev/null +++ b/mindspore/core/ops/hshrink.cc @@ -0,0 +1,58 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "ops/hshrink.h" +#include "utils/check_convert_utils.h" +#include "abstract/primitive_infer_map.h" + +namespace mindspore { +namespace ops { + +namespace { +abstract::ShapePtr InferShape(const PrimitivePtr &primitive, const std::vector &input_args) { + MS_EXCEPTION_IF_NULL(primitive); + for (const auto &item : input_args) { + MS_EXCEPTION_IF_NULL(item); + } + auto in_shape = CheckAndConvertUtils::ConvertShapePtrToShapeMap(input_args[0]->GetShapeTrack())[kShape]; + return std::make_shared(in_shape); +} +TypePtr InferType(const PrimitivePtr &primitive, const std::vector &input_args) { + if (std::any_of(input_args.begin(), input_args.end(), [](const AbstractBasePtr &a) { return a == nullptr; })) { + MS_LOG(EXCEPTION) << "nullptr"; + } + std::map types; + const std::set valid_types = {kFloat16, kFloat32}; + types.emplace("input_x", input_args[0]->BuildType()); + return CheckAndConvertUtils::CheckTensorTypeSame(types, valid_types, primitive->name()); +} + +} // namespace + +AbstractBasePtr HShrinkInfer(const abstract::AnalysisEnginePtr &, const PrimitivePtr &primitive, + const std::vector &input_args) { + return std::make_shared(InferType(primitive, input_args), + InferShape(primitive, input_args)->shape()); +} + +REGISTER_PRIMITIVE_EVAL_IMPL(HShrink, prim::kPrimHShrink, HShrinkInfer, nullptr, true); +} // namespace ops +} // namespace mindspore diff --git a/mindspore/core/ops/hshrink.h b/mindspore/core/ops/hshrink.h new file mode 100644 index 00000000000..dc296c6d8aa --- /dev/null +++ b/mindspore/core/ops/hshrink.h @@ -0,0 +1,44 @@ +/** + * Copyright 2020-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_CORE_OPS_HSHRINK_H +#define MINDSPORE_CORE_OPS_HSHRINK_H + +#include +#include + +#include "ops/primitive_c.h" +#include "abstract/abstract_value.h" +#include "utils/check_convert_utils.h" + +namespace mindspore { +namespace ops { +constexpr auto kNameHShrink = "HShrink"; +class HShrink : public PrimitiveC { + public: + HShrink() : PrimitiveC(kNameHShrink) { + InitIOName({"input_x"}, {"output"}); + } + ~HShrink() = default; + MS_DECLARE_PARENT(HShrink, PrimitiveC); +}; + +AbstractBasePtr HShrinkInfer(const abstract::AnalysisEnginePtr &, const PrimitivePtr &primitive, + const std::vector &input_args); +using PrimHShrinkPtr = std::shared_ptr; +} // namespace ops +} // namespace mindspore + +#endif // MINDSPORE_CORE_OPS_HSHRINK_H diff --git a/mindspore/nn/layer/activation.py b/mindspore/nn/layer/activation.py index b947e5eb873..79a073c44a1 100644 --- a/mindspore/nn/layer/activation.py +++ b/mindspore/nn/layer/activation.py @@ -40,6 +40,7 @@ __all__ = ['Softmax', 'ELU', 'LogSigmoid', 'SoftShrink', + 'HShrink', ] @@ -803,6 +804,49 @@ class SoftShrink(Cell): output = self.softshrink(input_x) return output +class HShrink(Cell): + r""" + Applies the hard shrinkage function element-wise, each element comply the follow function: + + .. math:: + \text{HardShrink}(x) = + \begin{cases} + x, & \text{ if } x > \lambda \\ + x, & \text{ if } x < -\lambda \\ + 0, & \text{ otherwise } + \end{cases} + + Args: + lambd (float): The value for the Hardshrink formulation. Default: 0.5 + + Inputs: + - **input_x** (Tensor) - The input of hshrink with data type of float16 or float32. + + Outputs: + Tensor, the same shape as the input. + + Supported Platforms: + ``Ascend`` ``GPU`` ``CPU`` + + Raises: + TypeError: If `lambd` is not a float. + TypeError: If dtype of `input_x` is neither float16 nor float32. + + Examples: + >>> input_x = Tensor(np.array([[ 0.5, 1, 2.0],[0.0533,0.0776,-2.1233]]),mstype.float32) + >>> hshrink = nn.HShrink() + >>> output = hshrink(input_x) + >>> print(output) + [[ 0. 1. 2. ] + [ 0. 0. -2.1233]] + """ + def __init__(self, lambd=0.5): + super(HShrink, self).__init__() + self.hshrink = P.HShrink(lambd) + def construct(self, input_x): + return self.hshrink(input_x) + + _activation = { 'softmax': Softmax, 'logsoftmax': LogSoftmax, @@ -819,6 +863,7 @@ _activation = { 'hsigmoid': HSigmoid, 'logsigmoid': LogSigmoid, 'softshrink': SoftShrink, + 'hshrink': HShrink, } diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py index a017bc4d416..a52810073f0 100644 --- a/mindspore/ops/_op_impl/tbe/__init__.py +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -394,3 +394,4 @@ from .soft_shrink import _soft_shrink_tbe from .soft_shrink_grad import _soft_shrink_grad_tbe from .hsigmoid_grad import _hsigmoid_grad_tbe from .hsigmoid import _hsigmoid_tbe +from .hshrink import _hshrink_tbe diff --git a/mindspore/ops/_op_impl/tbe/hshrink.py b/mindspore/ops/_op_impl/tbe/hshrink.py new file mode 100644 index 00000000000..261741ea418 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/hshrink.py @@ -0,0 +1,33 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""HardShrink op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType +hshrink_op_info = TBERegOp("HShrink") \ + .fusion_type("ELEMWISE") \ + .async_flag(False) \ + .binfile_name("hard_shrink.so") \ + .compute_cost(10) \ + .kernel_name("hard_shrink") \ + .partial_flag(True) \ + .attr("lambd", "optional", "float", "all", "0.5") \ + .input(0, "input_x", False, "required", "all") \ + .output(0, "output", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + +@op_info_register(hshrink_op_info) +def _hshrink_tbe(): + return diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index bf79430ed1c..4ce999320c7 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -86,7 +86,7 @@ from .nn_ops import (LSTM, SGD, Adam, FusedSparseAdam, FusedSparseLazyAdam, Adam FusedSparseFtrl, FusedSparseProximalAdagrad, ApplyAdaMax, ApplyAdadelta, ApplyAdagrad, ApplyAdagradV2, ApplyAddSign, ApplyPowerSign, ApplyGradientDescent, ApplyProximalGradientDescent, - ApplyRMSProp, ApplyCenteredRMSProp, BasicLSTMCell, InTopK, AdaptiveAvgPool2D, SoftShrink) + ApplyRMSProp, ApplyCenteredRMSProp, BasicLSTMCell, InTopK, AdaptiveAvgPool2D, SoftShrink, HShrink) from . import _quant_ops from ._quant_ops import * from .other_ops import (Assign, InplaceAssign, IOU, BoundingBoxDecode, BoundingBoxEncode, @@ -486,6 +486,8 @@ __all__ = [ "SoftShrink", "FFT3D", "IFFT3D" + "HShrink" + ] __all__.sort() diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 49be3591e0f..3a72e605679 100755 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -8665,7 +8665,6 @@ class SoftShrink(Primitive): x + \lambda, & \text{ if } x < -\lambda \\ 0, & \text{ otherwise } \end{cases} - Args: lambd: the :math:`\lambda` must be no less than zero value for the Softshrink formulation. Default: 0.5. @@ -8699,3 +8698,45 @@ class SoftShrink(Primitive): """Initialize SoftShrink""" validator.check_value_type("lambd", lambd, [float], self.name) validator.check_number("lambd", lambd, 0, Rel.GE, self.name) + + +class HShrink(Primitive): + r""" + Applies the hard shrinkage function element-wise, each element comply the follow function: + + .. math:: + \text{HardShrink}(x) = + \begin{cases} + x, & \text{ if } x > \lambda \\ + x, & \text{ if } x < -\lambda \\ + 0, & \text{ otherwise } + \end{cases} + + Args: + lambd (float): The value for the Hardshrink formulation. Default: 0.5 + + Inputs: + - **input_x** (Tensor) - The input of hshrink with data type of float16 or float32. + + Outputs: + Tensor, the same shape as the input. + + Supported Platforms: + ``Ascend`` ``GPU`` ``CPU`` + + Raises: + TypeError: If `lambd` is not a float. + TypeError: If dtype of `input_x` is neither float16 nor float32. + + Examples: + >>> input_x = Tensor(np.array([[ 0.5, 1, 2.0],[0.0533,0.0776,-2.1233]]),mstype.float32) + >>> hshrink = P.HShrink() + >>> output = hshrink(input_x) + >>> print(output) + [[ 0. 1. 2. ] + [ 0. 0. -2.1233]] + """ + @prim_attr_register + def __init__(self, lambd=0.5): + """Initialize HShrink""" + validator.check_value_type('lambd', lambd, [float], self.name) diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index be3c5f16432..19ac1e81987 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -2204,6 +2204,11 @@ test_case_nn_ops = [ 'desc_inputs': [Tensor(np.array([[-4, 4, 1]]), mstype.float32)], 'desc_bprop': [Tensor(np.array([[0, 1, 0.6666]]), mstype.float32)], 'skip': ['backward']}), + ('HardShrink', { + 'block': P.HShrink(), + 'desc_inputs': [Tensor(np.array([[0.5, 1, 2.0], [0.0533, 0.0776, -2.1233]]), mstype.float32)], + 'desc_bprop': [], + 'skip': ['backward']}), ] test_case_array_ops = [ From b04036e13c480608d9d1edaf27b3ac0682228d03 Mon Sep 17 00:00:00 2001 From: zhangjie <543376780@qq.com> Date: Fri, 11 Jun 2021 20:53:04 +0800 Subject: [PATCH 2/3] [feat][assistant][I3PYD4] add new data operator HShrink and HShrinkGrad --- mindspore/core/base/core_ops.h | 1 + mindspore/core/ops/grad/hshrink_grad.cc | 63 +++++++++++++++++++ mindspore/core/ops/grad/hshrink_grad.h | 43 +++++++++++++ .../ops/_grad_experimental/grad_nn_ops.py | 12 ++++ mindspore/ops/_op_impl/tbe/__init__.py | 1 + mindspore/ops/_op_impl/tbe/hshrink_grad.py | 37 +++++++++++ mindspore/ops/operations/_grad_ops.py | 31 +++++++++ tests/ut/python/ops/test_ops.py | 5 ++ 8 files changed, 193 insertions(+) create mode 100644 mindspore/core/ops/grad/hshrink_grad.cc create mode 100644 mindspore/core/ops/grad/hshrink_grad.h create mode 100644 mindspore/ops/_op_impl/tbe/hshrink_grad.py diff --git a/mindspore/core/base/core_ops.h b/mindspore/core/base/core_ops.h index 74e1c1b08a7..becc48fece6 100644 --- a/mindspore/core/base/core_ops.h +++ b/mindspore/core/base/core_ops.h @@ -376,6 +376,7 @@ inline const PrimitivePtr kFusedMulAdd = std::make_shared("FusedMulAd inline const PrimitivePtr kPrimSoftShrink = std::make_shared("SoftShrink"); inline const PrimitivePtr kPrimSoftShrinkGrad = std::make_shared("SoftShrinkGrad"); inline const PrimitivePtr kPrimHShrink = std::make_shared("HShrink"); +inline const PrimitivePtr kPrimHShrinkGrad = std::make_shared("HShrinkGrad"); // Comm ops inline const PrimitivePtr kPrimMirror = std::make_shared("_MirrorOperator"); diff --git a/mindspore/core/ops/grad/hshrink_grad.cc b/mindspore/core/ops/grad/hshrink_grad.cc new file mode 100644 index 00000000000..0c85f9f4030 --- /dev/null +++ b/mindspore/core/ops/grad/hshrink_grad.cc @@ -0,0 +1,63 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ops/grad/hshrink_grad.h" +#include +#include +#include +#include +#include +#include +#include "ops/op_utils.h" +#include "utils/check_convert_utils.h" +#include "abstract/primitive_infer_map.h" + +namespace mindspore { +namespace ops { + +abstract::ShapePtr HShrinkGradInferShape(const PrimitivePtr &primitive, + const std::vector &input_args) { + MS_EXCEPTION_IF_NULL(primitive); + auto prim_name = primitive->name(); + auto gradients_shape = CheckAndConvertUtils::ConvertShapePtrToShapeMap(input_args[0]->BuildShape())[kShape]; + auto features_shape = CheckAndConvertUtils::ConvertShapePtrToShapeMap(input_args[1]->BuildShape())[kShape]; + + CheckAndConvertUtils::Check("gradients_shape", gradients_shape, kEqual, "features_shape", features_shape, prim_name, + TypeError); + return std::make_shared(gradients_shape); +} + +TypePtr HShrinkGradInferType(const PrimitivePtr &prim, const std::vector &input_args) { + MS_EXCEPTION_IF_NULL(prim); + CheckAndConvertUtils::CheckInteger("input number", input_args.size(), kEqual, 2, prim->name()); + for (const auto &item : input_args) { + MS_EXCEPTION_IF_NULL(item); + } + std::map types; + const std::set valid_types = {kFloat16, kFloat32}; + types.emplace("gradients", input_args[0]->BuildType()); + types.emplace("features", input_args[1]->BuildType()); + return CheckAndConvertUtils::CheckTensorTypeSame(types, valid_types, prim->name()); +} + +AbstractBasePtr HShrinkGradInfer(const abstract::AnalysisEnginePtr &, const PrimitivePtr &primitive, + const std::vector &input_args) { + return std::make_shared(HShrinkGradInferType(primitive, input_args), + HShrinkGradInferShape(primitive, input_args)->shape()); +} +REGISTER_PRIMITIVE_EVAL_IMPL(HShrinkGrad, prim::kPrimHShrinkGrad, HShrinkGradInfer, nullptr, true); +} // namespace ops +} // namespace mindspore diff --git a/mindspore/core/ops/grad/hshrink_grad.h b/mindspore/core/ops/grad/hshrink_grad.h new file mode 100644 index 00000000000..70a62ba9518 --- /dev/null +++ b/mindspore/core/ops/grad/hshrink_grad.h @@ -0,0 +1,43 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CORE_OPS_HShrink_GRAD_H_ +#define MINDSPORE_CORE_OPS_HShrink_GRAD_H_ +#include +#include +#include +#include +#include "ops/primitive_c.h" +#include "abstract/abstract_value.h" +#include "utils/check_convert_utils.h" + +namespace mindspore { +namespace ops { +constexpr auto kNameHShrinkGrad = "HShrinkGrad"; +class HShrinkGrad : public PrimitiveC { + public: + HShrinkGrad() : PrimitiveC(kNameHShrinkGrad) { InitIOName({"gradients", "features"}, {"backprops"}); } + ~HShrinkGrad() = default; + MS_DECLARE_PARENT(HShrinkGrad, PrimitiveC); +}; + +AbstractBasePtr HShrinkGradInfer(const abstract::AnalysisEnginePtr &, const PrimitivePtr &primitive, + const std::vector &input_args); +using PrimHShrinkGradPtr = std::shared_ptr; +} // namespace ops +} // namespace mindspore + +#endif // MINDSPORE_CORE_OPS_HShrink_GRAD_H_ diff --git a/mindspore/ops/_grad_experimental/grad_nn_ops.py b/mindspore/ops/_grad_experimental/grad_nn_ops.py index acb3f84dc31..58106b6c88b 100644 --- a/mindspore/ops/_grad_experimental/grad_nn_ops.py +++ b/mindspore/ops/_grad_experimental/grad_nn_ops.py @@ -44,3 +44,15 @@ def get_bprop_softshrink(self): return (dx,) return bprop + + +@bprop_getters.register(P.HShrink) +def get_bprop_hshrink(self): + """Grad definition for `HShrinkGrad` operation.""" + grad = G.HShrinkGrad() + + def bprop(features, out, gradients): + dx = grad(gradients, features) + return (dx,) + + return bprop diff --git a/mindspore/ops/_op_impl/tbe/__init__.py b/mindspore/ops/_op_impl/tbe/__init__.py index a52810073f0..91144d3176f 100644 --- a/mindspore/ops/_op_impl/tbe/__init__.py +++ b/mindspore/ops/_op_impl/tbe/__init__.py @@ -395,3 +395,4 @@ from .soft_shrink_grad import _soft_shrink_grad_tbe from .hsigmoid_grad import _hsigmoid_grad_tbe from .hsigmoid import _hsigmoid_tbe from .hshrink import _hshrink_tbe +from .hshrink_grad import _hshrink_grad_tbe diff --git a/mindspore/ops/_op_impl/tbe/hshrink_grad.py b/mindspore/ops/_op_impl/tbe/hshrink_grad.py new file mode 100644 index 00000000000..3d162e35aa0 --- /dev/null +++ b/mindspore/ops/_op_impl/tbe/hshrink_grad.py @@ -0,0 +1,37 @@ +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""HShrinkGrad op""" +from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType + +hshrink_grad_op_info = TBERegOp("HShrinkGrad") \ + .fusion_type("OPAQUE") \ + .async_flag(False) \ + .binfile_name("hard_shrink_grad.so") \ + .compute_cost(10) \ + .kernel_name("hard_shrink_grad") \ + .partial_flag(True) \ + .attr("lambda", "optional", "float", "all", "0.5") \ + .input(0, "gradients", False, "required", "all") \ + .input(1, "features", False, "required", "all") \ + .output(0, "backprops", False, "required", "all") \ + .dtype_format(DataType.F16_Default, DataType.F16_Default, DataType.F16_Default) \ + .dtype_format(DataType.F32_Default, DataType.F32_Default, DataType.F32_Default) \ + .get_op_info() + + +@op_info_register(hshrink_grad_op_info) +def _hshrink_grad_tbe(): + """HShrinkGrad TBE register""" + return diff --git a/mindspore/ops/operations/_grad_ops.py b/mindspore/ops/operations/_grad_ops.py index 22f361f7060..b64f1157f97 100644 --- a/mindspore/ops/operations/_grad_ops.py +++ b/mindspore/ops/operations/_grad_ops.py @@ -2212,3 +2212,34 @@ class SoftShrinkGrad(Primitive): self.init_prim_io_names(inputs=['input_grad', 'input_x'], outputs=['output']) validator.check_value_type("lambd", lambd, [float], self.name) validator.check_number("lambd", lambd, 0, Rel.GE, self.name) + + +class HShrinkGrad(Primitive): + """ + Computes gradients for HShrinkGrad operation. + + Args: + lambd (float): the λ value for the Hardshrink formulation. Default: 0.5 + + Inputs: + - **gradients** (Tensor) - the gradients of loss to output of HShrink function. + Currently gradients data type only support float16 and float32. + - **features** (Tensor) - Must be the input `input_x` of the forward operator HSHrink. + Currently features data type only support float16 and float32. + + Outputs: + backprops - Tensor, with the same shape and data type as `features`. + + Rasise: + TypeError: If `lambd` is not a float. + TypeError: If shape of `gradients` is not the same as `features`. + TypeError: If dtype of `gradients` is not the same as `features`. + TypeError: If dtype of `gradients` or `features` is neither float16 nor float32. + + Supported Platforms: + ``Ascend`` + """ + + @prim_attr_register + def __init__(self, lambd=0.5): + validator.check_value_type("lambd", lambd, [float], self.name) diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index 19ac1e81987..bbff59c946f 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -2209,6 +2209,11 @@ test_case_nn_ops = [ 'desc_inputs': [Tensor(np.array([[0.5, 1, 2.0], [0.0533, 0.0776, -2.1233]]), mstype.float32)], 'desc_bprop': [], 'skip': ['backward']}), + ('HShrinkGrad', { + 'block': G.HShrinkGrad(), + 'desc_inputs': [Tensor(np.array([[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]), mstype.float16), + Tensor(np.array([[-4, -3, -2], [1, 2, 4]]), mstype.float16)], + 'skip': ['backward']}), ] test_case_array_ops = [ From 556e67402d22f2bc948e687da7561fbf736967f4 Mon Sep 17 00:00:00 2001 From: danansheng Date: Tue, 20 Jul 2021 15:26:16 +0800 Subject: [PATCH 3/3] [fix][assistant][I3PYD4] fix bug in Ascend operator HShrink and HShrinkGrad --- mindspore/core/ops/grad/hshrink_grad.cc | 4 +--- mindspore/core/ops/grad/hshrink_grad.h | 9 +++------ mindspore/core/ops/hshrink.cc | 11 ++++++----- mindspore/core/ops/hshrink.h | 9 +++------ mindspore/nn/layer/activation.py | 12 +++++++----- mindspore/ops/_grad_experimental/grad_nn_ops.py | 2 +- mindspore/ops/_op_impl/tbe/hshrink.py | 2 +- mindspore/ops/_op_impl/tbe/hshrink_grad.py | 2 +- mindspore/ops/operations/__init__.py | 6 +++--- mindspore/ops/operations/_grad_ops.py | 13 ++++++++----- mindspore/ops/operations/nn_ops.py | 16 ++++++++++------ 11 files changed, 44 insertions(+), 42 deletions(-) diff --git a/mindspore/core/ops/grad/hshrink_grad.cc b/mindspore/core/ops/grad/hshrink_grad.cc index 0c85f9f4030..878ab7b4617 100644 --- a/mindspore/core/ops/grad/hshrink_grad.cc +++ b/mindspore/core/ops/grad/hshrink_grad.cc @@ -18,19 +18,17 @@ #include #include #include -#include #include -#include #include "ops/op_utils.h" #include "utils/check_convert_utils.h" #include "abstract/primitive_infer_map.h" namespace mindspore { namespace ops { - abstract::ShapePtr HShrinkGradInferShape(const PrimitivePtr &primitive, const std::vector &input_args) { MS_EXCEPTION_IF_NULL(primitive); + CheckAndConvertUtils::CheckInteger("input number", input_args.size(), kEqual, 2, primitive->name()); auto prim_name = primitive->name(); auto gradients_shape = CheckAndConvertUtils::ConvertShapePtrToShapeMap(input_args[0]->BuildShape())[kShape]; auto features_shape = CheckAndConvertUtils::ConvertShapePtrToShapeMap(input_args[1]->BuildShape())[kShape]; diff --git a/mindspore/core/ops/grad/hshrink_grad.h b/mindspore/core/ops/grad/hshrink_grad.h index 70a62ba9518..210b8b47965 100644 --- a/mindspore/core/ops/grad/hshrink_grad.h +++ b/mindspore/core/ops/grad/hshrink_grad.h @@ -14,11 +14,9 @@ * limitations under the License. */ -#ifndef MINDSPORE_CORE_OPS_HShrink_GRAD_H_ -#define MINDSPORE_CORE_OPS_HShrink_GRAD_H_ -#include +#ifndef MINDSPORE_CORE_OPS_HSHRINK_GRAD_H_ +#define MINDSPORE_CORE_OPS_HSHRINK_GRAD_H_ #include -#include #include #include "ops/primitive_c.h" #include "abstract/abstract_value.h" @@ -39,5 +37,4 @@ AbstractBasePtr HShrinkGradInfer(const abstract::AnalysisEnginePtr &, const Prim using PrimHShrinkGradPtr = std::shared_ptr; } // namespace ops } // namespace mindspore - -#endif // MINDSPORE_CORE_OPS_HShrink_GRAD_H_ +#endif // MINDSPORE_CORE_OPS_HSHRINK_GRAD_H_ diff --git a/mindspore/core/ops/hshrink.cc b/mindspore/core/ops/hshrink.cc index 34bf26893ca..636a5a393f2 100644 --- a/mindspore/core/ops/hshrink.cc +++ b/mindspore/core/ops/hshrink.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020-2021 Huawei Technologies Co., Ltd + * Copyright 2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,10 +25,10 @@ namespace mindspore { namespace ops { - namespace { abstract::ShapePtr InferShape(const PrimitivePtr &primitive, const std::vector &input_args) { MS_EXCEPTION_IF_NULL(primitive); + CheckAndConvertUtils::CheckInteger("input number", input_args.size(), kEqual, 1, primitive->name()); for (const auto &item : input_args) { MS_EXCEPTION_IF_NULL(item); } @@ -36,13 +36,14 @@ abstract::ShapePtr InferShape(const PrimitivePtr &primitive, const std::vector(in_shape); } TypePtr InferType(const PrimitivePtr &primitive, const std::vector &input_args) { + MS_EXCEPTION_IF_NULL(primitive); + CheckAndConvertUtils::CheckInteger("input number", input_args.size(), kEqual, 1, primitive->name()); if (std::any_of(input_args.begin(), input_args.end(), [](const AbstractBasePtr &a) { return a == nullptr; })) { MS_LOG(EXCEPTION) << "nullptr"; } - std::map types; const std::set valid_types = {kFloat16, kFloat32}; - types.emplace("input_x", input_args[0]->BuildType()); - return CheckAndConvertUtils::CheckTensorTypeSame(types, valid_types, primitive->name()); + return CheckAndConvertUtils::CheckTensorTypeValid("input_x", input_args[0]->BuildType(), valid_types, + primitive->name()); } } // namespace diff --git a/mindspore/core/ops/hshrink.h b/mindspore/core/ops/hshrink.h index dc296c6d8aa..582e8847dea 100644 --- a/mindspore/core/ops/hshrink.h +++ b/mindspore/core/ops/hshrink.h @@ -1,5 +1,5 @@ /** - * Copyright 2020-2021 Huawei Technologies Co., Ltd + * Copyright 2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,17 +28,14 @@ namespace ops { constexpr auto kNameHShrink = "HShrink"; class HShrink : public PrimitiveC { public: - HShrink() : PrimitiveC(kNameHShrink) { - InitIOName({"input_x"}, {"output"}); - } + HShrink() : PrimitiveC(kNameHShrink) { InitIOName({"input_x"}, {"output"}); } ~HShrink() = default; MS_DECLARE_PARENT(HShrink, PrimitiveC); }; AbstractBasePtr HShrinkInfer(const abstract::AnalysisEnginePtr &, const PrimitivePtr &primitive, - const std::vector &input_args); + const std::vector &input_args); using PrimHShrinkPtr = std::shared_ptr; } // namespace ops } // namespace mindspore - #endif // MINDSPORE_CORE_OPS_HSHRINK_H diff --git a/mindspore/nn/layer/activation.py b/mindspore/nn/layer/activation.py index 79a073c44a1..634d3d0ee07 100644 --- a/mindspore/nn/layer/activation.py +++ b/mindspore/nn/layer/activation.py @@ -806,7 +806,7 @@ class SoftShrink(Cell): class HShrink(Cell): r""" - Applies the hard shrinkage function element-wise, each element comply the follow function: + Applies the hard shrinkage function element-wise, each element complies the follow function: .. math:: \text{HardShrink}(x) = @@ -817,16 +817,16 @@ class HShrink(Cell): \end{cases} Args: - lambd (float): The value for the Hardshrink formulation. Default: 0.5 + lambd (float): The value for the HardShrink formulation. Default: 0.5 Inputs: - - **input_x** (Tensor) - The input of hshrink with data type of float16 or float32. + - **input_x** (Tensor) - The input of HardShrink with data type of float16 or float32. Outputs: - Tensor, the same shape as the input. + Tensor, the same shape and data type as the input. Supported Platforms: - ``Ascend`` ``GPU`` ``CPU`` + ``Ascend`` Raises: TypeError: If `lambd` is not a float. @@ -840,9 +840,11 @@ class HShrink(Cell): [[ 0. 1. 2. ] [ 0. 0. -2.1233]] """ + def __init__(self, lambd=0.5): super(HShrink, self).__init__() self.hshrink = P.HShrink(lambd) + def construct(self, input_x): return self.hshrink(input_x) diff --git a/mindspore/ops/_grad_experimental/grad_nn_ops.py b/mindspore/ops/_grad_experimental/grad_nn_ops.py index 58106b6c88b..341f1bc90a1 100644 --- a/mindspore/ops/_grad_experimental/grad_nn_ops.py +++ b/mindspore/ops/_grad_experimental/grad_nn_ops.py @@ -49,7 +49,7 @@ def get_bprop_softshrink(self): @bprop_getters.register(P.HShrink) def get_bprop_hshrink(self): """Grad definition for `HShrinkGrad` operation.""" - grad = G.HShrinkGrad() + grad = G.HShrinkGrad(self.lambd) def bprop(features, out, gradients): dx = grad(gradients, features) diff --git a/mindspore/ops/_op_impl/tbe/hshrink.py b/mindspore/ops/_op_impl/tbe/hshrink.py index 261741ea418..accc40667af 100644 --- a/mindspore/ops/_op_impl/tbe/hshrink.py +++ b/mindspore/ops/_op_impl/tbe/hshrink.py @@ -15,7 +15,7 @@ """HardShrink op""" from mindspore.ops.op_info_register import op_info_register, TBERegOp, DataType hshrink_op_info = TBERegOp("HShrink") \ - .fusion_type("ELEMWISE") \ + .fusion_type("OPAQUE") \ .async_flag(False) \ .binfile_name("hard_shrink.so") \ .compute_cost(10) \ diff --git a/mindspore/ops/_op_impl/tbe/hshrink_grad.py b/mindspore/ops/_op_impl/tbe/hshrink_grad.py index 3d162e35aa0..82a06ce00c3 100644 --- a/mindspore/ops/_op_impl/tbe/hshrink_grad.py +++ b/mindspore/ops/_op_impl/tbe/hshrink_grad.py @@ -22,7 +22,7 @@ hshrink_grad_op_info = TBERegOp("HShrinkGrad") \ .compute_cost(10) \ .kernel_name("hard_shrink_grad") \ .partial_flag(True) \ - .attr("lambda", "optional", "float", "all", "0.5") \ + .attr("lambd", "optional", "float", "all", "0.5") \ .input(0, "gradients", False, "required", "all") \ .input(1, "features", False, "required", "all") \ .output(0, "backprops", False, "required", "all") \ diff --git a/mindspore/ops/operations/__init__.py b/mindspore/ops/operations/__init__.py index 4ce999320c7..0f6f4132929 100644 --- a/mindspore/ops/operations/__init__.py +++ b/mindspore/ops/operations/__init__.py @@ -76,7 +76,7 @@ from .nn_ops import (LSTM, SGD, Adam, FusedSparseAdam, FusedSparseLazyAdam, Adam MaxPool, DataFormatDimMap, AvgPool, Conv2DBackpropInput, ComputeAccidentalHits, MaxPoolWithArgmax, OneHot, Pad, MirrorPad, Mish, PReLU, ReLU, ReLU6, ReLUV2, HSwish, HSigmoid, - ResizeBilinear, Sigmoid, SeLU, + ResizeBilinear, Sigmoid, SeLU, HShrink, SigmoidCrossEntropyWithLogits, NLLLoss, BCEWithLogitsLoss, SmoothL1Loss, Softmax, Softsign, Softplus, LRN, RNNTLoss, DynamicRNN, DynamicGRUV2, SoftmaxCrossEntropyWithLogits, ROIAlign, @@ -86,7 +86,7 @@ from .nn_ops import (LSTM, SGD, Adam, FusedSparseAdam, FusedSparseLazyAdam, Adam FusedSparseFtrl, FusedSparseProximalAdagrad, ApplyAdaMax, ApplyAdadelta, ApplyAdagrad, ApplyAdagradV2, ApplyAddSign, ApplyPowerSign, ApplyGradientDescent, ApplyProximalGradientDescent, - ApplyRMSProp, ApplyCenteredRMSProp, BasicLSTMCell, InTopK, AdaptiveAvgPool2D, SoftShrink, HShrink) + ApplyRMSProp, ApplyCenteredRMSProp, BasicLSTMCell, InTopK, AdaptiveAvgPool2D, SoftShrink) from . import _quant_ops from ._quant_ops import * from .other_ops import (Assign, InplaceAssign, IOU, BoundingBoxDecode, BoundingBoxEncode, @@ -485,7 +485,7 @@ __all__ = [ "TensorScatterSub", "SoftShrink", "FFT3D", - "IFFT3D" + "IFFT3D", "HShrink" ] diff --git a/mindspore/ops/operations/_grad_ops.py b/mindspore/ops/operations/_grad_ops.py index b64f1157f97..ebc15f08451 100644 --- a/mindspore/ops/operations/_grad_ops.py +++ b/mindspore/ops/operations/_grad_ops.py @@ -2219,20 +2219,20 @@ class HShrinkGrad(Primitive): Computes gradients for HShrinkGrad operation. Args: - lambd (float): the λ value for the Hardshrink formulation. Default: 0.5 + Lambd (float): the λ value for the Hardshrink formulation. Default: 0.5 Inputs: - - **gradients** (Tensor) - the gradients of loss to output of HShrink function. + - **Gradients** (Tensor) - the gradients of loss to output of HShrink function. Currently gradients data type only support float16 and float32. - - **features** (Tensor) - Must be the input `input_x` of the forward operator HSHrink. + - **Features** (Tensor) - Must be the input `input_x` of the forward operator HSHrink. Currently features data type only support float16 and float32. Outputs: backprops - Tensor, with the same shape and data type as `features`. Rasise: - TypeError: If `lambd` is not a float. - TypeError: If shape of `gradients` is not the same as `features`. + ValueError: If `lambd` is not a float. + ValueError: If shape of `gradients` is not the same as `features`. TypeError: If dtype of `gradients` is not the same as `features`. TypeError: If dtype of `gradients` or `features` is neither float16 nor float32. @@ -2243,3 +2243,6 @@ class HShrinkGrad(Primitive): @prim_attr_register def __init__(self, lambd=0.5): validator.check_value_type("lambd", lambd, [float], self.name) + if lambd < 0.0: + lambd = 0.0 + self.add_prim_attr('lambd', lambd) diff --git a/mindspore/ops/operations/nn_ops.py b/mindspore/ops/operations/nn_ops.py index 3a72e605679..7d834e30e09 100755 --- a/mindspore/ops/operations/nn_ops.py +++ b/mindspore/ops/operations/nn_ops.py @@ -8702,7 +8702,7 @@ class SoftShrink(Primitive): class HShrink(Primitive): r""" - Applies the hard shrinkage function element-wise, each element comply the follow function: + Applies the hard shrinkage function element-wise, each element complies the follow function: .. math:: \text{HardShrink}(x) = @@ -8711,18 +8711,18 @@ class HShrink(Primitive): x, & \text{ if } x < -\lambda \\ 0, & \text{ otherwise } \end{cases} - + Args: - lambd (float): The value for the Hardshrink formulation. Default: 0.5 + lambd (float): The value for the HardShrink formulation. Default: 0.5 Inputs: - - **input_x** (Tensor) - The input of hshrink with data type of float16 or float32. + - **input_x** (Tensor) - The input of HardShrink with data type of float16 or float32. Outputs: - Tensor, the same shape as the input. + Tensor, the same shape and data type as the input. Supported Platforms: - ``Ascend`` ``GPU`` ``CPU`` + ``Ascend`` Raises: TypeError: If `lambd` is not a float. @@ -8736,7 +8736,11 @@ class HShrink(Primitive): [[ 0. 1. 2. ] [ 0. 0. -2.1233]] """ + @prim_attr_register def __init__(self, lambd=0.5): """Initialize HShrink""" validator.check_value_type('lambd', lambd, [float], self.name) + if lambd < 0.0: + lambd = 0.0 + self.add_prim_attr('lambd', lambd)