!26810 [MS][LITE][r1.5] sync fuzz bugfix and npu bugfix

Merge pull request !26810 from XianglongZeng/r1.5

mindspore/ccsrc/backend/kernel_compiler/cpu/nnacl/infer/broadcast_to_infer.c

@@ -169,6 +169,9 @@ int BroadcastToInferShape(const TensorC *const *inputs, size_t inputs_size, Tens
     }
   } else {
     const TensorC *shape_tensor = inputs[1];
+    if (shape_tensor->data_ == NULL) {
+      return NNACL_INFER_INVALID;
+    }
     dst_shape_size = GetElementNum(shape_tensor);
     if (dst_shape_size > MAX_SHAPE_SIZE) {
       return NNACL_INPUT_TENSOR_ERROR;

mindspore/lite/src/common/tensor_util.cc

@@ -173,6 +173,7 @@ int GenerateOutTensorC(const OpParameter *const parameter, const std::vector<lit
       parameter->type_ == mindspore::schema::PrimitiveType_TensorListSetItem) {
 #ifndef CONTROLFLOW_TENSORLIST_CLIP
     // TensorListC ->TensorC
+    MS_CHECK_TRUE_RET(!outputs.empty() && outputs.front()->data_type() == TypeId::kObjectTypeTensorType, RET_ERROR);
     auto *tensor_list_c = reinterpret_cast<TensorListC *>(malloc(sizeof(TensorListC)));
     if (tensor_list_c == nullptr) {
       return RET_ERROR;

mindspore/lite/src/delegate/npu/npu_converter_utils.h

@@ -31,6 +31,8 @@
 namespace mindspore {
 enum NCHW_SHAPE { NCHW_INVALID = -1, NCHW_N = 0, NCHW_C = 1, NCHW_H = 2, NCHW_W = 3 };
 enum NHWC_SHAPE { NHWC_N = 0, NHWC_H = 1, NHWC_W = 2, NHWC_C = 3 };
+inline const std::vector<int> NHWC2NCHW_PERM = {0, 3, 1, 2};
+inline const std::vector<int> NCHW2NHWC_PERM = {0, 2, 3, 1};
 
 enum NPU_ACTIVATION_MODE {
   ACTIVATION_INVALID = -1,

mindspore/lite/src/delegate/npu/npu_delegate.cc

@@ -51,6 +51,7 @@
 #include "src/delegate/npu/op/tile_npu.h"
 #include "src/delegate/npu/op/transpose_npu.h"
 #include "src/delegate/npu/op/unsqueeze_npu.h"
+#include "src/delegate/npu/op/abs_npu.h"
 #include "src/delegate/npu/npu_graph.h"
 #include "src/delegate/delegate_utils.h"
 #include "src/delegate/npu/pass/npu_transform_pass.h"
@@ -165,6 +166,7 @@ Status NPUDelegate::Init() {
     {schema::PrimitiveType_TileFusion, GetNPUOp<TileNPUOp>},
     {schema::PrimitiveType_Transpose, GetNPUOp<TransposeNPUOp>},
     {schema::PrimitiveType_Unsqueeze, GetNPUOp<UnsqueezeNPUOp>},
+    {schema::PrimitiveType_Abs, GetNPUOp<AbsNPUOp>},
   };
   return mindspore::kSuccess;
 }

mindspore/lite/src/delegate/npu/op/abs_npu.cc

@@ -0,0 +1,58 @@
+/**
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/delegate/npu/op/abs_npu.h"
+#include "include/graph/op/all_ops.h"
+#include "src/kernel_registry.h"
+
+namespace mindspore {
+int AbsNPUOp::Init(const schema::Primitive *primitive, const std::vector<mindspore::MSTensor> &in_tensors,
+                   const std::vector<mindspore::MSTensor> &out_tensors) {
+  // NPU ddk does not support Abs op in fact. Square and Sqrt are utilized to realize it.
+  square_ = new (std::nothrow) hiai::op::Square(name_ + "_square");
+  if (square_ == nullptr) {
+    MS_LOG(ERROR) << name_ << "_square op is nullptr";
+    return RET_ERROR;
+  }
+  sqrt_ = new (std::nothrow) hiai::op::Sqrt(name_ + "_sqrt");
+  if (sqrt_ == nullptr) {
+    MS_LOG(ERROR) << name_ << "_sqrt op is nullptr";
+    return RET_ERROR;
+  }
+  return RET_OK;
+}
+
+int AbsNPUOp::SetNPUInputs(const std::vector<mindspore::MSTensor> &in_tensors,
+                           const std::vector<mindspore::MSTensor> &out_tensors,
+                           const std::vector<ge::Operator *> &npu_inputs) {
+  square_->set_input_x(*npu_inputs[0]);
+  sqrt_->set_input_x(*square_);
+  return RET_OK;
+}
+
+ge::Operator *AbsNPUOp::GetNPUOp() { return this->sqrt_; }
+
+AbsNPUOp::~AbsNPUOp() {
+  if (square_ != nullptr) {
+    delete square_;
+    square_ = nullptr;
+  }
+  if (sqrt_ != nullptr) {
+    delete sqrt_;
+    sqrt_ = nullptr;
+  }
+}
+}  // namespace mindspore

mindspore/lite/src/delegate/npu/op/abs_npu.h

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2020-2021 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_LITE_SRC_RUNTIME_DELEGATE_NPU_OP_ABS_NPU_H_
+#define MINDSPORE_LITE_SRC_RUNTIME_DELEGATE_NPU_OP_ABS_NPU_H_
+#include <vector>
+#include <string>
+#include "include/graph/op/all_ops.h"
+#include "src/delegate/npu/op/npu_op.h"
+
+namespace mindspore {
+class AbsNPUOp : public NPUOp {
+ public:
+  AbsNPUOp(const schema::Primitive *primitive, const std::vector<mindspore::MSTensor> &in_tensors,
+           const std::vector<mindspore::MSTensor> &out_tensors, std::string name)
+      : NPUOp(primitive, in_tensors, out_tensors, name) {}
+
+  ~AbsNPUOp() override;
+
+  int IsSupport(const schema::Primitive *primitive, const std::vector<mindspore::MSTensor> &in_tensors,
+                const std::vector<mindspore::MSTensor> &out_tensors) override {
+    return RET_OK;
+  }
+
+  int Init(const schema::Primitive *primitive, const std::vector<mindspore::MSTensor> &in_tensors,
+           const std::vector<mindspore::MSTensor> &out_tensors) override;
+
+  int SetNPUInputs(const std::vector<mindspore::MSTensor> &in_tensors,
+                   const std::vector<mindspore::MSTensor> &out_tensors,
+                   const std::vector<ge::Operator *> &npu_inputs) override;
+
+  ge::Operator *GetNPUOp() override;
+
+ private:
+  hiai::op::Square *square_ = nullptr;
+  hiai::op::Sqrt *sqrt_ = nullptr;
+};
+}  // namespace mindspore
+#endif  // MINDSPORE_LITE_SRC_RUNTIME_DELEGATE_NPU_OP_ABS_NPU_H_

mindspore/lite/src/delegate/npu/op/deconvolution_npu.cc

@@ -95,6 +95,13 @@ int DeconvolutionNPUOp::Init(const schema::Primitive *primitive, const std::vect
       return RET_ERROR;
     }
   }
+  // The shape should be specified not after op Init method since the tensor shape and format may be changed after pass.
+  auto output_shape = out_tensors.at(0).Shape();
+  auto output_batch = static_cast<int32_t>(output_shape.at(NHWC_N));
+  auto output_channel = static_cast<int32_t>(output_shape.at(NHWC_C));
+  auto output_height = static_cast<int32_t>(output_shape.at(NHWC_H));
+  auto output_width = static_cast<int32_t>(output_shape.at(NHWC_W));
+  out_shape_value_ = {output_batch, output_channel, output_height, output_width};
   return RET_OK;
 }
 
@@ -110,6 +117,18 @@ int DeconvolutionNPUOp::SetNPUInputs(const std::vector<mindspore::MSTensor> &in_
   }
   CHECK_NULL_RETURN(weight_);
   deconv_->set_input_filter(*weight_);
+
+  ge::TensorDesc out_shape_desc(ge::Shape({NPU_SHAPE_SIZE}), ge::FORMAT_NCHW, ge::DT_INT32);
+  ge::TensorPtr out_shape_tensor = std::make_shared<hiai::Tensor>(out_shape_desc);
+  out_shape_tensor->SetData(reinterpret_cast<uint8_t *>(out_shape_value_.data()), NPU_SHAPE_SIZE * sizeof(int32_t));
+  out_shape_ = new (std::nothrow) hiai::op::Const(name_ + "_output_shape");
+  if (out_shape_ == nullptr) {
+    MS_LOG(ERROR) << "create const NPU op failed for " << name_ + "_output_shape";
+    return RET_ERROR;
+  }
+  out_shape_->set_attr_value(out_shape_tensor);
+  deconv_->set_input_output_shape(*out_shape_);
+
   if (in_tensors.size() == CONV_INPUT_SIZE) {
     ret = InitBiasConst(in_tensors);
     if (ret != RET_OK) {
@@ -136,5 +155,9 @@ DeconvolutionNPUOp::~DeconvolutionNPUOp() {
     delete deconv_;
     deconv_ = nullptr;
   }
+  if (out_shape_ != nullptr) {
+    delete out_shape_;
+    out_shape_ = nullptr;
+  }
 }
 }  // namespace mindspore

mindspore/lite/src/delegate/npu/op/deconvolution_npu.h

@@ -18,6 +18,7 @@
 
 #include <vector>
 #include <string>
+#include <memory>
 #include "include/graph/op/all_ops.h"
 #include "src/delegate/npu/op/convolution_base_npu.h"
 
@@ -44,7 +45,9 @@ class DeconvolutionNPUOp : public ConvolutionBaseNPUOp {
  private:
   int SetDeconvParam(const schema::Conv2dTransposeFusion *conv_prim);
   schema::ActivationType act_type_ = schema::ActivationType_NO_ACTIVATION;
+  std::vector<int32_t> out_shape_value_{};
   hiai::op::ConvTranspose *deconv_ = nullptr;
+  hiai::op::Const *out_shape_ = nullptr;
 };
 }  // namespace mindspore
 #endif  // MINDSPORE_LITE_SRC_RUNTIME_DELEGATE_NPU_OP_DECONVOLUTION_NPU_H_

mindspore/lite/src/delegate/npu/op/resize_npu.cc

@@ -20,8 +20,6 @@
 #include "src/delegate/npu/npu_manager.h"
 
 namespace mindspore {
-constexpr int SHAPE_SIZE = 4;
-
 int ResizeNPUOp::IsSupport(const schema::Primitive *primitive, const std::vector<mindspore::MSTensor> &in_tensors,
                            const std::vector<mindspore::MSTensor> &out_tensors) {
   auto resize_prim = primitive->value_as_Resize();
@@ -51,10 +49,10 @@ int ResizeNPUOp::Init(const schema::Primitive *primitive, const std::vector<mind
   auto new_height = static_cast<float>(out_tensors.at(0).Shape().at(NHWC_H));
   auto new_width = static_cast<float>(out_tensors.at(0).Shape().at(NHWC_W));
 
-  ge::TensorDesc sizeTensorDesc(ge::Shape({SHAPE_SIZE}), ge::FORMAT_ND, ge::DT_FLOAT);
+  ge::TensorDesc sizeTensorDesc(ge::Shape({NPU_SHAPE_SIZE}), ge::FORMAT_ND, ge::DT_FLOAT);
   ge::TensorPtr sizeTensor = std::make_shared<hiai::Tensor>(sizeTensorDesc);
-  vector<float> dataValue = {1, 1, new_height / org_height, new_width / org_width};
-  sizeTensor->SetData(reinterpret_cast<uint8_t *>(dataValue.data()), SHAPE_SIZE * sizeof(float));
+  std::vector<float> dataValue = {1, 1, new_height / org_height, new_width / org_width};
+  sizeTensor->SetData(reinterpret_cast<uint8_t *>(dataValue.data()), NPU_SHAPE_SIZE * sizeof(float));
   out_size_ = new (std::nothrow) hiai::op::Const(name_ + "_size");
   if (out_size_ == nullptr) {
     MS_LOG(ERROR) << "create const NPU op failed for " << name_;

mindspore/lite/src/delegate/npu/op/scale_npu.cc

@@ -36,7 +36,7 @@ int ScaleNPUOp::IsSupport(const schema::Primitive *primitive, const std::vector<
   }
   if (axis_ != NHWC_C && axis_ != NCHW_C) {
     if (in_tensors.size() <= BIAS_INDEX) {
-      MS_LOG(INFO) << "Npu Scale op does not support axis: " << axis_ << ", try to convert to Mul op.";
+      MS_LOG(INFO) << "Npu Scale op does not support axis: " << axis_ << ", trying to convert to Mul op.";
       use_mul_ = true;
     } else {
       MS_LOG(WARNING) << "Npu Scale axis attr only support 1 or channel, now is " << axis_;
@@ -136,7 +136,7 @@ int ScaleNPUOp::ConvertScaleToMul(const std::vector<ge::Operator *> &npu_inputs,
     }
     reshape_ = new (std::nothrow) hiai::op::Reshape(name_ + "_reshape");
     if (reshape_ == nullptr) {
-      MS_LOG(ERROR) << "New Reshape npu operator for op " << name_ << " failed.";
+      MS_LOG(ERROR) << "New Reshape npu operator for op " << name_ << "_reshape failed.";
       return RET_ERROR;
     }
     std::shared_ptr<ge::Tensor> shape_tensor = std::make_shared<ge::Tensor>();

mindspore/lite/src/delegate/npu/transpose_kernel.cc

@@ -59,9 +59,7 @@ void PackNCHWToNHWCFp32(const void *src, void *dst, int batch, int plane, int ch
 }
 
 int TransposeNPUKernel::Execute() {
-  std::vector<int> nh2nc_perm = {0, 3, 1, 2};
-  std::vector<int> nc2nh_perm = {0, 2, 3, 1};
-  if (perm_ != nh2nc_perm && perm_ != nc2nh_perm) {
+  if (perm_ != NHWC2NCHW_PERM && perm_ != NCHW2NHWC_PERM) {
     MS_LOG(ERROR) << "NPU transpose op only supports nhwc->nchw or nchw->nhwc.";
     return RET_ERROR;
   }
@@ -76,9 +74,9 @@ int TransposeNPUKernel::Execute() {
   MS_ASSERT(input);
   auto output = out_tensor.MutableData();
   MS_ASSERT(output);
-  if (perm_ == nh2nc_perm) {
+  if (perm_ == NHWC2NCHW_PERM) {
     PackNHWCToNCHWFp32(input, output, shape[NHWC_N], shape[NHWC_H] * shape[NHWC_W], shape[NHWC_C]);
-  } else if (perm_ == nc2nh_perm) {
+  } else if (perm_ == NCHW2NHWC_PERM) {
     PackNCHWToNHWCFp32(input, output, shape[NCHW_N], shape[NCHW_H] * shape[NCHW_W], shape[NCHW_C]);
   } else {
     MS_LOG(ERROR) << "NPU transpose op only supports nhwc->nchw or nchw->nhwc.";

mindspore/lite/src/lite_model.cc

@@ -261,6 +261,20 @@ int LiteModel::SubGraphVerify() const {
       MS_LOG(ERROR) << "Index of graph->node_indices_ is beyond node_size.";
       return RET_ERROR;
     }
+    // Check the graph valid
+    for (auto output : graph->output_indices_) {
+      bool found_output = std::any_of(graph->node_indices_.begin(), graph->node_indices_.end(), [&](uint32_t node_idx) {
+        auto node = this->all_nodes_.at(node_idx);
+        return std::any_of(node->output_indices_.begin(), node->output_indices_.end(),
+                           [&output](uint32_t idx) { return output == idx; });
+      });
+      bool is_input = std::any_of(graph->input_indices_.begin(), graph->input_indices_.end(),
+                                  [&output](uint32_t idx) { return output == idx; });
+      if (!found_output && !is_input) {
+        MS_LOG(ERROR) << "The output is not valid.";
+        return RET_ERROR;
+      }
+    }
   }
   return RET_OK;
 }

mindspore/lite/src/lite_session.cc

@@ -166,6 +166,10 @@ int LiteSession::ConvertTensorsData(const lite::Model *model, size_t tensor_inde
 
   auto ret = DecompressTensor(*src_tensor, dst_tensor);
   if (ret == RET_NO_CHANGE) {
+    if (dst_tensor->Size() == 0 || src_tensor->data()->size() < dst_tensor->Size()) {
+      MS_LOG(ERROR) << "Tensor data shape invalid";
+      return RET_ERROR;
+    }
     dst_tensor->set_data(const_cast<unsigned char *>(src_tensor->data()->data()));
     dst_tensor->set_own_data(false);
   } else if (ret != RET_OK) {

mindspore/lite/src/runtime/kernel/arm/fp32/l2_norm_fp32.cc

@@ -113,6 +113,8 @@ int L2NormCPUKernel::DivSqrtSum(int task_id) {
 
 int L2NormCPUKernel::CalcL2NormTrailingAxis(int task_id) {
   auto input = in_tensors_.at(0);
+  CHECK_NULL_RETURN(input);
+  MS_CHECK_TRUE_RET(!input->shape().empty(), RET_ERROR);
   if (input->shape().back() == 0) {
     MS_LOG(ERROR) << "input->shape().back() is 0";
     return RET_ERROR;

mindspore/lite/src/runtime/kernel/arm/fp32/reverse_sequence_fp32.cc

@@ -66,6 +66,7 @@ int ReverseSequenceCPUKernel::ReSize() {
 
   ConvertAxisToPositive(input0->shape(), &(para->batch_axis_));
   ConvertAxisToPositive(input0->shape(), &(para->seq_axis_));
+  MS_CHECK_TRUE_RET(para->batch_axis_ >= 0 && para->seq_axis_ >= 0, RET_ERROR);
 
   para->ndim_ = input0->shape().size();
   for (int i = 0; i < para->ndim_; i++) {

mindspore/lite/src/runtime/kernel/arm/fp32/scale_fp32.cc

@@ -94,7 +94,7 @@ int ScaleCPUKernel::CalculateParameter() {
   if (scale_param_->axis_ < 0) {
     scale_param_->axis_ = scale_param_->axis_ + in_shape.size();
   }
-  if (scale_shape.size() + scale_param_->axis_ > in_shape.size()) {
+  if (scale_param_->axis_ < 0 || scale_shape.size() + scale_param_->axis_ > in_shape.size()) {
     MS_LOG(ERROR) << "Scale tensor shape is incorrect.";
     return RET_ERROR;
   }

mindspore/lite/src/runtime/kernel/arm/int8/add_int8.cc

@@ -19,6 +19,7 @@
 #include "src/kernel_registry.h"
 #include "include/errorcode.h"
 #include "src/common/file_utils.h"
+#include "src/common/log_util.h"
 
 using mindspore::lite::KernelRegistrar;
 using mindspore::lite::RET_ERROR;
@@ -48,6 +49,12 @@ int QuantizedAddCPUKernel::Init() {
   auto *input1 = in_tensors_.at(1);
   auto *output = out_tensors_.at(0);
 
+  CHECK_NULL_RETURN(input0);
+  CHECK_NULL_RETURN(input1);
+  CHECK_NULL_RETURN(output);
+  MS_CHECK_TRUE_RET(!input0->quant_params().empty(), RET_ERROR);
+  MS_CHECK_TRUE_RET(!input1->quant_params().empty(), RET_ERROR);
+  MS_CHECK_TRUE_RET(!output->quant_params().empty(), RET_ERROR);
   para_->in0_args_.zp_ = input0->quant_params().front().zeroPoint * -1;
   para_->in1_args_.zp_ = input1->quant_params().front().zeroPoint * -1;
   para_->out_zp_ = output->quant_params().front().zeroPoint;

mindspore/lite/src/runtime/kernel/arm/int8/convolution_int8.cc

@@ -101,6 +101,7 @@ int ConvolutionInt8CPUKernel::InitWeightBias() {
   if (in_tensors_.size() == kInputSize2) {
     auto ori_bias = reinterpret_cast<int32_t *>(in_tensors_.at(kBiasIndex)->data());
     CHECK_NULL_RETURN(ori_bias);
+    MS_CHECK_GT(output_channel, 0, RET_ERROR);
     memcpy(bias_data_, ori_bias, static_cast<size_t>(output_channel) * sizeof(int32_t));
   } else {
     MS_ASSERT(in_tensors_.size() == kInputSize1);

mindspore/lite/src/runtime/kernel/arm/int8/transpose_int8.cc

@@ -55,10 +55,13 @@ int TransposeInt8CPUKernel::ReSize() {
 
   // get perm data
   auto perm_tensor = in_tensors_.at(1);
+  MS_CHECK_TRUE_RET(perm_tensor->data_type() == kNumberTypeInt32 || perm_tensor->data_type() == kNumberTypeInt,
+                    RET_ERROR);
   int *perm_data = reinterpret_cast<int *>(perm_tensor->data());
   CHECK_NULL_RETURN(perm_data);
   transpose_param_->num_axes_ = perm_tensor->ElementsNum();
   for (int i = 0; i < transpose_param_->num_axes_; ++i) {
+    MS_CHECK_LT(perm_data[i], static_cast<int>(in_shape.size()), RET_ERROR);
     transpose_param_->perm_[i] = perm_data[i];
   }