From 58e740a48a016cc76bb63cd5a29ac6216ee1663f Mon Sep 17 00:00:00 2001 From: Liu_Xuu Date: Fri, 30 Jul 2021 17:07:47 +0800 Subject: [PATCH] [MSLITE] fix bugs and code check --- .../tensorrt/op/activation_tensorrt.cc | 5 + .../delegate/tensorrt/op/concate_tensorrt.cc | 6 +- .../tensorrt/op/convolution_tensorrt.cc | 15 ++- .../tensorrt/op/deconvolution_tensorrt.cc | 15 ++- .../tensorrt/op/elementwise_tensorrt.cc | 41 ++++--- .../tensorrt/op/elementwise_tensorrt.h | 6 +- .../delegate/tensorrt/op/gather_tensorrt.cc | 5 + .../delegate/tensorrt/op/matmul_tensorrt.cc | 14 ++- .../src/delegate/tensorrt/op/pad_tensorrt.cc | 5 + .../src/delegate/tensorrt/op/pool_tensorrt.cc | 5 + .../delegate/tensorrt/op/reduce_tensorrt.cc | 4 + .../delegate/tensorrt/op/scale_tensorrt.cc | 4 + .../delegate/tensorrt/op/shape_tensorrt.cc | 5 + .../delegate/tensorrt/op/shuffle_tensorrt.cc | 101 +++++++++++++----- .../delegate/tensorrt/op/shuffle_tensorrt.h | 1 + .../delegate/tensorrt/op/slice_tensorrt.cc | 15 +-- .../src/delegate/tensorrt/op/slice_tensorrt.h | 2 + .../delegate/tensorrt/op/softmax_tensorrt.cc | 4 + .../src/delegate/tensorrt/op/tensorrt_op.cc | 11 ++ .../src/delegate/tensorrt/op/tensorrt_op.h | 2 + .../delegate/tensorrt/op/unary_tensorrt.cc | 4 + .../delegate/tensorrt/tensorrt_delegate.cc | 19 ++-- .../delegate/tensorrt/tensorrt_subgraph.cc | 2 + .../src/delegate/tensorrt/tensorrt_subgraph.h | 6 +- .../src/delegate/tensorrt/tensorrt_utils.cc | 2 +- .../src/delegate/tensorrt/tensorrt_utils.h | 2 +- 26 files changed, 227 insertions(+), 74 deletions(-) diff --git a/mindspore/lite/src/delegate/tensorrt/op/activation_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/activation_tensorrt.cc index f81e797efcc..a26c09c2e4a 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/activation_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/activation_tensorrt.cc @@ -21,6 +21,10 @@ namespace mindspore::lite { int ActivationTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (in_tensors.size() != 1) { MS_LOG(ERROR) << "Unsupported input tensor size, size is " << in_tensors.size(); return RET_ERROR; @@ -62,6 +66,7 @@ int ActivationTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { } activation_layer->setName(op_name_.c_str()); + activation_layer->getOutput(0)->setName(out_tensors_[0].Name().c_str()); this->AddInnerOutTensors(activation_layer->getOutput(0)); return RET_OK; diff --git a/mindspore/lite/src/delegate/tensorrt/op/concate_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/concate_tensorrt.cc index 994980e5b29..b156b125dd4 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/concate_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/concate_tensorrt.cc @@ -20,6 +20,10 @@ namespace mindspore::lite { int ConcateTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (in_tensors.size() < 1) { MS_LOG(ERROR) << "Unsupported input tensor size, size is " << in_tensors.size(); return RET_ERROR; @@ -41,7 +45,6 @@ int ConcateTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { MS_LOG(ERROR) << "concate_op convert failed"; return RET_ERROR; } - MS_LOG(INFO) << "in tensort size of concate: " << tensorrt_in_tensors_.size(); if (tensorrt_in_tensors_.size() != in_tensors_.size()) { MS_LOG(ERROR) << "concate_op in tensor is invalid"; return RET_ERROR; @@ -64,6 +67,7 @@ int ConcateTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { concate_layer->setAxis(axis); } concate_layer->setName(op_name_.c_str()); + concate_layer->getOutput(0)->setName(out_tensors_[0].Name().c_str()); this->AddInnerOutTensors(concate_layer->getOutput(0)); return RET_OK; diff --git a/mindspore/lite/src/delegate/tensorrt/op/convolution_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/convolution_tensorrt.cc index 4cbfbd3f207..649158a5365 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/convolution_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/convolution_tensorrt.cc @@ -24,6 +24,10 @@ constexpr int BIAS_INDEX = 2; int ConvolutionTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (in_tensors.size() != 2 && in_tensors.size() != 3) { MS_LOG(ERROR) << "Unsupported input tensor size, size is " << in_tensors.size(); return RET_ERROR; @@ -53,8 +57,12 @@ int ConvolutionTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { } transpose_layer_in->setName((op_name_ + "_transpose2NCHW").c_str()); + // transpose weight + const mindspore::MSTensor &weight_tensor = in_tensors_[1]; + nvinfer1::Weights kernelWeights = lite::TransposeWeight(weight_tensor, &pack_weight_); + // conv - int nbOutputMaps = conv_op->out_channel(); + int nbOutputMaps = weight_tensor.Shape()[0]; if (nbOutputMaps <= 0) { MS_LOG(ERROR) << "out_channel is invalid"; return RET_ERROR; @@ -67,9 +75,6 @@ int ConvolutionTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { } nvinfer1::Dims kernelSize = lite::ConvertCudaDims(std::vector(kernel_size->begin(), kernel_size->end())); - // transpose weight - nvinfer1::Weights kernelWeights = lite::TransposeWeight(in_tensors_[1], &pack_weight_); - // bias nvinfer1::Weights biasWeights{}; if (in_tensors_.size() >= INPUT_SIZE3) { @@ -113,7 +118,7 @@ int ConvolutionTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { return RET_ERROR; } transpose_layer_out->setName((op_name_ + "_transpose2NHWC").c_str()); - + transpose_layer_out->getOutput(0)->setName(out_tensors_[0].Name().c_str()); this->AddInnerOutTensors(transpose_layer_out->getOutput(0)); return RET_OK; } diff --git a/mindspore/lite/src/delegate/tensorrt/op/deconvolution_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/deconvolution_tensorrt.cc index 98d62a5eb9b..8b863ba8349 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/deconvolution_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/deconvolution_tensorrt.cc @@ -23,6 +23,10 @@ namespace mindspore::lite { int DeconvolutionTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (in_tensors.size() != 2 && in_tensors.size() != 3) { MS_LOG(ERROR) << "Unsupported input tensor size, size is " << in_tensors.size(); return RET_ERROR; @@ -51,8 +55,12 @@ int DeconvolutionTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { } transpose_layer_in->setName((op_name_ + "_transpose2NCHW").c_str()); + // transpose weight + const mindspore::MSTensor &weight_tensor = in_tensors_[1]; + nvinfer1::Weights kernelWeights = lite::TransposeWeight(weight_tensor, &pack_weight_); + // deconv basic params - int nbOutputMaps = deconv_op->out_channel(); + int nbOutputMaps = weight_tensor.Shape()[0]; if (nbOutputMaps <= 0) { MS_LOG(ERROR) << "out_channel is invalid"; return RET_ERROR; @@ -65,9 +73,6 @@ int DeconvolutionTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { } nvinfer1::Dims kernelSize = lite::ConvertCudaDims(std::vector(kernel_size->begin(), kernel_size->end())); - // transpose weight - nvinfer1::Weights kernelWeights = lite::TransposeWeight(in_tensors_[1], &pack_weight_); - // bias nvinfer1::Weights biasWeights{}; if (in_tensors_.size() >= 3) { @@ -111,7 +116,7 @@ int DeconvolutionTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { return RET_ERROR; } transpose_layer_out->setName((op_name_ + "_transpose2NHWC").c_str()); - + transpose_layer_out->getOutput(0)->setName(out_tensors_[0].Name().c_str()); this->AddInnerOutTensors(transpose_layer_out->getOutput(0)); return RET_OK; } diff --git a/mindspore/lite/src/delegate/tensorrt/op/elementwise_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/elementwise_tensorrt.cc index 8f0f2fa2894..2b64aad520c 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/elementwise_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/elementwise_tensorrt.cc @@ -21,6 +21,10 @@ namespace mindspore::lite { int ElementWiseTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } std::map element_wise_ops = { {schema::PrimitiveType_AddFusion, nvinfer1::ElementWiseOperation::kSUM}, {schema::PrimitiveType_PowFusion, nvinfer1::ElementWiseOperation::kPOW}, @@ -61,6 +65,13 @@ int ElementWiseTensorRT::IsSupport(const schema::Primitive *primitive, MS_LOG(ERROR) << "invalid output tensort size: " << out_tensors.size(); return RET_ERROR; } + + // if constant tensor is scalar, it needs to know another input tensor's shape to broadcast + if (in_tensors[0].Shape()[0] == -1 && in_tensors[1].Shape().size() == 0) { + MS_LOG(ERROR) << "invalid all input tensor shape unknown for: " << op_name_; + return RET_ERROR; + } + return RET_OK; } @@ -69,23 +80,25 @@ int ElementWiseTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { MS_LOG(ERROR) << "network or input tensor size is invalid"; return RET_ERROR; } - // create ITensor from MS scalar - if (this->in_tensors_[1].Shape().size() == 0) { - nvinfer1::ITensor *scalar_input = - lite::ConvertScalarToITensor(network, this->in_tensors_[0].Shape().size(), this->in_tensors_[1].MutableData()); - if (scalar_input == nullptr) { - MS_LOG(ERROR) << "create Itensor from scalar failed"; - return RET_ERROR; - } - this->AddInnerInTensors(scalar_input); - } + first_in_tensor_index_ = strcmp(tensorrt_in_tensors_[0]->getName(), in_tensors_[0].Name().c_str()) == 0 ? 0 : 1; // add elementwise if (this->tensorrt_in_tensors_.size() != 2) { - MS_LOG(ERROR) << "invalid inner in tensors cnt: " << this->tensorrt_in_tensors_.size(); - return RET_ERROR; + // create ITensor from MS constant tensor of index 1 - first_in_tensor_index_ + nvinfer1::ITensor *constant_input = nullptr; + if (this->in_tensors_[1 - first_in_tensor_index_].Shape().size() == 0) { + constant_input = lite::ConvertScalarToITensor(network, this->in_tensors_[first_in_tensor_index_].Shape().size(), + in_tensors_[1 - first_in_tensor_index_].Data().get()); + } else { + constant_input = lite::ConvertConstantTensor(network, in_tensors_[1 - first_in_tensor_index_]); + } + if (constant_input == nullptr) { + MS_LOG(ERROR) << "create Itensor from constant tensor failed: " << op_name_; + return RET_ERROR; + } + this->AddInnerInTensors(constant_input); } - nvinfer1::IElementWiseLayer *cal_layer = - network->addElementWise(*tensorrt_in_tensors_[0], *tensorrt_in_tensors_[1], element_wise_op_); + nvinfer1::IElementWiseLayer *cal_layer = network->addElementWise( + *tensorrt_in_tensors_[first_in_tensor_index_], *tensorrt_in_tensors_[1 - first_in_tensor_index_], element_wise_op_); if (cal_layer == nullptr) { MS_LOG(ERROR) << "addElementWise failed for TensorRT."; diff --git a/mindspore/lite/src/delegate/tensorrt/op/elementwise_tensorrt.h b/mindspore/lite/src/delegate/tensorrt/op/elementwise_tensorrt.h index a370c80ca5f..c927ab074dd 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/elementwise_tensorrt.h +++ b/mindspore/lite/src/delegate/tensorrt/op/elementwise_tensorrt.h @@ -35,8 +35,12 @@ class ElementWiseTensorRT : public TensorRTOp { const std::vector &out_tensors) override; private: - nvinfer1::ElementWiseOperation element_wise_op_; nvinfer1::ITensor *AddActivation(nvinfer1::INetworkDefinition *network, nvinfer1::ITensor *in_tensor); + + nvinfer1::ElementWiseOperation element_wise_op_; + + // index of first input MSTensor in the trt input tensor vector + size_t first_in_tensor_index_ = 0; }; } // namespace mindspore::lite #endif // MINDSPORE_LITE_SRC_DELEGATE_TENSORRT_OP_ELEMENTWISE_TENSORRT_H_ diff --git a/mindspore/lite/src/delegate/tensorrt/op/gather_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/gather_tensorrt.cc index 410854f0e78..6bdbc2ea740 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/gather_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/gather_tensorrt.cc @@ -22,6 +22,10 @@ constexpr int AXIS_INDEX = 2; int GatherTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (in_tensors.size() != 3) { MS_LOG(ERROR) << "invalid input tensor size: " << in_tensors.size(); return RET_ERROR; @@ -61,6 +65,7 @@ int GatherTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { return RET_ERROR; } gather_layer->setName(op_name_.c_str()); + gather_layer->getOutput(0)->setName(out_tensors_[0].Name().c_str()); this->AddInnerOutTensors(gather_layer->getOutput(0)); return RET_OK; } diff --git a/mindspore/lite/src/delegate/tensorrt/op/matmul_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/matmul_tensorrt.cc index e56a4f3eec8..07a9cf4c7aa 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/matmul_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/matmul_tensorrt.cc @@ -22,6 +22,10 @@ constexpr int BIAS_INDEX = 2; int MatMulTensorRT::IsSupport(const mindspore::schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (in_tensors.size() != 2 && in_tensors.size() != 3) { MS_LOG(ERROR) << "Unsupported input tensor size, size is " << in_tensors.size(); return RET_ERROR; @@ -41,16 +45,18 @@ int MatMulTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { auto matmul_layer = network->addMatrixMultiply(*tensorrt_in_tensors_[0], transpose_a_, *weight, transpose_b_); matmul_layer->setName(op_name_.c_str()); + nvinfer1::ITensor *out_tensor = matmul_layer->getOutput(0); - if (in_tensors_.size() == 3) { + if (in_tensors_.size() == BIAS_INDEX + 1) { auto bias = ConvertTensorWithExpandDims(network, in_tensors_[BIAS_INDEX], in_tensors_[0].Shape().size()); auto bias_layer = network->addElementWise(*matmul_layer->getOutput(0), *bias, nvinfer1::ElementWiseOperation::kSUM); auto bias_layer_name = op_name_ + "_bias"; bias_layer->setName(bias_layer_name.c_str()); - this->AddInnerOutTensors(bias_layer->getOutput(0)); - } else { - this->AddInnerOutTensors(matmul_layer->getOutput(0)); + out_tensor = bias_layer->getOutput(0); } + + out_tensor->setName(out_tensors_[0].Name().c_str()); + this->AddInnerOutTensors(out_tensor); return RET_OK; } } // namespace mindspore::lite diff --git a/mindspore/lite/src/delegate/tensorrt/op/pad_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/pad_tensorrt.cc index d5565765c98..5e1e2e72a66 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/pad_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/pad_tensorrt.cc @@ -23,6 +23,10 @@ namespace mindspore::lite { int PadTensorRT::IsSupport(const mindspore::schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (in_tensors.size() != 2 && in_tensors.size() != 3) { MS_LOG(ERROR) << "Unsupported input tensor size, size is " << in_tensors.size(); return RET_ERROR; @@ -98,6 +102,7 @@ int PadTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { return RET_ERROR; } transpose_layer_out->setName((op_name_ + "_transpose2NHWC").c_str()); + transpose_layer_out->getOutput(0)->setName(out_tensors_[0].Name().c_str()); this->AddInnerOutTensors(transpose_layer_out->getOutput(0)); return RET_OK; diff --git a/mindspore/lite/src/delegate/tensorrt/op/pool_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/pool_tensorrt.cc index 4263755c2fc..3ade0a4834b 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/pool_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/pool_tensorrt.cc @@ -22,6 +22,10 @@ namespace mindspore::lite { int PoolTensorRT::IsSupport(const mindspore::schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (in_tensors.size() != 1) { MS_LOG(ERROR) << "Unsupported input tensor size, size is " << in_tensors.size(); return RET_ERROR; @@ -89,6 +93,7 @@ int PoolTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { return RET_ERROR; } transpose_layer_out->setName((op_name_ + "_transpose2NHWC").c_str()); + transpose_layer_out->getOutput(0)->setName(out_tensors_[0].Name().c_str()); this->AddInnerOutTensors(transpose_layer_out->getOutput(0)); return RET_OK; } diff --git a/mindspore/lite/src/delegate/tensorrt/op/reduce_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/reduce_tensorrt.cc index 8be59ee52d3..3cf38700868 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/reduce_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/reduce_tensorrt.cc @@ -19,6 +19,10 @@ namespace mindspore::lite { int ReduceTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } auto reduce_op = primitive->value_as_ReduceFusion(); if (reduce_op == nullptr) { MS_LOG(ERROR) << "convert failed"; diff --git a/mindspore/lite/src/delegate/tensorrt/op/scale_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/scale_tensorrt.cc index b665c65fc7b..f0135bc2ef1 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/scale_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/scale_tensorrt.cc @@ -26,6 +26,10 @@ constexpr int POWER_INDEX = 3; int ScaleTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (in_tensors.size() != 2 && in_tensors.size() != 3 && in_tensors.size() != 4) { MS_LOG(ERROR) << "Unsupported input tensor size, size is: " << in_tensors.size(); return RET_ERROR; diff --git a/mindspore/lite/src/delegate/tensorrt/op/shape_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/shape_tensorrt.cc index 4db3722db10..9e006341215 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/shape_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/shape_tensorrt.cc @@ -19,6 +19,10 @@ namespace mindspore::lite { int ShapeTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (in_tensors.size() != 1) { MS_LOG(ERROR) << "Unsupported input tensor size, size is " << in_tensors.size(); return RET_ERROR; @@ -41,6 +45,7 @@ int ShapeTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { return RET_ERROR; } shape_layer->setName(op_name_.c_str()); + shape_layer->getOutput(0)->setName(out_tensors_[0].Name().c_str()); this->AddInnerOutTensors(shape_layer->getOutput(0)); return RET_OK; } diff --git a/mindspore/lite/src/delegate/tensorrt/op/shuffle_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/shuffle_tensorrt.cc index d5d21cf9270..21b3ae2e66e 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/shuffle_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/shuffle_tensorrt.cc @@ -16,19 +16,49 @@ #include "src/delegate/tensorrt/op/shuffle_tensorrt.h" #include +#include +#include namespace mindspore::lite { int ShuffleTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { - if ((type_ == schema::PrimitiveType::PrimitiveType_Squeeze || - type_ == schema::PrimitiveType::PrimitiveType_Unsqueeze) && - in_tensors.size() != 1) { - MS_LOG(ERROR) << "invalid input tensort size: " << in_tensors.size(); + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; return RET_ERROR; } - if ((type_ == schema::PrimitiveType::PrimitiveType_Transpose) && in_tensors.size() != 2) { - MS_LOG(ERROR) << "invalid input tensort size: " << in_tensors.size(); - return RET_ERROR; + switch (type_) { + case schema::PrimitiveType_Flatten: + case schema::PrimitiveType_Squeeze: + case schema::PrimitiveType_Unsqueeze: { + if (in_tensors.size() != 1) { + MS_LOG(ERROR) << "Unsupported in_tensors size " << in_tensors.size() << " of " + << schema::EnumNamePrimitiveType(type_); + return RET_ERROR; + } + break; + } + case schema::PrimitiveType_Reshape: { + if (in_tensors.size() != 2) { + MS_LOG(ERROR) << "PrimitiveType_Transpose Unsupported in_tensors size: " << in_tensors.size(); + return RET_ERROR; + } + break; + } + case schema::PrimitiveType_Transpose: { + if (in_tensors.size() != 2) { + MS_LOG(ERROR) << "PrimitiveType_Transpose Unsupported in_tensors size: " << in_tensors.size(); + return RET_ERROR; + } + if (in_tensors[1].Data() == nullptr) { + MS_LOG(ERROR) << "Unsupported shape tensor of " << schema::EnumNamePrimitiveType(type_); + return RET_ERROR; + } + break; + } + default: { + MS_LOG(ERROR) << "Unsupported op type:" << schema::EnumNamePrimitiveType(type_); + return RET_ERROR; + } } if (out_tensors.size() != 1) { MS_LOG(ERROR) << "invalid output tensort size: " << out_tensors.size(); @@ -49,7 +79,7 @@ int ShuffleTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { } shuffle_layer->setName(op_name_.c_str()); - switch (this->type()) { + switch (type_) { case schema::PrimitiveType_Unsqueeze: { int ret = AddUnsqueezeOp(shuffle_layer); if (ret != RET_OK) { @@ -82,6 +112,14 @@ int ShuffleTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { } break; } + case schema::PrimitiveType_Flatten: { + int ret = AddFlattenOp(shuffle_layer); + if (ret != RET_OK) { + MS_LOG(ERROR) << "AddFlattenOp failed."; + return ret; + } + break; + } default: MS_LOG(ERROR) << "Unsupported op type."; return RET_ERROR; @@ -148,7 +186,6 @@ int ShuffleTensorRT::AddUnsqueezeOp(nvinfer1::IShuffleLayer *shuffle_layer) { } nvinfer1::Dims unsqueeze_dims = lite::ConvertCudaDims(unsqueeze_shape); - MS_LOG(INFO) << "AddUnsqueezeOp: " << op_name_ << " unsqueeze_dims.nbDims: " << unsqueeze_dims.nbDims; shuffle_layer->setReshapeDimensions(unsqueeze_dims); return shuffle_layer->getOutput(0) == nullptr ? RET_ERROR : RET_OK; @@ -166,8 +203,8 @@ int ShuffleTensorRT::AddTransposeOp(nvinfer1::IShuffleLayer *shuffle_layer) { } // perm mindspore::MSTensor perm_ternsor = in_tensors_[1]; - if (perm_ternsor.Data() == nullptr || perm_ternsor.ElementNum() != tensorrt_in_tensors_[0]->getDimensions().nbDims) { - MS_LOG(ERROR) << "AddTransposeOp perm_ternsor data is invalid."; + if (perm_ternsor.Data() == nullptr) { + MS_LOG(ERROR) << "AddTransposeOp perm_ternsor data is invalid: " << op_name_; return RET_ERROR; } int *perm_data = reinterpret_cast(perm_ternsor.MutableData()); @@ -180,26 +217,38 @@ int ShuffleTensorRT::AddTransposeOp(nvinfer1::IShuffleLayer *shuffle_layer) { shuffle_layer->setFirstTranspose(perm); return RET_OK; } + int ShuffleTensorRT::AddReshapeOp(nvinfer1::IShuffleLayer *shuffle_layer) { - auto reshape_op = this->op_primitive_->value_as_Reshape(); - if (reshape_op == nullptr) { - MS_LOG(ERROR) << "AddReshapeOp convert failed"; - return RET_ERROR; - } - if (in_tensors_.size() != 2) { - MS_LOG(ERROR) << "AddReshapeOp size of in tensort needs check: " << in_tensors_.size(); - return RET_ERROR; - } mindspore::MSTensor &shape_tensor = in_tensors_[1]; - nvinfer1::Dims reshape_dims = ConvertCudaDims(shape_tensor.Data().get(), shape_tensor.ElementNum()); - int ret = InferReshapeDims(tensorrt_in_tensors_[0]->getDimensions(), &reshape_dims); - if (ret != RET_OK) { - MS_LOG(ERROR) << "invalid dims for reshape " << op_name_; - return ret; + if (shape_tensor.Data() != nullptr) { + // static shuffle layer + nvinfer1::Dims reshape_dims = lite::ConvertCudaDims(shape_tensor.Data().get(), shape_tensor.ElementNum()); + int ret = InferReshapeDims(tensorrt_in_tensors_[0]->getDimensions(), &reshape_dims); + if (ret != RET_OK) { + MS_LOG(ERROR) << "invalid dims for reshape " << op_name_; + return ret; + } + shuffle_layer->setReshapeDimensions(reshape_dims); + } else { + if (tensorrt_in_tensors_.size() != 2) { + MS_LOG(ERROR) << "invalid shape tensor for reshape " << op_name_; + return RET_ERROR; + } + shuffle_layer->setInput(1, *tensorrt_in_tensors_[1]); } - shuffle_layer->setReshapeDimensions(reshape_dims); return RET_OK; } + +int ShuffleTensorRT::AddFlattenOp(nvinfer1::IShuffleLayer *shuffle_layer) { + nvinfer1::Dims flatten_dims; + const std::vector &input_shape = in_tensors_[0].Shape(); + flatten_dims.nbDims = 2; + flatten_dims.d[0] = input_shape[0]; + flatten_dims.d[1] = std::accumulate(input_shape.begin() + 1, input_shape.end(), 1, std::multiplies()); + shuffle_layer->setReshapeDimensions(flatten_dims); + return RET_OK; +} + int ShuffleTensorRT::InferReshapeDims(nvinfer1::Dims input_dims, nvinfer1::Dims *reshape_dims) { int infer_index = -1; int known_cnt = 1; diff --git a/mindspore/lite/src/delegate/tensorrt/op/shuffle_tensorrt.h b/mindspore/lite/src/delegate/tensorrt/op/shuffle_tensorrt.h index 98d90d9ac2c..e799a7dcaee 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/shuffle_tensorrt.h +++ b/mindspore/lite/src/delegate/tensorrt/op/shuffle_tensorrt.h @@ -39,6 +39,7 @@ class ShuffleTensorRT : public TensorRTOp { int AddUnsqueezeOp(nvinfer1::IShuffleLayer *shuffle_layer); int AddTransposeOp(nvinfer1::IShuffleLayer *shuffle_layer); int AddReshapeOp(nvinfer1::IShuffleLayer *shuffle_layer); + int AddFlattenOp(nvinfer1::IShuffleLayer *shuffle_layer); int InferReshapeDims(nvinfer1::Dims input_dims, nvinfer1::Dims *reshape_dims); }; } // namespace mindspore::lite diff --git a/mindspore/lite/src/delegate/tensorrt/op/slice_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/slice_tensorrt.cc index 4946fa0b501..a5e172e0dc5 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/slice_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/slice_tensorrt.cc @@ -21,7 +21,11 @@ namespace mindspore::lite { int SliceTensorRT::IsSupport(const mindspore::schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { - if (in_tensors.size() != 4 && in_tensors.size() != 5) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } + if (in_tensors.size() < STRIDE_INDEX + 1) { MS_LOG(ERROR) << "Unsupported input tensor size, size is " << in_tensors.size(); return RET_ERROR; } @@ -29,8 +33,8 @@ int SliceTensorRT::IsSupport(const mindspore::schema::Primitive *primitive, MS_LOG(ERROR) << "Unsupported output tensor size, size is " << out_tensors.size(); return RET_ERROR; } - if (in_tensors_[1].Data() == nullptr) { - MS_LOG(ERROR) << "invalid pad tensor for: " << op_name_; + if (in_tensors_[BEGIN_INDEX].Data() == nullptr || in_tensors_[STRIDE_INDEX].Data() == nullptr) { + MS_LOG(ERROR) << "invalid pad or stride tensor for: " << op_name_; return RET_ERROR; } return RET_OK; @@ -42,9 +46,8 @@ int SliceTensorRT::AddInnerOp(nvinfer1::INetworkDefinition *network) { MS_LOG(ERROR) << "convert StridedSlice failed: " << op_name_; return RET_ERROR; } - const mindspore::MSTensor &begin = in_tensors_[1]; - // mindspore::MSTensor &end = in_tensors_[2]; - const mindspore::MSTensor &stride = in_tensors_[3]; + const mindspore::MSTensor &begin = in_tensors_[BEGIN_INDEX]; + const mindspore::MSTensor &stride = in_tensors_[STRIDE_INDEX]; nvinfer1::Dims start_dims = lite::ConvertCudaDims(begin.Data().get(), begin.ElementNum()); nvinfer1::Dims size_dims = lite::ConvertCudaDims(out_tensors_[0].Shape()); diff --git a/mindspore/lite/src/delegate/tensorrt/op/slice_tensorrt.h b/mindspore/lite/src/delegate/tensorrt/op/slice_tensorrt.h index 7bedfaf2adf..856f4d50712 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/slice_tensorrt.h +++ b/mindspore/lite/src/delegate/tensorrt/op/slice_tensorrt.h @@ -20,6 +20,8 @@ #include "src/delegate/tensorrt/op/tensorrt_op.h" namespace mindspore::lite { +constexpr int BEGIN_INDEX = 1; +constexpr int STRIDE_INDEX = 3; class SliceTensorRT : public TensorRTOp { public: SliceTensorRT(const schema::Primitive *primitive, const std::vector &in_tensors, diff --git a/mindspore/lite/src/delegate/tensorrt/op/softmax_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/softmax_tensorrt.cc index 6f3d418fd34..e65508276f7 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/softmax_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/softmax_tensorrt.cc @@ -19,6 +19,10 @@ namespace mindspore::lite { int SoftMaxTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (primitive->value_type() == schema::PrimitiveType::PrimitiveType_LogSoftmax) { with_log_ = true; auto softmax_op = primitive->value_as_LogSoftmax(); diff --git a/mindspore/lite/src/delegate/tensorrt/op/tensorrt_op.cc b/mindspore/lite/src/delegate/tensorrt/op/tensorrt_op.cc index 5acc69ef559..4f7b3ca8164 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/tensorrt_op.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/tensorrt_op.cc @@ -42,4 +42,15 @@ void TensorRTOp::set_out_ops(const std::vector &out_ops) { this->o const std::vector &TensorRTOp::in_ops() const { return this->in_ops_; } const std::vector &TensorRTOp::out_ops() const { return this->out_ops_; } + +bool TensorRTOp::IsShapeKnown() { + if (this->in_tensors_[0].Shape().size() == 0) { + return false; + } else { + if (this->in_tensors_[0].Shape()[0] == -1) { + return false; + } + } + return true; +} } // namespace mindspore::lite diff --git a/mindspore/lite/src/delegate/tensorrt/op/tensorrt_op.h b/mindspore/lite/src/delegate/tensorrt/op/tensorrt_op.h index 91e73de901f..9cc77218988 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/tensorrt_op.h +++ b/mindspore/lite/src/delegate/tensorrt/op/tensorrt_op.h @@ -75,6 +75,8 @@ class TensorRTOp { const std::vector &out_ops() const; protected: + bool IsShapeKnown(); + std::vector layers_; const schema::Primitive *op_primitive_; diff --git a/mindspore/lite/src/delegate/tensorrt/op/unary_tensorrt.cc b/mindspore/lite/src/delegate/tensorrt/op/unary_tensorrt.cc index 4549a8f5498..c5f59da7825 100644 --- a/mindspore/lite/src/delegate/tensorrt/op/unary_tensorrt.cc +++ b/mindspore/lite/src/delegate/tensorrt/op/unary_tensorrt.cc @@ -19,6 +19,10 @@ namespace mindspore::lite { int UnaryTensorRT::IsSupport(const schema::Primitive *primitive, const std::vector &in_tensors, const std::vector &out_tensors) { + if (!IsShapeKnown()) { + MS_LOG(ERROR) << "Unsupported input tensor unknown shape: " << op_name_; + return RET_ERROR; + } if (in_tensors.size() != 1) { MS_LOG(ERROR) << "Unsupported input tensor size, size is " << in_tensors.size(); } diff --git a/mindspore/lite/src/delegate/tensorrt/tensorrt_delegate.cc b/mindspore/lite/src/delegate/tensorrt/tensorrt_delegate.cc index 4965a6c1059..e295c34ef3f 100644 --- a/mindspore/lite/src/delegate/tensorrt/tensorrt_delegate.cc +++ b/mindspore/lite/src/delegate/tensorrt/tensorrt_delegate.cc @@ -69,9 +69,6 @@ int TensorRTDelegate::Init() { op_func_lists_.clear(); op_func_lists_ = { {schema::PrimitiveType_Activation, GetTensorRTOp}, - {schema::PrimitiveType_Unsqueeze, GetTensorRTOp}, - {schema::PrimitiveType_Squeeze, GetTensorRTOp}, - {schema::PrimitiveType_Reshape, GetTensorRTOp}, {schema::PrimitiveType_Concat, GetTensorRTOp}, {schema::PrimitiveType_Conv2DFusion, GetTensorRTOp}, {schema::PrimitiveType_Conv2dTransposeFusion, GetTensorRTOp}, @@ -81,14 +78,20 @@ int TensorRTDelegate::Init() { {schema::PrimitiveType_AddFusion, GetTensorRTOp}, {schema::PrimitiveType_MulFusion, GetTensorRTOp}, {schema::PrimitiveType_Eltwise, GetTensorRTOp}, - {schema::PrimitiveType_Transpose, GetTensorRTOp}, - {schema::PrimitiveType_ReduceFusion, GetTensorRTOp}, - {schema::PrimitiveType_Sqrt, GetTensorRTOp}, + {schema::PrimitiveType_Gather, GetTensorRTOp}, {schema::PrimitiveType_MatMul, GetTensorRTOp}, - {schema::PrimitiveType_ScaleFusion, GetTensorRTOp}, - {schema::PrimitiveType_StridedSlice, GetTensorRTOp}, {schema::PrimitiveType_AvgPoolFusion, GetTensorRTOp}, {schema::PrimitiveType_PadFusion, GetTensorRTOp}, + {schema::PrimitiveType_ReduceFusion, GetTensorRTOp}, + {schema::PrimitiveType_ScaleFusion, GetTensorRTOp}, + {schema::PrimitiveType_StridedSlice, GetTensorRTOp}, + {schema::PrimitiveType_Shape, GetTensorRTOp}, + {schema::PrimitiveType_Unsqueeze, GetTensorRTOp}, + {schema::PrimitiveType_Squeeze, GetTensorRTOp}, + {schema::PrimitiveType_Reshape, GetTensorRTOp}, + {schema::PrimitiveType_Transpose, GetTensorRTOp}, + {schema::PrimitiveType_Flatten, GetTensorRTOp}, + {schema::PrimitiveType_Sqrt, GetTensorRTOp}, }; return RET_OK; } diff --git a/mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.cc b/mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.cc index 1c3ce666941..2be96a83f27 100644 --- a/mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.cc +++ b/mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.cc @@ -158,6 +158,7 @@ int TensorRTSubGraph::BuildTensorRTGraph() { return RET_ERROR; } trt_tensor = lite::ConvertConstantTensor(this->network_, in_tensor); + MS_LOG(INFO) << "auto convert constant tensor for: " << cur_op->GetOpName(); cur_op->AddInnerInTensors(trt_tensor); } } else { @@ -178,6 +179,7 @@ int TensorRTSubGraph::BuildTensorRTGraph() { for (size_t index = 0; index < out_op->outputs().size(); index++) { if (out_op->outputs()[index] == out_tensor) { out_op->GetInnerOutTensor()[index]->setName(out_tensor.Name().c_str()); + MS_LOG(INFO) << "markOutput for: " << out_tensor.Name(); this->network_->markOutput(*out_op->GetInnerOutTensor()[index]); } } diff --git a/mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.h b/mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.h index 80ed386df7d..cd9163112c2 100644 --- a/mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.h +++ b/mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.h @@ -37,8 +37,10 @@ class TensorRTSubGraph : public kernel::Kernel { trt_specific_weight_nodes_ = { schema::PrimitiveType_Conv2DFusion, schema::PrimitiveType_ReduceFusion, schema::PrimitiveType_Transpose, schema::PrimitiveType_Gather, schema::PrimitiveType_Reshape, schema::PrimitiveType_PowFusion, - schema::PrimitiveType_DivFusion, schema::PrimitiveType_MatMul, schema::PrimitiveType_ScaleFusion, - schema::PrimitiveType_MulFusion, schema::PrimitiveType_StridedSlice, schema::PrimitiveType_PadFusion}; + schema::PrimitiveType_AddFusion, schema::PrimitiveType_DivFusion, schema::PrimitiveType_SubFusion, + schema::PrimitiveType_MatMul, schema::PrimitiveType_PowFusion, schema::PrimitiveType_Eltwise, + schema::PrimitiveType_ScaleFusion, schema::PrimitiveType_MulFusion, schema::PrimitiveType_StridedSlice, + schema::PrimitiveType_PadFusion}; } ~TensorRTSubGraph() override; diff --git a/mindspore/lite/src/delegate/tensorrt/tensorrt_utils.cc b/mindspore/lite/src/delegate/tensorrt/tensorrt_utils.cc index 230c35c829d..52ea5952adb 100644 --- a/mindspore/lite/src/delegate/tensorrt/tensorrt_utils.cc +++ b/mindspore/lite/src/delegate/tensorrt/tensorrt_utils.cc @@ -108,7 +108,7 @@ nvinfer1::ITensor *ConvertConstantTensor(nvinfer1::INetworkDefinition *network, return constant_tensor->getOutput(0); } -nvinfer1::ITensor *ConvertScalarToITensor(nvinfer1::INetworkDefinition *network, size_t shape_size, void *value) { +nvinfer1::ITensor *ConvertScalarToITensor(nvinfer1::INetworkDefinition *network, size_t shape_size, const void *value) { nvinfer1::Dims dims = ConvertCudaDims(1, shape_size); nvinfer1::Weights weights{nvinfer1::DataType::kFLOAT, value, 1}; nvinfer1::IConstantLayer *constant_tensor = network->addConstant(dims, weights); diff --git a/mindspore/lite/src/delegate/tensorrt/tensorrt_utils.h b/mindspore/lite/src/delegate/tensorrt/tensorrt_utils.h index aacaed8534a..ae0a583faee 100644 --- a/mindspore/lite/src/delegate/tensorrt/tensorrt_utils.h +++ b/mindspore/lite/src/delegate/tensorrt/tensorrt_utils.h @@ -51,7 +51,7 @@ nvinfer1::ITensor *ConvertConstantTensor(nvinfer1::INetworkDefinition *network, nvinfer1::ITensor *ConvertTensorWithExpandDims(nvinfer1::INetworkDefinition *network, const mindspore::MSTensor &ms_tensor, size_t expand_shape_size); -nvinfer1::ITensor *ConvertScalarToITensor(nvinfer1::INetworkDefinition *network, size_t shape_size, void *value); +nvinfer1::ITensor *ConvertScalarToITensor(nvinfer1::INetworkDefinition *network, size_t shape_size, const void *value); nvinfer1::Weights TransposeWeight(const mindspore::MSTensor &ms_tensor, float **pack_weight);