ssk015
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!20327 [MSLITE] add support fp16 tag for tensorrt 

Merge pull request !20327 from Liu_Xuu/trt_0714_fp16
This commit is contained in:
i-robot 2021-07-19 02:47:26 +00:00 committed by Gitee
parent 2c94b631a7 82191be350
commit 9c3e4a408b
5 changed files with 87 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
mindspore/lite/src/delegate/tensorrt/tensorrt_delegate.cc
View File

@ -49,6 +49,20 @@ int TensorRTDelegate::Init() {
if (!IsHardwareSupport()) {
return RET_NOT_SUPPORT;
}
std::vector<std::shared_ptr<DeviceInfoContext>> device_list = context_->MutableDeviceInfo();
auto iter = std::find_if(device_list.begin(), device_list.end(), [](std::shared_ptr<DeviceInfoContext> device) {
return device->GetDeviceType() == DeviceType::kGPU;
});
if (iter == device_list.end()) {
MS_LOG(ERROR) << "no gpu device info found for TensorRT.";
return RET_ERROR;
}
auto gpu_info = (*iter)->Cast<GPUDeviceInfo>();
if (gpu_info == nullptr) {
MS_LOG(ERROR) << "no gpu device info found for TensorRT.";
return RET_ERROR;
}
device_info_ = gpu_info;
op_func_lists_.clear();
op_func_lists_ = {
{schema::PrimitiveType_Activation, GetTensorRTOp<ActivationTensorRT>},
@ -132,7 +146,7 @@ TensorRTSubGraph *TensorRTDelegate::CreateTensorRTGraph(const std::vector<Tensor
KernelIter from, KernelIter end) {
auto in_tensors = GraphInTensors<TensorRTOp>(ops, model, from, end);
auto out_tensors = GraphOutTensors<TensorRTOp>(ops, model, from, end);
auto *tensorrt_graph = new (std::nothrow) TensorRTSubGraph(ops, in_tensors, out_tensors);
auto *tensorrt_graph = new (std::nothrow) TensorRTSubGraph(ops, in_tensors, out_tensors, context_, device_info_);
if (tensorrt_graph == nullptr) {
MS_LOG(ERROR) << "new tensorrt_graph failed.";
return nullptr;

8
mindspore/lite/src/delegate/tensorrt/tensorrt_delegate.h
View File

@ -18,11 +18,13 @@
#include <string>
#include <vector>
#include <map>
#include <memory>
#include "include/api/delegate.h"
#include "src/delegate/tensorrt/tensorrt_subgraph.h"
#include "include/api/kernel.h"
#include "include/errorcode.h"
#include "src/common/log_adapter.h"
#include "include/api/context.h"
namespace mindspore::lite {
typedef TensorRTOp *(*TensorRTGetOp)(const schema::Primitive *primitive,
@ -31,7 +33,7 @@ typedef TensorRTOp *(*TensorRTGetOp)(const schema::Primitive *primitive,
class TensorRTDelegate : public Delegate {
public:
TensorRTDelegate() = default;
explicit TensorRTDelegate(mindspore::Context *context) : context_(context) {}
~TensorRTDelegate() override = default;
@ -46,6 +48,10 @@ class TensorRTDelegate : public Delegate {
KernelIter end);
std::map<schema::PrimitiveType, TensorRTGetOp> op_func_lists_;
mindspore::Context *context_;
std::shared_ptr<GPUDeviceInfo> device_info_{nullptr};
};
} // namespace mindspore::lite
#endif // MINDSPORE_LITE_SRC_RUNTIME_DELEGATE_TENSORRT_DELEGATE_

56
mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.cc
View File

@ -15,6 +15,7 @@
*/
#include "src/delegate/tensorrt/tensorrt_subgraph.h"
#include <cuda_runtime_api.h>
#include <string>
#include <vector>
#include <set>
@ -30,9 +31,9 @@ TensorRTSubGraph::~TensorRTSubGraph() {
config_->destroy();
config_ = nullptr;
}
if (context_ != nullptr) {
context_->destroy();
context_ = nullptr;
if (trt_context_ != nullptr) {
trt_context_->destroy();
trt_context_ = nullptr;
}
if (engine_ != nullptr) {
engine_->destroy();
@ -74,9 +75,6 @@ int TensorRTSubGraph::BuildEngine() {
MS_LOG(ERROR) << "create builder config failed.";
return RET_ERROR;
}
// config setup
// setMaxWorkspaceSize to x MB
this->config_->setMaxWorkspaceSize(16 * (1 << 20));
// print all network ops
MS_LOG(INFO) << "build engine for tensorrt network: " << this->network_->getName();
for (int i = 0; i < this->network_->getNbLayers(); i++) {
@ -84,6 +82,9 @@ int TensorRTSubGraph::BuildEngine() {
}
MS_LOG(DEBUG) << "end of tensorrt network: " << this->network_->getName();
if (SetDeviceConfig() != RET_OK) {
MS_LOG(WARNING) << "set tensorrt config failed.";
}
engine_ = runtime_->GetBuilder()->buildEngineWithConfig(*this->network_, *this->config_);
if (engine_ == nullptr) {
MS_LOG(ERROR) << "Create engine failed in TensorRT network";
@ -92,6 +93,43 @@ int TensorRTSubGraph::BuildEngine() {
return RET_OK;
}
int TensorRTSubGraph::SetDeviceConfig() {
// set fp16
if (device_info_->GetEnableFP16() && SupportFP16()) {
config_->setFlag(nvinfer1::BuilderFlag::kFP16);
}
// config setMaxWorkspaceSize to x MB
config_->setMaxWorkspaceSize(256 * (1 << 20));
return RET_OK;
}
bool TensorRTSubGraph::SupportFP16() {
int deviceCnt = 0;
cudaError ret = cudaGetDeviceCount(&deviceCnt);
if (ret != cudaSuccess) {
MS_LOG(ERROR) << "cudaGetDeviceCount failed.";
return false;
}
std::vector<std::string> supportFP16_versions{"5.3", "6.0", "6.2", "7.0", "7.2", "7.5", "8.0", "8.6"};
cudaDeviceProp prop;
std::string version;
for (int dev = 0; dev < deviceCnt; dev++) {
ret = cudaGetDeviceProperties(&prop, dev);
if (ret != cudaSuccess) {
MS_LOG(ERROR) << "cuDeviceGetAttribute failed.";
return false;
}
version = std::to_string(prop.major) + "." + std::to_string(prop.minor);
if (std::find(supportFP16_versions.begin(), supportFP16_versions.end(), version) != supportFP16_versions.end()) {
MS_LOG(INFO) << "cuda device version is: " << version << ", support FP16, set enable FP16 tag successful";
return true;
}
}
MS_LOG(WARNING) << "cuda device version is: " << version << ", don't support FP16, set enable FP16 tag failed";
return false;
}
int TensorRTSubGraph::BuildTensorRTGraph() {
MS_ASSERT(!all_ops_.empty());
// Connect NetWork.
@ -163,8 +201,8 @@ int TensorRTSubGraph::Prepare() {
MS_LOG(ERROR) << "engine_ is null in this builder_";
return RET_ERROR;
}
this->context_ = this->engine_->createExecutionContext();
if (this->context_ == nullptr) {
this->trt_context_ = this->engine_->createExecutionContext();
if (this->trt_context_ == nullptr) {
MS_LOG(ERROR) << "TensorRTSubGraph create context failed.";
return RET_ERROR;
}
@ -196,7 +234,7 @@ int TensorRTSubGraph::Execute() {
for (size_t i = 0; i < inputs_.size(); i++) {
runtime_->GetAllocator()->SyncMemInHostAndDevice(inputs_[i], trt_in_tensor_name_[i], true);
}
auto ret = this->context_->executeV2(tensor_bindings_);
auto ret = this->trt_context_->executeV2(tensor_bindings_);
if (!ret) {
MS_LOG(ERROR) << "TensorRT execute failed.";
return RET_ERROR;

14
mindspore/lite/src/delegate/tensorrt/tensorrt_subgraph.h
View File

@ -19,9 +19,11 @@
#include <set>
#include <string>
#include <vector>
#include <memory>
#include "include/api/kernel.h"
#include "src/delegate/tensorrt/tensorrt_runtime.h"
#include "src/delegate/tensorrt/tensorrt_utils.h"
#include "include/api/context.h"
namespace mindspore::lite {
using mindspore::lite::RET_ERROR;
@ -29,8 +31,9 @@ using mindspore::lite::RET_OK;
class TensorRTSubGraph : public kernel::Kernel {
public:
TensorRTSubGraph(std::vector<TensorRTOp *> ops, const std::vector<mindspore::MSTensor> &inputs,
const std::vector<mindspore::MSTensor> &outputs)
: kernel::Kernel(inputs, outputs, nullptr, nullptr), all_ops_(std::move(ops)) {
const std::vector<mindspore::MSTensor> &outputs, const mindspore::Context *ctx,
std::shared_ptr<GPUDeviceInfo> device_info)
: kernel::Kernel(inputs, outputs, nullptr, ctx), all_ops_(std::move(ops)), device_info_(device_info) {
trt_specific_weight_nodes_ = {
schema::PrimitiveType_Conv2DFusion, schema::PrimitiveType_ReduceFusion, schema::PrimitiveType_Transpose,
schema::PrimitiveType_Gather, schema::PrimitiveType_Reshape, schema::PrimitiveType_PowFusion,
@ -55,6 +58,10 @@ class TensorRTSubGraph : public kernel::Kernel {
private:
int BuildEngine();
int SetDeviceConfig();
bool SupportFP16();
static nvinfer1::ITensor *FindTensorRTInputs(TensorRTOp *cur_op, mindspore::MSTensor in_tensor);
TensorRTRuntime *runtime_{nullptr};
@ -66,6 +73,7 @@ class TensorRTSubGraph : public kernel::Kernel {
std::vector<TensorRTOp *> out_ops_{};
void **tensor_bindings_{nullptr};
std::shared_ptr<GPUDeviceInfo> device_info_{nullptr};
std::set<mindspore::schema::PrimitiveType> trt_specific_weight_nodes_;
@ -76,7 +84,7 @@ class TensorRTSubGraph : public kernel::Kernel {
nvinfer1::INetworkDefinition *network_{nullptr};
nvinfer1::IBuilderConfig *config_{nullptr};
nvinfer1::ICudaEngine *engine_{nullptr};
nvinfer1::IExecutionContext *context_{nullptr};
nvinfer1::IExecutionContext *trt_context_{nullptr};
};
} // namespace mindspore::lite
#endif // MINDSPORE_LITE_SRC_RUNTIME_DELEGATE_TENSORRT_SUB_GTAPH_

14
mindspore/lite/src/lite_session.cc
View File

@ -653,6 +653,12 @@ int LiteSession::Init(const Context *context) {
if (context->delegate != nullptr) {
delegate_ = context->delegate;
}
ms_context_ = MSContextFromContext(context);
if (ms_context_ == nullptr) {
MS_LOG(ERROR) << "transfer context to ms context failed.";
is_running_.store(false);
return RET_NULL_PTR;
}
#if SUPPORT_NPU
if (delegate_ == nullptr && context_->IsNpuEnabled()) {
delegate_ = std::shared_ptr<NPUDelegate>(new (std::nothrow) NPUDelegate(context_->GetNpuInfo()));
@ -664,7 +670,7 @@ int LiteSession::Init(const Context *context) {
#endif
#if GPU_TENSORRT
if (delegate_ == nullptr && context_->IsGpuEnabled()) {
delegate_ = std::shared_ptr<TensorRTDelegate>(new (std::nothrow) TensorRTDelegate());
delegate_ = std::shared_ptr<TensorRTDelegate>(new (std::nothrow) TensorRTDelegate(ms_context_));
if (delegate_ == nullptr) {
MS_LOG(ERROR) << "New tensorrt delegate_ failed";
return RET_ERROR;
@ -694,12 +700,6 @@ int LiteSession::Init(const Context *context) {
is_running_.store(false);
return ret;
}
ms_context_ = MSContextFromContext(context);
if (ms_context_ == nullptr) {
MS_LOG(ERROR) << "transfer context to ms context failed.";
is_running_.store(false);
return RET_NULL_PTR;
}
is_running_.store(false);
return RET_OK;
}