From 733f76506330ae5cfe4c4b5b40d51c2186e721e9 Mon Sep 17 00:00:00 2001 From: buxue Date: Tue, 20 Jul 2021 16:20:00 +0800 Subject: [PATCH] add security compilation options --- build.sh | 44 ++++++++++++++++++- cmake/options.cmake | 5 +++ .../ccsrc/backend/session/cpu_session.cc | 2 + mindspore/ccsrc/debug/CMakeLists.txt | 21 ++++++--- .../runtime/device/cpu/cpu_device_address.cc | 2 + .../runtime/device/cpu/cpu_kernel_runtime.cc | 10 ++++- .../hardware/cpu/cpu_device_context.cc | 2 + mindspore/ccsrc/utils/security_py.cc | 35 +++++++++++++++ mindspore/ccsrc/vm/backend.cc | 2 + 9 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 mindspore/ccsrc/utils/security_py.cc diff --git a/build.sh b/build.sh index 5193516fac2..17dc8f19629 100755 --- a/build.sh +++ b/build.sh @@ -46,6 +46,7 @@ usage() echo " -i Enable increment building, default off" echo " -j[n] Set the threads when building (Default: -j8)" echo " -e Use cpu, gpu or ascend" + echo " -s Enable security, default off" echo " -P Enable dump anf graph to file in ProtoBuffer format, default on" echo " -D Enable dumping of function graph ir, default on" echo " -z Compile dataset & mindrecord, default on" @@ -83,6 +84,7 @@ checkopts() THREAD_NUM=8 DEBUG_MODE="off" VERBOSE="" + ENABLE_SECURITY="off" ENABLE_COVERAGE="off" RUN_TESTCASES="off" RUN_CPP_ST_TESTS="off" @@ -118,8 +120,10 @@ checkopts() DEVICE="" ENABLE_HIDDEN="on" TENSORRT_HOME="" + USER_ENABLE_DUMP_IR=false + USER_ENABLE_DEBUGGER=false # Process the options - while getopts 'drvj:c:t:hsb:a:g:p:ie:m:l:I:RP:D:zM:V:K:B:En:A:S:k:W:H:L:' opt + while getopts 'drvj:c:t:hb:s:a:g:p:ie:m:l:I:RP:D:zM:V:K:B:En:A:S:k:W:H:L:' opt do CASE_SENSIVE_ARG=${OPTARG} OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') @@ -205,6 +209,25 @@ checkopts() fi TRAIN_MODE=$(echo "$OPTARG" | tr '[a-z]' '[A-Z]') ;; + s) + check_on_off $OPTARG s + if [[ "X$OPTARG" == "Xon" ]]; then + if [[ $USER_ENABLE_DUMP_IR == true ]]; then + echo "enable security, the dump ir is not available" + usage + exit 1 + fi + if [[ $USER_ENABLE_DEBUGGER == true ]]; then + echo "enable security, the debugger is not available" + usage + exit 1 + fi + ENABLE_DUMP_IR="off" + ENABLE_DEBUGGER="off" + fi + ENABLE_SECURITY="$OPTARG" + echo "enable security" + ;; R) ENABLE_TIMELINE="on" echo "enable time_line record" @@ -236,6 +259,14 @@ checkopts() ;; D) check_on_off $OPTARG D + if [[ "X$OPTARG" == "Xon" ]]; then + if [[ "X$ENABLE_SECURITY" == "Xon" ]]; then + echo "enable security, the dump ir is not available" + usage + exit 1 + fi + USER_ENABLE_DUMP_IR=true + fi ENABLE_DUMP_IR="$OPTARG" echo "enable dump function graph ir" ;; @@ -271,6 +302,14 @@ checkopts() ;; B) check_on_off $OPTARG B + if [[ "X$OPTARG" == "Xon" ]]; then + if [[ "X$ENABLE_SECURITY" == "Xon" ]]; then + echo "enable security, the debugger is not available" + usage + exit 1 + fi + USER_ENABLE_DEBUGGER=true + fi ENABLE_DEBUGGER="$OPTARG" ;; E) @@ -409,6 +448,9 @@ build_mindspore() if [[ "X$ENABLE_PROFILE" = "Xon" ]]; then CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_PROFILE=ON" fi + if [[ "X$ENABLE_SECURITY" = "Xon" ]]; then + CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_SECURITY=ON" + fi if [[ "X$ENABLE_TIMELINE" = "Xon" ]]; then CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_TIMELINE=ON" fi diff --git a/cmake/options.cmake b/cmake/options.cmake index 2a201475812..c4bd42b3223 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -11,6 +11,7 @@ option(ENABLE_ASAN "Enable Google Sanitizer to find memory bugs") option(ENABLE_LOAD_ANF_IR "Enable load ANF-IR as input of 'infer' stage of pipeline" OFF) option(ENABLE_COVERAGE "Enable code coverage report" OFF) option(USE_GLOG "Use glog to output log" OFF) +option(ENABLE_SECURITY "Enable security, maintenance function will be disabled, default off" OFF) option(ENABLE_PROFILE "Enable pipeline profile, default off" OFF) option(ENABLE_TIMELINE "Enable time line record" OFF) option(ENABLE_DUMP_PROTO "Enable dump anf graph to file in ProtoBuffer format, default on" ON) @@ -100,6 +101,10 @@ if(ENABLE_PROFILE) add_compile_definitions(ENABLE_PROFILE) endif() +if(ENABLE_SECURITY) + add_compile_definitions(ENABLE_SECURITY) +endif() + if(ENABLE_TIMELINE) add_compile_definitions(ENABLE_TIMELINE) endif() diff --git a/mindspore/ccsrc/backend/session/cpu_session.cc b/mindspore/ccsrc/backend/session/cpu_session.cc index 3ab9d41e929..3677c1a98da 100644 --- a/mindspore/ccsrc/backend/session/cpu_session.cc +++ b/mindspore/ccsrc/backend/session/cpu_session.cc @@ -46,10 +46,12 @@ namespace mindspore { namespace session { void CPUSession::Init(uint32_t device_id) { +#ifndef ENABLE_SECURITY // Dump json config file if dump is enabled auto &json_parser = DumpJsonParser::GetInstance(); json_parser.Parse(); json_parser.CopyMSCfgJsonToDir(rank_id_); +#endif InitExecutor(kCPUDevice, device_id); } diff --git a/mindspore/ccsrc/debug/CMakeLists.txt b/mindspore/ccsrc/debug/CMakeLists.txt index 5a18012fcbd..091c0b4041b 100644 --- a/mindspore/ccsrc/debug/CMakeLists.txt +++ b/mindspore/ccsrc/debug/CMakeLists.txt @@ -9,10 +9,6 @@ set(_DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/trace.cc" "${CMAKE_CURRENT_SOURCE_DIR}/common.cc" "${CMAKE_CURRENT_SOURCE_DIR}/env_config_parser.cc" - "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_json_parser.cc" - "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/cpu_e2e_dump.cc" - "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_utils.cc" - "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/npy_header.cc" ) set(_OFFLINE_SRC_LIST @@ -45,9 +41,20 @@ if(ENABLE_DEBUGGER) "${CMAKE_CURRENT_SOURCE_DIR}/debugger/debugger_utils.cc" ) endif() -if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows") - list(APPEND _DEBUG_SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/common.cc") - list(APPEND _DEBUG_SRC_LIST "data_dump/e2e_dump.cc") + +if(NOT ENABLE_SECURITY) + list(APPEND _DEBUG_SRC_LIST + "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/cpu_e2e_dump.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_json_parser.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/dump_utils.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/npy_header.cc" + ) + if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows") + list(APPEND _DEBUG_SRC_LIST + "${CMAKE_CURRENT_SOURCE_DIR}/common.cc" + "${CMAKE_CURRENT_SOURCE_DIR}/data_dump/e2e_dump.cc" + ) + endif() endif() set_property(SOURCE ${_DEBUG_SRC_LIST} ${_RDR_SRC_LIST} PROPERTY COMPILE_DEFINITIONS diff --git a/mindspore/ccsrc/runtime/device/cpu/cpu_device_address.cc b/mindspore/ccsrc/runtime/device/cpu/cpu_device_address.cc index a2ed31d0e1e..da771370f9b 100644 --- a/mindspore/ccsrc/runtime/device/cpu/cpu_device_address.cc +++ b/mindspore/ccsrc/runtime/device/cpu/cpu_device_address.cc @@ -37,6 +37,7 @@ CPUDeviceAddress::~CPUDeviceAddress() { bool CPUDeviceAddress::DumpMemToFile(const std::string &filepath, const std::string &, const ShapeVector &host_shape, TypeId host_type, bool) const { bool ret = false; +#ifndef ENABLE_SECURITY if (filepath.empty()) { MS_LOG(ERROR) << "Dump file path is null!"; return ret; @@ -44,6 +45,7 @@ bool CPUDeviceAddress::DumpMemToFile(const std::string &filepath, const std::str std::string path = filepath + '.' + format_; MS_LOG(DEBUG) << "E2E Dump path is " << path; ret = DumpJsonParser::DumpToFile(path, ptr_, size_, host_shape, host_type); +#endif return ret; } diff --git a/mindspore/ccsrc/runtime/device/cpu/cpu_kernel_runtime.cc b/mindspore/ccsrc/runtime/device/cpu/cpu_kernel_runtime.cc index f313d20c412..ac9fb079b5a 100644 --- a/mindspore/ccsrc/runtime/device/cpu/cpu_kernel_runtime.cc +++ b/mindspore/ccsrc/runtime/device/cpu/cpu_kernel_runtime.cc @@ -405,9 +405,11 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) { auto profiler_inst = profiler::cpu::CPUProfiler::GetInstance(); MS_EXCEPTION_IF_NULL(profiler_inst); +#ifndef ENABLE_SECURITY auto &dump_json_parser = DumpJsonParser::GetInstance(); bool iter_dump_flag = dump_json_parser.GetIterDumpFlag(); uint32_t graph_id = kernel_graph->graph_id(); +#endif for (const auto &kernel : kernels) { #ifdef ENABLE_PROFILE @@ -448,9 +450,11 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) { } catch (std::exception &e) { MS_LOG(EXCEPTION) << e.what() << "\nTrace:" << trace::DumpSourceLines(kernel); } +#ifndef ENABLE_SECURITY if (iter_dump_flag) { CPUE2eDump::DumpCNodeData(kernel, graph_id); } +#endif if (profiler_inst->GetEnableFlag()) { profiler_inst->OpDataProducerEnd(); } @@ -466,10 +470,14 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) { MS_LOG(INFO) << "cpu kernel: " << kernel->fullname_with_scope() << " costs " << cost_time * 1e6 << " us"; #endif } +#ifndef ENABLE_SECURITY if (iter_dump_flag) { CPUE2eDump::DumpParametersAndConst(kernel_graph, graph_id); } - dump_json_parser.UpdateDumpIter(); + if (graph_id == 0) { + dump_json_parser.UpdateDumpIter(); + } +#endif return true; } } // namespace cpu diff --git a/mindspore/ccsrc/runtime/hardware/cpu/cpu_device_context.cc b/mindspore/ccsrc/runtime/hardware/cpu/cpu_device_context.cc index 511ef13844f..813f8b2d34c 100644 --- a/mindspore/ccsrc/runtime/hardware/cpu/cpu_device_context.cc +++ b/mindspore/ccsrc/runtime/hardware/cpu/cpu_device_context.cc @@ -45,12 +45,14 @@ bool CPUDeviceContext::Initialize() { mem_manager_ = std::make_shared(); MS_EXCEPTION_IF_NULL(mem_manager_); +#ifndef ENABLE_SECURITY // Dump json config file if dump is enabled. auto rank_id = GetRankID(); auto &json_parser = DumpJsonParser::GetInstance(); json_parser.Parse(); json_parser.CopyJsonToDir(rank_id); json_parser.CopyMSCfgJsonToDir(rank_id); +#endif initialized_ = true; return true; diff --git a/mindspore/ccsrc/utils/security_py.cc b/mindspore/ccsrc/utils/security_py.cc new file mode 100644 index 00000000000..9be39768322 --- /dev/null +++ b/mindspore/ccsrc/utils/security_py.cc @@ -0,0 +1,35 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils/utils.h" +#include "pybind_api/api_register.h" + +namespace mindspore { +// Get whether security is enable +bool EnableSecurity() { +#ifdef ENABLE_SECURITY + return true; +#else + return false; +#endif +} + +// Define python wrapper to judge security enable. +REGISTER_PYBIND_DEFINE(security, ([](py::module *const m) { + auto m_sub = m->def_submodule("security", "submodule for security"); + (void)m_sub.def("enable_security", &EnableSecurity, "enable security"); + })); +} // namespace mindspore diff --git a/mindspore/ccsrc/vm/backend.cc b/mindspore/ccsrc/vm/backend.cc index 30bb60899a4..a45e96069b8 100644 --- a/mindspore/ccsrc/vm/backend.cc +++ b/mindspore/ccsrc/vm/backend.cc @@ -676,6 +676,7 @@ void PrepareForDebuggr(const GraphCompilerInfo &graph_compiler_info) { } #endif +#ifndef ENABLE_SECURITY if (DumpJsonParser::GetInstance().e2e_dump_enabled()) { DumpJsonParser::GetInstance().ClearGraph(); for (size_t i = 0; i < graph_compiler_info.graphs_.size(); ++i) { @@ -684,6 +685,7 @@ void PrepareForDebuggr(const GraphCompilerInfo &graph_compiler_info) { } } } +#endif } } // namespace