From 177f3f75bff98bef7dbe652ea46f8f73583eaa22 Mon Sep 17 00:00:00 2001 From: yanghaitao1 Date: Mon, 13 Sep 2021 09:36:20 -0400 Subject: [PATCH] remove profiler if compiled with -s on --- mindspore/ccsrc/CMakeLists.txt | 14 ++++--- .../gpu/data/dataset_iterator_kernel.cc | 2 + .../ccsrc/backend/optimizer/somas/somas.cc | 6 ++- .../ccsrc/backend/optimizer/somas/somas.h | 2 + .../ccsrc/backend/session/ascend_session.cc | 2 +- mindspore/ccsrc/profiler/CMakeLists.txt | 42 ++++++++++--------- .../device/ascend/ascend_memory_manager.cc | 8 +++- .../runtime/device/cpu/cpu_kernel_runtime.cc | 8 ++-- .../runtime/device/gpu/gpu_kernel_runtime.cc | 8 +++- .../profiler/parser/memory_usage_parser.py | 40 +++++++++++------- mindspore/profiler/profiling.py | 5 +++ tests/st/profiler/test_ascend_profiler.py | 1 + tests/st/profiler/test_profiler.py | 3 ++ tests/ut/cpp/CMakeLists.txt | 5 +++ 14 files changed, 95 insertions(+), 51 deletions(-) diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index f4fe9d464e4..ecf8913a9bd 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -153,12 +153,14 @@ include_directories("${CMAKE_BINARY_DIR}/ps/core") file(GLOB_RECURSE COMM_PROTO_IN RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "ps/core/protos/*.proto") ms_protobuf_generate(COMM_PROTO_SRCS COMM_PROTO_HDRS ${COMM_PROTO_IN}) list(APPEND MINDSPORE_PROTO_LIST ${COMM_PROTO_SRCS}) - -include_directories("${CMAKE_BINARY_DIR}/profiler/device/ascend") -file(GLOB_RECURSE PROFILER_PROTO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} - "profiler/device/ascend/memory_profiling.proto") -ms_protobuf_generate_py(PROFILER_MEM_PROTO_PY PROFILER_MEM_PROTO_HDRS_PY PROFILER_MEM_PROTO_PYS ${PROFILER_PROTO_LIST}) -list(APPEND MINDSPORE_PROTO_LIST ${PROFILER_MEM_PROTO_PY}) +if(NOT ENABLE_SECURITY) + include_directories("${CMAKE_BINARY_DIR}/profiler/device/ascend") + file(GLOB_RECURSE PROFILER_PROTO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "profiler/device/ascend/memory_profiling.proto") + ms_protobuf_generate_py(PROFILER_MEM_PROTO_PY PROFILER_MEM_PROTO_HDRS_PY PROFILER_MEM_PROTO_PYS + ${PROFILER_PROTO_LIST}) + list(APPEND MINDSPORE_PROTO_LIST ${PROFILER_MEM_PROTO_PY}) +endif() if(ENABLE_DEBUGGER) # debugger: compile proto files diff --git a/mindspore/ccsrc/backend/kernel_compiler/gpu/data/dataset_iterator_kernel.cc b/mindspore/ccsrc/backend/kernel_compiler/gpu/data/dataset_iterator_kernel.cc index db72eafaa67..96ea1f6b8bb 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/gpu/data/dataset_iterator_kernel.cc +++ b/mindspore/ccsrc/backend/kernel_compiler/gpu/data/dataset_iterator_kernel.cc @@ -73,6 +73,7 @@ bool DatasetIteratorKernel::Init(const CNodePtr &kernel_node) { MS_LOG(EXCEPTION) << "Gpu Queue(" << queue_name_ << ") Open Failed"; } +#ifndef ENABLE_SECURITY auto profiler_inst = profiler::gpu::GPUProfiler::GetInstance(); MS_EXCEPTION_IF_NULL(profiler_inst); profiling_enable_ = profiler_inst->GetEnableFlag(); @@ -82,6 +83,7 @@ bool DatasetIteratorKernel::Init(const CNodePtr &kernel_node) { MS_EXCEPTION_IF_NULL(profiling_op_); profiler_inst->RegisterProfilingOp(profiling_op_); } +#endif return true; } diff --git a/mindspore/ccsrc/backend/optimizer/somas/somas.cc b/mindspore/ccsrc/backend/optimizer/somas/somas.cc index 98bcd1ed9ee..c334bc2caa9 100644 --- a/mindspore/ccsrc/backend/optimizer/somas/somas.cc +++ b/mindspore/ccsrc/backend/optimizer/somas/somas.cc @@ -37,12 +37,13 @@ #include "debug/rdr/running_data_recorder.h" #endif #include "common/thread_pool.h" +#ifndef ENABLE_SECURITY #include "profiler/device/ascend/memory_profiling.h" using mindspore::profiler::ascend::MemoryProfiling; using mindspore::profiler::ascend::NodeMemory; using mindspore::profiler::ascend::TensorMemory; - +#endif namespace mindspore { namespace somas { constexpr auto kGapSize = 512; @@ -1772,7 +1773,7 @@ uint8_t *Somas::GetNodeWorkSpacePtr(const AnfNodePtr &node, size_t index) const } return ptr; } - +#ifndef ENABLE_SECURITY void Somas::ConvertToProfilingNode(uint32_t graph_id) { #ifdef ENABLE_D auto graph_node = MemoryProfiling::GetInstance().GetGraphMemoryNode(graph_id); @@ -1810,5 +1811,6 @@ void Somas::ConvertToProfilingNode(uint32_t graph_id) { } #endif } +#endif } // namespace somas } // namespace mindspore diff --git a/mindspore/ccsrc/backend/optimizer/somas/somas.h b/mindspore/ccsrc/backend/optimizer/somas/somas.h index bcf4d63c2af..5c72510e020 100644 --- a/mindspore/ccsrc/backend/optimizer/somas/somas.h +++ b/mindspore/ccsrc/backend/optimizer/somas/somas.h @@ -55,7 +55,9 @@ class Somas { void DumpSomasMemoryIR(const string filename) const; static bool NodeSort(SomasNodePtr, SomasNodePtr); +#ifndef ENABLE_SECURITY void ConvertToProfilingNode(uint32_t graph_id); +#endif private: std::vector reuse_matrix_; diff --git a/mindspore/ccsrc/backend/session/ascend_session.cc b/mindspore/ccsrc/backend/session/ascend_session.cc index c2cb0033ed0..6406b00eec8 100644 --- a/mindspore/ccsrc/backend/session/ascend_session.cc +++ b/mindspore/ccsrc/backend/session/ascend_session.cc @@ -81,9 +81,9 @@ #include "ps/ps_cache/ps_cache_manager.h" #endif #include "runtime/device/ascend/ascend_bucket.h" +#ifndef ENABLE_SECURITY #include "profiler/device/ascend/memory_profiling.h" -#ifndef ENABLE_SECURITY using mindspore::device::ascend::ProfilingManager; using mindspore::profiler::ascend::MemoryProfiling; #endif diff --git a/mindspore/ccsrc/profiler/CMakeLists.txt b/mindspore/ccsrc/profiler/CMakeLists.txt index a9d314e8ae9..ec64b31d7bb 100644 --- a/mindspore/ccsrc/profiler/CMakeLists.txt +++ b/mindspore/ccsrc/profiler/CMakeLists.txt @@ -1,25 +1,27 @@ -if(ENABLE_GPU) - file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} - "device/gpu/*.cc" "device/cpu/*.cc") -endif() +if(NOT ENABLE_SECURITY) + if(ENABLE_GPU) + file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "device/gpu/*.cc" "device/cpu/*.cc") + endif() -if(ENABLE_D) - file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} - "device/cpu/*.cc" "device/ascend/*.cc") -endif() + if(ENABLE_D) + file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "device/cpu/*.cc" "device/ascend/*.cc") + endif() -if(ENABLE_CPU AND NOT (ENABLE_D OR ENABLE_GPU)) - file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} - "device/cpu/*.cc") -endif() + if(ENABLE_CPU AND NOT (ENABLE_D OR ENABLE_GPU)) + file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + "device/cpu/*.cc") + endif() -list(APPEND PROFILER_SRC_LIST ${CMAKE_CURRENT_SOURCE_DIR}/device/profiling.cc - ${CMAKE_CURRENT_SOURCE_DIR}/device/data_saver.cc) + list(APPEND PROFILER_SRC_LIST ${CMAKE_CURRENT_SOURCE_DIR}/device/profiling.cc + ${CMAKE_CURRENT_SOURCE_DIR}/device/data_saver.cc) -set_property(SOURCE ${PROFILER_SRC_LIST} PROPERTY COMPILE_DEFINITIONS - SUBMODULE_ID=mindspore::SubModuleId::SM_PROFILER) -add_library(_mindspore_profiler_obj OBJECT ${PROFILER_SRC_LIST}) + set_property(SOURCE ${PROFILER_SRC_LIST} PROPERTY COMPILE_DEFINITIONS + SUBMODULE_ID=mindspore::SubModuleId::SM_PROFILER) + add_library(_mindspore_profiler_obj OBJECT ${PROFILER_SRC_LIST}) -if(ENABLE_D) - add_dependencies(_mindspore_profiler_obj mindspore::protobuf) -endif() + if(ENABLE_D) + add_dependencies(_mindspore_profiler_obj mindspore::protobuf) + endif() +endif() \ No newline at end of file diff --git a/mindspore/ccsrc/runtime/device/ascend/ascend_memory_manager.cc b/mindspore/ccsrc/runtime/device/ascend/ascend_memory_manager.cc index aca15006313..79cac3326ab 100644 --- a/mindspore/ccsrc/runtime/device/ascend/ascend_memory_manager.cc +++ b/mindspore/ccsrc/runtime/device/ascend/ascend_memory_manager.cc @@ -18,11 +18,13 @@ #include "runtime/device/ascend/ascend_memory_pool.h" #include "utils/ms_context.h" #include "runtime/mem.h" +#ifndef ENABLE_SECURITY #include "runtime/device/ascend/profiling/profiling_manager.h" #include "profiler/device/ascend/memory_profiling.h" using mindspore::device::ascend::ProfilingManager; using mindspore::profiler::ascend::MemoryProfiling; +#endif namespace mindspore { namespace device { @@ -139,7 +141,7 @@ uint8_t *AscendMemoryManager::MallocStaticMem(size_t size, bool communication_me << "], Pool statistics: pool total size [" << AscendMemoryPool::GetInstance().total_mem_statistics() << "] used [" << AscendMemoryPool::GetInstance().used_mem_statistics() << "] communication_mem:" << communication_mem; - +#ifndef ENABLE_SECURITY if (MemoryProfiling::GetInstance().IsMemoryProfilingEnable() && graph_id != kInvalidGraphId) { auto node = MemoryProfiling::GetInstance().GetGraphMemoryNode(graph_id); if (node == nullptr) { @@ -149,7 +151,7 @@ uint8_t *AscendMemoryManager::MallocStaticMem(size_t size, bool communication_me node->AddStaticMemorySize(SizeToUint(align_size)); } - +#endif if (communication_mem) { // create protect area [kMemAlignSize -- data -- kMemAlignSize] uint8_t *alloc_address = reinterpret_cast(AscendMemoryPool::GetInstance().AllocTensorMem(align_size)); @@ -191,9 +193,11 @@ uint8_t *AscendMemoryManager::MallocDynamicMem(size_t size, bool communication_m void AscendMemoryManager::MallocSomasDynamicMem(const session::KernelGraph *graph) { MemoryManager::MallocSomasDynamicMem(graph); +#ifndef ENABLE_SECURITY if (MemoryProfiling::GetInstance().IsMemoryProfilingEnable()) { somas_reuse_util_ptr_->ConvertToProfilingNode(graph->graph_id()); } +#endif } // communication memory: [512align_size + data + 512align_size] diff --git a/mindspore/ccsrc/runtime/device/cpu/cpu_kernel_runtime.cc b/mindspore/ccsrc/runtime/device/cpu/cpu_kernel_runtime.cc index abb066b2c2e..5d3a00c2614 100644 --- a/mindspore/ccsrc/runtime/device/cpu/cpu_kernel_runtime.cc +++ b/mindspore/ccsrc/runtime/device/cpu/cpu_kernel_runtime.cc @@ -410,8 +410,6 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) { static_cast(mem_manager_.get())->IncreaseAddressRefCount(kernel_graph); auto kernels = kernel_graph->execution_order(); - auto profiler_inst = profiler::cpu::CPUProfiler::GetInstance(); - MS_EXCEPTION_IF_NULL(profiler_inst); #ifndef ENABLE_SECURITY auto &dump_json_parser = DumpJsonParser::GetInstance(); @@ -452,10 +450,14 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) { AddRuntimeAddress(device_address, &kernel_workspaces); } bool ret = true; +#ifndef ENABLE_SECURITY + auto profiler_inst = profiler::cpu::CPUProfiler::GetInstance(); + MS_EXCEPTION_IF_NULL(profiler_inst); if (profiler_inst->GetEnableFlag()) { uint32_t pid = getpid(); profiler_inst->OpDataProducerBegin(kernel->fullname_with_scope(), pid); } +#endif #ifdef ENABLE_DUMP_IR kernel::KernelLaunchInfo mem_info = {kernel_inputs, kernel_workspaces, kernel_outputs}; std::string op_name = kernel->fullname_with_scope(); @@ -470,10 +472,10 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) { if (iter_dump_flag) { CPUE2eDump::DumpCNodeData(kernel, graph_id); } -#endif if (profiler_inst->GetEnableFlag()) { profiler_inst->OpDataProducerEnd(); } +#endif if (!ret) { #ifdef ENABLE_DUMP_IR mindspore::RDR::TriggerAll(); diff --git a/mindspore/ccsrc/runtime/device/gpu/gpu_kernel_runtime.cc b/mindspore/ccsrc/runtime/device/gpu/gpu_kernel_runtime.cc index 9a0e8b5149c..1461d6b9d91 100644 --- a/mindspore/ccsrc/runtime/device/gpu/gpu_kernel_runtime.cc +++ b/mindspore/ccsrc/runtime/device/gpu/gpu_kernel_runtime.cc @@ -826,7 +826,7 @@ void GPUKernelRuntime::LaunchKernelWithoutMock(const session::KernelGraph *graph const AddressPtrList &outputs, bool profiling) { MS_EXCEPTION_IF_NULL(graph); MS_EXCEPTION_IF_NULL(kernel); - +#ifndef ENABLE_SECURITY auto profiler_inst = profiler::gpu::GPUProfiler::GetInstance(); MS_EXCEPTION_IF_NULL(profiler_inst); @@ -835,11 +835,13 @@ void GPUKernelRuntime::LaunchKernelWithoutMock(const session::KernelGraph *graph profiler::gpu::ProfilingUtils::GetProfilingTraceFromEnv(NOT_NULL(graph)); profiler_inst->SetStepTraceOpName(profiling_trace); } - +#endif if (!profiling) { +#ifndef ENABLE_SECURITY if (profiler_inst->GetEnableFlag()) { profiler_inst->OpDataProducerBegin(kernel->fullname_with_scope(), stream_); } +#endif auto kernel_mod = AnfAlgo::GetKernelMod(kernel); MS_EXCEPTION_IF_NULL(kernel_mod); if (!kernel_mod->Launch(inputs, workspaces, outputs, stream_)) { @@ -848,12 +850,14 @@ void GPUKernelRuntime::LaunchKernelWithoutMock(const session::KernelGraph *graph #endif MS_LOG(EXCEPTION) << "Launch kernel failed: " << kernel->fullname_with_scope(); } +#ifndef ENABLE_SECURITY if (profiler_inst->GetEnableFlag()) { profiler_inst->OpDataProducerEnd(); if (profiler_inst->GetSyncEnableFlag()) { CHECK_OP_RET_WITH_ERROR(SyncStream(), "Profiler SyncStream failed."); } } +#endif } else { LaunchKernelWithTimeProfiling(kernel, inputs, workspaces, outputs); } diff --git a/mindspore/profiler/parser/memory_usage_parser.py b/mindspore/profiler/parser/memory_usage_parser.py index 5b003eed1dc..a77f0baa439 100644 --- a/mindspore/profiler/parser/memory_usage_parser.py +++ b/mindspore/profiler/parser/memory_usage_parser.py @@ -17,6 +17,7 @@ from collections import OrderedDict import json import os import stat +from importlib import import_module from google.protobuf.text_format import ParseError @@ -27,7 +28,12 @@ from mindspore.profiler.common.validator.validate_path import validate_and_norma from mindspore.profiler.parser.container import MemoryGraph as Graph from mindspore.profiler.parser.container import MemoryNode as Node from mindspore.profiler.parser.container import MemoryTensor as Tensor -from mindspore.train.memory_profiling_pb2 import MemoryProto +import mindspore._c_expression as c_expression + +try: + MemoryProto = import_module("mindspore.train.memory_profiling_pb2.MemoryProto") +except ModuleNotFoundError: + MemoryProto = None GIGABYTES = 1024 * 1024 * 1024 @@ -84,22 +90,26 @@ class MemoryUsageParser: raise ProfilerIOException # Parse memory raw data from file. - memory_proto = MemoryProto() - try: - memory_proto.ParseFromString(content) - except ParseError as err: - msg = "Fail to parse memory proto file." - logger.error("Cannot parse the memory file. Please check the file schema.\n%s", err) - raise ProfilerRawFileException(msg) + if not c_expression.security.enable_security(): + if not MemoryProto: + raise ProfilerRawFileException("Can not find memory profiling pb file.") - # Parse memory details based on graphs in the network. - graphs = memory_proto.graph_mem - self._parse_graph_memory(graphs) - # Update memory summary information. - self._mem_summary['capacity'] = memory_proto.total_mem / GIGABYTES - self._mem_summary['peak_mem'] = self._peak_mem + memory_proto = MemoryProto() + try: + memory_proto.ParseFromString(content) + except ParseError as err: + msg = "Fail to parse memory proto file." + logger.error("Cannot parse the memory file. Please check the file schema.\n%s", err) + raise ProfilerRawFileException(msg) - logger.info('Finished processing memory usage data.') + # Parse memory details based on graphs in the network. + graphs = memory_proto.graph_mem + self._parse_graph_memory(graphs) + # Update memory summary information. + self._mem_summary['capacity'] = memory_proto.total_mem / GIGABYTES + self._mem_summary['peak_mem'] = self._peak_mem + + logger.info('Finished processing memory usage data.') def _parse_graph_memory(self, graphs): """Parse memory usage based on subgraphs.""" diff --git a/mindspore/profiler/profiling.py b/mindspore/profiler/profiling.py index b068edcef13..9470ea0fb55 100644 --- a/mindspore/profiler/profiling.py +++ b/mindspore/profiler/profiling.py @@ -121,6 +121,9 @@ class Profiler: _aicpu_op_output_filename_target = "output_data_preprocess_aicpu_" def __init__(self, **kwargs): + if c_expression.security.enable_security(): + raise Runtime("Profiler is not supported if compiled with \'-s on\'") + # get device_id and device_target self._get_devid_rankid_and_devtarget() self._get_output_path(kwargs) @@ -225,6 +228,8 @@ class Profiler: """ Collect and analyse performance data, called after training or during training. The example shows above. """ + if c_expression.security.enable_security(): + raise Runtime("Profiler is not supported if compiled with \'-s on\'") self._cpu_profiler.stop() _stop_dataset_profiler() if self._device_target and self._device_target == "GPU": diff --git a/tests/st/profiler/test_ascend_profiler.py b/tests/st/profiler/test_ascend_profiler.py index 9cb0406a056..e711f0f9fa8 100644 --- a/tests/st/profiler/test_ascend_profiler.py +++ b/tests/st/profiler/test_ascend_profiler.py @@ -39,6 +39,7 @@ y = np.random.randn(1, 3, 3, 4).astype(np.float32) @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_ascend_training @pytest.mark.env_onecard +@pytest.mark.security_off def test_ascend_profiling(): if os.path.isdir("./data_ascend_profiler"): shutil.rmtree("./data_ascend_profiler") diff --git a/tests/st/profiler/test_profiler.py b/tests/st/profiler/test_profiler.py index 32d339d4932..3e0cbac0933 100644 --- a/tests/st/profiler/test_profiler.py +++ b/tests/st/profiler/test_profiler.py @@ -15,6 +15,7 @@ import os import shutil +from tests.security_utils import security_off_wrap import pytest from mindspore import dataset as ds @@ -137,6 +138,7 @@ class TestProfiler: @pytest.mark.level1 @pytest.mark.platform_x86_gpu_training @pytest.mark.env_onecard + @security_off_wrap def test_gpu_profiler(self): self._train_with_profiler(device_target="GPU") self._check_gpu_profiling_file() @@ -145,6 +147,7 @@ class TestProfiler: @pytest.mark.platform_arm_ascend_training @pytest.mark.platform_x86_ascend_training @pytest.mark.env_onecard + @security_off_wrap def test_ascend_profiler(self): self._train_with_profiler(device_target="Ascend") self._check_d_profiling_file() diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 68e30a88e36..824dea36b37 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -186,6 +186,11 @@ list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/backend/optimizer/ list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/backend/optimizer/gpu/batch_norm_relu_fusion.cc") list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/backend/optimizer/gpu/batch_norm_relu_grad_fusion.cc") list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/backend/kernel_compiler/tbe/ascend_kernel_compile.cc") +if(ENABLE_SECURITY) + list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/profiling.cc") + list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/ascend/memory_profiling.cc") + list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/ascend/ascend_profiling.cc") +endif() add_library(_ut_mindspore_obj OBJECT ${MINDSPORE_SRC_LIST}) add_library(_ut_ut_obj OBJECT ${UT_SRCS})