Col_lin
/
mindspore2022
forked from huawei/mindspore2022
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove profiler if compiled with -s on

This commit is contained in:
yanghaitao1 2021-09-13 09:36:20 -04:00
parent f5bc717c4c
commit 177f3f75bf
14 changed files with 95 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
mindspore/ccsrc/CMakeLists.txt
View File

@ -153,12 +153,14 @@ include_directories("${CMAKE_BINARY_DIR}/ps/core")
file(GLOB_RECURSE COMM_PROTO_IN RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "ps/core/protos/*.proto")
ms_protobuf_generate(COMM_PROTO_SRCS COMM_PROTO_HDRS ${COMM_PROTO_IN})
list(APPEND MINDSPORE_PROTO_LIST ${COMM_PROTO_SRCS})
include_directories("${CMAKE_BINARY_DIR}/profiler/device/ascend")
file(GLOB_RECURSE PROFILER_PROTO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"profiler/device/ascend/memory_profiling.proto")
ms_protobuf_generate_py(PROFILER_MEM_PROTO_PY PROFILER_MEM_PROTO_HDRS_PY PROFILER_MEM_PROTO_PYS ${PROFILER_PROTO_LIST})
list(APPEND MINDSPORE_PROTO_LIST ${PROFILER_MEM_PROTO_PY})
if(NOT ENABLE_SECURITY)
include_directories("${CMAKE_BINARY_DIR}/profiler/device/ascend")
file(GLOB_RECURSE PROFILER_PROTO_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"profiler/device/ascend/memory_profiling.proto")
ms_protobuf_generate_py(PROFILER_MEM_PROTO_PY PROFILER_MEM_PROTO_HDRS_PY PROFILER_MEM_PROTO_PYS
${PROFILER_PROTO_LIST})
list(APPEND MINDSPORE_PROTO_LIST ${PROFILER_MEM_PROTO_PY})
endif()
if(ENABLE_DEBUGGER)
# debugger: compile proto files

2
mindspore/ccsrc/backend/kernel_compiler/gpu/data/dataset_iterator_kernel.cc
View File

@ -73,6 +73,7 @@ bool DatasetIteratorKernel::Init(const CNodePtr &kernel_node) {
MS_LOG(EXCEPTION) << "Gpu Queue(" << queue_name_ << ") Open Failed";
}
#ifndef ENABLE_SECURITY
auto profiler_inst = profiler::gpu::GPUProfiler::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_inst);
profiling_enable_ = profiler_inst->GetEnableFlag();
@ -82,6 +83,7 @@ bool DatasetIteratorKernel::Init(const CNodePtr &kernel_node) {
MS_EXCEPTION_IF_NULL(profiling_op_);
profiler_inst->RegisterProfilingOp(profiling_op_);
}
#endif
return true;
}

6
mindspore/ccsrc/backend/optimizer/somas/somas.cc
View File

@ -37,12 +37,13 @@
#include "debug/rdr/running_data_recorder.h"
#endif
#include "common/thread_pool.h"
#ifndef ENABLE_SECURITY
#include "profiler/device/ascend/memory_profiling.h"
using mindspore::profiler::ascend::MemoryProfiling;
using mindspore::profiler::ascend::NodeMemory;
using mindspore::profiler::ascend::TensorMemory;
#endif
namespace mindspore {
namespace somas {
constexpr auto kGapSize = 512;
@ -1772,7 +1773,7 @@ uint8_t *Somas::GetNodeWorkSpacePtr(const AnfNodePtr &node, size_t index) const
}
return ptr;
}
#ifndef ENABLE_SECURITY
void Somas::ConvertToProfilingNode(uint32_t graph_id) {
#ifdef ENABLE_D
auto graph_node = MemoryProfiling::GetInstance().GetGraphMemoryNode(graph_id);
@ -1810,5 +1811,6 @@ void Somas::ConvertToProfilingNode(uint32_t graph_id) {
}
#endif
}
#endif
} // namespace somas
} // namespace mindspore

2
mindspore/ccsrc/backend/optimizer/somas/somas.h
View File

@ -55,7 +55,9 @@ class Somas {
void DumpSomasMemoryIR(const string filename) const;
static bool NodeSort(SomasNodePtr, SomasNodePtr);
#ifndef ENABLE_SECURITY
void ConvertToProfilingNode(uint32_t graph_id);
#endif
private:
std::vector<DynamicBitSet> reuse_matrix_;

2
mindspore/ccsrc/backend/session/ascend_session.cc
View File

@ -81,9 +81,9 @@
#include "ps/ps_cache/ps_cache_manager.h"
#endif
#include "runtime/device/ascend/ascend_bucket.h"
#ifndef ENABLE_SECURITY
#include "profiler/device/ascend/memory_profiling.h"
#ifndef ENABLE_SECURITY
using mindspore::device::ascend::ProfilingManager;
using mindspore::profiler::ascend::MemoryProfiling;
#endif

42
mindspore/ccsrc/profiler/CMakeLists.txt
View File

@ -1,25 +1,27 @@
if(ENABLE_GPU)
file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"device/gpu/*.cc" "device/cpu/*.cc")
endif()
if(NOT ENABLE_SECURITY)
if(ENABLE_GPU)
file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"device/gpu/*.cc" "device/cpu/*.cc")
endif()
if(ENABLE_D)
file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"device/cpu/*.cc" "device/ascend/*.cc")
endif()
if(ENABLE_D)
file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"device/cpu/*.cc" "device/ascend/*.cc")
endif()
if(ENABLE_CPU AND NOT (ENABLE_D OR ENABLE_GPU))
file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"device/cpu/*.cc")
endif()
if(ENABLE_CPU AND NOT (ENABLE_D OR ENABLE_GPU))
file(GLOB_RECURSE PROFILER_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"device/cpu/*.cc")
endif()
list(APPEND PROFILER_SRC_LIST ${CMAKE_CURRENT_SOURCE_DIR}/device/profiling.cc
${CMAKE_CURRENT_SOURCE_DIR}/device/data_saver.cc)
list(APPEND PROFILER_SRC_LIST ${CMAKE_CURRENT_SOURCE_DIR}/device/profiling.cc
${CMAKE_CURRENT_SOURCE_DIR}/device/data_saver.cc)
set_property(SOURCE ${PROFILER_SRC_LIST} PROPERTY COMPILE_DEFINITIONS
SUBMODULE_ID=mindspore::SubModuleId::SM_PROFILER)
add_library(_mindspore_profiler_obj OBJECT ${PROFILER_SRC_LIST})
set_property(SOURCE ${PROFILER_SRC_LIST} PROPERTY COMPILE_DEFINITIONS
SUBMODULE_ID=mindspore::SubModuleId::SM_PROFILER)
add_library(_mindspore_profiler_obj OBJECT ${PROFILER_SRC_LIST})
if(ENABLE_D)
add_dependencies(_mindspore_profiler_obj mindspore::protobuf)
endif()
if(ENABLE_D)
add_dependencies(_mindspore_profiler_obj mindspore::protobuf)
endif()
endif()

8
mindspore/ccsrc/runtime/device/ascend/ascend_memory_manager.cc
View File

@ -18,11 +18,13 @@
#include "runtime/device/ascend/ascend_memory_pool.h"
#include "utils/ms_context.h"
#include "runtime/mem.h"
#ifndef ENABLE_SECURITY
#include "runtime/device/ascend/profiling/profiling_manager.h"
#include "profiler/device/ascend/memory_profiling.h"
using mindspore::device::ascend::ProfilingManager;
using mindspore::profiler::ascend::MemoryProfiling;
#endif
namespace mindspore {
namespace device {
@ -139,7 +141,7 @@ uint8_t *AscendMemoryManager::MallocStaticMem(size_t size, bool communication_me
<< "], Pool statistics: pool total size [" << AscendMemoryPool::GetInstance().total_mem_statistics()
<< "] used [" << AscendMemoryPool::GetInstance().used_mem_statistics()
<< "] communication_mem:" << communication_mem;
#ifndef ENABLE_SECURITY
if (MemoryProfiling::GetInstance().IsMemoryProfilingEnable() && graph_id != kInvalidGraphId) {
auto node = MemoryProfiling::GetInstance().GetGraphMemoryNode(graph_id);
if (node == nullptr) {
@ -149,7 +151,7 @@ uint8_t *AscendMemoryManager::MallocStaticMem(size_t size, bool communication_me
node->AddStaticMemorySize(SizeToUint(align_size));
}
#endif
if (communication_mem) {
// create protect area [kMemAlignSize -- data -- kMemAlignSize]
uint8_t *alloc_address = reinterpret_cast<uint8_t *>(AscendMemoryPool::GetInstance().AllocTensorMem(align_size));
@ -191,9 +193,11 @@ uint8_t *AscendMemoryManager::MallocDynamicMem(size_t size, bool communication_m
void AscendMemoryManager::MallocSomasDynamicMem(const session::KernelGraph *graph) {
MemoryManager::MallocSomasDynamicMem(graph);
#ifndef ENABLE_SECURITY
if (MemoryProfiling::GetInstance().IsMemoryProfilingEnable()) {
somas_reuse_util_ptr_->ConvertToProfilingNode(graph->graph_id());
}
#endif
}
// communication memory: [512align_size + data + 512align_size]

8
mindspore/ccsrc/runtime/device/cpu/cpu_kernel_runtime.cc
View File

@ -410,8 +410,6 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) {
static_cast<CPUMemoryManager *>(mem_manager_.get())->IncreaseAddressRefCount(kernel_graph);
auto kernels = kernel_graph->execution_order();
auto profiler_inst = profiler::cpu::CPUProfiler::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_inst);
#ifndef ENABLE_SECURITY
auto &dump_json_parser = DumpJsonParser::GetInstance();
@ -452,10 +450,14 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) {
AddRuntimeAddress(device_address, &kernel_workspaces);
}
bool ret = true;
#ifndef ENABLE_SECURITY
auto profiler_inst = profiler::cpu::CPUProfiler::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_inst);
if (profiler_inst->GetEnableFlag()) {
uint32_t pid = getpid();
profiler_inst->OpDataProducerBegin(kernel->fullname_with_scope(), pid);
}
#endif
#ifdef ENABLE_DUMP_IR
kernel::KernelLaunchInfo mem_info = {kernel_inputs, kernel_workspaces, kernel_outputs};
std::string op_name = kernel->fullname_with_scope();
@ -470,10 +472,10 @@ bool CPUKernelRuntime::Run(session::KernelGraph *kernel_graph, bool) {
if (iter_dump_flag) {
CPUE2eDump::DumpCNodeData(kernel, graph_id);
}
#endif
if (profiler_inst->GetEnableFlag()) {
profiler_inst->OpDataProducerEnd();
}
#endif
if (!ret) {
#ifdef ENABLE_DUMP_IR
mindspore::RDR::TriggerAll();

8
mindspore/ccsrc/runtime/device/gpu/gpu_kernel_runtime.cc
View File

@ -826,7 +826,7 @@ void GPUKernelRuntime::LaunchKernelWithoutMock(const session::KernelGraph *graph
const AddressPtrList &outputs, bool profiling) {
MS_EXCEPTION_IF_NULL(graph);
MS_EXCEPTION_IF_NULL(kernel);
#ifndef ENABLE_SECURITY
auto profiler_inst = profiler::gpu::GPUProfiler::GetInstance();
MS_EXCEPTION_IF_NULL(profiler_inst);
@ -835,11 +835,13 @@ void GPUKernelRuntime::LaunchKernelWithoutMock(const session::KernelGraph *graph
profiler::gpu::ProfilingUtils::GetProfilingTraceFromEnv(NOT_NULL(graph));
profiler_inst->SetStepTraceOpName(profiling_trace);
}
#endif
if (!profiling) {
#ifndef ENABLE_SECURITY
if (profiler_inst->GetEnableFlag()) {
profiler_inst->OpDataProducerBegin(kernel->fullname_with_scope(), stream_);
}
#endif
auto kernel_mod = AnfAlgo::GetKernelMod(kernel);
MS_EXCEPTION_IF_NULL(kernel_mod);
if (!kernel_mod->Launch(inputs, workspaces, outputs, stream_)) {
@ -848,12 +850,14 @@ void GPUKernelRuntime::LaunchKernelWithoutMock(const session::KernelGraph *graph
#endif
MS_LOG(EXCEPTION) << "Launch kernel failed: " << kernel->fullname_with_scope();
}
#ifndef ENABLE_SECURITY
if (profiler_inst->GetEnableFlag()) {
profiler_inst->OpDataProducerEnd();
if (profiler_inst->GetSyncEnableFlag()) {
CHECK_OP_RET_WITH_ERROR(SyncStream(), "Profiler SyncStream failed.");
}
}
#endif
} else {
LaunchKernelWithTimeProfiling(kernel, inputs, workspaces, outputs);
}

40
mindspore/profiler/parser/memory_usage_parser.py
View File

@ -17,6 +17,7 @@ from collections import OrderedDict
import json
import os
import stat
from importlib import import_module
from google.protobuf.text_format import ParseError
@ -27,7 +28,12 @@ from mindspore.profiler.common.validator.validate_path import validate_and_norma
from mindspore.profiler.parser.container import MemoryGraph as Graph
from mindspore.profiler.parser.container import MemoryNode as Node
from mindspore.profiler.parser.container import MemoryTensor as Tensor
from mindspore.train.memory_profiling_pb2 import MemoryProto
import mindspore._c_expression as c_expression
try:
MemoryProto = import_module("mindspore.train.memory_profiling_pb2.MemoryProto")
except ModuleNotFoundError:
MemoryProto = None
GIGABYTES = 1024 * 1024 * 1024
@ -84,22 +90,26 @@ class MemoryUsageParser:
raise ProfilerIOException
# Parse memory raw data from file.
memory_proto = MemoryProto()
try:
memory_proto.ParseFromString(content)
except ParseError as err:
msg = "Fail to parse memory proto file."
logger.error("Cannot parse the memory file. Please check the file schema.\n%s", err)
raise ProfilerRawFileException(msg)
if not c_expression.security.enable_security():
if not MemoryProto:
raise ProfilerRawFileException("Can not find memory profiling pb file.")
# Parse memory details based on graphs in the network.
graphs = memory_proto.graph_mem
self._parse_graph_memory(graphs)
# Update memory summary information.
self._mem_summary['capacity'] = memory_proto.total_mem / GIGABYTES
self._mem_summary['peak_mem'] = self._peak_mem
memory_proto = MemoryProto()
try:
memory_proto.ParseFromString(content)
except ParseError as err:
msg = "Fail to parse memory proto file."
logger.error("Cannot parse the memory file. Please check the file schema.\n%s", err)
raise ProfilerRawFileException(msg)
logger.info('Finished processing memory usage data.')
# Parse memory details based on graphs in the network.
graphs = memory_proto.graph_mem
self._parse_graph_memory(graphs)
# Update memory summary information.
self._mem_summary['capacity'] = memory_proto.total_mem / GIGABYTES
self._mem_summary['peak_mem'] = self._peak_mem
logger.info('Finished processing memory usage data.')
def _parse_graph_memory(self, graphs):
"""Parse memory usage based on subgraphs."""

5
mindspore/profiler/profiling.py
View File

@ -121,6 +121,9 @@ class Profiler:
_aicpu_op_output_filename_target = "output_data_preprocess_aicpu_"
def __init__(self, **kwargs):
if c_expression.security.enable_security():
raise Runtime("Profiler is not supported if compiled with \'-s on\'")
# get device_id and device_target
self._get_devid_rankid_and_devtarget()
self._get_output_path(kwargs)
@ -225,6 +228,8 @@ class Profiler:
"""
Collect and analyse performance data, called after training or during training. The example shows above.
"""
if c_expression.security.enable_security():
raise Runtime("Profiler is not supported if compiled with \'-s on\'")
self._cpu_profiler.stop()
_stop_dataset_profiler()
if self._device_target and self._device_target == "GPU":

1
tests/st/profiler/test_ascend_profiler.py
View File

@ -39,6 +39,7 @@ y = np.random.randn(1, 3, 3, 4).astype(np.float32)
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@pytest.mark.security_off
def test_ascend_profiling():
if os.path.isdir("./data_ascend_profiler"):
shutil.rmtree("./data_ascend_profiler")

3
tests/st/profiler/test_profiler.py
View File

@ -15,6 +15,7 @@
import os
import shutil
from tests.security_utils import security_off_wrap
import pytest
from mindspore import dataset as ds
@ -137,6 +138,7 @@ class TestProfiler:
@pytest.mark.level1
@pytest.mark.platform_x86_gpu_training
@pytest.mark.env_onecard
@security_off_wrap
def test_gpu_profiler(self):
self._train_with_profiler(device_target="GPU")
self._check_gpu_profiling_file()
@ -145,6 +147,7 @@ class TestProfiler:
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@security_off_wrap
def test_ascend_profiler(self):
self._train_with_profiler(device_target="Ascend")
self._check_d_profiling_file()

5
tests/ut/cpp/CMakeLists.txt
View File

@ -186,6 +186,11 @@ list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/backend/optimizer/
list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/backend/optimizer/gpu/batch_norm_relu_fusion.cc")
list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/backend/optimizer/gpu/batch_norm_relu_grad_fusion.cc")
list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/backend/kernel_compiler/tbe/ascend_kernel_compile.cc")
if(ENABLE_SECURITY)
list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/profiling.cc")
list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/ascend/memory_profiling.cc")
list(REMOVE_ITEM MINDSPORE_SRC_LIST "../../../mindspore/ccsrc/profiler/device/ascend/ascend_profiling.cc")
endif()
add_library(_ut_mindspore_obj OBJECT ${MINDSPORE_SRC_LIST})
add_library(_ut_ut_obj OBJECT ${UT_SRCS})