From 2e622c1428f1ddd37a41a24a4b8e7e2cbb156ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=87=A7=E5=BA=86=E9=A6=99?= Date: Tue, 11 Jan 2022 10:22:46 +0800 Subject: [PATCH] Adapt to the old data format of aicpu --- .../profiler/parser/aicpu_data_parser.py | 38 +++++++++++++++++- .../DATA_PREPROCESS.dev.AICPU.0.slice_0 | Bin 0 -> 968 bytes .../output_data_preprocess_aicpu_0.txt | 5 +++ .../profiler/parser/test_aicpu_parser.py | 36 +++++++++++++---- 4 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 tests/ut/data/profiler_data/JOB_AICPU/data_txt/DATA_PREPROCESS.dev.AICPU.0.slice_0 create mode 100644 tests/ut/data/profiler_data/JOB_AICPU/expect_txt/output_data_preprocess_aicpu_0.txt diff --git a/mindspore/python/mindspore/profiler/parser/aicpu_data_parser.py b/mindspore/python/mindspore/profiler/parser/aicpu_data_parser.py index 00d239d70a3..2c804bbfcb0 100644 --- a/mindspore/python/mindspore/profiler/parser/aicpu_data_parser.py +++ b/mindspore/python/mindspore/profiler/parser/aicpu_data_parser.py @@ -144,7 +144,10 @@ class DataPreProcessParser: with open(self._source_file_name, 'rb') as ai_cpu_data: content = ai_cpu_data.read() - ai_cpu_total_time_summary, result_list = self.parser_binary_file(content) + if content[0:2].hex().upper() == "5A5A": + ai_cpu_total_time_summary, result_list = self.parser_binary_file(content) + else: + ai_cpu_total_time_summary, result_list = self.parser_txt_file(content) os.chmod(self._source_file_name, stat.S_IREAD) @@ -188,6 +191,39 @@ class DataPreProcessParser: return ai_cpu_total_time_summary, result_list + def parser_txt_file(self, content): + """Parse txt format file.""" + ai_cpu_str = str(content.replace(b'\n\x00', b' ___ ').replace(b'\x00', b' ___ '))[2:-1] + ai_cpu_lines = ai_cpu_str.split(" ___ ") + result_list = list() + ai_cpu_total_time_summary = 0 + # Node serial number. + serial_number = 1 + for i in range(len(ai_cpu_lines) - 1): + node_line = ai_cpu_lines[i] + thread_line = ai_cpu_lines[i + 1] + if "Node" in node_line and "Thread" in thread_line: + # Get the node data from node_line + result = self._get_kernel_result( + serial_number, + node_line.split(','), + thread_line.split(',') + ) + + if result is None: + continue + + result_list.append(result) + # Calculate the total time. + total_time = result[2] + ai_cpu_total_time_summary += total_time + # Increase node serial number. + serial_number += 1 + elif "Node" in node_line and "Thread" not in thread_line: + node_type_name = node_line.split(',')[0].split(':')[-1] + logger.warning("The node type:%s cannot find thread data", node_type_name) + return ai_cpu_total_time_summary, result_list + def query_aicpu_data(self): """ Get execution time of AI CPU operator. diff --git a/tests/ut/data/profiler_data/JOB_AICPU/data_txt/DATA_PREPROCESS.dev.AICPU.0.slice_0 b/tests/ut/data/profiler_data/JOB_AICPU/data_txt/DATA_PREPROCESS.dev.AICPU.0.slice_0 new file mode 100644 index 0000000000000000000000000000000000000000..ac6d4db8279057a543ab895f725dd04f269757c4 GIT binary patch literal 968 zcmb`Fy>5dr6h`wXegN8DY;5dE85%`uyJV=?5Q7CqK>>`6iQ2cX`7KS;F0I61`TOn# z9Zrm~aF3vrCFrO#wSB1zdyrOo=(VenmBwyefn$IG1o}Jzj15(1bAb{*7u*qG6P=8b zS&aMyHAbhZZBeE!MXF**wz5c@E=G{Jm$YZ(_ISt^>G@zTk#3%4nePY>DZvaN$}BZtt`hI z%0hQ_zlkvc95NyJ4CzVPqk6MzMpK1gG`ERkwqi8zfTiFN{hd+ZNV5=g7QhWgQz`=X zjnNp9kT5a~E{ukN None: + """Clear output file.""" + if os.path.exists(self.output_path): + shutil.rmtree(self.output_path) + + def test_aicpu_parser_binary(self): + """Test the class of aicpu binary data Parser.""" self.profiling_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../../data/profiler_data/' 'JOB_AICPU/data')) @@ -40,13 +49,6 @@ class TestAicpuParser: "DropoutGenMask-op280" } - def teardown_method(self) -> None: - """Clear output file.""" - if os.path.exists(self.output_path): - shutil.rmtree(self.output_path) - - def test_aicpu_parser(self): - """Test the class of Aicpu Parser.""" data = DataPreProcessParser(self.profiling_dir, self.output_file, self.op_task_dict) data.execute() with open(self.expect_file, 'r') as fp: @@ -54,3 +56,23 @@ class TestAicpuParser: with open(self.output_file, 'r') as fp: result = fp.read() assert expect_result == result + + def test_aicpu_parser_txt(self): + """Test the class of aicpu txt data Parser.""" + self.profiling_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), + '../../../data/profiler_data/' + 'JOB_AICPU/data_txt')) + self.expect_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), + '../../../data/profiler_data/' + 'JOB_AICPU/expect_txt')) + self.output_path = tempfile.mkdtemp(prefix='output_data_preprocess_aicpu_') + self.output_file = os.path.join(self.output_path, 'output_data_preprocess_aicpu_0.txt') + self.expect_file = os.path.join(self.expect_dir, 'output_data_preprocess_aicpu_0.txt') + + data = DataPreProcessParser(self.profiling_dir, self.output_file, None) + data.execute() + with open(self.expect_file, 'r') as fp: + expect_result = fp.read() + with open(self.output_file, 'r') as fp: + result = fp.read() + assert expect_result == result