diff --git a/mindspore/_extends/parallel_compile/tbe_compiler/tbe_adapter.py b/mindspore/_extends/parallel_compile/tbe_compiler/tbe_adapter.py index 1c9d5613004..29fadc833f7 100644 --- a/mindspore/_extends/parallel_compile/tbe_compiler/tbe_adapter.py +++ b/mindspore/_extends/parallel_compile/tbe_compiler/tbe_adapter.py @@ -467,7 +467,7 @@ def ga_tune(job: TbeJob): """ l1_size = job.content["l1_size"] op_kernel_name = job.content["fusion_op_name"] - dispatch_autotune_task(job.source_id, job.id, l1_size, json.dumps(job.content), [], op_kernel_name) + dispatch_autotune_task(job.source_id, job.id, l1_size, json.dumps(job.content), {}, op_kernel_name) job.status = JobStatus.JOB_RUNNING return True @@ -519,7 +519,7 @@ def rl_tune_single_op(job: TbeJob): pack_args = pack_op_args(inputs, outputs, attrs) res = dispatch_single_tune_task(job.source_id, job.id, l1_size, base_kernel, op_kernel_name, full_name, tune_op_module_name, op_func_name, op_type, pack_args) - res = _process_rl_tune_result(job, res) + res = _process_rl_tune_result(job, op_type, res) return res @@ -546,19 +546,24 @@ def rl_tune_fusion_op(job: TbeJob): base_kernel = job.content["SocInfo"]["op_debug_dir"] + "/kernel_meta/" + op_kernel_name + ".o" compute_op_list = get_compute_op_list(job.content) op_module_names_str = "" + op_type_set = set() for op in compute_op_list: op_module_names_str = op_module_names_str + "," + get_module_name(op) + op_type_set.add(op["type"]) op_module_names_str = op_module_names_str[1:] + op_type = "__".join(list(op_type_set)) from schedule_search.rl_online_tune import dispatch_fusion_tune_task res = dispatch_fusion_tune_task(job.source_id, job.id, l1_size, base_kernel, op_kernel_name, op_module_names_str, json.dumps(job.content)) - res = _process_rl_tune_result(job, res) + res = _process_rl_tune_result(job, op_type, res) return res -def _process_rl_tune_result(job, res): +def _process_rl_tune_result(job, op_type, res): if not res: - res = bool(job.sys_offline_tune or os.getenv("REPEAT_TUNE", "False").lower() != "true") + from schedule_search.tune_util import filter_black_op_type + res = bool(job.sys_offline_tune or os.getenv("REPEAT_TUNE", "False").lower() != "true" or filter_black_op_type( + op_type)) else: job.status = JobStatus.JOB_RUNNING res = True diff --git a/mindspore/ccsrc/backend/kernel_compiler/tbe/ascend_kernel_compile.cc b/mindspore/ccsrc/backend/kernel_compiler/tbe/ascend_kernel_compile.cc index 7e5beec7ae6..9ecae282dbf 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/tbe/ascend_kernel_compile.cc +++ b/mindspore/ccsrc/backend/kernel_compiler/tbe/ascend_kernel_compile.cc @@ -344,15 +344,16 @@ nlohmann::json AscendKernelCompileManager::TurnStrToJson(const std::string &stri return json; } -void AscendKernelCompileManager::ParseTargetJobStatus(const std::string &type, const std::string &build_result, +void AscendKernelCompileManager::ParseTargetJobStatus(const std::string &type, const std::string &job_result, std::vector *success_job) { MS_EXCEPTION_IF_NULL(success_job); - auto json_obj = TurnStrToJson(build_result); + auto json_obj = TurnStrToJson(job_result); if (json_obj.at(kStatus) == kSuccess) { nlohmann::json query_obj; if (!ParseJson(GetJsonValue(json_obj, kResult), &query_obj)) { MS_LOG(EXCEPTION) << "Parse query result error."; } + auto kernel_name = GetJsonValue(query_obj, kFusionOpName); struct TargetJobStatus task_info; QueryResultProcess(json_obj, &task_info); if (task_info.job_status == kSuccess) { @@ -367,16 +368,16 @@ void AscendKernelCompileManager::ParseTargetJobStatus(const std::string &type, c } else if (task_info.job_status == kFailed) { if (type == kPreCompile) { success_job->emplace_back(task_info.target_job_id); - MS_LOG(WARNING) << "Single op pre build failed ,res: " << query_obj; + MS_LOG(WARNING) << "Single op pre build failed ,op: " << kernel_name; } else { ResetOldTask(); single_processed_kernels_.clear(); - MS_LOG(EXCEPTION) << "Single op compile failed ,res: " << query_obj; + MS_LOG(EXCEPTION) << "Single op compile failed ,op: " << kernel_name; } } } else { auto file_name = GetJsonValue(json_obj, kJobType) + "_" + json_obj.at(kJobId).dump(); - TbeUtils::SaveJsonInfo(file_name, build_result); + TbeUtils::SaveJsonInfo(file_name, job_result); MS_LOG(EXCEPTION) << "Query job failed"; } } @@ -391,9 +392,9 @@ void AscendKernelCompileManager::QueryFinishJob(const std::string &job_type) { nlohmann::json query_json; auto kernel_json = iter->second; JsonAssemble(kQuery, kernel_json, &query_json); - auto build_result = build_manager_->ProcessTbeJob(query_json); + auto job_result = build_manager_->ProcessTbeJob(query_json); query_cnt++; - ParseTargetJobStatus(job_type, build_result, &success_job); + ParseTargetJobStatus(job_type, job_result, &success_job); iter++; } for (auto k : success_job) { diff --git a/mindspore/ccsrc/backend/kernel_compiler/tbe/ascend_kernel_compile.h b/mindspore/ccsrc/backend/kernel_compiler/tbe/ascend_kernel_compile.h index 203d7f4ec29..a6f6bc9f883 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/tbe/ascend_kernel_compile.h +++ b/mindspore/ccsrc/backend/kernel_compiler/tbe/ascend_kernel_compile.h @@ -66,7 +66,7 @@ class AscendKernelCompileManager { ~AscendKernelCompileManager(); void GetAllAscendNodes(const std::shared_ptr &kernel_graph, std::vector *tbe_nodes); void QueryFinishJob(const std::string &type); - void ParseTargetJobStatus(const std::string &type, const std::string &build_res, std::vector *success_job); + void ParseTargetJobStatus(const std::string &type, const std::string &job_result, std::vector *success_job); void QueryPreBuildFinishJob(); void QueryFusionFinishJob(KernelModMap *kernel_mode_ret); void PrintProcessLog(const nlohmann::json &json, int adjust_log_level); diff --git a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_json/tbe_json_creator.cc b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_json/tbe_json_creator.cc index 69cc855f2cd..b212d153c7e 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_json/tbe_json_creator.cc +++ b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_json/tbe_json_creator.cc @@ -301,6 +301,7 @@ bool TbeJsonCreator::GenAttrsDescJson(const AnfNodePtr &anf_node, nlohmann::json auto op_info_ptr = tbe::TbeDynamicShapeUtil::FindOp(op_name, cnode); nlohmann::json attrs_json; GenAttrsJson(cnode, op_info_ptr, &attrs_json); + (*compute_json)[kJAttrs] = attrs_json; nlohmann::json attrs_desc; for (const auto &attr : attrs_json) { diff --git a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_json/tbe_json_utils.h b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_json/tbe_json_utils.h index 6e49f1d135f..fcd26879181 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_json/tbe_json_utils.h +++ b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_json/tbe_json_utils.h @@ -78,6 +78,7 @@ constexpr auto kJModuleName = "module_name"; constexpr auto kJModuleNamePrefix = "impl."; constexpr auto kJPattern = "pattern"; constexpr auto kJPyModulePath = "py_module_path"; +constexpr auto kJAttrs = "attrs"; constexpr auto kJAttrDesc = "attr_desc"; constexpr auto kJSocInfo = "SocInfo"; constexpr auto kJFusionOpName = "fusion_op_name"; diff --git a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_utils.cc b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_utils.cc index f72a8eef553..a0eb2d62c51 100644 --- a/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_utils.cc +++ b/mindspore/ccsrc/backend/kernel_compiler/tbe/tbe_utils.cc @@ -112,8 +112,9 @@ std::string TbeUtils::GetOpDebugPath() { if (!old_build.empty()) { if (config_path[config_path.length() - 1] == '/') { debug_path = config_path; + } else { + debug_path = config_path + "/"; } - debug_path = config_path + "/"; return debug_path; } else { std::string rank_id_str = common::GetEnv(kRankID); @@ -123,8 +124,9 @@ std::string TbeUtils::GetOpDebugPath() { } if (config_path[config_path.length() - 1] == '/') { debug_path = config_path + "rank_" + rank_id_str + "/"; + } else { + debug_path = config_path + "/" + "rank_" + rank_id_str + "/"; } - debug_path = config_path + "/" + "rank_" + rank_id_str + "/"; return debug_path; } } @@ -382,12 +384,17 @@ void TbeUtils::GetCompileInfo(const AnfNodePtr &node, std::string *compile_info, std::ifstream file(path.c_str()); std::string ori_file = std::string((std::istreambuf_iterator(file)), std::istreambuf_iterator()); if (!ParseJson(ori_file, &read_new_json)) { - MS_LOG(EXCEPTION) << "Parse compile info error."; + MS_LOG(EXCEPTION) << "Parse compile info error :" << ori_file; } - *compile_info = read_new_json[kBuildRes].dump(); + auto build_res_str = GetJsonValue(read_new_json, kBuildRes); + nlohmann::json build_res_json; + if (!ParseJson(build_res_str, &build_res_json)) { + MS_LOG(EXCEPTION) << "Parse build result for " << node->fullname_with_scope() << " error :" << build_res_str; + } + *compile_info = build_res_json.dump(); file.close(); file.clear(); - MS_LOG(INFO) << "Get compile info from json file success"; + MS_LOG(INFO) << "Get compile info from json file success :" << *compile_info; } void TbeUtils::SaveCompileInfo(const std::string &json_name, const std::string &build_res, bool *save_flag) { @@ -407,11 +414,7 @@ void TbeUtils::SaveCompileInfo(const std::string &json_name, const std::string & } file.close(); file.clear(); - if (build_res.empty()) { - save_new_json[kBuildRes] = build_res; - } else { - save_new_json[kBuildRes] = nlohmann::json::parse(build_res); - } + save_new_json[kBuildRes] = build_res; std::ofstream file_write; file_write.open(path); if (!file_write.is_open()) { diff --git a/tests/ut/cpp/tbe/tbe_json_creator_test.cc b/tests/ut/cpp/tbe/tbe_json_creator_test.cc index 7af91ab2c94..937ef13c83a 100644 --- a/tests/ut/cpp/tbe/tbe_json_creator_test.cc +++ b/tests/ut/cpp/tbe/tbe_json_creator_test.cc @@ -76,11 +76,11 @@ TEST_F(TestHWTBEJsonCreator, test_tbe_single_common) { auto tbe_json_creator_build = std::make_shared(); nlohmann::json kernel_json; EXPECT_TRUE(tbe_json_creator_select->GenJson(relu1, &kernel_json)); - EXPECT_EQ(tbe_json_creator_select->GetJsonHash(), 7996236493612266030U); + EXPECT_EQ(tbe_json_creator_select->GetJsonHash(), 4297213426602035622U); EXPECT_TRUE(tbe_json_creator_check->GenJson(relu1, &kernel_json)); - EXPECT_EQ(tbe_json_creator_check->GetJsonHash(), 16463039402039306442U); + EXPECT_EQ(tbe_json_creator_check->GetJsonHash(), 5131870964632527075U); EXPECT_TRUE(tbe_json_creator_build->GenJson(relu1, &kernel_json)); - EXPECT_EQ(tbe_json_creator_build->GetJsonHash(), 8871925407866693227U); + EXPECT_EQ(tbe_json_creator_build->GetJsonHash(), 6011570351795510237U); } TEST_F(TestHWTBEJsonCreator, test_tbe_single_conv2d_backprop_filter) { @@ -119,11 +119,11 @@ TEST_F(TestHWTBEJsonCreator, test_tbe_single_conv2d_backprop_filter) { auto tbe_json_creator_build = std::make_shared(); nlohmann::json kernel_json; EXPECT_TRUE(tbe_json_creator_select->GenJson(conv2d_backprop_filter, &kernel_json)); - EXPECT_EQ(tbe_json_creator_select->GetJsonHash(), 16997569423579290131U); + EXPECT_EQ(tbe_json_creator_select->GetJsonHash(), 3804649253898608226U); EXPECT_TRUE(tbe_json_creator_check->GenJson(conv2d_backprop_filter, &kernel_json)); - EXPECT_EQ(tbe_json_creator_check->GetJsonHash(), 1051090390656699050U); + EXPECT_EQ(tbe_json_creator_check->GetJsonHash(), 5736923382341947495U); EXPECT_TRUE(tbe_json_creator_build->GenJson(conv2d_backprop_filter, &kernel_json)); - EXPECT_EQ(tbe_json_creator_build->GetJsonHash(), 3388833908101709327U); + EXPECT_EQ(tbe_json_creator_build->GetJsonHash(), 4580870880229487185U); } TEST_F(TestHWTBEJsonCreator, test_tbe_single_dynamic_rnn) { @@ -177,11 +177,11 @@ TEST_F(TestHWTBEJsonCreator, test_tbe_single_dynamic_rnn) { auto tbe_json_creator_build = std::make_shared(); nlohmann::json kernel_json; EXPECT_TRUE(tbe_json_creator_select->GenJson(dynamic_rnn, &kernel_json)); - EXPECT_EQ(tbe_json_creator_select->GetJsonHash(), 599064190761596566U); + EXPECT_EQ(tbe_json_creator_select->GetJsonHash(), 13058640182660031121U); EXPECT_TRUE(tbe_json_creator_check->GenJson(dynamic_rnn, &kernel_json)); - EXPECT_EQ(tbe_json_creator_check->GetJsonHash(), 13855166034392728379U); + EXPECT_EQ(tbe_json_creator_check->GetJsonHash(), 5110289197661808901U); EXPECT_TRUE(tbe_json_creator_build->GenJson(dynamic_rnn, &kernel_json)); - EXPECT_EQ(tbe_json_creator_build->GetJsonHash(), 12346685554589275353U); + EXPECT_EQ(tbe_json_creator_build->GetJsonHash(), 4729701784171992376U); } TEST_F(TestHWTBEJsonCreator, test_tbe_single_layer_norm) { @@ -231,11 +231,11 @@ TEST_F(TestHWTBEJsonCreator, test_tbe_single_layer_norm) { auto tbe_json_creator_build = std::make_shared(); nlohmann::json kernel_json; EXPECT_TRUE(tbe_json_creator_select->GenJson(layer_norm, &kernel_json)); - EXPECT_EQ(tbe_json_creator_select->GetJsonHash(), 13056848426482724958U); + EXPECT_EQ(tbe_json_creator_select->GetJsonHash(), 1114128635775386802U); EXPECT_TRUE(tbe_json_creator_check->GenJson(layer_norm, &kernel_json)); - EXPECT_EQ(tbe_json_creator_check->GetJsonHash(), 3069436317069842619U); + EXPECT_EQ(tbe_json_creator_check->GetJsonHash(), 2636386772926575020U); EXPECT_TRUE(tbe_json_creator_build->GenJson(layer_norm, &kernel_json)); - EXPECT_EQ(tbe_json_creator_build->GetJsonHash(), 18320131482846743097U); + EXPECT_EQ(tbe_json_creator_build->GetJsonHash(), 9247341733773157591U); } TEST_F(TestHWTBEJsonCreator, test_tbe_fusion_common) { @@ -306,7 +306,7 @@ TEST_F(TestHWTBEJsonCreator, test_tbe_fusion_common) { nlohmann::json fusion_json; auto tbe_json_creator = std::make_shared(); EXPECT_TRUE(tbe_json_creator->GenJson(fusion_scope_info, &fusion_json)); - EXPECT_EQ(tbe_json_creator->GetJsonHash(), 4464178465553346953U); + EXPECT_EQ(tbe_json_creator->GetJsonHash(), 2048601293894159116U); } TEST_F(TestHWTBEJsonCreator, test_fusion_add_conv2d) { @@ -365,7 +365,7 @@ TEST_F(TestHWTBEJsonCreator, test_fusion_add_conv2d) { nlohmann::json fusion_json; auto tbe_json_creator = std::make_shared(); EXPECT_TRUE(tbe_json_creator->GenJson(fusion_scope_info, &fusion_json)); - EXPECT_EQ(tbe_json_creator->GetJsonHash(), 6707165667078013944U); + EXPECT_EQ(tbe_json_creator->GetJsonHash(), 5525866309367625497U); } } // namespace mindspore::kernel