forked from huawei/mindspore2022
remove mindspore's log
This commit is contained in:
jjfeing
parent
bf87af2db0
commit
988bf44dff
|
|
@ -25,7 +25,6 @@ import tbe.common.context.op_context as op_context
|
|||
from tbe.common.buildcfg import build_config
|
||||
import te.platform.vector_random_buff as vector_random_buff
|
||||
import te.platform.cube_random_buff as cube_random_buff
|
||||
from mindspore import log
|
||||
from .tbe_common import check_kernel_info, TBEException
|
||||
from .helper import _op_select_format, _check_supported
|
||||
|
||||
|
|
@ -148,7 +147,6 @@ class TbeProcess:
|
|||
self.auto_tune_op_list = None
|
||||
self.tune_ops_name = os.getenv("TUNE_OPS_NAME")
|
||||
self.selected_tune_ops = self.tune_ops_name.split(",") if self.tune_ops_name is not None else None
|
||||
log.info("Selected to tune ops list:{}".format(self.selected_tune_ops))
|
||||
|
||||
def __del__(self):
|
||||
if self.__pool is not None:
|
||||
|
|
@ -189,9 +187,7 @@ class TbeProcess:
|
|||
self.tune_mode = tune_mode
|
||||
if os.getenv("ENABLE_TUNE_DUMP", "").lower() == "true":
|
||||
self.offline_tune = True
|
||||
log.info("Tune offline mode is on...")
|
||||
if self.tune_mode == NO_TUNE and not self.offline_tune:
|
||||
log.info("[NO_TUNE] There is no need to initialize auto_tune related variables.")
|
||||
return "Success"
|
||||
|
||||
try:
|
||||
|
|
@ -199,7 +195,6 @@ class TbeProcess:
|
|||
import auto_tune_main
|
||||
import schedule_search # pylint: disable=unused-import
|
||||
self.auto_tune_op_list = auto_tune_main.enable_auto_tune_support()
|
||||
log.info("auto tune GA support ops list:{}".format(self.auto_tune_op_list))
|
||||
except ImportError:
|
||||
res = "TBEException", \
|
||||
"No module named `auto_tune` or `schedule_search`. If you want tune your op's performance," \
|
||||
|
|
@ -241,17 +236,14 @@ class TbeProcess:
|
|||
"""
|
||||
exit tbe process
|
||||
"""
|
||||
log.info("start to exit tbe process...")
|
||||
if self.__pool is not None:
|
||||
stop_thread = threading.Thread(target=self.close_pool)
|
||||
stop_thread.daemon = True
|
||||
stop_thread.start()
|
||||
log.info("tbe process poll exited.")
|
||||
if self.__tuner is not None:
|
||||
stop_tuner = threading.Thread(target=self.close_tuner)
|
||||
stop_tuner.daemon = True
|
||||
stop_tuner.start()
|
||||
log.info("tbe process tuner exited.")
|
||||
|
||||
def _if_tune_ops(self, op_json):
|
||||
"""
|
||||
|
|
@ -346,14 +338,11 @@ class TbeProcess:
|
|||
task_future = self.__pool.apply_async(func=run_compiler, args=(op_json,))
|
||||
self.__running_tasks.append((task_id, task_future))
|
||||
else:
|
||||
log.info("start_compile_op: task id: {} op json:\n {}".format(task_id, op_json))
|
||||
if self.__tuner is None:
|
||||
log.error("Please confirm that the mode isn't NO_TUNE and auto_tune already initialized.")
|
||||
return error_id
|
||||
if not self.__tuner.tune_init:
|
||||
status = self.__tuner.init_tune_interface(op_json, self.tune_process_num)
|
||||
if not status:
|
||||
log.error("Auto tune init failed, place check your hardware config or go back to normal compile!")
|
||||
self.tune_init = False
|
||||
return error_id
|
||||
self.__reset_op_info = self.get_reset_op_info()
|
||||
|
|
@ -381,7 +370,6 @@ class TbeProcess:
|
|||
elif tune_mode == GA_TUNE:
|
||||
self.__tuner.ga_tune(task_id, op_json)
|
||||
else:
|
||||
log.error("Unsupported Tune Mode!")
|
||||
return error_id
|
||||
|
||||
return task_id
|
||||
|
|
@ -416,11 +404,7 @@ class TbeProcess:
|
|||
if not ret:
|
||||
query_count = query_count + 1
|
||||
time.sleep(30)
|
||||
log.info("{} of {} Task is Tuning({} Tasks tune fail),wait more 30 seconds...".format(
|
||||
len(self.__running_tune_tasks),
|
||||
len(self.__all_tune_tasks), len(self.__failed_tune_task)))
|
||||
else:
|
||||
log.info("get finish tasks:[{}]".format(ret))
|
||||
for item in ret:
|
||||
task_id = item['task_id']
|
||||
status_code = item['status_code']
|
||||
|
|
@ -430,21 +414,12 @@ class TbeProcess:
|
|||
res = task_id, "Success", compile_info
|
||||
else:
|
||||
self.__failed_tune_task.append(task_id)
|
||||
log.info("task_id:{}, json:{}".format(task_id, self.__task_info[task_id]))
|
||||
res = task_id, "Failed", compile_info
|
||||
self.__finish_tune_task.append(res)
|
||||
self.__running_tune_tasks.remove(task_id)
|
||||
ret = self.__finish_tune_task.pop()
|
||||
return ret
|
||||
log.error("Tune Task Timeout!!!")
|
||||
log.error("AllTaskNum:{}, RunningTaskNum:{}, FailedTaskNum:{}".format(len(self.__all_tune_tasks),
|
||||
len(self.__running_tune_tasks),
|
||||
len(self.__failed_tune_task)))
|
||||
return 0, "Failed", "Failed"
|
||||
log.error("All Task Is Done!!!")
|
||||
log.error("AllTaskNum:{}, RunningTaskNum:{}, FailedTaskNum:{}".format(len(self.__all_tune_tasks),
|
||||
len(self.__running_tune_tasks),
|
||||
len(self.__failed_tune_task)))
|
||||
return -1, "Failed", "Failed"
|
||||
|
||||
def reset_task_info(self):
|
||||
|
|
|
|||
|
|
@ -16,10 +16,7 @@
|
|||
import os
|
||||
import datetime
|
||||
import json
|
||||
import sys
|
||||
import traceback
|
||||
from tbe.common.rl_bank.bank_manager import set_current_op_name
|
||||
from te.platform.cce_conf import te_set_version
|
||||
from te_fusion.fusion_util import fusion_op, dump_fusion_json
|
||||
from te_fusion.parallel_compilation import init_multi_process_env, get_finished_compilation_task, \
|
||||
deinit_multi_process_env, start_ga_multi_process
|
||||
|
|
@ -27,7 +24,6 @@ from te_fusion.compile_task_manager import dispatch_autotune_task, import_py_mod
|
|||
import auto_tune
|
||||
from schedule_search.rl_online_tune import rl_tune_init, dispatch_fusion_tune_task, dispatch_single_tune_task, \
|
||||
rl_tune_deinit
|
||||
from mindspore import log
|
||||
from .compiler import build_op
|
||||
from .re_construct_json import single_to_fusion, fusion_to_fusion
|
||||
|
||||
|
|
@ -64,13 +60,9 @@ class TbeTuner:
|
|||
"""
|
||||
json_info = json.loads(json_str)
|
||||
soc_info = self.get_soc_info(json_info)
|
||||
cur_cce_product_params = te_set_version(*soc_info)
|
||||
if cur_cce_product_params is None:
|
||||
log.warning("Set Soc Info failed.")
|
||||
tune_mode = self.get_tune_mode(json_info)
|
||||
ret = self.parallel_compilation_init(soc_info, tune_mode, process_num)
|
||||
if not ret:
|
||||
log.error("Init parallel compilation env failed")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
|
@ -136,10 +128,8 @@ class TbeTuner:
|
|||
tune_bank_flag = False
|
||||
|
||||
if not os.path.isdir(base_custom_path):
|
||||
log.error("Check whether the tuning path [{}] exists.".format(base_custom_path))
|
||||
return
|
||||
if not os.access(base_custom_path, os.R_OK | os.W_OK | os.X_OK):
|
||||
log.error("Check whether the permission on the tuning path [{}] is correct.".format(base_custom_path))
|
||||
return
|
||||
|
||||
if not tune_bank_flag:
|
||||
|
|
@ -182,10 +172,8 @@ class TbeTuner:
|
|||
env_count = process_num
|
||||
if "TE_PARALLEL_COMPILER" in os.environ:
|
||||
env_count = os.getenv("TE_PARALLEL_COMPILER")
|
||||
log.info("TE_PARALLEL_COMPILER is set to {}".format(env_count))
|
||||
if int(env_count) > process_num:
|
||||
env_count = process_num
|
||||
log.info("change process count to {}".format(process_num))
|
||||
os.environ["TE_PARALLEL_COMPILER"] = str(int(env_count))
|
||||
pid_str = os.getpid()
|
||||
time_str = datetime.datetime.now().strftime('%Y%m%d_%H%M%S%f')[:-3]
|
||||
|
|
@ -204,34 +192,24 @@ class TbeTuner:
|
|||
else:
|
||||
# pylint: disable=no-else-return
|
||||
if te_log_level.isdigit() and int(te_log_level) >= len(TE_LOG_LEVEL):
|
||||
log.error(f"Invalid environment TE_LOGLEVEL, the value should be in [0, 4) if it is a digit, but got : "
|
||||
f"{te_log_level}")
|
||||
return False
|
||||
elif te_log_level.upper() not in TE_LOG_LEVEL:
|
||||
log.error(f"Invalid environment TE_LOGLEVEL, the value should be one of [DEBUG, INFO, WARNING, ERROR] "
|
||||
f"if it is a string, but got :{te_log_level}")
|
||||
return False
|
||||
global_loglevel = int(te_log_level) if te_log_level.isdigit() else TE_LOG_LEVEL.index(te_log_level.upper())
|
||||
ret = init_multi_process_env(embedding, soc_info, tune_mode, global_loglevel, enable_event, pid_ts)
|
||||
if ret is None:
|
||||
log.error("Init multiprocess env failed")
|
||||
return False
|
||||
self.multi_init = True
|
||||
process_count = ret[0]
|
||||
log.info("Init multiprocess env success with {} process".format(process_count))
|
||||
if "RL" in tune_mode:
|
||||
res_queue = ret[1]
|
||||
live_checker = ret[2]
|
||||
termin_event = ret[3]
|
||||
ret = rl_tune_init(soc_info, res_queue, live_checker, termin_event, global_loglevel, pid_ts)
|
||||
if not ret:
|
||||
log.error("RL env init failed!")
|
||||
return False
|
||||
self.rl_init = True
|
||||
log.info("RL Tune init success.")
|
||||
if "GA" in tune_mode:
|
||||
start_ga_multi_process(tune_mode)
|
||||
log.info("GA Tune init success.")
|
||||
return True
|
||||
|
||||
def sync_fusion_env(self):
|
||||
|
|
@ -302,9 +280,6 @@ class TbeTuner:
|
|||
compile_info, op_args, op_module_name = build_op(OP_BUILD, json.dumps(json_info), tune_mode)
|
||||
# pylint: disable=broad-except
|
||||
except Exception:
|
||||
exc_type, exc_value, _ = sys.exc_info()
|
||||
log.error(
|
||||
"exc_type:{}, exc_value:{}, exc_traceback:{}".format(exc_type, exc_value, traceback.format_exc()))
|
||||
return False, job_type, compile_info
|
||||
finally:
|
||||
pass
|
||||
|
|
@ -344,9 +319,6 @@ class TbeTuner:
|
|||
fusion_op(converted_json, reset_op_info=reset_op_info)
|
||||
# pylint: disable=broad-except
|
||||
except Exception:
|
||||
exc_type, exc_value, _ = sys.exc_info()
|
||||
log.error(
|
||||
"exc_type:{}, exc_value:{}, exc_traceback:{}".format(exc_type, exc_value, traceback.format_exc()))
|
||||
return False, job_type, compile_info
|
||||
if self.offline_tune:
|
||||
job_type = RL_OFFLINE
|
||||
|
|
|
|||
Loading…
Reference in New Issue