From 991925bfe178edb69536865fd8fe94369af026cf Mon Sep 17 00:00:00 2001 From: xinran Date: Sun, 14 Aug 2022 22:30:25 +0800 Subject: [PATCH] ADD file via upload --- .../algorithm/duration_time_model/template.py | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 src/gausskernel/dbmind/tools/components/sqldiag/algorithm/duration_time_model/template.py diff --git a/src/gausskernel/dbmind/tools/components/sqldiag/algorithm/duration_time_model/template.py b/src/gausskernel/dbmind/tools/components/sqldiag/algorithm/duration_time_model/template.py new file mode 100644 index 00000000..41b000ac --- /dev/null +++ b/src/gausskernel/dbmind/tools/components/sqldiag/algorithm/duration_time_model/template.py @@ -0,0 +1,137 @@ +""" +Copyright (c) 2020 Huawei Technologies Co.,Ltd. + +openGauss is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + + http://license.coscl.org.cn/MulanPSL2 + +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +""" +import heapq +import json +import logging +import os +import stat +from functools import reduce +from collections import defaultdict + +from ..sql_similarity import calc_sql_distance +from ...preprocessing import get_sql_template, templatize_sql +from ...utils import check_illegal_sql, LRUCache +from . import AbstractModel + + +class TemplateModel(AbstractModel): + # Initialize algorithm parameters + def __init__(self, params): + super().__init__(params) + self.bias = 1e-5 + self.__hash_table = dict(INSERT=dict(), UPDATE=dict(), DELETE=dict(), SELECT=dict(), + OTHER=dict()) + self.time_list_size = params.time_list_size + self.knn_number = params.knn_number + self.similarity_algorithm = calc_sql_distance(params.similarity_algorithm) + + # training method for template model + def fit(self, data): + for sql, duration_time in data: + if check_illegal_sql(sql): + continue + sql_template = templatize_sql(sql) + sql_prefix = sql_template.split()[0] + if sql_prefix not in self.__hash_table: + sql_prefix = 'OTHER' + if sql_template not in self.__hash_table[sql_prefix]: + self.__hash_table[sql_prefix][sql_template] = dict(time_list=[], count=0, mean_time=0.0, iter_time=0.0) + self.__hash_table[sql_prefix][sql_template]['count'] += 1 + self.__hash_table[sql_prefix][sql_template]['time_list'].append(duration_time) + + for sql_prefix, sql_prefix_info in self.__hash_table.items(): + for sql_template, sql_template_info in sql_prefix_info.items(): + del sql_template_info['time_list'][:-self.time_list_size] + sql_template_info['mean_time'] = sum(sql_template_info['time_list']) / len(sql_template_info['time_list']) + sql_template_info['iter_time'] = reduce(lambda x, y: (x+y)/2, sql_template_info['time_list']) + + + def transform(self, data): + predict_result_dict = defaultdict(list) + for sql in data: + sql_, status, predict_time, top_similarity_sql = self.predict_duration_time(sql) + predict_result_dict[status].append([sql_, predict_time, top_similarity_sql]) + for key, value in predict_result_dict.items(): + if value: + value.sort(key=lambda item: item[1], reverse=True) + return predict_result_dict + + @LRUCache(max_size=1024) + def predict_duration_time(self, sql): + top_similarity_sql = None + if check_illegal_sql(sql): + predict_time = -1 + status = 'Suspect illegal sql' + return sql, status, predict_time, top_similarity_sql + + sql_template = templatize_sql(sql) + # get 'sql_template' of SQL + sql_prefix = sql_template.strip().split()[0] + if sql_prefix not in self.__hash_table: + sql_prefix = 'OTHER' + if not self.__hash_table[sql_prefix]: + status = 'No SQL information' + predict_time = -1 + elif sql_template not in self.__hash_table[sql_prefix]: + similarity_info = [] + """ + if the template does not exist in the hash table, + then calculate the possible execution time based on template + similarity and KNN algorithm in all other templates + """ + status = 'No SQL template found' + for local_sql_template, local_sql_template_info in self.__hash_table[sql_prefix].items(): + similarity_info.append( + (self.similarity_algorithm(sql_template, local_sql_template), + local_sql_template_info['mean_time'], local_sql_template)) + topn_similarity_info = heapq.nlargest(self.knn_number, similarity_info) + sum_similarity_scores = sum(item[0] for item in topn_similarity_info) + if not sum_similarity_scores: + sum_similarity_scores = self.bias + top_similarity_sql = '\n'.join([item[2] for item in topn_similarity_info]) + similarity_proportions = [item[0] / sum_similarity_scores for item in + topn_similarity_info] + topn_duration_time = [item[1] for item in topn_similarity_info] + predict_time = reduce(lambda x, y: x + y, + map(lambda x, y: x * y, similarity_proportions, + topn_duration_time)) + + else: + status = 'Fine match' + predict_time = self.__hash_table[sql_prefix][sql_template]['iter_time'] + top_similarity_sql = sql_template + + return sql, status, predict_time, top_similarity_sql + + def load(self, filepath): + realpath = os.path.realpath(filepath) + if os.path.exists(realpath): + template_path = os.path.join(realpath, 'template.json') + with open(template_path, mode='r') as f: + self.__hash_table = json.load(f) + else: + logging.error("{} not exist.".format(realpath)) + + def save(self, filepath): + realpath = os.path.realpath(filepath) + if not os.path.exists(realpath): + os.makedirs(realpath, mode=0o700) + if oct(os.stat(realpath).st_mode)[-3:] != '700': + os.chmod(realpath, stat.S_IRWXU) + template_path = os.path.join(realpath, 'template.json') + with open(template_path, mode='w') as f: + json.dump(self.__hash_table, f, indent=4) + print("Template model is stored in '{}'".format(realpath)) +