From 6021c26abe79ea22d7b719850afdbdc2c3795a4c Mon Sep 17 00:00:00 2001 From: xinran Date: Sun, 14 Aug 2022 22:29:48 +0800 Subject: [PATCH] Delete 'src/gausskernel/dbmind/tools/components/sqldiag/algorithm/duration_time_model/template.py' --- .../algorithm/duration_time_model/template.py | 136 ------------------ 1 file changed, 136 deletions(-) delete mode 100644 src/gausskernel/dbmind/tools/components/sqldiag/algorithm/duration_time_model/template.py diff --git a/src/gausskernel/dbmind/tools/components/sqldiag/algorithm/duration_time_model/template.py b/src/gausskernel/dbmind/tools/components/sqldiag/algorithm/duration_time_model/template.py deleted file mode 100644 index ec20d0ede..000000000 --- a/src/gausskernel/dbmind/tools/components/sqldiag/algorithm/duration_time_model/template.py +++ /dev/null @@ -1,136 +0,0 @@ -""" -Copyright (c) 2020 Huawei Technologies Co.,Ltd. - -openGauss is licensed under Mulan PSL v2. -You can use this software according to the terms and conditions of the Mulan PSL v2. -You may obtain a copy of Mulan PSL v2 at: - - http://license.coscl.org.cn/MulanPSL2 - -THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, -EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, -MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. -See the Mulan PSL v2 for more details. -""" -import heapq -import json -import logging -import os -import stat -from functools import reduce -from collections import defaultdict - -from ..sql_similarity import calc_sql_distance -from ...preprocessing import get_sql_template, templatize_sql -from ...utils import check_illegal_sql, LRUCache -from . import AbstractModel - - -class TemplateModel(AbstractModel): - def __init__(self, params): - super().__init__(params) - self.bias = 1e-5 - self.__hash_table = dict(INSERT=dict(), UPDATE=dict(), DELETE=dict(), SELECT=dict(), - OTHER=dict()) - self.time_list_size = params.time_list_size - self.knn_number = params.knn_number - self.similarity_algorithm = calc_sql_distance(params.similarity_algorithm) - - # training method for template model - def fit(self, data): - for sql, duration_time in data: - if check_illegal_sql(sql): - continue - sql_template = templatize_sql(sql) - sql_prefix = sql_template.split()[0] - if sql_prefix not in self.__hash_table: - sql_prefix = 'OTHER' - if sql_template not in self.__hash_table[sql_prefix]: - self.__hash_table[sql_prefix][sql_template] = dict(time_list=[], count=0, mean_time=0.0, iter_time=0.0) - self.__hash_table[sql_prefix][sql_template]['count'] += 1 - self.__hash_table[sql_prefix][sql_template]['time_list'].append(duration_time) - - for sql_prefix, sql_prefix_info in self.__hash_table.items(): - for sql_template, sql_template_info in sql_prefix_info.items(): - del sql_template_info['time_list'][:-self.time_list_size] - sql_template_info['mean_time'] = sum(sql_template_info['time_list']) / len(sql_template_info['time_list']) - sql_template_info['iter_time'] = reduce(lambda x, y: (x+y)/2, sql_template_info['time_list']) - - - def transform(self, data): - predict_result_dict = defaultdict(list) - for sql in data: - sql_, status, predict_time, top_similarity_sql = self.predict_duration_time(sql) - predict_result_dict[status].append([sql_, predict_time, top_similarity_sql]) - for key, value in predict_result_dict.items(): - if value: - value.sort(key=lambda item: item[1], reverse=True) - return predict_result_dict - - @LRUCache(max_size=1024) - def predict_duration_time(self, sql): - top_similarity_sql = None - if check_illegal_sql(sql): - predict_time = -1 - status = 'Suspect illegal sql' - return sql, status, predict_time, top_similarity_sql - - sql_template = templatize_sql(sql) - # get 'sql_template' of SQL - sql_prefix = sql_template.strip().split()[0] - if sql_prefix not in self.__hash_table: - sql_prefix = 'OTHER' - if not self.__hash_table[sql_prefix]: - status = 'No SQL information' - predict_time = -1 - elif sql_template not in self.__hash_table[sql_prefix]: - similarity_info = [] - """ - if the template does not exist in the hash table, - then calculate the possible execution time based on template - similarity and KNN algorithm in all other templates - """ - status = 'No SQL template found' - for local_sql_template, local_sql_template_info in self.__hash_table[sql_prefix].items(): - similarity_info.append( - (self.similarity_algorithm(sql_template, local_sql_template), - local_sql_template_info['mean_time'], local_sql_template)) - topn_similarity_info = heapq.nlargest(self.knn_number, similarity_info) - sum_similarity_scores = sum(item[0] for item in topn_similarity_info) - if not sum_similarity_scores: - sum_similarity_scores = self.bias - top_similarity_sql = '\n'.join([item[2] for item in topn_similarity_info]) - similarity_proportions = [item[0] / sum_similarity_scores for item in - topn_similarity_info] - topn_duration_time = [item[1] for item in topn_similarity_info] - predict_time = reduce(lambda x, y: x + y, - map(lambda x, y: x * y, similarity_proportions, - topn_duration_time)) - - else: - status = 'Fine match' - predict_time = self.__hash_table[sql_prefix][sql_template]['iter_time'] - top_similarity_sql = sql_template - - return sql, status, predict_time, top_similarity_sql - - def load(self, filepath): - realpath = os.path.realpath(filepath) - if os.path.exists(realpath): - template_path = os.path.join(realpath, 'template.json') - with open(template_path, mode='r') as f: - self.__hash_table = json.load(f) - else: - logging.error("{} not exist.".format(realpath)) - - def save(self, filepath): - realpath = os.path.realpath(filepath) - if not os.path.exists(realpath): - os.makedirs(realpath, mode=0o700) - if oct(os.stat(realpath).st_mode)[-3:] != '700': - os.chmod(realpath, stat.S_IRWXU) - template_path = os.path.join(realpath, 'template.json') - with open(template_path, mode='w') as f: - json.dump(self.__hash_table, f, indent=4) - print("Template model is stored in '{}'".format(realpath)) -