Compare commits
3 Commits
| Author | SHA1 | Date |
|---|---|---|
|
|
478c99f128 | |
|
|
7cfa2f839e | |
|
|
7cfaa5b963 |
|
|
@ -4454,7 +4454,16 @@ void getSubscriptions(Archive *fout)
|
|||
}
|
||||
|
||||
if (!isExecUserSuperRole(fout)) {
|
||||
write_msg(NULL, "WARNING: subscriptions not dumped because current user is not a superuser\n");
|
||||
res = ExecuteSqlQuery(fout,
|
||||
"SELECT count(*) FROM pg_subscription "
|
||||
"WHERE subdbid = (SELECT oid FROM pg_catalog.pg_database"
|
||||
" WHERE datname = current_database())",
|
||||
PGRES_TUPLES_OK);
|
||||
uint64 n = (res != NULL) ? strtoul(PQgetvalue(res, 0, 0), NULL, 10) : 0;
|
||||
if (n > 0) {
|
||||
write_msg(NULL, "WARNING: subscriptions not dumped because current user is not a superuser\n");
|
||||
}
|
||||
PQclear(res);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -10786,11 +10795,6 @@ static void dumpDirectory(Archive* fout)
|
|||
char* dirpath = NULL;
|
||||
char* diracl = NULL;
|
||||
|
||||
if (!isExecUserSuperRole(fout)) {
|
||||
write_msg(NULL, "WARNING: directory not dumped because current user is not a superuser\n");
|
||||
return;
|
||||
}
|
||||
|
||||
/* Make sure we are in proper schema */
|
||||
selectSourceSchema(fout, "pg_catalog");
|
||||
|
||||
|
|
@ -21400,11 +21404,6 @@ static void dumpSynonym(Archive* fout)
|
|||
PQExpBuffer q;
|
||||
PQExpBuffer delq;
|
||||
|
||||
if (!isExecUserSuperRole(fout)) {
|
||||
write_msg(NULL, "WARNING: synonym not dumped because current user is not a superuser\n");
|
||||
return;
|
||||
}
|
||||
|
||||
selectSourceSchema(fout, "pg_catalog");
|
||||
query = createPQExpBuffer();
|
||||
printfPQExpBuffer(query,
|
||||
|
|
|
|||
|
|
@ -47,8 +47,7 @@ static void InternalAggIsSupported(const char *aggName)
|
|||
"json_agg",
|
||||
"json_object_agg",
|
||||
"st_summarystatsagg",
|
||||
"st_union",
|
||||
"wm_concat"
|
||||
"st_union"
|
||||
};
|
||||
|
||||
uint len = lengthof(supportList);
|
||||
|
|
|
|||
|
|
@ -309,7 +309,6 @@ bool pg_md5_encrypt(const char* passwd, const char* salt, size_t salt_len, char*
|
|||
{
|
||||
size_t passwd_len = strlen(passwd);
|
||||
errno_t rc = EOK;
|
||||
/* the length of salt and password is <= SIZE_MAX */
|
||||
#ifndef WIN32
|
||||
if (unlikely(passwd_len >= SIZE_MAX - salt_len)) {
|
||||
return false;
|
||||
|
|
@ -323,7 +322,6 @@ bool pg_md5_encrypt(const char* passwd, const char* salt, size_t salt_len, char*
|
|||
char* crypt_buf = (char*)malloc(passwd_len + salt_len + 1);
|
||||
bool ret = false;
|
||||
|
||||
/* the buffer is not exist */
|
||||
if (crypt_buf == NULL)
|
||||
return false;
|
||||
|
||||
|
|
|
|||
|
|
@ -772,15 +772,6 @@ bool pg_sha256_encrypt_for_md5(const char* password, const char* salt, size_t sa
|
|||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* @Description: calculate the encrypted password for GsSm3.
|
||||
* @const char* password : the password need be encrypted.
|
||||
* @const char* salt_s : the content fo the slat.
|
||||
* @size_t salt_len : the length fo the slat.
|
||||
* @char* buf : the buffer to store the encrypted key with GsSm3.
|
||||
* @char* client_key_buf : the buffer to store the key of client.
|
||||
* @int iteration_count : to record the number of the iteration.
|
||||
*/
|
||||
bool GsSm3Encrypt(
|
||||
const char* password, const char* salt_s, size_t salt_len, char* buf, char* client_key_buf, int iteration_count)
|
||||
{
|
||||
|
|
@ -808,7 +799,6 @@ bool GsSm3Encrypt(
|
|||
}
|
||||
|
||||
password_len = strlen(password);
|
||||
/* Tranform string(64Bytes) to binary(32Bytes) */
|
||||
sha_hex_to_bytes32(salt, (char*)salt_s);
|
||||
/* calculate k */
|
||||
pkcs_ret = PKCS5_PBKDF2_HMAC((char*)password,
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -14,23 +14,11 @@ import os
|
|||
|
||||
from . import feature_mapping
|
||||
from . import features
|
||||
# To import file feature_mapping and features from parent folder
|
||||
|
||||
#function name: load_feature_lib
|
||||
#description: Print the variable FEATURE_LIB in the file-- features
|
||||
#return value: The value of FEATURE_LIB
|
||||
#date: 2022/8/2
|
||||
#contact: 1865997821
|
||||
|
||||
def load_feature_lib():
|
||||
return features.FEATURE_LIB
|
||||
|
||||
#function name: get_feature_mapper
|
||||
#description: Get the item and value of a dictionary type in the file-- feature_mapping and output it as a generator.
|
||||
#return value: The item and value in _dict_ variable
|
||||
#note:Dictionary key-value pairs must start with C then the item and value will be return.
|
||||
#date: 2022/8/2
|
||||
#contact: 1865997821
|
||||
|
||||
def get_feature_mapper():
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -11,27 +11,22 @@
|
|||
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
||||
# See the Mulan PSL v2 for more details.
|
||||
import csv
|
||||
#import csv packet
|
||||
from collections import defaultdict
|
||||
from typing import List
|
||||
# To import defaultdict in the parent floder collections and List in the parent floder typing
|
||||
|
||||
import numpy as np
|
||||
# import numpy packet as the name np
|
||||
|
||||
from ..analyzer import _euclid_distance as euclid_distance
|
||||
from dbmind.common.utils import ExceptionCatch
|
||||
#To import private function-- _euclid_distance as euclid_distance
|
||||
|
||||
#function name: calculate_weight
|
||||
#description: This function will output feature_weight (= residual_vector / the sum of residual_vector)
|
||||
#The data used for the calculation is from the features_labels_dict, and the key value pairs of the features_labels_dict are filtered
|
||||
#arguments: np.ndarray and np.ndarray
|
||||
#return value: weight_matrix
|
||||
#date: 2022/8/2
|
||||
#contact: 1865997821
|
||||
|
||||
def calculate_weight(features: np.ndarray, labels: np.ndarray) -> List:
|
||||
"""
|
||||
Calculate weight matrix based on feature set
|
||||
:param features: feature set
|
||||
:param labels: label set
|
||||
:return: weight_matrix
|
||||
"""
|
||||
normalize_features, normalize_labels = [], []
|
||||
features_labels_dict = defaultdict(list)
|
||||
for i in range(len(labels)):
|
||||
|
|
@ -61,16 +56,6 @@ def calculate_weight(features: np.ndarray, labels: np.ndarray) -> List:
|
|||
return weight_matrix
|
||||
|
||||
|
||||
# function name: build_model
|
||||
# description: Create two variables-- features and labels.There are refer to two numpy array(all elements are zero)
|
||||
# The features array's size is feature_number and dimension is feature_dimension
|
||||
# This function will read the two arrays and write it as a matrix in a csv file(the save path is './features_new.npz')
|
||||
# And then it will call the function calculate_weight to calculate the matrix
|
||||
# arguments: feature_path, feature_number, feature_dimension
|
||||
# return value: None
|
||||
# note:A ExceptionCatch function modifier is used
|
||||
# date: 2022/8/2
|
||||
#contact: 1865997821
|
||||
@ExceptionCatch(strategy='exit', name='FEATURE')
|
||||
def build_model(feature_path: str, feature_number: int, feature_dimension: int,
|
||||
save_path: str = './features_new.npz') -> None:
|
||||
|
|
|
|||
|
|
@ -11,13 +11,6 @@
|
|||
# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
|
||||
# See the Mulan PSL v2 for more details.
|
||||
|
||||
#function name: detect
|
||||
#description: if the method is "bool" type, then call the functions sum_detect、avg_detect、ks_detect to diagnose errors
|
||||
#These functions are in the parent slow_sql/significance_detection
|
||||
#arguments: data1(array), data2(array), method
|
||||
#return value: bool type
|
||||
#date: 2022/8/2
|
||||
#contact: 1865997821
|
||||
|
||||
def detect(data1, data2, method='bool', threshold=0.01, p_value=0.5):
|
||||
if method == 'bool':
|
||||
|
|
|
|||
|
|
@ -12,16 +12,17 @@
|
|||
# See the Mulan PSL v2 for more details.
|
||||
|
||||
alpha = 1e-10
|
||||
#Define a minimum number of errors
|
||||
|
||||
#function name: detect
|
||||
#description: Calculate whether the data has abrupt changes based on the average value
|
||||
#arguments: data1, data2, threshold,method
|
||||
#return value: bool
|
||||
#date: 2022/8/
|
||||
#contact: 1865997821
|
||||
|
||||
def detect(data1, data2, threshold=0.5, method='bool'):
|
||||
"""
|
||||
Calculate whether the data has abrupt changes based on the average value
|
||||
:param data1: input data array
|
||||
:param data2: input data array
|
||||
:param threshold: Mutation rate
|
||||
:param method: The way to calculate the mutation
|
||||
:return: bool
|
||||
"""
|
||||
if not isinstance(data1, list) or not isinstance(data2, list):
|
||||
raise TypeError("The format of the input data is wrong.")
|
||||
avg1 = sum(data1) / len(data1) if data1 else 0
|
||||
|
|
|
|||
|
|
@ -13,14 +13,8 @@
|
|||
import sys
|
||||
|
||||
from .cli import DBMindRun
|
||||
#To import DBMindRun method from the parent file cli
|
||||
|
||||
#function name: main
|
||||
#description: Get the system command parameters, pass to the DBMindRun and call this function,if an InterruptedError is reported, the program will exit( sys.exit(1)).
|
||||
#arguments: None
|
||||
#return value: None
|
||||
#date: 2022/8/3
|
||||
#contact: 1865997821
|
||||
|
||||
def main() -> None:
|
||||
try:
|
||||
DBMindRun(sys.argv[1:])
|
||||
|
|
|
|||
|
|
@ -55,12 +55,7 @@ CONFIG_OPTIONS = {
|
|||
'LOG-level': ['DEBUG', 'INFO', 'WARNING', 'ERROR']
|
||||
}
|
||||
|
||||
#function name: check_config_validity
|
||||
#description: Checks the validity of the passed parameter
|
||||
#arguments: section, option, value
|
||||
#return value: bool and string
|
||||
#date: 2022/8/
|
||||
#contact: 1865997821
|
||||
|
||||
def check_config_validity(section, option, value):
|
||||
config_item = '%s-%s' % (section, option)
|
||||
# exceptional cases:
|
||||
|
|
@ -92,16 +87,6 @@ def check_config_validity(section, option, value):
|
|||
return True, None
|
||||
|
||||
|
||||
|
||||
#function name: load_sys_configs
|
||||
#description: Create and load the modification file
|
||||
#arguments: The configuration to modify
|
||||
#return value: a new configuration file
|
||||
#note:To facilitate the user to modify the configuration items through the
|
||||
#configuration file easily, we add inline comments to the file, but we need to remove the inline comments while parsing.
|
||||
#Otherwise, it will cause the read configuration items to be wrong.
|
||||
#date: 2022/8/
|
||||
#contact: 1865997821
|
||||
def load_sys_configs(confile):
|
||||
# Note: To facilitate the user to modify the configuration items through the
|
||||
# configuration file easily, we add inline comments to the file, but we need
|
||||
|
|
@ -111,8 +96,6 @@ def load_sys_configs(confile):
|
|||
with open(file=confile, mode='r') as fp:
|
||||
configs.read_file(fp)
|
||||
|
||||
|
||||
# Define a class that encapsulates the modification item
|
||||
class ConfigWrapper(object):
|
||||
def __getattribute__(self, name):
|
||||
try:
|
||||
|
|
@ -139,7 +122,7 @@ def load_sys_configs(confile):
|
|||
|
||||
return ConfigWrapper()
|
||||
|
||||
# Defines a class that updates the encapsulated modification file
|
||||
|
||||
class ConfigUpdater:
|
||||
def __init__(self, filepath):
|
||||
self.config = ConfigParser(inline_comment_prefixes=None)
|
||||
|
|
@ -187,7 +170,7 @@ class ConfigUpdater:
|
|||
self.fp.flush()
|
||||
self.fp.close()
|
||||
|
||||
# Defines a class that dynamically displays a modified item
|
||||
|
||||
class DynamicConfig:
|
||||
@staticmethod
|
||||
def get(*args, **kwargs):
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ except ImportError:
|
|||
|
||||
SKIP_LIST = ('COMMENT', 'LOG')
|
||||
|
||||
# The global variable acts as a switch that controls whether the program runs
|
||||
dbmind_master_should_exit = False
|
||||
|
||||
|
||||
|
|
@ -58,16 +57,8 @@ def _process_clean(force=False):
|
|||
global_vars.worker.terminate(cancel_futures=force)
|
||||
TimedTaskManager.stop()
|
||||
|
||||
#function name: signal_handler
|
||||
#description: The function processes the received signal parameters, reassigns variable x according to different signals
|
||||
#or calls other functions to complete the content indicated by signals
|
||||
#arguments: signum, frame
|
||||
#return value: bool (dbmind_master_should_exit)
|
||||
#date: 2022/8/3
|
||||
#contact: 1865997821
|
||||
|
||||
def signal_handler(signum, frame):
|
||||
# The global variable dbmind_master_should_exit can be modified in this function to continue to play a control role
|
||||
global dbmind_master_should_exit
|
||||
|
||||
if signum == signal.SIGINT or signum == signal.SIGHUP:
|
||||
|
|
@ -157,12 +148,10 @@ class DBMindMain(Daemon):
|
|||
time.sleep(1)
|
||||
logging.info('DBMind will close.')
|
||||
|
||||
# Emptying the execution pool
|
||||
def clean(self):
|
||||
if os.path.exists(self.pid_file):
|
||||
os.unlink(self.pid_file)
|
||||
|
||||
# Reload the execution pool and solve the error
|
||||
|
||||
def reload(self):
|
||||
pid = read_dbmind_pid_file(self.pid_file)
|
||||
if pid > 0:
|
||||
|
|
|
|||
|
|
@ -27,17 +27,6 @@ def do_after(rt_result):
|
|||
def do_exception(exception):
|
||||
"""Nothing"""
|
||||
|
||||
|
||||
#function name: around
|
||||
#description: Preserve the function properties and prevent an error from terminating the program
|
||||
#arguments: One or more functions
|
||||
#return value: none
|
||||
#note: Decorators are implemented in such a way that the function being decorated is actually another function (the function name and other properties change).
|
||||
#To avoid this, Python's FuncTools package provides a decorator called wraps to remove such side effects.
|
||||
#When writing a decorator, it is a good idea to wrap FuncTools before implementing it.
|
||||
#It preserves the name and properties of the original function
|
||||
#date: 2022/8/4
|
||||
#contact: 1865997821
|
||||
def around(func, *args, **kw):
|
||||
@wraps(func)
|
||||
def wrapper():
|
||||
|
|
|
|||
|
|
@ -15,11 +15,7 @@ from typing import Optional, Iterable, Union
|
|||
from .root_cause import RootCause
|
||||
from .enumerations import ALARM_TYPES, ALARM_LEVEL
|
||||
|
||||
#Define an Alarm class that takes the error parameters entered by the user and displays the error content and cause
|
||||
#method:Display the error content and suggestions, and retrieve suggestions provided by the system. If there are no suggestions, return “ no suggestions”
|
||||
#note:The property decorator turns a method into a property call.(root_causes、suggestions)
|
||||
#date:2022/8/4
|
||||
#contact:18365997821
|
||||
|
||||
class Alarm:
|
||||
def __init__(self,
|
||||
host: Union[str],
|
||||
|
|
|
|||
|
|
@ -12,11 +12,7 @@
|
|||
# See the Mulan PSL v2 for more details.
|
||||
from .root_cause import RootCause
|
||||
|
||||
#Define anSlowQuery class thatSlow query accepts user input commands and performs operations on the database
|
||||
#method:Display the error content and suggestions, and retrieve suggestions provided by the system. If there are no suggestions, return “ no suggestions”
|
||||
#note:The property decorator turns a method into a property call.(root_causes、suggestions)
|
||||
#date:2022/8/4
|
||||
#contact:18365997821
|
||||
|
||||
class SlowQuery:
|
||||
def __init__(self, db_host, db_port, db_name, schema_name, query, start_timestamp, duration_time,
|
||||
hit_rate=None, fetch_rate=None, cpu_time=None, data_io_time=None, template_id=None, sort_count=None,
|
||||
|
|
|
|||
|
|
@ -18,19 +18,13 @@ import psycopg2
|
|||
from .execute_factory import ExecuteFactory
|
||||
from .execute_factory import IndexInfo
|
||||
|
||||
#class name: DriverExecute (Inherits from the parent class ExecuteFactory)
|
||||
#description: The SQL statement performs the operations associated with the call
|
||||
#date: 2022/8/10
|
||||
#contact: 1865997821
|
||||
|
||||
class DriverExecute(ExecuteFactory):
|
||||
def __init__(self, *arg):
|
||||
#Call the arguments of the parent class __init__ method
|
||||
super(DriverExecute, self).__init__(*arg)
|
||||
self.conn = None
|
||||
self.cur = None
|
||||
|
||||
#Connecting to the database
|
||||
def init_conn_handle(self):
|
||||
self.conn = psycopg2.connect(dbname=self.dbname,
|
||||
user=self.user,
|
||||
|
|
@ -39,7 +33,6 @@ class DriverExecute(ExecuteFactory):
|
|||
port=self.port)
|
||||
self.cur = self.conn.cursor()
|
||||
|
||||
#If an error occurs after the SQL statement is executed, the error information is reported to the user
|
||||
def execute(self, sql):
|
||||
try:
|
||||
self.cur.execute(sql)
|
||||
|
|
@ -48,13 +41,11 @@ class DriverExecute(ExecuteFactory):
|
|||
except Exception:
|
||||
self.conn.commit()
|
||||
|
||||
#Disconnecting from the database
|
||||
def close_conn(self):
|
||||
if self.conn and self.cur:
|
||||
self.cur.close()
|
||||
self.conn.close()
|
||||
|
||||
#Check whether multiple nodes exist
|
||||
def is_multi_node(self):
|
||||
self.init_conn_handle()
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -13,11 +13,6 @@
|
|||
|
||||
import re
|
||||
|
||||
#class name: IndexInfo
|
||||
#description: Define information about table indexes
|
||||
#methods: __init__
|
||||
#date: 2022/8/10
|
||||
#contact: 1865997821
|
||||
|
||||
class IndexInfo:
|
||||
def __init__(self, schema, table, indexname, columns, indexdef):
|
||||
|
|
@ -29,9 +24,7 @@ class IndexInfo:
|
|||
self.primary_key = False
|
||||
self.redundant_obj = []
|
||||
|
||||
#class name: ExecuteFactory
|
||||
#date: 2022/8/10
|
||||
#contact: 1865997821
|
||||
|
||||
class ExecuteFactory:
|
||||
def __init__(self, dbname, user, password, host, port, schema, multi_node, max_index_storage):
|
||||
self.dbname = dbname
|
||||
|
|
@ -43,11 +36,11 @@ class ExecuteFactory:
|
|||
self.max_index_storage = max_index_storage
|
||||
self.multi_node = multi_node
|
||||
|
||||
# Record redundant indexes
|
||||
@staticmethod
|
||||
def record_redundant_indexes(cur_table_indexes, redundant_indexes):
|
||||
cur_table_indexes = sorted(cur_table_indexes,
|
||||
key=lambda index_obj: len(index_obj.columns.split(',')))
|
||||
# record redundant indexes
|
||||
for pos, index in enumerate(cur_table_indexes[:-1]):
|
||||
is_redundant = False
|
||||
for candidate_index in cur_table_indexes[pos + 1:]:
|
||||
|
|
@ -59,7 +52,6 @@ class ExecuteFactory:
|
|||
if is_redundant:
|
||||
redundant_indexes.append(index)
|
||||
|
||||
#Match the name of the table against the index of the query
|
||||
@staticmethod
|
||||
def match_table_name(table_name, query_index_dict):
|
||||
for elem in query_index_dict.keys():
|
||||
|
|
@ -74,7 +66,6 @@ class ExecuteFactory:
|
|||
return False, table_name
|
||||
return True, table_name
|
||||
|
||||
#Retrieves a valid index based on the regular expression, adding the corresponding index and empty element if none exists
|
||||
@staticmethod
|
||||
def get_valid_indexes(record, hypoid_table_column, valid_indexes):
|
||||
tokens = record.split(' ')
|
||||
|
|
@ -97,7 +88,6 @@ class ExecuteFactory:
|
|||
if columns not in valid_indexes[table_name]:
|
||||
valid_indexes[table_name].append((columns, index_type))
|
||||
|
||||
#Record invalid SQL statements and returns the corresponding help information that matches the corresponding SQL statement
|
||||
@staticmethod
|
||||
def record_ineffective_negative_sql(candidate_index, obj, ind):
|
||||
cur_table = candidate_index.table
|
||||
|
|
@ -135,7 +125,6 @@ class ExecuteFactory:
|
|||
candidate_index.ineffective_pos.append(ind)
|
||||
candidate_index.total_sql_num += obj.frequency
|
||||
|
||||
#Returns the last input and the corresponding result
|
||||
@staticmethod
|
||||
def match_last_result(table_name, index_column, history_indexes, history_invalid_indexes):
|
||||
for column in history_indexes.get(table_name, dict()):
|
||||
|
|
@ -153,7 +142,6 @@ class ExecuteFactory:
|
|||
if not history_indexes[table_name]:
|
||||
del history_indexes[table_name]
|
||||
|
||||
#Correcting SQL statements
|
||||
@staticmethod
|
||||
def make_single_advisor_sql(ori_sql):
|
||||
sql = 'select gs_index_advise(\''
|
||||
|
|
|
|||
|
|
@ -23,16 +23,12 @@ from .execute_factory import IndexInfo
|
|||
|
||||
BASE_CMD = None
|
||||
|
||||
#class name: GSqlExecute
|
||||
#description: Solve the optimization problem of GSQL statement execution
|
||||
#date: 2022/8/11
|
||||
#contact: 1865997821
|
||||
|
||||
class GSqlExecute(ExecuteFactory):
|
||||
def __init__(self, *args):
|
||||
super(GSqlExecute, self).__init__(*args)
|
||||
|
||||
def init_conn_handle(self):
|
||||
#define a global variable BASE_CMD,it is a connection command statement
|
||||
global BASE_CMD
|
||||
BASE_CMD = 'gsql -p ' + str(self.port) + ' -d ' + self.dbname
|
||||
if self.host:
|
||||
|
|
@ -42,7 +38,6 @@ class GSqlExecute(ExecuteFactory):
|
|||
if self.password:
|
||||
BASE_CMD += ' -W ' + self.password
|
||||
|
||||
#Run the shell command in BASE_CMD
|
||||
def run_shell_cmd(self, target_sql_list):
|
||||
cmd = BASE_CMD + ' -c \"'
|
||||
if self.schema:
|
||||
|
|
@ -52,7 +47,6 @@ class GSqlExecute(ExecuteFactory):
|
|||
cmd += '\"'
|
||||
proc = subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
|
||||
#Read data from stdout and stderr,If an error message is displayed, an error message is displayed
|
||||
(stdout, stderr) = proc.communicate()
|
||||
stdout, stderr = stdout.decode(), stderr.decode()
|
||||
if 'gsql: FATAL:' in stderr or 'failed to connect' in stderr:
|
||||
|
|
@ -80,7 +74,6 @@ class GSqlExecute(ExecuteFactory):
|
|||
print(e.output.decode(), file=sys.stderr)
|
||||
return int(ret.decode().strip().split()[2]) > 0
|
||||
|
||||
#Parse the recommended result returned
|
||||
@staticmethod
|
||||
def parse_single_advisor_result(res, table_index_dict):
|
||||
if len(res) > 2 and res[0:2] == ' (':
|
||||
|
|
@ -190,7 +183,6 @@ class GSqlExecute(ExecuteFactory):
|
|||
total_cost = 0
|
||||
found_plan = False
|
||||
hypo_index = False
|
||||
# create hypo-indexes
|
||||
for line in res:
|
||||
if 'QUERY PLAN' in line:
|
||||
found_plan = True
|
||||
|
|
@ -230,7 +222,6 @@ class GSqlExecute(ExecuteFactory):
|
|||
i += 1
|
||||
return total_cost
|
||||
|
||||
#Production workflows consume report files
|
||||
def estimate_workload_cost_file(self, workload, index_config=None, ori_indexes_name=None):
|
||||
sql_file = str(time.time()) + '.sql'
|
||||
is_computed = False
|
||||
|
|
@ -273,7 +264,6 @@ class GSqlExecute(ExecuteFactory):
|
|||
|
||||
return total_cost
|
||||
|
||||
#Check for empty indexes and note them to optimize the table structure
|
||||
def check_useless_index(self, history_indexes, history_invalid_indexes):
|
||||
schemas = [elem.lower()
|
||||
for elem in filter(None, self.schema.split(','))]
|
||||
|
|
|
|||
|
|
@ -539,14 +539,13 @@ class RnnModel():
|
|||
keras.backend.clear_session()
|
||||
set_session(self.session)
|
||||
with self.graph.as_default():
|
||||
# Judge whether the model needs to be initialized according to the changes of the model input and output dimensions.
|
||||
feature, label, need_init = self.parse(filename)
|
||||
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
|
||||
epsilon = self.model_info.make_epsilon()
|
||||
if need_init:# Cold start training
|
||||
if need_init:
|
||||
epoch_start = 0
|
||||
self.model = self._build_model(epsilon)
|
||||
else:# Incremental training
|
||||
else:
|
||||
epoch_start = int(self.model_info.last_epoch)
|
||||
ratio_error = ratio_error_loss_wrapper(epsilon)
|
||||
ratio_acc_2 = ratio_error_acc_wrapper(epsilon, 2)
|
||||
|
|
@ -557,16 +556,12 @@ class RnnModel():
|
|||
log_path = os.path.realpath(os.path.join(settings.PATH_LOG, self.model_info.model_name + '_log.json'))
|
||||
if not os.path.exists(log_path):
|
||||
os.mknod(log_path, mode=0o600)
|
||||
# Training logging callback function
|
||||
json_logging_callback = LossHistory(log_path, self.model_info.model_name, self.model_info.last_epoch)
|
||||
# Data segmentation
|
||||
X_train, X_val, y_train, y_val = \
|
||||
train_test_split(feature, label, test_size=0.1)
|
||||
# model training
|
||||
self.model.fit(X_train, y_train, epochs=self.model_info.last_epoch,
|
||||
batch_size=int(self.model_info.batch_size), validation_data=(X_val, y_val),
|
||||
verbose=0, initial_epoch=epoch_start, callbacks=[json_logging_callback])
|
||||
# save model
|
||||
self.model.save(self.model_info.model_path)
|
||||
val_pred = self.model.predict(X_val)
|
||||
val_re = get_ratio_errors_general(val_pred, y_val, epsilon)
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ from . import AbstractModel
|
|||
|
||||
|
||||
class TemplateModel(AbstractModel):
|
||||
# Initialize algorithm parameters
|
||||
def __init__(self, params):
|
||||
super().__init__(params)
|
||||
self.bias = 1e-5
|
||||
|
|
|
|||
|
|
@ -173,16 +173,11 @@ def procedure_main(mode, db_info, config):
|
|||
def rl_model(mode, env, config):
|
||||
# Lazy loading. Because loading Tensorflow takes a long time.
|
||||
from tuner.algorithms.rl_agent import RLAgent
|
||||
# Start reinforcement learning agent class.
|
||||
rl = RLAgent(env, alg=config['rl_algorithm'])
|
||||
# The two modes of training and tuning correspond to different execution processes.
|
||||
# The model needs to be trained before it can be used for tuning. The output of the training and tuning process is the list of parameters to be tuned. Because they share a set of models, it is required that the list of parameters to be tuned must be consistent in the two modes, otherwise exceptions with different output dimensions will be thrown.
|
||||
if mode == 'train':
|
||||
logging.warning('The list of tuned knobs in the training mode '
|
||||
'based on the reinforcement learning algorithm must be the same as '
|
||||
'that in the tuning mode. ')
|
||||
# The key parameter is the maximum iteration round rl_ steps, theoretically, the longer the more accurate, but also more time-consuming.
|
||||
# max_episode_steps is the maximum number of rounds in each round of reinforcement learning algorithm. In the implementation of x-tuner, this parameter is weakened, and it is generally default.
|
||||
rl.fit(config['rl_steps'], nb_max_episode_steps=config['max_episode_steps'])
|
||||
rl.save(config['rl_model_path'])
|
||||
logging.info('Saved reinforcement learning model at %s.', config['rl_model_path'])
|
||||
|
|
@ -205,7 +200,6 @@ def rl_model(mode, env, config):
|
|||
|
||||
def global_search(env, config):
|
||||
method = config['gop_algorithm']
|
||||
# Determine which algorithm to use.
|
||||
if method == 'bayes':
|
||||
from bayes_opt import BayesianOptimization
|
||||
|
||||
|
|
@ -213,13 +207,6 @@ def global_search(env, config):
|
|||
pbound = {name: (0, 1) for name in env.db.ordered_knob_list}
|
||||
|
||||
def performance_function(**params):
|
||||
"""
|
||||
function name: performance_function
|
||||
description: Define a black box function to adapt to the interface of the third-party library.
|
||||
author: Li Xinran
|
||||
date: 2022/8/4
|
||||
contact: 19154068808
|
||||
"""
|
||||
if not len(params) == env.nb_actions:
|
||||
raise AssertionError('Failed to check the input feature dimension.')
|
||||
|
||||
|
|
@ -235,21 +222,12 @@ def global_search(env, config):
|
|||
pbounds=pbound
|
||||
)
|
||||
optimizer.maximize(
|
||||
# The larger the maximum iteration round, the more accurate the result is, but it is also more time-consuming.
|
||||
n_iter=config['max_iterations']
|
||||
)
|
||||
elif method == 'pso':
|
||||
from tuner.algorithms.pso import Pso
|
||||
|
||||
def performance_function(v):
|
||||
"""
|
||||
function name: performance_function
|
||||
description: Find the global minimum value.
|
||||
note: Because the implementation of PSO algorithm is to find the global minimum value, take the opposite number here, so we need to change to take the global maximum value.
|
||||
author: Li Xinran
|
||||
date: 2022/8/4
|
||||
contact: 19154068808
|
||||
"""
|
||||
s, r, d, _ = env.step(v)
|
||||
return -r # Use -reward because PSO wishes to minimize.
|
||||
|
||||
|
|
@ -259,7 +237,6 @@ def global_search(env, config):
|
|||
particle_nums=config['particle_nums'],
|
||||
# max_iterations on the PSO indicates the maximum number of iterations per particle,
|
||||
# so it must be divided by the number of particles to be consistent with Bayes.
|
||||
# The larger the maximum iteration round is, the more accurate the result is, but also the more time-consuming.
|
||||
max_iteration=config['max_iterations'] // config['particle_nums'],
|
||||
x_min=0, x_max=1, max_vel=0.5
|
||||
)
|
||||
|
|
|
|||
|
|
@ -90,10 +90,7 @@ static void DropExtensionInListIsSupported(List* objname)
|
|||
}
|
||||
}
|
||||
|
||||
/* Enable DROP operation of the above objects during inplace upgrade or support_extended_features is true */
|
||||
if (!u_sess->attr.attr_common.IsInplaceUpgrade && !g_instance.attr.attr_common.support_extended_features) {
|
||||
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("EXTENSION is not yet supported.")));
|
||||
}
|
||||
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("EXTENSION is not yet supported.")));
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -5911,7 +5911,6 @@ Datum calculate_encrypted_combined_password(const char* password, const char* ro
|
|||
errno_t rc = EOK;
|
||||
|
||||
/* For PG ecological compatibility, we stored both sha256 and md5 password. */
|
||||
/* the encrypted method of sha256 */
|
||||
if (!pg_sha256_encrypt(password,
|
||||
salt_string,
|
||||
strlen(salt_string),
|
||||
|
|
@ -5922,7 +5921,7 @@ Datum calculate_encrypted_combined_password(const char* password, const char* ro
|
|||
securec_check(rc, "\0", "\0");
|
||||
ereport(ERROR, (errcode(ERRCODE_INVALID_PASSWORD), errmsg("first stage encryption password failed")));
|
||||
}
|
||||
/* the encrypted method of md5 */
|
||||
|
||||
if (!pg_md5_encrypt(password, rolname, strlen(rolname), encrypted_md5_password)) {
|
||||
rc = memset_s(encrypted_md5_password, MD5_PASSWD_LEN + 1, 0, MD5_PASSWD_LEN + 1);
|
||||
securec_check(rc, "\0", "\0");
|
||||
|
|
@ -6053,7 +6052,6 @@ static Datum gs_calculate_encrypted_sm3_password(const char* password, const cha
|
|||
Datum calculate_encrypted_password(bool is_encrypted, const char* password, const char* rolname,
|
||||
const char* salt_string)
|
||||
{
|
||||
/* If the password is '\0' or not exist */
|
||||
if (password == NULL || password[0] == '\0') {
|
||||
ereport(ERROR, (errcode(ERRCODE_INVALID_PASSWORD), errmsg("The password could not be NULL.")));
|
||||
}
|
||||
|
|
@ -6061,7 +6059,6 @@ Datum calculate_encrypted_password(bool is_encrypted, const char* password, cons
|
|||
char encrypted_md5_password[MD5_PASSWD_LEN + 1] = {0};
|
||||
Datum datum_value;
|
||||
|
||||
/* If the password has encrypted */
|
||||
if (!is_encrypted || isPWDENCRYPTED(password)) {
|
||||
return CStringGetTextDatum(password);
|
||||
}
|
||||
|
|
@ -6071,7 +6068,6 @@ Datum calculate_encrypted_password(bool is_encrypted, const char* password, cons
|
|||
* if Password_encryption_type is 0, the encrypted password is md5.
|
||||
* if Password_encryption_type is 1, the encrypted password is sha256 + md5.
|
||||
* if Password_encryption_type is 2, the encrypted password is sha256.
|
||||
* if Password_encryption_type is 3, the encrypted password is SM3.
|
||||
*/
|
||||
if (u_sess->attr.attr_security.Password_encryption_type == 0) {
|
||||
if (!pg_md5_encrypt(password, rolname, strlen(rolname), encrypted_md5_password)) {
|
||||
|
|
|
|||
|
|
@ -1901,13 +1901,6 @@ static void set_result_for_plpgsql_language_function_with_outparam(FuncExprState
|
|||
* @bool has_cursor_return - need store out-args cursor info.
|
||||
* @bool has_refcursor - need store in-args cursor info.
|
||||
* @bool isSetReturnFunc - indicate function returns a set.
|
||||
*The execution process of the ExecMakeFunctionResult function is as follows.
|
||||
* (1) Check whether funcResultStore exists, if so, get the result and return it
|
||||
(2) The calculated parameter values are stored in fcinfo.
|
||||
(3) Pass the parameter into the expression function to calculate the expression,
|
||||
first determine whether the parameter args exists null, and then determine the return mode of the function that returns the set,
|
||||
SFRM_ValuePerCall mode is to return a value each time the call, The SFRM_Materialize schema is the result set instantiated in Tuplestore.
|
||||
(4) Calculate and return results according to different modes.
|
||||
*/
|
||||
template <bool has_refcursor, bool has_cursor_return, bool isSetReturnFunc>
|
||||
static Datum ExecMakeFunctionResult(FuncExprState* fcache, ExprContext* econtext, bool* isNull, ExprDoneCond* isDone)
|
||||
|
|
@ -3089,10 +3082,6 @@ no_function_result:
|
|||
/* ----------------------------------------------------------------
|
||||
* ExecEvalFunc
|
||||
* ----------------------------------------------------------------
|
||||
*The execution process of the ExecEvalFunc function is as follows.
|
||||
(1) Initialize the FuncExprState node by init_fcache function, including initialization parameters, memory management, etc.
|
||||
(2) Judge whether the returned result is of set type according to the data in the FuncExprState function,
|
||||
and call the corresponding function to calculate the result.
|
||||
*/
|
||||
static Datum ExecEvalFunc(FuncExprState* fcache, ExprContext* econtext, bool* isNull, ExprDoneCond* isDone)
|
||||
{
|
||||
|
|
@ -3537,10 +3526,6 @@ static Datum ExecEvalNot(BoolExprState* notclause, ExprContext* econtext, bool*
|
|||
/* ----------------------------------------------------------------
|
||||
* ExecEvalOr
|
||||
* ----------------------------------------------------------------
|
||||
*The main execution process of ExecEvalOr function is as follows.
|
||||
(1) Traverse child expression clauses.
|
||||
(2) Use the function ExecEvalExpr to call the expression calculation function in clause and calculate the result.
|
||||
(3) To judge the results, if there is a result in the or expression that meets the conditions, it will jump out of the loop and return directly.
|
||||
*/
|
||||
static Datum ExecEvalOr(BoolExprState* orExpr, ExprContext* econtext, bool* isNull, ExprDoneCond* isDone)
|
||||
{
|
||||
|
|
@ -5179,11 +5164,6 @@ Datum ExecEvalExprSwitchContext(ExprState* expression, ExprContext* econtext, bo
|
|||
* 'parent' may be NULL if we are preparing an expression that is not
|
||||
* associated with a plan tree. (If so, it can't have aggs or subplans.)
|
||||
* This case should usually come through ExecPrepareExpr, not directly here.
|
||||
*The execution process of the ExecInitExpr function is as follows.
|
||||
(1) Determine whether the input node is empty. If it is empty,return NULL directly, indicating that there is no restriction for expression.
|
||||
(2) According to the type of node input,Initialize variable evalfunc which is the execution function corresponding to node,
|
||||
If the node has parameters or expressions, the function ExecInitExpr will be recursively called and ExprState tree will be generated.
|
||||
(3) Return ExprState tree, and execute the expression recursively according to ExprState tree.
|
||||
*/
|
||||
ExprState* ExecInitExpr(Expr* node, PlanState* parent)
|
||||
{
|
||||
|
|
@ -6143,10 +6123,6 @@ Datum fetch_lob_value_from_tuple(varatt_lob_pointer* lob_pointer, Oid update_oid
|
|||
* of *isDone = ExprMultipleResult signifies a set element, and a return
|
||||
* of *isDone = ExprEndResult signifies end of the set of tuple.
|
||||
* We assume that *isDone has been initialized to ExprSingleResult by caller.
|
||||
* The execution process of the ExecTargetList function is as follows.
|
||||
(1) Iterate over the expressions in targetlist.
|
||||
(2) Calculation of expression results.
|
||||
(3) Judge the itemIsDone[resind] parameter in the results and generate the final tuple.
|
||||
*/
|
||||
static bool ExecTargetList(List* targetlist, ExprContext* econtext, Datum* values, bool* isnull,
|
||||
ExprDoneCond* itemIsDone, ExprDoneCond* isDone)
|
||||
|
|
|
|||
|
|
@ -72,7 +72,9 @@
|
|||
extern char* nodeTagToString(NodeTag type);
|
||||
|
||||
typedef VectorBatch* (*VectorEngineFunc)(PlanState* node);
|
||||
|
||||
/*
|
||||
This is a test.
|
||||
*/
|
||||
VectorBatch* UnSupportVectorRunner(PlanState* node)
|
||||
{
|
||||
ereport(ERROR,
|
||||
|
|
|
|||
|
|
@ -2947,9 +2947,9 @@ void SetOneOfCompressOption(DefElem* defElem, TableCreateSupport* tableCreateSup
|
|||
} else if (pg_strcasecmp(defname, "compress_level") == 0) {
|
||||
tableCreateSupport->compressLevel = true;
|
||||
} else if (pg_strcasecmp(defname, "compress_byte_convert") == 0) {
|
||||
tableCreateSupport->compressByteConvert = defGetBoolean(defElem);
|
||||
tableCreateSupport->compressByteConvert = true;
|
||||
} else if (pg_strcasecmp(defname, "compress_diff_convert") == 0) {
|
||||
tableCreateSupport->compressDiffConvert = defGetBoolean(defElem);
|
||||
tableCreateSupport->compressDiffConvert = true;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -394,6 +394,8 @@ void mdcreate(SMgrRelation reln, ForkNumber forkNum, bool isRedo)
|
|||
}
|
||||
|
||||
if (fd < 0) {
|
||||
int save_errno = errno;
|
||||
|
||||
/*
|
||||
* During bootstrap, there are cases where a system relation will be
|
||||
* accessed (by internal backend processes) before the bootstrap
|
||||
|
|
@ -405,7 +407,26 @@ void mdcreate(SMgrRelation reln, ForkNumber forkNum, bool isRedo)
|
|||
* new catalogs can by no means be used by other relations, we simply
|
||||
* truncate them.
|
||||
*/
|
||||
fd = RetryDataFileIdOpenFile(isRedo, path, filenode, flags);
|
||||
if (isRedo || IsBootstrapProcessingMode() ||
|
||||
(u_sess->attr.attr_common.IsInplaceUpgrade && filenode.rnode.node.relNode < FirstNormalObjectId)) {
|
||||
ADIO_RUN()
|
||||
{
|
||||
flags = O_RDWR | PG_BINARY | O_DIRECT | (u_sess->attr.attr_common.IsInplaceUpgrade ? O_TRUNC : 0);
|
||||
}
|
||||
ADIO_ELSE()
|
||||
{
|
||||
flags = O_RDWR | PG_BINARY | (u_sess->attr.attr_common.IsInplaceUpgrade ? O_TRUNC : 0);
|
||||
}
|
||||
ADIO_END();
|
||||
|
||||
fd = DataFileIdOpenFile(path, filenode, flags, 0600);
|
||||
}
|
||||
|
||||
if (fd < 0) {
|
||||
/* be sure to report the error reported by create, not open */
|
||||
errno = save_errno;
|
||||
ereport(ERROR, (errcode_for_file_access(), errmsg("could not create file \"%s\": %m", path)));
|
||||
}
|
||||
}
|
||||
|
||||
File fd_pca = -1;
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
# src/test/ipv6/create_server.py
|
||||
#
|
||||
# ---------------------------------------------------------------------------------------
|
||||
# 增加注释
|
||||
|
||||
import getopt, sys, os
|
||||
import shutil
|
||||
import time
|
||||
|
|
|
|||
|
|
@ -132,14 +132,11 @@ DROP EXTENSION test_extension_exists;
|
|||
ERROR: extension "test_extension_exists" does not exist
|
||||
DROP EXTENSION IF EXISTS test_extension_exists;
|
||||
NOTICE: extension "test_extension_exists" does not exist, skipping
|
||||
-- only extension in white list support drop, but the guc support_extended_features is true in pg_regress mode, so hdfs_fdw can be droped
|
||||
CREATE DATABASE ext_test_db;
|
||||
\c ext_test_db
|
||||
DROP EXTENSION hdfs_fdw;
|
||||
DROP EXTENSION IF EXISTS hdfs_fdw;
|
||||
NOTICE: extension "hdfs_fdw" does not exist, skipping
|
||||
\c regression
|
||||
DROP DATABASE ext_test_db;
|
||||
-- exists but doesn't support drop
|
||||
DROP EXTENSION plpgsql;
|
||||
ERROR: EXTENSION is not yet supported.
|
||||
DROP EXTENSION IF EXISTS plpgsql;
|
||||
ERROR: EXTENSION is not yet supported.
|
||||
-- functions
|
||||
DROP FUNCTION test_function_exists();
|
||||
ERROR: function test_function_exists does not exist
|
||||
|
|
|
|||
|
|
@ -89,8 +89,5 @@ ERROR: Can not use compress option in ustore index.
|
|||
-- segment
|
||||
CREATE TABLE unspported_feature.segment_table(id int, c1 text) WITH(compresstype=2, segment=on); --failed
|
||||
ERROR: only row orientation table support compresstype.
|
||||
CREATE INDEX on unspported_feature.index_test(c1) WITH(compresstype=2, segment=on); --failed
|
||||
CREATE INDEX on unspported_feature.index_test(c1) WITH(compresstype=2, segment=on); --faled
|
||||
ERROR: Can not use compress option in segment storage.
|
||||
-- set compress_diff_convert
|
||||
create table unspported_feature.compress_byte_test(id int) with (compresstype=2, compress_byte_convert=false, compress_diff_convert = true); -- failed
|
||||
ERROR: compress_diff_convert should be used with compress_byte_convert.
|
||||
|
|
|
|||
|
|
@ -145,13 +145,9 @@ DROP TEXT SEARCH CONFIGURATION test_tsconfig_exists;
|
|||
-- doesn't exists
|
||||
DROP EXTENSION test_extension_exists;
|
||||
DROP EXTENSION IF EXISTS test_extension_exists;
|
||||
-- only extension in white list support drop, but the guc support_extended_features is true in pg_regress mode, so hdfs_fdw can be droped
|
||||
CREATE DATABASE ext_test_db;
|
||||
\c ext_test_db
|
||||
DROP EXTENSION hdfs_fdw;
|
||||
DROP EXTENSION IF EXISTS hdfs_fdw;
|
||||
\c regression
|
||||
DROP DATABASE ext_test_db;
|
||||
-- exists but doesn't support drop
|
||||
DROP EXTENSION plpgsql;
|
||||
DROP EXTENSION IF EXISTS plpgsql;
|
||||
|
||||
-- functions
|
||||
DROP FUNCTION test_function_exists();
|
||||
|
|
|
|||
|
|
@ -57,6 +57,4 @@ CREATE TABLE unspported_feature.ustore_table(id int, c1 text) WITH(compresstype=
|
|||
CREATE INDEX tbl_pc_idx1 on unspported_feature.index_test(c1) WITH(compresstype=2, storage_type=ustore); --failed
|
||||
-- segment
|
||||
CREATE TABLE unspported_feature.segment_table(id int, c1 text) WITH(compresstype=2, segment=on); --failed
|
||||
CREATE INDEX on unspported_feature.index_test(c1) WITH(compresstype=2, segment=on); --failed
|
||||
-- set compress_diff_convert
|
||||
create table unspported_feature.compress_byte_test(id int) with (compresstype=2, compress_byte_convert=false, compress_diff_convert = true); -- failed
|
||||
CREATE INDEX on unspported_feature.index_test(c1) WITH(compresstype=2, segment=on); --faled
|
||||
Loading…
Reference in New Issue