Nemoo
/
openGauss-server
forked from nuoya/openGauss-server
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!1080 回合unique_sql自动淘汰特性到2.0.0分支 

Merge pull request !1080 from TotaJ/test_2
This commit is contained in:
opengauss-bot 2021-07-12 07:39:37 +00:00 committed by Gitee
parent 8145a970b8 7f68b9106d
commit 217801b8e6
12 changed files with 243 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/bin/gs_guc/cluster_guc.conf
View File

@ -660,6 +660,8 @@ max_concurrent_autonomous_transactions|int|0,262143|NULL|NULL|
sync_config_strategy|enum|all_node,only_sync_node,none_node|NULL|Synchronization strategy for configuration files between host and standby.|
time_to_target_rpo|int|0,3600|NULL|NULL|
disable_memory_protect|bool|0,0|NULL|NULL|
unique_sql_clean_ratio|real|0,0.2|NULL|NULL|
enable_auto_clean_unique_sql|bool|0,0|NULL|NULL|
[cmserver]
log_dir|string|0,0|NULL|NULL|
log_file_size|int|0,2047|MB|NULL|

1
src/bin/gs_guc/cluster_guc.cpp
View File

@ -1778,7 +1778,6 @@ char* get_AZ_value(const char* value, const char* data_dir)
char delims[] = ",";
char* vptr = NULL;
char emptyvalue[] = "''";
int resultStatus = 0;
bool isNodeName = false;
if (az1 != NULL) {

3
src/common/backend/nodes/copyfuncs.cpp
View File

@ -4211,6 +4211,9 @@ static Query* _copyQuery(const Query* from)
COPY_SCALAR_FIELD(use_star_targets);
COPY_SCALAR_FIELD(is_from_full_join_rewrite);
COPY_SCALAR_FIELD(uniqueSQLId);
#ifndef ENABLE_MULTIPLE_NODES
COPY_STRING_FIELD(unique_sql_text);
#endif
COPY_SCALAR_FIELD(can_push);
COPY_SCALAR_FIELD(unique_check);

51
src/common/backend/utils/misc/guc.cpp
View File

@ -174,6 +174,8 @@
#define CONFIG_EXEC_PARAMS_NEW "global/config_exec_params.new"
#endif
#define DEFAULT_CLEAN_RATIO 0.1
/* upper limit for GUC variables measured in kilobytes of memory */
/* note that various places assume the byte size fits in a "long" variable */
#if SIZEOF_SIZE_T > 4 && SIZEOF_LONG > 4
@ -548,6 +550,10 @@ static bool logging_module_check(char** newval, void** extra, GucSource source);
static void logging_module_guc_assign(const char* newval, void* extra);
static void plog_merge_age_assign(int newval, void* extra);
#ifndef ENABLE_MULTIPLE_NODES
static bool CheckUniqueSqlCleanRatio(double* newval, void** extra, GucSource source);
#endif
/* Inplace Upgrade GUC hooks */
static bool check_is_upgrade(bool* newval, void** extra, GucSource source);
static void assign_is_inplace_upgrade(const bool newval, void* extra);
@ -3278,6 +3284,17 @@ static void InitConfigureNamesBool()
NULL,
NULL,
NULL},
{{"enable_auto_clean_unique_sql",
PGC_POSTMASTER,
INSTRUMENTS_OPTIONS,
gettext_noop("Enable auto clean unique sql entry when the UniquesQl hash table is full."),
NULL},
&g_instance.attr.attr_common.enable_auto_clean_unique_sql,
false,
NULL,
NULL,
NULL},
#endif
{{"enable_partition_opfusion", PGC_USERSET, QUERY_TUNING_METHOD,
@ -6933,7 +6950,24 @@ static void InitConfigureNamesReal()
{
struct config_real localConfigureNamesReal[] =
{{{"seq_page_cost",
{
#ifndef ENABLE_MULTIPLE_NODES
{{"unique_sql_clean_ratio",
PGC_SIGHUP,
INSTRUMENTS_OPTIONS,
gettext_noop("The percentage of the UniquesQl hash table that will be "
"automatically eliminated when the UniquesQl hash table "
"is full."),
NULL},
&u_sess->attr.attr_common.unique_sql_clean_ratio,
DEFAULT_CLEAN_RATIO,
0,
0.2,
CheckUniqueSqlCleanRatio,
NULL,
NULL},
#endif
{{"seq_page_cost",
PGC_USERSET,
QUERY_TUNING_COST,
gettext_noop("Sets the planner's estimate of the cost of a "
@ -19137,6 +19171,21 @@ static void plog_merge_age_assign(int newval, void* extra)
t_thrd.log_cxt.plog_msg_switch_tm.tv_usec = (newval - MS_PER_S * t_thrd.log_cxt.plog_msg_switch_tm.tv_sec) * 1000;
}
#ifndef ENABLE_MULTIPLE_NODES
static bool CheckUniqueSqlCleanRatio(double* newval, void** extra, GucSource source)
{
if (g_instance.attr.attr_common.enable_auto_clean_unique_sql && *newval == 0) {
ereport(WARNING,
(errmsg("Can't set unique_sql_clean_ratio to 0 when enable_auto_clean_unique_sql is true. "
"Reset it's value to default(%lf). If you want to disable auto clean unique sql, "
"please set enable_auto_clean_unique_sql to false.",
DEFAULT_CLEAN_RATIO)));
*newval = DEFAULT_CLEAN_RATIO;
}
return true;
}
#endif
/* ------------------------------------------------------------ */
/* GUC hooks for inplace/grey upgrade */
/* ------------------------------------------------------------ */

168
src/gausskernel/cbb/instruments/unique_sql/instr_unique_sql.cpp
View File

@ -88,6 +88,18 @@ typedef struct {
ListCell *curr_cell;
} UniqueSQLResults;
#ifndef ENABLE_MULTIPLE_NODES
/* The mapping of UniqueSQLKey to updated_time */
typedef struct {
UniqueSQLKey key;
TimestampTz updated_time; /* latest update time for the unique sql entry */
} KeyUpdatedtime;
static KeyUpdatedtime* GetSortedEntryList();
static int KeyUpdatedtimeCmp(const void* a, const void* b);
static bool AutoRecycleUniqueSQLEntry();
#endif
/* ---------Thread Local Variable---------- */
/* save prev-hooks */
static post_parse_analyze_hook_type g_prev_post_parse_analyze_hook = NULL;
@ -648,10 +660,10 @@ void UpdateUniqueSQLStat(Query* query, const char* sql, int64 elapse_start_time,
* 2, on local CN, if entry not existed, only can insert new entry when query(Query *)
* is not NULL
*/
#ifdef ENABLE_MULTIPLE_NODES
if (is_local_unique_sql() && query == NULL) {
return;
}
/* control unique sql number by instr_unique_sql_count. */
long totalCount = hash_get_num_entries(g_instance.stat_cxt.UniqueSQLHashtbl);
if (totalCount >= u_sess->attr.attr_common.instr_unique_sql_count) {
@ -659,6 +671,25 @@ void UpdateUniqueSQLStat(Query* query, const char* sql, int64 elapse_start_time,
(errmodule(MOD_INSTR), errmsg("[UniqueSQL] Failed to insert unique sql for up to limit")));
return;
}
#else
if (is_local_unique_sql() && query == NULL && u_sess->unique_sql_cxt.unique_sql_text == NULL) {
return;
}
/* control unique sql number by instr_unique_sql_count. */
long totalCount = hash_get_num_entries(g_instance.stat_cxt.UniqueSQLHashtbl);
if (totalCount >= u_sess->attr.attr_common.instr_unique_sql_count) {
if (g_instance.attr.attr_common.enable_auto_clean_unique_sql) {
if (!AutoRecycleUniqueSQLEntry()) {
return;
}
} else {
ereport(DEBUG2,
(errmodule(MOD_INSTR), errmsg("[UniqueSQL] Failed to insert unique sql for up to limit")));
return;
}
}
#endif
(void)LockUniqueSQLHashPartition(hashCode, LW_EXCLUSIVE);
entry = (UniqueSQL*)hash_search(g_instance.stat_cxt.UniqueSQLHashtbl, &key, HASH_ENTER, &found);
@ -670,7 +701,21 @@ void UpdateUniqueSQLStat(Query* query, const char* sql, int64 elapse_start_time,
if (!found) {
resetUniqueSQLEntry(entry);
#ifndef ENABLE_MULTIPLE_NODES
if (query == NULL && u_sess->unique_sql_cxt.unique_sql_text != NULL) {
entry->unique_sql = (char*)(entry + 1);
errno_t rc = memset_s(entry->unique_sql, UNIQUE_SQL_MAX_LEN, 0, UNIQUE_SQL_MAX_LEN);
securec_check(rc, "\0", "\0");
entry->is_local = true;
int unique_sql_textLen = strlen(u_sess->unique_sql_cxt.unique_sql_text);
rc = memcpy_s(entry->unique_sql, UNIQUE_SQL_MAX_LEN, u_sess->unique_sql_cxt.unique_sql_text, unique_sql_textLen);
securec_check(rc, "\0", "\0");
} else {
set_unique_sql_string_in_entry(entry, query, sql, u_sess->unique_sql_cxt.multi_sql_offset);
}
#else
set_unique_sql_string_in_entry(entry, query, sql, u_sess->unique_sql_cxt.multi_sql_offset);
#endif
}
}
@ -1760,6 +1805,14 @@ static void SetLocalUniqueSQLId(List* query_list)
}
u_sess->unique_sql_cxt.unique_sql_user_id = GetUserId();
#ifndef ENABLE_MULTIPLE_NODES
/* store the normalized uniquesq text into u_sess->Unique_sql_cxt in stage B
* or E of PBE, only if auto-cleanup is enabled
*/
if (g_instance.attr.attr_common.enable_auto_clean_unique_sql) {
u_sess->unique_sql_cxt.unique_sql_text = query->unique_sql_text;
}
#endif
instr_stmt_report_query(u_sess->unique_sql_cxt.unique_sql_id);
/* for BE message, only can have one SQL each time,
@ -2411,6 +2464,116 @@ static int CheckParameter(const char* global, const char* cleanType, int64 value
return cleantype;
}
#ifndef ENABLE_MULTIPLE_NODES
/*
* AutoRecycleUniqueSQLEntry
* This function is called when the hash table is full, and then
* randomly cleans a certain number of unique SQL in the hash table,
* which is instr_unique_sql_count * ratio + the bloated number.
* return:
* true - successful or there is still free space
* fasle - failed
*/
static bool AutoRecycleUniqueSQLEntry()
{
LWLockAcquire(UniqueSqlEvictLock, LW_EXCLUSIVE);
/* Clean is not needed if there is still free space in unique SQL hash table. */
long totalCount = hash_get_num_entries(g_instance.stat_cxt.UniqueSQLHashtbl);
if (totalCount < u_sess->attr.attr_common.instr_unique_sql_count - 1) {
LWLockRelease(UniqueSqlEvictLock);
ereport(DEBUG1,
(errmodule(MOD_INSTR), errmsg("[UniqueSQL] There is still free space in unique SQL hash table and no need to clean it up.")));
return true;
}
int instr_unique_sql_count = u_sess->attr.attr_common.instr_unique_sql_count;
/* If the number of entries is too large, it may cause the problem of
* applying for large memory. Aotu-cleanup is not performed in this situation.
* maxEntryNum - The maximum number of KeyUpdatedtime that 1G memory can store.
*/
long maxEntryNum = long(1024 * 1024 * 1024) / sizeof(KeyUpdatedtime);
if (totalCount >= maxEntryNum) {
LWLockRelease(UniqueSqlEvictLock);
ereport(WARNING,
(errmodule(MOD_INSTR), errcode(ERRCODE_LOG), errmsg("[UniqueSQL] instr_unique_sql_count is too large, uniquesql auto-clean will not happen.")));
return false;
}
double ratio = u_sess->attr.attr_common.unique_sql_clean_ratio;
int cleanCount = Max(int(ratio * instr_unique_sql_count + totalCount - instr_unique_sql_count), 1);
/* get remove entry list */
KeyUpdatedtime* removeList = GetSortedEntryList();
if (removeList == NULL) {
LWLockRelease(UniqueSqlEvictLock);
return false;
}
/* clean */
for (int i = 0; i < cleanCount; ++i) {
UniqueSQLKey* key = &(removeList[i].key);
uint32 hashCode = uniqueSQLHashCode(key, sizeof(UniqueSQLKey));
(void)LockUniqueSQLHashPartition(hashCode, LW_EXCLUSIVE);
/* remove entry, need update valid stat timestamp */
UpdateUniqueSQLValidStatTimestamp();
hash_search(g_instance.stat_cxt.UniqueSQLHashtbl, key, HASH_REMOVE, NULL);
UnlockUniqueSQLHashPartition(hashCode);
}
pfree(removeList);
LWLockRelease(UniqueSqlEvictLock);
ereport(DEBUG1,
(errmodule(MOD_INSTR), errmsg("[UniqueSQL] Auto-cleanup over, %d uniquesqls are recycled.", cleanCount)));
return true;
}
/*
* GetSortedEntryList
* get all of entry, and sort them by entry's updated_time
*/
static KeyUpdatedtime* GetSortedEntryList()
{
UniqueSQL* entry = NULL;
HASH_SEQ_STATUS hash_seq;
int j = 0;
int i;
for (i = 0; i < NUM_UNIQUE_SQL_PARTITIONS; i++) {
LWLockAcquire(GetMainLWLockByIndex(FirstUniqueSQLMappingLock + i), LW_SHARED);
}
long totalCount = hash_get_num_entries(g_instance.stat_cxt.UniqueSQLHashtbl);
KeyUpdatedtime* removeList = NULL;
removeList = (KeyUpdatedtime*)palloc0_noexcept(totalCount * sizeof(KeyUpdatedtime));
if (removeList == NULL) {
for (i = 0; i < NUM_UNIQUE_SQL_PARTITIONS; i++) {
LWLockRelease(GetMainLWLockByIndex(FirstUniqueSQLMappingLock + i));
}
return NULL;
}
hash_seq_init(&hash_seq, g_instance.stat_cxt.UniqueSQLHashtbl);
while ((entry = (UniqueSQL*)hash_seq_search(&hash_seq)) != NULL) {
KeyUpdatedtime keyUpdatedtime;
keyUpdatedtime.key.cn_id = entry->key.cn_id;
keyUpdatedtime.key.user_id = entry->key.user_id;
keyUpdatedtime.key.unique_sql_id = entry->key.unique_sql_id;
keyUpdatedtime.updated_time = entry->updated_time;
removeList[j++] = keyUpdatedtime;
}
for (i = 0; i < NUM_UNIQUE_SQL_PARTITIONS; i++) {
LWLockRelease(GetMainLWLockByIndex(FirstUniqueSQLMappingLock + i));
}
qsort((void*)removeList, j, sizeof(KeyUpdatedtime), KeyUpdatedtimeCmp);
return removeList;
}
static int KeyUpdatedtimeCmp(const void* a, const void* b)
{
const KeyUpdatedtime* i1 = (const KeyUpdatedtime*)a;
const KeyUpdatedtime* i2 = (const KeyUpdatedtime*)b;
if (i1->updated_time < i2->updated_time)
return -1;
else if (i1->updated_time == i2->updated_time)
return 0;
else
return 1;
}
#endif
/* Reset unique sql stat info for the current database */
Datum reset_unique_sql(PG_FUNCTION_ARGS)
{
@ -2484,6 +2647,9 @@ void ResetCurrentUniqueSQL(bool need_reset_cn_id)
/* used when nested portal calling case */
u_sess->unique_sql_cxt.portal_nesting_level = 0;
#ifndef ENABLE_MULTIPLE_NODES
u_sess->unique_sql_cxt.unique_sql_text = NULL;
#endif
}
void FindUniqueSQL(UniqueSQLKey key, char* unique_sql)

7
src/gausskernel/process/tcop/postgres.cpp
View File

@ -3371,7 +3371,14 @@ static void exec_parse_message(const char* query_string, /* string to execute */
if (u_sess->attr.attr_common.log_parser_stats)
ShowUsage("PARSE ANALYSIS STATISTICS");
#ifndef ENABLE_MULTIPLE_NODES
/* store normalized uniquesQl text into Query in P phase of PBE, only if auto-cleanup is enabled */
if (is_unique_sql_enabled() && g_instance.attr.attr_common.enable_auto_clean_unique_sql) {
query->unique_sql_text = FindCurrentUniqueSQL();
}
#endif
querytree_list = pg_rewrite_query(query);
#ifdef ENABLE_MULTIPLE_NODES
if (IS_PGXC_COORDINATOR && !IsConnFromCoord()) {
ListCell* lc = NULL;

3
src/gausskernel/process/threadpool/knl_session.cpp
View File

@ -903,6 +903,9 @@ static void knl_u_unique_sql_init(knl_u_unique_sql_context* unique_sql_cxt)
unique_sql_cxt->unique_sql_sort_instr->has_sorthash = false;
unique_sql_cxt->unique_sql_hash_instr->has_sorthash = false;
unique_sql_cxt->portal_nesting_level = 0;
#ifndef ENABLE_MULTIPLE_NODES
unique_sql_cxt->unique_sql_text = NULL;
#endif
}
static void knl_u_percentile_init(knl_u_percentile_context* percentile_cxt)

1
src/gausskernel/storage/lmgr/lwlocknames.txt
View File

@ -106,3 +106,4 @@ DeleteCompactionLock 98
DeleteConsumerLock 99
ConsumerStateLock 100
HypoIndexLock 101
UniqueSqlEvictLock 102

1
src/include/knl/knl_guc/knl_instance_attr_common.h
View File

@ -94,6 +94,7 @@ typedef struct knl_instance_attr_common {
#endif
#ifndef ENABLE_MULTIPLE_NODES
int sync_config_strategy;
bool enable_auto_clean_unique_sql;
#endif
} knl_instance_attr_common;

3
src/include/knl/knl_guc/knl_session_attr_common.h
View File

@ -172,6 +172,9 @@ typedef struct knl_session_attr_common {
/* instrumentation guc parameters */
int instr_unique_sql_count;
#ifndef ENABLE_MULTIPLE_NODES
double unique_sql_clean_ratio;
#endif
bool enable_instr_cpu_timer;
int unique_sql_track_type;
bool enable_instr_track_wait;

3
src/include/knl/knl_session.h
View File

@ -1837,6 +1837,9 @@ typedef struct knl_u_unique_sql_context {
/* handle nested portal calling */
uint32 portal_nesting_level;
#ifndef ENABLE_MULTIPLE_NODES
char* unique_sql_text;
#endif
} knl_u_unique_sql_context;
typedef struct unique_sql_sorthash_instr {

3
src/include/nodes/parsenodes_common.h
View File

@ -1617,6 +1617,9 @@ typedef struct Query {
* Please refer to subquery_planner.
*/
uint64 uniqueSQLId; /* used by unique sql id */
#ifndef ENABLE_MULTIPLE_NODES
char* unique_sql_text; /* used by unique sql plain text */
#endif
bool can_push;
bool unique_check; /* true if the subquery is generated by general
* sublink pullup, and scalar output is needed */