zpc_gitlink
/
openGauss-server
forked from huawei/openGauss-server
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

!1012 charry-pick: 增加memory peak facility, 

Merge pull request !1012 from gentle_hu/2.0.0
This commit is contained in:
opengauss-bot 2021-06-03 20:20:35 +08:00 committed by Gitee
parent d97c0e8a59 974d93ac1a
commit 86c7b50454
14 changed files with 47 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/common/backend/utils/misc/guc.cpp
View File

@ -847,6 +847,7 @@ static const struct config_enum_entry codegen_strategy_option[] = {
{"partial", CODEGEN_PARTIAL, false}, {"pure", CODEGEN_PURE, false}, {NULL, 0, false}};
/*change the char * memory_tracking_mode to enum*/
static const struct config_enum_entry memory_tracking_option[] = {{"none", MEMORY_TRACKING_NONE, false},
{"peak", MEMORY_TRACKING_PEAKMEMORY, false},
{"normal", MEMORY_TRACKING_NORMAL, false},
{"executor", MEMORY_TRACKING_EXECUTOR, false},
{"fullexec", MEMORY_TRACKING_FULLEXEC, false},

2
src/common/backend/utils/mmgr/aset.cpp
View File

@ -540,7 +540,7 @@ MemoryContext GenericMemoryAllocator::AllocSetContextCreate(MemoryContext parent
value |= IS_PROTECT;
/* only track the unshared context after t_thrd.mem_cxt.mem_track_mem_cxt is created */
if (func == &GenericFunctions && parent && MEMORY_TRACKING_MODE && t_thrd.mem_cxt.mem_track_mem_cxt &&
if (func == &GenericFunctions && parent && (MEMORY_TRACKING_MODE > MEMORY_TRACKING_PEAKMEMORY) && t_thrd.mem_cxt.mem_track_mem_cxt &&
(t_thrd.utils_cxt.ExecutorMemoryTrack == NULL || ((AllocSet)parent)->track)) {
isTracked = true;
value |= IS_TRACKED;

2
src/common/backend/utils/mmgr/asetalg.cpp
View File

@ -123,7 +123,7 @@ MemoryContext AlignMemoryAllocator::AllocSetContextCreate(MemoryContext parent,
func = &SessionFunctions;
/* only track the unshared context after t_thrd.mem_cxt.mem_track_mem_cxt is created */
if (func == &GenericFunctions && parent && MEMORY_TRACKING_MODE && t_thrd.mem_cxt.mem_track_mem_cxt &&
if (func == &GenericFunctions && parent && (MEMORY_TRACKING_MODE > MEMORY_TRACKING_PEAKMEMORY) && t_thrd.mem_cxt.mem_track_mem_cxt &&
(t_thrd.utils_cxt.ExecutorMemoryTrack == NULL || ((AllocSet)parent)->track)) {
isTracked = true;
value |= IS_TRACKED;

2
src/common/backend/utils/mmgr/asetstk.cpp
View File

@ -139,7 +139,7 @@ MemoryContext StackMemoryAllocator::AllocSetContextCreate(MemoryContext parent,
func = &SessionFunctions;
/* only track the memory context after t_thrd.mem_cxt.mem_track_mem_cxt is created */
if (func == &GenericFunctions && parent && u_sess->attr.attr_memory.memory_tracking_mode &&
if (func == &GenericFunctions && parent && u_sess->attr.attr_memory.memory_tracking_mode > MEMORY_TRACKING_PEAKMEMORY &&
t_thrd.mem_cxt.mem_track_mem_cxt &&
(t_thrd.utils_cxt.ExecutorMemoryTrack == NULL || ((AllocSet)parent)->track)) {
isTracked = true;

3
src/common/backend/utils/mmgr/mcxt.cpp
View File

@ -143,6 +143,9 @@ void MemoryContextInit(void)
/* init the thread memory track object */
t_thrd.utils_cxt.trackedMemChunks = 0;
t_thrd.utils_cxt.trackedBytes = 0;
t_thrd.utils_cxt.peakedBytesInQueryLifeCycle = 0;
t_thrd.utils_cxt.basedBytesInQueryLifeCycle = 0;
/*
* Initialize t_thrd.top_mem_cxt as an AllocSetContext with slow growth rate

6
src/common/backend/utils/mmgr/memprot.cpp
View File

@ -689,6 +689,10 @@ bool memTracker_ReserveMem(int64 requestedBytes, bool needProtect)
else
u_sess->stat_cxt.trackedMemChunks = newszChunk;
gs_atomic_add_32(&dynmicTrackedMemChunks, needChunk);
t_thrd.utils_cxt.basedBytesInQueryLifeCycle += requestedBytes;
if(t_thrd.utils_cxt.basedBytesInQueryLifeCycle > t_thrd.utils_cxt.peakedBytesInQueryLifeCycle)
t_thrd.utils_cxt.peakedBytesInQueryLifeCycle = t_thrd.utils_cxt.basedBytesInQueryLifeCycle;
}
}
@ -752,11 +756,13 @@ void memTracker_ReleaseMem(int64 toBeFreedRequested)
} else if (type == MEM_THRD) {
tc = t_thrd.utils_cxt.trackedMemChunks;
t_thrd.utils_cxt.trackedBytes -= toBeFreed;
t_thrd.utils_cxt.basedBytesInQueryLifeCycle -= toBeFreed;
tb = t_thrd.utils_cxt.trackedBytes;
} else {
tc = u_sess->stat_cxt.trackedMemChunks;
u_sess->stat_cxt.trackedBytes -= toBeFreed;
tb = u_sess->stat_cxt.trackedBytes;
t_thrd.utils_cxt.basedBytesInQueryLifeCycle -= toBeFreed;
}
int newszChunk = (uint64)tb >> chunkSizeInBits;

2
src/common/backend/utils/mmgr/memtrack.cpp
View File

@ -454,7 +454,7 @@ void MemoryTrackingNodeFree(MemoryTrack track)
*/
void MemoryTrackingOutputFile(void)
{
if (u_sess->attr.attr_memory.memory_tracking_mode && t_thrd.utils_cxt.ExecutorMemoryTrack) {
if (u_sess->attr.attr_memory.memory_tracking_mode > MEMORY_TRACKING_PEAKMEMORY && t_thrd.utils_cxt.ExecutorMemoryTrack) {
/*
* output the memory context information into log file
*/

6
src/gausskernel/optimizer/commands/explain.cpp
View File

@ -1171,8 +1171,10 @@ void ExplainOnePlan(
if (es->analyze) {
if (es->format == EXPLAIN_FORMAT_TEXT) {
if (t_thrd.explain_cxt.explain_perf_mode == EXPLAIN_NORMAL)
appendStringInfo(es->str, "Total runtime: %.3f ms\n", 1000.0 * totaltime);
if (t_thrd.explain_cxt.explain_perf_mode == EXPLAIN_NORMAL) {
appendStringInfo(es->str, "Total runtime: %.3f ms, Peak Memory :%ld (KB)\n", 1000.0 * totaltime, (int64)(t_thrd.utils_cxt.peakedBytesInQueryLifeCycle/1024));
}
else if (es->planinfo != NULL && es->planinfo->m_query_summary) {
appendStringInfo(es->planinfo->m_query_summary->info_str,
"Coordinator executor start time: %.3f ms\n",

2
src/gausskernel/process/stream/streamMain.cpp
View File

@ -607,6 +607,8 @@ void ResetStreamEnv()
t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->initMemInChunks = t_thrd.utils_cxt.trackedMemChunks;
t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->queryMemInChunks = t_thrd.utils_cxt.trackedMemChunks;
t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->peakChunksQuery = t_thrd.utils_cxt.trackedMemChunks;
t_thrd.utils_cxt.peakedBytesInQueryLifeCycle = 0;
t_thrd.utils_cxt.basedBytesInQueryLifeCycle = 0;
t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->spillCount = 0;
t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->spillSize = 0;
t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->broadcastSize = 0;

20
src/gausskernel/process/tcop/postgres.cpp
View File

@ -4892,6 +4892,9 @@ static void exec_execute_message(const char* portal_name, long max_rows)
break;
}
if(MEMORY_TRACKING_QUERY_PEAK)
ereport(LOG, (errmsg("execute portal %s, peak memory %ld(kb)", sourceText, (int64)(t_thrd.utils_cxt.peakedBytesInQueryLifeCycle/1024))));
if (save_log_statement_stats)
ShowUsage("EXECUTE MESSAGE STATISTICS");
@ -7764,7 +7767,15 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam
#ifdef ENABLE_MULTIPLE_NODES
// reset some flag related to stream
ResetStreamEnv();
#else
t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->initMemInChunks = t_thrd.utils_cxt.trackedMemChunks;
t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->queryMemInChunks = t_thrd.utils_cxt.trackedMemChunks;
t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->peakChunksQuery = t_thrd.utils_cxt.trackedMemChunks;
t_thrd.utils_cxt.peakedBytesInQueryLifeCycle = 0;
t_thrd.utils_cxt.basedBytesInQueryLifeCycle = 0;
#endif
t_thrd.codegen_cxt.codegen_IRload_thr_count = 0;
IsExplainPlanStmt = false;
t_thrd.codegen_cxt.g_runningInFmgr = false;
@ -7901,6 +7912,9 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam
}
#endif
exec_simple_query(query_string, QUERY_MESSAGE, &input_message); /* @hdfs Add the second parameter */
if(MEMORY_TRACKING_QUERY_PEAK)
ereport(LOG, (errmsg("query_string %s, peak memory %ld(kb)", query_string, (int64)(t_thrd.utils_cxt.peakedBytesInQueryLifeCycle/1024))));
u_sess->debug_query_id = 0;
send_ready_for_query = true;
} break;
@ -8272,6 +8286,8 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam
pq_putemptymessage('s');
}
pfree_ext(completionTag);
if(MEMORY_TRACKING_QUERY_PEAK)
ereport(LOG, (errmsg("execute opfusion, peak memory %ld(kb)", (int64)(t_thrd.utils_cxt.peakedBytesInQueryLifeCycle/1024))));
break;
}
pfree_ext(completionTag);
@ -11014,6 +11030,10 @@ static void exec_batch_bind_execute(StringInfo input_message)
break;
}
if(MEMORY_TRACKING_QUERY_PEAK)
ereport(LOG, (errmsg("execute batch execute %s, peak memory %ld(kb)", psrc->query_string, (int64)(t_thrd.utils_cxt.peakedBytesInQueryLifeCycle/1024))));
if (save_log_statement_stats) {
ShowUsage("BATCH BIND MESSAGE STATISTICS");
}

2
src/gausskernel/runtime/executor/instrument.cpp
View File

@ -438,7 +438,7 @@ void CalculateContextSize(MemoryContext ctx, int64* memory_size)
return;
/* to return the accurate value when memory tracking is enable */
if (u_sess->attr.attr_memory.memory_tracking_mode && aset->track)
if (u_sess->attr.attr_memory.memory_tracking_mode > MEMORY_TRACKING_PEAKMEMORY && aset->track)
*memory_size = aset->track->allBytesPeak;
else {
/* calculate MemoryContext Stats */

3
src/include/knl/knl_thread.h
View File

@ -1643,6 +1643,9 @@ typedef struct knl_t_utils_context {
/* Track memory usage in bytes at individual thread level */
int64 trackedBytes;
int64 peakedBytesInQueryLifeCycle;
int64 basedBytesInQueryLifeCycle;
/* Per thread/query quota in chunks */
int32 maxChunksPerThread; /* Will be updated by CostSize */

3
src/include/postgres.h
View File

@ -246,7 +246,8 @@ typedef enum {
} CodegenStrategy;
typedef enum {
MEMORY_TRACKING_NONE, /* not to track the memory usage */
MEMORY_TRACKING_NONE = 0, /* not to track the memory usage */
MEMORY_TRACKING_PEAKMEMORY,
MEMORY_TRACKING_NORMAL, /* just update the peak information internal */
MEMORY_TRACKING_EXECUTOR, /* to logging the memory information in executor */
MEMORY_TRACKING_FULLEXEC /* to logging the all memory context information in executor */

1
src/include/utils/memutils.h
View File

@ -188,6 +188,7 @@ private:
#define IS_USESS_AVAILABLE (likely(u_sess != NULL))
#define GS_MP_INITED (t_thrd.utils_cxt.gs_mp_inited)
#define MEMORY_TRACKING_MODE (IS_USESS_AVAILABLE ? u_sess->attr.attr_memory.memory_tracking_mode : 0)
#define MEMORY_TRACKING_QUERY_PEAK (IS_USESS_AVAILABLE ? (u_sess->attr.attr_memory.memory_tracking_mode == MEMORY_TRACKING_PEAKMEMORY ) : 0)
#define ENABLE_MEMORY_CONTEXT_CONTROL \
(IS_USESS_AVAILABLE ? u_sess->attr.attr_memory.enable_memory_context_control : false)
#define MEMORY_FAULT_PERCENT (IS_USESS_AVAILABLE ? u_sess->attr.attr_resource.memory_fault_percent : 0)