From 974d93ac1a40e124bdfaaa72ef3178ab476e1c08 Mon Sep 17 00:00:00 2001 From: zankyfun4 <48098892@qq.com> Date: Wed, 2 Jun 2021 20:55:13 +0800 Subject: [PATCH] add peak memory information for query --- src/common/backend/utils/misc/guc.cpp | 1 + src/common/backend/utils/mmgr/aset.cpp | 2 +- src/common/backend/utils/mmgr/asetalg.cpp | 2 +- src/common/backend/utils/mmgr/asetstk.cpp | 2 +- src/common/backend/utils/mmgr/mcxt.cpp | 3 +++ src/common/backend/utils/mmgr/memprot.cpp | 6 ++++++ src/common/backend/utils/mmgr/memtrack.cpp | 2 +- .../optimizer/commands/explain.cpp | 6 ++++-- src/gausskernel/process/stream/streamMain.cpp | 2 ++ src/gausskernel/process/tcop/postgres.cpp | 20 +++++++++++++++++++ .../runtime/executor/instrument.cpp | 2 +- src/include/knl/knl_thread.h | 3 +++ src/include/postgres.h | 3 ++- src/include/utils/memutils.h | 1 + 14 files changed, 47 insertions(+), 8 deletions(-) diff --git a/src/common/backend/utils/misc/guc.cpp b/src/common/backend/utils/misc/guc.cpp index c422b46db..05983a717 100644 --- a/src/common/backend/utils/misc/guc.cpp +++ b/src/common/backend/utils/misc/guc.cpp @@ -847,6 +847,7 @@ static const struct config_enum_entry codegen_strategy_option[] = { {"partial", CODEGEN_PARTIAL, false}, {"pure", CODEGEN_PURE, false}, {NULL, 0, false}}; /*change the char * memory_tracking_mode to enum*/ static const struct config_enum_entry memory_tracking_option[] = {{"none", MEMORY_TRACKING_NONE, false}, + {"peak", MEMORY_TRACKING_PEAKMEMORY, false}, {"normal", MEMORY_TRACKING_NORMAL, false}, {"executor", MEMORY_TRACKING_EXECUTOR, false}, {"fullexec", MEMORY_TRACKING_FULLEXEC, false}, diff --git a/src/common/backend/utils/mmgr/aset.cpp b/src/common/backend/utils/mmgr/aset.cpp index b9b99605a..cb954ea76 100644 --- a/src/common/backend/utils/mmgr/aset.cpp +++ b/src/common/backend/utils/mmgr/aset.cpp @@ -540,7 +540,7 @@ MemoryContext GenericMemoryAllocator::AllocSetContextCreate(MemoryContext parent value |= IS_PROTECT; /* only track the unshared context after t_thrd.mem_cxt.mem_track_mem_cxt is created */ - if (func == &GenericFunctions && parent && MEMORY_TRACKING_MODE && t_thrd.mem_cxt.mem_track_mem_cxt && + if (func == &GenericFunctions && parent && (MEMORY_TRACKING_MODE > MEMORY_TRACKING_PEAKMEMORY) && t_thrd.mem_cxt.mem_track_mem_cxt && (t_thrd.utils_cxt.ExecutorMemoryTrack == NULL || ((AllocSet)parent)->track)) { isTracked = true; value |= IS_TRACKED; diff --git a/src/common/backend/utils/mmgr/asetalg.cpp b/src/common/backend/utils/mmgr/asetalg.cpp index 53b170d3f..b90634ec1 100644 --- a/src/common/backend/utils/mmgr/asetalg.cpp +++ b/src/common/backend/utils/mmgr/asetalg.cpp @@ -123,7 +123,7 @@ MemoryContext AlignMemoryAllocator::AllocSetContextCreate(MemoryContext parent, func = &SessionFunctions; /* only track the unshared context after t_thrd.mem_cxt.mem_track_mem_cxt is created */ - if (func == &GenericFunctions && parent && MEMORY_TRACKING_MODE && t_thrd.mem_cxt.mem_track_mem_cxt && + if (func == &GenericFunctions && parent && (MEMORY_TRACKING_MODE > MEMORY_TRACKING_PEAKMEMORY) && t_thrd.mem_cxt.mem_track_mem_cxt && (t_thrd.utils_cxt.ExecutorMemoryTrack == NULL || ((AllocSet)parent)->track)) { isTracked = true; value |= IS_TRACKED; diff --git a/src/common/backend/utils/mmgr/asetstk.cpp b/src/common/backend/utils/mmgr/asetstk.cpp index e38ed3717..4c69dab4e 100644 --- a/src/common/backend/utils/mmgr/asetstk.cpp +++ b/src/common/backend/utils/mmgr/asetstk.cpp @@ -139,7 +139,7 @@ MemoryContext StackMemoryAllocator::AllocSetContextCreate(MemoryContext parent, func = &SessionFunctions; /* only track the memory context after t_thrd.mem_cxt.mem_track_mem_cxt is created */ - if (func == &GenericFunctions && parent && u_sess->attr.attr_memory.memory_tracking_mode && + if (func == &GenericFunctions && parent && u_sess->attr.attr_memory.memory_tracking_mode > MEMORY_TRACKING_PEAKMEMORY && t_thrd.mem_cxt.mem_track_mem_cxt && (t_thrd.utils_cxt.ExecutorMemoryTrack == NULL || ((AllocSet)parent)->track)) { isTracked = true; diff --git a/src/common/backend/utils/mmgr/mcxt.cpp b/src/common/backend/utils/mmgr/mcxt.cpp index 9f2f2e841..e9fec14c7 100644 --- a/src/common/backend/utils/mmgr/mcxt.cpp +++ b/src/common/backend/utils/mmgr/mcxt.cpp @@ -143,6 +143,9 @@ void MemoryContextInit(void) /* init the thread memory track object */ t_thrd.utils_cxt.trackedMemChunks = 0; t_thrd.utils_cxt.trackedBytes = 0; + t_thrd.utils_cxt.peakedBytesInQueryLifeCycle = 0; + t_thrd.utils_cxt.basedBytesInQueryLifeCycle = 0; + /* * Initialize t_thrd.top_mem_cxt as an AllocSetContext with slow growth rate diff --git a/src/common/backend/utils/mmgr/memprot.cpp b/src/common/backend/utils/mmgr/memprot.cpp index a6dab5789..04dea7372 100644 --- a/src/common/backend/utils/mmgr/memprot.cpp +++ b/src/common/backend/utils/mmgr/memprot.cpp @@ -689,6 +689,10 @@ bool memTracker_ReserveMem(int64 requestedBytes, bool needProtect) else u_sess->stat_cxt.trackedMemChunks = newszChunk; gs_atomic_add_32(&dynmicTrackedMemChunks, needChunk); + + t_thrd.utils_cxt.basedBytesInQueryLifeCycle += requestedBytes; + if(t_thrd.utils_cxt.basedBytesInQueryLifeCycle > t_thrd.utils_cxt.peakedBytesInQueryLifeCycle) + t_thrd.utils_cxt.peakedBytesInQueryLifeCycle = t_thrd.utils_cxt.basedBytesInQueryLifeCycle; } } @@ -752,11 +756,13 @@ void memTracker_ReleaseMem(int64 toBeFreedRequested) } else if (type == MEM_THRD) { tc = t_thrd.utils_cxt.trackedMemChunks; t_thrd.utils_cxt.trackedBytes -= toBeFreed; + t_thrd.utils_cxt.basedBytesInQueryLifeCycle -= toBeFreed; tb = t_thrd.utils_cxt.trackedBytes; } else { tc = u_sess->stat_cxt.trackedMemChunks; u_sess->stat_cxt.trackedBytes -= toBeFreed; tb = u_sess->stat_cxt.trackedBytes; + t_thrd.utils_cxt.basedBytesInQueryLifeCycle -= toBeFreed; } int newszChunk = (uint64)tb >> chunkSizeInBits; diff --git a/src/common/backend/utils/mmgr/memtrack.cpp b/src/common/backend/utils/mmgr/memtrack.cpp index 99cc28459..a245afc2b 100644 --- a/src/common/backend/utils/mmgr/memtrack.cpp +++ b/src/common/backend/utils/mmgr/memtrack.cpp @@ -454,7 +454,7 @@ void MemoryTrackingNodeFree(MemoryTrack track) */ void MemoryTrackingOutputFile(void) { - if (u_sess->attr.attr_memory.memory_tracking_mode && t_thrd.utils_cxt.ExecutorMemoryTrack) { + if (u_sess->attr.attr_memory.memory_tracking_mode > MEMORY_TRACKING_PEAKMEMORY && t_thrd.utils_cxt.ExecutorMemoryTrack) { /* * output the memory context information into log file */ diff --git a/src/gausskernel/optimizer/commands/explain.cpp b/src/gausskernel/optimizer/commands/explain.cpp index ff685d568..f0d502473 100644 --- a/src/gausskernel/optimizer/commands/explain.cpp +++ b/src/gausskernel/optimizer/commands/explain.cpp @@ -1171,8 +1171,10 @@ void ExplainOnePlan( if (es->analyze) { if (es->format == EXPLAIN_FORMAT_TEXT) { - if (t_thrd.explain_cxt.explain_perf_mode == EXPLAIN_NORMAL) - appendStringInfo(es->str, "Total runtime: %.3f ms\n", 1000.0 * totaltime); + if (t_thrd.explain_cxt.explain_perf_mode == EXPLAIN_NORMAL) { + appendStringInfo(es->str, "Total runtime: %.3f ms, Peak Memory :%ld (KB)\n", 1000.0 * totaltime, (int64)(t_thrd.utils_cxt.peakedBytesInQueryLifeCycle/1024)); + + } else if (es->planinfo != NULL && es->planinfo->m_query_summary) { appendStringInfo(es->planinfo->m_query_summary->info_str, "Coordinator executor start time: %.3f ms\n", diff --git a/src/gausskernel/process/stream/streamMain.cpp b/src/gausskernel/process/stream/streamMain.cpp index ae26c8397..85bd8247d 100644 --- a/src/gausskernel/process/stream/streamMain.cpp +++ b/src/gausskernel/process/stream/streamMain.cpp @@ -607,6 +607,8 @@ void ResetStreamEnv() t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->initMemInChunks = t_thrd.utils_cxt.trackedMemChunks; t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->queryMemInChunks = t_thrd.utils_cxt.trackedMemChunks; t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->peakChunksQuery = t_thrd.utils_cxt.trackedMemChunks; + t_thrd.utils_cxt.peakedBytesInQueryLifeCycle = 0; + t_thrd.utils_cxt.basedBytesInQueryLifeCycle = 0; t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->spillCount = 0; t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->spillSize = 0; t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->broadcastSize = 0; diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index 2eb0f000d..7ab332de7 100644 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -4892,6 +4892,9 @@ static void exec_execute_message(const char* portal_name, long max_rows) break; } + if(MEMORY_TRACKING_QUERY_PEAK) + ereport(LOG, (errmsg("execute portal %s, peak memory %ld(kb)", sourceText, (int64)(t_thrd.utils_cxt.peakedBytesInQueryLifeCycle/1024)))); + if (save_log_statement_stats) ShowUsage("EXECUTE MESSAGE STATISTICS"); @@ -7764,7 +7767,15 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam #ifdef ENABLE_MULTIPLE_NODES // reset some flag related to stream ResetStreamEnv(); +#else + t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->initMemInChunks = t_thrd.utils_cxt.trackedMemChunks; + t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->queryMemInChunks = t_thrd.utils_cxt.trackedMemChunks; + t_thrd.shemem_ptr_cxt.mySessionMemoryEntry->peakChunksQuery = t_thrd.utils_cxt.trackedMemChunks; + t_thrd.utils_cxt.peakedBytesInQueryLifeCycle = 0; + t_thrd.utils_cxt.basedBytesInQueryLifeCycle = 0; #endif + + t_thrd.codegen_cxt.codegen_IRload_thr_count = 0; IsExplainPlanStmt = false; t_thrd.codegen_cxt.g_runningInFmgr = false; @@ -7901,6 +7912,9 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam } #endif exec_simple_query(query_string, QUERY_MESSAGE, &input_message); /* @hdfs Add the second parameter */ + + if(MEMORY_TRACKING_QUERY_PEAK) + ereport(LOG, (errmsg("query_string %s, peak memory %ld(kb)", query_string, (int64)(t_thrd.utils_cxt.peakedBytesInQueryLifeCycle/1024)))); u_sess->debug_query_id = 0; send_ready_for_query = true; } break; @@ -8272,6 +8286,8 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam pq_putemptymessage('s'); } pfree_ext(completionTag); + if(MEMORY_TRACKING_QUERY_PEAK) + ereport(LOG, (errmsg("execute opfusion, peak memory %ld(kb)", (int64)(t_thrd.utils_cxt.peakedBytesInQueryLifeCycle/1024)))); break; } pfree_ext(completionTag); @@ -11014,6 +11030,10 @@ static void exec_batch_bind_execute(StringInfo input_message) break; } + + if(MEMORY_TRACKING_QUERY_PEAK) + ereport(LOG, (errmsg("execute batch execute %s, peak memory %ld(kb)", psrc->query_string, (int64)(t_thrd.utils_cxt.peakedBytesInQueryLifeCycle/1024)))); + if (save_log_statement_stats) { ShowUsage("BATCH BIND MESSAGE STATISTICS"); } diff --git a/src/gausskernel/runtime/executor/instrument.cpp b/src/gausskernel/runtime/executor/instrument.cpp index ea11573fd..b9578c297 100644 --- a/src/gausskernel/runtime/executor/instrument.cpp +++ b/src/gausskernel/runtime/executor/instrument.cpp @@ -438,7 +438,7 @@ void CalculateContextSize(MemoryContext ctx, int64* memory_size) return; /* to return the accurate value when memory tracking is enable */ - if (u_sess->attr.attr_memory.memory_tracking_mode && aset->track) + if (u_sess->attr.attr_memory.memory_tracking_mode > MEMORY_TRACKING_PEAKMEMORY && aset->track) *memory_size = aset->track->allBytesPeak; else { /* calculate MemoryContext Stats */ diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index e17cf2ba3..1b37b4c4e 100644 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -1643,6 +1643,9 @@ typedef struct knl_t_utils_context { /* Track memory usage in bytes at individual thread level */ int64 trackedBytes; + int64 peakedBytesInQueryLifeCycle; + int64 basedBytesInQueryLifeCycle; + /* Per thread/query quota in chunks */ int32 maxChunksPerThread; /* Will be updated by CostSize */ diff --git a/src/include/postgres.h b/src/include/postgres.h index 7f14dd180..2d1da032c 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -246,7 +246,8 @@ typedef enum { } CodegenStrategy; typedef enum { - MEMORY_TRACKING_NONE, /* not to track the memory usage */ + MEMORY_TRACKING_NONE = 0, /* not to track the memory usage */ + MEMORY_TRACKING_PEAKMEMORY, MEMORY_TRACKING_NORMAL, /* just update the peak information internal */ MEMORY_TRACKING_EXECUTOR, /* to logging the memory information in executor */ MEMORY_TRACKING_FULLEXEC /* to logging the all memory context information in executor */ diff --git a/src/include/utils/memutils.h b/src/include/utils/memutils.h index 872131503..c3da09937 100644 --- a/src/include/utils/memutils.h +++ b/src/include/utils/memutils.h @@ -188,6 +188,7 @@ private: #define IS_USESS_AVAILABLE (likely(u_sess != NULL)) #define GS_MP_INITED (t_thrd.utils_cxt.gs_mp_inited) #define MEMORY_TRACKING_MODE (IS_USESS_AVAILABLE ? u_sess->attr.attr_memory.memory_tracking_mode : 0) +#define MEMORY_TRACKING_QUERY_PEAK (IS_USESS_AVAILABLE ? (u_sess->attr.attr_memory.memory_tracking_mode == MEMORY_TRACKING_PEAKMEMORY ) : 0) #define ENABLE_MEMORY_CONTEXT_CONTROL \ (IS_USESS_AVAILABLE ? u_sess->attr.attr_memory.enable_memory_context_control : false) #define MEMORY_FAULT_PERCENT (IS_USESS_AVAILABLE ? u_sess->attr.attr_resource.memory_fault_percent : 0)