From 0dc835c2228bd6cabceedab1670eb3db22af3f0f Mon Sep 17 00:00:00 2001 From: liuxyu2 Date: Thu, 5 Oct 2023 23:12:46 +0800 Subject: [PATCH] explain --- .../dbmind/db4ai/executor/direct.cpp | 11 ++- .../db4ai/executor/distance_functions.cpp | 9 ++- .../dbmind/db4ai/executor/fp_ops.cpp | 8 +++ .../dbmind/db4ai/executor/gd/gd.cpp | 4 ++ .../dbmind/db4ai/executor/gd/linregr.cpp | 3 + .../db4ai/executor/gd/optimizer_ngd.cpp | 2 + .../db4ai/executor/gd/optimizer_ova.cpp | 2 + .../db4ai/executor/gd/optimizer_pca.cpp | 2 + .../dbmind/db4ai/executor/gd/pca.cpp | 2 + .../dbmind/db4ai/executor/gd/predict.cpp | 3 + .../db4ai/executor/gd/shuffle_cache.cpp | 1 + .../dbmind/db4ai/executor/gd/svm.cpp | 1 + .../executor/hyperparameter_validation.cpp | 2 + .../dbmind/db4ai/executor/kernel.cpp | 1 + .../dbmind/db4ai/executor/matrix.cpp | 2 + .../xgboost_gs/xgboost_sklearn.sql_in | 10 +++ .../dbmind/kernel/hypopg_index.cpp | 3 + .../dbmind/kernel/index_advisor.cpp | 3 + src/gausskernel/dbmind/tools/app/timed_app.py | 71 +++++++++---------- .../dbmind/tools/cmd/config_utils.py | 9 +++ 20 files changed, 107 insertions(+), 42 deletions(-) diff --git a/src/gausskernel/dbmind/db4ai/executor/direct.cpp b/src/gausskernel/dbmind/db4ai/executor/direct.cpp index d893a16e0..8c4eacce8 100644 --- a/src/gausskernel/dbmind/db4ai/executor/direct.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/direct.cpp @@ -89,14 +89,21 @@ Model *model_fit(const char *name, AlgorithmML algorithm, const Hyperparameter * } struct DirectModelPredictor { - AlgorithmAPI *palgo; - ModelPredictor predictor; + AlgorithmAPI *palgo;// Pointer to the AlgorithmAPI object + ModelPredictor predictor;// Instance of the ModelPredictor class }; +// Function to prepare a model for prediction and return a ModelPredictor + ModelPredictor model_prepare_predict(const Model* model) { + // Allocate memory for the DirectModelPredictor struct DirectModelPredictor *pred = (DirectModelPredictor*) palloc(sizeof(DirectModelPredictor)); + + // Get the algorithm-specific API for the model's algorithm pred->palgo = get_algorithm_api(model->algorithm); + + // Prepare the predictor using the algorithm API, model data, and return type pred->predictor = pred->palgo->prepare_predict(pred->palgo, &model->data, model->return_type); return (ModelPredictor)pred; } diff --git a/src/gausskernel/dbmind/db4ai/executor/distance_functions.cpp b/src/gausskernel/dbmind/db4ai/executor/distance_functions.cpp index af759c590..fefd4f77c 100644 --- a/src/gausskernel/dbmind/db4ai/executor/distance_functions.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/distance_functions.cpp @@ -44,6 +44,8 @@ IDENTIFICATION * are not available or for the the case that the dimension is not a multiple * of the width of the registers */ + // Function to calculate L1 distance between two points without vectorization + static force_inline double l1_non_vectorized(double const * p, double const * q, uint32_t const dimension) { double term = 0.; @@ -51,11 +53,13 @@ static force_inline double l1_non_vectorized(double const * p, double const * q, double distance = 0.; double distance_correction = 0.; + // Calculate the difference between the first dimension of p and q twoDiff(q[0], p[0], &term, &term_correction); term += term_correction; // absolute value of the difference (hopefully done by clearing the sign bit) distance = std::abs(term); - + + // Iterate over the remaining dimensions for (uint32_t d = 1; d < dimension; ++d) { twoDiff(q[d], p[d], &term, &term_correction); term += term_correction; @@ -63,7 +67,8 @@ static force_inline double l1_non_vectorized(double const * p, double const * q, twoSum(distance, term, &distance, &term_correction); distance_correction += term_correction; } - + + // Return the final distance return distance + distance_correction; } diff --git a/src/gausskernel/dbmind/db4ai/executor/fp_ops.cpp b/src/gausskernel/dbmind/db4ai/executor/fp_ops.cpp index 9fe9ab972..e59ef262d 100644 --- a/src/gausskernel/dbmind/db4ai/executor/fp_ops.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/fp_ops.cpp @@ -63,6 +63,7 @@ void twoDiff(double const a, double const b, double *sub, double *e) #endif } +// Function to split a double precision number into high and low parts using Veltkamp's splitting algorithm static force_inline void veltkamp_split(double p, double *p_hi, double *p_low) { uint32_t const shift = 27U; // ceil(53 / 2) @@ -134,6 +135,7 @@ IncrementalStatistics IncrementalStatistics::operator + (IncrementalStatistics c return sum; } +// Overloaded subtraction operator for IncrementalStatistics IncrementalStatistics IncrementalStatistics::operator - (IncrementalStatistics const & rhs) const { IncrementalStatistics minus = *this; @@ -166,6 +168,7 @@ IncrementalStatistics &IncrementalStatistics::operator += (IncrementalStatistics return *this; } +// Overloaded compound subtraction operator for IncrementalStatistics IncrementalStatistics &IncrementalStatistics::operator -= (IncrementalStatistics const & rhs) { uint64_t const current_population = population - rhs.population; @@ -221,6 +224,7 @@ double IncrementalStatistics::getEmpiricalMean() const return mean; } +// Function to calculate the empirical variance of the IncrementalStatistics object double IncrementalStatistics::getEmpiricalVariance() const { double variance = 0.; @@ -229,6 +233,8 @@ double IncrementalStatistics::getEmpiricalVariance() const return variance; } +// Retrieve the empirical standard deviation from the IncrementalStatistics object + double IncrementalStatistics::getEmpiricalStdDev() const { double const std_dev = getEmpiricalVariance(); @@ -237,6 +243,8 @@ double IncrementalStatistics::getEmpiricalStdDev() const return std_dev < 0. ? 0. : std::sqrt(getEmpiricalVariance()); } +// Function to reset the IncrementalStatistics object to its initial state + bool IncrementalStatistics::reset() { errno_t rc = memset_s(this, sizeof(IncrementalStatistics), 0, sizeof(IncrementalStatistics)); diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/gd.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/gd.cpp index 293221fa7..f1a8d85c2 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/gd.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/gd.cpp @@ -37,6 +37,7 @@ #define GD_TARGET_COL 0 // predefined, always the first +// Function to get the name of an OptimizerML enum value const char *gd_get_optimizer_name(OptimizerML optimizer) { static const char* names[] = { "gd", "ngd" }; @@ -47,6 +48,7 @@ const char *gd_get_optimizer_name(OptimizerML optimizer) return names[optimizer]; } +// Function to get the GradientDescent algorithm based on the AlgorithmML enum value GradientDescent *gd_get_algorithm(AlgorithmML algorithm) { GradientDescent *gd_algorithm = nullptr; @@ -143,6 +145,8 @@ static double gd_get_score(GradientDescentState *gd_state, MetricML metric, bool return score; } +// Copy data from a PostgreSQL array to a destination array + void gd_copy_pg_array_data(float8 *dest, Datum const source_datum, int32_t const num_entries) { ArrayType *pg_array = DatumGetArrayTypeP(source_datum); diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/linregr.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/linregr.cpp index 96e9f1ed3..350b1e5ab 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/linregr.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/linregr.cpp @@ -23,6 +23,7 @@ #include "db4ai/gd.h" +// Function to compute gradients for linear regression static void linear_reg_gradients(GradientsConfig *cfg) { Assert(cfg->features->rows > 0); @@ -43,6 +44,7 @@ static void linear_reg_gradients(GradientsConfig *cfg) matrix_release(&loss); } +// Function to test linear regression and calculate the loss static double linear_reg_test(const GradientDescentState* gd_state, const Matrix *features, const Matrix *dep_var, const Matrix *weights, Scores *scores) { @@ -76,6 +78,7 @@ static Datum linear_reg_predict(const Matrix *features, const Matrix *weights, //////////////////////////////////////////////////////////////////////////////////////////////////// + GradientDescent gd_linear_regression = { { LINEAR_REGRESSION, diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ngd.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ngd.cpp index a67a2ed65..d100566b0 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ngd.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ngd.cpp @@ -39,6 +39,8 @@ typedef struct OptimizerNormalize { Matrix scale_gradients; } OptimizerNormalize; +// Function to be called at the end of each iteration in the NGD (Normalized Gradient Descent) optimizer + static void opt_ngd_end_iteration(OptimizerGD *optimizer) { OptimizerNormalize *opt = (OptimizerNormalize *)optimizer; diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ova.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ova.cpp index a0334c23f..15ee2924e 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ova.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ova.cpp @@ -32,6 +32,8 @@ static void opt_gd_ova_start_iteration(OptimizerGD *optimizer) opt->parent->start_iteration(opt->parent); } +// Signal the end of an iteration for a Gradient Descent-based optimizer in the OVA framework + static void opt_gd_ova_end_iteration(OptimizerGD *optimizer) { OptimizerOVA *opt = (OptimizerOVA *)optimizer; diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_pca.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_pca.cpp index c9f76e628..6395222c3 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_pca.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_pca.cpp @@ -91,6 +91,8 @@ static void gd_pca_update_batch(OptimizerGD *optimizer, Matrix const *features, hyperp->lambda = cfg_pca.batch_error; } +// Release resources associated with a Principal Component Analysis (PCA) optimizer + force_inline static void gd_pca_release(OptimizerGD *optimizer) { auto pca_opt = reinterpret_cast(optimizer); diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp index 7abbe9171..0eacbab38 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp @@ -190,6 +190,8 @@ static void pca_gradients(GradientsConfig *cfg) cfg_pca->batch_error = error + error_correction; } +// Define a function 'pca_test' that is marked for forced inlining + force_inline static double pca_test(const GradientDescentState* gd_state, const Matrix *features, const Matrix *dep_var, const Matrix *weights, Scores *scores) { diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/predict.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/predict.cpp index 4c0ebde3e..9dffbe459 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/predict.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/predict.cpp @@ -32,8 +32,11 @@ extern bool verify_pgarray(ArrayType const * pg_array, int32_t n); +// Define a function 'gd_predict_prepare' that prepares a model predictor + ModelPredictor gd_predict_prepare(AlgorithmAPI *self, const SerializedModel *model, Oid return_type) { + // Allocate memory for a SerializedModelGD structure and initialize it with zeros SerializedModelGD *gdp = (SerializedModelGD *)palloc0(sizeof(SerializedModelGD)); gd_deserialize((GradientDescent*)self, model, return_type, gdp); if (gdp->algorithm->prepare_kernel != nullptr) { diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/shuffle_cache.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/shuffle_cache.cpp index fe80a6eec..f8b514ab0 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/shuffle_cache.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/shuffle_cache.cpp @@ -52,6 +52,7 @@ */ typedef struct ShuffleCache { + // Represents a ShuffleGD object (details are in the ShuffleGD struct) ShuffleGD shf; int cache_size; int *cache_batch; diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/svm.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/svm.cpp index cbf7014be..2c227d709 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/svm.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/svm.cpp @@ -61,6 +61,7 @@ static void svmc_gradients(GradientsConfig *cfg) static double svmc_test(const GradientDescentState* gd_state, const Matrix *features, const Matrix *dep_var, const Matrix *weights, Scores *scores) { + // Compute the loss for a Support Vector Machine (SVM) classifier Assert(features->rows > 0); Matrix distances; diff --git a/src/gausskernel/dbmind/db4ai/executor/hyperparameter_validation.cpp b/src/gausskernel/dbmind/db4ai/executor/hyperparameter_validation.cpp index e91ac8990..6da6454d4 100644 --- a/src/gausskernel/dbmind/db4ai/executor/hyperparameter_validation.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/hyperparameter_validation.cpp @@ -38,6 +38,8 @@ } +// Retrieve hyperparameter definitions for a given machine learning algorithm + const HyperparameterDefinition* get_hyperparameter_definitions(AlgorithmML algorithm, int32_t *result_size) { AlgorithmAPI* api = get_algorithm_api(algorithm); diff --git a/src/gausskernel/dbmind/db4ai/executor/kernel.cpp b/src/gausskernel/dbmind/db4ai/executor/kernel.cpp index 56ef4d74f..20f0c7033 100644 --- a/src/gausskernel/dbmind/db4ai/executor/kernel.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/kernel.cpp @@ -24,6 +24,7 @@ #include "db4ai/kernel.h" +// Release resources associated with a Gaussian kernel within a KernelTransformer static void kernel_gaussian_release(struct KernelTransformer *kernel) { KernelGaussian *kernel_g = (KernelGaussian *)kernel; diff --git a/src/gausskernel/dbmind/db4ai/executor/matrix.cpp b/src/gausskernel/dbmind/db4ai/executor/matrix.cpp index 60144d1af..62be82435 100644 --- a/src/gausskernel/dbmind/db4ai/executor/matrix.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/matrix.cpp @@ -52,6 +52,8 @@ void matrix_init_random_gaussian(Matrix *matrix, int rows, int columns, float8 m } } +// Initialize matrices for a Gaussian kernel in a machine learning model + void matrix_init_kernel_gaussian(int features, int components, float8 gamma, int seed, Matrix *weights, Matrix *offsets) { matrix_init_random_gaussian(weights, features, components, 0.0, sqrt(2.0 * gamma), seed); diff --git a/src/gausskernel/dbmind/deepsql/madlib_modules/xgboost_gs/xgboost_sklearn.sql_in b/src/gausskernel/dbmind/deepsql/madlib_modules/xgboost_gs/xgboost_sklearn.sql_in index 0b760cced..92609a57a 100644 --- a/src/gausskernel/dbmind/deepsql/madlib_modules/xgboost_gs/xgboost_sklearn.sql_in +++ b/src/gausskernel/dbmind/deepsql/madlib_modules/xgboost_gs/xgboost_sklearn.sql_in @@ -9,12 +9,19 @@ m4_include(`SQLCommon.m4') ---------------------------------------------------------------------------------------------- ------ Help messages ---------------------------------------------------------------------------------------------- +-- Create or replace a function named xgboost_sk_classifier in the MADlib schema +-- The function takes a parameter 'message' of type TEXT +-- The function returns TEXT CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier( message TEXT ) RETURNS TEXT AS $XX$ + -- Call a Python function named PythonFunction with arguments xgboost_gs, xgboost_sklearn, and xgboost_sk_classifier_help_message PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_classifier_help_message) $XX$ LANGUAGE plpythonu IMMUTABLE; ------------------------------------------------------------- +-- Create or replace a function named xgboost_sk_classifier in the MADlib schema +-- This version of the function doesn't take any parameters +-- The function returns TEXT CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier() RETURNS TEXT AS $XX$ SELECT MADLIB_SCHEMA.xgboost_sk_classifier(''::TEXT); @@ -22,12 +29,15 @@ $XX$ LANGUAGE sql IMMUTABLE; ----------------------------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor( message TEXT ) RETURNS TEXT AS $XX$ PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_regressor_help_message) $XX$ LANGUAGE plpythonu IMMUTABLE; + ------------------------------------------------------------- + CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor() RETURNS TEXT AS $XX$ diff --git a/src/gausskernel/dbmind/kernel/hypopg_index.cpp b/src/gausskernel/dbmind/kernel/hypopg_index.cpp index 62fad5d84..3d0f5793b 100644 --- a/src/gausskernel/dbmind/kernel/hypopg_index.cpp +++ b/src/gausskernel/dbmind/kernel/hypopg_index.cpp @@ -94,10 +94,12 @@ static void hypo_injectHypotheticalIndex(PlannerInfo *root, Oid relationObjectId static List *get_table_indexes(Oid oid); static List *get_index_attrnum(Oid oid); +// Initialize the HypoPG extension void InitHypopg() { // init memory context if (g_instance.hypo_cxt.HypopgContext == NULL) { + // Create a new memory context named "HypopgContext" within the instance context g_instance.hypo_cxt.HypopgContext = AllocSetContextCreate(g_instance.instance_context, "HypopgContext", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE, SHARED_CONTEXT); } @@ -113,6 +115,7 @@ void set_hypopg_prehook(ProcessUtility_hook_type func) prev_utility_hook = func; } +// Register hooks for the HypoPG extension void hypopg_register_hook() { // register hooks diff --git a/src/gausskernel/dbmind/kernel/index_advisor.cpp b/src/gausskernel/dbmind/kernel/index_advisor.cpp index 125ec911e..8b9ed9706 100644 --- a/src/gausskernel/dbmind/kernel/index_advisor.cpp +++ b/src/gausskernel/dbmind/kernel/index_advisor.cpp @@ -170,11 +170,14 @@ static void adv_get_info_from_plan_hook(Node* node, List* rtable); static void get_join_condition_from_plan(Node* node, List* rtable); static void get_order_condition_from_plan(Node* node); +// Define a PostgreSQL Datum-returning function to provide index advice + Datum gs_index_advise(PG_FUNCTION_ARGS) { FuncCallContext *func_ctx = NULL; SuggestedIndex *array = NULL; + // Retrieve the input query as a C string char *query = PG_GETARG_CSTRING(0); if (query == NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("you must enter a query statement."))); diff --git a/src/gausskernel/dbmind/tools/app/timed_app.py b/src/gausskernel/dbmind/tools/app/timed_app.py index 31e6790f9..ee2c92dcb 100644 --- a/src/gausskernel/dbmind/tools/app/timed_app.py +++ b/src/gausskernel/dbmind/tools/app/timed_app.py @@ -20,90 +20,85 @@ from dbmind.common.dispatcher import timer from dbmind.service import dai from dbmind.common import utils -# Read the metric value range configuration from a simple config file metric_value_range_map = utils.read_simple_config_file(constants.METRIC_VALUE_RANGE_CONFIG) -# Get the detection interval from the global configuration -detection_interval = global_vars.configs.getint('SELF-MONITORING', 'detection_interval') +detection_interval = global_vars.configs.getint( + 'SELF-MONITORING', 'detection_interval' +) -# Get the last detection time in minutes from the global configuration -last_detection_minutes = global_vars.configs.getint('SELF-MONITORING', 'last_detection_time') / 60 +last_detection_minutes = global_vars.configs.getint( + 'SELF-MONITORING', 'last_detection_time' +) / 60 -# Get the time to forecast into the future in minutes from the global configuration -how_long_to_forecast_minutes = global_vars.configs.getint('SELF-MONITORING', 'forecasting_future_time') / 60 +how_long_to_forecast_minutes = global_vars.configs.getint( + 'SELF-MONITORING', 'forecasting_future_time' +) / 60 -""" -The Four Golden Signals: +"""The Four Golden Signals: https://sre.google/sre-book/monitoring-distributed-systems/#xref_monitoring_golden-signals """ -# Get the golden key performance indicators (KPIs) from the global configuration -golden_kpi = list(map(str.strip, global_vars.configs.get('SELF-MONITORING', 'golden_kpi').split(','))) +golden_kpi = list(map( + str.strip, + global_vars.configs.get( + 'SELF-MONITORING', 'golden_kpi' + ).split(',') +)) + def quickly_forecast_wrapper(sequence, forecasting_minutes): - # Call the quickly_forecast function with the given sequence and forecasting time + # Call the quickly_forecast function with the given sequence and forecasting_minutes forecast_result = quickly_forecast(sequence, forecasting_minutes) - - # Retrieve the metric value range for the sequence from the metric_value_range_map + + # Get the metric value range for the given sequence from the metric_value_range_map metric_value_range = metric_value_range_map.get(sequence.name) + + # Check if metric_value_range and forecast_result are available - # Check if both metric value range and forecast result are available if metric_value_range and forecast_result: - # Split the metric value range into low and high values metric_value_range = metric_value_range.split(",") try: - # Convert the low and high values to floats + # Convert the low and high range values to floats metric_value_low = float(metric_value_range[0]) metric_value_high = float(metric_value_range[1]) except ValueError as ex: - # Log a warning if there is a value error and return the forecast result without clipping - logging.warning("quickly_forecast_wrapper value error:%s, so forecast_result will not be clipped." % ex) + logging.warning("quickly_forecast_wrapper value error:%s," + " so forecast_result will not be cliped." % ex) return forecast_result - - # Get the forecast values as a list + f_values = list(forecast_result.values) - - # Iterate over the forecast values and clip them to the metric value range if necessary for i in range(len(f_values)): if f_values[i] < metric_value_low: f_values[i] = metric_value_low if f_values[i] > metric_value_high: f_values[i] = metric_value_high - - # Update the forecast result values with the clipped values forecast_result.values = tuple(f_values) - - # Return the forecast result return forecast_result -//backend_timed_task + + @timer(detection_interval) def self_monitoring(): - # Check if the slow query diagnosis task is in the backend timed task list + # diagnose for slow queries if constants.SLOW_QUERY_DIAGNOSIS_NAME in global_vars.backend_timed_task: - # Retrieve all slow queries within the last detection minutes slow_query_collection = dai.get_all_slow_queries(last_detection_minutes) logging.debug('The length of slow_query_collection is %d.', len(slow_query_collection)) - - # Save the slow queries by executing the diagnose_query function in parallel dai.save_slow_queries( global_vars.worker.parallel_execute( diagnose_query, ((slow_query,) for slow_query in slow_query_collection) ) ) + @timer(how_long_to_forecast_minutes * 60) def forecast_kpi(): - # Check if the forecast task is in the backend timed task list if constants.FORECAST_NAME not in global_vars.backend_timed_task: return - - # Calculate the required history length for training, considering the expansion factor + + # The general training length is at least three times the forecasting length. expansion_factor = 5 enough_history_minutes = how_long_to_forecast_minutes * expansion_factor - - # Check if the enough_history_minutes value is valid if enough_history_minutes <= 0: logging.error( - 'The value of enough_history_minutes is less than or equal to 0 ' + 'The value of enough_history_minutes less than or equal to 0 ' 'and DBMind has ignored it.' ) return diff --git a/src/gausskernel/dbmind/tools/cmd/config_utils.py b/src/gausskernel/dbmind/tools/cmd/config_utils.py index 2c2b73d36..a3b8854ac 100644 --- a/src/gausskernel/dbmind/tools/cmd/config_utils.py +++ b/src/gausskernel/dbmind/tools/cmd/config_utils.py @@ -142,9 +142,17 @@ def load_sys_configs(confile): # Defines a class that updates the encapsulated modification file class ConfigUpdater: def __init__(self, filepath): + + # Initialize a ConfigParser object self.config = ConfigParser(inline_comment_prefixes=None) + + # Get the absolute path of the file self.filepath = os.path.realpath(filepath) + + # Initialize the file pointer as None self.fp = None + + # Set the readonly flag to True self.readonly = True def get(self, section, option): @@ -206,6 +214,7 @@ def set_config_parameter(confpath, section: str, option: str, value: str): if section.isupper(): with ConfigUpdater(os.path.join(confpath, constants.CONFILE_NAME)) as config: # If not found, raise NoSectionError or NoOptionError. + try: old_value, comment = config.get(section, option) except (NoSectionError, NoOptionError):