diff --git a/src/gausskernel/dbmind/db4ai/executor/direct.cpp b/src/gausskernel/dbmind/db4ai/executor/direct.cpp index d893a16e0..8c4eacce8 100644 --- a/src/gausskernel/dbmind/db4ai/executor/direct.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/direct.cpp @@ -89,14 +89,21 @@ Model *model_fit(const char *name, AlgorithmML algorithm, const Hyperparameter * } struct DirectModelPredictor { - AlgorithmAPI *palgo; - ModelPredictor predictor; + AlgorithmAPI *palgo;// Pointer to the AlgorithmAPI object + ModelPredictor predictor;// Instance of the ModelPredictor class }; +// Function to prepare a model for prediction and return a ModelPredictor + ModelPredictor model_prepare_predict(const Model* model) { + // Allocate memory for the DirectModelPredictor struct DirectModelPredictor *pred = (DirectModelPredictor*) palloc(sizeof(DirectModelPredictor)); + + // Get the algorithm-specific API for the model's algorithm pred->palgo = get_algorithm_api(model->algorithm); + + // Prepare the predictor using the algorithm API, model data, and return type pred->predictor = pred->palgo->prepare_predict(pred->palgo, &model->data, model->return_type); return (ModelPredictor)pred; } diff --git a/src/gausskernel/dbmind/db4ai/executor/distance_functions.cpp b/src/gausskernel/dbmind/db4ai/executor/distance_functions.cpp index af759c590..fefd4f77c 100644 --- a/src/gausskernel/dbmind/db4ai/executor/distance_functions.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/distance_functions.cpp @@ -44,6 +44,8 @@ IDENTIFICATION * are not available or for the the case that the dimension is not a multiple * of the width of the registers */ + // Function to calculate L1 distance between two points without vectorization + static force_inline double l1_non_vectorized(double const * p, double const * q, uint32_t const dimension) { double term = 0.; @@ -51,11 +53,13 @@ static force_inline double l1_non_vectorized(double const * p, double const * q, double distance = 0.; double distance_correction = 0.; + // Calculate the difference between the first dimension of p and q twoDiff(q[0], p[0], &term, &term_correction); term += term_correction; // absolute value of the difference (hopefully done by clearing the sign bit) distance = std::abs(term); - + + // Iterate over the remaining dimensions for (uint32_t d = 1; d < dimension; ++d) { twoDiff(q[d], p[d], &term, &term_correction); term += term_correction; @@ -63,7 +67,8 @@ static force_inline double l1_non_vectorized(double const * p, double const * q, twoSum(distance, term, &distance, &term_correction); distance_correction += term_correction; } - + + // Return the final distance return distance + distance_correction; } diff --git a/src/gausskernel/dbmind/db4ai/executor/fp_ops.cpp b/src/gausskernel/dbmind/db4ai/executor/fp_ops.cpp index 9fe9ab972..e59ef262d 100644 --- a/src/gausskernel/dbmind/db4ai/executor/fp_ops.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/fp_ops.cpp @@ -63,6 +63,7 @@ void twoDiff(double const a, double const b, double *sub, double *e) #endif } +// Function to split a double precision number into high and low parts using Veltkamp's splitting algorithm static force_inline void veltkamp_split(double p, double *p_hi, double *p_low) { uint32_t const shift = 27U; // ceil(53 / 2) @@ -134,6 +135,7 @@ IncrementalStatistics IncrementalStatistics::operator + (IncrementalStatistics c return sum; } +// Overloaded subtraction operator for IncrementalStatistics IncrementalStatistics IncrementalStatistics::operator - (IncrementalStatistics const & rhs) const { IncrementalStatistics minus = *this; @@ -166,6 +168,7 @@ IncrementalStatistics &IncrementalStatistics::operator += (IncrementalStatistics return *this; } +// Overloaded compound subtraction operator for IncrementalStatistics IncrementalStatistics &IncrementalStatistics::operator -= (IncrementalStatistics const & rhs) { uint64_t const current_population = population - rhs.population; @@ -221,6 +224,7 @@ double IncrementalStatistics::getEmpiricalMean() const return mean; } +// Function to calculate the empirical variance of the IncrementalStatistics object double IncrementalStatistics::getEmpiricalVariance() const { double variance = 0.; @@ -229,6 +233,8 @@ double IncrementalStatistics::getEmpiricalVariance() const return variance; } +// Retrieve the empirical standard deviation from the IncrementalStatistics object + double IncrementalStatistics::getEmpiricalStdDev() const { double const std_dev = getEmpiricalVariance(); @@ -237,6 +243,8 @@ double IncrementalStatistics::getEmpiricalStdDev() const return std_dev < 0. ? 0. : std::sqrt(getEmpiricalVariance()); } +// Function to reset the IncrementalStatistics object to its initial state + bool IncrementalStatistics::reset() { errno_t rc = memset_s(this, sizeof(IncrementalStatistics), 0, sizeof(IncrementalStatistics)); diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/gd.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/gd.cpp index 293221fa7..f1a8d85c2 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/gd.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/gd.cpp @@ -37,6 +37,7 @@ #define GD_TARGET_COL 0 // predefined, always the first +// Function to get the name of an OptimizerML enum value const char *gd_get_optimizer_name(OptimizerML optimizer) { static const char* names[] = { "gd", "ngd" }; @@ -47,6 +48,7 @@ const char *gd_get_optimizer_name(OptimizerML optimizer) return names[optimizer]; } +// Function to get the GradientDescent algorithm based on the AlgorithmML enum value GradientDescent *gd_get_algorithm(AlgorithmML algorithm) { GradientDescent *gd_algorithm = nullptr; @@ -143,6 +145,8 @@ static double gd_get_score(GradientDescentState *gd_state, MetricML metric, bool return score; } +// Copy data from a PostgreSQL array to a destination array + void gd_copy_pg_array_data(float8 *dest, Datum const source_datum, int32_t const num_entries) { ArrayType *pg_array = DatumGetArrayTypeP(source_datum); diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/linregr.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/linregr.cpp index 96e9f1ed3..350b1e5ab 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/linregr.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/linregr.cpp @@ -23,6 +23,7 @@ #include "db4ai/gd.h" +// Function to compute gradients for linear regression static void linear_reg_gradients(GradientsConfig *cfg) { Assert(cfg->features->rows > 0); @@ -43,6 +44,7 @@ static void linear_reg_gradients(GradientsConfig *cfg) matrix_release(&loss); } +// Function to test linear regression and calculate the loss static double linear_reg_test(const GradientDescentState* gd_state, const Matrix *features, const Matrix *dep_var, const Matrix *weights, Scores *scores) { @@ -76,6 +78,7 @@ static Datum linear_reg_predict(const Matrix *features, const Matrix *weights, //////////////////////////////////////////////////////////////////////////////////////////////////// + GradientDescent gd_linear_regression = { { LINEAR_REGRESSION, diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ngd.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ngd.cpp index a67a2ed65..d100566b0 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ngd.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ngd.cpp @@ -39,6 +39,8 @@ typedef struct OptimizerNormalize { Matrix scale_gradients; } OptimizerNormalize; +// Function to be called at the end of each iteration in the NGD (Normalized Gradient Descent) optimizer + static void opt_ngd_end_iteration(OptimizerGD *optimizer) { OptimizerNormalize *opt = (OptimizerNormalize *)optimizer; diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ova.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ova.cpp index a0334c23f..15ee2924e 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ova.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_ova.cpp @@ -32,6 +32,8 @@ static void opt_gd_ova_start_iteration(OptimizerGD *optimizer) opt->parent->start_iteration(opt->parent); } +// Signal the end of an iteration for a Gradient Descent-based optimizer in the OVA framework + static void opt_gd_ova_end_iteration(OptimizerGD *optimizer) { OptimizerOVA *opt = (OptimizerOVA *)optimizer; diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_pca.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_pca.cpp index c9f76e628..6395222c3 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_pca.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/optimizer_pca.cpp @@ -91,6 +91,8 @@ static void gd_pca_update_batch(OptimizerGD *optimizer, Matrix const *features, hyperp->lambda = cfg_pca.batch_error; } +// Release resources associated with a Principal Component Analysis (PCA) optimizer + force_inline static void gd_pca_release(OptimizerGD *optimizer) { auto pca_opt = reinterpret_cast(optimizer); diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp index 7abbe9171..0eacbab38 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp @@ -190,6 +190,8 @@ static void pca_gradients(GradientsConfig *cfg) cfg_pca->batch_error = error + error_correction; } +// Define a function 'pca_test' that is marked for forced inlining + force_inline static double pca_test(const GradientDescentState* gd_state, const Matrix *features, const Matrix *dep_var, const Matrix *weights, Scores *scores) { diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/predict.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/predict.cpp index 4c0ebde3e..9dffbe459 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/predict.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/predict.cpp @@ -32,8 +32,11 @@ extern bool verify_pgarray(ArrayType const * pg_array, int32_t n); +// Define a function 'gd_predict_prepare' that prepares a model predictor + ModelPredictor gd_predict_prepare(AlgorithmAPI *self, const SerializedModel *model, Oid return_type) { + // Allocate memory for a SerializedModelGD structure and initialize it with zeros SerializedModelGD *gdp = (SerializedModelGD *)palloc0(sizeof(SerializedModelGD)); gd_deserialize((GradientDescent*)self, model, return_type, gdp); if (gdp->algorithm->prepare_kernel != nullptr) { diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/shuffle_cache.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/shuffle_cache.cpp index fe80a6eec..f8b514ab0 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/shuffle_cache.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/shuffle_cache.cpp @@ -52,6 +52,7 @@ */ typedef struct ShuffleCache { + // Represents a ShuffleGD object (details are in the ShuffleGD struct) ShuffleGD shf; int cache_size; int *cache_batch; diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/svm.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/svm.cpp index cbf7014be..2c227d709 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/svm.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/svm.cpp @@ -61,6 +61,7 @@ static void svmc_gradients(GradientsConfig *cfg) static double svmc_test(const GradientDescentState* gd_state, const Matrix *features, const Matrix *dep_var, const Matrix *weights, Scores *scores) { + // Compute the loss for a Support Vector Machine (SVM) classifier Assert(features->rows > 0); Matrix distances; diff --git a/src/gausskernel/dbmind/db4ai/executor/hyperparameter_validation.cpp b/src/gausskernel/dbmind/db4ai/executor/hyperparameter_validation.cpp index e91ac8990..6da6454d4 100644 --- a/src/gausskernel/dbmind/db4ai/executor/hyperparameter_validation.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/hyperparameter_validation.cpp @@ -38,6 +38,8 @@ } +// Retrieve hyperparameter definitions for a given machine learning algorithm + const HyperparameterDefinition* get_hyperparameter_definitions(AlgorithmML algorithm, int32_t *result_size) { AlgorithmAPI* api = get_algorithm_api(algorithm); diff --git a/src/gausskernel/dbmind/db4ai/executor/kernel.cpp b/src/gausskernel/dbmind/db4ai/executor/kernel.cpp index 56ef4d74f..20f0c7033 100644 --- a/src/gausskernel/dbmind/db4ai/executor/kernel.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/kernel.cpp @@ -24,6 +24,7 @@ #include "db4ai/kernel.h" +// Release resources associated with a Gaussian kernel within a KernelTransformer static void kernel_gaussian_release(struct KernelTransformer *kernel) { KernelGaussian *kernel_g = (KernelGaussian *)kernel; diff --git a/src/gausskernel/dbmind/db4ai/executor/matrix.cpp b/src/gausskernel/dbmind/db4ai/executor/matrix.cpp index 60144d1af..62be82435 100644 --- a/src/gausskernel/dbmind/db4ai/executor/matrix.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/matrix.cpp @@ -52,6 +52,8 @@ void matrix_init_random_gaussian(Matrix *matrix, int rows, int columns, float8 m } } +// Initialize matrices for a Gaussian kernel in a machine learning model + void matrix_init_kernel_gaussian(int features, int components, float8 gamma, int seed, Matrix *weights, Matrix *offsets) { matrix_init_random_gaussian(weights, features, components, 0.0, sqrt(2.0 * gamma), seed); diff --git a/src/gausskernel/dbmind/deepsql/madlib_modules/xgboost_gs/xgboost_sklearn.sql_in b/src/gausskernel/dbmind/deepsql/madlib_modules/xgboost_gs/xgboost_sklearn.sql_in index 0b760cced..92609a57a 100644 --- a/src/gausskernel/dbmind/deepsql/madlib_modules/xgboost_gs/xgboost_sklearn.sql_in +++ b/src/gausskernel/dbmind/deepsql/madlib_modules/xgboost_gs/xgboost_sklearn.sql_in @@ -9,12 +9,19 @@ m4_include(`SQLCommon.m4') ---------------------------------------------------------------------------------------------- ------ Help messages ---------------------------------------------------------------------------------------------- +-- Create or replace a function named xgboost_sk_classifier in the MADlib schema +-- The function takes a parameter 'message' of type TEXT +-- The function returns TEXT CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier( message TEXT ) RETURNS TEXT AS $XX$ + -- Call a Python function named PythonFunction with arguments xgboost_gs, xgboost_sklearn, and xgboost_sk_classifier_help_message PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_classifier_help_message) $XX$ LANGUAGE plpythonu IMMUTABLE; ------------------------------------------------------------- +-- Create or replace a function named xgboost_sk_classifier in the MADlib schema +-- This version of the function doesn't take any parameters +-- The function returns TEXT CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_classifier() RETURNS TEXT AS $XX$ SELECT MADLIB_SCHEMA.xgboost_sk_classifier(''::TEXT); @@ -22,12 +29,15 @@ $XX$ LANGUAGE sql IMMUTABLE; ----------------------------------------------------------------------------------------------- + CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor( message TEXT ) RETURNS TEXT AS $XX$ PythonFunction(xgboost_gs, xgboost_sklearn, xgboost_sk_regressor_help_message) $XX$ LANGUAGE plpythonu IMMUTABLE; + ------------------------------------------------------------- + CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.xgboost_sk_regressor() RETURNS TEXT AS $XX$ diff --git a/src/gausskernel/dbmind/kernel/hypopg_index.cpp b/src/gausskernel/dbmind/kernel/hypopg_index.cpp index 62fad5d84..3d0f5793b 100644 --- a/src/gausskernel/dbmind/kernel/hypopg_index.cpp +++ b/src/gausskernel/dbmind/kernel/hypopg_index.cpp @@ -94,10 +94,12 @@ static void hypo_injectHypotheticalIndex(PlannerInfo *root, Oid relationObjectId static List *get_table_indexes(Oid oid); static List *get_index_attrnum(Oid oid); +// Initialize the HypoPG extension void InitHypopg() { // init memory context if (g_instance.hypo_cxt.HypopgContext == NULL) { + // Create a new memory context named "HypopgContext" within the instance context g_instance.hypo_cxt.HypopgContext = AllocSetContextCreate(g_instance.instance_context, "HypopgContext", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE, SHARED_CONTEXT); } @@ -113,6 +115,7 @@ void set_hypopg_prehook(ProcessUtility_hook_type func) prev_utility_hook = func; } +// Register hooks for the HypoPG extension void hypopg_register_hook() { // register hooks diff --git a/src/gausskernel/dbmind/kernel/index_advisor.cpp b/src/gausskernel/dbmind/kernel/index_advisor.cpp index 125ec911e..8b9ed9706 100644 --- a/src/gausskernel/dbmind/kernel/index_advisor.cpp +++ b/src/gausskernel/dbmind/kernel/index_advisor.cpp @@ -170,11 +170,14 @@ static void adv_get_info_from_plan_hook(Node* node, List* rtable); static void get_join_condition_from_plan(Node* node, List* rtable); static void get_order_condition_from_plan(Node* node); +// Define a PostgreSQL Datum-returning function to provide index advice + Datum gs_index_advise(PG_FUNCTION_ARGS) { FuncCallContext *func_ctx = NULL; SuggestedIndex *array = NULL; + // Retrieve the input query as a C string char *query = PG_GETARG_CSTRING(0); if (query == NULL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("you must enter a query statement."))); diff --git a/src/gausskernel/dbmind/tools/app/timed_app.py b/src/gausskernel/dbmind/tools/app/timed_app.py index 8945cd969..ee2c92dcb 100644 --- a/src/gausskernel/dbmind/tools/app/timed_app.py +++ b/src/gausskernel/dbmind/tools/app/timed_app.py @@ -46,11 +46,18 @@ golden_kpi = list(map( def quickly_forecast_wrapper(sequence, forecasting_minutes): + # Call the quickly_forecast function with the given sequence and forecasting_minutes forecast_result = quickly_forecast(sequence, forecasting_minutes) + + # Get the metric value range for the given sequence from the metric_value_range_map metric_value_range = metric_value_range_map.get(sequence.name) + + # Check if metric_value_range and forecast_result are available + if metric_value_range and forecast_result: metric_value_range = metric_value_range.split(",") try: + # Convert the low and high range values to floats metric_value_low = float(metric_value_range[0]) metric_value_high = float(metric_value_range[1]) except ValueError as ex: diff --git a/src/gausskernel/dbmind/tools/cmd/config_utils.py b/src/gausskernel/dbmind/tools/cmd/config_utils.py index 2c2b73d36..a3b8854ac 100644 --- a/src/gausskernel/dbmind/tools/cmd/config_utils.py +++ b/src/gausskernel/dbmind/tools/cmd/config_utils.py @@ -142,9 +142,17 @@ def load_sys_configs(confile): # Defines a class that updates the encapsulated modification file class ConfigUpdater: def __init__(self, filepath): + + # Initialize a ConfigParser object self.config = ConfigParser(inline_comment_prefixes=None) + + # Get the absolute path of the file self.filepath = os.path.realpath(filepath) + + # Initialize the file pointer as None self.fp = None + + # Set the readonly flag to True self.readonly = True def get(self, section, option): @@ -206,6 +214,7 @@ def set_config_parameter(confpath, section: str, option: str, value: str): if section.isupper(): with ConfigUpdater(os.path.join(confpath, constants.CONFILE_NAME)) as config: # If not found, raise NoSectionError or NoOptionError. + try: old_value, comment = config.get(section, option) except (NoSectionError, NoOptionError):