From 8f54badbddc2c1ab6170a80e01ef694a1861d0db Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Tue, 15 Aug 2023 10:20:47 +0800 Subject: [PATCH 01/48] Update chkpass.cpp --- contrib/chkpass/chkpass.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/chkpass/chkpass.cpp b/contrib/chkpass/chkpass.cpp index 512273aa3..cf57c6046 100644 --- a/contrib/chkpass/chkpass.cpp +++ b/contrib/chkpass/chkpass.cpp @@ -3,7 +3,7 @@ * Written by D'Arcy J.M. Cain * darcy@druid.net * http://www.druid.net/darcy/ - * + *111 * contrib/chkpass/chkpass.c * best viewed with tabs set to 4 */ -- 2.34.1 From 74add04db070316187ed5509cfb7cd62870476a5 Mon Sep 17 00:00:00 2001 From: Auweafan Date: Sun, 27 Aug 2023 20:43:23 +0800 Subject: [PATCH 02/48] Update curl_utils.cpp --- .../security/gs_policy/curl_utils.cpp | 59 +++++++++++-------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/src/gausskernel/security/gs_policy/curl_utils.cpp b/src/gausskernel/security/gs_policy/curl_utils.cpp index 5720f85e8..a8976fc9b 100644 --- a/src/gausskernel/security/gs_policy/curl_utils.cpp +++ b/src/gausskernel/security/gs_policy/curl_utils.cpp @@ -25,25 +25,29 @@ * ------------------------------------------------------------------------- */ - -#include -#include -#include -#include -#include -#include -#include "curl/curl.h" -#include "gs_policy/curl_utils.h" -#include "utils/elog.h" +/*与网络通信和HTTP请求相关的功能。 +它封装了libcurl库，提供了一组方便的函数来发送HTTP请求、处理响应并处理与网络通信相关的任务。 +创建和释放CURL句柄、设置请求选项（如URL、请求方法、超时时间等）、 +添加请求头信息、发送请求、接收响应、获取响应状态码和响应头信息、以及处理错误等。 +*/ +#include //用于文件输入输出操作。 +#include //用于流缓冲区操作 +#include //用于线程操作 +#include //用于互斥锁操作 +#include //用于向量容器操作 +#include //用于字符串操作 +#include "curl/curl.h" //用于使用CURL库进行网络通信。 +#include "gs_policy/curl_utils.h" //gs_policy模块中与curl通信相关的工具函数定义。 +#include "utils/elog.h" //用于错误日志记录。 -static std::mutex g_i_mutex; +static std::mutex g_i_mutex; //静态互斥锁对象 -CurlUtils::CurlUtils() : m_withSSL(false), - m_certificate(""), - m_user(""), - m_password(""), - m_curlForPost(NULL) +CurlUtils::CurlUtils() : m_withSSL(false), //用于指示是否启用SSL连接，默认为 false。 + m_certificate(""), //存储SSL证书的文件路径 + m_user(""), //存储用户名称 + m_password(""), //存储用户密码 + m_curlForPost(NULL)//用于执行POST请求的CURL句柄 { } @@ -52,6 +56,7 @@ CurlUtils::~CurlUtils() curl_easy_cleanup(m_curlForPost); } +//初始化 CurlUtils 类的实例 void CurlUtils::initialize(bool withSSL, const std::string certificate, const std::string user, const std::string password) { @@ -67,14 +72,19 @@ void CurlUtils::initialize(bool withSSL, const std::string certificate, const st */ bool CurlUtils::http_post_file_request(const std::string url, const std::string fileName, bool connection_testing) { + // 输出调试信息，包括URL和文件名 ereport(INFO, (errmsg("Url = %s, fileName = %s", url.c_str(), fileName.c_str()))); + // 从文件中读取内容并存储为字符串 std::ifstream t(fileName); std::string str((std::istreambuf_iterator(t)), std::istreambuf_iterator()); if (m_curlForPost != NULL) { + // 设置请求头 struct curl_slist *slist1 = NULL; slist1 = curl_slist_append(slist1, "Content-Type: application/json"); + + // 设置CURL选项，包括URL、进度通知设置、请求体等 (void)curl_easy_setopt(m_curlForPost, CURLOPT_URL, url.c_str()); (void)curl_easy_setopt(m_curlForPost, CURLOPT_NOPROGRESS, 1L); (void)curl_easy_setopt(m_curlForPost, CURLOPT_POSTFIELDS, str.c_str()); @@ -92,25 +102,26 @@ bool CurlUtils::http_post_file_request(const std::string url, const std::string (void)curl_easy_setopt(m_curlForPost, CURLOPT_TCP_KEEPALIVE, 1L); /* a simply connection test to server, just verify the connection without any data transfer */ + // 进行连接测试，仅验证连接而不进行数据传输 if (connection_testing) { (void)curl_easy_setopt(m_curlForPost, CURLOPT_CONNECT_ONLY, 1L); } /* perform a file transfer */ - CURLcode res = curl_easy_perform(m_curlForPost); - if (res != CURLE_OK) { + CURLcode res = curl_easy_perform(m_curlForPost);//使用curl_easy_perform()函数来执行网络请求。该函数将阻塞当前线程，直到请求完成或发生错误。 + if (res != CURLE_OK) {// 处理请求失败的情况 /* * we will not generate error which will make the audit thread restarted * but make an error and free related resource to make user deal with the connection issue */ - if (connection_testing) { - ereport(PANIC, - (errmsg("make sure connection to elastic_search_ip_addr, error info: %s\n", - curl_easy_strerror(res)))); + if (connection_testing) {//判断是否正在进行连接测试 + ereport(PANIC, //错误的严重级别，表示遇到了无法继续执行的致命错误，需要立即终止程序 + (errmsg("make sure connection to elastic_search_ip_addr, error info: %s\n", //错误消息的格式化部分 + curl_easy_strerror(res))));/*将错误码转换为对应的错误描述字符串。*/ } - curl_slist_free_all(slist1); - curl_easy_reset(m_curlForPost); + curl_slist_free_all(slist1);//释放整个链表及其节点所占用的内存 + curl_easy_reset(m_curlForPost);//重置，将其恢复为初始状态以进行新的请求。 ereport(WARNING, (errmsg("Connection issue happended, post file error: %s\n", curl_easy_strerror(res)))); return false; } -- 2.34.1 From 0cd2edfb52732db6a7a29d57dcc87697fb76dc00 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Sun, 27 Aug 2023 22:31:59 +0800 Subject: [PATCH 03/48] Update blockchain.cpp --- .../security/gs_ledger/blockchain.cpp | 263 +++++++++++------- 1 file changed, 155 insertions(+), 108 deletions(-) diff --git a/src/gausskernel/security/gs_ledger/blockchain.cpp b/src/gausskernel/security/gs_ledger/blockchain.cpp index 23c0d946f..bd0c20449 100644 --- a/src/gausskernel/security/gs_ledger/blockchain.cpp +++ b/src/gausskernel/security/gs_ledger/blockchain.cpp @@ -32,61 +32,72 @@ #include "utils/snapmgr.h" /* - * gen_global_hash -- generate globalhash of gchain + * gen_global_hash -- generate globalhash of gchain //生成gchain的全局哈希值 * - * hash_buffer: the buffer that ready to fill generated globalhash. - * info_string: operate info string of current block. - * exist: whether previous block exists. - * prev_hash: the address of previous hash value. + * hash_buffer: the buffer that ready to fill generated globalhash.//准备填充生成的全局哈希值的缓冲区 + * info_string: operate info string of current block.//当前块的操作信息字符串。 + * exist: whether previous block exists.//前一个块是否存在。 + * prev_hash: the address of previous hash value.//前一个哈希值的地址。 * * Note: globalhash is generated by operate info and previous globalhash using md5. + * 全局哈希值是通过使用md5算法，基于操作信息和前一个全局哈希值生成的。 */ +//该函数的作用是根据区块链中块的信息和相关参数，生成全局哈希用于验证和识别区块链的完整性和链式结构。(判断、生成块与MD5哈希值) bool gen_global_hash(hash32_t *hash_buffer, const char *info_string, bool exist, const hash32_t *prev_hash) { errno_t rc = EOK; int comb_strlen; char *comb_string = NULL; + /* * Previous block not exists means current insertion block is genesis, * then we use global systable as origin combine string for globalhash * generation. If previous block exists, we will use previous global * hash as combine string to calculate globalhash. */ - if (!exist) { + /* 如果前一个块不存在，表示当前插入的块是创世块，使用全局systable作为全局哈希生成的组合字符串； + 如果前一个块存在，我们将使用前一个全局哈希作为组合字符串来计算全局哈希。 */ + + if (!exist) {//前一个块不存在 /* generate genesis block globalhash */ - comb_strlen = strlen(GCHAIN_NAME) + strlen(info_string) + 1; - comb_string = (char *)palloc0(comb_strlen); - rc = snprintf_s(comb_string, comb_strlen, comb_strlen - 1, "%s%s", GCHAIN_NAME, info_string); - securec_check_ss(rc, "", ""); + /* 生成创世块的全局哈希 */ + comb_strlen = strlen(GCHAIN_NAME) + strlen(info_string) + 1;//计算组合字符串的长度 + comb_string = (char *)palloc0(comb_strlen);//分配足够的内存给组合字符串，并初始化为0 + rc = snprintf_s(comb_string, comb_strlen, comb_strlen - 1, "%s%s", GCHAIN_NAME, info_string);// 将GCHAIN_NAME和info_string拼接到组合字符串中 + //comb_string为存储位置，comb_strlen为最大允许字符数 + securec_check_ss(rc, "", "");//检查snprintf_s是否成功——如果发生错误并返回值为-1，表示目标缓冲区或格式字符串是一个空指针，或者无效的参数句柄被调用。此时，宏函数会释放分配给缓冲区和其他可变参数的内存，并在日志中输出错误信息，提供文件名和行号。 + //函数来源 src\include\gtm\utils\elog.h } else { /* use previous globalhash and current block info to calculate globalhash. */ - char *pre_hash_str = DatumGetCString(DirectFunctionCall1(hash32out, HASH32GetDatum(prev_hash))); - comb_strlen = strlen(pre_hash_str) + strlen(info_string) + 1; - comb_string = (char *)palloc0(comb_strlen); - rc = snprintf_s(comb_string, comb_strlen, comb_strlen - 1, "%s%s", info_string, pre_hash_str); - securec_check_ss(rc, "", ""); - pfree_ext(pre_hash_str); + /* 使用前一个全局哈希和当前块信息计算全局哈希 */ + char *pre_hash_str = DatumGetCString(DirectFunctionCall1(hash32out, HASH32GetDatum(prev_hash)));//将prev_hash转换为字符串形式 + comb_strlen = strlen(pre_hash_str) + strlen(info_string) + 1;//计算组合字符串的长度 + comb_string = (char *)palloc0(comb_strlen);//分配足够的内存给组合字符串，并初始化为0 + rc = snprintf_s(comb_string, comb_strlen, comb_strlen - 1, "%s%s", info_string, pre_hash_str); //将info_string和pre_hash_str拼接到组合字符串中 + securec_check_ss(rc, "", "");//检查snprintf_s是否成功 + pfree_ext(pre_hash_str);//释放pre_hash_str占用的内存 } - - if (!pg_md5_binary(comb_string, comb_strlen - 1, hash_buffer->data)) { - pfree(comb_string); - ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Failed to generate globalhash, out of memory"))); + /* 使用pg_md5_binary函数计算组合字符串的MD5哈希值，并存储在hash_buffer中 */ + if (!pg_md5_binary(comb_string, comb_strlen - 1, hash_buffer->data)) {//函数来源src\common\backend\libpq\md5.cpp + pfree(comb_string);//释放组合字符串占用的内存 + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Failed to generate globalhash, out of memory")));// 报错，内存不足 return false; } - pfree(comb_string); - return true; + pfree(comb_string);//释放组合字符串占用的内存 + return true;//返回全局哈希生成是否成功的结果 } /* - * set_gchain_comb_string -- combine block informations. + * set_gchain_comb_string -- combine block informations.//组合块信息 * - * db_name: the database name where executes cmd. - * user_name: the user name who executes cmd - * nsp_name: namespace name of usertable - * rel_name: rel_name of usertable - * cmd_text: the command query which modified user table. - * rel_hash: rel_hash of current block. + * db_name: the database name where executes cmd.//执行命令的数据库名称 + * user_name: the user name who executes cmd//执行命令的用户名 + * nsp_name: namespace name of usertable//用户表的命名空间名称 + * rel_name: rel_name of usertable//用户表的关系名称 + * cmd_text: the command query which modified user table.//修改用户表的命令查询 + * rel_hash: rel_hash of current block.//当前块的关系哈希值 */ +//将多个字符串和数值拼接在一起,设置区块链全局哈希生成所需的组合字符串。 char *set_gchain_comb_string(const char *db_name, const char *user_name, const char *nsp_name, const char *rel_name, const char *cmd_text, uint64 rel_hash) { @@ -95,97 +106,110 @@ char *set_gchain_comb_string(const char *db_name, const char *user_name, } int comb_len = strlen(db_name) + strlen(user_name) + strlen(nsp_name) + strlen(rel_name) + strlen(cmd_text) + PREVIOUS_HASH_LEN + 1; - char *comb_str = (char *)palloc0(sizeof(char) * comb_len); - errno_t rc = snprintf_s(comb_str, comb_len, comb_len - 1, "%s%s%s%s%s%lu", + char *comb_str = (char *)palloc0(sizeof(char) * comb_len);//分配足够的内存给组合字符串，并初始化为0 + errno_t rc = snprintf_s(comb_str, comb_len, comb_len - 1, "%s%s%s%s%s%lu",//将各个字符串和数值按指定格式拼接到组合字符串中，存入comb_str db_name, user_name, nsp_name, rel_name, cmd_text, rel_hash); - securec_check_ss(rc, "\0", "\0"); - return comb_str; + securec_check_ss(rc, "\0", "\0");//检查snprintf_s是否执行成功 + return comb_str;//返回生成的组合字符串 } /* - * ledger_gchain_append -- record a block to gchain. + * ledger_gchain_append -- record a block to gchain.//将一个块记录到gchain * - * relid: relation oid of usertable. - * query_string: original query which modified usertable. - * cn_hash: rel_hash in hist table generated by query_string. + * relid: relation oid of usertable.//用户表的关系OID + * query_string: original query which modified usertable.//修改用户表的原始查询语句 + * cn_hash: rel_hash in hist table generated by query_string.//通过query_string生成的hist表中的关系哈希值 * * Note: after block inserted into gchain, its globalhash will flush * into gchain cache for next block. Thus, previous global hash is * come from cache directly. + * 注意：在块插入gchain之后，它的全局哈希值将被刷新到gchain缓存中以供下一个块使用。 + 因此，前一个全局哈希值来自缓存直接提取。 */ +//用于向全局链表中追加记录,为区块链的增长提供了支持 void ledger_gchain_append(Oid relid, const char *query_string, uint64 cn_hash) { - Datum current_time; - Datum values[Natts_gs_global_chain] = {0}; - bool nulls[Natts_gs_global_chain] = {false}; + Datum current_time;//当前时间 + Datum values[Natts_gs_global_chain] = {0};//存储要插入的数据值 + //Natts_gs_global_chain定义为10，来源src\include\catalog\gs_global_chain.h + bool nulls[Natts_gs_global_chain] = {false};//标记是否为 NULL char *db_name = NULL; char *user_name = NULL; char *nsp_name = NULL; char *rel_name = NULL; - char *combine_string = NULL; - HeapTuple tup = NULL; - Relation rel_gchain = NULL; - GlobalPrevBlock current_block; + char *combine_string = NULL;//存储字符串 + HeapTuple tup = NULL; //堆元组 + Relation rel_gchain = NULL;//全局链表关系 + GlobalPrevBlock current_block;//当前块 /* get basic informations. */ - db_name = get_database_name(u_sess->proc_cxt.MyDatabaseId); - user_name = GetUserNameFromId(GetCurrentUserId()); - current_time = TimestampTzGetDatum(GetCurrentTimestamp()); - nsp_name = get_namespace_name(get_rel_namespace(relid)); - rel_name = get_rel_name(relid); + //获取基本信息 + db_name = get_database_name(u_sess->proc_cxt.MyDatabaseId);//根据数据库的OID获取当前数据库名，来源src\gausskernel\optimizer\commands\dbcommands.cpp + user_name = GetUserNameFromId(GetCurrentUserId());//根据用户OID获取当前用户名，来源src\common\backend\utils\init\miscinit.cpp + current_time = TimestampTzGetDatum(GetCurrentTimestamp());//获取当前时间戳 + nsp_name = get_namespace_name(get_rel_namespace(relid));//根据给定的命名空间OID获取关系所在命名空间名，来源src\common\backend\utils\cache\lsyscache.cpp + rel_name = get_rel_name(relid);//根据给定的关系OID获取关系名，来源src\common\backend\utils\cache\lsyscache.cpp /* Make combine string of current record: rel_name + nsp_name + query_string + rel_hash */ + /* 生成当前记录的组合字符串: rel_name + nsp_name + query_string + rel_hash */ + //调用本文件中的函数 combine_string = set_gchain_comb_string(db_name, user_name, nsp_name, rel_name, query_string, cn_hash); /* - * rel_hash: sum of hash in DN which generated by this query_string. - * globalhash: hash for last record of gs_global_chain, it means blockchain prevhash. + * rel_hash: sum of hash in DN which generated by this query_string.//由此查询字符串在DN中生成的哈希值之和 + * globalhash: hash for last record of gs_global_chain, it means blockchain prevhash.//gs_global_chain的最后一条记录的哈希值，也即区块链的前一个哈希值 */ - current_block.blocknum = get_next_g_blocknum(); - gen_global_hash(¤t_block.globalhash, combine_string, false, NULL); + current_block.blocknum = get_next_g_blocknum();// 获取下一个区块号 + gen_global_hash(¤t_block.globalhash, combine_string, false, NULL);// 创建下一个块并生成MD5哈希值 + // 准备要插入到全局链表中的数据 + values[Anum_gs_global_chain_blocknum - 1] = UInt64GetDatum(current_block.blocknum);//将当前块的块号（blocknum）转换为UInt64类型，并将其存储在values数组的对应位置上。 + values[Anum_gs_global_chain_dbname - 1] = DirectFunctionCall1(namein, CStringGetDatum(db_name));//将数据库名（db_name）转换为namein函数所需的输入参数类型，并将其存储在values数组的对应位置上 + values[Anum_gs_global_chain_username - 1] = DirectFunctionCall1(namein, CStringGetDatum(user_name));//将用户名（user_name）转换为namein函数所需的输入参数类型，并将其存储在values数组的对应位置上 + values[Anum_gs_global_chain_starttime - 1] = current_time;//将当前时间（current_time）存储在values数组的对应位置上 + values[Anum_gs_global_chain_relid - 1] = ObjectIdGetDatum(relid);//将关系ID（relid）转换为ObjectId类型，并将其存储在values数组的对应位置上 + values[Anum_gs_global_chain_relnsp - 1] = DirectFunctionCall1(namein, CStringGetDatum(nsp_name));//将命名空间名（nsp_name）转换为namein函数所需的输入参数类型，并将其存储在values数组的对应位置上 + values[Anum_gs_global_chain_relname - 1] = DirectFunctionCall1(namein, CStringGetDatum(rel_name));//将关系名（rel_name）转换为namein函数所需的输入参数类型，并将其存储在values数组的对应位置上 + values[Anum_gs_global_chain_relhash - 1] = UInt64GetDatum(cn_hash);//将关系哈希值（cn_hash）转换为UInt64类型，并将其存储在values数组的对应位置上 + values[Anum_gs_global_chain_globalhash - 1] = HASH32GetDatum(¤t_block.globalhash);//将当前块的全局哈希值（current_block.globalhash）转换为HASH32类型，并将其存储在values数组的对应位置上 + values[Anum_gs_global_chain_txcommand - 1] = CStringGetTextDatum(query_string);//将查询字符串（query_string）转换为text类型，并将其存储在values数组的对应位置上 - values[Anum_gs_global_chain_blocknum - 1] = UInt64GetDatum(current_block.blocknum); - values[Anum_gs_global_chain_dbname - 1] = DirectFunctionCall1(namein, CStringGetDatum(db_name)); - values[Anum_gs_global_chain_username - 1] = DirectFunctionCall1(namein, CStringGetDatum(user_name)); - values[Anum_gs_global_chain_starttime - 1] = current_time; - values[Anum_gs_global_chain_relid - 1] = ObjectIdGetDatum(relid); - values[Anum_gs_global_chain_relnsp - 1] = DirectFunctionCall1(namein, CStringGetDatum(nsp_name)); - values[Anum_gs_global_chain_relname - 1] = DirectFunctionCall1(namein, CStringGetDatum(rel_name)); - values[Anum_gs_global_chain_relhash - 1] = UInt64GetDatum(cn_hash); - values[Anum_gs_global_chain_globalhash - 1] = HASH32GetDatum(¤t_block.globalhash); - values[Anum_gs_global_chain_txcommand - 1] = CStringGetTextDatum(query_string); + rel_gchain = heap_open(GsGlobalChainRelationId, RowExclusiveLock);// 打开全局链表关系 + tup = heap_form_tuple(rel_gchain->rd_att, values, nulls);//创建新的堆元组 - rel_gchain = heap_open(GsGlobalChainRelationId, RowExclusiveLock); - tup = heap_form_tuple(rel_gchain->rd_att, values, nulls); - - simple_heap_insert(rel_gchain, tup); - heap_freetuple(tup); + simple_heap_insert(rel_gchain, tup);//插入堆元组到全局链表中 + //插入方法来源src\gausskernel\storage\access\heap\heapam.cpp + //通过获取事务标识符，检查冲突，准备缓冲区，将元组插入到关系中，并处理可见性和日志记录等步骤，实现了数据的插入 + heap_freetuple(tup);//释放堆元组内存 /* set latest previous global chain block */ - heap_close(rel_gchain, RowExclusiveLock); - pfree(combine_string); + /* 设置最新的上一个全局链块 */ + + heap_close(rel_gchain, RowExclusiveLock);//关闭全局链表关系 + pfree(combine_string);//释放组合字符串内存 } /* - * ledger_output_append_hash -- append relhash to response tag. + * ledger_output_append_hash -- append relhash to response tag.//将relhash附加到响应标签中 * - * resp_tag: response tag address. - * operation: command operation. - * hash: the hash that prepare to append. + * resp_tag: response tag address.//响应标签的地址 + * operation: command operation.//命令操作类型 + * hash: the hash that prepare to append.//准备附加的哈希值 */ +//该函数的作用是在执行插入、更新和删除操作时，将关系哈希值追加到响应标签中，以便后续处理和记录。 static void ledger_output_append_hash(char *resp_tag, CmdType operation, uint64 hash) { - Assert(resp_tag != NULL); - size_t len = strlen(resp_tag); - errno_t ret = EOK; + Assert(resp_tag != NULL); //确保resp_tag不为空 + size_t len = strlen(resp_tag);//获取resp_tag的长度 + errno_t ret = EOK;//初始化错误号 switch (operation) { case CMD_INSERT: case CMD_UPDATE: case CMD_DELETE: + //将哈希值转换为字符串，并追加到resp_tag字符串末尾 ret = snprintf_s(resp_tag + len, COMPLETION_TAG_BUFSIZE - len, COMPLETION_TAG_BUFSIZE - len - 1, - " %lu\0", hash); - securec_check_ss(ret, "\0", "\0"); + " %lu\0", hash);// 追加的位置是resp_tag的末尾，COMPLETION_TAG_BUFSIZE - len为可添加的最大字符数，COMPLETION_TAG_BUFSIZE - len - 1为要写入的字符串的最大长度 + securec_check_ss(ret, "\0", "\0");//检查函数调用是否成功 break; default: break; @@ -193,34 +217,39 @@ static void ledger_output_append_hash(char *resp_tag, CmdType operation, uint64 } /* - * ledger_ExecutorEnd -- record block to gchain. + * ledger_ExecutorEnd -- record block to gchain.//将块记录到gchain中 * - * query_desc: query descript of executor. + * query_desc: query descript of executor.//执行器的查询描述 * - * Note: append new block to gchain in CN or singlenode. + * Note: append new block to gchain in CN or singlenode.//在CN或单节点中，将新的块追加到gchain中 * each DN will link its relhash to es_modifiedRowHash, CN or singlenode * use es_modifiedRowHash to receive all DN relhash and accumulate them * as cn_relhash for insertion. + * 每个DN将其关系哈希链接到es_modifiedRowHash，CN或单节点使用es_modifiedRowHash接收所有DN的关系哈希并将它们累积为cn_relhash进行插入 */ +//是一个钩子函数，在查询执行完成后被调用。它主要的功能是记录查询的相关信息到全局链表中， +//包括关系ID、查询字符串、哈希值等。同时，它还检查并将哈希值追加到响应标签中。 +//负责触发记录操作 static void ledger_ExecutorEnd(QueryDesc *query_desc) { - uint64 hashsum; - bool has_remote_hash = query_desc->estate->es_modifiedRowHash != NIL; - hashsum = hash_combiner(query_desc->estate->es_modifiedRowHash); + uint64 hashsum; //哈希值的总和 + bool has_remote_hash = query_desc->estate->es_modifiedRowHash != NIL;//是否存在远程哈希 + hashsum = hash_combiner(query_desc->estate->es_modifiedRowHash);//组合远程哈希的哈希值 if ((IS_PGXC_COORDINATOR || g_instance.role == VSINGLENODE) && has_remote_hash) { + //如果当前节点为PGXC_COORDINATOR（协调节点）或VSINGLENODE（单节点），并且是否存在远程哈希 Oid relid = InvalidOid; Relation rel = NULL; - int relnum = query_desc->estate->es_num_result_relations; + int relnum = query_desc->estate->es_num_result_relations;//结果关系数量 if (relnum > 0) { rel = query_desc->estate->es_result_relations->ri_RelationDesc; /* gs_global_chain only records following actions */ - switch (query_desc->operation) { + switch (query_desc->operation) {//根据查询到的操作类型 case CMD_INSERT: case CMD_DELETE: case CMD_UPDATE: - relid = RelationGetRelid(rel); - if (rel->rd_isblockchain) { - ledger_gchain_append(relid, query_desc->sourceText, hashsum); + relid = RelationGetRelid(rel); //获取关系的ID + if (rel->rd_isblockchain) {//如果是块链表 + ledger_gchain_append(relid, query_desc->sourceText, hashsum);//将关系哈希信息追加到gchain中 } break; default: @@ -230,57 +259,68 @@ static void ledger_ExecutorEnd(QueryDesc *query_desc) } if (u_sess->ledger_cxt.resp_tag != NULL && has_remote_hash && !IsConnFromApp()) { + //如果相应标签地址的指针不为空&&存在远程哈希并且&&非应用程序连接，则将哈希值追加到响应标签中 ledger_output_append_hash(u_sess->ledger_cxt.resp_tag, query_desc->operation, hashsum); - u_sess->ledger_cxt.resp_tag = NULL; + u_sess->ledger_cxt.resp_tag = NULL;//清空响应标签 } if (t_thrd.security_ledger_cxt.prev_ExecutorEnd) { - ((ExecutorEnd_hook_type)t_thrd.security_ledger_cxt.prev_ExecutorEnd)(query_desc); + ((ExecutorEnd_hook_type)t_thrd.security_ledger_cxt.prev_ExecutorEnd)(query_desc); // 调用前一个ExecutorEnd钩子函数 } else { - standard_ExecutorEnd(query_desc); + standard_ExecutorEnd(query_desc); //执行标准的ExecutorEnd操作 + //函数来源src\gausskernel\runtime\executor\execMain.cpp + //该函数功能为释放快照、LLVM 编译清理、切换上下文并释放内存、重置查询描述的字段、输出内存追踪信息到文件、收集指令计数信息、重置永久空间（perm space）的全局值 } } /* - * light_ledger_ExecutorEnd -- record block to gchain in light proxy. + * light_ledger_ExecutorEnd -- record block to gchain in light proxy.//在轻量级代理中将块记录到全局链（gchain）中 * - * query: query of executor. - * relhash: sum of all DN relhash for insertion. + * query: query of executor.//执行器的查询 + * relhash: sum of all DN relhash for insertion.//插入操作的所有分布式节点关系哈希值的总和 * * Note: in light proxy logical, process will extract and * accumulate relhash from response text of DNs as cn_relhash. + * 在轻量级代理的逻辑中，处理过程会从分布式节点的响应文本中提取并累加关系哈希值作为cn_relhash */ +//根据传入的查询信息判断关系是否为用户表，如果是，则将关系的哈希信息追加到gchain中，以记录区块链中的数据变动。 void light_ledger_ExecutorEnd(Query *query, uint64 relhash) { + //如果不是PGXC协调器节点，并且当前节点的角色不是单节点（VSINGLENODE） if (!IS_PGXC_COORDINATOR && g_instance.role != VSINGLENODE) { return; } - Oid relid = InvalidOid; + Oid relid = InvalidOid;//关系ID，默认为无效ID - switch (query->commandType) { + switch (query->commandType) {//获取查询对象的命令类型 case CMD_INSERT: case CMD_DELETE: case CMD_UPDATE: - relid = get_target_query_relid(query->rtable, query->resultRelation); - if (is_ledger_usertable(relid)) { - ledger_gchain_append(relid, query->sql_statement, relhash); + relid = get_target_query_relid(query->rtable, query->resultRelation);//获取目标查询的关系ID + //函数来源src\gausskernel\security\gs_ledger\ledger_utils.cpp + if (is_ledger_usertable(relid)) {//根据关系ID的有效性、关系类型和所属命名空间判断关系是否为用户表 + //函数来源src\gausskernel\security\gs_ledger\ledger_utils.cpp + ledger_gchain_append(relid, query->sql_statement, relhash);//将关系哈希信息追加到gchain中 } break; /* Not support others */ + //其他指令不做处理 default: break; } } /* - * light_ledger_ExecutorEnd -- record block to gchain in opfusion. + * light_ledger_ExecutorEnd -- record block to gchain in opfusion.//在操作融合（opfusion）中将块记录到全局链（gchain）中 * - * fusiontype: operator type. - * relid: relation oid of usertable. - * query: original query which modified usertable. - * relhash: relhash in hist table generated by sourceText. + * fusiontype: operator type.//操作类型 + * relid: relation oid of usertable.//用户表的关系OID + * query: original query which modified usertable.//修改了用户表的原始查询 + * relhash: relhash in hist table generated by sourceText.//由源文本生成的历史表中的关系哈希值 */ +//和前一个函数相比，此函数判断执行条件时要求当前节点的角色是虚拟数据节点，或者关系ID对应的表不是用户表。。 void opfusion_ledger_ExecutorEnd(FusionType fusiontype, Oid relid, const char *query, uint64 relhash) { + //当前节点的角色是VDATANODE（虚拟数据节点）或者关系ID对应的表不是用户表 if (g_instance.role == VDATANODE || !is_ledger_usertable(relid)) { return; } @@ -289,8 +329,8 @@ void opfusion_ledger_ExecutorEnd(FusionType fusiontype, Oid relid, const char *q case INSERT_FUSION: case UPDATE_FUSION: case DELETE_FUSION: - if (is_ledger_usertable(relid)) { - ledger_gchain_append(relid, query, relhash); + if (is_ledger_usertable(relid)) {//根据关系ID的有效性、关系类型和所属命名空间判断关系是否为用户表 + ledger_gchain_append(relid, query, relhash);//将关系哈希信息追加到gchain中 } break; /* Not support others */ @@ -302,16 +342,23 @@ void opfusion_ledger_ExecutorEnd(FusionType fusiontype, Oid relid, const char *q /* * ledger_hook_init -- install of gchain block record hook. */ +//初始化钩子函数，将自定义的执行器结束钩子函数 ledger_ExecutorEnd 替换掉原有的钩子函数 ExecutorEnd_hook， +//从而在执行器结束时触发自定义的操作。这样可以实现对关系的哈希信息追加到 gchain 中的功能。 void ledger_hook_init(void) { + //将原先的 ExecutorEnd_hook 函数保存到 prev_ExecutorEnd 变量中 t_thrd.security_ledger_cxt.prev_ExecutorEnd = (void *)ExecutorEnd_hook; + //将 ledger_ExecutorEnd 函数赋值给 ExecutorEnd_hook，以替换原有的钩子函数 ExecutorEnd_hook = ledger_ExecutorEnd; } /* * ledger_hook_fini -- uninstall of gchain block record hook. */ +//恢复原始的钩子函数，将之前保存的 prev_ExecutorEnd 变量的值重新赋值给 ExecutorEnd_hook，以恢复原始的钩子函数的功能。这样可以确保在钩子函数替换后，再次恢复原有的钩子函数，避免对系统功能产生影响。 void ledger_hook_fini(void) { + //将prev_ExecutorEnd 变量的值赋值给 ExecutorEnd_hook，恢复原始的钩子函数 ExecutorEnd_hook = (ExecutorEnd_hook_type)t_thrd.security_ledger_cxt.prev_ExecutorEnd; } + -- 2.34.1 From e9ff3a3412ba30678af25164b1fbd648c16329c1 Mon Sep 17 00:00:00 2001 From: Auweafan Date: Sun, 27 Aug 2023 22:33:24 +0800 Subject: [PATCH 04/48] Update gs_policy_audit.cpp --- .../security/gs_policy/gs_policy_audit.cpp | 374 ++++++++++-------- 1 file changed, 216 insertions(+), 158 deletions(-) diff --git a/src/gausskernel/security/gs_policy/gs_policy_audit.cpp b/src/gausskernel/security/gs_policy/gs_policy_audit.cpp index 6209b6bd4..b509d9442 100644 --- a/src/gausskernel/security/gs_policy/gs_policy_audit.cpp +++ b/src/gausskernel/security/gs_policy/gs_policy_audit.cpp @@ -48,19 +48,20 @@ #include "utils/syscache.h" #include "pgaudit.h" -LoadPoliciesPtr load_audit_policies_hook = NULL; -LoadPolicyAccessPtr load_policy_access_hook = NULL; -LoadPolicyPrivilegesPtr load_policy_privileges_hook = NULL; -LoadPolicyFilterPtr load_policy_filter_hook = NULL; -THR_LOCAL LightUnifiedAuditExecutorPtr light_unified_audit_executor_hook = NULL; -OpFusionUnifiedAuditExecutorPtr opfusion_unified_audit_executor_hook = NULL; -OpFusionUnifiedAuditFlushLogsPtr opfusion_unified_audit_flush_logs_hook = NULL; +//函数指针变量 +LoadPoliciesPtr load_audit_policies_hook = NULL; //加载审计策略。 +LoadPolicyAccessPtr load_policy_access_hook = NULL; //加载策略的访问权限。 +LoadPolicyPrivilegesPtr load_policy_privileges_hook = NULL; //加载策略的权限。 +LoadPolicyFilterPtr load_policy_filter_hook = NULL; //加载策略的过滤器。 +THR_LOCAL LightUnifiedAuditExecutorPtr light_unified_audit_executor_hook = NULL; //用于执行轻量级统一审计。 +OpFusionUnifiedAuditExecutorPtr opfusion_unified_audit_executor_hook = NULL; //执行操作融合的统一审计 +OpFusionUnifiedAuditFlushLogsPtr opfusion_unified_audit_flush_logs_hook = NULL; //刷新操作融合的统一审计日志。 static const char* privileges_type[] = { "alter", "analyze", "comment", "create", "drop", "grant", "revoke", - "set", "show", "login_any", "login_failure", "login_success", "logout"}; + "set", "show", "login_any", "login_failure", "login_success", "logout"};//用于描述权限的种类 static const char* access_type[] = {"copy", "deallocate", "delete", "execute", "insert", "prepare", "reindex", - "select", "truncate", "update"}; + "select", "truncate", "update"};//用于描述访问的方式 /* reserve for masking with function name get_option_type */ /* reserver for maksing with function name construct_resource_name */ @@ -74,7 +75,12 @@ static const char* access_type[] = {"copy", "deallocate", "delete", "execute", " * @policyOid : policy id */ static void add_action_type(bool is_access, const char *action_type, const gs_stl::gs_string target_name_s, - Relation relation, Oid policyOid) + Relation relation, Oid policyOid)//向指定的关系中添加访问配置或特权配置 + /*is_access（布尔类型，表示是否添加访问配置） + action_type（指向字符的指针，表示要添加的配置类型） + target_name_s（gs_string 类型，表示目标名称） + relation（Relation 类型，表示关系对象） + policyOid（Oid 类型，表示策略ID）*/ { HeapTuple policy_htup = NULL; if (is_access) { @@ -95,9 +101,10 @@ static void add_action_type(bool is_access, const char *action_type, const gs_st policy_htup = heap_form_tuple(relation->rd_att, pol_values, pol_nulls); } /* Do the insertion */ - (void)simple_heap_insert(relation, policy_htup); - CatalogUpdateIndexes(relation, policy_htup); - heap_freetuple(policy_htup); + //将创建的元组插入到指定的关系中，并更新关系的索引。 + (void)simple_heap_insert(relation, policy_htup);//将新的行数据插入到指定的堆表中 + CatalogUpdateIndexes(relation, policy_htup);//对指定的目录索引进行更新 + heap_freetuple(policy_htup);//释放 policy_htup 所占用的内存空间 } /* @@ -105,19 +112,23 @@ static void add_action_type(bool is_access, const char *action_type, const gs_st */ static void handle_add_remove_all_types(int opt_type, privileges_access_set *add_actions, privileges_access_set *rem_actions, const privileges_access_set *exist_actions, bool is_add, long long polID, - const char *object) + const char *object)//处理添加和删除权限操作的函数 { /* add all supported type */ PgPolicyPrivilegesAccessStruct item; item.m_label_name = "all"; item.m_policy_oid = polID; - if (strcasecmp(object, "all")) { + + // 如果 object 不是 "all"，执行单个类型的操作 + if (strcasecmp(object, "all")) {//比较两个字符串（不区分大小写），判断它们是否相等。 item.m_type = object; if (is_add) { + // 如果该类型不存在于已有权限中，将其添加到 add_actions 中 if (exist_actions->find(item) == exist_actions->end()) { (void)add_actions->insert(item); } } else { + // 将该类型的权限从 exist_actions 中移除并加入到 rem_actions 中 privileges_access_set::const_iterator it = exist_actions->find(item); if (it != exist_actions->end()) { (void)rem_actions->insert(*it); @@ -125,17 +136,19 @@ static void handle_add_remove_all_types(int opt_type, privileges_access_set *add } return; } - + // 遍历所有支持的类型，执行全部类型的操作 int array_size = (opt_type == POLICY_OPT_ACCESS) ? (sizeof(access_type) / sizeof(access_type[0])) : (sizeof(privileges_type) / sizeof(privileges_type[0])); for (int i = 0; i < array_size; ++i) { item.m_type = (opt_type == POLICY_OPT_ACCESS) ? access_type[i] : privileges_type[i]; if (is_add) { + // 如果该类型不存在于已有权限中，将其添加到 add_actions 中 if (exist_actions->find(item) == exist_actions->end()) { (void)add_actions->insert(item); } } else { + // 将该类型的权限从 exist_actions 中移除并加入到 rem_actions 中 (void)rem_actions->insert(item); } } @@ -153,11 +166,11 @@ static inline void add_all_supported_types(bool is_access, const gs_stl::gs_stri { /* add all supported types */ int array_size = is_access ? (sizeof(access_type) / sizeof(access_type[0])) : - (sizeof(privileges_type) / sizeof(privileges_type[0])); + (sizeof(privileges_type) / sizeof(privileges_type[0]));//计算了数组的大小 for (int i = 0; i < array_size; ++i) { const char *action_type = is_access ? access_type[i] : privileges_type[i]; add_action_type(is_access, action_type, target_name_s, relation, policyOid); - } + }//遍历数组中的元素，将每个元素传递给 add_action_type 函数进行后续处理。 } /** @@ -175,7 +188,7 @@ static void add_labels_to_policy(bool is_access, const char *action_type, DefEle ListCell *target_name = NULL; /* no targets - means for all db objects */ - if (targets == NULL) { + if (targets == NULL) {//针对所有数据库对象进行操作 if (strcasecmp(action_type, "all") == 0) { add_all_supported_types(is_access, "all", relation, policyOid); } else { @@ -185,7 +198,7 @@ static void add_labels_to_policy(bool is_access, const char *action_type, DefEle } /* one or more targets */ - foreach (target_name, targets) { + foreach (target_name, targets) {//有一个或多个资源标签。函数使用 foreach 循环遍历 targets 列表中的每个元素。 RangeVar *rel = (RangeVar*)lfirst(target_name); gs_stl::gs_string target_name_s; construct_resource_name(rel, &target_name_s); @@ -198,7 +211,7 @@ static void add_labels_to_policy(bool is_access, const char *action_type, DefEle } else { if (verify_label_hook) { /* validate whether this label exists */ if (!verify_label_hook(target_name_s.c_str())) { - heap_close(relation, RowExclusiveLock); + heap_close(relation, RowExclusiveLock);//关闭打开的关系，并释放与之相关的资源。 ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("[%s] no such label found", target_name_s.c_str()))); } @@ -221,21 +234,22 @@ static void add_labels_to_policy(bool is_access, const char *action_type, DefEle * @relation : access catalog or privilege catalog. */ static void add_labels_to_privileges_access(bool is_access, const privileges_access_set *actions, - const GsPolicyStruct *policy, Relation relation) + const GsPolicyStruct *policy, Relation relation)//将标签添加到权限访问列表中。 { + //遍历权限访问列表 actions for (privileges_access_set::const_iterator it = actions->begin(); it != actions->end(); ++it) { - HeapTuple policy_htup = NULL; + HeapTuple policy_htup = NULL;//存储堆元组 const char *action_type = it->m_type.c_str(); gs_stl::gs_string target_name_s = it->m_label_name; Oid policyOid = policy->m_id; - if (verify_label_hook) { - if (!verify_label_hook(target_name_s.c_str())) { + if (verify_label_hook) {// + if (!verify_label_hook(target_name_s.c_str())) {//用于检查 target_name_s 是否存在有效的标签。 heap_close(relation, RowExclusiveLock); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("[%s] no such label found", target_name_s.c_str()))); } } - + //根据is_access的值创建堆元组 if (is_access) { bool policy_nulls[Natts_gs_auditing_policy_acc] = {false}; Datum policy_values[Natts_gs_auditing_policy_acc] = {0}; @@ -254,25 +268,27 @@ static void add_labels_to_privileges_access(bool is_access, const privileges_acc policy_htup = heap_form_tuple(relation->rd_att, policy_values, policy_nulls); } /* Do the insertion */ + //堆元组插入关系中 (void)simple_heap_insert(relation, policy_htup); - + //更新索引 CatalogUpdateIndexes(relation, policy_htup); - heap_freetuple(policy_htup); + heap_freetuple(policy_htup);//释放堆元组内存： } } static bool remove_labels_from_privileges_access(bool is_access, const privileges_access_set *actions, - privileges_access_set *existing_actions, Relation relation, gs_stl::gs_string *err_msg) + privileges_access_set *existing_actions, Relation relation, gs_stl::gs_string *err_msg)//从权限访问列表中移除标签 { bool is_deleted = false; - for (privileges_access_set::const_iterator it = actions->begin(); it != actions->end(); ++it) { + for (privileges_access_set::const_iterator it = actions->begin(); it != actions->end(); ++it) {//遍历要移除的标签 actions /* Removing access or privilege from policy having only one item is not allowed */ - if (existing_actions->size() == 1) { + if (existing_actions->size() == 1) {//检查存在的权限访问列表 existing_actions 的大小，如果只有一个标签，则不允许移除操作，并设置错误消息 *err_msg = (is_access) ? "Removing auditing access from policy with a single item not allowed" : "Removing auditing privilege from policy with a single item not allowed"; break; } - privileges_access_set::iterator i_it = existing_actions->find(*it); + privileges_access_set::iterator i_it = existing_actions->find(*it);//在现有的权限访问列表 existing_actions 中查找要移除的标签 + //如果找到要移除的标签，则从关系中进行删除操作，并更新权限访问列表 existing_actions if (i_it != existing_actions->end()) { if (!scan_to_delete_from_relation(i_it->m_id, relation, is_access ? GsAuditingPolicyAccessOidIndexId : GsAuditingPolicyPrivilegesOidIndexId)) @@ -281,13 +297,13 @@ static bool remove_labels_from_privileges_access(bool is_access, const privilege is_deleted = true; } } - return is_deleted; + return is_deleted;//返回是否有标签被成功移除的标志位 } /** * Add filter expr information into auditing policy. */ -static void add_filters(const filters_set *filters_to_add, Relation relation) +static void add_filters(const filters_set *filters_to_add, Relation relation)//在审计策略中添加过滤器信息。 { Datum curtime; HeapTuple policy_filters_htup; @@ -295,25 +311,29 @@ static void add_filters(const filters_set *filters_to_add, Relation relation) Datum policy_filters_values[Natts_gs_auditing_policy_filters]; errno_t rc; /* Get current timestamp */ - curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp()); + curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp());//获取当前的时间戳 + //遍历要添加的过滤器表达式集合 for (filters_set::const_iterator it = filters_to_add->begin(); it != filters_to_add->end(); ++it) { - /* restore values and nulls for insert new node group record */ + //为插入新的节点组记录恢复值和空值 rc = memset_s(policy_filters_values, sizeof(policy_filters_values), 0, sizeof(policy_filters_values)); securec_check(rc, "\0", "\0"); rc = memset_s(policy_filters_nulls, sizeof(policy_filters_nulls), false, sizeof(policy_filters_nulls)); securec_check(rc, "\0", "\0"); + //设置要插入的字段值 policy_filters_values[Anum_gs_auditing_policy_fltr_filter_type - 1] = DirectFunctionCall1(namein, CStringGetDatum(it->m_type.c_str())); policy_filters_values[Anum_gs_auditing_policy_fltr_label_name - 1] = DirectFunctionCall1(namein, CStringGetDatum(it->m_label_name.c_str())); policy_filters_values[Anum_gs_auditing_policy_fltr_logical_operator - 1] = CStringGetTextDatum(it->m_tree_string.c_str()); policy_filters_values[Anum_gs_auditing_policy_fltr_policy_oid - 1] = ObjectIdGetDatum(it->m_policy_oid); policy_filters_values[Anum_gs_auditing_policy_fltr_modify_date - 1] = curtime; + //创建一个堆元组 policy_filters_htup = heap_form_tuple(relation->rd_att, policy_filters_values, policy_filters_nulls); - /* Do the insertion */ + //进行插入操作 (void)simple_heap_insert(relation, policy_filters_htup); - + //更新索引 CatalogUpdateIndexes(relation, policy_filters_htup); + //释放堆元组的内存 heap_freetuple(policy_filters_htup); } } @@ -321,21 +341,27 @@ static void add_filters(const filters_set *filters_to_add, Relation relation) /** * Update filter expr information into auditing policy. */ -static void update_filters(const filters_set *filters_to_update, Relation policy_filters_relation) +static void update_filters(const filters_set *filters_to_update, Relation policy_filters_relation)//更新过滤器信息到审计策略 { + //进入循环，遍历要更新的过滤器表达式集合 filters_to_update for (filters_set::const_iterator it = filters_to_update->begin(); it != filters_to_update->end(); ++it) { ScanKeyData scanKey[1]; + //设置扫描键值 scanKey ScanKeyInit(&scanKey[0], Anum_gs_auditing_policy_fltr_policy_oid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(it->m_policy_oid)); /* Search tuple by index */ + //使用索引开始扫描匹配的元组 SysScanDesc scanDesc = systable_beginscan(policy_filters_relation, GsAuditingPolicyFiltersPolicyOidIndexId, true, NULL, 1, scanKey); - + //获取下一个匹配的元组 HeapTuple auditingPolicyTuple = systable_getnext(scanDesc); + //检查是否获取到有效的元组。如果没有，调用 add_filters 函数添加新的过滤器表达式记录 if (!HeapTupleIsValid(auditingPolicyTuple)) { add_filters(filters_to_update, policy_filters_relation); /* curtime */ - } else { + } + //如果获取到有效的元组，创建值、空值和替换数组，并设置要更新的字段值 + else { Datum values[Natts_gs_auditing_policy_filters] = { 0 }; bool nulls[Natts_gs_auditing_policy_filters] = { false }; bool replaces[Natts_gs_auditing_policy_filters] = { false }; @@ -350,27 +376,32 @@ static void update_filters(const filters_set *filters_to_update, Relation policy values[Anum_gs_auditing_policy_fltr_logical_operator - 1] = CStringGetTextDatum(it->m_tree_string.c_str()); nulls[Anum_gs_auditing_policy_fltr_logical_operator - 1] = false; replaces[Anum_gs_auditing_policy_fltr_logical_operator - 1] = true; - + + //使用新的字段值、空值和替换数组创建新的堆元组 newtuple HeapTuple newtuple = heap_modify_tuple(auditingPolicyTuple, RelationGetDescr(policy_filters_relation), values, nulls, replaces); + //执行堆更新操作 simple_heap_update(policy_filters_relation, &newtuple->t_self, newtuple); + //更新索引 CatalogUpdateIndexes(policy_filters_relation, newtuple); } + //结束扫描 systable_endscan(scanDesc); } } static void handle_alter_add_update_filter(List *filter, Oid policyOid, bool to_add) { + //检查过滤器是否为空。如果为空，直接返回 if (filter == NULL) { return; } - + //处理新的过滤器，并将过滤器转换成扁平树的字符串形式 gs_stl::gs_string flat_tree; if (!process_new_filters(filter, &flat_tree)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("Unsupported policy filter values"))); } - + //创建一个过滤器集合 filters_to_alter，并将转换后的过滤器信息插入到集合中 filters_set filters_to_alter; if (flat_tree.size() > 0) { PgPolicyFiltersStruct item; @@ -380,13 +411,14 @@ static void handle_alter_add_update_filter(List *filter, Oid policyOid, bool to_ item.m_policy_oid = policyOid; (void)filters_to_alter.insert(item); } + //检查是否需要执行更新操作 if (filters_to_alter.size() > 0) { - Relation policy_filters_relation = heap_open(GsAuditingPolicyFiltersRelationId, RowExclusiveLock); - if (policy_filters_relation) { - if (to_add) { + Relation policy_filters_relation = heap_open(GsAuditingPolicyFiltersRelationId, RowExclusiveLock);//检查是否需要执行更新操作 + if (policy_filters_relation) {//打开审计策略过滤器关系 + if (to_add) {//调用 add_filters 函数添加新的过滤器表达式记录 add_filters(&filters_to_alter, policy_filters_relation); - } else { + } else {//调用 update_filters 函数更新过滤器表达式记录 update_filters(&filters_to_alter, policy_filters_relation); } heap_close(policy_filters_relation, RowExclusiveLock); @@ -397,27 +429,32 @@ static void handle_alter_add_update_filter(List *filter, Oid policyOid, bool to_ /* * Load existing auditing policy about DML synatax. */ -static void load_existing_privileges(privileges_access_set *privs, long long policy_oid) +static void load_existing_privileges(privileges_access_set *privs, long long policy_oid)//加载关于DML语法的现有审计策略 { + //打开审计策略特权关系的关系对象 Relation relation = heap_open(GsAuditingPolicyPrivilegesRelationId, RowExclusiveLock); if (relation == NULL) { - return; + return;//如果关系对象为空，则直接返回 } HeapTuple rtup; Form_gs_auditing_policy_privileges rel_data; PgPolicyPrivilegesAccessStruct item; + //对审计策略特权关系进行扫描，并获取下一个 tuple TableScanDesc scan = tableam_scan_begin(relation, SnapshotNow, 0, NULL); while (scan && (rtup = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection))) { rel_data = (Form_gs_auditing_policy_privileges)GETSTRUCT(rtup); if (rel_data == NULL) { continue; } + //将 tuple 中的相关信息存入 item 结构体中 item.m_id = HeapTupleGetOid(rtup); item.m_type = rel_data->privilegetype.data; item.m_label_name = rel_data->labelname.data; item.m_policy_oid = (long long)(rel_data->policyoid); + /* load only matching privileges to policy id */ + //仅加载与给定的 policy_oid 相匹配的特权信息，并将其插入到 privs 集合中 if (item.m_policy_oid == policy_oid) { (void)privs->insert(item); } @@ -432,14 +469,16 @@ static void load_existing_privileges(privileges_access_set *privs, long long pol */ static void load_existing_access(privileges_access_set *acc, long long policy_oid) { + //打开审计策略访问权限关系的关系对象 relation Relation relation = heap_open(GsAuditingPolicyAccessRelationId, RowExclusiveLock); - if (!relation) { + if (!relation) {//关系对象为空，则直接返回 return; } HeapTuple rtup; Form_gs_auditing_policy_access rel_data; PgPolicyPrivilegesAccessStruct item; + //开始对审计策略访问权限关系进行扫描，并获取下一个 tuple TableScanDesc scan = tableam_scan_begin(relation, SnapshotNow, 0, NULL); while (scan && (rtup = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection))) { rel_data = (Form_gs_auditing_policy_access)GETSTRUCT(rtup); @@ -451,6 +490,7 @@ static void load_existing_access(privileges_access_set *acc, long long policy_oi item.m_label_name = rel_data->labelname.data; item.m_policy_oid = (long long)(rel_data->policyoid); /* load only matching access to policy id */ + //仅加载与给定的 policy_oid 相匹配的特权信息，并将其插入到 privs 集合中 if (item.m_policy_oid == policy_oid) { (void)acc->insert(item); } @@ -462,7 +502,7 @@ static void load_existing_access(privileges_access_set *acc, long long policy_oi static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool policy_status_changed, - gs_stl::gs_string *err_msg) + gs_stl::gs_string *err_msg)//更新审计策略的函数 { bool policy_nulls[Natts_gs_auditing_policy]; bool policy_replaces[Natts_gs_auditing_policy]; @@ -471,6 +511,7 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool errno_t rc; /* restore values and nulls for insert new node group record */ + //初始化 rc = memset_s(policy_values, sizeof(policy_values), 0, sizeof(policy_values)); securec_check(rc, "", ""); rc = memset_s(policy_nulls, sizeof(policy_nulls), false, sizeof(policy_nulls)); @@ -478,6 +519,7 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool rc = memset_s(policy_replaces, sizeof(policy_replaces), false, sizeof(policy_replaces)); securec_check(rc, "", ""); + //设置用于扫描的键值 ScanKeyData skey; /* Find the policy row to update */ ScanKeyInit(&skey, ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(policy->m_id)); @@ -485,10 +527,11 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool /* * set up for heap-or-index scan, not need to check tgscan as systable_getnext will deal with sysscan->irel = NULL */ + //开始一个堆表扫描，并获取下一个符合条件的 tuple SysScanDesc tgscan = systable_beginscan(relation, GsAuditingPolicyOidIndexId, true, NULL, 1, &skey); HeapTuple tup; tup = systable_getnext(tgscan); - if (!tup || !HeapTupleIsValid(tup)) { + if (!tup || !HeapTupleIsValid(tup)) {//没有找到符合条件的 tuple，则结束扫描，返回错误信息 systable_endscan(tgscan); (void)err_msg->append("could not find tuple for policy "); (void)err_msg->append(policy->m_name); @@ -496,8 +539,10 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool } /* Get current timestamp */ + //获取当前时间戳 curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp()); + //根据 policy_status_changed 的值来决定哪些字段需要更新以及它们的值 if (policy_status_changed) { policy_replaces[Anum_gs_auditing_policy_pol_enabled - 1] = true; policy_values[Anum_gs_auditing_policy_pol_enabled - 1] = BoolGetDatum(policy->m_enabled); @@ -510,13 +555,15 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool policy_values[Anum_gs_auditing_policy_pol_modify_date - 1] = curtime; HeapTuple newtuple = heap_modify_tuple(tup, RelationGetDescr(relation), policy_values, policy_nulls, policy_replaces); + //更新关系中的 tuple simple_heap_update(relation, &newtuple->t_self, newtuple); + //更新索引 CatalogUpdateIndexes(relation, newtuple); + //结束扫描，返回成功 systable_endscan(tgscan); return true; } - /* * function name: create_audit_policy * description : create auditing policy @@ -524,13 +571,14 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool void create_audit_policy(CreateAuditPolicyStmt *stmt) { /* check that if has access to config audit policy */ - if (!is_policy_enabled()) { + // 检查是否有配置审计策略的权限 + if (!is_policy_enabled()) {//没有权限则返回错误 ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); return; } - + //获取当前用户的用户名和会话IP，并保存相关信息。 char user_name[USERNAME_LEN] = {0}; (void)GetRoleName(GetCurrentUserId(), user_name, sizeof(user_name)); char buff[512] = {0}; @@ -561,8 +609,8 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt) securec_check(rc, "", ""); /* start to process policy */ - Relation policy_relation = heap_open(GsAuditingPolicyRelationId, RowExclusiveLock); - if (!policy_relation) { + Relation policy_relation = heap_open(GsAuditingPolicyRelationId, RowExclusiveLock);//检查是否已打开审计策略的关系 + if (!policy_relation) {//未能成功打开，则返回错误。 /* generate an error */ send_manage_message(AUDIT_FAILED); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("%s", "failed to open policies relation"))); @@ -570,7 +618,8 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt) } /* no more than MAX_POLICIES_NUM is allowed */ - if (get_num_of_existing_policies(policy_relation) >= MAX_POLICIES_NUM) { + //检查是否已达到最大策略数量限制 + if (get_num_of_existing_policies(policy_relation) >= MAX_POLICIES_NUM) {//超过了最大限制，则返回错误。 heap_close(policy_relation, RowExclusiveLock); /* generate an error */ send_manage_message(AUDIT_FAILED); @@ -580,6 +629,7 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt) } /* check whether such policy exists */ + //检查是否已存在同名的策略 policies_set existing_policies; load_existing_policies(policy_relation, &existing_policies); @@ -587,16 +637,16 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt) cur_policy.m_name = policy_name; policies_set::iterator it = existing_policies.find(cur_policy); if (it != existing_policies.end()) { /* policy already exists */ - heap_close(policy_relation, RowExclusiveLock); + heap_close(policy_relation, RowExclusiveLock);// /* while the 'if not exists' is specified generate a notice, else an error */ if (stmt->if_not_exists == true) { send_manage_message(AUDIT_OK); - ereport(NOTICE, (errmsg("%s policy already exists, create skipping", policy_name))); + ereport(NOTICE, (errmsg("%s policy already exists, create skipping", policy_name)));//生成通知 } else { send_manage_message(AUDIT_FAILED); ereport(ERROR, (errcode(ERRCODE_DUPLICATE_POLICY), - errmsg("%s policy already exists, create failed", policy_name))); + errmsg("%s policy already exists, create failed", policy_name)));//返回错误 } return; } @@ -604,6 +654,7 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt) /* Get current timestamp */ curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp()); + //创建审计策略的元组并插入到关系中。 policy_values[Anum_gs_auditing_policy_pol_name - 1] = DirectFunctionCall1(namein, CStringGetDatum(policy_name)); policy_values[Anum_gs_auditing_policy_pol_comments - 1] = DirectFunctionCall1(namein, CStringGetDatum("")); policy_values[Anum_gs_auditing_policy_pol_modify_date - 1] = curtime; @@ -617,56 +668,59 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt) heap_close(policy_relation, RowExclusiveLock); /* Start to Process PRIVILEGES(DML) && ACCESS(DDL) exprs */ + //根据策略类型处理策略目标，将标签添加到相应的关系中 int opt_type = get_option_type(policy_type); /* Extract policy targets from the statement node tree */ + //从语句节点树中提取策略目标 foreach (policy_target_item, stmt->policy_targets) { DefElem *defel = (DefElem *) lfirst(policy_target_item); const char *action_type = defel->defname; /* action: DELETE, INSERT, UPDATE, etc. */ DefElem *policy_items = (DefElem *) defel->arg; - switch (opt_type) { - case POLICY_OPT_PRIVILEGES: { - Relation priv_relation = heap_open(GsAuditingPolicyPrivilegesRelationId, RowExclusiveLock); - if (priv_relation) { - add_labels_to_policy(false, action_type, policy_items, priv_relation, policyOid); - heap_close(priv_relation, RowExclusiveLock); + switch (opt_type) {//根据选项类型选择不同的处理方式。 + case POLICY_OPT_PRIVILEGES: {//策略类型为权限 + Relation priv_relation = heap_open(GsAuditingPolicyPrivilegesRelationId, RowExclusiveLock);//使用独占锁（RowExclusiveLock）进行访问 + if (priv_relation) {//打开成功 + add_labels_to_policy(false, action_type, policy_items, priv_relation, policyOid);//DML操作 + heap_close(priv_relation, RowExclusiveLock);//使用独占锁关闭了权限关系 } } break; - case POLICY_OPT_ACCESS: { - Relation acc_relation = heap_open(GsAuditingPolicyAccessRelationId, RowExclusiveLock); - if (acc_relation) { - add_labels_to_policy(true, action_type, policy_items, acc_relation, policyOid); - heap_close(acc_relation, RowExclusiveLock); + case POLICY_OPT_ACCESS: {//策略访问控制 + Relation acc_relation = heap_open(GsAuditingPolicyAccessRelationId, RowExclusiveLock);//使用独占锁（RowExclusiveLock）进行访问 + if (acc_relation) {//打开成功 + add_labels_to_policy(true, action_type, policy_items, acc_relation, policyOid);//DDL操作 + heap_close(acc_relation, RowExclusiveLock);//使用独占锁关闭了权限关系 } } break; /* handled later */ - case POLICY_OPT_FILTER: + case POLICY_OPT_FILTER://类型为策略过滤器 break; - default: { + default: {//未知的策略类型。 /* report about unknown policy type */ - ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("Unsupported policy type"))); + ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("Unsupported policy type")));//报错 } break; } } - handle_alter_add_update_filter(stmt->policy_filters, policyOid, true /* add new filter row */); - CommandCounterIncrement(); - send_manage_message(AUDIT_OK); + handle_alter_add_update_filter(stmt->policy_filters, policyOid, true /* add new filter row */);//处理策略过滤器。 + CommandCounterIncrement();//增加命令计数器 + send_manage_message(AUDIT_OK);//发送管理消息，表示审核操作已成功。 - if (load_policy_privileges_hook) { + //钩子函数的调用 + if (load_policy_privileges_hook) {//判断是否注册了策略权限的加载钩子函数。 load_policy_privileges_hook(false); } - if (load_policy_access_hook) { + if (load_policy_access_hook) {//判断是否注册了策略访问的加载钩子函数。 load_policy_access_hook(false); } - if (load_audit_policies_hook) { + if (load_audit_policies_hook) {//判断是否注册了策略审计的的加载钩子函数。 load_audit_policies_hook(false); } /* load filters must be last */ - if (load_policy_filter_hook) { + if (load_policy_filter_hook) {////判断是否注册了策略过滤器的的加载钩子函数。 load_policy_filter_hook(false); } } @@ -677,14 +731,14 @@ static void handle_alter_audit_node(AlterAuditPolicyStmt *stmt, gs_stl::gs_strin privileges_access_set& existing_privileges, privileges_access_set& existing_access) { /* Extract policy items from the statement node tree */ - if (stmt->policy_type != NULL) { + if (stmt->policy_type != NULL) {//不为空，则进入循环处理策略项 ListCell *policy_item = NULL; - bool is_add = strcasecmp(stmt->policy_action, "add") == 0; /* action: add or remove */ - int opt_type = get_option_type(stmt->policy_type); - foreach (policy_item, stmt->policy_items) { + bool is_add = strcasecmp(stmt->policy_action, "add") == 0;//判断操作类型是否为添加（"add"）操作 /* action: add or remove */ + int opt_type = get_option_type(stmt->policy_type);//根据策略类型获取选项类型。 + foreach (policy_item, stmt->policy_items) {//遍历策略项列表，对每个策略项进行处理。 DefElem *pol_option_item = (DefElem *) lfirst(policy_item); /* policy option & list of targets */ if (pol_option_item == NULL) { /* for optional parts of statement */ - continue; + continue;//如果策略项为空，则跳过当前循环。 } /* pol_option_item->defname; copy, ... */ bool ret = true; @@ -693,21 +747,21 @@ static void handle_alter_audit_node(AlterAuditPolicyStmt *stmt, gs_stl::gs_strin ListCell *target = NULL; if (targets && list_length(targets) > 0) { /* arguments->defname; LABEL */ - foreach (target, targets) { + foreach (target, targets) {//遍历目标列表 if (!(ret = handle_target(target, opt_type, is_add, &err_msg, &access_to_add, &access_to_remove, &privs_to_add, &privs_to_remove, &existing_labels, &cur_policy, pol_option_item->defname))) { break; } } - } else { /* all objects */ + } else { /* all objects *///目标列表为空或不存在 switch (opt_type) { case POLICY_OPT_ACCESS: handle_add_remove_all_types(opt_type, &access_to_add, &access_to_remove, &existing_access, is_add, - cur_policy.m_id, pol_option_item->defname); + cur_policy.m_id, pol_option_item->defname);//对访问权限进行添加或移除操作。 break; case POLICY_OPT_PRIVILEGES: handle_add_remove_all_types(opt_type, &privs_to_add, &privs_to_remove, &existing_privileges, is_add, - cur_policy.m_id, pol_option_item->defname); + cur_policy.m_id, pol_option_item->defname);//对访问权限进行添加或移除操作。 break; default: break; @@ -715,7 +769,7 @@ static void handle_alter_audit_node(AlterAuditPolicyStmt *stmt, gs_stl::gs_strin } /* validations, If anything is added/removed to label, label must exist unless it's for ALL */ - if (!ret) { + if (!ret) {//验证失败，则生成错误并返回。 /* generate an error */ ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("%s", err_msg.c_str()))); return; @@ -729,39 +783,40 @@ static void update_audit_policy_actions(privileges_access_set& privs_to_add, pri privileges_access_set& existing_privileges, privileges_access_set& existing_access, gs_stl::gs_string& err_msg) { - if ((privs_to_add.size() > 0) || (privs_to_remove.size() > 0)) { - Relation priv_relation = heap_open(GsAuditingPolicyPrivilegesRelationId, RowExclusiveLock); + if ((privs_to_add.size() > 0) || (privs_to_remove.size() > 0)) {//判断待添加或待移除的特权是否存在 + Relation priv_relation = heap_open(GsAuditingPolicyPrivilegesRelationId, RowExclusiveLock);//打开审核策略特权关系表 if (priv_relation != NULL) { /* add privileges */ if (privs_to_add.size() > 0) { - add_labels_to_privileges_access(false, &privs_to_add, &cur_policy, priv_relation); + add_labels_to_privileges_access(false, &privs_to_add, &cur_policy, priv_relation);//将特权添加到当前策略项中 } else if (!remove_labels_from_privileges_access(false, &privs_to_remove, &existing_privileges, priv_relation, - &err_msg)) { /* remove privileges */ + &err_msg)) { /* remove privileges *///删除权限 heap_close(priv_relation, RowExclusiveLock); /* generate an error */ - ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("%s", (err_msg.size() > 0) ? err_msg.c_str() : "No matching privilege to delete found"))); + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),//移除特权操作返回错误 + errmsg("%s", (err_msg.size() > 0) ? err_msg.c_str() : "No matching privilege to delete found")));// 生成错误并终止 return; } - heap_close(priv_relation, RowExclusiveLock); + heap_close(priv_relation, RowExclusiveLock);//关闭特权关系表。 + } } - if ((access_to_add.size() > 0) || (access_to_remove.size() > 0)) { - Relation acc_relation = heap_open(GsAuditingPolicyAccessRelationId, RowExclusiveLock); + if ((access_to_add.size() > 0) || (access_to_remove.size() > 0)) {//判断待添加或待移除的访问权限是否存在 + Relation acc_relation = heap_open(GsAuditingPolicyAccessRelationId, RowExclusiveLock);//打开审核策略访问权限关系表 if (acc_relation != NULL) { /* add access */ if (access_to_add.size() > 0) { - add_labels_to_privileges_access(true, &access_to_add, &cur_policy, acc_relation); + add_labels_to_privileges_access(true, &access_to_add, &cur_policy, acc_relation);//将访问权限添加到当前策略项中 } else if (!remove_labels_from_privileges_access(true, &access_to_remove, &existing_access, acc_relation, - &err_msg)) { /* remove access */ + &err_msg)) { /* remove access *///移除访问权限 heap_close(acc_relation, RowExclusiveLock); /* generate an error */ - ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("%s", (err_msg.size() > 0) ? err_msg.c_str() : "No matching access to delete found"))); + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),//移除访问权限操作返回错误 + errmsg("%s", (err_msg.size() > 0) ? err_msg.c_str() : "No matching access to delete found")));//生成错误并终止。 return; } - heap_close(acc_relation, RowExclusiveLock); + heap_close(acc_relation, RowExclusiveLock);//关闭访问权限关系表。 } } } @@ -773,11 +828,12 @@ static void update_audit_policy_actions(privileges_access_set& privs_to_add, pri void alter_audit_policy(AlterAuditPolicyStmt *stmt) { /* check that if has access to config audit policy */ - if (!is_policy_enabled()) { + // 检查是否有配置审计策略的权限 + if (!is_policy_enabled()) {//没有权限则返回错误 ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); return; } - + //获取当前用户的用户名和会话IP，并保存相关信息。 char buff[512] = {0}; char user_name[USERNAME_LEN] = {0}; char session_ip[MAX_IP_LEN] = {0}; @@ -790,17 +846,17 @@ void alter_audit_policy(AlterAuditPolicyStmt *stmt) user_name, u_sess->attr.attr_common.application_name, session_ip, stmt->policy_name, stmt->policy_action); securec_check_ss(rc, "", ""); save_manage_message(buff); - const char *policy_name = stmt->policy_name; - policies_set existing_policies; - policy_labels_map existing_labels; + const char *policy_name = stmt->policy_name;//指向审计策略名称的指针 + policies_set existing_policies;//存储已存在的策略的数据结构。 + policy_labels_map existing_labels;//存储已存在的标签的数据结构。 - privileges_access_set existing_privileges; - privileges_access_set privs_to_remove; - privileges_access_set privs_to_add; + privileges_access_set existing_privileges;//存储已存在的权限的数据结构。 + privileges_access_set privs_to_remove;//存储待删除权限的数据结构。 + privileges_access_set privs_to_add;//存储待添加权限的数据结构。 - privileges_access_set existing_access; - privileges_access_set access_to_remove; - privileges_access_set access_to_add; + privileges_access_set existing_access;//存储已存在的访问记录的数据结构。 + privileges_access_set access_to_remove;//存储待删除访问记录的数据结构。 + privileges_access_set access_to_add;//存储待添加访问记录的数据结构。 Relation policy_relation = NULL; Relation labels_relation = NULL; @@ -810,6 +866,7 @@ void alter_audit_policy(AlterAuditPolicyStmt *stmt) load_existing_policies(policy_relation, &existing_policies); /* first check whether such policy exists */ + //检查是否已存在同名的策略，如果存在则返回错误，如果不存在则在关系中插入新的策略元组。 GsPolicyStruct cur_policy; cur_policy.m_name = policy_name; policies_set::iterator it = existing_policies.find(cur_policy); @@ -817,13 +874,13 @@ void alter_audit_policy(AlterAuditPolicyStmt *stmt) heap_close(policy_relation, RowExclusiveLock); if (stmt->missing_ok) { /* IF EXISTS is specified, generate a notice */ send_manage_message(AUDIT_OK); - ereport(NOTICE, (errmsg("%s policy not found, alter skipping", policy_name))); + ereport(NOTICE, (errmsg("%s policy not found, alter skipping", policy_name)));//生成通知 } else { /* generate an error */ send_manage_message(AUDIT_FAILED); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("%s no such policy found, alter failed", policy_name))); + errmsg("%s no such policy found, alter failed", policy_name)));//返回错误 } return; } @@ -831,68 +888,68 @@ void alter_audit_policy(AlterAuditPolicyStmt *stmt) cur_policy.m_enabled = it->m_enabled; /* Update policy if needed */ - bool policy_status_changed = false; - if (stmt->policy_enabled != NULL) { + bool policy_status_changed = false;//标记审计策略的状态是否发生了改变 + if (stmt->policy_enabled != NULL) {//审计策略的状态 DefElem *defel = (DefElem *) stmt->policy_enabled; if (strcasecmp(defel->defname, "status") == 0) { bool policy_new_status = (strcasecmp(strVal(defel->arg), "enable") == 0) ? true : false; - if (it->m_enabled != policy_new_status) { + if (it->m_enabled != policy_new_status) {//状态发生了改变,更新当前策略对象状态 policy_status_changed = true; cur_policy.m_enabled = policy_new_status; } } } - if (((stmt->policy_comments != NULL) && (strlen(stmt->policy_comments) > 0)) || policy_status_changed) { + if (((stmt->policy_comments != NULL) && (strlen(stmt->policy_comments) > 0)) || policy_status_changed) {//需要更新策略的注释或状态。 cur_policy.m_comments = stmt->policy_comments; gs_stl::gs_string err_msg; - if (!update_policy(&cur_policy, policy_relation, policy_status_changed, &err_msg)) { - heap_close(policy_relation, RowExclusiveLock); + if (!update_policy(&cur_policy, policy_relation, policy_status_changed, &err_msg)) {//更新策略失败 + heap_close(policy_relation, RowExclusiveLock);//关闭策略关系对象 /* generate an error */ - send_manage_message(AUDIT_FAILED); - ereport(ERROR, + send_manage_message(AUDIT_FAILED);//发送一个审计管理消息 + ereport(ERROR,// (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("%s", err_msg.c_str()))); + errmsg("%s", err_msg.c_str())));//生成一个错误，并返回。 return; } } heap_close(policy_relation, RowExclusiveLock); - labels_relation = heap_open(GsPolicyLabelRelationId, RowExclusiveLock); - load_existing_labels(labels_relation, &existing_labels); - heap_close(labels_relation, RowExclusiveLock); + labels_relation = heap_open(GsPolicyLabelRelationId, RowExclusiveLock); //存储策略标签的关系 + load_existing_labels(labels_relation, &existing_labels);//加载已存在的策略标签 + heap_close(labels_relation, RowExclusiveLock);//关闭指定的关系对象 /* Get current timestamp */ - (void)DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp()); - load_existing_privileges(&existing_privileges, cur_policy.m_id); - load_existing_access(&existing_access, cur_policy.m_id); + (void)DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp());//获取当前时间戳 + load_existing_privileges(&existing_privileges, cur_policy.m_id);//加载已存在的权限 + load_existing_access(&existing_access, cur_policy.m_id);//加载已存在的访问记录 /* Extract policy items from the statement node tree */ - gs_stl::gs_string err_msg; + gs_stl::gs_string err_msg;//保存错误消息。 handle_alter_audit_node(stmt, err_msg, cur_policy, access_to_add, access_to_remove, privs_to_add, privs_to_remove, - existing_labels, existing_privileges, existing_access); + existing_labels, existing_privileges, existing_access);//处理修改审计策略的节点，并传入相关的参数。 update_audit_policy_actions(privs_to_add, privs_to_remove, access_to_add, access_to_remove, cur_policy, - existing_privileges, existing_access, err_msg); - handle_alter_add_update_filter(stmt->policy_filters, cur_policy.m_id, false /* update filter */); + existing_privileges, existing_access, err_msg);//用于更新审计策略的权限和访问记录，并传入相关的参数。 + handle_alter_add_update_filter(stmt->policy_filters, cur_policy.m_id, false /* update filter */);//处理添加或更新策略的过滤条件，并传入相关的参数 - if (stmt->policy_action && !strcmp(stmt->policy_action, "drop_filter")) { + if (stmt->policy_action && !strcmp(stmt->policy_action, "drop_filter")) {//删除相关的过滤条件。 drop_policy_reference(GsAuditingPolicyFiltersRelationId, cur_policy.m_id); } - CommandCounterIncrement(); - send_manage_message(AUDIT_OK); + CommandCounterIncrement();//增加命令计数器 + send_manage_message(AUDIT_OK);//发送管理消息，表示审核操作已成功。 - if (load_policy_access_hook) { + if (load_policy_access_hook) {//判断是否注册了策略权限的加载钩子函数。 load_policy_access_hook(false); } - if (load_policy_privileges_hook) { + if (load_policy_privileges_hook) {//判断是否注册了策略访问的加载钩子函数。 load_policy_privileges_hook(false); } - if (load_audit_policies_hook) { + if (load_audit_policies_hook) {//判断是否注册了策略审计的的加载钩子函数。 load_audit_policies_hook(false); } /* load filters must be last */ - if (load_policy_filter_hook) { + if (load_policy_filter_hook) {//判断是否注册了策略过滤器的的加载钩子函数。 load_policy_filter_hook(false); } } @@ -904,7 +961,7 @@ void alter_audit_policy(AlterAuditPolicyStmt *stmt) void drop_audit_policy(DropAuditPolicyStmt *stmt) { /* check that if has access to config audit policy */ - if (!is_policy_enabled()) { + if (!is_policy_enabled()) {//检查是否有权限访问配置审计策略，如果没有权限，则生成一个错误消息并返回。 ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); @@ -912,6 +969,8 @@ void drop_audit_policy(DropAuditPolicyStmt *stmt) } /* save Mng logs */ + //保存管理日志信息 + //据传入的删除策略语句，获取当前用户信息、应用程序名称和会话IP，并将相关信息保存到管理日志中。 ListCell* policy_obj = NULL; foreach (policy_obj, stmt->policy_names) { const char* polname = (const char *)(((Value*)lfirst(policy_obj))->val.str); @@ -932,12 +991,12 @@ void drop_audit_policy(DropAuditPolicyStmt *stmt) foreach (policy_obj, stmt->policy_names) { const char* polname = (const char *)(((Value*)lfirst(policy_obj))->val.str); gs_stl::gs_set ids; - drop_policy_by_name(GsAuditingPolicyRelationId, polname, &ids); + drop_policy_by_name(GsAuditingPolicyRelationId, polname, &ids);//通过策略名称查询策略信息，并得到关联的对象ID。 if (ids.empty()) { - if (stmt->missing_ok) { + if (stmt->missing_ok) {//输出一个通知消息并跳过后续操作。 ereport(NOTICE, (errmsg("%s policy does not exist, drop skipping", polname))); continue; - } else { + } else {//发送一个管理消息，并生成一个错误消息，然后返回 send_manage_message(AUDIT_FAILED); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), @@ -945,7 +1004,7 @@ void drop_audit_policy(DropAuditPolicyStmt *stmt) return; } } - for (long long _id : ids) { + for (long long _id : ids) {//遍历对象ID集合，依次删除以下三项相关的目录信息 /* drop gs_auditing_policy_access catalog information */ drop_policy_reference(GsAuditingPolicyAccessRelationId, _id); /* drop gs_auditing_policy_privilege catalog information */ @@ -955,21 +1014,20 @@ void drop_audit_policy(DropAuditPolicyStmt *stmt) } } - CommandCounterIncrement(); - send_manage_message(AUDIT_OK); + CommandCounterIncrement(); // 增加命令计数器 + send_manage_message(AUDIT_OK);//发送管理消息，表示审核操作已成功。 - if (load_policy_access_hook) { + if (load_policy_access_hook) {//判断是否注册了策略权限的加载钩子函数。 load_policy_access_hook(false); } - if (load_policy_privileges_hook) { + if (load_policy_privileges_hook) {//判断是否注册了策略访问的加载钩子函数。 load_policy_privileges_hook(false); } - if (load_audit_policies_hook) { + if (load_audit_policies_hook) {//判断是否注册了策略审计的的加载钩子函数。 load_audit_policies_hook(false); } /* load filters must be last */ - if (load_policy_filter_hook) { + if (load_policy_filter_hook) {//判断是否注册了策略过滤器的的加载钩子函数。 load_policy_filter_hook(false); } } - -- 2.34.1 From 2c29b99fa56d85cfa66b633f551cfd6447458a35 Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Sun, 27 Aug 2023 22:33:46 +0800 Subject: [PATCH 05/48] Update iprange.cpp --- src/gausskernel/security/iprange/iprange.cpp | 293 +++++++++++++------ 1 file changed, 205 insertions(+), 88 deletions(-) diff --git a/src/gausskernel/security/iprange/iprange.cpp b/src/gausskernel/security/iprange/iprange.cpp index ca7b66ef9..d904ec94e 100644 --- a/src/gausskernel/security/iprange/iprange.cpp +++ b/src/gausskernel/security/iprange/iprange.cpp @@ -22,8 +22,8 @@ * * --------------------------------------------------------------------------------------- */ -#include -#include +#include //用于网络编程的标准头文件之一。该头文件提供了与网络相关的结构体、宏定义和函数原型。 +#include //提供了一些网络编程相关的函数和数据结构的定义，包括 IP 地址转换、网络字节序和主机字节序之间的转换等操作 #include #include #include @@ -36,13 +36,23 @@ using namespace std; -#define IPRANGE_IS_IPV4(ip) ((ip).ip_32.b == 0x0000FFFF) +//判断给定的IP是否为IPv4地址 +#define IPRANGE_IS_IPV4(ip) ((ip).ip_32.b == 0x0000FFFF) //通过比较IP的低32位的值是否等于0x0000FFFF来判断 如果相等，则表示该IP是IPv4地址 +/* +* 类型声明 Mask from Look-Up Table 查找表中的掩码 +* 定义了一个名为 MASK_FROM_LUT 的 std::vector 对象 +* MASK_FROM_LUT 数组存储了从最长子网掩码（32位）到最短子网掩码（1位）的掩码值 +* +* 每个元素都是通过将0xFFFFFFFF（32个1）向左移动不同的位数来生成的 +* +* 这样的一组掩码用于对IPv4地址进行子网掩码操作，以实现网络地址和主机地址的分离 +*/ std::vector MASK_FROM_LUT { - (uint32_t)0xFFFFFFFF, - (uint32_t)0xFFFFFFFF << 31, - (uint32_t)0xFFFFFFFF << 30, - (uint32_t)0xFFFFFFFF << 29, + (uint32_t)0xFFFFFFFF, // MASK_FROM_LUT[0] + (uint32_t)0xFFFFFFFF << 31, // MASK_FROM_LUT[1] 0xFFFFFFFE + (uint32_t)0xFFFFFFFF << 30, // MASK_FROM_LUT[2] 0xFFFFFFFC + (uint32_t)0xFFFFFFFF << 29, // MASK_FROM_LUT[3] 0xFFFFFFF8 (uint32_t)0xFFFFFFFF << 28, (uint32_t)0xFFFFFFFF << 27, (uint32_t)0xFFFFFFFF << 26, @@ -70,15 +80,24 @@ std::vector MASK_FROM_LUT { (uint32_t)0xFFFFFFFF << 4, (uint32_t)0xFFFFFFFF << 3, (uint32_t)0xFFFFFFFF << 2, - (uint32_t)0xFFFFFFFF << 1, + (uint32_t)0xFFFFFFFF << 1, // MASK_FROM_LUT[31] 0x80000000 (uint32_t)0xFFFFFFFF, }; +/* + * 类型声明 Mask to Look-Up Table 掩码到查找表 + * 定义了一个名为 MASK_TO_LUT 的 std::vector 对象 + * MASK_TO_LUT 数组存储了从最短子网掩码（1位）到最长子网掩码（32位）的掩码值 + * + * 每个元素都是通过将0xFFFFFFFF（32个1）向左移动不同的位数 并按位取反 来生成的 + * + * 这样的一组掩码用于对IPv4地址进行子网掩码操作，以实现网络地址和主机地址的分离 + */ std::vector MASK_TO_LUT { - (uint32_t)0, - ~((uint32_t)0xFFFFFFFF << 31), - ~((uint32_t)0xFFFFFFFF << 30), - ~((uint32_t)0xFFFFFFFF << 29), + (uint32_t)0, // MASK_TO_LUT[0] 0x00000000 + ~((uint32_t)0xFFFFFFFF << 31), // MASK_TO_LUT[1] 0x00000001 + ~((uint32_t)0xFFFFFFFF << 30), // MASK_TO_LUT[2] 0x00000003 + ~((uint32_t)0xFFFFFFFF << 29), // MASK_TO_LUT[3] 0x00000007 ~((uint32_t)0xFFFFFFFF << 28), ~((uint32_t)0xFFFFFFFF << 27), ~((uint32_t)0xFFFFFFFF << 26), @@ -106,12 +125,17 @@ std::vector MASK_TO_LUT { ~((uint32_t)0xFFFFFFFF << 4), ~((uint32_t)0xFFFFFFFF << 3), ~((uint32_t)0xFFFFFFFF << 2), - ~((uint32_t)0xFFFFFFFF << 1), + ~((uint32_t)0xFFFFFFFF << 1), // MASK_TO_LUT[31] 0x7FFFFFFF (uint32_t)0xFFFFFFFF }; -const IPV6 localhost_ipv6 ({0x0000000000000001, 0x0000000000000000}); -const IPV6 localhost_ipv4 ({0xffff7f000001, 0x0}); +/* +* 本地主机的 IPv6 地址和 IPv4 地址（表示本地回环地址） +* IPv6 128位二进制 4*8位十六进制 +* IPv4 32位二进制 8位十六进制 +*/ +const IPV6 localhost_ipv6 ({0x0000000000000001, 0x0000000000000000});//2*16 +const IPV6 localhost_ipv4 ({0xffff7f000001, 0x0}); //0x0000ffff 是 IPv4 兼容地址的前缀 0x7f000001是IPv4本地回环地址 #define IP_MAX_LEN 128 @@ -123,77 +147,119 @@ IPRange::~IPRange() { } +/* +* IP 地址转换的函数 +* 从网络字节序（大端字节序）转换为主机字节序（主机的字节序，通常是小端字节序） +*/ +//接受一个指向 IPV6 对象的指针 ip +//和一个指向 sockaddr_in6 结构体的指针 sa void IPRange::net_ipv6_to_host_order(IPV6 *ip, const struct sockaddr_in6 *sa) const { IPV6 tmp_ip; + //使用 memcpy_s 函数将 sa->sin6_addr 的内容拷贝到 tmp_ip.ip_64 中（拷贝的字节数为 sizeof(tmp_ip.ip_64)） + // memcpy_s 是一个安全版本的内存拷贝函数，用于防止内存溢出和错误的内存操作 int rc = memcpy_s(&(tmp_ip.ip_64), sizeof(tmp_ip.ip_64), &(sa->sin6_addr), sizeof(tmp_ip.ip_64)); - securec_check(rc, "\0", "\0"); + securec_check(rc, "\0", "\0"); //检查并确保字符串操作不会导致缓冲区溢出或无效的操作 + /* + * 调用 ntohl 函数对 tmp_ip.ip_32 的每个成员进行字节序转换 + * 将它们从网络字节序转换为主机字节序 + * 并分别赋值给 ip 对应的成员 + */ ip->ip_32.a = ntohl(tmp_ip.ip_32.d); ip->ip_32.b = ntohl(tmp_ip.ip_32.c); ip->ip_32.c = ntohl(tmp_ip.ip_32.b); ip->ip_32.d = ntohl(tmp_ip.ip_32.a); + /* + * ntohl 是一个网络字节序和主机字节序之间转换的函数 + * 全称是 "network to host long" + * 在头文件 中声明，属于 POSIX 标准的一部分 + * 作用是将一个 32 位整数从网络字节序（大端字节序）转换为主机字节序（与主机平台相关的字节序，通常是小端字节序） + * 函数的原型 uint32_t ntohl(uint32_t netlong); + */ } - +// 接受一个指向 IPV6 对象的指针 ip +// 和一个指向 in_addr 结构体的指针 addr void IPRange::net_ipv4_to_host_order(IPV6 *ip, const struct in_addr *addr) const { + /* + * 调用 ntohl 函数对 addr->s_addr 进行字节序转换，并将结果赋值给 ip->ip_32.a + * 并分别赋值 b c d + */ ip->ip_32.a = ntohl(addr->s_addr); ip->ip_32.b = 0x0000FFFF; ip->ip_32.c = ip->ip_32.d = 0; } +//IP 地址字符串转换为 IP 结构体的函数 +//接受一个指向 'IP 地址字符串' 的 指针 ip_str 和 一个指向 'IPV6 结构体' 的 指针 ip bool IPRange::str_to_ip(const char* ip_str, IPV6 *ip) { struct in_addr addr; struct sockaddr_in6 sa; - if (inet_pton(AF_INET6, ip_str, &sa.sin6_addr) > 0) { - net_ipv6_to_host_order(ip, &sa); - } else if (inet_pton(AF_INET, ip_str, &addr) > 0) { + /* + * inet_pton 函数是一个网络编程中常用的函数 + * 全称是 "Internet Presentation to Network" + * 在头文件 中声明，属于 POSIX 标准的一部分 + * 作用是将 IP 地址字符串转换为网络地址结构 + * 函数的原型 int inet_pton(int af, const char *src, void *dst); + * af 表示地址族（Address Family），可以是 AF_INET（IPv4 地址）或 AF_INET6（IPv6 地址） + * src 是待转换的 IP 地址字符串，dst 是用于存储转换结果的目标缓冲区 + * + * 转换成功，返回值为 1 + * 转换失败，返回值为 0 或 -1，并设置全局变量 errno 来指示具体出错原因 + */ + if (inet_pton(AF_INET6, ip_str, &sa.sin6_addr) > 0) { //有效的 IPv6 地址 + net_ipv6_to_host_order(ip, &sa); //将该地址从网络字节序转换为主机字节序，并将结果存储在 ip 中 + } else if (inet_pton(AF_INET, ip_str, &addr) > 0) { // 有效的 IPv4 地址 net_ipv4_to_host_order(ip, &addr); - } else { + } else { //传入的字符串格式无效或超出了合法的 IP 范围 /* * Note that even the format keep the same as ipv6 or ipv4 * still recognize it as invalid ip if ip exceed the valid range */ - m_err_str = "invalid ip: " + std::string(ip_str); + m_err_str = "invalid ip: " + std::string(ip_str); //将错误信息存储在成员变量 m_err_str 中 return false; } return true; } +// IP 地址范围掩码函数 +// 用于 根据给定的 CIDR 前缀长度对 IP 范围进行掩码处理，并更新输入的起始和结束 IP 地址值，以限制范围内的 IP 地址 +// 接受一个指向 Range 结构体 的指针 和一个 表示 CIDR 前缀长度 的 无符号短整型参数 cidr bool IPRange::mask_range(Range *range, unsigned short cidr) { if (IPRANGE_IS_IPV4(range->from)) { /* ipv4 */ - if (cidr < 1 || cidr > 32) { + if (cidr < 1 || cidr > 32) { //cidr 不在有效的范围内 m_err_str = "invalid cidr for ipv4: " + cidr; return false; } - - range->from.ip_32.a &= MASK_FROM_LUT[cidr]; - range->to.ip_32.a |= MASK_TO_LUT[cidr]; + //更新输入的起始和结束 IP 地址值 + range->from.ip_32.a &= MASK_FROM_LUT[cidr]; //起始 IP 地址的相应部分 与 掩码 进行 与运算 + range->to.ip_32.a |= MASK_TO_LUT[cidr]; //结束 IP 地址的相应部分 与 掩码 进行 或运算 } else { /* ipv6 */ if (cidr < 1 || cidr > 128) { m_err_str = "invalid cidr for ipv6: " + cidr; return false; } unsigned short complement = cidr % 32; /* the result is less or equal to 31 */ - - if (cidr > 96) { + //d c b a + if (cidr > 96) { //只处理第一部分 range->from.ip_32.a &= MASK_FROM_LUT[complement]; range->to.ip_32.a |= MASK_TO_LUT[complement]; - } else if (cidr > 64) { + } else if (cidr > 64) { //处理前两部分 range->from.ip_32.b &= MASK_FROM_LUT[complement]; range->to.ip_32.b |= MASK_TO_LUT[complement]; range->from.ip_32.a = 0; range->to.ip_32.a = 0xFFFFFFFF; - } else if (cidr > 32) { + } else if (cidr > 32) { //处理前三部分 range->from.ip_32.c &= MASK_FROM_LUT[complement]; range->to.ip_32.c |= MASK_TO_LUT[complement]; range->from.ip_32.b = 0; range->to.ip_32.b = 0xFFFFFFFF; range->from.ip_32.a = 0; range->to.ip_32.a = 0xFFFFFFFF; - } else { + } else { //处理所有四部分 range->from.ip_32.d &= MASK_FROM_LUT[complement]; range->to.ip_32.d |= MASK_TO_LUT[complement]; range->from.ip_32.c = 0; @@ -211,23 +277,33 @@ bool IPRange::mask_range(Range *range, unsigned short cidr) * parse the ip with mask into range sturst , format is as below: * x.x.x.x|x, ptr is the postion of "|" */ +/* + * 解析带有掩码的 IP 地址范围的函数 + * 根据传入的字符串范围和分隔符 "|" 的位置将其拆分为源 IP 和掩码 IP + * 并将结果存储在 Range 结构体的对象 new_range 中 + * + * const char *ptr 指针传入的是 "|" 的位置 + * range 参数被用作表示 IP 地址范围的字符串的起始位置 + * range_len 表示 x.x.x.x|x 长度 + */ bool IPRange::parse_mask(const char* range, size_t range_len, const char *ptr, Range *new_range) { - if (range_len > 100) { + if (range_len > 100) { //检查范围字符串的长度是否有效（小于等于100） m_err_str = "the range string length is not valid: " + range_len; return false; } char first_ip_str[IP_MAX_LEN] = {0}; char mask_ip_str[IP_MAX_LEN] = {0}; - size_t first_ip_str_len = ptr - range; - size_t mask_ip_str_len = range_len - 1 - first_ip_str_len; + size_t first_ip_str_len = ptr - range; //源 IP 字符串长度 + size_t mask_ip_str_len = range_len - 1 - first_ip_str_len; //掩码 IP 字符串长度 /* copy the first ip */ copy_without_spaces(first_ip_str, sizeof(first_ip_str), range, first_ip_str_len); /* get the other ip */ copy_without_spaces(mask_ip_str, sizeof(mask_ip_str), ptr + 1, mask_ip_str_len); IPV6 ip; IPV6 mask_ip; + //调用str_to_ip函数，将字符串表示的 IP 地址转换为 结构体 if (!str_to_ip(first_ip_str, &ip)) { m_err_str = "failed to convert ip: " + std::string(first_ip_str); return false; @@ -236,6 +312,7 @@ bool IPRange::parse_mask(const char* range, size_t range_len, const char *ptr, R m_err_str = "failed to convert mask ip: " + std::string(mask_ip_str); return false; } + // 0x0000ffff 是 IPv4 兼容地址的前缀 if (mask_ip.ip_32.b == 0x0000FFFF) { /* ipv4 */ new_range->from.ip_32.a = ip.ip_32.a & mask_ip.ip_32.a; new_range->from.ip_32.b = 0x0000FFFF; @@ -243,19 +320,21 @@ bool IPRange::parse_mask(const char* range, size_t range_len, const char *ptr, R new_range->to.ip_32.a = ip.ip_32.a | ~mask_ip.ip_32.a; new_range->to.ip_32.b = 0x0000FFFF; new_range->to.ip_64.upper = 0; - } else { + } else {/* ipv6 */ new_range->from = ip & mask_ip; new_range->to = ip | ~mask_ip; } return true; } +//解析单个IP范围，并将结果存储在Range结构体中 bool IPRange::parse_single(const char* range, size_t range_len, Range *new_range) { if (range_len > 100) { m_err_str = "the range string length is not valid: " + range_len; return false; } + //buf数组，用于存储去除空格后的IP char buf[IP_MAX_LEN] = {0}; /* copy the ip part to buf */ copy_without_spaces(buf, sizeof(buf), range, range_len); @@ -263,10 +342,12 @@ bool IPRange::parse_single(const char* range, size_t range_len, Range *new_range m_err_str = "failed to convert ip: " + std::string(buf); return false; } - new_range->to = new_range->from; + new_range->to = new_range->from; //IP范围只包含单个IP return true; } +//解析带有斜杠表示的CIDR形式的IP范围，并将结果存储在Range结构体中 +//ptr是指向IP范围字符串中斜杠符号("/")后的数字部分的指针 bool IPRange::parse_slash(const char* range, size_t range_len, const char *ptr, Range *new_range) { if (range_len > 100) { @@ -284,10 +365,12 @@ bool IPRange::parse_slash(const char* range, size_t range_len, const char *ptr, return false; } new_range->to = new_range->from; - (void)mask_range(new_range, cidr); + (void)mask_range(new_range, cidr); //调用mask_range函数，根据CIDR值对IP范围进行掩码操作，将范围限制在指定的子网中 return true; } +//解析形如 "x.x.x.x-y.y.y.y" 格式的 IP 范围字符串 +//ptr：指向 IP 范围字符串中短横线 - 的位置 bool IPRange::parse_hyphen(const char* range, size_t range_len, const char *ptr, Range *new_range) { if (range_len > 100) { @@ -304,6 +387,7 @@ bool IPRange::parse_hyphen(const char* range, size_t range_len, const char *ptr, copy_without_spaces(first_ip, sizeof(first_ip), range, first_ip_len); /* get the other ip */ copy_without_spaces(second_ip, sizeof(second_ip), ptr + 1, second_ip_len); + //str_to_ip IP 地址字符串转换为 IP 结构体 if (!str_to_ip(first_ip, &(new_range->from))) { m_err_str = "failed to parse ip: " + std::string(first_ip); return false; @@ -312,7 +396,7 @@ bool IPRange::parse_hyphen(const char* range, size_t range_len, const char *ptr, m_err_str = "failed to parse ip: " + std::string(second_ip); return false; } - if (new_range->from > new_range->to) { + if (new_range->from > new_range->to) { //第一个 IP 大于第二个 IP m_err_str = "the first ip (" + std::string(first_ip) + ") is bigger than the other (" + std::string(second_ip) + ")"; return false; @@ -320,16 +404,25 @@ bool IPRange::parse_hyphen(const char* range, size_t range_len, const char *ptr, return true; } +/* +* IP 范围处理函数的实现，用于处理 IP 范围之间的交集 +* +* new_ranges：存储处理后的 IP 范围的容器指针 +* remove_range：要移除的 IP 范围 +* exist_range：已存在的 IP 范围 +*/ void IPRange::handle_remove_intersection(Ranges_t *new_ranges, const Range *remove_range, Range *exist_range) { + // 求 remove_range 和 exist_range 的交集 IPV6 range_min = std::max(remove_range->from, exist_range->from); IPV6 range_max = std::min(remove_range->to, exist_range->to); - if (range_min > range_max) { + if (range_min > range_max) { //没有交集 /* no intersaction */ - new_ranges->push_back(*exist_range); + new_ranges->push_back(*exist_range); //向容器尾部添加新元素的成员函数 return; } /* there is an intersaction the remove_range includes the exist_range */ + //移除范围 remove_range 完全包含了已存在范围 exist_range if ((remove_range->from <= exist_range->from) && (remove_range->to >= exist_range->to)) { /* remove the exist range */ return; @@ -338,29 +431,32 @@ void IPRange::handle_remove_intersection(Ranges_t *new_ranges, const Range *remo // exist_range: 2 - 5 // remove_range: 1 - 3 // expected result: 4 - 5 - if (remove_range->from <= exist_range->from) { - exist_range->from = remove_range->to + 1; + if (remove_range->from <= exist_range->from) { // 移除范围的起始位置 小于或等于 已存在范围的起始位置 + exist_range->from = remove_range->to + 1; //更新 exist_range 的起始位置为 remove_range 的结束位置加1 new_ranges->push_back(*exist_range); return; } - if (remove_range->to >= exist_range->to) { - exist_range->to = remove_range->from - 1; + if (remove_range->to >= exist_range->to) { //移除范围的结束位置 大于或等于 已存在范围的结束位置 + exist_range->to = remove_range->from - 1; //更新 exist_range 的结束位置为 remove_range 的起始位置减1 new_ranges->push_back(*exist_range); return; } /* the remove range is inside the exist one */ + //移除范围 remove_range 完全位于已存在范围 exist_range 内部 new_ranges->push_back({exist_range->from, remove_range->from - 1}); new_ranges->push_back({remove_range->to + 1, exist_range->to}); } +// 处理 IP 范围之间的交集 并 添加新的范围 bool IPRange::handle_add_intersection(Range *new_range, const Range *exist_range) { + // 求 new_range 和 exist_range 的交集 IPV6 range_min = std::max(new_range->from, exist_range->from); IPV6 range_max = std::min(new_range->to, exist_range->to); - if (range_min > range_max) { + if (range_min > range_max) { //没有交集 return false; } - + //更新 new_range 的范围 new_range->from = std::min(new_range->from, exist_range->from); new_range->to = std::max(new_range->to, exist_range->to); return true; @@ -373,6 +469,7 @@ bool IPRange::handle_add_intersection(Range *new_range, const Range *exist_range * cidr ip : 192.168.10.1/21 * mask ip : 192.168.10.1/255.255.255.2 */ +// 解析 IP 范围的不同格式，并将其转换为 Range 结构 bool IPRange::parse_range(const char *range, size_t range_len, Range *new_range) { /* handle format of "ip/cidr" */ @@ -412,9 +509,11 @@ bool IPRange::parse_range(const char *range, size_t range_len, Range *new_range) return false; } +//向 IP 范围集合中添加多个 IP 范围 +//函数接受一个 std::unordered_set 类型的参数 ranges，其中包含要添加的 IP 范围字符串集合 bool IPRange::add_ranges(const std::unordered_set ranges) { - for (const std::string range : ranges) { + for (const std::string range : ranges) { //使用 for 循环遍历 ranges 集合中的每个 IP 范围字符串 if (!add_range(range.c_str(), range.length())) { return false; } @@ -422,6 +521,7 @@ bool IPRange::add_ranges(const std::unordered_set ranges) return true; } +//从 IP 范围集合中移除多个 IP 范围 bool IPRange::remove_ranges(const std::unordered_set ranges) { for (const std::string range : ranges) { @@ -432,46 +532,49 @@ bool IPRange::remove_ranges(const std::unordered_set ranges) return true; } +//向 IP 范围列表中添加新的范围 bool IPRange::add_range(Range *new_range) { /* adding the new range */ - if (m_ranges.size() == 0 || new_range->to < m_ranges[0].from) { - (void)m_ranges.insert(m_ranges.begin(), *new_range); + if (m_ranges.size() == 0 || new_range->to < m_ranges[0].from) { //列表为空或者新范围的结束值小于列表中第一个范围的开始值 + (void)m_ranges.insert(m_ranges.begin(), *new_range); //将新范围插入到列表的开头 return true; - } else if (new_range->from > m_ranges.back().to) { - m_ranges.push_back(*new_range); + } else if (new_range->from > m_ranges.back().to) { //新范围的开始值大于列表中最后一个范围的结束值 + m_ranges.push_back(*new_range); //将新范围追加到列表的末尾 return true; } Ranges_t new_ranges; bool we_had_intersection = false; uint32_t i = 0; - /* interate over the ranges and check for intersection or the place to add the new range */ + /* interate over the ranges and check for intersection or the place to add the new range + 在范围上进行交互，并检查是否有交集或添加新范围的地方 */ while (i < m_ranges.size()) { - /* in case of intersaction update the new range */ + /* in case of intersaction update the new range 在交互的情况下，更新新的范围 */ if (handle_add_intersection(new_range, &m_ranges[i])) { we_had_intersection = true; ++i; continue; } - /* just insert the intersaction */ + /* just insert the intersaction 只需插入交互 */ if (we_had_intersection) { we_had_intersection = false; new_ranges.push_back(*new_range); break; } else if (new_range->to < m_ranges[i].from) { - /* we got the plcae to put the new range */ + /* we got the plcae to put the new range 我们找到了放新靶场的地方 */ new_ranges.push_back(*new_range); break; } - /* just add the old range */ + /* just add the old range 只要加上旧的范围 */ new_ranges.push_back(m_ranges[i]); ++i; } - /* if the intersection was until the end of the list - add it now */ + /* if the intersection was until the end of the list - add it now + 如果交集是直到列表的末尾-现在添加它 */ if (we_had_intersection) { new_ranges.push_back(*new_range); } - /* copy the rest of the list if exist */ + /* copy the rest of the list if exist 如果存在，复制列表的其余部分 */ while (i < m_ranges.size()) { new_ranges.push_back(m_ranges[i]); ++i; @@ -480,6 +583,7 @@ bool IPRange::add_range(Range *new_range) return true; } +//检查给定的 IP 范围字符串是否有效 bool IPRange::is_range_valid(const std::string range) { IPRange tmp; @@ -487,39 +591,44 @@ bool IPRange::is_range_valid(const std::string range) return tmp.parse_range(range.c_str(), range.size(), &new_range); } +//用于向 IP 范围集合中添加单个 IP 范围 bool IPRange::add_range(const char* range, size_t range_len) { Range new_range; - m_err_str.clear(); + m_err_str.clear(); //清空错误信息 if (!parse_range(range, range_len, &new_range)) { return false; } return add_range(&new_range); } +//从 IP 范围集合中移除单个 IP 范围 bool IPRange::remove_range(const char *range, size_t range_len) { Ranges_t new_ranges; Range remove_range; m_err_str.clear(); - if (!parse_range(range, range_len, &remove_range)) { + if (!parse_range(range, range_len, &remove_range)) { //解析传入的范围字符串 return false; } - for (Range exist_range : m_ranges) { + for (Range exist_range : m_ranges) { //通过遍历存储在 m_ranges 中的每个范围对象 exist_range + //移除指定范围和当前范围的交集 handle_remove_intersection(&new_ranges, &remove_range, &exist_range); } - m_ranges.swap(new_ranges); + m_ranges.swap(new_ranges); // 新的范围集合 new_ranges 替换原有的范围集合 m_ranges return true; } +// 将给定的IPv6地址转换为字符串表示形式 std::string IPRange::ip_to_str(const IPV6 *ip) const { - char ip_str[INET6_ADDRSTRLEN]; + char ip_str[INET6_ADDRSTRLEN]; //字符数组 ip_str，用于存储转换后的IP地址字符串 /* now get it back and print it */ - if (IPRANGE_IS_IPV4(*ip)) { - uint32_t tmp = htonl(ip->ip_32.a); + if (IPRANGE_IS_IPV4(*ip)) { //ipv4 + uint32_t tmp = htonl(ip->ip_32.a); //将一个 32 位无符号整数 从 主机字节顺序 转换为 网络字节顺序 + // 将网络字节顺序表示的 IP 地址转换为字符串形式的 IP 地址 (void)inet_ntop(AF_INET, &tmp, ip_str, INET_ADDRSTRLEN); - } else { + } else { // ipv6 IPV6 tmp_ip = *ip; tmp_ip.ip_32.a = htonl(ip->ip_32.d); tmp_ip.ip_32.b = htonl(ip->ip_32.c); @@ -530,6 +639,7 @@ std::string IPRange::ip_to_str(const IPV6 *ip) const return std::string(ip_str); } +//在 IP 范围列表中进行二分查找，判断给定的 IP 是否在某个范围内 bool IPRange::binary_search(const IPV6 ip) const { /* do a binary search */ @@ -538,76 +648,84 @@ bool IPRange::binary_search(const IPV6 ip) const size_t mid = 0; while (i != j) { mid = (i + j) / 2; - if (ip >= m_ranges[mid].from && ip <= m_ranges[mid].to) { + if (ip >= m_ranges[mid].from && ip <= m_ranges[mid].to) { //给定的 IP 在中间范围 m_ranges[mid] 内 return true; } - if (ip < m_ranges[mid].from) { + if (ip < m_ranges[mid].from) { // IP在后半部分 j = (mid > 0) ? (mid - 1) : mid; } else { i = mid + 1; } } + // 判断给定的 IP 是否在最后剩余的范围内（即 m_ranges[i]） return (ip >= m_ranges[i].from && ip <= m_ranges[i].to); } +//判断当前 IP 范围列表与另一个 IP 范围对象所表示的范围是否存在交集 bool IPRange::is_intersect(const IPRange *arg) { - for (size_t i = 0; i < m_ranges.size(); ++i) { + for (size_t i = 0; i < m_ranges.size(); ++i) { // 遍历当前 IP 范围列表中的每个范围 Range tmp(m_ranges[i].from, m_ranges[i].to); - for (size_t j = 0 ; j < arg->m_ranges.size(); ++j) { - if (handle_add_intersection(&tmp, &arg->m_ranges[j])) { - return true; + for (size_t j = 0 ; j < arg->m_ranges.size(); ++j) { // 遍历传入的另一个 IP 范围对象 arg 中的每个范围 + if (handle_add_intersection(&tmp, &arg->m_ranges[j])) { // 调用 handle_add_intersection 函数 + return true; //判断两个范围是否有交集 } } } return false; } +/* 三个重载的 IPRange::is_in_range 函数，用于判断给定的 IP 是否在 IP 范围内 */ +//字符串形式表示的 IP 地址 ip_str 作为参数 bool IPRange::is_in_range(const char *ip_str) { IPV6 ip; - if (!str_to_ip(ip_str, &ip)) { + if (!str_to_ip(ip_str, &ip)) { //将 ip_str 转换为 IPV6 结构体类型的 IP 地址 ip return false; } - return is_in_range(&ip); + return is_in_range(&ip); //调用第二个重载函数 is_in_range(&ip) 进行判断，并返回结果 } - +//指向 IPV6 结构体类型的 IP 地址 ip 的指针作为参数 bool IPRange::is_in_range(const IPV6 *ip) { - if (m_ranges.size() == 0) { + if (m_ranges.size() == 0) { //判范围为空 m_err_str = "there are no ranges in this object"; return false; } - m_err_str.clear(); - if (*ip == localhost_ipv4 || *ip == localhost_ipv6) { + m_err_str.clear(); //清空错误字符串 m_err_str + if (*ip == localhost_ipv4 || *ip == localhost_ipv6) { // 等于本地主机 IPv4 地址或本地主机 IPv6 地址 + //调用 binary_search 函数分别在 IP 范围列表中查找对应的范围 return binary_search(localhost_ipv4) || binary_search(localhost_ipv6); } return binary_search(*ip); } - +//uint32_t 类型的 IPv4 地址 ipv4 作为参数 bool IPRange::is_in_range(const uint32_t ipv4) { IPV6 ip; + //将 IPv4 地址转换为 IPV6 结构体类型的 IP 地址 ip net_ipv4_to_host_order(&ip, (struct in_addr*)&ipv4); - return is_in_range(&ip); + return is_in_range(&ip); // 调用第二个重载函数 is_in_range(&ip) 进行判断，并返回结果 } +// 获取当前范围集合的字符串表示形式 std::unordered_set IPRange::get_ranges_set() { - std::unordered_set rslt; + std::unordered_set rslt; //空的无序集合 rslt，用于存储结果 + //循环遍历 m_ranges 中的每个范围 range for (Range range : m_ranges) { - if (ip_to_str(&range.from).compare(ip_to_str(&range.to)) == 0) { - rslt.insert(ip_to_str(&range.from)); + if (ip_to_str(&range.from).compare(ip_to_str(&range.to)) == 0) { //IP 范围只有一个 IP + rslt.insert(ip_to_str(&range.from));//将该 IP 地址作为字符串添加到结果集合中 } else { - rslt.insert(ip_to_str(&range.from) + "-" + ip_to_str(&range.to)); + rslt.insert(ip_to_str(&range.from) + "-" + ip_to_str(&range.to));//起始 IP 地址 - 结束 IP 地址 的形式作为字符串添加到结果集合中 } } return rslt; } - +// 将输入字符串中的空格字符去除，并将结果存储在目标缓冲区中 void IPRange::copy_without_spaces(char buf[], size_t buf_len, const char *original, size_t original_len) const { - if (original_len == 0 || original_len > buf_len) { + if (original_len == 0 || original_len > buf_len) { //输入字符串的长度为0 或 超过目标缓冲区的长度 return; } char *p = buf; @@ -618,9 +736,8 @@ void IPRange::copy_without_spaces(char buf[], size_t buf_len, const char *origin } *p = '\0'; } - +// 检查当前范围集合是否为空 bool IPRange::empty() const { return m_ranges.empty(); } - -- 2.34.1 From 6c15a750f0f1568bb8d389bf3ba3656e490d9006 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Sun, 27 Aug 2023 22:33:58 +0800 Subject: [PATCH 06/48] Update ledger_archive.cpp --- .../security/gs_ledger/ledger_archive.cpp | 441 +++++++++++------- 1 file changed, 284 insertions(+), 157 deletions(-) diff --git a/src/gausskernel/security/gs_ledger/ledger_archive.cpp b/src/gausskernel/security/gs_ledger/ledger_archive.cpp index 62647711b..3762a62b1 100644 --- a/src/gausskernel/security/gs_ledger/ledger_archive.cpp +++ b/src/gausskernel/security/gs_ledger/ledger_archive.cpp @@ -51,31 +51,42 @@ #endif /* - * prepare_histback_dir -- create hist_back dir under pg_audit. + * prepare_histback_dir -- create hist_back dir under pg_audit.//在 pg_audit 目录下创建 hist_back 文件夹 */ +//准备历史备份目录的函数 static void prepare_histback_dir(void) { + // 使用安全的方式构建历史备份目录路径 char ledger_histback_dir[MAXPGPATH] = {0}; int rc = snprintf_s(ledger_histback_dir, MAXPGPATH, MAXPGPATH - 1, "%s/hist_bak", g_instance.attr.attr_security.Audit_directory); - securec_check_ss(rc, "\0", "\0"); + //使用snprintf_s函数，确保路径字符串不会超过MAXPGPATH的长度限制 + securec_check_ss(rc, "\0", "\0");//检查snprintf_s是否成功 /* * Create histback directory if not present; ignore errors + * 创建历史备份目录，如果目录不存在则创建；忽略错误 */ + + //创建主目录，用于存放历史备份目录 (void)pg_mkdir_p(g_instance.attr.attr_security.Audit_directory, S_IRWXU); + //创建历史备份目录 (void)pg_mkdir_p(ledger_histback_dir, S_IRWXU); } /* - * ledger_copytable -- copy rows of hist table. + * ledger_copytable -- copy rows of hist table.//复制历史表中的行数据 * - * cstate: copy information state + * cstate: copy information state//复制信息状态 * * Note: this function is only used to copy user hist table or gchain. * we only copy commited data, thus for each history row, we will * recalculate prehash to ensure the consistency of front-to-back order. + * 此函数仅用于复制用户历史表或 gchain（全局链）。 + 我们只复制已提交的数据，因此对于每个历史行，我们将重新计算prehash，以确保前后顺序的一致性。 */ +//将指定表的数据复制到输出流的功能，并且在复制过程中根据特定逻辑计算哈希值。 +//用于复制数据库中表数据 static uint64 ledger_copytable(CopyState cstate) { Relation cur_rel; @@ -86,28 +97,34 @@ static uint64 ledger_copytable(CopyState cstate) uint64 processed = 0; bool is_gchain; - cur_rel = cstate->curPartionRel; - is_gchain = RelationGetRelid(cur_rel) == GsGlobalChainRelationId; - tuple_desc = RelationGetDescr(cur_rel); - attr = tuple_desc->attrs; - num_phys_attrs = tuple_desc->natts; - cstate->null_print_client = cstate->null_print; + cur_rel = cstate->curPartionRel;//获取当前分区关系 + is_gchain = RelationGetRelid(cur_rel) == GsGlobalChainRelationId;//判断是否为全局链关系 + tuple_desc = RelationGetDescr(cur_rel);//获取关系的元组描述符 + attr = tuple_desc->attrs;//获取元组描述符中的属性数组 + num_phys_attrs = tuple_desc->natts;//获取属性数量 + cstate->null_print_client = cstate->null_print;//复制null_print到null_print_client /* We use fe_msgbuf as a per-row buffer regardless of copy_dest */ - if (cstate->fe_msgbuf == NULL) { - cstate->fe_msgbuf = makeStringInfo(); - if (IS_PGXC_COORDINATOR || g_instance.role == VSINGLENODE) - ProcessFileHeader(cstate); + //在复制表数据的过程中使用一个缓冲区来存储每一行的数据，并且无论复制的目标是什么，都会使用fe_msgbuf作为每行的缓冲区 + if (cstate->fe_msgbuf == NULL) {//如果fe_msgbuf为空，则创建一个StringInfo对象，并发送文件头信息 + cstate->fe_msgbuf = makeStringInfo();//创建一个StringInfo对象 + if (IS_PGXC_COORDINATOR || g_instance.role == VSINGLENODE)//在协调器模式或单节点模式下 + ProcessFileHeader(cstate);//处理文件头信息 + //函数来源src\gausskernel\optimizer\commands\copy.cpp + //如果文件头信息是从文件中读取的，则打开文件并读取相应的内容，并进行一些格式验证和处理。 + //如果文件头信息是直接提供的，则根据分隔符的不同输出相应的文件头信息。 } /* For each column type, get its out function. */ - cstate->out_functions = (FmgrInfo*)palloc(num_phys_attrs * sizeof(FmgrInfo)); - foreach (cur, cstate->attnumlist) { - int attnum = lfirst_int(cur); - Oid out_func_oid; - bool isvarlena = false; - getTypeOutputInfo(attr[attnum - 1]->atttypid, &out_func_oid, &isvarlena); - fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]); + // 为每列类型获取其输出函数 + cstate->out_functions = (FmgrInfo*)palloc(num_phys_attrs * sizeof(FmgrInfo));//分配存储输出函数的数组 + foreach (cur, cstate->attnumlist) {//对于每个属性列，遍历列表 + int attnum = lfirst_int(cur);//获取属性列号 + Oid out_func_oid;//获取函数的OID + bool isvarlena = false;//是否为可变长度类型 + getTypeOutputInfo(attr[attnum - 1]->atttypid, &out_func_oid, &isvarlena);//获取属性列的输出函数和是否为可变长度类型，是否具有变长属性的信息存入isvarlena + //函数来源src\common\backend\utils\cache\lsyscache.cpp + fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);//获取输出函数的函数管理信息，并将其存储在cstate->out_functions数组中的相应位置 } /* @@ -116,15 +133,25 @@ static uint64 ledger_copytable(CopyState cstate) * datatype output routines, and should be faster than retail pfree's * anyway. (We don't need a whole econtext as CopyFrom does.) */ + //创建一个临时内存上下文，用于在每行处理完成后重置并释放分配的内存 + //CurrentMemoryContext:新创建的内存上下文将使用当前内存上下文作为父上下文 + //"COPY TO":内存上下文的名称 + //ALLOCSET_DEFAULT_MINSIZE:指定了内存上下文的初始大小 + //ALLOCSET_DEFAULT_INITSIZE:指定了内存上下文初始分配的大小 + //ALLOCSET_DEFAULT_MAXSIZE:指定了内存上下文可申请的最大内存量 cstate->rowcontext = AllocSetContextCreate( CurrentMemoryContext, "COPY TO", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); - + /* * For non-binary copy, we need to convert null_print to file * encoding, because it will be sent directly with CopySendString. */ - if (cstate->need_transcoding) { + if (cstate->need_transcoding) { //对于非二进制复制，需要将null_print转换为文件编码 + //确保在进行非二进制复制时，null_print能够以正确的文件编码进行发送和处理 cstate->null_print_client = pg_server_to_any(cstate->null_print, cstate->null_print_len, cstate->file_encoding); + //函数来源src\common\backend\utils\mb\mbutils.cpp + //如果当前会话上下文中的客户端编码和目标编码相同，则调用perform_default_encoding_conversion函数来执行默认的编码转换，并返回结果字符串 + //否则，将通过调用pg_do_encoding_conversion函数来进行编码转换。该函数将原始字符串s从当前会话上下文中的数据库编码转换为目标编码，并返回结果字符串。 } Tuple tuple; @@ -134,51 +161,54 @@ static uint64 ledger_copytable(CopyState cstate) Datum *values = NULL; bool *nulls = NULL; - values = (Datum*)palloc0(num_phys_attrs * sizeof(Datum)); - nulls = (bool*)palloc0(num_phys_attrs * sizeof(bool)); + values = (Datum*)palloc0(num_phys_attrs * sizeof(Datum));//分配存储值的数组 + nulls = (bool*)palloc0(num_phys_attrs * sizeof(bool));//分配存储空值标志的数组 - scan_desc = scan_handler_tbl_beginscan(cur_rel, GetActiveSnapshot(), 0, NULL); + scan_desc = scan_handler_tbl_beginscan(cur_rel, GetActiveSnapshot(), 0, NULL);//开始表扫描 /* For each row, we will recalculate previous hash. */ + //对于每一行，重新计算之前的哈希值 while ((tuple = scan_handler_tbl_getnext(scan_desc, ForwardScanDirection, cur_rel)) != NULL) { - CHECK_FOR_INTERRUPTS(); + CHECK_FOR_INTERRUPTS();//每次循环检查是否有中断请求，确保在进行长时间运行的操作时，可以及时响应中断信号 /* Deconstruct the tuple ... faster than repeated heap_getattr */ + //解析元组并获取列值和空值标记，存储在数组values和nulls中 tableam_tops_deform_tuple2(tuple, tuple_desc, values, nulls, GetTableScanDesc(scan_desc, cur_rel)->rs_cbuf); - if (!is_gchain) { + if (!is_gchain) {//对于非全局链关系，设置记录数和上一个哈希值的属性值 char comb_str[NAMEDATALEN] = {0}; - uint64 t_ins = nulls[USERCHAIN_COLUMN_HASH_INS] ? 0 : DatumGetUInt64(values[USERCHAIN_COLUMN_HASH_INS]); - uint64 t_del = nulls[USERCHAIN_COLUMN_HASH_DEL] ? 0 : DatumGetUInt64(values[USERCHAIN_COLUMN_HASH_DEL]); - errno_t rc = sprintf_s(comb_str, NAMEDATALEN, "%lu%lu", t_ins, t_del); + uint64 t_ins = nulls[USERCHAIN_COLUMN_HASH_INS] ? 0 : DatumGetUInt64(values[USERCHAIN_COLUMN_HASH_INS]);//哈希插入值 + uint64 t_del = nulls[USERCHAIN_COLUMN_HASH_DEL] ? 0 : DatumGetUInt64(values[USERCHAIN_COLUMN_HASH_DEL]);//哈希删除值 + errno_t rc = sprintf_s(comb_str, NAMEDATALEN, "%lu%lu", t_ins, t_del);//将插入值和删除值组合为字符串，存储在comb_str中 securec_check_ss(rc, "", ""); - gen_hist_tuple_hash(RelationGetRelid(cur_rel), comb_str, rec_num > 0, &pre_hash, &pre_hash); - values[USERCHAIN_COLUMN_REC_NUM] = UInt64GetDatum(rec_num); - values[USERCHAIN_COLUMN_PREVHASH] = HASH32GetDatum(&pre_hash); - } else { - char *db_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_dbname - 1])); - char *user_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_username - 1])); - char *rel_nsp = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_relnsp - 1])); - char *rel_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_relname - 1])); - char *query_string = TextDatumGetCString(values[Anum_gs_global_chain_txcommand - 1]); - uint64 rel_hash = DatumGetUInt64(values[Anum_gs_global_chain_relhash - 1]); - char *comb_str = set_gchain_comb_string(db_name, user_name, rel_nsp, rel_name, query_string, rel_hash); - gen_global_hash(&pre_hash, comb_str, rec_num > 0, &pre_hash); + gen_hist_tuple_hash(RelationGetRelid(cur_rel), comb_str, rec_num > 0, &pre_hash, &pre_hash);//函数来源src\gausskernel\security\gs_ledger\userchain.cpp + values[USERCHAIN_COLUMN_REC_NUM] = UInt64GetDatum(rec_num);//设置记录数属性值 + values[USERCHAIN_COLUMN_PREVHASH] = HASH32GetDatum(&pre_hash);//设置上一个哈希值属性值 + } else {//对于全局链关系，设置块号和全局哈希的属性值 + char *db_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_dbname - 1]));//数据库名 + char *user_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_username - 1]));//用户名 + char *rel_nsp = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_relnsp - 1]));//模式名 + char *rel_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_relname - 1]));//关系名 + char *query_string = TextDatumGetCString(values[Anum_gs_global_chain_txcommand - 1]);//查询字符串 + uint64 rel_hash = DatumGetUInt64(values[Anum_gs_global_chain_relhash - 1]);//关系哈希值 + char *comb_str = set_gchain_comb_string(db_name, user_name, rel_nsp, rel_name, query_string, rel_hash);//组合字符串 + gen_global_hash(&pre_hash, comb_str, rec_num > 0, &pre_hash);//计算全局哈希 values[Anum_gs_global_chain_blocknum - 1] = UInt64GetDatum(rec_num); values[Anum_gs_global_chain_globalhash - 1] = HASH32GetDatum(&pre_hash); pfree(comb_str); } /* Format and send the data */ - CopyOneRowTo(cstate, HeapTupleGetOid((HeapTuple)tuple), values, nulls); + CopyOneRowTo(cstate, HeapTupleGetOid((HeapTuple)tuple), values, nulls);//发送数据 + //函数来源src\gausskernel\optimizer\commands\copy.cpp rec_num++; processed++; } - scan_handler_tbl_endscan(scan_desc); - + scan_handler_tbl_endscan(scan_desc);//结束表扫描 + //释放内存 pfree_ext(values); pfree_ext(nulls); - MemoryContextDelete(cstate->rowcontext); + MemoryContextDelete(cstate->rowcontext);//删除临时内存上下文 - return processed; + return processed;//返回已处理的记录数 } /* @@ -190,6 +220,7 @@ static uint64 ledger_copytable(CopyState cstate) * Note: notice that this function is only used for copying hist table. * DO NOT use in other copy scenarios. */ +//将数据从一个关系（表）复制到另一个地方 static uint64 ledger_docopy(CopyStmt *stmt, const char *queryString) { Relation rel; @@ -201,73 +232,92 @@ static uint64 ledger_docopy(CopyStmt *stmt, const char *queryString) int attnum; /* Open and lock the relation, using the appropriate lock type. */ + //打开关系（表）并获取锁定访问权限 rel = heap_openrv(stmt->relation, AccessShareLock); + //创建范围表项并设置相关属性 rte = makeNode(RangeTblEntry); - rte->rtekind = RTE_RELATION; - rte->relid = RelationGetRelid(rel); - rte->relkind = rel->rd_rel->relkind; - rte->requiredPerms = ACL_SELECT; + rte->rtekind = RTE_RELATION;//表示该范围表项对应的是一个关系（表） + rte->relid = RelationGetRelid(rel);//获取打开关系（表）的标识符 + rte->relkind = rel->rd_rel->relkind;//获取关系（表）的类型 + rte->requiredPerms = ACL_SELECT;//该范围表项需要执行SELECT操作的权限 + //获取关系（表）的元组描述符 tup_desc = RelationGetDescr(rel); + //根据关系ID确定需要选择的列数 attnum = (rte->relid == GsGlobalChainRelationId) ? Natts_gs_global_chain : USERCHAIN_COLUMN_NUM; /* add columns that need select permission. */ + //添加需要选择权限的列至范围表项中 for (int i = 1; i <= attnum; ++i) { + //从1循环到attnum（需要选择的列数），根据FirstLowInvalidHeapAttributeNumber（表中的第一个有效列号）计算出对应的列号attno int attno = i - FirstLowInvalidHeapAttributeNumber; - rte->selectedCols = bms_add_member(rte->selectedCols, attno); + rte->selectedCols = bms_add_member(rte->selectedCols, attno);//函数来源src\common\backend\nodes\bitmapset.cpp + //记录需要选择的列 } + //检查是否具有查询权限 (void)ExecCheckRTPerms(list_make1(rte), true); + //创建CopyState结构，并执行COPY操作 cstate = BeginCopyTo(rel, query, queryString, stmt->filename, stmt->attlist, stmt->options); cstate->range_table = list_make1(rte); cstate->curPartionRel = cstate->rel; - processed = ledger_copytable(cstate); + processed = ledger_copytable(cstate);//复制数据库中表数据 EndCopyTo(cstate); + //关闭关系（表），释放锁定访问权限 if (rel != NULL) { heap_close(rel, AccessShareLock); } - + //返回处理的行数 return processed; } /* - * get_current_timestamp_text -- generate time text for name appending + * get_current_timestamp_text -- generate time text for name appending//生成用于名称附加的时间文本 * - * time_str: time text buffer to fill. + * time_str: time text buffer to fill.//用于填充时间文本的缓冲区 * - * Note: get current server time and remove dot. + * Note: get current server time and remove dot.//获取当前服务器时间并去除小数点。 */ +//获取当前时间戳的整数部分，并将其转换为字符串形式 static void get_current_timestamp_text(char *time_str) { - const char *now = timestamptz_to_str(GetCurrentTimestamp()); - size_t time_len = strlen(now); - size_t pos = 0; + const char *now = timestamptz_to_str(GetCurrentTimestamp());//获取当前时间戳字符串 + size_t time_len = strlen(now);//时间戳字符串的长度 + size_t pos = 0;//字符写入位置 + //遍历时间戳字符串的每个字符 for (size_t i = 0; i < time_len; ++i) { - if (now[i] >= '0' && now[i] <= '9') { - time_str[pos++] = now[i]; - } else if (now[i] == '.') { - break; + if (now[i] >= '0' && now[i] <= '9') { //如果当前字符为数字 + time_str[pos++] = now[i];//将数字字符写入结果字符串 + } else if (now[i] == '.') {//如果当前字符为小数点 + break;//结束遍历，不再写入小数部分 } } - time_str[pos] = '\0'; + time_str[pos] = '\0';//在结果字符串的末尾添加结束符 } /* - * copy_local_hist_table -- copy history table to hist_back dir. + * copy_local_hist_table -- copy history table to hist_back dir.//将历史表复制到hist_back目录 * - * relid: oid of usertable - * histname: the name of history table - * time: time text which will append to copyfile name. + * relid: oid of usertable//用户表的OID + * histname: the name of history table//历史表的名称 + * time: time text which will append to copyfile name.//将附加到复制文件名的时间文本 */ +//这段代码通过构造COPY语句，并指定表名、备份文件路径等参数，然后调用ledger_docopy函数执行COPY操作，将本地历史表的内容复制到指定路径下的文件中 +//将历史表复制到hist_back目录 static void copy_local_hist_table(Oid relid, char *histname, const char *time) { - errno_t rc; - char path[MAXPGPATH] = {0}; - StringInfoData buf; - initStringInfo(&buf); + errno_t rc;//定义一个用于检查返回码的变量 + char path[MAXPGPATH] = {0};//定义一个数组用于存储备份文件的路径，并初始化为全零 + StringInfoData buf;//定义一个用于构造COPY语句的字符串缓冲区变量 + initStringInfo(&buf);//初始化字符串缓冲区 + + //创建CopyStmt节点并设置成员 CopyStmt *stmt = makeNode(CopyStmt); - RangeVar *relation = makeRangeVar("blockchain", histname, -1); + RangeVar *relation = makeRangeVar("blockchain", histname, -1);//指向历史表 + + //构造备份文件路径 if (!is_absolute_path(g_instance.attr.attr_security.Audit_directory)) { + //将路径格式化为字符串 rc = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/%s/hist_bak/%s_%u_%s.hist", t_thrd.proc_cxt.DataDir, g_instance.attr.attr_security.Audit_directory, histname, relid, time); } else { @@ -275,128 +325,165 @@ static void copy_local_hist_table(Oid relid, char *histname, const char *time) g_instance.attr.attr_security.Audit_directory, histname, relid, time); } securec_check_ss(rc, "", ""); + + //构造COPY语句 appendStringInfo(&buf, "COPY blockchain.%s to \'%s\'", histname, path); + //设置CopyStmt成员 stmt->relation = relation; stmt->is_from = false; stmt->filename = path; + //调用ledger_docopy函数执行COPY操作 ledger_docopy((CopyStmt *)stmt, buf.data); } /* - * open_histback_dir -- open hist_back dir. + * open_histback_dir -- open hist_back dir.//打开 hist_back 目录 * - * dir_path: hist_back dir path. + * dir_path: hist_back dir path.//hist_back 目录的路径 * - * Note: caller need to close hist_back dir later. + * Note: caller need to close hist_back dir later.//调用者需要稍后关闭 hist_back 目录 */ +//根据备份文件目录的相对或绝对路径构造历史备份目录的完整路径，并使用AllocateDir函数打开该目录，然后返回一个指向该目录的DIR指针。 +//用于打开历史备份目录 static DIR *open_histback_dir(char *dir_path) { + //声明变量和初始化 DIR *dir = NULL; errno_t rc; + //判断是否为绝对路径来构造历史备份目录路径 if (!is_absolute_path(g_instance.attr.attr_security.Audit_directory)) { + //如果不是绝对路径，则使用相对路径构造历史备份目录的完整路径 rc = snprintf_s(dir_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s/hist_bak", t_thrd.proc_cxt.DataDir, g_instance.attr.attr_security.Audit_directory); } else { + //如果是绝对路径，则直接使用该路径作为历史备份目录的完整路径 rc = snprintf_s(dir_path, MAXPGPATH, MAXPGPATH - 1, "%s/hist_bak", g_instance.attr.attr_security.Audit_directory); } - securec_check_ss(rc, "", ""); + securec_check_ss(rc, "", "");//检查路径构造过程中的返回码，如果出现错误则报错 + //使用AllocateDir函数打开历史备份目录，将返回的DIR指针赋值给dir dir = AllocateDir(dir_path); + //返回DIR指针 return dir; } /* - * get_histback_dir_filesize -- count all file size of hist_back dir. + * get_histback_dir_filesize -- count all file size of hist_back dir.//计算hist_back目录中所有文件的总大小 */ +//打开历史备份目录并遍历其中的文件，对普通文件累加文件大小，最后返回历史备份目录中所有文件的总大小 static uint64 get_histback_dir_filesize() { - DIR *dir = NULL; - struct dirent *file = NULL; - char dir_path[MAXPGPATH] = {0}; - errno_t rc = EOK; - uint64 size = 0; + DIR *dir = NULL;//定义指向目录的指针，初始化为NULL + struct dirent *file = NULL;//定义指向目录项结构体的指针，初始化为NULL + char dir_path[MAXPGPATH] = {0};//定义保存目录路径的字符数组，并初始化为0 + errno_t rc = EOK;//用于检查返回码的变量，初始化为 EOK。 + uint64 size = 0;//定义保存文件大小的变量，初始值为0 + + //调用open_histback_dir函数打开历史备份目录，将返回的DIR指针赋值给dir dir = open_histback_dir(dir_path); + //如果打开历史备份目录失败，则返回0 if (dir == NULL) { return 0; } - + //循环遍历历史备份目录中的文件 while ((file = ReadDir(dir, dir_path)) != NULL) { - struct stat statbuf; - char filepath[MAXPGPATH]; + struct stat statbuf;//声明保存文件属性的结构体 + char filepath[MAXPGPATH];//声明保存文件完整路径的字符数组 + + //判断当前文件是否为.或..目录，如果是则跳过本次循环 if (strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0) { continue; } + //构造文件的完整路径 rc = snprintf_s(filepath, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dir_path, file->d_name); securec_check_ss(rc, "", ""); - + //判断当前文件是否是普通文件，并通过stat函数获取文件属性 if (file->d_type == DT_REG && stat(filepath, &statbuf) == 0) { - size += (uint64)statbuf.st_size; + size += (uint64)statbuf.st_size;//累加文件大小到size变量 } } + //关闭目录 FreeDir(dir); + //返回历史备份目录中所有文件的总大小 return size; } /* - * remove_oldest_histback_file -- remove oldest file in hist_back + * remove_oldest_histback_file -- remove oldest file in hist_back//删除 hist_back 目录中最早创建的文件 * * Note: the defination of 'oldest' here is the earliest file that * was created. this function will find and try to delete this file. */ +//这段代码打开历史备份目录并遍历其中的文件，找到最早创建的普通文件，删除该文件并返回其大小 static uint64 remove_oldest_histback_file() { - DIR *dir = NULL; - errno_t rc = EOK; - uint64 filesize = 0; - long min_ctime = LONG_MAX; - char dir_path[MAXPGPATH] = {0}; - char del_file[MAXPGPATH] = {0}; - struct dirent *file = NULL; - struct stat stat_buf; + DIR *dir = NULL;//定义指向目录的指针，初始化为NULL + errno_t rc = EOK;//用于检查返回码的变量 + uint64 filesize = 0;//定义保存文件大小的变量，初始值为0 + long min_ctime = LONG_MAX;//保存最早创建时间的变量，初始值为LONG_MAX + char dir_path[MAXPGPATH] = {0};//定义保存目录路径的字符数组，并初始化为0 + char del_file[MAXPGPATH] = {0};//定义保存待删除文件路径的字符数组，并初始化为0 + struct dirent *file = NULL;//定义指向目录项结构体的指针，初始化为NULL + struct stat stat_buf;//声明保存文件属性的结构体 + //调用open_histback_dir函数打开历史备份目录，将返回的DIR指针赋值给dir dir = open_histback_dir(dir_path); + //如果打开历史备份目录失败，则返回0 if (dir == NULL) { return 0; } - + //循环遍历历史备份目录中的文件 while ((file = ReadDir(dir, dir_path)) != NULL) { - char file_path[MAXPGPATH]; + char file_path[MAXPGPATH];//声明保存文件完整路径的字符数组 + + //判断当前文件是否为.或..目录，如果是则跳过本次循环 if (strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0) { continue; } + //构造文件的完整路径 rc = snprintf_s(file_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dir_path, file->d_name); securec_check_ss(rc, "", ""); + // 判断当前文件是否是普通文件，并通过stat函数获取文件属性 if (file->d_type == DT_REG && stat(file_path, &stat_buf) == 0) { + //如果当前文件的创建时间较早，则更新最早创建时间和待删除文件路径 if (min_ctime >= stat_buf.st_ctime) { min_ctime = stat_buf.st_ctime; rc = snprintf_s(del_file, MAXPGPATH, MAXPGPATH - 1, "%s", file_path); securec_check_ss(rc, "", ""); - filesize = (uint64)stat_buf.st_size; + filesize = (uint64)stat_buf.st_size;//更新要返回的文件大小 } } } + //关闭目录 FreeDir(dir); + //使用unlink函数删除待删除文件，如果失败则输出警告信息 if (unlink(del_file) < 0) { ereport(WARNING, (errmsg("could not remove histbak file: %s", del_file))); } + //返回被删除文件的大小 return filesize; } /* - * ledger_hist_archive -- interface for history table archive + * ledger_hist_archive -- interface for history table archive//历史表归档接口 * - * param1: the namespace of usertable. - * param2: the rel_name of usertable. + * param1: the namespace of usertable.//用户表的命名空间 + * param2: the rel_name of usertable.//用户表的关系名称 * * Note: this function will copy and unify hash_ins and hash_del * of history table belonging to given usertable. + * 注意：该函数将复制并统一给定用户表所属的历史表的 hash_ins 和 hash_del 操作。 */ +//归档用户历史表 Datum ledger_hist_archive(PG_FUNCTION_ARGS) { + //检查当前用户是否具有足够的权限 + //如果用户不是超级用户且不是审计管理员，则抛出错误 if (!isRelSuperuser() && !isAuditadmin(GetUserId())) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); } + //从参数中获取表的命名空间和名称 text *rel_nsp = PG_GETARG_TEXT_PP(0); text *rel_name = PG_GETARG_TEXT_PP(1); char *table_name; @@ -406,28 +493,39 @@ Datum ledger_hist_archive(PG_FUNCTION_ARGS) Oid relid; bool res = false; /* Init and prepare bak dictionary. */ + //准备历史表备份目录 prepare_histback_dir(); - + //将命名空间和表名转换为C字符串 table_nsp = text_to_cstring(rel_nsp); table_name = text_to_cstring(rel_name); + //获取命名空间的OID，并获取表的OID Oid nspoid = get_namespace_oid(table_nsp, false); relid = get_relname_relid(table_name, nspoid); + //检查用户表是否存在并且属于账本（ledger）命名空间 + //函数来源src\gausskernel\security\gs_ledger\ledger_utils.cpp ledger_usertable_check(relid, nspoid, table_name, table_nsp); - + //获取历史表的名称 + //函数来源src\gausskernel\security\gs_ledger\ledger_utils.cpp get_hist_name(relid, table_name, hist_name, nspoid, table_nsp); + get_current_timestamp_text(current_time); + + //如果当前角色不是协调器，则执行以下操作 if (g_instance.role != VCOORDINATOR) { /* - * Step 1. Copy user history table. + * Step 1. Copy user history table.复制用户历史表。 */ + //获取历史备份目录的总大小 uint64 total_histback_size = get_histback_dir_filesize(); + //如果总大小超过限制，则删除最旧的历史备份文件，直到总大小小于限制 while (total_histback_size >= (uint64)(u_sess->attr.attr_security.Audit_SpaceLimit * 1024L)) { total_histback_size -= remove_oldest_histback_file(); } /* Copy history table. */ + //复制历史表至本地 copy_local_hist_table(relid, hist_name, current_time); /* - * Step 2. Do unify and truncate. + * Step 2. Do unify and truncate. 进行统一和截断操作。 */ Datum values[USERCHAIN_COLUMN_NUM] = {0}; bool nulls[USERCHAIN_COLUMN_NUM] = {true}; @@ -440,21 +538,25 @@ Datum ledger_hist_archive(PG_FUNCTION_ARGS) TableScanDesc scan; HeapTuple tuple; /* sum all hash_ins and hash_del for unification. */ - lock_hist_hash_cache(LW_EXCLUSIVE); - Relation histRel = heap_open(get_relname_relid(hist_name, PG_BLOCKCHAIN_NAMESPACE), AccessExclusiveLock); - scan = heap_beginscan(histRel, SnapshotNow, 0, NULL); - while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { + // 对历史表进行扫描，计算hash_ins和hash_del的总和，并找到最大的记录数和对应的prevhash值 + lock_hist_hash_cache(LW_EXCLUSIVE);//获取对历史哈希缓存的独占锁 + Relation histRel = heap_open(get_relname_relid(hist_name, PG_BLOCKCHAIN_NAMESPACE), AccessExclusiveLock);//打开历史表,获取对该表的独占访问权限 + scan = heap_beginscan(histRel, SnapshotNow, 0, NULL);//开始对历史表进行扫描 + while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) {//扫描方向为正向（ForwardScanDirection） hist_empty = false; + //获取hash_ins值并累加 Datum value = heap_getattr(tuple, USERCHAIN_COLUMN_HASH_INS + 1, histRel->rd_att, &is_null); if (!is_null) { hash_ins += DatumGetUInt64(value); nulls[USERCHAIN_COLUMN_HASH_INS] = false; } + //获取hash_del值并累加 value = heap_getattr(tuple, USERCHAIN_COLUMN_HASH_DEL + 1, histRel->rd_att, &is_null); if (!is_null) { hash_del += DatumGetUInt64(value); nulls[USERCHAIN_COLUMN_HASH_DEL] = false; } + //获取当前记录数，并更新最大记录数和对应的prevhash值 cur_rec_num = DatumGetUInt64(heap_getattr(tuple, USERCHAIN_COLUMN_REC_NUM + 1, histRel->rd_att, &is_null)); if (max_rec_num <= cur_rec_num) { max_rec_num = cur_rec_num; @@ -462,82 +564,95 @@ Datum ledger_hist_archive(PG_FUNCTION_ARGS) heap_getattr(tuple, USERCHAIN_COLUMN_PREVHASH + 1, histRel->rd_att, &is_null); } } - heap_endscan(scan); + heap_endscan(scan);//结束对历史表的扫描操作 /* Empty table should not truncate and archive any more. */ + //如果历史表为空，则不执行统一和截断操作 if (hist_empty) { - heap_close(histRel, AccessExclusiveLock); - release_hist_hash_cache(); + heap_close(histRel, AccessExclusiveLock);//关闭历史表，并释放对该表的独占访问权限 + release_hist_hash_cache();//释放对历史哈希缓存的锁 res = true; return BoolGetDatum(res); } + //更新统一行的值 values[USERCHAIN_COLUMN_REC_NUM] = UInt64GetDatum(max_rec_num); values[USERCHAIN_COLUMN_HASH_INS] = UInt64GetDatum(hash_ins); values[USERCHAIN_COLUMN_HASH_DEL] = UInt64GetDatum(hash_del); nulls[USERCHAIN_COLUMN_REC_NUM] = false; nulls[USERCHAIN_COLUMN_PREVHASH] = false; - tuple = heap_form_tuple(RelationGetDescr(histRel), values, nulls); + tuple = heap_form_tuple(RelationGetDescr(histRel), values, nulls);//使用历史表的描述符（RelationGetDescr(histRel)）以及values和nulls数组来构建新的元组 /* Do real truncate. */ + //执行真正的截断操作，对历史表进行截断 heap_truncate_one_rel(histRel); /* Do insertion for unified row. */ - simple_heap_insert(histRel, tuple); - heap_freetuple(tuple); + //插入统一行 + simple_heap_insert(histRel, tuple);//将之前构建的元组插入到历史表中 + heap_freetuple(tuple);//释放对元组的内存空间 /* - * Step 3. Flush history hash table cache. + * Step 3. Flush history hash table cache.刷新历史哈希表缓存。 */ - remove_hist_recnum_cache(RelationGetRelid(histRel)); - heap_close(histRel, AccessExclusiveLock); - release_hist_hash_cache(); + remove_hist_recnum_cache(RelationGetRelid(histRel));//移除历史记录数量缓存 + heap_close(histRel, AccessExclusiveLock);//关闭历史表，并释放对该表的独占访问权限 + release_hist_hash_cache();//释放对历史哈希缓存的锁 res = true; } return BoolGetDatum(res); } /* - * ledger_gchain_archive -- archive gs_global_chain and unify each user rel + * ledger_gchain_archive -- archive gs_global_chain and unify each user rel//将gs_global_chain表归档并统一每个用户表 * * Note: this function will copy gchain to hist_back dir and accumulate * rel_hash for each relid. Additionally, recalculate globalhash to ensure the * consistency of front-to-back order. + * 注意：此函数将会将gchain表复制到hist_back目录，并为每个relid累加rel_hash + * 另外，重新计算globalhash以确保前后顺序的一致性。 */ +//用于归档全局链表(gs_global_chain)的函数 Datum ledger_gchain_archive(PG_FUNCTION_ARGS) { + //检查权限,只有超级用户或审核管理员才有权限执行该函数 if (!isRelSuperuser() && !isAuditadmin(GetUserId())) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); } bool res = false; /* Init and prepare bak dictionary. */ + //初始化并准备历史备份目录 prepare_histback_dir(); /* gs_global_chain table should only archived in CN */ + /* 只有在协调器或单节点角色下才对 gs_global_chain 表进行归档 */ if (g_instance.role != VCOORDINATOR && g_instance.role != VSINGLENODE) { return BoolGetDatum(res); } - + //获取历史备份目录的总大小 uint64 total_histback_size = get_histback_dir_filesize(); + //当总大小超过设定的审计空间限制时，删除最旧的历史备份文件 while (total_histback_size >= (uint64)(u_sess->attr.attr_security.Audit_SpaceLimit * 1024L)) { total_histback_size -= remove_oldest_histback_file(); } /* - * Step 1. Using CopyStmt to copy global chain. + * Step 1. Using CopyStmt to copy global chain.使用 CopyStmt 复制全局链表数据到备份文件中 */ CopyStmt *stmt = makeNode(CopyStmt); - RangeVar *relation = makeRangeVar("pg_catalog", GCHAIN_NAME, -1); + RangeVar *relation = makeRangeVar("pg_catalog", GCHAIN_NAME, -1);//pg_catalog表示表所在的namespace，GCHAIN_NAME是一个宏定义，表示表名 char path[MAXPGPATH] = {0}; StringInfoData buf; initStringInfo(&buf); errno_t rc; char current_time[NAMEDATALEN + 1]; - get_current_timestamp_text(current_time); - if (!is_absolute_path(g_instance.attr.attr_security.Audit_directory)) { + get_current_timestamp_text(current_time);//获取当前时间的文本形式 + + //根据绝对路径或相对路径生成备份文件路径 + if (!is_absolute_path(g_instance.attr.attr_security.Audit_directory)) {//相对路径 rc = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/%s/hist_bak/%s_%s.bak", t_thrd.proc_cxt.DataDir, g_instance.attr.attr_security.Audit_directory, GCHAIN_NAME, current_time); - } else { + } else {//绝对路径 rc = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/hist_bak/%s_%s.bak", g_instance.attr.attr_security.Audit_directory, GCHAIN_NAME, current_time); } @@ -546,10 +661,11 @@ Datum ledger_gchain_archive(PG_FUNCTION_ARGS) stmt->relation = relation; stmt->is_from = false; stmt->filename = path; + //执行复制操作 ledger_docopy((CopyStmt *)stmt, buf.data); /* - * Step 2. Do unify and truncate. + * Step 2. Do unify and truncate.统一和截断gs_global_chain表。 */ TableScanDesc scan; HeapTuple tuple; @@ -558,28 +674,35 @@ Datum ledger_gchain_archive(PG_FUNCTION_ARGS) bool is_null = false; gs_stl::gs_vector user_rel_arr; - rc = memset_s(&hash_ctl, sizeof(hash_ctl), 0, sizeof(hash_ctl)); + rc = memset_s(&hash_ctl, sizeof(hash_ctl), 0, sizeof(hash_ctl));//使用 memset_s 函数将 hash_ctl 清零 securec_check(rc, "\0", "\0"); /* Using hash table to do unify, each hash_entry refers to one relid informations. */ - hash_ctl.keysize = sizeof(Oid); - hash_ctl.entrysize = sizeof(GChainArchEntry); - hash_ctl.hash = oid_hash; - hash_ctl.hcxt = THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY); - HTAB *global_map = hash_create("Global Archive Hash", 256, &hash_ctl, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); + //使用哈希表进行统一，每个哈希项对应一个relid的信息 + hash_ctl.keysize = sizeof(Oid);//设置为 Oid 类型的大小 + hash_ctl.entrysize = sizeof(GChainArchEntry);//设置为GChainArchEntry结构体的大小 + hash_ctl.hash = oid_hash;//设置为Oid类型的哈希函数 + hash_ctl.hcxt = THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY);//获取安全内存上下文 + HTAB *global_map = hash_create("Global Archive Hash", 256, &hash_ctl, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT);//创建哈希表 global_map，并进行错误检查 if (global_map == NULL) { ereport(ERROR, (errcode(ERRCODE_INITIALIZE_FAILED), errmsg("could not initialize Global Archive Hash table."))); } /* Split gs_global_chain by relid, and accumulate rel_hash to a new record for each rel. */ - LWLockAcquire(GlobalPrevHashLock, LW_EXCLUSIVE); - Relation global_rel = heap_open(GsGlobalChainRelationId, AccessExclusiveLock); - scan = heap_beginscan(global_rel, SnapshotNow, 0, NULL); + //按 relid 分割gs_global_chain表，并将关联记录进行累加 + LWLockAcquire(GlobalPrevHashLock, LW_EXCLUSIVE);//调用 LWLockAcquire 函数获取全局互斥锁GlobalPrevHashLock。这是为了确保在进行统一和截断操作时，不会有其他进程同时访问gs_global_chain表。 + Relation global_rel = heap_open(GsGlobalChainRelationId, AccessExclusiveLock);//打开关系表gs_global_chain，并加上访问锁 AccessExclusiveLock + scan = heap_beginscan(global_rel, SnapshotNow, 0, NULL);//扫描时使用的快照是 SnapshotNow，表示当前的快照 while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { + //遍历 gs_global_chain 表中的所有记录，根据 relid 执行不同的操作。 + //如果 relid 是第一次出现，创建一个新的项并记录数据；如果 relid 已经存在，将 rel_hash 进行累加并更新记录数量。 + //这样就完成了 gs_global_chain 表的统一和截断操作。 Oid relid = DatumGetObjectId(heap_getattr(tuple, Anum_gs_global_chain_relid, global_rel->rd_att, &is_null)); - GChainArchEntry *item = (GChainArchEntry *)hash_search(global_map, &relid, HASH_ENTER, &found); + GChainArchEntry *item = (GChainArchEntry *)hash_search(global_map, &relid, HASH_ENTER, &found);//在哈希表global_map中查找相应的项,如果找到了对应的项，则将found标志设置为 true；如果没有找到，则创建一个新的项，并将found标志设置为false。同时，将relid添加数组中 if (!found) { /* If not exist in hash table, create an entry and records Datums. */ + /* 如果哈希表中不存在对应的项，则创建一个新的项并记录数据 */ user_rel_arr.push_back(relid); - heap_deform_tuple(tuple, RelationGetDescr(global_rel), item->val.values, item->val.nulls); + heap_deform_tuple(tuple, RelationGetDescr(global_rel), item->val.values, item->val.nulls);//将元组拆解并赋值 + //用DirectFunctionCall1函数将某些字符串类型转换成合适的数据类型 item->val.values[Anum_gs_global_chain_dbname - 1] = DirectFunctionCall1(namein, item->val.values[Anum_gs_global_chain_dbname - 1]); item->val.values[Anum_gs_global_chain_username - 1] = @@ -592,8 +715,9 @@ Datum ledger_gchain_archive(PG_FUNCTION_ARGS) item->val.rec_nums = 1; } else { /* If exists in hash table, just add rel_hash. */ + /* 如果已存在哈希表中，则将 rel_hash 累加 */ uint64 rel_hash = - DatumGetUInt64(heap_getattr(tuple, Anum_gs_global_chain_relhash, global_rel->rd_att, &is_null)); + DatumGetUInt64(heap_getattr(tuple, Anum_gs_global_chain_relhash, global_rel->rd_att, &is_null));//从元组中获取 rel_hash 的值，并将其加上之前项中的值 rel_hash += DatumGetUInt64(item->val.values[Anum_gs_global_chain_relhash - 1]); item->val.values[Anum_gs_global_chain_relhash - 1] = UInt64GetDatum(rel_hash); ++item->val.rec_nums; @@ -601,24 +725,27 @@ Datum ledger_gchain_archive(PG_FUNCTION_ARGS) } heap_endscan(scan); - if (user_rel_arr.empty()) { - heap_close(global_rel, AccessExclusiveLock); - LWLockRelease(GlobalPrevHashLock); - hash_destroy(global_map); + if (user_rel_arr.empty()) {//如果数组为空，表示进行统一和截断操作时没有找到任何匹配的relid记录 + heap_close(global_rel, AccessExclusiveLock);//闭全局表 global_rel，并释放访问锁 AccessExclusiveLock + LWLockRelease(GlobalPrevHashLock);//释放全局互斥锁 GlobalPrevHashLock + hash_destroy(global_map);//销毁哈希表 global_map res = true; return BoolGetDatum(res); } /* Do rel truncate. */ + // 截断 gs_global_chain 表 heap_truncate_one_rel(global_rel); /* Insert newest record to gchain order by relid. */ + //按 relid 排序插入最新记录到 gchain 表中 hash32_t global_hash; uint64 blocknum = -1; - for (int i = user_rel_arr.size() - 1; i >= 0; --i) { - GChainArchEntry *item = (GChainArchEntry *)hash_search(global_map, &user_rel_arr[i], HASH_FIND, &found); + for (int i = user_rel_arr.size() - 1; i >= 0; --i) {//从后向前处理每个元素 + GChainArchEntry *item = (GChainArchEntry *)hash_search(global_map, &user_rel_arr[i], HASH_FIND, &found);//在哈希表global_map中查找与当前relid相对应的项，并将结果存储在item /* Prepare common string */ const char *query_string = "Archived."; + //用DirectFunctionCall1函数将item中的特定列值转换成合适的字符串类型 char *db_name = DatumGetCString(DirectFunctionCall1(nameout, item->val.values[Anum_gs_global_chain_dbname - 1])); char *user_name = @@ -630,24 +757,24 @@ Datum ledger_gchain_archive(PG_FUNCTION_ARGS) uint64 rel_hash = DatumGetUInt64(item->val.values[Anum_gs_global_chain_relhash - 1]); char *com_str = set_gchain_comb_string(db_name, user_name, rel_nsp, rel_name, query_string, rel_hash); /* Generate global_hash. */ - gen_global_hash(&global_hash, com_str, i < (int)user_rel_arr.size() - 1, &global_hash); + gen_global_hash(&global_hash, com_str, i < (int)user_rel_arr.size() - 1, &global_hash);//生成全局哈希 blocknum += item->val.rec_nums; item->val.values[Anum_gs_global_chain_blocknum - 1] = UInt64GetDatum(blocknum); item->val.values[Anum_gs_global_chain_globalhash - 1] = HASH32GetDatum(&global_hash); tuple = heap_form_tuple(RelationGetDescr(global_rel), item->val.values, item->val.nulls); - simple_heap_insert(global_rel, tuple); - CatalogUpdateIndexes(global_rel, tuple); + simple_heap_insert(global_rel, tuple);//将tuple插入到global_rel表中 + CatalogUpdateIndexes(global_rel, tuple);//更新global_rel表的索引 heap_freetuple(tuple); pfree(com_str); } - hash_destroy(global_map); + hash_destroy(global_map);//销毁哈希表，释放相关资源 /* - * Step 3. Flush global_hash cache. + * Step 3. Flush global_hash cache.刷新 global_hash 缓存 */ - reset_g_blocknum(); - heap_close(global_rel, AccessExclusiveLock); - LWLockRelease(GlobalPrevHashLock); + reset_g_blocknum();//刷新global_hash缓存 + heap_close(global_rel, AccessExclusiveLock);//关闭全局表 global_rel，并释放访问锁AccessExclusiveLock + LWLockRelease(GlobalPrevHashLock);//释放全局互斥锁 GlobalPrevHashLock res = true; return BoolGetDatum(res); -- 2.34.1 From 097306bf47fc58ab22b371ee443d7c9ab20fc9fe Mon Sep 17 00:00:00 2001 From: Auweafan Date: Sun, 27 Aug 2023 22:34:45 +0800 Subject: [PATCH 07/48] Update gs_string.cpp --- .../security/gs_policy/gs_string.cpp | 94 ++++++++++--------- 1 file changed, 50 insertions(+), 44 deletions(-) diff --git a/src/gausskernel/security/gs_policy/gs_string.cpp b/src/gausskernel/security/gs_policy/gs_string.cpp index a2d21f6ed..664112136 100644 --- a/src/gausskernel/security/gs_policy/gs_string.cpp +++ b/src/gausskernel/security/gs_policy/gs_string.cpp @@ -29,65 +29,71 @@ namespace gs_stl { #define MIN_STR_CAPACITY 16 -MemoryContext GetStringMemory() +MemoryContext GetStringMemory()//获取字符串内存的内存上下文 { - if (t_thrd.security_policy_cxt.StringMemoryContext == NULL) { + if (t_thrd.security_policy_cxt.StringMemoryContext == NULL) {//没有分配内存上下文，需要进行内存分配 t_thrd.security_policy_cxt.StringMemoryContext = AllocSetContextCreate(THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY), "StringMemory", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); + /* + #define ALLOCSET_DEFAULT_MINSIZE 0 + #define ALLOCSET_DEFAULT_INITSIZE (8 * 1024) + #define ALLOCSET_DEFAULT_MAXSIZE (8 * 1024 * 1024) + */ } return t_thrd.security_policy_cxt.StringMemoryContext; } -void DeleteStringMemory() + +void DeleteStringMemory()//释放之前分配的字符串内存的内存上下文 { - if (t_thrd.security_policy_cxt.StringMemoryContext != NULL) { - MemoryContextDelete(t_thrd.security_policy_cxt.StringMemoryContext); - t_thrd.security_policy_cxt.StringMemoryContext = NULL; + if (t_thrd.security_policy_cxt.StringMemoryContext != NULL) {//已经分配了字符串内存的内存上下文，需要进行内存释放 + MemoryContextDelete(t_thrd.security_policy_cxt.StringMemoryContext);//删除之前分配的内存上下文。 + t_thrd.security_policy_cxt.StringMemoryContext = NULL;//表示内存上下文已被释放 } } -MemoryContext GetVectorMemory() +MemoryContext GetVectorMemory()//获取向量内存的内存上下文 { if (t_thrd.security_policy_cxt.VectorMemoryContext == NULL) { t_thrd.security_policy_cxt.VectorMemoryContext = AllocSetContextCreate(THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY), "VectorMemory", - ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); + ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE);//创建一个内存上下文，传入了一些参数 } return t_thrd.security_policy_cxt.VectorMemoryContext; } -void DeleteVectorMemory() +void DeleteVectorMemory()//释放之前分配的向量内存的内存上下文 { if (t_thrd.security_policy_cxt.VectorMemoryContext != NULL) { - MemoryContextDelete(t_thrd.security_policy_cxt.VectorMemoryContext); + MemoryContextDelete(t_thrd.security_policy_cxt.VectorMemoryContext);//删除之前分配的内存上下文。 t_thrd.security_policy_cxt.VectorMemoryContext = NULL; } } -MemoryContext GetMapMemory() +MemoryContext GetMapMemory()//获取映射内存的内存上下文 { - if (!t_thrd.security_policy_cxt.MapMemoryContext) { + if (!t_thrd.security_policy_cxt.MapMemoryContext) {//没有分配内存上下文，需要进行内存分配。 t_thrd.security_policy_cxt.MapMemoryContext = AllocSetContextCreate(TopMemoryContext, "MapMemory", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); } return t_thrd.security_policy_cxt.MapMemoryContext; } -void DeleteMapMemory() +void DeleteMapMemory()//释放之前分配的映射内存的内存上下文 { if (t_thrd.security_policy_cxt.MapMemoryContext) { - MemoryContextDelete(t_thrd.security_policy_cxt.MapMemoryContext); + MemoryContextDelete(t_thrd.security_policy_cxt.MapMemoryContext);//删除之前分配的内存上下文 t_thrd.security_policy_cxt.MapMemoryContext = nullptr; } } -void *_HashMapAllocFunc(Size request) +void *_HashMapAllocFunc(Size request)//在映射内存上下文中分配指定大小的内存空间 { - return MemoryContextAlloc(GetMapMemory(), request); + return MemoryContextAlloc(GetMapMemory(), request);//传入映射内存的内存上下文和请求的内存大小，来分配内存空间 } -MemoryContext GetSetMemory() +MemoryContext GetSetMemory()//存储了集合内存的内存上下文 { if (!t_thrd.security_policy_cxt.SetMemoryContext) { t_thrd.security_policy_cxt.SetMemoryContext = AllocSetContextCreate(TopMemoryContext, "SetMemory", @@ -96,7 +102,7 @@ MemoryContext GetSetMemory() return t_thrd.security_policy_cxt.SetMemoryContext; } -void DeleteSetMemory() +void DeleteSetMemory()//释放之前分配的集合内存的内存上下文 { if (t_thrd.security_policy_cxt.SetMemoryContext) { MemoryContextDelete(t_thrd.security_policy_cxt.SetMemoryContext); @@ -104,17 +110,17 @@ void DeleteSetMemory() } } -void *_HashSetAllocFunc(Size request) +void *_HashSetAllocFunc(Size request)//在集合内存上下文中分配指定大小的内存空间 { return MemoryContextAlloc(GetSetMemory(), request); } -int matchStr(const void *key1, const void *key2, Size keysize) +int matchStr(const void *key1, const void *key2, Size keysize)//比较两个字符串是否匹配。 { - return strncasecmp((const char *)key1, (const char *)key2, keysize - 1); + return strncasecmp((const char *)key1, (const char *)key2, keysize - 1);//比较两个字符串的前"keysize - 1"个字符，忽略大小写 } -int gs_stringCompareKeyFunc(const void *keyA, const void *keyB) +int gs_stringCompareKeyFunc(const void *keyA, const void *keyB)//比较两个字符串键的大小 { if (*(const gs_string *)keyA < *(const gs_string *)keyB) { return -1; @@ -126,7 +132,7 @@ int gs_stringCompareKeyFunc(const void *keyA, const void *keyB) } // string implementation -inline bool gs_string::InitBuff(const char *str, size_t len) +inline bool gs_string::InitBuff(const char *str, size_t len)//初始化字符串缓冲区。 { if (m_buff == NULL) { size_t init_len = (len > 0) ? (len + 1) : (strlen(str) + 1); @@ -139,12 +145,12 @@ inline bool gs_string::InitBuff(const char *str, size_t len) } return false; } -gs_string::gs_string(const char *str, size_t len) : m_buff(NULL), m_len(0), m_capacity(0) -{ +gs_string::gs_string(const char *str, size_t len) : m_buff(NULL), m_len(0), m_capacity(0)// 构造函数 +{//创建字符串对象 (void)InitBuff(str, len); } -gs_string::~gs_string() +gs_string::~gs_string()//析构函数，销毁字符串对象 { /* * Note that: container destruction will be called by system depending on @@ -162,13 +168,13 @@ gs_string::~gs_string() } gs_string::gs_string(const gs_string &arg) : m_buff(NULL), m_len(0), m_capacity(0) -{ +{//拷贝构造函数 operator = (arg); } -gs_string &gs_string::operator = (const gs_string &arg) +gs_string &gs_string::operator = (const gs_string &arg)//赋值运算符重载函数 { - if (&arg == this) { + if (&arg == this) {//传入的对象地址与当前对象地址相同 return *this; } @@ -187,7 +193,7 @@ gs_string &gs_string::operator = (const gs_string &arg) return *this; } -int gs_string::operator - (const gs_string &arg) const +int gs_string::operator - (const gs_string &arg) const//减法运算符重载函数实现 { if (this == &arg) { return 0; @@ -201,7 +207,7 @@ int gs_string::operator - (const gs_string &arg) const return 0; } -gs_string &gs_string::append(const gs_string &str) +gs_string &gs_string::append(const gs_string &str)//在字符串末尾追加字符串 { return append(str.c_str(), str.size()); } @@ -220,7 +226,7 @@ gs_string &gs_string::append(const char *str, size_t len) return *this; } -void gs_string::push_back(char ch) +void gs_string::push_back(char ch)//字符串末尾添加一个字符 { char t_chr[2] = {0}; t_chr[1] = ch; @@ -233,14 +239,14 @@ void gs_string::push_back(char ch) } } -void gs_string::pop_back() +void gs_string::pop_back()//字符串末尾删除一个字符 { if (m_len > 0) { m_buff[--m_len] = '\0'; } } -char gs_string::operator[](int idx) const +char gs_string::operator[](int idx) const//访问字符串中的指定位置的字符 { if (idx > (int)m_len) { return '\0'; @@ -248,7 +254,7 @@ char gs_string::operator[](int idx) const return m_buff[idx]; } -void gs_string::clear() +void gs_string::clear()//清空字符串 { if (m_buff != NULL) { m_buff[0] = '\0'; @@ -256,7 +262,7 @@ void gs_string::clear() } } -size_t gs_string::find(char arg, size_t start) const +size_t gs_string::find(char arg, size_t start) const//find { for (; start < m_len; ++start) { if (m_buff[start] == arg) { @@ -266,7 +272,7 @@ size_t gs_string::find(char arg, size_t start) const return npos; } -char gs_string::back() const +char gs_string::back() const//返回字符串的最后一个字符。 { if (m_len > 0) { return m_buff[m_len - 1]; @@ -274,7 +280,7 @@ char gs_string::back() const return m_buff[0]; } -gs_string gs_string::substr(size_t pos, size_t len) const +gs_string gs_string::substr(size_t pos, size_t len) const//返回从指定位置起始的指定长度子字符串。 { if ((pos + len) < m_len) { return gs_string((const char *)(m_buff + pos), len); @@ -285,7 +291,7 @@ gs_string gs_string::substr(size_t pos, size_t len) const return gs_string((const char *)m_buff, m_len); } -gs_string &gs_string::replace(size_t pos, size_t len, const char *s) +gs_string &gs_string::replace(size_t pos, size_t len, const char *s)//替换从指定位置开始的指定长度的子字符串。 { if (pos < m_len) { size_t rep_len = strlen(s) + 1; @@ -315,7 +321,7 @@ gs_string &gs_string::replace(size_t pos, size_t len, const char *s) return *this; } -void gs_string::erase(size_t pos, size_t len) +void gs_string::erase(size_t pos, size_t len)//删除从指定位置开始的指定长度的子字符串。 { if (m_len == 0 || (pos >= m_len)) { return; @@ -332,7 +338,7 @@ void gs_string::erase(size_t pos, size_t len) m_buff[m_len] = '\0'; } -bool gs_string::operator == (const gs_string &arg) const +bool gs_string::operator == (const gs_string &arg) const//比较两个字符串对象是否相等 { if (m_len != arg.m_len) { return false; @@ -343,17 +349,17 @@ bool gs_string::operator == (const gs_string &arg) const return m_len == 0; } -bool gs_string::operator < (const gs_string &arg) const +bool gs_string::operator < (const gs_string &arg) const//比较两个字符串对象大小关系。 { return strcasecmp(m_buff, arg.m_buff) < 0; } -inline char *gs_string::AllocFunc(size_t _size) const +inline char *gs_string::AllocFunc(size_t _size) const//用于分配字符串缓冲区 { return (char *)MemoryContextAlloc(GetStringMemory(), _size); } -inline char *gs_string::ReallocFunc(size_t _size) +inline char *gs_string::ReallocFunc(size_t _size)//用于重新分配字符串缓冲区 { m_capacity = _size; -- 2.34.1 From 3f12371f1d91dc1ff706cf096fb2813ba24ed4f1 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Sun, 27 Aug 2023 22:35:49 +0800 Subject: [PATCH 08/48] Update ledger_check.cpp --- .../security/gs_ledger/ledger_check.cpp | 184 ++++++++++-------- 1 file changed, 100 insertions(+), 84 deletions(-) diff --git a/src/gausskernel/security/gs_ledger/ledger_check.cpp b/src/gausskernel/security/gs_ledger/ledger_check.cpp index c64a351b4..1c7d07047 100644 --- a/src/gausskernel/security/gs_ledger/ledger_check.cpp +++ b/src/gausskernel/security/gs_ledger/ledger_check.cpp @@ -47,99 +47,106 @@ #include "gs_ledger/blockchain.h" /* - * gen_usertable_hash_sum -- calculate sum(hash) of the relation + * gen_usertable_hash_sum -- calculate sum(hash) of the relation//计算用户关系的hash总和 * - * rel: user relation + * rel: user relation//用户关系 */ +// 计算用户表的哈希总和 static uint64 gen_usertable_hash_sum(Relation rel) { - uint64 rel_hash = 0; - bool is_null = false; - int hash_natt = user_hash_attrno(rel->rd_att); - Assert(hash_natt >= 0); - HeapTuple tuple; - TupleDesc desc = rel->rd_att; - Snapshot snapshot = GetActiveSnapshot(); - TableScanDesc scan; - if (RELATION_CREATE_BUCKET(rel)) { + uint64 rel_hash = 0;//初始化表的哈希总和为0 + bool is_null = false;//初始化一个布尔变量，用于指示获取的属性值是否为空 + int hash_natt = user_hash_attrno(rel->rd_att);//获取哈希属性在表描述符中的位置 + Assert(hash_natt >= 0);//使用断言确保哈希属性的位置非负 + HeapTuple tuple;//声明一个堆元组变量，用于存储从表中获取的每个元组 + TupleDesc desc = rel->rd_att;//获取表的描述符 + Snapshot snapshot = GetActiveSnapshot();//获取当前活动快照 + TableScanDesc scan;//声明一个表扫描描述符 + if (RELATION_CREATE_BUCKET(rel)) {//如果表是分桶表 Relation bucket_rel = NULL; - oidvector *bucket_list = searchHashBucketByOid(rel->rd_bucketoid); + oidvector *bucket_list = searchHashBucketByOid(rel->rd_bucketoid);//获取分桶表的列表 + //遍历分桶表列表，每次取出一个分桶表的 OID，并调用bucketGetRelation函数获取该分桶表的关系对象 for (int i = 0; i < bucket_list->dim1; i++) { - bucket_rel = bucketGetRelation(rel, NULL, bucket_list->values[i]); - scan = heap_beginscan(bucket_rel, snapshot, 0, NULL); + bucket_rel = bucketGetRelation(rel, NULL, bucket_list->values[i]);//获取分桶表的关系对象 + scan = heap_beginscan(bucket_rel, snapshot, 0, NULL);//开始对分桶表进行堆扫描 + //遍历分桶表中的每个元组，使用 heap_getnext 函数获取下一个元组，并根据哈希属性的位置获取属性值，将其累加到哈希总和上 while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { rel_hash += DatumGetUInt64(heap_getattr(tuple, hash_natt + 1, desc, &is_null)); } - heap_endscan(scan); - bucketCloseRelation(bucket_rel); + heap_endscan(scan);//结束对分桶表的扫描 + bucketCloseRelation(bucket_rel);//关闭分桶表的关系对象 } - } else { - scan = heap_beginscan(rel, snapshot, 0, NULL); + } else {//如果表不是分桶表 + scan = heap_beginscan(rel, snapshot, 0, NULL);//开始对表进行堆扫描 + //遍历表中的每个元组，使用 heap_getnext 函数获取下一个元组，并根据哈希属性的位置获取属性值，将其累加到哈希总和上 while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { rel_hash += DatumGetUInt64(heap_getattr(tuple, hash_natt + 1, desc, &is_null)); } - heap_endscan(scan); + heap_endscan(scan);//结束扫描 } return rel_hash; } /* - * get_usertable_hash_sum -- calculate sum(hash) of the relation of specified oid + * get_usertable_hash_sum -- calculate sum(hash) of the relation of specified oid//计算指定 OID 的关系对象的哈希总和（sum(hash） * - * relid: user table oid + * relid: user table oid//用户表的 OID * - * NOTICE: if user table is partition table, just use the original relation id. + * NOTICE: if user table is partition table, just use the original relation id.//注意：如果用户表是分区表，则只使用原始关系 ID */ +//计算指定关系（表）的哈希总和 static uint64 get_usertable_hash_sum(Oid relid) { - uint64 rel_hash = 0; - Relation rel = NULL; + uint64 rel_hash = 0;//存储最终的哈希总和 + Relation rel = NULL;//存储打开的关系对象 rel = heap_open(relid, AccessShareLock); - if (!RelationIsPartitioned(rel)) { + if (!RelationIsPartitioned(rel)) {//判断打开的关系是否为分区表，如果不是，直接计算哈希总和 rel_hash = gen_usertable_hash_sum(rel); - } else { - List *partition_list = NIL; - ListCell *lc = NULL; - Partition part; - Relation fake_rel; - partition_list = relationGetPartitionList(rel, AccessShareLock); + } else {//关系为分区表 + List *partition_list = NIL;//存储分区列表 + ListCell *lc = NULL;//遍历分区列表的指针变量 + Partition part;//存储当前分区 + Relation fake_rel;//存储伪关系对象，即分区对应的关系对象 + partition_list = relationGetPartitionList(rel, AccessShareLock);//获取分区列表并使用共享锁 foreach (lc, partition_list) { part = (Partition)lfirst(lc); - fake_rel = partitionGetRelation(rel, part); + fake_rel = partitionGetRelation(rel, part);//通过分区和关系获取相应的伪关系对象 rel_hash += gen_usertable_hash_sum(fake_rel); - releaseDummyRelation(&fake_rel); + releaseDummyRelation(&fake_rel);//释放伪关系对象的资源 } - releasePartitionList(rel, &partition_list, AccessShareLock); + releasePartitionList(rel, &partition_list, AccessShareLock);//释放分区列表的资源 } heap_close(rel, AccessShareLock); return rel_hash; } /* - * get_histtable_hash_sum -- calculate sum(hash_ins) - sum(hash_del) of the history relation of specified oid + * get_histtable_hash_sum -- calculate sum(hash_ins) - sum(hash_del) of the history relation of specified oid// 计算指定 OID 的历史表的 sum(hash_ins) - sum(hash_del) * - * relid: history table oid + * relid: history table oid//历史表的 OID */ +//遍历所有历史表的行，并计算每行的 hash_ins 列和 hash_del 列的差值 static uint64 get_histtable_hash_sum(Oid hist_oid) { - uint64 rel_hash = 0; - bool is_null = false; - Relation hist_rel; - TableScanDesc scan; - HeapTuple tuple; - Snapshot snapshot = GetActiveSnapshot(); + uint64 rel_hash = 0;//初始化哈希总和为0 + bool is_null = false;//用于判断属性值是否为NULL + Relation hist_rel;//关系对象，用于表示分区表 + TableScanDesc scan;//表扫描描述符 + HeapTuple tuple;//元组，用于获取表中的每一行数据 + Snapshot snapshot = GetActiveSnapshot();//获取当前活动快照 - hist_rel = heap_open(hist_oid, AccessShareLock); - scan = heap_beginscan(hist_rel, snapshot, 0, NULL); + hist_rel = heap_open(hist_oid, AccessShareLock);//打开分区表并获取关系对象 + scan = heap_beginscan(hist_rel, snapshot, 0, NULL);//开始表扫描 while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { + // 从元组中获取hash_ins列的值 Datum value = heap_getattr(tuple, USERCHAIN_COLUMN_HASH_INS + 1, hist_rel->rd_att, &is_null); if (!is_null) { - rel_hash += DatumGetUInt64(value); + rel_hash += DatumGetUInt64(value);//如果列值不为NULL，则将其加到哈希总和中 } - + //从元组中获取 hash_del 列的值 value = heap_getattr(tuple, USERCHAIN_COLUMN_HASH_DEL + 1, hist_rel->rd_att, &is_null); if (!is_null) { - rel_hash -= DatumGetUInt64(value); + rel_hash -= DatumGetUInt64(value);//如果列值不为NULL，则将其从哈希总和中减去 } } heap_endscan(scan); @@ -148,17 +155,20 @@ static uint64 get_histtable_hash_sum(Oid hist_oid) } /* - * has_ledger_consistent_privilege -- calculate sum(hash_ins) - sum(hash_del) of the history relation of specified oid + * has_ledger_consistent_privilege -- calculate sum(hash_ins) - sum(hash_del) of the history relation of specified oid//计算指定OID的历史关系的sum(hash_ins) - sum(hash_del) * - * relid: history table oid + * relid: history table oid//历史表的OID */ +//验证当前用户是否具有对指定命名空间和关系表的权限.如果用户缺少任何一个权限，则认为其不具备一致权限 static bool has_ledger_consistent_privilege(Oid relid, Oid namespaceId) { + //检查当前用户是否具有对命名空间的ACL_USAGE权限 if (pg_namespace_aclcheck(namespaceId, GetUserId(), ACL_USAGE) != ACLCHECK_OK) { - return false; + return false;//没有权限 } + //检查当前用户是否具有对关系表的ACL_SELECT权限 if (pg_class_aclcheck(relid, GetUserId(), ACL_SELECT) != ACLCHECK_OK) { - return false; + return false;//没有权限 } return true; } @@ -169,28 +179,34 @@ static bool has_ledger_consistent_privilege(Oid relid, Oid namespaceId) * relid: user table oid * res_hash: hash sum of history table */ +//检查用户表的哈希总和和历史表的哈希总和是否相等 bool is_hist_hash_identity(Oid relid, uint64 *res_hash) { uint64 user_hash_sum; uint64 hist_hash_sum; - char hist_name[NAMEDATALEN]; + char hist_name[NAMEDATALEN];//存储历史表的名称 char *rel_name = get_rel_name(relid); + //获取历史表的名称并存储到hist_name数组中 if (!get_hist_name(relid, rel_name, hist_name)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("get hist table name failed."))); } + //根据历史表名称和命名空间获取历史表的对象标识符 Oid histoid = get_relname_relid(hist_name, PG_BLOCKCHAIN_NAMESPACE); - if (!OidIsValid(histoid)) { + if (!OidIsValid(histoid)) {//判断获取到的历史表的对象标识符是否无效（即未找到对应的历史表） + //抛出一个错误,找不到历史表 ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("could not find hist table of \"%s\".", rel_name))); } - + //获取用户表的哈希总和和历史表的哈希总和 user_hash_sum = get_usertable_hash_sum(relid); hist_hash_sum = get_histtable_hash_sum(histoid); - *res_hash = hist_hash_sum; - return user_hash_sum == hist_hash_sum; + *res_hash = hist_hash_sum;//将历史表的哈希总和赋值给结果哈希总和指针 + return user_hash_sum == hist_hash_sum;//返回用户表哈希总和和历史表哈希总和是否相等的结果 } - +//#ifdef ENABLE_MULTIPLE_NODES 和 #endif 是条件编译预处理指令，用于在编译代码时根据定义的宏来选择性地包含或排除一些代码片段 #ifdef ENABLE_MULTIPLE_NODES +//定义了两个静态函数 StrategyFuncAnd 和 StrategyFuncUInt64Sum， +//这些函数用于在并行计算的情况下收集远程节点的计算结果 /* * StrategyFuncAnd -- collect remote nodes with AND * @@ -202,27 +218,27 @@ bool is_hist_hash_identity(Oid relid, uint64 *res_hash) static void StrategyFuncAnd(ParallelFunctionState* state) { TupleTableSlot* slot = NULL; - bool result = true; + bool result = true;//初始化结果为真 Assert(state); Assert(state->tupstore); Assert(state->tupdesc); - slot = MakeSingleTupleTableSlot(state->tupdesc); + slot = MakeSingleTupleTableSlot(state->tupdesc);//创建一个单个元组的插槽 while (true) { - bool isnull = false; + bool isnull = false;//判断属性是否为NULL - if (!tuplestore_gettupleslot(state->tupstore, true, false, slot)) + if (!tuplestore_gettupleslot(state->tupstore, true, false, slot))//从元组存储中获取下一个元组 break; - if (!DatumGetBool(tableam_tslot_getattr(slot, 1, &isnull))) { - result = false; - break; + if (!DatumGetBool(tableam_tslot_getattr(slot, 1, &isnull))) {//获取元组的第一个属性值并判断是否为假 + result = false;//如果有一个节点的结果为假，则将结果设置为假 + break;//跳出循环 } - (void)ExecClearTuple(slot); + (void)ExecClearTuple(slot);//清空插槽中的元组 } - state->result = result; + state->result = result;//将最终的结果赋值给函数状态的 result 属性 } /* @@ -239,19 +255,19 @@ static void StrategyFuncUInt64Sum(ParallelFunctionState* state) int64 result = 0; Assert(state && state->tupstore && state->tupdesc); - slot = MakeSingleTupleTableSlot(state->tupdesc); + slot = MakeSingleTupleTableSlot(state->tupdesc);//创建一个单个元组的插槽 while (true) { bool isnull = false; - if (!tuplestore_gettupleslot(state->tupstore, true, false, slot)) + if (!tuplestore_gettupleslot(state->tupstore, true, false, slot))//从元组存储中获取下一个元组 break; - result += DatumGetUInt64(tableam_tslot_getattr(slot, 1, &isnull)); - ExecClearTuple(slot); + result += DatumGetUInt64(tableam_tslot_getattr(slot, 1, &isnull));//将元组的第一个属性值累加到结果上 + ExecClearTuple(slot);//清空插槽中的元组 } - state->result = result; + state->result = result;//将最终的结果赋值给函数状态的 result 属性 } #endif @@ -269,30 +285,30 @@ Datum ledger_hist_check(PG_FUNCTION_ARGS) bool res = false; char *table_name; char *table_nsp; - text *rel_nsp = PG_GETARG_TEXT_PP(0); - text *rel_name = PG_GETARG_TEXT_PP(1); + text *rel_nsp = PG_GETARG_TEXT_PP(0);//获取传入的命名空间参数 + text *rel_name = PG_GETARG_TEXT_PP(1);//获取传入的用户表名参数 - table_nsp = text_to_cstring(rel_nsp); - table_name = text_to_cstring(rel_name); - nsp_oid = get_namespace_oid(table_nsp, false); - relid = get_relname_relid(table_name, nsp_oid); - ledger_usertable_check(relid, nsp_oid, table_name, table_nsp); + table_nsp = text_to_cstring(rel_nsp);//将命名空间参数转换为C字符串 + table_name = text_to_cstring(rel_name);//将用户表名参数转换为C字符串 + nsp_oid = get_namespace_oid(table_nsp, false);//获取命名空间的OID + relid = get_relname_relid(table_name, nsp_oid);//根据用户表名和命名空间的OID获取用户表的OID + ledger_usertable_check(relid, nsp_oid, table_name, table_nsp);//检查用户表的存在性和合法性 - if (!has_ledger_consistent_privilege(relid, nsp_oid)) { + if (!has_ledger_consistent_privilege(relid, nsp_oid)) {//检查当前用户对用户表是否有足够的权限 ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); } - res = is_hist_hash_identity(relid, &res_hash); + res = is_hist_hash_identity(relid, &res_hash);//检查用户表的哈希值和历史表的哈希值是否相等 #ifdef ENABLE_MULTIPLE_NODES - if (!IsConnFromCoord()) { + if (!IsConnFromCoord()) {//非协调节点，则进行并行计算 StringInfoData buf; ParallelFunctionState* state = NULL; initStringInfo(&buf); appendStringInfo(&buf, "SELECT pg_catalog.ledger_hist_check('%s', '%s')", table_nsp, table_name); /* Get all hash diffs from DNs in distribute scenairo. */ - state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncAnd); - res &= state->result; - FreeParallelFunctionState(state); + state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncAnd);//调用远程函数并处理结果 + res &= state->result;//将远程节点的结果与本地结果进行逻辑与操作 + FreeParallelFunctionState(state);//释放并行函数状态的内存 } #endif return BoolGetDatum(res); -- 2.34.1 From 6a606596beddc0f790c6c4574953c2ac1bf02458 Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Sun, 27 Aug 2023 22:37:32 +0800 Subject: [PATCH 09/48] Update iprange.h --- src/include/iprange/iprange.h | 76 ++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/src/include/iprange/iprange.h b/src/include/iprange/iprange.h index ea2eb9b45..076586fca 100644 --- a/src/include/iprange/iprange.h +++ b/src/include/iprange/iprange.h @@ -30,22 +30,29 @@ #include #include +// ipv6arr定义为一个长度为16的无符号字符数组类型，表示IPv6地址的字节序列 typedef unsigned char ipv6arr[16]; - +// 128=2*64 typedef struct IPV6_64_s { uint64_t lower; uint64_t upper; } IPV6_64; - +// 128=4*32 typedef struct IPV6_32_s { + //书写顺序 d , c , b , a uint32_t a; uint32_t b; uint32_t c; uint32_t d; } IPV6_32; - +/* +* 定义了一个名为IPV6的联合体类型 +* 包含了两个成员变量ip_64和ip_32 +* 为了在不同的数据类型之间进行转换，可以将同一块数据以不同的方式进行表示 +* 即可以将IPv6地址表示为64位整数形式，也可以表示为32位整数形式 +*/ typedef union IPV6_s { IPV6_64 ip_64; @@ -75,14 +82,14 @@ inline const IPV6 operator+(const IPV6 lhs, const IPV6 rhs) uint64_t tmp; IPV6 new_ip {0,0}; - if (lhs.ip_32.b == 0x0000FFFF) { + if (lhs.ip_32.b == 0x0000FFFF) { // ipv4 new_ip.ip_32.a = lhs.ip_32.a + rhs.ip_32.a; new_ip.ip_32.b = 0x0000FFFF; return new_ip; } - + // ipv6 new_ip.ip_32.a = tmp = ((int64_t)lhs.ip_32.a) + rhs.ip_32.a; - new_ip.ip_32.b = tmp = ((int64_t)lhs.ip_32.b) + rhs.ip_32.b + (tmp >> 32); + new_ip.ip_32.b = tmp = ((int64_t)lhs.ip_32.b) + rhs.ip_32.b + (tmp >> 32); //溢出 new_ip.ip_32.c = tmp = ((int64_t)lhs.ip_32.c) + rhs.ip_32.c + (tmp >> 32); new_ip.ip_32.d = ((int64_t)lhs.ip_32.d) + rhs.ip_32.d + (tmp >> 32); return new_ip; @@ -97,13 +104,13 @@ inline const IPV6 operator+(const IPV6 lhs, const int i) inline const IPV6 operator-(const IPV6 lhs, const IPV6 rhs) { - if (lhs.ip_32.b == 0x0000FFFF) { + if (lhs.ip_32.b == 0x0000FFFF) { // ipv4 IPV6 new_ip {0,0}; new_ip.ip_32.a = lhs.ip_32.a - rhs.ip_32.a; new_ip.ip_32.b = 0x0000FFFF; return new_ip; } - + // ipv6 uint64_t tmp; IPV6 new_ip {0,0}; new_ip.ip_32.a = tmp = ((int64_t)lhs.ip_32.a) - rhs.ip_32.a; @@ -136,6 +143,7 @@ inline const IPV6 operator|(const IPV6 lhs, const IPV6 rhs) return new_ip; } +//用于表示 IP 地址范围 typedef struct Range_s { Range_s(const IPV6 _from = IPV6(), const IPV6 _to = IPV6()):from(_from), to(_to){} @@ -146,46 +154,48 @@ typedef struct Range_s class IPRange { public: - - IPRange(); + + IPRange(); //构造函数 ~IPRange(); - static bool is_range_valid(const std::string range); - bool add_ranges(const std::unordered_set ranges); - bool add_range(Range *new_range); - bool add_range(const char *range, size_t range_len); - bool remove_ranges(const std::unordered_set ranges); - bool remove_range(const char *range, size_t range_len); - std::unordered_set get_ranges_set(); + static bool is_range_valid(const std::string range); //检查给定的 IP 范围字符串是否有效 + bool add_ranges(const std::unordered_set ranges); //将一组 IP 地址范围添加到当前范围集合中 + bool add_range(Range *new_range); //将一个 IP 地址范围添加到当前范围集合中 + bool add_range(const char *range, size_t range_len); //将一个以字符串形式表示的 IP 地址范围添加到当前范围集合中 + bool remove_ranges(const std::unordered_set ranges); //从当前范围集合中移除一组 IP 地址范围 + bool remove_range(const char *range, size_t range_len); //从当前范围集合中移除一个以字符串形式表示的 IP 地址范围 + std::unordered_set get_ranges_set(); //获取当前范围集合的字符串表示形式 - bool is_in_range(const char *ip_str); - bool is_in_range(const IPV6 *ip); - bool is_in_range(const uint32_t ipv4); - bool is_intersect(const IPRange *arg); - bool empty() const; + bool is_in_range(const char *ip_str); //检查给定的 IP 地址字符串是否在当前范围集合内 + bool is_in_range(const IPV6 *ip); //检查给定的 IPv6 地址是否在当前范围集合内 + bool is_in_range(const uint32_t ipv4); //检查给定的 IPv4 地址是否在当前范围集合内 + bool is_intersect(const IPRange *arg); //检查当前范围集合与另一个 IP 范围集合是否有交集 + bool empty() const; //检查当前范围集合是否为空 - const std::string& get_err_str() { return m_err_str; } - std::string ip_to_str(const IPV6 *ip) const; - bool str_to_ip(const char* ip_str, IPV6 *ip); + const std::string& get_err_str() { return m_err_str; } //获取最近一次操作的错误信息 + std::string ip_to_str(const IPV6 *ip) const; //将给定的 IPv6 地址转换为字符串表示形式 + bool str_to_ip(const char* ip_str, IPV6 *ip); //将给定的 IP 地址字符串转换为 IPv6 地址 private: typedef std::vector Ranges_t; - Ranges_t m_ranges; - std::string m_err_str; + Ranges_t m_ranges; //存储 IP 范围的向量 + std::string m_err_str; //存储最近一次操作的错误信息 + //各种形式的 IP 地址字符串转换为 IP 结构体 bool parse_range(const char* range, size_t range_len, Range *new_range); bool parse_slash(const char* range, size_t range_len, const char *ptr, Range *new_range); bool parse_hyphen(const char* range, size_t range_len, const char *ptr, Range *new_range); bool parse_mask(const char* range, size_t range_len, const char *ptr, Range *new_range); bool parse_single(const char* range, size_t range_len, Range *new_range); + //二分查找算法在范围集合中查找给定的 IPv6 地址 bool binary_search(const IPV6 ip) const; - bool mask_range(Range *range, unsigned short cidr); - void handle_remove_intersection(Ranges_t *new_ranges, const Range *remove_range, Range *exist_range); - bool handle_add_intersection(Range *new_range, const Range *exist_range); - void copy_without_spaces(char buf[], size_t buf_len, const char *original, size_t original_len) const; - void net_ipv6_to_host_order(IPV6 *ip, const struct sockaddr_in6 *sa) const; - void net_ipv4_to_host_order(IPV6 *ip, const struct in_addr *addr) const; + bool mask_range(Range *range, unsigned short cidr); //根据给定的 CIDR 前缀长度对 IP 范围进行掩码处理，并更新输入的起始和结束 IP 地址值 + void handle_remove_intersection(Ranges_t *new_ranges, const Range *remove_range, Range *exist_range); //处理移除范围时的交叉情况 + bool handle_add_intersection(Range *new_range, const Range *exist_range); //处理添加范围时的交叉情况 + void copy_without_spaces(char buf[], size_t buf_len, const char *original, size_t original_len) const; //将原始字符串复制到缓冲区中，同时去除空格字符 + void net_ipv6_to_host_order(IPV6 *ip, const struct sockaddr_in6 *sa) const; //将网络字节序（大端序）的 IPv6 地址转换为主机字节序 + void net_ipv4_to_host_order(IPV6 *ip, const struct in_addr *addr) const; //将网络字节序（大端序）的 IPv4 地址转换为主机字节序 }; #endif // IPRANGE_AUDIT_H_ -- 2.34.1 From 595bb69c5d749b77782d042e0ff05163f0a0a26c Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Sat, 30 Sep 2023 00:44:20 +0800 Subject: [PATCH 10/48] Update ledger_check.cpp --- .../security/gs_ledger/ledger_check.cpp | 200 +++++++++++------- 1 file changed, 120 insertions(+), 80 deletions(-) diff --git a/src/gausskernel/security/gs_ledger/ledger_check.cpp b/src/gausskernel/security/gs_ledger/ledger_check.cpp index 1c7d07047..7ffc7987a 100644 --- a/src/gausskernel/security/gs_ledger/ledger_check.cpp +++ b/src/gausskernel/security/gs_ledger/ledger_check.cpp @@ -51,7 +51,7 @@ * * rel: user relation//用户关系 */ -// 计算用户表的哈希总和 +// 计算用户表的哈希总和（只计算给定关系的哈希总和） static uint64 gen_usertable_hash_sum(Relation rel) { uint64 rel_hash = 0;//初始化表的哈希总和为0 @@ -94,7 +94,7 @@ static uint64 gen_usertable_hash_sum(Relation rel) * * NOTICE: if user table is partition table, just use the original relation id.//注意：如果用户表是分区表，则只使用原始关系 ID */ -//计算指定关系（表）的哈希总和 +//计算指定关系（表）的哈希总和（可以处理分区表，并计算整个分区表的哈希总和） static uint64 get_usertable_hash_sum(Oid relid) { uint64 rel_hash = 0;//存储最终的哈希总和 @@ -143,7 +143,7 @@ static uint64 get_histtable_hash_sum(Oid hist_oid) if (!is_null) { rel_hash += DatumGetUInt64(value);//如果列值不为NULL，则将其加到哈希总和中 } - //从元组中获取 hash_del 列的值 + //从元组中获取hash_del列的值 value = heap_getattr(tuple, USERCHAIN_COLUMN_HASH_DEL + 1, hist_rel->rd_att, &is_null); if (!is_null) { rel_hash -= DatumGetUInt64(value);//如果列值不为NULL，则将其从哈希总和中减去 @@ -272,13 +272,14 @@ static void StrategyFuncUInt64Sum(ParallelFunctionState* state) #endif /* - * ledger_hist_check -- check whether user table hash and history table hash are equal + * ledger_hist_check -- check whether user table hash and history table hash are equal//检查用户表的哈希总和和历史表的哈希总和是否相等 * - * parameter1: user table name [type: text] - * parameter2: namespace of user table [type: text] + * parameter1: user table name [type: text]//用户表名称 + * parameter2: namespace of user table [type: text]//用户表的命名空间 */ Datum ledger_hist_check(PG_FUNCTION_ARGS) { + //定义变量 Oid relid; Oid nsp_oid; uint64 res_hash; @@ -303,6 +304,7 @@ Datum ledger_hist_check(PG_FUNCTION_ARGS) if (!IsConnFromCoord()) {//非协调节点，则进行并行计算 StringInfoData buf; ParallelFunctionState* state = NULL; + //初始化字符串缓冲区 initStringInfo(&buf); appendStringInfo(&buf, "SELECT pg_catalog.ledger_hist_check('%s', '%s')", table_nsp, table_name); /* Get all hash diffs from DNs in distribute scenairo. */ @@ -315,94 +317,97 @@ Datum ledger_hist_check(PG_FUNCTION_ARGS) } /* - * get_gchain_relhash_sum -- calculate relhash from gs_global_chain + * get_gchain_relhash_sum -- calculate relhash from gs_global_chain//从gs_global_chain计算relid对应的用户表哈希总和 * - * relid: user table oid + * relid: user table oid//用户表的对象标识符 */ static uint64 get_gchain_relhash_sum(Oid relid) { - uint64 relhash = 0; - HeapTuple tuple = NULL; + uint64 relhash = 0;//用户表哈希总和的初始值 + HeapTuple tuple = NULL;//堆元组指针，用于遍历gs_global_chain目录 /* scan the gs_global_chain catalog by relid */ + //打开gs_global_chain目录 Relation gchain_rel = heap_open(GsGlobalChainRelationId, AccessShareLock); - Form_gs_global_chain rdata = NULL; + Form_gs_global_chain rdata = NULL;//gs_global_chain目录中的元组数据结构体指针 + //扫描gs_global_chain目录 TableScanDesc scan = heap_beginscan(gchain_rel, SnapshotNow, 0, NULL); while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { - rdata = (Form_gs_global_chain)GETSTRUCT(tuple); + rdata = (Form_gs_global_chain)GETSTRUCT(tuple);//获取当前遍历的元组数据 if (rdata == NULL || rdata->relid != relid) { - continue; + continue;//若元组数据为空或relid不匹配，则跳过继续下一次遍历 } - relhash += rdata->relhash; + relhash += rdata->relhash;//累加当前匹配的relid的哈希值到用户表哈希总和 } - heap_endscan(scan); - heap_close(gchain_rel, AccessShareLock); - return relhash; + heap_endscan(scan);//扫描结束 + heap_close(gchain_rel, AccessShareLock);//关闭gs_global_chain目录 + return relhash;//返回用户表哈希总和 } /* - * get_dn_hist_relhash -- calculate relhash from gs_global_chain or history table + * get_dn_hist_relhash -- calculate relhash from gs_global_chain or history table//从gs_global_chain或history表中计算relhash * - * parameter1: user table name [type: text] - * parameter2: namespace of user table [type: text] + * parameter1: user table name [type: text]//用户表名称 + * parameter2: namespace of user table [type: text]//用户表的命名空间 * * NOTICE: if current node is cn, it's return the relhash calculated from gs_global_chain, * if current node is dn, it's return the relhash calculated from history table. + * 注意：如果当前节点是cn，则返回从gs_global_chain计算得到的relhash；如果当前节点是dn，则返回从history表计算得到的relhash。 */ Datum get_dn_hist_relhash(PG_FUNCTION_ARGS) { -#ifndef ENABLE_MULTIPLE_NODES - DISTRIBUTED_FEATURE_NOT_SUPPORTED(); - return UInt64GetDatum(0); +#ifndef ENABLE_MULTIPLE_NODES//如果不支持多节点部署，那么调用该函数将会立即返回一个哈希总和为0的结果，而不会继续执行后续的代码逻辑 + DISTRIBUTED_FEATURE_NOT_SUPPORTED();//宏，用于抛出一个错误，表示不支持分布式特性 + return UInt64GetDatum(0);//返回一个表示哈希总和为0的结果 #else - if (!IsConnFromCoord()) { + if (!IsConnFromCoord()) {//如果当前连接不是协调器连接 ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); return UInt64GetDatum(0); } Oid user_relid; Oid nsp_oid; uint64 res_hash; - text *rel_nsp = PG_GETARG_TEXT_PP(0); - text *rel_name = PG_GETARG_TEXT_PP(1); + text *rel_nsp = PG_GETARG_TEXT_PP(0);//获取命名空间名称 + text *rel_name = PG_GETARG_TEXT_PP(1);//获取表名 char *table_name; char *table_nsp; - table_nsp = text_to_cstring(rel_nsp); - table_name = text_to_cstring(rel_name); - nsp_oid = get_namespace_oid(table_nsp, false); - user_relid = get_relname_relid(table_name, nsp_oid); - ledger_usertable_check(user_relid, nsp_oid, table_name, table_nsp); + table_nsp = text_to_cstring(rel_nsp);//将命名空间转换为C字符串 + table_name = text_to_cstring(rel_name);//将表名转换为C字符串 + nsp_oid = get_namespace_oid(table_nsp, false);//获取命名空间的Oid + user_relid = get_relname_relid(table_name, nsp_oid);//获取用户表的Oid + ledger_usertable_check(user_relid, nsp_oid, table_name, table_nsp);//检查用户表的合法性 - if (!has_ledger_consistent_privilege(user_relid, nsp_oid)) { + if (!has_ledger_consistent_privilege(user_relid, nsp_oid)) {//没有一致性账本特权 ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); } - if (IS_PGXC_DATANODE) { - if (!is_hist_hash_identity(user_relid, &res_hash)) { + if (IS_PGXC_DATANODE) {//如果当前节点是数据节点 + if (!is_hist_hash_identity(user_relid, &res_hash)) {//判断用户表是否具有历史哈希标识 res_hash = 0; } - } else { - res_hash = get_gchain_relhash_sum(user_relid); + } else {//如果当前节点是协调器节点 + res_hash = get_gchain_relhash_sum(user_relid);//从gs_global_chain目录中计算用户表的哈希总和 } - return UInt64GetDatum(res_hash); + return UInt64GetDatum(res_hash);//返回res_hash的值 #endif } /* - * ledger_gchain_check -- calculate relation hash from all cn and all dn, and return cn_hash == dn_hash + * ledger_gchain_check -- calculate relation hash from all cn and all dn, and return cn_hash == dn_hash//从所有的协调器节点和所有的数据节点计算关系哈希，并返回协调器节点的哈希值是否等于数据节点的哈希值 * - * parameter1: user table name [type: text] - * parameter2: namespace of user table [type: text] + * parameter1: user table name [type: text]//用户表名 + * parameter2: namespace of user table [type: text]//用户表所属命名空间 */ Datum ledger_gchain_check(PG_FUNCTION_ARGS) { #ifdef ENABLE_MULTIPLE_NODES - if (!IS_PGXC_COORDINATOR || IsConnFromCoord()) { - ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); + if (!IS_PGXC_COORDINATOR || IsConnFromCoord()) {//如果不是协调器或者是协调器连接 + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));//抛出权限不足的错误 } #endif - text *rel_nsp = PG_GETARG_TEXT_PP(0); - text *rel_name = PG_GETARG_TEXT_PP(1); + text *rel_nsp = PG_GETARG_TEXT_PP(0);//获取命名空间名称 + text *rel_name = PG_GETARG_TEXT_PP(1);//获取表名 char *table_name; char *table_nsp; Oid user_relid; @@ -411,80 +416,94 @@ Datum ledger_gchain_check(PG_FUNCTION_ARGS) uint64 dn_hash; bool res; - table_nsp = text_to_cstring(rel_nsp); - table_name = text_to_cstring(rel_name); - nsp_oid = get_namespace_oid(table_nsp, false); - user_relid = get_relname_relid(table_name, nsp_oid); - ledger_usertable_check(user_relid, nsp_oid, table_name, table_nsp); + table_nsp = text_to_cstring(rel_nsp);//将命名空间转换为C字符串 + table_name = text_to_cstring(rel_name);//将表名转换为C字符串 + nsp_oid = get_namespace_oid(table_nsp, false);//获取命名空间的Oid + user_relid = get_relname_relid(table_name, nsp_oid);//获取用户表的Oid + ledger_usertable_check(user_relid, nsp_oid, table_name, table_nsp);//检查用户表的合法性 - if (!has_ledger_consistent_privilege(user_relid, nsp_oid)) { - ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); + if (!has_ledger_consistent_privilege(user_relid, nsp_oid)) {//没有一致性账本特权 + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));//抛出权限不足的错误 } - res = is_hist_hash_identity(user_relid, &dn_hash); + res = is_hist_hash_identity(user_relid, &dn_hash);//判断用户表是否具有历史哈希标识 if (!res) { - return BoolGetDatum(res); + return BoolGetDatum(res);//如果没有历史哈希标识，返回false } - cn_hash = get_gchain_relhash_sum(user_relid); + cn_hash = get_gchain_relhash_sum(user_relid);//从gs_global_chain目录中计算用户表的哈希总和 #ifdef ENABLE_MULTIPLE_NODES ParallelFunctionState* state = NULL; StringInfoData buf; initStringInfo(&buf); + //将格式化后的字符串 "SELECT pg_catalog.get_dn_hist_relhash('%s', '%s')" 和 table_nsp、table_name 的值追加到 buf 中 appendStringInfo(&buf, "SELECT pg_catalog.get_dn_hist_relhash('%s', '%s')", table_nsp, table_name); /* Get all hash diffs from DNs in distribute scenairo. */ + /* 在分布式场景中，从所有的数据节点获取哈希差异。 */ + // 调用RemoteFunctionResultHandler函数，传入buf.data（SQL查询语句）、NULL（没有附加参数）和StrategyFuncUInt64Sum（累加函数）来获取数据节点的哈希差异，并将结果赋给state->result state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncUInt64Sum); - dn_hash += state->result; - FreeParallelFunctionState(state); - if (GetAllCoordNodes() != NIL) { + dn_hash += state->result;//将state->result（数据节点的哈希差异）累加到数据节点的哈希值变量dn_hash + FreeParallelFunctionState(state);//释放内存，回收state占用的资源 + if (GetAllCoordNodes() != NIL) {//如果存在协调器节点，则执行下面的代码块 /* Get and accumulate all cnhash from all CNs. */ + /* 获取并累加所有协调器节点的哈希值。 */ state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncUInt64Sum, true, EXEC_ON_COORDS); - cn_hash += state->result; + cn_hash += state->result;//将state->result（协调器节点的哈希值）累加到协调器节点的哈希值变量cn_hash中 FreeParallelFunctionState(state); } #endif - return BoolGetDatum(dn_hash == cn_hash); + return BoolGetDatum(dn_hash == cn_hash);//返回协调器节点的哈希值是否等于数据节点的哈希值 } /* - * repaire_hist_table_internal -- compare hash and repair hist table + * repaire_hist_table_internal -- compare hash and repair hist table//比较哈希值并修复历史表 * - * relid: user table oid - * rel_name: user table name - * nspoid: user table namespace oid - * option: choose return tuple_hash_sum if true, or hash_diff if false. + * relid: user table oid//用户表的对象标识符 + * rel_name: user table name//用户表的名称 + * nspoid: user table namespace oid//用户表的命名空间对象标识符 + * option: choose return tuple_hash_sum if true, or hash_diff if false.//如果为true，则返回tuple_hash_sum；如果为false，则返回hash_diff * - * Note: the diff means tuple_hash_sum - hist_hash_sum. + * Note: the diff means tuple_hash_sum - hist_hash_sum.//注意：diff表示tuple_hash_sum - hist_hash_sum */ static uint64 repaire_hist_table_internal(Oid relid, char *rel_name, Oid nspoid, bool option) { + //声明并初始化变量 uint64 rel_hash; uint64 hash_diff; char histname[NAMEDATALEN]; + //获取历史表的名称 get_hist_name(relid, rel_name, histname, nspoid); + //获取历史表的对象标识符（OID） Oid histoid = get_relname_relid(histname, PG_BLOCKCHAIN_NAMESPACE); if (!OidIsValid(histoid)) { + //如果历史表的OID无效，则报错 ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("The hist table of \"%s\" is not exist.", rel_name))); } + //获取用户表的哈希值 rel_hash = get_usertable_hash_sum(relid); + //计算哈希差异（diff = tuple_hash_sum - hist_hash_sum） hash_diff = rel_hash - get_histtable_hash_sum(histoid); if (hash_diff != 0) { /* Do hist table repair. */ + /* 进行历史表修复 */ hist_table_record_internal(histoid, &hash_diff, NULL); } + //根据选项返回结果：如果option为true，返回用户表的哈希值；如果option为false，返回哈希差异。 return option ? rel_hash : hash_diff; } /* - * ledger_hist_repair -- repair the history table of specified user table + * ledger_hist_repair -- repair the history table of specified user table//修复指定用户表的历史表 * - * parameter1: user table name [type: text] - * parameter2: namespace of user table [type: text] + * parameter1: user table name [type: text]//用户表名称 + * parameter2: namespace of user table [type: text]//用户表所在的命名空间 */ Datum ledger_hist_repair(PG_FUNCTION_ARGS) { + //检查当前用户是否具有足够的权限 if (!isRelSuperuser() && !isAuditadmin(GetUserId())) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); } + //获取函数参数：用户表所在的命名空间和名称 text *rel_nsp = PG_GETARG_TEXT_PP(0); text *rel_name = PG_GETARG_TEXT_PP(1); char *table_name; @@ -492,26 +511,31 @@ Datum ledger_hist_repair(PG_FUNCTION_ARGS) Oid relid; Oid nspoid; uint64 delta = 0; - + //将参数转换为C字符串 table_nsp = text_to_cstring(rel_nsp); table_name = text_to_cstring(rel_name); + //获取命名空间的 OID nspoid = get_namespace_oid(table_nsp, false); + //获取用户表的对象标识符（OID） relid = get_relname_relid(table_name, nspoid); + //检查用户表是否存在，并且当前用户是否具有足够的权限 ledger_usertable_check(relid, nspoid, table_name, table_nsp); /* * Repair hist table of current datanode. Get hash sum of hist - * table and rel_hash of usertable, append the difference to hist table. + * table and rel_hash of usertable, append the difference to hist table.//修复当前数据节点的历史表。获取历史表的哈希值和用户表的哈希值，将差异追加到历史表中。 */ + //当前节点的角色是数据节点或者单点模式节点 if (g_instance.role == VDATANODE || g_instance.role == VSINGLENODE) { delta = repaire_hist_table_internal(relid, table_name, nspoid, false); } - + //如果当前节点是协调器或单点模式 if (g_instance.role == VCOORDINATOR || g_instance.role == VSINGLENODE) { #ifdef ENABLE_MULTIPLE_NODES ParallelFunctionState* state = NULL; StringInfoData buf; initStringInfo(&buf); + //生成并执行用于获取历史表差异的查询语句 appendStringInfo(&buf, "SELECT pg_catalog.ledger_hist_repair('%s', '%s')", table_nsp, table_name); /* Get all hash diffs from DNs in distribute scenairo. */ state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncUInt64Sum); @@ -519,25 +543,28 @@ Datum ledger_hist_repair(PG_FUNCTION_ARGS) FreeParallelFunctionState(state); #endif + //如果差异不为零，则将修复后的差异追加到全局链上 if (delta != 0) { ledger_gchain_append(relid, "HIST REPAIR.", delta); } } - + //返回修复的差异值 return UInt64GetDatum(delta); } /* - * ledger_gchain_repair -- repair gs_global_chain of specified user table + * ledger_gchain_repair -- repair gs_global_chain of specified user table//修复指定用户表的gs_global_chain * - * parameter1: user table name [type: text] - * parameter2: namespace of user table [type: text] + * parameter1: user table name [type: text]//用户表名称 + * parameter2: namespace of user table [type: text]//用户表所在的命名空间 */ Datum ledger_gchain_repair(PG_FUNCTION_ARGS) { + //检查当前用户是否具有足够的权限 if (!isRelSuperuser() && !isAuditadmin(GetUserId())) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); } + //获取函数参数：用户表所在的命名空间和名称 text *rel_nsp = PG_GETARG_TEXT_PP(0); text *rel_name = PG_GETARG_TEXT_PP(1); char *table_name; @@ -545,27 +572,35 @@ Datum ledger_gchain_repair(PG_FUNCTION_ARGS) Oid relid; Oid nspoid; uint64 dn_hash = 0; - + //将参数转换为C字符串 table_nsp = text_to_cstring(rel_nsp); table_name = text_to_cstring(rel_name); + //获取命名空间的 OID nspoid = get_namespace_oid(table_nsp, false); + //获取用户表的对象标识符（OID） relid = get_relname_relid(table_name, nspoid); + //检查用户表是否存在，并且当前用户是否具有足够的权限 ledger_usertable_check(relid, nspoid, table_name, table_nsp); /* * Repair hist table of current datanode. Get hash sum of hist * table and rel_hash of usertable, append the difference to hist table. */ + /* + * 修复当前数据节点的历史表。获取历史表的哈希值和用户表的哈希值，将差异追加到历史表中。 + */ + //如果当前节点的角色是数据节点或者单点模式节点，则执行修复历史表的操作，并将修复后的历史表哈希值赋给变量dn_hash if (g_instance.role == VDATANODE || g_instance.role == VSINGLENODE) { dn_hash = repaire_hist_table_internal(relid, table_name, nspoid, true); } uint64 rel_hash = dn_hash; uint64 cn_hash = 0; + //如果当前节点是协调器或单点模式 if (g_instance.role == VCOORDINATOR || g_instance.role == VSINGLENODE) { uint64 delta = 0; - cn_hash = get_gchain_relhash_sum(relid); - if (!IsConnFromCoord()) { + cn_hash = get_gchain_relhash_sum(relid);//获取当前节点的全局链哈希值并将结果保存在变量 cn_hash 中 + if (!IsConnFromCoord()) {//判断当前连接是否来自协调器节点 #ifdef ENABLE_MULTIPLE_NODES /* * CN accumulate all gchain cn_hash from all CNs, get all dn_hash from all DNs. @@ -574,25 +609,30 @@ Datum ledger_gchain_repair(PG_FUNCTION_ARGS) ParallelFunctionState* state = NULL; StringInfoData buf; initStringInfo(&buf); + //生成并执行用于获取历史表差异的查询语句 appendStringInfo(&buf, "SELECT pg_catalog.ledger_gchain_repair('%s', '%s')", table_nsp, table_name); /* Get and accumulate all dn_hash from all DNs. */ + //获取并累加所有数据节点中的历史表哈希值 state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncUInt64Sum); dn_hash += state->result; FreeParallelFunctionState(state); - if (GetAllCoordNodes() != NIL) { + if (GetAllCoordNodes() != NIL) {//如果存在其他的协调器节点 /* Get and accumulate all cn_hash from all CNs. */ + //获取并累加所有协调器节点中的全局链哈希值 state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncUInt64Sum, true, EXEC_ON_COORDS); cn_hash += state->result; FreeParallelFunctionState(state); } #endif + //计算差异值 delta = dn_hash - cn_hash; + //如果差异不为零，则将修复后的差异追加到全局链上 if (delta != 0) { ledger_gchain_append(relid, "GCHAIN REPAIR.", delta); } } rel_hash = cn_hash; } - + //返回修复后的用户表哈希值 return UInt64GetDatum(rel_hash); } \ No newline at end of file -- 2.34.1 From 86f1c843dc2adb0fd6dc6f85523562f90c53bc37 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Sat, 30 Sep 2023 00:44:55 +0800 Subject: [PATCH 11/48] Update ledger_utils.cpp --- .../security/gs_ledger/ledger_utils.cpp | 422 ++++++++++-------- 1 file changed, 247 insertions(+), 175 deletions(-) diff --git a/src/gausskernel/security/gs_ledger/ledger_utils.cpp b/src/gausskernel/security/gs_ledger/ledger_utils.cpp index b3e1bb548..8cdbce77e 100644 --- a/src/gausskernel/security/gs_ledger/ledger_utils.cpp +++ b/src/gausskernel/security/gs_ledger/ledger_utils.cpp @@ -25,129 +25,143 @@ #include "gs_ledger/ledger_utils.h" #include "catalog/gs_global_chain.h" -static pg_atomic_uint64 g_blocknum = 0; -static HTAB *g_recnum_cache = NULL; +static pg_atomic_uint64 g_blocknum = 0;//记录区块的编号 +static HTAB *g_recnum_cache = NULL;//指向哈希表的指针，用于缓存记录编号 /* - * reload_next_g_blocknum -- load next blocknum from gchain. + * reload_next_g_blocknum -- load next blocknum from gchain.//加载下一个区块编号（blocknum）的函数 * - * Note:If gchain is empty, next blocknum will start from 0. + * Note:If gchain is empty, next blocknum will start from 0.//如果全局链（gchain）为空，则下一个区块编号将从0开始 */ static uint32 reload_next_g_blocknum() { - Relation gchain_rel = NULL; - HeapTuple tup = NULL; - TableScanDesc scan; - uint32 blocknum; - uint32 max_num = 0; - bool isnull = false; + Relation gchain_rel = NULL;//全局链关系指针 + HeapTuple tup = NULL;//堆元组指针，用于遍历全局链关系中的元组 + TableScanDesc scan;//表扫描描述符 + uint32 blocknum;//当前堆元组表示的区块编号 + uint32 max_num = 0;//最大的区块编号 + bool isnull = false;//是否为空 - gchain_rel = heap_open(GsGlobalChainRelationId, RowExclusiveLock); - scan = heap_beginscan(gchain_rel, SnapshotAny, 0, NULL); - while ((tup = heap_getnext(scan, BackwardScanDirection)) != NULL) { + gchain_rel = heap_open(GsGlobalChainRelationId, RowExclusiveLock);//打开全局链关系 + scan = heap_beginscan(gchain_rel, SnapshotAny, 0, NULL);//开始对全局链关系进行表扫描 + while ((tup = heap_getnext(scan, BackwardScanDirection)) != NULL) {//循环遍历全局链关系中的堆元组 blocknum = DatumGetUInt32(heap_getattr(tup, Anum_gs_global_chain_blocknum, - RelationGetDescr(gchain_rel), &isnull)); - if (blocknum > max_num) { - max_num = blocknum; + RelationGetDescr(gchain_rel), &isnull));//获取堆元组中的区块编号属性值 + if (blocknum > max_num) {//如果当前区块编号大于最大区块编号 + max_num = blocknum;//更新最大区块编号 } else { - break; + break;//否则跳出循环 } } - heap_endscan(scan); - heap_close(gchain_rel, RowExclusiveLock); - return max_num; + heap_endscan(scan); //结束表扫描 + heap_close(gchain_rel, RowExclusiveLock);//关闭全局链关系 + return max_num;//返回最大的区块编号 } /* - * get_next_g_blocknum -- get next blocknum for gchain record. + * get_next_g_blocknum -- get next blocknum for gchain record.//获取全局链记录的下一个区块编号 * - * Note:provide next blocknum and auto increment itself. + * Note:provide next blocknum and auto increment itself.//获取下一个区块编号，并自动递增 */ uint64 get_next_g_blocknum() { - uint64 res = 0; - if (g_blocknum == 0) { - LWLockAcquire(GlobalPrevHashLock, LW_EXCLUSIVE); - if (g_blocknum == 0) { - pg_atomic_fetch_add_u64(&g_blocknum, 1); - int cur_num = reload_next_g_blocknum(); - pg_atomic_fetch_add_u64(&g_blocknum, cur_num); + uint64 res = 0;//存储结果的变量 + if (g_blocknum == 0) {//如果当前区块编号为0 + LWLockAcquire(GlobalPrevHashLock, LW_EXCLUSIVE);//获取全局前一个哈希锁（独占模式） + if (g_blocknum == 0) {//双重检查，确保其他线程未更新区块编号 + pg_atomic_fetch_add_u64(&g_blocknum, 1);//原子操作：将区块编号加1 + int cur_num = reload_next_g_blocknum();//获取当前最大的区块编号 + pg_atomic_fetch_add_u64(&g_blocknum, cur_num);//原子操作：将当前最大区块编号加到全局区块编号上 } - LWLockRelease(GlobalPrevHashLock); + LWLockRelease(GlobalPrevHashLock);//释放全局前一个哈希锁 } - LWLockAcquire(GlobalPrevHashLock, LW_SHARED); - res = pg_atomic_fetch_add_u64(&g_blocknum, 1); - LWLockRelease(GlobalPrevHashLock); - return res; + LWLockAcquire(GlobalPrevHashLock, LW_SHARED);//获取全局前一个哈希锁（共享模式） + res = pg_atomic_fetch_add_u64(&g_blocknum, 1);//原子操作：将区块编号加1 + LWLockRelease(GlobalPrevHashLock);//释放全局前一个哈希锁 + return res;//返回获取的区块编号 } - +//将全局区块编号g_blocknum重置为0 void reset_g_blocknum() { g_blocknum = 0; } /* - * reload_g_rec_num -- load next rec_num from hist table. + * reload_g_rec_num -- load next rec_num from hist table.//从hist表中加载下一个rec_num * - * histoid: hist table oid. + * histoid: hist table oid.//hist表的对象标识符 * - * Note:return next rec_num and auto increment. + * Note:return next rec_num and auto increment.//返回下一个rec_num的值和自增量 */ uint64 reload_g_rec_num(Oid histoid) { + //检查histoid的有效性 if (!OidIsValid(histoid)) { return false; } - Relation histRelation = NULL; - HeapTuple tup = NULL; - TableScanDesc scan; - uint64 max_rec_num = 0; - uint64 rec_num; - bool hist_empty = true; - bool isnull = false; - bool found; - + Relation histRelation = NULL;//hist表的关系对象 + HeapTuple tup = NULL;//堆元组 + TableScanDesc scan;//表扫描描述符 + uint64 max_rec_num = 0;//最大的rec_num值 + uint64 rec_num;//当前的rec_num值 + bool hist_empty = true;//hist表是否为空 + bool isnull = false;//判断属性是否为NULL + bool found;//是否在哈希表中找到项 + //打开hist表 histRelation = heap_open(histoid, AccessShareLock); + //开始扫描hist表 scan = heap_beginscan(histRelation, SnapshotNow, 0, NULL); while ((tup = heap_getnext(scan, BackwardScanDirection)) != NULL) { + //获取堆元组中的rec_num属性值 rec_num = DatumGetUInt64(heap_getattr(tup, 1, RelationGetDescr(histRelation), &isnull)); + //如果当前rec_num大于等于最大的rec_num，则更新最大的rec_num值 if (rec_num >= max_rec_num) { max_rec_num = rec_num; hist_empty = false; } } + //结束扫描hist表 heap_endscan(scan); + //关闭hist表 heap_close(histRelation, AccessShareLock); + //在g_recnum_cache哈希表中查找或插入histoid对应的RecNumItem RecNumItem *item = (RecNumItem *)hash_search(g_recnum_cache, &histoid, HASH_ENTER, &found); + //如果hist表为空，则rec_num设为0；否则，设置为最大rec_num+1 if (hist_empty) { rec_num = 0; } else { rec_num = max_rec_num + 1; } + //更新哈希表中rec_num的值，并返回rec_num item->rec_num = rec_num + 1; return rec_num; } /* - * get_next_recnum -- provide next rec_num. + * get_next_recnum -- provide next rec_num.//提供下一个rec_num值 * * histoid: hist table oid. */ uint64 get_next_recnum(Oid histoid) { + //检查全局recnum缓存g_recnum_cache是否已创建，如果未创建，则进行初始化 if (g_recnum_cache == NULL) { errno_t rc; HASHCTL ctl; LWLockAcquire(BlockchainVersionLock, LW_EXCLUSIVE); /* To avoid multiple threads double create hash table. */ + //双重检查锁机制：再次检查g_recnum_cache是否为NULL if (g_recnum_cache == NULL) { + //初始化哈希表的配置项 rc = memset_s(&ctl, sizeof(ctl), 0, sizeof(ctl)); securec_check(rc, "\0", "\0"); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(RecNumItem); ctl.hash = oid_hash; ctl.hcxt = INSTANCE_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY); + //创建哈希表g_recnum_cache g_recnum_cache = hash_create("global recnum cache", 256, &ctl, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); + //如果创建失败，则报错并抛出异常 if (g_recnum_cache == NULL) { ereport(ERROR, (errcode(ERRCODE_INITIALIZE_FAILED), errmsg("could not init global recnum cache"))); } @@ -158,27 +172,32 @@ uint64 get_next_recnum(Oid histoid) bool found = false; LWLockAcquire(BlockchainVersionLock, LW_SHARED); uint64 res; + //在g_recnum_cache哈希表中查找histoid对应的RecNumItem RecNumItem *item = (RecNumItem *)hash_search(g_recnum_cache, &histoid, HASH_FIND, &found); + //如果在哈希表中找到了对应项，则将res设置为对应的rec_num值，并将rec_num自增1 if (found) { res = item->rec_num; pg_atomic_add_fetch_u64(&item->rec_num, 1); } else { + //如果在哈希表中未找到对应项，则需要重新加载hist表并获取下一个rec_num值 LWLockRelease(BlockchainVersionLock); LWLockAcquire(BlockchainVersionLock, LW_EXCLUSIVE); res = reload_g_rec_num(histoid); } LWLockRelease(BlockchainVersionLock); - return res; + return res;//返回获取到的rec_num值 } - +//移除指定histoid对应的缓存项 bool remove_hist_recnum_cache(Oid histoid) { + //检查histoid是否有效，以及全局recnum缓存g_recnum_cache是否已创建 if (!OidIsValid(histoid) || g_recnum_cache == NULL) { - return false; + return false;//如果 istoid无效或者缓存未创建，则返回false } + //在哈希表g_recnum_cache中查找histoid对应的项，并将其从哈希表中移除 hash_search(g_recnum_cache, &histoid, HASH_REMOVE, NULL); - return true; + return true;//返回true表示成功移除缓存项 } /* @@ -186,14 +205,18 @@ bool remove_hist_recnum_cache(Oid histoid) * * mode: lockmode */ +//使用指定的锁模式（mode）锁定g_blocknum缓存 void lock_gchain_cache(LWLockMode mode) { LWLockAcquire(GlobalPrevHashLock, mode); + //GlobalPrevHashLock:用于确保对g_blocknum缓存的访问互斥 + //mode:锁的模式。可以是LW_SHARED（共享锁，允许多个线程同时获取）或LW_EXCLUSIVE（排他锁，只允许一个线程获取） } /* * release_gchain_cache -- release g_blocknum cache. */ +//释放g_blocknum缓存的锁 void release_gchain_cache() { LWLockRelease(GlobalPrevHashLock); @@ -202,380 +225,429 @@ void release_gchain_cache() /* * lock_hist_hash_cache -- load hist cache. */ +//加载历史哈希缓存 void lock_hist_hash_cache(LWLockMode mode) { LWLockAcquire(BlockchainVersionLock, mode); + //BlockchainVersionLock:用于确保历史哈希缓存的访问互斥 } /* * release_hist_hash_cache -- release hist cache. */ +//释放历史哈希缓存的锁 void release_hist_hash_cache() { LWLockRelease(BlockchainVersionLock); } /* - * get_target_query_relid -- get result relation oid. + * get_target_query_relid -- get result relation oid.//获取目标关系的OID * - * rte_list: range table entry list. - * resultRelation: the index of target relation in rte_list. + * rte_list: range table entry list.//范围表条目列表 + * resultRelation: the index of target relation in rte_list.//目标关系在rte_list中的索引 */ Oid get_target_query_relid(List* rte_list, int resultRelation) { Oid relid = InvalidOid; - + //检查resultRelation是否大于0 if (resultRelation > 0) { + //从rte_list中获取目标关系的表达式对象 RangeTblEntry *rte = (RangeTblEntry *)list_nth(rte_list, resultRelation - 1); + //检查表达式对象的relkind是否为RELKIND_RELATION，即是否为普通关系（表） if (rte->relkind == RELKIND_RELATION) { + //如果是普通关系，则获取关系的OID relid = rte->relid; } } + //返回关系的OID return relid; } /* - * is_ledger_usertable -- whether relid is ledger user table. + * is_ledger_usertable -- whether relid is ledger user table.//判断给定的relid是否为账本用户表 * - * relid: usertable oid. + * relid: usertable oid.//用户表的OID * * Note: ledger usertable has correct hist table in blockchain schema. * Or rather, the schema of ledger usertable has WITH BLOCKCHAIN option. + * 注意：账本用户表在区块链模式下具有正确的历史表。 + * 或者更确切地说，账本用户表的模式中包含 WITH BLOCKCHAIN 选项。 */ bool is_ledger_usertable(Oid relid) { + //检查relid是否有效，如果无效则返回false if (!OidIsValid(relid)) { return false; } - + //获取关系所属的命名空间OID Oid nspid = get_rel_namespace(relid); + //获取关系的relkind，即关系类型 char relkind = get_rel_relkind(relid); /* only table has its user chain table */ + //只有表（RELKIND_RELATION）才有其用户链表 if (relkind != RELKIND_RELATION) { return false; } /* check table belong to blockchain schema */ + //检查表是否属于账本模式的命名空间 + return IsLedgerNameSpace(nspid); } /* - * hash_combiner -- combine all relhash list items. + * hash_combiner -- combine all relhash list items.//合并所有relhash列表项 * - * relhash_list: the list contains several relhash. + * relhash_list: the list contains several relhash.//包含多个relhash的列表 */ uint64 hash_combiner(List *relhash_list) { - uint64 relhash_sum = 0; - ListCell *lc = NULL; - if (relhash_list == NIL) { + uint64 relhash_sum = 0;//用于保存累加后的哈希值总和 + ListCell *lc = NULL;//遍历链表的指针 + if (relhash_list == NIL) {//如果传入的链表为空，则直接返回0 return relhash_sum; } - foreach (lc, relhash_list) { - Datum *value = (Datum *)lfirst(lc); - relhash_sum += DatumGetUInt64(value); + foreach (lc, relhash_list) {//遍历链表中的每个元素 + Datum *value = (Datum *)lfirst(lc);//获取当前元素的值 + relhash_sum += DatumGetUInt64(value);//将当前值累加到哈希值总和中 } - return relhash_sum; + return relhash_sum;//返回累加后的哈希值 } /* - * is_ledger_hist_table -- whether relation is ledger hist table. + * is_ledger_hist_table -- whether relation is ledger hist table.//判断关系是否为历史总账表 * - * relid: relation oid. + * relid: relation oid.//关系的对象标识符（OID） */ bool is_ledger_hist_table(Oid relid) { - if (!OidIsValid(relid) || IsInitdb) { + if (!OidIsValid(relid) || IsInitdb) {//判断传入的关系 OID 是否有效，以及是否处在Initdb阶段 return false; } - Oid relnsp = get_rel_namespace(relid); - char relkind = get_rel_relkind(relid); + Oid relnsp = get_rel_namespace(relid);//获取关系所在的命名空间OID + char relkind = get_rel_relkind(relid);//获取关系的类型 /* check namespace oid of relation to verify hist table. */ + /* 检查关系的命名空间 OID，并验证其是否为 PG_BLOCKCHAIN_NAMESPACE，同时检查关系的类型，并验证其是否为 + * RELKIND_RELATION。*/ return relnsp == PG_BLOCKCHAIN_NAMESPACE && relkind == RELKIND_RELATION; } /* - * is_ledger_related_rel -- whether relation is ledger related table. + * is_ledger_related_rel -- whether relation is ledger related table.//判断关系是否为与总账相关的表 * - * rel: table relation. + * rel: table relation.//表关系 * - * Note: ledger related rel means: ledger usertable or ledger hist table or gs_global_chain. + * Note: ledger related rel means: ledger usertable or ledger hist table or gs_global_chain.//注意：与总账相关的表指的是总账用户表、历史总账表或 gs_global_chain 表 */ bool is_ledger_related_rel(Relation rel) { - Oid nspoid = RelationGetNamespace(rel); - char relkind = RelationGetRelkind(rel); - bool is_ledger_usertable = rel->rd_isblockchain; - bool is_ledger_histtable = (nspoid == PG_BLOCKCHAIN_NAMESPACE && relkind == RELKIND_RELATION); - bool is_ledger_gchain = (RelationGetRelid(rel) == GsGlobalChainRelationId); - return is_ledger_usertable || is_ledger_histtable || is_ledger_gchain; + Oid nspoid = RelationGetNamespace(rel);//获取关系所在的命名空间OID + char relkind = RelationGetRelkind(rel);//获取关系的类型 + bool is_ledger_usertable = rel->rd_isblockchain;//判断关系是否为总账用户表 + bool is_ledger_histtable = (nspoid == PG_BLOCKCHAIN_NAMESPACE && relkind == RELKIND_RELATION);//判断关系是否为历史总账表 + bool is_ledger_gchain = (RelationGetRelid(rel) == GsGlobalChainRelationId);//判断关系是否为gs_global_chain表 + return is_ledger_usertable || is_ledger_histtable || + is_ledger_gchain; // 只要满足总账用户表、历史总账表或gs_global_chain表中的任意一种情况，就返回true } /* - * ledger_usertable_check -- check relation is ledger user table. + * ledger_usertable_check -- check relation is ledger user table. //检查关系是否为总账用户表 */ bool ledger_usertable_check(Oid relid, Oid nspoid, const char *tablename, const char *tablensp) { - if (!OidIsValid(relid)) { + if (!OidIsValid(relid)) {//如果关系 OID无效，则报错 ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("table %s.%s not exists.", tablensp, tablename))); } - if (!IsLedgerNameSpace(nspoid)) { + if (!IsLedgerNameSpace(nspoid)) {//如果关系所在的命名空间不是总账命名空间，则报错 ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("table %s.%s is not ledger user table.", tablensp, tablename))); } - return true; + return true;//否则返回true，表示该关系是总账用户表 } /* - * namespace_get_depended_relid -- get relid under given schema oid. + * namespace_get_depended_relid -- get relid under given schema oid.//获取给定模式OID下的关系OID * - * nspid: schema oid. + * nspid: schema oid.//模式OID */ List *namespace_get_depended_relid(Oid nspid) { - List *relid_list = NIL; + List *relid_list = NIL;//创建一个空的关系OID列表 Relation pg_class_rel = NULL; ScanKeyData skey[1]; SysScanDesc sysscan; HeapTuple tuple; Oid tupid = InvalidOid; + //初始化扫描键值，通过模式OID进行等值匹配查找 ScanKeyInit(&skey[0], Anum_pg_class_relnamespace, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(nspid)); - pg_class_rel = heap_open(RelationRelationId, AccessShareLock); - sysscan = systable_beginscan(pg_class_rel, ClassNameNspIndexId, true, SnapshotNow, 1, skey); - - while (HeapTupleIsValid(tuple = systable_getnext(sysscan))) { - Form_pg_class reltup = (Form_pg_class)GETSTRUCT(tuple); - if (reltup->relkind == RELKIND_RELATION) { + pg_class_rel = heap_open(RelationRelationId, AccessShareLock);//打开pg_class表 + sysscan = systable_beginscan(pg_class_rel, ClassNameNspIndexId, true, SnapshotNow, 1, skey);//开始扫描 + //循环遍历扫描结果集 + while (HeapTupleIsValid(tuple = systable_getnext(sysscan))) {//获取下一个系统表的元组并检查取得的元组是否有效 + Form_pg_class reltup = (Form_pg_class)GETSTRUCT(tuple);//转换为Form_pg_class结构的指针，以便访问元组中的字段 + if (reltup->relkind == RELKIND_RELATION) {//仅处理关系类型为RELKIND_RELATION的关系 tupid = HeapTupleGetOid(tuple); - relid_list = lappend_oid(relid_list, tupid); + relid_list = lappend_oid(relid_list, tupid);//将关系OID添加到列表中 } } - systable_endscan(sysscan); - heap_close(pg_class_rel, AccessShareLock); - return relid_list; + systable_endscan(sysscan);//结束扫描 + heap_close(pg_class_rel, AccessShareLock);//关闭pg_class表 + return relid_list;//返回关系OID列表 } /* - * get_ledger_msg_hash -- extract relhash from response message. + * get_ledger_msg_hash -- extract relhash from response message.//从响应消息中提取关系哈希值 * - * message: response message. - * hash: buffer to load extracted relhash. + * message: response message.//响应消息 + * hash: buffer to load extracted relhash.//用于存储提取的关系哈希值的缓冲区 */ bool get_ledger_msg_hash(char *message, uint64 *hash, int *msg_len) { - bool is_from_dml = false; - int hash_offset; + bool is_from_dml = false;//标记消息是否来自数据操作语言（DML） + int hash_offset;//提取哈希值的偏移量 static hash_offset_pair hash_offset_map[] = { - {"INSERT", 3}, /* INSERT 0 1 HASH */ - {"UPDATE", 2}, /* UPDATE 1 HASH */ - {"DELETE", 2} /* DELETE 1 HASH */ + {"INSERT", 3}, /* INSERT 0 1 HASH */ //插入操作的哈希值在第3个空格后面 + {"UPDATE", 2}, /* UPDATE 1 HASH */ //更新操作的哈希值在第2个空格后面 + {"DELETE", 2} /* DELETE 1 HASH */ //删除操作的哈希值在第2个空格后面 }; - if (message == NULL || strlen(message) == 0) { + if (message == NULL || strlen(message) == 0) {//如果消息为空或长度为0，则返回false return false; } /* match space times. */ + /* 匹配操作类型 */ for (size_t i = 0; i < lengthof(hash_offset_map); i++) { - if (strncmp(message, hash_offset_map[i].name, 6) == 0) { /* 6: string length of INSERT/UPDATE/DELETE */ - hash_offset = hash_offset_map[i].hash_offset; - is_from_dml = true; + if (strncmp(message, hash_offset_map[i].name, 6) == 0) { /* 6: string length of INSERT/UPDATE/DELETE */ // 操作类型匹配成功 + hash_offset = hash_offset_map[i].hash_offset;//当前操作类型对应的提取哈希值的偏移量 + is_from_dml = true;//表示消息来自数据操作语言（DML） break; } } - if (is_from_dml) { + if (is_from_dml) {//如果消息来自DML操作 size_t pos; size_t len = strlen(message); for (pos = 0; pos < len && hash_offset > 0; ++pos) { - if (message[pos] == ' ') { + if (message[pos] == ' ') {//在消息中查找空格 --hash_offset; } } /* message contains Hash value */ - if (hash_offset == 0) { - size_t remain_len = len - pos; + /* 消息包含哈希值 */ + if (hash_offset == 0) {//找到了哈希值的位置 + size_t remain_len = len - pos;//哈希值后面的剩余长度 if (remain_len > 0) { - *hash = strtoul(message + pos, NULL, 10); /* 10: Decimal */ - message[pos - 1] = '\0'; /* remove appended hash string. */ - *msg_len = *msg_len - remain_len - 1; + *hash = strtoul(message + pos, NULL, 10); /* 10: Decimal */ //将哈希值转换为无符号长整型 + message[pos - 1] = '\0'; /* remove appended hash string. */ //移除追加的哈希字符串 + *msg_len = *msg_len - remain_len - 1;//更新消息长度 return true; } } } - return false; + return false;//没有提取到哈希值，返回false } /* - * get_hist_name -- get hist table name of ledger user table. + * get_hist_name -- get hist table name of ledger user table.//获取账本用户表的历史表名称 * - * relid: user table oid. - * rel_name: user table rel_name. - * hist_name: buffer to load hist table name. - * nsp_oid: schema oid of user table. - * nsp_name: schema name of user table. + * relid: user table oid.//用户表的OID + * rel_name: user table rel_name.//用户表的rel_name + * hist_name: buffer to load hist table name.//用于加载历史表名称的缓冲区 + * nsp_oid: schema oid of user table.//用户表的模式OID + * nsp_name: schema name of user table.//用户表的模式名称 * - * Note: relid is used for getting namespace. + * Note: relid is used for getting namespace.//注意：relid用于获取命名空间 * This function is also used for generate hist name when - * rename user table, then rel_name is newest user table name. - * Thus, we should not use relid to get user table name. + * rename user table, then rel_name is newest user table name.//当重命名用户表时，此函数也用于生成hist名称，因此，rel_name是最新的用户表名称。 + * Thus, we should not use relid to get user table name.//因此，我们不应使用relid来获取用户表名称。 */ bool get_hist_name(Oid relid, const char *rel_name, char *hist_name, Oid nsp_oid, const char *nsp_name) { errno_t rc; - if (!OidIsValid(relid) || rel_name == NULL) { + if (!OidIsValid(relid) || rel_name == NULL) {// 检查输入参数的合法性，如果reli无效或者rel_name为空，则直接返回false return false; } - nsp_oid = OidIsValid(nsp_oid) ? nsp_oid : get_rel_namespace(relid); - nsp_name = (nsp_name == NULL) ? get_namespace_name(nsp_oid) : nsp_name; - int part_hist_name_len = strlen(rel_name) + strlen(nsp_name) + 1; - if (part_hist_name_len + strlen("_hist") >= NAMEDATALEN) { + nsp_oid = OidIsValid(nsp_oid) ? nsp_oid : get_rel_namespace(relid);// 如果nsp_oid合法，则直接使用；否则通过get_rel_namespace函数获取用户表所属的模式OID + nsp_name = (nsp_name == NULL) ? get_namespace_name(nsp_oid) : nsp_name; //如果nsp_name为空，则通过get_namespace_name函数获取指定的模式名称；否则直接使用输入的nsp_name + int part_hist_name_len = strlen(rel_name) + strlen(nsp_name) + 1;//计算历史表名称的长度：长度为rel_name和nsp_name字符串长度之和再加上一个下划线的长度 + if (part_hist_name_len + strlen("_hist") >= NAMEDATALEN) {//根据历史表名称的长度判断是否超出NAMEDATALEN的上限，如果超出，则使用relid和nsp_oid拼接历史表名称； rc = snprintf_s(hist_name, NAMEDATALEN, NAMEDATALEN - 1, "%d_%d_hist", nsp_oid, relid); securec_check_ss(rc, "", ""); - } else { + } else {//否则使用nsp_name和rel_name拼接历史表名称，并在最后添加"_hist"字符串 rc = snprintf_s(hist_name, NAMEDATALEN, NAMEDATALEN - 1, "%s_%s_hist", nsp_name, rel_name); securec_check_ss(rc, "", ""); } - return true; + + return true;//返回true表示成功生成历史表名称 } /* - * querydesc_contains_ledger_usertable -- check querydesc result relation. + * querydesc_contains_ledger_usertable -- check querydesc result relation.//检查querydesc结果关系 * - * Note: check result relation of querydesc is ledger user table. + * Note: check result relation of querydesc is ledger user table.//检查querydesc结果关系是否为账本用户表 */ bool querydesc_contains_ledger_usertable(QueryDesc *query_desc) { + //检查输入参数的合法性，如果query_desc为空或者query_desc->estate为空，则直接返回false if (query_desc == NULL || query_desc->estate == NULL) { return false; } + //获取EState对象的指针 EState *estate = query_desc->estate; + //获取结果关系的数量 int relnum = estate->es_num_result_relations; + //如果结果关系的数量为0或者es_result_relations为空，则直接返回false if (relnum == 0 || estate->es_result_relations == NULL) { return false; } + //遍历所有结果关系 for (int i = 0; i < relnum; ++i) { + //如果结果关系的rd_isblockchain字段为true，表示该结果关系是账本用户表，返回true if (estate->es_result_relations[i].ri_RelationDesc->rd_isblockchain) { return true; } } + //遍历完所有结果关系后仍未找到账本用户表，返回false return false; } /* - * ledger_check_switch_schema -- check two schema has same blockchain option. + * ledger_check_switch_schema -- check two schema has same blockchain option.//检查两个模式是否具有相同的区块链选项 */ void ledger_check_switch_schema(Oid old_nsp, Oid new_nsp) { + //检查旧模式和新模式的区块链选项是否不一致 if (IsLedgerNameSpace(old_nsp) != IsLedgerNameSpace(new_nsp)) { + //如果不一致，则报错，不支持在账本模式和普通模式之间切换模式 ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Unsupport to switch schema of a table between ledger schema and normal schema."))); } } /* - * is_ledger_rowstore -- check withOpt of CreateStmt. + * is_ledger_rowstore -- check withOpt of CreateStmt.//检查CreateStmt中的withOpt * * defList: stmt->options. * * Note: we will check whether option contains orientation, we only support ORIENTATION ROW. + * 注意：我们会检查选项中是否包含方向（orientation），我们仅支持ORIENTATION ROW */ bool is_ledger_rowstore(List *defList) { ListCell *lc = NULL; /* Scan list to see if orientation was ROW store. */ + // 遍历defList列表，查看是否包含方向（orientation）为ROW存储。 foreach (lc, defList) { DefElem* def = (DefElem*)lfirst(lc); + //检查当前def是否是orientation选项，并且其值不是ORIENTATION_ROW。 if (pg_strcasecmp(def->defname, "orientation") == 0 && pg_strcasecmp(defGetString(def), ORIENTATION_ROW) != 0) { return false; } } + //如果没有找到不支持的方向选项，则返回true return true; } - +//在CreateStmt语句中检查是否指定了哈希存储相关的选项 bool is_ledger_hashbucketstore(List *defList) { ListCell *lc = NULL; /* Scan list to see if hashbucket store. */ + //遍历defList列表，查看是否包含哈希存储选项。 foreach (lc, defList) { DefElem* def = (DefElem*)lfirst(lc); + //检查当前def是否是bucketcnt选项，或者是hashbucket选项且其值为true。 if (pg_strcasecmp(def->defname, "bucketcnt") == 0 || (pg_strcasecmp(def->defname, "hashbucket") == 0 && defGetBoolean(def))) { return true; } } + //如果循环结束时没有找到哈希存储选项，则返回false。 return false; } /* - * check_ledger_attrs_support -- check attrs is ledger supported. + * check_ledger_attrs_support -- check attrs is ledger supported.//检查属性是否受到账本支持 * - * attrs: list of columns. + * attrs: list of columns.//列的列表 * * Note: because ledger tuple hash is generated by hash_any,so we * should check whether table column types is supported by hash calculate. + * 注意：由于账本元组哈希是由hash_any生成的，我们应该检查表列的类型是否适用于哈希计算。 */ void check_ledger_attrs_support(List *attrs) { + //检查传入的列属性列表是否为空 if (attrs == NIL) { return; } ListCell *lc = NULL; + //遍历属性列表中的每个列定义 foreach (lc, attrs) { + //获取当前列的定义 ColumnDef *colDef = (ColumnDef*)lfirst(lc); /* skip the hash column */ + /* 跳过哈希列 */ + //如果列名为"hash"，则跳过该列的检查，因为哈希列是由数据库自动生成的 if (strcmp(colDef->colname, "hash") == 0) { continue; } + //获取列的数据类型OID Oid typid = colDef->typname->typeOid; + //如果数据类型OID无效，尝试通过列类型名称获取类型元数据 if (!OidIsValid(typid)) { Type ctype = typenameType(NULL, colDef->typname, NULL); if (ctype != NULL) { + //将类型元数据转换为类型OID typid = typeTypeId(ctype); ReleaseSysCache(ctype); } } + //检查列的数据类型是否受支持 switch (typid) { - case INT8OID: - case INT1OID: - case INT2OID: - case OIDOID: - case INT4OID: - case BOOLOID: - case CHAROID: - case NAMEOID: - case INT2VECTOROID: - case CLOBOID: - case NVARCHAR2OID: - case VARCHAROID: - case TEXTOID: - case OIDVECTOROID: - case FLOAT4OID: - case FLOAT8OID: - case ABSTIMEOID: - case RELTIMEOID: - case CASHOID: - case BPCHAROID: - case RAWOID: - case BYTEAOID: - case BYTEAWITHOUTORDERCOLOID: - case BYTEAWITHOUTORDERWITHEQUALCOLOID: - case INTERVALOID: - case TIMEOID: - case TIMESTAMPOID: - case TIMESTAMPTZOID: - case DATEOID: - case TIMETZOID: - case SMALLDATETIMEOID: - case NUMERICOID: - case UUIDOID: + case INT8OID://8字节整数 + case INT1OID://1字节整数 + case INT2OID://2字节整数 + case OIDOID://对象标识符 + case INT4OID://4字节整数 + case BOOLOID://布尔类型 + case CHAROID://固定长度字符 + case NAMEOID://名称类型 + case INT2VECTOROID://整数数组 + case CLOBOID://可变长度字符 + case NVARCHAR2OID://可变长度Unicode字符 + case VARCHAROID://可变长度字符 + case TEXTOID://文本类型 + case OIDVECTOROID://对象标识符数组 + case FLOAT4OID://4字节浮点数 + case FLOAT8OID://8字节浮点数 + case ABSTIMEOID://绝对时间 + case RELTIMEOID://相对时间 + case CASHOID://货币类型 + case BPCHAROID://空格填充的固定长度字符 + case RAWOID://二进制数据 + case BYTEAOID://字节数组 + case BYTEAWITHOUTORDERCOLOID://无序字节数组 + case BYTEAWITHOUTORDERWITHEQUALCOLOID://带有相等操作符的无序字节数组 + case INTERVALOID://时间间隔 + case TIMEOID://时间 + case TIMESTAMPOID://时间戳 + case TIMESTAMPTZOID://带时区的时间戳 + case DATEOID://日期 + case TIMETZOID://带时区的时间 + case SMALLDATETIMEOID://日期和时间 + case NUMERICOID://任意精度数值 + case UUIDOID://通用唯一标识符 + //如果数据类型受支持，则继续下一列的检查 break; default: + //如果数据类型不受支持，则抛出错误异常 ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("Unsupport column type \"%s\" of ledger user table.", TypeNameToString(colDef->typname)))); -- 2.34.1 From 975b177d85c4ecb03a4f01ae4d53763ca4e2dba5 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Sat, 30 Sep 2023 00:45:16 +0800 Subject: [PATCH 12/48] Update userchain.cpp --- .../security/gs_ledger/userchain.cpp | 430 +++++++++++------- 1 file changed, 258 insertions(+), 172 deletions(-) diff --git a/src/gausskernel/security/gs_ledger/userchain.cpp b/src/gausskernel/security/gs_ledger/userchain.cpp index 68dc66dcd..9240ae49d 100644 --- a/src/gausskernel/security/gs_ledger/userchain.cpp +++ b/src/gausskernel/security/gs_ledger/userchain.cpp @@ -41,114 +41,152 @@ #include "catalog/gs_global_chain.h" /* - * create_hist_relation -- create a hist table based on the original user table. + * create_hist_relation -- create a hist table based on the original user table.//基于原始用户表创建一个hist表 * - * rel: The original user relation - * reloptions: relation options used to define new relation - * mainTblStmt: Some statement of the query when create the new relation. + * rel: The original user relation//原始用户表 + * reloptions: relation options used to define new relation//用于定义新关系的关系选项 + * mainTblStmt: Some statement of the query when create the new relation.//创建新关系时的某个查询语句 */ +//根据给定的目标表（rel）创建历史链表 void create_hist_relation(Relation rel, Datum reloptions, CreateStmt *mainTblStmt) { errno_t rc; char hist_name[NAMEDATALEN]; - Oid relid = RelationGetRelid(rel); - Oid nsp_oid = PG_BLOCKCHAIN_NAMESPACE; + Oid relid = RelationGetRelid(rel);//获取目标表的OID + Oid nsp_oid = PG_BLOCKCHAIN_NAMESPACE;//命名空间的OID Oid hist_oid; Oid collationObjectId[1]; Oid classObjectId[1]; int16 coloptions[1]; - bool shared_relation = rel->rd_rel->relisshared; + bool shared_relation = rel->rd_rel->relisshared;//判断目标表是否为共享关系 - get_hist_name(relid, get_rel_name(relid), hist_name); + get_hist_name(relid, get_rel_name(relid), hist_name);//获取历史链表的名称 /* * history chain table contains all the columns from the origin user table, and then need to * record the command type, blocknum, and hash value of last block record. + * 历史链表包含了原始用户表的所有列，并且还需要记录上一个区块记录的命令类型、区块号和哈希值。 */ - TupleDesc chain_desc = CreateTemplateTupleDesc(USERCHAIN_COLUMN_NUM, false); + TupleDesc chain_desc = CreateTemplateTupleDesc(USERCHAIN_COLUMN_NUM, false);//创建历史链表的元组描述符 - /* Now consider the additional columns and initilize the description */ - TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_REC_NUM + 1, "rec_num", INT8OID, -1, 0); - TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_HASH_INS + 1, "hash_ins", HASH16OID, -1, 0); - TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_HASH_DEL + 1, "hash_del", HASH16OID, -1, 0); - TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_PREVHASH + 1, "pre_hash", HASH32OID, -1, 0); + /* Now consider the additional columns and initilize the description */ //初始化历史链表的额外列 + TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_REC_NUM + 1, "rec_num", INT8OID, -1, 0);//添加记录号列 + TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_HASH_INS + 1, "hash_ins", HASH16OID, -1, 0);//添加插入操作的哈希值列 + TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_HASH_DEL + 1, "hash_del", HASH16OID, -1, 0);//添加删除操作的哈希值列 + TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_PREVHASH + 1, "pre_hash", HASH32OID, -1, 0);//添加上一个区块的哈希值列 reloptions = AddInternalOption(reloptions, INTERNAL_MASK_DALTER | INTERNAL_MASK_DDELETE | - INTERNAL_MASK_DINSERT | INTERNAL_MASK_DUPDATE); - + INTERNAL_MASK_DINSERT | INTERNAL_MASK_DUPDATE);//添加内部选项,通过使用按位或操作符（|），将INTERNAL_MASK_DALTER、INTERNAL_MASK_DDELETE、INTERNAL_MASK_DINSERT和INTERNAL_MASK_DUPDATE四个标志位添加到reloptions中 + //添加这些内部选项可以用于在处理历史链表时进行相应的操作追踪和记录。 + //使用给定的历史链表名称(hist_name)、命名空间OID(nsp_oid)、表空间OID(rel->rd_rel->reltablespace)等参数创建历史链表。 + //使用给定的chain_desc元组描述符来定义历史链表的列和属性。 + //使用指定的参数（如所有者、持久性、共享关系标志等）来配置历史链表。 + //注册历史链表的元数据到系统目录中，以便后续可以在数据库中访问和操作该表 + //将创建的历史链表的对象ID（OID）赋值给hist_oid变量，以便后续的处理和引用 hist_oid = heap_create_with_catalog(hist_name, nsp_oid, rel->rd_rel->reltablespace, InvalidOid, InvalidOid, InvalidOid, rel->rd_rel->relowner, chain_desc, NIL, 'r', (rel->rd_rel->relpersistence == 't') ? 'u' : rel->rd_rel->relpersistence, shared_relation, false, true, 0, ONCOMMIT_NOOP, reloptions, false, true, - NULL, REL_CMPRS_NOT_SUPPORT, NULL, false); + NULL, REL_CMPRS_NOT_SUPPORT, NULL, false);//创建历史链表 /* make the history chain relation visible, else heap_open will fail */ - CommandCounterIncrement(); + CommandCounterIncrement();//提交事务，使历史链表对后续操作可见 -#ifdef ENABLE_MULTIPLE_NODES +#ifdef ENABLE_MULTIPLE_NODES//启用了多节点（分布式）功能 bool is_initdb_on_dn = false; /* Add to pgxc_class */ /* When the sum of shmemNumDataNodes and shmemNumCoords equals to one, * the create table command is executed on datanode during initialization . * In this case, we do not write created table info in pgxc_class. */ + /*当shmemNumDataNodes和shmemNumCoords的总和等于1时， + * 在初始化期间，在datanode上执行创建表的命令 + * 在这种情况下，我们不会将创建的表信息写入pgxc_class。 + */ if ((*t_thrd.pgxc_cxt.shmemNumDataNodes + *t_thrd.pgxc_cxt.shmemNumCoords) == 1) { is_initdb_on_dn = true; } /* only support normal table, do not support foreign table (can be supported in the future) */ + //仅支持普通表 if ((!u_sess->attr.attr_common.IsInplaceUpgrade || !IsSystemNamespace(nsp_oid)) && - (IS_PGXC_COORDINATOR || (isRestoreMode && mainTblStmt->distributeby != NULL && !is_initdb_on_dn))) { + (IS_PGXC_COORDINATOR || (isRestoreMode && mainTblStmt->distributeby != NULL && !is_initdb_on_dn))) {//不是在升级模式下或者不在系统命名空间下,是协调节点,是恢复模式且主表语句中的distributeby不为空且不是在datanode上执行初始化操作 AddRelationDistribution(hist_name, hist_oid, NULL, mainTblStmt->subcluster, - InvalidOid, chain_desc, true); - CommandCounterIncrement(); + InvalidOid, chain_desc, true);//将历史链表添加到pgxc_class中以支持分布式 + CommandCounterIncrement();//提交事务，使pgxc_class对后续操作可见 /* Make sure locator info gets rebuilt */ - RelationCacheInvalidateEntry(hist_oid); + RelationCacheInvalidateEntry(hist_oid);//刷新历史链表的关系缓存 } #endif /* now create index for this new history table */ + //为这个新的历史表创建索引 char hist_index_name[NAMEDATALEN]; - rc = snprintf_s(hist_index_name, NAMEDATALEN, NAMEDATALEN - 1, "gs_hist_%u_index", relid); + rc = snprintf_s(hist_index_name, NAMEDATALEN, NAMEDATALEN - 1, "gs_hist_%u_index", relid);//创建历史链表的索引名称 securec_check_ss(rc, "", ""); /* open the previous created history chain table */ - Relation hist_rel = heap_open(hist_oid, ShareLock); - IndexInfo *hist_index = makeNode(IndexInfo); + Relation hist_rel = heap_open(hist_oid, ShareLock);//打开先前创建的历史链表 + IndexInfo *hist_index = makeNode(IndexInfo);//创建索引信息 hist_index->ii_NumIndexAttrs = 1; - hist_index->ii_NumIndexKeyAttrs = 1; - hist_index->ii_KeyAttrNumbers[0] = 1; + hist_index->ii_NumIndexKeyAttrs = 1;//索引的属性数量和键属性数量都为1 + hist_index->ii_KeyAttrNumbers[0] = 1;//索引的键属性为第一个属性 hist_index->ii_Expressions = NIL; hist_index->ii_ExpressionsState = NIL; hist_index->ii_Predicate = NIL; - hist_index->ii_PredicateState = NIL; + hist_index->ii_PredicateState = NIL;//设置为空列表，表示索引没有谓词限制 hist_index->ii_ExclusionOps = NULL; hist_index->ii_ExclusionProcs = NULL; - hist_index->ii_ExclusionStrats = NULL; - hist_index->ii_Unique = true; - hist_index->ii_ReadyForInserts = true; - hist_index->ii_Concurrent = false; - hist_index->ii_BrokenHotChain = false; - hist_index->ii_PgClassAttrId = Anum_pg_class_relhasindex; + hist_index->ii_ExclusionStrats = NULL;//表示索引没有排除约束 + hist_index->ii_Unique = true;//索引是唯一的 + hist_index->ii_ReadyForInserts = true;//索引已准备好接收插入操作 + hist_index->ii_Concurrent = false;//索引不支持并发操作 + hist_index->ii_BrokenHotChain = false;//索引的热点链未破裂 + hist_index->ii_PgClassAttrId = Anum_pg_class_relhasindex;//设置索引信息的各个字段值,表示索引对应于pg_class表的relhasindex字段 - collationObjectId[0] = InvalidOid; - classObjectId[0] = INT4_BTREE_OPS_OID; - coloptions[0] = 0; + + collationObjectId[0] = InvalidOid;//设置索引的排序规则对象标识符为无效值InvalidOid，表示索引不使用任何特定的排序规则。 + classObjectId[0] = INT4_BTREE_OPS_OID;//设置索引的操作符类对象标识符为INT4_BTREE_OPS_OID，表示索引使用整型数据的 B-tree 操作符。 + coloptions[0] = 0;//索引没有额外的列选项 IndexCreateExtraArgs extra; - extra.existingPSortOid = InvalidOid; - extra.isPartitionedIndex = false; - extra.isGlobalPartitionedIndex = false; + extra.existingPSortOid = InvalidOid;//该索引不依赖于任何预排序对象 + extra.isPartitionedIndex = false;//表示该索引不是分区索引 + extra.isGlobalPartitionedIndex = false;//该索引不是全局分区索引 + /* + hist_rel：被创建索引的关系对象。 + hist_index_name：索引的名称。 + InvalidOid：既有预排序对象的对象标识符，由于此处没有使用预排序对象，设置为无效值 InvalidOid。 + InvalidOid：分区表的对象标识符，由于此处不是分区索引，设置为无效值 InvalidOid。 + hist_index：指向索引元数据的 IndexInfo 结构体指针。 + list_make1((void *)"rec_num")：索引的键属性名。此处只有一个键属性，为字符串 "rec_num"。 + BTREE_AM_OID：索引访问方法的对象标识符，表示使用 B-tree 索引方法。 + rel->rd_rel->reltablespace：索引所属的表空间。 + collationObjectId：排序规则对象的标识符数组。 + classObjectId：操作符类对象的标识符数组。 + coloptions：列选项数组。 + (Datum) 0：索引的存储选项，此处为默认值。 + true：指示索引是唯一的。 + false：指示关闭并发索引创建。 + false：指示热点链未破裂。 + false：指示该索引不是分区索引。 + true：指示索引已准备好接受插入操作。 + false：指示该索引不是全局分区索引。 + false：指示索引不依赖于任何预排序对象。 + &extra：额外的索引创建参数结构体。 + false：指示创建非关系的索引。 + */ index_create(hist_rel, hist_index_name, InvalidOid, InvalidOid, hist_index, list_make1((void *)"rec_num"), BTREE_AM_OID, rel->rd_rel->reltablespace, collationObjectId, classObjectId, coloptions, (Datum) 0, true, false, false, false, - true, false, false, &extra, false); + true, false, false, &extra, false);//在历史链表上创建索引 - heap_close(hist_rel, NoLock); + heap_close(hist_rel, NoLock);//关闭历史链表关系 /* Specify dependent between history table and origin table with depend option audo. */ + //使用depend选项将历史表和原始表之间的依赖关系指定为自动依赖。 ObjectAddress myself; ObjectAddress referenced; myself.classId = RelationRelationId; @@ -157,84 +195,92 @@ void create_hist_relation(Relation rel, Datum reloptions, CreateStmt *mainTblStm referenced.classId = RelationRelationId; referenced.objectId = relid; referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_AUTO); + recordDependencyOn(&myself, &referenced, DEPENDENCY_AUTO);//指定历史链表和原始表之间的依赖关系为自动依赖,当被依赖的对象发生变化时，依赖关系会自动更新 - pfree_ext(chain_desc); + pfree_ext(chain_desc);//释放内存 /* * Make changes visible */ - CommandCounterIncrement(); + CommandCounterIncrement();//提交事务，使更改对后续操作可见 } /* - * rename_hist_by_usertable -- rename hist table name by its user table and new usertable name + * rename_hist_by_usertable -- rename hist table name by its user table and new usertable name//根据用户表和新的用户表名重命名历史表名 * - * relid: relation oid of user table - * new_usertable_name: the new table name of user table + * relid: relation oid of user table//用户表的关系oid + * new_usertable_name: the new table name of user table//用户表的新表名 * * Note: This function is used after origin user table renamed, and then caller * can use this function to rename the corresponding hist table name. + * 注意：此函数用于在原始用户表被重命名后，调用者可以使用该函数来重命名对应的历史表名。 */ void rename_hist_by_usertable(Oid relid, const char *new_usertable_name) { + //获取历史表的OID Oid hist_oid = get_hist_oid(relid); + //创建一个保存新历史表名的字符数组 char new_hist_name[NAMEDATALEN]; + //根据用户表和新的用户表名生成新的历史表名 get_hist_name(relid, new_usertable_name, new_hist_name); /* Do rename hist table. */ + //执行历史表重命名操作 RenameRelationInternal(hist_oid, new_hist_name); } /* - * rename_hist_by_newnsp -- rename one hist table while altering schema name + * rename_hist_by_newnsp -- rename one hist table while altering schema name//通过修改模式名称重命名一个历史表 * - * user_relid: relation oid of user table - * new_nsp_name: the new schema name of user table + * user_relid: relation oid of user table//用户表的关系oid + * new_nsp_name: the new schema name of user table//用户表的新模式名称 */ void rename_hist_by_newnsp(Oid user_relid, const char *new_nsp_name) { Oid hist_oid; char old_hist_name[NAMEDATALEN] = {0}; char new_hist_name[NAMEDATALEN] = {0}; + //获取旧的历史表名称 get_hist_name(user_relid, get_rel_name(user_relid), old_hist_name); + //根据旧的历史表名称和指定的命名空间获取历史表的oid hist_oid = get_relname_relid(old_hist_name, PG_BLOCKCHAIN_NAMESPACE); /* Some especial tables such as foreign tables have no hist table. So make sure hist exists. */ - if (!OidIsValid(hist_oid)) { + if (!OidIsValid(hist_oid)) {//如果历史表不存在，则直接返回 return; } + //构造新的历史表名称 get_hist_name(user_relid, get_rel_name(user_relid), new_hist_name, get_rel_namespace(user_relid), new_nsp_name); - + //执行重命名操作 RenameRelationInternal(hist_oid, new_hist_name); } /* - * rename_histlist_by_newnsp -- rename a list of hist table while altering schema name + * rename_histlist_by_newnsp -- rename a list of hist table while altering schema name//在修改模式名称的同时重命名历史表列表 * - * usertable_oid_list: relation oid list of user tables - * new_nsp_name: the new schema name of user table + * usertable_oid_list: relation oid list of user tables//用户表的关系oid列表 + * new_nsp_name: the new schema name of user table//用户表的新模式名称 */ void rename_histlist_by_newnsp(List *usertable_oid_list, const char *new_nsp_name) { ListCell *lc = NULL; - foreach (lc, usertable_oid_list) { - Oid relid = (Oid)lfirst_oid(lc); - rename_hist_by_newnsp(relid, new_nsp_name); + foreach (lc, usertable_oid_list) {//批量处理 + Oid relid = (Oid)lfirst_oid(lc);//将当前元素转换为Oid类型的relid + rename_hist_by_newnsp(relid, new_nsp_name);//用新的模式名称new_nsp_name重命名历史表 } } /* - * user_hash_attrno -- get the attribute number of user table's hash column. + * user_hash_attrno -- get the attribute number of user table's hash column.//获取用户表哈希列的属性编号 * - * rd_att: tuple description of user table + * rd_att: tuple description of user table//用户表的元组描述 */ int user_hash_attrno(const TupleDesc rd_att) { int hash_natt = -1; - Form_pg_attribute rel_attr = NULL; - for (int i = rd_att->natts - 1; i >= 0; i--) { + Form_pg_attribute rel_attr = NULL;//遍历用户表的属性 + for (int i = rd_att->natts - 1; i >= 0; i--) {//从最后一个属性开始向前查找 rel_attr = rd_att->attrs[i]; - if (strcmp(rel_attr->attname.data, "hash") == 0) { + if (strcmp(rel_attr->attname.data, "hash") == 0) {//如果两者相等，则说明找到了哈希列 hash_natt = i; break; } @@ -243,11 +289,11 @@ int user_hash_attrno(const TupleDesc rd_att) } /* - * hash_combine_tuple_data -- generate hash of each attribute and return the combination string. + * hash_combine_tuple_data -- generate hash of each attribute and return the combination string.//计算每个属性的哈希值，并返回组合字符串 * - * data_string: combination of hash that calculate from each attribute - * tabledesc: tuple description of user table - * tuple: row data of user table + * data_string: combination of hash that calculate from each attribute//由每个属性计算得到的哈希值的组合字符串 + * tabledesc: tuple description of user table//用户表的元组描述 + * tuple: row data of user table//用户表的行数据 */ static void hash_combine_tuple_data(char *buf, int buf_size, TupleDesc tabledesc, HeapTuple tuple) { @@ -257,178 +303,190 @@ static void hash_combine_tuple_data(char *buf, int buf_size, TupleDesc tabledesc char hash_str[UINT64STRSIZE + 1] = {0}; Datum *values = (Datum *) palloc0(natts * sizeof(Datum)); bool *nulls = (bool *) palloc0(natts * sizeof(bool)); + //将堆元组解码为属性值和空值标志 heap_deform_tuple(tuple, tabledesc, values, nulls); for (int i = 0; i < natts - 1; ++i) { /* except 'hash' column. */ if (nulls[i]) { continue; } - + //计算属性的哈希值 uint64 col_hash = compute_hash(tabledesc->attrs[i]->atttypid, values[i], LOCATOR_TYPE_HASH); + //将哈希值转换为字符串 rc = snprintf_s(hash_str, UINT64STRSIZE + 1, UINT64STRSIZE, "%lu", col_hash); securec_check_ss(rc, "", ""); + //将哈希值拼接到结果字符串中 rc = snprintf_s(buf + buflen, buf_size - buflen, buf_size - buflen - 1, "%s", hash_str); securec_check_ss(rc, "", ""); buflen += strlen(hash_str); } + //释放内存 pfree_ext(values); pfree_ext(nulls); } /* - * get_user_tuple_hash -- get the hash value of usertable's tuple. + * get_user_tuple_hash -- get the hash value of usertable's tuple.//获取用户表元组的哈希值 * - * tuple: row data of user table - * desc: tuple description of user table + * tuple: row data of user table//用户表的行数据 + * desc: tuple description of user table//用户表的元组描述 */ uint64 get_user_tuple_hash(HeapTuple tuple, TupleDesc desc) { Datum value; bool isnull = false; + //获取哈希列的列号 int hash_attno = user_hash_attrno(desc); + //从元组中获取哈希列的值（最后一列） value = heap_getattr(tuple, hash_attno + 1, desc, &isnull); /* get last column. */ - Assert(!isnull); + Assert(!isnull);//确保获取的值不为空 + //将值转换为uint64类型，并返回 return DatumGetUInt64(value); } /* - * gen_user_tuple_hash -- generate hash of each user table's tuple. + * gen_user_tuple_hash -- generate hash of each user table's tuple.//生成用户表中每个元组的哈希值 * - * rel: user table - * tuple: row data of user table + * rel: user table//用户表 + * tuple: row data of user table//用户表的行数据 */ static uint64 gen_user_tuple_hash(Relation rel, HeapTuple tuple) { + //获取用户表的元组描述符 TupleDesc tabledesc = RelationGetDescr(rel); + //计算用于存储哈希值以及中间结果的字符串需要的大小 int data_size = UINT64STRSIZE * tabledesc->natts + 1; + //分配内存存储字符串，并清空 char *data_string = (char *)palloc0(data_size * sizeof(char)); + //计算元组数据的哈希值并将其拼接到字符串中 hash_combine_tuple_data(data_string, data_size, tabledesc, tuple); - + //计算字符串的MD5哈希值，存储在sum数组中 uint8 sum[16]; if (pg_md5_binary(data_string, strlen(data_string), sum) == false) { pfree_ext(data_string); ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); } + //将sum数组的前8个字节（即16个字节中的第5到第12个字节）拼接成一个64位整数 uint64 result = 0; for (int i = 0; i < 7; i++) { result |= sum[4 + i]; result = (result << 8); } result |= sum[11]; - + //释放分配的内存并返回哈希值 pfree_ext(data_string); return result; } /* - * set_user_tuple_hash -- calculate and fill the hash attribute of user table's tuple. + * set_user_tuple_hash -- calculate and fill the hash attribute of user table's tuple.//计算并填充用户表元组的哈希属性 * - * tup: row data of user table - * rel: user table - * hash_exists: whether tuple comes with tuplehash. + * tup: row data of user table//用户表的行数据 + * rel: user table//用户表 + * hash_exists: whether tuple comes with tuplehash.//元组是否包含tuplehash * * Note: if hash_exists is true, we should recompute - * tuple hash and compare with tuplehash of itself. + * tuple hash and compare with tuplehash of itself.//如果hash_exists为真，则我们应该重新计算元组的哈希值，并与自身的tuplehash进行比较 */ HeapTuple set_user_tuple_hash(HeapTuple tup, Relation rel, bool hash_exists) { - uint64 row_hash = gen_user_tuple_hash(rel, tup); - int hash_attrno = user_hash_attrno(rel->rd_att); - if (hash_exists) { + uint64 row_hash = gen_user_tuple_hash(rel, tup);//生成该行数据的哈希值 + int hash_attrno = user_hash_attrno(rel->rd_att);//获取哈希属性在用户表中的属性编号 + if (hash_exists) {//如果元组已经包含tuplehash bool is_null; - Datum hash = heap_getattr(tup, hash_attrno + 1, rel->rd_att, &is_null); - if (is_null || row_hash != DatumGetUInt64(hash)) { - ereport(ERROR, (errcode(ERRCODE_OPERATE_INVALID_PARAM), errmsg("Invalid tuple hash."))); + Datum hash = heap_getattr(tup, hash_attrno + 1, rel->rd_att, &is_null);//从元组中获取已存在的哈希值 + if (is_null || row_hash != DatumGetUInt64(hash)) {//比较新计算的哈希值和已存在的哈希值是否相等 + ereport(ERROR, (errcode(ERRCODE_OPERATE_INVALID_PARAM), errmsg("Invalid tuple hash.")));//哈希值不一致，抛出错误 } - return tup; + return tup;//返回原始的元组 } Datum *values = NULL; bool *nulls = NULL; bool *replaces = NULL; - /* Build modified tuple */ + /* Build modified tuple */ //构建修改后的元组 int2 nattrs = RelationGetNumberOfAttributes(rel); values = (Datum*)palloc0(nattrs * sizeof(Datum)); nulls = (bool*)palloc0(nattrs * sizeof(bool)); replaces = (bool*)palloc0(nattrs * sizeof(bool)); - values[hash_attrno] = UInt64GetDatum(row_hash); - replaces[hash_attrno] = true; - HeapTuple newtup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); + values[hash_attrno] = UInt64GetDatum(row_hash); //将新计算的哈希值存入对应的属性中 + replaces[hash_attrno] = true;//标记该属性为被替换 + HeapTuple newtup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces);//修改元组 pfree_ext(values); pfree_ext(nulls); pfree_ext(replaces); - return newtup; + return newtup;//返回修改后的元组 } /* - * get_hist_oid -- get the oid of history table by oid, name and namespace name of user table + * get_hist_oid -- get the oid of history table by oid, name and namespace name of user table//根据用户表的OID、名称和命名空间名称获取历史表的OID * - * relid: relation oid of user table - * rel_name: relation name of user table - * rel_nsp: namespace name of user table + * relid: relation oid of user table//用户表的关系OID + * rel_name: relation name of user table//用户表的关系名称 + * rel_nsp: namespace name of user table//用户表的命名空间名称 */ Oid get_hist_oid(Oid relid, const char *rel_name, Oid rel_nsp) { - if (rel_name == NULL) { - rel_name = get_rel_name(relid); + if (rel_name == NULL) {//如果为NULL + rel_name = get_rel_name(relid);//通过get_rel_name(relid)函数获取用户表（user table）的名称，并将其赋值给rel_name } char hist_name[NAMEDATALEN]; - get_hist_name(relid, rel_name, hist_name, rel_nsp); - Oid hist_oid = get_relname_relid(hist_name, PG_BLOCKCHAIN_NAMESPACE); + get_hist_name(relid, rel_name, hist_name, rel_nsp);//根据用户表的OID、名称和命名空间名称生成历史表的名称 + Oid hist_oid = get_relname_relid(hist_name, PG_BLOCKCHAIN_NAMESPACE);//根据历史表的名称和预定义的命名空间（PG_BLOCKCHAIN_NAMESPACE）获取历史表的OID return hist_oid; } /* - * get_user_tupleid_hash -- get the hash value of usertable's tupleid + * get_user_tupleid_hash -- get the hash value of usertable's tupleid//获取用户表元组的哈希值 * - * relation: relation of user table - * tupleid: tupleid of user tuple + * relation: relation of user table//用户表的关系 + * tupleid: tupleid of user tuple//用户元组的tupleid */ uint64 get_user_tupleid_hash(Relation relation, ItemPointer tupleid) { - BlockNumber block; - Buffer buffer; - Buffer vmbuffer = InvalidBuffer; - Page page; - ItemId lp; - HeapTupleData tp; - TupleDesc tabledescr; - uint64 result; + BlockNumber block; //块号 + Buffer buffer;//缓冲区 + Buffer vmbuffer = InvalidBuffer;//可见度映射缓冲区 + Page page;//页面 + ItemId lp;//元组标识符 + HeapTupleData tp;//堆元组数据 + TupleDesc tabledescr;//表描述符 + uint64 result;//哈希值 - tabledescr = RelationGetDescr(relation); - /* get tuple use tupleid */ - block = ItemPointerGetBlockNumber(tupleid); - buffer = ReadBuffer(relation, block); - page = BufferGetPage(buffer); + tabledescr = RelationGetDescr(relation);//获取表描述符 + /* get tuple use tupleid */ //使用元组标识符（TupleId）获取元组 + block = ItemPointerGetBlockNumber(tupleid);//获取元组所在的块号 + buffer = ReadBuffer(relation, block);//读取关系中的块 + page = BufferGetPage(buffer);//获取块对应的页 if (PageIsAllVisible(page)) { - visibilitymap_pin(relation, block, &vmbuffer); + visibilitymap_pin(relation, block, &vmbuffer);//锁定页面 } - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);//对缓冲区进行互斥锁定 - lp = PageGetItemId(page, ItemPointerGetOffsetNumber(tupleid)); - tp.t_tableOid = RelationGetRelid(relation); - tp.t_data = (HeapTupleHeader) PageGetItem(page, lp); - tp.t_len = ItemIdGetLength(lp); - tp.t_self = *tupleid; + lp = PageGetItemId(page, ItemPointerGetOffsetNumber(tupleid));//获取元组标识符 + tp.t_tableOid = RelationGetRelid(relation);//设置表OID + tp.t_data = (HeapTupleHeader) PageGetItem(page, lp);//设置元组的数据 + tp.t_len = ItemIdGetLength(lp);//设置元组长度 + tp.t_self = *tupleid;//设置元组标识符 - result = get_user_tuple_hash(&tp, tabledescr); + result = get_user_tuple_hash(&tp, tabledescr);//计算元组的哈希值 - UnlockReleaseBuffer(buffer); + UnlockReleaseBuffer(buffer);//解锁并释放缓冲区 if (vmbuffer != InvalidBuffer) { - ReleaseBuffer(vmbuffer); + ReleaseBuffer(vmbuffer);//如果可见度映射缓冲区不为空，则释放它 } - return result; + return result;//返回哈希值 } /* - * gen_hist_tuple_hash -- calculate pre_hash of hist table + * gen_hist_tuple_hash -- calculate pre_hash of hist table//计算历史表的预散列哈希值 * - * relid: relation oid of history table - * current_block_data: data of current row, includes hash_ins and hash_del - * pre_row_exist: when current row is the first row, it's true - * pre_row_hash: the pre_hash value of previous row - * hash: the result hash + * relid: relation oid of history table//历史表的关系对象标识符 + * current_block_data: data of current row, includes hash_ins and hash_del//当前行的数据，包括要插入和要删除的哈希值 + * pre_row_exist: when current row is the first row, it's true//当当前行是第一行时，它的值为 true + * pre_row_hash: the pre_hash value of previous row//前一行的预散列哈希值。 + * hash: the result hash//计算结果的哈希值 */ void gen_hist_tuple_hash(Oid relid, char *current_block_data, bool pre_row_exist, hash32_t *pre_row_hash, hash32_t *hash) @@ -436,16 +494,22 @@ void gen_hist_tuple_hash(Oid relid, char *current_block_data, bool pre_row_exist errno_t rc; int buf_size = strlen(current_block_data) + NAMEDATALEN + 1; char *data_string = (char *)palloc0(buf_size * sizeof(char)); + //如果当前行是第一行 if (pre_row_exist) { + //将前一行的预散列哈希值转换为字符串 char *pre_hash_str = DatumGetCString(DirectFunctionCall1(hash32out, HASH32GetDatum(pre_row_hash))); + //将当前行的数据和前一行的预散列哈希值连接起来 rc = snprintf_s(data_string, buf_size, buf_size - 1, "%s%s", current_block_data, pre_hash_str); + //释放前一行的预散列哈希值字符串 pfree_ext(pre_hash_str); } else { + //获取历史表的关系名 char *rel_name = get_rel_name(relid); + //将当前行的数据和关系名连接起来 rc = snprintf_s(data_string, buf_size, buf_size - 1, "%s%s", current_block_data, rel_name); } securec_check_ss(rc, "", ""); - + //使用MD5算法计算数据字符串的哈希值 if (!pg_md5_binary(data_string, strlen(data_string), hash->data)) { pfree_ext(data_string); ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); @@ -454,12 +518,12 @@ void gen_hist_tuple_hash(Oid relid, char *current_block_data, bool pre_row_exist } /* - * fill_hist_block -- get newest preblock in cache, and prepare new block for flushing. + * fill_hist_block -- get newest preblock in cache, and prepare new block for flushing.//从缓存中获取最新的预块，并准备新的块用于刷新 * - * histoid: relation oid of history table - * hash_ins: hash_ins value of current row - * hash_del: hash_del value of current row - * block: row id and pre_hash from current row + * histoid: relation oid of history table//历史表的关系OID + * hash_ins: hash_ins value of current row//当前行的hash_ins值 + * hash_del: hash_del value of current row//当前行的hash_del值 + * block: row id and pre_hash from current row//当前行的行ID和预哈希值 */ static void fill_hist_block(Oid histoid, uint64 hash_ins, uint64 hash_del, HistBlock *block) { @@ -467,6 +531,7 @@ static void fill_hist_block(Oid histoid, uint64 hash_ins, uint64 hash_del, HistB block->rec_num = get_next_recnum(histoid); /* Before generate previous hash, we should get current block information */ + //在生成前一个哈希值之前，我们应该获取当前块的信息 error_t rc = sprintf_s(data, NAMEDATALEN, "%lu%lu%lu", block->rec_num, hash_ins, hash_del); securec_check_ss(rc, "", ""); @@ -474,20 +539,24 @@ static void fill_hist_block(Oid histoid, uint64 hash_ins, uint64 hash_del, HistB } /* - * hist_table_record_internal -- append record to history table when user table is modified + * hist_table_record_internal -- append record to history table when user table is modified//当用户表被修改时，将记录追加到历史表中 * - * hist_oid: relation oid of history table - * hash_ins: hash of row that added by current operation - * hash_del: hash of row that deleted by current operation + * hist_oid: relation oid of history table//历史表的关系OID + * hash_ins: hash of row that added by current operation//当前操作添加的行的哈希值 + * hash_del: hash of row that deleted by current operation//当前操作删除的行的哈希值 */ bool hist_table_record_internal(Oid hist_oid, const uint64 *hash_ins, const uint64 *hash_del) { + //定义一个values数组来存储要插入的字段值，同时定义一个nulls数组来表示每个字段是否为NULL。 Datum values[USERCHAIN_COLUMN_NUM] = {0}; bool nulls[USERCHAIN_COLUMN_NUM] = {false}; + //判断hash_ins和hash_del是否为NULL，并分别赋值给ins_null和del_null。 bool ins_null = hash_ins == NULL; bool del_null = hash_del == NULL; + //将实际的哈希值赋给t_ins和t_del，如果对应的指针是NULL，则赋值为0。 uint64 t_ins = ins_null ? 0 : *hash_ins; uint64 t_del = del_null ? 0 : *hash_del; + //声明一个HistBlock结构体变量block，并调用fill_hist_block函数获取当前块的信息。 HistBlock block; if (!OidIsValid(hist_oid)) { @@ -497,7 +566,7 @@ bool hist_table_record_internal(Oid hist_oid, const uint64 *hash_ins, const uint /* Before generate previous hash, we should get current block information */ fill_hist_block(hist_oid, t_ins, t_del, &block); - + //将block中的字段值赋给values数组对应的位置，设置相应的nulls标记。 values[USERCHAIN_COLUMN_REC_NUM] = UInt64GetDatum(block.rec_num); values[USERCHAIN_COLUMN_HASH_INS] = UInt64GetDatum(t_ins); values[USERCHAIN_COLUMN_HASH_DEL] = UInt64GetDatum(t_del); @@ -506,98 +575,115 @@ bool hist_table_record_internal(Oid hist_oid, const uint64 *hash_ins, const uint nulls[USERCHAIN_COLUMN_HASH_INS] = ins_null; nulls[USERCHAIN_COLUMN_HASH_DEL] = del_null; nulls[USERCHAIN_COLUMN_PREVHASH] = false; - + //打开并加锁历史表，并获取其描述符。 Relation hist_rel = heap_open(hist_oid, RowExclusiveLock); TupleDesc hist_desc = RelationGetDescr(hist_rel); - + //使用heap_form_tuple函数根据hist_desc和values、nulls数组创建一个HeapTuple对象，并将其插入到历史表中。 HeapTuple tuple = heap_form_tuple(hist_desc, values, nulls); simple_heap_insert(hist_rel, tuple); - heap_freetuple(tuple); + heap_freetuple(tuple);//释放tuple的内存 heap_close(hist_rel, RowExclusiveLock); return true; } /* - * hist_table_record_insert -- append a record while inserting into user table + * hist_table_record_insert -- append a record while inserting into user table//在向用户表插入记录时，追加一条记录 * - * rel: relation of user table - * tup: the tuple which is inserted into user table - * res_hash: delta of user table hash + * rel: relation of user table//用户表的关系对象 + * tup: the tuple which is inserted into user table//要插入到用户表的元组 + * res_hash: delta of user table hash//用户表哈希的增量 */ bool hist_table_record_insert(Relation rel, HeapTuple tup, uint64 *res_hash) { /* check all inputs are avaliable */ + //检查所有输入是否有效 if (tup == NULL || rel == NULL) { - return false; /* Do some thing */ + return false; /* Do some thing */ //可以进行适当的错误处理 } - + //计算要插入的元组的哈希值 uint64 hash_ins = get_user_tuple_hash(tup, rel->rd_att); + //获取历史表对象的OID Oid hist_oid = get_hist_oid(RelationGetRelid(rel), RelationGetRelationName(rel), RelationGetNamespace(rel)); + //将插入哈希值赋值给res_hash *res_hash = hash_ins; + //调用hist_table_record_internal函数向历史表中插入记录 return hist_table_record_internal(hist_oid, &hash_ins, NULL); } /* - * hist_table_record_delete -- append a record while deleting from user table + * hist_table_record_delete -- append a record while deleting from user table//在从用户表中删除记录时追加一条记录 * - * rel: relation of user table - * hash_del: the hash of deleted tuple - * res_hash: delta of user table hash + * rel: relation of user table//用户表的关系对象 + * hash_del: the hash of deleted tuple//被删除元组的哈希值 + * res_hash: delta of user table hash//用户表哈希的增量 */ bool hist_table_record_delete(Relation rel, uint64 hash_del, uint64 *res_hash) { + //获取历史表对象的OID Oid hist_oid = get_hist_oid(RelationGetRelid(rel), RelationGetRelationName(rel), RelationGetNamespace(rel)); + //将被删除元组的哈希值取负值，并赋给res_hash作为用户表哈希的增量 *res_hash = -hash_del; /* insert history record into userchain table */ + //调用hist_table_record_internal函数执行向历史表插入记录的操作 return hist_table_record_internal(hist_oid, NULL, &hash_del); } /* - * hist_table_record_update -- append a record while updating user table + * hist_table_record_update -- append a record while updating user table//在更新用户表时追加一条记录 * - * rel: relation of user table - * newtup: the tupleid which is updated from user table - * hash_del: the hash of deleted tuple - * res_hash: delta of user table hash + * rel: relation of user table//用户表的关系对象 + * newtup: the tupleid which is updated from user table//从用户表中更新的元组ID + * hash_del: the hash of deleted tuple//被删除元组的哈希值 + * res_hash: delta of user table hash//用户表哈希的增量 */ bool hist_table_record_update(Relation rel, HeapTuple newtup, uint64 hash_del, uint64 *res_hash) { + //获取新元组的哈希值 uint64 hash_ins = get_user_tuple_hash(newtup, rel->rd_att); + //获取历史表对象的OID Oid hist_oid = get_hist_oid(RelationGetRelid(rel), RelationGetRelationName(rel), RelationGetNamespace(rel)); + //计算用户表哈希的增量 *res_hash = hash_ins - hash_del; /* insert history record into userchain table */ + //调用hist_table_record_internal函数执行向历史表插入记录的操作 return hist_table_record_internal(hist_oid, &hash_ins, &hash_del); } /* - * get_copyfrom_line_relhash -- extract hash from each copyfrom line + * get_copyfrom_line_relhash -- extract hash from each copyfrom line//从每个COPY FROM行中提取哈希值 * - * row_data: string line from txt - * len: length of row_data - * hash_colno: the number of split char before hash text - * split: split char - * hash: the buffer of getting hash. + * row_data: string line from txt//来自txt文件的字符串行 + * len: length of row_data//row_data的长度 + * hash_colno: the number of split char before hash text//哈希文本之前的分割字符数 + * split: split char//分割字符 + * hash: the buffer of getting hash.//存储哈希值的缓冲区 */ bool get_copyfrom_line_relhash(const char *row_data, int len, int hash_colno, char split, uint64 *hash) { int pos; /* Not found hash column. */ + //如果未找到哈希列，则返回false if (hash_colno == -1) { return false; } for (pos = 0; pos < len && hash_colno > 0; ++pos) { + //遍历行数据，直到找到分割字符，减少hash_colno的计数 if (row_data[pos] == split) { --hash_colno; } } + //检查是否找到哈希列 if (hash_colno == 0) { int remain_len = len - pos; const char *hash_str = row_data + pos; + //提取哈希值并存储到指定地址 if (remain_len > 0) { + //如果剩余长度大于0，则将哈希字符串转换为uint64，并存储在hash指针指向的地址上，返回true *hash = DatumGetUInt64(DirectFunctionCall1(hash16in, CStringGetDatum(hash_str))); return true; } } + //到达此处表示未成功提取哈希值，返回false return false; } -- 2.34.1 From 5ba9dbee5092ca924fd16bb1e43148d39cf58504 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Mon, 2 Oct 2023 19:56:25 +0800 Subject: [PATCH 13/48] Update bufpage.cpp --- src/gausskernel/storage/page/bufpage.cpp | 196 ++++++++++++++++------- 1 file changed, 140 insertions(+), 56 deletions(-) diff --git a/src/gausskernel/storage/page/bufpage.cpp b/src/gausskernel/storage/page/bufpage.cpp index f89eaf940..256de24bf 100644 --- a/src/gausskernel/storage/page/bufpage.cpp +++ b/src/gausskernel/storage/page/bufpage.cpp @@ -51,23 +51,38 @@ static const uint16 PAGE_CHECKSUM_MAGIC = 0xFFFF; * treat such a page as empty and without free space. Eventually, VACUUM * will clean up such a page and make it usable. */ +/* +* typedef struct { + PageXLogRecPtr pd_lsn;//表示页面的LSN（Log Sequence Number），即页面上一次修改记录的最后一个字节之后的位置 + uint16 pd_checksum; //页面的校验和 + uint16 pd_flags;//页面的标志位，用于表示页面的状态和属性 + LocationIndex pd_lower; //指向空闲空间起始位置的偏移量 + LocationIndex pd_upper; //指向空闲空间结束位置的偏移量 + LocationIndex pd_special;//指向特殊空间起始位置的偏移量 + uint16 pd_pagesize_version;//页面大小和版本号 + ShortTransactionId pd_prune_xid;//最旧可修剪的事务ID（Transaction ID），如果没有可修剪的事务，则为零 + ItemIdData pd_linp[FLEXIBLE_ARRAY_MEMBER]; //行指针数组的起始位置 +} PageHeaderData; +*/ +//验证页面（Page）的合法性和完整性 bool PageIsVerified(Page page, BlockNumber blkno) { - PageHeader p = (PageHeader)page; - size_t* pagebytes = NULL; + PageHeader p = (PageHeader)page;//页面头指针 + size_t* pagebytes = NULL;//页面字节指针 int i; - bool checksum_failure = false; - bool header_sane = false; - bool all_zeroes = false; - uint16 checksum = 0; + bool checksum_failure = false;//校验和失败标记 + bool header_sane = false;//头部是否合法标记 + bool all_zeroes = false;//是否全是零标记 + uint16 checksum = 0;//校验和 /* * Don't verify page data unless the page passes basic non-zero test */ + //不验证页面数据，除非页面通过基本的非零测试 if (CheckPageZeroCases((PageHeader)page)) { - checksum = pg_checksum_page((char*)page, blkno); - if (checksum != p->pd_checksum) { - checksum_failure = true; + checksum = pg_checksum_page((char*)page, blkno);//计算页面校验和 + if (checksum != p->pd_checksum) {//比较校验和与页面头中保存的校验和 + checksum_failure = true;//校验和失败 } /* @@ -76,13 +91,19 @@ bool PageIsVerified(Page page, BlockNumber blkno) * the block can still reveal problems, which is why we offer the * checksum option. */ + /* + * 以下检查不能证明头部是正确的，只能证明它看起来足够合理以允许进入缓冲池。 + * 后续对块的使用仍然可能暴露问题，这就是为什么我们提供了校验和选项。 + */ if ((p->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && p->pd_lower <= p->pd_upper && p->pd_upper <= p->pd_special && p->pd_special <= BLCKSZ && p->pd_special == MAXALIGN(p->pd_special)) { - header_sane = true; + //PD_VALID_FLAG_BITS表示页面头部有效标志位的掩码，~PD_VALID_FLAG_BITS则表示去除有效标志位后的掩码。 + //如果结果等于0，说明除了有效标志位外，其他标志位都为0，即页面头部的标志位合法。 + header_sane = true; //头部合法 } if (header_sane && !checksum_failure) { - return true; + return true;//验证通过 } } @@ -91,26 +112,29 @@ bool PageIsVerified(Page page, BlockNumber blkno) * multiple of size_t - and it's much faster to compare memory using the * native word size. */ + // 检查全零情况。幸运的是，BLCKSZ保证总是sizeof(size_t)的倍数， + // 使用本机字大小比较内存的速度更快。 StaticAssertStmt( BLCKSZ == (BLCKSZ / sizeof(size_t)) * sizeof(size_t), "BLCKSZ has to be a multiple of sizeof(size_t)"); - all_zeroes = true; - pagebytes = (size_t*)page; + all_zeroes = true;//全零标记 + pagebytes = (size_t*)page;//将页面转换为size_t指针 for (i = 0; i < (int)(BLCKSZ / sizeof(size_t)); i++) { - if (pagebytes[i] != 0) { + if (pagebytes[i] != 0) {//如果出现非零字节，说明不是全零页面 all_zeroes = false; break; } } if (all_zeroes) { - return true; + return true;//验证通过 } /* * Throw a WARNING if the checksum fails, but only after we've checked for * the all-zeroes case. */ + //如果校验和失败，则抛出警告，但只在我们检查全零情况后才抛出。 if (checksum_failure) { ereport(WARNING, (ERRCODE_DATA_CORRUPTED, @@ -120,19 +144,25 @@ bool PageIsVerified(Page page, BlockNumber blkno) blkno))); if (header_sane && u_sess->attr.attr_common.ignore_checksum_failure) { - return true; + return true;//验证通过（如果忽略了校验和失败） + //如果启用了忽略校验和失败的选项，并且页面头部被认为是合法的，那么就会跳过校验和验证失败的处理，直接返回true表示通过了校验。 } } - return false; + return false;//验证失败 } /* * PageHeaderIsValid - * Check that the header fields of a page appear valid. + * Check that the header fields of a page appear valid.//检查页面头部字段是否有效。 * - * This is called when a page modify in memory. - * if a page has just been read in from disk, should use PageIsVerified + * This is called when a page modify in memory.//当内存中的页面被修改时调用此函数。 + * if a page has just been read in from disk, should use PageIsVerified//如果页面刚刚从磁盘上读取出来，应该使用PageIsVerified函数 + */ +/* + pagebytes：将页面头部转换为char指针，以字节为单位进行访问 + headersize：页面头部大小 + headeroff：头部偏移量 */ bool PageHeaderIsValid(PageHeader page) { @@ -143,13 +173,15 @@ bool PageHeaderIsValid(PageHeader page) headersize = GetPageHeaderSize(page); /* Check normal case */ - if (PageGetPageSize(page) == BLCKSZ && + /* 检查正常情况 */ + if (PageGetPageSize(page) == BLCKSZ &&//页面大小为标准块大小 (PageGetPageLayoutVersion(page) == PG_COMM_PAGE_LAYOUT_VERSION || PageGetPageLayoutVersion(page) == PG_HEAP_PAGE_LAYOUT_VERSION || PageGetPageLayoutVersion(page) == PG_SEGMENT_PAGE_LAYOUT_VERSION) && - (page->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && page->pd_lower >= headersize && - page->pd_lower <= page->pd_upper && page->pd_upper <= page->pd_special && page->pd_special <= BLCKSZ && - page->pd_special == MAXALIGN(page->pd_special)) + (page->pd_flags & ~PD_VALID_FLAG_BITS) == 0 //pd_flags中只包含有效标志位 + && page->pd_lower >= headersize &&// pd_lower大于等于头部大小 + page->pd_lower <= page->pd_upper && page->pd_upper <= page->pd_special && page->pd_special <= BLCKSZ &&//保证各个指针的顺序正确，并且不超过块大小 + page->pd_special == MAXALIGN(page->pd_special))//pd_special按对齐方式对齐 return true; /* @@ -159,26 +191,35 @@ bool PageHeaderIsValid(PageHeader page) * storage also sets LSN when creating a new page. * So we skip these three variables and test reset variables in the header. */ + /* + * 检查新页面的全零情况； + * 目前，即使是新页面，pd_flags、lsn和checksum可能不为零。例如，在重做log_new_page时，新页面可以设置PD_JUST_AFTER_FPW标志；然后在刷新到磁盘时设置校验和。段页面存储在创建新页面时也设置LSN。 + * 因此，我们跳过这三个变量，检查头部中的重置变量。 + */ if (page->pd_lower != 0 || page->pd_upper != 0 || page->pd_special != 0 || page->pd_pagesize_version != 0) { return false; } + //将页面转换为char指针，以字节为单位进行访问 pagebytes = (char*)page; + //根据页面是否为8字节事务堆版本，确定头部的偏移量 headeroff = PageIs8BXidHeapVersion(page) ? offsetof(HeapPageHeaderData, pd_linp) : offsetof(PageHeaderData, pd_linp); + //检查头部之后的每个字节是否为零 for (i = headeroff; i < BLCKSZ; i++) { if (pagebytes[i] != 0) - return false; + return false;//如果发现任何一个字节不为零，则返回false，表示页面头部不合法。 } + //所有字节都为零，表示页面头部合法 return true; } /* * UPageHeaderIsValid - * Check that the header fields of a page appear valid. + * Check that the header fields of a page appear valid.//检查页面头部字段是否合法。 * - * This is called when a page modify in memory. - * if a page has just been read in from disk, should use PageIsVerified + * This is called when a page modify in memory.//当对内存中的页面进行修改时调用此函数 + * if a page has just been read in from disk, should use PageIsVerified//如果刚刚从磁盘读入页面，则应该使用 PageIsVerified 函数。 */ bool UPageHeaderIsValid(const UHeapPageHeaderData* page) { @@ -189,12 +230,14 @@ bool UPageHeaderIsValid(const UHeapPageHeaderData* page) headersize = SizeOfUHeapPageHeaderData; /* Check normal case */ + // 检查正常情况 if (page->pd_lower >= headersize && page->pd_lower <= page->pd_upper && page->pd_upper <= page->pd_special && page->pd_special <= BLCKSZ && page->pd_special == MAXALIGN(page->pd_special)) return true; /* Check all-zeroes case */ + // 检查全零情况 if (page->pd_lsn.xlogid != 0 || page->pd_lsn.xrecoff != 0 || (page->pd_flags & UHEAP_VALID_FLAG_BITS) != 0 || page->pd_lower != 0 || page->pd_upper != 0 || page->pd_special != 0 || page->td_count != 0 || page->pd_prune_xid != 0) { @@ -205,7 +248,7 @@ bool UPageHeaderIsValid(const UHeapPageHeaderData* page) headeroff = offsetof(UHeapPageHeaderData, reserved); for (i = headeroff; i < BLCKSZ; i++) { if (pagebytes[i] != 0) - return false; + return false;//页面头部不合法 } return true; } @@ -215,15 +258,16 @@ bool UPageHeaderIsValid(const UHeapPageHeaderData* page) * Get a temporary page in local memory for special processing. * The returned page is not initialized at all; caller must do that. */ +//获取一个本地内存中的临时页面，用于特殊处理。返回的页面未被初始化，调用者必须自行进行初始化。 Page PageGetTempPage(Page page) { - Size pageSize; - Page temp; + Size pageSize;// 定义页面大小变量 + Page temp;// 定义临时页面指针变量 - pageSize = PageGetPageSize(page); - temp = (Page)palloc(pageSize); + pageSize = PageGetPageSize(page);//获取已存在页面的大小 + temp = (Page)palloc(pageSize);//分配与已存在页面大小相同的内存，并将返回的地址转换为Page类型 - return temp; + return temp;//返回临时页面指针 } /* @@ -231,6 +275,8 @@ Page PageGetTempPage(Page page) * Get a temporary page in local memory for special processing. * The page is initialized by copying the contents of the given page. */ +//获取一个本地内存中的临时页面，用于特殊处理。该页面通过复制给定页面的内容进行初始化。 +//该函数与上一个函数（PageGetTempPage）的区别在于使用了memcpy_s函数来复制给定页面的内容到临时页面。 Page PageGetTempPageCopy(Page page) { Size pageSize; @@ -240,7 +286,7 @@ Page PageGetTempPageCopy(Page page) pageSize = PageGetPageSize(page); temp = (Page)palloc(pageSize); - rc = memcpy_s(temp, pageSize, page, pageSize); + rc = memcpy_s(temp, pageSize, page, pageSize);//memcpy_s函数是一个安全的内存复制函数，在拷贝时会检查目标地址是否越界，因此能够避免缓冲区溢出漏洞的风险。 securec_check(rc, "\0", "\0"); return temp; } @@ -251,6 +297,7 @@ Page PageGetTempPageCopy(Page page) * Returns the size of the free (allocatable) space on a page, * without any consideration for adding/removing line pointers. */ +//返回页面上可用的自由空间大小，不考虑添加/删除行指针。 Size PageGetExactFreeSpace(Page page) { int space; @@ -259,6 +306,7 @@ Size PageGetExactFreeSpace(Page page) * Use signed arithmetic here so that we behave sensibly if pd_lower > * pd_upper. */ + //使用有符号数进行计算，以确保在 pd_lower > pd_upper 的情况下能够正确处理。 space = (int)((PageHeader)page)->pd_upper - (int)((PageHeader)page)->pd_lower; if (space < 0) { @@ -268,27 +316,42 @@ Size PageGetExactFreeSpace(Page page) return (Size)(uint32)space; } +//分配页面拷贝内存空间 +/* + t_thrd.storage_cxt.pageCopy: 表示页面复制的内存空间。这是一个指向char类型的指针变量。 + t_thrd.storage_cxt.segPageCopy: 表示段页面复制的内存空间。也是一个指向char类型的指针变量。 + ADIO_RUN(): 条件编译宏，用于判断存储设备是否可用。当存储设备可用时，会执行与之相关的代码块。 + ADIO_ELSE(): 条件编译宏的另一分支，当存储设备不可用时，会执行与之相关的代码块。 + adio_align_alloc(): 一个用于内存分配的函数，用于在存储设备可用时对齐分配内存空间。 + MemoryContextAlloc(): 一个用于在指定上下文中分配内存的函数，用于在存储设备不可用时进行内存分配。 + THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE): 用于获取存储内存上下文组的标识，用来指定在哪个上下文中进行内存分配。 + BLCKSZ: 表示内存块的大小，为一个预定义的常量。 +*/ static inline void AllocPageCopyMem() { - if (t_thrd.storage_cxt.pageCopy == NULL) { + if (t_thrd.storage_cxt.pageCopy == NULL) {//页面复制的内存空间为空 ADIO_RUN() { + //使用adio_align_alloc()函数进行内存分配 t_thrd.storage_cxt.pageCopy = (char*)adio_align_alloc(BLCKSZ); } ADIO_ELSE() { + //使用MemoryContextAlloc()函数在存储内存上下文中进行分配 t_thrd.storage_cxt.pageCopy = (char*)MemoryContextAlloc( THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE), BLCKSZ); } ADIO_END(); } - if (t_thrd.storage_cxt.segPageCopy == NULL) { + if (t_thrd.storage_cxt.segPageCopy == NULL) {//段页面复制的内存空间为空 ADIO_RUN() { + //使用adio_align_alloc()函数进行内存分配 t_thrd.storage_cxt.segPageCopy = (char*)adio_align_alloc(BLCKSZ); } ADIO_ELSE() { + //使用MemoryContextAlloc()函数在存储内存上下文中进行分配 t_thrd.storage_cxt.segPageCopy = (char*)MemoryContextAlloc( THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE), BLCKSZ); } @@ -303,6 +366,12 @@ static inline void AllocPageCopyMem() * buffer, other processes might be updating hint bits in it, so we must * copy the page to private storage if we are going to change it. */ +/* +数据块加密，我们为每个线程分配内存空间，该空间在线程退出之前会被重复使用。 +由于我们只对缓冲区有共享锁，其他进程可能会在其中更新提示位， +所以如果我们要修改它，我们必须将页面复制到私有存储空间中。 +*/ +//此函数根据指定的条件对数据块进行加密。 char* PageDataEncryptIfNeed(Page page, TdeInfo* tde_info, bool need_copy, bool is_segbuf) { size_t plainLength = 0; @@ -311,18 +380,19 @@ char* PageDataEncryptIfNeed(Page page, TdeInfo* tde_info, bool need_copy, bool i int retval = 0; TdePageInfo tde_page_info; char* dst = NULL; - + //如果页面是新的页面，或者不需要加密，或者TDE功能未启用，则直接返回原始页面 if (PageIsNew(page) || !PageIsTDE(page) || !g_instance.attr.attr_security.enable_tde) { return (char*)page; } Assert(!PageIsEncrypt(page)); - + //计算明文数据的长度，并生成随机初始化向量 plainLength = ((PageHeader)page)->pd_special - ((PageHeader)page)->pd_upper; retval = RAND_priv_bytes(tde_info->iv, RANDOM_IV_LEN); if (retval != 1) { ereport(WARNING, (errmodule(MOD_SEC_TDE), errmsg("generate random iv for tde failed, errcode:%d", retval))); return (char*)page; } + //如果需要复制页面，分配新的内存空间，并将页面内容复制到该空间中 if (need_copy) { AllocPageCopyMem(); dst = is_segbuf ? t_thrd.storage_cxt.segPageCopy : t_thrd.storage_cxt.pageCopy; @@ -333,13 +403,14 @@ char* PageDataEncryptIfNeed(Page page, TdeInfo* tde_info, bool need_copy, bool i } /* at this part, do the real encryption */ + //进行实际的加密操作 encryptBlockOrCUData(dst + ((PageHeader)dst)->pd_upper, plainLength, dst + ((PageHeader)dst)->pd_upper, &cipherLength, tde_info); Assert(plainLength == cipherLength); - + //将TDE信息转换为页信息，并将其存储在页面的末尾 ret = memset_s(&tde_page_info, sizeof(TdePageInfo), 0, sizeof(TdePageInfo)); securec_check(ret, "\0", "\0"); transformTdeInfoToPage(tde_info, &tde_page_info); @@ -347,37 +418,41 @@ char* PageDataEncryptIfNeed(Page page, TdeInfo* tde_info, bool need_copy, bool i securec_check(ret, "\0", "\0"); /* set the encryption flag */ + //设置页面的加密标志 PageSetEncrypt((Page)dst); return dst; } - +//对数据块进行解密 void PageDataDecryptIfNeed(Page page) { TdeInfo tde_info = {0}; TdePageInfo* tde_page_info = NULL; - + // 检查页面是否同时满足TDE加密和加密标志位的条件 /* whether this page is both TDE page and encrypted */ if (PageIsEncrypt(page) && PageIsTDE(page)) { size_t plainLength = 0; size_t cipherLength = ((PageHeader)page)->pd_special - ((PageHeader)page)->pd_upper; + // 获取存储在页面末尾的TDE信息，并将其转换为内部结构 tde_page_info = (TdePageInfo*)((char*)(page) + BLCKSZ - sizeof(TdePageInfo)); transformTdeInfoFromPage(&tde_info, tde_page_info); /* at this part, do the real decryption */ - decryptBlockOrCUData(page + ((PageHeader)page)->pd_upper, - cipherLength, + // 进行实际的解密操作 + decryptBlockOrCUData(page + ((PageHeader)page)->pd_upper,//要进行解密的数据块的起始位置和解密后的明文的存储位置 + cipherLength,//密文的长度，用于指示要解密的数据块的大小 page + ((PageHeader)page)->pd_upper, - &plainLength, - &tde_info); - Assert(cipherLength == plainLength); + &plainLength,//存储解密后的明文长度的变量的地址 + &tde_info);//指向TDE信息结构的指针，包含解密所需的密钥和其他相关信息 + Assert(cipherLength == plainLength);//确保解密后的明文长度与密文长度相等。如果不相等，将触发断言失败并产生错误信息，表示解密操作出现问题。 /* clear the encryption flag */ + // 清除加密标志位 PageClearEncrypt(page); } } /* - * Set checksum for a page in shared buffers. + * Set checksum for a page in shared buffers.//为共享缓冲区中的一个页面设置校验和。 * * If checksums are disabled, or if the page is not initialized, just return * the input. Otherwise, we must make a copy of the page before calculating @@ -393,6 +468,7 @@ void PageDataDecryptIfNeed(Page page) char* PageSetChecksumCopy(Page page, BlockNumber blkno, bool is_segbuf) { /* If we don't need a checksum, just return the passed-in data */ + //如果不需要校验和，直接返回传入的数据 if (!CheckPageZeroCases((PageHeader)page)) { return (char*)page; } @@ -403,16 +479,18 @@ char* PageSetChecksumCopy(Page page, BlockNumber blkno, bool is_segbuf) * array, is first to ensure adequate alignment for the checksumming code * and second to avoid wasting space in processes that never call this. */ + //分配一次拷贝空间，并在每次调用时重复使用 AllocPageCopyMem(); char *dst = is_segbuf ? t_thrd.storage_cxt.segPageCopy : t_thrd.storage_cxt.pageCopy; - + //拷贝原页面数据到拷贝空间 errno_t rc = memcpy_s(dst, BLCKSZ, (char*)page, BLCKSZ); securec_check(rc, "", ""); /* set page->pd_flags mark using FNV1A for checksum */ + //使用FNV1A算法设置页面标志位的校验和 PageSetChecksumByFNV1A(dst); - + //计算页面的校验和，并存储到页面头部的pd_checksum中 ((PageHeader)dst)->pd_checksum = pg_checksum_page(dst, blkno); return dst; @@ -424,27 +502,31 @@ char* PageSetChecksumCopy(Page page, BlockNumber blkno, bool is_segbuf) * This must only be used when we know that no other process can be modifying * the page buffer. */ +//为私有内存中的一个页面设置校验和 void PageSetChecksumInplace(Page page, BlockNumber blkno) { /* If we don't need a checksum, just return */ + /* 如果不需要校验和，直接返回 */ if (!CheckPageZeroCases((PageHeader)page)) { return; } /* set page->pd_flags mark using FNV1A for checksum */ + /* 使用FNV1A算法设置页面标志位的校验和 */ PageSetChecksumByFNV1A(page); - + /* 计算页面的校验和，并存储在pd_checksum中 */ ((PageHeader)page)->pd_checksum = pg_checksum_page((char*)page, blkno); } /* * PageGetFreeSpaceForMultipleTuples * Returns the size of the free (allocatable) space on a page, - * reduced by the space needed for multiple new line pointers. + * reduced by the space needed for multiple new line pointers.//返回页面上可用的空闲空间大小，减去多个新行指针所需的空间。 * * Note: this should usually only be used on index pages. Use - * PageGetHeapFreeSpace on heap pages. + * PageGetHeapFreeSpace on heap pages.//注意：这个函数通常只用于索引页面。对于堆页面，请使用PageGetHeapFreeSpace。 */ + Size PageGetFreeSpaceForMultipleTuples(Page page, int ntups) { int space; @@ -453,11 +535,13 @@ Size PageGetFreeSpaceForMultipleTuples(Page page, int ntups) * Use signed arithmetic here so that we behave sensibly if pd_lower > * pd_upper. */ + //在这里使用有符号算术，以便在pd_lower > pd_upper的情况下能正确处理。 + //有符号的减法运算，得到的结果即为初始的可用空间大小。 space = (int)((PageHeader)page)->pd_upper - (int)((PageHeader)page)->pd_lower; - + //判断剩余空间是否足够容纳多个新行指针 if (space < (int)(ntups * sizeof(ItemIdData))) - return 0; - space -= ntups * sizeof(ItemIdData); + return 0;//表示无可用空间 + space -= ntups * sizeof(ItemIdData);//根据新行指针的数量，将所需的空间从初始的可用空间中减去 return (Size) space; } -- 2.34.1 From 83e595ee5633716748efa0e6d4e4c8f4ba8ca40c Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Mon, 2 Oct 2023 19:56:55 +0800 Subject: [PATCH 14/48] Update checksum.cpp --- src/gausskernel/storage/page/checksum.cpp | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/gausskernel/storage/page/checksum.cpp b/src/gausskernel/storage/page/checksum.cpp index 30e37b30f..719b1e607 100644 --- a/src/gausskernel/storage/page/checksum.cpp +++ b/src/gausskernel/storage/page/checksum.cpp @@ -13,10 +13,9 @@ * ------------------------------------------------------------------------- */ -#include "postgres.h" -#include "knl/knl_variable.h" - -#include "storage/checksum.h" +#include "postgres.h"//主要的PostgreSQL头文件，其中包含了大量的声明和定义，用于构建和操作数据库系统 +#include "knl/knl_variable.h"//用于包含knernel实例、会话和线程的变量 +#include "storage/checksum.h"//声明计算校验和的函数 /* * The actual code is in storage/checksum_impl.h. This is done so that @@ -24,3 +23,13 @@ * that file from the exported openGauss headers. (Compare our CRC code.) */ #include "storage/checksum_impl.h" +//存储校验和的实现代码 +/* N_SUMS是并行计算校验和的数量。 + FNV_PRIME是FNV -1a哈希的质数乘数。 + CHECKSUM_CACL_ROUNDS是计算校验和的轮数。 + g_checksumBaseOffsets是每个并行FNV哈希的初始状态的基本偏移量数组。 + CHECKSUM_COMP是计算校验和的宏，用于进行一轮校验和计算。 + pg_checksum_block是块校验和算法的函数原型，用于计算数据块的校验和。 + DataBlockChecksum是数据块校验和算法的函数原型，用于计算数据块的校验和（可选地进行全零化）。 + pg_checksum_page是页面校验和算法的函数原型，用于计算页面的校验和。 +*/ \ No newline at end of file -- 2.34.1 From abe39f5288ab1d36a1b949bc79fd6fbf287a0a8c Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Mon, 2 Oct 2023 19:57:18 +0800 Subject: [PATCH 15/48] Update checksum_impl.cpp --- .../storage/page/checksum_impl.cpp | 79 +++++++++++-------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/src/gausskernel/storage/page/checksum_impl.cpp b/src/gausskernel/storage/page/checksum_impl.cpp index bd7b4df4f..6efa7e2f0 100644 --- a/src/gausskernel/storage/page/checksum_impl.cpp +++ b/src/gausskernel/storage/page/checksum_impl.cpp @@ -15,52 +15,54 @@ #include "postgres.h" #include "knl/knl_variable.h" #include "storage/checksum_impl.h" - +//用于对数据进行补零并计算校验和 void ChecksumForZeroPadding(uint32 *sums, const uint32 *dataArr, uint32 currentLeft, uint32 alignSize); - +//对初始种子和值进行校验和初始化 static inline uint32 pg_checksum_init(uint32 seed, uint32 value) { - CHECKSUM_COMP(seed, value); + CHECKSUM_COMP(seed, value);//将种子和值作为参数传入CHECKSUM_COMP宏中,进行校验和计算 return seed; } - +//计算数据块的校验和 uint32 DataBlockChecksum(char* data, uint32 size, bool zeroing) { - uint32 sums[N_SUMS]; - uint32* dataArr = (uint32*)data; - uint32 result = 0; - uint32 i, j; - uint32 currentLeft = size; + uint32 sums[N_SUMS];//定义数组存储部分校验和 + uint32* dataArr = (uint32*)data;//将数据块转换为uint32数组 + uint32 result = 0;//定义变量保存最终的校验和结果 + uint32 i, j;//定义循环变量 + uint32 currentLeft = size;//初始化剩余未处理的数据大小 /* ensure that the size is compatible with the algorithm */ + //确保输入的数据大小符合校验和算法的要求 uint32 alignSize = sizeof(uint32) * N_SUMS; Assert(zeroing || (size % alignSize == 0)); /* initialize partial checksums to their corresponding offsets */ - auto realSize = size < alignSize ? size : alignSize; + auto realSize = size < alignSize ? size : alignSize;//实际的校验和计算大小 uint32 *initUint32 = NULL; char usedForInit[sizeof(uint32) * N_SUMS] = {0}; - if (zeroing && size < alignSize) { + if (zeroing && size < alignSize) {//判断是否需要补零 errno_t rc = memcpy_s(usedForInit, alignSize, (char *) dataArr, realSize); securec_check(rc, "", ""); - currentLeft -= realSize; - initUint32 = (uint32*)usedForInit; + currentLeft -= realSize;//更新剩余未处理数据的大小 + initUint32 = (uint32*)usedForInit;//用复制的数据作为初始值 } else { initUint32 = dataArr; currentLeft -= alignSize; } for (j = 0; j < N_SUMS; j += 2) { - sums[j] = pg_checksum_init(g_checksumBaseOffsets[j], initUint32[j]); + sums[j] = pg_checksum_init(g_checksumBaseOffsets[j], initUint32[j]);// 初始化部分校验和 sums[j + 1] = pg_checksum_init(g_checksumBaseOffsets[j + 1], initUint32[j + 1]); } dataArr += N_SUMS; /* main checksum calculation */ + // 主要的校验和计算 for (i = 1; i < size / alignSize; i++) { for (j = 0; j < N_SUMS; j += 2) { - CHECKSUM_COMP(sums[j], dataArr[j]); + CHECKSUM_COMP(sums[j], dataArr[j]);// 计算部分校验和 CHECKSUM_COMP(sums[j + 1], dataArr[j + 1]); } dataArr += N_SUMS; @@ -69,7 +71,7 @@ uint32 DataBlockChecksum(char* data, uint32 size, bool zeroing) /* checksum for zero padding */ currentLeft -= alignSize * (i - 1); if (currentLeft > 0 && currentLeft < alignSize && zeroing) { - ChecksumForZeroPadding(sums, dataArr, currentLeft, alignSize); + ChecksumForZeroPadding(sums, dataArr, currentLeft, alignSize);// 补零并计算校验和 } /* finally add in two rounds of zeroes for additional mixing */ @@ -78,62 +80,67 @@ uint32 DataBlockChecksum(char* data, uint32 size, bool zeroing) CHECKSUM_COMP(sums[j], 0); /* xor fold partial checksums together */ - result ^= sums[j]; + result ^= sums[j];//对部分校验和结果进行混合得到最终的校验和结果 } return result; } - +//对剩余数据进行补零并计算校验和 void ChecksumForZeroPadding(uint32 *sums, const uint32 *dataArr, uint32 currentLeft, uint32 alignSize) { + // 定义变量maxLen，保存部分校验和数组的大小 auto maxLen = sizeof(uint32) * N_SUMS; + // 创建临时字符数组currentLeftChars，初始化为零，并将剩余的数据复制到该数组中。 char currentLeftChars[maxLen] = {0}; errno_t rc = memcpy_s(currentLeftChars, maxLen, (char *)dataArr, currentLeft); securec_check(rc, "", ""); + // 使用两层循环，对currentLeftChars数组的数据进行校验和计算，并更新部分校验和数组sums的值 for (int j = 0; j < N_SUMS; j += 2) { + // 对于每个部分校验和，分别计算两个32位整数的校验和 + // 使用宏CHECKSUM_COMP对sums中的值与currentLeftChars数组中对应位置的值进行校验和计算，计算结果更新到sums中 CHECKSUM_COMP(sums[j], ((uint32 *)currentLeftChars)[j]); CHECKSUM_COMP(sums[j + 1], ((uint32 *)currentLeftChars)[j + 1]); } } - +//对数据块进行校验和计算 uint32 pg_checksum_block(char* data, uint32 size) { - uint32 sums[N_SUMS]; - uint32* dataArr = (uint32*)data; - uint32 result = 0; + uint32 sums[N_SUMS];// 部分校验和数组 + uint32* dataArr = (uint32*)data;// 数据块指针转换为uint32类型指针 + uint32 result = 0;// 最终校验和结果 uint32 i, j; #ifndef ROACH_COMMON /* ensure that the size is compatible with the algorithm */ - Assert((size % (sizeof(uint32) * N_SUMS)) == 0); + Assert((size % (sizeof(uint32) * N_SUMS)) == 0);// 确保数据块大小能够被算法处理 #endif /* initialize partial checksums to their corresponding offsets */ for (j = 0; j < N_SUMS; j += 2) { - sums[j] = pg_checksum_init(g_checksumBaseOffsets[j], dataArr[j]); + sums[j] = pg_checksum_init(g_checksumBaseOffsets[j], dataArr[j]);// 使用初始偏移量计算部分校验和 sums[j + 1] = pg_checksum_init(g_checksumBaseOffsets[j + 1], dataArr[j + 1]); } - dataArr += N_SUMS; + dataArr += N_SUMS;// 移动数据指针到下一个数据块 /* main checksum calculation */ - for (i = 1; i < size / (sizeof(uint32) * N_SUMS); i++) { + for (i = 1; i < size / (sizeof(uint32) * N_SUMS); i++) {// 处理每个数据块 for (j = 0; j < N_SUMS; j += 2) { - CHECKSUM_COMP(sums[j], dataArr[j]); + CHECKSUM_COMP(sums[j], dataArr[j]);// 计算并更新部分校验和 CHECKSUM_COMP(sums[j + 1], dataArr[j + 1]); } - dataArr += N_SUMS; + dataArr += N_SUMS;// 移动数据指针到下一个数据块 } /* finally add in two rounds of zeroes for additional mixing */ for (j = 0; j < N_SUMS; j++) { - CHECKSUM_COMP(sums[j], 0); + CHECKSUM_COMP(sums[j], 0);// 添加两轮零值进行混合计算 CHECKSUM_COMP(sums[j], 0); /* xor fold partial checksums together */ - result ^= sums[j]; + result ^= sums[j];// 将部分校验和结果异或操作 } - return result; + return result;// 返回最终的校验和结果 } /* @@ -144,11 +151,12 @@ uint32 pg_checksum_block(char* data, uint32 size) * somehow moved to a different location), the page header (excluding the * checksum itself), and the page data. */ +//计算一个openGauss页面的校验和 uint16 pg_checksum_page(char* page, BlockNumber blkno) { PageHeader phdr = (PageHeader)page; - uint16 save_checksum; - uint32 checksum; + uint16 save_checksum;//保存原始校验和值 + uint32 checksum;//校验和计算结果 /* * Save pd_checksum and temporarily set it to zero, so that the checksum @@ -156,17 +164,22 @@ uint16 pg_checksum_page(char* page, BlockNumber blkno) * Restore it after, because actually updating the checksum is NOT part of * the API of this function. */ + //保存pd_checksum并将其临时置零，以避免受页面上旧校验和的影响 save_checksum = phdr->pd_checksum; phdr->pd_checksum = 0; + //计算除了校验和本身以外的部分的校验和 checksum = pg_checksum_block(page, BLCKSZ); + //恢复保存的原始pd_checksum值到页面头部 phdr->pd_checksum = save_checksum; /* Mix in the block number to detect transposed pages */ + //使用异或操作混合块号，以检测页面是否发生了位置变换 checksum ^= blkno; /* * Reduce to a uint16 (to fit in the pd_checksum field) with an offset of * one. That avoids checksums of zero, which seems like a good idea. */ + // 将计算得到的校验和值进行取模运算，限制在UINT16_MAX范围内，并加一作为最终的校验和值 return (checksum % UINT16_MAX) + 1; } -- 2.34.1 From 8c03bf5b3954719e44ede2e151f9d93a8c5173a0 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Mon, 2 Oct 2023 19:57:35 +0800 Subject: [PATCH 16/48] Update gs_xlogdump.cpp --- src/gausskernel/storage/page/gs_xlogdump.cpp | 146 +++++++++++++++---- 1 file changed, 121 insertions(+), 25 deletions(-) diff --git a/src/gausskernel/storage/page/gs_xlogdump.cpp b/src/gausskernel/storage/page/gs_xlogdump.cpp index 3f3e92aa5..8243a391d 100644 --- a/src/gausskernel/storage/page/gs_xlogdump.cpp +++ b/src/gausskernel/storage/page/gs_xlogdump.cpp @@ -54,59 +54,83 @@ typedef struct XLogFilter { BlockNumber blocknum; } XLogFilter; +//通过组合数据目录路径、起始LSN和结束LSN，生成一个格式化的输出文件名 static void GenerateOutputFileName(char *outputFilename, char *start_lsn_str, char *end_lsn_str) { List *elemlist = NIL; + //将起始LSN字符串按照'/'进行分割，将元素存储在elemlist中 SplitIdentifierString(start_lsn_str, '/', &elemlist); + //获取elemlist中第二个元素，赋值给start_lsn_str_p2变量 char *start_lsn_str_p2 = (char *)lsecond(elemlist); + //释放elemlist占用的内存 list_free_ext(elemlist); + //将结束LSN字符串按照'/'进行分割，将元素存储在elemlist中 SplitIdentifierString(end_lsn_str, '/', &elemlist); + //获取elemlist中第二个元素，赋值给end_lsn_str_p2变量 char *end_lsn_str_p2 = (char *)lsecond(elemlist); + //格式化输出文件名，并将结果写入outputFilename的末尾 int rc = snprintf_s(outputFilename + (int)strlen(outputFilename), MAXFILENAME, MAXFILENAME - 1, "%s/%s_%s.xlog", t_thrd.proc_cxt.DataDir, start_lsn_str_p2, end_lsn_str_p2); + //检查格式化操作是否成功执行 securec_check_ss(rc, "\0", "\0"); } - +//验证LSN字符串，并将其转换为XLogRecPtr类型的LSN指针 static void ValidateLSN(char *lsn_str, XLogRecPtr *lsn_ptr) { uint32 hi = 0; uint32 lo = 0; + //调用validate_xlog_location函数验证LSN字符串的有效性 validate_xlog_location(lsn_str); - + //使用sscanf_s函数从LSN字符串中解析出hi和lo两个部分的值 if (sscanf_s(lsn_str, "%X/%X", &hi, &lo) != TWO) + //如果解析失败，则抛出错误并提示无法解析LSN字符串 ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("could not parse xlog location \"%s\"", lsn_str))); + //将hi和lo组合成64位的LSN值，存储到lsn_ptr中 *lsn_ptr = (((uint64)hi) << XIDTHIRTYTWO) | lo; } - +//验证起始LSN和结束LSN，并将它们转换为XLogRecPtr类型的LSN指针 static void ValidateStartEndLSN(char *start_lsn_str, char *end_lsn_str, XLogRecPtr *start_lsn, XLogRecPtr *end_lsn) { + //验证起始LSN字符串并将其转换为XLogRecPtr类型的起始LSN指针 ValidateLSN(start_lsn_str, start_lsn); + //验证结束LSN字符串并将其转换为XLogRecPtr类型的结束LSN指针 ValidateLSN(end_lsn_str, end_lsn); - + //检查结束LSN是否小于起始LSN，如果是则抛出错误 if (XLByteLT(*end_lsn, *start_lsn)) + /* + * 如果结束LSN小于起始LSN，则抛出错误。 + * 错误信息包含起始LSN和结束LSN的具体数值。 + * 使用% X格式将高位32位和低位32位转换为十六进制数值。 + */ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("start xlog location %X/%X should be smaller than or equal to end xlog location %X/%X", (uint32)(*start_lsn >> XIDTHIRTYTWO), (uint32)(*start_lsn), (uint32)(*end_lsn >> XIDTHIRTYTWO), (uint32)(*end_lsn)))); } - +//获取当前数据库日志的最小LSN，用于作为起始LSN的参考值 static XLogRecPtr GetMinLSN() { + //获取最后一个已移除的日志段号 XLogSegNo lastRemovedSegNo = XLogGetLastRemovedSegno(); + //计算当前LSN XLogRecPtr current_recptr = (lastRemovedSegNo + 1) * XLogSegSize; + //返回当前LSN return current_recptr; } - +//获取当前数据库日志的最大LSN，用于作为结束LSN的参考值 static XLogRecPtr GetMaxLSN() { + //如果正在进行恢复，则抛出错误并提示无法获取最大LSN if (RecoveryInProgress()) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("recovery is in progress"), errhint("Can't get local max LSN during recovery."))); + //获取当前日志记录的写入指针位置 XLogRecPtr current_recptr = GetXLogWriteRecPtr(); + //返回当前日志记录的写入指针位置，即最大LSN return current_recptr; } - +//将XLog记录信息输出到字符串 static void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput) { errno_t rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, @@ -118,8 +142,10 @@ static void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput) StringInfoData buf; initStringInfo(&buf); RmgrTable[XLogRecGetRmid(record)].rm_desc(&buf, record); + //输出块引用的信息 rc = strcat_s(strOutput, MAXOUTPUTLEN, buf.data); securec_check(rc, "\0", "\0"); + //如果是bucket文件节点，输出bucketNode信息 if (!XLogRecHasAnyBlockRefs(record)) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "\n\n"); securec_check(rc, "\0", "\0"); @@ -154,6 +180,7 @@ static void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput) rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, ", blk %u", blk); securec_check_ss(rc, "\0", "\0"); /* others: lastlsn %X/%X" */ + // 输出lastlsn信息 XLogRecPtr lsn; XLogRecGetBlockLastLsn(record, block_id, &lsn); rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, ", lastlsn %X/%X", @@ -166,58 +193,89 @@ static void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput) void CheckOpenFile(FILE *outputfile, char *outputFilename) { + // 检查文件打开操作的结果 if (outputfile == NULL) ereport(ERROR, (errcode(ERRCODE_FILE_READ_FAILED), (errmsg("Cannot read %s", outputFilename)))); } void CheckWriteFile(int result, int cnt_len, char *outputFilename) { + // 检查文件写入操作的结果 if (result != cnt_len) ereport(ERROR, (errcode(ERRCODE_FILE_WRITE_FAILED), (errmsg("Cannot write %s", outputFilename)))); } void CheckCloseFile(int result, char *outputFilename) { + //检查文件关闭操作的结果 if (0 != result) ereport(ERROR, (errcode(ERRCODE_IO_ERROR), (errmsg("Cannot close %s", outputFilename)))); } - +//检查当前XLog记录是否符合一组给定的筛选条件 static bool CheckValidRecord(XLogReaderState *xlogreader_state, XLogFilter *filter) { - bool found = false; + bool found = false;//初始化 found 变量，表示是否找到满足条件的块 + //循环遍历 XLog 中的每个块 for (int i = 0; i <= xlogreader_state->max_block_id; i++) { RelFileNode rnode; ForkNumber forknum; BlockNumber blk; + //获取当前块的标记 if (!XLogRecGetBlockTag(xlogreader_state, i, &rnode, &forknum, &blk)) - continue; + continue;//如果返回 false，则说明没有找到块标记，跳过该块继续循环 + //检查块标记是否符合查询条件 if (RelFileNodeEquals(rnode, filter->by_relfilenode)) /* if equal to specific block or check all blocks, found = ture; */ found = (!filter->by_block) || (blk == filter->blocknum); + //如果找到一个符合条件的块，则返回true表示当前记录有效 if (found) { return found; } } + //没有找到任何符合条件的块，则返回false表示当前记录无效 return found; } - +//在一系列连续的日志记录中查找下一个有效的日志记录的LSN static XLogRecPtr UpdateNextLSN(XLogRecPtr cur_lsn, XLogRecPtr end_lsn, XLogReaderState *xlogreader_state, bool *found) { - XLogRecPtr next_record = InvalidXLogRecPtr; + XLogRecPtr next_record = InvalidXLogRecPtr;// 初始化 next_record，表示下一个记录的LSN + // 使用循环尝试多次查找下一个记录的LSN for (int tryTimes = 0; tryTimes < FIVE; tryTimes++) { + // 计算起始 LSN XLogRecPtr start_lsn = Max(cur_lsn, (g_instance.comm_cxt.predo_cxt.redoPf.oldest_segment) * XLogSegSize); + // 查找下一个记录的LSN next_record = XLogFindNextRecord(xlogreader_state, start_lsn); + // 检查找到的下一个记录的LSN是否有效并且小于结束LSN if (!XLByteEQ(next_record, InvalidXLogRecPtr) && XLByteLT(next_record, end_lsn)) { - *found = true; - return next_record; + *found = true;// 设置 found 为 true，表示找到了下一个记录的LSN + return next_record;// 返回找到的下一个记录的LSN } } + // 如果没有找到下一个记录的LSN，则返回 InvalidXLogRecPtr return next_record; } +/* +start_lsn: 起始位置的LSN（Log Sequence Number） +end_lsn: 结束位置的LSN +filter: 筛选器，用于选择要输出的记录 +outputFilename: 目标输出文件的名称 +readprivate: 一个XLogPrivate类型的结构体，用于初始化XLogReaderState +xlogreader_state: XLogReaderState类型的对象，用于读取WAL文件 +first_record: 第一个有效记录的位置，即起始位置之后的第一个有效记录位置 +valid_start_lsn: 真正处理的起始位置，由于可能找不到第一个有效记录，因此可能与给定的起始位置不同。 +valid_end_lsn: 当前读取的有效记录的结束位置 +outputfile: 目标输出文件的指针 +strOutput: 字符串缓冲区，用于记录输出信息 +count: 输出记录的计数器 +errormsg: 出错时的错误信息 +record: XLogRecord类型的对象，表示读取的记录 +*/ +//将WAL（Write-Ahead Log）文件中的记录转储到指定的输出文件中 static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filter, char *outputFilename) { /* start reading */ + // 初始化XLogReaderState，用于读取WAL文件 errno_t rc = EOK; XLogPrivate readprivate; rc = memset_s(&readprivate, sizeof(XLogPrivate), 0, sizeof(XLogPrivate)); @@ -230,9 +288,11 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte (errmsg("memory is temporarily unavailable while allocate xlog reader")))); /* get the first valid xlog record location */ + // 找到起始位置之后的第一个有效记录位置 XLogRecPtr first_record = XLogFindNextRecord(xlogreader_state, start_lsn); /* if we are recycling or removing log files concurrently, we can't find the next record right after. * Hence, we need to update the min_lsn */ + // 如果找不到第一个有效记录，则更新min_lsn，并再次尝试查找 if (XLByteEQ(first_record, InvalidXLogRecPtr)) { ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("XLogFindNextRecord: could not find a valid record after %X/%X. Retry.", @@ -249,9 +309,10 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte XLogRecPtr valid_start_lsn = first_record; XLogRecPtr valid_end_lsn = valid_start_lsn; - + // 打开目标输出文件 FILE *outputfile = fopen(outputFilename, "w"); CheckOpenFile(outputfile, outputFilename); + // 输出一些转储信息 char *strOutput = (char *)palloc(MAXOUTPUTLEN * sizeof(char)); rc = memset_s(strOutput, MAXOUTPUTLEN, 0, MAXOUTPUTLEN); securec_check(rc, "\0", "\0"); @@ -269,12 +330,14 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte int count = 0; char *errormsg = NULL; XLogRecord *record = NULL; + // 不断读取WAL记录，直到结束位置 while (XLByteLT(xlogreader_state->EndRecPtr, end_lsn)) { record = XLogReadRecord(xlogreader_state, first_record, &errormsg); valid_end_lsn = xlogreader_state->EndRecPtr; if (!record && XLByteLT(valid_end_lsn, end_lsn)) { /* if we are recycling or removing log files concurrently, and we can't find the next record right after. * In this case, we try to read from the current oldest xlog file. */ + // 如果无法读取下一条记录，则更新min_lsn并继续尝试读取 bool found = false; XLogRecPtr temp_start_lsn = Max(xlogreader_state->EndRecPtr, start_lsn); first_record = UpdateNextLSN(temp_start_lsn, end_lsn, xlogreader_state, &found); @@ -303,6 +366,7 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte break; } first_record = InvalidXLogRecPtr; /* No explicit start point; read the record after the one we just read */ + // 根据过滤器筛选记录，并输出 if (filter->by_xid_enabled && filter->by_xid != record->xl_xid) { continue; } @@ -325,7 +389,7 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte CheckWriteFile(fwrite(strOutput, 1, strlen(strOutput), outputfile), strlen(strOutput), outputFilename); pfree_ext(strOutput); } - + // 输出结束信息并关闭文件 XLogReaderFree(xlogreader_state); strOutput = (char *)palloc(MAXOUTPUTLEN * sizeof(char)); rc = memset_s(strOutput, MAXOUTPUTLEN, 0, MAXOUTPUTLEN); @@ -345,14 +409,20 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte } /* There are only two parameters in PG_FUNCTION_ARGS: start_lsn and end_lsn */ +/* + 这是一个PostgreSQL函数"gs_xlogdump_lsn"。 + 它接收起始LSN和结束LSN作为输入参数，并对指定的WAL文件进行转储。 +*/ Datum gs_xlogdump_lsn(PG_FUNCTION_ARGS) { errno_t rc = EOK; /* check user's right */ + /* 检查用户权限 */ const char fName[MAXFNAMELEN] = "gs_xlogdump_lsn"; CheckUser(fName); /* read in parameters */ + /* 读取并验证起始LSN和结束LSN */ char *start_lsn_str = text_to_cstring(PG_GETARG_TEXT_P(0)); char *end_lsn_str = text_to_cstring(PG_GETARG_TEXT_P(1)); @@ -360,12 +430,21 @@ Datum gs_xlogdump_lsn(PG_FUNCTION_ARGS) XLogRecPtr start_lsn, end_lsn; ValidateStartEndLSN(start_lsn_str, end_lsn_str, &start_lsn, &end_lsn); + /* + 根据起始LSN和结束LSN生成输出文件名 + 这里使用了GenerateOutputFileName函数 + */ + char *outputFilename = (char *)palloc(MAXFILENAME * sizeof(char)); rc = memset_s(outputFilename, MAXFILENAME, 0, MAXFILENAME); securec_check(rc, "\0", "\0"); GenerateOutputFileName(outputFilename, start_lsn_str, end_lsn_str); /* update start_lsn and end_lsn based on cur min_lsn and max_lsn */ + /* + 根据当前系统中的min_lsn和max_lsn更新起始LSN和结束LSN + 这是为了避免转储过程中出现不一致的情况。 + */ XLogRecPtr min_lsn = GetMinLSN(); XLogRecPtr max_lsn = GetMaxLSN(); @@ -375,31 +454,38 @@ Datum gs_xlogdump_lsn(PG_FUNCTION_ARGS) if (XLByteLT(max_lsn, end_lsn)) { end_lsn = max_lsn; } - + /* 初始化一个XLogFilter结构体，用于筛选日志记录 */ XLogFilter filter; rc = memset_s(&filter, sizeof(XLogFilter), 0, sizeof(XLogFilter)); securec_check_ss(rc, "\0", "\0"); + /* 对指定的WAL文件进行转储 */ XLogDump(start_lsn, end_lsn, &filter, outputFilename); + /* 返回输出文件名 */ PG_RETURN_TEXT_P(cstring_to_text(outputFilename)); } /* There are only one parameter in PG_FUNCTION_ARGS: c_xid */ +//基于事务ID对WAL文件进行转储 Datum gs_xlogdump_xid(PG_FUNCTION_ARGS) { errno_t rc = EOK; /* check user's right */ + /* 检查用户是否有权限调用该函数 */ const char fName[MAXFNAMELEN] = "gs_xlogdump_xid"; CheckUser(fName); /* read in parameters */ + /* 读取并验证传入的事务ID */ TransactionId c_xid = PG_GETARG_TRANSACTIONID(0); /* check parameters */ + /* 获取系统当前的最大事务ID，以确保传入的事务ID有效 */ TransactionId topXid = GetTopTransactionId(); if (TransactionIdPrecedes(topXid, c_xid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("Xid should less than or equal to %lu.", topXid)))); /* generate output file name */ + /* 生成输出文件名 */ char *outputFilename = (char *)palloc(MAXFILENAME * sizeof(char)); rc = memset_s(outputFilename, MAXFILENAME, 0, MAXFILENAME); securec_check(rc, "\0", "\0"); @@ -407,22 +493,27 @@ Datum gs_xlogdump_xid(PG_FUNCTION_ARGS) t_thrd.proc_cxt.DataDir, c_xid); securec_check_ss(rc, "\0", "\0"); /* update start_lsn and end_lsn based on cur min_lsn and max_lsn */ + /* 获取最小LSN和最大LSN，并更新转储范围 */ XLogRecPtr min_lsn = GetMinLSN(); XLogRecPtr max_lsn = GetMaxLSN(); + /* 初始化XLogFilter结构体，设定需要根据事务ID来筛选记录 */ XLogFilter filter; rc = memset_s(&filter, sizeof(XLogFilter), 0, sizeof(XLogFilter)); securec_check(rc, "\0", "\0"); filter.by_xid_enabled = true; filter.by_xid = c_xid; + /* 调用XLogDump函数进行转储 */ XLogDump(min_lsn, max_lsn, &filter, outputFilename); + /* 返回输出文件名 */ PG_RETURN_TEXT_P(cstring_to_text(outputFilename)); } - +//用于将指定路径和块号（或所有块）下的 WAL 日志记录转储到输出文件中 /* There are only three parameters in PG_FUNCTION_ARGS: path, blocknum, relation_type */ Datum gs_xlogdump_tablepath(PG_FUNCTION_ARGS) { errno_t rc = EOK; /* check user's right */ + //查用户是否有执行该函数的权限 const char fName[MAXFNAMELEN] = "gs_xlogdump_tablepath"; CheckUser(fName); @@ -431,6 +522,7 @@ Datum gs_xlogdump_tablepath(PG_FUNCTION_ARGS) int64 blocknum = PG_GETARG_INT64(1); char *relation_type = text_to_cstring(PG_GETARG_TEXT_P(2)); /* check parameters */ + //检查块号是否在合法范围内（-1 到 MaxBlockNumber 之间），如果不在范围内，则抛出错误 if (blocknum > MaxBlockNumber || blocknum < -1) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("Blocknum should be between -1 and %u.", MaxBlockNumber)))); @@ -443,7 +535,7 @@ Datum gs_xlogdump_tablepath(PG_FUNCTION_ARGS) securec_check_c(rc, "\0", "\0"); filter.by_tablepath_enabled = true; - if (blocknum == -1) { /* care all blocks */ + if (blocknum == -1) { /* care all blocks */ //示处理所有块 filter.by_block = false; } else { /* only one block */ filter.by_block = true; @@ -452,18 +544,20 @@ Datum gs_xlogdump_tablepath(PG_FUNCTION_ARGS) /* generate output file name */ char *outputFilename = (char *)palloc(MAXFILENAME * sizeof(char)); - rc = memset_s(outputFilename, MAXFILENAME, 0, MAXFILENAME); + rc = memset_s(outputFilename, MAXFILENAME, 0, MAXFILENAME);//确保输出文件名为空字符串 securec_check(rc, "\0", "\0"); - PrepForRead(path, blocknum, relation_type, outputFilename, &(filter.by_relfilenode), false); + PrepForRead(path, blocknum, relation_type, outputFilename, &(filter.by_relfilenode), false);//根据 filter.by_relfilenode验证路径和关联关系类型参数的合法性 ValidateParameterPath(filter.by_relfilenode, path); + //执行转储操作 XLogDump(min_lsn, max_lsn, &filter, outputFilename); - PG_RETURN_TEXT_P(cstring_to_text(outputFilename)); + PG_RETURN_TEXT_P(cstring_to_text(outputFilename));//将输出文件名转换为 PostgreSQL 文本类型 } - +//解析和转储数据库中的WAL（Write-Ahead Log）日志记录和数据页 Datum gs_xlogdump_parsepage_tablepath(PG_FUNCTION_ARGS) { /* check user's right */ + //检查用户权限 const char fName[MAXFNAMELEN] = "gs_xlogdump_parsepage_tablepath"; CheckUser(fName); @@ -475,6 +569,7 @@ Datum gs_xlogdump_parsepage_tablepath(PG_FUNCTION_ARGS) int rc = -1; /* page part, copy path since for segment we will edit path */ + //读取传入的函数参数，并进行相应的数据类型转换 char *path_cpy = (char *)palloc(MAXFILENAME + 1); rc = strcpy_s(path_cpy, MAXFILENAME + 1, path); securec_check(rc, "\0", "\0"); @@ -482,7 +577,7 @@ Datum gs_xlogdump_parsepage_tablepath(PG_FUNCTION_ARGS) /* In order to avoid querying the shared buffer and applying LW locks, blocking the business. */ /* In the case of finding all pages, force to check disk */ if (blocknum == -1) { - read_memory = false; + read_memory = false;//强制从磁盘读取数据 } char *outputFilenamePage = ParsePage(path_cpy, blocknum, relation_type, read_memory); pfree_ext(path_cpy); @@ -504,10 +599,11 @@ Datum gs_xlogdump_parsepage_tablepath(PG_FUNCTION_ARGS) rc = memset_s(outputFilename, MAXFILENAME, 0, MAXFILENAME); securec_check(rc, "\0", "\0"); PrepForRead(path, blocknum, relation_type, outputFilename, &(filter.by_relfilenode), false); + //获取最小和最大 LSN 并调用 XLogDump 函数将符合筛选条件的 WAL 日志记录转储到输出文件中。 XLogRecPtr min_lsn = GetMinLSN(); XLogRecPtr max_lsn = GetMaxLSN(); XLogDump(min_lsn, max_lsn, &filter, outputFilename); - + //计算生成结果字符串所需的长度，并为结果字符串分配内存 int outputLen = strlen(outputFilename) + strlen(outputFilenamePage) + 100; if (outputLen <= 0) ereport(ERROR, (errcode(ERRCODE_IO_ERROR), (errmsg("Cannot generate right output file name.")))); -- 2.34.1 From 2d539e3df3b2e51b91e9857040d9ad9f601f6a4c Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Mon, 2 Oct 2023 19:57:53 +0800 Subject: [PATCH 17/48] Update itemptr.cpp --- src/gausskernel/storage/page/itemptr.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/storage/page/itemptr.cpp b/src/gausskernel/storage/page/itemptr.cpp index 4f3ab240f..b56238689 100644 --- a/src/gausskernel/storage/page/itemptr.cpp +++ b/src/gausskernel/storage/page/itemptr.cpp @@ -13,6 +13,7 @@ * * ------------------------------------------------------------------------- */ +//这些函数提供了对OpenGauss数据库中物理地址指针进行比较和判断相等的功能，用于支持索引和有序集合的操作 #include "postgres.h" #include "knl/knl_variable.h" @@ -26,15 +27,18 @@ * Note: * Asserts that the disk item pointers are both valid! */ +//实现了处理OpenGauss数据库中的物理地址指针（ItemPointer）的功能 bool ItemPointerEquals(ItemPointer pointer1, ItemPointer pointer2) { + //比较两个物理地址指针 pointer1 和 pointer2 是否指向相同的数据库条目 + //通过比较两个指针的块号和偏移量来判断它们是否相等 if (ItemPointerGetBlockNumber(pointer1) == ItemPointerGetBlockNumber(pointer2) && ItemPointerGetOffsetNumber(pointer1) == ItemPointerGetOffsetNumber(pointer2)) return true; else return false; } - +//与上一个函数功能相同，但不进行有效性检查。它假定传入的物理地址指针是有效的。 bool ItemPointerEqualsNoCheck(ItemPointer pointer1, ItemPointer pointer2) { if ((ItemPointerGetBlockNumberNoCheck(pointer1) == ItemPointerGetBlockNumberNoCheck(pointer2)) && @@ -48,6 +52,7 @@ bool ItemPointerEqualsNoCheck(ItemPointer pointer1, ItemPointer pointer2) * ItemPointerCompare * Generic btree-style comparison for item pointers. */ +//实现一种通用的比较方法，在索引操作和有序集合操作中对物理地址指针（ItemPointer）进行比较 int32 ItemPointerCompare(ItemPointer arg1, ItemPointer arg2) { /* @@ -61,6 +66,7 @@ int32 ItemPointerCompare(ItemPointer arg1, ItemPointer arg2) return -1; else if (b1 > b2) return 1; + //如果块号相同，则按位置标识符进行比较 else if (arg1->ip_posid < arg2->ip_posid) return -1; else if (arg1->ip_posid > arg2->ip_posid) -- 2.34.1 From c72ec6b588ba22c48c2bc55851fa13c5f016de49 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Mon, 2 Oct 2023 19:58:24 +0800 Subject: [PATCH 18/48] Update pagecompress.cpp --- src/gausskernel/storage/page/pagecompress.cpp | 5582 +++++++++-------- 1 file changed, 2947 insertions(+), 2635 deletions(-) diff --git a/src/gausskernel/storage/page/pagecompress.cpp b/src/gausskernel/storage/page/pagecompress.cpp index f1d2112e0..e4c621788 100644 --- a/src/gausskernel/storage/page/pagecompress.cpp +++ b/src/gausskernel/storage/page/pagecompress.cpp @@ -1,2635 +1,2947 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * pagecompress.cpp - * page compression manager routines - * - * - * IDENTIFICATION - * src/gausskernel/storage/page/pagecompress.cpp - * - * ------------------------------------------------------------------------- - */ -#include "postgres.h" -#include "knl/knl_variable.h" -#include "storage/pagecompress.h" -#include "utils/memutils.h" -#include "utils/numeric.h" -#include "utils/numeric_gs.h" -#include "utils/dynahash.h" -#include "access/hash.h" -#include "access/tuptoaster.h" -#include "access/tableam.h" -#include "utils/rel.h" -#include "utils/rel_gs.h" - -static int int16Compare(Datum arg1, Datum arg2); -static int int32Compare(Datum arg1, Datum arg2); -static int uint32Compare(Datum arg1, Datum arg2); -static int int64Compare(Datum arg1, Datum arg2); - -static int64 int16Minus(Datum arg1, Datum arg2); -static int64 int32Minus(Datum arg1, Datum arg2); -static int64 uint32Minus(Datum arg1, Datum arg2); -static int64 int64Minus(Datum arg1, Datum arg2); - -static Datum int16Plus(Datum arg1, int64 arg2); -static Datum int32Plus(Datum arg1, int64 arg2); -static Datum uint32Plus(Datum arg1, int64 arg2); -static Datum int64Plus(Datum arg1, int64 arg2); - -static const NumberTypeOpt* binarySearch(Oid typeOid); - -/* const array for number type. */ -static const NumberTypeOpt g_number_types[] = { - { 20, int64Compare, int64Minus, int64Plus }, /* int8 */ - { 21, int16Compare, int16Minus, int16Plus }, /* int2 */ - { 23, int32Compare, int32Minus, int32Plus }, /* int4 */ - { 26, uint32Compare, uint32Minus, uint32Plus }, /* oid */ - { 1082, int32Compare, int32Minus, int32Plus }, /* date */ - { 1083, int64Compare, int64Minus, int64Plus }, /* time */ - { 1114, int64Compare, int64Minus, int64Plus }, /* timestamp */ - { 1184, int64Compare, int64Minus, int64Plus }, /* timestamptz */ - { 12089, int64Compare, int64Minus, int64Plus }, /* time_stamp */ -}; -static const int g_number_type_count = sizeof(g_number_types) / sizeof(NumberTypeOpt); - -/* Map about max values in byte for delta compression */ -static const int64 g_max_value_in_bytes[] = {0x0000000000000000, - 0x00000000000000FF, - 0x000000000000FFFF, - 0x0000000000FFFFFF, - 0x00000000FFFFFFFF, - 0x000000FFFFFFFFFF, - 0x0000FFFFFFFFFFFF, - 0x00FFFFFFFFFFFFFF - }; -static const int64 g_number_max_value_in_bytes = (int64)(sizeof(g_max_value_in_bytes) / sizeof(int64)); - -#define getTupleSpace(_tuple) (sizeof(ItemIdData) + MAXALIGN((_tuple)->t_len)) - -/* How many bits must be used at least for storing one delta-based data */ -static uint32 getBitsOfDelta(uint64 delta) -{ - /* guard against too-large input, which would put us into infinite loop */ - if (delta > ULLONG_MAX / 2) { - return 64; - } - - uint32 i = 0; - for (uint64 limit = 1; limit < delta; limit <<= 1) { - i++; - } - - return i; -} - -/* Given the number of bits, compute the size of bytes. */ -#define getSizeOfBits(_bits) (((_bits) + 7) >> 3) - -/* MAX_PREFIX_LEN limits the length of prefix string */ -#define MAX_PREFIX_LEN 256 - -/* judge whether a char is digital */ -#define isDigital(_ch) (((_ch) >= '0') && ((_ch) <= '9')) - -/* judge whether it's a taost tuple */ -#define isToastTuple(_tup) (HeapTupleHasExternal(_tup) || (_tup)->t_len > TOAST_TUPLE_THRESHOLD) - -/* judge whether this attribute is dropped */ -#define isAttrDropped(_attr) (InvalidOid == ((_attr)->atttypid)) - -/* one data repeats at least such times so that it will be added into dict. */ -#define DICTITEM_MIN_REPEATS 3 -#define DICTITEM_NUM_FACTOR 10 - -/* Limit for Dictionary Compression. - * 1. the max number of dict item is , whose id starts from 0; - * 2. after compress, one attribute is stored by byte; - */ -#define MAX_DICITEMID 256 -#define DICT_ITEMID_SIZE 1 -#define DICT_MISS_THRESHOLD 2 - -static int matchDictItem(const void* left, const void* right, Size keysize) -{ - const DictItemData* leftItem = (DictItemData*)left; - const DictItemData* rightItem = (DictItemData*)right; - Assert(leftItem != NULL && rightItem != NULL); - - /* we just care whether the result is 0 or not. */ - if (leftItem->itemSize != rightItem->itemSize) { - return 1; - } - - return memcmp(leftItem->itemData, rightItem->itemData, leftItem->itemSize); -} - -static uint32 hashDictItem(const void* key, Size keysize) -{ - const DictItemData* item = (const DictItemData*)key; - return DatumGetUInt32(hash_any((const unsigned char*)item->itemData, item->itemSize)); -} - -/* create hash table for building dictionary */ -static HTAB* hashCreate(const char* name, MemoryContext memCxt, int nelem, int keysize, int entrysize) -{ - HASHCTL hashCtrl; - errno_t rc = memset_s(&hashCtrl, sizeof(hashCtrl), 0, sizeof(hashCtrl)); - securec_check(rc, "", ""); - hashCtrl.hash = (HashValueFunc)hashDictItem; - hashCtrl.match = (HashCompareFunc)matchDictItem; - hashCtrl.keysize = (Size)keysize; - hashCtrl.entrysize = entrysize; - hashCtrl.hcxt = memCxt; - - int flags = (HASH_FUNCTION | HASH_COMPARE | HASH_ELEM | HASH_CONTEXT); - - return hash_create(name, nelem, &hashCtrl, flags); -} - -void DictCmprMeta::BuildHashTbl(MemoryContext memCxt) -{ - m_hash = hashCreate("DictInfo Hash", memCxt, dictItemNum + 16, sizeof(DictItemData), sizeof(ItemEntry)); - - Assert(dictItemNum <= MAX_DICITEMID); - for (int i = 0; i < dictItemNum; ++i) { - ItemEntry* dicItemEntry = (ItemEntry*)hash_search(m_hash, (const void*)(dictItems + i), HASH_ENTER, NULL); - Assert(dicItemEntry); - dicItemEntry->id = (unsigned char)i; - } -} - -unsigned char DictCmprMeta::GetDictItemId(DictItemData* key, bool& found) -{ - ItemEntry* dicItemEntry = (ItemEntry*)hash_search(m_hash, (const void*)key, HASH_FIND, &found); - if (found) - return dicItemEntry->id; - return 0; -} - -void DictCmprMeta::Destroy() -{ - if (m_hash) { - hash_destroy(m_hash); - m_hash = NULL; - } -} - -/* write all the dict metadata into given buffer */ -void DictCmprMeta::GetDictItems(char* buf) const -{ - int offset = 0; - errno_t rc = EOK; - for (int i = 0; i < dictItemNum; ++i) { - rc = memcpy_s(buf + offset, (size_t)(BLCKSZ - offset), dictItems[i].itemData, (size_t)(dictItems[i].itemSize)); - securec_check(rc, "", ""); - offset += dictItems[i].itemSize; - } -} - -template -Datum PageCompress::NumstrCompress(Datum src, int& outSize) -{ - uint32 evenLen; - int destLen = 0; - uint32 srcLen = 0; - char* srcPtr = NULL; - unsigned char* outBuf = NULL; - unsigned char* destPtr = NULL; - - Assert(attlen == -1 || attlen == -2); - if (attlen == -1) { - char* p = DatumGetPointer(src); - Assert(p != NULL); - srcPtr = VARDATA_ANY(p); - srcLen = VARSIZE_ANY_EXHDR(p); - } else { - srcPtr = DatumGetPointer(src); - srcLen = (uint32)strlen(srcPtr); - } - - /* for empty string, no need to compress */ - if (srcLen == 0) { - outSize = 0; - return 0; - } - - Assert(srcPtr != NULL && srcLen > 0); - - /* - * If the srcLen is odd, we need set the highest 4 bits for 1 for last byte - * in order to distinguish the last byte is one original byte - */ - unsigned char* srcData = (unsigned char*)srcPtr; - if (srcLen % 2 != 0) { - if (!isDigital(srcData[srcLen - 1])) { - outSize = 0; - return 0; - } - - evenLen = srcLen - 1; - destLen = (int)((evenLen >> 1) + 1); - } else { - evenLen = srcLen; - destLen = (int)(evenLen >> 1); - } - Assert(destLen >= 0); - - if (attlen == -1) { - outBuf = (unsigned char*)palloc((Size)(destLen + VARHDRSZ)); - if (destLen + VARHDRSZ_SHORT <= VARATT_SHORT_MAX) { - outSize = (int)(destLen + VARHDRSZ_SHORT); - SET_VARSIZE_SHORT(outBuf, outSize); - destPtr = outBuf + VARHDRSZ_SHORT; - } else { - outSize = destLen + VARHDRSZ; - SET_VARSIZE(outBuf, outSize); - destPtr = outBuf + VARHDRSZ; - } - } else { - /* Last byte has been set '\0' */ - outSize = destLen + 1; - destPtr = outBuf = (unsigned char*)palloc0((Size)outSize); - } - - /* Two bytes combine one byte */ - int pos = 0; - for (uint32 i = 0; i < evenLen; i = i + 2) { - if (isDigital(srcData[i]) && isDigital(srcData[i + 1])) { - destPtr[pos] = (srcData[i] - '0') | (((unsigned int)(srcData[i + 1] - '0')) << 4); - /* - * If attlen is -2, we can't use 0 represent '0''0', because '\0' is 0 - * So we guarantee all code > 0 - */ - destPtr[pos] += 1; - ++pos; - } else { - pfree(outBuf); - outSize = 0; - return (Datum)0; - } - } - - /* Deal with last byte if we need */ - if (srcLen > evenLen) { - destPtr[pos++] = (srcData[srcLen - 1] - '0') | 0xF0; - } - Assert(pos == destLen); - - return PointerGetDatum(outBuf); -} - -Datum PageCompress::NumstrUncompress( - Datum src, int attlen, int& cmprValSize, char* uncmprValBuf, int* dataLenWithPadding, char attalign) -{ - Assert(attlen == -1 || attlen == -2); - - uint32 srcLen = 0; - char* srcPtr = NULL; - if (attlen == -1) { - char* p = DatumGetPointer(src); - Assert(p != NULL); - srcPtr = VARDATA_ANY(p); - srcLen = VARSIZE_ANY_EXHDR(p); - cmprValSize = (int)(VARSIZE_ANY(src)); - } else { - srcPtr = DatumGetPointer(src); - srcLen = (uint32)strlen(srcPtr); - cmprValSize = (int)(srcLen + 1); - } - Assert(srcPtr != NULL && srcLen > 0 && cmprValSize > 0); - - unsigned char* outBuf = NULL; - unsigned char* destPtr = NULL; - unsigned char* srcData = (unsigned char*)srcPtr; - int destLen; - int padding_size = 0; - int datum_size = 0; - unsigned char lastByte = srcData[srcLen - 1]; - - if ((lastByte & 0xF0) == 0xF0) - destLen = (int)(((srcLen - 1) << 1) + 1); - else - destLen = (int)(srcLen << 1); - Assert(destLen > 0); - - if (attlen == -1) { - /* - * if the datum header datum_size is 4B, - * 1. aligned is required. - 2. the first byte of padding must be set 0 if padding exists. - * NB: nothing to do if the datum header is 1B; - */ - if (uncmprValBuf) { - outBuf = (unsigned char*)uncmprValBuf; - - Assert(dataLenWithPadding != NULL); - *outBuf = 0; - outBuf = (unsigned char*)att_align_nominal(outBuf, attalign); - padding_size = int(outBuf - (unsigned char*)uncmprValBuf); - } else { - outBuf = (unsigned char*)palloc((Size)(destLen + VARHDRSZ)); - } - - datum_size = destLen + VARHDRSZ; - SET_VARSIZE(outBuf, datum_size); - destPtr = outBuf + VARHDRSZ; - } else { - /* Last byte has been set '\0' */ - datum_size = destLen + 1; - if (uncmprValBuf) - destPtr = outBuf = (unsigned char*)uncmprValBuf; - else - destPtr = outBuf = (unsigned char*)palloc((Size)datum_size); - - errno_t rc = memset_s(destPtr, (size_t)datum_size, 0, (size_t)datum_size); - securec_check(rc, "", ""); - } - - /* One byte split into two bytes */ - int pos = 0; - for (uint32 i = 0; i < srcLen - 1; ++i) { - destPtr[pos++] = ((srcData[i] - 1) & 0x0F) + '0'; - destPtr[pos++] = (((srcData[i] - 1) & 0xF0) >> 4) + '0'; - } - - /* Deal with last byte if we need */ - if ((lastByte & 0xF0) == 0xF0) - destPtr[pos++] = (lastByte & 0x0F) + '0'; - else { - destPtr[pos++] = ((lastByte - 1) & 0x0F) + '0'; - destPtr[pos++] = (((lastByte - 1) & 0xF0) >> 4) + '0'; - } - - if (dataLenWithPadding) { - Assert(uncmprValBuf != NULL); - Assert(padding_size >= 0 && datum_size > 0); - *dataLenWithPadding = padding_size + datum_size; - } - - return PointerGetDatum(outBuf); -} - -PageCompress::PageCompress(Relation rel, MemoryContext memCtx) -{ - m_rel = rel; - m_inTups = NULL; - m_inTupsNum = 0; - m_current = 0; - m_outputTupsNum = 0; - m_mappedTupsNum = 0; - m_uncmprTupIdx = 0; - m_cmprTupsNum = 0; - m_toastTupsNum = 0; - m_cmprHeaderSize = 0; - m_outputTups = NULL; - m_cmprTups = NULL; - m_toastTups = NULL; - m_mappedTups = NULL; - m_nextBatch = false; - m_last = false; - - /* these member will be resident in parent memory context. */ - MemoryContext oldMemCnxt = MemoryContextSwitchTo(memCtx); - - m_parentMemCxt = memCtx; - m_cmprHeaderData = (char*)palloc(BLCKSZ); - m_pageMemCnxt = AllocSetContextCreate( - memCtx, "PageCompression", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); - - int nattrs = RelationGetNumberOfAttributes(m_rel); - m_cmprMode = (char*)palloc0((Size)(nattrs * sizeof(char))); - m_cmprMeta = (void**)palloc0((Size)(nattrs * sizeof(void*))); - m_numTypeOpt = (const NumberTypeOpt**)palloc((Size)(nattrs * sizeof(NumberTypeOpt*))); - m_dictMiss = (uint32*)palloc0((Size)(nattrs * sizeof(uint32))); - m_dictItemId = (unsigned char*)palloc((Size)(nattrs * sizeof(unsigned char))); - m_deformVals = (Datum*)palloc((Size)(nattrs * sizeof(Datum))); - m_deformNulls = (bool*)palloc((Size)(nattrs * sizeof(bool))); - - (void)MemoryContextSwitchTo(oldMemCnxt); - - Form_pg_attribute* attrs = m_rel->rd_att->attrs; - for (int col = 0; col < nattrs; ++col) { - m_numTypeOpt[col] = binarySearch(attrs[col]->atttypid); - } - -#ifdef TRACE_COMPRESS - nLoops = 0; -#endif -} - -PageCompress::~PageCompress() -{ - ResetPerPage(); - pfree_ext(m_cmprHeaderData); - pfree_ext(m_cmprMode); - pfree_ext(m_cmprMeta); - pfree_ext(m_numTypeOpt); - pfree_ext(m_dictItemId); - pfree_ext(m_dictMiss); - pfree_ext(m_deformVals); - pfree_ext(m_deformNulls); - pfree_ext(m_cmprTups); - pfree_ext(m_toastTups); - pfree_ext(m_mappedTups); - m_inTups = NULL; - m_rel = NULL; - m_parentMemCxt = NULL; - - MemoryContextDelete(m_pageMemCnxt); - m_pageMemCnxt = NULL; -} - -ColArray* PageCompress::DeformTups(HeapTuple* tups, int nTups) -{ - int attrNum = RelationGetNumberOfAttributes(m_rel); - TupleDesc desc = RelationGetDescr(m_rel); - - ColArray* colArray = (ColArray*)palloc((Size)(sizeof(ColArray) * attrNum)); - - for (int i = 0; i < attrNum; ++i) { - colArray[i].nulls = (bool*)palloc0((Size)(sizeof(bool) * nTups)); - colArray[i].val = (Datum*)palloc0((Size)(sizeof(Datum) * nTups)); - colArray[i].num = nTups; - } - - /* storage format is as followings: - * - * {col_1, row_1} {col_1, row_2} {col_1, row_3} {col_1, row_4} ... - * {col_2, row_1} {col_2, row_2} {col_2, row_3} {col_2, row_4} ... - * {col_3, row_1} {col_3, row_2} {col_3, row_3} {col_3, row_4} ... - * .... - * .... - */ - for (int row = 0; row < nTups; ++row) { - heap_deform_tuple(tups[row], desc, m_deformVals, m_deformNulls); - - for (int col = 0; col < attrNum; ++col) { - colArray[col].val[row] = m_deformVals[col]; - colArray[col].nulls[row] = m_deformNulls[col]; - } - } - - return colArray; -} - -/* - * --------------------------------------------------------------- - * Delta Compression/Decompression Methods - * --------------------------------------------------------------- - */ -DeltaCmprMeta* PageCompress::DeltaGetCmprMeta(ColArray* colArray, const NumberTypeOpt* opt) const -{ - Datum minVal, maxVal; - Datum* values = colArray->val; - bool* isnulls = colArray->nulls; - - /* init min/max value. */ - int i = 0; - while (i < colArray->num && isnulls[i]) - ++i; - /* if all the values are NULLs, return NULL */ - if (i == colArray->num) - return NULL; - - /* loop and get the min/max value in colArray. */ - maxVal = minVal = values[i]; - for (i++; i < colArray->num; i++) { - if (isnulls[i]) - continue; - - if (opt->compare(minVal, values[i]) > 0) - minVal = values[i]; - else if (opt->compare(maxVal, values[i]) < 0) - maxVal = values[i]; - } - - int64 diff = opt->minus(maxVal, minVal); - int bytes = (int)getSizeOfBits(getBitsOfDelta((uint64)diff)); - if (bytes >= g_number_max_value_in_bytes) { - return NULL; - } - - DeltaCmprMeta* deltaInfo = (DeltaCmprMeta*)palloc(sizeof(DeltaCmprMeta)); - deltaInfo->MinVal = minVal; - deltaInfo->MaxVal = maxVal; - deltaInfo->bytes = bytes; - - return deltaInfo; -} - -/* - * Given both delta meta and a value, compute and return its delta data. - * returning true means that delta compress successes, and deltaBuf is valid whose size would be - * computed by metaInfo->bytes. - * metaInfo, opt and val are the input parameters; and deltaBuf is the output - * returning false means that delta compress fails. - */ -bool PageCompress::DeltaCompress( - DeltaCmprMeta* metaInfo, const NumberTypeOpt* opt, Datum val, unsigned char* deltaBuf) const -{ - int64 diff = opt->minus(val, metaInfo->MinVal); - - /* - * if this diff is negative, don't compress; - * if this diff is out of range, don't compress; - */ - Assert(metaInfo->bytes < g_number_max_value_in_bytes); - if ((diff < 0) || (diff > g_max_value_in_bytes[metaInfo->bytes])) { - return false; - } - - /* - * until here, we know that: - * 1. diff is greater than 0; - * 2. the highest byte is 0x00, yes, at most 7 bytes is used and valid. - * 3. diff can be changed to uint64 safely. - */ - Assert((diff >= 0) && ((diff >> 56) == 0x00)); - - uint64 udiff = (uint64)diff; - for (int i = metaInfo->bytes - 1; i >= 0; i--) { - deltaBuf[i] = (udiff & 0xFF); - udiff = udiff >> 8; - } - - return true; -} - -/* [IN]: metaInfo && opt && deltaBuf[] - * delta decompress. - */ -Datum PageCompress::DeltaUncompress(DeltaCmprMeta* metaInfo, const NumberTypeOpt* opt, unsigned char* deltaBuf) -{ - uint64 diff = 0; - unsigned char* delta = (unsigned char*)deltaBuf; - - diff |= delta[0]; - for (int i = 1; i < metaInfo->bytes; i++) { - diff = (diff << 8) | delta[i]; - } - - Assert((diff >> 56) == 0x00); - return opt->plus(metaInfo->MinVal, (int64)diff); -} - -Datum PageCompress::DictUncompress(DictCmprMeta* metaInfo, const char* compressBuf, int attlen) -{ - Datum res = 0; - unsigned char id = *((unsigned char*)compressBuf); - DictItemData* val = metaInfo->dictItems + id; - - /* refer to fetch_att() */ - if (attlen > 0) { - switch (attlen) { - case sizeof(char): { - Assert(0); - break; - } - case sizeof(int16): { - res = Int16GetDatum(*(int16*)val->itemData); - break; - } - case sizeof(int32): { - res = Int32GetDatum(*(int32*)val->itemData); - break; - } - case sizeof(Datum): { - res = *(Datum*)val->itemData; - break; - } - default: - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported byval length: %d", (int)(attlen)))); - } - } else - res = PointerGetDatum(val->itemData); - return res; -} - -/* - * --------------------------------------------------------------- - * Prefix Compression/Decompression Methods - * --------------------------------------------------------------- - */ -template -int PageCompress::PrefixLen(Datum str1, Datum str2) -{ - char* strData1 = NULL; - char* strData2 = NULL; - int maxPrefixLen = 0; - int prefixLen = 0; - - /* Note that numeric column should not do this. */ - if (attlen == -1) { - char* ptr1 = DatumGetPointer(str1); - char* ptr2 = DatumGetPointer(str2); - strData1 = VARDATA_ANY(ptr1); - strData2 = VARDATA_ANY(ptr2); - maxPrefixLen = Min(VARSIZE_ANY_EXHDR(ptr1), VARSIZE_ANY_EXHDR(ptr2)); - } else if (attlen == -2) { - strData1 = DatumGetCString(str1); - strData2 = DatumGetCString(str2); - size_t str_len1 = strlen(strData1); - size_t str_len2 = strlen(strData2); - maxPrefixLen = (int)Min(str_len1, str_len2); - } - - while (prefixLen < maxPrefixLen && strData1[prefixLen] == strData2[prefixLen]) - ++prefixLen; - - return prefixLen; -} - -/* - * Get common prefix from colArray - * This function outputs prefixLen and prefix string - */ -template -char* PageCompress::PrefixGetCmprMeta(ColArray* colArray, int& prefixLen) -{ - int prev = 0; - int next = 0; - int arrayLen = colArray->num; - char* prefixStr = NULL; - - /* Initialize the prefixLen by comparing the first and second tuple */ - while (prev < arrayLen && colArray->nulls[prev]) - ++prev; - next = prev + 1; - - while (next < arrayLen && colArray->nulls[next]) - ++next; - - if (next < arrayLen) { - prefixLen = PrefixLen(colArray->val[prev], colArray->val[next]); - if (prefixLen == 0) { - /* - * if the first two have no common prefix, we don't continue to advance. - * maybe this trick is unreasonalbe. - */ - return NULL; - } - - Assert(prev < arrayLen); - if (attlen == -1) - prefixStr = VARDATA_ANY(DatumGetPointer(colArray->val[prev])); - else - prefixStr = DatumGetCString(colArray->val[prev]); - - /* Loop for geting the final prefixLen */ - prev = next; - for (++next; next < arrayLen; ++next) { - while (next < arrayLen && colArray->nulls[next]) { - ++next; - } - if (next == arrayLen) { - break; - } - - int len = PrefixLen(colArray->val[prev], colArray->val[next]); - if (prefixLen > len && len > 0) { - prefixLen = len; - } - prev = next; - } - } - return prefixStr; -} - -template -bool PageCompress::PrefixCompress(PrefixCmprMeta* prefix, Datum val, Datum& cmprsVal, int& cmprsValSize) -{ - char* attStart = NULL; - int attSize; - - if (-1 == attlen) { - char* ptr = DatumGetPointer(val); - Assert(ptr != NULL); - attStart = VARDATA_ANY(ptr); - attSize = VARSIZE_ANY_EXHDR(ptr); - } else { - attStart = DatumGetCString(val); - attSize = (int)(strlen(attStart) + 1); - } - Assert(attStart != NULL && attSize >= 0); - Assert(prefix != NULL && prefix->len > 0 && prefix->prefixStr != NULL); - - if (prefix->len > attSize || memcmp(prefix->prefixStr, attStart, (size_t)prefix->len)) { - return false; - } - - if (-1 == attlen) { - int suffixLen = attSize - prefix->len; - Assert(suffixLen >= 0); - char* suffix = (char*)palloc((Size)(suffixLen + VARHDRSZ)); - errno_t errorno = EOK; - - /* use one byte var-header as possible as much */ - if (suffixLen + VARHDRSZ_SHORT <= VARATT_SHORT_MAX) { - SET_VARSIZE_SHORT(suffix, suffixLen + VARHDRSZ_SHORT); - if (suffixLen > 0) { - errorno = memcpy_s((suffix + VARHDRSZ_SHORT), (size_t)suffixLen, (attStart + prefix->len), (size_t)suffixLen); - securec_check(errorno, "\0", "\0"); - } - cmprsValSize = (int)(suffixLen + VARHDRSZ_SHORT); - } else { - SET_VARSIZE(suffix, suffixLen + VARHDRSZ); - errorno = memcpy_s((suffix + VARHDRSZ), (size_t)suffixLen, (attStart + prefix->len), (size_t)suffixLen); - securec_check(errorno, "\0", "\0"); - cmprsValSize = suffixLen + VARHDRSZ; - } - cmprsVal = PointerGetDatum(suffix); - } else { - cmprsVal = CStringGetDatum(attStart + prefix->len); - cmprsValSize = (int)(strlen(attStart + prefix->len) + 1); - } - Assert(cmprsValSize > 0); - - return true; -} - -Datum PageCompress::PrefixUncompress(int attlen, PrefixCmprMeta* prefix, char* start, int* attrCmprsValSize, - char* uncmprValBuf, int* dataLenWithPadding, char attalign) -{ - int attsize; - char* attdata = NULL; - char* attr = NULL; - int datum_size; - int padding_size = 0; - errno_t rc = EOK; - - Assert(-1 == attlen || -2 == attlen); - Assert(prefix != NULL && prefix->len > 0 && prefix->prefixStr != NULL); - Assert(start != NULL && attrCmprsValSize != NULL); - - if (-1 == attlen) { - char* ptr = DatumGetPointer(start); - attdata = VARDATA_ANY(ptr); - attsize = VARSIZE_ANY_EXHDR(ptr); - *attrCmprsValSize = VARSIZE_ANY(ptr); - Assert(attdata != NULL && attsize >= 0 && *attrCmprsValSize > 0); - - int attRawLen = prefix->len + attsize; - - /* - * if the datum header datum_size is 4B, - * 1. aligned is required. - * 2. the first byte of padding must be set 0 if padding exists. - * NB: nothing to do if the datum header is 1B; - */ - if (uncmprValBuf) { - attr = uncmprValBuf; - - Assert(dataLenWithPadding != NULL); - *attr = 0; - attr = (char*)att_align_nominal(attr, attalign); - padding_size = (int)(attr - uncmprValBuf); - } else { - attr = (char*)palloc((Size)(attRawLen + VARHDRSZ)); - } - - datum_size = attRawLen + VARHDRSZ; - SET_VARSIZE(attr, datum_size); - int offset = VARHDRSZ; - rc = memcpy_s((attr + offset), (size_t)(datum_size - offset), prefix->prefixStr, (size_t)prefix->len); - securec_check(rc, "", ""); - offset += prefix->len; - if (attsize > 0) { - rc = memcpy_s((attr + offset), (size_t)(datum_size - offset), attdata, (size_t)attsize); - securec_check(rc, "", ""); - } - } else { - /* compressed attribute's info */ - attdata = DatumGetCString(start); - attsize = (int)strlen(attdata); - *attrCmprsValSize = attsize + 1; - Assert(attdata != NULL && attsize >= 0 && *attrCmprsValSize > 0); - - datum_size = prefix->len + attsize + 1; - if (uncmprValBuf) - attr = uncmprValBuf; - else - attr = (char*)palloc((Size)datum_size); - rc = memcpy_s(attr, (size_t)datum_size, prefix->prefixStr, (size_t)prefix->len); - securec_check(rc, "", ""); - if (attsize > 0) { - rc = memcpy_s((attr + prefix->len), datum_size - prefix->len, attdata, attsize); - securec_check(rc, "", ""); - } - attr[datum_size - 1] = '\0'; - } - - if (dataLenWithPadding) { - Assert(uncmprValBuf != NULL); - Assert(padding_size >= 0 && datum_size > 0); - *dataLenWithPadding = padding_size + datum_size; - } - - return PointerGetDatum(attr); -} - -/* choose compression method between delta and dict. - * we prefer delta method to dict method. - */ -void PageCompress::ChooseMethodForFixedLenType(ColArray* colArray, int col) -{ - int hintMode = m_rel->rd_att->attrs[col]->attcmprmode; - - /* first, try to use the user's hint */ - if (SetCmprMethod(colArray, col, hintMode)) - return; - - /* then, try to check whether delta value encoding is proper */ - if (CMPR_DELTA != hintMode && SetCmprMethod(colArray, col, CMPR_DELTA)) - return; - - /* then, try to check dictionary compress method */ - if (CMPR_DICT != hintMode && SetCmprMethod(colArray, col, CMPR_DICT)) - return; - - (void)SetCmprMethod(colArray, col, CMPR_NONE); -} - -template -void PageCompress::FillHashKey(DictItemData& key, const int& attlen, const Datum& attval) -{ - if (attFlag > 0) { - key.itemSize = (int16)attlen; - key.itemData = (char*)palloc((Size)attlen); - store_att_byval(key.itemData, attval, attlen); - } else if (-1 == attFlag) { - key.itemData = DatumGetPointer(attval); - key.itemSize = (int16)VARSIZE_ANY(key.itemData); - } else if (-2 == attFlag) { - key.itemData = DatumGetPointer(attval); - key.itemSize = (int16)(strlen(key.itemData) + 1); - } -} - -/* Choose dictinary items and put it into DictCmprMeta */ -template -DictCmprMeta* PageCompress::DictGetCmprMeta(ColArray* colArray, int col, int attlen) -{ - int i; - - /* - * If the times of failing to choose dictionary compression exceeds - * the threshold, we shouldn't waste CPU to try again. - */ - if (m_dictMiss[col] > DICT_MISS_THRESHOLD) - return NULL; - - DictCmprMeta* dicInfo = (DictCmprMeta*)palloc0(sizeof(DictCmprMeta)); - int pos = 0; - bool found = false; - HTAB* dictHash = - hashCreate("DictHash", m_pageMemCnxt, colArray->num + 16, sizeof(DictItemData), sizeof(DictItemEntry)); - - for (i = 0; i < colArray->num; ++i) { - if (colArray->nulls[i]) { - continue; - } - - DictItemData key; - if (attFlag > 0) { - FillHashKey<1>(key, attlen, colArray->val[i]); - } else if (attFlag == -1) { - FillHashKey<-1>(key, attlen, colArray->val[i]); - } else if (attFlag == -2) { - FillHashKey<-2>(key, attlen, colArray->val[i]); - } - - DictItemEntry* dicItemEntry = (DictItemEntry*)hash_search(dictHash, (const void*)&key, HASH_ENTER, &found); - Assert(dicItemEntry != NULL); - if (!found) { - dicItemEntry->hitCount = 1; - dicItemEntry->beChoose = false; - continue; - } - ++dicItemEntry->hitCount; - - /* - * If the repeated times exceeds DICTITEM_NUM_FACTOR, then we should - * choose it as dictionary item - * Description: use a priority queue to keep Top N dictionary item by hitCount - */ - if (!dicItemEntry->beChoose && - (dicItemEntry->hitCount >= Max(colArray->num / DICTITEM_NUM_FACTOR, DICTITEM_MIN_REPEATS))) { - dicItemEntry->beChoose = true; - - dicInfo->dictItems[pos++] = key; - dicInfo->dictItemNum = pos; - dicInfo->totalBytes += key.itemSize; - Assert(dicInfo->dictItemNum <= (colArray->num >> 1)); - - if (dicInfo->dictItemNum >= MAX_DICITEMID) - break; - } - } - - hash_destroy(dictHash); - if (pos > 0) - return dicInfo; - - ++m_dictMiss[col]; - pfree(dicInfo); - return NULL; -} - -template -bool PageCompress::NumstrEvaluate(ColArray* colArray) -{ - int n = colArray->num; - int numStr = 0; - bool flag = true; - Datum* val = colArray->val; - bool* nulls = colArray->nulls; - - for (int i = 0; i < n; ++i) { - if (nulls[i]) - continue; - - int len = 0; - char* dataPtr = NULL; - - if (attlen == -1) { - char* p = DatumGetPointer(val[i]); - dataPtr = VARDATA_ANY(p); - len = VARSIZE_ANY_EXHDR(p); - } else if (attlen == -2) { - dataPtr = DatumGetPointer(val[i]); - len = (int)strlen(dataPtr); - } - - /* do not count empty string */ - if (len == 0) { - continue; - } - - for (int j = 0; j < len; ++j) { - if (!isDigital(dataPtr[j])) { - flag = false; - break; - } - } - - if (flag == false) - break; - - /* - * a little trick: we needn't check all the values, and if the - * the first some values are suitable for compression, retrun - * and try to compress them later. - */ - if (++numStr > 2) { - flag = true; - break; - } - } - - /* no need to compress if all sample values are null or empty string */ - return (0 == numStr) ? false : flag; -} - -/* choose compression method between prefix and dict. - * we prefer prefix method to dict method. - */ -void PageCompress::ChooseMethodForVaryLenType(ColArray* colArray, int col) -{ - int hintMode = m_rel->rd_att->attrs[col]->attcmprmode; - - /* first, try to use the user's hint */ - if (SetCmprMethod(colArray, col, hintMode)) - return; - - /* then, try to check dictionary compress method */ - if (CMPR_DICT != hintMode && SetCmprMethod(colArray, col, CMPR_DICT)) - return; - - /* - * Try to get prefix. - * Note that numeric column should not do this - */ - if (CMPR_PREFIX != hintMode && SetCmprMethod(colArray, col, CMPR_PREFIX)) - return; - - /* then, try to number-string compress method */ - if (CMPR_NUMSTR != hintMode && SetCmprMethod(colArray, col, CMPR_NUMSTR)) - return; - - (void)SetCmprMethod(colArray, col, CMPR_NONE); -} - -/* - * if user has specify the compress method by CREATE TABLE sql clause, - * we take it as a hint, and first try to get compression metadata using it. - * but maybe it fails, and we have to try the other methods then. - */ -bool PageCompress::SetCmprMethod(ColArray* colArray, int col, int mode) -{ - Form_pg_attribute* attrs = m_rel->rd_att->attrs; - int attlen = attrs[col]->attlen; - - switch (mode) { - case CMPR_DELTA: { - Assert(m_numTypeOpt[col] != NULL && attlen > 1); - DeltaCmprMeta* deltaInfo = DeltaGetCmprMeta(colArray, m_numTypeOpt[col]); - - if (deltaInfo == NULL) - return false; - - /* - * this if condition tells us that it's always true that - * deltaInfo->bytes < 8 - */ - Assert(deltaInfo->bytes <= g_number_max_value_in_bytes); - if (deltaInfo->bytes >= attlen) { - pfree(deltaInfo); - return false; - } - - m_cmprMode[col] = CMPR_DELTA; - m_cmprMeta[col] = deltaInfo; - return true; - } - - case CMPR_DICT: { - DictCmprMeta* dicInfo = NULL; - if (-2 == attlen) - dicInfo = DictGetCmprMeta<-2>(colArray, col, attlen); - else if (-1 == attlen) - dicInfo = DictGetCmprMeta<-1>(colArray, col, attlen); - else - dicInfo = DictGetCmprMeta<1>(colArray, col, attlen); - - if (dicInfo == NULL) - return false; - - dicInfo->BuildHashTbl(m_pageMemCnxt); - m_cmprMode[col] = CMPR_DICT; - m_cmprMeta[col] = dicInfo; - return true; - } - - case CMPR_PREFIX: { - int prefixLen = 0; - char* prefixStr = NULL; - - Assert(-1 == attlen || -2 == attlen); - if (-2 == attlen) - prefixStr = PrefixGetCmprMeta<-2>(colArray, prefixLen); - else if (-1 == attlen) - prefixStr = PrefixGetCmprMeta<-1>(colArray, prefixLen); - - /* If this array has common prefix, we fill prefix compression meta information */ - if (prefixLen > 0 && prefixLen < MAX_PREFIX_LEN) { - PrefixCmprMeta* prefixInfo = (PrefixCmprMeta*)palloc(sizeof(PrefixCmprMeta)); - prefixInfo->len = prefixLen; - prefixInfo->prefixStr = (char*)palloc((Size)prefixLen); - errno_t rc = memcpy_s(prefixInfo->prefixStr, (size_t)prefixLen, prefixStr, (size_t)prefixLen); - securec_check(rc, "", ""); - - m_cmprMeta[col] = prefixInfo; - m_cmprMode[col] = CMPR_PREFIX; - return true; - } - return false; - } - - case CMPR_NUMSTR: { - Assert(-1 == attlen || -2 == attlen); - bool NumerStringFlag = (-1 == attlen) ? NumstrEvaluate<-1>(colArray) : NumstrEvaluate<-2>(colArray); - if (NumerStringFlag) { - m_cmprMode[col] = CMPR_NUMSTR; - m_cmprMeta[col] = NULL; - return true; - } - return false; - } - - case CMPR_UNDEF: - return false; - - case CMPR_NONE: - default: - m_cmprMode[col] = CMPR_NONE; - m_cmprMeta[col] = NULL; - return true; - } -} - -void PageCompress::ChooseCmprMethod(ColArray* colArrays) -{ - int attrNum = RelationGetNumberOfAttributes(m_rel); - Form_pg_attribute* attrs = m_rel->rd_att->attrs; - - for (int col = 0; col < attrNum; ++col) { - /* continue if this attribute shouldn't be compressed specified by user; */ - if (CMPR_NONE == attrs[col]->attcmprmode) - continue; - - /* continue if this attribute is dropped by user */ - if (isAttrDropped(attrs[col])) { - (void)SetCmprMethod(colArrays + col, col, CMPR_NONE); - continue; - } - if (m_numTypeOpt[col] != NULL) { - Assert(attrs[col]->attlen > 1); - ChooseMethodForFixedLenType(colArrays + col, col); - } else if (attrs[col]->attlen > 0) { - /* These types are not compressed. */ - (void)SetCmprMethod(colArrays + col, col, CMPR_NONE); - } else { - Assert(attrs[col]->attlen == -1 || attrs[col]->attlen == -2); - ChooseMethodForVaryLenType(colArrays + col, col); - } - } -} - -void PageCompress::SetCmprHeaderData(void) -{ - Form_pg_attribute* atts = RelationGetDescr(m_rel)->attrs; - int nattrs = RelationGetNumberOfAttributes(m_rel); - int offset = 0; - int rawCols = 0; - errno_t rc = EOK; - - for (int col = 0; col < nattrs; ++col) { - m_cmprHeaderData[offset++] = m_cmprMode[col]; /* compression mode */ - - switch (m_cmprMode[col]) { - case CMPR_DELTA: { - DeltaCmprMeta* deltaInfo = (DeltaCmprMeta*)m_cmprMeta[col]; - int attrLen = atts[col]->attlen; - Assert(deltaInfo->bytes < g_number_max_value_in_bytes); - Assert(attrLen > 0); - - /* how many bytes is used for delta value */ - m_cmprHeaderData[offset++] = (char)deltaInfo->bytes; - /* the min/base value */ - store_att_byval(m_cmprHeaderData + offset, deltaInfo->MinVal, attrLen); - offset += attrLen; - break; - } - - case CMPR_DICT: { - DictCmprMeta* dicInfo = (DictCmprMeta*)m_cmprMeta[col]; - Assert(dicInfo->dictItemNum < 0xFF); - /* how many dictionary items */ - m_cmprHeaderData[offset++] = (char)dicInfo->dictItemNum; - - dicInfo->GetDictItems(m_cmprHeaderData + offset); - offset += dicInfo->totalBytes; - break; - } - - case CMPR_PREFIX: { - Assert(-2 == atts[col]->attlen || -1 == atts[col]->attlen); - PrefixCmprMeta* prefixInfo = (PrefixCmprMeta*)m_cmprMeta[col]; - Assert(prefixInfo->len > 0 && prefixInfo->len < MAX_PREFIX_LEN); - /* how many bytes is used for prefix string */ - m_cmprHeaderData[offset++] = (char)prefixInfo->len; - /* data of prefix string */ - rc = memcpy_s(m_cmprHeaderData + offset, (size_t)(BLCKSZ - offset), prefixInfo->prefixStr, (size_t)(prefixInfo->len)); - securec_check(rc, "", ""); - offset += prefixInfo->len; - break; - } - - case CMPR_NUMSTR: - break; - - case CMPR_NONE: - ++rawCols; - break; - default: - break; - } - } - - /* - * the same to special space in page, start address of compression header - * is aligned using MAXALIGN(). - */ - m_cmprHeaderSize = (nattrs == rawCols) ? 0 : (int)MAXALIGN((uint32)offset); - Assert(m_cmprHeaderSize < BLCKSZ); -} - -/* fetch and parse the compression meta data, then return meta infomation object. - * metaStart [IN]: start address of compression metadata within page for each attribute; - * attrRawLen [IN]: fixed length of this attribute, it's from pg_attr->attlen; - * metaSize [OUT]: size of compression metadata within page for each attribute; - * mode [OUT]: compression method; - */ -void* PageCompress::FetchAttrCmprMeta(char* metaStart, const int attrRawLen, int* metaSize, char* mode) -{ - int offset = 1; - *mode = metaStart[0]; - void* retInfo = NULL; - - switch (*mode) { - case CMPR_DELTA: { - DeltaCmprMeta* deltaInfo = &t_thrd.storage_cxt.cmprMetaInfo->deltaInfo; - - /* how many bytes is used for delta value */ - deltaInfo->bytes = metaStart[offset]; - ++offset; - - Assert(attrRawLen > 0); - /* read and fetch the min/base value */ - deltaInfo->MinVal = fetch_att(metaStart + offset, true, attrRawLen); - offset += attrRawLen; - *metaSize = offset; - retInfo = (void*)deltaInfo; - break; - } - case CMPR_DICT: { - DictCmprMeta* dicInfo = &t_thrd.storage_cxt.cmprMetaInfo->dicInfo; - dicInfo->dictItemNum = metaStart[offset++]; - - if (attrRawLen > 0) { - for (int i = 0; i < dicInfo->dictItemNum; ++i) { - dicInfo->dictItems[i].itemData = metaStart + offset; - dicInfo->dictItems[i].itemSize = (int16)attrRawLen; - offset += attrRawLen; - } - } else if (attrRawLen == -1) { - for (int i = 0; i < dicInfo->dictItemNum; ++i) { - dicInfo->dictItems[i].itemData = metaStart + offset; - dicInfo->dictItems[i].itemSize = (int16)VARSIZE_ANY(dicInfo->dictItems[i].itemData); - offset += dicInfo->dictItems[i].itemSize; - } - } else if (attrRawLen == -2) { - for (int i = 0; i < dicInfo->dictItemNum; ++i) { - dicInfo->dictItems[i].itemData = metaStart + offset; - dicInfo->dictItems[i].itemSize = (int16)strlen(dicInfo->dictItems[i].itemData) + 1; - offset += dicInfo->dictItems[i].itemSize; - } - } - *metaSize = offset; - retInfo = (void*)dicInfo; - break; - } - - case CMPR_PREFIX: { - Assert(-2 == attrRawLen || -1 == attrRawLen); - PrefixCmprMeta* prefixInfo = &t_thrd.storage_cxt.cmprMetaInfo->prefixInfo; - prefixInfo->len = (metaStart[offset] & 0x00FF); - ++offset; - prefixInfo->prefixStr = metaStart + offset; - offset += prefixInfo->len; - *metaSize = offset; - retInfo = (void*)prefixInfo; - break; - } - - case CMPR_NUMSTR: - case CMPR_NONE: { - *metaSize = offset; - retInfo = (void*)NULL; - break; - } - - default: { - Assert(false); - retInfo = (void*)NULL; - break; - } - } - - return retInfo; -} - -/* according to compression mode and meta information, return the value size - * of this attribute in the tuple. - */ -int PageCompress::GetAttrCmprValSize(char mode, int attlen, void* metaInfo, const char* attrStart) -{ - switch (mode) { - case CMPR_DELTA: { - DeltaCmprMeta* deltaInfo = (DeltaCmprMeta*)metaInfo; - return deltaInfo->bytes; - } - - case CMPR_DICT: { - DictCmprMeta* dicInfo = (DictCmprMeta*)metaInfo; - return (dicInfo->dictItemNum == 1) ? 0 : DICT_ITEMID_SIZE; - } - - case CMPR_PREFIX: - case CMPR_NUMSTR: { - return (attlen == -1) ? VARSIZE_ANY(attrStart) : (strlen(attrStart) + 1); - } - - case CMPR_NONE: { - Assert(false); - break; - } - default: - break; - } - - Assert(false); - return 0; -} - -/* - * external buffer must be offered for some compress method, in order to - * reduce the times of multi memcpy() calls. - */ -bool PageCompress::NeedExternalBuf(char mode) -{ - return ((mode == CMPR_PREFIX) || (mode == CMPR_NUMSTR)); -} - -int PageCompress::UncompressOneAttr( - char mode, char attalign, int attlen, void* metaInfo, char* attrStart, int* attrCmprsValSize, char* uncmprValBuf) -{ - int dataLenWithPadding = 0; - - Assert((attlen == -1) || (attlen == -2 && attalign == 'c')); - Assert(NeedExternalBuf(mode)); - - switch (mode) { - case CMPR_PREFIX: { - PrefixCmprMeta* prefixInfo = (PrefixCmprMeta*)metaInfo; - (void)PrefixUncompress( - attlen, prefixInfo, attrStart, attrCmprsValSize, uncmprValBuf, &dataLenWithPadding, attalign); - break; - } - - case CMPR_NUMSTR: { - (void)NumstrUncompress( - PointerGetDatum(attrStart), attlen, *attrCmprsValSize, uncmprValBuf, &dataLenWithPadding, attalign); - break; - } - - default: - Assert(false); - break; - } - - return dataLenWithPadding; -} - -/* accordint to compression mode and meta information, set the length of compressed value, - * decompress this attribute in the tuple, and return both of them. - * mode [IN]: compression mode; - * metaInfo [IN]: compression meta information; - * typeOid [IN]: type oid of the attribute; - * attlen [IN]: attr len of this attribute; - * attrStart [IN]: start of attr data; - * attrCmprsValSize [OUT]: length of compressed value; - */ -Datum PageCompress::UncompressOneAttr( - char mode, void* metaInfo, Oid typeOid, int attlen, char* attrStart, int* attrCmprsValSize) -{ - switch (mode) { - case CMPR_DELTA: { - DeltaCmprMeta* deltaInfo = (DeltaCmprMeta*)metaInfo; - if (deltaInfo->bytes == 0) { - *attrCmprsValSize = 0; - return deltaInfo->MinVal; - } - - *attrCmprsValSize = deltaInfo->bytes; - - /* - * typeOid is InvalidOid when this attribute is dropped. But its - * typlen and typalign are also always kept. in this case, its - * value cann't be accessed and computed because of lost atttype. - * so that we just only skip it, and continue to access next valid attr. - * - * please refer to RemoveAttributeById(). - */ - if (InvalidOid == typeOid) { - return (Datum)0; - } - - const NumberTypeOpt* opt = binarySearch(typeOid); - return DeltaUncompress(deltaInfo, opt, (unsigned char*)attrStart); - } - - case CMPR_DICT: { - DictCmprMeta* dicInfo = (DictCmprMeta*)metaInfo; - - if (dicInfo->dictItemNum == 1) { - *attrCmprsValSize = 0; - char id = 0; - return DictUncompress(dicInfo, &id, attlen); - } - *attrCmprsValSize = DICT_ITEMID_SIZE; - return DictUncompress(dicInfo, attrStart, attlen); - } - - case CMPR_PREFIX: { - PrefixCmprMeta* prefixInfo = (PrefixCmprMeta*)metaInfo; - return PrefixUncompress(attlen, prefixInfo, attrStart, attrCmprsValSize); - } - - case CMPR_NUMSTR: { - return NumstrUncompress(PointerGetDatum(attrStart), attlen, *attrCmprsValSize); - } - - case CMPR_NONE: { - Assert(false); - return (Datum)0; - } - default: - break; - } - - Assert(false); - return (Datum)0; -} - -/* Get dictionary item ID */ -bool PageCompress::DictCompress(int col, Datum val, unsigned char& dicItemId) -{ - bool found = false; - int attlen = m_rel->rd_att->attrs[col]->attlen; - DictCmprMeta* dicInfo = (DictCmprMeta*)m_cmprMeta[col]; - DictItemData key; - - if (attlen > 0) - FillHashKey<1>(key, attlen, val); - else if (-1 == attlen) - FillHashKey<-1>(key, attlen, val); - else if (-2 == attlen) - FillHashKey<-2>(key, attlen, val); - - dicItemId = dicInfo->GetDictItemId(&key, found); - - if (attlen > 0) - pfree(key.itemData); - - return found; -} - -/* compress the col's attribute of one tuple, ouput data is written into formTuple */ -void PageCompress::CompressOneAttr(Datum val, bool null, FormCmprTupleData* formTuple, int col) -{ - formTuple->compressed[col] = false; - - if (null) { - formTuple->isnulls[col] = true; - formTuple->values[col] = (Datum)0; - return; - } - - Form_pg_attribute* attrs = m_rel->rd_att->attrs; - formTuple->values[col] = val; - formTuple->isnulls[col] = false; - - switch (m_cmprMode[col]) { - case CMPR_DELTA: { - DeltaCmprMeta* metaInfo = (DeltaCmprMeta*)m_cmprMeta[col]; - const NumberTypeOpt* opt = m_numTypeOpt[col]; - unsigned char* deltaBuf = (unsigned char*)palloc((Size)metaInfo->bytes); - if (DeltaCompress(metaInfo, opt, val, deltaBuf)) { - formTuple->compressed[col] = true; - if (metaInfo->bytes == 0) { - /* - * this means that all values of this column within page are equal to. - * we will store only the one into compression header, and null in every - * column for tuples. - * Notice that: keep this flag formTuple->isnulls[col] is false. - */ - formTuple->valsize[col] = 0; - formTuple->values[col] = (Datum)0; - pfree(deltaBuf); - } else { - formTuple->valsize[col] = metaInfo->bytes; - formTuple->values[col] = PointerGetDatum(deltaBuf); - } - } else - pfree(deltaBuf); - break; - } - - case CMPR_DICT: { - if (DictCompress(col, val, m_dictItemId[col])) { - formTuple->compressed[col] = true; - DictCmprMeta* dicInfo = (DictCmprMeta*)m_cmprMeta[col]; - if (dicInfo->dictItemNum == 1) { - formTuple->valsize[col] = 0; - formTuple->values[col] = (Datum)0; - } else { - formTuple->valsize[col] = DICT_ITEMID_SIZE; - formTuple->values[col] = PointerGetDatum(m_dictItemId + col); - } - } - break; - } - - case CMPR_PREFIX: { - int attrLen = attrs[col]->attlen; - Assert(attrLen == -1 || attrLen == -2); - PrefixCmprMeta* prefixInfo = (PrefixCmprMeta*)m_cmprMeta[col]; - Datum cmprsVal; - int cmprsValSize; - bool successful = false; - - /* - * we think that prefix method doesn't work well on those data, - * which are compressed by lz method internally. so skipp it. - */ - if (VARATT_IS_4B_C(DatumGetPointer(val))) { - break; - } - - if (-1 == attrLen) - successful = PrefixCompress<-1>(prefixInfo, val, cmprsVal, cmprsValSize); - else - successful = PrefixCompress<-2>(prefixInfo, val, cmprsVal, cmprsValSize); - - if (successful) { - formTuple->compressed[col] = true; - formTuple->valsize[col] = cmprsValSize; - formTuple->values[col] = cmprsVal; - } - break; - } - - case CMPR_NUMSTR: { - int attrLen = attrs[col]->attlen; - Assert(-1 == attrLen || -2 == attrLen); - - int outSize = 0; - Datum outValue; - - if (-1 == attrLen) { - outValue = NumstrCompress<-1>(val, outSize); - Assert(outSize == 0 || (Size)outSize == VARSIZE_ANY(DatumGetPointer(outValue))); - } else { - outValue = NumstrCompress<-2>(val, outSize); - Assert(outSize == 0 || (Size)outSize == strlen(DatumGetPointer(outValue)) + 1); - } - if (outSize > 0) { - formTuple->compressed[col] = true; - formTuple->valsize[col] = outSize; - formTuple->values[col] = outValue; - } - break; - } - - case CMPR_NONE: { - break; - } - default: - break; - } -} - -bool PageCompress::CompressOnePage(void) -{ - const int blksize = RelationGetTargetPageUsage(m_rel, HEAP_DEFAULT_FILLFACTOR); - int attrNum = RelationGetNumberOfAttributes(m_rel); - int size, row; - int prevToastNum = m_toastTupsNum; - int thisToasts; - - MemoryContext oldMemCnxt; - ColArray* colArray = NULL; - TupleDesc desc = RelationGetDescr(m_rel); - FormCmprTupleData formTupleData; - Datum* values = NULL; - bool* isnulls = NULL; - bool fillOnePage = false; - bool hascmpr = false; - bool remainBeforeCmpr = true; - errno_t rc = EOK; - - /* reset local mem-context after handling one page tuples. */ - ResetPerPage(); - - oldMemCnxt = MemoryContextSwitchTo(m_pageMemCnxt); - - /* at the end of the current batch, only toast tuples need to be written into pages. */ - if ((m_current == m_inTupsNum) || m_nextBatch) { - Assert(m_mappedTupsNum == 0); - ResetCmprMeta(); - /* jump to branch handle_toasts only if m_current is equal to m_inTupsNum */ - goto handle_toasts; - } - - /* - * Step 1: how many rows are compressed and form one page - * We need evaluate proper row numbers. - */ - size = SizeOfHeapPageHeaderData; - for (row = m_current; row < m_inTupsNum; ++row) { - /* collect all the toast tuples, and write once at the end of each batch */ - if (isToastTuple(m_inTups[row])) - m_toastTups[m_toastTupsNum++] = m_inTups[row]; - else { - if (size + (int)getTupleSpace(m_inTups[row]) > blksize) - break; - - /* collect all the untoast tuples, and enter comperss flow */ - m_mappedTups[m_mappedTupsNum++] = m_inTups[row]; - size += (int)getTupleSpace(m_inTups[row]); - } - } - - /* - * raw tuples cann't fill one page, then we must check m_last flag to - * write these ones. and toast tuples would be taken care. - */ - if (row == m_inTupsNum) { - ResetCmprMeta(); - remainBeforeCmpr = true; - goto handle_remains; - } - - /* Step 2: deform tuples into ColArray */ - colArray = DeformTups(m_mappedTups, m_mappedTupsNum); - - /* - * Step 3: Sampling data of each column and choose proper compression method - * We need generate compression meta information for step 3. - */ - ChooseCmprMethod(colArray); - SetCmprHeaderData(); - if (m_cmprHeaderSize == 0) { - /* all the tuples within the current batch are not suitable for compress. */ - OutputRawTupsBeforeCmpr(m_toastTupsNum - prevToastNum); - (void)MemoryContextSwitchTo(oldMemCnxt); - return true; - } - -#ifdef USE_ASSERT_CHECKING - CheckCmprHeaderData(); -#endif - - /* Step 4: Compress the current batch rows. */ - formTupleData.compressed = (bool*)palloc((Size)(sizeof(bool) * attrNum)); - formTupleData.valsize = (int*)palloc((Size)(sizeof(int) * attrNum)); - formTupleData.values = (Datum*)palloc((Size)(sizeof(Datum) * attrNum)); - formTupleData.isnulls = (bool*)palloc((Size)(sizeof(bool) * attrNum)); - size = (int)MAXALIGN(SizeOfHeapPageHeaderData) + (int)MAXALIGN((uint32)m_cmprHeaderSize); - - for (row = 0; row < m_mappedTupsNum; ++row) { - hascmpr = false; - - for (int col = 0; col < attrNum; ++col) { - CompressOneAttr(colArray[col].val[row], colArray[col].nulls[row], &formTupleData, col); - hascmpr = (hascmpr || formTupleData.compressed[col]); -#ifdef USE_ASSERT_CHECKING - CheckCmprAttr(colArray[col].val[row], colArray[col].nulls[row], col, &formTupleData); -#endif - } - - /* - * form compressed tuple and then dispatch until part of tuples fill one page, - * or all compressed tuples fill part of one page. - */ - if (hascmpr) { - fillOnePage = Dispatch(m_mappedTups[row], (HeapTuple)tableam_tops_form_cmprs_tuple(desc, &formTupleData, HEAP_TUPLE), size, blksize); - } else { - fillOnePage = Dispatch(m_mappedTups[row], size, blksize); - } - - if (fillOnePage) - break; - } - - /* - * Step 5: Judge whether compressed data is enough to fill one page - * If not enough, we compress the next tuples. But it is different from - * Step 4, if one column is compressed by delta value encoding, we can't - * change delta value. - */ - if (fillOnePage) { - Assert(row < m_mappedTupsNum); - ResetCmprMeta(); - OutputRawTupsAfterCmpr(m_toastTupsNum - prevToastNum); - (void)MemoryContextSwitchTo(oldMemCnxt); - return true; - } - Assert(row == m_mappedTupsNum); - - values = (Datum*)palloc((Size)(sizeof(Datum) * attrNum)); - isnulls = (bool*)palloc((Size)(sizeof(bool) * attrNum)); - - /* continue to read and compress following tuples, until one page is filled */ - for (row = m_current + m_mappedTupsNum + (m_toastTupsNum - prevToastNum); row < m_inTupsNum; ++row) { - HeapTuple raw = m_inTups[row]; - if (isToastTuple(raw)) { - m_toastTups[m_toastTupsNum++] = raw; - continue; - } - - heap_deform_tuple(raw, desc, values, isnulls); - hascmpr = false; - - for (int col = 0; col < attrNum; ++col) { - CompressOneAttr(values[col], isnulls[col], &formTupleData, col); - hascmpr = (hascmpr || formTupleData.compressed[col]); -#ifdef USE_ASSERT_CHECKING - CheckCmprAttr(values[col], isnulls[col], col, &formTupleData); -#endif - } - - if (hascmpr) { - fillOnePage = Dispatch(raw, (HeapTuple)tableam_tops_form_cmprs_tuple(desc, &formTupleData, HEAP_TUPLE), size, blksize); - } else { - fillOnePage = Dispatch(raw, size, blksize); - } - - if (fillOnePage) - break; - } - - if (fillOnePage || m_last) { - Assert(fillOnePage || (row == m_inTupsNum)); - OutputCmprTups(m_toastTupsNum - prevToastNum); - (void)MemoryContextSwitchTo(oldMemCnxt); - return true; - } - - /* - * after compress, remain tuples cann't fill one page. - * we think that cotinue to read and compress more tuples from next batch - * maybe is better. otherwise this page will holds one tuple at least. - */ - Assert(row == m_inTupsNum && m_last == false); - m_cmprHeaderSize = 0; - remainBeforeCmpr = false; - -handle_remains: - - thisToasts = m_toastTupsNum - prevToastNum; - if (m_mappedTupsNum == 0) { - /* case 1: only toast tuples remain. */ - Assert(thisToasts == (m_inTupsNum - m_current)); - m_current = m_inTupsNum; - } else if (m_last) { - /* - * case 2: for the last batch, - * first write all the raw tuples left into pages; - * then handle the remain toast tuples; - */ - Assert(remainBeforeCmpr); - OutputRawTupsBeforeCmpr(thisToasts); - Assert(m_current == m_inTupsNum); - (void)MemoryContextSwitchTo(oldMemCnxt); - return true; - } else { - /* - * case 3: if not the last batch - * first write all toasts of the current batch into pages; - * then move the remain into next batch; - */ - m_nextBatch = true; - - /* - * case 3.1: toats and raw tuples must be collected sperately; - */ - if (thisToasts > 0) { - /* - * step 1: collect these toasts in the front, make sure they are free - * after they are written into pages by heap_multi_insert(). - */ - rc = memcpy_s(m_inTups + m_current, - (size_t)(sizeof(HeapTuple) * thisToasts), - m_toastTups + prevToastNum, - (size_t)(sizeof(HeapTuple) * thisToasts)); - securec_check(rc, "", ""); - m_current += thisToasts; - } - - /* step 2: collect raw tuples in the end, make sure they are moved ahead next buffer. */ - if (remainBeforeCmpr) { - /* case 3.2.1: raw tuples are all in m_mappedTups[m_mappedTupsNum] */ - rc = memcpy_s(m_inTups + m_current, - (size_t)(sizeof(HeapTuple) * m_mappedTupsNum), - m_mappedTups, - (size_t)(sizeof(HeapTuple) * m_mappedTupsNum)); - securec_check(rc, "", ""); - } else { - /* - * case 3.2.2: some are int m_mappedTups[m_cmprTupsNum], and the others - * are in m_cmprTups[m_uncmprTupIdx, m_inTupsNum) - */ - int offset = m_current; - if (m_cmprTupsNum > 0) { - rc = memcpy_s(m_inTups + offset, - (size_t)(sizeof(HeapTuple) * m_cmprTupsNum), - m_mappedTups, - (size_t)(sizeof(HeapTuple) * m_cmprTupsNum)); - securec_check(rc, "", ""); - offset += m_cmprTupsNum; - m_cmprTupsNum = 0; - } - - int uncmpr = m_inTupsNum - m_uncmprTupIdx; - if (uncmpr > 0) { - rc = memcpy_s(m_inTups + offset, (size_t)(sizeof(HeapTuple) * uncmpr), m_cmprTups + m_uncmprTupIdx, - (size_t)(sizeof(HeapTuple) * uncmpr)); - securec_check(rc, "", ""); - } - } - } - -handle_toasts: - - if (m_toastTupsNum > 0) { - OutputToasts(); - (void)MemoryContextSwitchTo(oldMemCnxt); - return true; - } - - /* all tuples of this batch have been handled. */ - pfree(m_toastTups); - pfree(m_mappedTups); - pfree(m_cmprTups); - m_toastTups = NULL; - m_mappedTups = NULL; - m_cmprTups = NULL; - - (void)MemoryContextSwitchTo(oldMemCnxt); - return false; -} - -void PageCompress::OutputRawTupsBeforeCmpr(int nToasts) -{ - /* - * write all the tuples left out in the last batch. - * make m_current equal to m_inTupsNum so that toast tuples - * will be written into pages at the last time. - */ - m_outputTups = m_mappedTups; - m_outputTupsNum = m_mappedTupsNum; - -#ifdef USE_ASSERT_CHECKING - CheckOutputTups(NULL, 0, m_mappedTups, m_mappedTupsNum, nToasts); -#endif -#ifdef TRACE_COMPRESS - ++nLoops; - TraceCmprPage(0, NULL, NULL, 0, m_mappedTups, m_mappedTupsNum, nToasts); -#endif - - m_current += (m_outputTupsNum + nToasts); - m_cmprHeaderSize = 0; - m_cmprTupsNum = 0; -} - -void PageCompress::OutputRawTupsAfterCmpr(int nToasts) -{ - /* - * after compress, raw tuples exist in both m_mappedTups[] and m_cmprTups[]. - * and the second range is [m_uncmprTupIdx, m_inTupsNum). - * we have to collect all raw tuples into m_mappedTups[]. - */ - m_outputTups = m_mappedTups; - int uncmprNum = m_inTupsNum - m_uncmprTupIdx; - if (uncmprNum > 0) { - errno_t rc = memcpy_s(m_mappedTups + m_cmprTupsNum, - (size_t)(sizeof(HeapTuple) * uncmprNum), - m_cmprTups + m_uncmprTupIdx, - (size_t)(sizeof(HeapTuple) * uncmprNum)); - securec_check(rc, "", ""); - } - Assert((m_cmprTupsNum + uncmprNum) <= m_mappedTupsNum); - m_outputTupsNum = m_mappedTupsNum; - -#ifdef USE_ASSERT_CHECKING - CheckOutputTups( - m_mappedTups, m_cmprTupsNum, (m_mappedTups + m_cmprTupsNum), (m_mappedTupsNum - m_cmprTupsNum), nToasts); -#endif -#ifdef TRACE_COMPRESS - ++nLoops; - TraceCmprPage(1, NULL, NULL, 0, m_outputTups, m_outputTupsNum, nToasts); -#endif - - m_current += (m_outputTupsNum + nToasts); - m_cmprHeaderSize = 0; - m_cmprTupsNum = 0; -} - -void PageCompress::OutputCmprTups(int nToasts) -{ - /* - * after compress, output tuples exist in only m_cmprTups[], both compressed and some raws. - * the previous range is [0, m_cmprTupsNum) and the other is [m_uncmprTupIdx, m_inTupsNum). - * we have to append the second part to the first part. - */ - Assert(m_cmprTupsNum <= m_uncmprTupIdx); - m_outputTups = m_cmprTups; - int uncmprNum = m_inTupsNum - m_uncmprTupIdx; - if (0 == uncmprNum) - m_outputTupsNum = m_cmprTupsNum; - else if (m_cmprTupsNum == m_uncmprTupIdx) - m_outputTupsNum = m_inTupsNum; - else { - errno_t rc = memmove_s(m_cmprTups + m_cmprTupsNum, - (size_t)(sizeof(HeapTuple) * uncmprNum), - m_cmprTups + m_uncmprTupIdx, - (size_t)(sizeof(HeapTuple) * uncmprNum)); - securec_check(rc, "", ""); - m_outputTupsNum = m_cmprTupsNum + uncmprNum; - } - -#ifdef USE_ASSERT_CHECKING - Assert(m_outputTupsNum >= m_mappedTupsNum); - CheckOutputTups(m_mappedTups, m_cmprTupsNum, (m_cmprTups + m_cmprTupsNum), uncmprNum, nToasts); -#endif -#ifdef TRACE_COMPRESS - ++nLoops; - TraceCmprPage(2, m_cmprTups, m_mappedTups, m_cmprTupsNum, (m_cmprTups + m_cmprTupsNum), uncmprNum, nToasts); -#endif - - m_current += (m_outputTupsNum + nToasts); -} - -void PageCompress::OutputToasts(void) -{ - /* - * toast tuples within each batch are collected togather, and written once - * into pages before next SetBatchTuples() is called. - */ - m_outputTups = m_toastTups; - m_outputTupsNum = m_toastTupsNum; -#ifdef TRACE_COMPRESS - ++nLoops; - ereport(INFO, (errmsg("Loop %d, Tag: Toast Tuples, num %d", nLoops, m_toastTupsNum))); -#endif - m_toastTupsNum = 0; - Assert((m_last == false) || (m_current == m_inTupsNum)); - Assert(m_cmprHeaderSize == 0); - Assert(m_cmprTupsNum == 0); -} - -void PageCompress::SetBatchTuples(HeapTuple* tups, int ntups, bool isLast) -{ - m_inTups = tups; - m_inTupsNum = ntups; - m_current = 0; - m_last = isLast; - - /* - * we know the number of input tuples, so alloc the arrays for handling toast - * tuples. after this batch has been handled, free these memory. - */ - MemoryContext oldMemCnxt = MemoryContextSwitchTo(m_parentMemCxt); - - m_toastTups = (HeapTuple*)palloc((Size)(sizeof(HeapTuple) * ntups)); - m_mappedTups = (HeapTuple*)palloc((Size)(sizeof(HeapTuple) * ntups)); - m_cmprTups = (HeapTuple*)palloc((Size)(sizeof(HeapTuple) * ntups)); - m_toastTupsNum = 0; - m_nextBatch = false; - - (void)MemoryContextSwitchTo(oldMemCnxt); -} - -void PageCompress::ResetPerPage(void) -{ - int attNum = m_rel->rd_att->natts; - - /* Free everything by destroying dictionary */ - for (int att = 0; att < attNum; ++att) { - if (m_cmprMode[att] == CMPR_DICT && m_cmprMeta[att]) { - ((DictCmprMeta*)m_cmprMeta[att])->Destroy(); - m_cmprMeta[att] = NULL; - } - } - - MemoryContextReset(m_pageMemCnxt); - /* m_cmprHeaderData would be fixed during the entire compress, keep it. */ - m_cmprHeaderSize = 0; - m_outputTups = NULL; - m_outputTupsNum = 0; - - m_mappedTupsNum = 0; - m_cmprTupsNum = 0; - m_uncmprTupIdx = m_inTupsNum; -} - -/* - * reset all compress meta info when tuples are not compressed and output. - * 1. before OutputRawTupsAfterCmpr() is called; - * 2. before OutputRawTupsBeforeCmpr() is called, optional; - * 3. before branch handle_remains is jumpped to; - * 4. before branch handle_toasts is jumpped to; - */ -void PageCompress::ResetCmprMeta(void) -{ - int attNum = m_rel->rd_att->natts; - for (int i = 0; i < attNum; ++i) { - m_cmprMode[i] = CMPR_NONE; - m_cmprMeta[i] = NULL; - } - m_cmprHeaderSize = 0; -} - -/* - * it's different from BackWrite(), which transforms compressed tuples' info - * backward into raw tuples. but this method will transforms raw tuples' info - * forward into compressed tuples. - * NB: now this method applies to VACUUM FULL && CLUSTER - */ -void PageCompress::ForwardWrite(void) -{ - if (m_cmprTupsNum == 0) - return; - errno_t rc = EOK; - - Assert(m_cmprHeaderSize > 0); - for (int i = 0; i < m_cmprTupsNum; i++) { - HeapTupleHeader dest = m_cmprTups[i]->t_data; - HeapTupleHeader src = m_mappedTups[i]->t_data; - - Assert(HEAP_TUPLE_IS_COMPRESSED(dest)); - Assert(dest->t_hoff == src->t_hoff); - Assert(HeapTupleHeaderGetNatts(dest, m_rel->rd_att) == HeapTupleHeaderGetNatts(src, m_rel->rd_att)); - Assert((0x000F & dest->t_infomask) == (0x000F & src->t_infomask)); - Assert( - (0 == (dest->t_infomask & HEAP_HASNULL)) || - 0 == memcmp( - (char*)dest->t_bits, (char*)src->t_bits, BITMAPLEN(HeapTupleHeaderGetNatts(src, m_rel->rd_att)))); - - rc = memcpy_s((char*)dest, src->t_hoff, (char*)src, src->t_hoff); - securec_check(rc, "", ""); - HEAP_TUPLE_SET_COMPRESSED(dest); - } -} - -/* - * after heap_multi_insert() is done, we should write infomask back - * to the raw tuples. these information will be used by trigger and - * index build. - * this api exists just to reduce public methods as much as possible. - */ -void PageCompress::BackWrite(void) -{ - /* compressed tuples exist and are valid only if m_cmprHeaderSize > 0 */ - if (m_cmprTupsNum == 0) - return; - - errno_t rc = EOK; - - Assert(m_cmprHeaderSize > 0); - for (int i = 0; i < m_cmprTupsNum; i++) { - HeapTupleHeader dest = m_mappedTups[i]->t_data; - HeapTupleHeader src = m_cmprTups[i]->t_data; - - Assert(dest->t_hoff == src->t_hoff); - Assert((0x07FF & dest->t_infomask2) == (0x07FF & src->t_infomask2)); - Assert((0x000F & dest->t_infomask) == (0x000F & src->t_infomask)); - Assert(HeapTupleHasNulls(m_mappedTups[i]) == HeapTupleHasNulls(m_cmprTups[i])); - - rc = memcpy_s((char*)dest, src->t_hoff, (char*)src, src->t_hoff); - securec_check(rc, "", ""); - HEAP_TUPLE_CLEAR_COMPRESSED(dest); - m_mappedTups[i]->t_self = m_cmprTups[i]->t_self; - m_mappedTups[i]->t_tableOid = m_cmprTups[i]->t_tableOid; -#ifdef PGXC - m_mappedTups[i]->t_xc_node_id = m_cmprTups[i]->t_xc_node_id; -#endif - } -} - -bool PageCompress::Dispatch(HeapTuple raw, HeapTuple cmpr, int& nowSize, const int& blksize) -{ - Assert(HEAP_TUPLE_IS_COMPRESSED(cmpr->t_data)); - int needSize = (int)getTupleSpace(cmpr); - if (nowSize + needSize > blksize) - return true; - - Assert(m_cmprTupsNum < m_uncmprTupIdx); - HeapTupleCopyBase(cmpr, raw); - m_cmprTups[m_cmprTupsNum] = cmpr; - m_mappedTups[m_cmprTupsNum] = raw; - ++m_cmprTupsNum; -#ifdef USE_ASSERT_CHECKING - CheckCmprTuple(raw, cmpr); -#endif - nowSize += needSize; - return false; -} - -bool PageCompress::Dispatch(HeapTuple raw, int& nowSize, const int& blksize) -{ - Assert(!HEAP_TUPLE_IS_COMPRESSED(raw->t_data)); - int needSize = (int)getTupleSpace(raw); - if (nowSize + needSize > blksize) - return true; - - Assert(m_uncmprTupIdx > m_cmprTupsNum); - --m_uncmprTupIdx; - m_cmprTups[m_uncmprTupIdx] = raw; - nowSize += needSize; - return false; -} - -#ifdef USE_ASSERT_CHECKING - -void PageCompress::CheckCmprDatum(Datum arg1, Datum arg2, Form_pg_attribute attr) -{ - int attlen = attr->attlen; - errno_t rc = EOK; - - if (attr->attbyval) { - Assert(attlen > 0 && attlen <= SIZEOF_DATUM); - Assert(arg1 == arg2); - return; - } - - if (attlen > 0) { - Assert(0 == memcmp(DatumGetPointer(arg1), DatumGetPointer(arg2), attlen)); - return; - } - - /* - * we pack compressed data as many as possible, even its storage type is PLAIN; - * so raw data must be first packed before comparing; - */ - char packedBuf1[VARATT_SHORT_MAX] = {0}; - char packedBuf2[VARATT_SHORT_MAX] = {0}; - Pointer pOldValue1 = DatumGetPointer(arg1); - Pointer pOldValue2 = DatumGetPointer(arg2); - - if (-1 == attr->attlen) { - if (!VARATT_IS_EXTERNAL(pOldValue1) && !VARATT_IS_SHORT(pOldValue1) && - VARATT_CONVERTED_SHORT_SIZE(pOldValue1) <= VARATT_SHORT_MAX) { - Size len = VARATT_CONVERTED_SHORT_SIZE(pOldValue1); - Assert(len <= VARATT_SHORT_MAX); - SET_VARSIZE_SHORT(&packedBuf1[0], len); - rc = memcpy_s(&packedBuf1[1], VARATT_SHORT_MAX - VARHDRSZ_SHORT, VARDATA(pOldValue1), len - 1); - securec_check(rc, "", ""); - arg1 = PointerGetDatum(packedBuf1); - } - - if (!VARATT_IS_EXTERNAL(pOldValue2) && !VARATT_IS_SHORT(pOldValue2) && - VARATT_CONVERTED_SHORT_SIZE(pOldValue2) <= VARATT_SHORT_MAX) { - Size len = VARATT_CONVERTED_SHORT_SIZE(pOldValue2); - Assert(len <= VARATT_SHORT_MAX); - SET_VARSIZE_SHORT(&packedBuf2[0], len); - rc = memcpy_s(&packedBuf2[1], VARATT_SHORT_MAX - VARHDRSZ_SHORT, VARDATA(pOldValue2), len - 1); - securec_check(rc, "", ""); - arg2 = PointerGetDatum(packedBuf2); - } - } - - char* ptr1 = DatumGetPointer(arg1); - char* ptr2 = DatumGetPointer(arg2); - int size1 = 0; - int size2 = 0; - - if (-1 == attlen) { - size1 = VARSIZE_ANY(ptr1); - size2 = VARSIZE_ANY(ptr2); - } else if (-2 == attlen) { - size1 = strlen(ptr1) + 1; - size2 = strlen(ptr2) + 1; - } - - Assert(size1 == size2); - Assert(0 == memcmp(ptr2, ptr1, size1)); - return; -} - -void PageCompress::CheckCmprAttr(Datum rawVal, bool rawNull, int col, FormCmprTupleData* formTuple) -{ - Assert(rawNull == formTuple->isnulls[col]); - if (rawNull) { - Assert((Datum)0 == formTuple->values[col]); - return; - } - - if (formTuple->compressed[col] == false) { - Assert(rawVal == formTuple->values[col]); - return; - } - - Assert(m_cmprMode[col] != CMPR_UNDEF); - Form_pg_attribute* atts = RelationGetDescr(m_rel)->attrs; - Assert(1 != atts[col]->attlen); - int cmprSize; - Datum value = UncompressOneAttr(m_cmprMode[col], - m_cmprMeta[col], - atts[col]->atttypid, - atts[col]->attlen, - DatumGetPointer(formTuple->values[col]), - &cmprSize); - - Assert(cmprSize == formTuple->valsize[col]); - CheckCmprDatum(rawVal, value, atts[col]); -} - -void PageCompress::CheckCmprTuple(HeapTuple rawTup, HeapTuple cmprTup) -{ - TupleDesc desc = RelationGetDescr(m_rel); - Form_pg_attribute* atts = desc->attrs; - int nattrs = desc->natts; - - Datum* values = (Datum*)palloc(sizeof(Datum) * nattrs); - Datum* values2 = (Datum*)palloc(sizeof(Datum) * nattrs); - bool* isnulls = (bool*)palloc(sizeof(bool) * nattrs); - bool* isnulls2 = (bool*)palloc(sizeof(bool) * nattrs); - - heap_deform_cmprs_tuple(cmprTup, desc, values, isnulls, m_cmprHeaderData); - heap_deform_tuple(rawTup, desc, values2, isnulls2); - - for (int col = 0; col < nattrs; ++col) { - Assert(isnulls[col] == isnulls2[col]); - if (isnulls[col]) { - Assert((Datum)0 == values[col]); - Assert((Datum)0 == values2[col]); - continue; - } - - CheckCmprDatum(values[col], values2[col], atts[col]); - } - - pfree(values); - pfree(isnulls); - pfree(values2); - pfree(isnulls2); -} - -void PageCompress::CheckCmprHeaderData(void) -{ - if (m_cmprHeaderSize == 0) { - return; - } - - Form_pg_attribute* atts = RelationGetDescr(m_rel)->attrs; - int nattrs = RelationGetNumberOfAttributes(m_rel); - int cmprsOff = 0; /* pointer to the start of compression meta */ - void* metaInfo = NULL; - char mode = 0; - - for (int col = 0; col < nattrs; ++col) { - int metaSize = 0; - metaInfo = PageCompress::FetchAttrCmprMeta(m_cmprHeaderData + cmprsOff, atts[col]->attlen, &metaSize, &mode); - cmprsOff += metaSize; - - Assert(mode != CMPR_UNDEF && m_cmprMode[col] == mode); - switch (mode) { - case CMPR_DELTA: { - DeltaCmprMeta* deltaInfo = (DeltaCmprMeta*)metaInfo; - DeltaCmprMeta* target = (DeltaCmprMeta*)m_cmprMeta[col]; - Assert(deltaInfo->bytes == target->bytes); - Assert(deltaInfo->MinVal == target->MinVal); - - break; - } - - case CMPR_DICT: { - DictCmprMeta* dictMeta = (DictCmprMeta*)metaInfo; - DictCmprMeta* target = (DictCmprMeta*)m_cmprMeta[col]; - Assert(dictMeta->dictItemNum == target->dictItemNum); - - for (int i = 0; i < dictMeta->dictItemNum; ++i) { - Assert(dictMeta->dictItems[i].itemSize == target->dictItems[i].itemSize); - Assert(0 == memcmp(dictMeta->dictItems[i].itemData, target->dictItems[i].itemData, - target->dictItems[i].itemSize)); - } - - break; - } - - case CMPR_PREFIX: { - PrefixCmprMeta* prefixMeta = (PrefixCmprMeta*)metaInfo; - PrefixCmprMeta* target = (PrefixCmprMeta*)m_cmprMeta[col]; - Assert(prefixMeta->len == target->len); - Assert(0 == memcmp(prefixMeta->prefixStr, target->prefixStr, target->len)); - - break; - } - - case CMPR_NUMSTR: - case CMPR_NONE: - break; - default: - break; - } - } - - Assert(m_cmprHeaderSize == (int)MAXALIGN(cmprsOff)); - Assert(m_cmprHeaderSize < BLCKSZ); -} - -/* - * check the number and lossless for input tuples. - * 1. dispatch to toast arrays; - * 2. dispatch to uncompressed arrays; - * 3. compress and then mapped to mapped arrays; - * Important: call before m_current is modified and changed. - */ -void PageCompress::CheckOutputTups(HeapTuple* mapped, int nMapped, HeapTuple* uncmpr, int nUncmpr, int nToasts) -{ - int start = m_current; - int end = m_current + m_outputTupsNum + nToasts; - int realToasts = 0; - int realCmpr = 0; - int realUncmpr = 0; - - Assert((m_outputTupsNum == nToasts) || (m_outputTupsNum == (nMapped + nUncmpr))); - - for (; start < end; ++start) { - HeapTuple check = m_inTups[start]; - bool found = false; - - if (isToastTuple(check)) { - for (int i = m_toastTupsNum - nToasts; i < m_toastTupsNum; ++i) { - if (check != m_toastTups[i]) - continue; - found = true; - ++realToasts; - break; - } - - Assert(found == true); - continue; - } - - for (int i = 0; i < nMapped; ++i) { - if (check != mapped[i]) - continue; - found = true; - ++realCmpr; - break; - } - - if (found == true) - continue; - - for (int i = 0; i < nUncmpr; ++i) { - if (check != uncmpr[i]) - continue; - found = true; - ++realUncmpr; - break; - } - Assert(found == true); - } - - Assert(nToasts == realToasts); - Assert(nMapped == realCmpr); - Assert(nUncmpr == realUncmpr); -} - -#endif - -#ifdef TRACE_COMPRESS - -void PageCompress::TraceCmprPage( - int flag, HeapTuple* cmpr, HeapTuple* mapped, int nCmpr, HeapTuple* uncmpr, int nUncmpr, int nToasts) -{ - int start = m_current; - int end = m_current + m_outputTupsNum + nToasts; - int nattrs = RelationGetNumberOfAttributes(m_rel); - Form_pg_attribute* atts = RelationGetDescr(m_rel)->attrs; - - char* tag = NULL; - if (flag == 0) - tag = "Tag: Raw Tuples Before Compress"; - else if (flag == 1) - tag = "Tag: Raw Tuples After Compress"; - else if (flag == 2) - tag = "Tag: Compressed Tuples Of one page"; - else - tag = "Tag: Toast Tuples"; - ereport(INFO, (errmsg("Loop %d, range[%d - %d], %s", nLoops, start, (end - 1), tag))); - - DeltaCmprMeta* deltaMeta = NULL; - PrefixCmprMeta* prefixMeta = NULL; - DictCmprMeta* dictMeta = NULL; - - for (int col = 0; col < nattrs; ++col) { - switch (m_cmprMode[col]) { - case CMPR_NONE: - ereport(INFO, (errmsg("COL %d: None Compress", col))); - break; - - case CMPR_DELTA: - deltaMeta = (DeltaCmprMeta*)m_cmprMeta[col]; - ereport(INFO, - (errmsg("COL %d: Delta Compress, attrlen %d, delta bytes %d", - col, - atts[col]->attlen, - deltaMeta->bytes))); - break; - - case CMPR_NUMSTR: - ereport(INFO, (errmsg("COL %d: Number String Compress", col))); - break; - - case CMPR_PREFIX: - prefixMeta = (PrefixCmprMeta*)m_cmprMeta[col]; - ereport(INFO, (errmsg("COL %d: Prefix Compress, len %d", col, prefixMeta->len))); - break; - - case CMPR_DICT: - dictMeta = (DictCmprMeta*)m_cmprMeta[col]; - char str[0xFF * (sizeof(int16) + 1) + 1] = {0}; - char* p = str; - for (int i = 0; i < dictMeta->dictItemNum; ++i) { - (void)sprintf_s(p, sizeof(str), "%2d ", dictMeta->dictItems[i].itemSize); - p = p + 3; - } - ereport(INFO, (errmsg("COL %d: Dict Compress, num %d, len {%s}", col, dictMeta->dictItemNum, str))); - break; - default: - break; - } - } - - for (; start < end; ++start) { - HeapTuple check = m_inTups[start]; - - if (isToastTuple(check)) { - for (int i = m_toastTupsNum - nToasts; i < m_toastTupsNum; ++i) { - if (check != m_toastTups[i]) - continue; - ereport(INFO, (errmsg(" [%d] TOAST, raw len %d", start, check->t_len))); - break; - } - continue; - } - - for (int i = 0; i < nCmpr; ++i) { - if (check != mapped[i]) - continue; - ereport(INFO, (errmsg(" [%d] CMPR, raw len %d, cmpr len %d", start, check->t_len, cmpr[i]->t_len))); - continue; - } - - for (int i = 0; i < nUncmpr; ++i) { - if (check != uncmpr[i]) - continue; - ereport(INFO, (errmsg(" [%d] UNCMPR, raw len %d", start, check->t_len))); - break; - } - } - - ereport(INFO, (errmsg("Summary: cmpr %d, uncmpr %d, toasts %d", nCmpr, nUncmpr, nToasts))); -} - -#endif - -/* binary search. - * negative will be returned if search fails. Otherwise return the position of oid. - */ -static const NumberTypeOpt* binarySearch(Oid typeOid) -{ - int left = 0; - int right = g_number_type_count - 1; - - while (left <= right) { - int middle = left + ((right - left) / 2); - - if (g_number_types[middle].typeOid > typeOid) { - right = middle - 1; - } else if (g_number_types[middle].typeOid < typeOid) { - left = middle + 1; - } else - return (g_number_types + middle); - } - - return NULL; -} - -/* int16 Compare && Minus && Plus functions. */ -static int int16Compare(Datum arg1, Datum arg2) -{ - int16 val1 = DatumGetInt16(arg1); - int16 val2 = DatumGetInt16(arg2); - if (val1 > val2) - return (1); - if (val1 < val2) - return (-1); - return (0); -} - -static int64 int16Minus(Datum arg1, Datum arg2) -{ - int16 val1 = DatumGetInt16(arg1); - int16 val2 = DatumGetInt16(arg2); - - PG_RETURN_INT64((int64)(val1 - val2)); -} - -/* we don't care overflow problem about the plus, it's because: - * arg1 must be the result returned by int16Minus(); - * arg2 is from within int16Minus(); - */ -static Datum int16Plus(Datum arg1, int64 arg2) -{ - int16 val1 = DatumGetInt16(arg1); - int16 val2 = (int16)arg2; - - PG_RETURN_INT16((int16)(val1 + val2)); -} - -/* int32 Compare && Minus && Plus functions. */ -static int int32Compare(Datum arg1, Datum arg2) -{ - int32 val1 = DatumGetInt32(arg1); - int32 val2 = DatumGetInt32(arg2); - if (val1 > val2) - return 1; - if (val1 < val2) - return -1; - return 0; -} - -static int64 int32Minus(Datum arg1, Datum arg2) -{ - int32 val1 = DatumGetInt32(arg1); - int32 val2 = DatumGetInt32(arg2); - - PG_RETURN_INT64((int64)(val1 - val2)); -} - -/* we don't care overflow problem about the plus, it's because: - * arg1 must be the result returned by int32Minus(); - * arg2 is from within int32Minus(); - */ -static Datum int32Plus(Datum arg1, int64 arg2) -{ - int32 val1 = DatumGetInt32(arg1); - int32 val2 = (int32)arg2; - - PG_RETURN_INT32((int32)(val1 + val2)); -} - -/* uint32 Compare && Minus && Plus functions. */ -static int uint32Compare(Datum arg1, Datum arg2) -{ - uint32 val1 = DatumGetUInt32(arg1); - uint32 val2 = DatumGetUInt32(arg2); - if (val1 > val2) - return 1; - if (val1 < val2) - return -1; - return 0; -} - -static int64 uint32Minus(Datum arg1, Datum arg2) -{ - uint32 val1 = DatumGetUInt32(arg1); - uint32 val2 = DatumGetUInt32(arg2); - - PG_RETURN_INT64((int64)(val1 - val2)); -} - -/* we don't care overflow problem about the plus, it's because: - * arg1 must be the result returned by uint32Minus(); - * arg2 is from within uint32Minus(); - */ -static Datum uint32Plus(Datum arg1, int64 arg2) -{ - uint32 val1 = DatumGetUInt32(arg1); - uint32 val2 = (uint32)arg2; - - PG_RETURN_UINT32((uint32)(val1 + val2)); -} - -/* int64 Compare && Minus && Plus functions. */ -static int int64Compare(Datum arg1, Datum arg2) -{ - int64 val1 = DatumGetInt64(arg1); - int64 val2 = DatumGetInt64(arg2); - if (val1 > val2) - return 1; - if (val1 < val2) - return -1; - return 0; -} - -static int64 int64Minus(Datum arg1, Datum arg2) -{ - int64 val1 = DatumGetInt64(arg1); - int64 val2 = DatumGetInt64(arg2); - - PG_RETURN_INT64((int64)(val1 - val2)); -} - -/* we don't care overflow problem about the plus, it's because: - * arg1 must be the result returned by int64Minus(); - * arg2 is from within int64Minus(); - */ -static Datum int64Plus(Datum arg1, int64 arg2) -{ - int64 val1 = DatumGetInt64(arg1); - - PG_RETURN_INT64((int64)(val1 + arg2)); -} +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * pagecompress.cpp + * page compression manager routines + * + * + * IDENTIFICATION + * src/gausskernel/storage/page/pagecompress.cpp + * + * ------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "knl/knl_variable.h" +#include "storage/pagecompress.h" +#include "utils/memutils.h" +#include "utils/numeric.h" +#include "utils/numeric_gs.h" +#include "utils/dynahash.h" +#include "access/hash.h" +#include "access/tuptoaster.h" +#include "access/tableam.h" +#include "utils/rel.h" +#include "utils/rel_gs.h" + +static int int16Compare(Datum arg1, Datum arg2); +static int int32Compare(Datum arg1, Datum arg2); +static int uint32Compare(Datum arg1, Datum arg2); +static int int64Compare(Datum arg1, Datum arg2); + +static int64 int16Minus(Datum arg1, Datum arg2); +static int64 int32Minus(Datum arg1, Datum arg2); +static int64 uint32Minus(Datum arg1, Datum arg2); +static int64 int64Minus(Datum arg1, Datum arg2); + +static Datum int16Plus(Datum arg1, int64 arg2); +static Datum int32Plus(Datum arg1, int64 arg2); +static Datum uint32Plus(Datum arg1, int64 arg2); +static Datum int64Plus(Datum arg1, int64 arg2); + +static const NumberTypeOpt* binarySearch(Oid typeOid); + +/* const array for number type. */ +//定义一个名为 g_number_types 的常量数组，用于存储不同数值类型的信息，包括类型编号和比较、减法、加法等操作的函数指针 +static const NumberTypeOpt g_number_types[] = { + { 20, int64Compare, int64Minus, int64Plus }, /* int8 */ + { 21, int16Compare, int16Minus, int16Plus }, /* int2 */ + { 23, int32Compare, int32Minus, int32Plus }, /* int4 */ + { 26, uint32Compare, uint32Minus, uint32Plus }, /* oid */ + { 1082, int32Compare, int32Minus, int32Plus }, /* date */ + { 1083, int64Compare, int64Minus, int64Plus }, /* time */ + { 1114, int64Compare, int64Minus, int64Plus }, /* timestamp */ + { 1184, int64Compare, int64Minus, int64Plus }, /* timestamptz */ + { 12089, int64Compare, int64Minus, int64Plus }, /* time_stamp */ +}; +//数组中元素的个数 +static const int g_number_type_count = sizeof(g_number_types) / sizeof(NumberTypeOpt); + +/* Map about max values in byte for delta compression */ +//存储了在进行增量压缩时每个字节能表示的最大值 +static const int64 g_max_value_in_bytes[] = {0x0000000000000000, + 0x00000000000000FF, + 0x000000000000FFFF, + 0x0000000000FFFFFF, + 0x00000000FFFFFFFF, + 0x000000FFFFFFFFFF, + 0x0000FFFFFFFFFFFF, + 0x00FFFFFFFFFFFFFF + }; +//用于计算给定元组 _tuple 所占用的空间大小，包括存储 ItemIdData 的空间和对齐后的实际数据大小 +static const int64 g_number_max_value_in_bytes = (int64)(sizeof(g_max_value_in_bytes) / sizeof(int64)); + +#define getTupleSpace(_tuple) (sizeof(ItemIdData) + MAXALIGN((_tuple)->t_len)) + +/* How many bits must be used at least for storing one delta-based data */ +//用于计算给定增量值 delta 需要使用的最少位数。它通过不断左移限制值 limit，直到超过增量值 delta 时停止，并返回左移次数 +static uint32 getBitsOfDelta(uint64 delta) +{ + /* guard against too-large input, which would put us into infinite loop */ + if (delta > ULLONG_MAX / 2) { + return 64; + } + + uint32 i = 0; + for (uint64 limit = 1; limit < delta; limit <<= 1) { + i++; + } + + return i; +} + +/* Given the number of bits, compute the size of bytes. */ +//计算给定位数 _bits 需要使用的字节数，通过将位数加上 7 后除以 8 来取整 +#define getSizeOfBits(_bits) (((_bits) + 7) >> 3) + +/* MAX_PREFIX_LEN limits the length of prefix string */ +//定义了前缀字符串的最大长度为 256 +#define MAX_PREFIX_LEN 256 + +/* judge whether a char is digital */ +//判断一个字符 _ch 是否为数字字符 +#define isDigital(_ch) (((_ch) >= '0') && ((_ch) <= '9')) + +/* judge whether it's a taost tuple */ +//判断给定的元组 _tup 是否为 TOAST 形式的元组，即是否有外部存储或数据长度超过阈值 TOAST_TUPLE_THRESHOLD +#define isToastTuple(_tup) (HeapTupleHasExternal(_tup) || (_tup)->t_len > TOAST_TUPLE_THRESHOLD) + +/* judge whether this attribute is dropped */ +//宏用于判断给定的属性 _attr 是否被删除，即其类型标识符是否为 InvalidOid +#define isAttrDropped(_attr) (InvalidOid == ((_attr)->atttypid)) + +/* one data repeats at least such times so that it will be added into dict. */ +//定义了用于字典压缩的常量，DICTITEM_MIN_REPEATS 表示一个数据至少重复多少次才会被添加到字典中，DICTITEM_NUM_FACTOR 是一个因子常量 +#define DICTITEM_MIN_REPEATS 3 +#define DICTITEM_NUM_FACTOR 10 + +/* Limit for Dictionary Compression. + * 1. the max number of dict item is , whose id starts from 0; + * 2. after compress, one attribute is stored by byte; + */ +//定义了用于字典压缩的限制常量，MAX_DICITEMID 表示字典中的最大项数，DICT_ITEMID_SIZE 表示每个属性在压缩后存储时所占用的字节数，DICT_MISS_THRESHOLD 是一个阈值常量 +#define MAX_DICITEMID 256 +#define DICT_ITEMID_SIZE 1 +#define DICT_MISS_THRESHOLD 2 +//比较哈希表中两个字典项是否相等 +static int matchDictItem(const void* left, const void* right, Size keysize) +{ + /* 将 left 和 right 转换成 DictItemData 类型并进行非空指针检查 */ + const DictItemData* leftItem = (DictItemData*)left; + const DictItemData* rightItem = (DictItemData*)right; + Assert(leftItem != NULL && rightItem != NULL); + + /* we just care whether the result is 0 or not. */ + /* 比较左右两个字典项的长度 */ + if (leftItem->itemSize != rightItem->itemSize) { + return 1;/* 不相等则不匹配 */ + } + /* 调用 memcmp 函数比较左右两个字典项的数据 */ + return memcmp(leftItem->itemData, rightItem->itemData, leftItem->itemSize); +} +//计算哈希表中字典项的哈希值 +static uint32 hashDictItem(const void* key, Size keysize) +{ + /* 将输入键值 key 转换成 DictItemData 类型 */ + const DictItemData* item = (const DictItemData*)key; + /* 计算字典项数据的哈希值，并返回 unsigned int 类型结果 */ + return DatumGetUInt32(hash_any((const unsigned char*)item->itemData, item->itemSize)); +} + +/* create hash table for building dictionary */ +//创建哈希表并返回哈希表的指针 +static HTAB* hashCreate(const char* name, MemoryContext memCxt, int nelem, int keysize, int entrysize) +{ + /* 定义哈希控制结构体 hashCtrl 并初始化为零 */ + HASHCTL hashCtrl; + errno_t rc = memset_s(&hashCtrl, sizeof(hashCtrl), 0, sizeof(hashCtrl)); + securec_check(rc, "", ""); + /* 设置哈希控制结构体的各个字段 */ + hashCtrl.hash = (HashValueFunc)hashDictItem;/* 设置哈希函数为 hashDictItem */ + hashCtrl.match = (HashCompareFunc)matchDictItem;/* 设置比较函数为 matchDictItem */ + hashCtrl.keysize = (Size)keysize;/* 设置键值大小 */ + hashCtrl.entrysize = entrysize;/* 设置哈希表每个条目的大小 */ + hashCtrl.hcxt = memCxt;/* 设置内存上下文 */ + + int flags = (HASH_FUNCTION | HASH_COMPARE | HASH_ELEM | HASH_CONTEXT); + /* 调用 hash_create 函数创建哈希表并返回 */ + return hash_create(name, nelem, &hashCtrl, flags); +} +//构建一个哈希表，用于存储词典中的词条信息 +void DictCmprMeta::BuildHashTbl(MemoryContext memCxt) +{ + /* 创建哈希表，并将其保存在 m_hash 成员变量中 */ + m_hash = hashCreate("DictInfo Hash", memCxt, dictItemNum + 16, sizeof(DictItemData), sizeof(ItemEntry)); + /* 检查字典词条数量是否超过上限 */ + Assert(dictItemNum <= MAX_DICITEMID); + /* 遍历字典中的词条并插入哈希表 */ + for (int i = 0; i < dictItemNum; ++i) { + /* 在哈希表中查找词条项 */ + ItemEntry* dicItemEntry = (ItemEntry*)hash_search(m_hash, (const void*)(dictItems + i), HASH_ENTER, NULL); + /* 断言查找结果不为空 */ + Assert(dicItemEntry); + /* 设置词条项 ID */ + dicItemEntry->id = (unsigned char)i; + } +} +//根据给定的词条数据，在词典元数据对象的哈希表中查找对应的词条项 +unsigned char DictCmprMeta::GetDictItemId(DictItemData* key, bool& found) +{ + /* 在哈希表中查找词条项 */ + ItemEntry* dicItemEntry = (ItemEntry*)hash_search(m_hash, (const void*)key, HASH_FIND, &found); + /* 如果找到词条项，则返回其 ID */ + if (found) + return dicItemEntry->id; + return 0;//如果未找到词条项，则返回0 +} +//销毁词典元数据对象的哈希表 +void DictCmprMeta::Destroy() +{ + /* 销毁哈希表 */ + if (m_hash) { + hash_destroy(m_hash); + m_hash = NULL; + } +} + +/* write all the dict metadata into given buffer */ +//将所有的字典项数据写入给定的缓冲区，通过循环遍历每个字典项，使用memcpy_s函数将数据复制到缓冲区中，并使用偏移量来确保复制位置的正确性 +void DictCmprMeta::GetDictItems(char* buf) const +{ + int offset = 0;//当前写入缓冲区的偏移量 + errno_t rc = EOK;//保存memcpy_s函数的返回值，以检查是否发生错误 + for (int i = 0; i < dictItemNum; ++i) { + rc = memcpy_s(buf + offset, (size_t)(BLCKSZ - offset), dictItems[i].itemData, (size_t)(dictItems[i].itemSize)); + securec_check(rc, "", ""); + offset += dictItems[i].itemSize; + } +} +//进行数字字符串压缩 +template +Datum PageCompress::NumstrCompress(Datum src, int& outSize) +{ + uint32 evenLen; + int destLen = 0; + uint32 srcLen = 0; + char* srcPtr = NULL; + unsigned char* outBuf = NULL; + unsigned char* destPtr = NULL; + // 检查输入的attlen是否符合预期 + Assert(attlen == -1 || attlen == -2); + // 根据attlen的不同，确定要压缩的源数据和长度 + if (attlen == -1) { + char* p = DatumGetPointer(src); + Assert(p != NULL); + srcPtr = VARDATA_ANY(p); + srcLen = VARSIZE_ANY_EXHDR(p); + } else { + srcPtr = DatumGetPointer(src); + srcLen = (uint32)strlen(srcPtr); + } + + /* for empty string, no need to compress */ + // 如果源数据长度为0，则无需进行压缩 + if (srcLen == 0) { + outSize = 0; + return 0; + } + + Assert(srcPtr != NULL && srcLen > 0); + + /* + * If the srcLen is odd, we need set the highest 4 bits for 1 for last byte + * in order to distinguish the last byte is one original byte + */ + // 将源数据指针转换为无符号字符指针 + unsigned char* srcData = (unsigned char*)srcPtr; + // 如果源数据长度为奇数，则需要在最后一个字节的高4位设置为1， + // 以区分最后一个字节是原始字节还是压缩字节 + if (srcLen % 2 != 0) { + if (!isDigital(srcData[srcLen - 1])) { + outSize = 0; + return 0; + } + + evenLen = srcLen - 1; + destLen = (int)((evenLen >> 1) + 1); + } else { + evenLen = srcLen; + destLen = (int)(evenLen >> 1); + } + Assert(destLen >= 0); + // 根据attlen的不同，分配输出缓冲区 + if (attlen == -1) { + // 如果attlen为-1，则表示压缩后的数据将作为VARLENA类型存储 + outBuf = (unsigned char*)palloc((Size)(destLen + VARHDRSZ)); + if (destLen + VARHDRSZ_SHORT <= VARATT_SHORT_MAX) { + outSize = (int)(destLen + VARHDRSZ_SHORT); + SET_VARSIZE_SHORT(outBuf, outSize); + destPtr = outBuf + VARHDRSZ_SHORT; + } else { + // 如果attlen为-2，则表示压缩后的数据将作为C字符串存储，最后一个字节为'\0' + outSize = destLen + VARHDRSZ; + SET_VARSIZE(outBuf, outSize); + destPtr = outBuf + VARHDRSZ; + } + } else { + /* Last byte has been set '\0' */ + outSize = destLen + 1; + destPtr = outBuf = (unsigned char*)palloc0((Size)outSize); + } + + /* Two bytes combine one byte */ + // 将两个字节组合为一个字节进行压缩 + int pos = 0; + for (uint32 i = 0; i < evenLen; i = i + 2) { + if (isDigital(srcData[i]) && isDigital(srcData[i + 1])) { + // 将两个数字字符转换为一个压缩字节 + destPtr[pos] = (srcData[i] - '0') | (((unsigned int)(srcData[i + 1] - '0')) << 4); + /* + * If attlen is -2, we can't use 0 represent '0''0', because '\0' is 0 + * So we guarantee all code > 0 + */ + /* + * 如果attlen为-2，我们不能使用0表示'0''0'，因为'\0'的ASCII值为0, + * 所以我们保证所有的代码都大于0 + */ + destPtr[pos] += 1; + ++pos; + } else { + // 如果源数据中包含非数字字符，则释放分配的内存并返回 + pfree(outBuf); + outSize = 0; + return (Datum)0; + } + } + + /* Deal with last byte if we need */ + // 处理最后一个字节 + if (srcLen > evenLen) { + destPtr[pos++] = (srcData[srcLen - 1] - '0') | 0xF0; + } + Assert(pos == destLen); + // 返回压缩后的数据指针 + return PointerGetDatum(outBuf); +} +//对压缩的数据进行解压缩，将其还原为原始的数据 +Datum PageCompress::NumstrUncompress( + Datum src, int attlen, int& cmprValSize, char* uncmprValBuf, int* dataLenWithPadding, char attalign) +{ + Assert(attlen == -1 || attlen == -2);//断言，确保attlen的取值只能为-1或-2 + + uint32 srcLen = 0; + char* srcPtr = NULL; + if (attlen == -1) { + char* p = DatumGetPointer(src);//将输入参数src转换为指针类型 + Assert(p != NULL);// 断言，确保指针不为空 + srcPtr = VARDATA_ANY(p);// 获取指针p指向数据的有效部分（去掉头部） + srcLen = VARSIZE_ANY_EXHDR(p);// 获取数据的长度（不包括头部） + cmprValSize = (int)(VARSIZE_ANY(src));// 获取压缩数据的大小（包括头部） + } else { + srcPtr = DatumGetPointer(src);// 将输入参数src转换为指针类型 + srcLen = (uint32)strlen(srcPtr);// 获取指针srcPtr指向数据的长度 + cmprValSize = (int)(srcLen + 1);// 获取压缩数据的大小（加一是为了存储结束符'\0'） + } + Assert(srcPtr != NULL && srcLen > 0 && cmprValSize > 0);// 断言，确保数据指针、长度和大小都有效 + + unsigned char* outBuf = NULL; + unsigned char* destPtr = NULL; + unsigned char* srcData = (unsigned char*)srcPtr;// 将数据指针转换为unsigned char*类型 + int destLen; + int padding_size = 0; + int datum_size = 0; + unsigned char lastByte = srcData[srcLen - 1];// 获取数据的最后一个字节 + + if ((lastByte & 0xF0) == 0xF0) + destLen = (int)(((srcLen - 1) << 1) + 1);// 压缩格式特定情况下，计算解压后数据的长度 + else + destLen = (int)(srcLen << 1);// 计算解压后数据的长度 + Assert(destLen > 0);// 断言，确保解压后数据的长度大于零 + + if (attlen == -1) { + /* + * if the datum header datum_size is 4B, + * 1. aligned is required. + 2. the first byte of padding must be set 0 if padding exists. + * NB: nothing to do if the datum header is 1B; + */ + if (uncmprValBuf) { + outBuf = (unsigned char*)uncmprValBuf; + + Assert(dataLenWithPadding != NULL);// 断言，确保dataLenWithPadding指针不为空 + *outBuf = 0; + outBuf = (unsigned char*)att_align_nominal(outBuf, attalign);// 对齐指针位置，按照attalign的要求 + padding_size = int(outBuf - (unsigned char*)uncmprValBuf);// 计算填充大小 + } else { + outBuf = (unsigned char*)palloc((Size)(destLen + VARHDRSZ));// 分配内存空间，用于存储解压后的数据 + } + + datum_size = destLen + VARHDRSZ;// 数据大小（包括数据头） + SET_VARSIZE(outBuf, datum_size);// 设置数据头的大小 + destPtr = outBuf + VARHDRSZ;// 设置存储解压后数据的指针位置 + } else { + /* Last byte has been set '\0' */ + datum_size = destLen + 1; + if (uncmprValBuf) + destPtr = outBuf = (unsigned char*)uncmprValBuf; + else + destPtr = outBuf = (unsigned char*)palloc((Size)datum_size); + + errno_t rc = memset_s(destPtr, (size_t)datum_size, 0, (size_t)datum_size);// 初始化输出缓冲区为零 + securec_check(rc, "", "");// 检查初始化是否成功 + } + + /* One byte split into two bytes */ + int pos = 0; + for (uint32 i = 0; i < srcLen - 1; ++i) { + destPtr[pos++] = ((srcData[i] - 1) & 0x0F) + '0';// 解压缩数据并转换为ASCII码 + destPtr[pos++] = (((srcData[i] - 1) & 0xF0) >> 4) + '0';// 解压缩数据并转换为ASCII码 + } + + /* Deal with last byte if we need */ + if ((lastByte & 0xF0) == 0xF0) + destPtr[pos++] = (lastByte & 0x0F) + '0';// 解压缩数据并转换为ASCII码 + else { + destPtr[pos++] = ((lastByte - 1) & 0x0F) + '0';// 解压缩数据并转换为ASCII码 + destPtr[pos++] = (((lastByte - 1) & 0xF0) >> 4) + '0';// 解压缩数据并转换为ASCII码 + } + + if (dataLenWithPadding) { + Assert(uncmprValBuf != NULL);// 断言，确保uncmprValBuf指针不为空 + Assert(padding_size >= 0 && datum_size > 0);// 断言，确保填充大小和数据大小都有效 + *dataLenWithPadding = padding_size + datum_size;// 计算数据大小（包括填充大小） + } + + return PointerGetDatum(outBuf);// 返回解压后的数据指针 +} +//整个构造函数的目的是为相关成员变量分配内存空间，并进行必要的初始化操作 +PageCompress::PageCompress(Relation rel, MemoryContext memCtx) +{ + //初始化成员变量 + m_rel = rel; + m_inTups = NULL; + m_inTupsNum = 0; + m_current = 0; + m_outputTupsNum = 0; + m_mappedTupsNum = 0; + m_uncmprTupIdx = 0; + m_cmprTupsNum = 0; + m_toastTupsNum = 0; + m_cmprHeaderSize = 0; + m_outputTups = NULL; + m_cmprTups = NULL; + m_toastTups = NULL; + m_mappedTups = NULL; + m_nextBatch = false; + m_last = false; + + /* these member will be resident in parent memory context. */ + //将上下文切换为指定的内存上下文，并创建子上下文 + MemoryContext oldMemCnxt = MemoryContextSwitchTo(memCtx); + + m_parentMemCxt = memCtx; + m_cmprHeaderData = (char*)palloc(BLCKSZ); + m_pageMemCnxt = AllocSetContextCreate( + memCtx, "PageCompression", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); + + int nattrs = RelationGetNumberOfAttributes(m_rel); + // 分配存储压缩模式、压缩元数据等信息的内存空间，并进行初始化 + m_cmprMode = (char*)palloc0((Size)(nattrs * sizeof(char))); + m_cmprMeta = (void**)palloc0((Size)(nattrs * sizeof(void*))); + m_numTypeOpt = (const NumberTypeOpt**)palloc((Size)(nattrs * sizeof(NumberTypeOpt*))); + m_dictMiss = (uint32*)palloc0((Size)(nattrs * sizeof(uint32))); + m_dictItemId = (unsigned char*)palloc((Size)(nattrs * sizeof(unsigned char))); + m_deformVals = (Datum*)palloc((Size)(nattrs * sizeof(Datum))); + m_deformNulls = (bool*)palloc((Size)(nattrs * sizeof(bool))); + //切换回之前的内存上下文 + (void)MemoryContextSwitchTo(oldMemCnxt); + + Form_pg_attribute* attrs = m_rel->rd_att->attrs; + // 对于每个属性，进行二分查找，获取类型对应的NumberTypeOpt结构体指针，保存在m_numTypeOpt数组中 + for (int col = 0; col < nattrs; ++col) { + m_numTypeOpt[col] = binarySearch(attrs[col]->atttypid); + } + +#ifdef TRACE_COMPRESS + nLoops = 0; +#endif +} +//是PageCompress类的析构函数，用于在对象销毁时释放成员变量占用的内存空间 +PageCompress::~PageCompress() +{ + //释放所有未释放的分配到子上下文的内存空间 + ResetPerPage(); + // 释放分配到当前上下文的所有内存空间 + pfree_ext(m_cmprHeaderData); + pfree_ext(m_cmprMode); + pfree_ext(m_cmprMeta); + pfree_ext(m_numTypeOpt); + pfree_ext(m_dictItemId); + pfree_ext(m_dictMiss); + pfree_ext(m_deformVals); + pfree_ext(m_deformNulls); + pfree_ext(m_cmprTups); + pfree_ext(m_toastTups); + pfree_ext(m_mappedTups); + // 将相关指针设置为NULL + m_inTups = NULL; + m_rel = NULL; + m_parentMemCxt = NULL; + // 删除子上下文 + MemoryContextDelete(m_pageMemCnxt); + m_pageMemCnxt = NULL; +} +//将一组HeapTuple对象解析为列数组 +ColArray* PageCompress::DeformTups(HeapTuple* tups, int nTups) +{ + // 获取关系的属性数量和描述符 + int attrNum = RelationGetNumberOfAttributes(m_rel); + TupleDesc desc = RelationGetDescr(m_rel); + // 分配存储列数组的内存空间 + ColArray* colArray = (ColArray*)palloc((Size)(sizeof(ColArray) * attrNum)); + // 遍历每个属性，为每个属性分配存储值和空值的内存空间，并初始化num值 + for (int i = 0; i < attrNum; ++i) { + colArray[i].nulls = (bool*)palloc0((Size)(sizeof(bool) * nTups)); + colArray[i].val = (Datum*)palloc0((Size)(sizeof(Datum) * nTups)); + colArray[i].num = nTups; + } + + /* storage format is as followings:存储格式如下： + * + * {col_1, row_1} {col_1, row_2} {col_1, row_3} {col_1, row_4} ... + * {col_2, row_1} {col_2, row_2} {col_2, row_3} {col_2, row_4} ... + * {col_3, row_1} {col_3, row_2} {col_3, row_3} {col_3, row_4} ... + * .... + * .... + */ + for (int row = 0; row < nTups; ++row) { + // 使用heap_deform_tuple将HeapTuple解析为列数组 + heap_deform_tuple(tups[row], desc, m_deformVals, m_deformNulls); + // 将解析后的值和空值存储到对应的列数组中 + for (int col = 0; col < attrNum; ++col) { + colArray[col].val[row] = m_deformVals[col]; + colArray[col].nulls[row] = m_deformNulls[col]; + } + } + + return colArray; +} +//Delta压缩/解压缩方法 +/* + * --------------------------------------------------------------- + * Delta Compression/Decompression Methods + * --------------------------------------------------------------- + */ +//获取数据列的Delta压缩元信息，即最小值、最大值和所需字节数 +DeltaCmprMeta* PageCompress::DeltaGetCmprMeta(ColArray* colArray, const NumberTypeOpt* opt) const +{ + // 定义变量存储最小值和最大值 + Datum minVal, maxVal; + // 获取值数组和空值数组 + Datum* values = colArray->val; + bool* isnulls = colArray->nulls; + + /* init min/max value. */ + int i = 0; + while (i < colArray->num && isnulls[i])// 寻找第一个非NULL值 + ++i; + /* if all the values are NULLs, return NULL */// 所有值都是NULL + if (i == colArray->num) + return NULL; + + /* loop and get the min/max value in colArray. */ + // 初始化最小值和最大值 + maxVal = minVal = values[i]; + // 遍历列数组，更新最小值和最大值 + for (i++; i < colArray->num; i++) { + // 跳过NULL值 + if (isnulls[i]) + continue; + // 更新最小值 + if (opt->compare(minVal, values[i]) > 0) + minVal = values[i]; + // 更新最大值 + else if (opt->compare(maxVal, values[i]) < 0) + maxVal = values[i]; + } + // 计算差值 + int64 diff = opt->minus(maxVal, minVal); + // 计算所需字节数 + int bytes = (int)getSizeOfBits(getBitsOfDelta((uint64)diff)); + // 检查是否超过最大字节数 + if (bytes >= g_number_max_value_in_bytes) { + return NULL; + } + // 分配DeltaCmprMeta结构体空间 + DeltaCmprMeta* deltaInfo = (DeltaCmprMeta*)palloc(sizeof(DeltaCmprMeta)); + // 设置最小值、最大值和字节数 + deltaInfo->MinVal = minVal; + deltaInfo->MaxVal = maxVal; + deltaInfo->bytes = bytes; + + return deltaInfo; +} + +/* + * Given both delta meta and a value, compute and return its delta data. + * returning true means that delta compress successes, and deltaBuf is valid whose size would be + * computed by metaInfo->bytes. + * metaInfo, opt and val are the input parameters; and deltaBuf is the output + * returning false means that delta compress fails. + */ +//将给定的数值进行Delta压缩 +bool PageCompress::DeltaCompress( + DeltaCmprMeta* metaInfo, const NumberTypeOpt* opt, Datum val, unsigned char* deltaBuf) const +{ + //计算差值diff，即val和最小值之差 + int64 diff = opt->minus(val, metaInfo->MinVal); + + /* + * if this diff is negative, don't compress; + * if this diff is out of range, don't compress; + */ + /* + * 如果差值diff为负数或超出预设范围，则无法进行压缩，返回false。 + * metaInfo->bytes表示预设的最大字节数，g_number_max_value_in_bytes表示允许的最大字节数。 + */ + Assert(metaInfo->bytes < g_number_max_value_in_bytes); + if ((diff < 0) || (diff > g_max_value_in_bytes[metaInfo->bytes])) { + return false; + } + + /* + * until here, we know that: + * 1. diff is greater than 0; + * 2. the highest byte is 0x00, yes, at most 7 bytes is used and valid. + * 3. diff can be changed to uint64 safely. + */ + /* + * 到这里，我们知道： + * 1. 差值diff大于0； + * 2. 最高字节为0x00，最多使用7个字节。 + * 3. diff可以安全地转换为uint64类型。 + */ + Assert((diff >= 0) && ((diff >> 56) == 0x00)); + // 将差值逐字节存储到deltaBuf中，采用逆序存储的方式。 + uint64 udiff = (uint64)diff; + for (int i = metaInfo->bytes - 1; i >= 0; i--) { + deltaBuf[i] = (udiff & 0xFF);// 获取udiff的最低字节并存入deltaBuf + udiff = udiff >> 8;// 将udiff右移8位，准备获取下一个字节 + } + //返回true表示成功进行了Delta压缩 + return true; +} + +/* [IN]: metaInfo && opt && deltaBuf[] + * delta decompress. + */ +//是Delta压缩的逆算法，即将Delta压缩后的数据解压缩还原成原始的数值 +Datum PageCompress::DeltaUncompress(DeltaCmprMeta* metaInfo, const NumberTypeOpt* opt, unsigned char* deltaBuf) +{ + // 初始化差值为0，并获取Delta压缩后的数据缓冲区指针。 + uint64 diff = 0; + unsigned char* delta = (unsigned char*)deltaBuf; + // 依次读取缓冲区中存储的差值字节，并按字节逆序组合成一个64位无符号整数差值 diff。 + diff |= delta[0]; + for (int i = 1; i < metaInfo->bytes; i++) { + diff = (diff << 8) | delta[i]; + } + // 检查差值最高字节是否为0x00，如果不是则产生异常。 + Assert((diff >> 56) == 0x00); + // 将计算出的差值 diff 和最小值进行加法操作， + // 得到原始的数值，并作为函数返回值返回。 + return opt->plus(metaInfo->MinVal, (int64)diff); +} +//字典解压缩的算法，用于将通过字典压缩得到的数据解压缩还原成原始的数值或指针。 +Datum PageCompress::DictUncompress(DictCmprMeta* metaInfo, const char* compressBuf, int attlen) +{ + Datum res = 0; + unsigned char id = *((unsigned char*)compressBuf); + DictItemData* val = metaInfo->dictItems + id; + + /* refer to fetch_att() */ + // 根据输入的字节长度判断数据类型，然后将压缩后的数据解压缩成原始的数值或指针。 + if (attlen > 0) { + switch (attlen) { + case sizeof(char): { + Assert(0); + break; + } + case sizeof(int16): { + //将字典项中的int16类型数据转换为对应的Datum类型，并将结果赋值给res。 + res = Int16GetDatum(*(int16*)val->itemData); + break; + } + case sizeof(int32): { + //将字典项中的int32类型数据转换为对应的Datum类型，并将结果赋值给res。 + res = Int32GetDatum(*(int32*)val->itemData); + break; + } + case sizeof(Datum): { + res = *(Datum*)val->itemData; + break; + } + default: + //报错，不支持的字节长度。 + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported byval length: %d", (int)(attlen)))); + } + } else + //将字典项中存储的指针类型赋值给res。 + res = PointerGetDatum(val->itemData); + return res;// 返回解压缩后的结果。 +} +//前缀压缩/解压缩方法 +/* + * --------------------------------------------------------------- + * Prefix Compression/Decompression Methods + * --------------------------------------------------------------- + */ +//计算两个字符串类型（压缩前数据）的最大公共前缀长度 +template +int PageCompress::PrefixLen(Datum str1, Datum str2) +{ + char* strData1 = NULL; + char* strData2 = NULL; + int maxPrefixLen = 0; + int prefixLen = 0; + + /* Note that numeric column should not do this. */ + // 如果attlen为-1，说明压缩前的数据是变长类型（如text），需要进行特殊处理。 + if (attlen == -1) { + char* ptr1 = DatumGetPointer(str1); + char* ptr2 = DatumGetPointer(str2); + strData1 = VARDATA_ANY(ptr1);// 获取str1的数据部分 + strData2 = VARDATA_ANY(ptr2);// 获取str2的数据部分 + maxPrefixLen = Min(VARSIZE_ANY_EXHDR(ptr1), VARSIZE_ANY_EXHDR(ptr2)); + // 如果attlen为-2，说明压缩前的数据是以C字符串形式存储的，也需要特殊处理。 + } else if (attlen == -2) { + strData1 = DatumGetCString(str1);// 将str1转换为C字符串 + strData2 = DatumGetCString(str2);// 将str2转换为C字符串 + size_t str_len1 = strlen(strData1);// 计算str1的长度 + size_t str_len2 = strlen(strData2);// 计算str2的长度 + maxPrefixLen = (int)Min(str_len1, str_len2);// 计算str1和str2的最大前缀长度 + } + //从头开始比较str1和str2的字符，直到出现不相同的字符或达到最大前缀长度。 + while (prefixLen < maxPrefixLen && strData1[prefixLen] == strData2[prefixLen]) + ++prefixLen; + + return prefixLen; +} + +/* + * Get common prefix from colArray + * This function outputs prefixLen and prefix string + */ +//在数据压缩中用于获取存储在 colArray 中数据的公共前缀。通过获取公共前缀，可以提高数据的压缩率，减少存储空间的占用 +template +char* PageCompress::PrefixGetCmprMeta(ColArray* colArray, int& prefixLen) +{ + int prev = 0;// 指向前一个元组的索引 + int next = 0;// 指向下一个元组的索引 + int arrayLen = colArray->num;// 数组的长度 + char* prefixStr = NULL;// 存储公共前缀的字符串 + + /* Initialize the prefixLen by comparing the first and second tuple */ + while (prev < arrayLen && colArray->nulls[prev]) + ++prev;// 跳过可能的null值元组 + next = prev + 1; + + while (next < arrayLen && colArray->nulls[next]) + ++next; + + if (next < arrayLen) { + // 获取第一个和第二个元组的公共前缀长度 + prefixLen = PrefixLen(colArray->val[prev], colArray->val[next]); + if (prefixLen == 0) { + /* + * if the first two have no common prefix, we don't continue to advance. + * maybe this trick is unreasonalbe. + */ + return NULL; + } + + Assert(prev < arrayLen); + // 根据 attlen 的不同情况获取第一个元组的前缀字符串 + if (attlen == -1) + prefixStr = VARDATA_ANY(DatumGetPointer(colArray->val[prev])); + else + prefixStr = DatumGetCString(colArray->val[prev]); + + /* Loop for geting the final prefixLen */ + prev = next; + for (++next; next < arrayLen; ++next) { + // 跳过可能的null值元组 + while (next < arrayLen && colArray->nulls[next]) { + ++next; + } + if (next == arrayLen) { + break; + } + // 计算当前元组与前一个元组的公共前缀长度 + int len = PrefixLen(colArray->val[prev], colArray->val[next]); + if (prefixLen > len && len > 0) { + prefixLen = len;//更新公共前缀的长度为较小的 len 值 + } + prev = next; + } + } + return prefixStr;//返回最终的公共前缀字符串 +} +//对属性值进行前缀压缩，并返回压缩后的结果 +template +bool PageCompress::PrefixCompress(PrefixCmprMeta* prefix, Datum val, Datum& cmprsVal, int& cmprsValSize) +{ + char* attStart = NULL;// 属性值的起始位置 + int attSize;// 属性值的大小 + /* 根据属性长度进行处理 */ + if (-1 == attlen) { + char* ptr = DatumGetPointer(val); + Assert(ptr != NULL); + attStart = VARDATA_ANY(ptr);// 获取属性值的数据部分 + attSize = VARSIZE_ANY_EXHDR(ptr);// 获取属性值的大小（不包括头部） + } else { + attStart = DatumGetCString(val);// 获取属性值字符串的起始位置 + attSize = (int)(strlen(attStart) + 1);// 获取属性值字符串的大小（包括结尾的'\0'） + } + Assert(attStart != NULL && attSize >= 0); + Assert(prefix != NULL && prefix->len > 0 && prefix->prefixStr != NULL); + /* 检查属性值是否以指定的前缀开头 */ + if (prefix->len > attSize || memcmp(prefix->prefixStr, attStart, (size_t)prefix->len)) { + return false; + } + /* 进行压缩 */ + if (-1 == attlen) { + int suffixLen = attSize - prefix->len;// 计算后缀的大小 + Assert(suffixLen >= 0); + char* suffix = (char*)palloc((Size)(suffixLen + VARHDRSZ));// 申请后缀存储空间 + errno_t errorno = EOK; + + /* use one byte var-header as possible as much */ + /* 尽可能使用短变长头部 */ + if (suffixLen + VARHDRSZ_SHORT <= VARATT_SHORT_MAX) { + SET_VARSIZE_SHORT(suffix, suffixLen + VARHDRSZ_SHORT);// 设置短变长头部的大小 + if (suffixLen > 0) { + errorno = memcpy_s((suffix + VARHDRSZ_SHORT), (size_t)suffixLen, (attStart + prefix->len), (size_t)suffixLen); + securec_check(errorno, "\0", "\0"); + } + cmprsValSize = (int)(suffixLen + VARHDRSZ_SHORT);// 设置压缩后的大小 + } else { + SET_VARSIZE(suffix, suffixLen + VARHDRSZ);// 设置普通变长头部的大小 + errorno = memcpy_s((suffix + VARHDRSZ), (size_t)suffixLen, (attStart + prefix->len), (size_t)suffixLen); + securec_check(errorno, "\0", "\0"); + cmprsValSize = suffixLen + VARHDRSZ;//设置压缩后的大小 + } + cmprsVal = PointerGetDatum(suffix);//设置压缩后的属性值 + } else { + cmprsVal = CStringGetDatum(attStart + prefix->len);//设置压缩后的属性值 + cmprsValSize = (int)(strlen(attStart + prefix->len) + 1);//设置压缩后的大小 + } + Assert(cmprsValSize > 0); + + return true; +} +//对属性值进行前缀解压缩，将压缩后的属性值恢复为原始的属性值 +Datum PageCompress::PrefixUncompress(int attlen, PrefixCmprMeta* prefix, char* start, int* attrCmprsValSize, + char* uncmprValBuf, int* dataLenWithPadding, char attalign) +{ + int attsize; + char* attdata = NULL; + char* attr = NULL; + int datum_size; + int padding_size = 0; + errno_t rc = EOK; + + Assert(-1 == attlen || -2 == attlen); + Assert(prefix != NULL && prefix->len > 0 && prefix->prefixStr != NULL); + Assert(start != NULL && attrCmprsValSize != NULL); + // 判断属性值是否为二进制数据类型 + if (-1 == attlen) { + char* ptr = DatumGetPointer(start); + attdata = VARDATA_ANY(ptr);// 获取属性值数据部分的起始位置 + attsize = VARSIZE_ANY_EXHDR(ptr);// 获取属性值的大小（不包括头部） + *attrCmprsValSize = VARSIZE_ANY(ptr);// 设置属性压缩后的大小 + Assert(attdata != NULL && attsize >= 0 && *attrCmprsValSize > 0); + + int attRawLen = prefix->len + attsize;// 计算属性值解压缩后的大小 + + /* + * if the datum header datum_size is 4B, + * 1. aligned is required. + * 2. the first byte of padding must be set 0 if padding exists. + * NB: nothing to do if the datum header is 1B; + */ + /* + * 如果属性值的头部大小为 4 字节， + * 1. 需要进行对齐操作。 + * 2. 如果存在填充字节，填充字节的第一个字节必须设置为 0。 + * 注意：如果属性值的头部大小为 1 字节，则无需进行任何操作。 + */ + if (uncmprValBuf) { + attr = uncmprValBuf; + + Assert(dataLenWithPadding != NULL); + *attr = 0; + attr = (char*)att_align_nominal(attr, attalign);// 进行对齐操作 + padding_size = (int)(attr - uncmprValBuf);// 计算填充字节的大小 + } else { + attr = (char*)palloc((Size)(attRawLen + VARHDRSZ));// 分配存储解压缩后属性值的空间 + } + + datum_size = attRawLen + VARHDRSZ;// 计算解压缩后属性值的总大小（包括头部） + SET_VARSIZE(attr, datum_size);// 设置解压缩后属性值的大小 + int offset = VARHDRSZ; + rc = memcpy_s((attr + offset), (size_t)(datum_size - offset), prefix->prefixStr, (size_t)prefix->len); + securec_check(rc, "", ""); + offset += prefix->len; + if (attsize > 0) { + rc = memcpy_s((attr + offset), (size_t)(datum_size - offset), attdata, (size_t)attsize); + securec_check(rc, "", ""); + } + } else { + /* compressed attribute's info */ + /* 压缩属性的信息 */ + attdata = DatumGetCString(start);// 获取属性值字符串 + attsize = (int)strlen(attdata);// 获取属性值的大小 + *attrCmprsValSize = attsize + 1;// 设置属性压缩后的大小 + Assert(attdata != NULL && attsize >= 0 && *attrCmprsValSize > 0); + + datum_size = prefix->len + attsize + 1;// 计算解压缩后属性值的总大小 + if (uncmprValBuf) + attr = uncmprValBuf; + else + attr = (char*)palloc((Size)datum_size);// 分配存储解压缩后属性值的空间 + rc = memcpy_s(attr, (size_t)datum_size, prefix->prefixStr, (size_t)prefix->len);// 拷贝前缀字符串 + securec_check(rc, "", ""); + if (attsize > 0) { + rc = memcpy_s((attr + prefix->len), datum_size - prefix->len, attdata, attsize);// 拷贝属性值 + securec_check(rc, "", ""); + } + attr[datum_size - 1] = '\0';// 设置字符串结尾的 \0 字符 + } + + if (dataLenWithPadding) { + Assert(uncmprValBuf != NULL); + Assert(padding_size >= 0 && datum_size > 0); + *dataLenWithPadding = padding_size + datum_size;// 计算解压缩后属性值的总大小（包括填充字节） + } + + return PointerGetDatum(attr);// 返回解压缩后的属性值指针 +} + +/* choose compression method between delta and dict.选择Delta和字典之间的压缩方法。 + * we prefer delta method to dict method.我们更倾向于使用Delta方法而不是字典方法。 + */ +//选择固定长度类型属性的压缩方法 +void PageCompress::ChooseMethodForFixedLenType(ColArray* colArray, int col) +{ + int hintMode = m_rel->rd_att->attrs[col]->attcmprmode; + + /* first, try to use the user's hint */ + /* 首先，尝试使用用户的提示 */ + if (SetCmprMethod(colArray, col, hintMode)) + return; + + /* then, try to check whether delta value encoding is proper */ + /* 然后，尝试检查是否适合使用Delta值编码的压缩方法 */ + if (CMPR_DELTA != hintMode && SetCmprMethod(colArray, col, CMPR_DELTA)) + return; + + /* then, try to check dictionary compress method */ + /* 接着，尝试检查字典压缩方法 */ + if (CMPR_DICT != hintMode && SetCmprMethod(colArray, col, CMPR_DICT)) + return; + + (void)SetCmprMethod(colArray, col, CMPR_NONE); +} +//用于填充字典项数据结构的 key 值 +template +void PageCompress::FillHashKey(DictItemData& key, const int& attlen, const Datum& attval) +{ + //根据不同的attFlag值选择不同的处理方式 + if (attFlag > 0) { + //处理固定长度类型的属性 + key.itemSize = (int16)attlen; + key.itemData = (char*)palloc((Size)attlen); + store_att_byval(key.itemData, attval, attlen);//将属性值存储在key的itemData中 + } else if (-1 == attFlag) { + //处理二进制数据类型的属性 + key.itemData = DatumGetPointer(attval);//将属性值指针存储在key的itemData中 + key.itemSize = (int16)VARSIZE_ANY(key.itemData);//设置属性大小为VARSIZE_ANY(key.itemData) + } else if (-2 == attFlag) { + //处理字符串类型的属性 + key.itemData = DatumGetPointer(attval);// 将属性值指针存储在key的itemData中 + key.itemSize = (int16)(strlen(key.itemData) + 1);// 设置属性大小为字符串长度加1 + } +} +//评估一个列数组中的数值型字符串是否适合进行压缩 +/* Choose dictinary items and put it into DictCmprMeta */ +template +DictCmprMeta* PageCompress::DictGetCmprMeta(ColArray* colArray, int col, int attlen) +{ + int i; + + /* + * If the times of failing to choose dictionary compression exceeds + * the threshold, we shouldn't waste CPU to try again. + */ + /* + * 如果选择字典压缩方法失败的次数超过了阈值，则直接返回 NULL + * 防止浪费 CPU 时间 + */ + if (m_dictMiss[col] > DICT_MISS_THRESHOLD) + return NULL; + // 创建一个哈希表 dictHash 用于存储字典项 + DictCmprMeta* dicInfo = (DictCmprMeta*)palloc0(sizeof(DictCmprMeta)); + int pos = 0; + bool found = false; + HTAB* dictHash = + hashCreate("DictHash", m_pageMemCnxt, colArray->num + 16, sizeof(DictItemData), sizeof(DictItemEntry)); + // 遍历 colArray 中的每个元素 + for (i = 0; i < colArray->num; ++i) { + if (colArray->nulls[i]) { + continue; + } + // 根据 attFlag 计算字典项的 key 值 + DictItemData key; + if (attFlag > 0) { + FillHashKey<1>(key, attlen, colArray->val[i]); + } else if (attFlag == -1) { + FillHashKey<-1>(key, attlen, colArray->val[i]); + } else if (attFlag == -2) { + FillHashKey<-2>(key, attlen, colArray->val[i]); + } + // 查找哈希表 dictHash，如果不存在该字典项则插入该字典项 + DictItemEntry* dicItemEntry = (DictItemEntry*)hash_search(dictHash, (const void*)&key, HASH_ENTER, &found); + Assert(dicItemEntry != NULL); + if (!found) { + // 如果是第一次出现该字典项，则初始化 hitCount 和 beChoose 计数器 + dicItemEntry->hitCount = 1; + dicItemEntry->beChoose = false; + continue; + } + ++dicItemEntry->hitCount; + + /* + * If the repeated times exceeds DICTITEM_NUM_FACTOR, then we should + * choose it as dictionary item + * Description: use a priority queue to keep Top N dictionary item by hitCount + */ + /* + * 如果该字典项的 hitCount 计数器达到一定阈值，则选择该字典项作为字典项 + * 最后存储在 dicInfo 结构中，并更新总字节数和 pos 计数器 + */ + if (!dicItemEntry->beChoose && + (dicItemEntry->hitCount >= Max(colArray->num / DICTITEM_NUM_FACTOR, DICTITEM_MIN_REPEATS))) { + dicItemEntry->beChoose = true; + + dicInfo->dictItems[pos++] = key; + dicInfo->dictItemNum = pos; + dicInfo->totalBytes += key.itemSize; + Assert(dicInfo->dictItemNum <= (colArray->num >> 1)); + + if (dicInfo->dictItemNum >= MAX_DICITEMID) + break; + } + } + // 销毁哈希表 dictHash，并判断是否成功选择了字典项 + hash_destroy(dictHash); + if (pos > 0) + return dicInfo; + // 增加选择字典压缩方法失败次数的计数器，释放内存并返回 NULL + ++m_dictMiss[col]; + pfree(dicInfo); + return NULL; +} +/** + * 该函数用于评估 colArray 中的数值型字符串是否适合进行压缩。 + * + * @param colArray 待评估的列数组 + * @return 是否适合压缩 + */ +template +bool PageCompress::NumstrEvaluate(ColArray* colArray) +{ + int n = colArray->num;// 获取元素数量 + int numStr = 0;// 符合要求的数值型字符串数量 + bool flag = true;// 是否符合要求的标志 + Datum* val = colArray->val;// 数据值指针数组 + bool* nulls = colArray->nulls;// 空值标志数组 + + for (int i = 0; i < n; ++i) { + if (nulls[i]) + continue;// 跳过空值元素 + + int len = 0; + char* dataPtr = NULL; + + if (attlen == -1) { + char* p = DatumGetPointer(val[i]); + dataPtr = VARDATA_ANY(p);// 获取非定长字符串数据指针 + len = VARSIZE_ANY_EXHDR(p);// 获取非定长字符串长度 + } else if (attlen == -2) { + dataPtr = DatumGetPointer(val[i]); + len = (int)strlen(dataPtr);// 获取定长字符串长度 + } + + /* do not count empty string */ + if (len == 0) { + continue;// 跳过空字符串 + } + + for (int j = 0; j < len; ++j) { + if (!isDigital(dataPtr[j])) { + flag = false;// 包含非数字字符，将标志设为false + break; + } + } + + if (flag == false) + break;// 如果有非数字字符，跳出循环 + + /* + * a little trick: we needn't check all the values, and if the + * the first some values are suitable for compression, retrun + * and try to compress them later. + */ + if (++numStr > 2) { + flag = true;// 数值型字符串数量大于2，将标志设为true + + break; + } + } + + /* no need to compress if all sample values are null or empty string */ + return (0 == numStr) ? false : flag;// 如果符合要求的数值型字符串数量为0，则返回false；否则返回标志位 +} +} + +/* choose compression method between prefix and dict. + * we prefer prefix method to dict method. + */ +//选择压缩方法来对变长类型的列进行压缩 +//设置列的压缩方法和保存相应的压缩元数据 +void PageCompress::ChooseMethodForVaryLenType(ColArray* colArray, int col) +{ + int hintMode = m_rel->rd_att->attrs[col]->attcmprmode; + + /* first, try to use the user's hint */ + if (SetCmprMethod(colArray, col, hintMode))// 尝试使用用户提示的压缩模式 + return; + + /* then, try to check dictionary compress method */ + if (CMPR_DICT != hintMode && SetCmprMethod(colArray, col, CMPR_DICT))// 如果用户提示不是字典压缩模式，则尝试使用字典压缩方法 + return; + + /* + * Try to get prefix. + * Note that numeric column should not do this + */ + if (CMPR_PREFIX != hintMode && SetCmprMethod(colArray, col, CMPR_PREFIX))// 如果用户提示不是前缀压缩模式且列不是数值型，则尝试使用前缀压缩方法 + return; + + /* then, try to number-string compress method */ + if (CMPR_NUMSTR != hintMode && SetCmprMethod(colArray, col, CMPR_NUMSTR))// 如果用户提示不是数值型字符串压缩模式，则尝试使用数值型字符串压缩方法 + return; + + (void)SetCmprMethod(colArray, col, CMPR_NONE);// 最后设置为无压缩 +} + +/* + * if user has specify the compress method by CREATE TABLE sql clause, + * we take it as a hint, and first try to get compression metadata using it. + * but maybe it fails, and we have to try the other methods then. + */ +bool PageCompress::SetCmprMethod(ColArray* colArray, int col, int mode) +{ + Form_pg_attribute* attrs = m_rel->rd_att->attrs; + int attlen = attrs[col]->attlen; + + switch (mode) { + case CMPR_DELTA: { + // 检查列是否为数值类型且长度大于1 + Assert(m_numTypeOpt[col] != NULL && attlen > 1); + // 获取列的增量压缩元数据 + DeltaCmprMeta* deltaInfo = DeltaGetCmprMeta(colArray, m_numTypeOpt[col]); + + if (deltaInfo == NULL) + return false; + + /* + * this if condition tells us that it's always true that + * deltaInfo->bytes < 8 + */ + /* + * 这个if条件告诉我们， + * deltaInfo->bytes永远小于8 + */ + Assert(deltaInfo->bytes <= g_number_max_value_in_bytes); + if (deltaInfo->bytes >= attlen) { + pfree(deltaInfo); + return false; + } + // 设置压缩方法为CMPR_DELTA并保存元数据 + m_cmprMode[col] = CMPR_DELTA; + m_cmprMeta[col] = deltaInfo; + return true; + } + + case CMPR_DICT: { + DictCmprMeta* dicInfo = NULL; + // 根据列长度选择字典压缩方法 + if (-2 == attlen) + dicInfo = DictGetCmprMeta<-2>(colArray, col, attlen); + else if (-1 == attlen) + dicInfo = DictGetCmprMeta<-1>(colArray, col, attlen); + else + dicInfo = DictGetCmprMeta<1>(colArray, col, attlen); + + if (dicInfo == NULL) + return false; + // 构建字典压缩的哈希表 + dicInfo->BuildHashTbl(m_pageMemCnxt); + // 设置压缩方法为CMPR_DICT并保存元数据 + m_cmprMode[col] = CMPR_DICT; + m_cmprMeta[col] = dicInfo; + return true; + } + + case CMPR_PREFIX: { + int prefixLen = 0; + char* prefixStr = NULL; + + Assert(-1 == attlen || -2 == attlen); + // 根据列长度选择前缀压缩方法 + if (-2 == attlen) + prefixStr = PrefixGetCmprMeta<-2>(colArray, prefixLen); + else if (-1 == attlen) + prefixStr = PrefixGetCmprMeta<-1>(colArray, prefixLen); + + /* If this array has common prefix, we fill prefix compression meta information */ + /* 如果该数组具有公共前缀，则填充前缀压缩的元数据 */ + if (prefixLen > 0 && prefixLen < MAX_PREFIX_LEN) { + PrefixCmprMeta* prefixInfo = (PrefixCmprMeta*)palloc(sizeof(PrefixCmprMeta)); + prefixInfo->len = prefixLen; + prefixInfo->prefixStr = (char*)palloc((Size)prefixLen); + errno_t rc = memcpy_s(prefixInfo->prefixStr, (size_t)prefixLen, prefixStr, (size_t)prefixLen); + securec_check(rc, "", ""); + // 设置压缩方法为CMPR_PREFIX并保存元数据 + m_cmprMeta[col] = prefixInfo; + m_cmprMode[col] = CMPR_PREFIX; + return true; + } + return false; + } + + case CMPR_NUMSTR: { + Assert(-1 == attlen || -2 == attlen); + // 根据列长度评估数值字符串压缩方法 + bool NumerStringFlag = (-1 == attlen) ? NumstrEvaluate<-1>(colArray) : NumstrEvaluate<-2>(colArray); + if (NumerStringFlag) { + // 设置压缩方法为CMPR_NUMSTR + m_cmprMode[col] = CMPR_NUMSTR; + m_cmprMeta[col] = NULL; + return true; + } + return false; + } + + case CMPR_UNDEF: + return false; + + case CMPR_NONE: + default: + // 设置压缩方法为CMPR_NONE + m_cmprMode[col] = CMPR_NONE; + m_cmprMeta[col] = NULL; + return true; + } +} +//根据属性列的特性选择适当的压缩方法 +void PageCompress::ChooseCmprMethod(ColArray* colArrays) +{ + int attrNum = RelationGetNumberOfAttributes(m_rel); + Form_pg_attribute* attrs = m_rel->rd_att->attrs; + + for (int col = 0; col < attrNum; ++col) { + // 如果该属性列的attcmprmode为CMPR_NONE，表示用户指定不进行压缩，则跳过该属性列的处理。 + /* continue if this attribute shouldn't be compressed specified by user; */ + if (CMPR_NONE == attrs[col]->attcmprmode) + continue; + // 如果该属性列被用户删除，则将该属性列的压缩方法设置为CMPR_NONE。 + /* continue if this attribute is dropped by user */ + if (isAttrDropped(attrs[col])) { + (void)SetCmprMethod(colArrays + col, col, CMPR_NONE); + continue; + } + // 如果该属性列是固定长度类型（m_numTypeOpt[col] != NULL），则调用ChooseMethodForFixedLenType函数选择压缩方法。 + if (m_numTypeOpt[col] != NULL) { + Assert(attrs[col]->attlen > 1); + ChooseMethodForFixedLenType(colArrays + col, col); + } else if (attrs[col]->attlen > 0) {// 如果该属性列的长度大于0，表示该属性列类型不进行压缩，则将该属性列的压缩方法设置为CMPR_NONE。 + /* These types are not compressed. */ + // 这些类型不进行压缩。 + (void)SetCmprMethod(colArrays + col, col, CMPR_NONE); + } else { + // 否则，该属性列是可变长度类型，调用ChooseMethodForVaryLenType函数选择压缩方法。 + else { + // 否则，该属性列是可变长度类型，调用ChooseMethodForVaryLenType函数选择压缩方法。 + Assert(attrs[col]->attlen == -1 || attrs[col]->attlen == -2); + ChooseMethodForVaryLenType(colArrays + col, col); + } + } +} + +void PageCompress::SetCmprHeaderData(void) +{ + Form_pg_attribute* atts = RelationGetDescr(m_rel)->attrs; + int nattrs = RelationGetNumberOfAttributes(m_rel); + int offset = 0; + int rawCols = 0; + errno_t rc = EOK; + + for (int col = 0; col < nattrs; ++col) { + m_cmprHeaderData[offset++] = m_cmprMode[col]; /* compression mode *//* 压缩模式 */ + + switch (m_cmprMode[col]) { + case CMPR_DELTA: { + DeltaCmprMeta* deltaInfo = (DeltaCmprMeta*)m_cmprMeta[col]; + int attrLen = atts[col]->attlen; + Assert(deltaInfo->bytes < g_number_max_value_in_bytes);// 断言：delta值占用的字节数小于g_number_max_value_in_bytes + Assert(attrLen > 0);// 断言：属性列长度大于0 + + /* how many bytes is used for delta value */ + m_cmprHeaderData[offset++] = (char)deltaInfo->bytes;// 存储delta值占用的字节数到压缩头部数据 + /* the min/base value */ + store_att_byval(m_cmprHeaderData + offset, deltaInfo->MinVal, attrLen);// 存储最小/基准值到压缩头部数据 + offset += attrLen; + break; + } + + case CMPR_DICT: { + DictCmprMeta* dicInfo = (DictCmprMeta*)m_cmprMeta[col]; + Assert(dicInfo->dictItemNum < 0xFF);// 断言：字典项数量小于0xFF + /* how many dictionary items */ + m_cmprHeaderData[offset++] = (char)dicInfo->dictItemNum;// 存储字典项数量到压缩头部数据 + + dicInfo->GetDictItems(m_cmprHeaderData + offset);// 存储字典项数据到压缩头部数据 + offset += dicInfo->totalBytes; + break; + } + + case CMPR_PREFIX: { + Assert(-2 == atts[col]->attlen || -1 == atts[col]->attlen);// 断言：属性列长度为-2或-1 + PrefixCmprMeta* prefixInfo = (PrefixCmprMeta*)m_cmprMeta[col]; + Assert(prefixInfo->len > 0 && prefixInfo->len < MAX_PREFIX_LEN);// 断言：前缀长度大于0且小于MAX_PREFIX_LEN + /* how many bytes is used for prefix string */ + m_cmprHeaderData[offset++] = (char)prefixInfo->len;// 存储前缀长度到压缩头部数据 + /* data of prefix string */ + rc = memcpy_s(m_cmprHeaderData + offset, (size_t)(BLCKSZ - offset), prefixInfo->prefixStr, (size_t)(prefixInfo->len));// 存储前缀字符串到压缩头部数据 + securec_check(rc, "", ""); + offset += prefixInfo->len; + break; + } + + case CMPR_NUMSTR: + break; + + case CMPR_NONE: + ++rawCols;// 记录原始列的数量 + break; + default: + break; + } + } + + /* + * the same to special space in page, start address of compression header + * is aligned using MAXALIGN(). + */ + m_cmprHeaderSize = (nattrs == rawCols) ? 0 : (int)MAXALIGN((uint32)offset); + Assert(m_cmprHeaderSize < BLCKSZ);// 断言：压缩头部数据的大小小于页的大小 +} + +/* fetch and parse the compression meta data, then return meta infomation object. + * metaStart [IN]: start address of compression metadata within page for each attribute; + * attrRawLen [IN]: fixed length of this attribute, it's from pg_attr->attlen; + * metaSize [OUT]: size of compression metadata within page for each attribute; + * mode [OUT]: compression method; + */ +//从存储页的压缩元数据中提取并解析所需的压缩信息 +void* PageCompress::FetchAttrCmprMeta(char* metaStart, const int attrRawLen, int* metaSize, char* mode) +{ + int offset = 1; + *mode = metaStart[0];// 获取压缩模式 + void* retInfo = NULL;// 返回的元信息对象指针 + + switch (*mode) { + case CMPR_DELTA: {// Delta压缩模式 + DeltaCmprMeta* deltaInfo = &t_thrd.storage_cxt.cmprMetaInfo->deltaInfo; + + /* how many bytes is used for delta value */ + deltaInfo->bytes = metaStart[offset];// 获取delta值占用的字节数 + ++offset; + + Assert(attrRawLen > 0);// 断言属性列长度大于0 + /* read and fetch the min/base value */ + deltaInfo->MinVal = fetch_att(metaStart + offset, true, attrRawLen); + offset += attrRawLen; + *metaSize = offset;// 设置元数据大小 + retInfo = (void*)deltaInfo;// 返回Delta元信息对象 + break; + } + case CMPR_DICT: {// 字典压缩模式 + DictCmprMeta* dicInfo = &t_thrd.storage_cxt.cmprMetaInfo->dicInfo; + dicInfo->dictItemNum = metaStart[offset++];// 获取字典项数量 + + if (attrRawLen > 0) {// 属性列长度大于0 + for (int i = 0; i < dicInfo->dictItemNum; ++i) { + dicInfo->dictItems[i].itemData = metaStart + offset;// 设置字典项数据地址 + dicInfo->dictItems[i].itemSize = (int16)attrRawLen;// 设置字典项数据大小 + offset += attrRawLen;// 偏移量增加 + } + } else if (attrRawLen == -1) {// 属性列长度为-1，即变长类型（TOAST） + for (int i = 0; i < dicInfo->dictItemNum; ++i) { + dicInfo->dictItems[i].itemData = metaStart + offset;// 设置字典项数据地址 + dicInfo->dictItems[i].itemSize = (int16)VARSIZE_ANY(dicInfo->dictItems[i].itemData);// 设置字典项数据大小 + offset += dicInfo->dictItems[i].itemSize;// 偏移量增加 + } + } else if (attrRawLen == -2) {// 属性列长度为-2，即C字符串 + for (int i = 0; i < dicInfo->dictItemNum; ++i) { + dicInfo->dictItems[i].itemData = metaStart + offset;// 设置字典项数据地址 + dicInfo->dictItems[i].itemSize = (int16)strlen(dicInfo->dictItems[i].itemData) + 1;// 设置字典项数据大小 + offset += dicInfo->dictItems[i].itemSize;// 偏移量增加 + } + } + *metaSize = offset;// 设置元数据大小 + retInfo = (void*)dicInfo;// 返回字典元信息对象 + break; + } + + case CMPR_PREFIX: {// 前缀压缩模式 + Assert(-2 == attrRawLen || -1 == attrRawLen);// 断言属性列长度为-2或-1 + PrefixCmprMeta* prefixInfo = &t_thrd.storage_cxt.cmprMetaInfo->prefixInfo; + prefixInfo->len = (metaStart[offset] & 0x00FF);// 获取前缀长度 + ++offset; + prefixInfo->prefixStr = metaStart + offset;// 设置前缀字符串地址 + offset += prefixInfo->len;// 偏移量增加 + *metaSize = offset;// 设置元数据大小 + retInfo = (void*)prefixInfo;// 返回前缀元信息对象 + break; + } + + case CMPR_NUMSTR: + case CMPR_NONE: { + *metaSize = offset;// 设置元数据大小 + retInfo = (void*)NULL;// 返回空指针 + break; + } + + default: { + Assert(false);// 触发断言 + retInfo = (void*)NULL;// 返回空指针 + break; + } + } + + return retInfo;// 返回元信息对象 +} + +/* according to compression mode and meta information, return the value size + * of this attribute in the tuple. + */ +//根据压缩模式和元信息，给定一个属性值，计算出在经过压缩后，该属性值所占用的空间大小 +int PageCompress::GetAttrCmprValSize(char mode, int attlen, void* metaInfo, const char* attrStart) +{ + switch (mode) { + case CMPR_DELTA: { + DeltaCmprMeta* deltaInfo = (DeltaCmprMeta*)metaInfo; + return deltaInfo->bytes;// 返回 Delta 压缩模式中指定的 delta 值字节数 + } + + case CMPR_DICT: { + DictCmprMeta* dicInfo = (DictCmprMeta*)metaInfo; + return (dicInfo->dictItemNum == 1) ? 0 : DICT_ITEMID_SIZE;// 如果字典项数量为1，则返回0；否则返回字典项ID的大小 + } + + case CMPR_PREFIX: + case CMPR_NUMSTR: { + return (attlen == -1) ? VARSIZE_ANY(attrStart) : (strlen(attrStart) + 1);// 根据属性列长度的不同，返回不同类型压缩后的值大小 + } + + case CMPR_NONE: { + Assert(false);// 断言触发，无压缩模式不应该调用该函数 + break; + } + default: + break; + } + + Assert(false);// 断言触发，无法识别的压缩模式 + return 0; +} + +/* + * external buffer must be offered for some compress method, in order to + * reduce the times of multi memcpy() calls. + */ +//判断给定的压缩模式是否需要提供外部缓冲区 +bool PageCompress::NeedExternalBuf(char mode) +{ + // 检查给定的压缩模式是否需要提供外部缓冲区 + return ((mode == CMPR_PREFIX) || (mode == CMPR_NUMSTR)); +} +//用于对属性值进行解压缩操作 +int PageCompress::UncompressOneAttr( + char mode, char attalign, int attlen, void* metaInfo, char* attrStart, int* attrCmprsValSize, char* uncmprValBuf) +{ + int dataLenWithPadding = 0; + // 断言属性长度attlen只能为-1（变长类型TOAST）或者-2（固定长度类型，且对齐方式为c字符型） + Assert((attlen == -1) || (attlen == -2 && attalign == 'c')); + // 确保给定的压缩模式需要提供外部缓冲区 + Assert(NeedExternalBuf(mode)); + + switch (mode) { + case CMPR_PREFIX: { + PrefixCmprMeta* prefixInfo = (PrefixCmprMeta*)metaInfo; + // 调用PrefixUncompress函数对属性值进行解压缩 + // 将解压缩后的数据存储在uncmprValBuf中 + // 将解压缩后的数据长度（包括填充字节）保存在dataLenWithPadding中 + (void)PrefixUncompress( + attlen, prefixInfo, attrStart, attrCmprsValSize, uncmprValBuf, &dataLenWithPadding, attalign); + break; + } + + case CMPR_NUMSTR: { + // 调用NumstrUncompress函数对属性值进行解压缩 + // 将attrStart作为PointerGetDatum传入，并指定attlen和attrCmprsValSize + // 将解压缩后的数据存储在uncmprValBuf中 + // 将解压缩后的数据长度（包括填充字节）保存在dataLenWithPadding中 + (void)NumstrUncompress( + PointerGetDatum(attrStart), attlen, *attrCmprsValSize, uncmprValBuf, &dataLenWithPadding, attalign); + break; + } + + default: + Assert(false);// 如果无法识别给定的压缩模式，则触发断言错误 + break; + } + // 返回解压缩后的数据长度（包括填充字节） + return dataLenWithPadding; +} + +/* accordint to compression mode and meta information, set the length of compressed value, + * decompress this attribute in the tuple, and return both of them. + * mode [IN]: compression mode; + * metaInfo [IN]: compression meta information; + * typeOid [IN]: type oid of the attribute; + * attlen [IN]: attr len of this attribute; + * attrStart [IN]: start of attr data; + * attrCmprsValSize [OUT]: length of compressed value; + */ +//对给定的压缩模式和元信息进行解压缩，将压缩后的属性值恢复为原始的未压缩形式 +Datum PageCompress::UncompressOneAttr( + char mode, void* metaInfo, Oid typeOid, int attlen, char* attrStart, int* attrCmprsValSize) +{ + switch (mode) { + case CMPR_DELTA: { + DeltaCmprMeta* deltaInfo = (DeltaCmprMeta*)metaInfo; + if (deltaInfo->bytes == 0) {// 如果bytes为0，表示属性值未压缩 + *attrCmprsValSize = 0; + return deltaInfo->MinVal; // 直接返回MinVal字段作为解压缩后的属性值 + } + + *attrCmprsValSize = deltaInfo->bytes;// 设置压缩值的长度为bytes字段的值 + + /* + * typeOid is InvalidOid when this attribute is dropped. But its + * typlen and typalign are also always kept. in this case, its + * value cann't be accessed and computed because of lost atttype. + * so that we just only skip it, and continue to access next valid attr. + * + * please refer to RemoveAttributeById(). + */ + if (InvalidOid == typeOid) {// 如果typeOid为InvalidOid，表示该属性已被删除 + return (Datum)0;// 返回空值 + } + + const NumberTypeOpt* opt = binarySearch(typeOid);// 根据typeOid进行二分查找 + return DeltaUncompress(deltaInfo, opt, (unsigned char*)attrStart);// 调用DeltaUncompress函数进行解压缩 + } + + case CMPR_DICT: { + DictCmprMeta* dicInfo = (DictCmprMeta*)metaInfo; + + if (dicInfo->dictItemNum == 1) {// 如果dictItemNum为1，表示属性值未压缩 + *attrCmprsValSize = 0; + char id = 0; + return DictUncompress(dicInfo, &id, attlen);// 直接返回字典的第一个值作为解压缩后的属性值 + } + *attrCmprsValSize = DICT_ITEMID_SIZE;// 设置压缩值的长度为DICT_ITEMID_SIZE + return DictUncompress(dicInfo, attrStart, attlen);// 调用DictUncompress函数进行解压缩 + } + + case CMPR_PREFIX: { + PrefixCmprMeta* prefixInfo = (PrefixCmprMeta*)metaInfo; + return PrefixUncompress(attlen, prefixInfo, attrStart, attrCmprsValSize);// 调用PrefixUncompress函数进行解压缩 + } + + case CMPR_NUMSTR: { + return NumstrUncompress(PointerGetDatum(attrStart), attlen, *attrCmprsValSize);// 调用NumstrUncompress函数进行解压缩 + } + + case CMPR_NONE: { + Assert(false);// 调用NumstrUncompress函数进行解压缩 + return (Datum)0; + } + default: + break; + } + + Assert(false);// 触发断言错误，表示不支持的压缩模式 + return (Datum)0; +} + +/* Get dictionary item ID */ +//据给定的列号col和属性值val进行字典压缩，并返回对应的字典项ID +bool PageCompress::DictCompress(int col, Datum val, unsigned char& dicItemId) +{ + bool found = false; + int attlen = m_rel->rd_att->attrs[col]->attlen; + DictCmprMeta* dicInfo = (DictCmprMeta*)m_cmprMeta[col]; + DictItemData key; + // 根据属性长度和值填充字典项键 + if (attlen > 0) + FillHashKey<1>(key, attlen, val); + else if (-1 == attlen) + FillHashKey<-1>(key, attlen, val); + else if (-2 == attlen) + FillHashKey<-2>(key, attlen, val); + // 从字典压缩元信息中查找字典项ID + dicItemId = dicInfo->GetDictItemId(&key, found); + // 如果属性长度大于0，释放字典项键内存 + if (attlen > 0) + pfree(key.itemData); + + return found;// 返回是否找到字典项的标志位 +} + +/* compress the col's attribute of one tuple, ouput data is written into formTuple */ +void PageCompress::CompressOneAttr(Datum val, bool null, FormCmprTupleData* formTuple, int col) +{ + formTuple->compressed[col] = false;// 初始化该列未压缩 + + if (null) {// 如果该列的值为null + formTuple->isnulls[col] = true;// 设isnulls为true + formTuple->values[col] = (Datum)0;// 该列不存储实际数据，设为0 + return; + } + + Form_pg_attribute* attrs = m_rel->rd_att->attrs; + formTuple->values[col] = val;// 保存原始的值 + formTuple->isnulls[col] = false;// 设isnulls为false + + switch (m_cmprMode[col]) {// 根据压缩模式进行不同的处理 + case CMPR_DELTA: {// Delta压缩模式 + DeltaCmprMeta* metaInfo = (DeltaCmprMeta*)m_cmprMeta[col];// 获取压缩元数据 + const NumberTypeOpt* opt = m_numTypeOpt[col];// 获取数据类型选项 + unsigned char* deltaBuf = (unsigned char*)palloc((Size)metaInfo->bytes);// 分配缓存 + if (DeltaCompress(metaInfo, opt, val, deltaBuf)) {// 进行Delta压缩 + formTuple->compressed[col] = true;// 标记该列已压缩 + if (metaInfo->bytes == 0) {// 如果数据全相同，则只存储一个值 + /* + * this means that all values of this column within page are equal to. + * we will store only the one into compression header, and null in every + * column for tuples. + * Notice that: keep this flag formTuple->isnulls[col] is false. + */ + formTuple->valsize[col] = 0;// 不存储实际数据，设为0 + formTuple->values[col] = (Datum)0; + pfree(deltaBuf);// 释放缓存 + } else { + formTuple->valsize[col] = metaInfo->bytes; + formTuple->values[col] = PointerGetDatum(deltaBuf);// 将压缩后的数据指针存入values中 + } + } else + pfree(deltaBuf);// 释放缓存 + break; + } + + case CMPR_DICT: {// Dict压缩模式 + if (DictCompress(col, val, m_dictItemId[col])) {// 进行Dict压缩 + formTuple->compressed[col] = true;// 标记该列已压缩 + DictCmprMeta* dicInfo = (DictCmprMeta*)m_cmprMeta[col]; + if (dicInfo->dictItemNum == 1) {// 如果只有一个字典项，则不存储实际数据 + formTuple->valsize[col] = 0; + formTuple->values[col] = (Datum)0; + } else { + formTuple->valsize[col] = DICT_ITEMID_SIZE;// 否则存储字典项编号 + formTuple->values[col] = PointerGetDatum(m_dictItemId + col); + } + } + break; + } + + case CMPR_PREFIX: {// Prefix压缩模式 + int attrLen = attrs[col]->attlen; + Assert(attrLen == -1 || attrLen == -2); + PrefixCmprMeta* prefixInfo = (PrefixCmprMeta*)m_cmprMeta[col]; + Datum cmprsVal; + int cmprsValSize; + bool successful = false; + + /* + * we think that prefix method doesn't work well on those data, + * which are compressed by lz method internally. so skipp it. + */ + // 对于使用lz方法进行压缩的数据，不使用prefix方法 + if (VARATT_IS_4B_C(DatumGetPointer(val))) { + break; + } + + if (-1 == attrLen) + successful = PrefixCompress<-1>(prefixInfo, val, cmprsVal, cmprsValSize);// 进行Prefix压缩 + else + successful = PrefixCompress<-2>(prefixInfo, val, cmprsVal, cmprsValSize); + + if (successful) {// 压缩成功 + formTuple->compressed[col] = true;// 标记该列已压缩 + formTuple->valsize[col] = cmprsValSize; + formTuple->values[col] = cmprsVal;// 将压缩后的数据存入values中 + } + break; + } + + case CMPR_NUMSTR: {// Numstr压缩模式 + int attrLen = attrs[col]->attlen; + Assert(-1 == attrLen || -2 == attrLen); + + int outSize = 0; + Datum outValue; + + if (-1 == attrLen) { + outValue = NumstrCompress<-1>(val, outSize);// 进行Numstr压缩 + Assert(outSize == 0 || (Size)outSize == VARSIZE_ANY(DatumGetPointer(outValue))); + } else { + outValue = NumstrCompress<-2>(val, outSize); + Assert(outSize == 0 || (Size)outSize == strlen(DatumGetPointer(outValue)) + 1); + } + if (outSize > 0) {// 压缩成功 + formTuple->compressed[col] = true;// 标记该列已压缩 + formTuple->valsize[col] = outSize; + formTuple->values[col] = outValue;// 将压缩后的数据存入values中 + } + break; + } + + case CMPR_NONE: {// 不压缩模式 + break; + } + default: + break; + } +} +//页面压缩算法，将给定的数据按照特定规则进行压缩，并尽量填满一个页面的空间 +bool PageCompress::CompressOnePage(void) +{ + //初始化相关变量和内存环境 + const int blksize = RelationGetTargetPageUsage(m_rel, HEAP_DEFAULT_FILLFACTOR); + int attrNum = RelationGetNumberOfAttributes(m_rel); + int size, row; + int prevToastNum = m_toastTupsNum; + int thisToasts; + + MemoryContext oldMemCnxt; + ColArray* colArray = NULL; + TupleDesc desc = RelationGetDescr(m_rel); + FormCmprTupleData formTupleData; + Datum* values = NULL; + bool* isnulls = NULL; + bool fillOnePage = false; + bool hascmpr = false; + bool remainBeforeCmpr = true; + errno_t rc = EOK; + + /* reset local mem-context after handling one page tuples. */ + // 重置每页的相关内存上下文 + ResetPerPage(); + // 切换到m_pageMemCnxt上下文 + oldMemCnxt = MemoryContextSwitchTo(m_pageMemCnxt); + + /* at the end of the current batch, only toast tuples need to be written into pages. */ + // 如果当前是最后一个批次，则跳转到处理toast tuples的分支 + if ((m_current == m_inTupsNum) || m_nextBatch) { + Assert(m_mappedTupsNum == 0); + ResetCmprMeta(); + /* jump to branch handle_toasts only if m_current is equal to m_inTupsNum */ + goto handle_toasts; + } + + /* + * Step 1: how many rows are compressed and form one page + * We need evaluate proper row numbers. + */ + // Step 1: 统计可以压缩的行数，并计算大小是否超过块大小 + size = SizeOfHeapPageHeaderData; + for (row = m_current; row < m_inTupsNum; ++row) { + /* collect all the toast tuples, and write once at the end of each batch */ + // 收集所有的toast tuples，并在每个批次结束时写入页面 + if (isToastTuple(m_inTups[row])) + m_toastTups[m_toastTupsNum++] = m_inTups[row]; + else { + if (size + (int)getTupleSpace(m_inTups[row]) > blksize) + break; + // 收集所有未压缩的tuples，并进入压缩流程 + /* collect all the untoast tuples, and enter comperss flow */ + m_mappedTups[m_mappedTupsNum++] = m_inTups[row]; + size += (int)getTupleSpace(m_inTups[row]); + } + } + + /* + * raw tuples cann't fill one page, then we must check m_last flag to + * write these ones. and toast tuples would be taken care. + */ + // 如果剩余的行数无法填满一个页面，则将剩余的行写入页面，处理toast tuples + if (row == m_inTupsNum) { + ResetCmprMeta(); + remainBeforeCmpr = true; + goto handle_remains; + } + // Step 2: 将tuples转换为ColArray + /* Step 2: deform tuples into ColArray */ + colArray = DeformTups(m_mappedTups, m_mappedTupsNum); + + /* + * Step 3: Sampling data of each column and choose proper compression method + * We need generate compression meta information for step 3. + */ + // Step 3: 对每一列进行采样并选择合适的压缩方法 + ChooseCmprMethod(colArray); + SetCmprHeaderData(); + if (m_cmprHeaderSize == 0) { + // 当前批次中的所有tuples都不适合压缩 + /* all the tuples within the current batch are not suitable for compress. */ + OutputRawTupsBeforeCmpr(m_toastTupsNum - prevToastNum); + (void)MemoryContextSwitchTo(oldMemCnxt); + return true; + } + +#ifdef USE_ASSERT_CHECKING + CheckCmprHeaderData(); +#endif + // Step 4: 压缩当前批次的行 + /* Step 4: Compress the current batch rows. */ + formTupleData.compressed = (bool*)palloc((Size)(sizeof(bool) * attrNum)); + formTupleData.valsize = (int*)palloc((Size)(sizeof(int) * attrNum)); + formTupleData.values = (Datum*)palloc((Size)(sizeof(Datum) * attrNum)); + formTupleData.isnulls = (bool*)palloc((Size)(sizeof(bool) * attrNum)); + size = (int)MAXALIGN(SizeOfHeapPageHeaderData) + (int)MAXALIGN((uint32)m_cmprHeaderSize); + + for (row = 0; row < m_mappedTupsNum; ++row) { + hascmpr = false; + // 循环遍历列 + for (int col = 0; col < attrNum; ++col) { + // 对当前行的每列数据进行压缩 + CompressOneAttr(colArray[col].val[row], colArray[col].nulls[row], &formTupleData, col); + // 判断该列是否被压缩 + hascmpr = (hascmpr || formTupleData.compressed[col]); +#ifdef USE_ASSERT_CHECKING + CheckCmprAttr(colArray[col].val[row], colArray[col].nulls[row], col, &formTupleData); +#endif + } + + /* + * form compressed tuple and then dispatch until part of tuples fill one page, + * or all compressed tuples fill part of one page. + */ + // 根据是否发生压缩进行分发 + if (hascmpr) { + fillOnePage = Dispatch(m_mappedTups[row], (HeapTuple)tableam_tops_form_cmprs_tuple(desc, &formTupleData, HEAP_TUPLE), size, blksize); + } else { + fillOnePage = Dispatch(m_mappedTups[row], size, blksize); + } + + if (fillOnePage) + break; + } + + /* + * Step 5: Judge whether compressed data is enough to fill one page + * If not enough, we compress the next tuples. But it is different from + * Step 4, if one column is compressed by delta value encoding, we can't + * change delta value. + */ + // Step 5: 判断是否已经填满一个页面 + if (fillOnePage) { + Assert(row < m_mappedTupsNum); + ResetCmprMeta(); + OutputRawTupsAfterCmpr(m_toastTupsNum - prevToastNum); + (void)MemoryContextSwitchTo(oldMemCnxt); + return true; + } + Assert(row == m_mappedTupsNum); + + values = (Datum*)palloc((Size)(sizeof(Datum) * attrNum)); + isnulls = (bool*)palloc((Size)(sizeof(bool) * attrNum)); + // 如果页面未填满，则继续读取并压缩下一批次的行，直到填满一个页面或达到输入的最后一个批次 + /* continue to read and compress following tuples, until one page is filled */ + for (row = m_current + m_mappedTupsNum + (m_toastTupsNum - prevToastNum); row < m_inTupsNum; ++row) { + HeapTuple raw = m_inTups[row]; + if (isToastTuple(raw)) { + m_toastTups[m_toastTupsNum++] = raw; + continue; + } + // 将当前行数据解析为各列值和是否为null的标记 + heap_deform_tuple(raw, desc, values, isnulls); + hascmpr = false; + // 对当前行的每列数据进行压缩 + for (int col = 0; col < attrNum; ++col) { + CompressOneAttr(values[col], isnulls[col], &formTupleData, col); + hascmpr = (hascmpr || formTupleData.compressed[col]); +#ifdef USE_ASSERT_CHECKING + CheckCmprAttr(values[col], isnulls[col], col, &formTupleData); +#endif + } + // 根据是否发生压缩进行分发 + if (hascmpr) { + fillOnePage = Dispatch(raw, (HeapTuple)tableam_tops_form_cmprs_tuple(desc, &formTupleData, HEAP_TUPLE), size, blksize); + } else { + fillOnePage = Dispatch(raw, size, blksize); + } + + if (fillOnePage) + break; + } + + if (fillOnePage || m_last) { + Assert(fillOnePage || (row == m_inTupsNum)); + OutputCmprTups(m_toastTupsNum - prevToastNum); + (void)MemoryContextSwitchTo(oldMemCnxt); + return true; + } + + /* + * after compress, remain tuples cann't fill one page. + * we think that cotinue to read and compress more tuples from next batch + * maybe is better. otherwise this page will holds one tuple at least. + */ + // Step 6: 处理剩余的行 + Assert(row == m_inTupsNum && m_last == false); + m_cmprHeaderSize = 0; + remainBeforeCmpr = false; + +handle_remains: + + thisToasts = m_toastTupsNum - prevToastNum;// 计算当前批次中剩余的toast tuples数量 + if (m_mappedTupsNum == 0) { + /* case 1: only toast tuples remain. */ + Assert(thisToasts == (m_inTupsNum - m_current)); + m_current = m_inTupsNum; + // 如果没有原始行存在，只有toast tuples剩余，直接将m_current更新为m_inTupsNum表示所有行都已处理完毕 + } else if (m_last) { + /* + * case 2: for the last batch, + * first write all the raw tuples left into pages; + * then handle the remain toast tuples; + */ + Assert(remainBeforeCmpr); + OutputRawTupsBeforeCmpr(thisToasts); + Assert(m_current == m_inTupsNum); + (void)MemoryContextSwitchTo(oldMemCnxt); + return true; + // 如果是最后一个批次，先将剩余的原始行写入页面，然后处理剩余的toast tuples + } else { + /* + * case 3: if not the last batch + * first write all toasts of the current batch into pages; + * then move the remain into next batch; + */ + m_nextBatch = true; + // 标记下一个批次准备就绪 + /* + * case 3.1: toats and raw tuples must be collected sperately; + */ + if (thisToasts > 0) { + /* + * step 1: collect these toasts in the front, make sure they are free + * after they are written into pages by heap_multi_insert(). + */ + rc = memcpy_s(m_inTups + m_current, + (size_t)(sizeof(HeapTuple) * thisToasts), + m_toastTups + prevToastNum, + (size_t)(sizeof(HeapTuple) * thisToasts)); + securec_check(rc, "", ""); + m_current += thisToasts; + // 将剩余的toast tuples移到当前批次的最前面 + } + + /* step 2: collect raw tuples in the end, make sure they are moved ahead next buffer. */ + if (remainBeforeCmpr) { + /* case 3.2.1: raw tuples are all in m_mappedTups[m_mappedTupsNum] */ + rc = memcpy_s(m_inTups + m_current, + (size_t)(sizeof(HeapTuple) * m_mappedTupsNum), + m_mappedTups, + (size_t)(sizeof(HeapTuple) * m_mappedTupsNum)); + securec_check(rc, "", ""); + // 将剩余的原始行移到当前批次的最后 + } else { + /* + * case 3.2.2: some are int m_mappedTups[m_cmprTupsNum], and the others + * are in m_cmprTups[m_uncmprTupIdx, m_inTupsNum) + */ + int offset = m_current; + if (m_cmprTupsNum > 0) { + rc = memcpy_s(m_inTups + offset, + (size_t)(sizeof(HeapTuple) * m_cmprTupsNum), + m_mappedTups, + (size_t)(sizeof(HeapTuple) * m_cmprTupsNum)); + securec_check(rc, "", ""); + offset += m_cmprTupsNum; + m_cmprTupsNum = 0; + } + // 将一部分原始行从m_mappedTups复制到当前批次的最后，并更新偏移量 + int uncmpr = m_inTupsNum - m_uncmprTupIdx; + if (uncmpr > 0) { + rc = memcpy_s(m_inTups + offset, (size_t)(sizeof(HeapTuple) * uncmpr), m_cmprTups + m_uncmprTupIdx, + (size_t)(sizeof(HeapTuple) * uncmpr)); + securec_check(rc, "", ""); + } + // 将剩余的原始行从m_cmprTups复制到当前批次的最后 + } + } + +handle_toasts: + /* + * 如果还有未处理的toast tuples（大对象行），调用OutputToasts()函数来处理它们。 + * 处理完成后，切换回原始内存上下文并返回true。 + */ + + if (m_toastTupsNum > 0) { + OutputToasts(); + (void)MemoryContextSwitchTo(oldMemCnxt); + return true; + } + + /* all tuples of this batch have been handled. */ + pfree(m_toastTups); + pfree(m_mappedTups); + pfree(m_cmprTups); + m_toastTups = NULL; + m_mappedTups = NULL; + m_cmprTups = NULL; + + (void)MemoryContextSwitchTo(oldMemCnxt); + return false; +} +//输出上一批次剩余未压缩的行数据（raw tuples） +void PageCompress::OutputRawTupsBeforeCmpr(int nToasts) +{ + /* + * write all the tuples left out in the last batch. + * make m_current equal to m_inTupsNum so that toast tuples + * will be written into pages at the last time. + */ + /* + * 输出上一批次剩余未压缩的行数据。 + * 将 m_mappedTups 指针赋值给 m_outputTups，以进行输出。 + * 将 m_mappedTupsNum 赋值给 m_outputTupsNum，表示待输出的行数。 + */ + m_outputTups = m_mappedTups; + m_outputTupsNum = m_mappedTupsNum; + +#ifdef USE_ASSERT_CHECKING + // 在启用断言检查的条件编译下，对输出行数据进行检查。 + CheckOutputTups(NULL, 0, m_mappedTups, m_mappedTupsNum, nToasts); +#endif +#ifdef TRACE_COMPRESS + // 在启用压缩追踪的条件编译下，追踪压缩页的状态。 + ++nLoops; + TraceCmprPage(0, NULL, NULL, 0, m_mappedTups, m_mappedTupsNum, nToasts); +#endif + // 更新当前处理的行数，将已输出的行数和大对象行数相加。 + m_current += (m_outputTupsNum + nToasts); + // 将压缩头部大小初始化为0。 + m_cmprHeaderSize = 0; + // 将压缩行数初始化为0。 + m_cmprTupsNum = 0; +} +//将压缩后的元组和未压缩的原始元组收集到一个数组中，并更新相关的计数器和指针 +void PageCompress::OutputRawTupsAfterCmpr(int nToasts) +{ + /* + * after compress, raw tuples exist in both m_mappedTups[] and m_cmprTups[]. + * and the second range is [m_uncmprTupIdx, m_inTupsNum). + * we have to collect all raw tuples into m_mappedTups[]. + */ + /* + * 压缩完成后，原始元组存在于 m_mappedTups[] 和 m_cmprTups[]。 + * 第二个范围为 [m_uncmprTupIdx, m_inTupsNum)。 + * 我们需要将所有的原始元组收集到 m_mappedTups[] 中。 + */ + m_outputTups = m_mappedTups; + int uncmprNum = m_inTupsNum - m_uncmprTupIdx; + if (uncmprNum > 0) { + errno_t rc = memcpy_s(m_mappedTups + m_cmprTupsNum, + (size_t)(sizeof(HeapTuple) * uncmprNum), + m_cmprTups + m_uncmprTupIdx, + (size_t)(sizeof(HeapTuple) * uncmprNum)); + securec_check(rc, "", ""); + } + Assert((m_cmprTupsNum + uncmprNum) <= m_mappedTupsNum); + // 将 m_mappedTups 设置为输出行数据指针，表示输出原始数据。 + m_outputTupsNum = m_mappedTupsNum; + +#ifdef USE_ASSERT_CHECKING + // 在启用断言检查的条件编译下，对输出行数据进行检查。 + CheckOutputTups( + m_mappedTups, m_cmprTupsNum, (m_mappedTups + m_cmprTupsNum), (m_mappedTupsNum - m_cmprTupsNum), nToasts); +#endif +#ifdef TRACE_COMPRESS + // 在启用压缩追踪的条件编译下，追踪压缩页的状态。 + ++nLoops; + TraceCmprPage(1, NULL, NULL, 0, m_outputTups, m_outputTupsNum, nToasts); +#endif + // 更新当前处理的行数，将已输出的行数和大对象行数相加。 + m_current += (m_outputTupsNum + nToasts); + // 将压缩头部大小初始化为0。 + m_cmprHeaderSize = 0; + // 将压缩行数初始化为0。 + m_cmprTupsNum = 0; +} +//将压缩后的元组和未压缩的原始元组输出到一个数组中，并更新相关的计数器和指针。它是数据压缩过程中的一个关键步骤，确保所有的元组都被正确输出 +void PageCompress::OutputCmprTups(int nToasts) +{ + /* + * after compress, output tuples exist in only m_cmprTups[], both compressed and some raws. + * the previous range is [0, m_cmprTupsNum) and the other is [m_uncmprTupIdx, m_inTupsNum). + * we have to append the second part to the first part. + */ + // 检查压缩元组数量小于等于未压缩元组索引，确保前半部分的范围是[0, m_cmprTupsNum)，后半部分是[m_uncmprTupIdx,m_inTupsNum)。 + Assert(m_cmprTupsNum <= m_uncmprTupIdx); + // 将输出元组指针设置为压缩后的元组数组，表明输出的是压缩后的数据。 + m_outputTups = m_cmprTups; + // 计算未压缩元组数量 + int uncmprNum = m_inTupsNum - m_uncmprTupIdx; + // 根据不同的情况确定输出元组数量 + if (0 == uncmprNum) + m_outputTupsNum = m_cmprTupsNum; + else if (m_cmprTupsNum == m_uncmprTupIdx) + m_outputTupsNum = m_inTupsNum; + else { + // 将未压缩元组追加到压缩元组数组的末尾 + errno_t rc = memmove_s(m_cmprTups + m_cmprTupsNum, + (size_t)(sizeof(HeapTuple) * uncmprNum), + m_cmprTups + m_uncmprTupIdx, + (size_t)(sizeof(HeapTuple) * uncmprNum)); + securec_check(rc, "", ""); + m_outputTupsNum = m_cmprTupsNum + uncmprNum; + } + +#ifdef USE_ASSERT_CHECKING + // 在启用断言检查的条件编译下，确保输出元组数量大于等于已映射的元组数量 + Assert(m_outputTupsNum >= m_mappedTupsNum); + CheckOutputTups(m_mappedTups, m_cmprTupsNum, (m_cmprTups + m_cmprTupsNum), uncmprNum, nToasts); +#endif +#ifdef TRACE_COMPRESS + // 在启用压缩追踪的条件编译下，追踪压缩页的状态 + ++nLoops; + TraceCmprPage(2, m_cmprTups, m_mappedTups, m_cmprTupsNum, (m_cmprTups + m_cmprTupsNum), uncmprNum, nToasts); +#endif + // 更新当前处理的行数，包括已输出的行数和大对象行数 + m_current += (m_outputTupsNum + nToasts); +} +//将大对象元组输出到压缩页中，并进行相关状态和计数的更新 +void PageCompress::OutputToasts(void) +{ + /* + * toast tuples within each batch are collected togather, and written once + * into pages before next SetBatchTuples() is called. + */ + m_outputTups = m_toastTups;// 设置输出元组指针为大对象元组数组 + m_outputTupsNum = m_toastTupsNum;// 设置输出元组数量为大对象元组数量 +#ifdef TRACE_COMPRESS + ++nLoops; + ereport(INFO, (errmsg("Loop %d, Tag: Toast Tuples, num %d", nLoops, m_toastTupsNum))); +#endif + m_toastTupsNum = 0;// 清空大对象元组计数器 + Assert((m_last == false) || (m_current == m_inTupsNum));// 确保当前处理的行数已经等于所有输入行数 + Assert(m_cmprHeaderSize == 0);// 确保压缩头大小为0 + Assert(m_cmprTupsNum == 0);// 确保压缩元组数量为0 +} +//设置输入批处理元组的相关信息，并分配内存用于处理大对象元组、映射元组和压缩元组 +void PageCompress::SetBatchTuples(HeapTuple* tups, int ntups, bool isLast) +{ + // 设置输入批处理元组的相关信息 + m_inTups = tups;// 将输入批处理元组指针设置为输入的元组数组 + m_inTupsNum = ntups;// 设置输入批处理元组的数量 + m_current = 0;// 将当前处理的元组索引设置为0 + m_last = isLast;// 设置是否为最后一批元组的标志位 + + /* + * we know the number of input tuples, so alloc the arrays for handling toast + * tuples. after this batch has been handled, free these memory. + */ + MemoryContext oldMemCnxt = MemoryContextSwitchTo(m_parentMemCxt);// 在父内存上下文中切换内存上下文，并将旧的内存上下文保存起来 + // 分配用于处理大对象元组、映射元组和压缩元组的数组内存，数组大小为输入批处理元组数量的大小 + m_toastTups = (HeapTuple*)palloc((Size)(sizeof(HeapTuple) * ntups));// 将大对象元组计数器重置为初始值 + m_mappedTups = (HeapTuple*)palloc((Size)(sizeof(HeapTuple) * ntups)); + m_cmprTups = (HeapTuple*)palloc((Size)(sizeof(HeapTuple) * ntups)); + m_toastTupsNum = 0; + m_nextBatch = false;// 将下一批处理标志位重置为初始值 + + (void)MemoryContextSwitchTo(oldMemCnxt);// 切换回旧的内存上下文 +} +//重置页压缩对象的状态，清空已分配的资源，将各种计数器和标志位恢复到初始状态，为进行下一轮压缩做好准备 +void PageCompress::ResetPerPage(void) +{ + int attNum = m_rel->rd_att->natts; + + /* Free everything by destroying dictionary */ + // 遍历属性，如果是字典压缩模式并且存在压缩元数据，则销毁字典压缩元数据 + for (int att = 0; att < attNum; ++att) { + if (m_cmprMode[att] == CMPR_DICT && m_cmprMeta[att]) { + ((DictCmprMeta*)m_cmprMeta[att])->Destroy(); + m_cmprMeta[att] = NULL; + } + } + // 重置页压缩对象的内存上下文 + MemoryContextReset(m_pageMemCnxt); + /* m_cmprHeaderData would be fixed during the entire compress, keep it. */ + // 重置其他状态变量 + m_cmprHeaderSize = 0; + m_outputTups = NULL; + m_outputTupsNum = 0; + + m_mappedTupsNum = 0; + m_cmprTupsNum = 0; + m_uncmprTupIdx = m_inTupsNum; +} + +/* + * reset all compress meta info when tuples are not compressed and output. + * 1. before OutputRawTupsAfterCmpr() is called; + * 2. before OutputRawTupsBeforeCmpr() is called, optional; + * 3. before branch handle_remains is jumpped to; + * 4. before branch handle_toasts is jumpped to; + */ +//在元组没有被压缩且输出之前，重置所有压缩元数据的信息 +void PageCompress::ResetCmprMeta(void) +{ + int attNum = m_rel->rd_att->natts; + // 遍历所有属性，重置压缩模式和压缩元数据指针 + for (int i = 0; i < attNum; ++i) { + m_cmprMode[i] = CMPR_NONE;// 将压缩模式设置为无压缩 + m_cmprMeta[i] = NULL;// 将压缩元数据指针设置为空 + } + m_cmprHeaderSize = 0;// 将压缩头部大小设置为0 +} + +/* + * it's different from BackWrite(), which transforms compressed tuples' info + * backward into raw tuples. but this method will transforms raw tuples' info + * forward into compressed tuples. + * NB: now this method applies to VACUUM FULL && CLUSTER + */ +/* + * 该函数是 PageCompress 的一个成员函数，用于将原始元组信息转化为 + * 压缩后的元组信息，常用于 VACUUM FULL 和 CLUSTER 操作中。与 BackWrite() + * 函数不同，BackWrite() 函数是将已压缩的元组信息转换回原始元组信息。 + */ +//将页面中需要进行压缩的元组从原始的元组信息转换为压缩后的元组信息，常用于 VACUUM FULL 和 CLUSTER 操作中 +void PageCompress::ForwardWrite(void) +{ + // 判断是否存在需要压缩的元组，如果没有，则直接返回 + if (m_cmprTupsNum == 0) + return; + errno_t rc = EOK; + // 断言：压缩头部大小必须大于0 + Assert(m_cmprHeaderSize > 0); + // 遍历所有需要压缩的元组 + for (int i = 0; i < m_cmprTupsNum; i++) { + // 获取目标元组和源元组的 HeapTupleHeader 对象 + HeapTupleHeader dest = m_cmprTups[i]->t_data; + HeapTupleHeader src = m_mappedTups[i]->t_data; + // 断言：目标元组已经被标记为压缩状态 + Assert(HEAP_TUPLE_IS_COMPRESSED(dest)); + // 断言：源元组和目标元组的 t_hoff 偏移必须相同 + Assert(dest->t_hoff == src->t_hoff); + // 断言：元组的字段数必须一致 + Assert(HeapTupleHeaderGetNatts(dest, m_rel->rd_att) == HeapTupleHeaderGetNatts(src, m_rel->rd_att)); + // 断言：元组的标志位必须一致 + Assert((0x000F & dest->t_infomask) == (0x000F & src->t_infomask)); + // 断言：如果存在 NULL 值，位图内容也必须一致 + Assert( + (0 == (dest->t_infomask & HEAP_HASNULL)) || + 0 == memcmp( + (char*)dest->t_bits, (char*)src->t_bits, BITMAPLEN(HeapTupleHeaderGetNatts(src, m_rel->rd_att)))); + // 使用 memcpy_s() 函数将源元组的数据拷贝到目标元组中 + rc = memcpy_s((char*)dest, src->t_hoff, (char*)src, src->t_hoff); + securec_check(rc, "", ""); + // 标记目标元组为已压缩状态 + HEAP_TUPLE_SET_COMPRESSED(dest); + } +} + +/* + * after heap_multi_insert() is done, we should write infomask back + * to the raw tuples. these information will be used by trigger and + * index build. + * this api exists just to reduce public methods as much as possible. + */ +//将压缩后的元组信息回写至原始的元组中 +void PageCompress::BackWrite(void) +{ + /* compressed tuples exist and are valid only if m_cmprHeaderSize > 0 */ + if (m_cmprTupsNum == 0) + return; + + errno_t rc = EOK; + + Assert(m_cmprHeaderSize > 0); + for (int i = 0; i < m_cmprTupsNum; i++) { + HeapTupleHeader dest = m_mappedTups[i]->t_data; + HeapTupleHeader src = m_cmprTups[i]->t_data; + // 校验目标元组的t_hoff与源元组的t_hoff相等 + Assert(dest->t_hoff == src->t_hoff); + // 校验目标元组的t_infomask2低11位与源元组的t_infomask2低11位相等 + Assert((0x07FF & dest->t_infomask2) == (0x07FF & src->t_infomask2)); + // 校验目标元组的t_infomask低4位与源元组的t_infomask低4位相等 + Assert((0x000F & dest->t_infomask) == (0x000F & src->t_infomask)); + // 校验目标元组的是否包含NULL位图与源元组的是否包含NULL位图相等 + Assert(HeapTupleHasNulls(m_mappedTups[i]) == HeapTupleHasNulls(m_cmprTups[i])); + // 将源元组的数据拷贝回目标元组中 + rc = memcpy_s((char*)dest, src->t_hoff, (char*)src, src->t_hoff); + securec_check(rc, "", ""); + // 清除目标元组的已压缩状态 + HEAP_TUPLE_CLEAR_COMPRESSED(dest); + // 更新目标元组的t_self、t_tableOid和t_xc_node_id等字段值，使其与压缩元组保持一致 + m_mappedTups[i]->t_self = m_cmprTups[i]->t_self; + m_mappedTups[i]->t_tableOid = m_cmprTups[i]->t_tableOid; +#ifdef PGXC + m_mappedTups[i]->t_xc_node_id = m_cmprTups[i]->t_xc_node_id; +#endif + } +} +//将压缩后的元组添加到数据页面中，并更新相关的统计信息 +bool PageCompress::Dispatch(HeapTuple raw, HeapTuple cmpr, int& nowSize, const int& blksize) +{ + // 断言该元组已经被压缩 + Assert(HEAP_TUPLE_IS_COMPRESSED(cmpr->t_data)); + // 获取压缩元组所需的空间大小 + int needSize = (int)getTupleSpace(cmpr); + // 判断页面空间是否足够存放压缩元组 + if (nowSize + needSize > blksize) + return true; + // 断言已压缩元组数量未达到非压缩元组索引 + Assert(m_cmprTupsNum < m_uncmprTupIdx); + // 将原始元组作为基础复制到压缩元组中 + HeapTupleCopyBase(cmpr, raw); + // 将压缩元组和原始元组分别存储到数组中，更新压缩元组数量 + m_cmprTups[m_cmprTupsNum] = cmpr; + m_mappedTups[m_cmprTupsNum] = raw; + ++m_cmprTupsNum; +#ifdef USE_ASSERT_CHECKING + CheckCmprTuple(raw, cmpr);// 调用CheckCmprTuple()函数进行校验 +#endif + nowSize += needSize;// 更新页面已使用空间大小 + return false;// 返回false，表示压缩元组成功分发到页面中 +} +//将非压缩的元组添加到数据页面中，并更新相关的统计信息 +bool PageCompress::Dispatch(HeapTuple raw, int& nowSize, const int& blksize) +{ + // 断言元组未被压缩 + Assert(!HEAP_TUPLE_IS_COMPRESSED(raw->t_data)); + // 获取元组所需的空间大小 + int needSize = (int)getTupleSpace(raw); + // 判断元组添加后页面是否溢出 + if (nowSize + needSize > blksize) + return true; + // 断言非压缩元组索引大于已经压缩的元组数量 + Assert(m_uncmprTupIdx > m_cmprTupsNum); + // 存储元组到压缩元组数组中，并更新已有元素数量 + --m_uncmprTupIdx; + m_cmprTups[m_uncmprTupIdx] = raw; + // 更新当前页面已使用空间 + nowSize += needSize; + // 返回false表示元组成功添加到数据页面中 + return false; +} + +#ifdef USE_ASSERT_CHECKING +// 检查压缩数据的函数 +void PageCompress::CheckCmprDatum(Datum arg1, Datum arg2, Form_pg_attribute attr) +{ + int attlen = attr->attlen; + errno_t rc = EOK; + + if (attr->attbyval) { + // 如果属性按值传递 + Assert(attlen > 0 && attlen <= SIZEOF_DATUM);// 确保属性长度在有效范围内 + Assert(arg1 == arg2);// 确保两个参数相等 + return; + } + + if (attlen > 0) { + // 如果属性是定长的 + Assert(0 == memcmp(DatumGetPointer(arg1), DatumGetPointer(arg2), attlen));// 比较两个参数的值是否相等 + return; + } + + /* + * we pack compressed data as many as possible, even its storage type is PLAIN; + * so raw data must be first packed before comparing; + */ + /* + * 对于存储类型为PLIAN的压缩数据，我们尽可能多地进行打包； + * 所以在比较之前，先将原始数据打包； + */ + char packedBuf1[VARATT_SHORT_MAX] = {0}; + char packedBuf2[VARATT_SHORT_MAX] = {0}; + Pointer pOldValue1 = DatumGetPointer(arg1); + Pointer pOldValue2 = DatumGetPointer(arg2); + + if (-1 == attr->attlen) { + if (!VARATT_IS_EXTERNAL(pOldValue1) && !VARATT_IS_SHORT(pOldValue1) && + VARATT_CONVERTED_SHORT_SIZE(pOldValue1) <= VARATT_SHORT_MAX) { + Size len = VARATT_CONVERTED_SHORT_SIZE(pOldValue1); + Assert(len <= VARATT_SHORT_MAX);// 确保打包后的长度在有效范围内 + SET_VARSIZE_SHORT(&packedBuf1[0], len); + rc = memcpy_s(&packedBuf1[1], VARATT_SHORT_MAX - VARHDRSZ_SHORT, VARDATA(pOldValue1), len - 1); + securec_check(rc, "", ""); + arg1 = PointerGetDatum(packedBuf1); + } + + if (!VARATT_IS_EXTERNAL(pOldValue2) && !VARATT_IS_SHORT(pOldValue2) && + VARATT_CONVERTED_SHORT_SIZE(pOldValue2) <= VARATT_SHORT_MAX) { + Size len = VARATT_CONVERTED_SHORT_SIZE(pOldValue2); + Assert(len <= VARATT_SHORT_MAX);// 确保打包后的长度在有效范围内 + SET_VARSIZE_SHORT(&packedBuf2[0], len); + rc = memcpy_s(&packedBuf2[1], VARATT_SHORT_MAX - VARHDRSZ_SHORT, VARDATA(pOldValue2), len - 1); + securec_check(rc, "", ""); + arg2 = PointerGetDatum(packedBuf2); + } + } + + char* ptr1 = DatumGetPointer(arg1); + char* ptr2 = DatumGetPointer(arg2); + int size1 = 0; + int size2 = 0; + + if (-1 == attlen) { + // 对于变长的PLIAN格式，获取数据的大小 + size1 = VARSIZE_ANY(ptr1); + size2 = VARSIZE_ANY(ptr2); + } else if (-2 == attlen) { + // 对于变长的C格式，获取字符串的长度 + size1 = strlen(ptr1) + 1; + size2 = strlen(ptr2) + 1; + } + + Assert(size1 == size2);// 确保两个参数的大小相等 + Assert(0 == memcmp(ptr2, ptr1, size1));// 比较两个参数的值是否相等 + return; +} +//检查压缩属性的正确性。通过比较原始值和压缩元组中对应属性的值，以及对压缩后的值进行解压缩和比较，来确保压缩和解压缩的过程没有出错 +void PageCompress::CheckCmprAttr(Datum rawVal, bool rawNull, int col, FormCmprTupleData* formTuple) +{ + // 检查原始值是否为NULL，与压缩元组中相应属性的NUll标记是否一致 + Assert(rawNull == formTuple->isnulls[col]); + if (rawNull) { + // 如果原始值为NULL，则检查压缩元组中相应属性的值是否为0（表示NULL） + Assert((Datum)0 == formTuple->values[col]); + return; + } + + if (formTuple->compressed[col] == false) { + // 如果属性未被压缩，则检查原始值与压缩元组中相应属性的值是否相等 + Assert(rawVal == formTuple->values[col]); + return; + } + // 检查压缩模式是否已定义 + Assert(m_cmprMode[col] != CMPR_UNDEF); + Form_pg_attribute* atts = RelationGetDescr(m_rel)->attrs; + // 检查属性是否为定长类型 + Assert(1 != atts[col]->attlen); + int cmprSize; + // 对压缩后的属性值进行解压缩，得到解压后的值和大小 + Datum value = UncompressOneAttr(m_cmprMode[col], + m_cmprMeta[col], + atts[col]->atttypid, + atts[col]->attlen, + DatumGetPointer(formTuple->values[col]), + &cmprSize); + // 检查解压缩后的大小与压缩元组中相应属性的值大小是否相等 + Assert(cmprSize == formTuple->valsize[col]); + // 检查原始值与解压缩后的值是否相等 + CheckCmprDatum(rawVal, value, atts[col]); +} +//检查压缩前后的元组是否一致。通过将压缩元组和原始元组解析成属性值和Null标记数组，然后逐个比较每个属性的值和Null标记是否一致来进行检查。 +void PageCompress::CheckCmprTuple(HeapTuple rawTup, HeapTuple cmprTup) +{ + TupleDesc desc = RelationGetDescr(m_rel); + Form_pg_attribute* atts = desc->attrs; + int nattrs = desc->natts; + // 申请内存用于保存属性值和Null标记数组 + Datum* values = (Datum*)palloc(sizeof(Datum) * nattrs); + Datum* values2 = (Datum*)palloc(sizeof(Datum) * nattrs); + bool* isnulls = (bool*)palloc(sizeof(bool) * nattrs); + bool* isnulls2 = (bool*)palloc(sizeof(bool) * nattrs); + // 解析压缩元组和原始元组，得到属性值和Null标记数组 + heap_deform_cmprs_tuple(cmprTup, desc, values, isnulls, m_cmprHeaderData); + heap_deform_tuple(rawTup, desc, values2, isnulls2); + + for (int col = 0; col < nattrs; ++col) { + // 检查压缩前后的Null标记是否一致 + Assert(isnulls[col] == isnulls2[col]); + if (isnulls[col]) { + // 如果属性为NULL，则对应的值应该为0 + Assert((Datum)0 == values[col]); + Assert((Datum)0 == values2[col]); + continue; + } + // 对于非NULL属性，调用CheckCmprDatum函数比较其值是否相等 + CheckCmprDatum(values[col], values2[col], atts[col]); + } + // 释放申请的内存 + pfree(values); + pfree(isnulls); + pfree(values2); + pfree(isnulls2); +} +//检查压缩表头中的压缩元数据是否正确 +void PageCompress::CheckCmprHeaderData(void) +{ + if (m_cmprHeaderSize == 0) { + return; + } + // 获取属性描述信息 + Form_pg_attribute* atts = RelationGetDescr(m_rel)->attrs; + int nattrs = RelationGetNumberOfAttributes(m_rel); + // 记录当前解析到的压缩信息的偏移量 + int cmprsOff = 0; /* pointer to the start of compression meta */ + void* metaInfo = NULL; + char mode = 0; + // 遍历所有属性 + for (int col = 0; col < nattrs; ++col) { + // 获取该属性的压缩元数据 + int metaSize = 0; + metaInfo = PageCompress::FetchAttrCmprMeta(m_cmprHeaderData + cmprsOff, atts[col]->attlen, &metaSize, &mode); + cmprsOff += metaSize; + // 检查压缩模式是否一致 + Assert(mode != CMPR_UNDEF && m_cmprMode[col] == mode); + // 根据不同的压缩模式进行比较 + switch (mode) { + case CMPR_DELTA: { + // Delta模式需要比较压缩的字节数和最小值 + DeltaCmprMeta* deltaInfo = (DeltaCmprMeta*)metaInfo; + DeltaCmprMeta* target = (DeltaCmprMeta*)m_cmprMeta[col]; + Assert(deltaInfo->bytes == target->bytes); + Assert(deltaInfo->MinVal == target->MinVal); + + break; + } + + case CMPR_DICT: { + // Dict模式需要比较词典项数量、每个词典项的长度和内容等 + DictCmprMeta* dictMeta = (DictCmprMeta*)metaInfo; + DictCmprMeta* target = (DictCmprMeta*)m_cmprMeta[col]; + Assert(dictMeta->dictItemNum == target->dictItemNum); + + for (int i = 0; i < dictMeta->dictItemNum; ++i) { + Assert(dictMeta->dictItems[i].itemSize == target->dictItems[i].itemSize); + Assert(0 == memcmp(dictMeta->dictItems[i].itemData, target->dictItems[i].itemData, + target->dictItems[i].itemSize)); + } + + break; + } + + case CMPR_PREFIX: { + // Prefix模式需要比较前缀的长度和内容 + PrefixCmprMeta* prefixMeta = (PrefixCmprMeta*)metaInfo; + PrefixCmprMeta* target = (PrefixCmprMeta*)m_cmprMeta[col]; + Assert(prefixMeta->len == target->len); + Assert(0 == memcmp(prefixMeta->prefixStr, target->prefixStr, target->len)); + + break; + } + + case CMPR_NUMSTR: + case CMPR_NONE: + break; + default: + break; + } + } + //检查整个压缩表头的大小是否正确，并且不超过页面大小 + Assert(m_cmprHeaderSize == (int)MAXALIGN(cmprsOff)); + Assert(m_cmprHeaderSize < BLCKSZ); +} + +/* + * check the number and lossless for input tuples. + * 1. dispatch to toast arrays; + * 2. dispatch to uncompressed arrays; + * 3. compress and then mapped to mapped arrays; + * Important: call before m_current is modified and changed. + */ +//检查输入元组的数量和无损性。它用于验证输入参数中指定的 TOAST、压缩和未压缩元组的数量是否与实际存在的元组数量一致，并确保所有输入的元组都能在已映射元组数组、未压缩元组数组和 TOAST 元组数组中找到对应的匹配元组 +void PageCompress::CheckOutputTups(HeapTuple* mapped, int nMapped, HeapTuple* uncmpr, int nUncmpr, int nToasts) +{ + int start = m_current; + int end = m_current + m_outputTupsNum + nToasts; + int realToasts = 0; + int realCmpr = 0; + int realUncmpr = 0; + // 校验输入参数是否合理 + Assert((m_outputTupsNum == nToasts) || (m_outputTupsNum == (nMapped + nUncmpr))); + + for (; start < end; ++start) { + HeapTuple check = m_inTups[start]; + bool found = false; + + if (isToastTuple(check)) { + // 遍历 TOAST 元组数组，检查是否存在匹配的元组 + for (int i = m_toastTupsNum - nToasts; i < m_toastTupsNum; ++i) { + if (check != m_toastTups[i]) + continue; + found = true; + ++realToasts; + break; + } + // 确保找到了匹配的 TOAST 元组 + Assert(found == true); + continue; + } + // 遍历已映射元组数组，检查是否存在匹配的元组 + for (int i = 0; i < nMapped; ++i) { + if (check != mapped[i]) + continue; + found = true; + ++realCmpr; + break; + } + // 如果在已映射元组数组中找不到匹配，则遍历未压缩元组数组进行匹配 + if (found == true) + continue; + + for (int i = 0; i < nUncmpr; ++i) { + if (check != uncmpr[i]) + continue; + found = true; + ++realUncmpr; + break; + } + // 确保找到了匹配的元组 + Assert(found == true); + } + // 断言验证实际存在的 TOAST、压缩和未压缩元组数量与指定的数量是否一致 + Assert(nToasts == realToasts); + Assert(nMapped == realCmpr); + Assert(nUncmpr == realUncmpr); +} + +#endif + +#ifdef TRACE_COMPRESS +//这个函数的作用是在堆表(page)压缩过程中，跟踪和打印各种数据信息，包括循环次数、范围、标记、属性的压缩方式和元数据、压缩后的元组、未压缩的元组、TOAST元组的位置、长度等信息 +void PageCompress::TraceCmprPage( + int flag, HeapTuple* cmpr, HeapTuple* mapped, int nCmpr, HeapTuple* uncmpr, int nUncmpr, int nToasts) +{ + int start = m_current;// 当前位置 + int end = m_current + m_outputTupsNum + nToasts;// 结束位置，考虑了TOAST元组 + int nattrs = RelationGetNumberOfAttributes(m_rel);// 属性数量 + Form_pg_attribute* atts = RelationGetDescr(m_rel)->attrs;// 属性描述符 + + char* tag = NULL; + if (flag == 0) + tag = "Tag: Raw Tuples Before Compress";// 标记：压缩前的原始元组 + else if (flag == 1) + tag = "Tag: Raw Tuples After Compress";// 标记：压缩后的原始元组 + else if (flag == 2) + tag = "Tag: Compressed Tuples Of one page";// 标记：单页压缩后的元组 + else + tag = "Tag: Toast Tuples";// 标记：TOAST元组 + ereport(INFO, (errmsg("Loop %d, range[%d - %d], %s", nLoops, start, (end - 1), tag))); + + DeltaCmprMeta* deltaMeta = NULL;// Delta压缩元数据指针 + PrefixCmprMeta* prefixMeta = NULL;// 前缀压缩元数据指针 + DictCmprMeta* dictMeta = NULL;// 字典压缩元数据指针 + // 遍历属性列 + for (int col = 0; col < nattrs; ++col) { + switch (m_cmprMode[col]) { + case CMPR_NONE: + ereport(INFO, (errmsg("COL %d: None Compress", col)));// 无压缩的属性 + break; + + case CMPR_DELTA: + deltaMeta = (DeltaCmprMeta*)m_cmprMeta[col]; + ereport(INFO, + (errmsg("COL %d: Delta Compress, attrlen %d, delta bytes %d", + col, + atts[col]->attlen, + deltaMeta->bytes)));// Delta压缩的属性，输出属性长度和Delta字节数 + break; + + case CMPR_NUMSTR: + ereport(INFO, (errmsg("COL %d: Number String Compress", col)));// 数字字符串压缩的属性 + break; + + case CMPR_PREFIX: + prefixMeta = (PrefixCmprMeta*)m_cmprMeta[col]; + ereport(INFO, (errmsg("COL %d: Prefix Compress, len %d", col, prefixMeta->len)));// 前缀压缩的属性，输出前缀长度 + break; + + case CMPR_DICT: + dictMeta = (DictCmprMeta*)m_cmprMeta[col]; + char str[0xFF * (sizeof(int16) + 1) + 1] = {0}; + char* p = str; + for (int i = 0; i < dictMeta->dictItemNum; ++i) { + (void)sprintf_s(p, sizeof(str), "%2d ", dictMeta->dictItems[i].itemSize); + p = p + 3; + } + ereport(INFO, (errmsg("COL %d: Dict Compress, num %d, len {%s}", col, dictMeta->dictItemNum, str)));// 字典压缩的属性，输出字典项数量和每个字典项的长度 + break; + default: + break; + } + } + // 遍历元组数组 + for (; start < end; ++start) { + HeapTuple check = m_inTups[start]; + + if (isToastTuple(check)) {// TOAST元组 + for (int i = m_toastTupsNum - nToasts; i < m_toastTupsNum; ++i) { + if (check != m_toastTups[i])// 匹配TOAST元组 + continue; + ereport(INFO, (errmsg(" [%d] TOAST, raw len %d", start, check->t_len)));// 输出TOAST元组的位置和原始长度 + break; + } + continue; + } + + for (int i = 0; i < nCmpr; ++i) { + if (check != mapped[i])// 匹配压缩后的元组 + continue; + ereport(INFO, (errmsg(" [%d] CMPR, raw len %d, cmpr len %d", start, check->t_len, cmpr[i]->t_len)));// 输出压缩后的元组的位置、原始长度和压缩后长度 + continue; + } + + for (int i = 0; i < nUncmpr; ++i) { + if (check != uncmpr[i])// 匹配未压缩的元组 + continue; + ereport(INFO, (errmsg(" [%d] UNCMPR, raw len %d", start, check->t_len)));// 输出未压缩的元组的位置和原始长度 + break; + } + } + + ereport(INFO, (errmsg("Summary: cmpr %d, uncmpr %d, toasts %d", nCmpr, nUncmpr, nToasts)));// 输出压缩和未压缩元组、TOAST元组的总数 +} + +#endif + +/* binary search. + * negative will be returned if search fails. Otherwise return the position of oid. + */ +//实现了一个基于二分查找算法的函数，用于在一个数字类型选项数组中查找给定 OID 对应的选项信息 +static const NumberTypeOpt* binarySearch(Oid typeOid) +{ + int left = 0;// 查找区间左端点 + int right = g_number_type_count - 1;// 查找区间右端点 + // 二分查找 + while (left <= right) { + int middle = left + ((right - left) / 2);// 中间位置 + + if (g_number_types[middle].typeOid > typeOid) {// OID值在当前元素左边 + right = middle - 1;// 缩小查找区间到左半部分 + } else if (g_number_types[middle].typeOid < typeOid) {// OID值在当前元素右边 + left = middle + 1;// 缩小查找区间到右半部分 + } else + return (g_number_types + middle);// 找到所需项，返回其指针 + } + + return NULL;// 查找失败，返回NULL +} + +/* int16 Compare && Minus && Plus functions. */ +//实现了比较两个 int16 类型数据的函数 int16Compare +static int int16Compare(Datum arg1, Datum arg2) +{ + int16 val1 = DatumGetInt16(arg1);// 从参数 arg1 中获取 int16 值 + int16 val2 = DatumGetInt16(arg2);// 从参数 arg2 中获取 int16 值 + if (val1 > val2)// 比较 val1 和 val2 的大小 + return (1);// 如果 val1 大于 val2，返回 1 + if (val1 < val2)// 比较 val1 和 val2 的大小 + return (-1);// 如果 val1 小于 val2，返回 -1 + return (0);// val1 等于 val2，返回 0 +} +//实现了两个 int16 类型数据相减的功能 +static int64 int16Minus(Datum arg1, Datum arg2) +{ + int16 val1 = DatumGetInt16(arg1);// 从参数 arg1 中获取 int16 值 + int16 val2 = DatumGetInt16(arg2);// 从参数 arg2 中获取 int16 值 + + PG_RETURN_INT64((int64)(val1 - val2));// 返回 val1 减去 val2 的结果，并将结果转换为 int64 类型 +} + +/* we don't care overflow problem about the plus, it's because: + * arg1 must be the result returned by int16Minus(); + * arg2 is from within int16Minus(); + */ +/* + * 该函数实现了 int16 类型数据和 int64 类型数据相加的功能。 + * 函数接受两个参数 arg1 和 arg2，其中 arg1 的类型为 Datum，表示 PostgreSQL 数据库内部的通用数据类型。 + */ +static Datum int16Plus(Datum arg1, int64 arg2) +{ + int16 val1 = DatumGetInt16(arg1);// 将 arg1 转换为 int16 类型的值，并赋给 val1 + int16 val2 = (int16)arg2;// 将 arg2 强制转换为 int16 类型，并赋给 val2 + + PG_RETURN_INT16((int16)(val1 + val2));// 返回 val1 和 val2 相加的结果 +} + +/* int32 Compare && Minus && Plus functions. */ +/* + * 该函数实现了 int32 类型数据的比较功能。函数接受两个参数 arg1 和 arg2， + * 这两个参数的类型为 Datum，表示 PostgreSQL 数据库内部的通用数据类型。 + */ +static int int32Compare(Datum arg1, Datum arg2) +{ + int32 val1 = DatumGetInt32(arg1);// 将 arg1 转换为 int32 类型的值，并赋给 val1 + int32 val2 = DatumGetInt32(arg2);// 将 arg2 转换为 int32 类型的值，并赋给 val2 + if (val1 > val2)// 如果 val1 大于 val2，则返回 1 + return 1; + if (val1 < val2)// 如果 val1 小于 val2，则返回 -1 + return -1; + return 0;// 如果 val1 等于 val2，则返回 0 +} +/* + * 该函数实现了两个 int32 类型数据相减的功能。函数接受两个参数 arg1 和 arg2， + * 这两个参数的类型为 Datum，表示 PostgreSQL 数据库内部的通用数据类型。 + */ +static int64 int32Minus(Datum arg1, Datum arg2) +{ + int32 val1 = DatumGetInt32(arg1);// 将 arg1 转换为 int32 类型的值，并赋给 val1 + int32 val2 = DatumGetInt32(arg2);// 将 arg2 转换为 int32 类型的值，并赋给 val2 + + PG_RETURN_INT64((int64)(val1 - val2));// 返回 val1 减去 val2 的结果，并转换为 int64 类型 +} + +/* we don't care overflow problem about the plus, it's because: + * arg1 must be the result returned by int32Minus(); + * arg2 is from within int32Minus(); + */ +/* + * 该函数实现了 int32 类型数据和 int64 类型数据相加的功能。 + * 函数接受两个参数 arg1 和 arg2，其中 arg1 的类型为 Datum，表示 PostgreSQL 数据库内部的通用数据类型。 + * 这里假设 arg1 是 int32Minus() 函数的返回值，arg2 是 int32Minus() 函数内部的一个参数 。 + * 这个函数不考虑溢出问题。 + */ +static Datum int32Plus(Datum arg1, int64 arg2) +{ + int32 val1 = DatumGetInt32(arg1);// 将 arg1 转换为 int32 类型的值，并赋给 val1 + int32 val2 = (int32)arg2; // 将 arg2 强制转换为 int32 类型，并赋给 val2 + + PG_RETURN_INT32((int32)(val1 + val2));// 返回 val1 和 val2 相加的结果 +} + +/* uint32 Compare && Minus && Plus functions. */ +/* + * 该函数实现了 uint32 类型数据的比较功能。函数接受两个参数 arg1 和 arg2， + * 这两个参数的类型为 Datum，表示 PostgreSQL 数据库内部的通用数据类型。 + */ +static int uint32Compare(Datum arg1, Datum arg2) +{ + uint32 val1 = DatumGetUInt32(arg1);// 将 arg1 转换为 uint32 类型的值，并赋给 val1 + uint32 val2 = DatumGetUInt32(arg2);// 将 arg2 转换为 uint32 类型的值，并赋给 val2 + if (val1 > val2)// 如果 val1 大于 val2，则返回 1 + return 1; + if (val1 < val2)// 如果 val1 小于 val2，则返回 -1 + return -1; + return 0;// 如果 val1 等于 val2，则返回 0 +} +/* + * 该函数实现了两个 uint32 类型数据相减的功能。函数接受两个参数 arg1 和 arg2， + * 这两个参数的类型为 Datum，表示 PostgreSQL 数据库内部的通用数据类型。 + */ +static int64 uint32Minus(Datum arg1, Datum arg2) +{ + uint32 val1 = DatumGetUInt32(arg1);// 将 arg1 转换为 uint32 类型的值，并赋给 val1 + uint32 val2 = DatumGetUInt32(arg2);// 将 arg2 转换为 uint32 类型的值，并赋给 val2 + + PG_RETURN_INT64((int64)(val1 - val2));// 返回 val1 减去 val2 的结果，并转换为 int64 类型 +} + +/* we don't care overflow problem about the plus, it's because: + * arg1 must be the result returned by uint32Minus(); + * arg2 is from within uint32Minus(); + */ +/* + * 该函数实现了 uint32 类型数据和 int64 类型数据相加的功能。 + * 函数接受两个参数 arg1 和 arg2，其中 arg1 的类型为 Datum，表示 PostgreSQL 数据库内部的通用数据类型。 + * 这里假设 arg1 是 uint32Minus() 函数的返回值，arg2 是 uint32Minus() 函数内部的一个参数 。 + * 这个函数不考虑溢出问题。 + */ +static Datum uint32Plus(Datum arg1, int64 arg2) +{ + uint32 val1 = DatumGetUInt32(arg1);// 将 arg1 转换为 uint32 类型的值，并赋给 val1 + uint32 val2 = (uint32)arg2;// 将 arg2 强制转换为 uint32 类型，并赋给 val2 + + PG_RETURN_UINT32((uint32)(val1 + val2));// 返回 val1 和 val2 相加的结果，转换为 uint32 类型 +} + +/* int64 Compare && Minus && Plus functions. */ +/* + * 该函数实现了 int64 类型数据的比较功能。函数接受两个参数 arg1 和 arg2， + * 这两个参数的类型为 Datum，表示 PostgreSQL 数据库内部的通用数据类型。 + */ +static int int64Compare(Datum arg1, Datum arg2) +{ + int64 val1 = DatumGetInt64(arg1);// 将 arg1 转换为 int64 类型的值，并赋给 val1 + int64 val2 = DatumGetInt64(arg2);// 将 arg2 转换为 int64 类型的值，并赋给 val2 + if (val1 > val2)// 如果 val1 大于 val2，则返回 1 + return 1; + if (val1 < val2)// 如果 val1 小于 val2，则返回 -1 + return -1; + return 0; // 如果 val1 等于 val2，则返回 0 +} +/* + * 该函数实现了两个 int64 类型数据相减的功能。函数接受两个参数 arg1 和 arg2， + * 这两个参数的类型为 Datum，表示 PostgreSQL 数据库内部的通用数据类型。 + */ +static int64 int64Minus(Datum arg1, Datum arg2) +{ + int64 val1 = DatumGetInt64(arg1);// 将 arg1 转换为 int64 类型的值，并赋给 val1 + int64 val2 = DatumGetInt64(arg2);// 将 arg2 转换为 int64 类型的值，并赋给 val2 + + PG_RETURN_INT64((int64)(val1 - val2));// 返回 val1 减去 val2 的结果，并转换为 int64 类型 +} + +/* we don't care overflow problem about the plus, it's because: + * arg1 must be the result returned by int64Minus(); + * arg2 is from within int64Minus(); + */ +/* + * 该函数实现了 int64 类型数据和 int64 类型数据相加的功能。 + * 函数接受两个参数 arg1 和 arg2，其中 arg1 的类型为 Datum，表示 PostgreSQL 数据库内部的通用数据类型。 + * 这里假设 arg1 是 int64Minus() 函数的返回值，arg2 是 int64Minus() 函数内部的一个参数 。 + * 这个函数不考虑溢出问题。 + */ +static Datum int64Plus(Datum arg1, int64 arg2) +{ + int64 val1 = DatumGetInt64(arg1);// 将 arg1 转换为 int64 类型的值，并赋给 val1 + + PG_RETURN_INT64((int64)(val1 + arg2));// 返回 val1 和 arg2 相加的结果 +} -- 2.34.1 From 8436b49b49a081cb9cc64ec01c289a4afd6c123b Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Mon, 2 Oct 2023 19:58:42 +0800 Subject: [PATCH 19/48] Update pageparse.cpp --- src/gausskernel/storage/page/pageparse.cpp | 464 ++++++++++++++------- 1 file changed, 319 insertions(+), 145 deletions(-) diff --git a/src/gausskernel/storage/page/pageparse.cpp b/src/gausskernel/storage/page/pageparse.cpp index 7cfd4130a..0c3b96a5c 100644 --- a/src/gausskernel/storage/page/pageparse.cpp +++ b/src/gausskernel/storage/page/pageparse.cpp @@ -53,7 +53,10 @@ typedef enum { UBTREE_INDEX, INDEX_BOTT } INDEX_TYPE; - +/* +* 作用：检查用户权限，只有系统管理员（superuser）或者操作员管理员（operator admin）并且处于操作模式（operation +* mode）下才能使用指定的函数（fName）。 参数 fName：指定的函数名称。 +*/ void CheckUser(const char *fName) { if (!superuser() && !(isOperatoradmin(GetUserId()) && u_sess->attr.attr_security.operation_mode)) @@ -82,29 +85,35 @@ static void ParseIndexPageSpecialInfo(void *page, int type, char *output); static void ParseIndexPageData(void *page, int type, char *output); static void ParseIndexPage(void *page, int type, BlockNumber blkno, BlockNumber endBlk, char *output); - +//将一个字节数组（字符串）转换为二进制位表示的字符串。具体来说，它会遍历字节数组中的每个字节，并将每个二进制位转换为对应的字符(bit1 或 bit0)，然后将这些字符存储在指定的输出字符数组中。 static void formatBitmap(const unsigned char *start, int len, char bit1, char bit0, char *strOutput) { errno_t rc = EOK; + // 遍历字节数组中的每一个字节 for (int i = 0; i < len; ++i) { unsigned char ch = start[i]; unsigned char bitmask = 1; /* print 8 bits within a loop */ + // 打印 8 个二进制位的循环 do { + // 将当前二进制位对应的字符（bit1 或 bit0）添加到输出字符数组中 rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "%c", ((ch & bitmask) ? bit1 : bit0)); securec_check_ss(rc, "\0", "\0"); + // 将位掩码向左移动一位，以便添加下一个二进制位的字符 bitmask <<= 1; } while (bitmask != 0); + // 在输出字符数组中添加一个空格，表示一个字节的转换已经结束 rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, " "); securec_check_ss(rc, "\0", "\0"); } } /* init RelFileNode and outputFilename */ +/* 初始化RelFileNode和outputFilename */ void PrepForRead(char *path, int64 blocknum, char *relation_type, char *outputFilename, RelFileNode *relnode, bool parse_page) -{ +{ // 检查压缩表文件 if (CalculateCompressMainForkSize(path, true) != 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("compressed table file is not allowed here.")))); @@ -112,12 +121,12 @@ void PrepForRead(char *path, int64 blocknum, char *relation_type, char *outputFi char *pathFirstpart = (char *)palloc(MAXFNAMELEN * sizeof(char)); errno_t rc = memset_s(pathFirstpart, MAXFNAMELEN, 0, MAXFNAMELEN); securec_check(rc, "\0", "\0"); - + // 如果是分段关系类型，则解析bucketNode if ((strcmp(relation_type, "segment") == 0)) { relnode->bucketNode = SegmentBktId; char *bucketNodestr = strstr(path, "_b"); if (NULL != bucketNodestr) { - bucketNodestr += TWO; /* delete first two chars: _b */ + bucketNodestr += TWO; /* delete first two chars: _b *//* 删除前两个字符: _b */ relnode->bucketNode = (int4)pg_strtouint64(bucketNodestr, NULL, TENBASE); rc = strncpy_s(pathFirstpart, strlen(path) - strlen(bucketNodestr) - 1, path, strlen(path) - strlen(bucketNodestr) - TWO); @@ -127,32 +136,38 @@ void PrepForRead(char *path, int64 blocknum, char *relation_type, char *outputFi relnode->bucketNode = InvalidBktId; } RelFileNodeForkNum relfilenode; + // 根据路径获取文件节点 if (strlen(pathFirstpart) == 0) { relfilenode = relpath_to_filenode(path); } else { relfilenode = relpath_to_filenode(pathFirstpart); } + // 检查表空间Oid是否为0 if (relfilenode.rnode.node.spcNode == 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("The tablespace oid is 0. Please check the first parameter path. " "If you are not sure about the table path, please check pg_relation_filepath.")))); + // 将文件节点信息拷贝到relnode中 RelFileNodeRelCopy(*relnode, relfilenode.rnode.node); relnode->opt = 0; char *pagesuffix = "page"; char *xlogsuffix = "xlog"; + // 生成输出文件名 rc = snprintf_s(outputFilename + (int)strlen(outputFilename), MAXFILENAME, MAXFILENAME - 1, "%s/%u_%u_%u_%d.%s", t_thrd.proc_cxt.DataDir, relnode->spcNode, relnode->dbNode, relnode->relNode, blocknum, (parse_page ? pagesuffix : xlogsuffix)); securec_check_ss(rc, "\0", "\0"); - pfree_ext(pathFirstpart); + pfree_ext(pathFirstpart);// 释放内存 } - +// 输出页面的大小和版本信息，以及pd_xid_base、pd_multi_base、pd_prune_xid三个关键字段的值 static void ParseHeapHeader(const PageHeader page, char *strOutput, BlockNumber blockNum, BlockNumber block_endpoint) { + // 输出页面块号和终止块号信息 errno_t rc = EOK; rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "Page information of block %u/%u", blockNum, block_endpoint); securec_check_ss(rc, "\0", "\0"); + // 输出页面LSN信息和校验和验证结果 rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "\n\tpd_lsn: %X/%X", (uint32)(PageGetLSN(page) >> XIDTHIRTYTWO), (uint32)PageGetLSN(page)); securec_check_ss(rc, "\0", "\0"); @@ -165,7 +180,7 @@ static void ParseHeapHeader(const PageHeader page, char *strOutput, BlockNumber rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "\n\tpd_checksum: 0x%X, verify %s", page->pd_checksum, checksum_matched ? "success" : "fail"); securec_check_ss(rc, "\0", "\0"); - + // 根据页面标志位输出页面属性信息 rc = strcat_s(strOutput, MAXOUTPUTLEN, "\n\tpd_flags: "); securec_check(rc, "\0", "\0"); if (PageHasFreeLinePointers(page)) { @@ -192,6 +207,7 @@ static void ParseHeapHeader(const PageHeader page, char *strOutput, BlockNumber rc = strcat_s(strOutput, MAXOUTPUTLEN, "PD_ENCRYPT_PAGE "); securec_check(rc, "\0", "\0"); } + // 输出页面的pd_lower、pd_upper、pd_special信息 rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "\n\tpd_lower: %u, %s", page->pd_lower, PageIsEmpty(page) ? "empty" : "non-empty"); securec_check_ss(rc, "\0", "\0"); @@ -201,6 +217,7 @@ static void ParseHeapHeader(const PageHeader page, char *strOutput, BlockNumber rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "\n\tpd_special: %u, size %u", page->pd_special, PageGetSpecialSize(page)); securec_check_ss(rc, "\0", "\0"); + // 输出页面的大小和版本信息，以及pd_xid_base、pd_multi_base、pd_prune_xid三个关键字段的值 rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "\n\tPage size & version: %u, %u", (uint16)PageGetPageSize(page), (uint16)PageGetPageLayoutVersion(page)); securec_check_ss(rc, "\0", "\0"); @@ -212,67 +229,82 @@ static void ParseHeapHeader(const PageHeader page, char *strOutput, BlockNumber ((HeapPageHeader)(page))->pd_prune_xid + ((HeapPageHeader)(page))->pd_xid_base); securec_check_ss(rc, "\0", "\0"); } - +//根据tup中的位标志获取一系列的信息，并将它们存储到strOutput中 static void PrintInfomask(HeapTupleHeader tup, char *strOutput) { errno_t rc = EOK; + // 检查是否存在NULL值 if (tup->t_infomask & HEAP_HASNULL) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_HASNULL "); securec_check(rc, "\0", "\0"); } + // 检查是否存在可变宽度的字段 if (tup->t_infomask & HEAP_HASVARWIDTH) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_HASVARWIDTH "); securec_check(rc, "\0", "\0"); } + // 检查是否存在外部存储的字段 if (tup->t_infomask & HEAP_HASEXTERNAL) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_HASEXTERNAL "); securec_check(rc, "\0", "\0"); } + // 检查是否存在OID字段，并获取其值 if (tup->t_infomask & HEAP_HASOID) { rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "HEAP_HASOID(%d) ", HeapTupleHeaderGetOid(tup)); securec_check_ss(rc, "\0", "\0"); } + // 检查是否被压缩 if (tup->t_infomask & HEAP_COMPRESSED) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_COMPRESSED "); securec_check(rc, "\0", "\0"); } + // 检查是否为ComboCID if (tup->t_infomask & HEAP_COMBOCID) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_COMBOCID "); securec_check(rc, "\0", "\0"); } + // 检查是否具有Xmax排他锁 if (tup->t_infomask & HEAP_XMAX_EXCL_LOCK) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_XMAX_EXCL_LOCK "); securec_check(rc, "\0", "\0"); } + // 检查是否具有Xmax共享锁 if (tup->t_infomask & HEAP_XMAX_SHARED_LOCK) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_XMAX_SHARED_LOCK "); securec_check(rc, "\0", "\0"); } + // 检查是否为已提交的Xmin事务ID if (tup->t_infomask & HEAP_XMIN_COMMITTED) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_XMIN_COMMITTED "); securec_check(rc, "\0", "\0"); } + // 检查是否为无效的Xmin事务ID if (tup->t_infomask & HEAP_XMIN_INVALID) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_XMIN_INVALID "); securec_check(rc, "\0", "\0"); } + // 检查是否为已提交的Xmax事务ID if (tup->t_infomask & HEAP_XMAX_COMMITTED) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_XMAX_COMMITTED "); securec_check(rc, "\0", "\0"); } + // 检查是否为无效的Xmax事务ID if (tup->t_infomask & HEAP_XMAX_INVALID) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_XMAX_INVALID "); securec_check(rc, "\0", "\0"); } + // 检查Xmax事务ID是否为多个 if (tup->t_infomask & HEAP_XMAX_IS_MULTI) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_XMAX_IS_MULTI "); securec_check(rc, "\0", "\0"); } + // 检查是否已被更新 if (tup->t_infomask & HEAP_UPDATED) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_UPDATED "); securec_check(rc, "\0", "\0"); } + // 检查是否存在8字节的用户ID字段 if ((tup->t_infomask & HEAP_HAS_8BYTE_UID)) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_HAS_8BYTE_UID "); securec_check(rc, "\0", "\0"); @@ -280,22 +312,26 @@ static void PrintInfomask(HeapTupleHeader tup, char *strOutput) rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_HAS_NO_UID "); securec_check(rc, "\0", "\0"); } + // 追加t_infomask2的标志说明 rc = strcat_s(strOutput, MAXOUTPUTLEN, "\n\t\t\tt_infomask2: "); securec_check(rc, "\0", "\0"); + // 检查是否为热更新的堆块 if (tup->t_infomask2 & HEAP_HOT_UPDATED) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_HOT_UPDATED "); securec_check(rc, "\0", "\0"); } + // 检查是否只有tuple数据，没有堆块头 if (tup->t_infomask2 & HEAP_ONLY_TUPLE) { rc = strcat_s(strOutput, MAXOUTPUTLEN, "HEAP_ONLY_TUPLE "); securec_check(rc, "\0", "\0"); } } - +//解析给定页上指定行号的元组头信息，并将解析结果存储在strOutput字符串数组中 static void ParseTupleHeader(const PageHeader page, uint lineno, char *strOutput) { errno_t rc = EOK; ItemId lp = PageGetItemId(page, lineno); + // 将元组的长度和偏移量等信息追加到strOutput中 rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "\n\n\t\tTuple #%u is normal: length %u, offset %u", lineno, ItemIdGetLength(lp), ItemIdGetOffset(lp)); securec_check_ss(rc, "\0", "\0"); @@ -305,10 +341,12 @@ static void ParseTupleHeader(const PageHeader page, uint lineno, char *strOutput dummyTuple.t_data = tup; dummyTuple.t_xid_base = ((HeapPageHeader)(page))->pd_xid_base; dummyTuple.t_multi_base = ((HeapPageHeader)(page))->pd_multi_base; + // 将t_xmin、t_xmax和t_cid信息追加到strOutput中 rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "\n\t\t\tt_xmin/t_xmax/t_cid: %lu/%lu/%u", HeapTupleGetRawXmin(&dummyTuple), HeapTupleGetRawXmax(&dummyTuple), HeapTupleHeaderGetRawCommandId(tup)); securec_check_ss(rc, "\0", "\0"); + // 将ctid信息追加到strOutput中 rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "\n\t\t\tctid:(block %u/%u, offset %u)", tup->t_ctid.ip_blkid.bi_hi, tup->t_ctid.ip_blkid.bi_lo, tup->t_ctid.ip_posid); @@ -316,26 +354,27 @@ static void ParseTupleHeader(const PageHeader page, uint lineno, char *strOutput rc = strcat_s(strOutput, MAXOUTPUTLEN, "\n\t\t\tt_infomask: "); securec_check(rc, "\0", "\0"); - + // 调用PrintInfomask函数将t_infomask的详细解释追加到strOutput中 PrintInfomask(tup, strOutput); - + // 将Attrs Num追加到strOutput中 rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "Attrs Num: %d", HeapTupleHeaderGetNatts(tup, NULL)); securec_check_ss(rc, "\0", "\0"); - + // 将t_hoff追加到strOutput中 rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "\n\t\t\tt_hoff: %u", tup->t_hoff); securec_check_ss(rc, "\0", "\0"); rc = strcat_s(strOutput, MAXOUTPUTLEN, "\n\t\t\tt_bits: "); securec_check(rc, "\0", "\0"); - + // 调用formatBitmap函数将t_bits的信息追加到strOutput中 formatBitmap((const unsigned char *)tup->t_bits, BITMAPLEN(HeapTupleHeaderGetNatts(tup, NULL)), 'V', 'N', strOutput); } - +//解析给定页的堆页面信息，并将解析结果存储在strOutput字符串数组中 static void ParseHeapPage(const PageHeader page, BlockNumber blockNum, char *strOutput, BlockNumber block_endpoint) { errno_t rc = EOK; + /* 检查页面数据指针范围，判断是否存在数据损坏的情况 */ if (page->pd_lower < GetPageHeaderSize(page) || page->pd_lower > page->pd_upper || page->pd_upper > page->pd_special || page->pd_special > BLCKSZ || page->pd_special != MAXALIGN(page->pd_special)) { @@ -349,18 +388,21 @@ static void ParseHeapPage(const PageHeader page, BlockNumber blockNum, char *str securec_check_ss(rc, "\0", "\0"); return; } - + /* 解析堆页头信息 */ ParseHeapHeader(page, strOutput, blockNum, block_endpoint); /* parse tuple header */ + /* 输出堆页面上元组的信息 */ rc = strcat_s(strOutput, MAXOUTPUTLEN, "\n\n\tHeap tuple information on this page"); securec_check(rc, "\0", "\0"); uint nline = PageGetMaxOffsetNumber((Page)page); uint nunused = 0, nnormal = 0, ndead = 0; + /* 遍历堆页面上的每个元组 */ for (uint i = (OffsetNumber)1; i <= nline; i++) { ItemId lp = PageGetItemId(page, i); if (ItemIdIsNormal(lp)) { nnormal++; + /* 解析元组头信息 */ ParseTupleHeader(page, i, strOutput); } else if (ItemIdIsDead(lp)) { ndead++; @@ -374,17 +416,20 @@ static void ParseHeapPage(const PageHeader page, BlockNumber blockNum, char *str securec_check_ss(rc, "\0", "\0"); } } + /* 输出元组的统计信息 */ rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, "\n\tSummary (%u total): %u normal, %u unused, %u dead\n\nNormal Heap Page, special space is %u\n\n", nline, nnormal, nunused, ndead, (uint)PageGetSpecialSize(page)); securec_check_ss(rc, "\0", "\0"); } - +//解析给定页的信息并将解析结果存储在字符串数组 strOutput 中 static void ParseOnePage(const PageHeader page, BlockNumber blockNum, char *strOutput, char *relation_type, BlockNumber block_endpoint) { + // 根据关系类型判断页面类型 errno_t rc = EOK; if (strcmp(relation_type, "heap") == 0) { + // 堆页面解析逻辑 if (PG_HEAP_PAGE_LAYOUT_VERSION != (uint16)PageGetPageLayoutVersion(page) || PageGetSpecialSize(page) != 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("The target page is not heap, the given page version is: %u", @@ -392,6 +437,7 @@ static void ParseOnePage(const PageHeader page, BlockNumber blockNum, char *strO } ParseHeapPage(page, blockNum, strOutput, block_endpoint); } else if (strcmp(relation_type, "uheap") == 0) { + // UHeap页面解析逻辑 if (PG_UHEAP_PAGE_LAYOUT_VERSION != (uint16)PageGetPageLayoutVersion(page) || PageGetSpecialSize(page) != 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("The target page is not uheap, the given page version is: %u", @@ -399,6 +445,7 @@ static void ParseOnePage(const PageHeader page, BlockNumber blockNum, char *strO } ParseUHeapPage((void *)page, blockNum, block_endpoint, strOutput); } else if (strcmp(relation_type, "btree") == 0) { + // B-Tree页面解析逻辑 if (PG_COMM_PAGE_LAYOUT_VERSION != (uint16)PageGetPageLayoutVersion(page) || PageGetSpecialSize(page) == 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("The target page is not btree, the given page version is: %u", @@ -406,6 +453,7 @@ static void ParseOnePage(const PageHeader page, BlockNumber blockNum, char *strO } ParseIndexPage((void *)page, BTREE_INDEX, blockNum, block_endpoint, strOutput); } else if (strcmp(relation_type, "ubtree") == 0) { + // UB-Tree页面解析逻辑 if (PG_COMM_PAGE_LAYOUT_VERSION != (uint16)PageGetPageLayoutVersion(page) || PageGetSpecialSize(page) == 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("The target page is not ubtree, the given page version is: %u", @@ -413,30 +461,38 @@ static void ParseOnePage(const PageHeader page, BlockNumber blockNum, char *strO } ParseIndexPage((void *)page, UBTREE_INDEX, blockNum, block_endpoint, strOutput); } else if (strcmp(relation_type, "segment") == 0) { + // 分段表页面解析逻辑 rc = strcat_s(strOutput, MAXOUTPUTLEN, "Parse segment table."); securec_check(rc, "\0", "\0"); ParseHeapPage(page, blockNum, strOutput, block_endpoint); } else { + // 关系类型错误，输出错误信息 ereport(ERROR, (errcode(ERRCODE_UNDEFINED_PARAMETER), (errmsg("Only support heap, uheap, btree, ubtree, segment.")))); } } - +//根据给定的读取状态和页面信息，检查并生成相应的错误信息，并将结果存储在指定的字符串中 static void check_rdStatus(SMGR_READ_STATUS rdStatus, BlockNumber blockNum, const PageHeader page, char *strOutput1) { - errno_t rc = EOK; + errno_t rc = EOK;// 用于保存 snprintf_s 的返回值，即输出字符数 + // 检查读取状态 if (rdStatus == SMGR_RD_CRC_ERROR) { + // 当读取状态为 CRC 错误时，计算当前页的校验和并生成相关错误信息 uint16 checksum = pg_checksum_page((char *)page, blockNum); rc = snprintf_s(strOutput1, SHORTOUTPUTLEN, SHORTOUTPUTLEN - 1, "\nFor page %u, page verification failed, calculated checksum 0x%X but expected 0x%X.\n", blockNum, checksum, page->pd_checksum); - securec_check_ss(rc, "\0", "\0"); + securec_check_ss(rc, "\0", "\0");// 检查 snprintf_s 是否执行成功 } else if (rdStatus == SMGR_RD_NO_BLOCK) { + // 当读取状态为无块（不存在）时，生成相关错误信息 rc = snprintf_s(strOutput1, SHORTOUTPUTLEN, SHORTOUTPUTLEN - 1, "\tThe page %u does not exist.\n", blockNum); - securec_check_ss(rc, "\0", "\0"); + securec_check_ss(rc, "\0", "\0");// 检查 snprintf_s 是否执行成功 } } - +/* +这段代码的作用是从内存中读取指定的页， +如果读取成功，则返回 true，并将相应的页面信息存储在一个指定的字符串中（strOutput）。 +*/ static bool readFromMemory(SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum, char *relation_type, char *strOutput, FILE *outputfile, char *outputFilename) { @@ -444,15 +500,22 @@ static bool readFromMemory(SMgrRelation smgr, ForkNumber forkNum, BlockNumber bl SegPageLocation loc; loc.blocknum = 0; /* for segment, update to physical blocknum */ + // 如果目标页是段（segment）相关，则更新目标页面的物理块号 loc if (strcmp(relation_type, "segment") == 0) { loc = seg_get_physical_location(smgr->smgr_rnode.node, forkNum, blockNum); } + // 初始化目标页的缓存标记 new_tag，以及在缓冲池中的散列值 new_hash BufferTag new_tag; INIT_BUFFERTAG(new_tag, smgr->smgr_rnode.node, forkNum, blockNum); uint32 new_hash = BufTableHashCode(&new_tag); LWLock *partition_lock = BufMappingPartitionLock(new_hash); + // 资源管理器与缓存管理器之间的协调， + // 增加当前资源所有者使用的缓冲区数， + // 以便管理缓冲池时，更准确地估计所需缓冲区的数量 ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); /* LW lock to avoid concurrent read/write */ + // LW_LOCK：轻量级锁，避免并发读写冲突。 + // 这里先请求 partition_lock 共享锁，避免锁竞争 (void)LWLockAcquire(partition_lock, LW_SHARED); int buf_id = BufTableLookup(&new_tag, new_hash); @@ -462,50 +525,62 @@ static bool readFromMemory(SMgrRelation smgr, ForkNumber forkNum, BlockNumber bl bool valid = (strcmp(relation_type, "segment") == 0) ? SegPinBuffer(bufDesc) : PinBuffer(bufDesc, NULL); if (!valid) break; /* pin failed, read disk */ + // 释放 partition_lock 共享锁 LWLockRelease(partition_lock); + // 获取缓冲区页面的 content_lock 排他锁 (void)LWLockAcquire(BufferDescriptorGetContentLock(bufDesc), LW_SHARED); /* acquire content_lock */ Buffer buf = BufferDescriptorGetBuffer(bufDesc); const PageHeader page = (const PageHeader)BufferGetPage(buf); + // 将结果字符串 strOutput 中添加“来自内存”信息 rc = strcat_s(strOutput, MAXOUTPUTLEN, "The target page is from memory. "); securec_check(rc, "\0", "\0"); if (strcmp(relation_type, "segment") == 0) { + // 如果是 segment 相关，则解析物理块号为 loc.blocknum 的页面 ParseOnePage(page, loc.blocknum, strOutput, relation_type, loc.blocknum); /* physical blocknum */ - LWLockRelease(BufferDescriptorGetContentLock(bufDesc)); /* release content_lock */ + LWLockRelease(BufferDescriptorGetContentLock(bufDesc)); /* release content_lock */ // 释放 content_lock SegUnpinBuffer(bufDesc); } else { + // 否则，解析逻辑块号为 blockNum 的页面 ParseOnePage(page, blockNum, strOutput, relation_type, blockNum); LWLockRelease(BufferDescriptorGetContentLock(bufDesc)); /* release content_lock */ UnpinBuffer(bufDesc, true); } + // 将解析结果写入输出文件 outputfile，并释放动态分配的字符串内存 CheckWriteFile(fwrite(strOutput, 1, strlen(strOutput), outputfile), strlen(strOutput), outputFilename); pfree_ext(strOutput); return true; } } while (0); + // 如果目标页不在缓冲池中，需要释放 partition_lock 共享锁，并返回 false LWLockRelease(partition_lock); return false; } - +//检查给定的关系文件节点中指定页是否存在，并验证该页的头部信息是否与关系文件类型相匹配 static void CheckSegment(RelFileNode *relnode, ForkNumber forkNum) { + //打开关系文件所在的段（spc），并判断文件是否存在，如果不存在则抛出错误。 SegSpace *spc = spc_open(relnode->spcNode, relnode->dbNode, false); if (spc == NULL || !spc_datafile_exist(spc, 1, forkNum)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("Page doesn't exist.")))); + //获取关系文件在段中的大小，并检查指定页是否小于该大小，如果大于则抛出错误。 BlockNumber size = spc_size(spc, 1, forkNum); if (relnode->relNode >= size) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("The target page %u doesn't exist, the current max is %u.", relnode->relNode, size - 1)))); + //创建一个新的关系文件节点 relnodeHead，并初始化为与原节点关联的段的头部，然后读取该段的头部信息，如果读取失败则抛出错误。 RelFileNode *relnodeHead = (RelFileNode *)palloc(sizeof(RelFileNode)); relnodeHead->spcNode = relnode->spcNode; relnodeHead->dbNode = relnode->dbNode; - relnodeHead->relNode = 1; + relnodeHead->relNode = 1;// 头部的页号始终是1 relnodeHead->bucketNode = relnode->bucketNode; relnodeHead->opt = relnode->opt; + // 读取指定页的头部信息 Buffer buffer_temp = ReadBufferFast(spc, *relnodeHead, forkNum, relnode->relNode, RBM_NORMAL); if (!BufferIsValid(buffer_temp)) ereport(ERROR, (errcode_for_file_access(), errmsg("Segment Head is invalid %u/%u/%u %d %u", relnodeHead->spcNode, relnodeHead->dbNode, relnodeHead->relNode, forkNum, relnode->relNode))); pfree_ext(relnodeHead); + //验证读取到的头部信息是否与给定的关系文件类型匹配，如果匹配则说明给定的页是有效的，否则抛出错误。 SegmentHead *head = (SegmentHead *)PageGetContents(BufferGetPage(buffer_temp)); SegReleaseBuffer(buffer_temp); if (!(IsNormalSegmentHead(head) && relnode->bucketNode == SegmentBktId) && @@ -514,23 +589,29 @@ static void CheckSegment(RelFileNode *relnode, ForkNumber forkNum) (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("Page header does not match with corresponding segment type.")))); } - +/* + * 功能：验证传入的文件路径是否与指定的关系文件节点匹配。 + * 匹配规则为：通过 relpathbackend() 函数获取关系文件所在的路径，并将其与传入的路径进行比较， + * 如果两者不相等，则抛出错误。 + */ void ValidateParameterPath(RelFileNode rnode, char *str) { - char *path = relpathbackend(rnode, InvalidBackendId, MAIN_FORKNUM); + char *path = relpathbackend(rnode, InvalidBackendId, MAIN_FORKNUM);//获取指定关系文件节点的物理路径 if ((strcmp(path, str) != 0)) ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("\"%s\" is invalid input", str))); } - +//解析给定路径下指定块的页面数据，并将解析结果写入文件 char *ParsePage(char *path, int64 blocknum, char *relation_type, bool read_memory) { errno_t rc = EOK; /* check parameters */ + /* 检查参数 */ if (blocknum > MaxBlockNumber || blocknum < -1) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("Blocknum should be between -1 and %u.", MaxBlockNumber)))); /* initialize */ + /* 初始化 */ char *strOutput = (char *)palloc(MAXOUTPUTLEN * sizeof(char)); rc = memset_s(strOutput, MAXOUTPUTLEN, 0, MAXOUTPUTLEN); securec_check(rc, "\0", "\0"); @@ -540,7 +621,7 @@ char *ParsePage(char *path, int64 blocknum, char *relation_type, bool read_memor rc = memset_s(outputFilename, MAXFILENAME, 0, MAXFILENAME); securec_check(rc, "\0", "\0"); - + // 准备读取文件 PrepForRead(path, blocknum, relation_type, outputFilename, relnode, true); ValidateParameterPath(*relnode, path); @@ -549,9 +630,11 @@ char *ParsePage(char *path, int64 blocknum, char *relation_type, bool read_memor ForkNumber forkNum = MAIN_FORKNUM; SMgrRelation smgr = smgropen(*relnode, InvalidBackendId, GetColumnNum(forkNum)); + // 检查关系类型是否为"segment" if (strcmp(relation_type, "segment") == 0) CheckSegment(relnode, forkNum); BlockNumber maxBlockNum = smgrnblocks(smgr, forkNum) - 1; + // 检查blocknum是否合法 if (blocknum > maxBlockNum) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("Blocknum should be between -1 and %u.", maxBlockNum)))); @@ -560,57 +643,65 @@ char *ParsePage(char *path, int64 blocknum, char *relation_type, bool read_memor BlockNumber blockNum = 0; BlockNumber block_endpoint = 0; /* only parse one block, check if it is in memory */ + /* 只解析一个块，检查是否在内存中 */ if (read_memory) { blockNum = (BlockNumber)blocknum; + // 如果内存中存在要解析的块，则直接返回结果 if (readFromMemory(smgr, forkNum, blockNum, relation_type, strOutput, outputfile, outputFilename)) { /* found in memory */ + /* 在内存中找到 */ CheckCloseFile(fclose(outputfile), outputFilename); smgrcloseall(); return outputFilename; } } + /* 在内存中找到 */ /* read from disk */ rc = strcat_s(strOutput, MAXOUTPUTLEN, "The target page is from disk. "); securec_check(rc, "\0", "\0"); CheckWriteFile(fwrite(strOutput, 1, strlen(strOutput), outputfile), strlen(strOutput), outputFilename); pfree_ext(strOutput); - if (blocknum >= 0) { /* only parse one block */ + if (blocknum >= 0) { /* only parse one block */ /* 只解析一个块 */ blockNum = blocknum; block_endpoint = blocknum; - } else { /* blocknum == -1, parse all blocks */ + } else { /* blocknum == -1, parse all blocks */ /* blocknum == -1，解析所有块 */ block_endpoint = maxBlockNum; } /* if not declare a single block, then loop all blocks */ + /* 如果没有声明单个块，则循环遍历所有块 */ while (blockNum <= block_endpoint) { - char *strOutput = (char *)palloc(MAXOUTPUTLEN * sizeof(char)); - rc = memset_s(strOutput, MAXOUTPUTLEN, 0, MAXOUTPUTLEN); + char *strOutput = (char *)palloc(MAXOUTPUTLEN * sizeof(char));// 分配输出字符串内存 + rc = memset_s(strOutput, MAXOUTPUTLEN, 0, MAXOUTPUTLEN);// 初始化输出字符串为0 securec_check(rc, "\0", "\0"); - char *buffer = (char *)palloc0(BLCKSZ); - SMGR_READ_STATUS rdStatus = smgrread(smgr, forkNum, blockNum, buffer); - const PageHeader page = (const PageHeader)buffer; - char *strOutput1 = (char *)palloc(MAXOUTPUTLEN * sizeof(char)); - rc = memset_s(strOutput1, SHORTOUTPUTLEN, 0, SHORTOUTPUTLEN); + char *buffer = (char *)palloc0(BLCKSZ);// 分配缓冲区内存，并将其初始化为0 + SMGR_READ_STATUS rdStatus = smgrread(smgr, forkNum, blockNum, buffer);// 从存储管理器中读取块数据到缓冲区中 + const PageHeader page = (const PageHeader)buffer;// 将缓冲区转换成页面头的结构体指针 + char *strOutput1 = (char *)palloc(MAXOUTPUTLEN * sizeof(char));// 分配临时输出字符串内存 + rc = memset_s(strOutput1, SHORTOUTPUTLEN, 0, SHORTOUTPUTLEN);// 初始化临时输出字符串为0 securec_check(rc, "\0", "\0"); - check_rdStatus(rdStatus, blockNum, page, strOutput1); - CheckWriteFile(fwrite(strOutput1, 1, strlen(strOutput1), outputfile), strlen(strOutput1), outputFilename); - pfree_ext(strOutput1); + check_rdStatus(rdStatus, blockNum, page, strOutput1);// 检查读取状态并将相应信息写入临时输出字符串 + CheckWriteFile(fwrite(strOutput1, 1, strlen(strOutput1), outputfile), strlen(strOutput1), outputFilename);// 将临时输出字符串写入输出文件 + pfree_ext(strOutput1);// 释放临时输出字符串的内存 + /* 根据关系类型选择解析物理块号或逻辑块号 */ if (strcmp(relation_type, "segment") == 0) { - SegPageLocation loc = seg_get_physical_location(smgr->smgr_rnode.node, forkNum, blockNum); - SegPageLocation endloc = seg_get_physical_location(smgr->smgr_rnode.node, forkNum, block_endpoint); - ParseOnePage(page, loc.blocknum, strOutput, relation_type, endloc.blocknum); /* physical blocknum */ + SegPageLocation loc = seg_get_physical_location(smgr->smgr_rnode.node, forkNum, blockNum); // 获取物理块号 + SegPageLocation endloc = seg_get_physical_location(smgr->smgr_rnode.node, forkNum, block_endpoint);// 获取结束块的物理块号 + // 解析物理块号 + ParseOnePage(page, loc.blocknum, strOutput, relation_type, endloc.blocknum); /* physical blocknum */ // 解析指定物理块号的页面 } else - ParseOnePage(page, blockNum, strOutput, relation_type, block_endpoint); + // 解析逻辑块号 + ParseOnePage(page, blockNum, strOutput, relation_type, block_endpoint);// 解析指定物理块号的页面 - CheckWriteFile(fwrite(strOutput, 1, strlen(strOutput), outputfile), strlen(strOutput), outputFilename); - pfree_ext(strOutput); - pfree_ext(buffer); - blockNum++; + CheckWriteFile(fwrite(strOutput, 1, strlen(strOutput), outputfile), strlen(strOutput), outputFilename);// 将解析结果写入输出文件 + pfree_ext(strOutput); // 释放输出字符串的内存 + pfree_ext(buffer);// 释放缓冲区的内存 + blockNum++;// 递增块号，进入下一次循环 } - CheckCloseFile(fclose(outputfile), outputFilename); - smgrcloseall(); - return outputFilename; + CheckCloseFile(fclose(outputfile), outputFilename);// 关闭输出文件 + smgrcloseall();// 关闭所有存储管理器 + return outputFilename;// 返回输出文件名 } - +//解析UHeap页面头部信息 static void ParseUHeapPageHeader(void *page, BlockNumber blkno, BlockNumber endBlk, char *output) { errno_t rc = EOK; @@ -621,14 +712,16 @@ static void ParseUHeapPageHeader(void *page, BlockNumber blkno, BlockNumber endB return; } - pageHeader = (UHeapPageHeader)page; + pageHeader = (UHeapPageHeader)page;// 将页面数据转换为UHeap页面头部结构体 + // 拼接页面块号和结束页面块号的信息到输出字符串 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "Page information of block %u/%u\n", blkno, endBlk); securec_check_ss(rc, "\0", "\0"); + // 获取页面LSN并格式化到输出字符串 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tpd_lsn: %X/%X\n", (uint32)(PageGetLSN(pageHeader) >> XIDTHIRTYTWO), (uint32)PageGetLSN(pageHeader)); securec_check_ss(rc, "\0", "\0"); - + // 如果满足CheckPageZeroCases条件，计算页面校验和并判断校验结果是否成功，并格式化到输出字符串 if (CheckPageZeroCases((PageHeader)pageHeader)) { uint16 checksum = pg_checksum_page((char *)pageHeader, (BlockNumber)blkno); chksumResult = (checksum == pageHeader->pd_checksum); @@ -637,7 +730,8 @@ static void ParseUHeapPageHeader(void *page, BlockNumber blkno, BlockNumber endB rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tpd_checksum: 0x%X, verify %s\n", pageHeader->pd_checksum, chksumResult ? "success" : "fail"); securec_check_ss(rc, "\0", "\0"); - + // 如果页面有自由行指针，将相应信息拼接到输出字符串中 + // 如果页面已满，将相应信息拼接到输出字符串中 if (UPageHasFreeLinePointers(pageHeader)) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "PAGE_HAS_FREE_LINES."); securec_check_ss(rc, "\0", "\0"); @@ -647,15 +741,17 @@ static void ParseUHeapPageHeader(void *page, BlockNumber blkno, BlockNumber endB rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "PAGE_FULL."); securec_check_ss(rc, "\0", "\0"); } - + // 将pd_lower、pd_upper、pd_special等信息格式化到输出字符串 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tpd_lower: %u, %s\n\tpd_upper: %u, %s\n\tpd_special: %u, size %u\n", pageHeader->pd_lower, PageIsEmpty(pageHeader) ? "empty" : "non-empty", pageHeader->pd_upper, PageIsNew(pageHeader) ? "new page" : "old page", pageHeader->pd_special, PageGetSpecialSize(pageHeader)); securec_check_ss(rc, "\0", "\0"); + // 将页面的大小和版本信息格式化到输出字符串 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tPage size & version: %u, %u\n", (uint16)PageGetPageSize(pageHeader), (uint16)PageGetPageLayoutVersion(pageHeader)); securec_check_ss(rc, "\0", "\0"); + // 将potential_freespace、td_count、pd_prune_xid、pd_xid_base和pd_multi_base等信息格式化到输出字符串 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tpotential_freespace: %u\n", pageHeader->potential_freespace); securec_check_ss(rc, "\0", "\0"); @@ -669,7 +765,7 @@ static void ParseUHeapPageHeader(void *page, BlockNumber blkno, BlockNumber endB securec_check_ss(rc, "\0", "\0"); return; } - +//解析UHeap页面（无锁堆存储页面）中的事务数据（Transaction Data，简称TD）信息，并将结果输出到指定的字符串中 static void ParseUHeapPageTDInfo(void *page, char *output) { TD *tdInfo = NULL; @@ -678,41 +774,42 @@ static void ParseUHeapPageTDInfo(void *page, char *output) UHeapPageHeader pageHeader = NULL; errno_t rc = EOK; - if (page == NULL || output == NULL) { + if (page == NULL || output == NULL) {// 输入参数检查 return; } - pageHeader = (UHeapPageHeader)page; - tdPtr = (UHeapPageTDData *)PageGetTDPointer(page); - tdCount = pageHeader->td_count; + pageHeader = (UHeapPageHeader)page;// 将页面数据转换为页头结构体 + tdPtr = (UHeapPageTDData *)PageGetTDPointer(page);// 获取TD指针 + tdCount = pageHeader->td_count;// 获取TD数量 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, - "\n\n\tUHeap Page TD information, nTDSlots = %u\n", tdCount); - securec_check_ss(rc, "\0", "\0"); + "\n\n\tUHeap Page TD information, nTDSlots = %u\n", tdCount);// 将TD数量信息格式化到输出字符串中 + securec_check_ss(rc, "\0", "\0");// 检查snprintf_s函数的返回值 - for (uint16 i = 0; i < tdCount; i++) { + for (uint16 i = 0; i < tdCount; i++) {// 遍历每个TD槽，将其信息格式化到输出字符串中 tdInfo = &(tdPtr->td_info[i]); rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\t TD Slot #%d, xid:%lu, urp:%lu\n", i + 1, tdInfo->xactid, tdInfo->undo_record_ptr); securec_check_ss(rc, "\0", "\0"); } - return; + return;// 返回函数 } - +//解析UHeapPageItem，并将其相关信息追加到输出缓冲区output中 static void ParseUHeapPageItem(Item item, UHeapTuple tuple, char *output) { if (item == NULL || tuple == NULL || output == NULL) { return; } - + // 将Item对象转换为UHeapDiskTuple类型 UHeapDiskTuple diskTuple = (UHeapDiskTuple)item; errno_t rc = EOK; tuple->disk_tuple = diskTuple; + // 将一些信息追加到output中，包括xid（事务ID）、td_id（事务ID标识符）、locker_td_id（锁定者事务ID标识符）等内容 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\t\t\txid:%lu, td:%d, locker_td %d\n", UHeapTupleGetRawXid(tuple), diskTuple->td_id, diskTuple->locker_td_id); securec_check_ss(rc, "\0", "\0"); rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\t\t\tFlag:%u", diskTuple->flag); securec_check_ss(rc, "\0", "\0"); - + // 根据diskTuple的flag进行判断，将相应的信息追加到output中 if (diskTuple->flag & UHEAP_HAS_NULL) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\t\tUHEAP_HASNULL"); securec_check_ss(rc, "\0", "\0"); @@ -758,7 +855,7 @@ static void ParseUHeapPageItem(Item item, UHeapTuple tuple, char *output) "\n\t\t\tSINGLE_LOCKER_XID_IS_SUBXACT "); securec_check_ss(rc, "\0", "\0"); } - + // 将一些其他信息追加到output中，如列数（Number of columns）和t_hoff等信息 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\t\tFlag2:"); securec_check_ss(rc, "\0", "\0"); rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\t\tNumber of columns:: %d\n", @@ -768,59 +865,68 @@ static void ParseUHeapPageItem(Item item, UHeapTuple tuple, char *output) securec_check_ss(rc, "\0", "\0"); return; } - +//解析UHeap页面数据，并将相关信息追加到输出缓冲区output中 static void ParseUHeapPageData(void *page, char *output) { - errno_t rc; - uint32 rowPtrCnt = 0; - uint32 storeCnt = 0; - uint32 unusedCnt = 0; - uint32 normalCnt = 0; - uint32 deadCnt = 0; - uint32 redirectCnt = 0; - Item item; - RowPtr *rowptr; - UHeapTupleData utuple; - UHeapPageHeader pageHeader = NULL; + errno_t rc;// 错误码 + uint32 rowPtrCnt = 0;// 行指针个数 + uint32 storeCnt = 0;// 存储行指针个数 + uint32 unusedCnt = 0;// 未使用行指针个数 + uint32 normalCnt = 0;// 正常行指针个数 + uint32 deadCnt = 0;// 已删除行指针个数 + uint32 redirectCnt = 0;// 重定向行指针个数 + Item item;// UHeapPageItem对象 + RowPtr *rowptr;// 行指针 + UHeapTupleData utuple;// UHeap元组数据 + UHeapPageHeader pageHeader = NULL;// UHeap页面头部结构体指针 if (page == NULL || output == NULL) { - return; + return;// 如果页或输出为空，直接返回 } - pageHeader = (UHeapPageHeader)page; + pageHeader = (UHeapPageHeader)page;// 强制类型转换为UHeap页面头部结构体指针 + // 计算行指针个数 if (pageHeader->pd_lower <= SizeOfUHeapPageHeaderData) { rowPtrCnt = 0; } else { rowPtrCnt = (pageHeader->pd_lower - (SizeOfUHeapPageHeaderData + SizeOfUHeapTDData(pageHeader))) / sizeof(RowPtr); } - + // 将UHeap tuple信息追加到输出缓冲区 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\tUHeap tuple information on this page\n"); securec_check_ss(rc, "\0", "\0"); - + // 遍历行指针 for (uint32 i = FirstOffsetNumber; i <= rowPtrCnt; i++) { rowptr = UPageGetRowPtr(pageHeader, i); + // 如果行指针被使用 if (RowPtrIsUsed(rowptr)) { + // 如果行指针有存储 if (RowPtrHasStorage(rowptr)) { storeCnt++; } - + // 如果行指针是正常的 if (RowPtrIsNormal(rowptr)) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\tTuple #%u is normal: length %u, offset %u\n", i, RowPtrGetLen(rowptr), RowPtrGetOffset(rowptr)); securec_check_ss(rc, "\0", "\0"); normalCnt++; + // 获取UHeapPageItem对象，并复制基本数据到utuple item = UPageGetRowData(pageHeader, rowptr); UHeapTupleCopyBaseFromPage(&utuple, pageHeader); + // 解析UHeapPageItem并将信息追加到输出缓冲区 ParseUHeapPageItem(item, &utuple, output); - } else if (RowPtrIsDead(rowptr)) { + } + // 如果行指针已删除 + else if (RowPtrIsDead(rowptr)) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\tTuple #%u is dead: length %u, offset %u", i, RowPtrGetLen(rowptr), RowPtrGetOffset(rowptr)); securec_check_ss(rc, "\0", "\0"); deadCnt++; - } else { + } + // 如果行指针是重定向的 + else { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\tTuple #%u is redirected: length %u, offset %u", i, RowPtrGetLen(rowptr), RowPtrGetOffset(rowptr)); @@ -833,126 +939,151 @@ static void ParseUHeapPageData(void *page, char *output) securec_check_ss(rc, "\0", "\0"); } } + // 将摘要信息追加到输出缓冲区 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tSummary (%u total): %u unused, %u normal, %u dead, %u redirect\n", rowPtrCnt, unusedCnt, normalCnt, deadCnt, redirectCnt); securec_check_ss(rc, "\0", "\0"); } - +//解析UHeap页面的特殊信息，并将相关信息追加到输出缓冲区中 static void ParseUHeapPageSpecialInfo(void *page, char *output) { - errno_t rc = 0; - + errno_t rc = 0;// 错误码 + // 如果page或output为空，直接返回 if (page == NULL || output == NULL) { return; } - + // 将"Special area information on this page"追加到输出缓冲区 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\tSpecial area information on this page\n"); securec_check_ss(rc, "\0", "\0"); + // 将"Not used currently."追加到输出缓冲区 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\tNot used currently.\n"); securec_check_ss(rc, "\0", "\0"); } - +/** + * @brief 解析UHeap页面，包括页面头部、td slot信息、数据项和特殊区域，并将相关信息追加到输出缓冲区中 + * + * @param page UHeap页面数据 + * @param blkno 当前页面的块号 + * @param endBlk 结束块号 + * @param output 输出缓冲区 + */ static void ParseUHeapPage(void *page, BlockNumber blkno, BlockNumber endBlk, char *output) { - uint16 pageHeaderSize = 0; - UHeapPageHeader pageHeader = NULL; - errno_t rc; - + uint16 pageHeaderSize = 0;// 页面头部大小 + UHeapPageHeader pageHeader = NULL;// UHeap页面头部结构体指针 + errno_t rc;// 错误码 + // 如果page或output为空，直接返回 if (page == NULL || output == NULL) { return; } /* Parse header of uheap page. */ + // 解析UHeap页面的头部信息 pageHeader = (UHeapPageHeader)page; /* Check invalidation of uheap page head. */ + // 检查UHeap页面头部的有效性 pageHeaderSize = GetPageHeaderSize(pageHeader); if (pageHeader->pd_lower < pageHeaderSize || pageHeader->pd_lower > pageHeader->pd_upper || pageHeader->pd_upper > pageHeader->pd_special || pageHeader->pd_special > BLCKSZ || pageHeader->pd_special != MAXALIGN(pageHeader->pd_special)) { + // 如果UHeap页面头部无效，将错误信息追加到输出缓冲区并返回 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "The page data is corrupted, corrupted page pointers: lower = %u, upper = %u, special = %u\n", pageHeader->pd_lower, pageHeader->pd_upper, pageHeader->pd_special); securec_check_ss(rc, "\0", "\0"); return; } - + // 解析UHeap页面头部信息 ParseUHeapPageHeader((void *)pageHeader, blkno, endBlk, output); + // 如果页面是新页面，直接返回 if (PageIsNew(pageHeader)) { return; } /* Parse td slot info of uheap page. */ + // 解析UHeap页面的td slot信息 ParseUHeapPageTDInfo(pageHeader, output); /* Parse items of uheap page. */ + // 解析UHeap页面的数据项 ParseUHeapPageData(pageHeader, output); /* Parse special area of uheap page. */ + // 解析UHeap页面的特殊区域 ParseUHeapPageSpecialInfo(pageHeader, output); } - +//解析索引页面的头部信息，并将解析结果追加到输出字符串中 static void ParseIndexPageHeader(void *page, int type, BlockNumber blkno, BlockNumber endBlk, char *output) { errno_t rc = 0; bool chksumResult = false; PageHeader pageHeader = NULL; - + // 检查输入参数是否为空 if (page == NULL || output == NULL) { return; } pageHeader = (PageHeader)page; + // 将索引页面类型、块号和结束块号信息追加到输出字符串中 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "%s index page information of block %u/%u\n", (type == BTREE_INDEX) ? "Btree" : "UBtree", blkno, endBlk); securec_check_ss(rc, "\0", "\0"); + // 将页面LSN信息追加到输出字符串中 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tpd_lsn: %X/%X\n", (uint32)(PageGetLSN((Page)pageHeader) >> XIDTHIRTYTWO), (uint32)PageGetLSN((Page)pageHeader)); securec_check_ss(rc, "\0", "\0"); - + // 校验页面校验和 if (CheckPageZeroCases(pageHeader)) { uint16 checksum = pg_checksum_page((char *)pageHeader, (BlockNumber)blkno); chksumResult = (checksum == pageHeader->pd_checksum); } - + // 将校验和验证结果追加到输出字符串中 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tpd_checksum: 0x%X, verify %s\n", pageHeader->pd_checksum, chksumResult ? "success" : "fail"); securec_check_ss(rc, "\0", "\0"); - + // 判断页面是否具有自由行指针，并将结果追加到输出字符串中 if (PageHasFreeLinePointers((Page)pageHeader)) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "PD_HAS_FREE_LINES."); securec_check_ss(rc, "\0", "\0"); } + // 判断页面是否已满，并将结果追加到输出字符串中 if (PageIsFull(pageHeader)) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "PD_PAGE_FULL."); securec_check_ss(rc, "\0", "\0"); } + // 判断页面是否全部可见，并将结果追加到输出字符串中 if (PageIsAllVisible(pageHeader)) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "PD_ALL_VISIBLE."); securec_check_ss(rc, "\0", "\0"); } + // 判断页面是否压缩，并将结果追加到输出字符串中 if (PageIsCompressed(pageHeader)) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "PD_COMPRESSED_PAGE."); securec_check_ss(rc, "\0", "\0"); } + // 判断页面是否逻辑页面，并将结果追加到输出字符串中 if (PageIsLogical(pageHeader)) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "PD_LOGICAL_PAGE."); securec_check_ss(rc, "\0", "\0"); } + // 判断页面是否加密，并将结果追加到输出字符串中 if (PageIsEncrypt(pageHeader)) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "PD_ENCRYPT_PAGE."); securec_check_ss(rc, "\0", "\0"); } - + // 将pd_lower、pd_upper、pd_special等字段的值追加到输出字符串中 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tpd_lower: %u, %s\n\tpd_upper: %u, %s\n\tpd_special: %u, size %u\n", pageHeader->pd_lower, PageIsEmpty((Page)pageHeader) ? "empty" : "non-empty", pageHeader->pd_upper, PageIsNew((Page)pageHeader) ? "new page" : "old page", pageHeader->pd_special, PageGetSpecialSize((Page)pageHeader)); securec_check_ss(rc, "\0", "\0"); + // 将页面大小和版本号追加到输出字符串中 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tPage size & version: %u, %u\n", (uint16)PageGetPageSize(page), (uint16)PageGetPageLayoutVersion(page)); securec_check_ss(rc, "\0", "\0"); + // 将pd_prune_xid、pd_xid_base和pd_multi_base的值追加到输出字符串中 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tpd_prune_xid: %lu\n", pageHeader->pd_prune_xid + ((HeapPageHeader)(pageHeader))->pd_xid_base); securec_check_ss(rc, "\0", "\0"); @@ -963,9 +1094,17 @@ static void ParseIndexPageHeader(void *page, int type, BlockNumber blkno, BlockN return; } - +/** + * 解析索引页面中的条目（Item） + * + * @param item 条目指针 + * @param type 索引类型（UB树索引或B树索引） + * @param len 条目长度 + * @param output 输出字符串 + */ static void ParseIndexPageItem(Item item, int type, uint32 len, char *output) { + // 检查输入参数是否为空 if (item == NULL || output == NULL) { return; } @@ -974,6 +1113,7 @@ static void ParseIndexPageItem(Item item, int type, uint32 len, char *output) IndexTuple itup = (IndexTuple)item; bool hasnull = (itup->t_info & INDEX_NULL_MASK); unsigned int tuplen = (itup->t_info & INDEX_SIZE_MASK); + // 验证条目长度是否与给定长度相等 if (tuplen != len) { if (type == UBTREE_INDEX) { UstoreIndexXid uxid = (UstoreIndexXid)UstoreIndexTupleGetXid(itup); @@ -991,12 +1131,12 @@ static void ParseIndexPageItem(Item item, int type, uint32 len, char *output) rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\t\t\tLength: %u", tuplen); securec_check_ss(rc, "\0", "\0"); - + // 判断条目是否具有可变宽度的属性 if (itup->t_info & INDEX_VAR_MASK) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, ", has var-width attrs"); securec_check_ss(rc, "\0", "\0"); } - + // 判断条目是否具有空值 if (hasnull) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, ", has nulls "); securec_check_ss(rc, "\0", "\0"); @@ -1006,39 +1146,48 @@ static void ParseIndexPageItem(Item item, int type, uint32 len, char *output) securec_check_ss(rc, "\0", "\0"); return; } - +/** + * 解析索引页面的特殊信息并将结果输出到字符数组output中。 + * + * @param page 指向页面的指针 + * @param type 页面类型 + * @param output 输出结果的字符数组 + */ static void ParseIndexPageSpecialInfo(void *page, int type, char *output) { errno_t rc = 0; BTPageOpaqueInternal opaque = NULL; UBTPageOpaqueInternal uopaque = NULL; PageHeader pageHeader = NULL; - + // 参数合法性检查 if (page == NULL || output == NULL || type >= INDEX_BOTT) { return; } pageHeader = (PageHeader)page; opaque = (BTPageOpaqueInternal)PageGetSpecialPointer((Page)pageHeader); + // 根据特殊大小判断是否获取额外的特殊指针 if (PageGetSpecialSize((Page)pageHeader) > MAXALIGN(sizeof(BTPageOpaqueData))) { uopaque = (UBTPageOpaqueInternal)opaque; } - + // 添加索引类型信息到输出 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n%s index special information:\n", (type == BTREE_INDEX) ? "BTree" : "UBTree"); securec_check_ss(rc, "\0", "\0"); + // 添加左兄弟和右兄弟的信息到输出 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tbtree left sibling: %u\n", opaque->btpo_prev); securec_check_ss(rc, "\0", "\0"); rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tbtree right sibling: %u\n", opaque->btpo_next); securec_check_ss(rc, "\0", "\0"); - + // 如果页面未被删除，则添加B树级别信息到输出 if (!P_ISDELETED(opaque)) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tbtree tree level: %u\n", opaque->btpo.level); securec_check_ss(rc, "\0", "\0"); } else { + // 如果存在额外的特殊指针，则添加已删除页面的事务ID信息到输出 if (uopaque) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tnext txid (deleted): %lu\n", ((UBTPageOpaque)uopaque)->xact); @@ -1049,7 +1198,7 @@ static void ParseIndexPageSpecialInfo(void *page, int type, char *output) securec_check_ss(rc, "\0", "\0"); } } - + // 添加页面标志信息到输出 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tbtree flag: "); securec_check_ss(rc, "\0", "\0"); @@ -1078,11 +1227,11 @@ static void ParseIndexPageSpecialInfo(void *page, int type, char *output) } rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n"); securec_check_ss(rc, "\0", "\0"); - + // 添加循环ID信息到输出 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tbtree cycle ID: %u\n ", opaque->btpo_cycleid); securec_check_ss(rc, "\0", "\0"); - + // 如果存在额外的特殊指针，则添加UB树的活跃元组数量信息到输出 if (uopaque) { rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\tubtree active tuples: %d\n", uopaque->activeTupleCount); @@ -1091,16 +1240,17 @@ static void ParseIndexPageSpecialInfo(void *page, int type, char *output) return; } - +//解析索引页面数据，并将解析结果输出到字符数组中 static void ParseIndexPageData(void *page, int type, char *output) { errno_t rc; - uint32 rowPtrCnt = 0; - uint32 storeCnt = 0; - uint32 notusedCnt = 0; - uint32 normalCnt = 0; - uint32 deadCnt = 0; - uint32 redirectCnt = 0; + uint32 rowPtrCnt = 0;// 页面中的最大偏移量 + uint32 storeCnt = 0;// 存储项计数 + uint32 notusedCnt = 0;// 未使用项计数 + uint32 normalCnt = 0;// 正常项计数 + uint32 deadCnt = 0;// 已删除项计数 + uint32 redirectCnt = 0;// 重定向项计数 + Item item; ItemId lp; PageHeader pageHeader = NULL; @@ -1110,40 +1260,40 @@ static void ParseIndexPageData(void *page, int type, char *output) } pageHeader = (PageHeader)page; - rowPtrCnt = PageGetMaxOffsetNumber((Page)pageHeader); + rowPtrCnt = PageGetMaxOffsetNumber((Page)pageHeader);// 获取页面的最大偏移量 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\tIndex tuple information on this page\n"); securec_check_ss(rc, "\0", "\0"); - for (uint32 i = FirstOffsetNumber; i <= rowPtrCnt; i++) { - lp = PageGetItemId((Page)pageHeader, i); - if (ItemIdIsUsed(lp)) { - if (ItemIdHasStorage(lp)) { - storeCnt++; + for (uint32 i = FirstOffsetNumber; i <= rowPtrCnt; i++) {// 遍历每个偏移量 + lp = PageGetItemId((Page)pageHeader, i);// 获取ItemId + if (ItemIdIsUsed(lp)) {// 判断ItemId是否被使用 + if (ItemIdHasStorage(lp)) {// 判断ItemId是否具有存储 + storeCnt++;// 存储项计数加1 } - if (ItemIdIsNormal(lp) || (IndexItemIdIsFrozen(lp))) { + if (ItemIdIsNormal(lp) || (IndexItemIdIsFrozen(lp))) {// 判断ItemId是否正常或被冻结 rc = ItemIdIsNormal(lp) ? (snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\tTuple #%u is normal: length %u, offset %u\n", i, ItemIdGetLength(lp), ItemIdGetOffset(lp))) : (snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\tTuple #%u is frozen: length %u, offset %u\n", i, ItemIdGetLength(lp), ItemIdGetOffset(lp))); securec_check_ss(rc, "\0", "\0"); - normalCnt++; + normalCnt++;// 正常项计数加1 item = PageGetItem((Page)pageHeader, lp); - ParseIndexPageItem(item, type, ItemIdGetLength(lp), output); - } else if (ItemIdIsDead(lp)) { + ParseIndexPageItem(item, type, ItemIdGetLength(lp), output);// 解析索引元组的具体项并添加到输出中 + } else if (ItemIdIsDead(lp)) {// 判断ItemId是否已删除 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\tTuple #%u is dead: length %u, offset %u", i, ItemIdGetLength(lp), ItemIdGetOffset(lp)); securec_check_ss(rc, "\0", "\0"); - deadCnt++; - } else { + deadCnt++;// 已删除项计数加1 + } else {// ItemId是重定向的 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\tTuple #%u is redirected: length %u, offset %u", i, ItemIdGetLength(lp), ItemIdGetOffset(lp)); securec_check_ss(rc, "\0", "\0"); - redirectCnt++; + redirectCnt++;// 重定向项计数加1 } - } else { - notusedCnt++; + } else {// ItemId未使用 + notusedCnt++;// 未使用项计数加1 rc = snprintf_s(output + (int)strlen(output), MAXOUTPUTLEN, MAXOUTPUTLEN, "\n\t\tTuple #%u is unused\n", i); securec_check_ss(rc, "\0", "\0"); } @@ -1155,7 +1305,16 @@ static void ParseIndexPageData(void *page, int type, char *output) securec_check_ss(rc, "\0", "\0"); return; } - +/** + * @brief 解析索引页并输出解析结果 + * + * @param page 索引页的指针 + * @param type 索引类型 + * @param blkno 块号 + * @param endBlk 结束块号 + * @param output 输出字符数组 + */ +//解析给定的索引页，并将解析结果输出到字符数组中 static void ParseIndexPage(void *page, int type, BlockNumber blkno, BlockNumber endBlk, char *output) { PageHeader pageHeader = NULL; @@ -1168,6 +1327,7 @@ static void ParseIndexPage(void *page, int type, BlockNumber blkno, BlockNumber pageHeader = (PageHeader)page; headersize = GetPageHeaderSize(pageHeader); + // 检查页面指针是否损坏 if (pageHeader->pd_lower < headersize || pageHeader->pd_lower > pageHeader->pd_upper || pageHeader->pd_upper > pageHeader->pd_special || pageHeader->pd_special > BLCKSZ || pageHeader->pd_special != MAXALIGN(pageHeader->pd_special)) { @@ -1177,34 +1337,48 @@ static void ParseIndexPage(void *page, int type, BlockNumber blkno, BlockNumber securec_check_ss(rc, "\0", "\0"); return; } - + // 解析页头信息 ParseIndexPageHeader((void *)page, type, blkno, endBlk, output); + // 如果是新页面，则结束解析 if (PageIsNew(pageHeader)) { return; } - + // 解析页面数据 ParseIndexPageData((void *)page, type, output); + // 解析特殊信息 ParseIndexPageSpecialInfo((void *)page, type, output); return; } - +/** + * @brief 解析给定路径下的数据库页面，并返回解析结果。 + * + * @param[in] fcinfo PG_FUNCTION_ARGS类型的函数参数上下文，包含所有输入参数和返回值信息。 + * @return 返回解析结果的文本形式。 + * + * @note 该函数用于解析指定路径下的页面，通常用于调试和故障排除。 + */ Datum gs_parse_page_bypath(PG_FUNCTION_ARGS) { /* check user's right */ + /* 检查用户权限 */ const char fName[MAXFNAMELEN] = "gs_parse_page_bypath"; CheckUser(fName); /* read in parameters */ - char *path = text_to_cstring(PG_GETARG_TEXT_P(0)); - int64 blocknum = PG_GETARG_INT64(1); - char *relation_type = text_to_cstring(PG_GETARG_TEXT_P(2)); - bool read_memory = PG_GETARG_BOOL(3); + /* 读取输入参数 */ + char *path = text_to_cstring(PG_GETARG_TEXT_P(0));// 页面路径 + int64 blocknum = PG_GETARG_INT64(1);// 块号 + char *relation_type = text_to_cstring(PG_GETARG_TEXT_P(2));// 关系类型 + bool read_memory = PG_GETARG_BOOL(3);// 是否从内存中读取页面 /* In order to avoid querying the shared buffer and applying LW locks, blocking the business. */ /* In the case of finding all pages, force to check disk */ + /* 如果需要查找所有页面，则强制从磁盘读取页面，以避免对共享缓冲区进行查询并应用LW锁，从而阻塞业务。 */ if (blocknum == -1) { read_memory = false; } + /* 调用ParsePage函数解析给定路径下的页面，并将解析结果写入文本文件中。 */ char *outputFilename = ParsePage(path, blocknum, relation_type, read_memory); + /* 将解析结果读入文本形式，并作为函数返回值返回。 */ PG_RETURN_TEXT_P(cstring_to_text(outputFilename)); } -- 2.34.1 From 9e251c113db48c37ee3082cd94d1d250ac83cc18 Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:41:52 +0800 Subject: [PATCH 20/48] Update CMakeLists.txt --- src/gausskernel/storage/remote/CMakeLists.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/storage/remote/CMakeLists.txt b/src/gausskernel/storage/remote/CMakeLists.txt index fada81877..5cefff42f 100755 --- a/src/gausskernel/storage/remote/CMakeLists.txt +++ b/src/gausskernel/storage/remote/CMakeLists.txt @@ -1,6 +1,10 @@ #This is the main CMAKE for build bin. +#用于构建bin库的CMake文件。它包含了一些变量和指令，用于定义构建过程中所需的源代码、包含路径和编译选项等。 + +#将当前目录下的源文件添加到变量TGT_remote_SRC中 AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_remote_SRC) +#设置目标库需要包含的头文件路径 set(TGT_remote_INC ${PROJECT_OPENGS_DIR}/contrib/log_fdw ${PROJECT_TRUNK_DIR}/distribute/bin/gds @@ -15,8 +19,11 @@ set(TGT_remote_INC ${ZLIB_INCLUDE_PATH} ) +#设置目标库的宏选项 set(remote_DEF_OPTIONS ${MACRO_OPTIONS}) +#设置目标库的编译选项，包括优化选项、操作系统选项、保护选项、警告选项、二进制安全选项和检查选项 set(remote_COMPILE_OPTIONS ${OPTIMIZE_OPTIONS} ${OS_OPTIONS} ${PROTECT_OPTIONS} ${WARNING_OPTIONS} ${BIN_SECURE_OPTIONS} ${CHECK_OPTIONS}) +#设置目标库的链接选项 set(remote_LINK_OPTIONS ${BIN_LINK_OPTIONS}) +#添加静态库目标，并指定源文件、包含路径和编译链接选项等 add_static_objtarget(gausskernel_storage_remote TGT_remote_SRC TGT_remote_INC "${remote_DEF_OPTIONS}" "${remote_COMPILE_OPTIONS}" "${remote_LINK_OPTIONS}") - -- 2.34.1 From e0c15d35aa298c54a80717ac0d757de88f8c175f Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:42:50 +0800 Subject: [PATCH 21/48] Update Makefile --- src/gausskernel/storage/remote/Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/storage/remote/Makefile b/src/gausskernel/storage/remote/Makefile index d06f4ae38..26f617739 100644 --- a/src/gausskernel/storage/remote/Makefile +++ b/src/gausskernel/storage/remote/Makefile @@ -21,10 +21,17 @@ # # --------------------------------------------------------------------------------------- + +#通过该Makefile文件，可以在openGauss数据库的存储/远程组件目录中，进行编译、链接和生成目标二进制文件等操作 + +#子目录和顶层构建目录：指明该Makefile所在的子目录路径和顶层构建目录路径。 subdir = src/gausskernel/storage/remote top_builddir = ../../../.. +#包含全局Makefile文件：包含顶层构建目录下的Makefile.global文件，该文件定义了全局的编译选项和规则 include $(top_builddir)/src/Makefile.global +#条件判断：根据当前执行的Make命令来判断是否需要包含依赖文件（DEPEND） +#如果当前执行的是"clean"或"distclean"命令，则不包含依赖文件 ifneq "$(MAKECMDGOALS)" "clean" ifneq "$(MAKECMDGOALS)" "distclean" ifneq "$(shell which g++ |grep hutaf_llt |wc -l)" "1" @@ -32,6 +39,7 @@ ifneq "$(MAKECMDGOALS)" "clean" endif endif endif +#目标对象（OBJS）：定义要编译生成的目标对象文件列表 OBJS = remote_adapter.o remote_read.o - +#包含公共Makefile文件：包含openGauss数据库中的common.mk文件，该文件定义了具体的编译规则和链接选项等 include $(top_srcdir)/src/gausskernel/common.mk -- 2.34.1 From c122078fe8b10d83ea9c337ea61bc295e88d682d Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:43:44 +0800 Subject: [PATCH 22/48] Update remote_adapter.cpp --- .../storage/remote/remote_adapter.cpp | 324 ++++++++++++++---- 1 file changed, 258 insertions(+), 66 deletions(-) diff --git a/src/gausskernel/storage/remote/remote_adapter.cpp b/src/gausskernel/storage/remote/remote_adapter.cpp index 0adcf7d73..8f78a652e 100755 --- a/src/gausskernel/storage/remote/remote_adapter.cpp +++ b/src/gausskernel/storage/remote/remote_adapter.cpp @@ -54,17 +54,24 @@ int ReadCOrCsnFileForRemote(RelFileNode rnode, bytea** fileData); * @Return: remote read error code * @See also: */ +//用于等待指定LSN（日志序列号）被回放到本地 +// 常见的应用场景是在流复制（Streaming Replication）中，等待从节点接收并成功应用指定的事务日志 +// primary_insert_lsn：远程请求的LSN（日志序列号） int XLogWaitForReplay(uint64 primary_insert_lsn, int timeout = DEFAULT_WAIT_TIMES) { int wait_times = 0; /* local replayed lsn */ + //获取本地正在回放的LSN XLogRecPtr standby_replay_lsn = GetXLogReplayRecPtr(NULL, NULL); /* if primary_insert_lsn > standby_replay_lsn then need wait */ + //循环判断如果primary_insert_lsn大于standby_replay_lsn，则需要继续等待 while (!XLByteLE(primary_insert_lsn, standby_replay_lsn)) { /* if sleep to much times */ + //判断等待次数是否超过超时时间 if (wait_times >= timeout) { + //如果超过则返回错误并打印相应日志信息 ereport(LOG, (errmodule(MOD_REMOTE), errmsg("replay slow. requre lsn %X/%X, replayed lsn %X/%X", @@ -72,41 +79,50 @@ int XLogWaitForReplay(uint64 primary_insert_lsn, int timeout = DEFAULT_WAIT_TIME (uint32)primary_insert_lsn, (uint32)(standby_replay_lsn >> 32), (uint32)standby_replay_lsn))); + //等待超时，则返回REMOTE_READ_NEED_WAIT（远程读取需要等待） return REMOTE_READ_NEED_WAIT; } /* sleep 1s */ + //进行1秒的延迟 pg_usleep(1000000L); + //递增等待次数 ++wait_times; /* get current replay lsn again */ + //再次获取当前的回放LSN，更新 (void)GetXLogReplayRecPtr(NULL, &standby_replay_lsn); } - + //直到primary_insert_lsn小于等于standby_replay_lsn，表示LSN已经回放完成 + //成功回放，则返回REMOTE_READ_OK（远程读取完成） return REMOTE_READ_OK; } /* * Read block from buffer from primary, returning it as bytea */ +//实现了从主节点读取缓冲区中的数据块，并将其作为bytea类型返回 Datum gs_read_block_from_remote(PG_FUNCTION_ARGS) { - uint32 spcNode; - uint32 dbNode; - uint32 relNode; - int16 bucketNode; - int32 forkNum; - uint64 blockNum; - uint32 blockSize; - uint64 lsn; - bool isForCU = false; + uint32 spcNode; //文件空间节点号 + uint32 dbNode; //数据库节点号 + uint32 relNode; //关系节点号 + int16 bucketNode; //哈希桶节点号 + int32 forkNum; //分叉号 + uint64 blockNum; //块号 + uint32 blockSize; //块大小 + uint64 lsn; //日志序列号 + bool isForCU = false; //是否是针对CU（压缩单元）块的请求 bytea* result = NULL; int timeout = 0; + //检查当前用户是否为超级用户 if (GetUserId() != BOOTSTRAP_SUPERUSERID) { + //如果不是则抛出异常 ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be initial account to read files")))); } /* handle optional arguments */ + //解析传入的参数 spcNode = PG_GETARG_UINT32(0); dbNode = PG_GETARG_UINT32(1); relNode = PG_GETARG_UINT32(2); @@ -118,6 +134,7 @@ Datum gs_read_block_from_remote(PG_FUNCTION_ARGS) isForCU = PG_GETARG_BOOL(8); timeout = PG_GETARG_INT32(9); + //创建修复块的键（RepairBlockKey）对象，并填充相关属性 RepairBlockKey key; key.relfilenode.spcNode = spcNode; key.relfilenode.dbNode = dbNode; @@ -128,16 +145,21 @@ Datum gs_read_block_from_remote(PG_FUNCTION_ARGS) key.blocknum = blockNum; /* get block from local buffer */ + // 从本地缓冲区读取块 if (isForCU) { + // 从本地缓冲区读取 CU 块 /* if request to read CU block, we use forkNum column to replace colid. */ (void)StandbyReadCUforPrimary(key, blockNum, blockSize, lsn, timeout, &result); } else { + //从本地缓冲区读取普通页面块 (void)StandbyReadPageforPrimary(key, blockSize, lsn, &result, timeout, NULL); } if (NULL != result) { + //成功获取到数据块，则将其转换为bytea类型并返回 PG_RETURN_BYTEA_P(result); } else { + //未获取到数据块，则返回NULL PG_RETURN_NULL(); } } @@ -145,41 +167,50 @@ Datum gs_read_block_from_remote(PG_FUNCTION_ARGS) /* * Read block from buffer from primary, returning it as bytea */ +//用于从主节点的压缩块缓冲区中读取数据，并将其以bytea类型返回 Datum gs_read_block_from_remote_compress(PG_FUNCTION_ARGS) { - RepairBlockKey key; - uint32 blockSize; - uint64 lsn; - int timeout = 0; - bool isForCU = false; - bytea* result = NULL; + RepairBlockKey key; // 修复块的关键信息 + uint32 blockSize; // 块大小 + uint64 lsn; // 日志序列号 + int timeout = 0; // 超时时间 + bool isForCU = false; // 是否请求读取CU块 + bytea* result = NULL; // 返回的块数据 + // 检查当前用户是否为超级用户 if (GetUserId() != BOOTSTRAP_SUPERUSERID) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be initial account to read files")))); } /* handle optional arguments */ - key.relfilenode.spcNode = PG_GETARG_UINT32(0); - key.relfilenode.dbNode = PG_GETARG_UINT32(1); - key.relfilenode.relNode = PG_GETARG_UINT32(2); - key.relfilenode.bucketNode = PG_GETARG_INT16(3); - key.relfilenode.opt = PG_GETARG_UINT16(4); - key.forknum = PG_GETARG_INT32(5); - key.blocknum = (uint64)PG_GETARG_TRANSACTIONID(6); - blockSize = PG_GETARG_UINT32(7); - lsn = (uint64)PG_GETARG_TRANSACTIONID(8); - isForCU = PG_GETARG_BOOL(9); - timeout = PG_GETARG_INT32(10); + // 获取传入的参数值 + key.relfilenode.spcNode = PG_GETARG_UINT32(0); // 表空间OID + key.relfilenode.dbNode = PG_GETARG_UINT32(1); // 数据库OID + key.relfilenode.relNode = PG_GETARG_UINT32(2); // 关系OID + key.relfilenode.bucketNode = PG_GETARG_INT16(3); // 桶OID + key.relfilenode.opt = PG_GETARG_UINT16(4); // 扩展选项 + key.forknum = PG_GETARG_INT32(5); // 分支号 + key.blocknum = (uint64)PG_GETARG_TRANSACTIONID(6); // 块号 + blockSize = PG_GETARG_UINT32(7); // 块大小 + lsn = (uint64)PG_GETARG_TRANSACTIONID(8); // 日志序列号 + isForCU = PG_GETARG_BOOL(9); // 是否请求读取CU块 + timeout = PG_GETARG_INT32(10); // 超时时间 + /* get block from local buffer */ if (isForCU) { + // 从本地缓冲区读取 CU 块 /* if request to read CU block, we use forkNum column to replace colid. */ (void)StandbyReadCUforPrimary(key, key.blocknum, blockSize, lsn, timeout, &result); } else { + // 从本地缓冲区读取页块 (void)StandbyReadPageforPrimary(key, blockSize, lsn, &result, timeout, NULL); } + // 检查块是否成功获取 if (NULL != result) { + // 返回块数据 PG_RETURN_BYTEA_P(result); } else { + // 返回空值 PG_RETURN_NULL(); } } @@ -198,54 +229,71 @@ Datum gs_read_block_from_remote_compress(PG_FUNCTION_ARGS) * @Return: remote read error code * @See also: */ +//用于从主节点读取某个文件中指定块号的CU数据块 int StandbyReadCUforPrimary(RepairBlockKey key, uint64 offset, int32 size, uint64 lsn, int32 timeout, bytea** cudata) { + // 确保输出参数cudata存在 Assert(cudata); - + // 初始化返回值 int ret_code = REMOTE_READ_OK; /* wait request lsn for replay */ + //等待请求的LSN回放完成 + // 检查是否处于恢复进程中 if (RecoveryInProgress()) { + // 等待指定LSN的日志重放，超时时间为timeout ret_code = XLogWaitForReplay(lsn, timeout); if (ret_code != REMOTE_READ_OK) { + // 如果错误码不等于REMOTE_READ_OK，则发生了错误，记录错误日志并返回错误码 ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("could not redo request lsn."))); return ret_code; } } + // 使用传入的参数构造关系文件节点RelFileNode对象 RelFileNode relfilenode {key.relfilenode.spcNode, key.relfilenode.dbNode, key.relfilenode.relNode, InvalidBktId}; - + // 在匿名块中执行以下语句 { /* read from disk */ + //创建CFileNode对象，用于标识具体的文件和文件块 CFileNode cfilenode(relfilenode, key.forknum, MAIN_FORKNUM); + // 创建CUStorage对象，用于管理和访问CU数据 CUStorage* custorage = New(CurrentMemoryContext) CUStorage(cfilenode); + // 创建CU对象，用于加载和处理CU数据 CU* cu = New(CurrentMemoryContext) CU(); cu->m_inCUCache = false; + // 从磁盘加载指定偏移量和大小的CU块到内存中的CU对象中 custorage->LoadCU(cu, offset, size, false, false); /* check crc */ + // 检查CU数据的CRC校验码是否正确 if (ret_code == REMOTE_READ_OK) { if (!cu->CheckCrc()) { + // 如果CRC校验失败，则记录错误日志并更新返回值为REMOTE_READ_CRC_ERROR ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("check CU crc error."))); ret_code = REMOTE_READ_CRC_ERROR; } else { + // 复制CU数据到返回值缓冲区 bytea* buf = (bytea*)palloc0(VARHDRSZ + size); SET_VARSIZE(buf, size + VARHDRSZ); errno_t rc = memcpy_s(VARDATA(buf), size, cu->m_compressedLoadBuf, size); if (rc != EOK) { + // 如果复制过程中发生错误，则记录错误日志并更新返回值为REMOTE_READ_MEMCPY_ERROR ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("memcpy_s error, retcode=%d", rc))); ret_code = REMOTE_READ_MEMCPY_ERROR; } + // 将buf对象赋值给输出参数cudata *cudata = buf; + // 释放内存，删除custorage和cu对象 DELETE_EX(custorage); DELETE_EX(cu); } } } - + // 返回错误代码 return ret_code; } @@ -263,18 +311,24 @@ int StandbyReadCUforPrimary(RepairBlockKey key, uint64 offset, int32 size, uint6 * @Return: remote read error code * @See also: */ +//读取主库的数据页，并将数据存储在pagedata中 int StandbyReadPageforPrimary(RepairBlockKey key, uint32 blocksize, uint64 lsn, bytea** pagedata, int timeout, const XLogPhyBlock *pblk) { + //通过断言语句，检查pagedata是否为空指针 Assert(pagedata); + //传入的数据块大小是否与定义的常量BLCKSZ相同 if (unlikely(blocksize != BLCKSZ)) + //不同则返回错误码 return REMOTE_READ_BLCKSZ_NOT_SAME; int ret_code = REMOTE_READ_OK; /* wait request lsn for replay */ + //如果系统正在恢复中 if (RecoveryInProgress()) { + //等待请求的LSN（日志序列号）进行重放 ret_code = XLogWaitForReplay(lsn, timeout); if (ret_code != REMOTE_READ_OK) { ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("could not redo request lsn."))); @@ -282,14 +336,20 @@ int StandbyReadPageforPrimary(RepairBlockKey key, uint32 blocksize, uint64 lsn, } } + //该对象用于标识要读取数据页的关系文件在数据库中的位置 RelFileNode relfilenode{key.relfilenode.spcNode, key.relfilenode.dbNode, key.relfilenode.relNode, key.relfilenode.bucketNode, key.relfilenode.opt}; - + //如果指针 pblk 不为空，则表示当前正在读取的是备库上的数据块 + // 这时需要验证备库中的数据块位置与主库是否一致 if (NULL != pblk) { + //获取分段物理位置信息 SegPageLocation loc = seg_get_physical_location(relfilenode, key.forknum, key.blocknum); + //将loc.extent_size（扩展大小）转换为一个无符号8位整数 uint8 standby_relNode = (uint8) EXTENT_SIZE_TO_TYPE(loc.extent_size); BlockNumber standby_block = loc.blocknum; + //验证备库中的关系文件和数据块的位置与主库是否一致 if (standby_relNode != pblk->relNode || standby_block != pblk->block) { + //如果不一致，将报告错误并返回相应的错误码 ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("Standby page file is invalid! Standby relnode is %u, " "master is %u; Standby block is %u, master is %u.", @@ -297,73 +357,91 @@ int StandbyReadPageforPrimary(RepairBlockKey key, uint32 blocksize, uint64 lsn, return REMOTE_READ_BLCKSZ_NOT_SAME; } } - + //如果 pblk 为空，则表示当前正在读取主库上的数据块，不需要进行验证 + //分配了一个大小为 BLCKSZ + VARHDRSZ 的内存空间，并设置 pageData 的头部为该空间的有效大小 bytea* pageData = (bytea*)palloc(BLCKSZ + VARHDRSZ); SET_VARSIZE(pageData, BLCKSZ + VARHDRSZ); + //判断关系文件节点是否属于分段物理模型 if (IsSegmentPhysicalRelNode(relfilenode)) { Buffer buffer = InvalidBuffer; + //如果是，则使用 spc_open 函数打开对应的分段空间，并获取关系文件的数据块总数 SegSpace *spc = spc_open(relfilenode.spcNode, relfilenode.dbNode, false, false); BlockNumber spc_nblocks = spc_size(spc, relfilenode.relNode, key.forknum); - + //查给定的数据块号 key.blocknum 是否小于数据块总数 spc_nblocks if (key.blocknum < spc_nblocks) { + //读取磁盘上对应数据块的内容 buffer = ReadBufferFast(spc, relfilenode, key.forknum, key.blocknum, RBM_FOR_REMOTE); } - + //检查 Buffer 是否有效（即是否成功读取到了数据块） if (BufferIsValid(buffer)) { + //如果有效，则通过 LockBuffer 将该 Buffer 锁定，并获取其对应的 Block LockBuffer(buffer, BUFFER_LOCK_SHARE); Block block = BufferGetBlock(buffer); - + //将 Block 的内容拷贝到 pageData 中 errno_t rc = memcpy_s(VARDATA(pageData), BLCKSZ, block, BLCKSZ); + //如果 memcpy_s 函数执行失败 if (rc != EOK) { + //释放之前分配的内存空间 pageData，通过 ereport 报告错误 pfree(pageData); ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("memcpy_s error, retcode=%d", rc))); + //将返回值 ret_code 设置为 REMOTE_READ_MEMCPY_ERROR ret_code = REMOTE_READ_MEMCPY_ERROR; } - + //解锁并释放 Buffer LockBuffer(buffer, BUFFER_LOCK_UNLOCK); ReleaseBuffer(buffer); } else { + //没有成功读取到数据块，则将 ret_code 设置为 REMOTE_READ_SIZE_ERROR ret_code = REMOTE_READ_SIZE_ERROR; } - } else { + } else { //关系文件节点不是分段物理关系 bool hit = false; - + //读取指定的数据块 /* read page, if PageIsVerified failed will long jump to PG_CATCH() */ Buffer buf = ReadBufferForRemote(relfilenode, key.forknum, key.blocknum, RBM_FOR_REMOTE, NULL, &hit, pblk); - + //如果读取成功，则获取对应的 Buffer if (BufferIsInvalid(buf)) { + //则通过 ereport 报告错误并设置返回值 ret_code 为 REMOTE_READ_BLCKSZ_NOT_SAME ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("standby page buffer is invalid!"))); return REMOTE_READ_BLCKSZ_NOT_SAME; } + //通过 LockBuffer 锁定该 Buffer，并获取其对应的 Block LockBuffer(buf, BUFFER_LOCK_SHARE); Block block = BufferGetBlock(buf); - + //将 Block 的内容拷贝到 pageData 中 errno_t rc = memcpy_s(VARDATA(pageData), BLCKSZ, block, BLCKSZ); + //如果 memcpy_s 函数执行失败 if (rc != EOK) { + //释放之前分配的内存空间 pageData，通过 ereport 报告错误 pfree(pageData); ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("memcpy_s error, retcode=%d", rc))); + //将返回值 ret_code 设置为 REMOTE_READ_MEMCPY_ERROR ret_code = REMOTE_READ_MEMCPY_ERROR; } - + //解锁并释放 Buffer LockBuffer(buf, BUFFER_LOCK_UNLOCK); ReleaseBuffer(buf); } + //读取数据块成功的情况下执行 if (ret_code == REMOTE_READ_OK) { + //外部调用者就可以获取到读取到的数据块的内容 *pagedata = pageData; + //计算并设置该页的校验和 PageSetChecksumInplace((Page) VARDATA(*pagedata), key.blocknum); } - + //如果读取操作成功，返回的结果是 REMOTE_READ_OK，否则返回的是之前判断出的其他错误码 return ret_code; } const int RES_COL_NUM = 2; +//从远程读取文件，并返回结果 Datum gs_read_file_from_remote(PG_FUNCTION_ARGS) { - RelFileNode rnode; + RelFileNode rnode; //文件节点信息 RemoteReadFileKey key; - int32 forknum; - uint32 blockstart; + int32 forknum; //分片号 + uint32 blockstart; //起始块号 uint64 lsn; bytea* result = NULL; Datum values[RES_COL_NUM]; @@ -375,10 +453,14 @@ Datum gs_read_file_from_remote(PG_FUNCTION_ARGS) int parano = 0; XLogRecPtr current_lsn = InvalidXLogRecPtr; + //检查当前用户是否具有足够的权限执行该函数 if (GetUserId() != BOOTSTRAP_SUPERUSERID) { + //如果不是超级用户，则会抛出错误并中断执行 ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be initial account to read files")))); } /* handle optional arguments */ + //使用 PG_GETARG_* 宏来获取每个参数并将它们存储在对应的变量中 + //这些参数是通过调用该函数的 SQL 语句传递给函数的 rnode.spcNode = PG_GETARG_UINT32(parano++); rnode.dbNode = PG_GETARG_UINT32(parano++); rnode.relNode = PG_GETARG_UINT32(parano++); @@ -390,42 +472,54 @@ Datum gs_read_file_from_remote(PG_FUNCTION_ARGS) timeout = PG_GETARG_INT32(parano++); if (rnode.spcNode != 1 && rnode.spcNode != 2) { + //存储文件的表空间节点号不是 1 或 2，需要获取数据文件 /* get tale data file */ + //初始化了一个 key 变量，并将传入的参数赋值给 key 的相应字段 key.relfilenode = rnode; key.forknum = forknum; key.blockstart = blockstart; + //检查 forknum 的值是否等于 MAIN_FORKNUM if (forknum != MAIN_FORKNUM) { + //如果不是，会发出一个警告并返回 NULL ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("Forknum should be 0. Now is %d. \n", forknum)))); PG_RETURN_NULL(); } + //forknum 等于 MAIN_FORKNUM，远程读取文件 + //读取指定位置的文件内容 ret_code = ReadFileForRemote(&key, lsn, &result, timeout); - } else { + } else {//rnode.spcNode 的值为 1 或 2 + //读取与 rnode 相关的文件内容 ret_code = ReadCOrCsnFileForRemote(rnode, &result); } if (ret_code != REMOTE_READ_OK) { + //如果文件读取失败，返回 NULL PG_RETURN_NULL(); } if (!RecoveryInProgress()) { + //如果当前没有进行恢复操作，函数会获取当前的逻辑日志序列号（LSN） current_lsn = GetXLogInsertRecPtr(); } - + //创建一个元组描述符（TupleDesc）对象，并设置两个字段：file 和 lsn tupdesc = CreateTemplateTupleDesc(RES_COL_NUM, false, TAM_HEAP); parano = 1; TupleDescInitEntry(tupdesc, (AttrNumber)parano++, "file", BYTEAOID, -1, 0); TupleDescInitEntry(tupdesc, (AttrNumber)parano++, "lsn", XIDOID, -1, 0); + //values 数组存储了对应字段的值，nulls 数组表示对应字段是否为 NULL values[0] = PointerGetDatum(result); nulls[0] = false; values[1] = UInt64GetDatum(current_lsn); nulls[1] = false; + //将函数的返回值转换为合适的格式，并返回给调用者 tupdesc = BlessTupleDesc(tupdesc); tuple = heap_form_tuple(tupdesc, values, nulls); PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); } +//获取指定文件在远程节点上的大小 Datum gs_read_file_size_from_remote(PG_FUNCTION_ARGS) { RelFileNode rnode; @@ -436,10 +530,13 @@ Datum gs_read_file_size_from_remote(PG_FUNCTION_ARGS) int parano = 0; int ret_code = REMOTE_READ_OK; + //检查当前用户 id 是否是 Bootstrap 超级用户 if (GetUserId() != BOOTSTRAP_SUPERUSERID) { + //如果不是，则抛出一个权限不足的错误 ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be initial account to read files")))); } /* handle optional arguments */ + //从输入参数中获取文件所在的 RelFileNode、forknum、lsn 和 timeout 等信息 rnode.spcNode = PG_GETARG_UINT32(parano++); rnode.dbNode = PG_GETARG_UINT32(parano++); rnode.relNode = PG_GETARG_UINT32(parano++); @@ -450,14 +547,18 @@ Datum gs_read_file_size_from_remote(PG_FUNCTION_ARGS) timeout = PG_GETARG_INT32(parano++); /* get file size */ + //尝试在远程节点上获取指定文件的大小 ret_code = ReadFileSizeForRemote(rnode, forknum, lsn, &size, timeout); if (ret_code == REMOTE_READ_OK) { + //如果成功获取文件大小，则通过 PG_RETURN_INT64 宏将 size 的值作为函数的返回值返回给调用者 PG_RETURN_INT64(size); } else { + //如果获取文件大小失败，则返回 null 值 PG_RETURN_NULL(); } } +//在远程节点上获取指定文件的大小，并将结果通过 res 参数返回 int ReadFileSizeForRemote(RelFileNode rnode, int32 forknum, XLogRecPtr lsn, int64* res, int timeout) { SMgrRelation smgr = NULL; @@ -465,41 +566,56 @@ int ReadFileSizeForRemote(RelFileNode rnode, int32 forknum, XLogRecPtr lsn, int6 int ret_code = REMOTE_READ_OK; /* wait request lsn for replay */ + //检查是否处于恢复状态 if (RecoveryInProgress()) { + //如果正在进行恢复，等待请求的 lsn 进行重放 ret_code = XLogWaitForReplay(lsn, timeout); + //如果等待超时或发生错误，函数会抛出一个错误并返回相应的错误码 if (ret_code != REMOTE_READ_OK) { ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("could not redo request lsn."))); return ret_code; } } + //发送请求进行立即强制的检查点，并等待完成 RequestCheckpoint(CHECKPOINT_IMMEDIATE | CHECKPOINT_FORCE | CHECKPOINT_WAIT); /* check whether the file exists. not exist, return size -1 */ + //检查文件是否存在 struct stat statBuf; + //根据 rnode 和 forknum 构建文件路径 char* path = relpathperm(rnode, forknum); + //调用 stat 函数检查文件的状态信息 if (stat(path, &statBuf) < 0 && errno == ENOENT) { + //如果文件不存在，则将 res 设置为 -1，并直接返回 *res = -1; pfree(path); return ret_code; } pfree(path); + //如果文件存在，且不属于分段文件 if (!IsSegmentFileNode(rnode)) { + //打开相应的 SMgrRelation smgr = smgropen(rnode, InvalidBackendId); + //获取文件的块数量 nblock = smgrnblocks(smgr, forknum); + //关闭所有 SMgrRelation smgrcloseall(); - } else { + } else { //如果文件属于分段文件 + //打开相应的 SegSpace SegSpace *spc = spc_open(rnode.spcNode, rnode.dbNode, true, true); spc_datafile_create(spc, rnode.relNode, forknum); - + //获取分段文件的大小 nblock = spc_size(spc, rnode.relNode, forknum); } + //将 nblock 乘以 BLCKSZ，得到文件的大小 *res = nblock * BLCKSZ; return ret_code; } +//从远程文件中读取指定块的内容，并将读取到的数据复制到 pageData 中的相应位置 int ReadFileByReadBufferComom(RemoteReadFileKey *key, bytea* pageData, uint32 nblock) { int ret_code = REMOTE_READ_OK; @@ -507,52 +623,68 @@ int ReadFileByReadBufferComom(RemoteReadFileKey *key, bytea* pageData, uint32 nb uint32 j = 0; uint32 blk_start; uint32 blk_end; - bool hit = false; + bool hit = false; //标志是否命中 char* bufBlock = NULL; errno_t rc; /* get the segno file block start and block end */ + //计算要读取的段号文件块的起始块和终止块 blk_start = key->blockstart; blk_end = (nblock >= blk_start + MAX_BATCH_READ_BLOCKNUM ? blk_start + MAX_BATCH_READ_BLOCKNUM : nblock); - + //使用循环从起始块到终止块，依次读取每个块的内容 for (i = blk_start, j = 0; i < blk_end; i++, j++) { /* read page, if PageIsVerified failed will long jump */ + //来获取访问策略，将策略指针赋给 bstrategy 变量 BufferAccessStrategy bstrategy = GetAccessStrategy(BAS_REPAIR); + //获取指定块的缓冲区，并通过 Buffer 类型的变量 buf 进行引用 Buffer buf = ReadBufferForRemote(key->relfilenode, key->forknum, i, RBM_FOR_REMOTE, bstrategy, &hit, NULL); + //如果 buf 无效，即无法获取有效的缓冲区 if (BufferIsInvalid(buf)) { + //返回错误码 REMOTE_READ_BLCKSZ_NOT_SAME ereport(WARNING, (errmodule(MOD_REMOTE), errmsg("repair file failed!"))); return REMOTE_READ_BLCKSZ_NOT_SAME; } + //对缓冲区进行共享锁定 LockBuffer(buf, BUFFER_LOCK_SHARE); bufBlock = (char*)BufferGetBlock(buf); + //将缓冲区的内容复制到 pageData 的相应位置 rc = memcpy_s(VARDATA(pageData) + j * BLCKSZ, BLCKSZ, bufBlock, BLCKSZ); + //memcpy_s 函数调用失败 if (rc != EOK) { ereport(WARNING, (errmodule(MOD_REMOTE), errmsg("repair file failed, memcpy_s error, retcode=%d", rc))); + //设置 ret_code 为 REMOTE_READ_MEMCPY_ERROR ret_code = REMOTE_READ_MEMCPY_ERROR; + //返回之前解锁并释放缓冲区 LockBuffer(buf, BUFFER_LOCK_UNLOCK); ReleaseBuffer(buf); return ret_code; } + //解锁并释放缓冲区 LockBuffer(buf, BUFFER_LOCK_UNLOCK); ReleaseBuffer(buf); + //每个块的内容复制完成后，为复制的页面计算并设置校验和 PageSetChecksumInplace((Page) (VARDATA(pageData) + j * BLCKSZ), i); } - + //返回 ret_code 表示操作的结果 return ret_code; } const int MAX_RETRY_TIMES = 60; +//根据指定的参数从磁盘或者远程位置读取指定块的数据 int ReadFileByReadDisk(SegSpace* spc, RemoteReadFileKey *key, char* bufBlock, BlockNumber blocknum) { int ret_code = REMOTE_READ_OK; - int pageStatus; - int retryTimes = 0; + int pageStatus; //页面状态 + int retryTimes = 0; //重试次数 + //如果传入的文件节点是段文件节点 if (IsSegmentFileNode(key->relfilenode)) { + //创建一个伪造的文件节点 fakenode，将其属性设置为传入的文件节点属性 + // 但将 bucketNode 属性设置为 SegmentBktId RelFileNode fakenode = { .spcNode = key->relfilenode.spcNode, .dbNode = key->relfilenode.dbNode, @@ -560,47 +692,63 @@ int ReadFileByReadDisk(SegSpace* spc, RemoteReadFileKey *key, char* bufBlock, Bl .bucketNode = SegmentBktId, .opt = 0 }; -SEG_RETRY: + SEG_RETRY: + //从远程位置读取指定块的数据，并将数据存储到 bufBlock 中 seg_physical_read(spc, fakenode, key->forknum, blocknum, (char *)bufBlock); retryTimes++; + //检查页面的校验和是否匹配 if (PageIsVerified((Page)bufBlock, blocknum)) { + //如果匹配，则将页面状态设置为 SMGR_RD_OK pageStatus = SMGR_RD_OK; } else { + //如果校验和不匹配，将页面状态设置为 SMGR_RD_CRC_ERROR pageStatus = SMGR_RD_CRC_ERROR; + //如果重试次数小于最大重试次数 if (retryTimes < MAX_RETRY_TIMES) { /* sleep 10ms */ pg_usleep(10000L); + //等待 10 毫秒，并跳转到标签 SEG_RETRY 处再次尝试读取数据 goto SEG_RETRY; - } else { + } else { //如果重试次数达到最大重试次数 + //释放 bufBlock 的内存，并输出警告信息 pfree(bufBlock); ereport(WARNING, (errmodule(MOD_REMOTE), errmsg("repair file failed, read page crc check error, page: %u/%u/%u/%d, " "forknum is %d, block num is %u", key->relfilenode.spcNode, key->relfilenode.dbNode, key->relfilenode.relNode, key->relfilenode.bucketNode, key->forknum, blocknum))); + //将返回码 ret_code 设置为 REMOTE_READ_CRC_ERROR，并返回该返回码 ret_code = REMOTE_READ_CRC_ERROR; return ret_code; } } - } else { + } else { //传入的文件节点不是段文件节点 + //打开指定文件节点对应的存储管理器关系对象 smgr SMgrRelation smgr = smgropen(key->relfilenode, InvalidBackendId); /* standby read page, replay finish, there will be no synchronous changes. */ + //从磁盘读取指定块的数据，并将数据存储到 bufBlock 中 pageStatus = smgrread(smgr, key->forknum, blocknum, (char *)bufBlock); retryTimes++; + //检查页面状态是否为 SMGR_RD_OK if (pageStatus != SMGR_RD_OK) { + //如果不是，则释放 bufBlock 的内存，并输出警告信息 pfree(bufBlock); ereport(WARNING, (errmodule(MOD_REMOTE), errmsg("repair file failed, read page crc check error, page: %u/%u/%u/%d, " "forknum is %d, block num is %u", key->relfilenode.spcNode, key->relfilenode.dbNode, key->relfilenode.relNode, key->relfilenode.bucketNode, key->forknum, blocknum))); + //然后将返回码 ret_code 设置为 REMOTE_READ_CRC_ERROR ret_code = REMOTE_READ_CRC_ERROR; + //关闭存储管理器关系对象 smgr smgrclose(smgr); return ret_code; } + //关闭存储管理器关系对象 smgr smgrclose(smgr); } return ret_code; } +//远程读取指定文件的数据块，并将读取的数据存储到bytea类型的变量中 int ReadFileForRemote(RemoteReadFileKey *key, XLogRecPtr lsn, bytea** fileData, int timeout) { int ret_code = REMOTE_READ_OK; @@ -616,23 +764,33 @@ int ReadFileForRemote(RemoteReadFileKey *key, XLogRecPtr lsn, bytea** fileData, errno_t rc; /* wait request lsn for replay */ + //检查是否在恢复模式中 if (RecoveryInProgress()) { + //如果是，则等待请求的日志序列号（lsn）进行重放 ret_code = XLogWaitForReplay(lsn, timeout); if (ret_code != REMOTE_READ_OK) { + //如果超时或出错，返回错误代码 ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("could not redo request lsn."))); return ret_code; } } + //如果在恢复模式中或者目标文件是段文件节点 if (RecoveryInProgress() || IsSegmentFileNode(key->relfilenode)) { + //发送请求进行检查点 RequestCheckpoint(CHECKPOINT_WAIT | CHECKPOINT_FORCE | CHECKPOINT_IMMEDIATE); } /* get block num */ + //如果不是段文件节点 if (!IsSegmentFileNode(key->relfilenode)) { + //打开对应的SMgrRelation smgr = smgropen(key->relfilenode, InvalidBackendId); + //获取块数 nblock = smgrnblocks(smgr, key->forknum); + //关闭SMgrRelation smgrclose(smgr); - } else { + } else { //如果是段文件节点 + //打开对应的SegSpace spc = spc_open(key->relfilenode.spcNode, key->relfilenode.dbNode, false, false); if (!spc) { ereport(WARNING, (errmodule(MOD_REMOTE), @@ -640,50 +798,66 @@ int ReadFileForRemote(RemoteReadFileKey *key, XLogRecPtr lsn, bytea** fileData, key->relfilenode.spcNode, key->relfilenode.dbNode))); return REMOTE_READ_IO_ERROR; } + //获取块数 nblock = spc_size(spc, key->relfilenode.relNode, key->forknum); } + //检查请求的起始块（blockstart）是否超出文件块数 if (nblock <= key->blockstart) { ret_code = REMOTE_READ_SIZE_ERROR; return ret_code; } /* get the segno file block start and block end */ + //根据起始块和最大批量读取块数计算出要读取的段（segno）的起始块（blk_start）和结束块（blk_end） blk_start = key->blockstart; blk_end = (nblock >= blk_start + MAX_BATCH_READ_BLOCKNUM ? blk_start + MAX_BATCH_READ_BLOCKNUM : nblock); + //分配用于存储读取的数据的内存空间 pageData = (bytea*)palloc((blk_end - blk_start) * BLCKSZ + VARHDRSZ); SET_VARSIZE(pageData, ((blk_end - blk_start) * BLCKSZ + VARHDRSZ)); /* primary read page, need read page by ReadBuffer_common */ + //如果不是段文件节点并且不在恢复模式中 if (!IsSegmentFileNode(key->relfilenode) && !RecoveryInProgress()) { + //读取主数据节点的页数据 ret_code = ReadFileByReadBufferComom(key, pageData, nblock); if (ret_code != REMOTE_READ_OK) { + //如果读取失败，则释放分配的内存空间，返回错误代码 pfree(pageData); ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("read file failed!"))); return ret_code; } - } else { + } else { //如果是段文件节点或者在恢复模式中 + // 判断是否使用ADIO（Asynchronous Direct I/O）读取数据 ADIO_RUN() { + //如果是，则使用adio_align_alloc函数分配对齐的内存块bufBlock，大小为BLCKSZ bufBlock = (Page)adio_align_alloc(BLCKSZ); } ADIO_ELSE() { + //如果不是，则使用palloc函数分配内存块bufBlock，大小为BLCKSZ bufBlock = (Page)palloc(BLCKSZ); } ADIO_END(); + //用循环从起始块（blk_start）到结束块（blk_end）依次读取数据 for (i = blk_start, j = 0; i < blk_end; i++, j++) { + //并存储到bufBlock中 ret_code = ReadFileByReadDisk(spc, key, bufBlock, i); if (ret_code != REMOTE_READ_OK) { + //读取失败，则释放分配的内存块bufBlock和pageData pfree(bufBlock); pfree(pageData); ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("repair file failed, read block %u error, retcode=%d", i, rc))); + //返回错误代码 return ret_code; } + //使用memcpy_s函数将bufBlock中的数据复制到pageData的合适位置上 rc = memcpy_s(VARDATA(pageData) + j * BLCKSZ, BLCKSZ, bufBlock, BLCKSZ); if (rc != EOK) { + //复制失败，则释放分配的内存块bufBlock和pageData，返回错误代码 pfree(bufBlock); pfree(pageData); ret_code = REMOTE_READ_MEMCPY_ERROR; @@ -691,22 +865,25 @@ int ReadFileForRemote(RemoteReadFileKey *key, XLogRecPtr lsn, bytea** fileData, return ret_code; } } + //释放分配的内存块bufBlock pfree(bufBlock); + //如果读取失败，则释放分配的内存空间pageData，返回错误代码 if (ret_code != REMOTE_READ_OK) { pfree(pageData); ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("read file failed!"))); return ret_code; } } - + //将读取的数据赋值给fileData指针 *fileData = pageData; - + //返回读取结果代码 return ret_code; } const int REGR_MCR_SIZE_1MB = 1048576; const int CLOG_NODE = 1; const int CSN_NODE = 2; +//读取指定文件（根据RelFileNode确定）的数据，并将数据存储到bytea类型的变量中 int ReadCOrCsnFileForRemote(RelFileNode rnode, bytea** fileData) { uint32 flags = O_RDWR | PG_BINARY; @@ -718,6 +895,7 @@ int ReadCOrCsnFileForRemote(RelFileNode rnode, bytea** fileData) char *buffer = (char*)palloc(logSize); int result = -1; + //根据给定的文件类型（rnode.spcNode），确定文件所在的路径（logType） if (rnode.spcNode == CLOG_NODE) { logType = "pg_clog"; } else if (rnode.spcNode == CSN_NODE) { @@ -726,42 +904,54 @@ int ReadCOrCsnFileForRemote(RelFileNode rnode, bytea** fileData) ereport(LOG, (errmodule(MOD_SEGMENT_PAGE), errmsg("File type\"%u\" does not exist, stop read here.", rnode.spcNode))); } - + //构建完整的文件路径（path），格式为"logType/relNode"，其中relNode是文件的节点号 rc = snprintf_s(path, MAX_PATH, MAX_PATH - 1, "%s/%012u", logType, rnode.relNode); securec_check_ss(rc, "\0", "\0"); + //调用BasicOpenFile函数以读写模式打开文件，并获取文件描述符（fd） fd = BasicOpenFile(path, flags, S_IWUSR | S_IRUSR); if (fd < 0) { + //如果打开失败 pfree(buffer); - if (errno != ENOENT) { + if (errno != ENOENT) { //且错误码不是ENOENT ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path))); } // The file does not exist, break. ereport(LOG, (errmodule(MOD_SEGMENT_PAGE), errmsg("File \"%s\" does not exist, stop read here.", path))); pfree(path); + //则报错并返回-1 return -1; } + //为了记录这次读取文件数据的等待事件 pgstat_report_waitevent(WAIT_EVENT_DATA_FILE_READ); + //分配用于存储文件数据的内存空间（buffer），大小为logSize，其中logSize默认为16 MB + //使用pread函数从文件中读取数据，将读取的数据存储到buffer中。读取的字节数由nbytes返回 uint32 nbytes = pread(fd, buffer, logSize, 0); + //结束等待事件的记录 pgstat_report_waitevent(WAIT_EVENT_END); - if (close(fd)) { + + if (close(fd)) { //关闭文件，并检查关闭是否成功 pfree(path); pfree(buffer); ereport(ERROR, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", path))); } - + //检查实际读取的字节数（nbytes）是否大于logSize if (nbytes > logSize) { + //如果是，则表示读取的数据超过了预设的大小 pfree(buffer); ereport(ERROR, (errcode(MOD_SEGMENT_PAGE), errcode_for_file_access(), errmsg("could not read file %s. nbytes:%u, logSize:%u", path, nbytes, logSize))); pfree(path); + //报错并返回-1 return -1; } else { + //分配用于存储读取的数据的内存空间（pageData），大小为nbytes + VARHDRSZ，其中VARHDRSZ是页头变长字段的大小 bytea* pageData = (bytea*)palloc(nbytes + VARHDRSZ); SET_VARSIZE(pageData, (nbytes + VARHDRSZ)); + //将读取的数据从buffer复制到pageData中 rc = memcpy_s(VARDATA(pageData), nbytes, buffer, nbytes); if (rc != EOK) { pfree(path); @@ -769,12 +959,14 @@ int ReadCOrCsnFileForRemote(RelFileNode rnode, bytea** fileData) pfree(buffer); ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("repair file failed, memcpy_s error, retcode=%d", rc))); return -1; - } else { + } else { //将pageData赋值给fileData指针 *fileData = pageData; result = 0; } } + //释放不再使用的内存空间 pfree(path); pfree(buffer); + //返回读取结果代码 return result; } -- 2.34.1 From cab4c90f8b195dba4e5327e9586a6cf0d5f2f78d Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:44:28 +0800 Subject: [PATCH 23/48] Update remote_read.cpp --- .../storage/remote/remote_read.cpp | 72 ++++++++++++++++--- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/src/gausskernel/storage/remote/remote_read.cpp b/src/gausskernel/storage/remote/remote_read.cpp index 505e159a7..da61578ec 100755 --- a/src/gausskernel/storage/remote/remote_read.cpp +++ b/src/gausskernel/storage/remote/remote_read.cpp @@ -22,6 +22,9 @@ * * --------------------------------------------------------------------------------------- */ + +//提供了一些函数来处理远程读取数据的相关操作 +// 并且根据数据库运行模式和配置文件的设置来确定是否开启远程读取功能 #include "postgres.h" #include "knl/knl_variable.h" @@ -37,9 +40,13 @@ * @IN error_code: remote read error code * @Return: remote read error message */ +//用于根据远程读取的错误码返回相应的错误消息 +//error_code，远程读取的错误码 const char* RemoteReadErrMsg(int error_code) { + //const char*，远程读取的错误消息 const char* error_msg = ""; + //根据不同的错误码对应设置不同的错误消息 switch (error_code) { case REMOTE_READ_OK: error_msg = "normal"; @@ -79,35 +86,51 @@ const char* RemoteReadErrMsg(int error_code) return error_msg; } +//用于获取主服务的地址 +//远程主机的地址，远程主机的端口号,格式化写入address数组中 +//address，存储主服务地址的字符数组指针 +//address_len，address数组的长度 void GetPrimaryServiceAddress(char *address, size_t address_len) { + //参数的有效性检查 if (address == NULL || address_len == 0 || t_thrd.walreceiverfuncs_cxt.WalRcv == NULL) + //如果传入的address为空指针、address_len为0或者全局变量t_thrd.walreceiverfuncs_cxt.WalRcv为空，则直接返回 return; bool is_running = false; volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; int rc = 0; + //取walrcv->mutex自旋锁 SpinLockAcquire(&walrcv->mutex); + //通过读取walrcv->isRunning的值来确定WAL接收是否正在运行 is_running = walrcv->isRuning; + //释放自旋锁 SpinLockRelease(&walrcv->mutex); + //如果walrcv->pid等于0或者is_running为false if (walrcv->pid == 0 || !is_running) + //表示WAL接收未启动或者已停止，直接返回 return; + //获取walrcv->mutex自旋锁 SpinLockAcquire(&walrcv->mutex); + //通过snprintf_s函数将主服务地址的字符串格式化写入address数组中 + // 格式为"%s@%d"，其中%s代表远程主机的地址，%d代表远程主机的端口号 rc = snprintf_s(address, address_len, (address_len - 1), "%s@%d", walrcv->conn_channel.remotehost, walrcv->conn_channel.remoteport); securec_check_ss(rc, "\0", "\0"); + //释放自旋锁 SpinLockRelease(&walrcv->mutex); } /* * @Description: get remote address - * @IN/OUT first_address: first address - * @IN/OUT second_address: second_address - * @IN address_len: address len + * @IN/OUT first_address: first address firstAddress，存储第一个地址的字符数组指针 + * @IN/OUT second_address: second_address secondAddress，存储第二个地址的字符数组指针 + * @IN address_len: address len addressLen，firstAddress和secondAddress数组的长度 */ +//用于获取远程读取地址 void GetRemoteReadAddress(char* firstAddress, char* secondAddress, size_t addressLen) { char ip[MAX_IPADDR_LEN] = {0}; @@ -115,40 +138,52 @@ void GetRemoteReadAddress(char* firstAddress, char* secondAddress, size_t addres errno_t rc = EOK; /* make sure first_address is correct */ + //进行参数的有效性检查 if (firstAddress == NULL || addressLen == 0) + //传入的firstAddress为空指针、addressLen为0，则直接返回 return; - volatile HaShmemData* hashmdata = t_thrd.postmaster_cxt.HaShmData; - ServerMode serverMode = hashmdata->current_mode; + volatile HaShmemData* hashmdata = t_thrd.postmaster_cxt.HaShmData; //该结构体包含了与共享内存相关的数据 + ServerMode serverMode = hashmdata->current_mode; //存储当前服务器的工作模式 if (IS_DN_DUMMY_STANDYS_MODE()) { + //如果是虚拟备模式 if (serverMode == PRIMARY_MODE && !IsPrimaryStandByReadyToRemoteRead()) { + /// 当前服务器模式为主模式且主备切换已完成，则直接返回 return; } - + //如果不满足上面的条件，则检查第二个备份节点的连接是否存在 if (t_thrd.postmaster_cxt.ReplConnArray[1]) { + //将其地址信息格式化为字符串并存储到firstAddress中 rc = snprintf_s(firstAddress, addressLen, (addressLen - 1), "%s@%d", t_thrd.postmaster_cxt.ReplConnArray[1]->remotehost, t_thrd.postmaster_cxt.ReplConnArray[1]->remoteport); securec_check_ss(rc, "", ""); } } else if (IS_DN_MULTI_STANDYS_MODE()) { + // 如果处于多节点备份模式 if (serverMode == PRIMARY_MODE) { + // 如果当前服务器模式为主模式 + // 获取优选备份节点的地址信息 GetFastestReplayStandByServiceAddress(firstAddress, secondAddress, addressLen); if (firstAddress[0] != '\0') { + // 如果第一个地址不为空，则将其格式化为ip:port的形式并存储到firstAddress中 GetIPAndPort(firstAddress, ip, port, MAX_IPADDR_LEN); rc = snprintf_s(firstAddress, addressLen, (addressLen - 1), "%s@%s", ip, port); securec_check_ss(rc, "", ""); } if (secondAddress[0] != '\0') { + // 如果第二个地址不为空，则将其格式化为ip:port的形式并存储到secondAddress中 GetIPAndPort(secondAddress, ip, port, MAX_IPADDR_LEN); rc = snprintf_s(secondAddress, addressLen, (addressLen - 1), "%s@%s", ip, port); securec_check_ss(rc, "", ""); } } else if (serverMode == STANDBY_MODE) { + // 如果当前服务器模式为备模式 GetPrimaryServiceAddress(firstAddress, addressLen); if (firstAddress[0] != '\0') { + // 如果地址不为空，则将其格式化为ip:port的形式并存储到firstAddress中 GetIPAndPort(firstAddress, ip, port, MAX_IPADDR_LEN); rc = snprintf_s(firstAddress, addressLen, (addressLen - 1), "%s@%s", ip, port); securec_check_ss(rc, "", ""); @@ -159,6 +194,11 @@ void GetRemoteReadAddress(char* firstAddress, char* secondAddress, size_t addres } } +//从地址字符串中提取IP地址和端口号 +//address表示待处理的地址字符串 +// ip表示存储提取的IP地址 +// port表示存储提取的端口号 +// len表示ip和port缓冲区的长度 void GetIPAndPort(char* address, char* ip, char* port, size_t len) { char* outerPtr = NULL; @@ -166,10 +206,13 @@ void GetIPAndPort(char* address, char* ip, char* port, size_t len) char* tmpIp; char* tempPort; + //对address进行分割，以'@'为分隔符，将地址字符串分割为IP地址和端口号两部分 tmpIp = strtok_r(address, "@", &outerPtr); tempPort = strtok_r(NULL, "@", &outerPtr); + //判断提取的IP地址和端口号是否有效，即不为空且长度不超过缓冲区的限制 if (tmpIp != NULL && tmpIp[0] != '\0' && tempPort != NULL && tempPort[0] != '\0' && strlen(tmpIp) + strlen(tempPort) + 1 < len) { + //如果满足条件，则使用strcpy_s函数将提取的IP地址和端口号拷贝到ip和port缓冲区中 rc = strcpy_s(ip, MAX_IPADDR_LEN, tmpIp); securec_check(rc, "", ""); rc = strcpy_s(port, MAX_IPADDR_LEN, tempPort); @@ -182,13 +225,17 @@ void GetIPAndPort(char* address, char* ip, char* port, size_t len) * @Description: have remote node to read * @Return: true if have remote node */ +//用于判断是否存在远程节点可供读取 bool CanRemoteRead() { volatile HaShmemData* hashmdata = t_thrd.postmaster_cxt.HaShmData; + //获取当前的服务器模式 ServerMode serveMode = hashmdata->current_mode; - + //检查远程读取模式是否打开 && 是否不处于不含备机的数据节点模式 && 是否是PGXC数据节点 + //检查服务器模式是否不是NORMAL_MODE、PENDING_MODE和STANDBY_MODE if (IsRemoteReadModeOn() && !IS_DN_WITHOUT_STANDBYS_MODE() && IS_PGXC_DATANODE && serveMode != NORMAL_MODE && serveMode != PENDING_MODE && serveMode != STANDBY_MODE) { + //如果上述所有条件都满足，则返回true表示存在远程节点可供读取 return true; } return false; @@ -198,8 +245,11 @@ bool CanRemoteRead() * @Description: remote mode is on * @Return: true if emote read is on */ +//用于判断远程读取模式是否开启 bool IsRemoteReadModeOn() { + //获取到当前的远程读取模式，并将其与REMOTE_READ_OFF进行比较 + //果当前远程读取模式不等于REMOTE_READ_OFF，则表示远程读取模式开启，函数返回true return g_instance.attr.attr_storage.remote_read_mode != REMOTE_READ_OFF; } @@ -207,16 +257,22 @@ bool IsRemoteReadModeOn() * @Description: set remote mode off and get old remote_read_mode * @Return: old remote_read_mode */ +//用于 关闭远程读取模式 并 返回被关闭远程读取模式 int SetRemoteReadModeOffAndGetOldMode() { + //获取到当前的远程读取模式 int oldRemoteRead = g_instance.attr.attr_storage.remote_read_mode; - + //关闭远程读取 g_instance.attr.attr_storage.remote_read_mode = REMOTE_READ_OFF; + //返回被关闭远程读取模式 return oldRemoteRead; } +//用于设置远程读取模式 +//接受一个整型参数mode，用于指定要设置的远程读取模式 void SetRemoteReadMode(int mode) { + //将远程读取模式设置为指定的值 g_instance.attr.attr_storage.remote_read_mode = mode; return; -- 2.34.1 From f28aca29e27f06e53bc10dc5e45d16eb7bf5f578 Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:45:33 +0800 Subject: [PATCH 24/48] Update ckms_message.cpp --- .../tde_key_management/ckms_message.cpp | 188 +++++++++++++++++- 1 file changed, 177 insertions(+), 11 deletions(-) diff --git a/src/gausskernel/security/tde_key_management/ckms_message.cpp b/src/gausskernel/security/tde_key_management/ckms_message.cpp index 2d8b6429b..6a9a88e3c 100755 --- a/src/gausskernel/security/tde_key_management/ckms_message.cpp +++ b/src/gausskernel/security/tde_key_management/ckms_message.cpp @@ -22,7 +22,7 @@ * --------------------------------------------------------------------------------------- */ -#include +#include //用于文件读写 #include #include "tde_key_management/ckms_message.h" #include "utils/elog.h" @@ -32,6 +32,8 @@ namespace TDE { using namespace std; + +//构造函数，用于初始化对象的成员变量 CKMSMessage::CKMSMessage() { tde_message_mem = nullptr; @@ -43,27 +45,35 @@ CKMSMessage::CKMSMessage() token_timestamp = 0; } +//析构函数，用于释放对象的资源 +//调用了 clear() 函数，用于清理对象内部申请的动态内存 CKMSMessage::~CKMSMessage() { clear(); } +//初始化对象的上下文（tde_message_mem） void CKMSMessage::init() { - if (tde_message_mem == nullptr) { + if (tde_message_mem == nullptr) {//如果 tde_message_mem 为空 + //调用 AllocSetContextCreate() 创建一个新的上下文，并将其赋值给 tde_message_mem tde_message_mem = AllocSetContextCreate(g_instance.cache_cxt.global_cache_mem, "TDE_MESSAGE_CONTEXT", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE, SHARED_CONTEXT); } } +//清理对象的资源 void CKMSMessage::clear() { errno_t rc = 0; + //首先将 token_info->password 的内容清零 if (token_info->password != NULL) { rc = memset_s(token_info->password, strlen(token_info->password), 0, strlen(token_info->password)); securec_check(rc, "\0", "\0"); } + + //然后调用 pfree_ext() 释放对象的各个指针成员所指向的内存块 pfree_ext(tde_token); pfree_ext(tde_agency_token); @@ -83,10 +93,13 @@ void CKMSMessage::clear() pfree_ext(kms_info); } +//检查当前对象中的身份令牌是否有效 bool CKMSMessage::check_token_valid() { TimestampTz current_time = 0; + //获取了当前时间戳 current_time = GetCurrentTimestamp(); + //次检查 tde_token、tde_agency_token 和令牌时间戳是否为空或者是否超过了有效时间 if (tde_token == NULL) { return false; } @@ -99,6 +112,7 @@ bool CKMSMessage::check_token_valid() return true; } +//用于从文件中读取 KMS（Key Management Service）信息 char* CKMSMessage::read_kms_info_from_file() { char file_path[PATH_MAX] = {0}; @@ -110,40 +124,50 @@ char* CKMSMessage::read_kms_info_from_file() errno_t rc = EOK; data_directory = g_instance.attr.attr_common.data_directory; + //检查 data_directory 是否为空 if (data_directory == NULL) { + //如果为空，则会产生错误报告，并返回 NULL ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("get gaussdb data directory path is NULL"), errdetail("N/A"), errcause("data directory path not set"), erraction("check if guc data_directory is exist"))); return NULL; } + //拼接 data_directory、tde_config 和 token_file 构造完整的文件路径 path_len = strlen(data_directory) + strlen(tde_config) + strlen(token_file); rc = snprintf_s(file_path, PATH_MAX, path_len, "%s%s%s", data_directory, tde_config, token_file); securec_check_ss(rc, "\0", "\0"); + + //尝试以二进制的方式打开指定路径的文件 fstream json_file(file_path, ios::in | ios::binary); + //打开失败，则会产生错误报告，并返回相应的错误信息 if (!json_file) { ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_FILE_READ_FAILED), errmsg("unable to open kms_iam_info.json file"), errdetail("file path: %s", file_path), errcause("file not exist or broken"), erraction("check the kms_iam_info.json file"))); } - + //如果文件打开成功，函数会获取文件的长度，并进行一些长度检查 json_file.seekg(0, ios::end); json_len = json_file.tellg(); - if (json_len > max_json_len) { - json_file.close(); + if (json_len > max_json_len) { //如果文件长度超过 max_json_len + json_file.close(); //关闭文件 + //产生错误报告，并返回相应的错误信息 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_FILE_READ_FAILED), errmsg("kms_iam_info.json file length is bigger than max_len"), errdetail("file path: $TDE_PATH/tde_config/kms_iam_info.json"), errcause("file context is wrong"), erraction("check the kms_iam_info.json file"))); } + //根据文件长度动态分配一个大小适合的缓冲区，并将文件内容读入该缓冲区 json_file.seekg(0, ios::beg); buffer = (char*)palloc0(json_len + 1); json_file.read(buffer, json_len + 1); + //关闭文件，并返回读取到的文件内容的指针 json_file.close(); return buffer; } +//获取密码的明文 GS_UCHAR* CKMSMessage::get_cipher_rand() { GS_UCHAR* plain_text = NULL; @@ -153,21 +177,33 @@ GS_UCHAR* CKMSMessage::get_cipher_rand() int path_len = 0; errno_t rc = EOK; + //获取数据目录路径 data_directory = g_instance.attr.attr_common.data_directory; + //如果 data_directory 为 NULL，则报错并返回 NULL if (data_directory == NULL) { + //报错信息中包含了错误码、错误详情以及可能的修复措施 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("get gaussdb data directory path is NULL"), errdetail("N/A"), errcause("data directory path not set"), erraction("check if guc data_directory is exist"))); return NULL; } + + //计算 path_len 的长度 path_len = strlen(data_directory) + strlen(tde_config) + strlen(key) + 1; + //动态分配了一个大小为 path_len 的字符数组，并将其初始化为全零。该数组存储了拼接后的路径 key_dir = (char*)palloc0(path_len); + //拼接成完整的路径 rc = snprintf_s(key_dir, path_len, (path_len - 1), "%s%s%s", data_directory, tde_config, key); securec_check_ss(rc, "\0", "\0"); + + //动态分配了一个大小为 CIPHER_LEN + 1 的 GS_UCHAR 类型的数组，并将其初始化为全零。该数组用于存储密码的明文 plain_text = (GS_UCHAR*)palloc0(CIPHER_LEN + 1); + //解码密文文件并将结果存储在 plain_text 中 decode_cipher_files(keymode, NULL, key_dir, plain_text); + pfree_ext(key_dir); + //如果明文数组 plain_text 的长度为 0，则报错 if (strlen((char*)plain_text) == 0) { ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("can not get password plaintext"), errdetail("N/A"), errcause("file not exist or broken"), @@ -176,6 +212,7 @@ GS_UCHAR* CKMSMessage::get_cipher_rand() return plain_text; } +//用于填充 TokenInfo 结构体的信息 void CKMSMessage::fill_token_info(cJSON* internal_json) { char* username = NULL; @@ -183,7 +220,8 @@ void CKMSMessage::fill_token_info(cJSON* internal_json) char* domain_name = NULL; char* project_name = NULL; errno_t rc = EOK; - + //通过解析传入的 internal_json 参数，获取其中的 "username"、"domain_name" 和 "project_name" 字段的值 + //如果这些字段为空，那么会抛出错误并返回 if ((cJSON_GetObjectItem(internal_json, "username") == NULL) || (cJSON_GetObjectItem(internal_json, "domain_name") == NULL) || (cJSON_GetObjectItem(internal_json, "project_name") == NULL)) { @@ -192,30 +230,43 @@ void CKMSMessage::fill_token_info(cJSON* internal_json) errcause("IAM info value error"), erraction("check tde_config kms_iam_info.json file"))); } + + //将当前内存上下文切换为 tde_message_mem 上下文 MemoryContext old = MemoryContextSwitchTo(tde_message_mem); + //分配了 sizeof(TokenInfo) 大小的内存来存储 TokenInfo 结构体 token_info = (TokenInfo*)palloc0(sizeof(TokenInfo)); + + //从 internal_json 中获取到的 "username"、"domain_name" 和 "project_name" 字段的值拷贝到相应的结构体成员中，并对拷贝操作进行错误检查 username = cJSON_GetObjectItem(internal_json, "username")->valuestring; token_info->user_name = (char*)palloc0(strlen(username) + 1); rc = memcpy_s(token_info->user_name, (strlen(username) + 1), username, (strlen(username) + 1)); securec_check(rc, "\0", "\0"); + domain_name = cJSON_GetObjectItem(internal_json, "domain_name")->valuestring; token_info->domain_name = (char*)palloc0(strlen(domain_name) + 1); rc = memcpy_s(token_info->domain_name, (strlen(domain_name) + 1), domain_name, (strlen(domain_name) + 1)); securec_check(rc, "\0", "\0"); + project_name = cJSON_GetObjectItem(internal_json, "project_name")->valuestring; token_info->project_name = (char*)palloc0(strlen(project_name) + 1); rc = memcpy_s(token_info->project_name, (strlen(project_name) + 1), project_name, (strlen(project_name) + 1)); securec_check(rc, "\0", "\0"); + + //调用 get_cipher_rand 函数获取随机密码 password = (char*)get_cipher_rand(); - if (password == NULL) { + if (password == NULL) { //获取密码失败（返回 NULL），则抛出错误并返回 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("get internal password is NULL"), errdetail("N/A"), errcause("cipher rand file missing"), erraction("check password cipher rand file"))); return; } + //配了与 password 长度相同大小的内存给 token_info->password token_info->password = (char*)palloc0(strlen(password) + 1); + //将 password 的内容拷贝到 token_info->password 中，并进行错误检查 rc = memcpy_s(token_info->password, (strlen(password) + 1), password, (strlen(password) + 1)); securec_check(rc, "\0", "\0"); + + //将内存上下文切换回之前的上下文，并清空并释放 password 指针所指向的内存 MemoryContextSwitchTo(old); rc = memset_s(password, strlen(password), 0, strlen(password)); securec_check(rc, "\0", "\0"); @@ -223,6 +274,7 @@ void CKMSMessage::fill_token_info(cJSON* internal_json) return; } +//用于填充 AgencyTokenInfo 结构体的信息 void CKMSMessage::fill_agency_info(cJSON* agency_json) { char* domain_name = NULL; @@ -230,6 +282,8 @@ void CKMSMessage::fill_agency_info(cJSON* agency_json) char* project_name = NULL; errno_t rc = EOK; + //通过解析传入的 agency_json 参数，获取其中的 "domain_name"、"agency_name" 和 "project_name" 字段的值 + //如果这些字段为空，那么会抛出错误并返回 if ((cJSON_GetObjectItem(agency_json, "domain_name") == NULL) || (cJSON_GetObjectItem(agency_json, "agency_name") == NULL) || (cJSON_GetObjectItem(agency_json, "project_name") == NULL)) { @@ -238,33 +292,45 @@ void CKMSMessage::fill_agency_info(cJSON* agency_json) errcause("IAM info value error"), erraction("check tde_config kms_iam_info.json file"))); } + //将当前内存上下文切换为 tde_message_mem 上下文 MemoryContext old = MemoryContextSwitchTo(tde_message_mem); + //分配了 sizeof(AgencyTokenInfo) 大小的内存来存储 AgencyTokenInfo 结构体 agency_token_info = (AgencyTokenInfo*)palloc0(sizeof(AgencyTokenInfo)); + + //分别将从 agency_json 中获取到的 "domain_name"、"agency_name" 和 "project_name" 字段的值拷贝到相应的结构体成员中 + // 并对拷贝操作进行错误检查 domain_name = cJSON_GetObjectItem(agency_json, "domain_name")->valuestring; agency_token_info->domain_name = (char*)palloc0(strlen(domain_name) + 1); rc = memcpy_s(agency_token_info->domain_name, (strlen(domain_name) + 1), domain_name, (strlen(domain_name) + 1)); securec_check(rc, "\0", "\0"); + agency_name = cJSON_GetObjectItem(agency_json, "agency_name")->valuestring; agency_token_info->agency_name = (char*)palloc0(strlen(agency_name) + 1); rc = memcpy_s(agency_token_info->agency_name, (strlen(agency_name) + 1), agency_name, (strlen(agency_name) + 1)); securec_check(rc, "\0", "\0"); + project_name = cJSON_GetObjectItem(agency_json, "project_name")->valuestring; agency_token_info->project_name = (char*)palloc0(strlen(project_name) + 1); rc = memcpy_s(agency_token_info->project_name, (strlen(project_name) + 1), project_name, (strlen(project_name) + 1)); securec_check(rc, "\0", "\0"); + + //将内存上下文切换回之前的上下文，并返回 MemoryContextSwitchTo(old); return; } +//填充 KmsInfo 结构体的信息 void CKMSMessage::fill_kms_info(cJSON* kms_json) { char* project_name = NULL; char* project_id = NULL; errno_t rc = EOK; + //通过解析传入的 kms_json 参数，获取其中的 "project_name" 和 "project_id" 字段的值 + // 如果这些字段为空，那么会抛出错误并返回 if ((cJSON_GetObjectItem(kms_json, "project_name") == NULL) || (cJSON_GetObjectItem(kms_json, "project_id") == NULL)) { ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), @@ -272,22 +338,32 @@ void CKMSMessage::fill_kms_info(cJSON* kms_json) errcause("KMS info value error"), erraction("check tde_config kms_iam_info.json file"))); } + + //将当前内存上下文切换为 tde_message_mem 上下文 MemoryContext old = MemoryContextSwitchTo(tde_message_mem); + //分配了 sizeof(KmsInfo) 大小的内存来存储 KmsInfo 结构体 kms_info = (KmsInfo*)palloc0(sizeof(KmsInfo)); + + //分别将从 kms_json 中获取到的 "project_name" 和 "project_id" 字段的值拷贝到相应的结构体成员中 + // 并对拷贝操作进行错误检查 project_name = cJSON_GetObjectItem(kms_json, "project_name")->valuestring; kms_info->project_name = (char*)palloc0(strlen(project_name) + 1); rc = memcpy_s(kms_info->project_name, (strlen(project_name) + 1), project_name, (strlen(project_name) + 1)); securec_check(rc, "\0", "\0"); + project_id = cJSON_GetObjectItem(kms_json, "project_id")->valuestring; kms_info->project_id = (char*)palloc0(strlen(project_id) + 1); rc = memcpy_s(kms_info->project_id, (strlen(project_id) + 1), project_id, (strlen(project_id) + 1)); securec_check(rc, "\0", "\0"); + + //将内存上下文切换回之前的上下文，并返回 MemoryContextSwitchTo(old); return; } +//解析 JSON 文件并根据文件内容填充相关信息 void CKMSMessage::parser_json_file(char* buffer) { cJSON* json_root = NULL; @@ -295,42 +371,57 @@ void CKMSMessage::parser_json_file(char* buffer) cJSON* agency_json = NULL; cJSON* kms_json = NULL; + //调用 cJSON_Parse 函数解析传入的 buffer 字符串 + // 将解析结果存储在 json_root 中 json_root = cJSON_Parse(buffer); if (!json_root) { + //如果解析失败，则会释放 buffer 的内存，并抛出错误 pfree_ext(buffer); ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_FILE_READ_FAILED), errmsg("unable to get json file"), errdetail("N/A"), errcause("parse json file failed"), erraction("check the kms_iam_info.json file format"))); } + //从 json_root 中获取 "internal_user_info" 字段的值，并通过调用 fill_token_info 函数填充相关的令牌信息 internal_json = cJSON_GetObjectItem(json_root, "internal_user_info"); if (internal_json) { fill_token_info(internal_json); } + //从 json_root 中获取 "agency_user_info" 字段的值，并通过调用 fill_agency_info 函数填充相关的机构信息 agency_json = cJSON_GetObjectItem(json_root, "agency_user_info"); if (agency_json) { fill_agency_info(agency_json); } + //从 json_root 中获取 "kms_info" 字段的值，并通过调用 fill_kms_info 函数填充相关的 KMS（密钥管理服务）信息 kms_json = cJSON_GetObjectItem(json_root, "kms_info"); if (kms_json) { fill_kms_info(kms_json); } + + //释放 buffer 的内存，删除 json_root 对象 pfree_ext(buffer); cJSON_Delete(json_root); return; } +//用于加载用户信息 +//从文件中读取用户信息，并进行解析和填充 void CKMSMessage::load_user_info() { char* read_buffer = NULL; /* read KMS and IAM info from kms_iam_info.json */ + //调用 read_kms_info_from_file 函数读取 kms_iam_info.json 文件的内容，并将结果存储在 read_buffer 中 read_buffer = read_kms_info_from_file(); /* parser json file */ + //调用 parser_json_file 函数解析 read_buffer 中的 JSON 内容，以填充相关的用户信息 parser_json_file(read_buffer); return; } + +//用于生成 KMS（密钥管理服务）所需的 JSON 字符串 char* CKMSMessage::get_kms_json(ReplaceJsonValue input_json[], KmsHttpMsgType json_type, size_t count) { + //ret 是一个枚举类型的变量，用于存储 HTTP 请求的返回值 HttpErrCode ret = TDE_HTTP_SUCCEED; cJSON* json = NULL; char* json_string = NULL; @@ -338,51 +429,79 @@ char* CKMSMessage::get_kms_json(ReplaceJsonValue input_json[], KmsHttpMsgType js errno_t rc = EOK; /* json_type is KmsHttpMsgType */ + //调用 get_json_temp 函数获取指定 json_type 类型的 JSON 树 json = get_json_temp(json_type); - if (json == NULL) { + if (json == NULL) { //如果返回的 json 为 NULL，则会产生错误报告并返回 NULL ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("get JSON tree is NULL"), errdetail("N/A"), errcause("get KMS JSON tree failed"), erraction("check input prarmeter or config.ini file"))); return NULL; } + + //调用 traverse_jsontree_with_raplace_value 函数遍历 json 树 + //并根据 input_json 数组中的替换值进行替换 ret = traverse_jsontree_with_raplace_value(json, input_json, count); + //如果遍历操作出现错误，那么将删除 json 对象并返回 NULL if (ret != TDE_HTTP_SUCCEED) { cJSON_Delete(json); return NULL; } + + //调用 cJSON_Print 函数将替换完成后的 json 转换为字符串 temp_json = cJSON_Print(json); - if (temp_json == NULL) { + if (temp_json == NULL) {//如果转换失败，则会产生错误报告 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("get JSON tree is NULL"), errdetail("N/A"), errcause("get KMS JSON tree failed"), erraction("check input prarmeter or config.ini file"))); } + + //根据 temp_json 的长度动态分配内存，并将 temp_json 的内容复制到 json_string 中 json_string = (char*)palloc0(strlen(temp_json) + 1); rc = memcpy_s(json_string, (strlen(temp_json) + 1), temp_json, (strlen(temp_json) + 1)); securec_check(rc, "\0", "\0"); + + //释放 temp_json 的内存，删除 json 对象，并返回生成的 JSON 字符串 json_string cJSON_free(temp_json); cJSON_Delete(json); return json_string; } +//user +//用于生成用于 IAM（身份和访问管理）授权请求的 JSON 字符串 char* CKMSMessage::get_iam_token_json() { char* token = NULL; size_t count = 0; + //代表替换值的数组 json + // json 数组包含了一些键值对 + // 键是要替换的占位符 + // 值是通过 token_info 对象获取的具体信息（例如用户名、密码、域名和项目名） ReplaceJsonValue json[] = { {"$user_name$", token_info->user_name}, {"$password$", token_info->password}, {"$domain_name$", token_info->domain_name}, {"$project_name$", token_info->project_name}, }; + + //通过计算数组的大小除以单个元素的大小，得到数组元素的数量并赋值给 count 变量 count = sizeof(json) / sizeof(json[0]); + //调用 get_kms_json 函数，并将 json 数组、IAM_AUTH_REQ （IAM 授权请求类型）和 count 参数传递进去 + //会根据 json 数组中的值进行替换，并生成相应的 JSON 字符串 token = get_kms_json(json, IAM_AUTH_REQ, count); + //返回生成的 JSON 字符串 token return token; } +//agency +// 用于生成用于 IAM（身份和访问管理）授权请求的 JSON 字符串 char* CKMSMessage::get_iam_agency_token_json() { char* agency_token = NULL; size_t count = 0; + // 代表替换值的数组 json + // json 数组包含了一些键值对 + // 键是要替换的占位符 + // 值是通过 agency_token_info 对象获取的具体信息（例如域名、代理名称和项目名称） ReplaceJsonValue json[] = { {"$domain_name$", agency_token_info->domain_name}, {"$agency_name$", agency_token_info->agency_name}, @@ -390,77 +509,116 @@ char* CKMSMessage::get_iam_agency_token_json() }; count = sizeof(json) / sizeof(json[0]); agency_token = get_kms_json(json, IAM_AGENCY_TOKEN_REQ, count); + //返回生成的 JSON 字符串 agency_token return agency_token; } +//用于生成创建数据加密密钥（DEK）的请求 JSON 字符串 +//接受一个 cmk_id 参数，用于指定创建 DEK 所使用的主密钥 ID char* CKMSMessage::get_create_dek_json(const char* cmk_id) { char* req_body = NULL; size_t count = 0; + //创建了一个用于替换值的 json 数组 + // 包含了一个键值对 + // 即占位符 $cmk_id$ 和传入的 cmk_id 值 ReplaceJsonValue json[] = { {"$cmk_id$", cmk_id}, }; count = sizeof(json) / sizeof(json[0]); + // 将 json 数组、TDE_GEN_DEK_REQ 枚举类型（表示创建 DEK 请求类型）和 count 参数传递 req_body = get_kms_json(json, TDE_GEN_DEK_REQ, count); + // 返回 用于生成创建数据加密密钥（DEK）的请求 JSON 字符串 return req_body; } +//用于生成解密数据加密密钥（DEK）的请求 JSON 字符串 +//接受两个参数 cmk_id 是用于指定解密 DEK 所使用的主密钥 ID dek_cipher 是 DEK 的密文 char* CKMSMessage::get_decrypt_dek_json(const char* cmk_id, const char* dek_cipher) { char* req_body = NULL; size_t count = 0; + // 创建了一个用于替换值的 json 数组 + // 包含了两个键值对 + // 占位符 $dek_cipher$ 和 $cmk_id$ 分别对应传入的 dek_cipher 和 cmk_id 值 ReplaceJsonValue json[] = { {"$dek_cipher$", dek_cipher}, {"$cmk_id$", cmk_id}, }; count = sizeof(json) / sizeof(json[0]); + //将 json 数组、TDE_DEC_DEK_REQ 枚举类型（表示解密 DEK 请求类型）和 count 参数传递 req_body = get_kms_json(json, TDE_DEC_DEK_REQ, count); + //返回 生成解密数据加密密钥（DEK）的请求 JSON 字符串 return req_body; } +//用于保存 TDE token 和代理 token bool CKMSMessage::save_token(const char* token, const char* agency_token) { errno_t rc = EOK; - + //对传入的参数进行非空校验 if ((token == NULL) || (agency_token == NULL)) { return false; } + + //将上下文切换到 tde_message_mem MemoryContext old = MemoryContextSwitchTo(tde_message_mem); + //在该内存上创建 tde_token 和 tde_agency_token 变量，它们分别用于保存 TDE token 和代理 token if (tde_token == NULL) { tde_token = (char*)palloc0(strlen(token) + 1); } + //token 参数复制到 tde_token变量中 rc = memcpy_s(tde_token, (strlen(token) + 1), token, (strlen(token) + 1)); securec_check(rc, "\0", "\0"); + if (tde_agency_token == NULL) { tde_agency_token = (char*)palloc0(strlen(agency_token) + 1); } + //agency_token 参数复制到 tde_agency_token 变量中 rc = memcpy_s(tde_agency_token, (strlen(agency_token) + 1), agency_token, (strlen(agency_token) + 1)); securec_check(rc, "\0", "\0"); + + //获取当前时间戳 token_timestamp = GetCurrentTimestamp(); + //切换回原来的上下文 MemoryContextSwitchTo(old); return true; } +//用于遍历 JSON 树并替换特定的值 +// json_tree 是要遍历的 JSON 树的根节点 +// replace_rules 是一个包含替换规则的数组 +// rule_cnt 表示替换规则的数量。 HttpErrCode CKMSMessage::traverse_jsontree_with_raplace_value(cJSON *json_tree, ReplaceJsonValue replace_rules[], size_t rule_cnt) { char *new_value = NULL; HttpErrCode ret = TDE_HTTP_SUCCEED; + //对传入的 json_tree 进行非空校验 if (json_tree == NULL) { return TDE_HTTP_SUCCEED; } + //判断 json_tree 是否为字符串类型的节点 if (cJSON_IsString(json_tree)) { + //如果是，则遍历替换规则数组 replace_rules for (size_t i = 0; i < rule_cnt; i++) { + //检查当前节点的值是否为 NULL if (cJSON_GetStringValue(json_tree) == NULL) { + //如果是，则抛出错误 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("failed to get json tree"), errdetail("N/A"), errcause("config.ini json tree error"), erraction("check input prarmeter or config.ini file"))); } + //将当前节点的值与替换规则的源值进行比较 if (strcmp(replace_rules[i].src_value, cJSON_GetStringValue(json_tree)) == 0) { + //如果相等，则使用 cJSON_SetValuestring 函数 + // 将当前节点的值替换为替换规则的目标值，并将返回的新值保存到 new_value 变量中 new_value = cJSON_SetValuestring(json_tree, replace_rules[i].dest_value); + ////检查当前]新节点的值是否为 NULL if (new_value == NULL) { + // 如果是，则抛出错误 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("failed to set the value of json tree"), errdetail("N/A"), errcause("config.ini json tree error"), @@ -470,15 +628,23 @@ HttpErrCode CKMSMessage::traverse_jsontree_with_raplace_value(cJSON *json_tree, } } } + //递归调用 traverse_jsontree_with_raplace_value 函数分别对当前节点的下一个兄弟节点和子节点进行遍历 ret = traverse_jsontree_with_raplace_value(json_tree->next, replace_rules, rule_cnt); + //递归调用返回的结果不等于 TDE_HTTP_SUCCEED if (ret != TDE_HTTP_SUCCEED) { + //则将该结果作为函数的返回值 return ret; } + //否则，继续对当前节点的子节点进行遍历，并将结果作为函数的返回值 return traverse_jsontree_with_raplace_value(json_tree->child, replace_rules, rule_cnt); } +//根据传入的 json_tree_type 参数返回相应的 cJSON 对象指针 +//接受一个 KmsHttpMsgType 类型的参数 json_tree_type，用于确定返回哪个 cJSON 对象指针 cJSON *CKMSMessage::get_json_temp(KmsHttpMsgType json_tree_type) { + //使用 switch 语句根据 json_tree_type 的值进行匹配 + // 返回 匹配 字符串解析后的 cJSON 对象指针 switch (json_tree_type) { case IAM_AUTH_REQ: return cJSON_Parse(iam_auth_token_req); @@ -489,7 +655,7 @@ cJSON *CKMSMessage::get_json_temp(KmsHttpMsgType json_tree_type) case TDE_DEC_DEK_REQ: return cJSON_Parse(kms_decrypt_dek_req); default: - break; + break; //与上述情况都不匹配，则执行默认的 break，然后返回 NULL } return NULL; } -- 2.34.1 From 757c6de347212e2197b8483dbda80d0f8be626ff Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:46:10 +0800 Subject: [PATCH 25/48] Update CMakeLists.txt --- .../security/tde_key_management/CMakeLists.txt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/gausskernel/security/tde_key_management/CMakeLists.txt b/src/gausskernel/security/tde_key_management/CMakeLists.txt index 3d04bed80..bfb517489 100644 --- a/src/gausskernel/security/tde_key_management/CMakeLists.txt +++ b/src/gausskernel/security/tde_key_management/CMakeLists.txt @@ -1,6 +1,12 @@ #This is the main CMAKE for build bin. +#这是用于构建二进制文件的主要 CMake 配置。 + +#首先，使用 `AUX_SOURCE_DIRECTORY` 函数 +#将 `${CMAKE_CURRENT_SOURCE_DIR}` 目录下的源文件添加到 `TGT_tde_key_management_SRC` 变量中 AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_tde_key_management_SRC) +#然后，设置 `TGT_tde_key_management_INC` 变量，包含了一些源文件的头文件路径 +#这些路径可能包括与通信、IP 范围、通讯库、项目包含文件、跟踪等相关的路径 set(TGT_tde_key_management_INC ${PROJECT_SRC_DIR}/gausskernel/cbb/communication ${PROJECT_SRC_DIR}/include/iprange @@ -17,7 +23,16 @@ set(TGT_tde_key_management_INC ${LIBOPENSSL_INCLUDE_PATH} ) +#分别存储了编译选项和链接选项 +#这些选项可能包括宏定义、优化选项、操作系统选项、保护选项、警告选项、二进制安全选项、检查选项等 set(tde_key_management_DEF_OPTIONS ${MACRO_OPTIONS}) set(tde_key_management_COMPILE_OPTIONS ${OPTIMIZE_OPTIONS} ${OS_OPTIONS} ${PROTECT_OPTIONS} ${WARNING_OPTIONS} ${BIN_SECURE_OPTIONS} ${CHECK_OPTIONS}) set(tde_key_management_LINK_OPTIONS ${BIN_LINK_OPTIONS}) + +#最后，使用 `add_static_objtarget` 函数创建名为 `gausskernel_security_tde_key_management` 的静态目标 +#并将 `TGT_tde_key_management_SRC` 和 `TGT_tde_key_management_INC` 参数传递给该函数，同时传递编译和链接选项 add_static_objtarget(gausskernel_security_tde_key_management TGT_tde_key_management_SRC TGT_tde_key_management_INC "${tde_key_management_DEF_OPTIONS}" "${tde_key_management_COMPILE_OPTIONS}" "${tde_key_management_LINK_OPTIONS}") + + +#通过以上配置，可以将 `TGT_tde_key_management_SRC` 中的源文件编译成名为 `gausskernel_security_tde_key_management` 的静态库对象 +#并根据配置的编译和链接选项进行相应的处理和设置 \ No newline at end of file -- 2.34.1 From bebc90731ed95c72ec6c37ced20e5ef80d9935f3 Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:46:45 +0800 Subject: [PATCH 26/48] Update data_common.cpp --- .../tde_key_management/data_common.cpp | 106 +++++++++++++++--- 1 file changed, 91 insertions(+), 15 deletions(-) diff --git a/src/gausskernel/security/tde_key_management/data_common.cpp b/src/gausskernel/security/tde_key_management/data_common.cpp index 43374dd87..b5f304561 100644 --- a/src/gausskernel/security/tde_key_management/data_common.cpp +++ b/src/gausskernel/security/tde_key_management/data_common.cpp @@ -22,22 +22,34 @@ * ------------------------------------------------------------------------- */ +/* +主要是用来实现字符串列表的添加、获取、删除等操作，以及 TDEData 结构体的构造和析构函数。 +在具体业务场景中，这些函数可以用于处理命令参数、配置项等需要存储为字符串列表形式的数据。 +*/ + #include "tde_key_management/data_common.h" #include "utils/elog.h" +//为 AdvStrList 分配内存，并初始化成员变量 AdvStrList *malloc_advstr_list(void) { AdvStrList *new_list = NULL; - + //分配 sizeof(AdvStrList) 大小的内存空间 new_list = (AdvStrList *)palloc0(sizeof(AdvStrList)); + //检查 new_list 是否为 NULL if (new_list == NULL) { + //如果为 NULL，则说明内存分配失败，函数直接返回 NULL return NULL; } - new_list->node_cnt = 0; - new_list->first_node = NULL; + //如果分配成功，则将 new_list 的成员变量进行初始化 + new_list->node_cnt = 0; //表示当前链表中的节点个数为 0 + new_list->first_node = NULL; //表示链表中没有任何节点 + //返回指向该结构体的指针 return new_list; } +//向列表中追加一个字符串节点 +// 从列表的末尾开始遍历，最后将新的字符串节点插入到链表的末尾 void tde_append_node(AdvStrList *list, const char *str_val) { /* last_node -> next = new_node */ @@ -46,58 +58,87 @@ void tde_append_node(AdvStrList *list, const char *str_val) char *new_node_str_val = NULL; errno_t rc = 0; + //分配内存大小为 strlen(str_val) + 1 的空间 new_node_str_val = (char *)palloc0(strlen(str_val) + 1); if (new_node_str_val == NULL) { - return; + return; //如果为 NULL，则说明内存分配失败，函数直接返回 } + + //分配内存大小为 sizeof(AdvStrNode) 的空间 new_node = (AdvStrNode *)palloc0(sizeof(AdvStrNode)); if (new_node == NULL) { pfree_ext(new_node_str_val); - return; + return; //如果为 NULL，则说明内存分配失败，需要释放前面分配的 new_node_str_val 并直接返回 } + //将 str_val 复制到 new_node_str_val 中 rc = strcpy_s(new_node_str_val, strlen(str_val) + 1, str_val); securec_check(rc, "\0", "\0"); - new_node->str_val = new_node_str_val; - new_node->next = NULL; + + new_node->str_val = new_node_str_val; //表示该节点存储的字符串值 + new_node->next = NULL; //表示该节点是链表中的最后一个节点 + if (list->first_node == NULL) { + //如果链表为空 list->first_node = new_node; - } else { + } else { //如果链表不为空 + //将链表的第一个节点 list->first_node 赋值给 last_node，作为遍历链表的起始节点 last_node = list->first_node; + //从第一个节点开始遍历到倒数第二个节点 + //循环结束后，last_node 就指向了链表中的最后一个节点 for (int i = 0; i < (list->node_cnt - 1); i++) { last_node = last_node->next; } + //将新节点添加到最后一个节点的后面，完成节点的追加操作 last_node->next = new_node; } + //更新链表的节点数量 list->node_cnt++; } +//返回 AdvStrList 链表中字符串节点的个数 size_t tde_list_len(AdvStrList *list) { - return list->node_cnt; + return list->node_cnt; //node_cnt 记录了链表中字符串节点的个数 } +//将指定的字符串按照指定的分隔符进行切割 +// 并将每一个切割后的子串存放到 AdvStrList 中的一个字符串节点中 AdvStrList *tde_split_node(const char *str, char split_char) { AdvStrList *substr_list = NULL; char *cur_substr = NULL; size_t str_start = 0; errno_t rc = 0; + + //创建一个 AdvStrList 链表 substr_list = malloc_advstr_list(); if ((substr_list == NULL) || (str == NULL)) { + //如果创建失败或者输入参数 str 为 NULL，则直接返回 NULL return NULL; } + //从字符串 str 的第一个字符开始遍历每个字符 for (size_t i = 0; i < strlen(str); i++) { + //果遍历到的字符是分隔符 split_char 或者已经遍历到了字符串的最后一个字符 + //则可以认为遍历到了一个子字符串的结尾 if (str[i] == split_char || i == strlen(str) - 1) { + //分配内存，将子字符串复制到 cur_substr 中 + // 字符串长度为 i-str_start+1 cur_substr = (char *) palloc0(i - str_start + 1); rc = strncpy_s(cur_substr, i - str_start + 1, str + str_start, i - str_start); securec_check(rc, "\0", "\0"); + //需要再加上一个字节存放字符串结尾标志 '\0' cur_substr[i - str_start] = '\0'; - str_start = i + 1; - tde_append_node(substr_list, cur_substr); + str_start = i + 1; //设为当前子字符串的下一个字符位置，用于计算下一个子字符串的长度和内容 + //将新的节点 cur_substr 添加到链表 substr_list 的末尾 + tde_append_node(substr_list, cur_substr); + //释放 cur_substr 占用的内存 pfree_ext(cur_substr); } } + //完成链表的构建后，判断链表是否为空 if (tde_list_len(substr_list) == 0) { + //如果链表为空，则说明没有找到任何子字符串 + // 需要释放空间并返回 NULL free_advstr_list(substr_list); substr_list = NULL; return NULL; @@ -105,62 +146,90 @@ AdvStrList *tde_split_node(const char *str, char split_char) return substr_list; } +//释放 AdvStrList 占用的内存，包括链表节点、字符串值等 void free_advstr_list(AdvStrList *list) { AdvStrNode *cur_node = NULL; AdvStrNode *to_free = NULL; + //如果输入的链表指针 list 为 NULL if (list == NULL) { - return; + return; //直接返回，不需要进行释放操作 } - cur_node = list->first_node; + //指向链表的第一个节点 + cur_node = list->first_node; + //循环进行遍历，直到遍历完所有的链表节点 while (cur_node != NULL) { to_free = cur_node; cur_node = cur_node->next; - pfree_ext(to_free->str_val); - pfree_ext(to_free); + pfree_ext(to_free->str_val); //释放 to_free->str_val 所指向的 字符串数据 的内存空间 + pfree_ext(to_free); //释放 to_free 所指向的 节点 的内存空间 } + //释放链表头节点 list 的内存空间 pfree_ext(list); } +//根据列表中的位置获取对应的字符串值 char *tde_get_val(AdvStrList *list, int list_pos) { AdvStrNode *target_node = NULL; + // 如果传入的 list_pos 参数为 -1，则表示获取列表中的最后一个节点的值 if (list_pos == -1) { + //将 list_pos 的值设为链表节点数量减 1 list_pos = list->node_cnt - 1; } + + //进行判断，如果 list_pos 的值小于 0 或者大于等于 链表节点数量 list->node_cnt if (list_pos < 0 || list_pos >= list->node_cnt) { + //位置超出范围 return NULL; } + + //指向链表的第一个节点 target_node = list->first_node; + //循环进行遍历，直到找到目标节点位置 for (int i = 0; i < list_pos; i++) { target_node = target_node->next; } + //循环执行了 list_pos 次后，target_node 就指向了目标节点 + //返回目标节点的字符串值 target_node->str_val return target_node->str_val; } +//该函数和 free_advstr_list 相似 +// 区别在于该函数在释放链表时会跳过指定位置上的节点，将其它节点都释放掉 +// 这个函数可以用于一些特定的场景，例如虽然不需要列表中所有的元素，但是有一些需要保留的重要信息 void free_advstr_list_with_skip(AdvStrList *list, int list_pos) { AdvStrNode *cur_node = NULL; AdvStrNode *to_free = NULL; + //如果参数 list_pos 的值为 -1，表示要保留链表中的最后一个节点 if (list_pos == -1) { + //将 list_pos 的值设为链表节点数量减 1 list_pos = list->node_cnt - 1; } + + //指向链表的第一个节点 cur_node = list->first_node; + //循环进行遍历，直到遍历完所有的链表节点 for (int i = 0; i < (int)tde_list_len(list); i++) { to_free = cur_node; cur_node = cur_node->next; + //如果当前节点的位置 i 不等于参数 list_pos if (i != list_pos) { + //说明需要释放该节点和其所包含的字符串数据 pfree_ext(to_free->str_val); pfree_ext(to_free); } } + //释放链表头节点 list 的内存空间 pfree_ext(list); } +//构造函数会为成员变量分配内存，并将它们初始化 TDEData::TDEData() { cmk_id = NULL; @@ -168,13 +237,20 @@ TDEData::TDEData() dek_plaintext = NULL; } +//释放成员变量占用的内存，并清空敏感、私有数据 TDEData::~TDEData() { errno_t rc = 0; + if (dek_plaintext != NULL) { + //dek_plaintext 不为 NULL，表示存在明文密钥数据 + //调用 memset_s 函数对 dek_plaintext 所指向的内存空间进行安全的清零操作 + //使用 strlen(dek_plaintext) 获取需要清零的内存大小 rc = memset_s(dek_plaintext, strlen(dek_plaintext), 0, strlen(dek_plaintext)); + //使用 securec_check 宏检查 memset_s 函数的返回值 rc 是否为成功状态，如果不成功，则会触发错误处理 securec_check(rc, "\0", "\0"); } + //分别使用 pfree_ext 函数释放成员变量 cmk_id、dek_cipher和dek_plaintext 所指向的内存空间 pfree_ext(cmk_id); pfree_ext(dek_cipher); pfree_ext(dek_plaintext); -- 2.34.1 From 7eeb5bc495533d8d2078bbe62961057e2295ab90 Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:47:13 +0800 Subject: [PATCH 27/48] Update http_common.cpp --- .../tde_key_management/http_common.cpp | 107 +++++++++++++++++- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/src/gausskernel/security/tde_key_management/http_common.cpp b/src/gausskernel/security/tde_key_management/http_common.cpp index 390b82a29..4f51a0656 100644 --- a/src/gausskernel/security/tde_key_management/http_common.cpp +++ b/src/gausskernel/security/tde_key_management/http_common.cpp @@ -22,6 +22,12 @@ * ------------------------------------------------------------------------- */ +/* +包含了一个完整的 HTTP 请求过程，包括 +配置 CURL对象、设置请求信息、发送请求、接收响应、解析响应等步骤 +用户可以使用 http_request() 函数来方便地进行 HTTP 请求，并获取相应的结果 +*/ + #include #include #include @@ -29,67 +35,100 @@ #include "securec.h" #include "utils/elog.h" +//是整个流程的入口函数，用户通过调用该函数来发起 HTTP 请求，并返回相应的结果 +// 输入参数包括 http_req_msg（HTTP 请求消息体） +// http_config（HTTP 配置信息） +// http_res_list（HTTP 响应消息列表） HttpErrCode HttpCommon::http_request(HttpReqMsg* http_req_msg, HttpConfig *http_config, AdvStrList **http_res_list) { CURL *sender = NULL; struct curl_slist *http_header = NULL; HttpErrCode ret = TDE_HTTP_SUCCEED; + //全局初始化 libcurl curl_global_init(CURL_GLOBAL_ALL); + //创建 CURL 句柄 sender，使用 curl_easy_init() 初始化 sender = curl_easy_init(); if (sender == NULL) { + //创建 CURL 句柄失败，则抛出错误并返回 TDE_CURL_ERR 错误码 ereport(ERROR, (errcode(ERRCODE_CANNOT_CONNECT_NOW), errmsg("failed to init curl"))); return TDE_CURL_ERR; } + + //设置 HTTP 请求的配置信息 ret = set_http_config(sender, http_config); if (ret != TDE_HTTP_SUCCEED) { curl_easy_cleanup(sender); return ret; } + //设置 HTTP 请求的请求行信息 ret = set_http_reqline(sender, http_req_msg->method, http_req_msg->url); if (ret != TDE_HTTP_SUCCEED) { curl_easy_cleanup(sender); return ret; } + //设置 HTTP 请求的请求头信息 ret = set_http_reqheader(sender, http_req_msg->header_list, http_header); + //释放请求头链表 http_header curl_slist_free_all(http_header); if (ret != TDE_HTTP_SUCCEED) { curl_easy_cleanup(sender); return ret; } + //设置 HTTP 请求的请求体信息 ret = set_http_reqbody(sender, http_req_msg->body); if (ret != TDE_HTTP_SUCCEED) { curl_easy_cleanup(sender); return ret; } + //发送 HTTP 请求并获取响应信息 ret = get_http_resmsg(sender, http_config->res_part, http_res_list); + //清理 CURL 句柄 curl_easy_cleanup(sender); if (ret != TDE_HTTP_SUCCEED) { return ret; } + //全局清理 libcurl curl_global_cleanup(); return TDE_HTTP_SUCCEED; } +//用于设置 CURL 对象的配置信息，包括超时时间、SSL 验证等 +//使用 libcurl 库提供的函数 curl_easy_setopt() 来设置 CURL 对象的选项 HttpErrCode HttpCommon::set_http_config(CURL *http_obj, HttpConfig *http_config) { - CURLcode curl_ret = CURLE_OK; - const int connect_timeout = 10; + //通过检查 curl_ret 的值，可以判断函数调用是否成功执行，或者是否发生了错误 + CURLcode curl_ret = CURLE_OK; //CURLE_OK 是 libcurl 库定义的一个枚举常量，表示没有发生错误的状态码 + const int connect_timeout = 10; //表示连接超时时间为 10 秒 + //通过 curl_easy_setopt() 函数 + //设置 CURL 对象的 CURLOPT_TIMEOUT 选项 + //将 HTTP 请求的超时时间设置为 http_config 结构体中指定的值，并检查返回值是否为 CURLE_OK curl_ret = curl_easy_setopt(http_obj, CURLOPT_TIMEOUT, http_config->timeout); + // 并检查返回值是否为 CURLE_OK check_curl_ret(curl_ret); + + //设置 CURL 对象的 CURLOPT_CONNECTTIMEOUT 选项，将连接超时时间设置为 connect_timeout curl_ret = curl_easy_setopt(http_obj, CURLOPT_CONNECTTIMEOUT, connect_timeout); check_curl_ret(curl_ret); + + //设置 CURL 对象的 CURLOPT_SSL_VERIFYPEER 选项，禁用对服务器 SSL 证书的验证 curl_ret = curl_easy_setopt(http_obj, CURLOPT_SSL_VERIFYPEER, false); check_curl_ret(curl_ret); + + //设置 CURL 对象的 CURLOPT_SSL_VERIFYHOST 选项，禁用对服务器主机名的验证 curl_ret = curl_easy_setopt(http_obj, CURLOPT_SSL_VERIFYHOST, false); check_curl_ret(curl_ret); + return TDE_HTTP_SUCCEED; } +//用于检查 CURL 请求的返回值，并进行相应的处理，例如输出日志、抛出异常等 void HttpCommon::check_curl_ret(CURLcode curl_ret) { + //通过比较 curl_ret 和 CURLE_OK 来判断请求是否成功 if (curl_ret != CURLE_OK) { + //输出错误信息，包括错误码和错误消息 ereport(ERROR, (errcode(ERRCODE_CANNOT_CONNECT_NOW), errmsg("curl error. err code: '%lu', err msg: '%s.", (long unsigned int)curl_ret, curl_easy_strerror(curl_ret)))); @@ -97,26 +136,41 @@ void HttpCommon::check_curl_ret(CURLcode curl_ret) return; } +//用于设置 设置 HTTP 请求的请求行 （ CURL 对象的请求方法和 URL ） +//http_obj 是一个指向 CURL 对象的指针，表示要设置请求行的 HTTP 对象 +//method 是一个枚举类型 HttpMethod，表示 HTTP 请求的方法，可以是 HTTP_POST 或 HTTP_GET +//url 是一个指向字符串的指针，表示请求的 URL HttpErrCode HttpCommon::set_http_reqline(CURL *http_obj, HttpMethod method, const char *url) { CURLcode curl_ret = CURLE_OK; + //根据 method 的值来设置对应的 CURL 选项 switch (method) { case HTTP_POST: + //调用 curl_easy_setopt() 函数并将 CURLOPT_HTTPPOST 设置为 1，表示使用 POST 方法发送请求 curl_ret = curl_easy_setopt(http_obj, CURLOPT_HTTPPOST, 1); break; case HTTP_GET: + //调用 curl_easy_setopt() 函数并将 CURLOPT_HTTPGET 设置为 1，表示使用 GET 方法发送请求 curl_ret = curl_easy_setopt(http_obj, CURLOPT_HTTPGET, 1); break; default: break; } check_curl_ret(curl_ret); + + //设置请求的 URL，将 url 参数作为参数传入 curl_ret = curl_easy_setopt(http_obj, CURLOPT_URL, url); check_curl_ret(curl_ret); + + //返回一个枚举类型 TDE_HTTP_SUCCEED，表示设置请求行成功 return TDE_HTTP_SUCCEED; } +//函数用于设置 HTTP 请求的请求头 ( CURL 对象的请求头部信息 ) +//http_header_list 是一个指向字符串数组的指针，表示要设置的 HTTP 请求头列表 +//数组中的每个元素都是一个字符串，表示一个 HTTP 请求头 +//ret_header_list 是一个指向 curl_slist 结构体的指针，表示返回的请求头列表 HttpErrCode HttpCommon::set_http_reqheader(CURL *http_obj, const char *http_heaser_list[], struct curl_slist* ret_header_list) { @@ -124,57 +178,88 @@ HttpErrCode HttpCommon::set_http_reqheader(CURL *http_obj, const char *http_heas CURLcode curl_ret = CURLE_OK; for (size_t i = 0; http_heaser_list[i] != NULL; i++) { + //将当前的 HTTP 请求头字符串添加到临时列表 tmp_http_header 中 tmp_http_header = curl_slist_append(tmp_http_header, http_heaser_list[i]); + //添加失败（返回值为 NULL） if (tmp_http_header == NULL) { ereport(ERROR, (errcode(ERRCODE_CANNOT_CONNECT_NOW), errmsg("failed to set http header: '%s'", http_heaser_list[i]))); - return TDE_CURL_ERR; + return TDE_CURL_ERR; //输出错误信息并返回 TDE_CURL_ERR，表示设置请求头失败 } } + //将临时请求头列表 tmp_http_header 设置为 CURL 对象的请求头选项 CURLOPT_HTTPHEADER curl_ret = curl_easy_setopt(http_obj, CURLOPT_HTTPHEADER, tmp_http_header); check_curl_ret(curl_ret); ret_header_list = tmp_http_header; + //返回 TDE_HTTP_SUCCEED，表示设置请求头成功 return TDE_HTTP_SUCCEED; } +//用于设置 HTTP 请求的请求体 +//http_body 是一个指向字符串的指针，表示请求的消息体内容 HttpErrCode HttpCommon::set_http_reqbody(CURL *http_obj, const char *http_body) { + //请求体内容 http_body 设置为 CURL 对象的请求体选项 CURLOPT_POSTFIELDS + // 该选项用于指定 HTTP 请求的消息体内容 CURLcode curl_ret = curl_easy_setopt(http_obj, CURLOPT_POSTFIELDS, http_body); check_curl_ret(curl_ret); return TDE_HTTP_SUCCEED; } +//用于获取 CURL 对象的响应消息，并将其解析成字符串列表 +//res_part 是一个枚举类型的变量，表示要获取的响应部分（如响应行、响应头、响应体） +//response_msg 是一个指向 AdvStrList 指针的指针，表示用于存储响应消息体内容的字符串列表 HttpErrCode HttpCommon::get_http_resmsg(CURL *http_obj, HttpResListType res_part, AdvStrList **response_msg) { + //明了一个名为 http_res_list 的结构体变量，用于存储 HTTP 响应的相关信息，包括响应部分、响应行、响应头以及响应体等 HttpResList http_res_list = {"", res_part, HTTP_RESLINE, NULL, 0, 0, false}; const int status_code = 300; - long http_status_code = 0; + long http_status_code = 0; //用于存储实际的 HTTP 响应状态码 CURLcode curl_ret = CURLE_OK; + //通过调用 malloc_advstr_list() 函数来分配内存以创建一个字符串列表，用于存储响应消息体的每一行内容 http_res_list.str_list = malloc_advstr_list(); - if (http_res_list.str_list == NULL) { + if (http_res_list.str_list == NULL) { //分配内存失败 ereport(ERROR, (errcode(ERRCODE_CANNOT_CONNECT_NOW), errmsg("failed to malloc memory"))); return TDE_MALLOC_MEM_ERR; } + + //调用 curl_easy_setopt() 函数将相应的选项设置为 http_obj + //设置 CURLOPT_HEADER 选项为 1，表示希望在响应中包含头部信息 + //该选项告诉 libcurl 将头部信息写入到数据回调函数中 curl_ret = curl_easy_setopt(http_obj, CURLOPT_HEADER, 1); check_curl_ret(curl_ret); + //将 dump_http_response_msg_callback 函数设置为数据回调函数（write function） + //libcurl 在接收到响应数据时将调用此回调函数，用于处理接收到的响应数据。 curl_ret = curl_easy_setopt(http_obj, CURLOPT_WRITEFUNCTION, dump_http_response_msg_callback); check_curl_ret(curl_ret); + // 设置 CURLOPT_WRITEDATA 选项为 &http_res_list + // 表示希望将 http_res_list 的指针作为用户定义的指针传递给数据回调函数 + // 这样，在数据回调函数中可以通过该指针来访问并存储响应数据 curl_ret = curl_easy_setopt(http_obj, CURLOPT_WRITEDATA, (void *) &http_res_list); check_curl_ret(curl_ret); + + //调用 curl_easy_perform() 函数执行 HTTP 请求，并将响应结果保存到 http_res_list 中 curl_ret = curl_easy_perform(http_obj); check_curl_ret(curl_ret); + + //调用 curl_easy_getinfo() 函数获取实际的 HTTP 响应状态码 curl_ret = curl_easy_getinfo(http_obj, CURLINFO_RESPONSE_CODE, &http_status_code); check_curl_ret(curl_ret); + //如果实际状态码大于 status_code if (http_status_code > status_code) { + //输出错误信息 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_CANNOT_CONNECT_NOW), errmsg("curl error code is %d", (int)http_status_code), errdetail("N/A"), errcause("http request status error"), erraction("check curl retrun code status"))); } + //将响应消息体内容的字符串列表传递给调用者 *response_msg = http_res_list.str_list; return TDE_HTTP_SUCCEED; } +//是一个回调函数，用于处理 CURL 响应消息的回调，将响应消息按照指定规则进行解析和保存 +//接收 libcurl 库返回的响应消息，并根据设置的过滤条件来判断是否需要将消息保存到 HttpResList 结构体中 size_t HttpCommon::dump_http_response_msg_callback(void *tmp_cur_res_str, size_t chr_size, size_t cur_res_size, void *tmp_http_res_list) { @@ -182,17 +267,27 @@ size_t HttpCommon::dump_http_response_msg_callback(void *tmp_cur_res_str, size_t const int loop_num = 2; HttpResList *http_res_list = NULL; + //判断输入参数 tmp_cur_res_str 和 tmp_http_res_list 是否为空，如果为空则返回 0 if (tmp_cur_res_str == NULL || tmp_http_res_list == NULL) { return 0; } + //转换为 HttpResList 结构体指针 http_res_list = (HttpResList *)tmp_http_res_list; if (http_res_list->is_stop_dump) { + //如果 http_res_list->is_stop_dump 为 true，则直接返回当前响应消息的大小 return chr_size * cur_res_size; } + //转换为 const char* 类型 const char *cur_res_str = (const char *)tmp_cur_res_str; + //根据设置的过滤条件 http_res_list->filter_res_type + // 和当前情况 http_res_list->cur_pos + // 判断是否需要保存当前响应消息 if (http_res_list->filter_res_type == HTTP_MSG) { + // 过滤条件 为 HTTP_MSG，则需要保存当前响应消息 is_need_dump = true; } else { + // 根据 当前情况 http_res_list->cur_pos 的不同取值，判断是否需要保存当前响应消息 + // 并更新 http_res_list->cur_pos 的值 switch (http_res_list->cur_pos) { case HTTP_RESLINE: if (http_res_list->filter_res_type == HTTP_RESLINE) { @@ -222,7 +317,9 @@ size_t HttpCommon::dump_http_response_msg_callback(void *tmp_cur_res_str, size_t break; } } + //如果需要保存当前响应消息 if (is_need_dump) { + //将消息添加到 http_res_list->str_list 链表中 tde_append_node(http_res_list->str_list, cur_res_str); } return chr_size * cur_res_size; -- 2.34.1 From d191245bb714103ce856ae4ddd34ffa16d989146 Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:47:42 +0800 Subject: [PATCH 28/48] Update kms_httpmsg.ini --- .../security/tde_key_management/kms_httpmsg.ini | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/gausskernel/security/tde_key_management/kms_httpmsg.ini b/src/gausskernel/security/tde_key_management/kms_httpmsg.ini index 68ecb1ef7..aa1fe9859 100644 --- a/src/gausskernel/security/tde_key_management/kms_httpmsg.ini +++ b/src/gausskernel/security/tde_key_management/kms_httpmsg.ini @@ -1,3 +1,5 @@ +;存储了身份认证的请求字符串，用于获取身份认证的令牌 +;包含用户名、密码和域名等信息 const char *iam_auth_token_req = "{" " \"auth\": {" " \"identity\": {" @@ -20,6 +22,9 @@ const char *iam_auth_token_req = "{" " }" "}"; + +;存储了代理认证的请求字符串，用于获取代理认证的令牌 +;包含域名、代理名称和项目名称等信息 const char *iam_agency_token_req = "{" " \"auth\": {" " \"identity\": {" @@ -37,11 +42,17 @@ const char *iam_agency_token_req = "{" " }" "}"; + +;存储了创建数据密钥的请求字符串，用于生成一个新的数据加密密钥 +;包含数据密钥的长度和主密钥的 ID 等信息 const char *kms_create_dek_req = "{" " \"datakey_length\": \"128\"," " \"key_id\": \"$cmk_id$\"" "}"; + +;存储了解密数据密钥的请求字符串，用于解密已加密的数据密钥 +;包含密文、密文长度和主密钥的 ID 等信息 const char *kms_decrypt_dek_req = "{" " \"cipher_text\": \"$dek_cipher$\"," " \"datakey_cipher_length\": \"16\"," -- 2.34.1 From 8c4a8f418dc6c3d99af4e9cfe50c6e6c98834872 Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:48:09 +0800 Subject: [PATCH 29/48] Update kms_interface.cpp --- .../tde_key_management/kms_interface.cpp | 121 +++++++++++++++++- 1 file changed, 119 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/security/tde_key_management/kms_interface.cpp b/src/gausskernel/security/tde_key_management/kms_interface.cpp index 1bb8dc65d..0ff70358e 100755 --- a/src/gausskernel/security/tde_key_management/kms_interface.cpp +++ b/src/gausskernel/security/tde_key_management/kms_interface.cpp @@ -31,20 +31,27 @@ KMSInterface::KMSInterface() KMSInterface::~KMSInterface() {} +// 创建一个 KMS DEK（Data Encryption Key） DekInfo* KMSInterface::create_kms_dek(const char* cmk_id) { DekInfo* create_dek_info = NULL; AdvStrList *http_msg_list = NULL; /* check token valid */ + //检查令牌是否有效 if (!(TDE::CKMSMessage::get_instance().check_token_valid())) { /* get token */ + //如果令牌无效，获取新的令牌 get_kms_token(); } /* create KMS DEK */ + //向 KMS 发送请求，获取 KMS DEK 的信息 http_msg_list = kms_restful_get_dek(cmk_id, NULL, KMS_GEN_DEK); + //对返回的消息列表进行解析 create_dek_info = parser_http_array(http_msg_list, KMS_GEN_DEK); + //释放 http_msg_list 占用的内存 tde_list_free(http_msg_list); if (create_dek_info == NULL) { + //reate_dek_info ，如果为 NULL，则会报错并返回 NULL 值 ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("creating KMS DEK is NULL"))); return NULL; @@ -52,18 +59,24 @@ DekInfo* KMSInterface::create_kms_dek(const char* cmk_id) return create_dek_info; } +//获取 KMS DEK（Data Encryption Key）的明文 char* KMSInterface::get_kms_dek(const char* cmk_id, const char* dek_cipher) { char* dek_plain = NULL; AdvStrList *http_msg_list = NULL; /* check token valid */ + // 检查令牌是否有效 if (!(TDE::CKMSMessage::get_instance().check_token_valid())) { /* get token */ + // 如果令牌无效，获取新的令牌 get_kms_token(); } /* create KMS DEK */ + //向 KMS 发送请求，获取指定 CMK ID 的 KMS DEK 的信息 http_msg_list = kms_restful_get_dek(cmk_id, dek_cipher, KMS_GET_DEK); + //返回的消息列表进行解析，得到 KMS DEK 的明文 dek_plain = parser_http_string(http_msg_list, KMS_GET_DEK); + //释放 http_msg_list 占用的内存 tde_list_free(http_msg_list); if (dek_plain == NULL) { ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), @@ -73,6 +86,7 @@ char* KMSInterface::get_kms_dek(const char* cmk_id, const char* dek_cipher) return dek_plain; } +//获取KMS令牌（Token） void KMSInterface::get_kms_token() { bool result = false; @@ -86,25 +100,38 @@ void KMSInterface::get_kms_token() int len = 0; /* get internal user B token */ + //向 KMS 发送请求获取内部用户 B 的令牌 http_msg_list = kms_restful_token(); + //对返回的消息列表进行解析，得到用户 B 的令牌的值 temp_token = parser_http_string(http_msg_list, IAM_TOKEN); + /* combine token string */ + //根据 req_token_tag 和 temp_token 的值，将它们组合成完整的令牌字符串 len = strlen(req_token_tag) + 1 + strlen(temp_token) + 1; token = (char*)palloc0(len); rc = snprintf_s(token, len, len, "%s:%s", req_token_tag, temp_token); securec_check_ss(rc, "\0", "\0"); + /* get user A agency token */ + //向 KMS 发送请求获取用户 A 代理令牌 http_agency_msg_list = kms_restful_agency_token(token); + //对返回的消息列表进行解析，得到用户 A 代理令牌的值 temp_agency_token = parser_http_string(http_agency_msg_list, IAM_AGENCY_TOKEN); + /* combine agency token string */ + //根据 req_token_tag 和 temp_agency_token 的值，将它们组合成完整的代理令牌字符串，保存在 agency_token 中 len = strlen(req_token_tag) + 1 + strlen(temp_agency_token) + 1; agency_token = (char*)palloc0(len); rc = snprintf_s(agency_token, len, len, "%s:%s", req_token_tag, temp_agency_token); securec_check_ss(rc, "\0", "\0"); + /* cache all token */ + //将用户 B 的令牌和用户 A 的代理令牌缓存起来 result = TDE::CKMSMessage::get_instance().save_token(token, agency_token); + // 释放 http_msg_list 和 http_agency_msg_list 占用的内存 tde_list_free(http_msg_list); tde_list_free(http_agency_msg_list); + //保存令牌的操作失败（result 为 false），则释放所有的指针变量，并报错 if (result == false) { pfree_ext(token); pfree_ext(agency_token); @@ -114,6 +141,7 @@ void KMSInterface::get_kms_token() errmsg("Could KMS token caching is failed"))); return; } + //释放所有的指针变量，并返回 pfree_ext(token); pfree_ext(agency_token); pfree_ext(temp_token); @@ -121,6 +149,7 @@ void KMSInterface::get_kms_token() return; } +//向KMS（Key Management Service）发送HTTP请求 ( 获取IAM令牌 ) 获取Token的功能 AdvStrList* KMSInterface::kms_restful_token() { HttpErrCode ret = TDE_HTTP_SUCCEED; @@ -131,25 +160,38 @@ AdvStrList* KMSInterface::kms_restful_token() int len = 0; errno_t rc = EOK; + //获取CKMSMessage类的单例对象，然后通过该对象的token_info成员变量访问到token_info对象 + // 最后获取该对象的project_name成员变量的值 project_name = TDE::CKMSMessage::get_instance().token_info->project_name; + //构建token_url字符串，该字符串是组合了url_iam_head、project_name和url_token的URL len = strlen(url_iam_head) + strlen(url_token) + strlen(project_name) + 1; token_url = (char*)palloc0(len); rc = memcpy_s(token_url, len, url_iam_head, strlen(url_iam_head)); securec_check(rc, "\0", "\0"); + rc = memcpy_s((token_url + strlen(url_iam_head)), len, project_name, strlen(project_name)); securec_check(rc, "\0", "\0"); + rc = memcpy_s((token_url + strlen(url_iam_head) + strlen(project_name)), len, url_token, (strlen(url_token) + 1)); securec_check(rc, "\0", "\0"); + //获取IAM令牌的JSON字符串 http_body = TDE::CKMSMessage::get_instance().get_iam_token_json(); + //设置HTTP请求的头部信息，其中content_type是一个常量字符串 const char* http_head_list[] = {content_type, NULL}; + //构建http_req_msg结构体，包括请求方法（POST）、URL、头部信息和请求体 HttpReqMsg http_req_msg = {HTTP_POST, token_url, NULL, http_body, http_head_list}; + //构建http_config结构体，设置HTTP请求的超时时间和响应消息类型 HttpConfig http_config = {time_out, HTTP_MSG}; + //发送HTTP请求，并获取响应结果 ret = HttpCommon::http_request(&http_req_msg, &http_config, &http_msg_list); /* token_url is not NULL */ + //释放动态分配的内存，包括token_url和http_body pfree_ext(token_url); pfree_ext(http_body); + //判断HTTP请求的返回状态，如果不成功，抛出错误并输出相应的错误信息 if (ret != TDE_HTTP_SUCCEED) { + // 释放响应消息列表内存空间 tde_list_free(http_msg_list); ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("http request failed"), errdetail("N/A"), errcause("http request error"), @@ -158,6 +200,7 @@ AdvStrList* KMSInterface::kms_restful_token() return http_msg_list; } +//向KMS（Key Management Service）发送HTTP请求获取代理Token的功能 AdvStrList* KMSInterface::kms_restful_agency_token(char* token) { HttpErrCode ret = TDE_HTTP_SUCCEED; @@ -168,26 +211,40 @@ AdvStrList* KMSInterface::kms_restful_agency_token(char* token) int len = 0; errno_t rc = EOK; + // 获取项目名称 project_name = TDE::CKMSMessage::get_instance().agency_token_info->project_name; + // 计算URL长度 len = strlen(url_iam_head) + strlen(url_token) + strlen(project_name) + 1; + // 分配URL内存空间，并初始化为0 agency_token_url = (char*)palloc0(len); + // 将url_iam_head复制到agency_token_url中 rc = memcpy_s(agency_token_url, len, url_iam_head, strlen(url_iam_head)); securec_check(rc, "\0", "\0"); + // 将project_name追加到agency_token_url中 rc = memcpy_s((agency_token_url + strlen(url_iam_head)), len, project_name, strlen(project_name)); securec_check(rc, "\0", "\0"); + // 将url_token追加到agency_token_url中 rc = memcpy_s((agency_token_url + strlen(url_iam_head) + strlen(project_name)), len, url_token, (strlen(url_token) + 1)); securec_check(rc, "\0", "\0"); + // 获取IAM代理Token的JSON请求体内容 http_body = TDE::CKMSMessage::get_instance().get_iam_agency_token_json(); + // HTTP头列表，包含内容类型和身份认证 const char* http_head_list[] = {content_type, token, NULL}; + // 构造HTTP请求消息 HttpReqMsg http_req_msg = {HTTP_POST, agency_token_url, NULL, http_body, http_head_list}; + // HTTP配置，包含超时时间和消息类型 HttpConfig http_config = {time_out, HTTP_MSG}; + //// 发送HTTP请求，并获得响应消息列表 ret = HttpCommon::http_request(&http_req_msg, &http_config, &http_msg_list); /* agency_token_url is not NULL */ + // 释放内存空间 pfree_ext(agency_token_url); pfree_ext(http_body); + // 判断HTTP请求的返回状态，如果不成功，抛出错误并输出相应的错误信息 if (ret != TDE_HTTP_SUCCEED) { + // 释放响应消息列表内存空间 tde_list_free(http_msg_list); ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("http request failed"), errdetail("N/A"), errcause("http request error"), @@ -196,6 +253,7 @@ AdvStrList* KMSInterface::kms_restful_agency_token(char* token) return http_msg_list; } +//使用KMS接口发送HTTP请求获取DEK（Data Encryption Key） AdvStrList* KMSInterface::kms_restful_get_dek(const char* cmk_id, const char* dek_cipher, ResetApiType api_type) { HttpErrCode ret = TDE_HTTP_SUCCEED; @@ -207,40 +265,59 @@ AdvStrList* KMSInterface::kms_restful_get_dek(const char* cmk_id, const char* de int len = 0; errno_t rc = EOK; + //获取项目名称和项目ID project_name = TDE::CKMSMessage::get_instance().kms_info->project_name; project_id = TDE::CKMSMessage::get_instance().kms_info->project_id; + + //根据api_type判断是生成DEK还是获取DEK，并构造相应的URL和HTTP请求体 if (api_type == KMS_GEN_DEK) { + //计算DEK的URL和请求体内容 + //拼接url_kms_head、project_name、url_kms_path、project_id和url_create_dek len = strlen(url_kms_head) + strlen(project_name) + strlen(url_kms_path) + strlen(project_id) + strlen(url_create_dek) + 1; dek_url = (char*)palloc0(len); rc = snprintf_s(dek_url, len, len, "%s%s%s%s%s", url_kms_head, project_name, url_kms_path, project_id, url_create_dek); securec_check_ss(rc, "\0", "\0"); + //生成创建DEK的JSON请求体 http_body = TDE::CKMSMessage::get_instance().get_create_dek_json(cmk_id); } else if (api_type == KMS_GET_DEK) { + //计算并构造了获取DEK的URL和HTTP请求体内容 len = strlen(url_kms_head) + strlen(project_name) + strlen(url_kms_path) + strlen(project_id) + strlen(url_get_dek) + 1; dek_url = (char*)palloc0(len); rc = snprintf_s(dek_url, len, len, "%s%s%s%s%s", url_kms_head, project_name, url_kms_path, project_id, url_get_dek); securec_check_ss(rc, "\0", "\0"); + //生成解密DEK的JSON请求体 http_body = TDE::CKMSMessage::get_instance().get_decrypt_dek_json(cmk_id, dek_cipher); } + + //定义了常量字符指针数组http_head_list，包含了两个HTTP头内容：content_type和TDE::CKMSMessage::get_instance().tde_agency_token const char* http_head_list[] = {content_type, TDE::CKMSMessage::get_instance().tde_agency_token, NULL}; + //定义了HttpReqMsg结构体http_req_msg，设置请求方法为HTTP_POST，URL为dek_url，请求体为http_body，HTTP头列表为http_head_list HttpReqMsg http_req_msg = {HTTP_POST, dek_url, NULL, http_body, http_head_list}; + //定义了HttpConfig结构体http_config，设置超时时间为time_out，消息类型为HTTP_RESBODY HttpConfig http_config = {time_out, HTTP_RESBODY}; + //发送HTTP请求，并将响应消息列表保存到http_msg_list中 ret = HttpCommon::http_request(&http_req_msg, &http_config, &http_msg_list); + + //释放dek_url和http_body的内存空间 pfree_ext(dek_url); pfree_ext(http_body); + //如果HTTP请求失败 if (ret != TDE_HTTP_SUCCEED) { + //释放http_msg_list的内存空间，报告错误信息 tde_list_free(http_msg_list); ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("http request failed"), errdetail("N/A"), errcause("http request error"), erraction("check KMS or IAM connect or config parameter"))); } + //返回http_msg_list作为响应结果 return http_msg_list; } +//用于解析HTTP响应消息列表，根据不同的API类型提取特定的信息，并返回解析结果 char* KMSInterface::parser_http_string(AdvStrList* http_msg_list, ResetApiType api_type) { char* result = NULL; @@ -249,40 +326,57 @@ char* KMSInterface::parser_http_string(AdvStrList* http_msg_list, ResetApiType a errno_t rc = EOK; if ((api_type == IAM_TOKEN) || (api_type == IAM_AGENCY_TOKEN)) { + //api_type为 IAM_TOKEN 或 IAM_AGENCY_TOKEN + //在http_msg_list中查找指定的标签（get_token_tag） token_tmp = find_resheader(http_msg_list, get_token_tag); if (token_tmp == NULL) { + //token_tmp为NULL，则发出错误报告指示无法获取IAM token或IAM agency token，并终止执行 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("get iam token or iam agency token is NULL"), errdetail("N/A"), errcause("connect IAM failed"), erraction("check if your env can connect with IAM server"))); } - result = (char*)palloc0(strlen(token_tmp) + 1); + //为result分配内存空间，大小为token_tmp的长度加1 + result = (char *)palloc0(strlen(token_tmp) + 1); + // 将token_tmp的内容复制到result中 rc = memcpy_s(result, (strlen(token_tmp) + 1), token_tmp, (strlen(token_tmp) + 1)); securec_check(rc, "\0", "\0"); } else if (api_type == KMS_GET_DEK) { + //api_type 为 KMS_GET_DEK + //从http_msg_list中获取第一个元素（即KMS返回的JSON字符串） + // 解析该JSON字符串 cJSON *dek_json = cJSON_Parse(tde_get_val(http_msg_list, 0)); if (cJSON_GetObjectItem(dek_json, data_key) == NULL) { + //如果解析后的JSON对象中没有包含指定的key（data_key） + // 发出错误报告指示KMS返回的dek json key为空，并终止执行 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("KMS dek json key is NULL"), errdetail("N/A"), errcause("KMS return value error"), erraction("check KMS config paramenter"))); } + //从解析后的JSON对象中获取指定key（data_key）的值（即DEK的JSON字符串） json_string = cJSON_GetObjectItem(dek_json, data_key)->valuestring; if (json_string == NULL) { + //如果该值为空，则发出错误报告指示无法获取KMS的DEK，并终止执行 + //删除JSON对象（dek_json） cJSON_Delete(dek_json); ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("get kms dek is NULL"), errdetail("N/A"), errcause("connect KMS failed"), erraction("check if your env can connect with KMS server"))); } + //为result分配内存空间，大小为json_string的长度加1 result = (char*)palloc0(strlen(json_string) + 1); + //将json_string的内容复制到result中 rc = memcpy_s(result, (strlen(json_string) + 1), json_string, (strlen(json_string) + 1)); securec_check(rc, "\0", "\0"); + //删除解析后的JSON对象（dek_json） cJSON_Delete(dek_json); } return result; } +//用于解析HTTP响应消息列表，并根据不同的API类型提取DEK（Data Encryption Key）的明文和密文信息 DekInfo* KMSInterface::parser_http_array(AdvStrList* http_msg_list, ResetApiType api_type) { DekInfo* dek_info = NULL; @@ -291,21 +385,32 @@ DekInfo* KMSInterface::parser_http_array(AdvStrList* http_msg_list, ResetApiType char* cipher_json = NULL; errno_t rc = EOK; + //如果api_type为KMS_GEN_DEK if (api_type == KMS_GEN_DEK) { + //调用tde_get_val函数从http_msg_list中获取第一个元素（即KMS返回的JSON字符串） + // 并使用cJSON_Parse函数解析该JSON字符串 dek_json = cJSON_Parse(tde_get_val(http_msg_list, 0)); + //判断解析后的JSON对象中是否包含指定的key（plain_text和cipher_text） if ((cJSON_GetObjectItem(dek_json, plain_text) == NULL) || (cJSON_GetObjectItem(dek_json, cipher_text) == NULL)) { + //没有或者对应的值为空，则发出错误报告指示 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("KMS dek json key is NULL"), errdetail("N/A"), errcause("KMS return value error"), erraction("check KMS config paramenter"))); } + //从解析后的JSON对象中获取指定key（plain_text和cipher_text）的值 plain_json = cJSON_GetObjectItem(dek_json, plain_text)->valuestring; cipher_json = cJSON_GetObjectItem(dek_json, cipher_text)->valuestring; if ((plain_json == NULL) || (cipher_json == NULL)) { + //如果这两个值中有任何一个为空，则删除解析后的JSON对象（dek_json），并返回NULL cJSON_Delete(dek_json); return NULL; } + //为dek_info分配内存空间 + //并分别为dek_info->plain和dek_info->cipher分配内存空间 + //大小分别为plain_json和cipher_json的长度加1 + //并使用memcpy_s函数将对应的JSON字符串复制到dek_info相应的成员变量中 dek_info = (DekInfo*)palloc0(sizeof(DekInfo)); dek_info->plain = (char*)palloc0(strlen(plain_json) + 1); rc = memcpy_s(dek_info->plain, (strlen(plain_json) + 1), plain_json, (strlen(plain_json) + 1)); @@ -313,28 +418,40 @@ DekInfo* KMSInterface::parser_http_array(AdvStrList* http_msg_list, ResetApiType dek_info->cipher = (char*)palloc0(strlen(cipher_json) + 1); rc = memcpy_s(dek_info->cipher, (strlen(cipher_json) + 1), cipher_json, (strlen(cipher_json) + 1)); securec_check(rc, "\0", "\0"); + //删除解析后的JSON对象（dek_json） cJSON_Delete(dek_json); } + //返回dek_info作为解析结果 return dek_info; } +//在HTTP响应消息头（resheader）列表中查找特定类型的消息头，并返回对应的消息值 char *KMSInterface::find_resheader(AdvStrList *resheader_list, const char *resheader_type) { char *ret = NULL; + //使用for循环遍历resheader_list中的每个元素 for (size_t i = 0; i < tde_list_len(resheader_list); i++) { + //将当前元素按照冒号（:）进行分割，并将分割后得到的header信息保存在cur_header中 AdvStrList *cur_header = tde_split_node(tde_get_val(resheader_list, i), ':'); + //判断cur_header和cur_header的第一个元素（即header类型）是否为空 if ((cur_header == NULL) || (tde_get_val(cur_header, 0) == NULL)) { + //如果有任何一个为空，则发出错误报告指示获取HTTP响应头失败，并终止执行 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("get http header is NULL"), errdetail("N/A"), errcause("http request failed"), erraction("check IAM config parameter"))); } + //如果resheader_list长度大于1且当前消息头的类型等于指定的resheader_type if (tde_list_len(resheader_list) > 1 && strcmp(resheader_type, tde_get_val(cur_header, 0)) == 0) { + //将ret赋值为cur_header的第二个元素（即消息值） ret = tde_get_val(cur_header, 1); + //释放cur_header free_advstr_list_with_skip(cur_header, 1); + //跳出循环 break; - } else { + } else {//如果条件不满足 + //释放cur_header free_advstr_list(cur_header); } } -- 2.34.1 From 028a2299f835996ed5a5662d1113969d4e000dcb Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:49:00 +0800 Subject: [PATCH 30/48] Update Makefile --- .../security/tde_key_management/Makefile | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/security/tde_key_management/Makefile b/src/gausskernel/security/tde_key_management/Makefile index 864f16f0a..8e8a4f5e5 100644 --- a/src/gausskernel/security/tde_key_management/Makefile +++ b/src/gausskernel/security/tde_key_management/Makefile @@ -21,6 +21,23 @@ # # --------------------------------------------------------------------------------------- +#这是一个 TDE 密钥管理模块的 Makefile 文件 +# +#首先，定义了一些变量，包括 subdir、top_builddir，并包含了顶层 Makefile 文件 Makefile.global +# +#接着，使用 override CPPFLAGS 命令将 CPPFLAGS 变量设置为指定的值 +#其中包括 -I $(top_builddir)/src/include/tde_key_management，用于指定头文件的搜索路径 +# +#然后，通过条件判断和命令执行来处理依赖文件 $(DEPEND) 的引入 +#如果当前执行的目标不是 clean 或 distclean，并且系统中存在 g++ 编译器，则包含依赖文件 +#这样可以自动生成依赖关系，确保当源文件变化时能够正确地重新编译 +# +#接下来，定义了一个变量 OBJS，包含了需要编译的目标文件列表 +# +#最后，通过包含 common.mk 文件来完成编译规则的定义和编译目标的生成 +# +#该 Makefile 文件用于编译 TDE 密钥管理模块的源代码，并生成目标文件 + subdir=src/gausskernel/security/tde_key_management top_builddir = ../../../.. include $(top_builddir)/src/Makefile.global @@ -37,4 +54,3 @@ endif OBJS= ckms_message.o kms_interface.o data_common.o tde_key_manager.o tde_key_storage.o http_common.o include $(top_srcdir)/src/gausskernel/common.mk - -- 2.34.1 From 53a8f126c301a42b8ed53d0c876c05cda40f1336 Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:49:47 +0800 Subject: [PATCH 31/48] Update tde_key_manager.cpp --- .../tde_key_management/tde_key_manager.cpp | 65 +++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/src/gausskernel/security/tde_key_management/tde_key_manager.cpp b/src/gausskernel/security/tde_key_management/tde_key_manager.cpp index e2ce065d8..9e3fc9051 100755 --- a/src/gausskernel/security/tde_key_management/tde_key_manager.cpp +++ b/src/gausskernel/security/tde_key_management/tde_key_manager.cpp @@ -31,6 +31,7 @@ #include "utils/memutils.h" #include "knl/knl_session.h" +//默认的构造函数和析构函数，用于初始化和销毁相关资源 TDEKeyManager::TDEKeyManager() { tde_create_data = NULL; @@ -54,6 +55,13 @@ TDEKeyManager::~TDEKeyManager() } } +/* +init()函数用于初始化TDEKeyManager对象 +包括初始化TDE缓存、创建KMSInterface对象以及创建TDEData对象 + +如果TDE::TDEKeyStorage::get_instance().empty() +返回true，则调用TDE::TDEKeyStorage::get_instance().init()进行初始化 +*/ void TDEKeyManager::init() { if (TDE::TDEKeyStorage::get_instance().empty()) { @@ -65,6 +73,8 @@ void TDEKeyManager::init() return; } +//使用KMS接口创建数据加密密钥(DEK)，过KMSInterface对象的create_kms_dek函数创建DEK +// 并将DEK信息保存在tde_create_data中，然后返回tde_create_data指针 const TDEData* TDEKeyManager::create_dek() { errno_t rc = EOK; @@ -72,60 +82,81 @@ const TDEData* TDEKeyManager::create_dek() DekInfo* dek_info = NULL; /* use KMS interface to create DEK */ - cmk_id = get_cmk_id(); - tde_create_data->cmk_id = (char*)palloc0(strlen(cmk_id) + 1); - rc = memcpy_s(tde_create_data->cmk_id, (strlen(cmk_id) + 1), cmk_id, (strlen(cmk_id) + 1)); + cmk_id = get_cmk_id();// 调用get_cmk_id()函数获取cmk_id + tde_create_data->cmk_id = (char*)palloc0(strlen(cmk_id) + 1);// 分配内存并将cmk_id复制到tde_create_data->cmk_id中 + rc = memcpy_s(tde_create_data->cmk_id, (strlen(cmk_id) + 1), cmk_id, (strlen(cmk_id) + 1));// 复制cmk_id到tde_create_data->cmk_id securec_check(rc, "\0", "\0"); + //使用KMSInterface对象的create_kms_dek函数创建DEK，返回一个DekInfo结构体指针dek_info dek_info = kms_instance->create_kms_dek(tde_create_data->cmk_id); + //如果返回值为NULL，则抛出错误，并提供相关错误信息 if (dek_info == NULL) { ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("create KMS dek failed"), errdetail("N/A"), errcause("KMS error"), erraction("check KMS connect or config parameter"))); } + // 分配内存并将dek_info->cipher复制到tde_create_data->dek_cipher中 tde_create_data->dek_cipher = (char*)palloc0(strlen(dek_info->cipher) + 1); rc = memcpy_s(tde_create_data->dek_cipher, (strlen(dek_info->cipher) + 1), dek_info->cipher, (strlen(dek_info->cipher) + 1)); securec_check(rc, "\0", "\0"); + // 分配内存并将dek_info->plain复制到tde_create_data->dek_plaintext中 tde_create_data->dek_plaintext = (char*)palloc0(strlen(dek_info->plain) + 1); + securec_check(rc, "\0", "\0"); rc = memcpy_s(tde_create_data->dek_plaintext, (strlen(dek_info->plain) + 1), dek_info->plain, (strlen(dek_info->plain) + 1)); securec_check(rc, "\0", "\0"); + // 将dek_info->cipher的内容清空 rc = memset_s(dek_info->cipher, strlen(dek_info->cipher), 0, strlen(dek_info->cipher)); securec_check(rc, "\0", "\0"); + + // 释放占用的内存 pfree_ext(dek_info->cipher); pfree_ext(dek_info->plain); pfree_ext(dek_info); + return tde_create_data; } +//获取指定cmk_id和dek_cipher对应的DEK +//通过KMSInterface对象的get_kms_dek函数获取DEK，并将DEK信息保存在tde_get_data变量中 const TDEData* TDEKeyManager::get_dek(const char* cmk_id, const char* dek_cipher) { errno_t rc = EOK; char* dek_plain = NULL; /* use KMS interface to get DEK */ + // 分配内存并将cmk_id复制到tde_get_data->cmk_id中 tde_get_data->cmk_id = (char*)palloc0(strlen(cmk_id) + 1); rc = memcpy_s(tde_get_data->cmk_id, (strlen(cmk_id) + 1), cmk_id, (strlen(cmk_id) + 1)); securec_check(rc, "\0", "\0"); + // 使用KMS实例的get_kms_dek函数获取DEK，返回DEK明文dek_plain dek_plain = kms_instance->get_kms_dek(tde_get_data->cmk_id, dek_cipher); + // 如果返回值为NULL，则抛出错误，并提供相关错误信息 if (dek_plain == NULL) { ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("get KMS dek failed"), errdetail("N/A"), errcause("KMS error"), erraction("check KMS connect or config parameter"))); } + // 分配内存并将dek_cipher复制到tde_get_data->dek_cipher中 tde_get_data->dek_cipher = (char*)palloc0(strlen(dek_cipher) + 1); rc = memcpy_s(tde_get_data->dek_cipher, (strlen(dek_cipher) + 1), dek_cipher, (strlen(dek_cipher) + 1)); securec_check(rc, "\0", "\0"); + // 分配内存并将dek_plain复制到tde_get_data->dek_plaintext中 tde_get_data->dek_plaintext = (char*)palloc0(strlen(dek_plain) + 1); rc = memcpy_s(tde_get_data->dek_plaintext, (strlen(dek_plain) + 1), dek_plain, (strlen(dek_plain) + 1)); securec_check(rc, "\0", "\0"); + // 将dek_plain的内容清空 rc = memset_s(dek_plain, strlen(dek_plain), 0, strlen(dek_plain)); securec_check(rc, "\0", "\0"); + // 释放dek_plain占用的内存 pfree_ext(dek_plain); return tde_get_data; } +//用于获取cmk_id char* TDEKeyManager::get_cmk_id() { + //检查u_sess->attr.attr_security.tde_cmk_id是否为NULL或空字符串 + //如果是则抛出错误 if (u_sess->attr.attr_security.tde_cmk_id == NULL || strlen(u_sess->attr.attr_security.tde_cmk_id) == 0) { ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("get cmk id failed for Transparent Data Encryption"), @@ -134,6 +165,7 @@ char* TDEKeyManager::get_cmk_id() return u_sess->attr.attr_security.tde_cmk_id; } +//用于保存DEK信息到TDE缓存中 bool TDEKeyManager::save_key(const TDEData* tde_data) { errno_t rc = EOK; @@ -142,43 +174,61 @@ bool TDEKeyManager::save_key(const TDEData* tde_data) TimestampTz cur_timestamp = 0; /* prepare TDE cache entry */ + /* + 创建了一个 TDECacheEntry 对象，分配了适当的内存空间， + 并将传入的 TDEData 的属性值复制到对应的成员变量中。 + 具体来说，它使用 palloc0 函数分配了足够的内存空间来存储 DEK 密文和 DEK 明文， + 并使用 memcpy_s 函数将数据从 tde_data 中复制到 kms_cache_entry 对象中。 + */ kms_cache_entry = (TDECacheEntry*)palloc0(sizeof(TDECacheEntry)); kms_cache_entry->key_cipher = (char*)palloc0(strlen(tde_data->dek_cipher) + 1); rc = memcpy_s(kms_cache_entry->key_cipher, (strlen(tde_data->dek_cipher) + 1), tde_data->dek_cipher, (strlen(tde_data->dek_cipher) + 1)); securec_check(rc, "\0", "\0"); + kms_cache_entry->dek_plaintext = (char*)palloc0(strlen(tde_data->dek_plaintext) + 1); rc = memcpy_s(kms_cache_entry->dek_plaintext, (strlen(tde_data->dek_plaintext) + 1), tde_data->dek_plaintext, (strlen(tde_data->dek_plaintext) + 1)); securec_check(rc, "\0", "\0"); + //获取当前的时间戳，并将其赋值给 kms_cache_entry 的 timestamp 成员变量 cur_timestamp = GetCurrentTimestamp(); kms_cache_entry->timestamp = cur_timestamp; /* insert TDE key into TDE cache */ result = TDE::TDEKeyStorage::get_instance().insert_cache(kms_cache_entry); + //插入失败，代码会触发错误处理并返回错误信息 if (!result) { ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("TDE cache insert failed"))); } + //将 kms_cache_entry 的敏感数据 DEK 明文清零，以保护数据的安全性 rc = memset_s(kms_cache_entry->dek_plaintext, strlen(kms_cache_entry->dek_plaintext), 0, strlen(kms_cache_entry->dek_plaintext)); securec_check(rc, "\0", "\0"); + //释放 kms_cache_entry 的内存空间 pfree_ext(kms_cache_entry->key_cipher); pfree_ext(kms_cache_entry->dek_plaintext); pfree_ext(kms_cache_entry); + return result; } +//获取指定cmk_id和dek_cipher对应的DEK密钥 +//首先在TDE缓存中查找是否存在该DEK密钥，如果存在则直接返回 +//如果不存在，则调用get_dek函数获取DEK，并将其保存到TDE缓存中 const char* TDEKeyManager::get_key(const char* cmk_id, const char* dek_cipher) { errno_t rc = EOK; char* kms_cipher = NULL; char* kms_plaintext = NULL; - + //使用 palloc0 函数分配足够的内存空间来存储 dek_cipher 的副本 + // 并使用 memcpy_s 函数将 dek_cipher 的数据复制到 kms_cipher 中 kms_cipher = (char*)palloc0(strlen(dek_cipher) + 1); rc = memcpy_s(kms_cipher, (strlen(dek_cipher) + 1), dek_cipher, (strlen(dek_cipher) + 1)); securec_check(rc, "\0", "\0"); + //调用 TDE::TDEKeyStorage::get_instance().search_cache() 函数，在缓存中搜索与 kms_cipher 匹配的密钥 + // 如果找到了对应的明文密钥，代码会释放 kms_cipher 的内存空间，并直接返回该明文密钥 kms_plaintext = TDE::TDEKeyStorage::get_instance().search_cache(kms_cipher); if (kms_plaintext != NULL) { /* cache has key */ @@ -186,19 +236,26 @@ const char* TDEKeyManager::get_key(const char* cmk_id, const char* dek_cipher) return kms_plaintext; } else { /* key not found */ + //如果在缓存中没有找到匹配的密钥 const TDEData* tde_data = NULL; + //调用 get_dek(cmk_id, kms_cipher) 函数尝试从外部密钥管理系统获取 DEK 数据 tde_data = get_dek(cmk_id, kms_cipher); + //获取失败（返回值为 NULL） if (tde_data == NULL) { + //释放 kms_cipher 的内存空间，并触发错误处理，返回错误信息 pfree_ext(kms_cipher); ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("get KMS DEK is NULL"), errdetail("N/A"), errcause("get KMS dek_plaintext failed"), erraction("check KMS network or cipher is right"))); return kms_plaintext; } + //成功获取到 DEK 数据 /* update to cache */ save_key(tde_data); + //表示获取到了正确的明文密钥 kms_plaintext = tde_data->dek_plaintext; } + //释放 kms_cipher 的内存空间 pfree_ext(kms_cipher); return kms_plaintext; } -- 2.34.1 From 6a7effc925e170100ea15ada8f8f852d238a1be1 Mon Sep 17 00:00:00 2001 From: RrrrrR <13611135068@163.com> Date: Tue, 3 Oct 2023 16:50:12 +0800 Subject: [PATCH 32/48] Update tde_key_storage.cpp --- .../tde_key_management/tde_key_storage.cpp | 218 +++++++++++++++--- 1 file changed, 189 insertions(+), 29 deletions(-) diff --git a/src/gausskernel/security/tde_key_management/tde_key_storage.cpp b/src/gausskernel/security/tde_key_management/tde_key_storage.cpp index fffd9f8e4..6dc5bca6e 100755 --- a/src/gausskernel/security/tde_key_management/tde_key_storage.cpp +++ b/src/gausskernel/security/tde_key_management/tde_key_storage.cpp @@ -29,255 +29,368 @@ #include "access/hash.h" namespace TDE { + +//TDE 模块启动时，初始化 TDE key 管理哈希表 tde_cache，以便后续程序可以向其中添加、查询或删除 TDE key void TDEKeyStorage::init() { + // 检查 tde_cache_mem 是否为空指针 if (tde_cache_mem == nullptr) { + // 如果为空指针，则在全局缓存上下文中创建一个名为 "TDE_CACHE_CONTEXT" 的内存上下文 + // 并将其赋值给 tde_cache_mem 变量 tde_cache_mem = AllocSetContextCreate(g_instance.cache_cxt.global_cache_mem, "TDE_CACHE_CONTEXT", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE, SHARED_CONTEXT); } + // 检查 tde_cache 是否为空指针 if (tde_cache != NULL) { + // 如果不为空指针，表示 TDE 缓存已经初始化过了，直接返回 return; } + // 构建一个 HASHCTL 结构体 tde_ctl HASHCTL tde_ctl; + //将 tde_ctl 的内存块清零 errno_t rc = memset_s(&tde_ctl, sizeof(tde_ctl), 0, sizeof(tde_ctl)); securec_check(rc, "", ""); - tde_ctl.keysize = sizeof(char *); - tde_ctl.entrysize = sizeof(TDECacheEntry); - tde_ctl.hash = (HashValueFunc)tde_cache_entry_hash_func; - tde_ctl.match = (HashCompareFunc)tde_cache_entry_match_func; - tde_ctl.hcxt = tde_cache_mem; + + // 设置 tde_ctl 中的字段值 + tde_ctl.keysize = sizeof(char *); // 哈希表键的大小，这里是指针变量的大小 + tde_ctl.entrysize = sizeof(TDECacheEntry); // 每个哈希表条目的大小 + tde_ctl.hash = (HashValueFunc)tde_cache_entry_hash_func; + tde_ctl.match = (HashCompareFunc)tde_cache_entry_match_func; + tde_ctl.hcxt = tde_cache_mem; // 哈希表使用的内存上下文，之前创建的 tde_cache_mem 上下文 + + //告知该函数在创建哈希表时要使用哪些功能和设置 int flags = HASH_ELEM | HASH_CONTEXT | HASH_FUNCTION | HASH_COMPARE; + // 使用 hash_create 函数创建哈希表 tde_cache tde_cache = hash_create(tde_cache_name, max_bukect, &tde_ctl, flags); + //// 检查哈希表是否创建成功 if (tde_cache == NULL) { + //// 检查哈希表是否创建成功 ereport(ERROR, (errcode(ERRCODE_FUNCTION_HASH_NOT_INITED), errmsg("could not initialize TDE key manager hash table"))); } } +//针对 TDE 缓存哈希表中的每个键值对 +// 基于键的字符串内容计算出一个哈希值 +// 用于确定该键值对在哈希表中的存储位置 uint32 TDEKeyStorage::tde_cache_entry_hash_func(const void* key, Size keysize) { + //通过断言（Assert）来确保传入的参数 keysize 大于 0，并且 key 不为空指针（NULL） Assert(keysize > 0); Assert(key != NULL); + //强制转换为 const char* 类型 const char *dek = *(char **)key; + //计算字符串 dek 的长度 int s_len = strlen(dek); + //使用 PostgreSQL 提供的 hash_any 函数 + // 将 dek 强制转换为 const unsigned char* 类型 + // 并指定字符串长度 s_len，计算出哈希值 + // 然后，使用 DatumGetUInt32 将哈希值转换为 uint32 类型，并返回哈希值结果 return (DatumGetUInt32(hash_any((const unsigned char *)dek, s_len))); } +//用于比较两个密钥缓存条目的匹配情况 int TDEKeyStorage::tde_cache_entry_match_func(const void* key1, const void* key2, Size keySize) { + //通过断言（Assert）来确保输入参数的有效性 Assert(keySize > 0); Assert(key1 != NULL); Assert(key2 != NULL); /* de-reference char ** to char * to do comparison */ + //强制转换为 char ** 类型的指针，并通过解引用操作得到 char * 类型的指针 const char *dek1 = *(char **)key1; const char *dek2 = *(char **)key2; + //计算 dek1 和 dek2 字符串的长度 int dek1_len = strlen(dek1); int dek2_len = strlen(dek2); + //基于字符串内容比较，较短的字符串会被自动填充到与较长字符串相同的长度再进行比较 + // 返回值为 0 表示匹配，非零值表示不匹配 return (strncmp(dek1, dek2, ((dek1_len > dek2_len) ? dek1_len : dek2_len))); } +//构造函数 TDEKeyStorage::TDEKeyStorage() { - tde_cache_mem = nullptr; - tde_cache = NULL; + tde_cache_mem = nullptr; //表示缓存内存上下文还未分配 + tde_cache = NULL; //表示缓存对象未初始化 } +//析构函数 TDEKeyStorage::~TDEKeyStorage() { - reset(); - if (tde_cache_mem != nullptr) { - MemoryContextDelete(tde_cache_mem); - tde_cache_mem = nullptr; + reset(); //将 TDEKeyStorage 对象重置为初始状态 + if (tde_cache_mem != nullptr) { //如果不为空 + MemoryContextDelete(tde_cache_mem); //删除内存上下文 + tde_cache_mem = nullptr; //表示内存上下文已被删除 } } +//用于检查 TDE 缓存是否为空 bool TDEKeyStorage::empty() { if (tde_cache == NULL) { + //如果返回 true，则表示缓存为空 return true; } return false; } +//重置 TDE 缓存，并释放相关资源 void TDEKeyStorage::reset() { + //清空缓存中的数据 clear(); + //检查 tde_cache 是否为 NULL if (tde_cache != NULL) { + //获取 TDEKeyCacheLock 锁，以确保在访问缓存期间不会发生冲突 LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE); + //将 tde_cache 重置为初始状态 HeapMemResetHash(tde_cache, tde_cache_name); - tde_cache = NULL; + tde_cache = NULL; //表示缓存已经被重置 + //释放 TDEKeyCacheLock 锁，确保其他线程可以访问缓存 LWLockRelease(TDEKeyCacheLock); } } +//清空 TDEKeyStorage 对象中的缓存数据，以便重新使用或释放内存 void TDEKeyStorage::clear() { + //检查 tde_cache 是否为非空指针 if (tde_cache != NULL) { + // 获取 TDEKeyCacheLock 互斥锁 LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE); + + // 初始化哈希表扫描器 HASH_SEQ_STATUS scan_state; hash_seq_init(&scan_state, tde_cache); + + // 遍历哈希表中的所有条目，并删除它们 TDECacheEntry* item = NULL; while ((item = reinterpret_cast(hash_seq_search(&scan_state))) != NULL) { + // 清空缓存中数据，并释放其占用的内存空间 clean_cache_entry_value(item->dek_plaintext); + // 删除当前条目 if (hash_search(tde_cache, (const void*)&item->key_cipher, HASH_REMOVE, NULL) == NULL) { + // 如果删除失败，则释放互斥锁并触发错误 LWLockRelease(TDEKeyCacheLock); ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("TDE Key storage clear cache: remove entry failed"))); } } + // 释放 TDEKeyCacheLock 互斥锁 LWLockRelease(TDEKeyCacheLock); } } +//将指定的缓存对象（即 dek_plaintext）中的内容清空为全零 void TDEKeyStorage::reset_dek_plaintext(char* dek_plaintext) { errno_t rc = 0; + //调用标准库函数 memset_s()，将 dek_plaintext 中的内容设置为全零 rc = memset_s(dek_plaintext, strlen(dek_plaintext), 0, strlen(dek_plaintext)); securec_check(rc, "\0", "\0"); } +//将指定的缓存对象（即 dek_plaintext）中的内容清空为全零，并释放其占用的内存空间 void TDEKeyStorage::clean_cache_entry_value(char* dek_plaintext) { + //检查 dek_plaintext 是否为非空指针 if (dek_plaintext != NULL) { + //调用 reset_dek_plaintext() 方法，将其内容清空为全零 reset_dek_plaintext(dek_plaintext); + //释放 dek_plaintext 占用的内存空间 pfree_ext(dek_plaintext); } } +//用于向 tde_cache 哈希表中插入或更新缓存数据 bool TDEKeyStorage::insert_cache(TDECacheEntry* tde_cache_entry) { bool found = false; errno_t rc = EOK; + //切换内存上下文至 tde_cache_mem MemoryContext old = MemoryContextSwitchTo(tde_cache_mem); /* build tde cache key */ + //根据 tde_cache_entry 的 key_cipher 字段的长度动态分配内存 char* tde_cache_key = (char*)palloc0(strlen(tde_cache_entry->key_cipher) + 1); + //将 key_cipher 字符串拷贝到新分配的内存中 rc = memcpy_s(tde_cache_key, (strlen(tde_cache_entry->key_cipher) + 1), tde_cache_entry->key_cipher, (strlen(tde_cache_entry->key_cipher) + 1)); securec_check(rc, "\0", "\0"); /* insert hash table */ + //获取 TDEKeyCacheLock 读写锁 LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE); TDECacheEntry* entry = NULL; + //调用 hash_search() 函数在哈希表 tde_cache 中查找是否存在 tde_cache_key 等于 tde_cache_entry->key_cipher 的缓存项 entry = reinterpret_cast(hash_search(tde_cache, (const void*)&tde_cache_key, HASH_FIND, &found)); if (!found) { + //如果未找到对应的缓存项，则调用 hash_search() 函数以 HASH_ENTER 操作插入新的缓存项 entry = reinterpret_cast (hash_search(tde_cache, (const void*)&tde_cache_key, HASH_ENTER, &found)); if (entry == NULL) { + //如果插入失败（返回 NULL），则释放相关内存 pfree_ext(tde_cache_key); - LWLockRelease(TDEKeyCacheLock); - MemoryContextSwitchTo(old); + LWLockRelease(TDEKeyCacheLock); //释放锁 + MemoryContextSwitchTo(old); //恢复内存上下文 + //抛出异常报错 ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmsg("build TDE key cache hash table failed"))); return false; } /* insert new cache key-value */ + //如果插入成功，分配内存并将 dek_plaintext 字符串拷贝到新分配的内存中 entry->dek_plaintext = (char*)palloc0(strlen(tde_cache_entry->dek_plaintext) + 1); rc = memcpy_s(entry->dek_plaintext, (strlen(tde_cache_entry->dek_plaintext) + 1), tde_cache_entry->dek_plaintext, (strlen(tde_cache_entry->dek_plaintext) + 1)); securec_check(rc, "\0", "\0"); + //更新缓存的时间戳 entry->timestamp = tde_cache_entry->timestamp; - } else { + } else { //如果找到对应的缓存项 /* update cache key-value */ + //更新缓存中的 dek_plaintext 字段，以及时间戳字段 rc = memcpy_s(entry->dek_plaintext, (strlen(tde_cache_entry->dek_plaintext) + 1), tde_cache_entry->dek_plaintext, (strlen(tde_cache_entry->dek_plaintext) + 1)); securec_check(rc, "\0", "\0"); entry->timestamp = tde_cache_entry->timestamp; } + //恢复先前的内存上下文，并释放锁 MemoryContextSwitchTo(old); LWLockRelease(TDEKeyCacheLock); + //返回 true，表示插入或更新缓存成功 return true; } +//根据给定的 dek_cipher 在哈希表中搜索缓存项，并返回找到的缓存项的明文密钥 char* TDEKeyStorage::search_cache(const char* dek_cipher) { bool found = false; + //切换到 tde_cache_mem 内存上下文 MemoryContext old = MemoryContextSwitchTo(tde_cache_mem); + //获取锁 LWLockAcquire(TDEKeyCacheLock, LW_SHARED); + TDECacheEntry* entry = NULL; + //通过 hash_search() 函数在哈希表 tde_cache 中查找与给定的 dek_cipher 相关的缓存项 entry = reinterpret_cast(hash_search(tde_cache, (const void*)&dek_cipher, HASH_FIND, &found)); if (!found) { + //如果未找到对应的缓存项，则恢复到之前的内存上下文 old MemoryContextSwitchTo(old); + //释放锁 TDEKeyCacheLock，并返回 NULL，表示未找到缓存项 LWLockRelease(TDEKeyCacheLock); return NULL; } + //找到了对应的缓存项，同样恢复到之前的内存上下文 old，释放锁 MemoryContextSwitchTo(old); LWLockRelease(TDEKeyCacheLock); + //返回缓存项的明文密钥 dek_plaintext 字符串 return entry->dek_plaintext; } +//实现了定期清理过期的缓存项，保持 TDE 密钥缓存的有效性 void TDEKeyStorage::cache_watch_dog() { TimestampTz cur_timestamp = 0; if (tde_cache != NULL) { + //如果哈希表 tde_cache 不为空，则获取当前时间戳并赋值给 cur_timestamp cur_timestamp = GetCurrentTimestamp(); + //以独占模式获取锁 LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE); + //初始化哈希表遍历的状态 scan_state，并使用 hash_seq_init() 函数初始化遍历状态，将其与哈希表 tde_cache 相关联 HASH_SEQ_STATUS scan_state; hash_seq_init(&scan_state, tde_cache); + TDECacheEntry* item = NULL; + //通过循环调用 hash_seq_search() 函数遍历哈希表中的缓存项 while ((item = reinterpret_cast(hash_seq_search(&scan_state))) != NULL) { + //如果遍历到了缓存项，则进入循环体中的代码块 if ((cur_timestamp - item->timestamp) > USECS_PER_HOUR) { + //检查当前缓存项的时间戳与当前时间戳之差是否大于一个小时（即判断是否过期） + //清理缓存项的值 clean_cache_entry_value(item->dek_plaintext); + //使用 hash_search() 函数在哈希表中删除该缓存项 if (hash_search(tde_cache, (const void*)&item->key_cipher, HASH_REMOVE, NULL) == NULL) { + //如果删除失败，则释放锁 TDEKeyCacheLock，抛出异常报错，提示删除缓存项失败 LWLockRelease(TDEKeyCacheLock); ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("TDE Key storage WatchDog cache: remove entry failed"))); } } } + //释放锁 LWLockRelease(TDEKeyCacheLock); } return; } + /* TDE buffer rel file node cache management */ +//构造函数 TDEBufferCache::TDEBufferCache() { - tde_buffer_mem = nullptr; - tde_buffer_cache = NULL; + tde_buffer_mem = nullptr; //表示内存上下文还未分配 + tde_buffer_cache = NULL; //表示缓存对象未初始化 } - +//析构函数 TDEBufferCache::~TDEBufferCache() { - reset(); + reset(); //进行重置操作 + //判断是否分配了内存上下文 if (tde_buffer_mem != nullptr) { - MemoryContextDelete(tde_buffer_mem); - tde_buffer_mem = nullptr; + MemoryContextDelete(tde_buffer_mem); //删除该内存上下文 + tde_buffer_mem = nullptr; //将 tde_buffer_mem 置为 nullptr，以释放相关的内存资源 } } +//用于检查 TDE 缓存是否为空 bool TDEBufferCache::empty() { if (tde_buffer_cache == NULL) { + //如果 TDE 缓存中没有任何缓存条目，返回 true return true; } + //否则，返回 false，表示缓存非空 return false; } +//实现了重置 TDE 缓冲区缓存的功能，包括清空缓存和重新初始化相关数据结构 void TDEBufferCache::reset() { - clear(); + clear(); //清空缓存 + //判断是否已经分配了缓存结构 if (tde_buffer_cache != NULL) { + //以独占模式获取锁 LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE); + //重新初始化缓存的哈希表结构 tde_buffer_cache HeapMemResetHash(tde_buffer_cache, tde_buffer_name); - tde_buffer_cache = NULL; + tde_buffer_cache = NULL; //表示缓存已被重置 + //释放锁 LWLockRelease(TDEKeyCacheLock); } } +//实现了清空 TDE 缓冲区缓存的功能 void TDEBufferCache::clear() { + //检查 tde_buffer_cache 是否为 NULL，判断是否已经分配了缓存结构 if (tde_buffer_cache != NULL) { + //以独占模式获取锁 LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE); + //初始化哈希表扫描状态 scan_state，并将其与缓存结构 tde_buffer_cache 相关联 HASH_SEQ_STATUS scan_state; hash_seq_init(&scan_state, tde_buffer_cache); - TdeFileNodeEntry* item = NULL; + //用 hash_seq_search() 函数遍历哈希表中的每个项，并将当前项的指针保存在变量 item 中 + TdeFileNodeEntry *item = NULL; + // 在循环中，检查变量 item 是否为空 while ((item = reinterpret_cast(hash_seq_search(&scan_state))) != NULL) { + // 使用 hash_search() 函数将当前项从哈希表中移除 if (hash_search(tde_buffer_cache, (const void*)&item->tde_node, HASH_REMOVE, NULL) == NULL) { + //如果移除失败（返回值为 NULL），则释放锁 ，并抛出一个错误报告 LWLockRelease(TDEKeyCacheLock); ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_DATA_EXCEPTION), errmsg("clear TDE buffer cache key failed"), errdetail("N/A"), @@ -285,103 +398,150 @@ void TDEBufferCache::clear() erraction("check TDE_BUFFER_CACHE_CONTEXT or system cache"))); } } + //如果为空，表示已经遍历完哈希表中的所有项，退出循环 + //释放锁 LWLockRelease(TDEKeyCacheLock); } } +//用于创建和初始化缓存结构，并分配相应的内存空间 void TDEBufferCache::init() { + //检查全局变量 tde_buffer_mem 是否为空 if (tde_buffer_mem == nullptr) { + //如果为空，创建一个分配集合上下文 TDE_BUFFER_CACHE_CONTEXT + // 该上下文使用全局缓存内存 g_instance.cache_cxt.global_cache_mem，并设置默认的内存大小 tde_buffer_mem = AllocSetContextCreate(g_instance.cache_cxt.global_cache_mem, "TDE_BUFFER_CACHE_CONTEXT", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE, SHARED_CONTEXT); } + //检查缓存结构 tde_buffer_cache 是否为空 if (tde_buffer_cache != NULL) { + //如果不为空，则说明已经初始化过，直接返回，不再重复初始化 return; } + //如果 tde_buffer_cache 为空，则继续执行以下操作 /* initialize the hash table */ + //初始化 HASHCTL 结构体 tde_buffer_ctl，并将其清零 HASHCTL tde_buffer_ctl; errno_t rc = memset_s(&tde_buffer_ctl, sizeof(tde_buffer_ctl), 0, sizeof(tde_buffer_ctl)); securec_check(rc, "", ""); + //设置哈希表的键大小 keysize 为 sizeof(RelFileNode) + // 条目大小 entrysize 为 sizeof(TdeFileNodeEntry) + // 设置哈希函数 hash 为 tag_hash + // 上下文 hcxt 为 tde_buffer_mem tde_buffer_ctl.keysize = sizeof(RelFileNode); tde_buffer_ctl.entrysize = sizeof(TdeFileNodeEntry); tde_buffer_ctl.hash = tag_hash; tde_buffer_ctl.hcxt = tde_buffer_mem; + //创建哈希表 tde_buffer_cache，指定哈希表的名称为 tde_buffer_name，最大桶的数量为 max_bucket + // 并使用上述设置的 tde_buffer_ctl 进行初始化 tde_buffer_cache = hash_create(tde_buffer_name, max_bucket, &tde_buffer_ctl, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); + //如果创建哈希表失败（返回值为 NULL） if (tde_buffer_cache == NULL) { + //抛出一个错误报告 ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_FUNCTION_HASH_NOT_INITED), errmsg("init TDE buffer cache is NULL"), errdetail("N/A"), errcause("initialize cache is failed"), erraction("check TDE_BUFFER_CACHE_CONTEXT or system cache"))); } } +//向TDE缓冲区缓存中插入或更新缓存项 bool TDEBufferCache::insert_cache(RelFileNode tde_rnode, TdeInfo *tde_info) { bool found = false; errno_t rc = EOK; + //切换为TDE缓冲区缓存的上下文tde_buffer_mem MemoryContext old = MemoryContextSwitchTo(tde_buffer_mem); /* build tde cache key */ + //根据传入的关系文件节点RelFileNode tde_rnode构建TDE缓存的键RelFileNode tde_key RelFileNode tde_key = tde_rnode; /* insert hash table */ + //获取TDE缓冲区缓存的写锁 LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE); + TdeFileNodeEntry *entry = NULL; + //调用hash_search()函数在TDE缓冲区哈希表中查找关键字为tde_key的缓存项 entry = reinterpret_cast(hash_search(tde_buffer_cache, (const void*)&tde_key, HASH_FIND, &found)); - if (!found) { + if (!found) { //如果未找到缓存项，则表示需要插入新的缓存项 + //调用hash_search()函数，并将操作设置为HASH_ENTER，将新的关键字-值对插入到TDE缓冲区哈希表中 entry = reinterpret_cast(hash_search(tde_buffer_cache, (const void*)&tde_key, HASH_ENTER, &found)); - if (entry == NULL) { + if (entry == NULL) { //如果插入失败（返回值为NULL） + //释放写锁，恢复之前的内存上下文，并返回false表示插入失败 LWLockRelease(TDEKeyCacheLock); MemoryContextSwitchTo(old); return false; } /* insert new cache key-value */ + //如果插入成功，就需要给新的缓存项分配内存 entry->tde_info = (TdeInfo*)palloc0(sizeof(TdeInfo)); + //并将传入的tde_info中的字段值复制到缓存项对应的字段中 rc = strcpy_s(entry->tde_info->dek_cipher, DEK_CIPHER_LEN, tde_info->dek_cipher); securec_check(rc, "\0", "\0"); rc = strcpy_s(entry->tde_info->cmk_id, CMK_ID_LEN, tde_info->cmk_id); securec_check(rc, "\0", "\0"); entry->tde_info->algo = tde_info->algo; - } else { + } else { //在找到了缓存项的情况下，更新该缓存项的逻辑 /* update cache key-value */ + //通过比较缓存项中的dek_cipher字段和传入的tde_info->dek_cipher字段的值，判断是否需要更新 if (strncmp(entry->tde_info->dek_cipher, tde_info->dek_cipher, DEK_CIPHER_LEN) != 0) { + //如果不相等，将传入的tde_info->dek_cipher复制到缓存项的dek_cipher字段中 rc = strcpy_s(entry->tde_info->dek_cipher, DEK_CIPHER_LEN, tde_info->dek_cipher); securec_check(rc, "\0", "\0"); } + //通过对比缓存项中的cmk_id字段和传入的tde_info->cmk_id字段的值，判断是否需要更新 if (strncmp(entry->tde_info->cmk_id, tde_info->cmk_id, CMK_ID_LEN) != 0) { + //果不相等，则使用strcpy_s()函数将传入的tde_info->cmk_id复制到缓存项的cmk_id字段中 rc = strcpy_s(entry->tde_info->cmk_id, CMK_ID_LEN, tde_info->cmk_id); securec_check(rc, "\0", "\0"); } + //对比缓存项中的algo字段和传入的tde_info->algo字段的值，判断是否需要更新 if (entry->tde_info->algo != tde_info->algo) { + //如果不相等，则将传入的tde_info->algo赋值给缓存项的algo字段 entry->tde_info->algo = tde_info->algo; } } + //释放写锁，并将内存上下文切换回之前的上下文 MemoryContextSwitchTo(old); LWLockRelease(TDEKeyCacheLock); + //返回true表示插入或更新操作成功 return true; } +//在TDE缓冲区的哈希表中搜索与传入的tde_rnode相匹配的缓存项 +//如果找到了缓存项，则将缓存项中的tde_info复制到传入的tde_info中 +// 如果未找到缓存项，则直接返回 void TDEBufferCache::search_cache(RelFileNode tde_rnode, TdeInfo *tde_info) { bool found = false; + //传入的tde_rnode的bucketNode字段不是无效的桶ID(InvalidBktId)，则将其设置为SegmentBktId if (tde_rnode.bucketNode != InvalidBktId) { tde_rnode.bucketNode = SegmentBktId; } + //切换到tde_buffer_mem内存上下文 MemoryContext old = MemoryContextSwitchTo(tde_buffer_mem); + //获取一个共享锁，即读取锁 LWLockAcquire(TDEKeyCacheLock, LW_SHARED); + TdeFileNodeEntry* entry = NULL; + //在TDE缓冲区哈希表中查找与传入的tde_rnode对应的缓存项 entry = reinterpret_cast(hash_search(tde_buffer_cache, (const void*)&tde_rnode, HASH_FIND, &found)); - if (!found) { + if (!found) { //没有找到缓存项 + //恢复之前的内存上下文，释放锁，并直接返回 MemoryContextSwitchTo(old); LWLockRelease(TDEKeyCacheLock); return; } - + //如果找到了缓存项 + // 使用memcpy_s() 函数将缓存项中的tde_info复制到传入的tde_info中 errno_t rc = memcpy_s(tde_info, sizeof(TdeInfo), entry->tde_info, sizeof(TdeInfo)); securec_check(rc, "\0", "\0"); + //恢复之前的内存上下文，释放锁，并返回 MemoryContextSwitchTo(old); LWLockRelease(TDEKeyCacheLock); return; -- 2.34.1 From 49edc13014fd8ee2a40e8ac6daeda77b07b7b74e Mon Sep 17 00:00:00 2001 From: Auweafan Date: Tue, 3 Oct 2023 17:48:06 +0800 Subject: [PATCH 33/48] Update gs_policy_masking.cpp --- .../security/gs_policy/gs_policy_masking.cpp | 954 ++++++++++++------ 1 file changed, 637 insertions(+), 317 deletions(-) diff --git a/src/gausskernel/security/gs_policy/gs_policy_masking.cpp b/src/gausskernel/security/gs_policy/gs_policy_masking.cpp index 0af803079..b8bbce7f3 100644 --- a/src/gausskernel/security/gs_policy/gs_policy_masking.cpp +++ b/src/gausskernel/security/gs_policy/gs_policy_masking.cpp @@ -49,12 +49,22 @@ #include "utils/snapmgr.h" -LoadPoliciesPtr load_masking_policies_hook = NULL; -LoadPolicyAccessPtr load_masking_policy_actions_hook = NULL; -LoadPolicyFilterPtr load_masking_policy_filter_hook = NULL; -ValidateBehaviourPtr validate_masking_behaviour_hook = NULL; -VerifyLabelsByPolicy gs_verify_labels_by_policy_hook = NULL; +LoadPoliciesPtr load_masking_policies_hook = NULL;/* 定义函数钩子 LoadPoliciesPtr，用于加载脱敏策略 */ +LoadPolicyAccessPtr load_masking_policy_actions_hook = NULL;/* 定义函数钩子 LoadPolicyAccessPtr，用于加载脱敏策略的操作权限 */ +LoadPolicyFilterPtr load_masking_policy_filter_hook = NULL;/* 定义函数钩子 LoadPolicyFilterPtr，用于加载脱敏策略的过滤条件 */ +ValidateBehaviourPtr validate_masking_behaviour_hook = NULL;/* 定义函数钩子 ValidateBehaviourPtr，用于验证脱敏行为 */ +VerifyLabelsByPolicy gs_verify_labels_by_policy_hook = NULL;/* 定义函数钩子 VerifyLabelsByPolicy，用于通过脱敏策略验证标签 */ + +/* +"creditcardmasking" - 信用卡脱敏 +"basicemailmasking" - 基本电子邮件脱敏 +"fullemailmasking" - 完整电子邮件脱敏 +"alldigitsmasking" - 所有数字脱敏 +"shufflemasking" - 打乱顺序脱敏 +"randommasking" - 随机脱敏 +"regexpmasking" - 正则表达式脱敏 +*/ static const char* g_maskFunctions[] = { "creditcardmasking", "basicemailmasking", @@ -70,50 +80,72 @@ static void reload_masking_policies() { if (!OidIsValid(u_sess->proc_cxt.MyDatabaseId)) { return; - } + }//检查当前会话的数据库ID是否有效，如果无效，则直接返回，不进行后续操作。 if (load_masking_policies_hook != NULL) { - load_masking_policies_hook(false); + load_masking_policies_hook(false);//调用该函数以重新加载脱敏策略,调用时传入的参数是 false，表示不强制重新加载。 } if (load_masking_policy_actions_hook != NULL) { - load_masking_policy_actions_hook(false); + load_masking_policy_actions_hook(false);//调用该函数以重新加载脱敏策略的动作权限,调用时传入的参数是 false，表示不强制重新加载。 } /* load filters must be last */ if (load_masking_policy_filter_hook != NULL) { - load_masking_policy_filter_hook(false); + load_masking_policy_filter_hook(false);//调用该函数以重新加载脱敏策略的过滤条件,调用时传入的参数是 false，表示不强制重新加载。 } } #define BUFFSIZE 512 -bool PgPolicyMaskingActionStruct::operator == (const PgPolicyMaskingActionStruct &arg) const +bool PgPolicyMaskingActionStruct::operator==(const PgPolicyMaskingActionStruct &arg) const { + // 使用 < 运算符重载函数判断对象自身是否小于参数对象 if (*this < arg) { return false; - } else if (arg < *this) { + } + // 使用 < 运算符重载函数判断参数对象是否小于对象自身 + else if (arg < *this) { return false; } else { + // 对象相等 return true; } } -bool PgPolicyMaskingActionStruct::operator < (const PgPolicyMaskingActionStruct &arg) const +bool PgPolicyMaskingActionStruct::operator<(const PgPolicyMaskingActionStruct &arg) const { + // 比较对象自身的 m_type 和参数对象的 m_type 字符串 int res = strcasecmp(m_type.c_str(), arg.m_type.c_str()); - if (res < 0)return true; - if (res > 0)return false; + // 如果对象自身的 m_type 小于参数对象的 m_type，返回 true + if (res < 0) + return true; + // 如果对象自身的 m_type 大于参数对象的 m_type，返回 false + if (res > 0) + return false; + + // 比较对象自身的 m_label_name 和参数对象的 m_label_name 字符串 res = strcasecmp(m_label_name.c_str(), arg.m_label_name.c_str()); - if (res < 0)return true; - if (res > 0)return false; + // 如果对象自身的 m_label_name 小于参数对象的 m_label_name，返回 true + if (res < 0) + return true; + // 如果对象自身的 m_label_name 大于参数对象的 m_label_name，返回 false + if (res > 0) + return false; + + // 如果 m_type 和 m_label_name 都相等，比较 m_policy_oid return m_policy_oid < arg.m_policy_oid; } -int PgPolicyMaskingActionStruct::operator - (const PgPolicyMaskingActionStruct &arg) const + +int PgPolicyMaskingActionStruct::operator-(const PgPolicyMaskingActionStruct &arg) const { + // 使用 < 运算符重载函数判断对象自身是否小于参数对象 if (*this < arg) { return -1; - } else if (arg < *this) { + } + // 使用 < 运算符重载函数判断参数对象是否小于对象自身 + else if (arg < *this) { return 1; } else { + // 对象相等 return 0; } } @@ -127,13 +159,13 @@ typedef gs_stl::gs_mapactiontype.data; - item.m_params = rel_data->actparams.data; - item.m_label_name = rel_data->actlabelname.data; - item.m_policy_oid = (long long)(rel_data->policyoid); - /* - * when policy_oid is zero - load all privileges - * when policy_oid greater than zero, load only matching + + item.m_id = HeapTupleGetOid(rtup); // 获取操作项的 OID + item.m_type = rel_data->actiontype.data; // 获取操作项的类型 + item.m_params = rel_data->actparams.data; // 获取操作项的参数 + item.m_label_name = rel_data->actlabelname.data; // 获取操作项的标签名称 + item.m_policy_oid = (long long)(rel_data->policyoid); // 获取操作项的策略 OID + + /* + * 当 policy_oid 为零时，加载所有权限； + * 当 policy_oid 大于零时，只加载匹配的权限。 */ if ((policy_oid == 0) || ((policy_oid > 0) && (item.m_policy_oid == policy_oid))) { - actions->insert(item); - /* we want to track all masking actions for each label */ - (*labels_to_actions)[item.m_label_name].push_back(item); + actions->insert(item); // 将操作项添加到操作集合中 + /* 我们想要跟踪每个标签的所有脱敏操作 */ + (*labels_to_actions)[item.m_label_name].push_back(item); // 将操作项添加到标签到操作集合的映射中 } } - tableam_scan_end(scan); - heap_close(relation, RowExclusiveLock); + tableam_scan_end(scan); // 结束扫描 + + heap_close(relation, RowExclusiveLock); // 关闭表格 } +// 检查属性类型是否支持脱敏 static bool check_masking_attrtype(Oid atttypeid) { + // 判断传入的属性类型 ID 是否合法，如果不合法直接返回 false if (!OidIsValid(atttypeid)) { return false; } + // 定义一个用于存放结果的变量，初始值为 false bool result = false; + // 使用 switch 语句针对不同的属性类型进行比较 switch (atttypeid) { - case BOOLOID: - case RELTIMEOID: - case TIMEOID: - case TIMETZOID: - case INTERVALOID: - case TIMESTAMPOID: - case TIMESTAMPTZOID: - case SMALLDATETIMEOID: - case ABSTIMEOID: - case TEXTOID: - case BPCHAROID: - case VARCHAROID: - case NVARCHAR2OID: - case NAMEOID: - case INT8OID: - case INT4OID: - case INT2OID: - case INT1OID: - case NUMERICOID: - case FLOAT4OID: - case FLOAT8OID: + // 如果属性类型等于以下任意一种类型，则认为该属性类型支持脱敏 + case BOOLOID://表示布尔类型 + case RELTIMEOID://表示相对时间（以毫秒为单位）。 + case TIMEOID://表示时间 + case TIMETZOID://表示带有时区信息的时间。 + case INTERVALOID://表示时间间隔 + case TIMESTAMPOID:// 表示时间戳 + case TIMESTAMPTZOID://表示带有时区信息的时间戳。 + case SMALLDATETIMEOID://表示用户定义的日期/时间格式。 + case ABSTIMEOID://表示绝对时间 + case TEXTOID://表示字符串类型。 + case BPCHAROID://表示字符串类型。 + case VARCHAROID://表示字符串类型。 + case NVARCHAR2OID://表示字符串类型。 + case NAMEOID://表示字符串类型。 + case INT8OID://表示整数类型 + case INT4OID://表示整数类型 + case INT2OID://表示整数类型 + case INT1OID://表示整数类型 + case NUMERICOID://表示任意精度数字类型 + case FLOAT4OID://表示浮点数类型。 + case FLOAT8OID://表示浮点数类型。 + // 如果属性类型支持脱敏，将结果变量设置为 true result = true; + // 执行到 break 后跳出 switch 语句 break; default: + // 如果属性类型不支持脱敏，不做任何操作 break; } + // 返回最终的结果 return result; } + +// 判断指定表的某一列是否允许进行脱敏操作 static bool column_allow_to_masking(Oid relid, const char *column) { + // 判断传入的表ID是否合法，如果不合法直接返回false if (!OidIsValid(relid)) { return false; } + // 定义用于存储列类型ID的变量 Oid column_typeid; + // 声明一个指向表的属性结构的指针，初始值为NULL Form_pg_attribute attr_struct = NULL; + // 在系统缓存中查找指定表的指定列的元组 HeapTuple attr_tuple = SearchSysCacheAttName(relid, column); + // 如果未能找到指定的列元组，则抛出错误并提示该列不存在 if (!HeapTupleIsValid(attr_tuple)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" does not exist", column))); } + + // 从列元组中获取属性结构信息 attr_struct = (Form_pg_attribute)GETSTRUCT(attr_tuple); + // 获取列的类型ID column_typeid = attr_struct->atttypid; + // 释放列元组的系统缓存 ReleaseSysCache(attr_tuple); + + // 检查获取到的列类型ID是否支持脱敏，并返回结果 return check_masking_attrtype(column_typeid); } + /* * add_labels_to_masking_action * * insert row of catalog gs_masking_policy_actions */ static inline void add_labels_to_masking_action(const gs_stl::gs_string action_type, - const gs_stl::gs_string action_params, const gs_stl::gs_string label_name, - Relation relation, Oid policyOid, Datum curtime) + const gs_stl::gs_string action_params, + const gs_stl::gs_string label_name, Relation relation, Oid policyOid, + Datum curtime) { + // 定义用于存储是否为空的布尔数组和用于存储值的数据数组 bool policy_actions_nulls[Natts_gs_masking_policy_actions] = {false}; Datum policy_actions_values[Natts_gs_masking_policy_actions] = {0}; + // 验证标签钩子函数 if (verify_label_hook) { if (!verify_label_hook(label_name.c_str())) { + // 如果标签不存在，关闭表并抛出错误 heap_close(relation, RowExclusiveLock); - ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("[%s] no such label found", label_name.c_str()))); + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("[%s] no such label found", label_name.c_str()))); } } - /* for now mask on table/view is not allowed */ + // 目前不允许在表/视图上进行脱敏操作 Relation label_rel = NULL; policy_labels_map existing_labels; label_rel = heap_open(GsPolicyLabelRelationId, RowExclusiveLock); @@ -284,38 +351,42 @@ static inline void add_labels_to_masking_action(const gs_stl::gs_string action_t policy_labels_set::iterator eit = (lbit->second)->end(); for (; lit != eit; ++lit) { Oid fqdnOid = lit->m_data_value_fqdn.m_value_object; - /* masking only allowed operator on column */ + // 只允许在列上进行脱敏操作 if (strcmp(lit->m_data_type.c_str(), "column") != 0) { - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("Masking policy can only operate on column object."))); + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("Masking policy can only operate on column object."))); } - /* masked column should belong to an ordinary table */ + // 被脱敏的列应属于普通表 if (OidIsValid(fqdnOid)) { char relkind = get_rel_relkind(fqdnOid); if (relkind != RELKIND_RELATION || get_rel_persistence(fqdnOid) != 'p') { - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("Masking policy can only operate on column of ordinary table."))); + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("Masking policy can only operate on column of ordinary table."))); } const char *column_name = lit->m_data_value_fqdn.m_value_object_attrib.c_str(); + // 检查列是否支持脱敏操作 if (!column_allow_to_masking(fqdnOid, column_name)) { - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("Type of column %s is not supported to mask.", column_name))); + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("Type of column %s is not supported to mask.", column_name))); } } } } - policy_actions_values[Anum_gs_masking_policy_act_action_type - 1] = DirectFunctionCall1(namein, CStringGetDatum(action_type.c_str())); - policy_actions_values[Anum_gs_masking_policy_act_action_params - 1] = DirectFunctionCall1(namein, CStringGetDatum(action_params.c_str())); - policy_actions_values[Anum_gs_masking_policy_act_label_name - 1] = DirectFunctionCall1(namein, CStringGetDatum(label_name.c_str())); - policy_actions_values[Anum_gs_masking_policy_act_policy_oid - 1] = ObjectIdGetDatum(policyOid); - policy_actions_values[Anum_gs_masking_policy_act_modify_date - 1] = curtime; + // 将参数转换成对应的数据类型，并存入数据数组中 + policy_actions_values[Anum_gs_masking_policy_act_action_type - 1] = + DirectFunctionCall1(namein, CStringGetDatum(action_type.c_str())); + policy_actions_values[Anum_gs_masking_policy_act_action_params - 1] = + DirectFunctionCall1(namein, CStringGetDatum(action_params.c_str())); + policy_actions_values[Anum_gs_masking_policy_act_label_name - 1] = + DirectFunctionCall1(namein, CStringGetDatum(label_name.c_str())); + policy_actions_values[Anum_gs_masking_policy_act_policy_oid - 1] = ObjectIdGetDatum(policyOid); + policy_actions_values[Anum_gs_masking_policy_act_modify_date - 1] = curtime; + // 创建一个HeapTuple HeapTuple policy_htup = heap_form_tuple(relation->rd_att, policy_actions_values, policy_actions_nulls); + // 将HeapTuple插入表中 simple_heap_insert(relation, policy_htup); CatalogUpdateIndexes(relation, policy_htup); heap_freetuple(policy_htup); @@ -325,33 +396,39 @@ static inline void add_labels_to_masking_action(const gs_stl::gs_string action_t * add_masking_filters * insert a set of filters into gs_masking_policy_filters */ -static inline void add_masking_filters(const filters_set* filters_to_add, Relation relation) +static inline void add_masking_filters(const filters_set *filters_to_add, Relation relation) { - Datum curtime; - HeapTuple policy_filters_htup; - bool policy_filters_nulls[Natts_gs_masking_policy_filters] = {false}; - Datum policy_filters_values[Natts_gs_masking_policy_filters] = {0}; - errno_t rc = EOK; + Datum curtime; // 当前时间戳 + HeapTuple policy_filters_htup; // 用于插入的HeapTuple节 + bool policy_filters_nulls[Natts_gs_masking_policy_filters] = {false}; // 插入的HeapTuple空值标志数组 + Datum policy_filters_values[Natts_gs_masking_policy_filters] = {0}; // 插入的HeapTuple数据数组 + errno_t rc = EOK; // 错误代码 - /* Get current timestamp */ + /* 获取当前时间戳 */ curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp()); + /* 遍历要添加的过滤器集合 */ for (filters_set::const_iterator it = filters_to_add->begin(); it != filters_to_add->end(); ++it) { - /* restore values and nulls for insert new node group record */ + /* 重置插入新节点组记录的值和空值 */ rc = memset_s(policy_filters_values, sizeof(policy_filters_values), 0, sizeof(policy_filters_values)); securec_check(rc, "\0", "\0"); - rc = memset_s(policy_filters_nulls, sizeof(policy_filters_nulls), false, sizeof(policy_filters_nulls)); + rc = memset_s(policy_filters_nulls, sizeof(policy_filters_nulls), false, sizeof(policy_filters_nulls)); securec_check(rc, "\0", "\0"); - policy_filters_values[Anum_gs_masking_policy_fltr_filter_type - 1] = DirectFunctionCall1(namein, CStringGetDatum(it->m_type.c_str())); - policy_filters_values[Anum_gs_masking_policy_fltr_label_name - 1] = DirectFunctionCall1(namein, CStringGetDatum(it->m_label_name.c_str())); - policy_filters_values[Anum_gs_masking_policy_fltr_logical_operator - 1] = CStringGetTextDatum(it->m_tree_string.c_str()); - policy_filters_values[Anum_gs_masking_policy_fltr_policy_oid - 1] = ObjectIdGetDatum(it->m_policy_oid); - policy_filters_values[Anum_gs_masking_policy_fltr_modify_date - 1] = curtime; + /* 将过滤器类型转换为对应的数据类型，并存入数据数组中 */ + policy_filters_values[Anum_gs_masking_policy_fltr_filter_type - 1] = + DirectFunctionCall1(namein, CStringGetDatum(it->m_type.c_str())); + policy_filters_values[Anum_gs_masking_policy_fltr_label_name - 1] = + DirectFunctionCall1(namein, CStringGetDatum(it->m_label_name.c_str())); + policy_filters_values[Anum_gs_masking_policy_fltr_logical_operator - 1] = + CStringGetTextDatum(it->m_tree_string.c_str()); + policy_filters_values[Anum_gs_masking_policy_fltr_policy_oid - 1] = ObjectIdGetDatum(it->m_policy_oid); + policy_filters_values[Anum_gs_masking_policy_fltr_modify_date - 1] = curtime; + /* 创建一个HeapTuple */ policy_filters_htup = heap_form_tuple(relation->rd_att, policy_filters_values, policy_filters_nulls); - /* Do the insertion */ + /* 将HeapTuple插入表中 */ simple_heap_insert(relation, policy_filters_htup); CatalogUpdateIndexes(relation, policy_filters_htup); @@ -360,83 +437,87 @@ static inline void add_masking_filters(const filters_set* filters_to_add, Relati } -struct MaskingActionItem { - gs_stl::gs_string m_action; - gs_stl::gs_string m_params; - policy_labelname_set m_labels; + +struct MaskingActionItem { + gs_stl::gs_string m_action; // 脱敏动作名称 + gs_stl::gs_string m_params; // 脱敏操作的参数 + policy_labelname_set m_labels; // 标签集合，用于指定要应用脱敏操作的列或字段 }; + /* * parse_masking_action * for input item, verify the using function is a masking function * and parse function params into m_params, parse relate labels into m_labels */ -static bool parse_masking_action(DefElem* policy_item, MaskingActionItem* item, gs_stl::gs_string* err_msg) +static bool parse_masking_action(DefElem *policy_item, MaskingActionItem *item, gs_stl::gs_string *err_msg) { if (policy_item == NULL) { return true; } - int index = 0; - bool is_found = false; - if (policy_item->defnamespace != NULL) { - item->m_action.append(policy_item->defnamespace); + int index = 0; // 用于追踪循环中的索引 + bool is_found = false; // 表示是否找到了匹配的脱敏函数 + if (policy_item->defnamespace != NULL) { // 判断策略项的命名空间是否为空 + item->m_action.append(policy_item->defnamespace); // 将命名空间添加到脱敏动作中去 item->m_action.append("."); } else { - for (int i = 0; g_maskFunctions[i] != NULL; ++i) { - if (strcmp(g_maskFunctions[i], (policy_item->defname)) == 0 || + for (int i = 0; g_maskFunctions[i] != NULL; ++i) { // 遍历已定义的脱敏函数数组 + if (strcmp(g_maskFunctions[i], (policy_item->defname)) == + 0 || // 判断策略项的函数名称是否匹配已定义的脱敏函数 strcasecmp("maskall", (policy_item->defname)) == 0) { - is_found = true; + is_found = true; // 标记已找到匹配的脱敏函数 break; } } - if (!is_found) { - item->m_action.append(get_namespace_name(GetOidBySchemaName())); + if (!is_found) { // 如果未找到匹配的脱敏函数 + item->m_action.append(get_namespace_name(GetOidBySchemaName())); // 获取当前模式的名称并添加到脱敏动作中 item->m_action.append("."); } } - item->m_action.append(policy_item->defname); - bool is_validated = false; - ListCell *item_list = NULL; - foreach(item_list, (List *)policy_item->arg) { - if (index == 0) { /* masking function params */ - List *action_params = (List *)(lfirst(item_list)); - ListCell *param = NULL; - foreach(param, action_params) { - DefElem *param_value_node = (DefElem *)(lfirst(param)); + item->m_action.append(policy_item->defname); // 将策略项的函数名称添加到脱敏动作中 + + bool is_validated = false; // 标记是否已验证脱敏行为 + ListCell *item_list = NULL; + foreach (item_list, (List *)policy_item->arg) { // 遍历策略项的参数列表 + if (index == 0) { // 脱敏函数的参数 + List *action_params = (List *)(lfirst(item_list)); // 获取参数列表 + ListCell *param = NULL; + foreach (param, action_params) { // 遍历参数列表 + DefElem *param_value_node = (DefElem *)(lfirst(param)); // 获取参数节点 if (param_value_node) { - const char * param_type = param_value_node->defname; - const char * param_val = strVal(param_value_node->arg); - (void)item->m_params.append(param_type); + const char *param_type = param_value_node->defname; // 获取参数名称 + const char *param_val = strVal(param_value_node->arg); // 获取参数的值 + (void)item->m_params.append(param_type); // 将参数名称添加到脱敏操作的参数中 (void)item->m_params.append(":"); - (void)item->m_params.append(param_val); + (void)item->m_params.append(param_val); // 将参数值添加到脱敏操作的参数中 (void)item->m_params.append(","); } } - if (item->m_params.size()) { - item->m_params.pop_back(); /* remove last ',' */ + if (item->m_params.size()) { // 如果存在参数，则移除最后一个逗号 + item->m_params.pop_back(); } - if (validate_masking_behaviour_hook) { + if (validate_masking_behaviour_hook) { // 如果存在验证脱敏行为的钩子函数 is_validated = true; - /* check function by parameters */ + // 调用验证脱敏行为的钩子函数，检查脱敏函数及其参数是否合法 bool invalid_params = false; - if (!validate_masking_behaviour_hook(item->m_action.c_str(), item->m_params.c_str(), - &invalid_params, false)) { - ereport(ERROR, (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), errmodule(MOD_UDF), - errmsg("failed to mask"), - errdetail("wrong masking function %s", invalid_params ? ": invalid parameters" : " "))); + if (!validate_masking_behaviour_hook(item->m_action.c_str(), item->m_params.c_str(), &invalid_params, + false)) { + ereport(ERROR, + (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), errmodule(MOD_UDF), errmsg("failed to mask"), + errdetail("wrong masking function %s", invalid_params ? ": invalid parameters" : " "))); } } - } else if (index == 1) { /* labels to apply the policy */ - DefElem *label_defel = (DefElem *)(lfirst(item_list)); - ListCell *label_name_item = NULL; - foreach(label_name_item, (List *)label_defel->arg) { - RangeVar *rel = (RangeVar*)lfirst(label_name_item); + } else if (index == 1) { // 应用策略的标签 + DefElem *label_defel = (DefElem *)(lfirst(item_list)); // 获取标签定义节点 + ListCell *label_name_item = NULL; + foreach (label_name_item, (List *)label_defel->arg) { // 遍历标签名称列表 + RangeVar *rel = (RangeVar *)lfirst(label_name_item); // 获取关系变量 gs_stl::gs_string label_name; - construct_resource_name(rel, &label_name); - item->m_labels.insert(label_name); + construct_resource_name(rel, &label_name); // 构造资源名称 + item->m_labels.insert(label_name); // 将标签名称添加到标签集合中 } - } else { /* wrong num of params */ + } else { // 参数数量错误 (*err_msg) = "wrong number of policy arguments"; return false; } @@ -445,85 +526,85 @@ static bool parse_masking_action(DefElem* policy_item, MaskingActionItem* item, return true; } + /* * update_masking_filters * * update gs_masking_policy_filters using filters_to_update */ -static void update_masking_filters(const filters_set* filters_to_update, Relation policy_filters_relation) +static void update_masking_filters(const filters_set *filters_to_update, Relation policy_filters_relation) { for (filters_set::const_iterator it = filters_to_update->begin(); it != filters_to_update->end(); ++it) { + // 设置扫描键，根据策略 OID 进行匹配 ScanKeyData scanKey[1]; - ScanKeyInit(&scanKey[0], - Anum_gs_masking_policy_fltr_policy_oid, - BTEqualStrategyNumber, F_OIDEQ, + ScanKeyInit(&scanKey[0], Anum_gs_masking_policy_fltr_policy_oid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(it->m_policy_oid)); - /* Search tuple by index */ - SysScanDesc scanDesc = systable_beginscan(policy_filters_relation, - GsMaskingPolicyFiltersPolicyOidIndexId, - true, - NULL, - 1, - scanKey); + // 使用索引进行扫描 + SysScanDesc scanDesc = + systable_beginscan(policy_filters_relation, GsMaskingPolicyFiltersPolicyOidIndexId, true, NULL, 1, scanKey); + // 获取匹配的元组 HeapTuple maskingPolicyTuple = systable_getnext(scanDesc); if (!HeapTupleIsValid(maskingPolicyTuple)) { + // 如果未找到匹配的元组，则调用 add_masking_filters 函数添加策略项 add_masking_filters(filters_to_update, policy_filters_relation); } else { + // 否则，更新元组中的逻辑运算符字段 Datum values[Natts_gs_masking_policy_filters] = {0}; - bool nulls[Natts_gs_masking_policy_filters] = {false}; - bool replaces[Natts_gs_masking_policy_filters] = {false}; + bool nulls[Natts_gs_masking_policy_filters] = {false}; + bool replaces[Natts_gs_masking_policy_filters] = {false}; + // 使用策略项中的树形字符串更新逻辑运算符字段的值 values[Anum_gs_masking_policy_fltr_logical_operator - 1] = CStringGetTextDatum(it->m_tree_string.c_str()); nulls[Anum_gs_masking_policy_fltr_logical_operator - 1] = false; replaces[Anum_gs_masking_policy_fltr_logical_operator - 1] = true; - HeapTuple newtuple = heap_modify_tuple(maskingPolicyTuple, - RelationGetDescr(policy_filters_relation), - values, - nulls, - replaces); + // 根据新的元组数据创建新的元组 + HeapTuple newtuple = heap_modify_tuple(maskingPolicyTuple, RelationGetDescr(policy_filters_relation), + values, nulls, replaces); + // 进行堆更新和目录索引更新操作 simple_heap_update(policy_filters_relation, &newtuple->t_self, newtuple); CatalogUpdateIndexes(policy_filters_relation, newtuple); } + // 结束扫描 systable_endscan(scanDesc); } } static void get_policy_filter(gs_stl::gs_string *filter, Oid polid) { + // 打开 GsMaskingPolicyFiltersId 表 Relation rel = heap_open(GsMaskingPolicyFiltersId, RowExclusiveLock); if (rel == NULL) { return; } + // 设置扫描键，根据策略 OID 进行匹配 ScanKeyData scanKey[1]; - ScanKeyInit(&scanKey[0], - Anum_gs_masking_policy_fltr_policy_oid, - BTEqualStrategyNumber, - F_OIDEQ, + ScanKeyInit(&scanKey[0], Anum_gs_masking_policy_fltr_policy_oid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(polid)); - SysScanDesc scanDesc = systable_beginscan(rel, - GsMaskingPolicyFiltersPolicyOidIndexId, - true, - NULL, - 1, - scanKey); + // 使用索引进行扫描 + SysScanDesc scanDesc = systable_beginscan(rel, GsMaskingPolicyFiltersPolicyOidIndexId, true, NULL, 1, scanKey); + // 获取匹配的元组 HeapTuple rtup = systable_getnext(scanDesc); if (HeapTupleIsValid(rtup)) { bool isNull = true; const char *logical_operator = ""; - Datum logical_operator_datum = heap_getattr(rtup, Anum_gs_masking_policy_fltr_logical_operator, - RelationGetDescr(rel), &isNull); + // 获取逻辑运算符字段的值 + Datum logical_operator_datum = + heap_getattr(rtup, Anum_gs_masking_policy_fltr_logical_operator, RelationGetDescr(rel), &isNull); if (!isNull) { logical_operator = TextDatumGetCString(logical_operator_datum); } + // 存储逻辑运算符字段的值到 filter 参数中 (*filter) = logical_operator; } + // 结束扫描 systable_endscan(scanDesc); + // 关闭表 heap_close(rel, RowExclusiveLock); } @@ -535,19 +616,23 @@ static void get_policy_filter(gs_stl::gs_string *filter, Oid polid) static void handle_alter_add_or_update_filter(List* filter, Oid policyOid, policy_labelname_set *policy_labels, bool to_add, bool is_alter) { - /* If defined policy stmt has no filters - * 1. create stmt but not filters, then using filter "" to verify - * 2. alter stmt but not filters, then get filters of this policy oid, and then verify + /* 如果定义的策略语句没有过滤器 + * 1. 创建语句但没有过滤器，则使用过滤器 "" 进行验证 + * 2. 修改语句但没有过滤器，则获取该策略 OID 的过滤器，然后进行验证 */ if (filter == NULL) { if (!policy_labels->empty()) { + // 创建语句但没有过滤器，使用过滤器 "" 进行验证 gs_stl::gs_string flat_tree = ""; if (is_alter) { + // 修改语句但没有过滤器，获取该策略 OID 的过滤器 get_policy_filter(&flat_tree, policyOid); } if (gs_verify_labels_by_policy_hook != NULL) { gs_stl::gs_string polname; + // 根据过滤器和标签信息进行验证 if (!gs_verify_labels_by_policy_hook(flat_tree.c_str(), policy_labels, policyOid, &polname)) { + // 报错：当前策略与已存在的策略冲突 ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("current policy is conflict with exist policy: %s", polname.c_str()))); @@ -558,12 +643,14 @@ static void handle_alter_add_or_update_filter(List* filter, Oid policyOid, } gs_stl::gs_string flatten_tree; - /* Process new filters from parser tree and tranform them into string */ + /* 处理来自解析树的新过滤器并将其转换为字符串格式 */ if (!process_new_filters(filter, &flatten_tree)) { + // 报错：不支持的策略过滤器值 ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unsupported policy filter values"))); } filters_set filters_to_alter; if (flatten_tree.size() > 0) { + // 构造一个逻辑表达式节点 PgPolicyFiltersStruct item; item.m_type = "logical_expr"; item.m_label_name = "logical_expr"; @@ -571,21 +658,24 @@ static void handle_alter_add_or_update_filter(List* filter, Oid policyOid, item.m_policy_oid = policyOid; filters_to_alter.insert(item); } - /* when stmt has filter clause, just do verify using it's filter */ + /* 当语句有过滤器子句时，只需使用其过滤器进行验证 */ if (gs_verify_labels_by_policy_hook != NULL) { gs_stl::gs_string polname; - /* If it is alter stmt and this stmt is alter filter statement*/ + /* 如果是修改语句且该语句是修改过滤器语句 */ if (is_alter && policy_labels->empty()) { policy_labelname_set cur_labels; load_existing_masking_labels(&cur_labels, policyOid); + // 根据当前过滤器和已存在的标签信息进行验证 if (!gs_verify_labels_by_policy_hook(flatten_tree.c_str(), &cur_labels, policyOid, &polname)) { + // 报错：当前策略与已存在的策略冲突 ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("current policy is conflict with exist policy: %s", polname.c_str()))); } - /* if it is create stmt with filter_clause and have intersect */ + /* 如果是有过滤器子句的创建语句并发生交集 */ } else if (!policy_labels->empty() && !gs_verify_labels_by_policy_hook(flatten_tree.c_str(), policy_labels, policyOid, &polname)) { + // 报错：当前策略与已存在的策略冲突 ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("current policy is conflict with exist policy: %s", polname.c_str()))); @@ -595,6 +685,7 @@ static void handle_alter_add_or_update_filter(List* filter, Oid policyOid, if (filters_to_alter.size() > 0) { Relation policy_filters_relation = heap_open(GsMaskingPolicyFiltersId, RowExclusiveLock); if (policy_filters_relation) { + // 根据 to_add 参数进行添加或更新操作 if (to_add) { add_masking_filters(&filters_to_alter, policy_filters_relation); } else { @@ -606,15 +697,16 @@ static void handle_alter_add_or_update_filter(List* filter, Oid policyOid, } } + /* * create_masking_policy * * handle create masking policy command */ void create_masking_policy(CreateMaskingPolicyStmt *stmt) -{ +{//用于处理创建遮蔽策略的命令。它接受一个CreateMaskingPolicyStmt结构作为参数，该结构包含了创建策略的相关信息。 /* check that if has access to config masking policy */ - if (!is_policy_enabled()) { + if (!is_policy_enabled()) {//检查当前用户是否有权限配置遮蔽策略 ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); @@ -626,25 +718,26 @@ void create_masking_policy(CreateMaskingPolicyStmt *stmt) char buff[BUFFSIZE] = {0}; char session_ip[MAX_IP_LEN] = {0}; get_session_ip(session_ip, MAX_IP_LEN); - + //获取当前用户的名称、应用程序名称和会话的IP地址，并将这些信息组合成一条审计日志。 int rc = snprintf_s(buff, sizeof(buff), sizeof(buff) - 1, "user name: [%s], app_name: [%s], ip: [%s], CREATE MASKING POLICY [%s]", user_name, u_sess->attr.attr_common.application_name, session_ip, stmt->policy_name); securec_check_ss(rc, "\0", "\0"); save_manage_message(buff); + //从传入的参数中获取策略名称和启用状态，并声明一个curtime变量用于存储当前时间戳。 const char *policy_name = stmt->policy_name; bool policy_enabled = stmt->policy_enabled; Datum curtime; - HeapTuple policy_htup; - bool policy_nulls[Natts_gs_masking_policy] = {false}; - Datum policy_values[Natts_gs_masking_policy] = {0}; + HeapTuple policy_htup;//用于存储遮蔽策略的元组 + bool policy_nulls[Natts_gs_masking_policy] = {false};//表示属性是否为空 + Datum policy_values[Natts_gs_masking_policy] = {0};//属性值 /* Get current timestamp */ curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp()); - Relation policy_relation = heap_open(GsMaskingPolicyRelationId, RowExclusiveLock); + Relation policy_relation = heap_open(GsMaskingPolicyRelationId, RowExclusiveLock);//打开遮蔽策略的关系表。 if (policy_relation == NULL) { send_manage_message(AUDIT_FAILED); ereport(ERROR, @@ -653,6 +746,7 @@ void create_masking_policy(CreateMaskingPolicyStmt *stmt) return; } /* no more than MAX_POLICIES_NUM is allowed */ + //检查是否已经达到了策略数量的最大限制。 if (get_num_of_existing_policies(policy_relation) >= MAX_POLICIES_NUM) { /* out of limit */ heap_close(policy_relation, RowExclusiveLock); @@ -664,6 +758,7 @@ void create_masking_policy(CreateMaskingPolicyStmt *stmt) } /* check whether such policy exists */ + //检查该策略是否已经存在。 if (stmt->if_not_exists == true) { policies_set existing_policies; load_existing_policies(policy_relation, &existing_policies); @@ -680,6 +775,7 @@ void create_masking_policy(CreateMaskingPolicyStmt *stmt) } } + //构建遮蔽策略的元组，并将相应的属性值赋给policy_values数组。 policy_values[Anum_gs_masking_policy_pol_name - 1] = DirectFunctionCall1(namein, CStringGetDatum(policy_name)); policy_values[Anum_gs_masking_policy_pol_comments - 1] = DirectFunctionCall1(namein, CStringGetDatum("")); policy_values[Anum_gs_masking_policy_pol_modify_date - 1] = curtime; @@ -693,6 +789,7 @@ void create_masking_policy(CreateMaskingPolicyStmt *stmt) heap_close(policy_relation, RowExclusiveLock); /* Process masking actions */ + //打开遮蔽动作关系表 Relation masking_actions_relation = heap_open(GsMaskingPolicyActionsId, RowExclusiveLock); if (masking_actions_relation == NULL) { send_manage_message(AUDIT_FAILED); @@ -730,6 +827,7 @@ void create_masking_policy(CreateMaskingPolicyStmt *stmt) heap_close(masking_actions_relation, RowExclusiveLock); /* Process masking policy filters */ + //遍历语句节点树中的每个策略项，并解析出策略目标信息。 handle_alter_add_or_update_filter(stmt->policy_filters, policyOid, &policy_labels, true, false); CommandCounterIncrement(); send_manage_message(AUDIT_OK); @@ -742,72 +840,107 @@ void create_masking_policy(CreateMaskingPolicyStmt *stmt) * * update masking policy into catalog gs_masking_policy */ -static bool update_policy(const GsPolicyStruct* policy, Relation relation, - bool policy_status_changed, gs_stl::gs_string* err_msg) +/* +policy：要更新的遮蔽策略的结构体指针。 +relation：表示gs_masking_policy表的关系对象。 +policy_status_changed：一个布尔值，指示遮蔽策略的状态是否发生了变化。 +err_msg：一个指向std::string的指针，用于保存错误信息。 +*/ + +static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool policy_status_changed, + gs_stl::gs_string *err_msg) { - bool policy_nulls[Natts_gs_masking_policy] = {false}; - bool policy_replaces[Natts_gs_masking_policy] = {false}; - Datum policy_values[Natts_gs_masking_policy] = {0}; - Datum curtime; + // 用于标记列是否为空 + bool policy_nulls[Natts_gs_masking_policy] = {false}; + // 用于标记列是否需要替换 + bool policy_replaces[Natts_gs_masking_policy] = {false}; + // 列的值 + Datum policy_values[Natts_gs_masking_policy] = {0}; + // 当前时间戳 + Datum curtime; ScanKeyData skey; - /* Find the policy row to update */ - ScanKeyInit(&skey, - ObjectIdAttributeNumber, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(policy->m_id)); + // 创建扫描键，按照策略的ID进行相等匹配 + ScanKeyInit(&skey, ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(policy->m_id)); + // 开始在关系上执行扫描 SysScanDesc tgscan = systable_beginscan(relation, GsMaskingPolicyOidIndexId, true, NULL, 1, &skey); HeapTuple tup = NULL; if (tgscan == NULL) { + // 扫描索引失败，记录错误信息 (void)err_msg->append("could not scan index GsMaskingPolicyOidIndexId."); return false; } + // 获取下一个扫描到的元组 tup = systable_getnext(tgscan); if (!tup || !HeapTupleIsValid(tup)) { + // 未找到待更新的元组，记录错误信息 systable_endscan(tgscan); (void)err_msg->append("could not find tuple for policy "); (void)err_msg->append(policy->m_name.c_str()); return false; } - /* Get current timestamp */ + // 获取当前时间戳 curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp()); + // 如果策略的状态发生了变化 if (policy_status_changed) { - policy_replaces[Anum_gs_masking_policy_pol_enabled - 1] = true; - policy_values[Anum_gs_masking_policy_pol_enabled - 1] = BoolGetDatum(policy->m_enabled); + // 设置要替换的列和对应的值 + policy_replaces[Anum_gs_masking_policy_pol_enabled - 1] = true; + policy_values[Anum_gs_masking_policy_pol_enabled - 1] = BoolGetDatum(policy->m_enabled); } else { + // 设置要替换的列和对应的值 policy_replaces[Anum_gs_masking_policy_pol_comments - 1] = true; - policy_values[Anum_gs_masking_policy_pol_comments - 1] = + policy_values[Anum_gs_masking_policy_pol_comments - 1] = DirectFunctionCall1(namein, CStringGetDatum(policy->m_comments.c_str())); } + // 设置要替换的列和对应的值 policy_replaces[Anum_gs_masking_policy_pol_modify_date - 1] = true; policy_values[Anum_gs_masking_policy_pol_modify_date - 1] = curtime; - HeapTuple newtuple = heap_modify_tuple(tup, RelationGetDescr(relation), policy_values, policy_nulls, policy_replaces); + // 创建一个新的元组并进行修改 + HeapTuple newtuple = + heap_modify_tuple(tup, RelationGetDescr(relation), policy_values, policy_nulls, policy_replaces); + // 更新关系中的元组 simple_heap_update(relation, &newtuple->t_self, newtuple); + // 更新索引 CatalogUpdateIndexes(relation, newtuple); + // 结束扫描 systable_endscan(tgscan); return true; } + /* * update_labels_in_masking_action * * update labels of actions and verify whether label can insert */ -static inline bool update_labels_in_masking_action(const PgPolicyMaskingActionStruct* action_item, - const masking_label_to_actions_map* labels_to_actions, Relation relation, gs_stl::gs_string* err_msg) +/* +action_item：要更新的遮蔽动作的结构体指针。 +labels_to_actions：一个映射关系，将标签名映射到对应的动作集合。 +relation：表示gs_masking_policy_actions表的关系对象。 +err_msg：一个指向std::string的指针，用于保存错误信息。 +*/ +static inline bool update_labels_in_masking_action(const PgPolicyMaskingActionStruct *action_item, + const masking_label_to_actions_map *labels_to_actions, + Relation relation, gs_stl::gs_string *err_msg) { + // 用于标记列是否为空 bool policy_actions_nulls[Natts_gs_masking_policy_actions] = {false}; + // 用于标记列是否需要替换 bool policy_actions_replaces[Natts_gs_masking_policy_actions] = {false}; + // 列的值 Datum policy_actions_values[Natts_gs_masking_policy_actions] = {0}; - Datum curtime; + // 当前时间戳 + Datum curtime; + // 在映射中查找给定标签名对应的动作集合 masking_label_to_actions_map::const_iterator it = labels_to_actions->find(action_item->m_label_name); if (it == labels_to_actions->end()) { /* no such label */ + // 找不到对应的标签，记录错误信息 err_msg->clear(); (void)err_msg->append("No such label ["); (void)err_msg->append(action_item->m_label_name); @@ -816,6 +949,7 @@ static inline bool update_labels_in_masking_action(const PgPolicyMaskingActionSt } else { /* should not happen */ if (it->second->size() == 0) { + // 标签对应的动作集合为空，记录错误信息 err_msg->clear(); (void)err_msg->append("No masking actions for label ["); (void)err_msg->append(action_item->m_label_name); @@ -824,6 +958,7 @@ static inline bool update_labels_in_masking_action(const PgPolicyMaskingActionSt } /* should not happen */ if (it->second->size() > 1) { + // 标签对应的动作集合过多，记录错误信息 err_msg->clear(); (void)err_msg->append("Too many masking actions for label ["); (void)err_msg->append(action_item->m_label_name); @@ -833,84 +968,112 @@ static inline bool update_labels_in_masking_action(const PgPolicyMaskingActionSt } ScanKeyData skey; - /* Find the policy row to update */ - ScanKeyInit(&skey, - ObjectIdAttributeNumber, - BTEqualStrategyNumber, F_OIDEQ, + // 创建扫描键，按照动作的ID进行相等匹配 + ScanKeyInit(&skey, ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(it->second->begin()->m_id)); + // 开始在关系上执行扫描 SysScanDesc tgscan = systable_beginscan(relation, GsMaskingPolicyActionsOidIndexId, true, NULL, 1, &skey); HeapTuple tup = NULL; if (tgscan == NULL) { + // 扫描索引失败，记录错误信息 (void)err_msg->append("could not scan index GsMaskingPolicyActionsOidIndexId."); return false; } + // 获取下一个扫描到的元组 tup = systable_getnext(tgscan); if (!tup || !HeapTupleIsValid(tup)) { + // 未找到待更新的元组，记录错误信息 systable_endscan(tgscan); (void)err_msg->append("could not find tuple for masking action "); (void)err_msg->append(action_item->m_type); return false; } + // 获取当前时间戳 curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp()); - policy_actions_replaces[Anum_gs_masking_policy_act_action_type - 1] = true; - policy_actions_replaces[Anum_gs_masking_policy_act_action_params - 1] = true; - policy_actions_replaces[Anum_gs_masking_policy_act_modify_date - 1] = true; - policy_actions_values[Anum_gs_masking_policy_act_action_type - 1] = DirectFunctionCall1(namein, CStringGetDatum(action_item->m_type.c_str())); - policy_actions_values[Anum_gs_masking_policy_act_action_params - 1] = DirectFunctionCall1(namein, CStringGetDatum(action_item->m_params.c_str())); - policy_actions_values[Anum_gs_masking_policy_act_modify_date - 1] = curtime; + // 设置要替换的列和对应的值 + policy_actions_replaces[Anum_gs_masking_policy_act_action_type - 1] = true; + policy_actions_replaces[Anum_gs_masking_policy_act_action_params - 1] = true; + policy_actions_replaces[Anum_gs_masking_policy_act_modify_date - 1] = true; + policy_actions_values[Anum_gs_masking_policy_act_action_type - 1] = + DirectFunctionCall1(namein, CStringGetDatum(action_item->m_type.c_str())); + policy_actions_values[Anum_gs_masking_policy_act_action_params - 1] = + DirectFunctionCall1(namein, CStringGetDatum(action_item->m_params.c_str())); + policy_actions_values[Anum_gs_masking_policy_act_modify_date - 1] = curtime; - HeapTuple newtuple = heap_modify_tuple(tup, RelationGetDescr(relation), policy_actions_values, policy_actions_nulls, policy_actions_replaces); + // 创建一个新的元组并进行修改 + HeapTuple newtuple = heap_modify_tuple(tup, RelationGetDescr(relation), policy_actions_values, policy_actions_nulls, + policy_actions_replaces); + // 更新关系中的元组 simple_heap_update(relation, &newtuple->t_self, newtuple); + // 更新索引 CatalogUpdateIndexes(relation, newtuple); + // 结束扫描 systable_endscan(tgscan); return true; } -static inline bool remove_labels_from_masking_action(const PgPolicyMaskingActionStruct* action_item, - masking_actions_set* existing_actions, Relation relation, gs_stl::gs_string* err_msg) +/* +action_item：要移除的遮蔽动作的结构体指针。 +existing_actions：已存在的遮蔽动作集合。 +relation：表示gs_masking_policy_actions表的关系对象。 +err_msg：一个指向std::string的指针，用于保存错误信息。 +*/ +static inline bool remove_labels_from_masking_action(const PgPolicyMaskingActionStruct *action_item, + masking_actions_set *existing_actions, Relation relation, + gs_stl::gs_string *err_msg) { - /* Removing masking action from policy having only one action is not allowed */ + // 如果只有一个遮蔽动作，不允许从策略中移除遮蔽动作 if (existing_actions->size() == 1) { (*err_msg) = "Removing masking action from policy with a single masking action not allowed"; return false; } + + // 在现有的遮蔽动作集合中查找要移除的动作 masking_actions_set::iterator i_it = existing_actions->find(*action_item); if (i_it != existing_actions->end()) { + // 找到了要移除的动作，调用扫描函数在关系上进行删除操作 if (scan_to_delete_from_relation(i_it->m_id, relation, GsMaskingPolicyActionsOidIndexId)) - existing_actions->erase(i_it); + existing_actions->erase(i_it); // 从集合中移除动作 } else { + // 找不到要移除的动作，记录错误信息 err_msg->clear(); (void)err_msg->append("No such masking action ["); (void)err_msg->append(action_item->m_type); (void)err_msg->append("] found"); return false; } + return true; } -static void handle_drop_filter_and_condition(const char* policy_action, Oid policyOid) + +static void handle_drop_filter_and_condition(const char *policy_action, Oid policyOid) { if (policy_action == NULL) { return; } + // 如果策略动作是"drop_filter"，则删除相应的行 if (!strcmp(policy_action, "drop_filter")) { drop_policy_reference(GsMaskingPolicyFiltersId, policyOid); } } + static void handle_alter_policy_params(const char* policy_name, Node* policy_enabled, const char* policy_comments, Oid& policyOid) { + //打开遮蔽策略的关系（Relation），并加载现有的遮蔽策略。 policies_set existing_policies; Relation policy_relation = heap_open(GsMaskingPolicyRelationId, RowExclusiveLock); load_existing_policies(policy_relation, &existing_policies); /* first check whether such policy exists */ + //检查待更改的遮蔽策略是否存在于已加载的遮蔽策略中。 GsPolicyStruct current_policy; current_policy.m_name = policy_name; policies_set::iterator it = existing_policies.find(current_policy); @@ -919,11 +1082,12 @@ static void handle_alter_policy_params(const char* policy_name, Node* policy_ena ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("policy %s not exists.", policy_name))); return; } - + //如果策略存在，获取该策略的名称、ID和启用状态，并保存在current_policy中。 current_policy.m_id = policyOid = it->m_id; current_policy.m_enabled = it->m_enabled; /* Update policy status (enable/disable) if needed */ + //传入的policy_enabled参数不为空，则更新策略的状态（启用或禁用） bool policy_status_changed = false; if (policy_enabled != NULL) { DefElem *defel = (DefElem *) policy_enabled; @@ -937,6 +1101,7 @@ static void handle_alter_policy_params(const char* policy_name, Node* policy_ena } /* Update policy comments if needed */ + //如果传入的policy_comments参数不为空，或者策略状态发生了改变，则更新策略的注释。 if (((policy_comments != NULL) && (strlen(policy_comments) > 0)) || policy_status_changed) { current_policy.m_comments = policy_comments; gs_stl::gs_string err_msg; @@ -948,6 +1113,7 @@ static void handle_alter_policy_params(const char* policy_name, Node* policy_ena return; } } + //最后关闭遮蔽策略的关系。 heap_close(policy_relation, RowExclusiveLock); } @@ -958,19 +1124,35 @@ static void handle_alter_policy_params(const char* policy_name, Node* policy_ena * insert into currect sets for update, add, remove item * for each modify set, insert update catalog gs_masking_policy_actions */ -static void handle_alter_add_remove_modify_action(List* policy_items, const char* policy_action, Oid policyOid, - policy_labelname_set *policy_labels, masking_actions_set* existing_actions, - const policy_labels_map* existing_labels, masking_label_to_actions_map* labels_to_actions, Datum curtime) +/* +policy_items: 控制策略项列表 +policy_action: 策略动作类型（增加、删除或修改） +policyOid: 策略对象的标识符 +policy_labels: 策略标签集合 +existing_actions: 已存在的掩码动作集合 +existing_labels: 已存在的标签映射表 +labels_to_actions: 标签到动作的映射表 +curtime: 当前时间戳 +*/ +static void handle_alter_add_remove_modify_action(List *policy_items, const char *policy_action, Oid policyOid, + policy_labelname_set *policy_labels, + masking_actions_set *existing_actions, + const policy_labels_map *existing_labels, + masking_label_to_actions_map *labels_to_actions, Datum curtime) { + // 待删除的掩码动作集合 masking_actions_set actions_to_remove; + // 待更新的掩码动作集合 masking_actions_set actions_to_update; + // 待添加的掩码动作集合 masking_actions_set actions_to_add; - ListCell *policy_item_iter = NULL; + ListCell *policy_item_iter = NULL; + // 获取策略动作类型 int alter_opt = get_policy_command_type(policy_action); - foreach(policy_item_iter, policy_items) - { - DefElem *pol_option_item = (DefElem *) lfirst(policy_item_iter); + // 遍历策略项列表 + foreach (policy_item_iter, policy_items) { + DefElem *pol_option_item = (DefElem *)lfirst(policy_item_iter); if (pol_option_item == NULL) { continue; } @@ -978,13 +1160,15 @@ static void handle_alter_add_remove_modify_action(List* policy_items, const char bool ret = true; gs_stl::gs_string err_msg; - /* handle masking actions */ + /* 处理掩码动作 */ MaskingActionItem action_item; + // 解析掩码动作 ret = parse_masking_action(pol_option_item, &action_item, &err_msg); if (ret) { - /* validate that such label exists */ + /* 验证标签是否存在 */ gs_stl::gs_set::const_iterator it; for (it = action_item.m_labels.begin(); it != action_item.m_labels.end(); ++it) { + // 如果标签不存在，则报错 if (existing_labels->find(*it) == existing_labels->end()) { err_msg.clear(); (void)err_msg.append("Trying to bind masking policy ["); @@ -1000,6 +1184,7 @@ static void handle_alter_add_remove_modify_action(List* policy_items, const char item.m_params = action_item.m_params; item.m_label_name = *it; item.m_policy_oid = policyOid; + // 根据策略动作类型插入到相应的集合中 switch (alter_opt) { case POLICY_CMD_ADD: actions_to_add.insert(item); @@ -1018,7 +1203,7 @@ static void handle_alter_add_remove_modify_action(List* policy_items, const char } } - /* If anything is added OR removed to label, label must exist unless it's for ALL */ + // 如果添加或删除了任何标签，则标签必须存在，除非是针对ALL的操作 if (!ret) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("%s", err_msg.c_str()))); return; @@ -1026,16 +1211,21 @@ static void handle_alter_add_remove_modify_action(List* policy_items, const char } gs_stl::gs_string err_msg; - /* alter masking actions */ + /* 处理掩码动作 */ if ((actions_to_add.size() > 0) || (actions_to_remove.size() > 0) || (actions_to_update.size() > 0)) { + // 打开并锁定掩码策略动作关系表 Relation masking_actions_relation = heap_open(GsMaskingPolicyActionsId, RowExclusiveLock); if (masking_actions_relation) { + // 处理待添加的动作集合 for (masking_actions_set::const_iterator it = actions_to_add.begin(); it != actions_to_add.end(); ++it) { - add_labels_to_masking_action(it->m_type, it->m_params, it->m_label_name, - masking_actions_relation, policyOid, curtime); + add_labels_to_masking_action(it->m_type, it->m_params, it->m_label_name, masking_actions_relation, + policyOid, curtime); } - for (masking_actions_set::const_iterator it = actions_to_remove.begin(); it != actions_to_remove.end(); ++it) { + // 处理待删除的动作集合 + for (masking_actions_set::const_iterator it = actions_to_remove.begin(); it != actions_to_remove.end(); + ++it) { + // 从掩码策略动作关系表中删除相应的标签，并验证是否删除成功 if (!remove_labels_from_masking_action(&(*it), existing_actions, masking_actions_relation, &err_msg)) { heap_close(masking_actions_relation, RowExclusiveLock); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("%s", err_msg.c_str()))); @@ -1043,7 +1233,10 @@ static void handle_alter_add_remove_modify_action(List* policy_items, const char } } - for (masking_actions_set::const_iterator it = actions_to_update.begin(); it != actions_to_update.end(); ++it) { + // 处理待更新的动作集合 + for (masking_actions_set::const_iterator it = actions_to_update.begin(); it != actions_to_update.end(); + ++it) { + // 更新掩码策略动作关系表中相应的标签，并验证是否更新成功 if (!update_labels_in_masking_action(&(*it), labels_to_actions, masking_actions_relation, &err_msg)) { heap_close(masking_actions_relation, RowExclusiveLock); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("%s", err_msg.c_str()))); @@ -1051,6 +1244,7 @@ static void handle_alter_add_remove_modify_action(List* policy_items, const char } } + // 关闭掩码策略动作关系表 heap_close(masking_actions_relation, RowExclusiveLock); } } @@ -1061,9 +1255,21 @@ static void handle_alter_add_remove_modify_action(List* policy_items, const char * * handle alter masking stmt */ +/* +buff：大小为BUFFSIZE的字符数组，用于存储消息文本。 +user_name：大小为USERNAME_LEN的字符数组，用于存储用户名称。 +session_ip：大小为MAX_IP_LEN的字符数组，用于存储会话IP地址。 +labels_relation：一个指向关系的指针，用于管理掩码策略标签的关系。 +existing_labels：一个映射（map），用于存储现有掩码策略标签的名称和OID。 +existing_actions：一个集合（set），用于存储现有的掩码策略动作。 +labels_to_actions：一个映射，将掩码策略标签与相应的动作关联起来。 +policy_labels：一个集合，用于存储掩码策略的标签名称。 +policyOid：一个对象标识符（OID），表示当前掩码策略的OID。 +*/ void alter_masking_policy(AlterMaskingPolicyStmt *stmt) { /* check that if have access to config masking policy */ + //检查当前用户是否有权限修改掩码策略 if (!is_policy_enabled()) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), @@ -1071,6 +1277,7 @@ void alter_masking_policy(AlterMaskingPolicyStmt *stmt) return; } + //检查当前用户是否有权限修改掩码策略 char buff[BUFFSIZE] = {0}; char user_name[USERNAME_LEN] = {0}; char session_ip[MAX_IP_LEN] = {0}; @@ -1078,12 +1285,14 @@ void alter_masking_policy(AlterMaskingPolicyStmt *stmt) get_session_ip(session_ip, MAX_IP_LEN); (void)GetRoleName(GetCurrentUserId(), user_name, sizeof(user_name)); + //生成日志消息，记录用户名称、应用程序名称、IP地址和执行的ALTER MASKING POLICY语句，然后将该日志消息保存。 int rc = snprintf_s(buff, sizeof(buff), sizeof(buff) - 1, "user name: [%s], app_name: [%s], ip: [%s], ALTER MASKING POLICY [%s]", user_name, u_sess->attr.attr_common.application_name, session_ip, stmt->policy_name); securec_check_ss(rc, "\0", "\0"); save_manage_message(buff); + //声明一些关系和映射的变量，用于存储加载的掩码策略标签和动作的信息，并初始化一些其他变量。 Relation labels_relation = NULL; policy_labels_map existing_labels; masking_actions_set existing_actions; @@ -1094,6 +1303,7 @@ void alter_masking_policy(AlterMaskingPolicyStmt *stmt) handle_alter_policy_params(stmt->policy_name, stmt->policy_enabled, stmt->policy_comments, policyOid); /* load existing labels for further validation */ + //加载现有的掩码策略标签和动作信息，并获取当前时间戳。 labels_relation = heap_open(GsPolicyLabelRelationId, RowExclusiveLock); load_existing_labels(labels_relation, &existing_labels); heap_close(labels_relation, RowExclusiveLock); @@ -1107,6 +1317,7 @@ void alter_masking_policy(AlterMaskingPolicyStmt *stmt) } /* if ALTER MASKING POLICY DROP FILTER, should load all labelname of policyOid */ + //如果ALTER MASKING POLICY语句中包含删除筛选条件的操作，则加载该策略下的所有标签名。 if (get_policy_command_type(stmt->policy_action) == POLICY_CMD_DROP_FILTER) { load_existing_masking_labels(&policy_labels, policyOid); } @@ -1116,262 +1327,371 @@ void alter_masking_policy(AlterMaskingPolicyStmt *stmt) send_manage_message(AUDIT_OK); reload_masking_policies(); + //提交事务、发送审计消息、重新加载掩码策略。 } /* * drop_masking_policy - * - * handle drop masking policy stmt + * + * 处理 drop masking policy 语句 */ void drop_masking_policy(DropMaskingPolicyStmt *stmt) { - /* check that if have access to config masking policy */ + /* 检查是否有配置掩码策略的权限 */ if (!is_policy_enabled()) { - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("Permission denied."))); + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); return; } - ListCell* policy_obj = NULL; - /* save Mng logs */ - foreach(policy_obj, stmt->policy_names) { - const char* polname = (const char *)(((Value*)lfirst(policy_obj))->val.str); + ListCell *policy_obj = NULL; + /* 保存管理日志 */ + foreach (policy_obj, stmt->policy_names) { + // 获取掩码策略名称 + const char *polname = (const char *)(((Value *)lfirst(policy_obj))->val.str); char buff[BUFFSIZE] = {0}; char user_name[USERNAME_LEN] = {0}; char session_ip[MAX_IP_LEN] = {0}; + // 获取会话的IP地址和当前用户的名称 get_session_ip(session_ip, MAX_IP_LEN); (void)GetRoleName(GetCurrentUserId(), user_name, sizeof(user_name)); + // 构建管理日志消息 int rc = snprintf_s(buff, sizeof(buff), sizeof(buff) - 1, - "user name: [%s], app_name: [%s], ip: [%s], DROP MASKING POLICY [%s]", - user_name, u_sess->attr.attr_common.application_name, session_ip, polname); + "user name: [%s], app_name: [%s], ip: [%s], DROP MASKING POLICY [%s]", user_name, + u_sess->attr.attr_common.application_name, session_ip, polname); securec_check_ss(rc, "\0", "\0"); + + // 保存管理日志消息 save_manage_message(buff); } + // 打开掩码策略关系的堆表 Relation policy_relation = heap_open(GsMaskingPolicyRelationId, RowExclusiveLock); policies_set existing_policies; + // 加载已有的策略信息到 existing_policies 中 load_existing_policies(policy_relation, &existing_policies); + // 关闭掩码策略关系的堆表 heap_close(policy_relation, RowExclusiveLock); - foreach(policy_obj, stmt->policy_names) { - const char* polname = (const char *)(((Value*)lfirst(policy_obj))->val.str); + /* 遍历掩码策略名称列表 */ + foreach (policy_obj, stmt->policy_names) { + // 获取掩码策略名称 + const char *polname = (const char *)(((Value *)lfirst(policy_obj))->val.str); GsPolicyStruct cur_policy; cur_policy.m_name = polname; + // 检查当前策略是否存在于 existing_policies 中 policies_set::iterator it = existing_policies.find(cur_policy); + if (it == existing_policies.end()) { + // 策略不存在且 if_exists 标志为 false，则发送一条管理消息表示审核失败，并抛出一个错误 if (!stmt->if_exists) { send_manage_message(AUDIT_FAILED); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("policy %s not exists.", polname))); return; } + // 策略不存在但 if_exists 标志为 true，则发送一条通知消息表示策略不存在，并跳过当前策略的删除操作 ereport(NOTICE, (errmsg("policy %s not exists, skipping", polname))); } + gs_stl::gs_set ids; + // 删除指定名称的策略，并删除相关的过滤器和动作 drop_policy_by_name(GsMaskingPolicyRelationId, polname, &ids); + // 删除指定策略的相关过滤器和动作 for (long long _id : ids) { - /* for masking filters */ + /* 删除掩码策略的相关过滤器 */ drop_policy_reference(GsMaskingPolicyFiltersId, _id); - /* for masking actions */ + /* 删除掩码策略的相关动作 */ drop_policy_reference(GsMaskingPolicyActionsId, _id); } } + CommandCounterIncrement(); + // 发送一条管理消息表示删除成功 send_manage_message(AUDIT_OK); + // 重新加载掩码策略 reload_masking_policies(); } +/* +Relation label_rel = NULL;：策略标签关系表的关系对象。 +List *label_list = NIL;：用于存储策略标签名称的列表。 +TableScanDesc scan = NULL;：表扫描描述符，用于扫描策略标签关系表。 +HeapTuple tup = NULL;：堆元组，表示策略标签关系表中的一行数据。 +Form_gs_policy_label label_data = NULL;：指向策略标签关系表行数据的结构体指针。 +*/ static List *relid_get_policy_label_namelist(Oid relid, const char *column = NULL) { + // 定义相关变量 Relation label_rel = NULL; List *label_list = NIL; TableScanDesc scan = NULL; HeapTuple tup = NULL; Form_gs_policy_label label_data = NULL; + // 打开策略标签关系表 label_rel = heap_open(GsPolicyLabelRelationId, AccessShareLock); + + // 开始扫描策略标签关系表 scan = tableam_scan_begin(label_rel, SnapshotNow, 0, NULL); - while ((tup = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL) { + + // 迭代扫描策略标签关系表的每一行数据 + while ((tup = (HeapTuple)tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL) { + // 获取当前行的策略标签数据 label_data = (Form_gs_policy_label)GETSTRUCT(tup); + + // 判断策略标签是否与传入的表OID和列名匹配 if (label_data->fqdnid == relid && (column == NULL || strcasecmp(column, label_data->relcolumn.data) == 0)) { + // 复制标签名称，并添加到标签名称列表中 char *label_name = pstrdup(label_data->labelname.data); label_list = lappend(label_list, label_name); } } + + // 结束扫描 tableam_scan_end(scan); + + // 关闭策略标签关系表 heap_close(label_rel, AccessShareLock); + // 返回标签名称列表 return label_list; } +/* +oid_list 表示一个空的列表。 +policy_rel 表示关系表对象为空。 +scan 表示表扫描器对象为空。 +tup 表示堆元组对象为空。 +policy_data 表示脱敏策略数据结构指针为空。 +*/ static List *get_masking_policy_oidlist(bool check_enabled) { - List *oid_list = NIL; - Relation policy_rel = NULL; - TableScanDesc scan = NULL; - HeapTuple tup = NULL; - Form_gs_masking_policy policy_data = NULL; + List *oid_list = NIL; // 创建一个空的 OID 列表 + Relation policy_rel = NULL; // 脱敏策略关系表的关系对象 + TableScanDesc scan = NULL; // 表扫描器对象 + HeapTuple tup = NULL; // 堆元组对象 + Form_gs_masking_policy policy_data = NULL; // 脱敏策略的数据结构指针 - policy_rel = heap_open(GsMaskingPolicyRelationId, AccessShareLock); - scan = tableam_scan_begin(policy_rel, SnapshotNow, 0, NULL); + policy_rel = heap_open(GsMaskingPolicyRelationId, AccessShareLock); // 打开脱敏策略关系表并获取访问锁 + scan = tableam_scan_begin(policy_rel, SnapshotNow, 0, NULL); // 开始对表进行扫描 while ((tup = (HeapTuple)tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL) { - policy_data = (Form_gs_masking_policy)GETSTRUCT(tup); + // 遍历表中的每一行记录 + policy_data = (Form_gs_masking_policy)GETSTRUCT(tup); // 获取脱敏策略的数据 if (check_enabled && !policy_data->polenabled) { + // 如果需要检查启用状态且当前策略未启用，则跳过该策略 continue; } - oid_list = lappend_oid(oid_list, HeapTupleGetOid(tup)); + oid_list = lappend_oid(oid_list, HeapTupleGetOid(tup)); // 添加策略的 OID 到列表中 } - tableam_scan_end(scan); - heap_close(policy_rel, AccessShareLock); + tableam_scan_end(scan); // 结束表扫描 + heap_close(policy_rel, AccessShareLock); // 关闭脱敏策略关系表 - return oid_list; + return oid_list; // 返回 OID 列表 } +/* +is_masked 表示是否被脱敏。 +labels 用于存储脱敏策略的标签名。 +lc 用于遍历列表。 +action_rel 用于操作脱敏策略关系表。 +scan 用于扫描脱敏策略关系表。 +tup 用于存储扫描结果中的一行数据。 +action_data 是一个指向脱敏策略行数据的指针。 +*/ bool is_masked_relation(Oid relid, const char *column) { - bool is_masked = false; - List *labels = NIL; - ListCell *lc = NULL; - Relation action_rel = NULL; - TableScanDesc scan = NULL; - HeapTuple tup = NULL; - Form_gs_masking_policy_actions action_data = NULL; + bool is_masked = false; // 标识是否被脱敏 + List *labels = NIL; // 存储脱敏策略标签名列表 + ListCell *lc = NULL; // 列表遍历指针 + Relation action_rel = NULL; // 关系表对象 + TableScanDesc scan = NULL; // 表扫描器对象 + HeapTuple tup = NULL; // 堆元组对象 + Form_gs_masking_policy_actions action_data = NULL; // 脱敏策略行数据 - labels = relid_get_policy_label_namelist(relid, column); + labels = relid_get_policy_label_namelist(relid, column); // 获取指定列的脱敏策略标签名列表 if (labels == NIL) { - return false; + return false; // 如果标签名列表为空，则说明该列未被脱敏，直接返回false } - action_rel = heap_open(GsMaskingPolicyActionsId, AccessShareLock); - scan = tableam_scan_begin(action_rel, SnapshotNow, 0, NULL); + action_rel = heap_open(GsMaskingPolicyActionsId, AccessShareLock); // 打开脱敏策略关系表 + scan = tableam_scan_begin(action_rel, SnapshotNow, 0, NULL); // 开始表扫描 - while ((tup = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL && !is_masked) { - action_data = (Form_gs_masking_policy_actions)GETSTRUCT(tup); + while ((tup = (HeapTuple)tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL && !is_masked) { + action_data = (Form_gs_masking_policy_actions)GETSTRUCT(tup); // 获取脱敏策略行数据 foreach (lc, labels) { const char *labelname = (const char *)lfirst(lc); if (strcmp(labelname, action_data->actlabelname.data) == 0) { - is_masked = true; - break; + is_masked = true; // 如果找到与标签名匹配的脱敏策略行，设置is_masked为true + break; // 结束循环 } } } - tableam_scan_end(scan); - heap_close(action_rel, AccessShareLock); - list_free_deep(labels); + tableam_scan_end(scan); // 结束表扫描 + heap_close(action_rel, AccessShareLock); // 关闭关系表 + list_free_deep(labels); // 释放标签名列表的内存 - return is_masked; + return is_masked; // 返回是否被脱敏的结果 } + + bool is_masked_relation_enabled(Oid relid) { - bool is_match = false; - List *label_list = NIL; - List *polid_list = NIL; - ListCell *lc = NULL; - Relation action_rel = NULL; - TableScanDesc scan = NULL; - HeapTuple tuple = NULL; - Form_gs_masking_policy_actions action_data = NULL; + bool is_match = false; // 标识是否匹配脱敏策略 + List *label_list = NIL; // 存储关系的脱敏策略标签名列表 + List *polid_list = NIL; // 存储存在脱敏策略的OID列表 + ListCell *lc = NULL; // 列表遍历指针 + Relation action_rel = NULL; // 关系表对象 + TableScanDesc scan = NULL; // 表扫描器对象 + HeapTuple tuple = NULL; // 堆元组对象 + Form_gs_masking_policy_actions action_data = NULL; // 脱敏策略行数据 - polid_list = get_masking_policy_oidlist(true); - label_list = relid_get_policy_label_namelist(relid); - /* No masking policies or no relative resource labels */ + polid_list = get_masking_policy_oidlist(true); // 获取所有存在脱敏策略的OID列表 + label_list = relid_get_policy_label_namelist(relid); // 获取关系的脱敏策略标签名列表 + /* 没有脱敏策略或没有与资源相关的标签 */ if (polid_list == NIL || label_list == NIL) { list_free(polid_list); list_free(label_list); - return false; + return false; // 如果不存在脱敏策略或者标签，则直接返回false } - action_rel = heap_open(GsMaskingPolicyActionsId, AccessShareLock); - scan = tableam_scan_begin(action_rel, SnapshotNow, 0, NULL); + action_rel = heap_open(GsMaskingPolicyActionsId, AccessShareLock); // 打开脱敏策略关系表 + scan = tableam_scan_begin(action_rel, SnapshotNow, 0, NULL); // 开始表扫描 - while ((tuple = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL && !is_match) { - action_data = (Form_gs_masking_policy_actions)GETSTRUCT(tuple); + while ((tuple = (HeapTuple)tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL && !is_match) { + action_data = (Form_gs_masking_policy_actions)GETSTRUCT(tuple); // 获取脱敏策略行数据 foreach (lc, label_list) { const char *labelname = (const char *)lfirst(lc); if (strcmp(labelname, action_data->actlabelname.data) == 0 && list_member_oid(polid_list, action_data->policyoid)) { - is_match = true; - break; + is_match = true; // 如果存在匹配的脱敏策略，则设置is_match为true + break; // 结束循环 } } } - tableam_scan_end(scan); - heap_close(action_rel, AccessShareLock); - list_free_deep(label_list); - list_free(polid_list); + tableam_scan_end(scan); // 结束表扫描 + heap_close(action_rel, AccessShareLock); // 关闭关系表 + list_free_deep(label_list); // 释放标签名列表的内存 + list_free(polid_list); // 释放OID列表的内存 - return is_match; + return is_match; // 返回是否存在匹配的脱敏策略的结果 } + /* separate namespace and function name when user input to create masking policy */ -void parse_function_name(const char* func_name, char** parsed_funcname, Oid& schemaid) +void parse_function_name(const char *func_name, char **parsed_funcname, Oid &schemaid) { + // 定义变量rc，用于存储操作结果 errno_t rc = EOK; - const char* sch = strchr(func_name, '.'); + + // 在func_name中查找"."字符，并返回其位置 + const char *sch = strchr(func_name, '.'); + + // 为parsed_funcname分配内存空间，大小为func_name的长度加1 *parsed_funcname = (char *)palloc(strlen(func_name) + 1); + + // 如果sch存在，则表示函数名包括了命名空间 if (sch) { /* function name includes schema */ + // 计算命名空间在字符串中的位置 int name_pos = (sch - func_name); + + // 将函数名复制到parsed_funcname中，从name_pos+1位置开始复制，复制的长度为strlen(func_name) - name_pos rc = strncpy_s(*parsed_funcname, strlen(func_name) + 1, func_name + name_pos + 1, strlen(func_name) - name_pos); securec_check(rc, "\0", "\0"); - char* schema_name = NULL; + // 定义变量schema_name，用于存储命名空间名称 + char *schema_name = NULL; + // 为schema_name分配内存空间，大小为name_pos + 1 schema_name = (char *)palloc(name_pos + 1); + // 将命名空间复制到schema_name中，复制的长度为name_pos rc = strncpy_s(schema_name, name_pos + 1, func_name, name_pos); securec_check(rc, "\0", "\0"); + + // 调用SchemaNameGetSchemaOid函数，获取命名空间的OID，将结果赋值给schemaid schemaid = SchemaNameGetSchemaOid(schema_name, true); + + // 释放schema_name占用的内存空间 pfree(schema_name); } else { + // 如果函数名中不包含命名空间，则直接将函数名复制到parsed_funcname中 rc = strncpy_s(*parsed_funcname, strlen(func_name) + 1, func_name, strlen(func_name)); securec_check(rc, "\0", "\0"); + + // 将schemaid设置为默认的pg_catalog命名空间的OID schemaid = PG_CATALOG_NAMESPACE; } } bool IsMaskingFunctionOid(Oid funcid) { + // 如果函数OID无效，则返回false if (!OidIsValid(funcid)) { return false; } + + // 获取函数的命名空间OID Oid nspoid = get_func_namespace(funcid); + + // 获取函数的名称 char *funcname = get_func_name(funcid); + + // 如果命名空间OID为PG_CATALOG_NAMESPACE（表示函数在pg_catalog命名空间中） if (nspoid == PG_CATALOG_NAMESPACE) { + // 遍历g_maskFunctions数组，查找与函数名称相同的掩码函数 for (int i = 0; g_maskFunctions[i] != NULL; ++i) { if (strcmp(g_maskFunctions[i], funcname) == 0) { return true; } } } - return IsUDF(funcname, nspoid); + + // 调用IsUDF函数判断函数是否为用户定义的函数 + return IsUDF(funcname, nspoid); } + bool IsUDF(const char *funcname, Oid nspoid) { Relation rel = NULL; + + // 打开指定表 rel = heap_open(GsMaskingPolicyActionsId, AccessShareLock); - - TableScanDesc scan = heap_beginscan(rel, SnapshotNow, 0, NULL); - HeapTuple rtup = NULL; + + // 开始表扫描 + TableScanDesc scan = heap_beginscan(rel, SnapshotNow, 0, NULL); + HeapTuple rtup = NULL; Form_gs_masking_policy_actions rel_data = NULL; bool is_found = false; char *parsed_funcname = NULL; Oid nsp_oid = InvalidOid; - /* verify whether this function is used to be a masking function by searching gs_masking_policy_actions */ + + // 在gs_masking_policy_actions表中查找是否存在使用该函数作为掩码函数的记录 while ((rtup = heap_getnext(scan, ForwardScanDirection)) && !is_found) { rel_data = (Form_gs_masking_policy_actions)GETSTRUCT(rtup); if (rel_data == NULL) { continue; } + + // 解析存储在actiontype字段中的函数名 parse_function_name(rel_data->actiontype.data, &parsed_funcname, nsp_oid); + + // 比较解析得到的函数名和给定的函数名，以及命名空间OID是否匹配 if (strcasecmp(parsed_funcname, funcname) == 0 && nspoid == nsp_oid) { is_found = true; } + + // 释放解析得到的函数名内存 pfree(parsed_funcname); } + + // 结束表扫描 heap_endscan(scan); + + // 关闭表 heap_close(rel, AccessShareLock); + + // 返回是否找到匹配的记录 return is_found; } -- 2.34.1 From 103e5ffaa34c25e69282e351d68a5d28282bdeb3 Mon Sep 17 00:00:00 2001 From: Auweafan Date: Tue, 3 Oct 2023 17:49:00 +0800 Subject: [PATCH 34/48] Update gs_policy_utils.cpp --- .../security/gs_policy/gs_policy_utils.cpp | 360 +++++++++++------- 1 file changed, 232 insertions(+), 128 deletions(-) diff --git a/src/gausskernel/security/gs_policy/gs_policy_utils.cpp b/src/gausskernel/security/gs_policy/gs_policy_utils.cpp index fdc289cf9..69568f6c5 100755 --- a/src/gausskernel/security/gs_policy/gs_policy_utils.cpp +++ b/src/gausskernel/security/gs_policy/gs_policy_utils.cpp @@ -55,35 +55,40 @@ #define static #endif -GsSaveManagementEvent gs_save_mng_event_hook = NULL; -GsSendManagementEvent gs_send_mng_event_hook = NULL; +GsSaveManagementEvent gs_save_mng_event_hook = NULL;//保存管理事件的回调函数 +GsSendManagementEvent gs_send_mng_event_hook = NULL;//发送管理事件的回调函数 -void save_manage_message(const char* message) +void save_manage_message(const char* message)//用于保存管理事件 { if (gs_save_mng_event_hook != NULL) { gs_save_mng_event_hook(message); } } -void send_manage_message(AuditResult result_type) +void send_manage_message(AuditResult result_type)//用于发送管理事件 { if (gs_send_mng_event_hook != NULL) { gs_send_mng_event_hook(result_type); } } - -bool GsPolicyStruct::operator == (const GsPolicyStruct &arg) const +/* +这些函数是用于比较和排序操作的重载运算符，用于比较和操作不同结构体对象之间的关系。 +GsPolicyStruct的重载函数用于比较两个对象是否相等、判断大小关系以及执行减法运算。其中，比较是基于对象的名称进行的，使用了忽略大小写的字符串比较函数strcasecmp。 +PgPolicyFiltersStruct和PgPolicyPrivilegesAccessStruct的重载函数也是类似的，比较的依据包括类型、标签名称等属性，并根据字符串比较的结果以及整数比较的结果来确定对象之间的相等性、大小关系和减法结果。 +这些函数可以方便地在有序容器中进行插入、查找和删除操作，提高程序的效率。 +*/ +bool GsPolicyStruct::operator == (const GsPolicyStruct &arg) const//比较两个GsPolicyStruct对象是否相等 { return strcasecmp(m_name.c_str(), arg.m_name.c_str()) == 0; } -bool GsPolicyStruct::operator < (const GsPolicyStruct &arg) const +bool GsPolicyStruct::operator < (const GsPolicyStruct &arg) const//比较两个GsPolicyStruct对象的大小关系 { /* compare by name */ return strcasecmp(m_name.c_str(), arg.m_name.c_str()) < 0; } -int GsPolicyStruct::operator - (const GsPolicyStruct &arg) const +int GsPolicyStruct::operator - (const GsPolicyStruct &arg) const//执行两个GsPolicyStruct对象之间的减法运算，返回一个整数结果。 { if (*this < arg) { return -1; @@ -95,7 +100,7 @@ int GsPolicyStruct::operator - (const GsPolicyStruct &arg) const } bool PgPolicyFiltersStruct::operator == (const PgPolicyFiltersStruct &arg) const -{ +{//比较两个PgPolicyFiltersStruct对象是否相等 if (*this < arg) { return false; } else if (arg < *this) { @@ -106,7 +111,7 @@ bool PgPolicyFiltersStruct::operator == (const PgPolicyFiltersStruct &arg) const } bool PgPolicyFiltersStruct::operator < (const PgPolicyFiltersStruct &arg) const -{ +{//判断两个对象的大小关系 int res = strcasecmp(m_type.c_str(), arg.m_type.c_str()); if (res < 0) { return true; @@ -125,7 +130,7 @@ bool PgPolicyFiltersStruct::operator < (const PgPolicyFiltersStruct &arg) const } int PgPolicyFiltersStruct::operator - (const PgPolicyFiltersStruct &arg) const -{ +{//执行两个PgPolicyFiltersStruct对象之间的减法运算，返回一个整数结果。 if (*this < arg) { return -1; } else if (arg < *this) { @@ -136,13 +141,13 @@ int PgPolicyFiltersStruct::operator - (const PgPolicyFiltersStruct &arg) const } bool PgPolicyPrivilegesAccessStruct::operator == (const PgPolicyPrivilegesAccessStruct &arg) const -{ +{//比较两个PgPolicyPrivilegesAccessStruct对象是否相等。 return (strcasecmp(m_type.c_str(), arg.m_type.c_str()) == 0) && (strcasecmp(m_label_name.c_str(), arg.m_label_name.c_str()) == 0); } bool PgPolicyPrivilegesAccessStruct::operator < (const PgPolicyPrivilegesAccessStruct &arg) const -{ +{//operator<比较两个PgPolicyPrivilegesAccessStruct对象的大小关系。 int res = strcasecmp(m_type.c_str(), arg.m_type.c_str()); if (res < 0) { return true; @@ -154,7 +159,7 @@ bool PgPolicyPrivilegesAccessStruct::operator < (const PgPolicyPrivilegesAccessS } int PgPolicyPrivilegesAccessStruct::operator - (const PgPolicyPrivilegesAccessStruct &arg) const -{ +{//operator-执行两个PgPolicyPrivilegesAccessStruct对象之间的减法运算，返回一个整数结果。 if (*this < arg) { return -1; } else if (arg < *this) { @@ -164,177 +169,225 @@ int PgPolicyPrivilegesAccessStruct::operator - (const PgPolicyPrivilegesAccessSt } } + + + /* Process new filters from parser tree and tranform them into string */ +/* +* process_new_filters函数处理策略过滤器 +scan_to_delete_from_relation函数从关系中删除行 +construct_resource_name函数构建资源名称。 +都涉及对数据库、关系或数据的操作和处理,在一起使用时能够提供更全面和综合的数据库操作。 +*/ bool process_new_filters(const List *policy_filters, gs_stl::gs_string *flat_tree) { - if (!policy_filters) + if (!policy_filters) // 如果策略过滤器链表为空，则不进行处理，直接返回true return true; - flat_tree->clear(); - ListCell *policy_filter_item = NULL; - gs_stl::gs_vector nodes; + flat_tree->clear(); // 清空输出字符串 - foreach(policy_filter_item, policy_filters) { - PolicyFilterNode *root = (PolicyFilterNode *) lfirst(policy_filter_item); - nodes.push_back(root); - while (nodes.size() > 0) { - PolicyFilterNode* n = nodes.back(); - nodes.pop_back(); - /* operator type node */ - if (!strcmp(n->node_type, "op")) { - if (!strcmp(n->op_value, "and")) { - (void)flat_tree->append("*"); - } else if (!strcmp(n->op_value, "or")) { - (void)flat_tree->append("+"); - } else { /* unsupported operator */ - return false; + ListCell *policy_filter_item = NULL; // 定义策略过滤器链表节点指针 + gs_stl::gs_vector nodes; // 定义PolicyFilterNode指针的向量 + + foreach (policy_filter_item, policy_filters) { // 遍历策略过滤器链表中的每个节点 + PolicyFilterNode *root = (PolicyFilterNode *)lfirst(policy_filter_item); // 获取当前节点的指针 + nodes.push_back(root); // 将当前节点指针添加到nodes向量中 + while (nodes.size() > 0) { // 当nodes向量不为空时进行循环处理 + PolicyFilterNode *n = nodes.back(); // 获取nodes向量最后一个节点的指针 + nodes.pop_back(); // 移除nodes向量最后一个节点 + // operator type node 操作符类型节点 + if (!strcmp(n->node_type, "op")) { // 判断当前节点类型是否为操作符类型 + if (!strcmp(n->op_value, "and")) { // 如果是and操作符 + (void)flat_tree->append("*"); // 向输出字符串追加* + } else if (!strcmp(n->op_value, "or")) { // 如果是or操作符 + (void)flat_tree->append("+"); // 向输出字符串追加+ + } else { // 不支持的操作符类型 + return false; // 返回false表示处理失败 } - nodes.push_back((PolicyFilterNode *)n->right); - nodes.push_back((PolicyFilterNode *)n->left); - } else if (!strcmp(n->node_type, "filter")) { /* value type node */ - if (n->has_not_operator == true) { - (void)flat_tree->append("!"); + nodes.push_back((PolicyFilterNode *)n->right); // 将右子节点指针添加到nodes向量中 + nodes.push_back((PolicyFilterNode *)n->left); // 将左子节点指针添加到nodes向量中 + } else if (!strcmp(n->node_type, "filter")) { // value type node 值类型节点 + if (n->has_not_operator == true) { // 如果节点具有not操作符 + (void)flat_tree->append("!"); // 向输出字符串追加! } - (void)flat_tree->append(n->filter_type); - (void)flat_tree->append("["); - List *filter_item_objects = (List *) n->values; - ListCell *filter_obj = NULL; - foreach(filter_obj, filter_item_objects) { - const char *filter_value = (const char *)(((Value*)lfirst(filter_obj))->val.str); - if (!verify_ip_role_app(n->filter_type, filter_value, flat_tree)) { - return false; + (void)flat_tree->append(n->filter_type); // 向输出字符串追加节点的filter_type属性 + (void)flat_tree->append("["); // 向输出字符串追加[ + List *filter_item_objects = (List *)n->values; // 获取节点的值列表 + ListCell *filter_obj = NULL; // 定义值列表节点指针 + foreach (filter_obj, filter_item_objects) { // 遍历值列表中的每个节点 + const char *filter_value = + (const char *)(((Value *)lfirst(filter_obj))->val.str); // 获取当前节点的值并转为字符串 + if (!verify_ip_role_app(n->filter_type, filter_value, + flat_tree)) { // 验证节点的filter_type和filter_value是否有效 + return false; // 返回false表示处理失败 } - (void)flat_tree->append(","); + (void)flat_tree->append(","); // 向输出字符串追加逗号 } - if (flat_tree->back() == ',') { - flat_tree->pop_back(); + if (flat_tree->back() == ',') { // 如果输出字符串的最后一个字符是逗号 + flat_tree->pop_back(); // 移除输出字符串的最后一个字符（逗号） } - (void)flat_tree->append("]"); + (void)flat_tree->append("]"); // 向输出字符串追加] } } } - return true; + return true; // 返回true表示成功处理策略过滤器 } bool scan_to_delete_from_relation(long long row_id, Relation relation, unsigned int index_id) { - if (relation == NULL) { + if (relation == NULL) { // 如果关系为空则返回false return false; } - ScanKeyData skey; - /* Find the row to delete. */ - ScanKeyInit(&skey, - ObjectIdAttributeNumber, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(row_id)); + // 通过指定的行ID构建扫描键 + ScanKeyData skey; + ScanKeyInit(&skey, ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(row_id)); + + // 开始扫描关系表 SysScanDesc tgscan = systable_beginscan(relation, index_id, true, NULL, 1, &skey); + // 获取匹配的行 HeapTuple tup = systable_getnext(tgscan); - if (!HeapTupleIsValid(tup)) { + if (!HeapTupleIsValid(tup)) { // 如果获取失败则结束扫描并返回false systable_endscan(tgscan); return false; } - /* Delete the label tuple */ + + // 删除对应的元组 simple_heap_delete(relation, &tup->t_self); + // 结束扫描并返回true表示成功删除行 systable_endscan(tgscan); - return true; } void construct_resource_name(const RangeVar *rel, gs_stl::gs_string *target_name_s) { - if (rel->catalogname) { + if (rel->catalogname) { // 如果关系存在目录名 + // 向目标名称字符串追加目录名 (void)target_name_s->append((const char *)rel->catalogname); - target_name_s->push_back('.'); + target_name_s->push_back('.'); // 向目标名称字符串追加点号（表示目录和模式之间的分隔符） } - if (rel->schemaname) { + if (rel->schemaname) { // 如果关系存在模式名 + // 向目标名称字符串追加模式名 (void)target_name_s->append((const char *)rel->schemaname); - target_name_s->push_back('.'); + target_name_s->push_back('.'); // 向目标名称字符串追加点号（表示模式和表名之间的分隔符） } - if (rel->relname) { + if (rel->relname) { // 如果关系存在表名 + // 向目标名称字符串追加表名 (void)target_name_s->append(rel->relname); } } + /** * Check if current app is valid or not. */ bool verify_app_filter(const char* obj_value) { + // 检查字符串长度是否为0 if (strlen(obj_value) == 0) { + // 如果长度为0，报错并返回false ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("app: [%s] is invalid", obj_value))); return false; } /* The first character id numbers or dollar */ + // 获取字符串的第一个字符 char c = obj_value[0]; + // 如果第一个字符是数字或者美元符号，认为不合法，报错并返回false if ((c >= '0' && c <= '9') || c == '$') { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("app: [%s] is invalid", obj_value))); return false; } + // 获取字符串的长度 int len = strlen(obj_value); + // 遍历字符串的每个字符 for (int i = 0; i < len; i++) { c = obj_value[i]; + // 如果字符是字母、数字、下划线或美元符号，继续下一次循环 if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '$') { continue; } else { + // 否则，认为不合法，报错并返回false ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("app: [%s] is invalid", obj_value))); return false; } } + // 字符串通过所有验证规则，认为合法，返回true return true; } + /** * Check the effective of the filter information. * @ obj_type : filter type, which inlcudes IP, ROLES, and APPS. * @ obj_value : the actual filter information. * @ return_value : record about the filter information. */ -bool verify_ip_role_app(const char* obj_type, const char* obj_value, gs_stl::gs_string *return_value) +bool verify_ip_role_app(const char *obj_type, const char *obj_value, gs_stl::gs_string *return_value) { + // 检查对象类型是否为 "ip" if (!strcasecmp(obj_type, "ip")) { - const char* check_value = obj_value; + const char *check_value = obj_value; + // 检查 IP 范围是否合法 if (!IPRange::is_range_valid(check_value)) { - ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("ip range: [%s] is invalid, please identify", obj_value))); + // 如果不合法，报错并返回false + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("ip range: [%s] is invalid, please identify", obj_value))); return false; } + // 将合法的 IP 范围添加到返回值中 (void)return_value->append(check_value); return true; - } else if (!strcasecmp(obj_type, "roles")) { + } + // 检查对象类型是否为 "roles" + else if (!strcasecmp(obj_type, "roles")) { + // 获取角色的 Oid Oid uid = get_role_oid(obj_value, true); + // 检查角色是否有效 if (!OidIsValid(uid)) { + // 如果无效，报错并返回false ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("role: [%s] is invalid", obj_value))); return false; } char buffer[64]; /* buffer to store the oid int as string. 64 is the max length of oid. */ + // 将 Oid 转换为字符串 int nRet = snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, "%d", uid); securec_check_ss(nRet, "\0", "\0"); + // 将转换后的字符串添加到返回值中 (void)return_value->append(buffer); return true; - } else if (!strcasecmp(obj_type, "app")) { + } + // 检查对象类型是否为 "app" + else if (!strcasecmp(obj_type, "app")) { + // 验证应用程序过滤器是否合法 bool is_valid_app = verify_app_filter(obj_value); if (!is_valid_app) { + // 如果不合法，返回false return false; } } + // 将对象值添加到返回值中 (void)return_value->append(obj_value); return true; } -static bool add_privileges_access(const char *action_type, const char *label_name, - privileges_access_set *actions, const policy_labels_map *existing_labels, const GsPolicyStruct *policy, - gs_stl::gs_string *err_msg) + +static bool add_privileges_access(const char *action_type, const char *label_name, privileges_access_set *actions, + const policy_labels_map *existing_labels, const GsPolicyStruct *policy, + gs_stl::gs_string *err_msg) { + // 创建一个 PgPolicyPrivilegesAccessStruct 对象 PgPolicyPrivilegesAccessStruct item; item.m_type = action_type; item.m_label_name = label_name; item.m_policy_oid = policy->m_id; /* validate that such label exists */ + // 验证给定的标签是否存在于 existing_labels 中 if (existing_labels->find(label_name) == existing_labels->end()) { + // 如果标签不存在，生成错误消息并返回false err_msg->clear(); (void)err_msg->append("Trying to add/remove privilege/access ["); (void)err_msg->append(action_type); @@ -343,89 +396,109 @@ static bool add_privileges_access(const char *action_type, const char *label_nam (void)err_msg->append("]"); return false; } + // 将 PgPolicyPrivilegesAccessStruct 对象插入到 actions 中 (void)actions->insert(item); return true; } -bool handle_target(ListCell *target, - int opt_type, - bool is_add, - gs_stl::gs_string *err_msg, + +bool handle_target(ListCell *target, int opt_type, bool is_add, gs_stl::gs_string *err_msg, privileges_access_set *access_to_add, privileges_access_set *access_to_remove, privileges_access_set *privs_to_add, privileges_access_set *privs_to_remove, - const policy_labels_map *existing_labels, - const GsPolicyStruct *policy, const char *acc_action_type) + const policy_labels_map *existing_labels, const GsPolicyStruct *policy, const char *acc_action_type) { bool ret = false; switch (opt_type) { + // 处理 POLICY_OPT_PRIVILEGES 选项 case POLICY_OPT_PRIVILEGES: { - RangeVar *rel = (RangeVar*)lfirst(target); + RangeVar *rel = (RangeVar *)lfirst(target); gs_stl::gs_string target_name_s; - construct_resource_name((const RangeVar*)rel, &target_name_s); + construct_resource_name((const RangeVar *)rel, &target_name_s); if (is_add) { - ret = add_privileges_access(acc_action_type, target_name_s.c_str(), privs_to_add, - existing_labels, policy, err_msg); + // 调用 add_privileges_access 函数，添加权限信息到 privs_to_add + ret = add_privileges_access(acc_action_type, target_name_s.c_str(), privs_to_add, existing_labels, + policy, err_msg); } else { - ret = add_privileges_access(acc_action_type, target_name_s.c_str(), privs_to_remove, existing_labels, + // 调用 add_privileges_access 函数，添加权限信息到 privs_to_remove + ret = add_privileges_access(acc_action_type, target_name_s.c_str(), privs_to_remove, existing_labels, policy, err_msg); } - } - break; + } break; + // 处理 POLICY_OPT_ACCESS 选项 case POLICY_OPT_ACCESS: { - RangeVar *rel = (RangeVar*)lfirst(target); + RangeVar *rel = (RangeVar *)lfirst(target); gs_stl::gs_string target_name_s; - construct_resource_name((const RangeVar*)rel, &target_name_s); + construct_resource_name((const RangeVar *)rel, &target_name_s); if (is_add) { + // 调用 add_privileges_access 函数，添加访问控制信息到 access_to_add ret = add_privileges_access(acc_action_type, target_name_s.c_str(), access_to_add, existing_labels, policy, err_msg); } else { + // 调用 add_privileges_access 函数，添加访问控制信息到 access_to_remove ret = add_privileges_access(acc_action_type, target_name_s.c_str(), access_to_remove, existing_labels, policy, err_msg); } - } - break; - // : this is not handled here... + } break; + // 默认情况下不处理 default: break; } return ret; } -static bool parse_values(const gs_stl::gs_string logical_expr_str, int *offset, const char* obj_type) + +static bool parse_values(const gs_stl::gs_string logical_expr_str, int *offset, const char *obj_type) { + /* + logical_expr_str：逻辑表达式字符串，需要解析的字符串。 + offset：解析起始位置的偏移量指针，用于记录解析的进度，并在函数内更新偏移量的值。 + obj_type：目标对象类型，一个字符串参数，表示需要验证的目标对象类型。 + */ std::size_t found = gs_stl::gs_string::npos; + // 创建一个缓冲区 char buff[512] = {0}; + // 在逻辑表达式字符串中查找下一个 ']' 的位置，限制范围的末尾 size_t limit_pos = logical_expr_str.find(']', *offset); + // 临时存储解析结果的字符串 gs_stl::gs_string tmp_res; + // 解析是否成功的标志 bool parsed = true; + // 过滤器有效性的标志 bool filter_valid = false; int nRet; - /* not finding last ']' means error */ + + // 如果没有找到最后一个 ']'，则表示错误 if (limit_pos == gs_stl::gs_string::npos) { - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("filter: [%s] is invalid", logical_expr_str.c_str()))); + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("filter: [%s] is invalid", logical_expr_str.c_str()))); return false; } + + // 在逗号 ',' 出现之前以及限制范围内，循环解析 while ((found = logical_expr_str.find(',', *offset)) != gs_stl::gs_string::npos && found < limit_pos) { + // 将解析出的部分拷贝到缓冲区中 nRet = snprintf_s(buff, sizeof(buff), sizeof(buff) - 1, "%.*s", (int)(found - *offset), - logical_expr_str.c_str() + *offset); + logical_expr_str.c_str() + *offset); securec_check_ss(nRet, "\0", "\0"); + // 验证 IP、角色、应用程序等，返回过滤器的有效性，并将结果存储在 tmp_res 中 filter_valid = verify_ip_role_app(obj_type, buff, &tmp_res); + // 更新 parsed 的值，如果有一个解析失败，则 parsed 为 false parsed = parsed && filter_valid; *offset = found + 1; } + // 如果解析还未结束，处理剩余部分 if (*offset < (int)limit_pos) { nRet = snprintf_s(buff, sizeof(buff), sizeof(buff) - 1, "%.*s", (int)(limit_pos - *offset), - logical_expr_str.c_str() + *offset); + logical_expr_str.c_str() + *offset); securec_check_ss(nRet, "\0", "\0"); filter_valid = verify_ip_role_app(obj_type, buff, &tmp_res); parsed = parsed && filter_valid; *offset = limit_pos + 1; return parsed; } - /* getting here means error */ + + // 执行到这里表示错误，抛出错误信息并返回 false ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("filter: [%s] is invalid", logical_expr_str.c_str()))); return false; } @@ -435,85 +508,116 @@ bool validate_logical_expression(const gs_stl::gs_string logical_expr_str, int * { int logical_expr_len = logical_expr_str.size(); - while (*offset < logical_expr_len) { + while (*offset < logical_expr_len) { // 循环遍历表达式字符串 /* AND/OR node */ - if ((logical_expr_str[*offset] == '*') || (logical_expr_str[*offset] == '+')) { - (*offset)++; - return (validate_logical_expression(logical_expr_str, offset) /* go left */ - && validate_logical_expression(logical_expr_str, offset)); /* go right */ - } else if (logical_expr_str[*offset] == '!') { /* NOT operator */ - (*offset)++; - } else if (logical_expr_str[*offset] == 'i') { /* IP filter node */ - *offset += 3; /* 3 : skip 'ip[' */ - return parse_values(logical_expr_str, offset, "ip"); - } else if (logical_expr_str[*offset] == 'r') { /* ROLE filter node */ - *offset += 6; /* 6 : skip 'roles[' */ - return parse_values(logical_expr_str, offset, "roles"); - } else if (logical_expr_str[*offset] == 'a') { /* APPLICATION filter node */ - *offset += 4; /* 4 : skip 'app[' */ - return parse_values(logical_expr_str, offset, "app"); + if ((logical_expr_str[*offset] == '*') || (logical_expr_str[*offset] == '+')) { // 如果当前字符为*或+ + (*offset)++; // 偏移量自增1 + return (validate_logical_expression(logical_expr_str, offset) /* go left */ + && validate_logical_expression(logical_expr_str, offset)); /* go right */ + // 递归调用validate_logical_expression，分别处理左右两个子表达式，并将结果进行AND/OR操作返回 + } else if (logical_expr_str[*offset] == '!') { /* NOT operator */ // 如果当前字符为! + (*offset)++; // 偏移量自增1 + } else if (logical_expr_str[*offset] == 'i') { /* IP filter node */ // 如果当前字符为i + *offset += 3; /* 3 : skip 'ip[' */ // 偏移量增加3，跳过'ip[' + return parse_values(logical_expr_str, offset, "ip"); // 调用parse_values函数解析IP值，并返回结果 + } else if (logical_expr_str[*offset] == 'r') { /* ROLE filter node */ // 如果当前字符为r + *offset += 6; /* 6 : skip 'roles[' */ // 偏移量增加6，跳过'roles[' + return parse_values(logical_expr_str, offset, "roles"); // 调用parse_values函数解析角色值，并返回结果 + } else if (logical_expr_str[*offset] == 'a') { /* APPLICATION filter node */ // 如果当前字符为a + *offset += 4; /* 4 : skip 'app[' */ // 偏移量增加4，跳过'app[' + return parse_values(logical_expr_str, offset, "app"); // 调用parse_values函数解析应用程序值，并返回结果 } } - return false; + return false; // 如果表达式字符串已经遍历完毕，则返回false } + void get_session_ip(char *session_ip, int len) { + // 检查提供的缓冲区长度是否足够 if (len < MAX_IP_LEN) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("length of session ip buffer should more than 128"))); } - struct sockaddr* remote_addr = (struct sockaddr *)&u_sess->proc_cxt.MyProcPort->raddr.addr; + + // 获取远程地址 + struct sockaddr *remote_addr = (struct sockaddr *)&u_sess->proc_cxt.MyProcPort->raddr.addr; + + // 错误码变量 errno_t rc = EOK; - /* parse the remote ip address */ + + // 解析远程IP地址 if (AF_UNIX == remote_addr->sa_family) { + // 如果远程地址是UNIX域套接字(AF_UNIX)，将session_ip设置为"local" char *localstr = "local"; rc = memcpy_s(session_ip, len, localstr, strlen(localstr)); securec_check(rc, "\0", "\0"); } else { + // 否则调用get_client_ip函数获取客户端IP地址，并将结果存储在session_ip中 get_client_ip(remote_addr, session_ip); } } -void get_client_ip(const struct sockaddr* remote_addr, char *ip_str) + +void get_client_ip(const struct sockaddr *remote_addr, char *ip_str) { - /* parse the remote ip address */ + /* 解析远程IP地址 */ + + // 如果远程地址为IPv6（AF_INET6），使用inet_ntop函数将IPv6地址转换为字符串形式 if (AF_INET6 == remote_addr->sa_family) { - (void)inet_ntop(AF_INET6, &((struct sockaddr_in6*)remote_addr)->sin6_addr, ip_str, MAX_IP_LEN - 1); - } else if (AF_INET == remote_addr->sa_family) { - (void)inet_ntop(AF_INET, &((struct sockaddr_in*)remote_addr)->sin_addr, ip_str, MAX_IP_LEN - 1); + (void)inet_ntop(AF_INET6, &((struct sockaddr_in6 *)remote_addr)->sin6_addr, ip_str, MAX_IP_LEN - 1); + } + // 如果远程地址为IPv4（AF_INET），使用inet_ntop函数将IPv4地址转换为字符串形式 + else if (AF_INET == remote_addr->sa_family) { + (void)inet_ntop(AF_INET, &((struct sockaddr_in *)remote_addr)->sin_addr, ip_str, MAX_IP_LEN - 1); } } -bool is_database_valid(const char* dbname) + +bool is_database_valid(const char *dbname) { + // 检查数据库名是否为空 if (dbname == NULL) { return false; } + // 获取数据库的OID（对象标识符） Oid db_oid = get_database_oid(dbname, false); + // 检查数据库的OID是否有效 if (OidIsValid(db_oid)) { return true; } + // 如果数据库的OID无效, 返回false return false; } -ResourceOwnerData* create_temp_resourceowner() +ResourceOwnerData *create_temp_resourceowner() { - ResourceOwner tmpOwner = ResourceOwnerCreate(t_thrd.utils_cxt.CurrentResourceOwner, - "CheckUserOid", THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY)); + // 创建临时资源拥有者 + ResourceOwner tmpOwner = ResourceOwnerCreate(t_thrd.utils_cxt.CurrentResourceOwner, "CheckUserOid", + THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY)); + // 保存当前资源拥有者 ResourceOwner currentOwner = t_thrd.utils_cxt.CurrentResourceOwner; + // 将当前资源拥有者切换为临时资源拥有者 t_thrd.utils_cxt.CurrentResourceOwner = tmpOwner; + // 返回当前资源拥有者 return currentOwner; } -void release_temp_resourceowner(ResourceOwnerData* resource_owner) + +void release_temp_resourceowner(ResourceOwnerData *resource_owner) { + // 保存当前资源拥有者到临时变量 ResourceOwner tmpOwner = t_thrd.utils_cxt.CurrentResourceOwner; + // 释放锁之前需要释放的资源 ResourceOwnerRelease(tmpOwner, RESOURCE_RELEASE_BEFORE_LOCKS, true, true); + // 释放锁占用的资源 ResourceOwnerRelease(tmpOwner, RESOURCE_RELEASE_LOCKS, true, true); + // 释放锁之后需要释放的资源 ResourceOwnerRelease(tmpOwner, RESOURCE_RELEASE_AFTER_LOCKS, true, true); + // 恢复之前的资源拥有者 t_thrd.utils_cxt.CurrentResourceOwner = resource_owner; + // 删除临时资源拥有者 ResourceOwnerDelete(tmpOwner); } -- 2.34.1 From 6902501b654a8f2e34bc7b8d66a887e33250cab8 Mon Sep 17 00:00:00 2001 From: Auweafan Date: Tue, 3 Oct 2023 17:49:29 +0800 Subject: [PATCH 35/48] Update policy_common.cpp --- .../security/gs_policy/policy_common.cpp | 813 ++++++++++++------ 1 file changed, 538 insertions(+), 275 deletions(-) diff --git a/src/gausskernel/security/gs_policy/policy_common.cpp b/src/gausskernel/security/gs_policy/policy_common.cpp index e3d1ce968..8e5b7b1ba 100644 --- a/src/gausskernel/security/gs_policy/policy_common.cpp +++ b/src/gausskernel/security/gs_policy/policy_common.cpp @@ -58,6 +58,7 @@ #define MAX_MSG_BUFF_SIZE 512 #define POLICY_VIEWS_NUM 5 +//用于存储对应函数的地址，以便在程序运行过程中调用这些函数。 LoadLabelsPtr load_labels_hook = NULL; CheckPolicyPrivilegesForLabelPtr check_audit_policy_privileges_for_label_hook = NULL; CheckPolicyAccessForLabelPtr check_audit_policy_access_for_label_hook = NULL; @@ -67,32 +68,35 @@ IsLabelExistPtr verify_label_hook = NULL; bool GsPolicyLabel::operator == (const GsPolicyLabel &arg) const { + // 比较数据值和数据类型是否相等 return (strcasecmp(m_data_value.c_str(), arg.m_data_value.c_str()) == 0) && (strcasecmp(m_data_type.c_str(), arg.m_data_type.c_str()) == 0); } -bool GsPolicyLabel::operator < (const GsPolicyLabel& arg) const + +bool GsPolicyLabel::operator<(const GsPolicyLabel &arg) const { - /* first compare by data value */ + // 首先按照数据值进行比较 int res = strcasecmp(m_data_value.c_str(), arg.m_data_value.c_str()); if (res < 0) { - return true; + return true; // 如果当前对象的数据值小于参数对象的数据值，则返回true } if (res > 0) { - return false; + return false; // 如果当前对象的数据值大于参数对象的数据值，则返回false } - /* if data value is equal, compare by data type */ + // 若数据值相等，按照数据类型进行比较 return (strcasecmp(m_data_type.c_str(), arg.m_data_type.c_str()) < 0); } -int GsPolicyLabel::operator - (const GsPolicyLabel& arg) const + +int GsPolicyLabel::operator-(const GsPolicyLabel &arg) const { if (*this < arg) { - return -1; + return -1; // 如果当前对象小于参数对象，则返回-1 } else if (arg < *this) { - return 1; + return 1; // 如果当前对象大于参数对象，则返回1 } else { - return 0; + return 0; // 如果当前对象等于参数对象，则返回0 } } @@ -105,11 +109,14 @@ static inline void append_schema(GsPolicyFQDN *fqdn, const char *schema) { if (schema != NULL) { fqdn->m_value_schema = get_namespace_oid(schema, true); + // 如果schema不为NULL，使用get_namespace_oid函数获取schema的oid } else { fqdn->m_value_schema = SchemaNameGetSchemaOid(NULL); + // 如果schema为NULL，使用SchemaNameGetSchemaOid(NULL)获取当前schema的oid } } + /* * append_object * @@ -119,22 +126,24 @@ static bool append_object(GsPolicyFQDN *fqdn, const char *object, const char *la { fqdn->is_function = false; if (object == NULL) { - return false; + return false; // object为NULL返回false } if (!strcasecmp(label_item_type, "function")) { fqdn->m_value_object = get_func_oid(object, fqdn->m_value_schema, NULL); fqdn->is_function = true; - } else if (!strcasecmp(label_item_type, "table") || - !strcasecmp(label_item_type, "view") || + // 如果label_item_type为"function"，调用get_func_oid函数获取函数的OID，并将is_function设置为true + } else if (!strcasecmp(label_item_type, "table") || !strcasecmp(label_item_type, "view") || !strcasecmp(label_item_type, "column")) { fqdn->m_value_object = get_relname_relid(object, fqdn->m_value_schema); + // 如果label_item_type为"table"、"view"或"column"，调用get_relname_relid函数获取表、视图或列的OID } if (OidIsValid(fqdn->m_value_object)) { return true; - } + } // 如果获取到的OID有效，返回true；否则返回false return false; } + /* * fqdn_get_name * @@ -142,6 +151,7 @@ static bool append_object(GsPolicyFQDN *fqdn, const char *object, const char *la */ static bool fqdn_get_name(const char *label_item_type, GsPolicyFQDN *fqdn, RangeVar *rel) { + // 如果 label_item_type 是 "schema"，则将 rel->relname 添加到 fqdn 标签中作为 schema 名称 if (!strcasecmp(label_item_type, "schema")) { append_schema(fqdn, rel->relname); return true; @@ -149,22 +159,27 @@ static bool fqdn_get_name(const char *label_item_type, GsPolicyFQDN *fqdn, Range bool is_column = !strcasecmp(label_item_type, "column"); if (is_column) { + // 如果是列，先将 rel->catalogname 添加到 fqdn 标签中作为 catalog 名称 append_schema(fqdn, rel->catalogname); + // 然后将 rel->schemaname 添加到 fqdn 标签中作为 schema 名称，并添加列对象的 OID 到标签中 if (!append_object(fqdn, rel->schemaname, label_item_type)) { return false; } + // 最后将 rel->relname（列名）添加到 fqdn 标签中作为列属性名称 if (rel->relname != NULL) { fqdn->m_value_object_attrib = rel->relname; } else { return false; } } else { + // 如果不是列，则将 rel->schemaname 添加到 fqdn 标签中作为 schema 名称，并添加对象的 OID 到标签中 append_schema(fqdn, rel->schemaname); if (!append_object(fqdn, rel->relname, label_item_type)) { return false; } } + // 如果是列，并且未能获取到列的对象 OID，则返回 false if (is_column && fqdn->m_value_object == 0) { return false; } @@ -176,20 +191,34 @@ static bool fqdn_get_name(const char *label_item_type, GsPolicyFQDN *fqdn, Range * * get oid from schemaname or relname */ +/* + * get_relation_id + * + * 从 RangeVar 中获取 schemaname 或 relname 的对象 OID（Object ID）。 + */ static Oid get_relation_id(RangeVar *rel, bool is_column) { + // 根据是列还是表来确定 schemaname 和 relname 的取值 const char *schemaname = is_column ? rel->catalogname : rel->schemaname; const char *relname = is_column ? rel->schemaname : rel->relname; + + // 如果 schemaname 不为空，则获取命名空间的 OID if (schemaname != NULL) { Oid namespaceId = get_namespace_oid(schemaname, true); + + // 如果命名空间的 OID 有效，则获取 relname 对应的对象 OID if (OidIsValid(namespaceId)) { return get_relname_relid(relname, namespaceId); } + return 0; } + + // 如果 relname 不为空，则直接获取 relname 对应的对象 OID return (relname != NULL) ? RelnameGetRelid(relname) : 0; } + /* * verify_function_name * @@ -198,13 +227,15 @@ static Oid get_relation_id(RangeVar *rel, bool is_column) static bool verify_function_name(Oid namespaceId, const char *funcname) { if (!OidIsValid(namespaceId)) { - return false; + return false; // 如果命名空间ID无效，返回false } - bool is_found = false; + bool is_found = false; // 初始化是否找到函数的标志位 + + CatCList *catlist = NULL; // 定义一个指向CatCList结构体的指针 - CatCList *catlist = NULL; #ifndef ENABLE_MULTIPLE_NODES + // 根据t_thrd.proc->workingVersionNum的值判断调用哪个系统缓存搜索函数 if (t_thrd.proc->workingVersionNum < 92470) { catlist = SearchSysCacheList1(PROCNAMEARGSNSP, CStringGetDatum(funcname)); } else { @@ -213,19 +244,65 @@ static bool verify_function_name(Oid namespaceId, const char *funcname) #else catlist = SearchSysCacheList1(PROCNAMEARGSNSP, CStringGetDatum(funcname)); #endif - if (catlist != NULL) { + + if (catlist != NULL) { // 如果系统缓存列表不为空 for (int i = 0; i < catlist->n_members && !is_found; ++i) { - HeapTuple proctup = t_thrd.lsc_cxt.FetchTupleFromCatCList(catlist, i); - Form_pg_proc procform = (Form_pg_proc) GETSTRUCT(proctup); + // 检索系统缓存列表中的成员 + HeapTuple proctup = t_thrd.lsc_cxt.FetchTupleFromCatCList(catlist, i); + Form_pg_proc procform = (Form_pg_proc)GETSTRUCT(proctup); + + // 检查procform是否有效且pronamespace是否等于给定的namespaceId if (procform != NULL && procform->pronamespace == namespaceId) { Oid funcid = HeapTupleGetOid(proctup); - is_found = OidIsValid(funcid); + is_found = OidIsValid(funcid); // 更新是否找到函数的标志位 } } - ReleaseSysCacheList(catlist); + + ReleaseSysCacheList(catlist); // 释放系统缓存列表资源 } - return is_found; + + return is_found; // 返回是否找到函数的结果 } +static bool verify_function_name(Oid namespaceId, const char *funcname) +{ + if (!OidIsValid(namespaceId)) { + return false; // 如果命名空间ID无效，返回false + } + + bool is_found = false; // 初始化是否找到函数的标志位 + + CatCList *catlist = NULL; // 定义一个指向CatCList结构体的指针 + +#ifndef ENABLE_MULTIPLE_NODES + // 根据t_thrd.proc->workingVersionNum的值判断调用哪个系统缓存搜索函数 + if (t_thrd.proc->workingVersionNum < 92470) { + catlist = SearchSysCacheList1(PROCNAMEARGSNSP, CStringGetDatum(funcname)); + } else { + catlist = SearchSysCacheList1(PROCALLARGS, CStringGetDatum(funcname)); + } +#else + catlist = SearchSysCacheList1(PROCNAMEARGSNSP, CStringGetDatum(funcname)); +#endif + + if (catlist != NULL) { // 如果系统缓存列表不为空 + for (int i = 0; i < catlist->n_members && !is_found; ++i) { + // 检索系统缓存列表中的成员 + HeapTuple proctup = t_thrd.lsc_cxt.FetchTupleFromCatCList(catlist, i); + Form_pg_proc procform = (Form_pg_proc)GETSTRUCT(proctup); + + // 检查procform是否有效且pronamespace是否等于给定的namespaceId + if (procform != NULL && procform->pronamespace == namespaceId) { + Oid funcid = HeapTupleGetOid(proctup); + is_found = OidIsValid(funcid); // 更新是否找到函数的标志位 + } + } + + ReleaseSysCacheList(catlist); // 释放系统缓存列表资源 + } + + return is_found; // 返回是否找到函数的结果 +} + /* * get_view_rules @@ -252,40 +329,50 @@ static List *get_view_rules(RuleLock *rules) */ static bool verify_column_name(Oid relid, const char *colname) { - bool is_found = false; - Relation tbl_rel = relation_open(relid, AccessShareLock); - if (RelationIsValid(tbl_rel)) { - if (tbl_rel->rd_rules != NULL) { /* view */ - List *locks = get_view_rules(tbl_rel->rd_rules); - if (locks != NULL) { + bool is_found = false; // 初始化是否找到列名的标志位为false + + Relation tbl_rel = relation_open(relid, AccessShareLock); // 打开关系对象（表或视图） + + if (RelationIsValid(tbl_rel)) { // 判断关系对象是否有效 + + if (tbl_rel->rd_rules != NULL) { // 判断关系对象是否是视图 + + List *locks = get_view_rules(tbl_rel->rd_rules); // 获取视图规则列表 + if (locks != NULL) { // 判断视图规则列表是否为空 + ListCell *item = NULL; - foreach (item, locks) { - RewriteRule *rule = (RewriteRule *)lfirst(item); - Query *rule_action = (Query *)linitial(rule->actions); + foreach (item, locks) { // 遍历视图规则列表中的每个规则 + RewriteRule *rule = (RewriteRule *)lfirst(item); // 获取当前规则 + Query *rule_action = (Query *)linitial(rule->actions); // 获取规则动作 ListCell *target_item = NULL; - foreach (target_item, rule_action->targetList) { - TargetEntry *old_tle = (TargetEntry *)lfirst(target_item); - if (!strcasecmp(old_tle->resname, colname)) { - heap_close(tbl_rel, AccessShareLock); - return true; + foreach (target_item, rule_action->targetList) { // 遍历规则动作的目标列表 + TargetEntry *old_tle = (TargetEntry *)lfirst(target_item); // 获取目标项 + if (!strcasecmp(old_tle->resname, colname)) { // 比较目标项名称与列名是否相等 + heap_close(tbl_rel, AccessShareLock); // 关闭关系对象 + return true; // 如果找到匹配的列名，则返回true } } } - list_free(locks); + + list_free(locks); // 释放视图规则列表 } } - /* verify column */ - if (tbl_rel->rd_att != NULL) { - for (int i = 0; i < tbl_rel->rd_att->natts && !is_found; ++i) { - Form_pg_attribute attr = tbl_rel->rd_att->attrs[i]; - is_found = (strcasecmp(attr->attname.data, colname) == 0); + + // 验证列名 + if (tbl_rel->rd_att != NULL) { // 判断关系对象是否具有属性信息 + for (int i = 0; i < tbl_rel->rd_att->natts && !is_found; ++i) { // 遍历属性信息中的每个属性 + Form_pg_attribute attr = tbl_rel->rd_att->attrs[i]; // 获取当前属性 + is_found = (strcasecmp(attr->attname.data, colname) == 0); // 比较属性名称与列名是否相等 } } - heap_close(tbl_rel, AccessShareLock); + + heap_close(tbl_rel, AccessShareLock); // 关闭关系对象 } - return is_found; + + return is_found; // 返回是否找到列名的结果 } + /* * verify_temp_table * @@ -293,38 +380,43 @@ static bool verify_column_name(Oid relid, const char *colname) */ static void verify_temp_table(Oid relid, const char *relname, Relation relation, bool ignore = false) { - HeapTuple tuple = SearchSysCache1(RELOID, relid); - if (!HeapTupleIsValid(tuple)) { - heap_close(relation, RowExclusiveLock); - ereport(ERROR, - (errcode(ERRCODE_CACHE_LOOKUP_FAILED), - errmsg("Cache lookup failed for relation %u", relid))); + HeapTuple tuple = SearchSysCache1(RELOID, relid); // 在系统缓存中搜索关系对象元组 + if (!HeapTupleIsValid(tuple)) { // 判断关系对象元组是否有效 + heap_close(relation, RowExclusiveLock); // 关闭关系对象 + ereport(ERROR, (errcode(ERRCODE_CACHE_LOOKUP_FAILED), + errmsg("Cache lookup failed for relation %u", relid))); // 报告缓存查找失败的错误信息 } - Form_pg_class reltup = (Form_pg_class) GETSTRUCT(tuple); - /* Current object must be a normal table. - * 'ignore' means whether report error when resource type is not TABLE. - */ - if (reltup->relkind != RELKIND_RELATION && !ignore) { - ReleaseSysCache(tuple); - heap_close(relation, RowExclusiveLock); - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is not a normal table", relname))); + Form_pg_class reltup = (Form_pg_class)GETSTRUCT(tuple); // 获取关系对象元组的结构体形式 + + /* 当前对象必须是普通表。'ignore'表示在资源类型不是TABLE时是否报告错误。*/ + if (reltup->relkind != RELKIND_RELATION && !ignore) { // 判断关系对象是否是普通表，并检查是否应忽略错误 + ReleaseSysCache(tuple); // 释放关系对象元组的系统缓存 + heap_close(relation, RowExclusiveLock); // 关闭关系对象 + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not a normal table", relname))); // 报告错误，指定对象不是普通表 } - /* Check temp table or not */ - if (reltup->relpersistence == RELPERSISTENCE_TEMP) { - ReleaseSysCache(tuple); - heap_close(relation, RowExclusiveLock); + /* 检查是否为临时表 */ + if (reltup->relpersistence == RELPERSISTENCE_TEMP) { // 判断关系对象是否为临时表 + ReleaseSysCache(tuple); // 释放关系对象元组的系统缓存 + heap_close(relation, RowExclusiveLock); // 关闭关系对象 ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("Do not support policy label on temp table \"%s\"", relname))); + (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("Do not support policy label on temp table \"%s\"", + relname))); // 报告错误，不支持在临时表上设置策略标签 } - /* relase sys cache tuple */ + /* 释放系统缓存中的关系对象元组 */ ReleaseSysCache(tuple); } +/* +T_TABLE：表示标签类型为表格。 +T_COLUMN：表示标签类型为列。 +T_FUNCTION：表示标签类型为函数。 +T_VIEW：表示标签类型为视图。 +T_SCHEMA：表示标签类型为模式。 +T_UNKNOWN：表示标签类型未知。 +*/ enum LabelTypeEnum { T_TABLE, T_COLUMN, @@ -339,85 +431,124 @@ static int get_label_type(const char *label_type) if (!strcasecmp(label_type, "table")) { return T_TABLE; } + //判断标签类型是否为“table”，不区分大小写，如果是，则返回T_TABLE枚举常量 + if (!strcasecmp(label_type, "column")) { return T_COLUMN; } + //判断标签类型是否为“column”，不区分大小写，如果是，则返回T_COLUMN枚举常量 + if (!strcasecmp(label_type, "function")) { return T_FUNCTION; } + //判断标签类型是否为“function”，不区分大小写，如果是，则返回T_FUNCTION枚举常量 + if (!strcasecmp(label_type, "view")) { return T_VIEW; } + //判断标签类型是否为“view”，不区分大小写，如果是，则返回T_VIEW枚举常量 + if (!strcasecmp(label_type, "schema")) { return T_SCHEMA; } + //判断标签类型是否为“schema”，不区分大小写，如果是，则返回T_SCHEMA枚举常量 + return T_UNKNOWN; + //如果标签类型不属于上述几种类型，则返回T_UNKNOWN枚举常量 } + static void check_schema_type_exists(RangeVar *rel, Relation relation) { + //判断当前节点是否是PGXC协调器或单节点，并且指定的命名空间OID不存在 if ((IS_PGXC_COORDINATOR || IS_SINGLE_NODE) && !get_namespace_oid(rel->relname, true)) { + //释放关系并抛出错误信息，提醒找不到指定的模式 heap_close(relation, RowExclusiveLock); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("[%s] no such schema found", rel->relname))); } } + static void check_table_type_exists(RangeVar *rel, Relation relation) { + //获取给定 RangeVar 表示的关系的 OID Oid relid = get_relation_id(rel, false); + + //如果 OID 无效，说明找不到该关系 if (!OidIsValid(relid)) { if (IS_PGXC_COORDINATOR || IS_SINGLE_NODE) { /* return error */ heap_close(relation, RowExclusiveLock); - const char *schemaname = rel->schemaname ? - rel->schemaname : get_namespace_name(SchemaNameGetSchemaOid(NULL)); - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("[%s.%s] no such relation found", schemaname, rel->relname))); + + //获取要查询的关系的模式名，如果没有指定，则返回当前默认模式的名称 + const char *schemaname = + rel->schemaname ? rel->schemaname : get_namespace_name(SchemaNameGetSchemaOid(NULL)); + + //抛出错误信息，提醒找不到指定的关系 + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("[%s.%s] no such relation found", schemaname, rel->relname))); } } else { + //如果找到了该关系，则验证是否是一个临时表，并返回结果 verify_temp_table(relid, rel->relname, relation); } } + static void check_view_type_exists(RangeVar *rel, Relation relation) { + //获取给定 RangeVar 表示的关系的 OID Oid relid = get_relation_id(rel, false); + + //如果 OID 无效，说明找不到该视图 if ((IS_PGXC_COORDINATOR || IS_SINGLE_NODE) && !OidIsValid(relid)) { heap_close(relation, RowExclusiveLock); - const char *schemaname = rel->schemaname ? - rel->schemaname : get_namespace_name(SchemaNameGetSchemaOid(NULL)); + + //获取要查询的视图的模式名，如果没有指定，则返回当前默认模式的名称 + const char *schemaname = rel->schemaname ? rel->schemaname : get_namespace_name(SchemaNameGetSchemaOid(NULL)); + + //抛出错误信息，提醒找不到指定的视图 ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("[%s.%s] no such view found", schemaname, rel->relname))); + (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("[%s.%s] no such view found", schemaname, rel->relname))); } } + static void check_column_type_exists(RangeVar *rel, Relation relation) { + //获取给定 RangeVar 表示的关系的 OID，同时检查是否是一个临时表 Oid relid = get_relation_id(rel, true); + /* Create resource label on temp table is not allowed. */ + //如果当前节点不是 VDATANODE 节点，且找到了该关系，则验证是否是一个临时表 if (g_instance.role != VDATANODE && OidIsValid(relid)) { verify_temp_table(relid, rel->schemaname, relation, true); } + //如果 OID 无效，说明找不到该关系，并且当前节点是 PGXC 协调器或单节点 if ((IS_PGXC_COORDINATOR || IS_SINGLE_NODE) && !OidIsValid(relid)) { /* return error */ heap_close(relation, RowExclusiveLock); - const char *schemaname = rel->catalogname ? - rel->catalogname : get_namespace_name(SchemaNameGetSchemaOid(NULL)); - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("[%s.%s] no such relation found", schemaname, rel->schemaname))); + + //获取要查询的关系的模式名和目录名，如果没有指定，则返回当前默认模式的名称 + const char *schemaname = rel->catalogname ? rel->catalogname : get_namespace_name(SchemaNameGetSchemaOid(NULL)); + + //抛出错误信息，提醒找不到指定的关系 + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("[%s.%s] no such relation found", schemaname, rel->schemaname))); } + + //如果 OID 有效，当前节点是 PGXC 协调器或单节点，并且找不到指定的列 if ((IS_PGXC_COORDINATOR || IS_SINGLE_NODE) && !verify_column_name(relid, rel->relname)) { /* return error */ heap_close(relation, RowExclusiveLock); - const char *schemaname = rel->catalogname ? - rel->catalogname : get_namespace_name(SchemaNameGetSchemaOid(NULL)); - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("[%s.%s.%s] no such relation column found", schemaname, rel->schemaname, rel->relname))); + + //获取要查询的关系的模式名和目录名，如果没有指定，则返回当前默认模式的名称 + const char *schemaname = rel->catalogname ? rel->catalogname : get_namespace_name(SchemaNameGetSchemaOid(NULL)); + + //抛出错误信息，提醒找不到指定的列 + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("[%s.%s.%s] no such relation column found", schemaname, rel->schemaname, rel->relname))); } } @@ -446,10 +577,12 @@ static void check_function_type_exists(RangeVar *rel, Relation relation) */ static void verify_label_resource(const char *label_item_type, RangeVar *rel, Relation relation) { + //如果 rel->relname 为 "all"，则不需要验证资源 if (!strcasecmp(rel->relname, "all")) { return; } + //获取标签类型，根据不同的类型调用相应的函数进行验证 int label_type = get_label_type(label_item_type); switch (label_type) { case T_VIEW: @@ -479,80 +612,102 @@ static void verify_label_resource(const char *label_item_type, RangeVar *rel, Re */ void verify_column_is_bound_to_label(policy_labels_map *existing_labels, GsPolicyFQDN *fqdn) { + //遍历现有标签映射表中的每一个标签 policy_labels_map::iterator lit = existing_labels->begin(); policy_labels_map::iterator elit = existing_labels->end(); for (; lit != elit; ++lit) { + //遍历当前标签对应的标签集合中的每一个标签 policy_labels_set::iterator sit = lit->second->begin(); policy_labels_set::iterator esit = lit->second->end(); for (; sit != esit; ++sit) { + //判断当前标签是否与给定的列关联 if (sit->m_data_value_fqdn.m_value_schema == fqdn->m_value_schema && sit->m_data_value_fqdn.m_value_object == fqdn->m_value_object && sit->m_data_value_fqdn.m_value_object_attrib == fqdn->m_value_object_attrib) { - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("Column %s already in other label", fqdn->m_value_object_attrib.c_str()))); + //如果关联了其他标签，则抛出错误信息 + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("Column %s already in other label", fqdn->m_value_object_attrib.c_str()))); return; } } } } + /* * insert_policy_tuple * * insert a label tuple into relation, if is_empty, just insert blank */ -static void insert_policy_tuple(CreatePolicyLabelStmt *stmt, Relation relation, const char *label_item_type, +static void insert_policy_tuple(CreatePolicyLabelStmt *stmt, Relation relation, const char *label_item_type, GsPolicyFQDN *fqdn, bool is_empty) { - const char *label_name = stmt->label_name; - const char *label_type = stmt->label_type; + const char *label_name = stmt->label_name; // 获取标签名称 + const char *label_type = stmt->label_type; // 获取标签类型 HeapTuple htup; - bool nulls[Natts_gs_policy_label] = {false}; - Datum values[Natts_gs_policy_label] = {0}; + bool nulls[Natts_gs_policy_label] = {false}; // 记录每个属性是否为空 + Datum values[Natts_gs_policy_label] = {0}; // 属性的值 - values[Anum_gs_policy_label_labelname - 1] = DirectFunctionCall1(namein, CStringGetDatum(label_name)); - values[Anum_gs_policy_label_labeltype - 1] = DirectFunctionCall1(namein, CStringGetDatum(label_type)); - values[Anum_gs_policy_label_fqdnnamespace - 1] = ObjectIdGetDatum(fqdn->m_value_schema); - values[Anum_gs_policy_label_fqdnid - 1] = ObjectIdGetDatum(fqdn->m_value_object); + // 设置属性的值 + values[Anum_gs_policy_label_labelname - 1] = + DirectFunctionCall1(namein, CStringGetDatum(label_name)); // 设置 labelname 属性 + values[Anum_gs_policy_label_labeltype - 1] = + DirectFunctionCall1(namein, CStringGetDatum(label_type)); // 设置 labeltype 属性 + values[Anum_gs_policy_label_fqdnnamespace - 1] = ObjectIdGetDatum(fqdn->m_value_schema); // 设置 fqdnnamespace 属性 + values[Anum_gs_policy_label_fqdnid - 1] = ObjectIdGetDatum(fqdn->m_value_object); // 设置 fqdnid 属性 if (is_empty) { - values[Anum_gs_policy_label_relcolumn - 1] = DirectFunctionCall1(namein, CStringGetDatum("")); + values[Anum_gs_policy_label_relcolumn - 1] = + DirectFunctionCall1(namein, CStringGetDatum("")); // 如果为空，设置 relcolumn 属性为空字符串 } else { - values[Anum_gs_policy_label_relcolumn - 1] = DirectFunctionCall1(namein, CStringGetDatum(fqdn->m_value_object_attrib.c_str())); + values[Anum_gs_policy_label_relcolumn - 1] = DirectFunctionCall1( + namein, CStringGetDatum(fqdn->m_value_object_attrib.c_str())); // 如果不为空，设置 relcolumn 属性为列的属性 } - values[Anum_gs_policy_label_fqdntype - 1] = DirectFunctionCall1(namein, CStringGetDatum(label_item_type)); + values[Anum_gs_policy_label_fqdntype - 1] = + DirectFunctionCall1(namein, CStringGetDatum(label_item_type)); // 设置 fqdntype 属性 - htup = heap_form_tuple(relation->rd_att, values, nulls); + htup = heap_form_tuple(relation->rd_att, values, nulls); // 根据属性的值和是否为空，创建一个 HeapTuple 结构体 /* Do the insertion */ - simple_heap_insert(relation, htup); - CatalogUpdateIndexes(relation, htup); - heap_freetuple(htup); + simple_heap_insert(relation, htup); // 将 HeapTuple 插入到 relation 中 + CatalogUpdateIndexes(relation, htup); // 更新 relation 的索引信息 + heap_freetuple(htup); // 释放 HeapTuple 占用的内存 } + /* * insert_stmt_label_item * * insert label items from CreatePolicyLabelStmt into relation, * if is a column without table, close relation and report error. */ -static void insert_stmt_label_item(CreatePolicyLabelStmt *stmt, Relation relation, policy_labels_map *existing_labels) +static void insert_stmt_label_item(CreatePolicyLabelStmt *stmt, Relation relation, policy_labels_map *existing_labels) { GsPolicyFQDN fqdn; + + // 如果标签项目为空，则插入一个空标签元组 if (stmt->label_items == NULL) { /* empty label */ insert_policy_tuple(stmt, relation, "", &fqdn, true); } - ListCell *label_item_itr = NULL; - ListCell *label_obj = NULL; + + ListCell *label_item_itr = NULL; // 标签项目迭代器 + ListCell *label_obj = NULL; // 标签对象迭代器 + + // 遍历标签项目列表 foreach (label_item_itr, stmt->label_items) { - DefElem *defel = (DefElem *) lfirst(label_item_itr); - const char *label_item_type = defel->defname; - List *label_item_objects = (List *) defel->arg; + DefElem *defel = (DefElem *)lfirst(label_item_itr); // 获取标签项目定义 + const char *label_item_type = defel->defname; // 获取标签项目类型 + List *label_item_objects = (List *)defel->arg; // 获取标签项目对象列表 + + // 如果当前节点是数据节点且标签项目类型为视图，则跳过 if (g_instance.role == VDATANODE && !strcasecmp(label_item_type, "view")) { continue; } + + // 遍历标签项目对象列表 foreach (label_obj, label_item_objects) { - RangeVar *rel = (RangeVar*)lfirst(label_obj); - verify_label_resource(label_item_type, rel, relation); + RangeVar *rel = (RangeVar *)lfirst(label_obj); // 获取标签对象 + verify_label_resource(label_item_type, rel, relation); // 验证标签资源是否有效 + + // 获取标签对象的全限定名 if (!fqdn_get_name(label_item_type, &fqdn, rel)) { if (IS_PGXC_COORDINATOR || IS_SINGLE_NODE) { heap_close(relation, RowExclusiveLock); @@ -560,15 +715,19 @@ static void insert_stmt_label_item(CreatePolicyLabelStmt *stmt, Relation relatio } continue; } - /* verify Column is unique in label catalog */ + + // 如果标签项目类型为列，则验证列是否绑定到其他标签 if (!strcasecmp(label_item_type, "column")) { verify_column_is_bound_to_label(existing_labels, &fqdn); } + + // 插入标签元组 insert_policy_tuple(stmt, relation, label_item_type, &fqdn, false); } } } + /* * create policy label * @@ -577,11 +736,9 @@ static void insert_stmt_label_item(CreatePolicyLabelStmt *stmt, Relation relatio */ void create_policy_label(CreatePolicyLabelStmt *stmt) { - /* check that if has access to config resource label */ + // 检查当前用户是否有访问资源标签的权限 if (!is_policy_enabled()) { - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("Permission denied."))); + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); return; } @@ -592,9 +749,11 @@ void create_policy_label(CreatePolicyLabelStmt *stmt) get_session_ip(session_ip, MAX_IP_LEN); (void)GetRoleName(GetCurrentUserId(), user_name, sizeof(user_name)); - errno_t rc = snprintf_s(buff, MAX_MSG_BUFF_SIZE, MAX_MSG_BUFF_SIZE - 1, - "user name: [%s], app_name: [%s], ip: [%s], CREATE RESOURCE LABEL [%s], TYPE: [%s]", - user_name, u_sess->attr.attr_common.application_name, session_ip, stmt->label_name, stmt->label_type); + // 记录审计日志 + errno_t rc = + snprintf_s(buff, MAX_MSG_BUFF_SIZE, MAX_MSG_BUFF_SIZE - 1, + "user name: [%s], app_name: [%s], ip: [%s], CREATE RESOURCE LABEL [%s], TYPE: [%s]", user_name, + u_sess->attr.attr_common.application_name, session_ip, stmt->label_name, stmt->label_type); securec_check_ss(rc, "\0", "\0"); save_manage_message(buff); @@ -602,16 +761,16 @@ void create_policy_label(CreatePolicyLabelStmt *stmt) Relation relation; policy_labels_map existing_labels; - /* Open the relation for read and insertion */ + // 打开策略标签关系对象，并加锁 relation = heap_open(GsPolicyLabelRelationId, RowExclusiveLock); - /* validate that there is no such label name */ + // 加载已存在的策略标签，验证标签名是否已存在 load_existing_labels(relation, &existing_labels); - policy_labels_map::iterator it = existing_labels.find(label_name); + if (it != existing_labels.end()) { + // 如果标签名已存在，根据if_not_exists参数设置决定是抛出错误还是跳过 heap_close(relation, RowExclusiveLock); - /* generate an error or notice */ if (stmt->if_not_exists == true) { send_manage_message(AUDIT_OK); ereport(NOTICE, (errmsg("%s label already defined, skipping", label_name))); @@ -621,17 +780,22 @@ void create_policy_label(CreatePolicyLabelStmt *stmt) } return; } + + // 插入标签元组 insert_stmt_label_item(stmt, relation, &existing_labels); heap_close(relation, RowExclusiveLock); + // 增加计数器，发送审计成功消息 CommandCounterIncrement(); send_manage_message(AUDIT_OK); + // 如果存在加载标签的钩子函数，则执行它 if (load_labels_hook != NULL) { load_labels_hook(false); } } + /* * fqdn_to_string * @@ -639,31 +803,39 @@ void create_policy_label(CreatePolicyLabelStmt *stmt) */ static void fqdn_to_string(const GsPolicyFQDN *fqdn, gs_stl::gs_string *res, bool is_function) { + // 如果fqdn的m_value_schema有效 if (OidIsValid(fqdn->m_value_schema)) { const char *schema = get_namespace_name(fqdn->m_value_schema); + // 如果schema不为空且长度大于0 if (schema != NULL && strlen(schema)) { - res->append(schema); + res->append(schema); // 将schema追加到结果字符串中 } else { - return; + return; // 否则返回 } } else { - return; + return; // 如果fqdn的m_value_schema无效，则返回 } + + // 如果fqdn的m_value_object有效 if (OidIsValid(fqdn->m_value_object)) { const char *object = is_function ? get_func_name(fqdn->m_value_object) : get_rel_name(fqdn->m_value_object); + // 如果object不为空且长度大于0 if (object != NULL && strlen(object)) { - res->push_back('.'); - res->append(object); + res->push_back('.'); // 在结果字符串中添加一个点号 + res->append(object); // 将object追加到结果字符串中 } } else { - return; + return; // 如果fqdn的m_value_object无效，则返回 } + + // 如果fqdn的m_value_object_attrib非空 if (!fqdn->m_value_object_attrib.empty()) { - res->append("."); - res->append(fqdn->m_value_object_attrib); + res->append("."); // 在结果字符串中添加一个点号 + res->append(fqdn->m_value_object_attrib); // 将m_value_object_attrib追加到结果字符串中 } } + /* * load_existing_labels * @@ -671,32 +843,46 @@ static void fqdn_to_string(const GsPolicyFQDN *fqdn, gs_stl::gs_string *res, boo */ void load_existing_labels(Relation rel, policy_labels_map *existing_labels) { + // 如果关系为空，则直接返回 if (rel == NULL) { return; } - HeapTuple rtup; - Form_gs_policy_label rel_data; + HeapTuple rtup; // 堆元组指针 + Form_gs_policy_label rel_data; // 关系的元数据 + + // 开始扫描关系 TableScanDesc scan = tableam_scan_begin(rel, SnapshotNow, 0, NULL); - while (scan && (rtup = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection))) { - rel_data = (Form_gs_policy_label)GETSTRUCT(rtup); + while (scan && (rtup = (HeapTuple)tableam_scan_getnexttuple(scan, ForwardScanDirection))) { + rel_data = (Form_gs_policy_label)GETSTRUCT(rtup); // 获取元数据 + + // 如果元数据为空，则继续下一次循环 if (rel_data == NULL) { continue; } - GsPolicyLabel item; - item.m_id = HeapTupleGetOid(rtup); - item.m_name = rel_data->labelname.data; - item.m_type = rel_data->labeltype.data; - item.m_data_value_fqdn.m_value_schema = rel_data->fqdnnamespace; - item.m_data_value_fqdn.m_value_object = rel_data->fqdnid; - item.m_data_value_fqdn.m_value_object_attrib = rel_data->relcolumn.data; - fqdn_to_string(&item.m_data_value_fqdn, &item.m_data_value, !strcasecmp(rel_data->fqdntype.data, "function")); - item.m_data_type = rel_data->fqdntype.data; - (*existing_labels)[item.m_name].insert(item); + + GsPolicyLabel item; // 新的标签对象 + item.m_id = HeapTupleGetOid(rtup); // 设置标签的id + item.m_name = rel_data->labelname.data; // 设置标签的名称 + item.m_type = rel_data->labeltype.data; // 设置标签的类型 + item.m_data_value_fqdn.m_value_schema = rel_data->fqdnnamespace; // 设置标签关联对象所在的命名空间 + item.m_data_value_fqdn.m_value_object = rel_data->fqdnid; // 设置标签关联的对象id + item.m_data_value_fqdn.m_value_object_attrib = rel_data->relcolumn.data; // 设置标签关联对象的属性 + fqdn_to_string(&item.m_data_value_fqdn, &item.m_data_value, + !strcasecmp(rel_data->fqdntype.data, + "function")); // 将标签关联对象的信息转换为字符串，并存储在item.m_data_value中 + item.m_data_type = rel_data->fqdntype.data; // 设置标签的关联对象类型 + (*existing_labels)[item.m_name].insert(item); // 将标签对象添加到标签名称对应的集合中 } - tableam_scan_end(scan); + + tableam_scan_end(scan); // 结束扫描 } +/* +S_UNKNOWN表示未知状态，可用作默认值或错误情况下的状态。 +S_REMOVE表示要移除或删除的状态。 +S_ADD表示要添加或新增的状态。 +*/ enum { S_UNKNOWN, S_REMOVE, @@ -705,40 +891,48 @@ enum { static inline int get_stmt_type(const char *stmt_type) { + // 比较stmt_type和字符串"remove"，忽略大小写 if (!strcasecmp(stmt_type, "remove")) { - return S_REMOVE; + return S_REMOVE; // 如果匹配，则返回S_REMOVE } + // 比较stmt_type和字符串"add"，忽略大小写 if (!strcasecmp(stmt_type, "add")) { - return S_ADD; + return S_ADD; // 如果匹配，则返回S_ADD } + // 如果stmt_type不匹配任何已知类型，则返回S_UNKNOWN return S_UNKNOWN; } + /* * add_policy_label * * add a GsPolicyLabel item to policy label table, if this item exists, just report an error. */ -static void add_policy_label(Relation relation, GsPolicyLabel *item, - policy_labels_map *existing_labels, bool is_exist) +static void add_policy_label(Relation relation, GsPolicyLabel *item, policy_labels_map *existing_labels, bool is_exist) { - /* if the label exists in relation, just return an error */ + /* 如果标记已经存在于pg_gs_policy_label表中，直接返回错误 */ if (is_exist) { heap_close(relation, RowExclusiveLock); - ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("Resource %s %s already exists in label %s", - item->m_data_type.c_str(), item->m_data_value.c_str(), item->m_name.c_str()))); + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("Resource %s %s already exists in label %s", item->m_data_type.c_str(), + item->m_data_value.c_str(), item->m_name.c_str()))); return; } + + /* 定义一个布尔类型的null数组和一个数据类型的数组values，并初始化为false和0 */ bool nulls[Natts_gs_policy_label] = {false}; Datum values[Natts_gs_policy_label] = {0}; + + /* 取出要添加的标记信息中所表示的列名 */ const char *column = item->m_data_value_fqdn.m_value_object_attrib.c_str(); - /* verify Column is unique in label catalog */ + /* 如果标记类型为"column"，则需要验证该列是否已经被绑定到其他标记上 */ if (!strcasecmp(item->m_data_type.c_str(), "column")) { verify_column_is_bound_to_label(existing_labels, &item->m_data_value_fqdn); } - /* generate label name from item */ + /* 根据标记的名称、类型、所属schema、所属表、所表示的列等信息生成标记名称，并将相关信息存储在values数组中 */ values[Anum_gs_policy_label_labelname - 1] = DirectFunctionCall1(namein, CStringGetDatum(item->m_name.c_str())); values[Anum_gs_policy_label_labeltype - 1] = DirectFunctionCall1(namein, CStringGetDatum(item->m_type.c_str())); values[Anum_gs_policy_label_fqdnnamespace - 1] = ObjectIdGetDatum(item->m_data_value_fqdn.m_value_schema); @@ -746,42 +940,57 @@ static void add_policy_label(Relation relation, GsPolicyLabel *item, values[Anum_gs_policy_label_relcolumn - 1] = DirectFunctionCall1(namein, CStringGetDatum(column)); values[Anum_gs_policy_label_fqdntype - 1] = DirectFunctionCall1(namein, CStringGetDatum(item->m_data_type.c_str())); - /* Insert Data correctly */ + /* 将存储标记信息的数组values转为元组htup */ HeapTuple htup = heap_form_tuple(relation->rd_att, values, nulls); - /* Do the insertion */ + /* 将新元组插入到pg_gs_policy_label表中 */ simple_heap_insert(relation, htup); + /* 更新索引 */ CatalogUpdateIndexes(relation, htup); + + /* 释放htup占用的内存 */ heap_freetuple(htup); } + /* * remove_policy_label * * delete a GsPolicyLabel item in policy label table, if this item doesn't exists, just report an error. */ -static void remove_policy_label(Relation relation, GsPolicyLabel *item, - policy_labels_map *existing_labels, bool is_exist) +static void remove_policy_label(Relation relation, GsPolicyLabel *item, policy_labels_map *existing_labels, + bool is_exist) { - /* if the label doesn't exist in relation, just return an error */ + /* 如果标记在pg_gs_policy_label表中不存在则返回错误 */ if (!is_exist) { heap_close(relation, RowExclusiveLock); - ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("No such resource %s %s found in label %s", - item->m_data_type.c_str(), item->m_data_value.c_str(), item->m_name.c_str()))); + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("No such resource %s %s found in label %s", item->m_data_type.c_str(), + item->m_data_value.c_str(), item->m_name.c_str()))); return; } + + /* 定义一个ScanKeyData结构体，指定扫描pk_gs_policy_label_oid索引所用的筛选条件 */ ScanKeyData skey; - /* Find the label row to delete. */ ScanKeyInit(&skey, ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(item->m_id)); + + /* 开始使用指定筛选条件对pg_gs_policy_label表进行扫描 */ SysScanDesc tgscan = systable_beginscan(relation, GsPolicyLabelOidIndexId, true, NULL, 1, &skey); + + /* 获取符合条件的元组 */ HeapTuple tup = systable_getnext(tgscan); + + /* 如果没有获取到符合条件的元组则报错 */ if (!HeapTupleIsValid(tup)) { - ereport(ERROR, (errcode(ERRCODE_TRIGGERED_INVALID_TUPLE), - errmsg("Could not find tuple for label %lld", item->m_id))); + ereport(ERROR, + (errcode(ERRCODE_TRIGGERED_INVALID_TUPLE), errmsg("Could not find tuple for label %lld", item->m_id))); } - /* Delete the label tuple */ + + /* 删除符合条件的元组 */ simple_heap_delete(relation, &tup->t_self); + + /* 完成条件扫描 */ systable_endscan(tgscan); } @@ -792,29 +1001,30 @@ static void remove_policy_label(Relation relation, GsPolicyLabel *item, * deal with statements about alter label, include add label and remove label. * input relation is GsPolicyLabelRelationId, and existing_labels include all labels in GsPolicyLabelRelationId. */ -static void alter_stmt_label_item(AlterPolicyLabelStmt *stmt, Relation relation, - policy_labels_map *existing_labels) +static void alter_stmt_label_item(AlterPolicyLabelStmt *stmt, Relation relation, policy_labels_map *existing_labels) { - const char *label_name = stmt->label_name; - gs_stl::gs_string label_type = "resource"; - policy_labels_map::iterator it = existing_labels->find(label_name); - int stmt_type = get_stmt_type(stmt->stmt_type); /* ADD or REMOVE */ - ListCell *label_item_itr = NULL; - ListCell *label_obj = NULL; + const char *label_name = stmt->label_name; // 获取标记名称 + gs_stl::gs_string label_type = "resource"; // 标记类型，默认为"resource" + policy_labels_map::iterator it = existing_labels->find(label_name); // 查找标记在existing_labels中的位置 + int stmt_type = get_stmt_type(stmt->stmt_type); // 获取语句类型，是ADD还是REMOVE + ListCell *label_item_itr = NULL; // 用于遍历标记项列表 + ListCell *label_obj = NULL; // 用于遍历标记项中的对象列表 foreach (label_item_itr, stmt->label_items) { - DefElem *defel = (DefElem *) lfirst(label_item_itr); - const char *label_item_type = defel->defname; - List *label_item_objects = (List *) defel->arg; + DefElem *defel = (DefElem *)lfirst(label_item_itr); // 获取标记项 + const char *label_item_type = defel->defname; // 获取标记项的类型 + List *label_item_objects = (List *)defel->arg; // 获取标记项的对象列表 if (g_instance.role == VDATANODE && !strcasecmp(label_item_type, "view")) { continue; } + foreach (label_obj, label_item_objects) { - RangeVar *rel = (RangeVar *)lfirst(label_obj); + RangeVar *rel = (RangeVar *)lfirst(label_obj); // 获取标记项的对象 GsPolicyLabel item; bool item_founded = false; - verify_label_resource(label_item_type, rel, relation); - /* validate that provided resource is legal */ + verify_label_resource(label_item_type, rel, relation); // 验证提供的资源是否合法 + + // 验证提供的资源是否合法，如果不合法则报错 if (!fqdn_get_name(label_item_type, &item.m_data_value_fqdn, rel)) { if (IS_PGXC_COORDINATOR || IS_SINGLE_NODE) { heap_close(relation, RowExclusiveLock); @@ -824,34 +1034,35 @@ static void alter_stmt_label_item(AlterPolicyLabelStmt *stmt, Relation relation, continue; } - /* fill in the attributes of item */ + /* 填充item的属性 */ item.m_name = label_name; item.m_type = label_type; item.m_data_type = label_item_type; fqdn_to_string(&item.m_data_value_fqdn, &item.m_data_value, !strcasecmp(label_item_type, "function")); - + /* - * Whether labelname exists has been verified before. - * So empty label means the label for view type in DN (nothing records). - * If label not found here, we just create a new label when adding a label item. - */ + * 标记是否存在已经在之前验证过。 + * 因此，空标记表示DN中视图类型的标记（无记录）。 + * 如果此处未找到标记，则在添加标记项时创建新标记。 + */ if (it != existing_labels->end()) { - policy_labels_set::iterator i_it = it->second->find(item); - item_founded = (i_it != it->second->end()); + policy_labels_set::iterator i_it = it->second->find(item); // 查找标记项在existing_labels中的位置 + item_founded = (i_it != it->second->end()); // 判断标记项是否已存在 if (item_founded) { - item.m_id = i_it->m_id; + item.m_id = i_it->m_id; // 获取标记项的ID } } - /* - * if item founded in existing_labels, we can only remove it, - * and if not found, we can only add it. + + /* + * 如果标记项在existing_labels中找到了，那么只能删除它， + * 如果没有找到，只能添加它。 */ switch (stmt_type) { case S_REMOVE: - remove_policy_label(relation, &item, existing_labels, item_founded); + remove_policy_label(relation, &item, existing_labels, item_founded); // 删除标记项 return; case S_ADD: - add_policy_label(relation, &item, existing_labels, item_founded); + add_policy_label(relation, &item, existing_labels, item_founded); // 添加标记项 break; default: break; @@ -860,6 +1071,7 @@ static void alter_stmt_label_item(AlterPolicyLabelStmt *stmt, Relation relation, } } + /* * alter policy label * @@ -868,11 +1080,9 @@ static void alter_stmt_label_item(AlterPolicyLabelStmt *stmt, Relation relation, */ void alter_policy_label(AlterPolicyLabelStmt *stmt) { - /* check that if has access to config resource label */ + /* 检查是否有权限访问配置资源标记 */ if (!is_policy_enabled()) { - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("Permission denied."))); + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); return; } @@ -885,32 +1095,31 @@ void alter_policy_label(AlterPolicyLabelStmt *stmt) char user_name[USERNAME_LEN] = {0}; char session_ip[MAX_IP_LEN] = {0}; - get_session_ip(session_ip, MAX_IP_LEN); - (void)GetRoleName(GetCurrentUserId(), user_name, sizeof(user_name)); + get_session_ip(session_ip, MAX_IP_LEN); // 获取当前会话的IP地址 + (void)GetRoleName(GetCurrentUserId(), user_name, sizeof(user_name)); // 获取当前用户的角色名称 rc = snprintf_s(buff, MAX_MSG_BUFF_SIZE, MAX_MSG_BUFF_SIZE - 1, - "user name: [%s], app_name: [%s], ip: [%s], ALTER RESOURCE LABEL [%s], FOR %s", - user_name, u_sess->attr.attr_common.application_name, session_ip, stmt->label_name, stmt->stmt_type); + "user name: [%s], app_name: [%s], ip: [%s], ALTER RESOURCE LABEL [%s], FOR %s", user_name, + u_sess->attr.attr_common.application_name, session_ip, stmt->label_name, stmt->stmt_type); securec_check_ss(rc, "\0", "\0"); - save_manage_message(buff); + save_manage_message(buff); // 保存管理消息日志 - /* Open the relation for read and insertion */ - relation = heap_open(GsPolicyLabelRelationId, RowExclusiveLock); + /* 打开关系对象以进行读取和插入操作 */ + relation = heap_open(GsPolicyLabelRelationId, RowExclusiveLock); // 打开关系对象 - load_existing_labels(relation, &existing_labels); + load_existing_labels(relation, &existing_labels); // 加载已存在的标记项 - /* first check whether such label exists */ + /* 首先检查是否存在这样的标记 */ policy_labels_map::iterator it = existing_labels.find(label_name); if (g_instance.role != VDATANODE && it == existing_labels.end()) { heap_close(relation, RowExclusiveLock); send_manage_message(AUDIT_FAILED); - /* generate an error */ - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("%s no such label found", label_name))); + /* 生成错误 */ + ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("%s no such label found", label_name))); // 标记项不存在的错误提示 return; } - alter_stmt_label_item(stmt, relation, &existing_labels); + alter_stmt_label_item(stmt, relation, &existing_labels); // 修改标记项 heap_close(relation, RowExclusiveLock); @@ -930,21 +1139,28 @@ void alter_policy_label(AlterPolicyLabelStmt *stmt) static void remove_label(const policy_labels_set *label_resources, Relation relation) { for (policy_labels_set::const_iterator it = label_resources->begin(); it != label_resources->end(); ++it) { + // 遍历标签资源集合 ScanKeyData skey; + // 定义扫描键 /* - * Find the label row to delete. + * 使用标签项的 ID 作为索引条件，查找要删除的标签行。 */ ScanKeyInit(&skey, ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(it->m_id)); + // 初始化扫描键 SysScanDesc tgscan = systable_beginscan(relation, GsPolicyLabelOidIndexId, true, NULL, 1, &skey); + // 开始系统扫描 HeapTuple tup = systable_getnext(tgscan); + // 获取下一个HeapTuple if (!HeapTupleIsValid(tup)) - ereport(ERROR, - (errcode(ERRCODE_TRIGGERED_INVALID_TUPLE), - errmsg("Could not find tuple for label %lld", it->m_id))); + // 如果获取到的HeapTuple无效，则抛出异常 + ereport(ERROR, (errcode(ERRCODE_TRIGGERED_INVALID_TUPLE), + errmsg("Could not find tuple for label %lld", it->m_id))); - /* Delete the label tuple */ + /* 删除标签元组 */ simple_heap_delete(relation, &tup->t_self); + // 删除标签元组 systable_endscan(tgscan); + // 结束系统扫描 } } @@ -955,90 +1171,110 @@ static void remove_label(const policy_labels_set *label_resources, Relation rela */ static bool check_labels_to_drop(const policy_labels_map *labels_to_drop) { - /* check label in accesses */ + /* 检查标签在访问权限中 */ if (check_audit_policy_privileges_for_label_hook != NULL) { + // 如果存在 check_audit_policy_privileges_for_label_hook 钩子函数 if (check_audit_policy_privileges_for_label_hook(labels_to_drop)) { - return false; - } - } - /* check label in privileges */ - if (check_audit_policy_access_for_label_hook != NULL) { - if (check_audit_policy_access_for_label_hook(labels_to_drop)) { - return false; - } - } - /* check label in masking actions */ - if (check_masking_policy_actions_for_label_hook != NULL) { - if (check_masking_policy_actions_for_label_hook(labels_to_drop)) { + // 调用钩子函数检查标签，在访问权限中，如果返回 true，则返回 false return false; } } + /* 检查标签在权限中 */ + if (check_audit_policy_access_for_label_hook != NULL) { + // 如果存在 check_audit_policy_access_for_label_hook 钩子函数 + if (check_audit_policy_access_for_label_hook(labels_to_drop)) { + // 调用钩子函数检查标签，在权限中，如果返回 true，则返回 false + return false; + } + } + + /* 检查标签在遮蔽操作中 */ + if (check_masking_policy_actions_for_label_hook != NULL) { + // 如果存在 check_masking_policy_actions_for_label_hook 钩子函数 + if (check_masking_policy_actions_for_label_hook(labels_to_drop)) { + // 调用钩子函数检查标签，在遮蔽操作中，如果返回 true，则返回 false + return false; + } + } + + // 如果以上所有检查都通过，则返回 true return true; } + /* * drop_policy_label * * go through all the labels to validate whether can drop it safely, then update the label buffer */ -void drop_policy_label(DropPolicyLabelStmt *stmt) +void drop_policy_label(DropPolicyLabelStmt stmt) { - /* check that if has access to config resource label */ - if (!is_policy_enabled()) { - ereport(ERROR, - (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), - errmsg("Permission denied."))); + / 检查是否有权限对策略进行操作 * / if (!is_policy_enabled()) + { + ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied."))); return; } - ListCell *label_obj = NULL; - policy_labels_map existing_labels; - policy_labels_map labels_to_drop; - Relation labels_relation; + ListCell *label_obj = NULL; // 声明列表元素指针 + policy_labels_map existing_labels; // 存储已存在标签信息的 map 容器 + policy_labels_map labels_to_drop; // 存储待删除标签信息的 map 容器 + Relation labels_relation; // 存储标签关系的 relation 对象 - foreach(label_obj, stmt->label_names) { - const char* label_name = (const char *)(((Value*)lfirst(label_obj))->val.str); + /* 遍历所有要删除的标签名 */ + foreach (label_obj, stmt->label_names) { + const char *label_name = (const char *)(((Value *)lfirst(label_obj))->val.str); - char buff[MAX_MSG_BUFF_SIZE] = {0}; - char user_name[USERNAME_LEN] = {0}; - char session_ip[MAX_IP_LEN] = {0}; + char buff[MAX_MSG_BUFF_SIZE] = {0}; // 初始化字符数组 + char user_name[USERNAME_LEN] = {0}; // 初始化用户名数组 + char session_ip[MAX_IP_LEN] = {0}; // 初始化会话 IP 地址数组 + /* 获取当前会话 IP 地址 */ get_session_ip(session_ip, MAX_IP_LEN); + /* 获取当前用户的用户名 */ (void)GetRoleName(GetCurrentUserId(), user_name, sizeof(user_name)); + /* 使用 snprintf_s 函数将消息存入 buff 数组中 */ errno_t rc = snprintf_s(buff, MAX_MSG_BUFF_SIZE, MAX_MSG_BUFF_SIZE - 1, - "user name: [%s], app_name: [%s], ip: [%s], DROP RESOURCE LABEL [%s]", - user_name, u_sess->attr.attr_common.application_name, session_ip, label_name); + "user name: [%s], app_name: [%s], ip: [%s], DROP RESOURCE LABEL [%s]", user_name, + u_sess->attr.attr_common.application_name, session_ip, label_name); securec_check_ss(rc, "\0", "\0"); + /* 将操作记录保存到 accessaudit 表 */ save_manage_message(buff); } - /* Open the relation for read and insertion */ + /* 打开标签关系 relation 对象 */ labels_relation = heap_open(GsPolicyLabelRelationId, RowExclusiveLock); + /* 读取已存在的标签信息到 existing_labels 容器中 */ load_existing_labels(labels_relation, &existing_labels); + /* 遍历所有要删除的标签名 */ foreach (label_obj, stmt->label_names) { - const char *label_name = (const char *)(((Value*)lfirst(label_obj))->val.str); - /* first check whether such label exists */ + const char *label_name = (const char *)(((Value *)lfirst(label_obj))->val.str); + /* 先检查该标签是否存在于 existing_labels 容器中 */ policy_labels_map::iterator it = existing_labels.find(label_name); - if (it == existing_labels.end()) { + if (it == existing_labels.end()) { // 若不存在 + /* 若当前节点为数据节点，则直接跳过 */ if (g_instance.role == VDATANODE) { continue; } + /* 若不存在并且设置了 if_exists 参数为 false，报错并返回 */ if (stmt->if_exists == false) { heap_close(labels_relation, RowExclusiveLock); send_manage_message(AUDIT_FAILED); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("%s no such label found", label_name))); return; } + /* 若不存在并且设置了 if_exists 参数为 true，则发出提示信息 */ ereport(NOTICE, (errmsg("%s no such label found, skipping", label_name))); - } else { - /* add this label to list of labels to be removed */ + } else { // 若存在 + /* 将待删除标签信息添加到 labels_to_drop 容器中 */ labels_to_drop[*(it->first)] = *(it->second); } } + + /* 若没有标签信息需要删除，则直接返回 */ if (labels_to_drop.size() == 0) { heap_close(labels_relation, RowExclusiveLock); if (IS_PGXC_COORDINATOR || IS_SINGLE_NODE) { @@ -1047,61 +1283,78 @@ void drop_policy_label(DropPolicyLabelStmt *stmt) } return; } - /* validate that this label is not bound to any policy */ + + /* 检查待删除的标签是否与任何策略绑定，若有则报错并返回 */ if (!check_labels_to_drop(&labels_to_drop)) { heap_close(labels_relation, RowExclusiveLock); send_manage_message(AUDIT_FAILED); - ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("One or more labels bound to policies - cannot drop"))); + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("One or more labels bound to policies - cannot drop"))); return; } + /* 删除待删除的标签信息 */ for (policy_labels_map::iterator it = labels_to_drop.begin(); it != labels_to_drop.end(); ++it) { remove_label(it->second, labels_relation); } heap_close(labels_relation, RowExclusiveLock); - CommandCounterIncrement(); - send_manage_message(AUDIT_OK); + send_manage_message(AUDIT_OK); // 记录操作记录到 accessaudit 表 + /* 若设置了回调函数 load_labels_hook 则触发 */ if (load_labels_hook != NULL) { load_labels_hook(false); } } + /* * check that policy plugin/feature is enabled, only used policy configurating by client * return error if policy plugin is off directly */ bool is_policy_enabled() { - if (superuser() || isPolicyadmin(GetUserId())) { - return true; + // 检查是否是超级用户（具有所有权限） + if (superuser()) { + return true; // 如果是超级用户，则策略启用 } - return false; + + // 检查当前用户是否是策略管理员 + if (isPolicyadmin(GetUserId())) { + return true; // 如果是策略管理员，则策略启用 + } + + return false; // 如果都不是，则策略禁用 } + bool is_security_policy_relation(Oid relid) { + // 检查关系id是否有效 if (!OidIsValid(relid)) { - return false; + return false; // 无效的关系id，返回false } + // 获取关系名称 char *relname = get_rel_name(relid); if (relname == NULL) { - return false; + return false; // 获取不到关系名称，返回false } - const char *namelist[POLICY_VIEWS_NUM] = {"gs_labels", "gs_masking", - "gs_auditing", "gs_auditing_access", "gs_auditing_privilege"}; + // 定义安全策略视图的名称列表 + const char *namelist[POLICY_VIEWS_NUM] = {"gs_labels", "gs_masking", "gs_auditing", "gs_auditing_access", + "gs_auditing_privilege"}; + + // 遍历安全策略视图的名称列表 for (int i = 0; i < POLICY_VIEWS_NUM; ++i) { + // 如果关系名称与安全策略视图名称相等，则返回true if (strcmp(relname, namelist[i]) == 0) { return true; } } - /* check relation id is security policy catalog */ + // 检查关系id是否是安全策略目录 switch (relid) { case GsAuditingPolicyRelationId: case GsAuditingPolicyAccessRelationId: @@ -1111,16 +1364,17 @@ bool is_security_policy_relation(Oid relid) case GsMaskingPolicyActionsId: case GsMaskingPolicyFiltersId: case GsPolicyLabelRelationId: - return true; + return true; // 是安全策略目录，返回true default: break; } - return false; + return false; // 不是安全策略目录，返回false } + #ifndef WIN32 -bool get_files_list(const char *dir, std::vector& files, const char *ext, int queue_size) +bool get_files_list(const char *dir, std::vector &files, const char *ext, int queue_size) { struct dirent **namelist = NULL; int n; @@ -1129,12 +1383,16 @@ bool get_files_list(const char *dir, std::vector& files, const char return false; } int ext_len = strlen(ext); - bool next_file = true; + bool next_file = true; while (n-- && next_file) { + // 获取文件名的长度 int filelen = strlen(namelist[n]->d_name); + // 检查文件名的后缀是否与指定的后缀相同 if (!strncasecmp(namelist[n]->d_name + filelen - ext_len, ext, ext_len)) { + // 将符合条件的文件名添加到files向量中 files.emplace_back(namelist[n]->d_name); if (queue_size > 0 && queue_size < (int)files.size()) { + // 如果文件数量超过了队列大小，则停止继续添加文件 next_file = false; } } @@ -1144,7 +1402,7 @@ bool get_files_list(const char *dir, std::vector& files, const char return files.size() > 0; } #else -bool get_files_list(const char *dir,std::vector& files, const char *ext, int queue_size) +bool get_files_list(const char *dir, std::vector &files, const char *ext, int queue_size) { string path; string pattern; @@ -1156,26 +1414,31 @@ bool get_files_list(const char *dir,std::vector& files, const char filedir = path; pos = filedir.rfind('/'); if (pos != string::npos) { + // 提取出文件名的pattern pattern = path.substr(pos + 1); + // 提取出文件目录 filedir = filedir.erase(pos); } if (pattern.size() == 0) gs_ereport(LEVEL_ERROR, "Invalid audit policy dir: \"%s\"", dir); std::replace(filedir.begin(), filedir.end(), '/', '\\'); int ext_len = strlen(ext); + // 使用FindFirstFile函数来查找文件 hFind = FindFirstFile(path.c_str(), &fd); if (hFind != INVALID_HANDLE_VALUE) { do { /* directory */ + // 如果是目录，则跳过 if ((fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == FILE_ATTRIBUTE_DIRECTORY) { continue; } int filelen = strlen(fd.cFileName); + // 检查文件名的后缀是否与指定的后缀相同 if (!strncasecmp(fd.cFileName + filelen - ext_len, ext, ext_len)) + // 将符合条件的文件名添加到files向量中 files.push_back(fd.cFileName); - }while (FindNextFile(hFind, &fd)); + } while (FindNextFile(hFind, &fd)); FindClose(hFind); } } #endif - -- 2.34.1 From cefa81d8fee99f10fc438ea806e3f27ad4a12744 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Wed, 4 Oct 2023 05:17:49 +0800 Subject: [PATCH 36/48] Update cache_mgr.cpp --- src/gausskernel/storage/cmgr/cache_mgr.cpp | 739 ++++++++++++++++----- 1 file changed, 573 insertions(+), 166 deletions(-) diff --git a/src/gausskernel/storage/cmgr/cache_mgr.cpp b/src/gausskernel/storage/cmgr/cache_mgr.cpp index 048232838..c06536efc 100644 --- a/src/gausskernel/storage/cmgr/cache_mgr.cpp +++ b/src/gausskernel/storage/cmgr/cache_mgr.cpp @@ -46,7 +46,7 @@ typedef struct CacheLookupEnt { // 2097152 = 2 * 1024 * 1024 = 2M #define MAX_CACHE_SLOT_COUNT 2097152 - +//检查缓存槽的状态，并在一些条件下触发相应的错误处理或返回缓存块的无效索引 #define CHECK_CACHE_SLOT_STATUS() \ { \ if (m_csweep == start) { \ @@ -78,6 +78,7 @@ typedef struct CacheLookupEnt { * @Return: lock number * @See also: */ +//获取CacheMgr实例所需的LW锁数量 int CacheMgrNumLocks(int64 cache_size, uint32 each_block_size) { /* One LW Lock for each cache block include IO lock and compress lock */ @@ -91,16 +92,17 @@ int CacheMgrNumLocks(int64 cache_size, uint32 each_block_size) * @Return: cache size * @See also: */ +//根据缓存类型计算缓存大小 int64 CacheMgrCalcSizeByType(MgrCacheType type) { - int64 size = g_instance.attr.attr_storage.cstore_buffers * 1024LL * 1 / 4; - int64 cache_size = Min(size, MAX_METADATA_CACHE_SIZE); - if (!g_instance.attr.attr_sql.enable_orc_cache) { + int64 size = g_instance.attr.attr_storage.cstore_buffers * 1024LL * 1 / 4;// 计算缓存大小为cstore_buffers的1/4 + int64 cache_size = Min(size, MAX_METADATA_CACHE_SIZE);// 设置缓存大小为计算大小和最大元数据缓存大小之间的较小值 + if (!g_instance.attr.attr_sql.enable_orc_cache) {// 若不启用ORC缓存，则将缓存大小设置为1MB cache_size = 1024 * 1024; } - if (type == MGR_CACHE_TYPE_DATA) { - cache_size = g_instance.attr.attr_storage.cstore_buffers * 1024LL - cache_size; + if (type == MGR_CACHE_TYPE_DATA) {// 如果缓存类型是数据缓存 + cache_size = g_instance.attr.attr_storage.cstore_buffers * 1024LL - cache_size;// 数据缓存大小为cstore_buffers减去元数据缓存大小 } return cache_size; } @@ -113,6 +115,11 @@ int64 CacheMgrCalcSizeByType(MgrCacheType type) * @IN each_slot_size: slot struct size, ep:sizeof(CU) * @See also: */ +// 初始化cache实例的资源 +// cache_size：总的cache大小 +// each_block_size：每个cache块的大小，用于划分cache槽位 +// type：cache类型 +// each_slot_length：每个槽位的结构体大小，例如sizeof(CU) void CacheMgr::Init(int64 cache_size, uint32 each_block_size, MgrCacheType type, uint32 each_slot_length) { int i = 0; @@ -120,17 +127,20 @@ void CacheMgr::Init(int64 cache_size, uint32 each_block_size, MgrCacheType type, int32 total_slots = 0; m_cache_type = type; + // 一定要大于0 /* Must be greater than 0 */ m_CaccheSlotMax = 1; m_cstoreCurrentSize = 0; m_cstoreMaxSize = cache_size; - + // 计算需要多少个槽位 total_slots = Min(cache_size / each_block_size, MAX_CACHE_SLOT_COUNT); + // 分配槽位和描述符 m_CacheSlots = (char *)palloc0(total_slots * each_slot_length); m_CacheDesc = (CacheDesc *)palloc0(total_slots * sizeof(CacheDesc)); m_CacheSlotsNum = total_slots; m_slot_length = each_slot_length; + // 初始化每个槽位的描述符信息 for (i = 0; i < total_slots; ++i) { m_CacheDesc[i].m_usage_count = 0; m_CacheDesc[i].m_refcount = 0; @@ -139,11 +149,13 @@ void CacheMgr::Init(int64 cache_size, uint32 each_block_size, MgrCacheType type, m_CacheDesc[i].m_freeNext = i + 1; m_CacheDesc[i].m_cache_tag.type = CACHE_TYPE_NONE; m_CacheDesc[i].m_flag = CACHE_BLOCK_FREE; + // 根据cache类型设置锁类型 if (type == MGR_CACHE_TYPE_DATA) { trancheId = (int)LWTRANCHE_DATA_CACHE; } else if (type == MGR_CACHE_TYPE_INDEX) { trancheId = (int)LWTRANCHE_META_CACHE; } + // 初始化锁、刷新标志等描述符信息 m_CacheDesc[i].m_iobusy_lock = LWLockAssign(trancheId); m_CacheDesc[i].m_compress_lock = LWLockAssign(trancheId); m_CacheDesc[i].m_refreshing = false; @@ -153,6 +165,7 @@ void CacheMgr::Init(int64 cache_size, uint32 each_block_size, MgrCacheType type, } /* Cache Slot Free List */ + // Cache Slot Free List为空，就将最后一个槽位的next_free设为无效索引 m_CacheDesc[total_slots - 1].m_freeNext = CACHE_BLOCK_INVALID_IDX; m_freeListHead = 0; m_freeListTail = total_slots - 1; @@ -160,6 +173,7 @@ void CacheMgr::Init(int64 cache_size, uint32 each_block_size, MgrCacheType type, SpinLockInit(&m_memsize_lock); /* Clock Sweep Starting point */ + // 初始化Clock Sweep信息，用于回收缓存数据 m_csweep = 0; m_csweep_lock = CStoreCUCacheSweepLock; m_partition_lock = FirstCacheSlotMappingLock; @@ -167,7 +181,7 @@ void CacheMgr::Init(int64 cache_size, uint32 each_block_size, MgrCacheType type, m_csweep_lock = MetaCacheSweepLock; m_partition_lock = FirstCacheSlotMappingLock + NUM_CACHE_BUFFER_PARTITIONS / 2; } - + // 初始化cache查询表，建立哈希表 HASHCTL info; errno_t rc = memset_s(&info, sizeof(info), 0, sizeof(info)); securec_check(rc, "\0", "\0"); @@ -177,11 +191,12 @@ void CacheMgr::Init(int64 cache_size, uint32 each_block_size, MgrCacheType type, securec_check_ss(rc, "\0", "\0"); /* BufferTag maps to Buffer */ + // 缓存标记（CacheTag）映射到缓存描述符，用于查找 info.keysize = sizeof(CacheTag); info.entrysize = sizeof(CacheLookupEnt); info.hash = tag_hash; info.num_partitions = NUM_CACHE_BUFFER_PARTITIONS / 2; - + // 初始化哈希表 m_hash = HeapMemInitHash(hash_name, total_slots, total_slots, &info, HASH_ELEM | HASH_FUNCTION | HASH_PARTITION); } @@ -190,21 +205,25 @@ void CacheMgr::Init(int64 cache_size, uint32 each_block_size, MgrCacheType type, * excluding this instance itself. * @See also: */ +//销毁缓存实例，释放所有相关资源 void CacheMgr::Destroy(void) { /* reset and clear up hash table */ + // 重置和清理哈希表 HeapMemResetHash(m_hash, "Cache Buffer Lookup Table"); for (int i = 0; i < m_CacheSlotsNum; ++i) { + // 释放属于此槽位的所有内存 /* free all memory belonged to this slot */ FreeCacheBlockMem(i); SpinLockFree(&m_CacheDesc[i].m_slot_hdr_lock); } /* free spin lock resource */ + // 释放自旋锁资源 SpinLockFree(&m_memsize_lock); SpinLockFree(&m_freeList_lock); - + // 释放内存 pfree_ext(m_CacheSlots); pfree_ext(m_CacheDesc); } @@ -217,14 +236,16 @@ void CacheMgr::Destroy(void) * @IN type: type of key (data or meta) * @See also: */ +// 初始化缓存标记 void CacheMgr::InitCacheBlockTag(CacheTag *cacheTag, int32 type, const void *key, int32 length) const { + // 断言缓存类型和标记类型一致性 Assert(((m_cache_type == MGR_CACHE_TYPE_DATA) && (type == CACHE_COlUMN_DATA || type == CACHE_ORC_DATA || type == CACHE_OBS_DATA)) || ((m_cache_type == MGR_CACHE_TYPE_INDEX) && (type == CACHE_ORC_INDEX || type == CACHE_CARBONDATA_METADATA))); - + // 断言缓存标记长度不超过最大长度 Assert(length <= MAX_CACHE_TAG_LEN); - + // 初始化缓存标记类型和键值 cacheTag->type = type; errno_t rc = memcpy_s(cacheTag->key, MAX_CACHE_TAG_LEN, key, length); securec_check(rc, "\0", "\0"); @@ -238,49 +259,64 @@ void CacheMgr::InitCacheBlockTag(CacheTag *cacheTag, int32 type, const void *key * @Return: the block desc and pinned if found, null not found * @See also: */ +/* + * @Description: 在哈希表中查找缓存块 + * @IN cacheTag: 块的唯一标识 + * @IN first_enter_block: 标志是否需要增加使用计数，当块第一次使用时，可能需要增加使用计数 + * @Return: 如果找到缓存块，则返回块描述和固定标记；如果未找到，则返回null + * @See also: + */ +//在缓存管理器中查找缓存块 CacheSlotId_t CacheMgr::FindCacheBlock(CacheTag *cacheTag, bool first_enter_block) { - CacheLookupEnt *result = NULL; - CacheSlotId_t slotId = CACHE_BLOCK_INVALID_IDX; - uint32 hashCode = 0; + // 初始化变量 + CacheLookupEnt *result = NULL;// 初始化变量 + CacheSlotId_t slotId = CACHE_BLOCK_INVALID_IDX;// 缓存块的槽位编号 + uint32 hashCode = 0;// 缓存块标识的哈希值 /* invalid tag */ + /* 无效的标记 */ + // 断言缓存标记类型的有效性 Assert(cacheTag->type > CACHE_TYPE_NONE && cacheTag->type <= CACHE_CARBONDATA_METADATA); - - hashCode = GetHashCode(cacheTag); - (void)LockHashPartion(hashCode, LW_SHARED); + // 计算哈希码并锁定哈希分区 + hashCode = GetHashCode(cacheTag);// 获取哈希码 + (void)LockHashPartion(hashCode, LW_SHARED);// 锁定哈希分区 + // 在哈希表中根据哈希值进行查找 result = (CacheLookupEnt *)hash_search_with_hash_value(m_hash, (void *)cacheTag, hashCode, HASH_FIND, NULL); + // 固定缓存块，并检查是否有效 if (result != NULL) { - bool valid = PinCacheBlock(result->slot_id); - if (!valid) { + bool valid = PinCacheBlock(result->slot_id);// 缓存块无效 + if (!valid) {// 解锁哈希分区 UnLockHashPartion(hashCode); return slotId; } - slotId = result->slot_id; + slotId = result->slot_id; // 缓存块的槽位编号 + // 断言缓存类型和标记类型的一致性 Assert(((m_cache_type == MGR_CACHE_TYPE_DATA) && (m_CacheDesc[slotId].m_cache_tag.type == CACHE_COlUMN_DATA || m_CacheDesc[slotId].m_cache_tag.type == CACHE_ORC_DATA || m_CacheDesc[slotId].m_cache_tag.type == CACHE_OBS_DATA)) || ((m_cache_type == MGR_CACHE_TYPE_INDEX) && (m_CacheDesc[slotId].m_cache_tag.type == CACHE_ORC_INDEX || m_CacheDesc[slotId].m_cache_tag.type == CACHE_CARBONDATA_METADATA))); - + // 锁定缓存块描述头，并根据需要增加使用计数 LockCacheDescHeader(slotId); if (first_enter_block && (m_CacheDesc[slotId].m_usage_count < CACHE_BLOCK_MAX_USAGE)) { m_CacheDesc[slotId].m_usage_count += 1; } UnLockCacheDescHeader(slotId); - + // 断言缓存块的槽位编号和相关信息是合法的 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); CacheDesc *cacheDesc = m_CacheDesc + slotId; Assert(cacheDesc->m_slot_id == slotId); Assert(cacheTag->type == cacheDesc->m_cache_tag.type); Assert(memcmp(cacheTag->key, cacheDesc->m_cache_tag.key, MAX_CACHE_TAG_LEN) == 0); + // 输出日志，表示在缓存中找到了块 ereport(DEBUG1, (errmodule(MOD_CACHE), errmsg("Find block in cache, slot(%d), type(%d)", cacheDesc->m_slot_id, cacheTag->type))); } - UnLockHashPartion(hashCode); - return slotId; + UnLockHashPartion(hashCode);// 解锁哈希分区 + return slotId;// 返回缓存块的槽位编号 } /* @@ -288,44 +324,47 @@ CacheSlotId_t CacheMgr::FindCacheBlock(CacheTag *cacheTag, bool first_enter_bloc * @IN cacheTag: block unique identification * @See also: */ +//从缓存管理器中删除指定的缓存块 void CacheMgr::InvalidateCacheBlock(CacheTag *cacheTag) { - int retryTimes = 0; - CacheDesc *slotHdr = NULL; - int slotId = CACHE_BLOCK_INVALID_IDX; - int blockSize = 0; + int retryTimes = 0;// 重试次数 + CacheDesc *slotHdr = NULL;// 缓存块描述头指针 + int slotId = CACHE_BLOCK_INVALID_IDX;// 缓存槽位ID + int blockSize = 0;// 缓存块大小 - Assert(t_thrd.int_cxt.ImmediateInterruptOK == false); + Assert(t_thrd.int_cxt.ImmediateInterruptOK == false);// 断言中断状态为不可立即中断 while (1) { /* search and pin the found cache block */ - slotId = FindCacheBlock(cacheTag, false); + slotId = FindCacheBlock(cacheTag, false);// 在哈希表中查找缓存块的槽位ID（不锁定） if (!IsValidCacheSlotID(slotId)) - return; + return;// 如果槽位ID无效，则直接返回 /* the other backends now is using it, so wait. */ Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); - slotHdr = m_CacheDesc + slotId; + slotHdr = m_CacheDesc + slotId;// 获取缓存块描述头指针 if (slotHdr->m_refcount != 1) { - UnPinCacheBlock(slotId); + UnPinCacheBlock(slotId);// 取消固定该缓存块 /* it means this backend has to wait continuely. so that try some times and then make myself sleeping, keep CPU away from wasting as possible. */ - if (++retryTimes >= 64) { + if (++retryTimes >= 64) {// 如果重试次数达到上限，则休眠一段时间 retryTimes = 0; pg_usleep(2000L); } - continue; + continue;// 继续下一轮循环 } Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); /* delete from hash table */ + // 从哈希表中删除缓存块 DeleteCacheBlock(cacheTag); ereport(DEBUG1, (errmodule(MOD_CACHE), errmsg("drop cache block, solt(%d), flag(%d - %d)", slotId, m_CacheDesc[slotId].m_flag, CACHE_BLOCK_FREE))); /* update slot flag */ + // 更新槽位标记 LockCacheDescHeader(slotId); blockSize = m_CacheDesc[slotId].m_datablock_size; m_CacheDesc[slotId].m_flag = CACHE_BLOCK_FREE; @@ -333,13 +372,13 @@ void CacheMgr::InvalidateCacheBlock(CacheTag *cacheTag) UnLockCacheDescHeader(slotId); /* free this block cache and update its size before unpin this slot id */ - FreeCacheBlockMem(slotId); - ReleaseCacheMem(blockSize); - UnPinCacheBlock(slotId); + FreeCacheBlockMem(slotId);// 释放缓存块的内存 + ReleaseCacheMem(blockSize);// 更新缓存内存大小 + UnPinCacheBlock(slotId);// 解除对该缓存块的固定 /* put it into free list */ - PutFreeListCache(slotId); - break; + PutFreeListCache(slotId);// 将该缓存块放入空闲块列表 + break;// 退出循环 } } @@ -348,15 +387,25 @@ void CacheMgr::InvalidateCacheBlock(CacheTag *cacheTag) * @IN cacheTag: block unique identification * @See also: */ +/* + * @Description: 删除哈希表中的缓存块 + * @IN cacheTag: 缓存块的唯一标识 + * @See also: + */ +//从哈希表中删除指定的缓存块 void CacheMgr::DeleteCacheBlock(CacheTag *cacheTag) { /* invalid tag */ Assert(cacheTag->type > CACHE_TYPE_NONE && cacheTag->type <= CACHE_CARBONDATA_METADATA); - + // 断言缓存块标识类型的有效性 uint32 hashCode = GetHashCode(cacheTag); + // 计算缓存块标识的哈希值 (void)LockHashPartion(hashCode, LW_EXCLUSIVE); + // 锁定哈希表分区，以互斥方式 (void)hash_search_with_hash_value(m_hash, (void *)cacheTag, hashCode, HASH_REMOVE, NULL); + // 在哈希表中根据哈希值和缓存块标识查找并删除该缓存块 UnLockHashPartion(hashCode); + // 解锁哈希表分区 return; } @@ -366,13 +415,16 @@ void CacheMgr::DeleteCacheBlock(CacheTag *cacheTag) * @Return: slot id * @See also: */ +//使用时钟替换算法从缓存池中选择一个缓存块进行驱逐。通常在缓存空间不足时被调用，以便给新的缓存块腾出空间。 CacheSlotId_t CacheMgr::EvictCacheBlock(int size, int retryNum) { + // 初始化变量 CacheSlotId_t slotId = CACHE_BLOCK_INVALID_IDX; /* If there is insufficient memory or no slots available * we must evict one of the cache blocks using the clock sweep. * Get the m_csweep_lock, only one sweeper searches the list. */ + // 获取扫描器锁，以执行驱逐过程 LockSweep(); /* Initialize statistics for each new sweep */ @@ -390,7 +442,9 @@ CacheSlotId_t CacheMgr::EvictCacheBlock(int size, int retryNum) while (1) { /* Set the start slot to the current sweep position(m_csweep), Then advance the current sweep position to the next */ + // 将当前扫描位置（m_csweep）设置为起始槽位，然后将当前扫描位置前移至下一个槽位 slotId = m_csweep; + // 如果缓存块当前正在用于I/O操作，则跳过该缓存块 if (m_csweep++ >= m_CaccheSlotMax) { m_csweep = 0; } @@ -406,26 +460,30 @@ CacheSlotId_t CacheMgr::EvictCacheBlock(int size, int retryNum) pg_usleep(2); continue; } - + // 锁定此槽位上的缓存块描述头 LockCacheDescHeader(slotId); scanned++; /* skip invalid and error cache blocks */ if ((m_CacheDesc[slotId].m_flag & CACHE_BLOCK_VALID) || (m_CacheDesc[slotId].m_flag & CACHE_BLOCK_ERROR)) { /* skip pinned cache blocks */ + // 如果缓存块被锁定，则跳过该缓存块 if (m_CacheDesc[slotId].m_refcount == 0) { unpinned++; /* skip cache blocks with usage count > 0 */ + // 如果缓存块的使用计数大于0，则跳过该缓存块 if (m_CacheDesc[slotId].m_usage_count == 0) { /* skip cache blocks that are in another ring , 1 in my ring, 0 no ring */ + // 如果缓存块位于不同的优先级队列中，则跳过该缓存块 if (m_CacheDesc[slotId].m_ring_count == 0) { ereport(DEBUG2, (errmodule(MOD_CACHE), errmsg("evict cache block, solt(%d), flag(%d - %d)", slotId, m_CacheDesc[slotId].m_flag, CACHE_BLOCK_INFREE))); - + // 将缓存块标记为无效 m_CacheDesc[slotId].m_flag = CACHE_BLOCK_INFREE; // !Valid + // 释放缓存块描述头的锁 PinCacheBlock_Locked(slotId); // Released header lock found++; - + // 找到一个可重用的槽位，需要释放并重新使用其缓存空间 /* Found a slot to be reused with some buffer space, * the space must be freed and reused. * The slot is Invalid and Pinned!!! */ @@ -433,6 +491,7 @@ CacheSlotId_t CacheMgr::EvictCacheBlock(int size, int retryNum) } reserved++; } else { + // 减少使用计数以更新缓存块的访问时间 /* decrement the usage count to age the entry */ m_CacheDesc[slotId].m_usage_count--; } @@ -444,7 +503,7 @@ CacheSlotId_t CacheMgr::EvictCacheBlock(int size, int retryNum) if ((m_CacheDesc[slotId].m_flag & CACHE_BLOCK_FREE) && (m_CacheDesc[slotId].m_refcount > 0)) { freepinned++; } - } + }// 解锁缓存块描述头 UnLockCacheDescHeader(slotId); /* A full sweep should rarely happen @@ -460,11 +519,15 @@ CacheSlotId_t CacheMgr::EvictCacheBlock(int size, int retryNum) * returns CACHE_BLOCK_INVALID_IDX, it will retry to find freespace in GetFreeCacheBlock * function. */ + // 如果没有可用的未锁定缓存块，则需要进行驱逐。 + // 如果扫描位置回到起始位置，则表示已对缓存池进行了完整扫描。 CHECK_CACHE_SLOT_STATUS(); } + // 释放扫描器锁 UnlockSweep(); // purposely returning pinned Invalid slot !!! + // 返回被驱逐的缓存块的槽位ID，如果没有找到则返回CACHE_BLOCK_INVALID_IDX return slotId; } @@ -475,6 +538,7 @@ CacheSlotId_t CacheMgr::EvictCacheBlock(int size, int retryNum) * @Return: return valid block index with pinned or error return * @See also: */ +//根据指定的缓存内存大小选择合适的缓存块,从缓存池中获取 CacheSlotId_t CacheMgr::GetFreeCacheBlock(int size) { CacheSlotId_t slotId = CACHE_BLOCK_INVALID_IDX; @@ -484,8 +548,10 @@ RETRY_FIND_FREESPACE: retryNum++; /* If there is memory available, and slots on the free list, just return one from there */ + // 如果有可用的内存，并且空闲链表中有槽位，则直接从空闲链表中获取一个空闲槽位 if (ReserveCacheMem(size)) { if ((slotId = GetFreeListCache()) != CACHE_BLOCK_INVALID_IDX) { + // 更新最大槽位编号 LockSweep(); if (slotId > m_CaccheSlotMax) { m_CaccheSlotMax = slotId; @@ -493,20 +559,23 @@ RETRY_FIND_FREESPACE: UnlockSweep(); LockCacheDescHeader(slotId); - m_CacheDesc[slotId].m_flag = CACHE_BLOCK_NEW; // is !Valid + m_CacheDesc[slotId].m_flag = CACHE_BLOCK_NEW; // is !Valid // 表示新申请的但未初始化的缓存块 - PinCacheBlock_Locked(slotId); // Released header lock + PinCacheBlock_Locked(slotId); // Released header lock // 释放缓存块描述头的锁 /* purposely returning pinned Invalid slot !!! * Returning from here means we have reserved the memory and * a new free slot */ + // 返回一个被固定（pinned）的无效槽位 + // 返回该值意味着我们已经预留了内存并获得了一个新的空闲槽位 return slotId; } else { + // 现在释放内存，并稍后再次获取，一旦我们驱逐一个缓存块后就可以再次获取内存 /* We release the memory now, and get it back again later once we evict a buffer */ ReleaseCacheMem(size); } } - + // 驱逐一个缓存块来腾出空间 slotId = EvictCacheBlock(size, retryNum); /* * If the slotId is CACHE_BLOCK_INVALID_IDX, it means there is not proper slot to replace. @@ -514,6 +583,11 @@ RETRY_FIND_FREESPACE: * to find freespace. The default retry time is 3, and if there is not proper slot to find after * three times retry, it reports ERROR as before. */ + /* + * 如果slotId等于CACHE_BLOCK_INVALID_IDX，表示没有可替换的合适槽位。 + * 然而，在这种情况下，cstore缓冲区可能有空闲空间，因此我们需要重试查找空闲空间。 + * 默认的重试次数是3次，在经过三次重试后仍然找不到合适的槽位，则返回错误。 + */ if (slotId == CACHE_BLOCK_INVALID_IDX) { goto RETRY_FIND_FREESPACE; } @@ -527,6 +601,7 @@ RETRY_FIND_FREESPACE: * @Return: hash code * @See also: */ +//生成给定缓存块唯一标识的哈希值 uint32 CacheMgr::GetHashCode(CacheTag *cacheTag) { return get_hash_value(m_hash, (void *)cacheTag); @@ -538,76 +613,89 @@ uint32 CacheMgr::GetHashCode(CacheTag *cacheTag) * @IN slot: cache block index * @See also: */ +/* + * @Description: 释放缓存块内存，该函数在重新初始化（切换）时被调用，并尝试释放所有缓存列表槽的内存， + * 因此槽的数量可能等于 m_CacheSlotsNum。 + * @IN slot: 缓存块索引 + * @See also: + */ +//函数根据不同的缓存块类型执行不同的内存释放操作，确保释放内存并重置相关变量，防止内存泄漏和悬挂指针问题 void CacheMgr::FreeCacheBlockMem(CacheSlotId_t slot) { Assert(slot >= 0 && slot < m_CacheSlotsNum); if (m_CacheDesc[slot].m_cache_tag.type == CACHE_ORC_INDEX) { + // 如果缓存块类型是 CACHE_ORC_INDEX OrcMetadataValue *value = (OrcMetadataValue *)(&m_CacheSlots[slot * m_slot_length]); if (value->postScript != NULL) { - delete value->postScript; + delete value->postScript;// 释放 postScript 内存 value->postScript = NULL; } if (value->fileFooter != NULL) { - delete value->fileFooter; + delete value->fileFooter;// 释放 fileFooter 内存 value->fileFooter = NULL; } if (value->stripeFooter != NULL) { - delete value->stripeFooter; + delete value->stripeFooter;// 释放 stripeFooter 内存 value->stripeFooter = NULL; } if (value->rowIndex != NULL) { - delete value->rowIndex; + delete value->rowIndex;// 释放 rowIndex 内存 value->rowIndex = NULL; } if (value->fileName != NULL) { - free(value->fileName); + free(value->fileName);// 释放 fileName 内存 value->fileName = NULL; } if (value->dataDNA != NULL) { - free(value->dataDNA); + free(value->dataDNA);// 释放 dataDNA 内存 value->dataDNA = NULL; } - value->footerStart = 0; - value->size = 0; + value->footerStart = 0;// 重置 footerStart + value->size = 0;// 重置 size } else if (m_CacheDesc[slot].m_cache_tag.type == CACHE_CARBONDATA_METADATA) { + // 如果缓存块类型是 CACHE_CARBONDATA_METADATA CarbonMetadataValue* value = (CarbonMetadataValue*)(&m_CacheSlots[slot * m_slot_length]); if (NULL != value->fileHeader) { - free(value->fileHeader); + free(value->fileHeader);// 释放 fileHeader 内存 value->fileHeader = NULL; } if (NULL != value->fileFooter) { - free(value->fileFooter); + free(value->fileFooter);// 释放 fileFooter 内存 value->fileFooter = NULL; } if (NULL != value->fileName) { - free(value->fileName); + free(value->fileName);// 释放 fileName 内存 value->fileName = NULL; } if (NULL != value->dataDNA) { - free(value->dataDNA); + free(value->dataDNA);// 释放 dataDNA 内存 value->dataDNA = NULL; } - value->headerSize = 0; - value->footerSize = 0; - value->size = 0; + value->headerSize = 0;// 重置 headerSize + value->footerSize = 0;// 重置 footerSize + value->size = 0;// 重置 size } else if (m_CacheDesc[slot].m_cache_tag.type == CACHE_COlUMN_DATA) { /* Important: the memory with the same slot may be hold and used by CACHE_ORC_DATA, * CACHE_COlUMN_DATA and CACHE_ORC_INDEX, so we must confirm that the memory should * be set to 0 after the slot is released. at least memory of * min(sizeof(CU), sizeof(OrcDataValue)) size must be cleanned up. */ + // 如果缓存块类型是 CACHE_COlUMN_DATA + /* 注意：同一个槽中的内存可能被 CACHE_ORC_DATA、CACHE_COlUMN_DATA 和 CACHE_ORC_INDEX 使用和持有， + * 因此在释放槽之后，必须确保将内存设置为 0。至少需要清理 sizeof(CU) 或 sizeof(OrcDataValue) 大小的内存。 */ CU *cu = (CU *)(&m_CacheSlots[slot * m_slot_length]); - cu->FreeMem(); - cu->Reset(); + cu->FreeMem();// 释放 CU 内存 + cu->Reset();// 重置 CU } else if (m_CacheDesc[slot].m_cache_tag.type == CACHE_ORC_DATA || m_CacheDesc[slot].m_cache_tag.type == CACHE_OBS_DATA) { + // 如果缓存块类型是 CACHE_ORC_DATA 或 CACHE_OBS_DATA OrcDataValue *orc_data = (OrcDataValue *)(&m_CacheSlots[slot * m_slot_length]); if (orc_data->value != NULL) { - CStoreMemAlloc::Pfree(orc_data->value, false); + CStoreMemAlloc::Pfree(orc_data->value, false);// 释放 orc_data value 内存 orc_data->value = NULL; } - orc_data->size = 0; + orc_data->size = 0;// 重置 size } else { Assert(m_CacheDesc[slot].m_cache_tag.type == CACHE_TYPE_NONE); } @@ -620,28 +708,39 @@ void CacheMgr::FreeCacheBlockMem(CacheSlotId_t slot) * @Return: size of block memory * @See also: */ +/* + * @Description: 获取缓存块的内存大小 + * @IN slot: 缓存块索引 + * @Return: 内存大小 + * @See also: + */ +//根据缓存块的类型，选择不同的处理逻辑获取内存大小 int CacheMgr::GetCacheBlockMemSize(CacheSlotId_t slot) { Assert(slot >= 0 && slot <= m_CaccheSlotMax && slot < m_CacheSlotsNum); int slot_size = 0; if (m_CacheDesc[slot].m_cache_tag.type == CACHE_ORC_INDEX) { + // 如果缓存块类型是 CACHE_ORC_INDEX OrcMetadataValue *value = (OrcMetadataValue *)(&m_CacheSlots[slot * m_slot_length]); - slot_size = value->size; + slot_size = value->size;// 获取 OrcMetadataValue 结构体中记录的内存大小 } else if (m_CacheDesc[slot].m_cache_tag.type == CACHE_CARBONDATA_METADATA) { + // 如果缓存块类型是 CACHE_CARBONDATA_METADATA CarbonMetadataValue* value = (CarbonMetadataValue*)(&m_CacheSlots[slot * m_slot_length]); - return value->size; + return value->size;// 直接返回 CarbonMetadataValue 结构体中记录的内存大小 } else if (m_CacheDesc[slot].m_cache_tag.type == CACHE_COlUMN_DATA) { + // 如果缓存块类型是 CACHE_COlUMN_DATA CU *cu = (CU *)(&m_CacheSlots[slot * m_slot_length]); if (!cu->m_cache_compressed) { - slot_size = cu->GetUncompressBufSize(); + slot_size = cu->GetUncompressBufSize();// 获取未压缩缓存单元的大小 } else { - slot_size = cu->GetCompressBufSize(); + slot_size = cu->GetCompressBufSize();// 获取压缩缓存单元的大小 } } else if (m_CacheDesc[slot].m_cache_tag.type == CACHE_ORC_DATA || m_CacheDesc[slot].m_cache_tag.type == CACHE_OBS_DATA) { + // 如果缓存块类型是 CACHE_ORC_DATA 或 CACHE_OBS_DATA OrcDataValue *orc_data = (OrcDataValue *)(&m_CacheSlots[slot * m_slot_length]); - slot_size = orc_data->size; + slot_size = orc_data->size;// 获取 OrcDataValue 结构体中记录的内存大小 } else { Assert(m_CacheDesc[slot].m_cache_tag.type == CACHE_TYPE_NONE); } @@ -661,15 +760,16 @@ int CacheMgr::GetCacheBlockMemSize(CacheSlotId_t slot) * @Return: true -- enough cache memory and reserve , flase --no free memory * @See also: */ +//如果缓存管理器预留的内存足够，就将请求的内存大小加入到当前已使用的内存大小中，并返回 true；否则返回 false，表示没有足够的空闲内存 bool CacheMgr::ReserveCacheMem(int size) { - SpinLockAcquire(&m_memsize_lock); + SpinLockAcquire(&m_memsize_lock);// 获取互斥锁 if ((m_cstoreCurrentSize + size) <= m_cstoreMaxSize) { - m_cstoreCurrentSize += size; - SpinLockRelease(&m_memsize_lock); + m_cstoreCurrentSize += size;// 如果预留内存加上请求大小小于等于最大允许内存，则将请求内存大小加入当前已使用的内存大小中 + SpinLockRelease(&m_memsize_lock);// 释放互斥锁 return true; } - SpinLockRelease(&m_memsize_lock); + SpinLockRelease(&m_memsize_lock);// 如果没有足够的空闲内存，则直接释放互斥锁并返回 false return false; } @@ -678,12 +778,13 @@ bool CacheMgr::ReserveCacheMem(int size) * @IN size: size to release * @See also: */ +//根据传入的大小，减少预留的内存大小 void CacheMgr::ReleaseCacheMem(int size) { - SpinLockAcquire(&m_memsize_lock); - Assert(m_cstoreCurrentSize >= size); - m_cstoreCurrentSize -= size; - SpinLockRelease(&m_memsize_lock); + SpinLockAcquire(&m_memsize_lock);// 获取互斥锁 + Assert(m_cstoreCurrentSize >= size);// 断言当前已使用的内存大小大于等于释放的大小 + m_cstoreCurrentSize -= size;// 从当前已使用的内存大小中减去释放的大小 + SpinLockRelease(&m_memsize_lock);// 释放互斥锁 } /* @@ -695,14 +796,15 @@ void CacheMgr::ReleaseCacheMem(int size) * @Param[IN] newSize: new block size * @See also: */ +//调整先前分配给缓存块的内存大小。调整操作可能会导致预留内存（m_cstoreCurrentSize）略微超过允许的最大值（m_cstoreMaxSize），直到下一个块被分配为止 void CacheMgr::AdjustCacheMem(CacheSlotId_t slotId, int oldSize, int newSize) { - this->ReleaseCacheMem(oldSize - newSize); + this->ReleaseCacheMem(oldSize - newSize);// 释放需要减少的内存大小 - LockCacheDescHeader(slotId); - Assert(m_CacheDesc[slotId].m_datablock_size == oldSize); - m_CacheDesc[slotId].m_datablock_size = newSize; - UnLockCacheDescHeader(slotId); + LockCacheDescHeader(slotId);// 锁定缓存描述头 + Assert(m_CacheDesc[slotId].m_datablock_size == oldSize);// 断言缓存块的旧大小与参数中的旧大小一致 + m_CacheDesc[slotId].m_datablock_size = newSize;// 更新缓存块的大小为新大小 + UnLockCacheDescHeader(slotId);// 解锁缓存描述头 } /* @@ -710,6 +812,7 @@ void CacheMgr::AdjustCacheMem(CacheSlotId_t slotId, int oldSize, int newSize) * @Return: true -- use out, false -- still have * @See also: */ +//检查缓存的空闲块链表是否为空 bool CacheMgr::CacheFreeListEmpty() const { /* FreeList is terminated with CACHE_BLOCK_INVALID_IDX */ @@ -722,23 +825,24 @@ bool CacheMgr::CacheFreeListEmpty() const * @Return: CACHE_BLOCK_INVALID_IDX -- no free block slot, others normal block slot * @See also: */ +//从空闲槽列表中获取一个空闲的槽位，并将其从列表的头部移除。空闲槽列表和 m_freeNext 均受 m_freeList_lock 保护 int CacheMgr::GetFreeListCache() { int freeSlotIdx = CACHE_BLOCK_INVALID_IDX; - SpinLockAcquire(&m_freeList_lock); + SpinLockAcquire(&m_freeList_lock);// 获取自旋锁，保证并发访问的正确性 if (m_freeListHead != CACHE_BLOCK_INVALID_IDX) { - freeSlotIdx = m_freeListHead; - Assert(freeSlotIdx >= 0 && freeSlotIdx < m_CacheSlotsNum); - m_freeListHead = m_CacheDesc[freeSlotIdx].m_freeNext; - m_CacheDesc[freeSlotIdx].m_freeNext = CACHE_BLOCK_NO_LIST; + freeSlotIdx = m_freeListHead;// 取出空闲槽列表的头部索引 + Assert(freeSlotIdx >= 0 && freeSlotIdx < m_CacheSlotsNum);// 断言空闲槽索引的有效性 + m_freeListHead = m_CacheDesc[freeSlotIdx].m_freeNext;// 将下一个空闲槽的索引赋值给头部索引 + m_CacheDesc[freeSlotIdx].m_freeNext = CACHE_BLOCK_NO_LIST;// 将当前槽位的下一个指针设置为无列表状态 if (m_freeListHead == CACHE_BLOCK_INVALID_IDX) { - m_freeListTail = CACHE_BLOCK_INVALID_IDX; + m_freeListTail = CACHE_BLOCK_INVALID_IDX;// 如果头部索引为空，将尾部索引也设置为空 } } - SpinLockRelease(&m_freeList_lock); + SpinLockRelease(&m_freeList_lock);// 释放自旋锁 - return freeSlotIdx; + return freeSlotIdx;// 返回获取到的空闲槽位索引 } /* @@ -747,14 +851,22 @@ int CacheMgr::GetFreeListCache() * @IN freeSlotIdx: index of cache block * @See also: */ +/* + * @Description: 将缓存块添加到空闲槽列表中，m_freeNext 和列表由 m_freeList_lock 保护， + * 列表按照堆栈的顺序排列。我们重用之前使用过的条目以保持活动集更加紧凑。 + * @IN freeSlotIdx: 缓存块的索引 + * @See also: + */ +//将缓存块添加到空闲槽列表中 void CacheMgr::PutFreeListCache(CacheSlotId_t freeSlotIdx) { Assert(freeSlotIdx >= 0 && freeSlotIdx <= m_CaccheSlotMax && freeSlotIdx < m_CacheSlotsNum); // Put the slot at the head of the list + // 将槽位放在列表的头部 SpinLockAcquire(&m_freeList_lock); - m_CacheDesc[freeSlotIdx].m_freeNext = m_freeListHead; - m_freeListHead = freeSlotIdx; + m_CacheDesc[freeSlotIdx].m_freeNext = m_freeListHead;// 将当前槽位的下一个索引设置为原头部索引 + m_freeListHead = freeSlotIdx;// 更新头部索引为当前槽位索引 if (m_freeListTail == CACHE_BLOCK_INVALID_IDX) { m_freeListTail = m_freeListHead; } @@ -768,11 +880,20 @@ void CacheMgr::PutFreeListCache(CacheSlotId_t freeSlotIdx) * @Return: lock* * @See also: */ +/* + * @Description: 使用分区锁对哈希表进行加锁 + * @IN hashCode: 哈希码 + * @IN lockMode: 锁模式 + * @Return: 锁对象指针 + * @See also: + */ +//使用分区锁对哈希表进行加锁 LWLock* CacheMgr::LockHashPartion(uint32 hashCode, LWLockMode lockMode) const { + // 根据哈希码计算分区索引，并获取相应的主锁 LWLock *partionLock = GetMainLWLockByIndex(m_partition_lock + (hashCode % (NUM_CACHE_BUFFER_PARTITIONS / 2))); - (void)LWLockAcquire(partionLock, lockMode); - return partionLock; + (void)LWLockAcquire(partionLock, lockMode);// 获取分区锁 + return partionLock;// 返回分区锁对象指针 } /* @@ -780,10 +901,17 @@ LWLock* CacheMgr::LockHashPartion(uint32 hashCode, LWLockMode lockMode) const * @IN hashCode: hash code * @See also: */ +/* + * @Description: 使用分区锁对哈希表进行解锁 + * @IN hashCode: 哈希码 + * @See also: + */ +//使用分区锁对哈希表进行解锁 void CacheMgr::UnLockHashPartion(uint32 hashCode) const { + // 根据哈希码计算分区索引，并获取相应的主锁 LWLock *partionLock = GetMainLWLockByIndex(m_partition_lock + (hashCode % (NUM_CACHE_BUFFER_PARTITIONS / 2))); - LWLockRelease(partionLock); + LWLockRelease(partionLock);// 释放分区锁 } /* @@ -791,10 +919,16 @@ void CacheMgr::UnLockHashPartion(uint32 hashCode) const * @IN slotId: cache block index * @See also: */ +/* + * @Description: 使用自旋锁对缓存块描述符头部进行加锁 + * @IN slotId: 缓存块索引 + * @See also: + */ +//使用自旋锁对缓存块的描述符头部进行加锁 void CacheMgr::LockCacheDescHeader(CacheSlotId_t slotId) { Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); - SpinLockAcquire(&m_CacheDesc[slotId].m_slot_hdr_lock); + SpinLockAcquire(&m_CacheDesc[slotId].m_slot_hdr_lock);// 获取自旋锁 } /* @@ -802,10 +936,16 @@ void CacheMgr::LockCacheDescHeader(CacheSlotId_t slotId) * @IN slotId: cache block index * @See also: */ +/* + * @Description: 使用自旋锁对缓存块描述符头部进行解锁 + * @IN slotId: 缓存块索引 + * @See also: + */ +//使用自旋锁对缓存块的描述符头部进行解锁 void CacheMgr::UnLockCacheDescHeader(CacheSlotId_t slotId) { Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); - SpinLockRelease(&m_CacheDesc[slotId].m_slot_hdr_lock); + SpinLockRelease(&m_CacheDesc[slotId].m_slot_hdr_lock);// 释放自旋锁 } /* @@ -814,10 +954,17 @@ void CacheMgr::UnLockCacheDescHeader(CacheSlotId_t slotId) * @Return: true -- add, false --not use * @See also: */ +/* + * @Description: 检查缓存块描述符是否被固定（其他线程正在使用） + * @IN slotId: 缓存块索引 + * @Return: true -- 被固定，false -- 没有被使用 + * @See also: + */ +//检查特定缓存块的描述符是否被固定，即是否有其他线程正在使用该缓存块 bool CacheMgr::CacheBlockIsPinned(CacheSlotId_t slotId) const { Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); - return (m_CacheDesc[slotId].m_refcount > 0); + return (m_CacheDesc[slotId].m_refcount > 0);// 检查引用计数是否大于0 } /* @@ -826,6 +973,13 @@ bool CacheMgr::CacheBlockIsPinned(CacheSlotId_t slotId) const * @Return: return false if the slot is not marked valid, this means the block may soon be removed or changed. * @See also: */ +/* + * @Description: Pin缓存块，增加引用计数。 + * @IN slotId: 缓存块索引 + * @Return: 如果缓存块未标记为有效，则返回false，表示该块可能很快被删除或更改。 + * @See also: + */ +//固定（Pin）特定的缓存块，增加其引用计数 bool CacheMgr::PinCacheBlock(CacheSlotId_t slotId) { Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); @@ -836,20 +990,22 @@ bool CacheMgr::PinCacheBlock(CacheSlotId_t slotId) (m_CacheDesc[slotId].m_cache_tag.type == CACHE_ORC_INDEX || m_CacheDesc[slotId].m_cache_tag.type == CACHE_CARBONDATA_METADATA))); - LockCacheDescHeader(slotId); + LockCacheDescHeader(slotId);// 锁定缓存块描述符头部 /* Pinning an invalid slot returns false and does not pin the slot */ + /* 如果缓存块无效，则返回false并不固定该块 */ if (!(m_CacheDesc[slotId].m_flag & CACHE_BLOCK_VALID)) { - UnLockCacheDescHeader(slotId); + UnLockCacheDescHeader(slotId);// 解锁缓存块描述符头部 return false; } - + /* 增加引用计数以固定缓存块 */ /* increase refcount to pin cache block */ m_CacheDesc[slotId].m_refcount++; Assert(m_CacheDesc[slotId].m_refcount > 0); /* Increment the usage count */ - UnLockCacheDescHeader(slotId); + /* 增加使用计数 */ + UnLockCacheDescHeader(slotId);// 解锁缓存块描述符头部 ereport(DEBUG2, (errmodule(MOD_CACHE), errmsg("pin cache block, slot(%d), type(%d) ,flag(%hhu), refcount(%u)", slotId, m_CacheDesc[slotId].m_cache_tag.type, m_CacheDesc[slotId].m_flag, m_CacheDesc[slotId].m_refcount))); @@ -870,13 +1026,20 @@ bool CacheMgr::PinCacheBlock(CacheSlotId_t slotId) * @See also: header lock must be held on enty. This is an internal pin, we do not want to * count it used yet. We know what we are doing here, there is no need to check whether the Block is valid. */ +/* + * @Description: 锁定缓存块描述符头部的同时，固定（Pin）特定的缓存块，并增加其引用计数。 + * @IN slotId: 缓存块索引 + * @See also: + * 在进入函数时，必须已经持有缓存块描述符头部的锁定。这是一个内部的固定操作，我们不希望将其计入使用计数中。我们已经清楚自己在做什么，没有必要检查缓存块是否有效。 + */ +//是在持有缓存块描述符头部的锁定状态下，固定（Pin）特定的缓存块，并增加其引用计数 void CacheMgr::PinCacheBlock_Locked(CacheSlotId_t slotId) { Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); - + // 增加缓存块描述符的引用计数 m_CacheDesc[slotId].m_refcount++; - UnLockCacheDescHeader(slotId); - + UnLockCacheDescHeader(slotId);// 解锁缓存块描述符头部 + // 根据缓存类型，扩展并记住元数据缓存槽或数据缓存槽 if (m_cache_type == MGR_CACHE_TYPE_INDEX) { ResourceOwnerEnlargeMetaCacheSlot(t_thrd.utils_cxt.CurrentResourceOwner); ResourceOwnerRememberMetaCacheSlot(t_thrd.utils_cxt.CurrentResourceOwner, slotId); @@ -884,6 +1047,7 @@ void CacheMgr::PinCacheBlock_Locked(CacheSlotId_t slotId) ResourceOwnerEnlargeDataCacheSlot(t_thrd.utils_cxt.CurrentResourceOwner); ResourceOwnerRememberDataCacheSlot(t_thrd.utils_cxt.CurrentResourceOwner, slotId); } + // 记录当前固定的缓存块的信息 ereport(DEBUG2, (errmodule(MOD_CACHE), errmsg("pin locked cache block, slot(%d), refcount(%u)", slotId, m_CacheDesc[slotId].m_refcount))); } @@ -893,65 +1057,81 @@ void CacheMgr::PinCacheBlock_Locked(CacheSlotId_t slotId) * @IN slotId: cache block index * @See also: */ +/* + * @Description: 释放缓存块的引用，使其引用计数减一 + * @IN slotId: 缓存块索引 + * @See also: + */ +//释放缓存块的引用，使其引用计数减一 void CacheMgr::UnPinCacheBlock(CacheSlotId_t slotId) { Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 确保缓存块类型符合要求 Assert(((m_cache_type == MGR_CACHE_TYPE_DATA) && (m_CacheDesc[slotId].m_cache_tag.type == CACHE_COlUMN_DATA || m_CacheDesc[slotId].m_cache_tag.type == CACHE_ORC_DATA || m_CacheDesc[slotId].m_cache_tag.type == CACHE_OBS_DATA)) || ((m_cache_type == MGR_CACHE_TYPE_INDEX) && (m_CacheDesc[slotId].m_cache_tag.type == CACHE_ORC_INDEX || m_CacheDesc[slotId].m_cache_tag.type == CACHE_CARBONDATA_METADATA))); - + // 根据缓存类型，从当前资源拥有者的缓存槽中删除该缓存块 if (m_cache_type == MGR_CACHE_TYPE_INDEX) { ResourceOwnerForgetMetaCacheSlot(t_thrd.utils_cxt.CurrentResourceOwner, slotId); } else { ResourceOwnerForgetDataCacheSlot(t_thrd.utils_cxt.CurrentResourceOwner, slotId); } - LockCacheDescHeader(slotId); - Assert(m_CacheDesc[slotId].m_refcount > 0); - m_CacheDesc[slotId].m_refcount--; + LockCacheDescHeader(slotId);// 锁定缓存块描述符头部，防止其他线程修改其状态 + Assert(m_CacheDesc[slotId].m_refcount > 0);// 确保引用计数大于0 + m_CacheDesc[slotId].m_refcount--;// 缓存块引用计数减一 - UnLockCacheDescHeader(slotId); + UnLockCacheDescHeader(slotId);// 解锁缓存块描述符头部 + // 记录释放后的缓存块信息，包括索引、类型和引用计数 ereport(DEBUG2, (errmodule(MOD_CACHE), errmsg("unpin cache block, slot(%d), type(%d) ,refcount(%u)", slotId, m_CacheDesc[slotId].m_cache_tag.type, m_CacheDesc[slotId].m_refcount))); } - +//等待指定的缓存块被释放，直到没有其他会话正在引用该缓存块 void CacheMgr::WaitEvictSlot(CacheSlotId_t slotId) { + // 确保传入的 slotId 合法 Assert(slotId != CACHE_BLOCK_INVALID_IDX); /* we must wait for other session not reference the slot */ while (1) { + // 锁定缓存块描述符头部 LockCacheDescHeader(slotId); /* no other session referenced to this slot anymore */ + // 如果当前缓存块的引用计数为1，表示只有当前会话在使用该缓存块 if (m_CacheDesc[slotId].m_refcount == 1) { - m_CacheDesc[slotId].m_flag = CACHE_BLOCK_INFREE; // !Valid + m_CacheDesc[slotId].m_flag = CACHE_BLOCK_INFREE; // !Valid // 将缓存块标记为释放状态 /* * The slot will be reused with some buffer space, * the space must be freed and reused. * The slot is Invalid and Pinned!!! */ + // 解锁缓存块描述符头部，并跳出循环 UnLockCacheDescHeader(slotId); break; } + // 解锁缓存块描述符头部 UnLockCacheDescHeader(slotId); } } - +//从缓存池中分配一个空闲的缓存块，并返回其插槽号 slotId void CacheMgr::AllocateBlockFromCacheWithSlotId(CacheSlotId_t slotId) { + // 等待缓存块被释放 WaitEvictSlot(slotId); Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); Assert(m_CacheDesc[slotId].m_refcount == 1); Assert(m_CacheDesc[slotId].m_flag == CACHE_BLOCK_INFREE); - + // 锁定缓存块描述符头部 LockCacheDescHeader(slotId); + // 将缓存块的标记设置为可用状态 m_CacheDesc[slotId].m_flag = CACHE_BLOCK_FREE; + // 解锁缓存块描述符头部 UnLockCacheDescHeader(slotId); /* when return we still pinned the slotId cache */ @@ -966,76 +1146,99 @@ void CacheMgr::AllocateBlockFromCacheWithSlotId(CacheSlotId_t slotId) * @Return: block index * @See also: */ +/* + * @Description: 从缓存中分配一个块 + * @Param[IN] cacheTag: 块的唯一标识 + * @Param[IN/OUT] hasFound: 是否在缓存中找到 + * @Param[IN] hashCode: 哈希码 + * @Param[IN] size: 块的大小 + * @Return: 块的索引 + * @See also: + */ +//从缓存中分配一个内存块，用于存储数据 CacheSlotId_t CacheMgr::AllocateBlockFromCache(CacheTag *cacheTag, uint32 hashCode, int size, bool &hasFound) { - CacheLookupEnt *result = NULL; + CacheLookupEnt *result = NULL;// 用于保存在哈希表中找到的缓存块的信息 int old_size = 0; int slot; - Assert(size > 0); + Assert(size > 0);// 校验块大小是否大于0 /* try allocate block from free list */ + /* 尝试从空闲列表中分配块 */ while (1) { - slot = GetFreeCacheBlock(size); + slot = GetFreeCacheBlock(size);// 获取一个空闲的缓存槽位 Assert(slot >= 0 && slot <= m_CaccheSlotMax && slot < m_CacheSlotsNum); - Assert(m_CacheDesc[slot].m_refcount == 1); // Only ours + Assert(m_CacheDesc[slot].m_refcount == 1); // Only ours // 只能有我们引用 /* If it is a new slot, the buffer space is already reserved, * and we still need add it to hash table */ + /* 如果是一个新的槽位，则缓存空间已经被预留，需要将其添加到哈希表中 */ if (m_CacheDesc[slot].m_flag == CACHE_BLOCK_NEW) { /* for new block, cache type is CACHE_TYPE_NONE, we must set * the type first, becase this is key to distinguish code logic */ + /* 对于新块，缓存类型是CACHE_TYPE_NONE，我们必须先设置类型，因为这是区分代码逻辑的关键 */ m_CacheDesc[slot].m_cache_tag.type = cacheTag->type; break; } - + /* 缓存块不在空闲列表中，从空闲缓存中转移，旧的缓存块已经无效并且需要释放 */ /* cache block not on the free list, evict from free cache, * so the old cache block has changed to invalid and need to free */ Assert(m_CacheDesc[slot].m_flag == CACHE_BLOCK_INFREE); - + /* 移除哈希表中的条目 */ /* remove the hash table entry */ DeleteCacheBlock(&m_CacheDesc[slot].m_cache_tag); /* mark the block free */ + /* 标记块为空闲 */ LockCacheDescHeader(slot); - old_size = m_CacheDesc[slot].m_datablock_size; - m_CacheDesc[slot].m_flag = CACHE_BLOCK_FREE; // !Valid and Free + old_size = m_CacheDesc[slot].m_datablock_size;// 保存旧的块大小 + m_CacheDesc[slot].m_flag = CACHE_BLOCK_FREE; // !Valid and Free // 非有效，空闲状态 m_CacheDesc[slot].m_datablock_size = 0; UnLockCacheDescHeader(slot); /* free the cache block memory */ + /* 释放缓存块的内存 */ FreeCacheBlockMem(slot); /* return the old size and get the new size * so we could have a net gain here */ - if (ReserveCacheMem(size - old_size)) { + /* 返回旧的大小并获取新的大小，我们可能会有净增益 */ + if (ReserveCacheMem(size - old_size)) {// 根据新的大小重新预留缓存内存 /* We have reserved the memory, we will use this slot */ + /* 我们已经预留了内存，将使用该槽位 */ break; } /* no enough cache memory, just release the old size and go back for another * Now that the slot is dead no need to pin it anymore */ - ReleaseCacheMem(old_size); - UnPinCacheBlock(slot); + /* 没有足够的缓存内存，释放旧的大小，并继续寻找下一个 + * 现在槽位已经无效，不需要再引用 */ + ReleaseCacheMem(old_size);// 释放旧的缓存内存 + UnPinCacheBlock(slot);// 取消引用该槽位的缓存块 /* noy used any more, so add to free list */ + /* 不再使用，因此加入空闲列表 */ PutFreeListCache(slot); } - + /* 检查是否仍然被引用 */ /* check still pinned */ Assert(m_CacheDesc[slot].m_refcount > 0); TRYAGAIN: - (void)LockHashPartion(hashCode, LW_EXCLUSIVE); + (void)LockHashPartion(hashCode, LW_EXCLUSIVE);// 加锁访问哈希表 + /* 在哈希表中查找对应的缓存标识 */ result = (CacheLookupEnt *)hash_search_with_hash_value(m_hash, (void *)cacheTag, hashCode, HASH_ENTER, &hasFound); /* if the tag is already in the table somebody beat us to creating one * if the block is invalid, the found block could be in the process of being removed, so try again */ + /* 如果在表中找到了标识，则检查缓存块是否有效，如果无效，则重试 */ if (hasFound) { - LockCacheDescHeader(result->slot_id); + LockCacheDescHeader(result->slot_id);// 锁定缓存块头部信息 if (m_CacheDesc[result->slot_id].m_flag & CACHE_BLOCK_ERROR) { hasFound = false; } UnLockCacheDescHeader(result->slot_id); if (hasFound) { + /* 必须在使用之前引用该块 */ /* must pin the block before use */ bool valid = PinCacheBlock(result->slot_id); if (!valid) { @@ -1046,23 +1249,24 @@ TRYAGAIN: goto NOT_FOUND; } UnLockHashPartion(hashCode); - + /* 在这一步，我们发现缓存中已经存在该块 + * 因此我们需要释放自己的块并使用缓存中的块 */ /* at this point, we discover the block already in the cache // so we need to free ours and use the cached one */ - ReleaseCacheMem(size); + ReleaseCacheMem(size);// 释放自己分配的内存 ereport(DEBUG2, (errmodule(MOD_CACHE), errmsg("find cache block already in cache, solt(%d), change flag(%d - %d)", slot, m_CacheDesc[slot].m_flag, CACHE_BLOCK_FREE))); - + /* 标记块为空闲，并取消引用该块，将其放入空闲列表 */ /* mark the block free, unpin the block and put it to free list */ LockCacheDescHeader(slot); - m_CacheDesc[slot].m_flag = CACHE_BLOCK_FREE; // !Valid and Free + m_CacheDesc[slot].m_flag = CACHE_BLOCK_FREE; // !Valid and Free // 非有效，空闲状态 m_CacheDesc[slot].m_datablock_size = 0; UnLockCacheDescHeader(slot); UnPinCacheBlock(slot); - PutFreeListCache(slot); - + PutFreeListCache(slot);// 加入空闲列表 + /* 返回找到的被引用的槽位 */ /* return the found pinned slot */ return result->slot_id; } @@ -1090,9 +1294,23 @@ NOT_FOUND: * other session want to load this slod and cache data, they must be * waiting for IO complete. */ +/* + * @Description: 释放由slotId表示的缓存块 + * @Param[IN] slotId: 要释放的缓存块的槽位索引 + * @Return: true表示会话可以释放缓存块。 + * false表示其他会话正在并发更新该缓存块。 + * 在某些情况下，可能会有两个或多个线程同时更新缓存块。 + * 为了避免这种情况，我们在函数开始时判断缓存状态，以测试是否可以继续释放和更新缓存。 + * 当m_CacheDesc[slotId].m_refreshing为true时，表示其他会话正在更新相关的缓存。 + * 如果不是，则锁定缓存描述符并将其从false更改为true，以使其他后来的会话无法更新缓存。 + * 在清除槽位、释放缓存块内容并将其置于IO_busy状态后，将m_CacheDesc[slotId].m_refreshing设置为false。 + * 此时，如果其他会话想要加载此槽位和缓存数据，它们必须等待IO完成。 + */ +//用于释放由slotId表示的缓存块 bool CacheMgr::ReserveCacheBlockWithSlotId(CacheSlotId_t slotId) { /* judge if there is any other thread to fresh this slot content */ + /* 判断是否有其他线程正在刷新该槽位内容 */ LockCacheDescHeader(slotId); if (m_CacheDesc[slotId].m_refreshing == true) { @@ -1102,12 +1320,12 @@ bool CacheMgr::ReserveCacheBlockWithSlotId(CacheSlotId_t slotId) m_CacheDesc[slotId].m_refreshing = true; UnLockCacheDescHeader(slotId); - + /* 可以安全地清除槽位 */ /* we can safely evict the slot */ AllocateBlockFromCacheWithSlotId(slotId); Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); - + // 锁定缓存描述符，设置缓存块的状态 LockCacheDescHeader(slotId); m_CacheDesc[slotId].m_usage_count = 1; @@ -1115,7 +1333,7 @@ bool CacheMgr::ReserveCacheBlockWithSlotId(CacheSlotId_t slotId) m_CacheDesc[slotId].m_flag = CACHE_BLOCK_VALID | CACHE_BLOCK_IOBUSY; UnLockCacheDescHeader(slotId); - + /* 清除缓存块数据，并稍后填充 */ /* clear the cache block data and fill in it later, */ FreeCacheBlockMem(slotId); @@ -1125,8 +1343,12 @@ bool CacheMgr::ReserveCacheBlockWithSlotId(CacheSlotId_t slotId) * block momentarily, when racing with * another insert/reserve */ + /* + * 在I/O完成之前，阻止对数据的后续访问。 + * 当与插入/保留竞争时，此调用可能会暂时阻塞。 + */ (void)LWLockAcquire(m_CacheDesc[slotId].m_iobusy_lock, LW_EXCLUSIVE); - + /* 已经将slotId标记为繁忙，可以安全地将m_refreshing更改为false */ /* already make slotId busy, we can safely change m_refreshing to false */ LockCacheDescHeader(slotId); @@ -1135,8 +1357,8 @@ bool CacheMgr::ReserveCacheBlockWithSlotId(CacheSlotId_t slotId) UnLockCacheDescHeader(slotId); - IncRingCount(slotId); // record io state - Assert(m_CacheDesc[slotId].m_refcount > 0); // pinned + IncRingCount(slotId); // record io state // 记录I/O状态 + Assert(m_CacheDesc[slotId].m_refcount > 0); // pinned // 引用计数大于0 return true; } @@ -1155,8 +1377,9 @@ bool CacheMgr::ReserveCacheBlockWithSlotId(CacheSlotId_t slotId) bool CacheMgr::ReserveCstoreCacheBlockWithSlotId(CacheSlotId_t slotId) { /* judge if there is any other thread to fresh this slot content */ + // 判断该槽位是否已被标记为IOBUSY LockCacheDescHeader(slotId); - + // 将该缓存块标记为IO繁忙状态和有效状态 if (m_CacheDesc[slotId].m_flag & CACHE_BLOCK_IOBUSY) { UnLockCacheDescHeader(slotId); return false; @@ -1172,8 +1395,9 @@ bool CacheMgr::ReserveCstoreCacheBlockWithSlotId(CacheSlotId_t slotId) * block momentarily, when racing with * another insert/reserve */ + // 获取IO锁，阻塞对数据的后续访问，直到I/O操作完成 (void)LWLockAcquire(m_CacheDesc[slotId].m_iobusy_lock, LW_EXCLUSIVE); - + // 增加环计数，记录IO状态，并确保缓存块仍然被引用 IncRingCount(slotId); // record io state Assert(m_CacheDesc[slotId].m_refcount > 0); // pinned @@ -1208,11 +1432,20 @@ bool CacheMgr::ReserveCstoreCacheBlockWithSlotId(CacheSlotId_t slotId) * @Return: * @See also: */ +/* + * @Description: 将缓存块保留在缓存中 + * @IN cacheTag: 块的唯一标识符 + * @OUT hasFound: 缓存中是否找到块 + * @IN size: 缓存块的内存大小 + * @Return: 缓存槽位的ID + * @See also: + */ +//将缓存块保留在缓存中 CacheSlotId_t CacheMgr::ReserveCacheBlock(CacheTag *cacheTag, int size, bool &hasFound) { int slot; uint32 hashCode = GetHashCode(cacheTag); - + // 获取一个缓存块槽位 slot = AllocateBlockFromCache(cacheTag, hashCode, size, hasFound); Assert(slot >= 0 && slot <= m_CaccheSlotMax && slot < m_CacheSlotsNum); if (hasFound) { @@ -1230,6 +1463,7 @@ CacheSlotId_t CacheMgr::ReserveCacheBlock(CacheTag *cacheTag, int size, bool &ha errmsg("Reserve cache block, add IOBUSY flag, slot(%d), type(%d)", slot, cacheTag->type))); /* now our block is in the cache. initialize it before anybody sees it */ + //初始化新的块 LockCacheDescHeader(slot); InitCacheBlockTag(&(m_CacheDesc[slot].m_cache_tag), cacheTag->type, cacheTag->key, MAX_CACHE_TAG_LEN); @@ -1239,6 +1473,7 @@ CacheSlotId_t CacheMgr::ReserveCacheBlock(CacheTag *cacheTag, int size, bool &ha UnLockCacheDescHeader(slot); /* clear the block now, and fill it in later, */ + //将新的块放入哈希表中 FreeCacheBlockMem(slot); /* Block subsequent accesses to the data until the I/O is done @@ -1258,11 +1493,20 @@ CacheSlotId_t CacheMgr::ReserveCacheBlock(CacheTag *cacheTag, int size, bool &ha * @IN slotId: cache block index * @See also: */ +/* + * @Description: 获取缓存块 + * @IN slotId: 缓存块索引 + * @Return: 缓存块指针 + * @See also: + */ +//获取缓存块的缓存标识符（CacheTag） void *CacheMgr::GetCacheBlock(CacheSlotId_t slotId) { + // 检查参数是否合法 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 检查缓存块是否被固定 Assert(CacheBlockIsPinned(slotId)); - + // 返回缓存块指针 return &(m_CacheSlots[slotId * m_slot_length]); } @@ -1273,18 +1517,38 @@ void *CacheMgr::GetCacheBlock(CacheSlotId_t slotId) * @Return: * @See also: */ +/* + * @Description: 获取缓存块的缓存标签 + * @IN refcount: 缓存块的引用计数 + * @IN slotId: 缓存块索引 + * @Return: 指向缓存标签的指针 + * @See also: + */ +//获取缓存块的缓存标签（CacheTag） const CacheTag *CacheMgr::GetCacheBlockTag(CacheSlotId_t slotId, uint32 *refcount) const { + // 检查参数是否合法 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 获取缓存块的引用计数 *refcount = m_CacheDesc[slotId].m_refcount; + // 返回缓存块的缓存标签 return &(m_CacheDesc[slotId].m_cache_tag); } - +/* + * @Description: 将缓存块的缓存标签安全地复制到输出缓冲区 + * @IN slotId: 缓存块索引 + * @INOUT outTag: 输出缓冲区指针，用于存储复制后的缓存标签 + * @See also: + */ /* copy cache block tag to out buffer safely */ +//将缓存块的缓存标签（CacheTag）安全地复制到输出缓冲区中 void CacheMgr::CopyCacheBlockTag(CacheSlotId_t slotId, CacheTag *outTag) { + // 锁定缓存块的描述头部，确保安全复制 LockCacheDescHeader(slotId); + // 复制缓存块的缓存标签到输出缓冲区 *outTag = m_CacheDesc[slotId].m_cache_tag; + // 解锁缓存块的描述头部 UnLockCacheDescHeader(slotId); } @@ -1292,8 +1556,14 @@ void CacheMgr::CopyCacheBlockTag(CacheSlotId_t slotId, CacheTag *outTag) * @Description: lock cache buffer before evict start * @See also: */ +/* + * @Description: 在进行缓存驱逐操作前锁定缓存缓冲区 + * @See also: + */ +//获取具有排他性质的轻量级锁（LWLock），以确保在进行缓存驱逐操作期间的数据一致性和互斥访问 void CacheMgr::LockSweep() { + // 获取共享资源锁，模式为LW_EXCLUSIVE (void)LWLockAcquire(m_csweep_lock, LW_EXCLUSIVE); } @@ -1301,9 +1571,15 @@ void CacheMgr::LockSweep() * @Description: unlock cache buffer after evict end * @See also: */ +/* + * @Description: 在缓存驱逐操作结束后解锁缓存缓冲区 + * @See also: + */ +//释放先前获取的轻量级锁（LWLock），以便其他线程可以继续访问缓存缓冲区。这样做可以确保在缓存驱逐操作结束后，其他操作可以正常进行 void CacheMgr::UnlockSweep() { - LWLockRelease(m_csweep_lock); + + LWLockRelease(m_csweep_lock);// 释放缓存驱逐锁 } /* @@ -1311,11 +1587,21 @@ void CacheMgr::UnlockSweep() * @Param[IN] slot: cache block index * @See also: */ +/* + * @Description: 增加环形计数 + * @Param[IN] slot: 缓存块索引 + * @See also: + */ +//通过增加缓存块的环形计数来记录缓存块的使用情况 void CacheMgr::IncRingCount(CacheSlotId_t slot) { + // 断言确保缓存块索引在有效范围内 Assert(slot >= 0 && slot <= m_CaccheSlotMax && slot < m_CacheSlotsNum); + // 锁定缓存块的描述头部，确保安全操作 LockCacheDescHeader(slot); + // 增加缓存块的环形计数 m_CacheDesc[slot].m_ring_count++; + // 解锁缓存块的描述头部 UnLockCacheDescHeader(slot); } @@ -1324,13 +1610,23 @@ void CacheMgr::IncRingCount(CacheSlotId_t slot) * @Param[IN] slot: slot id * @See also: */ +/* + * @Description: 减少环形计数 + * @Param[IN] slot: 缓存块ID + * @See also: + */ +//通过减少缓存块的环形计数来调整缓存块的使用情况 void CacheMgr::DecRingCount(CacheSlotId_t slot) { + // 断言确保缓存块ID在有效范围内 Assert(slot >= 0 && slot <= m_CaccheSlotMax && slot < m_CacheSlotsNum); - + // 锁定缓存块的描述头部，确保安全操作 LockCacheDescHeader(slot); + // 断言确保缓存块的环形计数大于0 Assert(m_CacheDesc[slot].m_ring_count > 0); + // 减少缓存块的环形计数 m_CacheDesc[slot].m_ring_count--; + // 解锁缓存块的描述头部 UnLockCacheDescHeader(slot); } @@ -1339,28 +1635,36 @@ void CacheMgr::DecRingCount(CacheSlotId_t slot) * @Param[IN] slot: slot id * @See also: */ +/* + * @Description: 预取资源时，如果保留缓存块出现错误，则执行资源清理操作 + * @Param[IN] slot: 缓存块ID + * @See also: + */ +//在预取（prefetch）资源时，如果保留缓存块出现错误，则执行资源清理操作 void CacheMgr::AbortCacheBlock(CacheSlotId_t slot) { // check slot + // 检查缓存块ID是否有效 if (slot == CACHE_BLOCK_INVALID_IDX) { return; } Assert(slot >= 0 && slot <= m_CaccheSlotMax && slot < m_CacheSlotsNum); - + // 如果当前线程没有锁定缓存块，则先获取锁定 if (!LockHeldByMe(slot)) { LockOwn(slot); + // 释放IO繁忙锁 LWLockRelease(m_CacheDesc[slot].m_iobusy_lock); ereport(DEBUG1, (errmodule(MOD_CACHE), errmsg("abort cache block, release iobusy lock, slot(%d), type(%d)", slot, m_CacheDesc[slot].m_cache_tag.type))); } - + // 将缓存块设置为错误状态 SetCacheBlockErrorState(slot); - + // 减少缓存块的环形计数 LockCacheDescHeader(slot); Assert(m_CacheDesc[slot].m_ring_count > 0); m_CacheDesc[slot].m_ring_count--; UnLockCacheDescHeader(slot); - + // 输出日志 ereport(DEBUG1, (errmodule(MOD_CACHE), errmsg("abort cache block, slot(%d), type(%d), flag(%hhu), refcount(%u)", slot, m_CacheDesc[slot].m_cache_tag.type, m_CacheDesc[slot].m_flag, m_CacheDesc[slot].m_refcount))); @@ -1371,14 +1675,22 @@ void CacheMgr::AbortCacheBlock(CacheSlotId_t slot) * @Param[IN] slot: cache block index * @See also: */ +/* + * @Description: 将缓存块设置为错误状态 + * @Param[IN] slot: 缓存块索引 + * @See also: + */ +//将缓存块设置为错误状态 void CacheMgr::SetCacheBlockErrorState(CacheSlotId_t slot) { Assert(slot >= 0 && slot <= m_CaccheSlotMax && slot < m_CacheSlotsNum); - + // 锁定缓存块的头部 LockCacheDescHeader(slot); + // 设置缓存块的使用计数为0，标志位为CACHE_BLOCK_ERROR m_CacheDesc[slot].m_usage_count = 0; m_CacheDesc[slot].m_flag = CACHE_BLOCK_ERROR; UnLockCacheDescHeader(slot); + // 输出日志 ereport(DEBUG1, (errmodule(MOD_CACHE), errmsg("set cache block error state, slot(%d), type(%d) ,flag(%d)", slot, m_CacheDesc[slot].m_cache_tag.type, m_CacheDesc[slot].m_flag))); } @@ -1388,11 +1700,15 @@ void CacheMgr::SetCacheBlockErrorState(CacheSlotId_t slot) * @Return: cache memory size * @See also: */ +//返回当前缓存内存使用量 int64 CacheMgr::GetCurrentMemSize() { int64 memSize = 0; + // 获取缓存内存使用量锁 SpinLockAcquire(&m_memsize_lock); + // 记录缓存内存使用量 memSize = m_cstoreCurrentSize; + // 释放缓存内存使用量锁 SpinLockRelease(&m_memsize_lock); return memSize; @@ -1406,16 +1722,25 @@ int64 CacheMgr::GetCurrentMemSize() * @Return:true --in IO state; false --not in IO state * @See also: */ +/* + * @Description: 检查缓存块是否处于IO忙碌状态 + * @Param[IN] slotId: 缓存块编号 + * @Return: true表示在IO状态，false表示不在IO状态 + * @See also: + */ +//检查指定的缓存块是否处于IO忙碌状态 bool CacheMgr::IsIOBusy(CacheSlotId_t slotId) { bool ret = false; - + // 确保缓存块编号的有效性 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); - + // 锁定缓存块的头部 LockCacheDescHeader(slotId); + // 如果缓存块的标志位包含CACHE_BLOCK_IOBUSY，则表示在IO状态 if (m_CacheDesc[slotId].m_flag & CACHE_BLOCK_IOBUSY) { ret = true; } + // 解锁缓存块的头部 UnLockCacheDescHeader(slotId); return ret; @@ -1430,28 +1755,42 @@ bool CacheMgr::IsIOBusy(CacheSlotId_t slotId) * @Return: true, means error happen in preftech * @See also: */ +/* + * @Description: 等待IO操作完成 + * 如果缓存块处于IO忙碌状态，则进入睡眠状态，等待IO忙碌锁的释放。 + * 当被唤醒时，释放锁并再次检查缓存块的状态。 + * 该方法基于行存储的WaitIO()方法进行修改。 + * @Param[IN] slotId: 缓存块编号 + * @Return: true表示预取期间发生错误 + * @See also: + */ +//等待缓存块的IO完成。如果缓存块处于IO忙碌状态，则线程会进入睡眠状态，等待IO忙碌锁的释放。当被唤醒时，会释放锁并再次检查缓存块的状态。 bool CacheMgr::WaitIO(CacheSlotId_t slotId) { bool check_error = false; CacheFlags flag; - + // 确保缓存块编号的有效性 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); for (;;) { + // 锁定缓存块的头部 LockCacheDescHeader(slotId); + // 获取缓存块的标志位 flag = m_CacheDesc[slotId].m_flag; + // 如果标志位包含CACHE_BLOCK_ERROR，则表示在模拟中找到CU但已取消 if (flag & CACHE_BLOCK_ERROR) { // this means find cu in simulate but cancled check_error = true; break; } - + // 如果标志位不包含CACHE_BLOCK_IOBUSY，则表示不在IO忙碌状态，可以退出循环 if (!(flag & CACHE_BLOCK_IOBUSY)) break; - + // 解锁缓存块的头部，然后以共享模式获取IO忙碌锁，并释放锁 UnLockCacheDescHeader(slotId); (void)LWLockAcquire(m_CacheDesc[slotId].m_iobusy_lock, LW_SHARED); LWLockRelease(m_CacheDesc[slotId].m_iobusy_lock); } + // 解锁缓存块的头部 UnLockCacheDescHeader(slotId); return check_error; @@ -1464,28 +1803,42 @@ bool CacheMgr::WaitIO(CacheSlotId_t slotId) * @Param[IN] slotId: slot id * @See also: */ +/* + * @Description: CompleteIO + * 当IO操作完成时，将缓存块标记为非忙碌状态，并唤醒下一个等待者。 + * 该方法类似于列存储的TerminateBufferIO方法。 + * @Param[IN] slotId: 缓存块编号 + * @See also: + */ +//在IO操作完成后，将缓存块标记为非忙碌状态，并唤醒下一个等待者 void CacheMgr::CompleteIO(CacheSlotId_t slotId) { + // 确保缓存块编号的有效性 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 确保缓存块类型与缓存管理器类型匹配 Assert(((m_cache_type == MGR_CACHE_TYPE_DATA) && (m_CacheDesc[slotId].m_cache_tag.type == CACHE_COlUMN_DATA || m_CacheDesc[slotId].m_cache_tag.type == CACHE_ORC_DATA || m_CacheDesc[slotId].m_cache_tag.type == CACHE_OBS_DATA)) || ((m_cache_type == MGR_CACHE_TYPE_INDEX) && (m_CacheDesc[slotId].m_cache_tag.type == CACHE_ORC_INDEX || m_CacheDesc[slotId].m_cache_tag.type == CACHE_CARBONDATA_METADATA))); - + // 锁定缓存块的头部 LockCacheDescHeader(slotId); + // 如果缓存块不处于IO忙碌状态，则等待一段时间并报告错误 if (!(m_CacheDesc[slotId].m_flag & CACHE_BLOCK_IOBUSY)) { pg_usleep(5000); ereport(PANIC, (errmsg("complete IO, slot(%d), flag(%d)", slotId, m_CacheDesc[slotId].m_flag))); } - + // 将缓存块的标志位设置为非IO忙碌状态 m_CacheDesc[slotId].m_flag &= ~CACHE_BLOCK_IOBUSY; + // 解锁缓存块的头部 UnLockCacheDescHeader(slotId); + // 输出调试信息 ereport(DEBUG1, (errmodule(MOD_CACHE), errmsg("complete IO, slot(%d), flag(%d)", slotId, m_CacheDesc[slotId].m_flag))); + // 释放IO忙碌锁 LWLockRelease(m_CacheDesc[slotId].m_iobusy_lock); - + // 减少环形计数器的值 DecRingCount(slotId); return; @@ -1497,9 +1850,12 @@ void CacheMgr::CompleteIO(CacheSlotId_t slotId) * @Return: true -- lock; false --not lock * @See also: */ +//检查当前线程是否持有指定缓存块的IO忙碌锁。 bool CacheMgr::LockHeldByMe(CacheSlotId_t slotId) { + // 确保缓存块编号的有效性 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 调用LWLockHeldByMe函数检查当前线程是否持有指定缓存块的IO忙碌锁 return LWLockHeldByMe(m_CacheDesc[slotId].m_iobusy_lock); } @@ -1508,9 +1864,12 @@ bool CacheMgr::LockHeldByMe(CacheSlotId_t slotId) * @Param[IN] slotId: slot id * @See also: */ +//获取指定缓存块的IO忙碌锁。通过调用LWLockOwn函数来获取指定的锁 void CacheMgr::LockOwn(CacheSlotId_t slotId) { + // 确保缓存块编号的有效性 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 调用LWLockOwn函数获取指定缓存块的IO忙碌锁 LWLockOwn(m_CacheDesc[slotId].m_iobusy_lock); } @@ -1519,9 +1878,17 @@ void CacheMgr::LockOwn(CacheSlotId_t slotId) * @Param[IN] slotId: slot id * @See also: */ +// 该函数的作用是释放指定缓存块的IO忙碌锁。 +/* + * @Description: disown the lock + * @Param[IN] slotId: 缓存块编号 + * @See also: + */ void CacheMgr::LockDisown(CacheSlotId_t slotId) { + // 确保缓存块编号的有效性 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 调用LWLockDisown函数释放指定缓存块的IO忙碌锁 LWLockDisown(m_CacheDesc[slotId].m_iobusy_lock); } @@ -1529,9 +1896,12 @@ void CacheMgr::LockDisown(CacheSlotId_t slotId) * @Description: add cstore IO lock * @Param[IN] slotId: slot id */ +// 该函数的作用是获取Cstore IO锁并将其添加到缓存块。 void CacheMgr::AcquireCstoreIOLock(CacheSlotId_t slotId) { + // 确保缓存块编号的有效性 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 调用LWLockAcquire函数将Cstore IO锁以独占模式添加到指定缓存块 (void)LWLockAcquire(m_CacheDesc[slotId].m_iobusy_lock, LW_EXCLUSIVE); } @@ -1539,10 +1909,15 @@ void CacheMgr::AcquireCstoreIOLock(CacheSlotId_t slotId) * @Description: release cstore IO lock * @Param[IN] slotId: slot id: */ +// 该函数的作用是释放指定缓存块的Cstore IO锁。 +/* + * @Description: release cstore IO lock + * @Param[IN] slotId: 缓存块编号 + */ void CacheMgr::RealeseCstoreIOLock(CacheSlotId_t slotId) { - Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); - LWLockRelease(m_CacheDesc[slotId].m_iobusy_lock); + Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum);// 确保缓存块编号的有效性 + LWLockRelease(m_CacheDesc[slotId].m_iobusy_lock);// 调用LWLockRelease函数释放指定缓存块的Cstore IO锁 } /* @@ -1550,9 +1925,17 @@ void CacheMgr::RealeseCstoreIOLock(CacheSlotId_t slotId) * @Param[IN] slotId:slot id * @Return: true -- lock; false --not lock */ +// 该函数的作用是检查当前是否由当前线程持有指定缓存块的Cstore IO锁。 +/* + * @Description: check whether the IO lock is held by me + * @Param[IN] slotId: 缓存块编号 + * @Return: true -- 锁被持有；false -- 锁未被持有 + */ bool CacheMgr::CstoreIOLockHeldByMe(CacheSlotId_t slotId) { + // 确保缓存块编号的有效性 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 调用LWLockHeldByMe函数检查当前线程是否持有指定缓存块的Cstore IO锁 return LWLockHeldByMe(m_CacheDesc[slotId].m_iobusy_lock); } @@ -1561,9 +1944,17 @@ bool CacheMgr::CstoreIOLockHeldByMe(CacheSlotId_t slotId) * @Param[IN] slotId: slot id * @See also: */ +// 该函数的作用是获取指定缓存块的压缩锁。 +/* + * @Description: add compress lock + * @Param[IN] slotId: 缓存块编号 + * @See also: + */ void CacheMgr::AcquireCompressLock(CacheSlotId_t slotId) { + // 确保缓存块编号的有效性 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 调用LWLockAcquire函数以排他方式获取指定缓存块的压缩锁 (void)LWLockAcquire(m_CacheDesc[slotId].m_compress_lock, LW_EXCLUSIVE); } @@ -1572,9 +1963,17 @@ void CacheMgr::AcquireCompressLock(CacheSlotId_t slotId) * @Param[IN] slotId: slot id * @See also: */ +// 该函数的作用是释放指定缓存块的压缩锁。 +/* + * @Description: release compress lock + * @Param[IN] slotId: 缓存块编号 + * @See also: + */ void CacheMgr::RealeseCompressLock(CacheSlotId_t slotId) { + // 确保缓存块编号的有效性 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 调用LWLockRelease函数释放指定缓存块的压缩锁 LWLockRelease(m_CacheDesc[slotId].m_compress_lock); } @@ -1583,8 +1982,16 @@ void CacheMgr::RealeseCompressLock(CacheSlotId_t slotId) * @Param[IN] slotId:slot id * @Return: true -- lock; false --not lock */ +// 该函数的作用是检查当前线程是否持有指定缓存块的压缩锁。 +/* + * @Description: check whether the compress lock by me + * @Param[IN] slotId: 缓存块编号 + * @Return: 如果当前线程持有指定缓存块的压缩锁，返回true；否则返回false。 + */ bool CacheMgr::CompressLockHeldByMe(CacheSlotId_t slotId) { + // 确保缓存块编号的有效性 Assert(slotId >= 0 && slotId <= m_CaccheSlotMax && slotId < m_CacheSlotsNum); + // 调用LWLockHeldByMe函数检查当前线程是否持有指定缓存块的压缩锁 return LWLockHeldByMe(m_CacheDesc[slotId].m_compress_lock); } -- 2.34.1 From 907076b9f09431f792dd8d3be8772cd49c3d8289 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Wed, 4 Oct 2023 05:19:33 +0800 Subject: [PATCH 37/48] Update cstore_allocspace.cpp --- .../storage/cstore/cstore_allocspace.cpp | 338 +++++++++++------- 1 file changed, 210 insertions(+), 128 deletions(-) diff --git a/src/gausskernel/storage/cstore/cstore_allocspace.cpp b/src/gausskernel/storage/cstore/cstore_allocspace.cpp index 0d0710779..f26ce5ee3 100644 --- a/src/gausskernel/storage/cstore/cstore_allocspace.cpp +++ b/src/gausskernel/storage/cstore/cstore_allocspace.cpp @@ -37,42 +37,46 @@ #include "utils/snapmgr.h" HTAB* CStoreColspaceCache = NULL; - +//用于描述CStore列文件的相关信息 typedef struct { - CStoreColumnFileTag tag; - uint64 maxOffset; - uint32 maxCuid; - uint64 extendOffset; + CStoreColumnFileTag tag;//用于标识CStore列文件的标签 + uint64 maxOffset;//表示CStore列文件的最大偏移量 + uint32 maxCuid;//表示CStore列文件的最大CUID（Column Unique Identifier） + uint64 extendOffset;//表示CStore列文件的扩展偏移量 } CStoreColFileDesc; - +//计算CStore分配器共享内存空间的大小 Size CStoreAllocatorShmSize() { - Size size = 0; + Size size = 0;// 初始化共享内存大小为0 + // 估算以256为预期哈希表项数和CStoreColFileDesc结构体大小为基础的哈希表占用内存空间的估算值 + // 并将估算值累加到size变量中 size = add_size(size, hash_estimate_size(256, sizeof(CStoreColFileDesc))); - return size; + return size;// 返回CStore分配器共享内存空间的大小 } - +//初始化CStore列空间缓存（CStoreColspaceCache）的哈希表 void CStoreAllocator::InitColSpaceCache(void) { - HASHCTL ctl; + HASHCTL ctl;// 哈希表控制结构体 + + if (CStoreColspaceCache == NULL) {// 如果CStoreColspaceCache为空 + errno_t rc = memset_s(&ctl, sizeof(ctl), 0, sizeof(ctl));// 将ctl结构体初始化为0 - if (CStoreColspaceCache == NULL) { - errno_t rc = memset_s(&ctl, sizeof(ctl), 0, sizeof(ctl)); securec_check(rc, "", ""); - ctl.keysize = sizeof(CStoreColumnFileTag); - ctl.entrysize = sizeof(CStoreColFileDesc); - ctl.hash = tag_hash; + ctl.keysize = sizeof(CStoreColumnFileTag);// 设置哈希表键的大小为CStoreColumnFileTag结构体的大小 + ctl.entrysize = sizeof(CStoreColFileDesc);// 设置哈希表项的大小为CStoreColFileDesc结构体的大小 + ctl.hash = tag_hash;// 设置哈希函数为tag_hash + // 创建CStore Column Space Cache哈希表，并初始化为指定的大小和配置项 CStoreColspaceCache = HeapMemInitHash("CStore Column Space Cache", 40960, 81920, &ctl, HASH_ELEM | HASH_FUNCTION); - if (CStoreColspaceCache == NULL) + if (CStoreColspaceCache == NULL)// 如果创建哈希表失败 ereport(PANIC, (errmsg("could not initialize CStore Column space desc hash table"))); } } - +//重置CStore列空间缓存（CStoreColspaceCache）的哈希表 void CStoreAllocator::ResetColSpaceCache(void) { - if (CStoreColspaceCache != NULL) { - HeapMemResetHash(CStoreColspaceCache, "CStore Column Space Cache"); + if (CStoreColspaceCache != NULL) {// 如果CStoreColspaceCache不为空 + HeapMemResetHash(CStoreColspaceCache, "CStore Column Space Cache");// 重置CStore列空间缓存的哈希表 } } @@ -81,29 +85,32 @@ CStoreAllocator::CStoreAllocator() CStoreAllocator::~CStoreAllocator() {} - +//获取下一个可用的CUID（Column Unit ID） uint32 CStoreAllocator::GetNextCUID(Relation rel) { bool found = false; CStoreColFileDesc* entry = NULL; uint32 cuid = InValidCUID; - CStoreColumnFileTag tag(rel->rd_node, VirtualSpaceCacheColID, MAIN_FORKNUM); + CStoreColumnFileTag tag(rel->rd_node, VirtualSpaceCacheColID, MAIN_FORKNUM);// 创建CStore列文件标记 - (void)LWLockAcquire(CStoreColspaceCacheLock, LW_EXCLUSIVE); + (void)LWLockAcquire(CStoreColspaceCacheLock, LW_EXCLUSIVE);// 获取CStore列空间缓存锁 + // 在CStore列空间缓存中查找指定的CStore列文件标记，并返回对应的哈希表项 entry = (CStoreColFileDesc*)hash_search(CStoreColspaceCache, (void*)&tag, HASH_FIND, &found); + // 断言找到了指定的哈希表项 Assert(found); + // 获取CStore列文件的最大CUID cuid = entry->maxCuid; - if (cuid == MaxCUID) + if (cuid == MaxCUID)// 如果没有剩余的CUID可用 ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("No CUID is left for new CU in relation \"%u\". Please execute the VACUUM FULL before do " "anything else", rel->rd_id))); - if (cuid > CUIDWarningThreshold && cuid < MaxCUID) + if (cuid > CUIDWarningThreshold && cuid < MaxCUID)// 如果CUID接近极限值 ereport(WARNING, (errmsg("CUID is almost to be used up in relation \"%u\"", rel->rd_id))); - entry->maxCuid++; - LWLockRelease(CStoreColspaceCacheLock); - return cuid; + entry->maxCuid++;// 将CStore列文件的最大CUID增加1 + LWLockRelease(CStoreColspaceCacheLock);// 释放CStore列空间缓存锁 + return cuid;// 返回获取的CUID值 } /* @@ -114,21 +121,31 @@ uint32 CStoreAllocator::GetNextCUID(Relation rel) * @Return:0 -- no need extend, others extend size * @See also: */ +/* + * @Description: 根据已分配空间的情况计算扩展大小 + * @Param[IN] cu_offset: 最大CU偏移量 + * @Param[IN] cu_size: 写入的CU大小 + * @Param[IN] extend_offset: 记录的扩展偏移量 + * @Return: 0 -- 不需要扩展，其他值表示需要扩展的大小 + * @See also: + */ +//根据已分配空间的情况计算扩展大小 uint32 CStoreAllocator::CalcExtendSize(uint64 cu_offset, uint32 cu_size, uint64 extend_offset) { - uint32 extend_segment = (uint32)(u_sess->attr.attr_storage.fast_extend_file_size * 1024LL); - uint32 extend_size = 0; - uint32 need_file_size = 0; - uint32 left_extend_size = 0; + uint32 extend_segment = (uint32)(u_sess->attr.attr_storage.fast_extend_file_size * 1024LL);// 定义扩展段大小（以字节为单位） + uint32 extend_size = 0;// 初始化扩展大小为0 + uint32 need_file_size = 0;// 初始化需求文件大小为0 + uint32 left_extend_size = 0;// 初始化剩余扩展大小为0 - Assert(cu_offset <= extend_offset); - left_extend_size = extend_offset - cu_offset; + Assert(cu_offset <= extend_offset);// 断言最大CU偏移量小于等于记录的扩展偏移量 + left_extend_size = extend_offset - cu_offset;// 计算剩余扩展大小 if (cu_size <= left_extend_size) { - return 0; // no need fast entend + return 0; // no need fast entend // 不需要快速扩展 } - need_file_size = cu_size - left_extend_size; + need_file_size = cu_size - left_extend_size;// 计算需求文件大小 + // 若需求文件大小小于等于扩展段大小，则直接扩展到扩展段大小；否则，计算合适的扩展大小 if (need_file_size <= extend_segment) { extend_size = extend_segment; } else { @@ -136,7 +153,7 @@ uint32 CStoreAllocator::CalcExtendSize(uint64 cu_offset, uint32 cu_size, uint64 extend_size = need_file_size + extend_segment - remainder; } - return extend_size; + return extend_size;// 返回计算得到的扩展大小 } /* @@ -145,20 +162,32 @@ uint32 CStoreAllocator::CalcExtendSize(uint64 cu_offset, uint32 cu_size, uint64 * @Param[IN] size: cu size * @See also: */ +/* + * @Description: 分配文件空间 + * @Param[IN] cnode: 文件节点 + * @Param[IN] extend_offset: CU指针 + * @Param[IN] cu_offset: 最大CU偏移量 + * @Param[IN] cu_size：写入的CU大小 + * @Return: 实际分配的文件空间大小 + * @See also: + */ +//分配文件空间 uint32 CStoreAllocator::AcquireFileSpace(const CFileNode& cnode, uint64 extend_offset, uint64 cu_offset, uint32 cu_size) { uint32 extend_size = 0; CUStorage* cuStorage = New(CurrentMemoryContext) CUStorage(cnode); - + // 根据配置参数external_enable决定使用libaio还是pread()/pwrite()来读写文件 ADIO_RUN() { if (u_sess->attr.attr_sql.enable_fast_allocate) { + // 如果开启了快速内存分配，则调用CalcExtendSize函数计算扩展大小 extend_size = CStoreAllocator::CalcExtendSize(cu_offset, (uint32)cu_size, extend_offset); if (extend_size != 0) { - cuStorage->FastExtendFile(extend_offset, extend_size, true); + cuStorage->FastExtendFile(extend_offset, extend_size, true);// 先进行快速扩展 } - cuStorage->FastExtendFile(cu_offset, cu_size, false); + cuStorage->FastExtendFile(cu_offset, cu_size, false);// 再写入数据 } else { + // 如果没有开启快速内存分配，则使用palloc0函数分配内存，并将分配的空间清零。然后将数据写入该空间，最后释放内存。 char* buffer = (char*)adio_align_alloc(cu_size); errno_t rc = memset_s(buffer, cu_size, 0, cu_size); securec_check(rc, "\0", "\0"); @@ -169,6 +198,7 @@ uint32 CStoreAllocator::AcquireFileSpace(const CFileNode& cnode, uint64 extend_o } ADIO_ELSE() { + // 如果external_enable为false，则使用pread()/pwrite()进行文件读写 char* buffer = (char*)palloc0(cu_size); cuStorage->SaveCU(buffer, cu_offset, cu_size, false, true); pfree(buffer); @@ -177,11 +207,11 @@ uint32 CStoreAllocator::AcquireFileSpace(const CFileNode& cnode, uint64 extend_o } ADIO_END(); - DELETE_EX(cuStorage); + DELETE_EX(cuStorage);// 删除CUStorage对象，释放内存 - return extend_size; + return extend_size;// 返回实际分配的文件空间大小 } - +//为一个列存储文件节点（cnode）分配大小为size的空间，返回分配的空间在文件中的偏移量 uint64 CStoreAllocator::AcquireSpace(const CFileNode& cnode, Size size, int align_size) { Assert(align_size > 0); @@ -190,29 +220,31 @@ uint64 CStoreAllocator::AcquireSpace(const CFileNode& cnode, Size size, int alig uint64 offset = InvalidCStoreOffset; uint32 extend_size = 0; - LWLockAcquire(CStoreColspaceCacheLock, LW_EXCLUSIVE); + LWLockAcquire(CStoreColspaceCacheLock, LW_EXCLUSIVE);// 获取互斥锁 + /* 在哈希表中查找文件节点 */ entry = (CStoreColFileDesc*)hash_search(CStoreColspaceCache, (const void*)&cnode, HASH_FIND, &found); Assert(found); if (found) { offset = entry->maxOffset; + // 当升级时，最后一个CU需要添加填充。因此，cu_point必须向后对齐 // when upgrade, last cu need add padding. so cu_point must align backward int remainder = offset % align_size; if (remainder != 0) { ereport(WARNING, (errmsg("AcquireSpace: find un align size(%lu)", offset))); - offset = offset + align_size - remainder; - entry->maxOffset = offset; + offset = offset + align_size - remainder;// 对齐offset + entry->maxOffset = offset;// 更新maxOffset } - + // 必须在更新entry之前完成快速扩展，因为不能留下空洞 // must finish fast extend here before update, because we can not leave hole extend_size = CStoreAllocator::AcquireFileSpace(cnode, entry->extendOffset, entry->maxOffset, size); - entry->maxOffset += size; - entry->extendOffset += extend_size; + entry->maxOffset += size;// 更新maxOffset + entry->extendOffset += extend_size;// 更新maxOffset } - LWLockRelease(CStoreColspaceCacheLock); - return offset; + LWLockRelease(CStoreColspaceCacheLock);// 释放互斥锁 + return offset;// 返回offset } - +//从自由空间映射（fsm）中尝试获取大小为size的空间，并进行对齐，返回分配的空间在文件中的偏移量 uint64 CStoreAllocator::TryAcquireSpaceFromFSM(CStoreFreeSpace* fsm, Size size, int align_size) { CStoreFreeSpaceDesc desc; @@ -220,38 +252,41 @@ uint64 CStoreAllocator::TryAcquireSpaceFromFSM(CStoreFreeSpace* fsm, Size size, Assert(fsm != NULL); Assert(align_size > 0); + // 检查自由空间映射（fsm）是否有足够的空间来满足需求 if (!fsm->HasEnoughSpace(size + align_size)) - return offset; - + return offset;// 如果没有足够的空间，返回InvalidCStoreOffset表示分配失败 + // 从自由空间映射（fsm）中弹出大小最大的空闲块描述符（desc） fsm->PopDescWithMaxSize(desc); - offset = desc.beginOffset; + offset = desc.beginOffset;// 获取该空闲块的起始偏移量 // when upgrade, last cu need add padding. so cu_point must align backward + // 当进行升级时，最后一个CU需要添加填充。因此，cu_point必须向后对齐 int remainder = offset % align_size; if (remainder != 0) { ereport(WARNING, (errmsg("TryAcquireSpaceFromFSM: find un align size(%lu)", offset))); - offset = offset + align_size - remainder; - desc.beginOffset = offset; - desc.size -= remainder; + offset = offset + align_size - remainder;// 对齐offset + desc.beginOffset = offset;// 更新desc的起始偏移量 + desc.size -= remainder;// 更新desc的大小 } - + // 更新desc的起始偏移量和大小 desc.beginOffset += size; desc.size -= size; + // 如果仍然有剩余空间，将剩余空间的描述符压入自由空间映射（fsm）中 if (desc.size > 0) fsm->Push(desc); - return offset; + return offset;// 返回分配的空间在文件中的偏移量 } - +//为了获取空间而锁定关系（rel） void CStoreAllocator::LockRelForAcquireSpace(Relation rel) { - LockRelationForExtension(rel, ExclusiveLock); + LockRelationForExtension(rel, ExclusiveLock);//使用独占锁（ExclusiveLock）来锁定关系 } - +//为了释放获取空间而锁定的关系（rel） void CStoreAllocator::ReleaseRelForAcquireSpace(Relation rel) { - UnlockRelationForExtension(rel, ExclusiveLock); + UnlockRelationForExtension(rel, ExclusiveLock);//使用独占锁（ExclusiveLock）来释放关系的锁 } - +//无效化列空间缓存（CStoreColspaceCache）中与给定cnode相关的缓存条目 void CStoreAllocator::InvalidColSpaceCache(const CFileNode& cnode) { LWLockAcquire(CStoreColspaceCacheLock, LW_EXCLUSIVE); @@ -260,34 +295,44 @@ void CStoreAllocator::InvalidColSpaceCache(const CFileNode& cnode) } // build space cache for attrno[ attrNum ]. +// 函数作用：为给定关系（`heapRel`）和属性编号数组（`attrIds`）中的属性创建列空间缓存。 +// 缓存是针对每个属性单独创建的，并且使用锁定方式保证同步性。 +// 如果缓存已经存在，则不会重复创建。 +// 在构建缓存之前，该函数会锁定关系，以防止插入新的元组，并找到最大的CU ID（`maxCUID`）和CU指针的偏移量（`offset`）。 +// 如果存在Gin/BTree索引，还需要查找存储在BTree索引中的最大CU ID（`maxIdxCUID`），并将其与maxCUID比较，选择最大的值。 +// 使用这些信息，缓存将被创建，然后释放关系锁。 +// 最后，释放分配的内存。 void CStoreAllocator::BuildColSpaceCacheForRel(_in_ Relation heapRel, _in_ AttrNumber* attrIds, // equal to attrno[] _in_ int attrNum, _in_ List* indexRel) { + // 创建文件节点结构的数组 CFileNode* cFileNode = (CFileNode*)palloc(sizeof(CFileNode) * attrNum); + // 根据给定的属性编号构造cFileNode数组中每个属性的文件节点 for (int i = 0; i < attrNum; ++i) { cFileNode[i].m_rnode = heapRel->rd_node; cFileNode[i].m_forkNum = MAIN_FORKNUM; cFileNode[i].m_attid = attrIds[i]; } - + // 如果缓存不存在，就需要构建缓存 if (!CStoreAllocator::ColSpaceCacheExist(cFileNode, attrNum)) { uint64* offset = (uint64*)palloc(sizeof(uint64) * attrNum); // it's very important to make maxCUID and all the maxCUPointers the newest and biggest. // so lock and forbit this relation inserting new tuples, see also SaveAll() method. - // + // 防止关系在创建缓存期间插入新的元组，使用独占锁 LockRelationForExtension(heapRel, ExclusiveLock); - + // 获取最大CU ID Oid cudesOid = heapRel->rd_rel->relcudescrelid; uint32 maxCUID = CStore::GetMaxCUID(cudesOid, heapRel->rd_att) + 1; - + // 如果存在Gin/BTree索引，则获取BTree索引中的最大CU ID，用于比较选择最大的CU ID /* If there is gin/btree index, we need to find the biggest CU ID stored in the btree index. */ if (indexRel != NULL) { uint32 maxIdxCUID = CStore::GetMaxIndexCUID(heapRel, indexRel) + 1; if (maxIdxCUID > maxCUID) maxCUID = maxIdxCUID; } + // 查找每个属性的CU指针偏移量 for (int i = 0; i < attrNum; ++i) { if (!heapRel->rd_att->attrs[i]->attisdropped) { offset[i] = CStore::GetMaxCUPointer(attrIds[i], heapRel); @@ -295,13 +340,14 @@ void CStoreAllocator::BuildColSpaceCacheForRel(_in_ Relation heapRel, offset[i] = 0; } } + // 创建缓存 CStoreAllocator::BuildColSpaceCacheForRel(cFileNode, attrNum, offset, maxCUID); - + // 释放关系锁 UnlockRelationForExtension(heapRel, ExclusiveLock); - + // 释放分配的内存 pfree_ext(offset); } - + // 释放分配的内存 pfree_ext(cFileNode); } @@ -311,17 +357,27 @@ void CStoreAllocator::BuildColSpaceCacheForRel(_in_ Relation heapRel, * @Return: extend offset * @See also: */ +/* + * @Description: 计算快速扩展的偏移量 + * @Param[IN] max_offset: 文件的最大CU指针 + * @Return: 扩展的偏移量 + * @See also: + */ +//根据文件的最大CU指针，计算快速扩展的偏移量。偏移量是按照预定义的段大小对齐的，以提高存储空间的利用率 uint64 CStoreAllocator::GetExtendOffset(uint64 max_offset) { + // 获取快速扩展的段大小（单位：字节） int extend_segment = (int)(u_sess->attr.attr_storage.fast_extend_file_size * 1024LL); + // 计算当前偏移量所在的段的起始偏移量 uint64 offset = CU_FILE_OFFSET(max_offset); uint64 remainder = offset % extend_segment; if (remainder != 0) { + // 如果当前偏移量不是段的起始位置，则计算下一个段的起始偏移量 max_offset = max_offset + extend_segment - remainder; } return max_offset; } - +//构建列空间的全局缓存，将列对应的文件偏移量写入缓存，以提供分配空间时的参考 void CStoreAllocator::BuildColSpaceCacheForRel(const CFileNode* cnodes, int nColumn, uint64* offsets, uint32 maxCUID) { CFileNode tag(cnodes[0].m_rnode, VirtualSpaceCacheColID, MAIN_FORKNUM); @@ -334,6 +390,10 @@ void CStoreAllocator::BuildColSpaceCacheForRel(const CFileNode* cnodes, int nCol // If not, update them. // If yes, skip. // + // 构建列空间的全局缓存 + // 将列对应的文件偏移量写入缓存，以提供分配空间时的参考 + // 缓存中记录的是每个列的最大文件偏移量、对应的最大CU指针和扩展偏移量 + // entry = (CStoreColFileDesc*)hash_search(CStoreColspaceCache, (void*)&tag, HASH_ENTER, &found); if (entry == NULL) ereport(PANIC, (errmsg("build global column space cache hash table failed"))); @@ -341,6 +401,9 @@ void CStoreAllocator::BuildColSpaceCacheForRel(const CFileNode* cnodes, int nCol // Other session has insert some columns or all columns into hash table // !!!Note that we reuse variable 'found' // + // 判断是否需要更新缓存 + // 如果存在已经缓存的数据，则需要检查其是否已经完整 + // if (found) { for (int i = 0; i < nColumn; i++) { hash_search(CStoreColspaceCache, (void*)&cnodes[i], HASH_FIND, &found); @@ -348,15 +411,18 @@ void CStoreAllocator::BuildColSpaceCacheForRel(const CFileNode* cnodes, int nCol // Other session has insert some columns // It is incomplete // + // 如果某个列的空间信息不存在，则代表其他进程还没有往缓存中插入 + // 全部信息不完整，需要更新缓存 + // if (!found) break; } } - + // 如果缓存中没有列对应的空间信息，则需要更新缓存 if (!found) { entry->maxCuid = maxCUID; entry->maxOffset = InvalidCStoreOffset; - + // 逐一将每个列的空间信息填入缓存 for (int i = 0; i < nColumn; i++) { entry = (CStoreColFileDesc*)hash_search(CStoreColspaceCache, (void*)&cnodes[i], HASH_ENTER, NULL); if (entry == NULL) @@ -369,19 +435,22 @@ void CStoreAllocator::BuildColSpaceCacheForRel(const CFileNode* cnodes, int nCol LWLockRelease(CStoreColspaceCacheLock); } - +//用于检查给定的列空间信息是否存在于缓存中 bool CStoreAllocator::ColSpaceCacheExist(const CFileNode* cnodes, int nColumn) { bool found = false; - + // 获取共享锁，以防止其他线程对缓存进行修改 LWLockAcquire(CStoreColspaceCacheLock, LW_SHARED); + // 遍历每个列的CFileNode，查找其在缓存中是否存在 + // 如果某个列的空间信息不存在，则found为false，退出循环 for (int i = 0; i < nColumn; i++) { hash_search(CStoreColspaceCache, (void*)&cnodes[i], HASH_FIND, &found); if (!found) break; } - + // 释放共享锁 LWLockRelease(CStoreColspaceCacheLock); + // 返回是否所有列的空间信息都存在于缓存中的布尔值 return found; } @@ -390,29 +459,31 @@ bool CStoreAllocator::ColSpaceCacheExist(const CFileNode* cnodes, int nColumn) * here we recheck max cuid located in index * we want to make sure if there is on another larger cuid in index */ +//用于重新检查最大的cuid值 uint32 CStoreAllocator::recheck_max_cuid(Relation m_rel, uint32 max_cuid, int index_num, Relation* m_idxRelation) { bool find = false; List* index_rel_list = NIL; - + // 筛选出索引类型为B树或GIN的关联关系，将其添加到索引关联关系列表中 for (int i = 0; i < index_num; ++i) { Oid am_oid = m_idxRelation[i]->rd_rel->relam; if (am_oid == CBTREE_AM_OID || am_oid == CGIN_AM_OID) { index_rel_list = lappend(index_rel_list, m_idxRelation[i]); } } - + // 若索引关联关系列表为空，则直接返回原始的最大cuid值 if (list_length(index_rel_list) == 0) { return max_cuid; } + // 获取索引关联关系列表中的最大索引cuid，并在释放列表内存后返回 uint32 max_idx_cuid = CStore::GetMaxIndexCUID(m_rel, index_rel_list) + 1; list_free_ext(index_rel_list); - + // 如果最大索引cuid等于MaxCUID，表示没有剩余的cuid可供新的CU使用，报错 if (max_idx_cuid == MaxCUID) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("No CUID is left for new CU in relation \"%u\".", m_rel->rd_id))); } - + // 如果最大索引cuid大于原始的最大cuid值，更新缓存中对应的列文件描述项的最大cuid，并返回最大索引cuid if (max_idx_cuid > max_cuid) { CStoreColFileDesc* entry = NULL; CStoreColumnFileTag tag(m_rel->rd_node, VirtualSpaceCacheColID, MAIN_FORKNUM); @@ -421,104 +492,107 @@ uint32 CStoreAllocator::recheck_max_cuid(Relation m_rel, uint32 max_cuid, int in Assert(find); entry->maxCuid = max_idx_cuid + 1; LWLockRelease(CStoreColspaceCacheLock); - return max_idx_cuid; + return max_idx_cuid;// 如果最大索引cuid不大于原始的最大cuid值，则直接返回原始的最大cuid值 } return max_cuid; } - +//初始化CStoreFreeSpace对象 void CStoreFreeSpace::Initialize(int maxSize) { - m_maxSize = maxSize; - m_descNum = 0; - m_descs = (CStoreFreeSpaceDesc*)palloc0(sizeof(CStoreFreeSpaceDesc) * (m_maxSize + 1)); + m_maxSize = maxSize;// 设置最大尺寸 + m_descNum = 0;// 描述项数量初始化为0 + m_descs = (CStoreFreeSpaceDesc*)palloc0(sizeof(CStoreFreeSpaceDesc) * (m_maxSize + 1));// 分配描述项数组内存 +} } CStoreFreeSpace::~CStoreFreeSpace() { - m_descs = NULL; + m_descs = NULL;// 将描述项数组指针置空 } - +//销毁CStoreFreeSpace对象 void CStoreFreeSpace::Destroy() { - pfree(m_descs); - m_descs = NULL; + pfree(m_descs);// 释放描述项数组内存 + m_descs = NULL;// 将描述项数组指针置空 } - +//向CStoreFreeSpace对象的描述项数组中压入一个新的描述项 void CStoreFreeSpace::Push(const CStoreFreeSpaceDesc& desc) { int i; - if (m_maxSize == m_descNum) + if (m_maxSize == m_descNum)// 描述项数量已达到最大尺寸，无法继续添加 return; - i = ++m_descNum; - while (i != 1 && desc.size > m_descs[i / 2].size) { - m_descs[i] = m_descs[i / 2]; - i /= 2; + i = ++m_descNum;// 描述项数量加1，并将当前位置索引赋值给变量i + while (i != 1 && desc.size > m_descs[i / 2].size) {// 描述项的尺寸比父节点的尺寸大，进行上移操作 + m_descs[i] = m_descs[i / 2];// 将父节点的描述项下移到当前位置 + i /= 2;// 更新索引为父节点的索引 } - m_descs[i] = desc; + m_descs[i] = desc;// 将待插入的描述项存放到最终位置 } - +//用于弹出具有最大尺寸的描述项，并将其赋值给传入的参数desc void CStoreFreeSpace::PopDescWithMaxSize(CStoreFreeSpaceDesc& desc) { CStoreFreeSpaceDesc tmp; int i = 1; int subi = 2; - if (m_descNum == 0) + if (m_descNum == 0)// 描述项数量为0，无法弹出 return; - desc = m_descs[1]; - tmp = m_descs[m_descNum--]; + desc = m_descs[1];// 将根节点的描述项赋值给传入的参数desc + tmp = m_descs[m_descNum--];// 将最后一个描述项赋值给临时变量tmp，并将描述项数量减1 - while (subi <= m_descNum) { - if (subi < m_descNum && m_descs[subi].size < m_descs[subi + 1].size) + while (subi <= m_descNum) {// 子节点索引未超出描述项数组范围 + if (subi < m_descNum && m_descs[subi].size < m_descs[subi + 1].size)// 右子节点的尺寸更大，选择右子节点 subi++; - if (tmp.size >= m_descs[subi].size) + if (tmp.size >= m_descs[subi].size)// 临时描述项的尺寸大于等于子节点的尺寸，退出循环 break; - m_descs[i] = m_descs[subi]; - i = subi; - subi *= 2; + m_descs[i] = m_descs[subi];// 将子节点的描述项上移到当前位置 + i = subi;// 更新索引为子节点索引 + subi *= 2;// 计算下一个子节点的索引 } - m_descs[i] = tmp; + m_descs[i] = tmp;// 将临时描述项存放到最终确定的位置 } - +//用于获取具有最大尺寸的描述项，并将其赋值给传入的参数desc void CStoreFreeSpace::GetDescWithMaxSize(_out_ CStoreFreeSpaceDesc& desc) { - if (m_descNum == 0) + if (m_descNum == 0)// 如果描述项数量为0，则说明空间已满，返回size最大值 desc.size = ~0; - else + else// 否则，获取具有最大尺寸的描述项，并赋值给传入的参数desc desc = m_descs[1]; } - +//用于判断空闲空间是否足够放下指定大小的数据块。如果当前为空闲空间 bool CStoreFreeSpace::HasEnoughSpace(Size size) { - return IsEmpty() ? false : size <= m_descs[1].size; + return IsEmpty() ? false : size <= m_descs[1].size;// 判断空闲空间是否足够放下指定大小的数据块 } // compute free space data for the *attrno* attribute, which // belongs to the relation specified by *cudescHeapRel*. // *cudescIndexRel* used to index-scan. -// +//计算指定关系中指定属性的空闲空间数据 void CStoreFreeSpace::ComputeFreeSpace( _in_ AttrNumber attrno, _in_ Relation cudescHeapRel, _in_ Relation cudescIndexRel, __inout CStoreFreeSpace* fspace) { - bool isnull = false; - List* beginOffsetOrderedList = NIL; - ListCell* currCell = NULL; - ListCell* prevCell = NULL; - ListCell* nextCell = NULL; - TupleDesc cudescTupDesc = RelationGetDescr(cudescHeapRel); + bool isnull = false;// 是否为NULL值 + List* beginOffsetOrderedList = NIL;// 按beginoffset排序的列表 + ListCell* currCell = NULL;// 当前列表项 + ListCell* prevCell = NULL;// 前一个列表项 + ListCell* nextCell = NULL;// 后一个列表项 + TupleDesc cudescTupDesc = RelationGetDescr(cudescHeapRel);// CUDesc表的元组描述符 #ifdef USE_ASSERT_CHECKING - List* tupList = NIL; + List* tupList = NIL;// 用于断言检查的元组列表 #endif // Setup scan key to fetch from the index by col_id. + // 设置扫描键，按照col_id从索引中获取数据 ScanKeyData key; ScanKeyInit(&key, (AttrNumber)CUDescColIDAttr, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(attrno)); // DIRTY snapshot will be used so that we can get the newest data. + // 使用DIRTY快照初始化扫描描述符 SnapshotData SnapshotDirty; InitDirtySnapshot(SnapshotDirty); SysScanDesc cudescScan = systable_beginscan_ordered(cudescHeapRel, cudescIndexRel, &SnapshotDirty, 1, &key); @@ -531,6 +605,7 @@ void CStoreFreeSpace::ComputeFreeSpace( // Note: if the number of holds in the column > MaxNumOfHoleFSM, we should // give up the scan, and go back to the 'APPEND_ONLY'. // + //从CUDesc表中扫描列的CUDesc，并按beginoffset排序放入列表中，然后合并相邻的CUDesc HeapTuple tup = NULL; while ((tup = systable_getnext_ordered(cudescScan, BackwardScanDirection)) != NULL) { CStoreSpaceDesc spaceDesc; @@ -538,6 +613,7 @@ void CStoreFreeSpace::ComputeFreeSpace( char* cuPointer = DatumGetPointer(fastgetattr(tup, CUDescCUPointerAttr, cudescTupDesc, &isnull)); // skip cuPointer is null + // 跳过cuPointer为NULL的情况 if (isnull) continue; Assert(cuPointer); @@ -546,6 +622,7 @@ void CStoreFreeSpace::ComputeFreeSpace( spaceDesc.size = DatumGetInt32(fastgetattr(tup, CUDescSizeAttr, cudescTupDesc, &isnull)); // skip those special CUs with total NULL or the SAME value. + // 跳过特殊情况下size为0的CUDesc if (spaceDesc.size == 0) continue; @@ -555,6 +632,7 @@ void CStoreFreeSpace::ComputeFreeSpace( #endif // try to merge the descs. + // 尝试合并CUDesc for (currCell = list_head(beginOffsetOrderedList), prevCell = NULL; currCell != NULL; currCell = nextCell) { CStoreSpaceDesc* curEntry = (CStoreSpaceDesc*)lfirst(currCell); nextCell = lnext(currCell); @@ -575,10 +653,11 @@ void CStoreFreeSpace::ComputeFreeSpace( } // mark spaceDesc as handled. + // 标记spaceDesc已处理 spaceDesc.beginOffset = InvalidCStoreOffset; break; } - + // 使用插入排序 // do Insertion sort if (spaceDesc.beginOffset < curEntry->beginOffset) { // if |-- spaceDesc --|-- curEntry --| @@ -597,10 +676,11 @@ void CStoreFreeSpace::ComputeFreeSpace( } // check the size of beginOffsetOrderedList to avoid too many space + // 检查beginOffsetOrderedList的大小，避免空间过多 if (list_length(beginOffsetOrderedList) > MaxNumOfHoleFSM) goto scan_end; } - + // 标记spaceDesc已处理 // mark spaceDesc as handled. spaceDesc.beginOffset = InvalidCStoreOffset; break; @@ -609,7 +689,7 @@ void CStoreFreeSpace::ComputeFreeSpace( Assert(spaceDesc.beginOffset > curEntry->beginOffset + curEntry->size); prevCell = currCell; } - + // 在此处将spaceDesc追加到列表末尾 // by here, we should append the spaceDesc at the end of list. if (spaceDesc.beginOffset != InvalidCStoreOffset) { CStoreSpaceDesc* newEntry = (CStoreSpaceDesc*)palloc0(sizeof(CStoreSpaceDesc)); @@ -624,6 +704,8 @@ void CStoreFreeSpace::ComputeFreeSpace( // Step2 : Calculate the space hole of the column // Check if there's a hole at the begin of Column. + // 步骤2：计算列的空洞 + // 检查列的开头是否存在空洞 currCell = list_head(beginOffsetOrderedList); if (currCell != NULL) { CStoreSpaceDesc* entry = (CStoreSpaceDesc*)lfirst(currCell); @@ -634,7 +716,7 @@ void CStoreFreeSpace::ComputeFreeSpace( fspace->Push(desc); } } - + // 计算列的空洞 // Calculate the space hole of the column for (currCell = list_head(beginOffsetOrderedList); currCell != NULL; currCell = nextCell) { CStoreSpaceDesc *curEntry = NULL, *nextEntry = NULL; @@ -660,13 +742,13 @@ void CStoreFreeSpace::ComputeFreeSpace( scan_end: #ifdef USE_ASSERT_CHECKING - list_free_deep(tupList); + list_free_deep(tupList);// 释放断言检查用的元组列表 tupList = NIL; #endif - list_free_deep(beginOffsetOrderedList); + list_free_deep(beginOffsetOrderedList);// 释放排序后的列表 beginOffsetOrderedList = NIL; - systable_endscan_ordered(cudescScan); + systable_endscan_ordered(cudescScan);// 结束扫描 cudescScan = NULL; } -- 2.34.1 From d31b22c465ed21d09db2a658c220ec124e5401a8 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Wed, 4 Oct 2023 05:19:56 +0800 Subject: [PATCH 38/48] Update cstore_delete.cpp --- .../storage/cstore/cstore_delete.cpp | 391 +++++++++++------- 1 file changed, 237 insertions(+), 154 deletions(-) diff --git a/src/gausskernel/storage/cstore/cstore_delete.cpp b/src/gausskernel/storage/cstore/cstore_delete.cpp index 7575c6360..2864bcd38 100644 --- a/src/gausskernel/storage/cstore/cstore_delete.cpp +++ b/src/gausskernel/storage/cstore/cstore_delete.cpp @@ -49,52 +49,55 @@ #define SORT_COL_TID 2 #define ARRAY_2_LEN 2 - +//初始化CStoreDelete对象的成员变量 CStoreDelete::CStoreDelete( _in_ Relation rel, _in_ EState* estate, _in_ bool is_update_cu, _in_ Plan* plan, _in_ MemInfoArg* ArgmemInfo) - : m_deltaRealtion(NULL), - m_partDeltaOids(NIL), - m_estate(estate), - m_sortTupDesc(NULL), - m_deleteSortState(NULL), - m_partidIdx(0), - m_ctidIdx(0), - m_isTupDescCreateBySelf(false), - m_sortBatch(NULL), - m_rowOffset(NULL), - m_curSortedNum(0), - m_totalDeleteNum(0), - m_isRptRepeatTupErrForUpdate(false) + : m_deltaRealtion(NULL),// 初始化m_deltaRealtion为NULL + m_partDeltaOids(NIL),// 初始化m_partDeltaOids为空列表 + m_estate(estate),// 设置m_estate为传入的estate参数 + m_sortTupDesc(NULL),// 初始化m_sortTupDesc为NULL + m_deleteSortState(NULL),// 初始化m_deleteSortState为NULL + m_partidIdx(0),// 初始化m_partidIdx为0 + m_ctidIdx(0),// 初始化m_ctidIdx为0 + m_isTupDescCreateBySelf(false),//初始化m_isTupDescCreateBySelf为false + m_sortBatch(NULL),// 初始化m_sortBatch为NULL + m_rowOffset(NULL),// 初始化m_rowOffset为NULL + m_curSortedNum(0),// 初始化m_curSortedNum为0 + m_totalDeleteNum(0),// 初始化m_totalDeleteNum为0 + m_isRptRepeatTupErrForUpdate(false)// 初始化m_isRptRepeatTupErrForUpdate为false { - m_relation = rel; - m_DelMemInfo = NULL; + m_relation = rel;// 设置m_relation为传入的rel参数 + m_DelMemInfo = NULL;// 初始化m_DelMemInfo为NULL /* get partial sort num */ + // 获取部分排序行数 m_maxSortNum = RelationGetPartialClusterRows(rel); - + // 临时缓冲区 /* temp buffer */ m_rowOffset = (int*)palloc(sizeof(int) * RelMaxFullCuSize); - InitDeleteMemArg(plan, ArgmemInfo); + InitDeleteMemArg(plan, ArgmemInfo);// 初始化删除内存参数 /* function routine */ - bool isPartition = RELATION_IS_PARTITIONED(rel); + bool isPartition = RELATION_IS_PARTITIONED(rel);// 判断表是否为分区表 if (!isPartition) { - m_InitDeleteSortStatePtr = &CStoreDelete::InitDeleteSortStateForTable; - m_ExecDeletePtr = &CStoreDelete::ExecDeleteForTable; - m_deltaRealtion = heap_open(m_relation->rd_rel->reldeltarelid, RowExclusiveLock); + m_InitDeleteSortStatePtr = + &CStoreDelete:: + InitDeleteSortStateForTable; // 设置函数指针m_InitDeleteSortStatePtr为InitDeleteSortStateForTable + m_ExecDeletePtr = &CStoreDelete::ExecDeleteForTable;// 设置函数指针m_ExecDeletePtr为ExecDeleteForTable + m_deltaRealtion = heap_open(m_relation->rd_rel->reldeltarelid, RowExclusiveLock);// 打开主表的delta表 } else { - m_InitDeleteSortStatePtr = &CStoreDelete::InitDeleteSortStateForPartition; - m_ExecDeletePtr = &CStoreDelete::ExecDeleteForPartition; - CollectPartDeltaOids(); + m_InitDeleteSortStatePtr = &CStoreDelete::InitDeleteSortStateForPartition;// 设置函数指针m_InitDeleteSortStatePtr为InitDeleteSortStateForPartition + m_ExecDeletePtr = &CStoreDelete::ExecDeleteForPartition; // 设置函数指针m_ExecDeletePtr为ExecDeleteForPartition + CollectPartDeltaOids();// 收集所有分区的delta表OID } if (RelationIsPAXFormat(rel)) { - m_PutDeleteBatchPtr = &CStoreDelete::PutDeleteBatchForTable; + m_PutDeleteBatchPtr = &CStoreDelete::PutDeleteBatchForTable;// 设置函数指针m_PutDeleteBatchPtr为PutDeleteBatchForTable } else { - m_PutDeleteBatchPtr = &CStoreDelete::PutDeleteBatchForPartition; + m_PutDeleteBatchPtr = &CStoreDelete::PutDeleteBatchForPartition;// 设置函数指针m_PutDeleteBatchPtr为PutDeleteBatchForPartition } /* set update flag */ - m_isUpdate = is_update_cu; + m_isUpdate = is_update_cu;// 设置更新标志位m_isUpdate为传入的is_update_cu参数 } CStoreDelete::~CStoreDelete() @@ -109,58 +112,58 @@ CStoreDelete::~CStoreDelete() m_deltaRealtion = NULL; m_relation = NULL; } - +//销毁CStoreDelete对象，释放相关的内存资源 void CStoreDelete::Destroy() { if (m_sortTupDesc && m_isTupDescCreateBySelf) { - FreeTupleDesc(m_sortTupDesc); + FreeTupleDesc(m_sortTupDesc);// 释放由自身创建的排序元组描述符 m_sortTupDesc = NULL; } if (m_sortBatch) { - pfree(m_sortBatch); + pfree(m_sortBatch);// 释放排序批次内存 m_sortBatch = NULL; } if (m_rowOffset) { - pfree(m_rowOffset); + pfree(m_rowOffset);// 释放行偏移数组内存 m_rowOffset = NULL; } if (m_DelMemInfo) { - pfree_ext(m_DelMemInfo); + pfree_ext(m_DelMemInfo);// 释放删除内存信息 } if (m_deleteSortState) { - batchsort_end(m_deleteSortState); + batchsort_end(m_deleteSortState);// 结束批量排序 m_deleteSortState = NULL; } if (m_deltaRealtion) { - heap_close(m_deltaRealtion, NoLock); + heap_close(m_deltaRealtion, NoLock);// 关闭delta表 m_deltaRealtion = NULL; } if (list_length(m_partDeltaOids) > 0) { - list_free(m_partDeltaOids); + list_free(m_partDeltaOids);// 释放分区delta表OID列表 m_partDeltaOids = NIL; } } - +//用于收集分区表的delta表OID信息，将收集到的OID存储在m_partDeltaOids中 void CStoreDelete::CollectPartDeltaOids() { - Relation pgpartition = NULL; - TableScanDesc scan = NULL; - HeapTuple tuple = NULL; - ScanKeyData keys[ARRAY_2_LEN]; - Form_pg_partition partitionFrom = NULL; + Relation pgpartition = NULL;// 访问PgPartition元组的关系对象 + TableScanDesc scan = NULL;// 访问PgPartition元组的关系对象 + HeapTuple tuple = NULL;// 获取的元组 + ScanKeyData keys[ARRAY_2_LEN];// 扫描键值数组 + Form_pg_partition partitionFrom = NULL;// 声明指向PgPartition元组的指针 /* Process all partitions of this partitiond table */ ScanKeyInit(&keys[0], Anum_pg_partition_parttype, BTEqualStrategyNumber, F_CHAREQ, - CharGetDatum(PART_OBJ_TYPE_TABLE_PARTITION)); + CharGetDatum(PART_OBJ_TYPE_TABLE_PARTITION));// 初始化扫描键值，按类型为表分区和父表ID等于m_relation的条件进行过滤 ScanKeyInit(&keys[1], Anum_pg_partition_parentid, @@ -168,15 +171,15 @@ void CStoreDelete::CollectPartDeltaOids() F_OIDEQ, ObjectIdGetDatum(RelationGetRelid(m_relation))); - pgpartition = heap_open(PartitionRelationId, AccessShareLock); - scan = tableam_scan_begin(pgpartition, SnapshotNow, ARRAY_2_LEN, keys); + pgpartition = heap_open(PartitionRelationId, AccessShareLock);// 打开PgPartition表 + scan = tableam_scan_begin(pgpartition, SnapshotNow, ARRAY_2_LEN, keys);// 开始扫描 - while ((tuple = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL) { - partitionFrom = (Form_pg_partition)GETSTRUCT(tuple); - m_partDeltaOids = lappend_oid(m_partDeltaOids, partitionFrom->reldeltarelid); + while ((tuple = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL) {// 遍历扫描结果 + partitionFrom = (Form_pg_partition)GETSTRUCT(tuple);// 获取PgPartition元组地址，并转换为Form_pg_partition结构体 + m_partDeltaOids = lappend_oid(m_partDeltaOids, partitionFrom->reldeltarelid);// 获取分区delta表OID，并加入到m_partDeltaOids列表末尾 } - tableam_scan_end(scan); - heap_close(pgpartition, AccessShareLock); + tableam_scan_end(scan);// 结束扫描 + heap_close(pgpartition, AccessShareLock);// 关闭PgPartition表 } /* @@ -188,11 +191,20 @@ void CStoreDelete::CollectPartDeltaOids() * @Return: void * @See also: InitInsertMemArg */ +/* + * @Description: 初始化CStoreDelete的删除内存信息。有三个分支：plan是传递给存储层执行的优化器估计参数； + * ArgmemInfo用于将参数从上层传递给删除操作符执行；其他是不受控制的内存。 + * @IN plan: 如果直接使用删除操作符，则将计划的mem_info参数传递给执行。 + * @IN ArgmemInfo: ArgmemInfo用于传递mem_info参数来执行，例如更新操作。 + * @Return: void + * @See also: InitInsertMemArg + */ +//初始化CStoreDelete的删除内存信息 void CStoreDelete::InitDeleteMemArg(Plan* plan, MemInfoArg* ArgmemInfo) { - m_DelMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg)); + m_DelMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg));// 分配并初始化删除内存信息结构体 if (plan != NULL && plan->operatorMemKB[0] > 0) { - m_DelMemInfo->canSpreadmaxMem = plan->operatorMaxMem; + m_DelMemInfo->canSpreadmaxMem = plan->operatorMaxMem;// 设置可以扩散的最大内存 m_DelMemInfo->MemInsert = 0; m_DelMemInfo->MemSort = plan->operatorMemKB[0] ? plan->operatorMemKB[0] : u_sess->attr.attr_storage.psort_work_mem; @@ -203,9 +215,9 @@ void CStoreDelete::InitDeleteMemArg(Plan* plan, MemInfoArg* ArgmemInfo) m_DelMemInfo->MemSort, m_DelMemInfo->canSpreadmaxMem); } else if (ArgmemInfo != NULL) { - m_DelMemInfo->canSpreadmaxMem = ArgmemInfo->canSpreadmaxMem; + m_DelMemInfo->canSpreadmaxMem = ArgmemInfo->canSpreadmaxMem;// 设置可以扩散的最大内存 m_DelMemInfo->MemInsert = ArgmemInfo->MemInsert; - m_DelMemInfo->MemSort = ArgmemInfo->MemSort; + m_DelMemInfo->MemSort = ArgmemInfo->MemSort; // 设置排序内存 m_DelMemInfo->spreadNum = ArgmemInfo->spreadNum; MEMCTL_LOG(DEBUG2, "CStoreDelete(init ArgmemInfo):Insert workmem is : %dKB, sort workmem: %dKB,can spread maxMem is %dKB.", @@ -215,17 +227,19 @@ void CStoreDelete::InitDeleteMemArg(Plan* plan, MemInfoArg* ArgmemInfo) } else { m_DelMemInfo->canSpreadmaxMem = 0; m_DelMemInfo->MemInsert = 0; - m_DelMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem; + m_DelMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem;// 设置排序内存 m_DelMemInfo->spreadNum = 0; } - m_DelMemInfo->partitionNum = 1; + m_DelMemInfo->partitionNum = 1;// 分区数初始化为1 } - +//初始化删除操作符的排序信息 void CStoreDelete::InitSortState() { // init tupledesc, use partid and tid for sort + // 初始化TupleDesc，创建包含两个属性的TupleDesc，分别为partid和tid。 m_sortTupDesc = CreateTemplateTupleDesc(ARRAY_2_LEN, false); m_isTupDescCreateBySelf = true; + // 将partid和tid的列号设置为常量。 TupleDescInitEntry(m_sortTupDesc, SORT_COL_PARTID, "partid", OIDOID, -1, 0); TupleDescInitEntry(m_sortTupDesc, SORT_COL_TID, "tid", TIDOID, -1, 0); @@ -233,32 +247,39 @@ void CStoreDelete::InitSortState() m_ctidIdx = SORT_COL_TID; // sort cache + // 分配内存初始化m_sortBatch，用于进行排序操作时的缓存。 m_sortBatch = New(CurrentMemoryContext) VectorBatch(CurrentMemoryContext, m_sortTupDesc); // init sort state + // 调用m_InitDeleteSortStatePtr成员变量指向的函数来初始化排序状态。 (this->*m_InitDeleteSortStatePtr)(m_sortTupDesc, SORT_COL_PARTID, SORT_COL_TID); } - +//根据传入的参数初始化删除操作符的排序状态，并将m_PutDeleteBatchPtr指向PutDeleteBatchForUpdate函数 void CStoreDelete::InitSortState(TupleDesc sortTupDesc, int partidIdx, int ctidIdx) { + // 将传入的参数赋值给成员变量 m_sortTupDesc = sortTupDesc; m_partidIdx = partidIdx; m_ctidIdx = ctidIdx; // sort cache + // 分配内存初始化m_sortBatch，用于进行排序操作时的缓存。 m_sortBatch = New(CurrentMemoryContext) VectorBatch(CurrentMemoryContext, m_sortTupDesc); // init sort state + // 调用m_InitDeleteSortStatePtr成员变量指向的函数来初始化排序状态。 (this->*m_InitDeleteSortStatePtr)(sortTupDesc, partidIdx, ctidIdx); // change m_PutDeleteBatchPtr + // 更改m_PutDeleteBatchPtr指向的函数为PutDeleteBatchForUpdate。 m_PutDeleteBatchPtr = &CStoreDelete::PutDeleteBatchForUpdate; } - +//用于调用删除操作并判断是否需要进行部分删除 void CStoreDelete::PutDeleteBatch(_in_ VectorBatch* batch, _in_ JunkFilter* junkfilter) { + // 调用m_PutDeleteBatchPtr成员变量指向的函数来执行删除操作。 (this->*m_PutDeleteBatchPtr)(batch, junkfilter); - + // 如果删除操作符已经满了，则进行部分删除。 if (IsFull()) { // execute delete if parital sort is full PartialDelete(); @@ -266,61 +287,68 @@ void CStoreDelete::PutDeleteBatch(_in_ VectorBatch* batch, _in_ JunkFilter* junk return; } - +//执行部分删除和重置排序状态 void CStoreDelete::PartialDelete() { + // 执行部分删除操作，并更新m_totalDeleteNum统计变量。 m_totalDeleteNum += (uint64)(this->*m_ExecDeletePtr)(); - + // 递增命令计数器。 // parital delete need commad id ++ CommandCounterIncrement(); - + // 重置排序状态，为下一轮排序做准备。 // reset sort state ResetSortState(); } - +//执行删除操作并返回删除记录总数 uint64 CStoreDelete::ExecDelete() { + // 执行删除操作，并更新m_totalDeleteNum统计变量。 m_totalDeleteNum += (uint64)(this->*m_ExecDeletePtr)(); - + // 清空虚假关系缓存 // clean partition fake rel cache if (m_estate->esfRelations) { FakeRelationCacheDestroy(m_estate->esfRelations); } - + // 返回删除记录的总数。 return m_totalDeleteNum; } - +//初始化用于表格删除操作的排序状态 void CStoreDelete::InitDeleteSortStateForTable(TupleDesc sortTupDesc, int /* partidAttNo */, int ctidAttNo) { + // 断言sortTupDesc不为空。 Assert(sortTupDesc); + // 断言ctidAttNo小于等于sortTupDesc的属性数量。 Assert(ctidAttNo <= sortTupDesc->natts); - + // 初始化排序需要的参数 const int nkeys = 1; AttrNumber attNums[1]; Oid sortCollations[1]; bool nullsFirstFlags[1]; int SortMem = u_sess->attr.attr_storage.psort_work_mem; int maxMem = 0; - + // 查找TID类型的缓存条目 TypeCacheEntry* typeEntry = lookup_type_cache(TIDOID, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); - + // 设置排序的属性号、排序规则和空值排序规则 attNums[0] = ctidAttNo; sortCollations[0] = sortTupDesc->attrs[ctidAttNo - 1]->attcollation; nullsFirstFlags[0] = false; - + // 根据配置设置排序所需的内存大小和最大内存限制 SortMem = m_DelMemInfo->MemSort > 0 ? m_DelMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem; maxMem = m_DelMemInfo->canSpreadmaxMem > 0 ? m_DelMemInfo->canSpreadmaxMem : 0; - + // 使用batchsort_begin_heap函数开始一个基于堆的批量排序 m_deleteSortState = batchsort_begin_heap( sortTupDesc, nkeys, attNums, &typeEntry->lt_opr, sortCollations, nullsFirstFlags, SortMem, false, maxMem); } - +//为分区删除操作初始化排序状态 void CStoreDelete::InitDeleteSortStateForPartition(TupleDesc sortTupDesc, int partidAttNo, int ctidAttNo) { + // 断言sortTupDesc不为空。 Assert(sortTupDesc); + // 断言partidAttNo小于等于sortTupDesc的属性数量。 Assert(partidAttNo <= sortTupDesc->natts); + // 断言ctidAttNo小于等于sortTupDesc的属性数量。 Assert(ctidAttNo <= sortTupDesc->natts); - + // 设置排序所需的参数 const int nkeys = ARRAY_2_LEN; AttrNumber attNums[ARRAY_2_LEN]; Oid sortOperators[ARRAY_2_LEN]; @@ -328,123 +356,141 @@ void CStoreDelete::InitDeleteSortStateForPartition(TupleDesc sortTupDesc, int pa bool nullsFirstFlags[ARRAY_2_LEN]; int SortMem = u_sess->attr.attr_storage.psort_work_mem; int maxMem = 0; - + // 设置排序的属性号 attNums[0] = partidAttNo; attNums[1] = ctidAttNo; - + // 查找OID类型和TID类型的缓存条目 TypeCacheEntry* typeEntry = lookup_type_cache(OIDOID, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); sortOperators[0] = typeEntry->lt_opr; typeEntry = lookup_type_cache(TIDOID, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); sortOperators[1] = typeEntry->lt_opr; - + // 设置排序的排序规则 sortCollations[0] = sortTupDesc->attrs[partidAttNo - 1]->attcollation; sortCollations[1] = sortTupDesc->attrs[ctidAttNo - 1]->attcollation; - + // 设置排序的空值排序规则 nullsFirstFlags[0] = false; nullsFirstFlags[1] = false; - + // 根据配置设置排序所需的内存大小和最大内存限制 SortMem = m_DelMemInfo->MemSort > 0 ? m_DelMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem; maxMem = m_DelMemInfo->canSpreadmaxMem > 0 ? m_DelMemInfo->canSpreadmaxMem : 0; - + // 使用batchsort_begin_heap函数开始一个基于堆的批量排序 m_deleteSortState = batchsort_begin_heap( m_sortTupDesc, nkeys, attNums, sortOperators, sortCollations, nullsFirstFlags, SortMem, false, maxMem); } - +//将待删除的批次数据放入排序状态中进行排序 void CStoreDelete::PutDeleteBatchForTable(_in_ VectorBatch* batch, _in_ JunkFilter* junkfilter) { + // 断言参数不为空 Assert(batch && junkfilter && m_sortBatch && m_deleteSortState); - + // 获取TID字段的属性索引 int tididx = junkfilter->jf_junkAttNo; Assert(tididx - 1 < batch->m_cols); // shallow copy + // 将待删除的TID字段浅拷贝到排序批次中 m_sortBatch->m_arr[m_ctidIdx - 1].copy(&batch->m_arr[tididx - 1]); - + // 设置排序批次的行数 m_sortBatch->m_rows = batch->m_rows; // put batch + // 将排序批次加入排序状态中 m_deleteSortState->sort_putbatch(m_deleteSortState, m_sortBatch, 0, m_sortBatch->m_rows); - + // 更新已排序的记录数量 m_curSortedNum += batch->m_rows; } - +//将待删除的分区批次数据放入排序状态中进行排序 void CStoreDelete::PutDeleteBatchForPartition(_in_ VectorBatch* batch, _in_ JunkFilter* junkfilter) { + // 断言参数不为空 Assert(batch && junkfilter && m_sortBatch && m_deleteSortState); - + // 获取TID字段和分区字段的属性索引 int tididx = junkfilter->jf_junkAttNo; int partidx = junkfilter->jf_xc_part_id; Assert(tididx - 1 < batch->m_cols); Assert(partidx - 1 < batch->m_cols); // shallow copy + // 将待删除的分区字段和TID字段浅拷贝到排序批次中 m_sortBatch->m_arr[m_partidIdx - 1].copy(&batch->m_arr[partidx - 1]); m_sortBatch->m_arr[m_ctidIdx - 1].copy(&batch->m_arr[tididx - 1]); - + // 设置排序批次的行数 m_sortBatch->m_rows = batch->m_rows; // put batch + // 将排序批次加入排序状态中 m_deleteSortState->sort_putbatch(m_deleteSortState, m_sortBatch, 0, m_sortBatch->m_rows); - + // 更新已排序的记录数量 m_curSortedNum += batch->m_rows; } - +//将待更新的批次数据放入排序状态中进行排序 void CStoreDelete::PutDeleteBatchForUpdate(_in_ VectorBatch* batch, _in_ JunkFilter* /* junkfilter */) { + // 断言参数不为空 Assert(batch && m_sortBatch && m_deleteSortState); // shallow copy all batch data + // 浅拷贝整个批次数据到排序批次中 m_sortBatch->Copy(batch); // put batch + // 将排序批次加入排序状态中 m_deleteSortState->sort_putbatch(m_deleteSortState, m_sortBatch, 0, m_sortBatch->m_rows); + // 更新已排序的记录数量 m_curSortedNum += batch->m_rows; } - +//将待更新批次中指定行的数据放入排序状态中进行排序 void CStoreDelete::PutDeleteBatchForUpdate(_in_ VectorBatch* batch, _in_ int startIdx, _in_ int endIdx) { + // 断言参数不为空 Assert(batch && m_sortBatch && m_deleteSortState); - + /* 拷贝指定行的批次数据 */ /* copy batch data */ m_sortBatch->Copy(batch, startIdx, endIdx); - + /* 将排序批次加入排序状态中 */ /* put batch */ m_deleteSortState->sort_putbatch(m_deleteSortState, m_sortBatch, 0, m_sortBatch->m_rows); + /* 更新已排序的记录数量 */ m_curSortedNum += m_sortBatch->m_rows; } - +//执行表的删除操作 uint64 CStoreDelete::ExecDeleteForTable() { + // 断言参数不为空 Assert(m_relation && m_estate && m_sortBatch && m_deleteSortState && m_rowOffset); - uint32 lastCUID = InValidCUID; - uint32 lastOffset = 0; - int delRowNum = 0; - uint64 delTotalRowNum = 0; - Oid deltaTableid = RelationGetRelid(m_deltaRealtion); + uint32 lastCUID = InValidCUID;// 上一个数据块的CUID + uint32 lastOffset = 0;// 上一个偏移量 + int delRowNum = 0;// 删除的行数 + uint64 delTotalRowNum = 0;// 总共删除的行数 + Oid deltaTableid = RelationGetRelid(m_deltaRealtion);// Delta表的OID - (void)GetCurrentTransactionId(); - - m_sortBatch->Reset(true); + (void)GetCurrentTransactionId();// 获取当前事务ID + m_sortBatch->Reset(true);// 重置排序批次 + // 运行排序 // run sort batchsort_performsort(m_deleteSortState); - + // 获取排序后的批次 batchsort_getbatch(m_deleteSortState, true, m_sortBatch); while (!BatchIsNull(m_sortBatch)) { // get tid col - ScalarValue* tableidValues = m_sortBatch->m_arr[m_partidIdx - 1].m_vals; - ScalarValue* tidValues = m_sortBatch->m_arr[m_ctidIdx - 1].m_vals; - ItemPointer lastRowTid = NULL; + ScalarValue* tableidValues = m_sortBatch->m_arr[m_partidIdx - 1].m_vals;// 表ID列的值 + ScalarValue* tidValues = m_sortBatch->m_arr[m_ctidIdx - 1].m_vals;// TID列的值 + ItemPointer lastRowTid = NULL;// 上一行的TID for (int i = 0; i < m_sortBatch->m_rows; i++) { - ItemPointer tid = (ItemPointer)(tidValues + i); + ItemPointer tid = (ItemPointer)(tidValues + i);// 当前行的TID if (deltaTableid == tableidValues[i]) { /* * for delete, allow replicated tid; * for update, don't allow replicated tid; */ + // 如果是在Delta表上的删除操作 + /* + * 对于删除操作，允许重复的TID； + * 对于更新操作，不允许重复的TID； + */ if (lastRowTid != NULL && ItemPointerEquals(lastRowTid, tid)) { if (m_isRptRepeatTupErrForUpdate) { ereport(ERROR, (errcode(ERRCODE_CARDINALITY_VIOLATION), @@ -454,29 +500,33 @@ uint64 CStoreDelete::ExecDeleteForTable() continue; } - simple_heap_delete(m_deltaRealtion, tid); + simple_heap_delete(m_deltaRealtion, tid);// 在Delta表上执行删除操作 lastRowTid = tid; delTotalRowNum++; continue; } - uint32 curCUID = ItemPointerGetBlockNumber(tid); - uint32 curOffset = ItemPointerGetOffsetNumber(tid) - 1; + uint32 curCUID = ItemPointerGetBlockNumber(tid);// 当前数据块的CUID + uint32 curOffset = ItemPointerGetOffsetNumber(tid) - 1;// 当前偏移量 Assert(IsValidCUID(curCUID)); if (!IsValidCUID(lastCUID)) { + // 第一个数据块 // first CU lastCUID = curCUID; } else if (lastCUID != curCUID) { + // 修改同一数据块中的删除位图 // modify delete bitmap in same CU UpdateVCBitmap(m_relation, lastCUID, m_rowOffset, delRowNum, m_estate->es_snapshot); // switch CU + // 切换到下一个数据块 lastCUID = curCUID; delTotalRowNum += delRowNum; delRowNum = 0; } else if (lastOffset == curOffset) { + // 重复的TID // repeat ctid Assert(lastCUID == curCUID); @@ -490,12 +540,14 @@ uint64 CStoreDelete::ExecDeleteForTable() } // record offset + // 记录偏移量 m_rowOffset[delRowNum++] = curOffset; lastOffset = curOffset; Assert(delRowNum <= DefaultFullCUSize); } // get next sorted batch + // 获取下一个排序后的批次 batchsort_getbatch(m_deleteSortState, true, m_sortBatch); } @@ -506,9 +558,12 @@ uint64 CStoreDelete::ExecDeleteForTable() return delTotalRowNum; } - +/** + * 在分区表中执行删除操作 + */ uint64 CStoreDelete::ExecDeleteForPartition() { + // 断言关系、执行状态、排序批次、删除排序状态和行偏移量都存在 Assert(m_relation && m_estate && m_sortBatch && m_deleteSortState && m_rowOffset); Relation partFakeRel = NULL; @@ -520,30 +575,32 @@ uint64 CStoreDelete::ExecDeleteForPartition() int delRowNum = 0; uint64 delTotalRowNum = 0; - + // 获取当前事务的ID (void)GetCurrentTransactionId(); - + // 重置排序批次 m_sortBatch->Reset(true); - + // 执行排序 // run sort batchsort_performsort(m_deleteSortState); - + // 获取排序后的批次 batchsort_getbatch(m_deleteSortState, true, m_sortBatch); /* * ExecDeleteForPartition may be invoked by CStore update multi times. * We should close the m_deltaRelation opened last time. */ + // 如果之前有打开过的m_deltaRelation，则需要关闭它 if (m_deltaRealtion != NULL) { relation_close(m_deltaRealtion, NoLock); m_deltaRealtion = NULL; } - + // 循环处理排序后的批次，直到批次为空 while (!BatchIsNull(m_sortBatch)) { + // 定义分区ID和行ID的值，并初始化最后一行数据的行ID ScalarValue* partidValues = m_sortBatch->m_arr[m_partidIdx - 1].m_vals; ScalarValue* tidValues = m_sortBatch->m_arr[m_ctidIdx - 1].m_vals; ItemPointer lastRowTid = NULL; - + // 处理每一行数据 for (int i = 0; i < m_sortBatch->m_rows; i++) { Oid curPartID = DatumGetObjectId(*(partidValues + i)); Assert(curPartID != InvalidOid); @@ -551,6 +608,7 @@ uint64 CStoreDelete::ExecDeleteForPartition() ItemPointer tid = (ItemPointer)(tidValues + i); /* for delta table */ + // 如果是分区表中的删除操作，则需要打开对应的delta分区，并关闭之前打开过的delta分区 if (list_member_oid(m_partDeltaOids, curPartID)) { if (lastPartID == InvalidOid) { // first Partition @@ -576,6 +634,7 @@ uint64 CStoreDelete::ExecDeleteForPartition() * for delete, allow replicated tid; * for update, don't allow replicated tid; */ + // 如果行ID与上一行重复，则报错 if (lastRowTid != NULL && ItemPointerEquals(lastRowTid, tid)) { if (m_isRptRepeatTupErrForUpdate) { ereport(ERROR, (errcode(ERRCODE_CARDINALITY_VIOLATION), @@ -584,16 +643,16 @@ uint64 CStoreDelete::ExecDeleteForPartition() } continue; } - + // 执行删除操作并增加删除的行数 simple_heap_delete(m_deltaRealtion, tid); delTotalRowNum++; continue; } - + // 获取当前CU的ID和偏移量 uint32 curCUID = ItemPointerGetBlockNumber(tid); uint32 curOffset = ItemPointerGetOffsetNumber(tid) - 1; Assert(IsValidCUID(curCUID)); - + // 对于非分区表中的删除操作，如果当前处理的分区和CU与上一行不同，则需要修改删除位图 if (lastPartID == InvalidOid) { // first Partition and CU lastPartID = curPartID; @@ -655,46 +714,51 @@ uint64 CStoreDelete::ExecDeleteForPartition() continue; } - + // 记录当前行的偏移量 // record offset m_rowOffset[delRowNum++] = curOffset; lastOffset = curOffset; Assert(delRowNum <= DefaultFullCUSize); } - + // 获取下一个排序批次 // get next sorted batch batchsort_getbatch(m_deleteSortState, true, m_sortBatch); } - + // 处理最后一个CU的删除位图 if (delRowNum > 0) { UpdateVCBitmap(partFakeRel, lastCUID, m_rowOffset, delRowNum, m_estate->es_snapshot); delTotalRowNum += delRowNum; } - + // 返回删除的总行数 return delTotalRowNum; } // ExecDelete // Mark del_bitmap of CUDesc according to vecRowId +// ExecDelete函数用于执行删除操作 void CStoreDelete::ExecDelete(_in_ Relation rel, _in_ ScalarVector* vecRowId, _in_ Snapshot snapshot, _in_ Oid tableOid) { - uint32 lastCUID = InValidCUID; - int delRowNum = 0; - ScalarValue* values = vecRowId->m_vals; - uint32 curCUID, curOffset; - Oid deltaOid = RelationGetRelid(m_deltaRealtion); - ItemPointer lastRowTid = NULL; - GetCurrentTransactionId(); - + uint32 lastCUID = InValidCUID;// 上一行的CU ID，初始设置为无效值 + int delRowNum = 0;// 当前CU中待删除的行数 + ScalarValue* values = vecRowId->m_vals;// 删除行的rowid数组 + uint32 curCUID, curOffset;// 当前行的CU ID和偏移量 + Oid deltaOid = RelationGetRelid(m_deltaRealtion);// delta表的OID + ItemPointer lastRowTid = NULL;// 上一行的TID + GetCurrentTransactionId();// 获取当前事务的ID + // 遍历每一行删除数据 for (int i = 0; i < vecRowId->m_rows; ++i) { - ItemPointer tid = (ItemPointer)(values + i); - + ItemPointer tid = (ItemPointer)(values + i);// 获取当前行的TID + /* 对于delta表 */ /* for delta table */ if (tableOid == deltaOid) { /* * for delete, allow replicated tid; * for update, don't allow replicated tid; */ + /* + * 对于删除操作，允许重复的TID； + * 对于更新操作，不允许重复的TID； + */ if (lastRowTid != NULL && ItemPointerEquals(lastRowTid, tid)) { if (m_isRptRepeatTupErrForUpdate) { ereport(ERROR, (errcode(ERRCODE_CARDINALITY_VIOLATION), @@ -703,28 +767,28 @@ void CStoreDelete::ExecDelete(_in_ Relation rel, _in_ ScalarVector* vecRowId, _i } continue; } - simple_heap_delete(m_deltaRealtion, tid); + simple_heap_delete(m_deltaRealtion, tid);// 执行删除操作 continue; } - curCUID = ItemPointerGetBlockNumber(tid); - curOffset = ItemPointerGetOffsetNumber(tid) - 1; + curCUID = ItemPointerGetBlockNumber(tid);// 获取当前行的CU ID + curOffset = ItemPointerGetOffsetNumber(tid) - 1;// 获取当前行的偏移量 - Assert(IsValidCUID(curCUID)); + Assert(IsValidCUID(curCUID));// 断言当前行的CU ID是有效的 - if (!IsValidCUID(lastCUID)) + if (!IsValidCUID(lastCUID))// 如果上一行的CU ID无效，则设置为当前行的CU ID lastCUID = curCUID; - else if (lastCUID != curCUID) { + else if (lastCUID != curCUID) {// 如果上一行的CU ID与当前行的CU ID不相同，则需要更新删除位图 UpdateVCBitmap(rel, lastCUID, m_rowOffset, delRowNum, snapshot); lastCUID = curCUID; delRowNum = 0; } - m_rowOffset[delRowNum++] = curOffset; + m_rowOffset[delRowNum++] = curOffset;// 记录待删除行的偏移量 } if (delRowNum > 0) { - UpdateVCBitmap(rel, lastCUID, m_rowOffset, delRowNum, snapshot); + UpdateVCBitmap(rel, lastCUID, m_rowOffset, delRowNum, snapshot);// 更新最后一个CU的删除位图 } } @@ -733,10 +797,12 @@ void CStoreDelete::ExecDelete(_in_ Relation rel, _in_ ScalarVector* vecRowId, _i * This function change the bitmap of corresponding CUs. * And update the bitmap of CUs of VC. */ +//更新虚拟列（VC）对应的删除位图 void CStoreDelete::UpdateVCBitmap( _in_ Relation rel, _in_ uint32 cuid, _in_ const int* rowoffset, _in_ int delRowNum, _in_ Snapshot snapshot) { Retry: + // 扫描CUDesc表的两个条件 ScanKeyData key[2]; HeapTuple tmpTup = NULL, oldTup = NULL, newTup = NULL; bool isnull = false; @@ -747,12 +813,13 @@ Retry: /* * Open the CUDesc relation and its index */ + // 打开CUDesc表和其索引 Relation cudesc_rel = heap_open(rel->rd_rel->relcudescrelid, RowExclusiveLock); Relation idx_rel = index_open(cudesc_rel->rd_rel->relcudescidx, RowExclusiveLock); TupleDesc cudesc_tupdesc = cudesc_rel->rd_att; Assert(CUDescMaxAttrNum == cudesc_tupdesc->natts); - + // 初始化扫描CUDesc表时的条件 ScanKeyInit(&key[0], (AttrNumber)CUDescColIDAttr, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(VitrualDelColID)); ScanKeyInit(&key[1], (AttrNumber)CUDescCUIDAttr, BTEqualStrategyNumber, F_OIDEQ, UInt32GetDatum(cuid)); @@ -765,19 +832,20 @@ Retry: // just need get one to deal with, modify while->if 20150508 // If get the old one, it might show 'delete or update row conflict' // If get the new one, it will continue to deal with the new one + // Step 1: 获取并修改删除位图 ItemPointerData oldTupCtid; if ((tmpTup = systable_getnext_ordered(cudesc_scan, ForwardScanDirection)) != NULL) { Assert(newTup == NULL); oldTup = tmpTup; uint32 rowCount = DatumGetUInt32(fastgetattr(oldTup, CUDescRowCountAttr, cudesc_tupdesc, &isnull)); Assert(isnull == false); - + // 获取删除位图 // Get delMask int delMaskBytes = (rowCount + 7) / 8; delMask = (unsigned char*)palloc0(delMaskBytes); char* cuPtr = DatumGetPointer(fastgetattr(oldTup, CUDescCUPointerAttr, cudesc_tupdesc, &isnull)); - + // 当前CU已存在删除位图 // Delbitmap is not null // This CU has deleted rows before if (isnull == false) { @@ -785,7 +853,7 @@ Retry: Assert((int)VARSIZE_ANY_EXHDR(detoastPtr) == delMaskBytes); rc = memcpy_s(delMask, delMaskBytes, VARDATA_ANY(detoastPtr), VARSIZE_ANY_EXHDR(detoastPtr)); securec_check(rc, "", ""); - + // 如果*detoastPtr*指向新的空间，将其释放 // if *detoastPtr* is a new space, we will free it // the first time when it's useless. if (detoastPtr != cuPtr) { @@ -796,11 +864,11 @@ Retry: oldDelMask = (unsigned char*)palloc(delMaskBytes); rc = memcpy_s(oldDelMask, delMaskBytes, delMask, delMaskBytes); securec_check(rc, "", ""); - + // 修改删除位图 // Modify delMask for (int i = 0; i < delRowNum; ++i) { uint32 row = (uint32)rowoffset[i]; - + // 如果该行已被其他事务删除 // This row have been deleted by other transaction if (oldDelMask[row >> 3] & (1 << (row % 8))) { ereport( @@ -811,7 +879,7 @@ Retry: } pfree(oldDelMask); - + // 构造修改后的删除位图对应的新tuple // Form new tuple using new delMask newTup = CStore::FormVCCUDescTup(cudesc_tupdesc, (char*)delMask, cuid, rowCount, GetCurrentTransactionIdIfAny()); @@ -820,7 +888,7 @@ Retry: systable_endscan_ordered(cudesc_scan); index_close(idx_rel, RowExclusiveLock); - + // Step 2: 更新删除位图 // Step 2: update del_bitmap TM_Result result = TM_Invisible; TM_FailureData tmfd; @@ -836,14 +904,15 @@ Retry: true, NULL, &tmfd, - NULL, // we don't need update_indexes - NULL, // we don't meed modifiedIdxAttrs + NULL, // we don't need update_indexes // 不需要更新索引 + NULL, // we don't meed modifiedIdxAttrs // 不需要更新的列 false); switch (result) { case TM_SelfUpdated: case TM_SelfModified: { // Now It is TM_SelfModified + // 更新失败，出现锁冲突 ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), (errmsg("delete or update failed because lock conflict")))); break; @@ -880,6 +949,12 @@ Retry: * so it's possible that no tuple is found druing this search with * SnapshotNow and index scan. once it happens, error reporting. */ + /* + * 由于快照为SnapshotNow，存在以下情况： + * 1. 在BTREE索引正向扫描期间，可能先匹配到新的tuple再匹配到旧的tuple； + * 2. 对于SnapshotNow，事务处理期间xid状态可能从'processing'变为'committed'。 + * 因此，使用SnapshotNow和索引扫描时，有可能找不到任何tuple，如果出现这种情况，则报错。 + */ ereport(ERROR, (errcode(ERRCODE_CARDINALITY_VIOLATION), errmsg("delete or update row conflict"))); } heap_close(cudesc_rel, NoLock); @@ -888,26 +963,34 @@ Retry: * dead_tuple will increase when commit, see * function AtEOXact_PgStat. */ + // 如果不是更新操作，则增加删除行数统计 if (!m_isUpdate) pgstat_count_cu_delete(rel, delRowNum); } - +//判断当前是否已满排序状态 bool CStoreDelete::IsFull() const { // m_maxSortNum <= 0 means full sort + // 判断当前是否已满排序状态 + // 如果m_maxSortNum <= 0，表示无排序限制，因此不满 + // 否则，比较当前已排序的行数m_curSortedNum和最大排序数m_maxSortNum + // 若已排序的行数大于等于最大排序数，则表示已满 return m_maxSortNum <= 0 ? false : m_curSortedNum >= m_maxSortNum; } - +//重置排序状态 void CStoreDelete::ResetSortState() { + // 重置排序状态 + // // 释放之前的排序状态 // release last sort state if (m_deleteSortState) { batchsort_end(m_deleteSortState); m_deleteSortState = NULL; } - m_curSortedNum = 0; + m_curSortedNum = 0;// 当前已排序行数归零 // init new sort state + // 初始化新的排序状态，根据函数指针m_InitDeleteSortStatePtr调用相应的初始化函数 return (this->*m_InitDeleteSortStatePtr)(m_sortTupDesc, m_partidIdx, m_ctidIdx); } -- 2.34.1 From c15a35f028b14a3e757dede639a0ab056695706f Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Wed, 4 Oct 2023 05:20:18 +0800 Subject: [PATCH 39/48] Update cstore_delta.cpp --- .../storage/cstore/cstore_delta.cpp | 215 ++++++++++++------ 1 file changed, 143 insertions(+), 72 deletions(-) diff --git a/src/gausskernel/storage/cstore/cstore_delta.cpp b/src/gausskernel/storage/cstore/cstore_delta.cpp index 5c8d5a77b..79c9d9da9 100644 --- a/src/gausskernel/storage/cstore/cstore_delta.cpp +++ b/src/gausskernel/storage/cstore/cstore_delta.cpp @@ -42,6 +42,7 @@ static Oid GetCUIdxFromDeltaIdx(Relation deltaIdx, bool isPartition); * @IN parentRel: if rel is a partition of a cstore, parentRel is * its parent relation. Otherwise parentRel is NULL. */ +//将delta表中的数据移动到对应的列存储表中 void MoveDeltaDataToCU(Relation rel, Relation parentRel) { if (RELATION_IS_PARTITIONED(rel)) { @@ -65,13 +66,14 @@ void MoveDeltaDataToCU(Relation rel, Relation parentRel) list_free_ext(partitionList); return; } - + // 打开delta表 Relation deltaRel = heap_open(RelationGetDeltaRelId(rel), RowExclusiveLock); TableScanDesc deltaScanDesc = tableam_scan_begin(deltaRel, GetActiveSnapshot(), 0, NULL); InsertArg args; HeapTuple deltaTup = NULL; ResultRelInfo *resultRelInfo = NULL; if (rel->rd_rel->relhasindex) { + // 如果列存储表有索引，则初始化结果关系信息结构体，并构建索引 resultRelInfo = makeNode(ResultRelInfo); if (parentRel != NULL) { InitResultRelInfo(resultRelInfo, parentRel, 1, 0); @@ -80,23 +82,26 @@ void MoveDeltaDataToCU(Relation rel, Relation parentRel) } ExecOpenIndices(resultRelInfo, false); } + // 初始化插入参数和列存储插入类 CStoreInsert::InitInsertArg(rel, resultRelInfo, true, args); CStoreInsert cstoreInsert(rel, args, false, NULL, NULL); TupleDesc tupDesc = rel->rd_att; Datum* val = (Datum*)palloc(sizeof(Datum) * tupDesc->natts); bool* null = (bool*)palloc(sizeof(bool) * tupDesc->natts); bulkload_rows batchRow(tupDesc, RelationGetMaxBatchRows(rel), true); - + // 遍历delta表中的每一条记录 while ((deltaTup = (HeapTuple) tableam_scan_getnexttuple(deltaScanDesc, ForwardScanDirection)) != NULL) { tableam_tops_deform_tuple(deltaTup, tupDesc, val, null); /* ignore returned value because only one tuple is appended into */ + // 将记录添加到批量加载变量中 (void)batchRow.append_one_tuple(val, null, tupDesc); - + // 从delta表中删除当前记录 /* delete the current tuple from delta table */ simple_heap_delete(deltaRel, &deltaTup->t_self); if (batchRow.full_rownum()) { + // 当批量加载变量达到最大存储记录数时，插入到列存储表中 /* insert into main table */ cstoreInsert.BatchInsert(&batchRow, 0); batchRow.reset(true); @@ -104,12 +109,15 @@ void MoveDeltaDataToCU(Relation rel, Relation parentRel) } if (batchRow.m_rows_curnum > 0) { + // 将剩余的记录插入到列存储表中 cstoreInsert.BatchInsert(&batchRow, 0); } cstoreInsert.SetEndFlag(); + // 关闭扫描描述符和delta表 tableam_scan_end(deltaScanDesc); /* clean cstore insert */ + // 释放内存和资源 pfree(val); pfree(null); CStoreInsert::DeInitInsertArg(args); @@ -130,38 +138,46 @@ void MoveDeltaDataToCU(Relation rel, Relation parentRel) * @IN indexRelationId: the index on CU. * @IN parentRel: the parent relation of the CU if relationId is a partition of cstore. */ +//在Delta表上定义唯一索引 void DefineDeltaUniqueIndex(Oid relationId, IndexStmt* stmt, Oid indexRelationId, Relation parentRel) { + // 检查是否声明了唯一索引，如果未声明，则直接返回 if (!stmt->unique) { return; } - - Relation rel = NULL; - Relation deltaRelation = NULL; - Partition partition = NULL; - int numberOfKeyAttributes = 0; - List* indexColNames = NIL; - char deltaIndexRelName[NAMEDATALEN] = {'\0'}; + // 初始化变量 + Relation rel = NULL;// Delta表的关系对象 + Relation deltaRelation = NULL;// Delta表的关系对象 + Partition partition = NULL;// 分区对象，仅在relationId是cstore分区时使用 + int numberOfKeyAttributes = 0;// 索引键的数量 + List* indexColNames = NIL;// 索引键的名称列表 + char deltaIndexRelName[NAMEDATALEN] = {'\0'};// Delta索引的名称 error_t ret = 0; - + // 对于分区表，获取Delta表的关联关系 if (parentRel) { + // 对于分区表，relationId是一个分区ID。使用AccessShareLock锁定分区。 /* For partiontioned table. relationId is a partition id. */ /* We will build index on partioned delta table, so we just use AccessShareLock on partition. */ partition = partitionOpen(parentRel, relationId, AccessShareLock); rel = partitionGetRelation(parentRel, partition); partitionClose(parentRel, partition, NoLock); } else { + // 对于非分区表，直接打开关系对象 /* For non-partioned table. */ rel = relation_open(relationId, AccessShareLock); } - + // 如果Delta表是分区表，对每个分区的Delta表定义唯一索引 if (RELATION_IS_PARTITIONED(rel)) { + // 对于分区表 /* For partitioned table. */ List* partitionList = relationGetPartitionOidList(rel); ListCell* cell = NULL; Oid partitionOid = InvalidOid; Oid partIdxOid = InvalidOid; - + /* + * 全局分区索引不支持列存储， + * 每个分区块都有一个索引。我们在每个分区上构建索引。 + */ /* * Global partition index does not support column store, * so each partition has an index. We build index on each @@ -177,12 +193,12 @@ void DefineDeltaUniqueIndex(Oid relationId, IndexStmt* stmt, Oid indexRelationId relation_close(rel, NoLock); return; } - + // 打开Delta表 deltaRelation = heap_open(RelationGetDeltaRelId(rel), ShareLock); - + // 获取索引键的数量和名称列表 numberOfKeyAttributes = list_length(stmt->indexParams); indexColNames = ChooseIndexColumnNames(stmt->indexParams); - + // 构造Delta索引的名称 if (!parentRel) { ret = snprintf_s(deltaIndexRelName, sizeof(deltaIndexRelName), sizeof(deltaIndexRelName) - 1, "pg_delta_index_%u", indexRelationId); @@ -191,7 +207,7 @@ void DefineDeltaUniqueIndex(Oid relationId, IndexStmt* stmt, Oid indexRelationId sizeof(deltaIndexRelName) - 1, "pg_delta_part_index_%u", indexRelationId); } securec_check_ss_c(ret, "\0", "\0"); - + // 创建索引的信息对象 IndexInfo* indexInfo = NULL; indexInfo = makeNode(IndexInfo); indexInfo->ii_NumIndexAttrs = numberOfKeyAttributes; @@ -208,12 +224,12 @@ void DefineDeltaUniqueIndex(Oid relationId, IndexStmt* stmt, Oid indexRelationId indexInfo->ii_Concurrent = false; indexInfo->ii_BrokenHotChain = false; indexInfo->ii_PgClassAttrId = 0; - + // 创建Delta唯一索引 (void)CreateDeltaUniqueIndex(deltaRelation, deltaIndexRelName, indexInfo, stmt->indexParams, indexColNames, stmt->primary); - + // 关闭Delta表 heap_close(deltaRelation, NoLock); - + // 释放关联关系对象 if (parentRel) { releaseDummyRelation(&rel); } else { @@ -230,6 +246,17 @@ void DefineDeltaUniqueIndex(Oid relationId, IndexStmt* stmt, Oid indexRelationId * @IN indexColNames: the list contains name of each index column * @IN isPrimary: is primary index? */ +/* + * @Description: 在Delta表上创建唯一索引。 + * @IN deltaRel: Delta表的关系对象。 + * @IN deltaIndexName: Delta表索引的名称。 + * @IN indexInfo: CU索引的IndexInfo。 + * @IN indexElemList: 包含每个索引列属性的列表。 + * @IN indexColNames: 包含每个索引列名称的列表。 + * @IN isPrimary: 是否为主索引？ + * @return: 新创建的Delta索引的OID。 + */ +//在Delta表上创建唯一索引 Oid CreateDeltaUniqueIndex(Relation deltaRel, const char* deltaIndexName, IndexInfo* indexInfo, List* indexElemList, List* indexColNames, bool isPrimary) { @@ -238,20 +265,20 @@ Oid CreateDeltaUniqueIndex(Relation deltaRel, const char* deltaIndexName, IndexI Oid* collationObjectId = (Oid*)palloc(numIndexCols * sizeof(Oid)); Oid* classObjectId = (Oid*)palloc(numIndexCols * sizeof(Oid)); int16* coloptions = (int16*)palloc(numIndexCols * sizeof(int16)); - + // 计算索引列的属性值 ComputeIndexAttrs(indexInfo, typeObjectId, collationObjectId, classObjectId, coloptions, indexElemList, NULL, RelationGetRelid(deltaRel), "btree", BTREE_AM_OID, true, false); - + // 设置索引创建额外参数 IndexCreateExtraArgs extra; SetIndexCreateExtraArgs(&extra, InvalidOid, false, false); - + // 创建Delta唯一索引 Oid deltaIndexOid = index_create(deltaRel, deltaIndexName, InvalidOid, InvalidOid, indexInfo, indexColNames, BTREE_AM_OID, deltaRel->rd_rel->reltablespace, collationObjectId, classObjectId, coloptions, (Datum)0, isPrimary, false, false, false, true, false, false, &extra); - + // 释放内存 pfree(typeObjectId); pfree(collationObjectId); pfree(classObjectId); @@ -266,6 +293,7 @@ Oid CreateDeltaUniqueIndex(Relation deltaRel, const char* deltaIndexName, IndexI * @IN newRelOid: the new cstore after relation files swap. * @IN parentOid: parent oid for partitioned table, otherwise InvalidOid. */ +//删除旧delta表上的索引，并在新delta表上建立索引 void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid) { Relation oldRel = NULL; @@ -273,16 +301,19 @@ void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid) Relation parentRel = NULL; Oid oldDeltaOid = InvalidOid; Oid newDeltaOid = InvalidOid; - + // 打开旧的cstore并获取旧的delta表的oid oldRel = heap_open(oldRelOid, AccessShareLock); oldDeltaOid = oldRel->rd_rel->reldeltarelid; heap_close(oldRel, NoLock); if (OidIsValid(parentOid)) { /* For partitioned table. */ + // 如果是分区表，则打开父关系 parentRel = heap_open(parentOid, AccessShareLock); heap_close(parentRel, NoLock); + // 打开分区表对应的分区并获取新的关系 Partition partition = partitionOpen(parentRel, newRelOid, AccessShareLock); + // 如果是非分区表，则直接打开新的cstore newRel = partitionGetRelation(parentRel, partition); partitionClose(parentRel, partition, NoLock); } else { @@ -299,6 +330,7 @@ void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid) } /* Apply AccssShareLock because we only get information from old delta. */ + // 打开旧的delta表 Relation oldDelta = heap_open(oldDeltaOid, AccessShareLock); TupleDesc tupleDesc = RelationGetDescr(oldRel); @@ -313,6 +345,7 @@ void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid) heap_close(oldDelta, NoLock); /* Apply ShareLock because we will build index on new delta. */ + // 打开新的delta表 Relation newDelta = heap_open(newDeltaOid, ShareLock); ListCell* cell = NULL; @@ -336,11 +369,12 @@ void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid) * Must delete the index on old delta table, or conflicts will happen between * new index name and old index name. */ + // 删除旧delta表上的索引，避免新索引与旧索引名称冲突 indexObject.classId = RelationRelationId; indexObject.objectId = idxOid; indexObject.objectSubId = 0; performDeletion(&indexObject, DROP_RESTRICT, 0); - + // 在新的delta表上定义唯一索引 DefineDeltaUniqueIndex(newRelOid, indexStmt, CUIdxOid, parentRel); } } @@ -359,13 +393,21 @@ void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid) partition delta index. Otherwise reindex single partition delta index * For non-partioned cstore, indexPartId=InvaldOid. */ +/* + * @Description: 重建CU表上的Delta表索引 + * @IN indexId: CU上的索引OID + * @IN indexPartId: 指定分区CU索引的OID。对于分区cstore，indexPartId = + * InvalidOid表示重新生成所有分区delta索引，否则只重新生成单个分区的delta索引。对于非分区cstore，indexPartId = + * InvaldOid。 + */ +//重建CU表上的Delta表索引 void ReindexDeltaIndex(Oid indexId, Oid indexPartId) { Oid heapId = IndexGetRelation(indexId, false); - + /* 因为我们将在Delta表上构建索引，所以在CU表上获取AccessShareLock。 */ /* We get AccessShareLock on CU table because we will build index on delta table. */ Relation heapRelation = heap_open(heapId, AccessShareLock); - + /* 关闭CU表，但保持锁定状态。 */ /* Close CU table, but keep locks. */ heap_close(heapRelation, NoLock); @@ -373,14 +415,18 @@ void ReindexDeltaIndex(Oid indexId, Oid indexPartId) if (!RELATION_IS_PARTITIONED(heapRelation)) { /* For non partioned cstore table. */ + /* 对于非分区cstore表。 */ deltaIdxOid = GetDeltaIdxFromCUIdx(indexId, false); reindex_index(deltaIdxOid, InvalidOid, false, NULL, false); } else { + /* 对于分区cstore表。 */ /* For partitioned cstore table. */ if (OidIsValid(indexPartId)) { + /* 仅重新生成单个分区的delta索引。 */ deltaIdxOid = GetDeltaIdxFromCUIdx(indexPartId, true); reindex_index(deltaIdxOid, InvalidOid, false, NULL, false); } else { + /* 重新生成所有分区上的delta索引。 */ /* Reindex all indexes on part delta table. */ List* indexPartOidList = NIL; ListCell* partCell = NULL; @@ -404,35 +450,36 @@ void ReindexDeltaIndex(Oid indexId, Oid indexPartId) * @IN indexOid: parent index on CU * @IN partOid: partition CU oid */ +//重建分区Delta表的索引 void ReindexPartDeltaIndex(Oid indexOid, Oid partOid) { - Relation pg_partition = NULL; + Relation pg_partition = NULL;// 打开pg_partition系统表 ScanKeyData scanKey; SysScanDesc partScan; HeapTuple partTuple = NULL; Form_pg_partition partForm = NULL; Oid indexPartOid = InvalidOid; - pg_partition = heap_open(PartitionRelationId, AccessShareLock); + pg_partition = heap_open(PartitionRelationId, AccessShareLock);// 打开pg_partition系统表 ScanKeyInit(&scanKey, Anum_pg_partition_indextblid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(partOid)); - partScan = systable_beginscan(pg_partition, PartitionIndexTableIdIndexId, true, SnapshotNow, 1, &scanKey); - while ((partTuple = systable_getnext(partScan)) != NULL) { + partScan = systable_beginscan(pg_partition, PartitionIndexTableIdIndexId, true, SnapshotNow, 1, &scanKey);// 根据分区CU OID创建系统扫描 + while ((partTuple = systable_getnext(partScan)) != NULL) {// 遍历扫描结果 partForm = (Form_pg_partition)GETSTRUCT(partTuple); - if (partForm->parentid == indexOid) { + if (partForm->parentid == indexOid) {// 查找与给定索引OID匹配的分区 indexPartOid = HeapTupleGetOid(partTuple); break; } } - systable_endscan(partScan); - heap_close(pg_partition, AccessShareLock); + systable_endscan(partScan);// 结束系统扫描 + heap_close(pg_partition, AccessShareLock);// 关闭pg_partition系统表 - if (!OidIsValid(indexPartOid)) { + if (!OidIsValid(indexPartOid)) {// 若未找到匹配的分区OID，则抛出错误 ereport(ERROR, (errcode(ERRCODE_CACHE_LOOKUP_FAILED), errmsg("cache lookup failed for partitioned index %u", indexOid))); } - ReindexDeltaIndex(indexOid, indexPartOid); + ReindexDeltaIndex(indexOid, indexPartOid);// 重新构建指定分区CU索引的Delta表索引 } /* @@ -442,25 +489,35 @@ void ReindexPartDeltaIndex(Oid indexOid, Oid partOid) * false means delta index name is like pg_delta_index_xxx. * @Reture: the CU index oid */ +/* + * @Description: 从Delta表索引获取其对应的CU索引OID。 + * @IN deltaIdx: Delta表索引。 + * @IN isPartition: + * true表示Delta表索引命名方式为pg_delta_part_index_xxx，false表示Delta表索引命名方式为pg_delta_index_xxx。 + * @Return: CU索引OID。 + */ +//从Delta表索引获取与其对应的CU索引OID static Oid GetCUIdxFromDeltaIdx(Relation deltaIdx, bool isPartition) { - Oid CUIdxOid = 0; - int curIdx = 0; - const char* deltaIdxName = RelationGetRelationName(deltaIdx); + Oid CUIdxOid = 0;// 定义CU索引OID + int curIdx = 0;// 记录当前遍历到第几个字符 + const char* deltaIdxName = RelationGetRelationName(deltaIdx);// 获取Delta表索引名称 if (isPartition) { /* For partitioned delta table index. */ + /* 对于分区Delta表索引。*/ curIdx = strlen("pg_delta_part_index_"); } else { /* For non partitioned delta table index. */ + /* 对于非分区Delta表索引。*/ curIdx = strlen("pg_delta_index_"); } - while (deltaIdxName[curIdx] != '\0') { - CUIdxOid = CUIdxOid * 10 + (deltaIdxName[curIdx] - '0'); - curIdx++; + while (deltaIdxName[curIdx] != '\0') {// 遍历Delta表索引名称 + CUIdxOid = CUIdxOid * 10 + (deltaIdxName[curIdx] - '0');// 解析CU索引OID + curIdx++;// 指向下一个字符 } - return CUIdxOid; + return CUIdxOid;// 返回解析出的CU索引OID } /* @@ -470,31 +527,39 @@ static Oid GetCUIdxFromDeltaIdx(Relation deltaIdx, bool isPartition) * @IN suppressMiss: return InvalidOid if true when delta index is not found * @Reture: the delta index oid */ +/* + * @Description: 从对应的CU索引获取Delta索引OID。 + * @IN CUIndexOid: CU索引OID。 + * @IN isPartitioned: 表示CU索引是否为分区索引。 + * @IN suppressMiss: 如果为true，则在未找到Delta索引时返回InvalidOid。 + * @Return: Delta索引OID。 + */ +//从对应的CU索引获取Delta索引OID Oid GetDeltaIdxFromCUIdx(Oid CUIndexOid, bool isPartitioned, bool suppressMiss) { - char deltaIdxName[NAMEDATALEN] = {'\0'}; - Relation pgclass = NULL; - ScanKeyData scanKey[1]; - SysScanDesc scan = NULL; - HeapTuple tup = NULL; - Oid deltaIdxOid = InvalidOid; + char deltaIdxName[NAMEDATALEN] = {'\0'};// 定义Delta索引名称 + Relation pgclass = NULL;// 用于访问pg_class系统表 + ScanKeyData scanKey[1];// 扫描键 + SysScanDesc scan = NULL;// 扫描描述符 + HeapTuple tup = NULL;// 堆元组 + Oid deltaIdxOid = InvalidOid;// 定义Delta索引OID error_t ret = 0; if (!isPartitioned) { - ret = snprintf_s(deltaIdxName, sizeof(deltaIdxName), - sizeof(deltaIdxName) - 1, "pg_delta_index_%u", CUIndexOid); + ret = snprintf_s(deltaIdxName, sizeof(deltaIdxName), sizeof(deltaIdxName) - 1, "pg_delta_index_%u", + CUIndexOid); // 根据CU索引OID构建Delta索引名称 } else { ret = snprintf_s(deltaIdxName, sizeof(deltaIdxName), - sizeof(deltaIdxName) - 1, "pg_delta_part_index_%u", CUIndexOid); + sizeof(deltaIdxName) - 1, "pg_delta_part_index_%u", CUIndexOid);// 根据CU索引OID构建分区Delta索引名称 } securec_check_ss_c(ret, "\0", "\0"); - ScanKeyInit(&scanKey[0], Anum_pg_class_relname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(deltaIdxName)); - pgclass = heap_open(RelationRelationId, AccessShareLock); - scan = systable_beginscan(pgclass, ClassNameNspIndexId, true, SnapshotNow, 1, scanKey); - tup = systable_getnext(scan); + ScanKeyInit(&scanKey[0], Anum_pg_class_relname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(deltaIdxName));// 初始化扫描键 + pgclass = heap_open(RelationRelationId, AccessShareLock);// 打开pg_class系统表 + scan = systable_beginscan(pgclass, ClassNameNspIndexId, true, SnapshotNow, 1, scanKey);// 开始扫描pg_class系统表 + tup = systable_getnext(scan);// 获取下一个符合条件的堆元组 if (HeapTupleIsValid(tup)) { - deltaIdxOid = HeapTupleGetOid(tup); + deltaIdxOid = HeapTupleGetOid(tup);// 获取Delta索引OID } else { if (suppressMiss) { systable_endscan(scan); @@ -502,12 +567,12 @@ Oid GetDeltaIdxFromCUIdx(Oid CUIndexOid, bool isPartitioned, bool suppressMiss) return InvalidOid; } ereport( - ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation \"%s\" does not exist", deltaIdxName))); + ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation \"%s\" does not exist", deltaIdxName)));// 报告错误，指定的Delta索引不存在 } - systable_endscan(scan); - heap_close(pgclass, AccessShareLock); + systable_endscan(scan);// 结束扫描 + heap_close(pgclass, AccessShareLock);// 关闭pg_class系统表 - return deltaIdxOid; + return deltaIdxOid;// 返回Delta索引OID } /* @@ -515,35 +580,41 @@ Oid GetDeltaIdxFromCUIdx(Oid CUIndexOid, bool isPartitioned, bool suppressMiss) * @IN deltaIdx: delta index which name stores CU index oid * @Return: CU index name */ +/* + * @Description: 从Delta索引获取CU索引名称。 + * @IN deltaIdx: 存储CU索引OID的Delta索引。 + * @Return: CU索引名称。 + */ +//从Delta索引获取CU索引名称 char* GetCUIdxNameFromDeltaIdx(Relation deltaIdx) { - const char* deltaIdxName = RelationGetRelationName(deltaIdx); + const char* deltaIdxName = RelationGetRelationName(deltaIdx);// 获取Delta索引的名称 bool partitioned; if (pg_strncasecmp(deltaIdxName, "pg_delta_part_index_", strlen("pg_delta_part_index_")) == 0) { - partitioned = true; + partitioned = true;// 判断Delta索引是否为分区Delta索引 } else { partitioned = false; } - Oid CUIdxOid = GetCUIdxFromDeltaIdx(deltaIdx, partitioned); + Oid CUIdxOid = GetCUIdxFromDeltaIdx(deltaIdx, partitioned);// 通过Delta索引获取CU索引的OID char* CUIdxName = NULL; if (!partitioned) { - CUIdxName = get_rel_name(CUIdxOid); + CUIdxName = get_rel_name(CUIdxOid);// 获取非分区CU索引的名称 } else { - HeapTuple tuple = SearchSysCacheCopy1(PARTRELID, ObjectIdGetDatum(CUIdxOid)); + HeapTuple tuple = SearchSysCacheCopy1(PARTRELID, ObjectIdGetDatum(CUIdxOid));// 通过分区CU索引的OID搜索系统缓存 if (!HeapTupleIsValid(tuple)) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("could not find tuple for partition index %u", CUIdxOid))); + errmsg("could not find tuple for partition index %u", CUIdxOid)));// 如果未找到分区CU索引的元组，则报错 } Form_pg_partition partForm = (Form_pg_partition)GETSTRUCT(tuple); - Oid parentIdxOid = partForm->parentid; + Oid parentIdxOid = partForm->parentid;// 获取分区CU索引的父索引OID - CUIdxName = get_rel_name(parentIdxOid); + CUIdxName = get_rel_name(parentIdxOid);// 获取分区CU索引的父索引名称 - heap_freetuple(tuple); + heap_freetuple(tuple);// 释放缓存的元组 } - return CUIdxName; + return CUIdxName;// 返回CU索引名称 } -- 2.34.1 From f380fd6dfcd0bac57f4dce5172ab76f314b714ef Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Wed, 4 Oct 2023 05:20:38 +0800 Subject: [PATCH 40/48] Update cstore_mem_alloc.cpp --- .../storage/cstore/cstore_mem_alloc.cpp | 199 +++++++++++------- 1 file changed, 124 insertions(+), 75 deletions(-) diff --git a/src/gausskernel/storage/cstore/cstore_mem_alloc.cpp b/src/gausskernel/storage/cstore/cstore_mem_alloc.cpp index 30c8cc659..9010d4289 100644 --- a/src/gausskernel/storage/cstore/cstore_mem_alloc.cpp +++ b/src/gausskernel/storage/cstore/cstore_mem_alloc.cpp @@ -30,25 +30,25 @@ THR_LOCAL uint64 CStoreMemAlloc::m_count = 0; THR_LOCAL uint32 CStoreMemAlloc::m_ptrNodeCacheCount = 0; THR_LOCAL PointerNode* CStoreMemAlloc::m_ptrNodeCache[MaxPtrNodeCacheLen]; THR_LOCAL PointerList CStoreMemAlloc::m_tab[MaxPointersArryLen]; - +//在内存中分配一段连续的空间，并返回分配的内存的首地址 static inline void* InnerMalloc(Size size) { void* ptr = NULL; - ADIO_RUN() + ADIO_RUN()// 判断是否启用了Async IO { - int ret = posix_memalign((void**)&(ptr), SYS_LOGICAL_BLOCK_SIZE, (size_t)(size)); + int ret = posix_memalign((void**)&(ptr), SYS_LOGICAL_BLOCK_SIZE, (size_t)(size));// 采用posix_memalign分配内存 if (ret != 0) { ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("posix_memalign fails, The alignment argument was not a power of two, or was not a multiple " - "of sizeof(void *)"))); + "of sizeof(void *)")));// 如果分配内存失败，则报错 } } - ADIO_ELSE() + ADIO_ELSE()// 如果没有启用Async IO，则采用malloc分配内存 { ptr = malloc(size); } ADIO_END(); - return ptr; + return ptr;// 返回分配的内存的首地址 } /* @@ -58,40 +58,58 @@ static inline void* InnerMalloc(Size size) * if this memory is process-managered scope, set false. * @See also: */ +/* + * @Description: 从系统中申请一段内存空间，并返回分配的内存的首地址。 + * @Param[IN] size: 需要申请的内存大小。 + * @Param[IN] toRegister: 如果这段内存是线程管理的范围，则设置为true; + * 如果这段内存是进程管理的范围，则设置为false。 + * @See also: + */ +//在内存中分配一段连续的空间，并返回分配的内存的首地址 void* CStoreMemAlloc::Palloc(Size size, bool toRegister) { - Assert(size > 0); - void* ptr = InnerMalloc(size); + Assert(size > 0);// 断言需要申请的内存大小大于0 + void* ptr = InnerMalloc(size);// 调用InnerMalloc函数分配内存 if (ptr == NULL) { - ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("malloc fails, out of memory: size %lu", size))); + ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("malloc fails, out of memory: size %lu", size)));// 如果分配内存失败，则报错 } if (toRegister) { /* Step 2: register pointer */ - Register(ptr); + Register(ptr);// 如果需要将该内存注册到线程管理的范围，则调用Register函数进行注册 } - return ptr; + return ptr;// 返回分配的内存的首地址 } - +/* + * @Description: 从缓存中分配一个 PointerNode 结构体，或者从系统中申请一个 PointerNode 结构体。 + * @Param: 无 + * @See also: + */ +//从缓存中分配一个 PointerNode 结构体，或者从系统中申请一个 PointerNode 结构体 void* CStoreMemAlloc::AllocPointerNode() { PointerNode* ptr = NULL; - if (m_ptrNodeCacheCount > 0) { + if (m_ptrNodeCacheCount > 0) {// 如果缓存中有 PointerNode 结构体，则从缓存中分配一个结构体 ptr = m_ptrNodeCache[--m_ptrNodeCacheCount]; - } else { + } else {// 如果缓存中没有 PointerNode 结构体，则从系统中申请一个结构体 ptr = (PointerNode*)malloc(sizeof(PointerNode)); if (ptr == NULL) { ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("malloc fails, out of memory"))); } } - return ptr; + return ptr;// 返回分配的 PointerNode 结构体的首地址 } - +/* + * @Description: 释放一个 PointerNode 结构体，并将其放入缓存中或直接释放内存。 + * @Param[IN] ptr: 要释放的 PointerNode 结构体的首地址。 + * @See also: + */ +//释放一个 PointerNode 结构体，并将其放入缓存中或直接释放内存 void CStoreMemAlloc::FreePointerNode(PointerNode* ptr) { - if (m_ptrNodeCacheCount < MaxPtrNodeCacheLen) { + if (m_ptrNodeCacheCount < MaxPtrNodeCacheLen) {// 如果缓存中还有空间，则将 PointerNode 放入缓存中 m_ptrNodeCache[m_ptrNodeCacheCount++] = ptr; - } else { + } else {// 如果缓存已满，则直接释放 PointerNode 的内存 free(ptr); } } @@ -105,28 +123,37 @@ void CStoreMemAlloc::FreePointerNode(PointerNode* ptr) * false if passing false to Palloc(); * @See also: */ +/* + * @Description: 重新分配内存空间，类似于系统的realloc函数。 + * @Param[IN] old_size: 原始内存块的大小。 + * @Param[IN] size: 新的内存块的大小。 + * @Param[IN] pointer: 要重新分配的内存块的首地址。 + * @Param[IN] registered: 如果传递 true 给 Palloc()，则为 true；如果传递 false 给 Palloc()，则为 false。 + * @See also: + */ +//重新分配内存空间，类似于系统的realloc函数 void* CStoreMemAlloc::Repalloc(void* pointer, Size size, Size old_size, bool registered) { - Assert(pointer != NULL); - Assert(size > 0); + Assert(pointer != NULL);// 断言：指针不为空 + Assert(size > 0);// 断言：新的内存块大小大于0 - void* ptr = InnerMalloc(size); - if (ptr == NULL) { + void* ptr = InnerMalloc(size);// 分配新的内存块 + if (ptr == NULL) {// 如果分配失败，则报错 ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); } - errno_t rc = memcpy_s(ptr, size, pointer, old_size); + errno_t rc = memcpy_s(ptr, size, pointer, old_size);// 将旧的内存块内容拷贝到新的内存块中 securec_check_c(rc, "\0", "\0"); - if (registered) { + if (registered) {// 如果传入的 registered 参数为 true，则将新的内存块注册，同时解除旧的内存块的注册 Register(ptr); Unregister(pointer); Assert(m_count > 0); } - free(pointer); + free(pointer);// 释放旧的内存块的内存空间 - return ptr; + return ptr;// 返回重新分配的内存块的首地址 } /* @@ -136,83 +163,103 @@ void* CStoreMemAlloc::Repalloc(void* pointer, Size size, Size old_size, bool reg * false if passing false to Palloc(); * @See also: */ +/* + * @Description: 释放内存空间，类似于系统的free函数。 + * @Param[IN] pointer: 要释放的内存块的首地址。 + * @Param[IN] registered: 如果传递 true 给 Palloc()，则为 true；如果传递 false 给 Palloc()，则为 false。 + * @See also: + */ +//释放内存空间，类似于系统的free函数 void CStoreMemAlloc::Pfree(void* pointer, bool registered) { - Assert(pointer); - if (registered) { + Assert(pointer);// 断言：指针不为空 + if (registered) {// 如果传入的 registered 参数为 true，则解除内存块的注册 Unregister(pointer); } - free(pointer); + free(pointer);// 释放内存块的内存空间 } - +/* + * @Description: 注册内存块的指针地址，并添加到哈希表中。 + * @Param[IN] pointer: 要注册的内存块的指针地址。 + * @See also: + */ +//注册内存块的指针地址，并将其添加到哈希表中 void CStoreMemAlloc::Register(void* pointer) { - Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0); - int idx = PointerGetDatum(pointer) & (MaxPointersArryLen - 1); - PointerNode* nodePtr = m_tab[idx].tail; + Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0);// 断言：MaxPointersArryLen 是2的幂 + int idx = PointerGetDatum(pointer) & (MaxPointersArryLen - 1);// 通过指针的数值计算哈希表的索引 + PointerNode* nodePtr = m_tab[idx].tail;// 取出哈希表对应索引的链表的末尾节点 - if (nodePtr == NULL) { + if (nodePtr == NULL) { // 如果末尾节点为空，则说明该索引处还没有节点，需要新建一个节点并将其设置为头尾指针 Assert(m_tab[idx].header == NULL); m_tab[idx].header = m_tab[idx].tail = (PointerNode*)AllocPointerNode(); m_tab[idx].header->ptr = pointer; m_tab[idx].header->next = NULL; - } else { + } else {// 如果末尾节点不为空，则新建一个节点，并添加到链表的末尾 nodePtr->next = (PointerNode*)AllocPointerNode(); nodePtr->next->ptr = pointer; nodePtr->next->next = NULL; m_tab[idx].tail = nodePtr->next; } - ++m_count; + ++m_count;// 统计注册的内存块的数量 } - +/* + * @Description: 取消注册内存块的指针地址，并从哈希表中移除。 + * @Param[IN] pointer: 要取消注册的内存块的指针地址。 + * @See also: + */ +//取消注册内存块的指针地址，并从哈希表中移除 void CStoreMemAlloc::Unregister(const void* pointer) { - Assert(pointer && m_count > 0); + Assert(pointer && m_count > 0);// 断言：指针不能为空且已注册的内存块数量大于0 // Step 1: which node list include this pointer // - Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0); - int idx = PointerGetDatum(pointer) & (MaxPointersArryLen - 1); + Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0); // 断言：MaxPointersArryLen 是2的幂 + int idx = PointerGetDatum(pointer) & (MaxPointersArryLen - 1);// 通过指针的数值计算哈希表的索引 // Step 2: Search node list // - PointerNode* nodePtr = m_tab[idx].header; + PointerNode* nodePtr = m_tab[idx].header;// 获取哈希表索引处链表的头指针 Assert(nodePtr); - PointerNode* prePtr = NULL; - while (nodePtr != NULL) { - if (nodePtr->ptr == pointer) { - if (prePtr == NULL) { + PointerNode* prePtr = NULL;// 初始化前一个节点指针为NULL + while (nodePtr != NULL) {// 遍历链表，寻找要取消注册的指针地址对应的节点 + if (nodePtr->ptr == pointer) {// 如果找到了要取消注册的节点 + if (prePtr == NULL) {// 如果该节点是头指针 m_tab[idx].header = nodePtr->next; // If this list has only one node // - if (m_tab[idx].tail == nodePtr) { + if (m_tab[idx].tail == nodePtr) {// 如果该链表只有一个节点 Assert(m_tab[idx].header == NULL); - m_tab[idx].tail = NULL; + m_tab[idx].tail = NULL;// 将尾指针设为NULL } - } else { + } else {// 如果该节点不是头指针 prePtr->next = nodePtr->next; // We need modify tail pointer if free tail node // - if (m_tab[idx].tail == nodePtr) { - m_tab[idx].tail = prePtr; + if (m_tab[idx].tail == nodePtr) {// 如果该节点是尾指针 + m_tab[idx].tail = prePtr;// 将尾指针设为前一个节点 Assert(m_tab[idx].tail->next == NULL); } } - nodePtr->Reset(); - FreePointerNode(nodePtr); + nodePtr->Reset();// 重置节点的内容 + FreePointerNode(nodePtr);// 释放节点内存 break; } prePtr = nodePtr; - nodePtr = nodePtr->next; + nodePtr = nodePtr->next;// 继续遍历下一个节点 } - --m_count; - Assert(nodePtr != NULL); + --m_count;// 更新已注册的内存块数量 + Assert(nodePtr != NULL);// 断言：找到了要取消注册的节点 } - +/* + * @Description: 重置内存分配器的状态，释放已注册的内存块和缓存的节点。 + */ +//重置内存分配器的状态，释放已注册的内存块和缓存的节点 void CStoreMemAlloc::Reset() { uint32 i = 0; @@ -221,41 +268,43 @@ void CStoreMemAlloc::Reset() // Step 1: free each list in m_tab // for (i = 0; i < MaxPointersArryLen; ++i) { - PointerNode* nodePtr = m_tab[i].header; + PointerNode* nodePtr = m_tab[i].header;// 获取哈希表索引处链表的头指针 PointerNode* tmpPtr = NULL; - while ((nodePtr != NULL) && (nodePtr->ptr != NULL)) { - free(nodePtr->ptr); - tmpPtr = nodePtr->next; - free(nodePtr); - nodePtr = tmpPtr; - ++freeNum; + while ((nodePtr != NULL) && (nodePtr->ptr != NULL)) {// 遍历链表，释放节点和节点中的内存块 + free(nodePtr->ptr);// 释放节点中的内存块 + tmpPtr = nodePtr->next;// 保存下一个节点指针 + free(nodePtr);// 释放节点内存 + nodePtr = tmpPtr;// 继续处理下一个节点 + ++freeNum;// 更新释放的节点数量 } // Note that We must reset NULL // Because thread can be reused // - m_tab[i].header = NULL; - m_tab[i].tail = NULL; + m_tab[i].header = NULL;// 将头指针设为NULL + m_tab[i].tail = NULL;// 将尾指针设为NULL } - Assert(m_count == freeNum); - m_count = 0; + Assert(m_count == freeNum);// 断言：已注册的内存块数量等于释放的内存块数量 + m_count = 0;// 将已注册的内存块数量设为0 } // Step 2: free cached node if need // for (i = 0; i < m_ptrNodeCacheCount; ++i) { - free(m_ptrNodeCache[i]); + free(m_ptrNodeCache[i]);// 释放缓存的节点内存 } - m_ptrNodeCacheCount = 0; + m_ptrNodeCacheCount = 0;// 将缓存的节点数量设为0 } - +/* + * @Description: 初始化内存分配器的状态，设置初始值。 + */ void CStoreMemAlloc::Init() { - m_count = 0; - Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0); + m_count = 0;// 将已注册的内存块数量设为0 + Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0);// 断言：确保MaxPointersArryLen是2的幂次方 for (uint32 i = 0; i < MaxPointersArryLen; ++i) { - m_tab[i].header = NULL; - m_tab[i].tail = NULL; + m_tab[i].header = NULL;// 将哈希表索引处链表的头指针设为NULL + m_tab[i].tail = NULL;// 将哈希表索引处链表的尾指针设为NULL } - m_ptrNodeCacheCount = 0; + m_ptrNodeCacheCount = 0;// 将缓存的节点数量设为0 } -- 2.34.1 From 309fb71880bf60dd2fc34cbb3814e6bcc0230619 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Wed, 4 Oct 2023 05:20:59 +0800 Subject: [PATCH 41/48] Update cstore_minmax_func.cpp --- .../storage/cstore/cstore_minmax_func.cpp | 427 +++++++++++++++--- 1 file changed, 360 insertions(+), 67 deletions(-) diff --git a/src/gausskernel/storage/cstore/cstore_minmax_func.cpp b/src/gausskernel/storage/cstore/cstore_minmax_func.cpp index 0d360fd6b..5fee5e62a 100644 --- a/src/gausskernel/storage/cstore/cstore_minmax_func.cpp +++ b/src/gausskernel/storage/cstore/cstore_minmax_func.cpp @@ -32,18 +32,18 @@ /* Min/Max Option Function info */ struct FuncSetMinMaxInfo { /* data-type OID */ - Oid typeOid; + Oid typeOid;// 数据类型的对象标识符 /* Min/Max Set functions * Notice: we retain this member to avoid modifying many codes. */ - FuncSetMinMax set_minmax_func; + FuncSetMinMax set_minmax_func;// 最小/最大设置函数 /* substitute compare_datum_func for set_minmax_func ! * compare datum and set min/max info. */ - CompareDatum compare_datum_func; - FinishCompareDatum finish_compare_datum_func; + CompareDatum compare_datum_func;// 用于比较数据和设置最小/最大信息的函数 + FinishCompareDatum finish_compare_datum_func;// 完成比较数据函数 }; /* two offsets for var-length string @@ -85,13 +85,15 @@ static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first, * @IN minval: min value info * @See also: */ +//实际上没有任何实际操作，只是为了让编译器保持安静 static void FinishCompareDummy(const char* minval, const char* maxval, CUDesc* cuDescPtr) { /* just keep compiler silent */ - UNUSED_ARG(minval); - UNUSED_ARG(maxval); - UNUSED_ARG(cuDescPtr); - return; + /* 只是为了让编译器保持安静 */ + UNUSED_ARG(minval);// 未使用的参数minval + UNUSED_ARG(maxval);// 未使用的参数maxval + UNUSED_ARG(cuDescPtr); // 未使用的参数cuDescPtr + return;// 返回 } /* @@ -101,12 +103,16 @@ static void FinishCompareDummy(const char* minval, const char* maxval, CUDesc* c * @IN minval: min value info * @See also: */ +//用于固定长度数据类型的设置CUDesc最小/最大值 static void FinishCompareFixedLength(const char* minval, const char* maxval, CUDesc* cuDescPtr) { - errno_t rc = EOK; - rc = memcpy_s(cuDescPtr->cu_min, MIN_MAX_LEN, minval, MIN_MAX_LEN); + errno_t rc = EOK;// 定义errno_t变量rc并赋初值EOK + /* 将最小值信息复制到CU描述信息的最小值位置上 */ + rc = memcpy_s(cuDescPtr->cu_min, MIN_MAX_LEN, minval, MIN_MAX_LEN);// 将minval的MIN_MAX_LEN长度的数据复制到cuDescPtr->cu_min + securec_check(rc, "\0", "\0"); + /* 将最大值信息复制到CU描述信息的最大值位置上 */ + rc = memcpy_s(cuDescPtr->cu_max, MIN_MAX_LEN, maxval, MIN_MAX_LEN);// 将maxval的MIN_MAX_LEN长度的数据复制到cuDescPtr->cu_max securec_check(rc, "\0", "\0"); - rc = memcpy_s(cuDescPtr->cu_max, MIN_MAX_LEN, maxval, MIN_MAX_LEN); securec_check(rc, "\0", "\0"); } @@ -117,52 +123,60 @@ static void FinishCompareFixedLength(const char* minval, const char* maxval, CUD * @IN minval: min value info * @See also: */ +/* + * @Description: 用于变长数据类型的设置CUDesc最小/最大值的函数 + * @OUT cuDescPtr: CU描述信息 + * @IN maxval: 最大值信息 + * @IN minval: 最小值信息 + * @See also: + */ static void FinishCompareVarStrType(const char* minval, const char* maxval, CUDesc* cuDescPtr) { - char* dat = NULL; - errno_t rc = EOK; + char* dat = NULL;// 定义char类型指针dat，并初始化为NULL + errno_t rc = EOK;// 定义errno_t变量rc并赋初值EOK - /* if all values are NULLs, data point is 0. */ + /* if all values are NULLs, data point is 0. */ /* 如果所有值都为NULLs，则数据点为0。 */ if (*(Datum*)minval != 0) { - Assert(minval[varstr_dat_length] < MIN_MAX_LEN); - Assert(minval[varstr_dat_offset] == 1 || minval[varstr_dat_offset] == 4); + Assert(minval[varstr_dat_length] < MIN_MAX_LEN);// 断言minval的varstr_dat_length小于MIN_MAX_LEN + Assert(minval[varstr_dat_offset] == 1 || minval[varstr_dat_offset] == 4);// 断言minval的varstr_dat_offset等于1或4 - /* remember the length of min value */ + /* remember the length of min value */ /* 记录最小值的长度 */ cuDescPtr->cu_min[0] = minval[varstr_dat_length]; dat = DatumGetPointer(*(Datum*)minval) + minval[varstr_dat_offset]; - /* remember the min value */ - rc = memcpy_s(cuDescPtr->cu_min + 1, (MIN_MAX_LEN - 1), dat, minval[varstr_dat_length]); + /* remember the min value */ /* 记录最小值 */ + rc = memcpy_s(cuDescPtr->cu_min + 1, (MIN_MAX_LEN - 1), dat, + minval[varstr_dat_length]); // 将dat的minval[varstr_dat_length]长度的数据复制到cuDescPtr->cu_min securec_check(rc, "\0", "\0"); - Assert(maxval[varstr_dat_length] < MIN_MAX_LEN); - Assert(maxval[varstr_dat_offset] == 1 || maxval[varstr_dat_offset] == 4); + Assert(maxval[varstr_dat_length] < MIN_MAX_LEN);// 断言maxval的varstr_dat_length小于MIN_MAX_LEN + Assert(maxval[varstr_dat_offset] == 1 || maxval[varstr_dat_offset] == 4);// 断言maxval的varstr_dat_offset等于1或4 - /* remember the length of max value */ + /* remember the length of max value */ /* 记录最大值的长度 */ cuDescPtr->cu_max[0] = maxval[varstr_dat_length]; dat = DatumGetPointer(*(Datum*)maxval) + maxval[varstr_dat_offset]; - /* remember the max value */ - rc = memcpy_s(cuDescPtr->cu_max + 1, (MIN_MAX_LEN - 1), dat, maxval[varstr_dat_length]); + /* remember the max value */ /* 记录最大值 */ + rc = memcpy_s(cuDescPtr->cu_max + 1, (MIN_MAX_LEN - 1), dat, maxval[varstr_dat_length]);// 将dat的maxval[varstr_dat_length]长度的数据复制到cuDescPtr->cu_max securec_check(rc, "\0", "\0"); } else { - /* cuDescPtr->cu_min/cu_max have been set to 0, so nothing to do */ + /* cuDescPtr->cu_min/cu_max have been set to 0, so nothing to do */ /* cuDescPtr->cu_min/cu_max已经设置为0，因此不需要做任何操作 */ } } - +// 定义了一个结构体数组g_FuncTabSetMinMax，用于存储不同数据类型的最小/最大值设置函数 static FuncSetMinMaxInfo g_FuncTabSetMinMax[] = { { BOOLOID, /* OID = 16 bool */ - NULL, - CompareDummy, - FinishCompareDummy + NULL,// 最小/最大值设置函数为空 + CompareDummy,// 比较函数为CompareDummy + FinishCompareDummy// 结束比较函数为FinishCompareDummy }, { BYTEAOID, /* OID = 17 bytea */ - SetMinMaxVarStrType, - CompareVarStrType, - FinishCompareVarStrType + SetMinMaxVarStrType,// 最小/最大值设置函数为SetMinMaxVarStrType + CompareVarStrType,// 比较函数为CompareVarStrType + FinishCompareVarStrType// 结束比较函数为FinishCompareVarStrType }, { CHAROID, /* OID = 18 char */ - SetMinMaxChar, - CompareChar, - FinishCompareFixedLength + SetMinMaxChar,// 最小/最大值设置函数为SetMinMaxChar + CompareChar,// 比较函数为CompareChar + FinishCompareFixedLength// 结束比较函数为FinishCompareFixedLength }, { NAMEOID, /* OID = 19 name */ NULL, @@ -242,25 +256,27 @@ static FuncSetMinMaxInfo g_FuncTabSetMinMax[] = { }; const int FuncSetMinMaxTabSize = sizeof(g_FuncTabSetMinMax) / sizeof(FuncSetMinMaxInfo); - +//使用二分查找法来查找数据类型的OID在g_FuncTabSetMinMax结构体数组中的索引 /* Search and return the index of type OID given */ static int BinarySearch(Oid typeOid) { - int left = 0; - int mid = -1; - int right = FuncSetMinMaxTabSize - 1; + int left = 0;// 左边界 + int mid = -1;// 中间位置 + int right = FuncSetMinMaxTabSize - 1;// 右边界 while (left <= right) { - mid = left + ((right - left) / 2); - + mid = left + ((right - left) / 2);// 计算中间位置的索引 + // 如果中间位置的OID大于要查找的OID，则说明要查找的OID在左半部分，更新右边界 if (g_FuncTabSetMinMax[mid].typeOid > typeOid) right = mid - 1; + // 如果中间位置的OID小于要查找的OID，则说明要查找的OID在右半部分，更新左边界 else if (g_FuncTabSetMinMax[mid].typeOid < typeOid) left = mid + 1; + // 如果中间位置的OID等于要查找的OID，则直接返回中间位置的索引 else return mid; } - + // 如果未找到，则返回数组的大小FuncSetMinMaxTabSize return FuncSetMinMaxTabSize; } @@ -270,9 +286,13 @@ static int BinarySearch(Oid typeOid) * @Return: function about min/max setting * @See also: */ +// 查找指定数据类型的最小/最大值设置函数 +// 参数 typeOid：数据类型 OID +// 返回值：与最小/最大值设置相关的函数（FuncSetMinMax） FuncSetMinMax GetMinMaxFunc(Oid typeOid) { - int idx = BinarySearch(typeOid); + int idx = BinarySearch(typeOid);// 使用二分查找法，查找该数据类型在结构体数组中的索引 + // 如果找到了指定数据类型的最小/最大值设置函数，则返回该函数 return (idx < FuncSetMinMaxTabSize) ? g_FuncTabSetMinMax[idx].set_minmax_func : NULL; } @@ -282,9 +302,13 @@ FuncSetMinMax GetMinMaxFunc(Oid typeOid) * @Return: function about datum comparing * @See also: */ +// 查找指定数据类型的值比较函数 +// 参数 typeOid：数据类型 OID +// 返回值：与值比较相关的函数（CompareDatum） CompareDatum GetCompareDatumFunc(Oid typeOid) { - int idx = BinarySearch(typeOid); + int idx = BinarySearch(typeOid);// 使用二分查找法，查找该数据类型在结构体数组中的索引 + // 如果找到了指定数据类型的值比较函数，则返回该函数 return (idx < FuncSetMinMaxTabSize) ? g_FuncTabSetMinMax[idx].compare_datum_func : CompareDummy; } @@ -294,9 +318,14 @@ CompareDatum GetCompareDatumFunc(Oid typeOid) * @Return: finishing function about datum comparing * @See also: */ +// 查找指定数据类型的值比较完成函数 +// 参数 typeOid：数据类型 OID +// 返回值：与值比较完成相关的函数（FinishCompareDatum） +//查找指定数据类型的值比较完成函数。函数的参数为数据类型OID typeOid，返回值为与值比较完成相关的函数指针FinishCompareDatum FinishCompareDatum GetFinishCompareDatum(Oid typeOid) { - int idx = BinarySearch(typeOid); + int idx = BinarySearch(typeOid);// 使用二分查找法，查找该数据类型在结构体数组中的索引 + // 如果找到了指定数据类型的值比较完成函数，则返回该函数 return (idx < FuncSetMinMaxTabSize) ? g_FuncTabSetMinMax[idx].finish_compare_datum_func : FinishCompareDummy; } @@ -306,9 +335,12 @@ FinishCompareDatum GetFinishCompareDatum(Oid typeOid) * @Return: true if f is a dummy function; otherwise false. * @See also: */ +// 检查函数 f 是否为占位函数 +// 参数 f：值比较函数（CompareDatum） +// 返回值：如果 f 是占位函数返回 true，否则返回 false bool IsCompareDatumDummyFunc(CompareDatum f) { - return (CompareDummy == f); + return (CompareDummy == f); // 判断函数指针 f 是否等于占位函数 CompareDummy } /* @@ -320,14 +352,22 @@ bool IsCompareDatumDummyFunc(CompareDatum f) * @IN/OUT varstr_maxlen: max length about var-length string * @See also: */ +// 占位的值比较函数 +// 参数： +// - first: 指示是否是第一个值的标志（输入输出参数） +// - maxval: 最新的最大值（输入输出参数） +// - minval: 最新的最小值（输入输出参数） +// - v: 新的值 +// - varstr_maxlen: 变长字符串的最大长度（输入输出参数） +// 该函数没有实际的比较操作，只是将所有参数标记为未使用，并返回。 static void CompareDummy(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { - UNUSED_ARG(minval); + UNUSED_ARG(minval);// 标记未使用的参数 UNUSED_ARG(maxval); UNUSED_ARG(v); UNUSED_ARG(first); UNUSED_ARG(varstr_maxlen); - return; + return;// 返回 } /* @@ -339,21 +379,29 @@ static void CompareDummy(char* minval, char* maxval, Datum v, bool* first, int* * @IN/OUT varstr_maxlen: max length about var-length string * @See also: */ +// 用于 int8 类型数据的值比较函数 +// 参数： +// - first: 指示是否是第一个值的标志（输入输出参数） +// - maxval: 最新的最大值（输入输出参数） +// - minval: 最新的最小值（输入输出参数） +// - v: 新的 int8 类型值 +// - varstr_maxlen: 变长字符串的最大长度（输入输出参数） +// 该函数将输入值与当前最大值和最小值进行比较，并更新这些值。 static void CompareChar(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { - char chVal = DatumGetChar(v); + char chVal = DatumGetChar(v);// 将 int8 类型的值转换为 char 类型 - if (!*first) { - if (chVal < minval[0]) { - minval[0] = chVal; - } else if (chVal > maxval[0]) { - maxval[0] = chVal; + if (!*first) {// 如果不是第一个值 + if (chVal < minval[0]) {// 如果新值比最小值小 + minval[0] = chVal;// 更新最小值为新值 + } else if (chVal > maxval[0]) {// 如果新值比最大值大 + maxval[0] = chVal;// 更新最大值为新值 } - } else { - minval[0] = maxval[0] = chVal; - *first = false; + } else {// 如果是第一个值 + minval[0] = maxval[0] = chVal;// 最小值和最大值都等于新值 + *first = false;// 设置第一个值的标志为 false } - UNUSED_ARG(varstr_maxlen); + UNUSED_ARG(varstr_maxlen);// 标记未使用的参数 } /* @@ -363,9 +411,15 @@ static void CompareChar(char* minval, char* maxval, Datum v, bool* first, int* v * @IN v: the new int8 value * @See also: */ +// 用于设置 int8 类型的最小值和最大值 +// 参数： +// - cuDescPtr: CU 描述符（输入输出参数） +// - first: 第一个值的标志（输入输出参数） +// - v: 新的 int8 类型值 +// 该函数调用 CompareChar 函数，将新值与当前最小值和最大值进行比较，并更新这些值。 static void SetMinMaxChar(Datum v, CUDesc* cuDescPtr, bool* first) { - CompareChar(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL); + CompareChar(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL);// 调用 CompareChar 函数 } /* @@ -382,6 +436,38 @@ static void SetMinMaxChar(Datum v, CUDesc* cuDescPtr, bool* first) * 1B --> Offset to Var-Data, or Var-Head size * 1B --> Compare Data Size */ +/* + * @Description: comparing function for data-type var-length string + * 比较变长字符串类型数据的函数 + * + * @IN/OUT first: indicate the first value + * 第一个值的标志（输入输出参数） + * + * @IN/OUT maxval: the newest max value + * 最新的最大值（输入输出参数） + * + * @IN/OUT minval: the newest min value + * 最新的最小值（输入输出参数） + * + * @IN v: the new var-length string value + * 新的变长字符串值 + * + * @IN/OUT varstr_maxlen: max length about var-length string + * 变长字符串的最大长度（输入输出参数） + * + * @See also: + * + * minval/maxval struct: + * 8B --> Datum, pointer to min/max value + * 1B --> Offset to Var-Data, or Var-Head size + * 1B --> Compare Data Size + * + * minval/maxval 结构： + * 8B --> Datum，指向最小/最大值的指针 + * 1B --> 偏移量，指向变长数据或变长头的大小 + * 1B --> 比较数据的大小 + */ +//比较变长字符串类型的数据 static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { char* v_hdr = DatumGetPointer(v); @@ -396,6 +482,7 @@ static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first, Assert(v != (Datum)0); if (!*first) { /* Update Var-String max length info */ + // 更新变长字符串的最大长度信息 if (v_total_len > *varstr_maxlen) { *varstr_maxlen = v_total_len; } @@ -408,10 +495,13 @@ static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first, ret = memcmp(v_dat, curmin_dat, cmp_size); if (ret < 0 || (ret == 0 && copy_size < minval[varstr_dat_length])) { /* Update min value, and remember its pointer */ + // 更新最小值，并记住其指针 *(Datum*)minval = v; /* Remember its data offset */ + // 记住其数据偏移量 minval[varstr_dat_offset] = (char)(v_dat - v_hdr); /* Remember its data length */ + // 记住其数据长度 minval[varstr_dat_length] = copy_size; return; } @@ -424,21 +514,28 @@ static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first, ret = memcmp(v_dat, curmax_dat, cmp_size); if (ret > 0 || (ret == 0 && copy_size > maxval[varstr_dat_length])) { /* Update max value, and remember its pointer */ + // 更新最大值，并记住其指针 *(Datum*)maxval = v; /* Remember its data offset */ + // 记住其数据偏移量 maxval[varstr_dat_offset] = (char)(v_dat - v_hdr); /* Remember its data length */ + // 记住其数据长度 maxval[varstr_dat_length] = copy_size; } } else { /* Init Var-String max length info */ + // 初始化变长字符串的最大长度信息 *varstr_maxlen = v_total_len; /* Remember its pointer */ + // 记住其指针 *(Datum*)minval = *(Datum*)maxval = v; /* Remember its data offset */ + // 记住其数据偏移量 minval[varstr_dat_offset] = maxval[varstr_dat_offset] = (char)(v_dat - v_hdr); /* Remember its data length */ + // 记住其数据长度 minval[varstr_dat_length] = maxval[varstr_dat_length] = copy_size; *first = false; @@ -453,6 +550,25 @@ static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first, * @IN v: the new var-length string value * @See also: */ +/* + * @Description: set min/max for data-type var-length string + * 设置变长字符串类型的数据的最小值和最大值 + * + * @IN/OUT first: the first flag + * 第一个值的标志（输入输出参数） + * + * @IN/OUT maxval: the newest max value + * 最新的最大值（输入输出参数） + * + * @IN/OUT minval: the newest min value + * 最新的最小值（输入输出参数） + * + * @IN v: the new var-length string value + * 新的变长字符串值 + * + * @See also: + */ +//设置变长字符串类型数据的最小值和最大值 static void SetMinMaxVarStrTypeInternal(char* minval, char* maxval, Datum v, bool* first) { char* v_hdr = DatumGetPointer(v); @@ -466,11 +582,13 @@ static void SetMinMaxVarStrTypeInternal(char* minval, char* maxval, Datum v, boo copySize = Min(v_exchdr_len, (MIN_MAX_LEN - 1)); if (!*first) { + // 第二次及以后的值，更新最小值 Assert(minval[0] < MIN_MAX_LEN && maxval[0] < MIN_MAX_LEN); int cmpSize = Min(copySize, minval[0]); int ret = memcmp(v_dat, b + 1, cmpSize); if (ret < 0 || (ret == 0 && copySize < minval[0])) { + // 更新最小值的数据长度和数据内容 minval[0] = copySize; rc = memcpy_s(minval + 1, (MIN_MAX_LEN - 1), v_dat, copySize); securec_check(rc, "\0", "\0"); @@ -478,15 +596,17 @@ static void SetMinMaxVarStrTypeInternal(char* minval, char* maxval, Datum v, boo } b = maxval; - + // 更新最大值 cmpSize = Min(copySize, maxval[0]); ret = memcmp(v_dat, b + 1, cmpSize); if (ret > 0 || (ret == 0 && copySize > maxval[0])) { + // 更新最大值的数据长度和数据内容 maxval[0] = copySize; rc = memcpy_s(maxval + 1, (MIN_MAX_LEN - 1), v_dat, copySize); securec_check(rc, "\0", "\0"); } } else { + // 第一个值，初始化最大值和最小值 maxval[0] = minval[0] = copySize; rc = memcpy_s(maxval + 1, (MIN_MAX_LEN - 1), v_dat, copySize); securec_check(rc, "\0", "\0"); @@ -503,8 +623,10 @@ static void SetMinMaxVarStrTypeInternal(char* minval, char* maxval, Datum v, boo * @IN v: the new var-length string value * @See also: */ +//为变长字符串类型的数据设置最小值和最大值 static void SetMinMaxVarStrType(Datum v, CUDesc* cuDescPtr, bool* first) { + // 调用内部函数SetMinMaxVarStrTypeInternal，设置最小值和最大值 SetMinMaxVarStrTypeInternal(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first); } @@ -517,16 +639,23 @@ static void SetMinMaxVarStrType(Datum v, CUDesc* cuDescPtr, bool* first) * @IN/OUT varstr_maxlen: max length about var-length string * @See also: */ +//比较日期类型的数据，并更新最小值和最大值 static void CompareDate(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { + // 如果不是第一个值 if (!*first) { DateADT a = DatumGetDateADT(v); + // 比较新值和最小值，更新最小值 if (a < *(DateADT*)(minval)) { *((DateADT*)(minval)) = a; - } else if (a > *(DateADT*)(maxval)) { + } + // 比较新值和最大值，更新最大值 + else if (a > *(DateADT*)(maxval)) { *((DateADT*)(maxval)) = a; } + // 如果是第一个值 } else { + // 将最小值和最大值都设置为新值 *((DateADT*)(minval)) = *((DateADT*)(maxval)) = DatumGetDateADT(v); *first = false; } @@ -540,8 +669,10 @@ static void CompareDate(char* minval, char* maxval, Datum v, bool* first, int* v * @IN v: the new date value * @See also: */ +//设置日期类型数据的最小值和最大值 static void SetMinMaxDate(Datum v, CUDesc* cuDescPtr, bool* first) { + // 调用CompareDate函数，比较并更新最小值和最大值 CompareDate(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL); } @@ -554,14 +685,18 @@ static void SetMinMaxDate(Datum v, CUDesc* cuDescPtr, bool* first) * @IN/OUT varstr_maxlen: max length about var-length string * @See also: */ +//比较时间类型数据的值，并根据比较结果更新最小值和最大值 static void CompareTime(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { + // 检查是否为第一个值 if (!*first) { + // 将传入的值和当前最小值、最大值进行比较 TimeADT tVal = DatumGetTimeADT(v); TimeADT curMin = *(TimeADT*)(minval); TimeADT curMax = *(TimeADT*)(maxval); #ifdef HAVE_INT64_TIMESTAMP + // 对于64位时间戳的情况，如果新值小于当前最小值，则更新最小值；如果新值大于当前最大值，则更新最大值 if (tVal < curMin) { *((TimeADT*)(minval)) = tVal; } else if (tVal > curMax) { @@ -573,14 +708,22 @@ static void CompareTime(char* minval, char* maxval, Datum v, bool* first, int* v * 2. NAN == NAN * 3. the ordinary comparing rules */ + /* 时间比较规则： + * 1. NAN > 非NAN + * 2. NAN == NAN + * 3. 普通比较规则 + */ + // 对于非64位时间戳的情况，如果当前最小值是NaN（不是数字），或者新值非NaN且小于当前最小值，则更新最小值 if (isnan(curMin) || (!isnan(tVal) && (tVal < curMin))) *((TimeADT*)(minval)) = tVal; - + /* 不使用ELSE分支，以便分别更新最小值和最大值 */ /* Never use ELSE branch so update min/max value separately */ + // 如果新值是NaN，或者当前最大值非NaN且新值大于当前最大值，则更新最大值 if (isnan(tVal) || (!isnan(curMax) && (tVal > curMax))) *((TimeADT*)(maxval)) = tVal; #endif } else { + // 第一个值的情况下，直接将最小值和最大值都设置为传入的值，并将first标记设为false *((TimeADT*)(minval)) = *((TimeADT*)(maxval)) = DatumGetTimeADT(v); *first = false; } @@ -594,8 +737,14 @@ static void CompareTime(char* minval, char* maxval, Datum v, bool* first, int* v * @IN v: the new time value * @See also: */ +// 用于为时间类型数据设置最小值和最大值 +// 参数： +// - v：新的时间值 +// - cuDescPtr：列存储单元描述符的指针，包含了当前最小值和最大值 +// - first：第一个值的标志，输入输出参数 static void SetMinMaxTime(Datum v, CUDesc* cuDescPtr, bool* first) { + // 调用CompareTime函数比较时间类型数据的值，并根据比较结果更新最小值和最大值 CompareTime(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL); } @@ -608,8 +757,16 @@ static void SetMinMaxTime(Datum v, CUDesc* cuDescPtr, bool* first) * @IN/OUT varstr_maxlen: max length about var-length string * @See also: timestamp_cmp_internal() */ +// 用于比较时间戳类型的数据值，并根据比较结果更新最小值和最大值 +// 参数： +// - first：第一个值的标志，输入输出参数 +// - maxval：当前最大值，输入输出参数 +// - minval：当前最小值，输入输出参数 +// - v：新的时间戳值 +// - varstr_maxlen：varstr类型数据的最大长度，未被使用 static void CompareTimestamp(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { + // 如果不是第一个值，则比较新值与当前最小值、最大值之间的大小关系，并根据比较结果更新最小值和最大值 if (!*first) { Timestamp tVal = DatumGetTimestamp(v); Timestamp curMin = *(Timestamp*)(minval); @@ -622,6 +779,11 @@ static void CompareTimestamp(char* minval, char* maxval, Datum v, bool* first, i *((Timestamp*)(maxval)) = tVal; } #else + /* the rules of comparing are + * 1. NAN > non-NAN + * 2. NAN == NAN + * 3. the ordinary comparing rules + */ /* the rules of comparing are * 1. NAN > non-NAN * 2. NAN == NAN @@ -634,11 +796,11 @@ static void CompareTimestamp(char* minval, char* maxval, Datum v, bool* first, i if (isnan(tVal) || (!isnan(curMax) && (tVal > curMax))) *((Timestamp*)(maxval)) = tVal; #endif - } else { + } else {// 如果是第一个值，则直接将其设置为最小值和最大值 *((Timestamp*)(minval)) = *((Timestamp*)(maxval)) = DatumGetTimestamp(v); - *first = false; + *first = false;// 将第一个值的标志置为false } - UNUSED_ARG(varstr_maxlen); + UNUSED_ARG(varstr_maxlen);// 未使用的参数，被忽略掉 } /* @@ -648,8 +810,17 @@ static void CompareTimestamp(char* minval, char* maxval, Datum v, bool* first, i * @IN v: the new timestamp value * @See also: */ +/* + * @Description: 对于数据类型为时间戳的值，设置最小值和最大值。 + * @IN/OUT cuDescPtr: CU描述符 + * @IN/OUT first: 第一个值的标志，用于指示是否是第一个值 + * @IN v: 新的时间戳值 + * @See also: + */ +//处理数据类型为时间戳的值，它接受一个新的时间戳值 v，并根据比较结果更新最小值和最大值 static void SetMinMaxTimestamp(Datum v, CUDesc* cuDescPtr, bool* first) { + // 调用CompareTimestamp函数，比较新的时间戳值与当前最小值和最大值，根据比较结果更新最小值和最大值 CompareTimestamp(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL); } @@ -662,14 +833,27 @@ static void SetMinMaxTimestamp(Datum v, CUDesc* cuDescPtr, bool* first) * @IN/OUT varstr_maxlen: max length about var-length string * @See also: timestamp_cmp_internal() */ +/* + * @Description: 用于比较数据类型为带有时区的时间戳的比较函数。 + * @IN/OUT first: 表示是否是第一个值的标志位 + * @IN/OUT maxval: 当前最大值 + * @IN/OUT minval: 当前最小值 + * @IN v: 新的带有时区的时间戳值 + * @IN/OUT varstr_maxlen: 变长字符串的最大长度 + * @See also: timestamp_cmp_internal() + */ +//比较数据类型为带有时区的时间戳的比较函数。它接受一个新的带有时区的时间戳值 v，并根据比较结果更新最小值和最大值 static void CompareTimestamptz(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { + // 如果不是第一个值 if (!*first) { + // 将传入的参数转换为TimestampTz类型 TimestampTz tVal = DatumGetTimestampTz(v); TimestampTz curMin = *(TimestampTz*)(minval); TimestampTz curMax = *(TimestampTz*)(maxval); #ifdef HAVE_INT64_TIMESTAMP + // 如果是64位时间戳 if (tVal < curMin) { *((TimestampTz*)(minval)) = tVal; } else if (tVal > curMax) { @@ -681,14 +865,20 @@ static void CompareTimestamptz(char* minval, char* maxval, Datum v, bool* first, * 2. NAN == NAN * 3. the ordinary comparing rules */ + /* 比较规则如下： + * 1. NAN > 非NAN + * 2. NAN == NAN + * 3. 按照普通的比较规则 + */ if (isnan(curMin) || (!isnan(tVal) && (tVal < curMin))) *((TimestampTz*)(minval)) = tVal; - + // 不要使用ELSE分支，以便分别更新最小值和最大值 /* Never use ELSE branch so update min/max value separately */ if (isnan(tVal) || (!isnan(curMax) && (tVal > curMax))) *((TimestampTz*)(maxval)) = tVal; #endif } else { + // 如果是第一个值，则将最小值和最大值都设置为新值 *((TimestampTz*)(minval)) = *((TimestampTz*)(maxval)) = DatumGetTimestampTz(v); *first = false; } @@ -702,21 +892,41 @@ static void CompareTimestamptz(char* minval, char* maxval, Datum v, bool* first, * @IN v: the new timestamp with time zone value * @See also: */ +/* + * @Description: 设置带有时区的时间戳数据类型的最小值和最大值 + * @IN/OUT cuDescPtr: CU描述符 + * @IN/OUT first: 第一个值的标志位 + * @IN v: 新的带有时区的时间戳值 + * @See also: CompareTimestamptz() + */ static void SetMinMaxTimestamptz(Datum v, CUDesc* cuDescPtr, bool* first) { + // 调用CompareTimestamptz()函数，比较并更新最小值和最大值 CompareTimestamptz(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL); } - +/* + * @Description: 用于比较数据类型为uint64的比较函数。 + * @IN/OUT minval: 当前最小值 + * @IN/OUT maxval: 当前最大值 + * @IN v: 新的uint64值 + * @IN/OUT first: 表示是否是第一个值的标志位 + * @IN/OUT varstr_maxlen: 变长字符串的最大长度 + * @See also: + */ static void CompareUint64(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { + // 如果不是第一个值 if (!*first) { uint64 intVal = (uint64)v; if (intVal < *(uint64*)(minval)) { + // 更新最小值 *(uint64*)(minval) = intVal; } else if (intVal > *(uint64*)(maxval)) { + // 更新最大值 *(uint64*)(maxval) = intVal; } } else { + // 如果是第一个值，则将最小值和最大值都设置为新值 *(uint64*)(minval) = *(uint64*)(maxval) = (uint64)(v); *first = false; } @@ -730,8 +940,16 @@ static void CompareUint64(char* minval, char* maxval, Datum v, bool* first, int* * @IN v: the new uint64 value * @See also: */ +/* + * @Description: 用于设置数据类型为 uint64 的最小值和最大值。 + * @IN v: 新的 uint64 值 + * @IN/OUT cuDescPtr: CU 描述符 + * @IN/OUT flag: 表示是否是第一个值的标志位 + * @See also: + */ static void SetMinMaxUint64(Datum v, CUDesc* cuDescPtr, bool* flag) { + // 调用 CompareUint64 函数，更新最小值和最大值 CompareUint64(cuDescPtr->cu_min, cuDescPtr->cu_max, v, flag, NULL); } @@ -744,16 +962,29 @@ static void SetMinMaxUint64(Datum v, CUDesc* cuDescPtr, bool* flag) * @IN/OUT varstr_maxlen: max length about var-length string * @See also: */ +/* + * @Description: 用于比较数据类型为 int64 的比较函数。 + * @IN/OUT minval: 当前最小值 + * @IN/OUT maxval: 当前最大值 + * @IN v: 新的 int64 值 + * @IN/OUT first: 表示是否是第一个值的标志位 + * @IN/OUT varstr_maxlen: 变长字符串的最大长度 + * @See also: + */ static void CompareInt64(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { + // 如果不是第一个值 if (!*first) { int64 intVal = DatumGetInt64(v); if (intVal < *(int64*)(minval)) { + // 更新最小值 *(int64*)(minval) = intVal; } else if (intVal > *(int64*)(maxval)) { + // 更新最大值 *(int64*)(maxval) = intVal; } } else { + // 如果是第一个值，则将最小值和最大值都设置为新值 *(int64*)(minval) = *(int64*)(maxval) = DatumGetInt64(v); *first = false; } @@ -767,8 +998,16 @@ static void CompareInt64(char* minval, char* maxval, Datum v, bool* first, int* * @IN v: the new int64 value * @See also: */ +/* + * @Description: 用于设置数据类型为 int64 的最小值和最大值。 + * @IN/OUT cuDescPtr: CU 描述符 + * @IN/OUT first: 第一个值的标志位 + * @IN v: 新的 int64 值 + * @See also: + */ static void SetMinMaxInt64(Datum v, CUDesc* cuDescPtr, bool* first) { + // 调用 CompareInt64 函数，将新的 int64 值与当前的最小值和最大值进行比较和更新 CompareInt64(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL); } @@ -781,16 +1020,29 @@ static void SetMinMaxInt64(Datum v, CUDesc* cuDescPtr, bool* first) * @IN/OUT varstr_maxlen: max length about var-length string * @See also: */ +/* + * @Description: 用于比较数据类型为 int32 的比较函数。 + * @IN/OUT first: 表示是否是第一个值的标志位 + * @IN/OUT maxval: 当前最大值 + * @IN/OUT minval: 当前最小值 + * @IN v: 新的 int32 值 + * @IN/OUT varstr_maxlen: 变长字符串的最大长度 + * @See also: + */ static void CompareInt32(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { + // 如果不是第一个值 if (!*first) { int32 intVal = DatumGetInt32(v); if (intVal < *(int32*)(minval)) { + // 更新最小值 *(int32*)(minval) = intVal; } else if (intVal > *(int32*)(maxval)) { + // 更新最大值 *(int32*)(maxval) = intVal; } } else { + // 如果是第一个值，则将最小值和最大值都设置为新值 *(int32*)(minval) = *(int32*)(maxval) = DatumGetInt32(v); *first = false; } @@ -804,8 +1056,16 @@ static void CompareInt32(char* minval, char* maxval, Datum v, bool* first, int* * @IN v: the new int32 value * @See also: */ +/* + * @Description: 用于设置数据类型为 int32 的最小值和最大值。 + * @IN/OUT cuDescPtr: CU 描述符 + * @IN/OUT first: 第一个值的标志位 + * @IN v: 新的 int32 值 + * @See also: + */ static void SetMinMaxInt32(Datum v, CUDesc* cuDescPtr, bool* first) { + // 调用 CompareInt32 函数，将新的 int32 值与当前的最小值和最大值进行比较，并更新最小值和最大值 CompareInt32(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL); } @@ -818,16 +1078,29 @@ static void SetMinMaxInt32(Datum v, CUDesc* cuDescPtr, bool* first) * @IN/OUT varstr_maxlen: max length about var-length string * @See also: */ +/* + * @Description: 用于比较数据类型为 uint32 的比较函数。 + * @IN/OUT first: 表示是否是第一个值的标志位 + * @IN/OUT maxval: 当前最大值 + * @IN/OUT minval: 当前最小值 + * @IN v: 新的 uint32 值 + * @IN/OUT varstr_maxlen: 变长字符串的最大长度 + * @See also: + */ static void CompareUint32(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { + // 如果不是第一个值 if (!*first) { uint32 uintVal = DatumGetUInt32(v); if (uintVal < *(uint32*)(minval)) { + // 更新最小值 *(uint32*)(minval) = uintVal; } else if (uintVal > *(uint32*)(maxval)) { + // 更新最大值 *(uint32*)(maxval) = uintVal; } } else { + // 如果是第一个值，则将最小值和最大值都设置为新值 *(uint32*)(minval) = *(uint32*)(maxval) = DatumGetUInt32(v); *first = false; } @@ -841,8 +1114,16 @@ static void CompareUint32(char* minval, char* maxval, Datum v, bool* first, int* * @IN v: the new uint32 value * @See also: */ +/* + * @Description: 用于设置数据类型为 uint32 的最小值和最大值。 + * @IN/OUT cuDescPtr: CU 描述符 + * @IN/OUT first: 第一个值的标志位 + * @IN v: 新的 uint32 值 + * @See also: + */ void SetMinMaxUint32(Datum v, CUDesc* cuDescPtr, bool* first) { + // 调用 CompareUint32 函数来比较并更新最小值和最大值 CompareUint32(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL); } @@ -857,14 +1138,18 @@ void SetMinMaxUint32(Datum v, CUDesc* cuDescPtr, bool* first) */ static void CompareInt16(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen) { + // 如果不是第一个值 if (!*first) { int16 intVal = DatumGetInt16(v); if (intVal < *(int16*)minval) { + // 更新最小值 *(int16*)(minval) = intVal; } else if (intVal > *(int16*)maxval) { + // 更新最大值 *(int16*)(maxval) = intVal; } } else { + // 如果是第一个值，则将最小值和最大值都设置为新值 *(int16*)(minval) = *(int16*)(maxval) = DatumGetInt16(v); *first = false; } @@ -878,7 +1163,15 @@ static void CompareInt16(char* minval, char* maxval, Datum v, bool* first, int* * @IN v: the new int16 value * @See also: */ +/* + * @Description: 用于设置数据类型为 int16 的最小值和最大值。 + * @IN/OUT cuDescPtr: CU 描述符 + * @IN/OUT first: 第一个值的标志位 + * @IN v: 新的 int16 值 + * @See also: + */ static void SetMinMaxInt16(Datum v, CUDesc* cuDescPtr, bool* first) { + // 调用 CompareInt16 函数，设置最小值和最大值 CompareInt16(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL); } -- 2.34.1 From 9a08f5f0bceb529639f3a3e8df306ef79feaa468 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Wed, 4 Oct 2023 05:21:18 +0800 Subject: [PATCH 42/48] Update cstore_psort.cpp --- .../storage/cstore/cstore_psort.cpp | 270 ++++++++++-------- 1 file changed, 148 insertions(+), 122 deletions(-) diff --git a/src/gausskernel/storage/cstore/cstore_psort.cpp b/src/gausskernel/storage/cstore/cstore_psort.cpp index def061569..2e74ced86 100644 --- a/src/gausskernel/storage/cstore/cstore_psort.cpp +++ b/src/gausskernel/storage/cstore/cstore_psort.cpp @@ -31,18 +31,18 @@ #include "utils/typcache.h" CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int type, MemInfoArg* m_memInfo) - : m_tupleSortState(NULL), - m_batchSortState(NULL), - m_vecBatch(NULL), - m_tupleSlot(NULL), - m_type(type), - m_rel(rel), - m_sortKeys(sortKeys), - m_keyNum(keyNum), - m_curSortedRowNum(0), - m_vecBatchCursor(InvalidBathCursor) + : m_tupleSortState(NULL),// 初始化元组排序状态为NULL + m_batchSortState(NULL),// 初始化批次排序状态为NULL + m_vecBatch(NULL),// 初始化向量批次为NULL + m_tupleSlot(NULL),// 初始化元组插槽为NULL + m_type(type),// 初始化排序类型 + m_rel(rel),// 初始化关系对象 + m_sortKeys(sortKeys),// 初始化排序键数组 + m_keyNum(keyNum),// 初始化排序键数量 + m_curSortedRowNum(0),// 初始化当前已排序行数为0 + m_vecBatchCursor(InvalidBathCursor)// 初始化向量批次游标为无效值 { - m_tupDesc = m_rel->rd_att; + m_tupDesc = m_rel->rd_att;// 获取关系的属性描述符 Form_pg_attribute* attr = m_tupDesc->attrs; m_psortMemInfo = NULL; @@ -55,6 +55,7 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ // Note that these variables should be in parent memoryContex. // please free them in deconstructor method. // + // 为排序操作符和排序规则分配内存并设置相应的变量 m_sortOperators = (Oid*)palloc(sizeof(Oid) * keyNum); m_sortCollations = (Oid*)palloc(sizeof(Oid) * keyNum); for (int i = 0; i < keyNum; ++i) { @@ -64,7 +65,7 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ TypeCacheEntry* typeEntry = lookup_type_cache(attr[colIdx]->atttypid, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); m_sortOperators[i] = typeEntry->lt_opr; } - + // 分配内存并初始化相应的变量 m_nullsFirst = (bool*)palloc(sizeof(bool) * m_keyNum); errno_t rc = memset_s(m_nullsFirst, m_keyNum * sizeof(bool), false, m_keyNum * sizeof(bool)); securec_check(rc, "", ""); @@ -72,7 +73,7 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ m_val = (Datum*)palloc(sizeof(Datum) * m_tupDesc->natts); m_null = (bool*)palloc(sizeof(bool) * m_tupDesc->natts); - InitPsortMemArg(m_memInfo); + InitPsortMemArg(m_memInfo);// 初始化排序所需的内存信息 int sortMem = m_psortMemInfo->MemSort > 0 ? m_psortMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem; int canSpreadmaxMem = m_psortMemInfo->canSpreadmaxMem; /* @@ -81,6 +82,7 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ * so for some partitions with large data volume, we can expand by 20% by default. * In extreme cases, it is possible to exceed 20% per zone for canSpreadmaxMem. */ + // 根据排序类型创建相应类型的排序状态对象并初始化相应的函数指针 if (canSpreadmaxMem && m_psortMemInfo->partitionNum > 1) { canSpreadmaxMem = (int)((double)m_psortMemInfo->canSpreadmaxMem / (double)m_psortMemInfo->partitionNum * PSORT_SPREAD_MAXMEM_RATIO); @@ -95,9 +97,10 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ // WARNING: all the following variables should be regenerated // after m_psortMemContext is reset. // - AutoContextSwitch memContextGuard(m_psortMemContext); + AutoContextSwitch memContextGuard(m_psortMemContext);// 创建自动内存上下文切换器 - if (m_type == TUPLE_SORT) { + if (m_type == TUPLE_SORT) {// 如果排序类型为元组排序 + // 创建元组排序状态对象，并设置相应的回调函数指针 m_tupleSortState = tuplesort_begin_heap(m_tupDesc, m_keyNum, m_sortKeys, @@ -108,12 +111,13 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ false, canSpreadmaxMem); - m_funcGetBatchValue = &CStorePSort::GetBatchValueFromTupleSort; + m_funcGetBatchValue = &CStorePSort::GetBatchValueFromTupleSort;// 设置获取批次值的回调函数指针 - m_funcReset = &CStorePSort::ResetTupleSortState; + m_funcReset = &CStorePSort::ResetTupleSortState;// 设置重置排序状态的回调函数指针 - m_tupleSlot = MakeSingleTupleTableSlot(m_tupDesc); - } else { + m_tupleSlot = MakeSingleTupleTableSlot(m_tupDesc);// 创建单个元组插槽 + } else {// 否则，即排序类型为批次排序 + // 创建批次排序状态对象，并设置相应的回调函数指针 m_batchSortState = batchsort_begin_heap(m_tupDesc, m_keyNum, m_sortKeys, @@ -124,11 +128,11 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ false, canSpreadmaxMem); - m_funcGetBatchValue = &CStorePSort::GetBatchValueFromBatchSort; + m_funcGetBatchValue = &CStorePSort::GetBatchValueFromBatchSort;// 设置获取批次值的回调函数指针 - m_funcReset = &CStorePSort::ResetBatchSortState; + m_funcReset = &CStorePSort::ResetBatchSortState;// 设置重置排序状态的回调函数指针 - m_vecBatch = New(CurrentMemoryContext) VectorBatch(CurrentMemoryContext, m_tupDesc); + m_vecBatch = New(CurrentMemoryContext) VectorBatch(CurrentMemoryContext, m_tupDesc);// 创建新的向量批次 } } @@ -149,26 +153,26 @@ CStorePSort::~CStorePSort() m_sortCollations = NULL; m_psortMemContext = NULL; } - +//销毁CStorePSort对象，释放相关的内存资源 void CStorePSort::Destroy() { if (m_type == TUPLE_SORT && m_tupleSlot != NULL) { - ExecDropSingleTupleTableSlot(m_tupleSlot); + ExecDropSingleTupleTableSlot(m_tupleSlot);// 释放单个元组插槽的内存 m_tupleSlot = NULL; } - + // 释放在非 psortMemContext 内分配的内存 // free memory not alloc in m_psortMemContext - pfree_ext(m_sortOperators); - pfree_ext(m_sortCollations); - pfree_ext(m_nullsFirst); - pfree_ext(m_val); - pfree_ext(m_null); + pfree_ext(m_sortOperators);// 释放排序操作符数组的内存 + pfree_ext(m_sortCollations);// 释放排序规则数组的内存 + pfree_ext(m_nullsFirst);// 释放空值排序顺序数组的内存 + pfree_ext(m_val);// 释放值数组的内存 + pfree_ext(m_null);// 释放空值标志数组的内存 if (m_psortMemInfo) { - pfree_ext(m_psortMemInfo); + pfree_ext(m_psortMemInfo);// 释放排序内存信息结构体的内存 } - + // 释放在 psortMemContext 内分配的内存 // free memory alloc in m_psortMemContext - MemoryContextDelete(m_psortMemContext); + MemoryContextDelete(m_psortMemContext);// 删除排序内存上下文 } /* @@ -179,8 +183,10 @@ void CStorePSort::Destroy() * @sort mem for cstore table and partition tables. * @Return: void */ +// void CStorePSort::InitPsortMemArg(MemInfoArg* ArgmemInfo) { + // 初始化psort的内存信息 m_psortMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg)); if (ArgmemInfo != NULL) { Assert(ArgmemInfo->partitionNum > 0); @@ -188,6 +194,7 @@ void CStorePSort::InitPsortMemArg(MemInfoArg* ArgmemInfo) * m_psortMemInfo->canSpreadmaxMem is all partitions sort spread Mem for partition table. * m_psortMemInfo->MemSort is one partition sort Mem for partition table. */ + // 计算可扩展的最大内存和分区排序内存 m_psortMemInfo->canSpreadmaxMem = (ArgmemInfo->canSpreadmaxMem - ArgmemInfo->MemInsert > 0) ? ArgmemInfo->canSpreadmaxMem - ArgmemInfo->MemInsert : 0; @@ -195,12 +202,14 @@ void CStorePSort::InitPsortMemArg(MemInfoArg* ArgmemInfo) m_psortMemInfo->MemSort = ArgmemInfo->MemSort; m_psortMemInfo->spreadNum = ArgmemInfo->spreadNum; m_psortMemInfo->partitionNum = ArgmemInfo->partitionNum; + // 打印初始化的内存信息 MEMCTL_LOG(DEBUG2, "CStorePSort(init ArgmemInfo):Insert workmem is : %dKB, sort workmem: %dKB,can spread maxMem is %dKB.", m_psortMemInfo->MemInsert, m_psortMemInfo->MemSort, m_psortMemInfo->canSpreadmaxMem); } else { + // 未传入有效的内存信息参数，使用默认值 m_psortMemInfo->canSpreadmaxMem = 0; m_psortMemInfo->MemInsert = 0; m_psortMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem; @@ -208,152 +217,156 @@ void CStorePSort::InitPsortMemArg(MemInfoArg* ArgmemInfo) m_psortMemInfo->partitionNum = 1; } } - +//重置排序状态 void CStorePSort::Reset(bool endFlag) { - return (this->*m_funcReset)(endFlag); + return (this->*m_funcReset)(endFlag);// 通过函数指针调用Reset函数的具体实现，传入endFlag参数 } - +//将向量批次数据放入排序状态中 void CStorePSort::PutVecBatch(Relation rel, VectorBatch* pVecBatch) { - Assert(m_batchSortState && m_type == BATCH_SORT); + Assert(m_batchSortState && m_type == BATCH_SORT);// 断言确保批次排序状态不为空且类型为BATCH_SORT + // 调用sort_putbatch函数将向量批次数据放入批次排序状态中 m_batchSortState->sort_putbatch(m_batchSortState, pVecBatch, 0, pVecBatch->m_rows); + // 更新已排序行数 m_curSortedRowNum += pVecBatch->m_rows; } - +// 将元组数据放入排序状态中 void CStorePSort::PutTuple(Datum* values, bool* nulls) { - Assert(m_tupleSortState && m_type == TUPLE_SORT); + Assert(m_tupleSortState && m_type == TUPLE_SORT);// 断言确保元组排序状态不为空且类型为TUPLE_SORT - AutoContextSwitch memContextGuard(m_psortMemContext); + AutoContextSwitch memContextGuard(m_psortMemContext);// 切换上下文到psort内存上下文 - HeapTuple tuple = (HeapTuple)tableam_tops_form_tuple(m_tupDesc, values, nulls, HEAP_TUPLE); + HeapTuple tuple = (HeapTuple)tableam_tops_form_tuple(m_tupDesc, values, nulls, HEAP_TUPLE);// 从元组描述符，值和空值创建堆元组 - TupleTableSlot* slot = MakeSingleTupleTableSlot(m_tupDesc); + TupleTableSlot* slot = MakeSingleTupleTableSlot(m_tupDesc);// 创建单个元组表格插槽 - (void)ExecStoreTuple(tuple, slot, InvalidBuffer, false); + (void)ExecStoreTuple(tuple, slot, InvalidBuffer, false);// 将堆元组存储到表格插槽中 - tuplesort_puttupleslot(m_tupleSortState, slot); + tuplesort_puttupleslot(m_tupleSortState, slot);// 将表格插槽的元组加入元组排序状态中 - ExecDropSingleTupleTableSlot(slot); + ExecDropSingleTupleTableSlot(slot);// 释放表格插槽 - heap_freetuple(tuple); + heap_freetuple(tuple);// 释放堆元组 tuple = NULL; } - +// 将单个元组数据放入排序状态中，并更新已排序行数 void CStorePSort::PutSingleTuple(Datum* values, bool* nulls) { - PutTuple(values, nulls); - m_curSortedRowNum++; + PutTuple(values, nulls);// 调用PutTuple将元组数据放入排序状态中 + m_curSortedRowNum++;// 更新已排序行数 } - +// 将批次数据放入排序状态中 void CStorePSort::PutBatchValues(bulkload_rows* batchRowPtr) { - AutoContextSwitch memContextGuard(m_psortMemContext); + AutoContextSwitch memContextGuard(m_psortMemContext);// 切换上下文到psort内存上下文 bulkload_rows_iter iter; - iter.begin(batchRowPtr); + iter.begin(batchRowPtr);// 设置迭代器的起点 while (iter.not_end()) { /* fetch next tuple */ - iter.next(m_val, m_null); + iter.next(m_val, m_null);// 从批次中获取下一个元组 /* put tuple into sort processor */ - PutTuple(m_val, m_null); + PutTuple(m_val, m_null);// 将元组数据放入排序状态中 } - iter.end(); + iter.end();// 设置迭代器的终点 - m_curSortedRowNum += batchRowPtr->m_rows_curnum; + m_curSortedRowNum += batchRowPtr->m_rows_curnum;// 更新已排序行数 } - +// 执行排序操作 void CStorePSort::RunSort() { - AutoContextSwitch memContextGuard(m_psortMemContext); + AutoContextSwitch memContextGuard(m_psortMemContext);// 切换上下文到psort内存上下文 if (m_type == TUPLE_SORT) - tuplesort_performsort(m_tupleSortState); + tuplesort_performsort(m_tupleSortState);// 对元组排序状态执行排序操作 else - batchsort_performsort(m_batchSortState); + batchsort_performsort(m_batchSortState);// 对批次排序状态执行排序操作 } - +// 获取批次值 void CStorePSort::GetBatchValue(bulkload_rows* batchRowsPtr) { - return (this->*m_funcGetBatchValue)(batchRowsPtr); + return (this->*m_funcGetBatchValue)(batchRowsPtr);// 调用函数指针指向的函数，获取批次值 } - +// 获取向量批次 VectorBatch* CStorePSort::GetVectorBatch() { - Assert(m_batchSortState && m_vecBatch && m_type == BATCH_SORT); + Assert(m_batchSortState && m_vecBatch && m_type == BATCH_SORT);// 断言：批次排序状态、向量批次和排序类型必须符合条件 if (unlikely(!(m_tupleSortState && m_tupleSlot && m_type == TUPLE_SORT))) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Invalid tupleSortState, tupleSlot, or sorting strategy while getting tuple in cstore psort."))); } - m_vecBatch->Reset(true); - batchsort_getbatch(m_batchSortState, true, m_vecBatch); - return m_vecBatch; + m_vecBatch->Reset(true);// 重置向量批次，清空数据 + batchsort_getbatch(m_batchSortState, true, m_vecBatch);// 获取批次排序状态中的批次数据 + return m_vecBatch;// 返回向量批次 } - +// 获取元组 TupleTableSlot* CStorePSort::GetTuple() { - Assert(m_tupleSortState && m_tupleSlot && m_type == TUPLE_SORT); + Assert(m_tupleSortState && m_tupleSlot && m_type == TUPLE_SORT);// 断言：元组排序状态、元组插槽和排序类型必须符合条件 if (unlikely(!(m_tupleSortState && m_tupleSlot && m_type == TUPLE_SORT))) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("Invalid tupleSortState, tupleSlot, or sorting strategy while getting tuple in cstore psort."))); } - if (!TupIsNull(m_tupleSlot)) + if (!TupIsNull(m_tupleSlot))// 如果当前元组插槽不为空，则清空插槽中的元组数据 (void)ExecClearTuple(m_tupleSlot); - (void)tuplesort_gettupleslot(m_tupleSortState, true, m_tupleSlot, NULL); - return m_tupleSlot; + (void)tuplesort_gettupleslot(m_tupleSortState, true, m_tupleSlot, NULL);// 从元组排序状态中获取下一个元组，并存储在元组插槽中 + return m_tupleSlot;// 返回元组插槽 } - +// 检查排序是否已满 bool CStorePSort::IsFull() const { - return m_curSortedRowNum >= m_partialClusterRowNum; + return m_curSortedRowNum >= m_partialClusterRowNum;// 如果当前已排序行数大于等于部分簇的行数，则认为已满，返回true，否则返回false } - +// 获取已排序的行数 int CStorePSort::GetRowNum() const { - return m_curSortedRowNum; + return m_curSortedRowNum;// 返回当前已排序的行数 } #ifdef USE_ASSERT_CHECKING +// 在使用断言进行检查时，确保部分簇的行数大于等于完整簇的大小，并且部分簇的行数必须是完整簇大小的整数倍 inline void CStorePSort::AssertCheck() { - Assert((m_partialClusterRowNum >= m_fullCUSize) && ((m_partialClusterRowNum % m_fullCUSize) == 0)); + Assert((m_partialClusterRowNum >= m_fullCUSize) && ((m_partialClusterRowNum % m_fullCUSize) == 0));// 断言：部分簇的行数大于等于完整簇的大小，并且部分簇的行数必须是完整簇大小的整数倍 } #endif - +//重置元组排序状态 void CStorePSort::ResetTupleSortState(bool endFlag) { - AutoContextSwitch memContextGuard(m_psortMemContext); - int sortMem = u_sess->attr.attr_storage.psort_work_mem; - int canSpreadMaxMem = 0; + AutoContextSwitch memContextGuard(m_psortMemContext);// 自动内存上下文切换保护对象 + int sortMem = u_sess->attr.attr_storage.psort_work_mem;// 排序使用的内存大小，默认为系统设置的psort_work_mem值 + int canSpreadMaxMem = 0;// 可以扩展的最大内存大小，默认为0 if (m_tupleSortState) { - sortMem = m_psortMemInfo->MemSort > 0 ? m_psortMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem; - canSpreadMaxMem = m_psortMemInfo->canSpreadmaxMem > 0 ? m_psortMemInfo->canSpreadmaxMem : 0; + sortMem = m_psortMemInfo->MemSort > 0 ? m_psortMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem;// 如果设置了排序内存大小，则使用设置值；否则使用系统设置的psort_work_mem值 + canSpreadMaxMem = m_psortMemInfo->canSpreadmaxMem > 0 ? m_psortMemInfo->canSpreadmaxMem : 0;// 如果设置了可以扩展的最大内存大小，则使用设置值；否则默认为0 - if (canSpreadMaxMem && m_psortMemInfo->partitionNum > 1) { + if (canSpreadMaxMem && m_psortMemInfo->partitionNum > 1) {// 如果可以扩展的最大内存大小大于0，并且分区数大于1 canSpreadMaxMem = (int)((double)m_psortMemInfo->canSpreadmaxMem / (double)m_psortMemInfo->partitionNum * - PSORT_SPREAD_MAXMEM_RATIO); + PSORT_SPREAD_MAXMEM_RATIO);// 根据分区数比例计算每个分区可以扩展的最大内存大小 } - tuplesort_end(m_tupleSortState); + tuplesort_end(m_tupleSortState);// 结束元组排序 m_tupleSortState = NULL; } - + // 清除 m_tupleSlot // clear m_tupleSlot if (m_tupleSlot != NULL) { - ExecDropSingleTupleTableSlot(m_tupleSlot); + ExecDropSingleTupleTableSlot(m_tupleSlot);// 释放单个元组表插槽 m_tupleSlot = NULL; } // clear memory contex - MemoryContextReset(m_psortMemContext); + // 清除内存上下文 + MemoryContextReset(m_psortMemContext);// 重置内存上下文 - if (!endFlag) { - m_tupleSortState = tuplesort_begin_heap(m_tupDesc, + if (!endFlag) {// 如果不是结束标志 + m_tupleSortState = tuplesort_begin_heap(m_tupDesc,// 开始堆排序 m_keyNum, m_sortKeys, m_sortOperators, @@ -363,32 +376,35 @@ void CStorePSort::ResetTupleSortState(bool endFlag) false, canSpreadMaxMem); - m_tupleSlot = MakeSingleTupleTableSlot(m_tupDesc); + m_tupleSlot = MakeSingleTupleTableSlot(m_tupDesc);// 创建单个元组表插槽 } - m_curSortedRowNum = 0; + m_curSortedRowNum = 0;// 当前已排序的行数归零 } - +//重置批量排序状态 void CStorePSort::ResetBatchSortState(bool endFlag) { - AutoContextSwitch memContextGuard(m_psortMemContext); - int sortMem = u_sess->attr.attr_storage.psort_work_mem; - int canSpreadMaxMem = 0; + AutoContextSwitch memContextGuard(m_psortMemContext);// 切换到排序过程所使用的内存上下文。 + int sortMem = u_sess->attr.attr_storage.psort_work_mem;// 获取排序所使用的内存大小。 + int canSpreadMaxMem = 0;// 可扩展的最大内存大小先设为0。 - if (m_batchSortState) { + if (m_batchSortState) {// 如果已存在批量排序状态(m_batchSortState不为NULL)。 + // 根据情况确定排序使用的内存大小和可扩展的最大内存大小。 sortMem = m_psortMemInfo->MemSort > 0 ? m_psortMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem; + // 判断是否有可扩展的最大内存，并且分区数大于1，则设定可扩展的最大内存大小。 canSpreadMaxMem = m_psortMemInfo->canSpreadmaxMem > 0 ? m_psortMemInfo->canSpreadmaxMem : 0; if (canSpreadMaxMem && m_psortMemInfo->partitionNum > 1) { canSpreadMaxMem = (int)((double)m_psortMemInfo->canSpreadmaxMem / (double)m_psortMemInfo->partitionNum * PSORT_SPREAD_MAXMEM_RATIO); } - batchsort_end(m_batchSortState); - m_batchSortState = NULL; + batchsort_end(m_batchSortState);// 结束排序。 + m_batchSortState = NULL;// 将批量排序状态置为NULL。 } // clear memory contex - MemoryContextReset(m_psortMemContext); + MemoryContextReset(m_psortMemContext);// 重置排序过程所使用的内存上下文。 - if (!endFlag) { + if (!endFlag) {// 如果不是结束排序(endFlag为false)。 + // 通过batchsort_begin_heap函数开始堆排序，设定元组描述符、键值个数、排序键、排序算子、分区方法、是否nullsFirst、排序内存大小、是否并行排序、可扩展的最大内存大小。 m_batchSortState = batchsort_begin_heap(m_tupDesc, m_keyNum, m_sortKeys, @@ -398,49 +414,52 @@ void CStorePSort::ResetBatchSortState(bool endFlag) sortMem, false, canSpreadMaxMem); - + // 创建与元组描述符对应的空批次(m_vecBatch)。 m_vecBatch = New(CurrentMemoryContext) VectorBatch(CurrentMemoryContext, m_tupDesc); } - m_curSortedRowNum = 0; - m_vecBatchCursor = InvalidBathCursor; + m_curSortedRowNum = 0;// 将当前已排序的行数(m_curSortedRowNum)归零。 + m_vecBatchCursor = InvalidBathCursor;// 将批次游标指向无效位置。 } - +//在外部排序中，从元组排序中逐个读取批次的元组数据，并将其添加到bulkload_rows类实例中。循环直到读取完所有元组或者批次数据已满 void CStorePSort::GetBatchValueFromTupleSort(bulkload_rows* batchRowsPtr) { - AutoContextSwitch memContextGuard(m_psortMemContext); - TupleTableSlot* slot = MakeSingleTupleTableSlot(m_tupDesc); + AutoContextSwitch memContextGuard(m_psortMemContext);// 切换到排序过程所使用的内存上下文。 + TupleTableSlot* slot = MakeSingleTupleTableSlot(m_tupDesc);// 创建一个单个元组的TupleTableSlot。 // here it isn't a dead loop. // when batchRowsPtr is full, break down the loops. - while (true) { - if (!tuplesort_gettupleslot(m_tupleSortState, true, slot, NULL)) { - break; + while (true) {// 循环读取批次数据。 + if (!tuplesort_gettupleslot(m_tupleSortState, true, slot, NULL)) {// 从元组排序中获取下一个元组至slot中。 + break;// 如果获取不到元组，则退出循环。 } - heap_deform_tuple((HeapTuple)slot->tts_tuple, m_tupDesc, m_val, m_null); - if (batchRowsPtr->append_one_tuple(m_val, m_null, m_tupDesc)) - break; + heap_deform_tuple((HeapTuple)slot->tts_tuple, m_tupDesc, m_val, m_null);// 将元组解压为属性值和NULL标记。 + if (batchRowsPtr->append_one_tuple(m_val, m_null, m_tupDesc))// 将属性值和NULL标记添加到bulkload_rows中。 + break;// 如果添加完成，则退出循环。 } - ExecDropSingleTupleTableSlot(slot); + ExecDropSingleTupleTableSlot(slot);// 释放TupleTableSlot的资源。 } - +//从批量排序中逐个读取批次的元组数据，并将其添加到bulkload_rows类实例中。循环直到读取完所有元组或者批次数据已满 void CStorePSort::GetBatchValueFromBatchSort(bulkload_rows* batchRowsPtr) { - AutoContextSwitch memContextGuard(m_psortMemContext); + AutoContextSwitch memContextGuard(m_psortMemContext);// 切换到排序过程所使用的内存上下文。 // precondition for the following WHILE loop // - Assert(batchRowsPtr->m_rows_maxnum > 0); - Assert(batchRowsPtr->m_rows_maxnum % BatchMaxSize == 0); - bool is_start = true; + Assert(batchRowsPtr->m_rows_maxnum > 0);// 确保批次中的行数大于0。 + Assert(batchRowsPtr->m_rows_maxnum % BatchMaxSize == 0);// 确保批次的行数是BatchMaxSize的整数倍。 + bool is_start = true;// 标记是否是第一次获取批次数据。 if (BathCursorIsValid(m_vecBatchCursor)) { // first fetch tuples from last vector batch. + // 从上次处理的向量批次中获取元组。 if (batchRowsPtr->append_one_vector(m_tupDesc, m_vecBatch, &m_vecBatchCursor)) { // it's filled with part of VectBatch tuples. // m_vecBatchCursor has been updated, and we have to // care the special case. reset when all the m_vecBatch is handled. + // 如果批次已满，则部分向量批次已填充。 + // 更新m_vecBatchCursor，需要注意特殊情况下的重置操作。 if (m_vecBatchCursor == m_vecBatch->m_rows) { m_vecBatchCursor = InvalidBathCursor; } @@ -449,33 +468,40 @@ void CStorePSort::GetBatchValueFromBatchSort(bulkload_rows* batchRowsPtr) // it's fine to continue to hold more tuples. // first of all reset this cursor. + // 继续获取更多元组。 + // 首先重置游标。 m_vecBatchCursor = InvalidBathCursor; is_start = false; } // here it isn't a dead loop. // when batchRowsPtr is full, break down the loops. + // 进入循环直至批次数据填满或无可用元组。 while (true) { Assert(m_vecBatchCursor == InvalidBathCursor); + // 从批量排序中获取向量批次。 batchsort_getbatch(m_batchSortState, true, m_vecBatch); if (BatchIsNull(m_vecBatch)) { + // 如果没有可用的元组，则跳出循环。 // break if there isn't any tuple break; } if (is_start) { - Assert(batchRowsPtr->m_rows_curnum % BatchMaxSize == 0); + Assert(batchRowsPtr->m_rows_curnum % BatchMaxSize == 0);// 确保当前批次的行数是BatchMaxSize的整数倍。 } // we have to copy datum, because it belongs to // m_vecBatch which is reset by batchsort_getbatch(). // free all the space by calling Reset(), so see its references. // - m_vecBatchCursor = 0; + m_vecBatchCursor = 0;// 将向量批次游标重置为0。 if (batchRowsPtr->append_one_vector(m_tupDesc, m_vecBatch, &m_vecBatchCursor)) { // ok, it's filled with part of VectBatch tuples. // m_vecBatchCursor has been updated, and we have to // care the special case. reset when all the m_vecBatch is handled. + // 批次已满，部分向量批次已填充。 + // 更新m_vecBatchCursor，需要注意特殊情况下的重置操作。 if (m_vecBatchCursor == m_vecBatch->m_rows) { m_vecBatchCursor = InvalidBathCursor; } -- 2.34.1 From f0a0a6f0ee8e1d11b0e605c4368508c505c8bcaa Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Wed, 4 Oct 2023 05:21:38 +0800 Subject: [PATCH 43/48] Update cstore_roughcheck_func.cpp --- .../storage/cstore/cstore_roughcheck_func.cpp | 168 ++++++++++-------- 1 file changed, 91 insertions(+), 77 deletions(-) diff --git a/src/gausskernel/storage/cstore/cstore_roughcheck_func.cpp b/src/gausskernel/storage/cstore/cstore_roughcheck_func.cpp index a0d71829a..14ef4634c 100644 --- a/src/gausskernel/storage/cstore/cstore_roughcheck_func.cpp +++ b/src/gausskernel/storage/cstore/cstore_roughcheck_func.cpp @@ -52,12 +52,13 @@ template bool RoughCheckIntCU(const T& min, const T& max, const T& arg); template bool RoughCheckFloatCU(const T& min, const T& max, const T& arg); - +//根据给定的typeOid（数据类型OID）、strategy（比较策略）和collation（排序规则OID），返回对应的RoughCheckFunc函数指针 RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation) { switch (typeOid) { case CHAROID: { switch (strategy) { + // 在CHAR类型下，根据不同的比较策略返回不同的RoughCheckCharCU函数指针 case CStoreLessStrategyNumber: return RoughCheckCharCU; case CStoreLessEqualStrategyNumber: @@ -75,6 +76,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation) } case INT2OID: { switch (strategy) { + // 在INT2类型下，根据不同的比较策略返回不同的RoughCheckInt16CU函数指针 case CStoreLessStrategyNumber: return RoughCheckInt16CU; case CStoreLessEqualStrategyNumber: @@ -92,6 +94,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation) } case INT4OID: { switch (strategy) { + // 在INT4类型下，根据不同的比较策略返回不同的RoughCheckInt32CU函数指针 case CStoreLessStrategyNumber: return RoughCheckInt32CU; case CStoreLessEqualStrategyNumber: @@ -109,6 +112,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation) } case INT8OID: { switch (strategy) { + // 在INT8类型下，根据不同的比较策略返回不同的RoughCheckInt64CU函数指针 case CStoreLessStrategyNumber: return RoughCheckInt64CU; case CStoreLessEqualStrategyNumber: @@ -126,6 +130,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation) } case OIDOID: { switch (strategy) { + // 在OID类型下，根据不同的比较策略返回不同的RoughCheckUInt32CU函数指针 case CStoreLessStrategyNumber: return RoughCheckUInt32CU; case CStoreLessEqualStrategyNumber: @@ -143,6 +148,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation) } case DATEOID: { switch (strategy) { + // 在DATE类型下，根据不同的比较策略返回不同的RoughCheckDateCU函数指针 case CStoreLessStrategyNumber: return RoughCheckDateCU; case CStoreLessEqualStrategyNumber: @@ -160,6 +166,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation) } case TIMEOID: { switch (strategy) { + // 在TIME类型下，根据不同的比较策略返回不同的RoughCheckTimeCU函数指针 case CStoreLessStrategyNumber: return RoughCheckTimeCU; case CStoreLessEqualStrategyNumber: @@ -177,6 +184,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation) } case TIMESTAMPOID: { switch (strategy) { + // 在TIMESTAMP类型下，根据不同的比较策略返回不同的RoughCheckTimestampCU函数指针 case CStoreLessStrategyNumber: return RoughCheckTimestampCU; case CStoreLessEqualStrategyNumber: @@ -194,6 +202,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation) } case TIMESTAMPTZOID: { switch (strategy) { + // 在TIMESTAMPTZ类型下，根据不同的比较策略返回不同的RoughCheckTimestampTzCU函数指针 case CStoreLessStrategyNumber: return RoughCheckTimestampTzCU; case CStoreLessEqualStrategyNumber: @@ -216,6 +225,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation) return RoughCheckAllThrough; switch (strategy) { + // 在BPCHAR、VARCHAR和TEXT类型下，根据不同的比较策略返回不同的RoughCheckStringCU函数指针 case CStoreLessStrategyNumber: return RoughCheckStringCU; case CStoreLessEqualStrategyNumber: @@ -236,180 +246,184 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation) } } } - +//对char类型的参数进行粗略检查 template bool RoughCheckCharCU(CUDesc* cudesc, Datum arg) { - char min = *(char*)cudesc->cu_min; - char max = *(char*)cudesc->cu_max; + char min = *(char*)cudesc->cu_min;// 获取CUDesc结构体中的最小值 + char max = *(char*)cudesc->cu_max;// 获取CUDesc结构体中的最大值 - return RoughCheckIntCU(min, max, DatumGetChar(arg)); + return RoughCheckIntCU(min, max, DatumGetChar(arg));// 调用RoughCheckIntCU函数，对char类型的值进行粗略检查 } - +//对int16类型的参数进行粗略检查 template bool RoughCheckInt16CU(CUDesc* cudesc, Datum arg) { - int16 min = *(int16*)cudesc->cu_min; - int16 max = *(int16*)cudesc->cu_max; + int16 min = *(int16*)cudesc->cu_min;// 获取CUDesc结构体中的最小值 + int16 max = *(int16*)cudesc->cu_max;// 获取CUDesc结构体中的最大值 - return RoughCheckIntCU(min, max, DatumGetInt64(arg)); + return RoughCheckIntCU(min, max, DatumGetInt64(arg));// 调用RoughCheckIntCU函数，对int16类型的值进行粗略检查 } - +//对int32类型的参数进行粗略检查 template bool RoughCheckInt32CU(CUDesc* cudesc, Datum arg) { - int32 min = *(int32*)cudesc->cu_min; - int32 max = *(int32*)cudesc->cu_max; + int32 min = *(int32*)cudesc->cu_min;// 获取CUDesc结构体中的最小值 + int32 max = *(int32*)cudesc->cu_max;// 获取CUDesc结构体中的最大值 - return RoughCheckIntCU(min, max, DatumGetInt64(arg)); + return RoughCheckIntCU(min, max, DatumGetInt64(arg));// 调用RoughCheckIntCU函数，对int32类型的值进行粗略检查 } - +//对int64类型的参数进行粗略检查 template bool RoughCheckInt64CU(CUDesc* cudesc, Datum arg) { - int64 min = *(int64*)cudesc->cu_min; - int64 max = *(int64*)cudesc->cu_max; + int64 min = *(int64*)cudesc->cu_min;// 获取CUDesc结构体中的最小值 + int64 max = *(int64*)cudesc->cu_max;// 获取CUDesc结构体中的最大值 - return RoughCheckIntCU(min, max, DatumGetInt64(arg)); + return RoughCheckIntCU(min, max, DatumGetInt64(arg));// 调用RoughCheckIntCU函数，对int64类型的值进行粗略检查 } - +//对uint32类型的参数进行粗略检查 template bool RoughCheckUInt32CU(CUDesc* cudesc, Datum arg) { - uint32 min = *(uint32*)cudesc->cu_min; - uint32 max = *(uint32*)cudesc->cu_max; + uint32 min = *(uint32*)cudesc->cu_min;// 获取CUDesc结构体中的最小值 + uint32 max = *(uint32*)cudesc->cu_max;// 获取CUDesc结构体中的最大值 - return RoughCheckIntCU(min, max, DatumGetInt64(arg)); + return RoughCheckIntCU(min, max, DatumGetInt64(arg));// 调用RoughCheckIntCU函数，对uint32类型的值进行粗略检查 } - +} +//对DateADT类型（日期类型）的参数进行粗略检查 template bool RoughCheckDateCU(CUDesc* cudesc, Datum arg) { - DateADT min = *(DateADT*)cudesc->cu_min; - DateADT max = *(DateADT*)cudesc->cu_max; + DateADT min = *(DateADT*)cudesc->cu_min;// 获取CUDesc结构体中的最小值 + DateADT max = *(DateADT*)cudesc->cu_max;// 获取CUDesc结构体中的最大值 - return RoughCheckIntCU(min, max, DatumGetDateADT(arg)); + return RoughCheckIntCU( + min, max, DatumGetDateADT(arg)); // 调用RoughCheckIntCU函数，对DateADT类型的值进行粗略检查 } - +//对TimeADT类型（时间类型）的参数进行粗略检查 template bool RoughCheckTimeCU(CUDesc* cudesc, Datum arg) { - TimeADT min = *(TimeADT*)cudesc->cu_min; - TimeADT max = *(TimeADT*)cudesc->cu_max; + TimeADT min = *(TimeADT*)cudesc->cu_min;// 获取CUDesc结构体中的最小值 + TimeADT max = *(TimeADT*)cudesc->cu_max;// 获取CUDesc结构体中的最大值 #ifdef HAVE_INT64_TIMESTAMP - return RoughCheckIntCU(min, max, DatumGetTimeADT(arg)); + return RoughCheckIntCU( + min, max, DatumGetTimeADT(arg)); // 调用RoughCheckIntCU函数，对TimeADT类型的值进行粗略检查 #else // do not support for RoughCheck return true; #endif } - +//对Timestamp类型（时间戳类型）的参数进行粗略检查 template bool RoughCheckTimestampCU(CUDesc* cudesc, Datum arg) { - Timestamp min = *(Timestamp*)cudesc->cu_min; - Timestamp max = *(Timestamp*)cudesc->cu_max; + Timestamp min = *(Timestamp*)cudesc->cu_min;// 获取CUDesc结构体中的最小值 + Timestamp max = *(Timestamp*)cudesc->cu_max;// 获取CUDesc结构体中的最大值 #ifdef HAVE_INT64_TIMESTAMP - return RoughCheckIntCU(min, max, DatumGetTimestamp(arg)); + return RoughCheckIntCU(min, max, DatumGetTimestamp(arg));// 调用RoughCheckIntCU函数，对Timestamp类型的值进行粗略检查 #else // do not support for RoughCheck return true; #endif } - +//对带时区的TimestampTz类型（带时区的时间戳类型）的参数进行粗略检查 template bool RoughCheckTimestampTzCU(CUDesc* cudesc, Datum arg) { - TimestampTz min = *(TimestampTz*)cudesc->cu_min; - TimestampTz max = *(TimestampTz*)cudesc->cu_max; + TimestampTz min = *(TimestampTz*)cudesc->cu_min;// 获取CUDesc结构体中的最小值 + TimestampTz max = *(TimestampTz*)cudesc->cu_max;// 获取CUDesc结构体中的最大值 #ifdef HAVE_INT64_TIMESTAMP - return RoughCheckIntCU(min, max, DatumGetTimestampTz(arg)); + return RoughCheckIntCU(min, max, DatumGetTimestampTz(arg));// 调用RoughCheckIntCU函数，对TimestampTz类型的值进行粗略检查 #else // do not support for RoughCheck return true; #endif } - +//对整型参数进行粗略检查。根据传入的策略（strategy）值和最小值（min）、最大值（max），它会判断参数（arg）是否满足策略要求 template bool RoughCheckIntCU(const T& min, const T& max, const T& arg) { - bool hitCU = true; + bool hitCU = true;// 初始化命中标志为true，表示参数满足策略要求 - if (strategy == CStoreLessStrategyNumber) { - if (min >= arg) { - hitCU = false; + if (strategy == CStoreLessStrategyNumber) {// 如果策略是小于号策略 + if (min >= arg) {// 如果最小值大于等于参数值 + hitCU = false;// 设置命中标志为false，表示参数不满足策略要求 } - } else if (strategy == CStoreLessEqualStrategyNumber) { - if (min > arg) { - hitCU = false; + } else if (strategy == CStoreLessEqualStrategyNumber) {// 如果策略是小于等于号策略 + if (min > arg) {// 如果最小值大于参数值 + hitCU = false;// 设置命中标志为false，表示参数不满足策略要求 } - } else if (strategy == CStoreEqualStrategyNumber) { - if (arg < min || arg > max) { - hitCU = false; + } else if (strategy == CStoreEqualStrategyNumber) {// 如果策略是等于号策略 + if (arg < min || arg > max) {// 如果参数值小于最小值或者大于最大值 + hitCU = false;// 设置命中标志为false，表示参数不满足策略要求 } - } else if (strategy == CStoreGreaterEqualStrategyNumber) { - if (max < arg) { - hitCU = false; + } else if (strategy == CStoreGreaterEqualStrategyNumber) {// 如果策略是大于等于号策略 + if (max < arg) {// 如果最大值小于参数值 + hitCU = false;// 设置命中标志为false，表示参数不满足策略要求 } - } else if (strategy == CStoreGreaterStrategyNumber) { - if (max <= arg) { - hitCU = false; + } else if (strategy == CStoreGreaterStrategyNumber) {// 如果策略是大于号策略 + if (max <= arg) {// 如果最大值小于等于参数值 + hitCU = false;// 设置命中标志为false，表示参数不满足策略要求 } } - return hitCU; + return hitCU;// 返回命中标志，表示参数是否满足策略要求 } - +//根据传入的策略（strategy）值和最小值（cudesc->cu_min）、最大值（cudesc->cu_max），以及参数值（arg），它会判断参数是否满足策略要求 template bool RoughCheckStringCU(CUDesc* cudesc, Datum arg) { - bool hitCU = true; + bool hitCU = true;// 初始化命中标志为true，表示参数满足策略要求 - char* min = cudesc->cu_min; - char* max = cudesc->cu_max; + char* min = cudesc->cu_min;// 获取最小值字符串指针 + char* max = cudesc->cu_max;// 获取最大值字符串指针 - char* argStr = VARDATA_ANY(DatumGetPointer(arg)); - int argLen = VARSIZE_ANY_EXHDR(arg); + char* argStr = VARDATA_ANY(DatumGetPointer(arg));// 获取参数字符串指针 + int argLen = VARSIZE_ANY_EXHDR(arg);// 获取参数字符串长度 - int minLen = *(char*)min; - int maxLen = *(char*)max; + int minLen = *(char*)min;// 最小值字符串的长度 + int maxLen = *(char*)max;// 最大值字符串的长度 - char* minStr = min + 1; - char* maxStr = max + 1; + char* minStr = min + 1;// 最小值字符串的起始地址 + char* maxStr = max + 1;// 最大值字符串的起始地址 - int cmpMinLen = minLen > argLen ? argLen : minLen; - int cmpMaxLen = maxLen > argLen ? argLen : maxLen; + int cmpMinLen = minLen > argLen ? argLen : minLen;// 比较最小长度 + int cmpMaxLen = maxLen > argLen ? argLen : maxLen;// 比较最大长度 switch (strategy) { case CStoreLessStrategyNumber: - case CStoreLessEqualStrategyNumber: { - if (memcmp(minStr, argStr, cmpMinLen) > 0) - hitCU = false; + case CStoreLessEqualStrategyNumber: {// 如果策略是小于号或者小于等于号 + if (memcmp(minStr, argStr, cmpMinLen) > 0)// 如果最小字符串与参数字符串的前cmpMinLen个字符比较结果大于0 + hitCU = false;// 设置命中标志为false，表示参数不满足策略要求 } break; - case CStoreEqualStrategyNumber: { - if (memcmp(minStr, argStr, cmpMinLen) > 0 || memcmp(maxStr, argStr, cmpMaxLen) < 0) - hitCU = false; + case CStoreEqualStrategyNumber: {// 如果策略是等于号 + if (memcmp(minStr, argStr, cmpMinLen) > 0 || memcmp(maxStr, argStr, cmpMaxLen) < 0)// 如果最小字符串与参数字符串的前cmpMinLen个字符比较结果大于0，或者最大字符串与参数字符串的前cmpMaxLen个字符比较结果小于0 + hitCU = false;// 设置命中标志为false，表示参数不满足策略要求 } break; case CStoreGreaterEqualStrategyNumber: - case CStoreGreaterStrategyNumber: { - if (memcmp(maxStr, argStr, cmpMaxLen) < 0) - hitCU = false; + case CStoreGreaterStrategyNumber: {// 如果策略是大于等于号或者大于号 + if (memcmp(maxStr, argStr, cmpMaxLen) < 0)// 如果最大字符串与参数字符串的前cmpMaxLen个字符比较结果小于0 + hitCU = false;// 设置命中标志为false，表示参数不满足策略要求 } break; default: break; } - return hitCU; + return hitCU;// 返回命中标志，表示参数是否满足策略要求 } /* * In some unsupport type or invaild strategy number, make the CU through the rough check */ +//在遇到不支持的数据类型或无效的策略号时，通过粗略检查来生成计算单位（CU） bool RoughCheckAllThrough(CUDesc* cudesc, Datum arg) { return true; -- 2.34.1 From 7d1fd6229830f8c2ec4263ebade26c16d4aa0194 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Wed, 4 Oct 2023 05:21:54 +0800 Subject: [PATCH 44/48] Update cstore_update.cpp --- .../storage/cstore/cstore_update.cpp | 244 +++++++++--------- 1 file changed, 125 insertions(+), 119 deletions(-) diff --git a/src/gausskernel/storage/cstore/cstore_update.cpp b/src/gausskernel/storage/cstore/cstore_update.cpp index 0312ad6ee..82ec03c53 100644 --- a/src/gausskernel/storage/cstore/cstore_update.cpp +++ b/src/gausskernel/storage/cstore/cstore_update.cpp @@ -34,37 +34,38 @@ extern void ExecVecConstraints(ResultRelInfo* resultRelInfo, VectorBatch* batch, CStoreUpdate::CStoreUpdate(_in_ Relation rel, _in_ EState* estate, _in_ Plan* plan) : m_estate(estate) { + // 确保estate不为空 Assert(estate); - m_relation = rel; - m_resultRelInfo = estate->es_result_relation_info; - m_isPartition = RELATION_IS_PARTITIONED(rel); - m_delMemInfo = NULL; - m_insMemInfo = NULL; + m_relation = rel;// 初始化表关系 + m_resultRelInfo = estate->es_result_relation_info;// 初始化结果关系信息 + m_isPartition = RELATION_IS_PARTITIONED(rel);// 判断表是否分区 + m_delMemInfo = NULL;// 初始化删除内存信息 + m_insMemInfo = NULL;// 初始化插入内存信息 /* init memory, memory info will be used to init delete and insert. */ - InitUpdateMemArg(plan); + InitUpdateMemArg(plan);// 初始化更新的内存参数 /* init delete */ - m_delete = New(CurrentMemoryContext) CStoreDelete(rel, estate, true, NULL, m_delMemInfo); - m_delete->setReportErrorForUpdate(true); + m_delete = New(CurrentMemoryContext) CStoreDelete(rel, estate, true, NULL, m_delMemInfo);// 初始化删除操作 + m_delete->setReportErrorForUpdate(true);// 设置报告更新错误 /* init insert */ - if (!m_isPartition) { - CStoreInsert::InitInsertArg(rel, m_resultRelInfo, true, m_insert_args); - m_insert_args.sortType = BATCH_SORT; + if (!m_isPartition) {// 如果表不是分区表 + CStoreInsert::InitInsertArg(rel, m_resultRelInfo, true, m_insert_args);// 初始化插入操作参数 + m_insert_args.sortType = BATCH_SORT; // 设置插入操作的排序类型为批量排序 - m_insert = New(CurrentMemoryContext) CStoreInsert(rel, m_insert_args, true, NULL, m_insMemInfo); - m_partionInsert = NULL; - } else { + m_insert = New(CurrentMemoryContext) CStoreInsert(rel, m_insert_args, true, NULL, m_insMemInfo);// 初始化插入操作 + m_partionInsert = NULL;// 分区插入操作为空 + } else {// 如果表是分区表 m_partionInsert = - New(CurrentMemoryContext) CStorePartitionInsert(rel, m_resultRelInfo, TUPLE_SORT, true, NULL, m_insMemInfo); - + New(CurrentMemoryContext) CStorePartitionInsert(rel, m_resultRelInfo, TUPLE_SORT, true, NULL, m_insMemInfo);// 初始化分区插入操作，排序类型为元组排序 + // 在切换分区时使用闪存缓存的数据更新 /* update using flash cached data when switch partition when insert data */ m_partionInsert->SetPartitionCacheStrategy(FLASH_WHEN_SWICH_PARTITION); - m_insert = NULL; + m_insert = NULL;// 插入操作为空 } - m_hasUniqueIdx = CheckHasUniqueIdx(); + m_hasUniqueIdx = CheckHasUniqueIdx();// 检查是否有唯一索引 } CStoreUpdate::~CStoreUpdate() @@ -82,24 +83,24 @@ CStoreUpdate::~CStoreUpdate() void CStoreUpdate::Destroy() { if (m_delete) { - DELETE_EX(m_delete); + DELETE_EX(m_delete);// 如果删除操作存在，则删除。 } if (m_insert) { - DELETE_EX(m_insert); - CStoreInsert::DeInitInsertArg(m_insert_args); + DELETE_EX(m_insert);// 如果插入操作存在，则删除。 + CStoreInsert::DeInitInsertArg(m_insert_args);// 释放插入操作参数的空间。 } if (m_partionInsert) { - DELETE_EX(m_partionInsert); + DELETE_EX(m_partionInsert);// 如果分区插入操作存在，则删除。 } if (m_delMemInfo) { - pfree_ext(m_delMemInfo); + pfree_ext(m_delMemInfo);// 释放删除内存信息的空间。 } if (m_insMemInfo) { - pfree_ext(m_insMemInfo); + pfree_ext(m_insMemInfo);// 释放插入内存信息的空间。 } } @@ -111,36 +112,37 @@ void CStoreUpdate::Destroy() * @Return: void * @See also: InitInsertMemArg */ +//初始化CStoreUpdate类的内存参数 void CStoreUpdate::InitUpdateMemArg(Plan* plan) { - int maxbatchRows = RelationGetMaxBatchRows(m_relation); - int partialClusterRows = RelationGetPartialClusterRows(m_relation); - int partitionNum = 1; - List* partitionList = NIL; + int maxbatchRows = RelationGetMaxBatchRows(m_relation);// 获取最大批量行数 + int partialClusterRows = RelationGetPartialClusterRows(m_relation);// 获取部分聚集行数 + int partitionNum = 1;// 分区数，默认为1 + List* partitionList = NIL;// 分区列表 - m_delMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg)); - m_insMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg)); + m_delMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg));// 初始化删除内存参数 + m_insMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg));// 初始化插入内存参数 if (m_isPartition) { - partitionList = relationGetPartitionList(m_relation, AccessShareLock); + partitionList = relationGetPartitionList(m_relation, AccessShareLock);// 获取分区列表 /* get partition number */ - partitionNum = list_length(partitionList); - releasePartitionList(m_relation, &partitionList, NoLock); + partitionNum = list_length(partitionList);// 计算分区数 + releasePartitionList(m_relation, &partitionList, NoLock); // 释放分区列表 } - + // 初始化删除内存参数和插入内存参数的值 /* init mem = delete mem + insert mem. */ if (plan != NULL && plan->operatorMemKB[0] > 0) { - m_delMemInfo->canSpreadmaxMem = plan->operatorMaxMem; - m_delMemInfo->MemInsert = 0; - m_delMemInfo->spreadNum = 0; - m_delMemInfo->partitionNum = 1; + m_delMemInfo->canSpreadmaxMem = plan->operatorMaxMem;// 可扩展的最大内存 + m_delMemInfo->MemInsert = 0;// 删除操作的插入内存 + m_delMemInfo->spreadNum = 0;// 扩展次数 + m_delMemInfo->partitionNum = 1;// 分区数，默认为1 if (m_isPartition) { /* for partition insert. */ - m_insMemInfo->canSpreadmaxMem = plan->operatorMaxMem; - m_insMemInfo->MemInsert = plan->operatorMemKB[0] * 1 / 3; - m_insMemInfo->MemSort = plan->operatorMemKB[0] - m_insMemInfo->MemInsert; - m_insMemInfo->spreadNum = 0; - m_insMemInfo->partitionNum = partitionNum; - m_delMemInfo->partitionNum = partitionNum; + m_insMemInfo->canSpreadmaxMem = plan->operatorMaxMem;// 可扩展的最大内存 + m_insMemInfo->MemInsert = plan->operatorMemKB[0] * 1 / 3;// 插入操作的插入内存 + m_insMemInfo->MemSort = plan->operatorMemKB[0] - m_insMemInfo->MemInsert;// 排序内存 + m_insMemInfo->spreadNum = 0;// 扩展次数 + m_insMemInfo->partitionNum = partitionNum;// 分区数 + m_delMemInfo->partitionNum = partitionNum;// 分区数 m_delMemInfo->MemSort = m_insMemInfo->MemSort; MEMCTL_LOG(DEBUG2, "UpdateForCStorePartDelete(init plan):Insert workmem is : %dKB, sort workmem: %dKB," @@ -157,13 +159,13 @@ void CStoreUpdate::InitUpdateMemArg(Plan* plan) m_insMemInfo->partitionNum, m_insMemInfo->canSpreadmaxMem); } else { - m_insMemInfo->canSpreadmaxMem = plan->operatorMaxMem; + m_insMemInfo->canSpreadmaxMem = plan->operatorMaxMem;// 可扩展的最大内存 m_insMemInfo->MemInsert = static_cast((double)plan->operatorMemKB[0] * (double)(maxbatchRows * BATCHROW_TIMES) / - (double)(maxbatchRows + partialClusterRows)); - m_insMemInfo->MemSort = plan->operatorMemKB[0] - m_insMemInfo->MemInsert; - m_insMemInfo->spreadNum = 0; - m_insMemInfo->partitionNum = 1; + (double)(maxbatchRows + partialClusterRows));// 插入操作的插入内存 + m_insMemInfo->MemSort = plan->operatorMemKB[0] - m_insMemInfo->MemInsert;// 排序内存 + m_insMemInfo->spreadNum = 0;// 扩展次数 + m_insMemInfo->partitionNum = 1;// 分区数，默认为1 m_delMemInfo->MemSort = m_insMemInfo->MemSort; MEMCTL_LOG(DEBUG2, "UpdateForCStoreDelete(init plan):Insert workmem is : %dKB, sort workmem: %dKB," @@ -185,47 +187,49 @@ void CStoreUpdate::InitUpdateMemArg(Plan* plan) * For static load, a single partition of sort Mem is 512MB, and there is no need to subdivide sort Mem. * So, set the partitionNum is 1 for all partition table. */ - m_delMemInfo->canSpreadmaxMem = 0; - m_delMemInfo->MemInsert = 0; - m_delMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem; - m_delMemInfo->spreadNum = 0; - m_delMemInfo->partitionNum = 1; - m_insMemInfo->canSpreadmaxMem = 0; - m_insMemInfo->MemInsert = u_sess->attr.attr_storage.partition_max_cache_size; - m_insMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem; - m_insMemInfo->spreadNum = 0; - m_insMemInfo->partitionNum = 1; + // 静态加载的情况 + m_delMemInfo->canSpreadmaxMem = 0;// 可扩展的最大内存 + m_delMemInfo->MemInsert = 0;// 删除操作的插入内存 + m_delMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem;// 排序内存 + m_delMemInfo->spreadNum = 0;// 扩展次数 + m_delMemInfo->partitionNum = 1;// 分区数，默认为1 + m_insMemInfo->canSpreadmaxMem = 0;// 可扩展的最大内存 + m_insMemInfo->MemInsert = u_sess->attr.attr_storage.partition_max_cache_size;// 插入操作的插入内存 + m_insMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem;// 排序内存 + m_insMemInfo->spreadNum = 0;// 扩展次数 + m_insMemInfo->partitionNum = 1;// 分区数，默认为1 } } - +//初始化删除操作的排序状态 void CStoreUpdate::InitSortState(TupleDesc sortTupDesc) { - Assert(sortTupDesc && m_resultRelInfo && m_delete); + Assert(sortTupDesc && m_resultRelInfo && m_delete);// 断言，确保sortTupDesc、m_resultRelInfo和m_delete不为空 - JunkFilter* junkfilter = m_resultRelInfo->ri_junkFilter; + JunkFilter* junkfilter = m_resultRelInfo->ri_junkFilter;// 获取结果关系信息中的垃圾过滤器 // init delete sort state m_delete->InitSortState(sortTupDesc, junkfilter->jf_xc_part_id, junkfilter->jf_junkAttNo); + // 初始化删除操作的排序状态，传入排序元组描述、垃圾过滤器中的分区ID和垃圾属性编号 } - +//执行数据更新操作 uint64 CStoreUpdate::ExecUpdate(_in_ VectorBatch* batch, _in_ int options) { - Assert(batch && m_resultRelInfo && m_delete); - Assert((m_isPartition && m_partionInsert) || (!m_isPartition && m_insert)); + Assert(batch && m_resultRelInfo && m_delete);// 断言，确保batch、m_resultRelInfo和m_delete不为空 + Assert((m_isPartition && m_partionInsert) || (!m_isPartition && m_insert));// 断言，确保如果有分区相关变量，则相应插入操作也不能为空；否则，如果没有分区，则相应插入操作不能为空 - JunkFilter* junkfilter = m_resultRelInfo->ri_junkFilter; + JunkFilter* junkfilter = m_resultRelInfo->ri_junkFilter;// 获取结果关系信息中的垃圾过滤器 // delete if (!m_hasUniqueIdx) { - m_delete->PutDeleteBatch(batch, junkfilter); + m_delete->PutDeleteBatch(batch, junkfilter);// 调用删除操作的PutDeleteBatch方法，传入批量数据和垃圾过滤器，实现删除操作 } int oriCols = batch->m_cols; - batch->m_cols = junkfilter->jf_cleanTupType->natts; + batch->m_cols = junkfilter->jf_cleanTupType->natts;// 设置批量数据的列数为清理后的元组类型中的属性数量 // Check the constraints of the batch if (m_relation->rd_att->constr) - ExecVecConstraints(m_resultRelInfo, batch, m_estate); + ExecVecConstraints(m_resultRelInfo, batch, m_estate);// 检查批量数据的约束条件 // insert then batch if (!m_hasUniqueIdx) { @@ -235,9 +239,9 @@ uint64 CStoreUpdate::ExecUpdate(_in_ VectorBatch* batch, _in_ int options) * firstly insert and then delete. */ if (m_isPartition) { - m_partionInsert->BatchInsert(batch, options); + m_partionInsert->BatchInsert(batch, options);// 调用分区插入操作的BatchInsert方法，传入批量数据和选项，实现插入操作 } else { - m_insert->BatchInsert(batch, options); + m_insert->BatchInsert(batch, options);// 调用插入操作的BatchInsert方法，传入批量数据和选项，实现插入操作 } } else { /* @@ -245,33 +249,34 @@ uint64 CStoreUpdate::ExecUpdate(_in_ VectorBatch* batch, _in_ int options) * or the index key of new data and old data may violate. */ if (m_isPartition) { - PartitionBatchDeleteAndInsert(batch, oriCols, options, junkfilter); + PartitionBatchDeleteAndInsert(batch, oriCols, options, junkfilter);// 调用分区批量删除和插入操作，传入批量数据、原始列数、选项和垃圾过滤器，实现删除和插入操作 } else { - BatchDeleteAndInsert(batch, oriCols, options, junkfilter); + BatchDeleteAndInsert(batch, oriCols, options, junkfilter);// 调用批量删除和插入操作，传入批量数据、原始列数、选项和垃圾过滤器，实现删除和插入操作 } } - batch->m_cols = oriCols; + batch->m_cols = oriCols; // 恢复批量数据的列数为原始列数 - return (uint64)(uint32)batch->m_rows; + return (uint64)(uint32)batch->m_rows; // 返回批量数据的行数 } /* * @Description: Do batch delete and insert. In this function, the number of rows those are deleted is * strictly equals number of rows will be inserted. It is usually used when cstore has unique index. */ +//执行批量删除和插入操作 void CStoreUpdate::BatchDeleteAndInsert(VectorBatch *batch, int oriBatchCols, int options, JunkFilter *junkfilter) { - int currentCols = batch->m_cols; + int currentCols = batch->m_cols;// 获取当前批量数据的列数 /* keep memory space from leaking during bulk-insert */ - MemoryContext insertCnxt = m_insert->GetTmpMemCnxt(); - MemoryContext updateCnxt = MemoryContextSwitchTo(insertCnxt); + MemoryContext insertCnxt = m_insert->GetTmpMemCnxt();// 获取插入操作中的临时内存上下文 + MemoryContext updateCnxt = MemoryContextSwitchTo(insertCnxt);// 切换当前内存上下文为插入操作中的临时内存上下文 - CStorePSort* sorter = m_insert->GetSorter(); + CStorePSort* sorter = m_insert->GetSorter();// 获取插入操作中的排序器 if (sorter != NULL) { /* Step 1: relation has partial cluster key */ - Assert(batch->m_cols == m_insert->m_relation->rd_att->natts); + Assert(batch->m_cols == m_insert->m_relation->rd_att->natts);// 断言，确保批量数据中的列数等于插入操作中关系信息中属性的数量 { AutoContextSwitch updateContext(updateCnxt); @@ -280,44 +285,45 @@ void CStoreUpdate::BatchDeleteAndInsert(VectorBatch *batch, int oriBatchCols, in batch->m_cols = currentCols; } - sorter->PutVecBatch(m_insert->m_relation, batch); + sorter->PutVecBatch(m_insert->m_relation, batch);// 将批量数据传递给排序器 if (sorter->IsFull()) { { AutoContextSwitch updateContext(updateCnxt); - m_delete->PartialDelete(); + m_delete->PartialDelete();// 调用删除操作的PartialDelete方法，实现部分删除操作 } - m_insert->SortAndInsert(options); + m_insert->SortAndInsert(options);// 调用插入操作的SortAndInsert方法，传入选项，实现排序和插入操作 } } else { /* Step 2: relation doesn't have partial cluster key */ - bulkload_rows* bufferedBatchRows = m_insert->GetBufferedBatchRows(); + bulkload_rows* bufferedBatchRows = m_insert->GetBufferedBatchRows();// 获取插入操作中的批量行 - Assert(batch->m_rows <= BatchMaxSize); - Assert(batch->m_cols && m_insert->m_relation->rd_att->natts); - Assert(bufferedBatchRows->m_rows_maxnum > 0); - Assert(bufferedBatchRows->m_rows_maxnum % BatchMaxSize == 0); + Assert(batch->m_rows <= BatchMaxSize);// 断言，确保批量数据的行数小于等于最大批量大小 + Assert(batch->m_cols && m_insert->m_relation->rd_att->natts);// 断言，确保批量数据的列数和插入操作中关系信息的属性数量都不为零 + Assert(bufferedBatchRows->m_rows_maxnum > 0);// 断言，确保批量行的最大行数大于零 + Assert(bufferedBatchRows->m_rows_maxnum % BatchMaxSize == 0);// 断言，确保批量行的最大行数是最大批量大小的倍数 int startIdx = 0; int lastStartIdx = startIdx; for (;;) { /* we need cache data until batchrows is full */ bool needInsert = bufferedBatchRows->append_one_vector( - RelationGetDescr(m_relation), batch, &startIdx, m_insert->m_cstorInsertMem); + RelationGetDescr(m_relation), batch, &startIdx, m_insert->m_cstorInsertMem);// 将批量数据追加到批量行中，同时更新其起始索引 if (startIdx > lastStartIdx) { AutoContextSwitch updateContext(updateCnxt); batch->m_cols = oriBatchCols; - m_delete->PutDeleteBatchForUpdate(batch, lastStartIdx, startIdx); + m_delete->PutDeleteBatchForUpdate(batch, lastStartIdx, startIdx);// 调用删除操作的PutDeleteBatchForUpdate方法，传入批量数据、起始索引和结束索引，实现删除操作 batch->m_cols = currentCols; } if (needInsert) { { AutoContextSwitch updateContext(updateCnxt); - m_delete->PartialDelete(); + m_delete->PartialDelete();// 调用删除操作的PartialDelete方法，实现部分删除操作 + } - m_insert->BatchInsertCommon(bufferedBatchRows, options); - bufferedBatchRows->reset(true); + m_insert->BatchInsertCommon(bufferedBatchRows, options);// 调用插入操作的BatchInsertCommon方法，传入批量行和选项，实现批量插入操作 + bufferedBatchRows->reset(true);// 重置批量行状态，参数指定是否释放内存 lastStartIdx = startIdx; } else { break; @@ -325,52 +331,52 @@ void CStoreUpdate::BatchDeleteAndInsert(VectorBatch *batch, int oriBatchCols, in } } - MemoryContextReset(insertCnxt); - (void)MemoryContextSwitchTo(updateCnxt); + MemoryContextReset(insertCnxt);// 重置临时内存上下文 + (void)MemoryContextSwitchTo(updateCnxt);// 恢复原始内存上下文 } - +//执行分区的批量删除和插入操作 void CStoreUpdate::PartitionBatchDeleteAndInsert(VectorBatch *batch, int oriBatchCols, int options, JunkFilter *junkfilter) { - int currentCols = batch->m_cols; - batch->m_cols = oriBatchCols; + int currentCols = batch->m_cols;// 获取当前批量数据的列数 + batch->m_cols = oriBatchCols;// 将当前批量数据的列数设置为原始批量数据的列数 - m_delete->PutDeleteBatch(batch, junkfilter); - m_delete->PartialDelete(); + m_delete->PutDeleteBatch(batch, junkfilter);// 调用删除操作的PutDeleteBatch方法，传入批量数据和垃圾过滤器，实现删除操作 + m_delete->PartialDelete();// 调用删除操作的PartialDelete方法，实现部分删除操作 - batch->m_cols = currentCols; + batch->m_cols = currentCols;// 将批量数据的列数恢复为当前批量数据的列数 - m_partionInsert->BatchInsert(batch, options); + m_partionInsert->BatchInsert(batch, options);// 调用分区插入操作的BatchInsert方法，传入批量数据和选项，实现批量插入操作 } - +//检查关系信息中是否存在唯一索引 bool CStoreUpdate::CheckHasUniqueIdx() { - bool hasUniqueIdx = false; - if (m_resultRelInfo && m_resultRelInfo->ri_NumIndices > 0) { - IndexInfo** indexInfos = m_resultRelInfo->ri_IndexRelationInfo; - for (int i = 0; i < m_resultRelInfo->ri_NumIndices; ++i) { - if (indexInfos[i]->ii_Unique) { - hasUniqueIdx = true; - break; + bool hasUniqueIdx = false;// 初始化是否存在唯一索引的标志为假 + if (m_resultRelInfo && m_resultRelInfo->ri_NumIndices > 0) {// 如果存在关系信息且索引数量大于零 + IndexInfo** indexInfos = m_resultRelInfo->ri_IndexRelationInfo;// 获取索引关系信息 + for (int i = 0; i < m_resultRelInfo->ri_NumIndices; ++i) {// 遍历索引关系信息 + if (indexInfos[i]->ii_Unique) {// 如果当前索引是唯一索引 + hasUniqueIdx = true;// 将是否存在唯一索引的标志设置为真 + break;// 结束循环 } } } - return hasUniqueIdx; + return hasUniqueIdx;// 返回是否存在唯一索引的标志 } - +//用于结束更新操作 void CStoreUpdate::EndUpdate(_in_ int options) { - Assert(m_delete); - Assert((m_isPartition && m_partionInsert) || (!m_isPartition && m_insert)); + Assert(m_delete);// 断言删除操作不为空 + Assert((m_isPartition && m_partionInsert) || (!m_isPartition && m_insert));// 断言如果是分区更新，则分区插入操作不为空；否则，插入操作不为空 // end delete - m_delete->ExecDelete(); + m_delete->ExecDelete();// 执行删除操作 // end insert - if (m_isPartition) { - m_partionInsert->EndBatchInsert(); + if (m_isPartition) {// 如果是分区更新 + m_partionInsert->EndBatchInsert();// 结束分区批量插入操作 } else { - m_insert->SetEndFlag(); - m_insert->BatchInsert((VectorBatch*)NULL, options); + m_insert->SetEndFlag();// 设置插入操作的结束标志 + m_insert->BatchInsert((VectorBatch*)NULL, options);// 执行插入操作，传入空的批量数据和选项 } } -- 2.34.1 From a857f4f4a16ef627f1a7993ff5f3c81ced548bc2 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Wed, 4 Oct 2023 05:22:17 +0800 Subject: [PATCH 45/48] Update cucache_mgr.cpp --- .../storage/cstore/cucache_mgr.cpp | 384 +++++++++++++----- 1 file changed, 291 insertions(+), 93 deletions(-) diff --git a/src/gausskernel/storage/cstore/cucache_mgr.cpp b/src/gausskernel/storage/cstore/cucache_mgr.cpp index 65ac4ac76..e3f88ea73 100644 --- a/src/gausskernel/storage/cstore/cucache_mgr.cpp +++ b/src/gausskernel/storage/cstore/cucache_mgr.cpp @@ -46,10 +46,11 @@ DataCacheMgr* DataCacheMgr::m_data_cache = NULL; * @Return: lock number * @See also: */ +//获取数据缓存管理器中锁的数量 int DataCacheMgrNumLocks() { - int64 cache_size = CacheMgrCalcSizeByType(MGR_CACHE_TYPE_DATA); - return CacheMgrNumLocks(cache_size, BLCKSZ); + int64 cache_size = CacheMgrCalcSizeByType(MGR_CACHE_TYPE_DATA);// 计算数据缓存大小 + return CacheMgrNumLocks(cache_size, BLCKSZ);// 返回根据缓存大小和块大小计算得到的锁的数量 } /* @@ -57,41 +58,47 @@ int DataCacheMgrNumLocks() * @Return: CU cache instance * @See also: */ +//获取数据缓存管理器的单例实例 DataCacheMgr* DataCacheMgr::GetInstance(void) { - Assert(m_data_cache != NULL); - return m_data_cache; + Assert(m_data_cache != NULL);// 断言确保数据缓存实例不为空 + return m_data_cache;// 返回数据缓存实例 } /* * @Description: create or recreate the Singleton Instance of CU cache * @See also: */ +//创建或重新创建数据缓存管理器的单例实例 void DataCacheMgr::NewSingletonInstance(void) { int64 cache_size = 0; + // 只有 Postmaster 才有权利创建或重新创建 CU 缓存实例 /* only Postmaster has the privilege to create or recreate CU cache instance */ - if (IsUnderPostmaster) - return; + if (IsUnderPostmaster)// 如果处于 Postmaster 下 + return;// 直接返回 - int condition = sizeof(DataSlotTagKey) != MAX_CACHE_TAG_LEN ? 1 : 0; - BUILD_BUG_ON_CONDITION(condition); + int condition = sizeof(DataSlotTagKey) != MAX_CACHE_TAG_LEN ? 1 : 0;// 检查 DataSlotTagKey 的大小是否等于 MAX_CACHE_TAG_LEN + BUILD_BUG_ON_CONDITION(condition);// 在编译时检查条件，如果为真，则产生编译错误 - if (m_data_cache == NULL) { + if (m_data_cache == NULL) {// 如果数据缓存实例为空 + // 第一次创建该实例 /* create this instance at the first time */ - m_data_cache = New(CurrentMemoryContext) DataCacheMgr; - m_data_cache->m_cache_mgr = New(CurrentMemoryContext) CacheMgr; + m_data_cache = New(CurrentMemoryContext) DataCacheMgr;// 创建 DataCacheMgr 实例 + m_data_cache->m_cache_mgr = New(CurrentMemoryContext) CacheMgr;// 创建 CacheMgr 实例 } else { + // 销毁实例的所有资源 /* destroy all resources of its members */ - m_data_cache->m_cache_mgr->Destroy(); - SpinLockFree(&m_data_cache->m_adio_write_cache_lock); + m_data_cache->m_cache_mgr->Destroy();// 销毁缓存管理器的资源 + SpinLockFree(&m_data_cache->m_adio_write_cache_lock);// 释放自旋锁的内存资源 } - cache_size = CacheMgrCalcSizeByType(MGR_CACHE_TYPE_DATA); - m_data_cache->m_cstoreMaxSize = cache_size; - SpinLockInit(&m_data_cache->m_adio_write_cache_lock); + cache_size = CacheMgrCalcSizeByType(MGR_CACHE_TYPE_DATA);// 计算数据缓存的大小 + m_data_cache->m_cstoreMaxSize = cache_size;// 设置数据缓存的最大大小 + SpinLockInit(&m_data_cache->m_adio_write_cache_lock);// 初始化自旋锁 /* init or reset this instance */ + // 初始化或重置实例 m_data_cache->m_cache_mgr->Init(cache_size, BLCKSZ, MGR_CACHE_TYPE_DATA, Max(sizeof(CU), sizeof(OrcDataValue))); - ereport(LOG, (errmodule(MOD_CACHE), errmsg("set data cache size(%ld)", cache_size))); + ereport(LOG, (errmodule(MOD_CACHE), errmsg("set data cache size(%ld)", cache_size)));// 记录日志，设置数据缓存的大小 } /* @@ -103,15 +110,21 @@ void DataCacheMgr::NewSingletonInstance(void) * @Return: cu slot tag * @See also: */ +//初始化 CU（Column Unit）的唯一标志符（tag） DataSlotTag DataCacheMgr::InitCUSlotTag(RelFileNodeOld* rnode, int colid, uint32 cuid, CUPointer cuPtr) { + // 初始化 CU 的唯一标志符（tag） DataSlotTag tag; + // 设置 CU 的文件节点信息 tag.slotTag.cuSlotTag.m_rnode = *rnode; + // 设置 CU 的标识符、列 ID、偏移量和填充字段 tag.slotTag.cuSlotTag.m_CUId = cuid; tag.slotTag.cuSlotTag.m_colId = colid; tag.slotTag.cuSlotTag.m_cuPtr = cuPtr; tag.slotTag.cuSlotTag.m_padding = 0; + // 设置 slot 类型为 CACHE_COlUMN_DATA tag.slotType = CACHE_COlUMN_DATA; + // 返回初始化后的 CU slot 标签 return tag; } @@ -124,28 +137,39 @@ DataSlotTag DataCacheMgr::InitCUSlotTag(RelFileNodeOld* rnode, int colid, uint32 * @Return: orc data slot tag * @See also: */ +//初始化 ORC（Optimized Row Columnar）数据的唯一标志符（tag） DataSlotTag DataCacheMgr::InitORCSlotTag(RelFileNode* rnode, int32 fileid, uint64 offset, uint64 length) { + // 初始化 ORC 数据的唯一标志符（tag） DataSlotTag tag; + // 设置 ORC 数据的文件节点信息 tag.slotTag.orcSlotTag.m_rnode = *(RelFileNodeOld*)rnode; + // 设置 ORC 数据的关系 ID、偏移量和长度 tag.slotTag.orcSlotTag.m_fileId = fileid; tag.slotTag.orcSlotTag.m_offset = offset; tag.slotTag.orcSlotTag.m_length = length; + // 设置 slot 类型为 CACHE_ORC_DATA tag.slotType = CACHE_ORC_DATA; + // 返回初始化后的 ORC 数据 slot 标签 return tag; } - +//初始化 OBS（Object Storage Service）数据的唯一标志符（tag） DataSlotTag DataCacheMgr::InitOBSSlotTag(uint32 hostNameHash, uint32 bucketNameHash, uint32 fileFirstHalfHash, uint32 fileSecondHalfHash, uint64 offset, uint64 length) const { + // 初始化 OBS 数据的唯一标志符（tag） DataSlotTag tag; + // 设置 OBS 数据的服务器名哈希、桶名哈希、文件名前半部分哈希和文件名后半部分哈希 tag.slotTag.obsSlotTag.m_serverHash = hostNameHash; tag.slotTag.obsSlotTag.m_bucketHash = bucketNameHash; tag.slotTag.obsSlotTag.m_fileFirstHash = fileFirstHalfHash; tag.slotTag.obsSlotTag.m_fileSecondHash = fileSecondHalfHash; + // 设置 OBS 数据的偏移量和长度 tag.slotTag.obsSlotTag.m_offset = offset; tag.slotTag.obsSlotTag.m_length = length; + // 设置 slot 类型为 CACHE_OBS_DATA tag.slotType = CACHE_OBS_DATA; + // 返回初始化后的 OBS 数据 slot 标签 return tag; } @@ -158,12 +182,17 @@ DataSlotTag DataCacheMgr::InitOBSSlotTag(uint32 hostNameHash, uint32 bucketNameH */ CacheSlotId_t DataCacheMgr::FindDataBlock(DataSlotTag* dataSlotTag, bool first_enter_block) { + // 声明一个 slot id 并初始化为 CACHE_BLOCK_INVALID_IDX CacheSlotId_t slot = CACHE_BLOCK_INVALID_IDX; + // 声明一个 CacheTag 对象并初始化为 0 CacheTag cacheTag = {0}; - + // 初始化 CacheTag 对象，使用 slot 的类型和 tag 来初始化 + // 由于 tag 可能是复合类型，需要传入 tag 的大小 m_cache_mgr->InitCacheBlockTag(&cacheTag, dataSlotTag->slotType, &dataSlotTag->slotTag, sizeof(DataSlotTagKey)); + // 根据 CacheTag 在 cache 中查找对应的 slot id + // 如果 first_enter_block 为 true，则表示第一次使用该 block，需要申请新的 slot slot = m_cache_mgr->FindCacheBlock(&cacheTag, first_enter_block); - + // 返回查找到的 slot id return slot; } @@ -178,25 +207,36 @@ CacheSlotId_t DataCacheMgr::FindDataBlock(DataSlotTag* dataSlotTag, bool first_e */ void DataCacheMgr::InvalidateCU(RelFileNodeOld* rnode, int colId, uint32 cuId, CUPointer cuPtr) { + // 声明一个 CacheTag 对象并初始化为 0 CacheTag cacheTag = {0}; + // 根据传入的参数初始化 DataSlotTag 对象 DataSlotTag dataSlotTag = InitCUSlotTag(rnode, colId, cuId, cuPtr); - + // 初始化 CacheTag 对象，使用 slot 的类型和 tag 来初始化 + // 由于 tag 可能是复合类型，需要传入 tag 的大小 m_cache_mgr->InitCacheBlockTag(&cacheTag, dataSlotTag.slotType, &dataSlotTag.slotTag, sizeof(DataSlotTagKey)); + // invalid cache 中的对应 slot m_cache_mgr->InvalidateCacheBlock(&cacheTag); return; } /* invalid all CUs from data cache which belongs to this column relation */ +//使所有属于该列关系的 CU 在 cache 中对应的 slot 设置为无效 void DataCacheMgr::DropRelationCUCache(const RelFileNode& rnode) { + // 声明一个 CacheTag 对象并初始化为 0 CacheTag tag = {0}; + // 声明一个 CUSlotTag 对象，初始化为 {InvalidOid, InvalidOid, InvalidOid}、0、0、0、0 CUSlotTag cuTag = {{InvalidOid, InvalidOid, InvalidOid}, 0, 0, 0, 0}; + // 获取已使用的 cache slot 的数量 const int maxSlot = m_cache_mgr->GetUsedCacheSlotNum(); - + // 遍历所有 cache slot，并检查是否属于该列关系，是则使其失效 for (CacheSlotId_t slot = 0; slot <= maxSlot; slot++) { + // 复制当前 slot 的 tag 到 tag 变量中 m_cache_mgr->CopyCacheBlockTag(slot, &tag); + // 将 tag 的 key 转换为 CUSlotTag 类型 cuTag = *(CUSlotTag*)tag.key; + // 如果当前 slot 是 CACHE_COlUMN_DATA 类型且是该列关系，则使其失效 if (CACHE_COlUMN_DATA == tag.type && RelFileColumnNodeRelEquals(rnode, cuTag.m_rnode)) { /* try to invalid this CU */ InvalidateCU(&cuTag.m_rnode, cuTag.m_colId, cuTag.m_CUId, cuTag.m_cuPtr); @@ -210,8 +250,10 @@ void DataCacheMgr::DropRelationCUCache(const RelFileNode& rnode) * @Return: CU data pointer * @See also: */ +//获取指定 cuSlotId 对应的 CU（压缩单元）数据指针 CU* DataCacheMgr::GetCUBuf(int cuSlotId) { + // 返回指定 cuSlotId 对应的 CU 数据指针 return (CU*)m_cache_mgr->GetCacheBlock(cuSlotId); } @@ -221,8 +263,10 @@ CU* DataCacheMgr::GetCUBuf(int cuSlotId) * @Return: ORC data pointer * @See also: */ +//获取指定 cuSlotId 对应的 ORC 数据指针 OrcDataValue* DataCacheMgr::GetORCDataBuf(int cuSlotId) { + // 返回指定 cuSlotId 对应的 ORC 数据指针 return (OrcDataValue*)m_cache_mgr->GetCacheBlock(cuSlotId); } @@ -232,11 +276,14 @@ OrcDataValue* DataCacheMgr::GetORCDataBuf(int cuSlotId) * @Return: true for the session can evict the cache. * false for some other session is updating it concurrently. */ +//预留指定的缓存块 slotId bool DataCacheMgr::ReserveDataBlockWithSlotId(CacheSlotId_t slotId) { + // 预留指定的缓存块 slot if (m_cache_mgr->ReserveCacheBlockWithSlotId(slotId)) { Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO) || t_thrd.storage_cxt.CacheBlockInProgressIO == slotId); + // 设置当前正在 IO 的缓存块的 ID t_thrd.storage_cxt.CacheBlockInProgressIO = slotId; return true; } @@ -250,11 +297,17 @@ bool DataCacheMgr::ReserveDataBlockWithSlotId(CacheSlotId_t slotId) * @Return: true for the session can evict the cache. * false for the slot is IOBUSY. */ +// 函数作用：对于远程读取，从缓存中驱逐某个由slotId表示的缓存块。 +// 参数 slotId：要驱逐的目标缓存块的槽位ID。 +// 返回值：true表示会话可以驱逐缓存块；false表示该槽位正在进行IO操作，无法驱逐。 bool DataCacheMgr::ReserveCstoreDataBlockWithSlotId(CacheSlotId_t slotId) { + // 调用缓存管理器的函数来驱逐具有指定槽位ID的Cstore缓存块 if (m_cache_mgr->ReserveCstoreCacheBlockWithSlotId(slotId)) { + // 确认当前正在进行IO操作的缓存块是否为有效的槽位ID，如果是，则断言失败 Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO) || t_thrd.storage_cxt.CacheBlockInProgressIO == slotId); + // 将正在进行IO操作的缓存块设置为指定的槽位ID t_thrd.storage_cxt.CacheBlockInProgressIO = slotId; return true; } @@ -270,18 +323,26 @@ bool DataCacheMgr::ReserveCstoreDataBlockWithSlotId(CacheSlotId_t slotId) * @Return: slot id * @See also: */ +// 函数作用：从数据缓存管理器中预留数据块 +// 参数 dataSlotTag：数据槽位的标签 +// 参数 size：需要的块大小 +// 参数 hasFound：是否找到了预留的块 +// 返回值：槽位ID CacheSlotId_t DataCacheMgr::ReserveDataBlock(DataSlotTag* dataSlotTag, int size, bool& hasFound) { CacheSlotId_t slot = CACHE_BLOCK_INVALID_IDX; CacheTag cacheTag = {0}; - + // 使用数据槽位标签初始化缓存块标签 m_cache_mgr->InitCacheBlockTag(&cacheTag, dataSlotTag->slotType, &dataSlotTag->slotTag, sizeof(DataSlotTagKey)); + // 调用缓存管理器的函数来预留指定大小的缓存块 slot = m_cache_mgr->ReserveCacheBlock(&cacheTag, size, hasFound); if (!hasFound) { + // 如果没有找到预留的块，则记录正在进行IO操作的缓存块为当前预留的槽位ID /* remember block slot in process */ Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO)); t_thrd.storage_cxt.CacheBlockInProgressIO = slot; } + // 打印调试信息 if (cacheTag.type == CACHE_COlUMN_DATA) { CUSlotTag* cuslotTag = &dataSlotTag->slotTag.cuSlotTag; ereport(DEBUG1, @@ -334,8 +395,11 @@ CacheSlotId_t DataCacheMgr::ReserveDataBlock(DataSlotTag* dataSlotTag, int size, * @IN CUSlotId: slot id * @See also: */ +// 函数作用：取消数据块的固定状态，解除数据块的Pin。 +// 参数 CUSlotId：数据块的槽位ID。 void DataCacheMgr::UnPinDataBlock(int CUSlotId) { + // 调用缓存管理器的函数来取消指定槽位ID的缓存块的固定状态，解除Pin。 m_cache_mgr->UnPinCacheBlock(CUSlotId); } @@ -344,8 +408,11 @@ void DataCacheMgr::UnPinDataBlock(int CUSlotId) * @Param[IN] slot: slot id * @See also: */ +// 函数作用：如果预留数据块失败，则进行资源清理（设置块状态和释放锁，如果需要）。 +// 参数 slot：槽位ID void DataCacheMgr::AbortCU(CacheSlotId_t slot) { + // 调用缓存管理器的函数来设置指定槽位ID的缓存块状态，并释放相关的锁。 m_cache_mgr->AbortCacheBlock(slot); return; } @@ -355,15 +422,19 @@ void DataCacheMgr::AbortCU(CacheSlotId_t slot) * @Param[IN] abort: whether to delete this CU from cache * @See also: */ +// 函数作用：终止或中止当前处理的CU（Compressed Unit）。 +// 参数 abort：是否从缓存中删除该CU。 void DataCacheMgr::TerminateCU(bool abort) { if (abort) { if (IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO)) { + // 如果当前存在进行IO操作的缓存块，将其设置为无效，并包括IO状态和锁的所有者信息。 /* invalid this block slot, and include IO state and lock owner */ Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressUncompress)); AbortCU(t_thrd.storage_cxt.CacheBlockInProgressIO); } if (IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressUncompress)) { + // 如果当前存在进行解压缩的缓存块，将其设置为无效，并不关心IO状态或锁的所有者信息。 /* invalid this block slot, and don't care IO state or lock owner */ Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO)); m_cache_mgr->SetCacheBlockErrorState(t_thrd.storage_cxt.CacheBlockInProgressUncompress); @@ -371,6 +442,7 @@ void DataCacheMgr::TerminateCU(bool abort) cuPtr->FreeSrcBuf(); } } + // 清除记录 /* clear record */ t_thrd.storage_cxt.CacheBlockInProgressIO = CACHE_BLOCK_INVALID_IDX; t_thrd.storage_cxt.CacheBlockInProgressUncompress = CACHE_BLOCK_INVALID_IDX; @@ -379,27 +451,30 @@ void DataCacheMgr::TerminateCU(bool abort) /* * @Description: terminate or abort this CU in verify process.. */ +//终止或中止在验证过程中的当前CU（压缩单元） void DataCacheMgr::TerminateVerifyCU() { if (IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO)) { /* invalid this block slot, and include IO state and lock owner */ + // 如果存在进行IO操作的缓存块，将其设置为无效，并包括IO状态和锁的所有者信息。 Assert (!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressUncompress)); Assert(m_cache_mgr->CstoreIOLockHeldByMe(t_thrd.storage_cxt.CacheBlockInProgressIO)); AbortCU(t_thrd.storage_cxt.CacheBlockInProgressIO); - HOLD_INTERRUPTS(); /* match the upcoming RESUME_INTERRUPTS */ + HOLD_INTERRUPTS(); /* match the upcoming RESUME_INTERRUPTS */ // 匹配即将到来的 RESUME_INTERRUPTS m_cache_mgr->RealeseCstoreIOLock(t_thrd.storage_cxt.CacheBlockInProgressIO); } if (IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressUncompress)) { /* invalid this block slot, and don't care IO state or lock owner */ + // 如果存在进行解压缩的缓存块，将其设置为无效，并不关心IO状态或锁的所有者信息。 Assert (!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO)); Assert (m_cache_mgr->CompressLockHeldByMe(t_thrd.storage_cxt.CacheBlockInProgressUncompress)); m_cache_mgr->SetCacheBlockErrorState(t_thrd.storage_cxt.CacheBlockInProgressUncompress); CU* cuPtr = GetCUBuf(t_thrd.storage_cxt.CacheBlockInProgressUncompress); cuPtr->FreeSrcBuf(); - HOLD_INTERRUPTS(); /* match the upcoming RESUME_INTERRUPTS */ + HOLD_INTERRUPTS(); /* match the upcoming RESUME_INTERRUPTS */ // 匹配即将到来的 RESUME_INTERRUPTS m_cache_mgr->RealeseCompressLock(t_thrd.storage_cxt.CacheBlockInProgressUncompress); } - + // 清除记录 /* clear record */ t_thrd.storage_cxt.CacheBlockInProgressIO = CACHE_BLOCK_INVALID_IDX; t_thrd.storage_cxt.CacheBlockInProgressUncompress = CACHE_BLOCK_INVALID_IDX; @@ -414,45 +489,64 @@ void DataCacheMgr::TerminateVerifyCU() * @Return: CUUncompressedRetCode value * @See also: */ +/* + * @Description: CU cache将解压缩CU的原始数据。 + * @Param[IN] cuDescPtr: CU描述信息指针 + * @Param[IN] slotId: CU槽位ID + * @Return: CUUncompressedRetCode枚举值 + * @See also: + */ +//对CU进行解压缩 CUUncompressedRetCode DataCacheMgr::StartUncompressCU( CUDesc* cuDescPtr, CacheSlotId_t slotId, int planNodeId, bool timing, int align_size) { - CU* cuPtr = GetCUBuf(slotId); + CU* cuPtr = GetCUBuf(slotId);// 获取CU缓存块指针 - m_cache_mgr->AcquireCompressLock(slotId); + m_cache_mgr->AcquireCompressLock(slotId);// 获取解压缩锁 /* * if the slot flag is set to CACHE_BLOCK_ERROR by some abort threads, * the compressed buffer is also valid no matter the CU comes from primary * node or remote mode. So, the thread can continue to uncompress. */ + /* + * 如果槽位标志被设置为CACHE_BLOCK_ERROR，说明有其他终止线程将其设置为无效。 + * 不管CU来自主节点还是远程模式，压缩缓冲区都是有效的。因此，线程可以继续解压缩。 + */ if (m_cache_mgr->IsIOBusy(slotId)) { /* * the CU is being reloading by remote read thread, * return CU_RELOADING and retry to load CU. */ + /* + * CU正在被远程读线程重新加载， + * 返回CU_RELOADING并重试加载CU。 + */ m_cache_mgr->RealeseCompressLock(slotId); return CU_RELOADING; } if (!cuPtr->m_cache_compressed) { + /* 另一个线程已解压缩此CU数据 */ /* another therad have decompressed this CU data */ m_cache_mgr->RealeseCompressLock(slotId); return CU_OK; } if (cuPtr->m_adio_error) { + /* IO错误 */ /* IO error */ this->m_cache_mgr->RealeseCompressLock(slotId); return CU_ERR_ADIO; } - + /* 记录当前正在解压缩的槽位ID */ /* remember this slot id */ Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressUncompress)); t_thrd.storage_cxt.CacheBlockInProgressUncompress = slotId; if (cuPtr->CheckCrc() == false) { /* CRC check failed */ + /* CRC检查失败 */ m_cache_mgr->RealeseCompressLock(slotId); return CU_ERR_CRC; } @@ -461,7 +555,7 @@ CUUncompressedRetCode DataCacheMgr::StartUncompressCU( m_cache_mgr->RealeseCompressLock(slotId); return CU_ERR_MAGIC; } - + /* 用于跟踪cstore扫描的宏 */ /* macro for tracing cstore scan */ #define UNCOMPRESS_TRACE(A) \ do { \ @@ -469,26 +563,29 @@ CUUncompressedRetCode DataCacheMgr::StartUncompressCU( A; \ } \ } while (0) - + /* 始终假设磁盘上是压缩数据，缓存中是未压缩数据。 */ /* Always presume compressed disk and uncompressed cache. */ UNCOMPRESS_TRACE(TRACK_START(planNodeId, UNCOMPRESS_CU)); cuPtr->UnCompress(cuDescPtr->row_count, cuDescPtr->magic, align_size); UNCOMPRESS_TRACE(TRACK_END(planNodeId, UNCOMPRESS_CU)); - + /* 不要将压缩缓冲区放入缓存中 + * 将来我们可能在缓存中保存压缩数据或让调用者选择，但目前我们只保留未压缩数据。 + */ /* Do not put the compressedBuf in the cache * In the future we may have compressed data in the cache or let the * caller choose, but for now we just keep the uncompressed data. */ - cuPtr->FreeCompressBuf(); + cuPtr->FreeCompressBuf();// 释放压缩缓冲区 /* Adjust the allocation reservation to take into account * compression or expansion. */ + /* 调整内存分配预留，考虑压缩或扩展 */ int cu_uncompress_size = cuPtr->GetUncompressBufSize(); m_cache_mgr->AdjustCacheMem(slotId, cuDescPtr->cu_size, cu_uncompress_size); - m_cache_mgr->RealeseCompressLock(slotId); + m_cache_mgr->RealeseCompressLock(slotId);// 释放解压缩锁 - TerminateCU(false); + TerminateCU(false);// 终止当前CU的处理 return CU_OK; } @@ -497,6 +594,9 @@ CUUncompressedRetCode DataCacheMgr::StartUncompressCU( * @Return: cache used size * @See also: */ +// get data cache manage current memory cache used size +// 返回数据缓存管理器当前的内存缓存使用大小 + int64 DataCacheMgr::GetCurrentMemSize() { if (!IS_SINGLE_NODE) @@ -514,6 +614,12 @@ int64 DataCacheMgr::GetCurrentMemSize() * @Return: true, means error happen in preftech * @See also: */ +// DataBlockWaitIO +/* 如果CU处于IOBUSY状态，则睡眠等待IO busy锁。 + * 如果在等待期间被唤醒，则释放锁并再次检查。 + * 该方法基于rowstore中的WaitIO()函数。 + */ +//等待指定槽位ID的CU IO操作完成 bool DataCacheMgr::DataBlockWaitIO(CacheSlotId_t slotId) { return m_cache_mgr->WaitIO(slotId); @@ -526,6 +632,10 @@ bool DataCacheMgr::DataBlockWaitIO(CacheSlotId_t slotId) * @Param[IN] slotId: slot id * @See also: */ +// DataBlockCompleteIO +/* 当I/O操作完成时，将CU标记为非忙状态并唤醒下一个等待者。 + * 该方法类似于列存储中的TerminateBufferIO函数。 + */ void DataCacheMgr::DataBlockCompleteIO(CacheSlotId_t slotId) { m_cache_mgr->CompleteIO(slotId); @@ -542,6 +652,7 @@ void DataCacheMgr::DataBlockCompleteIO(CacheSlotId_t slotId) * @Return: true -- lock; false --not lock * @See also: */ +// 检查当前线程是否持有指定槽位ID的CU描述符锁 bool DataCacheMgr::CULWLockHeldByMe(CacheSlotId_t slotId) { return m_cache_mgr->LockHeldByMe(slotId); @@ -552,6 +663,7 @@ bool DataCacheMgr::CULWLockHeldByMe(CacheSlotId_t slotId) * @Param[IN] slotId: slot id * @See also: */ +// 获取指定槽位ID的CU描述符锁 void DataCacheMgr::CULWLockOwn(CacheSlotId_t slotId) { return m_cache_mgr->LockOwn(slotId); @@ -562,6 +674,7 @@ void DataCacheMgr::CULWLockOwn(CacheSlotId_t slotId) * @Param[IN] slotId: slot id * @See also: */ +// 释放指定槽位ID的CU描述符锁 void DataCacheMgr::CULWLockDisown(CacheSlotId_t slotId) { return m_cache_mgr->LockDisown(slotId); @@ -571,6 +684,7 @@ void DataCacheMgr::CULWLockDisown(CacheSlotId_t slotId) * @Description: used for adio write * @See also: */ +// 用于adio写操作时锁住私有缓存 void DataCacheMgr::LockPrivateCache() { SpinLockAcquire(&m_adio_write_cache_lock); @@ -580,6 +694,7 @@ void DataCacheMgr::LockPrivateCache() * @Description: used for adio write * @See also: */ +// 用于adio写操作时释放私有缓存锁 void DataCacheMgr::UnLockPrivateCache() { SpinLockRelease(&m_adio_write_cache_lock); @@ -590,59 +705,64 @@ void DataCacheMgr::UnLockPrivateCache() * @IN slotId: slot id * @See also: */ +//在发现数据缓存资源泄漏时打印相关的警告信息 void DataCacheMgr::PrintDataCacheSlotLeakWarning(CacheSlotId_t slotId) { - DataSlotTag tag = {}; - uint32 refcount = 0; + // 标记和引用计数 + DataSlotTag tag = {};// 数据槽位标记 + uint32 refcount = 0; // 引用计数 - Assert(IsValidCacheSlotID(slotId)); - - const CacheTag* cacheTag = m_cache_mgr->GetCacheBlockTag(slotId, &refcount); + Assert(IsValidCacheSlotID(slotId));// 检查槽位ID的有效性 + // 获取缓存标签 + const CacheTag *cacheTag = m_cache_mgr->GetCacheBlockTag(slotId, &refcount); // 获取缓存标签和引用计数 + // 如果缓存标签为空，则输出错误信息 if (cacheTag == NULL) { ereport(ERROR, (errmsg("No cache found when print data cache slot leak warning: slotId:%d", slotId))); } + // 复制标签到tag中 errno_t rc = memcpy_s(&(tag.slotTag), sizeof(DataSlotTagKey), cacheTag->key, sizeof(DataSlotTagKey)); securec_check(rc, "\0", "\0"); - if (cacheTag->type == CACHE_COlUMN_DATA) { - CUSlotTag* cuslotTag = &tag.slotTag.cuSlotTag; + // 根据不同的缓存类型，输出不同的警告信息 + if (cacheTag->type == CACHE_COlUMN_DATA) { // 列式数据缓存 + CUSlotTag* cuslotTag = &tag.slotTag.cuSlotTag;// 列式数据槽位标记 ereport(WARNING, (errmsg("CUCache refcount leak: type:%d, spaceNode: %u, dbNode: %u, relNode: %u, colId: %d, cuId: %d, " "cuPoint: %lu, refcount: %u, slotId:%d", cacheTag->type, - cuslotTag->m_rnode.spcNode, - cuslotTag->m_rnode.dbNode, - cuslotTag->m_rnode.relNode, - cuslotTag->m_colId, - cuslotTag->m_CUId, - cuslotTag->m_cuPtr, - refcount, - slotId))); - } else if (cacheTag->type == CACHE_ORC_DATA) { - ORCSlotTag* orcslotTag = &tag.slotTag.orcSlotTag; + cuslotTag->m_rnode.spcNode,// 表空间节点 + cuslotTag->m_rnode.dbNode,// 数据库节点 + cuslotTag->m_rnode.relNode,// 表节点 + cuslotTag->m_colId,// 列ID + cuslotTag->m_CUId,// CU ID + cuslotTag->m_cuPtr,// CU指针 + refcount,// 引用计数 + slotId)));// 槽位ID + } else if (cacheTag->type == CACHE_ORC_DATA) { // ORC数据缓存 + ORCSlotTag* orcslotTag = &tag.slotTag.orcSlotTag;// ORC数据槽位标记 ereport(WARNING, (errmsg("ORCCache refcount leak: type:%d, spaceNode: %u, dbNode: %u, relNode: %u, fileId: %d, length: %lu, " "offset: %lu, refcount: %u, slotId:%d", cacheTag->type, - orcslotTag->m_rnode.spcNode, - orcslotTag->m_rnode.dbNode, - orcslotTag->m_rnode.relNode, - orcslotTag->m_fileId, - orcslotTag->m_length, - orcslotTag->m_offset, - refcount, - slotId))); - } else if (cacheTag->type == CACHE_OBS_DATA) { - OBSSlotTag* obsslotTag = &tag.slotTag.obsSlotTag; + orcslotTag->m_rnode.spcNode,// 表空间节点 + orcslotTag->m_rnode.dbNode,// 数据库节点 + orcslotTag->m_rnode.relNode,// 表节点 + orcslotTag->m_fileId,// 文件ID + orcslotTag->m_length,// 长度 + orcslotTag->m_offset,// 偏移量 + refcount,// 引用计数 + slotId)));// 槽位ID + } else if (cacheTag->type == CACHE_OBS_DATA) {// OBS数据缓存 + OBSSlotTag* obsslotTag = &tag.slotTag.obsSlotTag;// OBS数据槽位标记 ereport(WARNING, (errmsg("OBSCache refcount leak: type:%d, server hash: %u, bucket hash: %u, file hash1: %u, file hash2: " "%u, length: %lu, offset: %lu", cacheTag->type, - obsslotTag->m_serverHash, - obsslotTag->m_bucketHash, - obsslotTag->m_fileFirstHash, - obsslotTag->m_fileSecondHash, - obsslotTag->m_length, - obsslotTag->m_offset))); + obsslotTag->m_serverHash,// 服务器哈希值 + obsslotTag->m_bucketHash,// 桶哈希值 + obsslotTag->m_fileFirstHash,// 文件哈希值1 + obsslotTag->m_fileSecondHash,// 文件哈希值2 + obsslotTag->m_length, // 长度 + obsslotTag->m_offset))); // 偏移量 } return; @@ -655,13 +775,22 @@ void DataCacheMgr::PrintDataCacheSlotLeakWarning(CacheSlotId_t slotId) * @IN slotId: slot id * @See also: */ +/* + * @Description: 设置数据块的值，用于ORC数据缓存 + * @IN buffer: 缓冲区指针 + * @IN size: 缓冲区大小 + * @IN slotId: 槽位ID + * @See also: + */ +//设置ORC数据缓存块的值 void DataCacheMgr::SetORCDataBlockValue(CacheSlotId_t slotId, const void* buffer, uint64 size) { + // 获取ORC数据缓存块 OrcDataValue* orcDataValue = GetORCDataBuf(slotId); - + // 设置ORC数据缓存块的大小和值 orcDataValue->size = size; orcDataValue->value = (char*)CStoreMemAlloc::Palloc(size, false); - + // 将缓冲区的数据复制到ORC数据缓存块中 errno_t rc = memcpy_s(orcDataValue->value, size, buffer, size); securec_check(rc, "\0", "\0"); @@ -674,7 +803,7 @@ void DataCacheMgr::SetOBSDataBlockValue( CacheSlotId_t slotId, const void* buffer, uint64 size, const char* prefix, const char* dataDNA) { OrcDataValue* orcDataValue = GetORCDataBuf(slotId); - + // 计算缓冲区总大小，包括前缀、数据DNA和数据本身 /* prefix + '\0' + dataDNA + '\0' + data */ size_t prefixLen = strlen(prefix); size_t dataDNALen = strlen(dataDNA); @@ -682,28 +811,28 @@ void DataCacheMgr::SetOBSDataBlockValue( orcDataValue->size = size + prefixLen + 1 + dataDNALen + 1; orcDataValue->value = (char*)CStoreMemAlloc::Palloc(orcDataValue->size, false); - + // 将缓冲区的值置为0 /* set whole buffer to 0 */ rc = memset_s(orcDataValue->value, orcDataValue->size, 0, orcDataValue->size); securec_check(rc, "\0", "\0"); - + // 复制前缀到缓冲区 /* copy prefix */ rc = memcpy_s(orcDataValue->value, orcDataValue->size, prefix, prefixLen); securec_check(rc, "\0", "\0"); - + // 复制数据DNA到缓冲区 /* copy data DNA */ rc = memcpy_s((char*)orcDataValue->value + prefixLen + 1, orcDataValue->size - (prefixLen + 1), dataDNA, dataDNALen); securec_check(rc, "\0", "\0"); - + // 复制数据到缓冲区 /* copy data */ rc = memcpy_s((char*)orcDataValue->value + prefixLen + 1 + dataDNALen + 1, size, buffer, size); securec_check(rc, "\0", "\0"); - + // 输出调试信息，包括设置的槽位ID和大小 ereport(DEBUG1, (errmodule(MOD_CACHE), errmsg("set orc data block slot(%d), size(%lu)", slotId, size))); return; @@ -714,6 +843,9 @@ void DataCacheMgr::SetOBSDataBlockValue( * @IN slotId: slot id * @See also: */ +// 函数作用：获取压缩锁 +// 参数： +// - slotId: 槽位ID void DataCacheMgr::AcquireCompressLock(CacheSlotId_t slotId) { return m_cache_mgr->AcquireCompressLock(slotId); @@ -724,6 +856,9 @@ void DataCacheMgr::AcquireCompressLock(CacheSlotId_t slotId) * @IN slotId: slot id * @See also: */ +// 函数作用：释放压缩锁 +// 参数： +// - slotId: 槽位ID void DataCacheMgr::RealeseCompressLock(CacheSlotId_t slotId) { return m_cache_mgr->RealeseCompressLock(slotId); @@ -736,12 +871,17 @@ void DataCacheMgr::RealeseCompressLock(CacheSlotId_t slotId) * @Return: always success * @See also: */ +// 函数作用：cstore预取读取完成的回调函数 +// 参数： +// - aioDesc: aio描述符 +// - res: 读取到的字节数 +// 返回值：始终返回成功（0） int CompltrReadCUReq(void* aioDesc, long res) { AioDispatchCUDesc_t* desc = (AioDispatchCUDesc_t*)aioDesc; START_CRIT_SECTION(); - + // 检查读取结果是否与期望的CU大小一致 if (res != desc->cuDesc.size) { *(desc->cuDesc.io_error) = true; ereport(WARNING, @@ -752,11 +892,13 @@ int CompltrReadCUReq(void* aioDesc, long res) } else { *(desc->cuDesc.io_error) = false; } - + // 断言缓存槽位ID有效 Assert(IsValidCacheSlotID(desc->cuDesc.slotId)); + // 拥有CULWLock（读写锁）的所有权 CUCache->CULWLockOwn(desc->cuDesc.slotId); + // 完成对数据块的IO操作 CUCache->DataBlockCompleteIO(desc->cuDesc.slotId); - + // 输出调试信息，包括槽位ID ereport(DEBUG1, (errmodule(MOD_ADIO), errmsg("CompltrReadCUReq!slotid(%d) ", desc->cuDesc.slotId))); END_CRIT_SECTION(); @@ -772,11 +914,19 @@ int CompltrReadCUReq(void* aioDesc, long res) * @Return: always success * @See also: */ +/** + * @Description: cstore回写完成的回调函数 + * @Param[IN] aioDesc: aio描述符 + * @Param[IN] res: 写入的字节数 + * @Return: 始终返回成功（0） + * @See also: + */ int CompltrWriteCUReq(void* aioDesc, long res) { AioDispatchCUDesc_t* desc = (AioDispatchCUDesc_t*)aioDesc; - + // 开启临界区 START_CRIT_SECTION(); + // 检查写入结果是否与期望的CU大小一致 if (res != desc->cuDesc.size) { *(desc->cuDesc.io_error) = true; ereport(WARNING, @@ -787,7 +937,9 @@ int CompltrWriteCUReq(void* aioDesc, long res) } else { *(desc->cuDesc.io_error) = false; } + // 标记IO操作已完成 desc->cuDesc.io_finish = true; + // 结束临界区 END_CRIT_SECTION(); return 0; @@ -798,8 +950,10 @@ int CompltrWriteCUReq(void* aioDesc, long res) * @IN CUSlotId: slot id * @See also: */ +//释放数据块的引用 void ReleaseORCBlock(CacheSlotId_t slot) { + // 调用ORCCache的UnPinDataBlock函数，解除对数据块的引用 ORCCache->UnPinDataBlock(slot); } @@ -815,15 +969,31 @@ void ReleaseORCBlock(CacheSlotId_t slot) * @See also: we do not want to ereport(error) when found error, becasue this function called by * liborc(orc.HdfsCacheFileInputStream.read) */ +/** + * @Description: 分配ORC数据缓存块 + * @Param[IN] rnode: 文件节点 + * @Param[IN] relid: 关系ID + * @Param[IN] offset: 偏移量 + * @Param[IN] length: 长度 + * @Param[IN/OUT] found: 是否找到槽位 + * @Param[IN/OUT] err_found: 是否发现错误 + * @Return: ORC数据槽位ID + * @See also: + * 当发现错误时，我们不希望通过ereport(error)报告错误，因为该函数由liborc(orc.HdfsCacheFileInputStream.read)调用 + */ +//分配ORC数据缓存块 CacheSlotId_t ORCCacheAllocBlock( RelFileNode* rnode, int32 relid, uint64 offset, uint64 length, bool& found, bool& err_found) { err_found = false; int maxRetry = 3; + // 初始化ORC数据槽位标签 DataSlotTag dataSlotTag = ORCCache->InitORCSlotTag(rnode, relid, offset, length); + // 在ORC缓存中查找数据块 CacheSlotId_t slotId = ORCCache->FindDataBlock(&dataSlotTag, true); if (IsValidCacheSlotID(slotId)) { found = true; + // 预留数据块 } else { found = false; slotId = ORCCache->ReserveDataBlock(&dataSlotTag, length, found); @@ -831,9 +1001,11 @@ CacheSlotId_t ORCCacheAllocBlock( while (found) { if (ORCCache->DataBlockWaitIO(slotId)) { + // 等待IO操作完成后解除对数据块的引用 ORCCache->UnPinDataBlock(slotId); if (maxRetry-- <= 0) { err_found = true; + // 输出日志，报告分配ORC数据时发生错误的信息 ereport(LOG, (errmodule(MOD_ORC), errmsg("wait IO find an error when allocate orc data, slotID(%d), spcID(%u), dbID(%u), " @@ -847,6 +1019,7 @@ CacheSlotId_t ORCCacheAllocBlock( length))); break; } else { + // 重新预留数据块 slotId = ORCCache->ReserveDataBlock(&dataSlotTag, length, found); continue; } @@ -863,6 +1036,11 @@ CacheSlotId_t ORCCacheAllocBlock( * @Return: true for the session can evict the cache. * false for some other session is updating it concurrently. */ +/** + * @Description: 释放缓存并从OBS重新加载数据 + * @Param[IN] slotID: 要驱逐的缓存槽位ID + * @Return: 如果会话可以驱逐缓存，则返回true。如果其他会话正在并发更新该缓存，则返回false。 + */ bool OBSCacheRenewBlock(CacheSlotId_t slotID) { return (OBSCache->ReserveDataBlockWithSlotId(slotID)); @@ -880,62 +1058,82 @@ bool OBSCacheRenewBlock(CacheSlotId_t slotID) * @See also: we do not want to ereport(error) when found error, becasue this function called by * liborc(orc.HdfsCacheFileInputStream.read) */ +//在ORC缓存中分配一个数据缓存块 CacheSlotId_t OBSCacheAllocBlock(const char* hostName, const char* bucketName, const char* prefixName, uint64 offset, uint64 length, bool& found, bool& err_found) { Assert(hostName && bucketName && prefixName); - + // 获取主机名、存储桶名称和文件前缀名的长度 uint32 hostNameLen = strlen(hostName); uint32 buckectNameLen = strlen(bucketName); uint32 prefixNameLen = strlen(prefixName); - + // 计算主机名和存储桶名称的哈希值 uint32 hostNameHash = string_hash((void*)hostName, hostNameLen + 1); uint32 bucketNameHash = string_hash((void*)bucketName, buckectNameLen + 1); - + // 计算文件前半部分和后半部分的哈希值 uint32 fileFirstHalfHash = string_hash((void*)prefixName, prefixNameLen >> 1); uint32 fileSecondHalfHash = string_hash((void*)(prefixName + (prefixNameLen >> 1)), strlen(prefixName + (prefixNameLen >> 1)) + 1); - + // 初始化DataSlotTag结构体，填入哈希值和偏移量、长度等信息 err_found = false; int maxRetry = 3; DataSlotTag dataSlotTag = OBSCache->InitOBSSlotTag(hostNameHash, bucketNameHash, fileFirstHalfHash, fileSecondHalfHash, offset, length); + // 尝试从缓存中查找数据槽位 CacheSlotId_t slotId = OBSCache->FindDataBlock(&dataSlotTag, true); if (IsValidCacheSlotID(slotId)) { - found = true; + found = true;// 找到了有效的缓存槽位 } else { - found = false; + found = false;// 没有找到有效的缓存槽位 + // 预留一个数据块 slotId = OBSCache->ReserveDataBlock(&dataSlotTag, length, found); } - + // 如果缓存槽位正在被其他会话并发更新，则等待IO完成 while (found) { if (OBSCache->DataBlockWaitIO(slotId)) { - OBSCache->UnPinDataBlock(slotId); + OBSCache->UnPinDataBlock(slotId);// 解锁缓存块 if (maxRetry-- > 0) { + // 再次尝试预留数据块 slotId = OBSCache->ReserveDataBlock(&dataSlotTag, length, found); continue; } else { - err_found = true; + err_found = true;// 达到最大重试次数，表示发生错误 break; } } else { break; } } - return slotId; + return slotId;// 返回缓存槽位ID } - +/** + * @Description: 根据缓存块ID获取ORC数据缓存块的指针 + * @Param[IN] slot: 缓存槽位ID + * @Return: 返回ORC数据缓存块的指针 + */ OrcDataValue* ORCCacheGetBlock(CacheSlotId_t slot) { return ORCCache->GetORCDataBuf(slot); } - +/** + * @Description: 将指定大小的数据写入ORC数据缓存块，并标记IO操作完成 + * @Param[IN] slotId: 缓存槽位ID + * @Param[IN] buffer: 待写入数据的缓冲区指针 + * @Param[IN] size: 待写入数据的字节数 + */ void ORCCacheSetBlock(CacheSlotId_t slotId, const void* buffer, uint64 size) { ORCCache->SetORCDataBlockValue(slotId, buffer, size); ORCCache->DataBlockCompleteIO(slotId); } - +/** + * @Description: 将指定大小的数据写入OBS数据缓存块，并标记IO操作完成 + * @Param[IN] slotId: 缓存槽位ID + * @Param[IN] buffer: 待写入数据的缓冲区指针 + * @Param[IN] size: 待写入数据的字节数 + * @Param[IN] prefix: 文件前缀名 + * @Param[IN] dataDNA: 数据的DNA值 + */ void OBSCacheSetBlock(CacheSlotId_t slotId, const void* buffer, uint64 size, const char* prefix, const char* dataDNA) { OBSCache->SetOBSDataBlockValue(slotId, buffer, size, prefix, dataDNA); -- 2.34.1 From badf1e6b54096d35fff635b5632c08640acb42c2 Mon Sep 17 00:00:00 2001 From: Auweafan Date: Wed, 4 Oct 2023 16:47:11 +0800 Subject: [PATCH 46/48] Update xlog_share_storage.cpp --- .../xlog_share_storage/xlog_share_storage.cpp | 677 +++++++++++++----- 1 file changed, 496 insertions(+), 181 deletions(-) diff --git a/src/gausskernel/storage/xlog_share_storage/xlog_share_storage.cpp b/src/gausskernel/storage/xlog_share_storage/xlog_share_storage.cpp index 0eb3b6429..78278baa0 100644 --- a/src/gausskernel/storage/xlog_share_storage/xlog_share_storage.cpp +++ b/src/gausskernel/storage/xlog_share_storage/xlog_share_storage.cpp @@ -1,4 +1,4 @@ -/* +7/* * Copyright (c) 2020 Huawei Technologies Co.,Ltd. * * openGauss is licensed under Mulan PSL v2. @@ -44,7 +44,18 @@ #include "replication/shared_storage_walreceiver.h" #include "replication/walreceiver.h" -static int copyFile = -1; + /* + 静态变量和常量定义: + copyFile 复制文件的文件描述符 + copyFileTLI 复制文件的时间线ID。 + copySegNo 复制文件的起始段号。 + copyOff 复制文件的起始偏移量。 + LOCK_LEN 锁的长度 + CHECK_LOCK_INTERVAL 检查锁的时间间隔 + MAX_SIZE_CAN_COPY_TO_SHARE 可以复制到共享存储的最大文件大小 + PTR_PRINT_SHIFT_SIZE 输出指针时的位移大小 + */ + static int copyFile = -1; static TimeLineID copyFileTLI = 0; static XLogSegNo copySegNo = 0; static uint32 copyOff = 0; @@ -56,30 +67,35 @@ const static uint32 PTR_PRINT_SHIFT_SIZE = 32; static void SharedStorageXlogCopyBackendSigHupHandler(SIGNAL_ARGS) { - int save_errno = errno; + int save_errno = errno; // 保存当前的错误号，以便后续恢复。 - t_thrd.sharestoragexlogcopyer_cxt.got_SIGHUP = true; - if (t_thrd.proc) { - SetLatch(&t_thrd.proc->procLatch); + t_thrd.sharestoragexlogcopyer_cxt.got_SIGHUP = true; // 将全局变量got_SIGHUP设置为true，表示接收到了SIGHUP信号。 + + if (t_thrd.proc) { // 如果当前线程的进程控制块存在。 + SetLatch(&t_thrd.proc->procLatch); // 设置该进程的procLatch，唤醒其处理程序。 } - errno = save_errno; + + errno = save_errno; // 恢复之前保存的错误号。 } + static void SharedStorageXlogCopyBackendShutdownHandler(SIGNAL_ARGS) { - int save_errno = errno; + int save_errno = errno; // 保存当前的错误号，以便后续恢复。 - t_thrd.sharestoragexlogcopyer_cxt.shutdown_requested = true; + t_thrd.sharestoragexlogcopyer_cxt.shutdown_requested = + true; // 将全局变量shutdown_requested设置为true，表示请求进行关闭操作。 - if (t_thrd.proc) - SetLatch(&t_thrd.proc->procLatch); + if (t_thrd.proc) // 如果当前线程的进程控制块存在。 + SetLatch(&t_thrd.proc->procLatch); // 设置该进程的procLatch，唤醒其处理程序。 - errno = save_errno; + errno = save_errno; // 恢复之前保存的错误号。 } + static void SharedStorageXlogCopyBackendQuickDie(SIGNAL_ARGS) { - gs_signal_setmask(&t_thrd.libpq_cxt.BlockSig, NULL); + gs_signal_setmask(&t_thrd.libpq_cxt.BlockSig, NULL);// 解除对指定信号的阻塞。 /* * We DO NOT want to run proc_exit() callbacks -- we're here because @@ -89,7 +105,9 @@ static void SharedStorageXlogCopyBackendQuickDie(SIGNAL_ARGS) * things by calling exit() directly, we have to reset the callbacks * explicitly to make this work as intended. */ - on_exit_reset(); + on_exit_reset();// 重置退出时的回调函数，避免执行进程退出处理过程。 + + /* /* * Note we do exit(2) not exit(0). This is to force the postmaster into a @@ -99,53 +117,56 @@ static void SharedStorageXlogCopyBackendQuickDie(SIGNAL_ARGS) * should ensure the postmaster sees this as a crash, too, but no harm in * being doubly sure.) */ - exit(2); + exit(2); // 以退出代码2结束该进程。 } static void SharedStorageXlogCopyBackendSigUsr1Handler(SIGNAL_ARGS) { - int saveErrno = errno; + int saveErrno = errno; // 保存当前的 errno 值。 - latch_sigusr1_handler(); + latch_sigusr1_handler(); // 调用 latch_sigusr1_handler() 处理 SIGUSR1 信号。 - errno = saveErrno; + errno = saveErrno; // 恢复之前保存的 errno 值。 } + void LocalXLogRead(char *buf, XLogRecPtr startptr, Size count) { - char *p = NULL; - XLogRecPtr recptr; - Size nbytes; + char *p = NULL; // 定义字符指针变量p，用于指向读取到的数据的存储位置 + XLogRecPtr recptr; // 定义XLogRecPtr类型变量recptr，表示当前读取到的XLog记录的位置 + Size nbytes; // 定义Size类型变量nbytes，表示还需要读取的字节数 - p = buf; - recptr = startptr; - nbytes = count; + p = buf; // 将buf赋值给p，即将p指向待存储数据的位置 + recptr = startptr; // 将startptr赋值给recptr，初始化recptr为待读取位置 + nbytes = count; // 将count赋值给nbytes，初始化nbytes为待读取字节数 while (nbytes > 0) { - uint32 startoff; - int segbytes; - int readbytes; + uint32 startoff; // 定义uint32类型变量startoff，表示当前XLog记录在所在数据段中的偏移量 + int segbytes; // 定义int类型变量segbytes，表示当前数据段中的可读字节数 + int readbytes; // 定义int类型变量readbytes，表示本次实际读取的字节数 - startoff = recptr % XLogSegSize; + startoff = recptr % XLogSegSize; // 计算出recptr相对于数据段的偏移量 - /* Do we need to switch to a different xlog segment? */ + // 是否需要切换到下一个数据段？ if (t_thrd.sharestoragexlogcopyer_cxt.readFile < 0 || !XLByteInSeg(recptr, t_thrd.sharestoragexlogcopyer_cxt.readSegNo)) { - char path[MAXPGPATH]; + + char path[MAXPGPATH]; // 定义字符数组path，用于存储数据段的路径名 if (t_thrd.sharestoragexlogcopyer_cxt.readFile >= 0) { - (void)close(t_thrd.sharestoragexlogcopyer_cxt.readFile); + (void)close(t_thrd.sharestoragexlogcopyer_cxt.readFile); // 关闭当前打开的文件 } - XLByteToSeg(recptr, t_thrd.sharestoragexlogcopyer_cxt.readSegNo); - XLogFilePath(path, MAXPGPATH, t_thrd.xlog_cxt.ThisTimeLineID, t_thrd.sharestoragexlogcopyer_cxt.readSegNo); + XLByteToSeg(recptr, t_thrd.sharestoragexlogcopyer_cxt.readSegNo); // 根据recptr计算出待读取的数据段号 + XLogFilePath(path, MAXPGPATH, t_thrd.xlog_cxt.ThisTimeLineID, + t_thrd.sharestoragexlogcopyer_cxt.readSegNo); // 构造数据段的完整路径名 + // 打开数据段文件 t_thrd.sharestoragexlogcopyer_cxt.readFile = BasicOpenFile(path, O_RDONLY | PG_BINARY, 0); + if (t_thrd.sharestoragexlogcopyer_cxt.readFile < 0) { /* - * If the file is not found, assume it's because the standby - * asked for a too old WAL segment that has already been - * removed or recycled. + * 如果文件未找到，则说明已经请求了一个已被删除或回收的过时WAL段。 */ if (errno == ENOENT) { ereport(ERROR, (errcode_for_file_access(), @@ -153,42 +174,49 @@ void LocalXLogRead(char *buf, XLogRecPtr startptr, Size count) XLogFileNameP(t_thrd.xlog_cxt.ThisTimeLineID, t_thrd.sharestoragexlogcopyer_cxt.readSegNo)))); } else { + // 其他情况下报错 ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\" (log segment %s): %m", path, XLogFileNameP(t_thrd.xlog_cxt.ThisTimeLineID, t_thrd.sharestoragexlogcopyer_cxt.readSegNo)))); } } - t_thrd.sharestoragexlogcopyer_cxt.readOff = 0; + + t_thrd.sharestoragexlogcopyer_cxt.readOff = 0; // 初始化读取偏移量为0 } - /* Need to seek in the file? */ + // 需要进行文件定位吗？ if (t_thrd.sharestoragexlogcopyer_cxt.readOff != startoff) { if (lseek(t_thrd.sharestoragexlogcopyer_cxt.readFile, (off_t)startoff, SEEK_SET) < 0) { (void)close(t_thrd.sharestoragexlogcopyer_cxt.readFile); t_thrd.sharestoragexlogcopyer_cxt.readFile = -1; + + // 文件定位失败，报错 ereport(ERROR, (errcode_for_file_access(), errmsg("could not seek in log segment %s to offset %u: %m", XLogFileNameP(t_thrd.xlog_cxt.ThisTimeLineID, t_thrd.sharestoragexlogcopyer_cxt.readSegNo), startoff))); } - t_thrd.sharestoragexlogcopyer_cxt.readOff = startoff; + t_thrd.sharestoragexlogcopyer_cxt.readOff = startoff; // 更新当前读取偏移量 } - /* How many bytes are within this segment? */ + // 当前段中还有多少字节可读？ if (nbytes > (XLogSegSize - startoff)) { segbytes = XLogSegSize - startoff; } else { segbytes = nbytes; } - pgstat_report_waitevent(WAIT_EVENT_WAL_READ); - readbytes = read(t_thrd.sharestoragexlogcopyer_cxt.readFile, p, segbytes); - pgstat_report_waitevent(WAIT_EVENT_END); + pgstat_report_waitevent(WAIT_EVENT_WAL_READ); // 上报统计信息，表示正在等待WAL读取过程 + readbytes = read(t_thrd.sharestoragexlogcopyer_cxt.readFile, p, segbytes); // 从文件中读取字节数据 + pgstat_report_waitevent(WAIT_EVENT_END); // 上报统计信息，表示WAL读取过程结束 + if (readbytes <= 0) { (void)close(t_thrd.sharestoragexlogcopyer_cxt.readFile); t_thrd.sharestoragexlogcopyer_cxt.readFile = -1; + + // 读取失败，报错 ereport(ERROR, (errcode_for_file_access(), errmsg("could not read from log segment %s, offset %u, length %lu: %m", @@ -196,128 +224,215 @@ void LocalXLogRead(char *buf, XLogRecPtr startptr, Size count) t_thrd.sharestoragexlogcopyer_cxt.readOff, INT2ULONG(segbytes)))); } - /* Update state for read */ - XLByteAdvance(recptr, readbytes); - - t_thrd.sharestoragexlogcopyer_cxt.readOff += readbytes; - nbytes -= readbytes; - p += readbytes; + // 更新读取状态 + XLByteAdvance(recptr, readbytes); // 更新recptr，表示已经读取到该位置 + t_thrd.sharestoragexlogcopyer_cxt.readOff += readbytes; // 更新读取偏移量 + nbytes -= readbytes; // 更新待读取字节数 + p += readbytes; // 移动指针p，指向下一个存储位置 } } + void NotifySyncWaiters(XLogRecPtr newPos) { + // 获取全局变量WalSndCtl的指针 volatile WalSndCtlData *walsndctl = t_thrd.walsender_cxt.WalSndCtl; + // 获取同步锁SyncRepLock (void)LWLockAcquire(SyncRepLock, LW_EXCLUSIVE); /* * Set the lsn first so that when we wake backends they will release up to * this location. */ + + // 判断SYNC_REP_WAIT_RECEIVE同步队列中的最大位置是否小于newPos，如果是，则更新同步队列中的最大位置，并唤醒等待队列中的所有进程 if (XLByteLT(walsndctl->lsn[SYNC_REP_WAIT_RECEIVE], newPos)) { walsndctl->lsn[SYNC_REP_WAIT_RECEIVE] = newPos; (void)SyncRepWakeQueue(false, SYNC_REP_WAIT_RECEIVE); } + + // 判断SYNC_REP_WAIT_WRITE同步队列中的最大位置是否小于newPos，如果是，则更新同步队列中的最大位置，并唤醒等待队列中的所有进程 if (XLByteLT(walsndctl->lsn[SYNC_REP_WAIT_WRITE], newPos)) { walsndctl->lsn[SYNC_REP_WAIT_WRITE] = newPos; (void)SyncRepWakeQueue(false, SYNC_REP_WAIT_WRITE); } + + // 判断SYNC_REP_WAIT_FLUSH同步队列中的最大位置是否小于newPos，如果是，则更新同步队列中的最大位置，并唤醒等待队列中的所有进程 if (XLByteLT(walsndctl->lsn[SYNC_REP_WAIT_FLUSH], newPos)) { walsndctl->lsn[SYNC_REP_WAIT_FLUSH] = newPos; (void)SyncRepWakeQueue(false, SYNC_REP_WAIT_FLUSH); } + // 释放同步锁 LWLockRelease(SyncRepLock); } + void PushCtlLsn(XLogRecPtr flushPtr) { + // 获取共享存储XLog控制块的指针 ShareStorageXLogCtl *sharestorageCtl = g_instance.xlog_cxt.shareStorageXLogCtl; + + // 将flushPtr设置为insertHead sharestorageCtl->insertHead = flushPtr; + + // 检查是否需要更新insertTail，如果flushPtr减去insertTail大于xlogFileSize，则需要更新insertTail if ((sharestorageCtl->insertHead - sharestorageCtl->insertTail) > sharestorageCtl->xlogFileSize) { sharestorageCtl->insertTail = sharestorageCtl->insertHead - sharestorageCtl->xlogFileSize; } + + // 计算新的共享存储XLog控制块的crc sharestorageCtl->crc = CalShareStorageCtlInfoCrc(sharestorageCtl); + + // 更新共享存储XLog控制块的信息到磁盘 UpdateShareStorageCtlInfo(sharestorageCtl); + + // 将XLog数据刷新到共享存储 FsyncXlogToShareStorage(); + + // 根据配置的日志级别决定输出日志信息，打印插入位置的LSN（高32位和低32位） int mode = u_sess->attr.attr_storage.HaModuleDebug ? LOG : DEBUG2; const uint32 shftSz = 32; ereport(mode, (errmsg("[PushCtlLsn]insertHead lsn %08X/%08X\n", (uint32)(flushPtr >> shftSz), (uint32)flushPtr))); + + // 通知等待同步的进程，传递最新的写入位置 NotifySyncWaiters(flushPtr); } + static inline int CalcWriteLen(XLogRecPtr startWrite, XLogRecPtr endPtr) { + // 检查起始位置是否为XLOG块大小的整数倍 Assert((startWrite % XLOG_BLCKSZ) == 0); + + // 对齐写入结束位置 XLogRecPtr alignWriteEnd = startWrite - startWrite % ShareStorageBufSize + ShareStorageBufSize; + + // 如果对齐后的结束位置大于实际结束位置，则返回实际结束位置到起始位置的距离 if (alignWriteEnd > endPtr) { XLogRecPtr ActualCopyEnd = TYPEALIGN(XLOG_BLCKSZ, endPtr); return static_cast(ActualCopyEnd - startWrite); - } else { + } + // 否则返回对齐后的结束位置到起始位置的距离 + else { return static_cast(alignWriteEnd - startWrite); } } + XLogRecPtr GetMaxPosCanOverWrite() { if (!IsUnderPostmaster) { return MAX_XLOG_REC_PTR; } + // 如果不在Postmaster进程下，直接返回最大的XLog位置（可以覆盖任何位置的XLog） XLogRecPtr maxFlush = InvalidXLogRecPtr; + // 初始化最大刷新位置为无效值 + for (int i = 0; i < g_instance.attr.attr_storage.max_wal_senders; i++) { + // 遍历所有的WAL发送进程 + /* use volatile pointer to prevent code rearrangement */ volatile WalSnd *walsnd = &t_thrd.walsender_cxt.WalSndCtl->walsnds[i]; + // 使用volatile指针防止代码重排 + SpinLockAcquire(&walsnd->mutex); + // 获取WAL发送进程的互斥锁（mutex） + if (walsnd->pid != 0 && walsnd->peer_role == STANDBY_CLUSTER_MODE) { + // 如果该WAL发送进程的进程ID不为0，并且角色为STANDBY_CLUSTER_MODE + if (XLByteLT(maxFlush, walsnd->flush)) { maxFlush = walsnd->flush; } + // 更新最大刷新位置为当前WAL发送进程的刷新位置（如果大于之前的最大刷新位置） } + SpinLockRelease(&walsnd->mutex); + // 释放WAL发送进程的互斥锁 } if (maxFlush == InvalidXLogRecPtr) { return MAX_XLOG_REC_PTR; } + // 如果找不到符合条件的WAL发送进程，直接返回最大的XLog位置（可以覆盖任何位置的XLog） return g_instance.attr.attr_storage.xlog_file_size + maxFlush; + // 返回XLog文件大小加上最大的刷新位置，表示可以覆盖到的最大位置 } -void AddXLogPageHeader(char* buf, XLogRecPtr startWrite, int writeLen, XLogRecPtr endPtr) + +void AddXLogPageHeader(char *buf, XLogRecPtr startWrite, int writeLen, XLogRecPtr endPtr) { int offset = 0; - while (offset < writeLen) { + // 初始化偏移量为0 + + while (offset < writeLen) { + // 当偏移量小于写入长度时循环 + XLogPageHeader xlogPageHeader = (XLogPageHeader)(buf + offset); + // 将buf偏移量位置的地址转换为XLogPageHeader类型指针 + xlogPageHeader->xlp_total_len = XLOG_BLCKSZ; + // 设置XLogPageHeader的xlp_total_len字段为XLOG_BLCKSZ（XLog页面大小） + offset += XLOG_BLCKSZ; + // 增加偏移量一个XLog页面大小 } if (XLByteLT(endPtr, startWrite + writeLen)) { + // 如果endPtr小于startWrite + writeLen + XLogPageHeader xlogPageHeader = (XLogPageHeader)(buf + writeLen - XLOG_BLCKSZ); + // 将buf偏移量为writeLen - XLOG_BLCKSZ的地址转换为XLogPageHeader类型指针 + xlogPageHeader->xlp_total_len = endPtr % XLOG_BLCKSZ; + // 设置XLogPageHeader的xlp_total_len字段为endPtr对XLOG_BLCKSZ取模的结果 } } + void DoXlogCopy(XLogRecPtr targetPtr) { uint64 writeLength = 0; ShareStorageXLogCtl *sharestorageCtl = g_instance.xlog_cxt.shareStorageXLogCtl; + // 获取共享存储相关控制信息 + Assert(targetPtr != InvalidXLogRecPtr); + // 断言传入的目标位置不为InvalidXLogRecPtr + if (XLByteLE(targetPtr, sharestorageCtl->insertHead)) { return; } + // 如果目标位置在当前插入点的前面，则返回。 XLogRecPtr startWrite = sharestorageCtl->insertHead - (sharestorageCtl->insertHead % XLOG_BLCKSZ); + // 计算写入的起始位置，要求从XLOG页面对齐的位置开始 + while (XLByteLT(startWrite, targetPtr)) { int writeLen = CalcWriteLen(startWrite, targetPtr); + // 计算需要被写入的字节数 + LocalXLogRead(t_thrd.sharestoragexlogcopyer_cxt.buf, startWrite, static_cast(writeLen)); + // 从本地XLOG读取对应字节数的数据，存入t_thrd.sharestoragexlogcopyer_cxt.buf中 + AddXLogPageHeader(t_thrd.sharestoragexlogcopyer_cxt.buf, startWrite, writeLen, targetPtr); + // 给buf中添加XLog页面头信息 + (void)WriteXlogToShareStorage(startWrite, t_thrd.sharestoragexlogcopyer_cxt.buf, writeLen); + // 将buf中的数据写入共享存储 + startWrite += writeLen; + // 修改写入的起始位置 + writeLength += writeLen; + // 累加写入的字节数 + if ((writeLength >= XLogSegSize) || (XLByteLT(targetPtr, startWrite))) { PushCtlLsn(XLByteLE(startWrite, targetPtr) ? startWrite : targetPtr); + // 将LSN信息推送给共享存储的控制进程 + writeLength = 0; } } @@ -325,58 +440,82 @@ void DoXlogCopy(XLogRecPtr targetPtr) if (writeLength > 0) { PushCtlLsn(targetPtr); } + // 如果剩余的字节数大于0，也将其LSN信息推送给共享存储的控制进程 } + static bool GetLock() { + // 检查是否配置了XLog锁文件路径，如果没有配置，直接返回true if (g_instance.attr.attr_storage.xlog_lock_file_path == NULL) { return true; } + // 尝试获取锁文件的写入锁，如果成功获取锁，则返回true if (LockNasWriteFile(g_instance.xlog_cxt.shareStorageLockFd)) { return true; } + + // 如果获取锁失败，则报告致命错误，并给出相应的错误提示信息 ereport(FATAL, (errmsg("could not lock lock file(%d) %s", g_instance.xlog_cxt.shareStorageLockFd, - g_instance.attr.attr_storage.xlog_lock_file_path))); + g_instance.attr.attr_storage.xlog_lock_file_path))); + + // 返回false，表示获取锁失败 return false; } + static bool ReleaseLock() { + // 检查是否配置了XLog锁文件路径，如果没有配置，直接返回true if (g_instance.attr.attr_storage.xlog_lock_file_path == NULL) { return true; } + // 尝试释放锁文件的写入锁，如果成功释放锁，则返回true if (UnlockNasWriteFile(g_instance.xlog_cxt.shareStorageLockFd)) { return true; } + // 如果释放锁失败，则报告致命错误，并给出相应的错误提示信息 ereport(FATAL, (errmsg("could not unlock lock file(%d) %s", g_instance.xlog_cxt.shareStorageLockFd, - g_instance.attr.attr_storage.xlog_lock_file_path))); + g_instance.attr.attr_storage.xlog_lock_file_path))); + + // 返回false，表示释放锁失败 return false; } + bool CheckAndCopyXLog(bool forceCopy) { + // 获取共享存储控制信息 ShareStorageXLogCtl *sharestorageCtl = g_instance.xlog_cxt.shareStorageXLogCtl; do { XLogRecPtr localFlush = InvalidXLogRecPtr; + // 判断主备模式是否是主节点，如果是，则获取主节点刷盘位置，否则获取本地刷盘位置 if (IS_SHARED_STORAGE_MAIN_STANDBY_MODE) { localFlush = GetFlushMainStandby(); } else { localFlush = GetFlushRecPtr(); } + + // 判断本地刷盘位置是否小于等于共享存储控制信息中的插入头部位置，如果是，则直接返回true if (XLByteLE(localFlush, sharestorageCtl->insertHead)) { return true; } + // 获取可以覆盖的最大位置 XLogRecPtr maxPosCanWrite = GetMaxPosCanOverWrite(); + // 判断插入头部位置是否小于最大可以覆盖位置，如果是，则进行xlog拷贝操作 if (XLByteLT(sharestorageCtl->insertHead, maxPosCanWrite)) { + // 计算期望的拷贝位置 XLogRecPtr expectPos = XLByteLT(localFlush, maxPosCanWrite) ? localFlush : maxPosCanWrite; + // 如果期望的拷贝位置与插入头部位置之间的距离超过了最大可拷贝的大小，就将期望位置设置为插入头部位置加上最大可拷贝的大小 if ((expectPos - sharestorageCtl->insertHead) > MAX_SIZE_CAN_COPY_TO_SHARE) { expectPos = sharestorageCtl->insertHead + MAX_SIZE_CAN_COPY_TO_SHARE; } + // 进行xlog拷贝操作 DoXlogCopy(expectPos); } } while (forceCopy && GetLock()); @@ -384,12 +523,16 @@ bool CheckAndCopyXLog(bool forceCopy) return false; } + + void ShutdownShareStorageXLogCopy() { + // 如果没有配置xlog文件路径，则直接返回 if (g_instance.attr.attr_storage.xlog_file_path == NULL) { return; } + // 分配共享存储拷贝操作相关的缓冲区 if (t_thrd.sharestoragexlogcopyer_cxt.originBuf == NULL) { t_thrd.sharestoragexlogcopyer_cxt.originBuf = (char *)palloc(ShareStorageBufSize + g_instance.xlog_cxt.shareStorageopCtl.blkSize); @@ -397,12 +540,15 @@ void ShutdownShareStorageXLogCopy() t_thrd.sharestoragexlogcopyer_cxt.originBuf); } + // 读取共享存储控制信息，并进行xlog拷贝操作 ReadShareStorageCtlInfo(g_instance.xlog_cxt.shareStorageXLogCtl); CheckAndCopyXLog(true); + // 关闭读取共享存储控制信息的文件描述符 if (t_thrd.sharestoragexlogcopyer_cxt.readFile >= 0) { (void)close(t_thrd.sharestoragexlogcopyer_cxt.readFile); } + // 释放共享存储拷贝操作相关的内存，并清除进程信号量 if (t_thrd.sharestoragexlogcopyer_cxt.originBuf != NULL) { pfree(t_thrd.sharestoragexlogcopyer_cxt.originBuf); t_thrd.sharestoragexlogcopyer_cxt.originBuf = NULL; @@ -411,16 +557,21 @@ void ShutdownShareStorageXLogCopy() g_instance.proc_base->ShareStoragexlogCopyerLatch = NULL; ereport(LOG, (errmsg("stopped xlog copy at %X/%X", - (uint32)(g_instance.xlog_cxt.shareStorageXLogCtl->insertHead >> PTR_PRINT_SHIFT_SIZE), - (uint32)g_instance.xlog_cxt.shareStorageXLogCtl->insertHead))); + (uint32)(g_instance.xlog_cxt.shareStorageXLogCtl->insertHead >> PTR_PRINT_SHIFT_SIZE), + (uint32)g_instance.xlog_cxt.shareStorageXLogCtl->insertHead))); } + bool FileSizeCanUpdate() { + // 获取endCheckpointPtr XLogRecPtr endCheckpointPtr = sizeof(CheckPointPlus) + t_thrd.shemem_ptr_cxt.ControlFile->checkPoint; + // 获取ctlInfo ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.shareStorageXLogCtl; + // 计算checkpointPos和headPos uint64 checkpointPos = endCheckpointPtr % ctlInfo->xlogFileSize; uint64 headPos = ctlInfo->insertHead % ctlInfo->xlogFileSize; + // 检查条件并返回结果 if ((XLByteLE(endCheckpointPtr, ctlInfo->insertHead) && checkpointPos <= headPos) || XLByteLT(ctlInfo->insertHead, endCheckpointPtr)) { return true; @@ -429,15 +580,20 @@ bool FileSizeCanUpdate() return false; } + void CheckShareStorageCtlInfo(XLogRecPtr localEnd) { + // 如果不是共享存储模式，则直接返回 if (!IS_SHARED_STORAGE_MODE) { return; } + // 获取ctlInfo ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.shareStorageXLogCtl; + // 读取共享存储的控制信息 ReadShareStorageCtlInfo(ctlInfo); + // 检查系统标识符是否一致，如果不一致则报错 if (ctlInfo->systemIdentifier != GetSystemIdentifier()) { ereport(FATAL, (errmsg("database system version is different between shared storage %lu and local %lu", ctlInfo->systemIdentifier, GetSystemIdentifier()))); @@ -447,11 +603,15 @@ void CheckShareStorageCtlInfo(XLogRecPtr localEnd) XLogRecPtr shareStorageLatestRecordStart = InvalidXLogRecPtr; int shareStorageLatestRecordLen; pg_crc32 shareStorageLatestRecordCrc; + // 在共享存储上查找最新的记录的信息 FindLastRecordCheckInfoOnShareStorage(&shareStorageLatestRecordStart, &shareStorageLatestRecordCrc, &shareStorageLatestRecordLen); + // 断言保证本地的end位置小于等于共享存储的insertHead位置 Assert(XLByteLE(shareStorageLatestRecordStart + shareStorageLatestRecordLen, ctlInfo->insertHead)); + // 如果本地的end位置小于共享存储的insertHead位置 if (XLByteLT(localEnd, ctlInfo->insertHead)) { + // 如果共享存储的最后一条记录的起始位置为无效值，或者本地的end位置和共享存储的最后一条记录的结束位置不相等，则报错 if (XLByteEQ(shareStorageLatestRecordStart, InvalidXLogRecPtr) || !XLByteEQ(localEnd, shareStorageLatestRecordStart + MAXALIGN(shareStorageLatestRecordLen))) { ereport(FATAL, (errmsg("the local's head is smaller than The shared storage's head"), @@ -469,6 +629,7 @@ void CheckShareStorageCtlInfo(XLogRecPtr localEnd) (errmsg("modify share storage head from %X/%X to %X/%X", static_cast(ctlInfo->insertHead >> shiftSize), static_cast(ctlInfo->insertHead), static_cast(localEnd >> shiftSize), static_cast(localEnd)))); + // 更新共享存储的insertHead位置 ctlInfo->insertHead = localEnd; ctlInfo->crc = CalShareStorageCtlInfoCrc(ctlInfo); UpdateShareStorageCtlInfo(ctlInfo); @@ -480,6 +641,7 @@ void CheckShareStorageCtlInfo(XLogRecPtr localEnd) XLogFilePath(path, MAXPGPATH, t_thrd.xlog_cxt.ThisTimeLineID, sendSegNo); struct stat stat_buf; + // 如果获取文件信息失败，则报错 if (stat(path, &stat_buf) != 0) { ereport(FATAL, (errmsg("the local's tail is bigger than The shared storage's head %X/%X", static_cast(ctlInfo->insertHead >> shiftSize), @@ -488,6 +650,7 @@ void CheckShareStorageCtlInfo(XLogRecPtr localEnd) bool crcValid = false; + // 获取本地记录的校验码，并与共享存储的最后一条记录的校验码进行比较，如果不相等则报错 pg_crc32 localCheckCrc = GetXlogRecordCrc(shareStorageLatestRecordStart, crcValid, XLogPageRead, 0); if (shareStorageLatestRecordCrc != localCheckCrc) { ereport(FATAL, (errmsg("shared storage request lsn[%X/%X]'s crc mismatched (share, local):[%u,%u].", @@ -497,8 +660,9 @@ void CheckShareStorageCtlInfo(XLogRecPtr localEnd) } } - uint32 localTerm = Max(g_instance.comm_cxt.localinfo_cxt.term_from_file, - g_instance.comm_cxt.localinfo_cxt.term_from_xlog); + // 获取本地term和ctlInfo中的term中的较大值，并更新ctlInfo中的term和crc + uint32 localTerm = + Max(g_instance.comm_cxt.localinfo_cxt.term_from_file, g_instance.comm_cxt.localinfo_cxt.term_from_xlog); if (localTerm > ctlInfo->term) { ctlInfo->term = localTerm; ctlInfo->crc = CalShareStorageCtlInfoCrc(ctlInfo); @@ -507,34 +671,44 @@ void CheckShareStorageCtlInfo(XLogRecPtr localEnd) } } + void UpdateShareStorageCtlInfo() { bool changed = false; ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.shareStorageXLogCtl; + // 读取共享存储的控制信息 ReadShareStorageCtlInfo(ctlInfo); + // 检查版本号是否一致，如果不一致则更新版本号并标记状态为已更改 if (ctlInfo->version != CURRENT_SHARE_STORAGE_CTL_VERSION) { ctlInfo->version = CURRENT_SHARE_STORAGE_CTL_VERSION; changed = true; } + // 检查长度是否一致，如果不一致则更新长度并标记状态为已更改 if (ctlInfo->length != SizeOfShareStorageXLogCtl) { ctlInfo->length = SizeOfShareStorageXLogCtl; changed = true; } + // 检查XLOG文件大小是否一致，如果不一致则进行相应处理 if (ctlInfo->xlogFileSize != (uint64)g_instance.attr.attr_storage.xlog_file_size) { + // 断言保证当前操作的ctl中的XLOG文件大小与ctlInfo中的XLOG文件大小一致 Assert(g_instance.xlog_cxt.shareStorageopCtl.xlogFileSize == ctlInfo->xlogFileSize); + // 判断文件大小是否可以更新，如果不可更新则报错 if (!FileSizeCanUpdate()) { ereport(FATAL, (errmsg("could not update share storage size."), errdetail("current size:%lu, new size:%lu", ctlInfo->xlogFileSize, g_instance.attr.attr_storage.xlog_file_size))); } + // 更新当前操作的ctl中的XLOG文件大小 g_instance.xlog_cxt.shareStorageopCtl.xlogFileSize = g_instance.attr.attr_storage.xlog_file_size; + // 更新ctlInfo中的XLOG文件大小，并标记状态为已更改 ctlInfo->xlogFileSize = g_instance.xlog_cxt.shareStorageopCtl.xlogFileSize; changed = true; } + // 如果状态已更改，则计算crc并更新共享存储的控制信息 if (changed) { ctlInfo->crc = CalShareStorageCtlInfoCrc(ctlInfo); UpdateShareStorageCtlInfo(ctlInfo); @@ -543,58 +717,95 @@ void UpdateShareStorageCtlInfo() static void SharedStorageXlogCopyBackendQuitAndClean(int code, Datum arg) { + // 输出日志信息，解锁文件 ereport(LOG, (errmsg("SharedStorageXlogCopyBackendMain unlock file(%d) %s", g_instance.xlog_cxt.shareStorageLockFd, - g_instance.attr.attr_storage.xlog_lock_file_path))); + g_instance.attr.attr_storage.xlog_lock_file_path))); ReleaseLock(); } static void InitThreadSignal() { - (void)gspqsignal(SIGHUP, SharedStorageXlogCopyBackendSigHupHandler); /* reload config file */ - (void)gspqsignal(SIGINT, SIG_IGN); /* ignore query cancel */ - (void)gspqsignal(SIGTERM, SharedStorageXlogCopyBackendShutdownHandler); /* request shutdown */ - (void)gspqsignal(SIGQUIT, SharedStorageXlogCopyBackendQuickDie); /* hard crash time */ - (void)gspqsignal(SIGALRM, SIG_IGN); - (void)gspqsignal(SIGPIPE, SIG_IGN); - (void)gspqsignal(SIGUSR1, SharedStorageXlogCopyBackendSigUsr1Handler); - (void)gspqsignal(SIGUSR2, SIG_IGN); - /* - * Reset some signals that are accepted by postmaster but not here - */ - (void)gspqsignal(SIGCHLD, SIG_DFL); - (void)gspqsignal(SIGTTIN, SIG_DFL); - (void)gspqsignal(SIGTTOU, SIG_DFL); - (void)gspqsignal(SIGCONT, SIG_DFL); - (void)gspqsignal(SIGWINCH, SIG_DFL); + // 初始化线程信号处理函数 + // 当收到SIGHUP信号时，调用SharedStorageXlogCopyBackendSigHupHandler函数重新加载配置文件 + (void)gspqsignal(SIGHUP, SharedStorageXlogCopyBackendSigHupHandler); + + // 忽略SIGINT信号，即忽略用户的终端中断信号 + (void)gspqsignal(SIGINT, SIG_IGN); + + // 当收到SIGTERM信号时，调用SharedStorageXlogCopyBackendShutdownHandler函数请求关闭 + (void)gspqsignal(SIGTERM, SharedStorageXlogCopyBackendShutdownHandler); + + // 当收到SIGQUIT信号时，调用SharedStorageXlogCopyBackendQuickDie函数进行强制崩溃 + (void)gspqsignal(SIGQUIT, SharedStorageXlogCopyBackendQuickDie); + + // 忽略SIGALRM信号，即忽略定时器信号 + (void)gspqsignal(SIGALRM, SIG_IGN); + + // 忽略SIGPIPE信号，即忽略管道破裂信号 + (void)gspqsignal(SIGPIPE, SIG_IGN); + + // 当收到SIGUSR1信号时，调用SharedStorageXlogCopyBackendSigUsr1Handler函数进行相应处理 + (void)gspqsignal(SIGUSR1, SharedStorageXlogCopyBackendSigUsr1Handler); + + // 忽略SIGUSR2信号，即忽略用户定义的信号2 + (void)gspqsignal(SIGUSR2, SIG_IGN); + + /* + * 重置一些信号的处理方式，这些信号在postmaster中被接受，但在此处使用默认的处理方式 + */ + (void)gspqsignal(SIGCHLD, SIG_DFL); // 子进程终止或停止 + (void)gspqsignal(SIGTTIN, SIG_DFL); // 后台进程从控制终端读取 + (void)gspqsignal(SIGTTOU, SIG_DFL); // 后台进程向控制终端写入 + (void)gspqsignal(SIGCONT, SIG_DFL); // 发送给正在运行或停止的进程以使其继续运行 + (void)gspqsignal(SIGWINCH, SIG_DFL); // 窗口大小更改 + + // 设置信号掩码，解除对信号的屏蔽 gs_signal_setmask(&t_thrd.libpq_cxt.UnBlockSig, NULL); + + // 解除对SIGUSR2信号的屏蔽 (void)gs_signal_unblock_sigusr2(); } + + void SharedStorageXlogCopyBackendMain(void) { + // 初始化线程信号处理函数 InitThreadSignal(); + + // 如果当前为共享存储的主备模式，则检查共享存储的写锁 if (IS_SHARED_STORAGE_MAIN_STANDBY_MODE) { CheckShareStorageWriteLock(); } + + // 如果正在进行恢复，并且当前不是共享存储的主备模式，则停止XLog的拷贝，并退出进程 if (RecoveryInProgress() && !IS_SHARED_STORAGE_MAIN_STANDBY_MODE) { ereport(LOG, (errmsg("stopped xlog copy in recovery"))); proc_exit(0); /* done */ } + // 当进程退出时，执行SharedStorageXlogCopyBackendQuitAndClean函数清理资源 on_proc_exit(SharedStorageXlogCopyBackendQuitAndClean, 0); + // 将当前进程的procLatch与共享存储XLog拷贝进程的Latch关联起来 g_instance.proc_base->ShareStoragexlogCopyerLatch = &t_thrd.proc->procLatch; + // 为共享存储XLog拷贝进程分配缓冲区 t_thrd.sharestoragexlogcopyer_cxt.originBuf = (char *)palloc(ShareStorageBufSize + g_instance.xlog_cxt.shareStorageopCtl.blkSize); - t_thrd.sharestoragexlogcopyer_cxt.buf = (char *)TYPEALIGN(g_instance.xlog_cxt.shareStorageopCtl.blkSize, - t_thrd.sharestoragexlogcopyer_cxt.originBuf); - UpdateShareStorageCtlInfo(); - ereport(LOG, (errmsg("start xlog copy at %X/%X", - (uint32)(g_instance.xlog_cxt.shareStorageXLogCtl->insertHead >> PTR_PRINT_SHIFT_SIZE), - (uint32)g_instance.xlog_cxt.shareStorageXLogCtl->insertHead))); + t_thrd.sharestoragexlogcopyer_cxt.buf = + (char *)TYPEALIGN(g_instance.xlog_cxt.shareStorageopCtl.blkSize, t_thrd.sharestoragexlogcopyer_cxt.originBuf); + // 更新共享存储XLog控制信息 + UpdateShareStorageCtlInfo(); + + // 记录日志，表示共享存储XLog拷贝进程已启动 + ereport(LOG, (errmsg("start xlog copy at %X/%X", + (uint32)(g_instance.xlog_cxt.shareStorageXLogCtl->insertHead >> PTR_PRINT_SHIFT_SIZE), + (uint32)g_instance.xlog_cxt.shareStorageXLogCtl->insertHead))); + + // 根据当前模式确定当前时间轴ID if (IS_SHARED_STORAGE_MAIN_STANDBY_MODE) { t_thrd.xlog_cxt.ThisTimeLineID = GetRecoveryTargetTLI(); } else { @@ -602,17 +813,27 @@ void SharedStorageXlogCopyBackendMain(void) } Assert(t_thrd.xlog_cxt.ThisTimeLineID != 0); + // 用于检查共享存储锁的计数器 uint64 checkLockCount = 0; + + // 上报当前状态为IDLE pgstat_report_activity(STATE_IDLE, NULL); + + // 进入循环 for (;;) { + // 重置进程的Latch ResetLatch(&t_thrd.proc->procLatch); + + // 上报当前状态为RUNNING pgstat_report_activity(STATE_RUNNING, NULL); + // 检查是否需要获取锁 ++checkLockCount; if (checkLockCount & CHECK_LOCK_INTERVAL) { GetLock(); } + // 如果收到了关闭信号，则执行关闭操作并退出进程 if (t_thrd.sharestoragexlogcopyer_cxt.shutdown_requested) { t_thrd.sharestoragexlogcopyer_cxt.shutdown_requested = false; g_instance.proc_base->ShareStoragexlogCopyerLatch = NULL; @@ -620,53 +841,81 @@ void SharedStorageXlogCopyBackendMain(void) proc_exit(0); /* done */ } + // 如果收到了SIGHUP信号，重新加载配置文件 if (t_thrd.sharestoragexlogcopyer_cxt.got_SIGHUP) { t_thrd.sharestoragexlogcopyer_cxt.got_SIGHUP = false; ProcessConfigFile(PGC_SIGHUP); } + // 检查并拷贝XLog，如果需要等待拷贝完成，则继续循环 bool waitCopy = CheckAndCopyXLog(false); if (waitCopy) { continue; } + // 上报当前状态为IDLE pgstat_report_activity(STATE_IDLE, NULL); + + // 休眠一段时间 const long sleepTime = 1000L; (void)WaitLatch(&t_thrd.proc->procLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, sleepTime); } + // 运行到这里说明循环已退出，清空Latch，并记录日志 g_instance.proc_base->ShareStoragexlogCopyerLatch = NULL; ereport(LOG, (errmsg("stopped xlog copy at %X/%X", - (uint32)(g_instance.xlog_cxt.shareStorageXLogCtl->insertHead >> PTR_PRINT_SHIFT_SIZE), - (uint32)g_instance.xlog_cxt.shareStorageXLogCtl->insertHead))); + (uint32)(g_instance.xlog_cxt.shareStorageXLogCtl->insertHead >> PTR_PRINT_SHIFT_SIZE), + (uint32)g_instance.xlog_cxt.shareStorageXLogCtl->insertHead))); proc_exit(0); } +// 唤醒共享存储XLog拷贝进程的函数 void WakeUpXLogCopyerBackend() { + // 如果共享存储XLog拷贝进程的Latch存在，则唤醒它 if (g_instance.proc_base->ShareStoragexlogCopyerLatch != NULL) { SetLatch(g_instance.proc_base->ShareStoragexlogCopyerLatch); } } + static void DoXLogCopyFromLocal(XLogRecPtr copyStart, XLogRecPtr copyEnd) { + // 32位右移的位数 uint32 shiftSize = 32; + + // 获取共享存储XLog控制信息 ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.shareStorageXLogCtl; + + // 获取XLog文件大小 uint64 xlogFileSize = static_cast(g_instance.attr.attr_storage.xlog_file_size); + // 检查共享存储的写锁 CheckShareStorageWriteLock(); + + // 初始化共享存储XLog控制信息，并设置系统标识符 InitShareStorageCtlInfo(ctlInfo, ctlInfo->systemIdentifier); + + // 根据拷贝的起始位置和结束位置确定insertHead的值 if (copyEnd - copyStart > xlogFileSize) { ctlInfo->insertHead = copyEnd - xlogFileSize; } else { ctlInfo->insertHead = copyStart; } + + // 将insertHead对齐到一个XLogSegSize的边界 ctlInfo->insertHead = ctlInfo->insertHead - ctlInfo->insertHead % XLogSegSize; + + // 计算共享存储XLog控制信息的校验和 ctlInfo->crc = CalShareStorageCtlInfoCrc(ctlInfo); + + // 更新共享存储XLog控制信息 UpdateShareStorageCtlInfo(ctlInfo); + + // 将XLog同步到共享存储 FsyncXlogToShareStorage(); + // 如果共享存储XLog拷贝进程的缓冲区为空，分配缓冲区 if (t_thrd.sharestoragexlogcopyer_cxt.originBuf == NULL) { t_thrd.sharestoragexlogcopyer_cxt.originBuf = (char *)palloc(ShareStorageBufSize + g_instance.xlog_cxt.shareStorageopCtl.blkSize); @@ -674,66 +923,90 @@ static void DoXLogCopyFromLocal(XLogRecPtr copyStart, XLogRecPtr copyEnd) t_thrd.sharestoragexlogcopyer_cxt.originBuf); } + // 记录日志，表示开始从本地覆盖拷贝XLog到共享存储 ereport(LOG, (errmsg("start to overwrite xlog from local to shared storage, start: %X/%X, end: %X/%X.", static_cast(ctlInfo->insertHead >> shiftSize), static_cast(ctlInfo->insertHead), static_cast(copyEnd >> shiftSize), static_cast(copyEnd)))); + // 执行XLog的拷贝 DoXlogCopy(copyEnd); + + // 关闭读取XLog文件的文件描述符 if (t_thrd.sharestoragexlogcopyer_cxt.readFile >= 0) { (void)close(t_thrd.sharestoragexlogcopyer_cxt.readFile); } + + // 释放共享存储XLog拷贝进程的缓冲区 if (t_thrd.sharestoragexlogcopyer_cxt.originBuf != NULL) { pfree(t_thrd.sharestoragexlogcopyer_cxt.originBuf); t_thrd.sharestoragexlogcopyer_cxt.originBuf = NULL; t_thrd.sharestoragexlogcopyer_cxt.buf = NULL; } + // 记录日志，表示成功将本地的XLog覆盖拷贝到共享存储，并显示当前共享存储的insertHead位置 ereport(LOG, (errmsg("successfully overwrite xlog from local to shared storage, shared storage's head is %X/%X", static_cast(ctlInfo->insertHead >> shiftSize), static_cast(ctlInfo->insertHead)))); } +/* +ctlInfo // 共享存储的控制信息 +localMaxLSN // 本地最大LSN +localMinLSN // 本地最小LSN +localMaxEnd // 本地最大LSN的结束位置 +localMaxLsnCrc // 本地最大LSN的校验和 +localMinLsnCrc // 本地最小LSN的校验和 +localMaxLsnLen // 本地最大LSN的长度 +sharedStorageRecCrc // 共享存储记录的校验和 +maxLsnMsg[XLOG_READER_MAX_MSGLENTH] // 最大LSN的错误信息 +minLsnMsg[XLOG_READER_MAX_MSGLENTH] // 最小LSN的错误信息 +tli // 时间线ID +crcValid // 校验和是否有效 +shiftSize // 移位大小 + */ bool XLogOverwriteFromLocal(bool force) { - ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.shareStorageXLogCtl; - XLogRecPtr localMaxLSN = InvalidXLogRecPtr; - XLogRecPtr localMinLSN = InvalidXLogRecPtr; - XLogRecPtr localMaxEnd = InvalidXLogRecPtr; - pg_crc32 localMaxLsnCrc = 0; - pg_crc32 localMinLsnCrc = 0; - uint32 localMaxLsnLen = 0; - pg_crc32 sharedStorageRecCrc = 0; - char maxLsnMsg[XLOG_READER_MAX_MSGLENTH] = {0}; - char minLsnMsg[XLOG_READER_MAX_MSGLENTH] = {0}; - TimeLineID tli = 0; - bool crcValid = false; - uint32 shiftSize = 32; + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.shareStorageXLogCtl; // 共享存储的控制信息 + XLogRecPtr localMaxLSN = InvalidXLogRecPtr; // 本地最大LSN + XLogRecPtr localMinLSN = InvalidXLogRecPtr; // 本地最小LSN + XLogRecPtr localMaxEnd = InvalidXLogRecPtr; // 本地最大LSN的结束位置 + pg_crc32 localMaxLsnCrc = 0; // 本地最大LSN的校验和 + pg_crc32 localMinLsnCrc = 0; // 本地最小LSN的校验和 + uint32 localMaxLsnLen = 0; // 本地最大LSN的长度 + pg_crc32 sharedStorageRecCrc = 0; // 共享存储记录的校验和 + char maxLsnMsg[XLOG_READER_MAX_MSGLENTH] = {0}; // 最大LSN的错误信息 + char minLsnMsg[XLOG_READER_MAX_MSGLENTH] = {0}; // 最小LSN的错误信息 + TimeLineID tli = 0; // 时间线ID + bool crcValid = false; // 校验和是否有效 + uint32 shiftSize = 32; // 移位大小 log_min_messages = LOG; - ReadShareStorageCtlInfo(ctlInfo); - if (ctlInfo->systemIdentifier != GetSystemIdentifier()) { + ReadShareStorageCtlInfo(ctlInfo); // 读取共享存储的控制信息 + + if (ctlInfo->systemIdentifier != GetSystemIdentifier()) { // 共享存储和本地的系统版本不一致 ereport(WARNING, (errmsg("database system version is different between shared storage and local"), - errdetail("The shared storage's system version is %lu, the local's system version is %lu.", - ctlInfo->systemIdentifier, GetSystemIdentifier()))); - if (!force) { + errdetail("The shared storage's system version is %lu, the local's system version is %lu.", + ctlInfo->systemIdentifier, GetSystemIdentifier()))); + if (!force) { // 不强制执行则返回失败 return false; } } localMaxLSN = FindMaxLSN(t_thrd.proc_cxt.DataDir, maxLsnMsg, XLOG_READER_MAX_MSGLENTH, &localMaxLsnCrc, - &localMaxLsnLen, &tli); - if (XLogRecPtrIsInvalid(localMaxLSN)) { + &localMaxLsnLen, &tli); // 查找本地最大LSN + if (XLogRecPtrIsInvalid(localMaxLSN)) { // 本地最大LSN无效 ereport(WARNING, (errmsg("find local max lsn fail: %s", maxLsnMsg))); return false; } else { ereport(LOG, (errmsg("find local max lsn success: %X/%X", static_cast(localMaxLSN >> shiftSize), static_cast(localMaxLSN)))); } - localMaxEnd = MAXALIGN(localMaxLSN + localMaxLsnLen); + localMaxEnd = MAXALIGN(localMaxLSN + localMaxLsnLen); // 计算本地最大LSN的结束位置 - localMinLSN = FindMinLSN(t_thrd.proc_cxt.DataDir, minLsnMsg, XLOG_READER_MAX_MSGLENTH, &localMinLsnCrc); - if (XLogRecPtrIsInvalid(localMinLSN)) { + localMinLSN = + FindMinLSN(t_thrd.proc_cxt.DataDir, minLsnMsg, XLOG_READER_MAX_MSGLENTH, &localMinLsnCrc); // 查找本地最小LSN + if (XLogRecPtrIsInvalid(localMinLSN)) { // 本地最小LSN无效 ereport(WARNING, (errmsg("find local min lsn fail: %s", minLsnMsg))); return false; } else { @@ -741,15 +1014,15 @@ bool XLogOverwriteFromLocal(bool force) static_cast(localMinLSN)))); } - t_thrd.xlog_cxt.ThisTimeLineID = tli; + t_thrd.xlog_cxt.ThisTimeLineID = tli; // 设置当前时间线ID - if (force == true && ctlInfo->systemIdentifier != GetSystemIdentifier()) { + if (force == true && ctlInfo->systemIdentifier != GetSystemIdentifier()) { // 强制执行并且系统版本不一致 ctlInfo->systemIdentifier = GetSystemIdentifier(); - DoXLogCopyFromLocal(localMinLSN, localMaxEnd); + DoXLogCopyFromLocal(localMinLSN, localMaxEnd); // 从本地复制XLog到共享存储 return true; } - /* ctl info lsn may not be accurate */ + // 共享存储的最新记录的起始位置、校验和和长度 XLogRecPtr shareStorageLatestRecordStart = InvalidXLogRecPtr; XLogRecPtr shareStorageLatestRecordEnd = InvalidXLogRecPtr; int shareStorageLatestRecordLen = 0; @@ -759,9 +1032,10 @@ bool XLogOverwriteFromLocal(bool force) Assert(shareStorageLatestRecordStart != InvalidXLogRecPtr); shareStorageLatestRecordEnd = MAXALIGN(shareStorageLatestRecordStart + shareStorageLatestRecordLen); + if (shareStorageLatestRecordStart != InvalidXLogRecPtr) { - // check local maxLSN - if (XLByteLE(localMaxEnd, shareStorageLatestRecordEnd)) { + // 检查本地最大LSN + if (XLByteLE(localMaxEnd, shareStorageLatestRecordEnd)) { // 本地最大LSN小于等于共享存储的最新记录的结束位置 sharedStorageRecCrc = GetXlogRecordCrc(localMaxLSN, crcValid, SharedStorageXLogPageRead, g_instance.xlog_cxt.shareStorageopCtl.blkSize); ereport( @@ -772,7 +1046,7 @@ bool XLogOverwriteFromLocal(bool force) static_cast(shareStorageLatestRecordEnd >> shiftSize), static_cast(shareStorageLatestRecordEnd), static_cast(localMaxEnd >> shiftSize), static_cast(localMaxEnd), localMaxLsnCrc, sharedStorageRecCrc))); - if (sharedStorageRecCrc == localMaxLsnCrc) { + if (sharedStorageRecCrc == localMaxLsnCrc) { // 校验和相同，则更新控制信息并返回成功 ctlInfo->insertHead = localMaxEnd; if (ctlInfo->insertHead > (ctlInfo->xlogFileSize + XLogSegSize)) { ctlInfo->insertTail = ctlInfo->insertHead - ctlInfo->xlogFileSize; @@ -784,7 +1058,7 @@ bool XLogOverwriteFromLocal(bool force) FsyncXlogToShareStorage(); return true; } - } else { + } else { // 本地最大LSN大于共享存储的最新记录的结束位置 pg_crc32 localCheckCrc = GetXlogRecordCrc(shareStorageLatestRecordStart, crcValid, XLogPageRead, 0); ereport( LOG, @@ -793,37 +1067,39 @@ bool XLogOverwriteFromLocal(bool force) static_cast(shareStorageLatestRecordEnd >> shiftSize), static_cast(shareStorageLatestRecordEnd), localCheckCrc, shareStorageLatestRecordCrc, static_cast(localMaxEnd >> shiftSize), static_cast(localMaxEnd)))); - if (shareStorageLatestRecordCrc == localCheckCrc) { + if (shareStorageLatestRecordCrc == localCheckCrc) { // 校验和相同，则从共享存储复制XLog到本地 DoXLogCopyFromLocal(shareStorageLatestRecordEnd, localMaxEnd); return true; } } } - // check local minLSN + + // 检查本地最小LSN sharedStorageRecCrc = GetXlogRecordCrc(localMinLSN, crcValid, SharedStorageXLogPageRead, g_instance.xlog_cxt.shareStorageopCtl.blkSize); - if (sharedStorageRecCrc != localMinLsnCrc) { + if (sharedStorageRecCrc != localMinLsnCrc) { // 校验和不相同 ereport(WARNING, (errmsg("the local minLsn(%X/%X) 's crc mismatched with shared storage, crc[local, share]:[%u,%u].", (uint32)(localMinLSN >> shiftSize), (uint32)localMinLSN, localMinLsnCrc, sharedStorageRecCrc))); } - DoXLogCopyFromLocal(localMinLSN, localMaxEnd); + DoXLogCopyFromLocal(localMinLSN, localMaxEnd); // 从本地复制XLog到共享存储 return true; } + static bool XLogCopyWrite(char *buf, int nbytes, XLogRecPtr recptr) { - uint64 startoff; - int byteswritten; + uint64 startoff; // 日志在文件中的起始偏移量 + int byteswritten; // 实际写入的字节数 t_thrd.xlog_cxt.ThisTimeLineID = copyFileTLI; while (nbytes > 0) { int segbytes; - + // 如果当前的copyFile小于0或者recptr所在的段不在copySegNo中 if (copyFile < 0 || !XLByteInSeg(recptr, copySegNo)) { bool use_existent = false; - + // 如果copyFile>=0，则关闭文件 if (copyFile >= 0) { if (close(copyFile) != 0) { ereport(WARNING, (errcode_for_file_access(), @@ -834,6 +1110,7 @@ static bool XLogCopyWrite(char *buf, int nbytes, XLogRecPtr recptr) copyFile = -1; /* Create/use new log file */ + /* 创建/使用新的日志文件 */ XLByteToSeg(recptr, copySegNo); use_existent = true; copyFile = XLogFileInit(copySegNo, &use_existent, true); @@ -841,14 +1118,17 @@ static bool XLogCopyWrite(char *buf, int nbytes, XLogRecPtr recptr) ereport(LOG, (errmsg("create or open log file %s.", XLogFileNameP(copyFileTLI, copySegNo)))); } /* Calculate the start offset of the logs */ + // 计算日志在段中的起始偏移量 startoff = recptr % XLogSegSize; + // 计算本次写入的字节数 if (startoff + nbytes > XLogSegSize) segbytes = XLogSegSize - startoff; else segbytes = nbytes; /* Need to seek in the file? */ + // 如果copyOff与起始偏移量不一致，则需要进行seek操作 if (copyOff != (uint32)startoff) { if (lseek(copyFile, (off_t)startoff, SEEK_SET) < 0) { ereport(WARNING, (errcode_for_file_access(), errmsg("could not seek in log file %s to offset %lu: %m", @@ -859,11 +1139,13 @@ static bool XLogCopyWrite(char *buf, int nbytes, XLogRecPtr recptr) } /* OK to write the logs */ + // 写入日志 errno = 0; byteswritten = write(copyFile, buf, segbytes); if (byteswritten <= 0) { /* if write didn't set errno, assume no disk space */ + /* 如果write没有设置errno，则假设是磁盘空间不足 */ if (errno == 0) { errno = ENOSPC; } @@ -872,6 +1154,7 @@ static bool XLogCopyWrite(char *buf, int nbytes, XLogRecPtr recptr) XLogFileNameP(copyFileTLI, copySegNo), copyOff, INT2ULONG(segbytes)))); return false; } + // 检查日志文件的正确性 if (copyOff == (uint32)0 && segbytes >= (int)sizeof(XLogPageHeaderData)) { if (((XLogPageHeader)buf)->xlp_magic == XLOG_PAGE_MAGIC && (copySegNo * XLogSegSize) != ((XLogPageHeader)buf)->xlp_pageaddr) { @@ -882,33 +1165,45 @@ static bool XLogCopyWrite(char *buf, int nbytes, XLogRecPtr recptr) } } /* Update state for write */ + // 更新写入状态 XLByteAdvance(recptr, byteswritten); copyOff += byteswritten; nbytes -= byteswritten; buf += byteswritten; - + // 执行fsync操作，确保数据持久化到磁盘 issue_xlog_fsync(copyFile, copySegNo); } return true; } +/* + +ctlInfo // 共享存储的控制信息 +bufOrigin// 缓冲区起始地址 +buf// 对齐后的缓冲区地址 +xlogreader// XLog读取器 +readPtr// 当前读取的日志位置 +read_len// 读取的字节数 +shiftSize// 移位大小 + */ static bool DoXLogCopyFromShare(XLogRecPtr copyStart) { - ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.shareStorageXLogCtl; - char *bufOrigin = (char *)palloc(ShareStorageBufSize + g_instance.xlog_cxt.shareStorageopCtl.blkSize); - char *buf = (char *)TYPEALIGN(g_instance.xlog_cxt.shareStorageopCtl.blkSize, bufOrigin); - XLogReaderState *xlogreader = NULL; - XLogRecPtr readPtr = copyStart; - int read_len = 0; - uint32 shiftSize = 32; + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.shareStorageXLogCtl; // 共享存储的控制信息 + char *bufOrigin = (char *)palloc(ShareStorageBufSize + g_instance.xlog_cxt.shareStorageopCtl.blkSize);// 缓冲区起始地址 + char *buf = (char *)TYPEALIGN(g_instance.xlog_cxt.shareStorageopCtl.blkSize, bufOrigin);// 对齐后的缓冲区地址 + XLogReaderState *xlogreader = NULL; // XLog读取器 + XLogRecPtr readPtr = copyStart;// 当前读取的日志位置 + int read_len = 0;// 读取的字节数 + uint32 shiftSize = 32;// 移位大小 ereport(LOG, (errmsg("start to overwrite xlog from shared storage to local, start: %X/%X, end: %X/%X.", static_cast(copyStart >> shiftSize), static_cast(copyStart), static_cast(ctlInfo->insertHead >> shiftSize), static_cast(ctlInfo->insertHead)))); + // 分配XLog读取器 xlogreader = XLogReaderAllocate(SharedStorageXLogPageRead, 0, g_instance.xlog_cxt.shareStorageopCtl.blkSize); if (xlogreader == NULL) { ereport(WARNING, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"), @@ -919,21 +1214,25 @@ static bool DoXLogCopyFromShare(XLogRecPtr copyStart) while (XLByteLT(readPtr, ctlInfo->insertHead)) { // read xlog from shared storage to buf + // 从共享存储读取日志到缓冲区 readPtr = readPtr - readPtr % XLOG_BLCKSZ; bool readResult = SharedStorageXlogReadCheck(xlogreader, ctlInfo->insertHead, readPtr, buf, &read_len); // write xlog from buf to local pg_xlog + // 将缓冲区中的日志写入到本地 bool writeResult = XLogCopyWrite(buf, read_len, readPtr); if (!writeResult) { return false; } XLByteAdvance(readPtr, read_len); + // 如果readResult为false，表示共享存储中已经没有更多的日志可读取 if (!readResult) { break; } } + // 关闭文件和释放内存 if (copyFile >= 0) { if (close(copyFile) != 0) { ereport(WARNING, (errcode_for_file_access(), @@ -950,57 +1249,72 @@ static bool DoXLogCopyFromShare(XLogRecPtr copyStart) return true; } +/* +ctlInfo// 获取共享存储的控制信息指针 +localMaxLSN // 本地最大LSN值，初始值为无效值 +localMaxEnd // 本地最大LSN的结束位置，初始值为无效值 +localMaxLsnCrc // 本地最大LSN的CRC校验值，初始值为0 +localMaxLsnLen// 本地最大LSN的长度，初始值为0 +sharedStorageRecCrc // 共享存储记录的CRC校验值，初始值为0 +maxLsnMsg[XLOG_READER_MAX_MSGLENTH] // 最大LSN的消息缓冲区 +crcValid // CRC校验是否有效 +shiftSize // 位移大小，用于打印日志时将LSN转换为可读形式 +log_min_messages // 设置日志级别为LOG + */ bool XLogOverwriteFromShare() { - ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.shareStorageXLogCtl; - XLogRecPtr localMaxLSN = InvalidXLogRecPtr; - XLogRecPtr localMaxEnd = InvalidXLogRecPtr; - pg_crc32 localMaxLsnCrc = 0; - uint32 localMaxLsnLen = 0; - pg_crc32 sharedStorageRecCrc = 0; - char maxLsnMsg[XLOG_READER_MAX_MSGLENTH] = {0}; - bool crcValid = false; - uint32 shiftSize = 32; - log_min_messages = LOG; + ShareStorageXLogCtl *ctlInfo = g_instance.xlog_cxt.shareStorageXLogCtl; // 获取共享存储的控制信息指针 + XLogRecPtr localMaxLSN = InvalidXLogRecPtr; // 本地最大LSN值，初始值为无效值 + XLogRecPtr localMaxEnd = InvalidXLogRecPtr; // 本地最大LSN的结束位置，初始值为无效值 + pg_crc32 localMaxLsnCrc = 0; // 本地最大LSN的CRC校验值，初始值为0 + uint32 localMaxLsnLen = 0; // 本地最大LSN的长度，初始值为0 + pg_crc32 sharedStorageRecCrc = 0; // 共享存储记录的CRC校验值，初始值为0 + char maxLsnMsg[XLOG_READER_MAX_MSGLENTH] = {0}; // 最大LSN的消息缓冲区 + bool crcValid = false; // CRC校验是否有效 + uint32 shiftSize = 32; // 位移大小，用于打印日志时将LSN转换为可读形式 + log_min_messages = LOG; // 设置日志级别为LOG - ReadShareStorageCtlInfo(ctlInfo); + ReadShareStorageCtlInfo(ctlInfo); // 读取共享存储的控制信息 if (ctlInfo->systemIdentifier != GetSystemIdentifier()) { ereport(WARNING, (errmsg("database system version is different between shared storage and local"), - errdetail("The shared storage's system version is %lu, the local's system version is %lu.", - ctlInfo->systemIdentifier, GetSystemIdentifier()))); - return false; + errdetail("The shared storage's system version is %lu, the local's system version is %lu.", + ctlInfo->systemIdentifier, GetSystemIdentifier()))); + return false; // 若共享存储与本地系统版本不匹配，则返回错误 } - /* The local log is consistent with the local data and cannot be simply overwritten from the shared storage. - If the verification fails, the build repair should be triggered */ + /* + 本地日志与本地数据一致，不能从共享存储简单地覆盖本地日志。 + 如果验证失败，则应触发构建修复 + */ localMaxLSN = FindMaxLSN(t_thrd.proc_cxt.DataDir, maxLsnMsg, XLOG_READER_MAX_MSGLENTH, &localMaxLsnCrc, - &localMaxLsnLen, ©FileTLI); + &localMaxLsnLen, ©FileTLI); // 查找本地最大LSN if (XLogRecPtrIsInvalid(localMaxLSN)) { ereport(WARNING, (errmsg("find local max lsn fail: %s", maxLsnMsg))); - return false; + return false; // 若查找本地最大LSN失败，则返回错误 } else { ereport(LOG, (errmsg("find local max lsn success: %X/%X", static_cast(localMaxLSN >> shiftSize), static_cast(localMaxLSN)))); } - localMaxEnd = MAXALIGN(localMaxLSN + localMaxLsnLen); + localMaxEnd = MAXALIGN(localMaxLSN + localMaxLsnLen); // 计算本地最大LSN的结束位置 if (XLByteLE(localMaxEnd, ctlInfo->insertHead)) { ereport(LOG, (errmsg("the shared storage's head(%X/%X) is greater than or equal to the local maxEnd(%X/%X).", static_cast(ctlInfo->insertHead >> shiftSize), static_cast(ctlInfo->insertHead), static_cast(localMaxEnd >> shiftSize), static_cast(localMaxEnd)))); - sharedStorageRecCrc = GetXlogRecordCrc(localMaxLSN, crcValid, SharedStorageXLogPageRead, - g_instance.xlog_cxt.shareStorageopCtl.blkSize); + sharedStorageRecCrc = + GetXlogRecordCrc(localMaxLSN, crcValid, SharedStorageXLogPageRead, + g_instance.xlog_cxt.shareStorageopCtl.blkSize); // 获取共享存储记录的CRC校验值 if (sharedStorageRecCrc != localMaxLsnCrc) { ereport(WARNING, (errmsg("crc[local,share]:[%u,%u] mismatch.", localMaxLsnCrc, sharedStorageRecCrc))); - return false; + return false; // 若本地最大LSN的CRC校验值与共享存储的CRC校验值不匹配，则返回错误 } - return DoXLogCopyFromShare(localMaxEnd); + return DoXLogCopyFromShare(localMaxEnd); // 从共享存储复制日志到本地 } else { ereport(WARNING, (errmsg("the shared storage's head(%X/%X) is less than the local maxEnd(%X/%X).", static_cast(ctlInfo->insertHead >> shiftSize), static_cast(ctlInfo->insertHead), static_cast(localMaxEnd >> shiftSize), static_cast(localMaxEnd)))); - return false; + return false; // 若共享存储的头部位置小于本地最大LSN的结束位置，则返回错误 } } @@ -1015,51 +1329,52 @@ Size CalShareStorageCtlSize() ShareStorageXLogCtl *AlignAllocShareStorageCtl() { - Assert(g_instance.attr.attr_storage.xlog_file_path != NULL); - Assert(g_instance.xlog_cxt.shareStorageopCtl.blkSize != 0); - uint32 blkSize = g_instance.xlog_cxt.shareStorageopCtl.blkSize; - void *tmpBuf = palloc(sizeof(ShareStorageXLogCtlAllocBlock) + blkSize); - ShareStorageXLogCtlAllocBlock *ctlInfo = (ShareStorageXLogCtlAllocBlock *)TYPEALIGN(blkSize, tmpBuf); - ctlInfo->originPointer = tmpBuf; - return &(ctlInfo->ctlInfo); + Assert(g_instance.attr.attr_storage.xlog_file_path != NULL); // 断言xlog文件路径不为空 + Assert(g_instance.xlog_cxt.shareStorageopCtl.blkSize != 0); // 断言共享存储的块大小不为0 + uint32 blkSize = g_instance.xlog_cxt.shareStorageopCtl.blkSize; // 获取块大小 + void *tmpBuf = palloc(sizeof(ShareStorageXLogCtlAllocBlock) + blkSize); // 分配内存空间 + ShareStorageXLogCtlAllocBlock *ctlInfo = + (ShareStorageXLogCtlAllocBlock *)TYPEALIGN(blkSize, tmpBuf); // 对齐内存空间，并返回共享存储控制信息的指针 + ctlInfo->originPointer = tmpBuf; // 存储原始指针 + return &(ctlInfo->ctlInfo); // 返回共享存储控制信息指针 } void AlignFreeShareStorageCtl(ShareStorageXLogCtl *ctlInfo) { - ShareStorageXLogCtlAllocBlock *ctlInfoBlock = reinterpret_cast(ctlInfo); - void *tmpBuf = ctlInfoBlock->originPointer; - pfree(tmpBuf); + ShareStorageXLogCtlAllocBlock *ctlInfoBlock = reinterpret_cast( + ctlInfo); // 强制类型转换，将共享存储控制信息指针转换为共享存储控制信息的内存块指针 + void *tmpBuf = ctlInfoBlock->originPointer; // 获取原始指针 + pfree(tmpBuf); // 释放内存空间 } bool LockNasWriteFile(int fd) { - struct flock lock; - lock.l_type = F_WRLCK; - lock.l_start = 0; - lock.l_whence = SEEK_SET; - lock.l_len = LOCK_LEN; - lock.l_pid = getpid(); + struct flock lock; // 文件锁结构体 + lock.l_type = F_WRLCK; // 读写锁类型为写锁 + lock.l_start = 0; // 锁的起始位置 + lock.l_whence = SEEK_SET; // 基准位置 + lock.l_len = LOCK_LEN; // 要锁定区域的长度 + lock.l_pid = getpid(); // 锁定进程的PID - if (fcntl(fd, F_SETLK, &lock) == 0) { + if (fcntl(fd, F_SETLK, &lock) == 0) { // 设置文件锁 return true; } - ereport(WARNING, (errcode_for_file_access(), errmsg("could not lock lock file : %m"))); + ereport(WARNING, (errcode_for_file_access(), errmsg("could not lock lock file : %m"))); // 日志记录 return false; } bool UnlockNasWriteFile(int fd) { - struct flock lock; - lock.l_type = F_UNLCK; - lock.l_start = 0; - lock.l_whence = SEEK_SET; - lock.l_len = LOCK_LEN; - lock.l_pid = getpid(); + struct flock lock; // 文件锁结构体 + lock.l_type = F_UNLCK; // 解除锁定 + lock.l_start = 0; // 锁的起始位置 + lock.l_whence = SEEK_SET; // 基准位置 + lock.l_len = LOCK_LEN; // 要锁定区域的长度 + lock.l_pid = getpid(); // 锁定进程的PID - if (fcntl(fd, F_SETLKW, &lock) == 0) { + if (fcntl(fd, F_SETLKW, &lock) == 0) { // 解除文件锁 return true; } - ereport(WARNING, (errcode_for_file_access(), errmsg("could not unlock lock file : %m"))); + ereport(WARNING, (errcode_for_file_access(), errmsg("could not unlock lock file : %m"))); // 日志记录 return false; } - -- 2.34.1 From 9124cc9411eaf9dfefe2f859ee30f63b6d0a7cf3 Mon Sep 17 00:00:00 2001 From: hyt15810656168 Date: Wed, 4 Oct 2023 18:00:53 +0800 Subject: [PATCH 47/48] Update custorage.cpp --- src/gausskernel/storage/cstore/custorage.cpp | 636 ++++++++++++------- 1 file changed, 409 insertions(+), 227 deletions(-) diff --git a/src/gausskernel/storage/cstore/custorage.cpp b/src/gausskernel/storage/cstore/custorage.cpp index d9964c5c3..660fec927 100755 --- a/src/gausskernel/storage/cstore/custorage.cpp +++ b/src/gausskernel/storage/cstore/custorage.cpp @@ -49,22 +49,28 @@ * The max size for single data file */ const uint64 MAX_FILE_SIZE = (uint64)RELSEG_SIZE * BLCKSZ; - +/** + * @Description: CUStorage类的构造函数，用于初始化CUStorage对象 + * @Param[IN] cFileNode: CFileNode对象，表示文件节点 + * @Param[IN] strategy: CStoreAllocateStrategy对象，表示存储分配策略 + */ CUStorage::CUStorage(const CFileNode& cFileNode, CStoreAllocateStrategy strategy) : m_cnode(cFileNode), m_fd(FILE_INVALID), m_strategy(strategy), append_only(false), is_2byte_align(false) { - InitFileNamePrefix(cFileNode); - m_fileName[0] = '\0'; // empty string - InitCstoreFreeSpace(m_strategy); + InitFileNamePrefix(cFileNode);// 初始化文件名前缀 + m_fileName[0] = '\0'; // empty string // 设置文件名为空字符串 + InitCstoreFreeSpace(m_strategy);// 初始化存储空间 } - +/** + * @Description: 销毁CUStorage对象，释放资源 + */ void CUStorage::Destroy() { if (m_fd != FILE_INVALID) - CloseFile(m_fd); + CloseFile(m_fd);// 关闭文件 if (m_freespace != NULL) { - DELETE_EX(m_freespace); + DELETE_EX(m_freespace);// 释放存储空间 } } @@ -76,12 +82,19 @@ CUStorage::~CUStorage() } /* create file, if file exists and isRedo is false then jump elog ERROR, the caller should table care to close fd */ +/** + * @Description: 创建文件，如果文件已存在且isRedo为false，则抛出错误；调用者需要负责关闭fd文件描述符 + * @Param[IN] file_name: 文件名 + * @Param[IN] fileId: 文件ID + * @Param[IN] isRedo: 是否处于重做模式，true表示是，false表示否 + * @Return: 成功创建的文件的文件描述符 + */ File CUStorage::CreateFile(char* file_name, int fileId, bool isRedo) const { File fd = -1; - int fileFlags = O_RDWR | O_CREAT | O_EXCL | PG_BINARY; + int fileFlags = O_RDWR | O_CREAT | O_EXCL | PG_BINARY;// 文件标志 - fd = open(file_name, fileFlags, 0600); + fd = open(file_name, fileFlags, 0600);// 创建文件 if (fd < 0) { int save_errno = errno; @@ -91,13 +104,15 @@ File CUStorage::CreateFile(char* file_name, int fileId, bool isRedo) const * script nominally creates it. Therefore, allow the file to exist * already, even if isRedo is not set. */ + // 在引导过程中，有些情况下系统关系可能在引导脚本正式创建之前就被内部后台进程访问到，因此即使isRedo未设置，允许该文件已存在 if (isRedo || IsBootstrapProcessingMode()) { - fileFlags = O_RDWR | PG_BINARY; + fileFlags = O_RDWR | PG_BINARY;// 文件标志 - fd = open(file_name, fileFlags, 0600); + fd = open(file_name, fileFlags, 0600);// 打开文件 } /* be sure to report the error reported by create, not open */ + // 确保报告的是创建文件时的错误，而不是打开文件的错误 errno = save_errno; if (fd < 0) { ereport(ERROR, (errcode_for_file_access(), errmsg("Could not open file \"%s\": %m", file_name))); @@ -108,58 +123,78 @@ File CUStorage::CreateFile(char* file_name, int fileId, bool isRedo) const } /* open file, if file not exists then create it */ +/** + * @Description: 打开文件，如果文件不存在则创建它 + * @Param[IN] file_name: 文件名 + * @Param[IN] fileId: 文件ID + * @Param[IN] direct_flag: 是否启用直接IO模式，true表示启用，false表示不启用 + * @Return: 打开文件的文件描述符 + */ File CUStorage::OpenFile(char* file_name, int fileId, bool direct_flag) { - File fd = WSOpenFile(file_name, fileId, direct_flag); + File fd = WSOpenFile(file_name, fileId, direct_flag);// 调用WSOpenFile打开文件 if (fd < 0) { - ereport(ERROR, (errcode_for_file_access(), errmsg("Could not open file \"%s\": %m", file_name))); + ereport(ERROR, (errcode_for_file_access(), errmsg("Could not open file \"%s\": %m", file_name)));// 抛出错误信息 } return fd; } - +/** + * @Description: 打开文件，如果文件不存在则创建它 + * @Param[IN] file_name: 文件名 + * @Param[IN] fileId: 文件ID + * @Param[IN] direct_flag: 是否启用直接IO模式，true表示启用，false表示不启用 + * @Return: 打开文件的文件描述符 + */ /* open file, if file not exists then create it */ File CUStorage::WSOpenFile(char* file_name, int fileId, bool direct_flag) { - uint32 fileFlags = O_RDWR | PG_BINARY; + uint32 fileFlags = O_RDWR | PG_BINARY;// 文件标志，默认为读写模式 struct stat st; /* file may create by other, so fileFlags without O_EXCL */ if (lstat((const char*)file_name, &st) == -1) { - fileFlags = O_RDWR | O_CREAT | PG_BINARY; + fileFlags = O_RDWR | O_CREAT | PG_BINARY;// 如果文件不存在，则添加创建标志 } ADIO_RUN() { if (direct_flag) { - fileFlags |= O_DIRECT; + fileFlags |= O_DIRECT;// 如果启用直接IO模式，则添加O_DIRECT标志 } } ADIO_END(); RelFileNodeForkNum filenode; - filenode.rnode.node = m_cnode.m_rnode; + filenode.rnode.node = m_cnode.m_rnode;// 设置文件所属的节点 filenode.rnode.backend = InvalidBackendId; - filenode.segno = fileId; - filenode.storage = COLUMN_STORE; + filenode.segno = fileId;// 设置文件的段号 + filenode.storage = COLUMN_STORE;// 设置文件的存储方式 if (m_cnode.m_attid <= 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_ATTRIBUTE), - errmsg("validate user defined attribute failed!"))); + errmsg("validate user defined attribute failed!")));// 验证用户定义的属性失败，抛出错误信息 } - filenode.forknumber = (ForkNumber)(m_cnode.m_attid + FirstColForkNum); + filenode.forknumber = (ForkNumber)(m_cnode.m_attid + FirstColForkNum);// 设置文件的分叉号 - return DataFileIdOpenFile(file_name, filenode, (int)fileFlags, 0600); + return DataFileIdOpenFile(file_name, filenode, (int)fileFlags, 0600);// 调用DataFileIdOpenFile函数打开文件，并返回文件描述符 } - +/** + * @Description: 关闭文件 + * @Param[IN] fd: 文件描述符 + */ void CUStorage::CloseFile(File fd) const { - FileClose(fd); + FileClose(fd);// 调用FileClose函数关闭文件 } - +/** + * @Description: 获取BCM文件名 + * @Param[OUT] bcmfile: 存储BCM文件名的字符数组 + * @Param[IN] fileId: 文件ID + */ void CUStorage::GetBcmFileName(_out_ char* bcmfile, _in_ int fileId) const { - Assert(fileId >= 0); + Assert(fileId >= 0);// 断言：文件ID必须大于等于0 // bcm file list: 16385_C1_bcm 16385_C1_bcm.1 16385_C1_bcm.2 .... int rc = 0; @@ -169,69 +204,80 @@ void CUStorage::GetBcmFileName(_out_ char* bcmfile, _in_ int fileId) const } else { rc = snprintf_s(bcmfile, MAXPGPATH, MAXPGPATH - 1, "%s_%s", m_fileNamePrefix, forkNames[BCM_FORKNUM]); } - securec_check_ss(rc, "", ""); + securec_check_ss(rc, "", "");// 检查字符串格式化函数的返回值是否出错 bcmfile[MAXPGPATH - 1] = '\0'; } - +/** + * @Description: 获取列文件名 + * @Return: 列文件名 + */ +//获取列文件名，即文件名前缀 const char* CUStorage::GetColumnFileName() const { // cu file name without segment no. :16385_C1 - return (const char*)m_fileNamePrefix; + return (const char*)m_fileNamePrefix;// 返回文件名前缀作为列文件名 } - +/** + * @Description: 获取文件名 + * @Param[OUT] fileName: 存储文件名的字符数组 + * @Param[IN] capacity: 字符数组的容量 + * @Param[IN] fileId: 文件ID + */ +//根据文件ID获取文件名，主要用于列文件的命名 void CUStorage::GetFileName(_out_ char* fileName, _in_ const size_t capacity, _in_ const int fileId) const { - Assert(fileId >= 0); + Assert(fileId >= 0);// 断言：文件ID必须大于等于0 // it means a cu file for one column of a reltaion. // it's different from bcm file name, its file list: // 16385_C1.0 16385_C1.1 16385_C1.2 ... int rc = snprintf_s(fileName, capacity, capacity - 1, "%s.%d", m_fileNamePrefix, fileId); - securec_check_ss(rc, "", ""); + securec_check_ss(rc, "", "");// 检查字符串格式化函数的返回值是否出错 fileName[capacity - 1] = '\0'; } - +//根据给定的策略初始化m_freespace变量 void CUStorage::InitCstoreFreeSpace(CStoreAllocateStrategy strategy) { - if (USING_FREE_SPACE == strategy) { - m_freespace = New(CurrentMemoryContext) CStoreFreeSpace(); + if (USING_FREE_SPACE == strategy) {// 如果策略为USING_FREE_SPACE + m_freespace = New(CurrentMemoryContext) CStoreFreeSpace();// 在当前内存上下文中创建一个CStoreFreeSpace对象，并将指针赋值给m_freespace } else { - m_freespace = NULL; + m_freespace = NULL;// 否则将m_freespace置为空指针 } } // Get common FileName prefix only once. -// +// 仅获取一次通用的文件名前缀 void CUStorage::InitFileNamePrefix(_in_ const CFileNode& cFileNode) { - int pathlen; - char attr_name[32]; - errno_t rc = EOK; + int pathlen;// 路径长度变量 + char attr_name[32];// 属性名数组，长度为32 + errno_t rc = EOK;// 错误码初始化为EOK - m_fileNamePrefix[0] = '\0'; // empty string + m_fileNamePrefix[0] = '\0'; // empty string // 置空字符串 - rc = sprintf_s(attr_name, sizeof(attr_name), "C%d", cFileNode.m_attid); - securec_check_ss(rc, "", ""); + rc = sprintf_s(attr_name, sizeof(attr_name), "C%d", cFileNode.m_attid);// 将"cFileNode.m_attid"格式化为字符串，存储在attr_name中 + securec_check_ss(rc, "", "");// 检查sprintf_s函数是否执行成功，如果失败则抛出异常 - Oid spcoid, dboid, reloid; - spcoid = cFileNode.m_rnode.spcNode; - dboid = cFileNode.m_rnode.dbNode; - reloid = cFileNode.m_rnode.relNode; + Oid spcoid, dboid, reloid;// 表空间OID、数据库OID、关系OID + spcoid = cFileNode.m_rnode.spcNode;// 设置表空间OID + dboid = cFileNode.m_rnode.dbNode;// 设置数据库OID + reloid = cFileNode.m_rnode.relNode;// 设置关系OID - if (spcoid == GLOBALTABLESPACE_OID) { - /* Shared system relations live in {datadir}/global */ - Assert(dboid == 0); - pathlen = strlen("global") + 1 + OIDCHARS + 1 + strlen(attr_name) + 1; - rc = snprintf_s(m_fileNamePrefix, sizeof(m_fileNamePrefix), pathlen, "global/%u_%s", reloid, attr_name); - securec_check_ss(rc, "", ""); - } else if (spcoid == DEFAULTTABLESPACE_OID) { - /* The default tablespace is {datadir}/base */ - pathlen = strlen("base") + 1 + OIDCHARS + 1 + OIDCHARS + 1 + strlen(attr_name) + 1; - rc = snprintf_s(m_fileNamePrefix, sizeof(m_fileNamePrefix), pathlen, "base/%u/%u_%s", dboid, reloid, attr_name); - securec_check_ss(rc, "", ""); + if (spcoid == GLOBALTABLESPACE_OID) {// 如果表空间OID为GLOBALTABLESPACE_OID + /* Shared system relations live in {datadir}/global */ /* 共享系统关系位于{datadir}/global */ + Assert(dboid == 0);// 断言数据库OID为0 + pathlen = strlen("global") + 1 + OIDCHARS + 1 + strlen(attr_name) + 1;// 计算路径长度 + rc = snprintf_s(m_fileNamePrefix, sizeof(m_fileNamePrefix), pathlen, "global/%u_%s", reloid, attr_name);// 格式化文件名，存储在m_fileNamePrefix中 + securec_check_ss(rc, "", "");// 检查snprintf_s函数是否执行成功，如果失败则抛出异常 + } else if (spcoid == DEFAULTTABLESPACE_OID) {// 如果表空间OID为DEFAULTTABLESPACE_OID + /* The default tablespace is {datadir}/base */ /* 默认表空间是{datadir}/base */ + pathlen = strlen("base") + 1 + OIDCHARS + 1 + OIDCHARS + 1 + strlen(attr_name) + 1;// 计算路径长度 + rc = snprintf_s(m_fileNamePrefix, sizeof(m_fileNamePrefix), pathlen, "base/%u/%u_%s", dboid, reloid, attr_name);// 格式化文件名，存储在m_fileNamePrefix中 + securec_check_ss(rc, "", "");// 检查snprintf_s函数是否执行成功，如果失败则抛出异常 } else { /* All other tablespaces are accessed via symlinks */ + /* 所有其他表空间通过符号链接访问 */ rc = snprintf_s(m_fileNamePrefix, sizeof(m_fileNamePrefix), sizeof(m_fileNamePrefix) - 1, @@ -241,11 +287,11 @@ void CUStorage::InitFileNamePrefix(_in_ const CFileNode& cFileNode) g_instance.attr.attr_common.PGXCNodeName, dboid, reloid, - attr_name); - securec_check_ss(rc, "", ""); + attr_name);// 格式化文件名，存储在m_fileNamePrefix中 + securec_check_ss(rc, "", "");// 检查snprintf_s函数是否执行成功，如果失败则抛出异常 } } - +//在CU报告的I/O错误情况下保存错误信息 static void SaveCUReportIOError( const char* fileName, uint64 writeOffset, int writtenBtyes, int expectedBytes, int expectedTotalBytes, int align_size) { @@ -253,6 +299,7 @@ static void SaveCUReportIOError( Assert(align_size > 0); if (writtenBtyes % align_size != 0) { /* later load will report error about not-matched align */ + /* 如果写入字节数符合对齐大小的要求，抛出异常并提示CU部分写入 */ ereport(ERROR, (errcode_for_file_access(), errmsg("CU partial write, file \"%s\", offset(%lu), total(%d), expected(%d), actual(%d): %m", @@ -263,6 +310,10 @@ static void SaveCUReportIOError( writtenBtyes), errhint("CU file will be not page aligned, Check free disk space"))); } else { + /* + * 如果写入字节数不符合对齐大小的要求，抛出异常并提示CU文件没有进行页对 + * 齐建议检查磁盘剩余空间。 + */ ereport(ERROR, (errcode_for_file_access(), errmsg("CU partial write, file \"%s\", offset(%lu), total(%d), expected(%d), actual(%d): %m", @@ -274,6 +325,7 @@ static void SaveCUReportIOError( errhint("Check free disk space."))); } } else { + /* 如果写入字节数小于等于0，抛出异常并提示无法扩展文件 */ ereport(ERROR, (errcode_for_file_access(), errmsg("could not extend file \"%s\", offset(%lu), total(%d), expected(%d), actual(%d): %m", @@ -285,12 +337,13 @@ static void SaveCUReportIOError( errhint("Check free disk space."))); } } - +//加载CU报告时的I/O错误处理 static void LoadCUReportIOError( const char* fileName, uint64 readOffset, int readBytes, int expectedBytes, int expectedTotalBytes) { if (errno == ENOENT) { /* file not exists */ + // 文件不存在 ereport(ERROR, (errcode_for_file_access(), errmsg("read file \"%s\" failed, %m", fileName), @@ -301,6 +354,7 @@ static void LoadCUReportIOError( readBytes))); } else if (readBytes >= 0) { if (readBytes == 0) { + // 文件中没有读取到数据 /* no data is read from */ ereport(ERROR, (errcode_for_file_access(), @@ -312,6 +366,7 @@ static void LoadCUReportIOError( expectedBytes, readBytes))); } else { + // 文件中只读取了部分数据 /* only some data is read from */ ereport(ERROR, (errcode_for_file_access(), @@ -327,6 +382,7 @@ static void LoadCUReportIOError( "maybe you should upgrade cstore data files first, or there were CU written IO error before"))); } } else { + // 无法读取文件 ereport(ERROR, (errcode_for_file_access(), errmsg("could not read file \"%s\", offset(%lu), load size(%d), expected read(%d), actual read(%d): %m", @@ -338,94 +394,102 @@ static void LoadCUReportIOError( } } - +//计算两个数的最小值 static inline int min(int a, int b) { - return (a > b ? b : a); + return (a > b ? b : a);// 返回a和b中的较小值 } - +//通过指定的偏移量和大小，将数据写入CU文件 void CUStorage::SaveCU(char* write_buf, _in_ uint64 offset, _in_ int size, bool direct_flag, bool for_extension) { - int writeFileId = offset / MAX_FILE_SIZE; - uint64 writeOffset = offset % MAX_FILE_SIZE; - int write_size = min(size, (int)(MAX_FILE_SIZE - writeOffset)); - int left_size = size - write_size; - Oid tableSpaceOid = m_cnode.m_rnode.spcNode; - char tmpFileName[MAXPGPATH] = {0}; - errno_t rc = 0; + int writeFileId = offset / MAX_FILE_SIZE;// 计算写入位置所在的文件编号 + uint64 writeOffset = offset % MAX_FILE_SIZE;// 计算写入位置在文件中的偏移量 + int write_size = min(size, (int)(MAX_FILE_SIZE - writeOffset));// 计算每次写入的大小，不能超过文件剩余空间 + int left_size = size - write_size;// 计算剩余要写入的数据大小 + Oid tableSpaceOid = m_cnode.m_rnode.spcNode;// 获取表空间Oid + char tmpFileName[MAXPGPATH] = {0};// 临时文件名 + errno_t rc = 0;// 错误码 - if (append_only) - TableSpaceUsageManager::IsExceedMaxsize(tableSpaceOid, size, false); + if (append_only)// 如果是追加写入模式 + TableSpaceUsageManager::IsExceedMaxsize(tableSpaceOid, size, false);// 检查表空间使用情况是否超过最大限制 while (write_size > 0) { - GetFileName(tmpFileName, MAXPGPATH, writeFileId); + GetFileName(tmpFileName, MAXPGPATH, writeFileId);// 获取当前写入文件的文件名 if (strcmp(tmpFileName, m_fileName) != 0) { if (m_fd != FILE_INVALID) { + /* + * Flush data if switch data file. We don't do so during + * file extension since we'll soon fsync the actual data. + */ /* * Flush data if switch data file. We don't do so during * file extension since we'll soon fsync the actual data. */ if (!for_extension) - FlushDataFile(); - FileClose(m_fd); + FlushDataFile();// 切换文件时刷新缓冲区中的数据到磁盘 + FileClose(m_fd);// 关闭之前的文件句柄 } - m_fd = OpenFile(tmpFileName, writeFileId, direct_flag); - Assert(m_fd != FILE_INVALID); - rc = strcpy_s(m_fileName, MAXPGPATH, tmpFileName); - securec_check_c(rc, "\0", "\0"); + m_fd = OpenFile(tmpFileName, writeFileId, direct_flag);// 打开新的文件句柄 + Assert(m_fd != FILE_INVALID);// 断言文件句柄有效性 + rc = strcpy_s(m_fileName, MAXPGPATH, tmpFileName);// 更新当前文件名 + securec_check_c(rc, "\0", "\0");// 检查字符串拷贝操作是否成功 } - Assert(m_fd != FILE_INVALID); + Assert(m_fd != FILE_INVALID);// 断言文件句柄有效性 - int writtenBytes = FilePWrite(m_fd, write_buf, write_size, writeOffset); + int writtenBytes = FilePWrite(m_fd, write_buf, write_size, writeOffset);// 在文件指定位置写入数据 if (writtenBytes != write_size) { int align_size = is_2byte_align ? ALIGNOF_TIMESERIES_CUSIZE : ALIGNOF_CUSIZE; SaveCUReportIOError(tmpFileName, writeOffset, writtenBytes, write_size, size, align_size); - } + }// 检查写入是否成功，如果不成功，抛出异常并报告错误信息 - ++writeFileId; - writeOffset = 0; - write_buf += write_size; - write_size = (((unsigned int)left_size > MAX_FILE_SIZE) ? MAX_FILE_SIZE : left_size); - left_size -= write_size; + ++writeFileId;// 更新文件编号 + writeOffset = 0;// 重置写入偏移量为0 + write_buf += write_size;// 更新写入缓冲区指针 + write_size = (((unsigned int)left_size > MAX_FILE_SIZE) ? MAX_FILE_SIZE : left_size);// 计算下一次写入的大小，不能超过文件大小 + left_size -= write_size;// 更新剩余待写入数据大小 } if (left_size != 0) { ereport(ERROR, (errcode_for_file_access(), errmsg("write file \"%s\" failed in savecu!", tmpFileName))); - } + }// 计算下一次写入的大小，不能超过文件大小 } - +//从远程读取CU数据后，覆盖写入本地CU文件 // After remote read CU, need overwrite local CU void CUStorage::OverwriteCU( _in_ char* write_buf, _in_ uint64 offset, _in_ int size, bool direct_flag, bool for_extension) { // check offset and size - int writeFileId = offset / MAX_FILE_SIZE; - uint64 writeOffset = offset % MAX_FILE_SIZE; - int write_size = min(size, (int)(MAX_FILE_SIZE - writeOffset)); - int left_size = size - write_size; - char tmpFileName[MAXPGPATH] = {0}; - errno_t rc = 0; - + // 检查偏移量和大小 + int writeFileId = offset / MAX_FILE_SIZE;// 计算写入位置所在的文件编号 + uint64 writeOffset = offset % MAX_FILE_SIZE;// 计算写入位置在文件中的偏移量 + int write_size = min(size, (int)(MAX_FILE_SIZE - writeOffset));// 计算每次写入的大小，不能超过文件剩余空间 + int left_size = size - write_size;// 计算剩余待写入数据大小 + char tmpFileName[MAXPGPATH] = {0};// 临时文件名 + errno_t rc = 0;// 错误码 + // 覆盖写入CU文件，不增加最大大小 // Overwirte CU, do not increase the max size while (write_size > 0) { - GetFileName(tmpFileName, MAXPGPATH, writeFileId); + GetFileName(tmpFileName, MAXPGPATH, writeFileId);// 获取当前写入文件的文件名 if (strcmp(tmpFileName, m_fileName) != 0) { if (m_fd != FILE_INVALID) { /* * Flush data if switch data file. We don't do so during * file extension since we'll soon fsync the actual data. */ + /* + * 切换数据文件时刷新数据，但在文件扩展期间不这样做，因为我们很快会将实际数据进行fsync。 + */ if (!for_extension) - FlushDataFile(); - FileClose(m_fd); + FlushDataFile();// 切换文件时刷新缓冲区中的数据到磁盘 + FileClose(m_fd);// 关闭之前的文件句柄 } - m_fd = OpenFile(tmpFileName, writeFileId, direct_flag); - Assert(m_fd != FILE_INVALID); - rc = strcpy_s(m_fileName, MAXPGPATH, tmpFileName); - securec_check(rc, "\0", "\0"); + m_fd = OpenFile(tmpFileName, writeFileId, direct_flag);// 打开新的文件句柄 + Assert(m_fd != FILE_INVALID);// 断言文件句柄有效性 + rc = strcpy_s(m_fileName, MAXPGPATH, tmpFileName);// 更新当前文件名 + securec_check(rc, "\0", "\0");// 检查字符串拷贝操作是否成功 } if (m_fd == FILE_INVALID) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", tmpFileName))); @@ -434,6 +498,7 @@ void CUStorage::OverwriteCU( int nbytes = 0; if ((nbytes = FilePWrite(m_fd, write_buf, write_size, writeOffset)) != write_size) { // just warning + // 只是警告 ereport(WARNING, (errcode_for_file_access(), errmsg("Overwrite CU failed. file \"%s\" , offset(%lu), size(%d), expect_write_size(%d), " @@ -445,11 +510,11 @@ void CUStorage::OverwriteCU( nbytes), handle_in_client(true))); } - ++writeFileId; - writeOffset = 0; - write_buf += write_size; - write_size = (((unsigned int)left_size > MAX_FILE_SIZE) ? MAX_FILE_SIZE : left_size); - left_size -= write_size; + ++writeFileId;// 更新文件编号 + writeOffset = 0;// 重置写入偏移量为0 + write_buf += write_size;// 更新写入缓冲区指针 + write_size = (((unsigned int)left_size > MAX_FILE_SIZE) ? MAX_FILE_SIZE : left_size);// 计算下一次写入的大小，不能超过文件大小 + left_size -= write_size;// 更新剩余待写入数据大小 } if (left_size != 0) { @@ -457,64 +522,73 @@ void CUStorage::OverwriteCU( errmsg("write file \"%s\" failed in OverwriteCU!", tmpFileName))); } } - +//从CU文件中读取数据并填充到输出缓冲区 void CUStorage::Load(_in_ uint64 offset, _in_ int size, __inout char* outbuf, bool direct_flag) { - int readFileId = CU_FILE_ID(offset); - uint64 readOffset = CU_FILE_OFFSET(offset); - int read_size = min(size, (int)(MAX_FILE_SIZE - readOffset)); - int left_size = size - read_size; + int readFileId = CU_FILE_ID(offset);// 获取读取位置所在的文件编号 + uint64 readOffset = CU_FILE_OFFSET(offset);// 获取读取位置在文件中的偏移量 + int read_size = min(size, (int)(MAX_FILE_SIZE - readOffset));// 计算每次读取的大小，不能超过文件剩余空间 + int left_size = size - read_size;// 计算剩余待读取数据大小 - char* read_buf = outbuf; - char tmpFileName[MAXPGPATH]; - errno_t rc = 0; + char* read_buf = outbuf;// 设置读缓冲区指针为输出缓冲区指针 + char tmpFileName[MAXPGPATH];// 临时文件名 + errno_t rc = 0;// 错误码 while (read_size > 0) { - GetFileName(tmpFileName, MAXPGPATH, readFileId); + GetFileName(tmpFileName, MAXPGPATH, readFileId);// 获取当前读取文件的文件名 if (strcmp(tmpFileName, m_fileName) != 0) { if (m_fd != FILE_INVALID) - FileClose(m_fd); - m_fd = OpenFile(tmpFileName, readFileId, direct_flag); + FileClose(m_fd);// 关闭之前的文件句柄 + m_fd = OpenFile(tmpFileName, readFileId, direct_flag);// 打开新的文件句柄 if (m_fd == FILE_INVALID) { ereport(ERROR, (errcode_for_file_access(), errmsg("Could not open file \"%s\"", tmpFileName))); } - rc = strcpy_s(m_fileName, MAXPGPATH, tmpFileName); - securec_check_c(rc, "\0", "\0"); + rc = strcpy_s(m_fileName, MAXPGPATH, tmpFileName);// 更新当前文件名 + securec_check_c(rc, "\0", "\0");// 检查字符串拷贝操作是否成功 } - int nbytes = FilePRead(m_fd, read_buf, read_size, readOffset); + int nbytes = FilePRead(m_fd, read_buf, read_size, readOffset);// 在当前文件句柄指定的位置进行数据读取 if (nbytes != read_size) { - LoadCUReportIOError(tmpFileName, readOffset, nbytes, read_size, size); + LoadCUReportIOError(tmpFileName, readOffset, nbytes, read_size, size);// 报告读取错误信息 } - ++readFileId; - readOffset = 0; - read_buf += read_size; - read_size = (((unsigned int)left_size > MAX_FILE_SIZE) ? MAX_FILE_SIZE : left_size); - left_size -= read_size; + ++readFileId;// 更新文件编号 + readOffset = 0;// 重置读取偏移量为0 + read_buf += read_size;// 更新读缓冲区指针 + read_size = (((unsigned int)left_size > MAX_FILE_SIZE) ? MAX_FILE_SIZE : left_size);// 计算下一次读取的大小，不能超过文件大小 + left_size -= read_size;// 更新剩余待读取数据大小 } if (left_size != 0) { ereport(ERROR, (errcode_for_file_access(), - errmsg("read file \"%s\" failed in load!", tmpFileName))); + errmsg("read file \"%s\" failed in load!", tmpFileName)));// 报告读取失败的错误信息 } } - +/** + * @brief 从CU文件中读取指定数据到输出缓冲区，支持对齐操作。 + * + * @param offset 读取位置在CU文件中的偏移量。 + * @param size 待读取数据大小。 + * @param outbuf 输出缓冲区。 + * @param direct_flag 是否使用O_DIRECT进行读取。 + * + * @return 返回已读取的数据大小。如果读取失败，则返回0。 + */ int CUStorage::WSLoad(_in_ uint64 offset, _in_ int size, __inout char* outbuf, bool direct_flag) { - int readFileId = CU_FILE_ID(offset); - uint64 readOffset = CU_FILE_OFFSET(offset); - int read_size = min(size, (int)(MAX_FILE_SIZE - readOffset)); - int left_size = size - read_size; - errno_t rc = 0; + int readFileId = CU_FILE_ID(offset);// 获取读取位置所在的文件编号 + uint64 readOffset = CU_FILE_OFFSET(offset);// 获取读取位置在文件中的偏移量 + int read_size = min(size, (int)(MAX_FILE_SIZE - readOffset));// 计算每次读取的大小，不能超过文件剩余空间 + int left_size = size - read_size;// 计算剩余待读取数据大小 + errno_t rc = 0;// 错误码 - char* read_buf = outbuf; - char tmpFileName[MAXPGPATH]; - bool isCUReadSizeValid = false; + char* read_buf = outbuf;// 设置读缓冲区指针为输出缓冲区指针 + char tmpFileName[MAXPGPATH];// 临时文件名 + bool isCUReadSizeValid = false;// 标记读取大小是否有效 - const int CUALIGNSIZE = is_2byte_align ? ALIGNOF_TIMESERIES_CUSIZE : ALIGNOF_CUSIZE; - isCUReadSizeValid = (read_size > 0 && 0 == read_size % CUALIGNSIZE); + const int CUALIGNSIZE = is_2byte_align ? ALIGNOF_TIMESERIES_CUSIZE : ALIGNOF_CUSIZE;// 计算CU对齐大小 + isCUReadSizeValid = (read_size > 0 && 0 == read_size % CUALIGNSIZE);// 判断读取大小是否有效 if (!isCUReadSizeValid) { ereport(ERROR, @@ -531,27 +605,27 @@ int CUStorage::WSLoad(_in_ uint64 offset, _in_ int size, __inout char* outbuf, b } while (read_size > 0) { - GetFileName(tmpFileName, MAXPGPATH, readFileId); + GetFileName(tmpFileName, MAXPGPATH, readFileId);// 获取当前读取文件的文件名 if (strcmp(tmpFileName, m_fileName) != 0) { if (m_fd != FILE_INVALID) - FileClose(m_fd); - m_fd = WSOpenFile(tmpFileName, readFileId, direct_flag); + FileClose(m_fd);// 关闭之前的文件句柄 + m_fd = WSOpenFile(tmpFileName, readFileId, direct_flag);// 打开新的文件句柄 if (FILE_INVALID == m_fd) return 0; - rc = strcpy_s(m_fileName, MAXPGPATH, tmpFileName); - securec_check(rc, "\0", "\0"); + rc = strcpy_s(m_fileName, MAXPGPATH, tmpFileName);// 更新当前文件名 + securec_check(rc, "\0", "\0");// 检查字符串拷贝操作是否成功 } int nbytes = 0; /* IO collector and IO scheduler for cstore insert */ if (ENABLE_WORKLOAD_CONTROL) - IOSchedulerAndUpdate(IO_TYPE_READ, 1, IO_TYPE_COLUMN); + IOSchedulerAndUpdate(IO_TYPE_READ, 1, IO_TYPE_COLUMN);// 进行IO调度 - if ((nbytes = FilePRead(m_fd, read_buf, read_size, readOffset)) != read_size) { - if (0 == nbytes) { + if ((nbytes = FilePRead(m_fd, read_buf, read_size, readOffset)) != read_size) {// 在当前文件句柄指定位置进行数据读取 + if (0 == nbytes) {// 对于读到0字节的情况进行异常处理 if (u_sess->attr.attr_storage.HaModuleDebug) ereport(NOTICE, (errcode_for_file_access(), @@ -560,7 +634,7 @@ int CUStorage::WSLoad(_in_ uint64 offset, _in_ int size, __inout char* outbuf, b return 0; } - if (nbytes % CUALIGNSIZE != 0) { + if (nbytes % CUALIGNSIZE != 0) {// 如果读取大小无法对齐，则抛出异常 ereport(ERROR, (errcode_for_file_access(), errmsg("read file \"%s\" failed, offset(%lu), size(%d), expect_read_size(%d), " @@ -570,7 +644,7 @@ int CUStorage::WSLoad(_in_ uint64 offset, _in_ int size, __inout char* outbuf, b size, read_size, nbytes))); - } else { + } else {// 如果读取出错，则抛出异常 ereport(ERROR, (errcode_for_file_access(), errmsg("could not read file \"%s\", offset(%lu), size(%d), expect_read_size(%d), " @@ -583,19 +657,19 @@ int CUStorage::WSLoad(_in_ uint64 offset, _in_ int size, __inout char* outbuf, b } } - ++readFileId; - readOffset = 0; - read_buf += read_size; - read_size = (((unsigned int)left_size > MAX_FILE_SIZE) ? MAX_FILE_SIZE : left_size); - left_size -= read_size; + ++readFileId;// 更新文件编号 + readOffset = 0;// 重置读取偏移量为0 + read_buf += read_size;// 更新读缓冲区指针 + read_size = (((unsigned int)left_size > MAX_FILE_SIZE) ? MAX_FILE_SIZE : left_size);// 计算下一次读取的大小，不能超过文件大小 + left_size -= read_size;// 更新剩余待读取数据大小 } - if (left_size != 0) { + if (left_size != 0) {// 如果还有剩余数据未读取，则抛出异常 ereport(ERROR, (errcode_for_file_access(), errmsg("read file \"%s\" failed in wsload!", tmpFileName))); } - return size; + return size;// 返回已读取的数据大小。如果读取失败，则返回0。 } /* @@ -604,14 +678,20 @@ int CUStorage::WSLoad(_in_ uint64 offset, _in_ int size, __inout char* outbuf, b * @Return: virtual fd * @See also: */ +/** + * @brief 从CU文件中获取存储的CU的文件句柄。 + * + * @param offset CU指针的偏移量。 + * @return 返回CU文件的文件句柄。 + */ File CUStorage::GetCUFileFd(uint64 offset) { char tmpFileName[MAXPGPATH]; File file; - int readFileId = CU_FILE_ID(offset); - GetFileName(tmpFileName, MAXPGPATH, readFileId); - file = OpenFile(tmpFileName, readFileId, true); + int readFileId = CU_FILE_ID(offset);// 获取读取位置所在的文件编号 + GetFileName(tmpFileName, MAXPGPATH, readFileId);// 获取当前文件的文件名 + file = OpenFile(tmpFileName, readFileId, true);// 打开CU文件并获取文件句柄 return file; } @@ -621,10 +701,16 @@ File CUStorage::GetCUFileFd(uint64 offset) * @Return: file offset * @See also: */ +/** + * @brief 从CU文件中获取存储的CU的偏移量。 + * + * @param offset CU指针的偏移量。 + * @return 返回CU在文件中的偏移量。 + */ uint64 CUStorage::GetCUOffsetInFile(uint64 offset) const { - uint64 readOffset = CU_FILE_OFFSET(offset); - return readOffset; + uint64 readOffset = CU_FILE_OFFSET(offset);// 获取CU在文件中的偏移量 + return readOffset;// 返回CU在文件中的偏移量 } /* @@ -634,15 +720,22 @@ uint64 CUStorage::GetCUOffsetInFile(uint64 offset) const * @Return: true -- in one file; false -- not in one file * @See also: */ +/** + * @brief 判断一个CU是否存储在同一个数据文件中。 + * + * @param offset CU指针的偏移量。 + * @param size CU的大小。 + * @return 如果一个CU存储在同一个数据文件中，则返回true，否则返回false。 + */ bool CUStorage::IsCUStoreInOneFile(uint64 offset, int size) const { - uint64 readOffset = CU_FILE_OFFSET(offset); - int left_size = MAX_FILE_SIZE - readOffset; + uint64 readOffset = CU_FILE_OFFSET(offset);// 获取CU在文件中的偏移量 + int left_size = MAX_FILE_SIZE - readOffset;// 计算当前文件剩余的空闲空间 if (size > left_size) { - return false; + return false;// 如果CU大小大于当前文件的剩余空间，则表示CU不会存储在同一个数据文件中，返回false } - return true; + return true;// 否则返回true } /* @@ -651,6 +744,13 @@ bool CUStorage::IsCUStoreInOneFile(uint64 offset, int size) const * @Param[IN] size: extend size * @See also: */ +/** + * @brief 为ADIO模式下的文件预分配空间，以保证文件物理地址的连续性。 + * + * @param extend_offset 扩展偏移量。 + * @param size 扩展大小。 + * @param keep_size 是否保留原文件大小。 + */ void CUStorage::FastExtendFile(uint64 extend_offset, uint32 size, bool keep_size) { File fd; @@ -659,19 +759,19 @@ void CUStorage::FastExtendFile(uint64 extend_offset, uint32 size, bool keep_size uint32 extend_size; while (size > 0) { - file_offset = CU_FILE_OFFSET(extend_offset); - left_size = MAX_FILE_SIZE - file_offset; - extend_size = (size < left_size) ? size : left_size; + file_offset = CU_FILE_OFFSET(extend_offset);// 获取CU在文件中的偏移量 + left_size = MAX_FILE_SIZE - file_offset;// 计算当前文件剩余的空闲空间 + extend_size = (size < left_size) ? size : left_size;// 计算需要预分配的空间大小 - fd = GetCUFileFd(extend_offset); - FileFastExtendFile(fd, file_offset, extend_size, keep_size); - CloseFile(fd); + fd = GetCUFileFd(extend_offset);// 获取预分配空间所在的文件句柄 + FileFastExtendFile(fd, file_offset, extend_size, keep_size);// 预分配空间 + CloseFile(fd);// 关闭文件句柄 - extend_offset += extend_size; - Assert(size >= extend_size); - size -= extend_size; + extend_offset += extend_size;// 更新扩展偏移量 + Assert(size >= extend_size);// 断言扩展大小是否大于空间分配大小 + size -= extend_size;// 减去已经分配的空间大小 } - Assert(size == 0); + Assert(size == 0);// 断言已经分配完了所有的空间 return; } @@ -681,22 +781,29 @@ void CUStorage::FastExtendFile(uint64 extend_offset, uint32 size, bool keep_size * @Return: cu offset * @See also: */ +/** + * @brief 根据给定的CU偏移量获取对齐后的CU偏移量，用于升级过程。 + * + * @param offset CU指针的偏移量。 + * @return 对齐后的CU偏移量。 + * @see None + */ uint64 CUStorage::GetAlignCUOffset(uint64 offset) const { const int CUALIGNSIZE = is_2byte_align ? ALIGNOF_TIMESERIES_CUSIZE : ALIGNOF_CUSIZE; - uint64 readOffset = CU_FILE_OFFSET(offset); - uint64 remainder = readOffset % CUALIGNSIZE; + uint64 readOffset = CU_FILE_OFFSET(offset);// 获取CU在文件中的偏移量 + uint64 remainder = readOffset % CUALIGNSIZE;// 计算偏移量除以对齐大小的余数 if (remainder != 0) { ereport(DEBUG1, (errmodule(MOD_ADIO), errmsg("GetAlignCUOffset: find un align offset(%lu)", offset))); if (readOffset > (uint64)CUALIGNSIZE) { - return offset - remainder; + return offset - remainder;// 偏移量大于对齐大小，则返回差值 } else if (readOffset < (uint64)CUALIGNSIZE) { - return offset - readOffset; + return offset - readOffset;// 偏移量小于对齐大小，则返回偏移量本身 } else { - Assert(0); + Assert(0);// 其他情况不应该出现，断言失败 } } - return offset; + return offset;// 无需对齐，直接返回原偏移量 } /* @@ -705,15 +812,22 @@ uint64 CUStorage::GetAlignCUOffset(uint64 offset) const * @Return: cu size * @See also: */ +/** + * @brief 根据给定的CU大小获取对齐后的CU大小，用于升级过程。 + * + * @param size CU的大小。 + * @return 对齐后的CU大小。 + * @see None + */ int CUStorage::GetAlignCUSize(int size) const { const int CUALIGNSIZE = is_2byte_align ? ALIGNOF_TIMESERIES_CUSIZE : ALIGNOF_CUSIZE; - int remainder = size % CUALIGNSIZE; + int remainder = size % CUALIGNSIZE;// 计算大小除以对齐大小的余数 if (remainder != 0) { ereport(DEBUG1, (errmodule(MOD_ADIO), errmsg("GetAlignCUOffset: find un align size(%d)", size))); - return size + CUALIGNSIZE - remainder; + return size + CUALIGNSIZE - remainder;// 返回对齐后的大小 } - return size; + return size;// 无需对齐，直接返回原大小 } /* @@ -725,6 +839,16 @@ int CUStorage::GetAlignCUSize(int size) const * @Param[IN] size: CU data size * @See also: */ +/** + * @brief 从CU文件加载CU数据。 + * + * @param cuPtr 要加载数据的CU对象。 + * @param offset CU数据在逻辑文件中的逻辑偏移量。 + * @param size CU数据的大小。 + * @param direct_flag 是否启用ADIO特性，如果启用，则使用DIO。 + * @param inCUCache 是否在CU缓存中。 + * @see None + */ void CUStorage::LoadCU(_in_ CU* cuPtr, _in_ uint64 offset, _in_ int size, bool direct_flag, bool inCUCache) { if (size < 0 || (uint64)size > MAX_FILE_SIZE) { @@ -742,19 +866,21 @@ void CUStorage::LoadCU(_in_ CU* cuPtr, _in_ uint64 offset, _in_ int size, bool d uint64 load_offset; int load_size; - load_offset = GetAlignCUOffset(offset); - cuPtr->m_head_padding_size = offset - load_offset; - load_size = GetAlignCUSize(cuPtr->m_head_padding_size + size); + load_offset = GetAlignCUOffset(offset);// 获取对齐后的偏移量 + cuPtr->m_head_padding_size = offset - load_offset;// 计算头部填充大小 + load_size = GetAlignCUSize(cuPtr->m_head_padding_size + size);// 获取对齐后的大小 // !!! FUTURE CASE.. We can optimize memory allocation. // // In order to avoid over-boundary read in the function readData, more 8 byte memory is allocated. + // 为了避免在readData函数中越界读取，额外分配8字节内存 cuPtr->m_compressedLoadBuf = (char*)CStoreMemAlloc::Palloc(load_size + 8, !inCUCache); - Load(load_offset, load_size, cuPtr->m_compressedLoadBuf, direct_flag); + Load(load_offset, load_size, cuPtr->m_compressedLoadBuf, direct_flag);// 加载数据 // the complete CU data has been loaded, so set the cu size. // we will check this value during decompressing cu data. - // + // 完整的CU数据已加载，因此设置CU大小。 + // 在解压缩CU数据时，我们将检查该值。 cuPtr->m_compressedBuf = cuPtr->m_compressedLoadBuf + cuPtr->m_head_padding_size; cuPtr->SetCUSize(size); cuPtr->m_compressedBufSize = size; @@ -770,20 +896,34 @@ void CUStorage::LoadCU(_in_ CU* cuPtr, _in_ uint64 offset, _in_ int size, bool d * @Param[IN] size: CU data size * @See also: */ +/* + * @Description: 从远程节点加载CU数据 + * @Param[IN] cuPtr: 要加载数据的CU对象 + * @Param[IN] direct_flag: 如果启用ADIO特性, 则使用DIO + * @Param[IN] inCUCache: 指示cuPtr是否在CU缓存中 + * @Param[IN] offset: CU数据在逻辑文件中的逻辑偏移量 + * @Param[IN] size: CU数据的大小 + * @See also: + */ +//从远程节点加载CU数据 void CUStorage::RemoteLoadCU(_in_ CU* cuPtr, _in_ uint64 offset, _in_ int size, bool direct_flag, bool inCUCache) { /* the caller should alloc the m_compressedLoadBuf */ + /* 调用方应该分配m_compressedLoadBuf */ Assert(cuPtr->m_compressedLoadBuf != NULL); /* get offset and size */ + /* 获取偏移量和大小 */ uint64 load_offset = GetAlignCUOffset(offset); cuPtr->m_head_padding_size = offset - load_offset; int load_size = GetAlignCUSize(cuPtr->m_head_padding_size + size); /* get current xlog insert lsn */ + /* 获取当前XLog插入LSN */ XLogRecPtr cur_lsn = GetInsertRecPtr(); /* get remote address */ + /* 获取远程地址 */ char remote_address1[MAXPGPATH] = { 0 }; /* remote_address1[0] = '\0'; */ char remote_address2[MAXPGPATH] = { 0 }; /* remote_address2[0] = '\0'; */ GetRemoteReadAddress(remote_address1, remote_address2, MAXPGPATH); @@ -832,17 +972,19 @@ retry: handle_in_client(true))); /* Check interrupts */ + /* 检查中断 */ CHECK_FOR_INTERRUPTS(); remote_address = remote_address2; ++retry_times; - goto retry; /* jump out retry_times >= 1 */ + goto retry; /* jump out retry_times >= 1 */ /* 跳出 retry_times >= 1 */ } } // the complete CU data has been loaded, so set the cu size. // we will check this value during decompressing cu data. - // + // CU数据已经完全加载，设置CU的大小 + // 在解压缩CU数据时会检查这个值 cuPtr->m_compressedBuf = cuPtr->m_compressedLoadBuf + cuPtr->m_head_padding_size; cuPtr->SetCUSize(size); cuPtr->m_compressedBufSize = size; @@ -851,7 +993,8 @@ retry: // Flush data to disk // FileSync respect sync_method variable -// +// 函数作用：将数据刷新到磁盘 +// FileSync函数根据sync_method变量来执行同步操作 void CUStorage::FlushDataFile() const { if (m_fd != FILE_INVALID) { @@ -860,20 +1003,25 @@ void CUStorage::FlushDataFile() const } } } - +// 函数作用：分配空间给CU数据 +// 参数size：要分配的数据大小 +// 返回值：分配得到的偏移量 uint64 CUStorage::AllocSpace(_in_ int size) { uint64 offset = InvalidCStoreOffset; int align_size = is_2byte_align ? ALIGNOF_TIMESERIES_CUSIZE : ALIGNOF_CUSIZE; switch (m_strategy) { case USING_FREE_SPACE: + // 尝试从已释放的空间中获取合适的空间 offset = CStoreAllocator::TryAcquireSpaceFromFSM(m_freespace, size, align_size); if (offset != InvalidCStoreOffset) { + // 如果找到了合适的空间，则设置append_only标志为false append_only = false; break; } /* if no free space fits, use the APPEND_ONLY way */ case APPEND_ONLY: + // 如果采用追加模式则直接向后分配空间 offset = CStoreAllocator::AcquireSpace(m_cnode, size, align_size); append_only = true; break; @@ -883,7 +1031,10 @@ uint64 CUStorage::AllocSpace(_in_ int size) } return offset; } - +// 函数作用：创建CU存储文件 +// 参数fileId：文件id +// 参数isRedo：是否为重做操作 +// 根据文件id和isRedo参数生成文件路径，并使用CreateFile函数创建文件，然后关闭文件描述 void CUStorage::CreateStorage(int fileId, bool isRedo) const { File fd; @@ -893,7 +1044,10 @@ void CUStorage::CreateStorage(int fileId, bool isRedo) const fd = CreateFile(filePath, fileId, isRedo); close(fd); } - +// 函数作用：判断给定的CU存储文件是否存在 +// 参数fileId：文件id +// 返回值：判断结果，存在返回true，不存在返回false +// 首先根据文件id生成文件路径，然后使用lstat函数判断文件是否存在 bool CUStorage::IsDataFileExist(int fileId) const { char tmpFileName[MAXPGPATH]; @@ -905,14 +1059,14 @@ bool CUStorage::IsDataFileExist(int fileId) const return true; } - +//判断给定的BCM文件是否存在 bool CUStorage::IsBcmFileExist(_in_ int fileId) const { char tmpFileName[MAXPGPATH]; GetBcmFileName(tmpFileName, fileId); struct stat st; - if (lstat((const char*)tmpFileName, &st) == -1) + if (lstat((const char*)tmpFileName, &st) == -1)// 使用lstat函数判断BCM文件是否存在 return false; return true; @@ -921,21 +1075,22 @@ bool CUStorage::IsBcmFileExist(_in_ int fileId) const /* * @Description: truncate column data files which relation CREATE and TRUNCATE in same XACT block */ +//截断数据文件 void CUStorage::TruncateDataFile() { int fileId = 0; char tmpFileName[MAXPGPATH]; while (1) { - if (!IsDataFileExist(fileId)) + if (!IsDataFileExist(fileId))// 调用IsDataFileExist函数判断数据文件是否存在 break; GetFileName(tmpFileName, MAXPGPATH, fileId); File vfd = OpenFile(tmpFileName, fileId, false); - if (FileTruncate(vfd, 0)) { - ereport(WARNING, (errmsg("could not ftruncate file \"%s\": %m", tmpFileName))); + if (FileTruncate(vfd, 0)) {// 执行文件截断操作 + ereport(WARNING, (errmsg("could not ftruncate file \"%s\": %m", tmpFileName)));// 输出警告信息 } - CloseFile(vfd); + CloseFile(vfd);// 关闭文件描述符 ++fileId; } @@ -944,51 +1099,56 @@ void CUStorage::TruncateDataFile() /* * @Description: truncate column bcm files which relation CREATE and TRUNCATE in same XACT block */ +//在同一个事务块中遍历所有BCM文件并逐个截断（将其大小截断为0） void CUStorage::TruncateBcmFile() { int fileId = 0; char tmpFileName[MAXPGPATH]; - + // 遍历所有的BCM文件，逐个截断 while (1) { - if (!IsBcmFileExist(fileId)) + if (!IsBcmFileExist(fileId))// 判断该BCM文件是否存在 break; - GetBcmFileName(tmpFileName, fileId); - File vfd = OpenFile(tmpFileName, fileId, false); - if (FileTruncate(vfd, 0)) { + GetBcmFileName(tmpFileName, fileId);// 获取该BCM文件的路径 + File vfd = OpenFile(tmpFileName, fileId, false);// 打开该BCM文件 + if (FileTruncate(vfd, 0)) {// 截断该BCM文 ereport(WARNING, (errmsg("could not ftruncate file \"%s\": %m", tmpFileName))); } - CloseFile(vfd); + CloseFile(vfd);// 关闭该BCM文件 - ++fileId; + ++fileId;// 处理下一个BCM文件 } } - +// 设置2字节对齐模式 void CUStorage::Set2ByteAlign(bool is_2byte_align) { - this->is_2byte_align = is_2byte_align; + this->is_2byte_align = is_2byte_align;// 更新内部状态 } - +// 判断当前是否是2字节对齐模式 bool CUStorage::Is2ByteAlign() { - return is_2byte_align; + return is_2byte_align;// 返回内部状态值 } CUFile::CUFile(const RelFileNode& fileNode, int col) { + // 根据传入的fileNode、col和MAIN_FORKNUM创建CFileNode对象 CFileNode cFileNode(fileNode, col, MAIN_FORKNUM); + // 使用CUStorage类创建m_colStorage对象，并传入cFileNode作为参数 m_colStorage = New(CurrentMemoryContext) CUStorage(cFileNode); - + // 根据编译选项决定使用的内存分配方式 ADIO_RUN() { + // 使用adio_align_alloc分配内存给m_buffer m_buffer = (char*)adio_align_alloc(CUFILE_MAX_BUF_SIZE); } ADIO_ELSE() { + // 使用palloc分配内存给m_buffer m_buffer = (char*)palloc(CUFILE_MAX_BUF_SIZE); } ADIO_END(); - + // 初始化其他成员变量 m_maxpos = 0; m_maxFileSize = 0; m_curFileOffset = 0; @@ -997,45 +1157,57 @@ CUFile::CUFile(const RelFileNode& fileNode, int col) CUFile::~CUFile() { + // 将m_colStorage设置为NULL m_colStorage = NULL; - m_buffer = NULL; + m_buffer = NULL;// 将m_buffer设置为NULL } - +// 销毁CUFile对象 void CUFile::Destroy() { + // 删除m_colStorage对象 DELETE_EX(m_colStorage); ADIO_RUN() { + // 使用adio_align_free释放m_buffer所分配的内存 adio_align_free(m_buffer); } ADIO_ELSE() { + // 使用pfree释放m_buffer所分配的内存 pfree(m_buffer); + // 将m_buffer设置为NULL m_buffer = NULL; } ADIO_END(); } - +//从CUFile对象所对应的文件中读取指定偏移量的数据 char* CUFile::Read(uint64 offset, int wantSize, int* realSize, int align_size) { + // 确保align_size大于0 Assert(align_size > 0); + // 确保wantSize是align_size的整数倍 Assert(wantSize % align_size == 0); + // 确保CUFILE_MAX_BUF_SIZE是align_size的整数倍 Assert(CUFILE_MAX_BUF_SIZE % align_size == 0); - + // 判断是否未初始化 const bool notInited = (-1 == m_fileId); + // 判断是否需要切换文件 bool switchId = (m_fileId != (int)(CU_FILE_ID(offset))); - + // 计算缓冲区上界和下界 uint64 bufUpBound = MAX_FILE_SIZE * m_fileId + m_curFileOffset; uint64 bufDownOffset = bufUpBound - m_maxpos; + // 判断是否在缓冲区内 bool notInBuf = (offset < bufDownOffset || offset >= bufUpBound); if (notInited || switchId || notInBuf) { + // 更新文件ID和文件偏移量 m_fileId = CU_FILE_ID(offset); m_curFileOffset = CU_FILE_OFFSET(offset); // we don't call FileClose() because Load() will do the real work // if (notInited || switchId || (m_curFileOffset >= m_maxFileSize)) { + // 获取临时文件名并获取文件大小 char tmpFileName[MAXPGPATH]; m_colStorage->GetFileName(tmpFileName, MAXPGPATH, m_fileId); @@ -1043,14 +1215,16 @@ char* CUFile::Read(uint64 offset, int wantSize, int* realSize, int align_size) if (lstat((const char*)tmpFileName, &st) == 0) { m_maxFileSize = st.st_size; } else { + // 报告错误，无法读取文件 ereport(ERROR, (errcode_for_file_access(), errmsg("could not read file \"%s\": %m", tmpFileName))); } } // load data into buffer - // + // 加载数据到缓冲区 m_maxpos = Min((m_maxFileSize - m_curFileOffset), CUFILE_MAX_BUF_SIZE); if (m_maxpos == 0) { + // 文件已读取完毕，设置realSize为0，返回NULL *realSize = 0; return NULL; } else { @@ -1058,41 +1232,49 @@ char* CUFile::Read(uint64 offset, int wantSize, int* realSize, int align_size) m_curFileOffset += (uint32)m_maxpos; // recompute down and up bound of this buffer after load data - // + // 重新计算加载数据后的缓冲区下界和上界 bufUpBound = MAX_FILE_SIZE * m_fileId + m_curFileOffset; bufDownOffset = bufUpBound - m_maxpos; } } // read data from this buffer and its start position is readPos - // + // 从缓冲区中读取数据，其起始位置是readPos int readPos = offset - bufDownOffset; char* outdata = m_buffer + readPos; *realSize = Min(wantSize, (m_maxpos - readPos)); return outdata; } - +// 函数作用：获取每列的数据文件大小，通过循环遍历每个列的段文件 /* Get the data file size of each column by forloop each segment file of column */ uint64 GetColDataFileSize(Relation rel, int attid) { + // 创建临时CFileNode对象 CFileNode tmpNode(rel->rd_node, attid, MAIN_FORKNUM); + // 创建CUStorage对象 CUStorage custore(tmpNode); + // 初始化路径名和文件大小 char pathname[MAXPGPATH] = "\0"; uint64 size = 0; errno = 0; + // 段文件ID必须连续 /* segment file id must be continues */ for (int i = 0;; i++) { struct stat fst; + // 获取段文件名 custore.GetFileName(pathname, MAXPGPATH, i); if (stat(pathname, &fst) < 0) { /* pathname file is not exist */ + // 段文件不存在 if (errno == ENOENT) break; else + // 报告错误，无法获取文件信息 ereport(ERROR, (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", pathname))); } + // 累加文件大小 size += fst.st_size; } custore.Destroy(); -- 2.34.1 From f6ddcc93b39dfd9e86df3c96c08c294a380c842e Mon Sep 17 00:00:00 2001 From: Auweafan Date: Wed, 4 Oct 2023 18:47:53 +0800 Subject: [PATCH 48/48] Update inv_api.cpp --- .../storage/large_object/inv_api.cpp | 539 +++++++++--------- 1 file changed, 282 insertions(+), 257 deletions(-) diff --git a/src/gausskernel/storage/large_object/inv_api.cpp b/src/gausskernel/storage/large_object/inv_api.cpp index 1f69c5dc9..7f7d1c830 100644 --- a/src/gausskernel/storage/large_object/inv_api.cpp +++ b/src/gausskernel/storage/large_object/inv_api.cpp @@ -62,121 +62,135 @@ static void open_lo_relation(void) if (t_thrd.storage_cxt.lo_heap_r && t_thrd.storage_cxt.lo_index_r) return; /* already open in current xact */ - /* Arrange for the top xact to own these relation references */ + /* 获取当前的资源拥有者 */ currentOwner = t_thrd.utils_cxt.CurrentResourceOwner; PG_TRY(); { + /* 将顶层事务的资源拥有者设置为当前资源拥有者 */ t_thrd.utils_cxt.CurrentResourceOwner = t_thrd.utils_cxt.TopTransactionResourceOwner; - /* Use RowExclusiveLock since we might either read or write */ + /* 使用RowExclusiveLock，因为我们可能需要读取或写入 */ if (t_thrd.storage_cxt.lo_heap_r == NULL) + /* 打开LargeObjectRelationId对应的heap关系，并使用RowExclusiveLock */ t_thrd.storage_cxt.lo_heap_r = heap_open(LargeObjectRelationId, RowExclusiveLock); if (t_thrd.storage_cxt.lo_index_r == NULL) + /* 打开LargeObjectLOidPNIndexId对应的index关系，并使用RowExclusiveLock */ t_thrd.storage_cxt.lo_index_r = index_open(LargeObjectLOidPNIndexId, RowExclusiveLock); } PG_CATCH(); { - /* Ensure CurrentResourceOwner is restored on error */ + /* 在错误发生时恢复CurrentResourceOwner的值 */ t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; PG_RE_THROW(); } PG_END_TRY(); + /* 恢复CurrentResourceOwner的值 */ t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; } + /* * Clean up at main transaction end */ void close_lo_relation(bool isCommit) { - if (t_thrd.storage_cxt.lo_heap_r || t_thrd.storage_cxt.lo_index_r) { + if (t_thrd.storage_cxt.lo_heap_r || t_thrd.storage_cxt.lo_index_r) { // 如果lo_heap_r或lo_index_r不为空 /* - * Only bother to close if committing; else abort cleanup will handle - * it + * 只有在提交事务时才关闭，否则由于中止清理处理 */ - if (isCommit) { - ResourceOwner currentOwner; + if (isCommit) { // 如果是提交事务 + ResourceOwner currentOwner; // 当前资源拥有者 - currentOwner = t_thrd.utils_cxt.CurrentResourceOwner; - PG_TRY(); + currentOwner = t_thrd.utils_cxt.CurrentResourceOwner; // 保存当前资源拥有者 + PG_TRY(); // 尝试执行以下代码块 { - t_thrd.utils_cxt.CurrentResourceOwner = t_thrd.utils_cxt.TopTransactionResourceOwner; + t_thrd.utils_cxt.CurrentResourceOwner = + t_thrd.utils_cxt.TopTransactionResourceOwner; // 设置当前资源拥有者为顶级事务资源拥有者 - if (t_thrd.storage_cxt.lo_index_r) - index_close(t_thrd.storage_cxt.lo_index_r, NoLock); - if (t_thrd.storage_cxt.lo_heap_r) - heap_close(t_thrd.storage_cxt.lo_heap_r, NoLock); + if (t_thrd.storage_cxt.lo_index_r) // 如果lo_index_r不为空 + index_close(t_thrd.storage_cxt.lo_index_r, NoLock); // 关闭索引关系 + if (t_thrd.storage_cxt.lo_heap_r) // 如果lo_heap_r不为空 + heap_close(t_thrd.storage_cxt.lo_heap_r, NoLock); // 关闭堆关系 } - PG_CATCH(); + PG_CATCH(); // 捕获异常 { - /* Ensure CurrentResourceOwner is restored on error */ - t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; - PG_RE_THROW(); + /* 确保错误时恢复CurrentResourceOwner */ + t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; // 恢复当前资源拥有者 + PG_RE_THROW(); // 重新抛出异常 } PG_END_TRY(); - t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; + + t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; // 恢复当前资源拥有者 } - t_thrd.storage_cxt.lo_heap_r = NULL; - t_thrd.storage_cxt.lo_index_r = NULL; + + t_thrd.storage_cxt.lo_heap_r = NULL; // 将lo_heap_r置为NULL + t_thrd.storage_cxt.lo_index_r = NULL; // 将lo_index_r置为NULL } } + /* * Same as pg_largeobject.c's LargeObjectExists(), except snapshot to * read with can be specified. */ static bool myLargeObjectExists(Oid loid, Snapshot snapshot) { - Relation pg_lo_meta; - ScanKeyData skey[1]; - SysScanDesc sd; - HeapTuple tuple; - bool retval = false; + Relation pg_lo_meta; // LargeObjectMetadataRelationId 对应的关系 + ScanKeyData skey[1]; // 扫描键 + SysScanDesc sd; // 系统扫描描述符 + HeapTuple tuple; // 堆元组 + bool retval = false; // 返回值，默认为false + // 初始化扫描键，设置ObjectIdAttributeNumber（对象ID属性编号）等于loid ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(loid)); + // 打开LargeObjectMetadataRelationId对应的heap关系，并使用AccessShareLock进行锁定 pg_lo_meta = heap_open(LargeObjectMetadataRelationId, AccessShareLock); + // 开始对pg_lo_meta关系进行系统扫描，使用LargeObjectMetadataOidIndexId对应的索引， + // 忽略无效元组，指定快照为snapshot，扫描键数量为1，扫描键为skey sd = systable_beginscan(pg_lo_meta, LargeObjectMetadataOidIndexId, true, snapshot, 1, skey); + // 获取下一个堆元组 tuple = systable_getnext(sd); if (HeapTupleIsValid(tuple)) - retval = true; + retval = true; // 如果堆元组有效，则将retval设为true + // 结束系统扫描 systable_endscan(sd); + // 关闭pg_lo_meta关系，释放AccessShareLock锁定的资源 heap_close(pg_lo_meta, AccessShareLock); - return retval; + return retval; // 返回retval值 } + /* * Extract data field from a pg_largeobject tuple, detoasting if needed * and verifying that the length is sane. Returns data pointer (a bytea *), * data length, and an indication of whether to pfree the data pointer. */ -static void getdatafield(Form_pg_largeobject tuple, bytea** pdatafield, int* plen, bool* pfreeit) +static void getdatafield(Form_pg_largeobject tuple, bytea **pdatafield, int *plen, bool *pfreeit) { - bytea* datafield = NULL; - int len; - bool freeit = false; + bytea *datafield = NULL; // 数据域指针 + int len; // 数据域长度 + bool freeit = false; // 是否需要释放数据域 - datafield = &(tuple->data); /* see note at top of file */ - if (VARATT_IS_EXTENDED(datafield)) { - datafield = (bytea*)heap_tuple_untoast_attr((struct varlena*)datafield); - freeit = true; + datafield = &(tuple->data); /* 见文件顶部的注释 */ + if (VARATT_IS_EXTENDED(datafield)) { // 如果数据域使用扩展存储 + datafield = + (bytea *)heap_tuple_untoast_attr((struct varlena *)datafield); // 将扩展存储的数据域解压成栈上数据域 + freeit = true; // 设置需要释放数据域 } - len = VARSIZE(datafield) - VARHDRSZ; - if (len < 0 || len > LOBLKSIZE) - ereport(ERROR, - (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("pg_largeobject entry for OID %u, page %d has invalid data field size %d", - tuple->loid, - tuple->pageno, - len))); - *pdatafield = datafield; - *plen = len; - *pfreeit = freeit; + len = VARSIZE(datafield) - VARHDRSZ; // 获取数据域长度 + if (len < 0 || len > LOBLKSIZE) // 如果数据域长度不正确 + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("pg_largeobject entry for OID %u, page %d has invalid data field size %d", tuple->loid, + tuple->pageno, len))); + *pdatafield = datafield; // 返回数据域指针 + *plen = len; // 返回数据域长度 + *pfreeit = freeit; // 返回是否需要释放数据域 } /* @@ -197,6 +211,7 @@ Oid inv_create(Oid lobjId) /* * Create a new largeobject with empty data pages + 创建一个新的largeobject型对象，并设置为空数据页。 */ lobjId_new = LargeObjectCreate(lobjId); @@ -230,51 +245,62 @@ Oid inv_create(Oid lobjId) * and subsidiary data are allocated in the specified memory context, * which must be suitably long-lived for the caller's purposes. */ -LargeObjectDesc* inv_open(Oid lobjId, int flags, MemoryContext mcxt) +LargeObjectDesc *inv_open(Oid lobjId, int flags, MemoryContext mcxt) { - LargeObjectDesc* retval = NULL; + LargeObjectDesc *retval = NULL; // 返回的LargeObjectDesc型对象描述符指针 - retval = (LargeObjectDesc*)MemoryContextAlloc(mcxt, sizeof(LargeObjectDesc)); + retval = + (LargeObjectDesc *)MemoryContextAlloc(mcxt, sizeof(LargeObjectDesc)); // 分配内存空间用于存储LargeObjectDesc型对象描述符 - retval->id = lobjId; - retval->subid = GetCurrentSubTransactionId(); - retval->offset = 0; + retval->id = lobjId; // 设置LargeObjectDesc型对象的OID + retval->subid = GetCurrentSubTransactionId(); // 设置当前子事务ID + retval->offset = 0; // 设置偏移量 - if (flags & INV_WRITE) { - retval->snapshot = SnapshotNow; - retval->flags = IFS_WRLOCK | IFS_RDLOCK; - } else if (flags & INV_READ) { + if (flags & INV_WRITE) { // 如果标志包含INV_WRITE，表示以写模式打开LargeObjectDesc型对象 + retval->snapshot = SnapshotNow; // 使用当前快照 + retval->flags = IFS_WRLOCK | IFS_RDLOCK; // 设置写锁和读锁标志 + } else if (flags & INV_READ) { // 如果标志包含INV_READ，表示以读模式打开LargeObjectDesc型对象 /* - * We must register the snapshot in TopTransaction's resowner, because - * it must stay alive until the LO is closed rather than until the - * current portal shuts down. + * 必须在TopTransaction的资源拥有者中注册快照， + * 因为它必须在LO关闭之前保持活动状态，而不是在当前portal关闭时。 */ - retval->snapshot = RegisterSnapshotOnOwner(GetActiveSnapshot(), t_thrd.utils_cxt.TopTransactionResourceOwner); - retval->flags = IFS_RDLOCK; - } else - ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("invalid flags: %d", flags))); + retval->snapshot = RegisterSnapshotOnOwner( + GetActiveSnapshot(), t_thrd.utils_cxt.TopTransactionResourceOwner); // 在资源拥有者中注册快照 + retval->flags = IFS_RDLOCK; // 设置读锁标志 + } else { + ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("invalid flags: %d", flags))); // 抛出错误，标志无效 + } - /* Can't use LargeObjectExists here because it always uses SnapshotNow */ - if (!myLargeObjectExists(lobjId, retval->snapshot)) - ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("large object %u does not exist", lobjId))); + // 不能使用LargeObjectExists，因为它总是使用SnapshotNow + if (!myLargeObjectExists(lobjId, retval->snapshot)) { // 如果大型对象不存在 + ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("large object %u does not exist", lobjId))); // 抛出错误，大型对象不存在 + } - return retval; + return retval; // 返回LargeObjectDesc型对象描述符指针 } /* * Closes a large object descriptor previously made by inv_open(), and * releases the long-term memory used by it. */ -void inv_close(LargeObjectDesc* obj_desc) +void inv_close(LargeObjectDesc *obj_desc) { Assert(PointerIsValid(obj_desc)); + /* + * 检查obj_desc是否有效 + */ if (obj_desc->snapshot != SnapshotNow) UnregisterSnapshotFromOwner(obj_desc->snapshot, t_thrd.utils_cxt.TopTransactionResourceOwner); + /* + * 释放obj_desc的内存空间 + */ pfree(obj_desc); } + /* * Destroys an existing large object (not to be confused with a descriptor!) * @@ -287,18 +313,19 @@ int inv_drop(Oid lobjId) /* * Delete any comments and dependencies on the large object */ - object.classId = LargeObjectRelationId; - object.objectId = lobjId; - object.objectSubId = 0; - performDeletion(&object, DROP_CASCADE, 0); + object.classId = LargeObjectRelationId; // 设置对象类别为大型对象 + object.objectId = lobjId; // 设置对象ID为目标大型对象的OID + object.objectSubId = 0; // 设置对象子ID为0 + performDeletion(&object, DROP_CASCADE, 0); // 执行删除操作，级联删除依赖项 /* * Advance command counter so that tuple removal will be seen by later * large-object operations in this transaction. + * 提升命令计数器以便后续事务中的元组移除可以被看到 */ - CommandCounterIncrement(); + CommandCounterIncrement();// 提升命令计数器 - return 1; + return 1;// 返回1表示删除成功 } /* @@ -314,14 +341,24 @@ static uint32 inv_getsize(LargeObjectDesc* obj_desc) SysScanDesc sd; HeapTuple tuple; + /* + * 检查obj_desc是否有效 + */ Assert(PointerIsValid(obj_desc)); + /* 打开pg_largeobject关系 */ open_lo_relation(); + /* + * 初始化扫描键信息 + */ ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id)); - sd = systable_beginscan_ordered( - t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 1, skey); + /* + * 有序扫描pg_largeobject索引，查找对应的元组。 + */ + sd = systable_beginscan_ordered(t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 1, + skey); /* * Because the pg_largeobject index is on both loid and pageno, but we @@ -336,136 +373,152 @@ static uint32 inv_getsize(LargeObjectDesc* obj_desc) int len; bool pfreeit = false; + /* 如果存在null值，则错误 */ if (HeapTupleHasNulls(tuple)) /* paranoia */ ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("null field found in pg_largeobject"))); + /* 获取元组中的数据 */ data = (Form_pg_largeobject)GETSTRUCT(tuple); getdatafield(data, &datafield, &len, &pfreeit); + + /* 计算最后一个字节的位置 */ lastbyte = data->pageno * LOBLKSIZE + len; + + /* 释放datafield指向的内存空间 */ if (pfreeit) pfree(datafield); } + /* + * 终止扫描 + */ systable_endscan_ordered(sd); return lastbyte; } -int inv_seek(LargeObjectDesc* obj_desc, int offset, int whence) +int inv_seek(LargeObjectDesc *obj_desc, int offset, int whence) { - Assert(PointerIsValid(obj_desc)); + Assert(PointerIsValid(obj_desc)); // 断言大型对象描述符指针有效 switch (whence) { - case SEEK_SET: - if (offset < 0) + case SEEK_SET: // 从文件开头开始计算offset + if (offset < 0) // 如果offset为负数，则抛出错误 ereport(ERROR, (errcode_for_file_access(), errmsg("invalid seek offset: %d", offset))); - obj_desc->offset = offset; + obj_desc->offset = offset; // 设置偏移量为offset break; - case SEEK_CUR: - if (offset < 0 && obj_desc->offset < ((uint32)(-offset))) + case SEEK_CUR: // 相对于当前位置计算offset + if (offset < 0 && + obj_desc->offset < ((uint32)(-offset))) // 如果offset为负数且偏移量小于(-offset)，则抛出错误 ereport(ERROR, (errcode_for_file_access(), errmsg("invalid seek offset: %d", offset))); - obj_desc->offset += offset; + obj_desc->offset += offset; // 偏移量增加offset break; - case SEEK_END: { - uint32 size = inv_getsize(obj_desc); - if (offset < 0 && size < ((uint32)(-offset))) + case SEEK_END: { // 相对于文件末尾计算offset + uint32 size = inv_getsize(obj_desc); // 获取大型对象的大小 + if (offset < 0 && size < ((uint32)(-offset))) // 如果offset为负数且对象大小小于(-offset)，则抛出错误 ereport(ERROR, (errcode_for_file_access(), errmsg("invalid seek offset: %d", offset))); - obj_desc->offset = size + offset; + obj_desc->offset = size + offset; // 偏移量设置为对象大小加offset } break; default: - ereport(ERROR, (errcode_for_file_access(), errmsg("invalid whence: %d", whence))); + ereport(ERROR, (errcode_for_file_access(), + errmsg("invalid whence: %d", whence))); // 如果whence参数无效，则抛出错误 } + return obj_desc->offset; // 返回当前偏移量 +} + +int inv_tell(LargeObjectDesc *obj_desc) +{ + // 检查obj_desc是否有效 + Assert(PointerIsValid(obj_desc)); + + // 返回obj_desc中的offset字段值 return obj_desc->offset; } -int inv_tell(LargeObjectDesc* obj_desc) -{ - Assert(PointerIsValid(obj_desc)); - return obj_desc->offset; -} - -int inv_read(LargeObjectDesc* obj_desc, char* buf, int nbytes) +int inv_read(LargeObjectDesc *obj_desc, char *buf, int nbytes) { - Assert(PointerIsValid(obj_desc)); - int nread = 0; + Assert(PointerIsValid(obj_desc)); // 断言大型对象描述符指针有效 + + int nread = 0; // 已读取的字节数 int n; int off; int len; - int32 pageno = (int32)(obj_desc->offset / LOBLKSIZE); + int32 pageno = (int32)(obj_desc->offset / LOBLKSIZE); // 根据偏移量计算页号 uint32 pageoff; ScanKeyData skey[2]; SysScanDesc sd; HeapTuple tuple; errno_t rc = EOK; - Assert(buf != NULL); + Assert(buf != NULL); // 断言缓冲区指针有效 - if (nbytes <= 0) { + if (nbytes <= 0) { // 如果要读取的字节数小于等于0，则直接返回0 return 0; } - open_lo_relation(); + open_lo_relation(); // 打开大型对象关系表 + // 初始化扫描键值 ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id)); - ScanKeyInit(&skey[1], Anum_pg_largeobject_pageno, BTGreaterEqualStrategyNumber, F_INT4GE, Int32GetDatum(pageno)); - sd = systable_beginscan_ordered( - t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2, skey); + // 开始有序扫描大型对象索引表 + sd = systable_beginscan_ordered(t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2, + skey); while ((tuple = systable_getnext_ordered(sd, ForwardScanDirection)) != NULL) { Form_pg_largeobject data; - bytea* datafield = NULL; + bytea *datafield = NULL; bool pfreeit = false; - if (HeapTupleHasNulls(tuple)) /* paranoia */ + if (HeapTupleHasNulls(tuple)) // 如果元组有空字段，抛出错误 ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("null field found in pg_largeobject"))); - data = (Form_pg_largeobject)GETSTRUCT(tuple); + data = (Form_pg_largeobject)GETSTRUCT(tuple); // 获取大型对象元组的数据 /* * We expect the indexscan will deliver pages in order. However, * there may be missing pages if the LO contains unwritten "holes". We * want missing sections to read out as zeroes. */ - pageoff = ((uint32)data->pageno) * LOBLKSIZE; - if (pageoff > obj_desc->offset) { + pageoff = ((uint32)data->pageno) * LOBLKSIZE; // 计算页的偏移量 + if (pageoff > obj_desc->offset) { // 如果页的偏移量大于当前偏移量，则说明存在缺失的部分，将其读出为零 n = pageoff - obj_desc->offset; n = (n <= (nbytes - nread)) ? n : (nbytes - nread); - rc = memset_s(buf + nread, n, '\0', n); + rc = memset_s(buf + nread, n, '\0', n); // 将buf中的一部分置为零 securec_check(rc, "", ""); nread += n; obj_desc->offset += n; } if (nread < nbytes) { - off = (int)(obj_desc->offset - pageoff); - if (off < 0 || off >= LOBLKSIZE) { - ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("invalid offset num:%d", off))); + off = (int)(obj_desc->offset - pageoff); // 计算在当前页内的偏移量 + if (off < 0 || off >= LOBLKSIZE) { // 如果偏移量不合法，抛出错误 + ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("invalid offset num:%d", off))); } - getdatafield(data, &datafield, &len, &pfreeit); - if (len > off) { + getdatafield(data, &datafield, &len, &pfreeit); // 获取大型对象数据字段 + if (len > off) { // 如果数据字段内有足够的数据可供读取 n = len - off; n = (n <= (nbytes - nread)) ? n : (nbytes - nread); - rc = memcpy_s(buf + nread, n, VARDATA(datafield) + off, n); + rc = memcpy_s(buf + nread, n, VARDATA(datafield) + off, n); // 将数据复制到buf中 securec_check(rc, "", ""); nread += n; obj_desc->offset += n; } if (pfreeit) - pfree(datafield); + pfree(datafield); // 释放大型对象数据字段的内存 } if (nread >= nbytes) break; } - systable_endscan_ordered(sd); + systable_endscan_ordered(sd); // 结束扫描 - return nread; + return nread; // 返回已读取的字节数 } + void check_obj_desc(const LargeObjectDesc* obj_desc) { /* enforce writability because snapshot is probably wrong otherwise */ @@ -480,56 +533,57 @@ void check_obj_desc(const LargeObjectDesc* obj_desc) ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("large object %u was already dropped", obj_desc->id))); } -int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes) +int inv_write(LargeObjectDesc *obj_desc, const char *buf, int nbytes) { if (nbytes <= 0) { return 0; } - int nwritten = 0; - int n; - int off; - int len; - int32 pageno; - ScanKeyData skey[2]; - SysScanDesc sd; - HeapTuple oldtuple = NULL; - Form_pg_largeobject olddata = NULL; - bool neednextpage = true; - bytea* datafield = NULL; - bool pfreeit = false; + int nwritten = 0; // 已写入的字节数 + int n; // 当前写入的字节数 + int off; // 偏移量，表示当前写入的位置在页面中的偏移量 + int len; // 旧页中的有效数据长度 + int32 pageno; // 页面号 + ScanKeyData skey[2]; // 扫描键 + SysScanDesc sd; // 系统扫描描述符 + HeapTuple oldtuple = NULL; // 旧元组 + Form_pg_largeobject olddata = NULL; // 旧的pg_largeobject元组 + bool neednextpage = true; // 是否需要获取下一个已存在的页面 + bytea *datafield = NULL; // 数据字段 + bool pfreeit = false; // 是否需要释放datafield struct { bytea hdr; - char data[LOBLKSIZE]; /* make struct big enough */ - int32 align_it; /* ensure struct is aligned well enough */ - } workbuf; - char* workb = VARDATA(&workbuf.hdr); - HeapTuple newtup; - Datum values[Natts_pg_largeobject]; - bool nulls[Natts_pg_largeobject]; - bool replace[Natts_pg_largeobject]; - CatalogIndexState indstate; + char data[LOBLKSIZE]; /* make struct big enough */ + int32 align_it; /* ensure struct is aligned well enough */ + } workbuf; // 工作缓冲区 + char *workb = VARDATA(&workbuf.hdr); // 缓冲区数据部分的指针 + HeapTuple newtup; // 新元组 + Datum values[Natts_pg_largeobject]; // 插入的值 + bool nulls[Natts_pg_largeobject]; // 是否为NULL + bool replace[Natts_pg_largeobject]; // 是否替换 + CatalogIndexState indstate; // 目录索引状态 errno_t rc; if (unlikely(!PointerIsValid(obj_desc))) { return 0; } - pageno = (int32)(obj_desc->offset / LOBLKSIZE); + pageno = (int32)(obj_desc->offset / LOBLKSIZE); // 计算页面号 - rc = memset_s(&workbuf, sizeof(workbuf), 0, sizeof(workbuf)); + rc = memset_s(&workbuf, sizeof(workbuf), 0, sizeof(workbuf)); // 清零工作缓冲区 securec_check(rc, "\0", "\0"); - Assert(buf != NULL); + Assert(buf != NULL); // 断言buf不为空 - check_obj_desc(obj_desc); - open_lo_relation(); + check_obj_desc(obj_desc); // 检查对象描述符 + open_lo_relation(); // 打开large object关系 - indstate = CatalogOpenIndexes(t_thrd.storage_cxt.lo_heap_r); + indstate = CatalogOpenIndexes(t_thrd.storage_cxt.lo_heap_r); // 打开目录索引 + // 初始化扫描键 ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id)); - ScanKeyInit(&skey[1], Anum_pg_largeobject_pageno, BTGreaterEqualStrategyNumber, F_INT4GE, Int32GetDatum(pageno)); - sd = systable_beginscan_ordered( - t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2, skey); + // 开始有序扫描 + sd = systable_beginscan_ordered(t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2, + skey); while (nwritten < nbytes) { /* @@ -556,8 +610,8 @@ int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes) * * First, load old data into workbuf */ - getdatafield(olddata, &datafield, &len, &pfreeit); - rc = memcpy_s(workb, len, VARDATA(datafield), len); + getdatafield(olddata, &datafield, &len, &pfreeit); // 获取旧数据 + rc = memcpy_s(workb, len, VARDATA(datafield), len); // 拷贝旧数据到工作缓冲区 securec_check(rc, "", ""); if (pfreeit) pfree(datafield); @@ -565,25 +619,25 @@ int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes) /* * Fill any hole */ - off = (int)(obj_desc->offset % LOBLKSIZE); + off = (int)(obj_desc->offset % LOBLKSIZE); // 计算偏移量 if (off > len) { - rc = memset_s(workb + len, off - len, '\0', off - len); + rc = memset_s(workb + len, off - len, '\0', off - len); // 填充空洞 securec_check(rc, "", ""); } /* * Insert appropriate portion of new data */ - n = LOBLKSIZE - off; - n = (n <= (nbytes - nwritten)) ? n : (nbytes - nwritten); - rc = memcpy_s(workb + off, n, buf + nwritten, n); + n = LOBLKSIZE - off; // 可写入的长度 + n = (n <= (nbytes - nwritten)) ? n : (nbytes - nwritten); // 取较小的长度 + rc = memcpy_s(workb + off, n, buf + nwritten, n); // 拷贝数据到工作缓冲区 securec_check(rc, "", ""); - nwritten += n; - obj_desc->offset += n; + nwritten += n; // 更新已写入的字节数 + obj_desc->offset += n; // 更新偏移量 off += n; /* compute valid length of new page */ len = (len >= off) ? len : off; - SET_VARSIZE(&workbuf.hdr, len + VARHDRSZ); + SET_VARSIZE(&workbuf.hdr, len + VARHDRSZ); // 设置新页的长度 /* * Form and insert updated tuple @@ -596,11 +650,11 @@ int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes) securec_check(rc, "", ""); values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf); replace[Anum_pg_largeobject_data - 1] = true; - newtup = - heap_modify_tuple(oldtuple, RelationGetDescr(t_thrd.storage_cxt.lo_heap_r), values, nulls, replace); - simple_heap_update(t_thrd.storage_cxt.lo_heap_r, &newtup->t_self, newtup); - CatalogIndexInsert(indstate, newtup); - heap_freetuple(newtup); + newtup = heap_modify_tuple(oldtuple, RelationGetDescr(t_thrd.storage_cxt.lo_heap_r), values, nulls, + replace); // 修改元组 + simple_heap_update(t_thrd.storage_cxt.lo_heap_r, &newtup->t_self, newtup); // 更新堆中的元组 + CatalogIndexInsert(indstate, newtup); // 在索引中插入元组 + heap_freetuple(newtup); // 释放新元组 /* * We're done with this old page. @@ -614,24 +668,24 @@ int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes) * * First, fill any hole */ - off = (int)(obj_desc->offset % LOBLKSIZE); + off = (int)(obj_desc->offset % LOBLKSIZE); // 计算偏移量 if (off > 0) { - rc = memset_s(workb, off, '\0', off); + rc = memset_s(workb, off, '\0', off); // 填充空洞 securec_check(rc, "", ""); } /* * Insert appropriate portion of new data */ - n = LOBLKSIZE - off; - n = (n <= (nbytes - nwritten)) ? n : (nbytes - nwritten); - rc = memcpy_s(workb + off, n, buf + nwritten, n); + n = LOBLKSIZE - off; // 可写入的长度 + n = (n <= (nbytes - nwritten)) ? n : (nbytes - nwritten); // 取较小的长度 + rc = memcpy_s(workb + off, n, buf + nwritten, n); // 拷贝数据到工作缓冲区 securec_check(rc, "", ""); - nwritten += n; - obj_desc->offset += n; + nwritten += n; // 更新已写入的字节数 + obj_desc->offset += n; // 更新偏移量 /* compute valid length of new page */ len = off + n; - SET_VARSIZE(&workbuf.hdr, len + VARHDRSZ); + SET_VARSIZE(&workbuf.hdr, len + VARHDRSZ); // 设置新页的长度 /* * Form and insert updated tuple @@ -643,171 +697,142 @@ int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes) values[Anum_pg_largeobject_loid - 1] = ObjectIdGetDatum(obj_desc->id); values[Anum_pg_largeobject_pageno - 1] = Int32GetDatum(pageno); values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf); - newtup = heap_form_tuple(t_thrd.storage_cxt.lo_heap_r->rd_att, values, nulls); - (void)simple_heap_insert(t_thrd.storage_cxt.lo_heap_r, newtup); - CatalogIndexInsert(indstate, newtup); - heap_freetuple(newtup); + newtup = heap_form_tuple(t_thrd.storage_cxt.lo_heap_r->rd_att, values, nulls); // 创建新元组 + (void)simple_heap_insert(t_thrd.storage_cxt.lo_heap_r, newtup); // 插入新元组 + CatalogIndexInsert(indstate, newtup); // 在索引中插入元组 + heap_freetuple(newtup); // 释放新元组 } pageno++; } - systable_endscan_ordered(sd); + systable_endscan_ordered(sd); // 结束有序扫描 - CatalogCloseIndexes(indstate); + CatalogCloseIndexes(indstate); // 关闭目录索引 /* * Advance command counter so that my tuple updates will be seen by later * large-object operations in this transaction. */ - CommandCounterIncrement(); + CommandCounterIncrement(); // 提升命令计数器，以确保后续事务可以看到我更新的元组 return nwritten; } -void inv_truncate(LargeObjectDesc* obj_desc, int len) +void inv_truncate(LargeObjectDesc *obj_desc, int len) { - int32 pageno = (int32)(len / LOBLKSIZE); + int32 pageno = (int32)(len / LOBLKSIZE); // 计算页号 + int off; ScanKeyData skey[2]; SysScanDesc sd; HeapTuple oldtuple; Form_pg_largeobject olddata; + struct { bytea hdr; - char data[LOBLKSIZE]; /* make struct big enough */ - int32 align_it; /* ensure struct is aligned well enough */ + char data[LOBLKSIZE]; // 用于存储数据的缓冲区 + int32 align_it; } workbuf; - char* workb = VARDATA(&workbuf.hdr); + char *workb = VARDATA(&workbuf.hdr); + HeapTuple newtup; Datum values[Natts_pg_largeobject] = {0, 0, 0}; - bool nulls[Natts_pg_largeobject] = {false, false, false}; + bool nulls[Natts_pg_largeobject] = {false, false, false}; bool replace[Natts_pg_largeobject] = {false, false, false}; CatalogIndexState indstate; errno_t rc; - rc = memset_s(&workbuf, sizeof(workbuf), 0, sizeof(workbuf)); - securec_check(rc, "\0", "\0"); - Assert(PointerIsValid(obj_desc)); - check_obj_desc(obj_desc); - open_lo_relation(); + rc = memset_s(&workbuf, sizeof(workbuf), 0, sizeof(workbuf)); // 将workbuf缓冲区清零 + securec_check(rc, "\0", "\0"); // 检查内存操作是否成功 - indstate = CatalogOpenIndexes(t_thrd.storage_cxt.lo_heap_r); + Assert(PointerIsValid(obj_desc)); // 检查obj_desc指针是否有效 - /* - * Set up to find all pages with desired loid and pageno >= target - */ + check_obj_desc(obj_desc); // 检查obj_desc的描述是否有效 + open_lo_relation(); // 打开大型对象关系表 + + indstate = CatalogOpenIndexes(t_thrd.storage_cxt.lo_heap_r); // 打开索引 + + /* 设置扫描键，查找指定loid和pageno大于等于目标的所有页 */ ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id)); ScanKeyInit(&skey[1], Anum_pg_largeobject_pageno, BTGreaterEqualStrategyNumber, F_INT4GE, Int32GetDatum(pageno)); - sd = systable_beginscan_ordered( - t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2, skey); + sd = systable_beginscan_ordered(t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2, + skey); // 开始有序扫描 - /* - * If possible, get the page the truncation point is in. The truncation - * point may be beyond the end of the LO or in a hole. - */ olddata = NULL; if ((oldtuple = systable_getnext_ordered(sd, ForwardScanDirection)) != NULL) { - if (HeapTupleHasNulls(oldtuple)) /* paranoia */ + if (HeapTupleHasNulls(oldtuple)) // 检查旧元组是否有空值（出于谨慎） ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("null field found in pg_largeobject"))); olddata = (Form_pg_largeobject)GETSTRUCT(oldtuple); - Assert(olddata->pageno >= pageno); + Assert(olddata->pageno >= pageno); // 检查pageno是否大于等于目标页号 } - /* - * If we found the page of the truncation point we need to truncate the - * data in it. Otherwise if we're in a hole, we need to create a page to - * mark the end of data. - */ if (olddata != NULL && olddata->pageno == pageno) { - /* First, load old data into workbuf */ - bytea* datafield = NULL; + bytea *datafield = NULL; bool pfreeit = false; int pagelen; + /* 将旧数据加载到workbuf中 */ getdatafield(olddata, &datafield, &pagelen, &pfreeit); rc = memcpy_s(workb, pagelen, VARDATA(datafield), pagelen); securec_check(rc, "", ""); if (pfreeit) pfree(datafield); - /* - * Fill any hole - */ off = len % LOBLKSIZE; + + /* 填充任何空隙 */ if (off > pagelen) { rc = memset_s(workb + pagelen, off - pagelen, '\0', off - pagelen); securec_check(rc, "", ""); } - /* compute length of new page */ - SET_VARSIZE(&workbuf.hdr, off + VARHDRSZ); + SET_VARSIZE(&workbuf.hdr, off + VARHDRSZ); // 计算新页的长度 - /* - * Form and insert updated tuple - */ values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf); replace[Anum_pg_largeobject_data - 1] = true; - newtup = heap_modify_tuple(oldtuple, RelationGetDescr(t_thrd.storage_cxt.lo_heap_r), values, nulls, replace); - simple_heap_update(t_thrd.storage_cxt.lo_heap_r, &newtup->t_self, newtup); - CatalogIndexInsert(indstate, newtup); - heap_freetuple(newtup); + newtup = heap_modify_tuple(oldtuple, RelationGetDescr(t_thrd.storage_cxt.lo_heap_r), values, nulls, + replace); // 修改旧的元组 + simple_heap_update(t_thrd.storage_cxt.lo_heap_r, &newtup->t_self, newtup); // 更新堆上的元组 + CatalogIndexInsert(indstate, newtup); // 插入索引 + heap_freetuple(newtup); // 释放元组内存 } else { - /* - * If the first page we found was after the truncation point, we're in - * a hole that we'll fill, but we need to delete the later page - * because the loop below won't visit it again. - */ if (olddata != NULL) { Assert(olddata->pageno > pageno); - simple_heap_delete(t_thrd.storage_cxt.lo_heap_r, &oldtuple->t_self); + simple_heap_delete(t_thrd.storage_cxt.lo_heap_r, &oldtuple->t_self); // 删除后续页面 } - /* - * Write a brand new page. - * - * Fill the hole up to the truncation point - */ off = len % LOBLKSIZE; + + /* 填充空隙 */ if (off > 0) { rc = memset_s(workb, off, '\0', off); securec_check(rc, "", ""); } - /* compute length of new page */ - SET_VARSIZE(&workbuf.hdr, off + VARHDRSZ); + SET_VARSIZE(&workbuf.hdr, off + VARHDRSZ); // 计算新页的长度 - /* - * Form and insert new tuple - */ + /* 插入新的元组 */ values[Anum_pg_largeobject_loid - 1] = ObjectIdGetDatum(obj_desc->id); values[Anum_pg_largeobject_pageno - 1] = Int32GetDatum(pageno); values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf); newtup = heap_form_tuple(t_thrd.storage_cxt.lo_heap_r->rd_att, values, nulls); - (void)simple_heap_insert(t_thrd.storage_cxt.lo_heap_r, newtup); - CatalogIndexInsert(indstate, newtup); - heap_freetuple(newtup); + (void)simple_heap_insert(t_thrd.storage_cxt.lo_heap_r, newtup); // 在堆上插入元组 + CatalogIndexInsert(indstate, newtup); // 插入索引 + heap_freetuple(newtup); // 释放元组内存 } - /* - * Delete any pages after the truncation point. If the initial search - * didn't find a page, then of course there's nothing more to do. - */ if (olddata != NULL) { while ((oldtuple = systable_getnext_ordered(sd, ForwardScanDirection)) != NULL) { - simple_heap_delete(t_thrd.storage_cxt.lo_heap_r, &oldtuple->t_self); + simple_heap_delete(t_thrd.storage_cxt.lo_heap_r, &oldtuple->t_self); // 删除截断点后的页面 } } - systable_endscan_ordered(sd); + systable_endscan_ordered(sd); // 结束有序扫描 - CatalogCloseIndexes(indstate); + CatalogCloseIndexes(indstate); // 关闭索引 - /* - * Advance command counter so that tuple updates will be seen by later - * large-object operations in this transaction. - */ - CommandCounterIncrement(); + CommandCounterIncrement(); // 提升命令计数器，以便后续事务中的元组更新可见 } -- 2.34.1