Compare commits

..

48 Commits

Author SHA1 Message Date
Auweafan f6ddcc93b3 Update inv_api.cpp 2023-10-04 18:47:53 +08:00
hyt15810656168 9124cc9411 Update custorage.cpp 2023-10-04 18:00:53 +08:00
Auweafan badf1e6b54 Update xlog_share_storage.cpp 2023-10-04 16:47:11 +08:00
hyt15810656168 a857f4f4a1 Update cucache_mgr.cpp 2023-10-04 05:22:17 +08:00
hyt15810656168 7d1fd62298 Update cstore_update.cpp 2023-10-04 05:21:54 +08:00
hyt15810656168 f0a0a6f0ee Update cstore_roughcheck_func.cpp 2023-10-04 05:21:38 +08:00
hyt15810656168 9a08f5f0bc Update cstore_psort.cpp 2023-10-04 05:21:18 +08:00
hyt15810656168 309fb71880 Update cstore_minmax_func.cpp 2023-10-04 05:20:59 +08:00
hyt15810656168 f380fd6dfc Update cstore_mem_alloc.cpp 2023-10-04 05:20:38 +08:00
hyt15810656168 c15a35f028 Update cstore_delta.cpp 2023-10-04 05:20:18 +08:00
hyt15810656168 d31b22c465 Update cstore_delete.cpp 2023-10-04 05:19:56 +08:00
hyt15810656168 907076b9f0 Update cstore_allocspace.cpp 2023-10-04 05:19:33 +08:00
hyt15810656168 cefa81d8fe Update cache_mgr.cpp 2023-10-04 05:17:49 +08:00
Auweafan 6902501b65 Update policy_common.cpp 2023-10-03 17:49:29 +08:00
Auweafan 103e5ffaa3 Update gs_policy_utils.cpp 2023-10-03 17:49:00 +08:00
Auweafan 49edc13014 Update gs_policy_masking.cpp 2023-10-03 17:48:06 +08:00
RrrrrR 6a7effc925 Update tde_key_storage.cpp 2023-10-03 16:50:12 +08:00
RrrrrR 53a8f126c3 Update tde_key_manager.cpp 2023-10-03 16:49:47 +08:00
RrrrrR 028a2299f8 Update Makefile 2023-10-03 16:49:00 +08:00
RrrrrR 8c4a8f418d Update kms_interface.cpp 2023-10-03 16:48:09 +08:00
RrrrrR d191245bb7 Update kms_httpmsg.ini 2023-10-03 16:47:42 +08:00
RrrrrR 7eeb5bc495 Update http_common.cpp 2023-10-03 16:47:13 +08:00
RrrrrR bebc90731e Update data_common.cpp 2023-10-03 16:46:45 +08:00
RrrrrR 757c6de347 Update CMakeLists.txt 2023-10-03 16:46:10 +08:00
RrrrrR f28aca29e2 Update ckms_message.cpp 2023-10-03 16:45:33 +08:00
RrrrrR cab4c90f8b Update remote_read.cpp 2023-10-03 16:44:28 +08:00
RrrrrR c122078fe8 Update remote_adapter.cpp 2023-10-03 16:43:44 +08:00
RrrrrR e0c15d35aa Update Makefile 2023-10-03 16:42:50 +08:00
RrrrrR 9e251c113d Update CMakeLists.txt 2023-10-03 16:41:52 +08:00
hyt15810656168 8436b49b49 Update pageparse.cpp 2023-10-02 19:58:42 +08:00
hyt15810656168 c72ec6b588 Update pagecompress.cpp 2023-10-02 19:58:24 +08:00
hyt15810656168 2d539e3df3 Update itemptr.cpp 2023-10-02 19:57:53 +08:00
hyt15810656168 8c03bf5b39 Update gs_xlogdump.cpp 2023-10-02 19:57:35 +08:00
hyt15810656168 abe39f5288 Update checksum_impl.cpp 2023-10-02 19:57:18 +08:00
hyt15810656168 83e595ee56 Update checksum.cpp 2023-10-02 19:56:55 +08:00
hyt15810656168 5ba9dbee50 Update bufpage.cpp 2023-10-02 19:56:25 +08:00
hyt15810656168 975b177d85 Update userchain.cpp 2023-09-30 00:45:16 +08:00
hyt15810656168 86f1c843dc Update ledger_utils.cpp 2023-09-30 00:44:55 +08:00
hyt15810656168 595bb69c5d Update ledger_check.cpp 2023-09-30 00:44:20 +08:00
RrrrrR 6a606596be Update iprange.h 2023-08-27 22:37:32 +08:00
hyt15810656168 3f12371f1d Update ledger_check.cpp 2023-08-27 22:35:49 +08:00
Auweafan 097306bf47 Update gs_string.cpp 2023-08-27 22:34:45 +08:00
hyt15810656168 6c15a750f0 Update ledger_archive.cpp 2023-08-27 22:33:58 +08:00
RrrrrR 2c29b99fa5 Update iprange.cpp 2023-08-27 22:33:46 +08:00
Auweafan e9ff3a3412 Update gs_policy_audit.cpp 2023-08-27 22:33:24 +08:00
hyt15810656168 0cd2edfb52 Update blockchain.cpp 2023-08-27 22:31:59 +08:00
Auweafan 74add04db0 Update curl_utils.cpp 2023-08-27 20:43:23 +08:00
hyt15810656168 8f54badbdd Update chkpass.cpp 2023-08-15 10:20:47 +08:00
47 changed files with 11321 additions and 6621 deletions

View File

@ -3,7 +3,7 @@
* Written by D'Arcy J.M. Cain
* darcy@druid.net
* http://www.druid.net/darcy/
*
*111
* contrib/chkpass/chkpass.c
* best viewed with tabs set to 4
*/

View File

@ -32,61 +32,72 @@
#include "utils/snapmgr.h"
/*
* gen_global_hash -- generate globalhash of gchain
* gen_global_hash -- generate globalhash of gchain //生成gchain的全局哈希值
*
* hash_buffer: the buffer that ready to fill generated globalhash.
* info_string: operate info string of current block.
* exist: whether previous block exists.
* prev_hash: the address of previous hash value.
* hash_buffer: the buffer that ready to fill generated globalhash.//准备填充生成的全局哈希值的缓冲区
* info_string: operate info string of current block.//当前块的操作信息字符串。
* exist: whether previous block exists.//前一个块是否存在。
* prev_hash: the address of previous hash value.//前一个哈希值的地址。
*
* Note: globalhash is generated by operate info and previous globalhash using md5.
* 使md5算法
*/
//该函数的作用是根据区块链中块的信息和相关参数,生成全局哈希用于验证和识别区块链的完整性和链式结构。(判断、生成块与MD5哈希值)
bool gen_global_hash(hash32_t *hash_buffer, const char *info_string, bool exist, const hash32_t *prev_hash)
{
errno_t rc = EOK;
int comb_strlen;
char *comb_string = NULL;
/*
* Previous block not exists means current insertion block is genesis,
* then we use global systable as origin combine string for globalhash
* generation. If previous block exists, we will use previous global
* hash as combine string to calculate globalhash.
*/
if (!exist) {
/* 如果前一个块不存在表示当前插入的块是创世块使用全局systable作为全局哈希生成的组合字符串
使 */
if (!exist) {//前一个块不存在
/* generate genesis block globalhash */
comb_strlen = strlen(GCHAIN_NAME) + strlen(info_string) + 1;
comb_string = (char *)palloc0(comb_strlen);
rc = snprintf_s(comb_string, comb_strlen, comb_strlen - 1, "%s%s", GCHAIN_NAME, info_string);
securec_check_ss(rc, "", "");
/* 生成创世块的全局哈希 */
comb_strlen = strlen(GCHAIN_NAME) + strlen(info_string) + 1;//计算组合字符串的长度
comb_string = (char *)palloc0(comb_strlen);//分配足够的内存给组合字符串并初始化为0
rc = snprintf_s(comb_string, comb_strlen, comb_strlen - 1, "%s%s", GCHAIN_NAME, info_string);// 将GCHAIN_NAME和info_string拼接到组合字符串中
//comb_string为存储位置comb_strlen为最大允许字符数
securec_check_ss(rc, "", "");//检查snprintf_s是否成功——如果发生错误并返回值为-1表示目标缓冲区或格式字符串是一个空指针或者无效的参数句柄被调用。此时宏函数会释放分配给缓冲区和其他可变参数的内存并在日志中输出错误信息提供文件名和行号。
//函数来源 src\include\gtm\utils\elog.h
} else {
/* use previous globalhash and current block info to calculate globalhash. */
char *pre_hash_str = DatumGetCString(DirectFunctionCall1(hash32out, HASH32GetDatum(prev_hash)));
comb_strlen = strlen(pre_hash_str) + strlen(info_string) + 1;
comb_string = (char *)palloc0(comb_strlen);
rc = snprintf_s(comb_string, comb_strlen, comb_strlen - 1, "%s%s", info_string, pre_hash_str);
securec_check_ss(rc, "", "");
pfree_ext(pre_hash_str);
/* 使用前一个全局哈希和当前块信息计算全局哈希 */
char *pre_hash_str = DatumGetCString(DirectFunctionCall1(hash32out, HASH32GetDatum(prev_hash)));//将prev_hash转换为字符串形式
comb_strlen = strlen(pre_hash_str) + strlen(info_string) + 1;//计算组合字符串的长度
comb_string = (char *)palloc0(comb_strlen);//分配足够的内存给组合字符串并初始化为0
rc = snprintf_s(comb_string, comb_strlen, comb_strlen - 1, "%s%s", info_string, pre_hash_str); //将info_string和pre_hash_str拼接到组合字符串中
securec_check_ss(rc, "", "");//检查snprintf_s是否成功
pfree_ext(pre_hash_str);//释放pre_hash_str占用的内存
}
if (!pg_md5_binary(comb_string, comb_strlen - 1, hash_buffer->data)) {
pfree(comb_string);
ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Failed to generate globalhash, out of memory")));
/* 使用pg_md5_binary函数计算组合字符串的MD5哈希值并存储在hash_buffer中 */
if (!pg_md5_binary(comb_string, comb_strlen - 1, hash_buffer->data)) {//函数来源src\common\backend\libpq\md5.cpp
pfree(comb_string);//释放组合字符串占用的内存
ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("Failed to generate globalhash, out of memory")));// 报错,内存不足
return false;
}
pfree(comb_string);
return true;
pfree(comb_string);//释放组合字符串占用的内存
return true;//返回全局哈希生成是否成功的结果
}
/*
* set_gchain_comb_string -- combine block informations.
* set_gchain_comb_string -- combine block informations.//组合块信息
*
* db_name: the database name where executes cmd.
* user_name: the user name who executes cmd
* nsp_name: namespace name of usertable
* rel_name: rel_name of usertable
* cmd_text: the command query which modified user table.
* rel_hash: rel_hash of current block.
* db_name: the database name where executes cmd.//执行命令的数据库名称
* user_name: the user name who executes cmd//执行命令的用户名
* nsp_name: namespace name of usertable//用户表的命名空间名称
* rel_name: rel_name of usertable//用户表的关系名称
* cmd_text: the command query which modified user table.//修改用户表的命令查询
* rel_hash: rel_hash of current block.//当前块的关系哈希值
*/
//将多个字符串和数值拼接在一起,设置区块链全局哈希生成所需的组合字符串。
char *set_gchain_comb_string(const char *db_name, const char *user_name,
const char *nsp_name, const char *rel_name, const char *cmd_text, uint64 rel_hash)
{
@ -95,97 +106,110 @@ char *set_gchain_comb_string(const char *db_name, const char *user_name,
}
int comb_len = strlen(db_name) + strlen(user_name) + strlen(nsp_name) +
strlen(rel_name) + strlen(cmd_text) + PREVIOUS_HASH_LEN + 1;
char *comb_str = (char *)palloc0(sizeof(char) * comb_len);
errno_t rc = snprintf_s(comb_str, comb_len, comb_len - 1, "%s%s%s%s%s%lu",
char *comb_str = (char *)palloc0(sizeof(char) * comb_len);//分配足够的内存给组合字符串并初始化为0
errno_t rc = snprintf_s(comb_str, comb_len, comb_len - 1, "%s%s%s%s%s%lu",//将各个字符串和数值按指定格式拼接到组合字符串中存入comb_str
db_name, user_name, nsp_name, rel_name, cmd_text, rel_hash);
securec_check_ss(rc, "\0", "\0");
return comb_str;
securec_check_ss(rc, "\0", "\0");//检查snprintf_s是否执行成功
return comb_str;//返回生成的组合字符串
}
/*
* ledger_gchain_append -- record a block to gchain.
* ledger_gchain_append -- record a block to gchain.//将一个块记录到gchain
*
* relid: relation oid of usertable.
* query_string: original query which modified usertable.
* cn_hash: rel_hash in hist table generated by query_string.
* relid: relation oid of usertable.//用户表的关系OID
* query_string: original query which modified usertable.//修改用户表的原始查询语句
* cn_hash: rel_hash in hist table generated by query_string.//通过query_string生成的hist表中的关系哈希值
*
* Note: after block inserted into gchain, its globalhash will flush
* into gchain cache for next block. Thus, previous global hash is
* come from cache directly.
* gchain之后gchain缓存中以供下一个块使用
*/
//用于向全局链表中追加记录,为区块链的增长提供了支持
void ledger_gchain_append(Oid relid, const char *query_string, uint64 cn_hash)
{
Datum current_time;
Datum values[Natts_gs_global_chain] = {0};
bool nulls[Natts_gs_global_chain] = {false};
Datum current_time;//当前时间
Datum values[Natts_gs_global_chain] = {0};//存储要插入的数据值
//Natts_gs_global_chain定义为10来源src\include\catalog\gs_global_chain.h
bool nulls[Natts_gs_global_chain] = {false};//标记是否为 NULL
char *db_name = NULL;
char *user_name = NULL;
char *nsp_name = NULL;
char *rel_name = NULL;
char *combine_string = NULL;
HeapTuple tup = NULL;
Relation rel_gchain = NULL;
GlobalPrevBlock current_block;
char *combine_string = NULL;//存储字符串
HeapTuple tup = NULL; //堆元组
Relation rel_gchain = NULL;//全局链表关系
GlobalPrevBlock current_block;//当前块
/* get basic informations. */
db_name = get_database_name(u_sess->proc_cxt.MyDatabaseId);
user_name = GetUserNameFromId(GetCurrentUserId());
current_time = TimestampTzGetDatum(GetCurrentTimestamp());
nsp_name = get_namespace_name(get_rel_namespace(relid));
rel_name = get_rel_name(relid);
//获取基本信息
db_name = get_database_name(u_sess->proc_cxt.MyDatabaseId);//根据数据库的OID获取当前数据库名来源src\gausskernel\optimizer\commands\dbcommands.cpp
user_name = GetUserNameFromId(GetCurrentUserId());//根据用户OID获取当前用户名来源src\common\backend\utils\init\miscinit.cpp
current_time = TimestampTzGetDatum(GetCurrentTimestamp());//获取当前时间戳
nsp_name = get_namespace_name(get_rel_namespace(relid));//根据给定的命名空间OID获取关系所在命名空间名来源src\common\backend\utils\cache\lsyscache.cpp
rel_name = get_rel_name(relid);//根据给定的关系OID获取关系名来源src\common\backend\utils\cache\lsyscache.cpp
/* Make combine string of current record: rel_name + nsp_name + query_string + rel_hash */
/* 生成当前记录的组合字符串: rel_name + nsp_name + query_string + rel_hash */
//调用本文件中的函数
combine_string = set_gchain_comb_string(db_name, user_name, nsp_name, rel_name, query_string, cn_hash);
/*
* rel_hash: sum of hash in DN which generated by this query_string.
* globalhash: hash for last record of gs_global_chain, it means blockchain prevhash.
* rel_hash: sum of hash in DN which generated by this query_string.//由此查询字符串在DN中生成的哈希值之和
* globalhash: hash for last record of gs_global_chain, it means blockchain prevhash.//gs_global_chain的最后一条记录的哈希值也即区块链的前一个哈希值
*/
current_block.blocknum = get_next_g_blocknum();
gen_global_hash(&current_block.globalhash, combine_string, false, NULL);
current_block.blocknum = get_next_g_blocknum();// 获取下一个区块号
gen_global_hash(&current_block.globalhash, combine_string, false, NULL);// 创建下一个块并生成MD5哈希值
// 准备要插入到全局链表中的数据
values[Anum_gs_global_chain_blocknum - 1] = UInt64GetDatum(current_block.blocknum);//将当前块的块号blocknum转换为UInt64类型并将其存储在values数组的对应位置上。
values[Anum_gs_global_chain_dbname - 1] = DirectFunctionCall1(namein, CStringGetDatum(db_name));//将数据库名db_name转换为namein函数所需的输入参数类型并将其存储在values数组的对应位置上
values[Anum_gs_global_chain_username - 1] = DirectFunctionCall1(namein, CStringGetDatum(user_name));//将用户名user_name转换为namein函数所需的输入参数类型并将其存储在values数组的对应位置上
values[Anum_gs_global_chain_starttime - 1] = current_time;//将当前时间current_time存储在values数组的对应位置上
values[Anum_gs_global_chain_relid - 1] = ObjectIdGetDatum(relid);//将关系IDrelid转换为ObjectId类型并将其存储在values数组的对应位置上
values[Anum_gs_global_chain_relnsp - 1] = DirectFunctionCall1(namein, CStringGetDatum(nsp_name));//将命名空间名nsp_name转换为namein函数所需的输入参数类型并将其存储在values数组的对应位置上
values[Anum_gs_global_chain_relname - 1] = DirectFunctionCall1(namein, CStringGetDatum(rel_name));//将关系名rel_name转换为namein函数所需的输入参数类型并将其存储在values数组的对应位置上
values[Anum_gs_global_chain_relhash - 1] = UInt64GetDatum(cn_hash);//将关系哈希值cn_hash转换为UInt64类型并将其存储在values数组的对应位置上
values[Anum_gs_global_chain_globalhash - 1] = HASH32GetDatum(&current_block.globalhash);//将当前块的全局哈希值current_block.globalhash转换为HASH32类型并将其存储在values数组的对应位置上
values[Anum_gs_global_chain_txcommand - 1] = CStringGetTextDatum(query_string);//将查询字符串query_string转换为text类型并将其存储在values数组的对应位置上
values[Anum_gs_global_chain_blocknum - 1] = UInt64GetDatum(current_block.blocknum);
values[Anum_gs_global_chain_dbname - 1] = DirectFunctionCall1(namein, CStringGetDatum(db_name));
values[Anum_gs_global_chain_username - 1] = DirectFunctionCall1(namein, CStringGetDatum(user_name));
values[Anum_gs_global_chain_starttime - 1] = current_time;
values[Anum_gs_global_chain_relid - 1] = ObjectIdGetDatum(relid);
values[Anum_gs_global_chain_relnsp - 1] = DirectFunctionCall1(namein, CStringGetDatum(nsp_name));
values[Anum_gs_global_chain_relname - 1] = DirectFunctionCall1(namein, CStringGetDatum(rel_name));
values[Anum_gs_global_chain_relhash - 1] = UInt64GetDatum(cn_hash);
values[Anum_gs_global_chain_globalhash - 1] = HASH32GetDatum(&current_block.globalhash);
values[Anum_gs_global_chain_txcommand - 1] = CStringGetTextDatum(query_string);
rel_gchain = heap_open(GsGlobalChainRelationId, RowExclusiveLock);// 打开全局链表关系
tup = heap_form_tuple(rel_gchain->rd_att, values, nulls);//创建新的堆元组
rel_gchain = heap_open(GsGlobalChainRelationId, RowExclusiveLock);
tup = heap_form_tuple(rel_gchain->rd_att, values, nulls);
simple_heap_insert(rel_gchain, tup);
heap_freetuple(tup);
simple_heap_insert(rel_gchain, tup);//插入堆元组到全局链表中
//插入方法来源src\gausskernel\storage\access\heap\heapam.cpp
//通过获取事务标识符,检查冲突,准备缓冲区,将元组插入到关系中,并处理可见性和日志记录等步骤,实现了数据的插入
heap_freetuple(tup);//释放堆元组内存
/* set latest previous global chain block */
heap_close(rel_gchain, RowExclusiveLock);
pfree(combine_string);
/* 设置最新的上一个全局链块 */
heap_close(rel_gchain, RowExclusiveLock);//关闭全局链表关系
pfree(combine_string);//释放组合字符串内存
}
/*
* ledger_output_append_hash -- append relhash to response tag.
* ledger_output_append_hash -- append relhash to response tag.//将relhash附加到响应标签中
*
* resp_tag: response tag address.
* operation: command operation.
* hash: the hash that prepare to append.
* resp_tag: response tag address.//响应标签的地址
* operation: command operation.//命令操作类型
* hash: the hash that prepare to append.//准备附加的哈希值
*/
//该函数的作用是在执行插入、更新和删除操作时,将关系哈希值追加到响应标签中,以便后续处理和记录。
static void ledger_output_append_hash(char *resp_tag, CmdType operation, uint64 hash)
{
Assert(resp_tag != NULL);
size_t len = strlen(resp_tag);
errno_t ret = EOK;
Assert(resp_tag != NULL); //确保resp_tag不为空
size_t len = strlen(resp_tag);//获取resp_tag的长度
errno_t ret = EOK;//初始化错误号
switch (operation) {
case CMD_INSERT:
case CMD_UPDATE:
case CMD_DELETE:
//将哈希值转换为字符串并追加到resp_tag字符串末尾
ret = snprintf_s(resp_tag + len, COMPLETION_TAG_BUFSIZE - len, COMPLETION_TAG_BUFSIZE - len - 1,
" %lu\0", hash);
securec_check_ss(ret, "\0", "\0");
" %lu\0", hash);// 追加的位置是resp_tag的末尾COMPLETION_TAG_BUFSIZE - len为可添加的最大字符数COMPLETION_TAG_BUFSIZE - len - 1为要写入的字符串的最大长度
securec_check_ss(ret, "\0", "\0");//检查函数调用是否成功
break;
default:
break;
@ -193,34 +217,39 @@ static void ledger_output_append_hash(char *resp_tag, CmdType operation, uint64
}
/*
* ledger_ExecutorEnd -- record block to gchain.
* ledger_ExecutorEnd -- record block to gchain.//将块记录到gchain中
*
* query_desc: query descript of executor.
* query_desc: query descript of executor.//执行器的查询描述
*
* Note: append new block to gchain in CN or singlenode.
* Note: append new block to gchain in CN or singlenode.//在CN或单节点中将新的块追加到gchain中
* each DN will link its relhash to es_modifiedRowHash, CN or singlenode
* use es_modifiedRowHash to receive all DN relhash and accumulate them
* as cn_relhash for insertion.
* DN将其关系哈希链接到es_modifiedRowHashCN或单节点使用es_modifiedRowHash接收所有DN的关系哈希并将它们累积为cn_relhash进行插入
*/
//是一个钩子函数,在查询执行完成后被调用。它主要的功能是记录查询的相关信息到全局链表中,
//包括关系ID、查询字符串、哈希值等。同时它还检查并将哈希值追加到响应标签中。
//负责触发记录操作
static void ledger_ExecutorEnd(QueryDesc *query_desc)
{
uint64 hashsum;
bool has_remote_hash = query_desc->estate->es_modifiedRowHash != NIL;
hashsum = hash_combiner(query_desc->estate->es_modifiedRowHash);
uint64 hashsum; //哈希值的总和
bool has_remote_hash = query_desc->estate->es_modifiedRowHash != NIL;//是否存在远程哈希
hashsum = hash_combiner(query_desc->estate->es_modifiedRowHash);//组合远程哈希的哈希值
if ((IS_PGXC_COORDINATOR || g_instance.role == VSINGLENODE) && has_remote_hash) {
//如果当前节点为PGXC_COORDINATOR协调节点或VSINGLENODE单节点并且是否存在远程哈希
Oid relid = InvalidOid;
Relation rel = NULL;
int relnum = query_desc->estate->es_num_result_relations;
int relnum = query_desc->estate->es_num_result_relations;//结果关系数量
if (relnum > 0) {
rel = query_desc->estate->es_result_relations->ri_RelationDesc;
/* gs_global_chain only records following actions */
switch (query_desc->operation) {
switch (query_desc->operation) {//根据查询到的操作类型
case CMD_INSERT:
case CMD_DELETE:
case CMD_UPDATE:
relid = RelationGetRelid(rel);
if (rel->rd_isblockchain) {
ledger_gchain_append(relid, query_desc->sourceText, hashsum);
relid = RelationGetRelid(rel); //获取关系的ID
if (rel->rd_isblockchain) {//如果是块链表
ledger_gchain_append(relid, query_desc->sourceText, hashsum);//将关系哈希信息追加到gchain中
}
break;
default:
@ -230,57 +259,68 @@ static void ledger_ExecutorEnd(QueryDesc *query_desc)
}
if (u_sess->ledger_cxt.resp_tag != NULL && has_remote_hash && !IsConnFromApp()) {
//如果相应标签地址的指针不为空&&存在远程哈希并且&&非应用程序连接,则将哈希值追加到响应标签中
ledger_output_append_hash(u_sess->ledger_cxt.resp_tag, query_desc->operation, hashsum);
u_sess->ledger_cxt.resp_tag = NULL;
u_sess->ledger_cxt.resp_tag = NULL;//清空响应标签
}
if (t_thrd.security_ledger_cxt.prev_ExecutorEnd) {
((ExecutorEnd_hook_type)t_thrd.security_ledger_cxt.prev_ExecutorEnd)(query_desc);
((ExecutorEnd_hook_type)t_thrd.security_ledger_cxt.prev_ExecutorEnd)(query_desc); // 调用前一个ExecutorEnd钩子函数
} else {
standard_ExecutorEnd(query_desc);
standard_ExecutorEnd(query_desc); //执行标准的ExecutorEnd操作
//函数来源src\gausskernel\runtime\executor\execMain.cpp
//该函数功能为释放快照、LLVM 编译清理、切换上下文并释放内存、重置查询描述的字段、输出内存追踪信息到文件、收集指令计数信息、重置永久空间perm space的全局值
}
}
/*
* light_ledger_ExecutorEnd -- record block to gchain in light proxy.
* light_ledger_ExecutorEnd -- record block to gchain in light proxy.//在轻量级代理中将块记录到全局链gchain
*
* query: query of executor.
* relhash: sum of all DN relhash for insertion.
* query: query of executor.//执行器的查询
* relhash: sum of all DN relhash for insertion.//插入操作的所有分布式节点关系哈希值的总和
*
* Note: in light proxy logical, process will extract and
* accumulate relhash from response text of DNs as cn_relhash.
* cn_relhash
*/
//根据传入的查询信息判断关系是否为用户表如果是则将关系的哈希信息追加到gchain中以记录区块链中的数据变动。
void light_ledger_ExecutorEnd(Query *query, uint64 relhash)
{
//如果不是PGXC协调器节点并且当前节点的角色不是单节点VSINGLENODE
if (!IS_PGXC_COORDINATOR && g_instance.role != VSINGLENODE) {
return;
}
Oid relid = InvalidOid;
Oid relid = InvalidOid;//关系ID默认为无效ID
switch (query->commandType) {
switch (query->commandType) {//获取查询对象的命令类型
case CMD_INSERT:
case CMD_DELETE:
case CMD_UPDATE:
relid = get_target_query_relid(query->rtable, query->resultRelation);
if (is_ledger_usertable(relid)) {
ledger_gchain_append(relid, query->sql_statement, relhash);
relid = get_target_query_relid(query->rtable, query->resultRelation);//获取目标查询的关系ID
//函数来源src\gausskernel\security\gs_ledger\ledger_utils.cpp
if (is_ledger_usertable(relid)) {//根据关系ID的有效性、关系类型和所属命名空间判断关系是否为用户表
//函数来源src\gausskernel\security\gs_ledger\ledger_utils.cpp
ledger_gchain_append(relid, query->sql_statement, relhash);//将关系哈希信息追加到gchain中
}
break;
/* Not support others */
//其他指令不做处理
default:
break;
}
}
/*
* light_ledger_ExecutorEnd -- record block to gchain in opfusion.
* light_ledger_ExecutorEnd -- record block to gchain in opfusion.//在操作融合opfusion中将块记录到全局链gchain
*
* fusiontype: operator type.
* relid: relation oid of usertable.
* query: original query which modified usertable.
* relhash: relhash in hist table generated by sourceText.
* fusiontype: operator type.//操作类型
* relid: relation oid of usertable.//用户表的关系OID
* query: original query which modified usertable.//修改了用户表的原始查询
* relhash: relhash in hist table generated by sourceText.//由源文本生成的历史表中的关系哈希值
*/
//和前一个函数相比此函数判断执行条件时要求当前节点的角色是虚拟数据节点或者关系ID对应的表不是用户表。。
void opfusion_ledger_ExecutorEnd(FusionType fusiontype, Oid relid, const char *query, uint64 relhash)
{
//当前节点的角色是VDATANODE虚拟数据节点或者关系ID对应的表不是用户表
if (g_instance.role == VDATANODE || !is_ledger_usertable(relid)) {
return;
}
@ -289,8 +329,8 @@ void opfusion_ledger_ExecutorEnd(FusionType fusiontype, Oid relid, const char *q
case INSERT_FUSION:
case UPDATE_FUSION:
case DELETE_FUSION:
if (is_ledger_usertable(relid)) {
ledger_gchain_append(relid, query, relhash);
if (is_ledger_usertable(relid)) {//根据关系ID的有效性、关系类型和所属命名空间判断关系是否为用户表
ledger_gchain_append(relid, query, relhash);//将关系哈希信息追加到gchain中
}
break;
/* Not support others */
@ -302,16 +342,23 @@ void opfusion_ledger_ExecutorEnd(FusionType fusiontype, Oid relid, const char *q
/*
* ledger_hook_init -- install of gchain block record hook.
*/
//初始化钩子函数,将自定义的执行器结束钩子函数 ledger_ExecutorEnd 替换掉原有的钩子函数 ExecutorEnd_hook
//从而在执行器结束时触发自定义的操作。这样可以实现对关系的哈希信息追加到 gchain 中的功能。
void ledger_hook_init(void)
{
//将原先的 ExecutorEnd_hook 函数保存到 prev_ExecutorEnd 变量中
t_thrd.security_ledger_cxt.prev_ExecutorEnd = (void *)ExecutorEnd_hook;
//将 ledger_ExecutorEnd 函数赋值给 ExecutorEnd_hook以替换原有的钩子函数
ExecutorEnd_hook = ledger_ExecutorEnd;
}
/*
* ledger_hook_fini -- uninstall of gchain block record hook.
*/
//恢复原始的钩子函数,将之前保存的 prev_ExecutorEnd 变量的值重新赋值给 ExecutorEnd_hook以恢复原始的钩子函数的功能。这样可以确保在钩子函数替换后再次恢复原有的钩子函数避免对系统功能产生影响。
void ledger_hook_fini(void)
{
//将prev_ExecutorEnd 变量的值赋值给 ExecutorEnd_hook恢复原始的钩子函数
ExecutorEnd_hook = (ExecutorEnd_hook_type)t_thrd.security_ledger_cxt.prev_ExecutorEnd;
}

View File

@ -51,31 +51,42 @@
#endif
/*
* prepare_histback_dir -- create hist_back dir under pg_audit.
* prepare_histback_dir -- create hist_back dir under pg_audit.//在 pg_audit 目录下创建 hist_back 文件夹
*/
//准备历史备份目录的函数
static void prepare_histback_dir(void)
{
// 使用安全的方式构建历史备份目录路径
char ledger_histback_dir[MAXPGPATH] = {0};
int rc = snprintf_s(ledger_histback_dir, MAXPGPATH, MAXPGPATH - 1,
"%s/hist_bak", g_instance.attr.attr_security.Audit_directory);
securec_check_ss(rc, "\0", "\0");
//使用snprintf_s函数确保路径字符串不会超过MAXPGPATH的长度限制
securec_check_ss(rc, "\0", "\0");//检查snprintf_s是否成功
/*
* Create histback directory if not present; ignore errors
*
*/
//创建主目录,用于存放历史备份目录
(void)pg_mkdir_p(g_instance.attr.attr_security.Audit_directory, S_IRWXU);
//创建历史备份目录
(void)pg_mkdir_p(ledger_histback_dir, S_IRWXU);
}
/*
* ledger_copytable -- copy rows of hist table.
* ledger_copytable -- copy rows of hist table.//复制历史表中的行数据
*
* cstate: copy information state
* cstate: copy information state//复制信息状态
*
* Note: this function is only used to copy user hist table or gchain.
* we only copy commited data, thus for each history row, we will
* recalculate prehash to ensure the consistency of front-to-back order.
* gchain
prehash
*/
//将指定表的数据复制到输出流的功能,并且在复制过程中根据特定逻辑计算哈希值。
//用于复制数据库中表数据
static uint64 ledger_copytable(CopyState cstate)
{
Relation cur_rel;
@ -86,28 +97,34 @@ static uint64 ledger_copytable(CopyState cstate)
uint64 processed = 0;
bool is_gchain;
cur_rel = cstate->curPartionRel;
is_gchain = RelationGetRelid(cur_rel) == GsGlobalChainRelationId;
tuple_desc = RelationGetDescr(cur_rel);
attr = tuple_desc->attrs;
num_phys_attrs = tuple_desc->natts;
cstate->null_print_client = cstate->null_print;
cur_rel = cstate->curPartionRel;//获取当前分区关系
is_gchain = RelationGetRelid(cur_rel) == GsGlobalChainRelationId;//判断是否为全局链关系
tuple_desc = RelationGetDescr(cur_rel);//获取关系的元组描述符
attr = tuple_desc->attrs;//获取元组描述符中的属性数组
num_phys_attrs = tuple_desc->natts;//获取属性数量
cstate->null_print_client = cstate->null_print;//复制null_print到null_print_client
/* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
if (cstate->fe_msgbuf == NULL) {
cstate->fe_msgbuf = makeStringInfo();
if (IS_PGXC_COORDINATOR || g_instance.role == VSINGLENODE)
ProcessFileHeader(cstate);
//在复制表数据的过程中使用一个缓冲区来存储每一行的数据并且无论复制的目标是什么都会使用fe_msgbuf作为每行的缓冲区
if (cstate->fe_msgbuf == NULL) {//如果fe_msgbuf为空则创建一个StringInfo对象并发送文件头信息
cstate->fe_msgbuf = makeStringInfo();//创建一个StringInfo对象
if (IS_PGXC_COORDINATOR || g_instance.role == VSINGLENODE)//在协调器模式或单节点模式下
ProcessFileHeader(cstate);//处理文件头信息
//函数来源src\gausskernel\optimizer\commands\copy.cpp
//如果文件头信息是从文件中读取的,则打开文件并读取相应的内容,并进行一些格式验证和处理。
//如果文件头信息是直接提供的,则根据分隔符的不同输出相应的文件头信息。
}
/* For each column type, get its out function. */
cstate->out_functions = (FmgrInfo*)palloc(num_phys_attrs * sizeof(FmgrInfo));
foreach (cur, cstate->attnumlist) {
int attnum = lfirst_int(cur);
Oid out_func_oid;
bool isvarlena = false;
getTypeOutputInfo(attr[attnum - 1]->atttypid, &out_func_oid, &isvarlena);
fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);
// 为每列类型获取其输出函数
cstate->out_functions = (FmgrInfo*)palloc(num_phys_attrs * sizeof(FmgrInfo));//分配存储输出函数的数组
foreach (cur, cstate->attnumlist) {//对于每个属性列,遍历列表
int attnum = lfirst_int(cur);//获取属性列号
Oid out_func_oid;//获取函数的OID
bool isvarlena = false;//是否为可变长度类型
getTypeOutputInfo(attr[attnum - 1]->atttypid, &out_func_oid, &isvarlena);//获取属性列的输出函数和是否为可变长度类型是否具有变长属性的信息存入isvarlena
//函数来源src\common\backend\utils\cache\lsyscache.cpp
fmgr_info(out_func_oid, &cstate->out_functions[attnum - 1]);//获取输出函数的函数管理信息并将其存储在cstate->out_functions数组中的相应位置
}
/*
@ -116,15 +133,25 @@ static uint64 ledger_copytable(CopyState cstate)
* datatype output routines, and should be faster than retail pfree's
* anyway. (We don't need a whole econtext as CopyFrom does.)
*/
//创建一个临时内存上下文,用于在每行处理完成后重置并释放分配的内存
//CurrentMemoryContext:新创建的内存上下文将使用当前内存上下文作为父上下文
//"COPY TO":内存上下文的名称
//ALLOCSET_DEFAULT_MINSIZE:指定了内存上下文的初始大小
//ALLOCSET_DEFAULT_INITSIZE:指定了内存上下文初始分配的大小
//ALLOCSET_DEFAULT_MAXSIZE:指定了内存上下文可申请的最大内存量
cstate->rowcontext = AllocSetContextCreate(
CurrentMemoryContext, "COPY TO", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE);
/*
* For non-binary copy, we need to convert null_print to file
* encoding, because it will be sent directly with CopySendString.
*/
if (cstate->need_transcoding) {
if (cstate->need_transcoding) { //对于非二进制复制需要将null_print转换为文件编码
//确保在进行非二进制复制时null_print能够以正确的文件编码进行发送和处理
cstate->null_print_client = pg_server_to_any(cstate->null_print, cstate->null_print_len, cstate->file_encoding);
//函数来源src\common\backend\utils\mb\mbutils.cpp
//如果当前会话上下文中的客户端编码和目标编码相同则调用perform_default_encoding_conversion函数来执行默认的编码转换并返回结果字符串
//否则将通过调用pg_do_encoding_conversion函数来进行编码转换。该函数将原始字符串s从当前会话上下文中的数据库编码转换为目标编码并返回结果字符串。
}
Tuple tuple;
@ -134,51 +161,54 @@ static uint64 ledger_copytable(CopyState cstate)
Datum *values = NULL;
bool *nulls = NULL;
values = (Datum*)palloc0(num_phys_attrs * sizeof(Datum));
nulls = (bool*)palloc0(num_phys_attrs * sizeof(bool));
values = (Datum*)palloc0(num_phys_attrs * sizeof(Datum));//分配存储值的数组
nulls = (bool*)palloc0(num_phys_attrs * sizeof(bool));//分配存储空值标志的数组
scan_desc = scan_handler_tbl_beginscan(cur_rel, GetActiveSnapshot(), 0, NULL);
scan_desc = scan_handler_tbl_beginscan(cur_rel, GetActiveSnapshot(), 0, NULL);//开始表扫描
/* For each row, we will recalculate previous hash. */
//对于每一行,重新计算之前的哈希值
while ((tuple = scan_handler_tbl_getnext(scan_desc, ForwardScanDirection, cur_rel)) != NULL) {
CHECK_FOR_INTERRUPTS();
CHECK_FOR_INTERRUPTS();//每次循环检查是否有中断请求,确保在进行长时间运行的操作时,可以及时响应中断信号
/* Deconstruct the tuple ... faster than repeated heap_getattr */
//解析元组并获取列值和空值标记存储在数组values和nulls中
tableam_tops_deform_tuple2(tuple, tuple_desc, values, nulls, GetTableScanDesc(scan_desc, cur_rel)->rs_cbuf);
if (!is_gchain) {
if (!is_gchain) {//对于非全局链关系,设置记录数和上一个哈希值的属性值
char comb_str[NAMEDATALEN] = {0};
uint64 t_ins = nulls[USERCHAIN_COLUMN_HASH_INS] ? 0 : DatumGetUInt64(values[USERCHAIN_COLUMN_HASH_INS]);
uint64 t_del = nulls[USERCHAIN_COLUMN_HASH_DEL] ? 0 : DatumGetUInt64(values[USERCHAIN_COLUMN_HASH_DEL]);
errno_t rc = sprintf_s(comb_str, NAMEDATALEN, "%lu%lu", t_ins, t_del);
uint64 t_ins = nulls[USERCHAIN_COLUMN_HASH_INS] ? 0 : DatumGetUInt64(values[USERCHAIN_COLUMN_HASH_INS]);//哈希插入值
uint64 t_del = nulls[USERCHAIN_COLUMN_HASH_DEL] ? 0 : DatumGetUInt64(values[USERCHAIN_COLUMN_HASH_DEL]);//哈希删除值
errno_t rc = sprintf_s(comb_str, NAMEDATALEN, "%lu%lu", t_ins, t_del);//将插入值和删除值组合为字符串存储在comb_str中
securec_check_ss(rc, "", "");
gen_hist_tuple_hash(RelationGetRelid(cur_rel), comb_str, rec_num > 0, &pre_hash, &pre_hash);
values[USERCHAIN_COLUMN_REC_NUM] = UInt64GetDatum(rec_num);
values[USERCHAIN_COLUMN_PREVHASH] = HASH32GetDatum(&pre_hash);
} else {
char *db_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_dbname - 1]));
char *user_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_username - 1]));
char *rel_nsp = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_relnsp - 1]));
char *rel_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_relname - 1]));
char *query_string = TextDatumGetCString(values[Anum_gs_global_chain_txcommand - 1]);
uint64 rel_hash = DatumGetUInt64(values[Anum_gs_global_chain_relhash - 1]);
char *comb_str = set_gchain_comb_string(db_name, user_name, rel_nsp, rel_name, query_string, rel_hash);
gen_global_hash(&pre_hash, comb_str, rec_num > 0, &pre_hash);
gen_hist_tuple_hash(RelationGetRelid(cur_rel), comb_str, rec_num > 0, &pre_hash, &pre_hash);//函数来源src\gausskernel\security\gs_ledger\userchain.cpp
values[USERCHAIN_COLUMN_REC_NUM] = UInt64GetDatum(rec_num);//设置记录数属性值
values[USERCHAIN_COLUMN_PREVHASH] = HASH32GetDatum(&pre_hash);//设置上一个哈希值属性值
} else {//对于全局链关系,设置块号和全局哈希的属性值
char *db_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_dbname - 1]));//数据库名
char *user_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_username - 1]));//用户名
char *rel_nsp = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_relnsp - 1]));//模式名
char *rel_name = DatumGetCString(DirectFunctionCall1(nameout, values[Anum_gs_global_chain_relname - 1]));//关系名
char *query_string = TextDatumGetCString(values[Anum_gs_global_chain_txcommand - 1]);//查询字符串
uint64 rel_hash = DatumGetUInt64(values[Anum_gs_global_chain_relhash - 1]);//关系哈希值
char *comb_str = set_gchain_comb_string(db_name, user_name, rel_nsp, rel_name, query_string, rel_hash);//组合字符串
gen_global_hash(&pre_hash, comb_str, rec_num > 0, &pre_hash);//计算全局哈希
values[Anum_gs_global_chain_blocknum - 1] = UInt64GetDatum(rec_num);
values[Anum_gs_global_chain_globalhash - 1] = HASH32GetDatum(&pre_hash);
pfree(comb_str);
}
/* Format and send the data */
CopyOneRowTo(cstate, HeapTupleGetOid((HeapTuple)tuple), values, nulls);
CopyOneRowTo(cstate, HeapTupleGetOid((HeapTuple)tuple), values, nulls);//发送数据
//函数来源src\gausskernel\optimizer\commands\copy.cpp
rec_num++;
processed++;
}
scan_handler_tbl_endscan(scan_desc);
scan_handler_tbl_endscan(scan_desc);//结束表扫描
//释放内存
pfree_ext(values);
pfree_ext(nulls);
MemoryContextDelete(cstate->rowcontext);
MemoryContextDelete(cstate->rowcontext);//删除临时内存上下文
return processed;
return processed;//返回已处理的记录数
}
/*
@ -190,6 +220,7 @@ static uint64 ledger_copytable(CopyState cstate)
* Note: notice that this function is only used for copying hist table.
* DO NOT use in other copy scenarios.
*/
//将数据从一个关系(表)复制到另一个地方
static uint64 ledger_docopy(CopyStmt *stmt, const char *queryString)
{
Relation rel;
@ -201,73 +232,92 @@ static uint64 ledger_docopy(CopyStmt *stmt, const char *queryString)
int attnum;
/* Open and lock the relation, using the appropriate lock type. */
//打开关系(表)并获取锁定访问权限
rel = heap_openrv(stmt->relation, AccessShareLock);
//创建范围表项并设置相关属性
rte = makeNode(RangeTblEntry);
rte->rtekind = RTE_RELATION;
rte->relid = RelationGetRelid(rel);
rte->relkind = rel->rd_rel->relkind;
rte->requiredPerms = ACL_SELECT;
rte->rtekind = RTE_RELATION;//表示该范围表项对应的是一个关系(表)
rte->relid = RelationGetRelid(rel);//获取打开关系(表)的标识符
rte->relkind = rel->rd_rel->relkind;//获取关系(表)的类型
rte->requiredPerms = ACL_SELECT;//该范围表项需要执行SELECT操作的权限
//获取关系(表)的元组描述符
tup_desc = RelationGetDescr(rel);
//根据关系ID确定需要选择的列数
attnum = (rte->relid == GsGlobalChainRelationId) ? Natts_gs_global_chain : USERCHAIN_COLUMN_NUM;
/* add columns that need select permission. */
//添加需要选择权限的列至范围表项中
for (int i = 1; i <= attnum; ++i) {
//从1循环到attnum需要选择的列数根据FirstLowInvalidHeapAttributeNumber表中的第一个有效列号计算出对应的列号attno
int attno = i - FirstLowInvalidHeapAttributeNumber;
rte->selectedCols = bms_add_member(rte->selectedCols, attno);
rte->selectedCols = bms_add_member(rte->selectedCols, attno);//函数来源src\common\backend\nodes\bitmapset.cpp
//记录需要选择的列
}
//检查是否具有查询权限
(void)ExecCheckRTPerms(list_make1(rte), true);
//创建CopyState结构并执行COPY操作
cstate = BeginCopyTo(rel, query, queryString, stmt->filename, stmt->attlist, stmt->options);
cstate->range_table = list_make1(rte);
cstate->curPartionRel = cstate->rel;
processed = ledger_copytable(cstate);
processed = ledger_copytable(cstate);//复制数据库中表数据
EndCopyTo(cstate);
//关闭关系(表),释放锁定访问权限
if (rel != NULL) {
heap_close(rel, AccessShareLock);
}
//返回处理的行数
return processed;
}
/*
* get_current_timestamp_text -- generate time text for name appending
* get_current_timestamp_text -- generate time text for name appending//生成用于名称附加的时间文本
*
* time_str: time text buffer to fill.
* time_str: time text buffer to fill.//用于填充时间文本的缓冲区
*
* Note: get current server time and remove dot.
* Note: get current server time and remove dot.//获取当前服务器时间并去除小数点。
*/
//获取当前时间戳的整数部分,并将其转换为字符串形式
static void get_current_timestamp_text(char *time_str)
{
const char *now = timestamptz_to_str(GetCurrentTimestamp());
size_t time_len = strlen(now);
size_t pos = 0;
const char *now = timestamptz_to_str(GetCurrentTimestamp());//获取当前时间戳字符串
size_t time_len = strlen(now);//时间戳字符串的长度
size_t pos = 0;//字符写入位置
//遍历时间戳字符串的每个字符
for (size_t i = 0; i < time_len; ++i) {
if (now[i] >= '0' && now[i] <= '9') {
time_str[pos++] = now[i];
} else if (now[i] == '.') {
break;
if (now[i] >= '0' && now[i] <= '9') { //如果当前字符为数字
time_str[pos++] = now[i];//将数字字符写入结果字符串
} else if (now[i] == '.') {//如果当前字符为小数点
break;//结束遍历,不再写入小数部分
}
}
time_str[pos] = '\0';
time_str[pos] = '\0';//在结果字符串的末尾添加结束符
}
/*
* copy_local_hist_table -- copy history table to hist_back dir.
* copy_local_hist_table -- copy history table to hist_back dir.//将历史表复制到hist_back目录
*
* relid: oid of usertable
* histname: the name of history table
* time: time text which will append to copyfile name.
* relid: oid of usertable//用户表的OID
* histname: the name of history table//历史表的名称
* time: time text which will append to copyfile name.//将附加到复制文件名的时间文本
*/
//这段代码通过构造COPY语句并指定表名、备份文件路径等参数然后调用ledger_docopy函数执行COPY操作将本地历史表的内容复制到指定路径下的文件中
//将历史表复制到hist_back目录
static void copy_local_hist_table(Oid relid, char *histname, const char *time)
{
errno_t rc;
char path[MAXPGPATH] = {0};
StringInfoData buf;
initStringInfo(&buf);
errno_t rc;//定义一个用于检查返回码的变量
char path[MAXPGPATH] = {0};//定义一个数组用于存储备份文件的路径,并初始化为全零
StringInfoData buf;//定义一个用于构造COPY语句的字符串缓冲区变量
initStringInfo(&buf);//初始化字符串缓冲区
//创建CopyStmt节点并设置成员
CopyStmt *stmt = makeNode(CopyStmt);
RangeVar *relation = makeRangeVar("blockchain", histname, -1);
RangeVar *relation = makeRangeVar("blockchain", histname, -1);//指向历史表
//构造备份文件路径
if (!is_absolute_path(g_instance.attr.attr_security.Audit_directory)) {
//将路径格式化为字符串
rc = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/%s/hist_bak/%s_%u_%s.hist",
t_thrd.proc_cxt.DataDir, g_instance.attr.attr_security.Audit_directory, histname, relid, time);
} else {
@ -275,128 +325,165 @@ static void copy_local_hist_table(Oid relid, char *histname, const char *time)
g_instance.attr.attr_security.Audit_directory, histname, relid, time);
}
securec_check_ss(rc, "", "");
//构造COPY语句
appendStringInfo(&buf, "COPY blockchain.%s to \'%s\'", histname, path);
//设置CopyStmt成员
stmt->relation = relation;
stmt->is_from = false;
stmt->filename = path;
//调用ledger_docopy函数执行COPY操作
ledger_docopy((CopyStmt *)stmt, buf.data);
}
/*
* open_histback_dir -- open hist_back dir.
* open_histback_dir -- open hist_back dir.//打开 hist_back 目录
*
* dir_path: hist_back dir path.
* dir_path: hist_back dir path.//hist_back 目录的路径
*
* Note: caller need to close hist_back dir later.
* Note: caller need to close hist_back dir later.//调用者需要稍后关闭 hist_back 目录
*/
//根据备份文件目录的相对或绝对路径构造历史备份目录的完整路径并使用AllocateDir函数打开该目录然后返回一个指向该目录的DIR指针。
//用于打开历史备份目录
static DIR *open_histback_dir(char *dir_path)
{
//声明变量和初始化
DIR *dir = NULL;
errno_t rc;
//判断是否为绝对路径来构造历史备份目录路径
if (!is_absolute_path(g_instance.attr.attr_security.Audit_directory)) {
//如果不是绝对路径,则使用相对路径构造历史备份目录的完整路径
rc = snprintf_s(dir_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s/hist_bak",
t_thrd.proc_cxt.DataDir, g_instance.attr.attr_security.Audit_directory);
} else {
//如果是绝对路径,则直接使用该路径作为历史备份目录的完整路径
rc = snprintf_s(dir_path, MAXPGPATH, MAXPGPATH - 1, "%s/hist_bak",
g_instance.attr.attr_security.Audit_directory);
}
securec_check_ss(rc, "", "");
securec_check_ss(rc, "", "");//检查路径构造过程中的返回码,如果出现错误则报错
//使用AllocateDir函数打开历史备份目录将返回的DIR指针赋值给dir
dir = AllocateDir(dir_path);
//返回DIR指针
return dir;
}
/*
* get_histback_dir_filesize -- count all file size of hist_back dir.
* get_histback_dir_filesize -- count all file size of hist_back dir.//计算hist_back目录中所有文件的总大小
*/
//打开历史备份目录并遍历其中的文件,对普通文件累加文件大小,最后返回历史备份目录中所有文件的总大小
static uint64 get_histback_dir_filesize()
{
DIR *dir = NULL;
struct dirent *file = NULL;
char dir_path[MAXPGPATH] = {0};
errno_t rc = EOK;
uint64 size = 0;
DIR *dir = NULL;//定义指向目录的指针初始化为NULL
struct dirent *file = NULL;//定义指向目录项结构体的指针初始化为NULL
char dir_path[MAXPGPATH] = {0};//定义保存目录路径的字符数组并初始化为0
errno_t rc = EOK;//用于检查返回码的变量,初始化为 EOK。
uint64 size = 0;//定义保存文件大小的变量初始值为0
//调用open_histback_dir函数打开历史备份目录将返回的DIR指针赋值给dir
dir = open_histback_dir(dir_path);
//如果打开历史备份目录失败则返回0
if (dir == NULL) {
return 0;
}
//循环遍历历史备份目录中的文件
while ((file = ReadDir(dir, dir_path)) != NULL) {
struct stat statbuf;
char filepath[MAXPGPATH];
struct stat statbuf;//声明保存文件属性的结构体
char filepath[MAXPGPATH];//声明保存文件完整路径的字符数组
//判断当前文件是否为.或..目录,如果是则跳过本次循环
if (strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0) {
continue;
}
//构造文件的完整路径
rc = snprintf_s(filepath, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dir_path, file->d_name);
securec_check_ss(rc, "", "");
//判断当前文件是否是普通文件并通过stat函数获取文件属性
if (file->d_type == DT_REG && stat(filepath, &statbuf) == 0) {
size += (uint64)statbuf.st_size;
size += (uint64)statbuf.st_size;//累加文件大小到size变量
}
}
//关闭目录
FreeDir(dir);
//返回历史备份目录中所有文件的总大小
return size;
}
/*
* remove_oldest_histback_file -- remove oldest file in hist_back
* remove_oldest_histback_file -- remove oldest file in hist_back//删除 hist_back 目录中最早创建的文件
*
* Note: the defination of 'oldest' here is the earliest file that
* was created. this function will find and try to delete this file.
*/
//这段代码打开历史备份目录并遍历其中的文件,找到最早创建的普通文件,删除该文件并返回其大小
static uint64 remove_oldest_histback_file()
{
DIR *dir = NULL;
errno_t rc = EOK;
uint64 filesize = 0;
long min_ctime = LONG_MAX;
char dir_path[MAXPGPATH] = {0};
char del_file[MAXPGPATH] = {0};
struct dirent *file = NULL;
struct stat stat_buf;
DIR *dir = NULL;//定义指向目录的指针初始化为NULL
errno_t rc = EOK;//用于检查返回码的变量
uint64 filesize = 0;//定义保存文件大小的变量初始值为0
long min_ctime = LONG_MAX;//保存最早创建时间的变量初始值为LONG_MAX
char dir_path[MAXPGPATH] = {0};//定义保存目录路径的字符数组并初始化为0
char del_file[MAXPGPATH] = {0};//定义保存待删除文件路径的字符数组并初始化为0
struct dirent *file = NULL;//定义指向目录项结构体的指针初始化为NULL
struct stat stat_buf;//声明保存文件属性的结构体
//调用open_histback_dir函数打开历史备份目录将返回的DIR指针赋值给dir
dir = open_histback_dir(dir_path);
//如果打开历史备份目录失败则返回0
if (dir == NULL) {
return 0;
}
//循环遍历历史备份目录中的文件
while ((file = ReadDir(dir, dir_path)) != NULL) {
char file_path[MAXPGPATH];
char file_path[MAXPGPATH];//声明保存文件完整路径的字符数组
//判断当前文件是否为.或..目录,如果是则跳过本次循环
if (strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0) {
continue;
}
//构造文件的完整路径
rc = snprintf_s(file_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s", dir_path, file->d_name);
securec_check_ss(rc, "", "");
// 判断当前文件是否是普通文件并通过stat函数获取文件属性
if (file->d_type == DT_REG && stat(file_path, &stat_buf) == 0) {
//如果当前文件的创建时间较早,则更新最早创建时间和待删除文件路径
if (min_ctime >= stat_buf.st_ctime) {
min_ctime = stat_buf.st_ctime;
rc = snprintf_s(del_file, MAXPGPATH, MAXPGPATH - 1, "%s", file_path);
securec_check_ss(rc, "", "");
filesize = (uint64)stat_buf.st_size;
filesize = (uint64)stat_buf.st_size;//更新要返回的文件大小
}
}
}
//关闭目录
FreeDir(dir);
//使用unlink函数删除待删除文件如果失败则输出警告信息
if (unlink(del_file) < 0) {
ereport(WARNING, (errmsg("could not remove histbak file: %s", del_file)));
}
//返回被删除文件的大小
return filesize;
}
/*
* ledger_hist_archive -- interface for history table archive
* ledger_hist_archive -- interface for history table archive//历史表归档接口
*
* param1: the namespace of usertable.
* param2: the rel_name of usertable.
* param1: the namespace of usertable.//用户表的命名空间
* param2: the rel_name of usertable.//用户表的关系名称
*
* Note: this function will copy and unify hash_ins and hash_del
* of history table belonging to given usertable.
* hash_ins hash_del
*/
//归档用户历史表
Datum ledger_hist_archive(PG_FUNCTION_ARGS)
{
//检查当前用户是否具有足够的权限
//如果用户不是超级用户且不是审计管理员,则抛出错误
if (!isRelSuperuser() && !isAuditadmin(GetUserId())) {
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));
}
//从参数中获取表的命名空间和名称
text *rel_nsp = PG_GETARG_TEXT_PP(0);
text *rel_name = PG_GETARG_TEXT_PP(1);
char *table_name;
@ -406,28 +493,39 @@ Datum ledger_hist_archive(PG_FUNCTION_ARGS)
Oid relid;
bool res = false;
/* Init and prepare bak dictionary. */
//准备历史表备份目录
prepare_histback_dir();
//将命名空间和表名转换为C字符串
table_nsp = text_to_cstring(rel_nsp);
table_name = text_to_cstring(rel_name);
//获取命名空间的OID并获取表的OID
Oid nspoid = get_namespace_oid(table_nsp, false);
relid = get_relname_relid(table_name, nspoid);
//检查用户表是否存在并且属于账本ledger命名空间
//函数来源src\gausskernel\security\gs_ledger\ledger_utils.cpp
ledger_usertable_check(relid, nspoid, table_name, table_nsp);
//获取历史表的名称
//函数来源src\gausskernel\security\gs_ledger\ledger_utils.cpp
get_hist_name(relid, table_name, hist_name, nspoid, table_nsp);
get_current_timestamp_text(current_time);
//如果当前角色不是协调器,则执行以下操作
if (g_instance.role != VCOORDINATOR) {
/*
* Step 1. Copy user history table.
* Step 1. Copy user history table.
*/
//获取历史备份目录的总大小
uint64 total_histback_size = get_histback_dir_filesize();
//如果总大小超过限制,则删除最旧的历史备份文件,直到总大小小于限制
while (total_histback_size >= (uint64)(u_sess->attr.attr_security.Audit_SpaceLimit * 1024L)) {
total_histback_size -= remove_oldest_histback_file();
}
/* Copy history table. */
//复制历史表至本地
copy_local_hist_table(relid, hist_name, current_time);
/*
* Step 2. Do unify and truncate.
* Step 2. Do unify and truncate.
*/
Datum values[USERCHAIN_COLUMN_NUM] = {0};
bool nulls[USERCHAIN_COLUMN_NUM] = {true};
@ -440,21 +538,25 @@ Datum ledger_hist_archive(PG_FUNCTION_ARGS)
TableScanDesc scan;
HeapTuple tuple;
/* sum all hash_ins and hash_del for unification. */
lock_hist_hash_cache(LW_EXCLUSIVE);
Relation histRel = heap_open(get_relname_relid(hist_name, PG_BLOCKCHAIN_NAMESPACE), AccessExclusiveLock);
scan = heap_beginscan(histRel, SnapshotNow, 0, NULL);
while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) {
// 对历史表进行扫描计算hash_ins和hash_del的总和并找到最大的记录数和对应的prevhash值
lock_hist_hash_cache(LW_EXCLUSIVE);//获取对历史哈希缓存的独占锁
Relation histRel = heap_open(get_relname_relid(hist_name, PG_BLOCKCHAIN_NAMESPACE), AccessExclusiveLock);//打开历史表,获取对该表的独占访问权限
scan = heap_beginscan(histRel, SnapshotNow, 0, NULL);//开始对历史表进行扫描
while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) {//扫描方向为正向ForwardScanDirection
hist_empty = false;
//获取hash_ins值并累加
Datum value = heap_getattr(tuple, USERCHAIN_COLUMN_HASH_INS + 1, histRel->rd_att, &is_null);
if (!is_null) {
hash_ins += DatumGetUInt64(value);
nulls[USERCHAIN_COLUMN_HASH_INS] = false;
}
//获取hash_del值并累加
value = heap_getattr(tuple, USERCHAIN_COLUMN_HASH_DEL + 1, histRel->rd_att, &is_null);
if (!is_null) {
hash_del += DatumGetUInt64(value);
nulls[USERCHAIN_COLUMN_HASH_DEL] = false;
}
//获取当前记录数并更新最大记录数和对应的prevhash值
cur_rec_num = DatumGetUInt64(heap_getattr(tuple, USERCHAIN_COLUMN_REC_NUM + 1, histRel->rd_att, &is_null));
if (max_rec_num <= cur_rec_num) {
max_rec_num = cur_rec_num;
@ -462,82 +564,95 @@ Datum ledger_hist_archive(PG_FUNCTION_ARGS)
heap_getattr(tuple, USERCHAIN_COLUMN_PREVHASH + 1, histRel->rd_att, &is_null);
}
}
heap_endscan(scan);
heap_endscan(scan);//结束对历史表的扫描操作
/* Empty table should not truncate and archive any more. */
//如果历史表为空,则不执行统一和截断操作
if (hist_empty) {
heap_close(histRel, AccessExclusiveLock);
release_hist_hash_cache();
heap_close(histRel, AccessExclusiveLock);//关闭历史表,并释放对该表的独占访问权限
release_hist_hash_cache();//释放对历史哈希缓存的锁
res = true;
return BoolGetDatum(res);
}
//更新统一行的值
values[USERCHAIN_COLUMN_REC_NUM] = UInt64GetDatum(max_rec_num);
values[USERCHAIN_COLUMN_HASH_INS] = UInt64GetDatum(hash_ins);
values[USERCHAIN_COLUMN_HASH_DEL] = UInt64GetDatum(hash_del);
nulls[USERCHAIN_COLUMN_REC_NUM] = false;
nulls[USERCHAIN_COLUMN_PREVHASH] = false;
tuple = heap_form_tuple(RelationGetDescr(histRel), values, nulls);
tuple = heap_form_tuple(RelationGetDescr(histRel), values, nulls);//使用历史表的描述符RelationGetDescr(histRel)以及values和nulls数组来构建新的元组
/* Do real truncate. */
//执行真正的截断操作,对历史表进行截断
heap_truncate_one_rel(histRel);
/* Do insertion for unified row. */
simple_heap_insert(histRel, tuple);
heap_freetuple(tuple);
//插入统一行
simple_heap_insert(histRel, tuple);//将之前构建的元组插入到历史表中
heap_freetuple(tuple);//释放对元组的内存空间
/*
* Step 3. Flush history hash table cache.
* Step 3. Flush history hash table cache.
*/
remove_hist_recnum_cache(RelationGetRelid(histRel));
heap_close(histRel, AccessExclusiveLock);
release_hist_hash_cache();
remove_hist_recnum_cache(RelationGetRelid(histRel));//移除历史记录数量缓存
heap_close(histRel, AccessExclusiveLock);//关闭历史表,并释放对该表的独占访问权限
release_hist_hash_cache();//释放对历史哈希缓存的锁
res = true;
}
return BoolGetDatum(res);
}
/*
* ledger_gchain_archive -- archive gs_global_chain and unify each user rel
* ledger_gchain_archive -- archive gs_global_chain and unify each user rel//将gs_global_chain表归档并统一每个用户表
*
* Note: this function will copy gchain to hist_back dir and accumulate
* rel_hash for each relid. Additionally, recalculate globalhash to ensure the
* consistency of front-to-back order.
* gchain表复制到hist_back目录relid累加rel_hash
* globalhash以确保前后顺序的一致性
*/
//用于归档全局链表(gs_global_chain)的函数
Datum ledger_gchain_archive(PG_FUNCTION_ARGS)
{
//检查权限,只有超级用户或审核管理员才有权限执行该函数
if (!isRelSuperuser() && !isAuditadmin(GetUserId())) {
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));
}
bool res = false;
/* Init and prepare bak dictionary. */
//初始化并准备历史备份目录
prepare_histback_dir();
/* gs_global_chain table should only archived in CN */
/* 只有在协调器或单节点角色下才对 gs_global_chain 表进行归档 */
if (g_instance.role != VCOORDINATOR && g_instance.role != VSINGLENODE) {
return BoolGetDatum(res);
}
//获取历史备份目录的总大小
uint64 total_histback_size = get_histback_dir_filesize();
//当总大小超过设定的审计空间限制时,删除最旧的历史备份文件
while (total_histback_size >= (uint64)(u_sess->attr.attr_security.Audit_SpaceLimit * 1024L)) {
total_histback_size -= remove_oldest_histback_file();
}
/*
* Step 1. Using CopyStmt to copy global chain.
* Step 1. Using CopyStmt to copy global chain.使 CopyStmt
*/
CopyStmt *stmt = makeNode(CopyStmt);
RangeVar *relation = makeRangeVar("pg_catalog", GCHAIN_NAME, -1);
RangeVar *relation = makeRangeVar("pg_catalog", GCHAIN_NAME, -1);//pg_catalog表示表所在的namespaceGCHAIN_NAME是一个宏定义表示表名
char path[MAXPGPATH] = {0};
StringInfoData buf;
initStringInfo(&buf);
errno_t rc;
char current_time[NAMEDATALEN + 1];
get_current_timestamp_text(current_time);
if (!is_absolute_path(g_instance.attr.attr_security.Audit_directory)) {
get_current_timestamp_text(current_time);//获取当前时间的文本形式
//根据绝对路径或相对路径生成备份文件路径
if (!is_absolute_path(g_instance.attr.attr_security.Audit_directory)) {//相对路径
rc = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/%s/hist_bak/%s_%s.bak",
t_thrd.proc_cxt.DataDir, g_instance.attr.attr_security.Audit_directory,
GCHAIN_NAME, current_time);
} else {
} else {//绝对路径
rc = snprintf_s(path, MAXPGPATH, MAXPGPATH - 1, "%s/hist_bak/%s_%s.bak",
g_instance.attr.attr_security.Audit_directory, GCHAIN_NAME, current_time);
}
@ -546,10 +661,11 @@ Datum ledger_gchain_archive(PG_FUNCTION_ARGS)
stmt->relation = relation;
stmt->is_from = false;
stmt->filename = path;
//执行复制操作
ledger_docopy((CopyStmt *)stmt, buf.data);
/*
* Step 2. Do unify and truncate.
* Step 2. Do unify and truncate.gs_global_chain表
*/
TableScanDesc scan;
HeapTuple tuple;
@ -558,28 +674,35 @@ Datum ledger_gchain_archive(PG_FUNCTION_ARGS)
bool is_null = false;
gs_stl::gs_vector<Oid, true> user_rel_arr;
rc = memset_s(&hash_ctl, sizeof(hash_ctl), 0, sizeof(hash_ctl));
rc = memset_s(&hash_ctl, sizeof(hash_ctl), 0, sizeof(hash_ctl));//使用 memset_s 函数将 hash_ctl 清零
securec_check(rc, "\0", "\0");
/* Using hash table to do unify, each hash_entry refers to one relid informations. */
hash_ctl.keysize = sizeof(Oid);
hash_ctl.entrysize = sizeof(GChainArchEntry);
hash_ctl.hash = oid_hash;
hash_ctl.hcxt = THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY);
HTAB *global_map = hash_create("Global Archive Hash", 256, &hash_ctl, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT);
//使用哈希表进行统一每个哈希项对应一个relid的信息
hash_ctl.keysize = sizeof(Oid);//设置为 Oid 类型的大小
hash_ctl.entrysize = sizeof(GChainArchEntry);//设置为GChainArchEntry结构体的大小
hash_ctl.hash = oid_hash;//设置为Oid类型的哈希函数
hash_ctl.hcxt = THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY);//获取安全内存上下文
HTAB *global_map = hash_create("Global Archive Hash", 256, &hash_ctl, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT);//创建哈希表 global_map并进行错误检查
if (global_map == NULL) {
ereport(ERROR, (errcode(ERRCODE_INITIALIZE_FAILED), errmsg("could not initialize Global Archive Hash table.")));
}
/* Split gs_global_chain by relid, and accumulate rel_hash to a new record for each rel. */
LWLockAcquire(GlobalPrevHashLock, LW_EXCLUSIVE);
Relation global_rel = heap_open(GsGlobalChainRelationId, AccessExclusiveLock);
scan = heap_beginscan(global_rel, SnapshotNow, 0, NULL);
//按 relid 分割gs_global_chain表并将关联记录进行累加
LWLockAcquire(GlobalPrevHashLock, LW_EXCLUSIVE);//调用 LWLockAcquire 函数获取全局互斥锁GlobalPrevHashLock。这是为了确保在进行统一和截断操作时不会有其他进程同时访问gs_global_chain表。
Relation global_rel = heap_open(GsGlobalChainRelationId, AccessExclusiveLock);//打开关系表gs_global_chain并加上访问锁 AccessExclusiveLock
scan = heap_beginscan(global_rel, SnapshotNow, 0, NULL);//扫描时使用的快照是 SnapshotNow表示当前的快照
while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) {
//遍历 gs_global_chain 表中的所有记录,根据 relid 执行不同的操作。
//如果 relid 是第一次出现,创建一个新的项并记录数据;如果 relid 已经存在,将 rel_hash 进行累加并更新记录数量。
//这样就完成了 gs_global_chain 表的统一和截断操作。
Oid relid = DatumGetObjectId(heap_getattr(tuple, Anum_gs_global_chain_relid, global_rel->rd_att, &is_null));
GChainArchEntry *item = (GChainArchEntry *)hash_search(global_map, &relid, HASH_ENTER, &found);
GChainArchEntry *item = (GChainArchEntry *)hash_search(global_map, &relid, HASH_ENTER, &found);//在哈希表global_map中查找相应的项,如果找到了对应的项则将found标志设置为 true如果没有找到则创建一个新的项并将found标志设置为false。同时将relid添加数组中
if (!found) {
/* If not exist in hash table, create an entry and records Datums. */
/* 如果哈希表中不存在对应的项,则创建一个新的项并记录数据 */
user_rel_arr.push_back(relid);
heap_deform_tuple(tuple, RelationGetDescr(global_rel), item->val.values, item->val.nulls);
heap_deform_tuple(tuple, RelationGetDescr(global_rel), item->val.values, item->val.nulls);//将元组拆解并赋值
//用DirectFunctionCall1函数将某些字符串类型转换成合适的数据类型
item->val.values[Anum_gs_global_chain_dbname - 1] =
DirectFunctionCall1(namein, item->val.values[Anum_gs_global_chain_dbname - 1]);
item->val.values[Anum_gs_global_chain_username - 1] =
@ -592,8 +715,9 @@ Datum ledger_gchain_archive(PG_FUNCTION_ARGS)
item->val.rec_nums = 1;
} else {
/* If exists in hash table, just add rel_hash. */
/* 如果已存在哈希表中,则将 rel_hash 累加 */
uint64 rel_hash =
DatumGetUInt64(heap_getattr(tuple, Anum_gs_global_chain_relhash, global_rel->rd_att, &is_null));
DatumGetUInt64(heap_getattr(tuple, Anum_gs_global_chain_relhash, global_rel->rd_att, &is_null));//从元组中获取 rel_hash 的值,并将其加上之前项中的值
rel_hash += DatumGetUInt64(item->val.values[Anum_gs_global_chain_relhash - 1]);
item->val.values[Anum_gs_global_chain_relhash - 1] = UInt64GetDatum(rel_hash);
++item->val.rec_nums;
@ -601,24 +725,27 @@ Datum ledger_gchain_archive(PG_FUNCTION_ARGS)
}
heap_endscan(scan);
if (user_rel_arr.empty()) {
heap_close(global_rel, AccessExclusiveLock);
LWLockRelease(GlobalPrevHashLock);
hash_destroy(global_map);
if (user_rel_arr.empty()) {//如果数组为空表示进行统一和截断操作时没有找到任何匹配的relid记录
heap_close(global_rel, AccessExclusiveLock);//闭全局表 global_rel并释放访问锁 AccessExclusiveLock
LWLockRelease(GlobalPrevHashLock);//释放全局互斥锁 GlobalPrevHashLock
hash_destroy(global_map);//销毁哈希表 global_map
res = true;
return BoolGetDatum(res);
}
/* Do rel truncate. */
// 截断 gs_global_chain 表
heap_truncate_one_rel(global_rel);
/* Insert newest record to gchain order by relid. */
//按 relid 排序插入最新记录到 gchain 表中
hash32_t global_hash;
uint64 blocknum = -1;
for (int i = user_rel_arr.size() - 1; i >= 0; --i) {
GChainArchEntry *item = (GChainArchEntry *)hash_search(global_map, &user_rel_arr[i], HASH_FIND, &found);
for (int i = user_rel_arr.size() - 1; i >= 0; --i) {//从后向前处理每个元素
GChainArchEntry *item = (GChainArchEntry *)hash_search(global_map, &user_rel_arr[i], HASH_FIND, &found);//在哈希表global_map中查找与当前relid相对应的项并将结果存储在item
/* Prepare common string */
const char *query_string = "Archived.";
//用DirectFunctionCall1函数将item中的特定列值转换成合适的字符串类型
char *db_name =
DatumGetCString(DirectFunctionCall1(nameout, item->val.values[Anum_gs_global_chain_dbname - 1]));
char *user_name =
@ -630,24 +757,24 @@ Datum ledger_gchain_archive(PG_FUNCTION_ARGS)
uint64 rel_hash = DatumGetUInt64(item->val.values[Anum_gs_global_chain_relhash - 1]);
char *com_str = set_gchain_comb_string(db_name, user_name, rel_nsp, rel_name, query_string, rel_hash);
/* Generate global_hash. */
gen_global_hash(&global_hash, com_str, i < (int)user_rel_arr.size() - 1, &global_hash);
gen_global_hash(&global_hash, com_str, i < (int)user_rel_arr.size() - 1, &global_hash);//生成全局哈希
blocknum += item->val.rec_nums;
item->val.values[Anum_gs_global_chain_blocknum - 1] = UInt64GetDatum(blocknum);
item->val.values[Anum_gs_global_chain_globalhash - 1] = HASH32GetDatum(&global_hash);
tuple = heap_form_tuple(RelationGetDescr(global_rel), item->val.values, item->val.nulls);
simple_heap_insert(global_rel, tuple);
CatalogUpdateIndexes(global_rel, tuple);
simple_heap_insert(global_rel, tuple);//将tuple插入到global_rel表中
CatalogUpdateIndexes(global_rel, tuple);//更新global_rel表的索引
heap_freetuple(tuple);
pfree(com_str);
}
hash_destroy(global_map);
hash_destroy(global_map);//销毁哈希表,释放相关资源
/*
* Step 3. Flush global_hash cache.
* Step 3. Flush global_hash cache. global_hash
*/
reset_g_blocknum();
heap_close(global_rel, AccessExclusiveLock);
LWLockRelease(GlobalPrevHashLock);
reset_g_blocknum();//刷新global_hash缓存
heap_close(global_rel, AccessExclusiveLock);//关闭全局表 global_rel并释放访问锁AccessExclusiveLock
LWLockRelease(GlobalPrevHashLock);//释放全局互斥锁 GlobalPrevHashLock
res = true;
return BoolGetDatum(res);

View File

@ -47,99 +47,106 @@
#include "gs_ledger/blockchain.h"
/*
* gen_usertable_hash_sum -- calculate sum(hash) of the relation
* gen_usertable_hash_sum -- calculate sum(hash) of the relation//计算用户关系的hash总和
*
* rel: user relation
* rel: user relation//用户关系
*/
// 计算用户表的哈希总和(只计算给定关系的哈希总和)
static uint64 gen_usertable_hash_sum(Relation rel)
{
uint64 rel_hash = 0;
bool is_null = false;
int hash_natt = user_hash_attrno(rel->rd_att);
Assert(hash_natt >= 0);
HeapTuple tuple;
TupleDesc desc = rel->rd_att;
Snapshot snapshot = GetActiveSnapshot();
TableScanDesc scan;
if (RELATION_CREATE_BUCKET(rel)) {
uint64 rel_hash = 0;//初始化表的哈希总和为0
bool is_null = false;//初始化一个布尔变量,用于指示获取的属性值是否为空
int hash_natt = user_hash_attrno(rel->rd_att);//获取哈希属性在表描述符中的位置
Assert(hash_natt >= 0);//使用断言确保哈希属性的位置非负
HeapTuple tuple;//声明一个堆元组变量,用于存储从表中获取的每个元组
TupleDesc desc = rel->rd_att;//获取表的描述符
Snapshot snapshot = GetActiveSnapshot();//获取当前活动快照
TableScanDesc scan;//声明一个表扫描描述符
if (RELATION_CREATE_BUCKET(rel)) {//如果表是分桶表
Relation bucket_rel = NULL;
oidvector *bucket_list = searchHashBucketByOid(rel->rd_bucketoid);
oidvector *bucket_list = searchHashBucketByOid(rel->rd_bucketoid);//获取分桶表的列表
//遍历分桶表列表,每次取出一个分桶表的 OID并调用bucketGetRelation函数获取该分桶表的关系对象
for (int i = 0; i < bucket_list->dim1; i++) {
bucket_rel = bucketGetRelation(rel, NULL, bucket_list->values[i]);
scan = heap_beginscan(bucket_rel, snapshot, 0, NULL);
bucket_rel = bucketGetRelation(rel, NULL, bucket_list->values[i]);//获取分桶表的关系对象
scan = heap_beginscan(bucket_rel, snapshot, 0, NULL);//开始对分桶表进行堆扫描
//遍历分桶表中的每个元组,使用 heap_getnext 函数获取下一个元组,并根据哈希属性的位置获取属性值,将其累加到哈希总和上
while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) {
rel_hash += DatumGetUInt64(heap_getattr(tuple, hash_natt + 1, desc, &is_null));
}
heap_endscan(scan);
bucketCloseRelation(bucket_rel);
heap_endscan(scan);//结束对分桶表的扫描
bucketCloseRelation(bucket_rel);//关闭分桶表的关系对象
}
} else {
scan = heap_beginscan(rel, snapshot, 0, NULL);
} else {//如果表不是分桶表
scan = heap_beginscan(rel, snapshot, 0, NULL);//开始对表进行堆扫描
//遍历表中的每个元组,使用 heap_getnext 函数获取下一个元组,并根据哈希属性的位置获取属性值,将其累加到哈希总和上
while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) {
rel_hash += DatumGetUInt64(heap_getattr(tuple, hash_natt + 1, desc, &is_null));
}
heap_endscan(scan);
heap_endscan(scan);//结束扫描
}
return rel_hash;
}
/*
* get_usertable_hash_sum -- calculate sum(hash) of the relation of specified oid
* get_usertable_hash_sum -- calculate sum(hash) of the relation of specified oid//计算指定 OID 的关系对象的哈希总和sum(hash
*
* relid: user table oid
* relid: user table oid//用户表的 OID
*
* NOTICE: if user table is partition table, just use the original relation id.
* NOTICE: if user table is partition table, just use the original relation id.//注意:如果用户表是分区表,则只使用原始关系 ID
*/
//计算指定关系(表)的哈希总和(可以处理分区表,并计算整个分区表的哈希总和)
static uint64 get_usertable_hash_sum(Oid relid)
{
uint64 rel_hash = 0;
Relation rel = NULL;
uint64 rel_hash = 0;//存储最终的哈希总和
Relation rel = NULL;//存储打开的关系对象
rel = heap_open(relid, AccessShareLock);
if (!RelationIsPartitioned(rel)) {
if (!RelationIsPartitioned(rel)) {//判断打开的关系是否为分区表,如果不是,直接计算哈希总和
rel_hash = gen_usertable_hash_sum(rel);
} else {
List *partition_list = NIL;
ListCell *lc = NULL;
Partition part;
Relation fake_rel;
partition_list = relationGetPartitionList(rel, AccessShareLock);
} else {//关系为分区表
List *partition_list = NIL;//存储分区列表
ListCell *lc = NULL;//遍历分区列表的指针变量
Partition part;//存储当前分区
Relation fake_rel;//存储伪关系对象,即分区对应的关系对象
partition_list = relationGetPartitionList(rel, AccessShareLock);//获取分区列表并使用共享锁
foreach (lc, partition_list) {
part = (Partition)lfirst(lc);
fake_rel = partitionGetRelation(rel, part);
fake_rel = partitionGetRelation(rel, part);//通过分区和关系获取相应的伪关系对象
rel_hash += gen_usertable_hash_sum(fake_rel);
releaseDummyRelation(&fake_rel);
releaseDummyRelation(&fake_rel);//释放伪关系对象的资源
}
releasePartitionList(rel, &partition_list, AccessShareLock);
releasePartitionList(rel, &partition_list, AccessShareLock);//释放分区列表的资源
}
heap_close(rel, AccessShareLock);
return rel_hash;
}
/*
* get_histtable_hash_sum -- calculate sum(hash_ins) - sum(hash_del) of the history relation of specified oid
* get_histtable_hash_sum -- calculate sum(hash_ins) - sum(hash_del) of the history relation of specified oid// 计算指定 OID 的历史表的 sum(hash_ins) - sum(hash_del)
*
* relid: history table oid
* relid: history table oid//历史表的 OID
*/
//遍历所有历史表的行,并计算每行的 hash_ins 列和 hash_del 列的差值
static uint64 get_histtable_hash_sum(Oid hist_oid)
{
uint64 rel_hash = 0;
bool is_null = false;
Relation hist_rel;
TableScanDesc scan;
HeapTuple tuple;
Snapshot snapshot = GetActiveSnapshot();
uint64 rel_hash = 0;//初始化哈希总和为0
bool is_null = false;//用于判断属性值是否为NULL
Relation hist_rel;//关系对象,用于表示分区表
TableScanDesc scan;//表扫描描述符
HeapTuple tuple;//元组,用于获取表中的每一行数据
Snapshot snapshot = GetActiveSnapshot();//获取当前活动快照
hist_rel = heap_open(hist_oid, AccessShareLock);
scan = heap_beginscan(hist_rel, snapshot, 0, NULL);
hist_rel = heap_open(hist_oid, AccessShareLock);//打开分区表并获取关系对象
scan = heap_beginscan(hist_rel, snapshot, 0, NULL);//开始表扫描
while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) {
// 从元组中获取hash_ins列的值
Datum value = heap_getattr(tuple, USERCHAIN_COLUMN_HASH_INS + 1, hist_rel->rd_att, &is_null);
if (!is_null) {
rel_hash += DatumGetUInt64(value);
rel_hash += DatumGetUInt64(value);//如果列值不为NULL则将其加到哈希总和中
}
//从元组中获取hash_del列的值
value = heap_getattr(tuple, USERCHAIN_COLUMN_HASH_DEL + 1, hist_rel->rd_att, &is_null);
if (!is_null) {
rel_hash -= DatumGetUInt64(value);
rel_hash -= DatumGetUInt64(value);//如果列值不为NULL则将其从哈希总和中减去
}
}
heap_endscan(scan);
@ -148,17 +155,20 @@ static uint64 get_histtable_hash_sum(Oid hist_oid)
}
/*
* has_ledger_consistent_privilege -- calculate sum(hash_ins) - sum(hash_del) of the history relation of specified oid
* has_ledger_consistent_privilege -- calculate sum(hash_ins) - sum(hash_del) of the history relation of specified oid//计算指定OID的历史关系的sum(hash_ins) - sum(hash_del)
*
* relid: history table oid
* relid: history table oid//历史表的OID
*/
//验证当前用户是否具有对指定命名空间和关系表的权限.如果用户缺少任何一个权限,则认为其不具备一致权限
static bool has_ledger_consistent_privilege(Oid relid, Oid namespaceId)
{
//检查当前用户是否具有对命名空间的ACL_USAGE权限
if (pg_namespace_aclcheck(namespaceId, GetUserId(), ACL_USAGE) != ACLCHECK_OK) {
return false;
return false;//没有权限
}
//检查当前用户是否具有对关系表的ACL_SELECT权限
if (pg_class_aclcheck(relid, GetUserId(), ACL_SELECT) != ACLCHECK_OK) {
return false;
return false;//没有权限
}
return true;
}
@ -169,28 +179,34 @@ static bool has_ledger_consistent_privilege(Oid relid, Oid namespaceId)
* relid: user table oid
* res_hash: hash sum of history table
*/
//检查用户表的哈希总和和历史表的哈希总和是否相等
bool is_hist_hash_identity(Oid relid, uint64 *res_hash)
{
uint64 user_hash_sum;
uint64 hist_hash_sum;
char hist_name[NAMEDATALEN];
char hist_name[NAMEDATALEN];//存储历史表的名称
char *rel_name = get_rel_name(relid);
//获取历史表的名称并存储到hist_name数组中
if (!get_hist_name(relid, rel_name, hist_name)) {
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("get hist table name failed.")));
}
//根据历史表名称和命名空间获取历史表的对象标识符
Oid histoid = get_relname_relid(hist_name, PG_BLOCKCHAIN_NAMESPACE);
if (!OidIsValid(histoid)) {
if (!OidIsValid(histoid)) {//判断获取到的历史表的对象标识符是否无效(即未找到对应的历史表)
//抛出一个错误,找不到历史表
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("could not find hist table of \"%s\".", rel_name)));
}
//获取用户表的哈希总和和历史表的哈希总和
user_hash_sum = get_usertable_hash_sum(relid);
hist_hash_sum = get_histtable_hash_sum(histoid);
*res_hash = hist_hash_sum;
return user_hash_sum == hist_hash_sum;
*res_hash = hist_hash_sum;//将历史表的哈希总和赋值给结果哈希总和指针
return user_hash_sum == hist_hash_sum;//返回用户表哈希总和和历史表哈希总和是否相等的结果
}
//#ifdef ENABLE_MULTIPLE_NODES 和 #endif 是条件编译预处理指令,用于在编译代码时根据定义的宏来选择性地包含或排除一些代码片段
#ifdef ENABLE_MULTIPLE_NODES
//定义了两个静态函数 StrategyFuncAnd 和 StrategyFuncUInt64Sum
//这些函数用于在并行计算的情况下收集远程节点的计算结果
/*
* StrategyFuncAnd -- collect remote nodes with AND
*
@ -202,27 +218,27 @@ bool is_hist_hash_identity(Oid relid, uint64 *res_hash)
static void StrategyFuncAnd(ParallelFunctionState* state)
{
TupleTableSlot* slot = NULL;
bool result = true;
bool result = true;//初始化结果为真
Assert(state);
Assert(state->tupstore);
Assert(state->tupdesc);
slot = MakeSingleTupleTableSlot(state->tupdesc);
slot = MakeSingleTupleTableSlot(state->tupdesc);//创建一个单个元组的插槽
while (true) {
bool isnull = false;
bool isnull = false;//判断属性是否为NULL
if (!tuplestore_gettupleslot(state->tupstore, true, false, slot))
if (!tuplestore_gettupleslot(state->tupstore, true, false, slot))//从元组存储中获取下一个元组
break;
if (!DatumGetBool(tableam_tslot_getattr(slot, 1, &isnull))) {
result = false;
break;
if (!DatumGetBool(tableam_tslot_getattr(slot, 1, &isnull))) {//获取元组的第一个属性值并判断是否为假
result = false;//如果有一个节点的结果为假,则将结果设置为假
break;//跳出循环
}
(void)ExecClearTuple(slot);
(void)ExecClearTuple(slot);//清空插槽中的元组
}
state->result = result;
state->result = result;//将最终的结果赋值给函数状态的 result 属性
}
/*
@ -239,154 +255,159 @@ static void StrategyFuncUInt64Sum(ParallelFunctionState* state)
int64 result = 0;
Assert(state && state->tupstore && state->tupdesc);
slot = MakeSingleTupleTableSlot(state->tupdesc);
slot = MakeSingleTupleTableSlot(state->tupdesc);//创建一个单个元组的插槽
while (true) {
bool isnull = false;
if (!tuplestore_gettupleslot(state->tupstore, true, false, slot))
if (!tuplestore_gettupleslot(state->tupstore, true, false, slot))//从元组存储中获取下一个元组
break;
result += DatumGetUInt64(tableam_tslot_getattr(slot, 1, &isnull));
ExecClearTuple(slot);
result += DatumGetUInt64(tableam_tslot_getattr(slot, 1, &isnull));//将元组的第一个属性值累加到结果上
ExecClearTuple(slot);//清空插槽中的元组
}
state->result = result;
state->result = result;//将最终的结果赋值给函数状态的 result 属性
}
#endif
/*
* ledger_hist_check -- check whether user table hash and history table hash are equal
* ledger_hist_check -- check whether user table hash and history table hash are equal//检查用户表的哈希总和和历史表的哈希总和是否相等
*
* parameter1: user table name [type: text]
* parameter2: namespace of user table [type: text]
* parameter1: user table name [type: text]//用户表名称
* parameter2: namespace of user table [type: text]//用户表的命名空间
*/
Datum ledger_hist_check(PG_FUNCTION_ARGS)
{
//定义变量
Oid relid;
Oid nsp_oid;
uint64 res_hash;
bool res = false;
char *table_name;
char *table_nsp;
text *rel_nsp = PG_GETARG_TEXT_PP(0);
text *rel_name = PG_GETARG_TEXT_PP(1);
text *rel_nsp = PG_GETARG_TEXT_PP(0);//获取传入的命名空间参数
text *rel_name = PG_GETARG_TEXT_PP(1);//获取传入的用户表名参数
table_nsp = text_to_cstring(rel_nsp);
table_name = text_to_cstring(rel_name);
nsp_oid = get_namespace_oid(table_nsp, false);
relid = get_relname_relid(table_name, nsp_oid);
ledger_usertable_check(relid, nsp_oid, table_name, table_nsp);
table_nsp = text_to_cstring(rel_nsp);//将命名空间参数转换为C字符串
table_name = text_to_cstring(rel_name);//将用户表名参数转换为C字符串
nsp_oid = get_namespace_oid(table_nsp, false);//获取命名空间的OID
relid = get_relname_relid(table_name, nsp_oid);//根据用户表名和命名空间的OID获取用户表的OID
ledger_usertable_check(relid, nsp_oid, table_name, table_nsp);//检查用户表的存在性和合法性
if (!has_ledger_consistent_privilege(relid, nsp_oid)) {
if (!has_ledger_consistent_privilege(relid, nsp_oid)) {//检查当前用户对用户表是否有足够的权限
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));
}
res = is_hist_hash_identity(relid, &res_hash);
res = is_hist_hash_identity(relid, &res_hash);//检查用户表的哈希值和历史表的哈希值是否相等
#ifdef ENABLE_MULTIPLE_NODES
if (!IsConnFromCoord()) {
if (!IsConnFromCoord()) {//非协调节点,则进行并行计算
StringInfoData buf;
ParallelFunctionState* state = NULL;
//初始化字符串缓冲区
initStringInfo(&buf);
appendStringInfo(&buf, "SELECT pg_catalog.ledger_hist_check('%s', '%s')", table_nsp, table_name);
/* Get all hash diffs from DNs in distribute scenairo. */
state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncAnd);
res &= state->result;
FreeParallelFunctionState(state);
state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncAnd);//调用远程函数并处理结果
res &= state->result;//将远程节点的结果与本地结果进行逻辑与操作
FreeParallelFunctionState(state);//释放并行函数状态的内存
}
#endif
return BoolGetDatum(res);
}
/*
* get_gchain_relhash_sum -- calculate relhash from gs_global_chain
* get_gchain_relhash_sum -- calculate relhash from gs_global_chain//从gs_global_chain计算relid对应的用户表哈希总和
*
* relid: user table oid
* relid: user table oid//用户表的对象标识符
*/
static uint64 get_gchain_relhash_sum(Oid relid)
{
uint64 relhash = 0;
HeapTuple tuple = NULL;
uint64 relhash = 0;//用户表哈希总和的初始值
HeapTuple tuple = NULL;//堆元组指针用于遍历gs_global_chain目录
/* scan the gs_global_chain catalog by relid */
//打开gs_global_chain目录
Relation gchain_rel = heap_open(GsGlobalChainRelationId, AccessShareLock);
Form_gs_global_chain rdata = NULL;
Form_gs_global_chain rdata = NULL;//gs_global_chain目录中的元组数据结构体指针
//扫描gs_global_chain目录
TableScanDesc scan = heap_beginscan(gchain_rel, SnapshotNow, 0, NULL);
while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) {
rdata = (Form_gs_global_chain)GETSTRUCT(tuple);
rdata = (Form_gs_global_chain)GETSTRUCT(tuple);//获取当前遍历的元组数据
if (rdata == NULL || rdata->relid != relid) {
continue;
continue;//若元组数据为空或relid不匹配则跳过继续下一次遍历
}
relhash += rdata->relhash;
relhash += rdata->relhash;//累加当前匹配的relid的哈希值到用户表哈希总和
}
heap_endscan(scan);
heap_close(gchain_rel, AccessShareLock);
return relhash;
heap_endscan(scan);//扫描结束
heap_close(gchain_rel, AccessShareLock);//关闭gs_global_chain目录
return relhash;//返回用户表哈希总和
}
/*
* get_dn_hist_relhash -- calculate relhash from gs_global_chain or history table
* get_dn_hist_relhash -- calculate relhash from gs_global_chain or history table//从gs_global_chain或history表中计算relhash
*
* parameter1: user table name [type: text]
* parameter2: namespace of user table [type: text]
* parameter1: user table name [type: text]//用户表名称
* parameter2: namespace of user table [type: text]//用户表的命名空间
*
* NOTICE: if current node is cn, it's return the relhash calculated from gs_global_chain,
* if current node is dn, it's return the relhash calculated from history table.
* cngs_global_chain计算得到的relhashdnhistory表计算得到的relhash
*/
Datum get_dn_hist_relhash(PG_FUNCTION_ARGS)
{
#ifndef ENABLE_MULTIPLE_NODES
DISTRIBUTED_FEATURE_NOT_SUPPORTED();
return UInt64GetDatum(0);
#ifndef ENABLE_MULTIPLE_NODES//如果不支持多节点部署那么调用该函数将会立即返回一个哈希总和为0的结果而不会继续执行后续的代码逻辑
DISTRIBUTED_FEATURE_NOT_SUPPORTED();//宏,用于抛出一个错误,表示不支持分布式特性
return UInt64GetDatum(0);//返回一个表示哈希总和为0的结果
#else
if (!IsConnFromCoord()) {
if (!IsConnFromCoord()) {//如果当前连接不是协调器连接
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));
return UInt64GetDatum(0);
}
Oid user_relid;
Oid nsp_oid;
uint64 res_hash;
text *rel_nsp = PG_GETARG_TEXT_PP(0);
text *rel_name = PG_GETARG_TEXT_PP(1);
text *rel_nsp = PG_GETARG_TEXT_PP(0);//获取命名空间名称
text *rel_name = PG_GETARG_TEXT_PP(1);//获取表名
char *table_name;
char *table_nsp;
table_nsp = text_to_cstring(rel_nsp);
table_name = text_to_cstring(rel_name);
nsp_oid = get_namespace_oid(table_nsp, false);
user_relid = get_relname_relid(table_name, nsp_oid);
ledger_usertable_check(user_relid, nsp_oid, table_name, table_nsp);
table_nsp = text_to_cstring(rel_nsp);//将命名空间转换为C字符串
table_name = text_to_cstring(rel_name);//将表名转换为C字符串
nsp_oid = get_namespace_oid(table_nsp, false);//获取命名空间的Oid
user_relid = get_relname_relid(table_name, nsp_oid);//获取用户表的Oid
ledger_usertable_check(user_relid, nsp_oid, table_name, table_nsp);//检查用户表的合法性
if (!has_ledger_consistent_privilege(user_relid, nsp_oid)) {
if (!has_ledger_consistent_privilege(user_relid, nsp_oid)) {//没有一致性账本特权
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));
}
if (IS_PGXC_DATANODE) {
if (!is_hist_hash_identity(user_relid, &res_hash)) {
if (IS_PGXC_DATANODE) {//如果当前节点是数据节点
if (!is_hist_hash_identity(user_relid, &res_hash)) {//判断用户表是否具有历史哈希标识
res_hash = 0;
}
} else {
res_hash = get_gchain_relhash_sum(user_relid);
} else {//如果当前节点是协调器节点
res_hash = get_gchain_relhash_sum(user_relid);//从gs_global_chain目录中计算用户表的哈希总和
}
return UInt64GetDatum(res_hash);
return UInt64GetDatum(res_hash);//返回res_hash的值
#endif
}
/*
* ledger_gchain_check -- calculate relation hash from all cn and all dn, and return cn_hash == dn_hash
* ledger_gchain_check -- calculate relation hash from all cn and all dn, and return cn_hash == dn_hash//从所有的协调器节点和所有的数据节点计算关系哈希,并返回协调器节点的哈希值是否等于数据节点的哈希值
*
* parameter1: user table name [type: text]
* parameter2: namespace of user table [type: text]
* parameter1: user table name [type: text]//用户表名
* parameter2: namespace of user table [type: text]//用户表所属命名空间
*/
Datum ledger_gchain_check(PG_FUNCTION_ARGS)
{
#ifdef ENABLE_MULTIPLE_NODES
if (!IS_PGXC_COORDINATOR || IsConnFromCoord()) {
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));
if (!IS_PGXC_COORDINATOR || IsConnFromCoord()) {//如果不是协调器或者是协调器连接
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));//抛出权限不足的错误
}
#endif
text *rel_nsp = PG_GETARG_TEXT_PP(0);
text *rel_name = PG_GETARG_TEXT_PP(1);
text *rel_nsp = PG_GETARG_TEXT_PP(0);//获取命名空间名称
text *rel_name = PG_GETARG_TEXT_PP(1);//获取表名
char *table_name;
char *table_nsp;
Oid user_relid;
@ -395,80 +416,94 @@ Datum ledger_gchain_check(PG_FUNCTION_ARGS)
uint64 dn_hash;
bool res;
table_nsp = text_to_cstring(rel_nsp);
table_name = text_to_cstring(rel_name);
nsp_oid = get_namespace_oid(table_nsp, false);
user_relid = get_relname_relid(table_name, nsp_oid);
ledger_usertable_check(user_relid, nsp_oid, table_name, table_nsp);
table_nsp = text_to_cstring(rel_nsp);//将命名空间转换为C字符串
table_name = text_to_cstring(rel_name);//将表名转换为C字符串
nsp_oid = get_namespace_oid(table_nsp, false);//获取命名空间的Oid
user_relid = get_relname_relid(table_name, nsp_oid);//获取用户表的Oid
ledger_usertable_check(user_relid, nsp_oid, table_name, table_nsp);//检查用户表的合法性
if (!has_ledger_consistent_privilege(user_relid, nsp_oid)) {
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));
if (!has_ledger_consistent_privilege(user_relid, nsp_oid)) {//没有一致性账本特权
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));//抛出权限不足的错误
}
res = is_hist_hash_identity(user_relid, &dn_hash);
res = is_hist_hash_identity(user_relid, &dn_hash);//判断用户表是否具有历史哈希标识
if (!res) {
return BoolGetDatum(res);
return BoolGetDatum(res);//如果没有历史哈希标识返回false
}
cn_hash = get_gchain_relhash_sum(user_relid);
cn_hash = get_gchain_relhash_sum(user_relid);//从gs_global_chain目录中计算用户表的哈希总和
#ifdef ENABLE_MULTIPLE_NODES
ParallelFunctionState* state = NULL;
StringInfoData buf;
initStringInfo(&buf);
//将格式化后的字符串 "SELECT pg_catalog.get_dn_hist_relhash('%s', '%s')" 和 table_nsp、table_name 的值追加到 buf 中
appendStringInfo(&buf, "SELECT pg_catalog.get_dn_hist_relhash('%s', '%s')", table_nsp, table_name);
/* Get all hash diffs from DNs in distribute scenairo. */
/* 在分布式场景中,从所有的数据节点获取哈希差异。 */
// 调用RemoteFunctionResultHandler函数传入buf.dataSQL查询语句、NULL没有附加参数和StrategyFuncUInt64Sum累加函数来获取数据节点的哈希差异并将结果赋给state->result
state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncUInt64Sum);
dn_hash += state->result;
FreeParallelFunctionState(state);
if (GetAllCoordNodes() != NIL) {
dn_hash += state->result;//将state->result数据节点的哈希差异累加到数据节点的哈希值变量dn_hash
FreeParallelFunctionState(state);//释放内存回收state占用的资源
if (GetAllCoordNodes() != NIL) {//如果存在协调器节点,则执行下面的代码块
/* Get and accumulate all cnhash from all CNs. */
/* 获取并累加所有协调器节点的哈希值。 */
state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncUInt64Sum, true, EXEC_ON_COORDS);
cn_hash += state->result;
cn_hash += state->result;//将state->result协调器节点的哈希值累加到协调器节点的哈希值变量cn_hash中
FreeParallelFunctionState(state);
}
#endif
return BoolGetDatum(dn_hash == cn_hash);
return BoolGetDatum(dn_hash == cn_hash);//返回协调器节点的哈希值是否等于数据节点的哈希值
}
/*
* repaire_hist_table_internal -- compare hash and repair hist table
* repaire_hist_table_internal -- compare hash and repair hist table//比较哈希值并修复历史表
*
* relid: user table oid
* rel_name: user table name
* nspoid: user table namespace oid
* option: choose return tuple_hash_sum if true, or hash_diff if false.
* relid: user table oid//用户表的对象标识符
* rel_name: user table name//用户表的名称
* nspoid: user table namespace oid//用户表的命名空间对象标识符
* option: choose return tuple_hash_sum if true, or hash_diff if false.//如果为true则返回tuple_hash_sum如果为false则返回hash_diff
*
* Note: the diff means tuple_hash_sum - hist_hash_sum.
* Note: the diff means tuple_hash_sum - hist_hash_sum.//注意diff表示tuple_hash_sum - hist_hash_sum
*/
static uint64 repaire_hist_table_internal(Oid relid, char *rel_name, Oid nspoid, bool option)
{
//声明并初始化变量
uint64 rel_hash;
uint64 hash_diff;
char histname[NAMEDATALEN];
//获取历史表的名称
get_hist_name(relid, rel_name, histname, nspoid);
//获取历史表的对象标识符OID
Oid histoid = get_relname_relid(histname, PG_BLOCKCHAIN_NAMESPACE);
if (!OidIsValid(histoid)) {
//如果历史表的OID无效则报错
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("The hist table of \"%s\" is not exist.", rel_name)));
}
//获取用户表的哈希值
rel_hash = get_usertable_hash_sum(relid);
//计算哈希差异diff = tuple_hash_sum - hist_hash_sum
hash_diff = rel_hash - get_histtable_hash_sum(histoid);
if (hash_diff != 0) {
/* Do hist table repair. */
/* 进行历史表修复 */
hist_table_record_internal(histoid, &hash_diff, NULL);
}
//根据选项返回结果如果option为true返回用户表的哈希值如果option为false返回哈希差异。
return option ? rel_hash : hash_diff;
}
/*
* ledger_hist_repair -- repair the history table of specified user table
* ledger_hist_repair -- repair the history table of specified user table//修复指定用户表的历史表
*
* parameter1: user table name [type: text]
* parameter2: namespace of user table [type: text]
* parameter1: user table name [type: text]//用户表名称
* parameter2: namespace of user table [type: text]//用户表所在的命名空间
*/
Datum ledger_hist_repair(PG_FUNCTION_ARGS)
{
//检查当前用户是否具有足够的权限
if (!isRelSuperuser() && !isAuditadmin(GetUserId())) {
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));
}
//获取函数参数:用户表所在的命名空间和名称
text *rel_nsp = PG_GETARG_TEXT_PP(0);
text *rel_name = PG_GETARG_TEXT_PP(1);
char *table_name;
@ -476,26 +511,31 @@ Datum ledger_hist_repair(PG_FUNCTION_ARGS)
Oid relid;
Oid nspoid;
uint64 delta = 0;
//将参数转换为C字符串
table_nsp = text_to_cstring(rel_nsp);
table_name = text_to_cstring(rel_name);
//获取命名空间的 OID
nspoid = get_namespace_oid(table_nsp, false);
//获取用户表的对象标识符OID
relid = get_relname_relid(table_name, nspoid);
//检查用户表是否存在,并且当前用户是否具有足够的权限
ledger_usertable_check(relid, nspoid, table_name, table_nsp);
/*
* Repair hist table of current datanode. Get hash sum of hist
* table and rel_hash of usertable, append the difference to hist table.
* table and rel_hash of usertable, append the difference to hist table.//修复当前数据节点的历史表。获取历史表的哈希值和用户表的哈希值,将差异追加到历史表中。
*/
//当前节点的角色是数据节点或者单点模式节点
if (g_instance.role == VDATANODE || g_instance.role == VSINGLENODE) {
delta = repaire_hist_table_internal(relid, table_name, nspoid, false);
}
//如果当前节点是协调器或单点模式
if (g_instance.role == VCOORDINATOR || g_instance.role == VSINGLENODE) {
#ifdef ENABLE_MULTIPLE_NODES
ParallelFunctionState* state = NULL;
StringInfoData buf;
initStringInfo(&buf);
//生成并执行用于获取历史表差异的查询语句
appendStringInfo(&buf, "SELECT pg_catalog.ledger_hist_repair('%s', '%s')", table_nsp, table_name);
/* Get all hash diffs from DNs in distribute scenairo. */
state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncUInt64Sum);
@ -503,25 +543,28 @@ Datum ledger_hist_repair(PG_FUNCTION_ARGS)
FreeParallelFunctionState(state);
#endif
//如果差异不为零,则将修复后的差异追加到全局链上
if (delta != 0) {
ledger_gchain_append(relid, "HIST REPAIR.", delta);
}
}
//返回修复的差异值
return UInt64GetDatum(delta);
}
/*
* ledger_gchain_repair -- repair gs_global_chain of specified user table
* ledger_gchain_repair -- repair gs_global_chain of specified user table//修复指定用户表的gs_global_chain
*
* parameter1: user table name [type: text]
* parameter2: namespace of user table [type: text]
* parameter1: user table name [type: text]//用户表名称
* parameter2: namespace of user table [type: text]//用户表所在的命名空间
*/
Datum ledger_gchain_repair(PG_FUNCTION_ARGS)
{
//检查当前用户是否具有足够的权限
if (!isRelSuperuser() && !isAuditadmin(GetUserId())) {
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));
}
//获取函数参数:用户表所在的命名空间和名称
text *rel_nsp = PG_GETARG_TEXT_PP(0);
text *rel_name = PG_GETARG_TEXT_PP(1);
char *table_name;
@ -529,27 +572,35 @@ Datum ledger_gchain_repair(PG_FUNCTION_ARGS)
Oid relid;
Oid nspoid;
uint64 dn_hash = 0;
//将参数转换为C字符串
table_nsp = text_to_cstring(rel_nsp);
table_name = text_to_cstring(rel_name);
//获取命名空间的 OID
nspoid = get_namespace_oid(table_nsp, false);
//获取用户表的对象标识符OID
relid = get_relname_relid(table_name, nspoid);
//检查用户表是否存在,并且当前用户是否具有足够的权限
ledger_usertable_check(relid, nspoid, table_name, table_nsp);
/*
* Repair hist table of current datanode. Get hash sum of hist
* table and rel_hash of usertable, append the difference to hist table.
*/
/*
*
*/
//如果当前节点的角色是数据节点或者单点模式节点则执行修复历史表的操作并将修复后的历史表哈希值赋给变量dn_hash
if (g_instance.role == VDATANODE || g_instance.role == VSINGLENODE) {
dn_hash = repaire_hist_table_internal(relid, table_name, nspoid, true);
}
uint64 rel_hash = dn_hash;
uint64 cn_hash = 0;
//如果当前节点是协调器或单点模式
if (g_instance.role == VCOORDINATOR || g_instance.role == VSINGLENODE) {
uint64 delta = 0;
cn_hash = get_gchain_relhash_sum(relid);
if (!IsConnFromCoord()) {
cn_hash = get_gchain_relhash_sum(relid);//获取当前节点的全局链哈希值并将结果保存在变量 cn_hash 中
if (!IsConnFromCoord()) {//判断当前连接是否来自协调器节点
#ifdef ENABLE_MULTIPLE_NODES
/*
* CN accumulate all gchain cn_hash from all CNs, get all dn_hash from all DNs.
@ -558,25 +609,30 @@ Datum ledger_gchain_repair(PG_FUNCTION_ARGS)
ParallelFunctionState* state = NULL;
StringInfoData buf;
initStringInfo(&buf);
//生成并执行用于获取历史表差异的查询语句
appendStringInfo(&buf, "SELECT pg_catalog.ledger_gchain_repair('%s', '%s')", table_nsp, table_name);
/* Get and accumulate all dn_hash from all DNs. */
//获取并累加所有数据节点中的历史表哈希值
state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncUInt64Sum);
dn_hash += state->result;
FreeParallelFunctionState(state);
if (GetAllCoordNodes() != NIL) {
if (GetAllCoordNodes() != NIL) {//如果存在其他的协调器节点
/* Get and accumulate all cn_hash from all CNs. */
//获取并累加所有协调器节点中的全局链哈希值
state = RemoteFunctionResultHandler(buf.data, NULL, StrategyFuncUInt64Sum, true, EXEC_ON_COORDS);
cn_hash += state->result;
FreeParallelFunctionState(state);
}
#endif
//计算差异值
delta = dn_hash - cn_hash;
//如果差异不为零,则将修复后的差异追加到全局链上
if (delta != 0) {
ledger_gchain_append(relid, "GCHAIN REPAIR.", delta);
}
}
rel_hash = cn_hash;
}
//返回修复后的用户表哈希值
return UInt64GetDatum(rel_hash);
}

View File

@ -25,129 +25,143 @@
#include "gs_ledger/ledger_utils.h"
#include "catalog/gs_global_chain.h"
static pg_atomic_uint64 g_blocknum = 0;
static HTAB *g_recnum_cache = NULL;
static pg_atomic_uint64 g_blocknum = 0;//记录区块的编号
static HTAB *g_recnum_cache = NULL;//指向哈希表的指针,用于缓存记录编号
/*
* reload_next_g_blocknum -- load next blocknum from gchain.
* reload_next_g_blocknum -- load next blocknum from gchain.//加载下一个区块编号blocknum的函数
*
* Note:If gchain is empty, next blocknum will start from 0.
* Note:If gchain is empty, next blocknum will start from 0.//如果全局链gchain为空则下一个区块编号将从0开始
*/
static uint32 reload_next_g_blocknum()
{
Relation gchain_rel = NULL;
HeapTuple tup = NULL;
TableScanDesc scan;
uint32 blocknum;
uint32 max_num = 0;
bool isnull = false;
Relation gchain_rel = NULL;//全局链关系指针
HeapTuple tup = NULL;//堆元组指针,用于遍历全局链关系中的元组
TableScanDesc scan;//表扫描描述符
uint32 blocknum;//当前堆元组表示的区块编号
uint32 max_num = 0;//最大的区块编号
bool isnull = false;//是否为空
gchain_rel = heap_open(GsGlobalChainRelationId, RowExclusiveLock);
scan = heap_beginscan(gchain_rel, SnapshotAny, 0, NULL);
while ((tup = heap_getnext(scan, BackwardScanDirection)) != NULL) {
gchain_rel = heap_open(GsGlobalChainRelationId, RowExclusiveLock);//打开全局链关系
scan = heap_beginscan(gchain_rel, SnapshotAny, 0, NULL);//开始对全局链关系进行表扫描
while ((tup = heap_getnext(scan, BackwardScanDirection)) != NULL) {//循环遍历全局链关系中的堆元组
blocknum = DatumGetUInt32(heap_getattr(tup, Anum_gs_global_chain_blocknum,
RelationGetDescr(gchain_rel), &isnull));
if (blocknum > max_num) {
max_num = blocknum;
RelationGetDescr(gchain_rel), &isnull));//获取堆元组中的区块编号属性值
if (blocknum > max_num) {//如果当前区块编号大于最大区块编号
max_num = blocknum;//更新最大区块编号
} else {
break;
break;//否则跳出循环
}
}
heap_endscan(scan);
heap_close(gchain_rel, RowExclusiveLock);
return max_num;
heap_endscan(scan); //结束表扫描
heap_close(gchain_rel, RowExclusiveLock);//关闭全局链关系
return max_num;//返回最大的区块编号
}
/*
* get_next_g_blocknum -- get next blocknum for gchain record.
* get_next_g_blocknum -- get next blocknum for gchain record.//获取全局链记录的下一个区块编号
*
* Note:provide next blocknum and auto increment itself.
* Note:provide next blocknum and auto increment itself.//获取下一个区块编号,并自动递增
*/
uint64 get_next_g_blocknum()
{
uint64 res = 0;
if (g_blocknum == 0) {
LWLockAcquire(GlobalPrevHashLock, LW_EXCLUSIVE);
if (g_blocknum == 0) {
pg_atomic_fetch_add_u64(&g_blocknum, 1);
int cur_num = reload_next_g_blocknum();
pg_atomic_fetch_add_u64(&g_blocknum, cur_num);
uint64 res = 0;//存储结果的变量
if (g_blocknum == 0) {//如果当前区块编号为0
LWLockAcquire(GlobalPrevHashLock, LW_EXCLUSIVE);//获取全局前一个哈希锁(独占模式)
if (g_blocknum == 0) {//双重检查,确保其他线程未更新区块编号
pg_atomic_fetch_add_u64(&g_blocknum, 1);//原子操作将区块编号加1
int cur_num = reload_next_g_blocknum();//获取当前最大的区块编号
pg_atomic_fetch_add_u64(&g_blocknum, cur_num);//原子操作:将当前最大区块编号加到全局区块编号上
}
LWLockRelease(GlobalPrevHashLock);
LWLockRelease(GlobalPrevHashLock);//释放全局前一个哈希锁
}
LWLockAcquire(GlobalPrevHashLock, LW_SHARED);
res = pg_atomic_fetch_add_u64(&g_blocknum, 1);
LWLockRelease(GlobalPrevHashLock);
return res;
LWLockAcquire(GlobalPrevHashLock, LW_SHARED);//获取全局前一个哈希锁(共享模式)
res = pg_atomic_fetch_add_u64(&g_blocknum, 1);//原子操作将区块编号加1
LWLockRelease(GlobalPrevHashLock);//释放全局前一个哈希锁
return res;//返回获取的区块编号
}
//将全局区块编号g_blocknum重置为0
void reset_g_blocknum()
{
g_blocknum = 0;
}
/*
* reload_g_rec_num -- load next rec_num from hist table.
* reload_g_rec_num -- load next rec_num from hist table.//从hist表中加载下一个rec_num
*
* histoid: hist table oid.
* histoid: hist table oid.//hist表的对象标识符
*
* Note:return next rec_num and auto increment.
* Note:return next rec_num and auto increment.//返回下一个rec_num的值和自增量
*/
uint64 reload_g_rec_num(Oid histoid)
{
//检查histoid的有效性
if (!OidIsValid(histoid)) {
return false;
}
Relation histRelation = NULL;
HeapTuple tup = NULL;
TableScanDesc scan;
uint64 max_rec_num = 0;
uint64 rec_num;
bool hist_empty = true;
bool isnull = false;
bool found;
Relation histRelation = NULL;//hist表的关系对象
HeapTuple tup = NULL;//堆元组
TableScanDesc scan;//表扫描描述符
uint64 max_rec_num = 0;//最大的rec_num值
uint64 rec_num;//当前的rec_num值
bool hist_empty = true;//hist表是否为空
bool isnull = false;//判断属性是否为NULL
bool found;//是否在哈希表中找到项
//打开hist表
histRelation = heap_open(histoid, AccessShareLock);
//开始扫描hist表
scan = heap_beginscan(histRelation, SnapshotNow, 0, NULL);
while ((tup = heap_getnext(scan, BackwardScanDirection)) != NULL) {
//获取堆元组中的rec_num属性值
rec_num = DatumGetUInt64(heap_getattr(tup, 1, RelationGetDescr(histRelation), &isnull));
//如果当前rec_num大于等于最大的rec_num则更新最大的rec_num值
if (rec_num >= max_rec_num) {
max_rec_num = rec_num;
hist_empty = false;
}
}
//结束扫描hist表
heap_endscan(scan);
//关闭hist表
heap_close(histRelation, AccessShareLock);
//在g_recnum_cache哈希表中查找或插入histoid对应的RecNumItem
RecNumItem *item = (RecNumItem *)hash_search(g_recnum_cache, &histoid, HASH_ENTER, &found);
//如果hist表为空则rec_num设为0否则设置为最大rec_num+1
if (hist_empty) {
rec_num = 0;
} else {
rec_num = max_rec_num + 1;
}
//更新哈希表中rec_num的值并返回rec_num
item->rec_num = rec_num + 1;
return rec_num;
}
/*
* get_next_recnum -- provide next rec_num.
* get_next_recnum -- provide next rec_num.//提供下一个rec_num值
*
* histoid: hist table oid.
*/
uint64 get_next_recnum(Oid histoid)
{
//检查全局recnum缓存g_recnum_cache是否已创建如果未创建则进行初始化
if (g_recnum_cache == NULL) {
errno_t rc;
HASHCTL ctl;
LWLockAcquire(BlockchainVersionLock, LW_EXCLUSIVE);
/* To avoid multiple threads double create hash table. */
//双重检查锁机制再次检查g_recnum_cache是否为NULL
if (g_recnum_cache == NULL) {
//初始化哈希表的配置项
rc = memset_s(&ctl, sizeof(ctl), 0, sizeof(ctl));
securec_check(rc, "\0", "\0");
ctl.keysize = sizeof(Oid);
ctl.entrysize = sizeof(RecNumItem);
ctl.hash = oid_hash;
ctl.hcxt = INSTANCE_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY);
//创建哈希表g_recnum_cache
g_recnum_cache = hash_create("global recnum cache", 256, &ctl, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT);
//如果创建失败,则报错并抛出异常
if (g_recnum_cache == NULL) {
ereport(ERROR, (errcode(ERRCODE_INITIALIZE_FAILED), errmsg("could not init global recnum cache")));
}
@ -158,27 +172,32 @@ uint64 get_next_recnum(Oid histoid)
bool found = false;
LWLockAcquire(BlockchainVersionLock, LW_SHARED);
uint64 res;
//在g_recnum_cache哈希表中查找histoid对应的RecNumItem
RecNumItem *item = (RecNumItem *)hash_search(g_recnum_cache, &histoid, HASH_FIND, &found);
//如果在哈希表中找到了对应项则将res设置为对应的rec_num值并将rec_num自增1
if (found) {
res = item->rec_num;
pg_atomic_add_fetch_u64(&item->rec_num, 1);
} else {
//如果在哈希表中未找到对应项则需要重新加载hist表并获取下一个rec_num值
LWLockRelease(BlockchainVersionLock);
LWLockAcquire(BlockchainVersionLock, LW_EXCLUSIVE);
res = reload_g_rec_num(histoid);
}
LWLockRelease(BlockchainVersionLock);
return res;
return res;//返回获取到的rec_num值
}
//移除指定histoid对应的缓存项
bool remove_hist_recnum_cache(Oid histoid)
{
//检查histoid是否有效以及全局recnum缓存g_recnum_cache是否已创建
if (!OidIsValid(histoid) || g_recnum_cache == NULL) {
return false;
return false;//如果 istoid无效或者缓存未创建则返回false
}
//在哈希表g_recnum_cache中查找histoid对应的项并将其从哈希表中移除
hash_search(g_recnum_cache, &histoid, HASH_REMOVE, NULL);
return true;
return true;//返回true表示成功移除缓存项
}
/*
@ -186,14 +205,18 @@ bool remove_hist_recnum_cache(Oid histoid)
*
* mode: lockmode
*/
//使用指定的锁模式mode锁定g_blocknum缓存
void lock_gchain_cache(LWLockMode mode)
{
LWLockAcquire(GlobalPrevHashLock, mode);
//GlobalPrevHashLock:用于确保对g_blocknum缓存的访问互斥
//mode:锁的模式。可以是LW_SHARED共享锁允许多个线程同时获取或LW_EXCLUSIVE排他锁只允许一个线程获取
}
/*
* release_gchain_cache -- release g_blocknum cache.
*/
//释放g_blocknum缓存的锁
void release_gchain_cache()
{
LWLockRelease(GlobalPrevHashLock);
@ -202,380 +225,429 @@ void release_gchain_cache()
/*
* lock_hist_hash_cache -- load hist cache.
*/
//加载历史哈希缓存
void lock_hist_hash_cache(LWLockMode mode)
{
LWLockAcquire(BlockchainVersionLock, mode);
//BlockchainVersionLock:用于确保历史哈希缓存的访问互斥
}
/*
* release_hist_hash_cache -- release hist cache.
*/
//释放历史哈希缓存的锁
void release_hist_hash_cache()
{
LWLockRelease(BlockchainVersionLock);
}
/*
* get_target_query_relid -- get result relation oid.
* get_target_query_relid -- get result relation oid.//获取目标关系的OID
*
* rte_list: range table entry list.
* resultRelation: the index of target relation in rte_list.
* rte_list: range table entry list.//范围表条目列表
* resultRelation: the index of target relation in rte_list.//目标关系在rte_list中的索引
*/
Oid get_target_query_relid(List* rte_list, int resultRelation)
{
Oid relid = InvalidOid;
//检查resultRelation是否大于0
if (resultRelation > 0) {
//从rte_list中获取目标关系的表达式对象
RangeTblEntry *rte = (RangeTblEntry *)list_nth(rte_list, resultRelation - 1);
//检查表达式对象的relkind是否为RELKIND_RELATION即是否为普通关系
if (rte->relkind == RELKIND_RELATION) {
//如果是普通关系则获取关系的OID
relid = rte->relid;
}
}
//返回关系的OID
return relid;
}
/*
* is_ledger_usertable -- whether relid is ledger user table.
* is_ledger_usertable -- whether relid is ledger user table.//判断给定的relid是否为账本用户表
*
* relid: usertable oid.
* relid: usertable oid.//用户表的OID
*
* Note: ledger usertable has correct hist table in blockchain schema.
* Or rather, the schema of ledger usertable has WITH BLOCKCHAIN option.
*
* WITH BLOCKCHAIN
*/
bool is_ledger_usertable(Oid relid)
{
//检查relid是否有效如果无效则返回false
if (!OidIsValid(relid)) {
return false;
}
//获取关系所属的命名空间OID
Oid nspid = get_rel_namespace(relid);
//获取关系的relkind即关系类型
char relkind = get_rel_relkind(relid);
/* only table has its user chain table */
//只有表RELKIND_RELATION才有其用户链表
if (relkind != RELKIND_RELATION) {
return false;
}
/* check table belong to blockchain schema */
//检查表是否属于账本模式的命名空间
return IsLedgerNameSpace(nspid);
}
/*
* hash_combiner -- combine all relhash list items.
* hash_combiner -- combine all relhash list items.//合并所有relhash列表项
*
* relhash_list: the list contains several relhash.
* relhash_list: the list contains several relhash.//包含多个relhash的列表
*/
uint64 hash_combiner(List *relhash_list)
{
uint64 relhash_sum = 0;
ListCell *lc = NULL;
if (relhash_list == NIL) {
uint64 relhash_sum = 0;//用于保存累加后的哈希值总和
ListCell *lc = NULL;//遍历链表的指针
if (relhash_list == NIL) {//如果传入的链表为空则直接返回0
return relhash_sum;
}
foreach (lc, relhash_list) {
Datum *value = (Datum *)lfirst(lc);
relhash_sum += DatumGetUInt64(value);
foreach (lc, relhash_list) {//遍历链表中的每个元素
Datum *value = (Datum *)lfirst(lc);//获取当前元素的值
relhash_sum += DatumGetUInt64(value);//将当前值累加到哈希值总和中
}
return relhash_sum;
return relhash_sum;//返回累加后的哈希值
}
/*
* is_ledger_hist_table -- whether relation is ledger hist table.
* is_ledger_hist_table -- whether relation is ledger hist table.//判断关系是否为历史总账表
*
* relid: relation oid.
* relid: relation oid.//关系的对象标识符OID
*/
bool is_ledger_hist_table(Oid relid)
{
if (!OidIsValid(relid) || IsInitdb) {
if (!OidIsValid(relid) || IsInitdb) {//判断传入的关系 OID 是否有效以及是否处在Initdb阶段
return false;
}
Oid relnsp = get_rel_namespace(relid);
char relkind = get_rel_relkind(relid);
Oid relnsp = get_rel_namespace(relid);//获取关系所在的命名空间OID
char relkind = get_rel_relkind(relid);//获取关系的类型
/* check namespace oid of relation to verify hist table. */
/* 检查关系的命名空间 OID并验证其是否为 PG_BLOCKCHAIN_NAMESPACE同时检查关系的类型并验证其是否为
* RELKIND_RELATION*/
return relnsp == PG_BLOCKCHAIN_NAMESPACE && relkind == RELKIND_RELATION;
}
/*
* is_ledger_related_rel -- whether relation is ledger related table.
* is_ledger_related_rel -- whether relation is ledger related table.//判断关系是否为与总账相关的表
*
* rel: table relation.
* rel: table relation.//表关系
*
* Note: ledger related rel means: ledger usertable or ledger hist table or gs_global_chain.
* Note: ledger related rel means: ledger usertable or ledger hist table or gs_global_chain.//注意:与总账相关的表指的是总账用户表、历史总账表或 gs_global_chain 表
*/
bool is_ledger_related_rel(Relation rel)
{
Oid nspoid = RelationGetNamespace(rel);
char relkind = RelationGetRelkind(rel);
bool is_ledger_usertable = rel->rd_isblockchain;
bool is_ledger_histtable = (nspoid == PG_BLOCKCHAIN_NAMESPACE && relkind == RELKIND_RELATION);
bool is_ledger_gchain = (RelationGetRelid(rel) == GsGlobalChainRelationId);
return is_ledger_usertable || is_ledger_histtable || is_ledger_gchain;
Oid nspoid = RelationGetNamespace(rel);//获取关系所在的命名空间OID
char relkind = RelationGetRelkind(rel);//获取关系的类型
bool is_ledger_usertable = rel->rd_isblockchain;//判断关系是否为总账用户表
bool is_ledger_histtable = (nspoid == PG_BLOCKCHAIN_NAMESPACE && relkind == RELKIND_RELATION);//判断关系是否为历史总账表
bool is_ledger_gchain = (RelationGetRelid(rel) == GsGlobalChainRelationId);//判断关系是否为gs_global_chain表
return is_ledger_usertable || is_ledger_histtable ||
is_ledger_gchain; // 只要满足总账用户表、历史总账表或gs_global_chain表中的任意一种情况就返回true
}
/*
* ledger_usertable_check -- check relation is ledger user table.
* ledger_usertable_check -- check relation is ledger user table. //检查关系是否为总账用户表
*/
bool ledger_usertable_check(Oid relid, Oid nspoid, const char *tablename, const char *tablensp)
{
if (!OidIsValid(relid)) {
if (!OidIsValid(relid)) {//如果关系 OID无效则报错
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("table %s.%s not exists.", tablensp, tablename)));
}
if (!IsLedgerNameSpace(nspoid)) {
if (!IsLedgerNameSpace(nspoid)) {//如果关系所在的命名空间不是总账命名空间,则报错
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("table %s.%s is not ledger user table.", tablensp, tablename)));
}
return true;
return true;//否则返回true表示该关系是总账用户表
}
/*
* namespace_get_depended_relid -- get relid under given schema oid.
* namespace_get_depended_relid -- get relid under given schema oid.//获取给定模式OID下的关系OID
*
* nspid: schema oid.
* nspid: schema oid.//模式OID
*/
List *namespace_get_depended_relid(Oid nspid)
{
List *relid_list = NIL;
List *relid_list = NIL;//创建一个空的关系OID列表
Relation pg_class_rel = NULL;
ScanKeyData skey[1];
SysScanDesc sysscan;
HeapTuple tuple;
Oid tupid = InvalidOid;
//初始化扫描键值通过模式OID进行等值匹配查找
ScanKeyInit(&skey[0], Anum_pg_class_relnamespace, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(nspid));
pg_class_rel = heap_open(RelationRelationId, AccessShareLock);
sysscan = systable_beginscan(pg_class_rel, ClassNameNspIndexId, true, SnapshotNow, 1, skey);
while (HeapTupleIsValid(tuple = systable_getnext(sysscan))) {
Form_pg_class reltup = (Form_pg_class)GETSTRUCT(tuple);
if (reltup->relkind == RELKIND_RELATION) {
pg_class_rel = heap_open(RelationRelationId, AccessShareLock);//打开pg_class表
sysscan = systable_beginscan(pg_class_rel, ClassNameNspIndexId, true, SnapshotNow, 1, skey);//开始扫描
//循环遍历扫描结果集
while (HeapTupleIsValid(tuple = systable_getnext(sysscan))) {//获取下一个系统表的元组并检查取得的元组是否有效
Form_pg_class reltup = (Form_pg_class)GETSTRUCT(tuple);//转换为Form_pg_class结构的指针以便访问元组中的字段
if (reltup->relkind == RELKIND_RELATION) {//仅处理关系类型为RELKIND_RELATION的关系
tupid = HeapTupleGetOid(tuple);
relid_list = lappend_oid(relid_list, tupid);
relid_list = lappend_oid(relid_list, tupid);//将关系OID添加到列表中
}
}
systable_endscan(sysscan);
heap_close(pg_class_rel, AccessShareLock);
return relid_list;
systable_endscan(sysscan);//结束扫描
heap_close(pg_class_rel, AccessShareLock);//关闭pg_class表
return relid_list;//返回关系OID列表
}
/*
* get_ledger_msg_hash -- extract relhash from response message.
* get_ledger_msg_hash -- extract relhash from response message.//从响应消息中提取关系哈希值
*
* message: response message.
* hash: buffer to load extracted relhash.
* message: response message.//响应消息
* hash: buffer to load extracted relhash.//用于存储提取的关系哈希值的缓冲区
*/
bool get_ledger_msg_hash(char *message, uint64 *hash, int *msg_len)
{
bool is_from_dml = false;
int hash_offset;
bool is_from_dml = false;//标记消息是否来自数据操作语言DML
int hash_offset;//提取哈希值的偏移量
static hash_offset_pair hash_offset_map[] = {
{"INSERT", 3}, /* INSERT 0 1 HASH */
{"UPDATE", 2}, /* UPDATE 1 HASH */
{"DELETE", 2} /* DELETE 1 HASH */
{"INSERT", 3}, /* INSERT 0 1 HASH */ //插入操作的哈希值在第3个空格后面
{"UPDATE", 2}, /* UPDATE 1 HASH */ //更新操作的哈希值在第2个空格后面
{"DELETE", 2} /* DELETE 1 HASH */ //删除操作的哈希值在第2个空格后面
};
if (message == NULL || strlen(message) == 0) {
if (message == NULL || strlen(message) == 0) {//如果消息为空或长度为0则返回false
return false;
}
/* match space times. */
/* 匹配操作类型 */
for (size_t i = 0; i < lengthof(hash_offset_map); i++) {
if (strncmp(message, hash_offset_map[i].name, 6) == 0) { /* 6: string length of INSERT/UPDATE/DELETE */
hash_offset = hash_offset_map[i].hash_offset;
is_from_dml = true;
if (strncmp(message, hash_offset_map[i].name, 6) == 0) { /* 6: string length of INSERT/UPDATE/DELETE */ // 操作类型匹配成功
hash_offset = hash_offset_map[i].hash_offset;//当前操作类型对应的提取哈希值的偏移量
is_from_dml = true;//表示消息来自数据操作语言DML
break;
}
}
if (is_from_dml) {
if (is_from_dml) {//如果消息来自DML操作
size_t pos;
size_t len = strlen(message);
for (pos = 0; pos < len && hash_offset > 0; ++pos) {
if (message[pos] == ' ') {
if (message[pos] == ' ') {//在消息中查找空格
--hash_offset;
}
}
/* message contains Hash value */
if (hash_offset == 0) {
size_t remain_len = len - pos;
/* 消息包含哈希值 */
if (hash_offset == 0) {//找到了哈希值的位置
size_t remain_len = len - pos;//哈希值后面的剩余长度
if (remain_len > 0) {
*hash = strtoul(message + pos, NULL, 10); /* 10: Decimal */
message[pos - 1] = '\0'; /* remove appended hash string. */
*msg_len = *msg_len - remain_len - 1;
*hash = strtoul(message + pos, NULL, 10); /* 10: Decimal */ //将哈希值转换为无符号长整型
message[pos - 1] = '\0'; /* remove appended hash string. */ //移除追加的哈希字符串
*msg_len = *msg_len - remain_len - 1;//更新消息长度
return true;
}
}
}
return false;
return false;//没有提取到哈希值返回false
}
/*
* get_hist_name -- get hist table name of ledger user table.
* get_hist_name -- get hist table name of ledger user table.//获取账本用户表的历史表名称
*
* relid: user table oid.
* rel_name: user table rel_name.
* hist_name: buffer to load hist table name.
* nsp_oid: schema oid of user table.
* nsp_name: schema name of user table.
* relid: user table oid.//用户表的OID
* rel_name: user table rel_name.//用户表的rel_name
* hist_name: buffer to load hist table name.//用于加载历史表名称的缓冲区
* nsp_oid: schema oid of user table.//用户表的模式OID
* nsp_name: schema name of user table.//用户表的模式名称
*
* Note: relid is used for getting namespace.
* Note: relid is used for getting namespace.//注意relid用于获取命名空间
* This function is also used for generate hist name when
* rename user table, then rel_name is newest user table name.
* Thus, we should not use relid to get user table name.
* rename user table, then rel_name is newest user table name.//当重命名用户表时此函数也用于生成hist名称因此rel_name是最新的用户表名称。
* Thus, we should not use relid to get user table name.//因此我们不应使用relid来获取用户表名称。
*/
bool get_hist_name(Oid relid, const char *rel_name, char *hist_name, Oid nsp_oid, const char *nsp_name)
{
errno_t rc;
if (!OidIsValid(relid) || rel_name == NULL) {
if (!OidIsValid(relid) || rel_name == NULL) {// 检查输入参数的合法性如果reli无效或者rel_name为空则直接返回false
return false;
}
nsp_oid = OidIsValid(nsp_oid) ? nsp_oid : get_rel_namespace(relid);
nsp_name = (nsp_name == NULL) ? get_namespace_name(nsp_oid) : nsp_name;
int part_hist_name_len = strlen(rel_name) + strlen(nsp_name) + 1;
if (part_hist_name_len + strlen("_hist") >= NAMEDATALEN) {
nsp_oid = OidIsValid(nsp_oid) ? nsp_oid : get_rel_namespace(relid);// 如果nsp_oid合法则直接使用否则通过get_rel_namespace函数获取用户表所属的模式OID
nsp_name = (nsp_name == NULL) ? get_namespace_name(nsp_oid) : nsp_name; //如果nsp_name为空则通过get_namespace_name函数获取指定的模式名称否则直接使用输入的nsp_name
int part_hist_name_len = strlen(rel_name) + strlen(nsp_name) + 1;//计算历史表名称的长度长度为rel_name和nsp_name字符串长度之和再加上一个下划线的长度
if (part_hist_name_len + strlen("_hist") >= NAMEDATALEN) {//根据历史表名称的长度判断是否超出NAMEDATALEN的上限如果超出则使用relid和nsp_oid拼接历史表名称
rc = snprintf_s(hist_name, NAMEDATALEN, NAMEDATALEN - 1, "%d_%d_hist", nsp_oid, relid);
securec_check_ss(rc, "", "");
} else {
} else {//否则使用nsp_name和rel_name拼接历史表名称并在最后添加"_hist"字符串
rc = snprintf_s(hist_name, NAMEDATALEN, NAMEDATALEN - 1, "%s_%s_hist", nsp_name, rel_name);
securec_check_ss(rc, "", "");
}
return true;
return true;//返回true表示成功生成历史表名称
}
/*
* querydesc_contains_ledger_usertable -- check querydesc result relation.
* querydesc_contains_ledger_usertable -- check querydesc result relation.//检查querydesc结果关系
*
* Note: check result relation of querydesc is ledger user table.
* Note: check result relation of querydesc is ledger user table.//检查querydesc结果关系是否为账本用户表
*/
bool querydesc_contains_ledger_usertable(QueryDesc *query_desc)
{
//检查输入参数的合法性如果query_desc为空或者query_desc->estate为空则直接返回false
if (query_desc == NULL || query_desc->estate == NULL) {
return false;
}
//获取EState对象的指针
EState *estate = query_desc->estate;
//获取结果关系的数量
int relnum = estate->es_num_result_relations;
//如果结果关系的数量为0或者es_result_relations为空则直接返回false
if (relnum == 0 || estate->es_result_relations == NULL) {
return false;
}
//遍历所有结果关系
for (int i = 0; i < relnum; ++i) {
//如果结果关系的rd_isblockchain字段为true表示该结果关系是账本用户表返回true
if (estate->es_result_relations[i].ri_RelationDesc->rd_isblockchain) {
return true;
}
}
//遍历完所有结果关系后仍未找到账本用户表返回false
return false;
}
/*
* ledger_check_switch_schema -- check two schema has same blockchain option.
* ledger_check_switch_schema -- check two schema has same blockchain option.//检查两个模式是否具有相同的区块链选项
*/
void ledger_check_switch_schema(Oid old_nsp, Oid new_nsp)
{
//检查旧模式和新模式的区块链选项是否不一致
if (IsLedgerNameSpace(old_nsp) != IsLedgerNameSpace(new_nsp)) {
//如果不一致,则报错,不支持在账本模式和普通模式之间切换模式
ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("Unsupport to switch schema of a table between ledger schema and normal schema.")));
}
}
/*
* is_ledger_rowstore -- check withOpt of CreateStmt.
* is_ledger_rowstore -- check withOpt of CreateStmt.//检查CreateStmt中的withOpt
*
* defList: stmt->options.
*
* Note: we will check whether option contains orientation, we only support ORIENTATION ROW.
* orientationORIENTATION ROW
*/
bool is_ledger_rowstore(List *defList)
{
ListCell *lc = NULL;
/* Scan list to see if orientation was ROW store. */
// 遍历defList列表查看是否包含方向orientation为ROW存储。
foreach (lc, defList) {
DefElem* def = (DefElem*)lfirst(lc);
//检查当前def是否是orientation选项并且其值不是ORIENTATION_ROW。
if (pg_strcasecmp(def->defname, "orientation") == 0 &&
pg_strcasecmp(defGetString(def), ORIENTATION_ROW) != 0) {
return false;
}
}
//如果没有找到不支持的方向选项则返回true
return true;
}
//在CreateStmt语句中检查是否指定了哈希存储相关的选项
bool is_ledger_hashbucketstore(List *defList)
{
ListCell *lc = NULL;
/* Scan list to see if hashbucket store. */
//遍历defList列表查看是否包含哈希存储选项。
foreach (lc, defList) {
DefElem* def = (DefElem*)lfirst(lc);
//检查当前def是否是bucketcnt选项或者是hashbucket选项且其值为true。
if (pg_strcasecmp(def->defname, "bucketcnt") == 0 ||
(pg_strcasecmp(def->defname, "hashbucket") == 0 &&
defGetBoolean(def))) {
return true;
}
}
//如果循环结束时没有找到哈希存储选项则返回false。
return false;
}
/*
* check_ledger_attrs_support -- check attrs is ledger supported.
* check_ledger_attrs_support -- check attrs is ledger supported.//检查属性是否受到账本支持
*
* attrs: list of columns.
* attrs: list of columns.//列的列表
*
* Note: because ledger tuple hash is generated by hash_any,so we
* should check whether table column types is supported by hash calculate.
* hash_any生成的
*/
void check_ledger_attrs_support(List *attrs)
{
//检查传入的列属性列表是否为空
if (attrs == NIL) {
return;
}
ListCell *lc = NULL;
//遍历属性列表中的每个列定义
foreach (lc, attrs) {
//获取当前列的定义
ColumnDef *colDef = (ColumnDef*)lfirst(lc);
/* skip the hash column */
/* 跳过哈希列 */
//如果列名为"hash",则跳过该列的检查,因为哈希列是由数据库自动生成的
if (strcmp(colDef->colname, "hash") == 0) {
continue;
}
//获取列的数据类型OID
Oid typid = colDef->typname->typeOid;
//如果数据类型OID无效尝试通过列类型名称获取类型元数据
if (!OidIsValid(typid)) {
Type ctype = typenameType(NULL, colDef->typname, NULL);
if (ctype != NULL) {
//将类型元数据转换为类型OID
typid = typeTypeId(ctype);
ReleaseSysCache(ctype);
}
}
//检查列的数据类型是否受支持
switch (typid) {
case INT8OID:
case INT1OID:
case INT2OID:
case OIDOID:
case INT4OID:
case BOOLOID:
case CHAROID:
case NAMEOID:
case INT2VECTOROID:
case CLOBOID:
case NVARCHAR2OID:
case VARCHAROID:
case TEXTOID:
case OIDVECTOROID:
case FLOAT4OID:
case FLOAT8OID:
case ABSTIMEOID:
case RELTIMEOID:
case CASHOID:
case BPCHAROID:
case RAWOID:
case BYTEAOID:
case BYTEAWITHOUTORDERCOLOID:
case BYTEAWITHOUTORDERWITHEQUALCOLOID:
case INTERVALOID:
case TIMEOID:
case TIMESTAMPOID:
case TIMESTAMPTZOID:
case DATEOID:
case TIMETZOID:
case SMALLDATETIMEOID:
case NUMERICOID:
case UUIDOID:
case INT8OID://8字节整数
case INT1OID://1字节整数
case INT2OID://2字节整数
case OIDOID://对象标识符
case INT4OID://4字节整数
case BOOLOID://布尔类型
case CHAROID://固定长度字符
case NAMEOID://名称类型
case INT2VECTOROID://整数数组
case CLOBOID://可变长度字符
case NVARCHAR2OID://可变长度Unicode字符
case VARCHAROID://可变长度字符
case TEXTOID://文本类型
case OIDVECTOROID://对象标识符数组
case FLOAT4OID://4字节浮点数
case FLOAT8OID://8字节浮点数
case ABSTIMEOID://绝对时间
case RELTIMEOID://相对时间
case CASHOID://货币类型
case BPCHAROID://空格填充的固定长度字符
case RAWOID://二进制数据
case BYTEAOID://字节数组
case BYTEAWITHOUTORDERCOLOID://无序字节数组
case BYTEAWITHOUTORDERWITHEQUALCOLOID://带有相等操作符的无序字节数组
case INTERVALOID://时间间隔
case TIMEOID://时间
case TIMESTAMPOID://时间戳
case TIMESTAMPTZOID://带时区的时间戳
case DATEOID://日期
case TIMETZOID://带时区的时间
case SMALLDATETIMEOID://日期和时间
case NUMERICOID://任意精度数值
case UUIDOID://通用唯一标识符
//如果数据类型受支持,则继续下一列的检查
break;
default:
//如果数据类型不受支持,则抛出错误异常
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("Unsupport column type \"%s\" of ledger user table.",
TypeNameToString(colDef->typname))));

View File

@ -41,114 +41,152 @@
#include "catalog/gs_global_chain.h"
/*
* create_hist_relation -- create a hist table based on the original user table.
* create_hist_relation -- create a hist table based on the original user table.//基于原始用户表创建一个hist表
*
* rel: The original user relation
* reloptions: relation options used to define new relation
* mainTblStmt: Some statement of the query when create the new relation.
* rel: The original user relation//原始用户表
* reloptions: relation options used to define new relation//用于定义新关系的关系选项
* mainTblStmt: Some statement of the query when create the new relation.//创建新关系时的某个查询语句
*/
//根据给定的目标表rel创建历史链表
void create_hist_relation(Relation rel, Datum reloptions, CreateStmt *mainTblStmt)
{
errno_t rc;
char hist_name[NAMEDATALEN];
Oid relid = RelationGetRelid(rel);
Oid nsp_oid = PG_BLOCKCHAIN_NAMESPACE;
Oid relid = RelationGetRelid(rel);//获取目标表的OID
Oid nsp_oid = PG_BLOCKCHAIN_NAMESPACE;//命名空间的OID
Oid hist_oid;
Oid collationObjectId[1];
Oid classObjectId[1];
int16 coloptions[1];
bool shared_relation = rel->rd_rel->relisshared;
bool shared_relation = rel->rd_rel->relisshared;//判断目标表是否为共享关系
get_hist_name(relid, get_rel_name(relid), hist_name);
get_hist_name(relid, get_rel_name(relid), hist_name);//获取历史链表的名称
/*
* history chain table contains all the columns from the origin user table, and then need to
* record the command type, blocknum, and hash value of last block record.
*
*/
TupleDesc chain_desc = CreateTemplateTupleDesc(USERCHAIN_COLUMN_NUM, false);
TupleDesc chain_desc = CreateTemplateTupleDesc(USERCHAIN_COLUMN_NUM, false);//创建历史链表的元组描述符
/* Now consider the additional columns and initilize the description */
TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_REC_NUM + 1, "rec_num", INT8OID, -1, 0);
TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_HASH_INS + 1, "hash_ins", HASH16OID, -1, 0);
TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_HASH_DEL + 1, "hash_del", HASH16OID, -1, 0);
TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_PREVHASH + 1, "pre_hash", HASH32OID, -1, 0);
/* Now consider the additional columns and initilize the description */ //初始化历史链表的额外列
TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_REC_NUM + 1, "rec_num", INT8OID, -1, 0);//添加记录号列
TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_HASH_INS + 1, "hash_ins", HASH16OID, -1, 0);//添加插入操作的哈希值列
TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_HASH_DEL + 1, "hash_del", HASH16OID, -1, 0);//添加删除操作的哈希值列
TupleDescInitEntry(chain_desc, USERCHAIN_COLUMN_PREVHASH + 1, "pre_hash", HASH32OID, -1, 0);//添加上一个区块的哈希值列
reloptions = AddInternalOption(reloptions, INTERNAL_MASK_DALTER | INTERNAL_MASK_DDELETE |
INTERNAL_MASK_DINSERT | INTERNAL_MASK_DUPDATE);
INTERNAL_MASK_DINSERT | INTERNAL_MASK_DUPDATE);//添加内部选项,通过使用按位或操作符(|将INTERNAL_MASK_DALTER、INTERNAL_MASK_DDELETE、INTERNAL_MASK_DINSERT和INTERNAL_MASK_DUPDATE四个标志位添加到reloptions中
//添加这些内部选项可以用于在处理历史链表时进行相应的操作追踪和记录。
//使用给定的历史链表名称(hist_name)、命名空间OID(nsp_oid)、表空间OID(rel->rd_rel->reltablespace)等参数创建历史链表。
//使用给定的chain_desc元组描述符来定义历史链表的列和属性。
//使用指定的参数(如所有者、持久性、共享关系标志等)来配置历史链表。
//注册历史链表的元数据到系统目录中,以便后续可以在数据库中访问和操作该表
//将创建的历史链表的对象IDOID赋值给hist_oid变量以便后续的处理和引用
hist_oid = heap_create_with_catalog(hist_name, nsp_oid, rel->rd_rel->reltablespace, InvalidOid,
InvalidOid, InvalidOid, rel->rd_rel->relowner, chain_desc, NIL, 'r',
(rel->rd_rel->relpersistence == 't') ? 'u' : rel->rd_rel->relpersistence,
shared_relation, false, true, 0, ONCOMMIT_NOOP, reloptions, false, true,
NULL, REL_CMPRS_NOT_SUPPORT, NULL, false);
NULL, REL_CMPRS_NOT_SUPPORT, NULL, false);//创建历史链表
/* make the history chain relation visible, else heap_open will fail */
CommandCounterIncrement();
CommandCounterIncrement();//提交事务,使历史链表对后续操作可见
#ifdef ENABLE_MULTIPLE_NODES
#ifdef ENABLE_MULTIPLE_NODES//启用了多节点(分布式)功能
bool is_initdb_on_dn = false;
/* Add to pgxc_class */
/* When the sum of shmemNumDataNodes and shmemNumCoords equals to one,
* the create table command is executed on datanode during initialization .
* In this case, we do not write created table info in pgxc_class.
*/
/*当shmemNumDataNodes和shmemNumCoords的总和等于1时
* datanode上执行创建表的命令
* pgxc_class
*/
if ((*t_thrd.pgxc_cxt.shmemNumDataNodes + *t_thrd.pgxc_cxt.shmemNumCoords) == 1) {
is_initdb_on_dn = true;
}
/* only support normal table, do not support foreign table (can be supported in the future) */
//仅支持普通表
if ((!u_sess->attr.attr_common.IsInplaceUpgrade || !IsSystemNamespace(nsp_oid)) &&
(IS_PGXC_COORDINATOR || (isRestoreMode && mainTblStmt->distributeby != NULL && !is_initdb_on_dn))) {
(IS_PGXC_COORDINATOR || (isRestoreMode && mainTblStmt->distributeby != NULL && !is_initdb_on_dn))) {//不是在升级模式下或者不在系统命名空间下,是协调节点,是恢复模式且主表语句中的distributeby不为空且不是在datanode上执行初始化操作
AddRelationDistribution(hist_name, hist_oid, NULL, mainTblStmt->subcluster,
InvalidOid, chain_desc, true);
CommandCounterIncrement();
InvalidOid, chain_desc, true);//将历史链表添加到pgxc_class中以支持分布式
CommandCounterIncrement();//提交事务使pgxc_class对后续操作可见
/* Make sure locator info gets rebuilt */
RelationCacheInvalidateEntry(hist_oid);
RelationCacheInvalidateEntry(hist_oid);//刷新历史链表的关系缓存
}
#endif
/* now create index for this new history table */
//为这个新的历史表创建索引
char hist_index_name[NAMEDATALEN];
rc = snprintf_s(hist_index_name, NAMEDATALEN, NAMEDATALEN - 1, "gs_hist_%u_index", relid);
rc = snprintf_s(hist_index_name, NAMEDATALEN, NAMEDATALEN - 1, "gs_hist_%u_index", relid);//创建历史链表的索引名称
securec_check_ss(rc, "", "");
/* open the previous created history chain table */
Relation hist_rel = heap_open(hist_oid, ShareLock);
IndexInfo *hist_index = makeNode(IndexInfo);
Relation hist_rel = heap_open(hist_oid, ShareLock);//打开先前创建的历史链表
IndexInfo *hist_index = makeNode(IndexInfo);//创建索引信息
hist_index->ii_NumIndexAttrs = 1;
hist_index->ii_NumIndexKeyAttrs = 1;
hist_index->ii_KeyAttrNumbers[0] = 1;
hist_index->ii_NumIndexKeyAttrs = 1;//索引的属性数量和键属性数量都为1
hist_index->ii_KeyAttrNumbers[0] = 1;//索引的键属性为第一个属性
hist_index->ii_Expressions = NIL;
hist_index->ii_ExpressionsState = NIL;
hist_index->ii_Predicate = NIL;
hist_index->ii_PredicateState = NIL;
hist_index->ii_PredicateState = NIL;//设置为空列表,表示索引没有谓词限制
hist_index->ii_ExclusionOps = NULL;
hist_index->ii_ExclusionProcs = NULL;
hist_index->ii_ExclusionStrats = NULL;
hist_index->ii_Unique = true;
hist_index->ii_ReadyForInserts = true;
hist_index->ii_Concurrent = false;
hist_index->ii_BrokenHotChain = false;
hist_index->ii_PgClassAttrId = Anum_pg_class_relhasindex;
hist_index->ii_ExclusionStrats = NULL;//表示索引没有排除约束
hist_index->ii_Unique = true;//索引是唯一的
hist_index->ii_ReadyForInserts = true;//索引已准备好接收插入操作
hist_index->ii_Concurrent = false;//索引不支持并发操作
hist_index->ii_BrokenHotChain = false;//索引的热点链未破裂
hist_index->ii_PgClassAttrId = Anum_pg_class_relhasindex;//设置索引信息的各个字段值,表示索引对应于pg_class表的relhasindex字段
collationObjectId[0] = InvalidOid;
classObjectId[0] = INT4_BTREE_OPS_OID;
coloptions[0] = 0;
collationObjectId[0] = InvalidOid;//设置索引的排序规则对象标识符为无效值InvalidOid表示索引不使用任何特定的排序规则。
classObjectId[0] = INT4_BTREE_OPS_OID;//设置索引的操作符类对象标识符为INT4_BTREE_OPS_OID表示索引使用整型数据的 B-tree 操作符。
coloptions[0] = 0;//索引没有额外的列选项
IndexCreateExtraArgs extra;
extra.existingPSortOid = InvalidOid;
extra.isPartitionedIndex = false;
extra.isGlobalPartitionedIndex = false;
extra.existingPSortOid = InvalidOid;//该索引不依赖于任何预排序对象
extra.isPartitionedIndex = false;//表示该索引不是分区索引
extra.isGlobalPartitionedIndex = false;//该索引不是全局分区索引
/*
hist_rel
hist_index_name
InvalidOid使 InvalidOid
InvalidOid InvalidOid
hist_index IndexInfo
list_make1((void *)"rec_num") "rec_num"
BTREE_AM_OID访使 B-tree
rel->rd_rel->reltablespace
collationObjectId
classObjectId
coloptions
(Datum) 0
true
false
false
false
true
false
false
&extra
false
*/
index_create(hist_rel, hist_index_name, InvalidOid, InvalidOid,
hist_index, list_make1((void *)"rec_num"), BTREE_AM_OID,
rel->rd_rel->reltablespace, collationObjectId, classObjectId,
coloptions, (Datum) 0, true, false, false, false,
true, false, false, &extra, false);
true, false, false, &extra, false);//在历史链表上创建索引
heap_close(hist_rel, NoLock);
heap_close(hist_rel, NoLock);//关闭历史链表关系
/* Specify dependent between history table and origin table with depend option audo. */
//使用depend选项将历史表和原始表之间的依赖关系指定为自动依赖。
ObjectAddress myself;
ObjectAddress referenced;
myself.classId = RelationRelationId;
@ -157,84 +195,92 @@ void create_hist_relation(Relation rel, Datum reloptions, CreateStmt *mainTblStm
referenced.classId = RelationRelationId;
referenced.objectId = relid;
referenced.objectSubId = 0;
recordDependencyOn(&myself, &referenced, DEPENDENCY_AUTO);
recordDependencyOn(&myself, &referenced, DEPENDENCY_AUTO);//指定历史链表和原始表之间的依赖关系为自动依赖,当被依赖的对象发生变化时,依赖关系会自动更新
pfree_ext(chain_desc);
pfree_ext(chain_desc);//释放内存
/*
* Make changes visible
*/
CommandCounterIncrement();
CommandCounterIncrement();//提交事务,使更改对后续操作可见
}
/*
* rename_hist_by_usertable -- rename hist table name by its user table and new usertable name
* rename_hist_by_usertable -- rename hist table name by its user table and new usertable name//根据用户表和新的用户表名重命名历史表名
*
* relid: relation oid of user table
* new_usertable_name: the new table name of user table
* relid: relation oid of user table//用户表的关系oid
* new_usertable_name: the new table name of user table//用户表的新表名
*
* Note: This function is used after origin user table renamed, and then caller
* can use this function to rename the corresponding hist table name.
* 使
*/
void rename_hist_by_usertable(Oid relid, const char *new_usertable_name)
{
//获取历史表的OID
Oid hist_oid = get_hist_oid(relid);
//创建一个保存新历史表名的字符数组
char new_hist_name[NAMEDATALEN];
//根据用户表和新的用户表名生成新的历史表名
get_hist_name(relid, new_usertable_name, new_hist_name);
/* Do rename hist table. */
//执行历史表重命名操作
RenameRelationInternal(hist_oid, new_hist_name);
}
/*
* rename_hist_by_newnsp -- rename one hist table while altering schema name
* rename_hist_by_newnsp -- rename one hist table while altering schema name//通过修改模式名称重命名一个历史表
*
* user_relid: relation oid of user table
* new_nsp_name: the new schema name of user table
* user_relid: relation oid of user table//用户表的关系oid
* new_nsp_name: the new schema name of user table//用户表的新模式名称
*/
void rename_hist_by_newnsp(Oid user_relid, const char *new_nsp_name)
{
Oid hist_oid;
char old_hist_name[NAMEDATALEN] = {0};
char new_hist_name[NAMEDATALEN] = {0};
//获取旧的历史表名称
get_hist_name(user_relid, get_rel_name(user_relid), old_hist_name);
//根据旧的历史表名称和指定的命名空间获取历史表的oid
hist_oid = get_relname_relid(old_hist_name, PG_BLOCKCHAIN_NAMESPACE);
/* Some especial tables such as foreign tables have no hist table. So make sure hist exists. */
if (!OidIsValid(hist_oid)) {
if (!OidIsValid(hist_oid)) {//如果历史表不存在,则直接返回
return;
}
//构造新的历史表名称
get_hist_name(user_relid, get_rel_name(user_relid), new_hist_name, get_rel_namespace(user_relid), new_nsp_name);
//执行重命名操作
RenameRelationInternal(hist_oid, new_hist_name);
}
/*
* rename_histlist_by_newnsp -- rename a list of hist table while altering schema name
* rename_histlist_by_newnsp -- rename a list of hist table while altering schema name//在修改模式名称的同时重命名历史表列表
*
* usertable_oid_list: relation oid list of user tables
* new_nsp_name: the new schema name of user table
* usertable_oid_list: relation oid list of user tables//用户表的关系oid列表
* new_nsp_name: the new schema name of user table//用户表的新模式名称
*/
void rename_histlist_by_newnsp(List *usertable_oid_list, const char *new_nsp_name)
{
ListCell *lc = NULL;
foreach (lc, usertable_oid_list) {
Oid relid = (Oid)lfirst_oid(lc);
rename_hist_by_newnsp(relid, new_nsp_name);
foreach (lc, usertable_oid_list) {//批量处理
Oid relid = (Oid)lfirst_oid(lc);//将当前元素转换为Oid类型的relid
rename_hist_by_newnsp(relid, new_nsp_name);//用新的模式名称new_nsp_name重命名历史表
}
}
/*
* user_hash_attrno -- get the attribute number of user table's hash column.
* user_hash_attrno -- get the attribute number of user table's hash column.//获取用户表哈希列的属性编号
*
* rd_att: tuple description of user table
* rd_att: tuple description of user table//用户表的元组描述
*/
int user_hash_attrno(const TupleDesc rd_att)
{
int hash_natt = -1;
Form_pg_attribute rel_attr = NULL;
for (int i = rd_att->natts - 1; i >= 0; i--) {
Form_pg_attribute rel_attr = NULL;//遍历用户表的属性
for (int i = rd_att->natts - 1; i >= 0; i--) {//从最后一个属性开始向前查找
rel_attr = rd_att->attrs[i];
if (strcmp(rel_attr->attname.data, "hash") == 0) {
if (strcmp(rel_attr->attname.data, "hash") == 0) {//如果两者相等,则说明找到了哈希列
hash_natt = i;
break;
}
@ -243,11 +289,11 @@ int user_hash_attrno(const TupleDesc rd_att)
}
/*
* hash_combine_tuple_data -- generate hash of each attribute and return the combination string.
* hash_combine_tuple_data -- generate hash of each attribute and return the combination string.//计算每个属性的哈希值,并返回组合字符串
*
* data_string: combination of hash that calculate from each attribute
* tabledesc: tuple description of user table
* tuple: row data of user table
* data_string: combination of hash that calculate from each attribute//由每个属性计算得到的哈希值的组合字符串
* tabledesc: tuple description of user table//用户表的元组描述
* tuple: row data of user table//用户表的行数据
*/
static void hash_combine_tuple_data(char *buf, int buf_size, TupleDesc tabledesc, HeapTuple tuple)
{
@ -257,178 +303,190 @@ static void hash_combine_tuple_data(char *buf, int buf_size, TupleDesc tabledesc
char hash_str[UINT64STRSIZE + 1] = {0};
Datum *values = (Datum *) palloc0(natts * sizeof(Datum));
bool *nulls = (bool *) palloc0(natts * sizeof(bool));
//将堆元组解码为属性值和空值标志
heap_deform_tuple(tuple, tabledesc, values, nulls);
for (int i = 0; i < natts - 1; ++i) { /* except 'hash' column. */
if (nulls[i]) {
continue;
}
//计算属性的哈希值
uint64 col_hash = compute_hash(tabledesc->attrs[i]->atttypid, values[i], LOCATOR_TYPE_HASH);
//将哈希值转换为字符串
rc = snprintf_s(hash_str, UINT64STRSIZE + 1, UINT64STRSIZE, "%lu", col_hash);
securec_check_ss(rc, "", "");
//将哈希值拼接到结果字符串中
rc = snprintf_s(buf + buflen, buf_size - buflen, buf_size - buflen - 1, "%s", hash_str);
securec_check_ss(rc, "", "");
buflen += strlen(hash_str);
}
//释放内存
pfree_ext(values);
pfree_ext(nulls);
}
/*
* get_user_tuple_hash -- get the hash value of usertable's tuple.
* get_user_tuple_hash -- get the hash value of usertable's tuple.//获取用户表元组的哈希值
*
* tuple: row data of user table
* desc: tuple description of user table
* tuple: row data of user table//用户表的行数据
* desc: tuple description of user table//用户表的元组描述
*/
uint64 get_user_tuple_hash(HeapTuple tuple, TupleDesc desc)
{
Datum value;
bool isnull = false;
//获取哈希列的列号
int hash_attno = user_hash_attrno(desc);
//从元组中获取哈希列的值(最后一列)
value = heap_getattr(tuple, hash_attno + 1, desc, &isnull); /* get last column. */
Assert(!isnull);
Assert(!isnull);//确保获取的值不为空
//将值转换为uint64类型并返回
return DatumGetUInt64(value);
}
/*
* gen_user_tuple_hash -- generate hash of each user table's tuple.
* gen_user_tuple_hash -- generate hash of each user table's tuple.//生成用户表中每个元组的哈希值
*
* rel: user table
* tuple: row data of user table
* rel: user table//用户表
* tuple: row data of user table//用户表的行数据
*/
static uint64 gen_user_tuple_hash(Relation rel, HeapTuple tuple)
{
//获取用户表的元组描述符
TupleDesc tabledesc = RelationGetDescr(rel);
//计算用于存储哈希值以及中间结果的字符串需要的大小
int data_size = UINT64STRSIZE * tabledesc->natts + 1;
//分配内存存储字符串,并清空
char *data_string = (char *)palloc0(data_size * sizeof(char));
//计算元组数据的哈希值并将其拼接到字符串中
hash_combine_tuple_data(data_string, data_size, tabledesc, tuple);
//计算字符串的MD5哈希值存储在sum数组中
uint8 sum[16];
if (pg_md5_binary(data_string, strlen(data_string), sum) == false) {
pfree_ext(data_string);
ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory")));
}
//将sum数组的前8个字节即16个字节中的第5到第12个字节拼接成一个64位整数
uint64 result = 0;
for (int i = 0; i < 7; i++) {
result |= sum[4 + i];
result = (result << 8);
}
result |= sum[11];
//释放分配的内存并返回哈希值
pfree_ext(data_string);
return result;
}
/*
* set_user_tuple_hash -- calculate and fill the hash attribute of user table's tuple.
* set_user_tuple_hash -- calculate and fill the hash attribute of user table's tuple.//计算并填充用户表元组的哈希属性
*
* tup: row data of user table
* rel: user table
* hash_exists: whether tuple comes with tuplehash.
* tup: row data of user table//用户表的行数据
* rel: user table//用户表
* hash_exists: whether tuple comes with tuplehash.//元组是否包含tuplehash
*
* Note: if hash_exists is true, we should recompute
* tuple hash and compare with tuplehash of itself.
* tuple hash and compare with tuplehash of itself.//如果hash_exists为真则我们应该重新计算元组的哈希值并与自身的tuplehash进行比较
*/
HeapTuple set_user_tuple_hash(HeapTuple tup, Relation rel, bool hash_exists)
{
uint64 row_hash = gen_user_tuple_hash(rel, tup);
int hash_attrno = user_hash_attrno(rel->rd_att);
if (hash_exists) {
uint64 row_hash = gen_user_tuple_hash(rel, tup);//生成该行数据的哈希值
int hash_attrno = user_hash_attrno(rel->rd_att);//获取哈希属性在用户表中的属性编号
if (hash_exists) {//如果元组已经包含tuplehash
bool is_null;
Datum hash = heap_getattr(tup, hash_attrno + 1, rel->rd_att, &is_null);
if (is_null || row_hash != DatumGetUInt64(hash)) {
ereport(ERROR, (errcode(ERRCODE_OPERATE_INVALID_PARAM), errmsg("Invalid tuple hash.")));
Datum hash = heap_getattr(tup, hash_attrno + 1, rel->rd_att, &is_null);//从元组中获取已存在的哈希值
if (is_null || row_hash != DatumGetUInt64(hash)) {//比较新计算的哈希值和已存在的哈希值是否相等
ereport(ERROR, (errcode(ERRCODE_OPERATE_INVALID_PARAM), errmsg("Invalid tuple hash.")));//哈希值不一致,抛出错误
}
return tup;
return tup;//返回原始的元组
}
Datum *values = NULL;
bool *nulls = NULL;
bool *replaces = NULL;
/* Build modified tuple */
/* Build modified tuple */ //构建修改后的元组
int2 nattrs = RelationGetNumberOfAttributes(rel);
values = (Datum*)palloc0(nattrs * sizeof(Datum));
nulls = (bool*)palloc0(nattrs * sizeof(bool));
replaces = (bool*)palloc0(nattrs * sizeof(bool));
values[hash_attrno] = UInt64GetDatum(row_hash);
replaces[hash_attrno] = true;
HeapTuple newtup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces);
values[hash_attrno] = UInt64GetDatum(row_hash); //将新计算的哈希值存入对应的属性中
replaces[hash_attrno] = true;//标记该属性为被替换
HeapTuple newtup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces);//修改元组
pfree_ext(values);
pfree_ext(nulls);
pfree_ext(replaces);
return newtup;
return newtup;//返回修改后的元组
}
/*
* get_hist_oid -- get the oid of history table by oid, name and namespace name of user table
* get_hist_oid -- get the oid of history table by oid, name and namespace name of user table//根据用户表的OID、名称和命名空间名称获取历史表的OID
*
* relid: relation oid of user table
* rel_name: relation name of user table
* rel_nsp: namespace name of user table
* relid: relation oid of user table//用户表的关系OID
* rel_name: relation name of user table//用户表的关系名称
* rel_nsp: namespace name of user table//用户表的命名空间名称
*/
Oid get_hist_oid(Oid relid, const char *rel_name, Oid rel_nsp)
{
if (rel_name == NULL) {
rel_name = get_rel_name(relid);
if (rel_name == NULL) {//如果为NULL
rel_name = get_rel_name(relid);//通过get_rel_name(relid)函数获取用户表user table的名称并将其赋值给rel_name
}
char hist_name[NAMEDATALEN];
get_hist_name(relid, rel_name, hist_name, rel_nsp);
Oid hist_oid = get_relname_relid(hist_name, PG_BLOCKCHAIN_NAMESPACE);
get_hist_name(relid, rel_name, hist_name, rel_nsp);//根据用户表的OID、名称和命名空间名称生成历史表的名称
Oid hist_oid = get_relname_relid(hist_name, PG_BLOCKCHAIN_NAMESPACE);//根据历史表的名称和预定义的命名空间PG_BLOCKCHAIN_NAMESPACE获取历史表的OID
return hist_oid;
}
/*
* get_user_tupleid_hash -- get the hash value of usertable's tupleid
* get_user_tupleid_hash -- get the hash value of usertable's tupleid//获取用户表元组的哈希值
*
* relation: relation of user table
* tupleid: tupleid of user tuple
* relation: relation of user table//用户表的关系
* tupleid: tupleid of user tuple//用户元组的tupleid
*/
uint64 get_user_tupleid_hash(Relation relation, ItemPointer tupleid)
{
BlockNumber block;
Buffer buffer;
Buffer vmbuffer = InvalidBuffer;
Page page;
ItemId lp;
HeapTupleData tp;
TupleDesc tabledescr;
uint64 result;
BlockNumber block; //块号
Buffer buffer;//缓冲区
Buffer vmbuffer = InvalidBuffer;//可见度映射缓冲区
Page page;//页面
ItemId lp;//元组标识符
HeapTupleData tp;//堆元组数据
TupleDesc tabledescr;//表描述符
uint64 result;//哈希值
tabledescr = RelationGetDescr(relation);
/* get tuple use tupleid */
block = ItemPointerGetBlockNumber(tupleid);
buffer = ReadBuffer(relation, block);
page = BufferGetPage(buffer);
tabledescr = RelationGetDescr(relation);//获取表描述符
/* get tuple use tupleid */ //使用元组标识符TupleId获取元组
block = ItemPointerGetBlockNumber(tupleid);//获取元组所在的块号
buffer = ReadBuffer(relation, block);//读取关系中的块
page = BufferGetPage(buffer);//获取块对应的页
if (PageIsAllVisible(page)) {
visibilitymap_pin(relation, block, &vmbuffer);
visibilitymap_pin(relation, block, &vmbuffer);//锁定页面
}
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);//对缓冲区进行互斥锁定
lp = PageGetItemId(page, ItemPointerGetOffsetNumber(tupleid));
tp.t_tableOid = RelationGetRelid(relation);
tp.t_data = (HeapTupleHeader) PageGetItem(page, lp);
tp.t_len = ItemIdGetLength(lp);
tp.t_self = *tupleid;
lp = PageGetItemId(page, ItemPointerGetOffsetNumber(tupleid));//获取元组标识符
tp.t_tableOid = RelationGetRelid(relation);//设置表OID
tp.t_data = (HeapTupleHeader) PageGetItem(page, lp);//设置元组的数据
tp.t_len = ItemIdGetLength(lp);//设置元组长度
tp.t_self = *tupleid;//设置元组标识符
result = get_user_tuple_hash(&tp, tabledescr);
result = get_user_tuple_hash(&tp, tabledescr);//计算元组的哈希值
UnlockReleaseBuffer(buffer);
UnlockReleaseBuffer(buffer);//解锁并释放缓冲区
if (vmbuffer != InvalidBuffer) {
ReleaseBuffer(vmbuffer);
ReleaseBuffer(vmbuffer);//如果可见度映射缓冲区不为空,则释放它
}
return result;
return result;//返回哈希值
}
/*
* gen_hist_tuple_hash -- calculate pre_hash of hist table
* gen_hist_tuple_hash -- calculate pre_hash of hist table//计算历史表的预散列哈希值
*
* relid: relation oid of history table
* current_block_data: data of current row, includes hash_ins and hash_del
* pre_row_exist: when current row is the first row, it's true
* pre_row_hash: the pre_hash value of previous row
* hash: the result hash
* relid: relation oid of history table//历史表的关系对象标识符
* current_block_data: data of current row, includes hash_ins and hash_del//当前行的数据,包括要插入和要删除的哈希值
* pre_row_exist: when current row is the first row, it's true//当当前行是第一行时,它的值为 true
* pre_row_hash: the pre_hash value of previous row//前一行的预散列哈希值。
* hash: the result hash//计算结果的哈希值
*/
void gen_hist_tuple_hash(Oid relid, char *current_block_data, bool pre_row_exist,
hash32_t *pre_row_hash, hash32_t *hash)
@ -436,16 +494,22 @@ void gen_hist_tuple_hash(Oid relid, char *current_block_data, bool pre_row_exist
errno_t rc;
int buf_size = strlen(current_block_data) + NAMEDATALEN + 1;
char *data_string = (char *)palloc0(buf_size * sizeof(char));
//如果当前行是第一行
if (pre_row_exist) {
//将前一行的预散列哈希值转换为字符串
char *pre_hash_str = DatumGetCString(DirectFunctionCall1(hash32out, HASH32GetDatum(pre_row_hash)));
//将当前行的数据和前一行的预散列哈希值连接起来
rc = snprintf_s(data_string, buf_size, buf_size - 1, "%s%s", current_block_data, pre_hash_str);
//释放前一行的预散列哈希值字符串
pfree_ext(pre_hash_str);
} else {
//获取历史表的关系名
char *rel_name = get_rel_name(relid);
//将当前行的数据和关系名连接起来
rc = snprintf_s(data_string, buf_size, buf_size - 1, "%s%s", current_block_data, rel_name);
}
securec_check_ss(rc, "", "");
//使用MD5算法计算数据字符串的哈希值
if (!pg_md5_binary(data_string, strlen(data_string), hash->data)) {
pfree_ext(data_string);
ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory")));
@ -454,12 +518,12 @@ void gen_hist_tuple_hash(Oid relid, char *current_block_data, bool pre_row_exist
}
/*
* fill_hist_block -- get newest preblock in cache, and prepare new block for flushing.
* fill_hist_block -- get newest preblock in cache, and prepare new block for flushing.//从缓存中获取最新的预块,并准备新的块用于刷新
*
* histoid: relation oid of history table
* hash_ins: hash_ins value of current row
* hash_del: hash_del value of current row
* block: row id and pre_hash from current row
* histoid: relation oid of history table//历史表的关系OID
* hash_ins: hash_ins value of current row//当前行的hash_ins值
* hash_del: hash_del value of current row//当前行的hash_del值
* block: row id and pre_hash from current row//当前行的行ID和预哈希值
*/
static void fill_hist_block(Oid histoid, uint64 hash_ins, uint64 hash_del, HistBlock *block)
{
@ -467,6 +531,7 @@ static void fill_hist_block(Oid histoid, uint64 hash_ins, uint64 hash_del, HistB
block->rec_num = get_next_recnum(histoid);
/* Before generate previous hash, we should get current block information */
//在生成前一个哈希值之前,我们应该获取当前块的信息
error_t rc = sprintf_s(data, NAMEDATALEN, "%lu%lu%lu", block->rec_num, hash_ins, hash_del);
securec_check_ss(rc, "", "");
@ -474,20 +539,24 @@ static void fill_hist_block(Oid histoid, uint64 hash_ins, uint64 hash_del, HistB
}
/*
* hist_table_record_internal -- append record to history table when user table is modified
* hist_table_record_internal -- append record to history table when user table is modified//当用户表被修改时,将记录追加到历史表中
*
* hist_oid: relation oid of history table
* hash_ins: hash of row that added by current operation
* hash_del: hash of row that deleted by current operation
* hist_oid: relation oid of history table//历史表的关系OID
* hash_ins: hash of row that added by current operation//当前操作添加的行的哈希值
* hash_del: hash of row that deleted by current operation//当前操作删除的行的哈希值
*/
bool hist_table_record_internal(Oid hist_oid, const uint64 *hash_ins, const uint64 *hash_del)
{
//定义一个values数组来存储要插入的字段值同时定义一个nulls数组来表示每个字段是否为NULL。
Datum values[USERCHAIN_COLUMN_NUM] = {0};
bool nulls[USERCHAIN_COLUMN_NUM] = {false};
//判断hash_ins和hash_del是否为NULL并分别赋值给ins_null和del_null。
bool ins_null = hash_ins == NULL;
bool del_null = hash_del == NULL;
//将实际的哈希值赋给t_ins和t_del如果对应的指针是NULL则赋值为0。
uint64 t_ins = ins_null ? 0 : *hash_ins;
uint64 t_del = del_null ? 0 : *hash_del;
//声明一个HistBlock结构体变量block并调用fill_hist_block函数获取当前块的信息。
HistBlock block;
if (!OidIsValid(hist_oid)) {
@ -497,7 +566,7 @@ bool hist_table_record_internal(Oid hist_oid, const uint64 *hash_ins, const uint
/* Before generate previous hash, we should get current block information */
fill_hist_block(hist_oid, t_ins, t_del, &block);
//将block中的字段值赋给values数组对应的位置设置相应的nulls标记。
values[USERCHAIN_COLUMN_REC_NUM] = UInt64GetDatum(block.rec_num);
values[USERCHAIN_COLUMN_HASH_INS] = UInt64GetDatum(t_ins);
values[USERCHAIN_COLUMN_HASH_DEL] = UInt64GetDatum(t_del);
@ -506,98 +575,115 @@ bool hist_table_record_internal(Oid hist_oid, const uint64 *hash_ins, const uint
nulls[USERCHAIN_COLUMN_HASH_INS] = ins_null;
nulls[USERCHAIN_COLUMN_HASH_DEL] = del_null;
nulls[USERCHAIN_COLUMN_PREVHASH] = false;
//打开并加锁历史表,并获取其描述符。
Relation hist_rel = heap_open(hist_oid, RowExclusiveLock);
TupleDesc hist_desc = RelationGetDescr(hist_rel);
//使用heap_form_tuple函数根据hist_desc和values、nulls数组创建一个HeapTuple对象并将其插入到历史表中。
HeapTuple tuple = heap_form_tuple(hist_desc, values, nulls);
simple_heap_insert(hist_rel, tuple);
heap_freetuple(tuple);
heap_freetuple(tuple);//释放tuple的内存
heap_close(hist_rel, RowExclusiveLock);
return true;
}
/*
* hist_table_record_insert -- append a record while inserting into user table
* hist_table_record_insert -- append a record while inserting into user table//在向用户表插入记录时,追加一条记录
*
* rel: relation of user table
* tup: the tuple which is inserted into user table
* res_hash: delta of user table hash
* rel: relation of user table//用户表的关系对象
* tup: the tuple which is inserted into user table//要插入到用户表的元组
* res_hash: delta of user table hash//用户表哈希的增量
*/
bool hist_table_record_insert(Relation rel, HeapTuple tup, uint64 *res_hash)
{
/* check all inputs are avaliable */
//检查所有输入是否有效
if (tup == NULL || rel == NULL) {
return false; /* Do some thing */
return false; /* Do some thing */ //可以进行适当的错误处理
}
//计算要插入的元组的哈希值
uint64 hash_ins = get_user_tuple_hash(tup, rel->rd_att);
//获取历史表对象的OID
Oid hist_oid = get_hist_oid(RelationGetRelid(rel), RelationGetRelationName(rel), RelationGetNamespace(rel));
//将插入哈希值赋值给res_hash
*res_hash = hash_ins;
//调用hist_table_record_internal函数向历史表中插入记录
return hist_table_record_internal(hist_oid, &hash_ins, NULL);
}
/*
* hist_table_record_delete -- append a record while deleting from user table
* hist_table_record_delete -- append a record while deleting from user table//在从用户表中删除记录时追加一条记录
*
* rel: relation of user table
* hash_del: the hash of deleted tuple
* res_hash: delta of user table hash
* rel: relation of user table//用户表的关系对象
* hash_del: the hash of deleted tuple//被删除元组的哈希值
* res_hash: delta of user table hash//用户表哈希的增量
*/
bool hist_table_record_delete(Relation rel, uint64 hash_del, uint64 *res_hash)
{
//获取历史表对象的OID
Oid hist_oid = get_hist_oid(RelationGetRelid(rel), RelationGetRelationName(rel), RelationGetNamespace(rel));
//将被删除元组的哈希值取负值并赋给res_hash作为用户表哈希的增量
*res_hash = -hash_del;
/* insert history record into userchain table */
//调用hist_table_record_internal函数执行向历史表插入记录的操作
return hist_table_record_internal(hist_oid, NULL, &hash_del);
}
/*
* hist_table_record_update -- append a record while updating user table
* hist_table_record_update -- append a record while updating user table//在更新用户表时追加一条记录
*
* rel: relation of user table
* newtup: the tupleid which is updated from user table
* hash_del: the hash of deleted tuple
* res_hash: delta of user table hash
* rel: relation of user table//用户表的关系对象
* newtup: the tupleid which is updated from user table//从用户表中更新的元组ID
* hash_del: the hash of deleted tuple//被删除元组的哈希值
* res_hash: delta of user table hash//用户表哈希的增量
*/
bool hist_table_record_update(Relation rel, HeapTuple newtup, uint64 hash_del, uint64 *res_hash)
{
//获取新元组的哈希值
uint64 hash_ins = get_user_tuple_hash(newtup, rel->rd_att);
//获取历史表对象的OID
Oid hist_oid = get_hist_oid(RelationGetRelid(rel), RelationGetRelationName(rel), RelationGetNamespace(rel));
//计算用户表哈希的增量
*res_hash = hash_ins - hash_del;
/* insert history record into userchain table */
//调用hist_table_record_internal函数执行向历史表插入记录的操作
return hist_table_record_internal(hist_oid, &hash_ins, &hash_del);
}
/*
* get_copyfrom_line_relhash -- extract hash from each copyfrom line
* get_copyfrom_line_relhash -- extract hash from each copyfrom line//从每个COPY FROM行中提取哈希值
*
* row_data: string line from txt
* len: length of row_data
* hash_colno: the number of split char before hash text
* split: split char
* hash: the buffer of getting hash.
* row_data: string line from txt//来自txt文件的字符串行
* len: length of row_data//row_data的长度
* hash_colno: the number of split char before hash text//哈希文本之前的分割字符数
* split: split char//分割字符
* hash: the buffer of getting hash.//存储哈希值的缓冲区
*/
bool get_copyfrom_line_relhash(const char *row_data, int len, int hash_colno, char split, uint64 *hash)
{
int pos;
/* Not found hash column. */
//如果未找到哈希列则返回false
if (hash_colno == -1) {
return false;
}
for (pos = 0; pos < len && hash_colno > 0; ++pos) {
//遍历行数据直到找到分割字符减少hash_colno的计数
if (row_data[pos] == split) {
--hash_colno;
}
}
//检查是否找到哈希列
if (hash_colno == 0) {
int remain_len = len - pos;
const char *hash_str = row_data + pos;
//提取哈希值并存储到指定地址
if (remain_len > 0) {
//如果剩余长度大于0则将哈希字符串转换为uint64并存储在hash指针指向的地址上返回true
*hash = DatumGetUInt64(DirectFunctionCall1(hash16in, CStringGetDatum(hash_str)));
return true;
}
}
//到达此处表示未成功提取哈希值返回false
return false;
}

View File

@ -25,25 +25,29 @@
* -------------------------------------------------------------------------
*/
#include <fstream>
#include <streambuf>
#include <thread>
#include <mutex>
#include <vector>
#include <string>
#include "curl/curl.h"
#include "gs_policy/curl_utils.h"
#include "utils/elog.h"
/*与网络通信和HTTP请求相关的功能。
libcurl库便HTTP请求
CURL句柄URL
*/
#include <fstream> //用于文件输入输出操作。
#include <streambuf> //用于流缓冲区操作
#include <thread> //用于线程操作
#include <mutex> //用于互斥锁操作
#include <vector> //用于向量容器操作
#include <string> //用于字符串操作
#include "curl/curl.h" //用于使用CURL库进行网络通信。
#include "gs_policy/curl_utils.h" //gs_policy模块中与curl通信相关的工具函数定义。
#include "utils/elog.h" //用于错误日志记录。
static std::mutex g_i_mutex;
static std::mutex g_i_mutex; //静态互斥锁对象
CurlUtils::CurlUtils() : m_withSSL(false),
m_certificate(""),
m_user(""),
m_password(""),
m_curlForPost(NULL)
CurlUtils::CurlUtils() : m_withSSL(false), //用于指示是否启用SSL连接默认为 false。
m_certificate(""), //存储SSL证书的文件路径
m_user(""), //存储用户名称
m_password(""), //存储用户密码
m_curlForPost(NULL)//用于执行POST请求的CURL句柄
{
}
@ -52,6 +56,7 @@ CurlUtils::~CurlUtils()
curl_easy_cleanup(m_curlForPost);
}
//初始化 CurlUtils 类的实例
void CurlUtils::initialize(bool withSSL, const std::string certificate, const std::string user,
const std::string password)
{
@ -67,14 +72,19 @@ void CurlUtils::initialize(bool withSSL, const std::string certificate, const st
*/
bool CurlUtils::http_post_file_request(const std::string url, const std::string fileName, bool connection_testing)
{
// 输出调试信息包括URL和文件名
ereport(INFO, (errmsg("Url = %s, fileName = %s", url.c_str(), fileName.c_str())));
// 从文件中读取内容并存储为字符串
std::ifstream t(fileName);
std::string str((std::istreambuf_iterator<char>(t)),
std::istreambuf_iterator<char>());
if (m_curlForPost != NULL) {
// 设置请求头
struct curl_slist *slist1 = NULL;
slist1 = curl_slist_append(slist1, "Content-Type: application/json");
// 设置CURL选项包括URL、进度通知设置、请求体等
(void)curl_easy_setopt(m_curlForPost, CURLOPT_URL, url.c_str());
(void)curl_easy_setopt(m_curlForPost, CURLOPT_NOPROGRESS, 1L);
(void)curl_easy_setopt(m_curlForPost, CURLOPT_POSTFIELDS, str.c_str());
@ -92,25 +102,26 @@ bool CurlUtils::http_post_file_request(const std::string url, const std::string
(void)curl_easy_setopt(m_curlForPost, CURLOPT_TCP_KEEPALIVE, 1L);
/* a simply connection test to server, just verify the connection without any data transfer */
// 进行连接测试,仅验证连接而不进行数据传输
if (connection_testing) {
(void)curl_easy_setopt(m_curlForPost, CURLOPT_CONNECT_ONLY, 1L);
}
/* perform a file transfer */
CURLcode res = curl_easy_perform(m_curlForPost);
if (res != CURLE_OK) {
CURLcode res = curl_easy_perform(m_curlForPost);//使用curl_easy_perform()函数来执行网络请求。该函数将阻塞当前线程,直到请求完成或发生错误。
if (res != CURLE_OK) {// 处理请求失败的情况
/*
* we will not generate error which will make the audit thread restarted
* but make an error and free related resource to make user deal with the connection issue
*/
if (connection_testing) {
ereport(PANIC,
(errmsg("make sure connection to elastic_search_ip_addr, error info: %s\n",
curl_easy_strerror(res))));
if (connection_testing) {//判断是否正在进行连接测试
ereport(PANIC, //错误的严重级别,表示遇到了无法继续执行的致命错误,需要立即终止程序
(errmsg("make sure connection to elastic_search_ip_addr, error info: %s\n", //错误消息的格式化部分
curl_easy_strerror(res))));/*将错误码转换为对应的错误描述字符串。*/
}
curl_slist_free_all(slist1);
curl_easy_reset(m_curlForPost);
curl_slist_free_all(slist1);//释放整个链表及其节点所占用的内存
curl_easy_reset(m_curlForPost);//重置,将其恢复为初始状态以进行新的请求。
ereport(WARNING, (errmsg("Connection issue happended, post file error: %s\n", curl_easy_strerror(res))));
return false;
}

View File

@ -48,19 +48,20 @@
#include "utils/syscache.h"
#include "pgaudit.h"
LoadPoliciesPtr load_audit_policies_hook = NULL;
LoadPolicyAccessPtr load_policy_access_hook = NULL;
LoadPolicyPrivilegesPtr load_policy_privileges_hook = NULL;
LoadPolicyFilterPtr load_policy_filter_hook = NULL;
THR_LOCAL LightUnifiedAuditExecutorPtr light_unified_audit_executor_hook = NULL;
OpFusionUnifiedAuditExecutorPtr opfusion_unified_audit_executor_hook = NULL;
OpFusionUnifiedAuditFlushLogsPtr opfusion_unified_audit_flush_logs_hook = NULL;
//函数指针变量
LoadPoliciesPtr load_audit_policies_hook = NULL; //加载审计策略。
LoadPolicyAccessPtr load_policy_access_hook = NULL; //加载策略的访问权限。
LoadPolicyPrivilegesPtr load_policy_privileges_hook = NULL; //加载策略的权限。
LoadPolicyFilterPtr load_policy_filter_hook = NULL; //加载策略的过滤器。
THR_LOCAL LightUnifiedAuditExecutorPtr light_unified_audit_executor_hook = NULL; //用于执行轻量级统一审计。
OpFusionUnifiedAuditExecutorPtr opfusion_unified_audit_executor_hook = NULL; //执行操作融合的统一审计
OpFusionUnifiedAuditFlushLogsPtr opfusion_unified_audit_flush_logs_hook = NULL; //刷新操作融合的统一审计日志。
static const char* privileges_type[] = { "alter", "analyze", "comment", "create", "drop", "grant", "revoke",
"set", "show", "login_any", "login_failure", "login_success", "logout"};
"set", "show", "login_any", "login_failure", "login_success", "logout"};//用于描述权限的种类
static const char* access_type[] = {"copy", "deallocate", "delete", "execute", "insert", "prepare", "reindex",
"select", "truncate", "update"};
"select", "truncate", "update"};//用于描述访问的方式
/* reserve for masking with function name get_option_type */
/* reserver for maksing with function name construct_resource_name */
@ -74,7 +75,12 @@ static const char* access_type[] = {"copy", "deallocate", "delete", "execute", "
* @policyOid : policy id
*/
static void add_action_type(bool is_access, const char *action_type, const gs_stl::gs_string target_name_s,
Relation relation, Oid policyOid)
Relation relation, Oid policyOid)//向指定的关系中添加访问配置或特权配置
/*is_access布尔类型表示是否添加访问配置
action_type
target_name_sgs_string
relationRelation
policyOidOid ID*/
{
HeapTuple policy_htup = NULL;
if (is_access) {
@ -95,9 +101,10 @@ static void add_action_type(bool is_access, const char *action_type, const gs_st
policy_htup = heap_form_tuple(relation->rd_att, pol_values, pol_nulls);
}
/* Do the insertion */
(void)simple_heap_insert(relation, policy_htup);
CatalogUpdateIndexes(relation, policy_htup);
heap_freetuple(policy_htup);
//将创建的元组插入到指定的关系中,并更新关系的索引。
(void)simple_heap_insert(relation, policy_htup);//将新的行数据插入到指定的堆表中
CatalogUpdateIndexes(relation, policy_htup);//对指定的目录索引进行更新
heap_freetuple(policy_htup);//释放 policy_htup 所占用的内存空间
}
/*
@ -105,19 +112,23 @@ static void add_action_type(bool is_access, const char *action_type, const gs_st
*/
static void handle_add_remove_all_types(int opt_type, privileges_access_set *add_actions,
privileges_access_set *rem_actions, const privileges_access_set *exist_actions, bool is_add, long long polID,
const char *object)
const char *object)//处理添加和删除权限操作的函数
{
/* add all supported type */
PgPolicyPrivilegesAccessStruct item;
item.m_label_name = "all";
item.m_policy_oid = polID;
if (strcasecmp(object, "all")) {
// 如果 object 不是 "all",执行单个类型的操作
if (strcasecmp(object, "all")) {//比较两个字符串(不区分大小写),判断它们是否相等。
item.m_type = object;
if (is_add) {
// 如果该类型不存在于已有权限中,将其添加到 add_actions 中
if (exist_actions->find(item) == exist_actions->end()) {
(void)add_actions->insert(item);
}
} else {
// 将该类型的权限从 exist_actions 中移除并加入到 rem_actions 中
privileges_access_set::const_iterator it = exist_actions->find(item);
if (it != exist_actions->end()) {
(void)rem_actions->insert(*it);
@ -125,17 +136,19 @@ static void handle_add_remove_all_types(int opt_type, privileges_access_set *add
}
return;
}
// 遍历所有支持的类型,执行全部类型的操作
int array_size = (opt_type == POLICY_OPT_ACCESS) ? (sizeof(access_type) / sizeof(access_type[0])) :
(sizeof(privileges_type) / sizeof(privileges_type[0]));
for (int i = 0; i < array_size; ++i) {
item.m_type = (opt_type == POLICY_OPT_ACCESS) ? access_type[i] : privileges_type[i];
if (is_add) {
// 如果该类型不存在于已有权限中,将其添加到 add_actions 中
if (exist_actions->find(item) == exist_actions->end()) {
(void)add_actions->insert(item);
}
} else {
// 将该类型的权限从 exist_actions 中移除并加入到 rem_actions 中
(void)rem_actions->insert(item);
}
}
@ -153,11 +166,11 @@ static inline void add_all_supported_types(bool is_access, const gs_stl::gs_stri
{
/* add all supported types */
int array_size = is_access ? (sizeof(access_type) / sizeof(access_type[0])) :
(sizeof(privileges_type) / sizeof(privileges_type[0]));
(sizeof(privileges_type) / sizeof(privileges_type[0]));//计算了数组的大小
for (int i = 0; i < array_size; ++i) {
const char *action_type = is_access ? access_type[i] : privileges_type[i];
add_action_type(is_access, action_type, target_name_s, relation, policyOid);
}
}//遍历数组中的元素,将每个元素传递给 add_action_type 函数进行后续处理。
}
/**
@ -175,7 +188,7 @@ static void add_labels_to_policy(bool is_access, const char *action_type, DefEle
ListCell *target_name = NULL;
/* no targets - means for all db objects */
if (targets == NULL) {
if (targets == NULL) {//针对所有数据库对象进行操作
if (strcasecmp(action_type, "all") == 0) {
add_all_supported_types(is_access, "all", relation, policyOid);
} else {
@ -185,7 +198,7 @@ static void add_labels_to_policy(bool is_access, const char *action_type, DefEle
}
/* one or more targets */
foreach (target_name, targets) {
foreach (target_name, targets) {//有一个或多个资源标签。函数使用 foreach 循环遍历 targets 列表中的每个元素。
RangeVar *rel = (RangeVar*)lfirst(target_name);
gs_stl::gs_string target_name_s;
construct_resource_name(rel, &target_name_s);
@ -198,7 +211,7 @@ static void add_labels_to_policy(bool is_access, const char *action_type, DefEle
} else {
if (verify_label_hook) { /* validate whether this label exists */
if (!verify_label_hook(target_name_s.c_str())) {
heap_close(relation, RowExclusiveLock);
heap_close(relation, RowExclusiveLock);//关闭打开的关系,并释放与之相关的资源。
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("[%s] no such label found", target_name_s.c_str())));
}
@ -221,21 +234,22 @@ static void add_labels_to_policy(bool is_access, const char *action_type, DefEle
* @relation : access catalog or privilege catalog.
*/
static void add_labels_to_privileges_access(bool is_access, const privileges_access_set *actions,
const GsPolicyStruct *policy, Relation relation)
const GsPolicyStruct *policy, Relation relation)//将标签添加到权限访问列表中。
{
//遍历权限访问列表 actions
for (privileges_access_set::const_iterator it = actions->begin(); it != actions->end(); ++it) {
HeapTuple policy_htup = NULL;
HeapTuple policy_htup = NULL;//存储堆元组
const char *action_type = it->m_type.c_str();
gs_stl::gs_string target_name_s = it->m_label_name;
Oid policyOid = policy->m_id;
if (verify_label_hook) {
if (!verify_label_hook(target_name_s.c_str())) {
if (verify_label_hook) {//
if (!verify_label_hook(target_name_s.c_str())) {//用于检查 target_name_s 是否存在有效的标签。
heap_close(relation, RowExclusiveLock);
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("[%s] no such label found", target_name_s.c_str())));
}
}
//根据is_access的值创建堆元组
if (is_access) {
bool policy_nulls[Natts_gs_auditing_policy_acc] = {false};
Datum policy_values[Natts_gs_auditing_policy_acc] = {0};
@ -254,25 +268,27 @@ static void add_labels_to_privileges_access(bool is_access, const privileges_acc
policy_htup = heap_form_tuple(relation->rd_att, policy_values, policy_nulls);
}
/* Do the insertion */
//堆元组插入关系中
(void)simple_heap_insert(relation, policy_htup);
//更新索引
CatalogUpdateIndexes(relation, policy_htup);
heap_freetuple(policy_htup);
heap_freetuple(policy_htup);//释放堆元组内存:
}
}
static bool remove_labels_from_privileges_access(bool is_access, const privileges_access_set *actions,
privileges_access_set *existing_actions, Relation relation, gs_stl::gs_string *err_msg)
privileges_access_set *existing_actions, Relation relation, gs_stl::gs_string *err_msg)//从权限访问列表中移除标签
{
bool is_deleted = false;
for (privileges_access_set::const_iterator it = actions->begin(); it != actions->end(); ++it) {
for (privileges_access_set::const_iterator it = actions->begin(); it != actions->end(); ++it) {//遍历要移除的标签 actions
/* Removing access or privilege from policy having only one item is not allowed */
if (existing_actions->size() == 1) {
if (existing_actions->size() == 1) {//检查存在的权限访问列表 existing_actions 的大小,如果只有一个标签,则不允许移除操作,并设置错误消息
*err_msg = (is_access) ? "Removing auditing access from policy with a single item not allowed" :
"Removing auditing privilege from policy with a single item not allowed";
break;
}
privileges_access_set::iterator i_it = existing_actions->find(*it);
privileges_access_set::iterator i_it = existing_actions->find(*it);//在现有的权限访问列表 existing_actions 中查找要移除的标签
//如果找到要移除的标签,则从关系中进行删除操作,并更新权限访问列表 existing_actions
if (i_it != existing_actions->end()) {
if (!scan_to_delete_from_relation(i_it->m_id, relation, is_access ? GsAuditingPolicyAccessOidIndexId :
GsAuditingPolicyPrivilegesOidIndexId))
@ -281,13 +297,13 @@ static bool remove_labels_from_privileges_access(bool is_access, const privilege
is_deleted = true;
}
}
return is_deleted;
return is_deleted;//返回是否有标签被成功移除的标志位
}
/**
* Add filter expr information into auditing policy.
*/
static void add_filters(const filters_set *filters_to_add, Relation relation)
static void add_filters(const filters_set *filters_to_add, Relation relation)//在审计策略中添加过滤器信息。
{
Datum curtime;
HeapTuple policy_filters_htup;
@ -295,25 +311,29 @@ static void add_filters(const filters_set *filters_to_add, Relation relation)
Datum policy_filters_values[Natts_gs_auditing_policy_filters];
errno_t rc;
/* Get current timestamp */
curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp());
curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp());//获取当前的时间戳
//遍历要添加的过滤器表达式集合
for (filters_set::const_iterator it = filters_to_add->begin(); it != filters_to_add->end(); ++it) {
/* restore values and nulls for insert new node group record */
//为插入新的节点组记录恢复值和空值
rc = memset_s(policy_filters_values, sizeof(policy_filters_values), 0, sizeof(policy_filters_values));
securec_check(rc, "\0", "\0");
rc = memset_s(policy_filters_nulls, sizeof(policy_filters_nulls), false, sizeof(policy_filters_nulls));
securec_check(rc, "\0", "\0");
//设置要插入的字段值
policy_filters_values[Anum_gs_auditing_policy_fltr_filter_type - 1] = DirectFunctionCall1(namein, CStringGetDatum(it->m_type.c_str()));
policy_filters_values[Anum_gs_auditing_policy_fltr_label_name - 1] = DirectFunctionCall1(namein, CStringGetDatum(it->m_label_name.c_str()));
policy_filters_values[Anum_gs_auditing_policy_fltr_logical_operator - 1] =
CStringGetTextDatum(it->m_tree_string.c_str());
policy_filters_values[Anum_gs_auditing_policy_fltr_policy_oid - 1] = ObjectIdGetDatum(it->m_policy_oid);
policy_filters_values[Anum_gs_auditing_policy_fltr_modify_date - 1] = curtime;
//创建一个堆元组
policy_filters_htup = heap_form_tuple(relation->rd_att, policy_filters_values, policy_filters_nulls);
/* Do the insertion */
//进行插入操作
(void)simple_heap_insert(relation, policy_filters_htup);
//更新索引
CatalogUpdateIndexes(relation, policy_filters_htup);
//释放堆元组的内存
heap_freetuple(policy_filters_htup);
}
}
@ -321,21 +341,27 @@ static void add_filters(const filters_set *filters_to_add, Relation relation)
/**
* Update filter expr information into auditing policy.
*/
static void update_filters(const filters_set *filters_to_update, Relation policy_filters_relation)
static void update_filters(const filters_set *filters_to_update, Relation policy_filters_relation)//更新过滤器信息到审计策略
{
//进入循环,遍历要更新的过滤器表达式集合 filters_to_update
for (filters_set::const_iterator it = filters_to_update->begin(); it != filters_to_update->end(); ++it) {
ScanKeyData scanKey[1];
//设置扫描键值 scanKey
ScanKeyInit(&scanKey[0], Anum_gs_auditing_policy_fltr_policy_oid, BTEqualStrategyNumber, F_OIDEQ,
ObjectIdGetDatum(it->m_policy_oid));
/* Search tuple by index */
//使用索引开始扫描匹配的元组
SysScanDesc scanDesc = systable_beginscan(policy_filters_relation, GsAuditingPolicyFiltersPolicyOidIndexId,
true, NULL, 1, scanKey);
//获取下一个匹配的元组
HeapTuple auditingPolicyTuple = systable_getnext(scanDesc);
//检查是否获取到有效的元组。如果没有,调用 add_filters 函数添加新的过滤器表达式记录
if (!HeapTupleIsValid(auditingPolicyTuple)) {
add_filters(filters_to_update, policy_filters_relation); /* curtime */
} else {
}
//如果获取到有效的元组,创建值、空值和替换数组,并设置要更新的字段值
else {
Datum values[Natts_gs_auditing_policy_filters] = { 0 };
bool nulls[Natts_gs_auditing_policy_filters] = { false };
bool replaces[Natts_gs_auditing_policy_filters] = { false };
@ -350,27 +376,32 @@ static void update_filters(const filters_set *filters_to_update, Relation policy
values[Anum_gs_auditing_policy_fltr_logical_operator - 1] = CStringGetTextDatum(it->m_tree_string.c_str());
nulls[Anum_gs_auditing_policy_fltr_logical_operator - 1] = false;
replaces[Anum_gs_auditing_policy_fltr_logical_operator - 1] = true;
//使用新的字段值、空值和替换数组创建新的堆元组 newtuple
HeapTuple newtuple = heap_modify_tuple(auditingPolicyTuple, RelationGetDescr(policy_filters_relation),
values, nulls, replaces);
//执行堆更新操作
simple_heap_update(policy_filters_relation, &newtuple->t_self, newtuple);
//更新索引
CatalogUpdateIndexes(policy_filters_relation, newtuple);
}
//结束扫描
systable_endscan(scanDesc);
}
}
static void handle_alter_add_update_filter(List *filter, Oid policyOid, bool to_add)
{
//检查过滤器是否为空。如果为空,直接返回
if (filter == NULL) {
return;
}
//处理新的过滤器,并将过滤器转换成扁平树的字符串形式
gs_stl::gs_string flat_tree;
if (!process_new_filters(filter, &flat_tree)) {
ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("Unsupported policy filter values")));
}
//创建一个过滤器集合 filters_to_alter并将转换后的过滤器信息插入到集合中
filters_set filters_to_alter;
if (flat_tree.size() > 0) {
PgPolicyFiltersStruct item;
@ -380,13 +411,14 @@ static void handle_alter_add_update_filter(List *filter, Oid policyOid, bool to_
item.m_policy_oid = policyOid;
(void)filters_to_alter.insert(item);
}
//检查是否需要执行更新操作
if (filters_to_alter.size() > 0) {
Relation policy_filters_relation = heap_open(GsAuditingPolicyFiltersRelationId, RowExclusiveLock);
if (policy_filters_relation) {
if (to_add) {
Relation policy_filters_relation = heap_open(GsAuditingPolicyFiltersRelationId, RowExclusiveLock);//检查是否需要执行更新操作
if (policy_filters_relation) {//打开审计策略过滤器关系
if (to_add) {//调用 add_filters 函数添加新的过滤器表达式记录
add_filters(&filters_to_alter, policy_filters_relation);
} else {
} else {//调用 update_filters 函数更新过滤器表达式记录
update_filters(&filters_to_alter, policy_filters_relation);
}
heap_close(policy_filters_relation, RowExclusiveLock);
@ -397,27 +429,32 @@ static void handle_alter_add_update_filter(List *filter, Oid policyOid, bool to_
/*
* Load existing auditing policy about DML synatax.
*/
static void load_existing_privileges(privileges_access_set *privs, long long policy_oid)
static void load_existing_privileges(privileges_access_set *privs, long long policy_oid)//加载关于DML语法的现有审计策略
{
//打开审计策略特权关系的关系对象
Relation relation = heap_open(GsAuditingPolicyPrivilegesRelationId, RowExclusiveLock);
if (relation == NULL) {
return;
return;//如果关系对象为空,则直接返回
}
HeapTuple rtup;
Form_gs_auditing_policy_privileges rel_data;
PgPolicyPrivilegesAccessStruct item;
//对审计策略特权关系进行扫描,并获取下一个 tuple
TableScanDesc scan = tableam_scan_begin(relation, SnapshotNow, 0, NULL);
while (scan && (rtup = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection))) {
rel_data = (Form_gs_auditing_policy_privileges)GETSTRUCT(rtup);
if (rel_data == NULL) {
continue;
}
//将 tuple 中的相关信息存入 item 结构体中
item.m_id = HeapTupleGetOid(rtup);
item.m_type = rel_data->privilegetype.data;
item.m_label_name = rel_data->labelname.data;
item.m_policy_oid = (long long)(rel_data->policyoid);
/* load only matching privileges to policy id */
//仅加载与给定的 policy_oid 相匹配的特权信息,并将其插入到 privs 集合中
if (item.m_policy_oid == policy_oid) {
(void)privs->insert(item);
}
@ -432,14 +469,16 @@ static void load_existing_privileges(privileges_access_set *privs, long long pol
*/
static void load_existing_access(privileges_access_set *acc, long long policy_oid)
{
//打开审计策略访问权限关系的关系对象 relation
Relation relation = heap_open(GsAuditingPolicyAccessRelationId, RowExclusiveLock);
if (!relation) {
if (!relation) {//关系对象为空,则直接返回
return;
}
HeapTuple rtup;
Form_gs_auditing_policy_access rel_data;
PgPolicyPrivilegesAccessStruct item;
//开始对审计策略访问权限关系进行扫描,并获取下一个 tuple
TableScanDesc scan = tableam_scan_begin(relation, SnapshotNow, 0, NULL);
while (scan && (rtup = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection))) {
rel_data = (Form_gs_auditing_policy_access)GETSTRUCT(rtup);
@ -451,6 +490,7 @@ static void load_existing_access(privileges_access_set *acc, long long policy_oi
item.m_label_name = rel_data->labelname.data;
item.m_policy_oid = (long long)(rel_data->policyoid);
/* load only matching access to policy id */
//仅加载与给定的 policy_oid 相匹配的特权信息,并将其插入到 privs 集合中
if (item.m_policy_oid == policy_oid) {
(void)acc->insert(item);
}
@ -462,7 +502,7 @@ static void load_existing_access(privileges_access_set *acc, long long policy_oi
static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool policy_status_changed,
gs_stl::gs_string *err_msg)
gs_stl::gs_string *err_msg)//更新审计策略的函数
{
bool policy_nulls[Natts_gs_auditing_policy];
bool policy_replaces[Natts_gs_auditing_policy];
@ -471,6 +511,7 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool
errno_t rc;
/* restore values and nulls for insert new node group record */
//初始化
rc = memset_s(policy_values, sizeof(policy_values), 0, sizeof(policy_values));
securec_check(rc, "", "");
rc = memset_s(policy_nulls, sizeof(policy_nulls), false, sizeof(policy_nulls));
@ -478,6 +519,7 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool
rc = memset_s(policy_replaces, sizeof(policy_replaces), false, sizeof(policy_replaces));
securec_check(rc, "", "");
//设置用于扫描的键值
ScanKeyData skey;
/* Find the policy row to update */
ScanKeyInit(&skey, ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(policy->m_id));
@ -485,10 +527,11 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool
/*
* set up for heap-or-index scan, not need to check tgscan as systable_getnext will deal with sysscan->irel = NULL
*/
//开始一个堆表扫描,并获取下一个符合条件的 tuple
SysScanDesc tgscan = systable_beginscan(relation, GsAuditingPolicyOidIndexId, true, NULL, 1, &skey);
HeapTuple tup;
tup = systable_getnext(tgscan);
if (!tup || !HeapTupleIsValid(tup)) {
if (!tup || !HeapTupleIsValid(tup)) {//没有找到符合条件的 tuple则结束扫描返回错误信息
systable_endscan(tgscan);
(void)err_msg->append("could not find tuple for policy ");
(void)err_msg->append(policy->m_name);
@ -496,8 +539,10 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool
}
/* Get current timestamp */
//获取当前时间戳
curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp());
//根据 policy_status_changed 的值来决定哪些字段需要更新以及它们的值
if (policy_status_changed) {
policy_replaces[Anum_gs_auditing_policy_pol_enabled - 1] = true;
policy_values[Anum_gs_auditing_policy_pol_enabled - 1] = BoolGetDatum(policy->m_enabled);
@ -510,13 +555,15 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool
policy_values[Anum_gs_auditing_policy_pol_modify_date - 1] = curtime;
HeapTuple newtuple = heap_modify_tuple(tup, RelationGetDescr(relation),
policy_values, policy_nulls, policy_replaces);
//更新关系中的 tuple
simple_heap_update(relation, &newtuple->t_self, newtuple);
//更新索引
CatalogUpdateIndexes(relation, newtuple);
//结束扫描,返回成功
systable_endscan(tgscan);
return true;
}
/*
* function name: create_audit_policy
* description : create auditing policy
@ -524,13 +571,14 @@ static bool update_policy(const GsPolicyStruct *policy, Relation relation, bool
void create_audit_policy(CreateAuditPolicyStmt *stmt)
{
/* check that if has access to config audit policy */
if (!is_policy_enabled()) {
// 检查是否有配置审计策略的权限
if (!is_policy_enabled()) {//没有权限则返回错误
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
errmsg("Permission denied.")));
return;
}
//获取当前用户的用户名和会话IP并保存相关信息。
char user_name[USERNAME_LEN] = {0};
(void)GetRoleName(GetCurrentUserId(), user_name, sizeof(user_name));
char buff[512] = {0};
@ -561,8 +609,8 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt)
securec_check(rc, "", "");
/* start to process policy */
Relation policy_relation = heap_open(GsAuditingPolicyRelationId, RowExclusiveLock);
if (!policy_relation) {
Relation policy_relation = heap_open(GsAuditingPolicyRelationId, RowExclusiveLock);//检查是否已打开审计策略的关系
if (!policy_relation) {//未能成功打开,则返回错误。
/* generate an error */
send_manage_message(AUDIT_FAILED);
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("%s", "failed to open policies relation")));
@ -570,7 +618,8 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt)
}
/* no more than MAX_POLICIES_NUM is allowed */
if (get_num_of_existing_policies<Form_gs_auditing_policy>(policy_relation) >= MAX_POLICIES_NUM) {
//检查是否已达到最大策略数量限制
if (get_num_of_existing_policies<Form_gs_auditing_policy>(policy_relation) >= MAX_POLICIES_NUM) {//超过了最大限制,则返回错误。
heap_close(policy_relation, RowExclusiveLock);
/* generate an error */
send_manage_message(AUDIT_FAILED);
@ -580,6 +629,7 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt)
}
/* check whether such policy exists */
//检查是否已存在同名的策略
policies_set existing_policies;
load_existing_policies<Form_gs_auditing_policy>(policy_relation, &existing_policies);
@ -587,16 +637,16 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt)
cur_policy.m_name = policy_name;
policies_set::iterator it = existing_policies.find(cur_policy);
if (it != existing_policies.end()) { /* policy already exists */
heap_close(policy_relation, RowExclusiveLock);
heap_close(policy_relation, RowExclusiveLock);//
/* while the 'if not exists' is specified generate a notice, else an error */
if (stmt->if_not_exists == true) {
send_manage_message(AUDIT_OK);
ereport(NOTICE, (errmsg("%s policy already exists, create skipping", policy_name)));
ereport(NOTICE, (errmsg("%s policy already exists, create skipping", policy_name)));//生成通知
} else {
send_manage_message(AUDIT_FAILED);
ereport(ERROR,
(errcode(ERRCODE_DUPLICATE_POLICY),
errmsg("%s policy already exists, create failed", policy_name)));
errmsg("%s policy already exists, create failed", policy_name)));//返回错误
}
return;
}
@ -604,6 +654,7 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt)
/* Get current timestamp */
curtime = DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp());
//创建审计策略的元组并插入到关系中。
policy_values[Anum_gs_auditing_policy_pol_name - 1] = DirectFunctionCall1(namein, CStringGetDatum(policy_name));
policy_values[Anum_gs_auditing_policy_pol_comments - 1] = DirectFunctionCall1(namein, CStringGetDatum(""));
policy_values[Anum_gs_auditing_policy_pol_modify_date - 1] = curtime;
@ -617,56 +668,59 @@ void create_audit_policy(CreateAuditPolicyStmt *stmt)
heap_close(policy_relation, RowExclusiveLock);
/* Start to Process PRIVILEGES(DML) && ACCESS(DDL) exprs */
//根据策略类型处理策略目标,将标签添加到相应的关系中
int opt_type = get_option_type(policy_type);
/* Extract policy targets from the statement node tree */
//从语句节点树中提取策略目标
foreach (policy_target_item, stmt->policy_targets) {
DefElem *defel = (DefElem *) lfirst(policy_target_item);
const char *action_type = defel->defname; /* action: DELETE, INSERT, UPDATE, etc. */
DefElem *policy_items = (DefElem *) defel->arg;
switch (opt_type) {
case POLICY_OPT_PRIVILEGES: {
Relation priv_relation = heap_open(GsAuditingPolicyPrivilegesRelationId, RowExclusiveLock);
if (priv_relation) {
add_labels_to_policy(false, action_type, policy_items, priv_relation, policyOid);
heap_close(priv_relation, RowExclusiveLock);
switch (opt_type) {//根据选项类型选择不同的处理方式。
case POLICY_OPT_PRIVILEGES: {//策略类型为权限
Relation priv_relation = heap_open(GsAuditingPolicyPrivilegesRelationId, RowExclusiveLock);//使用独占锁RowExclusiveLock进行访问
if (priv_relation) {//打开成功
add_labels_to_policy(false, action_type, policy_items, priv_relation, policyOid);//DML操作
heap_close(priv_relation, RowExclusiveLock);//使用独占锁关闭了权限关系
}
}
break;
case POLICY_OPT_ACCESS: {
Relation acc_relation = heap_open(GsAuditingPolicyAccessRelationId, RowExclusiveLock);
if (acc_relation) {
add_labels_to_policy(true, action_type, policy_items, acc_relation, policyOid);
heap_close(acc_relation, RowExclusiveLock);
case POLICY_OPT_ACCESS: {//策略访问控制
Relation acc_relation = heap_open(GsAuditingPolicyAccessRelationId, RowExclusiveLock);//使用独占锁RowExclusiveLock进行访问
if (acc_relation) {//打开成功
add_labels_to_policy(true, action_type, policy_items, acc_relation, policyOid);//DDL操作
heap_close(acc_relation, RowExclusiveLock);//使用独占锁关闭了权限关系
}
}
break;
/* handled later */
case POLICY_OPT_FILTER:
case POLICY_OPT_FILTER://类型为策略过滤器
break;
default: {
default: {//未知的策略类型。
/* report about unknown policy type */
ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("Unsupported policy type")));
ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("Unsupported policy type")));//报错
}
break;
}
}
handle_alter_add_update_filter(stmt->policy_filters, policyOid, true /* add new filter row */);
CommandCounterIncrement();
send_manage_message(AUDIT_OK);
handle_alter_add_update_filter(stmt->policy_filters, policyOid, true /* add new filter row */);//处理策略过滤器。
CommandCounterIncrement();//增加命令计数器
send_manage_message(AUDIT_OK);//发送管理消息,表示审核操作已成功。
if (load_policy_privileges_hook) {
//钩子函数的调用
if (load_policy_privileges_hook) {//判断是否注册了策略权限的加载钩子函数。
load_policy_privileges_hook(false);
}
if (load_policy_access_hook) {
if (load_policy_access_hook) {//判断是否注册了策略访问的加载钩子函数。
load_policy_access_hook(false);
}
if (load_audit_policies_hook) {
if (load_audit_policies_hook) {//判断是否注册了策略审计的的加载钩子函数。
load_audit_policies_hook(false);
}
/* load filters must be last */
if (load_policy_filter_hook) {
if (load_policy_filter_hook) {////判断是否注册了策略过滤器的的加载钩子函数。
load_policy_filter_hook(false);
}
}
@ -677,14 +731,14 @@ static void handle_alter_audit_node(AlterAuditPolicyStmt *stmt, gs_stl::gs_strin
privileges_access_set& existing_privileges, privileges_access_set& existing_access)
{
/* Extract policy items from the statement node tree */
if (stmt->policy_type != NULL) {
if (stmt->policy_type != NULL) {//不为空,则进入循环处理策略项
ListCell *policy_item = NULL;
bool is_add = strcasecmp(stmt->policy_action, "add") == 0; /* action: add or remove */
int opt_type = get_option_type(stmt->policy_type);
foreach (policy_item, stmt->policy_items) {
bool is_add = strcasecmp(stmt->policy_action, "add") == 0;//判断操作类型是否为添加("add")操作 /* action: add or remove */
int opt_type = get_option_type(stmt->policy_type);//根据策略类型获取选项类型。
foreach (policy_item, stmt->policy_items) {//遍历策略项列表,对每个策略项进行处理。
DefElem *pol_option_item = (DefElem *) lfirst(policy_item); /* policy option & list of targets */
if (pol_option_item == NULL) { /* for optional parts of statement */
continue;
continue;//如果策略项为空,则跳过当前循环。
}
/* pol_option_item->defname; copy, ... */
bool ret = true;
@ -693,21 +747,21 @@ static void handle_alter_audit_node(AlterAuditPolicyStmt *stmt, gs_stl::gs_strin
ListCell *target = NULL;
if (targets && list_length(targets) > 0) {
/* arguments->defname; LABEL */
foreach (target, targets) {
foreach (target, targets) {//遍历目标列表
if (!(ret = handle_target(target, opt_type, is_add, &err_msg, &access_to_add, &access_to_remove,
&privs_to_add, &privs_to_remove, &existing_labels, &cur_policy, pol_option_item->defname))) {
break;
}
}
} else { /* all objects */
} else { /* all objects *///目标列表为空或不存在
switch (opt_type) {
case POLICY_OPT_ACCESS:
handle_add_remove_all_types(opt_type, &access_to_add, &access_to_remove, &existing_access, is_add,
cur_policy.m_id, pol_option_item->defname);
cur_policy.m_id, pol_option_item->defname);//对访问权限进行添加或移除操作。
break;
case POLICY_OPT_PRIVILEGES:
handle_add_remove_all_types(opt_type, &privs_to_add, &privs_to_remove, &existing_privileges, is_add,
cur_policy.m_id, pol_option_item->defname);
cur_policy.m_id, pol_option_item->defname);//对访问权限进行添加或移除操作。
break;
default:
break;
@ -715,7 +769,7 @@ static void handle_alter_audit_node(AlterAuditPolicyStmt *stmt, gs_stl::gs_strin
}
/* validations, If anything is added/removed to label, label must exist unless it's for ALL */
if (!ret) {
if (!ret) {//验证失败,则生成错误并返回。
/* generate an error */
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("%s", err_msg.c_str())));
return;
@ -729,39 +783,40 @@ static void update_audit_policy_actions(privileges_access_set& privs_to_add, pri
privileges_access_set& existing_privileges, privileges_access_set& existing_access,
gs_stl::gs_string& err_msg)
{
if ((privs_to_add.size() > 0) || (privs_to_remove.size() > 0)) {
Relation priv_relation = heap_open(GsAuditingPolicyPrivilegesRelationId, RowExclusiveLock);
if ((privs_to_add.size() > 0) || (privs_to_remove.size() > 0)) {//判断待添加或待移除的特权是否存在
Relation priv_relation = heap_open(GsAuditingPolicyPrivilegesRelationId, RowExclusiveLock);//打开审核策略特权关系表
if (priv_relation != NULL) {
/* add privileges */
if (privs_to_add.size() > 0) {
add_labels_to_privileges_access(false, &privs_to_add, &cur_policy, priv_relation);
add_labels_to_privileges_access(false, &privs_to_add, &cur_policy, priv_relation);//将特权添加到当前策略项中
} else if (!remove_labels_from_privileges_access(false, &privs_to_remove, &existing_privileges,
priv_relation,
&err_msg)) { /* remove privileges */
&err_msg)) { /* remove privileges *///删除权限
heap_close(priv_relation, RowExclusiveLock);
/* generate an error */
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("%s", (err_msg.size() > 0) ? err_msg.c_str() : "No matching privilege to delete found")));
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),//移除特权操作返回错误
errmsg("%s", (err_msg.size() > 0) ? err_msg.c_str() : "No matching privilege to delete found")));// 生成错误并终止
return;
}
heap_close(priv_relation, RowExclusiveLock);
heap_close(priv_relation, RowExclusiveLock);//关闭特权关系表。
}
}
if ((access_to_add.size() > 0) || (access_to_remove.size() > 0)) {
Relation acc_relation = heap_open(GsAuditingPolicyAccessRelationId, RowExclusiveLock);
if ((access_to_add.size() > 0) || (access_to_remove.size() > 0)) {//判断待添加或待移除的访问权限是否存在
Relation acc_relation = heap_open(GsAuditingPolicyAccessRelationId, RowExclusiveLock);//打开审核策略访问权限关系表
if (acc_relation != NULL) {
/* add access */
if (access_to_add.size() > 0) {
add_labels_to_privileges_access(true, &access_to_add, &cur_policy, acc_relation);
add_labels_to_privileges_access(true, &access_to_add, &cur_policy, acc_relation);//将访问权限添加到当前策略项中
} else if (!remove_labels_from_privileges_access(true, &access_to_remove, &existing_access, acc_relation,
&err_msg)) { /* remove access */
&err_msg)) { /* remove access *///移除访问权限
heap_close(acc_relation, RowExclusiveLock);
/* generate an error */
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("%s", (err_msg.size() > 0) ? err_msg.c_str() : "No matching access to delete found")));
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),//移除访问权限操作返回错误
errmsg("%s", (err_msg.size() > 0) ? err_msg.c_str() : "No matching access to delete found")));//生成错误并终止。
return;
}
heap_close(acc_relation, RowExclusiveLock);
heap_close(acc_relation, RowExclusiveLock);//关闭访问权限关系表。
}
}
}
@ -773,11 +828,12 @@ static void update_audit_policy_actions(privileges_access_set& privs_to_add, pri
void alter_audit_policy(AlterAuditPolicyStmt *stmt)
{
/* check that if has access to config audit policy */
if (!is_policy_enabled()) {
// 检查是否有配置审计策略的权限
if (!is_policy_enabled()) {//没有权限则返回错误
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("Permission denied.")));
return;
}
//获取当前用户的用户名和会话IP并保存相关信息。
char buff[512] = {0};
char user_name[USERNAME_LEN] = {0};
char session_ip[MAX_IP_LEN] = {0};
@ -790,17 +846,17 @@ void alter_audit_policy(AlterAuditPolicyStmt *stmt)
user_name, u_sess->attr.attr_common.application_name, session_ip, stmt->policy_name, stmt->policy_action);
securec_check_ss(rc, "", "");
save_manage_message(buff);
const char *policy_name = stmt->policy_name;
policies_set existing_policies;
policy_labels_map existing_labels;
const char *policy_name = stmt->policy_name;//指向审计策略名称的指针
policies_set existing_policies;//存储已存在的策略的数据结构。
policy_labels_map existing_labels;//存储已存在的标签的数据结构。
privileges_access_set existing_privileges;
privileges_access_set privs_to_remove;
privileges_access_set privs_to_add;
privileges_access_set existing_privileges;//存储已存在的权限的数据结构。
privileges_access_set privs_to_remove;//存储待删除权限的数据结构。
privileges_access_set privs_to_add;//存储待添加权限的数据结构。
privileges_access_set existing_access;
privileges_access_set access_to_remove;
privileges_access_set access_to_add;
privileges_access_set existing_access;//存储已存在的访问记录的数据结构。
privileges_access_set access_to_remove;//存储待删除访问记录的数据结构。
privileges_access_set access_to_add;//存储待添加访问记录的数据结构。
Relation policy_relation = NULL;
Relation labels_relation = NULL;
@ -810,6 +866,7 @@ void alter_audit_policy(AlterAuditPolicyStmt *stmt)
load_existing_policies<Form_gs_auditing_policy>(policy_relation, &existing_policies);
/* first check whether such policy exists */
//检查是否已存在同名的策略,如果存在则返回错误,如果不存在则在关系中插入新的策略元组。
GsPolicyStruct cur_policy;
cur_policy.m_name = policy_name;
policies_set::iterator it = existing_policies.find(cur_policy);
@ -817,13 +874,13 @@ void alter_audit_policy(AlterAuditPolicyStmt *stmt)
heap_close(policy_relation, RowExclusiveLock);
if (stmt->missing_ok) { /* IF EXISTS is specified, generate a notice */
send_manage_message(AUDIT_OK);
ereport(NOTICE, (errmsg("%s policy not found, alter skipping", policy_name)));
ereport(NOTICE, (errmsg("%s policy not found, alter skipping", policy_name)));//生成通知
} else {
/* generate an error */
send_manage_message(AUDIT_FAILED);
ereport(ERROR,
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("%s no such policy found, alter failed", policy_name)));
errmsg("%s no such policy found, alter failed", policy_name)));//返回错误
}
return;
}
@ -831,68 +888,68 @@ void alter_audit_policy(AlterAuditPolicyStmt *stmt)
cur_policy.m_enabled = it->m_enabled;
/* Update policy if needed */
bool policy_status_changed = false;
if (stmt->policy_enabled != NULL) {
bool policy_status_changed = false;//标记审计策略的状态是否发生了改变
if (stmt->policy_enabled != NULL) {//审计策略的状态
DefElem *defel = (DefElem *) stmt->policy_enabled;
if (strcasecmp(defel->defname, "status") == 0) {
bool policy_new_status = (strcasecmp(strVal(defel->arg), "enable") == 0) ? true : false;
if (it->m_enabled != policy_new_status) {
if (it->m_enabled != policy_new_status) {//状态发生了改变,更新当前策略对象状态
policy_status_changed = true;
cur_policy.m_enabled = policy_new_status;
}
}
}
if (((stmt->policy_comments != NULL) && (strlen(stmt->policy_comments) > 0)) || policy_status_changed) {
if (((stmt->policy_comments != NULL) && (strlen(stmt->policy_comments) > 0)) || policy_status_changed) {//需要更新策略的注释或状态。
cur_policy.m_comments = stmt->policy_comments;
gs_stl::gs_string err_msg;
if (!update_policy(&cur_policy, policy_relation, policy_status_changed, &err_msg)) {
heap_close(policy_relation, RowExclusiveLock);
if (!update_policy(&cur_policy, policy_relation, policy_status_changed, &err_msg)) {//更新策略失败
heap_close(policy_relation, RowExclusiveLock);//关闭策略关系对象
/* generate an error */
send_manage_message(AUDIT_FAILED);
ereport(ERROR,
send_manage_message(AUDIT_FAILED);//发送一个审计管理消息
ereport(ERROR,//
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("%s", err_msg.c_str())));
errmsg("%s", err_msg.c_str())));//生成一个错误,并返回。
return;
}
}
heap_close(policy_relation, RowExclusiveLock);
labels_relation = heap_open(GsPolicyLabelRelationId, RowExclusiveLock);
load_existing_labels(labels_relation, &existing_labels);
heap_close(labels_relation, RowExclusiveLock);
labels_relation = heap_open(GsPolicyLabelRelationId, RowExclusiveLock); //存储策略标签的关系
load_existing_labels(labels_relation, &existing_labels);//加载已存在的策略标签
heap_close(labels_relation, RowExclusiveLock);//关闭指定的关系对象
/* Get current timestamp */
(void)DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp());
load_existing_privileges(&existing_privileges, cur_policy.m_id);
load_existing_access(&existing_access, cur_policy.m_id);
(void)DirectFunctionCall1(timestamptz_timestamp, GetCurrentTimestamp());//获取当前时间戳
load_existing_privileges(&existing_privileges, cur_policy.m_id);//加载已存在的权限
load_existing_access(&existing_access, cur_policy.m_id);//加载已存在的访问记录
/* Extract policy items from the statement node tree */
gs_stl::gs_string err_msg;
gs_stl::gs_string err_msg;//保存错误消息。
handle_alter_audit_node(stmt, err_msg, cur_policy, access_to_add, access_to_remove, privs_to_add, privs_to_remove,
existing_labels, existing_privileges, existing_access);
existing_labels, existing_privileges, existing_access);//处理修改审计策略的节点,并传入相关的参数。
update_audit_policy_actions(privs_to_add, privs_to_remove, access_to_add, access_to_remove, cur_policy,
existing_privileges, existing_access, err_msg);
handle_alter_add_update_filter(stmt->policy_filters, cur_policy.m_id, false /* update filter */);
existing_privileges, existing_access, err_msg);//用于更新审计策略的权限和访问记录,并传入相关的参数。
handle_alter_add_update_filter(stmt->policy_filters, cur_policy.m_id, false /* update filter */);//处理添加或更新策略的过滤条件,并传入相关的参数
if (stmt->policy_action && !strcmp(stmt->policy_action, "drop_filter")) {
if (stmt->policy_action && !strcmp(stmt->policy_action, "drop_filter")) {//删除相关的过滤条件。
drop_policy_reference<Form_gs_auditing_policy_filters>(GsAuditingPolicyFiltersRelationId, cur_policy.m_id);
}
CommandCounterIncrement();
send_manage_message(AUDIT_OK);
CommandCounterIncrement();//增加命令计数器
send_manage_message(AUDIT_OK);//发送管理消息,表示审核操作已成功。
if (load_policy_access_hook) {
if (load_policy_access_hook) {//判断是否注册了策略权限的加载钩子函数。
load_policy_access_hook(false);
}
if (load_policy_privileges_hook) {
if (load_policy_privileges_hook) {//判断是否注册了策略访问的加载钩子函数。
load_policy_privileges_hook(false);
}
if (load_audit_policies_hook) {
if (load_audit_policies_hook) {//判断是否注册了策略审计的的加载钩子函数。
load_audit_policies_hook(false);
}
/* load filters must be last */
if (load_policy_filter_hook) {
if (load_policy_filter_hook) {//判断是否注册了策略过滤器的的加载钩子函数。
load_policy_filter_hook(false);
}
}
@ -904,7 +961,7 @@ void alter_audit_policy(AlterAuditPolicyStmt *stmt)
void drop_audit_policy(DropAuditPolicyStmt *stmt)
{
/* check that if has access to config audit policy */
if (!is_policy_enabled()) {
if (!is_policy_enabled()) {//检查是否有权限访问配置审计策略,如果没有权限,则生成一个错误消息并返回。
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
errmsg("Permission denied.")));
@ -912,6 +969,8 @@ void drop_audit_policy(DropAuditPolicyStmt *stmt)
}
/* save Mng logs */
//保存管理日志信息
//据传入的删除策略语句获取当前用户信息、应用程序名称和会话IP并将相关信息保存到管理日志中。
ListCell* policy_obj = NULL;
foreach (policy_obj, stmt->policy_names) {
const char* polname = (const char *)(((Value*)lfirst(policy_obj))->val.str);
@ -932,12 +991,12 @@ void drop_audit_policy(DropAuditPolicyStmt *stmt)
foreach (policy_obj, stmt->policy_names) {
const char* polname = (const char *)(((Value*)lfirst(policy_obj))->val.str);
gs_stl::gs_set<long long> ids;
drop_policy_by_name<Form_gs_auditing_policy>(GsAuditingPolicyRelationId, polname, &ids);
drop_policy_by_name<Form_gs_auditing_policy>(GsAuditingPolicyRelationId, polname, &ids);//通过策略名称查询策略信息并得到关联的对象ID。
if (ids.empty()) {
if (stmt->missing_ok) {
if (stmt->missing_ok) {//输出一个通知消息并跳过后续操作。
ereport(NOTICE, (errmsg("%s policy does not exist, drop skipping", polname)));
continue;
} else {
} else {//发送一个管理消息,并生成一个错误消息,然后返回
send_manage_message(AUDIT_FAILED);
ereport(ERROR,
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
@ -945,7 +1004,7 @@ void drop_audit_policy(DropAuditPolicyStmt *stmt)
return;
}
}
for (long long _id : ids) {
for (long long _id : ids) {//遍历对象ID集合依次删除以下三项相关的目录信息
/* drop gs_auditing_policy_access catalog information */
drop_policy_reference<Form_gs_auditing_policy_access>(GsAuditingPolicyAccessRelationId, _id);
/* drop gs_auditing_policy_privilege catalog information */
@ -955,21 +1014,20 @@ void drop_audit_policy(DropAuditPolicyStmt *stmt)
}
}
CommandCounterIncrement();
send_manage_message(AUDIT_OK);
CommandCounterIncrement(); // 增加命令计数器
send_manage_message(AUDIT_OK);//发送管理消息,表示审核操作已成功。
if (load_policy_access_hook) {
if (load_policy_access_hook) {//判断是否注册了策略权限的加载钩子函数。
load_policy_access_hook(false);
}
if (load_policy_privileges_hook) {
if (load_policy_privileges_hook) {//判断是否注册了策略访问的加载钩子函数。
load_policy_privileges_hook(false);
}
if (load_audit_policies_hook) {
if (load_audit_policies_hook) {//判断是否注册了策略审计的的加载钩子函数。
load_audit_policies_hook(false);
}
/* load filters must be last */
if (load_policy_filter_hook) {
if (load_policy_filter_hook) {//判断是否注册了策略过滤器的的加载钩子函数。
load_policy_filter_hook(false);
}
}

File diff suppressed because it is too large Load Diff

View File

@ -55,35 +55,40 @@
#define static
#endif
GsSaveManagementEvent gs_save_mng_event_hook = NULL;
GsSendManagementEvent gs_send_mng_event_hook = NULL;
GsSaveManagementEvent gs_save_mng_event_hook = NULL;//保存管理事件的回调函数
GsSendManagementEvent gs_send_mng_event_hook = NULL;//发送管理事件的回调函数
void save_manage_message(const char* message)
void save_manage_message(const char* message)//用于保存管理事件
{
if (gs_save_mng_event_hook != NULL) {
gs_save_mng_event_hook(message);
}
}
void send_manage_message(AuditResult result_type)
void send_manage_message(AuditResult result_type)//用于发送管理事件
{
if (gs_send_mng_event_hook != NULL) {
gs_send_mng_event_hook(result_type);
}
}
bool GsPolicyStruct::operator == (const GsPolicyStruct &arg) const
/*
GsPolicyStruct的重载函数用于比较两个对象是否相等使strcasecmp
PgPolicyFiltersStruct和PgPolicyPrivilegesAccessStruct的重载函数也是类似的
便
*/
bool GsPolicyStruct::operator == (const GsPolicyStruct &arg) const//比较两个GsPolicyStruct对象是否相等
{
return strcasecmp(m_name.c_str(), arg.m_name.c_str()) == 0;
}
bool GsPolicyStruct::operator < (const GsPolicyStruct &arg) const
bool GsPolicyStruct::operator < (const GsPolicyStruct &arg) const//比较两个GsPolicyStruct对象的大小关系
{
/* compare by name */
return strcasecmp(m_name.c_str(), arg.m_name.c_str()) < 0;
}
int GsPolicyStruct::operator - (const GsPolicyStruct &arg) const
int GsPolicyStruct::operator - (const GsPolicyStruct &arg) const//执行两个GsPolicyStruct对象之间的减法运算返回一个整数结果。
{
if (*this < arg) {
return -1;
@ -95,7 +100,7 @@ int GsPolicyStruct::operator - (const GsPolicyStruct &arg) const
}
bool PgPolicyFiltersStruct::operator == (const PgPolicyFiltersStruct &arg) const
{
{//比较两个PgPolicyFiltersStruct对象是否相等
if (*this < arg) {
return false;
} else if (arg < *this) {
@ -106,7 +111,7 @@ bool PgPolicyFiltersStruct::operator == (const PgPolicyFiltersStruct &arg) const
}
bool PgPolicyFiltersStruct::operator < (const PgPolicyFiltersStruct &arg) const
{
{//判断两个对象的大小关系
int res = strcasecmp(m_type.c_str(), arg.m_type.c_str());
if (res < 0) {
return true;
@ -125,7 +130,7 @@ bool PgPolicyFiltersStruct::operator < (const PgPolicyFiltersStruct &arg) const
}
int PgPolicyFiltersStruct::operator - (const PgPolicyFiltersStruct &arg) const
{
{//执行两个PgPolicyFiltersStruct对象之间的减法运算返回一个整数结果。
if (*this < arg) {
return -1;
} else if (arg < *this) {
@ -136,13 +141,13 @@ int PgPolicyFiltersStruct::operator - (const PgPolicyFiltersStruct &arg) const
}
bool PgPolicyPrivilegesAccessStruct::operator == (const PgPolicyPrivilegesAccessStruct &arg) const
{
{//比较两个PgPolicyPrivilegesAccessStruct对象是否相等。
return (strcasecmp(m_type.c_str(), arg.m_type.c_str()) == 0) &&
(strcasecmp(m_label_name.c_str(), arg.m_label_name.c_str()) == 0);
}
bool PgPolicyPrivilegesAccessStruct::operator < (const PgPolicyPrivilegesAccessStruct &arg) const
{
{//operator<比较两个PgPolicyPrivilegesAccessStruct对象的大小关系。
int res = strcasecmp(m_type.c_str(), arg.m_type.c_str());
if (res < 0) {
return true;
@ -154,7 +159,7 @@ bool PgPolicyPrivilegesAccessStruct::operator < (const PgPolicyPrivilegesAccessS
}
int PgPolicyPrivilegesAccessStruct::operator - (const PgPolicyPrivilegesAccessStruct &arg) const
{
{//operator-执行两个PgPolicyPrivilegesAccessStruct对象之间的减法运算返回一个整数结果。
if (*this < arg) {
return -1;
} else if (arg < *this) {
@ -164,177 +169,225 @@ int PgPolicyPrivilegesAccessStruct::operator - (const PgPolicyPrivilegesAccessSt
}
}
/* Process new filters from parser tree and tranform them into string */
/*
* process_new_filters函数处理策略过滤器
scan_to_delete_from_relation函数从关系中删除行
construct_resource_name函数构建资源名称
,使
*/
bool process_new_filters(const List *policy_filters, gs_stl::gs_string *flat_tree)
{
if (!policy_filters)
if (!policy_filters) // 如果策略过滤器链表为空则不进行处理直接返回true
return true;
flat_tree->clear();
ListCell *policy_filter_item = NULL;
gs_stl::gs_vector<PolicyFilterNode *> nodes;
flat_tree->clear(); // 清空输出字符串
foreach(policy_filter_item, policy_filters) {
PolicyFilterNode *root = (PolicyFilterNode *) lfirst(policy_filter_item);
nodes.push_back(root);
while (nodes.size() > 0) {
PolicyFilterNode* n = nodes.back();
nodes.pop_back();
/* operator type node */
if (!strcmp(n->node_type, "op")) {
if (!strcmp(n->op_value, "and")) {
(void)flat_tree->append("*");
} else if (!strcmp(n->op_value, "or")) {
(void)flat_tree->append("+");
} else { /* unsupported operator */
return false;
ListCell *policy_filter_item = NULL; // 定义策略过滤器链表节点指针
gs_stl::gs_vector<PolicyFilterNode *> nodes; // 定义PolicyFilterNode指针的向量
foreach (policy_filter_item, policy_filters) { // 遍历策略过滤器链表中的每个节点
PolicyFilterNode *root = (PolicyFilterNode *)lfirst(policy_filter_item); // 获取当前节点的指针
nodes.push_back(root); // 将当前节点指针添加到nodes向量中
while (nodes.size() > 0) { // 当nodes向量不为空时进行循环处理
PolicyFilterNode *n = nodes.back(); // 获取nodes向量最后一个节点的指针
nodes.pop_back(); // 移除nodes向量最后一个节点
// operator type node 操作符类型节点
if (!strcmp(n->node_type, "op")) { // 判断当前节点类型是否为操作符类型
if (!strcmp(n->op_value, "and")) { // 如果是and操作符
(void)flat_tree->append("*"); // 向输出字符串追加*
} else if (!strcmp(n->op_value, "or")) { // 如果是or操作符
(void)flat_tree->append("+"); // 向输出字符串追加+
} else { // 不支持的操作符类型
return false; // 返回false表示处理失败
}
nodes.push_back((PolicyFilterNode *)n->right);
nodes.push_back((PolicyFilterNode *)n->left);
} else if (!strcmp(n->node_type, "filter")) { /* value type node */
if (n->has_not_operator == true) {
(void)flat_tree->append("!");
nodes.push_back((PolicyFilterNode *)n->right); // 将右子节点指针添加到nodes向量中
nodes.push_back((PolicyFilterNode *)n->left); // 将左子节点指针添加到nodes向量中
} else if (!strcmp(n->node_type, "filter")) { // value type node 值类型节点
if (n->has_not_operator == true) { // 如果节点具有not操作符
(void)flat_tree->append("!"); // 向输出字符串追加!
}
(void)flat_tree->append(n->filter_type);
(void)flat_tree->append("[");
List *filter_item_objects = (List *) n->values;
ListCell *filter_obj = NULL;
foreach(filter_obj, filter_item_objects) {
const char *filter_value = (const char *)(((Value*)lfirst(filter_obj))->val.str);
if (!verify_ip_role_app(n->filter_type, filter_value, flat_tree)) {
return false;
(void)flat_tree->append(n->filter_type); // 向输出字符串追加节点的filter_type属性
(void)flat_tree->append("["); // 向输出字符串追加[
List *filter_item_objects = (List *)n->values; // 获取节点的值列表
ListCell *filter_obj = NULL; // 定义值列表节点指针
foreach (filter_obj, filter_item_objects) { // 遍历值列表中的每个节点
const char *filter_value =
(const char *)(((Value *)lfirst(filter_obj))->val.str); // 获取当前节点的值并转为字符串
if (!verify_ip_role_app(n->filter_type, filter_value,
flat_tree)) { // 验证节点的filter_type和filter_value是否有效
return false; // 返回false表示处理失败
}
(void)flat_tree->append(",");
(void)flat_tree->append(","); // 向输出字符串追加逗号
}
if (flat_tree->back() == ',') {
flat_tree->pop_back();
if (flat_tree->back() == ',') { // 如果输出字符串的最后一个字符是逗号
flat_tree->pop_back(); // 移除输出字符串的最后一个字符(逗号)
}
(void)flat_tree->append("]");
(void)flat_tree->append("]"); // 向输出字符串追加]
}
}
}
return true;
return true; // 返回true表示成功处理策略过滤器
}
bool scan_to_delete_from_relation(long long row_id, Relation relation, unsigned int index_id)
{
if (relation == NULL) {
if (relation == NULL) { // 如果关系为空则返回false
return false;
}
ScanKeyData skey;
/* Find the row to delete. */
ScanKeyInit(&skey,
ObjectIdAttributeNumber,
BTEqualStrategyNumber, F_OIDEQ,
ObjectIdGetDatum(row_id));
// 通过指定的行ID构建扫描键
ScanKeyData skey;
ScanKeyInit(&skey, ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(row_id));
// 开始扫描关系表
SysScanDesc tgscan = systable_beginscan(relation, index_id, true, NULL, 1, &skey);
// 获取匹配的行
HeapTuple tup = systable_getnext(tgscan);
if (!HeapTupleIsValid(tup)) {
if (!HeapTupleIsValid(tup)) { // 如果获取失败则结束扫描并返回false
systable_endscan(tgscan);
return false;
}
/* Delete the label tuple */
// 删除对应的元组
simple_heap_delete(relation, &tup->t_self);
// 结束扫描并返回true表示成功删除行
systable_endscan(tgscan);
return true;
}
void construct_resource_name(const RangeVar *rel, gs_stl::gs_string *target_name_s)
{
if (rel->catalogname) {
if (rel->catalogname) { // 如果关系存在目录名
// 向目标名称字符串追加目录名
(void)target_name_s->append((const char *)rel->catalogname);
target_name_s->push_back('.');
target_name_s->push_back('.'); // 向目标名称字符串追加点号(表示目录和模式之间的分隔符)
}
if (rel->schemaname) {
if (rel->schemaname) { // 如果关系存在模式名
// 向目标名称字符串追加模式名
(void)target_name_s->append((const char *)rel->schemaname);
target_name_s->push_back('.');
target_name_s->push_back('.'); // 向目标名称字符串追加点号(表示模式和表名之间的分隔符)
}
if (rel->relname) {
if (rel->relname) { // 如果关系存在表名
// 向目标名称字符串追加表名
(void)target_name_s->append(rel->relname);
}
}
/**
* Check if current app is valid or not.
*/
bool verify_app_filter(const char* obj_value)
{
// 检查字符串长度是否为0
if (strlen(obj_value) == 0) {
// 如果长度为0报错并返回false
ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("app: [%s] is invalid", obj_value)));
return false;
}
/* The first character id numbers or dollar */
// 获取字符串的第一个字符
char c = obj_value[0];
// 如果第一个字符是数字或者美元符号认为不合法报错并返回false
if ((c >= '0' && c <= '9') || c == '$') {
ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("app: [%s] is invalid", obj_value)));
return false;
}
// 获取字符串的长度
int len = strlen(obj_value);
// 遍历字符串的每个字符
for (int i = 0; i < len; i++) {
c = obj_value[i];
// 如果字符是字母、数字、下划线或美元符号,继续下一次循环
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '$') {
continue;
} else {
// 否则认为不合法报错并返回false
ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("app: [%s] is invalid", obj_value)));
return false;
}
}
// 字符串通过所有验证规则认为合法返回true
return true;
}
/**
* Check the effective of the filter information.
* @ obj_type : filter type, which inlcudes IP, ROLES, and APPS.
* @ obj_value : the actual filter information.
* @ return_value : record about the filter information.
*/
bool verify_ip_role_app(const char* obj_type, const char* obj_value, gs_stl::gs_string *return_value)
bool verify_ip_role_app(const char *obj_type, const char *obj_value, gs_stl::gs_string *return_value)
{
// 检查对象类型是否为 "ip"
if (!strcasecmp(obj_type, "ip")) {
const char* check_value = obj_value;
const char *check_value = obj_value;
// 检查 IP 范围是否合法
if (!IPRange::is_range_valid(check_value)) {
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("ip range: [%s] is invalid, please identify", obj_value)));
// 如果不合法报错并返回false
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("ip range: [%s] is invalid, please identify", obj_value)));
return false;
}
// 将合法的 IP 范围添加到返回值中
(void)return_value->append(check_value);
return true;
} else if (!strcasecmp(obj_type, "roles")) {
}
// 检查对象类型是否为 "roles"
else if (!strcasecmp(obj_type, "roles")) {
// 获取角色的 Oid
Oid uid = get_role_oid(obj_value, true);
// 检查角色是否有效
if (!OidIsValid(uid)) {
// 如果无效报错并返回false
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("role: [%s] is invalid", obj_value)));
return false;
}
char buffer[64]; /* buffer to store the oid int as string. 64 is the max length of oid. */
// 将 Oid 转换为字符串
int nRet = snprintf_s(buffer, sizeof(buffer), sizeof(buffer) - 1, "%d", uid);
securec_check_ss(nRet, "\0", "\0");
// 将转换后的字符串添加到返回值中
(void)return_value->append(buffer);
return true;
} else if (!strcasecmp(obj_type, "app")) {
}
// 检查对象类型是否为 "app"
else if (!strcasecmp(obj_type, "app")) {
// 验证应用程序过滤器是否合法
bool is_valid_app = verify_app_filter(obj_value);
if (!is_valid_app) {
// 如果不合法返回false
return false;
}
}
// 将对象值添加到返回值中
(void)return_value->append(obj_value);
return true;
}
static bool add_privileges_access(const char *action_type, const char *label_name,
privileges_access_set *actions, const policy_labels_map *existing_labels, const GsPolicyStruct *policy,
gs_stl::gs_string *err_msg)
static bool add_privileges_access(const char *action_type, const char *label_name, privileges_access_set *actions,
const policy_labels_map *existing_labels, const GsPolicyStruct *policy,
gs_stl::gs_string *err_msg)
{
// 创建一个 PgPolicyPrivilegesAccessStruct 对象
PgPolicyPrivilegesAccessStruct item;
item.m_type = action_type;
item.m_label_name = label_name;
item.m_policy_oid = policy->m_id;
/* validate that such label exists */
// 验证给定的标签是否存在于 existing_labels 中
if (existing_labels->find(label_name) == existing_labels->end()) {
// 如果标签不存在生成错误消息并返回false
err_msg->clear();
(void)err_msg->append("Trying to add/remove privilege/access [");
(void)err_msg->append(action_type);
@ -343,89 +396,109 @@ static bool add_privileges_access(const char *action_type, const char *label_nam
(void)err_msg->append("]");
return false;
}
// 将 PgPolicyPrivilegesAccessStruct 对象插入到 actions 中
(void)actions->insert(item);
return true;
}
bool handle_target(ListCell *target,
int opt_type,
bool is_add,
gs_stl::gs_string *err_msg,
bool handle_target(ListCell *target, int opt_type, bool is_add, gs_stl::gs_string *err_msg,
privileges_access_set *access_to_add, privileges_access_set *access_to_remove,
privileges_access_set *privs_to_add, privileges_access_set *privs_to_remove,
const policy_labels_map *existing_labels,
const GsPolicyStruct *policy, const char *acc_action_type)
const policy_labels_map *existing_labels, const GsPolicyStruct *policy, const char *acc_action_type)
{
bool ret = false;
switch (opt_type) {
// 处理 POLICY_OPT_PRIVILEGES 选项
case POLICY_OPT_PRIVILEGES: {
RangeVar *rel = (RangeVar*)lfirst(target);
RangeVar *rel = (RangeVar *)lfirst(target);
gs_stl::gs_string target_name_s;
construct_resource_name((const RangeVar*)rel, &target_name_s);
construct_resource_name((const RangeVar *)rel, &target_name_s);
if (is_add) {
ret = add_privileges_access(acc_action_type, target_name_s.c_str(), privs_to_add,
existing_labels, policy, err_msg);
// 调用 add_privileges_access 函数,添加权限信息到 privs_to_add
ret = add_privileges_access(acc_action_type, target_name_s.c_str(), privs_to_add, existing_labels,
policy, err_msg);
} else {
ret = add_privileges_access(acc_action_type, target_name_s.c_str(), privs_to_remove, existing_labels,
// 调用 add_privileges_access 函数,添加权限信息到 privs_to_remove
ret = add_privileges_access(acc_action_type, target_name_s.c_str(), privs_to_remove, existing_labels,
policy, err_msg);
}
}
break;
} break;
// 处理 POLICY_OPT_ACCESS 选项
case POLICY_OPT_ACCESS: {
RangeVar *rel = (RangeVar*)lfirst(target);
RangeVar *rel = (RangeVar *)lfirst(target);
gs_stl::gs_string target_name_s;
construct_resource_name((const RangeVar*)rel, &target_name_s);
construct_resource_name((const RangeVar *)rel, &target_name_s);
if (is_add) {
// 调用 add_privileges_access 函数,添加访问控制信息到 access_to_add
ret = add_privileges_access(acc_action_type, target_name_s.c_str(), access_to_add, existing_labels,
policy, err_msg);
} else {
// 调用 add_privileges_access 函数,添加访问控制信息到 access_to_remove
ret = add_privileges_access(acc_action_type, target_name_s.c_str(), access_to_remove, existing_labels,
policy, err_msg);
}
}
break;
// : this is not handled here...
} break;
// 默认情况下不处理
default:
break;
}
return ret;
}
static bool parse_values(const gs_stl::gs_string logical_expr_str, int *offset, const char* obj_type)
static bool parse_values(const gs_stl::gs_string logical_expr_str, int *offset, const char *obj_type)
{
/*
logical_expr_str
offset
obj_type
*/
std::size_t found = gs_stl::gs_string::npos;
// 创建一个缓冲区
char buff[512] = {0};
// 在逻辑表达式字符串中查找下一个 ']' 的位置,限制范围的末尾
size_t limit_pos = logical_expr_str.find(']', *offset);
// 临时存储解析结果的字符串
gs_stl::gs_string tmp_res;
// 解析是否成功的标志
bool parsed = true;
// 过滤器有效性的标志
bool filter_valid = false;
int nRet;
/* not finding last ']' means error */
// 如果没有找到最后一个 ']',则表示错误
if (limit_pos == gs_stl::gs_string::npos) {
ereport(ERROR,
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("filter: [%s] is invalid", logical_expr_str.c_str())));
ereport(ERROR,
(errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("filter: [%s] is invalid", logical_expr_str.c_str())));
return false;
}
// 在逗号 ',' 出现之前以及限制范围内,循环解析
while ((found = logical_expr_str.find(',', *offset)) != gs_stl::gs_string::npos && found < limit_pos) {
// 将解析出的部分拷贝到缓冲区中
nRet = snprintf_s(buff, sizeof(buff), sizeof(buff) - 1, "%.*s", (int)(found - *offset),
logical_expr_str.c_str() + *offset);
logical_expr_str.c_str() + *offset);
securec_check_ss(nRet, "\0", "\0");
// 验证 IP、角色、应用程序等返回过滤器的有效性并将结果存储在 tmp_res 中
filter_valid = verify_ip_role_app(obj_type, buff, &tmp_res);
// 更新 parsed 的值,如果有一个解析失败,则 parsed 为 false
parsed = parsed && filter_valid;
*offset = found + 1;
}
// 如果解析还未结束,处理剩余部分
if (*offset < (int)limit_pos) {
nRet = snprintf_s(buff, sizeof(buff), sizeof(buff) - 1, "%.*s", (int)(limit_pos - *offset),
logical_expr_str.c_str() + *offset);
logical_expr_str.c_str() + *offset);
securec_check_ss(nRet, "\0", "\0");
filter_valid = verify_ip_role_app(obj_type, buff, &tmp_res);
parsed = parsed && filter_valid;
*offset = limit_pos + 1;
return parsed;
}
/* getting here means error */
// 执行到这里表示错误,抛出错误信息并返回 false
ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("filter: [%s] is invalid", logical_expr_str.c_str())));
return false;
}
@ -435,85 +508,116 @@ bool validate_logical_expression(const gs_stl::gs_string logical_expr_str, int *
{
int logical_expr_len = logical_expr_str.size();
while (*offset < logical_expr_len) {
while (*offset < logical_expr_len) { // 循环遍历表达式字符串
/* AND/OR node */
if ((logical_expr_str[*offset] == '*') || (logical_expr_str[*offset] == '+')) {
(*offset)++;
return (validate_logical_expression(logical_expr_str, offset) /* go left */
&& validate_logical_expression(logical_expr_str, offset)); /* go right */
} else if (logical_expr_str[*offset] == '!') { /* NOT operator */
(*offset)++;
} else if (logical_expr_str[*offset] == 'i') { /* IP filter node */
*offset += 3; /* 3 : skip 'ip[' */
return parse_values(logical_expr_str, offset, "ip");
} else if (logical_expr_str[*offset] == 'r') { /* ROLE filter node */
*offset += 6; /* 6 : skip 'roles[' */
return parse_values(logical_expr_str, offset, "roles");
} else if (logical_expr_str[*offset] == 'a') { /* APPLICATION filter node */
*offset += 4; /* 4 : skip 'app[' */
return parse_values(logical_expr_str, offset, "app");
if ((logical_expr_str[*offset] == '*') || (logical_expr_str[*offset] == '+')) { // 如果当前字符为*或+
(*offset)++; // 偏移量自增1
return (validate_logical_expression(logical_expr_str, offset) /* go left */
&& validate_logical_expression(logical_expr_str, offset)); /* go right */
// 递归调用validate_logical_expression分别处理左右两个子表达式并将结果进行AND/OR操作返回
} else if (logical_expr_str[*offset] == '!') { /* NOT operator */ // 如果当前字符为!
(*offset)++; // 偏移量自增1
} else if (logical_expr_str[*offset] == 'i') { /* IP filter node */ // 如果当前字符为i
*offset += 3; /* 3 : skip 'ip[' */ // 偏移量增加3跳过'ip['
return parse_values(logical_expr_str, offset, "ip"); // 调用parse_values函数解析IP值并返回结果
} else if (logical_expr_str[*offset] == 'r') { /* ROLE filter node */ // 如果当前字符为r
*offset += 6; /* 6 : skip 'roles[' */ // 偏移量增加6跳过'roles['
return parse_values(logical_expr_str, offset, "roles"); // 调用parse_values函数解析角色值并返回结果
} else if (logical_expr_str[*offset] == 'a') { /* APPLICATION filter node */ // 如果当前字符为a
*offset += 4; /* 4 : skip 'app[' */ // 偏移量增加4跳过'app['
return parse_values(logical_expr_str, offset, "app"); // 调用parse_values函数解析应用程序值并返回结果
}
}
return false;
return false; // 如果表达式字符串已经遍历完毕则返回false
}
void get_session_ip(char *session_ip, int len)
{
// 检查提供的缓冲区长度是否足够
if (len < MAX_IP_LEN) {
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("length of session ip buffer should more than 128")));
}
struct sockaddr* remote_addr = (struct sockaddr *)&u_sess->proc_cxt.MyProcPort->raddr.addr;
// 获取远程地址
struct sockaddr *remote_addr = (struct sockaddr *)&u_sess->proc_cxt.MyProcPort->raddr.addr;
// 错误码变量
errno_t rc = EOK;
/* parse the remote ip address */
// 解析远程IP地址
if (AF_UNIX == remote_addr->sa_family) {
// 如果远程地址是UNIX域套接字(AF_UNIX)将session_ip设置为"local"
char *localstr = "local";
rc = memcpy_s(session_ip, len, localstr, strlen(localstr));
securec_check(rc, "\0", "\0");
} else {
// 否则调用get_client_ip函数获取客户端IP地址并将结果存储在session_ip中
get_client_ip(remote_addr, session_ip);
}
}
void get_client_ip(const struct sockaddr* remote_addr, char *ip_str)
void get_client_ip(const struct sockaddr *remote_addr, char *ip_str)
{
/* parse the remote ip address */
/* 解析远程IP地址 */
// 如果远程地址为IPv6AF_INET6使用inet_ntop函数将IPv6地址转换为字符串形式
if (AF_INET6 == remote_addr->sa_family) {
(void)inet_ntop(AF_INET6, &((struct sockaddr_in6*)remote_addr)->sin6_addr, ip_str, MAX_IP_LEN - 1);
} else if (AF_INET == remote_addr->sa_family) {
(void)inet_ntop(AF_INET, &((struct sockaddr_in*)remote_addr)->sin_addr, ip_str, MAX_IP_LEN - 1);
(void)inet_ntop(AF_INET6, &((struct sockaddr_in6 *)remote_addr)->sin6_addr, ip_str, MAX_IP_LEN - 1);
}
// 如果远程地址为IPv4AF_INET使用inet_ntop函数将IPv4地址转换为字符串形式
else if (AF_INET == remote_addr->sa_family) {
(void)inet_ntop(AF_INET, &((struct sockaddr_in *)remote_addr)->sin_addr, ip_str, MAX_IP_LEN - 1);
}
}
bool is_database_valid(const char* dbname)
bool is_database_valid(const char *dbname)
{
// 检查数据库名是否为空
if (dbname == NULL) {
return false;
}
// 获取数据库的OID对象标识符
Oid db_oid = get_database_oid(dbname, false);
// 检查数据库的OID是否有效
if (OidIsValid(db_oid)) {
return true;
}
// 如果数据库的OID无效, 返回false
return false;
}
ResourceOwnerData* create_temp_resourceowner()
ResourceOwnerData *create_temp_resourceowner()
{
ResourceOwner tmpOwner = ResourceOwnerCreate(t_thrd.utils_cxt.CurrentResourceOwner,
"CheckUserOid", THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY));
// 创建临时资源拥有者
ResourceOwner tmpOwner = ResourceOwnerCreate(t_thrd.utils_cxt.CurrentResourceOwner, "CheckUserOid",
THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY));
// 保存当前资源拥有者
ResourceOwner currentOwner = t_thrd.utils_cxt.CurrentResourceOwner;
// 将当前资源拥有者切换为临时资源拥有者
t_thrd.utils_cxt.CurrentResourceOwner = tmpOwner;
// 返回当前资源拥有者
return currentOwner;
}
void release_temp_resourceowner(ResourceOwnerData* resource_owner)
void release_temp_resourceowner(ResourceOwnerData *resource_owner)
{
// 保存当前资源拥有者到临时变量
ResourceOwner tmpOwner = t_thrd.utils_cxt.CurrentResourceOwner;
// 释放锁之前需要释放的资源
ResourceOwnerRelease(tmpOwner, RESOURCE_RELEASE_BEFORE_LOCKS, true, true);
// 释放锁占用的资源
ResourceOwnerRelease(tmpOwner, RESOURCE_RELEASE_LOCKS, true, true);
// 释放锁之后需要释放的资源
ResourceOwnerRelease(tmpOwner, RESOURCE_RELEASE_AFTER_LOCKS, true, true);
// 恢复之前的资源拥有者
t_thrd.utils_cxt.CurrentResourceOwner = resource_owner;
// 删除临时资源拥有者
ResourceOwnerDelete(tmpOwner);
}

View File

@ -29,65 +29,71 @@
namespace gs_stl {
#define MIN_STR_CAPACITY 16
MemoryContext GetStringMemory()
MemoryContext GetStringMemory()//获取字符串内存的内存上下文
{
if (t_thrd.security_policy_cxt.StringMemoryContext == NULL) {
if (t_thrd.security_policy_cxt.StringMemoryContext == NULL) {//没有分配内存上下文,需要进行内存分配
t_thrd.security_policy_cxt.StringMemoryContext =
AllocSetContextCreate(THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY), "StringMemory",
ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE);
/*
#define ALLOCSET_DEFAULT_MINSIZE 0
#define ALLOCSET_DEFAULT_INITSIZE (8 * 1024)
#define ALLOCSET_DEFAULT_MAXSIZE (8 * 1024 * 1024)
*/
}
return t_thrd.security_policy_cxt.StringMemoryContext;
}
void DeleteStringMemory()
void DeleteStringMemory()//释放之前分配的字符串内存的内存上下文
{
if (t_thrd.security_policy_cxt.StringMemoryContext != NULL) {
MemoryContextDelete(t_thrd.security_policy_cxt.StringMemoryContext);
t_thrd.security_policy_cxt.StringMemoryContext = NULL;
if (t_thrd.security_policy_cxt.StringMemoryContext != NULL) {//已经分配了字符串内存的内存上下文,需要进行内存释放
MemoryContextDelete(t_thrd.security_policy_cxt.StringMemoryContext);//删除之前分配的内存上下文。
t_thrd.security_policy_cxt.StringMemoryContext = NULL;//表示内存上下文已被释放
}
}
MemoryContext GetVectorMemory()
MemoryContext GetVectorMemory()//获取向量内存的内存上下文
{
if (t_thrd.security_policy_cxt.VectorMemoryContext == NULL) {
t_thrd.security_policy_cxt.VectorMemoryContext =
AllocSetContextCreate(THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY), "VectorMemory",
ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE);
ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE);//创建一个内存上下文,传入了一些参数
}
return t_thrd.security_policy_cxt.VectorMemoryContext;
}
void DeleteVectorMemory()
void DeleteVectorMemory()//释放之前分配的向量内存的内存上下文
{
if (t_thrd.security_policy_cxt.VectorMemoryContext != NULL) {
MemoryContextDelete(t_thrd.security_policy_cxt.VectorMemoryContext);
MemoryContextDelete(t_thrd.security_policy_cxt.VectorMemoryContext);//删除之前分配的内存上下文。
t_thrd.security_policy_cxt.VectorMemoryContext = NULL;
}
}
MemoryContext GetMapMemory()
MemoryContext GetMapMemory()//获取映射内存的内存上下文
{
if (!t_thrd.security_policy_cxt.MapMemoryContext) {
if (!t_thrd.security_policy_cxt.MapMemoryContext) {//没有分配内存上下文,需要进行内存分配。
t_thrd.security_policy_cxt.MapMemoryContext = AllocSetContextCreate(TopMemoryContext, "MapMemory",
ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE);
}
return t_thrd.security_policy_cxt.MapMemoryContext;
}
void DeleteMapMemory()
void DeleteMapMemory()//释放之前分配的映射内存的内存上下文
{
if (t_thrd.security_policy_cxt.MapMemoryContext) {
MemoryContextDelete(t_thrd.security_policy_cxt.MapMemoryContext);
MemoryContextDelete(t_thrd.security_policy_cxt.MapMemoryContext);//删除之前分配的内存上下文
t_thrd.security_policy_cxt.MapMemoryContext = nullptr;
}
}
void *_HashMapAllocFunc(Size request)
void *_HashMapAllocFunc(Size request)//在映射内存上下文中分配指定大小的内存空间
{
return MemoryContextAlloc(GetMapMemory(), request);
return MemoryContextAlloc(GetMapMemory(), request);//传入映射内存的内存上下文和请求的内存大小,来分配内存空间
}
MemoryContext GetSetMemory()
MemoryContext GetSetMemory()//存储了集合内存的内存上下文
{
if (!t_thrd.security_policy_cxt.SetMemoryContext) {
t_thrd.security_policy_cxt.SetMemoryContext = AllocSetContextCreate(TopMemoryContext, "SetMemory",
@ -96,7 +102,7 @@ MemoryContext GetSetMemory()
return t_thrd.security_policy_cxt.SetMemoryContext;
}
void DeleteSetMemory()
void DeleteSetMemory()//释放之前分配的集合内存的内存上下文
{
if (t_thrd.security_policy_cxt.SetMemoryContext) {
MemoryContextDelete(t_thrd.security_policy_cxt.SetMemoryContext);
@ -104,17 +110,17 @@ void DeleteSetMemory()
}
}
void *_HashSetAllocFunc(Size request)
void *_HashSetAllocFunc(Size request)//在集合内存上下文中分配指定大小的内存空间
{
return MemoryContextAlloc(GetSetMemory(), request);
}
int matchStr(const void *key1, const void *key2, Size keysize)
int matchStr(const void *key1, const void *key2, Size keysize)//比较两个字符串是否匹配。
{
return strncasecmp((const char *)key1, (const char *)key2, keysize - 1);
return strncasecmp((const char *)key1, (const char *)key2, keysize - 1);//比较两个字符串的前"keysize - 1"个字符,忽略大小写
}
int gs_stringCompareKeyFunc(const void *keyA, const void *keyB)
int gs_stringCompareKeyFunc(const void *keyA, const void *keyB)//比较两个字符串键的大小
{
if (*(const gs_string *)keyA < *(const gs_string *)keyB) {
return -1;
@ -126,7 +132,7 @@ int gs_stringCompareKeyFunc(const void *keyA, const void *keyB)
}
// string implementation
inline bool gs_string::InitBuff(const char *str, size_t len)
inline bool gs_string::InitBuff(const char *str, size_t len)//初始化字符串缓冲区。
{
if (m_buff == NULL) {
size_t init_len = (len > 0) ? (len + 1) : (strlen(str) + 1);
@ -139,12 +145,12 @@ inline bool gs_string::InitBuff(const char *str, size_t len)
}
return false;
}
gs_string::gs_string(const char *str, size_t len) : m_buff(NULL), m_len(0), m_capacity(0)
{
gs_string::gs_string(const char *str, size_t len) : m_buff(NULL), m_len(0), m_capacity(0)// 构造函数
{//创建字符串对象
(void)InitBuff(str, len);
}
gs_string::~gs_string()
gs_string::~gs_string()//析构函数,销毁字符串对象
{
/*
* Note that: container destruction will be called by system depending on
@ -162,13 +168,13 @@ gs_string::~gs_string()
}
gs_string::gs_string(const gs_string &arg) : m_buff(NULL), m_len(0), m_capacity(0)
{
{//拷贝构造函数
operator = (arg);
}
gs_string &gs_string::operator = (const gs_string &arg)
gs_string &gs_string::operator = (const gs_string &arg)//赋值运算符重载函数
{
if (&arg == this) {
if (&arg == this) {//传入的对象地址与当前对象地址相同
return *this;
}
@ -187,7 +193,7 @@ gs_string &gs_string::operator = (const gs_string &arg)
return *this;
}
int gs_string::operator - (const gs_string &arg) const
int gs_string::operator - (const gs_string &arg) const//减法运算符重载函数实现
{
if (this == &arg) {
return 0;
@ -201,7 +207,7 @@ int gs_string::operator - (const gs_string &arg) const
return 0;
}
gs_string &gs_string::append(const gs_string &str)
gs_string &gs_string::append(const gs_string &str)//在字符串末尾追加字符串
{
return append(str.c_str(), str.size());
}
@ -220,7 +226,7 @@ gs_string &gs_string::append(const char *str, size_t len)
return *this;
}
void gs_string::push_back(char ch)
void gs_string::push_back(char ch)//字符串末尾添加一个字符
{
char t_chr[2] = {0};
t_chr[1] = ch;
@ -233,14 +239,14 @@ void gs_string::push_back(char ch)
}
}
void gs_string::pop_back()
void gs_string::pop_back()//字符串末尾删除一个字符
{
if (m_len > 0) {
m_buff[--m_len] = '\0';
}
}
char gs_string::operator[](int idx) const
char gs_string::operator[](int idx) const//访问字符串中的指定位置的字符
{
if (idx > (int)m_len) {
return '\0';
@ -248,7 +254,7 @@ char gs_string::operator[](int idx) const
return m_buff[idx];
}
void gs_string::clear()
void gs_string::clear()//清空字符串
{
if (m_buff != NULL) {
m_buff[0] = '\0';
@ -256,7 +262,7 @@ void gs_string::clear()
}
}
size_t gs_string::find(char arg, size_t start) const
size_t gs_string::find(char arg, size_t start) const//find
{
for (; start < m_len; ++start) {
if (m_buff[start] == arg) {
@ -266,7 +272,7 @@ size_t gs_string::find(char arg, size_t start) const
return npos;
}
char gs_string::back() const
char gs_string::back() const//返回字符串的最后一个字符。
{
if (m_len > 0) {
return m_buff[m_len - 1];
@ -274,7 +280,7 @@ char gs_string::back() const
return m_buff[0];
}
gs_string gs_string::substr(size_t pos, size_t len) const
gs_string gs_string::substr(size_t pos, size_t len) const//返回从指定位置起始的指定长度子字符串。
{
if ((pos + len) < m_len) {
return gs_string((const char *)(m_buff + pos), len);
@ -285,7 +291,7 @@ gs_string gs_string::substr(size_t pos, size_t len) const
return gs_string((const char *)m_buff, m_len);
}
gs_string &gs_string::replace(size_t pos, size_t len, const char *s)
gs_string &gs_string::replace(size_t pos, size_t len, const char *s)//替换从指定位置开始的指定长度的子字符串。
{
if (pos < m_len) {
size_t rep_len = strlen(s) + 1;
@ -315,7 +321,7 @@ gs_string &gs_string::replace(size_t pos, size_t len, const char *s)
return *this;
}
void gs_string::erase(size_t pos, size_t len)
void gs_string::erase(size_t pos, size_t len)//删除从指定位置开始的指定长度的子字符串。
{
if (m_len == 0 || (pos >= m_len)) {
return;
@ -332,7 +338,7 @@ void gs_string::erase(size_t pos, size_t len)
m_buff[m_len] = '\0';
}
bool gs_string::operator == (const gs_string &arg) const
bool gs_string::operator == (const gs_string &arg) const//比较两个字符串对象是否相等
{
if (m_len != arg.m_len) {
return false;
@ -343,17 +349,17 @@ bool gs_string::operator == (const gs_string &arg) const
return m_len == 0;
}
bool gs_string::operator < (const gs_string &arg) const
bool gs_string::operator < (const gs_string &arg) const//比较两个字符串对象大小关系。
{
return strcasecmp(m_buff, arg.m_buff) < 0;
}
inline char *gs_string::AllocFunc(size_t _size) const
inline char *gs_string::AllocFunc(size_t _size) const//用于分配字符串缓冲区
{
return (char *)MemoryContextAlloc(GetStringMemory(), _size);
}
inline char *gs_string::ReallocFunc(size_t _size)
inline char *gs_string::ReallocFunc(size_t _size)//用于重新分配字符串缓冲区
{
m_capacity = _size;

File diff suppressed because it is too large Load Diff

View File

@ -22,8 +22,8 @@
*
* ---------------------------------------------------------------------------------------
*/
#include <netinet/in.h>
#include <arpa/inet.h>
#include <netinet/in.h> //用于网络编程的标准头文件之一。该头文件提供了与网络相关的结构体、宏定义和函数原型。
#include <arpa/inet.h> //提供了一些网络编程相关的函数和数据结构的定义,包括 IP 地址转换、网络字节序和主机字节序之间的转换等操作
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
@ -36,13 +36,23 @@
using namespace std;
#define IPRANGE_IS_IPV4(ip) ((ip).ip_32.b == 0x0000FFFF)
//判断给定的IP是否为IPv4地址
#define IPRANGE_IS_IPV4(ip) ((ip).ip_32.b == 0x0000FFFF) //通过比较IP的低32位的值是否等于0x0000FFFF来判断 如果相等则表示该IP是IPv4地址
/*
* Mask from Look-Up Table
* MASK_FROM_LUT std::vector
* MASK_FROM_LUT 321
*
* 0xFFFFFFFF321
*
* IPv4地址进行子网掩码操作
*/
std::vector<uint32_t> MASK_FROM_LUT {
(uint32_t)0xFFFFFFFF,
(uint32_t)0xFFFFFFFF << 31,
(uint32_t)0xFFFFFFFF << 30,
(uint32_t)0xFFFFFFFF << 29,
(uint32_t)0xFFFFFFFF, // MASK_FROM_LUT[0]
(uint32_t)0xFFFFFFFF << 31, // MASK_FROM_LUT[1] 0xFFFFFFFE
(uint32_t)0xFFFFFFFF << 30, // MASK_FROM_LUT[2] 0xFFFFFFFC
(uint32_t)0xFFFFFFFF << 29, // MASK_FROM_LUT[3] 0xFFFFFFF8
(uint32_t)0xFFFFFFFF << 28,
(uint32_t)0xFFFFFFFF << 27,
(uint32_t)0xFFFFFFFF << 26,
@ -70,15 +80,24 @@ std::vector<uint32_t> MASK_FROM_LUT {
(uint32_t)0xFFFFFFFF << 4,
(uint32_t)0xFFFFFFFF << 3,
(uint32_t)0xFFFFFFFF << 2,
(uint32_t)0xFFFFFFFF << 1,
(uint32_t)0xFFFFFFFF << 1, // MASK_FROM_LUT[31] 0x80000000
(uint32_t)0xFFFFFFFF,
};
/*
* Mask to Look-Up Table
* MASK_TO_LUT std::vector
* MASK_TO_LUT 132
*
* 0xFFFFFFFF321
*
* IPv4地址进行子网掩码操作
*/
std::vector<uint32_t> MASK_TO_LUT {
(uint32_t)0,
~((uint32_t)0xFFFFFFFF << 31),
~((uint32_t)0xFFFFFFFF << 30),
~((uint32_t)0xFFFFFFFF << 29),
(uint32_t)0, // MASK_TO_LUT[0] 0x00000000
~((uint32_t)0xFFFFFFFF << 31), // MASK_TO_LUT[1] 0x00000001
~((uint32_t)0xFFFFFFFF << 30), // MASK_TO_LUT[2] 0x00000003
~((uint32_t)0xFFFFFFFF << 29), // MASK_TO_LUT[3] 0x00000007
~((uint32_t)0xFFFFFFFF << 28),
~((uint32_t)0xFFFFFFFF << 27),
~((uint32_t)0xFFFFFFFF << 26),
@ -106,12 +125,17 @@ std::vector<uint32_t> MASK_TO_LUT {
~((uint32_t)0xFFFFFFFF << 4),
~((uint32_t)0xFFFFFFFF << 3),
~((uint32_t)0xFFFFFFFF << 2),
~((uint32_t)0xFFFFFFFF << 1),
~((uint32_t)0xFFFFFFFF << 1), // MASK_TO_LUT[31] 0x7FFFFFFF
(uint32_t)0xFFFFFFFF
};
const IPV6 localhost_ipv6 ({0x0000000000000001, 0x0000000000000000});
const IPV6 localhost_ipv4 ({0xffff7f000001, 0x0});
/*
* IPv6 IPv4
* IPv6 128 4*8
* IPv4 32 8
*/
const IPV6 localhost_ipv6 ({0x0000000000000001, 0x0000000000000000});//2*16
const IPV6 localhost_ipv4 ({0xffff7f000001, 0x0}); //0x0000ffff 是 IPv4 兼容地址的前缀 0x7f000001是IPv4本地回环地址
#define IP_MAX_LEN 128
@ -123,77 +147,119 @@ IPRange::~IPRange()
{
}
/*
* IP
*
*/
//接受一个指向 IPV6 对象的指针 ip
//和一个指向 sockaddr_in6 结构体的指针 sa
void IPRange::net_ipv6_to_host_order(IPV6 *ip, const struct sockaddr_in6 *sa) const
{
IPV6 tmp_ip;
//使用 memcpy_s 函数将 sa->sin6_addr 的内容拷贝到 tmp_ip.ip_64 中(拷贝的字节数为 sizeof(tmp_ip.ip_64)
// memcpy_s 是一个安全版本的内存拷贝函数,用于防止内存溢出和错误的内存操作
int rc = memcpy_s(&(tmp_ip.ip_64), sizeof(tmp_ip.ip_64), &(sa->sin6_addr), sizeof(tmp_ip.ip_64));
securec_check(rc, "\0", "\0");
securec_check(rc, "\0", "\0"); //检查并确保字符串操作不会导致缓冲区溢出或无效的操作
/*
* ntohl tmp_ip.ip_32
*
* ip
*/
ip->ip_32.a = ntohl(tmp_ip.ip_32.d);
ip->ip_32.b = ntohl(tmp_ip.ip_32.c);
ip->ip_32.c = ntohl(tmp_ip.ip_32.b);
ip->ip_32.d = ntohl(tmp_ip.ip_32.a);
/*
* ntohl
* "network to host long"
* <arpa/inet.h> POSIX
* 32
* uint32_t ntohl(uint32_t netlong);
*/
}
// 接受一个指向 IPV6 对象的指针 ip
// 和一个指向 in_addr 结构体的指针 addr
void IPRange::net_ipv4_to_host_order(IPV6 *ip, const struct in_addr *addr) const
{
/*
* ntohl addr->s_addr ip->ip_32.a
* b c d
*/
ip->ip_32.a = ntohl(addr->s_addr);
ip->ip_32.b = 0x0000FFFF;
ip->ip_32.c = ip->ip_32.d = 0;
}
//IP 地址字符串转换为 IP 结构体的函数
//接受一个指向 'IP 地址字符串' 的 指针 ip_str 和 一个指向 'IPV6 结构体' 的 指针 ip
bool IPRange::str_to_ip(const char* ip_str, IPV6 *ip)
{
struct in_addr addr;
struct sockaddr_in6 sa;
if (inet_pton(AF_INET6, ip_str, &sa.sin6_addr) > 0) {
net_ipv6_to_host_order(ip, &sa);
} else if (inet_pton(AF_INET, ip_str, &addr) > 0) {
/*
* inet_pton
* "Internet Presentation to Network"
* <arpa/inet.h> POSIX
* IP
* int inet_pton(int af, const char *src, void *dst);
* af Address Family AF_INETIPv4 AF_INET6IPv6
* src IP dst
*
* 1
* 0 -1 errno
*/
if (inet_pton(AF_INET6, ip_str, &sa.sin6_addr) > 0) { //有效的 IPv6 地址
net_ipv6_to_host_order(ip, &sa); //将该地址从网络字节序转换为主机字节序,并将结果存储在 ip 中
} else if (inet_pton(AF_INET, ip_str, &addr) > 0) { // 有效的 IPv4 地址
net_ipv4_to_host_order(ip, &addr);
} else {
} else { //传入的字符串格式无效或超出了合法的 IP 范围
/*
* Note that even the format keep the same as ipv6 or ipv4
* still recognize it as invalid ip if ip exceed the valid range
*/
m_err_str = "invalid ip: " + std::string(ip_str);
m_err_str = "invalid ip: " + std::string(ip_str); //将错误信息存储在成员变量 m_err_str 中
return false;
}
return true;
}
// IP 地址范围掩码函数
// 用于 根据给定的 CIDR 前缀长度对 IP 范围进行掩码处理,并更新输入的起始和结束 IP 地址值,以限制范围内的 IP 地址
// 接受一个指向 Range 结构体 的指针 和一个 表示 CIDR 前缀长度 的 无符号短整型参数 cidr
bool IPRange::mask_range(Range *range, unsigned short cidr)
{
if (IPRANGE_IS_IPV4(range->from)) { /* ipv4 */
if (cidr < 1 || cidr > 32) {
if (cidr < 1 || cidr > 32) { //cidr 不在有效的范围内
m_err_str = "invalid cidr for ipv4: " + cidr;
return false;
}
range->from.ip_32.a &= MASK_FROM_LUT[cidr];
range->to.ip_32.a |= MASK_TO_LUT[cidr];
//更新输入的起始和结束 IP 地址值
range->from.ip_32.a &= MASK_FROM_LUT[cidr]; //起始 IP 地址的相应部分 与 掩码 进行 与运算
range->to.ip_32.a |= MASK_TO_LUT[cidr]; //结束 IP 地址的相应部分 与 掩码 进行 或运算
} else { /* ipv6 */
if (cidr < 1 || cidr > 128) {
m_err_str = "invalid cidr for ipv6: " + cidr;
return false;
}
unsigned short complement = cidr % 32; /* the result is less or equal to 31 */
if (cidr > 96) {
//d c b a
if (cidr > 96) { //只处理第一部分
range->from.ip_32.a &= MASK_FROM_LUT[complement];
range->to.ip_32.a |= MASK_TO_LUT[complement];
} else if (cidr > 64) {
} else if (cidr > 64) { //处理前两部分
range->from.ip_32.b &= MASK_FROM_LUT[complement];
range->to.ip_32.b |= MASK_TO_LUT[complement];
range->from.ip_32.a = 0;
range->to.ip_32.a = 0xFFFFFFFF;
} else if (cidr > 32) {
} else if (cidr > 32) { //处理前三部分
range->from.ip_32.c &= MASK_FROM_LUT[complement];
range->to.ip_32.c |= MASK_TO_LUT[complement];
range->from.ip_32.b = 0;
range->to.ip_32.b = 0xFFFFFFFF;
range->from.ip_32.a = 0;
range->to.ip_32.a = 0xFFFFFFFF;
} else {
} else { //处理所有四部分
range->from.ip_32.d &= MASK_FROM_LUT[complement];
range->to.ip_32.d |= MASK_TO_LUT[complement];
range->from.ip_32.c = 0;
@ -211,23 +277,33 @@ bool IPRange::mask_range(Range *range, unsigned short cidr)
* parse the ip with mask into range sturst , format is as below:
* x.x.x.x|x, ptr is the postion of "|"
*/
/*
* IP
* "|" IP IP
* Range new_range
*
* const char *ptr "|"
* range IP
* range_len x.x.x.x|x
*/
bool IPRange::parse_mask(const char* range, size_t range_len, const char *ptr, Range *new_range)
{
if (range_len > 100) {
if (range_len > 100) { //检查范围字符串的长度是否有效小于等于100
m_err_str = "the range string length is not valid: " + range_len;
return false;
}
char first_ip_str[IP_MAX_LEN] = {0};
char mask_ip_str[IP_MAX_LEN] = {0};
size_t first_ip_str_len = ptr - range;
size_t mask_ip_str_len = range_len - 1 - first_ip_str_len;
size_t first_ip_str_len = ptr - range; //源 IP 字符串长度
size_t mask_ip_str_len = range_len - 1 - first_ip_str_len; //掩码 IP 字符串长度
/* copy the first ip */
copy_without_spaces(first_ip_str, sizeof(first_ip_str), range, first_ip_str_len);
/* get the other ip */
copy_without_spaces(mask_ip_str, sizeof(mask_ip_str), ptr + 1, mask_ip_str_len);
IPV6 ip;
IPV6 mask_ip;
//调用str_to_ip函数将字符串表示的 IP 地址转换为 结构体
if (!str_to_ip(first_ip_str, &ip)) {
m_err_str = "failed to convert ip: " + std::string(first_ip_str);
return false;
@ -236,6 +312,7 @@ bool IPRange::parse_mask(const char* range, size_t range_len, const char *ptr, R
m_err_str = "failed to convert mask ip: " + std::string(mask_ip_str);
return false;
}
// 0x0000ffff 是 IPv4 兼容地址的前缀
if (mask_ip.ip_32.b == 0x0000FFFF) { /* ipv4 */
new_range->from.ip_32.a = ip.ip_32.a & mask_ip.ip_32.a;
new_range->from.ip_32.b = 0x0000FFFF;
@ -243,19 +320,21 @@ bool IPRange::parse_mask(const char* range, size_t range_len, const char *ptr, R
new_range->to.ip_32.a = ip.ip_32.a | ~mask_ip.ip_32.a;
new_range->to.ip_32.b = 0x0000FFFF;
new_range->to.ip_64.upper = 0;
} else {
} else {/* ipv6 */
new_range->from = ip & mask_ip;
new_range->to = ip | ~mask_ip;
}
return true;
}
//解析单个IP范围并将结果存储在Range结构体中
bool IPRange::parse_single(const char* range, size_t range_len, Range *new_range)
{
if (range_len > 100) {
m_err_str = "the range string length is not valid: " + range_len;
return false;
}
//buf数组用于存储去除空格后的IP
char buf[IP_MAX_LEN] = {0};
/* copy the ip part to buf */
copy_without_spaces(buf, sizeof(buf), range, range_len);
@ -263,10 +342,12 @@ bool IPRange::parse_single(const char* range, size_t range_len, Range *new_range
m_err_str = "failed to convert ip: " + std::string(buf);
return false;
}
new_range->to = new_range->from;
new_range->to = new_range->from; //IP范围只包含单个IP
return true;
}
//解析带有斜杠表示的CIDR形式的IP范围并将结果存储在Range结构体中
//ptr是指向IP范围字符串中斜杠符号("/")后的数字部分的指针
bool IPRange::parse_slash(const char* range, size_t range_len, const char *ptr, Range *new_range)
{
if (range_len > 100) {
@ -284,10 +365,12 @@ bool IPRange::parse_slash(const char* range, size_t range_len, const char *ptr,
return false;
}
new_range->to = new_range->from;
(void)mask_range(new_range, cidr);
(void)mask_range(new_range, cidr); //调用mask_range函数根据CIDR值对IP范围进行掩码操作将范围限制在指定的子网中
return true;
}
//解析形如 "x.x.x.x-y.y.y.y" 格式的 IP 范围字符串
//ptr指向 IP 范围字符串中短横线 - 的位置
bool IPRange::parse_hyphen(const char* range, size_t range_len, const char *ptr, Range *new_range)
{
if (range_len > 100) {
@ -304,6 +387,7 @@ bool IPRange::parse_hyphen(const char* range, size_t range_len, const char *ptr,
copy_without_spaces(first_ip, sizeof(first_ip), range, first_ip_len);
/* get the other ip */
copy_without_spaces(second_ip, sizeof(second_ip), ptr + 1, second_ip_len);
//str_to_ip IP 地址字符串转换为 IP 结构体
if (!str_to_ip(first_ip, &(new_range->from))) {
m_err_str = "failed to parse ip: " + std::string(first_ip);
return false;
@ -312,7 +396,7 @@ bool IPRange::parse_hyphen(const char* range, size_t range_len, const char *ptr,
m_err_str = "failed to parse ip: " + std::string(second_ip);
return false;
}
if (new_range->from > new_range->to) {
if (new_range->from > new_range->to) { //第一个 IP 大于第二个 IP
m_err_str =
"the first ip (" + std::string(first_ip) + ") is bigger than the other (" + std::string(second_ip) + ")";
return false;
@ -320,16 +404,25 @@ bool IPRange::parse_hyphen(const char* range, size_t range_len, const char *ptr,
return true;
}
/*
* IP IP
*
* new_ranges IP
* remove_range IP
* exist_range IP
*/
void IPRange::handle_remove_intersection(Ranges_t *new_ranges, const Range *remove_range, Range *exist_range)
{
// 求 remove_range 和 exist_range 的交集
IPV6 range_min = std::max(remove_range->from, exist_range->from);
IPV6 range_max = std::min(remove_range->to, exist_range->to);
if (range_min > range_max) {
if (range_min > range_max) { //没有交集
/* no intersaction */
new_ranges->push_back(*exist_range);
new_ranges->push_back(*exist_range); //向容器尾部添加新元素的成员函数
return;
}
/* there is an intersaction the remove_range includes the exist_range */
//移除范围 remove_range 完全包含了已存在范围 exist_range
if ((remove_range->from <= exist_range->from) && (remove_range->to >= exist_range->to)) {
/* remove the exist range */
return;
@ -338,29 +431,32 @@ void IPRange::handle_remove_intersection(Ranges_t *new_ranges, const Range *remo
// exist_range: 2 - 5
// remove_range: 1 - 3
// expected result: 4 - 5
if (remove_range->from <= exist_range->from) {
exist_range->from = remove_range->to + 1;
if (remove_range->from <= exist_range->from) { // 移除范围的起始位置 小于或等于 已存在范围的起始位置
exist_range->from = remove_range->to + 1; //更新 exist_range 的起始位置为 remove_range 的结束位置加1
new_ranges->push_back(*exist_range);
return;
}
if (remove_range->to >= exist_range->to) {
exist_range->to = remove_range->from - 1;
if (remove_range->to >= exist_range->to) { //移除范围的结束位置 大于或等于 已存在范围的结束位置
exist_range->to = remove_range->from - 1; //更新 exist_range 的结束位置为 remove_range 的起始位置减1
new_ranges->push_back(*exist_range);
return;
}
/* the remove range is inside the exist one */
//移除范围 remove_range 完全位于已存在范围 exist_range 内部
new_ranges->push_back({exist_range->from, remove_range->from - 1});
new_ranges->push_back({remove_range->to + 1, exist_range->to});
}
// 处理 IP 范围之间的交集 并 添加新的范围
bool IPRange::handle_add_intersection(Range *new_range, const Range *exist_range)
{
// 求 new_range 和 exist_range 的交集
IPV6 range_min = std::max(new_range->from, exist_range->from);
IPV6 range_max = std::min(new_range->to, exist_range->to);
if (range_min > range_max) {
if (range_min > range_max) { //没有交集
return false;
}
//更新 new_range 的范围
new_range->from = std::min(new_range->from, exist_range->from);
new_range->to = std::max(new_range->to, exist_range->to);
return true;
@ -373,6 +469,7 @@ bool IPRange::handle_add_intersection(Range *new_range, const Range *exist_range
* cidr ip : 192.168.10.1/21
* mask ip : 192.168.10.1/255.255.255.2
*/
// 解析 IP 范围的不同格式,并将其转换为 Range 结构
bool IPRange::parse_range(const char *range, size_t range_len, Range *new_range)
{
/* handle format of "ip/cidr" */
@ -412,9 +509,11 @@ bool IPRange::parse_range(const char *range, size_t range_len, Range *new_range)
return false;
}
//向 IP 范围集合中添加多个 IP 范围
//函数接受一个 std::unordered_set<std::string> 类型的参数 ranges其中包含要添加的 IP 范围字符串集合
bool IPRange::add_ranges(const std::unordered_set<std::string> ranges)
{
for (const std::string range : ranges) {
for (const std::string range : ranges) { //使用 for 循环遍历 ranges 集合中的每个 IP 范围字符串
if (!add_range(range.c_str(), range.length())) {
return false;
}
@ -422,6 +521,7 @@ bool IPRange::add_ranges(const std::unordered_set<std::string> ranges)
return true;
}
//从 IP 范围集合中移除多个 IP 范围
bool IPRange::remove_ranges(const std::unordered_set<std::string> ranges)
{
for (const std::string range : ranges) {
@ -432,46 +532,49 @@ bool IPRange::remove_ranges(const std::unordered_set<std::string> ranges)
return true;
}
//向 IP 范围列表中添加新的范围
bool IPRange::add_range(Range *new_range)
{
/* adding the new range */
if (m_ranges.size() == 0 || new_range->to < m_ranges[0].from) {
(void)m_ranges.insert(m_ranges.begin(), *new_range);
if (m_ranges.size() == 0 || new_range->to < m_ranges[0].from) { //列表为空或者新范围的结束值小于列表中第一个范围的开始值
(void)m_ranges.insert(m_ranges.begin(), *new_range); //将新范围插入到列表的开头
return true;
} else if (new_range->from > m_ranges.back().to) {
m_ranges.push_back(*new_range);
} else if (new_range->from > m_ranges.back().to) { //新范围的开始值大于列表中最后一个范围的结束值
m_ranges.push_back(*new_range); //将新范围追加到列表的末尾
return true;
}
Ranges_t new_ranges;
bool we_had_intersection = false;
uint32_t i = 0;
/* interate over the ranges and check for intersection or the place to add the new range */
/* interate over the ranges and check for intersection or the place to add the new range
*/
while (i < m_ranges.size()) {
/* in case of intersaction update the new range */
/* in case of intersaction update the new range 在交互的情况下,更新新的范围 */
if (handle_add_intersection(new_range, &m_ranges[i])) {
we_had_intersection = true;
++i;
continue;
}
/* just insert the intersaction */
/* just insert the intersaction 只需插入交互 */
if (we_had_intersection) {
we_had_intersection = false;
new_ranges.push_back(*new_range);
break;
} else if (new_range->to < m_ranges[i].from) {
/* we got the plcae to put the new range */
/* we got the plcae to put the new range 我们找到了放新靶场的地方 */
new_ranges.push_back(*new_range);
break;
}
/* just add the old range */
/* just add the old range 只要加上旧的范围 */
new_ranges.push_back(m_ranges[i]);
++i;
}
/* if the intersection was until the end of the list - add it now */
/* if the intersection was until the end of the list - add it now
- */
if (we_had_intersection) {
new_ranges.push_back(*new_range);
}
/* copy the rest of the list if exist */
/* copy the rest of the list if exist 如果存在,复制列表的其余部分 */
while (i < m_ranges.size()) {
new_ranges.push_back(m_ranges[i]);
++i;
@ -480,6 +583,7 @@ bool IPRange::add_range(Range *new_range)
return true;
}
//检查给定的 IP 范围字符串是否有效
bool IPRange::is_range_valid(const std::string range)
{
IPRange tmp;
@ -487,39 +591,44 @@ bool IPRange::is_range_valid(const std::string range)
return tmp.parse_range(range.c_str(), range.size(), &new_range);
}
//用于向 IP 范围集合中添加单个 IP 范围
bool IPRange::add_range(const char* range, size_t range_len)
{
Range new_range;
m_err_str.clear();
m_err_str.clear(); //清空错误信息
if (!parse_range(range, range_len, &new_range)) {
return false;
}
return add_range(&new_range);
}
//从 IP 范围集合中移除单个 IP 范围
bool IPRange::remove_range(const char *range, size_t range_len)
{
Ranges_t new_ranges;
Range remove_range;
m_err_str.clear();
if (!parse_range(range, range_len, &remove_range)) {
if (!parse_range(range, range_len, &remove_range)) { //解析传入的范围字符串
return false;
}
for (Range exist_range : m_ranges) {
for (Range exist_range : m_ranges) { //通过遍历存储在 m_ranges 中的每个范围对象 exist_range
//移除指定范围和当前范围的交集
handle_remove_intersection(&new_ranges, &remove_range, &exist_range);
}
m_ranges.swap(new_ranges);
m_ranges.swap(new_ranges); // 新的范围集合 new_ranges 替换原有的范围集合 m_ranges
return true;
}
// 将给定的IPv6地址转换为字符串表示形式
std::string IPRange::ip_to_str(const IPV6 *ip) const
{
char ip_str[INET6_ADDRSTRLEN];
char ip_str[INET6_ADDRSTRLEN]; //字符数组 ip_str用于存储转换后的IP地址字符串
/* now get it back and print it */
if (IPRANGE_IS_IPV4(*ip)) {
uint32_t tmp = htonl(ip->ip_32.a);
if (IPRANGE_IS_IPV4(*ip)) { //ipv4
uint32_t tmp = htonl(ip->ip_32.a); //将一个 32 位无符号整数 从 主机字节顺序 转换为 网络字节顺序
// 将网络字节顺序表示的 IP 地址转换为字符串形式的 IP 地址
(void)inet_ntop(AF_INET, &tmp, ip_str, INET_ADDRSTRLEN);
} else {
} else { // ipv6
IPV6 tmp_ip = *ip;
tmp_ip.ip_32.a = htonl(ip->ip_32.d);
tmp_ip.ip_32.b = htonl(ip->ip_32.c);
@ -530,6 +639,7 @@ std::string IPRange::ip_to_str(const IPV6 *ip) const
return std::string(ip_str);
}
//在 IP 范围列表中进行二分查找,判断给定的 IP 是否在某个范围内
bool IPRange::binary_search(const IPV6 ip) const
{
/* do a binary search */
@ -538,76 +648,84 @@ bool IPRange::binary_search(const IPV6 ip) const
size_t mid = 0;
while (i != j) {
mid = (i + j) / 2;
if (ip >= m_ranges[mid].from && ip <= m_ranges[mid].to) {
if (ip >= m_ranges[mid].from && ip <= m_ranges[mid].to) { //给定的 IP 在中间范围 m_ranges[mid] 内
return true;
}
if (ip < m_ranges[mid].from) {
if (ip < m_ranges[mid].from) { // IP在后半部分
j = (mid > 0) ? (mid - 1) : mid;
} else {
i = mid + 1;
}
}
// 判断给定的 IP 是否在最后剩余的范围内(即 m_ranges[i]
return (ip >= m_ranges[i].from && ip <= m_ranges[i].to);
}
//判断当前 IP 范围列表与另一个 IP 范围对象所表示的范围是否存在交集
bool IPRange::is_intersect(const IPRange *arg)
{
for (size_t i = 0; i < m_ranges.size(); ++i) {
for (size_t i = 0; i < m_ranges.size(); ++i) { // 遍历当前 IP 范围列表中的每个范围
Range tmp(m_ranges[i].from, m_ranges[i].to);
for (size_t j = 0 ; j < arg->m_ranges.size(); ++j) {
if (handle_add_intersection(&tmp, &arg->m_ranges[j])) {
return true;
for (size_t j = 0 ; j < arg->m_ranges.size(); ++j) { // 遍历传入的另一个 IP 范围对象 arg 中的每个范围
if (handle_add_intersection(&tmp, &arg->m_ranges[j])) { // 调用 handle_add_intersection 函数
return true; //判断两个范围是否有交集
}
}
}
return false;
}
/* 三个重载的 IPRange::is_in_range 函数,用于判断给定的 IP 是否在 IP 范围内 */
//字符串形式表示的 IP 地址 ip_str 作为参数
bool IPRange::is_in_range(const char *ip_str)
{
IPV6 ip;
if (!str_to_ip(ip_str, &ip)) {
if (!str_to_ip(ip_str, &ip)) { //将 ip_str 转换为 IPV6 结构体类型的 IP 地址 ip
return false;
}
return is_in_range(&ip);
return is_in_range(&ip); //调用第二个重载函数 is_in_range(&ip) 进行判断,并返回结果
}
//指向 IPV6 结构体类型的 IP 地址 ip 的指针作为参数
bool IPRange::is_in_range(const IPV6 *ip)
{
if (m_ranges.size() == 0) {
if (m_ranges.size() == 0) { //判范围为空
m_err_str = "there are no ranges in this object";
return false;
}
m_err_str.clear();
if (*ip == localhost_ipv4 || *ip == localhost_ipv6) {
m_err_str.clear(); //清空错误字符串 m_err_str
if (*ip == localhost_ipv4 || *ip == localhost_ipv6) { // 等于本地主机 IPv4 地址或本地主机 IPv6 地址
//调用 binary_search 函数分别在 IP 范围列表中查找对应的范围
return binary_search(localhost_ipv4) || binary_search(localhost_ipv6);
}
return binary_search(*ip);
}
//uint32_t 类型的 IPv4 地址 ipv4 作为参数
bool IPRange::is_in_range(const uint32_t ipv4)
{
IPV6 ip;
//将 IPv4 地址转换为 IPV6 结构体类型的 IP 地址 ip
net_ipv4_to_host_order(&ip, (struct in_addr*)&ipv4);
return is_in_range(&ip);
return is_in_range(&ip); // 调用第二个重载函数 is_in_range(&ip) 进行判断,并返回结果
}
// 获取当前范围集合的字符串表示形式
std::unordered_set<std::string> IPRange::get_ranges_set()
{
std::unordered_set<std::string> rslt;
std::unordered_set<std::string> rslt; //空的无序集合 rslt用于存储结果
//循环遍历 m_ranges 中的每个范围 range
for (Range range : m_ranges) {
if (ip_to_str(&range.from).compare(ip_to_str(&range.to)) == 0) {
rslt.insert(ip_to_str(&range.from));
if (ip_to_str(&range.from).compare(ip_to_str(&range.to)) == 0) { //IP 范围只有一个 IP
rslt.insert(ip_to_str(&range.from));//将该 IP 地址作为字符串添加到结果集合中
} else {
rslt.insert(ip_to_str(&range.from) + "-" + ip_to_str(&range.to));
rslt.insert(ip_to_str(&range.from) + "-" + ip_to_str(&range.to));//起始 IP 地址 - 结束 IP 地址 的形式作为字符串添加到结果集合中
}
}
return rslt;
}
// 将输入字符串中的空格字符去除,并将结果存储在目标缓冲区中
void IPRange::copy_without_spaces(char buf[], size_t buf_len, const char *original, size_t original_len) const
{
if (original_len == 0 || original_len > buf_len) {
if (original_len == 0 || original_len > buf_len) { //输入字符串的长度为0 或 超过目标缓冲区的长度
return;
}
char *p = buf;
@ -618,9 +736,8 @@ void IPRange::copy_without_spaces(char buf[], size_t buf_len, const char *origin
}
*p = '\0';
}
// 检查当前范围集合是否为空
bool IPRange::empty() const
{
return m_ranges.empty();
}

View File

@ -1,6 +1,12 @@
#This is the main CMAKE for build bin.
# CMake
#使 `AUX_SOURCE_DIRECTORY`
# `${CMAKE_CURRENT_SOURCE_DIR}` `TGT_tde_key_management_SRC`
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_tde_key_management_SRC)
# `TGT_tde_key_management_INC`
#IP
set(TGT_tde_key_management_INC
${PROJECT_SRC_DIR}/gausskernel/cbb/communication
${PROJECT_SRC_DIR}/include/iprange
@ -17,7 +23,16 @@ set(TGT_tde_key_management_INC
${LIBOPENSSL_INCLUDE_PATH}
)
#
#
set(tde_key_management_DEF_OPTIONS ${MACRO_OPTIONS})
set(tde_key_management_COMPILE_OPTIONS ${OPTIMIZE_OPTIONS} ${OS_OPTIONS} ${PROTECT_OPTIONS} ${WARNING_OPTIONS} ${BIN_SECURE_OPTIONS} ${CHECK_OPTIONS})
set(tde_key_management_LINK_OPTIONS ${BIN_LINK_OPTIONS})
#使 `add_static_objtarget` `gausskernel_security_tde_key_management`
# `TGT_tde_key_management_SRC` `TGT_tde_key_management_INC`
add_static_objtarget(gausskernel_security_tde_key_management TGT_tde_key_management_SRC TGT_tde_key_management_INC "${tde_key_management_DEF_OPTIONS}" "${tde_key_management_COMPILE_OPTIONS}" "${tde_key_management_LINK_OPTIONS}")
# `TGT_tde_key_management_SRC` `gausskernel_security_tde_key_management`
#

View File

@ -21,6 +21,23 @@
#
# ---------------------------------------------------------------------------------------
#这是一个 TDE 密钥管理模块的 Makefile 文件
#
#首先,定义了一些变量,包括 subdir、top_builddir并包含了顶层 Makefile 文件 Makefile.global
#
#接着,使用 override CPPFLAGS 命令将 CPPFLAGS 变量设置为指定的值
#其中包括 -I $(top_builddir)/src/include/tde_key_management用于指定头文件的搜索路径
#
#然后,通过条件判断和命令执行来处理依赖文件 $(DEPEND) 的引入
#如果当前执行的目标不是 clean 或 distclean并且系统中存在 g++ 编译器,则包含依赖文件
#这样可以自动生成依赖关系,确保当源文件变化时能够正确地重新编译
#
#接下来,定义了一个变量 OBJS包含了需要编译的目标文件列表
#
#最后,通过包含 common.mk 文件来完成编译规则的定义和编译目标的生成
#
#该 Makefile 文件用于编译 TDE 密钥管理模块的源代码,并生成目标文件
subdir=src/gausskernel/security/tde_key_management
top_builddir = ../../../..
include $(top_builddir)/src/Makefile.global
@ -37,4 +54,3 @@ endif
OBJS= ckms_message.o kms_interface.o data_common.o tde_key_manager.o tde_key_storage.o http_common.o
include $(top_srcdir)/src/gausskernel/common.mk

View File

@ -22,7 +22,7 @@
* ---------------------------------------------------------------------------------------
*/
#include <fstream>
#include <fstream> //用于文件读写
#include <iostream>
#include "tde_key_management/ckms_message.h"
#include "utils/elog.h"
@ -32,6 +32,8 @@
namespace TDE {
using namespace std;
//构造函数,用于初始化对象的成员变量
CKMSMessage::CKMSMessage()
{
tde_message_mem = nullptr;
@ -43,27 +45,35 @@ CKMSMessage::CKMSMessage()
token_timestamp = 0;
}
//析构函数,用于释放对象的资源
//调用了 clear() 函数,用于清理对象内部申请的动态内存
CKMSMessage::~CKMSMessage()
{
clear();
}
//初始化对象的上下文tde_message_mem
void CKMSMessage::init()
{
if (tde_message_mem == nullptr) {
if (tde_message_mem == nullptr) {//如果 tde_message_mem 为空
//调用 AllocSetContextCreate() 创建一个新的上下文,并将其赋值给 tde_message_mem
tde_message_mem = AllocSetContextCreate(g_instance.cache_cxt.global_cache_mem, "TDE_MESSAGE_CONTEXT",
ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE, SHARED_CONTEXT);
}
}
//清理对象的资源
void CKMSMessage::clear()
{
errno_t rc = 0;
//首先将 token_info->password 的内容清零
if (token_info->password != NULL) {
rc = memset_s(token_info->password, strlen(token_info->password), 0, strlen(token_info->password));
securec_check(rc, "\0", "\0");
}
//然后调用 pfree_ext() 释放对象的各个指针成员所指向的内存块
pfree_ext(tde_token);
pfree_ext(tde_agency_token);
@ -83,10 +93,13 @@ void CKMSMessage::clear()
pfree_ext(kms_info);
}
//检查当前对象中的身份令牌是否有效
bool CKMSMessage::check_token_valid()
{
TimestampTz current_time = 0;
//获取了当前时间戳
current_time = GetCurrentTimestamp();
//次检查 tde_token、tde_agency_token 和令牌时间戳是否为空或者是否超过了有效时间
if (tde_token == NULL) {
return false;
}
@ -99,6 +112,7 @@ bool CKMSMessage::check_token_valid()
return true;
}
//用于从文件中读取 KMSKey Management Service信息
char* CKMSMessage::read_kms_info_from_file()
{
char file_path[PATH_MAX] = {0};
@ -110,40 +124,50 @@ char* CKMSMessage::read_kms_info_from_file()
errno_t rc = EOK;
data_directory = g_instance.attr.attr_common.data_directory;
//检查 data_directory 是否为空
if (data_directory == NULL) {
//如果为空,则会产生错误报告,并返回 NULL
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("get gaussdb data directory path is NULL"), errdetail("N/A"),
errcause("data directory path not set"),
erraction("check if guc data_directory is exist")));
return NULL;
}
//拼接 data_directory、tde_config 和 token_file 构造完整的文件路径
path_len = strlen(data_directory) + strlen(tde_config) + strlen(token_file);
rc = snprintf_s(file_path, PATH_MAX, path_len, "%s%s%s", data_directory, tde_config, token_file);
securec_check_ss(rc, "\0", "\0");
//尝试以二进制的方式打开指定路径的文件
fstream json_file(file_path, ios::in | ios::binary);
//打开失败,则会产生错误报告,并返回相应的错误信息
if (!json_file) {
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_FILE_READ_FAILED),
errmsg("unable to open kms_iam_info.json file"), errdetail("file path: %s", file_path),
errcause("file not exist or broken"), erraction("check the kms_iam_info.json file")));
}
//如果文件打开成功,函数会获取文件的长度,并进行一些长度检查
json_file.seekg(0, ios::end);
json_len = json_file.tellg();
if (json_len > max_json_len) {
json_file.close();
if (json_len > max_json_len) { //如果文件长度超过 max_json_len
json_file.close(); //关闭文件
//产生错误报告,并返回相应的错误信息
ereport(ERROR,
(errmodule(MOD_SEC_TDE), errcode(ERRCODE_FILE_READ_FAILED),
errmsg("kms_iam_info.json file length is bigger than max_len"),
errdetail("file path: $TDE_PATH/tde_config/kms_iam_info.json"), errcause("file context is wrong"),
erraction("check the kms_iam_info.json file")));
}
//根据文件长度动态分配一个大小适合的缓冲区,并将文件内容读入该缓冲区
json_file.seekg(0, ios::beg);
buffer = (char*)palloc0(json_len + 1);
json_file.read(buffer, json_len + 1);
//关闭文件,并返回读取到的文件内容的指针
json_file.close();
return buffer;
}
//获取密码的明文
GS_UCHAR* CKMSMessage::get_cipher_rand()
{
GS_UCHAR* plain_text = NULL;
@ -153,21 +177,33 @@ GS_UCHAR* CKMSMessage::get_cipher_rand()
int path_len = 0;
errno_t rc = EOK;
//获取数据目录路径
data_directory = g_instance.attr.attr_common.data_directory;
//如果 data_directory 为 NULL则报错并返回 NULL
if (data_directory == NULL) {
//报错信息中包含了错误码、错误详情以及可能的修复措施
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("get gaussdb data directory path is NULL"), errdetail("N/A"),
errcause("data directory path not set"),
erraction("check if guc data_directory is exist")));
return NULL;
}
//计算 path_len 的长度
path_len = strlen(data_directory) + strlen(tde_config) + strlen(key) + 1;
//动态分配了一个大小为 path_len 的字符数组,并将其初始化为全零。该数组存储了拼接后的路径
key_dir = (char*)palloc0(path_len);
//拼接成完整的路径
rc = snprintf_s(key_dir, path_len, (path_len - 1), "%s%s%s", data_directory, tde_config, key);
securec_check_ss(rc, "\0", "\0");
//动态分配了一个大小为 CIPHER_LEN + 1 的 GS_UCHAR 类型的数组,并将其初始化为全零。该数组用于存储密码的明文
plain_text = (GS_UCHAR*)palloc0(CIPHER_LEN + 1);
//解码密文文件并将结果存储在 plain_text 中
decode_cipher_files(keymode, NULL, key_dir, plain_text);
pfree_ext(key_dir);
//如果明文数组 plain_text 的长度为 0则报错
if (strlen((char*)plain_text) == 0) {
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("can not get password plaintext"), errdetail("N/A"), errcause("file not exist or broken"),
@ -176,6 +212,7 @@ GS_UCHAR* CKMSMessage::get_cipher_rand()
return plain_text;
}
//用于填充 TokenInfo 结构体的信息
void CKMSMessage::fill_token_info(cJSON* internal_json)
{
char* username = NULL;
@ -183,7 +220,8 @@ void CKMSMessage::fill_token_info(cJSON* internal_json)
char* domain_name = NULL;
char* project_name = NULL;
errno_t rc = EOK;
//通过解析传入的 internal_json 参数,获取其中的 "username"、"domain_name" 和 "project_name" 字段的值
//如果这些字段为空,那么会抛出错误并返回
if ((cJSON_GetObjectItem(internal_json, "username") == NULL) ||
(cJSON_GetObjectItem(internal_json, "domain_name") == NULL) ||
(cJSON_GetObjectItem(internal_json, "project_name") == NULL)) {
@ -192,30 +230,43 @@ void CKMSMessage::fill_token_info(cJSON* internal_json)
errcause("IAM info value error"),
erraction("check tde_config kms_iam_info.json file")));
}
//将当前内存上下文切换为 tde_message_mem 上下文
MemoryContext old = MemoryContextSwitchTo(tde_message_mem);
//分配了 sizeof(TokenInfo) 大小的内存来存储 TokenInfo 结构体
token_info = (TokenInfo*)palloc0(sizeof(TokenInfo));
//从 internal_json 中获取到的 "username"、"domain_name" 和 "project_name" 字段的值拷贝到相应的结构体成员中,并对拷贝操作进行错误检查
username = cJSON_GetObjectItem(internal_json, "username")->valuestring;
token_info->user_name = (char*)palloc0(strlen(username) + 1);
rc = memcpy_s(token_info->user_name, (strlen(username) + 1), username, (strlen(username) + 1));
securec_check(rc, "\0", "\0");
domain_name = cJSON_GetObjectItem(internal_json, "domain_name")->valuestring;
token_info->domain_name = (char*)palloc0(strlen(domain_name) + 1);
rc = memcpy_s(token_info->domain_name, (strlen(domain_name) + 1), domain_name, (strlen(domain_name) + 1));
securec_check(rc, "\0", "\0");
project_name = cJSON_GetObjectItem(internal_json, "project_name")->valuestring;
token_info->project_name = (char*)palloc0(strlen(project_name) + 1);
rc = memcpy_s(token_info->project_name, (strlen(project_name) + 1), project_name, (strlen(project_name) + 1));
securec_check(rc, "\0", "\0");
//调用 get_cipher_rand 函数获取随机密码
password = (char*)get_cipher_rand();
if (password == NULL) {
if (password == NULL) { //获取密码失败(返回 NULL则抛出错误并返回
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("get internal password is NULL"), errdetail("N/A"), errcause("cipher rand file missing"),
erraction("check password cipher rand file")));
return;
}
//配了与 password 长度相同大小的内存给 token_info->password
token_info->password = (char*)palloc0(strlen(password) + 1);
//将 password 的内容拷贝到 token_info->password 中,并进行错误检查
rc = memcpy_s(token_info->password, (strlen(password) + 1), password, (strlen(password) + 1));
securec_check(rc, "\0", "\0");
//将内存上下文切换回之前的上下文,并清空并释放 password 指针所指向的内存
MemoryContextSwitchTo(old);
rc = memset_s(password, strlen(password), 0, strlen(password));
securec_check(rc, "\0", "\0");
@ -223,6 +274,7 @@ void CKMSMessage::fill_token_info(cJSON* internal_json)
return;
}
//用于填充 AgencyTokenInfo 结构体的信息
void CKMSMessage::fill_agency_info(cJSON* agency_json)
{
char* domain_name = NULL;
@ -230,6 +282,8 @@ void CKMSMessage::fill_agency_info(cJSON* agency_json)
char* project_name = NULL;
errno_t rc = EOK;
//通过解析传入的 agency_json 参数,获取其中的 "domain_name"、"agency_name" 和 "project_name" 字段的值
//如果这些字段为空,那么会抛出错误并返回
if ((cJSON_GetObjectItem(agency_json, "domain_name") == NULL) ||
(cJSON_GetObjectItem(agency_json, "agency_name") == NULL) ||
(cJSON_GetObjectItem(agency_json, "project_name") == NULL)) {
@ -238,33 +292,45 @@ void CKMSMessage::fill_agency_info(cJSON* agency_json)
errcause("IAM info value error"),
erraction("check tde_config kms_iam_info.json file")));
}
//将当前内存上下文切换为 tde_message_mem 上下文
MemoryContext old = MemoryContextSwitchTo(tde_message_mem);
//分配了 sizeof(AgencyTokenInfo) 大小的内存来存储 AgencyTokenInfo 结构体
agency_token_info = (AgencyTokenInfo*)palloc0(sizeof(AgencyTokenInfo));
//分别将从 agency_json 中获取到的 "domain_name"、"agency_name" 和 "project_name" 字段的值拷贝到相应的结构体成员中
// 并对拷贝操作进行错误检查
domain_name = cJSON_GetObjectItem(agency_json, "domain_name")->valuestring;
agency_token_info->domain_name = (char*)palloc0(strlen(domain_name) + 1);
rc = memcpy_s(agency_token_info->domain_name, (strlen(domain_name) + 1), domain_name,
(strlen(domain_name) + 1));
securec_check(rc, "\0", "\0");
agency_name = cJSON_GetObjectItem(agency_json, "agency_name")->valuestring;
agency_token_info->agency_name = (char*)palloc0(strlen(agency_name) + 1);
rc = memcpy_s(agency_token_info->agency_name, (strlen(agency_name) + 1), agency_name,
(strlen(agency_name) + 1));
securec_check(rc, "\0", "\0");
project_name = cJSON_GetObjectItem(agency_json, "project_name")->valuestring;
agency_token_info->project_name = (char*)palloc0(strlen(project_name) + 1);
rc = memcpy_s(agency_token_info->project_name, (strlen(project_name) + 1), project_name,
(strlen(project_name) + 1));
securec_check(rc, "\0", "\0");
//将内存上下文切换回之前的上下文,并返回
MemoryContextSwitchTo(old);
return;
}
//填充 KmsInfo 结构体的信息
void CKMSMessage::fill_kms_info(cJSON* kms_json)
{
char* project_name = NULL;
char* project_id = NULL;
errno_t rc = EOK;
//通过解析传入的 kms_json 参数,获取其中的 "project_name" 和 "project_id" 字段的值
// 如果这些字段为空,那么会抛出错误并返回
if ((cJSON_GetObjectItem(kms_json, "project_name") == NULL) ||
(cJSON_GetObjectItem(kms_json, "project_id") == NULL)) {
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
@ -272,22 +338,32 @@ void CKMSMessage::fill_kms_info(cJSON* kms_json)
errcause("KMS info value error"),
erraction("check tde_config kms_iam_info.json file")));
}
//将当前内存上下文切换为 tde_message_mem 上下文
MemoryContext old = MemoryContextSwitchTo(tde_message_mem);
//分配了 sizeof(KmsInfo) 大小的内存来存储 KmsInfo 结构体
kms_info = (KmsInfo*)palloc0(sizeof(KmsInfo));
//分别将从 kms_json 中获取到的 "project_name" 和 "project_id" 字段的值拷贝到相应的结构体成员中
// 并对拷贝操作进行错误检查
project_name = cJSON_GetObjectItem(kms_json, "project_name")->valuestring;
kms_info->project_name = (char*)palloc0(strlen(project_name) + 1);
rc = memcpy_s(kms_info->project_name, (strlen(project_name) + 1), project_name,
(strlen(project_name) + 1));
securec_check(rc, "\0", "\0");
project_id = cJSON_GetObjectItem(kms_json, "project_id")->valuestring;
kms_info->project_id = (char*)palloc0(strlen(project_id) + 1);
rc = memcpy_s(kms_info->project_id, (strlen(project_id) + 1), project_id,
(strlen(project_id) + 1));
securec_check(rc, "\0", "\0");
//将内存上下文切换回之前的上下文,并返回
MemoryContextSwitchTo(old);
return;
}
//解析 JSON 文件并根据文件内容填充相关信息
void CKMSMessage::parser_json_file(char* buffer)
{
cJSON* json_root = NULL;
@ -295,42 +371,57 @@ void CKMSMessage::parser_json_file(char* buffer)
cJSON* agency_json = NULL;
cJSON* kms_json = NULL;
//调用 cJSON_Parse 函数解析传入的 buffer 字符串
// 将解析结果存储在 json_root 中
json_root = cJSON_Parse(buffer);
if (!json_root) {
//如果解析失败,则会释放 buffer 的内存,并抛出错误
pfree_ext(buffer);
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_FILE_READ_FAILED),
errmsg("unable to get json file"), errdetail("N/A"), errcause("parse json file failed"),
erraction("check the kms_iam_info.json file format")));
}
//从 json_root 中获取 "internal_user_info" 字段的值,并通过调用 fill_token_info 函数填充相关的令牌信息
internal_json = cJSON_GetObjectItem(json_root, "internal_user_info");
if (internal_json) {
fill_token_info(internal_json);
}
//从 json_root 中获取 "agency_user_info" 字段的值,并通过调用 fill_agency_info 函数填充相关的机构信息
agency_json = cJSON_GetObjectItem(json_root, "agency_user_info");
if (agency_json) {
fill_agency_info(agency_json);
}
//从 json_root 中获取 "kms_info" 字段的值,并通过调用 fill_kms_info 函数填充相关的 KMS密钥管理服务信息
kms_json = cJSON_GetObjectItem(json_root, "kms_info");
if (kms_json) {
fill_kms_info(kms_json);
}
//释放 buffer 的内存,删除 json_root 对象
pfree_ext(buffer);
cJSON_Delete(json_root);
return;
}
//用于加载用户信息
//从文件中读取用户信息,并进行解析和填充
void CKMSMessage::load_user_info()
{
char* read_buffer = NULL;
/* read KMS and IAM info from kms_iam_info.json */
//调用 read_kms_info_from_file 函数读取 kms_iam_info.json 文件的内容,并将结果存储在 read_buffer 中
read_buffer = read_kms_info_from_file();
/* parser json file */
//调用 parser_json_file 函数解析 read_buffer 中的 JSON 内容,以填充相关的用户信息
parser_json_file(read_buffer);
return;
}
//用于生成 KMS密钥管理服务所需的 JSON 字符串
char* CKMSMessage::get_kms_json(ReplaceJsonValue input_json[], KmsHttpMsgType json_type, size_t count)
{
//ret 是一个枚举类型的变量,用于存储 HTTP 请求的返回值
HttpErrCode ret = TDE_HTTP_SUCCEED;
cJSON* json = NULL;
char* json_string = NULL;
@ -338,51 +429,79 @@ char* CKMSMessage::get_kms_json(ReplaceJsonValue input_json[], KmsHttpMsgType js
errno_t rc = EOK;
/* json_type is KmsHttpMsgType */
//调用 get_json_temp 函数获取指定 json_type 类型的 JSON 树
json = get_json_temp(json_type);
if (json == NULL) {
if (json == NULL) { //如果返回的 json 为 NULL则会产生错误报告并返回 NULL
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("get JSON tree is NULL"), errdetail("N/A"), errcause("get KMS JSON tree failed"),
erraction("check input prarmeter or config.ini file")));
return NULL;
}
//调用 traverse_jsontree_with_raplace_value 函数遍历 json 树
//并根据 input_json 数组中的替换值进行替换
ret = traverse_jsontree_with_raplace_value(json, input_json, count);
//如果遍历操作出现错误,那么将删除 json 对象并返回 NULL
if (ret != TDE_HTTP_SUCCEED) {
cJSON_Delete(json);
return NULL;
}
//调用 cJSON_Print 函数将替换完成后的 json 转换为字符串
temp_json = cJSON_Print(json);
if (temp_json == NULL) {
if (temp_json == NULL) {//如果转换失败,则会产生错误报告
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("get JSON tree is NULL"), errdetail("N/A"), errcause("get KMS JSON tree failed"),
erraction("check input prarmeter or config.ini file")));
}
//根据 temp_json 的长度动态分配内存,并将 temp_json 的内容复制到 json_string 中
json_string = (char*)palloc0(strlen(temp_json) + 1);
rc = memcpy_s(json_string, (strlen(temp_json) + 1), temp_json, (strlen(temp_json) + 1));
securec_check(rc, "\0", "\0");
//释放 temp_json 的内存,删除 json 对象,并返回生成的 JSON 字符串 json_string
cJSON_free(temp_json);
cJSON_Delete(json);
return json_string;
}
//user
//用于生成用于 IAM身份和访问管理授权请求的 JSON 字符串
char* CKMSMessage::get_iam_token_json()
{
char* token = NULL;
size_t count = 0;
//代表替换值的数组 json
// json 数组包含了一些键值对
// 键是要替换的占位符
// 值是通过 token_info 对象获取的具体信息(例如用户名、密码、域名和项目名)
ReplaceJsonValue json[] = {
{"$user_name$", token_info->user_name},
{"$password$", token_info->password},
{"$domain_name$", token_info->domain_name},
{"$project_name$", token_info->project_name},
};
//通过计算数组的大小除以单个元素的大小,得到数组元素的数量并赋值给 count 变量
count = sizeof(json) / sizeof(json[0]);
//调用 get_kms_json 函数,并将 json 数组、IAM_AUTH_REQ IAM 授权请求类型)和 count 参数传递进去
//会根据 json 数组中的值进行替换,并生成相应的 JSON 字符串
token = get_kms_json(json, IAM_AUTH_REQ, count);
//返回生成的 JSON 字符串 token
return token;
}
//agency
// 用于生成用于 IAM身份和访问管理授权请求的 JSON 字符串
char* CKMSMessage::get_iam_agency_token_json()
{
char* agency_token = NULL;
size_t count = 0;
// 代表替换值的数组 json
// json 数组包含了一些键值对
// 键是要替换的占位符
// 值是通过 agency_token_info 对象获取的具体信息(例如域名、代理名称和项目名称)
ReplaceJsonValue json[] = {
{"$domain_name$", agency_token_info->domain_name},
{"$agency_name$", agency_token_info->agency_name},
@ -390,77 +509,116 @@ char* CKMSMessage::get_iam_agency_token_json()
};
count = sizeof(json) / sizeof(json[0]);
agency_token = get_kms_json(json, IAM_AGENCY_TOKEN_REQ, count);
//返回生成的 JSON 字符串 agency_token
return agency_token;
}
//用于生成创建数据加密密钥DEK的请求 JSON 字符串
//接受一个 cmk_id 参数,用于指定创建 DEK 所使用的主密钥 ID
char* CKMSMessage::get_create_dek_json(const char* cmk_id)
{
char* req_body = NULL;
size_t count = 0;
//创建了一个用于替换值的 json 数组
// 包含了一个键值对
// 即占位符 $cmk_id$ 和传入的 cmk_id 值
ReplaceJsonValue json[] = {
{"$cmk_id$", cmk_id},
};
count = sizeof(json) / sizeof(json[0]);
// 将 json 数组、TDE_GEN_DEK_REQ 枚举类型(表示创建 DEK 请求类型)和 count 参数传递
req_body = get_kms_json(json, TDE_GEN_DEK_REQ, count);
// 返回 用于生成创建数据加密密钥DEK的请求 JSON 字符串
return req_body;
}
//用于生成解密数据加密密钥DEK的请求 JSON 字符串
//接受两个参数 cmk_id 是用于指定解密 DEK 所使用的主密钥 ID dek_cipher 是 DEK 的密文
char* CKMSMessage::get_decrypt_dek_json(const char* cmk_id, const char* dek_cipher)
{
char* req_body = NULL;
size_t count = 0;
// 创建了一个用于替换值的 json 数组
// 包含了两个键值对
// 占位符 $dek_cipher$ 和 $cmk_id$ 分别对应传入的 dek_cipher 和 cmk_id 值
ReplaceJsonValue json[] = {
{"$dek_cipher$", dek_cipher},
{"$cmk_id$", cmk_id},
};
count = sizeof(json) / sizeof(json[0]);
//将 json 数组、TDE_DEC_DEK_REQ 枚举类型(表示解密 DEK 请求类型)和 count 参数传递
req_body = get_kms_json(json, TDE_DEC_DEK_REQ, count);
//返回 生成解密数据加密密钥DEK的请求 JSON 字符串
return req_body;
}
//用于保存 TDE token 和代理 token
bool CKMSMessage::save_token(const char* token, const char* agency_token)
{
errno_t rc = EOK;
//对传入的参数进行非空校验
if ((token == NULL) || (agency_token == NULL)) {
return false;
}
//将上下文切换到 tde_message_mem
MemoryContext old = MemoryContextSwitchTo(tde_message_mem);
//在该内存上创建 tde_token 和 tde_agency_token 变量,它们分别用于保存 TDE token 和代理 token
if (tde_token == NULL) {
tde_token = (char*)palloc0(strlen(token) + 1);
}
//token 参数复制到 tde_token变量中
rc = memcpy_s(tde_token, (strlen(token) + 1), token, (strlen(token) + 1));
securec_check(rc, "\0", "\0");
if (tde_agency_token == NULL) {
tde_agency_token = (char*)palloc0(strlen(agency_token) + 1);
}
//agency_token 参数复制到 tde_agency_token 变量中
rc = memcpy_s(tde_agency_token, (strlen(agency_token) + 1), agency_token, (strlen(agency_token) + 1));
securec_check(rc, "\0", "\0");
//获取当前时间戳
token_timestamp = GetCurrentTimestamp();
//切换回原来的上下文
MemoryContextSwitchTo(old);
return true;
}
//用于遍历 JSON 树并替换特定的值
// json_tree 是要遍历的 JSON 树的根节点
// replace_rules 是一个包含替换规则的数组
// rule_cnt 表示替换规则的数量。
HttpErrCode CKMSMessage::traverse_jsontree_with_raplace_value(cJSON *json_tree, ReplaceJsonValue replace_rules[],
size_t rule_cnt)
{
char *new_value = NULL;
HttpErrCode ret = TDE_HTTP_SUCCEED;
//对传入的 json_tree 进行非空校验
if (json_tree == NULL) {
return TDE_HTTP_SUCCEED;
}
//判断 json_tree 是否为字符串类型的节点
if (cJSON_IsString(json_tree)) {
//如果是,则遍历替换规则数组 replace_rules
for (size_t i = 0; i < rule_cnt; i++) {
//检查当前节点的值是否为 NULL
if (cJSON_GetStringValue(json_tree) == NULL) {
//如果是,则抛出错误
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("failed to get json tree"), errdetail("N/A"),
errcause("config.ini json tree error"),
erraction("check input prarmeter or config.ini file")));
}
//将当前节点的值与替换规则的源值进行比较
if (strcmp(replace_rules[i].src_value, cJSON_GetStringValue(json_tree)) == 0) {
//如果相等,则使用 cJSON_SetValuestring 函数
// 将当前节点的值替换为替换规则的目标值,并将返回的新值保存到 new_value 变量中
new_value = cJSON_SetValuestring(json_tree, replace_rules[i].dest_value);
////检查当前]新节点的值是否为 NULL
if (new_value == NULL) {
// 如果是,则抛出错误
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("failed to set the value of json tree"), errdetail("N/A"),
errcause("config.ini json tree error"),
@ -470,15 +628,23 @@ HttpErrCode CKMSMessage::traverse_jsontree_with_raplace_value(cJSON *json_tree,
}
}
}
//递归调用 traverse_jsontree_with_raplace_value 函数分别对当前节点的下一个兄弟节点和子节点进行遍历
ret = traverse_jsontree_with_raplace_value(json_tree->next, replace_rules, rule_cnt);
//递归调用返回的结果不等于 TDE_HTTP_SUCCEED
if (ret != TDE_HTTP_SUCCEED) {
//则将该结果作为函数的返回值
return ret;
}
//否则,继续对当前节点的子节点进行遍历,并将结果作为函数的返回值
return traverse_jsontree_with_raplace_value(json_tree->child, replace_rules, rule_cnt);
}
//根据传入的 json_tree_type 参数返回相应的 cJSON 对象指针
//接受一个 KmsHttpMsgType 类型的参数 json_tree_type用于确定返回哪个 cJSON 对象指针
cJSON *CKMSMessage::get_json_temp(KmsHttpMsgType json_tree_type)
{
//使用 switch 语句根据 json_tree_type 的值进行匹配
// 返回 匹配 字符串解析后的 cJSON 对象指针
switch (json_tree_type) {
case IAM_AUTH_REQ:
return cJSON_Parse(iam_auth_token_req);
@ -489,7 +655,7 @@ cJSON *CKMSMessage::get_json_temp(KmsHttpMsgType json_tree_type)
case TDE_DEC_DEK_REQ:
return cJSON_Parse(kms_decrypt_dek_req);
default:
break;
break; //与上述情况都不匹配,则执行默认的 break然后返回 NULL
}
return NULL;
}

View File

@ -22,22 +22,34 @@
* -------------------------------------------------------------------------
*/
/*
TDEData
*/
#include "tde_key_management/data_common.h"
#include "utils/elog.h"
//为 AdvStrList 分配内存,并初始化成员变量
AdvStrList *malloc_advstr_list(void)
{
AdvStrList *new_list = NULL;
//分配 sizeof(AdvStrList) 大小的内存空间
new_list = (AdvStrList *)palloc0(sizeof(AdvStrList));
//检查 new_list 是否为 NULL
if (new_list == NULL) {
//如果为 NULL则说明内存分配失败函数直接返回 NULL
return NULL;
}
new_list->node_cnt = 0;
new_list->first_node = NULL;
//如果分配成功,则将 new_list 的成员变量进行初始化
new_list->node_cnt = 0; //表示当前链表中的节点个数为 0
new_list->first_node = NULL; //表示链表中没有任何节点
//返回指向该结构体的指针
return new_list;
}
//向列表中追加一个字符串节点
// 从列表的末尾开始遍历,最后将新的字符串节点插入到链表的末尾
void tde_append_node(AdvStrList *list, const char *str_val)
{
/* last_node -> next = new_node */
@ -46,58 +58,87 @@ void tde_append_node(AdvStrList *list, const char *str_val)
char *new_node_str_val = NULL;
errno_t rc = 0;
//分配内存大小为 strlen(str_val) + 1 的空间
new_node_str_val = (char *)palloc0(strlen(str_val) + 1);
if (new_node_str_val == NULL) {
return;
return; //如果为 NULL则说明内存分配失败函数直接返回
}
//分配内存大小为 sizeof(AdvStrNode) 的空间
new_node = (AdvStrNode *)palloc0(sizeof(AdvStrNode));
if (new_node == NULL) {
pfree_ext(new_node_str_val);
return;
return; //如果为 NULL则说明内存分配失败需要释放前面分配的 new_node_str_val 并直接返回
}
//将 str_val 复制到 new_node_str_val 中
rc = strcpy_s(new_node_str_val, strlen(str_val) + 1, str_val);
securec_check(rc, "\0", "\0");
new_node->str_val = new_node_str_val;
new_node->next = NULL;
new_node->str_val = new_node_str_val; //表示该节点存储的字符串值
new_node->next = NULL; //表示该节点是链表中的最后一个节点
if (list->first_node == NULL) {
//如果链表为空
list->first_node = new_node;
} else {
} else { //如果链表不为空
//将链表的第一个节点 list->first_node 赋值给 last_node作为遍历链表的起始节点
last_node = list->first_node;
//从第一个节点开始遍历到倒数第二个节点
//循环结束后last_node 就指向了链表中的最后一个节点
for (int i = 0; i < (list->node_cnt - 1); i++) {
last_node = last_node->next;
}
//将新节点添加到最后一个节点的后面,完成节点的追加操作
last_node->next = new_node;
}
//更新链表的节点数量
list->node_cnt++;
}
//返回 AdvStrList 链表中字符串节点的个数
size_t tde_list_len(AdvStrList *list)
{
return list->node_cnt;
return list->node_cnt; //node_cnt 记录了链表中字符串节点的个数
}
//将指定的字符串按照指定的分隔符进行切割
// 并将每一个切割后的子串存放到 AdvStrList 中的一个字符串节点中
AdvStrList *tde_split_node(const char *str, char split_char)
{
AdvStrList *substr_list = NULL;
char *cur_substr = NULL;
size_t str_start = 0;
errno_t rc = 0;
//创建一个 AdvStrList 链表
substr_list = malloc_advstr_list();
if ((substr_list == NULL) || (str == NULL)) {
//如果创建失败或者输入参数 str 为 NULL则直接返回 NULL
return NULL;
}
//从字符串 str 的第一个字符开始遍历每个字符
for (size_t i = 0; i < strlen(str); i++) {
//果遍历到的字符是分隔符 split_char 或者已经遍历到了字符串的最后一个字符
//则可以认为遍历到了一个子字符串的结尾
if (str[i] == split_char || i == strlen(str) - 1) {
//分配内存,将子字符串复制到 cur_substr 中
// 字符串长度为 i-str_start+1
cur_substr = (char *) palloc0(i - str_start + 1);
rc = strncpy_s(cur_substr, i - str_start + 1, str + str_start, i - str_start);
securec_check(rc, "\0", "\0");
//需要再加上一个字节存放字符串结尾标志 '\0'
cur_substr[i - str_start] = '\0';
str_start = i + 1;
tde_append_node(substr_list, cur_substr);
str_start = i + 1; //设为当前子字符串的下一个字符位置,用于计算下一个子字符串的长度和内容
//将新的节点 cur_substr 添加到链表 substr_list 的末尾
tde_append_node(substr_list, cur_substr);
//释放 cur_substr 占用的内存
pfree_ext(cur_substr);
}
}
//完成链表的构建后,判断链表是否为空
if (tde_list_len(substr_list) == 0) {
//如果链表为空,则说明没有找到任何子字符串
// 需要释放空间并返回 NULL
free_advstr_list(substr_list);
substr_list = NULL;
return NULL;
@ -105,62 +146,90 @@ AdvStrList *tde_split_node(const char *str, char split_char)
return substr_list;
}
//释放 AdvStrList 占用的内存,包括链表节点、字符串值等
void free_advstr_list(AdvStrList *list)
{
AdvStrNode *cur_node = NULL;
AdvStrNode *to_free = NULL;
//如果输入的链表指针 list 为 NULL
if (list == NULL) {
return;
return; //直接返回,不需要进行释放操作
}
cur_node = list->first_node;
//指向链表的第一个节点
cur_node = list->first_node;
//循环进行遍历,直到遍历完所有的链表节点
while (cur_node != NULL) {
to_free = cur_node;
cur_node = cur_node->next;
pfree_ext(to_free->str_val);
pfree_ext(to_free);
pfree_ext(to_free->str_val); //释放 to_free->str_val 所指向的 字符串数据 的内存空间
pfree_ext(to_free); //释放 to_free 所指向的 节点 的内存空间
}
//释放链表头节点 list 的内存空间
pfree_ext(list);
}
//根据列表中的位置获取对应的字符串值
char *tde_get_val(AdvStrList *list, int list_pos)
{
AdvStrNode *target_node = NULL;
// 如果传入的 list_pos 参数为 -1则表示获取列表中的最后一个节点的值
if (list_pos == -1) {
//将 list_pos 的值设为链表节点数量减 1
list_pos = list->node_cnt - 1;
}
//进行判断,如果 list_pos 的值小于 0 或者大于等于 链表节点数量 list->node_cnt
if (list_pos < 0 || list_pos >= list->node_cnt) {
//位置超出范围
return NULL;
}
//指向链表的第一个节点
target_node = list->first_node;
//循环进行遍历,直到找到目标节点位置
for (int i = 0; i < list_pos; i++) {
target_node = target_node->next;
}
//循环执行了 list_pos 次后target_node 就指向了目标节点
//返回目标节点的字符串值 target_node->str_val
return target_node->str_val;
}
//该函数和 free_advstr_list 相似
// 区别在于该函数在释放链表时会跳过指定位置上的节点,将其它节点都释放掉
// 这个函数可以用于一些特定的场景,例如虽然不需要列表中所有的元素,但是有一些需要保留的重要信息
void free_advstr_list_with_skip(AdvStrList *list, int list_pos)
{
AdvStrNode *cur_node = NULL;
AdvStrNode *to_free = NULL;
//如果参数 list_pos 的值为 -1表示要保留链表中的最后一个节点
if (list_pos == -1) {
//将 list_pos 的值设为链表节点数量减 1
list_pos = list->node_cnt - 1;
}
//指向链表的第一个节点
cur_node = list->first_node;
//循环进行遍历,直到遍历完所有的链表节点
for (int i = 0; i < (int)tde_list_len(list); i++) {
to_free = cur_node;
cur_node = cur_node->next;
//如果当前节点的位置 i 不等于参数 list_pos
if (i != list_pos) {
//说明需要释放该节点和其所包含的字符串数据
pfree_ext(to_free->str_val);
pfree_ext(to_free);
}
}
//释放链表头节点 list 的内存空间
pfree_ext(list);
}
//构造函数会为成员变量分配内存,并将它们初始化
TDEData::TDEData()
{
cmk_id = NULL;
@ -168,13 +237,20 @@ TDEData::TDEData()
dek_plaintext = NULL;
}
//释放成员变量占用的内存,并清空敏感、私有数据
TDEData::~TDEData()
{
errno_t rc = 0;
if (dek_plaintext != NULL) {
//dek_plaintext 不为 NULL表示存在明文密钥数据
//调用 memset_s 函数对 dek_plaintext 所指向的内存空间进行安全的清零操作
//使用 strlen(dek_plaintext) 获取需要清零的内存大小
rc = memset_s(dek_plaintext, strlen(dek_plaintext), 0, strlen(dek_plaintext));
//使用 securec_check 宏检查 memset_s 函数的返回值 rc 是否为成功状态,如果不成功,则会触发错误处理
securec_check(rc, "\0", "\0");
}
//分别使用 pfree_ext 函数释放成员变量 cmk_id、dek_cipher和dek_plaintext 所指向的内存空间
pfree_ext(cmk_id);
pfree_ext(dek_cipher);
pfree_ext(dek_plaintext);

View File

@ -22,6 +22,12 @@
* -------------------------------------------------------------------------
*/
/*
HTTP
CURL对象
使 http_request() 便 HTTP
*/
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
@ -29,67 +35,100 @@
#include "securec.h"
#include "utils/elog.h"
//是整个流程的入口函数,用户通过调用该函数来发起 HTTP 请求,并返回相应的结果
// 输入参数包括 http_req_msgHTTP 请求消息体)
// http_configHTTP 配置信息)
// http_res_listHTTP 响应消息列表)
HttpErrCode HttpCommon::http_request(HttpReqMsg* http_req_msg, HttpConfig *http_config, AdvStrList **http_res_list)
{
CURL *sender = NULL;
struct curl_slist *http_header = NULL;
HttpErrCode ret = TDE_HTTP_SUCCEED;
//全局初始化 libcurl
curl_global_init(CURL_GLOBAL_ALL);
//创建 CURL 句柄 sender使用 curl_easy_init() 初始化
sender = curl_easy_init();
if (sender == NULL) {
//创建 CURL 句柄失败,则抛出错误并返回 TDE_CURL_ERR 错误码
ereport(ERROR, (errcode(ERRCODE_CANNOT_CONNECT_NOW), errmsg("failed to init curl")));
return TDE_CURL_ERR;
}
//设置 HTTP 请求的配置信息
ret = set_http_config(sender, http_config);
if (ret != TDE_HTTP_SUCCEED) {
curl_easy_cleanup(sender);
return ret;
}
//设置 HTTP 请求的请求行信息
ret = set_http_reqline(sender, http_req_msg->method, http_req_msg->url);
if (ret != TDE_HTTP_SUCCEED) {
curl_easy_cleanup(sender);
return ret;
}
//设置 HTTP 请求的请求头信息
ret = set_http_reqheader(sender, http_req_msg->header_list, http_header);
//释放请求头链表 http_header
curl_slist_free_all(http_header);
if (ret != TDE_HTTP_SUCCEED) {
curl_easy_cleanup(sender);
return ret;
}
//设置 HTTP 请求的请求体信息
ret = set_http_reqbody(sender, http_req_msg->body);
if (ret != TDE_HTTP_SUCCEED) {
curl_easy_cleanup(sender);
return ret;
}
//发送 HTTP 请求并获取响应信息
ret = get_http_resmsg(sender, http_config->res_part, http_res_list);
//清理 CURL 句柄
curl_easy_cleanup(sender);
if (ret != TDE_HTTP_SUCCEED) {
return ret;
}
//全局清理 libcurl
curl_global_cleanup();
return TDE_HTTP_SUCCEED;
}
//用于设置 CURL 对象的配置信息包括超时时间、SSL 验证等
//使用 libcurl 库提供的函数 curl_easy_setopt() 来设置 CURL 对象的选项
HttpErrCode HttpCommon::set_http_config(CURL *http_obj, HttpConfig *http_config)
{
CURLcode curl_ret = CURLE_OK;
const int connect_timeout = 10;
//通过检查 curl_ret 的值,可以判断函数调用是否成功执行,或者是否发生了错误
CURLcode curl_ret = CURLE_OK; //CURLE_OK 是 libcurl 库定义的一个枚举常量,表示没有发生错误的状态码
const int connect_timeout = 10; //表示连接超时时间为 10 秒
//通过 curl_easy_setopt() 函数
//设置 CURL 对象的 CURLOPT_TIMEOUT 选项
//将 HTTP 请求的超时时间设置为 http_config 结构体中指定的值,并检查返回值是否为 CURLE_OK
curl_ret = curl_easy_setopt(http_obj, CURLOPT_TIMEOUT, http_config->timeout);
// 并检查返回值是否为 CURLE_OK
check_curl_ret(curl_ret);
//设置 CURL 对象的 CURLOPT_CONNECTTIMEOUT 选项,将连接超时时间设置为 connect_timeout
curl_ret = curl_easy_setopt(http_obj, CURLOPT_CONNECTTIMEOUT, connect_timeout);
check_curl_ret(curl_ret);
//设置 CURL 对象的 CURLOPT_SSL_VERIFYPEER 选项,禁用对服务器 SSL 证书的验证
curl_ret = curl_easy_setopt(http_obj, CURLOPT_SSL_VERIFYPEER, false);
check_curl_ret(curl_ret);
//设置 CURL 对象的 CURLOPT_SSL_VERIFYHOST 选项,禁用对服务器主机名的验证
curl_ret = curl_easy_setopt(http_obj, CURLOPT_SSL_VERIFYHOST, false);
check_curl_ret(curl_ret);
return TDE_HTTP_SUCCEED;
}
//用于检查 CURL 请求的返回值,并进行相应的处理,例如输出日志、抛出异常等
void HttpCommon::check_curl_ret(CURLcode curl_ret)
{
//通过比较 curl_ret 和 CURLE_OK 来判断请求是否成功
if (curl_ret != CURLE_OK) {
//输出错误信息,包括错误码和错误消息
ereport(ERROR, (errcode(ERRCODE_CANNOT_CONNECT_NOW),
errmsg("curl error. err code: '%lu', err msg: '%s.", (long unsigned int)curl_ret,
curl_easy_strerror(curl_ret))));
@ -97,26 +136,41 @@ void HttpCommon::check_curl_ret(CURLcode curl_ret)
return;
}
//用于设置 设置 HTTP 请求的请求行 CURL 对象的请求方法和 URL
//http_obj 是一个指向 CURL 对象的指针,表示要设置请求行的 HTTP 对象
//method 是一个枚举类型 HttpMethod表示 HTTP 请求的方法,可以是 HTTP_POST 或 HTTP_GET
//url 是一个指向字符串的指针,表示请求的 URL
HttpErrCode HttpCommon::set_http_reqline(CURL *http_obj, HttpMethod method, const char *url)
{
CURLcode curl_ret = CURLE_OK;
//根据 method 的值来设置对应的 CURL 选项
switch (method) {
case HTTP_POST:
//调用 curl_easy_setopt() 函数并将 CURLOPT_HTTPPOST 设置为 1表示使用 POST 方法发送请求
curl_ret = curl_easy_setopt(http_obj, CURLOPT_HTTPPOST, 1);
break;
case HTTP_GET:
//调用 curl_easy_setopt() 函数并将 CURLOPT_HTTPGET 设置为 1表示使用 GET 方法发送请求
curl_ret = curl_easy_setopt(http_obj, CURLOPT_HTTPGET, 1);
break;
default:
break;
}
check_curl_ret(curl_ret);
//设置请求的 URL将 url 参数作为参数传入
curl_ret = curl_easy_setopt(http_obj, CURLOPT_URL, url);
check_curl_ret(curl_ret);
//返回一个枚举类型 TDE_HTTP_SUCCEED表示设置请求行成功
return TDE_HTTP_SUCCEED;
}
//函数用于设置 HTTP 请求的请求头 ( CURL 对象的请求头部信息 )
//http_header_list 是一个指向字符串数组的指针,表示要设置的 HTTP 请求头列表
//数组中的每个元素都是一个字符串,表示一个 HTTP 请求头
//ret_header_list 是一个指向 curl_slist 结构体的指针,表示返回的请求头列表
HttpErrCode HttpCommon::set_http_reqheader(CURL *http_obj, const char *http_heaser_list[],
struct curl_slist* ret_header_list)
{
@ -124,57 +178,88 @@ HttpErrCode HttpCommon::set_http_reqheader(CURL *http_obj, const char *http_heas
CURLcode curl_ret = CURLE_OK;
for (size_t i = 0; http_heaser_list[i] != NULL; i++) {
//将当前的 HTTP 请求头字符串添加到临时列表 tmp_http_header 中
tmp_http_header = curl_slist_append(tmp_http_header, http_heaser_list[i]);
//添加失败(返回值为 NULL
if (tmp_http_header == NULL) {
ereport(ERROR, (errcode(ERRCODE_CANNOT_CONNECT_NOW),
errmsg("failed to set http header: '%s'", http_heaser_list[i])));
return TDE_CURL_ERR;
return TDE_CURL_ERR; //输出错误信息并返回 TDE_CURL_ERR表示设置请求头失败
}
}
//将临时请求头列表 tmp_http_header 设置为 CURL 对象的请求头选项 CURLOPT_HTTPHEADER
curl_ret = curl_easy_setopt(http_obj, CURLOPT_HTTPHEADER, tmp_http_header);
check_curl_ret(curl_ret);
ret_header_list = tmp_http_header;
//返回 TDE_HTTP_SUCCEED表示设置请求头成功
return TDE_HTTP_SUCCEED;
}
//用于设置 HTTP 请求的请求体
//http_body 是一个指向字符串的指针,表示请求的消息体内容
HttpErrCode HttpCommon::set_http_reqbody(CURL *http_obj, const char *http_body)
{
//请求体内容 http_body 设置为 CURL 对象的请求体选项 CURLOPT_POSTFIELDS
// 该选项用于指定 HTTP 请求的消息体内容
CURLcode curl_ret = curl_easy_setopt(http_obj, CURLOPT_POSTFIELDS, http_body);
check_curl_ret(curl_ret);
return TDE_HTTP_SUCCEED;
}
//用于获取 CURL 对象的响应消息,并将其解析成字符串列表
//res_part 是一个枚举类型的变量,表示要获取的响应部分(如响应行、响应头、响应体)
//response_msg 是一个指向 AdvStrList 指针的指针,表示用于存储响应消息体内容的字符串列表
HttpErrCode HttpCommon::get_http_resmsg(CURL *http_obj, HttpResListType res_part, AdvStrList **response_msg)
{
//明了一个名为 http_res_list 的结构体变量,用于存储 HTTP 响应的相关信息,包括响应部分、响应行、响应头以及响应体等
HttpResList http_res_list = {"", res_part, HTTP_RESLINE, NULL, 0, 0, false};
const int status_code = 300;
long http_status_code = 0;
long http_status_code = 0; //用于存储实际的 HTTP 响应状态码
CURLcode curl_ret = CURLE_OK;
//通过调用 malloc_advstr_list() 函数来分配内存以创建一个字符串列表,用于存储响应消息体的每一行内容
http_res_list.str_list = malloc_advstr_list();
if (http_res_list.str_list == NULL) {
if (http_res_list.str_list == NULL) { //分配内存失败
ereport(ERROR, (errcode(ERRCODE_CANNOT_CONNECT_NOW), errmsg("failed to malloc memory")));
return TDE_MALLOC_MEM_ERR;
}
//调用 curl_easy_setopt() 函数将相应的选项设置为 http_obj
//设置 CURLOPT_HEADER 选项为 1表示希望在响应中包含头部信息
//该选项告诉 libcurl 将头部信息写入到数据回调函数中
curl_ret = curl_easy_setopt(http_obj, CURLOPT_HEADER, 1);
check_curl_ret(curl_ret);
//将 dump_http_response_msg_callback 函数设置为数据回调函数write function
//libcurl 在接收到响应数据时将调用此回调函数,用于处理接收到的响应数据。
curl_ret = curl_easy_setopt(http_obj, CURLOPT_WRITEFUNCTION, dump_http_response_msg_callback);
check_curl_ret(curl_ret);
// 设置 CURLOPT_WRITEDATA 选项为 &http_res_list
// 表示希望将 http_res_list 的指针作为用户定义的指针传递给数据回调函数
// 这样,在数据回调函数中可以通过该指针来访问并存储响应数据
curl_ret = curl_easy_setopt(http_obj, CURLOPT_WRITEDATA, (void *) &http_res_list);
check_curl_ret(curl_ret);
//调用 curl_easy_perform() 函数执行 HTTP 请求,并将响应结果保存到 http_res_list 中
curl_ret = curl_easy_perform(http_obj);
check_curl_ret(curl_ret);
//调用 curl_easy_getinfo() 函数获取实际的 HTTP 响应状态码
curl_ret = curl_easy_getinfo(http_obj, CURLINFO_RESPONSE_CODE, &http_status_code);
check_curl_ret(curl_ret);
//如果实际状态码大于 status_code
if (http_status_code > status_code) {
//输出错误信息
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_CANNOT_CONNECT_NOW),
errmsg("curl error code is %d", (int)http_status_code), errdetail("N/A"),
errcause("http request status error"), erraction("check curl retrun code status")));
}
//将响应消息体内容的字符串列表传递给调用者
*response_msg = http_res_list.str_list;
return TDE_HTTP_SUCCEED;
}
//是一个回调函数,用于处理 CURL 响应消息的回调,将响应消息按照指定规则进行解析和保存
//接收 libcurl 库返回的响应消息,并根据设置的过滤条件来判断是否需要将消息保存到 HttpResList 结构体中
size_t HttpCommon::dump_http_response_msg_callback(void *tmp_cur_res_str, size_t chr_size, size_t cur_res_size,
void *tmp_http_res_list)
{
@ -182,17 +267,27 @@ size_t HttpCommon::dump_http_response_msg_callback(void *tmp_cur_res_str, size_t
const int loop_num = 2;
HttpResList *http_res_list = NULL;
//判断输入参数 tmp_cur_res_str 和 tmp_http_res_list 是否为空,如果为空则返回 0
if (tmp_cur_res_str == NULL || tmp_http_res_list == NULL) {
return 0;
}
//转换为 HttpResList 结构体指针
http_res_list = (HttpResList *)tmp_http_res_list;
if (http_res_list->is_stop_dump) {
//如果 http_res_list->is_stop_dump 为 true则直接返回当前响应消息的大小
return chr_size * cur_res_size;
}
//转换为 const char* 类型
const char *cur_res_str = (const char *)tmp_cur_res_str;
//根据设置的过滤条件 http_res_list->filter_res_type
// 和当前情况 http_res_list->cur_pos
// 判断是否需要保存当前响应消息
if (http_res_list->filter_res_type == HTTP_MSG) {
// 过滤条件 为 HTTP_MSG则需要保存当前响应消息
is_need_dump = true;
} else {
// 根据 当前情况 http_res_list->cur_pos 的不同取值,判断是否需要保存当前响应消息
// 并更新 http_res_list->cur_pos 的值
switch (http_res_list->cur_pos) {
case HTTP_RESLINE:
if (http_res_list->filter_res_type == HTTP_RESLINE) {
@ -222,7 +317,9 @@ size_t HttpCommon::dump_http_response_msg_callback(void *tmp_cur_res_str, size_t
break;
}
}
//如果需要保存当前响应消息
if (is_need_dump) {
//将消息添加到 http_res_list->str_list 链表中
tde_append_node(http_res_list->str_list, cur_res_str);
}
return chr_size * cur_res_size;

View File

@ -1,3 +1,5 @@
;存储了身份认证的请求字符串,用于获取身份认证的令牌
;包含用户名、密码和域名等信息
const char *iam_auth_token_req = "{"
" \"auth\": {"
" \"identity\": {"
@ -20,6 +22,9 @@ const char *iam_auth_token_req = "{"
" }"
"}";
;存储了代理认证的请求字符串,用于获取代理认证的令牌
;包含域名、代理名称和项目名称等信息
const char *iam_agency_token_req = "{"
" \"auth\": {"
" \"identity\": {"
@ -37,11 +42,17 @@ const char *iam_agency_token_req = "{"
" }"
"}";
;存储了创建数据密钥的请求字符串,用于生成一个新的数据加密密钥
;包含数据密钥的长度和主密钥的 ID 等信息
const char *kms_create_dek_req = "{"
" \"datakey_length\": \"128\","
" \"key_id\": \"$cmk_id$\""
"}";
;存储了解密数据密钥的请求字符串,用于解密已加密的数据密钥
;包含密文、密文长度和主密钥的 ID 等信息
const char *kms_decrypt_dek_req = "{"
" \"cipher_text\": \"$dek_cipher$\","
" \"datakey_cipher_length\": \"16\","

View File

@ -31,20 +31,27 @@ KMSInterface::KMSInterface()
KMSInterface::~KMSInterface()
{}
// 创建一个 KMS DEKData Encryption Key
DekInfo* KMSInterface::create_kms_dek(const char* cmk_id)
{
DekInfo* create_dek_info = NULL;
AdvStrList *http_msg_list = NULL;
/* check token valid */
//检查令牌是否有效
if (!(TDE::CKMSMessage::get_instance().check_token_valid())) {
/* get token */
//如果令牌无效,获取新的令牌
get_kms_token();
}
/* create KMS DEK */
//向 KMS 发送请求,获取 KMS DEK 的信息
http_msg_list = kms_restful_get_dek(cmk_id, NULL, KMS_GEN_DEK);
//对返回的消息列表进行解析
create_dek_info = parser_http_array(http_msg_list, KMS_GEN_DEK);
//释放 http_msg_list 占用的内存
tde_list_free(http_msg_list);
if (create_dek_info == NULL) {
//reate_dek_info ,如果为 NULL则会报错并返回 NULL 值
ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("creating KMS DEK is NULL")));
return NULL;
@ -52,18 +59,24 @@ DekInfo* KMSInterface::create_kms_dek(const char* cmk_id)
return create_dek_info;
}
//获取 KMS DEKData Encryption Key的明文
char* KMSInterface::get_kms_dek(const char* cmk_id, const char* dek_cipher)
{
char* dek_plain = NULL;
AdvStrList *http_msg_list = NULL;
/* check token valid */
// 检查令牌是否有效
if (!(TDE::CKMSMessage::get_instance().check_token_valid())) {
/* get token */
// 如果令牌无效,获取新的令牌
get_kms_token();
}
/* create KMS DEK */
//向 KMS 发送请求,获取指定 CMK ID 的 KMS DEK 的信息
http_msg_list = kms_restful_get_dek(cmk_id, dek_cipher, KMS_GET_DEK);
//返回的消息列表进行解析,得到 KMS DEK 的明文
dek_plain = parser_http_string(http_msg_list, KMS_GET_DEK);
//释放 http_msg_list 占用的内存
tde_list_free(http_msg_list);
if (dek_plain == NULL) {
ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
@ -73,6 +86,7 @@ char* KMSInterface::get_kms_dek(const char* cmk_id, const char* dek_cipher)
return dek_plain;
}
//获取KMS令牌Token
void KMSInterface::get_kms_token()
{
bool result = false;
@ -86,25 +100,38 @@ void KMSInterface::get_kms_token()
int len = 0;
/* get internal user B token */
//向 KMS 发送请求获取内部用户 B 的令牌
http_msg_list = kms_restful_token();
//对返回的消息列表进行解析,得到用户 B 的令牌的值
temp_token = parser_http_string(http_msg_list, IAM_TOKEN);
/* combine token string */
//根据 req_token_tag 和 temp_token 的值,将它们组合成完整的令牌字符串
len = strlen(req_token_tag) + 1 + strlen(temp_token) + 1;
token = (char*)palloc0(len);
rc = snprintf_s(token, len, len, "%s:%s", req_token_tag, temp_token);
securec_check_ss(rc, "\0", "\0");
/* get user A agency token */
//向 KMS 发送请求获取用户 A 代理令牌
http_agency_msg_list = kms_restful_agency_token(token);
//对返回的消息列表进行解析,得到用户 A 代理令牌的值
temp_agency_token = parser_http_string(http_agency_msg_list, IAM_AGENCY_TOKEN);
/* combine agency token string */
//根据 req_token_tag 和 temp_agency_token 的值,将它们组合成完整的代理令牌字符串,保存在 agency_token 中
len = strlen(req_token_tag) + 1 + strlen(temp_agency_token) + 1;
agency_token = (char*)palloc0(len);
rc = snprintf_s(agency_token, len, len, "%s:%s", req_token_tag, temp_agency_token);
securec_check_ss(rc, "\0", "\0");
/* cache all token */
//将用户 B 的令牌和用户 A 的代理令牌缓存起来
result = TDE::CKMSMessage::get_instance().save_token(token, agency_token);
// 释放 http_msg_list 和 http_agency_msg_list 占用的内存
tde_list_free(http_msg_list);
tde_list_free(http_agency_msg_list);
//保存令牌的操作失败result 为 false则释放所有的指针变量并报错
if (result == false) {
pfree_ext(token);
pfree_ext(agency_token);
@ -114,6 +141,7 @@ void KMSInterface::get_kms_token()
errmsg("Could KMS token caching is failed")));
return;
}
//释放所有的指针变量,并返回
pfree_ext(token);
pfree_ext(agency_token);
pfree_ext(temp_token);
@ -121,6 +149,7 @@ void KMSInterface::get_kms_token()
return;
}
//向KMSKey Management Service发送HTTP请求 ( 获取IAM令牌 ) 获取Token的功能
AdvStrList* KMSInterface::kms_restful_token()
{
HttpErrCode ret = TDE_HTTP_SUCCEED;
@ -131,25 +160,38 @@ AdvStrList* KMSInterface::kms_restful_token()
int len = 0;
errno_t rc = EOK;
//获取CKMSMessage类的单例对象然后通过该对象的token_info成员变量访问到token_info对象
// 最后获取该对象的project_name成员变量的值
project_name = TDE::CKMSMessage::get_instance().token_info->project_name;
//构建token_url字符串该字符串是组合了url_iam_head、project_name和url_token的URL
len = strlen(url_iam_head) + strlen(url_token) + strlen(project_name) + 1;
token_url = (char*)palloc0(len);
rc = memcpy_s(token_url, len, url_iam_head, strlen(url_iam_head));
securec_check(rc, "\0", "\0");
rc = memcpy_s((token_url + strlen(url_iam_head)), len, project_name, strlen(project_name));
securec_check(rc, "\0", "\0");
rc = memcpy_s((token_url + strlen(url_iam_head) + strlen(project_name)), len, url_token, (strlen(url_token) + 1));
securec_check(rc, "\0", "\0");
//获取IAM令牌的JSON字符串
http_body = TDE::CKMSMessage::get_instance().get_iam_token_json();
//设置HTTP请求的头部信息其中content_type是一个常量字符串
const char* http_head_list[] = {content_type, NULL};
//构建http_req_msg结构体包括请求方法POST、URL、头部信息和请求体
HttpReqMsg http_req_msg = {HTTP_POST, token_url, NULL, http_body, http_head_list};
//构建http_config结构体设置HTTP请求的超时时间和响应消息类型
HttpConfig http_config = {time_out, HTTP_MSG};
//发送HTTP请求并获取响应结果
ret = HttpCommon::http_request(&http_req_msg, &http_config, &http_msg_list);
/* token_url is not NULL */
//释放动态分配的内存包括token_url和http_body
pfree_ext(token_url);
pfree_ext(http_body);
//判断HTTP请求的返回状态如果不成功抛出错误并输出相应的错误信息
if (ret != TDE_HTTP_SUCCEED) {
// 释放响应消息列表内存空间
tde_list_free(http_msg_list);
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("http request failed"), errdetail("N/A"), errcause("http request error"),
@ -158,6 +200,7 @@ AdvStrList* KMSInterface::kms_restful_token()
return http_msg_list;
}
//向KMSKey Management Service发送HTTP请求获取代理Token的功能
AdvStrList* KMSInterface::kms_restful_agency_token(char* token)
{
HttpErrCode ret = TDE_HTTP_SUCCEED;
@ -168,26 +211,40 @@ AdvStrList* KMSInterface::kms_restful_agency_token(char* token)
int len = 0;
errno_t rc = EOK;
// 获取项目名称
project_name = TDE::CKMSMessage::get_instance().agency_token_info->project_name;
// 计算URL长度
len = strlen(url_iam_head) + strlen(url_token) + strlen(project_name) + 1;
// 分配URL内存空间并初始化为0
agency_token_url = (char*)palloc0(len);
// 将url_iam_head复制到agency_token_url中
rc = memcpy_s(agency_token_url, len, url_iam_head, strlen(url_iam_head));
securec_check(rc, "\0", "\0");
// 将project_name追加到agency_token_url中
rc = memcpy_s((agency_token_url + strlen(url_iam_head)), len, project_name, strlen(project_name));
securec_check(rc, "\0", "\0");
// 将url_token追加到agency_token_url中
rc = memcpy_s((agency_token_url + strlen(url_iam_head) + strlen(project_name)), len, url_token,
(strlen(url_token) + 1));
securec_check(rc, "\0", "\0");
// 获取IAM代理Token的JSON请求体内容
http_body = TDE::CKMSMessage::get_instance().get_iam_agency_token_json();
// HTTP头列表包含内容类型和身份认证
const char* http_head_list[] = {content_type, token, NULL};
// 构造HTTP请求消息
HttpReqMsg http_req_msg = {HTTP_POST, agency_token_url, NULL, http_body, http_head_list};
// HTTP配置包含超时时间和消息类型
HttpConfig http_config = {time_out, HTTP_MSG};
//// 发送HTTP请求并获得响应消息列表
ret = HttpCommon::http_request(&http_req_msg, &http_config, &http_msg_list);
/* agency_token_url is not NULL */
// 释放内存空间
pfree_ext(agency_token_url);
pfree_ext(http_body);
// 判断HTTP请求的返回状态如果不成功抛出错误并输出相应的错误信息
if (ret != TDE_HTTP_SUCCEED) {
// 释放响应消息列表内存空间
tde_list_free(http_msg_list);
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("http request failed"), errdetail("N/A"), errcause("http request error"),
@ -196,6 +253,7 @@ AdvStrList* KMSInterface::kms_restful_agency_token(char* token)
return http_msg_list;
}
//使用KMS接口发送HTTP请求获取DEKData Encryption Key
AdvStrList* KMSInterface::kms_restful_get_dek(const char* cmk_id, const char* dek_cipher, ResetApiType api_type)
{
HttpErrCode ret = TDE_HTTP_SUCCEED;
@ -207,40 +265,59 @@ AdvStrList* KMSInterface::kms_restful_get_dek(const char* cmk_id, const char* de
int len = 0;
errno_t rc = EOK;
//获取项目名称和项目ID
project_name = TDE::CKMSMessage::get_instance().kms_info->project_name;
project_id = TDE::CKMSMessage::get_instance().kms_info->project_id;
//根据api_type判断是生成DEK还是获取DEK并构造相应的URL和HTTP请求体
if (api_type == KMS_GEN_DEK) {
//计算DEK的URL和请求体内容
//拼接url_kms_head、project_name、url_kms_path、project_id和url_create_dek
len = strlen(url_kms_head) + strlen(project_name) + strlen(url_kms_path) + strlen(project_id) +
strlen(url_create_dek) + 1;
dek_url = (char*)palloc0(len);
rc = snprintf_s(dek_url, len, len, "%s%s%s%s%s", url_kms_head, project_name, url_kms_path, project_id,
url_create_dek);
securec_check_ss(rc, "\0", "\0");
//生成创建DEK的JSON请求体
http_body = TDE::CKMSMessage::get_instance().get_create_dek_json(cmk_id);
} else if (api_type == KMS_GET_DEK) {
//计算并构造了获取DEK的URL和HTTP请求体内容
len = strlen(url_kms_head) + strlen(project_name) + strlen(url_kms_path) + strlen(project_id) +
strlen(url_get_dek) + 1;
dek_url = (char*)palloc0(len);
rc = snprintf_s(dek_url, len, len, "%s%s%s%s%s", url_kms_head, project_name, url_kms_path, project_id,
url_get_dek);
securec_check_ss(rc, "\0", "\0");
//生成解密DEK的JSON请求体
http_body = TDE::CKMSMessage::get_instance().get_decrypt_dek_json(cmk_id, dek_cipher);
}
//定义了常量字符指针数组http_head_list包含了两个HTTP头内容content_type和TDE::CKMSMessage::get_instance().tde_agency_token
const char* http_head_list[] = {content_type, TDE::CKMSMessage::get_instance().tde_agency_token, NULL};
//定义了HttpReqMsg结构体http_req_msg设置请求方法为HTTP_POSTURL为dek_url请求体为http_bodyHTTP头列表为http_head_list
HttpReqMsg http_req_msg = {HTTP_POST, dek_url, NULL, http_body, http_head_list};
//定义了HttpConfig结构体http_config设置超时时间为time_out消息类型为HTTP_RESBODY
HttpConfig http_config = {time_out, HTTP_RESBODY};
//发送HTTP请求并将响应消息列表保存到http_msg_list中
ret = HttpCommon::http_request(&http_req_msg, &http_config, &http_msg_list);
//释放dek_url和http_body的内存空间
pfree_ext(dek_url);
pfree_ext(http_body);
//如果HTTP请求失败
if (ret != TDE_HTTP_SUCCEED) {
//释放http_msg_list的内存空间报告错误信息
tde_list_free(http_msg_list);
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("http request failed"), errdetail("N/A"), errcause("http request error"),
erraction("check KMS or IAM connect or config parameter")));
}
//返回http_msg_list作为响应结果
return http_msg_list;
}
//用于解析HTTP响应消息列表根据不同的API类型提取特定的信息并返回解析结果
char* KMSInterface::parser_http_string(AdvStrList* http_msg_list, ResetApiType api_type)
{
char* result = NULL;
@ -249,40 +326,57 @@ char* KMSInterface::parser_http_string(AdvStrList* http_msg_list, ResetApiType a
errno_t rc = EOK;
if ((api_type == IAM_TOKEN) || (api_type == IAM_AGENCY_TOKEN)) {
//api_type为 IAM_TOKEN 或 IAM_AGENCY_TOKEN
//在http_msg_list中查找指定的标签get_token_tag
token_tmp = find_resheader(http_msg_list, get_token_tag);
if (token_tmp == NULL) {
//token_tmp为NULL则发出错误报告指示无法获取IAM token或IAM agency token并终止执行
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("get iam token or iam agency token is NULL"), errdetail("N/A"),
errcause("connect IAM failed"),
erraction("check if your env can connect with IAM server")));
}
result = (char*)palloc0(strlen(token_tmp) + 1);
//为result分配内存空间大小为token_tmp的长度加1
result = (char *)palloc0(strlen(token_tmp) + 1);
// 将token_tmp的内容复制到result中
rc = memcpy_s(result, (strlen(token_tmp) + 1), token_tmp, (strlen(token_tmp) + 1));
securec_check(rc, "\0", "\0");
} else if (api_type == KMS_GET_DEK) {
//api_type 为 KMS_GET_DEK
//从http_msg_list中获取第一个元素即KMS返回的JSON字符串
// 解析该JSON字符串
cJSON *dek_json = cJSON_Parse(tde_get_val(http_msg_list, 0));
if (cJSON_GetObjectItem(dek_json, data_key) == NULL) {
//如果解析后的JSON对象中没有包含指定的keydata_key
// 发出错误报告指示KMS返回的dek json key为空并终止执行
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("KMS dek json key is NULL"), errdetail("N/A"),
errcause("KMS return value error"),
erraction("check KMS config paramenter")));
}
//从解析后的JSON对象中获取指定keydata_key的值即DEK的JSON字符串
json_string = cJSON_GetObjectItem(dek_json, data_key)->valuestring;
if (json_string == NULL) {
//如果该值为空则发出错误报告指示无法获取KMS的DEK并终止执行
//删除JSON对象dek_json
cJSON_Delete(dek_json);
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("get kms dek is NULL"), errdetail("N/A"),
errcause("connect KMS failed"),
erraction("check if your env can connect with KMS server")));
}
//为result分配内存空间大小为json_string的长度加1
result = (char*)palloc0(strlen(json_string) + 1);
//将json_string的内容复制到result中
rc = memcpy_s(result, (strlen(json_string) + 1), json_string, (strlen(json_string) + 1));
securec_check(rc, "\0", "\0");
//删除解析后的JSON对象dek_json
cJSON_Delete(dek_json);
}
return result;
}
//用于解析HTTP响应消息列表并根据不同的API类型提取DEKData Encryption Key的明文和密文信息
DekInfo* KMSInterface::parser_http_array(AdvStrList* http_msg_list, ResetApiType api_type)
{
DekInfo* dek_info = NULL;
@ -291,21 +385,32 @@ DekInfo* KMSInterface::parser_http_array(AdvStrList* http_msg_list, ResetApiType
char* cipher_json = NULL;
errno_t rc = EOK;
//如果api_type为KMS_GEN_DEK
if (api_type == KMS_GEN_DEK) {
//调用tde_get_val函数从http_msg_list中获取第一个元素即KMS返回的JSON字符串
// 并使用cJSON_Parse函数解析该JSON字符串
dek_json = cJSON_Parse(tde_get_val(http_msg_list, 0));
//判断解析后的JSON对象中是否包含指定的keyplain_text和cipher_text
if ((cJSON_GetObjectItem(dek_json, plain_text) == NULL) ||
(cJSON_GetObjectItem(dek_json, cipher_text) == NULL)) {
//没有或者对应的值为空,则发出错误报告指示
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("KMS dek json key is NULL"), errdetail("N/A"),
errcause("KMS return value error"),
erraction("check KMS config paramenter")));
}
//从解析后的JSON对象中获取指定keyplain_text和cipher_text的值
plain_json = cJSON_GetObjectItem(dek_json, plain_text)->valuestring;
cipher_json = cJSON_GetObjectItem(dek_json, cipher_text)->valuestring;
if ((plain_json == NULL) || (cipher_json == NULL)) {
//如果这两个值中有任何一个为空则删除解析后的JSON对象dek_json并返回NULL
cJSON_Delete(dek_json);
return NULL;
}
//为dek_info分配内存空间
//并分别为dek_info->plain和dek_info->cipher分配内存空间
//大小分别为plain_json和cipher_json的长度加1
//并使用memcpy_s函数将对应的JSON字符串复制到dek_info相应的成员变量中
dek_info = (DekInfo*)palloc0(sizeof(DekInfo));
dek_info->plain = (char*)palloc0(strlen(plain_json) + 1);
rc = memcpy_s(dek_info->plain, (strlen(plain_json) + 1), plain_json, (strlen(plain_json) + 1));
@ -313,28 +418,40 @@ DekInfo* KMSInterface::parser_http_array(AdvStrList* http_msg_list, ResetApiType
dek_info->cipher = (char*)palloc0(strlen(cipher_json) + 1);
rc = memcpy_s(dek_info->cipher, (strlen(cipher_json) + 1), cipher_json, (strlen(cipher_json) + 1));
securec_check(rc, "\0", "\0");
//删除解析后的JSON对象dek_json
cJSON_Delete(dek_json);
}
//返回dek_info作为解析结果
return dek_info;
}
//在HTTP响应消息头resheader列表中查找特定类型的消息头并返回对应的消息值
char *KMSInterface::find_resheader(AdvStrList *resheader_list, const char *resheader_type)
{
char *ret = NULL;
//使用for循环遍历resheader_list中的每个元素
for (size_t i = 0; i < tde_list_len(resheader_list); i++) {
//将当前元素按照冒号(:进行分割并将分割后得到的header信息保存在cur_header中
AdvStrList *cur_header = tde_split_node(tde_get_val(resheader_list, i), ':');
//判断cur_header和cur_header的第一个元素即header类型是否为空
if ((cur_header == NULL) || (tde_get_val(cur_header, 0) == NULL)) {
//如果有任何一个为空则发出错误报告指示获取HTTP响应头失败并终止执行
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("get http header is NULL"), errdetail("N/A"),
errcause("http request failed"),
erraction("check IAM config parameter")));
}
//如果resheader_list长度大于1且当前消息头的类型等于指定的resheader_type
if (tde_list_len(resheader_list) > 1 && strcmp(resheader_type, tde_get_val(cur_header, 0)) == 0) {
//将ret赋值为cur_header的第二个元素即消息值
ret = tde_get_val(cur_header, 1);
//释放cur_header
free_advstr_list_with_skip(cur_header, 1);
//跳出循环
break;
} else {
} else {//如果条件不满足
//释放cur_header
free_advstr_list(cur_header);
}
}

View File

@ -31,6 +31,7 @@
#include "utils/memutils.h"
#include "knl/knl_session.h"
//默认的构造函数和析构函数,用于初始化和销毁相关资源
TDEKeyManager::TDEKeyManager()
{
tde_create_data = NULL;
@ -54,6 +55,13 @@ TDEKeyManager::~TDEKeyManager()
}
}
/*
init()TDEKeyManager对象
TDE缓存KMSInterface对象以及创建TDEData对象
TDE::TDEKeyStorage::get_instance().empty()
trueTDE::TDEKeyStorage::get_instance().init()
*/
void TDEKeyManager::init()
{
if (TDE::TDEKeyStorage::get_instance().empty()) {
@ -65,6 +73,8 @@ void TDEKeyManager::init()
return;
}
//使用KMS接口创建数据加密密钥(DEK)过KMSInterface对象的create_kms_dek函数创建DEK
// 并将DEK信息保存在tde_create_data中然后返回tde_create_data指针
const TDEData* TDEKeyManager::create_dek()
{
errno_t rc = EOK;
@ -72,60 +82,81 @@ const TDEData* TDEKeyManager::create_dek()
DekInfo* dek_info = NULL;
/* use KMS interface to create DEK */
cmk_id = get_cmk_id();
tde_create_data->cmk_id = (char*)palloc0(strlen(cmk_id) + 1);
rc = memcpy_s(tde_create_data->cmk_id, (strlen(cmk_id) + 1), cmk_id, (strlen(cmk_id) + 1));
cmk_id = get_cmk_id();// 调用get_cmk_id()函数获取cmk_id
tde_create_data->cmk_id = (char*)palloc0(strlen(cmk_id) + 1);// 分配内存并将cmk_id复制到tde_create_data->cmk_id中
rc = memcpy_s(tde_create_data->cmk_id, (strlen(cmk_id) + 1), cmk_id, (strlen(cmk_id) + 1));// 复制cmk_id到tde_create_data->cmk_id
securec_check(rc, "\0", "\0");
//使用KMSInterface对象的create_kms_dek函数创建DEK返回一个DekInfo结构体指针dek_info
dek_info = kms_instance->create_kms_dek(tde_create_data->cmk_id);
//如果返回值为NULL则抛出错误并提供相关错误信息
if (dek_info == NULL) {
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("create KMS dek failed"), errdetail("N/A"), errcause("KMS error"),
erraction("check KMS connect or config parameter")));
}
// 分配内存并将dek_info->cipher复制到tde_create_data->dek_cipher中
tde_create_data->dek_cipher = (char*)palloc0(strlen(dek_info->cipher) + 1);
rc = memcpy_s(tde_create_data->dek_cipher, (strlen(dek_info->cipher) + 1), dek_info->cipher,
(strlen(dek_info->cipher) + 1));
securec_check(rc, "\0", "\0");
// 分配内存并将dek_info->plain复制到tde_create_data->dek_plaintext中
tde_create_data->dek_plaintext = (char*)palloc0(strlen(dek_info->plain) + 1);
securec_check(rc, "\0", "\0");
rc = memcpy_s(tde_create_data->dek_plaintext, (strlen(dek_info->plain) + 1), dek_info->plain,
(strlen(dek_info->plain) + 1));
securec_check(rc, "\0", "\0");
// 将dek_info->cipher的内容清空
rc = memset_s(dek_info->cipher, strlen(dek_info->cipher), 0, strlen(dek_info->cipher));
securec_check(rc, "\0", "\0");
// 释放占用的内存
pfree_ext(dek_info->cipher);
pfree_ext(dek_info->plain);
pfree_ext(dek_info);
return tde_create_data;
}
//获取指定cmk_id和dek_cipher对应的DEK
//通过KMSInterface对象的get_kms_dek函数获取DEK并将DEK信息保存在tde_get_data变量中
const TDEData* TDEKeyManager::get_dek(const char* cmk_id, const char* dek_cipher)
{
errno_t rc = EOK;
char* dek_plain = NULL;
/* use KMS interface to get DEK */
// 分配内存并将cmk_id复制到tde_get_data->cmk_id中
tde_get_data->cmk_id = (char*)palloc0(strlen(cmk_id) + 1);
rc = memcpy_s(tde_get_data->cmk_id, (strlen(cmk_id) + 1), cmk_id, (strlen(cmk_id) + 1));
securec_check(rc, "\0", "\0");
// 使用KMS实例的get_kms_dek函数获取DEK返回DEK明文dek_plain
dek_plain = kms_instance->get_kms_dek(tde_get_data->cmk_id, dek_cipher);
// 如果返回值为NULL则抛出错误并提供相关错误信息
if (dek_plain == NULL) {
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("get KMS dek failed"), errdetail("N/A"), errcause("KMS error"),
erraction("check KMS connect or config parameter")));
}
// 分配内存并将dek_cipher复制到tde_get_data->dek_cipher中
tde_get_data->dek_cipher = (char*)palloc0(strlen(dek_cipher) + 1);
rc = memcpy_s(tde_get_data->dek_cipher, (strlen(dek_cipher) + 1), dek_cipher, (strlen(dek_cipher) + 1));
securec_check(rc, "\0", "\0");
// 分配内存并将dek_plain复制到tde_get_data->dek_plaintext中
tde_get_data->dek_plaintext = (char*)palloc0(strlen(dek_plain) + 1);
rc = memcpy_s(tde_get_data->dek_plaintext, (strlen(dek_plain) + 1), dek_plain, (strlen(dek_plain) + 1));
securec_check(rc, "\0", "\0");
// 将dek_plain的内容清空
rc = memset_s(dek_plain, strlen(dek_plain), 0, strlen(dek_plain));
securec_check(rc, "\0", "\0");
// 释放dek_plain占用的内存
pfree_ext(dek_plain);
return tde_get_data;
}
//用于获取cmk_id
char* TDEKeyManager::get_cmk_id()
{
//检查u_sess->attr.attr_security.tde_cmk_id是否为NULL或空字符串
//如果是则抛出错误
if (u_sess->attr.attr_security.tde_cmk_id == NULL || strlen(u_sess->attr.attr_security.tde_cmk_id) == 0) {
ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("get cmk id failed for Transparent Data Encryption"),
@ -134,6 +165,7 @@ char* TDEKeyManager::get_cmk_id()
return u_sess->attr.attr_security.tde_cmk_id;
}
//用于保存DEK信息到TDE缓存中
bool TDEKeyManager::save_key(const TDEData* tde_data)
{
errno_t rc = EOK;
@ -142,43 +174,61 @@ bool TDEKeyManager::save_key(const TDEData* tde_data)
TimestampTz cur_timestamp = 0;
/* prepare TDE cache entry */
/*
TDECacheEntry
TDEData
使 palloc0 DEK DEK
使 memcpy_s tde_data kms_cache_entry
*/
kms_cache_entry = (TDECacheEntry*)palloc0(sizeof(TDECacheEntry));
kms_cache_entry->key_cipher = (char*)palloc0(strlen(tde_data->dek_cipher) + 1);
rc = memcpy_s(kms_cache_entry->key_cipher, (strlen(tde_data->dek_cipher) + 1), tde_data->dek_cipher,
(strlen(tde_data->dek_cipher) + 1));
securec_check(rc, "\0", "\0");
kms_cache_entry->dek_plaintext = (char*)palloc0(strlen(tde_data->dek_plaintext) + 1);
rc = memcpy_s(kms_cache_entry->dek_plaintext, (strlen(tde_data->dek_plaintext) + 1), tde_data->dek_plaintext,
(strlen(tde_data->dek_plaintext) + 1));
securec_check(rc, "\0", "\0");
//获取当前的时间戳,并将其赋值给 kms_cache_entry 的 timestamp 成员变量
cur_timestamp = GetCurrentTimestamp();
kms_cache_entry->timestamp = cur_timestamp;
/* insert TDE key into TDE cache */
result = TDE::TDEKeyStorage::get_instance().insert_cache(kms_cache_entry);
//插入失败,代码会触发错误处理并返回错误信息
if (!result) {
ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION),
errmsg("TDE cache insert failed")));
}
//将 kms_cache_entry 的敏感数据 DEK 明文清零,以保护数据的安全性
rc = memset_s(kms_cache_entry->dek_plaintext, strlen(kms_cache_entry->dek_plaintext), 0,
strlen(kms_cache_entry->dek_plaintext));
securec_check(rc, "\0", "\0");
//释放 kms_cache_entry 的内存空间
pfree_ext(kms_cache_entry->key_cipher);
pfree_ext(kms_cache_entry->dek_plaintext);
pfree_ext(kms_cache_entry);
return result;
}
//获取指定cmk_id和dek_cipher对应的DEK密钥
//首先在TDE缓存中查找是否存在该DEK密钥如果存在则直接返回
//如果不存在则调用get_dek函数获取DEK并将其保存到TDE缓存中
const char* TDEKeyManager::get_key(const char* cmk_id, const char* dek_cipher)
{
errno_t rc = EOK;
char* kms_cipher = NULL;
char* kms_plaintext = NULL;
//使用 palloc0 函数分配足够的内存空间来存储 dek_cipher 的副本
// 并使用 memcpy_s 函数将 dek_cipher 的数据复制到 kms_cipher 中
kms_cipher = (char*)palloc0(strlen(dek_cipher) + 1);
rc = memcpy_s(kms_cipher, (strlen(dek_cipher) + 1), dek_cipher,
(strlen(dek_cipher) + 1));
securec_check(rc, "\0", "\0");
//调用 TDE::TDEKeyStorage::get_instance().search_cache() 函数,在缓存中搜索与 kms_cipher 匹配的密钥
// 如果找到了对应的明文密钥,代码会释放 kms_cipher 的内存空间,并直接返回该明文密钥
kms_plaintext = TDE::TDEKeyStorage::get_instance().search_cache(kms_cipher);
if (kms_plaintext != NULL) {
/* cache has key */
@ -186,19 +236,26 @@ const char* TDEKeyManager::get_key(const char* cmk_id, const char* dek_cipher)
return kms_plaintext;
} else {
/* key not found */
//如果在缓存中没有找到匹配的密钥
const TDEData* tde_data = NULL;
//调用 get_dek(cmk_id, kms_cipher) 函数尝试从外部密钥管理系统获取 DEK 数据
tde_data = get_dek(cmk_id, kms_cipher);
//获取失败(返回值为 NULL
if (tde_data == NULL) {
//释放 kms_cipher 的内存空间,并触发错误处理,返回错误信息
pfree_ext(kms_cipher);
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("get KMS DEK is NULL"), errdetail("N/A"), errcause("get KMS dek_plaintext failed"),
erraction("check KMS network or cipher is right")));
return kms_plaintext;
}
//成功获取到 DEK 数据
/* update to cache */
save_key(tde_data);
//表示获取到了正确的明文密钥
kms_plaintext = tde_data->dek_plaintext;
}
//释放 kms_cipher 的内存空间
pfree_ext(kms_cipher);
return kms_plaintext;
}

View File

@ -29,255 +29,368 @@
#include "access/hash.h"
namespace TDE {
//TDE 模块启动时,初始化 TDE key 管理哈希表 tde_cache以便后续程序可以向其中添加、查询或删除 TDE key
void TDEKeyStorage::init()
{
// 检查 tde_cache_mem 是否为空指针
if (tde_cache_mem == nullptr) {
// 如果为空指针,则在全局缓存上下文中创建一个名为 "TDE_CACHE_CONTEXT" 的内存上下文
// 并将其赋值给 tde_cache_mem 变量
tde_cache_mem = AllocSetContextCreate(g_instance.cache_cxt.global_cache_mem, "TDE_CACHE_CONTEXT",
ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE, SHARED_CONTEXT);
}
// 检查 tde_cache 是否为空指针
if (tde_cache != NULL) {
// 如果不为空指针,表示 TDE 缓存已经初始化过了,直接返回
return;
}
// 构建一个 HASHCTL 结构体 tde_ctl
HASHCTL tde_ctl;
//将 tde_ctl 的内存块清零
errno_t rc = memset_s(&tde_ctl, sizeof(tde_ctl), 0, sizeof(tde_ctl));
securec_check(rc, "", "");
tde_ctl.keysize = sizeof(char *);
tde_ctl.entrysize = sizeof(TDECacheEntry);
tde_ctl.hash = (HashValueFunc)tde_cache_entry_hash_func;
tde_ctl.match = (HashCompareFunc)tde_cache_entry_match_func;
tde_ctl.hcxt = tde_cache_mem;
// 设置 tde_ctl 中的字段值
tde_ctl.keysize = sizeof(char *); // 哈希表键的大小,这里是指针变量的大小
tde_ctl.entrysize = sizeof(TDECacheEntry); // 每个哈希表条目的大小
tde_ctl.hash = (HashValueFunc)tde_cache_entry_hash_func;
tde_ctl.match = (HashCompareFunc)tde_cache_entry_match_func;
tde_ctl.hcxt = tde_cache_mem; // 哈希表使用的内存上下文,之前创建的 tde_cache_mem 上下文
//告知该函数在创建哈希表时要使用哪些功能和设置
int flags = HASH_ELEM | HASH_CONTEXT | HASH_FUNCTION | HASH_COMPARE;
// 使用 hash_create 函数创建哈希表 tde_cache
tde_cache = hash_create(tde_cache_name, max_bukect, &tde_ctl, flags);
//// 检查哈希表是否创建成功
if (tde_cache == NULL) {
//// 检查哈希表是否创建成功
ereport(ERROR, (errcode(ERRCODE_FUNCTION_HASH_NOT_INITED),
errmsg("could not initialize TDE key manager hash table")));
}
}
//针对 TDE 缓存哈希表中的每个键值对
// 基于键的字符串内容计算出一个哈希值
// 用于确定该键值对在哈希表中的存储位置
uint32 TDEKeyStorage::tde_cache_entry_hash_func(const void* key, Size keysize)
{
//通过断言Assert来确保传入的参数 keysize 大于 0并且 key 不为空指针NULL
Assert(keysize > 0);
Assert(key != NULL);
//强制转换为 const char* 类型
const char *dek = *(char **)key;
//计算字符串 dek 的长度
int s_len = strlen(dek);
//使用 PostgreSQL 提供的 hash_any 函数
// 将 dek 强制转换为 const unsigned char* 类型
// 并指定字符串长度 s_len计算出哈希值
// 然后,使用 DatumGetUInt32 将哈希值转换为 uint32 类型,并返回哈希值结果
return (DatumGetUInt32(hash_any((const unsigned char *)dek, s_len)));
}
//用于比较两个密钥缓存条目的匹配情况
int TDEKeyStorage::tde_cache_entry_match_func(const void* key1, const void* key2, Size keySize)
{
//通过断言Assert来确保输入参数的有效性
Assert(keySize > 0);
Assert(key1 != NULL);
Assert(key2 != NULL);
/* de-reference char ** to char * to do comparison */
//强制转换为 char ** 类型的指针,并通过解引用操作得到 char * 类型的指针
const char *dek1 = *(char **)key1;
const char *dek2 = *(char **)key2;
//计算 dek1 和 dek2 字符串的长度
int dek1_len = strlen(dek1);
int dek2_len = strlen(dek2);
//基于字符串内容比较,较短的字符串会被自动填充到与较长字符串相同的长度再进行比较
// 返回值为 0 表示匹配,非零值表示不匹配
return (strncmp(dek1, dek2, ((dek1_len > dek2_len) ? dek1_len : dek2_len)));
}
//构造函数
TDEKeyStorage::TDEKeyStorage()
{
tde_cache_mem = nullptr;
tde_cache = NULL;
tde_cache_mem = nullptr; //表示缓存内存上下文还未分配
tde_cache = NULL; //表示缓存对象未初始化
}
//析构函数
TDEKeyStorage::~TDEKeyStorage()
{
reset();
if (tde_cache_mem != nullptr) {
MemoryContextDelete(tde_cache_mem);
tde_cache_mem = nullptr;
reset(); //将 TDEKeyStorage 对象重置为初始状态
if (tde_cache_mem != nullptr) { //如果不为空
MemoryContextDelete(tde_cache_mem); //删除内存上下文
tde_cache_mem = nullptr; //表示内存上下文已被删除
}
}
//用于检查 TDE 缓存是否为空
bool TDEKeyStorage::empty()
{
if (tde_cache == NULL) {
//如果返回 true则表示缓存为空
return true;
}
return false;
}
//重置 TDE 缓存,并释放相关资源
void TDEKeyStorage::reset()
{
//清空缓存中的数据
clear();
//检查 tde_cache 是否为 NULL
if (tde_cache != NULL) {
//获取 TDEKeyCacheLock 锁,以确保在访问缓存期间不会发生冲突
LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE);
//将 tde_cache 重置为初始状态
HeapMemResetHash(tde_cache, tde_cache_name);
tde_cache = NULL;
tde_cache = NULL; //表示缓存已经被重置
//释放 TDEKeyCacheLock 锁,确保其他线程可以访问缓存
LWLockRelease(TDEKeyCacheLock);
}
}
//清空 TDEKeyStorage 对象中的缓存数据,以便重新使用或释放内存
void TDEKeyStorage::clear()
{
//检查 tde_cache 是否为非空指针
if (tde_cache != NULL) {
// 获取 TDEKeyCacheLock 互斥锁
LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE);
// 初始化哈希表扫描器
HASH_SEQ_STATUS scan_state;
hash_seq_init(&scan_state, tde_cache);
// 遍历哈希表中的所有条目,并删除它们
TDECacheEntry* item = NULL;
while ((item = reinterpret_cast<TDECacheEntry*>(hash_seq_search(&scan_state))) != NULL) {
// 清空缓存中数据,并释放其占用的内存空间
clean_cache_entry_value(item->dek_plaintext);
// 删除当前条目
if (hash_search(tde_cache, (const void*)&item->key_cipher, HASH_REMOVE, NULL) == NULL) {
// 如果删除失败,则释放互斥锁并触发错误
LWLockRelease(TDEKeyCacheLock);
ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION),
errmsg("TDE Key storage clear cache: remove entry failed")));
}
}
// 释放 TDEKeyCacheLock 互斥锁
LWLockRelease(TDEKeyCacheLock);
}
}
//将指定的缓存对象(即 dek_plaintext中的内容清空为全零
void TDEKeyStorage::reset_dek_plaintext(char* dek_plaintext)
{
errno_t rc = 0;
//调用标准库函数 memset_s(),将 dek_plaintext 中的内容设置为全零
rc = memset_s(dek_plaintext, strlen(dek_plaintext), 0, strlen(dek_plaintext));
securec_check(rc, "\0", "\0");
}
//将指定的缓存对象(即 dek_plaintext中的内容清空为全零并释放其占用的内存空间
void TDEKeyStorage::clean_cache_entry_value(char* dek_plaintext)
{
//检查 dek_plaintext 是否为非空指针
if (dek_plaintext != NULL) {
//调用 reset_dek_plaintext() 方法,将其内容清空为全零
reset_dek_plaintext(dek_plaintext);
//释放 dek_plaintext 占用的内存空间
pfree_ext(dek_plaintext);
}
}
//用于向 tde_cache 哈希表中插入或更新缓存数据
bool TDEKeyStorage::insert_cache(TDECacheEntry* tde_cache_entry)
{
bool found = false;
errno_t rc = EOK;
//切换内存上下文至 tde_cache_mem
MemoryContext old = MemoryContextSwitchTo(tde_cache_mem);
/* build tde cache key */
//根据 tde_cache_entry 的 key_cipher 字段的长度动态分配内存
char* tde_cache_key = (char*)palloc0(strlen(tde_cache_entry->key_cipher) + 1);
//将 key_cipher 字符串拷贝到新分配的内存中
rc = memcpy_s(tde_cache_key, (strlen(tde_cache_entry->key_cipher) + 1), tde_cache_entry->key_cipher,
(strlen(tde_cache_entry->key_cipher) + 1));
securec_check(rc, "\0", "\0");
/* insert hash table */
//获取 TDEKeyCacheLock 读写锁
LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE);
TDECacheEntry* entry = NULL;
//调用 hash_search() 函数在哈希表 tde_cache 中查找是否存在 tde_cache_key 等于 tde_cache_entry->key_cipher 的缓存项
entry = reinterpret_cast<TDECacheEntry*>(hash_search(tde_cache, (const void*)&tde_cache_key, HASH_FIND, &found));
if (!found) {
//如果未找到对应的缓存项,则调用 hash_search() 函数以 HASH_ENTER 操作插入新的缓存项
entry = reinterpret_cast<TDECacheEntry*>
(hash_search(tde_cache, (const void*)&tde_cache_key, HASH_ENTER, &found));
if (entry == NULL) {
//如果插入失败(返回 NULL则释放相关内存
pfree_ext(tde_cache_key);
LWLockRelease(TDEKeyCacheLock);
MemoryContextSwitchTo(old);
LWLockRelease(TDEKeyCacheLock); //释放锁
MemoryContextSwitchTo(old); //恢复内存上下文
//抛出异常报错
ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE),
errmsg("build TDE key cache hash table failed")));
return false;
}
/* insert new cache key-value */
//如果插入成功,分配内存并将 dek_plaintext 字符串拷贝到新分配的内存中
entry->dek_plaintext = (char*)palloc0(strlen(tde_cache_entry->dek_plaintext) + 1);
rc = memcpy_s(entry->dek_plaintext, (strlen(tde_cache_entry->dek_plaintext) + 1),
tde_cache_entry->dek_plaintext, (strlen(tde_cache_entry->dek_plaintext) + 1));
securec_check(rc, "\0", "\0");
//更新缓存的时间戳
entry->timestamp = tde_cache_entry->timestamp;
} else {
} else { //如果找到对应的缓存项
/* update cache key-value */
//更新缓存中的 dek_plaintext 字段,以及时间戳字段
rc = memcpy_s(entry->dek_plaintext, (strlen(tde_cache_entry->dek_plaintext) + 1),
tde_cache_entry->dek_plaintext, (strlen(tde_cache_entry->dek_plaintext) + 1));
securec_check(rc, "\0", "\0");
entry->timestamp = tde_cache_entry->timestamp;
}
//恢复先前的内存上下文,并释放锁
MemoryContextSwitchTo(old);
LWLockRelease(TDEKeyCacheLock);
//返回 true表示插入或更新缓存成功
return true;
}
//根据给定的 dek_cipher 在哈希表中搜索缓存项,并返回找到的缓存项的明文密钥
char* TDEKeyStorage::search_cache(const char* dek_cipher)
{
bool found = false;
//切换到 tde_cache_mem 内存上下文
MemoryContext old = MemoryContextSwitchTo(tde_cache_mem);
//获取锁
LWLockAcquire(TDEKeyCacheLock, LW_SHARED);
TDECacheEntry* entry = NULL;
//通过 hash_search() 函数在哈希表 tde_cache 中查找与给定的 dek_cipher 相关的缓存项
entry = reinterpret_cast<TDECacheEntry*>(hash_search(tde_cache, (const void*)&dek_cipher, HASH_FIND, &found));
if (!found) {
//如果未找到对应的缓存项,则恢复到之前的内存上下文 old
MemoryContextSwitchTo(old);
//释放锁 TDEKeyCacheLock并返回 NULL表示未找到缓存项
LWLockRelease(TDEKeyCacheLock);
return NULL;
}
//找到了对应的缓存项,同样恢复到之前的内存上下文 old释放锁
MemoryContextSwitchTo(old);
LWLockRelease(TDEKeyCacheLock);
//返回缓存项的明文密钥 dek_plaintext 字符串
return entry->dek_plaintext;
}
//实现了定期清理过期的缓存项,保持 TDE 密钥缓存的有效性
void TDEKeyStorage::cache_watch_dog()
{
TimestampTz cur_timestamp = 0;
if (tde_cache != NULL) {
//如果哈希表 tde_cache 不为空,则获取当前时间戳并赋值给 cur_timestamp
cur_timestamp = GetCurrentTimestamp();
//以独占模式获取锁
LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE);
//初始化哈希表遍历的状态 scan_state并使用 hash_seq_init() 函数初始化遍历状态,将其与哈希表 tde_cache 相关联
HASH_SEQ_STATUS scan_state;
hash_seq_init(&scan_state, tde_cache);
TDECacheEntry* item = NULL;
//通过循环调用 hash_seq_search() 函数遍历哈希表中的缓存项
while ((item = reinterpret_cast<TDECacheEntry*>(hash_seq_search(&scan_state))) != NULL) {
//如果遍历到了缓存项,则进入循环体中的代码块
if ((cur_timestamp - item->timestamp) > USECS_PER_HOUR) {
//检查当前缓存项的时间戳与当前时间戳之差是否大于一个小时(即判断是否过期)
//清理缓存项的值
clean_cache_entry_value(item->dek_plaintext);
//使用 hash_search() 函数在哈希表中删除该缓存项
if (hash_search(tde_cache, (const void*)&item->key_cipher, HASH_REMOVE, NULL) == NULL) {
//如果删除失败,则释放锁 TDEKeyCacheLock抛出异常报错提示删除缓存项失败
LWLockRelease(TDEKeyCacheLock);
ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION),
errmsg("TDE Key storage WatchDog cache: remove entry failed")));
}
}
}
//释放锁
LWLockRelease(TDEKeyCacheLock);
}
return;
}
/* TDE buffer rel file node cache management */
//构造函数
TDEBufferCache::TDEBufferCache()
{
tde_buffer_mem = nullptr;
tde_buffer_cache = NULL;
tde_buffer_mem = nullptr; //表示内存上下文还未分配
tde_buffer_cache = NULL; //表示缓存对象未初始化
}
//析构函数
TDEBufferCache::~TDEBufferCache()
{
reset();
reset(); //进行重置操作
//判断是否分配了内存上下文
if (tde_buffer_mem != nullptr) {
MemoryContextDelete(tde_buffer_mem);
tde_buffer_mem = nullptr;
MemoryContextDelete(tde_buffer_mem); //删除该内存上下文
tde_buffer_mem = nullptr; //将 tde_buffer_mem 置为 nullptr以释放相关的内存资源
}
}
//用于检查 TDE 缓存是否为空
bool TDEBufferCache::empty()
{
if (tde_buffer_cache == NULL) {
//如果 TDE 缓存中没有任何缓存条目,返回 true
return true;
}
//否则,返回 false表示缓存非空
return false;
}
//实现了重置 TDE 缓冲区缓存的功能,包括清空缓存和重新初始化相关数据结构
void TDEBufferCache::reset()
{
clear();
clear(); //清空缓存
//判断是否已经分配了缓存结构
if (tde_buffer_cache != NULL) {
//以独占模式获取锁
LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE);
//重新初始化缓存的哈希表结构 tde_buffer_cache
HeapMemResetHash(tde_buffer_cache, tde_buffer_name);
tde_buffer_cache = NULL;
tde_buffer_cache = NULL; //表示缓存已被重置
//释放锁
LWLockRelease(TDEKeyCacheLock);
}
}
//实现了清空 TDE 缓冲区缓存的功能
void TDEBufferCache::clear()
{
//检查 tde_buffer_cache 是否为 NULL判断是否已经分配了缓存结构
if (tde_buffer_cache != NULL) {
//以独占模式获取锁
LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE);
//初始化哈希表扫描状态 scan_state并将其与缓存结构 tde_buffer_cache 相关联
HASH_SEQ_STATUS scan_state;
hash_seq_init(&scan_state, tde_buffer_cache);
TdeFileNodeEntry* item = NULL;
//用 hash_seq_search() 函数遍历哈希表中的每个项,并将当前项的指针保存在变量 item 中
TdeFileNodeEntry *item = NULL;
// 在循环中,检查变量 item 是否为空
while ((item = reinterpret_cast<TdeFileNodeEntry*>(hash_seq_search(&scan_state))) != NULL) {
// 使用 hash_search() 函数将当前项从哈希表中移除
if (hash_search(tde_buffer_cache, (const void*)&item->tde_node, HASH_REMOVE, NULL) == NULL) {
//如果移除失败(返回值为 NULL则释放锁 ,并抛出一个错误报告
LWLockRelease(TDEKeyCacheLock);
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_DATA_EXCEPTION),
errmsg("clear TDE buffer cache key failed"), errdetail("N/A"),
@ -285,103 +398,150 @@ void TDEBufferCache::clear()
erraction("check TDE_BUFFER_CACHE_CONTEXT or system cache")));
}
}
//如果为空,表示已经遍历完哈希表中的所有项,退出循环
//释放锁
LWLockRelease(TDEKeyCacheLock);
}
}
//用于创建和初始化缓存结构,并分配相应的内存空间
void TDEBufferCache::init()
{
//检查全局变量 tde_buffer_mem 是否为空
if (tde_buffer_mem == nullptr) {
//如果为空,创建一个分配集合上下文 TDE_BUFFER_CACHE_CONTEXT
// 该上下文使用全局缓存内存 g_instance.cache_cxt.global_cache_mem并设置默认的内存大小
tde_buffer_mem = AllocSetContextCreate(g_instance.cache_cxt.global_cache_mem, "TDE_BUFFER_CACHE_CONTEXT",
ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE, SHARED_CONTEXT);
}
//检查缓存结构 tde_buffer_cache 是否为空
if (tde_buffer_cache != NULL) {
//如果不为空,则说明已经初始化过,直接返回,不再重复初始化
return;
}
//如果 tde_buffer_cache 为空,则继续执行以下操作
/* initialize the hash table */
//初始化 HASHCTL 结构体 tde_buffer_ctl并将其清零
HASHCTL tde_buffer_ctl;
errno_t rc = memset_s(&tde_buffer_ctl, sizeof(tde_buffer_ctl), 0, sizeof(tde_buffer_ctl));
securec_check(rc, "", "");
//设置哈希表的键大小 keysize 为 sizeof(RelFileNode)
// 条目大小 entrysize 为 sizeof(TdeFileNodeEntry)
// 设置哈希函数 hash 为 tag_hash
// 上下文 hcxt 为 tde_buffer_mem
tde_buffer_ctl.keysize = sizeof(RelFileNode);
tde_buffer_ctl.entrysize = sizeof(TdeFileNodeEntry);
tde_buffer_ctl.hash = tag_hash;
tde_buffer_ctl.hcxt = tde_buffer_mem;
//创建哈希表 tde_buffer_cache指定哈希表的名称为 tde_buffer_name最大桶的数量为 max_bucket
// 并使用上述设置的 tde_buffer_ctl 进行初始化
tde_buffer_cache = hash_create(tde_buffer_name, max_bucket, &tde_buffer_ctl,
HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT);
//如果创建哈希表失败(返回值为 NULL
if (tde_buffer_cache == NULL) {
//抛出一个错误报告
ereport(ERROR, (errmodule(MOD_SEC_TDE), errcode(ERRCODE_FUNCTION_HASH_NOT_INITED),
errmsg("init TDE buffer cache is NULL"), errdetail("N/A"),
errcause("initialize cache is failed"), erraction("check TDE_BUFFER_CACHE_CONTEXT or system cache")));
}
}
//向TDE缓冲区缓存中插入或更新缓存项
bool TDEBufferCache::insert_cache(RelFileNode tde_rnode, TdeInfo *tde_info)
{
bool found = false;
errno_t rc = EOK;
//切换为TDE缓冲区缓存的上下文tde_buffer_mem
MemoryContext old = MemoryContextSwitchTo(tde_buffer_mem);
/* build tde cache key */
//根据传入的关系文件节点RelFileNode tde_rnode构建TDE缓存的键RelFileNode tde_key
RelFileNode tde_key = tde_rnode;
/* insert hash table */
//获取TDE缓冲区缓存的写锁
LWLockAcquire(TDEKeyCacheLock, LW_EXCLUSIVE);
TdeFileNodeEntry *entry = NULL;
//调用hash_search()函数在TDE缓冲区哈希表中查找关键字为tde_key的缓存项
entry = reinterpret_cast<TdeFileNodeEntry*>(hash_search(tde_buffer_cache, (const void*)&tde_key,
HASH_FIND, &found));
if (!found) {
if (!found) { //如果未找到缓存项,则表示需要插入新的缓存项
//调用hash_search()函数并将操作设置为HASH_ENTER将新的关键字-值对插入到TDE缓冲区哈希表中
entry = reinterpret_cast<TdeFileNodeEntry*>(hash_search(tde_buffer_cache,
(const void*)&tde_key, HASH_ENTER, &found));
if (entry == NULL) {
if (entry == NULL) { //如果插入失败返回值为NULL
//释放写锁恢复之前的内存上下文并返回false表示插入失败
LWLockRelease(TDEKeyCacheLock);
MemoryContextSwitchTo(old);
return false;
}
/* insert new cache key-value */
//如果插入成功,就需要给新的缓存项分配内存
entry->tde_info = (TdeInfo*)palloc0(sizeof(TdeInfo));
//并将传入的tde_info中的字段值复制到缓存项对应的字段中
rc = strcpy_s(entry->tde_info->dek_cipher, DEK_CIPHER_LEN, tde_info->dek_cipher);
securec_check(rc, "\0", "\0");
rc = strcpy_s(entry->tde_info->cmk_id, CMK_ID_LEN, tde_info->cmk_id);
securec_check(rc, "\0", "\0");
entry->tde_info->algo = tde_info->algo;
} else {
} else { //在找到了缓存项的情况下,更新该缓存项的逻辑
/* update cache key-value */
//通过比较缓存项中的dek_cipher字段和传入的tde_info->dek_cipher字段的值判断是否需要更新
if (strncmp(entry->tde_info->dek_cipher, tde_info->dek_cipher, DEK_CIPHER_LEN) != 0) {
//如果不相等将传入的tde_info->dek_cipher复制到缓存项的dek_cipher字段中
rc = strcpy_s(entry->tde_info->dek_cipher, DEK_CIPHER_LEN, tde_info->dek_cipher);
securec_check(rc, "\0", "\0");
}
//通过对比缓存项中的cmk_id字段和传入的tde_info->cmk_id字段的值判断是否需要更新
if (strncmp(entry->tde_info->cmk_id, tde_info->cmk_id, CMK_ID_LEN) != 0) {
//果不相等则使用strcpy_s()函数将传入的tde_info->cmk_id复制到缓存项的cmk_id字段中
rc = strcpy_s(entry->tde_info->cmk_id, CMK_ID_LEN, tde_info->cmk_id);
securec_check(rc, "\0", "\0");
}
//对比缓存项中的algo字段和传入的tde_info->algo字段的值判断是否需要更新
if (entry->tde_info->algo != tde_info->algo) {
//如果不相等则将传入的tde_info->algo赋值给缓存项的algo字段
entry->tde_info->algo = tde_info->algo;
}
}
//释放写锁,并将内存上下文切换回之前的上下文
MemoryContextSwitchTo(old);
LWLockRelease(TDEKeyCacheLock);
//返回true表示插入或更新操作成功
return true;
}
//在TDE缓冲区的哈希表中搜索与传入的tde_rnode相匹配的缓存项
//如果找到了缓存项则将缓存项中的tde_info复制到传入的tde_info中
// 如果未找到缓存项,则直接返回
void TDEBufferCache::search_cache(RelFileNode tde_rnode, TdeInfo *tde_info)
{
bool found = false;
//传入的tde_rnode的bucketNode字段不是无效的桶ID(InvalidBktId)则将其设置为SegmentBktId
if (tde_rnode.bucketNode != InvalidBktId) {
tde_rnode.bucketNode = SegmentBktId;
}
//切换到tde_buffer_mem内存上下文
MemoryContext old = MemoryContextSwitchTo(tde_buffer_mem);
//获取一个共享锁,即读取锁
LWLockAcquire(TDEKeyCacheLock, LW_SHARED);
TdeFileNodeEntry* entry = NULL;
//在TDE缓冲区哈希表中查找与传入的tde_rnode对应的缓存项
entry = reinterpret_cast<TdeFileNodeEntry*>(hash_search(tde_buffer_cache, (const void*)&tde_rnode,
HASH_FIND, &found));
if (!found) {
if (!found) { //没有找到缓存项
//恢复之前的内存上下文,释放锁,并直接返回
MemoryContextSwitchTo(old);
LWLockRelease(TDEKeyCacheLock);
return;
}
//如果找到了缓存项
// 使用memcpy_s() 函数将缓存项中的tde_info复制到传入的tde_info中
errno_t rc = memcpy_s(tde_info, sizeof(TdeInfo), entry->tde_info, sizeof(TdeInfo));
securec_check(rc, "\0", "\0");
//恢复之前的内存上下文,释放锁,并返回
MemoryContextSwitchTo(old);
LWLockRelease(TDEKeyCacheLock);
return;

File diff suppressed because it is too large Load Diff

View File

@ -37,42 +37,46 @@
#include "utils/snapmgr.h"
HTAB* CStoreColspaceCache = NULL;
//用于描述CStore列文件的相关信息
typedef struct {
CStoreColumnFileTag tag;
uint64 maxOffset;
uint32 maxCuid;
uint64 extendOffset;
CStoreColumnFileTag tag;//用于标识CStore列文件的标签
uint64 maxOffset;//表示CStore列文件的最大偏移量
uint32 maxCuid;//表示CStore列文件的最大CUIDColumn Unique Identifier
uint64 extendOffset;//表示CStore列文件的扩展偏移量
} CStoreColFileDesc;
//计算CStore分配器共享内存空间的大小
Size CStoreAllocatorShmSize()
{
Size size = 0;
Size size = 0;// 初始化共享内存大小为0
// 估算以256为预期哈希表项数和CStoreColFileDesc结构体大小为基础的哈希表占用内存空间的估算值
// 并将估算值累加到size变量中
size = add_size(size, hash_estimate_size(256, sizeof(CStoreColFileDesc)));
return size;
return size;// 返回CStore分配器共享内存空间的大小
}
//初始化CStore列空间缓存CStoreColspaceCache的哈希表
void CStoreAllocator::InitColSpaceCache(void)
{
HASHCTL ctl;
HASHCTL ctl;// 哈希表控制结构体
if (CStoreColspaceCache == NULL) {// 如果CStoreColspaceCache为空
errno_t rc = memset_s(&ctl, sizeof(ctl), 0, sizeof(ctl));// 将ctl结构体初始化为0
if (CStoreColspaceCache == NULL) {
errno_t rc = memset_s(&ctl, sizeof(ctl), 0, sizeof(ctl));
securec_check(rc, "", "");
ctl.keysize = sizeof(CStoreColumnFileTag);
ctl.entrysize = sizeof(CStoreColFileDesc);
ctl.hash = tag_hash;
ctl.keysize = sizeof(CStoreColumnFileTag);// 设置哈希表键的大小为CStoreColumnFileTag结构体的大小
ctl.entrysize = sizeof(CStoreColFileDesc);// 设置哈希表项的大小为CStoreColFileDesc结构体的大小
ctl.hash = tag_hash;// 设置哈希函数为tag_hash
// 创建CStore Column Space Cache哈希表并初始化为指定的大小和配置项
CStoreColspaceCache =
HeapMemInitHash("CStore Column Space Cache", 40960, 81920, &ctl, HASH_ELEM | HASH_FUNCTION);
if (CStoreColspaceCache == NULL)
if (CStoreColspaceCache == NULL)// 如果创建哈希表失败
ereport(PANIC, (errmsg("could not initialize CStore Column space desc hash table")));
}
}
//重置CStore列空间缓存CStoreColspaceCache的哈希表
void CStoreAllocator::ResetColSpaceCache(void)
{
if (CStoreColspaceCache != NULL) {
HeapMemResetHash(CStoreColspaceCache, "CStore Column Space Cache");
if (CStoreColspaceCache != NULL) {// 如果CStoreColspaceCache不为空
HeapMemResetHash(CStoreColspaceCache, "CStore Column Space Cache");// 重置CStore列空间缓存的哈希表
}
}
@ -81,29 +85,32 @@ CStoreAllocator::CStoreAllocator()
CStoreAllocator::~CStoreAllocator()
{}
//获取下一个可用的CUIDColumn Unit ID
uint32 CStoreAllocator::GetNextCUID(Relation rel)
{
bool found = false;
CStoreColFileDesc* entry = NULL;
uint32 cuid = InValidCUID;
CStoreColumnFileTag tag(rel->rd_node, VirtualSpaceCacheColID, MAIN_FORKNUM);
CStoreColumnFileTag tag(rel->rd_node, VirtualSpaceCacheColID, MAIN_FORKNUM);// 创建CStore列文件标记
(void)LWLockAcquire(CStoreColspaceCacheLock, LW_EXCLUSIVE);
(void)LWLockAcquire(CStoreColspaceCacheLock, LW_EXCLUSIVE);// 获取CStore列空间缓存锁
// 在CStore列空间缓存中查找指定的CStore列文件标记并返回对应的哈希表项
entry = (CStoreColFileDesc*)hash_search(CStoreColspaceCache, (void*)&tag, HASH_FIND, &found);
// 断言找到了指定的哈希表项
Assert(found);
// 获取CStore列文件的最大CUID
cuid = entry->maxCuid;
if (cuid == MaxCUID)
if (cuid == MaxCUID)// 如果没有剩余的CUID可用
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_RESOURCES),
errmsg("No CUID is left for new CU in relation \"%u\". Please execute the VACUUM FULL before do "
"anything else",
rel->rd_id)));
if (cuid > CUIDWarningThreshold && cuid < MaxCUID)
if (cuid > CUIDWarningThreshold && cuid < MaxCUID)// 如果CUID接近极限值
ereport(WARNING, (errmsg("CUID is almost to be used up in relation \"%u\"", rel->rd_id)));
entry->maxCuid++;
LWLockRelease(CStoreColspaceCacheLock);
return cuid;
entry->maxCuid++;// 将CStore列文件的最大CUID增加1
LWLockRelease(CStoreColspaceCacheLock);// 释放CStore列空间缓存锁
return cuid;// 返回获取的CUID值
}
/*
@ -114,21 +121,31 @@ uint32 CStoreAllocator::GetNextCUID(Relation rel)
* @Return:0 -- no need extend, others extend size
* @See also:
*/
/*
* @Description:
* @Param[IN] cu_offset: CU偏移量
* @Param[IN] cu_size: CU大小
* @Param[IN] extend_offset:
* @Return: 0 --
* @See also:
*/
//根据已分配空间的情况计算扩展大小
uint32 CStoreAllocator::CalcExtendSize(uint64 cu_offset, uint32 cu_size, uint64 extend_offset)
{
uint32 extend_segment = (uint32)(u_sess->attr.attr_storage.fast_extend_file_size * 1024LL);
uint32 extend_size = 0;
uint32 need_file_size = 0;
uint32 left_extend_size = 0;
uint32 extend_segment = (uint32)(u_sess->attr.attr_storage.fast_extend_file_size * 1024LL);// 定义扩展段大小(以字节为单位)
uint32 extend_size = 0;// 初始化扩展大小为0
uint32 need_file_size = 0;// 初始化需求文件大小为0
uint32 left_extend_size = 0;// 初始化剩余扩展大小为0
Assert(cu_offset <= extend_offset);
left_extend_size = extend_offset - cu_offset;
Assert(cu_offset <= extend_offset);// 断言最大CU偏移量小于等于记录的扩展偏移量
left_extend_size = extend_offset - cu_offset;// 计算剩余扩展大小
if (cu_size <= left_extend_size) {
return 0; // no need fast entend
return 0; // no need fast entend // 不需要快速扩展
}
need_file_size = cu_size - left_extend_size;
need_file_size = cu_size - left_extend_size;// 计算需求文件大小
// 若需求文件大小小于等于扩展段大小,则直接扩展到扩展段大小;否则,计算合适的扩展大小
if (need_file_size <= extend_segment) {
extend_size = extend_segment;
} else {
@ -136,7 +153,7 @@ uint32 CStoreAllocator::CalcExtendSize(uint64 cu_offset, uint32 cu_size, uint64
extend_size = need_file_size + extend_segment - remainder;
}
return extend_size;
return extend_size;// 返回计算得到的扩展大小
}
/*
@ -145,20 +162,32 @@ uint32 CStoreAllocator::CalcExtendSize(uint64 cu_offset, uint32 cu_size, uint64
* @Param[IN] size: cu size
* @See also:
*/
/*
* @Description:
* @Param[IN] cnode:
* @Param[IN] extend_offset: CU指针
* @Param[IN] cu_offset: CU偏移量
* @Param[IN] cu_sizeCU大小
* @Return:
* @See also:
*/
//分配文件空间
uint32 CStoreAllocator::AcquireFileSpace(const CFileNode& cnode, uint64 extend_offset, uint64 cu_offset, uint32 cu_size)
{
uint32 extend_size = 0;
CUStorage* cuStorage = New(CurrentMemoryContext) CUStorage(cnode);
// 根据配置参数external_enable决定使用libaio还是pread()/pwrite()来读写文件
ADIO_RUN()
{
if (u_sess->attr.attr_sql.enable_fast_allocate) {
// 如果开启了快速内存分配则调用CalcExtendSize函数计算扩展大小
extend_size = CStoreAllocator::CalcExtendSize(cu_offset, (uint32)cu_size, extend_offset);
if (extend_size != 0) {
cuStorage->FastExtendFile(extend_offset, extend_size, true);
cuStorage->FastExtendFile(extend_offset, extend_size, true);// 先进行快速扩展
}
cuStorage->FastExtendFile(cu_offset, cu_size, false);
cuStorage->FastExtendFile(cu_offset, cu_size, false);// 再写入数据
} else {
// 如果没有开启快速内存分配则使用palloc0函数分配内存并将分配的空间清零。然后将数据写入该空间最后释放内存。
char* buffer = (char*)adio_align_alloc(cu_size);
errno_t rc = memset_s(buffer, cu_size, 0, cu_size);
securec_check(rc, "\0", "\0");
@ -169,6 +198,7 @@ uint32 CStoreAllocator::AcquireFileSpace(const CFileNode& cnode, uint64 extend_o
}
ADIO_ELSE()
{
// 如果external_enable为false则使用pread()/pwrite()进行文件读写
char* buffer = (char*)palloc0(cu_size);
cuStorage->SaveCU(buffer, cu_offset, cu_size, false, true);
pfree(buffer);
@ -177,11 +207,11 @@ uint32 CStoreAllocator::AcquireFileSpace(const CFileNode& cnode, uint64 extend_o
}
ADIO_END();
DELETE_EX(cuStorage);
DELETE_EX(cuStorage);// 删除CUStorage对象释放内存
return extend_size;
return extend_size;// 返回实际分配的文件空间大小
}
//为一个列存储文件节点cnode分配大小为size的空间返回分配的空间在文件中的偏移量
uint64 CStoreAllocator::AcquireSpace(const CFileNode& cnode, Size size, int align_size)
{
Assert(align_size > 0);
@ -190,29 +220,31 @@ uint64 CStoreAllocator::AcquireSpace(const CFileNode& cnode, Size size, int alig
uint64 offset = InvalidCStoreOffset;
uint32 extend_size = 0;
LWLockAcquire(CStoreColspaceCacheLock, LW_EXCLUSIVE);
LWLockAcquire(CStoreColspaceCacheLock, LW_EXCLUSIVE);// 获取互斥锁
/* 在哈希表中查找文件节点 */
entry = (CStoreColFileDesc*)hash_search(CStoreColspaceCache, (const void*)&cnode, HASH_FIND, &found);
Assert(found);
if (found) {
offset = entry->maxOffset;
// 当升级时最后一个CU需要添加填充。因此cu_point必须向后对齐
// when upgrade, last cu need add padding. so cu_point must align backward
int remainder = offset % align_size;
if (remainder != 0) {
ereport(WARNING, (errmsg("AcquireSpace: find un align size(%lu)", offset)));
offset = offset + align_size - remainder;
entry->maxOffset = offset;
offset = offset + align_size - remainder;// 对齐offset
entry->maxOffset = offset;// 更新maxOffset
}
// 必须在更新entry之前完成快速扩展因为不能留下空洞
// must finish fast extend here before update, because we can not leave hole
extend_size = CStoreAllocator::AcquireFileSpace(cnode, entry->extendOffset, entry->maxOffset, size);
entry->maxOffset += size;
entry->extendOffset += extend_size;
entry->maxOffset += size;// 更新maxOffset
entry->extendOffset += extend_size;// 更新maxOffset
}
LWLockRelease(CStoreColspaceCacheLock);
return offset;
LWLockRelease(CStoreColspaceCacheLock);// 释放互斥锁
return offset;// 返回offset
}
//从自由空间映射fsm中尝试获取大小为size的空间并进行对齐返回分配的空间在文件中的偏移量
uint64 CStoreAllocator::TryAcquireSpaceFromFSM(CStoreFreeSpace* fsm, Size size, int align_size)
{
CStoreFreeSpaceDesc desc;
@ -220,38 +252,41 @@ uint64 CStoreAllocator::TryAcquireSpaceFromFSM(CStoreFreeSpace* fsm, Size size,
Assert(fsm != NULL);
Assert(align_size > 0);
// 检查自由空间映射fsm是否有足够的空间来满足需求
if (!fsm->HasEnoughSpace(size + align_size))
return offset;
return offset;// 如果没有足够的空间返回InvalidCStoreOffset表示分配失败
// 从自由空间映射fsm中弹出大小最大的空闲块描述符desc
fsm->PopDescWithMaxSize(desc);
offset = desc.beginOffset;
offset = desc.beginOffset;// 获取该空闲块的起始偏移量
// when upgrade, last cu need add padding. so cu_point must align backward
// 当进行升级时最后一个CU需要添加填充。因此cu_point必须向后对齐
int remainder = offset % align_size;
if (remainder != 0) {
ereport(WARNING, (errmsg("TryAcquireSpaceFromFSM: find un align size(%lu)", offset)));
offset = offset + align_size - remainder;
desc.beginOffset = offset;
desc.size -= remainder;
offset = offset + align_size - remainder;// 对齐offset
desc.beginOffset = offset;// 更新desc的起始偏移量
desc.size -= remainder;// 更新desc的大小
}
// 更新desc的起始偏移量和大小
desc.beginOffset += size;
desc.size -= size;
// 如果仍然有剩余空间将剩余空间的描述符压入自由空间映射fsm
if (desc.size > 0)
fsm->Push(desc);
return offset;
return offset;// 返回分配的空间在文件中的偏移量
}
//为了获取空间而锁定关系rel
void CStoreAllocator::LockRelForAcquireSpace(Relation rel)
{
LockRelationForExtension(rel, ExclusiveLock);
LockRelationForExtension(rel, ExclusiveLock);//使用独占锁ExclusiveLock来锁定关系
}
//为了释放获取空间而锁定的关系rel
void CStoreAllocator::ReleaseRelForAcquireSpace(Relation rel)
{
UnlockRelationForExtension(rel, ExclusiveLock);
UnlockRelationForExtension(rel, ExclusiveLock);//使用独占锁ExclusiveLock来释放关系的锁
}
//无效化列空间缓存CStoreColspaceCache中与给定cnode相关的缓存条目
void CStoreAllocator::InvalidColSpaceCache(const CFileNode& cnode)
{
LWLockAcquire(CStoreColspaceCacheLock, LW_EXCLUSIVE);
@ -260,34 +295,44 @@ void CStoreAllocator::InvalidColSpaceCache(const CFileNode& cnode)
}
// build space cache for attrno[ attrNum ].
// 函数作用:为给定关系(`heapRel`)和属性编号数组(`attrIds`)中的属性创建列空间缓存。
// 缓存是针对每个属性单独创建的,并且使用锁定方式保证同步性。
// 如果缓存已经存在,则不会重复创建。
// 在构建缓存之前该函数会锁定关系以防止插入新的元组并找到最大的CU ID`maxCUID`和CU指针的偏移量`offset`)。
// 如果存在Gin/BTree索引还需要查找存储在BTree索引中的最大CU ID`maxIdxCUID`并将其与maxCUID比较选择最大的值。
// 使用这些信息,缓存将被创建,然后释放关系锁。
// 最后,释放分配的内存。
void CStoreAllocator::BuildColSpaceCacheForRel(_in_ Relation heapRel,
_in_ AttrNumber* attrIds, // equal to attrno[]
_in_ int attrNum, _in_ List* indexRel)
{
// 创建文件节点结构的数组
CFileNode* cFileNode = (CFileNode*)palloc(sizeof(CFileNode) * attrNum);
// 根据给定的属性编号构造cFileNode数组中每个属性的文件节点
for (int i = 0; i < attrNum; ++i) {
cFileNode[i].m_rnode = heapRel->rd_node;
cFileNode[i].m_forkNum = MAIN_FORKNUM;
cFileNode[i].m_attid = attrIds[i];
}
// 如果缓存不存在,就需要构建缓存
if (!CStoreAllocator::ColSpaceCacheExist(cFileNode, attrNum)) {
uint64* offset = (uint64*)palloc(sizeof(uint64) * attrNum);
// it's very important to make maxCUID and all the maxCUPointers the newest and biggest.
// so lock and forbit this relation inserting new tuples, see also SaveAll() method.
//
// 防止关系在创建缓存期间插入新的元组,使用独占锁
LockRelationForExtension(heapRel, ExclusiveLock);
// 获取最大CU ID
Oid cudesOid = heapRel->rd_rel->relcudescrelid;
uint32 maxCUID = CStore::GetMaxCUID(cudesOid, heapRel->rd_att) + 1;
// 如果存在Gin/BTree索引则获取BTree索引中的最大CU ID用于比较选择最大的CU ID
/* If there is gin/btree index, we need to find the biggest CU ID stored in the btree index. */
if (indexRel != NULL) {
uint32 maxIdxCUID = CStore::GetMaxIndexCUID(heapRel, indexRel) + 1;
if (maxIdxCUID > maxCUID)
maxCUID = maxIdxCUID;
}
// 查找每个属性的CU指针偏移量
for (int i = 0; i < attrNum; ++i) {
if (!heapRel->rd_att->attrs[i]->attisdropped) {
offset[i] = CStore::GetMaxCUPointer(attrIds[i], heapRel);
@ -295,13 +340,14 @@ void CStoreAllocator::BuildColSpaceCacheForRel(_in_ Relation heapRel,
offset[i] = 0;
}
}
// 创建缓存
CStoreAllocator::BuildColSpaceCacheForRel(cFileNode, attrNum, offset, maxCUID);
// 释放关系锁
UnlockRelationForExtension(heapRel, ExclusiveLock);
// 释放分配的内存
pfree_ext(offset);
}
// 释放分配的内存
pfree_ext(cFileNode);
}
@ -311,17 +357,27 @@ void CStoreAllocator::BuildColSpaceCacheForRel(_in_ Relation heapRel,
* @Return: extend offset
* @See also:
*/
/*
* @Description:
* @Param[IN] max_offset: CU指针
* @Return:
* @See also:
*/
//根据文件的最大CU指针计算快速扩展的偏移量。偏移量是按照预定义的段大小对齐的以提高存储空间的利用率
uint64 CStoreAllocator::GetExtendOffset(uint64 max_offset)
{
// 获取快速扩展的段大小(单位:字节)
int extend_segment = (int)(u_sess->attr.attr_storage.fast_extend_file_size * 1024LL);
// 计算当前偏移量所在的段的起始偏移量
uint64 offset = CU_FILE_OFFSET(max_offset);
uint64 remainder = offset % extend_segment;
if (remainder != 0) {
// 如果当前偏移量不是段的起始位置,则计算下一个段的起始偏移量
max_offset = max_offset + extend_segment - remainder;
}
return max_offset;
}
//构建列空间的全局缓存,将列对应的文件偏移量写入缓存,以提供分配空间时的参考
void CStoreAllocator::BuildColSpaceCacheForRel(const CFileNode* cnodes, int nColumn, uint64* offsets, uint32 maxCUID)
{
CFileNode tag(cnodes[0].m_rnode, VirtualSpaceCacheColID, MAIN_FORKNUM);
@ -334,6 +390,10 @@ void CStoreAllocator::BuildColSpaceCacheForRel(const CFileNode* cnodes, int nCol
// If not, update them.
// If yes, skip.
//
// 构建列空间的全局缓存
// 将列对应的文件偏移量写入缓存,以提供分配空间时的参考
// 缓存中记录的是每个列的最大文件偏移量、对应的最大CU指针和扩展偏移量
//
entry = (CStoreColFileDesc*)hash_search(CStoreColspaceCache, (void*)&tag, HASH_ENTER, &found);
if (entry == NULL)
ereport(PANIC, (errmsg("build global column space cache hash table failed")));
@ -341,6 +401,9 @@ void CStoreAllocator::BuildColSpaceCacheForRel(const CFileNode* cnodes, int nCol
// Other session has insert some columns or all columns into hash table
// !!!Note that we reuse variable 'found'
//
// 判断是否需要更新缓存
// 如果存在已经缓存的数据,则需要检查其是否已经完整
//
if (found) {
for (int i = 0; i < nColumn; i++) {
hash_search(CStoreColspaceCache, (void*)&cnodes[i], HASH_FIND, &found);
@ -348,15 +411,18 @@ void CStoreAllocator::BuildColSpaceCacheForRel(const CFileNode* cnodes, int nCol
// Other session has insert some columns
// It is incomplete
//
// 如果某个列的空间信息不存在,则代表其他进程还没有往缓存中插入
// 全部信息不完整,需要更新缓存
//
if (!found)
break;
}
}
// 如果缓存中没有列对应的空间信息,则需要更新缓存
if (!found) {
entry->maxCuid = maxCUID;
entry->maxOffset = InvalidCStoreOffset;
// 逐一将每个列的空间信息填入缓存
for (int i = 0; i < nColumn; i++) {
entry = (CStoreColFileDesc*)hash_search(CStoreColspaceCache, (void*)&cnodes[i], HASH_ENTER, NULL);
if (entry == NULL)
@ -369,19 +435,22 @@ void CStoreAllocator::BuildColSpaceCacheForRel(const CFileNode* cnodes, int nCol
LWLockRelease(CStoreColspaceCacheLock);
}
//用于检查给定的列空间信息是否存在于缓存中
bool CStoreAllocator::ColSpaceCacheExist(const CFileNode* cnodes, int nColumn)
{
bool found = false;
// 获取共享锁,以防止其他线程对缓存进行修改
LWLockAcquire(CStoreColspaceCacheLock, LW_SHARED);
// 遍历每个列的CFileNode查找其在缓存中是否存在
// 如果某个列的空间信息不存在则found为false退出循环
for (int i = 0; i < nColumn; i++) {
hash_search(CStoreColspaceCache, (void*)&cnodes[i], HASH_FIND, &found);
if (!found)
break;
}
// 释放共享锁
LWLockRelease(CStoreColspaceCacheLock);
// 返回是否所有列的空间信息都存在于缓存中的布尔值
return found;
}
@ -390,29 +459,31 @@ bool CStoreAllocator::ColSpaceCacheExist(const CFileNode* cnodes, int nColumn)
* here we recheck max cuid located in index
* we want to make sure if there is on another larger cuid in index
*/
//用于重新检查最大的cuid值
uint32 CStoreAllocator::recheck_max_cuid(Relation m_rel, uint32 max_cuid, int index_num, Relation* m_idxRelation)
{
bool find = false;
List* index_rel_list = NIL;
// 筛选出索引类型为B树或GIN的关联关系将其添加到索引关联关系列表中
for (int i = 0; i < index_num; ++i) {
Oid am_oid = m_idxRelation[i]->rd_rel->relam;
if (am_oid == CBTREE_AM_OID || am_oid == CGIN_AM_OID) {
index_rel_list = lappend(index_rel_list, m_idxRelation[i]);
}
}
// 若索引关联关系列表为空则直接返回原始的最大cuid值
if (list_length(index_rel_list) == 0) {
return max_cuid;
}
// 获取索引关联关系列表中的最大索引cuid并在释放列表内存后返回
uint32 max_idx_cuid = CStore::GetMaxIndexCUID(m_rel, index_rel_list) + 1;
list_free_ext(index_rel_list);
// 如果最大索引cuid等于MaxCUID表示没有剩余的cuid可供新的CU使用报错
if (max_idx_cuid == MaxCUID) {
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_RESOURCES),
errmsg("No CUID is left for new CU in relation \"%u\".", m_rel->rd_id)));
}
// 如果最大索引cuid大于原始的最大cuid值更新缓存中对应的列文件描述项的最大cuid并返回最大索引cuid
if (max_idx_cuid > max_cuid) {
CStoreColFileDesc* entry = NULL;
CStoreColumnFileTag tag(m_rel->rd_node, VirtualSpaceCacheColID, MAIN_FORKNUM);
@ -421,104 +492,107 @@ uint32 CStoreAllocator::recheck_max_cuid(Relation m_rel, uint32 max_cuid, int in
Assert(find);
entry->maxCuid = max_idx_cuid + 1;
LWLockRelease(CStoreColspaceCacheLock);
return max_idx_cuid;
return max_idx_cuid;// 如果最大索引cuid不大于原始的最大cuid值则直接返回原始的最大cuid值
}
return max_cuid;
}
//初始化CStoreFreeSpace对象
void CStoreFreeSpace::Initialize(int maxSize)
{
m_maxSize = maxSize;
m_descNum = 0;
m_descs = (CStoreFreeSpaceDesc*)palloc0(sizeof(CStoreFreeSpaceDesc) * (m_maxSize + 1));
m_maxSize = maxSize;// 设置最大尺寸
m_descNum = 0;// 描述项数量初始化为0
m_descs = (CStoreFreeSpaceDesc*)palloc0(sizeof(CStoreFreeSpaceDesc) * (m_maxSize + 1));// 分配描述项数组内存
}
}
CStoreFreeSpace::~CStoreFreeSpace()
{
m_descs = NULL;
m_descs = NULL;// 将描述项数组指针置空
}
//销毁CStoreFreeSpace对象
void CStoreFreeSpace::Destroy()
{
pfree(m_descs);
m_descs = NULL;
pfree(m_descs);// 释放描述项数组内存
m_descs = NULL;// 将描述项数组指针置空
}
//向CStoreFreeSpace对象的描述项数组中压入一个新的描述项
void CStoreFreeSpace::Push(const CStoreFreeSpaceDesc& desc)
{
int i;
if (m_maxSize == m_descNum)
if (m_maxSize == m_descNum)// 描述项数量已达到最大尺寸,无法继续添加
return;
i = ++m_descNum;
while (i != 1 && desc.size > m_descs[i / 2].size) {
m_descs[i] = m_descs[i / 2];
i /= 2;
i = ++m_descNum;// 描述项数量加1并将当前位置索引赋值给变量i
while (i != 1 && desc.size > m_descs[i / 2].size) {// 描述项的尺寸比父节点的尺寸大,进行上移操作
m_descs[i] = m_descs[i / 2];// 将父节点的描述项下移到当前位置
i /= 2;// 更新索引为父节点的索引
}
m_descs[i] = desc;
m_descs[i] = desc;// 将待插入的描述项存放到最终位置
}
//用于弹出具有最大尺寸的描述项并将其赋值给传入的参数desc
void CStoreFreeSpace::PopDescWithMaxSize(CStoreFreeSpaceDesc& desc)
{
CStoreFreeSpaceDesc tmp;
int i = 1;
int subi = 2;
if (m_descNum == 0)
if (m_descNum == 0)// 描述项数量为0无法弹出
return;
desc = m_descs[1];
tmp = m_descs[m_descNum--];
desc = m_descs[1];// 将根节点的描述项赋值给传入的参数desc
tmp = m_descs[m_descNum--];// 将最后一个描述项赋值给临时变量tmp并将描述项数量减1
while (subi <= m_descNum) {
if (subi < m_descNum && m_descs[subi].size < m_descs[subi + 1].size)
while (subi <= m_descNum) {// 子节点索引未超出描述项数组范围
if (subi < m_descNum && m_descs[subi].size < m_descs[subi + 1].size)// 右子节点的尺寸更大,选择右子节点
subi++;
if (tmp.size >= m_descs[subi].size)
if (tmp.size >= m_descs[subi].size)// 临时描述项的尺寸大于等于子节点的尺寸,退出循环
break;
m_descs[i] = m_descs[subi];
i = subi;
subi *= 2;
m_descs[i] = m_descs[subi];// 将子节点的描述项上移到当前位置
i = subi;// 更新索引为子节点索引
subi *= 2;// 计算下一个子节点的索引
}
m_descs[i] = tmp;
m_descs[i] = tmp;// 将临时描述项存放到最终确定的位置
}
//用于获取具有最大尺寸的描述项并将其赋值给传入的参数desc
void CStoreFreeSpace::GetDescWithMaxSize(_out_ CStoreFreeSpaceDesc& desc)
{
if (m_descNum == 0)
if (m_descNum == 0)// 如果描述项数量为0则说明空间已满返回size最大值
desc.size = ~0;
else
else// 否则获取具有最大尺寸的描述项并赋值给传入的参数desc
desc = m_descs[1];
}
//用于判断空闲空间是否足够放下指定大小的数据块。如果当前为空闲空间
bool CStoreFreeSpace::HasEnoughSpace(Size size)
{
return IsEmpty() ? false : size <= m_descs[1].size;
return IsEmpty() ? false : size <= m_descs[1].size;// 判断空闲空间是否足够放下指定大小的数据块
}
// compute free space data for the *attrno* attribute, which
// belongs to the relation specified by *cudescHeapRel*.
// *cudescIndexRel* used to index-scan.
//
//计算指定关系中指定属性的空闲空间数据
void CStoreFreeSpace::ComputeFreeSpace(
_in_ AttrNumber attrno, _in_ Relation cudescHeapRel, _in_ Relation cudescIndexRel, __inout CStoreFreeSpace* fspace)
{
bool isnull = false;
List* beginOffsetOrderedList = NIL;
ListCell* currCell = NULL;
ListCell* prevCell = NULL;
ListCell* nextCell = NULL;
TupleDesc cudescTupDesc = RelationGetDescr(cudescHeapRel);
bool isnull = false;// 是否为NULL值
List* beginOffsetOrderedList = NIL;// 按beginoffset排序的列表
ListCell* currCell = NULL;// 当前列表项
ListCell* prevCell = NULL;// 前一个列表项
ListCell* nextCell = NULL;// 后一个列表项
TupleDesc cudescTupDesc = RelationGetDescr(cudescHeapRel);// CUDesc表的元组描述符
#ifdef USE_ASSERT_CHECKING
List* tupList = NIL;
List* tupList = NIL;// 用于断言检查的元组列表
#endif
// Setup scan key to fetch from the index by col_id.
// 设置扫描键按照col_id从索引中获取数据
ScanKeyData key;
ScanKeyInit(&key, (AttrNumber)CUDescColIDAttr, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(attrno));
// DIRTY snapshot will be used so that we can get the newest data.
// 使用DIRTY快照初始化扫描描述符
SnapshotData SnapshotDirty;
InitDirtySnapshot(SnapshotDirty);
SysScanDesc cudescScan = systable_beginscan_ordered(cudescHeapRel, cudescIndexRel, &SnapshotDirty, 1, &key);
@ -531,6 +605,7 @@ void CStoreFreeSpace::ComputeFreeSpace(
// Note: if the number of holds in the column > MaxNumOfHoleFSM, we should
// give up the scan, and go back to the 'APPEND_ONLY'.
//
//从CUDesc表中扫描列的CUDesc并按beginoffset排序放入列表中然后合并相邻的CUDesc
HeapTuple tup = NULL;
while ((tup = systable_getnext_ordered(cudescScan, BackwardScanDirection)) != NULL) {
CStoreSpaceDesc spaceDesc;
@ -538,6 +613,7 @@ void CStoreFreeSpace::ComputeFreeSpace(
char* cuPointer = DatumGetPointer(fastgetattr(tup, CUDescCUPointerAttr, cudescTupDesc, &isnull));
// skip cuPointer is null
// 跳过cuPointer为NULL的情况
if (isnull)
continue;
Assert(cuPointer);
@ -546,6 +622,7 @@ void CStoreFreeSpace::ComputeFreeSpace(
spaceDesc.size = DatumGetInt32(fastgetattr(tup, CUDescSizeAttr, cudescTupDesc, &isnull));
// skip those special CUs with total NULL or the SAME value.
// 跳过特殊情况下size为0的CUDesc
if (spaceDesc.size == 0)
continue;
@ -555,6 +632,7 @@ void CStoreFreeSpace::ComputeFreeSpace(
#endif
// try to merge the descs.
// 尝试合并CUDesc
for (currCell = list_head(beginOffsetOrderedList), prevCell = NULL; currCell != NULL; currCell = nextCell) {
CStoreSpaceDesc* curEntry = (CStoreSpaceDesc*)lfirst(currCell);
nextCell = lnext(currCell);
@ -575,10 +653,11 @@ void CStoreFreeSpace::ComputeFreeSpace(
}
// mark spaceDesc as handled.
// 标记spaceDesc已处理
spaceDesc.beginOffset = InvalidCStoreOffset;
break;
}
// 使用插入排序
// do Insertion sort
if (spaceDesc.beginOffset < curEntry->beginOffset) {
// if |-- spaceDesc --|-- curEntry --|
@ -597,10 +676,11 @@ void CStoreFreeSpace::ComputeFreeSpace(
}
// check the size of beginOffsetOrderedList to avoid too many space
// 检查beginOffsetOrderedList的大小避免空间过多
if (list_length(beginOffsetOrderedList) > MaxNumOfHoleFSM)
goto scan_end;
}
// 标记spaceDesc已处理
// mark spaceDesc as handled.
spaceDesc.beginOffset = InvalidCStoreOffset;
break;
@ -609,7 +689,7 @@ void CStoreFreeSpace::ComputeFreeSpace(
Assert(spaceDesc.beginOffset > curEntry->beginOffset + curEntry->size);
prevCell = currCell;
}
// 在此处将spaceDesc追加到列表末尾
// by here, we should append the spaceDesc at the end of list.
if (spaceDesc.beginOffset != InvalidCStoreOffset) {
CStoreSpaceDesc* newEntry = (CStoreSpaceDesc*)palloc0(sizeof(CStoreSpaceDesc));
@ -624,6 +704,8 @@ void CStoreFreeSpace::ComputeFreeSpace(
// Step2 : Calculate the space hole of the column
// Check if there's a hole at the begin of Column.
// 步骤2计算列的空洞
// 检查列的开头是否存在空洞
currCell = list_head(beginOffsetOrderedList);
if (currCell != NULL) {
CStoreSpaceDesc* entry = (CStoreSpaceDesc*)lfirst(currCell);
@ -634,7 +716,7 @@ void CStoreFreeSpace::ComputeFreeSpace(
fspace->Push(desc);
}
}
// 计算列的空洞
// Calculate the space hole of the column
for (currCell = list_head(beginOffsetOrderedList); currCell != NULL; currCell = nextCell) {
CStoreSpaceDesc *curEntry = NULL, *nextEntry = NULL;
@ -660,13 +742,13 @@ void CStoreFreeSpace::ComputeFreeSpace(
scan_end:
#ifdef USE_ASSERT_CHECKING
list_free_deep(tupList);
list_free_deep(tupList);// 释放断言检查用的元组列表
tupList = NIL;
#endif
list_free_deep(beginOffsetOrderedList);
list_free_deep(beginOffsetOrderedList);// 释放排序后的列表
beginOffsetOrderedList = NIL;
systable_endscan_ordered(cudescScan);
systable_endscan_ordered(cudescScan);// 结束扫描
cudescScan = NULL;
}

View File

@ -49,52 +49,55 @@
#define SORT_COL_TID 2
#define ARRAY_2_LEN 2
//初始化CStoreDelete对象的成员变量
CStoreDelete::CStoreDelete(
_in_ Relation rel, _in_ EState* estate, _in_ bool is_update_cu, _in_ Plan* plan, _in_ MemInfoArg* ArgmemInfo)
: m_deltaRealtion(NULL),
m_partDeltaOids(NIL),
m_estate(estate),
m_sortTupDesc(NULL),
m_deleteSortState(NULL),
m_partidIdx(0),
m_ctidIdx(0),
m_isTupDescCreateBySelf(false),
m_sortBatch(NULL),
m_rowOffset(NULL),
m_curSortedNum(0),
m_totalDeleteNum(0),
m_isRptRepeatTupErrForUpdate(false)
: m_deltaRealtion(NULL),// 初始化m_deltaRealtion为NULL
m_partDeltaOids(NIL),// 初始化m_partDeltaOids为空列表
m_estate(estate),// 设置m_estate为传入的estate参数
m_sortTupDesc(NULL),// 初始化m_sortTupDesc为NULL
m_deleteSortState(NULL),// 初始化m_deleteSortState为NULL
m_partidIdx(0),// 初始化m_partidIdx为0
m_ctidIdx(0),// 初始化m_ctidIdx为0
m_isTupDescCreateBySelf(false),//初始化m_isTupDescCreateBySelf为false
m_sortBatch(NULL),// 初始化m_sortBatch为NULL
m_rowOffset(NULL),// 初始化m_rowOffset为NULL
m_curSortedNum(0),// 初始化m_curSortedNum为0
m_totalDeleteNum(0),// 初始化m_totalDeleteNum为0
m_isRptRepeatTupErrForUpdate(false)// 初始化m_isRptRepeatTupErrForUpdate为false
{
m_relation = rel;
m_DelMemInfo = NULL;
m_relation = rel;// 设置m_relation为传入的rel参数
m_DelMemInfo = NULL;// 初始化m_DelMemInfo为NULL
/* get partial sort num */
// 获取部分排序行数
m_maxSortNum = RelationGetPartialClusterRows(rel);
// 临时缓冲区
/* temp buffer */
m_rowOffset = (int*)palloc(sizeof(int) * RelMaxFullCuSize);
InitDeleteMemArg(plan, ArgmemInfo);
InitDeleteMemArg(plan, ArgmemInfo);// 初始化删除内存参数
/* function routine */
bool isPartition = RELATION_IS_PARTITIONED(rel);
bool isPartition = RELATION_IS_PARTITIONED(rel);// 判断表是否为分区表
if (!isPartition) {
m_InitDeleteSortStatePtr = &CStoreDelete::InitDeleteSortStateForTable;
m_ExecDeletePtr = &CStoreDelete::ExecDeleteForTable;
m_deltaRealtion = heap_open(m_relation->rd_rel->reldeltarelid, RowExclusiveLock);
m_InitDeleteSortStatePtr =
&CStoreDelete::
InitDeleteSortStateForTable; // 设置函数指针m_InitDeleteSortStatePtr为InitDeleteSortStateForTable
m_ExecDeletePtr = &CStoreDelete::ExecDeleteForTable;// 设置函数指针m_ExecDeletePtr为ExecDeleteForTable
m_deltaRealtion = heap_open(m_relation->rd_rel->reldeltarelid, RowExclusiveLock);// 打开主表的delta表
} else {
m_InitDeleteSortStatePtr = &CStoreDelete::InitDeleteSortStateForPartition;
m_ExecDeletePtr = &CStoreDelete::ExecDeleteForPartition;
CollectPartDeltaOids();
m_InitDeleteSortStatePtr = &CStoreDelete::InitDeleteSortStateForPartition;// 设置函数指针m_InitDeleteSortStatePtr为InitDeleteSortStateForPartition
m_ExecDeletePtr = &CStoreDelete::ExecDeleteForPartition; // 设置函数指针m_ExecDeletePtr为ExecDeleteForPartition
CollectPartDeltaOids();// 收集所有分区的delta表OID
}
if (RelationIsPAXFormat(rel)) {
m_PutDeleteBatchPtr = &CStoreDelete::PutDeleteBatchForTable;
m_PutDeleteBatchPtr = &CStoreDelete::PutDeleteBatchForTable;// 设置函数指针m_PutDeleteBatchPtr为PutDeleteBatchForTable
} else {
m_PutDeleteBatchPtr = &CStoreDelete::PutDeleteBatchForPartition;
m_PutDeleteBatchPtr = &CStoreDelete::PutDeleteBatchForPartition;// 设置函数指针m_PutDeleteBatchPtr为PutDeleteBatchForPartition
}
/* set update flag */
m_isUpdate = is_update_cu;
m_isUpdate = is_update_cu;// 设置更新标志位m_isUpdate为传入的is_update_cu参数
}
CStoreDelete::~CStoreDelete()
@ -109,58 +112,58 @@ CStoreDelete::~CStoreDelete()
m_deltaRealtion = NULL;
m_relation = NULL;
}
//销毁CStoreDelete对象释放相关的内存资源
void CStoreDelete::Destroy()
{
if (m_sortTupDesc && m_isTupDescCreateBySelf) {
FreeTupleDesc(m_sortTupDesc);
FreeTupleDesc(m_sortTupDesc);// 释放由自身创建的排序元组描述符
m_sortTupDesc = NULL;
}
if (m_sortBatch) {
pfree(m_sortBatch);
pfree(m_sortBatch);// 释放排序批次内存
m_sortBatch = NULL;
}
if (m_rowOffset) {
pfree(m_rowOffset);
pfree(m_rowOffset);// 释放行偏移数组内存
m_rowOffset = NULL;
}
if (m_DelMemInfo) {
pfree_ext(m_DelMemInfo);
pfree_ext(m_DelMemInfo);// 释放删除内存信息
}
if (m_deleteSortState) {
batchsort_end(m_deleteSortState);
batchsort_end(m_deleteSortState);// 结束批量排序
m_deleteSortState = NULL;
}
if (m_deltaRealtion) {
heap_close(m_deltaRealtion, NoLock);
heap_close(m_deltaRealtion, NoLock);// 关闭delta表
m_deltaRealtion = NULL;
}
if (list_length(m_partDeltaOids) > 0) {
list_free(m_partDeltaOids);
list_free(m_partDeltaOids);// 释放分区delta表OID列表
m_partDeltaOids = NIL;
}
}
//用于收集分区表的delta表OID信息将收集到的OID存储在m_partDeltaOids中
void CStoreDelete::CollectPartDeltaOids()
{
Relation pgpartition = NULL;
TableScanDesc scan = NULL;
HeapTuple tuple = NULL;
ScanKeyData keys[ARRAY_2_LEN];
Form_pg_partition partitionFrom = NULL;
Relation pgpartition = NULL;// 访问PgPartition元组的关系对象
TableScanDesc scan = NULL;// 访问PgPartition元组的关系对象
HeapTuple tuple = NULL;// 获取的元组
ScanKeyData keys[ARRAY_2_LEN];// 扫描键值数组
Form_pg_partition partitionFrom = NULL;// 声明指向PgPartition元组的指针
/* Process all partitions of this partitiond table */
ScanKeyInit(&keys[0],
Anum_pg_partition_parttype,
BTEqualStrategyNumber,
F_CHAREQ,
CharGetDatum(PART_OBJ_TYPE_TABLE_PARTITION));
CharGetDatum(PART_OBJ_TYPE_TABLE_PARTITION));// 初始化扫描键值按类型为表分区和父表ID等于m_relation的条件进行过滤
ScanKeyInit(&keys[1],
Anum_pg_partition_parentid,
@ -168,15 +171,15 @@ void CStoreDelete::CollectPartDeltaOids()
F_OIDEQ,
ObjectIdGetDatum(RelationGetRelid(m_relation)));
pgpartition = heap_open(PartitionRelationId, AccessShareLock);
scan = tableam_scan_begin(pgpartition, SnapshotNow, ARRAY_2_LEN, keys);
pgpartition = heap_open(PartitionRelationId, AccessShareLock);// 打开PgPartition表
scan = tableam_scan_begin(pgpartition, SnapshotNow, ARRAY_2_LEN, keys);// 开始扫描
while ((tuple = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL) {
partitionFrom = (Form_pg_partition)GETSTRUCT(tuple);
m_partDeltaOids = lappend_oid(m_partDeltaOids, partitionFrom->reldeltarelid);
while ((tuple = (HeapTuple) tableam_scan_getnexttuple(scan, ForwardScanDirection)) != NULL) {// 遍历扫描结果
partitionFrom = (Form_pg_partition)GETSTRUCT(tuple);// 获取PgPartition元组地址并转换为Form_pg_partition结构体
m_partDeltaOids = lappend_oid(m_partDeltaOids, partitionFrom->reldeltarelid);// 获取分区delta表OID并加入到m_partDeltaOids列表末尾
}
tableam_scan_end(scan);
heap_close(pgpartition, AccessShareLock);
tableam_scan_end(scan);// 结束扫描
heap_close(pgpartition, AccessShareLock);// 关闭PgPartition表
}
/*
@ -188,11 +191,20 @@ void CStoreDelete::CollectPartDeltaOids()
* @Return: void
* @See also: InitInsertMemArg
*/
/*
* @Description: CStoreDelete的删除内存信息plan是传递给存储层执行的优化器估计参数
* ArgmemInfo用于将参数从上层传递给删除操作符执行
* @IN plan: 使mem_info参数传递给执行
* @IN ArgmemInfo: ArgmemInfo用于传递mem_info参数来执行
* @Return: void
* @See also: InitInsertMemArg
*/
//初始化CStoreDelete的删除内存信息
void CStoreDelete::InitDeleteMemArg(Plan* plan, MemInfoArg* ArgmemInfo)
{
m_DelMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg));
m_DelMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg));// 分配并初始化删除内存信息结构体
if (plan != NULL && plan->operatorMemKB[0] > 0) {
m_DelMemInfo->canSpreadmaxMem = plan->operatorMaxMem;
m_DelMemInfo->canSpreadmaxMem = plan->operatorMaxMem;// 设置可以扩散的最大内存
m_DelMemInfo->MemInsert = 0;
m_DelMemInfo->MemSort =
plan->operatorMemKB[0] ? plan->operatorMemKB[0] : u_sess->attr.attr_storage.psort_work_mem;
@ -203,9 +215,9 @@ void CStoreDelete::InitDeleteMemArg(Plan* plan, MemInfoArg* ArgmemInfo)
m_DelMemInfo->MemSort,
m_DelMemInfo->canSpreadmaxMem);
} else if (ArgmemInfo != NULL) {
m_DelMemInfo->canSpreadmaxMem = ArgmemInfo->canSpreadmaxMem;
m_DelMemInfo->canSpreadmaxMem = ArgmemInfo->canSpreadmaxMem;// 设置可以扩散的最大内存
m_DelMemInfo->MemInsert = ArgmemInfo->MemInsert;
m_DelMemInfo->MemSort = ArgmemInfo->MemSort;
m_DelMemInfo->MemSort = ArgmemInfo->MemSort; // 设置排序内存
m_DelMemInfo->spreadNum = ArgmemInfo->spreadNum;
MEMCTL_LOG(DEBUG2,
"CStoreDelete(init ArgmemInfo):Insert workmem is : %dKB, sort workmem: %dKB,can spread maxMem is %dKB.",
@ -215,17 +227,19 @@ void CStoreDelete::InitDeleteMemArg(Plan* plan, MemInfoArg* ArgmemInfo)
} else {
m_DelMemInfo->canSpreadmaxMem = 0;
m_DelMemInfo->MemInsert = 0;
m_DelMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem;
m_DelMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem;// 设置排序内存
m_DelMemInfo->spreadNum = 0;
}
m_DelMemInfo->partitionNum = 1;
m_DelMemInfo->partitionNum = 1;// 分区数初始化为1
}
//初始化删除操作符的排序信息
void CStoreDelete::InitSortState()
{
// init tupledesc, use partid and tid for sort
// 初始化TupleDesc创建包含两个属性的TupleDesc分别为partid和tid。
m_sortTupDesc = CreateTemplateTupleDesc(ARRAY_2_LEN, false);
m_isTupDescCreateBySelf = true;
// 将partid和tid的列号设置为常量。
TupleDescInitEntry(m_sortTupDesc, SORT_COL_PARTID, "partid", OIDOID, -1, 0);
TupleDescInitEntry(m_sortTupDesc, SORT_COL_TID, "tid", TIDOID, -1, 0);
@ -233,32 +247,39 @@ void CStoreDelete::InitSortState()
m_ctidIdx = SORT_COL_TID;
// sort cache
// 分配内存初始化m_sortBatch用于进行排序操作时的缓存。
m_sortBatch = New(CurrentMemoryContext) VectorBatch(CurrentMemoryContext, m_sortTupDesc);
// init sort state
// 调用m_InitDeleteSortStatePtr成员变量指向的函数来初始化排序状态。
(this->*m_InitDeleteSortStatePtr)(m_sortTupDesc, SORT_COL_PARTID, SORT_COL_TID);
}
//根据传入的参数初始化删除操作符的排序状态并将m_PutDeleteBatchPtr指向PutDeleteBatchForUpdate函数
void CStoreDelete::InitSortState(TupleDesc sortTupDesc, int partidIdx, int ctidIdx)
{
// 将传入的参数赋值给成员变量
m_sortTupDesc = sortTupDesc;
m_partidIdx = partidIdx;
m_ctidIdx = ctidIdx;
// sort cache
// 分配内存初始化m_sortBatch用于进行排序操作时的缓存。
m_sortBatch = New(CurrentMemoryContext) VectorBatch(CurrentMemoryContext, m_sortTupDesc);
// init sort state
// 调用m_InitDeleteSortStatePtr成员变量指向的函数来初始化排序状态。
(this->*m_InitDeleteSortStatePtr)(sortTupDesc, partidIdx, ctidIdx);
// change m_PutDeleteBatchPtr
// 更改m_PutDeleteBatchPtr指向的函数为PutDeleteBatchForUpdate。
m_PutDeleteBatchPtr = &CStoreDelete::PutDeleteBatchForUpdate;
}
//用于调用删除操作并判断是否需要进行部分删除
void CStoreDelete::PutDeleteBatch(_in_ VectorBatch* batch, _in_ JunkFilter* junkfilter)
{
// 调用m_PutDeleteBatchPtr成员变量指向的函数来执行删除操作。
(this->*m_PutDeleteBatchPtr)(batch, junkfilter);
// 如果删除操作符已经满了,则进行部分删除。
if (IsFull()) {
// execute delete if parital sort is full
PartialDelete();
@ -266,61 +287,68 @@ void CStoreDelete::PutDeleteBatch(_in_ VectorBatch* batch, _in_ JunkFilter* junk
return;
}
//执行部分删除和重置排序状态
void CStoreDelete::PartialDelete()
{
// 执行部分删除操作并更新m_totalDeleteNum统计变量。
m_totalDeleteNum += (uint64)(this->*m_ExecDeletePtr)();
// 递增命令计数器。
// parital delete need commad id ++
CommandCounterIncrement();
// 重置排序状态,为下一轮排序做准备。
// reset sort state
ResetSortState();
}
//执行删除操作并返回删除记录总数
uint64 CStoreDelete::ExecDelete()
{
// 执行删除操作并更新m_totalDeleteNum统计变量。
m_totalDeleteNum += (uint64)(this->*m_ExecDeletePtr)();
// 清空虚假关系缓存
// clean partition fake rel cache
if (m_estate->esfRelations) {
FakeRelationCacheDestroy(m_estate->esfRelations);
}
// 返回删除记录的总数。
return m_totalDeleteNum;
}
//初始化用于表格删除操作的排序状态
void CStoreDelete::InitDeleteSortStateForTable(TupleDesc sortTupDesc, int /* partidAttNo */, int ctidAttNo)
{
// 断言sortTupDesc不为空。
Assert(sortTupDesc);
// 断言ctidAttNo小于等于sortTupDesc的属性数量。
Assert(ctidAttNo <= sortTupDesc->natts);
// 初始化排序需要的参数
const int nkeys = 1;
AttrNumber attNums[1];
Oid sortCollations[1];
bool nullsFirstFlags[1];
int SortMem = u_sess->attr.attr_storage.psort_work_mem;
int maxMem = 0;
// 查找TID类型的缓存条目
TypeCacheEntry* typeEntry = lookup_type_cache(TIDOID, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR);
// 设置排序的属性号、排序规则和空值排序规则
attNums[0] = ctidAttNo;
sortCollations[0] = sortTupDesc->attrs[ctidAttNo - 1]->attcollation;
nullsFirstFlags[0] = false;
// 根据配置设置排序所需的内存大小和最大内存限制
SortMem = m_DelMemInfo->MemSort > 0 ? m_DelMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem;
maxMem = m_DelMemInfo->canSpreadmaxMem > 0 ? m_DelMemInfo->canSpreadmaxMem : 0;
// 使用batchsort_begin_heap函数开始一个基于堆的批量排序
m_deleteSortState = batchsort_begin_heap(
sortTupDesc, nkeys, attNums, &typeEntry->lt_opr, sortCollations, nullsFirstFlags, SortMem, false, maxMem);
}
//为分区删除操作初始化排序状态
void CStoreDelete::InitDeleteSortStateForPartition(TupleDesc sortTupDesc, int partidAttNo, int ctidAttNo)
{
// 断言sortTupDesc不为空。
Assert(sortTupDesc);
// 断言partidAttNo小于等于sortTupDesc的属性数量。
Assert(partidAttNo <= sortTupDesc->natts);
// 断言ctidAttNo小于等于sortTupDesc的属性数量。
Assert(ctidAttNo <= sortTupDesc->natts);
// 设置排序所需的参数
const int nkeys = ARRAY_2_LEN;
AttrNumber attNums[ARRAY_2_LEN];
Oid sortOperators[ARRAY_2_LEN];
@ -328,123 +356,141 @@ void CStoreDelete::InitDeleteSortStateForPartition(TupleDesc sortTupDesc, int pa
bool nullsFirstFlags[ARRAY_2_LEN];
int SortMem = u_sess->attr.attr_storage.psort_work_mem;
int maxMem = 0;
// 设置排序的属性号
attNums[0] = partidAttNo;
attNums[1] = ctidAttNo;
// 查找OID类型和TID类型的缓存条目
TypeCacheEntry* typeEntry = lookup_type_cache(OIDOID, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR);
sortOperators[0] = typeEntry->lt_opr;
typeEntry = lookup_type_cache(TIDOID, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR);
sortOperators[1] = typeEntry->lt_opr;
// 设置排序的排序规则
sortCollations[0] = sortTupDesc->attrs[partidAttNo - 1]->attcollation;
sortCollations[1] = sortTupDesc->attrs[ctidAttNo - 1]->attcollation;
// 设置排序的空值排序规则
nullsFirstFlags[0] = false;
nullsFirstFlags[1] = false;
// 根据配置设置排序所需的内存大小和最大内存限制
SortMem = m_DelMemInfo->MemSort > 0 ? m_DelMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem;
maxMem = m_DelMemInfo->canSpreadmaxMem > 0 ? m_DelMemInfo->canSpreadmaxMem : 0;
// 使用batchsort_begin_heap函数开始一个基于堆的批量排序
m_deleteSortState = batchsort_begin_heap(
m_sortTupDesc, nkeys, attNums, sortOperators, sortCollations, nullsFirstFlags, SortMem, false, maxMem);
}
//将待删除的批次数据放入排序状态中进行排序
void CStoreDelete::PutDeleteBatchForTable(_in_ VectorBatch* batch, _in_ JunkFilter* junkfilter)
{
// 断言参数不为空
Assert(batch && junkfilter && m_sortBatch && m_deleteSortState);
// 获取TID字段的属性索引
int tididx = junkfilter->jf_junkAttNo;
Assert(tididx - 1 < batch->m_cols);
// shallow copy
// 将待删除的TID字段浅拷贝到排序批次中
m_sortBatch->m_arr[m_ctidIdx - 1].copy(&batch->m_arr[tididx - 1]);
// 设置排序批次的行数
m_sortBatch->m_rows = batch->m_rows;
// put batch
// 将排序批次加入排序状态中
m_deleteSortState->sort_putbatch(m_deleteSortState, m_sortBatch, 0, m_sortBatch->m_rows);
// 更新已排序的记录数量
m_curSortedNum += batch->m_rows;
}
//将待删除的分区批次数据放入排序状态中进行排序
void CStoreDelete::PutDeleteBatchForPartition(_in_ VectorBatch* batch, _in_ JunkFilter* junkfilter)
{
// 断言参数不为空
Assert(batch && junkfilter && m_sortBatch && m_deleteSortState);
// 获取TID字段和分区字段的属性索引
int tididx = junkfilter->jf_junkAttNo;
int partidx = junkfilter->jf_xc_part_id;
Assert(tididx - 1 < batch->m_cols);
Assert(partidx - 1 < batch->m_cols);
// shallow copy
// 将待删除的分区字段和TID字段浅拷贝到排序批次中
m_sortBatch->m_arr[m_partidIdx - 1].copy(&batch->m_arr[partidx - 1]);
m_sortBatch->m_arr[m_ctidIdx - 1].copy(&batch->m_arr[tididx - 1]);
// 设置排序批次的行数
m_sortBatch->m_rows = batch->m_rows;
// put batch
// 将排序批次加入排序状态中
m_deleteSortState->sort_putbatch(m_deleteSortState, m_sortBatch, 0, m_sortBatch->m_rows);
// 更新已排序的记录数量
m_curSortedNum += batch->m_rows;
}
//将待更新的批次数据放入排序状态中进行排序
void CStoreDelete::PutDeleteBatchForUpdate(_in_ VectorBatch* batch, _in_ JunkFilter* /* junkfilter */)
{
// 断言参数不为空
Assert(batch && m_sortBatch && m_deleteSortState);
// shallow copy all batch data
// 浅拷贝整个批次数据到排序批次中
m_sortBatch->Copy<false, false>(batch);
// put batch
// 将排序批次加入排序状态中
m_deleteSortState->sort_putbatch(m_deleteSortState, m_sortBatch, 0, m_sortBatch->m_rows);
// 更新已排序的记录数量
m_curSortedNum += batch->m_rows;
}
//将待更新批次中指定行的数据放入排序状态中进行排序
void CStoreDelete::PutDeleteBatchForUpdate(_in_ VectorBatch* batch, _in_ int startIdx, _in_ int endIdx) {
// 断言参数不为空
Assert(batch && m_sortBatch && m_deleteSortState);
/* 拷贝指定行的批次数据 */
/* copy batch data */
m_sortBatch->Copy<false, false>(batch, startIdx, endIdx);
/* 将排序批次加入排序状态中 */
/* put batch */
m_deleteSortState->sort_putbatch(m_deleteSortState, m_sortBatch, 0, m_sortBatch->m_rows);
/* 更新已排序的记录数量 */
m_curSortedNum += m_sortBatch->m_rows;
}
//执行表的删除操作
uint64 CStoreDelete::ExecDeleteForTable()
{
// 断言参数不为空
Assert(m_relation && m_estate && m_sortBatch && m_deleteSortState && m_rowOffset);
uint32 lastCUID = InValidCUID;
uint32 lastOffset = 0;
int delRowNum = 0;
uint64 delTotalRowNum = 0;
Oid deltaTableid = RelationGetRelid(m_deltaRealtion);
uint32 lastCUID = InValidCUID;// 上一个数据块的CUID
uint32 lastOffset = 0;// 上一个偏移量
int delRowNum = 0;// 删除的行数
uint64 delTotalRowNum = 0;// 总共删除的行数
Oid deltaTableid = RelationGetRelid(m_deltaRealtion);// Delta表的OID
(void)GetCurrentTransactionId();
m_sortBatch->Reset(true);
(void)GetCurrentTransactionId();// 获取当前事务ID
m_sortBatch->Reset(true);// 重置排序批次
// 运行排序
// run sort
batchsort_performsort(m_deleteSortState);
// 获取排序后的批次
batchsort_getbatch(m_deleteSortState, true, m_sortBatch);
while (!BatchIsNull(m_sortBatch)) {
// get tid col
ScalarValue* tableidValues = m_sortBatch->m_arr[m_partidIdx - 1].m_vals;
ScalarValue* tidValues = m_sortBatch->m_arr[m_ctidIdx - 1].m_vals;
ItemPointer lastRowTid = NULL;
ScalarValue* tableidValues = m_sortBatch->m_arr[m_partidIdx - 1].m_vals;// 表ID列的值
ScalarValue* tidValues = m_sortBatch->m_arr[m_ctidIdx - 1].m_vals;// TID列的值
ItemPointer lastRowTid = NULL;// 上一行的TID
for (int i = 0; i < m_sortBatch->m_rows; i++) {
ItemPointer tid = (ItemPointer)(tidValues + i);
ItemPointer tid = (ItemPointer)(tidValues + i);// 当前行的TID
if (deltaTableid == tableidValues[i]) {
/*
* for delete, allow replicated tid;
* for update, don't allow replicated tid;
*/
// 如果是在Delta表上的删除操作
/*
* TID
* TID
*/
if (lastRowTid != NULL && ItemPointerEquals(lastRowTid, tid)) {
if (m_isRptRepeatTupErrForUpdate) {
ereport(ERROR, (errcode(ERRCODE_CARDINALITY_VIOLATION),
@ -454,29 +500,33 @@ uint64 CStoreDelete::ExecDeleteForTable()
continue;
}
simple_heap_delete(m_deltaRealtion, tid);
simple_heap_delete(m_deltaRealtion, tid);// 在Delta表上执行删除操作
lastRowTid = tid;
delTotalRowNum++;
continue;
}
uint32 curCUID = ItemPointerGetBlockNumber(tid);
uint32 curOffset = ItemPointerGetOffsetNumber(tid) - 1;
uint32 curCUID = ItemPointerGetBlockNumber(tid);// 当前数据块的CUID
uint32 curOffset = ItemPointerGetOffsetNumber(tid) - 1;// 当前偏移量
Assert(IsValidCUID(curCUID));
if (!IsValidCUID(lastCUID)) {
// 第一个数据块
// first CU
lastCUID = curCUID;
} else if (lastCUID != curCUID) {
// 修改同一数据块中的删除位图
// modify delete bitmap in same CU
UpdateVCBitmap(m_relation, lastCUID, m_rowOffset, delRowNum, m_estate->es_snapshot);
// switch CU
// 切换到下一个数据块
lastCUID = curCUID;
delTotalRowNum += delRowNum;
delRowNum = 0;
} else if (lastOffset == curOffset) {
// 重复的TID
// repeat ctid
Assert(lastCUID == curCUID);
@ -490,12 +540,14 @@ uint64 CStoreDelete::ExecDeleteForTable()
}
// record offset
// 记录偏移量
m_rowOffset[delRowNum++] = curOffset;
lastOffset = curOffset;
Assert(delRowNum <= DefaultFullCUSize);
}
// get next sorted batch
// 获取下一个排序后的批次
batchsort_getbatch(m_deleteSortState, true, m_sortBatch);
}
@ -506,9 +558,12 @@ uint64 CStoreDelete::ExecDeleteForTable()
return delTotalRowNum;
}
/**
*
*/
uint64 CStoreDelete::ExecDeleteForPartition()
{
// 断言关系、执行状态、排序批次、删除排序状态和行偏移量都存在
Assert(m_relation && m_estate && m_sortBatch && m_deleteSortState && m_rowOffset);
Relation partFakeRel = NULL;
@ -520,30 +575,32 @@ uint64 CStoreDelete::ExecDeleteForPartition()
int delRowNum = 0;
uint64 delTotalRowNum = 0;
// 获取当前事务的ID
(void)GetCurrentTransactionId();
// 重置排序批次
m_sortBatch->Reset(true);
// 执行排序
// run sort
batchsort_performsort(m_deleteSortState);
// 获取排序后的批次
batchsort_getbatch(m_deleteSortState, true, m_sortBatch);
/*
* ExecDeleteForPartition may be invoked by CStore update multi times.
* We should close the m_deltaRelation opened last time.
*/
// 如果之前有打开过的m_deltaRelation则需要关闭它
if (m_deltaRealtion != NULL) {
relation_close(m_deltaRealtion, NoLock);
m_deltaRealtion = NULL;
}
// 循环处理排序后的批次,直到批次为空
while (!BatchIsNull(m_sortBatch)) {
// 定义分区ID和行ID的值并初始化最后一行数据的行ID
ScalarValue* partidValues = m_sortBatch->m_arr[m_partidIdx - 1].m_vals;
ScalarValue* tidValues = m_sortBatch->m_arr[m_ctidIdx - 1].m_vals;
ItemPointer lastRowTid = NULL;
// 处理每一行数据
for (int i = 0; i < m_sortBatch->m_rows; i++) {
Oid curPartID = DatumGetObjectId(*(partidValues + i));
Assert(curPartID != InvalidOid);
@ -551,6 +608,7 @@ uint64 CStoreDelete::ExecDeleteForPartition()
ItemPointer tid = (ItemPointer)(tidValues + i);
/* for delta table */
// 如果是分区表中的删除操作则需要打开对应的delta分区并关闭之前打开过的delta分区
if (list_member_oid(m_partDeltaOids, curPartID)) {
if (lastPartID == InvalidOid) {
// first Partition
@ -576,6 +634,7 @@ uint64 CStoreDelete::ExecDeleteForPartition()
* for delete, allow replicated tid;
* for update, don't allow replicated tid;
*/
// 如果行ID与上一行重复则报错
if (lastRowTid != NULL && ItemPointerEquals(lastRowTid, tid)) {
if (m_isRptRepeatTupErrForUpdate) {
ereport(ERROR, (errcode(ERRCODE_CARDINALITY_VIOLATION),
@ -584,16 +643,16 @@ uint64 CStoreDelete::ExecDeleteForPartition()
}
continue;
}
// 执行删除操作并增加删除的行数
simple_heap_delete(m_deltaRealtion, tid);
delTotalRowNum++;
continue;
}
// 获取当前CU的ID和偏移量
uint32 curCUID = ItemPointerGetBlockNumber(tid);
uint32 curOffset = ItemPointerGetOffsetNumber(tid) - 1;
Assert(IsValidCUID(curCUID));
// 对于非分区表中的删除操作如果当前处理的分区和CU与上一行不同则需要修改删除位图
if (lastPartID == InvalidOid) {
// first Partition and CU
lastPartID = curPartID;
@ -655,46 +714,51 @@ uint64 CStoreDelete::ExecDeleteForPartition()
continue;
}
// 记录当前行的偏移量
// record offset
m_rowOffset[delRowNum++] = curOffset;
lastOffset = curOffset;
Assert(delRowNum <= DefaultFullCUSize);
}
// 获取下一个排序批次
// get next sorted batch
batchsort_getbatch(m_deleteSortState, true, m_sortBatch);
}
// 处理最后一个CU的删除位图
if (delRowNum > 0) {
UpdateVCBitmap(partFakeRel, lastCUID, m_rowOffset, delRowNum, m_estate->es_snapshot);
delTotalRowNum += delRowNum;
}
// 返回删除的总行数
return delTotalRowNum;
}
// ExecDelete
// Mark del_bitmap of CUDesc according to vecRowId
// ExecDelete函数用于执行删除操作
void CStoreDelete::ExecDelete(_in_ Relation rel, _in_ ScalarVector* vecRowId, _in_ Snapshot snapshot, _in_ Oid tableOid)
{
uint32 lastCUID = InValidCUID;
int delRowNum = 0;
ScalarValue* values = vecRowId->m_vals;
uint32 curCUID, curOffset;
Oid deltaOid = RelationGetRelid(m_deltaRealtion);
ItemPointer lastRowTid = NULL;
GetCurrentTransactionId();
uint32 lastCUID = InValidCUID;// 上一行的CU ID初始设置为无效值
int delRowNum = 0;// 当前CU中待删除的行数
ScalarValue* values = vecRowId->m_vals;// 删除行的rowid数组
uint32 curCUID, curOffset;// 当前行的CU ID和偏移量
Oid deltaOid = RelationGetRelid(m_deltaRealtion);// delta表的OID
ItemPointer lastRowTid = NULL;// 上一行的TID
GetCurrentTransactionId();// 获取当前事务的ID
// 遍历每一行删除数据
for (int i = 0; i < vecRowId->m_rows; ++i) {
ItemPointer tid = (ItemPointer)(values + i);
ItemPointer tid = (ItemPointer)(values + i);// 获取当前行的TID
/* 对于delta表 */
/* for delta table */
if (tableOid == deltaOid) {
/*
* for delete, allow replicated tid;
* for update, don't allow replicated tid;
*/
/*
* TID
* TID
*/
if (lastRowTid != NULL && ItemPointerEquals(lastRowTid, tid)) {
if (m_isRptRepeatTupErrForUpdate) {
ereport(ERROR, (errcode(ERRCODE_CARDINALITY_VIOLATION),
@ -703,28 +767,28 @@ void CStoreDelete::ExecDelete(_in_ Relation rel, _in_ ScalarVector* vecRowId, _i
}
continue;
}
simple_heap_delete(m_deltaRealtion, tid);
simple_heap_delete(m_deltaRealtion, tid);// 执行删除操作
continue;
}
curCUID = ItemPointerGetBlockNumber(tid);
curOffset = ItemPointerGetOffsetNumber(tid) - 1;
curCUID = ItemPointerGetBlockNumber(tid);// 获取当前行的CU ID
curOffset = ItemPointerGetOffsetNumber(tid) - 1;// 获取当前行的偏移量
Assert(IsValidCUID(curCUID));
Assert(IsValidCUID(curCUID));// 断言当前行的CU ID是有效的
if (!IsValidCUID(lastCUID))
if (!IsValidCUID(lastCUID))// 如果上一行的CU ID无效则设置为当前行的CU ID
lastCUID = curCUID;
else if (lastCUID != curCUID) {
else if (lastCUID != curCUID) {// 如果上一行的CU ID与当前行的CU ID不相同则需要更新删除位图
UpdateVCBitmap(rel, lastCUID, m_rowOffset, delRowNum, snapshot);
lastCUID = curCUID;
delRowNum = 0;
}
m_rowOffset[delRowNum++] = curOffset;
m_rowOffset[delRowNum++] = curOffset;// 记录待删除行的偏移量
}
if (delRowNum > 0) {
UpdateVCBitmap(rel, lastCUID, m_rowOffset, delRowNum, snapshot);
UpdateVCBitmap(rel, lastCUID, m_rowOffset, delRowNum, snapshot);// 更新最后一个CU的删除位图
}
}
@ -733,10 +797,12 @@ void CStoreDelete::ExecDelete(_in_ Relation rel, _in_ ScalarVector* vecRowId, _i
* This function change the bitmap of corresponding CUs.
* And update the bitmap of CUs of VC.
*/
//更新虚拟列VC对应的删除位图
void CStoreDelete::UpdateVCBitmap(
_in_ Relation rel, _in_ uint32 cuid, _in_ const int* rowoffset, _in_ int delRowNum, _in_ Snapshot snapshot)
{
Retry:
// 扫描CUDesc表的两个条件
ScanKeyData key[2];
HeapTuple tmpTup = NULL, oldTup = NULL, newTup = NULL;
bool isnull = false;
@ -747,12 +813,13 @@ Retry:
/*
* Open the CUDesc relation and its index
*/
// 打开CUDesc表和其索引
Relation cudesc_rel = heap_open(rel->rd_rel->relcudescrelid, RowExclusiveLock);
Relation idx_rel = index_open(cudesc_rel->rd_rel->relcudescidx, RowExclusiveLock);
TupleDesc cudesc_tupdesc = cudesc_rel->rd_att;
Assert(CUDescMaxAttrNum == cudesc_tupdesc->natts);
// 初始化扫描CUDesc表时的条件
ScanKeyInit(&key[0], (AttrNumber)CUDescColIDAttr, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(VitrualDelColID));
ScanKeyInit(&key[1], (AttrNumber)CUDescCUIDAttr, BTEqualStrategyNumber, F_OIDEQ, UInt32GetDatum(cuid));
@ -765,19 +832,20 @@ Retry:
// just need get one to deal with, modify while->if 20150508
// If get the old one, it might show 'delete or update row conflict'
// If get the new one, it will continue to deal with the new one
// Step 1: 获取并修改删除位图
ItemPointerData oldTupCtid;
if ((tmpTup = systable_getnext_ordered(cudesc_scan, ForwardScanDirection)) != NULL) {
Assert(newTup == NULL);
oldTup = tmpTup;
uint32 rowCount = DatumGetUInt32(fastgetattr(oldTup, CUDescRowCountAttr, cudesc_tupdesc, &isnull));
Assert(isnull == false);
// 获取删除位图
// Get delMask
int delMaskBytes = (rowCount + 7) / 8;
delMask = (unsigned char*)palloc0(delMaskBytes);
char* cuPtr = DatumGetPointer(fastgetattr(oldTup, CUDescCUPointerAttr, cudesc_tupdesc, &isnull));
// 当前CU已存在删除位图
// Delbitmap is not null
// This CU has deleted rows before
if (isnull == false) {
@ -785,7 +853,7 @@ Retry:
Assert((int)VARSIZE_ANY_EXHDR(detoastPtr) == delMaskBytes);
rc = memcpy_s(delMask, delMaskBytes, VARDATA_ANY(detoastPtr), VARSIZE_ANY_EXHDR(detoastPtr));
securec_check(rc, "", "");
// 如果*detoastPtr*指向新的空间,将其释放
// if *detoastPtr* is a new space, we will free it
// the first time when it's useless.
if (detoastPtr != cuPtr) {
@ -796,11 +864,11 @@ Retry:
oldDelMask = (unsigned char*)palloc(delMaskBytes);
rc = memcpy_s(oldDelMask, delMaskBytes, delMask, delMaskBytes);
securec_check(rc, "", "");
// 修改删除位图
// Modify delMask
for (int i = 0; i < delRowNum; ++i) {
uint32 row = (uint32)rowoffset[i];
// 如果该行已被其他事务删除
// This row have been deleted by other transaction
if (oldDelMask[row >> 3] & (1 << (row % 8))) {
ereport(
@ -811,7 +879,7 @@ Retry:
}
pfree(oldDelMask);
// 构造修改后的删除位图对应的新tuple
// Form new tuple using new delMask
newTup =
CStore::FormVCCUDescTup(cudesc_tupdesc, (char*)delMask, cuid, rowCount, GetCurrentTransactionIdIfAny());
@ -820,7 +888,7 @@ Retry:
systable_endscan_ordered(cudesc_scan);
index_close(idx_rel, RowExclusiveLock);
// Step 2: 更新删除位图
// Step 2: update del_bitmap
TM_Result result = TM_Invisible;
TM_FailureData tmfd;
@ -836,14 +904,15 @@ Retry:
true,
NULL,
&tmfd,
NULL, // we don't need update_indexes
NULL, // we don't meed modifiedIdxAttrs
NULL, // we don't need update_indexes // 不需要更新索引
NULL, // we don't meed modifiedIdxAttrs // 不需要更新的列
false);
switch (result) {
case TM_SelfUpdated:
case TM_SelfModified: {
// Now It is TM_SelfModified
// 更新失败,出现锁冲突
ereport(ERROR,
(errcode(ERRCODE_LOCK_NOT_AVAILABLE), (errmsg("delete or update failed because lock conflict"))));
break;
@ -880,6 +949,12 @@ Retry:
* so it's possible that no tuple is found druing this search with
* SnapshotNow and index scan. once it happens, error reporting.
*/
/*
* SnapshotNow
* 1. BTREE索引正向扫描期间tuple再匹配到旧的tuple
* 2. SnapshotNowxid状态可能从'processing''committed'
* 使SnapshotNow和索引扫描时tuple
*/
ereport(ERROR, (errcode(ERRCODE_CARDINALITY_VIOLATION), errmsg("delete or update row conflict")));
}
heap_close(cudesc_rel, NoLock);
@ -888,26 +963,34 @@ Retry:
* dead_tuple will increase when commit, see
* function AtEOXact_PgStat.
*/
// 如果不是更新操作,则增加删除行数统计
if (!m_isUpdate)
pgstat_count_cu_delete(rel, delRowNum);
}
//判断当前是否已满排序状态
bool CStoreDelete::IsFull() const
{
// m_maxSortNum <= 0 means full sort
// 判断当前是否已满排序状态
// 如果m_maxSortNum <= 0表示无排序限制因此不满
// 否则比较当前已排序的行数m_curSortedNum和最大排序数m_maxSortNum
// 若已排序的行数大于等于最大排序数,则表示已满
return m_maxSortNum <= 0 ? false : m_curSortedNum >= m_maxSortNum;
}
//重置排序状态
void CStoreDelete::ResetSortState()
{
// 重置排序状态
// // 释放之前的排序状态
// release last sort state
if (m_deleteSortState) {
batchsort_end(m_deleteSortState);
m_deleteSortState = NULL;
}
m_curSortedNum = 0;
m_curSortedNum = 0;// 当前已排序行数归零
// init new sort state
// 初始化新的排序状态根据函数指针m_InitDeleteSortStatePtr调用相应的初始化函数
return (this->*m_InitDeleteSortStatePtr)(m_sortTupDesc, m_partidIdx, m_ctidIdx);
}

View File

@ -42,6 +42,7 @@ static Oid GetCUIdxFromDeltaIdx(Relation deltaIdx, bool isPartition);
* @IN parentRel: if rel is a partition of a cstore, parentRel is
* its parent relation. Otherwise parentRel is NULL.
*/
//将delta表中的数据移动到对应的列存储表中
void MoveDeltaDataToCU(Relation rel, Relation parentRel)
{
if (RELATION_IS_PARTITIONED(rel)) {
@ -65,13 +66,14 @@ void MoveDeltaDataToCU(Relation rel, Relation parentRel)
list_free_ext(partitionList);
return;
}
// 打开delta表
Relation deltaRel = heap_open(RelationGetDeltaRelId(rel), RowExclusiveLock);
TableScanDesc deltaScanDesc = tableam_scan_begin(deltaRel, GetActiveSnapshot(), 0, NULL);
InsertArg args;
HeapTuple deltaTup = NULL;
ResultRelInfo *resultRelInfo = NULL;
if (rel->rd_rel->relhasindex) {
// 如果列存储表有索引,则初始化结果关系信息结构体,并构建索引
resultRelInfo = makeNode(ResultRelInfo);
if (parentRel != NULL) {
InitResultRelInfo(resultRelInfo, parentRel, 1, 0);
@ -80,23 +82,26 @@ void MoveDeltaDataToCU(Relation rel, Relation parentRel)
}
ExecOpenIndices(resultRelInfo, false);
}
// 初始化插入参数和列存储插入类
CStoreInsert::InitInsertArg(rel, resultRelInfo, true, args);
CStoreInsert cstoreInsert(rel, args, false, NULL, NULL);
TupleDesc tupDesc = rel->rd_att;
Datum* val = (Datum*)palloc(sizeof(Datum) * tupDesc->natts);
bool* null = (bool*)palloc(sizeof(bool) * tupDesc->natts);
bulkload_rows batchRow(tupDesc, RelationGetMaxBatchRows(rel), true);
// 遍历delta表中的每一条记录
while ((deltaTup = (HeapTuple) tableam_scan_getnexttuple(deltaScanDesc, ForwardScanDirection)) != NULL) {
tableam_tops_deform_tuple(deltaTup, tupDesc, val, null);
/* ignore returned value because only one tuple is appended into */
// 将记录添加到批量加载变量中
(void)batchRow.append_one_tuple(val, null, tupDesc);
// 从delta表中删除当前记录
/* delete the current tuple from delta table */
simple_heap_delete(deltaRel, &deltaTup->t_self);
if (batchRow.full_rownum()) {
// 当批量加载变量达到最大存储记录数时,插入到列存储表中
/* insert into main table */
cstoreInsert.BatchInsert(&batchRow, 0);
batchRow.reset(true);
@ -104,12 +109,15 @@ void MoveDeltaDataToCU(Relation rel, Relation parentRel)
}
if (batchRow.m_rows_curnum > 0) {
// 将剩余的记录插入到列存储表中
cstoreInsert.BatchInsert(&batchRow, 0);
}
cstoreInsert.SetEndFlag();
// 关闭扫描描述符和delta表
tableam_scan_end(deltaScanDesc);
/* clean cstore insert */
// 释放内存和资源
pfree(val);
pfree(null);
CStoreInsert::DeInitInsertArg(args);
@ -130,38 +138,46 @@ void MoveDeltaDataToCU(Relation rel, Relation parentRel)
* @IN indexRelationId: the index on CU.
* @IN parentRel: the parent relation of the CU if relationId is a partition of cstore.
*/
//在Delta表上定义唯一索引
void DefineDeltaUniqueIndex(Oid relationId, IndexStmt* stmt, Oid indexRelationId, Relation parentRel)
{
// 检查是否声明了唯一索引,如果未声明,则直接返回
if (!stmt->unique) {
return;
}
Relation rel = NULL;
Relation deltaRelation = NULL;
Partition partition = NULL;
int numberOfKeyAttributes = 0;
List* indexColNames = NIL;
char deltaIndexRelName[NAMEDATALEN] = {'\0'};
// 初始化变量
Relation rel = NULL;// Delta表的关系对象
Relation deltaRelation = NULL;// Delta表的关系对象
Partition partition = NULL;// 分区对象仅在relationId是cstore分区时使用
int numberOfKeyAttributes = 0;// 索引键的数量
List* indexColNames = NIL;// 索引键的名称列表
char deltaIndexRelName[NAMEDATALEN] = {'\0'};// Delta索引的名称
error_t ret = 0;
// 对于分区表获取Delta表的关联关系
if (parentRel) {
// 对于分区表relationId是一个分区ID。使用AccessShareLock锁定分区。
/* For partiontioned table. relationId is a partition id. */
/* We will build index on partioned delta table, so we just use AccessShareLock on partition. */
partition = partitionOpen(parentRel, relationId, AccessShareLock);
rel = partitionGetRelation(parentRel, partition);
partitionClose(parentRel, partition, NoLock);
} else {
// 对于非分区表,直接打开关系对象
/* For non-partioned table. */
rel = relation_open(relationId, AccessShareLock);
}
// 如果Delta表是分区表对每个分区的Delta表定义唯一索引
if (RELATION_IS_PARTITIONED(rel)) {
// 对于分区表
/* For partitioned table. */
List* partitionList = relationGetPartitionOidList(rel);
ListCell* cell = NULL;
Oid partitionOid = InvalidOid;
Oid partIdxOid = InvalidOid;
/*
*
*
*/
/*
* Global partition index does not support column store,
* so each partition has an index. We build index on each
@ -177,12 +193,12 @@ void DefineDeltaUniqueIndex(Oid relationId, IndexStmt* stmt, Oid indexRelationId
relation_close(rel, NoLock);
return;
}
// 打开Delta表
deltaRelation = heap_open(RelationGetDeltaRelId(rel), ShareLock);
// 获取索引键的数量和名称列表
numberOfKeyAttributes = list_length(stmt->indexParams);
indexColNames = ChooseIndexColumnNames(stmt->indexParams);
// 构造Delta索引的名称
if (!parentRel) {
ret = snprintf_s(deltaIndexRelName, sizeof(deltaIndexRelName),
sizeof(deltaIndexRelName) - 1, "pg_delta_index_%u", indexRelationId);
@ -191,7 +207,7 @@ void DefineDeltaUniqueIndex(Oid relationId, IndexStmt* stmt, Oid indexRelationId
sizeof(deltaIndexRelName) - 1, "pg_delta_part_index_%u", indexRelationId);
}
securec_check_ss_c(ret, "\0", "\0");
// 创建索引的信息对象
IndexInfo* indexInfo = NULL;
indexInfo = makeNode(IndexInfo);
indexInfo->ii_NumIndexAttrs = numberOfKeyAttributes;
@ -208,12 +224,12 @@ void DefineDeltaUniqueIndex(Oid relationId, IndexStmt* stmt, Oid indexRelationId
indexInfo->ii_Concurrent = false;
indexInfo->ii_BrokenHotChain = false;
indexInfo->ii_PgClassAttrId = 0;
// 创建Delta唯一索引
(void)CreateDeltaUniqueIndex(deltaRelation, deltaIndexRelName, indexInfo, stmt->indexParams,
indexColNames, stmt->primary);
// 关闭Delta表
heap_close(deltaRelation, NoLock);
// 释放关联关系对象
if (parentRel) {
releaseDummyRelation(&rel);
} else {
@ -230,6 +246,17 @@ void DefineDeltaUniqueIndex(Oid relationId, IndexStmt* stmt, Oid indexRelationId
* @IN indexColNames: the list contains name of each index column
* @IN isPrimary: is primary index?
*/
/*
* @Description: Delta表上创建唯一索引
* @IN deltaRel: Delta表的关系对象
* @IN deltaIndexName: Delta表索引的名称
* @IN indexInfo: CU索引的IndexInfo
* @IN indexElemList:
* @IN indexColNames:
* @IN isPrimary:
* @return: Delta索引的OID
*/
//在Delta表上创建唯一索引
Oid CreateDeltaUniqueIndex(Relation deltaRel, const char* deltaIndexName, IndexInfo* indexInfo,
List* indexElemList, List* indexColNames, bool isPrimary)
{
@ -238,20 +265,20 @@ Oid CreateDeltaUniqueIndex(Relation deltaRel, const char* deltaIndexName, IndexI
Oid* collationObjectId = (Oid*)palloc(numIndexCols * sizeof(Oid));
Oid* classObjectId = (Oid*)palloc(numIndexCols * sizeof(Oid));
int16* coloptions = (int16*)palloc(numIndexCols * sizeof(int16));
// 计算索引列的属性值
ComputeIndexAttrs(indexInfo, typeObjectId, collationObjectId, classObjectId, coloptions,
indexElemList, NULL, RelationGetRelid(deltaRel), "btree", BTREE_AM_OID, true, false);
// 设置索引创建额外参数
IndexCreateExtraArgs extra;
SetIndexCreateExtraArgs(&extra, InvalidOid, false, false);
// 创建Delta唯一索引
Oid deltaIndexOid = index_create(deltaRel,
deltaIndexName, InvalidOid, InvalidOid, indexInfo,
indexColNames, BTREE_AM_OID, deltaRel->rd_rel->reltablespace,
collationObjectId, classObjectId, coloptions, (Datum)0,
isPrimary, false, false, false, true, false,
false, &extra);
// 释放内存
pfree(typeObjectId);
pfree(collationObjectId);
pfree(classObjectId);
@ -266,6 +293,7 @@ Oid CreateDeltaUniqueIndex(Relation deltaRel, const char* deltaIndexName, IndexI
* @IN newRelOid: the new cstore after relation files swap.
* @IN parentOid: parent oid for partitioned table, otherwise InvalidOid.
*/
//删除旧delta表上的索引并在新delta表上建立索引
void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid)
{
Relation oldRel = NULL;
@ -273,16 +301,19 @@ void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid)
Relation parentRel = NULL;
Oid oldDeltaOid = InvalidOid;
Oid newDeltaOid = InvalidOid;
// 打开旧的cstore并获取旧的delta表的oid
oldRel = heap_open(oldRelOid, AccessShareLock);
oldDeltaOid = oldRel->rd_rel->reldeltarelid;
heap_close(oldRel, NoLock);
if (OidIsValid(parentOid)) {
/* For partitioned table. */
// 如果是分区表,则打开父关系
parentRel = heap_open(parentOid, AccessShareLock);
heap_close(parentRel, NoLock);
// 打开分区表对应的分区并获取新的关系
Partition partition = partitionOpen(parentRel, newRelOid, AccessShareLock);
// 如果是非分区表则直接打开新的cstore
newRel = partitionGetRelation(parentRel, partition);
partitionClose(parentRel, partition, NoLock);
} else {
@ -299,6 +330,7 @@ void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid)
}
/* Apply AccssShareLock because we only get information from old delta. */
// 打开旧的delta表
Relation oldDelta = heap_open(oldDeltaOid, AccessShareLock);
TupleDesc tupleDesc = RelationGetDescr(oldRel);
@ -313,6 +345,7 @@ void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid)
heap_close(oldDelta, NoLock);
/* Apply ShareLock because we will build index on new delta. */
// 打开新的delta表
Relation newDelta = heap_open(newDeltaOid, ShareLock);
ListCell* cell = NULL;
@ -336,11 +369,12 @@ void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid)
* Must delete the index on old delta table, or conflicts will happen between
* new index name and old index name.
*/
// 删除旧delta表上的索引避免新索引与旧索引名称冲突
indexObject.classId = RelationRelationId;
indexObject.objectId = idxOid;
indexObject.objectSubId = 0;
performDeletion(&indexObject, DROP_RESTRICT, 0);
// 在新的delta表上定义唯一索引
DefineDeltaUniqueIndex(newRelOid, indexStmt, CUIdxOid, parentRel);
}
}
@ -359,13 +393,21 @@ void BuildIndexOnNewDeltaTable(Oid oldRelOid, Oid newRelOid, Oid parentOid)
partition delta index. Otherwise reindex single partition delta index
* For non-partioned cstore, indexPartId=InvaldOid.
*/
/*
* @Description: CU表上的Delta表索引
* @IN indexId: CU上的索引OID
* @IN indexPartId: CU索引的OIDcstoreindexPartId =
* InvalidOid表示重新生成所有分区delta索引delta索引cstoreindexPartId =
* InvaldOid
*/
//重建CU表上的Delta表索引
void ReindexDeltaIndex(Oid indexId, Oid indexPartId)
{
Oid heapId = IndexGetRelation(indexId, false);
/* 因为我们将在Delta表上构建索引所以在CU表上获取AccessShareLock。 */
/* We get AccessShareLock on CU table because we will build index on delta table. */
Relation heapRelation = heap_open(heapId, AccessShareLock);
/* 关闭CU表但保持锁定状态。 */
/* Close CU table, but keep locks. */
heap_close(heapRelation, NoLock);
@ -373,14 +415,18 @@ void ReindexDeltaIndex(Oid indexId, Oid indexPartId)
if (!RELATION_IS_PARTITIONED(heapRelation)) {
/* For non partioned cstore table. */
/* 对于非分区cstore表。 */
deltaIdxOid = GetDeltaIdxFromCUIdx(indexId, false);
reindex_index(deltaIdxOid, InvalidOid, false, NULL, false);
} else {
/* 对于分区cstore表。 */
/* For partitioned cstore table. */
if (OidIsValid(indexPartId)) {
/* 仅重新生成单个分区的delta索引。 */
deltaIdxOid = GetDeltaIdxFromCUIdx(indexPartId, true);
reindex_index(deltaIdxOid, InvalidOid, false, NULL, false);
} else {
/* 重新生成所有分区上的delta索引。 */
/* Reindex all indexes on part delta table. */
List* indexPartOidList = NIL;
ListCell* partCell = NULL;
@ -404,35 +450,36 @@ void ReindexDeltaIndex(Oid indexId, Oid indexPartId)
* @IN indexOid: parent index on CU
* @IN partOid: partition CU oid
*/
//重建分区Delta表的索引
void ReindexPartDeltaIndex(Oid indexOid, Oid partOid)
{
Relation pg_partition = NULL;
Relation pg_partition = NULL;// 打开pg_partition系统表
ScanKeyData scanKey;
SysScanDesc partScan;
HeapTuple partTuple = NULL;
Form_pg_partition partForm = NULL;
Oid indexPartOid = InvalidOid;
pg_partition = heap_open(PartitionRelationId, AccessShareLock);
pg_partition = heap_open(PartitionRelationId, AccessShareLock);// 打开pg_partition系统表
ScanKeyInit(&scanKey, Anum_pg_partition_indextblid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(partOid));
partScan = systable_beginscan(pg_partition, PartitionIndexTableIdIndexId, true, SnapshotNow, 1, &scanKey);
while ((partTuple = systable_getnext(partScan)) != NULL) {
partScan = systable_beginscan(pg_partition, PartitionIndexTableIdIndexId, true, SnapshotNow, 1, &scanKey);// 根据分区CU OID创建系统扫描
while ((partTuple = systable_getnext(partScan)) != NULL) {// 遍历扫描结果
partForm = (Form_pg_partition)GETSTRUCT(partTuple);
if (partForm->parentid == indexOid) {
if (partForm->parentid == indexOid) {// 查找与给定索引OID匹配的分区
indexPartOid = HeapTupleGetOid(partTuple);
break;
}
}
systable_endscan(partScan);
heap_close(pg_partition, AccessShareLock);
systable_endscan(partScan);// 结束系统扫描
heap_close(pg_partition, AccessShareLock);// 关闭pg_partition系统表
if (!OidIsValid(indexPartOid)) {
if (!OidIsValid(indexPartOid)) {// 若未找到匹配的分区OID则抛出错误
ereport(ERROR,
(errcode(ERRCODE_CACHE_LOOKUP_FAILED),
errmsg("cache lookup failed for partitioned index %u", indexOid)));
}
ReindexDeltaIndex(indexOid, indexPartOid);
ReindexDeltaIndex(indexOid, indexPartOid);// 重新构建指定分区CU索引的Delta表索引
}
/*
@ -442,25 +489,35 @@ void ReindexPartDeltaIndex(Oid indexOid, Oid partOid)
* false means delta index name is like pg_delta_index_xxx.
* @Reture: the CU index oid
*/
/*
* @Description: Delta表索引获取其对应的CU索引OID
* @IN deltaIdx: Delta表索引
* @IN isPartition:
* true表示Delta表索引命名方式为pg_delta_part_index_xxxfalse表示Delta表索引命名方式为pg_delta_index_xxx
* @Return: CU索引OID
*/
//从Delta表索引获取与其对应的CU索引OID
static Oid GetCUIdxFromDeltaIdx(Relation deltaIdx, bool isPartition)
{
Oid CUIdxOid = 0;
int curIdx = 0;
const char* deltaIdxName = RelationGetRelationName(deltaIdx);
Oid CUIdxOid = 0;// 定义CU索引OID
int curIdx = 0;// 记录当前遍历到第几个字符
const char* deltaIdxName = RelationGetRelationName(deltaIdx);// 获取Delta表索引名称
if (isPartition) {
/* For partitioned delta table index. */
/* 对于分区Delta表索引。*/
curIdx = strlen("pg_delta_part_index_");
} else {
/* For non partitioned delta table index. */
/* 对于非分区Delta表索引。*/
curIdx = strlen("pg_delta_index_");
}
while (deltaIdxName[curIdx] != '\0') {
CUIdxOid = CUIdxOid * 10 + (deltaIdxName[curIdx] - '0');
curIdx++;
while (deltaIdxName[curIdx] != '\0') {// 遍历Delta表索引名称
CUIdxOid = CUIdxOid * 10 + (deltaIdxName[curIdx] - '0');// 解析CU索引OID
curIdx++;// 指向下一个字符
}
return CUIdxOid;
return CUIdxOid;// 返回解析出的CU索引OID
}
/*
@ -470,31 +527,39 @@ static Oid GetCUIdxFromDeltaIdx(Relation deltaIdx, bool isPartition)
* @IN suppressMiss: return InvalidOid if true when delta index is not found
* @Reture: the delta index oid
*/
/*
* @Description: CU索引获取Delta索引OID
* @IN CUIndexOid: CU索引OID
* @IN isPartitioned: CU索引是否为分区索引
* @IN suppressMiss: trueDelta索引时返回InvalidOid
* @Return: Delta索引OID
*/
//从对应的CU索引获取Delta索引OID
Oid GetDeltaIdxFromCUIdx(Oid CUIndexOid, bool isPartitioned, bool suppressMiss)
{
char deltaIdxName[NAMEDATALEN] = {'\0'};
Relation pgclass = NULL;
ScanKeyData scanKey[1];
SysScanDesc scan = NULL;
HeapTuple tup = NULL;
Oid deltaIdxOid = InvalidOid;
char deltaIdxName[NAMEDATALEN] = {'\0'};// 定义Delta索引名称
Relation pgclass = NULL;// 用于访问pg_class系统表
ScanKeyData scanKey[1];// 扫描键
SysScanDesc scan = NULL;// 扫描描述符
HeapTuple tup = NULL;// 堆元组
Oid deltaIdxOid = InvalidOid;// 定义Delta索引OID
error_t ret = 0;
if (!isPartitioned) {
ret = snprintf_s(deltaIdxName, sizeof(deltaIdxName),
sizeof(deltaIdxName) - 1, "pg_delta_index_%u", CUIndexOid);
ret = snprintf_s(deltaIdxName, sizeof(deltaIdxName), sizeof(deltaIdxName) - 1, "pg_delta_index_%u",
CUIndexOid); // 根据CU索引OID构建Delta索引名称
} else {
ret = snprintf_s(deltaIdxName, sizeof(deltaIdxName),
sizeof(deltaIdxName) - 1, "pg_delta_part_index_%u", CUIndexOid);
sizeof(deltaIdxName) - 1, "pg_delta_part_index_%u", CUIndexOid);// 根据CU索引OID构建分区Delta索引名称
}
securec_check_ss_c(ret, "\0", "\0");
ScanKeyInit(&scanKey[0], Anum_pg_class_relname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(deltaIdxName));
pgclass = heap_open(RelationRelationId, AccessShareLock);
scan = systable_beginscan(pgclass, ClassNameNspIndexId, true, SnapshotNow, 1, scanKey);
tup = systable_getnext(scan);
ScanKeyInit(&scanKey[0], Anum_pg_class_relname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(deltaIdxName));// 初始化扫描键
pgclass = heap_open(RelationRelationId, AccessShareLock);// 打开pg_class系统表
scan = systable_beginscan(pgclass, ClassNameNspIndexId, true, SnapshotNow, 1, scanKey);// 开始扫描pg_class系统表
tup = systable_getnext(scan);// 获取下一个符合条件的堆元组
if (HeapTupleIsValid(tup)) {
deltaIdxOid = HeapTupleGetOid(tup);
deltaIdxOid = HeapTupleGetOid(tup);// 获取Delta索引OID
} else {
if (suppressMiss) {
systable_endscan(scan);
@ -502,12 +567,12 @@ Oid GetDeltaIdxFromCUIdx(Oid CUIndexOid, bool isPartitioned, bool suppressMiss)
return InvalidOid;
}
ereport(
ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation \"%s\" does not exist", deltaIdxName)));
ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation \"%s\" does not exist", deltaIdxName)));// 报告错误指定的Delta索引不存在
}
systable_endscan(scan);
heap_close(pgclass, AccessShareLock);
systable_endscan(scan);// 结束扫描
heap_close(pgclass, AccessShareLock);// 关闭pg_class系统表
return deltaIdxOid;
return deltaIdxOid;// 返回Delta索引OID
}
/*
@ -515,35 +580,41 @@ Oid GetDeltaIdxFromCUIdx(Oid CUIndexOid, bool isPartitioned, bool suppressMiss)
* @IN deltaIdx: delta index which name stores CU index oid
* @Return: CU index name
*/
/*
* @Description: Delta索引获取CU索引名称
* @IN deltaIdx: CU索引OID的Delta索引
* @Return: CU索引名称
*/
//从Delta索引获取CU索引名称
char* GetCUIdxNameFromDeltaIdx(Relation deltaIdx)
{
const char* deltaIdxName = RelationGetRelationName(deltaIdx);
const char* deltaIdxName = RelationGetRelationName(deltaIdx);// 获取Delta索引的名称
bool partitioned;
if (pg_strncasecmp(deltaIdxName, "pg_delta_part_index_", strlen("pg_delta_part_index_")) == 0) {
partitioned = true;
partitioned = true;// 判断Delta索引是否为分区Delta索引
} else {
partitioned = false;
}
Oid CUIdxOid = GetCUIdxFromDeltaIdx(deltaIdx, partitioned);
Oid CUIdxOid = GetCUIdxFromDeltaIdx(deltaIdx, partitioned);// 通过Delta索引获取CU索引的OID
char* CUIdxName = NULL;
if (!partitioned) {
CUIdxName = get_rel_name(CUIdxOid);
CUIdxName = get_rel_name(CUIdxOid);// 获取非分区CU索引的名称
} else {
HeapTuple tuple = SearchSysCacheCopy1(PARTRELID, ObjectIdGetDatum(CUIdxOid));
HeapTuple tuple = SearchSysCacheCopy1(PARTRELID, ObjectIdGetDatum(CUIdxOid));// 通过分区CU索引的OID搜索系统缓存
if (!HeapTupleIsValid(tuple)) {
ereport(ERROR,
(errcode(ERRCODE_WRONG_OBJECT_TYPE),
errmsg("could not find tuple for partition index %u", CUIdxOid)));
errmsg("could not find tuple for partition index %u", CUIdxOid)));// 如果未找到分区CU索引的元组则报错
}
Form_pg_partition partForm = (Form_pg_partition)GETSTRUCT(tuple);
Oid parentIdxOid = partForm->parentid;
Oid parentIdxOid = partForm->parentid;// 获取分区CU索引的父索引OID
CUIdxName = get_rel_name(parentIdxOid);
CUIdxName = get_rel_name(parentIdxOid);// 获取分区CU索引的父索引名称
heap_freetuple(tuple);
heap_freetuple(tuple);// 释放缓存的元组
}
return CUIdxName;
return CUIdxName;// 返回CU索引名称
}

View File

@ -30,25 +30,25 @@ THR_LOCAL uint64 CStoreMemAlloc::m_count = 0;
THR_LOCAL uint32 CStoreMemAlloc::m_ptrNodeCacheCount = 0;
THR_LOCAL PointerNode* CStoreMemAlloc::m_ptrNodeCache[MaxPtrNodeCacheLen];
THR_LOCAL PointerList CStoreMemAlloc::m_tab[MaxPointersArryLen];
//在内存中分配一段连续的空间,并返回分配的内存的首地址
static inline void* InnerMalloc(Size size)
{
void* ptr = NULL;
ADIO_RUN()
ADIO_RUN()// 判断是否启用了Async IO
{
int ret = posix_memalign((void**)&(ptr), SYS_LOGICAL_BLOCK_SIZE, (size_t)(size));
int ret = posix_memalign((void**)&(ptr), SYS_LOGICAL_BLOCK_SIZE, (size_t)(size));// 采用posix_memalign分配内存
if (ret != 0) {
ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH),
errmsg("posix_memalign fails, The alignment argument was not a power of two, or was not a multiple "
"of sizeof(void *)")));
"of sizeof(void *)")));// 如果分配内存失败,则报错
}
}
ADIO_ELSE()
ADIO_ELSE()// 如果没有启用Async IO则采用malloc分配内存
{
ptr = malloc(size);
}
ADIO_END();
return ptr;
return ptr;// 返回分配的内存的首地址
}
/*
@ -58,40 +58,58 @@ static inline void* InnerMalloc(Size size)
* if this memory is process-managered scope, set false.
* @See also:
*/
/*
* @Description:
* @Param[IN] size:
* @Param[IN] toRegister: 线true;
* false
* @See also:
*/
//在内存中分配一段连续的空间,并返回分配的内存的首地址
void* CStoreMemAlloc::Palloc(Size size, bool toRegister)
{
Assert(size > 0);
void* ptr = InnerMalloc(size);
Assert(size > 0);// 断言需要申请的内存大小大于0
void* ptr = InnerMalloc(size);// 调用InnerMalloc函数分配内存
if (ptr == NULL) {
ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("malloc fails, out of memory: size %lu", size)));
ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("malloc fails, out of memory: size %lu", size)));// 如果分配内存失败,则报错
}
if (toRegister) {
/* Step 2: register pointer */
Register(ptr);
Register(ptr);// 如果需要将该内存注册到线程管理的范围则调用Register函数进行注册
}
return ptr;
return ptr;// 返回分配的内存的首地址
}
/*
* @Description: PointerNode PointerNode
* @Param:
* @See also:
*/
//从缓存中分配一个 PointerNode 结构体,或者从系统中申请一个 PointerNode 结构体
void* CStoreMemAlloc::AllocPointerNode()
{
PointerNode* ptr = NULL;
if (m_ptrNodeCacheCount > 0) {
if (m_ptrNodeCacheCount > 0) {// 如果缓存中有 PointerNode 结构体,则从缓存中分配一个结构体
ptr = m_ptrNodeCache[--m_ptrNodeCacheCount];
} else {
} else {// 如果缓存中没有 PointerNode 结构体,则从系统中申请一个结构体
ptr = (PointerNode*)malloc(sizeof(PointerNode));
if (ptr == NULL) {
ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("malloc fails, out of memory")));
}
}
return ptr;
return ptr;// 返回分配的 PointerNode 结构体的首地址
}
/*
* @Description: PointerNode
* @Param[IN] ptr: PointerNode
* @See also:
*/
//释放一个 PointerNode 结构体,并将其放入缓存中或直接释放内存
void CStoreMemAlloc::FreePointerNode(PointerNode* ptr)
{
if (m_ptrNodeCacheCount < MaxPtrNodeCacheLen) {
if (m_ptrNodeCacheCount < MaxPtrNodeCacheLen) {// 如果缓存中还有空间,则将 PointerNode 放入缓存中
m_ptrNodeCache[m_ptrNodeCacheCount++] = ptr;
} else {
} else {// 如果缓存已满,则直接释放 PointerNode 的内存
free(ptr);
}
}
@ -105,28 +123,37 @@ void CStoreMemAlloc::FreePointerNode(PointerNode* ptr)
* false if passing false to Palloc();
* @See also:
*/
/*
* @Description: realloc函数
* @Param[IN] old_size:
* @Param[IN] size:
* @Param[IN] pointer:
* @Param[IN] registered: true Palloc() true false Palloc() false
* @See also:
*/
//重新分配内存空间类似于系统的realloc函数
void* CStoreMemAlloc::Repalloc(void* pointer, Size size, Size old_size, bool registered)
{
Assert(pointer != NULL);
Assert(size > 0);
Assert(pointer != NULL);// 断言:指针不为空
Assert(size > 0);// 断言新的内存块大小大于0
void* ptr = InnerMalloc(size);
if (ptr == NULL) {
void* ptr = InnerMalloc(size);// 分配新的内存块
if (ptr == NULL) {// 如果分配失败,则报错
ereport(ERROR, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory")));
}
errno_t rc = memcpy_s(ptr, size, pointer, old_size);
errno_t rc = memcpy_s(ptr, size, pointer, old_size);// 将旧的内存块内容拷贝到新的内存块中
securec_check_c(rc, "\0", "\0");
if (registered) {
if (registered) {// 如果传入的 registered 参数为 true则将新的内存块注册同时解除旧的内存块的注册
Register(ptr);
Unregister(pointer);
Assert(m_count > 0);
}
free(pointer);
free(pointer);// 释放旧的内存块的内存空间
return ptr;
return ptr;// 返回重新分配的内存块的首地址
}
/*
@ -136,83 +163,103 @@ void* CStoreMemAlloc::Repalloc(void* pointer, Size size, Size old_size, bool reg
* false if passing false to Palloc();
* @See also:
*/
/*
* @Description: free函数
* @Param[IN] pointer:
* @Param[IN] registered: true Palloc() true false Palloc() false
* @See also:
*/
//释放内存空间类似于系统的free函数
void CStoreMemAlloc::Pfree(void* pointer, bool registered)
{
Assert(pointer);
if (registered) {
Assert(pointer);// 断言:指针不为空
if (registered) {// 如果传入的 registered 参数为 true则解除内存块的注册
Unregister(pointer);
}
free(pointer);
free(pointer);// 释放内存块的内存空间
}
/*
* @Description:
* @Param[IN] pointer:
* @See also:
*/
//注册内存块的指针地址,并将其添加到哈希表中
void CStoreMemAlloc::Register(void* pointer)
{
Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0);
int idx = PointerGetDatum(pointer) & (MaxPointersArryLen - 1);
PointerNode* nodePtr = m_tab[idx].tail;
Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0);// 断言MaxPointersArryLen 是2的幂
int idx = PointerGetDatum(pointer) & (MaxPointersArryLen - 1);// 通过指针的数值计算哈希表的索引
PointerNode* nodePtr = m_tab[idx].tail;// 取出哈希表对应索引的链表的末尾节点
if (nodePtr == NULL) {
if (nodePtr == NULL) { // 如果末尾节点为空,则说明该索引处还没有节点,需要新建一个节点并将其设置为头尾指针
Assert(m_tab[idx].header == NULL);
m_tab[idx].header = m_tab[idx].tail = (PointerNode*)AllocPointerNode();
m_tab[idx].header->ptr = pointer;
m_tab[idx].header->next = NULL;
} else {
} else {// 如果末尾节点不为空,则新建一个节点,并添加到链表的末尾
nodePtr->next = (PointerNode*)AllocPointerNode();
nodePtr->next->ptr = pointer;
nodePtr->next->next = NULL;
m_tab[idx].tail = nodePtr->next;
}
++m_count;
++m_count;// 统计注册的内存块的数量
}
/*
* @Description:
* @Param[IN] pointer:
* @See also:
*/
//取消注册内存块的指针地址,并从哈希表中移除
void CStoreMemAlloc::Unregister(const void* pointer)
{
Assert(pointer && m_count > 0);
Assert(pointer && m_count > 0);// 断言指针不能为空且已注册的内存块数量大于0
// Step 1: which node list include this pointer
//
Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0);
int idx = PointerGetDatum(pointer) & (MaxPointersArryLen - 1);
Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0); // 断言MaxPointersArryLen 是2的幂
int idx = PointerGetDatum(pointer) & (MaxPointersArryLen - 1);// 通过指针的数值计算哈希表的索引
// Step 2: Search node list
//
PointerNode* nodePtr = m_tab[idx].header;
PointerNode* nodePtr = m_tab[idx].header;// 获取哈希表索引处链表的头指针
Assert(nodePtr);
PointerNode* prePtr = NULL;
while (nodePtr != NULL) {
if (nodePtr->ptr == pointer) {
if (prePtr == NULL) {
PointerNode* prePtr = NULL;// 初始化前一个节点指针为NULL
while (nodePtr != NULL) {// 遍历链表,寻找要取消注册的指针地址对应的节点
if (nodePtr->ptr == pointer) {// 如果找到了要取消注册的节点
if (prePtr == NULL) {// 如果该节点是头指针
m_tab[idx].header = nodePtr->next;
// If this list has only one node
//
if (m_tab[idx].tail == nodePtr) {
if (m_tab[idx].tail == nodePtr) {// 如果该链表只有一个节点
Assert(m_tab[idx].header == NULL);
m_tab[idx].tail = NULL;
m_tab[idx].tail = NULL;// 将尾指针设为NULL
}
} else {
} else {// 如果该节点不是头指针
prePtr->next = nodePtr->next;
// We need modify tail pointer if free tail node
//
if (m_tab[idx].tail == nodePtr) {
m_tab[idx].tail = prePtr;
if (m_tab[idx].tail == nodePtr) {// 如果该节点是尾指针
m_tab[idx].tail = prePtr;// 将尾指针设为前一个节点
Assert(m_tab[idx].tail->next == NULL);
}
}
nodePtr->Reset();
FreePointerNode(nodePtr);
nodePtr->Reset();// 重置节点的内容
FreePointerNode(nodePtr);// 释放节点内存
break;
}
prePtr = nodePtr;
nodePtr = nodePtr->next;
nodePtr = nodePtr->next;// 继续遍历下一个节点
}
--m_count;
Assert(nodePtr != NULL);
--m_count;// 更新已注册的内存块数量
Assert(nodePtr != NULL);// 断言:找到了要取消注册的节点
}
/*
* @Description:
*/
//重置内存分配器的状态,释放已注册的内存块和缓存的节点
void CStoreMemAlloc::Reset()
{
uint32 i = 0;
@ -221,41 +268,43 @@ void CStoreMemAlloc::Reset()
// Step 1: free each list in m_tab
//
for (i = 0; i < MaxPointersArryLen; ++i) {
PointerNode* nodePtr = m_tab[i].header;
PointerNode* nodePtr = m_tab[i].header;// 获取哈希表索引处链表的头指针
PointerNode* tmpPtr = NULL;
while ((nodePtr != NULL) && (nodePtr->ptr != NULL)) {
free(nodePtr->ptr);
tmpPtr = nodePtr->next;
free(nodePtr);
nodePtr = tmpPtr;
++freeNum;
while ((nodePtr != NULL) && (nodePtr->ptr != NULL)) {// 遍历链表,释放节点和节点中的内存块
free(nodePtr->ptr);// 释放节点中的内存块
tmpPtr = nodePtr->next;// 保存下一个节点指针
free(nodePtr);// 释放节点内存
nodePtr = tmpPtr;// 继续处理下一个节点
++freeNum;// 更新释放的节点数量
}
// Note that We must reset NULL
// Because thread can be reused
//
m_tab[i].header = NULL;
m_tab[i].tail = NULL;
m_tab[i].header = NULL;// 将头指针设为NULL
m_tab[i].tail = NULL;// 将尾指针设为NULL
}
Assert(m_count == freeNum);
m_count = 0;
Assert(m_count == freeNum);// 断言:已注册的内存块数量等于释放的内存块数量
m_count = 0;// 将已注册的内存块数量设为0
}
// Step 2: free cached node if need
//
for (i = 0; i < m_ptrNodeCacheCount; ++i) {
free(m_ptrNodeCache[i]);
free(m_ptrNodeCache[i]);// 释放缓存的节点内存
}
m_ptrNodeCacheCount = 0;
m_ptrNodeCacheCount = 0;// 将缓存的节点数量设为0
}
/*
* @Description:
*/
void CStoreMemAlloc::Init()
{
m_count = 0;
Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0);
m_count = 0;// 将已注册的内存块数量设为0
Assert((MaxPointersArryLen & (MaxPointersArryLen - 1)) == 0);// 断言确保MaxPointersArryLen是2的幂次方
for (uint32 i = 0; i < MaxPointersArryLen; ++i) {
m_tab[i].header = NULL;
m_tab[i].tail = NULL;
m_tab[i].header = NULL;// 将哈希表索引处链表的头指针设为NULL
m_tab[i].tail = NULL;// 将哈希表索引处链表的尾指针设为NULL
}
m_ptrNodeCacheCount = 0;
m_ptrNodeCacheCount = 0;// 将缓存的节点数量设为0
}

View File

@ -32,18 +32,18 @@
/* Min/Max Option Function info */
struct FuncSetMinMaxInfo {
/* data-type OID */
Oid typeOid;
Oid typeOid;// 数据类型的对象标识符
/* Min/Max Set functions
* Notice: we retain this member to avoid modifying many codes.
*/
FuncSetMinMax set_minmax_func;
FuncSetMinMax set_minmax_func;// 最小/最大设置函数
/* substitute compare_datum_func for set_minmax_func !
* compare datum and set min/max info.
*/
CompareDatum compare_datum_func;
FinishCompareDatum finish_compare_datum_func;
CompareDatum compare_datum_func;// 用于比较数据和设置最小/最大信息的函数
FinishCompareDatum finish_compare_datum_func;// 完成比较数据函数
};
/* two offsets for var-length string
@ -85,13 +85,15 @@ static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first,
* @IN minval: min value info
* @See also:
*/
//实际上没有任何实际操作,只是为了让编译器保持安静
static void FinishCompareDummy(const char* minval, const char* maxval, CUDesc* cuDescPtr)
{
/* just keep compiler silent */
UNUSED_ARG(minval);
UNUSED_ARG(maxval);
UNUSED_ARG(cuDescPtr);
return;
/* 只是为了让编译器保持安静 */
UNUSED_ARG(minval);// 未使用的参数minval
UNUSED_ARG(maxval);// 未使用的参数maxval
UNUSED_ARG(cuDescPtr); // 未使用的参数cuDescPtr
return;// 返回
}
/*
@ -101,12 +103,16 @@ static void FinishCompareDummy(const char* minval, const char* maxval, CUDesc* c
* @IN minval: min value info
* @See also:
*/
//用于固定长度数据类型的设置CUDesc最小/最大值
static void FinishCompareFixedLength(const char* minval, const char* maxval, CUDesc* cuDescPtr)
{
errno_t rc = EOK;
rc = memcpy_s(cuDescPtr->cu_min, MIN_MAX_LEN, minval, MIN_MAX_LEN);
errno_t rc = EOK;// 定义errno_t变量rc并赋初值EOK
/* 将最小值信息复制到CU描述信息的最小值位置上 */
rc = memcpy_s(cuDescPtr->cu_min, MIN_MAX_LEN, minval, MIN_MAX_LEN);// 将minval的MIN_MAX_LEN长度的数据复制到cuDescPtr->cu_min
securec_check(rc, "\0", "\0");
/* 将最大值信息复制到CU描述信息的最大值位置上 */
rc = memcpy_s(cuDescPtr->cu_max, MIN_MAX_LEN, maxval, MIN_MAX_LEN);// 将maxval的MIN_MAX_LEN长度的数据复制到cuDescPtr->cu_max
securec_check(rc, "\0", "\0");
rc = memcpy_s(cuDescPtr->cu_max, MIN_MAX_LEN, maxval, MIN_MAX_LEN);
securec_check(rc, "\0", "\0");
}
@ -117,52 +123,60 @@ static void FinishCompareFixedLength(const char* minval, const char* maxval, CUD
* @IN minval: min value info
* @See also:
*/
/*
* @Description: CUDesc最小/
* @OUT cuDescPtr: CU描述信息
* @IN maxval:
* @IN minval:
* @See also:
*/
static void FinishCompareVarStrType(const char* minval, const char* maxval, CUDesc* cuDescPtr)
{
char* dat = NULL;
errno_t rc = EOK;
char* dat = NULL;// 定义char类型指针dat并初始化为NULL
errno_t rc = EOK;// 定义errno_t变量rc并赋初值EOK
/* if all values are NULLs, data point is 0. */
/* if all values are NULLs, data point is 0. */ /* 如果所有值都为NULLs则数据点为0。 */
if (*(Datum*)minval != 0) {
Assert(minval[varstr_dat_length] < MIN_MAX_LEN);
Assert(minval[varstr_dat_offset] == 1 || minval[varstr_dat_offset] == 4);
Assert(minval[varstr_dat_length] < MIN_MAX_LEN);// 断言minval的varstr_dat_length小于MIN_MAX_LEN
Assert(minval[varstr_dat_offset] == 1 || minval[varstr_dat_offset] == 4);// 断言minval的varstr_dat_offset等于1或4
/* remember the length of min value */
/* remember the length of min value */ /* 记录最小值的长度 */
cuDescPtr->cu_min[0] = minval[varstr_dat_length];
dat = DatumGetPointer(*(Datum*)minval) + minval[varstr_dat_offset];
/* remember the min value */
rc = memcpy_s(cuDescPtr->cu_min + 1, (MIN_MAX_LEN - 1), dat, minval[varstr_dat_length]);
/* remember the min value */ /* 记录最小值 */
rc = memcpy_s(cuDescPtr->cu_min + 1, (MIN_MAX_LEN - 1), dat,
minval[varstr_dat_length]); // 将dat的minval[varstr_dat_length]长度的数据复制到cuDescPtr->cu_min
securec_check(rc, "\0", "\0");
Assert(maxval[varstr_dat_length] < MIN_MAX_LEN);
Assert(maxval[varstr_dat_offset] == 1 || maxval[varstr_dat_offset] == 4);
Assert(maxval[varstr_dat_length] < MIN_MAX_LEN);// 断言maxval的varstr_dat_length小于MIN_MAX_LEN
Assert(maxval[varstr_dat_offset] == 1 || maxval[varstr_dat_offset] == 4);// 断言maxval的varstr_dat_offset等于1或4
/* remember the length of max value */
/* remember the length of max value */ /* 记录最大值的长度 */
cuDescPtr->cu_max[0] = maxval[varstr_dat_length];
dat = DatumGetPointer(*(Datum*)maxval) + maxval[varstr_dat_offset];
/* remember the max value */
rc = memcpy_s(cuDescPtr->cu_max + 1, (MIN_MAX_LEN - 1), dat, maxval[varstr_dat_length]);
/* remember the max value */ /* 记录最大值 */
rc = memcpy_s(cuDescPtr->cu_max + 1, (MIN_MAX_LEN - 1), dat, maxval[varstr_dat_length]);// 将dat的maxval[varstr_dat_length]长度的数据复制到cuDescPtr->cu_max
securec_check(rc, "\0", "\0");
} else {
/* cuDescPtr->cu_min/cu_max have been set to 0, so nothing to do */
/* cuDescPtr->cu_min/cu_max have been set to 0, so nothing to do */ /* cuDescPtr->cu_min/cu_max已经设置为0因此不需要做任何操作 */
}
}
// 定义了一个结构体数组g_FuncTabSetMinMax用于存储不同数据类型的最小/最大值设置函数
static FuncSetMinMaxInfo g_FuncTabSetMinMax[] = {
{ BOOLOID, /* OID = 16 bool */
NULL,
CompareDummy,
FinishCompareDummy
NULL,// 最小/最大值设置函数为空
CompareDummy,// 比较函数为CompareDummy
FinishCompareDummy// 结束比较函数为FinishCompareDummy
},
{ BYTEAOID, /* OID = 17 bytea */
SetMinMaxVarStrType,
CompareVarStrType,
FinishCompareVarStrType
SetMinMaxVarStrType,// 最小/最大值设置函数为SetMinMaxVarStrType
CompareVarStrType,// 比较函数为CompareVarStrType
FinishCompareVarStrType// 结束比较函数为FinishCompareVarStrType
},
{ CHAROID, /* OID = 18 char */
SetMinMaxChar,
CompareChar,
FinishCompareFixedLength
SetMinMaxChar,// 最小/最大值设置函数为SetMinMaxChar
CompareChar,// 比较函数为CompareChar
FinishCompareFixedLength// 结束比较函数为FinishCompareFixedLength
},
{ NAMEOID, /* OID = 19 name */
NULL,
@ -242,25 +256,27 @@ static FuncSetMinMaxInfo g_FuncTabSetMinMax[] = {
};
const int FuncSetMinMaxTabSize = sizeof(g_FuncTabSetMinMax) / sizeof(FuncSetMinMaxInfo);
//使用二分查找法来查找数据类型的OID在g_FuncTabSetMinMax结构体数组中的索引
/* Search and return the index of type OID given */
static int BinarySearch(Oid typeOid)
{
int left = 0;
int mid = -1;
int right = FuncSetMinMaxTabSize - 1;
int left = 0;// 左边界
int mid = -1;// 中间位置
int right = FuncSetMinMaxTabSize - 1;// 右边界
while (left <= right) {
mid = left + ((right - left) / 2);
mid = left + ((right - left) / 2);// 计算中间位置的索引
// 如果中间位置的OID大于要查找的OID则说明要查找的OID在左半部分更新右边界
if (g_FuncTabSetMinMax[mid].typeOid > typeOid)
right = mid - 1;
// 如果中间位置的OID小于要查找的OID则说明要查找的OID在右半部分更新左边界
else if (g_FuncTabSetMinMax[mid].typeOid < typeOid)
left = mid + 1;
// 如果中间位置的OID等于要查找的OID则直接返回中间位置的索引
else
return mid;
}
// 如果未找到则返回数组的大小FuncSetMinMaxTabSize
return FuncSetMinMaxTabSize;
}
@ -270,9 +286,13 @@ static int BinarySearch(Oid typeOid)
* @Return: function about min/max setting
* @See also:
*/
// 查找指定数据类型的最小/最大值设置函数
// 参数 typeOid数据类型 OID
// 返回值:与最小/最大值设置相关的函数FuncSetMinMax
FuncSetMinMax GetMinMaxFunc(Oid typeOid)
{
int idx = BinarySearch(typeOid);
int idx = BinarySearch(typeOid);// 使用二分查找法,查找该数据类型在结构体数组中的索引
// 如果找到了指定数据类型的最小/最大值设置函数,则返回该函数
return (idx < FuncSetMinMaxTabSize) ? g_FuncTabSetMinMax[idx].set_minmax_func : NULL;
}
@ -282,9 +302,13 @@ FuncSetMinMax GetMinMaxFunc(Oid typeOid)
* @Return: function about datum comparing
* @See also:
*/
// 查找指定数据类型的值比较函数
// 参数 typeOid数据类型 OID
// 返回值与值比较相关的函数CompareDatum
CompareDatum GetCompareDatumFunc(Oid typeOid)
{
int idx = BinarySearch(typeOid);
int idx = BinarySearch(typeOid);// 使用二分查找法,查找该数据类型在结构体数组中的索引
// 如果找到了指定数据类型的值比较函数,则返回该函数
return (idx < FuncSetMinMaxTabSize) ? g_FuncTabSetMinMax[idx].compare_datum_func : CompareDummy;
}
@ -294,9 +318,14 @@ CompareDatum GetCompareDatumFunc(Oid typeOid)
* @Return: finishing function about datum comparing
* @See also:
*/
// 查找指定数据类型的值比较完成函数
// 参数 typeOid数据类型 OID
// 返回值与值比较完成相关的函数FinishCompareDatum
//查找指定数据类型的值比较完成函数。函数的参数为数据类型OID typeOid返回值为与值比较完成相关的函数指针FinishCompareDatum
FinishCompareDatum GetFinishCompareDatum(Oid typeOid)
{
int idx = BinarySearch(typeOid);
int idx = BinarySearch(typeOid);// 使用二分查找法,查找该数据类型在结构体数组中的索引
// 如果找到了指定数据类型的值比较完成函数,则返回该函数
return (idx < FuncSetMinMaxTabSize) ? g_FuncTabSetMinMax[idx].finish_compare_datum_func : FinishCompareDummy;
}
@ -306,9 +335,12 @@ FinishCompareDatum GetFinishCompareDatum(Oid typeOid)
* @Return: true if f is a dummy function; otherwise false.
* @See also:
*/
// 检查函数 f 是否为占位函数
// 参数 f值比较函数CompareDatum
// 返回值:如果 f 是占位函数返回 true否则返回 false
bool IsCompareDatumDummyFunc(CompareDatum f)
{
return (CompareDummy == f);
return (CompareDummy == f); // 判断函数指针 f 是否等于占位函数 CompareDummy
}
/*
@ -320,14 +352,22 @@ bool IsCompareDatumDummyFunc(CompareDatum f)
* @IN/OUT varstr_maxlen: max length about var-length string
* @See also:
*/
// 占位的值比较函数
// 参数:
// - first: 指示是否是第一个值的标志(输入输出参数)
// - maxval: 最新的最大值(输入输出参数)
// - minval: 最新的最小值(输入输出参数)
// - v: 新的值
// - varstr_maxlen: 变长字符串的最大长度(输入输出参数)
// 该函数没有实际的比较操作,只是将所有参数标记为未使用,并返回。
static void CompareDummy(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
UNUSED_ARG(minval);
UNUSED_ARG(minval);// 标记未使用的参数
UNUSED_ARG(maxval);
UNUSED_ARG(v);
UNUSED_ARG(first);
UNUSED_ARG(varstr_maxlen);
return;
return;// 返回
}
/*
@ -339,21 +379,29 @@ static void CompareDummy(char* minval, char* maxval, Datum v, bool* first, int*
* @IN/OUT varstr_maxlen: max length about var-length string
* @See also:
*/
// 用于 int8 类型数据的值比较函数
// 参数:
// - first: 指示是否是第一个值的标志(输入输出参数)
// - maxval: 最新的最大值(输入输出参数)
// - minval: 最新的最小值(输入输出参数)
// - v: 新的 int8 类型值
// - varstr_maxlen: 变长字符串的最大长度(输入输出参数)
// 该函数将输入值与当前最大值和最小值进行比较,并更新这些值。
static void CompareChar(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
char chVal = DatumGetChar(v);
char chVal = DatumGetChar(v);// 将 int8 类型的值转换为 char 类型
if (!*first) {
if (chVal < minval[0]) {
minval[0] = chVal;
} else if (chVal > maxval[0]) {
maxval[0] = chVal;
if (!*first) {// 如果不是第一个值
if (chVal < minval[0]) {// 如果新值比最小值小
minval[0] = chVal;// 更新最小值为新值
} else if (chVal > maxval[0]) {// 如果新值比最大值大
maxval[0] = chVal;// 更新最大值为新值
}
} else {
minval[0] = maxval[0] = chVal;
*first = false;
} else {// 如果是第一个值
minval[0] = maxval[0] = chVal;// 最小值和最大值都等于新值
*first = false;// 设置第一个值的标志为 false
}
UNUSED_ARG(varstr_maxlen);
UNUSED_ARG(varstr_maxlen);// 标记未使用的参数
}
/*
@ -363,9 +411,15 @@ static void CompareChar(char* minval, char* maxval, Datum v, bool* first, int* v
* @IN v: the new int8 value
* @See also:
*/
// 用于设置 int8 类型的最小值和最大值
// 参数:
// - cuDescPtr: CU 描述符(输入输出参数)
// - first: 第一个值的标志(输入输出参数)
// - v: 新的 int8 类型值
// 该函数调用 CompareChar 函数,将新值与当前最小值和最大值进行比较,并更新这些值。
static void SetMinMaxChar(Datum v, CUDesc* cuDescPtr, bool* first)
{
CompareChar(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL);
CompareChar(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL);// 调用 CompareChar 函数
}
/*
@ -382,6 +436,38 @@ static void SetMinMaxChar(Datum v, CUDesc* cuDescPtr, bool* first)
* 1B --> Offset to Var-Data, or Var-Head size
* 1B --> Compare Data Size
*/
/*
* @Description: comparing function for data-type var-length string
*
*
* @IN/OUT first: indicate the first value
*
*
* @IN/OUT maxval: the newest max value
*
*
* @IN/OUT minval: the newest min value
*
*
* @IN v: the new var-length string value
*
*
* @IN/OUT varstr_maxlen: max length about var-length string
*
*
* @See also:
*
* minval/maxval struct:
* 8B --> Datum, pointer to min/max value
* 1B --> Offset to Var-Data, or Var-Head size
* 1B --> Compare Data Size
*
* minval/maxval
* 8B --> Datum/
* 1B -->
* 1B -->
*/
//比较变长字符串类型的数据
static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
char* v_hdr = DatumGetPointer(v);
@ -396,6 +482,7 @@ static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first,
Assert(v != (Datum)0);
if (!*first) {
/* Update Var-String max length info */
// 更新变长字符串的最大长度信息
if (v_total_len > *varstr_maxlen) {
*varstr_maxlen = v_total_len;
}
@ -408,10 +495,13 @@ static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first,
ret = memcmp(v_dat, curmin_dat, cmp_size);
if (ret < 0 || (ret == 0 && copy_size < minval[varstr_dat_length])) {
/* Update min value, and remember its pointer */
// 更新最小值,并记住其指针
*(Datum*)minval = v;
/* Remember its data offset */
// 记住其数据偏移量
minval[varstr_dat_offset] = (char)(v_dat - v_hdr);
/* Remember its data length */
// 记住其数据长度
minval[varstr_dat_length] = copy_size;
return;
}
@ -424,21 +514,28 @@ static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first,
ret = memcmp(v_dat, curmax_dat, cmp_size);
if (ret > 0 || (ret == 0 && copy_size > maxval[varstr_dat_length])) {
/* Update max value, and remember its pointer */
// 更新最大值,并记住其指针
*(Datum*)maxval = v;
/* Remember its data offset */
// 记住其数据偏移量
maxval[varstr_dat_offset] = (char)(v_dat - v_hdr);
/* Remember its data length */
// 记住其数据长度
maxval[varstr_dat_length] = copy_size;
}
} else {
/* Init Var-String max length info */
// 初始化变长字符串的最大长度信息
*varstr_maxlen = v_total_len;
/* Remember its pointer */
// 记住其指针
*(Datum*)minval = *(Datum*)maxval = v;
/* Remember its data offset */
// 记住其数据偏移量
minval[varstr_dat_offset] = maxval[varstr_dat_offset] = (char)(v_dat - v_hdr);
/* Remember its data length */
// 记住其数据长度
minval[varstr_dat_length] = maxval[varstr_dat_length] = copy_size;
*first = false;
@ -453,6 +550,25 @@ static void CompareVarStrType(char* minval, char* maxval, Datum v, bool* first,
* @IN v: the new var-length string value
* @See also:
*/
/*
* @Description: set min/max for data-type var-length string
*
*
* @IN/OUT first: the first flag
*
*
* @IN/OUT maxval: the newest max value
*
*
* @IN/OUT minval: the newest min value
*
*
* @IN v: the new var-length string value
*
*
* @See also:
*/
//设置变长字符串类型数据的最小值和最大值
static void SetMinMaxVarStrTypeInternal(char* minval, char* maxval, Datum v, bool* first)
{
char* v_hdr = DatumGetPointer(v);
@ -466,11 +582,13 @@ static void SetMinMaxVarStrTypeInternal(char* minval, char* maxval, Datum v, boo
copySize = Min(v_exchdr_len, (MIN_MAX_LEN - 1));
if (!*first) {
// 第二次及以后的值,更新最小值
Assert(minval[0] < MIN_MAX_LEN && maxval[0] < MIN_MAX_LEN);
int cmpSize = Min(copySize, minval[0]);
int ret = memcmp(v_dat, b + 1, cmpSize);
if (ret < 0 || (ret == 0 && copySize < minval[0])) {
// 更新最小值的数据长度和数据内容
minval[0] = copySize;
rc = memcpy_s(minval + 1, (MIN_MAX_LEN - 1), v_dat, copySize);
securec_check(rc, "\0", "\0");
@ -478,15 +596,17 @@ static void SetMinMaxVarStrTypeInternal(char* minval, char* maxval, Datum v, boo
}
b = maxval;
// 更新最大值
cmpSize = Min(copySize, maxval[0]);
ret = memcmp(v_dat, b + 1, cmpSize);
if (ret > 0 || (ret == 0 && copySize > maxval[0])) {
// 更新最大值的数据长度和数据内容
maxval[0] = copySize;
rc = memcpy_s(maxval + 1, (MIN_MAX_LEN - 1), v_dat, copySize);
securec_check(rc, "\0", "\0");
}
} else {
// 第一个值,初始化最大值和最小值
maxval[0] = minval[0] = copySize;
rc = memcpy_s(maxval + 1, (MIN_MAX_LEN - 1), v_dat, copySize);
securec_check(rc, "\0", "\0");
@ -503,8 +623,10 @@ static void SetMinMaxVarStrTypeInternal(char* minval, char* maxval, Datum v, boo
* @IN v: the new var-length string value
* @See also:
*/
//为变长字符串类型的数据设置最小值和最大值
static void SetMinMaxVarStrType(Datum v, CUDesc* cuDescPtr, bool* first)
{
// 调用内部函数SetMinMaxVarStrTypeInternal设置最小值和最大值
SetMinMaxVarStrTypeInternal(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first);
}
@ -517,16 +639,23 @@ static void SetMinMaxVarStrType(Datum v, CUDesc* cuDescPtr, bool* first)
* @IN/OUT varstr_maxlen: max length about var-length string
* @See also:
*/
//比较日期类型的数据,并更新最小值和最大值
static void CompareDate(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
// 如果不是第一个值
if (!*first) {
DateADT a = DatumGetDateADT(v);
// 比较新值和最小值,更新最小值
if (a < *(DateADT*)(minval)) {
*((DateADT*)(minval)) = a;
} else if (a > *(DateADT*)(maxval)) {
}
// 比较新值和最大值,更新最大值
else if (a > *(DateADT*)(maxval)) {
*((DateADT*)(maxval)) = a;
}
// 如果是第一个值
} else {
// 将最小值和最大值都设置为新值
*((DateADT*)(minval)) = *((DateADT*)(maxval)) = DatumGetDateADT(v);
*first = false;
}
@ -540,8 +669,10 @@ static void CompareDate(char* minval, char* maxval, Datum v, bool* first, int* v
* @IN v: the new date value
* @See also:
*/
//设置日期类型数据的最小值和最大值
static void SetMinMaxDate(Datum v, CUDesc* cuDescPtr, bool* first)
{
// 调用CompareDate函数比较并更新最小值和最大值
CompareDate(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL);
}
@ -554,14 +685,18 @@ static void SetMinMaxDate(Datum v, CUDesc* cuDescPtr, bool* first)
* @IN/OUT varstr_maxlen: max length about var-length string
* @See also:
*/
//比较时间类型数据的值,并根据比较结果更新最小值和最大值
static void CompareTime(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
// 检查是否为第一个值
if (!*first) {
// 将传入的值和当前最小值、最大值进行比较
TimeADT tVal = DatumGetTimeADT(v);
TimeADT curMin = *(TimeADT*)(minval);
TimeADT curMax = *(TimeADT*)(maxval);
#ifdef HAVE_INT64_TIMESTAMP
// 对于64位时间戳的情况如果新值小于当前最小值则更新最小值如果新值大于当前最大值则更新最大值
if (tVal < curMin) {
*((TimeADT*)(minval)) = tVal;
} else if (tVal > curMax) {
@ -573,14 +708,22 @@ static void CompareTime(char* minval, char* maxval, Datum v, bool* first, int* v
* 2. NAN == NAN
* 3. the ordinary comparing rules
*/
/* 时间比较规则:
* 1. NAN > NAN
* 2. NAN == NAN
* 3.
*/
// 对于非64位时间戳的情况如果当前最小值是NaN不是数字或者新值非NaN且小于当前最小值则更新最小值
if (isnan(curMin) || (!isnan(tVal) && (tVal < curMin)))
*((TimeADT*)(minval)) = tVal;
/* 不使用ELSE分支以便分别更新最小值和最大值 */
/* Never use ELSE branch so update min/max value separately */
// 如果新值是NaN或者当前最大值非NaN且新值大于当前最大值则更新最大值
if (isnan(tVal) || (!isnan(curMax) && (tVal > curMax)))
*((TimeADT*)(maxval)) = tVal;
#endif
} else {
// 第一个值的情况下直接将最小值和最大值都设置为传入的值并将first标记设为false
*((TimeADT*)(minval)) = *((TimeADT*)(maxval)) = DatumGetTimeADT(v);
*first = false;
}
@ -594,8 +737,14 @@ static void CompareTime(char* minval, char* maxval, Datum v, bool* first, int* v
* @IN v: the new time value
* @See also:
*/
// 用于为时间类型数据设置最小值和最大值
// 参数:
// - v新的时间值
// - cuDescPtr列存储单元描述符的指针包含了当前最小值和最大值
// - first第一个值的标志输入输出参数
static void SetMinMaxTime(Datum v, CUDesc* cuDescPtr, bool* first)
{
// 调用CompareTime函数比较时间类型数据的值并根据比较结果更新最小值和最大值
CompareTime(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL);
}
@ -608,8 +757,16 @@ static void SetMinMaxTime(Datum v, CUDesc* cuDescPtr, bool* first)
* @IN/OUT varstr_maxlen: max length about var-length string
* @See also: timestamp_cmp_internal()
*/
// 用于比较时间戳类型的数据值,并根据比较结果更新最小值和最大值
// 参数:
// - first第一个值的标志输入输出参数
// - maxval当前最大值输入输出参数
// - minval当前最小值输入输出参数
// - v新的时间戳值
// - varstr_maxlenvarstr类型数据的最大长度未被使用
static void CompareTimestamp(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
// 如果不是第一个值,则比较新值与当前最小值、最大值之间的大小关系,并根据比较结果更新最小值和最大值
if (!*first) {
Timestamp tVal = DatumGetTimestamp(v);
Timestamp curMin = *(Timestamp*)(minval);
@ -622,6 +779,11 @@ static void CompareTimestamp(char* minval, char* maxval, Datum v, bool* first, i
*((Timestamp*)(maxval)) = tVal;
}
#else
/* the rules of comparing are
* 1. NAN > non-NAN
* 2. NAN == NAN
* 3. the ordinary comparing rules
*/
/* the rules of comparing are
* 1. NAN > non-NAN
* 2. NAN == NAN
@ -634,11 +796,11 @@ static void CompareTimestamp(char* minval, char* maxval, Datum v, bool* first, i
if (isnan(tVal) || (!isnan(curMax) && (tVal > curMax)))
*((Timestamp*)(maxval)) = tVal;
#endif
} else {
} else {// 如果是第一个值,则直接将其设置为最小值和最大值
*((Timestamp*)(minval)) = *((Timestamp*)(maxval)) = DatumGetTimestamp(v);
*first = false;
*first = false;// 将第一个值的标志置为false
}
UNUSED_ARG(varstr_maxlen);
UNUSED_ARG(varstr_maxlen);// 未使用的参数,被忽略掉
}
/*
@ -648,8 +810,17 @@ static void CompareTimestamp(char* minval, char* maxval, Datum v, bool* first, i
* @IN v: the new timestamp value
* @See also:
*/
/*
* @Description:
* @IN/OUT cuDescPtr: CU描述符
* @IN/OUT first:
* @IN v:
* @See also:
*/
//处理数据类型为时间戳的值,它接受一个新的时间戳值 v并根据比较结果更新最小值和最大值
static void SetMinMaxTimestamp(Datum v, CUDesc* cuDescPtr, bool* first)
{
// 调用CompareTimestamp函数比较新的时间戳值与当前最小值和最大值根据比较结果更新最小值和最大值
CompareTimestamp(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL);
}
@ -662,14 +833,27 @@ static void SetMinMaxTimestamp(Datum v, CUDesc* cuDescPtr, bool* first)
* @IN/OUT varstr_maxlen: max length about var-length string
* @See also: timestamp_cmp_internal()
*/
/*
* @Description:
* @IN/OUT first:
* @IN/OUT maxval:
* @IN/OUT minval:
* @IN v:
* @IN/OUT varstr_maxlen:
* @See also: timestamp_cmp_internal()
*/
//比较数据类型为带有时区的时间戳的比较函数。它接受一个新的带有时区的时间戳值 v并根据比较结果更新最小值和最大值
static void CompareTimestamptz(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
// 如果不是第一个值
if (!*first) {
// 将传入的参数转换为TimestampTz类型
TimestampTz tVal = DatumGetTimestampTz(v);
TimestampTz curMin = *(TimestampTz*)(minval);
TimestampTz curMax = *(TimestampTz*)(maxval);
#ifdef HAVE_INT64_TIMESTAMP
// 如果是64位时间戳
if (tVal < curMin) {
*((TimestampTz*)(minval)) = tVal;
} else if (tVal > curMax) {
@ -681,14 +865,20 @@ static void CompareTimestamptz(char* minval, char* maxval, Datum v, bool* first,
* 2. NAN == NAN
* 3. the ordinary comparing rules
*/
/* 比较规则如下:
* 1. NAN > NAN
* 2. NAN == NAN
* 3.
*/
if (isnan(curMin) || (!isnan(tVal) && (tVal < curMin)))
*((TimestampTz*)(minval)) = tVal;
// 不要使用ELSE分支以便分别更新最小值和最大值
/* Never use ELSE branch so update min/max value separately */
if (isnan(tVal) || (!isnan(curMax) && (tVal > curMax)))
*((TimestampTz*)(maxval)) = tVal;
#endif
} else {
// 如果是第一个值,则将最小值和最大值都设置为新值
*((TimestampTz*)(minval)) = *((TimestampTz*)(maxval)) = DatumGetTimestampTz(v);
*first = false;
}
@ -702,21 +892,41 @@ static void CompareTimestamptz(char* minval, char* maxval, Datum v, bool* first,
* @IN v: the new timestamp with time zone value
* @See also:
*/
/*
* @Description:
* @IN/OUT cuDescPtr: CU描述符
* @IN/OUT first:
* @IN v:
* @See also: CompareTimestamptz()
*/
static void SetMinMaxTimestamptz(Datum v, CUDesc* cuDescPtr, bool* first)
{
// 调用CompareTimestamptz()函数,比较并更新最小值和最大值
CompareTimestamptz(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL);
}
/*
* @Description: uint64的比较函数
* @IN/OUT minval:
* @IN/OUT maxval:
* @IN v: uint64值
* @IN/OUT first:
* @IN/OUT varstr_maxlen:
* @See also:
*/
static void CompareUint64(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
// 如果不是第一个值
if (!*first) {
uint64 intVal = (uint64)v;
if (intVal < *(uint64*)(minval)) {
// 更新最小值
*(uint64*)(minval) = intVal;
} else if (intVal > *(uint64*)(maxval)) {
// 更新最大值
*(uint64*)(maxval) = intVal;
}
} else {
// 如果是第一个值,则将最小值和最大值都设置为新值
*(uint64*)(minval) = *(uint64*)(maxval) = (uint64)(v);
*first = false;
}
@ -730,8 +940,16 @@ static void CompareUint64(char* minval, char* maxval, Datum v, bool* first, int*
* @IN v: the new uint64 value
* @See also:
*/
/*
* @Description: uint64
* @IN v: uint64
* @IN/OUT cuDescPtr: CU
* @IN/OUT flag:
* @See also:
*/
static void SetMinMaxUint64(Datum v, CUDesc* cuDescPtr, bool* flag)
{
// 调用 CompareUint64 函数,更新最小值和最大值
CompareUint64(cuDescPtr->cu_min, cuDescPtr->cu_max, v, flag, NULL);
}
@ -744,16 +962,29 @@ static void SetMinMaxUint64(Datum v, CUDesc* cuDescPtr, bool* flag)
* @IN/OUT varstr_maxlen: max length about var-length string
* @See also:
*/
/*
* @Description: int64
* @IN/OUT minval:
* @IN/OUT maxval:
* @IN v: int64
* @IN/OUT first:
* @IN/OUT varstr_maxlen:
* @See also:
*/
static void CompareInt64(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
// 如果不是第一个值
if (!*first) {
int64 intVal = DatumGetInt64(v);
if (intVal < *(int64*)(minval)) {
// 更新最小值
*(int64*)(minval) = intVal;
} else if (intVal > *(int64*)(maxval)) {
// 更新最大值
*(int64*)(maxval) = intVal;
}
} else {
// 如果是第一个值,则将最小值和最大值都设置为新值
*(int64*)(minval) = *(int64*)(maxval) = DatumGetInt64(v);
*first = false;
}
@ -767,8 +998,16 @@ static void CompareInt64(char* minval, char* maxval, Datum v, bool* first, int*
* @IN v: the new int64 value
* @See also:
*/
/*
* @Description: int64
* @IN/OUT cuDescPtr: CU
* @IN/OUT first:
* @IN v: int64
* @See also:
*/
static void SetMinMaxInt64(Datum v, CUDesc* cuDescPtr, bool* first)
{
// 调用 CompareInt64 函数,将新的 int64 值与当前的最小值和最大值进行比较和更新
CompareInt64(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL);
}
@ -781,16 +1020,29 @@ static void SetMinMaxInt64(Datum v, CUDesc* cuDescPtr, bool* first)
* @IN/OUT varstr_maxlen: max length about var-length string
* @See also:
*/
/*
* @Description: int32
* @IN/OUT first:
* @IN/OUT maxval:
* @IN/OUT minval:
* @IN v: int32
* @IN/OUT varstr_maxlen:
* @See also:
*/
static void CompareInt32(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
// 如果不是第一个值
if (!*first) {
int32 intVal = DatumGetInt32(v);
if (intVal < *(int32*)(minval)) {
// 更新最小值
*(int32*)(minval) = intVal;
} else if (intVal > *(int32*)(maxval)) {
// 更新最大值
*(int32*)(maxval) = intVal;
}
} else {
// 如果是第一个值,则将最小值和最大值都设置为新值
*(int32*)(minval) = *(int32*)(maxval) = DatumGetInt32(v);
*first = false;
}
@ -804,8 +1056,16 @@ static void CompareInt32(char* minval, char* maxval, Datum v, bool* first, int*
* @IN v: the new int32 value
* @See also:
*/
/*
* @Description: int32
* @IN/OUT cuDescPtr: CU
* @IN/OUT first:
* @IN v: int32
* @See also:
*/
static void SetMinMaxInt32(Datum v, CUDesc* cuDescPtr, bool* first)
{
// 调用 CompareInt32 函数,将新的 int32 值与当前的最小值和最大值进行比较,并更新最小值和最大值
CompareInt32(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL);
}
@ -818,16 +1078,29 @@ static void SetMinMaxInt32(Datum v, CUDesc* cuDescPtr, bool* first)
* @IN/OUT varstr_maxlen: max length about var-length string
* @See also:
*/
/*
* @Description: uint32
* @IN/OUT first:
* @IN/OUT maxval:
* @IN/OUT minval:
* @IN v: uint32
* @IN/OUT varstr_maxlen:
* @See also:
*/
static void CompareUint32(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
// 如果不是第一个值
if (!*first) {
uint32 uintVal = DatumGetUInt32(v);
if (uintVal < *(uint32*)(minval)) {
// 更新最小值
*(uint32*)(minval) = uintVal;
} else if (uintVal > *(uint32*)(maxval)) {
// 更新最大值
*(uint32*)(maxval) = uintVal;
}
} else {
// 如果是第一个值,则将最小值和最大值都设置为新值
*(uint32*)(minval) = *(uint32*)(maxval) = DatumGetUInt32(v);
*first = false;
}
@ -841,8 +1114,16 @@ static void CompareUint32(char* minval, char* maxval, Datum v, bool* first, int*
* @IN v: the new uint32 value
* @See also:
*/
/*
* @Description: uint32
* @IN/OUT cuDescPtr: CU
* @IN/OUT first:
* @IN v: uint32
* @See also:
*/
void SetMinMaxUint32(Datum v, CUDesc* cuDescPtr, bool* first)
{
// 调用 CompareUint32 函数来比较并更新最小值和最大值
CompareUint32(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL);
}
@ -857,14 +1138,18 @@ void SetMinMaxUint32(Datum v, CUDesc* cuDescPtr, bool* first)
*/
static void CompareInt16(char* minval, char* maxval, Datum v, bool* first, int* varstr_maxlen)
{
// 如果不是第一个值
if (!*first) {
int16 intVal = DatumGetInt16(v);
if (intVal < *(int16*)minval) {
// 更新最小值
*(int16*)(minval) = intVal;
} else if (intVal > *(int16*)maxval) {
// 更新最大值
*(int16*)(maxval) = intVal;
}
} else {
// 如果是第一个值,则将最小值和最大值都设置为新值
*(int16*)(minval) = *(int16*)(maxval) = DatumGetInt16(v);
*first = false;
}
@ -878,7 +1163,15 @@ static void CompareInt16(char* minval, char* maxval, Datum v, bool* first, int*
* @IN v: the new int16 value
* @See also:
*/
/*
* @Description: int16
* @IN/OUT cuDescPtr: CU
* @IN/OUT first:
* @IN v: int16
* @See also:
*/
static void SetMinMaxInt16(Datum v, CUDesc* cuDescPtr, bool* first)
{
// 调用 CompareInt16 函数,设置最小值和最大值
CompareInt16(cuDescPtr->cu_min, cuDescPtr->cu_max, v, first, NULL);
}

View File

@ -31,18 +31,18 @@
#include "utils/typcache.h"
CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int type, MemInfoArg* m_memInfo)
: m_tupleSortState(NULL),
m_batchSortState(NULL),
m_vecBatch(NULL),
m_tupleSlot(NULL),
m_type(type),
m_rel(rel),
m_sortKeys(sortKeys),
m_keyNum(keyNum),
m_curSortedRowNum(0),
m_vecBatchCursor(InvalidBathCursor)
: m_tupleSortState(NULL),// 初始化元组排序状态为NULL
m_batchSortState(NULL),// 初始化批次排序状态为NULL
m_vecBatch(NULL),// 初始化向量批次为NULL
m_tupleSlot(NULL),// 初始化元组插槽为NULL
m_type(type),// 初始化排序类型
m_rel(rel),// 初始化关系对象
m_sortKeys(sortKeys),// 初始化排序键数组
m_keyNum(keyNum),// 初始化排序键数量
m_curSortedRowNum(0),// 初始化当前已排序行数为0
m_vecBatchCursor(InvalidBathCursor)// 初始化向量批次游标为无效值
{
m_tupDesc = m_rel->rd_att;
m_tupDesc = m_rel->rd_att;// 获取关系的属性描述符
Form_pg_attribute* attr = m_tupDesc->attrs;
m_psortMemInfo = NULL;
@ -55,6 +55,7 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ
// Note that these variables should be in parent memoryContex.
// please free them in deconstructor method.
//
// 为排序操作符和排序规则分配内存并设置相应的变量
m_sortOperators = (Oid*)palloc(sizeof(Oid) * keyNum);
m_sortCollations = (Oid*)palloc(sizeof(Oid) * keyNum);
for (int i = 0; i < keyNum; ++i) {
@ -64,7 +65,7 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ
TypeCacheEntry* typeEntry = lookup_type_cache(attr[colIdx]->atttypid, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR);
m_sortOperators[i] = typeEntry->lt_opr;
}
// 分配内存并初始化相应的变量
m_nullsFirst = (bool*)palloc(sizeof(bool) * m_keyNum);
errno_t rc = memset_s(m_nullsFirst, m_keyNum * sizeof(bool), false, m_keyNum * sizeof(bool));
securec_check(rc, "", "");
@ -72,7 +73,7 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ
m_val = (Datum*)palloc(sizeof(Datum) * m_tupDesc->natts);
m_null = (bool*)palloc(sizeof(bool) * m_tupDesc->natts);
InitPsortMemArg(m_memInfo);
InitPsortMemArg(m_memInfo);// 初始化排序所需的内存信息
int sortMem = m_psortMemInfo->MemSort > 0 ? m_psortMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem;
int canSpreadmaxMem = m_psortMemInfo->canSpreadmaxMem;
/*
@ -81,6 +82,7 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ
* so for some partitions with large data volume, we can expand by 20% by default.
* In extreme cases, it is possible to exceed 20% per zone for canSpreadmaxMem.
*/
// 根据排序类型创建相应类型的排序状态对象并初始化相应的函数指针
if (canSpreadmaxMem && m_psortMemInfo->partitionNum > 1) {
canSpreadmaxMem = (int)((double)m_psortMemInfo->canSpreadmaxMem / (double)m_psortMemInfo->partitionNum *
PSORT_SPREAD_MAXMEM_RATIO);
@ -95,9 +97,10 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ
// WARNING: all the following variables should be regenerated
// after m_psortMemContext is reset.
//
AutoContextSwitch memContextGuard(m_psortMemContext);
AutoContextSwitch memContextGuard(m_psortMemContext);// 创建自动内存上下文切换器
if (m_type == TUPLE_SORT) {
if (m_type == TUPLE_SORT) {// 如果排序类型为元组排序
// 创建元组排序状态对象,并设置相应的回调函数指针
m_tupleSortState = tuplesort_begin_heap(m_tupDesc,
m_keyNum,
m_sortKeys,
@ -108,12 +111,13 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ
false,
canSpreadmaxMem);
m_funcGetBatchValue = &CStorePSort::GetBatchValueFromTupleSort;
m_funcGetBatchValue = &CStorePSort::GetBatchValueFromTupleSort;// 设置获取批次值的回调函数指针
m_funcReset = &CStorePSort::ResetTupleSortState;
m_funcReset = &CStorePSort::ResetTupleSortState;// 设置重置排序状态的回调函数指针
m_tupleSlot = MakeSingleTupleTableSlot(m_tupDesc);
} else {
m_tupleSlot = MakeSingleTupleTableSlot(m_tupDesc);// 创建单个元组插槽
} else {// 否则,即排序类型为批次排序
// 创建批次排序状态对象,并设置相应的回调函数指针
m_batchSortState = batchsort_begin_heap(m_tupDesc,
m_keyNum,
m_sortKeys,
@ -124,11 +128,11 @@ CStorePSort::CStorePSort(Relation rel, AttrNumber* sortKeys, int keyNum, int typ
false,
canSpreadmaxMem);
m_funcGetBatchValue = &CStorePSort::GetBatchValueFromBatchSort;
m_funcGetBatchValue = &CStorePSort::GetBatchValueFromBatchSort;// 设置获取批次值的回调函数指针
m_funcReset = &CStorePSort::ResetBatchSortState;
m_funcReset = &CStorePSort::ResetBatchSortState;// 设置重置排序状态的回调函数指针
m_vecBatch = New(CurrentMemoryContext) VectorBatch(CurrentMemoryContext, m_tupDesc);
m_vecBatch = New(CurrentMemoryContext) VectorBatch(CurrentMemoryContext, m_tupDesc);// 创建新的向量批次
}
}
@ -149,26 +153,26 @@ CStorePSort::~CStorePSort()
m_sortCollations = NULL;
m_psortMemContext = NULL;
}
//销毁CStorePSort对象释放相关的内存资源
void CStorePSort::Destroy()
{
if (m_type == TUPLE_SORT && m_tupleSlot != NULL) {
ExecDropSingleTupleTableSlot(m_tupleSlot);
ExecDropSingleTupleTableSlot(m_tupleSlot);// 释放单个元组插槽的内存
m_tupleSlot = NULL;
}
// 释放在非 psortMemContext 内分配的内存
// free memory not alloc in m_psortMemContext
pfree_ext(m_sortOperators);
pfree_ext(m_sortCollations);
pfree_ext(m_nullsFirst);
pfree_ext(m_val);
pfree_ext(m_null);
pfree_ext(m_sortOperators);// 释放排序操作符数组的内存
pfree_ext(m_sortCollations);// 释放排序规则数组的内存
pfree_ext(m_nullsFirst);// 释放空值排序顺序数组的内存
pfree_ext(m_val);// 释放值数组的内存
pfree_ext(m_null);// 释放空值标志数组的内存
if (m_psortMemInfo) {
pfree_ext(m_psortMemInfo);
pfree_ext(m_psortMemInfo);// 释放排序内存信息结构体的内存
}
// 释放在 psortMemContext 内分配的内存
// free memory alloc in m_psortMemContext
MemoryContextDelete(m_psortMemContext);
MemoryContextDelete(m_psortMemContext);// 删除排序内存上下文
}
/*
@ -179,8 +183,10 @@ void CStorePSort::Destroy()
* @sort mem for cstore table and partition tables.
* @Return: void
*/
//
void CStorePSort::InitPsortMemArg(MemInfoArg* ArgmemInfo)
{
// 初始化psort的内存信息
m_psortMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg));
if (ArgmemInfo != NULL) {
Assert(ArgmemInfo->partitionNum > 0);
@ -188,6 +194,7 @@ void CStorePSort::InitPsortMemArg(MemInfoArg* ArgmemInfo)
* m_psortMemInfo->canSpreadmaxMem is all partitions sort spread Mem for partition table.
* m_psortMemInfo->MemSort is one partition sort Mem for partition table.
*/
// 计算可扩展的最大内存和分区排序内存
m_psortMemInfo->canSpreadmaxMem = (ArgmemInfo->canSpreadmaxMem - ArgmemInfo->MemInsert > 0) ?
ArgmemInfo->canSpreadmaxMem - ArgmemInfo->MemInsert :
0;
@ -195,12 +202,14 @@ void CStorePSort::InitPsortMemArg(MemInfoArg* ArgmemInfo)
m_psortMemInfo->MemSort = ArgmemInfo->MemSort;
m_psortMemInfo->spreadNum = ArgmemInfo->spreadNum;
m_psortMemInfo->partitionNum = ArgmemInfo->partitionNum;
// 打印初始化的内存信息
MEMCTL_LOG(DEBUG2,
"CStorePSort(init ArgmemInfo):Insert workmem is : %dKB, sort workmem: %dKB,can spread maxMem is %dKB.",
m_psortMemInfo->MemInsert,
m_psortMemInfo->MemSort,
m_psortMemInfo->canSpreadmaxMem);
} else {
// 未传入有效的内存信息参数,使用默认值
m_psortMemInfo->canSpreadmaxMem = 0;
m_psortMemInfo->MemInsert = 0;
m_psortMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem;
@ -208,152 +217,156 @@ void CStorePSort::InitPsortMemArg(MemInfoArg* ArgmemInfo)
m_psortMemInfo->partitionNum = 1;
}
}
//重置排序状态
void CStorePSort::Reset(bool endFlag)
{
return (this->*m_funcReset)(endFlag);
return (this->*m_funcReset)(endFlag);// 通过函数指针调用Reset函数的具体实现传入endFlag参数
}
//将向量批次数据放入排序状态中
void CStorePSort::PutVecBatch(Relation rel, VectorBatch* pVecBatch)
{
Assert(m_batchSortState && m_type == BATCH_SORT);
Assert(m_batchSortState && m_type == BATCH_SORT);// 断言确保批次排序状态不为空且类型为BATCH_SORT
// 调用sort_putbatch函数将向量批次数据放入批次排序状态中
m_batchSortState->sort_putbatch(m_batchSortState, pVecBatch, 0, pVecBatch->m_rows);
// 更新已排序行数
m_curSortedRowNum += pVecBatch->m_rows;
}
// 将元组数据放入排序状态中
void CStorePSort::PutTuple(Datum* values, bool* nulls)
{
Assert(m_tupleSortState && m_type == TUPLE_SORT);
Assert(m_tupleSortState && m_type == TUPLE_SORT);// 断言确保元组排序状态不为空且类型为TUPLE_SORT
AutoContextSwitch memContextGuard(m_psortMemContext);
AutoContextSwitch memContextGuard(m_psortMemContext);// 切换上下文到psort内存上下文
HeapTuple tuple = (HeapTuple)tableam_tops_form_tuple(m_tupDesc, values, nulls, HEAP_TUPLE);
HeapTuple tuple = (HeapTuple)tableam_tops_form_tuple(m_tupDesc, values, nulls, HEAP_TUPLE);// 从元组描述符,值和空值创建堆元组
TupleTableSlot* slot = MakeSingleTupleTableSlot(m_tupDesc);
TupleTableSlot* slot = MakeSingleTupleTableSlot(m_tupDesc);// 创建单个元组表格插槽
(void)ExecStoreTuple(tuple, slot, InvalidBuffer, false);
(void)ExecStoreTuple(tuple, slot, InvalidBuffer, false);// 将堆元组存储到表格插槽中
tuplesort_puttupleslot(m_tupleSortState, slot);
tuplesort_puttupleslot(m_tupleSortState, slot);// 将表格插槽的元组加入元组排序状态中
ExecDropSingleTupleTableSlot(slot);
ExecDropSingleTupleTableSlot(slot);// 释放表格插槽
heap_freetuple(tuple);
heap_freetuple(tuple);// 释放堆元组
tuple = NULL;
}
// 将单个元组数据放入排序状态中,并更新已排序行数
void CStorePSort::PutSingleTuple(Datum* values, bool* nulls)
{
PutTuple(values, nulls);
m_curSortedRowNum++;
PutTuple(values, nulls);// 调用PutTuple将元组数据放入排序状态中
m_curSortedRowNum++;// 更新已排序行数
}
// 将批次数据放入排序状态中
void CStorePSort::PutBatchValues(bulkload_rows* batchRowPtr)
{
AutoContextSwitch memContextGuard(m_psortMemContext);
AutoContextSwitch memContextGuard(m_psortMemContext);// 切换上下文到psort内存上下文
bulkload_rows_iter iter;
iter.begin(batchRowPtr);
iter.begin(batchRowPtr);// 设置迭代器的起点
while (iter.not_end()) {
/* fetch next tuple */
iter.next(m_val, m_null);
iter.next(m_val, m_null);// 从批次中获取下一个元组
/* put tuple into sort processor */
PutTuple(m_val, m_null);
PutTuple(m_val, m_null);// 将元组数据放入排序状态中
}
iter.end();
iter.end();// 设置迭代器的终点
m_curSortedRowNum += batchRowPtr->m_rows_curnum;
m_curSortedRowNum += batchRowPtr->m_rows_curnum;// 更新已排序行数
}
// 执行排序操作
void CStorePSort::RunSort()
{
AutoContextSwitch memContextGuard(m_psortMemContext);
AutoContextSwitch memContextGuard(m_psortMemContext);// 切换上下文到psort内存上下文
if (m_type == TUPLE_SORT)
tuplesort_performsort(m_tupleSortState);
tuplesort_performsort(m_tupleSortState);// 对元组排序状态执行排序操作
else
batchsort_performsort(m_batchSortState);
batchsort_performsort(m_batchSortState);// 对批次排序状态执行排序操作
}
// 获取批次值
void CStorePSort::GetBatchValue(bulkload_rows* batchRowsPtr)
{
return (this->*m_funcGetBatchValue)(batchRowsPtr);
return (this->*m_funcGetBatchValue)(batchRowsPtr);// 调用函数指针指向的函数,获取批次值
}
// 获取向量批次
VectorBatch* CStorePSort::GetVectorBatch()
{
Assert(m_batchSortState && m_vecBatch && m_type == BATCH_SORT);
Assert(m_batchSortState && m_vecBatch && m_type == BATCH_SORT);// 断言:批次排序状态、向量批次和排序类型必须符合条件
if (unlikely(!(m_tupleSortState && m_tupleSlot && m_type == TUPLE_SORT))) {
ereport(ERROR,
(errcode(ERRCODE_INTERNAL_ERROR),
errmsg("Invalid tupleSortState, tupleSlot, or sorting strategy while getting tuple in cstore psort.")));
}
m_vecBatch->Reset(true);
batchsort_getbatch(m_batchSortState, true, m_vecBatch);
return m_vecBatch;
m_vecBatch->Reset(true);// 重置向量批次,清空数据
batchsort_getbatch(m_batchSortState, true, m_vecBatch);// 获取批次排序状态中的批次数据
return m_vecBatch;// 返回向量批次
}
// 获取元组
TupleTableSlot* CStorePSort::GetTuple()
{
Assert(m_tupleSortState && m_tupleSlot && m_type == TUPLE_SORT);
Assert(m_tupleSortState && m_tupleSlot && m_type == TUPLE_SORT);// 断言:元组排序状态、元组插槽和排序类型必须符合条件
if (unlikely(!(m_tupleSortState && m_tupleSlot && m_type == TUPLE_SORT))) {
ereport(ERROR,
(errcode(ERRCODE_INTERNAL_ERROR),
errmsg("Invalid tupleSortState, tupleSlot, or sorting strategy while getting tuple in cstore psort.")));
}
if (!TupIsNull(m_tupleSlot))
if (!TupIsNull(m_tupleSlot))// 如果当前元组插槽不为空,则清空插槽中的元组数据
(void)ExecClearTuple(m_tupleSlot);
(void)tuplesort_gettupleslot(m_tupleSortState, true, m_tupleSlot, NULL);
return m_tupleSlot;
(void)tuplesort_gettupleslot(m_tupleSortState, true, m_tupleSlot, NULL);// 从元组排序状态中获取下一个元组,并存储在元组插槽中
return m_tupleSlot;// 返回元组插槽
}
// 检查排序是否已满
bool CStorePSort::IsFull() const
{
return m_curSortedRowNum >= m_partialClusterRowNum;
return m_curSortedRowNum >= m_partialClusterRowNum;// 如果当前已排序行数大于等于部分簇的行数则认为已满返回true否则返回false
}
// 获取已排序的行数
int CStorePSort::GetRowNum() const
{
return m_curSortedRowNum;
return m_curSortedRowNum;// 返回当前已排序的行数
}
#ifdef USE_ASSERT_CHECKING
// 在使用断言进行检查时,确保部分簇的行数大于等于完整簇的大小,并且部分簇的行数必须是完整簇大小的整数倍
inline void CStorePSort::AssertCheck()
{
Assert((m_partialClusterRowNum >= m_fullCUSize) && ((m_partialClusterRowNum % m_fullCUSize) == 0));
Assert((m_partialClusterRowNum >= m_fullCUSize) && ((m_partialClusterRowNum % m_fullCUSize) == 0));// 断言:部分簇的行数大于等于完整簇的大小,并且部分簇的行数必须是完整簇大小的整数倍
}
#endif
//重置元组排序状态
void CStorePSort::ResetTupleSortState(bool endFlag)
{
AutoContextSwitch memContextGuard(m_psortMemContext);
int sortMem = u_sess->attr.attr_storage.psort_work_mem;
int canSpreadMaxMem = 0;
AutoContextSwitch memContextGuard(m_psortMemContext);// 自动内存上下文切换保护对象
int sortMem = u_sess->attr.attr_storage.psort_work_mem;// 排序使用的内存大小默认为系统设置的psort_work_mem值
int canSpreadMaxMem = 0;// 可以扩展的最大内存大小默认为0
if (m_tupleSortState) {
sortMem = m_psortMemInfo->MemSort > 0 ? m_psortMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem;
canSpreadMaxMem = m_psortMemInfo->canSpreadmaxMem > 0 ? m_psortMemInfo->canSpreadmaxMem : 0;
sortMem = m_psortMemInfo->MemSort > 0 ? m_psortMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem;// 如果设置了排序内存大小则使用设置值否则使用系统设置的psort_work_mem值
canSpreadMaxMem = m_psortMemInfo->canSpreadmaxMem > 0 ? m_psortMemInfo->canSpreadmaxMem : 0;// 如果设置了可以扩展的最大内存大小则使用设置值否则默认为0
if (canSpreadMaxMem && m_psortMemInfo->partitionNum > 1) {
if (canSpreadMaxMem && m_psortMemInfo->partitionNum > 1) {// 如果可以扩展的最大内存大小大于0并且分区数大于1
canSpreadMaxMem = (int)((double)m_psortMemInfo->canSpreadmaxMem / (double)m_psortMemInfo->partitionNum *
PSORT_SPREAD_MAXMEM_RATIO);
PSORT_SPREAD_MAXMEM_RATIO);// 根据分区数比例计算每个分区可以扩展的最大内存大小
}
tuplesort_end(m_tupleSortState);
tuplesort_end(m_tupleSortState);// 结束元组排序
m_tupleSortState = NULL;
}
// 清除 m_tupleSlot
// clear m_tupleSlot
if (m_tupleSlot != NULL) {
ExecDropSingleTupleTableSlot(m_tupleSlot);
ExecDropSingleTupleTableSlot(m_tupleSlot);// 释放单个元组表插槽
m_tupleSlot = NULL;
}
// clear memory contex
MemoryContextReset(m_psortMemContext);
// 清除内存上下文
MemoryContextReset(m_psortMemContext);// 重置内存上下文
if (!endFlag) {
m_tupleSortState = tuplesort_begin_heap(m_tupDesc,
if (!endFlag) {// 如果不是结束标志
m_tupleSortState = tuplesort_begin_heap(m_tupDesc,// 开始堆排序
m_keyNum,
m_sortKeys,
m_sortOperators,
@ -363,32 +376,35 @@ void CStorePSort::ResetTupleSortState(bool endFlag)
false,
canSpreadMaxMem);
m_tupleSlot = MakeSingleTupleTableSlot(m_tupDesc);
m_tupleSlot = MakeSingleTupleTableSlot(m_tupDesc);// 创建单个元组表插槽
}
m_curSortedRowNum = 0;
m_curSortedRowNum = 0;// 当前已排序的行数归零
}
//重置批量排序状态
void CStorePSort::ResetBatchSortState(bool endFlag)
{
AutoContextSwitch memContextGuard(m_psortMemContext);
int sortMem = u_sess->attr.attr_storage.psort_work_mem;
int canSpreadMaxMem = 0;
AutoContextSwitch memContextGuard(m_psortMemContext);// 切换到排序过程所使用的内存上下文。
int sortMem = u_sess->attr.attr_storage.psort_work_mem;// 获取排序所使用的内存大小。
int canSpreadMaxMem = 0;// 可扩展的最大内存大小先设为0。
if (m_batchSortState) {
if (m_batchSortState) {// 如果已存在批量排序状态(m_batchSortState不为NULL)。
// 根据情况确定排序使用的内存大小和可扩展的最大内存大小。
sortMem = m_psortMemInfo->MemSort > 0 ? m_psortMemInfo->MemSort : u_sess->attr.attr_storage.psort_work_mem;
// 判断是否有可扩展的最大内存并且分区数大于1则设定可扩展的最大内存大小。
canSpreadMaxMem = m_psortMemInfo->canSpreadmaxMem > 0 ? m_psortMemInfo->canSpreadmaxMem : 0;
if (canSpreadMaxMem && m_psortMemInfo->partitionNum > 1) {
canSpreadMaxMem = (int)((double)m_psortMemInfo->canSpreadmaxMem / (double)m_psortMemInfo->partitionNum *
PSORT_SPREAD_MAXMEM_RATIO);
}
batchsort_end(m_batchSortState);
m_batchSortState = NULL;
batchsort_end(m_batchSortState);// 结束排序。
m_batchSortState = NULL;// 将批量排序状态置为NULL。
}
// clear memory contex
MemoryContextReset(m_psortMemContext);
MemoryContextReset(m_psortMemContext);// 重置排序过程所使用的内存上下文。
if (!endFlag) {
if (!endFlag) {// 如果不是结束排序(endFlag为false)。
// 通过batchsort_begin_heap函数开始堆排序设定元组描述符、键值个数、排序键、排序算子、分区方法、是否nullsFirst、排序内存大小、是否并行排序、可扩展的最大内存大小。
m_batchSortState = batchsort_begin_heap(m_tupDesc,
m_keyNum,
m_sortKeys,
@ -398,49 +414,52 @@ void CStorePSort::ResetBatchSortState(bool endFlag)
sortMem,
false,
canSpreadMaxMem);
// 创建与元组描述符对应的空批次(m_vecBatch)。
m_vecBatch = New(CurrentMemoryContext) VectorBatch(CurrentMemoryContext, m_tupDesc);
}
m_curSortedRowNum = 0;
m_vecBatchCursor = InvalidBathCursor;
m_curSortedRowNum = 0;// 将当前已排序的行数(m_curSortedRowNum)归零。
m_vecBatchCursor = InvalidBathCursor;// 将批次游标指向无效位置。
}
//在外部排序中从元组排序中逐个读取批次的元组数据并将其添加到bulkload_rows类实例中。循环直到读取完所有元组或者批次数据已满
void CStorePSort::GetBatchValueFromTupleSort(bulkload_rows* batchRowsPtr)
{
AutoContextSwitch memContextGuard(m_psortMemContext);
TupleTableSlot* slot = MakeSingleTupleTableSlot(m_tupDesc);
AutoContextSwitch memContextGuard(m_psortMemContext);// 切换到排序过程所使用的内存上下文。
TupleTableSlot* slot = MakeSingleTupleTableSlot(m_tupDesc);// 创建一个单个元组的TupleTableSlot。
// here it isn't a dead loop.
// when batchRowsPtr is full, break down the loops.
while (true) {
if (!tuplesort_gettupleslot(m_tupleSortState, true, slot, NULL)) {
break;
while (true) {// 循环读取批次数据。
if (!tuplesort_gettupleslot(m_tupleSortState, true, slot, NULL)) {// 从元组排序中获取下一个元组至slot中。
break;// 如果获取不到元组,则退出循环。
}
heap_deform_tuple((HeapTuple)slot->tts_tuple, m_tupDesc, m_val, m_null);
if (batchRowsPtr->append_one_tuple(m_val, m_null, m_tupDesc))
break;
heap_deform_tuple((HeapTuple)slot->tts_tuple, m_tupDesc, m_val, m_null);// 将元组解压为属性值和NULL标记。
if (batchRowsPtr->append_one_tuple(m_val, m_null, m_tupDesc))// 将属性值和NULL标记添加到bulkload_rows中。
break;// 如果添加完成,则退出循环。
}
ExecDropSingleTupleTableSlot(slot);
ExecDropSingleTupleTableSlot(slot);// 释放TupleTableSlot的资源。
}
//从批量排序中逐个读取批次的元组数据并将其添加到bulkload_rows类实例中。循环直到读取完所有元组或者批次数据已满
void CStorePSort::GetBatchValueFromBatchSort(bulkload_rows* batchRowsPtr)
{
AutoContextSwitch memContextGuard(m_psortMemContext);
AutoContextSwitch memContextGuard(m_psortMemContext);// 切换到排序过程所使用的内存上下文。
// precondition for the following WHILE loop
//
Assert(batchRowsPtr->m_rows_maxnum > 0);
Assert(batchRowsPtr->m_rows_maxnum % BatchMaxSize == 0);
bool is_start = true;
Assert(batchRowsPtr->m_rows_maxnum > 0);// 确保批次中的行数大于0。
Assert(batchRowsPtr->m_rows_maxnum % BatchMaxSize == 0);// 确保批次的行数是BatchMaxSize的整数倍。
bool is_start = true;// 标记是否是第一次获取批次数据。
if (BathCursorIsValid(m_vecBatchCursor)) {
// first fetch tuples from last vector batch.
// 从上次处理的向量批次中获取元组。
if (batchRowsPtr->append_one_vector(m_tupDesc, m_vecBatch, &m_vecBatchCursor)) {
// it's filled with part of VectBatch tuples.
// m_vecBatchCursor has been updated, and we have to
// care the special case. reset when all the m_vecBatch is handled.
// 如果批次已满,则部分向量批次已填充。
// 更新m_vecBatchCursor需要注意特殊情况下的重置操作。
if (m_vecBatchCursor == m_vecBatch->m_rows) {
m_vecBatchCursor = InvalidBathCursor;
}
@ -449,33 +468,40 @@ void CStorePSort::GetBatchValueFromBatchSort(bulkload_rows* batchRowsPtr)
// it's fine to continue to hold more tuples.
// first of all reset this cursor.
// 继续获取更多元组。
// 首先重置游标。
m_vecBatchCursor = InvalidBathCursor;
is_start = false;
}
// here it isn't a dead loop.
// when batchRowsPtr is full, break down the loops.
// 进入循环直至批次数据填满或无可用元组。
while (true) {
Assert(m_vecBatchCursor == InvalidBathCursor);
// 从批量排序中获取向量批次。
batchsort_getbatch(m_batchSortState, true, m_vecBatch);
if (BatchIsNull(m_vecBatch)) {
// 如果没有可用的元组,则跳出循环。
// break if there isn't any tuple
break;
}
if (is_start) {
Assert(batchRowsPtr->m_rows_curnum % BatchMaxSize == 0);
Assert(batchRowsPtr->m_rows_curnum % BatchMaxSize == 0);// 确保当前批次的行数是BatchMaxSize的整数倍。
}
// we have to copy datum, because it belongs to
// m_vecBatch which is reset by batchsort_getbatch().
// free all the space by calling Reset(), so see its references.
//
m_vecBatchCursor = 0;
m_vecBatchCursor = 0;// 将向量批次游标重置为0。
if (batchRowsPtr->append_one_vector(m_tupDesc, m_vecBatch, &m_vecBatchCursor)) {
// ok, it's filled with part of VectBatch tuples.
// m_vecBatchCursor has been updated, and we have to
// care the special case. reset when all the m_vecBatch is handled.
// 批次已满,部分向量批次已填充。
// 更新m_vecBatchCursor需要注意特殊情况下的重置操作。
if (m_vecBatchCursor == m_vecBatch->m_rows) {
m_vecBatchCursor = InvalidBathCursor;
}

View File

@ -52,12 +52,13 @@ template <class T, int strategy>
bool RoughCheckIntCU(const T& min, const T& max, const T& arg);
template <class T, int strategy>
bool RoughCheckFloatCU(const T& min, const T& max, const T& arg);
//根据给定的typeOid数据类型OID、strategy比较策略和collation排序规则OID返回对应的RoughCheckFunc函数指针
RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation)
{
switch (typeOid) {
case CHAROID: {
switch (strategy) {
// 在CHAR类型下根据不同的比较策略返回不同的RoughCheckCharCU函数指针
case CStoreLessStrategyNumber:
return RoughCheckCharCU<CStoreLessStrategyNumber>;
case CStoreLessEqualStrategyNumber:
@ -75,6 +76,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation)
}
case INT2OID: {
switch (strategy) {
// 在INT2类型下根据不同的比较策略返回不同的RoughCheckInt16CU函数指针
case CStoreLessStrategyNumber:
return RoughCheckInt16CU<CStoreLessStrategyNumber>;
case CStoreLessEqualStrategyNumber:
@ -92,6 +94,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation)
}
case INT4OID: {
switch (strategy) {
// 在INT4类型下根据不同的比较策略返回不同的RoughCheckInt32CU函数指针
case CStoreLessStrategyNumber:
return RoughCheckInt32CU<CStoreLessStrategyNumber>;
case CStoreLessEqualStrategyNumber:
@ -109,6 +112,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation)
}
case INT8OID: {
switch (strategy) {
// 在INT8类型下根据不同的比较策略返回不同的RoughCheckInt64CU函数指针
case CStoreLessStrategyNumber:
return RoughCheckInt64CU<CStoreLessStrategyNumber>;
case CStoreLessEqualStrategyNumber:
@ -126,6 +130,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation)
}
case OIDOID: {
switch (strategy) {
// 在OID类型下根据不同的比较策略返回不同的RoughCheckUInt32CU函数指针
case CStoreLessStrategyNumber:
return RoughCheckUInt32CU<CStoreLessStrategyNumber>;
case CStoreLessEqualStrategyNumber:
@ -143,6 +148,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation)
}
case DATEOID: {
switch (strategy) {
// 在DATE类型下根据不同的比较策略返回不同的RoughCheckDateCU函数指针
case CStoreLessStrategyNumber:
return RoughCheckDateCU<CStoreLessStrategyNumber>;
case CStoreLessEqualStrategyNumber:
@ -160,6 +166,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation)
}
case TIMEOID: {
switch (strategy) {
// 在TIME类型下根据不同的比较策略返回不同的RoughCheckTimeCU函数指针
case CStoreLessStrategyNumber:
return RoughCheckTimeCU<CStoreLessStrategyNumber>;
case CStoreLessEqualStrategyNumber:
@ -177,6 +184,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation)
}
case TIMESTAMPOID: {
switch (strategy) {
// 在TIMESTAMP类型下根据不同的比较策略返回不同的RoughCheckTimestampCU函数指针
case CStoreLessStrategyNumber:
return RoughCheckTimestampCU<CStoreLessStrategyNumber>;
case CStoreLessEqualStrategyNumber:
@ -194,6 +202,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation)
}
case TIMESTAMPTZOID: {
switch (strategy) {
// 在TIMESTAMPTZ类型下根据不同的比较策略返回不同的RoughCheckTimestampTzCU函数指针
case CStoreLessStrategyNumber:
return RoughCheckTimestampTzCU<CStoreLessStrategyNumber>;
case CStoreLessEqualStrategyNumber:
@ -216,6 +225,7 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation)
return RoughCheckAllThrough;
switch (strategy) {
// 在BPCHAR、VARCHAR和TEXT类型下根据不同的比较策略返回不同的RoughCheckStringCU函数指针
case CStoreLessStrategyNumber:
return RoughCheckStringCU<CStoreLessStrategyNumber>;
case CStoreLessEqualStrategyNumber:
@ -236,180 +246,184 @@ RoughCheckFunc GetRoughCheckFunc(Oid typeOid, int strategy, Oid collation)
}
}
}
//对char类型的参数进行粗略检查
template <int strategy>
bool RoughCheckCharCU(CUDesc* cudesc, Datum arg)
{
char min = *(char*)cudesc->cu_min;
char max = *(char*)cudesc->cu_max;
char min = *(char*)cudesc->cu_min;// 获取CUDesc结构体中的最小值
char max = *(char*)cudesc->cu_max;// 获取CUDesc结构体中的最大值
return RoughCheckIntCU<char, strategy>(min, max, DatumGetChar(arg));
return RoughCheckIntCU<char, strategy>(min, max, DatumGetChar(arg));// 调用RoughCheckIntCU函数对char类型的值进行粗略检查
}
//对int16类型的参数进行粗略检查
template <int strategy>
bool RoughCheckInt16CU(CUDesc* cudesc, Datum arg)
{
int16 min = *(int16*)cudesc->cu_min;
int16 max = *(int16*)cudesc->cu_max;
int16 min = *(int16*)cudesc->cu_min;// 获取CUDesc结构体中的最小值
int16 max = *(int16*)cudesc->cu_max;// 获取CUDesc结构体中的最大值
return RoughCheckIntCU<int64, strategy>(min, max, DatumGetInt64(arg));
return RoughCheckIntCU<int64, strategy>(min, max, DatumGetInt64(arg));// 调用RoughCheckIntCU函数对int16类型的值进行粗略检查
}
//对int32类型的参数进行粗略检查
template <int strategy>
bool RoughCheckInt32CU(CUDesc* cudesc, Datum arg)
{
int32 min = *(int32*)cudesc->cu_min;
int32 max = *(int32*)cudesc->cu_max;
int32 min = *(int32*)cudesc->cu_min;// 获取CUDesc结构体中的最小值
int32 max = *(int32*)cudesc->cu_max;// 获取CUDesc结构体中的最大值
return RoughCheckIntCU<int64, strategy>(min, max, DatumGetInt64(arg));
return RoughCheckIntCU<int64, strategy>(min, max, DatumGetInt64(arg));// 调用RoughCheckIntCU函数对int32类型的值进行粗略检查
}
//对int64类型的参数进行粗略检查
template <int strategy>
bool RoughCheckInt64CU(CUDesc* cudesc, Datum arg)
{
int64 min = *(int64*)cudesc->cu_min;
int64 max = *(int64*)cudesc->cu_max;
int64 min = *(int64*)cudesc->cu_min;// 获取CUDesc结构体中的最小值
int64 max = *(int64*)cudesc->cu_max;// 获取CUDesc结构体中的最大值
return RoughCheckIntCU<int64, strategy>(min, max, DatumGetInt64(arg));
return RoughCheckIntCU<int64, strategy>(min, max, DatumGetInt64(arg));// 调用RoughCheckIntCU函数对int64类型的值进行粗略检查
}
//对uint32类型的参数进行粗略检查
template <int strategy>
bool RoughCheckUInt32CU(CUDesc* cudesc, Datum arg)
{
uint32 min = *(uint32*)cudesc->cu_min;
uint32 max = *(uint32*)cudesc->cu_max;
uint32 min = *(uint32*)cudesc->cu_min;// 获取CUDesc结构体中的最小值
uint32 max = *(uint32*)cudesc->cu_max;// 获取CUDesc结构体中的最大值
return RoughCheckIntCU<int64, strategy>(min, max, DatumGetInt64(arg));
return RoughCheckIntCU<int64, strategy>(min, max, DatumGetInt64(arg));// 调用RoughCheckIntCU函数对uint32类型的值进行粗略检查
}
}
//对DateADT类型日期类型的参数进行粗略检查
template <int strategy>
bool RoughCheckDateCU(CUDesc* cudesc, Datum arg)
{
DateADT min = *(DateADT*)cudesc->cu_min;
DateADT max = *(DateADT*)cudesc->cu_max;
DateADT min = *(DateADT*)cudesc->cu_min;// 获取CUDesc结构体中的最小值
DateADT max = *(DateADT*)cudesc->cu_max;// 获取CUDesc结构体中的最大值
return RoughCheckIntCU<DateADT, strategy>(min, max, DatumGetDateADT(arg));
return RoughCheckIntCU<DateADT, strategy>(
min, max, DatumGetDateADT(arg)); // 调用RoughCheckIntCU函数对DateADT类型的值进行粗略检查
}
//对TimeADT类型时间类型的参数进行粗略检查
template <int strategy>
bool RoughCheckTimeCU(CUDesc* cudesc, Datum arg)
{
TimeADT min = *(TimeADT*)cudesc->cu_min;
TimeADT max = *(TimeADT*)cudesc->cu_max;
TimeADT min = *(TimeADT*)cudesc->cu_min;// 获取CUDesc结构体中的最小值
TimeADT max = *(TimeADT*)cudesc->cu_max;// 获取CUDesc结构体中的最大值
#ifdef HAVE_INT64_TIMESTAMP
return RoughCheckIntCU<TimeADT, strategy>(min, max, DatumGetTimeADT(arg));
return RoughCheckIntCU<TimeADT, strategy>(
min, max, DatumGetTimeADT(arg)); // 调用RoughCheckIntCU函数对TimeADT类型的值进行粗略检查
#else
// do not support for RoughCheck
return true;
#endif
}
//对Timestamp类型时间戳类型的参数进行粗略检查
template <int strategy>
bool RoughCheckTimestampCU(CUDesc* cudesc, Datum arg)
{
Timestamp min = *(Timestamp*)cudesc->cu_min;
Timestamp max = *(Timestamp*)cudesc->cu_max;
Timestamp min = *(Timestamp*)cudesc->cu_min;// 获取CUDesc结构体中的最小值
Timestamp max = *(Timestamp*)cudesc->cu_max;// 获取CUDesc结构体中的最大值
#ifdef HAVE_INT64_TIMESTAMP
return RoughCheckIntCU<Timestamp, strategy>(min, max, DatumGetTimestamp(arg));
return RoughCheckIntCU<Timestamp, strategy>(min, max, DatumGetTimestamp(arg));// 调用RoughCheckIntCU函数对Timestamp类型的值进行粗略检查
#else
// do not support for RoughCheck
return true;
#endif
}
//对带时区的TimestampTz类型带时区的时间戳类型的参数进行粗略检查
template <int strategy>
bool RoughCheckTimestampTzCU(CUDesc* cudesc, Datum arg)
{
TimestampTz min = *(TimestampTz*)cudesc->cu_min;
TimestampTz max = *(TimestampTz*)cudesc->cu_max;
TimestampTz min = *(TimestampTz*)cudesc->cu_min;// 获取CUDesc结构体中的最小值
TimestampTz max = *(TimestampTz*)cudesc->cu_max;// 获取CUDesc结构体中的最大值
#ifdef HAVE_INT64_TIMESTAMP
return RoughCheckIntCU<TimestampTz, strategy>(min, max, DatumGetTimestampTz(arg));
return RoughCheckIntCU<TimestampTz, strategy>(min, max, DatumGetTimestampTz(arg));// 调用RoughCheckIntCU函数对TimestampTz类型的值进行粗略检查
#else
// do not support for RoughCheck
return true;
#endif
}
//对整型参数进行粗略检查。根据传入的策略strategy值和最小值min、最大值max它会判断参数arg是否满足策略要求
template <class T, int strategy>
bool RoughCheckIntCU(const T& min, const T& max, const T& arg)
{
bool hitCU = true;
bool hitCU = true;// 初始化命中标志为true表示参数满足策略要求
if (strategy == CStoreLessStrategyNumber) {
if (min >= arg) {
hitCU = false;
if (strategy == CStoreLessStrategyNumber) {// 如果策略是小于号策略
if (min >= arg) {// 如果最小值大于等于参数值
hitCU = false;// 设置命中标志为false表示参数不满足策略要求
}
} else if (strategy == CStoreLessEqualStrategyNumber) {
if (min > arg) {
hitCU = false;
} else if (strategy == CStoreLessEqualStrategyNumber) {// 如果策略是小于等于号策略
if (min > arg) {// 如果最小值大于参数值
hitCU = false;// 设置命中标志为false表示参数不满足策略要求
}
} else if (strategy == CStoreEqualStrategyNumber) {
if (arg < min || arg > max) {
hitCU = false;
} else if (strategy == CStoreEqualStrategyNumber) {// 如果策略是等于号策略
if (arg < min || arg > max) {// 如果参数值小于最小值或者大于最大值
hitCU = false;// 设置命中标志为false表示参数不满足策略要求
}
} else if (strategy == CStoreGreaterEqualStrategyNumber) {
if (max < arg) {
hitCU = false;
} else if (strategy == CStoreGreaterEqualStrategyNumber) {// 如果策略是大于等于号策略
if (max < arg) {// 如果最大值小于参数值
hitCU = false;// 设置命中标志为false表示参数不满足策略要求
}
} else if (strategy == CStoreGreaterStrategyNumber) {
if (max <= arg) {
hitCU = false;
} else if (strategy == CStoreGreaterStrategyNumber) {// 如果策略是大于号策略
if (max <= arg) {// 如果最大值小于等于参数值
hitCU = false;// 设置命中标志为false表示参数不满足策略要求
}
}
return hitCU;
return hitCU;// 返回命中标志,表示参数是否满足策略要求
}
//根据传入的策略strategy值和最小值cudesc->cu_min、最大值cudesc->cu_max以及参数值arg它会判断参数是否满足策略要求
template <int strategy>
bool RoughCheckStringCU(CUDesc* cudesc, Datum arg)
{
bool hitCU = true;
bool hitCU = true;// 初始化命中标志为true表示参数满足策略要求
char* min = cudesc->cu_min;
char* max = cudesc->cu_max;
char* min = cudesc->cu_min;// 获取最小值字符串指针
char* max = cudesc->cu_max;// 获取最大值字符串指针
char* argStr = VARDATA_ANY(DatumGetPointer(arg));
int argLen = VARSIZE_ANY_EXHDR(arg);
char* argStr = VARDATA_ANY(DatumGetPointer(arg));// 获取参数字符串指针
int argLen = VARSIZE_ANY_EXHDR(arg);// 获取参数字符串长度
int minLen = *(char*)min;
int maxLen = *(char*)max;
int minLen = *(char*)min;// 最小值字符串的长度
int maxLen = *(char*)max;// 最大值字符串的长度
char* minStr = min + 1;
char* maxStr = max + 1;
char* minStr = min + 1;// 最小值字符串的起始地址
char* maxStr = max + 1;// 最大值字符串的起始地址
int cmpMinLen = minLen > argLen ? argLen : minLen;
int cmpMaxLen = maxLen > argLen ? argLen : maxLen;
int cmpMinLen = minLen > argLen ? argLen : minLen;// 比较最小长度
int cmpMaxLen = maxLen > argLen ? argLen : maxLen;// 比较最大长度
switch (strategy) {
case CStoreLessStrategyNumber:
case CStoreLessEqualStrategyNumber: {
if (memcmp(minStr, argStr, cmpMinLen) > 0)
hitCU = false;
case CStoreLessEqualStrategyNumber: {// 如果策略是小于号或者小于等于号
if (memcmp(minStr, argStr, cmpMinLen) > 0)// 如果最小字符串与参数字符串的前cmpMinLen个字符比较结果大于0
hitCU = false;// 设置命中标志为false表示参数不满足策略要求
} break;
case CStoreEqualStrategyNumber: {
if (memcmp(minStr, argStr, cmpMinLen) > 0 || memcmp(maxStr, argStr, cmpMaxLen) < 0)
hitCU = false;
case CStoreEqualStrategyNumber: {// 如果策略是等于号
if (memcmp(minStr, argStr, cmpMinLen) > 0 || memcmp(maxStr, argStr, cmpMaxLen) < 0)// 如果最小字符串与参数字符串的前cmpMinLen个字符比较结果大于0或者最大字符串与参数字符串的前cmpMaxLen个字符比较结果小于0
hitCU = false;// 设置命中标志为false表示参数不满足策略要求
} break;
case CStoreGreaterEqualStrategyNumber:
case CStoreGreaterStrategyNumber: {
if (memcmp(maxStr, argStr, cmpMaxLen) < 0)
hitCU = false;
case CStoreGreaterStrategyNumber: {// 如果策略是大于等于号或者大于号
if (memcmp(maxStr, argStr, cmpMaxLen) < 0)// 如果最大字符串与参数字符串的前cmpMaxLen个字符比较结果小于0
hitCU = false;// 设置命中标志为false表示参数不满足策略要求
} break;
default:
break;
}
return hitCU;
return hitCU;// 返回命中标志,表示参数是否满足策略要求
}
/*
* In some unsupport type or invaild strategy number, make the CU through the rough check
*/
//在遇到不支持的数据类型或无效的策略号时通过粗略检查来生成计算单位CU
bool RoughCheckAllThrough(CUDesc* cudesc, Datum arg)
{
return true;

View File

@ -34,37 +34,38 @@ extern void ExecVecConstraints(ResultRelInfo* resultRelInfo, VectorBatch* batch,
CStoreUpdate::CStoreUpdate(_in_ Relation rel, _in_ EState* estate, _in_ Plan* plan) : m_estate(estate)
{
// 确保estate不为空
Assert(estate);
m_relation = rel;
m_resultRelInfo = estate->es_result_relation_info;
m_isPartition = RELATION_IS_PARTITIONED(rel);
m_delMemInfo = NULL;
m_insMemInfo = NULL;
m_relation = rel;// 初始化表关系
m_resultRelInfo = estate->es_result_relation_info;// 初始化结果关系信息
m_isPartition = RELATION_IS_PARTITIONED(rel);// 判断表是否分区
m_delMemInfo = NULL;// 初始化删除内存信息
m_insMemInfo = NULL;// 初始化插入内存信息
/* init memory, memory info will be used to init delete and insert. */
InitUpdateMemArg(plan);
InitUpdateMemArg(plan);// 初始化更新的内存参数
/* init delete */
m_delete = New(CurrentMemoryContext) CStoreDelete(rel, estate, true, NULL, m_delMemInfo);
m_delete->setReportErrorForUpdate(true);
m_delete = New(CurrentMemoryContext) CStoreDelete(rel, estate, true, NULL, m_delMemInfo);// 初始化删除操作
m_delete->setReportErrorForUpdate(true);// 设置报告更新错误
/* init insert */
if (!m_isPartition) {
CStoreInsert::InitInsertArg(rel, m_resultRelInfo, true, m_insert_args);
m_insert_args.sortType = BATCH_SORT;
if (!m_isPartition) {// 如果表不是分区表
CStoreInsert::InitInsertArg(rel, m_resultRelInfo, true, m_insert_args);// 初始化插入操作参数
m_insert_args.sortType = BATCH_SORT; // 设置插入操作的排序类型为批量排序
m_insert = New(CurrentMemoryContext) CStoreInsert(rel, m_insert_args, true, NULL, m_insMemInfo);
m_partionInsert = NULL;
} else {
m_insert = New(CurrentMemoryContext) CStoreInsert(rel, m_insert_args, true, NULL, m_insMemInfo);// 初始化插入操作
m_partionInsert = NULL;// 分区插入操作为空
} else {// 如果表是分区表
m_partionInsert =
New(CurrentMemoryContext) CStorePartitionInsert(rel, m_resultRelInfo, TUPLE_SORT, true, NULL, m_insMemInfo);
New(CurrentMemoryContext) CStorePartitionInsert(rel, m_resultRelInfo, TUPLE_SORT, true, NULL, m_insMemInfo);// 初始化分区插入操作,排序类型为元组排序
// 在切换分区时使用闪存缓存的数据更新
/* update using flash cached data when switch partition when insert data */
m_partionInsert->SetPartitionCacheStrategy(FLASH_WHEN_SWICH_PARTITION);
m_insert = NULL;
m_insert = NULL;// 插入操作为空
}
m_hasUniqueIdx = CheckHasUniqueIdx();
m_hasUniqueIdx = CheckHasUniqueIdx();// 检查是否有唯一索引
}
CStoreUpdate::~CStoreUpdate()
@ -82,24 +83,24 @@ CStoreUpdate::~CStoreUpdate()
void CStoreUpdate::Destroy()
{
if (m_delete) {
DELETE_EX(m_delete);
DELETE_EX(m_delete);// 如果删除操作存在,则删除。
}
if (m_insert) {
DELETE_EX(m_insert);
CStoreInsert::DeInitInsertArg(m_insert_args);
DELETE_EX(m_insert);// 如果插入操作存在,则删除。
CStoreInsert::DeInitInsertArg(m_insert_args);// 释放插入操作参数的空间。
}
if (m_partionInsert) {
DELETE_EX(m_partionInsert);
DELETE_EX(m_partionInsert);// 如果分区插入操作存在,则删除。
}
if (m_delMemInfo) {
pfree_ext(m_delMemInfo);
pfree_ext(m_delMemInfo);// 释放删除内存信息的空间。
}
if (m_insMemInfo) {
pfree_ext(m_insMemInfo);
pfree_ext(m_insMemInfo);// 释放插入内存信息的空间。
}
}
@ -111,36 +112,37 @@ void CStoreUpdate::Destroy()
* @Return: void
* @See also: InitInsertMemArg
*/
//初始化CStoreUpdate类的内存参数
void CStoreUpdate::InitUpdateMemArg(Plan* plan)
{
int maxbatchRows = RelationGetMaxBatchRows(m_relation);
int partialClusterRows = RelationGetPartialClusterRows(m_relation);
int partitionNum = 1;
List* partitionList = NIL;
int maxbatchRows = RelationGetMaxBatchRows(m_relation);// 获取最大批量行数
int partialClusterRows = RelationGetPartialClusterRows(m_relation);// 获取部分聚集行数
int partitionNum = 1;// 分区数默认为1
List* partitionList = NIL;// 分区列表
m_delMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg));
m_insMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg));
m_delMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg));// 初始化删除内存参数
m_insMemInfo = (MemInfoArg*)palloc0(sizeof(struct MemInfoArg));// 初始化插入内存参数
if (m_isPartition) {
partitionList = relationGetPartitionList(m_relation, AccessShareLock);
partitionList = relationGetPartitionList(m_relation, AccessShareLock);// 获取分区列表
/* get partition number */
partitionNum = list_length(partitionList);
releasePartitionList(m_relation, &partitionList, NoLock);
partitionNum = list_length(partitionList);// 计算分区数
releasePartitionList(m_relation, &partitionList, NoLock); // 释放分区列表
}
// 初始化删除内存参数和插入内存参数的值
/* init mem = delete mem + insert mem. */
if (plan != NULL && plan->operatorMemKB[0] > 0) {
m_delMemInfo->canSpreadmaxMem = plan->operatorMaxMem;
m_delMemInfo->MemInsert = 0;
m_delMemInfo->spreadNum = 0;
m_delMemInfo->partitionNum = 1;
m_delMemInfo->canSpreadmaxMem = plan->operatorMaxMem;// 可扩展的最大内存
m_delMemInfo->MemInsert = 0;// 删除操作的插入内存
m_delMemInfo->spreadNum = 0;// 扩展次数
m_delMemInfo->partitionNum = 1;// 分区数默认为1
if (m_isPartition) {
/* for partition insert. */
m_insMemInfo->canSpreadmaxMem = plan->operatorMaxMem;
m_insMemInfo->MemInsert = plan->operatorMemKB[0] * 1 / 3;
m_insMemInfo->MemSort = plan->operatorMemKB[0] - m_insMemInfo->MemInsert;
m_insMemInfo->spreadNum = 0;
m_insMemInfo->partitionNum = partitionNum;
m_delMemInfo->partitionNum = partitionNum;
m_insMemInfo->canSpreadmaxMem = plan->operatorMaxMem;// 可扩展的最大内存
m_insMemInfo->MemInsert = plan->operatorMemKB[0] * 1 / 3;// 插入操作的插入内存
m_insMemInfo->MemSort = plan->operatorMemKB[0] - m_insMemInfo->MemInsert;// 排序内存
m_insMemInfo->spreadNum = 0;// 扩展次数
m_insMemInfo->partitionNum = partitionNum;// 分区数
m_delMemInfo->partitionNum = partitionNum;// 分区数
m_delMemInfo->MemSort = m_insMemInfo->MemSort;
MEMCTL_LOG(DEBUG2,
"UpdateForCStorePartDelete(init plan):Insert workmem is : %dKB, sort workmem: %dKB,"
@ -157,13 +159,13 @@ void CStoreUpdate::InitUpdateMemArg(Plan* plan)
m_insMemInfo->partitionNum,
m_insMemInfo->canSpreadmaxMem);
} else {
m_insMemInfo->canSpreadmaxMem = plan->operatorMaxMem;
m_insMemInfo->canSpreadmaxMem = plan->operatorMaxMem;// 可扩展的最大内存
m_insMemInfo->MemInsert =
static_cast<int>((double)plan->operatorMemKB[0] * (double)(maxbatchRows * BATCHROW_TIMES) /
(double)(maxbatchRows + partialClusterRows));
m_insMemInfo->MemSort = plan->operatorMemKB[0] - m_insMemInfo->MemInsert;
m_insMemInfo->spreadNum = 0;
m_insMemInfo->partitionNum = 1;
(double)(maxbatchRows + partialClusterRows));// 插入操作的插入内存
m_insMemInfo->MemSort = plan->operatorMemKB[0] - m_insMemInfo->MemInsert;// 排序内存
m_insMemInfo->spreadNum = 0;// 扩展次数
m_insMemInfo->partitionNum = 1;// 分区数默认为1
m_delMemInfo->MemSort = m_insMemInfo->MemSort;
MEMCTL_LOG(DEBUG2,
"UpdateForCStoreDelete(init plan):Insert workmem is : %dKB, sort workmem: %dKB,"
@ -185,47 +187,49 @@ void CStoreUpdate::InitUpdateMemArg(Plan* plan)
* For static load, a single partition of sort Mem is 512MB, and there is no need to subdivide sort Mem.
* So, set the partitionNum is 1 for all partition table.
*/
m_delMemInfo->canSpreadmaxMem = 0;
m_delMemInfo->MemInsert = 0;
m_delMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem;
m_delMemInfo->spreadNum = 0;
m_delMemInfo->partitionNum = 1;
m_insMemInfo->canSpreadmaxMem = 0;
m_insMemInfo->MemInsert = u_sess->attr.attr_storage.partition_max_cache_size;
m_insMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem;
m_insMemInfo->spreadNum = 0;
m_insMemInfo->partitionNum = 1;
// 静态加载的情况
m_delMemInfo->canSpreadmaxMem = 0;// 可扩展的最大内存
m_delMemInfo->MemInsert = 0;// 删除操作的插入内存
m_delMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem;// 排序内存
m_delMemInfo->spreadNum = 0;// 扩展次数
m_delMemInfo->partitionNum = 1;// 分区数默认为1
m_insMemInfo->canSpreadmaxMem = 0;// 可扩展的最大内存
m_insMemInfo->MemInsert = u_sess->attr.attr_storage.partition_max_cache_size;// 插入操作的插入内存
m_insMemInfo->MemSort = u_sess->attr.attr_storage.psort_work_mem;// 排序内存
m_insMemInfo->spreadNum = 0;// 扩展次数
m_insMemInfo->partitionNum = 1;// 分区数默认为1
}
}
//初始化删除操作的排序状态
void CStoreUpdate::InitSortState(TupleDesc sortTupDesc)
{
Assert(sortTupDesc && m_resultRelInfo && m_delete);
Assert(sortTupDesc && m_resultRelInfo && m_delete);// 断言确保sortTupDesc、m_resultRelInfo和m_delete不为空
JunkFilter* junkfilter = m_resultRelInfo->ri_junkFilter;
JunkFilter* junkfilter = m_resultRelInfo->ri_junkFilter;// 获取结果关系信息中的垃圾过滤器
// init delete sort state
m_delete->InitSortState(sortTupDesc, junkfilter->jf_xc_part_id, junkfilter->jf_junkAttNo);
// 初始化删除操作的排序状态传入排序元组描述、垃圾过滤器中的分区ID和垃圾属性编号
}
//执行数据更新操作
uint64 CStoreUpdate::ExecUpdate(_in_ VectorBatch* batch, _in_ int options)
{
Assert(batch && m_resultRelInfo && m_delete);
Assert((m_isPartition && m_partionInsert) || (!m_isPartition && m_insert));
Assert(batch && m_resultRelInfo && m_delete);// 断言确保batch、m_resultRelInfo和m_delete不为空
Assert((m_isPartition && m_partionInsert) || (!m_isPartition && m_insert));// 断言,确保如果有分区相关变量,则相应插入操作也不能为空;否则,如果没有分区,则相应插入操作不能为空
JunkFilter* junkfilter = m_resultRelInfo->ri_junkFilter;
JunkFilter* junkfilter = m_resultRelInfo->ri_junkFilter;// 获取结果关系信息中的垃圾过滤器
// delete
if (!m_hasUniqueIdx) {
m_delete->PutDeleteBatch(batch, junkfilter);
m_delete->PutDeleteBatch(batch, junkfilter);// 调用删除操作的PutDeleteBatch方法传入批量数据和垃圾过滤器实现删除操作
}
int oriCols = batch->m_cols;
batch->m_cols = junkfilter->jf_cleanTupType->natts;
batch->m_cols = junkfilter->jf_cleanTupType->natts;// 设置批量数据的列数为清理后的元组类型中的属性数量
// Check the constraints of the batch
if (m_relation->rd_att->constr)
ExecVecConstraints(m_resultRelInfo, batch, m_estate);
ExecVecConstraints(m_resultRelInfo, batch, m_estate);// 检查批量数据的约束条件
// insert then batch
if (!m_hasUniqueIdx) {
@ -235,9 +239,9 @@ uint64 CStoreUpdate::ExecUpdate(_in_ VectorBatch* batch, _in_ int options)
* firstly insert and then delete.
*/
if (m_isPartition) {
m_partionInsert->BatchInsert(batch, options);
m_partionInsert->BatchInsert(batch, options);// 调用分区插入操作的BatchInsert方法传入批量数据和选项实现插入操作
} else {
m_insert->BatchInsert(batch, options);
m_insert->BatchInsert(batch, options);// 调用插入操作的BatchInsert方法传入批量数据和选项实现插入操作
}
} else {
/*
@ -245,33 +249,34 @@ uint64 CStoreUpdate::ExecUpdate(_in_ VectorBatch* batch, _in_ int options)
* or the index key of new data and old data may violate.
*/
if (m_isPartition) {
PartitionBatchDeleteAndInsert(batch, oriCols, options, junkfilter);
PartitionBatchDeleteAndInsert(batch, oriCols, options, junkfilter);// 调用分区批量删除和插入操作,传入批量数据、原始列数、选项和垃圾过滤器,实现删除和插入操作
} else {
BatchDeleteAndInsert(batch, oriCols, options, junkfilter);
BatchDeleteAndInsert(batch, oriCols, options, junkfilter);// 调用批量删除和插入操作,传入批量数据、原始列数、选项和垃圾过滤器,实现删除和插入操作
}
}
batch->m_cols = oriCols;
batch->m_cols = oriCols; // 恢复批量数据的列数为原始列数
return (uint64)(uint32)batch->m_rows;
return (uint64)(uint32)batch->m_rows; // 返回批量数据的行数
}
/*
* @Description: Do batch delete and insert. In this function, the number of rows those are deleted is
* strictly equals number of rows will be inserted. It is usually used when cstore has unique index.
*/
//执行批量删除和插入操作
void CStoreUpdate::BatchDeleteAndInsert(VectorBatch *batch, int oriBatchCols, int options, JunkFilter *junkfilter)
{
int currentCols = batch->m_cols;
int currentCols = batch->m_cols;// 获取当前批量数据的列数
/* keep memory space from leaking during bulk-insert */
MemoryContext insertCnxt = m_insert->GetTmpMemCnxt();
MemoryContext updateCnxt = MemoryContextSwitchTo(insertCnxt);
MemoryContext insertCnxt = m_insert->GetTmpMemCnxt();// 获取插入操作中的临时内存上下文
MemoryContext updateCnxt = MemoryContextSwitchTo(insertCnxt);// 切换当前内存上下文为插入操作中的临时内存上下文
CStorePSort* sorter = m_insert->GetSorter();
CStorePSort* sorter = m_insert->GetSorter();// 获取插入操作中的排序器
if (sorter != NULL) {
/* Step 1: relation has partial cluster key */
Assert(batch->m_cols == m_insert->m_relation->rd_att->natts);
Assert(batch->m_cols == m_insert->m_relation->rd_att->natts);// 断言,确保批量数据中的列数等于插入操作中关系信息中属性的数量
{
AutoContextSwitch updateContext(updateCnxt);
@ -280,44 +285,45 @@ void CStoreUpdate::BatchDeleteAndInsert(VectorBatch *batch, int oriBatchCols, in
batch->m_cols = currentCols;
}
sorter->PutVecBatch(m_insert->m_relation, batch);
sorter->PutVecBatch(m_insert->m_relation, batch);// 将批量数据传递给排序器
if (sorter->IsFull()) {
{
AutoContextSwitch updateContext(updateCnxt);
m_delete->PartialDelete();
m_delete->PartialDelete();// 调用删除操作的PartialDelete方法实现部分删除操作
}
m_insert->SortAndInsert(options);
m_insert->SortAndInsert(options);// 调用插入操作的SortAndInsert方法传入选项实现排序和插入操作
}
} else {
/* Step 2: relation doesn't have partial cluster key */
bulkload_rows* bufferedBatchRows = m_insert->GetBufferedBatchRows();
bulkload_rows* bufferedBatchRows = m_insert->GetBufferedBatchRows();// 获取插入操作中的批量行
Assert(batch->m_rows <= BatchMaxSize);
Assert(batch->m_cols && m_insert->m_relation->rd_att->natts);
Assert(bufferedBatchRows->m_rows_maxnum > 0);
Assert(bufferedBatchRows->m_rows_maxnum % BatchMaxSize == 0);
Assert(batch->m_rows <= BatchMaxSize);// 断言,确保批量数据的行数小于等于最大批量大小
Assert(batch->m_cols && m_insert->m_relation->rd_att->natts);// 断言,确保批量数据的列数和插入操作中关系信息的属性数量都不为零
Assert(bufferedBatchRows->m_rows_maxnum > 0);// 断言,确保批量行的最大行数大于零
Assert(bufferedBatchRows->m_rows_maxnum % BatchMaxSize == 0);// 断言,确保批量行的最大行数是最大批量大小的倍数
int startIdx = 0;
int lastStartIdx = startIdx;
for (;;) {
/* we need cache data until batchrows is full */
bool needInsert = bufferedBatchRows->append_one_vector(
RelationGetDescr(m_relation), batch, &startIdx, m_insert->m_cstorInsertMem);
RelationGetDescr(m_relation), batch, &startIdx, m_insert->m_cstorInsertMem);// 将批量数据追加到批量行中,同时更新其起始索引
if (startIdx > lastStartIdx) {
AutoContextSwitch updateContext(updateCnxt);
batch->m_cols = oriBatchCols;
m_delete->PutDeleteBatchForUpdate(batch, lastStartIdx, startIdx);
m_delete->PutDeleteBatchForUpdate(batch, lastStartIdx, startIdx);// 调用删除操作的PutDeleteBatchForUpdate方法传入批量数据、起始索引和结束索引实现删除操作
batch->m_cols = currentCols;
}
if (needInsert) {
{
AutoContextSwitch updateContext(updateCnxt);
m_delete->PartialDelete();
m_delete->PartialDelete();// 调用删除操作的PartialDelete方法实现部分删除操作
}
m_insert->BatchInsertCommon(bufferedBatchRows, options);
bufferedBatchRows->reset(true);
m_insert->BatchInsertCommon(bufferedBatchRows, options);// 调用插入操作的BatchInsertCommon方法传入批量行和选项实现批量插入操作
bufferedBatchRows->reset(true);// 重置批量行状态,参数指定是否释放内存
lastStartIdx = startIdx;
} else {
break;
@ -325,52 +331,52 @@ void CStoreUpdate::BatchDeleteAndInsert(VectorBatch *batch, int oriBatchCols, in
}
}
MemoryContextReset(insertCnxt);
(void)MemoryContextSwitchTo(updateCnxt);
MemoryContextReset(insertCnxt);// 重置临时内存上下文
(void)MemoryContextSwitchTo(updateCnxt);// 恢复原始内存上下文
}
//执行分区的批量删除和插入操作
void CStoreUpdate::PartitionBatchDeleteAndInsert(VectorBatch *batch, int oriBatchCols, int options,
JunkFilter *junkfilter)
{
int currentCols = batch->m_cols;
batch->m_cols = oriBatchCols;
int currentCols = batch->m_cols;// 获取当前批量数据的列数
batch->m_cols = oriBatchCols;// 将当前批量数据的列数设置为原始批量数据的列数
m_delete->PutDeleteBatch(batch, junkfilter);
m_delete->PartialDelete();
m_delete->PutDeleteBatch(batch, junkfilter);// 调用删除操作的PutDeleteBatch方法传入批量数据和垃圾过滤器实现删除操作
m_delete->PartialDelete();// 调用删除操作的PartialDelete方法实现部分删除操作
batch->m_cols = currentCols;
batch->m_cols = currentCols;// 将批量数据的列数恢复为当前批量数据的列数
m_partionInsert->BatchInsert(batch, options);
m_partionInsert->BatchInsert(batch, options);// 调用分区插入操作的BatchInsert方法传入批量数据和选项实现批量插入操作
}
//检查关系信息中是否存在唯一索引
bool CStoreUpdate::CheckHasUniqueIdx()
{
bool hasUniqueIdx = false;
if (m_resultRelInfo && m_resultRelInfo->ri_NumIndices > 0) {
IndexInfo** indexInfos = m_resultRelInfo->ri_IndexRelationInfo;
for (int i = 0; i < m_resultRelInfo->ri_NumIndices; ++i) {
if (indexInfos[i]->ii_Unique) {
hasUniqueIdx = true;
break;
bool hasUniqueIdx = false;// 初始化是否存在唯一索引的标志为假
if (m_resultRelInfo && m_resultRelInfo->ri_NumIndices > 0) {// 如果存在关系信息且索引数量大于零
IndexInfo** indexInfos = m_resultRelInfo->ri_IndexRelationInfo;// 获取索引关系信息
for (int i = 0; i < m_resultRelInfo->ri_NumIndices; ++i) {// 遍历索引关系信息
if (indexInfos[i]->ii_Unique) {// 如果当前索引是唯一索引
hasUniqueIdx = true;// 将是否存在唯一索引的标志设置为真
break;// 结束循环
}
}
}
return hasUniqueIdx;
return hasUniqueIdx;// 返回是否存在唯一索引的标志
}
//用于结束更新操作
void CStoreUpdate::EndUpdate(_in_ int options)
{
Assert(m_delete);
Assert((m_isPartition && m_partionInsert) || (!m_isPartition && m_insert));
Assert(m_delete);// 断言删除操作不为空
Assert((m_isPartition && m_partionInsert) || (!m_isPartition && m_insert));// 断言如果是分区更新,则分区插入操作不为空;否则,插入操作不为空
// end delete
m_delete->ExecDelete();
m_delete->ExecDelete();// 执行删除操作
// end insert
if (m_isPartition) {
m_partionInsert->EndBatchInsert();
if (m_isPartition) {// 如果是分区更新
m_partionInsert->EndBatchInsert();// 结束分区批量插入操作
} else {
m_insert->SetEndFlag();
m_insert->BatchInsert((VectorBatch*)NULL, options);
m_insert->SetEndFlag();// 设置插入操作的结束标志
m_insert->BatchInsert((VectorBatch*)NULL, options);// 执行插入操作,传入空的批量数据和选项
}
}

View File

@ -46,10 +46,11 @@ DataCacheMgr* DataCacheMgr::m_data_cache = NULL;
* @Return: lock number
* @See also:
*/
//获取数据缓存管理器中锁的数量
int DataCacheMgrNumLocks()
{
int64 cache_size = CacheMgrCalcSizeByType(MGR_CACHE_TYPE_DATA);
return CacheMgrNumLocks(cache_size, BLCKSZ);
int64 cache_size = CacheMgrCalcSizeByType(MGR_CACHE_TYPE_DATA);// 计算数据缓存大小
return CacheMgrNumLocks(cache_size, BLCKSZ);// 返回根据缓存大小和块大小计算得到的锁的数量
}
/*
@ -57,41 +58,47 @@ int DataCacheMgrNumLocks()
* @Return: CU cache instance
* @See also:
*/
//获取数据缓存管理器的单例实例
DataCacheMgr* DataCacheMgr::GetInstance(void)
{
Assert(m_data_cache != NULL);
return m_data_cache;
Assert(m_data_cache != NULL);// 断言确保数据缓存实例不为空
return m_data_cache;// 返回数据缓存实例
}
/*
* @Description: create or recreate the Singleton Instance of CU cache
* @See also:
*/
//创建或重新创建数据缓存管理器的单例实例
void DataCacheMgr::NewSingletonInstance(void)
{
int64 cache_size = 0;
// 只有 Postmaster 才有权利创建或重新创建 CU 缓存实例
/* only Postmaster has the privilege to create or recreate CU cache instance */
if (IsUnderPostmaster)
return;
if (IsUnderPostmaster)// 如果处于 Postmaster 下
return;// 直接返回
int condition = sizeof(DataSlotTagKey) != MAX_CACHE_TAG_LEN ? 1 : 0;
BUILD_BUG_ON_CONDITION(condition);
int condition = sizeof(DataSlotTagKey) != MAX_CACHE_TAG_LEN ? 1 : 0;// 检查 DataSlotTagKey 的大小是否等于 MAX_CACHE_TAG_LEN
BUILD_BUG_ON_CONDITION(condition);// 在编译时检查条件,如果为真,则产生编译错误
if (m_data_cache == NULL) {
if (m_data_cache == NULL) {// 如果数据缓存实例为空
// 第一次创建该实例
/* create this instance at the first time */
m_data_cache = New(CurrentMemoryContext) DataCacheMgr;
m_data_cache->m_cache_mgr = New(CurrentMemoryContext) CacheMgr;
m_data_cache = New(CurrentMemoryContext) DataCacheMgr;// 创建 DataCacheMgr 实例
m_data_cache->m_cache_mgr = New(CurrentMemoryContext) CacheMgr;// 创建 CacheMgr 实例
} else {
// 销毁实例的所有资源
/* destroy all resources of its members */
m_data_cache->m_cache_mgr->Destroy();
SpinLockFree(&m_data_cache->m_adio_write_cache_lock);
m_data_cache->m_cache_mgr->Destroy();// 销毁缓存管理器的资源
SpinLockFree(&m_data_cache->m_adio_write_cache_lock);// 释放自旋锁的内存资源
}
cache_size = CacheMgrCalcSizeByType(MGR_CACHE_TYPE_DATA);
m_data_cache->m_cstoreMaxSize = cache_size;
SpinLockInit(&m_data_cache->m_adio_write_cache_lock);
cache_size = CacheMgrCalcSizeByType(MGR_CACHE_TYPE_DATA);// 计算数据缓存的大小
m_data_cache->m_cstoreMaxSize = cache_size;// 设置数据缓存的最大大小
SpinLockInit(&m_data_cache->m_adio_write_cache_lock);// 初始化自旋锁
/* init or reset this instance */
// 初始化或重置实例
m_data_cache->m_cache_mgr->Init(cache_size, BLCKSZ, MGR_CACHE_TYPE_DATA, Max(sizeof(CU), sizeof(OrcDataValue)));
ereport(LOG, (errmodule(MOD_CACHE), errmsg("set data cache size(%ld)", cache_size)));
ereport(LOG, (errmodule(MOD_CACHE), errmsg("set data cache size(%ld)", cache_size)));// 记录日志,设置数据缓存的大小
}
/*
@ -103,15 +110,21 @@ void DataCacheMgr::NewSingletonInstance(void)
* @Return: cu slot tag
* @See also:
*/
//初始化 CUColumn Unit的唯一标志符tag
DataSlotTag DataCacheMgr::InitCUSlotTag(RelFileNodeOld* rnode, int colid, uint32 cuid, CUPointer cuPtr)
{
// 初始化 CU 的唯一标志符tag
DataSlotTag tag;
// 设置 CU 的文件节点信息
tag.slotTag.cuSlotTag.m_rnode = *rnode;
// 设置 CU 的标识符、列 ID、偏移量和填充字段
tag.slotTag.cuSlotTag.m_CUId = cuid;
tag.slotTag.cuSlotTag.m_colId = colid;
tag.slotTag.cuSlotTag.m_cuPtr = cuPtr;
tag.slotTag.cuSlotTag.m_padding = 0;
// 设置 slot 类型为 CACHE_COlUMN_DATA
tag.slotType = CACHE_COlUMN_DATA;
// 返回初始化后的 CU slot 标签
return tag;
}
@ -124,28 +137,39 @@ DataSlotTag DataCacheMgr::InitCUSlotTag(RelFileNodeOld* rnode, int colid, uint32
* @Return: orc data slot tag
* @See also:
*/
//初始化 ORCOptimized Row Columnar数据的唯一标志符tag
DataSlotTag DataCacheMgr::InitORCSlotTag(RelFileNode* rnode, int32 fileid, uint64 offset, uint64 length)
{
// 初始化 ORC 数据的唯一标志符tag
DataSlotTag tag;
// 设置 ORC 数据的文件节点信息
tag.slotTag.orcSlotTag.m_rnode = *(RelFileNodeOld*)rnode;
// 设置 ORC 数据的关系 ID、偏移量和长度
tag.slotTag.orcSlotTag.m_fileId = fileid;
tag.slotTag.orcSlotTag.m_offset = offset;
tag.slotTag.orcSlotTag.m_length = length;
// 设置 slot 类型为 CACHE_ORC_DATA
tag.slotType = CACHE_ORC_DATA;
// 返回初始化后的 ORC 数据 slot 标签
return tag;
}
//初始化 OBSObject Storage Service数据的唯一标志符tag
DataSlotTag DataCacheMgr::InitOBSSlotTag(uint32 hostNameHash, uint32 bucketNameHash, uint32 fileFirstHalfHash,
uint32 fileSecondHalfHash, uint64 offset, uint64 length) const
{
// 初始化 OBS 数据的唯一标志符tag
DataSlotTag tag;
// 设置 OBS 数据的服务器名哈希、桶名哈希、文件名前半部分哈希和文件名后半部分哈希
tag.slotTag.obsSlotTag.m_serverHash = hostNameHash;
tag.slotTag.obsSlotTag.m_bucketHash = bucketNameHash;
tag.slotTag.obsSlotTag.m_fileFirstHash = fileFirstHalfHash;
tag.slotTag.obsSlotTag.m_fileSecondHash = fileSecondHalfHash;
// 设置 OBS 数据的偏移量和长度
tag.slotTag.obsSlotTag.m_offset = offset;
tag.slotTag.obsSlotTag.m_length = length;
// 设置 slot 类型为 CACHE_OBS_DATA
tag.slotType = CACHE_OBS_DATA;
// 返回初始化后的 OBS 数据 slot 标签
return tag;
}
@ -158,12 +182,17 @@ DataSlotTag DataCacheMgr::InitOBSSlotTag(uint32 hostNameHash, uint32 bucketNameH
*/
CacheSlotId_t DataCacheMgr::FindDataBlock(DataSlotTag* dataSlotTag, bool first_enter_block)
{
// 声明一个 slot id 并初始化为 CACHE_BLOCK_INVALID_IDX
CacheSlotId_t slot = CACHE_BLOCK_INVALID_IDX;
// 声明一个 CacheTag 对象并初始化为 0
CacheTag cacheTag = {0};
// 初始化 CacheTag 对象,使用 slot 的类型和 tag 来初始化
// 由于 tag 可能是复合类型,需要传入 tag 的大小
m_cache_mgr->InitCacheBlockTag(&cacheTag, dataSlotTag->slotType, &dataSlotTag->slotTag, sizeof(DataSlotTagKey));
// 根据 CacheTag 在 cache 中查找对应的 slot id
// 如果 first_enter_block 为 true则表示第一次使用该 block需要申请新的 slot
slot = m_cache_mgr->FindCacheBlock(&cacheTag, first_enter_block);
// 返回查找到的 slot id
return slot;
}
@ -178,25 +207,36 @@ CacheSlotId_t DataCacheMgr::FindDataBlock(DataSlotTag* dataSlotTag, bool first_e
*/
void DataCacheMgr::InvalidateCU(RelFileNodeOld* rnode, int colId, uint32 cuId, CUPointer cuPtr)
{
// 声明一个 CacheTag 对象并初始化为 0
CacheTag cacheTag = {0};
// 根据传入的参数初始化 DataSlotTag 对象
DataSlotTag dataSlotTag = InitCUSlotTag(rnode, colId, cuId, cuPtr);
// 初始化 CacheTag 对象,使用 slot 的类型和 tag 来初始化
// 由于 tag 可能是复合类型,需要传入 tag 的大小
m_cache_mgr->InitCacheBlockTag(&cacheTag, dataSlotTag.slotType, &dataSlotTag.slotTag, sizeof(DataSlotTagKey));
// invalid cache 中的对应 slot
m_cache_mgr->InvalidateCacheBlock(&cacheTag);
return;
}
/* invalid all CUs from data cache which belongs to this column relation */
//使所有属于该列关系的 CU 在 cache 中对应的 slot 设置为无效
void DataCacheMgr::DropRelationCUCache(const RelFileNode& rnode)
{
// 声明一个 CacheTag 对象并初始化为 0
CacheTag tag = {0};
// 声明一个 CUSlotTag 对象,初始化为 {InvalidOid, InvalidOid, InvalidOid}、0、0、0、0
CUSlotTag cuTag = {{InvalidOid, InvalidOid, InvalidOid}, 0, 0, 0, 0};
// 获取已使用的 cache slot 的数量
const int maxSlot = m_cache_mgr->GetUsedCacheSlotNum();
// 遍历所有 cache slot并检查是否属于该列关系是则使其失效
for (CacheSlotId_t slot = 0; slot <= maxSlot; slot++) {
// 复制当前 slot 的 tag 到 tag 变量中
m_cache_mgr->CopyCacheBlockTag(slot, &tag);
// 将 tag 的 key 转换为 CUSlotTag 类型
cuTag = *(CUSlotTag*)tag.key;
// 如果当前 slot 是 CACHE_COlUMN_DATA 类型且是该列关系,则使其失效
if (CACHE_COlUMN_DATA == tag.type && RelFileColumnNodeRelEquals(rnode, cuTag.m_rnode)) {
/* try to invalid this CU */
InvalidateCU(&cuTag.m_rnode, cuTag.m_colId, cuTag.m_CUId, cuTag.m_cuPtr);
@ -210,8 +250,10 @@ void DataCacheMgr::DropRelationCUCache(const RelFileNode& rnode)
* @Return: CU data pointer
* @See also:
*/
//获取指定 cuSlotId 对应的 CU压缩单元数据指针
CU* DataCacheMgr::GetCUBuf(int cuSlotId)
{
// 返回指定 cuSlotId 对应的 CU 数据指针
return (CU*)m_cache_mgr->GetCacheBlock(cuSlotId);
}
@ -221,8 +263,10 @@ CU* DataCacheMgr::GetCUBuf(int cuSlotId)
* @Return: ORC data pointer
* @See also:
*/
//获取指定 cuSlotId 对应的 ORC 数据指针
OrcDataValue* DataCacheMgr::GetORCDataBuf(int cuSlotId)
{
// 返回指定 cuSlotId 对应的 ORC 数据指针
return (OrcDataValue*)m_cache_mgr->GetCacheBlock(cuSlotId);
}
@ -232,11 +276,14 @@ OrcDataValue* DataCacheMgr::GetORCDataBuf(int cuSlotId)
* @Return: true for the session can evict the cache.
* false for some other session is updating it concurrently.
*/
//预留指定的缓存块 slotId
bool DataCacheMgr::ReserveDataBlockWithSlotId(CacheSlotId_t slotId)
{
// 预留指定的缓存块 slot
if (m_cache_mgr->ReserveCacheBlockWithSlotId(slotId)) {
Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO) ||
t_thrd.storage_cxt.CacheBlockInProgressIO == slotId);
// 设置当前正在 IO 的缓存块的 ID
t_thrd.storage_cxt.CacheBlockInProgressIO = slotId;
return true;
}
@ -250,11 +297,17 @@ bool DataCacheMgr::ReserveDataBlockWithSlotId(CacheSlotId_t slotId)
* @Return: true for the session can evict the cache.
* false for the slot is IOBUSY.
*/
// 函数作用对于远程读取从缓存中驱逐某个由slotId表示的缓存块。
// 参数 slotId要驱逐的目标缓存块的槽位ID。
// 返回值true表示会话可以驱逐缓存块false表示该槽位正在进行IO操作无法驱逐。
bool DataCacheMgr::ReserveCstoreDataBlockWithSlotId(CacheSlotId_t slotId)
{
// 调用缓存管理器的函数来驱逐具有指定槽位ID的Cstore缓存块
if (m_cache_mgr->ReserveCstoreCacheBlockWithSlotId(slotId)) {
// 确认当前正在进行IO操作的缓存块是否为有效的槽位ID如果是则断言失败
Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO) ||
t_thrd.storage_cxt.CacheBlockInProgressIO == slotId);
// 将正在进行IO操作的缓存块设置为指定的槽位ID
t_thrd.storage_cxt.CacheBlockInProgressIO = slotId;
return true;
}
@ -270,18 +323,26 @@ bool DataCacheMgr::ReserveCstoreDataBlockWithSlotId(CacheSlotId_t slotId)
* @Return: slot id
* @See also:
*/
// 函数作用:从数据缓存管理器中预留数据块
// 参数 dataSlotTag数据槽位的标签
// 参数 size需要的块大小
// 参数 hasFound是否找到了预留的块
// 返回值槽位ID
CacheSlotId_t DataCacheMgr::ReserveDataBlock(DataSlotTag* dataSlotTag, int size, bool& hasFound)
{
CacheSlotId_t slot = CACHE_BLOCK_INVALID_IDX;
CacheTag cacheTag = {0};
// 使用数据槽位标签初始化缓存块标签
m_cache_mgr->InitCacheBlockTag(&cacheTag, dataSlotTag->slotType, &dataSlotTag->slotTag, sizeof(DataSlotTagKey));
// 调用缓存管理器的函数来预留指定大小的缓存块
slot = m_cache_mgr->ReserveCacheBlock(&cacheTag, size, hasFound);
if (!hasFound) {
// 如果没有找到预留的块则记录正在进行IO操作的缓存块为当前预留的槽位ID
/* remember block slot in process */
Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO));
t_thrd.storage_cxt.CacheBlockInProgressIO = slot;
}
// 打印调试信息
if (cacheTag.type == CACHE_COlUMN_DATA) {
CUSlotTag* cuslotTag = &dataSlotTag->slotTag.cuSlotTag;
ereport(DEBUG1,
@ -334,8 +395,11 @@ CacheSlotId_t DataCacheMgr::ReserveDataBlock(DataSlotTag* dataSlotTag, int size,
* @IN CUSlotId: slot id
* @See also:
*/
// 函数作用取消数据块的固定状态解除数据块的Pin。
// 参数 CUSlotId数据块的槽位ID。
void DataCacheMgr::UnPinDataBlock(int CUSlotId)
{
// 调用缓存管理器的函数来取消指定槽位ID的缓存块的固定状态解除Pin。
m_cache_mgr->UnPinCacheBlock(CUSlotId);
}
@ -344,8 +408,11 @@ void DataCacheMgr::UnPinDataBlock(int CUSlotId)
* @Param[IN] slot: slot id
* @See also:
*/
// 函数作用:如果预留数据块失败,则进行资源清理(设置块状态和释放锁,如果需要)。
// 参数 slot槽位ID
void DataCacheMgr::AbortCU(CacheSlotId_t slot)
{
// 调用缓存管理器的函数来设置指定槽位ID的缓存块状态并释放相关的锁。
m_cache_mgr->AbortCacheBlock(slot);
return;
}
@ -355,15 +422,19 @@ void DataCacheMgr::AbortCU(CacheSlotId_t slot)
* @Param[IN] abort: whether to delete this CU from cache
* @See also:
*/
// 函数作用终止或中止当前处理的CUCompressed Unit
// 参数 abort是否从缓存中删除该CU。
void DataCacheMgr::TerminateCU(bool abort)
{
if (abort) {
if (IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO)) {
// 如果当前存在进行IO操作的缓存块将其设置为无效并包括IO状态和锁的所有者信息。
/* invalid this block slot, and include IO state and lock owner */
Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressUncompress));
AbortCU(t_thrd.storage_cxt.CacheBlockInProgressIO);
}
if (IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressUncompress)) {
// 如果当前存在进行解压缩的缓存块将其设置为无效并不关心IO状态或锁的所有者信息。
/* invalid this block slot, and don't care IO state or lock owner */
Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO));
m_cache_mgr->SetCacheBlockErrorState(t_thrd.storage_cxt.CacheBlockInProgressUncompress);
@ -371,6 +442,7 @@ void DataCacheMgr::TerminateCU(bool abort)
cuPtr->FreeSrcBuf();
}
}
// 清除记录
/* clear record */
t_thrd.storage_cxt.CacheBlockInProgressIO = CACHE_BLOCK_INVALID_IDX;
t_thrd.storage_cxt.CacheBlockInProgressUncompress = CACHE_BLOCK_INVALID_IDX;
@ -379,27 +451,30 @@ void DataCacheMgr::TerminateCU(bool abort)
/*
* @Description: terminate or abort this CU in verify process..
*/
//终止或中止在验证过程中的当前CU压缩单元
void DataCacheMgr::TerminateVerifyCU()
{
if (IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO)) {
/* invalid this block slot, and include IO state and lock owner */
// 如果存在进行IO操作的缓存块将其设置为无效并包括IO状态和锁的所有者信息。
Assert (!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressUncompress));
Assert(m_cache_mgr->CstoreIOLockHeldByMe(t_thrd.storage_cxt.CacheBlockInProgressIO));
AbortCU(t_thrd.storage_cxt.CacheBlockInProgressIO);
HOLD_INTERRUPTS(); /* match the upcoming RESUME_INTERRUPTS */
HOLD_INTERRUPTS(); /* match the upcoming RESUME_INTERRUPTS */ // 匹配即将到来的 RESUME_INTERRUPTS
m_cache_mgr->RealeseCstoreIOLock(t_thrd.storage_cxt.CacheBlockInProgressIO);
}
if (IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressUncompress)) {
/* invalid this block slot, and don't care IO state or lock owner */
// 如果存在进行解压缩的缓存块将其设置为无效并不关心IO状态或锁的所有者信息。
Assert (!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressIO));
Assert (m_cache_mgr->CompressLockHeldByMe(t_thrd.storage_cxt.CacheBlockInProgressUncompress));
m_cache_mgr->SetCacheBlockErrorState(t_thrd.storage_cxt.CacheBlockInProgressUncompress);
CU* cuPtr = GetCUBuf(t_thrd.storage_cxt.CacheBlockInProgressUncompress);
cuPtr->FreeSrcBuf();
HOLD_INTERRUPTS(); /* match the upcoming RESUME_INTERRUPTS */
HOLD_INTERRUPTS(); /* match the upcoming RESUME_INTERRUPTS */ // 匹配即将到来的 RESUME_INTERRUPTS
m_cache_mgr->RealeseCompressLock(t_thrd.storage_cxt.CacheBlockInProgressUncompress);
}
// 清除记录
/* clear record */
t_thrd.storage_cxt.CacheBlockInProgressIO = CACHE_BLOCK_INVALID_IDX;
t_thrd.storage_cxt.CacheBlockInProgressUncompress = CACHE_BLOCK_INVALID_IDX;
@ -414,45 +489,64 @@ void DataCacheMgr::TerminateVerifyCU()
* @Return: CUUncompressedRetCode value
* @See also:
*/
/*
* @Description: CU cache将解压缩CU的原始数据
* @Param[IN] cuDescPtr: CU描述信息指针
* @Param[IN] slotId: CU槽位ID
* @Return: CUUncompressedRetCode枚举值
* @See also:
*/
//对CU进行解压缩
CUUncompressedRetCode DataCacheMgr::StartUncompressCU(
CUDesc* cuDescPtr, CacheSlotId_t slotId, int planNodeId, bool timing, int align_size)
{
CU* cuPtr = GetCUBuf(slotId);
CU* cuPtr = GetCUBuf(slotId);// 获取CU缓存块指针
m_cache_mgr->AcquireCompressLock(slotId);
m_cache_mgr->AcquireCompressLock(slotId);// 获取解压缩锁
/*
* if the slot flag is set to CACHE_BLOCK_ERROR by some abort threads,
* the compressed buffer is also valid no matter the CU comes from primary
* node or remote mode. So, the thread can continue to uncompress.
*/
/*
* CACHE_BLOCK_ERROR线
* CU来自主节点还是远程模式线
*/
if (m_cache_mgr->IsIOBusy(slotId)) {
/*
* the CU is being reloading by remote read thread,
* return CU_RELOADING and retry to load CU.
*/
/*
* CU正在被远程读线程重新加载
* CU_RELOADING并重试加载CU
*/
m_cache_mgr->RealeseCompressLock(slotId);
return CU_RELOADING;
}
if (!cuPtr->m_cache_compressed) {
/* 另一个线程已解压缩此CU数据 */
/* another therad have decompressed this CU data */
m_cache_mgr->RealeseCompressLock(slotId);
return CU_OK;
}
if (cuPtr->m_adio_error) {
/* IO错误 */
/* IO error */
this->m_cache_mgr->RealeseCompressLock(slotId);
return CU_ERR_ADIO;
}
/* 记录当前正在解压缩的槽位ID */
/* remember this slot id */
Assert(!IsValidCacheSlotID(t_thrd.storage_cxt.CacheBlockInProgressUncompress));
t_thrd.storage_cxt.CacheBlockInProgressUncompress = slotId;
if (cuPtr->CheckCrc() == false) {
/* CRC check failed */
/* CRC检查失败 */
m_cache_mgr->RealeseCompressLock(slotId);
return CU_ERR_CRC;
}
@ -461,7 +555,7 @@ CUUncompressedRetCode DataCacheMgr::StartUncompressCU(
m_cache_mgr->RealeseCompressLock(slotId);
return CU_ERR_MAGIC;
}
/* 用于跟踪cstore扫描的宏 */
/* macro for tracing cstore scan */
#define UNCOMPRESS_TRACE(A) \
do { \
@ -469,26 +563,29 @@ CUUncompressedRetCode DataCacheMgr::StartUncompressCU(
A; \
} \
} while (0)
/* 始终假设磁盘上是压缩数据,缓存中是未压缩数据。 */
/* Always presume compressed disk and uncompressed cache. */
UNCOMPRESS_TRACE(TRACK_START(planNodeId, UNCOMPRESS_CU));
cuPtr->UnCompress(cuDescPtr->row_count, cuDescPtr->magic, align_size);
UNCOMPRESS_TRACE(TRACK_END(planNodeId, UNCOMPRESS_CU));
/* 不要将压缩缓冲区放入缓存中
*
*/
/* Do not put the compressedBuf in the cache
* In the future we may have compressed data in the cache or let the
* caller choose, but for now we just keep the uncompressed data.
*/
cuPtr->FreeCompressBuf();
cuPtr->FreeCompressBuf();// 释放压缩缓冲区
/* Adjust the allocation reservation to take into account
* compression or expansion.
*/
/* 调整内存分配预留,考虑压缩或扩展 */
int cu_uncompress_size = cuPtr->GetUncompressBufSize();
m_cache_mgr->AdjustCacheMem(slotId, cuDescPtr->cu_size, cu_uncompress_size);
m_cache_mgr->RealeseCompressLock(slotId);
m_cache_mgr->RealeseCompressLock(slotId);// 释放解压缩锁
TerminateCU(false);
TerminateCU(false);// 终止当前CU的处理
return CU_OK;
}
@ -497,6 +594,9 @@ CUUncompressedRetCode DataCacheMgr::StartUncompressCU(
* @Return: cache used size
* @See also:
*/
// get data cache manage current memory cache used size
// 返回数据缓存管理器当前的内存缓存使用大小
int64 DataCacheMgr::GetCurrentMemSize()
{
if (!IS_SINGLE_NODE)
@ -514,6 +614,12 @@ int64 DataCacheMgr::GetCurrentMemSize()
* @Return: true, means error happen in preftech
* @See also:
*/
// DataBlockWaitIO
/* 如果CU处于IOBUSY状态则睡眠等待IO busy锁。
*
* rowstore中的WaitIO()
*/
//等待指定槽位ID的CU IO操作完成
bool DataCacheMgr::DataBlockWaitIO(CacheSlotId_t slotId)
{
return m_cache_mgr->WaitIO(slotId);
@ -526,6 +632,10 @@ bool DataCacheMgr::DataBlockWaitIO(CacheSlotId_t slotId)
* @Param[IN] slotId: slot id
* @See also:
*/
// DataBlockCompleteIO
/* 当I/O操作完成时将CU标记为非忙状态并唤醒下一个等待者。
* TerminateBufferIO函数
*/
void DataCacheMgr::DataBlockCompleteIO(CacheSlotId_t slotId)
{
m_cache_mgr->CompleteIO(slotId);
@ -542,6 +652,7 @@ void DataCacheMgr::DataBlockCompleteIO(CacheSlotId_t slotId)
* @Return: true -- lock; false --not lock
* @See also:
*/
// 检查当前线程是否持有指定槽位ID的CU描述符锁
bool DataCacheMgr::CULWLockHeldByMe(CacheSlotId_t slotId)
{
return m_cache_mgr->LockHeldByMe(slotId);
@ -552,6 +663,7 @@ bool DataCacheMgr::CULWLockHeldByMe(CacheSlotId_t slotId)
* @Param[IN] slotId: slot id
* @See also:
*/
// 获取指定槽位ID的CU描述符锁
void DataCacheMgr::CULWLockOwn(CacheSlotId_t slotId)
{
return m_cache_mgr->LockOwn(slotId);
@ -562,6 +674,7 @@ void DataCacheMgr::CULWLockOwn(CacheSlotId_t slotId)
* @Param[IN] slotId: slot id
* @See also:
*/
// 释放指定槽位ID的CU描述符锁
void DataCacheMgr::CULWLockDisown(CacheSlotId_t slotId)
{
return m_cache_mgr->LockDisown(slotId);
@ -571,6 +684,7 @@ void DataCacheMgr::CULWLockDisown(CacheSlotId_t slotId)
* @Description: used for adio write
* @See also:
*/
// 用于adio写操作时锁住私有缓存
void DataCacheMgr::LockPrivateCache()
{
SpinLockAcquire(&m_adio_write_cache_lock);
@ -580,6 +694,7 @@ void DataCacheMgr::LockPrivateCache()
* @Description: used for adio write
* @See also:
*/
// 用于adio写操作时释放私有缓存锁
void DataCacheMgr::UnLockPrivateCache()
{
SpinLockRelease(&m_adio_write_cache_lock);
@ -590,59 +705,64 @@ void DataCacheMgr::UnLockPrivateCache()
* @IN slotId: slot id
* @See also:
*/
//在发现数据缓存资源泄漏时打印相关的警告信息
void DataCacheMgr::PrintDataCacheSlotLeakWarning(CacheSlotId_t slotId)
{
DataSlotTag tag = {};
uint32 refcount = 0;
// 标记和引用计数
DataSlotTag tag = {};// 数据槽位标记
uint32 refcount = 0; // 引用计数
Assert(IsValidCacheSlotID(slotId));
const CacheTag* cacheTag = m_cache_mgr->GetCacheBlockTag(slotId, &refcount);
Assert(IsValidCacheSlotID(slotId));// 检查槽位ID的有效性
// 获取缓存标签
const CacheTag *cacheTag = m_cache_mgr->GetCacheBlockTag(slotId, &refcount); // 获取缓存标签和引用计数
// 如果缓存标签为空,则输出错误信息
if (cacheTag == NULL) {
ereport(ERROR, (errmsg("No cache found when print data cache slot leak warning: slotId:%d", slotId)));
}
// 复制标签到tag中
errno_t rc = memcpy_s(&(tag.slotTag), sizeof(DataSlotTagKey), cacheTag->key, sizeof(DataSlotTagKey));
securec_check(rc, "\0", "\0");
if (cacheTag->type == CACHE_COlUMN_DATA) {
CUSlotTag* cuslotTag = &tag.slotTag.cuSlotTag;
// 根据不同的缓存类型,输出不同的警告信息
if (cacheTag->type == CACHE_COlUMN_DATA) { // 列式数据缓存
CUSlotTag* cuslotTag = &tag.slotTag.cuSlotTag;// 列式数据槽位标记
ereport(WARNING,
(errmsg("CUCache refcount leak: type:%d, spaceNode: %u, dbNode: %u, relNode: %u, colId: %d, cuId: %d, "
"cuPoint: %lu, refcount: %u, slotId:%d",
cacheTag->type,
cuslotTag->m_rnode.spcNode,
cuslotTag->m_rnode.dbNode,
cuslotTag->m_rnode.relNode,
cuslotTag->m_colId,
cuslotTag->m_CUId,
cuslotTag->m_cuPtr,
refcount,
slotId)));
} else if (cacheTag->type == CACHE_ORC_DATA) {
ORCSlotTag* orcslotTag = &tag.slotTag.orcSlotTag;
cuslotTag->m_rnode.spcNode,// 表空间节点
cuslotTag->m_rnode.dbNode,// 数据库节点
cuslotTag->m_rnode.relNode,// 表节点
cuslotTag->m_colId,// 列ID
cuslotTag->m_CUId,// CU ID
cuslotTag->m_cuPtr,// CU指针
refcount,// 引用计数
slotId)));// 槽位ID
} else if (cacheTag->type == CACHE_ORC_DATA) { // ORC数据缓存
ORCSlotTag* orcslotTag = &tag.slotTag.orcSlotTag;// ORC数据槽位标记
ereport(WARNING,
(errmsg("ORCCache refcount leak: type:%d, spaceNode: %u, dbNode: %u, relNode: %u, fileId: %d, length: %lu, "
"offset: %lu, refcount: %u, slotId:%d",
cacheTag->type,
orcslotTag->m_rnode.spcNode,
orcslotTag->m_rnode.dbNode,
orcslotTag->m_rnode.relNode,
orcslotTag->m_fileId,
orcslotTag->m_length,
orcslotTag->m_offset,
refcount,
slotId)));
} else if (cacheTag->type == CACHE_OBS_DATA) {
OBSSlotTag* obsslotTag = &tag.slotTag.obsSlotTag;
orcslotTag->m_rnode.spcNode,// 表空间节点
orcslotTag->m_rnode.dbNode,// 数据库节点
orcslotTag->m_rnode.relNode,// 表节点
orcslotTag->m_fileId,// 文件ID
orcslotTag->m_length,// 长度
orcslotTag->m_offset,// 偏移量
refcount,// 引用计数
slotId)));// 槽位ID
} else if (cacheTag->type == CACHE_OBS_DATA) {// OBS数据缓存
OBSSlotTag* obsslotTag = &tag.slotTag.obsSlotTag;// OBS数据槽位标记
ereport(WARNING,
(errmsg("OBSCache refcount leak: type:%d, server hash: %u, bucket hash: %u, file hash1: %u, file hash2: "
"%u, length: %lu, offset: %lu",
cacheTag->type,
obsslotTag->m_serverHash,
obsslotTag->m_bucketHash,
obsslotTag->m_fileFirstHash,
obsslotTag->m_fileSecondHash,
obsslotTag->m_length,
obsslotTag->m_offset)));
obsslotTag->m_serverHash,// 服务器哈希值
obsslotTag->m_bucketHash,// 桶哈希值
obsslotTag->m_fileFirstHash,// 文件哈希值1
obsslotTag->m_fileSecondHash,// 文件哈希值2
obsslotTag->m_length, // 长度
obsslotTag->m_offset))); // 偏移量
}
return;
@ -655,13 +775,22 @@ void DataCacheMgr::PrintDataCacheSlotLeakWarning(CacheSlotId_t slotId)
* @IN slotId: slot id
* @See also:
*/
/*
* @Description: ORC数据缓存
* @IN buffer:
* @IN size:
* @IN slotId: ID
* @See also:
*/
//设置ORC数据缓存块的值
void DataCacheMgr::SetORCDataBlockValue(CacheSlotId_t slotId, const void* buffer, uint64 size)
{
// 获取ORC数据缓存块
OrcDataValue* orcDataValue = GetORCDataBuf(slotId);
// 设置ORC数据缓存块的大小和值
orcDataValue->size = size;
orcDataValue->value = (char*)CStoreMemAlloc::Palloc(size, false);
// 将缓冲区的数据复制到ORC数据缓存块中
errno_t rc = memcpy_s(orcDataValue->value, size, buffer, size);
securec_check(rc, "\0", "\0");
@ -674,7 +803,7 @@ void DataCacheMgr::SetOBSDataBlockValue(
CacheSlotId_t slotId, const void* buffer, uint64 size, const char* prefix, const char* dataDNA)
{
OrcDataValue* orcDataValue = GetORCDataBuf(slotId);
// 计算缓冲区总大小包括前缀、数据DNA和数据本身
/* prefix + '\0' + dataDNA + '\0' + data */
size_t prefixLen = strlen(prefix);
size_t dataDNALen = strlen(dataDNA);
@ -682,28 +811,28 @@ void DataCacheMgr::SetOBSDataBlockValue(
orcDataValue->size = size + prefixLen + 1 + dataDNALen + 1;
orcDataValue->value = (char*)CStoreMemAlloc::Palloc(orcDataValue->size, false);
// 将缓冲区的值置为0
/* set whole buffer to 0 */
rc = memset_s(orcDataValue->value, orcDataValue->size, 0, orcDataValue->size);
securec_check(rc, "\0", "\0");
// 复制前缀到缓冲区
/* copy prefix */
rc = memcpy_s(orcDataValue->value, orcDataValue->size, prefix, prefixLen);
securec_check(rc, "\0", "\0");
// 复制数据DNA到缓冲区
/* copy data DNA */
rc =
memcpy_s((char*)orcDataValue->value + prefixLen + 1, orcDataValue->size - (prefixLen + 1), dataDNA, dataDNALen);
securec_check(rc, "\0", "\0");
// 复制数据到缓冲区
/* copy data */
rc = memcpy_s((char*)orcDataValue->value + prefixLen + 1 + dataDNALen + 1, size, buffer, size);
securec_check(rc, "\0", "\0");
// 输出调试信息包括设置的槽位ID和大小
ereport(DEBUG1, (errmodule(MOD_CACHE), errmsg("set orc data block slot(%d), size(%lu)", slotId, size)));
return;
@ -714,6 +843,9 @@ void DataCacheMgr::SetOBSDataBlockValue(
* @IN slotId: slot id
* @See also:
*/
// 函数作用:获取压缩锁
// 参数:
// - slotId: 槽位ID
void DataCacheMgr::AcquireCompressLock(CacheSlotId_t slotId)
{
return m_cache_mgr->AcquireCompressLock(slotId);
@ -724,6 +856,9 @@ void DataCacheMgr::AcquireCompressLock(CacheSlotId_t slotId)
* @IN slotId: slot id
* @See also:
*/
// 函数作用:释放压缩锁
// 参数:
// - slotId: 槽位ID
void DataCacheMgr::RealeseCompressLock(CacheSlotId_t slotId)
{
return m_cache_mgr->RealeseCompressLock(slotId);
@ -736,12 +871,17 @@ void DataCacheMgr::RealeseCompressLock(CacheSlotId_t slotId)
* @Return: always success
* @See also:
*/
// 函数作用cstore预取读取完成的回调函数
// 参数:
// - aioDesc: aio描述符
// - res: 读取到的字节数
// 返回值始终返回成功0
int CompltrReadCUReq(void* aioDesc, long res)
{
AioDispatchCUDesc_t* desc = (AioDispatchCUDesc_t*)aioDesc;
START_CRIT_SECTION();
// 检查读取结果是否与期望的CU大小一致
if (res != desc->cuDesc.size) {
*(desc->cuDesc.io_error) = true;
ereport(WARNING,
@ -752,11 +892,13 @@ int CompltrReadCUReq(void* aioDesc, long res)
} else {
*(desc->cuDesc.io_error) = false;
}
// 断言缓存槽位ID有效
Assert(IsValidCacheSlotID(desc->cuDesc.slotId));
// 拥有CULWLock读写锁的所有权
CUCache->CULWLockOwn(desc->cuDesc.slotId);
// 完成对数据块的IO操作
CUCache->DataBlockCompleteIO(desc->cuDesc.slotId);
// 输出调试信息包括槽位ID
ereport(DEBUG1, (errmodule(MOD_ADIO), errmsg("CompltrReadCUReq!slotid(%d) ", desc->cuDesc.slotId)));
END_CRIT_SECTION();
@ -772,11 +914,19 @@ int CompltrReadCUReq(void* aioDesc, long res)
* @Return: always success
* @See also:
*/
/**
* @Description: cstore回写完成的回调函数
* @Param[IN] aioDesc: aio描述符
* @Param[IN] res:
* @Return: 0
* @See also:
*/
int CompltrWriteCUReq(void* aioDesc, long res)
{
AioDispatchCUDesc_t* desc = (AioDispatchCUDesc_t*)aioDesc;
// 开启临界区
START_CRIT_SECTION();
// 检查写入结果是否与期望的CU大小一致
if (res != desc->cuDesc.size) {
*(desc->cuDesc.io_error) = true;
ereport(WARNING,
@ -787,7 +937,9 @@ int CompltrWriteCUReq(void* aioDesc, long res)
} else {
*(desc->cuDesc.io_error) = false;
}
// 标记IO操作已完成
desc->cuDesc.io_finish = true;
// 结束临界区
END_CRIT_SECTION();
return 0;
@ -798,8 +950,10 @@ int CompltrWriteCUReq(void* aioDesc, long res)
* @IN CUSlotId: slot id
* @See also:
*/
//释放数据块的引用
void ReleaseORCBlock(CacheSlotId_t slot)
{
// 调用ORCCache的UnPinDataBlock函数解除对数据块的引用
ORCCache->UnPinDataBlock(slot);
}
@ -815,15 +969,31 @@ void ReleaseORCBlock(CacheSlotId_t slot)
* @See also: we do not want to ereport(error) when found error, becasue this function called by
* liborc(orc.HdfsCacheFileInputStream.read)
*/
/**
* @Description: ORC数据缓存块
* @Param[IN] rnode:
* @Param[IN] relid: ID
* @Param[IN] offset:
* @Param[IN] length:
* @Param[IN/OUT] found:
* @Param[IN/OUT] err_found:
* @Return: ORC数据槽位ID
* @See also:
* ereport(error)liborc(orc.HdfsCacheFileInputStream.read)
*/
//分配ORC数据缓存块
CacheSlotId_t ORCCacheAllocBlock(
RelFileNode* rnode, int32 relid, uint64 offset, uint64 length, bool& found, bool& err_found)
{
err_found = false;
int maxRetry = 3;
// 初始化ORC数据槽位标签
DataSlotTag dataSlotTag = ORCCache->InitORCSlotTag(rnode, relid, offset, length);
// 在ORC缓存中查找数据块
CacheSlotId_t slotId = ORCCache->FindDataBlock(&dataSlotTag, true);
if (IsValidCacheSlotID(slotId)) {
found = true;
// 预留数据块
} else {
found = false;
slotId = ORCCache->ReserveDataBlock(&dataSlotTag, length, found);
@ -831,9 +1001,11 @@ CacheSlotId_t ORCCacheAllocBlock(
while (found) {
if (ORCCache->DataBlockWaitIO(slotId)) {
// 等待IO操作完成后解除对数据块的引用
ORCCache->UnPinDataBlock(slotId);
if (maxRetry-- <= 0) {
err_found = true;
// 输出日志报告分配ORC数据时发生错误的信息
ereport(LOG,
(errmodule(MOD_ORC),
errmsg("wait IO find an error when allocate orc data, slotID(%d), spcID(%u), dbID(%u), "
@ -847,6 +1019,7 @@ CacheSlotId_t ORCCacheAllocBlock(
length)));
break;
} else {
// 重新预留数据块
slotId = ORCCache->ReserveDataBlock(&dataSlotTag, length, found);
continue;
}
@ -863,6 +1036,11 @@ CacheSlotId_t ORCCacheAllocBlock(
* @Return: true for the session can evict the cache.
* false for some other session is updating it concurrently.
*/
/**
* @Description: OBS重新加载数据
* @Param[IN] slotID: ID
* @Return: truefalse
*/
bool OBSCacheRenewBlock(CacheSlotId_t slotID)
{
return (OBSCache->ReserveDataBlockWithSlotId(slotID));
@ -880,62 +1058,82 @@ bool OBSCacheRenewBlock(CacheSlotId_t slotID)
* @See also: we do not want to ereport(error) when found error, becasue this function called by
* liborc(orc.HdfsCacheFileInputStream.read)
*/
//在ORC缓存中分配一个数据缓存块
CacheSlotId_t OBSCacheAllocBlock(const char* hostName, const char* bucketName, const char* prefixName, uint64 offset,
uint64 length, bool& found, bool& err_found)
{
Assert(hostName && bucketName && prefixName);
// 获取主机名、存储桶名称和文件前缀名的长度
uint32 hostNameLen = strlen(hostName);
uint32 buckectNameLen = strlen(bucketName);
uint32 prefixNameLen = strlen(prefixName);
// 计算主机名和存储桶名称的哈希值
uint32 hostNameHash = string_hash((void*)hostName, hostNameLen + 1);
uint32 bucketNameHash = string_hash((void*)bucketName, buckectNameLen + 1);
// 计算文件前半部分和后半部分的哈希值
uint32 fileFirstHalfHash = string_hash((void*)prefixName, prefixNameLen >> 1);
uint32 fileSecondHalfHash =
string_hash((void*)(prefixName + (prefixNameLen >> 1)), strlen(prefixName + (prefixNameLen >> 1)) + 1);
// 初始化DataSlotTag结构体填入哈希值和偏移量、长度等信息
err_found = false;
int maxRetry = 3;
DataSlotTag dataSlotTag =
OBSCache->InitOBSSlotTag(hostNameHash, bucketNameHash, fileFirstHalfHash, fileSecondHalfHash, offset, length);
// 尝试从缓存中查找数据槽位
CacheSlotId_t slotId = OBSCache->FindDataBlock(&dataSlotTag, true);
if (IsValidCacheSlotID(slotId)) {
found = true;
found = true;// 找到了有效的缓存槽位
} else {
found = false;
found = false;// 没有找到有效的缓存槽位
// 预留一个数据块
slotId = OBSCache->ReserveDataBlock(&dataSlotTag, length, found);
}
// 如果缓存槽位正在被其他会话并发更新则等待IO完成
while (found) {
if (OBSCache->DataBlockWaitIO(slotId)) {
OBSCache->UnPinDataBlock(slotId);
OBSCache->UnPinDataBlock(slotId);// 解锁缓存块
if (maxRetry-- > 0) {
// 再次尝试预留数据块
slotId = OBSCache->ReserveDataBlock(&dataSlotTag, length, found);
continue;
} else {
err_found = true;
err_found = true;// 达到最大重试次数,表示发生错误
break;
}
} else {
break;
}
}
return slotId;
return slotId;// 返回缓存槽位ID
}
/**
* @Description: ID获取ORC数据缓存块的指针
* @Param[IN] slot: ID
* @Return: ORC数据缓存块的指针
*/
OrcDataValue* ORCCacheGetBlock(CacheSlotId_t slot)
{
return ORCCache->GetORCDataBuf(slot);
}
/**
* @Description: ORC数据缓存块IO操作完成
* @Param[IN] slotId: ID
* @Param[IN] buffer:
* @Param[IN] size:
*/
void ORCCacheSetBlock(CacheSlotId_t slotId, const void* buffer, uint64 size)
{
ORCCache->SetORCDataBlockValue(slotId, buffer, size);
ORCCache->DataBlockCompleteIO(slotId);
}
/**
* @Description: OBS数据缓存块IO操作完成
* @Param[IN] slotId: ID
* @Param[IN] buffer:
* @Param[IN] size:
* @Param[IN] prefix:
* @Param[IN] dataDNA: DNA值
*/
void OBSCacheSetBlock(CacheSlotId_t slotId, const void* buffer, uint64 size, const char* prefix, const char* dataDNA)
{
OBSCache->SetOBSDataBlockValue(slotId, buffer, size, prefix, dataDNA);

File diff suppressed because it is too large Load Diff

View File

@ -62,121 +62,135 @@ static void open_lo_relation(void)
if (t_thrd.storage_cxt.lo_heap_r && t_thrd.storage_cxt.lo_index_r)
return; /* already open in current xact */
/* Arrange for the top xact to own these relation references */
/* 获取当前的资源拥有者 */
currentOwner = t_thrd.utils_cxt.CurrentResourceOwner;
PG_TRY();
{
/* 将顶层事务的资源拥有者设置为当前资源拥有者 */
t_thrd.utils_cxt.CurrentResourceOwner = t_thrd.utils_cxt.TopTransactionResourceOwner;
/* Use RowExclusiveLock since we might either read or write */
/* 使用RowExclusiveLock因为我们可能需要读取或写入 */
if (t_thrd.storage_cxt.lo_heap_r == NULL)
/* 打开LargeObjectRelationId对应的heap关系并使用RowExclusiveLock */
t_thrd.storage_cxt.lo_heap_r = heap_open(LargeObjectRelationId, RowExclusiveLock);
if (t_thrd.storage_cxt.lo_index_r == NULL)
/* 打开LargeObjectLOidPNIndexId对应的index关系并使用RowExclusiveLock */
t_thrd.storage_cxt.lo_index_r = index_open(LargeObjectLOidPNIndexId, RowExclusiveLock);
}
PG_CATCH();
{
/* Ensure CurrentResourceOwner is restored on error */
/* 在错误发生时恢复CurrentResourceOwner的值 */
t_thrd.utils_cxt.CurrentResourceOwner = currentOwner;
PG_RE_THROW();
}
PG_END_TRY();
/* 恢复CurrentResourceOwner的值 */
t_thrd.utils_cxt.CurrentResourceOwner = currentOwner;
}
/*
* Clean up at main transaction end
*/
void close_lo_relation(bool isCommit)
{
if (t_thrd.storage_cxt.lo_heap_r || t_thrd.storage_cxt.lo_index_r) {
if (t_thrd.storage_cxt.lo_heap_r || t_thrd.storage_cxt.lo_index_r) { // 如果lo_heap_r或lo_index_r不为空
/*
* Only bother to close if committing; else abort cleanup will handle
* it
*
*/
if (isCommit) {
ResourceOwner currentOwner;
if (isCommit) { // 如果是提交事务
ResourceOwner currentOwner; // 当前资源拥有者
currentOwner = t_thrd.utils_cxt.CurrentResourceOwner;
PG_TRY();
currentOwner = t_thrd.utils_cxt.CurrentResourceOwner; // 保存当前资源拥有者
PG_TRY(); // 尝试执行以下代码块
{
t_thrd.utils_cxt.CurrentResourceOwner = t_thrd.utils_cxt.TopTransactionResourceOwner;
t_thrd.utils_cxt.CurrentResourceOwner =
t_thrd.utils_cxt.TopTransactionResourceOwner; // 设置当前资源拥有者为顶级事务资源拥有者
if (t_thrd.storage_cxt.lo_index_r)
index_close(t_thrd.storage_cxt.lo_index_r, NoLock);
if (t_thrd.storage_cxt.lo_heap_r)
heap_close(t_thrd.storage_cxt.lo_heap_r, NoLock);
if (t_thrd.storage_cxt.lo_index_r) // 如果lo_index_r不为空
index_close(t_thrd.storage_cxt.lo_index_r, NoLock); // 关闭索引关系
if (t_thrd.storage_cxt.lo_heap_r) // 如果lo_heap_r不为空
heap_close(t_thrd.storage_cxt.lo_heap_r, NoLock); // 关闭堆关系
}
PG_CATCH();
PG_CATCH(); // 捕获异常
{
/* Ensure CurrentResourceOwner is restored on error */
t_thrd.utils_cxt.CurrentResourceOwner = currentOwner;
PG_RE_THROW();
/* 确保错误时恢复CurrentResourceOwner */
t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; // 恢复当前资源拥有者
PG_RE_THROW(); // 重新抛出异常
}
PG_END_TRY();
t_thrd.utils_cxt.CurrentResourceOwner = currentOwner;
t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; // 恢复当前资源拥有者
}
t_thrd.storage_cxt.lo_heap_r = NULL;
t_thrd.storage_cxt.lo_index_r = NULL;
t_thrd.storage_cxt.lo_heap_r = NULL; // 将lo_heap_r置为NULL
t_thrd.storage_cxt.lo_index_r = NULL; // 将lo_index_r置为NULL
}
}
/*
* Same as pg_largeobject.c's LargeObjectExists(), except snapshot to
* read with can be specified.
*/
static bool myLargeObjectExists(Oid loid, Snapshot snapshot)
{
Relation pg_lo_meta;
ScanKeyData skey[1];
SysScanDesc sd;
HeapTuple tuple;
bool retval = false;
Relation pg_lo_meta; // LargeObjectMetadataRelationId 对应的关系
ScanKeyData skey[1]; // 扫描键
SysScanDesc sd; // 系统扫描描述符
HeapTuple tuple; // 堆元组
bool retval = false; // 返回值默认为false
// 初始化扫描键设置ObjectIdAttributeNumber对象ID属性编号等于loid
ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(loid));
// 打开LargeObjectMetadataRelationId对应的heap关系并使用AccessShareLock进行锁定
pg_lo_meta = heap_open(LargeObjectMetadataRelationId, AccessShareLock);
// 开始对pg_lo_meta关系进行系统扫描使用LargeObjectMetadataOidIndexId对应的索引
// 忽略无效元组指定快照为snapshot扫描键数量为1扫描键为skey
sd = systable_beginscan(pg_lo_meta, LargeObjectMetadataOidIndexId, true, snapshot, 1, skey);
// 获取下一个堆元组
tuple = systable_getnext(sd);
if (HeapTupleIsValid(tuple))
retval = true;
retval = true; // 如果堆元组有效则将retval设为true
// 结束系统扫描
systable_endscan(sd);
// 关闭pg_lo_meta关系释放AccessShareLock锁定的资源
heap_close(pg_lo_meta, AccessShareLock);
return retval;
return retval; // 返回retval值
}
/*
* Extract data field from a pg_largeobject tuple, detoasting if needed
* and verifying that the length is sane. Returns data pointer (a bytea *),
* data length, and an indication of whether to pfree the data pointer.
*/
static void getdatafield(Form_pg_largeobject tuple, bytea** pdatafield, int* plen, bool* pfreeit)
static void getdatafield(Form_pg_largeobject tuple, bytea **pdatafield, int *plen, bool *pfreeit)
{
bytea* datafield = NULL;
int len;
bool freeit = false;
bytea *datafield = NULL; // 数据域指针
int len; // 数据域长度
bool freeit = false; // 是否需要释放数据域
datafield = &(tuple->data); /* see note at top of file */
if (VARATT_IS_EXTENDED(datafield)) {
datafield = (bytea*)heap_tuple_untoast_attr((struct varlena*)datafield);
freeit = true;
datafield = &(tuple->data); /* 见文件顶部的注释 */
if (VARATT_IS_EXTENDED(datafield)) { // 如果数据域使用扩展存储
datafield =
(bytea *)heap_tuple_untoast_attr((struct varlena *)datafield); // 将扩展存储的数据域解压成栈上数据域
freeit = true; // 设置需要释放数据域
}
len = VARSIZE(datafield) - VARHDRSZ;
if (len < 0 || len > LOBLKSIZE)
ereport(ERROR,
(errcode(ERRCODE_DATA_CORRUPTED),
errmsg("pg_largeobject entry for OID %u, page %d has invalid data field size %d",
tuple->loid,
tuple->pageno,
len)));
*pdatafield = datafield;
*plen = len;
*pfreeit = freeit;
len = VARSIZE(datafield) - VARHDRSZ; // 获取数据域长度
if (len < 0 || len > LOBLKSIZE) // 如果数据域长度不正确
ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED),
errmsg("pg_largeobject entry for OID %u, page %d has invalid data field size %d", tuple->loid,
tuple->pageno, len)));
*pdatafield = datafield; // 返回数据域指针
*plen = len; // 返回数据域长度
*pfreeit = freeit; // 返回是否需要释放数据域
}
/*
@ -197,6 +211,7 @@ Oid inv_create(Oid lobjId)
/*
* Create a new largeobject with empty data pages
largeobject型对象
*/
lobjId_new = LargeObjectCreate(lobjId);
@ -230,51 +245,62 @@ Oid inv_create(Oid lobjId)
* and subsidiary data are allocated in the specified memory context,
* which must be suitably long-lived for the caller's purposes.
*/
LargeObjectDesc* inv_open(Oid lobjId, int flags, MemoryContext mcxt)
LargeObjectDesc *inv_open(Oid lobjId, int flags, MemoryContext mcxt)
{
LargeObjectDesc* retval = NULL;
LargeObjectDesc *retval = NULL; // 返回的LargeObjectDesc型对象描述符指针
retval = (LargeObjectDesc*)MemoryContextAlloc(mcxt, sizeof(LargeObjectDesc));
retval =
(LargeObjectDesc *)MemoryContextAlloc(mcxt, sizeof(LargeObjectDesc)); // 分配内存空间用于存储LargeObjectDesc型对象描述符
retval->id = lobjId;
retval->subid = GetCurrentSubTransactionId();
retval->offset = 0;
retval->id = lobjId; // 设置LargeObjectDesc型对象的OID
retval->subid = GetCurrentSubTransactionId(); // 设置当前子事务ID
retval->offset = 0; // 设置偏移量
if (flags & INV_WRITE) {
retval->snapshot = SnapshotNow;
retval->flags = IFS_WRLOCK | IFS_RDLOCK;
} else if (flags & INV_READ) {
if (flags & INV_WRITE) { // 如果标志包含INV_WRITE表示以写模式打开LargeObjectDesc型对象
retval->snapshot = SnapshotNow; // 使用当前快照
retval->flags = IFS_WRLOCK | IFS_RDLOCK; // 设置写锁和读锁标志
} else if (flags & INV_READ) { // 如果标志包含INV_READ表示以读模式打开LargeObjectDesc型对象
/*
* We must register the snapshot in TopTransaction's resowner, because
* it must stay alive until the LO is closed rather than until the
* current portal shuts down.
* TopTransaction的资源拥有者中注册快照
* LO关闭之前保持活动状态portal关闭时
*/
retval->snapshot = RegisterSnapshotOnOwner(GetActiveSnapshot(), t_thrd.utils_cxt.TopTransactionResourceOwner);
retval->flags = IFS_RDLOCK;
} else
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("invalid flags: %d", flags)));
retval->snapshot = RegisterSnapshotOnOwner(
GetActiveSnapshot(), t_thrd.utils_cxt.TopTransactionResourceOwner); // 在资源拥有者中注册快照
retval->flags = IFS_RDLOCK; // 设置读锁标志
} else {
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("invalid flags: %d", flags))); // 抛出错误,标志无效
}
/* Can't use LargeObjectExists here because it always uses SnapshotNow */
if (!myLargeObjectExists(lobjId, retval->snapshot))
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("large object %u does not exist", lobjId)));
// 不能使用LargeObjectExists因为它总是使用SnapshotNow
if (!myLargeObjectExists(lobjId, retval->snapshot)) { // 如果大型对象不存在
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT),
errmsg("large object %u does not exist", lobjId))); // 抛出错误,大型对象不存在
}
return retval;
return retval; // 返回LargeObjectDesc型对象描述符指针
}
/*
* Closes a large object descriptor previously made by inv_open(), and
* releases the long-term memory used by it.
*/
void inv_close(LargeObjectDesc* obj_desc)
void inv_close(LargeObjectDesc *obj_desc)
{
Assert(PointerIsValid(obj_desc));
/*
* obj_desc是否有效
*/
if (obj_desc->snapshot != SnapshotNow)
UnregisterSnapshotFromOwner(obj_desc->snapshot, t_thrd.utils_cxt.TopTransactionResourceOwner);
/*
* obj_desc的内存空间
*/
pfree(obj_desc);
}
/*
* Destroys an existing large object (not to be confused with a descriptor!)
*
@ -287,18 +313,19 @@ int inv_drop(Oid lobjId)
/*
* Delete any comments and dependencies on the large object
*/
object.classId = LargeObjectRelationId;
object.objectId = lobjId;
object.objectSubId = 0;
performDeletion(&object, DROP_CASCADE, 0);
object.classId = LargeObjectRelationId; // 设置对象类别为大型对象
object.objectId = lobjId; // 设置对象ID为目标大型对象的OID
object.objectSubId = 0; // 设置对象子ID为0
performDeletion(&object, DROP_CASCADE, 0); // 执行删除操作,级联删除依赖项
/*
* Advance command counter so that tuple removal will be seen by later
* large-object operations in this transaction.
* 便
*/
CommandCounterIncrement();
CommandCounterIncrement();// 提升命令计数器
return 1;
return 1;// 返回1表示删除成功
}
/*
@ -314,14 +341,24 @@ static uint32 inv_getsize(LargeObjectDesc* obj_desc)
SysScanDesc sd;
HeapTuple tuple;
/*
* obj_desc是否有效
*/
Assert(PointerIsValid(obj_desc));
/* 打开pg_largeobject关系 */
open_lo_relation();
/*
*
*/
ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id));
sd = systable_beginscan_ordered(
t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 1, skey);
/*
* pg_largeobject索引
*/
sd = systable_beginscan_ordered(t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 1,
skey);
/*
* Because the pg_largeobject index is on both loid and pageno, but we
@ -336,136 +373,152 @@ static uint32 inv_getsize(LargeObjectDesc* obj_desc)
int len;
bool pfreeit = false;
/* 如果存在null值则错误 */
if (HeapTupleHasNulls(tuple)) /* paranoia */
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("null field found in pg_largeobject")));
/* 获取元组中的数据 */
data = (Form_pg_largeobject)GETSTRUCT(tuple);
getdatafield(data, &datafield, &len, &pfreeit);
/* 计算最后一个字节的位置 */
lastbyte = data->pageno * LOBLKSIZE + len;
/* 释放datafield指向的内存空间 */
if (pfreeit)
pfree(datafield);
}
/*
*
*/
systable_endscan_ordered(sd);
return lastbyte;
}
int inv_seek(LargeObjectDesc* obj_desc, int offset, int whence)
int inv_seek(LargeObjectDesc *obj_desc, int offset, int whence)
{
Assert(PointerIsValid(obj_desc));
Assert(PointerIsValid(obj_desc)); // 断言大型对象描述符指针有效
switch (whence) {
case SEEK_SET:
if (offset < 0)
case SEEK_SET: // 从文件开头开始计算offset
if (offset < 0) // 如果offset为负数则抛出错误
ereport(ERROR, (errcode_for_file_access(), errmsg("invalid seek offset: %d", offset)));
obj_desc->offset = offset;
obj_desc->offset = offset; // 设置偏移量为offset
break;
case SEEK_CUR:
if (offset < 0 && obj_desc->offset < ((uint32)(-offset)))
case SEEK_CUR: // 相对于当前位置计算offset
if (offset < 0 &&
obj_desc->offset < ((uint32)(-offset))) // 如果offset为负数且偏移量小于(-offset),则抛出错误
ereport(ERROR, (errcode_for_file_access(), errmsg("invalid seek offset: %d", offset)));
obj_desc->offset += offset;
obj_desc->offset += offset; // 偏移量增加offset
break;
case SEEK_END: {
uint32 size = inv_getsize(obj_desc);
if (offset < 0 && size < ((uint32)(-offset)))
case SEEK_END: { // 相对于文件末尾计算offset
uint32 size = inv_getsize(obj_desc); // 获取大型对象的大小
if (offset < 0 && size < ((uint32)(-offset))) // 如果offset为负数且对象大小小于(-offset),则抛出错误
ereport(ERROR, (errcode_for_file_access(), errmsg("invalid seek offset: %d", offset)));
obj_desc->offset = size + offset;
obj_desc->offset = size + offset; // 偏移量设置为对象大小加offset
} break;
default:
ereport(ERROR, (errcode_for_file_access(), errmsg("invalid whence: %d", whence)));
ereport(ERROR, (errcode_for_file_access(),
errmsg("invalid whence: %d", whence))); // 如果whence参数无效则抛出错误
}
return obj_desc->offset; // 返回当前偏移量
}
int inv_tell(LargeObjectDesc *obj_desc)
{
// 检查obj_desc是否有效
Assert(PointerIsValid(obj_desc));
// 返回obj_desc中的offset字段值
return obj_desc->offset;
}
int inv_tell(LargeObjectDesc* obj_desc)
{
Assert(PointerIsValid(obj_desc));
return obj_desc->offset;
}
int inv_read(LargeObjectDesc* obj_desc, char* buf, int nbytes)
int inv_read(LargeObjectDesc *obj_desc, char *buf, int nbytes)
{
Assert(PointerIsValid(obj_desc));
int nread = 0;
Assert(PointerIsValid(obj_desc)); // 断言大型对象描述符指针有效
int nread = 0; // 已读取的字节数
int n;
int off;
int len;
int32 pageno = (int32)(obj_desc->offset / LOBLKSIZE);
int32 pageno = (int32)(obj_desc->offset / LOBLKSIZE); // 根据偏移量计算页号
uint32 pageoff;
ScanKeyData skey[2];
SysScanDesc sd;
HeapTuple tuple;
errno_t rc = EOK;
Assert(buf != NULL);
Assert(buf != NULL); // 断言缓冲区指针有效
if (nbytes <= 0) {
if (nbytes <= 0) { // 如果要读取的字节数小于等于0则直接返回0
return 0;
}
open_lo_relation();
open_lo_relation(); // 打开大型对象关系表
// 初始化扫描键值
ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id));
ScanKeyInit(&skey[1], Anum_pg_largeobject_pageno, BTGreaterEqualStrategyNumber, F_INT4GE, Int32GetDatum(pageno));
sd = systable_beginscan_ordered(
t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2, skey);
// 开始有序扫描大型对象索引表
sd = systable_beginscan_ordered(t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2,
skey);
while ((tuple = systable_getnext_ordered(sd, ForwardScanDirection)) != NULL) {
Form_pg_largeobject data;
bytea* datafield = NULL;
bytea *datafield = NULL;
bool pfreeit = false;
if (HeapTupleHasNulls(tuple)) /* paranoia */
if (HeapTupleHasNulls(tuple)) // 如果元组有空字段,抛出错误
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("null field found in pg_largeobject")));
data = (Form_pg_largeobject)GETSTRUCT(tuple);
data = (Form_pg_largeobject)GETSTRUCT(tuple); // 获取大型对象元组的数据
/*
* We expect the indexscan will deliver pages in order. However,
* there may be missing pages if the LO contains unwritten "holes". We
* want missing sections to read out as zeroes.
*/
pageoff = ((uint32)data->pageno) * LOBLKSIZE;
if (pageoff > obj_desc->offset) {
pageoff = ((uint32)data->pageno) * LOBLKSIZE; // 计算页的偏移量
if (pageoff > obj_desc->offset) { // 如果页的偏移量大于当前偏移量,则说明存在缺失的部分,将其读出为零
n = pageoff - obj_desc->offset;
n = (n <= (nbytes - nread)) ? n : (nbytes - nread);
rc = memset_s(buf + nread, n, '\0', n);
rc = memset_s(buf + nread, n, '\0', n); // 将buf中的一部分置为零
securec_check(rc, "", "");
nread += n;
obj_desc->offset += n;
}
if (nread < nbytes) {
off = (int)(obj_desc->offset - pageoff);
if (off < 0 || off >= LOBLKSIZE) {
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT),
errmsg("invalid offset num:%d", off)));
off = (int)(obj_desc->offset - pageoff); // 计算在当前页内的偏移量
if (off < 0 || off >= LOBLKSIZE) { // 如果偏移量不合法,抛出错误
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("invalid offset num:%d", off)));
}
getdatafield(data, &datafield, &len, &pfreeit);
if (len > off) {
getdatafield(data, &datafield, &len, &pfreeit); // 获取大型对象数据字段
if (len > off) { // 如果数据字段内有足够的数据可供读取
n = len - off;
n = (n <= (nbytes - nread)) ? n : (nbytes - nread);
rc = memcpy_s(buf + nread, n, VARDATA(datafield) + off, n);
rc = memcpy_s(buf + nread, n, VARDATA(datafield) + off, n); // 将数据复制到buf中
securec_check(rc, "", "");
nread += n;
obj_desc->offset += n;
}
if (pfreeit)
pfree(datafield);
pfree(datafield); // 释放大型对象数据字段的内存
}
if (nread >= nbytes)
break;
}
systable_endscan_ordered(sd);
systable_endscan_ordered(sd); // 结束扫描
return nread;
return nread; // 返回已读取的字节数
}
void check_obj_desc(const LargeObjectDesc* obj_desc)
{
/* enforce writability because snapshot is probably wrong otherwise */
@ -480,56 +533,57 @@ void check_obj_desc(const LargeObjectDesc* obj_desc)
ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("large object %u was already dropped", obj_desc->id)));
}
int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes)
int inv_write(LargeObjectDesc *obj_desc, const char *buf, int nbytes)
{
if (nbytes <= 0) {
return 0;
}
int nwritten = 0;
int n;
int off;
int len;
int32 pageno;
ScanKeyData skey[2];
SysScanDesc sd;
HeapTuple oldtuple = NULL;
Form_pg_largeobject olddata = NULL;
bool neednextpage = true;
bytea* datafield = NULL;
bool pfreeit = false;
int nwritten = 0; // 已写入的字节数
int n; // 当前写入的字节数
int off; // 偏移量,表示当前写入的位置在页面中的偏移量
int len; // 旧页中的有效数据长度
int32 pageno; // 页面号
ScanKeyData skey[2]; // 扫描键
SysScanDesc sd; // 系统扫描描述符
HeapTuple oldtuple = NULL; // 旧元组
Form_pg_largeobject olddata = NULL; // 旧的pg_largeobject元组
bool neednextpage = true; // 是否需要获取下一个已存在的页面
bytea *datafield = NULL; // 数据字段
bool pfreeit = false; // 是否需要释放datafield
struct {
bytea hdr;
char data[LOBLKSIZE]; /* make struct big enough */
int32 align_it; /* ensure struct is aligned well enough */
} workbuf;
char* workb = VARDATA(&workbuf.hdr);
HeapTuple newtup;
Datum values[Natts_pg_largeobject];
bool nulls[Natts_pg_largeobject];
bool replace[Natts_pg_largeobject];
CatalogIndexState indstate;
char data[LOBLKSIZE]; /* make struct big enough */
int32 align_it; /* ensure struct is aligned well enough */
} workbuf; // 工作缓冲区
char *workb = VARDATA(&workbuf.hdr); // 缓冲区数据部分的指针
HeapTuple newtup; // 新元组
Datum values[Natts_pg_largeobject]; // 插入的值
bool nulls[Natts_pg_largeobject]; // 是否为NULL
bool replace[Natts_pg_largeobject]; // 是否替换
CatalogIndexState indstate; // 目录索引状态
errno_t rc;
if (unlikely(!PointerIsValid(obj_desc))) {
return 0;
}
pageno = (int32)(obj_desc->offset / LOBLKSIZE);
pageno = (int32)(obj_desc->offset / LOBLKSIZE); // 计算页面号
rc = memset_s(&workbuf, sizeof(workbuf), 0, sizeof(workbuf));
rc = memset_s(&workbuf, sizeof(workbuf), 0, sizeof(workbuf)); // 清零工作缓冲区
securec_check(rc, "\0", "\0");
Assert(buf != NULL);
Assert(buf != NULL); // 断言buf不为空
check_obj_desc(obj_desc);
open_lo_relation();
check_obj_desc(obj_desc); // 检查对象描述符
open_lo_relation(); // 打开large object关系
indstate = CatalogOpenIndexes(t_thrd.storage_cxt.lo_heap_r);
indstate = CatalogOpenIndexes(t_thrd.storage_cxt.lo_heap_r); // 打开目录索引
// 初始化扫描键
ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id));
ScanKeyInit(&skey[1], Anum_pg_largeobject_pageno, BTGreaterEqualStrategyNumber, F_INT4GE, Int32GetDatum(pageno));
sd = systable_beginscan_ordered(
t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2, skey);
// 开始有序扫描
sd = systable_beginscan_ordered(t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2,
skey);
while (nwritten < nbytes) {
/*
@ -556,8 +610,8 @@ int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes)
*
* First, load old data into workbuf
*/
getdatafield(olddata, &datafield, &len, &pfreeit);
rc = memcpy_s(workb, len, VARDATA(datafield), len);
getdatafield(olddata, &datafield, &len, &pfreeit); // 获取旧数据
rc = memcpy_s(workb, len, VARDATA(datafield), len); // 拷贝旧数据到工作缓冲区
securec_check(rc, "", "");
if (pfreeit)
pfree(datafield);
@ -565,25 +619,25 @@ int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes)
/*
* Fill any hole
*/
off = (int)(obj_desc->offset % LOBLKSIZE);
off = (int)(obj_desc->offset % LOBLKSIZE); // 计算偏移量
if (off > len) {
rc = memset_s(workb + len, off - len, '\0', off - len);
rc = memset_s(workb + len, off - len, '\0', off - len); // 填充空洞
securec_check(rc, "", "");
}
/*
* Insert appropriate portion of new data
*/
n = LOBLKSIZE - off;
n = (n <= (nbytes - nwritten)) ? n : (nbytes - nwritten);
rc = memcpy_s(workb + off, n, buf + nwritten, n);
n = LOBLKSIZE - off; // 可写入的长度
n = (n <= (nbytes - nwritten)) ? n : (nbytes - nwritten); // 取较小的长度
rc = memcpy_s(workb + off, n, buf + nwritten, n); // 拷贝数据到工作缓冲区
securec_check(rc, "", "");
nwritten += n;
obj_desc->offset += n;
nwritten += n; // 更新已写入的字节数
obj_desc->offset += n; // 更新偏移量
off += n;
/* compute valid length of new page */
len = (len >= off) ? len : off;
SET_VARSIZE(&workbuf.hdr, len + VARHDRSZ);
SET_VARSIZE(&workbuf.hdr, len + VARHDRSZ); // 设置新页的长度
/*
* Form and insert updated tuple
@ -596,11 +650,11 @@ int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes)
securec_check(rc, "", "");
values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf);
replace[Anum_pg_largeobject_data - 1] = true;
newtup =
heap_modify_tuple(oldtuple, RelationGetDescr(t_thrd.storage_cxt.lo_heap_r), values, nulls, replace);
simple_heap_update(t_thrd.storage_cxt.lo_heap_r, &newtup->t_self, newtup);
CatalogIndexInsert(indstate, newtup);
heap_freetuple(newtup);
newtup = heap_modify_tuple(oldtuple, RelationGetDescr(t_thrd.storage_cxt.lo_heap_r), values, nulls,
replace); // 修改元组
simple_heap_update(t_thrd.storage_cxt.lo_heap_r, &newtup->t_self, newtup); // 更新堆中的元组
CatalogIndexInsert(indstate, newtup); // 在索引中插入元组
heap_freetuple(newtup); // 释放新元组
/*
* We're done with this old page.
@ -614,24 +668,24 @@ int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes)
*
* First, fill any hole
*/
off = (int)(obj_desc->offset % LOBLKSIZE);
off = (int)(obj_desc->offset % LOBLKSIZE); // 计算偏移量
if (off > 0) {
rc = memset_s(workb, off, '\0', off);
rc = memset_s(workb, off, '\0', off); // 填充空洞
securec_check(rc, "", "");
}
/*
* Insert appropriate portion of new data
*/
n = LOBLKSIZE - off;
n = (n <= (nbytes - nwritten)) ? n : (nbytes - nwritten);
rc = memcpy_s(workb + off, n, buf + nwritten, n);
n = LOBLKSIZE - off; // 可写入的长度
n = (n <= (nbytes - nwritten)) ? n : (nbytes - nwritten); // 取较小的长度
rc = memcpy_s(workb + off, n, buf + nwritten, n); // 拷贝数据到工作缓冲区
securec_check(rc, "", "");
nwritten += n;
obj_desc->offset += n;
nwritten += n; // 更新已写入的字节数
obj_desc->offset += n; // 更新偏移量
/* compute valid length of new page */
len = off + n;
SET_VARSIZE(&workbuf.hdr, len + VARHDRSZ);
SET_VARSIZE(&workbuf.hdr, len + VARHDRSZ); // 设置新页的长度
/*
* Form and insert updated tuple
@ -643,171 +697,142 @@ int inv_write(LargeObjectDesc* obj_desc, const char* buf, int nbytes)
values[Anum_pg_largeobject_loid - 1] = ObjectIdGetDatum(obj_desc->id);
values[Anum_pg_largeobject_pageno - 1] = Int32GetDatum(pageno);
values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf);
newtup = heap_form_tuple(t_thrd.storage_cxt.lo_heap_r->rd_att, values, nulls);
(void)simple_heap_insert(t_thrd.storage_cxt.lo_heap_r, newtup);
CatalogIndexInsert(indstate, newtup);
heap_freetuple(newtup);
newtup = heap_form_tuple(t_thrd.storage_cxt.lo_heap_r->rd_att, values, nulls); // 创建新元组
(void)simple_heap_insert(t_thrd.storage_cxt.lo_heap_r, newtup); // 插入新元组
CatalogIndexInsert(indstate, newtup); // 在索引中插入元组
heap_freetuple(newtup); // 释放新元组
}
pageno++;
}
systable_endscan_ordered(sd);
systable_endscan_ordered(sd); // 结束有序扫描
CatalogCloseIndexes(indstate);
CatalogCloseIndexes(indstate); // 关闭目录索引
/*
* Advance command counter so that my tuple updates will be seen by later
* large-object operations in this transaction.
*/
CommandCounterIncrement();
CommandCounterIncrement(); // 提升命令计数器,以确保后续事务可以看到我更新的元组
return nwritten;
}
void inv_truncate(LargeObjectDesc* obj_desc, int len)
void inv_truncate(LargeObjectDesc *obj_desc, int len)
{
int32 pageno = (int32)(len / LOBLKSIZE);
int32 pageno = (int32)(len / LOBLKSIZE); // 计算页号
int off;
ScanKeyData skey[2];
SysScanDesc sd;
HeapTuple oldtuple;
Form_pg_largeobject olddata;
struct {
bytea hdr;
char data[LOBLKSIZE]; /* make struct big enough */
int32 align_it; /* ensure struct is aligned well enough */
char data[LOBLKSIZE]; // 用于存储数据的缓冲区
int32 align_it;
} workbuf;
char* workb = VARDATA(&workbuf.hdr);
char *workb = VARDATA(&workbuf.hdr);
HeapTuple newtup;
Datum values[Natts_pg_largeobject] = {0, 0, 0};
bool nulls[Natts_pg_largeobject] = {false, false, false};
bool nulls[Natts_pg_largeobject] = {false, false, false};
bool replace[Natts_pg_largeobject] = {false, false, false};
CatalogIndexState indstate;
errno_t rc;
rc = memset_s(&workbuf, sizeof(workbuf), 0, sizeof(workbuf));
securec_check(rc, "\0", "\0");
Assert(PointerIsValid(obj_desc));
check_obj_desc(obj_desc);
open_lo_relation();
rc = memset_s(&workbuf, sizeof(workbuf), 0, sizeof(workbuf)); // 将workbuf缓冲区清零
securec_check(rc, "\0", "\0"); // 检查内存操作是否成功
indstate = CatalogOpenIndexes(t_thrd.storage_cxt.lo_heap_r);
Assert(PointerIsValid(obj_desc)); // 检查obj_desc指针是否有效
/*
* Set up to find all pages with desired loid and pageno >= target
*/
check_obj_desc(obj_desc); // 检查obj_desc的描述是否有效
open_lo_relation(); // 打开大型对象关系表
indstate = CatalogOpenIndexes(t_thrd.storage_cxt.lo_heap_r); // 打开索引
/* 设置扫描键查找指定loid和pageno大于等于目标的所有页 */
ScanKeyInit(&skey[0], Anum_pg_largeobject_loid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(obj_desc->id));
ScanKeyInit(&skey[1], Anum_pg_largeobject_pageno, BTGreaterEqualStrategyNumber, F_INT4GE, Int32GetDatum(pageno));
sd = systable_beginscan_ordered(
t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2, skey);
sd = systable_beginscan_ordered(t_thrd.storage_cxt.lo_heap_r, t_thrd.storage_cxt.lo_index_r, obj_desc->snapshot, 2,
skey); // 开始有序扫描
/*
* If possible, get the page the truncation point is in. The truncation
* point may be beyond the end of the LO or in a hole.
*/
olddata = NULL;
if ((oldtuple = systable_getnext_ordered(sd, ForwardScanDirection)) != NULL) {
if (HeapTupleHasNulls(oldtuple)) /* paranoia */
if (HeapTupleHasNulls(oldtuple)) // 检查旧元组是否有空值(出于谨慎)
ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("null field found in pg_largeobject")));
olddata = (Form_pg_largeobject)GETSTRUCT(oldtuple);
Assert(olddata->pageno >= pageno);
Assert(olddata->pageno >= pageno); // 检查pageno是否大于等于目标页号
}
/*
* If we found the page of the truncation point we need to truncate the
* data in it. Otherwise if we're in a hole, we need to create a page to
* mark the end of data.
*/
if (olddata != NULL && olddata->pageno == pageno) {
/* First, load old data into workbuf */
bytea* datafield = NULL;
bytea *datafield = NULL;
bool pfreeit = false;
int pagelen;
/* 将旧数据加载到workbuf中 */
getdatafield(olddata, &datafield, &pagelen, &pfreeit);
rc = memcpy_s(workb, pagelen, VARDATA(datafield), pagelen);
securec_check(rc, "", "");
if (pfreeit)
pfree(datafield);
/*
* Fill any hole
*/
off = len % LOBLKSIZE;
/* 填充任何空隙 */
if (off > pagelen) {
rc = memset_s(workb + pagelen, off - pagelen, '\0', off - pagelen);
securec_check(rc, "", "");
}
/* compute length of new page */
SET_VARSIZE(&workbuf.hdr, off + VARHDRSZ);
SET_VARSIZE(&workbuf.hdr, off + VARHDRSZ); // 计算新页的长度
/*
* Form and insert updated tuple
*/
values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf);
replace[Anum_pg_largeobject_data - 1] = true;
newtup = heap_modify_tuple(oldtuple, RelationGetDescr(t_thrd.storage_cxt.lo_heap_r), values, nulls, replace);
simple_heap_update(t_thrd.storage_cxt.lo_heap_r, &newtup->t_self, newtup);
CatalogIndexInsert(indstate, newtup);
heap_freetuple(newtup);
newtup = heap_modify_tuple(oldtuple, RelationGetDescr(t_thrd.storage_cxt.lo_heap_r), values, nulls,
replace); // 修改旧的元组
simple_heap_update(t_thrd.storage_cxt.lo_heap_r, &newtup->t_self, newtup); // 更新堆上的元组
CatalogIndexInsert(indstate, newtup); // 插入索引
heap_freetuple(newtup); // 释放元组内存
} else {
/*
* If the first page we found was after the truncation point, we're in
* a hole that we'll fill, but we need to delete the later page
* because the loop below won't visit it again.
*/
if (olddata != NULL) {
Assert(olddata->pageno > pageno);
simple_heap_delete(t_thrd.storage_cxt.lo_heap_r, &oldtuple->t_self);
simple_heap_delete(t_thrd.storage_cxt.lo_heap_r, &oldtuple->t_self); // 删除后续页面
}
/*
* Write a brand new page.
*
* Fill the hole up to the truncation point
*/
off = len % LOBLKSIZE;
/* 填充空隙 */
if (off > 0) {
rc = memset_s(workb, off, '\0', off);
securec_check(rc, "", "");
}
/* compute length of new page */
SET_VARSIZE(&workbuf.hdr, off + VARHDRSZ);
SET_VARSIZE(&workbuf.hdr, off + VARHDRSZ); // 计算新页的长度
/*
* Form and insert new tuple
*/
/* 插入新的元组 */
values[Anum_pg_largeobject_loid - 1] = ObjectIdGetDatum(obj_desc->id);
values[Anum_pg_largeobject_pageno - 1] = Int32GetDatum(pageno);
values[Anum_pg_largeobject_data - 1] = PointerGetDatum(&workbuf);
newtup = heap_form_tuple(t_thrd.storage_cxt.lo_heap_r->rd_att, values, nulls);
(void)simple_heap_insert(t_thrd.storage_cxt.lo_heap_r, newtup);
CatalogIndexInsert(indstate, newtup);
heap_freetuple(newtup);
(void)simple_heap_insert(t_thrd.storage_cxt.lo_heap_r, newtup); // 在堆上插入元组
CatalogIndexInsert(indstate, newtup); // 插入索引
heap_freetuple(newtup); // 释放元组内存
}
/*
* Delete any pages after the truncation point. If the initial search
* didn't find a page, then of course there's nothing more to do.
*/
if (olddata != NULL) {
while ((oldtuple = systable_getnext_ordered(sd, ForwardScanDirection)) != NULL) {
simple_heap_delete(t_thrd.storage_cxt.lo_heap_r, &oldtuple->t_self);
simple_heap_delete(t_thrd.storage_cxt.lo_heap_r, &oldtuple->t_self); // 删除截断点后的页面
}
}
systable_endscan_ordered(sd);
systable_endscan_ordered(sd); // 结束有序扫描
CatalogCloseIndexes(indstate);
CatalogCloseIndexes(indstate); // 关闭索引
/*
* Advance command counter so that tuple updates will be seen by later
* large-object operations in this transaction.
*/
CommandCounterIncrement();
CommandCounterIncrement(); // 提升命令计数器,以便后续事务中的元组更新可见
}

View File

@ -51,23 +51,38 @@ static const uint16 PAGE_CHECKSUM_MAGIC = 0xFFFF;
* treat such a page as empty and without free space. Eventually, VACUUM
* will clean up such a page and make it usable.
*/
/*
* typedef struct {
PageXLogRecPtr pd_lsn;//表示页面的LSNLog Sequence Number即页面上一次修改记录的最后一个字节之后的位置
uint16 pd_checksum; //页面的校验和
uint16 pd_flags;//页面的标志位,用于表示页面的状态和属性
LocationIndex pd_lower; //指向空闲空间起始位置的偏移量
LocationIndex pd_upper; //指向空闲空间结束位置的偏移量
LocationIndex pd_special;//指向特殊空间起始位置的偏移量
uint16 pd_pagesize_version;//页面大小和版本号
ShortTransactionId pd_prune_xid;//最旧可修剪的事务IDTransaction ID如果没有可修剪的事务则为零
ItemIdData pd_linp[FLEXIBLE_ARRAY_MEMBER]; //行指针数组的起始位置
} PageHeaderData;
*/
//验证页面Page的合法性和完整性
bool PageIsVerified(Page page, BlockNumber blkno)
{
PageHeader p = (PageHeader)page;
size_t* pagebytes = NULL;
PageHeader p = (PageHeader)page;//页面头指针
size_t* pagebytes = NULL;//页面字节指针
int i;
bool checksum_failure = false;
bool header_sane = false;
bool all_zeroes = false;
uint16 checksum = 0;
bool checksum_failure = false;//校验和失败标记
bool header_sane = false;//头部是否合法标记
bool all_zeroes = false;//是否全是零标记
uint16 checksum = 0;//校验和
/*
* Don't verify page data unless the page passes basic non-zero test
*/
//不验证页面数据,除非页面通过基本的非零测试
if (CheckPageZeroCases((PageHeader)page)) {
checksum = pg_checksum_page((char*)page, blkno);
if (checksum != p->pd_checksum) {
checksum_failure = true;
checksum = pg_checksum_page((char*)page, blkno);//计算页面校验和
if (checksum != p->pd_checksum) {//比较校验和与页面头中保存的校验和
checksum_failure = true;//校验和失败
}
/*
@ -76,13 +91,19 @@ bool PageIsVerified(Page page, BlockNumber blkno)
* the block can still reveal problems, which is why we offer the
* checksum option.
*/
/*
*
* 使
*/
if ((p->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && p->pd_lower <= p->pd_upper && p->pd_upper <= p->pd_special &&
p->pd_special <= BLCKSZ && p->pd_special == MAXALIGN(p->pd_special)) {
header_sane = true;
//PD_VALID_FLAG_BITS表示页面头部有效标志位的掩码~PD_VALID_FLAG_BITS则表示去除有效标志位后的掩码。
//如果结果等于0说明除了有效标志位外其他标志位都为0即页面头部的标志位合法。
header_sane = true; //头部合法
}
if (header_sane && !checksum_failure) {
return true;
return true;//验证通过
}
}
@ -91,26 +112,29 @@ bool PageIsVerified(Page page, BlockNumber blkno)
* multiple of size_t - and it's much faster to compare memory using the
* native word size.
*/
// 检查全零情况。幸运的是BLCKSZ保证总是sizeof(size_t)的倍数,
// 使用本机字大小比较内存的速度更快。
StaticAssertStmt(
BLCKSZ == (BLCKSZ / sizeof(size_t)) * sizeof(size_t), "BLCKSZ has to be a multiple of sizeof(size_t)");
all_zeroes = true;
pagebytes = (size_t*)page;
all_zeroes = true;//全零标记
pagebytes = (size_t*)page;//将页面转换为size_t指针
for (i = 0; i < (int)(BLCKSZ / sizeof(size_t)); i++) {
if (pagebytes[i] != 0) {
if (pagebytes[i] != 0) {//如果出现非零字节,说明不是全零页面
all_zeroes = false;
break;
}
}
if (all_zeroes) {
return true;
return true;//验证通过
}
/*
* Throw a WARNING if the checksum fails, but only after we've checked for
* the all-zeroes case.
*/
//如果校验和失败,则抛出警告,但只在我们检查全零情况后才抛出。
if (checksum_failure) {
ereport(WARNING,
(ERRCODE_DATA_CORRUPTED,
@ -120,19 +144,25 @@ bool PageIsVerified(Page page, BlockNumber blkno)
blkno)));
if (header_sane && u_sess->attr.attr_common.ignore_checksum_failure) {
return true;
return true;//验证通过(如果忽略了校验和失败)
//如果启用了忽略校验和失败的选项并且页面头部被认为是合法的那么就会跳过校验和验证失败的处理直接返回true表示通过了校验。
}
}
return false;
return false;//验证失败
}
/*
* PageHeaderIsValid
* Check that the header fields of a page appear valid.
* Check that the header fields of a page appear valid.//检查页面头部字段是否有效。
*
* This is called when a page modify in memory.
* if a page has just been read in from disk, should use PageIsVerified
* This is called when a page modify in memory.//当内存中的页面被修改时调用此函数。
* if a page has just been read in from disk, should use PageIsVerified//如果页面刚刚从磁盘上读取出来应该使用PageIsVerified函数
*/
/*
pagebyteschar指针访
headersize
headeroff
*/
bool PageHeaderIsValid(PageHeader page)
{
@ -143,13 +173,15 @@ bool PageHeaderIsValid(PageHeader page)
headersize = GetPageHeaderSize(page);
/* Check normal case */
if (PageGetPageSize(page) == BLCKSZ &&
/* 检查正常情况 */
if (PageGetPageSize(page) == BLCKSZ &&//页面大小为标准块大小
(PageGetPageLayoutVersion(page) == PG_COMM_PAGE_LAYOUT_VERSION ||
PageGetPageLayoutVersion(page) == PG_HEAP_PAGE_LAYOUT_VERSION ||
PageGetPageLayoutVersion(page) == PG_SEGMENT_PAGE_LAYOUT_VERSION) &&
(page->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && page->pd_lower >= headersize &&
page->pd_lower <= page->pd_upper && page->pd_upper <= page->pd_special && page->pd_special <= BLCKSZ &&
page->pd_special == MAXALIGN(page->pd_special))
(page->pd_flags & ~PD_VALID_FLAG_BITS) == 0 //pd_flags中只包含有效标志位
&& page->pd_lower >= headersize &&// pd_lower大于等于头部大小
page->pd_lower <= page->pd_upper && page->pd_upper <= page->pd_special && page->pd_special <= BLCKSZ &&//保证各个指针的顺序正确,并且不超过块大小
page->pd_special == MAXALIGN(page->pd_special))//pd_special按对齐方式对齐
return true;
/*
@ -159,26 +191,35 @@ bool PageHeaderIsValid(PageHeader page)
* storage also sets LSN when creating a new page.
* So we skip these three variables and test reset variables in the header.
*/
/*
*
* 使pd_flagslsn和checksum可能不为零log_new_page时PD_JUST_AFTER_FPW标志LSN
*
*/
if (page->pd_lower != 0 || page->pd_upper != 0 || page->pd_special != 0 || page->pd_pagesize_version != 0) {
return false;
}
//将页面转换为char指针以字节为单位进行访问
pagebytes = (char*)page;
//根据页面是否为8字节事务堆版本确定头部的偏移量
headeroff =
PageIs8BXidHeapVersion(page) ? offsetof(HeapPageHeaderData, pd_linp) : offsetof(PageHeaderData, pd_linp);
//检查头部之后的每个字节是否为零
for (i = headeroff; i < BLCKSZ; i++) {
if (pagebytes[i] != 0)
return false;
return false;//如果发现任何一个字节不为零则返回false表示页面头部不合法。
}
//所有字节都为零,表示页面头部合法
return true;
}
/*
* UPageHeaderIsValid
* Check that the header fields of a page appear valid.
* Check that the header fields of a page appear valid.//检查页面头部字段是否合法。
*
* This is called when a page modify in memory.
* if a page has just been read in from disk, should use PageIsVerified
* This is called when a page modify in memory.//当对内存中的页面进行修改时调用此函数
* if a page has just been read in from disk, should use PageIsVerified//如果刚刚从磁盘读入页面,则应该使用 PageIsVerified 函数。
*/
bool UPageHeaderIsValid(const UHeapPageHeaderData* page)
{
@ -189,12 +230,14 @@ bool UPageHeaderIsValid(const UHeapPageHeaderData* page)
headersize = SizeOfUHeapPageHeaderData;
/* Check normal case */
// 检查正常情况
if (page->pd_lower >= headersize && page->pd_lower <= page->pd_upper &&
page->pd_upper <= page->pd_special && page->pd_special <= BLCKSZ &&
page->pd_special == MAXALIGN(page->pd_special))
return true;
/* Check all-zeroes case */
// 检查全零情况
if (page->pd_lsn.xlogid != 0 || page->pd_lsn.xrecoff != 0 || (page->pd_flags & UHEAP_VALID_FLAG_BITS) != 0 ||
page->pd_lower != 0 || page->pd_upper != 0 || page->pd_special != 0 ||
page->td_count != 0 || page->pd_prune_xid != 0) {
@ -205,7 +248,7 @@ bool UPageHeaderIsValid(const UHeapPageHeaderData* page)
headeroff = offsetof(UHeapPageHeaderData, reserved);
for (i = headeroff; i < BLCKSZ; i++) {
if (pagebytes[i] != 0)
return false;
return false;//页面头部不合法
}
return true;
}
@ -215,15 +258,16 @@ bool UPageHeaderIsValid(const UHeapPageHeaderData* page)
* Get a temporary page in local memory for special processing.
* The returned page is not initialized at all; caller must do that.
*/
//获取一个本地内存中的临时页面,用于特殊处理。返回的页面未被初始化,调用者必须自行进行初始化。
Page PageGetTempPage(Page page)
{
Size pageSize;
Page temp;
Size pageSize;// 定义页面大小变量
Page temp;// 定义临时页面指针变量
pageSize = PageGetPageSize(page);
temp = (Page)palloc(pageSize);
pageSize = PageGetPageSize(page);//获取已存在页面的大小
temp = (Page)palloc(pageSize);//分配与已存在页面大小相同的内存并将返回的地址转换为Page类型
return temp;
return temp;//返回临时页面指针
}
/*
@ -231,6 +275,8 @@ Page PageGetTempPage(Page page)
* Get a temporary page in local memory for special processing.
* The page is initialized by copying the contents of the given page.
*/
//获取一个本地内存中的临时页面,用于特殊处理。该页面通过复制给定页面的内容进行初始化。
//该函数与上一个函数PageGetTempPage的区别在于使用了memcpy_s函数来复制给定页面的内容到临时页面。
Page PageGetTempPageCopy(Page page)
{
Size pageSize;
@ -240,7 +286,7 @@ Page PageGetTempPageCopy(Page page)
pageSize = PageGetPageSize(page);
temp = (Page)palloc(pageSize);
rc = memcpy_s(temp, pageSize, page, pageSize);
rc = memcpy_s(temp, pageSize, page, pageSize);//memcpy_s函数是一个安全的内存复制函数在拷贝时会检查目标地址是否越界因此能够避免缓冲区溢出漏洞的风险。
securec_check(rc, "\0", "\0");
return temp;
}
@ -251,6 +297,7 @@ Page PageGetTempPageCopy(Page page)
* Returns the size of the free (allocatable) space on a page,
* without any consideration for adding/removing line pointers.
*/
//返回页面上可用的自由空间大小,不考虑添加/删除行指针。
Size PageGetExactFreeSpace(Page page)
{
int space;
@ -259,6 +306,7 @@ Size PageGetExactFreeSpace(Page page)
* Use signed arithmetic here so that we behave sensibly if pd_lower >
* pd_upper.
*/
//使用有符号数进行计算,以确保在 pd_lower > pd_upper 的情况下能够正确处理。
space = (int)((PageHeader)page)->pd_upper - (int)((PageHeader)page)->pd_lower;
if (space < 0) {
@ -268,27 +316,42 @@ Size PageGetExactFreeSpace(Page page)
return (Size)(uint32)space;
}
//分配页面拷贝内存空间
/*
t_thrd.storage_cxt.pageCopy: char类型的指针变量
t_thrd.storage_cxt.segPageCopy: char类型的指针变量
ADIO_RUN():
ADIO_ELSE():
adio_align_alloc():
MemoryContextAlloc():
THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE):
BLCKSZ:
*/
static inline void AllocPageCopyMem()
{
if (t_thrd.storage_cxt.pageCopy == NULL) {
if (t_thrd.storage_cxt.pageCopy == NULL) {//页面复制的内存空间为空
ADIO_RUN()
{
//使用adio_align_alloc()函数进行内存分配
t_thrd.storage_cxt.pageCopy = (char*)adio_align_alloc(BLCKSZ);
}
ADIO_ELSE()
{
//使用MemoryContextAlloc()函数在存储内存上下文中进行分配
t_thrd.storage_cxt.pageCopy = (char*)MemoryContextAlloc(
THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE), BLCKSZ);
}
ADIO_END();
}
if (t_thrd.storage_cxt.segPageCopy == NULL) {
if (t_thrd.storage_cxt.segPageCopy == NULL) {//段页面复制的内存空间为空
ADIO_RUN()
{
//使用adio_align_alloc()函数进行内存分配
t_thrd.storage_cxt.segPageCopy = (char*)adio_align_alloc(BLCKSZ);
}
ADIO_ELSE()
{
//使用MemoryContextAlloc()函数在存储内存上下文中进行分配
t_thrd.storage_cxt.segPageCopy = (char*)MemoryContextAlloc(
THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE), BLCKSZ);
}
@ -303,6 +366,12 @@ static inline void AllocPageCopyMem()
* buffer, other processes might be updating hint bits in it, so we must
* copy the page to private storage if we are going to change it.
*/
/*
线线退使
*/
//此函数根据指定的条件对数据块进行加密。
char* PageDataEncryptIfNeed(Page page, TdeInfo* tde_info, bool need_copy, bool is_segbuf)
{
size_t plainLength = 0;
@ -311,18 +380,19 @@ char* PageDataEncryptIfNeed(Page page, TdeInfo* tde_info, bool need_copy, bool i
int retval = 0;
TdePageInfo tde_page_info;
char* dst = NULL;
//如果页面是新的页面或者不需要加密或者TDE功能未启用则直接返回原始页面
if (PageIsNew(page) || !PageIsTDE(page) || !g_instance.attr.attr_security.enable_tde) {
return (char*)page;
}
Assert(!PageIsEncrypt(page));
//计算明文数据的长度,并生成随机初始化向量
plainLength = ((PageHeader)page)->pd_special - ((PageHeader)page)->pd_upper;
retval = RAND_priv_bytes(tde_info->iv, RANDOM_IV_LEN);
if (retval != 1) {
ereport(WARNING, (errmodule(MOD_SEC_TDE), errmsg("generate random iv for tde failed, errcode:%d", retval)));
return (char*)page;
}
//如果需要复制页面,分配新的内存空间,并将页面内容复制到该空间中
if (need_copy) {
AllocPageCopyMem();
dst = is_segbuf ? t_thrd.storage_cxt.segPageCopy : t_thrd.storage_cxt.pageCopy;
@ -333,13 +403,14 @@ char* PageDataEncryptIfNeed(Page page, TdeInfo* tde_info, bool need_copy, bool i
}
/* at this part, do the real encryption */
//进行实际的加密操作
encryptBlockOrCUData(dst + ((PageHeader)dst)->pd_upper,
plainLength,
dst + ((PageHeader)dst)->pd_upper,
&cipherLength,
tde_info);
Assert(plainLength == cipherLength);
//将TDE信息转换为页信息并将其存储在页面的末尾
ret = memset_s(&tde_page_info, sizeof(TdePageInfo), 0, sizeof(TdePageInfo));
securec_check(ret, "\0", "\0");
transformTdeInfoToPage(tde_info, &tde_page_info);
@ -347,37 +418,41 @@ char* PageDataEncryptIfNeed(Page page, TdeInfo* tde_info, bool need_copy, bool i
securec_check(ret, "\0", "\0");
/* set the encryption flag */
//设置页面的加密标志
PageSetEncrypt((Page)dst);
return dst;
}
//对数据块进行解密
void PageDataDecryptIfNeed(Page page)
{
TdeInfo tde_info = {0};
TdePageInfo* tde_page_info = NULL;
// 检查页面是否同时满足TDE加密和加密标志位的条件
/* whether this page is both TDE page and encrypted */
if (PageIsEncrypt(page) && PageIsTDE(page)) {
size_t plainLength = 0;
size_t cipherLength = ((PageHeader)page)->pd_special - ((PageHeader)page)->pd_upper;
// 获取存储在页面末尾的TDE信息并将其转换为内部结构
tde_page_info = (TdePageInfo*)((char*)(page) + BLCKSZ - sizeof(TdePageInfo));
transformTdeInfoFromPage(&tde_info, tde_page_info);
/* at this part, do the real decryption */
decryptBlockOrCUData(page + ((PageHeader)page)->pd_upper,
cipherLength,
// 进行实际的解密操作
decryptBlockOrCUData(page + ((PageHeader)page)->pd_upper,//要进行解密的数据块的起始位置和解密后的明文的存储位置
cipherLength,//密文的长度,用于指示要解密的数据块的大小
page + ((PageHeader)page)->pd_upper,
&plainLength,
&tde_info);
Assert(cipherLength == plainLength);
&plainLength,//存储解密后的明文长度的变量的地址
&tde_info);//指向TDE信息结构的指针包含解密所需的密钥和其他相关信息
Assert(cipherLength == plainLength);//确保解密后的明文长度与密文长度相等。如果不相等,将触发断言失败并产生错误信息,表示解密操作出现问题。
/* clear the encryption flag */
// 清除加密标志位
PageClearEncrypt(page);
}
}
/*
* Set checksum for a page in shared buffers.
* Set checksum for a page in shared buffers.//为共享缓冲区中的一个页面设置校验和。
*
* If checksums are disabled, or if the page is not initialized, just return
* the input. Otherwise, we must make a copy of the page before calculating
@ -393,6 +468,7 @@ void PageDataDecryptIfNeed(Page page)
char* PageSetChecksumCopy(Page page, BlockNumber blkno, bool is_segbuf)
{
/* If we don't need a checksum, just return the passed-in data */
//如果不需要校验和,直接返回传入的数据
if (!CheckPageZeroCases((PageHeader)page)) {
return (char*)page;
}
@ -403,16 +479,18 @@ char* PageSetChecksumCopy(Page page, BlockNumber blkno, bool is_segbuf)
* array, is first to ensure adequate alignment for the checksumming code
* and second to avoid wasting space in processes that never call this.
*/
//分配一次拷贝空间,并在每次调用时重复使用
AllocPageCopyMem();
char *dst = is_segbuf ? t_thrd.storage_cxt.segPageCopy : t_thrd.storage_cxt.pageCopy;
//拷贝原页面数据到拷贝空间
errno_t rc = memcpy_s(dst, BLCKSZ, (char*)page, BLCKSZ);
securec_check(rc, "", "");
/* set page->pd_flags mark using FNV1A for checksum */
//使用FNV1A算法设置页面标志位的校验和
PageSetChecksumByFNV1A(dst);
//计算页面的校验和并存储到页面头部的pd_checksum中
((PageHeader)dst)->pd_checksum = pg_checksum_page(dst, blkno);
return dst;
@ -424,27 +502,31 @@ char* PageSetChecksumCopy(Page page, BlockNumber blkno, bool is_segbuf)
* This must only be used when we know that no other process can be modifying
* the page buffer.
*/
//为私有内存中的一个页面设置校验和
void PageSetChecksumInplace(Page page, BlockNumber blkno)
{
/* If we don't need a checksum, just return */
/* 如果不需要校验和,直接返回 */
if (!CheckPageZeroCases((PageHeader)page)) {
return;
}
/* set page->pd_flags mark using FNV1A for checksum */
/* 使用FNV1A算法设置页面标志位的校验和 */
PageSetChecksumByFNV1A(page);
/* 计算页面的校验和并存储在pd_checksum中 */
((PageHeader)page)->pd_checksum = pg_checksum_page((char*)page, blkno);
}
/*
* PageGetFreeSpaceForMultipleTuples
* Returns the size of the free (allocatable) space on a page,
* reduced by the space needed for multiple new line pointers.
* reduced by the space needed for multiple new line pointers.//返回页面上可用的空闲空间大小,减去多个新行指针所需的空间。
*
* Note: this should usually only be used on index pages. Use
* PageGetHeapFreeSpace on heap pages.
* PageGetHeapFreeSpace on heap pages.//注意这个函数通常只用于索引页面。对于堆页面请使用PageGetHeapFreeSpace。
*/
Size PageGetFreeSpaceForMultipleTuples(Page page, int ntups)
{
int space;
@ -453,11 +535,13 @@ Size PageGetFreeSpaceForMultipleTuples(Page page, int ntups)
* Use signed arithmetic here so that we behave sensibly if pd_lower >
* pd_upper.
*/
//在这里使用有符号算术以便在pd_lower > pd_upper的情况下能正确处理。
//有符号的减法运算,得到的结果即为初始的可用空间大小。
space = (int)((PageHeader)page)->pd_upper - (int)((PageHeader)page)->pd_lower;
//判断剩余空间是否足够容纳多个新行指针
if (space < (int)(ntups * sizeof(ItemIdData)))
return 0;
space -= ntups * sizeof(ItemIdData);
return 0;//表示无可用空间
space -= ntups * sizeof(ItemIdData);//根据新行指针的数量,将所需的空间从初始的可用空间中减去
return (Size) space;
}

View File

@ -13,10 +13,9 @@
* -------------------------------------------------------------------------
*/
#include "postgres.h"
#include "knl/knl_variable.h"
#include "storage/checksum.h"
#include "postgres.h"//主要的PostgreSQL头文件其中包含了大量的声明和定义用于构建和操作数据库系统
#include "knl/knl_variable.h"//用于包含knernel实例、会话和线程的变量
#include "storage/checksum.h"//声明计算校验和的函数
/*
* The actual code is in storage/checksum_impl.h. This is done so that
@ -24,3 +23,13 @@
* that file from the exported openGauss headers. (Compare our CRC code.)
*/
#include "storage/checksum_impl.h"
//存储校验和的实现代码
/* N_SUMS是并行计算校验和的数量。
FNV_PRIME是FNV -1a哈希的质数乘数
CHECKSUM_CACL_ROUNDS是计算校验和的轮数
g_checksumBaseOffsets是每个并行FNV哈希的初始状态的基本偏移量数组
CHECKSUM_COMP是计算校验和的宏
pg_checksum_block是块校验和算法的函数原型
DataBlockChecksum是数据块校验和算法的函数原型
pg_checksum_page是页面校验和算法的函数原型
*/

View File

@ -15,52 +15,54 @@
#include "postgres.h"
#include "knl/knl_variable.h"
#include "storage/checksum_impl.h"
//用于对数据进行补零并计算校验和
void ChecksumForZeroPadding(uint32 *sums, const uint32 *dataArr, uint32 currentLeft, uint32 alignSize);
//对初始种子和值进行校验和初始化
static inline uint32 pg_checksum_init(uint32 seed, uint32 value)
{
CHECKSUM_COMP(seed, value);
CHECKSUM_COMP(seed, value);//将种子和值作为参数传入CHECKSUM_COMP宏中,进行校验和计算
return seed;
}
//计算数据块的校验和
uint32 DataBlockChecksum(char* data, uint32 size, bool zeroing)
{
uint32 sums[N_SUMS];
uint32* dataArr = (uint32*)data;
uint32 result = 0;
uint32 i, j;
uint32 currentLeft = size;
uint32 sums[N_SUMS];//定义数组存储部分校验和
uint32* dataArr = (uint32*)data;//将数据块转换为uint32数组
uint32 result = 0;//定义变量保存最终的校验和结果
uint32 i, j;//定义循环变量
uint32 currentLeft = size;//初始化剩余未处理的数据大小
/* ensure that the size is compatible with the algorithm */
//确保输入的数据大小符合校验和算法的要求
uint32 alignSize = sizeof(uint32) * N_SUMS;
Assert(zeroing || (size % alignSize == 0));
/* initialize partial checksums to their corresponding offsets */
auto realSize = size < alignSize ? size : alignSize;
auto realSize = size < alignSize ? size : alignSize;//实际的校验和计算大小
uint32 *initUint32 = NULL;
char usedForInit[sizeof(uint32) * N_SUMS] = {0};
if (zeroing && size < alignSize) {
if (zeroing && size < alignSize) {//判断是否需要补零
errno_t rc = memcpy_s(usedForInit, alignSize, (char *) dataArr, realSize);
securec_check(rc, "", "");
currentLeft -= realSize;
initUint32 = (uint32*)usedForInit;
currentLeft -= realSize;//更新剩余未处理数据的大小
initUint32 = (uint32*)usedForInit;//用复制的数据作为初始值
} else {
initUint32 = dataArr;
currentLeft -= alignSize;
}
for (j = 0; j < N_SUMS; j += 2) {
sums[j] = pg_checksum_init(g_checksumBaseOffsets[j], initUint32[j]);
sums[j] = pg_checksum_init(g_checksumBaseOffsets[j], initUint32[j]);// 初始化部分校验和
sums[j + 1] = pg_checksum_init(g_checksumBaseOffsets[j + 1], initUint32[j + 1]);
}
dataArr += N_SUMS;
/* main checksum calculation */
// 主要的校验和计算
for (i = 1; i < size / alignSize; i++) {
for (j = 0; j < N_SUMS; j += 2) {
CHECKSUM_COMP(sums[j], dataArr[j]);
CHECKSUM_COMP(sums[j], dataArr[j]);// 计算部分校验和
CHECKSUM_COMP(sums[j + 1], dataArr[j + 1]);
}
dataArr += N_SUMS;
@ -69,7 +71,7 @@ uint32 DataBlockChecksum(char* data, uint32 size, bool zeroing)
/* checksum for zero padding */
currentLeft -= alignSize * (i - 1);
if (currentLeft > 0 && currentLeft < alignSize && zeroing) {
ChecksumForZeroPadding(sums, dataArr, currentLeft, alignSize);
ChecksumForZeroPadding(sums, dataArr, currentLeft, alignSize);// 补零并计算校验和
}
/* finally add in two rounds of zeroes for additional mixing */
@ -78,62 +80,67 @@ uint32 DataBlockChecksum(char* data, uint32 size, bool zeroing)
CHECKSUM_COMP(sums[j], 0);
/* xor fold partial checksums together */
result ^= sums[j];
result ^= sums[j];//对部分校验和结果进行混合得到最终的校验和结果
}
return result;
}
//对剩余数据进行补零并计算校验和
void ChecksumForZeroPadding(uint32 *sums, const uint32 *dataArr, uint32 currentLeft, uint32 alignSize)
{
// 定义变量maxLen保存部分校验和数组的大小
auto maxLen = sizeof(uint32) * N_SUMS;
// 创建临时字符数组currentLeftChars初始化为零并将剩余的数据复制到该数组中。
char currentLeftChars[maxLen] = {0};
errno_t rc = memcpy_s(currentLeftChars, maxLen, (char *)dataArr, currentLeft);
securec_check(rc, "", "");
// 使用两层循环对currentLeftChars数组的数据进行校验和计算并更新部分校验和数组sums的值
for (int j = 0; j < N_SUMS; j += 2) {
// 对于每个部分校验和分别计算两个32位整数的校验和
// 使用宏CHECKSUM_COMP对sums中的值与currentLeftChars数组中对应位置的值进行校验和计算计算结果更新到sums中
CHECKSUM_COMP(sums[j], ((uint32 *)currentLeftChars)[j]);
CHECKSUM_COMP(sums[j + 1], ((uint32 *)currentLeftChars)[j + 1]);
}
}
//对数据块进行校验和计算
uint32 pg_checksum_block(char* data, uint32 size)
{
uint32 sums[N_SUMS];
uint32* dataArr = (uint32*)data;
uint32 result = 0;
uint32 sums[N_SUMS];// 部分校验和数组
uint32* dataArr = (uint32*)data;// 数据块指针转换为uint32类型指针
uint32 result = 0;// 最终校验和结果
uint32 i, j;
#ifndef ROACH_COMMON
/* ensure that the size is compatible with the algorithm */
Assert((size % (sizeof(uint32) * N_SUMS)) == 0);
Assert((size % (sizeof(uint32) * N_SUMS)) == 0);// 确保数据块大小能够被算法处理
#endif
/* initialize partial checksums to their corresponding offsets */
for (j = 0; j < N_SUMS; j += 2) {
sums[j] = pg_checksum_init(g_checksumBaseOffsets[j], dataArr[j]);
sums[j] = pg_checksum_init(g_checksumBaseOffsets[j], dataArr[j]);// 使用初始偏移量计算部分校验和
sums[j + 1] = pg_checksum_init(g_checksumBaseOffsets[j + 1], dataArr[j + 1]);
}
dataArr += N_SUMS;
dataArr += N_SUMS;// 移动数据指针到下一个数据块
/* main checksum calculation */
for (i = 1; i < size / (sizeof(uint32) * N_SUMS); i++) {
for (i = 1; i < size / (sizeof(uint32) * N_SUMS); i++) {// 处理每个数据块
for (j = 0; j < N_SUMS; j += 2) {
CHECKSUM_COMP(sums[j], dataArr[j]);
CHECKSUM_COMP(sums[j], dataArr[j]);// 计算并更新部分校验和
CHECKSUM_COMP(sums[j + 1], dataArr[j + 1]);
}
dataArr += N_SUMS;
dataArr += N_SUMS;// 移动数据指针到下一个数据块
}
/* finally add in two rounds of zeroes for additional mixing */
for (j = 0; j < N_SUMS; j++) {
CHECKSUM_COMP(sums[j], 0);
CHECKSUM_COMP(sums[j], 0);// 添加两轮零值进行混合计算
CHECKSUM_COMP(sums[j], 0);
/* xor fold partial checksums together */
result ^= sums[j];
result ^= sums[j];// 将部分校验和结果异或操作
}
return result;
return result;// 返回最终的校验和结果
}
/*
@ -144,11 +151,12 @@ uint32 pg_checksum_block(char* data, uint32 size)
* somehow moved to a different location), the page header (excluding the
* checksum itself), and the page data.
*/
//计算一个openGauss页面的校验和
uint16 pg_checksum_page(char* page, BlockNumber blkno)
{
PageHeader phdr = (PageHeader)page;
uint16 save_checksum;
uint32 checksum;
uint16 save_checksum;//保存原始校验和值
uint32 checksum;//校验和计算结果
/*
* Save pd_checksum and temporarily set it to zero, so that the checksum
@ -156,17 +164,22 @@ uint16 pg_checksum_page(char* page, BlockNumber blkno)
* Restore it after, because actually updating the checksum is NOT part of
* the API of this function.
*/
//保存pd_checksum并将其临时置零以避免受页面上旧校验和的影响
save_checksum = phdr->pd_checksum;
phdr->pd_checksum = 0;
//计算除了校验和本身以外的部分的校验和
checksum = pg_checksum_block(page, BLCKSZ);
//恢复保存的原始pd_checksum值到页面头部
phdr->pd_checksum = save_checksum;
/* Mix in the block number to detect transposed pages */
//使用异或操作混合块号,以检测页面是否发生了位置变换
checksum ^= blkno;
/*
* Reduce to a uint16 (to fit in the pd_checksum field) with an offset of
* one. That avoids checksums of zero, which seems like a good idea.
*/
// 将计算得到的校验和值进行取模运算限制在UINT16_MAX范围内并加一作为最终的校验和值
return (checksum % UINT16_MAX) + 1;
}

View File

@ -54,59 +54,83 @@ typedef struct XLogFilter {
BlockNumber blocknum;
} XLogFilter;
//通过组合数据目录路径、起始LSN和结束LSN生成一个格式化的输出文件名
static void GenerateOutputFileName(char *outputFilename, char *start_lsn_str, char *end_lsn_str)
{
List *elemlist = NIL;
//将起始LSN字符串按照'/'进行分割将元素存储在elemlist中
SplitIdentifierString(start_lsn_str, '/', &elemlist);
//获取elemlist中第二个元素赋值给start_lsn_str_p2变量
char *start_lsn_str_p2 = (char *)lsecond(elemlist);
//释放elemlist占用的内存
list_free_ext(elemlist);
//将结束LSN字符串按照'/'进行分割将元素存储在elemlist中
SplitIdentifierString(end_lsn_str, '/', &elemlist);
//获取elemlist中第二个元素赋值给end_lsn_str_p2变量
char *end_lsn_str_p2 = (char *)lsecond(elemlist);
//格式化输出文件名并将结果写入outputFilename的末尾
int rc = snprintf_s(outputFilename + (int)strlen(outputFilename), MAXFILENAME, MAXFILENAME - 1, "%s/%s_%s.xlog",
t_thrd.proc_cxt.DataDir, start_lsn_str_p2, end_lsn_str_p2);
//检查格式化操作是否成功执行
securec_check_ss(rc, "\0", "\0");
}
//验证LSN字符串并将其转换为XLogRecPtr类型的LSN指针
static void ValidateLSN(char *lsn_str, XLogRecPtr *lsn_ptr)
{
uint32 hi = 0;
uint32 lo = 0;
//调用validate_xlog_location函数验证LSN字符串的有效性
validate_xlog_location(lsn_str);
//使用sscanf_s函数从LSN字符串中解析出hi和lo两个部分的值
if (sscanf_s(lsn_str, "%X/%X", &hi, &lo) != TWO)
//如果解析失败则抛出错误并提示无法解析LSN字符串
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("could not parse xlog location \"%s\"", lsn_str)));
//将hi和lo组合成64位的LSN值存储到lsn_ptr中
*lsn_ptr = (((uint64)hi) << XIDTHIRTYTWO) | lo;
}
//验证起始LSN和结束LSN并将它们转换为XLogRecPtr类型的LSN指针
static void ValidateStartEndLSN(char *start_lsn_str, char *end_lsn_str, XLogRecPtr *start_lsn, XLogRecPtr *end_lsn)
{
//验证起始LSN字符串并将其转换为XLogRecPtr类型的起始LSN指针
ValidateLSN(start_lsn_str, start_lsn);
//验证结束LSN字符串并将其转换为XLogRecPtr类型的结束LSN指针
ValidateLSN(end_lsn_str, end_lsn);
//检查结束LSN是否小于起始LSN如果是则抛出错误
if (XLByteLT(*end_lsn, *start_lsn))
/*
* LSN小于起始LSN
* LSN和结束LSN的具体数值
* 使% X格式将高位32位和低位32位转换为十六进制数值
*/
ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("start xlog location %X/%X should be smaller than or equal to end xlog location %X/%X",
(uint32)(*start_lsn >> XIDTHIRTYTWO), (uint32)(*start_lsn), (uint32)(*end_lsn >> XIDTHIRTYTWO),
(uint32)(*end_lsn))));
}
//获取当前数据库日志的最小LSN用于作为起始LSN的参考值
static XLogRecPtr GetMinLSN()
{
//获取最后一个已移除的日志段号
XLogSegNo lastRemovedSegNo = XLogGetLastRemovedSegno();
//计算当前LSN
XLogRecPtr current_recptr = (lastRemovedSegNo + 1) * XLogSegSize;
//返回当前LSN
return current_recptr;
}
//获取当前数据库日志的最大LSN用于作为结束LSN的参考值
static XLogRecPtr GetMaxLSN()
{
//如果正在进行恢复则抛出错误并提示无法获取最大LSN
if (RecoveryInProgress())
ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("recovery is in progress"),
errhint("Can't get local max LSN during recovery.")));
//获取当前日志记录的写入指针位置
XLogRecPtr current_recptr = GetXLogWriteRecPtr();
//返回当前日志记录的写入指针位置即最大LSN
return current_recptr;
}
//将XLog记录信息输出到字符串
static void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput)
{
errno_t rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1,
@ -118,8 +142,10 @@ static void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput)
StringInfoData buf;
initStringInfo(&buf);
RmgrTable[XLogRecGetRmid(record)].rm_desc(&buf, record);
//输出块引用的信息
rc = strcat_s(strOutput, MAXOUTPUTLEN, buf.data);
securec_check(rc, "\0", "\0");
//如果是bucket文件节点输出bucketNode信息
if (!XLogRecHasAnyBlockRefs(record)) {
rc = strcat_s(strOutput, MAXOUTPUTLEN, "\n\n");
securec_check(rc, "\0", "\0");
@ -154,6 +180,7 @@ static void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput)
rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, ", blk %u", blk);
securec_check_ss(rc, "\0", "\0");
/* others: lastlsn %X/%X" */
// 输出lastlsn信息
XLogRecPtr lsn;
XLogRecGetBlockLastLsn(record, block_id, &lsn);
rc = snprintf_s(strOutput + (int)strlen(strOutput), MAXOUTPUTLEN, MAXOUTPUTLEN - 1, ", lastlsn %X/%X",
@ -166,58 +193,89 @@ static void XLogDumpDisplayRecord(XLogReaderState *record, char *strOutput)
void CheckOpenFile(FILE *outputfile, char *outputFilename)
{
// 检查文件打开操作的结果
if (outputfile == NULL)
ereport(ERROR, (errcode(ERRCODE_FILE_READ_FAILED), (errmsg("Cannot read %s", outputFilename))));
}
void CheckWriteFile(int result, int cnt_len, char *outputFilename)
{
// 检查文件写入操作的结果
if (result != cnt_len)
ereport(ERROR, (errcode(ERRCODE_FILE_WRITE_FAILED), (errmsg("Cannot write %s", outputFilename))));
}
void CheckCloseFile(int result, char *outputFilename)
{
//检查文件关闭操作的结果
if (0 != result)
ereport(ERROR, (errcode(ERRCODE_IO_ERROR), (errmsg("Cannot close %s", outputFilename))));
}
//检查当前XLog记录是否符合一组给定的筛选条件
static bool CheckValidRecord(XLogReaderState *xlogreader_state, XLogFilter *filter)
{
bool found = false;
bool found = false;//初始化 found 变量,表示是否找到满足条件的块
//循环遍历 XLog 中的每个块
for (int i = 0; i <= xlogreader_state->max_block_id; i++) {
RelFileNode rnode;
ForkNumber forknum;
BlockNumber blk;
//获取当前块的标记
if (!XLogRecGetBlockTag(xlogreader_state, i, &rnode, &forknum, &blk))
continue;
continue;//如果返回 false则说明没有找到块标记跳过该块继续循环
//检查块标记是否符合查询条件
if (RelFileNodeEquals(rnode, filter->by_relfilenode))
/* if equal to specific block or check all blocks, found = ture; */
found = (!filter->by_block) || (blk == filter->blocknum);
//如果找到一个符合条件的块则返回true表示当前记录有效
if (found) {
return found;
}
}
//没有找到任何符合条件的块则返回false表示当前记录无效
return found;
}
//在一系列连续的日志记录中查找下一个有效的日志记录的LSN
static XLogRecPtr UpdateNextLSN(XLogRecPtr cur_lsn, XLogRecPtr end_lsn, XLogReaderState *xlogreader_state, bool *found)
{
XLogRecPtr next_record = InvalidXLogRecPtr;
XLogRecPtr next_record = InvalidXLogRecPtr;// 初始化 next_record表示下一个记录的LSN
// 使用循环尝试多次查找下一个记录的LSN
for (int tryTimes = 0; tryTimes < FIVE; tryTimes++) {
// 计算起始 LSN
XLogRecPtr start_lsn = Max(cur_lsn, (g_instance.comm_cxt.predo_cxt.redoPf.oldest_segment) * XLogSegSize);
// 查找下一个记录的LSN
next_record = XLogFindNextRecord(xlogreader_state, start_lsn);
// 检查找到的下一个记录的LSN是否有效并且小于结束LSN
if (!XLByteEQ(next_record, InvalidXLogRecPtr) && XLByteLT(next_record, end_lsn)) {
*found = true;
return next_record;
*found = true;// 设置 found 为 true表示找到了下一个记录的LSN
return next_record;// 返回找到的下一个记录的LSN
}
}
// 如果没有找到下一个记录的LSN则返回 InvalidXLogRecPtr
return next_record;
}
/*
start_lsn: LSNLog Sequence Number
end_lsn: LSN
filter:
outputFilename:
readprivate: XLogPrivate类型的结构体XLogReaderState
xlogreader_state: XLogReaderState类型的对象WAL文件
first_record:
valid_start_lsn:
valid_end_lsn:
outputfile:
strOutput:
count:
errormsg:
record: XLogRecord类型的对象
*/
//将WALWrite-Ahead Log文件中的记录转储到指定的输出文件中
static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filter, char *outputFilename)
{
/* start reading */
// 初始化XLogReaderState用于读取WAL文件
errno_t rc = EOK;
XLogPrivate readprivate;
rc = memset_s(&readprivate, sizeof(XLogPrivate), 0, sizeof(XLogPrivate));
@ -230,9 +288,11 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte
(errmsg("memory is temporarily unavailable while allocate xlog reader"))));
/* get the first valid xlog record location */
// 找到起始位置之后的第一个有效记录位置
XLogRecPtr first_record = XLogFindNextRecord(xlogreader_state, start_lsn);
/* if we are recycling or removing log files concurrently, we can't find the next record right after.
* Hence, we need to update the min_lsn */
// 如果找不到第一个有效记录则更新min_lsn并再次尝试查找
if (XLByteEQ(first_record, InvalidXLogRecPtr)) {
ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
(errmsg("XLogFindNextRecord: could not find a valid record after %X/%X. Retry.",
@ -249,9 +309,10 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte
XLogRecPtr valid_start_lsn = first_record;
XLogRecPtr valid_end_lsn = valid_start_lsn;
// 打开目标输出文件
FILE *outputfile = fopen(outputFilename, "w");
CheckOpenFile(outputfile, outputFilename);
// 输出一些转储信息
char *strOutput = (char *)palloc(MAXOUTPUTLEN * sizeof(char));
rc = memset_s(strOutput, MAXOUTPUTLEN, 0, MAXOUTPUTLEN);
securec_check(rc, "\0", "\0");
@ -269,12 +330,14 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte
int count = 0;
char *errormsg = NULL;
XLogRecord *record = NULL;
// 不断读取WAL记录直到结束位置
while (XLByteLT(xlogreader_state->EndRecPtr, end_lsn)) {
record = XLogReadRecord(xlogreader_state, first_record, &errormsg);
valid_end_lsn = xlogreader_state->EndRecPtr;
if (!record && XLByteLT(valid_end_lsn, end_lsn)) {
/* if we are recycling or removing log files concurrently, and we can't find the next record right after.
* In this case, we try to read from the current oldest xlog file. */
// 如果无法读取下一条记录则更新min_lsn并继续尝试读取
bool found = false;
XLogRecPtr temp_start_lsn = Max(xlogreader_state->EndRecPtr, start_lsn);
first_record = UpdateNextLSN(temp_start_lsn, end_lsn, xlogreader_state, &found);
@ -303,6 +366,7 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte
break;
}
first_record = InvalidXLogRecPtr; /* No explicit start point; read the record after the one we just read */
// 根据过滤器筛选记录,并输出
if (filter->by_xid_enabled && filter->by_xid != record->xl_xid) {
continue;
}
@ -325,7 +389,7 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte
CheckWriteFile(fwrite(strOutput, 1, strlen(strOutput), outputfile), strlen(strOutput), outputFilename);
pfree_ext(strOutput);
}
// 输出结束信息并关闭文件
XLogReaderFree(xlogreader_state);
strOutput = (char *)palloc(MAXOUTPUTLEN * sizeof(char));
rc = memset_s(strOutput, MAXOUTPUTLEN, 0, MAXOUTPUTLEN);
@ -345,14 +409,20 @@ static void XLogDump(XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogFilter *filte
}
/* There are only two parameters in PG_FUNCTION_ARGS: start_lsn and end_lsn */
/*
PostgreSQL函数"gs_xlogdump_lsn"
LSN和结束LSN作为输入参数WAL文件进行转储
*/
Datum gs_xlogdump_lsn(PG_FUNCTION_ARGS)
{
errno_t rc = EOK;
/* check user's right */
/* 检查用户权限 */
const char fName[MAXFNAMELEN] = "gs_xlogdump_lsn";
CheckUser(fName);
/* read in parameters */
/* 读取并验证起始LSN和结束LSN */
char *start_lsn_str = text_to_cstring(PG_GETARG_TEXT_P(0));
char *end_lsn_str = text_to_cstring(PG_GETARG_TEXT_P(1));
@ -360,12 +430,21 @@ Datum gs_xlogdump_lsn(PG_FUNCTION_ARGS)
XLogRecPtr start_lsn, end_lsn;
ValidateStartEndLSN(start_lsn_str, end_lsn_str, &start_lsn, &end_lsn);
/*
LSN和结束LSN生成输出文件名
使GenerateOutputFileName函数
*/
char *outputFilename = (char *)palloc(MAXFILENAME * sizeof(char));
rc = memset_s(outputFilename, MAXFILENAME, 0, MAXFILENAME);
securec_check(rc, "\0", "\0");
GenerateOutputFileName(outputFilename, start_lsn_str, end_lsn_str);
/* update start_lsn and end_lsn based on cur min_lsn and max_lsn */
/*
min_lsn和max_lsn更新起始LSN和结束LSN
*/
XLogRecPtr min_lsn = GetMinLSN();
XLogRecPtr max_lsn = GetMaxLSN();
@ -375,31 +454,38 @@ Datum gs_xlogdump_lsn(PG_FUNCTION_ARGS)
if (XLByteLT(max_lsn, end_lsn)) {
end_lsn = max_lsn;
}
/* 初始化一个XLogFilter结构体用于筛选日志记录 */
XLogFilter filter;
rc = memset_s(&filter, sizeof(XLogFilter), 0, sizeof(XLogFilter));
securec_check_ss(rc, "\0", "\0");
/* 对指定的WAL文件进行转储 */
XLogDump(start_lsn, end_lsn, &filter, outputFilename);
/* 返回输出文件名 */
PG_RETURN_TEXT_P(cstring_to_text(outputFilename));
}
/* There are only one parameter in PG_FUNCTION_ARGS: c_xid */
//基于事务ID对WAL文件进行转储
Datum gs_xlogdump_xid(PG_FUNCTION_ARGS)
{
errno_t rc = EOK;
/* check user's right */
/* 检查用户是否有权限调用该函数 */
const char fName[MAXFNAMELEN] = "gs_xlogdump_xid";
CheckUser(fName);
/* read in parameters */
/* 读取并验证传入的事务ID */
TransactionId c_xid = PG_GETARG_TRANSACTIONID(0);
/* check parameters */
/* 获取系统当前的最大事务ID以确保传入的事务ID有效 */
TransactionId topXid = GetTopTransactionId();
if (TransactionIdPrecedes(topXid, c_xid))
ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
(errmsg("Xid should less than or equal to %lu.", topXid))));
/* generate output file name */
/* 生成输出文件名 */
char *outputFilename = (char *)palloc(MAXFILENAME * sizeof(char));
rc = memset_s(outputFilename, MAXFILENAME, 0, MAXFILENAME);
securec_check(rc, "\0", "\0");
@ -407,22 +493,27 @@ Datum gs_xlogdump_xid(PG_FUNCTION_ARGS)
t_thrd.proc_cxt.DataDir, c_xid);
securec_check_ss(rc, "\0", "\0");
/* update start_lsn and end_lsn based on cur min_lsn and max_lsn */
/* 获取最小LSN和最大LSN并更新转储范围 */
XLogRecPtr min_lsn = GetMinLSN();
XLogRecPtr max_lsn = GetMaxLSN();
/* 初始化XLogFilter结构体设定需要根据事务ID来筛选记录 */
XLogFilter filter;
rc = memset_s(&filter, sizeof(XLogFilter), 0, sizeof(XLogFilter));
securec_check(rc, "\0", "\0");
filter.by_xid_enabled = true;
filter.by_xid = c_xid;
/* 调用XLogDump函数进行转储 */
XLogDump(min_lsn, max_lsn, &filter, outputFilename);
/* 返回输出文件名 */
PG_RETURN_TEXT_P(cstring_to_text(outputFilename));
}
//用于将指定路径和块号(或所有块)下的 WAL 日志记录转储到输出文件中
/* There are only three parameters in PG_FUNCTION_ARGS: path, blocknum, relation_type */
Datum gs_xlogdump_tablepath(PG_FUNCTION_ARGS)
{
errno_t rc = EOK;
/* check user's right */
//查用户是否有执行该函数的权限
const char fName[MAXFNAMELEN] = "gs_xlogdump_tablepath";
CheckUser(fName);
@ -431,6 +522,7 @@ Datum gs_xlogdump_tablepath(PG_FUNCTION_ARGS)
int64 blocknum = PG_GETARG_INT64(1);
char *relation_type = text_to_cstring(PG_GETARG_TEXT_P(2));
/* check parameters */
//检查块号是否在合法范围内(-1 到 MaxBlockNumber 之间),如果不在范围内,则抛出错误
if (blocknum > MaxBlockNumber || blocknum < -1)
ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
(errmsg("Blocknum should be between -1 and %u.", MaxBlockNumber))));
@ -443,7 +535,7 @@ Datum gs_xlogdump_tablepath(PG_FUNCTION_ARGS)
securec_check_c(rc, "\0", "\0");
filter.by_tablepath_enabled = true;
if (blocknum == -1) { /* care all blocks */
if (blocknum == -1) { /* care all blocks */ //示处理所有块
filter.by_block = false;
} else { /* only one block */
filter.by_block = true;
@ -452,18 +544,20 @@ Datum gs_xlogdump_tablepath(PG_FUNCTION_ARGS)
/* generate output file name */
char *outputFilename = (char *)palloc(MAXFILENAME * sizeof(char));
rc = memset_s(outputFilename, MAXFILENAME, 0, MAXFILENAME);
rc = memset_s(outputFilename, MAXFILENAME, 0, MAXFILENAME);//确保输出文件名为空字符串
securec_check(rc, "\0", "\0");
PrepForRead(path, blocknum, relation_type, outputFilename, &(filter.by_relfilenode), false);
PrepForRead(path, blocknum, relation_type, outputFilename, &(filter.by_relfilenode), false);//根据 filter.by_relfilenode验证路径和关联关系类型参数的合法性
ValidateParameterPath(filter.by_relfilenode, path);
//执行转储操作
XLogDump(min_lsn, max_lsn, &filter, outputFilename);
PG_RETURN_TEXT_P(cstring_to_text(outputFilename));
PG_RETURN_TEXT_P(cstring_to_text(outputFilename));//将输出文件名转换为 PostgreSQL 文本类型
}
//解析和转储数据库中的WALWrite-Ahead Log日志记录和数据页
Datum gs_xlogdump_parsepage_tablepath(PG_FUNCTION_ARGS)
{
/* check user's right */
//检查用户权限
const char fName[MAXFNAMELEN] = "gs_xlogdump_parsepage_tablepath";
CheckUser(fName);
@ -475,6 +569,7 @@ Datum gs_xlogdump_parsepage_tablepath(PG_FUNCTION_ARGS)
int rc = -1;
/* page part, copy path since for segment we will edit path */
//读取传入的函数参数,并进行相应的数据类型转换
char *path_cpy = (char *)palloc(MAXFILENAME + 1);
rc = strcpy_s(path_cpy, MAXFILENAME + 1, path);
securec_check(rc, "\0", "\0");
@ -482,7 +577,7 @@ Datum gs_xlogdump_parsepage_tablepath(PG_FUNCTION_ARGS)
/* In order to avoid querying the shared buffer and applying LW locks, blocking the business. */
/* In the case of finding all pages, force to check disk */
if (blocknum == -1) {
read_memory = false;
read_memory = false;//强制从磁盘读取数据
}
char *outputFilenamePage = ParsePage(path_cpy, blocknum, relation_type, read_memory);
pfree_ext(path_cpy);
@ -504,10 +599,11 @@ Datum gs_xlogdump_parsepage_tablepath(PG_FUNCTION_ARGS)
rc = memset_s(outputFilename, MAXFILENAME, 0, MAXFILENAME);
securec_check(rc, "\0", "\0");
PrepForRead(path, blocknum, relation_type, outputFilename, &(filter.by_relfilenode), false);
//获取最小和最大 LSN 并调用 XLogDump 函数将符合筛选条件的 WAL 日志记录转储到输出文件中。
XLogRecPtr min_lsn = GetMinLSN();
XLogRecPtr max_lsn = GetMaxLSN();
XLogDump(min_lsn, max_lsn, &filter, outputFilename);
//计算生成结果字符串所需的长度,并为结果字符串分配内存
int outputLen = strlen(outputFilename) + strlen(outputFilenamePage) + 100;
if (outputLen <= 0)
ereport(ERROR, (errcode(ERRCODE_IO_ERROR), (errmsg("Cannot generate right output file name."))));

View File

@ -13,6 +13,7 @@
*
* -------------------------------------------------------------------------
*/
//这些函数提供了对OpenGauss数据库中物理地址指针进行比较和判断相等的功能用于支持索引和有序集合的操作
#include "postgres.h"
#include "knl/knl_variable.h"
@ -26,15 +27,18 @@
* Note:
* Asserts that the disk item pointers are both valid!
*/
//实现了处理OpenGauss数据库中的物理地址指针ItemPointer的功能
bool ItemPointerEquals(ItemPointer pointer1, ItemPointer pointer2)
{
//比较两个物理地址指针 pointer1 和 pointer2 是否指向相同的数据库条目
//通过比较两个指针的块号和偏移量来判断它们是否相等
if (ItemPointerGetBlockNumber(pointer1) == ItemPointerGetBlockNumber(pointer2) &&
ItemPointerGetOffsetNumber(pointer1) == ItemPointerGetOffsetNumber(pointer2))
return true;
else
return false;
}
//与上一个函数功能相同,但不进行有效性检查。它假定传入的物理地址指针是有效的。
bool ItemPointerEqualsNoCheck(ItemPointer pointer1, ItemPointer pointer2)
{
if ((ItemPointerGetBlockNumberNoCheck(pointer1) == ItemPointerGetBlockNumberNoCheck(pointer2)) &&
@ -48,6 +52,7 @@ bool ItemPointerEqualsNoCheck(ItemPointer pointer1, ItemPointer pointer2)
* ItemPointerCompare
* Generic btree-style comparison for item pointers.
*/
//实现一种通用的比较方法在索引操作和有序集合操作中对物理地址指针ItemPointer进行比较
int32 ItemPointerCompare(ItemPointer arg1, ItemPointer arg2)
{
/*
@ -61,6 +66,7 @@ int32 ItemPointerCompare(ItemPointer arg1, ItemPointer arg2)
return -1;
else if (b1 > b2)
return 1;
//如果块号相同,则按位置标识符进行比较
else if (arg1->ip_posid < arg2->ip_posid)
return -1;
else if (arg1->ip_posid > arg2->ip_posid)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,10 @@
#This is the main CMAKE for build bin.
#binCMake
#TGT_remote_SRC
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} TGT_remote_SRC)
#
set(TGT_remote_INC
${PROJECT_OPENGS_DIR}/contrib/log_fdw
${PROJECT_TRUNK_DIR}/distribute/bin/gds
@ -15,8 +19,11 @@ set(TGT_remote_INC
${ZLIB_INCLUDE_PATH}
)
#
set(remote_DEF_OPTIONS ${MACRO_OPTIONS})
#
set(remote_COMPILE_OPTIONS ${OPTIMIZE_OPTIONS} ${OS_OPTIONS} ${PROTECT_OPTIONS} ${WARNING_OPTIONS} ${BIN_SECURE_OPTIONS} ${CHECK_OPTIONS})
#
set(remote_LINK_OPTIONS ${BIN_LINK_OPTIONS})
#
add_static_objtarget(gausskernel_storage_remote TGT_remote_SRC TGT_remote_INC "${remote_DEF_OPTIONS}" "${remote_COMPILE_OPTIONS}" "${remote_LINK_OPTIONS}")

View File

@ -21,10 +21,17 @@
#
# ---------------------------------------------------------------------------------------
#通过该Makefile文件可以在openGauss数据库的存储/远程组件目录中,进行编译、链接和生成目标二进制文件等操作
#子目录和顶层构建目录指明该Makefile所在的子目录路径和顶层构建目录路径。
subdir = src/gausskernel/storage/remote
top_builddir = ../../../..
#包含全局Makefile文件包含顶层构建目录下的Makefile.global文件该文件定义了全局的编译选项和规则
include $(top_builddir)/src/Makefile.global
#条件判断根据当前执行的Make命令来判断是否需要包含依赖文件DEPEND
#如果当前执行的是"clean"或"distclean"命令,则不包含依赖文件
ifneq "$(MAKECMDGOALS)" "clean"
ifneq "$(MAKECMDGOALS)" "distclean"
ifneq "$(shell which g++ |grep hutaf_llt |wc -l)" "1"
@ -32,6 +39,7 @@ ifneq "$(MAKECMDGOALS)" "clean"
endif
endif
endif
#目标对象OBJS定义要编译生成的目标对象文件列表
OBJS = remote_adapter.o remote_read.o
#包含公共Makefile文件包含openGauss数据库中的common.mk文件该文件定义了具体的编译规则和链接选项等
include $(top_srcdir)/src/gausskernel/common.mk

View File

@ -54,17 +54,24 @@ int ReadCOrCsnFileForRemote(RelFileNode rnode, bytea** fileData);
* @Return: remote read error code
* @See also:
*/
//用于等待指定LSN日志序列号被回放到本地
// 常见的应用场景是在流复制Streaming Replication等待从节点接收并成功应用指定的事务日志
// primary_insert_lsn远程请求的LSN日志序列号
int XLogWaitForReplay(uint64 primary_insert_lsn, int timeout = DEFAULT_WAIT_TIMES)
{
int wait_times = 0;
/* local replayed lsn */
//获取本地正在回放的LSN
XLogRecPtr standby_replay_lsn = GetXLogReplayRecPtr(NULL, NULL);
/* if primary_insert_lsn > standby_replay_lsn then need wait */
//循环判断如果primary_insert_lsn大于standby_replay_lsn则需要继续等待
while (!XLByteLE(primary_insert_lsn, standby_replay_lsn)) {
/* if sleep to much times */
//判断等待次数是否超过超时时间
if (wait_times >= timeout) {
//如果超过则返回错误并打印相应日志信息
ereport(LOG,
(errmodule(MOD_REMOTE),
errmsg("replay slow. requre lsn %X/%X, replayed lsn %X/%X",
@ -72,41 +79,50 @@ int XLogWaitForReplay(uint64 primary_insert_lsn, int timeout = DEFAULT_WAIT_TIME
(uint32)primary_insert_lsn,
(uint32)(standby_replay_lsn >> 32),
(uint32)standby_replay_lsn)));
//等待超时则返回REMOTE_READ_NEED_WAIT远程读取需要等待
return REMOTE_READ_NEED_WAIT;
}
/* sleep 1s */
//进行1秒的延迟
pg_usleep(1000000L);
//递增等待次数
++wait_times;
/* get current replay lsn again */
//再次获取当前的回放LSN更新
(void)GetXLogReplayRecPtr(NULL, &standby_replay_lsn);
}
//直到primary_insert_lsn小于等于standby_replay_lsn表示LSN已经回放完成
//成功回放则返回REMOTE_READ_OK远程读取完成
return REMOTE_READ_OK;
}
/*
* Read block from buffer from primary, returning it as bytea
*/
//实现了从主节点读取缓冲区中的数据块并将其作为bytea类型返回
Datum gs_read_block_from_remote(PG_FUNCTION_ARGS)
{
uint32 spcNode;
uint32 dbNode;
uint32 relNode;
int16 bucketNode;
int32 forkNum;
uint64 blockNum;
uint32 blockSize;
uint64 lsn;
bool isForCU = false;
uint32 spcNode; //文件空间节点号
uint32 dbNode; //数据库节点号
uint32 relNode; //关系节点号
int16 bucketNode; //哈希桶节点号
int32 forkNum; //分叉号
uint64 blockNum; //块号
uint32 blockSize; //块大小
uint64 lsn; //日志序列号
bool isForCU = false; //是否是针对CU压缩单元块的请求
bytea* result = NULL;
int timeout = 0;
//检查当前用户是否为超级用户
if (GetUserId() != BOOTSTRAP_SUPERUSERID) {
//如果不是则抛出异常
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be initial account to read files"))));
}
/* handle optional arguments */
//解析传入的参数
spcNode = PG_GETARG_UINT32(0);
dbNode = PG_GETARG_UINT32(1);
relNode = PG_GETARG_UINT32(2);
@ -118,6 +134,7 @@ Datum gs_read_block_from_remote(PG_FUNCTION_ARGS)
isForCU = PG_GETARG_BOOL(8);
timeout = PG_GETARG_INT32(9);
//创建修复块的键RepairBlockKey对象并填充相关属性
RepairBlockKey key;
key.relfilenode.spcNode = spcNode;
key.relfilenode.dbNode = dbNode;
@ -128,16 +145,21 @@ Datum gs_read_block_from_remote(PG_FUNCTION_ARGS)
key.blocknum = blockNum;
/* get block from local buffer */
// 从本地缓冲区读取块
if (isForCU) {
// 从本地缓冲区读取 CU 块
/* if request to read CU block, we use forkNum column to replace colid. */
(void)StandbyReadCUforPrimary(key, blockNum, blockSize, lsn, timeout, &result);
} else {
//从本地缓冲区读取普通页面块
(void)StandbyReadPageforPrimary(key, blockSize, lsn, &result, timeout, NULL);
}
if (NULL != result) {
//成功获取到数据块则将其转换为bytea类型并返回
PG_RETURN_BYTEA_P(result);
} else {
//未获取到数据块则返回NULL
PG_RETURN_NULL();
}
}
@ -145,41 +167,50 @@ Datum gs_read_block_from_remote(PG_FUNCTION_ARGS)
/*
* Read block from buffer from primary, returning it as bytea
*/
//用于从主节点的压缩块缓冲区中读取数据并将其以bytea类型返回
Datum gs_read_block_from_remote_compress(PG_FUNCTION_ARGS)
{
RepairBlockKey key;
uint32 blockSize;
uint64 lsn;
int timeout = 0;
bool isForCU = false;
bytea* result = NULL;
RepairBlockKey key; // 修复块的关键信息
uint32 blockSize; // 块大小
uint64 lsn; // 日志序列号
int timeout = 0; // 超时时间
bool isForCU = false; // 是否请求读取CU块
bytea* result = NULL; // 返回的块数据
// 检查当前用户是否为超级用户
if (GetUserId() != BOOTSTRAP_SUPERUSERID) {
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be initial account to read files"))));
}
/* handle optional arguments */
key.relfilenode.spcNode = PG_GETARG_UINT32(0);
key.relfilenode.dbNode = PG_GETARG_UINT32(1);
key.relfilenode.relNode = PG_GETARG_UINT32(2);
key.relfilenode.bucketNode = PG_GETARG_INT16(3);
key.relfilenode.opt = PG_GETARG_UINT16(4);
key.forknum = PG_GETARG_INT32(5);
key.blocknum = (uint64)PG_GETARG_TRANSACTIONID(6);
blockSize = PG_GETARG_UINT32(7);
lsn = (uint64)PG_GETARG_TRANSACTIONID(8);
isForCU = PG_GETARG_BOOL(9);
timeout = PG_GETARG_INT32(10);
// 获取传入的参数值
key.relfilenode.spcNode = PG_GETARG_UINT32(0); // 表空间OID
key.relfilenode.dbNode = PG_GETARG_UINT32(1); // 数据库OID
key.relfilenode.relNode = PG_GETARG_UINT32(2); // 关系OID
key.relfilenode.bucketNode = PG_GETARG_INT16(3); // 桶OID
key.relfilenode.opt = PG_GETARG_UINT16(4); // 扩展选项
key.forknum = PG_GETARG_INT32(5); // 分支号
key.blocknum = (uint64)PG_GETARG_TRANSACTIONID(6); // 块号
blockSize = PG_GETARG_UINT32(7); // 块大小
lsn = (uint64)PG_GETARG_TRANSACTIONID(8); // 日志序列号
isForCU = PG_GETARG_BOOL(9); // 是否请求读取CU块
timeout = PG_GETARG_INT32(10); // 超时时间
/* get block from local buffer */
if (isForCU) {
// 从本地缓冲区读取 CU 块
/* if request to read CU block, we use forkNum column to replace colid. */
(void)StandbyReadCUforPrimary(key, key.blocknum, blockSize, lsn, timeout, &result);
} else {
// 从本地缓冲区读取页块
(void)StandbyReadPageforPrimary(key, blockSize, lsn, &result, timeout, NULL);
}
// 检查块是否成功获取
if (NULL != result) {
// 返回块数据
PG_RETURN_BYTEA_P(result);
} else {
// 返回空值
PG_RETURN_NULL();
}
}
@ -198,54 +229,71 @@ Datum gs_read_block_from_remote_compress(PG_FUNCTION_ARGS)
* @Return: remote read error code
* @See also:
*/
//用于从主节点读取某个文件中指定块号的CU数据块
int StandbyReadCUforPrimary(RepairBlockKey key, uint64 offset, int32 size, uint64 lsn, int32 timeout,
bytea** cudata)
{
// 确保输出参数cudata存在
Assert(cudata);
// 初始化返回值
int ret_code = REMOTE_READ_OK;
/* wait request lsn for replay */
//等待请求的LSN回放完成
// 检查是否处于恢复进程中
if (RecoveryInProgress()) {
// 等待指定LSN的日志重放超时时间为timeout
ret_code = XLogWaitForReplay(lsn, timeout);
if (ret_code != REMOTE_READ_OK) {
// 如果错误码不等于REMOTE_READ_OK则发生了错误记录错误日志并返回错误码
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("could not redo request lsn.")));
return ret_code;
}
}
// 使用传入的参数构造关系文件节点RelFileNode对象
RelFileNode relfilenode {key.relfilenode.spcNode, key.relfilenode.dbNode, key.relfilenode.relNode, InvalidBktId};
// 在匿名块中执行以下语句
{
/* read from disk */
//创建CFileNode对象用于标识具体的文件和文件块
CFileNode cfilenode(relfilenode, key.forknum, MAIN_FORKNUM);
// 创建CUStorage对象用于管理和访问CU数据
CUStorage* custorage = New(CurrentMemoryContext) CUStorage(cfilenode);
// 创建CU对象用于加载和处理CU数据
CU* cu = New(CurrentMemoryContext) CU();
cu->m_inCUCache = false;
// 从磁盘加载指定偏移量和大小的CU块到内存中的CU对象中
custorage->LoadCU(cu, offset, size, false, false);
/* check crc */
// 检查CU数据的CRC校验码是否正确
if (ret_code == REMOTE_READ_OK) {
if (!cu->CheckCrc()) {
// 如果CRC校验失败则记录错误日志并更新返回值为REMOTE_READ_CRC_ERROR
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("check CU crc error.")));
ret_code = REMOTE_READ_CRC_ERROR;
} else {
// 复制CU数据到返回值缓冲区
bytea* buf = (bytea*)palloc0(VARHDRSZ + size);
SET_VARSIZE(buf, size + VARHDRSZ);
errno_t rc = memcpy_s(VARDATA(buf), size, cu->m_compressedLoadBuf, size);
if (rc != EOK) {
// 如果复制过程中发生错误则记录错误日志并更新返回值为REMOTE_READ_MEMCPY_ERROR
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("memcpy_s error, retcode=%d", rc)));
ret_code = REMOTE_READ_MEMCPY_ERROR;
}
// 将buf对象赋值给输出参数cudata
*cudata = buf;
// 释放内存删除custorage和cu对象
DELETE_EX(custorage);
DELETE_EX(cu);
}
}
}
// 返回错误代码
return ret_code;
}
@ -263,18 +311,24 @@ int StandbyReadCUforPrimary(RepairBlockKey key, uint64 offset, int32 size, uint6
* @Return: remote read error code
* @See also:
*/
//读取主库的数据页并将数据存储在pagedata中
int StandbyReadPageforPrimary(RepairBlockKey key, uint32 blocksize, uint64 lsn, bytea** pagedata,
int timeout, const XLogPhyBlock *pblk)
{
//通过断言语句检查pagedata是否为空指针
Assert(pagedata);
//传入的数据块大小是否与定义的常量BLCKSZ相同
if (unlikely(blocksize != BLCKSZ))
//不同则返回错误码
return REMOTE_READ_BLCKSZ_NOT_SAME;
int ret_code = REMOTE_READ_OK;
/* wait request lsn for replay */
//如果系统正在恢复中
if (RecoveryInProgress()) {
//等待请求的LSN日志序列号进行重放
ret_code = XLogWaitForReplay(lsn, timeout);
if (ret_code != REMOTE_READ_OK) {
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("could not redo request lsn.")));
@ -282,14 +336,20 @@ int StandbyReadPageforPrimary(RepairBlockKey key, uint32 blocksize, uint64 lsn,
}
}
//该对象用于标识要读取数据页的关系文件在数据库中的位置
RelFileNode relfilenode{key.relfilenode.spcNode, key.relfilenode.dbNode, key.relfilenode.relNode,
key.relfilenode.bucketNode, key.relfilenode.opt};
//如果指针 pblk 不为空,则表示当前正在读取的是备库上的数据块
// 这时需要验证备库中的数据块位置与主库是否一致
if (NULL != pblk) {
//获取分段物理位置信息
SegPageLocation loc = seg_get_physical_location(relfilenode, key.forknum, key.blocknum);
//将loc.extent_size扩展大小转换为一个无符号8位整数
uint8 standby_relNode = (uint8) EXTENT_SIZE_TO_TYPE(loc.extent_size);
BlockNumber standby_block = loc.blocknum;
//验证备库中的关系文件和数据块的位置与主库是否一致
if (standby_relNode != pblk->relNode || standby_block != pblk->block) {
//如果不一致,将报告错误并返回相应的错误码
ereport(ERROR, (errmodule(MOD_REMOTE),
errmsg("Standby page file is invalid! Standby relnode is %u, "
"master is %u; Standby block is %u, master is %u.",
@ -297,73 +357,91 @@ int StandbyReadPageforPrimary(RepairBlockKey key, uint32 blocksize, uint64 lsn,
return REMOTE_READ_BLCKSZ_NOT_SAME;
}
}
//如果 pblk 为空,则表示当前正在读取主库上的数据块,不需要进行验证
//分配了一个大小为 BLCKSZ + VARHDRSZ 的内存空间,并设置 pageData 的头部为该空间的有效大小
bytea* pageData = (bytea*)palloc(BLCKSZ + VARHDRSZ);
SET_VARSIZE(pageData, BLCKSZ + VARHDRSZ);
//判断关系文件节点是否属于分段物理模型
if (IsSegmentPhysicalRelNode(relfilenode)) {
Buffer buffer = InvalidBuffer;
//如果是,则使用 spc_open 函数打开对应的分段空间,并获取关系文件的数据块总数
SegSpace *spc = spc_open(relfilenode.spcNode, relfilenode.dbNode, false, false);
BlockNumber spc_nblocks = spc_size(spc, relfilenode.relNode, key.forknum);
//查给定的数据块号 key.blocknum 是否小于数据块总数 spc_nblocks
if (key.blocknum < spc_nblocks) {
//读取磁盘上对应数据块的内容
buffer = ReadBufferFast(spc, relfilenode, key.forknum, key.blocknum, RBM_FOR_REMOTE);
}
//检查 Buffer 是否有效(即是否成功读取到了数据块)
if (BufferIsValid(buffer)) {
//如果有效,则通过 LockBuffer 将该 Buffer 锁定,并获取其对应的 Block
LockBuffer(buffer, BUFFER_LOCK_SHARE);
Block block = BufferGetBlock(buffer);
//将 Block 的内容拷贝到 pageData 中
errno_t rc = memcpy_s(VARDATA(pageData), BLCKSZ, block, BLCKSZ);
//如果 memcpy_s 函数执行失败
if (rc != EOK) {
//释放之前分配的内存空间 pageData通过 ereport 报告错误
pfree(pageData);
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("memcpy_s error, retcode=%d", rc)));
//将返回值 ret_code 设置为 REMOTE_READ_MEMCPY_ERROR
ret_code = REMOTE_READ_MEMCPY_ERROR;
}
//解锁并释放 Buffer
LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
ReleaseBuffer(buffer);
} else {
//没有成功读取到数据块,则将 ret_code 设置为 REMOTE_READ_SIZE_ERROR
ret_code = REMOTE_READ_SIZE_ERROR;
}
} else {
} else { //关系文件节点不是分段物理关系
bool hit = false;
//读取指定的数据块
/* read page, if PageIsVerified failed will long jump to PG_CATCH() */
Buffer buf = ReadBufferForRemote(relfilenode, key.forknum, key.blocknum, RBM_FOR_REMOTE, NULL, &hit, pblk);
//如果读取成功,则获取对应的 Buffer
if (BufferIsInvalid(buf)) {
//则通过 ereport 报告错误并设置返回值 ret_code 为 REMOTE_READ_BLCKSZ_NOT_SAME
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("standby page buffer is invalid!")));
return REMOTE_READ_BLCKSZ_NOT_SAME;
}
//通过 LockBuffer 锁定该 Buffer并获取其对应的 Block
LockBuffer(buf, BUFFER_LOCK_SHARE);
Block block = BufferGetBlock(buf);
//将 Block 的内容拷贝到 pageData 中
errno_t rc = memcpy_s(VARDATA(pageData), BLCKSZ, block, BLCKSZ);
//如果 memcpy_s 函数执行失败
if (rc != EOK) {
//释放之前分配的内存空间 pageData通过 ereport 报告错误
pfree(pageData);
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("memcpy_s error, retcode=%d", rc)));
//将返回值 ret_code 设置为 REMOTE_READ_MEMCPY_ERROR
ret_code = REMOTE_READ_MEMCPY_ERROR;
}
//解锁并释放 Buffer
LockBuffer(buf, BUFFER_LOCK_UNLOCK);
ReleaseBuffer(buf);
}
//读取数据块成功的情况下执行
if (ret_code == REMOTE_READ_OK) {
//外部调用者就可以获取到读取到的数据块的内容
*pagedata = pageData;
//计算并设置该页的校验和
PageSetChecksumInplace((Page) VARDATA(*pagedata), key.blocknum);
}
//如果读取操作成功,返回的结果是 REMOTE_READ_OK否则返回的是之前判断出的其他错误码
return ret_code;
}
const int RES_COL_NUM = 2;
//从远程读取文件,并返回结果
Datum gs_read_file_from_remote(PG_FUNCTION_ARGS)
{
RelFileNode rnode;
RelFileNode rnode; //文件节点信息
RemoteReadFileKey key;
int32 forknum;
uint32 blockstart;
int32 forknum; //分片号
uint32 blockstart; //起始块号
uint64 lsn;
bytea* result = NULL;
Datum values[RES_COL_NUM];
@ -375,10 +453,14 @@ Datum gs_read_file_from_remote(PG_FUNCTION_ARGS)
int parano = 0;
XLogRecPtr current_lsn = InvalidXLogRecPtr;
//检查当前用户是否具有足够的权限执行该函数
if (GetUserId() != BOOTSTRAP_SUPERUSERID) {
//如果不是超级用户,则会抛出错误并中断执行
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be initial account to read files"))));
}
/* handle optional arguments */
//使用 PG_GETARG_* 宏来获取每个参数并将它们存储在对应的变量中
//这些参数是通过调用该函数的 SQL 语句传递给函数的
rnode.spcNode = PG_GETARG_UINT32(parano++);
rnode.dbNode = PG_GETARG_UINT32(parano++);
rnode.relNode = PG_GETARG_UINT32(parano++);
@ -390,42 +472,54 @@ Datum gs_read_file_from_remote(PG_FUNCTION_ARGS)
timeout = PG_GETARG_INT32(parano++);
if (rnode.spcNode != 1 && rnode.spcNode != 2) {
//存储文件的表空间节点号不是 1 或 2需要获取数据文件
/* get tale data file */
//初始化了一个 key 变量,并将传入的参数赋值给 key 的相应字段
key.relfilenode = rnode;
key.forknum = forknum;
key.blockstart = blockstart;
//检查 forknum 的值是否等于 MAIN_FORKNUM
if (forknum != MAIN_FORKNUM) {
//如果不是,会发出一个警告并返回 NULL
ereport(WARNING, (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
(errmsg("Forknum should be 0. Now is %d. \n", forknum))));
PG_RETURN_NULL();
}
//forknum 等于 MAIN_FORKNUM远程读取文件
//读取指定位置的文件内容
ret_code = ReadFileForRemote(&key, lsn, &result, timeout);
} else {
} else {//rnode.spcNode 的值为 1 或 2
//读取与 rnode 相关的文件内容
ret_code = ReadCOrCsnFileForRemote(rnode, &result);
}
if (ret_code != REMOTE_READ_OK) {
//如果文件读取失败,返回 NULL
PG_RETURN_NULL();
}
if (!RecoveryInProgress()) {
//如果当前没有进行恢复操作函数会获取当前的逻辑日志序列号LSN
current_lsn = GetXLogInsertRecPtr();
}
//创建一个元组描述符TupleDesc对象并设置两个字段file 和 lsn
tupdesc = CreateTemplateTupleDesc(RES_COL_NUM, false, TAM_HEAP);
parano = 1;
TupleDescInitEntry(tupdesc, (AttrNumber)parano++, "file", BYTEAOID, -1, 0);
TupleDescInitEntry(tupdesc, (AttrNumber)parano++, "lsn", XIDOID, -1, 0);
//values 数组存储了对应字段的值nulls 数组表示对应字段是否为 NULL
values[0] = PointerGetDatum(result);
nulls[0] = false;
values[1] = UInt64GetDatum(current_lsn);
nulls[1] = false;
//将函数的返回值转换为合适的格式,并返回给调用者
tupdesc = BlessTupleDesc(tupdesc);
tuple = heap_form_tuple(tupdesc, values, nulls);
PG_RETURN_DATUM(HeapTupleGetDatum(tuple));
}
//获取指定文件在远程节点上的大小
Datum gs_read_file_size_from_remote(PG_FUNCTION_ARGS)
{
RelFileNode rnode;
@ -436,10 +530,13 @@ Datum gs_read_file_size_from_remote(PG_FUNCTION_ARGS)
int parano = 0;
int ret_code = REMOTE_READ_OK;
//检查当前用户 id 是否是 Bootstrap 超级用户
if (GetUserId() != BOOTSTRAP_SUPERUSERID) {
//如果不是,则抛出一个权限不足的错误
ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be initial account to read files"))));
}
/* handle optional arguments */
//从输入参数中获取文件所在的 RelFileNode、forknum、lsn 和 timeout 等信息
rnode.spcNode = PG_GETARG_UINT32(parano++);
rnode.dbNode = PG_GETARG_UINT32(parano++);
rnode.relNode = PG_GETARG_UINT32(parano++);
@ -450,14 +547,18 @@ Datum gs_read_file_size_from_remote(PG_FUNCTION_ARGS)
timeout = PG_GETARG_INT32(parano++);
/* get file size */
//尝试在远程节点上获取指定文件的大小
ret_code = ReadFileSizeForRemote(rnode, forknum, lsn, &size, timeout);
if (ret_code == REMOTE_READ_OK) {
//如果成功获取文件大小,则通过 PG_RETURN_INT64 宏将 size 的值作为函数的返回值返回给调用者
PG_RETURN_INT64(size);
} else {
//如果获取文件大小失败,则返回 null 值
PG_RETURN_NULL();
}
}
//在远程节点上获取指定文件的大小,并将结果通过 res 参数返回
int ReadFileSizeForRemote(RelFileNode rnode, int32 forknum, XLogRecPtr lsn, int64* res, int timeout)
{
SMgrRelation smgr = NULL;
@ -465,41 +566,56 @@ int ReadFileSizeForRemote(RelFileNode rnode, int32 forknum, XLogRecPtr lsn, int6
int ret_code = REMOTE_READ_OK;
/* wait request lsn for replay */
//检查是否处于恢复状态
if (RecoveryInProgress()) {
//如果正在进行恢复,等待请求的 lsn 进行重放
ret_code = XLogWaitForReplay(lsn, timeout);
//如果等待超时或发生错误,函数会抛出一个错误并返回相应的错误码
if (ret_code != REMOTE_READ_OK) {
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("could not redo request lsn.")));
return ret_code;
}
}
//发送请求进行立即强制的检查点,并等待完成
RequestCheckpoint(CHECKPOINT_IMMEDIATE | CHECKPOINT_FORCE | CHECKPOINT_WAIT);
/* check whether the file exists. not exist, return size -1 */
//检查文件是否存在
struct stat statBuf;
//根据 rnode 和 forknum 构建文件路径
char* path = relpathperm(rnode, forknum);
//调用 stat 函数检查文件的状态信息
if (stat(path, &statBuf) < 0 && errno == ENOENT) {
//如果文件不存在,则将 res 设置为 -1并直接返回
*res = -1;
pfree(path);
return ret_code;
}
pfree(path);
//如果文件存在,且不属于分段文件
if (!IsSegmentFileNode(rnode)) {
//打开相应的 SMgrRelation
smgr = smgropen(rnode, InvalidBackendId);
//获取文件的块数量
nblock = smgrnblocks(smgr, forknum);
//关闭所有 SMgrRelation
smgrcloseall();
} else {
} else { //如果文件属于分段文件
//打开相应的 SegSpace
SegSpace *spc = spc_open(rnode.spcNode, rnode.dbNode, true, true);
spc_datafile_create(spc, rnode.relNode, forknum);
//获取分段文件的大小
nblock = spc_size(spc, rnode.relNode, forknum);
}
//将 nblock 乘以 BLCKSZ得到文件的大小
*res = nblock * BLCKSZ;
return ret_code;
}
//从远程文件中读取指定块的内容,并将读取到的数据复制到 pageData 中的相应位置
int ReadFileByReadBufferComom(RemoteReadFileKey *key, bytea* pageData, uint32 nblock)
{
int ret_code = REMOTE_READ_OK;
@ -507,52 +623,68 @@ int ReadFileByReadBufferComom(RemoteReadFileKey *key, bytea* pageData, uint32 nb
uint32 j = 0;
uint32 blk_start;
uint32 blk_end;
bool hit = false;
bool hit = false; //标志是否命中
char* bufBlock = NULL;
errno_t rc;
/* get the segno file block start and block end */
//计算要读取的段号文件块的起始块和终止块
blk_start = key->blockstart;
blk_end = (nblock >= blk_start + MAX_BATCH_READ_BLOCKNUM ? blk_start + MAX_BATCH_READ_BLOCKNUM : nblock);
//使用循环从起始块到终止块,依次读取每个块的内容
for (i = blk_start, j = 0; i < blk_end; i++, j++) {
/* read page, if PageIsVerified failed will long jump */
//来获取访问策略,将策略指针赋给 bstrategy 变量
BufferAccessStrategy bstrategy = GetAccessStrategy(BAS_REPAIR);
//获取指定块的缓冲区,并通过 Buffer 类型的变量 buf 进行引用
Buffer buf = ReadBufferForRemote(key->relfilenode, key->forknum, i, RBM_FOR_REMOTE, bstrategy, &hit, NULL);
//如果 buf 无效,即无法获取有效的缓冲区
if (BufferIsInvalid(buf)) {
//返回错误码 REMOTE_READ_BLCKSZ_NOT_SAME
ereport(WARNING, (errmodule(MOD_REMOTE), errmsg("repair file failed!")));
return REMOTE_READ_BLCKSZ_NOT_SAME;
}
//对缓冲区进行共享锁定
LockBuffer(buf, BUFFER_LOCK_SHARE);
bufBlock = (char*)BufferGetBlock(buf);
//将缓冲区的内容复制到 pageData 的相应位置
rc = memcpy_s(VARDATA(pageData) + j * BLCKSZ, BLCKSZ, bufBlock, BLCKSZ);
//memcpy_s 函数调用失败
if (rc != EOK) {
ereport(WARNING, (errmodule(MOD_REMOTE), errmsg("repair file failed, memcpy_s error, retcode=%d", rc)));
//设置 ret_code 为 REMOTE_READ_MEMCPY_ERROR
ret_code = REMOTE_READ_MEMCPY_ERROR;
//返回之前解锁并释放缓冲区
LockBuffer(buf, BUFFER_LOCK_UNLOCK);
ReleaseBuffer(buf);
return ret_code;
}
//解锁并释放缓冲区
LockBuffer(buf, BUFFER_LOCK_UNLOCK);
ReleaseBuffer(buf);
//每个块的内容复制完成后,为复制的页面计算并设置校验和
PageSetChecksumInplace((Page) (VARDATA(pageData) + j * BLCKSZ), i);
}
//返回 ret_code 表示操作的结果
return ret_code;
}
const int MAX_RETRY_TIMES = 60;
//根据指定的参数从磁盘或者远程位置读取指定块的数据
int ReadFileByReadDisk(SegSpace* spc, RemoteReadFileKey *key, char* bufBlock, BlockNumber blocknum)
{
int ret_code = REMOTE_READ_OK;
int pageStatus;
int retryTimes = 0;
int pageStatus; //页面状态
int retryTimes = 0; //重试次数
//如果传入的文件节点是段文件节点
if (IsSegmentFileNode(key->relfilenode)) {
//创建一个伪造的文件节点 fakenode将其属性设置为传入的文件节点属性
// 但将 bucketNode 属性设置为 SegmentBktId
RelFileNode fakenode = {
.spcNode = key->relfilenode.spcNode,
.dbNode = key->relfilenode.dbNode,
@ -560,47 +692,63 @@ int ReadFileByReadDisk(SegSpace* spc, RemoteReadFileKey *key, char* bufBlock, Bl
.bucketNode = SegmentBktId,
.opt = 0
};
SEG_RETRY:
SEG_RETRY:
//从远程位置读取指定块的数据,并将数据存储到 bufBlock 中
seg_physical_read(spc, fakenode, key->forknum, blocknum, (char *)bufBlock);
retryTimes++;
//检查页面的校验和是否匹配
if (PageIsVerified((Page)bufBlock, blocknum)) {
//如果匹配,则将页面状态设置为 SMGR_RD_OK
pageStatus = SMGR_RD_OK;
} else {
//如果校验和不匹配,将页面状态设置为 SMGR_RD_CRC_ERROR
pageStatus = SMGR_RD_CRC_ERROR;
//如果重试次数小于最大重试次数
if (retryTimes < MAX_RETRY_TIMES) {
/* sleep 10ms */
pg_usleep(10000L);
//等待 10 毫秒,并跳转到标签 SEG_RETRY 处再次尝试读取数据
goto SEG_RETRY;
} else {
} else { //如果重试次数达到最大重试次数
//释放 bufBlock 的内存,并输出警告信息
pfree(bufBlock);
ereport(WARNING, (errmodule(MOD_REMOTE),
errmsg("repair file failed, read page crc check error, page: %u/%u/%u/%d, "
"forknum is %d, block num is %u", key->relfilenode.spcNode, key->relfilenode.dbNode,
key->relfilenode.relNode, key->relfilenode.bucketNode, key->forknum, blocknum)));
//将返回码 ret_code 设置为 REMOTE_READ_CRC_ERROR并返回该返回码
ret_code = REMOTE_READ_CRC_ERROR;
return ret_code;
}
}
} else {
} else { //传入的文件节点不是段文件节点
//打开指定文件节点对应的存储管理器关系对象 smgr
SMgrRelation smgr = smgropen(key->relfilenode, InvalidBackendId);
/* standby read page, replay finish, there will be no synchronous changes. */
//从磁盘读取指定块的数据,并将数据存储到 bufBlock 中
pageStatus = smgrread(smgr, key->forknum, blocknum, (char *)bufBlock);
retryTimes++;
//检查页面状态是否为 SMGR_RD_OK
if (pageStatus != SMGR_RD_OK) {
//如果不是,则释放 bufBlock 的内存,并输出警告信息
pfree(bufBlock);
ereport(WARNING, (errmodule(MOD_REMOTE),
errmsg("repair file failed, read page crc check error, page: %u/%u/%u/%d, "
"forknum is %d, block num is %u", key->relfilenode.spcNode, key->relfilenode.dbNode,
key->relfilenode.relNode, key->relfilenode.bucketNode, key->forknum, blocknum)));
//然后将返回码 ret_code 设置为 REMOTE_READ_CRC_ERROR
ret_code = REMOTE_READ_CRC_ERROR;
//关闭存储管理器关系对象 smgr
smgrclose(smgr);
return ret_code;
}
//关闭存储管理器关系对象 smgr
smgrclose(smgr);
}
return ret_code;
}
//远程读取指定文件的数据块并将读取的数据存储到bytea类型的变量中
int ReadFileForRemote(RemoteReadFileKey *key, XLogRecPtr lsn, bytea** fileData, int timeout)
{
int ret_code = REMOTE_READ_OK;
@ -616,23 +764,33 @@ int ReadFileForRemote(RemoteReadFileKey *key, XLogRecPtr lsn, bytea** fileData,
errno_t rc;
/* wait request lsn for replay */
//检查是否在恢复模式中
if (RecoveryInProgress()) {
//如果是则等待请求的日志序列号lsn进行重放
ret_code = XLogWaitForReplay(lsn, timeout);
if (ret_code != REMOTE_READ_OK) {
//如果超时或出错,返回错误代码
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("could not redo request lsn.")));
return ret_code;
}
}
//如果在恢复模式中或者目标文件是段文件节点
if (RecoveryInProgress() || IsSegmentFileNode(key->relfilenode)) {
//发送请求进行检查点
RequestCheckpoint(CHECKPOINT_WAIT | CHECKPOINT_FORCE | CHECKPOINT_IMMEDIATE);
}
/* get block num */
//如果不是段文件节点
if (!IsSegmentFileNode(key->relfilenode)) {
//打开对应的SMgrRelation
smgr = smgropen(key->relfilenode, InvalidBackendId);
//获取块数
nblock = smgrnblocks(smgr, key->forknum);
//关闭SMgrRelation
smgrclose(smgr);
} else {
} else { //如果是段文件节点
//打开对应的SegSpace
spc = spc_open(key->relfilenode.spcNode, key->relfilenode.dbNode, false, false);
if (!spc) {
ereport(WARNING, (errmodule(MOD_REMOTE),
@ -640,50 +798,66 @@ int ReadFileForRemote(RemoteReadFileKey *key, XLogRecPtr lsn, bytea** fileData,
key->relfilenode.spcNode, key->relfilenode.dbNode)));
return REMOTE_READ_IO_ERROR;
}
//获取块数
nblock = spc_size(spc, key->relfilenode.relNode, key->forknum);
}
//检查请求的起始块blockstart是否超出文件块数
if (nblock <= key->blockstart) {
ret_code = REMOTE_READ_SIZE_ERROR;
return ret_code;
}
/* get the segno file block start and block end */
//根据起始块和最大批量读取块数计算出要读取的段segno的起始块blk_start和结束块blk_end
blk_start = key->blockstart;
blk_end = (nblock >= blk_start + MAX_BATCH_READ_BLOCKNUM ? blk_start + MAX_BATCH_READ_BLOCKNUM : nblock);
//分配用于存储读取的数据的内存空间
pageData = (bytea*)palloc((blk_end - blk_start) * BLCKSZ + VARHDRSZ);
SET_VARSIZE(pageData, ((blk_end - blk_start) * BLCKSZ + VARHDRSZ));
/* primary read page, need read page by ReadBuffer_common */
//如果不是段文件节点并且不在恢复模式中
if (!IsSegmentFileNode(key->relfilenode) && !RecoveryInProgress()) {
//读取主数据节点的页数据
ret_code = ReadFileByReadBufferComom(key, pageData, nblock);
if (ret_code != REMOTE_READ_OK) {
//如果读取失败,则释放分配的内存空间,返回错误代码
pfree(pageData);
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("read file failed!")));
return ret_code;
}
} else {
} else { //如果是段文件节点或者在恢复模式中
// 判断是否使用ADIOAsynchronous Direct I/O读取数据
ADIO_RUN()
{
//如果是则使用adio_align_alloc函数分配对齐的内存块bufBlock大小为BLCKSZ
bufBlock = (Page)adio_align_alloc(BLCKSZ);
}
ADIO_ELSE()
{
//如果不是则使用palloc函数分配内存块bufBlock大小为BLCKSZ
bufBlock = (Page)palloc(BLCKSZ);
}
ADIO_END();
//用循环从起始块blk_start到结束块blk_end依次读取数据
for (i = blk_start, j = 0; i < blk_end; i++, j++) {
//并存储到bufBlock中
ret_code = ReadFileByReadDisk(spc, key, bufBlock, i);
if (ret_code != REMOTE_READ_OK) {
//读取失败则释放分配的内存块bufBlock和pageData
pfree(bufBlock);
pfree(pageData);
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("repair file failed, read block %u error, retcode=%d",
i, rc)));
//返回错误代码
return ret_code;
}
//使用memcpy_s函数将bufBlock中的数据复制到pageData的合适位置上
rc = memcpy_s(VARDATA(pageData) + j * BLCKSZ, BLCKSZ, bufBlock, BLCKSZ);
if (rc != EOK) {
//复制失败则释放分配的内存块bufBlock和pageData返回错误代码
pfree(bufBlock);
pfree(pageData);
ret_code = REMOTE_READ_MEMCPY_ERROR;
@ -691,22 +865,25 @@ int ReadFileForRemote(RemoteReadFileKey *key, XLogRecPtr lsn, bytea** fileData,
return ret_code;
}
}
//释放分配的内存块bufBlock
pfree(bufBlock);
//如果读取失败则释放分配的内存空间pageData返回错误代码
if (ret_code != REMOTE_READ_OK) {
pfree(pageData);
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("read file failed!")));
return ret_code;
}
}
//将读取的数据赋值给fileData指针
*fileData = pageData;
//返回读取结果代码
return ret_code;
}
const int REGR_MCR_SIZE_1MB = 1048576;
const int CLOG_NODE = 1;
const int CSN_NODE = 2;
//读取指定文件根据RelFileNode确定的数据并将数据存储到bytea类型的变量中
int ReadCOrCsnFileForRemote(RelFileNode rnode, bytea** fileData)
{
uint32 flags = O_RDWR | PG_BINARY;
@ -718,6 +895,7 @@ int ReadCOrCsnFileForRemote(RelFileNode rnode, bytea** fileData)
char *buffer = (char*)palloc(logSize);
int result = -1;
//根据给定的文件类型rnode.spcNode确定文件所在的路径logType
if (rnode.spcNode == CLOG_NODE) {
logType = "pg_clog";
} else if (rnode.spcNode == CSN_NODE) {
@ -726,42 +904,54 @@ int ReadCOrCsnFileForRemote(RelFileNode rnode, bytea** fileData)
ereport(LOG, (errmodule(MOD_SEGMENT_PAGE), errmsg("File type\"%u\" does not exist, stop read here.",
rnode.spcNode)));
}
//构建完整的文件路径path格式为"logType/relNode"其中relNode是文件的节点号
rc = snprintf_s(path, MAX_PATH, MAX_PATH - 1, "%s/%012u", logType, rnode.relNode);
securec_check_ss(rc, "\0", "\0");
//调用BasicOpenFile函数以读写模式打开文件并获取文件描述符fd
fd = BasicOpenFile(path, flags, S_IWUSR | S_IRUSR);
if (fd < 0) {
//如果打开失败
pfree(buffer);
if (errno != ENOENT) {
if (errno != ENOENT) { //且错误码不是ENOENT
ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", path)));
}
// The file does not exist, break.
ereport(LOG,
(errmodule(MOD_SEGMENT_PAGE), errmsg("File \"%s\" does not exist, stop read here.", path)));
pfree(path);
//则报错并返回-1
return -1;
}
//为了记录这次读取文件数据的等待事件
pgstat_report_waitevent(WAIT_EVENT_DATA_FILE_READ);
//分配用于存储文件数据的内存空间buffer大小为logSize其中logSize默认为16 MB
//使用pread函数从文件中读取数据将读取的数据存储到buffer中。读取的字节数由nbytes返回
uint32 nbytes = pread(fd, buffer, logSize, 0);
//结束等待事件的记录
pgstat_report_waitevent(WAIT_EVENT_END);
if (close(fd)) {
if (close(fd)) { //关闭文件,并检查关闭是否成功
pfree(path);
pfree(buffer);
ereport(ERROR, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", path)));
}
//检查实际读取的字节数nbytes是否大于logSize
if (nbytes > logSize) {
//如果是,则表示读取的数据超过了预设的大小
pfree(buffer);
ereport(ERROR,
(errcode(MOD_SEGMENT_PAGE),
errcode_for_file_access(),
errmsg("could not read file %s. nbytes:%u, logSize:%u", path, nbytes, logSize)));
pfree(path);
//报错并返回-1
return -1;
} else {
//分配用于存储读取的数据的内存空间pageData大小为nbytes + VARHDRSZ其中VARHDRSZ是页头变长字段的大小
bytea* pageData = (bytea*)palloc(nbytes + VARHDRSZ);
SET_VARSIZE(pageData, (nbytes + VARHDRSZ));
//将读取的数据从buffer复制到pageData中
rc = memcpy_s(VARDATA(pageData), nbytes, buffer, nbytes);
if (rc != EOK) {
pfree(path);
@ -769,12 +959,14 @@ int ReadCOrCsnFileForRemote(RelFileNode rnode, bytea** fileData)
pfree(buffer);
ereport(ERROR, (errmodule(MOD_REMOTE), errmsg("repair file failed, memcpy_s error, retcode=%d", rc)));
return -1;
} else {
} else { //将pageData赋值给fileData指针
*fileData = pageData;
result = 0;
}
}
//释放不再使用的内存空间
pfree(path);
pfree(buffer);
//返回读取结果代码
return result;
}

View File

@ -22,6 +22,9 @@
*
* ---------------------------------------------------------------------------------------
*/
//提供了一些函数来处理远程读取数据的相关操作
// 并且根据数据库运行模式和配置文件的设置来确定是否开启远程读取功能
#include "postgres.h"
#include "knl/knl_variable.h"
@ -37,9 +40,13 @@
* @IN error_code: remote read error code
* @Return: remote read error message
*/
//用于根据远程读取的错误码返回相应的错误消息
//error_code远程读取的错误码
const char* RemoteReadErrMsg(int error_code)
{
//const char*,远程读取的错误消息
const char* error_msg = "";
//根据不同的错误码对应设置不同的错误消息
switch (error_code) {
case REMOTE_READ_OK:
error_msg = "normal";
@ -79,35 +86,51 @@ const char* RemoteReadErrMsg(int error_code)
return error_msg;
}
//用于获取主服务的地址
//远程主机的地址,远程主机的端口号,格式化写入address数组中
//address存储主服务地址的字符数组指针
//address_lenaddress数组的长度
void GetPrimaryServiceAddress(char *address, size_t address_len)
{
//参数的有效性检查
if (address == NULL || address_len == 0 || t_thrd.walreceiverfuncs_cxt.WalRcv == NULL)
//如果传入的address为空指针、address_len为0或者全局变量t_thrd.walreceiverfuncs_cxt.WalRcv为空则直接返回
return;
bool is_running = false;
volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv;
int rc = 0;
//取walrcv->mutex自旋锁
SpinLockAcquire(&walrcv->mutex);
//通过读取walrcv->isRunning的值来确定WAL接收是否正在运行
is_running = walrcv->isRuning;
//释放自旋锁
SpinLockRelease(&walrcv->mutex);
//如果walrcv->pid等于0或者is_running为false
if (walrcv->pid == 0 || !is_running)
//表示WAL接收未启动或者已停止直接返回
return;
//获取walrcv->mutex自旋锁
SpinLockAcquire(&walrcv->mutex);
//通过snprintf_s函数将主服务地址的字符串格式化写入address数组中
// 格式为"%s@%d",其中%s代表远程主机的地址%d代表远程主机的端口号
rc = snprintf_s(address, address_len, (address_len - 1), "%s@%d", walrcv->conn_channel.remotehost,
walrcv->conn_channel.remoteport);
securec_check_ss(rc, "\0", "\0");
//释放自旋锁
SpinLockRelease(&walrcv->mutex);
}
/*
* @Description: get remote address
* @IN/OUT first_address: first address
* @IN/OUT second_address: second_address
* @IN address_len: address len
* @IN/OUT first_address: first address firstAddress
* @IN/OUT second_address: second_address secondAddress
* @IN address_len: address len addressLenfirstAddress和secondAddress数组的长度
*/
//用于获取远程读取地址
void GetRemoteReadAddress(char* firstAddress, char* secondAddress, size_t addressLen)
{
char ip[MAX_IPADDR_LEN] = {0};
@ -115,40 +138,52 @@ void GetRemoteReadAddress(char* firstAddress, char* secondAddress, size_t addres
errno_t rc = EOK;
/* make sure first_address is correct */
//进行参数的有效性检查
if (firstAddress == NULL || addressLen == 0)
//传入的firstAddress为空指针、addressLen为0则直接返回
return;
volatile HaShmemData* hashmdata = t_thrd.postmaster_cxt.HaShmData;
ServerMode serverMode = hashmdata->current_mode;
volatile HaShmemData* hashmdata = t_thrd.postmaster_cxt.HaShmData; //该结构体包含了与共享内存相关的数据
ServerMode serverMode = hashmdata->current_mode; //存储当前服务器的工作模式
if (IS_DN_DUMMY_STANDYS_MODE()) {
//如果是虚拟备模式
if (serverMode == PRIMARY_MODE && !IsPrimaryStandByReadyToRemoteRead()) {
/// 当前服务器模式为主模式且主备切换已完成,则直接返回
return;
}
//如果不满足上面的条件,则检查第二个备份节点的连接是否存在
if (t_thrd.postmaster_cxt.ReplConnArray[1]) {
//将其地址信息格式化为字符串并存储到firstAddress中
rc = snprintf_s(firstAddress, addressLen, (addressLen - 1),
"%s@%d", t_thrd.postmaster_cxt.ReplConnArray[1]->remotehost,
t_thrd.postmaster_cxt.ReplConnArray[1]->remoteport);
securec_check_ss(rc, "", "");
}
} else if (IS_DN_MULTI_STANDYS_MODE()) {
// 如果处于多节点备份模式
if (serverMode == PRIMARY_MODE) {
// 如果当前服务器模式为主模式
// 获取优选备份节点的地址信息
GetFastestReplayStandByServiceAddress(firstAddress, secondAddress, addressLen);
if (firstAddress[0] != '\0') {
// 如果第一个地址不为空则将其格式化为ip:port的形式并存储到firstAddress中
GetIPAndPort(firstAddress, ip, port, MAX_IPADDR_LEN);
rc = snprintf_s(firstAddress, addressLen, (addressLen - 1), "%s@%s", ip, port);
securec_check_ss(rc, "", "");
}
if (secondAddress[0] != '\0') {
// 如果第二个地址不为空则将其格式化为ip:port的形式并存储到secondAddress中
GetIPAndPort(secondAddress, ip, port, MAX_IPADDR_LEN);
rc = snprintf_s(secondAddress, addressLen, (addressLen - 1), "%s@%s", ip, port);
securec_check_ss(rc, "", "");
}
} else if (serverMode == STANDBY_MODE) {
// 如果当前服务器模式为备模式
GetPrimaryServiceAddress(firstAddress, addressLen);
if (firstAddress[0] != '\0') {
// 如果地址不为空则将其格式化为ip:port的形式并存储到firstAddress中
GetIPAndPort(firstAddress, ip, port, MAX_IPADDR_LEN);
rc = snprintf_s(firstAddress, addressLen, (addressLen - 1), "%s@%s", ip, port);
securec_check_ss(rc, "", "");
@ -159,6 +194,11 @@ void GetRemoteReadAddress(char* firstAddress, char* secondAddress, size_t addres
}
}
//从地址字符串中提取IP地址和端口号
//address表示待处理的地址字符串
// ip表示存储提取的IP地址
// port表示存储提取的端口号
// len表示ip和port缓冲区的长度
void GetIPAndPort(char* address, char* ip, char* port, size_t len)
{
char* outerPtr = NULL;
@ -166,10 +206,13 @@ void GetIPAndPort(char* address, char* ip, char* port, size_t len)
char* tmpIp;
char* tempPort;
//对address进行分割以'@'为分隔符将地址字符串分割为IP地址和端口号两部分
tmpIp = strtok_r(address, "@", &outerPtr);
tempPort = strtok_r(NULL, "@", &outerPtr);
//判断提取的IP地址和端口号是否有效即不为空且长度不超过缓冲区的限制
if (tmpIp != NULL && tmpIp[0] != '\0' && tempPort != NULL && tempPort[0] != '\0' &&
strlen(tmpIp) + strlen(tempPort) + 1 < len) {
//如果满足条件则使用strcpy_s函数将提取的IP地址和端口号拷贝到ip和port缓冲区中
rc = strcpy_s(ip, MAX_IPADDR_LEN, tmpIp);
securec_check(rc, "", "");
rc = strcpy_s(port, MAX_IPADDR_LEN, tempPort);
@ -182,13 +225,17 @@ void GetIPAndPort(char* address, char* ip, char* port, size_t len)
* @Description: have remote node to read
* @Return: true if have remote node
*/
//用于判断是否存在远程节点可供读取
bool CanRemoteRead()
{
volatile HaShmemData* hashmdata = t_thrd.postmaster_cxt.HaShmData;
//获取当前的服务器模式
ServerMode serveMode = hashmdata->current_mode;
//检查远程读取模式是否打开 && 是否不处于不含备机的数据节点模式 && 是否是PGXC数据节点
//检查服务器模式是否不是NORMAL_MODE、PENDING_MODE和STANDBY_MODE
if (IsRemoteReadModeOn() && !IS_DN_WITHOUT_STANDBYS_MODE() && IS_PGXC_DATANODE && serveMode != NORMAL_MODE &&
serveMode != PENDING_MODE && serveMode != STANDBY_MODE) {
//如果上述所有条件都满足则返回true表示存在远程节点可供读取
return true;
}
return false;
@ -198,8 +245,11 @@ bool CanRemoteRead()
* @Description: remote mode is on
* @Return: true if emote read is on
*/
//用于判断远程读取模式是否开启
bool IsRemoteReadModeOn()
{
//获取到当前的远程读取模式并将其与REMOTE_READ_OFF进行比较
//果当前远程读取模式不等于REMOTE_READ_OFF则表示远程读取模式开启函数返回true
return g_instance.attr.attr_storage.remote_read_mode != REMOTE_READ_OFF;
}
@ -207,16 +257,22 @@ bool IsRemoteReadModeOn()
* @Description: set remote mode off and get old remote_read_mode
* @Return: old remote_read_mode
*/
//用于 关闭远程读取模式 并 返回被关闭远程读取模式
int SetRemoteReadModeOffAndGetOldMode()
{
//获取到当前的远程读取模式
int oldRemoteRead = g_instance.attr.attr_storage.remote_read_mode;
//关闭远程读取
g_instance.attr.attr_storage.remote_read_mode = REMOTE_READ_OFF;
//返回被关闭远程读取模式
return oldRemoteRead;
}
//用于设置远程读取模式
//接受一个整型参数mode用于指定要设置的远程读取模式
void SetRemoteReadMode(int mode)
{
//将远程读取模式设置为指定的值
g_instance.attr.attr_storage.remote_read_mode = mode;
return;

View File

@ -30,22 +30,29 @@
#include <unordered_set>
#include <vector>
// ipv6arr定义为一个长度为16的无符号字符数组类型表示IPv6地址的字节序列
typedef unsigned char ipv6arr[16];
// 128=2*64
typedef struct IPV6_64_s
{
uint64_t lower;
uint64_t upper;
} IPV6_64;
// 128=4*32
typedef struct IPV6_32_s
{
//书写顺序 d , c , b , a
uint32_t a;
uint32_t b;
uint32_t c;
uint32_t d;
} IPV6_32;
/*
* IPV6的联合体类型
* ip_64和ip_32
*
* IPv6地址表示为64位整数形式32
*/
typedef union IPV6_s
{
IPV6_64 ip_64;
@ -75,14 +82,14 @@ inline const IPV6 operator+(const IPV6 lhs, const IPV6 rhs)
uint64_t tmp;
IPV6 new_ip {0,0};
if (lhs.ip_32.b == 0x0000FFFF) {
if (lhs.ip_32.b == 0x0000FFFF) { // ipv4
new_ip.ip_32.a = lhs.ip_32.a + rhs.ip_32.a;
new_ip.ip_32.b = 0x0000FFFF;
return new_ip;
}
// ipv6
new_ip.ip_32.a = tmp = ((int64_t)lhs.ip_32.a) + rhs.ip_32.a;
new_ip.ip_32.b = tmp = ((int64_t)lhs.ip_32.b) + rhs.ip_32.b + (tmp >> 32);
new_ip.ip_32.b = tmp = ((int64_t)lhs.ip_32.b) + rhs.ip_32.b + (tmp >> 32); //溢出
new_ip.ip_32.c = tmp = ((int64_t)lhs.ip_32.c) + rhs.ip_32.c + (tmp >> 32);
new_ip.ip_32.d = ((int64_t)lhs.ip_32.d) + rhs.ip_32.d + (tmp >> 32);
return new_ip;
@ -97,13 +104,13 @@ inline const IPV6 operator+(const IPV6 lhs, const int i)
inline const IPV6 operator-(const IPV6 lhs, const IPV6 rhs)
{
if (lhs.ip_32.b == 0x0000FFFF) {
if (lhs.ip_32.b == 0x0000FFFF) { // ipv4
IPV6 new_ip {0,0};
new_ip.ip_32.a = lhs.ip_32.a - rhs.ip_32.a;
new_ip.ip_32.b = 0x0000FFFF;
return new_ip;
}
// ipv6
uint64_t tmp;
IPV6 new_ip {0,0};
new_ip.ip_32.a = tmp = ((int64_t)lhs.ip_32.a) - rhs.ip_32.a;
@ -136,6 +143,7 @@ inline const IPV6 operator|(const IPV6 lhs, const IPV6 rhs)
return new_ip;
}
//用于表示 IP 地址范围
typedef struct Range_s
{
Range_s(const IPV6 _from = IPV6(), const IPV6 _to = IPV6()):from(_from), to(_to){}
@ -146,46 +154,48 @@ typedef struct Range_s
class IPRange
{
public:
IPRange();
IPRange(); //构造函数
~IPRange();
static bool is_range_valid(const std::string range);
bool add_ranges(const std::unordered_set<std::string> ranges);
bool add_range(Range *new_range);
bool add_range(const char *range, size_t range_len);
bool remove_ranges(const std::unordered_set<std::string> ranges);
bool remove_range(const char *range, size_t range_len);
std::unordered_set<std::string> get_ranges_set();
static bool is_range_valid(const std::string range); //检查给定的 IP 范围字符串是否有效
bool add_ranges(const std::unordered_set<std::string> ranges); //将一组 IP 地址范围添加到当前范围集合中
bool add_range(Range *new_range); //将一个 IP 地址范围添加到当前范围集合中
bool add_range(const char *range, size_t range_len); //将一个以字符串形式表示的 IP 地址范围添加到当前范围集合中
bool remove_ranges(const std::unordered_set<std::string> ranges); //从当前范围集合中移除一组 IP 地址范围
bool remove_range(const char *range, size_t range_len); //从当前范围集合中移除一个以字符串形式表示的 IP 地址范围
std::unordered_set<std::string> get_ranges_set(); //获取当前范围集合的字符串表示形式
bool is_in_range(const char *ip_str);
bool is_in_range(const IPV6 *ip);
bool is_in_range(const uint32_t ipv4);
bool is_intersect(const IPRange *arg);
bool empty() const;
bool is_in_range(const char *ip_str); //检查给定的 IP 地址字符串是否在当前范围集合内
bool is_in_range(const IPV6 *ip); //检查给定的 IPv6 地址是否在当前范围集合内
bool is_in_range(const uint32_t ipv4); //检查给定的 IPv4 地址是否在当前范围集合内
bool is_intersect(const IPRange *arg); //检查当前范围集合与另一个 IP 范围集合是否有交集
bool empty() const; //检查当前范围集合是否为空
const std::string& get_err_str() { return m_err_str; }
std::string ip_to_str(const IPV6 *ip) const;
bool str_to_ip(const char* ip_str, IPV6 *ip);
const std::string& get_err_str() { return m_err_str; } //获取最近一次操作的错误信息
std::string ip_to_str(const IPV6 *ip) const; //将给定的 IPv6 地址转换为字符串表示形式
bool str_to_ip(const char* ip_str, IPV6 *ip); //将给定的 IP 地址字符串转换为 IPv6 地址
private:
typedef std::vector<Range> Ranges_t;
Ranges_t m_ranges;
std::string m_err_str;
Ranges_t m_ranges; //存储 IP 范围的向量
std::string m_err_str; //存储最近一次操作的错误信息
//各种形式的 IP 地址字符串转换为 IP 结构体
bool parse_range(const char* range, size_t range_len, Range *new_range);
bool parse_slash(const char* range, size_t range_len, const char *ptr, Range *new_range);
bool parse_hyphen(const char* range, size_t range_len, const char *ptr, Range *new_range);
bool parse_mask(const char* range, size_t range_len, const char *ptr, Range *new_range);
bool parse_single(const char* range, size_t range_len, Range *new_range);
//二分查找算法在范围集合中查找给定的 IPv6 地址
bool binary_search(const IPV6 ip) const;
bool mask_range(Range *range, unsigned short cidr);
void handle_remove_intersection(Ranges_t *new_ranges, const Range *remove_range, Range *exist_range);
bool handle_add_intersection(Range *new_range, const Range *exist_range);
void copy_without_spaces(char buf[], size_t buf_len, const char *original, size_t original_len) const;
void net_ipv6_to_host_order(IPV6 *ip, const struct sockaddr_in6 *sa) const;
void net_ipv4_to_host_order(IPV6 *ip, const struct in_addr *addr) const;
bool mask_range(Range *range, unsigned short cidr); //根据给定的 CIDR 前缀长度对 IP 范围进行掩码处理,并更新输入的起始和结束 IP 地址值
void handle_remove_intersection(Ranges_t *new_ranges, const Range *remove_range, Range *exist_range); //处理移除范围时的交叉情况
bool handle_add_intersection(Range *new_range, const Range *exist_range); //处理添加范围时的交叉情况
void copy_without_spaces(char buf[], size_t buf_len, const char *original, size_t original_len) const; //将原始字符串复制到缓冲区中,同时去除空格字符
void net_ipv6_to_host_order(IPV6 *ip, const struct sockaddr_in6 *sa) const; //将网络字节序(大端序)的 IPv6 地址转换为主机字节序
void net_ipv4_to_host_order(IPV6 *ip, const struct in_addr *addr) const; //将网络字节序(大端序)的 IPv4 地址转换为主机字节序
};
#endif // IPRANGE_AUDIT_H_