From d7f0b9336845242fb13d01f3b0b2ac79ec4b6a37 Mon Sep 17 00:00:00 2001 From: shuimuzhihua Date: Sun, 20 Aug 2023 20:16:28 +0800 Subject: [PATCH 01/15] Update mc_tcp.h --- .../cbb/communication/libcomm_core/mc_tcp.h | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_core/mc_tcp.h b/src/gausskernel/cbb/communication/libcomm_core/mc_tcp.h index 83a71ffff..6147b1645 100644 --- a/src/gausskernel/cbb/communication/libcomm_core/mc_tcp.h +++ b/src/gausskernel/cbb/communication/libcomm_core/mc_tcp.h @@ -26,47 +26,49 @@ #include #include #include -#include -#include -#include -#include -#include +#include // 有关系统数据类型定义的头文件 +#include // 有关进程控制定义的头文件 +#include // 套接字编程的头文件 +#include // 网络操作的头文件 + +#include // 有关错误码定义的头文件 #include #include #include #include -#include + +#include // 提供网络地址和端口相关定义的头文件 #include #include "../libcomm_common.h" -#define TCP_LISTENQ 1024 +#define TCP_LISTENQ 1024 // 侦听队列的容量 -extern void mc_tcp_set_keepalive_param(int idle, int intvl, int count); +extern void mc_tcp_set_keepalive_param(int idle, int intvl, int count); // 设置TCP连接的保活参数 -extern void mc_tcp_set_timeout_param(int conn_timeout, int send_timeout); +extern void mc_tcp_set_timeout_param(int conn_timeout, int send_timeout); // 设置TCP连接超时参数 -extern int mc_tcp_get_connect_timeout(); +extern int mc_tcp_get_connect_timeout(); // 获取超时参数 -extern int mc_tcp_listen(const char* host, int port, socklen_t* addrlenp); +extern int mc_tcp_listen(const char* host, int port, socklen_t* addrlenp); // 在指定的主机和端口上进行TCP监听 -extern int mc_tcp_accept(int fd, struct sockaddr* sa, socklen_t* salenptr); +extern int mc_tcp_accept(int fd, struct sockaddr* sa, socklen_t* salenptr); // 服务端接受客户端的连接 -extern int mc_tcp_connect(const char* host, int port); +extern int mc_tcp_connect(const char* host, int port); // 创建一个TCP连接到指定的主机和端口 -extern int mc_tcp_get_peer_name(int fd, char* host, int* port); +extern int mc_tcp_get_peer_name(int fd, char* host, int* port); // 获取目标IP地址和端口号 -extern int mc_tcp_write_block(int fd, const void* data, int size); +extern int mc_tcp_write_block(int fd, const void* data, int size); // 以阻塞方式向指定的套接字写入数据 -extern int mc_tcp_write_noblock(int fd, const void* data, int size); +extern int mc_tcp_write_noblock(int fd, const void* data, int size); // 以非阻塞方式向指定的套接字写入数据 -extern int mc_tcp_read_block(int fd, void* data, int size, int flags); +extern int mc_tcp_read_block(int fd, void* data, int size, int flags); // 以阻塞方式从给定的套接字中读取数据 -extern int mc_tcp_read_nonblock(int fd, void* data, int size, int flags); +extern int mc_tcp_read_nonblock(int fd, void* data, int size, int flags); // 以非阻塞方式从给定的套接字中读取数据 -extern int mc_tcp_check_socket(int sock); +extern int mc_tcp_check_socket(int sock); // 检查套接字的状态 -extern void mc_tcp_close(int fd); +extern void mc_tcp_close(int fd); // 关闭一个套接字 -extern int mc_tcp_addr_init(const char* host, int port, struct sockaddr_storage* ss, int* in_len); +extern int mc_tcp_addr_init(const char* host, int port, struct sockaddr_storage* ss, int* in_len); // 设置主机和端口信息 #endif //_CORE_MC_TCP_H_ -- 2.34.1 From 27e9a8358c6d6d3b2666eb9329624488c68c7c14 Mon Sep 17 00:00:00 2001 From: shuimuzhihua Date: Sun, 20 Aug 2023 20:19:19 +0800 Subject: [PATCH 02/15] Update mc_tcp.cpp --- .../cbb/communication/libcomm_core/mc_tcp.cpp | 655 +++++++++++------- 1 file changed, 420 insertions(+), 235 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_core/mc_tcp.cpp b/src/gausskernel/cbb/communication/libcomm_core/mc_tcp.cpp index f4d8e3a65..b50df5007 100644 --- a/src/gausskernel/cbb/communication/libcomm_core/mc_tcp.cpp +++ b/src/gausskernel/cbb/communication/libcomm_core/mc_tcp.cpp @@ -24,60 +24,108 @@ #include "libcomm_utils/libcomm_util.h" #include "libcomm_utils/libcomm_err.h" -#ifdef USE_SSL -#include "knl/knl_session.h" -#include "libpq/libpq-be.h" -#include "libcomm/libcomm.h" -#include "../libcomm_common.h" +#ifdef USE_SSL // SSL(Secure Socket Layer)安全套接层 + #include "knl/knl_session.h" + #include "libpq/libpq-be.h" // libpq: postgreSQL + #include "libcomm/libcomm.h" + #include "../libcomm_common.h" #endif +/* +* 功能：设置套接字选项 +* fd：套接字描述符 +* level：选项层次 +* opname：选项名称 +* optval：指向要设置的选项值的指针，一般是一个特定类型的变量的指针 +* optlen：选项值的大小 +* 注：该函数封装了基本函数setsockopt +*/ void mc_tcp_setsockopt(int fd, int level, int optname, const void* optval, socklen_t optlen) { - if (setsockopt(fd, level, optname, optval, optlen) < 0) { - errno_assert(errno); - } + // 如果设置选项失败（返回值小于0），则会触发错误处理 + if (setsockopt(fd, level, optname, optval, optlen) < 0) { + errno_assert(errno); // 调用errno_assert函数来处理错误 + } } +/* +* 功能：设置TCP连接的保活参数 +* idle：空闲时间 +* intvl：保活探测间隔 +* count：保活探测次数 +*/ void mc_tcp_set_keepalive_param(int idle, int intvl, int count) { + // 设置TCP连接保活参数: 参数值大于0时有效，否则设置为0 g_instance.comm_cxt.mctcp_cxt.mc_tcp_keepalive_idle = (idle > 0) ? idle : 0; g_instance.comm_cxt.mctcp_cxt.mc_tcp_keepalive_interval = (intvl > 0) ? intvl : 0; g_instance.comm_cxt.mctcp_cxt.mc_tcp_keepalive_count = (count > 0) ? count : 0; } +/* +* 功能：设置TCP连接超时参数 +* conn_timeout：连接极限时长 +* send_timeout：发送极限时长 +*/ void mc_tcp_set_timeout_param(int conn_timeout, int send_timeout) { + // 设置TCP连接超时参数，参数值大于0时有效，否则设置为0 g_instance.comm_cxt.mctcp_cxt.mc_tcp_connect_timeout = (conn_timeout > 0) ? conn_timeout : 0; g_instance.comm_cxt.mctcp_cxt.mc_tcp_send_timeout = (send_timeout > 0) ? send_timeout : 0; } int mc_tcp_get_connect_timeout() { - return g_instance.comm_cxt.mctcp_cxt.mc_tcp_connect_timeout; + return g_instance.comm_cxt.mctcp_cxt.mc_tcp_connect_timeout; } +/* +* 功能：设置TCP连接的发送和接收超时时间 +* fd：套接字描述符 +* timeo：超时秒数 +*/ void mc_tcp_set_timeout(int fd, int timeo) { if (timeo == 0) { - return; + return; // 超时秒数为0直接返回 } - struct timeval timeout = {timeo, 0}; + // 构造一个timeval结构体，表示超时时间 + struct timeval timeout = {timeo, 0}; // 用seconds表示超时秒数，microseconds用不上所以为0 + + // 计算timeval结构体的大小 socklen_t len = sizeof(timeout); + + // 设置套接字选项SO_SNDTIMEO（发送超时） mc_tcp_setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, (const char*)&timeout, len); + + // 设置套接字选项SO_RCVTIMEO（接收超时） mc_tcp_setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&timeout, len); } +/* +* 功能：保持socket存活 +* fd：套接字描述符 +* 注：此函数用于在指定套接字上启用并配置TCP连接的保活机制，从而确保连接在空闲时也能保持活跃状态，提高连接的可靠性 +*/ void mc_tcp_set_keepalive(int fd) { + // 获取TCP保活参数 int on = 1; - int idle = g_instance.comm_cxt.mctcp_cxt.mc_tcp_keepalive_idle; - int interval = g_instance.comm_cxt.mctcp_cxt.mc_tcp_keepalive_interval; - int count = g_instance.comm_cxt.mctcp_cxt.mc_tcp_keepalive_count; + int idle = g_instance.comm_cxt.mctcp_cxt.mc_tcp_keepalive_idle; // 空闲时间 + int interval = g_instance.comm_cxt.mctcp_cxt.mc_tcp_keepalive_interval; // 保活探测间隔 + int count = g_instance.comm_cxt.mctcp_cxt.mc_tcp_keepalive_count; // 保活探测次数 + // 设置SO_KEEPALIVE套接字选项，使保活机制生效 mc_tcp_setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, (char*)&on, sizeof(on)); + + // 设置TCP_KEEPIDLE选项，表示连接空闲多长时间后开始发送保活探测 mc_tcp_setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, (char*)&idle, sizeof(idle)); + + // 设置TCP_KEEPINTVL选项，表示保活探测之间的间隔时间 mc_tcp_setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, (char*)&interval, sizeof(interval)); + + // 设置TCP_KEEPCNT选项，表示保活探测的次数 mc_tcp_setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, (char*)&count, sizeof(count)); } @@ -86,60 +134,88 @@ function name: mc_tcp_get_peer_name description: This function is used to obtain the host IP and port number of the host bound to the specific socket. arguments: The first argument is a descriptor to a specified socket. The second argument is used to store the host IP address bound to the socket determined by the first parameter, in dotted decimal. - The third parameter is used to store the port number bound to a specific socket, in the order of host bytes. + The third parameter is used to store the port number bound to a specific socket, in the order of host bytes. return value: Return 0 if the function runs successfully. When the call to the getpeername() function fails： - 1、Return EBADF if the socket argument is not a valid file descriptor. - 2、Return EINVAL if the socket has been shut down. - 3、Return ENOTCONN if the socket is not connected or otherwise has not had the peer pre-specified. - 4、Return ENOTSOCK if the socket argument does not refer to a socket. - 5、Return EOPNOTSUPP if the operation is not supported for the socket protocol. - 6、Return ENOBUFS if insufficient resources were available in the system to complete the call. - Return -2 when the host IP address belongs to IPv4 type, it fails to convert it to dotted decimal. - Return -3 when the host IP address belongs to IPv6 type, it fails to convert it to dotted decimal. - Return -4 when the error type is not any of the above. + 1、Return EBADF if the socket argument is not a valid file descriptor. + 2、Return EINVAL if the socket has been shut down. + 3、Return ENOTCONN if the socket is not connected or otherwise has not had the peer pre-specified. + 4、Return ENOTSOCK if the socket argument does not refer to a socket. + 5、Return EOPNOTSUPP if the operation is not supported for the socket protocol. + 6、Return ENOBUFS if insufficient resources were available in the system to complete the call. + Return -2 when the host IP address belongs to IPv4 type, it fails to convert it to dotted decimal. + Return -3 when the host IP address belongs to IPv6 type, it fails to convert it to dotted decimal. + Return -4 when the error type is not any of the above. note: Allocate a certain amount of memory space for the host and port pointers respectively in advance. date: 2022/8/9 contact tel: 18720816902 */ +/* +* 功能：获取目标IP地址和端口号 +* fd：套接字描述符 +* host：存放的IP地址 +* port：存放的端口号 +* 返回值： +* EBADF：套接字参数不是有效的文件描述符 +* EINVAL：套接字已关闭 +* ENOTCONN：套接字未连接或尚未预先指定对等方 +* ENOTSOCK：套接字参数不引用套接字 +* EOPNOTSUPP：套接字协议不支持该操作 +* ENOBUFS：系统中没有足够的资源来完成调用 +* -2：当主机IP地址属于IPv4类型时，无法将其转换为点分十进制 +* -3：当主机IP地址属于IPv6类型时，无法将其转换为点分十进制 +* -4：当错误类型不是以上任何一种时 +* 注：该函数封装了基本函数getpeername +*/ int mc_tcp_get_peer_name(int fd, char* host, int* port) { - struct sockaddr peeraddr = {0}; - socklen_t len = sizeof(struct sockaddr); + struct sockaddr peeraddr = {0}; // 定义用于存放对端地址信息的结构体 + socklen_t len = sizeof(struct sockaddr); // 定义用于接收地址信息的结构体大小（peeraddr的大小） + + // 使用getpeername函数获取与指定套接字关联的对端地址信息 int ret = getpeername(fd, (struct sockaddr*)&peeraddr, &len); if (ret < 0) { - return ret; + return ret; // 如果出现错误，直接返回错误码 } - - if (AF_INET == peeraddr.sa_family) { - struct sockaddr_in* paddr = (struct sockaddr_in*)&peeraddr; - *port = ntohs(paddr->sin_port); + + if (AF_INET == peeraddr.sa_family) { // 如果地址族是IPv4 + struct sockaddr_in* paddr = (struct sockaddr_in*)&peeraddr; // 转换为sockaddr_in结构体 + *port = ntohs(paddr->sin_port); // 将网络字节顺序的端口号转换为主机字节顺序 if (inet_ntop(AF_INET, &(paddr->sin_addr), host, HOST_ADDRSTRLEN) == NULL) { - ret = -2; + ret = -2; // 无法将IP地址转换为点分十进制格式 } - } else if (AF_INET6 == peeraddr.sa_family) { - struct sockaddr_in6* paddr = (struct sockaddr_in6*)&peeraddr; - *port = ntohs(paddr->sin6_port); + } else if (AF_INET6 == peeraddr.sa_family) { // 如果地址族是IPv6 + struct sockaddr_in6* paddr = (struct sockaddr_in6*)&peeraddr; // 转换为sockaddr_in6结构体 + *port = ntohs(paddr->sin6_port); // 将网络字节顺序的端口号转换为主机字节顺序 if (inet_ntop(AF_INET6, &(paddr->sin6_addr), host, HOST_ADDRSTRLEN) == NULL) { - ret = -3; + ret = -3; // 无法将IP地址转换为点分十进制格式 } } else { - ret = -4; + ret = -4; // 不支持的地址族 } - return ret; + return ret; // 返回执行结果 } -// set socket to NON-BLOCKING to epoll on it -// +/* +* 功能：设置TCP模式为非阻塞模式 +* fd：套接字描述符 +* 返回值：同set_socketopt，意义相同 +*/ int mc_tcp_set_nonblock(int fd) { - return set_socketopt(fd, 0, O_NONBLOCK); + return set_socketopt(fd, 0, O_NONBLOCK); // 调用set_socketopt函数设置为非阻塞模式 } +/* +* 功能：设置TCP模式为`close-on-exec`模式 +* fd：套接字描述符 +* 返回值：同set_socketopt，意义相同 +* 描述：执行exec系统调用时，该套接字会自动关闭，以防止在子进程中继承这个套接字 +*/ int mc_tcp_set_cloexec(int fd) { - return set_socketopt(fd, 1, FD_CLOEXEC); + return set_socketopt(fd, 1, FD_CLOEXEC); // 调用set_socketopt函数设置为`close-on-exec`模式 } /* @@ -147,36 +223,48 @@ function name: mc_tcp_accept description: This function will block the process by default until a client connection is established and returns a new available socket. arguments: The first argument is a socket descriptor to a specific socket. The second argument is a result parameter, which is used to accept a return value that specifies the address of the client. - The third argument is also a result argument, which is used to accept the size of the sockaddr structure. It indicates the number of bytes occupied by the sockaddr structure. + The third argument is also a result argument, which is used to accept the size of the sockaddr structure. It indicates the number of bytes occupied by the sockaddr structure. return value: Return a value less than 0 if an error occurred when call the function accept4(), else return the new fd of socket. note: none date: 2022/8/9 contact tel: 18720816902 */ +/* +* 功能：服务端接受客户端的连接 +* fd：套接字描述符 +* sa：保存着目的地端口和IP地址的数据结构 +* salenptr：指向addr缓冲区大小的指针，以字节为单位 +* 返回值：成功返回新fd（socket）；失败返回负值 +* 注：该函数封装了基本函数accept，默认情况下会阻塞进程 +*/ int mc_tcp_accept(int fd, struct sockaddr* sa, socklen_t* salenptr) { - int new_fd; + int new_fd; // 记录待返回的新fd again: + // 使用accept4函数接受一个新的连接，设置SOCK_CLOEXEC选项使其在执行exec系统调用时自动关闭 if ((new_fd = accept4(fd, sa, salenptr, SOCK_CLOEXEC)) < 0) { #ifdef EPROTO if (errno == EPROTO || errno == ECONNABORTED) { #else if (errno == ECONNABORTED) { #endif - goto again; + goto again; // 如果连接被中断，则重新尝试接受连接 } else { - errno_assert(errno); + errno_assert(errno); // 否则，输出错误信息并退出 } } - if (new_fd > 0) { - (void)mc_tcp_set_nonblock(new_fd); // set non-block + if (new_fd > 0) { // 如果成功接受连接 + (void)mc_tcp_set_nonblock(new_fd); // 将新连接设置为非阻塞模式 int no_delay = (g_instance.comm_cxt.commutil_cxt.g_no_delay) ? 1 : 0; - mc_tcp_setsockopt(new_fd, IPPROTO_TCP, TCP_NODELAY, &no_delay, sizeof(no_delay)); // set no delay - mc_tcp_set_keepalive(new_fd); + + // 根据no_delay的值设置TCP_NODELAY选项，用于禁用Nagle算法以减少延迟 + mc_tcp_setsockopt(new_fd, IPPROTO_TCP, TCP_NODELAY, &no_delay, sizeof(no_delay)); + + mc_tcp_set_keepalive(new_fd); // 设置保活机制 } - return (new_fd); + return new_fd; // 返回新的套接字描述符 } /* @@ -184,53 +272,71 @@ function name: mc_tcp_bind description: This function binds the specified socket to a specific IP address and port. arguments: The first argument indicates the socket descriptor that has been established. The second argument is a pointer to the sockaddr structure to socket. - The third argument is byte length of sockaddr structure. + The third argument is byte length of sockaddr structure. return value: Return errno, the return value is 0 if succeed, else one of other error types is returned for failure. note: none date: 2022/8/9 contact tel: 18720816902 */ +/* +* 功能：将fd的socket绑定到sa指定的地址和端口 +* fd：套接字描述符 +* sa：保存着目的地端口和IP地址的数据结构 +* salen：sizeof(sa)，sa结构体的大小 +* 返回值：绑定成功返回0；绑定失败返回其他错误类型之一(errno类型) +* 注：该函数封装了基本函数bind +*/ int mc_tcp_bind(int fd, const struct sockaddr* sa, socklen_t salen) { int error = -1; - int retry_time = MAX_BIND_RETRYS; - sockaddr_in* addr = (sockaddr_in*)sa; + int retry_time = MAX_BIND_RETRYS; // 设置最大重试次数（这里值为30） + sockaddr_in* addr = (sockaddr_in*)sa; // 将传入的地址信息转换为sockaddr_in结构体 while (retry_time--) { - error = bind(fd, sa, salen); + error = bind(fd, sa, salen); // 尝试在指定套接字上绑定地址信息 if (error == 0) { - break; + break; // 如果绑定成功，跳出循环 } - uint16 port = ntohs(addr->sin_port); + // 如果绑定失败 + uint16_t port = ntohs(addr->sin_port); // 获取端口号 + // 输出警告日志，说明绑定失败的原因，并提示可能的解决办法 LIBCOMM_ELOG(WARNING, - "(mc tcp listen)\tFailed to bind host:port[%s:%hu], errno[%d]:%s." - "Maybe port %hu is used, run 'netstat -anop|grep %hu' or " - "'lsof -i:%hu'(need root) to see who is using.", - inet_ntoa(addr->sin_addr), - port, - errno, - mc_strerror(errno), - port, - port, - port); + "(mc tcp listen)\tFailed to bind host:port[%s:%hu], errno[%d]:%s." + "Maybe port %hu is used, run 'netstat -anop|grep %hu' or " + "'lsof -i:%hu'(need root) to see who is using.", + inet_ntoa(addr->sin_addr), port, errno, mc_strerror(errno), port, port, port); - (void)sleep(5); + (void)sleep(5); // 休眠5秒后重试 } - return error; + return error; // 返回绑定结果 } +/* + * 功能：将fd的socket连至sa指定的服务端 + * fd：套接字描述符 + * sa：保存着目的地端口和IP地址的数据结构 + * salen：sizeof(sa)，sa结构体的大小 + * 返回值：成功返回0；失败返回-1 + * 注：该函数封装了基本函数connect；connect函数只用于客户端，且三次握手在connect后底层自动发生 + */ static int mc_tcp_do_connect(int fd, const struct sockaddr* sa, socklen_t salen) { - int error = connect(fd, sa, salen); - return error; + int error = connect(fd, sa, salen); // 在给定套接字上发起连接 + return error; // 返回连接结果 } +/* +* 功能：在指定的主机和端口上进行TCP监听 +* fd：套接字描述符 +* backlog：侦听队列容量（定义在mc_tcp.h中，值为1024） +* 注：该函数封装了基本函数listen +*/ static void mc_tcp_do_listen(int fd, int backlog) { if (listen(fd, backlog) < 0) { - errno_assert(errno); + errno_assert(errno); // 如果listen函数出错，输出错误信息并退出 } } @@ -240,63 +346,75 @@ description: This function receives data from the other end of TCP in a blocking process will not end until the data of size byte length is successfully received or a real error occurs in the receiving process arguments: The first argument indicates the specific socket that has been established. The second argument is a pointer to memory area, we use it to store received data. - The third argument is byte length of the memory area pointed to by the data pointer. - The fourth argument specifies additional operations in addition to the read operation. + The third argument is byte length of the memory area pointed to by the data pointer. + The fourth argument specifies additional operations in addition to the read operation. return value: If there is no error, it returns the byte length of the successfully read data. If an error occurs, return - 1. note: When the data is read successfully, the byte length of the data is greater than 0. date: 2022/8/10 contact tel: 18720816902 */ +/* +* 功能：以阻塞方式从给定的套接字中读取数据 +* fd：套接字描述符 +* data：接收数据的缓冲区的指针 +* size：缓冲的最大长度，单位为字节 +* flags：可选的标志参数 +* 返回值：接收成功，返回已接收数据的字节数；发生错误，返回-1 +* 注：该函数封装了基本函数recv；若此函数的返回值小于size值，则说明该数据未被接收完全 +*/ int mc_tcp_read_block(int fd, void* data, int size, int flags) { #ifdef LIBCOMM_FAULT_INJECTION_ENABLE + // 检查是否启用了通信错误注入 if (is_comm_fault_injection(LIBCOMM_FI_MC_TCP_READ_BLOCK_FAILED)) { LIBCOMM_ELOG(WARNING, "(mc tcp read block)\t[FAULT INJECTION]Failed to read block for %d.", fd); return -1; } #endif - uint64 time_enter, time_now; + + uint64_t time_enter, time_now; // 用于判断读取超时的时间游标 time_enter = mc_timers_ms(); ssize_t nbytes = 0; int rc = 0; #ifdef USE_SSL - SSL *ssl = NULL; + SSL* ssl = NULL; + // 在使用SSL的情况下查找SSL对象 LIBCOMM_FIND_SSL(ssl, fd, "(mc tcp read block)\tNot find ssl for sock "); #endif - // In our application, if we do not get an integrated message, we must continue receiving. - // + // 如果没有得到一个完整的消息，必须继续接收 while (nbytes != size) { #ifdef USE_SSL if (g_instance.attr.attr_network.comm_enable_SSL) { - rc = LibCommClientSSLRead(ssl, (char*)data + nbytes, size - nbytes); + // 如果启用了SSL，使用LibCommClientSSLRead进行读取 + rc = LibCommClientSSLRead(ssl, (char *)data + nbytes, size - nbytes); } else #endif { - rc = recv(fd, (char*)data + nbytes, size - nbytes, flags); + // 否则使用recv函数进行读取 + rc = recv(fd, (char *)data + nbytes, size - nbytes, flags); } if (rc > 0) { - if (((char*)data)[0] == '\0') { + if (((char *)data)[0] == '\0') { + // 检查是否收到非法消息 LIBCOMM_ELOG(ERROR, "(mc tcp read block)\tIllegal message from sock %d.", fd); - return -1; + return -1; // 返回错误码 } - nbytes = nbytes + rc; - } else if (rc == 0) { // Orderly shutdown by the other peer. + } else if (rc == 0) { // 对等方有序关闭连接 nbytes = 0; - break; + break; // 置nbytes为0后退出循环 } else if (rc < 0) { if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) { + // 处理非阻塞情况下的读取超时和错误 time_now = mc_timers_ms(); - // If can not receive data after 600 seconds later, - // we think this connection has problem if (((time_now - time_enter) > - ((uint64)(unsigned)g_instance.comm_cxt.mctcp_cxt.mc_tcp_send_timeout * SEC_TO_MICRO_SEC)) && + ((uint64_t)(unsigned)g_instance.comm_cxt.mctcp_cxt.mc_tcp_send_timeout * SEC_TO_MICRO_SEC)) && (time_now > time_enter)) { errno = ECOMMTCPSENDTIMEOUT; return -1; @@ -309,13 +427,13 @@ int mc_tcp_read_block(int fd, void* data, int size, int flags) } } } - /* Orderly shutdown by the other peer or Signalise peer failure. */ + // 对等方有序关闭连接或者标志对等方故障 if ((nbytes == 0) || (nbytes == -1 && (errno == ECONNRESET || errno == ECONNREFUSED || errno == ETIMEDOUT || - errno == EHOSTUNREACH))) { - return -1; + errno == EHOSTUNREACH))) { + return -1; // 返回错误码 } - return (size_t)nbytes; + return (size_t)nbytes; // 返回已接收数据的字节数 } /* @@ -324,17 +442,27 @@ description: This function receives data from the other end of TCP in a non bloc the data receiving process is only performed once. arguments: The first argument indicates the specific socket that has been established. The second argument is a pointer to memory area, we use it to store received data. - The third argument is byte length of the memory area pointed to by the data pointer. - The fourth argument specifies additional operations in addition to the read operation. + The third argument is byte length of the memory area pointed to by the data pointer. + The fourth argument specifies additional operations in addition to the read operation. return value: If the error type is one of the errors represented by EAGAIN, EWOULDBLOCK and EINTR, it returns 0; other error types return - 1; if there is no error, it returns the byte length of the successfully read data. note: When the data is read successfully, the byte length of the data is greater than 0. date: 2022/8/10 contact tel: 18720816902 */ +/* +* 功能：以非阻塞方式从给定的套接字中读取数据 +* fd：套接字描述符 +* data：接收数据的缓冲区的指针 +* size：缓冲的最大长度，单位为字节 +* flags：可选的标志参数 +* 返回值：接收成功，返回已接收数据的字节数；发生错误，返回-1 +* 注：该函数封装了基本函数recv；若此函数的返回值小于size值，则说明该数据未被接收完全 +*/ int mc_tcp_read_nonblock(int fd, void* data, int size, int flags) { #ifdef LIBCOMM_FAULT_INJECTION_ENABLE + // 检查是否启用了通信错误注入 if (is_comm_fault_injection(LIBCOMM_FI_MC_TCP_READ_NONBLOCK_FAILED)) { LIBCOMM_ELOG(WARNING, "(mc tcp read nonblock)\t[FAULT INJECTION]Failed to read nonblock for %d.", fd); return -1; @@ -345,36 +473,31 @@ int mc_tcp_read_nonblock(int fd, void* data, int size, int flags) #ifdef USE_SSL if (g_instance.attr.attr_network.comm_enable_SSL) { + // 在启用SSL的情况下，查找SSL对象 SSL *ssl = NULL; LIBCOMM_FIND_SSL(ssl, fd, "(mc tcp read nonblock)\tNot find ssl for sock "); nbytes = LibCommClientSSLRead(ssl, data, size); } else -#endif /* USE_SSL */ +#endif { + // 否则使用recv函数进行非阻塞读取 nbytes = recv(fd, data, size, flags); } - // Several errors are OK. When speculative read is being done we may not - // be able to read a single byte to the socket. Also, SIGSTOP issued - // by a debugging tool can result in EINTR error. - // + // 处理非阻塞读取返回值 if (nbytes == -1 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) { - return 0; + return 0; // 暂时没有数据可读 } - // Signalise peer failure. - // if (nbytes == -1) { - return -1; + return -1; // 返回读取错误 } - // Orderly shutdown by the other peer. - // if (nbytes == 0) { - return -1; + return -1; // 返回对等方关闭连接 } - return (size_t)nbytes; + return (size_t)nbytes; // 返回实际读取的字节数 } /* @@ -383,24 +506,31 @@ description: This function binds the specified socket to a specific IP address a arguments: The only argument indicates the specific socket that has been established. return value: Return -1 if when the recv function wait for the protocol to receive data, the other end of TCP closes the connection or a real error occurred while - reading data. In other cases, 0 is returned. + reading data. In other cases, 0 is returned. note: none date: 2022/8/10 contact tel: 18720816902 */ +/* +* 功能：检查套接字的状态 +* sock：套接字描述符 +* 返回值：成功返回0；失败返回-1 +* 注：此函数通过尝试从套接字中读取数据（使用recv函数），并根据读取的结果和错误码判断套接字的状态 +*/ int mc_tcp_check_socket(int sock) { char temp_buf[IOV_DATA_SIZE] = {0}; bool is_sock_err = false; int error = -1; - if (sock < 0) { + if (sock < 0) { // 套接字无效 return -1; } #ifdef USE_SSL - SSL *ssl = NULL; + SSL* ssl = NULL; + // 在使用SSL的情况下查找SSL对象 LIBCOMM_FIND_SSL(ssl, sock, "(mc tcp check socket)\tNot find ssl for sock "); #endif @@ -409,45 +539,47 @@ int mc_tcp_check_socket(int sock) while (false == is_sock_err) { #ifdef USE_SSL if (g_instance.attr.attr_network.comm_enable_SSL) { + // 在启用SSL的情况下使用LibCommClientSSLRead进行读取 error = LibCommClientSSLRead(ssl, temp_buf, IOV_DATA_SIZE); } else #endif { + // 否则使用recv函数进行读取 error = recv(sock, temp_buf, IOV_DATA_SIZE, 0); } if (error < 0) { - // no data to recieve, and no socket error + // 没有数据可接收，且没有套接字错误 if (errno == EAGAIN || errno == EWOULDBLOCK) { break; } - // need retry + // 需要重试 else if (errno == EINTR) { continue; } - // other errno means really error + // 其他errno表示真正的错误 else { is_sock_err = true; break; } } - // remote has closed + // 对端已关闭连接 if (error == 0) { is_sock_err = true; break; } if (error > 0) { - // something fault data + // 接收到错误的数据 continue; } } if (is_sock_err) { - return -1; + return -1; // 返回错误 } - return 0; + return 0; // 返回成功 } /* @@ -456,69 +588,77 @@ description: This function writes data to the specified socket in blocking mode, will not end until all the data are successfully sent or a real error occurs during the sending process arguments: The first argument indicates the specific socket that has been established. The second argument is a pointer to memory area, we use it to store data to be sent. - The third argument is byte length of data to be sent. + The third argument is byte length of data to be sent. return value: If there is no error, it returns the byte length of the successfully sent data. If an error occurs, return - 1. note: none date: 2022/8/10 contact tel: 18720816902 */ +/* +* 功能：以阻塞方式向指定的套接字写入数据 +* fd：套接字描述符 +* data：待发送数据的缓冲区的指针 +* size：待发送数据长度，单位为字节 +* 返回值：写入成功，返回已写入数据的字节数；发生错误，返回-1 +* 注：该函数封装了基本函数send；若此函数的返回结果小于size值，则说明该数据未被发送完全 +*/ int mc_tcp_write_block(int fd, const void* data, int size) { #ifdef LIBCOMM_FAULT_INJECTION_ENABLE + // 检查是否启用了通信错误注入 if (is_comm_fault_injection(LIBCOMM_FI_MC_TCP_WRITE_FAILED)) { LIBCOMM_ELOG(WARNING, "(mc tcp write)\t[FAULT INJECTION]Failed to write for %d.", fd); - shutdown(fd, SHUT_RDWR); - return -1; + shutdown(fd, SHUT_RDWR); // 关闭套接字连接 + return -1; // 返回错误 } #endif ssize_t nbytes; ssize_t nSend = 0; - const int flags = 0; + const int flags = 0; // 无特殊标志 uint64 time_enter, time_now; - time_enter = mc_timers_ms(); + time_enter = mc_timers_ms(); // 记录进入函数的时间 #ifdef USE_SSL - SSL *ssl = NULL; + SSL* ssl = NULL; + // 在使用SSL的情况下查找SSL对象 LIBCOMM_FIND_SSL(ssl, fd, "(mc tcp write block)\tNot find ssl for sock "); #endif - // Several errors are OK. When speculative write is being done we may not - // be able to write a single byte to the socket. Also, SIGSTOP issued - // by a debugging tool can result in EINTR error. - // + // 数据还未发完，必须继续发送 while (nSend != size) { #ifdef USE_SSL if (g_instance.attr.attr_network.comm_enable_SSL) { + // 在启用SSL的情况下使用LibCommClientSSLWrite进行写入 nbytes = LibCommClientSSLWrite(ssl, (void*)((char*)data + nSend), size - nSend); } else #endif { + // 否则使用send函数进行写入 nbytes = send(fd, (const void*)((char*)data + nSend), size - nSend, flags); } if (nbytes <= 0) { if (nbytes == -1 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR || errno == ENOBUFS)) { time_now = mc_timers_ms(); - // If can not receive data after 600 seconds later, - // we think this connection has problem + // 如果在超过指定时间后仍无法发送数据，可能存在连接问题 if (((time_now - time_enter) > - ((uint64)(unsigned)g_instance.comm_cxt.mctcp_cxt.mc_tcp_send_timeout * SEC_TO_MICRO_SEC)) && + ((uint64)(unsigned)g_instance.comm_cxt.mctcp_cxt.mc_tcp_send_timeout * SEC_TO_MICRO_SEC)) && (time_now > time_enter)) { - errno = ECOMMTCPSENDTIMEOUT; - return -1; + errno = ECOMMTCPSENDTIMEOUT; // 设置错误码 + return -1; // 返回错误 } continue; } - return -1; + return -1; // 返回错误 } else { - nSend += nbytes; + nSend += nbytes; // 更新已发送字节数 } } - return (size_t)nSend; + return (size_t)nSend; // 返回实际发送的字节数 } /* @@ -527,106 +667,133 @@ description: This function writes data to the specified socket in non blocking m the data transmission process is only performed once. arguments: The first argument indicates the specific socket that has been established. The second argument is a pointer to memory area, we use it to store data to be sent. - The third argument is byte length of data to be sent. + The third argument is byte length of data to be sent. return value: If the sending fails but the failure reason is one of the error types represented by EAGAIN、 EWOULDBLOCK、EINTR 和ENOBUFS, then 0 is returned; if the error type is other, then - 1 - is returned; If the transmission is successful, the byte length of the successfully transmitted - data is returned + is returned; If the transmission is successful, the byte length of the successfully transmitted + data is returned note: none date: 2022/8/10 contact tel: 18720816902 */ +/* +* 功能：以非阻塞方式从给定的套接字中读取数据，数据传输过程只执行一次 +* fd：套接字描述符 +* data：待发送数据的缓冲区的指针 +* size：待发送数据长度，单位为字节 +* 返回值：写入成功，返回已写入数据的字节数；发送失败，且失败原因为EAGAIN、EWOULDBLOCK、EINTR或ENOBUFS之一，返回0；否则，返回-1 +* 注：该函数封装了基本函数send；若此函数的返回结果小于size值，则说明该数据未被发送完全 +*/ int mc_tcp_write_noblock(int fd, const void* data, int size) { #ifdef LIBCOMM_FAULT_INJECTION_ENABLE + // 检查是否启用了通信错误注入 if (is_comm_fault_injection(LIBCOMM_FI_MC_TCP_WRITE_NONBLOCK_FAILED)) { LIBCOMM_ELOG(WARNING, "(mc tcp write noblock)\t[FAULT INJECTION]Failed to write nonblock for %d.", fd); - shutdown(fd, SHUT_RDWR); - return -1; + shutdown(fd, SHUT_RDWR); // 关闭套接字连接 + return -1; // 返回错误 } #endif ssize_t nbytes; - const int flags = 0; + const int flags = 0; // 无特殊标志 #ifdef USE_SSL - SSL *ssl = NULL; + SSL* ssl = NULL; + // 在使用SSL的情况下查找SSL对象 LIBCOMM_FIND_SSL(ssl, fd, "(mc tcp write nonblock)\tNot find ssl for sock "); #endif - // Several errors are OK. When speculative write is being done we may not - // be able to write a single byte to the socket. Also, SIGSTOP issued - // by a debugging tool can result in EINTR error. - // #ifdef USE_SSL if (g_instance.attr.attr_network.comm_enable_SSL) { + // 在启用SSL的情况下使用LibCommClientSSLWrite进行写入 nbytes = LibCommClientSSLWrite(ssl, data, size); } else #endif { + // 否则使用send函数进行写入 nbytes = send(fd, data, size, flags); } + // 发送失败，且失败原因为EAGAIN、EWOULDBLOCK、EINTR或ENOBUFS之一，返回0 if (nbytes == -1 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR || errno == ENOBUFS)) { - return 0; + return 0; // 返回0表示暂无数据发送 } + // 发送失败，或者nbytes小于等于0，返回-1 if (nbytes <= 0) { - return -1; + return -1; // 返回错误 } - return (size_t)nbytes; + return (size_t)nbytes; // 返回实际发送的字节数 } -// initialize Socket -// +/* +* 功能：创建一个套接字 +* family：表示协议族 +* type：表示套接字类型 +* protocol：表示协议 +* 返回值：成功则返回一个套接字描述符（int类型，值为正）；失败返回-1 +* 注：该函数封装了基本函数socket +*/ int mc_tcp_socket(int family, int type, int protocol) { int error = socket(family, type, protocol); - return (error); + return (error); // 返回套接字的文件描述符 } +/* +* 功能：关闭一个套接字 +* fd：套接字描述符 +* 注：该函数封装了基本函数close；如果关闭过程中出现错误，函数会输出一个错误日志，包括错误号对应的错误信息 +*/ void mc_tcp_close(int fd) { int error = 0; - LIBCOMM_ELOG(WARNING, "(mc tcp close)\tClose socket[%d].", fd); - error = close(fd); + LIBCOMM_ELOG(WARNING, "(mc tcp close)\tClose socket[%d].", fd); // 输出日志，表示正在关闭套接字 + error = close(fd); // 关闭套接字 if (error != 0) { - LIBCOMM_ELOG(WARNING, "(mc tcp close)\tFailed to close socket[%d]:%s.", fd, strerror(errno)); + LIBCOMM_ELOG(WARNING, "(mc tcp close)\tFailed to close socket[%d]:%s.", fd, strerror(errno)); // 输出错误日志 } return; } -/* init a sctp mc_tcp_addr_init with host and port */ +/* +* 功能：设置主机和端口信息 +* host：待连接的主机名或IP地址的字符串表示 +* port：待连接的端口号 +* ss：一个指向sockaddr_storage类型的指针，用于存储初始化后的地址结构 +* in_len：存储初始化后的地址结构的长度 +* 返回值：设置成功返回0；失败返回错误码 +*/ int mc_tcp_addr_init(const char* host, int port, struct sockaddr_storage* ss, int* in_len) { - mc_assert(port >= 0); + mc_assert(port >= 0); // 断言端口号非负 - struct sockaddr_in* t_addr = NULL; + struct sockaddr_in* t_addr = NULL; // 定义一个指向sockaddr_in结构体的指针 int len = 0; int error = 0; - // do not termination process when revieve SIGPIPE - // + // 忽略SIGPIPE信号，避免在套接字写入时出现异常 (void)signal(SIGPIPE, SIG_IGN); if (strcmp(host, "localhost") == 0) { - host = "0.0.0.0"; + host = "0.0.0.0"; // 如果host是"localhost"，将其替换为 "0.0.0.0" } - t_addr = (struct sockaddr_in*)ss; - t_addr->sin_family = AF_INET; - t_addr->sin_port = htons(port); - error = inet_pton(AF_INET, host, &t_addr->sin_addr); - len = sizeof(struct sockaddr_in); + t_addr = (struct sockaddr_in*)ss; // 将ss转换为sockaddr_in类型的指针 + t_addr->sin_family = AF_INET; // 设置地址族为IPv4 + t_addr->sin_port = htons(port); // 设置端口号 + error = inet_pton(AF_INET, host, &t_addr->sin_addr); // 将点分十进制的IP地址转换为二进制格式 + len = sizeof(struct sockaddr_in); // 地址结构长度 #ifdef __FreeBSD__ - t_addr->sin_len = len; + t_addr->sin_len = len; // 仅在FreeBSD平台下设置地址结构长度字段 #endif - *in_len = len; - return (error == 1) ? 0 : error; + *in_len = len; // 将地址结构长度传递出去 + return (error == 1) ? 0 : error; // 返回结果 } /* @@ -640,18 +807,24 @@ note: none date: 2022/8/10 contact tel: 18720816902 */ +/* +* 功能：创建一个非阻塞的TCP套接字并连接到指定的主机和端口 +* host：待连接的主机名或IP地址的字符串表示 +* port：待连接的端口号 +* 返回值：成功返回连接后的套接字描述符；失败返回-1 +*/ int mc_tcp_connect_nonblock(const char* host, int port) { int sockfd, n; struct addrinfo hints = {0}; - struct addrinfo *res = NULL; + struct addrinfo* res = NULL; errno_t ss_rc = memset_s(&hints, sizeof(hints), 0, sizeof(struct addrinfo)); securec_check(ss_rc, "\0", "\0"); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; + hints.ai_family = AF_UNSPEC; // 指定协议族 + hints.ai_socktype = SOCK_STREAM; // 指明socket发送和接收分组的形式 char serv[NI_MAXSERV]; int rc = snprintf_s(serv, NI_MAXSERV, NI_MAXSERV - 1, "%d", port); @@ -659,39 +832,39 @@ int mc_tcp_connect_nonblock(const char* host, int port) do { if (strcmp(host, "localhost") == 0) { - n = getaddrinfo(NULL, serv, &hints, &res); + n = getaddrinfo(NULL, serv, &hints, &res); // 解析主机名或IP地址 } else { - n = getaddrinfo(host, serv, &hints, &res); + n = getaddrinfo(host, serv, &hints, &res); // 解析指定的主机名或IP地址 } - } while (n == EAI_AGAIN); + } while (n == EAI_AGAIN); // 在出现EAI_AGAIN错误时重新尝试 if (n != 0) { - return -1; + return -1; // 解析失败，返回-1 } - sockfd = mc_tcp_socket(res->ai_family, res->ai_socktype, res->ai_protocol); + sockfd = mc_tcp_socket(res->ai_family, res->ai_socktype, res->ai_protocol); // 创建套接字 if (sockfd < 0) { freeaddrinfo(res); - return -1; + return -1; // 创建套接字失败，返回-1 } if (mc_tcp_set_nonblock(sockfd) < 0) { freeaddrinfo(res); - return -1; + return -1; // 设置非阻塞失败，返回-1 } errno = 0; - mc_tcp_do_connect(sockfd, res->ai_addr, res->ai_addrlen); + mc_tcp_do_connect(sockfd, res->ai_addr, res->ai_addrlen); // 发起非阻塞连接 freeaddrinfo(res); - return sockfd; + return sockfd; // 返回连接后的套接字描述符 } /* function name: mc_tcp_connect description: This function first obtains the ports of other hosts with the same domain name stored through the ports of specific hosts, and creates a socket to establish a connection with an appropriate - one of these ports. + one of these ports. arguments: The first parameter specifies a specific host, and the second parameter specifies a specific port of the host. return value: The key is to successfully establish a connection with a port in the linked list. If the connection is successful, the socket file descriptor connected to it will be returned. Otherwise, it will return - 1. @@ -699,28 +872,35 @@ note: We finally get the infomation of the ports of other hosts through a linked date: 2022/8/10 contact tel: 18720816902 */ -int mc_tcp_connect(const char* host, int port) +/* +* 功能：创建一个TCP连接到指定的主机和端口 +* host：待连接的主机名或IP地址的字符串表示 +* port：待连接的端口号 +* 返回值：成功返回连接后的套接字描述符；失败返回-1 +*/ +int mc_tcp_connect(const char *host, int port) { + int sockfd, n, error = 0; + #ifdef LIBCOMM_FAULT_INJECTION_ENABLE if (is_comm_fault_injection(LIBCOMM_FI_MC_TCP_CONNECT_FAILED)) { - LIBCOMM_ELOG( - WARNING, "(mc tcp connect)\t[FAULT INJECTION]Failed to do control tcp listen for %s:%d.", host, port); + LIBCOMM_ELOG(WARNING, "(mc tcp connect)\t[FAULT INJECTION]Failed to do control tcp listen for %s:%d.", host, + port); return -1; } #endif - mc_assert(port > 0); + + mc_assert(port > 0); // 断言端口号大于0 errno = 0; - int sockfd, n, error = 0; - struct addrinfo hints = {0}; struct addrinfo *res = NULL, *ressave = NULL; errno_t ss_rc = memset_s(&hints, sizeof(hints), 0, sizeof(struct addrinfo)); securec_check(ss_rc, "\0", "\0"); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; + hints.ai_family = AF_UNSPEC; // 指定协议族 + hints.ai_socktype = SOCK_STREAM; // 指明socket发送和接收分组的形式 char serv[NI_MAXSERV]; int rc = snprintf_s(serv, NI_MAXSERV, NI_MAXSERV - 1, "%d", port); @@ -728,9 +908,9 @@ int mc_tcp_connect(const char* host, int port) retry: if (strcmp(host, "localhost") == 0) { - n = getaddrinfo(NULL, serv, &hints, &res); + n = getaddrinfo(NULL, serv, &hints, &res); // 解析主机名或IP地址 } else { - n = getaddrinfo(host, serv, &hints, &res); + n = getaddrinfo(host, serv, &hints, &res); // 解析指定的主机名或IP地址 } if (n == EAI_AGAIN) { @@ -738,46 +918,38 @@ retry: } if (n != 0) { - LIBCOMM_ELOG(WARNING, - "(mc tcp connect)\tFailed to get address infomation for host:port[%s:%s] error[%d]:%s.", - host, - serv, - n, - mc_strerror(errno)); - return -1; + LIBCOMM_ELOG(WARNING, "(mc tcp connect)\tFailed to get address information for host:port[%s:%s] error[%d]:%s.", + host, serv, n, mc_strerror(errno)); + return -1; // 解析失败，返回-1 } ressave = res; do { - sockfd = mc_tcp_socket(res->ai_family, res->ai_socktype, res->ai_protocol); + sockfd = mc_tcp_socket(res->ai_family, res->ai_socktype, res->ai_protocol); // 创建套接字 if (sockfd < 0) { - continue; // ignore this one + continue; // 创建套接字失败，忽略此次尝试 } - // set tcp connection socket properties - // - int no_delay = (g_instance.comm_cxt.commutil_cxt.g_no_delay) ? 1 : 0; - mc_tcp_setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &no_delay, sizeof(no_delay)); - mc_tcp_set_keepalive(sockfd); - mc_tcp_set_timeout(sockfd, g_instance.comm_cxt.mctcp_cxt.mc_tcp_connect_timeout); - error = mc_tcp_set_cloexec(sockfd); + // 设置TCP连接套接字属性 + int no_delay = (g_instance.comm_cxt.commutil_cxt.g_no_delay) ? 1 : 0; + mc_tcp_setsockopt(sockfd, IPPROTO_TCP, TCP_NODELAY, &no_delay, sizeof(no_delay)); // 禁用Nagle算法 + mc_tcp_set_keepalive(sockfd); // 保持socket存活 + mc_tcp_set_timeout(sockfd, g_instance.comm_cxt.mctcp_cxt.mc_tcp_connect_timeout); // 设置连接超时时间 + + error = mc_tcp_set_cloexec(sockfd); // 设置FD_CLOEXEC属性 if (error == 0 && (mc_tcp_do_connect(sockfd, res->ai_addr, res->ai_addrlen) == 0)) { - error = mc_tcp_set_nonblock(sockfd); // set non-block + error = mc_tcp_set_nonblock(sockfd); // 设置非阻塞模式 if (error == 0) { - break; // success + break; // 成功建立连接 } } LIBCOMM_ELOG(WARNING, - "(mc tcp connect)\tFailed to build TCP connect to host:port[%s:%s], family[%d],error[%d]:%s.", - host, - serv, - res->ai_family, - errno, - mc_strerror(errno)); + "(mc tcp connect)\tFailed to build TCP connect to host:port[%s:%s], family[%d], error[%d]:%s.", + host, serv, res->ai_family, errno, mc_strerror(errno)); - mc_tcp_close(sockfd); // ignore this one + mc_tcp_close(sockfd); // 关闭连接失败的套接字 sockfd = -1; } while ((res = res->ai_next) != NULL); @@ -786,7 +958,7 @@ retry: freeaddrinfo(ressave); } - return (sockfd); + return sockfd; // 返回连接后的套接字描述符 } /* @@ -801,16 +973,26 @@ note: We finally get the infomation of the ports of other hosts through a linked date: 2022/8/10 contact tel: 18720816902 */ +/* +* 功能：在指定的主机和端口上进行TCP监听 +* host：待连接的主机名或IP地址的字符串表示 +* port：待连接的端口号 +* addrlenp：一个指向socklen_t类型的指针，用于在成功绑定套接字后存储协议地址的大小 +* 返回值：表示监听套接字的文件描述符 +*/ int mc_tcp_listen(const char* host, int port, socklen_t* addrlenp) { #ifdef LIBCOMM_FAULT_INJECTION_ENABLE + // 检查是否启用了通信故障注入 if (is_comm_fault_injection(LIBCOMM_FI_MC_TCP_LISTEN_FAILED)) { - LIBCOMM_ELOG( - WARNING, "(mc tcp listen)\t[FAULT INJECTION]Failed to do control tcp listen for %s:%d.", host, port); + // 打印警告日志，模拟故障情况 + LIBCOMM_ELOG(WARNING, "(mc tcp listen)\t[FAULT INJECTION]Failed to do control tcp listen for %s:%d.", host, + port); return -1; } #endif + // 初始化变量和结构体 errno = 0; int listenfd, n, error = 0; const int on = 1; @@ -826,15 +1008,17 @@ int mc_tcp_listen(const char* host, int port, socklen_t* addrlenp) ss_rc = memset_s(&hints, sizeof(hints), 0, sizeof(struct addrinfo)); securec_check(ss_rc, "\0", "\0"); + // 设置地址信息的参数 hints.ai_flags = AI_PASSIVE; hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; retry: + // 获取地址信息 if (strcmp(host, "localhost") == 0) { - n = getaddrinfo(NULL, serv, &hints, &res); + n = getaddrinfo(NULL, serv, &hints, &res); // 解析主机名或IP地址 } else { - n = getaddrinfo(host, serv, &hints, &res); + n = getaddrinfo(host, serv, &hints, &res); // 解析指定的主机名或IP地址 } if (n == EAI_AGAIN) { @@ -842,47 +1026,48 @@ retry: } if (n != 0 || res == NULL) { - LIBCOMM_ELOG(WARNING, - "(mc tcp listen)\tFailed to get address infomation for host:port[%s:%s] error[%d]:%s.", - host, - serv, - n, - mc_strerror(errno)); + // 打印警告日志，表示获取地址信息失败 + LIBCOMM_ELOG(WARNING, "(mc tcp listen)\tFailed to get address infomation for host:port[%s:%s] error[%d]:%s.", + host, serv, n, mc_strerror(errno)); return -1; } ressave = res; do { - + // 创建监听套接字 listenfd = mc_tcp_socket(res->ai_family, res->ai_socktype, res->ai_protocol); if (listenfd < 0) { - continue; // error, try next one + continue; // 创建套接字失败，尝试下一个 } - // set tcp connection socket properties - // + + // 设置套接字选项 mc_tcp_setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); error = mc_tcp_set_cloexec(listenfd); if (error == 0) { error = mc_tcp_set_nonblock(listenfd); } + // 绑定套接字到地址 if (error == 0 && (mc_tcp_bind(listenfd, res->ai_addr, res->ai_addrlen) == 0)) { - break; // success + break; // 绑定成功 } - mc_tcp_close(listenfd); // bind error, close and try next one + mc_tcp_close(listenfd); // 绑定失败，关闭套接字并尝试下一个 listenfd = -1; } while ((res = res->ai_next) != NULL); if (listenfd != -1) { + // 开始监听连接 mc_tcp_do_listen(listenfd, TCP_LISTENQ); if (addrlenp != NULL && res != NULL) { - *addrlenp = res->ai_addrlen; // return size of protocol address + *addrlenp = res->ai_addrlen; // 返回协议地址的大小 } + // 打印调试日志，表示成功开始监听 COMM_DEBUG_LOG("(mc tcp listen)\tControl tcp listen for %s:%s on socket[%d].", host, serv, listenfd); } else { + // 打印警告日志，表示监听失败 LIBCOMM_ELOG(WARNING, "(mc tcp listen)\tFailed to do control tcp listen for %s:%s.", host, serv); } -- 2.34.1 From dae34bd1b5bb8dfe91fc18042200dab22854b470 Mon Sep 17 00:00:00 2001 From: shuimuzhihua Date: Sat, 26 Aug 2023 14:47:36 +0800 Subject: [PATCH 03/15] Update mc_poller_epoll.h --- .../libcomm_core/mc_poller_epoll.h | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.h b/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.h index 2edd15160..6a8221f98 100644 --- a/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.h +++ b/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.h @@ -30,24 +30,37 @@ #define MC_POLLER_HAVE_ASYNC_ADD 1 -#define MC_POLLER_MAX_EVENTS 512 +#define MC_POLLER_MAX_EVENTS 512 // 目前正在处理事件（epoll_event）的最大数目 -#define MC_POLLER_FD_ID_OFFSET 32 +#define MC_POLLER_FD_ID_OFFSET 32 // fd的移位值（左移） #define MC_POLLER_FD_ID_MASK 0xffff -// Item of epoller list, including poller handler and list item. -// -struct mc_poller_hndl_item { +// 定义一个结构体mc_poller_hndl_item，用于存储poller句柄和列表项 +struct mc_poller_hndl_item +{ struct mc_poller_hndl hndl; struct mc_list_item item; }; - -struct mc_poller { - int ep; // Current pollset. - int nevents; // Number of events being processed at the moment. - int index; // Index of the event being processed at the moment. - struct epoll_event events[MC_POLLER_MAX_EVENTS]; // Events being processed at the moment. +/* +struct mc_poller_hndl +{ + int fd; // 文件描述符 + int id; // 与文件描述符相关联的标识符或索引(标识特定连接或资源的唯一标识符) + uint32_t events; // 事件 }; +struct mc_list_item +{ + struct mc_list_item* next; // 具有一个指向自己类型的指针 +}; +*/ + +struct mc_poller +{ + int ep; // 当前poller值 + int nevents; // 当前正在处理的事件数 + int index; // 当前正在处理的事件的索引 + struct epoll_event events[MC_POLLER_MAX_EVENTS]; // 目前正在处理事件 +}; #endif -- 2.34.1 From bd9e16656c3647b384b5e3fd8ac67f12e4a00df3 Mon Sep 17 00:00:00 2001 From: shuimuzhihua Date: Sat, 26 Aug 2023 14:48:39 +0800 Subject: [PATCH 04/15] Update mc_poller_epoll.h --- .../communication/libcomm_core/mc_poller_epoll.h | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.h b/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.h index 6a8221f98..d60650c77 100644 --- a/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.h +++ b/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.h @@ -42,19 +42,6 @@ struct mc_poller_hndl_item struct mc_poller_hndl hndl; struct mc_list_item item; }; -/* -struct mc_poller_hndl -{ - int fd; // 文件描述符 - int id; // 与文件描述符相关联的标识符或索引(标识特定连接或资源的唯一标识符) - uint32_t events; // 事件 -}; - -struct mc_list_item -{ - struct mc_list_item* next; // 具有一个指向自己类型的指针 -}; -*/ struct mc_poller { -- 2.34.1 From fb6f6b33cf19e5b045f1ce731a11ad1fc73d6bff Mon Sep 17 00:00:00 2001 From: shuimuzhihua Date: Sat, 26 Aug 2023 15:01:44 +0800 Subject: [PATCH 05/15] Update mc_poller.h --- .../communication/libcomm_core/mc_poller.h | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_core/mc_poller.h b/src/gausskernel/cbb/communication/libcomm_core/mc_poller.h index 027b208bf..15f282969 100644 --- a/src/gausskernel/cbb/communication/libcomm_core/mc_poller.h +++ b/src/gausskernel/cbb/communication/libcomm_core/mc_poller.h @@ -30,44 +30,43 @@ #include "libcomm_utils/libcomm_err.h" #include "mc_poller_epoll.h" -// Initialize poller. -// +// 初始化轮询器 int mc_poller_init(struct mc_poller* self); -// Terminate poller. -// + +// 终止轮询器 void mc_poller_term(struct mc_poller* self); -// Add fd to poller. -// + +// 向轮询器中添加socket void mc_poller_add(struct mc_poller* self, int fd, int id); -// Remove item from poller -// + +// 从轮询器中删除socket int mc_poller_rm(struct mc_poller* self, int fd); -// Set poller event as POLLIN. -// + +// 设置轮询事件 void mc_poller_set_in(struct mc_poller* self, struct mc_poller_hndl* hndl); -// Wait poller events happens. -// + +// 等待轮询事件发生 int mc_poller_wait(struct mc_poller* self, int timeout); -// declaration of Poller handler list. -// -class mc_poller_hndl_list { +// 事件轮询处理器列表 +class mc_poller_hndl_list +{ public: mc_poller_hndl_list(); ~mc_poller_hndl_list(); struct mc_poller* get_poller(); int get_socket_count(); int init(); - int add_fd(struct sock_id* fd_id); - int del_fd(struct sock_id* fd_id); - // Disable copy construction and assignment. + int add_fd(struct sock_id* fd_id); // 添加socket + int del_fd(struct sock_id* fd_id); // 删除socket + // 删除拷贝构造函数和赋值运算符重载 mc_poller_hndl_list(const mc_poller_hndl_list&) = delete; const mc_poller_hndl_list& operator=(const mc_poller_hndl_list&) = delete; - struct mc_poller* m_poller; - +public: + struct mc_poller* m_poller; // 存储事件的容器(事件数组) private: - pthread_mutex_t m_lock; - int m_socket_count; + pthread_mutex_t m_lock; // 互斥锁 + int m_socket_count; // 套接字个数 }; #endif // _MC_POLLER_H_ -- 2.34.1 From 3b67b1f4d304f649e8f170fbee045f024363bcd3 Mon Sep 17 00:00:00 2001 From: shuimuzhihua Date: Sat, 26 Aug 2023 15:02:48 +0800 Subject: [PATCH 06/15] Update mc_poller_epoll.cpp --- .../libcomm_core/mc_poller_epoll.cpp | 153 +++++++++++++----- 1 file changed, 110 insertions(+), 43 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.cpp b/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.cpp index facc0752b..18556b577 100644 --- a/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.cpp +++ b/src/gausskernel/cbb/communication/libcomm_core/mc_poller_epoll.cpp @@ -25,6 +25,12 @@ #include "mc_poller.h" #include "libcomm_common.h" +/* +* 功能：初始化一个事件轮询器 +* self：指向mc_poller结构的指针，用于管理事件轮询 +* 返回值：返回0表示成功初始化(但也可能抛出异常) +* 注：该函数封装了epoll_create函数 +*/ int mc_poller_init(struct mc_poller* self) { #ifndef EPOLL_CLOEXEC @@ -32,80 +38,116 @@ int mc_poller_init(struct mc_poller* self) #endif #ifdef EPOLL_CLOEXEC + // 创建epoll实例，并使用EPOLL_CLOEXEC标志 self->ep = epoll_create1(EPOLL_CLOEXEC); #else - // Size parameter is unused, we can safely set it to 1. - // - self->ep = epoll_create(1); + // 设置epoll实例的FD_CLOEXEC标志，避免子进程继承该文件描述符 rc = fcntl(self->ep, F_SETFD, FD_CLOEXEC); errno_assert(rc != -1); #endif + if (self->ep == -1) { + // 处理创建epoll失败的情况 if (errno == ENFILE || errno == EMFILE) { - return -EMFILE; + return -EMFILE; // 文件描述符不足 } - errno_assert(false); + errno_assert(false); // 发生其他错误，抛出异常 } + + // 初始化轮询器的nevents和index self->nevents = 0; self->index = 0; return 0; } +/* + * 功能：关闭轮询器描述符，释放资源 + * self：指向mc_poller结构的指针，用于管理事件轮询 + */ void mc_poller_term(struct mc_poller* self) { - close(self->ep); + close(self->ep); // 关闭轮询器描述符，释放资源 } +/* + * 功能：向事件轮询器中添加一个文件描述符，并进行相关的初始化操作 + * self：指向mc_poller结构的指针，用于管理事件轮询 + * fd：待添加到事件轮询器中的文件描述符 + * id：与文件描述符相关联的标识符或索引(标识特定连接或资源的唯一标识符) + * 注：该函数封装了epoll_ctl函数 + */ void mc_poller_add(struct mc_poller* self, int fd, int id) { int rc; struct epoll_event ev; errno_t ss_rc = 0; - // Initialise the handle and add the file descriptor to the pollset. - // + // 初始化事件结构 ss_rc = memset_s(&ev, sizeof(ev), 0, sizeof(struct epoll_event)); securec_check(ss_rc, "\0", "\0"); + + // 设置事件类型为EPOLLIN，表示可读事件 ev.events = EPOLLIN; + + // 将文件描述符和id组合成一个64位的整数，存储在事件的data.u64中 ev.data.u64 = (((uint64)(unsigned)(fd)) << MC_POLLER_FD_ID_OFFSET) + id; + + // 将文件描述符添加到事件轮询器中 rc = epoll_ctl(self->ep, EPOLL_CTL_ADD, fd, &ev); - errno_assert(rc == 0); + errno_assert(rc == 0); // 检查epoll_ctl是否成功 } -int mc_poller_rm(struct mc_poller* self, int fd) +/* + * 功能：从事件轮询器中移除文件描述符 + * self：指向mc_poller结构的指针，用于管理事件轮询 + * fd：待添加到事件轮询器中的文件描述符 + * 返回值：直接返回epoll_ctl结果 + * 注：该函数封装了epoll_ctl函数 + */ +int mc_poller_rm(struct mc_poller *self, int fd) { - // Remove the file descriptor from the pollset. - // + // 使用epoll_ctl函数执行删除操作 return epoll_ctl(self->ep, EPOLL_CTL_DEL, fd, NULL); } -int mc_poller_wait(struct mc_poller* self, int timeout) +/* + * 功能：在事件轮询器上等待事件的发生 + * self：指向mc_poller结构的指针，用于管理事件轮询 + * timeout：等待事件的超时时间(单位：毫秒) + * 返回值：返回0表示成功初始化(但也可能抛出异常) + * 注：该函数封装了epoll_wait函数 + */ +int mc_poller_wait(struct mc_poller *self, int timeout) { int nevents; - // Clear all existing events. - // + // 清除现有事件 self->nevents = 0; self->index = 0; - // Wait for new events. - // + // 等待新事件的发生 for (;;) { + // 使用epoll_wait函数等待事件，并将结果保存在self->events数组中 nevents = epoll_wait(self->ep, self->events, MC_POLLER_MAX_EVENTS, timeout); + + // 如果返回结果是-1且错误是EINTR，表示被中断，继续等待 if (mc_slow(nevents == -1 && errno == EINTR)) { continue; } else { break; } } + + // 检查self->nevents是否不等于-1，如果等于-1，抛出异常 errno_assert(self->nevents != -1); + + // 将实际获取的事件数量赋值给self->nevents self->nevents = nevents; return 0; } -// implementation of mc_poller_hndl_list -// +// mc_poller_hndl_list类的构造函数和析构函数 mc_poller_hndl_list::mc_poller_hndl_list() { m_poller = NULL; @@ -116,72 +158,97 @@ mc_poller_hndl_list::~mc_poller_hndl_list() { LIBCOMM_FREE(m_poller, sizeof(struct mc_poller)); } + +/* + * 功能：初始化事件轮询处理器列表 + * 返回值：返回0表示成功初始化，返回-1表示出错 + */ int mc_poller_hndl_list::init() { - LIBCOMM_PTHREAD_MUTEX_INIT(&m_lock, 0); + LIBCOMM_PTHREAD_MUTEX_INIT(&m_lock, 0); // 初始化互斥锁 - LIBCOMM_MALLOC(m_poller, (sizeof(struct mc_poller)), mc_poller); + LIBCOMM_MALLOC(m_poller, (sizeof(struct mc_poller)), mc_poller); // 分配内存存储轮询器 if (NULL == m_poller) { - return -1; + return -1; // 内存分配失败，返回错误码 } if (mc_poller_init(m_poller)) { - return -1; + return -1; // 轮询器初始化失败，返回错误码 } - m_socket_count = 0; + m_socket_count = 0; // 初始化socket数量 - return 0; + return 0; // 初始化成功，返回0 } + struct mc_poller* mc_poller_hndl_list::get_poller() { return m_poller; } +/* + * 功能：获取事件轮询处理器列表中的socket数量 + * 返回值：返回获取到的socket数量 + * 注：该读取操作线程不安全，所以需要使用互斥锁 + */ int mc_poller_hndl_list::get_socket_count() { int cnt = 0; - LIBCOMM_PTHREAD_MUTEX_LOCK(&m_lock); - cnt = m_socket_count; - LIBCOMM_PTHREAD_MUTEX_UNLOCK(&m_lock); - return cnt; + LIBCOMM_PTHREAD_MUTEX_LOCK(&m_lock); // 加锁 + cnt = m_socket_count; // 获取socket数量 + LIBCOMM_PTHREAD_MUTEX_UNLOCK(&m_lock); // 解锁 + return cnt; // 返回获取到的socket数量 } -int mc_poller_hndl_list::add_fd(struct sock_id* fd_id) +/* + * 功能：将文件描述符添加到事件轮询处理器列表中 + * fd_id：文件描述符 + * 返回值：成功返回0；失败返回-1 + */ +int mc_poller_hndl_list::add_fd(struct sock_id *fd_id) { if (fd_id == NULL) { - return -1; + return -1; // 如果传入的文件描述符为空，返回错误码-1 } + #ifdef LIBCOMM_FAULT_INJECTION_ENABLE if (is_comm_fault_injection(LIBCOMM_FI_POLLER_ADD_FD_FAILED)) { - LIBCOMM_ELOG( - WARNING, "(poller add fd)\t[FAULT INJECTION]Failed to save socket[%d] version[%d].", fd_id->fd, fd_id->id); + // 如果启用了错误注入并且出现故障注入，记录日志并返回错误码-1 + LIBCOMM_ELOG(WARNING, "(poller add fd)\t[FAULT INJECTION]Failed to save socket[%d] version[%d].", fd_id->fd, + fd_id->id); return -1; } #endif - LIBCOMM_PTHREAD_MUTEX_LOCK(&m_lock); - // add the the socket to do epoll - // + LIBCOMM_PTHREAD_MUTEX_LOCK(&m_lock); // 加锁 + + // 将文件描述符添加到事件轮询器中进行监听 mc_poller_add(m_poller, fd_id->fd, fd_id->id); + + // 增加监听的socket数量 m_socket_count++; - LIBCOMM_PTHREAD_MUTEX_UNLOCK(&m_lock); + LIBCOMM_PTHREAD_MUTEX_UNLOCK(&m_lock); // 解锁 return 0; } -int mc_poller_hndl_list::del_fd(struct sock_id* fd_id) +/* + * 功能：从事件轮询处理器列表中移除文件描述符 + * fd_id：文件描述符 + * 返回值：移除结果（mc_poller_rm函数的返回值） + */ +int mc_poller_hndl_list::del_fd(struct sock_id *fd_id) { if (fd_id == NULL) { - return -1; + return -1; // 如果传入的文件描述符标识符为空，返回错误码-1 } - LIBCOMM_PTHREAD_MUTEX_LOCK(&m_lock); + LIBCOMM_PTHREAD_MUTEX_LOCK(&m_lock); // 加锁 - int rc = mc_poller_rm(m_poller, fd_id->fd); + int rc = mc_poller_rm(m_poller, fd_id->fd); // 移除文件描述符 - LIBCOMM_PTHREAD_MUTEX_UNLOCK(&m_lock); + LIBCOMM_PTHREAD_MUTEX_UNLOCK(&m_lock); // 解锁 - return rc; + return rc; // 返回移除结果 } -- 2.34.1 From 45bbe9e64e973c9e40821f0be1133d93012c01ad Mon Sep 17 00:00:00 2001 From: richard_chen Date: Wed, 27 Sep 2023 21:44:39 +0800 Subject: [PATCH 07/15] =?UTF-8?q?=E5=AF=B9=E4=BA=8Elibcomm=5Fshakehands.cp?= =?UTF-8?q?p=E6=96=87=E4=BB=B6=E4=B8=AD=E7=9A=84=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E8=BF=9B=E8=A1=8C=E4=BA=86=E6=B3=A8=E9=87=8A=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../libcomm_utils/libcomm_shakehands.cpp | 652 +++++++++++------- 1 file changed, 413 insertions(+), 239 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_shakehands.cpp b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_shakehands.cpp index 64c53c4b8..113a56a31 100644 --- a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_shakehands.cpp +++ b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_shakehands.cpp @@ -65,7 +65,6 @@ #define static #endif - /* * function name : gs_r_build_reply_connection * description : as the connection between cn & dn is duplex. @@ -78,105 +77,137 @@ * return value : -1: error * : 0:Succeed */ -static int gs_r_build_reply_connection(FCMSG_T* fcmsgr, int local_version) +// 此函数的作用是在两个结点间建立反向连接，其中参数fcmsgr提供了socket套接字和远端的端口 +static int gs_r_build_reply_connection(FCMSG_T *fcmsgr, int local_version) { + // 定义变量ss_rc用于保存函数返回值，类型为errno_t，用于处理系统调用和库函数错误 errno_t ss_rc; + // 定义变量cpylen为uint32类型，用于保存复制的长度 uint32 cpylen; + // 从fcmsgr中获取node_idx，streamid和remote_version，并保存到本地变量中 int node_idx = fcmsgr->node_idx; int streamid = fcmsgr->streamid; int remote_version = fcmsgr->version; - // get remote nodename and host from global variable + // 初始化远程主机名和节点名的字符数组，长度为HOST_ADDRSTRLEN和NAMEDATALEN，初始值为0x0 char remote_host[HOST_ADDRSTRLEN] = {0x0}; char remote_nodename[NAMEDATALEN] = {0x0}; + // 从全局变量g_instance.comm_cxt.g_r_node_sock[node_idx].remote_host中获取cpylen，这是远程主机名的长度 cpylen = comm_get_cpylen(g_instance.comm_cxt.g_r_node_sock[node_idx].remote_host, HOST_ADDRSTRLEN); + // 使用memset_s函数将remote_host数组的值设置为0x0，长度为HOST_ADDRSTRLEN，函数返回值为ss_rc ss_rc = memset_s(remote_host, HOST_ADDRSTRLEN, 0x0, HOST_ADDRSTRLEN); + // 调用securec_check函数检查memset_s函数的返回值，如果返回值不为0，则报错并退出程序 securec_check(ss_rc, "\0", "\0"); - ss_rc = - strncpy_s(remote_host, HOST_ADDRSTRLEN, g_instance.comm_cxt.g_r_node_sock[node_idx].remote_host, cpylen + 1); + // 使用strncpy_s函数将g_instance.comm_cxt.g_r_node_sock[node_idx].remote_host中的内容复制到remote_host中，复制长度为cpylen+1，函数返回值为ss_rc + ss_rc = strncpy_s(remote_host, HOST_ADDRSTRLEN, g_instance.comm_cxt.g_r_node_sock[node_idx].remote_host, cpylen + 1); + // 调用securec_check函数检查strncpy_s函数的返回值，如果返回值不为0，则报错并退出程序 securec_check(ss_rc, "\0", "\0"); + // 在复制的内容后面添加一个'\0'字符，表示字符串的结束 remote_host[cpylen] = '\0'; + // 从全局变量g_instance.comm_cxt.g_r_node_sock[node_idx].remote_nodename中获取cpylen，这是远程节点名的长度 cpylen = comm_get_cpylen(g_instance.comm_cxt.g_r_node_sock[node_idx].remote_nodename, NAMEDATALEN); + // 使用memset_s函数将remote_nodename数组的值设置为0x0，长度为NAMEDATALEN，函数返回值为ss_rc ss_rc = memset_s(remote_nodename, NAMEDATALEN, 0x0, NAMEDATALEN); + // 调用securec_check函数检查memset_s函数的返回值，如果返回值不为0，则报错并退出程序 securec_check(ss_rc, "\0", "\0"); + // 使用strncpy_s函数将g_instance.comm_cxt.g_r_node_sock[node_idx].remote_nodename中的内容复制到remote_nodename中，复制长度为cpylen+1，函数返回值为ss_rc ss_rc = strncpy_s( remote_nodename, NAMEDATALEN, g_instance.comm_cxt.g_r_node_sock[node_idx].remote_nodename, cpylen + 1); + // 调用securec_check函数检查strncpy_s函数的返回值，如果返回值不为0，则报错并退出程序 securec_check(ss_rc, "\0", "\0"); + // 在复制的内容后面添加一个'\0'字符，表示字符串的结束 remote_nodename[cpylen] = '\0'; - // streamcap contain the ctrl port and data port of remote process. - // we use streamcap to send these ports - // becuase we do not want to add too much members to FCMSG_T - // so we usually use streamcap to send some extra msgs + // 从fcmsgr中获取streamcap，这是一个64位的整数，其中高32位保存了控制端口，低32位保存了数据端口 + // 我们使用streamcap来发送这些端口，是因为我们不想在FCMSG_T中添加太多成员，所以我们通常使用streamcap来发送一些额外的消息 int remote_ctrl_port = (int)(fcmsgr->streamcap >> 32); int remote_data_port = (int)(fcmsgr->streamcap); + // 初始化libcommaddrinfo结构体，该结构体保存了远程主机的信息 libcommaddrinfo libcomm_addrinfo; libcomm_addrinfo.host = remote_host; libcomm_addrinfo.ctrl_port = remote_ctrl_port; libcomm_addrinfo.listen_port = remote_data_port; + + // 获取远程节点名的长度，并保存到cpylen中 cpylen = comm_get_cpylen(remote_nodename, NAMEDATALEN); + // 使用memset_s函数将libcomm_addrinfo.nodename数组的值设置为0x0，长度为NAMEDATALEN，函数返回值为ss_rc ss_rc = memset_s(libcomm_addrinfo.nodename, NAMEDATALEN, 0x0, NAMEDATALEN); + // 调用securec_check函数检查memset_s函数的返回值，如果返回值不为0，则报错并退出程序 securec_check(ss_rc, "\0", "\0"); + // 使用strncpy_s函数将remote_nodename中的内容复制到libcomm_addrinfo.nodename中，复制长度为cpylen+1，函数返回值为ss_rc ss_rc = strncpy_s(libcomm_addrinfo.nodename, NAMEDATALEN, remote_nodename, cpylen + 1); + // 调用securec_check函数检查strncpy_s函数的返回值，如果返回值不为0，则报错并退出程序 securec_check(ss_rc, "\0", "\0"); + // 在复制的内容后面添加一个'\0'字符，表示字符串的结束 libcomm_addrinfo.nodename[cpylen] = '\0'; + // 输出调试日志，记录正在为哪个节点建立TCP连接 COMM_DEBUG_LOG("(r|build reply conn)\tBuild TCP connect for node[%d]:%s.", - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx)); + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx)); - if (unlikely(gs_s_check_connection(&libcomm_addrinfo, node_idx, true, CTRL_CHANNEL) == false)) { + // 检查是否成功与远程主机建立连接，如果连接失败，则返回false + if (unlikely(gs_s_check_connection(&libcomm_addrinfo, node_idx, true, CTRL_CHANNEL) == false)) + { + // 如果连接失败，则输出错误日志，记录失败的主机、端口和节点名 LIBCOMM_ELOG(WARNING, - "(r|build reply conn)\tFailed to connect to host:port[%s:%d], node name[%s].", - libcomm_addrinfo.host, - libcomm_addrinfo.ctrl_port, - remote_nodename); + "(r|build reply conn)\tFailed to connect to host:port[%s:%d], node name[%s].", + libcomm_addrinfo.host, + libcomm_addrinfo.ctrl_port, + remote_nodename); return -1; } - /* - * Check and build logic connection whit the remote point (if need) - */ + // 输出调试日志，记录正在为哪个节点建立数据连接 COMM_DEBUG_LOG("(r|build reply conn)\tBuild data connection for node[%d]:%s.", - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx)); + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx)); - // failed to build logic connection - if (unlikely(gs_s_check_connection(&libcomm_addrinfo, node_idx, true, DATA_CHANNEL) == false)) { + // 如果与远程主机的数据连接建立失败 + if (unlikely(gs_s_check_connection(&libcomm_addrinfo, node_idx, true, DATA_CHANNEL) == false)) + { + // 输出错误日志，记录失败的主机、端口和节点名，以及错误信息 LIBCOMM_ELOG(WARNING, - "(r|build reply conn)\tFailed to build data connection " - "to %s:%d for node[%d]:%s, detail:%s.", - libcomm_addrinfo.host, - libcomm_addrinfo.listen_port, - node_idx, - libcomm_addrinfo.nodename, - mc_strerror(errno)); + "(r|build reply conn)\tFailed to build data connection " + "to %s:%d for node[%d]:%s, detail:%s.", + libcomm_addrinfo.host, + libcomm_addrinfo.listen_port, + node_idx, + libcomm_addrinfo.nodename, + mc_strerror(errno)); errno = ECOMMTCPCONNFAIL; return -1; } + // 输出调试日志，记录已经成功建立了数据连接 COMM_DEBUG_LOG("(r|build reply conn)\tBuild data logical connection for node[%d]:%s.", - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx)); + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx)); - struct p_mailbox* pmailbox = &P_MAILBOX(node_idx, streamid); + // 获取pmailbox指针，它指向了P_MAILBOX(node_idx, streamid)的地址 + struct p_mailbox *pmailbox = &P_MAILBOX(node_idx, streamid); + // 加锁，确保对pmailbox的操作是线程安全的 LIBCOMM_PTHREAD_MUTEX_LOCK(&pmailbox->sinfo_lock); - if (pmailbox->state != MAIL_CLOSED) { + // 如果pmailbox的状态不是MAIL_CLOSED（即邮箱没有关闭） + if (pmailbox->state != MAIL_CLOSED) + { + // 输出错误日志，记录无法获取邮箱的节点索引和流ID MAILBOX_ELOG(pmailbox, - WARNING, - "(r|build reply conn)\tFailed to get mailbox for node[%d,%d]:%s.", - node_idx, - streamid, - REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx)); + WARNING, + "(r|build reply conn)\tFailed to get mailbox for node[%d,%d]:%s.", + node_idx, + streamid, + REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx)); + // 关闭逻辑连接，并设置关闭原因为ECOMMTCPREMOETECLOSE（表示远程邮箱已经关闭） gs_s_close_logic_connection(pmailbox, ECOMMTCPREMOETECLOSE, NULL); } - // inital pmailbox for sending msgs to cn + // 为向cn发送消息初始化pmailbox pmailbox->local_version = local_version; pmailbox->remote_version = remote_version; pmailbox->ctrl_tcp_sock = g_instance.comm_cxt.g_s_node_sock[node_idx].ctrl_tcp_sock; @@ -187,16 +218,22 @@ static int gs_r_build_reply_connection(FCMSG_T* fcmsgr, int local_version) pmailbox->local_thread_id = 0; pmailbox->peer_thread_id = 0; pmailbox->close_reason = 0; + // 调用COMM_STAT_CALL宏，记录pmailbox的统计信息，将当前时间赋值给pmailbox->statistic->start_time COMM_STAT_CALL(pmailbox, pmailbox->statistic->start_time = (uint32)mc_timers_ms()); - if (g_instance.comm_cxt.commutil_cxt.g_stat_mode && (NULL == pmailbox->statistic)) { + // 如果g_instance.comm_cxt.commutil_cxt.g_stat_mode为真且pmailbox->statistic为NULL + // 则动态分配pmailbox->statistic的内存空间，大小为sizeof(struct pmailbox_statistic) + if (g_instance.comm_cxt.commutil_cxt.g_stat_mode && (NULL == pmailbox->statistic)) + { LIBCOMM_MALLOC(pmailbox->statistic, sizeof(struct pmailbox_statistic), pmailbox_statistic); - if (NULL == pmailbox->statistic) { + // 如果动态分配内存失败，则设置errno为ECOMMTCPRELEASEMEM，解锁pmailbox->sinfo_lock，并返回-1 + if (NULL == pmailbox->statistic) + { errno = ECOMMTCPRELEASEMEM; LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); return -1; } } - + // 解锁pmailbox->sinfo_lock LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); return 0; @@ -211,50 +248,70 @@ static int gs_r_build_reply_connection(FCMSG_T* fcmsgr, int local_version) * arguments : _in_ cmailbox: libcomm conntion info. * _in_ close_reason: close reason. */ -void gs_r_close_logic_connection(struct c_mailbox* cmailbox, int close_reason, FCMSG_T* msg) +// 此函数的作用是消费者关闭逻辑连接并且重置邮箱。其中参数cmailbox中包含着连接的信息，也就是要关闭的信息，close_reason表示的是关闭的原因，msg为关闭控制信息。 +void gs_r_close_logic_connection(struct c_mailbox *cmailbox, int close_reason, FCMSG_T *msg) { + // 定义变量ss_rc用于保存函数返回值，类型为errno_t，用于处理系统调用和库函数错误 errno_t ss_rc; + // 定义变量cpylen用于保存复制的长度，类型为uint32 uint32 cpylen; - if (cmailbox->state == MAIL_CLOSED) { + // 检查邮箱状态，如果已经是关闭状态，则直接返回，不再执行后续操作 + if (cmailbox->state == MAIL_CLOSED) + { return; } - if (ENABLE_THREAD_POOL_DN_LOGICCONN) { + // 如果启用了线程池DN逻辑连接，则调用NotifyListener函数，通知监听器邮箱状态将要改变 + if (ENABLE_THREAD_POOL_DN_LOGICCONN) + { NotifyListener(cmailbox, true, __FUNCTION__); } - // 1, if tcp disconnect, we can not send control message on tcp channel, - // remote can receive disconnect event when flow control thread call epoll_wait. - // 2, close reason is ECOMMTCPREMOETECLOSE means remote send MAIL_CLOSED, - // we could not reply MAIL_CLOSED message. - if (IS_NOTIFY_REMOTE(close_reason) && msg) { + // 1. 如果TCP断开连接，我们不能在TCP通道上发送控制消息，当流控制线程调用epoll_wait时，远程可以接收到断开事件。 + // 2. 如果关闭原因是ECOMMTCPREMOETECLOSE，表示远程发送了MAIL_CLOSED，我们不能回复MAIL_CLOSED消息。 + if (IS_NOTIFY_REMOTE(close_reason) && msg) + { + // 设置msg的类型为CTRL_CLOSED，表示关闭控制消息 msg->type = CTRL_CLOSED; + // 设置msg的节点索引为cmailbox的索引 msg->node_idx = cmailbox->idx; + // 设置msg的流ID为cmailbox的流ID msg->streamid = cmailbox->streamid; + // 设置msg的流容量为0 msg->streamcap = 0; + // 设置msg的版本为cmailbox的远程版本 msg->version = cmailbox->remote_version; + // 设置msg的查询ID为cmailbox的查询ID msg->query_id = cmailbox->query_id; + // 获取本地节点名的长度并保存在cpylen中 cpylen = comm_get_cpylen(g_instance.comm_cxt.localinfo_cxt.g_self_nodename, NAMEDATALEN); + // 使用memset_s函数将msg的节点名设置为0，长度为NAMEDATALEN ss_rc = memset_s(msg->nodename, NAMEDATALEN, 0x0, NAMEDATALEN); + // 使用securec_check函数检查memset_s函数的返回值，如果返回错误，则终止程序 securec_check(ss_rc, "\0", "\0"); + // 使用strncpy_s函数将本地节点名复制到msg的节点名中，长度为cpylen + 1 ss_rc = strncpy_s(msg->nodename, NAMEDATALEN, g_instance.comm_cxt.localinfo_cxt.g_self_nodename, cpylen + 1); + // 使用securec_check函数检查strncpy_s函数的返回值，如果返回错误，则终止程序 securec_check(ss_rc, "\0", "\0"); + // 在msg的节点名后添加字符串结束标志'\0' msg->nodename[cpylen] = '\0'; } - // wake up the consumer who is waiting for the data at gs_wait_poll, + // 唤醒在gs_wait_poll处等待数据的消费者 gs_poll_signal(cmailbox->semaphore); - // At last, reset mailbox and clean hash table + // 最后，重置邮箱并清理哈希表 gs_r_reset_cmailbox(cmailbox, close_reason); return; } +// 此函数的作用是获得接收开始时间 uint64 gs_get_recv_ready_time() { + // 返回全局变量g_instance.comm_cxt.localinfo_cxt.g_r_first_recv_time的值 return g_instance.comm_cxt.localinfo_cxt.g_r_first_recv_time; } @@ -269,16 +326,19 @@ uint64 gs_get_recv_ready_time() * and initialize consumer mailbox, * then reply MAIL_READY message to producer. */ -void gs_receivers_flow_handle_ready_request(FCMSG_T* fcmsgr) +// 此函数的作用是在接收端接收到MAIL_READY消息时处理就绪请求 +void gs_receivers_flow_handle_ready_request(FCMSG_T *fcmsgr) { + // 从fcmsgr中获取流ID，并赋值给uint16类型的变量streamid uint16 streamid = fcmsgr->streamid; + // 从fcmsgr中获取结点索引，并赋值给uint16类型的变量node_idx uint16 node_idx = fcmsgr->node_idx; - // producer send pmailbox version as fcmsgr->version, - // now save it to cmailbox->remote_version. + int remote_verion = fcmsgr->version; int ctrl_socket = g_instance.comm_cxt.g_r_node_sock[node_idx].ctrl_tcp_sock; struct FCMSG_T fcmsgs = {0x0}; - struct c_mailbox* cmailbox = NULL; + // 初始化cmailbox传递载体 + struct c_mailbox *cmailbox = NULL; int rc = -1; uint64 time_now; uint64 time_callback_start = 0; @@ -298,35 +358,41 @@ void gs_receivers_flow_handle_ready_request(FCMSG_T* fcmsgr) COMM_TIMER_INIT(); #ifdef LIBCOMM_FAULT_INJECTION_ENABLE - if (is_comm_fault_injection(LIBCOMM_FI_CONSUMER_REJECT)) { + if (is_comm_fault_injection(LIBCOMM_FI_CONSUMER_REJECT)) + { LIBCOMM_ELOG(WARNING, "(r|flow ctrl)\t[FAULT INJECTION]Consumer can not accept."); errno = ECOMMTCPAPPCLOSE; goto accept_failed; } #endif - // the entry should be close, may be caused by cancel - // node_idx < 0 means data conntion has some errors + // 判断node_idx是否小于0，如果小于0，说明数据连接存在错误 Assert(node_idx >= 0); - // initialize consumer cmailbox + // 初始化消费者的邮箱 cmailbox = &C_MAILBOX(node_idx, streamid); + // 对消费者邮箱上锁，防止其他线程对其改变 LIBCOMM_PTHREAD_MUTEX_LOCK(&cmailbox->sinfo_lock); - if (cmailbox->state != MAIL_CLOSED) { + // 判断邮箱接收的是不是MAIL_CLOSED标志信息，如果不等于，进行日志记录 + if (cmailbox->state != MAIL_CLOSED) + { MAILBOX_ELOG(cmailbox, - WARNING, - "(r|flow ctrl)\tFailed to get mailbox for node[%d]:%s.", - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx)); + WARNING, + "(r|flow ctrl)\tFailed to get mailbox for node[%d]:%s.", + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx)); + // 关闭逻辑连接 gs_r_close_logic_connection(cmailbox, ECOMMTCPREMOETECLOSE, NULL); } - + // 将本地版本加1，如果大于等于最大邮箱版本，则重置为0 local_version = cmailbox->local_version + 1; - if (local_version >= MAX_MAILBOX_VERSION) { + if (local_version >= MAX_MAILBOX_VERSION) + { local_version = 0; } + // 初始化消费者邮箱的一系列属性 cmailbox->local_version = local_version; // producer send pmailbox version as fcmsgr->version, // now save it to cmailbox->remote_version. @@ -339,63 +405,88 @@ void gs_receivers_flow_handle_ready_request(FCMSG_T* fcmsgr) cmailbox->local_thread_id = 0; cmailbox->peer_thread_id = 0; cmailbox->close_reason = 0; - if (g_instance.comm_cxt.commutil_cxt.g_stat_mode && (cmailbox->statistic == NULL)) { + // 如果g_instance.comm_cxt.commutil_cxt.g_stat_mode为真且cmailbox的statistic为NULL，则执行下面的代码块 + if (g_instance.comm_cxt.commutil_cxt.g_stat_mode && (cmailbox->statistic == NULL)) + { + // 动态分配cmailbox的statistic成员变量的内存空间 LIBCOMM_MALLOC(cmailbox->statistic, sizeof(struct cmailbox_statistic), cmailbox_statistic); - if (cmailbox->statistic == NULL) { + // 如果动态分配内存失败，则解锁cmailbox的sinfo_lock互斥锁，设置errno为ECOMMTCPRELEASEMEM，并跳转到accept_failed标签处的代码 + if (cmailbox->statistic == NULL) + { + // 解锁消费者邮箱 LIBCOMM_PTHREAD_MUTEX_UNLOCK(&cmailbox->sinfo_lock); errno = ECOMMTCPRELEASEMEM; goto accept_failed; } } + // 获取当前时间，并将其赋值给time_now变量 time_now = COMM_STAT_TIME(); + // 调用COMM_STAT_CALL宏，将time_now赋值给cmailbox的statistic->start_time成员变量 COMM_STAT_CALL(cmailbox, cmailbox->statistic->start_time = (uint32)time_now); + // 打印调试日志，包含node_idx、streamid、g_instance.comm_cxt.g_r_node_sock[node_idx].remote_nodename和cmailbox的状态 COMM_DEBUG_LOG("(r|flow ctrl)\tNode[%d] stream[%d], node name[%s] is in state[%s].", - node_idx, - streamid, - g_instance.comm_cxt.g_r_node_sock[node_idx].remote_nodename, - stream_stat_string(cmailbox->state)); + node_idx, + streamid, + g_instance.comm_cxt.g_r_node_sock[node_idx].remote_nodename, + stream_stat_string(cmailbox->state)); + // 解锁cmailbox的sinfo_lock互斥锁 LIBCOMM_PTHREAD_MUTEX_UNLOCK(&cmailbox->sinfo_lock); + // 计算处理准备消息所花费的时间，单位为微秒 deal_time = ABS_SUB((long long int)time(NULL), (long long int)gs_get_recv_ready_time()); - if ((long long int)mc_tcp_get_connect_timeout() < (long long int)deal_time) { + // 如果连接超时时间小于处理准备消息所花费的时间，则输出警告日志 + if ((long long int)mc_tcp_get_connect_timeout() < (long long int)deal_time) + { LIBCOMM_ELOG(WARNING, "(r|flow ctrl)\tIt takes %lus to process the ready message, query id:%lu.", - deal_time, u_sess->debug_query_id); + deal_time, u_sess->debug_query_id); } - // for dual connection, we need to build an inverse logic conn with same gs_sock - if (fcmsgr->type == CTRL_CONN_DUAL) { + // 如果是双工连接，则需要使用相同的gs_sock建立反向逻辑连接 + if (fcmsgr->type == CTRL_CONN_DUAL) + { + // 设置数据节点数量 u_sess->pgxc_cxt.NumDataNodes = (int)(fcmsgr->extra_info); - // build pmailbox with the same version and remote_verion as cmailbox, - // when this connection is duplex. - if (gs_r_build_reply_connection(fcmsgr, local_version) != 0) { + // 如果是双工连接，则使用与cmailbox相同版本和remote_verion构建pmailbox + if (gs_r_build_reply_connection(fcmsgr, local_version) != 0) + { + // 锁定cmailbox的sinfo_lock互斥锁 LIBCOMM_PTHREAD_MUTEX_LOCK(&cmailbox->sinfo_lock); + // 关闭逻辑连接 gs_r_close_logic_connection(cmailbox, ECOMMTCPTCPDISCONNECT, NULL); + // 解锁cmailbox的sinfo_lock互斥锁 LIBCOMM_PTHREAD_MUTEX_UNLOCK(&cmailbox->sinfo_lock); goto accept_failed; } } + // 初始化gs_sock一系列属性 gs_sock.idx = node_idx; gs_sock.sid = streamid; gs_sock.ver = local_version; - - if (fcmsgr->type == CTRL_CONN_DUAL) { - // CN request this logic connection, is a dual channel + // 如果fcmsgr的类型是CTRL_CONN_DUAL，则执行下面的代码块 + if (fcmsgr->type == CTRL_CONN_DUAL) + { + // CN请求这个逻辑连接，这是一个双工通道 gs_sock.type = GSOCK_DAUL_CHANNEL; - rc = gs_send_msg_by_unix_domain((void*)&gs_sock, sizeof(gs_sock)); - if (rc <= 0) { + // 通过UNIX域套接字发送gs_sock消息 + rc = gs_send_msg_by_unix_domain((void *)&gs_sock, sizeof(gs_sock)); + if (rc <= 0) + { + // 如果发送失败，则输出警告日志，节点索引为node_idx，远程名称为g_instance.comm_cxt.g_r_node_sock[node_idx]的套接字[ctrl_socket]发送失败 LIBCOMM_ELOG(WARNING, - "(r|flow ctrl)\t fail to notify main thread from node[%d]:%s with socket[%d].", - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), - ctrl_socket); + "(r|flow ctrl)\t fail to notify main thread from node[%d]:%s with socket[%d].", + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), + ctrl_socket); + // 关闭gsocket gs_close_gsocket(&gs_sock); goto accept_failed; } } - // reply MAIL_READY message to producer + // 回复MAIL_READY消息给生产者 + // 初始化回复的消息的一系列属性 fcmsgs.type = CTRL_CONN_ACCEPT; fcmsgs.node_idx = node_idx; fcmsgs.streamid = streamid; @@ -404,6 +495,7 @@ void gs_receivers_flow_handle_ready_request(FCMSG_T* fcmsgr) fcmsgs.query_id = fcmsgr->query_id; fcmsgs.extra_info = local_version; + // 初始化回复的消息的结点名 cpylen = comm_get_cpylen(g_instance.comm_cxt.localinfo_cxt.g_self_nodename, NAMEDATALEN); ss_rc = memset_s(fcmsgs.nodename, NAMEDATALEN, 0x0, NAMEDATALEN); securec_check(ss_rc, "\0", "\0"); @@ -411,46 +503,59 @@ void gs_receivers_flow_handle_ready_request(FCMSG_T* fcmsgr) securec_check(ss_rc, "\0", "\0"); fcmsgs.nodename[cpylen] = '\0'; + // 发送控制消息 rc = gs_send_ctrl_msg(&g_instance.comm_cxt.g_r_node_sock[node_idx], &fcmsgs, ROLE_CONSUMER); - if (rc <= 0) { + // 如果发送失败，则输出警告日志，节点索引为node_idx，远程名称为g_instance.comm_cxt.g_r_node_sock[node_idx]，套接字为ctrl_socket，错误信息为mc_strerror(errno) + if (rc <= 0) + { errno = ECOMMTCPTCPDISCONNECT; LIBCOMM_ELOG(WARNING, - "(r|flow ctrl)\tFailed to send ready msg to node[%d]:%s with socket[%d]:%s.", - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), - ctrl_socket, - mc_strerror(errno)); + "(r|flow ctrl)\tFailed to send ready msg to node[%d]:%s with socket[%d]:%s.", + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), + ctrl_socket, + mc_strerror(errno)); return; } - + // 输出调试日志，成功向节点索引为node_idx，远程名称为g_instance.comm_cxt.g_r_node_sock[node_idx]的套接字ctrl_socket发送CTRL_CONN_ACCEPT消息 COMM_DEBUG_LOG("(r|flow ctrl)\tSuccess to send CTRL_CONN_ACCEPT msg to node[%d]:%s with socket[%d].", - node_idx, REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), ctrl_socket); + node_idx, REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), ctrl_socket); + // 计算处理准备消息所花费的时间，单位为微秒 deal_time = ABS_SUB((long long int)time(NULL), (long long int)gs_get_recv_ready_time()); - if ((uint64)(unsigned)mc_tcp_get_connect_timeout() < deal_time) { + // 如果连接超时时间小于处理准备消息所花费的时间，则输出警告日志 + if ((uint64)(unsigned)mc_tcp_get_connect_timeout() < deal_time) + { LIBCOMM_ELOG(WARNING, "(r|flow ctrl)\tIt takes %lus to process the ready message, query id:%lu.", - deal_time, u_sess->debug_query_id); + deal_time, u_sess->debug_query_id); } - if (fcmsgr->type == CTRL_CONN_DUAL) { + // 如果最初的请求消息的类型是CTRL_CONN_DUAL，直接结束 + if (fcmsgr->type == CTRL_CONN_DUAL) + { return; } #ifdef LIBCOMM_SPEED_TEST_ENABLE - else if (fcmsgr->stream_key.queryId == LIBCOMM_PERFORMANCE_PLAN_ID) { - ; // do nothing + else if (fcmsgr->stream_key.queryId == LIBCOMM_PERFORMANCE_PLAN_ID) + { + ; // do nothing } #endif - else { + else + { + // 断言激活消费者这个指针不为空 Assert(g_instance.comm_cxt.gs_wakeup_consumer != NULL); - // DN request this logic connection, is a single channel + // 设置gs_sock的类型为GSOCK_CONSUMER gs_sock.type = GSOCK_CONSUMER; - // wake up Conumser in executor to continue + // 初始化连接消息 ss_rc = memset_s(&connInfo, sizeof(StreamConnInfo), 0x0, sizeof(StreamConnInfo)); securec_check(ss_rc, "\0", "\0"); + // 初始化连接信息的一系列属性 connInfo.port.libcomm_layer.gsock = gs_sock; + // 初始化连接消息的结点名 cpylen = comm_get_cpylen(fcmsgr->nodename, NAMEDATALEN); ss_rc = memset_s(connInfo.nodeName, NAMEDATALEN, 0x0, NAMEDATALEN); securec_check(ss_rc, "\0", "\0"); @@ -458,41 +563,48 @@ void gs_receivers_flow_handle_ready_request(FCMSG_T* fcmsgr) securec_check(ss_rc, "\0", "\0"); connInfo.nodeName[cpylen] = '\0'; + // 记录开始时间 time_callback_start = time(NULL); + // 根据最初传送的请求信息激活消费者 bool wakeup_if = (*g_instance.comm_cxt.gs_wakeup_consumer)(fcmsgr->stream_key, connInfo); + // 记录结束时间 time_callback_end = time(NULL); - + // 计算处理回调函数所花费的时间，单位为微秒 deal_time = ABS_SUB(time_callback_end, time_callback_start); - if ((uint64)(unsigned)mc_tcp_get_connect_timeout() < deal_time) { + // 如果连接超时时间小于处理回调函数所花费的时间，则输出警告日志 + if ((uint64)(unsigned)mc_tcp_get_connect_timeout() < deal_time) + { LIBCOMM_ELOG(WARNING, "(r|flow ctrl)\tWake up consumer timeout for node[%d]:%s, it takes %lus, \ because the lock wait timeout, query id:%lu :%s.", node_idx, REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), deal_time, u_sess->debug_query_id, mc_strerror(errno)); } - - if (!wakeup_if) { + // 如果唤醒失败，则记录日志 + if (!wakeup_if) + { COMM_DEBUG_LOG("(r|flow ctrl)\tFailed to wake up consumer " "for node[%d]:%s, query maybe already quit:%s.", - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), - mc_strerror(errno)); + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), + mc_strerror(errno)); (void)gs_r_close_stream(&gs_sock); // we needn't to send Reject. return; } } + // 记录日志 COMM_TIMER_LOG("(r|flow ctrl)\tWake up consumer, node[%d, %d]:%s.", - node_idx, - streamid, - REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx)); + node_idx, + streamid, + REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx)); return; accept_failed: - // Failed to accept connction, - // reply CTRL_INIT message to producer + // 如果消费者接受就绪请求消息失败，那么给生产者回复一个CTRL_INIT消息 + // 初始化回复的消息的属性 fcmsgs.type = CTRL_CONN_REJECT; fcmsgs.node_idx = node_idx; fcmsgs.streamid = streamid; @@ -507,18 +619,21 @@ accept_failed: securec_check(ss_rc, "\0", "\0"); fcmsgs.nodename[cpylen] = '\0'; + // 向生产者发送消息 rc = gs_send_ctrl_msg(&g_instance.comm_cxt.g_r_node_sock[node_idx], &fcmsgs, ROLE_CONSUMER); - if (rc <= 0) { + if (rc <= 0) + { + // 如果发送错误则记录日志 LIBCOMM_ELOG(WARNING, - "(r|flow ctrl)\tFailed to send init msg to node[%d]:%s with socket[%d]:%s.", - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), - ctrl_socket, - mc_strerror(errno)); + "(r|flow ctrl)\tFailed to send init msg to node[%d]:%s with socket[%d]:%s.", + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), + ctrl_socket, + mc_strerror(errno)); } return; -} // gs_receivers_flow_handle_ready_request +} // gs_receivers_flow_handle_ready_request /* * function name : gs_receivers_flow_handle_close_request @@ -527,36 +642,40 @@ accept_failed: * Producer closed the stream, which indicate error happened when sender is sending data * if it is a Closed message, we should close local mailbox and tell the thread of executor to quit */ -void gs_receivers_flow_handle_close_request(FCMSG_T* fcmsgr) +// 此函数的作用是在接收端接收到MAIL_CLOSED消息时处理关闭请求。 +void gs_receivers_flow_handle_close_request(FCMSG_T *fcmsgr) { + // 从传入的信息中获得流ID与结点索引 int streamid = fcmsgr->streamid; int node_idx = fcmsgr->node_idx; - struct c_mailbox* cmailbox = NULL; + // 创建一个消费者邮箱 + struct c_mailbox *cmailbox = NULL; - // set node index and get stream index from hash table (g_r_htab_nodeid_skey_to_stream) - // + // 初始化消费者邮箱实例 cmailbox = &(C_MAILBOX(node_idx, streamid)); + // 将消费者邮箱上锁 LIBCOMM_PTHREAD_MUTEX_LOCK(&(cmailbox->sinfo_lock)); - // If the stream is closed already, may because of tcp error or upper consumer closed actively, - // we just try to delete the entry in hash table(g_r_htab_nodeid_skey_to_stream) - // - if (false == gs_check_mailbox(cmailbox->local_version, fcmsgr->version)) { + // 如果发送的请求的版本号和邮箱的版本号不匹配，那么直接解锁邮箱并且结束 + if (false == gs_check_mailbox(cmailbox->local_version, fcmsgr->version)) + { LIBCOMM_PTHREAD_MUTEX_UNLOCK(&(cmailbox->sinfo_lock)); return; } + // 记录日志 COMM_DEBUG_LOG("(r|flow ctrl)\tStream[%d] is closed " "by remote node[%d]:%s, query[%lu].", - streamid, - node_idx, - g_instance.comm_cxt.g_r_node_sock[node_idx].remote_nodename, - fcmsgr->query_id); - + streamid, + node_idx, + g_instance.comm_cxt.g_r_node_sock[node_idx].remote_nodename, + fcmsgr->query_id); + // 使用函数关闭逻辑连接 gs_r_close_logic_connection(cmailbox, ECOMMTCPREMOETECLOSE, NULL); + // 将消费者邮箱解锁 LIBCOMM_PTHREAD_MUTEX_UNLOCK(&(cmailbox->sinfo_lock)); return; -} // gs_receivers_flow_handle_close_request +} // gs_receivers_flow_handle_close_request /* * function name : gs_receivers_flow_handle_assert_fail_request @@ -566,24 +685,27 @@ void gs_receivers_flow_handle_close_request(FCMSG_T* fcmsgr) * if the sender assert fail and core, we will get a CTRL_ASSERT_FAIL message, * we should make the consumer core as well */ -void gs_receivers_flow_handle_assert_fail_request(FCMSG_T* fcmsgr) +// 此函数的作用是用于处理接收线程收到CTRL_ASSERT_FAIL时的断言失败情况 +void gs_receivers_flow_handle_assert_fail_request(FCMSG_T *fcmsgr) { - + // 从传入的信息中获得流ID与结点索引 int sidx = fcmsgr->streamid; int nidx = fcmsgr->node_idx; - - struct c_mailbox* cmailbox = NULL; - + // 创建消费者邮箱 + struct c_mailbox *cmailbox = NULL; + // 通过节点索引和流ID获取对应的c_mailbox实例，并赋值给cmailbox指针 cmailbox = &(C_MAILBOX(nidx, sidx)); + // 当遇到断言失败的情况，只需记录错误日志即可 LIBCOMM_ELOG(WARNING, - "(r|flow ctrl)\tNode[%d] stream[%d] assert fail, node name[%s] with state[%d] has bufCAP[%lu] and " - "buff_q->u_size[%lu].", - nidx, - sidx, - g_instance.comm_cxt.g_r_node_sock[nidx].remote_nodename, - cmailbox->state, - cmailbox->bufCAP, - cmailbox->buff_q->u_size); + "(r|flow ctrl)\tNode[%d] stream[%d] assert fail, node name[%s] with state[%d] has bufCAP[%lu] and " + "buff_q->u_size[%lu].", + nidx, + sidx, + g_instance.comm_cxt.g_r_node_sock[nidx].remote_nodename, + cmailbox->state, + cmailbox->bufCAP, + cmailbox->buff_q->u_size); + // 记录邮箱的错误日志 MAILBOX_ELOG(cmailbox, WARNING, "(r|flow ctrl)\tMailbox Info which assert fail."); Assert(0 != 0); @@ -595,39 +717,44 @@ void gs_receivers_flow_handle_assert_fail_request(FCMSG_T* fcmsgr) * arguments : _in_ fcmsgr: the message that senders_flow thread received. * return value : void */ -void gs_senders_flow_handle_tid_request(FCMSG_T* fcmsgr) +// 此函数的作用是用于处理发送线程收到CTRL_PEER_TID时保存对端线程ID的情况 +void gs_senders_flow_handle_tid_request(FCMSG_T *fcmsgr) { + // 获取消息的流ID，结点索引与版本号 int streamid = fcmsgr->streamid; int node_idx = fcmsgr->node_idx; int version = fcmsgr->version; - struct p_mailbox* pmailbox = NULL; + // 初始化生产者邮箱 + struct p_mailbox *pmailbox = NULL; pmailbox = &P_MAILBOX(node_idx, streamid); + // 将生产者邮箱上锁 LIBCOMM_PTHREAD_MUTEX_LOCK(&pmailbox->sinfo_lock); - // 1: check the pmailbox[idx][streamid] is correct - // - // check stream key - if (gs_check_mailbox(pmailbox->local_version, version) == false) { + // 检查pmailbox[idx][streamid]是否正确，即检查流密钥 + // 如果检查错误则直接记录邮箱的错误日志，并且解锁 + if (gs_check_mailbox(pmailbox->local_version, version) == false) + { MAILBOX_ELOG(pmailbox, WARNING, "(s|flow ctrl)\tStream has already closed."); LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); return; } - // 2: set the new quota and peer thread id to the pmailbox - // + // 设置新的配额和对端线程ID到pmailbox中 pmailbox->peer_thread_id = fcmsgr->extra_info; pmailbox->bufCAP += fcmsgr->streamcap; - // 3: set the new state to the pmailbox, then tell the executor thread who may be waiting in gs_send to continue - // - if (pmailbox->state == MAIL_HOLD) { + // 设置新的状态到pmailbox中，然后告诉可能在gs_send中等待的执行线程继续执行 + if (pmailbox->state == MAIL_HOLD) + { gs_poll_signal(pmailbox->semaphore); } + // 记录日志 COMM_DEBUG_LOG("(s|flow ctrl)\tWake up mailbox[%d][%d], node[%s], bufCAP[%lu] add[%ld] type[%s].", - node_idx, - streamid, - g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, - pmailbox->bufCAP, - fcmsgr->streamcap, - stream_stat_string(pmailbox->state)); + node_idx, + streamid, + g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, + pmailbox->bufCAP, + fcmsgr->streamcap, + stream_stat_string(pmailbox->state)); + // 将消费者邮箱解锁 LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); } @@ -639,20 +766,27 @@ void gs_senders_flow_handle_tid_request(FCMSG_T* fcmsgr) * if the receiver failed to get an usable stream index, we will get a CTRL_INIT message, * we should tell the executor thread to exit and report the error. */ -void gs_senders_flow_handle_init_request(FCMSG_T* fcmsgr) +// 此函数的作用是在发送端收到一个CTRL_INIT时，处理这个初始化请求 +void gs_senders_flow_handle_init_request(FCMSG_T *fcmsgr) { + // 从收到的消息中获取流ID和结点索引 int streamid = fcmsgr->streamid; int node_idx = fcmsgr->node_idx; - struct p_mailbox* pmailbox = NULL; + // 创建并且初始化一个生产者邮箱 + struct p_mailbox *pmailbox = NULL; pmailbox = &P_MAILBOX(node_idx, streamid); + // 将生产者邮箱上锁 LIBCOMM_PTHREAD_MUTEX_LOCK(&pmailbox->sinfo_lock); - if (gs_check_mailbox(pmailbox->local_version, fcmsgr->version) == false) { + // 如果邮箱的版本和消息的版本不匹配，那么直接解锁并且结束 + if (gs_check_mailbox(pmailbox->local_version, fcmsgr->version) == false) + { LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); return; } - // close logic conneion and sigal producer + // 关闭逻辑连接并且告诉消费者 gs_s_close_logic_connection(pmailbox, ECOMMTCPREJECTSTREAM, NULL); + // 将生产者解锁 LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); } @@ -664,46 +798,60 @@ void gs_senders_flow_handle_init_request(FCMSG_T* fcmsgr) * if the receiver succeed to get an usable stream index, we will get a MAIL_READY message, * we should change the state of pmailbox and tell the executor thread to continue */ -void gs_senders_flow_handle_ready_request(FCMSG_T* fcmsgr) +// 此函数的作用是当发送端接收到MAIL_READY时，处理这个就绪请求 +void gs_senders_flow_handle_ready_request(FCMSG_T *fcmsgr) { + // 从接收到的消息中获取流ID和结点索引 int streamid = fcmsgr->streamid; int node_idx = fcmsgr->node_idx; - struct p_mailbox* pmailbox = NULL; + // 初始化生产者邮箱 + struct p_mailbox *pmailbox = NULL; uint64 time_now = 0; // 1: check the pmailbox state is CLOSED, we will retry for 10 times + // 检查生产者邮箱的状态是不是关闭的，循环检查10次 pmailbox = &P_MAILBOX(node_idx, streamid); + // 将生产者邮箱上锁 LIBCOMM_PTHREAD_MUTEX_LOCK(&pmailbox->sinfo_lock); - if (gs_check_mailbox(pmailbox->local_version, fcmsgr->version) == false) { + // 如果邮箱的版本和消息的版本不匹配，那么直接解锁并且结束 + if (gs_check_mailbox(pmailbox->local_version, fcmsgr->version) == false) + { + LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); + return; + } + // 如果邮箱的状态不是就绪态，那么直接解锁然后结束 + if (pmailbox->state != MAIL_READY) + { LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); return; } - if (pmailbox->state != MAIL_READY) { - LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); - return; - } - - // 2: change the state of the mailbox + // 改变邮箱的状态为运行态 pmailbox->state = MAIL_RUN; + // 增加pmailbox的bufCAP值，即增加流容量 pmailbox->bufCAP += fcmsgr->streamcap; - // consumer send cmailbox version as fcmsgr->extra_info, - // now save it to pmailbox->remote_version. + // 将传入的消息中的extra_info字段赋值给pmailbox的remote_version字段，表示远程版本信息 pmailbox->remote_version = (uint16)(fcmsgr->extra_info); + // 打印调试日志，说明当前节点和流的状态以及流容量等信息 COMM_DEBUG_LOG("(s|flow ctrl)\tnode[%d] stream[%d] is in state[%s], node name[%s], bufCAP[%lu].", - node_idx, - streamid, - stream_stat_string(pmailbox->state), - g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, - pmailbox->bufCAP); + node_idx, + streamid, + stream_stat_string(pmailbox->state), + g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, + pmailbox->bufCAP); - // 3: signal the producer which should be waiting in gs_connect + // 发送信号给pmailbox的信号量，唤醒等待在该信号量上的线程，告诉它们可以继续执行 gs_poll_signal(pmailbox->semaphore); - if (pmailbox->statistic != NULL) { + // 如果pmailbox的统计信息不为空 + if (pmailbox->statistic != NULL) + { + // 获取当前时间 time_now = COMM_STAT_TIME(); + // 计算连接时间并存储在pmailbox的统计信息中 pmailbox->statistic->connect_time = ABS_SUB(time_now, pmailbox->statistic->connect_time); } + // 将邮箱解锁 LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); } @@ -715,39 +863,47 @@ void gs_senders_flow_handle_ready_request(FCMSG_T* fcmsgr) * if the receiver have enough buffer to continue receive, we will get a MAIL_RUN message, * we should change the state of pmailbox and tell the executor thread to continue */ -void gs_senders_flow_handle_resume_request(FCMSG_T* fcmsgr) +// 此函数的作用是处理发送线程收到MAIL_RUN请求时的情况 +void gs_senders_flow_handle_resume_request(FCMSG_T *fcmsgr) { + // 从请求中获取流ID和结点索引 int streamid = fcmsgr->streamid; int node_idx = fcmsgr->node_idx; - struct p_mailbox* pmailbox = NULL; + // 初始化生产者邮箱 + struct p_mailbox *pmailbox = NULL; // 1: check the pmailbox[idx][fcmsgr.streamid] is correct, if not wait a few seconds and retry // pmailbox = &P_MAILBOX(node_idx, streamid); + // 锁定一个互斥锁，用于保护共享数据，避免同时被多个线程访问导致数据不一致 LIBCOMM_PTHREAD_MUTEX_LOCK(&pmailbox->sinfo_lock); - if (gs_check_mailbox(pmailbox->local_version, fcmsgr->version) == - false) { // if the pmailbox is not matched, we will break here and report error - if (pmailbox->state != MAIL_CLOSED) { + // 如果检查失败，说明流已经关闭或者版本不匹配 + if (gs_check_mailbox(pmailbox->local_version, fcmsgr->version) == false) + { + // 如果邮箱不匹配，将邮箱解锁并且报告错误 + if (pmailbox->state != MAIL_CLOSED) + { MAILBOX_ELOG(pmailbox, WARNING, "(s|flow ctrl)\tStream not work."); } LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); return; } - // 2: set the new quota and state to the pmailbox - // + // 增加pmailbox的bufCAP值，即增加流容量 pmailbox->bufCAP += fcmsgr->streamcap; COMM_DEBUG_LOG("(s|flow ctrl)\tWake up node[%d] stream[%d], node name[%s], bufCAP[%lu] add[%ld].", - node_idx, - streamid, - g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, - pmailbox->bufCAP, - fcmsgr->streamcap); - // 3: tell the executor thread who may be waiting in gs_send to continue - // - if (pmailbox->state == MAIL_HOLD) { + node_idx, + streamid, + g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, + pmailbox->bufCAP, + fcmsgr->streamcap); + // 告知执行线程等待的可以被执行了 + if (pmailbox->state == MAIL_HOLD) + { + // 从生产者邮箱发送信号 gs_poll_signal(pmailbox->semaphore); } + // 解锁邮箱 LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); } @@ -759,26 +915,31 @@ void gs_senders_flow_handle_resume_request(FCMSG_T* fcmsgr) * if the receiver quit, we will get a MAIL_CLOSED message, * we should close the pmailbox and tell the executor thread to continue */ -void gs_senders_flow_handle_close_request(FCMSG_T* fcmsgr) +// 此函数的作用是处理发送线程收到MAIL_CLOSED请求时的情况 +void gs_senders_flow_handle_close_request(FCMSG_T *fcmsgr) { + // 从请求中获取流ID和结点索引 int streamid = fcmsgr->streamid; int node_idx = fcmsgr->node_idx; - struct p_mailbox* pmailbox = NULL; + // 初始化生产者邮箱 + struct p_mailbox *pmailbox = NULL; // 1: get the pmailbox[idx][fcmsgr.streamid] and check it is in correct state // pmailbox = &P_MAILBOX(node_idx, streamid); + // 锁定一个互斥锁，用于保护共享数据，避免同时被多个线程访问导致数据不一致 LIBCOMM_PTHREAD_MUTEX_LOCK(&pmailbox->sinfo_lock); + // 如果检查失败，说明流已经关闭或者版本不匹配 COMM_DEBUG_LOG("(s|flow ctrl)\tStream[%d] is closed " "by remote node[%d]:%s, query[%lu].", - streamid, - node_idx, - g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, - fcmsgr->query_id); + streamid, + node_idx, + g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, + fcmsgr->query_id); - // 2: if the pmailbox is not matched, we will break here and report error - // - if (gs_check_mailbox(pmailbox->local_version, fcmsgr->version) == false) { + // 如果检查失败，说明流已经关闭或者版本不匹配，将邮箱解锁并且报告错误 + if (gs_check_mailbox(pmailbox->local_version, fcmsgr->version) == false) + { LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); return; } @@ -786,7 +947,9 @@ void gs_senders_flow_handle_close_request(FCMSG_T* fcmsgr) // 3: if gs_connect already return correctly, delete the record in the hash table, // or, the entry will be deleted in gs_connect // + // 直接关闭逻辑连接 gs_s_close_logic_connection(pmailbox, ECOMMTCPREMOETECLOSE, NULL); + // 解锁邮箱 LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); } @@ -798,59 +961,70 @@ void gs_senders_flow_handle_close_request(FCMSG_T* fcmsgr) * if the receiver assert fail and core, we will get a CTRL_ASSERT_FAIL message, * we should make the producer core as well */ -void gs_senders_flow_handle_assert_fail_request(FCMSG_T* fcmsgr) +// 此函数的作用是处理发送线程收到CTRL_ASSERT_FAIL请求时的情况 +void gs_senders_flow_handle_assert_fail_request(FCMSG_T *fcmsgr) { + // 从请求中获取流ID和结点索引 int sidx = fcmsgr->streamid; int nidx = fcmsgr->node_idx; - - struct p_mailbox* pmailbox = NULL; + // 初始化生产者邮箱 + struct p_mailbox *pmailbox = NULL; pmailbox = &(P_MAILBOX(nidx, sidx)); - + // 记录报错日志 LIBCOMM_ELOG(WARNING, - "(s|flow ctrl)\tNode[%d] stream[%d] assert fail, node name[%s] with state[%d] has bufCAP[%lu].", - nidx, - sidx, - g_instance.comm_cxt.g_s_node_sock[nidx].remote_nodename, - pmailbox->state, - pmailbox->bufCAP); + "(s|flow ctrl)\tNode[%d] stream[%d] assert fail, node name[%s] with state[%d] has bufCAP[%lu].", + nidx, + sidx, + g_instance.comm_cxt.g_s_node_sock[nidx].remote_nodename, + pmailbox->state, + pmailbox->bufCAP); + // 收到这种失败时只需要记录邮箱日志即可 MAILBOX_ELOG(pmailbox, WARNING, "(s|flow ctrl)\tMailbox Info which assert fail."); Assert(0 != 0); } extern ThreadId getThreadIdForLibcomm(int logictid); -void gs_senders_flow_handle_stop_query_request(FCMSG_T* fcmsgr) +// 此函数的作用是处理发送线程收到stop_query_request时的情况 +void gs_senders_flow_handle_stop_query_request(FCMSG_T *fcmsgr) { + // 从请求中获取流ID和结点索引 int streamid = fcmsgr->streamid; int node_idx = fcmsgr->node_idx; - struct p_mailbox* pmailbox = NULL; + // 初始化生产者邮箱 + struct p_mailbox *pmailbox = NULL; // 1: get the pmailbox[idx][fcmsgr.streamid] and check it is in correct state // pmailbox = &P_MAILBOX(node_idx, streamid); + // 将邮箱上锁 LIBCOMM_PTHREAD_MUTEX_LOCK(&pmailbox->sinfo_lock); + // 记录日志 COMM_DEBUG_LOG("(s|flow ctrl)\tQuery[%lu] is stop " "by remote node[%d]:%s with stream[%d].", - fcmsgr->query_id, - node_idx, - g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, - streamid); + fcmsgr->query_id, + node_idx, + g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, + streamid); - // 2: if the pmailbox is not matched, we will break here and report error - // - if (gs_check_mailbox(pmailbox->local_version, fcmsgr->version) == false) { + // 如果检查失败，说明流已经关闭或者版本不匹配，将邮箱解锁并且报告错误 + if (gs_check_mailbox(pmailbox->local_version, fcmsgr->version) == false) + { LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); return; } // 3: send SIGUSR1 to backend, then set the stop flag to stop query - // + // 向后端发送一个用户指令，设置一个停止标志以停止询问 int logictid = (int)ntohl(fcmsgr->extra_info); + // 获得一个后端线程ID ThreadId backendTID = getThreadIdForLibcomm(logictid); - if (0 != backendTID) { + // 如果ID不是0，则停止询问 + if (0 != backendTID) + { StreamNodeGroup::stopAllThreadInNodeGroup(backendTID, fcmsgr->query_id); } + // 解锁邮箱 LIBCOMM_PTHREAD_MUTEX_UNLOCK(&pmailbox->sinfo_lock); -} - +} \ No newline at end of file -- 2.34.1 From b3f0eeb4ef399d495ba17f48aaf922b404b7c191 Mon Sep 17 00:00:00 2001 From: richard_chen Date: Wed, 27 Sep 2023 21:45:31 +0800 Subject: [PATCH 08/15] =?UTF-8?q?=E5=AF=B9libcomm=5Fmemory.cpp=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E4=B8=AD=E7=9A=84=E5=87=BD=E6=95=B0=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E4=BA=86=E6=B3=A8=E9=87=8A=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../libcomm_utils/libcomm_memory.cpp | 510 ++++++++++++------ 1 file changed, 337 insertions(+), 173 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_memory.cpp b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_memory.cpp index 388c88e83..e436a4052 100644 --- a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_memory.cpp +++ b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_memory.cpp @@ -65,7 +65,6 @@ #define static #endif - #define STREAM_SCAN_FINISH 'F' #define STREAM_SCAN_WAIT 'W' #define STREAM_SCAN_DATA 'D' @@ -73,9 +72,12 @@ extern bool executorEarlyStop(); /* release memory of communication layer, just for LLT */ +// 此函数的功能是释放通信层的内存 int gs_release_comm_memory() { + // 构造一个通信上下文的对象，切换到全局通信内存上下文，用于管理通信内存。 AutoContextSwitch commContext(g_instance.comm_cxt.comm_global_mem_cxt); + // 调用函数gs_r_release_comm_memory来释放通信内存。 gs_r_release_comm_memory(); return 0; } @@ -85,27 +87,41 @@ int gs_release_comm_memory() * * @param[IN] key_s: stream key */ -void gs_memory_init_entry(StreamSharedContext* sharedContext, int consumerNum, int producerNum) +// 此函数用于初始化一个关于流和内存使用情况的哈希表条目 +void gs_memory_init_entry(StreamSharedContext *sharedContext, int consumerNum, int producerNum) { - struct hash_entry* entry = NULL; - struct hash_entry** poll_entrys = NULL; - struct hash_entry*** quota_entrys = NULL; + // 定义一个指向哈希表条目的指针 + struct hash_entry *entry = NULL; + // 定义一个指向哈希表条目的指针的指针，用于存储与消费者相关的条目 + struct hash_entry **poll_entrys = NULL; + // 定义一个指向哈希表条目的指针的指针的指针，用于存储与生产者相关的条目 + struct hash_entry ***quota_entrys = NULL; - poll_entrys = (struct hash_entry**)palloc(sizeof(struct hash_entry*) * consumerNum); - quota_entrys = (struct hash_entry***)palloc(sizeof(struct hash_entry**) * consumerNum); - - for (int i = 0; i < consumerNum; i++) { - entry = (struct hash_entry*)palloc(sizeof(struct hash_entry)); + // 为每个消费者分配内存以存储哈希表条目指针 + poll_entrys = (struct hash_entry **)palloc(sizeof(struct hash_entry *) * consumerNum); + // 为每个消费者分配内存以存储生产者相关的哈希表条目指针 + quota_entrys = (struct hash_entry ***)palloc(sizeof(struct hash_entry **) * consumerNum); + // 遍历所有消费者 + for (int i = 0; i < consumerNum; i++) + { + // 为每个消费者分配一个哈希表条目，并初始化它 + entry = (struct hash_entry *)palloc(sizeof(struct hash_entry)); (void)entry->_init(); + // 将当前消费者的哈希表条目存储在poll_entrys中 poll_entrys[i] = entry; - quota_entrys[i] = (struct hash_entry**)palloc(sizeof(struct hash_entry*) * producerNum); - for (int j = 0; j < producerNum; j++) { - entry = (struct hash_entry*)palloc(sizeof(struct hash_entry)); + // 为每个生产者分配内存以存储哈希表条目指针 + quota_entrys[i] = (struct hash_entry **)palloc(sizeof(struct hash_entry *) * producerNum); + // 遍历所有的生产者 + for (int j = 0; j < producerNum; j++) + { + // 为每个生产者分配一个哈希表条目，并初始化它 + entry = (struct hash_entry *)palloc(sizeof(struct hash_entry)); (void)entry->_init(); + // 将当前生产者的哈希表条目存储在quota_entrys中 quota_entrys[i][j] = entry; } } - + // 将poll_entrys和quota_entrys存储在共享上下文中，以便后续使用 sharedContext->poll_entrys = poll_entrys; sharedContext->quota_entrys = quota_entrys; } @@ -117,38 +133,51 @@ void gs_memory_init_entry(StreamSharedContext* sharedContext, int consumerNum, i * @param[IN] sharedContext: context for shared memory stream * @param[IN] nthChannel: destination consumer */ -void gs_message_by_memory(StringInfo buf, StreamSharedContext* sharedContext, int nthChannel) +// 此函数的作用是通过内存发送错误/通知消息，其中参数buf为错误或者通知的字符串，sharedContext为共享内存流上下文，nthChannel为目标消费者 +void gs_message_by_memory(StringInfo buf, StreamSharedContext *sharedContext, int nthChannel) { + // 目标缓冲区，用于存储要发送的消息 StringInfo buf_dst = NULL; - struct hash_entry* entry = NULL; + // 哈希表条目，用于管理共享内存流 + struct hash_entry *entry = NULL; - /* Copy Error/Notice messages to shared context. */ + // 将错误/通知消息复制到共享上下文中 buf_dst = sharedContext->messages[nthChannel][u_sess->stream_cxt.smp_id]; - /* - * If producer is waked up and shared buffer has been consumed while waiting, - * it can continue to append data to its messages of sharedContext. - */ + // 如果生产者在等待期间被唤醒，并且共享缓冲区已被消耗，它可以继续将其数据追加到sharedContext的消息中 entry = sharedContext->quota_entrys[nthChannel][u_sess->stream_cxt.smp_id]; - while (buf_dst->len > 0) { + // 当目标缓冲区中还有未处理的数据时 + while (buf_dst->len > 0) + { + // 等待一段时间，直到可以继续处理数据 (void)entry->_timewait(SINGLE_WAITQUOTA); } + // 将源缓冲区的数据追加到目标缓冲区中 appendBinaryStringInfo(buf_dst, buf->data, buf->len); + // 更新目标缓冲区的游标位置 buf_dst->cursor = buf->cursor; - /* Send signal to dest consumer. */ + // 向目标消费者发送信号。 entry = sharedContext->poll_entrys[nthChannel]; + // 发送信号通知目标消费者有新消息到达 entry->_signal(); + // 释放源缓冲区的数据内存 pfree(buf->data); + // 将源缓冲区的数据指针置为NULL，避免悬挂指针 buf->data = NULL; } -void gs_memory_disconnect(StreamSharedContext* sharedContext, int nthChannel) +// 此函数的作用是断开内存连接 +void gs_memory_disconnect(StreamSharedContext *sharedContext, int nthChannel) { - struct hash_entry* entry = NULL; + // 定义一个指向哈希表条目的指针 + struct hash_entry *entry = NULL; + // 将指定通道的数据状态设置为连接错误 sharedContext->dataStatus[nthChannel][u_sess->stream_cxt.smp_id] = CONN_ERR; + // 获取指定通道的轮询条目 entry = sharedContext->poll_entrys[nthChannel]; + // 向轮询条目发送信号，通常用于通知其他进程或线程发生了某种事件或状态变化 entry->_signal(); } @@ -159,16 +188,27 @@ void gs_memory_disconnect(StreamSharedContext* sharedContext, int nthChannel) * @param[IN] sharedContext: context for shared memory stream * @param[IN] nthChannel: destination consumer */ -bool gs_is_databuff_empty(StreamSharedContext* sharedContext, int nthChannel) +// 此函数的作用是判断数据缓冲区是否为空，其中参数sharedContext为共享内存流上下文，nthChannel为目标消费者 +bool gs_is_databuff_empty(StreamSharedContext *sharedContext, int nthChannel) { - if (sharedContext->vectorized) { - VectorBatch* batch = sharedContext->sharedBatches[nthChannel][u_sess->stream_cxt.smp_id]; - if (batch->m_rows == 0) { + // 判断是否启用了向量化处理 + if (sharedContext->vectorized) + { + // 获取指定通道和会话的共享批处理对象 + VectorBatch *batch = sharedContext->sharedBatches[nthChannel][u_sess->stream_cxt.smp_id]; + // 如果批处理的行数为0，则缓冲区为空 + if (batch->m_rows == 0) + { return true; } - } else { - TupleVector* tupleVec = sharedContext->sharedTuples[nthChannel][u_sess->stream_cxt.smp_id]; - if (tupleVec->tuplePointer == 0) { + } + else + { + // 获取指定通道和会话的共享元组向量对象 + TupleVector *tupleVec = sharedContext->sharedTuples[nthChannel][u_sess->stream_cxt.smp_id]; + // 如果元组指针为0，则缓冲区为空 + if (tupleVec->tuplePointer == 0) + { return true; } } @@ -185,94 +225,134 @@ bool gs_is_databuff_empty(StreamSharedContext* sharedContext, int nthChannel) * @param[IN] nthChannel: destination consumer * @param[IN] nthRow: the Nth row to be sent in batch */ +// 此函数的作用是通过共享内存向本地消费者发送消息，参数tuple为要发送的元组，batchsrc为要发送的批次，sharedContext为共享内存流上下文，nthChannel为目标消费者，nthRow为批次中要发送的第n行 void gs_memory_send( - TupleTableSlot* tuple, VectorBatch* batchsrc, StreamSharedContext* sharedContext, int nthChannel, int nthRow) + TupleTableSlot *tuple, VectorBatch *batchsrc, StreamSharedContext *sharedContext, int nthChannel, int nthRow) { - VectorBatch* batch = NULL; - TupleVector* tupleVec = NULL; + // 定义一个批处理指针，用于存储批处理对象 + VectorBatch *batch = NULL; + // 定义一个元组向量指针，用于存储元组向量对象 + TupleVector *tupleVec = NULL; + // 定义一个布尔型变量，用于存储是否可以发送的状态 bool ready_to_send = false; + // 定义一个数据状态变量，用于存储数据的状态 DataStatus dataStatus; - struct hash_entry* entry = NULL; - + // 定义一个哈希表条目指针，用于存储共享内存流上下文的哈希表条目 + struct hash_entry *entry = NULL; + // 报告等待状态，将当前状态设置为等待刷新数据状态 WaitState oldStatus = pgstat_report_waitstatus_comm(STATE_WAIT_FLUSH_DATA, - u_sess->pgxc_cxt.PGXCNodeId, - -1, - u_sess->stream_cxt.producer_obj->getParentPlanNodeId(), - global_node_definition ? global_node_definition->num_nodes : -1); - + u_sess->pgxc_cxt.PGXCNodeId, + -1, + u_sess->stream_cxt.producer_obj->getParentPlanNodeId(), + global_node_definition ? global_node_definition->num_nodes : -1); + // 记录时间，开始发送数据 StreamTimeSendStart(t_thrd.pgxc_cxt.GlobalNetInstr); + // 获取指定通道和会话的共享内存流上下文的哈希表条目 entry = sharedContext->quota_entrys[nthChannel][u_sess->stream_cxt.smp_id]; - for (;;) { - /* Check for interrupt at the beginning of the loop. */ + // 进入无限循环，直到发送完成或发生中断等条件退出循环 + for (;;) + { + // 在循环开始处检查中断。如果发生中断，则立即退出循环 CHECK_FOR_INTERRUPTS(); - /* Check if we should early stop. */ - /* Quit if the connection close, especially in a early close case. */ - if (executorEarlyStop() || sharedContext->is_connect_end[nthChannel][u_sess->stream_cxt.smp_id]) { + // 检查是否需要提前停止。如果连接关闭，特别是在提前关闭的情况下，则退出循环 + if (executorEarlyStop() || sharedContext->is_connect_end[nthChannel][u_sess->stream_cxt.smp_id]) + { + // 恢复等待状态为原始状态 (void)pgstat_report_waitstatus(oldStatus); return; } - + // 获取指定通道和会话的数据状态 dataStatus = sharedContext->dataStatus[nthChannel][u_sess->stream_cxt.smp_id]; - /* Break the loop if we find quota. */ + // 如果数据状态为DATA_EMPTY且（在__aarch64__架构下，数据缓冲区为空），或者数据状态为DATA_PREPARE，则跳出循环 if ((dataStatus == DATA_EMPTY #ifdef __aarch64__ && gs_is_databuff_empty(sharedContext, nthChannel) #endif - ) || - dataStatus == DATA_PREPARE) { + ) || + dataStatus == DATA_PREPARE) + { break; } - + // 记录时间，开始等待配额 StreamTimeWaitQuotaStart(t_thrd.pgxc_cxt.GlobalNetInstr); + // 调用entry的_timewait方法，传入SINGLE_WAITQUOTA作为参数，等待配额 (void)entry->_timewait(SINGLE_WAITQUOTA); + // 记录时间，结束等待配额 StreamTimeWaitQuotaEnd(t_thrd.pgxc_cxt.GlobalNetInstr); } - + // 记录时间，开始复制数据 StreamTimeCopyStart(t_thrd.pgxc_cxt.GlobalNetInstr); - /* Copy data to shared context. */ - if (sharedContext->vectorized) { + // 将数据复制到共享上下文 + if (sharedContext->vectorized) + { + // 如果启用了向量化处理，则断言共享批处理对象不为空 Assert(sharedContext->sharedBatches != NULL); + // 获取指定通道和会话的共享批处理对象 batch = sharedContext->sharedBatches[nthChannel][u_sess->stream_cxt.smp_id]; - /* data copy */ - if (-1 == nthRow) { - /* Do deep copy of all rows, for local roundrobin & local broadcast. */ + // 如果nthRow为-1，则对所有行进行深度复制，用于本地循环和本地广播 + if (-1 == nthRow) + { + // Assert批处理的行数为0，因为要进行所有行的深度复制 Assert(batch->m_rows == 0); + // 进行深度复制 batch->Copy(batchsrc); + // 设置可以发送的状态为true ready_to_send = true; - } else { + } + else + { + // 复制指定行的数据 batch->CopyNth(batchsrc, nthRow); - if (BatchMaxSize == batch->m_rows) { + // 如果批处理的行数等于BatchMaxSize，则设置可以发送的状态为true + if (BatchMaxSize == batch->m_rows) + { ready_to_send = true; } } - } else { + } + else + { + // 如果未启用向量化处理，则断言共享元组向量对象不为空 Assert(sharedContext->sharedTuples != NULL); + // 获取指定通道和会话的共享元组向量对象 tupleVec = sharedContext->sharedTuples[nthChannel][u_sess->stream_cxt.smp_id]; + // 获取元组指针 int n = tupleVec->tuplePointer; + // 复制元组到元组向量 ExecCopySlot(tupleVec->tupleVector[n], tuple); + // 元组指针加1 tupleVec->tuplePointer++; - if (TupleVectorMaxSize == tupleVec->tuplePointer) { + // 如果元组指针等于TupleVectorMaxSize，则设置可以发送的状态为true + if (TupleVectorMaxSize == tupleVec->tuplePointer) + { ready_to_send = true; } } + // 记录时间，结束复制数据 StreamTimeCopyEnd(t_thrd.pgxc_cxt.GlobalNetInstr); - /* send the signal if copy finished */ - if (ready_to_send) { + // 如果数据已经准备好发送，则执行以下代码块 + if (ready_to_send) + { #ifdef __aarch64__ + // 在__aarch64__架构下，执行内存屏障操作，确保内存操作的正确顺序 pg_memory_barrier(); #endif - /* set flag */ + // 设置数据状态为DATA_READY，表示数据已经准备好 sharedContext->dataStatus[nthChannel][u_sess->stream_cxt.smp_id] = DATA_READY; /* send signal */ entry = sharedContext->poll_entrys[nthChannel]; entry->_signal(); - } else { + } + else + { + // 如果数据还没有准备好，则将数据状态设置为DATA_PREPARE，表示数据正在准备中 sharedContext->dataStatus[nthChannel][u_sess->stream_cxt.smp_id] = DATA_PREPARE; } + // 记录时间，结束发送数据 StreamTimeSendEnd(t_thrd.pgxc_cxt.GlobalNetInstr); - + // 恢复等待状态为原始状态 (void)pgstat_report_waitstatus(oldStatus); } @@ -282,17 +362,22 @@ void gs_memory_send( * @param[IN] node: stream state * @return bool: true -- found data */ +// 此函数用于从流状态的缓冲区中获取一个元组，参数node为流状态 FORCE_INLINE -bool gs_return_tuple(StreamState* node) +bool gs_return_tuple(StreamState *node) { - TupleVector* tupleVec = node->tempTupleVec; - - if (tupleVec->tuplePointer == 0) { + // 获取流状态的临时元组向量 + TupleVector *tupleVec = node->tempTupleVec; + // 如果元组指针为0，表示没有数据可返回 + if (tupleVec->tuplePointer == 0) + { return false; } - + // 元组指针减1，因为我们要返回的是当前指针指向的元组 tupleVec->tuplePointer--; + // 获取当前元组指针的索引 int n = tupleVec->tuplePointer; + // 将结果元组槽设置为当前元组指针指向的元组 node->ss.ps.ps_ResultTupleSlot = tupleVec->tupleVector[n]; return true; @@ -305,53 +390,67 @@ bool gs_return_tuple(StreamState* node) * @param[IN] loc: data location * @return bool: true -- found data */ -bool gs_consume_memory_data(StreamState* node, int loc) +// 此函数的作用是从共享内存中消费本地生产者的数据，参数node为流状态，loc为数据位置 +bool gs_consume_memory_data(StreamState *node, int loc) { - StreamSharedContext* sharedContext = node->sharedContext; - + // 获取流状态的共享上下文 + StreamSharedContext *sharedContext = node->sharedContext; + // 记录时间，开始网络工作时间拷贝 NetWorkTimeCopyStart(t_thrd.pgxc_cxt.GlobalNetInstr); - /* Take data from the shared context. */ - if (sharedContext->vectorized) { - VectorBatch* batchsrc = sharedContext->sharedBatches[u_sess->stream_cxt.smp_id][loc]; - VectorBatch* batchdst = ((VecStreamState*)node)->m_CurrentBatch; - - if (batchsrc->m_rows == 0) { + // 如果共享上下文已经向量化,从共享上下文中获取数据 + if (sharedContext->vectorized) + { + // 获取源批处理对象和目标批处理对象 + VectorBatch *batchsrc = sharedContext->sharedBatches[u_sess->stream_cxt.smp_id][loc]; + VectorBatch *batchdst = ((VecStreamState *)node)->m_CurrentBatch; + // 如果源批处理的行数为0，表示没有数据可消费，返回false + if (batchsrc->m_rows == 0) + { return false; } - + // 将源批处理的数据复制到目标批处理，进行深复制，不重置源批处理 batchdst->Copy(batchsrc); - + // 重置源批处理 batchsrc->Reset(); - } else { - TupleVector* tuplesrc = sharedContext->sharedTuples[u_sess->stream_cxt.smp_id][loc]; - TupleVector* tupledst = node->tempTupleVec; + } + else + { + // 如果共享上下文未向量化,获取源元组向量对象和目标元组向量对象 + TupleVector *tuplesrc = sharedContext->sharedTuples[u_sess->stream_cxt.smp_id][loc]; + TupleVector *tupledst = node->tempTupleVec; - if (tuplesrc->tuplePointer == 0) { + // 如果源元组指针为0，表示没有数据可消费，返回false + if (tuplesrc->tuplePointer == 0) + { return false; } - - for (int i = 0; i < tuplesrc->tuplePointer; i++) { + // 将源元组向量的数据复制到目标元组向量 + for (int i = 0; i < tuplesrc->tuplePointer; i++) + { (void)ExecCopySlot(tupledst->tupleVector[i], tuplesrc->tupleVector[i]); } - + // 设置目标元组指针为源元组指针，重置源元组指针 tupledst->tuplePointer = tuplesrc->tuplePointer; tuplesrc->tuplePointer = 0; + // 返回元组 (void)gs_return_tuple(node); } + // 记录时间，结束网络工作时间拷贝 NetWorkTimeCopyEnd(t_thrd.pgxc_cxt.GlobalNetInstr); - struct hash_entry* entry = NULL; + struct hash_entry *entry = NULL; + // 获取配额条目 entry = sharedContext->quota_entrys[u_sess->stream_cxt.smp_id][loc]; - + // 如果编译在aarch64架构下，执行内存屏障操作，确保内存操作的正确顺序 #ifdef __aarch64__ pg_memory_barrier(); #endif - /* Reset flag */ + // 重置标志位 sharedContext->dataStatus[u_sess->stream_cxt.smp_id][loc] = DATA_EMPTY; - /* send signal */ + // 发送信号 entry->_signal(); - + // 更新扫描位置 node->sharedContext->scanLoc[u_sess->stream_cxt.smp_id] = loc; return true; } @@ -364,97 +463,139 @@ bool gs_consume_memory_data(StreamState* node, int loc) * STREAM_SCAN_WAIT -- still need to poll to wait for data. * STREAM_SCAN_FINISH -- stream scan finished. */ -char gs_find_memory_data(StreamState* node, int* waitnode_count) +// 此函数的作用是从生产者状态扫描数据，参数node为流状态 +char gs_find_memory_data(StreamState *node, int *waitnode_count) { + // 定义一个数据状态变量 DataStatus dataStatus; + // 定义一个字符串信息变量，初始值为NULL StringInfo buf = NULL; + // 获取上次扫描的位置 int scanLoc = node->sharedContext->scanLoc[u_sess->stream_cxt.smp_id]; + // 定义一个计数器变量，初始值为上次扫描的位置 int i = scanLoc; + // 定义一个标志位，表示扫描是否完成，初始值为true bool finished = true; + // 定义一个标志位，表示连接是否结束，初始值为false bool is_conn_end = false; + // 定义一个计数器，用于统计需要等待的节点数量，初始值为0 int waitnodeCount = 0; - struct hash_entry* entry = NULL; + // 定义一个哈希表条目指针，初始值为NULL + struct hash_entry *entry = NULL; - /* Check if there is available data, and scan from last time location. */ - do { + // 检查是否存在目标数据，并且从最后位置开始寻找 + do + { i++; - if (i == node->conn_count) { + // 如果计数器等于连接数，则重置为0 + if (i == node->conn_count) + { i = 0; } - /* Update scan location. */ + // 更新扫描位置 node->sharedContext->scanLoc[u_sess->stream_cxt.smp_id] = i; + // 获取当前位置的数据状态 dataStatus = node->sharedContext->dataStatus[u_sess->stream_cxt.smp_id][i]; + // 获取当前位置的连接是否结束状态 is_conn_end = node->sharedContext->is_connect_end[u_sess->stream_cxt.smp_id][i]; - if (!is_conn_end) { + if (!is_conn_end) + { + // 设置扫描完成标志位为false finished = false; + // 需要等待的节点数量加1 waitnodeCount++; } - /* - * Firstly, we handle error or notice messages. - * If an error occured, we should stop scan now. - * If an notice occured, we can still receive data. - */ + // 首先，我们处理错误或通知消息。如果错误，我们应该立即停止。如果发生了通知，我们仍然可以接收数据 + // 从共享上下文中获取消息，这是一个字符串信息（StringInfo）结构，其中包含了消息的数据和长度等信息 buf = node->sharedContext->messages[u_sess->stream_cxt.smp_id][i]; - if (buf->len > 0) { - if (buf->cursor == 'E') { + // 如果消息的长度大于0，即存在消息 + if (buf->len > 0) + { + // 如果消息的游标为'E'，表示这是一个错误消息 + if (buf->cursor == 'E') + { + // 调用函数处理流错误，参数为节点，错误消息的数据和长度 HandleStreamError(node, buf->data, buf->len); + // 返回一个标识，表示流扫描结束 return STREAM_SCAN_FINISH; - } else if (buf->cursor == 'N') { + } + // 如果消息的游标为'N'，表示这是一个通知消息 + else if (buf->cursor == 'N') + { + // 调用函数处理流通知，参数为节点，通知消息的数据和长度 HandleStreamNotice(node, buf->data, buf->len); + // 重置字符串信息，清空游标和数据 resetStringInfo(buf); - /* After one notice message has handled, send signal and wake up the dest producer. */ + // 在处理完一个通知消息后，发送信号并唤醒目标生产者 entry = node->sharedContext->quota_entrys[u_sess->stream_cxt.smp_id][i]; + // 获取配额条目，可能是为了记录或控制生产者的行为 entry->_signal(); - + // 返回一个标识，表示流扫描需要等待 return STREAM_SCAN_WAIT; } } - - switch (dataStatus) { - case DATA_EMPTY: - break; - - case DATA_PREPARE: - /* Take the rest data away when the connection is end. */ - if (is_conn_end) { - /* Return data if any. */ - if (gs_consume_memory_data(node, i)) { - return STREAM_SCAN_DATA; - } - } - break; - - case DATA_READY: - if (gs_consume_memory_data(node, i)) { + // 根据dataStatus的值选择执行的代码块 + switch (dataStatus) + { + // 如果dataStatus的值为DATA_EMPTY,不执行任何操作 + case DATA_EMPTY: + break; + // 如果dataStatus的值为DATA_PREPARE + case DATA_PREPARE: + // 当连接结束的时候带走其余的数据 + if (is_conn_end) + { + /* Return data if any. */ + // 如果有数据，通过调用gs_consume_memory_data函数来消耗数据 + if (gs_consume_memory_data(node, i)) + { + // 返回STREAM_SCAN_DATA，表示成功从生产者找到数据 return STREAM_SCAN_DATA; - } else { - break; } + } + break; - case CONN_ERR: - ereport(ERROR, + case DATA_READY: + // 通过调用gs_consume_memory_data函数来消耗数据 + if (gs_consume_memory_data(node, i)) + { + // 返回STREAM_SCAN_DATA，表示成功从生产者找到数据 + return STREAM_SCAN_DATA; + } + else + { + break; + } + // 如果dataStatus的值为CONN_ERR，生成一个错误报告 + case CONN_ERR: + ereport(ERROR, (errcode(ERRCODE_STREAM_REMOTE_CLOSE_SOCKET), - errmsg("Failed to read response from Local Stream Node," - " Detail: Node %s, Plan Node ID %u, SMP ID %d", + errmsg("Failed to read response from Local Stream Node," + " Detail: Node %s, Plan Node ID %u, SMP ID %d", g_instance.attr.attr_common.PGXCNodeName, node->sharedContext->key_s.planNodeId, i))); - break; - // dataStatus is enum, - default: - break; + break; + // dataStatus is enum, + default: + break; } } while (i != scanLoc); - + // 将waitnodeCount的值赋给指针waitnode_count所指向的变量，为了返回等待节点的数量 *waitnode_count = waitnodeCount; - if (finished) { + if (finished) + { + // 返回STREAM_SCAN_FINISH，表示流扫描完成 return STREAM_SCAN_FINISH; - } else { + } + else + { + // 返回STREAM_SCAN_WAIT，表示仍然需要轮询等待数据 return STREAM_SCAN_WAIT; } } @@ -466,51 +607,68 @@ char gs_find_memory_data(StreamState* node, int* waitnode_count) * @return bool: true -- successed to find data and need more data. * false -- all connection finished or recerive error. */ -bool gs_memory_recv(StreamState* node) +// 此函数的作用是从共享内存中接收本地流的数据，返回是否成功找到数据并需要更多数据，或者所有连接已完成或接收错误 +bool gs_memory_recv(StreamState *node) { + // 存储操作结果的字符变量 char result; - struct hash_entry* entry = NULL; + // 哈希表条目指针，初始化为NULL + struct hash_entry *entry = NULL; + // 获取流状态对应的哈希表条目 entry = node->sharedContext->poll_entrys[u_sess->stream_cxt.smp_id]; bool re = true; + // 初始化等待节点数为0 int waitnode_count = 0; - /* If there is already tuple in buffer, return the data at once. */ - if (!node->sharedContext->vectorized && gs_return_tuple(node)) { + // 如果缓冲区中已有元组，则立即返回数据 + if (!node->sharedContext->vectorized && gs_return_tuple(node)) + { return true; } - for (;;) { - /* Check for interrupt at the beginning of the loop. */ + for (;;) + { + // 检查是否有中断请求 CHECK_FOR_INTERRUPTS(); - /* Check if we can early stop now. */ - if (executorEarlyStop()) { + // 检查是否可以提前结束循环 + if (executorEarlyStop()) + { re = false; break; } - /* Search all producers to find data. */ + // 搜索所有生产者以查找数据，同时更新等待节点数 result = gs_find_memory_data(node, &waitnode_count); - if (result == STREAM_SCAN_DATA) { + if (result == STREAM_SCAN_DATA) + { re = true; break; - } else if (result == STREAM_SCAN_FINISH) { + } + // 如果所有连接已完成或接收错误 + else if (result == STREAM_SCAN_FINISH) + { re = false; break; } - + // 定义一个旧的等待状态阶段变量，初始化为PHASE_NONE，表示当前没有等待状态 WaitStatePhase oldPhase = pgstat_report_waitstatus_phase(PHASE_NONE, true); + // 定义一个旧的等待状态变量，通过调用pgstat_report_waitstatus_comm函数来初始化。 + // 该函数将等待状态设置为STATE_WAIT_NODE，表示当前正在等待节点响应。 + // 还将当前节点的ID、等待节点的数量、计划节点的ID以及全局节点定义的数量作为参数传递给该函数 WaitState oldStatus = pgstat_report_waitstatus_comm(STATE_WAIT_NODE, - u_sess->pgxc_cxt.PGXCNodeId, - waitnode_count, - node->sharedContext->key_s.planNodeId, - global_node_definition ? global_node_definition->num_nodes : -1); + u_sess->pgxc_cxt.PGXCNodeId, + waitnode_count, + node->sharedContext->key_s.planNodeId, + global_node_definition ? global_node_definition->num_nodes : -1); - /* Poll to wait data from producers. */ + // 开始网络时间轮询，用于度量网络操作的耗时 NetWorkTimePollStart(t_thrd.pgxc_cxt.GlobalNetInstr); + // 调用entry的_timewait方法，传入SINGLE_WAITQUOTA作为参数，用于等待生产者提供数据 (void)entry->_timewait(SINGLE_WAITQUOTA); + // 结束网络时间轮询 NetWorkTimePollEnd(t_thrd.pgxc_cxt.GlobalNetInstr); - + // 重置等待状态阶段和等待状态为旧的状态 pgstat_reset_waitStatePhase(oldStatus, oldPhase); } @@ -523,16 +681,20 @@ bool gs_memory_recv(StreamState* node) * @param[IN] sharedContext: context for shared memory stream * @param[IN] connNum: producer connection number */ -void gs_memory_send_finish(StreamSharedContext* sharedContext, int connNum) +// 此函数用于通知所有相关的消费者没有更多数据可发送 +void gs_memory_send_finish(StreamSharedContext *sharedContext, int connNum) { - struct hash_entry* entry = NULL; + // 定义一个哈希表条目指针，初始化为NULL + struct hash_entry *entry = NULL; - for (int i = 0; i < connNum; i++) { - /* Set flags. */ + for (int i = 0; i < connNum; i++) + { + // 设置标志位，表示连接已经结束 sharedContext->is_connect_end[i][u_sess->stream_cxt.smp_id] = true; - /* send signal */ + // 获取当前连接对应的哈希表条目 entry = sharedContext->poll_entrys[i]; + // 调用哈希表条目的_signal方法，发送信号且通知等操作 entry->_signal(); } } @@ -544,12 +706,15 @@ void gs_memory_send_finish(StreamSharedContext* sharedContext, int connNum) * @param[IN] connNum: producer connection number * @param[IN] smpId: producer smp id */ -void gs_memory_close_conn(StreamSharedContext* sharedContext, int connNum, int consumerId) +// 此函数用于设置与特定生产者的所有连接关闭 +void gs_memory_close_conn(StreamSharedContext *sharedContext, int connNum, int consumerId) { - struct hash_entry* entry = NULL; + // 定义一个哈希表条目指针，初始化为NULL + struct hash_entry *entry = NULL; - for (int i = 0; i < connNum; i++) { - /* Set flags. */ + for (int i = 0; i < connNum; i++) + { + // 设置标志位，表示与特定生产者的连接已结束 sharedContext->is_connect_end[consumerId][i] = true; /* @@ -557,8 +722,7 @@ void gs_memory_close_conn(StreamSharedContext* sharedContext, int connNum, int c * in a query like "limit XXX", when consumer don't need data anymore, * but the producers haven't send all data yet. */ - entry = sharedContext->quota_entrys[consumerId][i]; - entry->_signal(); + entry = sharedContext->quota_entrys[consumerId][i]; // 获取当前连接对应的哈希表条目 + entry->_signal(); // 调用哈希表条目的_signal方法，发送信号且通知等操作 } -} - +} \ No newline at end of file -- 2.34.1 From b7bc23baf93842b064474960cfd2fa5f15b509a4 Mon Sep 17 00:00:00 2001 From: richard_chen Date: Wed, 27 Sep 2023 21:46:04 +0800 Subject: [PATCH 09/15] =?UTF-8?q?=E5=AF=B9libcomm=5Fadapter.cpp=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E4=B8=AD=E7=9A=84=E5=87=BD=E6=95=B0=E8=BF=9B=E8=A1=8C?= =?UTF-8?q?=E4=BA=86=E6=B3=A8=E9=87=8A=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../libcomm_utils/libcomm_adapter.cpp | 1324 ++++++++++------- 1 file changed, 801 insertions(+), 523 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_adapter.cpp b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_adapter.cpp index baa37d2b5..6a206ad02 100644 --- a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_adapter.cpp +++ b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_adapter.cpp @@ -72,22 +72,34 @@ #define MSG_HEAD_MAGIC_NUM2 0x3E #define MAX_NUMA_NODE 16 -// libcomm delay message number +// 初始化一个延迟消息编号为0 static int libcomm_delay_no = 0; +// libcomm适配器层实例，libcommAdaptLayer是一个自定义的类，表示libcomm适配器层。该实例被命名为g_libcomm_adapt LibcommAdaptLayer g_libcomm_adapt; -extern HTAB* g_htab_fd_id_node_idx; + +/* + 外部定义一个HTAB指针、fd_id_node_idx和互斥锁, + HTAB是OpenGauss中用于高效存储和管理键值对的哈希表数据结构。 + fd_id_node_idx是一个键，用于在哈希表中查找或存储数据。 + 互斥锁用于保护对哈希表的访问，以避免并发冲突。 +*/ +extern HTAB *g_htab_fd_id_node_idx; extern pthread_mutex_t g_htab_fd_id_node_idx_lock; -static int gs_tcp_write_noblock(int node_idx, int sock, const char* msg, int msg_len, int *send_count); -static int libcomm_build_tcp_connection(libcommaddrinfo* libcomm_addrinfo, int node_idx); -int gs_s_build_tcp_ctrl_connection(libcommaddrinfo* libcomm_addrinfo, int node_idx, bool is_reply); +static int gs_tcp_write_noblock(int node_idx, int sock, const char *msg, int msg_len, int *send_count); +static int libcomm_build_tcp_connection(libcommaddrinfo *libcomm_addrinfo, int node_idx); + +int gs_s_build_tcp_ctrl_connection(libcommaddrinfo *libcomm_addrinfo, int node_idx, bool is_reply); + +// 此静态函数用于在数据库中监听TCP连接 static int libcomm_tcp_listen() { + // 调用mc_tcp_listen函数来监听指定的主机和端口。其中第一个参数为本地主机名，第二个参数为服务器监听的端口后，第三个参数为空 return mc_tcp_listen(g_instance.comm_cxt.localinfo_cxt.g_local_host, - g_instance.comm_cxt.g_receivers->server_listen_conn.port, - NULL); + g_instance.comm_cxt.g_receivers->server_listen_conn.port, + NULL); } /* @@ -98,30 +110,38 @@ static int libcomm_tcp_listen() * return value : 0: malloc succeed * -1: malloc failed */ -int libcomm_malloc_iov_item(struct mc_lqueue_item** iov_item, int size) +// 此函数的功能是在libcomm中进行内存分配。 +// 其中iov_item指向被分配的内存，这块内存是为消息队列中的元素分配的，size指的是此次将要分配的内存的大小 +int libcomm_malloc_iov_item(struct mc_lqueue_item **iov_item, int size) { - struct mc_lqueue_item* item = NULL; - struct iovec* iov = NULL; + // mc_lqueue_item表示消息队列中的元素，其表示的是不同的消息，包含消息的一些属性。 + // iovec是用于存储I/O操作的数据的结构体，iov->base表示要接收或者发送的数据的指针，iov->size表示数据的大小。 + struct mc_lqueue_item *item = NULL; + struct iovec *iov = NULL; - /* get iov_item from memory pool */ - *iov_item = gs_memory_pool_queue_pop((char*)iov); - if (*iov_item != NULL) { + // 先尝试从内存池中获取iov_item，如果能分配则直接分配，否则使用malloc动态分配内存 + *iov_item = gs_memory_pool_queue_pop((char *)iov); + if (*iov_item != NULL) + { return 0; } + // 如果libcomm已使用的内存加上要分配的大小超过了总可用内存，则设置errno为ECOMMTCPMEMALLOC并返回-1，表示失败。否则，继续执行下面的代码。 /* - * IF libcomm_used_memory + size is more than g_total_usable_memory - * then errno = ECOMMTCPMEMALLOC and return -1 - */ - /* if memory pool is empty, malloc iov_item */ + 此处使用了三次LIBCOMM_MALLOC的原因是我们要传输一个数据的话，需要先对数据也就是iov进行分配内存，而分配时需要对数据结构体分配，然后再对iov->base也就是数据的实体分配，然后再将此数据放置到消息队列中，为此我们还需要给消息队列元素结构图分配一个内存来存储这个iov。 + */ LIBCOMM_MALLOC(iov, sizeof(struct iovec), iovec); - if (iov == NULL) { + if (iov == NULL) + { + // 如果分配iov失败，则设置errno为ECOMMTCPMEMALLOC并返回-1，表示失败。 errno = ECOMMTCPMEMALLOC; return -1; } LIBCOMM_MALLOC(iov->iov_base, (unsigned)size, void); - if (iov->iov_base == NULL) { + if (iov->iov_base == NULL) + { + // 如果分配iov_base失败，则释放iov占用的内存，并设置errno为ECOMMTCPMEMALLOC并返回-1，表示失败。 LIBCOMM_FREE(iov, sizeof(struct iovec)); errno = ECOMMTCPMEMALLOC; return -1; @@ -129,15 +149,22 @@ int libcomm_malloc_iov_item(struct mc_lqueue_item** iov_item, int size) iov->iov_len = 0; LIBCOMM_MALLOC(item, sizeof(struct mc_lqueue_item), mc_lqueue_item); - if (item == NULL) { + if (item == NULL) + { + // 如果分配item失败，则对于数据iov的分配也没用了，故释放iov_base和iov占用的内存，并设置errno为ECOMMTCPMEMALLOC并返回-1，表示失败。 LIBCOMM_FREE(iov->iov_base, size); LIBCOMM_FREE(iov, sizeof(struct iovec)); errno = ECOMMTCPMEMALLOC; return -1; } + + // 分配成功的话，将数据iov添加消息队列元素中。 item->element.add(iov); + // 将分配的item赋值给*iov_item，直接进行分配。 *iov_item = item; + + // 返回0，表示分配成功。 return 0; } @@ -147,31 +174,36 @@ int libcomm_malloc_iov_item(struct mc_lqueue_item** iov_item, int size) * arguments : iov_item: the pointer of free memory. * size: size of iov->iov_base */ -void libcomm_free_iov_item(struct mc_lqueue_item** iov_item, int size) +// 此函数的功能是在libcomm中进行内存的释放。其中iov_item表示要释放的消息队列中的元素，size表示释放的大小 +void libcomm_free_iov_item(struct mc_lqueue_item **iov_item, int size) { - struct mc_lqueue_item* item = *iov_item; - struct iovec* iov = NULL; + struct mc_lqueue_item *item = *iov_item; + struct iovec *iov = NULL; bool rc = false; - - if (unlikely(item == NULL)) { + // 如果要释放的内存为空，那么就没有能释放的东西了，所以直接结束。 + if (unlikely(item == NULL)) + { return; } - + // iov指的是消息队列元素中真正的在I/O过程中的数据 iov = item->element.data; - + // 此处判断一下数据iov是不是空的，如果空就直接报错。 Assert(iov != NULL && iov->iov_base != NULL); - + // 释放的过程中先将数据的大小给重置为0，即初始化一下 iov->iov_len = 0; - /* push pointer to memory pool */ - rc = gs_memory_pool_queue_push((char*)item); - /* if memory pool is full, free iov_item */ - if (!rc) { + // 将消息队列元素的内存返还到内存池中 + rc = gs_memory_pool_queue_push((char *)item); + + // 如果返还到内存池中失败，也就是rc标志为0的话，那就直接释放内存，这部分内存不要了。 + if (!rc) + { LIBCOMM_FREE(iov->iov_base, size); LIBCOMM_FREE(iov, sizeof(struct iovec)); LIBCOMM_FREE(item, sizeof(struct mc_lqueue_item)); } + // 为防止野指针直接给iov_item指向空 *iov_item = NULL; return; } @@ -185,154 +217,181 @@ void libcomm_free_iov_item(struct mc_lqueue_item** iov_item, int size) * msg_len: msg length * return value : length of msg had be sent */ -static int gs_tcp_write_noblock(int node_idx, int sock, const char* msg, int msg_len, int *send_count) +// 此函数的作用是在TCP协议的非阻塞模式下发送数据 +/* + node_idx:节点索引 + sock:socket套接字 + msg:发送的数据 + msg_len:数据大小 + send_count:指向发送计数器的指针，用来计数 +*/ +static int gs_tcp_write_noblock(int node_idx, int sock, const char *msg, int msg_len, int *send_count) { uint64 time_enter, time_now; int send_bytes = 0; int error = -1; - + // 用来统计发送时间 time_enter = mc_timers_ms(); - do { - /* - * we send data in non-block mode, - * but we will assure the data will - * be sent out if the network is ok - */ + do + { + // 在非阻塞模式下发布数据，网络正常的话数据可以被发送出去。 + // 调用mc_tcp_write_noblock函数，返回值为发送的字节数，如果发送的字节小于0说明没有数据被发送，故错误直接结束。 error = mc_tcp_write_noblock(sock, msg + send_bytes, msg_len - send_bytes); - if (error < 0) { + if (error < 0) + { errno = ECOMMTCPDISCONNECT; break; } - - if (send_count != NULL) { + // 如果发送了内容，则计数器加一 + if (send_count != NULL) + { (*send_count)++; } - /* - * when primary and the standby is switchover, - * the old connection is broken - */ - if (g_instance.comm_cxt.g_senders->sender_conn[node_idx].ip_changed == true) { + // 如果主节点和备用结点发生了切换，断开旧的连接，同时跳出循环 + if (g_instance.comm_cxt.g_senders->sender_conn[node_idx].ip_changed == true) + { errno = ECOMMTCPPEERCHANGED; break; } + // 记录此刻时间，计算发送所用的时间，如果此时间超过指定的发送超时时间，那么跳出循环 time_now = mc_timers_ms(); if (((time_now - time_enter) > - ((uint64)(unsigned)g_instance.comm_cxt.counters_cxt.g_comm_send_timeout * SEC_TO_MICRO_SEC)) && - (time_now > time_enter)) { + ((uint64)(unsigned)g_instance.comm_cxt.counters_cxt.g_comm_send_timeout * SEC_TO_MICRO_SEC)) && + (time_now > time_enter)) + { errno = ECOMMTCPSENDTIMEOUT; break; } - + // 统计已经发送了多少字节，当发送的字节数等于消息的长度时结束发送 send_bytes += error; } while (send_bytes != msg_len); - + // 返回发送的字节数 return send_bytes; } /* function name: libcomm_tcp_send description: This function is used to send the message including message head and message body, to - a specific socket. + a specific socket. arguments: send_ info is a pointer of LibcommRecvInfo* type, pointing to the memory storing the data - waiting to be sent. + waiting to be sent. return value: Data will be sent twice in total. Before sending data, if it is found that the socket to receive - data is not matched with the specified socket, then - 1 will be returned; If the sending of + data is not matched with the specified socket, then - 1 will be returned; If the sending of message head or message body fails, return - 1; If the function runs successfully, the byte length of the message body sent successfully is returned. note: none date: 2022/8/11 contact tel: 18720816902 */ -static int libcomm_tcp_send(LibcommSendInfo* send_info) +// 此函数的作用是使用TCP协议进行数据的发送，发送的数据中包含数据的的头部和主体。 +// 其中send_info是一个指针，指向将要发送的数据的内存块。LibcommSendInfo是发送的数据的结构体，里面包含套接字和节点索引等属性。 +static int libcomm_tcp_send(LibcommSendInfo *send_info) { + // 对于一系列属性例如套接字等进行初始化赋值 int sock = send_info->socket; int sock_id = send_info->socket_id; int version = send_info->version; int streamid = send_info->streamid; int node_idx = send_info->node_idx; int msg_len = send_info->msg_len; - char* msg = send_info->msg; + char *msg = send_info->msg; int error = -1; int send_bytes; int send_count = 0; struct sock_id fd_id = {0, 0}; MsgHead msg_head; + // 初始化数据的头部 msg_head.type = 'D'; msg_head.magic_num = MSG_HEAD_MAGIC_NUM; msg_head.version = version; msg_head.logic_id = streamid; msg_head.msg_len = msg_len; msg_head.checksum = MSG_HEAD_TEMP_CHECKSUM; - + // 获取读写锁中的写锁，确保只有一个线程可以访问共享资源并进行修改，从而避免了并发访问可能导致的数据竞争问题。 LIBCOMM_PTHREAD_RWLOCK_WRLOCK(&g_instance.comm_cxt.g_senders->sender_conn[node_idx].rwlock); - /* check socket version saved before, to prevent send msg to wrong remote node */ + // 检查套接字的版本号是否与之前的一致，以防止将消息发到错误的远程节点 if ((sock != g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket) || - (sock_id != g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket_id)) { + (sock_id != g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket_id)) + { + // 如果版本号不匹配，则记录日志并解锁该连接，返回-1 表示发送失败 COMM_DEBUG_LOG("(s|send)\tsocket version of node%d:%s mismatch old[%d,%d], new[%d,%d].", - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx), - sock, - sock_id, - g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket, - g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket_id); + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx), + sock, + sock_id, + g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket, + g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket_id); LIBCOMM_PTHREAD_RWLOCK_UNLOCK(&g_instance.comm_cxt.g_senders->sender_conn[node_idx].rwlock); return -1; } + // 使用上文提到的TCP协议发送数据函数将消息的头部以非阻塞的方式发送到指定的socket + send_bytes = gs_tcp_write_noblock(node_idx, sock, (char *)&msg_head, sizeof(MsgHead), NULL); - send_bytes = gs_tcp_write_noblock(node_idx, sock, (char*)&msg_head, sizeof(MsgHead), NULL); - if (send_bytes != sizeof(MsgHead)) { - /* close the bad socket when send failed */ + // 检查发送的字节数是否等于消息头部的字节数，如果不等则说明发送失败 + if (send_bytes != sizeof(MsgHead)) + { + // 获取连接的socket和socket ID fd_id.fd = g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket; fd_id.id = g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket_id; + // 当发送失败时，关闭有问题的socket gs_s_close_bad_data_socket(&fd_id, ECOMMTCPDISCONNECT, node_idx); + // 记录发送消息头部失败的警告日志，包含发送的字节数、错误号和错误信息 LIBCOMM_ELOG(WARNING, - "(s|send)\tsend msghead failed send_bytes[%d] errno[%d:%s].", - send_bytes, - errno, - mc_strerror(errno)); + "(s|send)\tsend msghead failed send_bytes[%d] errno[%d:%s].", + send_bytes, + errno, + mc_strerror(errno)); + // 解锁该连接的读写锁，并返回-1 表示发送失败 LIBCOMM_PTHREAD_RWLOCK_UNLOCK(&g_instance.comm_cxt.g_senders->sender_conn[node_idx].rwlock); return -1; } + // 记录日志，表示正在向指定节点发送数据 COMM_DEBUG_LOG("(s|send)\tsend to dn[%d]:%s head[%d, %d] on socket[%d].", - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx), - (int)sizeof(MsgHead), - error, - g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket); + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx), + (int)sizeof(MsgHead), + error, + g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket); + // 使用上文提到的TCP协议发送数据函数将消息的本体以非阻塞的方式发送到指定的socket send_bytes = gs_tcp_write_noblock(node_idx, sock, msg, msg_len, &send_count); - if (send_bytes > 0) { + // 检查发送的字节数是否大于0，如果是，则将发送的字节数添加到该节点的通信字节数中 + if (send_bytes > 0) + { g_instance.comm_cxt.g_senders->sender_conn[node_idx].comm_bytes += send_bytes; } + // 将发送的数量添加到该节点的通信总数中 g_instance.comm_cxt.g_senders->sender_conn[node_idx].comm_count += send_count; + // 记录日志，表示正在向指定节点发送数据 COMM_DEBUG_LOG("(s|send)\tsend to dn[%d]:%s data[%d, %d] on socket[%d].", - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx), - msg_len, - error, - g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket); - - if (send_bytes != msg_len) { - /* close the bad socket when send failed */ + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx), + msg_len, + error, + g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket); + // 如果发送的字节数不等于数据的长度，则关闭有问题的套接字 + if (send_bytes != msg_len) + { fd_id.fd = g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket; fd_id.id = g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket_id; + // 关闭有问题的套接字并记录到日志中 gs_s_close_bad_data_socket(&fd_id, errno, node_idx); LIBCOMM_ELOG(WARNING, - "(s|send)\tsend length mismatch send_bytes[%d] msg_len[%d] errno[%d:%s].", - send_bytes, - msg_len, - errno, - mc_strerror(errno)); + "(s|send)\tsend length mismatch send_bytes[%d] msg_len[%d] errno[%d:%s].", + send_bytes, + msg_len, + errno, + mc_strerror(errno)); send_bytes = -1; } - + // 解锁该连接的读写锁，并返回发送的字节数 LIBCOMM_PTHREAD_RWLOCK_UNLOCK(&g_instance.comm_cxt.g_senders->sender_conn[node_idx].rwlock); return send_bytes; } @@ -340,57 +399,63 @@ static int libcomm_tcp_send(LibcommSendInfo* send_info) /* function name: libcomm_tcp_recv_noidx description: This function is used to store the message transmitted from the sender, specifically to obtain - the message from a specific socket. + the message from a specific socket. arguments: recv_ info is a pointer of LibcommRecvInfo* type, pointing to the memory to store the data - received. + received. return value: If it fails to allocate memory for iov_ Item, return RECV_MEM_ERROR; - If it fails to obtain data, no matter it is a message header or a message body, from the specified socket in blocking mode, return RECV_NET_ERROR; + If it fails to obtain data, no matter it is a message header or a message body, from the specified socket in blocking mode, return RECV_NET_ERROR; If the function runs successfully, the byte length of the read message body is returned. note: none date: 2022/8/11 contact tel: 18720816902 */ -static int libcomm_tcp_recv_noidx(LibcommRecvInfo* recv_info) +// 此函数的作用是在没有具体节点索引的情况下按照TCP协议接收数据，根据与后文的函数对比，此函数是在接收器未确定的情况下接收数据。 +static int libcomm_tcp_recv_noidx(LibcommRecvInfo *recv_info) { + // 初始化套接字等基础信息 int sock = recv_info->socket; MsgHead msg_head = {0}; - struct mc_lqueue_item* iov_item = NULL; - struct iovec* iov = NULL; + struct mc_lqueue_item *iov_item = NULL; + struct iovec *iov = NULL; int error = -1; - // malloc 64 bytes for recv when node_idx < 0, need free - if (0 != libcomm_malloc_iov_item(&iov_item, IOV_DATA_SIZE)) { + // 想要接收数据首先需要分配一个内存来存储数据，如果分配失败直接不接收数据，结束 + if (0 != libcomm_malloc_iov_item(&iov_item, IOV_DATA_SIZE)) + { return RECV_MEM_ERROR; } iov = iov_item->element.data; - // recv poll event, recv msg head in block mode + // 使用mc_tcp_read_block函数以阻塞模式从socket接收消息头，并存储在msg_head中 error = mc_tcp_read_block(sock, &msg_head, sizeof(MsgHead), 0); - // must be a connect msg when node_idx < 0 + // 如果接收错误则记录日志 if (error < 0 || msg_head.type != 'C' || msg_head.magic_num != MSG_HEAD_MAGIC_NUM || - msg_head.checksum != MSG_HEAD_TEMP_CHECKSUM || msg_head.msg_len > IOV_DATA_SIZE) { + msg_head.checksum != MSG_HEAD_TEMP_CHECKSUM || msg_head.msg_len > IOV_DATA_SIZE) + { LIBCOMM_ELOG(WARNING, - "(r|inner recv)\tReceiver error msg head[%d] " - "from socket[%d] lid:%d type[%d], magic_num[%d], len[%u].", - error, - sock, - msg_head.logic_id, - msg_head.type, - msg_head.magic_num, - msg_head.msg_len); - + "(r|inner recv)\tReceiver error msg head[%d] " + "from socket[%d] lid:%d type[%d], magic_num[%d], len[%u].", + error, + sock, + msg_head.logic_id, + msg_head.type, + msg_head.magic_num, + msg_head.msg_len); + // 对于之前分配的内存进行释放 libcomm_free_iov_item(&iov_item, IOV_DATA_SIZE); return RECV_NET_ERROR; } - // recv msg head finish + // 使用mc_tcp_read_block函数以阻塞模式从socket接收消息内容 error = mc_tcp_read_block(sock, iov->iov_base, msg_head.msg_len, 0); - if (error < 0) { + // 如果接收错误则释放内存 + if (error < 0) + { libcomm_free_iov_item(&iov_item, IOV_DATA_SIZE); return RECV_NET_ERROR; } - + // 接收正确则将数据写到recv_info中并且返回接收到的字节数。 iov->iov_len = error; recv_info->iov_item = iov_item; recv_info->streamid = msg_head.logic_id; @@ -402,154 +467,175 @@ static int libcomm_tcp_recv_noidx(LibcommRecvInfo* recv_info) /* function name: libcomm_tcp_recv description: This function is used to store the message transmitted from the sender, specifically to obtain - the message from a specific socket. + the message from a specific socket. arguments: recv_ info is a pointer of LibcommRecvInfo* type, pointing to the memory to store the data - received. -return value: If the receiver has not been determined, call libcomm_tcp_recv_noidx() and take the return value - of (libcomm_tcp_recv_noidx (recv_info)); Return RECV_NET_ERROR if there is an error in the - process of reading the message heade or message body; If there is no data readable in the + received. +return value: If the receiver has not been determined, call libcomm_tcp_recv_noidx() and take the return value + of (libcomm_tcp_recv_noidx (recv_info)); Return RECV_NET_ERROR if there is an error in the + process of reading the message heade or message body; If there is no data readable in the receiving buffer of the specified socket at this time or the number of bytes of the data that - has been read is not enough, it returns RECV_NEED_RETRY; If iov_item is NULL, it returns - RECV_MEM_ERROR if it fails to allocate space for it; If the function runs successfully, then + has been read is not enough, it returns RECV_NEED_RETRY; If iov_item is NULL, it returns + RECV_MEM_ERROR if it fails to allocate space for it; If the function runs successfully, then the byte length of the read message head and message body is returned. note: none date: 2022/8/11 contact tel: 18720816902 */ -int libcomm_tcp_recv(LibcommRecvInfo* recv_info) +// 此函数的作用是在确定了节点索引的情况下使用TCP协议接收数据 +int libcomm_tcp_recv(LibcommRecvInfo *recv_info) { - MsgHead* msg_head = NULL; - struct iovec* iov = NULL; - struct mc_lqueue_item* iov_item = NULL; + MsgHead *msg_head = NULL; + struct iovec *iov = NULL; + struct mc_lqueue_item *iov_item = NULL; int sock = recv_info->socket; int node_idx = recv_info->node_idx; int recv_bytes = -1; - int* head_read_cursor = 0; + int *head_read_cursor = 0; int unread_head_len = 0; int unread_body_len = 0; - // first READY message, no have idx, can not buffer, block recv - if (node_idx < 0) { + // 如果没有具体的节点索引，直接使用上文的函数进行数据的接收 + if (node_idx < 0) + { return libcomm_tcp_recv_noidx(recv_info); } + // 获取特定节点索引的消息的头部 msg_head = &g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].msg_head; + // 获取特定节点索引的数据 iov_item = g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].iov_item; + // 获取特定节点索引的消息已经读取位置的标志 head_read_cursor = &(g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].head_read_cursor); + // 计算未读的消息头部长度 unread_head_len = sizeof(MsgHead) - *head_read_cursor; - if (unread_head_len > 0) { - // recv poll event, recv msg head in block mode - recv_bytes = mc_tcp_read_nonblock(sock, (char*)msg_head + *head_read_cursor, unread_head_len, 0); - if (recv_bytes < 0) { + // 如果有未读取的消息头部 + if (unread_head_len > 0) + { + // 以阻塞模式在TCP socket上进行非阻塞读取，以接收消息头 + recv_bytes = mc_tcp_read_nonblock(sock, (char *)msg_head + *head_read_cursor, unread_head_len, 0); + // 如果接收失败则返回错误 + if (recv_bytes < 0) + { return RECV_NET_ERROR; } - if (recv_bytes == 0) { + if (recv_bytes == 0) + { return RECV_NEED_RETRY; } - + // 更新消息头部读取标志，表示已读取的消息头部长度增长了 *head_read_cursor = *head_read_cursor + recv_bytes; - /* msg head not received complete, return to epoll_wait */ - if (recv_bytes < unread_head_len) { + // 如果接收到的字节数小于未读取的消息头部长度，表示消息头部尚未完全接收，需要返回epoll_wait等待更多的数据 + if (recv_bytes < unread_head_len) + { return RECV_NEED_RETRY; } } - /* recv msg head finish */ + // 判断是否读取完消息的头部，读取完则继续执行，否则报错 Assert(*head_read_cursor == sizeof(MsgHead)); - + // 如果消息的头部一些属性错误直接结束 if (msg_head->magic_num != MSG_HEAD_MAGIC_NUM || msg_head->checksum != MSG_HEAD_TEMP_CHECKSUM || - msg_head->msg_len > IOV_DATA_SIZE) { + msg_head->msg_len > IOV_DATA_SIZE) + { LIBCOMM_ELOG(WARNING, - "(r|inner recv)\tReceiver error msg head[%d] " - "from socket[%d] node[%d]:%s lid:%d len=%u, magic_num[%d].", - recv_bytes, - sock, - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), - msg_head->logic_id, - msg_head->msg_len, - msg_head->magic_num); + "(r|inner recv)\tReceiver error msg head[%d] " + "from socket[%d] node[%d]:%s lid:%d len=%u, magic_num[%d].", + recv_bytes, + sock, + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), + msg_head->logic_id, + msg_head->msg_len, + msg_head->magic_num); return RECV_NET_ERROR; } - if (iov_item == NULL) { - if (0 != libcomm_malloc_iov_item(&iov_item, IOV_DATA_SIZE)) { + if (iov_item == NULL) + { + // 分配内存失败的话就结束 + if (0 != libcomm_malloc_iov_item(&iov_item, IOV_DATA_SIZE)) + { return RECV_MEM_ERROR; } - - // save new malloc iov point + // 如果没有失败则将分配的内存给到特定节点索引处 g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].iov_item = iov_item; } iov = iov_item->element.data; - - if (msg_head->msg_len <= iov->iov_len) { + // 判断消息头部的长度是否小于数据的长度 + if (msg_head->msg_len <= iov->iov_len) + { Assert(msg_head->msg_len > iov->iov_len); LIBCOMM_ELOG(WARNING, - "(r|inner recv)\tReceiver error msg_len %u iov_len %lu.", - msg_head->msg_len, iov->iov_len); + "(r|inner recv)\tReceiver error msg_len %u iov_len %lu.", + msg_head->msg_len, iov->iov_len); return RECV_NET_ERROR; } - + // 计算未读取的消息本体的长度 unread_body_len = msg_head->msg_len - iov->iov_len; - #ifdef LIBCOMM_FAULT_INJECTION_ENABLE - if ((is_comm_fault_injection(LIBCOMM_FI_R_PACKAGE_SPLIT))) { - if (iov->iov_len == 0) { + if ((is_comm_fault_injection(LIBCOMM_FI_R_PACKAGE_SPLIT))) + { + if (iov->iov_len == 0) + { unread_body_len = unread_body_len / 2; } } #endif - - recv_bytes = mc_tcp_read_nonblock(sock, (char*)iov->iov_base + iov->iov_len, unread_body_len, 0); + // 使用非阻塞方式从TCP套接字接收数据，接收消息本体部分 + recv_bytes = mc_tcp_read_nonblock(sock, (char *)iov->iov_base + iov->iov_len, unread_body_len, 0); + // 将接收到的字节数添加到通信字节数中 g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].comm_bytes += recv_bytes; + // 通信计数加1 g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].comm_count += 1; - // real network errors, we should report it - // errno is not EAGAIN/EWOULDBLOCK/EINTR - if (recv_bytes < 0) { + // 如果接收到的消息的字节数小于或者等于0就报错 + if (recv_bytes < 0) + { return RECV_NET_ERROR; } - if (recv_bytes == 0) { + if (recv_bytes == 0) + { return RECV_NEED_RETRY; } - + // 数据的总长度增加了目前接收到的消息长度 iov->iov_len += recv_bytes; - /* msg body not received complete, return to epoll_wait */ - if (iov->iov_len < msg_head->msg_len) { + // 如果消息本体没有接收完，则进入epoll_wait等待 + if (iov->iov_len < msg_head->msg_len) + { return RECV_NEED_RETRY; } - /* recv msg body finish */ + // 检查是否接收完数据 Assert(iov->iov_len == msg_head->msg_len); + // 将接收到的数据返回 recv_info->iov_item = iov_item; recv_info->streamid = msg_head->logic_id; recv_info->version = msg_head->version; COMM_DEBUG_LOG("(r|inner recv)\tReceiver msg head[%d] " "from socket[%d] node[%d]:%s logic id:%d len=%u.", - recv_bytes, - sock, - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), - msg_head->logic_id, - msg_head->msg_len); - - // give up iov point that g_receivers had - // iov point must return and save to cmailbox + recv_bytes, + sock, + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_r_node_sock, node_idx), + msg_head->logic_id, + msg_head->msg_len); + // 对于相关的变量的状态进行重置 g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].iov_item = NULL; g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].head_read_cursor = 0; g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].msg_head.type = MSG_NULL; - if (g_ackchk_time) { + if (g_ackchk_time) + { g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].last_rcv_time = mc_timers_ms(); } - + // 返回接收到数据的长度 return iov->iov_len; } @@ -562,99 +648,128 @@ int libcomm_tcp_recv(LibcommRecvInfo* recv_info) * -1: net error * -2: mem error */ -int gs_accept_data_conntion(struct iovec* iov, const sock_id fd_id) +// 此函数的作用是接受客户端的 TCP 连接请求,并将连接信息存储到指定的iovec结构体中 +// 其中iov中提供了连接包的信息，fd_id表示逻辑连接的套接字 +int gs_accept_data_conntion(struct iovec *iov, const sock_id fd_id) { + // 初始化节点索引 int node_idx = -1; + // 初始化一个sock_id结构体的变量，用来保存之前的套接字描述符 struct sock_id old_fd_id; - struct libcomm_connect_package* connect_pkg = (struct libcomm_connect_package*)iov->iov_base; + // 将传入的连接包内的信息提取出来 + struct libcomm_connect_package *connect_pkg = (struct libcomm_connect_package *)iov->iov_base; - if (iov->iov_len < sizeof(struct libcomm_connect_package)) { + // 检查传入的连接包的数据长度是否小于 libcomm_connect_package 的大小，如果小于则说明返回网络接收错误 + if (iov->iov_len < sizeof(struct libcomm_connect_package)) + { LIBCOMM_ELOG(WARNING, - "(r|inner recv)\tIov len[%zu] is less than libcomm_connect_package[%zu].", - iov->iov_len, sizeof(struct libcomm_connect_package)); + "(r|inner recv)\tIov len[%zu] is less than libcomm_connect_package[%zu].", + iov->iov_len, sizeof(struct libcomm_connect_package)); Assert(iov->iov_len == sizeof(struct libcomm_connect_package)); return RECV_NET_ERROR; } - /* Network data is not trusted */ - if (connect_pkg->magic_num != MSG_HEAD_MAGIC_NUM2) { + // 判断连接包的magic_num是否正确，如果错误则说明传输过程中连接包被篡改，返回返回网络接收错误 + if (connect_pkg->magic_num != MSG_HEAD_MAGIC_NUM2) + { return RECV_NET_ERROR; } + // 将连接包中的 node_name 的最后一个字符设为 '\0'，确保它是一个字符串结束符 connect_pkg->node_name[NAMEDATALEN - 1] = '\0'; + // 将连接包中的 host 的最后一个字符设为 '\0'，确保它是一个字符串结束符 connect_pkg->host[HOST_ADDRSTRLEN - 1] = '\0'; + // 根据连接包的节点名称获取连接包的节点索引 node_idx = gs_get_node_idx(connect_pkg->node_name); - if (unlikely(node_idx < 0)) { + // 如果节点索引小于0即错误，那么返回返回网络接收错误 + if (unlikely(node_idx < 0)) + { LIBCOMM_ELOG(WARNING, - "(r|inner recv)\tFailed to get node index for %s: %s.", - connect_pkg->node_name, - mc_strerror(errno)); + "(r|inner recv)\tFailed to get node index for %s: %s.", + connect_pkg->node_name, + mc_strerror(errno)); return RECV_NET_ERROR; } - if (gs_map_sock_id_to_node_idx(fd_id, node_idx) < 0) { + // 检查是否能够通过套接字 ID（sock_id）映射到节点索引（node_idx），如果不能则返回网络接收错误 + if (gs_map_sock_id_to_node_idx(fd_id, node_idx) < 0) + { LIBCOMM_ELOG(WARNING, "(r|inner recv)\tFailed to save sock and sockid."); return RECV_NET_ERROR; } + // 获取目标节点索引的读写锁，并且进行写锁定 LIBCOMM_PTHREAD_RWLOCK_WRLOCK(&g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].rwlock); - // step6: if the old socket is ok, maybe the primary is changed, we should close the old connection - if (g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].socket >= 0) { + + // 如果旧的套接字是有效的，说明主节点已经改变，关闭旧的连接 + if (g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].socket >= 0) + { LIBCOMM_ELOG(WARNING, - "(r|inner recv)\tOld connection exist, maybe the primary is changed, old address of " - "node[%d] is:%s, new is:%s, the connection will be reset.", - node_idx, - g_instance.comm_cxt.g_r_node_sock[node_idx].remote_host, - connect_pkg->host); + "(r|inner recv)\tOld connection exist, maybe the primary is changed, old address of " + "node[%d] is:%s, new is:%s, the connection will be reset.", + node_idx, + g_instance.comm_cxt.g_r_node_sock[node_idx].remote_host, + connect_pkg->host); old_fd_id.fd = g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].socket; old_fd_id.id = g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].socket_id; + // 调用函数关闭旧的连接 gs_r_close_bad_data_socket(node_idx, old_fd_id, ECOMMTCPPEERCHANGED, false); } + // 将新的套接字 fd 赋给目标连接，并将连接的套接字 ID 设置为 fd 的 ID。 g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].socket = fd_id.fd; g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].socket_id = fd_id.id; + // 将连接的消息头类型设置为 MSG_NULL g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].msg_head.type = MSG_NULL; + // 将连接的头读取游标设置为0，表示从头开始读取数据 g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].head_read_cursor = 0; - if (g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].iov_item) { - struct iovec* iov_data = g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].iov_item->element.data; + // 如果目标连接已经有数据的话，那么先清空其中数据 + if (g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].iov_item) + { + struct iovec *iov_data = g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].iov_item->element.data; iov_data->iov_len = 0; } - + // 释放对指定的连接的写锁定 LIBCOMM_PTHREAD_RWLOCK_UNLOCK(&g_instance.comm_cxt.g_receivers->receiver_conn[node_idx].rwlock); - // step8: set the socket information + // 设置套接字信息，将连接包中的host复制到remote_host中，进行双方连接 int ss_rc = strcpy_s(g_instance.comm_cxt.g_r_node_sock[node_idx].remote_host, HOST_ADDRSTRLEN, connect_pkg->host); + // 进行连接后的安全检查 securec_check(ss_rc, "\0", "\0"); - // step 7: send back ack to tell the sender continue + // 发送一个ack确认帧告诉发送方已经连接可以继续 struct libcomm_accept_package ack_msg; ack_msg.type = LIBCOMM_PKG_TYPE_ACCEPT; ack_msg.result = 1; - if (g_libcomm_adapt.send_ack(fd_id.fd, (char*)&ack_msg, sizeof(ack_msg)) < 0) { + // 使用ack发送函数发送确认消息给发送方 + // 如果发送失败，则调用gs_r_close_bad_data_socket函数关闭套接字并返回网络接收错误 + if (g_libcomm_adapt.send_ack(fd_id.fd, (char *)&ack_msg, sizeof(ack_msg)) < 0) + { gs_r_close_bad_data_socket(node_idx, fd_id, ECOMMTCPDISCONNECT, true); return RECV_NET_ERROR; } - #ifdef LIBCOMM_FAULT_INJECTION_ENABLE - if (is_comm_fault_injection(LIBCOMM_FI_GSS_SCTP_FAILED)) { + // 检查是否开启特定的故障注入模式，如果是，则模拟GSS认证失败并返回网络接收错误 + if (is_comm_fault_injection(LIBCOMM_FI_GSS_SCTP_FAILED)) + { errno = ECOMMTCPGSSAUTHFAIL; LIBCOMM_ELOG(WARNING, - "(r|recv loop)\t[FAULT INJECTION]Data channel GSS authentication failed, listen socket[%d]:%s.", - fd_id.fd, - mc_strerror(errno)); + "(r|recv loop)\t[FAULT INJECTION]Data channel GSS authentication failed, listen socket[%d]:%s.", + fd_id.fd, + mc_strerror(errno)); return RECV_NET_ERROR; } #endif - + // 记录日志，表示接受到来自指定节点（node_idx）的连接请求，使用的套接字为fd_id.fd，ID为fd_id.id LIBCOMM_ELOG(LOG, - "(r|recv loop)\tAccept data connection for " - "node[%d]:%s with socket[%d,%d].", - node_idx, - g_instance.comm_cxt.g_r_node_sock[node_idx].remote_nodename, - fd_id.fd, - fd_id.id); + "(r|recv loop)\tAccept data connection for " + "node[%d]:%s with socket[%d,%d].", + node_idx, + g_instance.comm_cxt.g_r_node_sock[node_idx].remote_nodename, + fd_id.fd, + fd_id.id); return 0; } @@ -665,7 +780,9 @@ int gs_accept_data_conntion(struct iovec* iov, const sock_id fd_id) * arguments : _in_ t_fd_id: the fd and fd id for this connection. * _in_ fcmsgr: the message we received. */ -void gs_accept_ctrl_conntion(struct sock_id* t_fd_id, struct FCMSG_T* fcmsgr) +// 此函数的作用是检查新的连接，如果有新的连接，则关闭旧的连接 +// t_fd_id表示当前连接的文件描述符和文件描述符的 ID，fcmsgr表示接收到的消息 +void gs_accept_ctrl_conntion(struct sock_id *t_fd_id, struct FCMSG_T *fcmsgr) { int current_mode; int rc = -1; @@ -674,119 +791,147 @@ void gs_accept_ctrl_conntion(struct sock_id* t_fd_id, struct FCMSG_T* fcmsgr) uint32 cpylen; struct sock_id old_fd_id = {-1, -1}; + // 获取接收到的消息的指定的节点索引 uint16 idx = fcmsgr->node_idx; + // 检查节点索引是否小于目前的节点数，判断节点索引是否超出范围 Assert(idx < g_instance.comm_cxt.counters_cxt.g_cur_node_num); /* Network data is not trusted */ fcmsgr->nodename[NAMEDATALEN - 1] = '\0'; + // 获取特定套接字的锁，防止并发操作对共享数据的影响 g_instance.comm_cxt.g_r_node_sock[idx].lock(); + // 获取旧套接字的文件描述符和ID，并将其存储到old_fd_id结构中 old_fd_id.fd = g_instance.comm_cxt.g_r_node_sock[idx].ctrl_tcp_sock; old_fd_id.id = g_instance.comm_cxt.g_r_node_sock[idx].ctrl_tcp_sock_id; - // if the two sockets are the same, we need not close the socket, it is ok - if (old_fd_id.fd == t_fd_id->fd && old_fd_id.id == t_fd_id->id) { + /* 检测新旧套接字是否相同，如果相同则直接解锁，否则先解锁，然后关闭旧套接字， + 并记录警告日志。然后尝试将新套接字和套接字ID注册到对应的节点索引， + 如果失败则再次记录警告日志并关闭新套接字。 + */ + if (old_fd_id.fd == t_fd_id->fd && old_fd_id.id == t_fd_id->id) + { g_instance.comm_cxt.g_r_node_sock[idx].unlock(); - } else { + } + else + { g_instance.comm_cxt.g_r_node_sock[idx].unlock(); - if (old_fd_id.fd >= 0) { + if (old_fd_id.fd >= 0) + { /* close the old tcp socket, or it will be error and leak */ gs_r_close_bad_ctrl_tcp_sock(&old_fd_id, ECOMMTCPPEERCHANGED); LIBCOMM_ELOG(WARNING, - "(r|flow ctrl)\tOld connection exist, maybe the primary is changed, old address of " - "node[%d] is:%s, the connection will be reset.", - idx, - g_instance.comm_cxt.g_r_node_sock[idx].remote_host); + "(r|flow ctrl)\tOld connection exist, maybe the primary is changed, old address of " + "node[%d] is:%s, the connection will be reset.", + idx, + g_instance.comm_cxt.g_r_node_sock[idx].remote_host); } /* regist new sock and sock id to node idx */ - if (gs_map_sock_id_to_node_idx(*t_fd_id, idx) < 0) { + if (gs_map_sock_id_to_node_idx(*t_fd_id, idx) < 0) + { LIBCOMM_ELOG(WARNING, "(r|flow ctrl)\tFailed to save sock and sockid."); gs_r_close_bad_ctrl_tcp_sock(t_fd_id, ECOMMTCPTCPDISCONNECT); return; } } - + // 在处理完旧套接字后，再次对特定套接字上锁，防止在处理过程中被其他线程干扰 g_instance.comm_cxt.g_r_node_sock[idx].lock(); + // 将新的套接字的信息设置到指定节点索引中，更新连接信息 g_instance.comm_cxt.g_r_node_sock[idx].set_nl(t_fd_id->fd, CTRL_TCP_SOCK); g_instance.comm_cxt.g_r_node_sock[idx].set_nl(t_fd_id->id, CTRL_TCP_SOCK_ID); + // 获取节点名称的长度 cpylen = comm_get_cpylen(fcmsgr->nodename, NAMEDATALEN); + // 将远程节点名称设置为0 ss_rc = memset_s(g_instance.comm_cxt.g_r_node_sock[idx].remote_nodename, NAMEDATALEN, 0x0, NAMEDATALEN); securec_check(ss_rc, "\0", "\0"); + // 使用strncpy函数复制节点名称到远程节点名称，包括结束符'\0' ss_rc = strncpy_s(g_instance.comm_cxt.g_r_node_sock[idx].remote_nodename, NAMEDATALEN, fcmsgr->nodename, cpylen + 1); securec_check(ss_rc, "\0", "\0"); g_instance.comm_cxt.g_r_node_sock[idx].remote_nodename[cpylen] = '\0'; - + // 解锁对特定套接字的锁定 g_instance.comm_cxt.g_r_node_sock[idx].unlock(); - /* send response to remote, thus ready control msg arrived after connection has established */ - if (IS_PGXC_COORDINATOR) { + // 向远程发送响应，以便在连接建立后收到控制消息 + // 如果当前节点是PGXC协调者节点，则设置ack为'o'，并向远程发送该响应 + if (IS_PGXC_COORDINATOR) + { ack = 'o'; rc = mc_tcp_write_block(t_fd_id->fd, &ack, sizeof(ack)); - // if tcp send failed, close tcp connction - if (rc <= 0) { + // 如果TCP发送失败，则关闭TCP连接 + if (rc <= 0) + { LIBCOMM_ELOG(WARNING, "(r|flow ctrl)\tFailed to send ack, error:%s.", mc_strerror(errno)); gs_r_close_bad_ctrl_tcp_sock(t_fd_id, ECOMMTCPTCPDISCONNECT); return; } - } else if (fcmsgr->type == CTRL_CONN_REGIST_CN) { - if (g_instance.comm_cxt.g_ha_shm_data) { + } + else if (fcmsgr->type == CTRL_CONN_REGIST_CN) + { + // 获取当前模式 + if (g_instance.comm_cxt.g_ha_shm_data) + { current_mode = g_instance.comm_cxt.g_ha_shm_data->current_mode; - } else { + } + else + { current_mode = UNKNOWN_MODE; LIBCOMM_ELOG(WARNING, "(r|flow ctrl)\tCannot get current mode, postmaster exit."); } - - ack = (current_mode != STANDBY_MODE && current_mode != PENDING_MODE && current_mode != UNKNOWN_MODE) ? - 'o' : 'r'; - + // 根据当前模式设置ack确认帧的值 + ack = (current_mode != STANDBY_MODE && current_mode != PENDING_MODE && current_mode != UNKNOWN_MODE) ? 'o' : 'r'; + // 向远程发送ack响应 rc = mc_tcp_write_block(t_fd_id->fd, &ack, sizeof(ack)); - // if tcp send failed, close tcp connction - if (rc <= 0 || (ack == 'r')) { - if (current_mode == STANDBY_MODE) { + // 如果TCP发送失败，则关闭TCP连接 + if (rc <= 0 || (ack == 'r')) + { + if (current_mode == STANDBY_MODE) + { LIBCOMM_ELOG(WARNING, "(r|flow ctrl)\tCannot accept connection in standby mode."); - /* - * When the standby node is promoting, - * if not promoting immediately, - * wait to avoid tons of LIBCOMM log printed. - */ + // 获取当前节点的状态 DbState db_state = get_local_dbstate(); - if (db_state != PROMOTING_STATE) { + if (db_state != PROMOTING_STATE) + { (void)sleep(1); LIBCOMM_ELOG(WARNING, - "(r|flow ctrl)\tCannot accept connection because the standby is not promoting immediately."); + "(r|flow ctrl)\tCannot accept connection because the standby is not promoting immediately."); } - } else if (current_mode == PENDING_MODE) { + } + else if (current_mode == PENDING_MODE) + { // sleep 1 second to wait process starting (void)sleep(1); LIBCOMM_ELOG(WARNING, "(r|flow ctrl)\tCannot accept connection in pending mode."); - } else { + } + else + { LIBCOMM_ELOG(WARNING, "(r|flow ctrl)\tCannot accept connection in unknown mode."); } - if (rc <= 0) { + if (rc <= 0) + { // 如果TCP发送失败 LIBCOMM_ELOG(WARNING, "(r|flow ctrl)\tFailed to send ack, error:%s.", mc_strerror(errno)); } - + // 关闭TCP连接 gs_r_close_bad_ctrl_tcp_sock(t_fd_id, ECOMMTCPTCPDISCONNECT); return; } } - + // 记录连接的日志 LIBCOMM_ELOG(LOG, - "(r|flow ctrl)\tAccept control connection for " - "node[%d]:%s with socket[%d,%d].", - idx, - g_instance.comm_cxt.g_r_node_sock[idx].remote_nodename, - t_fd_id->fd, - t_fd_id->id); + "(r|flow ctrl)\tAccept control connection for " + "node[%d]:%s with socket[%d,%d].", + idx, + g_instance.comm_cxt.g_r_node_sock[idx].remote_nodename, + t_fd_id->fd, + t_fd_id->id); return; } @@ -798,45 +943,64 @@ void gs_accept_ctrl_conntion(struct sock_id* t_fd_id, struct FCMSG_T* fcmsgr) * return value : -1: error * : 0: succeed */ +// 此函数的作用是通过unix_domain套接字来与postmaster线程进行连接 static int gs_connect_by_unix_domain() { errno_t ss_rc; uint32 cpylen, maxlen; - // STEP1 create new socket - if ((g_instance.comm_cxt.localinfo_cxt.sock_to_server_loop = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) { + // 创建新的套接字，通过调用 socket 函数创建一个 Unix域套接字，创建失败则返回-1 + if ((g_instance.comm_cxt.localinfo_cxt.sock_to_server_loop = socket(AF_UNIX, SOCK_STREAM, 0)) < 0) + { LIBCOMM_ELOG(WARNING, "(SendUnixDomainMsg)\tCould not create socket."); return -1; } - - if (g_instance.comm_cxt.g_unix_path == NULL) { + // 检查 g_instance.comm_cxt.g_unix_path 是否为空 + if (g_instance.comm_cxt.g_unix_path == NULL) + { LIBCOMM_ELOG(WARNING, "(SendUnixDomainMsg)\tCould not get unix path."); return -1; } - // STEP2 set unix addr + // 设置unix_domain套接字的地址 + // 创建一个地址的实例，其中包含套接字的地址族(sun_family)&套接字的地址路径(sun_path) struct sockaddr_un unp; + // 使用 memset_s 函数将 unp 结构体填充为 0 ss_rc = memset_s(&unp, sizeof(unp), 0x0, sizeof(struct sockaddr_un)); + // 检查 memset_s 函数是否成功返回，如果返回错误，则通过 securec_check 函数打印错误信息，并终止程序运行 securec_check(ss_rc, "\0", "\0"); + // 设置 unp 结构体的 sun_family 成员为 AF_UNIX，表示该套接字是Unix域套接字 unp.sun_family = AF_UNIX; + // 最大长度赋值为套接字地址路径的长度 maxlen = sizeof(unp.sun_path); + // 使用 comm_get_cpylen 函数获取需要复制的长度 cpylen = comm_get_cpylen(g_instance.comm_cxt.g_unix_path, maxlen); + // 使用 memset_s 函数将 unp.sun_path 字符串填充为 0 ss_rc = memset_s(unp.sun_path, maxlen, 0x0, maxlen); securec_check(ss_rc, "\0", "\0"); + // 使用 strncpy_s 函数将目标路径复制给套接字地址路径 ss_rc = strncpy_s(unp.sun_path, maxlen, g_instance.comm_cxt.g_unix_path, cpylen + 1); securec_check(ss_rc, "\0", "\0"); + // 在套接字地址路径的结尾添加一个空字符（'\0'），表示字符串的结束 unp.sun_path[cpylen] = '\0'; - // STEP3 connecting, server loop will be waked up + // 通过套接字连接服务器，服务器循环将被唤醒 + // 使用 connect 函数通过套接字连接服务器 if (connect(g_instance.comm_cxt.localinfo_cxt.sock_to_server_loop, - (struct sockaddr*)&unp, - sizeof(struct sockaddr_un)) == -1) { + (struct sockaddr *)&unp, + sizeof(struct sockaddr_un)) == -1) + { + // 连接失败则打印日志 LIBCOMM_ELOG(WARNING, "(SendUnixDomainMsg)\tFailed to connect by unix socket, error: %s", mc_strerror(errno)); + // 关闭套接字 close(g_instance.comm_cxt.localinfo_cxt.sock_to_server_loop); + // 将套接字文件描述符设置为无效值 g_instance.comm_cxt.localinfo_cxt.sock_to_server_loop = INVALID_SOCK; + // 返回-1，表示连接失败 return -1; } + // 连接成功 return 0; } @@ -848,29 +1012,36 @@ static int gs_connect_by_unix_domain() * return value : -1: error * : other postive value: sent bytes */ -int gs_send_msg_by_unix_domain(const void* msg, int msg_len) +// 此函数的功能是通过Unix域套接字将消息发到postmaster线程 +int gs_send_msg_by_unix_domain(const void *msg, int msg_len) { int error = 0; bool is_retry = true; retry: - if (g_instance.comm_cxt.localinfo_cxt.sock_to_server_loop == INVALID_SOCK) { + // 如果循环监听的套接字是一个无效套接字，那么先使用连接函数与postmaster线程进行连接 + if (g_instance.comm_cxt.localinfo_cxt.sock_to_server_loop == INVALID_SOCK) + { error = gs_connect_by_unix_domain(); } - - if (error < 0) { + // 如果连接失败则返回一个错误 + if (error < 0) + { return error; } - - error = mc_tcp_write_block(g_instance.comm_cxt.localinfo_cxt.sock_to_server_loop, (const void*)msg, msg_len); - if (error <= 0) { + // 调用函数尝试通过套接字发送消息 + error = mc_tcp_write_block(g_instance.comm_cxt.localinfo_cxt.sock_to_server_loop, (const void *)msg, msg_len); + // 如果发送失败，先记录到日志中 + if (error <= 0) + { LIBCOMM_ELOG(WARNING, "(s|unix domain)\tFailed to send through unix socket, error: %s", mc_strerror(errno)); + // 关闭套接字 close(g_instance.comm_cxt.localinfo_cxt.sock_to_server_loop); + // 将套接字文件描述符设置为无效值 g_instance.comm_cxt.localinfo_cxt.sock_to_server_loop = INVALID_SOCK; - // send fail may due to socket is closed by postmaster - // try to make a new connection then send one more time - // if failed in second time, return error - if (is_retry) { + // 如果是第一次重试，那么直接重试回到retry，并且将is_retry设置为false阻止第二次重试。如果已经是第二次重试了，直接返回错误。 + if (is_retry) + { is_retry = false; goto retry; } @@ -887,30 +1058,35 @@ retry: * return value : -1: recv failed or value of gsocket is invalid * : sizeof(gsocket):succeed */ -int gs_recv_msg_by_unix_domain(int fd, gsocket* gs_sock) +// 此函数的功能是通过Unix域套接字接收消息 +int gs_recv_msg_by_unix_domain(int fd, gsocket *gs_sock) { int error; + // 正常情况应该接收到的消息的大小 int size = (int)sizeof(gsocket); + // 尝试使用阻塞模式的接收函数接收消息 error = mc_tcp_read_block(fd, gs_sock, size, 0); - // recv failed - if (error != size) { + // 如果接收到的消息的大小不等于正常情况的大小，即接收错误，则记录日志并且关闭TCP，返回-1 + if (error != size) + { LIBCOMM_ELOG(WARNING, - "(r|unix domain)\tfailed to recv gs_sock from unix domain, result: %d, error:%s.", - error, - gs_comm_strerror()); + "(r|unix domain)\tfailed to recv gs_sock from unix domain, result: %d, error:%s.", + error, + gs_comm_strerror()); mc_tcp_close(fd); return -1; } - // check the value of receiver gs_sock + // 检查套接字各项参数是否正常，如果不正确则记录报错日志并且返回-1 if ((gs_sock->type != GSOCK_DAUL_CHANNEL) || (gs_sock->idx >= g_instance.comm_cxt.counters_cxt.g_cur_node_num) || - (gs_sock->sid == 0) || (gs_sock->sid >= g_instance.comm_cxt.counters_cxt.g_max_stream_num)) { + (gs_sock->sid == 0) || (gs_sock->sid >= g_instance.comm_cxt.counters_cxt.g_max_stream_num)) + { LIBCOMM_ELOG(WARNING, - "(r|unix domain)\tinvalid gs_sock from unix domain, idx: %d, sid: %d, ver: %d, type: %d.", - gs_sock->idx, - gs_sock->sid, - gs_sock->ver, - gs_sock->type); + "(r|unix domain)\tinvalid gs_sock from unix domain, idx: %d, sid: %d, ver: %d, type: %d.", + gs_sock->idx, + gs_sock->sid, + gs_sock->ver, + gs_sock->type); mc_tcp_close(fd); return -1; } @@ -918,7 +1094,7 @@ int gs_recv_msg_by_unix_domain(int fd, gsocket* gs_sock) return error; } -static int libcomm_build_tcp_connection(libcommaddrinfo* libcomm_addrinfo, int node_idx) +static int libcomm_build_tcp_connection(libcommaddrinfo *libcomm_addrinfo, int node_idx) { struct sock_id fd_id = {-1, -1}; ip_key addr; @@ -935,30 +1111,32 @@ static int libcomm_build_tcp_connection(libcommaddrinfo* libcomm_addrinfo, int n * so we use sctp_port+1 for data connection for tcp mode. */ int sock = mc_tcp_connect(libcomm_addrinfo->host, libcomm_addrinfo->listen_port); - if (sock < 0) { + if (sock < 0) + { LIBCOMM_ELOG(WARNING, - "(s|build tcp connection)\tFailed to build data connection " - "to %s:%d for node[%d]:%s, error[%d:%d]:%s.", - libcomm_addrinfo->host, - libcomm_addrinfo->listen_port, - node_idx, - g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, - error, - errno, - mc_strerror(errno)); + "(s|build tcp connection)\tFailed to build data connection " + "to %s:%d for node[%d]:%s, error[%d:%d]:%s.", + libcomm_addrinfo->host, + libcomm_addrinfo->listen_port, + node_idx, + g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, + error, + errno, + mc_strerror(errno)); return -1; } #ifdef ENABLE_GSS /* Client side gss kerberos authentication for data connection. */ - if (g_instance.comm_cxt.localinfo_cxt.gs_krb_keyfile != NULL && GssClientAuth(sock, libcomm_addrinfo->host) < 0) { + if (g_instance.comm_cxt.localinfo_cxt.gs_krb_keyfile != NULL && GssClientAuth(sock, libcomm_addrinfo->host) < 0) + { LIBCOMM_ELOG(WARNING, - "(s|connect)\tData channel GSS authentication failed, " - "remote:%s[%s:%d]:%s.", - g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, - libcomm_addrinfo->host, - libcomm_addrinfo->listen_port, - mc_strerror(errno)); + "(s|connect)\tData channel GSS authentication failed, " + "remote:%s[%s:%d]:%s.", + g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, + libcomm_addrinfo->host, + libcomm_addrinfo->listen_port, + mc_strerror(errno)); errno = ECOMMTCPGSSAUTHFAIL; // Failed to build sctp connection mc_tcp_close(sock); @@ -968,10 +1146,12 @@ static int libcomm_build_tcp_connection(libcommaddrinfo* libcomm_addrinfo, int n g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]->socket = sock; #ifdef USE_SSL - if (g_instance.attr.attr_network.comm_enable_SSL) { + if (g_instance.attr.attr_network.comm_enable_SSL) + { LibcommPollingStatusType status = LibCommClientSecureOpen(g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1], false); - if (status != LIBCOMM_POLLING_OK) { + if (status != LIBCOMM_POLLING_OK) + { LIBCOMM_ELOG(LOG, "comm_data_channel_conn open ssl failed status:%d\n", status); } } @@ -979,9 +1159,11 @@ static int libcomm_build_tcp_connection(libcommaddrinfo* libcomm_addrinfo, int n fd_id.fd = sock; fd_id.id = 0; - if (gs_update_fd_to_htab_socket_version(&fd_id) < 0) { + if (gs_update_fd_to_htab_socket_version(&fd_id) < 0) + { #ifdef USE_SSL - if (g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]->ssl != NULL) { + if (g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]->ssl != NULL) + { LibCommClientSSLClose(g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]); } #endif @@ -1008,23 +1190,26 @@ static int libcomm_build_tcp_connection(libcommaddrinfo* libcomm_addrinfo, int n msg_head.version = 0; error = LibCommClientWriteBlock( - g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1], (char*)&msg_head, sizeof(MsgHead)); - if (error > 0) { + g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1], (char *)&msg_head, sizeof(MsgHead)); + if (error > 0) + { error = LibCommClientWriteBlock( - g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1], (char*)&connect_package, msg_len); + g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1], (char *)&connect_package, msg_len); } - if (error <= 0) { + if (error <= 0) + { LIBCOMM_ELOG(WARNING, - "(s|build tcp connection)\tFailed to send assoc id to %s:%d " - "for node[%d]:%s on socket[%d].", - libcomm_addrinfo->host, - libcomm_addrinfo->listen_port, - node_idx, - g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, - sock); + "(s|build tcp connection)\tFailed to send assoc id to %s:%d " + "for node[%d]:%s on socket[%d].", + libcomm_addrinfo->host, + libcomm_addrinfo->listen_port, + node_idx, + g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, + sock); #ifdef USE_SSL - if (g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]->ssl != NULL) { + if (g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]->ssl != NULL) + { LibCommClientSSLClose(g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]); } #endif @@ -1032,10 +1217,12 @@ static int libcomm_build_tcp_connection(libcommaddrinfo* libcomm_addrinfo, int n return -1; } - if (gs_map_sock_id_to_node_idx(fd_id, node_idx) < 0) { + if (gs_map_sock_id_to_node_idx(fd_id, node_idx) < 0) + { LIBCOMM_ELOG(WARNING, "(s|build tcp connection)\tFailed to save sock and sockid."); #ifdef USE_SSL - if (g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]->ssl != NULL) { + if (g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]->ssl != NULL) + { LibCommClientSSLClose(g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]); } #endif @@ -1046,26 +1233,30 @@ static int libcomm_build_tcp_connection(libcommaddrinfo* libcomm_addrinfo, int n retry_read: struct libcomm_accept_package ack_msg = {0, 0}; error = LibCommClientReadBlock(g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1], - &ack_msg, sizeof(ack_msg), 0); + &ack_msg, sizeof(ack_msg), 0); // if failed, we close the bad one and return -1 - if (error < 0 || ack_msg.result != 1 || ack_msg.type != LIBCOMM_PKG_TYPE_ACCEPT) { + if (error < 0 || ack_msg.result != 1 || ack_msg.type != LIBCOMM_PKG_TYPE_ACCEPT) + { LIBCOMM_ELOG(WARNING, - "(s|build tcp connection)\tFailed to recv assoc id from %s:%d " - "for node[%d]:%s on socket[%d] error:[%d].", - libcomm_addrinfo->host, - libcomm_addrinfo->listen_port, - node_idx, - g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, - sock, error); + "(s|build tcp connection)\tFailed to recv assoc id from %s:%d " + "for node[%d]:%s on socket[%d] error:[%d].", + libcomm_addrinfo->host, + libcomm_addrinfo->listen_port, + node_idx, + g_instance.comm_cxt.g_s_node_sock[node_idx].remote_nodename, + sock, error); #ifdef USE_SSL - if (g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]->ssl != NULL) { + if (g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]->ssl != NULL) + { LibCommClientSSLClose(g_instance.attr.attr_network.comm_data_channel_conn[node_idx - 1]); } #endif mc_tcp_close(sock); return -1; - } else if (error == 0) { + } + else if (error == 0) + { usleep(1000); goto retry_read; } @@ -1077,7 +1268,8 @@ retry_read: * make sure the connection of control * channel and data channel is same node */ - if (strcmp(g_instance.comm_cxt.g_s_node_sock[node_idx].remote_host, libcomm_addrinfo->host) != 0) { + if (strcmp(g_instance.comm_cxt.g_s_node_sock[node_idx].remote_host, libcomm_addrinfo->host) != 0) + { g_instance.comm_cxt.g_senders->sender_conn[node_idx].ip_changed = false; LIBCOMM_PTHREAD_RWLOCK_UNLOCK(&g_instance.comm_cxt.g_senders->sender_conn[node_idx].rwlock); return -1; @@ -1085,7 +1277,7 @@ retry_read: /* close old connection */ struct sock_id libcomm_fd_id = {g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket, - g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket_id}; + g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket_id}; gs_s_close_bad_data_socket(&libcomm_fd_id, ECOMMTCPPEERCHANGED, node_idx); cpylen = comm_get_cpylen(libcomm_addrinfo->host, HOST_ADDRSTRLEN); @@ -1093,9 +1285,9 @@ retry_read: g_instance.comm_cxt.g_senders->sender_conn[node_idx].remote_host, HOST_ADDRSTRLEN, 0x0, HOST_ADDRSTRLEN); securec_check(ss_rc, "\0", "\0"); ss_rc = strncpy_s(g_instance.comm_cxt.g_senders->sender_conn[node_idx].remote_host, - HOST_ADDRSTRLEN, - libcomm_addrinfo->host, - cpylen + 1); + HOST_ADDRSTRLEN, + libcomm_addrinfo->host, + cpylen + 1); securec_check(ss_rc, "\0", "\0"); g_instance.comm_cxt.g_senders->sender_conn[node_idx].remote_host[cpylen] = '\0'; @@ -1121,13 +1313,13 @@ retry_read: LIBCOMM_PTHREAD_RWLOCK_UNLOCK(&g_instance.comm_cxt.g_senders->sender_conn[node_idx].rwlock); LIBCOMM_ELOG(LOG, - "(s|build tcp connection)\tSucceed to connect %s:%d with socket[%d:%d] for node[%d]:%s.", - libcomm_addrinfo->host, - libcomm_addrinfo->listen_port, - fd_id.fd, - fd_id.id, - node_idx, - REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx)); + "(s|build tcp connection)\tSucceed to connect %s:%d with socket[%d:%d] for node[%d]:%s.", + libcomm_addrinfo->host, + libcomm_addrinfo->listen_port, + fd_id.fd, + fd_id.id, + node_idx, + REMOTE_NAME(g_instance.comm_cxt.g_s_node_sock, node_idx)); return 0; } @@ -1144,7 +1336,7 @@ retry_read: * -1: failed. * 0: succeed. */ -int gs_s_build_tcp_ctrl_connection(libcommaddrinfo* libcomm_addrinfo, int node_idx, bool is_reply) +int gs_s_build_tcp_ctrl_connection(libcommaddrinfo *libcomm_addrinfo, int node_idx, bool is_reply) { int tcp_sock = -1; int ctrl_sock = -1; @@ -1154,61 +1346,70 @@ int gs_s_build_tcp_ctrl_connection(libcommaddrinfo* libcomm_addrinfo, int node_i errno_t ss_rc; uint32 cpylen; char ack = 'r'; - char* remote_host = libcomm_addrinfo->host; + char *remote_host = libcomm_addrinfo->host; int remote_tcp_port = libcomm_addrinfo->ctrl_port; - char* remote_nodename = libcomm_addrinfo->nodename; + char *remote_nodename = libcomm_addrinfo->nodename; // do connect to remote tcp listening port tcp_sock = mc_tcp_connect(remote_host, remote_tcp_port); // failed to connect, report error - if (tcp_sock < 0) { + if (tcp_sock < 0) + { LIBCOMM_ELOG(WARNING, - "(s|connect)\tTCP connect failed to node:%s[%s:%d]:%s.", - remote_nodename, - remote_host, - remote_tcp_port, - mc_strerror(errno)); + "(s|connect)\tTCP connect failed to node:%s[%s:%d]:%s.", + remote_nodename, + remote_host, + remote_tcp_port, + mc_strerror(errno)); errno = ECOMMTCPCONNFAIL; return -1; } #ifdef ENABLE_GSS /* Client side gss kerberos authentication for tcp connection. */ - if (g_instance.comm_cxt.localinfo_cxt.gs_krb_keyfile != NULL && GssClientAuth(tcp_sock, remote_host) < 0) { + if (g_instance.comm_cxt.localinfo_cxt.gs_krb_keyfile != NULL && GssClientAuth(tcp_sock, remote_host) < 0) + { mc_tcp_close(tcp_sock); LIBCOMM_ELOG(WARNING, - "(s|connect)\tControl channel GSS authentication failed, remote:%s[%s:%d]:%s.", - remote_nodename, - remote_host, - remote_tcp_port, - mc_strerror(errno)); + "(s|connect)\tControl channel GSS authentication failed, remote:%s[%s:%d]:%s.", + remote_nodename, + remote_host, + remote_tcp_port, + mc_strerror(errno)); errno = ECOMMTCPGSSAUTHFAIL; return -1; - } else { + } + else + { COMM_DEBUG_LOG("(s|connect)\tControl channel GSS authentication SUCC, remote:%s[%s:%d]:%s.", - remote_nodename, - remote_host, - remote_tcp_port, - mc_strerror(errno)); + remote_nodename, + remote_host, + remote_tcp_port, + mc_strerror(errno)); } #endif - + g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->socket = tcp_sock; #ifdef USE_SSL - if (g_instance.attr.attr_network.comm_enable_SSL) { - LibcommPollingStatusType status = + if (g_instance.attr.attr_network.comm_enable_SSL) + { + LibcommPollingStatusType status = LibCommClientSecureOpen(g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1], false); - if (status != LIBCOMM_POLLING_OK) { + if (status != LIBCOMM_POLLING_OK) + { LIBCOMM_ELOG(LOG, "comm_ctrl_channel_conn secure open failed status:%d\n", status); } } #endif // wait ack from remote node, reject when the state of remote node is incorrect, such as standby mode; struct FCMSG_T fcmsgs = {0x0}; - if (IS_PGXC_COORDINATOR) { + if (IS_PGXC_COORDINATOR) + { fcmsgs.type = CTRL_CONN_REGIST_CN; - } else { + } + else + { fcmsgs.type = CTRL_CONN_REGIST; } @@ -1224,19 +1425,21 @@ int gs_s_build_tcp_ctrl_connection(libcommaddrinfo* libcomm_addrinfo, int node_i fcmsgs.nodename[cpylen] = '\0'; error = gs_send_ctrl_msg_by_socket(g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1], - &fcmsgs, node_idx); - if (error < 0) { + &fcmsgs, node_idx); + if (error < 0) + { #ifdef USE_SSL - if (g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->ssl != NULL) { + if (g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->ssl != NULL) + { LibCommClientSSLClose(g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]); } #endif mc_tcp_close(g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->socket); LIBCOMM_ELOG(WARNING, - "(s|connect)\tSend ctrl msg failed remote[%s] with addr[%s:%d].", - remote_nodename, - remote_host, - remote_tcp_port); + "(s|connect)\tSend ctrl msg failed remote[%s] with addr[%s:%d].", + remote_nodename, + remote_host, + remote_tcp_port); errno = ECOMMTCPCONNFAIL; return -1; } @@ -1244,43 +1447,50 @@ int gs_s_build_tcp_ctrl_connection(libcommaddrinfo* libcomm_addrinfo, int node_i // cn need to ask the remote datanode status when make connection // 'r' is received when remote is standby or pending mode // for conn between dns, skip this step, ip is given by executor - if (IS_PGXC_COORDINATOR) { + if (IS_PGXC_COORDINATOR) + { error = mc_tcp_read_block(tcp_sock, &ack, sizeof(char), 0); - if (error < 0 || ack != 'o') { + if (error < 0 || ack != 'o') + { #ifdef USE_SSL - if (g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->ssl != NULL) { + if (g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->ssl != NULL) + { LibCommClientSSLClose(g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]); } #endif mc_tcp_close(tcp_sock); LIBCOMM_ELOG(WARNING, - "(s|connect)\tControl channel connect reject by remote[%s] with addr[%s:%d], remote is not a primary " - "node.", - remote_nodename, - remote_host, - remote_tcp_port); + "(s|connect)\tControl channel connect reject by remote[%s] with addr[%s:%d], remote is not a primary " + "node.", + remote_nodename, + remote_host, + remote_tcp_port); errno = ECOMMTCPCONNFAIL; return -1; } - } else if (is_reply) { + } + else if (is_reply) + { /* when DN build reply connecion to CN * wait the reply of CN, to make sure CN has received ctrl connection request * then send ctrl msgs to CN */ error = LibCommClientReadBlock(g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1], - &ack, sizeof(char), 0); - if (error < 0 || ack != 'o') { + &ack, sizeof(char), 0); + if (error < 0 || ack != 'o') + { #ifdef USE_SSL - if (g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->ssl != NULL) { + if (g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->ssl != NULL) + { LibCommClientSSLClose(g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]); } #endif mc_tcp_close(tcp_sock); LIBCOMM_ELOG(WARNING, - "(s|connect)\tControl channel connect reject by remote[%s] with addr[%s:%d].", - remote_nodename, - remote_host, - remote_tcp_port); + "(s|connect)\tControl channel connect reject by remote[%s] with addr[%s:%d].", + remote_nodename, + remote_host, + remote_tcp_port); errno = ECOMMTCPCONNFAIL; return -1; } @@ -1288,9 +1498,11 @@ int gs_s_build_tcp_ctrl_connection(libcommaddrinfo* libcomm_addrinfo, int node_i struct sock_id fd_id = {tcp_sock, 0}; // if we successfully to connect, we should record the socket(fd) and the version(id) - if (gs_update_fd_to_htab_socket_version(&fd_id) < 0) { + if (gs_update_fd_to_htab_socket_version(&fd_id) < 0) + { #ifdef USE_SSL - if (g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->ssl != NULL) { + if (g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->ssl != NULL) + { LibCommClientSSLClose(g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]); } #endif @@ -1300,10 +1512,12 @@ int gs_s_build_tcp_ctrl_connection(libcommaddrinfo* libcomm_addrinfo, int node_i return -1; } - if (gs_map_sock_id_to_node_idx(fd_id, node_idx) < 0) { + if (gs_map_sock_id_to_node_idx(fd_id, node_idx) < 0) + { LIBCOMM_ELOG(WARNING, "(s|connect)\tFailed to save sock and sockid."); #ifdef USE_SSL - if (g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->ssl != NULL) { + if (g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]->ssl != NULL) + { LibCommClientSSLClose(g_instance.attr.attr_network.comm_ctrl_channel_conn[node_idx - 1]); } #endif @@ -1315,7 +1529,7 @@ int gs_s_build_tcp_ctrl_connection(libcommaddrinfo* libcomm_addrinfo, int node_i g_instance.comm_cxt.g_senders->sender_conn[node_idx].ip_changed = true; LIBCOMM_PTHREAD_RWLOCK_WRLOCK(&g_instance.comm_cxt.g_senders->sender_conn[node_idx].rwlock); struct sock_id libcomm_fd_id = {g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket, - g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket_id}; + g_instance.comm_cxt.g_senders->sender_conn[node_idx].socket_id}; gs_s_close_bad_data_socket(&libcomm_fd_id, ECOMMTCPPEERCHANGED, node_idx); LIBCOMM_PTHREAD_RWLOCK_UNLOCK(&g_instance.comm_cxt.g_senders->sender_conn[node_idx].rwlock); @@ -1349,7 +1563,8 @@ int gs_s_build_tcp_ctrl_connection(libcommaddrinfo* libcomm_addrinfo, int node_i g_instance.comm_cxt.g_s_node_sock[node_idx].set_nl(remote_tcp_port, CTRL_TCP_PORT); g_instance.comm_cxt.g_s_node_sock[node_idx].ip_changed = false; /* add the socket to the epoll list for monitoring network events */ - if (g_instance.comm_cxt.pollers_cxt.g_s_poller_list->add_fd(&fd_id) < 0) { + if (g_instance.comm_cxt.pollers_cxt.g_s_poller_list->add_fd(&fd_id) < 0) + { g_instance.comm_cxt.g_s_node_sock[node_idx].close_socket_nl(CTRL_TCP_SOCK); g_instance.comm_cxt.g_s_node_sock[node_idx].set_nl(-1, CTRL_TCP_PORT); g_instance.comm_cxt.g_s_node_sock[node_idx].unlock(); @@ -1375,15 +1590,15 @@ int gs_s_build_tcp_ctrl_connection(libcommaddrinfo* libcomm_addrinfo, int node_i g_instance.comm_cxt.g_s_node_sock[node_idx].unlock(); LIBCOMM_ELOG(LOG, - "(s|connect)\tTCP connect successed to node:%s[%s:%d] on socket[%d,%d] with node id[%d].", - remote_nodename, - remote_host, - remote_tcp_port, - fd_id.fd, - fd_id.id, - node_idx); + "(s|connect)\tTCP connect successed to node:%s[%s:%d] on socket[%d,%d] with node id[%d].", + remote_nodename, + remote_host, + remote_tcp_port, + fd_id.fd, + fd_id.id, + node_idx); return 0; -} // gs_s_build_tcp_ctrl_connection +} // gs_s_build_tcp_ctrl_connection // Registed a Consumer callback function to wake up the Consumer (thread at executor) when a Producer (thread at // executor) connected successfully @@ -1391,7 +1606,7 @@ int gs_s_build_tcp_ctrl_connection(libcommaddrinfo* libcomm_addrinfo, int node_i void gs_connect_regist_callback(wakeup_hook_type wakeup_callback) { g_instance.comm_cxt.gs_wakeup_consumer = wakeup_callback; -} // gs_connect_regist_callback +} // gs_connect_regist_callback void gs_init_adapt_layer() { @@ -1429,9 +1644,11 @@ void gs_delay_survey() securec_check(ss_rc, "\0", "\0"); msg.type = LIBCOMM_PKG_TYPE_DELAY_REQUEST; - for (node_idx = 0; node_idx < g_instance.comm_cxt.counters_cxt.g_cur_node_num; node_idx++) { + for (node_idx = 0; node_idx < g_instance.comm_cxt.counters_cxt.g_cur_node_num; node_idx++) + { // if the connection is not ready, continue - if (g_instance.comm_cxt.g_senders->sender_conn[node_idx].assoc_id == 0) { + if (g_instance.comm_cxt.g_senders->sender_conn[node_idx].assoc_id == 0) + { continue; } @@ -1446,7 +1663,7 @@ void gs_delay_survey() send_info.node_idx = node_idx; send_info.streamid = 0; send_info.version = 0; - send_info.msg = (char*)&msg; + send_info.msg = (char *)&msg; send_info.msg_len = sizeof(struct libcomm_delay_package); (void)g_libcomm_adapt.block_send(&send_info); @@ -1459,7 +1676,8 @@ int CommEpollCreate(int size) { CommSetEpollOption(CommEpollThreadPoolListener); int epfd = comm_epoll_create(size); - if (epfd < 0) { + if (epfd < 0) + { LIBCOMM_ELOG(WARNING, "Trace: CommEpollCreate epoll_create failed, detail: epfd[%d]%m", epfd); return epfd; } @@ -1471,20 +1689,25 @@ int CommEpollCreate(int size) int CommEpollCtl(int epfd, int op, int fd, struct epoll_event *event) { int rc = -1; - /* + /* * If the current connection is libcomm, the upper-layer fd value is -1. * When the current connection is libpq, the upper-layer fd value is a valid value. * The CN DN logical connection function must be compatible with the preceding two modes. */ - if (fd >= 0) { + if (fd >= 0) + { rc = comm_epoll_ctl(epfd, op, fd, event); - if (rc < 0) { + if (rc < 0) + { LIBCOMM_ELOG(WARNING, "Trace: CommEpollCtl epoll_ctl failed, detail: epfd[%d]%m", epfd); return rc; } - } else { + } + else + { CommEpollFd *comm_epfd = GetCommEpollFd(epfd); - if (comm_epfd != NULL) { + if (comm_epfd != NULL) + { rc = comm_epfd->EpollCtl(op, fd, event); } } @@ -1494,7 +1717,8 @@ int CommEpollCtl(int epfd, int op, int fd, struct epoll_event *event) int CommEpollWait(int epfd, struct epoll_event *event, int maxevents, int timeout) { CommEpollFd *comm_epfd = GetCommEpollFd(epfd); - if ((!comm_epfd) || (maxevents <= 0)) { + if ((!comm_epfd) || (maxevents <= 0)) + { LIBCOMM_ELOG(ERROR, "Trace: error, epfd %d not found or maxevents <= 0 (=%d)", epfd, maxevents); errno = (maxevents <= 0) ? EINVAL : EBADF; return -1; @@ -1506,7 +1730,8 @@ int CommEpollWait(int epfd, struct epoll_event *event, int maxevents, int timeou int CommEpollClose(int epfd) { - if (epfd < 0) { + if (epfd < 0) + { Assert(epfd >= 0); LIBCOMM_ELOG(ERROR, "Trace: CommEpollClose failed, detail epfd[%d]%m.", epfd); return -1; @@ -1522,18 +1747,19 @@ int CommEpollClose(int epfd) void InitCommLogicResource() { - if (g_instance.comm_logic_cxt.comm_fd_collection != NULL) { + if (g_instance.comm_logic_cxt.comm_fd_collection != NULL) + { Assert(0); LIBCOMM_ELOG(ERROR, "Trace: InitCommLogicResource failed, detail comm_fd_collection is null."); return; } g_instance.comm_logic_cxt.comm_logic_mem_cxt = AllocSetContextCreate(g_instance.instance_context, - "CommLogicMemCxt", - ALLOCSET_DEFAULT_MINSIZE, - ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE, - SHARED_CONTEXT); + "CommLogicMemCxt", + ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE, + SHARED_CONTEXT); g_instance.comm_logic_cxt.comm_fd_collection = New(g_instance.comm_logic_cxt.comm_logic_mem_cxt) FdCollection(); g_instance.comm_logic_cxt.comm_fd_collection->Init(); return; @@ -1541,14 +1767,17 @@ void InitCommLogicResource() void ProcessCommLogicTearDown() { - if (g_instance.comm_logic_cxt.comm_fd_collection) { + if (g_instance.comm_logic_cxt.comm_fd_collection) + { g_instance.comm_logic_cxt.comm_fd_collection->DeInit(); delete g_instance.comm_logic_cxt.comm_fd_collection; MemoryContextDelete(g_instance.comm_logic_cxt.comm_logic_mem_cxt); g_instance.comm_logic_cxt.comm_logic_mem_cxt = NULL; LIBCOMM_ELOG(LOG, "Trace: ProcessCommLogicTearDown success"); - } else { + } + else + { LIBCOMM_ELOG(WARNING, "Trace: ProcessCommLogicTearDown failed, detail comm_fd_collection is null."); } } @@ -1556,7 +1785,8 @@ void ProcessCommLogicTearDown() #define INNER_INTERFACES_API CommEpollFd *GetCommEpollFd(int epfd) { - if (g_instance.comm_logic_cxt.comm_fd_collection) { + if (g_instance.comm_logic_cxt.comm_fd_collection) + { return g_instance.comm_logic_cxt.comm_fd_collection->GetCommEpollFd(epfd); } return NULL; @@ -1564,14 +1794,16 @@ CommEpollFd *GetCommEpollFd(int epfd) void LogicEpollCreate(int epfd, int size) { - if (g_instance.comm_logic_cxt.comm_fd_collection) { + if (g_instance.comm_logic_cxt.comm_fd_collection) + { g_instance.comm_logic_cxt.comm_fd_collection->AddEpfd(epfd, size); } } void CloseLogicEpfd(int epfd) { - if (g_instance.comm_logic_cxt.comm_fd_collection) { + if (g_instance.comm_logic_cxt.comm_fd_collection) + { g_instance.comm_logic_cxt.comm_fd_collection->DelEpfd(epfd); } } @@ -1598,7 +1830,7 @@ void FdCollection::Init() /* Init epfd hash table */ HASHCTL hinfo; securec_check_c((memset_s(&hinfo, sizeof(hinfo), 0, sizeof(hinfo))), "", "") - hinfo.keysize = sizeof(int); + hinfo.keysize = sizeof(int); hinfo.entrysize = sizeof(CommEpFdInfo); hinfo.hcxt = g_instance.comm_logic_cxt.comm_logic_mem_cxt; m_epfd_htab = hash_create("epfd_htab", MAX_NUMA_NODE, &hinfo, HASH_ELEM | HASH_CONTEXT); @@ -1606,7 +1838,7 @@ void FdCollection::Init() /* Init logicfd hash table */ securec_check_c((memset_s(&hinfo, sizeof(hinfo), 0, sizeof(hinfo))), "", "") - hinfo.keysize = sizeof(LogicFd); + hinfo.keysize = sizeof(LogicFd); hinfo.entrysize = sizeof(SessionInfo); hinfo.hcxt = g_instance.comm_logic_cxt.comm_logic_mem_cxt; hinfo.hash = tag_hash; @@ -1628,7 +1860,8 @@ void FdCollection::DeInit() inline bool FdCollection::IsEpfdValid(int fd) { - if (fd < 0) { + if (fd < 0) + { return false; } return true; @@ -1636,12 +1869,14 @@ inline bool FdCollection::IsEpfdValid(int fd) int FdCollection::AddEpfd(int epfd, int size) { - if (!IsEpfdValid(epfd)) { + if (!IsEpfdValid(epfd)) + { return -1; } CommEpollFd *comm_epfd = GetCommEpollFd(epfd); - if (comm_epfd) { + if (comm_epfd) + { Assert(0); LIBCOMM_ELOG(ERROR, "Trace: AddEpfd failed, detail: epfd[%d] is already created.", epfd); return -1; @@ -1649,7 +1884,8 @@ int FdCollection::AddEpfd(int epfd, int size) AutoContextSwitch acontext(g_instance.comm_logic_cxt.comm_logic_mem_cxt); comm_epfd = New(g_instance.comm_logic_cxt.comm_logic_mem_cxt) CommEpollFd(epfd, size); - if (comm_epfd == NULL) { + if (comm_epfd == NULL) + { LIBCOMM_ELOG(ERROR, "Trace: FdCollection::AddEpfd New CommEpollFd failed, detail: epfd[%d]%m", epfd); return -1; } @@ -1657,8 +1893,9 @@ int FdCollection::AddEpfd(int epfd, int size) comm_epfd->Init(epfd); AutoMutexLock mutex(&m_epfd_htab_lock); mutex.lock(); - CommEpFdInfo *comm_epfd_info = (CommEpFdInfo*)hash_search(m_epfd_htab, &epfd, HASH_ENTER, NULL); - if (comm_epfd_info == NULL) { + CommEpFdInfo *comm_epfd_info = (CommEpFdInfo *)hash_search(m_epfd_htab, &epfd, HASH_ENTER, NULL); + if (comm_epfd_info == NULL) + { mutex.unLock(); LIBCOMM_ELOG(ERROR, "Trace: AddEpfd HASH_ENTER failed, detail: epfd[%d]%m", epfd); return -1; @@ -1675,24 +1912,29 @@ int FdCollection::DelEpfd(int epfd) CommEpollFd *comm_epfd = NULL; AutoMutexLock mutex(&m_epfd_htab_lock); mutex.lock(); - CommEpFdInfo *comm_epfd_info = (CommEpFdInfo*)hash_search(m_epfd_htab, &epfd, HASH_FIND, NULL); - if (comm_epfd_info) { + CommEpFdInfo *comm_epfd_info = (CommEpFdInfo *)hash_search(m_epfd_htab, &epfd, HASH_FIND, NULL); + if (comm_epfd_info) + { comm_epfd = comm_epfd_info->comm_epfd; comm_epfd_info->comm_epfd = NULL; - } else { + } + else + { LIBCOMM_ELOG(WARNING, "Trace: DelEpfd HASH_FIND failed, detail: epfd[%d] is already deleted.", epfd); mutex.unLock(); return 0; } - if (hash_search(m_epfd_htab, &epfd, HASH_REMOVE, NULL) == NULL) { + if (hash_search(m_epfd_htab, &epfd, HASH_REMOVE, NULL) == NULL) + { mutex.unLock(); LIBCOMM_ELOG(ERROR, "Trace: DelEpfd HASH_REMOVE failed, Detail epfd[%d]%m", epfd); return -1; } mutex.unLock(); - if (comm_epfd) { + if (comm_epfd) + { comm_epfd->DeInit(); delete comm_epfd; } @@ -1709,15 +1951,20 @@ void FdCollection::CleanEpfd() AutoMutexLock mutex(&m_epfd_htab_lock); mutex.lock(); hash_seq_init(&hash_seq_status, m_epfd_htab); - while ((entry = (CommEpFdInfo *)hash_seq_search(&hash_seq_status))) { - if (entry->comm_epfd) { + while ((entry = (CommEpFdInfo *)hash_seq_search(&hash_seq_status))) + { + if (entry->comm_epfd) + { delete entry->comm_epfd; entry->comm_epfd = NULL; - } else { + } + else + { LIBCOMM_ELOG(WARNING, "Trace: CleanEpfd failed, detail: epfd[%d] is already deleted", entry->epfd); } - if (hash_search(m_logicfd_htab, &entry->epfd, HASH_REMOVE, NULL) == NULL) { + if (hash_search(m_logicfd_htab, &entry->epfd, HASH_REMOVE, NULL) == NULL) + { LIBCOMM_ELOG(ERROR, "Trace: CleanEpfd HASH_REMOVE failed, detail: epfd[%d]", entry->epfd); } clear_cnt++; @@ -1728,47 +1975,50 @@ void FdCollection::CleanEpfd() return; } -inline CommEpollFd* FdCollection::GetCommEpollFd(int epfd) +inline CommEpollFd *FdCollection::GetCommEpollFd(int epfd) { - if (!IsEpfdValid(epfd)) { + if (!IsEpfdValid(epfd)) + { return NULL; } AutoMutexLock mutex(&m_epfd_htab_lock); mutex.lock(); - CommEpFdInfo *comm_epfd_info = (CommEpFdInfo*)hash_search(m_epfd_htab, &epfd, HASH_FIND, NULL); + CommEpFdInfo *comm_epfd_info = (CommEpFdInfo *)hash_search(m_epfd_htab, &epfd, HASH_FIND, NULL); mutex.unLock(); return (comm_epfd_info ? comm_epfd_info->comm_epfd : NULL); } int FdCollection::AddLogicFd(CommEpollFd *comm_epfd, void *session_ptr) { - knl_session_context *session = (knl_session_context*)session_ptr; + knl_session_context *session = (knl_session_context *)session_ptr; SessionInfo *session_info = NULL; LogicFd logic_fd = {session->proc_cxt.MyProcPort->gs_sock.idx, session->proc_cxt.MyProcPort->gs_sock.sid}; AutoMutexLock mutex(&m_logicfd_htab_lock); mutex.lock(); - session_info = (SessionInfo*)hash_search(m_logicfd_htab, &logic_fd, HASH_FIND, NULL); - if (session_info != NULL) { + session_info = (SessionInfo *)hash_search(m_logicfd_htab, &logic_fd, HASH_FIND, NULL); + if (session_info != NULL) + { LIBCOMM_ELOG(WARNING, - "Trace: AddLogicFd HASH_FIND session_info is not null, detail: epfd[%d], idx[%d], sid[%d], " - "in_hash: {epfd[%d], idx[%d], streamid[%d]}", - comm_epfd->m_epfd, logic_fd.idx, logic_fd.streamid, - session_info->comm_epfd_ptr->m_epfd, - session_info->commfd.logic_fd.idx, - session_info->commfd.logic_fd.sid); + "Trace: AddLogicFd HASH_FIND session_info is not null, detail: epfd[%d], idx[%d], sid[%d], " + "in_hash: {epfd[%d], idx[%d], streamid[%d]}", + comm_epfd->m_epfd, logic_fd.idx, logic_fd.streamid, + session_info->comm_epfd_ptr->m_epfd, + session_info->commfd.logic_fd.idx, + session_info->commfd.logic_fd.sid); mutex.unLock(); DelLogicFd(&logic_fd); mutex.lock(); } AutoContextSwitch acontext(g_instance.comm_logic_cxt.comm_logic_mem_cxt); - session_info = (SessionInfo*)hash_search(m_logicfd_htab, &logic_fd, HASH_ENTER, NULL); - if (session_info == NULL) { + session_info = (SessionInfo *)hash_search(m_logicfd_htab, &logic_fd, HASH_ENTER, NULL); + if (session_info == NULL) + { mutex.unLock(); LIBCOMM_ELOG(ERROR, - "Trace: AddLogicFd HASH_ENTER failed, detail: idx[%d], sid[%d]%m", logic_fd.idx, logic_fd.streamid); + "Trace: AddLogicFd HASH_ENTER failed, detail: idx[%d], sid[%d]%m", logic_fd.idx, logic_fd.streamid); return -1; } @@ -1783,12 +2033,12 @@ int FdCollection::AddLogicFd(CommEpollFd *comm_epfd, void *session_ptr) mutex.unLock(); session->session_info_ptr = session_info; LIBCOMM_ELOG(LOG, - "Trace: AddLogicFd success, detail: epfd[%d], idx[%d], sid[%d]" - "in_hash:{epfd[%d], idx[%d], streamid[%d]}, ", - comm_epfd->m_epfd, logic_fd.idx, logic_fd.streamid, - session_info->comm_epfd_ptr->m_epfd, - session_info->commfd.logic_fd.idx, - session_info->commfd.logic_fd.sid); + "Trace: AddLogicFd success, detail: epfd[%d], idx[%d], sid[%d]" + "in_hash:{epfd[%d], idx[%d], streamid[%d]}, ", + comm_epfd->m_epfd, logic_fd.idx, logic_fd.streamid, + session_info->comm_epfd_ptr->m_epfd, + session_info->commfd.logic_fd.idx, + session_info->commfd.logic_fd.sid); WakeupSession(session_info, false, __FUNCTION__); return 0; @@ -1798,27 +2048,29 @@ int FdCollection::DelLogicFd(const LogicFd *logic_fd) { AutoMutexLock mutex(&m_logicfd_htab_lock); mutex.lock(); - SessionInfo *session_info = (SessionInfo*)hash_search(m_logicfd_htab, logic_fd, HASH_FIND, NULL); - if (session_info == NULL) { + SessionInfo *session_info = (SessionInfo *)hash_search(m_logicfd_htab, logic_fd, HASH_FIND, NULL); + if (session_info == NULL) + { mutex.unLock(); LIBCOMM_ELOG(WARNING, - "Trace: DelLogicFd HASH_FIND failed, detail: idx[%d], sid[%d] is already deleted.", - logic_fd->idx, logic_fd->streamid); + "Trace: DelLogicFd HASH_FIND failed, detail: idx[%d], sid[%d] is already deleted.", + logic_fd->idx, logic_fd->streamid); return 0; } int epfd = session_info->comm_epfd_ptr->m_epfd; - if (hash_search(m_logicfd_htab, logic_fd, HASH_REMOVE, NULL) == NULL) { + if (hash_search(m_logicfd_htab, logic_fd, HASH_REMOVE, NULL) == NULL) + { mutex.unLock(); LIBCOMM_ELOG(ERROR, "Trace: DelLogicFd HASH_REMOVE failed, Detail epfd[%d], idx[%d], sid[%d]%m", - epfd, logic_fd->idx, logic_fd->streamid); + epfd, logic_fd->idx, logic_fd->streamid); return -1; } m_logicfd_nums--; mutex.unLock(); LIBCOMM_ELOG(LOG, "Trace: DelLogicFd HASH_REMOVE success, Detail epfd[%d], idx[%d], sid[%d]", - epfd, logic_fd->idx, logic_fd->streamid); + epfd, logic_fd->idx, logic_fd->streamid); return 0; } @@ -1831,7 +2083,8 @@ void FdCollection::CleanLogicFd() AutoMutexLock mutex(&m_logicfd_htab_lock); mutex.lock(); hash_seq_init(&hash_seq_status, m_logicfd_htab); - while ((entry = (SessionInfo *)hash_seq_search(&hash_seq_status))) { + while ((entry = (SessionInfo *)hash_seq_search(&hash_seq_status))) + { hash_search(m_logicfd_htab, &entry->logic_fd, HASH_REMOVE, NULL); clear_cnt++; } @@ -1841,11 +2094,11 @@ void FdCollection::CleanLogicFd() return; } -SessionInfo* FdCollection::GetSessionInfo(const LogicFd *logic_fd) +SessionInfo *FdCollection::GetSessionInfo(const LogicFd *logic_fd) { AutoMutexLock mutex(&m_logicfd_htab_lock); mutex.lock(); - SessionInfo *session_info = (SessionInfo*)hash_search(m_logicfd_htab, logic_fd, HASH_FIND, NULL); + SessionInfo *session_info = (SessionInfo *)hash_search(m_logicfd_htab, logic_fd, HASH_FIND, NULL); mutex.unLock(); return session_info; } @@ -1881,12 +2134,14 @@ int CommEpollFd::EpollCtl(int op, int fd, const struct epoll_event *event) int rc = 0; /* Input parameter validation */ - if (event == NULL) { + if (event == NULL) + { Assert(event); LIBCOMM_ELOG(ERROR, "Trace: CommEpollCtl op[%d]: event is null", op); return -1; } - if (op != EPOLL_CTL_ADD && op != EPOLL_CTL_MOD && op != EPOLL_CTL_DEL) { + if (op != EPOLL_CTL_ADD && op != EPOLL_CTL_MOD && op != EPOLL_CTL_DEL) + { Assert(0); LIBCOMM_ELOG(ERROR, "Trace: CommEpollCtl: Incorrect operator op[%d] ", op); return -1; @@ -1895,55 +2150,68 @@ int CommEpollFd::EpollCtl(int op, int fd, const struct epoll_event *event) knl_session_context *session = GetSessionBaseOnEvent(event); int idx = session->proc_cxt.MyProcPort->gs_sock.idx; int streamid = session->proc_cxt.MyProcPort->gs_sock.sid; - if (idx <= 0 || streamid <= 0) { + if (idx <= 0 || streamid <= 0) + { LIBCOMM_ELOG(WARNING, "Trace: CommEpollFd::EpollCtl invalid params, Detail idx[%d], streamid[%d]", - idx, streamid); + idx, streamid); return -1; } - if (op == EPOLL_CTL_ADD) { + if (op == EPOLL_CTL_ADD) + { rc = RegisterNewSession(session); - } else if (op == EPOLL_CTL_MOD) { + } + else if (op == EPOLL_CTL_MOD) + { rc = ResetSession(session); - } else if (op == EPOLL_CTL_DEL) { + } + else if (op == EPOLL_CTL_DEL) + { rc = UnRegisterSession(session); } - if (rc == 0) { + if (rc == 0) + { LIBCOMM_ELOG(LOG, "Trace: CommEpollFd::EpollCtl op[%d] success, Detail idx[%d], streamid[%d]", - op, idx, streamid); - } else { + op, idx, streamid); + } + else + { LIBCOMM_ELOG(ERROR, "Trace: CommEpollFd::EpollCtl op[%d] failed, Detail idx[%d], streamid[%d]", - op, idx, streamid); + op, idx, streamid); } return 0; } -int CommEpollFd::RegisterNewSession(knl_session_context* session) +int CommEpollFd::RegisterNewSession(knl_session_context *session) { return m_comm_fd_collection->AddLogicFd(this, session); } -int CommEpollFd::ResetSession(const knl_session_context* session) +int CommEpollFd::ResetSession(const knl_session_context *session) { SessionInfo *session_info = session->session_info_ptr; - if (gs_compare_and_swap_32(&session_info->is_idle, (int)false, (int)true) == false) { + if (gs_compare_and_swap_32(&session_info->is_idle, (int)false, (int)true) == false) + { LIBCOMM_ELOG(WARNING, "Trace: ResetSession, invalid status " - "detail: idx[%d], streamid[%d], is_idle[false]", - session_info->commfd.logic_fd.idx, - session_info->commfd.logic_fd.sid); + "detail: idx[%d], streamid[%d], is_idle[false]", + session_info->commfd.logic_fd.idx, + session_info->commfd.logic_fd.sid); return 0; } - if (CheckError(session_info)) { + if (CheckError(session_info)) + { WakeupSession(session_info, true, __FUNCTION__); - } else if (CheckEvent(session_info)) { + } + else if (CheckEvent(session_info)) + { WakeupSession(session_info, false, __FUNCTION__); } return 0; } -int CommEpollFd::UnRegisterSession(const knl_session_context* session) +int CommEpollFd::UnRegisterSession(const knl_session_context *session) { Assert(session); LogicFd logic_fd = {session->proc_cxt.MyProcPort->gs_sock.idx, session->proc_cxt.MyProcPort->gs_sock.sid}; @@ -1952,7 +2220,8 @@ int CommEpollFd::UnRegisterSession(const knl_session_context* session) void CommEpollFd::PushReadyEvent(SessionInfo *session_info, bool err_occurs) { - if (session_info == NULL) { + if (session_info == NULL) + { LIBCOMM_ELOG(ERROR, "Trace: PushReadyEvent: session_info is null, epfd[%d]", m_epfd); return; } @@ -1964,15 +2233,15 @@ void CommEpollFd::PushReadyEvent(SessionInfo *session_info, bool err_occurs) const bool CommEpollFd::CheckError(const SessionInfo *session_info) { - struct c_mailbox* cmailbox = &(C_MAILBOX(session_info->commfd.logic_fd.idx, session_info->commfd.logic_fd.sid)); + struct c_mailbox *cmailbox = &(C_MAILBOX(session_info->commfd.logic_fd.idx, session_info->commfd.logic_fd.sid)); return ((cmailbox != NULL && cmailbox->state == MAIL_CLOSED) || - g_instance.comm_cxt.g_receivers->receiver_conn[session_info->commfd.logic_fd.idx].socket == -1 || - (cmailbox != NULL && session_info->commfd.logic_fd.ver != cmailbox->local_version)); + g_instance.comm_cxt.g_receivers->receiver_conn[session_info->commfd.logic_fd.idx].socket == -1 || + (cmailbox != NULL && session_info->commfd.logic_fd.ver != cmailbox->local_version)); } const bool CommEpollFd::CheckEvent(const SessionInfo *session_info) { - struct c_mailbox* cmailbox = &(C_MAILBOX(session_info->commfd.logic_fd.idx, session_info->commfd.logic_fd.sid)); + struct c_mailbox *cmailbox = &(C_MAILBOX(session_info->commfd.logic_fd.idx, session_info->commfd.logic_fd.sid)); return (cmailbox != NULL && cmailbox->buff_q->is_empty == 0); } @@ -1987,25 +2256,32 @@ int CommEpollFd::EpollWait(int epfd, epoll_event *events, int maxevents, int tim Assert(epfd == m_epfd); m_all_ready_fds = 0; - do { + do + { rc = GetReadyEvents(); - if (rc > 0) { + if (rc > 0) + { return rc; } ready_fds = comm_epoll_wait(m_epfd, m_ready_events, m_maxevents, timeout); - if (ready_fds < 0) { + if (ready_fds < 0) + { LIBCOMM_ELOG(WARNING, "Trace: EpollWait epfd[%d], ready_fds[%d]", m_epfd, ready_fds); return ready_fds; } - for (i = 0; i < ready_fds; i++) { + for (i = 0; i < ready_fds; i++) + { fd = m_ready_events[i].data.fd; - if (IsWakeupFd(fd)) { + if (IsWakeupFd(fd)) + { rc = GetReadyEvents(); RemoveWakeupFd(); - } else { + } + else + { m_events[m_all_ready_fds++] = m_ready_events[i]; } } @@ -2016,9 +2292,11 @@ int CommEpollFd::EpollWait(int epfd, epoll_event *events, int maxevents, int tim int CommEpollFd::GetReadyEvents() { SessionInfo *item = NULL; - while (m_ready_events_queue.size()) { + while (m_ready_events_queue.size()) + { item = m_ready_events_queue.pop(item); - if (item == NULL) { + if (item == NULL) + { continue; } @@ -2028,4 +2306,4 @@ int CommEpollFd::GetReadyEvents() m_all_ready_fds++; } return m_all_ready_fds; -} +} \ No newline at end of file -- 2.34.1 From 5f5ef9b4eab31ad7b1f9e9adcf5dc30e5304477f Mon Sep 17 00:00:00 2001 From: dzlsgdhr Date: Tue, 3 Oct 2023 16:50:07 +0800 Subject: [PATCH 10/15] Update libcomm_util.cpp --- .../libcomm_utils/libcomm_util.cpp | 285 +++++++++++------- 1 file changed, 178 insertions(+), 107 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_util.cpp b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_util.cpp index 22461383e..34c305b73 100644 --- a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_util.cpp +++ b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_util.cpp @@ -36,22 +36,24 @@ extern long libpq_used_memory; extern long libcomm_used_memory; extern long comm_peak_used_memory; - +//这个函数设置了全局实例的通信上下文中的可用内存量。 +// 参数usable_memory表示可用内存量，单位是KB。函数内部将其乘以1024，转换为字节。 +// 然后将其保存在全局实例的通信上下文的commutil_cxt成员中的g_total_usable_memory成员变量中。 void gs_set_usable_memory(long usable_memory) { g_instance.comm_cxt.commutil_cxt.g_total_usable_memory = usable_memory * 1024; } - +//为数据恢复函数设立存储空间，进行数据的内容存放 void gs_set_memory_pool_size(long mem_pool) { g_instance.comm_cxt.commutil_cxt.g_memory_pool_size = mem_pool * 1024; } - +//获得内存空间，该内存空间为进行通信传输所需要的内存空间，分别为通信进行过程中的内存空间和提供应用程序接口的内存空间 long gs_get_comm_used_memory(void) { return libcomm_used_memory + libpq_used_memory; } - +//获取内存空间，该内存空间为通信传递所需的最大的内存空间，也就是为libcomm_used_memory + libpq_used_memory的最大值 long gs_get_comm_peak_memory(void) { return comm_peak_used_memory; @@ -65,40 +67,51 @@ Size gs_get_comm_context_memory(void) return 0; } } - -int cmp_addr(sockaddr_storage_t* addr1, sockaddr_storage_t* addr2) +/*这个函数是用来比较两个套接字地址（sockaddr_storage_t类型）是否相等的。 +首先如果这两个地址的协议族（IPv4和IPv6）不同，返回-1。如果相同开始进行判断，如果是IPv6的情况， +判断接口是否相等，如果相等返回IPv6的地址位置，地址大小。同理对IPv4进行判断，如果接口对应返回IPv6的地址位置， +地址大小。该函数为地址判断形函数，在满足地址相同的情况下对地址的接口类型进行判断，观察是否满足函数调用原则，如果满足则对函数的地址进行输出。*/ +int cmp_addr(sockaddr_storage_t *addr1, + sockaddr_storage_t *addr2) // 定义函数cmp_addr，接收两个指向sockaddr_storage_t类型的指针作为参数 { - if (addr1->sa.sa_family != addr2->sa.sa_family) { - return -1; + if (addr1->sa.sa_family != addr2->sa.sa_family) { // 如果这两个地址的协议族（比如IPv4和IPv6）不同 + return -1; // 返回-1表示这两个地址不相等 } - switch (addr1->sa.sa_family) { - case AF_INET6: - if (addr1->v6.sin6_port != addr2->v6.sin6_port) { - return -1; + switch (addr1->sa.sa_family) { // 根据协议族进行不同的比较 + case AF_INET6: // 如果是IPv6 + if (addr1->v6.sin6_port != addr2->v6.sin6_port) { // 如果这两个地址的端口号不同 + return -1; // 返回-1表示这两个地址不相等 } - return memcmp(&addr1->v6.sin6_addr, &addr2->v6.sin6_addr, sizeof(addr1->v6.sin6_addr)); - case AF_INET: - if (addr1->v4.sin_port != addr2->v4.sin_port) { - return -1; + return memcmp( + &addr1->v6.sin6_addr, &addr2->v6.sin6_addr, + sizeof(addr1->v6.sin6_addr)); // 使用memcmp函数比较IPv6地址部分是否相等，如果相等返回0，不相等返回非0值 + case AF_INET: // 如果是IPv4 + if (addr1->v4.sin_port != addr2->v4.sin_port) { // 如果这两个地址的端口号不同 + return -1; // 返回-1表示这两个地址不相等 } - return memcmp(&addr1->v4.sin_addr, &addr2->v4.sin_addr, sizeof(addr1->v4.sin_addr)); - default: - return -1; + return memcmp( + &addr1->v4.sin_addr, &addr2->v4.sin_addr, + sizeof(addr1->v4.sin_addr)); // 使用memcmp函数比较IPv4地址部分是否相等，如果相等返回0，不相等返回非0值 + default: // 如果协议族不是IPv4也不是IPv6 + return -1; // 返回-1表示这两个地址不相等 } } - +//查找程序错误的判断参数 void set_debug_mode(bool mod) { g_instance.comm_cxt.commutil_cxt.g_debug_mode = mod; } +//时间设置函数的判断参数 void set_timer_mode(bool mod) { g_instance.comm_cxt.commutil_cxt.g_timer_mode = mod; } +//生命函数的判断参数 void set_stat_mode(bool mod) { g_instance.comm_cxt.commutil_cxt.g_stat_mode = mod; } +//延时函数的判断参数 void set_no_delay(bool mod) { g_instance.comm_cxt.commutil_cxt.g_no_delay = mod; @@ -107,54 +120,55 @@ void set_no_delay(bool mod) // set FL or FD attribute of socket by parameter FL_OR_FD, // FL_OR_FD is 0 for setting FL (O_NONBLOCK) // FL_OR_FD is 1 for setting FD (FD_CLOEXEC) -// -int set_socketopt(int sock, int FL_OR_FD, long arg) +//这段文字是对设置socket的FL或FD属性的一种说明。其中，FL_OR_FD是一个参数，当它值为0时，表示设置FL属性（标记为O_NONBLOCK）。这个O_NONBLOCK标记表示该socket将处于非阻塞模式，意味着对socket的操作不会阻塞程序的执行。当FL_OR_FD的值为1时，表示设置FD属性（标记为FD_CLOEXEC）。这个FD_CLOEXEC标记表示在执行新的程序或命令时，这个socket将被关闭。 +//函数在开头对FL以及FD进行了选择，并且根据不同情况分别返回对应的FL或者FD的值。后面对socket的fcntl函数进行了判定，排除两种小于零的错误情况，最后返回RC值表示成功。 + // 该函数对socket的类型进行判定，表达了何时为非阻塞模式，何时为运行模式，该函数表明了进程的状态，防止进程出现错误。 int set_socketopt( + int sock, int FL_OR_FD, + long arg) // 定义一个函数set_socketopt，接收三个参数：socket文件描述符，标志位FL_OR_FD，以及长整型参数arg { - int flg, rc; - int get_cmd = 0; - int set_cmd = 0; - switch (FL_OR_FD) { - case 0: // FL - get_cmd = F_GETFL; - set_cmd = F_SETFL; + int flg, rc; // 定义两个整型变量flg和rc，用于存储fcntl函数的返回值和设置选项的结果 + int get_cmd = 0; // 定义一个整型变量get_cmd，并初始化为0，用于存储获取选项的命令 + int set_cmd = 0; // 定义一个整型变量set_cmd，并初始化为0，用于存储设置选项的命令 + switch (FL_OR_FD) { // 根据FL_OR_FD的值进行不同的操作 + case 0: // FL + get_cmd = F_GETFL; // 如果FL_OR_FD为0，即要获取的是文件状态标志，因此设置get_cmd为F_GETFL + set_cmd = F_SETFL; // 如果FL_OR_FD为0，即要设置的是文件状态标志，因此设置set_cmd为F_SETFL break; - case 1: // FD - get_cmd = F_GETFD; - set_cmd = F_SETFD; + case 1: // FD + get_cmd = F_GETFD; // 如果FL_OR_FD为1，即要获取的是文件描述符标志，因此设置get_cmd为F_GETFD + set_cmd = F_SETFD; // 如果FL_OR_FD为1，即要设置的是文件描述符标志，因此设置set_cmd为F_SETFD break; default: - get_cmd = -1; + get_cmd = -1; // 如果FL_OR_FD的值既不是0也不是1，即未知的情况，设置get_cmd和set_cmd为-1，表示错误 set_cmd = -1; break; } - if (get_cmd == -1 || set_cmd == -1) { - LIBCOMM_ELOG(WARNING, "(set socket opt)\tUnkown: command type[%d].", FL_OR_FD); - return -1; + if (get_cmd == -1 || set_cmd == -1) { // 如果get_cmd或set_cmd的值为-1，表示未知的命令类型 + LIBCOMM_ELOG(WARNING, "(set socket opt)\tUnkown: command type[%d].", + FL_OR_FD); // 使用日志库输出警告信息，说明未知的命令类型 + return -1; // 返回-1表示错误 } - if ((flg = fcntl(sock, get_cmd, 0)) < 0) { // F_GETFL, F_GETFD - LIBCOMM_ELOG(WARNING, - "(set socket opt)\tFail to get fcntl[%d:%ld] of socket[%d]:fcntl failed,return[%d].", - get_cmd, - arg, - sock, - flg); - return flg; + if ((flg = fcntl(sock, get_cmd, 0)) < 0) { // 调用fcntl函数获取socket的选项值，将返回值存储在flg中 + // 如果fcntl函数调用失败，即返回值小于0 + LIBCOMM_ELOG(WARNING, "(set socket opt)\tFail to get fcntl[%d:%ld] of socket[%d]:fcntl failed,return[%d].", + get_cmd, arg, sock, flg); + // 使用日志库输出警告信息，说明fcntl获取选项失败 + return flg; // 返回flg的值，表示错误 } - flg |= arg; // O_NONBLOCK, FD_CLOEXEC - if ((rc = fcntl(sock, set_cmd, flg)) < 0) { // F_SETFL, F_SETFL - LIBCOMM_ELOG(WARNING, - "(set socket opt)\tFail to set fcntl[%d:%ld] of socket[%d]:fcntl failed,return[%d].", - set_cmd, - arg, - sock, - flg); - return rc; + flg |= arg; // 将arg的值与flg的值进行按位或操作，即将arg的值添加到flg的值中，用于设置新的选项值 + // O_NONBLOCK为非阻塞标志，FD_CLOEXEC为执行时关闭文件描述符标志 + if ((rc = fcntl(sock, set_cmd, flg)) < 0) { // 调用fcntl函数设置socket的选项值，将返回值存储在rc中 + // 如果fcntl函数调用失败，即返回值小于0 + LIBCOMM_ELOG(WARNING, "(set socket opt)\tFail to set fcntl[%d:%ld] of socket[%d]:fcntl failed,return[%d].", + set_cmd, arg, sock, flg); + // 使用日志库输出警告信息，说明fcntl设置选项失败 + return rc; // 返回rc的值，表示错误 } - return rc; + return rc; // 返回rc的值，表示成功 } - +//32位字符复制函数，此处采取不会出现复制错误的复制程序 uint32 comm_get_cpylen(const char* src, uint32 max_len) { uint32 cpylen = 0; @@ -177,7 +191,7 @@ uint32 comm_get_cpylen(const char* src, uint32 max_len) return cpylen; } - +//本函数在数据库之中起到了字符流判定的作用，其中引入静态只读字符，并根据传入的type进行判定，当type< 0或者type> 0的情况则对函数报错，输出UNKNOWN值 static const char* MAILBOX_STAT[MAIL_MAX_TYPE] = {"UNKNOWN", "READY", "RUN", "HOLD", "CLOSED", "TO_CLOSED"}; const char* stream_stat_string(int type) @@ -188,7 +202,9 @@ const char* stream_stat_string(int type) return MAILBOX_STAT[type]; } - +//本函数起到了控制消息信号判定作用，当type在规定值以内，对函数进行判定， +// 返回CTRL_MSG_STAT之中的操作指令，如果函数并不在规定范围之内，则将type值赋0， +// 进行UNKNOWN判定 static const char* CTRL_MSG_STAT[CTRL_MAX_TYPE] = {"UNKNOWN", "REGIST", "REGIST_CN", @@ -212,7 +228,9 @@ const char* ctrl_msg_string(int type) return CTRL_MSG_STAT[type]; } - +/*该函数为消息控制台函数，对消息进行控制，的那个消息处于控制区间内，返回MSG_OPER_STAT之中静态定义值，请注意，此处返回为READ_DATA_FROM_LOGIC + 1数值， +READ_DATA_FROM_LOGIC + 1 的操作是因为数组 MSG_OPER_STAT 的索引是从 0 开始的。 +数组的长度为 READ_DATA_FROM_LOGIC + 1，这样可以确保数组的最后一个元素的索引为 READ_DATA_FROM_LOGIC。*/ static const char *MSG_OPER_STAT[READ_DATA_FROM_LOGIC + 1] = { "send_some", "secure_read", @@ -240,7 +258,10 @@ void printfcmsg(const char* caller, struct FCMSG_T* pfcmsg) pfcmsg->query_id, pfcmsg->nodename); } - +/*请注意，该函数极为复杂，该函数所讲述为一种传输机制cmailbox传输机制，该传输机制已在前面进行阐明， +printf_cmailbox_statistic函数用于输出cmailbox的统计信息，帮助用户了解cmailbox的使用情况和性能表现。 +通过调用该函数并传入cmailbox对象和节点名称作为参数，可以打印出相关的统计信息，例如已发送的消息数量、已接收的消息数量、消息发送的成功率等。 +该传输机制因其稳定且高效的原因在华为数据库opengauss之中得到了广泛的应用。*/ void printf_cmailbox_statistic(c_mailbox* cmailbox, char* nodename) { if (NULL == cmailbox->statistic || MAIL_UNKNOWN == cmailbox->state) { @@ -295,7 +316,9 @@ void printf_cmailbox_statistic(c_mailbox* cmailbox, char* nodename) cmailbox->statistic->recv_loop_time, cmailbox->statistic->recv_loop_count); } - +/*与cmailbox极为相似，pmailbox同样也是一种消息传递机制，也在前面有具体的解释。 +其中，printf_pmailbox_statistic函数用于输出pmailbox的统计信息，帮助用户了解pmailbox的使用情况和性能表现。 +通过调用该函数并传入pmailbox对象和节点名称作为参数，可以打印出相关的统计信息，例如已发送的消息数量、已接收的消息数量、消息发送的成功率等。*/ void printf_pmailbox_statistic(p_mailbox* pmailbox, char* nodename) { if (NULL == pmailbox->statistic || MAIL_UNKNOWN == pmailbox->state) { @@ -340,7 +363,7 @@ void printf_pmailbox_statistic(p_mailbox* pmailbox, char* nodename) pmailbox->statistic->os_send_overhead, pmailbox->statistic->producer_elapsed_time); } - +/*输出接口函数*/ void print_socket_info(int sock, struct tcp_info* info, bool sender) { mc_elog(LOG, @@ -370,7 +393,8 @@ void print_socket_info(int sock, struct tcp_info* info, bool sender) // error logging function // extern uint32 GetTopTransactionIdIfAny(void); - +//mc_elog 函数在 OpenGauss 数据库中的 libcomm_util.cpp 文件中是一个用于错误日志记录的函数。mc_elog +// 是 "multi-copy elog" 的缩写，其中 "elog" 是 PostgreSQL 中的一个错误日志系统。 void mc_elog(int elevel, const char* format, ...) { #define MSLEN 4 @@ -420,44 +444,71 @@ void mc_elog(int elevel, const char* format, ...) ss_rc = strncpy_s(timebuf + MSOFFSET, TIMELEN - MSOFFSET - 1, msbuf, MSLEN); securec_check(ss_rc, "\0", "\0"); + // 检查u_sess是否存在，u_sess代表一个用户会话 if (u_sess) { + // 如果u_sess存在，那么将相关信息打印到标准输出 fprintf(stdout, - "%s %s " /* timestamp with milliseconds */ - "%ld.%d " /* session ID */ - "%s " /* database name */ - "%lu " /* process ID */ - "%s " /* application name */ - "%u " /* transaction ID (0 if none) */ - "%s " /* DataNode name */ - "%s " /* SQL state */ - "%lu " /* u_sess->debug_query_id */ - "%s: " /* log level */ - "%s\n", /* error message */ - timebuf, - timebuf + MSOFFSET + MSLEN + 1, - (long)(t_thrd.proc_cxt.MyStartTime), - t_thrd.myLogicTid, - (u_sess->proc_cxt.MyProcPort && u_sess->proc_cxt.MyProcPort->database_name && - u_sess->proc_cxt.MyProcPort->database_name[0] != '\0') - ? u_sess->proc_cxt.MyProcPort->database_name - : "[unknown]", - t_thrd.proc_cxt.MyProcPid, - (u_sess->proc_cxt.MyProcPort && u_sess->attr.attr_common.application_name && - u_sess->attr.attr_common.application_name[0] != '\0') - ? u_sess->attr.attr_common.application_name - : "[unknown]", - GetTopTransactionIdIfAny(), - g_instance.attr.attr_common.PGXCNodeName ? g_instance.attr.attr_common.PGXCNodeName - : g_instance.comm_cxt.localinfo_cxt.g_self_nodename, - "00000", - u_sess->debug_query_id, - (elevel == LOG) ? "[LIBCOMM] LOG" : "[LIBCOMM] WARNING", - msg); + // 时间戳，包括毫秒部分 + "%s %s " /* timestamp with milliseconds */ + // 会话ID + "%ld.%d " /* session ID */ + // 数据库名称，如果未知则为"[unknown]" + "%s " /* database name */ + // 进程ID + "%lu " /* process ID */ + // 应用程序名称，如果未知则为"[unknown]" + "%s " /* application name */ + // 事务ID，如果没有则为0 + "%u " /* transaction ID (0 if none) */ + // DataNode名称，如果没有则为g_instance.comm_cxt.localinfo_cxt.g_self_nodename + "%s " /* DataNode name */ + // SQL状态，这里固定为"00000" + "%s " /* SQL state */ + // u_sess的debug_query_id + "%lu " /* u_sess->debug_query_id */ + // 日志级别，如果是LOG则为"[LIBCOMM] LOG"，否则为"[LIBCOMM] WARNING" + "%s: " /* log level */ + // 错误信息 + "%s\n", /* error message */ + // 时间戳 + timebuf, + // 时间戳的毫秒部分 + timebuf + MSOFFSET + MSLEN + 1, + // 会话开始时间的长整型表示 + (long)(t_thrd.proc_cxt.MyStartTime), + // 逻辑线程ID + t_thrd.myLogicTid, + // 数据库名称，如果未知则为"[unknown]" + (u_sess->proc_cxt.MyProcPort && u_sess->proc_cxt.MyProcPort->database_name && + u_sess->proc_cxt.MyProcPort->database_name[0] != '\0') + ? u_sess->proc_cxt.MyProcPort->database_name + : "[unknown]", + // 进程ID + t_thrd.proc_cxt.MyProcPid, + // 应用程序名称，如果未知则为"[unknown]" + (u_sess->proc_cxt.MyProcPort && u_sess->attr.attr_common.application_name && + u_sess->attr.attr_common.application_name[0] != '\0') + ? u_sess->attr.attr_common.application_name + : "[unknown]", + // 获取顶级事务ID，如果没有则为0 + GetTopTransactionIdIfAny(), + // DataNode名称，如果没有则为g_instance.comm_cxt.localinfo_cxt.g_self_nodename + g_instance.attr.attr_common.PGXCNodeName ? g_instance.attr.attr_common.PGXCNodeName + : g_instance.comm_cxt.localinfo_cxt.g_self_nodename, + // SQL状态，这里固定为"00000" + "00000", + // u_sess的debug_query_id + u_sess->debug_query_id, + // 日志级别，如果是LOG则为"[LIBCOMM] LOG"，否则为"[LIBCOMM] WARNING" + (elevel == LOG) ? "[LIBCOMM] LOG" : "[LIBCOMM] WARNING", + // 错误信息 + msg); } + // 刷新标准输出缓冲区，确保所有输出都被立即显示 (void)fflush(stdout); } - +//该函数记录程序运行时长 static inline int mc_clock_gettime(struct timespec* ts) { return clock_gettime(CLOCK_MONOTONIC, ts); @@ -465,54 +516,74 @@ static inline int mc_clock_gettime(struct timespec* ts) #define MC_NS_IN_SEC 1000000000ULL +// 定义一个名为mc_timers_us的函数，该函数没有参数，并返回一个无符号的64位整数，表示以微秒为单位的时间 uint64_t mc_timers_us(void) { + // 定义一个无符号的64位整数变量ns_monotonic，用于存储获取到的时间值 uint64_t ns_monotonic; + // 定义一个timespec结构体变量ts，并初始化其所有成员为0 struct timespec ts = {0}; + // 调用mc_clock_gettime函数，获取当前的单调时间，并将其存储在ts变量中 (void)mc_clock_gettime(&ts); + // 将获取到的时间值（纳秒为单位）转换为微秒，然后存储在ns_monotonic变量中 + // 这里的转换是先将秒数部分转换为纳秒（乘以MC_NS_IN_SEC，即1秒的纳秒数），再加上原有的纳秒部分，然后再除以1000得到微秒 ns_monotonic = (uint64)((ts.tv_sec * MC_NS_IN_SEC) + (uint64)ts.tv_nsec) / 1000; + // 返回转换后的时间值（微秒为单位） return ns_monotonic; } - +//同样为单位换算函数，换算单位为纳秒的千分之一 uint64 mc_timers_ms(void) { return mc_timers_us() / 1000; } +// 定义一个名为comm_ipc_log_get_time的函数，该函数接收一个字符指针now_date和一个整数time_len作为参数 void comm_ipc_log_get_time(char *now_date, int time_len) { + // 定义常量MS_LEN为4，表示毫秒字符串的长度 const int MS_LEN = 4; + // 定义常量BUF_LEN为MS_LEN的两倍，表示毫秒缓冲区的长度 const int BUF_LEN = MS_LEN * 2; + // 定义常量MS_OFFSET为19，表示毫秒字符串在now_date中的偏移量 const int MS_OFFSET = 19; - struct timeval tv = {0}; /* make time string start, like setup_formatted_log_time */ - struct pg_tm* localtime = NULL; + // 定义一个timeval结构体变量tv，并初始化其所有成员为0，用于获取当前时间 + struct timeval tv = {0}; /* make time string start, like setup_formatted_log_time */ + // 定义一个pg_tm结构体指针localtime，用于存储本地时间 + struct pg_tm *localtime = NULL; + // 定义一个pg_time_t类型变量stamp_time，用于存储时间戳 pg_time_t stamp_time; + // 定义一个长度为BUF_LEN的字符数组msbuf，用于存储毫秒字符串 char msbuf[BUF_LEN]; + // 定义一个errno_t类型变量ss_rc，用于存储函数调用返回值 errno_t ss_rc = 0; + // 调用gettimeofday函数获取当前时间，并将其存储在tv变量中 (void)gettimeofday(&tv, NULL); + // 将获取到的时间戳（秒为单位）转换为pg_time_t类型，并存储在stamp_time变量中 stamp_time = (pg_time_t)tv.tv_sec; + // 调用pg_localtime函数将时间戳转换为本地时间，并存储在localtime变量中 localtime = pg_localtime(&stamp_time, log_timezone); + // 如果localtime不为NULL，则调用pg_strftime函数将本地时间格式化为指定格式的字符串，并将其存储在now_date变量中 /* leave room for milliseconds. */ if (localtime != NULL) { - (void)pg_strftime(now_date, time_len, - "%Y-%m-%d %H:%M:%S", - localtime); + (void)pg_strftime(now_date, time_len, "%Y-%m-%d %H:%M:%S", localtime); } - /* 'paste' milliseconds into place. */ + // 将微秒转换为毫秒，并将其存储在msbuf变量中 + /* 'paste' milliseconds into place. */ ss_rc = snprintf_s(msbuf, sizeof(msbuf), MS_LEN, ".%03d", (int)(tv.tv_usec / 1000)); securec_check_ss(ss_rc, "\0", "\0"); + // 将毫秒字符串复制到now_date变量的指定位置中 ss_rc = strncpy_s(now_date + MS_OFFSET, time_len - MS_OFFSET - 1, msbuf, MS_LEN); securec_check(ss_rc, "\0", "\0"); } - +//唤醒通道类构造函数 WakeupPipe::WakeupPipe() { InitPipe(m_normal_wakeup_pipes); } - +//唤醒通道类唤醒函数 void WakeupPipe::DoWakeup() { int errno_tmp = errno; @@ -521,7 +592,7 @@ void WakeupPipe::DoWakeup() } errno = errno_tmp; } - +//唤醒通道类创建通道函数 void WakeupPipe::InitPipe(int *input_pipes) { if (input_pipes == NULL) { @@ -543,12 +614,12 @@ void WakeupPipe::InitPipe(int *input_pipes) m_ev.events = EPOLLIN; m_ev.data.fd = input_pipes[0]; } - +//唤醒通道类转化通道函数 void WakeupPipe::RemoveWakeupFd() { (void)epoll_ctl(m_epfd, EPOLL_CTL_DEL, m_normal_wakeup_pipes[WAKEUP_PIPE_START], NULL); } - +//唤醒通道类关闭通道函数 void WakeupPipe::ClosePipe(int *input_pipes) { close(input_pipes[WAKEUP_PIPE_START]); @@ -556,7 +627,7 @@ void WakeupPipe::ClosePipe(int *input_pipes) input_pipes[WAKEUP_PIPE_START] = INVALID_FD; input_pipes[WAKEUP_PIPE_END] = INVALID_FD; } - +//唤醒通道类析构函数 WakeupPipe::~WakeupPipe() { ClosePipe(m_normal_wakeup_pipes); -- 2.34.1 From 6fa2c047e2c06c4956dad92f44c3cb863a8952ab Mon Sep 17 00:00:00 2001 From: dzlsgdhr Date: Tue, 3 Oct 2023 16:52:12 +0800 Subject: [PATCH 11/15] Update libcomm_queue.cpp --- .../libcomm_utils/libcomm_queue.cpp | 180 +++++++++++------- 1 file changed, 107 insertions(+), 73 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_queue.cpp b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_queue.cpp index cfd49f98e..8f0512376 100644 --- a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_queue.cpp +++ b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_queue.cpp @@ -26,121 +26,155 @@ #include "libcomm_queue.h" #include "libcomm_common.h" -int mc_queue_pop(struct mc_queue* q, int* e) + +//该代码用于在环形队列之中取出一个元素，使用POP表达弹出一个元素，按照本函数意思，将环形队列Q中的头部指针q取出，将q中的值赋给e,同时，该代码使用了互斥锁（mutex）以确保在多线程环境下的安全操作。 +int mc_queue_pop( + struct mc_queue *q, + int *e) // 定义一个函数mc_queue_pop，输入参数为一个mc_queue结构体的指针q和一个整型指针e。函数返回一个整型值。 { - if (q == NULL) { + if (q == NULL) { // 如果q为NULL，表示队列没有初始化，返回错误代码-1。 return -1; } - LIBCOMM_PTHREAD_MUTEX_LOCK(&(q->lock)); - if (q->is_empty == 1) { + LIBCOMM_PTHREAD_MUTEX_LOCK(&(q->lock)); // 锁定队列的互斥锁，确保同一时间只有一个线程可以操作队列。 + + if (q->is_empty == 1) { // 如果队列为空，解锁互斥锁并返回0，表示队列为空。 LIBCOMM_PTHREAD_MUTEX_UNLOCK(&(q->lock)); return 0; } - *e = q->data[q->head]; - q->data[q->head] = -1; // unnormal value - ++(q->head); + *e = q->data[q->head]; // 取出队列头部的元素，并赋值给e指向的变量。 + q->data[q->head] = -1; // 将队列头部的元素设为-1，表示该元素已经被取出。 - if (q->head == q->size) { + ++(q->head); // 将队列头部指针向后移动一位。 + + if (q->head == q->size) { // 如果队列头部指针已经到达队列尾部，将其重置为0。 q->head = 0; } - if (q->head == q->tail) { + + if (q->head == q->tail) { // 如果队列头部指针和尾部指针重合，表示队列已经为空。 q->is_empty = 1; } - q->is_full = 0; - q->count--; + q->is_full = 0; // 队列取出一个元素后，队列不可能再是满的，所以将is_full设为0。 + q->count--; // 队列的元素数量减1。 - LIBCOMM_PTHREAD_MUTEX_UNLOCK(&(q->lock)); - return 1; + LIBCOMM_PTHREAD_MUTEX_UNLOCK(&(q->lock)); // 解锁互斥锁，允许其他线程访问队列。 + return 1; // 返回1，表示成功从队列中取出一个元素。 } - -int mc_queue_push(struct mc_queue* q, int e) +//该函数定义为向环形队列之中添加元素，按照函数之中定义为向环形队列Q的q处添加数值为e的元素，同样，该代码使用了互斥锁（mutex）以确保在多线程环境下的安全操作。 +int mc_queue_push(struct mc_queue* q, int e) // 定义一个函数mc_queue_push，输入参数为一个mc_queue结构体的指针q和一个整型值e。函数返回一个整型值。 +{ + if (q == NULL) { // 如果q为NULL，表示队列没有初始化，返回错误代码-1。 + return -1; + } + + LIBCOMM_PTHREAD_MUTEX_LOCK(&(q->lock)); // 锁定队列的互斥锁，确保同一时间只有一个线程可以操作队列。 + + if (q->is_full) { // 如果队列已满，解锁互斥锁并返回0，表示队列已满。 + LIBCOMM_PTHREAD_MUTEX_UNLOCK(&(q->lock)); + return 0; + } + + q->data[q->tail] = e; // 将元素e添加到队列的尾部。 + ++(q->tail); // 将队列尾部指针向后移动一位。 + + if (q->tail == q->size) { // 如果队列尾部指针已经到达队列的末尾，将其重置为0。 + q->tail = 0; + } + + if (q->tail == q->head) { // 如果队列尾部指针和头部指针重合，表示队列已满。 + q->is_full = 1; + } + + q->is_empty = 0; // 添加一个元素后，队列不可能再是空的，所以将is_empty设为0。 + q->count++; // 队列的元素数量加1。 + + LIBCOMM_PTHREAD_MUTEX_UNLOCK(&(q->lock)); // 解锁互斥锁，允许其他线程访问队列。 + return 1; // 返回1，表示成功将一个元素添加到队列中。 +} +//此函数意思为创建一个数据流循环队列。其中q为队列的初始指针，size为队列大小。同样，该代码使用了互斥锁（mutex）以确保在多线程环境下的安全操作。 +int mc_queue_init( + struct mc_queue *q, + int size) // 定义一个函数mc_queue_init，输入参数为一个mc_queue结构体的指针q和一个整型值size，表示队列的大小。函数返回一个整型值。 { - if (q == NULL) { + if (q == NULL || size <= 1) { // 如果q为NULL或size小于等于1，表示输入参数无效，返回错误代码-1。 return -1; } - LIBCOMM_PTHREAD_MUTEX_LOCK(&(q->lock)); - if (q->is_full) { - LIBCOMM_PTHREAD_MUTEX_UNLOCK(&(q->lock)); - return 0; - } - q->data[q->tail] = e; - ++(q->tail); - - if (q->tail == q->size) { - q->tail = 0; - } - if (q->tail == q->head) { - q->is_full = 1; - } - - q->is_empty = 0; - q->count++; - LIBCOMM_PTHREAD_MUTEX_UNLOCK(&(q->lock)); - return 1; -} - -int mc_queue_init(struct mc_queue* q, int size) -{ - if (q == NULL || size <= 1) { - return -1; - } - - q->size = size - 1; - q->data = NULL; - LIBCOMM_MALLOC(q->data, (q->size * sizeof(int)), int); + q->size = size - 1; // 队列的大小为size-1，因为环形队列的索引从0开始。 + q->data = NULL; // 初始化队列的数据指针为NULL。 + LIBCOMM_MALLOC(q->data, (q->size * sizeof(int)), + int); // 动态分配队列的数据内存空间，大小为(size-1)*sizeof(int)。如果分配失败，返回错误代码-1。 if (q->data == NULL) { return -1; } - LIBCOMM_PTHREAD_MUTEX_INIT(&(q->lock), 0); + LIBCOMM_PTHREAD_MUTEX_INIT(&(q->lock), 0); // 初始化队列的互斥锁。 + // 这里是一段注释，解释了该队列的设计目的：为了获取streamid，我们不使用stream 0。 // we desgin this for getting streamid, we do not use stream 0 // - q->is_empty = 1; - q->is_full = 0; - q->head = 0; - q->tail = 0; - q->count = 0; - q->pop = mc_queue_pop; - q->push = mc_queue_push; + q->is_empty = 1; // 初始化队列的状态标志位，表示队列为空。 + q->is_full = 0; // 初始化队列的状态标志位，表示队列不满。 + q->head = 0; // 初始化队列的头部指针为0。 + q->tail = 0; // 初始化队列的尾部指针为0。 + q->count = 0; // 初始化队列的元素数量为0。 + q->pop = mc_queue_pop; // 设置队列的出队函数为mc_queue_pop。 + q->push = mc_queue_push; // 设置队列的入队函数为mc_queue_push。 - return 0; + return 0; // 返回0，表示成功初始化队列。 } - -struct mc_queue* mc_queue_clear(struct mc_queue* q) +//此为清楚函数，清楚位于队列中位置q的元素，，并且最后返回清除之后的函数 +struct mc_queue *mc_queue_clear( + struct mc_queue + *q) // 定义一个函数mc_queue_clear，输入参数为一个mc_queue结构体的指针q，函数返回一个mc_queue结构体的指针。 { - if (q == NULL) { + if (q == NULL) { // 如果q为NULL，表示队列没有初始化，返回NULL。 return NULL; } - int data = 0; + int data = 0; // 定义一个整型变量data，用于暂存队列中的元素。 - while (!q->is_empty) { - (void)q->pop(q, &data); + while (!q->is_empty) { // 当队列不为空时，循环执行以下操作。 + (void)q->pop( + q, + &data); // 调用队列的出队函数pop，将队列头部的元素取出并赋值给data。使用(void)是为了防止函数返回值被忽略导致的警告。 } - return q; + return q; // 返回清空后的队列指针。 } - -struct mc_queue* mc_queue_destroy(struct mc_queue* q) +struct mc_queue *mc_queue_destroy( + struct mc_queue + *q) // 定义一个函数mc_queue_destroy，输入参数为一个mc_queue结构体的指针q，函数返回一个mc_queue结构体的指针。 { - if (q == NULL) { + if (q == NULL) { // 如果q为NULL，表示队列没有初始化，返回NULL。 return NULL; } - if (q->data == NULL) { + if (q->data == NULL) { // 如果队列的数据指针为NULL，表示队列未初始化或已经被销毁，直接返回队列指针。 return q; } - q = mc_queue_clear(q); - LIBCOMM_PTHREAD_MUTEX_LOCK(&(q->lock)); - mc_free(q->data); - q->data = NULL; - LIBCOMM_PTHREAD_MUTEX_UNLOCK(&(q->lock)); - LIBCOMM_PTHREAD_MUTEX_DESTORY(&(q->lock)); + q = mc_queue_clear(q); // 调用mc_queue_clear函数清空队列中的所有元素。 + LIBCOMM_PTHREAD_MUTEX_LOCK(&(q->lock)); // 锁定队列的互斥锁，确保同一时间只有一个线程可以访问队列。 + mc_free(q->data); // 释放队列的数据内存空间。 + q->data = NULL; // 将队列的数据指针设为NULL。 + LIBCOMM_PTHREAD_MUTEX_UNLOCK(&(q->lock)); // 解锁互斥锁，允许其他线程访问队列。 + LIBCOMM_PTHREAD_MUTEX_DESTORY(&(q->lock)); // 销毁互斥锁。 - return q; + return q; // 返回销毁后的队列指针。 } + + +/*libcomm_queue代码较为简单容易理解，其表达了对数据流环形队列的各种操作方法，其中使用五个基础函数，分别为 +* 1，弹出函数mc_queue_pop(struct mc_queue *q,int *e)：该函数所代表为将循环队列之中位于位置q的元素弹出循环队列数据流， +* 将弹出的数据的值赋值给e已达到弹出函数的效用 +* 2，插入函数mc_queue_push(struct mc_queue* q, int e)：该函数所代表为向循环队列数据列之中位置q处插入e之中的数值， +* 并将循环队列向后指，达到循环队列插入的效果 +* 3，创建函数mc_queue_init(struct mc_queue *q,int size)：该函数所代表创建循环队列数据流，在位置q处创建大小为size的数据流。 +* 并且利用LIBCOMM_MALLOC(q->data, (q->size * sizeof(int)),int); 对size大小的内存空间进行合理的空间分配 +* 4，清除函数struct mc_queue *mc_queue_clear(struct mc_queue *q)：利用调用弹出函数，并将值付给一块内存空间并删除该内存空间达到清除的效果， +* 该函数表明清楚循环队列数据流之中位于q的数据。 +* 5，销毁函数struct mc_queue *mc_queue_destroy(struct mc_queue*q)：调用清除函数来清除队列之中所有的数据。 + +*/ -- 2.34.1 From 1250fc5cf36fb6a53f6338a0c020d5114e929d84 Mon Sep 17 00:00:00 2001 From: dzlsgdhr Date: Tue, 3 Oct 2023 16:53:05 +0800 Subject: [PATCH 12/15] Update libcomm_err.cpp --- .../libcomm_utils/libcomm_err.cpp | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_err.cpp b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_err.cpp index fefbd9d12..56630f1e0 100644 --- a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_err.cpp +++ b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_err.cpp @@ -31,7 +31,7 @@ void mc_err_abort(void) static const char unknown_error[] = ""; const char* mc_strerror(int errnum) { - if (errnum >= 1000) { + if (errnum >= 1000) {//对不同的错误类型进行分类处理记录 return mc_comlib_strerror(errnum); // communication layer defined errors } else if (errnum == 0) { return unknown_error; @@ -39,3 +39,30 @@ const char* mc_strerror(int errnum) return strerror(errnum); // system defined errors } } + +/* +libcomm_err.cpp文件是关于对于数据库中出现的错误进行归类分析，其中定义了两个函数，分别是 +1，mc_err_abort函数，此函数多存在于libcomm_err.h之中，主要目的是对libcomm_err.h之中的宏定义进行 +终止处理，起到了制止函数运行的作用 +2，mc_strerror函数，此函数作用在libcomm_errno-comlib.h之中，对libcomm_errno-comlib.h传递数值， +让libcomm_errno-comlib.h对错误类型进行判断 +libcomm_err.cpp是一个查找错误的函数，其主要目的为找出数据库之中的错误并将其记录，保证了数据库的准确性。 + +下面将分别对libcomm_err.h文件和libcomm_errno-comlib.h文件进行注释 + +1，libcomm_err.h文件是将错误文件记录在文件夹之中的文件，并且中断函数的运行， +其中对不同的错误类型进行了区分，其中包含函数： + 1，mc_assert(x)宏定义：在win32之中仍存在一些不适配，并将其中的文件路径以及函数输出到屏幕之上 + 2，gai_assert(x)宏定义：对mc_unlikely(x)进行判定，若发现出现错误，将错误信息、错误号和源文件名以及行号打印到标准错误输出 + 3，posix_assert(x)宏定义：与2函数相同，将错误信息、错误号和源文件名以及行号打印到标准错误输出，但是输出错误号为strerror(x)错误号 + 4，mc_assert_state(obj, state_name)宏定义：对obj->name和state_name进行比较，将错误信息、错误号和源文件名以及行号打印到标准错误输出，同时对错误缓冲区进行刷新，确保下一次错误能被及时显示在屏幕上 + 5，alloc_assert(x)宏定义：对储存分配进行检查，如果发现储存未正确分配，将错误信息、错误号和源文件名以及行号打印到标准错误输出 + 6，errno_assert(x)宏定义：对通信的条件进行错误检查，如果发现出现错误，将错误信息、错误号和源文件名以及行号打印到标准错误输出 + 7，errnum_assert(cond, err)宏定义：对6宏定义进行检查，查看错误信息，错误号是否正确，如果发现错误，将错误信息、错误号和源文件名以及行号打印到标准错误输出 + 8、9，win_assert(x)，wsa_assert(x)宏定义：与6函数类似，均为对条件进行判断，如果条件中存在作物，进行错误信息输出，不同于6，其输出错误信息分别为最后接收错误以及WSA最后接受错误 + 10，mc_fsm_error(message, state, src, type)宏定义：分别将message, state, src, type四个点作为错误信息进行输出 + +2.libcomm_errno-comlib.h文件对错误类型进行判断，分别判断从错误类型1000到1063的错误类型，对错误类型进行记录 + +主函数不仅对于1000-1063的错误类型进行了判断，同样也对错误类型不在此范围的错误进行判断，其中有未知错误以及不同别的错误类型 +*/ \ No newline at end of file -- 2.34.1 From 6781341774eb7a831811d8e9855dca4f7eeab454 Mon Sep 17 00:00:00 2001 From: dzlsgdhr Date: Tue, 3 Oct 2023 16:54:09 +0800 Subject: [PATCH 13/15] Update libcomm_err.h --- .../communication/libcomm_utils/libcomm_err.h | 129 +++++++++++------- 1 file changed, 79 insertions(+), 50 deletions(-) diff --git a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_err.h b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_err.h index daa3930c2..24825ea0d 100644 --- a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_err.h +++ b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_err.h @@ -48,109 +48,138 @@ /* Same as system assert(). However, under Win32 assert has some deficiencies. Thus this macro. */ -#define mc_assert(x) \ +#define mc_assert(x) /*总体来说，这段代码的目的是在某些条件不满足时输出调试日志并终止程序，以确保程序的正确性。*/ \ do { \ if (mc_slow(!(x))) { \ COMM_DEBUG_LOG("sctp check failed: %s (%s:%d)\n", #x, __FILE__, __LINE__); \ - mc_err_abort(); \ - } \ - } while (0) + /*如果表达式 x 不满足，则输出一条调试日志，记录断言失败的表达式 x， + 以及该日志消息所在的源文件名和行号。__FILE__ 和 __LINE__ 是预定义的宏， + 分别表示当前源文件的文件名和行号。*/ + mc_err_abort(); \ + /*如果表达式 x 不满足，则调用 mc_err_abort 函数，进行错误处理并终止程序执行。*/ \ + } + } while (0)//由于条件为0所以只会执行一次 // Provides convenient way to check for errors from getaddrinfo. -#define gai_assert(x) \ +#define gai_assert(x) // 定义一个名为gai_assert的宏，参数为x ，如果x为假将会执行错误处理 do { \ if (mc_unlikely(x)) { \ - const char* errstr = gai_strerror(x); \ - fprintf(stderr, "%s (%s:%d)\n", errstr, __FILE__, __LINE__); \ - mc_err_abort(); \ - } \ + const char* errstr = gai_strerror(x); // 调用gai_strerror函数，将错误号转化为错误信息字符串 + fprintf(stderr, "%s (%s:%d)\n", errstr, __FILE__, __LINE__); // 将错误信息，文件名和行号打印到标准错误输出 + mc_err_abort(); //调用函数终止程序运行 \ + } } while (false) // Provides convenient way to check for POSIX errors. -#define posix_assert(x) \ - do { \ +#define posix_assert(x) //次宏定义和上一个宏定义有相似的效果，区别在于次宏定义打印的标准错误输出与之前存在不同 \ + do { if (mc_unlikely(x)) { \ - fprintf(stderr, "%s (%s:%d)\n", strerror(x), __FILE__, __LINE__); \ - mc_err_abort(); \ - } \ + fprintf(stderr, "%s (%s:%d)\n", strerror(x), __FILE__, __LINE__); //对strerror(x)进行打印输出 + mc_err_abort(); + } } while (false) #define mc_assert_state(obj, state_name) \ do { \ - if (mc_slow((obj)->state != (state_name))) { \ - fprintf(stderr, "Assertion failed: %d == %s (%s:%d)\n", (obj)->state, #state_name, __FILE__, __LINE__); \ - (void)fflush(stderr); \ - mc_err_abort(); \ - } \ - } while (0) + if (mc_slow((obj)->state != (state_name))) { // 使用mc_slow宏，用于优化可能性较小的条件判断，如果obj的状态不等于state_name + fprintf(stderr, "Assertion failed: %d == %s (%s:%d)\n", (obj)->state, #state_name, __FILE__, __LINE__);// 将错误信息，文件名和行号打印到标准错误输出 + (void)fflush(stderr); // 刷新标准错误输出缓冲区，确保错误信息立即显示 + mc_err_abort(); //调用函数终止程序运行 \ + } + } while (0)//由于条件为0所以只会执行一次 /* Checks whether memory allocation was successful. */ -#define alloc_assert(x) \ - do { \ - if (mc_slow(!(x))) { \ - fprintf(stderr, "Out of memory (%s:%d)\n", __FILE__, __LINE__); \ - (void)fflush(stderr); \ - mc_err_abort(); \ - } \ +#define alloc_assert(x) //宏定义，检查存储分配是否成功 + do { + if (mc_slow(!(x))) { //如果存储分配未成功， 将错误信息、错误号和源文件名以及行号打印到标准错误输出 + fprintf(stderr, "Out of memory (%s:%d)\n", __FILE__, __LINE__); + (void)fflush(stderr); // 刷新标准错误输出缓冲区，确保错误信息立即显示 + mc_err_abort(); //调用函数终止程序运行 + } } while (0) /* Check the condition. If false prints out the errno. */ -#define errno_assert(x) \ +// 定义一个名为errno_assert的宏，参数为x +#define errno_assert(x) +// 查看条件x，如果出现错误输出错误 do { \ + // 使用mc_slow宏来优化可能性较小的条件判断。如果x不为真，则执行下面的代码块 if (mc_slow(!(x))) { \ + // 将错误信息、错误号和源文件名以及行号打印到标准错误输出 fprintf(stderr, "%s [%d] (%s:%d)\n", mc_strerror(errno), (int)errno, __FILE__, __LINE__); \ + // 刷新标准错误输出缓冲区，确保错误信息立即显示 (void)fflush(stderr); \ + // 调用mc_err_abort函数终止程序运行 mc_err_abort(); \ } \ } while (0) - /* Checks whether supplied errno number is an error. */ -#define errnum_assert(cond, err) \ +// 定义一个名为errnum_assert的宏，参数为cond和err +#define errnum_assert(cond, err) //检查检查是否出错的出错码是否出错 \ + // 使用do-while循环来保证宏的使用像一个语句 do { \ + // 使用mc_slow宏来优化可能性较小的条件判断。如果cond不为真，则执行下面的代码块 if (mc_slow(!(cond))) { \ + // 将错误信息、错误号和源文件名以及行号打印到标准错误输出 fprintf(stderr, "%s [%d] (%s:%d)\n", mc_strerror(err), (int)(err), __FILE__, __LINE__); \ + // 刷新标准错误输出缓冲区，确保错误信息立即显示 (void)fflush(stderr); \ + // 调用mc_err_abort函数终止程序运行 mc_err_abort(); \ } \ } while (0) /* Checks the condition. If false prints out the GetLastError info. */ +// 定义一个名为win_assert的宏，参数为x #define win_assert(x) \ + // 使用do-while循环来保证宏的使用像一个语句 do { \ + // 使用mc_slow宏来优化可能性较小的条件判断。如果x不为真，则执行下面的代码块 if (mc_slow(!(x))) { \ + // 定义一个长度为256的字符数组errstr，用于存储错误信息 char errstr[256]; \ + // 调用mc_win_error函数，将错误信息存储在errstr中 mc_win_error((int)GetLastError(), errstr, 256); \ + // 将错误信息、错误号和源文件名以及行号打印到标准错误输出 fprintf(stderr, "%s [%d] (%s:%d)\n", errstr, (int)GetLastError(), __FILE__, __LINE__); \ + // 刷新标准错误输出缓冲区，确保错误信息立即显示 (void)fflush(stderr); \ + // 调用mc_err_abort函数终止程序运行 mc_err_abort(); \ } \ } while (0) - /* Checks the condition. If false prints out the WSAGetLastError info. */ -#define wsa_assert(x) \ - do { \ - if (mc_slow(!(x))) { \ - char errstr[256]; \ - mc_win_error(WSAGetLastError(), errstr, 256); \ - fprintf(stderr, "%s [%d] (%s:%d)\n", errstr, (int)WSAGetLastError(), __FILE__, __LINE__); \ - (void)fflush(stderr); \ - mc_err_abort(); \ - } \ - } while (0) +#define wsa_assert(x) //与之前宏定义类似，区别一上一个宏定义，该宏定义对于错误存储为存储WSAGetLastError()错误，存在差别 \ + do { // 开始一个do-while循环，这个循环至少会执行一次，即使条件不满足 + if (mc_slow(!(x))) { // 如果函数mc_slow的参数为假（也就是说，x不满足），那么执行下面的代码块 + char errstr[256]; // 定义一个长度为256的字符数组errstr，用于存储错误信息 + mc_win_error(WSAGetLastError(), errstr, 256); // 调用函数mc_win_error，获取Windows的最后一个错误，并将其存储在errstr中 + fprintf(stderr, "%s [%d] (%s:%d)\n", errstr, (int)WSAGetLastError(), __FILE__, __LINE__); // 将错误信息、错误号、当前文件名和行号打印到标准错误输出 + (void)fflush(stderr); // 刷新标准错误输出缓冲区，确保上面的错误信息立即显示 + mc_err_abort(); // 调用函数mc_err_abort，这个函数通常会导致程序终止 + } // 结束if语句 +} while (0); // 结束do-while循环，因为条件为0（即假），所以这个循环只会执行一次 /* Assertion-like macros for easier fsm debugging. */ -#define mc_fsm_error(message, state, src, type) \ - do { \ - fprintf(stderr, "%s: state=%d source=%d action=%d (%s:%d)\n", message, state, src, type, __FILE__, __LINE__); \ - (void)fflush(stderr); \ - mc_err_abort(); \ - } while (0) +#define mc_fsm_error(message, state, src, type) // 定义一个名为mc_fsm_error的宏，该宏接受四个参数：message（错误消息），state（当前状态），src（源），type（类型） + do { // 开始一个do-while循环，因为后面有一个分号，所以这个循环只会执行一次 + fprintf(stderr, "%s: state=%d source=%d action=%d (%s:%d)\n", message, state, src, type, __FILE__, __LINE__); // 将错误消息，当前状态，源，类型以及当前的文件名和行号输出到标准错误流 + (void)fflush(stderr); // 刷新标准错误流，确保上面的输出立即被显示 + mc_err_abort(); // 调用mc_err_abort函数，终止函数进程 + } while (0) // 结束do-while循环 +//皆为调用mc_fsm_error（）宏并将错误信息传入的宏定义 +#define mc_fsm_bad_action(state, src, type) mc_fsm_error("Unexpected action", state, src, type) // 如果在有限状态机中出现了预期之外的动作，调用mc_fsm_error宏，并传入错误消息"Unexpected action"以及当前的状态、源、类型 -#define mc_fsm_bad_action(state, src, type) mc_fsm_error("Unexpected action", state, src, type) -#define mc_fsm_bad_state(state, src, type) mc_fsm_error("Unexpected state", state, src, type) -#define mc_fsm_bad_source(state, src, type) mc_fsm_error("Unexpected source", state, src, type) +#define mc_fsm_bad_state(state, src, type) mc_fsm_error("Unexpected state", state, src, type) // 如果在有限状态机中出现了预期之外的状态，调用mc_fsm_error宏，并传入错误消息"Unexpected state"以及当前的状态、源、类型 + +#define mc_fsm_bad_source(state, src, type) mc_fsm_error("Unexpected source", state, src, type) // 如果在有限状态机中出现了预期之外的源，调用mc_fsm_error宏，并传入错误消息"Unexpected source"以及当前的状态、源、类型 void mc_err_abort(void); const char* mc_err_strerror(int errnum); #endif //_UTILS_ERR_H_ + +/* + + +*/ -- 2.34.1 From 2e649bc536f5f5fcd68f022a29de8aedf2b3c2df Mon Sep 17 00:00:00 2001 From: dzlsgdhr Date: Tue, 3 Oct 2023 16:54:42 +0800 Subject: [PATCH 14/15] Update libcomm_errno-comlib.h --- .../cbb/communication/libcomm_utils/libcomm_errno-comlib.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_errno-comlib.h b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_errno-comlib.h index b224ddef9..85b585f4c 100644 --- a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_errno-comlib.h +++ b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_errno-comlib.h @@ -23,7 +23,7 @@ #ifndef _MC_ERRNO_COMLIB_H_ #define _MC_ERRNO_COMLIB_H_ -static const char* comlib_error[] = { +static const char* comlib_error[] = {//进行了对错误类型的判断，从1000开始到1063分别为不同的错误类型 "1000 Reserved", "1001 Invalid argument", "1002 Memeory allocate error", -- 2.34.1 From 00f3d07a2f3a8a3d3059040ad1cf769531057c79 Mon Sep 17 00:00:00 2001 From: dzlsgdhr Date: Tue, 3 Oct 2023 17:25:31 +0800 Subject: [PATCH 15/15] Update libcomm_util.cpp --- .../cbb/communication/libcomm_utils/libcomm_util.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_util.cpp b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_util.cpp index 34c305b73..7201f6616 100644 --- a/src/gausskernel/cbb/communication/libcomm_utils/libcomm_util.cpp +++ b/src/gausskernel/cbb/communication/libcomm_utils/libcomm_util.cpp @@ -614,7 +614,7 @@ void WakeupPipe::InitPipe(int *input_pipes) m_ev.events = EPOLLIN; m_ev.data.fd = input_pipes[0]; } -//唤醒通道类转化通道函数 +//唤醒通道类移除通道开始函数 void WakeupPipe::RemoveWakeupFd() { (void)epoll_ctl(m_epfd, EPOLL_CTL_DEL, m_normal_wakeup_pipes[WAKEUP_PIPE_START], NULL); -- 2.34.1