diff --git a/contrib/postgres_fdw/deparse.cpp b/contrib/postgres_fdw/deparse.cpp index d4cc79a29..12fd99853 100644 --- a/contrib/postgres_fdw/deparse.cpp +++ b/contrib/postgres_fdw/deparse.cpp @@ -1151,7 +1151,6 @@ static void deparseRelation(StringInfo buf, Relation rel) const char *nspname = NULL; const char *relname = NULL; ListCell *lc; - char parttype = PARTTYPE_NON_PARTITIONED_RELATION; /* obtain additional catalog information. */ ForeignTable* table = GetForeignTable(RelationGetRelid(rel)); @@ -1180,31 +1179,34 @@ static void deparseRelation(StringInfo buf, Relation rel) relname = RelationGetRelationName(rel); } - /* foreign table could not be built from a partitioned table */ - UserMapping* user = GetUserMapping(rel->rd_rel->relowner, table->serverid); - ForeignServer *server = GetForeignServer(table->serverid); - PGconn* conn = GetConnection(server, user, false); + /* In current version, there are some unpredictable operations (delete/update, etc.) of foreign table built on + * partitioned table. We forbid all operations in this condition by default. */ + if (!ENABLE_SQL_BETA_FEATURE(PARTITION_FDW_ON)) { + char parttype = PARTTYPE_NON_PARTITIONED_RELATION; + UserMapping* user = GetUserMapping(GetUserId(), table->serverid); + ForeignServer *server = GetForeignServer(table->serverid); + PGconn* conn = GetConnection(server, user, false); - PQExpBuffer query = createPQExpBuffer(); - appendPQExpBuffer(query, - "SELECT c.parttype FROM pg_class c, pg_namespace n " - "WHERE c.relname = '%s' and c.relnamespace = n.oid and n.nspname = '%s'", - quote_identifier(relname), quote_identifier(nspname)); + PQExpBuffer query = createPQExpBuffer(); + appendPQExpBuffer(query, + "SELECT c.parttype FROM pg_class c, pg_namespace n " + "WHERE c.relname = '%s' and c.relnamespace = n.oid and n.nspname = '%s'", + quote_identifier(relname), quote_identifier(nspname)); - PGresult* res = pgfdw_exec_query(conn, query->data); - if (PQresultStatus(res) != PGRES_TUPLES_OK) { - pgfdw_report_error(ERROR, res, conn, true, query->data); - } - /* res may be empty as the relname/nspname validation is not checked */ - if (PQntuples(res) > 0) { - parttype = *PQgetvalue(res, 0, 0); - } - PQclear(res); - destroyPQExpBuffer(query); + PGresult* res = pgfdw_exec_query(conn, query->data); + if (PQresultStatus(res) != PGRES_TUPLES_OK) { + pgfdw_report_error(ERROR, res, conn, true, query->data); + } + /* res may be empty as the relname/nspname validation is not checked */ + if (PQntuples(res) > 0) { + parttype = *PQgetvalue(res, 0, 0); + } + PQclear(res); + destroyPQExpBuffer(query); - if (!ENABLE_SQL_BETA_FEATURE(PARTITION_FDW_ON) && - (parttype == PARTTYPE_PARTITIONED_RELATION || parttype == PARTTYPE_SUBPARTITIONED_RELATION)) { - ereport(ERROR, (errmsg("could not operate foreign table on partitioned table"))); + if ((parttype == PARTTYPE_PARTITIONED_RELATION || parttype == PARTTYPE_SUBPARTITIONED_RELATION)) { + ereport(ERROR, (errmsg("could not operate foreign table on partitioned table"))); + } } appendStringInfo(buf, "%s.%s", quote_identifier(nspname), quote_identifier(relname)); diff --git a/doc/src/sgml/ref/create_model.sgmlin b/doc/src/sgml/ref/create_model.sgmlin index 3cb22c661..523e23063 100644 --- a/doc/src/sgml/ref/create_model.sgmlin +++ b/doc/src/sgml/ref/create_model.sgmlin @@ -80,6 +80,7 @@ For example: * verbose: 0 (no output), 1 (less output), or 2 (full output) # Hyperparameter list for 'xgboost_regression_logistic', 'xgboost_binary_logistic', 'xgboost_regression_gamma' and 'xgboost_regression_squarederror': + * n_iter: Maximum iterations until convergence * batch_size: Number of tuples in each processing batch * booster: Which booster to use, e.g., gbtree, gblinear or dart (default: gbtree) * tree_method: The tree construction algorithm used in XGBoost. Choices: auto, exact, approx, hist, gpu_hist (gpu_hist only supported with GPU) diff --git a/src/bin/gs_guc/pg_guc.cpp b/src/bin/gs_guc/pg_guc.cpp index a075fb8ed..885c455da 100644 --- a/src/bin/gs_guc/pg_guc.cpp +++ b/src/bin/gs_guc/pg_guc.cpp @@ -2547,10 +2547,6 @@ int main(int argc, char** argv) hba_param = ((char**)pg_malloc_zero(arraysize * sizeof(char *))); config_value = ((char**)pg_malloc_zero(arraysize * sizeof(char*))); } - if (false == allocate_memory_list()) { - write_stderr(_("ERROR: Failed to allocate memory to list.\n")); - exit(1); - } key_mode = SERVER_MODE; /* @@ -2769,8 +2765,54 @@ int main(int argc, char** argv) do_advice(); exit(1); } + char arguments[MAX_BUF_SIZE] = {0x00}; + for (int i = 0; i < argc; i++) { + if ((strlen(arguments) + strlen(argv[i])) >= (MAX_BUF_SIZE - 2)) { + if (*arguments) { + (void)write_log("The gs_guc run with the following arguments: [%s].\n", arguments); + } + (void)write_log("The gs_guc run with the following arguments: [%s].\n", argv[i]); + rc = memset_s(arguments, MAX_BUF_SIZE, 0, MAX_BUF_SIZE - 1); + securec_check_c(rc, "\0", "\0"); + continue; + } + errno_t rc = strcat_s(arguments, MAX_BUF_SIZE, argv[i]); + size_t len = strlen(arguments); + if (rc != EOK) { + break; + } + arguments[len] = ' '; + arguments[len + 1] = '\0'; + } + if (*arguments) { + (void)write_log("The gs_guc run with the following arguments: [%s].\n", arguments); + } check_encrypt_options(); + + if (ctl_command == ENCRYPT_KEY_COMMAND) { + process_encrypt_cmd(pgdata_D, pgdata_C, pgdata_R); + (void)write_log("gs_guc encrypt %s\n", loginfo); + } else if (ctl_command == GENERATE_KEY_COMMAND) { + doGenerateOperation(pgdata_D, loginfo); + } + + if (ctl_command == ENCRYPT_KEY_COMMAND || ctl_command == GENERATE_KEY_COMMAND) { + GS_FREE(g_prefix); + GS_FREE(g_plainkey); + GS_FREE(g_cipherkey); + GS_FREE(key_username); + GS_FREE(pgdata_D); + GS_FREE(pgdata_R); + GS_FREE(pgdata_C); + return 0; + } + + if (false == allocate_memory_list()) { + write_stderr(_("ERROR: Failed to allocate memory to list.\n")); + exit(1); + } + if (ctl_command != ENCRYPT_KEY_COMMAND && ctl_command != GENERATE_KEY_COMMAND && (!bhave_param && !is_hba_conf)) { write_stderr(_("%s: the form of this command is incorrect\n"), progname); do_advice(); @@ -2832,30 +2874,6 @@ int main(int argc, char** argv) // log output redirect init_log(PROG_NAME); - /* print the log about arguments of gs_guc */ - char arguments[MAX_BUF_SIZE] = {0x00}; - for (int i = 0; i < argc; i++) { - if ((strlen(arguments) + strlen(argv[i])) >= (MAX_BUF_SIZE - 2)) { - if (*arguments) { - (void)write_log("The gs_guc run with the following arguments: [%s].\n", arguments); - } - (void)write_log("The gs_guc run with the following arguments: [%s].\n", argv[i]); - rc = memset_s(arguments, MAX_BUF_SIZE, 0, MAX_BUF_SIZE - 1); - securec_check_c(rc, "\0", "\0"); - continue; - } - errno_t rc = strcat_s(arguments, MAX_BUF_SIZE, argv[i]); - size_t len = strlen(arguments); - if (rc != EOK) { - break; - } - arguments[len] = ' '; - arguments[len + 1] = '\0'; - } - if (*arguments) { - (void)write_log("The gs_guc run with the following arguments: [%s].\n", arguments); - } - if ((true == is_hba_conf) && ((nodetype != INSTANCE_COORDINATOR) && (nodetype != INSTANCE_DATANODE))) { write_stderr(_("%s: authentication operation (-h) is not supported for \"gtm\" or \"gtm_proxy\"\n"), progname); @@ -2863,54 +2881,38 @@ int main(int argc, char** argv) exit(1); } - if (ctl_command == ENCRYPT_KEY_COMMAND) { - process_encrypt_cmd(pgdata_D, pgdata_C, pgdata_R); - (void)write_log("gs_guc encrypt %s\n", loginfo); - } else if (ctl_command == GENERATE_KEY_COMMAND) { - doGenerateOperation(pgdata_D, loginfo); - } else { - // the number of -Z is equal to 2 - if (node_type_number == LARGE_INSTANCE_NUM) { - if (node_type_value[0] == node_type_value[1]) { - (void)write_stderr("When the number of -Z is equal to 2, the value must be different.\n"); - exit(1); - } - - for (int index = 0; index < LARGE_INSTANCE_NUM; index++) { - if (node_type_value[index] != INSTANCE_COORDINATOR && node_type_value[index] != INSTANCE_DATANODE) { - (void)write_stderr("ERROR: When the number of -Z is equal to 2, the parameter value of -Z must be " - "coordinator or datanode.\n"); - exit(1); - } - checkLcName(node_type_value[index]); - } - nodetype = INSTANCE_COORDINATOR; - - if (0 != validate_cluster_guc_options(nodename, nodetype, instance_name, pgdata_D)) { - exit(1); - } - process_cluster_guc_option(nodename, nodetype, instance_name, pgdata_D); - } else { - checkLcName(nodetype); - if (0 != validate_cluster_guc_options(nodename, nodetype, instance_name, pgdata_D)) { - exit(1); - } - process_cluster_guc_option(nodename, nodetype, instance_name, pgdata_D); + // the number of -Z is equal to 2 + if (node_type_number == LARGE_INSTANCE_NUM) { + if (node_type_value[0] == node_type_value[1]) { + (void)write_stderr("When the number of -Z is equal to 2, the value must be different.\n"); + exit(1); } + + for (int index = 0; index < LARGE_INSTANCE_NUM; index++) { + if (node_type_value[index] != INSTANCE_COORDINATOR && node_type_value[index] != INSTANCE_DATANODE) { + (void)write_stderr("ERROR: When the number of -Z is equal to 2, the parameter value of -Z must be " + "coordinator or datanode.\n"); + exit(1); + } + checkLcName(node_type_value[index]); + } + nodetype = INSTANCE_COORDINATOR; + + if (0 != validate_cluster_guc_options(nodename, nodetype, instance_name, pgdata_D)) { + exit(1); + } + process_cluster_guc_option(nodename, nodetype, instance_name, pgdata_D); + } else { + checkLcName(nodetype); + if (0 != validate_cluster_guc_options(nodename, nodetype, instance_name, pgdata_D)) { + exit(1); + } + process_cluster_guc_option(nodename, nodetype, instance_name, pgdata_D); } - GS_FREE(g_prefix); - GS_FREE(g_plainkey); - GS_FREE(g_cipherkey); - GS_FREE(key_username); GS_FREE(pgdata_D); - GS_FREE(pgdata_R); - GS_FREE(pgdata_C); GS_FREE(instance_name); - if (ctl_command == ENCRYPT_KEY_COMMAND || ctl_command == GENERATE_KEY_COMMAND) - return 0; - nRet = print_guc_result((const char*)nodename); GS_FREE(nodename); clear_g_incorrect_nodeInfo(); diff --git a/src/bin/pg_ctl/pg_build.cpp b/src/bin/pg_ctl/pg_build.cpp index c004a06d2..59ce732bb 100755 --- a/src/bin/pg_ctl/pg_build.cpp +++ b/src/bin/pg_ctl/pg_build.cpp @@ -25,6 +25,7 @@ #include "bin/elog.h" #include "nodes/pg_list.h" +#include "replication/replicainternal.h" #include "storage/smgr/fd.h" #include "utils/builtins.h" #include "utils/datetime.h" @@ -64,6 +65,7 @@ char g_buildprimary_slotname[MAX_VALUE_LEN] = {0}; char g_str_replication_type[MAX_VALUE_LEN] = {0}; int g_replconn_idx = -1; int g_replication_type = -1; +bool is_cross_region_build = false; #define RT_WITH_DUMMY_STANDBY 0 #define RT_WITH_MULTI_STANDBY 1 @@ -576,7 +578,9 @@ void get_conninfo(const char* filename) exit(1); } - if (IS_CROSS_CLUSTER_BUILD) { + if (build_mode == CROSS_CLUSTER_FULL_BUILD || build_mode == CROSS_CLUSTER_INC_BUILD || + build_mode == CROSS_CLUSTER_STANDBY_FULL_BUILD) { + /* For shared storage cluster */ conninfo_para = config_para_cross_cluster_build; } else { conninfo_para = config_para_build; @@ -864,10 +868,11 @@ PGconn* check_and_conn(int conn_timeout, int recv_timeout, uint32 term) tnRet = memset_s(repl_conninfo_str, MAXPGPATH, 0, MAXPGPATH); securec_check_ss_c(tnRet, "", ""); + is_cross_region_build = false; if (register_username != NULL && register_password != NULL) { if (*register_username == '.') { register_username += 2; - } + } tnRet = snprintf_s(repl_conninfo_str, sizeof(repl_conninfo_str), sizeof(repl_conninfo_str) - 1, @@ -882,6 +887,7 @@ PGconn* check_and_conn(int conn_timeout, int recv_timeout, uint32 term) repl_conn_info.remoteport, conn_timeout, recv_timeout, register_username, register_password); + is_cross_region_build = true; } else { tnRet = snprintf_s(repl_conninfo_str, sizeof(repl_conninfo_str), diff --git a/src/bin/psql/input.cpp b/src/bin/psql/input.cpp index 8ed2f1d56..5b804f32a 100644 --- a/src/bin/psql/input.cpp +++ b/src/bin/psql/input.cpp @@ -244,7 +244,9 @@ bool SensitiveStrCheck(const char* target) if (strstr(target_copy, "PASSWORD") != NULL || strstr(target_copy, "IDENTIFIED") != NULL || strstr(target_copy, "GS_ENCRYPT_AES128") != NULL || strstr(target_copy, "GS_DECRYPT_AES128") != NULL || - strstr(target_copy, "GS_ENCRYPT") != NULL || strstr(target_copy, "GS_DECRYPT") != NULL) { + strstr(target_copy, "GS_ENCRYPT") != NULL || strstr(target_copy, "GS_DECRYPT") != NULL || + strstr(target_copy, "PG_CREATE_PHYSICAL_REPLICATION_SLOT_EXTERN") != NULL || + strstr(target_copy, "SECRETKEY") != NULL || strstr(target_copy, "CREATE_CREDENTIAL") != NULL) { free(target_copy); return TRUE; } else { diff --git a/src/bin/psql/startup.cpp b/src/bin/psql/startup.cpp index a444a3d70..c9dedc011 100644 --- a/src/bin/psql/startup.cpp +++ b/src/bin/psql/startup.cpp @@ -46,8 +46,6 @@ PsqlSettings pset; /* Used for change child process name in gsql parallel execute mode. */ char* argv_para; int argv_num; -static int PASSWORD_STR_LEN = 9; -static bool dbname_alloced = false; static bool is_pipeline = false; static bool is_interactive = true; #ifndef ENABLE_MULTIPLE_NODES @@ -630,6 +628,10 @@ int main(int argc, char* argv[]) /* Stored connection and guc info for new connections in gsql parallel mode. */ values_free[3] = true; + if (options.dbname != NULL) { + /* When we use a new dbname or exit the program, we need to free the value. */ + values_free[4] = true; /* The dbname index is 4 */ + } pset.connInfo.keywords = keywords; pset.connInfo.values = values; @@ -828,6 +830,12 @@ int main(int argc, char* argv[]) for (int i = 0; i < PARAMS_ARRAY_SIZE; i++) { if (pset.connInfo.values_free[i] && NULL != pset.connInfo.values[i]) { if (strlen(pset.connInfo.values[i]) != 0) { + /* Erase the connection information in the memory. */ + rc = memset_s(pset.connInfo.values[i], + strlen(pset.connInfo.values[i]), + 0, + strlen(pset.connInfo.values[i])); + securec_check_c(rc, "\0", "\0"); free(pset.connInfo.values[i]); pset.connInfo.values[i] = NULL; } @@ -859,13 +867,6 @@ int main(int argc, char* argv[]) ResetQueryRetryController(); EmptyRetryErrcodesList(pset.errcodes_list); - if (dbname_alloced) { - rc = memset_s(options.dbname, strlen(options.dbname), 0, strlen(options.dbname)); - securec_check_c(rc, "\0", "\0"); - free(options.dbname); - options.dbname = NULL; - } - return successResult; } @@ -1014,7 +1015,7 @@ static void parse_psql_options(int argc, char* const argv[], struct adhoc_opts* bool is_action_file = false; /* Database Security: Data importing/dumping support AES128. */ char* dencrypt_key = NULL; - char* needmask = NULL; + char* dbname = NULL; errno_t rc = EOK; #ifdef USE_READLINE useReadline = false; @@ -1054,9 +1055,7 @@ static void parse_psql_options(int argc, char* const argv[], struct adhoc_opts* } break; case 'd': - options->dbname = pg_strdup(optarg); - dbname_alloced = true; - needmask = optarg; + dbname = optarg; break; case 'e': if (!SetVariable(pset.vars, "ECHO", "queries")) { @@ -1263,10 +1262,8 @@ static void parse_psql_options(int argc, char* const argv[], struct adhoc_opts* * if we still have arguments, use it as the database name and username */ while (argc - optind >= 1) { - if (options->dbname == NULL) { - options->dbname = pg_strdup(argv[optind]); - dbname_alloced = true; - needmask = argv[optind]; + if (dbname == NULL) { + dbname = argv[optind]; } else if (options->username == NULL) { options->username = argv[optind]; } else if (!pset.quiet) { @@ -1276,21 +1273,25 @@ static void parse_psql_options(int argc, char* const argv[], struct adhoc_opts* optind++; } - if (needmask != NULL) { + /* mask Password information stored in dbname */ + if (dbname != NULL) { + /* Save a copy for connection before masking the password. */ + options->dbname = pg_strdup(dbname); + /* mask informations in URI string. */ - if (strncmp(options->dbname, "postgresql://", strlen("postgresql://")) == 0) { - char *off_argv = needmask + strlen("postgresql://"); + if (strncmp(dbname, "postgresql://", strlen("postgresql://")) == 0) { + char *off_argv = dbname + strlen("postgresql://"); rc = memset_s(off_argv, strlen(off_argv), '*', strlen(off_argv)); check_memset_s(rc); - } else if (strncmp(options->dbname, "postgres://", strlen("postgres://")) == 0) { - char *off_argv = needmask + strlen("postgres://"); + } else if (strncmp(dbname, "postgres://", strlen("postgres://")) == 0) { + char *off_argv = dbname + strlen("postgres://"); rc = memset_s(off_argv, strlen(off_argv), '*', strlen(off_argv)); check_memset_s(rc); } - /* mask password */ + /* mask password in key/value string. */ char *temp = NULL; - if ((temp = strstr(needmask, "password")) != NULL) { - char *off_argv = temp + PASSWORD_STR_LEN; + if ((temp = strstr(dbname, "password")) != NULL) { + char *off_argv = temp + strlen("password"); rc = memset_s(off_argv, strlen(off_argv), '*', strlen(off_argv)); check_memset_s(rc); } diff --git a/src/common/backend/catalog/builtin_funcs.ini b/src/common/backend/catalog/builtin_funcs.ini index 95ba20633..d489c8b2e 100755 --- a/src/common/backend/catalog/builtin_funcs.ini +++ b/src/common/backend/catalog/builtin_funcs.ini @@ -3031,6 +3031,10 @@ "get_byte", 1, AddBuiltinFunc(_0(721), _1("get_byte"), _2(2), _3(true), _4(false), _5(byteaGetByte), _6(23), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('i'), _19(0), _20(2, 17, 23), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("byteaGetByte"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33("get byte"), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), + AddFuncGroup( + "get_client_info", 1, + AddBuiltinFunc(_0(7732), _1("get_client_info"), _2(0), _3(false), _4(true), _5(get_client_info), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(100), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(0), _21(2, 20, 25), _22(2, 'o', 'o'), _23(2, "sid", "client_info"), _24(NULL), _25("get_client_info"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(true), _32(false), _33("read current client"), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + ), AddFuncGroup( "get_current_ts_config", 1, AddBuiltinFunc(_0(3759), _1("get_current_ts_config"), _2(0), _3(true), _4(false), _5(get_current_ts_config), _6(3734), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(0), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("get_current_ts_config"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33("get current tsearch configuration"), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) @@ -8173,14 +8177,14 @@ "pg_rotate_logfile", 1, AddBuiltinFunc(_0(2622), _1("pg_rotate_logfile"), _2(0), _3(true), _4(false), _5(pg_rotate_logfile), _6(16), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('v'), _19(0), _20(0), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("pg_rotate_logfile"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33("rotate log file - old version for adminpack 1.0"), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) ), - AddFuncGroup( - "pg_sequence_parameters", 1, - AddBuiltinFunc(_0(3078), _1("pg_sequence_parameters"), _2(1), _3(true), _4(false), _5(pg_sequence_parameters), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(1, 26), _21(6, 26, 34, 34, 34, 34, 16), _22(6, 'i', 'o', 'o', 'o', 'o', 'o'), _23(6, "sequence_oid", "start_value", "minimum_value", "maximum_value", "increment", "cycle_option"), _24(NULL), _25("pg_sequence_parameters"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33("sequence parameters, for use by information schema"), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) - ), AddFuncGroup( "pg_sequence_last_value", 1, AddBuiltinFunc(_0(3080), _1("pg_sequence_last_value"), _2(1), _3(true), _4(false), _5(pg_sequence_last_value), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(1, 26), _21(3, 26, 34, 34), _22(3, 'i', 'o', 'o'), _23(3, "sequence_oid", "cache_value", "last_value"), _24(NULL), _25("pg_sequence_last_value"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false)) ), + AddFuncGroup( + "pg_sequence_parameters", 1, + AddBuiltinFunc(_0(3078), _1("pg_sequence_parameters"), _2(1), _3(true), _4(false), _5(pg_sequence_parameters), _6(2249), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(0), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(1, 26), _21(6, 26, 34, 34, 34, 34, 16), _22(6, 'i', 'o', 'o', 'o', 'o', 'o'), _23(6, "sequence_oid", "start_value", "minimum_value", "maximum_value", "increment", "cycle_option"), _24(NULL), _25("pg_sequence_parameters"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33("sequence parameters, for use by information schema"), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) + ), AddFuncGroup( "pg_shared_memctx_detail", 1, AddBuiltinFunc(_0(3987), _1("pg_shared_memctx_detail"), _2(1), _3(false), _4(true), _5(pg_shared_memctx_detail), _6(16), _7(PG_CATALOG_NAMESPACE), _8(BOOTSTRAP_SUPERUSERID), _9(INTERNALlanguageId), _10(1), _11(100), _12(0), _13(0), _14(false), _15(false), _16(false), _17(false), _18('s'), _19(0), _20(1, 2275), _21(NULL), _22(NULL), _23(NULL), _24(NULL), _25("pg_shared_memctx_detail"), _26(NULL), _27(NULL), _28(NULL), _29(0), _30(false), _31(NULL), _32(false), _33(NULL), _34('f'), _35(NULL), _36(0), _37(false), _38(NULL), _39(NULL), _40(0)) diff --git a/src/common/backend/catalog/dependency.cpp b/src/common/backend/catalog/dependency.cpp index 4a73e2de5..1c21397fc 100644 --- a/src/common/backend/catalog/dependency.cpp +++ b/src/common/backend/catalog/dependency.cpp @@ -2509,6 +2509,8 @@ char* getObjectDescription(const ObjectAddress* object) initStringInfo(&buffer); + char* signature = NULL; + switch (getObjectClass(object)) { case OCLASS_CLASS: getRelationDescription(&buffer, object->objectId); @@ -2518,11 +2520,15 @@ char* getObjectDescription(const ObjectAddress* object) break; case OCLASS_PROC: - appendStringInfo(&buffer, _("function %s"), format_procedure(object->objectId)); + signature = format_procedure(object->objectId); + appendStringInfo(&buffer, _("function %s"), signature); + pfree_ext(signature); break; case OCLASS_PACKAGE: - appendStringInfo(&buffer, _("package %s"), format_procedure(object->objectId)); + signature = format_procedure(object->objectId); + appendStringInfo(&buffer, _("package %s"), signature); + pfree_ext(signature); break; case OCLASS_TYPE: @@ -2776,7 +2782,7 @@ char* getObjectDescription(const ObjectAddress* object) initStringInfo(&opfam); getOpFamilyDescription(&opfam, amprocForm->amprocfamily); - + signature = format_procedure(amprocForm->amproc); /* ------ translator: %d is the function number, the first two %s's are data type names, the third %s is the description of the @@ -2788,8 +2794,9 @@ char* getObjectDescription(const ObjectAddress* object) format_type_be(amprocForm->amproclefttype), format_type_be(amprocForm->amprocrighttype), opfam.data, - format_procedure(amprocForm->amproc)); + signature); + pfree_ext(signature); pfree_ext(opfam.data); systable_endscan(amscan); diff --git a/src/common/backend/catalog/gs_package.cpp b/src/common/backend/catalog/gs_package.cpp index f336b8ea3..d7c53a832 100644 --- a/src/common/backend/catalog/gs_package.cpp +++ b/src/common/backend/catalog/gs_package.cpp @@ -1105,8 +1105,29 @@ static List* BuildFuncInfoList(PLpgSQL_execstate* estate) void BuildSessionPackageRuntimeForAutoSession(uint64 sessionId, uint64 parentSessionId, PLpgSQL_execstate* estate, PLpgSQL_function* func) { - SessionPackageRuntime* parentSessionPkgs = NULL; + MemoryContext pkgRuntimeCtx = AllocSetContextCreate(CurrentMemoryContext, + "SessionPackageRuntime", + ALLOCSET_SMALL_MINSIZE, + ALLOCSET_SMALL_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + MemoryContext oldCtx = MemoryContextSwitchTo(pkgRuntimeCtx); + SessionPackageRuntime* resultSessionPkgs = (SessionPackageRuntime*)palloc0(sizeof(SessionPackageRuntime)); + resultSessionPkgs->context = pkgRuntimeCtx; + /* doing insert gs_source, no need to restore package value */ + if (func->is_insert_gs_source) { + resultSessionPkgs->is_insert_gs_source = true; + g_instance.global_session_pkg->Add(sessionId, resultSessionPkgs); + MemoryContextSwitchTo(oldCtx); + MemoryContextDelete(resultSessionPkgs->context); + return; + } + + if (u_sess->plsql_cxt.plpgsqlpkg_dlist_objects != NULL) { + CopyCurrentSessionPkgs(resultSessionPkgs, u_sess->plsql_cxt.plpgsqlpkg_dlist_objects); + } + + SessionPackageRuntime* parentSessionPkgs = NULL; /* get parent session pkgs, build current session pkgs need include them */ if (parentSessionId != 0) { if (!u_sess->plsql_cxt.not_found_parent_session_pkgs) { @@ -1120,17 +1141,6 @@ void BuildSessionPackageRuntimeForAutoSession(uint64 sessionId, uint64 parentSes } } } - MemoryContext pkgRuntimeCtx = AllocSetContextCreate(CurrentMemoryContext, - "SessionPackageRuntime", - ALLOCSET_SMALL_MINSIZE, - ALLOCSET_SMALL_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); - MemoryContext oldCtx = MemoryContextSwitchTo(pkgRuntimeCtx); - SessionPackageRuntime* resultSessionPkgs = (SessionPackageRuntime*)palloc0(sizeof(SessionPackageRuntime)); - resultSessionPkgs->context = pkgRuntimeCtx; - if (u_sess->plsql_cxt.plpgsqlpkg_dlist_objects != NULL) { - CopyCurrentSessionPkgs(resultSessionPkgs, u_sess->plsql_cxt.plpgsqlpkg_dlist_objects); - } if (parentSessionPkgs) { List* parentPkgList = parentSessionPkgs->runtimes; @@ -1387,6 +1397,9 @@ void initAutoSessionPkgsValue(uint64 sessionId) if (sessionPkgs->runtimes == NULL) { return; } + if (sessionPkgs->is_insert_gs_source) { + return; + } foreach(cell, sessionPkgs->runtimes) { pkgState = (PackageRuntimeState*)lfirst(cell); @@ -1510,11 +1523,7 @@ void initAutonomousPkgValue(PLpgSQL_package* targetPkg, uint64 sessionId) List *processAutonmSessionPkgs(PLpgSQL_function* func, PLpgSQL_execstate* estate, bool isAutonm) { - List *autonmsList = NULL; - /* ignore inline_code_block function */ - if (!OidIsValid(func->fn_oid)) { - return NULL; - } + List *autonmsList = NIL; uint64 currentSessionId = IS_THREAD_POOL_WORKER ? u_sess->session_id : t_thrd.proc_cxt.MyProcPid; @@ -1525,6 +1534,13 @@ List *processAutonmSessionPkgs(PLpgSQL_function* func, PLpgSQL_execstate* estate * sessionpkgs from g_instance.global_session_pkg */ uint64 automnSessionId = u_sess->SPI_cxt.autonomous_session->current_attach_sessionid; + if (func->is_insert_gs_source) { + /* doing insert gs_source, need do noting */ + g_instance.global_session_pkg->Remove(automnSessionId); + g_instance.global_session_pkg->Remove(currentSessionId); + return NIL; + } + SessionPackageRuntime* sessionpkgs = g_instance.global_session_pkg->Fetch(automnSessionId); RestoreAutonmSessionPkgs(sessionpkgs); if (sessionpkgs != NULL) { @@ -1546,7 +1562,11 @@ List *processAutonmSessionPkgs(PLpgSQL_function* func, PLpgSQL_execstate* estate * and restore package values by it. */ if (u_sess->is_autonomous_session == true && u_sess->SPI_cxt._connected == 0) { - BuildSessionPackageRuntimeForParentSession(currentSessionId, estate); + /* doing insert gs_source, need do noting */ + if (u_sess->plsql_cxt.auto_parent_session_pkgs == NULL + || !u_sess->plsql_cxt.auto_parent_session_pkgs->is_insert_gs_source) { + BuildSessionPackageRuntimeForParentSession(currentSessionId, estate); + } /* autonomous session will be reused by next autonomous procedure, need clean it */ if (u_sess->plsql_cxt.auto_parent_session_pkgs != NULL) { MemoryContextDelete(u_sess->plsql_cxt.auto_parent_session_pkgs->context); @@ -1560,11 +1580,6 @@ List *processAutonmSessionPkgs(PLpgSQL_function* func, PLpgSQL_execstate* estate void processAutonmSessionPkgsInException(PLpgSQL_function* func) { - /* ignore inline_code_block function */ - if (!OidIsValid(func->fn_oid)) { - return; - } - uint64 currentSessionId = IS_THREAD_POOL_WORKER ? u_sess->session_id : t_thrd.proc_cxt.MyProcPid; if (IsAutonomousTransaction(func->action->isAutonomous)) { @@ -1578,6 +1593,13 @@ void processAutonmSessionPkgsInException(PLpgSQL_function* func) return; } uint64 automnSessionId = u_sess->SPI_cxt.autonomous_session->current_attach_sessionid; + if (func->is_insert_gs_source) { + /* doing insert gs_source, need do noting */ + g_instance.global_session_pkg->Remove(automnSessionId); + g_instance.global_session_pkg->Remove(currentSessionId); + return; + } + SessionPackageRuntime* sessionpkgs = g_instance.global_session_pkg->Fetch(automnSessionId); RestoreAutonmSessionPkgs(sessionpkgs); if (sessionpkgs != NULL) { @@ -1596,7 +1618,11 @@ void processAutonmSessionPkgsInException(PLpgSQL_function* func) * and restore package values by it. */ if (u_sess->is_autonomous_session == true && u_sess->SPI_cxt._connected == 0) { - BuildSessionPackageRuntimeForParentSession(currentSessionId, NULL); + /* doing insert gs_source, need do noting */ + if (u_sess->plsql_cxt.auto_parent_session_pkgs == NULL + || !u_sess->plsql_cxt.auto_parent_session_pkgs->is_insert_gs_source) { + BuildSessionPackageRuntimeForParentSession(currentSessionId, NULL); + } /* autonomous session will be reused by next autonomous procedure, need clean it */ if (u_sess->plsql_cxt.auto_parent_session_pkgs != NULL) { MemoryContextDelete(u_sess->plsql_cxt.auto_parent_session_pkgs->context); diff --git a/src/common/backend/catalog/heap.cpp b/src/common/backend/catalog/heap.cpp index 2afa519e6..2452e6fa7 100644 --- a/src/common/backend/catalog/heap.cpp +++ b/src/common/backend/catalog/heap.cpp @@ -6245,14 +6245,21 @@ Oid AddNewIntervalPartition(Relation rel, void* insertTuple) CacheInvalidateRelcache(rel); } + /* + * to avoid dead lock, we should release AccessShareLock on ADD_PARTITION_ACTION + * locked by the transaction before aquire AccessExclusiveLock. + */ + UnlockRelationForAccessIntervalPartTabIfHeld(rel); /* it will accept invalidation messages generated by other sessions in lockRelationForAddIntervalPartition. */ - lockRelationForAddIntervalPartition(rel); + LockRelationForAddIntervalPartition(rel); partitionRoutingForTuple(rel, insertTuple, u_sess->catalog_cxt.route); /* if the partition exists, return partition's oid */ if (u_sess->catalog_cxt.route->fileExist) { Assert(OidIsValid(u_sess->catalog_cxt.route->partitionId)); - unLockRelationForAddIntervalPartition(rel); + /* we should take AccessShareLock again before release AccessExclusiveLock for consistency. */ + LockRelationForAccessIntervalPartitionTab(rel); + UnlockRelationForAddIntervalPartition(rel); return u_sess->catalog_cxt.route->partitionId; } @@ -6320,6 +6327,8 @@ Oid AddNewIntervalPartition(Relation rel, void* insertTuple) */ CommandCounterIncrement(); + UpdatePgObjectChangecsn(RelationGetRelid(rel), rel->rd_rel->relkind); + return newPartOid; } diff --git a/src/common/backend/catalog/index.cpp b/src/common/backend/catalog/index.cpp index 1613448a6..86fbacc1b 100644 --- a/src/common/backend/catalog/index.cpp +++ b/src/common/backend/catalog/index.cpp @@ -5862,7 +5862,7 @@ void AddGPIForSubPartition(Oid partTableOid, Oid partOid, Oid subPartOid) * @@GaussDB@@ * Target : This routine is used to scan partition tuples and delete them from all global partition indexes. */ -void ScanPartitionDeleteGPITuples(Relation partTableRel, Relation partRel, const List* indexRelList, +void ScanPartitionDeleteGPITuples(Relation partTableRel, Relation partRel, const List* indexRelList, const List* indexInfoList) { TableScanDesc scan = NULL; @@ -5893,9 +5893,8 @@ void ScanPartitionDeleteGPITuples(Relation partTableRel, Relation partRel, const Relation indexRel = (Relation)lfirst(cell); IndexInfo* indexInfo = static_cast(lfirst(cell1)); - if (!RelationIsUstoreIndex(indexRel)) { - continue; /* only ubtree have index_delete routine */ - } + /* only ubtree have index_delete routine */ + Assert(RelationIsUstoreIndex(indexRel)); Datum values[tupleDesc->natts]; bool isNull[tupleDesc->natts]; @@ -5988,7 +5987,7 @@ bool DeleteGPITuplesForPartition(Oid partTableOid, Oid partOid) * Description : * Notes : */ -void DeleteGPITuplesForSubPartition(Oid partTableOid, Oid partOid, Oid subPartOid) +bool DeleteGPITuplesForSubPartition(Oid partTableOid, Oid partOid, Oid subPartOid) { Relation partTableRel = NULL; Relation partRel = NULL; @@ -5999,6 +5998,7 @@ void DeleteGPITuplesForSubPartition(Oid partTableOid, Oid partOid, Oid subPartOi List* indexRelList = NIL; List* indexInfoList = NIL; ListCell* cell = NULL; + bool all_ubtree = true; partTableRel = heap_open(partTableOid, AccessShareLock); part = partitionOpen(partTableRel, partOid, AccessShareLock); @@ -6012,6 +6012,12 @@ void DeleteGPITuplesForSubPartition(Oid partTableOid, Oid partOid, Oid subPartOi Relation indexRel = relation_open(indexOid, RowExclusiveLock); IndexInfo* indexInfo = BuildIndexInfo(indexRel); + if (!RelationIsUstoreIndex(indexRel)) { + all_ubtree = false; + relation_close(indexRel, RowExclusiveLock); + continue; + } + indexRelList = lappend(indexRelList, indexRel); indexInfoList = lappend(indexInfoList, indexInfo); } @@ -6033,6 +6039,8 @@ void DeleteGPITuplesForSubPartition(Oid partTableOid, Oid partOid, Oid subPartOi releaseDummyRelation(&partRel); partitionClose(partTableRel, part, NoLock); heap_close(partTableRel, NoLock); + + return all_ubtree; } void mergeBTreeIndexes(List* mergingBtreeIndexes, List* srcPartMergeOffset, int2 bktId) diff --git a/src/common/backend/catalog/namespace.cpp b/src/common/backend/catalog/namespace.cpp index 36a80a56b..814dfc062 100644 --- a/src/common/backend/catalog/namespace.cpp +++ b/src/common/backend/catalog/namespace.cpp @@ -285,6 +285,7 @@ Oid RangeVarGetRelidExtended(const RangeVar* relation, LOCKMODE lockmode, bool m (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg("temporary tables cannot specify a schema name"))); } + pfree_ext(errDetail); errDetail = get_relname_relid_extend( relation->relname, u_sess->catalog_cxt.myTempNamespace, &relId, isSupportSynonym, refSynOid); } @@ -293,6 +294,7 @@ Oid RangeVarGetRelidExtended(const RangeVar* relation, LOCKMODE lockmode, bool m /* use exact schema given */ namespaceId = LookupExplicitNamespace(relation->schemaname); + pfree_ext(errDetail); errDetail = get_relname_relid_extend(relation->relname, namespaceId, &relId, isSupportSynonym, refSynOid); if (OidIsValid(relId) && namespaceId == u_sess->catalog_cxt.myTempNamespace) @@ -300,6 +302,7 @@ Oid RangeVarGetRelidExtended(const RangeVar* relation, LOCKMODE lockmode, bool m } else { /* search the namespace path */ if (isSupportSynonym) { + pfree_ext(errDetail); errDetail = RelnameGetRelidExtended(relation->relname, &relId, refSynOid, detailInfo); } else { relId = RelnameGetRelid(relation->relname, detailInfo); @@ -419,8 +422,8 @@ Oid RangeVarGetRelidExtended(const RangeVar* relation, LOCKMODE lockmode, bool m /* Skipping report error, but store the error detail info and report later. */ appendStringInfo(detailInfo, _("%s"), errDetail); } - pfree_ext(errDetail); } + pfree_ext(errDetail); if (!OidIsValid(relId) && !missing_ok) { if (relation->schemaname) diff --git a/src/common/backend/catalog/pg_object.cpp b/src/common/backend/catalog/pg_object.cpp index 7f90f0bfd..1d3b9fa6b 100644 --- a/src/common/backend/catalog/pg_object.cpp +++ b/src/common/backend/catalog/pg_object.cpp @@ -377,6 +377,9 @@ void UpdatePgObjectChangecsn(Oid objectOid, PgObjectType objectType) if (!CheckObjectExist(objectOid, objectType)) { return; } + if (u_sess->exec_cxt.isExecTrunc) { + return; + } relation = heap_open(PgObjectRelationId, RowExclusiveLock); tup = SearchSysCache2(PGOBJECTID, ObjectIdGetDatum(objectOid), CharGetDatum(objectType)); if (!HeapTupleIsValid(tup)) { diff --git a/src/common/backend/catalog/pg_partition.cpp b/src/common/backend/catalog/pg_partition.cpp index dc9747fa2..7298850a4 100644 --- a/src/common/backend/catalog/pg_partition.cpp +++ b/src/common/backend/catalog/pg_partition.cpp @@ -1425,3 +1425,41 @@ Oid GetBaseRelOidOfParition(Relation relation) return relation->parentId; } +/* NB: all operations on ADD_PARTITION_ACTION sequence lock must use TopTransactionResourceOwner. */ +void LockRelationForAddIntervalPartition(Relation rel) +{ + ResourceOwner currentOwner = t_thrd.utils_cxt.CurrentResourceOwner; + t_thrd.utils_cxt.CurrentResourceOwner = t_thrd.utils_cxt.TopTransactionResourceOwner; + LockPartition(RelationGetRelid(rel), ADD_PARTITION_ACTION, + AccessExclusiveLock, PARTITION_SEQUENCE_LOCK); + t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; +} + +void LockRelationForAccessIntervalPartitionTab(Relation rel) +{ + ResourceOwner currentOwner = t_thrd.utils_cxt.CurrentResourceOwner; + t_thrd.utils_cxt.CurrentResourceOwner = t_thrd.utils_cxt.TopTransactionResourceOwner; + LockPartition(RelationGetRelid(rel), ADD_PARTITION_ACTION, + AccessShareLock, PARTITION_SEQUENCE_LOCK); + t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; +} + +void UnlockRelationForAccessIntervalPartTabIfHeld(Relation rel) +{ + ResourceOwner currentOwner = t_thrd.utils_cxt.CurrentResourceOwner; + t_thrd.utils_cxt.CurrentResourceOwner = t_thrd.utils_cxt.TopTransactionResourceOwner; + + UnlockPartitionSeqIfHeld(RelationGetRelid(rel), ADD_PARTITION_ACTION, AccessShareLock); + t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; +} + +void UnlockRelationForAddIntervalPartition(Relation rel) +{ + ResourceOwner currentOwner = t_thrd.utils_cxt.CurrentResourceOwner; + t_thrd.utils_cxt.CurrentResourceOwner = t_thrd.utils_cxt.TopTransactionResourceOwner; + + UnlockPartition(RelationGetRelid(rel), ADD_PARTITION_ACTION, + AccessExclusiveLock, PARTITION_SEQUENCE_LOCK); + t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; +} + diff --git a/src/common/backend/catalog/system_views.sql b/src/common/backend/catalog/system_views.sql index 10bee9baf..058724cab 100644 --- a/src/common/backend/catalog/system_views.sql +++ b/src/common/backend/catalog/system_views.sql @@ -245,7 +245,7 @@ CREATE VIEW pg_rlspolicies AS END AS policypermissive, CASE WHEN pol.polroles = '{0}' THEN - string_to_array('public', ' ') + pg_catalog.string_to_array('public', ' ') ELSE ARRAY ( @@ -549,33 +549,33 @@ REVOKE ALL on pg_statistic FROM public; REVOKE ALL on pg_statistic_ext FROM public; CREATE VIEW pg_locks AS - SELECT * FROM pg_lock_status() AS L; + SELECT * FROM pg_catalog.pg_lock_status() AS L; CREATE VIEW pg_cursors AS - SELECT * FROM pg_cursor() AS C; + SELECT * FROM pg_catalog.pg_cursor() AS C; CREATE VIEW pg_available_extensions AS SELECT E.name, E.default_version, X.extversion AS installed_version, E.comment - FROM pg_available_extensions() AS E + FROM pg_catalog.pg_available_extensions() AS E LEFT JOIN pg_extension AS X ON E.name = X.extname; CREATE VIEW pg_available_extension_versions AS SELECT E.name, E.version, (X.extname IS NOT NULL) AS installed, E.superuser, E.relocatable, E.schema, E.requires, E.comment - FROM pg_available_extension_versions() AS E + FROM pg_catalog.pg_available_extension_versions() AS E LEFT JOIN pg_extension AS X ON E.name = X.extname AND E.version = X.extversion; CREATE VIEW pg_prepared_xacts AS SELECT P.transaction, P.gid, P.prepared, U.rolname AS owner, D.datname AS database - FROM pg_prepared_xact() AS P + FROM pg_catalog.pg_prepared_xact() AS P LEFT JOIN pg_authid U ON P.ownerid = U.oid LEFT JOIN pg_database D ON P.dbid = D.oid; CREATE VIEW pg_prepared_statements AS - SELECT * FROM pg_prepared_statement() AS P; + SELECT * FROM pg_catalog.pg_prepared_statement() AS P; CREATE VIEW pg_seclabels AS SELECT @@ -719,7 +719,7 @@ FROM JOIN pg_authid rol ON l.classoid = rol.tableoid AND l.objoid = rol.oid; CREATE VIEW pg_settings AS - SELECT * FROM pg_show_all_settings() AS A; + SELECT * FROM pg_catalog.pg_show_all_settings() AS A; CREATE RULE pg_settings_u AS ON UPDATE TO pg_settings @@ -733,13 +733,13 @@ CREATE RULE pg_settings_n AS GRANT SELECT, UPDATE ON pg_settings TO PUBLIC; CREATE VIEW pg_timezone_abbrevs AS - SELECT * FROM pg_timezone_abbrevs(); + SELECT * FROM pg_catalog.pg_timezone_abbrevs(); CREATE VIEW pg_timezone_names AS - SELECT * FROM pg_timezone_names(); + SELECT * FROM pg_catalog.pg_timezone_names(); CREATE VIEW pg_control_group_config AS - SELECT * FROM pg_control_group_config(); + SELECT * FROM pg_catalog.pg_control_group_config(); -- Statistics views @@ -824,7 +824,7 @@ CREATE VIEW pg_statio_all_tables AS pg_catalog.pg_stat_get_blocks_hit(C.oid) AS heap_blks_hit, pg_catalog.sum(pg_catalog.pg_stat_get_blocks_fetched(I.indexrelid) - pg_catalog.pg_stat_get_blocks_hit(I.indexrelid))::bigint AS idx_blks_read, - pg_catalog.sum(pg_stat_get_blocks_hit(I.indexrelid))::bigint AS idx_blks_hit, + pg_catalog.sum(pg_catalog.pg_stat_get_blocks_hit(I.indexrelid))::bigint AS idx_blks_hit, pg_catalog.pg_stat_get_blocks_fetched(T.oid) - pg_catalog.pg_stat_get_blocks_hit(T.oid) AS toast_blks_read, pg_catalog.pg_stat_get_blocks_hit(T.oid) AS toast_blks_hit, @@ -882,7 +882,7 @@ CREATE VIEW pg_statio_all_indexes AS N.nspname AS schemaname, C.relname AS relname, I.relname AS indexrelname, - pg_stat_get_blocks_fetched(I.oid) - + pg_catalog.pg_stat_get_blocks_fetched(I.oid) - pg_catalog.pg_stat_get_blocks_hit(I.oid) AS idx_blks_read, pg_catalog.pg_stat_get_blocks_hit(I.oid) AS idx_blks_hit FROM pg_class C JOIN @@ -906,7 +906,7 @@ CREATE VIEW pg_statio_all_sequences AS C.oid AS relid, N.nspname AS schemaname, C.relname AS relname, - pg_stat_get_blocks_fetched(C.oid) - + pg_catalog.pg_stat_get_blocks_fetched(C.oid) - pg_catalog.pg_stat_get_blocks_hit(C.oid) AS blks_read, pg_catalog.pg_stat_get_blocks_hit(C.oid) AS blks_hit FROM pg_class C @@ -1141,7 +1141,7 @@ SELECT FROM pg_stat_activity_ng AS S, pg_catalog.pg_stat_get_wlm_realtime_session_info(NULL) AS T WHERE S.pid = T.threadid; -CREATE OR REPLACE FUNCTION gs_wlm_get_all_user_resource_info() +CREATE OR REPLACE FUNCTION pg_catalog.gs_wlm_get_all_user_resource_info() RETURNS setof record AS $$ DECLARE @@ -1162,7 +1162,7 @@ DECLARE LANGUAGE 'plpgsql' NOT FENCED; CREATE VIEW pg_total_user_resource_info_oid AS -SELECT * FROM gs_wlm_get_all_user_resource_info() AS +SELECT * FROM pg_catalog.gs_wlm_get_all_user_resource_info() AS (userid Oid, used_memory int, total_memory int, @@ -1228,7 +1228,7 @@ create table gs_wlm_user_resource_history REVOKE all on gs_wlm_user_resource_history FROM public; -CREATE OR REPLACE FUNCTION gs_wlm_persistent_user_resource_info() +CREATE OR REPLACE FUNCTION pg_catalog.gs_wlm_persistent_user_resource_info() RETURNS setof record AS $$ DECLARE @@ -1270,7 +1270,7 @@ create table gs_wlm_instance_history REVOKE ALL on gs_wlm_instance_history FROM public; -CREATE OR REPLACE FUNCTION create_wlm_instance_statistics_info() +CREATE OR REPLACE FUNCTION pg_catalog.create_wlm_instance_statistics_info() RETURNS int AS $$ DECLARE @@ -1612,18 +1612,18 @@ CREATE VIEW gs_wlm_workload_records AS WHERE P.query_pid = S.threadpid AND S.usesysid = U.oid; -CREATE VIEW gs_os_run_info AS SELECT * FROM pv_os_run_info(); -CREATE VIEW gs_session_memory_context AS SELECT * FROM pv_session_memory_detail(); -CREATE VIEW gs_thread_memory_context AS SELECT * FROM pv_thread_memory_detail(); -CREATE VIEW gs_shared_memory_detail AS SELECT * FROM pg_shared_memory_detail(); -CREATE VIEW gs_instance_time AS SELECT * FROM pv_instance_time(); -CREATE VIEW gs_session_time AS SELECT * FROM pv_session_time(); -CREATE VIEW gs_session_memory AS SELECT * FROM pv_session_memory(); -CREATE VIEW gs_total_memory_detail AS SELECT * FROM pv_total_memory_detail(); -CREATE VIEW pg_total_memory_detail AS SELECT * FROM pv_total_memory_detail(); -CREATE VIEW gs_redo_stat AS SELECT * FROM pg_stat_get_redo_stat(); -CREATE VIEW gs_session_stat AS SELECT * FROM pv_session_stat(); -CREATE VIEW gs_file_stat AS SELECT * FROM pg_stat_get_file_stat(); +CREATE VIEW gs_os_run_info AS SELECT * FROM pg_catalog.pv_os_run_info(); +CREATE VIEW gs_session_memory_context AS SELECT * FROM pg_catalog.pv_session_memory_detail(); +CREATE VIEW gs_thread_memory_context AS SELECT * FROM pg_catalog.pv_thread_memory_detail(); +CREATE VIEW gs_shared_memory_detail AS SELECT * FROM pg_catalog.pg_shared_memory_detail(); +CREATE VIEW gs_instance_time AS SELECT * FROM pg_catalog.pv_instance_time(); +CREATE VIEW gs_session_time AS SELECT * FROM pg_catalog.pv_session_time(); +CREATE VIEW gs_session_memory AS SELECT * FROM pg_catalog.pv_session_memory(); +CREATE VIEW gs_total_memory_detail AS SELECT * FROM pg_catalog.pv_total_memory_detail(); +CREATE VIEW pg_total_memory_detail AS SELECT * FROM pg_catalog.pv_total_memory_detail(); +CREATE VIEW gs_redo_stat AS SELECT * FROM pg_catalog.pg_stat_get_redo_stat(); +CREATE VIEW gs_session_stat AS SELECT * FROM pg_catalog.pv_session_stat(); +CREATE VIEW gs_file_stat AS SELECT * FROM pg_catalog.pg_stat_get_file_stat(); CREATE OR REPLACE FUNCTION pg_catalog.gs_session_memory_detail_tp(OUT sessid TEXT, OUT sesstype TEXT, OUT contextname TEXT, OUT level INT2, OUT parent TEXT, OUT totalsize INT8, OUT freesize INT8, OUT usedsize INT8) RETURNS setof record @@ -1718,7 +1718,7 @@ BEGIN END; $$ LANGUAGE plpgsql NOT FENCED; -CREATE VIEW gs_session_memory_detail AS SELECT * FROM gs_session_memory_detail_tp() ORDER BY sessid; +CREATE VIEW gs_session_memory_detail AS SELECT * FROM pg_catalog.gs_session_memory_detail_tp() ORDER BY sessid; CREATE VIEW pg_stat_replication AS SELECT @@ -1754,7 +1754,7 @@ CREATE VIEW pg_replication_slots AS L.catalog_xmin, L.restart_lsn, L.dummy_standby - FROM pg_get_replication_slots() AS L + FROM pg_catalog.pg_get_replication_slots() AS L LEFT JOIN pg_database D ON (L.datoid = D.oid); @@ -1819,17 +1819,17 @@ CREATE VIEW pg_stat_xact_user_functions AS CREATE VIEW pg_stat_bgwriter AS SELECT - pg_stat_get_bgwriter_timed_checkpoints() AS checkpoints_timed, - pg_stat_get_bgwriter_requested_checkpoints() AS checkpoints_req, - pg_stat_get_checkpoint_write_time() AS checkpoint_write_time, - pg_stat_get_checkpoint_sync_time() AS checkpoint_sync_time, - pg_stat_get_bgwriter_buf_written_checkpoints() AS buffers_checkpoint, - pg_stat_get_bgwriter_buf_written_clean() AS buffers_clean, - pg_stat_get_bgwriter_maxwritten_clean() AS maxwritten_clean, - pg_stat_get_buf_written_backend() AS buffers_backend, - pg_stat_get_buf_fsync_backend() AS buffers_backend_fsync, - pg_stat_get_buf_alloc() AS buffers_alloc, - pg_stat_get_bgwriter_stat_reset_time() AS stats_reset; + pg_catalog.pg_stat_get_bgwriter_timed_checkpoints() AS checkpoints_timed, + pg_catalog.pg_stat_get_bgwriter_requested_checkpoints() AS checkpoints_req, + pg_catalog.pg_stat_get_checkpoint_write_time() AS checkpoint_write_time, + pg_catalog.pg_stat_get_checkpoint_sync_time() AS checkpoint_sync_time, + pg_catalog.pg_stat_get_bgwriter_buf_written_checkpoints() AS buffers_checkpoint, + pg_catalog.pg_stat_get_bgwriter_buf_written_clean() AS buffers_clean, + pg_catalog.pg_stat_get_bgwriter_maxwritten_clean() AS maxwritten_clean, + pg_catalog.pg_stat_get_buf_written_backend() AS buffers_backend, + pg_catalog.pg_stat_get_buf_fsync_backend() AS buffers_backend_fsync, + pg_catalog.pg_stat_get_buf_alloc() AS buffers_alloc, + pg_catalog.pg_stat_get_bgwriter_stat_reset_time() AS stats_reset; CREATE VIEW pg_user_mappings AS SELECT @@ -1856,7 +1856,7 @@ REVOKE ALL on pg_user_mapping FROM public; -- these functions are added for supporting default format transformation CREATE OR REPLACE FUNCTION pg_catalog.to_char(NUMERIC) RETURNS VARCHAR2 -AS $$ SELECT CAST(numeric_out($1) AS VARCHAR2) $$ +AS $$ SELECT CAST(pg_catalog.numeric_out($1) AS VARCHAR2) $$ LANGUAGE SQL STRICT IMMUTABLE NOT FENCED; CREATE OR REPLACE FUNCTION pg_catalog.to_char(INT2) @@ -1891,7 +1891,7 @@ LANGUAGE SQL STRICT IMMUTABLE NOT FENCED; CREATE OR REPLACE FUNCTION pg_catalog.to_number(TEXT) RETURNS NUMERIC -AS $$ SELECT pg_catalog.numeric_in(textout($1), 0::Oid, -1) $$ +AS $$ SELECT pg_catalog.numeric_in(pg_catalog.textout($1), 0::Oid, -1) $$ LANGUAGE SQL STRICT IMMUTABLE NOT FENCED; CREATE CAST (VARCHAR2 AS RAW) WITH FUNCTION pg_catalog.hextoraw(text) AS IMPLICIT; @@ -2413,7 +2413,7 @@ begin end if; --source string to source_array for i in 1..pg_catalog.length($1) loop - if substr($1,i,1) ~ '\n' then + if pg_catalog.substr($1,i,1) ~ '\n' then if position = i then source_array(source_line) := '\n'; else @@ -2469,12 +2469,12 @@ begin exit; end if; end loop; - if left($2,1) = '^' then + if pg_catalog.left($2,1) = '^' then regex_temp := pg_catalog.substr($2,2); else regex_temp := $2; end if; - if right($2,1) = '$' then + if pg_catalog.right($2,1) = '$' then regex_temp := pg_catalog.substr(regex_temp,1,pg_catalog.length(regex_temp)-1); end if; if flag then @@ -2497,7 +2497,7 @@ begin return false; end if; end loop; - case right($3, 1) + case pg_catalog.right($3, 1) when 'i' then return $1 ~* $2; when 'c' then return $1 ~ $2; when 'm' then return pg_catalog.regex_like_m($1,$2); @@ -2523,13 +2523,13 @@ SQL_STMT VARCHAR2(500); fail_cursor REFCURSOR; success_cursor REFCURSOR; BEGIN - SELECT text(oid) FROM pg_catalog.pg_authid WHERE rolname=SESSION_USER INTO user_id; + SELECT pg_catalog.text(oid) FROM pg_catalog.pg_authid WHERE rolname=SESSION_USER INTO user_id; SELECT SESSION_USER INTO user_name; SELECT pg_catalog.CURRENT_DATABASE() INTO db_name; IF flag = true THEN SQL_STMT := 'SELECT username,database,time,type,result,client_conninfo FROM pg_catalog.pg_query_audit(''1970-1-1'',''9999-12-31'') WHERE - type IN (''login_success'') AND username =' || quote_literal(user_name) || - ' AND database =' || quote_literal(db_name) || ' AND userid =' || quote_literal(user_id) || ';'; + type IN (''login_success'') AND username =' || pg_catalog.quote_literal(user_name) || + ' AND database =' || pg_catalog.quote_literal(db_name) || ' AND userid =' || pg_catalog.quote_literal(user_id) || ';'; OPEN success_cursor FOR EXECUTE SQL_STMT; --search bottom up for all the success login info FETCH LAST FROM success_cursor into username, database, logintime, mytype, result, client_conninfo; @@ -2540,8 +2540,8 @@ BEGIN CLOSE success_cursor; ELSE SQL_STMT := 'SELECT username,database,time,type,result,client_conninfo FROM pg_catalog.pg_query_audit(''1970-1-1'',''9999-12-31'') WHERE - type IN (''login_success'', ''login_failed'') AND username =' || quote_literal(user_name) || - ' AND database =' || quote_literal(db_name) || ' AND userid =' || quote_literal(user_id) || ';'; + type IN (''login_success'', ''login_failed'') AND username =' || pg_catalog.quote_literal(user_name) || + ' AND database =' || pg_catalog.quote_literal(db_name) || ' AND userid =' || pg_catalog.quote_literal(user_id) || ';'; OPEN fail_cursor FOR EXECUTE SQL_STMT; --search bottom up FETCH LAST FROM fail_cursor into username, database, logintime, mytype, result, client_conninfo; @@ -2580,15 +2580,15 @@ success_cursor REFCURSOR; mybackendid bigint; curSessionFound boolean; BEGIN - SELECT text(oid) FROM pg_catalog.pg_authid WHERE rolname=SESSION_USER INTO user_id; + SELECT pg_catalog.text(oid) FROM pg_catalog.pg_authid WHERE rolname=SESSION_USER INTO user_id; SELECT SESSION_USER INTO user_name; SELECT pg_catalog.CURRENT_DATABASE() INTO db_name; SELECT pg_catalog.pg_backend_pid() INTO mybackendid; curSessionFound = false; IF flag = true THEN SQL_STMT := 'SELECT username,database,time,type,result,client_conninfo, pg_catalog.split_part(thread_id,''@'',1) backendid FROM pg_catalog.pg_query_audit(''1970-1-1'',''9999-12-31'') WHERE - type IN (''login_success'') AND username =' || quote_literal(user_name) || - ' AND database =' || quote_literal(db_name) || ' AND userid =' || quote_literal(user_id) || ';'; + type IN (''login_success'') AND username =' || pg_catalog.quote_literal(user_name) || + ' AND database =' || pg_catalog.quote_literal(db_name) || ' AND userid =' || pg_catalog.quote_literal(user_id) || ';'; OPEN success_cursor FOR EXECUTE SQL_STMT; --search bottom up for all the success login info FETCH LAST FROM success_cursor into username, database, logintime, mytype, result, client_conninfo, backendid; @@ -2785,14 +2785,14 @@ DECLARE execute_query text; BEGIN if row_num = 0 then - EXECUTE 'select count(1) from ' || table_name into tolal_num; - execute_query = 'select seqNum, count(1) as num - from (select table_data_skewness(row(' || column_name ||'), ''H'') as seqNum from ' || table_name || + EXECUTE 'select pg_catalog.count(1) from ' || table_name into tolal_num; + execute_query = 'select seqNum, pg_catalog.count(1) as num + from (select pg_catalog.table_data_skewness(row(' || column_name ||'), ''H'') as seqNum from ' || table_name || ') group by seqNum order by num DESC'; else tolal_num = row_num; - execute_query = 'select seqNum, count(1) as num - from (select table_data_skewness(row(' || column_name ||'), ''H'') as seqNum from ' || table_name || + execute_query = 'select seqNum, pg_catalog.count(1) as num + from (select pg_catalog.table_data_skewness(row(' || column_name ||'), ''H'') as seqNum from ' || table_name || ' limit ' || row_num ||') group by seqNum order by num DESC'; end if; @@ -2965,11 +2965,11 @@ DECLARE ec_username, ec_query, ec_libodbc_type - FROM pg_stat_get_wlm_ec_operator_info(0) where ec_operator > 0'; + FROM pg_catalog.pg_stat_get_wlm_ec_operator_info(0) where ec_operator > 0'; - query_plan_str := 'SELECT * FROM gs_stat_get_wlm_plan_operator_info(0)'; + query_plan_str := 'SELECT * FROM pg_catalog.gs_stat_get_wlm_plan_operator_info(0)'; - query_str := 'SELECT * FROM pg_stat_get_wlm_operator_info(1)'; + query_str := 'SELECT * FROM pg_catalog.pg_stat_get_wlm_operator_info(1)'; IF flag > 0 THEN EXECUTE 'INSERT INTO gs_wlm_ec_operator_info ' || query_ec_str; @@ -3060,7 +3060,7 @@ DECLARE dop = row_data.query_dop; condition = row_data.condition; projection = row_data.projection; - query_str_encode := 'SELECT encode_plan_node($tag$'|| operation ||'$tag$,$tag$'|| orientation ||'$tag$,$tag$'|| strategy ||'$tag$,$tag$ '|| options || '$tag$,$tag$'|| dop ||'$tag$,$tag$' || condition || '$tag$,$tag$' || projection || '$tag$) as result;'; + query_str_encode := 'SELECT pg_catalog.encode_plan_node($tag$'|| operation ||'$tag$,$tag$'|| orientation ||'$tag$,$tag$'|| strategy ||'$tag$,$tag$ '|| options || '$tag$,$tag$'|| dop ||'$tag$,$tag$' || condition || '$tag$,$tag$' || projection || '$tag$) as result;'; EXECUTE query_str_encode INTO encoded_data; encode = encoded_data.result; return next; @@ -3255,7 +3255,7 @@ DECLARE IF row_info_data.parttype = 'n' THEN query_str := 'SELECT relname,oid from pg_class where oid= '||row_info_data.reldeltarelid||''; EXECUTE(query_str) INTO row_data; - query_select_str := 'select count(*) from cstore.' || row_data.relname || ''; + query_select_str := 'select pg_catalog.count(*) from cstore.' || row_data.relname || ''; EXECUTE (query_select_str) INTO live_tuple; query_size_str := 'select * from pg_catalog.pg_relation_size(' || row_data.oid || ')'; EXECUTE (query_size_str) INTO data_size; @@ -3268,7 +3268,7 @@ DECLARE query_str := 'SELECT relname,oid from pg_class where oid = '||row_part_info.reldeltarelid||''; part_name := row_part_info.relname; FOR row_data IN EXECUTE(query_str) LOOP - query_select_str := 'select count(*) from cstore.' || row_data.relname || ''; + query_select_str := 'select pg_catalog.count(*) from cstore.' || row_data.relname || ''; EXECUTE (query_select_str) INTO live_tuple; query_size_str := 'select * from pg_catalog.pg_relation_size(' || row_data.oid || ')'; EXECUTE (query_size_str) INTO data_size; @@ -3321,7 +3321,7 @@ DECLARE BEGIN query_database_oid := 'SELECT datname FROM pg_database WHERE datallowconn = true order by datname'; for databse_name in EXECUTE(query_database_oid) LOOP - unlock_str = format('SELECT * FROM pg_catalog.pgxc_unlock_for_sp_database(''%s'')', databse_name.datname); + unlock_str = pg_catalog.format('SELECT * FROM pg_catalog.pgxc_unlock_for_sp_database(''%s'')', databse_name.datname); begin EXECUTE(unlock_str) into unlock_result; if unlock_result = 'f' then @@ -3522,11 +3522,11 @@ CREATE VIEW pg_catalog.gs_db_privileges AS FROM pg_catalog.gs_db_privilege; CREATE OR REPLACE VIEW pg_catalog.gs_gsc_memory_detail AS - SELECT db_id, sum(totalsize) AS totalsize, sum(freesize) AS freesize, sum(usedsize) AS usedsize + SELECT db_id, pg_catalog.sum(totalsize) AS totalsize, pg_catalog.sum(freesize) AS freesize, pg_catalog.sum(usedsize) AS usedsize FROM ( SELECT - CASE WHEN contextname like '%GlobalSysDBCacheEntryMemCxt%' THEN substring(contextname, 29) - ELSE substring(parent, 29) END AS db_id, + CASE WHEN contextname like '%GlobalSysDBCacheEntryMemCxt%' THEN pg_catalog.substring(contextname, 29) + ELSE pg_catalog.substring(parent, 29) END AS db_id, totalsize, freesize, usedsize diff --git a/src/common/backend/catalog/toasting.cpp b/src/common/backend/catalog/toasting.cpp index 63c14f8fc..4369d70b6 100644 --- a/src/common/backend/catalog/toasting.cpp +++ b/src/common/backend/catalog/toasting.cpp @@ -745,9 +745,11 @@ static void InitTempToastNamespace(void) /* Advance command counter to make namespace visible */ CommandCounterIncrement(); - - Assert(OidIsValid(u_sess->catalog_cxt.myTempNamespace) && OidIsValid(u_sess->catalog_cxt.myTempToastNamespace)); - + if (!OidIsValid(u_sess->catalog_cxt.myTempToastNamespace)) { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Temp toast namespace create failed"))); + } u_sess->catalog_cxt.baseSearchPathValid = false; } diff --git a/src/common/backend/client_logic/client_logic.cpp b/src/common/backend/client_logic/client_logic.cpp index 7800f3cdc..479a84b4e 100644 --- a/src/common/backend/client_logic/client_logic.cpp +++ b/src/common/backend/client_logic/client_logic.cpp @@ -50,6 +50,7 @@ #include "tcop/utility.h" #include "pgxc/pgxc.h" #include "utils/fmgroids.h" +#include "funcapi.h" const size_t ENCRYPTED_VALUE_MIN_LENGTH = 170; const size_t ENCRYPTED_VALUE_MAX_LENGTH = 1024; @@ -1324,3 +1325,28 @@ ClientLogicColumnRef *get_column_enc_def(Oid rel_oid, const char *col_name) return enc_def; } + +Datum get_client_info(PG_FUNCTION_ARGS) +{ + ReturnSetInfo* rsinfo = (ReturnSetInfo*)fcinfo->resultinfo; + TupleDesc tupdesc; + Tuplestorestate* tupstore = NULL; + const int COLUMN_NUM = 2; + MemoryContext oldcontext = MemoryContextSwitchTo(rsinfo->econtext->ecxt_per_query_memory); + tupdesc = CreateTemplateTupleDesc(COLUMN_NUM, false, TAM_HEAP); + TupleDescInitEntry(tupdesc, (AttrNumber)1, "sid", INT8OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber)2, "client_info", TEXTOID, -1, 0); + + tupstore = tuplestore_begin_heap(true, false, u_sess->attr.attr_memory.work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = BlessTupleDesc(tupdesc); + + (void)MemoryContextSwitchTo(oldcontext); + if (ENABLE_THREAD_POOL) { + g_threadPoolControler->GetSessionCtrl()->getSessionClientInfo(rsinfo->setResult, rsinfo->setDesc); + } + + tuplestore_donestoring(rsinfo->setResult); + return (Datum)0; +} \ No newline at end of file diff --git a/src/common/backend/nodes/outfuncs.cpp b/src/common/backend/nodes/outfuncs.cpp index 1bd1ccb61..dbb4adb3d 100755 --- a/src/common/backend/nodes/outfuncs.cpp +++ b/src/common/backend/nodes/outfuncs.cpp @@ -678,6 +678,9 @@ static void _outPruningResult(StringInfo str, PruningResult* node) if (t_thrd.proc->workingVersionNum >= num) { WRITE_NODE_FIELD(expr); } + if (t_thrd.proc->workingVersionNum >= PBESINGLEPARTITION_VERSION_NUM) { + WRITE_BOOL_FIELD(isPbeSinlePartition); + } } static void _outSubPartitionPruningResult(StringInfo str, SubPartitionPruningResult* node) diff --git a/src/common/backend/nodes/readfuncs.cpp b/src/common/backend/nodes/readfuncs.cpp index 01d04dad2..e16993a4b 100755 --- a/src/common/backend/nodes/readfuncs.cpp +++ b/src/common/backend/nodes/readfuncs.cpp @@ -3312,6 +3312,9 @@ static PruningResult* _readPruningResult(PruningResult* local_node) if (t_thrd.proc->workingVersionNum >= num) { READ_NODE_FIELD(expr); } + IF_EXIST(isPbeSinlePartition) { + READ_BOOL_FIELD(isPbeSinlePartition); + } READ_DONE(); } diff --git a/src/common/backend/parser/gram.y b/src/common/backend/parser/gram.y index 4d57dcb6b..9d73d43e2 100644 --- a/src/common/backend/parser/gram.y +++ b/src/common/backend/parser/gram.y @@ -19590,6 +19590,11 @@ for_locking_items: for_locking_item: FOR UPDATE hint_string locked_rels_list opt_nowait { + if (u_sess->parser_cxt.isTimeCapsule) { + u_sess->parser_cxt.isTimeCapsule = false; + ereport(errstate, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("SELECT TIMECAPSULE FOR UPDATE is not supported."))); + } LockingClause *n = makeNode(LockingClause); n->lockedRels = $4; n->forUpdate = TRUE; @@ -20193,22 +20198,22 @@ timecapsule_clause: TIMECAPSULE opt_timecapsule_clause { $$ = $2; } opt_timecapsule_clause: - CSN a_expr + CSN {u_sess->parser_cxt.isTimeCapsule = true;} a_expr { TcapFeatureEnsure(); RangeTimeCapsule *n = makeNode(RangeTimeCapsule); n->tvtype = TV_VERSION_CSN; - n->tvver = (Node *)$2; - n->location = @2; + n->tvver = (Node *)$3; + n->location = @3; $$ = (Node *) n; } - | TIMESTAMP a_expr + | TIMESTAMP {u_sess->parser_cxt.isTimeCapsule = true;} a_expr { TcapFeatureEnsure(); RangeTimeCapsule *n = makeNode(RangeTimeCapsule); n->tvtype = TV_VERSION_TIMESTAMP; - n->tvver = (Node *)$2; - n->location = @2; + n->tvver = (Node *)$3; + n->location = @3; $$ = (Node *) n; } ; diff --git a/src/common/backend/parser/parse_hint.cpp b/src/common/backend/parser/parse_hint.cpp index a531309d7..90794a82f 100755 --- a/src/common/backend/parser/parse_hint.cpp +++ b/src/common/backend/parser/parse_hint.cpp @@ -402,6 +402,10 @@ static void PredpushHintDesc(PredpushHint* hint, StringInfo buf) relnamesToBuf(base_hint.relnames, buf); + if (hint->dest_name != NULL) { + appendStringInfo(buf, ", %s", hint->dest_name); + } + if (hint->candidates != NULL) appendStringInfo(buf, ")"); @@ -3325,6 +3329,50 @@ static void transform_skew_hint(PlannerInfo* root, Query* parse, List* skew_hint parse->hintState->skew_hint = skew_hint_transf_l; } +/* + * Check Predpush hint rely on each other. + */ +static void check_predpush_cycle_hint(PlannerInfo *root, + List *predpush_hint_list, + PredpushHint *predpush_hint) +{ + ListCell *lc = NULL; + + if (bms_num_members(predpush_hint->candidates) != 1) { + return; + } + + if (predpush_hint->dest_id == 0) { + return; + } + + int cur_dest = predpush_hint->dest_id; + foreach(lc, predpush_hint_list) { + PredpushHint *prev_hint = (PredpushHint *)lfirst(lc); + + if (prev_hint == predpush_hint) { + break; + } + + if (bms_num_members(prev_hint->candidates) != 1) { + continue; + } + + if (prev_hint->dest_id == 0) { + continue; + } + + int prev_dest = prev_hint->dest_id; + if (bms_is_member(prev_dest, predpush_hint->candidates) && + bms_is_member(cur_dest, prev_hint->candidates)) { + append_warning_to_list( + root, (Hint*)predpush_hint, "Error hint:%s, Predpush cannot rely on each other.", hint_string); + } + } + + return; +} + /* * @Description: Transform predpush hint into processible type, including: * transfrom subquery name @@ -3350,6 +3398,7 @@ static void transform_predpush_hint(PlannerInfo* root, Query* parse, List* predp } predpush_hint->dest_id = relid; + check_predpush_cycle_hint(root, predpush_hint_list, predpush_hint); } return; @@ -3866,4 +3915,4 @@ bool CheckNodeNameHint(HintState* hintstate) } } return false; -} \ No newline at end of file +} diff --git a/src/common/backend/parser/parse_relation.cpp b/src/common/backend/parser/parse_relation.cpp index 9acfa8817..6d6c73582 100755 --- a/src/common/backend/parser/parse_relation.cpp +++ b/src/common/backend/parser/parse_relation.cpp @@ -1134,6 +1134,11 @@ Relation parserOpenTable(ParseState *pstate, const RangeVar *relation, int lockm TryUnlockAllAccounts(); } + if (rel->partMap && rel->partMap->type == PART_TYPE_INTERVAL) { + /* take AccessShareLock on ADD_PARTITION_ACTION to avoid concurrency with new partition operations. */ + LockRelationForAccessIntervalPartitionTab(rel); + } + if (IS_PGXC_COORDINATOR && !IsConnFromCoord()) { if (u_sess->attr.attr_sql.enable_parallel_ddl && !isFirstNode && isCreateView) { UnlockRelation(rel, lockmode); diff --git a/src/common/backend/parser/parse_startwith.cpp b/src/common/backend/parser/parse_startwith.cpp index 36ba6938a..551321563 100644 --- a/src/common/backend/parser/parse_startwith.cpp +++ b/src/common/backend/parser/parse_startwith.cpp @@ -1091,10 +1091,12 @@ static void transformStartWithClause(StartWithTransformContext *context, SelectS raw_expression_tree_walker((Node*)context->connectByExpr, (bool (*)())pseudo_level_rownum_walker, (Node*)context->connectByExpr); checkConnectByExprValidity((Node*)connectByExpr); + StartWithWalker(context, connectByExpr); context->relInfoList = context->pstate->p_start_info; - context->connect_by_type = CONNECT_BY_LEVEL; - context->connectByLevelExpr = connectByExpr; - context->connectByOtherExpr = NULL; + if (context->connect_by_type != CONNECT_BY_PRIOR) { + context->connectByLevelExpr = connectByExpr; + context->connectByOtherExpr = NULL; + } } /* transform start with ... connect by's expr */ @@ -1103,12 +1105,24 @@ static void transformStartWithClause(StartWithTransformContext *context, SelectS StartWithWalker(context, connectByExpr); } - /* now handle where quals which could whole push down */ + /* + * now handle where quals which might need to be pushed down. + * 1. this is necessary only for implicitly joined tables + * such as ... FROM t1,t2 WHERE t1.xx = t2.yy ... + * 2. note that only join quals should be pushed down while + * non-join quals such as (t1.xx < n) should be kept in the outer loop + * of the CTE scan, otherwise the end-result will be different from + * those produced by the standard swcb syntax. + * (the issue here is that implicitly joined tables are difficult to handle + * in our implementation of start with .. connect by .. syntax, + * as we don't have a clear cut of join quals from the non-join quals at this stage. + * users should be encouraged to use explicity joined tables whenever + * possible before a clear-cut solution is implemented.) + */ int lens = list_length(context->pstate->p_start_info); if (lens != 1) { Node *whereClause = (Node *)copyObject(stmt->whereClause); context->whereClause = whereClause; - stmt->whereClause = NULL; } @@ -1256,6 +1270,36 @@ static void AddWithClauseToBranch(ParseState *pstate, SelectStmt *stmt, List *re return; } +static bool walker_to_exclude_non_join_quals(Node *node, Node *context_node) +{ + if (node == NULL) { + return false; + } + + if (!IsA(node, A_Expr)) { + return raw_expression_tree_walker(node, (bool (*)()) walker_to_exclude_non_join_quals, (void*)NULL); + } + + A_Expr* expr = (A_Expr*) node; + /* + * this is to achieve consistent result sets with those produced by the original + * start with .. connect by syntax, which does not push filter quals down to connect quals. + * if non-column item appears on any side of an operator, we guess that it is + * not a join qual so should not be filtered in sw op, and force it to be true. + * this rule is not always correct but should work fine most of the time. + * could be improved later on, e.g. find better ways to extract non-join quals + * from the where clause. + */ + if (expr->kind == AEXPR_OP && + (!IsA(expr->lexpr, ColumnRef) || !IsA(expr->rexpr, ColumnRef))) { + expr->lexpr = makeBoolAConst(true, -1); + expr->rexpr = makeBoolAConst(true, -1); + expr->kind = AEXPR_OR; + } + + return false; +} + /* * -------------------------------------------------------------------------------------- * @Brief: Create SWCB's conversion CTE's inner branch, normally we add ConnectByExpr to @@ -1314,6 +1358,10 @@ static SelectStmt *CreateStartWithCTEInnerBranch(ParseState* pstate, JoinExpr *final_join = (JoinExpr *)origin_table; /* pushdown requires deep copying of the quals */ Node *whereCopy = (Node *)copyObject(whereClause); + /* only join quals can be pushed down */ + raw_expression_tree_walker((Node *)whereCopy, + (bool (*)())walker_to_exclude_non_join_quals, (void*)NULL); + if (final_join->quals == NULL) { final_join->quals = whereCopy; } else { @@ -1420,11 +1468,18 @@ static SelectStmt *CreateStartWithCTEOuterBranch(ParseState *pstate, /* push whereClause down to init part, taking care to avoid NULL in expr. */ quals = (Node *)startWithExpr; + Node* whereClauseCopy = (Node *)copyObject(whereClause); + + if (whereClause != NULL) { + /* only join quals can be pushed down */ + raw_expression_tree_walker((Node*)whereClauseCopy, + (bool (*)())walker_to_exclude_non_join_quals, (void*)NULL); + } + if (quals == NULL) { - /* pushdown requires deep copying of the quals */ - quals = (Node *)copyObject(whereClause); + quals = whereClauseCopy; } else if (whereClause != NULL) { - quals = (Node *)makeA_Expr(AEXPR_AND, NULL, (Node *)copyObject(whereClause), + quals = (Node *)makeA_Expr(AEXPR_AND, NULL, whereClauseCopy, (Node*)startWithExpr, -1); } diff --git a/src/common/backend/parser/parser.cpp b/src/common/backend/parser/parser.cpp index ba75d042c..d91fb17cc 100644 --- a/src/common/backend/parser/parser.cpp +++ b/src/common/backend/parser/parser.cpp @@ -28,6 +28,11 @@ extern void resetOperatorPlusFlag(); +static void resetIsTimeCapsuleFlag() +{ + u_sess->parser_cxt.isTimeCapsule = false; +} + static void resetCreateFuncFlag() { u_sess->parser_cxt.isCreateFuncOrProc = false; @@ -48,6 +53,9 @@ List* raw_parser(const char* str, List** query_string_locationlist) /* reset u_sess->parser_cxt.stmt_contains_operator_plus */ resetOperatorPlusFlag(); + /* reset u_sess->parser_cxt.isTimeCapsule */ + resetIsTimeCapsuleFlag(); + /* reset u_sess->parser_cxt.isCreateFuncOrProc */ resetCreateFuncFlag(); diff --git a/src/common/backend/pgxc_single/barrier/barrier.cpp b/src/common/backend/pgxc_single/barrier/barrier.cpp index 8d1adb3a5..7b001da33 100755 --- a/src/common/backend/pgxc_single/barrier/barrier.cpp +++ b/src/common/backend/pgxc_single/barrier/barrier.cpp @@ -871,12 +871,14 @@ static void ExecuteBarrier(const char* id, bool isSwitchoverBarrier) /* Only obs-based disaster recovery needs the following processing */ if (g_instance.archive_obs_cxt.archive_slot_num != 0 && g_instance.archive_obs_cxt.barrier_lsn_info != NULL) { +#ifdef ENABLE_MULTIPLE_NODES if (IS_HADR_BARRIER(id) || IS_CSN_BARRIER(id)) { SpinLockAcquire(&g_instance.archive_obs_cxt.barrier_lock); SaveAllNodeBarrierLsnInfo(id, conn_handles); g_instance.archive_obs_cxt.barrier_lsn_info[connCnt].barrierLsn = recptr; SpinLockRelease(&g_instance.archive_obs_cxt.barrier_lock); } +#endif if (t_thrd.role == BARRIER_CREATOR && !isSwitchoverBarrier) { UpdateGlobalBarrierListOnMedia(id, g_instance.attr.attr_common.PGXCNodeName); @@ -1017,7 +1019,7 @@ static void RequestXLogFromStream() static void barrier_redo_pause(char* barrierId) { - if (!is_barrier_pausable(barrierId)) { + if (!is_barrier_pausable(barrierId) || t_thrd.xlog_cxt.recoveryTarget == RECOVERY_TARGET_TIME_OBS) { return; } volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; @@ -1035,7 +1037,7 @@ static void barrier_redo_pause(char* barrierId) SpinLockRelease(&walrcv->mutex); pg_usleep(1000L); RedoInterruptCallBack(); - if(IS_OBS_DISASTER_RECOVER_MODE) { + if (IS_OBS_DISASTER_RECOVER_MODE) { update_recovery_barrier(); } else if (IS_DISASTER_RECOVER_MODE) { RequestXLogFromStream(); diff --git a/src/common/backend/utils/adt/ruleutils.cpp b/src/common/backend/utils/adt/ruleutils.cpp index 61a5d376d..d7d5781c4 100644 --- a/src/common/backend/utils/adt/ruleutils.cpp +++ b/src/common/backend/utils/adt/ruleutils.cpp @@ -232,9 +232,9 @@ static char* pg_get_triggerdef_worker(Oid trigid, bool pretty); static void decompile_column_index_array(Datum column_index_array, Oid relId, StringInfo buf); static char* pg_get_ruledef_worker(Oid ruleoid, int prettyFlags); static char *pg_get_indexdef_worker(Oid indexrelid, int colno, const Oid *excludeOps, bool attrsOnly, bool showTblSpc, - int prettyFlags, bool dumpSchemaOnly = false, bool showSubpartitionLocal = true); + int prettyFlags, bool dumpSchemaOnly = false, bool showPartitionLocal = true, bool showSubpartitionLocal = true); static void pg_get_indexdef_partitions(Oid indexrelid, Form_pg_index idxrec, bool showTblSpc, StringInfoData *buf, - bool dumpSchemaOnly, bool showSubpartitionLocal); + bool dumpSchemaOnly, bool showPartitionLocal, bool showSubpartitionLocal); static char* pg_get_constraintdef_worker(Oid constraintId, bool fullCommand, int prettyFlags); static text* pg_get_expr_worker(text* expr, Oid relid, const char* relname, int prettyFlags); static int print_function_arguments(StringInfo buf, HeapTuple proctup, bool print_table_args, bool print_defaults); @@ -2960,7 +2960,7 @@ Datum pg_get_indexdef_for_dump(PG_FUNCTION_ARGS) bool dumpSchemaOnly = PG_GETARG_BOOL(1); PG_RETURN_TEXT_P(string_to_text(pg_get_indexdef_worker(indexrelid, 0, NULL, false, true, 0, dumpSchemaOnly, - false))); + true, false))); } Datum pg_get_indexdef_ext(PG_FUNCTION_ARGS) @@ -2972,7 +2972,7 @@ Datum pg_get_indexdef_ext(PG_FUNCTION_ARGS) prettyFlags = pretty ? (PRETTYFLAG_PAREN | PRETTYFLAG_INDENT) : 0; PG_RETURN_TEXT_P(string_to_text(pg_get_indexdef_worker(indexrelid, colno, NULL, colno != 0, true, prettyFlags, - false, false))); + false, false, false))); } /** @@ -3051,7 +3051,7 @@ static void GetIndexdefForIntervalPartTabDumpSchemaOnly(Oid indexrelid, RangePar } static void pg_get_indexdef_partitions(Oid indexrelid, Form_pg_index idxrec, bool showTblSpc, StringInfoData *buf, - bool dumpSchemaOnly, bool showSubpartitionLocal) + bool dumpSchemaOnly, bool showPartitionLocal, bool showSubpartitionLocal) { Oid relid = idxrec->indrelid; /* @@ -3067,12 +3067,12 @@ static void pg_get_indexdef_partitions(Oid indexrelid, Form_pg_index idxrec, boo appendStringInfo(buf, " LOCAL"); /* - * The LOCAL index information of the subpartition table is more. + * The LOCAL index information of the partition and subpartition table is more. * And the meta-statements (e.g. \d \d+ \dS) are used more. - * Therefore, when the meta-statement is called, the subpartition LOCAL index information is not displayed. + * Therefore, when the meta-statement is called, the LOCAL index information is not displayed. */ bool isSub = RelationIsSubPartitioned(rel); - if (isSub && !showSubpartitionLocal) { + if ((!isSub && !showPartitionLocal) || (isSub && !showSubpartitionLocal)) { heap_close(rel, NoLock); return; } @@ -3112,7 +3112,7 @@ static void pg_get_indexdef_partitions(Oid indexrelid, Form_pg_index idxrec, boo * NULL then it points to an array of exclusion operator OIDs. */ static char *pg_get_indexdef_worker(Oid indexrelid, int colno, const Oid *excludeOps, bool attrsOnly, bool showTblSpc, - int prettyFlags, bool dumpSchemaOnly, bool showSubpartitionLocal) + int prettyFlags, bool dumpSchemaOnly, bool showPartitionLocal, bool showSubpartitionLocal) { /* might want a separate isConstraint parameter later */ bool isConstraint = (excludeOps != NULL); @@ -3316,7 +3316,8 @@ static char *pg_get_indexdef_worker(Oid indexrelid, int colno, const Oid *exclud if (idxrelrec->parttype == PARTTYPE_PARTITIONED_RELATION && idxrelrec->relkind != RELKIND_GLOBAL_INDEX) { - pg_get_indexdef_partitions(indexrelid, idxrec, showTblSpc, &buf, dumpSchemaOnly, showSubpartitionLocal); + pg_get_indexdef_partitions(indexrelid, idxrec, showTblSpc, &buf, dumpSchemaOnly, + showPartitionLocal, showSubpartitionLocal); } /* diff --git a/src/common/backend/utils/adt/varlena.cpp b/src/common/backend/utils/adt/varlena.cpp index 2ba2ce1ae..64378b723 100644 --- a/src/common/backend/utils/adt/varlena.cpp +++ b/src/common/backend/utils/adt/varlena.cpp @@ -746,7 +746,7 @@ static Datum text_length_huge(Datum str) */ Datum textlen(PG_FUNCTION_ARGS) { - Datum str = PG_GETARG_DATUM(0); + Datum str = fetch_real_lob_if_need(PG_GETARG_DATUM(0)); if (VARATT_IS_HUGE_TOAST_POINTER((varlena *)DatumGetTextPP(str))) { return text_length_huge(str); @@ -1899,7 +1899,7 @@ Datum textne(PG_FUNCTION_ARGS) { Datum arg1 = PG_GETARG_DATUM(0); Datum arg2 = PG_GETARG_DATUM(1); - if (VARATT_IS_HUGE_TOAST_POINTER(DatumGetPointer(arg1)) || VARATT_IS_HUGE_TOAST_POINTER(DatumGetPointer(arg2))) { + if (VARATT_IS_HUGE_TOAST_POINTER(DatumGetPointer(arg1)) && VARATT_IS_HUGE_TOAST_POINTER(DatumGetPointer(arg2))) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("textne could not support more than 1GB clob/blob data"))); } @@ -2063,7 +2063,7 @@ Datum text_ge(PG_FUNCTION_ARGS) { text* arg1 = PG_GETARG_TEXT_PP(0); text* arg2 = PG_GETARG_TEXT_PP(1); - if (VARATT_IS_HUGE_TOAST_POINTER((varlena *)arg1) || VARATT_IS_HUGE_TOAST_POINTER((varlena *)arg2)) { + if (VARATT_IS_HUGE_TOAST_POINTER((varlena *)arg1) && VARATT_IS_HUGE_TOAST_POINTER((varlena *)arg2)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("text_ge could not support more than 1GB clob/blob data"))); } diff --git a/src/common/backend/utils/cache/knl_localpartdefcache.cpp b/src/common/backend/utils/cache/knl_localpartdefcache.cpp index a54bd21a3..a12167b92 100644 --- a/src/common/backend/utils/cache/knl_localpartdefcache.cpp +++ b/src/common/backend/utils/cache/knl_localpartdefcache.cpp @@ -211,7 +211,7 @@ void LocalPartDefCache::Init() } m_global_partdefcache = t_thrd.lsc_cxt.lsc->GetGlobalPartDefCache(); m_db_id = t_thrd.lsc_cxt.lsc->my_database_id; - part_cache_need_eoxact_work = false; + PartCacheNeedEOXActWork = false; m_is_inited = true; } @@ -298,7 +298,7 @@ void LocalPartDefCache::AtEOXact_PartitionCache(bool isCommit) * transaction, even though we could clear it at subtransaction end in * some cases. */ - if (!part_cache_need_eoxact_work + if (!GetPartCacheNeedEOXActWork() #ifdef USE_ASSERT_CHECKING && !assert_enabled #endif @@ -356,7 +356,7 @@ void LocalPartDefCache::AtEOXact_PartitionCache(bool isCommit) } /* Once done with the transaction, we can reset need_eoxact_work */ - part_cache_need_eoxact_work = false; + SetPartCacheNeedEOXActWork(false); } void LocalPartDefCache::AtEOSubXact_PartitionCache(bool isCommit, SubTransactionId mySubid, @@ -366,7 +366,7 @@ void LocalPartDefCache::AtEOSubXact_PartitionCache(bool isCommit, SubTransaction * Skip the relcache scan if nothing to do --- see notes for * AtEOXact_PartitionCache. */ - if (!part_cache_need_eoxact_work) + if (!GetPartCacheNeedEOXActWork()) return; Dlelem *bucket_elt; diff --git a/src/common/backend/utils/cache/knl_localsysdbcache.cpp b/src/common/backend/utils/cache/knl_localsysdbcache.cpp index aa58fdc8a..71c956b4d 100644 --- a/src/common/backend/utils/cache/knl_localsysdbcache.cpp +++ b/src/common/backend/utils/cache/knl_localsysdbcache.cpp @@ -432,7 +432,6 @@ Size *GetSizeVfdCachePtr() } } - int GetVfdNfile() { if (EnableLocalSysCache()) { @@ -524,19 +523,49 @@ bool LocalSysDBCache::LocalSysDBCacheNeedClearMyDB(Oid db_id, const char *db_nam strcmp(t_thrd.proc_cxt.MyProgName, "BootStrap") == 0 )); } + /* it is a weird design that we need access mydatabaseid before initsession. + * but for GSC mode, we do need aquire lock when cache hit and there are invalid msgs + * with session uninited and so u_sess->proc_cxt.MyDatabaseId is InvalidOid. + * 1 the publication feature will send all rels' invalmsgs even no refered ddl. + * 2 relations may have refcount leak, so we must rebuild them if cache hit. + * when we rebuild a relation, the session may be not uninited, + * and so u_sess->proc_cxt.MyDatabaseId is InvalidOid, + * so we use t_thrd.lsc_cxt.lsc->my_database_id on GSC mode */ + Assert(CheckMyDatabaseMatch()); + Assert(m_global_db != NULL); + /* if u_sess->proc_cxt.MyDatabaseId is InvalidOid, the session's status is uninit + * we will call SetDatabase to rewrite it. + * but beofre SetDatabase, we need the dbid to Accept Invalid msg */ + bool lock_db_advance = u_sess->proc_cxt.MyDatabaseId == InvalidOid && IS_THREAD_POOL_WORKER; /* cache hit, when initsession, we lock db to avoid alter db */ - Oid old_db_id = t_thrd.proc->databaseId; - if (u_sess->proc_cxt.MyDatabaseId == InvalidOid && IS_THREAD_POOL_WORKER) { - LockSharedObject(DatabaseRelationId, my_database_id, 0, RowExclusiveLock); + if (lock_db_advance) { + Assert(u_sess->proc_cxt.MyDatabaseTableSpace == InvalidOid); + Assert(u_sess->proc_cxt.DatabasePath == NULL); + Assert(t_thrd.proc->databaseId == InvalidOid); + + u_sess->proc_cxt.MyDatabaseId = my_database_id; + u_sess->proc_cxt.MyDatabaseTableSpace = my_database_tablespace; + /* use refer not copy, it will be rewritten when initsession */ + u_sess->proc_cxt.DatabasePath = my_database_path; + + /* we dont want to accept inval msg here, so use LockSharedObjectForSession to avoid it. */ + LockSharedObjectForSession(DatabaseRelationId, my_database_id, 0, RowExclusiveLock); t_thrd.proc->databaseId = my_database_id; - UnlockSharedObject(DatabaseRelationId, my_database_id, 0, RowExclusiveLock); - } - Assert(m_global_db != NULL); - if (m_global_db->m_isDead) { - t_thrd.proc->databaseId = old_db_id; + UnlockSharedObjectForSession(DatabaseRelationId, my_database_id, 0, RowExclusiveLock); + + /* when we acquired the dblock, alter db transaction happened, and we should clear cache of mydb. */ + if (m_global_db->m_isDead) { + t_thrd.proc->databaseId = InvalidOid; + u_sess->proc_cxt.MyDatabaseId = InvalidOid; + u_sess->proc_cxt.MyDatabaseTableSpace = InvalidOid; + u_sess->proc_cxt.DatabasePath = NULL; + return true; + } + } else if (m_global_db->m_isDead) { return true; } + return false; } @@ -564,7 +593,6 @@ void LocalSysDBCache::LocalSysDBCacheClearMyDB(Oid db_id, const char *db_name) MemoryContextResetAndDeleteChildren(lsc_mydb_memcxt); is_inited = false; - other_space = ((AllocSet)lsc_top_memcxt)->totalSpace + ((AllocSet)lsc_share_memcxt)->totalSpace; rel_index_rule_space = 0; } @@ -593,29 +621,26 @@ void LocalSysDBCache::LocalSysDBCacheReSet() UnRegisterRelCacheCallBack(&inval_cxt, RelfilenodeMapInvalidateCallback); UnRegisterSysCacheCallBack(&inval_cxt, TABLESPACEOID, InvalidateTableSpaceCacheCallback); - MemoryContextResetAndDeleteChildren(lsc_mydb_memcxt); - MemoryContextResetAndDeleteChildren(lsc_share_memcxt); + MemoryContextDelete(lsc_mydb_memcxt); + MemoryContextDelete(lsc_share_memcxt); + lsc_share_memcxt = + AllocSetContextCreate(lsc_top_memcxt, "LocalSysCacheShareMemoryContext", ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE, STANDARD_CONTEXT); + + lsc_mydb_memcxt = + AllocSetContextCreate(lsc_top_memcxt, "LocalSysCacheMyDBMemoryContext", ALLOCSET_DEFAULT_MINSIZE, + ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE, STANDARD_CONTEXT); MemoryContext old = MemoryContextSwitchTo(lsc_share_memcxt); knl_u_relmap_init(&relmap_cxt); MemoryContextSwitchTo(old); - other_space = ((AllocSet)lsc_top_memcxt)->totalSpace + ((AllocSet)lsc_share_memcxt)->totalSpace; rel_index_rule_space = 0; is_inited = false; is_closed = false; is_lsc_catbucket_created = false; } -static bool LSCMemroyOverflow(uint64 total_space) -{ - const uint32 kb_bit_double = 10; - if (unlikely(total_space < ((uint64)g_instance.attr.attr_memory.local_syscache_threshold << kb_bit_double))) { - return false; - } - return true; -} - bool LocalSysDBCache::LocalSysDBCacheNeedReBuild() { /* we have recovered from startup, redo may dont tell us inval msgs, so discard all lsc */ @@ -626,15 +651,21 @@ bool LocalSysDBCache::LocalSysDBCacheNeedReBuild() return true; } - /* it seems only fmgr_info_cxt has no resowner to avoid mem leak. - * we assum 1kb memory leaked once */ + /* we assum 1kb memory leaked once */ if (unlikely(abort_count > (uint64)g_instance.attr.attr_memory.local_syscache_threshold)) { return true; } - uint64 total_space = other_space + AllocSetContextUsedSpace((AllocSet)lsc_mydb_memcxt) + - AllocSetContextUsedSpace((AllocSet)u_sess->cache_mem_cxt) + rel_index_rule_space; - return LSCMemroyOverflow(total_space); + uint64 total_space = + ((AllocSet)lsc_top_memcxt)->totalSpace + + ((AllocSet)lsc_share_memcxt)->totalSpace + + ((AllocSet)lsc_mydb_memcxt)->totalSpace + + ((AllocSet)u_sess->cache_mem_cxt)->totalSpace + + rel_index_rule_space; + + uint64 memory_upper_limit = ((uint64)g_instance.attr.attr_memory.local_syscache_threshold) << 10; + + return total_space * (1 - MAX_LSC_FREESIZE_RATIO) > memory_upper_limit; } /* rebuild cache on memcxt of mydb or share, call it only before initsyscache */ @@ -664,8 +695,23 @@ void LocalSysDBCache::LocalSysDBCacheReBuild() bool LocalSysDBCache::LocalSysDBCacheNeedSwapOut() { - uint64 total_space = other_space + rel_index_rule_space + ((AllocSet)lsc_mydb_memcxt)->totalSpace; - return LSCMemroyOverflow(total_space); + uint64 used_space = + AllocSetContextUsedSpace((AllocSet)lsc_top_memcxt) + + AllocSetContextUsedSpace((AllocSet)lsc_share_memcxt) + + AllocSetContextUsedSpace((AllocSet)lsc_mydb_memcxt) + + AllocSetContextUsedSpace((AllocSet)u_sess->cache_mem_cxt) + + rel_index_rule_space; + uint64 memory_upper_limit = + (((uint64)g_instance.attr.attr_memory.local_syscache_threshold) << 10) * cur_swapout_ratio; + bool need_swapout = used_space > memory_upper_limit; + + /* swapout until memory used space is from MAX_LSC_SWAPOUT_RATIO=90% to MIN_LSC_SWAPOUT_RATIO=70% */ + if (unlikely(need_swapout && cur_swapout_ratio == MAX_LSC_SWAPOUT_RATIO)) { + cur_swapout_ratio = MIN_LSC_SWAPOUT_RATIO; + } else if (unlikely(!need_swapout && cur_swapout_ratio == MIN_LSC_SWAPOUT_RATIO)) { + cur_swapout_ratio = MAX_LSC_SWAPOUT_RATIO; + } + return need_swapout; } void LocalSysDBCache::CloseLocalSysDBCache() @@ -688,7 +734,6 @@ void LocalSysDBCache::ClearSysCacheIfNecessary(Oid db_id, const char *db_name) if (unlikely(!is_inited)) { return; } - other_space = ((AllocSet)lsc_top_memcxt)->totalSpace + ((AllocSet)lsc_share_memcxt)->totalSpace; /* rebuild if memory gt double of threshold */ if (unlikely(LocalSysDBCacheNeedReBuild())) { recovery_finished = g_instance.global_sysdbcache.recovery_finished; @@ -740,7 +785,6 @@ void LocalSysDBCache::CreateDBObject() knl_u_relmap_init(&relmap_cxt); MemoryContextSwitchTo(old); m_shared_global_db = g_instance.global_sysdbcache.GetSharedGSCEntry(); - other_space = ((AllocSet)lsc_top_memcxt)->totalSpace + ((AllocSet)lsc_share_memcxt)->totalSpace; rel_index_rule_space = 0; is_lsc_catbucket_created = false; } @@ -753,9 +797,9 @@ void LocalSysDBCache::CreateCatBucket() MemoryContext old = MemoryContextSwitchTo(lsc_share_memcxt); systabcache.CreateCatBuckets(); MemoryContextSwitchTo(old); - other_space = ((AllocSet)lsc_top_memcxt)->totalSpace + ((AllocSet)lsc_share_memcxt)->totalSpace; rel_index_rule_space = 0; is_lsc_catbucket_created = true; + cur_swapout_ratio = MAX_LSC_SWAPOUT_RATIO; } void LocalSysDBCache::SetDatabaseName(const char *db_name) @@ -924,6 +968,8 @@ LocalSysDBCache::LocalSysDBCache() got_pool_reload = false; m_shared_global_db = NULL; + cur_swapout_ratio = MAX_LSC_SWAPOUT_RATIO; + is_lsc_catbucket_created = false; is_closed = false; is_inited = false; diff --git a/src/common/backend/utils/cache/knl_localsystupcache.cpp b/src/common/backend/utils/cache/knl_localsystupcache.cpp index ddf196099..132aefc30 100644 --- a/src/common/backend/utils/cache/knl_localsystupcache.cpp +++ b/src/common/backend/utils/cache/knl_localsystupcache.cpp @@ -356,7 +356,7 @@ void LocalSysTupCache::InitPhase2Impl() { Assert(m_is_inited); Assert(!m_is_inited_phase2); - Assert(m_db_id == InvalidOid); + /* CacheIdGetGlobalSysTupCache maybe fail when memory fault */ Assert(m_global_systupcache == NULL); /* for now we even dont know which db to connect */ if (m_relinfo.cc_relisshared) { @@ -509,7 +509,7 @@ LocalCatCTup *LocalSysTupCache::SearchTupleInternal(int nkeys, Datum v1, Datum v /* if not found, search from global cache */ if (unlikely(!found)) { ct = SearchTupleFromGlobal(arguments, hash_value, hash_index, level); - if (ct == NULL) { + if (unlikely(ct == NULL)) { return NULL; } } @@ -529,8 +529,10 @@ LocalCatCTup *LocalSysTupCache::SearchTupleInternal(int nkeys, Datum v1, Datum v cc_neg_hits++; ct = NULL; } + if (unlikely(!found)) { + RemoveTailTupleElements(hash_index); + } - RemoveTailTupleElements(hash_index); return ct; } @@ -665,15 +667,16 @@ LocalCatCList *LocalSysTupCache::SearchListInternal(int nkeys, Datum v1, Datum v CACHE2_elog(DEBUG2, "SearchLocalCatCacheList(%s): found list", m_relinfo.cc_relname); cc_lhits++; found = true; + ResourceOwnerRememberLocalCatCList(LOCAL_SYSDB_RESOWNER, cl); break; } if (unlikely(!found)) { cl = SearchListFromGlobal(nkeys, arguments, hash_value, level); + ResourceOwnerRememberLocalCatCList(LOCAL_SYSDB_RESOWNER, cl); + RemoveTailListElements(); } - ResourceOwnerRememberLocalCatCList(LOCAL_SYSDB_RESOWNER, cl); - RemoveTailListElements(); return cl; } @@ -817,12 +820,14 @@ LocalCatCTup *LocalSysTupCache::SearchLocalCatCTupleForProcAllArgs( */ if (likely(ct->global_ct != NULL)) { CACHE3_elog(DEBUG2, "SearchLocalCatCache(%s): found in bucket %d", m_relinfo.cc_relname, hash_index); + ResourceOwnerEnlargeLocalCatCTup(LOCAL_SYSDB_RESOWNER); ct->refcount++; cc_hits++; ResourceOwnerRememberLocalCatCTup(LOCAL_SYSDB_RESOWNER, ct); } - - RemoveTailTupleElements(hash_index); + if (unlikely(!found)) { + RemoveTailTupleElements(hash_index); + } pfree_ext(argModes); return ct; diff --git a/src/common/backend/utils/cache/knl_localtabdefcache.cpp b/src/common/backend/utils/cache/knl_localtabdefcache.cpp index 46d3f77e0..6b989cde8 100644 --- a/src/common/backend/utils/cache/knl_localtabdefcache.cpp +++ b/src/common/backend/utils/cache/knl_localtabdefcache.cpp @@ -367,7 +367,7 @@ void LocalTabDefCache::Init() relcacheInvalsReceived = 0; initFileRelationIds = NIL; - need_eoxact_work = false; + RelCacheNeedEOXActWork = false; g_bucketmap_cache = NIL; max_bucket_map_size = BUCKET_MAP_SIZE; @@ -816,8 +816,9 @@ void LocalTabDefCache::RememberToFreeTupleDescAtEOX(TupleDesc td) { if (EOXactTupleDescArray == NULL) { MemoryContext oldcxt = MemoryContextSwitchTo(LocalMyDBCacheMemCxt()); - EOXactTupleDescArrayLen = 16; - EOXactTupleDescArray = (TupleDesc *)palloc(EOXactTupleDescArrayLen * sizeof(TupleDesc)); + const int default_len = 16; + EOXactTupleDescArray = (TupleDesc *)palloc(default_len * sizeof(TupleDesc)); + EOXactTupleDescArrayLen = default_len; NextEOXactTupleDescNum = 0; MemoryContextSwitchTo(oldcxt); } else if (NextEOXactTupleDescNum >= EOXactTupleDescArrayLen) { @@ -873,7 +874,7 @@ void LocalTabDefCache::AtEOXact_RelationCache(bool isCommit) * transaction, even though we could clear it at subtransaction end in * some cases. */ - if (!LocalRelCacheNeedEOXactWork() + if (!GetRelCacheNeedEOXActWork() #ifdef USE_ASSERT_CHECKING && !assert_enabled #endif @@ -962,7 +963,7 @@ void LocalTabDefCache::AtEOXact_RelationCache(bool isCommit) } } /* Once done with the transaction, we can reset u_sess->relcache_cxt.need_eoxact_work */ - SetLocalRelCacheNeedEOXactWork(false); + SetRelCacheNeedEOXActWork(false); } /* @@ -979,7 +980,7 @@ void LocalTabDefCache::AtEOSubXact_RelationCache(bool isCommit, SubTransactionId * Skip the relcache scan if nothing to do --- see notes for * AtEOXact_RelationCache. */ - if (!LocalRelCacheNeedEOXactWork()) + if (!GetRelCacheNeedEOXActWork()) return; Dlelem *bucket_elt; @@ -1117,7 +1118,7 @@ void LocalTabDefCache::ResetInitFlag() relcacheInvalsReceived = 0; initFileRelationIds = NIL; - need_eoxact_work = false; + RelCacheNeedEOXActWork = false; g_bucketmap_cache = NIL; max_bucket_map_size = 0; diff --git a/src/common/backend/utils/cache/partcache.cpp b/src/common/backend/utils/cache/partcache.cpp index 9b025ff8e..1015826fe 100644 --- a/src/common/backend/utils/cache/partcache.cpp +++ b/src/common/backend/utils/cache/partcache.cpp @@ -473,7 +473,7 @@ Partition PartitionBuildLocalPartition(const char *relname, Oid partid, Oid part part->pd_newRelfilenodeSubid = InvalidSubTransactionId; /* must flag that we have rels created in this transaction */ - SetPartCacheNeedEoxactWork(true); + SetPartCacheNeedEOXActWork(true); /* * initialize partition tuple form (caller may add/override data later) @@ -986,7 +986,7 @@ void AtEOXact_PartitionCache(bool isCommit) * transaction, even though we could clear it at subtransaction end in * some cases. */ - if (!u_sess->cache_cxt.part_cache_need_eoxact_work + if (!GetPartCacheNeedEOXActWork() #ifdef USE_ASSERT_CHECKING && !assert_enabled #endif @@ -1040,7 +1040,7 @@ void AtEOXact_PartitionCache(bool isCommit) } /* Once done with the transaction, we can reset need_eoxact_work */ - u_sess->cache_cxt.part_cache_need_eoxact_work = false; + SetPartCacheNeedEOXActWork(false); } /* @@ -1063,7 +1063,7 @@ void AtEOSubXact_PartitionCache(bool isCommit, SubTransactionId mySubid, SubTran * Skip the relcache scan if nothing to do --- see notes for * AtEOXact_PartitionCache. */ - if (!u_sess->cache_cxt.part_cache_need_eoxact_work) + if (!GetPartCacheNeedEOXActWork()) return; hash_seq_init(&status, u_sess->cache_cxt.PartitionIdCache); @@ -1705,7 +1705,7 @@ void PartitionSetNewRelfilenode(Relation parent, Partition part, TransactionId f part->pd_newRelfilenodeSubid = GetCurrentSubTransactionId(); /* ... and now we have eoxact cleanup work to do */ - SetPartCacheNeedEoxactWork(true); + SetPartCacheNeedEOXActWork(true); } static void PartitionParseRelOptions(Partition partition, HeapTuple tuple) diff --git a/src/common/backend/utils/cache/relcache.cpp b/src/common/backend/utils/cache/relcache.cpp index 822431120..f44b182b0 100644 --- a/src/common/backend/utils/cache/relcache.cpp +++ b/src/common/backend/utils/cache/relcache.cpp @@ -2456,7 +2456,7 @@ void RelationInitPhysicalAddr(Relation relation) * tables and on user tables declared as additional catalog * tables. */ - if (HistoricSnapshotActive() && RelationIsAccessibleInLogicalDecoding(relation) && IsTransactionState()) { + if (HistoricSnapshotActive() && RelationIsAccessibleInLogicalDecoding(relation)) { HeapTuple phys_tuple; Form_pg_class physrel; @@ -4161,7 +4161,7 @@ void AtEOXact_RelationCache(bool isCommit) * transaction, even though we could clear it at subtransaction end in * some cases. */ - if (!u_sess->relcache_cxt.need_eoxact_work + if (!GetRelCacheNeedEOXActWork() #ifdef USE_ASSERT_CHECKING && !assert_enabled #endif @@ -4253,7 +4253,7 @@ void AtEOXact_RelationCache(bool isCommit) } /* Once done with the transaction, we can reset u_sess->relcache_cxt.need_eoxact_work */ - u_sess->relcache_cxt.need_eoxact_work = false; + SetRelCacheNeedEOXActWork(false); } /* @@ -4276,7 +4276,7 @@ void AtEOSubXact_RelationCache(bool isCommit, SubTransactionId mySubid, SubTrans * Skip the relcache scan if nothing to do --- see notes for * AtEOXact_RelationCache. */ - if (!u_sess->relcache_cxt.need_eoxact_work) + if (!GetRelCacheNeedEOXActWork()) return; hash_seq_init(&status, u_sess->relcache_cxt.RelationIdCache); @@ -4416,7 +4416,7 @@ Relation RelationBuildLocalRelation(const char* relname, Oid relnamespace, Tuple rel->rd_newRelfilenodeSubid = InvalidSubTransactionId; /* must flag that we have rels created in this transaction */ - SetLocalRelCacheNeedEOXactWork(true); + SetRelCacheNeedEOXActWork(true); /* * create a new tuple descriptor from the one passed in. We do this @@ -4784,7 +4784,7 @@ void RelationSetNewRelfilenode(Relation relation, TransactionId freezeXid, Multi */ relation->rd_newRelfilenodeSubid = GetCurrentSubTransactionId(); /* ... and now we have eoxact cleanup work to do */ - SetLocalRelCacheNeedEOXactWork(true); + SetRelCacheNeedEOXActWork(true); } RelFileNodeBackend CreateNewRelfilenode(Relation relation, TransactionId freezeXid) @@ -6064,7 +6064,7 @@ void RelationSetIndexList(Relation relation, List* indexIds, Oid oidIndex) relation->rd_pkindex = InvalidOid; relation->rd_indexvalid = 2; /* mark list as forced */ /* must flag that we have a forced index list */ - SetLocalRelCacheNeedEOXactWork(true); + SetRelCacheNeedEOXActWork(true); } /* diff --git a/src/common/backend/utils/error/elog.cpp b/src/common/backend/utils/error/elog.cpp index f517e3500..cea46c2c1 100644 --- a/src/common/backend/utils/error/elog.cpp +++ b/src/common/backend/utils/error/elog.cpp @@ -3945,6 +3945,10 @@ void getElevelAndSqlstate(int* eLevel, int* sqlState) *sqlState = t_thrd.log_cxt.errordata[t_thrd.log_cxt.errordata_stack_depth].sqlerrcode; } +/* + * When the SQL statement is truncated, this function cannot perform normal password masking. + * maskPassword will return null if the statement does not need to be masked or any error occurs. + */ char* maskPassword(const char* query_string) { char* mask_string = NULL; @@ -4232,6 +4236,19 @@ static void inline ClearYylval(const core_YYSTYPE *yylval) securec_check(rc, "\0", "\0"); } +static int get_reallen_of_credential(char *param) +{ + int len = 0; + for (int i = 0; param[i] != '\0'; i++) { + if (param[i] == '\'') { + len += 2; + } else { + len++; + } + } + return len; +} + /* * Mask the password in statment CREATE ROLE, CREATE USER, ALTER ROLE, ALTER USER, CREATE GROUP * SET ROLE, CREATE DATABASE LINK, and some function @@ -4248,7 +4265,9 @@ static char* mask_Password_internal(const char* query_string) bool isPassword = false; char* mask_string = NULL; /* the function list need mask */ - const char* funcs[] = {"dblink_connect", "create_credential", "pg_create_physical_replication_slot_extern"}; + const char* funcs[] = {"dblink_connect", "create_credential"}; + bool is_create_credential = false; + bool is_create_credential_passwd = false; int funcNum = sizeof(funcs) / sizeof(funcs[0]); int position[16] = {0}; int length[16] = {0}; @@ -4260,7 +4279,8 @@ static char* mask_Password_internal(const char* query_string) YYLTYPE conninfoStartPos = 0; /* connection start postion for CreateSubscriptionStmt */ /* the functions need to mask all contents */ - const char* funCrypt[] = {"gs_encrypt_aes128", "gs_decrypt_aes128", "gs_encrypt", "gs_decrypt"}; + const char* funCrypt[] = {"gs_encrypt_aes128", "gs_decrypt_aes128", "gs_encrypt", "gs_decrypt", + "pg_create_physical_replication_slot_extern"}; int funCryptNum = sizeof(funCrypt) / sizeof(funCrypt[0]); bool isCryptFunc = false; @@ -4365,10 +4385,11 @@ static char* mask_Password_internal(const char* query_string) if (subQueryLen < childStmtLen) { /* Need more space, enlarge length is (childStmtLen - subQueryLen) */ maskStringLen += (childStmtLen - subQueryLen) + 1; - char* maskStrNew = (char*)selfpalloc0(maskStringLen); + char* maskStrNew = (char*)MemoryContextAllocZero( + SESS_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY), maskStringLen); rc = memcpy_s(maskStrNew, maskStringLen, mask_string, strlen(mask_string)); securec_check(rc, "\0", "\0"); - selfpfree(mask_string); + pfree_ext(mask_string); mask_string = maskStrNew; } @@ -4409,7 +4430,18 @@ static char* mask_Password_internal(const char* query_string) /* Calcute the difference between origin password length and mask password length */ position[idx] -= truncateLen; - length[idx] = strlen(yylval.str); + if (!is_create_credential) { + length[idx] = strlen(yylval.str); + } else if (isPassword) { + is_create_credential_passwd = true; + length[idx] = strlen(yylval.str); + } else { + if (idx == 2 && !is_create_credential_passwd) { + length[idx] = get_reallen_of_credential(yylval.str); + } else { + length[idx] = 0; + } + } ++idx; /* record the conninfo start pos, we will use it to calculate the actual length of conninfo */ @@ -4463,10 +4495,11 @@ static char* mask_Password_internal(const char* query_string) if (length[i] < maskLen) { /* need more space. */ int plen = strlen(mask_string) + maskLen - length[i] + 1; - char* maskStrNew = (char*)selfpalloc0(plen); + char* maskStrNew = (char*)MemoryContextAllocZero( + SESS_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_SECURITY), plen); rc = memcpy_s(maskStrNew, plen, mask_string, strlen(mask_string)); securec_check(rc, "\0", "\0"); - selfpfree(mask_string); + pfree_ext(mask_string); mask_string = maskStrNew; } @@ -4597,6 +4630,10 @@ static char* mask_Password_internal(const char* query_string) /* first, check funcs[] */ for (i = 0; i < funcNum; ++i) { if (pg_strcasecmp(yylval.str, funcs[i]) == 0) { + is_create_credential = false; + if (pg_strcasecmp(yylval.str, "create_credential") == 0) { + is_create_credential = true; + } curStmtType = 8; break; } @@ -4962,6 +4999,7 @@ static char* mask_Password_internal(const char* query_string) } return mask_string; + } static void eraseSingleQuotes(char* query_string) diff --git a/src/common/backend/utils/init/globals.cpp b/src/common/backend/utils/init/globals.cpp index 558c4b4ca..699992831 100644 --- a/src/common/backend/utils/init/globals.cpp +++ b/src/common/backend/utils/init/globals.cpp @@ -59,12 +59,13 @@ bool open_join_children = true; bool will_shutdown = false; /* hard-wired binary version number */ -const uint32 GRAND_VERSION_NUM = 92604; +const uint32 GRAND_VERSION_NUM = 92605; const uint32 PREDPUSH_SAME_LEVEL_VERSION_NUM = 92522; const uint32 UPSERT_WHERE_VERSION_NUM = 92514; const uint32 FUNC_PARAM_COL_VERSION_NUM = 92500; const uint32 SUBPARTITION_VERSION_NUM = 92436; +const uint32 PBESINGLEPARTITION_VERSION_NUM = 92523; const uint32 DEFAULT_MAT_CTE_NUM = 92429; const uint32 MATERIALIZED_CTE_NUM = 92424; const uint32 HINT_ENHANCEMENT_VERSION_NUM = 92359; @@ -113,6 +114,8 @@ const uint32 SUPPORT_HASH_XLOG_VERSION_NUM = 92603; /* This variable indicates wheather the instance is in progress of upgrade as a whole */ uint32 volatile WorkingGrandVersionNum = GRAND_VERSION_NUM; +const uint32 INVALID_INVISIBLE_TUPLE_VERSION = 92605; + const uint32 ENHANCED_TUPLE_LOCK_VERSION_NUM = 92583; const uint32 TWOPHASE_FILE_VERSION = 92414; @@ -129,6 +132,8 @@ bool InplaceUpgradePrecommit = false; const uint32 DISASTER_READ_VERSION_NUM = 92592; +const uint32 PITR_INIT_VERSION_NUM = 92599; + #ifdef PGXC bool useLocalXid = false; #endif diff --git a/src/common/backend/utils/init/postinit.cpp b/src/common/backend/utils/init/postinit.cpp index f448b775a..76a3563dc 100644 --- a/src/common/backend/utils/init/postinit.cpp +++ b/src/common/backend/utils/init/postinit.cpp @@ -66,6 +66,7 @@ #include "storage/ipc.h" #include "storage/smgr/knl_usync.h" #include "storage/lmgr.h" +#include "storage/predicate.h" #include "storage/proc.h" #include "storage/procarray.h" #include "storage/procsignal.h" @@ -2031,7 +2032,6 @@ void PostgresInitializer::InitThread() on_shmem_exit(ShutdownXLOG, 0); } } - void PostgresInitializer::InitLoadLocalSysCache(Oid db_oid, const char *db_name) { if(!EnableLocalSysCache()) { @@ -2068,6 +2068,11 @@ void PostgresInitializer::InitLoadLocalSysCache(Oid db_oid, const char *db_name) ResourceOwnerRelease(t_thrd.lsc_cxt.lsc->local_sysdb_resowner, RESOURCE_RELEASE_LOCKS, false, true); ResourceOwnerRelease(t_thrd.lsc_cxt.lsc->local_sysdb_resowner, RESOURCE_RELEASE_AFTER_LOCKS, false, true); + /* we are not in transaction, so resowner cannot help us release proclocks and predicatelocks. + * we do nothing above except init and load syscache, so no undowork need. ProcReleaseLocks always need + * */ + ProcReleaseLocks(false); + ReleasePredicateLocks(false); /* lwlocks arre released at sigsetjmp */ /* recovery CurrentResourceOwner */ @@ -2590,8 +2595,9 @@ void PostgresInitializer::SetDatabasePath() ValidatePgVersion(m_fullpath); - /* This should happen only once per process */ - Assert(!u_sess->proc_cxt.DatabasePath); + /* This should happen only once per process, for gsc, it may equal the pointer belongs to lsc */ + Assert(!u_sess->proc_cxt.DatabasePath || + (EnableLocalSysCache() && u_sess->proc_cxt.DatabasePath == t_thrd.lsc_cxt.lsc->my_database_path)); u_sess->proc_cxt.DatabasePath = MemoryContextStrdup( SESS_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_EXECUTOR), m_fullpath); if (EnableLocalSysCache()) { diff --git a/src/common/backend/utils/mb/mbutils.cpp b/src/common/backend/utils/mb/mbutils.cpp index 981331330..c204fd0d1 100644 --- a/src/common/backend/utils/mb/mbutils.cpp +++ b/src/common/backend/utils/mb/mbutils.cpp @@ -829,10 +829,6 @@ int pg_mbstrlen_with_len_toast(const char* mbstr, int* limit) { int len = 0; - /* optimization for single byte encoding */ - if (pg_database_encoding_max_length() == 1) { - return *limit; - } while (*limit > 0 && *mbstr) { int l = pg_mblen(mbstr); diff --git a/src/common/backend/utils/misc/oidrbtree.cpp b/src/common/backend/utils/misc/oidrbtree.cpp index d651a4c64..3d80cdecc 100644 --- a/src/common/backend/utils/misc/oidrbtree.cpp +++ b/src/common/backend/utils/misc/oidrbtree.cpp @@ -26,15 +26,6 @@ #include "utils/oidrbtree.h" #include "utils/memutils.h" -MemoryContext GetOidRBTreeMemory() -{ - if (!t_thrd.security_policy_cxt.OidRBTreeMemoryContext) { - t_thrd.security_policy_cxt.OidRBTreeMemoryContext = AllocSetContextCreate(TopMemoryContext, "OidRBTreeMemory", - ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); - } - return t_thrd.security_policy_cxt.OidRBTreeMemoryContext; -} - void DeleteOidRBTreeMemory() { if (t_thrd.security_policy_cxt.OidRBTreeMemoryContext != NULL) { @@ -72,9 +63,7 @@ static void OidRBCombine(RBNode* existing, const RBNode* newdata, void* arg) /* Allocator function for oid rbtree */ static RBNode* OidRBAlloc(void* arg) { - MemoryContext oldContext = MemoryContextSwitchTo(GetOidRBTreeMemory()); OidRBNode* oidRBNode = static_cast(palloc(sizeof(OidRBNode))); - (void)MemoryContextSwitchTo(oldContext); return (RBNode*)oidRBNode; } @@ -87,9 +76,7 @@ static void OidRBDealloc(RBNode* rbNode, void* arg) OidRBTree* CreateOidRBTree() { - MemoryContext oldContext = MemoryContextSwitchTo(GetOidRBTreeMemory()); OidRBTree* oidRBTree = rb_create(sizeof(OidRBNode), OidRBComparator, OidRBCombine, OidRBAlloc, OidRBDealloc, NULL); - (void)MemoryContextSwitchTo(oldContext); return oidRBTree; } @@ -97,14 +84,12 @@ static List* OidRBTreeGetNodeList(OidRBTree& oidRBtree) { List* nodeList = NIL; OidRBTree* tree = &oidRBtree; - MemoryContext oldContext = MemoryContextSwitchTo(GetOidRBTreeMemory()); rb_begin_iterate(tree, InvertedWalk); RBNode *node = rb_iterate(tree); while (node != NULL) { nodeList = lappend(nodeList, node); node = rb_iterate(tree); } - (void)MemoryContextSwitchTo(oldContext); return nodeList; } diff --git a/src/common/backend/utils/resowner/resowner.cpp b/src/common/backend/utils/resowner/resowner.cpp index 333e9e283..dd62b1d97 100755 --- a/src/common/backend/utils/resowner/resowner.cpp +++ b/src/common/backend/utils/resowner/resowner.cpp @@ -39,7 +39,9 @@ #include "catalog/pg_hashbucket_fn.h" /* - * ResourceOwner objects look like this + * ResourceOwner objects look like this. When tracking new types of resource, + * you must at least add the 'Remember' interface for that resource and adapt + * the 'ResourceOwnerConcat' function. */ typedef struct ResourceOwnerData { ResourceOwner parent; /* NULL if no parent (toplevel owner) */ @@ -542,6 +544,174 @@ void ResourceOwnerDelete(ResourceOwner owner) ResourceOwnerFreeOwner(owner, true); } +/* + * Part 1 of 'ResourceOwnerConcat'. Concatenate the top 12 resources of two owners. + */ +static void ResourceOwnerConcatPart1(ResourceOwner target, ResourceOwner source) +{ + int i; + + for (i = 0; i < source->nbuffers; i++) { + ResourceOwnerEnlargeBuffers(target); + ResourceOwnerRememberBuffer(target, source->buffers[i]); + } + + for (i = 0; i < source->nlocalcatclist; i++) { + ResourceOwnerEnlargeLocalCatCList(target); + ResourceOwnerRememberLocalCatCList(target, source->localcatclists[i]); + } + + for (i = 0; i < source->nlocalcatctup; i++) { + ResourceOwnerEnlargeLocalCatCTup(target); + ResourceOwnerRememberLocalCatCTup(target, source->localcatctups[i]); + } + + for (i = 0; i < source->nglobalcatctup; i++) { + ResourceOwnerEnlargeGlobalCatCTup(target); + ResourceOwnerRememberGlobalCatCTup(target, source->globalcatctups[i]); + } + + for (i = 0; i < source->nglobalcatclist; i++) { + ResourceOwnerEnlargeGlobalCatCList(target); + ResourceOwnerRememberGlobalCatCList(target, source->globalcatclists[i]); + } + + for (i = 0; i < source->nglobalbaseentry; i++) { + ResourceOwnerEnlargeGlobalBaseEntry(target); + ResourceOwnerRememberGlobalBaseEntry(target, source->globalbaseentries[i]); + } + + for (i = 0; i < source->nglobaldbentry; i++) { + ResourceOwnerEnlargeGlobalDBEntry(target); + ResourceOwnerRememberGlobalDBEntry(target, source->globaldbentries[i]); + } + + for (i = 0; i < source->nglobalisexclusive; i++) { + ResourceOwnerEnlargeGlobalIsExclusive(target); + ResourceOwnerRememberGlobalIsExclusive(target, source->globalisexclusives[i]); + } + + for (i = 0; i < source->ncatrefs; i++) { + ResourceOwnerEnlargeCatCacheRefs(target); + ResourceOwnerRememberCatCacheRef(target, source->catrefs[i]); + } + + for (i = 0; i < source->ncatlistrefs; i++) { + ResourceOwnerEnlargeCatCacheListRefs(target); + ResourceOwnerRememberCatCacheListRef(target, source->catlistrefs[i]); + } + + for (i = 0; i < source->nrelrefs; i++) { + ResourceOwnerEnlargeRelationRefs(target); + ResourceOwnerRememberRelationRef(target, source->relrefs[i]); + } + + for (i = 0; i < source->npartrefs; i++) { + ResourceOwnerEnlargePartitionRefs(target); + ResourceOwnerRememberPartitionRef(target, source->partrefs[i]); + } +} + +/* + * Part 2 of 'ResourceOwnerConcat'. Concatenate the remaining resources of two owners. + */ +static void ResourceOwnerConcatPart2(ResourceOwner target, ResourceOwner source) +{ + int i; + + for (i = 0; i < source->nfakerelrefs; i++) { + dlist_push_tail(&(target->fakerelrefs_list), dlist_pop_head_node(&(source->fakerelrefs_list))); + target->nfakerelrefs++; + } + + for (i = 0; i < source->nfakepartrefs; i++) { + ResourceOwnerEnlargeFakepartRefs(target); + ResourceOwnerRememberFakepartRef(target, source->fakepartrefs[i]); + } + + for (i = 0; i < source->nplanrefs; i++) { + ResourceOwnerEnlargePlanCacheRefs(target); + ResourceOwnerRememberPlanCacheRef(target, source->planrefs[i]); + } + + for (i = 0; i < source->ntupdescs; i++) { + ResourceOwnerEnlargeTupleDescs(target); + ResourceOwnerRememberTupleDesc(target, source->tupdescs[i]); + } + + for (i = 0; i < source->nsnapshots; i++) { + ResourceOwnerEnlargeSnapshots(target); + ResourceOwnerRememberSnapshot(target, source->snapshots[i]); + } + + for (i = 0; i < source->nfiles; i++) { + ResourceOwnerEnlargeFiles(target); + ResourceOwnerRememberFile(target, source->files[i]); + } + + for (i = 0; i < source->nDataCacheSlots; i++) { + ResourceOwnerEnlargeDataCacheSlot(target); + ResourceOwnerRememberDataCacheSlot(target, source->dataCacheSlots[i]); + } + + for (i = 0; i < source->nMetaCacheSlots; i++) { + ResourceOwnerEnlargeMetaCacheSlot(target); + ResourceOwnerRememberMetaCacheSlot(target, source->metaCacheSlots[i]); + } + + for (i = 0; i < source->nPthreadMutex; i++) { + ResourceOwnerEnlargePthreadMutex(target); + ResourceOwnerRememberPthreadMutex(target, source->pThdMutexs[i]); + } + + for (i = 0; i < source->nPthreadRWlock; i++) { + ResourceOwnerEnlargePthreadRWlock(target); + ResourceOwnerRememberPthreadRWlock(target, source->pThdRWlocks[i]); + } + + for (i = 0; i < source->npartmaprefs; i++) { + ResourceOwnerEnlargePartitionMapRefs(target); + ResourceOwnerRememberPartitionMapRef(target, source->partmaprefs[i]); + } + + for (i = 0; i < source->nglobalMemContext; i++) { + ResourceOwnerEnlargeGMemContext(target); + ResourceOwnerRememberGMemContext(target, source->globalMemContexts[i]); + } +} + +/* ResourceOwnerConcat + * Concatenate two owners. + * + * The resources traced by the 'source' are placed in the 'target' for tracing. + * The advantage is that the memory occupied by the 'source' owner can be released + * to reduce the memory consumed by tracing resources. When using a stream-plan, + * this is useful for preventing "memory is temporarily unavailable" error when + * executing a large number of SQLs in a single transaction/procedure. + * + * Note: After the invoking is complete, the memory of the 'source' should be release. + */ +void ResourceOwnerConcat(ResourceOwner target, ResourceOwner source) +{ + Assert(target && source); + /* + * When modifying the structure of ResourceOwnerData, note that the ResourceOwnerConcat + * function needs to be adapted when tracing new types of resources. + */ + Assert(sizeof(ResourceOwnerData) == 448); /* The current size of ResourceOwnerData is 448 */ + + while (source->firstchild != NULL) { + ResourceOwnerConcat(target, source->firstchild); + } + + /* + * ResourceOwner traces too many resources. To reduce cyclomatic complexity, + * the Concatenate operation is divided into two parts. + */ + ResourceOwnerConcatPart1(target, source); + ResourceOwnerConcatPart2(target, source); +} + /* * Fetch parent of a ResourceOwner (returns NULL if top-level owner) */ @@ -575,6 +745,14 @@ ResourceOwner ResourceOwnerGetFirstChild(ResourceOwner owner) return owner->firstchild; } +/* + * Fetch memory context of a ResourceOwner + */ +MemoryContext ResourceOwnerGetMemCxt(ResourceOwner owner) +{ + return owner->memCxt; +} + /* * Reassign a ResourceOwner to have a new parent */ diff --git a/src/common/pl/plpgsql/src/gram.y b/src/common/pl/plpgsql/src/gram.y index 17580149d..dad560b29 100755 --- a/src/common/pl/plpgsql/src/gram.y +++ b/src/common/pl/plpgsql/src/gram.y @@ -2958,11 +2958,18 @@ for_control : for_variable K_IN if ($1.rec) { #ifndef ENABLE_MULTIPLE_NODES - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && IMPLICIT_FOR_LOOP_VARIABLE) { + /* only A format and not in upgrade, IMPLICIT_FOR_LOOP_VARIABLE is valid */ + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT + && IMPLICIT_FOR_LOOP_VARIABLE + && u_sess->attr.attr_common.upgrade_mode == 0) { BuildForQueryVariable(expr1, &newp->row, &newp->rec, $1.name, $1.lineno); check_assignable((PLpgSQL_datum *)newp->rec ? (PLpgSQL_datum *)newp->rec : (PLpgSQL_datum *)newp->row, @1); } else { + /* check the sql */ + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && ALLOW_PROCEDURE_COMPILE_CHECK) { + (void)getCursorTupleDesc(expr1, false, true); + } newp->rec = $1.rec; check_assignable((PLpgSQL_datum *) newp->rec, @1); } @@ -2974,11 +2981,18 @@ for_control : for_variable K_IN else if ($1.row) { #ifndef ENABLE_MULTIPLE_NODES - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && IMPLICIT_FOR_LOOP_VARIABLE) { + /* only A format and not in upgrade, IMPLICIT_FOR_LOOP_VARIABLE is valid */ + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT + && IMPLICIT_FOR_LOOP_VARIABLE + && u_sess->attr.attr_common.upgrade_mode == 0) { BuildForQueryVariable(expr1, &newp->row, &newp->rec, $1.name, $1.lineno); check_assignable((PLpgSQL_datum *)newp->rec ? (PLpgSQL_datum *)newp->rec : (PLpgSQL_datum *)newp->row, @1); } else { + /* check the sql */ + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && ALLOW_PROCEDURE_COMPILE_CHECK) { + (void)getCursorTupleDesc(expr1, false, true); + } newp->row = $1.row; check_assignable((PLpgSQL_datum *) newp->row, @1); } @@ -2990,11 +3004,18 @@ for_control : for_variable K_IN else if ($1.scalar) { #ifndef ENABLE_MULTIPLE_NODES - if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && IMPLICIT_FOR_LOOP_VARIABLE) { + /* only A format and not in upgrade, IMPLICIT_FOR_LOOP_VARIABLE is valid */ + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT + && IMPLICIT_FOR_LOOP_VARIABLE + && u_sess->attr.attr_common.upgrade_mode == 0) { BuildForQueryVariable(expr1, &newp->row, &newp->rec, $1.name, $1.lineno); check_assignable((PLpgSQL_datum *)newp->rec ? (PLpgSQL_datum *)newp->rec : (PLpgSQL_datum *)newp->row, @1); } else { + /* check the sql */ + if (u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && ALLOW_PROCEDURE_COMPILE_CHECK) { + (void)getCursorTupleDesc(expr1, false, true); + } /* convert single scalar to list */ newp->row = make_scalar_list1($1.name, $1.scalar, $1.dno, $1.lineno, @1); /* no need for check_assignable */ @@ -4342,6 +4363,7 @@ stmt_open : K_OPEN cursor_variable yyerror("syntax error"); } } +#ifndef ENABLE_MULTIPLE_NODES if (newp->query != NULL && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && ALLOW_PROCEDURE_COMPILE_CHECK) { (void)getCursorTupleDesc(newp->query, false, true); } @@ -4349,6 +4371,7 @@ stmt_open : K_OPEN cursor_variable { (void)getCursorTupleDesc(newp->dynquery, false, true); } +#endif } else { @@ -8773,9 +8796,11 @@ make_execsql_stmt(int firsttoken, int location) pfree_ext(ds.data); check_sql_expr(expr->query, location, 0); +#ifndef ENABLE_MULTIPLE_NODES if (firsttoken == K_SELECT && u_sess->attr.attr_sql.sql_compatibility == A_FORMAT && ALLOW_PROCEDURE_COMPILE_CHECK) { (void)getCursorTupleDesc(expr, false, true); } +#endif execsql = (PLpgSQL_stmt_execsql *)palloc(sizeof(PLpgSQL_stmt_execsql)); execsql->cmd_type = PLPGSQL_STMT_EXECSQL; execsql->lineno = plpgsql_location_to_lineno(location); diff --git a/src/common/pl/plpgsql/src/pl_comp.cpp b/src/common/pl/plpgsql/src/pl_comp.cpp index 3d8b90fbc..528d51802 100644 --- a/src/common/pl/plpgsql/src/pl_comp.cpp +++ b/src/common/pl/plpgsql/src/pl_comp.cpp @@ -752,6 +752,7 @@ static PLpgSQL_function* do_compile(FunctionCallInfo fcinfo, HeapTuple proc_tup, func->resolve_option = GetResolveOption(); func->invalItems = NIL; func->is_autonomous = false; + func->is_insert_gs_source = false; func->pkg_oid = pkgoid; func->fn_searchpath->addCatalog = true; @@ -1420,6 +1421,7 @@ PLpgSQL_function* plpgsql_compile_inline(char* proc_source) func->fn_retbyval = true; func->fn_rettyplen = sizeof(int32); func->is_autonomous = false; + func->is_insert_gs_source = false; getTypeInputInfo(VOIDOID, &typinput, &func->fn_rettypioparam); fmgr_info(typinput, &(func->fn_retinput)); diff --git a/src/common/pl/plpgsql/src/pl_exec.cpp b/src/common/pl/plpgsql/src/pl_exec.cpp index 41ab2a0d4..a7699c812 100644 --- a/src/common/pl/plpgsql/src/pl_exec.cpp +++ b/src/common/pl/plpgsql/src/pl_exec.cpp @@ -258,7 +258,7 @@ static PLpgSQL_rec* copyPLpgsqlRec(PLpgSQL_rec* src); static PLpgSQL_recfield* copyPLpgsqlRecfield(PLpgSQL_recfield* src); static List* invalid_depend_func_and_packgae(Oid pkgOid); static void ReportCompileConcurrentError(const char* objName, bool isPackage); - +static Datum CopyFcinfoArgValue(Oid typOid, Datum value); /* ---------- * plpgsql_check_line_validity Called by the debugger plugin for * validating a given linenumber @@ -838,9 +838,7 @@ Datum plpgsql_exec_autonm_function(PLpgSQL_function* func, #ifndef ENABLE_MULTIPLE_NODES uint64 sessionId = IS_THREAD_POOL_WORKER ? u_sess->session_id : t_thrd.proc_cxt.MyProcPid; /* add session package values to global for autonm session, to restore package values */ - if (OidIsValid(func->fn_oid)) { - BuildSessionPackageRuntimeForAutoSession(sessionId, u_sess->autonomous_parent_sessionid, &estate, func); - } + BuildSessionPackageRuntimeForAutoSession(sessionId, u_sess->autonomous_parent_sessionid, &estate, func); #endif /* Statement concatenation. If the block is an anonymous block, the entire anonymous block is returned. */ @@ -1011,8 +1009,7 @@ Datum plpgsql_exec_function(PLpgSQL_function* func, FunctionCallInfo fcinfo, boo #ifndef ENABLE_MULTIPLE_NODES check_debug(func, &estate); - bool isExecAutoFunc = OidIsValid(func->fn_oid) && - u_sess->is_autonomous_session == true && u_sess->SPI_cxt._connected == 0; + bool isExecAutoFunc = u_sess->is_autonomous_session == true && u_sess->SPI_cxt._connected == 0; /* when exec autonomous transaction procedure, need update package values by parent session */ if (isExecAutoFunc) { initAutoSessionPkgsValue(u_sess->autonomous_parent_sessionid); @@ -1083,12 +1080,12 @@ Datum plpgsql_exec_function(PLpgSQL_function* func, FunctionCallInfo fcinfo, boo while (argmodes[outArgCnt] == PROARGMODE_OUT) { outArgCnt++; } - var->value = fcinfo->arg[outArgCnt]; + var->value = CopyFcinfoArgValue(fcinfo->argTypes[outArgCnt], fcinfo->arg[outArgCnt]); var->isnull = fcinfo->argnull[outArgCnt]; var->freeval = false; outArgCnt++; } else { - var->value = fcinfo->arg[i]; + var->value = CopyFcinfoArgValue(fcinfo->argTypes[i], fcinfo->arg[i]); var->isnull = fcinfo->argnull[i]; var->freeval = false; } @@ -1469,6 +1466,25 @@ Datum plpgsql_exec_function(PLpgSQL_function* func, FunctionCallInfo fcinfo, boo return estate.retval; } +static Datum CopyFcinfoArgValue(Oid typOid, Datum value) +{ +#ifdef ENABLE_MULTIPLE_NODES + return value; +#endif + if (!OidIsValid(typOid)) { + return value; + } + /* + * For centralized database, package value may be in param. + * In this case, the value should be copyed, becaues the ref + * value may be influenced by procedure. + */ + bool typByVal = false; + int16 typLen; + get_typlenbyval(typOid, &typLen, &typByVal); + return datumCopy(value, typByVal, typLen); +} + static void RecordSetGeneratedField(PLpgSQL_rec *recNew) { int natts = recNew->tupdesc->natts; @@ -4580,16 +4596,10 @@ static int exec_stmt_return(PLpgSQL_execstate* estate, PLpgSQL_stmt_return* stmt case PLPGSQL_DTYPE_VAR: { PLpgSQL_var* var = (PLpgSQL_var*)retvar; Datum value = var->value; - if (is_external_clob(var->datatype->typoid, var->isnull, value)) { - bool is_null = false; - bool is_have_huge_clob = false; - struct varatt_lob_pointer* lob_pointer = (varatt_lob_pointer*)(VARDATA_EXTERNAL(value)); - value = fetch_lob_value_from_tuple(lob_pointer, InvalidOid, &is_null, &is_have_huge_clob); - if (is_have_huge_clob) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("huge clob do not support as return parameter"))); - } + if (is_huge_clob(var->datatype->typoid, var->isnull, value)) { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("huge clob do not support as return parameter"))); } if (need_param_seperation) { estate->paramval = value; @@ -5435,7 +5445,7 @@ static int exec_stmt_execsql(PLpgSQL_execstate* estate, PLpgSQL_stmt_execsql* st bool has_alloc = false; TransactionId oldTransactionId = SPI_get_top_transaction_id(); - + /* * On the first call for this statement generate the plan, and detect * whether the statement is INSERT/UPDATE/DELETE/MERGE @@ -5467,7 +5477,19 @@ static int exec_stmt_execsql(PLpgSQL_execstate* estate, PLpgSQL_stmt_execsql* st if (ENABLE_CN_GPC && g_instance.plan_cache->CheckRecreateSPICachePlan(expr->plan)) { g_instance.plan_cache->RecreateSPICachePlan(expr->plan); } - +#ifndef ENABLE_MULTIPLE_NODES + ListCell* l = NULL; + bool isforbid = true; + bool savedisAllowCommitRollback = false; + bool needResetErrMsg = false; + foreach (l, SPI_plan_get_plan_sources(expr->plan)) { + CachedPlanSource* plansource = (CachedPlanSource*)lfirst(l); + isforbid = CheckElementParsetreeTag(plansource->raw_parse_tree); + if (isforbid) { + needResetErrMsg = stp_disable_xact_and_set_err_msg(&savedisAllowCommitRollback, STP_XACT_COMPL_SQL); + } + } +#endif /* * Set up ParamListInfo (hook function and possibly data values) */ @@ -5513,10 +5535,6 @@ static int exec_stmt_execsql(PLpgSQL_execstate* estate, PLpgSQL_stmt_execsql* st plpgsql_estate = estate; -#ifndef ENABLE_MULTIPLE_NODES - t_thrd.xact_cxt.isSelectInto = stmt->into; -#endif - /* * Execute the plan */ @@ -5529,7 +5547,11 @@ static int exec_stmt_execsql(PLpgSQL_execstate* estate, PLpgSQL_stmt_execsql* st // This is used for nested STP. If the transaction Id changed, // then need to create new econtext for the TopTransaction. stp_check_transaction_and_create_econtext(estate,oldTransactionId); - +#ifndef ENABLE_MULTIPLE_NODES + if (isforbid) { + stp_reset_xact_state_and_err_msg(savedisAllowCommitRollback, needResetErrMsg); + } +#endif plpgsql_estate = NULL; /* @@ -5721,7 +5743,6 @@ static int exec_stmt_execsql(PLpgSQL_execstate* estate, PLpgSQL_stmt_execsql* st estate->cursor_return_data = saved_cursor_data; estate->cursor_return_numbers = saved_cursor_numbers; - t_thrd.xact_cxt.isSelectInto = false; return PLPGSQL_RC_OK; } @@ -5900,6 +5921,9 @@ static int exec_stmt_dynexecute(PLpgSQL_execstate* estate, PLpgSQL_stmt_dynexecu bool savedisAllowCommitRollback = false; bool needResetErrMsg = false; needResetErrMsg = stp_disable_xact_and_set_err_msg(&savedisAllowCommitRollback, STP_XACT_USED_AS_EXPR); +#else + /* Saves the status of whether to send commandId. */ + bool saveSetSendCommandId = IsSendCommandId(); #endif /* * First we evaluate the string expression after the EXECUTE keyword. Its @@ -5994,6 +6018,8 @@ static int exec_stmt_dynexecute(PLpgSQL_execstate* estate, PLpgSQL_stmt_dynexecu FormatCallStack* plcallstack = t_thrd.log_cxt.call_stack; #ifndef ENABLE_MULTIPLE_NODES estate_cursor_set(plcallstack); +#else + SetSendCommandId(saveSetSendCommandId); #endif if (plcallstack != NULL) { t_thrd.log_cxt.call_stack = plcallstack->prev; @@ -6004,6 +6030,10 @@ static int exec_stmt_dynexecute(PLpgSQL_execstate* estate, PLpgSQL_stmt_dynexecu } PG_END_TRY(); +#ifdef ENABLE_MULTIPLE_NODES + SetSendCommandId(saveSetSendCommandId); +#endif + /* * This is used for nested STP. If the transaction Id changed, * then need to create new econtext for the TopTransaction. @@ -6485,7 +6515,10 @@ static int exec_stmt_open(PLpgSQL_execstate* estate, PLpgSQL_stmt_open* stmt) errmsg("cursor \"%s\" already in use in OPEN statement.", curname))); } } - +#ifdef ENABLE_MULTIPLE_NODES + /* In distributed mode, the commandId is sent when a cursor is opened. */ + SetSendCommandId(true); +#endif /* ---------- * Process the OPEN according to it's type. * ---------- @@ -7395,16 +7428,10 @@ void exec_assign_value(PLpgSQL_execstate* estate, PLpgSQL_datum* target, Datum v MemoryContext oldcontext = NULL; AttrNumber attrno = ((PLpgSQL_arrayelem*)target)->assignattrno; - if (is_external_clob(valtype, *isNull, value)) { - bool is_null = false; - bool is_have_huge_clob = false; - struct varatt_lob_pointer* lob_pointer = (varatt_lob_pointer*)(VARDATA_EXTERNAL(value)); - value = fetch_lob_value_from_tuple(lob_pointer, InvalidOid, &is_null, &is_have_huge_clob); - if (is_have_huge_clob) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("huge clob do not support as array element."))); - } + if (is_huge_clob(valtype, *isNull, value)) { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("huge clob do not support as array element."))); } /* * We need to do subscript evaluation, which might require @@ -7683,16 +7710,10 @@ void exec_assign_value(PLpgSQL_execstate* estate, PLpgSQL_datum* target, Datum v MemoryContext oldcontext = NULL; AttrNumber attrno = ((PLpgSQL_tableelem*)target)->assignattrno; - if (is_external_clob(valtype, *isNull, value)) { - bool is_null = false; - bool is_have_huge_clob = false; - struct varatt_lob_pointer* lob_pointer = (varatt_lob_pointer*)(VARDATA_EXTERNAL(value)); - value = fetch_lob_value_from_tuple(lob_pointer, InvalidOid, &is_null, &is_have_huge_clob); - if (is_have_huge_clob) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("huge clob do not support as table of element."))); - } + if (is_huge_clob(valtype, *isNull, value)) { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("huge clob do not support as table of element."))); } /* @@ -7965,17 +7986,6 @@ void exec_assign_value(PLpgSQL_execstate* estate, PLpgSQL_datum* target, Datum v /* * Target has a assign list */ - if (is_external_clob(valtype, *isNull, value)) { - bool is_null = false; - bool is_have_huge_clob = false; - struct varatt_lob_pointer* lob_pointer = (varatt_lob_pointer*)(VARDATA_EXTERNAL(value)); - value = fetch_lob_value_from_tuple(lob_pointer, InvalidOid, &is_null, &is_have_huge_clob); - if (is_have_huge_clob) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("huge clob do not support as record element."))); - } - } PLpgSQL_assignlist* assignvar = (PLpgSQL_assignlist*)target; List* assignlist = assignvar->assignlist; PLpgSQL_datum* assigntarget = estate->datums[assignvar->targetno]; @@ -9422,7 +9432,7 @@ static Datum exec_eval_expr(PLpgSQL_execstate* estate, PLpgSQL_expr* expr, bool* static int exec_run_select(PLpgSQL_execstate* estate, PLpgSQL_expr* expr, long maxtuples, Portal* portalP, bool isCollectParam) { - ParamListInfo paramLI; + ParamListInfo paramLI = NULL; int rc; /* @@ -9437,8 +9447,12 @@ static int exec_run_select(PLpgSQL_execstate* estate, PLpgSQL_expr* expr, long m /* * Set up ParamListInfo (hook function and possibly data values) + * For validation estate->datums should be NULL, since there is + * no parameter set vo validation */ - paramLI = setup_param_list(estate, expr); + if (estate->datums) { + paramLI = setup_param_list(estate, expr); + } /* * If a portal was requested, put the query into the portal @@ -10646,12 +10660,6 @@ static void exec_move_row(PLpgSQL_execstate* estate, valtype = InvalidOid; } - if (valtype == CLOBOID && !isnull && VARATT_IS_HUGE_TOAST_POINTER(value)) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("huge clob do not support as record element."))); - } - /* accept function's return value for cursor */ if (isnull == false && CheckTypeIsCursor(row, valtype, fnum) && estate->cursor_return_data != NULL) { @@ -10773,17 +10781,10 @@ HeapTuple make_tuple_from_row(PLpgSQL_execstate* estate, PLpgSQL_row* row, Tuple exec_eval_datum(estate, estate->datums[row->varnos[i]], &fieldtypeid, &fieldtypmod, &dvalues[i], &nulls[i]); } - if (is_external_clob(fieldtypeid, nulls[i], dvalues[i])) { - bool is_null = false; - bool is_have_huge_clob = false; - struct varatt_lob_pointer* lob_pointer = (varatt_lob_pointer*)(VARDATA_EXTERNAL(dvalues[i])); - dvalues[i] = fetch_lob_value_from_tuple(lob_pointer, InvalidOid, &is_null, &is_have_huge_clob); - if (is_have_huge_clob) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("huge clob do not support concat a row"))); - } - nulls[i] = is_null; + if (is_huge_clob(fieldtypeid, nulls[i], dvalues[i])) { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("huge clob do not support concat a row"))); } if (estate->is_exception && fieldtypeid == REFCURSOROID) { diff --git a/src/common/pl/plpgsql/src/pl_global_package_runtime_cache.cpp b/src/common/pl/plpgsql/src/pl_global_package_runtime_cache.cpp index 05e08da4a..4ee0bd190 100644 --- a/src/common/pl/plpgsql/src/pl_global_package_runtime_cache.cpp +++ b/src/common/pl/plpgsql/src/pl_global_package_runtime_cache.cpp @@ -186,6 +186,7 @@ static SessionPackageRuntime* CopySessionPackageRuntime(SessionPackageRuntime *r sessPkgRuntime->portalContext = CopyPortalContexts(runtime->portalContext); sessPkgRuntime->portalData = CopyPortalDatas(runtime); sessPkgRuntime->funcValInfo = CopyFuncInfoDatas(runtime); + sessPkgRuntime->is_insert_gs_source = runtime->is_insert_gs_source; MemoryContextSwitchTo(oldCtx); return sessPkgRuntime; } diff --git a/src/common/pl/plpgsql/src/pl_handler.cpp b/src/common/pl/plpgsql/src/pl_handler.cpp index 8be0a334b..2a95a7faf 100755 --- a/src/common/pl/plpgsql/src/pl_handler.cpp +++ b/src/common/pl/plpgsql/src/pl_handler.cpp @@ -219,7 +219,9 @@ static void InsertGsSource(Oid objId, Oid nspid, const char* name, const char* t { (void)CompileStatusSwtichTo(NONE_STATUS); u_sess->plsql_cxt.curr_compile_context = NULL; + u_sess->plsql_cxt.is_insert_gs_source = true; ExecuteDoStmt(stmt, true); + u_sess->plsql_cxt.is_insert_gs_source = false; } PG_CATCH(); { @@ -228,6 +230,7 @@ static void InsertGsSource(Oid objId, Oid nspid, const char* name, const char* t } (void)CompileStatusSwtichTo(save_compile_status); u_sess->plsql_cxt.curr_compile_context = save_compile_context; + u_sess->plsql_cxt.is_insert_gs_source = false; clearCompileContextList(save_compile_list_length); PG_RE_THROW(); } @@ -745,6 +748,8 @@ Datum plpgsql_call_handler(PG_FUNCTION_ARGS) u_sess->opt_cxt.is_stream = true; u_sess->opt_cxt.is_stream_support = true; } + /* Saves the status of whether to send commandId. */ + bool saveSetSendCommandId = IsSendCommandId(); #else int outerDop = u_sess->opt_cxt.query_dop; u_sess->opt_cxt.query_dop = 1; @@ -821,6 +826,7 @@ Datum plpgsql_call_handler(PG_FUNCTION_ARGS) PLpgSQL_compile_context* save_compile_context = u_sess->plsql_cxt.curr_compile_context; int save_compile_list_length = list_length(u_sess->plsql_cxt.compile_context_list); int save_compile_status = u_sess->plsql_cxt.compile_status; + PG_TRY(); { /* @@ -873,7 +879,6 @@ Datum plpgsql_call_handler(PG_FUNCTION_ARGS) estate_cursor_set(plcallstack); #endif - if (plcallstack != NULL) { t_thrd.log_cxt.call_stack = plcallstack->prev; } @@ -970,7 +975,9 @@ Datum plpgsql_call_handler(PG_FUNCTION_ARGS) if (u_sess->SPI_cxt._connected == 0) { t_thrd.utils_cxt.STPSavedResourceOwner = NULL; } - +#ifdef ENABLE_MULTIPLE_NODES + SetSendCommandId(saveSetSendCommandId); +#endif /* ErrorData could be allocted in SPI's MemoryContext, copy it. */ oldContext = MemoryContextSwitchTo(oldContext); ErrorData *edata = CopyErrorData(); @@ -992,6 +999,10 @@ Datum plpgsql_call_handler(PG_FUNCTION_ARGS) if (u_sess->SPI_cxt._connected == 0) { t_thrd.utils_cxt.STPSavedResourceOwner = NULL; } +#ifdef ENABLE_MULTIPLE_NODES + SetSendCommandId(saveSetSendCommandId); +#endif + /* * Disconnect from SPI manager */ @@ -1039,8 +1050,12 @@ Datum plpgsql_inline_handler(PG_FUNCTION_ARGS) #ifndef ENABLE_MULTIPLE_NODES int outerDop = u_sess->opt_cxt.query_dop; u_sess->opt_cxt.query_dop = 1; +#else + /* Saves the status of whether to send commandId. */ + bool saveSetSendCommandId = IsSendCommandId(); #endif + _PG_init(); AssertEreport(IsA(codeblock, InlineCodeBlock), MOD_PLSQL, "Inline code block is required."); @@ -1075,6 +1090,7 @@ Datum plpgsql_inline_handler(PG_FUNCTION_ARGS) PG_END_TRY(); PGSTAT_END_PLSQL_TIME_RECORD(PL_COMPILATION_TIME); + func->is_insert_gs_source = u_sess->plsql_cxt.is_insert_gs_source; /* Mark packages the function use, so them can't be deleted from under us */ AddPackageUseCount(func); /* Mark the function as busy, just pro forma */ @@ -1093,7 +1109,6 @@ Datum plpgsql_inline_handler(PG_FUNCTION_ARGS) fake_fcinfo.flinfo = &flinfo; flinfo.fn_oid = InvalidOid; flinfo.fn_mcxt = CurrentMemoryContext; - PGSTAT_START_PLSQL_TIME_RECORD(); /* save flag for nest plpgsql compile */ save_compile_context = u_sess->plsql_cxt.curr_compile_context; @@ -1135,11 +1150,18 @@ Datum plpgsql_inline_handler(PG_FUNCTION_ARGS) clearCompileContextList(save_compile_list_length); /* AutonomousSession Disconnecting and releasing resources */ DestoryAutonomousSession(true); +#ifdef ENABLE_MULTIPLE_NODES + SetSendCommandId(saveSetSendCommandId); +#endif PG_RE_THROW(); } PG_END_TRY(); +#ifdef ENABLE_MULTIPLE_NODES + SetSendCommandId(saveSetSendCommandId); +#endif + /* Disconnecting and releasing resources */ DestoryAutonomousSession(false); diff --git a/src/gausskernel/cbb/instruments/statement/instr_statement.cpp b/src/gausskernel/cbb/instruments/statement/instr_statement.cpp index 89e2d8ab2..2541fcc29 100755 --- a/src/gausskernel/cbb/instruments/statement/instr_statement.cpp +++ b/src/gausskernel/cbb/instruments/statement/instr_statement.cpp @@ -1399,12 +1399,10 @@ void instr_stmt_report_basic_info() } if (to_update_db_name || to_update_user_name || to_update_client_addr) { ResourceOwner old_cur_owner = t_thrd.utils_cxt.CurrentResourceOwner; - MemoryContext old_ctx = MemoryContextSwitchTo(t_thrd.mem_cxt.msg_mem_cxt); - t_thrd.utils_cxt.CurrentResourceOwner = ResourceOwnerCreate(NULL, "Full/Slow SQL", + t_thrd.utils_cxt.CurrentResourceOwner = ResourceOwnerCreate(old_cur_owner, "Full/Slow SQL", THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_DFX)); - (void)MemoryContextSwitchTo(old_ctx); - old_ctx = MemoryContextSwitchTo(u_sess->statement_cxt.stmt_stat_cxt); + MemoryContext old_ctx = MemoryContextSwitchTo(u_sess->statement_cxt.stmt_stat_cxt); if (to_update_db_name) { u_sess->statement_cxt.db_name = get_database_name(beentry->st_databaseid); } diff --git a/src/gausskernel/cbb/instruments/unique_sql/instr_unique_sql.cpp b/src/gausskernel/cbb/instruments/unique_sql/instr_unique_sql.cpp index d1cb635a3..06b8e24ad 100755 --- a/src/gausskernel/cbb/instruments/unique_sql/instr_unique_sql.cpp +++ b/src/gausskernel/cbb/instruments/unique_sql/instr_unique_sql.cpp @@ -494,30 +494,6 @@ void UpdateUniqueSQLVecSortStats(Batchsortstate* state, uint64 spill_count, Time } } -static void mask_unique_sql_str(UniqueSQL* unique_sql) -{ - errno_t rc; - - /* hide password */ - if (unique_sql->unique_sql != NULL) { - char* mask_str = NULL; - mask_str = maskPassword(unique_sql->unique_sql); - if (mask_str != NULL) { - rc = memset_s(unique_sql->unique_sql, UNIQUE_SQL_MAX_LEN - 1, 0, UNIQUE_SQL_MAX_LEN - 1); - securec_check(rc, "\0", "\0"); - - /* after calling maskPassword, mask_str can be longer than original string, - * now the length of masked password('*..*') is fixed to 'password_min_length'(GUC) */ - size_t valid_mask_len = strlen(mask_str) > (size_t)(UNIQUE_SQL_MAX_LEN - 1) - ? (size_t)(UNIQUE_SQL_MAX_LEN - 1) - : strlen(mask_str); - rc = memcpy_s(unique_sql->unique_sql, UNIQUE_SQL_MAX_LEN - 1, mask_str, valid_mask_len); - securec_check(rc, "\0", "\0"); - pfree(mask_str); - } - } -} - static void set_unique_sql_string_in_entry(UniqueSQL* entry, Query* query, const char* sql, int32 multi_sql_offset) { errno_t rc = EOK; @@ -536,7 +512,6 @@ static void set_unique_sql_string_in_entry(UniqueSQL* entry, Query* query, const // generate and store normalized query string if (normalized_unique_querystring(query, sql, entry->unique_sql, UNIQUE_SQL_MAX_LEN - 1, multi_sql_offset)) { - mask_unique_sql_str(entry); entry->unique_sql = trim(entry->unique_sql); } else { ereport(LOG, diff --git a/src/gausskernel/cbb/instruments/utils/unique_query.cpp b/src/gausskernel/cbb/instruments/utils/unique_query.cpp index 30a51c1d1..0afdfed52 100755 --- a/src/gausskernel/cbb/instruments/utils/unique_query.cpp +++ b/src/gausskernel/cbb/instruments/utils/unique_query.cpp @@ -203,12 +203,11 @@ bool normalized_unique_querystring(Query* query, const char* query_string, char* } bool result = true; - char* norm_query = NULL; + char *norm_query = NULL, *mask_str = NULL; int encoding = GetDatabaseEncoding(); int query_len; pgssJumbleState jstate; - errno_t rc; - rc = memset_s(&jstate, sizeof(jstate), 0, sizeof(jstate)); + errno_t rc = memset_s(&jstate, sizeof(jstate), 0, sizeof(jstate)); securec_check(rc, "\0", "\0"); query_len = strlen(query_string); @@ -221,6 +220,12 @@ bool normalized_unique_querystring(Query* query, const char* query_string, char* result = false; } } + } else { + mask_str = maskPassword(query_string); + if (mask_str != NULL) { + query_string = mask_str; + query_len = strlen(mask_str); + } } if (result) { @@ -235,6 +240,7 @@ bool normalized_unique_querystring(Query* query, const char* query_string, char* query_string = builtin_unique_sql->unique_sql; query_len = builtin_unique_sql->unique_sql_len; } + if (query_len > buf_len) { query_len = pg_encoding_mbcliplen(encoding, query_string, query_len, g_instance.attr.attr_common.pgstat_track_activity_query_size - 1); @@ -245,6 +251,7 @@ bool normalized_unique_querystring(Query* query, const char* query_string, char* } } + pfree_ext(mask_str); return result; } /* diff --git a/src/gausskernel/cbb/utils/partition/partitionmap.cpp b/src/gausskernel/cbb/utils/partition/partitionmap.cpp index 1f2989690..b5242fa97 100644 --- a/src/gausskernel/cbb/utils/partition/partitionmap.cpp +++ b/src/gausskernel/cbb/utils/partition/partitionmap.cpp @@ -936,7 +936,7 @@ void RebuildPartitonMap(PartitionMap* oldMap, PartitionMap* newMap) } } else { oldMap->isDirty = true; - SetLocalRelCacheNeedEOXactWork(true); + SetRelCacheNeedEOXActWork(true); elog(LOG, "map refcount is not zero when RebuildPartitonMap "); } } diff --git a/src/gausskernel/dbmind/CMakeLists.txt b/src/gausskernel/dbmind/CMakeLists.txt index 950b6928d..e1c8bec06 100755 --- a/src/gausskernel/dbmind/CMakeLists.txt +++ b/src/gausskernel/dbmind/CMakeLists.txt @@ -14,7 +14,7 @@ set(CMAKE_MODULE_PATH add_subdirectory(kernel) add_subdirectory(db4ai) -if(NOT "${ENABLE_LITE_MODE}" STREQUAL "ON") +if(NOT "${ENABLE_LITE_MODE}" STREQUAL "ON" AND "${ENABLE_MULTIPLE_NODES}" STREQUAL "OFF") install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tools/ DESTINATION bin/dbmind) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/gs_dbmind PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE diff --git a/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp b/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp index f4241344a..7abbe9171 100644 --- a/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/gd/pca.cpp @@ -257,8 +257,9 @@ static Datum pca_predict(const Matrix *features, const Matrix *weights, static HyperparameterDefinition pca_hyperparameter_definitions[] = { HYPERPARAMETER_INT4("number_components", 1, 1, true, INT32_MAX, true, HyperparametersGD, number_dimensions, HP_NO_AUTOML()), - HYPERPARAMETER_INT4("batch_size", 1000, 1, true, INT32_MAX, true, HyperparametersGD, batch_size, HP_NO_AUTOML()), - HYPERPARAMETER_INT4("max_iterations", 100, 1, true, INT32_MAX, true, HyperparametersGD, max_iterations, + HYPERPARAMETER_INT4("batch_size", 1000, 1, true, MAX_BATCH_SIZE, true, HyperparametersGD, batch_size, + HP_NO_AUTOML()), + HYPERPARAMETER_INT4("max_iterations", 100, 1, true, ITER_MAX, true, HyperparametersGD, max_iterations, HP_NO_AUTOML()), HYPERPARAMETER_INT4("max_seconds", 0, 0, true, INT32_MAX, true, HyperparametersGD, max_seconds, HP_NO_AUTOML()), HYPERPARAMETER_FLOAT8("tolerance", 0.0005, 0.0, true, DBL_MAX, true, HyperparametersGD, tolerance, HP_NO_AUTOML()), diff --git a/src/gausskernel/dbmind/db4ai/executor/kmeans/kmeans.cpp b/src/gausskernel/dbmind/db4ai/executor/kmeans/kmeans.cpp index 170b73c0e..0b8933966 100644 --- a/src/gausskernel/dbmind/db4ai/executor/kmeans/kmeans.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/kmeans/kmeans.cpp @@ -1041,10 +1041,11 @@ HyperparameterDefinition kmeans_hyperparameter_definitions[] = { HYPERPARAMETER_INT4("verbose", 0, 0, true, 2, true, HyperparametersKMeans, verbosity, HP_NO_AUTOML()), HYPERPARAMETER_INT4("num_centroids", 1, 1, true, 1000000, true, HyperparametersKMeans, num_centroids, HP_NO_AUTOML()), - HYPERPARAMETER_INT4("max_iterations", 10, 1, true, INT32_MAX, true, HyperparametersKMeans, num_iterations, + HYPERPARAMETER_INT4("max_iterations", 10, 1, true, ITER_MAX, true, HyperparametersKMeans, num_iterations, HP_NO_AUTOML()), HYPERPARAMETER_INT4("num_features", 0, 1, true, INT32_MAX, true, HyperparametersKMeans, n_features, HP_NO_AUTOML()), - HYPERPARAMETER_INT4("batch_size", 1000, 1, true, 1000000, true, HyperparametersKMeans, batch_size, HP_NO_AUTOML()), + HYPERPARAMETER_INT4("batch_size", 1000, 1, true, MAX_BATCH_SIZE, true, HyperparametersKMeans, batch_size, + HP_NO_AUTOML()), HYPERPARAMETER_INT4("seed", 0, 0, true, INT32_MAX, true, HyperparametersKMeans, external_seed, HP_AUTOML_INT(1, INT32_MAX, 1, ProbabilityDistribution::UNIFORM_RANGE)), HYPERPARAMETER_FLOAT8("tolerance", 0.00001, 0.0, false, 1.0, true, HyperparametersKMeans, tolerance, diff --git a/src/gausskernel/dbmind/db4ai/executor/xgboost/xgboost.cpp b/src/gausskernel/dbmind/db4ai/executor/xgboost/xgboost.cpp index 7f044417a..803a0cea4 100644 --- a/src/gausskernel/dbmind/db4ai/executor/xgboost/xgboost.cpp +++ b/src/gausskernel/dbmind/db4ai/executor/xgboost/xgboost.cpp @@ -80,8 +80,11 @@ static MemoryContext g_xgboostMcxt = NULL; #define safe_xgboost(call) { \ int err = (call); \ if (err != 0) { \ + char *str = const_cast(g_xgboostApi->XGBGetLastError()); \ + char *res = strchr(str, '\n'); \ + *res = '\0'; \ ereport(ERROR, (errmodule(MOD_DB4AI), errcode(ERRCODE_INVALID_PARAMETER_VALUE), \ - errmsg("%s:%d: error in %s: %s\n", __FILE__, __LINE__, #call, g_xgboostApi->XGBGetLastError()))); \ + errmsg("%s", strrchr(str, ':') + 1))); \ } \ } @@ -225,14 +228,15 @@ const char *xgboost_boost_str[] = {"gbtree", "gblinear", "dart"}; const char *xgboost_tree_method_str[] = {"auto", "exact", "approx", "hist", "gpu_hist"}; const char *xgboost_eval_metric_str[] = {"rmse", "rmsle", "map", "mae", "auc", "aucpr" }; static HyperparameterDefinition xgboost_hyperparameter_definitions[] = { - HYPERPARAMETER_INT4("n_iter", 10, 1, true, INT32_MAX, true, HyperparamsXGBoost, n_iterations, HP_NO_AUTOML()), - HYPERPARAMETER_INT4("batch_size", 10000, 1, true, INT32_MAX, true, HyperparamsXGBoost, batch_size, HP_NO_AUTOML()), + HYPERPARAMETER_INT4("n_iter", 10, 1, true, ITER_MAX, true, HyperparamsXGBoost, n_iterations, HP_NO_AUTOML()), + HYPERPARAMETER_INT4("batch_size", 10000, 1, true, MAX_BATCH_SIZE, true, HyperparamsXGBoost, batch_size, + HP_NO_AUTOML()), HYPERPARAMETER_INT4("max_depth", 5, 0, true, INT32_MAX, true, HyperparamsXGBoost, max_depth, HP_NO_AUTOML()), HYPERPARAMETER_INT4("min_child_weight", 1, 0, true, INT32_MAX, true, HyperparamsXGBoost, min_child_weight, HP_NO_AUTOML()), - HYPERPARAMETER_FLOAT8("gamma", 0.0, 0.0, true, 1, true, HyperparamsXGBoost, gamma, HP_NO_AUTOML()), + HYPERPARAMETER_FLOAT8("gamma", 0.0, 0.0, true, DBL_MAX, true, HyperparamsXGBoost, gamma, HP_NO_AUTOML()), HYPERPARAMETER_FLOAT8("eta", 0.3, 0.0, true, 1, true, HyperparamsXGBoost, eta, HP_NO_AUTOML()), - HYPERPARAMETER_INT4("nthread", 1, 0, true, INT32_MAX, true, HyperparamsXGBoost, nthread, HP_NO_AUTOML()), + HYPERPARAMETER_INT4("nthread", 1, 0, true, 100, true, HyperparamsXGBoost, nthread, HP_NO_AUTOML()), HYPERPARAMETER_INT4("verbosity", 1, 0, true, 3, true, HyperparamsXGBoost, verbosity, HP_NO_AUTOML()), HYPERPARAMETER_INT4("seed", 0, 0, true, INT32_MAX, true, HyperparamsXGBoost, seed, HP_AUTOML_INT(1, INT32_MAX, 1, ProbabilityDistribution::UNIFORM_RANGE)), @@ -548,12 +552,6 @@ static void xgboost_run(AlgorithmAPI *self, TrainModelState *pstate, Model **mod ModelTuple const *outer_tuple_slot = nullptr; - // Check max_depth parameter - if (xg_hyperp->max_depth == 0 && (0 != strcmp(xgboost_boost_str[BOOST_GBLINEAR_IDX], xg_hyperp->booster))) { - ereport(ERROR, (errmodule(MOD_DB4AI), - errmsg("Max_depth must be larger than 0 when booster is non_linear value."))); - } - load_xgboost_library(); // data holder for in-between (chunk-wise) invocation of XGBoost training algorithm diff --git a/src/gausskernel/dbmind/tools/__main__.py b/src/gausskernel/dbmind/tools/__main__.py index 73046464d..0343425fd 100644 --- a/src/gausskernel/dbmind/tools/__main__.py +++ b/src/gausskernel/dbmind/tools/__main__.py @@ -16,7 +16,7 @@ except ImportError: import sys import os - curr_path = os.path.dirname(os.path.abspath(__file__)) + curr_path = os.path.dirname(os.path.realpath(__file__)) root_path = os.path.dirname(curr_path) sys.path.append(root_path) from dbmind.cmd import main diff --git a/src/gausskernel/dbmind/tools/cmd/config_utils.py b/src/gausskernel/dbmind/tools/cmd/config_utils.py index 6fcd06247..e1c518186 100644 --- a/src/gausskernel/dbmind/tools/cmd/config_utils.py +++ b/src/gausskernel/dbmind/tools/cmd/config_utils.py @@ -11,25 +11,57 @@ # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. # See the Mulan PSL v2 for more details. import os -import re from configparser import ConfigParser from configparser import NoSectionError, NoOptionError from dbmind import constants from dbmind.common import security from dbmind.common.exceptions import InvalidPasswordException, ConfigSettingError -from dbmind.metadatabase.dao.dynamic_config import dynamic_config_get, dynamic_config_set from dbmind.common.utils import write_to_terminal +from dbmind.metadatabase.dao.dynamic_config import dynamic_config_get, dynamic_config_set + +DBMIND_CONF_HEADER = """\ +# Copyright (c) 2022 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +# Notice: +# 1. (null) explicitly represents empty or null. Meanwhile blank represents undefined. +# 2. DBMind encrypts password parameters. Hence, there is no plain-text password after initialization. +# 3. Users can only configure the plain-text password in this file before initializing +# (that is, using the --initialize option), +# and then if users want to modify the password-related information, +# users need to use the 'set' sub-command to achieve. +# 4. If users use relative path in this file, the current working directory is the directory where this file is located. +""" NULL_TYPE = '(null)' # empty text. ENCRYPTED_SIGNAL = 'Encrypted->' +# Used by check_config_validity(). +CONFIG_OPTIONS = { + 'TSDB-name': ['prometheus'], + 'METADATABASE-dbtype': ['sqlite', 'opengauss', 'postgresql'], + 'WORKER-type': ['local', 'dist'], + 'LOG-level': ['DEBUG', 'INFO', 'WARNING', 'ERROR'] +} -def check_config_validity(section, option, value, inline_comment=None): + +def check_config_validity(section, option, value): config_item = '%s-%s' % (section, option) # exceptional cases: if config_item == 'METADATABASE-port': return True, None + # normal inspection process: if 'port' in option: valid_port = str.isdigit(value) and 0 < int(value) <= 65535 @@ -38,16 +70,14 @@ def check_config_validity(section, option, value, inline_comment=None): if 'database' in option: if value == NULL_TYPE or value.strip() == '': return False, 'Unspecified database name' - if 'Options:' in inline_comment: - # determine setting option whether choose from option list. - results = re.findall(r'Options: (.*)?\.', inline_comment) - if len(results) > 0: - options = list(map(str.strip, results[0].split(','))) - if value not in options: - return False, 'Invalid choice: %s' % value + + options = CONFIG_OPTIONS.get(config_item) + if options and value not in options: + return False, 'Invalid choice: %s' % value + if 'dbtype' in option and value == 'opengauss': write_to_terminal( - 'WARN: default PostgresSQL connector (psycopg2-binary) does not support openGauss.\n' + 'WARN: default PostgreSQL connector (psycopg2-binary) does not support openGauss.\n' 'It would help if you compiled psycopg2 with openGauss manually or ' 'created a connection user after setting the GUC password_encryption_type to 1.', color='yellow' @@ -96,13 +126,16 @@ def load_sys_configs(confile): class ConfigUpdater: def __init__(self, filepath): self.config = ConfigParser(inline_comment_prefixes=None) - self.filepath = os.path.abspath(filepath) + self.filepath = os.path.realpath(filepath) self.fp = None self.readonly = True def get(self, section, option): value = self.config.get(section, option) - default_value, inline_comment = map(str.strip, value.rsplit('#', 1)) + try: + default_value, inline_comment = map(str.strip, value.rsplit('#', 1)) + except ValueError: + default_value, inline_comment = value.strip(), '' if default_value == '': default_value = NULL_TYPE return default_value, inline_comment @@ -132,10 +165,7 @@ class ConfigUpdater: # output configurations self.fp.truncate(0) self.fp.seek(0) - with open( - file=os.path.join(constants.MISC_PATH, constants.CONFILE_HEADER_NAME) - ) as header_fp: - self.fp.writelines(header_fp.readlines()) + self.fp.write(DBMIND_CONF_HEADER) self.config.write(self.fp) self.fp.flush() self.fp.close() @@ -163,7 +193,7 @@ def set_config_parameter(confpath, section: str, option: str, value: str): old_value, comment = config.get(section, option) except (NoSectionError, NoOptionError): raise ConfigSettingError('Not found the parameter %s-%s.' % (section, option)) - valid, reason = check_config_validity(section, option, value, comment) + valid, reason = check_config_validity(section, option, value) if not valid: raise ConfigSettingError('Incorrect value due to %s.' % reason) # If user wants to change password, we should encrypt the plain-text password first. diff --git a/src/gausskernel/dbmind/tools/cmd/edbmind.py b/src/gausskernel/dbmind/tools/cmd/edbmind.py index 95a5fbf1b..35b4f90a7 100644 --- a/src/gausskernel/dbmind/tools/cmd/edbmind.py +++ b/src/gausskernel/dbmind/tools/cmd/edbmind.py @@ -14,6 +14,7 @@ import logging import os +import threading import signal import sys import traceback @@ -55,7 +56,6 @@ def _check_confpath(confpath): def _process_clean(force=False): global_vars.worker.terminate(cancel_futures=force) TimedTaskManager.stop() - logging.shutdown() def signal_handler(signum, frame): @@ -67,7 +67,10 @@ def signal_handler(signum, frame): elif signum == signal.SIGUSR2: # used for debugging utils.write_to_terminal('Stack frames:', color='green') - traceback.print_stack(frame) + for th in threading.enumerate(): + print(th) + traceback.print_stack(sys._current_frames()[th.ident]) + print() elif signum == signal.SIGTERM: signal.signal(signal.SIGTERM, signal.SIG_IGN) logging.info('DBMind received exit signal.') @@ -87,7 +90,7 @@ class DBMindMain(Daemon): if not _check_confpath(confpath): raise SetupError("Invalid directory '%s', please set up first." % confpath) - self.confpath = os.path.abspath(confpath) + self.confpath = os.path.realpath(confpath) self.worker = None pid_file = os.path.join(confpath, constants.PIDFILE_NAME) diff --git a/src/gausskernel/dbmind/tools/cmd/setup.py b/src/gausskernel/dbmind/tools/cmd/setup.py index eecc14df4..b49f3f113 100644 --- a/src/gausskernel/dbmind/tools/cmd/setup.py +++ b/src/gausskernel/dbmind/tools/cmd/setup.py @@ -19,7 +19,7 @@ from dbmind import constants, global_vars from dbmind.cmd.config_utils import ( ConfigUpdater, check_config_validity, DynamicConfig, load_sys_configs, - NULL_TYPE, ENCRYPTED_SIGNAL + NULL_TYPE, ENCRYPTED_SIGNAL, DBMIND_CONF_HEADER ) from dbmind.cmd.edbmind import SKIP_LIST from dbmind.common import utils, security @@ -35,7 +35,7 @@ from dbmind.metadatabase.dao.dynamic_config import dynamic_config_set, dynamic_c def initialize_and_check_config(confpath, interactive=False): if not os.path.exists(confpath): raise SetupError('Not found the directory %s.' % confpath) - confpath = os.path.abspath(confpath) # in case of dir changed. + confpath = os.path.realpath(confpath) # in case of dir changed. os.chdir(confpath) dbmind_conf_path = os.path.join(confpath, constants.CONFILE_NAME) dynamic_config_path = os.path.join(confpath, constants.DYNAMIC_CONFIG) @@ -70,7 +70,7 @@ def initialize_and_check_config(confpath, interactive=False): for section, section_comment in config.sections(SKIP_LIST): for option, value, inline_comment in config.items(section): valid, invalid_reason = check_config_validity( - section, option, value, inline_comment + section, option, value ) if not valid: raise SetupError( @@ -151,7 +151,10 @@ def setup_directory_interactive(confpath): utils.write_to_terminal(section_comment, color='yellow') # Get each configuration item. for option, values in config.items(section): - default_value, inline_comment = map(str.strip, values.rsplit('#', 1)) + try: + default_value, inline_comment = map(str.strip, values.rsplit('#', 1)) + except ValueError: + default_value, inline_comment = values.strip(), '' # If not set default value, the default value is null. if default_value.strip() == '': default_value = NULL_TYPE @@ -170,7 +173,7 @@ def setup_directory_interactive(confpath): input_value = default_value valid, invalid_reason = check_config_validity( - section, option, input_value, inline_comment + section, option, input_value ) if not valid: utils.write_to_terminal( @@ -190,8 +193,7 @@ def setup_directory_interactive(confpath): # output configurations with open(file=config_dst, mode='w+') as fp: # Add header comments (including license and notice). - with open(file=os.path.join(confpath, constants.CONFILE_HEADER_NAME)) as header_fp: - fp.writelines(header_fp.readlines()) + fp.write(DBMIND_CONF_HEADER) config.write(fp) initialize_and_check_config(confpath, interactive=True) @@ -221,8 +223,7 @@ def setup_directory(confpath): with open(file=os.path.join(confpath, constants.CONFILE_NAME), mode='r+') as fp: old = fp.readlines() # Add header comments (including license and notice). - with open(file=os.path.join(constants.MISC_PATH, constants.CONFILE_HEADER_NAME)) as header_fp: - fp.seek(0) - fp.writelines(header_fp.readlines()) + fp.seek(0) + fp.write(DBMIND_CONF_HEADER) fp.writelines(old) utils.write_to_terminal("Configure directory '%s' has been created successfully." % confpath, color='green') diff --git a/src/gausskernel/dbmind/tools/common/daemon.py b/src/gausskernel/dbmind/tools/common/daemon.py index eaf6cfd41..1446765b2 100644 --- a/src/gausskernel/dbmind/tools/common/daemon.py +++ b/src/gausskernel/dbmind/tools/common/daemon.py @@ -63,7 +63,7 @@ class Daemon: RUNNING = 1 def __init__(self, pid_file, stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): - self.pid_file = os.path.abspath(pid_file) + self.pid_file = os.path.realpath(pid_file) self.stdin = stdin self.stdout = stdout self.stderr = stderr diff --git a/src/gausskernel/dbmind/tools/common/tsdb/prometheus_client.py b/src/gausskernel/dbmind/tools/common/tsdb/prometheus_client.py index a1d47fe97..6cc233ed3 100644 --- a/src/gausskernel/dbmind/tools/common/tsdb/prometheus_client.py +++ b/src/gausskernel/dbmind/tools/common/tsdb/prometheus_client.py @@ -293,3 +293,10 @@ class PrometheusClient(TsdbClient): "HTTP Status Code {} ({!r})".format(response.status_code, response.content) ) return _standardize(data) + + def timestamp(self): + seq = self.get_current_metric_value('prometheus_remote_storage_highest_timestamp_in_seconds') + if len(seq) == 0 or len(seq[0]) == 0: + return 0 + return seq[0].timestamps[0] + diff --git a/src/gausskernel/dbmind/tools/common/tsdb/tsdb_client.py b/src/gausskernel/dbmind/tools/common/tsdb/tsdb_client.py index 6a7c8c942..99f5e1168 100644 --- a/src/gausskernel/dbmind/tools/common/tsdb/tsdb_client.py +++ b/src/gausskernel/dbmind/tools/common/tsdb/tsdb_client.py @@ -11,6 +11,7 @@ # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. # See the Mulan PSL v2 for more details. +import time from datetime import datetime, timedelta @@ -48,3 +49,8 @@ class TsdbClient(object): params: dict = None): """get metric target from tsdb""" pass + + def timestamp(self): + """get the current unix-timestamp from the time-series database.""" + return int(time.time() * 1000) + diff --git a/src/gausskernel/dbmind/tools/common/tsdb/tsdb_client_factory.py b/src/gausskernel/dbmind/tools/common/tsdb/tsdb_client_factory.py index eac9b59a7..f4ba1f19d 100644 --- a/src/gausskernel/dbmind/tools/common/tsdb/tsdb_client_factory.py +++ b/src/gausskernel/dbmind/tools/common/tsdb/tsdb_client_factory.py @@ -12,6 +12,7 @@ # See the Mulan PSL v2 for more details. import threading +import time from dbmind.common.exceptions import ApiClientException from dbmind.common.tsdb.tsdb_client import TsdbClient @@ -46,3 +47,9 @@ class TsdbClientFactory(object): cls.tsdb_client = client if cls.tsdb_client is None: raise ApiClientException("Failed to init TSDB client, please check config file") + + if abs(cls.tsdb_client.timestamp() - time.time() * 1000) > 60 * 1000: # threshold is 1 minute. + raise ApiClientException('Found clock drift between TSDB client and server, ' + 'please check and synchronize system clocks.') + + diff --git a/src/gausskernel/dbmind/tools/common/utils.py b/src/gausskernel/dbmind/tools/common/utils.py index da5875c1f..acde5938d 100644 --- a/src/gausskernel/dbmind/tools/common/utils.py +++ b/src/gausskernel/dbmind/tools/common/utils.py @@ -153,25 +153,28 @@ class MultiProcessingRFHandler(RotatingFileHandler): self._queue = multiprocessing.Queue(-1) self._should_exit = False - self._receiv_thr = threading.Thread(target=self._receive) + self._receiv_thr = threading.Thread(target=self._receive, name='LoggingReceiverThread') + self._receiv_thr.daemon = True self._receiv_thr.start() def _receive(self): while True: try: - record = self._queue.get_nowait() + if self._should_exit and self._queue.empty(): + break + record = self._queue.get(timeout=.2) super().emit(record) - except Empty: - time.sleep(.1) except (KeyboardInterrupt, SystemExit): raise - except EOFError: + except (OSError, EOFError): break + except Empty: + pass except: traceback.print_exc(file=sys.stderr) - if self._should_exit and self._queue.empty(): - break - + self._queue.close() + self._queue.join_thread() + def _send(self, s): self._queue.put_nowait(s) @@ -181,6 +184,7 @@ class MultiProcessingRFHandler(RotatingFileHandler): record.msg = record.msg % record.args record.args = None if record.exc_info: + self.format(record) record.exc_info = None self._send(record) except (KeyboardInterrupt, SystemExit): @@ -189,8 +193,10 @@ class MultiProcessingRFHandler(RotatingFileHandler): self.handleError(record) def close(self): - super().close() - self._should_exit = True + if not self._should_exit: + self._should_exit = True + self._receiv_thr.join(5) + super().close() class ExceptionCatch: diff --git a/src/gausskernel/dbmind/tools/components/__init__.py b/src/gausskernel/dbmind/tools/components/__init__.py index 3daf8de60..b07f5b9c9 100644 --- a/src/gausskernel/dbmind/tools/components/__init__.py +++ b/src/gausskernel/dbmind/tools/components/__init__.py @@ -20,7 +20,7 @@ from dbmind.common.utils import where_am_i def list_components(): """Return all components in current directory.""" - curr_dir = os.path.abspath(os.path.dirname(__file__)) + curr_dir = os.path.realpath(os.path.dirname(__file__)) components = list( map(lambda tup: tup[1], pkgutil.iter_modules((curr_dir,))) diff --git a/src/gausskernel/dbmind/tools/components/forecast.py b/src/gausskernel/dbmind/tools/components/forecast.py index 74ae12ffa..aef2e7b38 100644 --- a/src/gausskernel/dbmind/tools/components/forecast.py +++ b/src/gausskernel/dbmind/tools/components/forecast.py @@ -57,7 +57,7 @@ def show(metric, host, start_time, end_time): for row_ in result: row = [str(getattr(row_, field)).strip() for field in field_names] csv_writer.writerow(row) - write_to_terminal('Dumped file is %s.' % os.path.abspath(dump_file_name)) + write_to_terminal('Dumped file is %s.' % os.path.realpath(dump_file_name)) elif char == 'N': print(output_table) print('(%d rows)' % nb_rows) diff --git a/src/gausskernel/dbmind/tools/components/index_advisor/__main__.py b/src/gausskernel/dbmind/tools/components/index_advisor/__main__.py new file mode 100644 index 000000000..3b09d8e49 --- /dev/null +++ b/src/gausskernel/dbmind/tools/components/index_advisor/__main__.py @@ -0,0 +1,24 @@ +# Copyright (c) 2022 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + +import sys +import os + +try: + from dbmind.components.index_advisor import main +except ImportError: + libpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') + sys.path.append(libpath) + from index_advisor import main + +main(sys.argv[1:]) diff --git a/src/gausskernel/dbmind/tools/components/index_advisor/index_advisor_workload.py b/src/gausskernel/dbmind/tools/components/index_advisor/index_advisor_workload.py index b2f8912a7..cbe136b2c 100644 --- a/src/gausskernel/dbmind/tools/components/index_advisor/index_advisor_workload.py +++ b/src/gausskernel/dbmind/tools/components/index_advisor/index_advisor_workload.py @@ -1,1018 +1,1019 @@ -""" -Copyright (c) 2020 Huawei Technologies Co.,Ltd. - -openGauss is licensed under Mulan PSL v2. -You can use this software according to the terms and conditions of the Mulan PSL v2. -You may obtain a copy of Mulan PSL v2 at: - - http://license.coscl.org.cn/MulanPSL2 - -THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, -EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, -MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. -See the Mulan PSL v2 for more details. -""" -import os -import sys -import argparse -import copy -import getpass -import random -import re -import json -import select -import logging - -try: - from .dao.gsql_execute import GSqlExecute - from .dao.execute_factory import ExecuteFactory -except ImportError: - from dao.gsql_execute import GSqlExecute - from dao.execute_factory import ExecuteFactory - -ENABLE_MULTI_NODE = False -SAMPLE_NUM = 5 -MAX_INDEX_COLUMN_NUM = 4 -MAX_INDEX_NUM = None -MAX_INDEX_STORAGE = None -FULL_ARRANGEMENT_THRESHOLD = 20 -NEGATIVE_RATIO_THRESHOLD = 0.2 -SHARP = '#' -JSON_TYPE = False -DRIVER = None -BLANK = ' ' -SQL_TYPE = ['select', 'delete', 'insert', 'update'] -SQL_PATTERN = [r'\((\s*(\d+(\.\d+)?\s*)[,]?)+\)', # match integer set in the IN collection - r'([^\\])\'((\')|(.*?([^\\])\'))', # match all content in single quotes - r'(([^<>]\s*=\s*)|([^<>]\s+))(\d+)(\.\d+)?'] # match single integer -SQL_DISPLAY_PATTERN = [r'\((\s*(\d+(\.\d+)?\s*)[,]?)+\)', # match integer set in the IN collection - r'\'((\')|(.*?\'))', # match all content in single quotes - r'([^\_\d])\d+(\.\d+)?'] # match single integer -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - - -class CheckValid(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - ill_character = [" ", "|", ";", "&", "$", "<", ">", "`", "\\", "'", "\"", - "{", "}", "(", ")", "[", "]", "~", "*", "?", "!", "\n"] - if not values.strip(): - return - if any(ill_char in values for ill_char in ill_character): - raise Exception( - "There are illegal characters in the %s." % self.dest) - setattr(namespace, self.dest, values) - - -def read_input_from_pipe(): - """ - Read stdin input if there is "echo 'str1 str2' | python xx.py", - return the input string - """ - input_str = "" - r_handle, _, _ = select.select([sys.stdin], [], [], 0) - if not r_handle: - return "" - - for item in r_handle: - if item == sys.stdin: - input_str = sys.stdin.read().strip() - return input_str - - -def get_password(): - password = read_input_from_pipe() - if password: - logging.warning("Read password from pipe.") - else: - password = getpass.getpass("Password for database user:") - if not password: - raise ValueError('Please input the password') - return password - - -class QueryItem: - def __init__(self, sql, freq): - self.statement = sql - self.frequency = freq - self.valid_index_list = [] - self.cost_list = [] - - -class IndexItem: - def __init__(self, tbl, cols, index_type=None): - self.table = tbl - self.columns = cols - self.atomic_pos = 0 - self.benefit = 0 - self.storage = 0 - self.positive_pos = [] - self.ineffective_pos = [] - self.negative_pos = [] - self.total_sql_num = 0 - self.insert_sql_num = 0 - self.update_sql_num = 0 - self.delete_sql_num = 0 - self.select_sql_num = 0 - self.index_type = index_type - self.is_candidate = False - - def __str__(self): - return f'{self.table} {self.columns} {self.index_type}' - - -def singleton(cls): - instances = {} - def _singleton(*args, **kwargs): - if not cls in instances: - instances[cls] = cls(*args, **kwargs) - return instances[cls] - return _singleton - - -@singleton -class IndexItemFactory: - def __init__(self): - self.indexes = {} - - def get_index(self, tbl, cols, index_type): - if not (tbl, tuple(cols), index_type) in self.indexes: - self.indexes[(tbl, tuple(cols), index_type)] = IndexItem(tbl, cols, index_type=index_type) - return self.indexes[(tbl, tuple(cols), index_type)] - - -class IndexAdvisor: - def __init__(self, db, workload_info, multi_iter_mode): - self.db = db - self.workload_info = workload_info - self.workload_used_index = set() - self.multi_iter_mode = multi_iter_mode - - self.determine_indexes = [] - self.integrate_indexes = {} - self.index_cost_total = [] - - self.display_detail_info = {} - - def retain_lower_cost_index(self, candidate_indexes): - remove_indexes = [] - for i in range(len(candidate_indexes)-1): - if candidate_indexes[i].table != candidate_indexes[i+1].table: - continue - if candidate_indexes[i].columns == candidate_indexes[i+1].columns: - if self.index_cost_total[candidate_indexes[i].atomic_pos] <= \ - self.index_cost_total[candidate_indexes[i+1].atomic_pos]: - remove_indexes.append(i+1) - else: - remove_indexes.append(i) - for index in remove_indexes[::-1]: - candidate_indexes.pop(index) - - def complex_index_advisor(self): - self.display_detail_info['workloadCount'] = self.workload_info[1] - print_header_boundary(" Generate candidate indexes ") - candidate_indexes = generate_candidate_indexes( - self.workload_info[0], self.db) - if DRIVER: - self.db.init_conn_handle() - if len(candidate_indexes) == 0: - print("No candidate indexes generated!") - self.db.estimate_workload_cost_file(self.workload_info[0], - ori_indexes_name=self.workload_used_index) - if DRIVER: - self.db.close_conn() - return None - - print_header_boundary(" Determine optimal indexes ") - atomic_config_total = generate_atomic_config(self.workload_info[0]) - if atomic_config_total and len(atomic_config_total[0]) != 0: - raise ValueError("The empty atomic config isn't generated!") - for atomic_config in atomic_config_total: - self.index_cost_total.append( - self.db.estimate_workload_cost_file(self.workload_info[0], atomic_config, - self.workload_used_index)) - if DRIVER: - self.db.close_conn() - - opt_config = greedy_determine_opt_config(self.workload_info[0], atomic_config_total, - candidate_indexes, self.index_cost_total[0]) - self.retain_lower_cost_index(candidate_indexes) - if len(opt_config) == 0: - print("No optimal indexes generated!") - return None - return opt_config - - def retain_high_benefit_index(self, candidate_indexes): - remove_indexes = [] - for i in range(len(candidate_indexes)-1): - candidate_indexes[i].cost_pos = i + 1 - if candidate_indexes[i].table != candidate_indexes[i+1].table: - continue - if candidate_indexes[i].columns == candidate_indexes[i+1].columns: - if candidate_indexes[i].benefit >= candidate_indexes[i+1].benefit: - remove_indexes.append(i+1) - else: - remove_indexes.append(i) - candidate_indexes[len(candidate_indexes)-1].cost_pos = len(candidate_indexes) - for index in remove_indexes[::-1]: - candidate_indexes.pop(index) - - def simple_index_advisor(self): - self.display_detail_info['workloadCount'] = self.workload_info[1] - print_header_boundary(" Generate candidate indexes ") - candidate_indexes = generate_candidate_indexes( - self.workload_info[0], self.db) - if DRIVER: - self.db.init_conn_handle() - if len(candidate_indexes) == 0: - print("No candidate indexes generated!") - self.db.estimate_workload_cost_file(self.workload_info[0], - ori_indexes_name=self.workload_used_index) - if DRIVER: - self.db.close_conn() - return None - - print_header_boundary(" Determine optimal indexes ") - ori_total_cost = \ - self.db.estimate_workload_cost_file(self.workload_info[0], - ori_indexes_name=self.workload_used_index) - self.index_cost_total.append(ori_total_cost) - for obj in candidate_indexes: - new_total_cost = self.db.estimate_workload_cost_file( - self.workload_info[0], [obj]) - obj.benefit = ori_total_cost - new_total_cost - self.index_cost_total.append(new_total_cost) - self.retain_high_benefit_index(candidate_indexes) - if DRIVER: - self.db.close_conn() - if len(self.index_cost_total) == 1: - print("No optimal indexes generated!") - return None - global MAX_INDEX_NUM - MAX_INDEX_NUM = MAX_INDEX_NUM or 10 - return candidate_indexes - - def filter_low_benefit_index(self, opt_indexes): - for key, index in enumerate(opt_indexes): - sql_optimzed = 0 - if self.multi_iter_mode: - cost_list_pos = index.atomic_pos - else: - cost_list_pos = index.cost_pos - # calculate the average benefit of each positive SQL - for pos in index.positive_pos: - sql_optimzed += 1 - self.workload_info[0][pos].cost_list[cost_list_pos] / \ - self.workload_info[0][pos].cost_list[0] - negative_sql_ratio = 0 - if index.total_sql_num: - negative_sql_ratio = (index.insert_sql_num + index.delete_sql_num + - index.update_sql_num) / index.total_sql_num - # filter the candidate indexes that do not meet the conditions of optimization - if not index.positive_pos: - continue - if sql_optimzed / len(index.positive_pos) < 0.1: - continue - if sql_optimzed / len(index.positive_pos) < \ - NEGATIVE_RATIO_THRESHOLD < negative_sql_ratio: - continue - self.determine_indexes.append(index) - - def record_info(self, index, sql_info, cost_list_pos, table_name, statement): - workload_optimized = (1 - self.index_cost_total[cost_list_pos] / - self.index_cost_total[0]) * 100 - sql_info['workloadOptimized'] = '%.2f' % \ - (workload_optimized if workload_optimized > 1 else 1) - sql_info['schemaName'] = index.table.split('.')[0] - sql_info['tbName'] = table_name - sql_info['columns'] = index.columns - sql_info['index_type'] = index.index_type - sql_info['statement'] = statement - sql_info['dmlCount'] = round(index.total_sql_num) - sql_info['selectRatio'] = 1 - sql_info['insertRatio'] = sql_info['deleteRatio'] = sql_info['updateRatio'] = 0 - if index.total_sql_num: - sql_info['selectRatio'] = round( - index.select_sql_num * 100 / index.total_sql_num, 2) - sql_info['insertRatio'] = round( - index.insert_sql_num * 100 / index.total_sql_num, 2) - sql_info['deleteRatio'] = round( - index.delete_sql_num * 100 / index.total_sql_num, 2) - sql_info['updateRatio'] = round(100 - sql_info['selectRatio'] - sql_info['insertRatio'] - - sql_info['deleteRatio'], 2) - self.display_detail_info['recommendIndexes'].append(sql_info) - - def computer_index_optimization_info(self, index, table_name, statement, opt_indexes): - if self.multi_iter_mode: - cost_list_pos = index.atomic_pos - else: - cost_list_pos = index.cost_pos - sql_info = {'sqlDetails': []} - benefit_types = [index.ineffective_pos, - index.positive_pos, index.negative_pos] - for category, sql_pos in enumerate(benefit_types): - sql_count = 0 - for item in sql_pos: - sql_count += self.workload_info[0][item].frequency - for pos in sql_pos: - sql_detail = {} - sql_template = self.workload_info[0][pos].statement - for pattern in SQL_DISPLAY_PATTERN: - sql_template = re.sub(pattern, '?', sql_template) - - sql_detail['sqlTemplate'] = sql_template - sql_detail['sql'] = self.workload_info[0][pos].statement - sql_detail['sqlCount'] = int(round(sql_count)) - if category == 1: - sql_optimzed = (self.workload_info[0][pos].cost_list[0] - - self.workload_info[0][pos].cost_list[cost_list_pos]) / \ - self.workload_info[0][pos].cost_list[cost_list_pos] - sql_detail['optimized'] = '%.3f' % sql_optimzed - sql_detail['correlationType'] = category - sql_info['sqlDetails'].append(sql_detail) - self.record_info(index, sql_info, cost_list_pos, table_name, statement) - - def display_advise_indexes_info(self, opt_indexes, show_detail): - index_current_storage = 0 - cnt = 0 - self.display_detail_info['recommendIndexes'] = [] - for key, index in enumerate(self.determine_indexes): - # constraints for Top-N algorithm - if MAX_INDEX_STORAGE and (index_current_storage + index.storage) > MAX_INDEX_STORAGE: - continue - if MAX_INDEX_NUM and cnt == MAX_INDEX_NUM: - break - if not self.multi_iter_mode and index.benefit <= 0: - continue - index_current_storage += index.storage - cnt += 1 - # display determine indexes - table_name = index.table.split('.')[-1] - index_name = 'idx_%s_%s%s' % (table_name, (index.index_type - + '_' if index.index_type else '') \ - ,'_'.join(index.columns.split(', '))) - statement = 'CREATE INDEX %s ON %s%s%s;' % (index_name, index.table, - '(' + index.columns + ')', - (' '+index.index_type if index.index_type else '')) - print(statement) - if show_detail: - # record detailed SQL optimization information for each index - self.computer_index_optimization_info( - index, table_name, statement, opt_indexes) - - def generate_incremental_index(self, history_advise_indexes): - self.integrate_indexes = copy.copy(history_advise_indexes) - self.integrate_indexes['currentIndexes'] = {} - for key, index in enumerate(self.determine_indexes): - self.integrate_indexes['currentIndexes'][index.table] = \ - self.integrate_indexes['currentIndexes'].get(index.table, []) - self.integrate_indexes['currentIndexes'][index.table].append( - (index.columns, index.index_type)) - - def generate_redundant_useless_indexes(self, history_invalid_indexes): - whole_indexes, redundant_indexes = get_whole_index(self.db, - self.integrate_indexes['historyIndexes'], - history_invalid_indexes, - self.display_detail_info) - display_useless_redundant_indexes(whole_indexes, redundant_indexes, - self.workload_used_index, self.display_detail_info) - - def display_incremental_index(self, history_invalid_indexes, - workload_file_path): - def rm_schema(table_name): - return table_name.split('.')[-1] - # display historical effective indexes - if self.integrate_indexes['historyIndexes']: - print_header_boundary(" Historical effective indexes ") - for table_name, index_list in self.integrate_indexes['historyIndexes'].items(): - for column in index_list: - index_name = 'idx_%s_%s%s' % (rm_schema(table_name), - (column[1] + '_' if column[1] else ''), - '_'.join(column[0].split(', '))) - statement = 'CREATE INDEX %s ON %s%s%s;' % (index_name, table_name, - '(' + column[0] + ')', (' ' + column[1] if column[1] else '')) - print(statement) - # display historical invalid indexes - if history_invalid_indexes: - print_header_boundary(" Historical invalid indexes ") - for table_name, index_list in history_invalid_indexes.items(): - for column in index_list: - index_name = 'idx_%s_%s%s' % (rm_schema(table_name), - (column[1] + '_' if column[1] else ''), - '_'.join(column[0].split(', '))) - statement = 'CREATE INDEX %s ON %s%s%s;' % (index_name, table_name, - '(' + column[0] + ')', (' ' + column[1] if column[1] else '')) - print(statement) - # save integrate indexes result - integrate_indexes_file = os.path.join(os.path.dirname(workload_file_path), - 'index_result.json') - for table, indexes in self.integrate_indexes['currentIndexes'].items(): - self.integrate_indexes['historyIndexes'][table] = \ - self.integrate_indexes['historyIndexes'].get(table, []) - self.integrate_indexes['historyIndexes'][table].extend(indexes) - self.integrate_indexes['historyIndexes'][table] = \ - list( - set(map(tuple, (self.integrate_indexes['historyIndexes'][table])))) - with open(integrate_indexes_file, 'w') as file: - json.dump(self.integrate_indexes['historyIndexes'], file) - - -def green(text): - return '\033[32m%s\033[0m' % text - - -def print_header_boundary(header): - # Output a header first, which looks more beautiful. - try: - term_width = os.get_terminal_size().columns - # The width of each of the two sides of the terminal. - side_width = (term_width - len(header)) // 2 - except (AttributeError, OSError): - side_width = 0 - title = SHARP * side_width + header + SHARP * side_width - print(green(title)) - - -def load_workload(file_path): - wd_dict = {} - workload = [] - global BLANK - with open(file_path, 'r') as file: - raw_text = ''.join(file.readlines()) - sqls = raw_text.split(';') - for sql in sqls: - if any(re.search(r'((\A|[\s\(,])%s[\s*\(])' % tp, sql.lower()) for tp in SQL_TYPE): - TWO_BLANKS = BLANK * 2 - while TWO_BLANKS in sql: - sql = sql.replace(TWO_BLANKS, BLANK) - if sql.strip() not in wd_dict.keys(): - wd_dict[sql.strip()] = 1 - else: - wd_dict[sql.strip()] += 1 - for sql, freq in wd_dict.items(): - workload.append(QueryItem(sql, freq)) - - return workload - - -def get_workload_template(workload): - templates = {} - placeholder = r'@@@' - - for item in workload: - sql_template = item.statement - for pattern in SQL_PATTERN: - sql_template = re.sub(pattern, placeholder, sql_template) - if sql_template not in templates: - templates[sql_template] = {} - templates[sql_template]['cnt'] = 0 - templates[sql_template]['samples'] = [] - templates[sql_template]['cnt'] += item.frequency - # reservoir sampling - if len(templates[sql_template]['samples']) < SAMPLE_NUM: - templates[sql_template]['samples'].append(item.statement) - else: - if random.randint(0, templates[sql_template]['cnt']) < SAMPLE_NUM: - templates[sql_template]['samples'][random.randint(0, SAMPLE_NUM - 1)] = \ - item.statement - - return templates - - -def workload_compression(input_path): - compressed_workload = [] - total_num = 0 - if JSON_TYPE: - with open(input_path, 'r') as file: - templates = json.load(file) - else: - workload = load_workload(input_path) - templates = get_workload_template(workload) - - for _, elem in templates.items(): - for sql in elem['samples']: - compressed_workload.append( - QueryItem(sql.strip(), elem['cnt'] / len(elem['samples']))) - total_num += elem['cnt'] - return compressed_workload, total_num - - -# enumerate the column combinations for a suggested index -def get_indexable_columns(table_index_dict): - query_indexable_columns = {} - if len(table_index_dict) == 0: - return query_indexable_columns - - for table in table_index_dict.keys(): - query_indexable_columns[table] = [] - for columns_tuple in table_index_dict[table]: - indexable_columns = columns_tuple[0].split(',') - for column in indexable_columns: - for ind, item in enumerate(query_indexable_columns[table]): - if column != item[0]: - continue - if columns_tuple[1] == item[1]: - query_indexable_columns[table].pop(ind) - break - query_indexable_columns[table].append( - (column, columns_tuple[1])) - - return query_indexable_columns - - -def add_column(valid_index_dict, table, columns_info, single_col_info): - columns, columns_index_type = columns_info - single_column, single_index_type = single_col_info - if columns_index_type.strip('"') != single_index_type.strip('"'): - add_column(valid_index_dict, table, (columns, 'local'), - (single_column, 'local')) - add_column(valid_index_dict, table, (columns, 'global'), - (single_column, 'global')) - else: - current_columns_tuple = ( - columns + ',' + single_column, columns_index_type) - if current_columns_tuple in valid_index_dict[table]: - return - if single_index_type == 'local': - global_columns_tuple = (columns + ',' + single_column, 'global') - if global_columns_tuple in valid_index_dict[table]: - global_pos = valid_index_dict[table].index( - global_columns_tuple) - valid_index_dict[table][global_pos] = current_columns_tuple - current_columns_tuple = global_columns_tuple - valid_index_dict[table].append(current_columns_tuple) - - -def get_valid_index_dict(table_index_dict, query, db): - need_check = False - query_indexable_columns = get_indexable_columns(table_index_dict) - valid_index_dict = db.query_index_check(query.statement, query_indexable_columns, - ENABLE_MULTI_NODE) - - for i in range(MAX_INDEX_COLUMN_NUM): - for table in valid_index_dict.keys(): - for columns, index_type in valid_index_dict[table]: - if columns.count(',') != i: - continue - need_check = True - for single_column, single_index_type in query_indexable_columns[table]: - if single_column not in columns: - add_column(valid_index_dict, table, (columns, index_type), - (single_column, single_index_type)) - if need_check: - valid_index_dict = db.query_index_check(query.statement, valid_index_dict, - ENABLE_MULTI_NODE) - need_check = False - else: - break - return valid_index_dict - - -def print_candidate_indexes(column_sqls, table, candidate_indexes): - if column_sqls[0][1]: - print("table: ", table, "columns: ",column_sqls[0][0], - "type: ", column_sqls[0][1]) - else: - print("table: ", table, "columns: ",column_sqls[0][0]) - if (table, tuple(column_sqls[0][0]), column_sqls[0][1]) not in IndexItemFactory().indexes: - index = IndexItemFactory().get_index(table, column_sqls[0][0], 'local') - index.index_type = 'global' - else: - index = IndexItemFactory().get_index(table, column_sqls[0][0], column_sqls[0][1]) - index.is_candidate = True - candidate_indexes.append(index) - - -def filter_redundant_indexes(index_dict): - candidate_indexes = [] - for table, column_sqls in index_dict.items(): - # sorted using index_type and columns - sorted_column_sqls = sorted( - column_sqls.items(), key=lambda item: (item[0][1], item[0][0])) - merged_column_sqls = [] - # merge sqls - for i in range(len(sorted_column_sqls) - 1): - if re.match(sorted_column_sqls[i][0][0] + ',', sorted_column_sqls[i+1][0][0]) and \ - sorted_column_sqls[i][0][1] == sorted_column_sqls[i+1][0][1]: - sorted_column_sqls[i+1][1].extend(sorted_column_sqls[i][1]) - else: - merged_column_sqls.append(sorted_column_sqls[i]) - else: - merged_column_sqls.append(sorted_column_sqls[-1]) - # sort using columns - merged_column_sqls.sort(key=lambda item: item[0][0]) - for i in range(len(merged_column_sqls)-1): - # same columns - if merged_column_sqls[i][0][0] == \ - merged_column_sqls[i+1][0][0]: - print_candidate_indexes(merged_column_sqls[i], - table, - candidate_indexes) - continue - # left match for the partation table - if re.match(merged_column_sqls[i][0][0] + ',', - merged_column_sqls[i+1][0][0]): - merged_column_sqls[i+1][1].extend( - merged_column_sqls[i][1]) - merged_column_sqls[i+1] = ((merged_column_sqls[i+1][0][0], 'global'), - merged_column_sqls[i+1][1]) - continue - print_candidate_indexes(merged_column_sqls[i], table, candidate_indexes) - else: - print_candidate_indexes(merged_column_sqls[-1], table, candidate_indexes) - return candidate_indexes - - -def filter_duplicate_indexes(valid_index_dict, index_dict, workload, pos): - for table in valid_index_dict.keys(): - if table not in index_dict.keys(): - index_dict[table] = {} - valid_index_dict[table].sort(key=lambda x: -len(x[0])) - for columns, index_type in valid_index_dict[table]: - if len(workload[pos].valid_index_list) >= FULL_ARRANGEMENT_THRESHOLD: - break - if (columns, index_type) in index_dict[table]: - index_dict[table][(columns, index_type)].append(pos) - else: - column_sql = {(columns, index_type): [pos]} - index_dict[table].update(column_sql) - workload[pos].valid_index_list.append( - IndexItemFactory().get_index(table, columns, index_type=index_type)) - - -def generate_candidate_indexes(workload, db): - index_dict = {} - if DRIVER: - db.init_conn_handle() - for k, query in enumerate(workload): - if not re.search(r'(\A|\s)select\s', query.statement.lower()): - continue - table_index_dict = db.query_index_advisor(query.statement) - valid_index_dict = get_valid_index_dict(table_index_dict, query, db) - # filter duplicate indexes - filter_duplicate_indexes(valid_index_dict, index_dict, workload, k) - - # filter redundant indexes - candidate_indexes = filter_redundant_indexes(index_dict) - if DRIVER: - db.close_conn() - return candidate_indexes - - -def get_atomic_config_for_query(indexes, config, ind, atomic_configs): - if ind == len(indexes): - table_count = {} - for index in config: - if index.table not in table_count.keys(): - table_count[index.table] = 1 - else: - table_count[index.table] += 1 - if len(table_count) > 2 or table_count[index.table] > 2: - return - atomic_configs.append(config) - - return - - get_atomic_config_for_query( - indexes, copy.copy(config), ind + 1, atomic_configs) - config.append(indexes[ind]) - get_atomic_config_for_query( - indexes, copy.copy(config), ind + 1, atomic_configs) - - -def is_same_config(config1, config2): - if len(config1) != len(config2): - return False - - for index1 in config1: - is_found = False - for index2 in config2: - if index1.table == index2.table and index1.columns == index2.columns \ - and index1.index_type == index2.index_type: - is_found = True - if not is_found: - return False - - return True - - -def generate_atomic_config(workload): - atomic_config_total = [] - - for query in workload: - if len(query.valid_index_list) == 0: - continue - - atomic_configs = [] - atomic_config = [] - get_atomic_config_for_query( - query.valid_index_list, atomic_config, 0, atomic_configs) - - is_found = False - for new_config in atomic_configs: - for exist_config in atomic_config_total: - if is_same_config(new_config, exist_config): - is_found = True - break - if not is_found: - atomic_config_total.append(new_config) - is_found = False - - return atomic_config_total - - -def atomic_config_is_valid(atomic_config, config): - is_exist = False - is_same = False - if len(atomic_config) == 1: - is_same = (config[-1] is atomic_config[0]) - for atomic_index in atomic_config: - is_exist = False - for index in config: - if index is atomic_index: - index.storage = atomic_index.storage - is_exist = True - break - if not is_exist: - break - return is_exist, is_same - - -# find the subsets of a given config in the atomic configs -def find_subsets_num(config, atomic_config_total): - atomic_subsets_num = [] - cur_index_atomic_pos = -1 - for i, atomic_config in enumerate(atomic_config_total): - if len(atomic_config) > len(config): - continue - is_exist, is_same = atomic_config_is_valid(atomic_config, config) - if is_same: - cur_index_atomic_pos = i - if is_exist: - atomic_subsets_num.append(i) - if cur_index_atomic_pos == -1: - raise ValueError("No atomic configs found for current config!") - return atomic_subsets_num, cur_index_atomic_pos - - -# infer the total cost of workload for a config according to the cost of atomic configs -def infer_workload_cost(workload, config, atomic_config_total): - total_cost = 0 - is_computed = False - atomic_subsets_num, cur_index_atomic_pos = find_subsets_num( - config, atomic_config_total) - if len(atomic_subsets_num) == 0: - raise ValueError("No atomic configs found for current config!") - if not config[-1].total_sql_num: - is_computed = True - for ind, obj in enumerate(workload): - if max(atomic_subsets_num) >= len(obj.cost_list): - raise ValueError("Wrong atomic config for current query!") - # compute the cost for selection - min_cost = obj.cost_list[0] - for num in atomic_subsets_num: - if num < len(obj.cost_list) and obj.cost_list[num] < min_cost: - min_cost = obj.cost_list[num] - total_cost += min_cost - - # record ineffective sql and negative sql for candidate indexes - if is_computed: - ExecuteFactory.record_ineffective_negative_sql( - config[-1], obj, ind) - return total_cost, cur_index_atomic_pos - - -def get_whole_index(db, history_indexes, history_invalid_index, detail_info): - if DRIVER: - db.init_conn_handle() - whole_index, redundant_indexes = \ - db.check_useless_index(history_indexes, history_invalid_index) - if DRIVER: - db.close_conn() - print_header_boundary(" Created indexes ") - detail_info['createdIndexes'] = [] - if not whole_index: - print("No created index!") - else: - for index in whole_index: - index_info = {'schemaName': index.schema, 'tbName': index.table, - 'columns': index.columns, 'statement': index.indexdef + ';'} - detail_info['createdIndexes'].append(index_info) - print("%s: %s;" % (index.schema, index.indexdef)) - return whole_index, redundant_indexes - - -def display_redundant_indexes(redundant_indexes, unused_index_columns, remove_list, detail_info): - for pos, index in enumerate(redundant_indexes): - is_redundant = False - for redundant_obj in index.redundant_obj: - # redundant objects are not in the useless index set or - # both redundant objects and redundant index in useless index must be redundant index - index_exist = redundant_obj.indexname not in unused_index_columns.keys() or \ - (unused_index_columns.get(redundant_obj.indexname) and - unused_index_columns.get(index.indexname)) - if index_exist: - is_redundant = True - if not is_redundant: - remove_list.append(pos) - for item in sorted(remove_list, reverse=True): - redundant_indexes.pop(item) - - if not redundant_indexes: - print("No redundant index!") - # redundant index - for index in redundant_indexes: - statement = "DROP INDEX %s.%s;" % \ - (index.schema, index.indexname) - print(statement) - existing_index = [item.indexname + ':' + - item.columns for item in index.redundant_obj] - redundant_index = {"schemaName": index.schema, "tbName": index.table, "type": 2, - "columns": index.columns, "statement": statement, - "existingIndex": existing_index} - detail_info['uselessIndexes'].append(redundant_index) - - -def display_useless_redundant_indexes(whole_indexes, redundant_indexes, - workload_indexes, detail_info): - indexes_name = set(index.indexname for index in whole_indexes) - unused_index = list(indexes_name.difference(workload_indexes)) - remove_list = [] - print_header_boundary(" Current workload useless indexes ") - detail_info['uselessIndexes'] = [] - # useless index not contain unique index - unused_index_columns = {} - has_unused_index = False - for cur_index in unused_index: - for index in whole_indexes: - if cur_index != index.indexname: - continue - # get useless index details from whole index - unused_index_columns[cur_index] = index.columns - if 'UNIQUE INDEX' not in index.indexdef: - has_unused_index = True - statement = "DROP INDEX %s;" % index.indexname - print(statement) - useless_index = {"schemaName": index.schema, "tbName": index.table, "type": 3, - "columns": index.columns, "statement": statement} - detail_info['uselessIndexes'].append(useless_index) - if not has_unused_index: - print("No useless index!") - print_header_boundary(" Redundant indexes ") - # filter redundant index - display_redundant_indexes( - redundant_indexes, unused_index_columns, remove_list, detail_info) - - -def greedy_determine_opt_config(workload, atomic_config_total, candidate_indexes, origin_sum_cost): - opt_config = [] - index_num_record = set() - min_cost = origin_sum_cost - for i in range(len(candidate_indexes)): - if i == 1 and min_cost == origin_sum_cost: - break - cur_min_cost = origin_sum_cost - cur_index = None - cur_index_num = -1 - for k, index in enumerate(candidate_indexes): - if k in index_num_record: - continue - cur_config = copy.copy(opt_config) - cur_config.append(index) - cur_estimated_cost, cur_index_atomic_pos = \ - infer_workload_cost(workload, cur_config, atomic_config_total) - if cur_estimated_cost < cur_min_cost: - cur_min_cost = cur_estimated_cost - cur_index = index - cur_index.atomic_pos = cur_index_atomic_pos - cur_index_num = k - if cur_index and cur_min_cost < min_cost: - if MAX_INDEX_STORAGE and sum([obj.storage for obj in opt_config]) + \ - cur_index.storage > MAX_INDEX_STORAGE: - candidate_indexes.remove(cur_index) - continue - if len(opt_config) == MAX_INDEX_NUM: - break - min_cost = cur_min_cost - opt_config.append(cur_index) - index_num_record.add(cur_index_num) - else: - break - - return opt_config - - -def get_last_indexes_result(input_path): - last_indexes_result_file = os.path.join( - os.path.dirname(input_path), 'index_result.json') - integrate_indexes = {'historyIndexes': {}} - if os.path.exists(last_indexes_result_file): - try: - with open(last_indexes_result_file, 'r') as file: - integrate_indexes['historyIndexes'] = json.load(file) - except json.JSONDecodeError: - return integrate_indexes - return integrate_indexes - - -def index_advisor_workload(history_advise_indexes, db, workload_file_path, - multi_iter_mode, show_detail): - workload_info = workload_compression(workload_file_path) - index_advisor = IndexAdvisor(db, workload_info, multi_iter_mode) - if multi_iter_mode: - opt_indexes = index_advisor.complex_index_advisor() - else: - opt_indexes = index_advisor.simple_index_advisor() - if opt_indexes: - index_advisor.filter_low_benefit_index(opt_indexes) - index_advisor.display_advise_indexes_info(opt_indexes, show_detail) - - index_advisor.generate_incremental_index(history_advise_indexes) - history_invalid_indexes = {} - index_advisor.generate_redundant_useless_indexes(history_invalid_indexes) - index_advisor.display_incremental_index( - history_invalid_indexes, workload_file_path) - if show_detail: - print_header_boundary(" Display detail information ") - sql_info = json.dumps( - index_advisor.display_detail_info, indent=4, separators=(',', ':')) - print(sql_info) - - -def check_parameter(args): - global MAX_INDEX_NUM, ENABLE_MULTI_NODE, MAX_INDEX_STORAGE, JSON_TYPE, DRIVER - if args.max_index_num is not None and args.max_index_num <= 0: - raise argparse.ArgumentTypeError("%s is an invalid positive int value" % - args.max_index_num) - if args.max_index_storage is not None and args.max_index_storage <= 0: - raise argparse.ArgumentTypeError("%s is an invalid positive int value" % - args.max_index_storage) - JSON_TYPE = args.json - MAX_INDEX_NUM = args.max_index_num - ENABLE_MULTI_NODE = args.multi_node - MAX_INDEX_STORAGE = args.max_index_storage - if args.U and args.U != getpass.getuser() and not args.W: - raise ValueError('Enter the \'-W\' parameter for user ' - + args.U + ' when executing the script.') - - -def main(argv): - arg_parser = argparse.ArgumentParser( - description='Generate index set for workload.') - arg_parser.add_argument("p", help="Port of database", type=int) - arg_parser.add_argument("d", help="Name of database", action=CheckValid) - arg_parser.add_argument( - "--h", help="Host for database", action=CheckValid) - arg_parser.add_argument( - "-U", help="Username for database log-in", action=CheckValid) - arg_parser.add_argument( - "f", help="File containing workload queries (One query per line)") - arg_parser.add_argument("--schema", help="Schema name for the current business data", - required=True, action=CheckValid) - arg_parser.add_argument( - "--max_index_num", help="Maximum number of suggested indexes", type=int) - arg_parser.add_argument("--max_index_storage", - help="Maximum storage of suggested indexes/MB", type=int) - arg_parser.add_argument("--multi_iter_mode", action='store_true', - help="Whether to use multi-iteration algorithm", default=False) - arg_parser.add_argument("--multi_node", action='store_true', - help="Whether to support distributed scenarios", default=False) - arg_parser.add_argument("--json", action='store_true', - help="Whether the workload file format is json", default=False) - arg_parser.add_argument("--driver", action='store_true', - help="Whether to employ python-driver", default=False) - arg_parser.add_argument("--show_detail", action='store_true', - help="Whether to show detailed sql information", default=False) - args = arg_parser.parse_args(argv) - - args.W = get_password() - check_parameter(args) - # Initialize the connection - global DRIVER - if args.driver: - try: - import psycopg2 - try: - from .dao.driver_execute import DriverExecute - except ImportError: - from dao.driver_execute import DriverExecute - db = DriverExecute(args.d, args.U, args.W, args.h, args.p, args.schema, - args.multi_node, args.max_index_storage) - except ImportError: - logging.warning('Python driver import failed, ' - 'the gsql mode will be selected to connect to the database.') - db = GSqlExecute(args.d, args.U, args.W, args.h, args.p, args.schema, - args.multi_node, args.max_index_storage) - db.init_conn_handle() - args.driver = None - else: - db = GSqlExecute(args.d, args.U, args.W, args.h, args.p, args.schema, - args.multi_node, args.max_index_storage) - db.init_conn_handle() - DRIVER = args.driver - if args.multi_node and not db.is_multi_node(): - raise argparse.ArgumentTypeError('--multi_node is only support for distributed database') - index_advisor_workload(get_last_indexes_result(args.f), db, args.f, - args.multi_iter_mode, args.show_detail) - - -if __name__ == '__main__': - main(sys.argv[1:]) +""" +Copyright (c) 2020 Huawei Technologies Co.,Ltd. + +openGauss is licensed under Mulan PSL v2. +You can use this software according to the terms and conditions of the Mulan PSL v2. +You may obtain a copy of Mulan PSL v2 at: + + http://license.coscl.org.cn/MulanPSL2 + +THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +See the Mulan PSL v2 for more details. +""" +import os +import sys +import argparse +import copy +import getpass +import random +import re +import json +import select +import logging + +try: + from .dao.gsql_execute import GSqlExecute + from .dao.execute_factory import ExecuteFactory +except ImportError: + from dao.gsql_execute import GSqlExecute + from dao.execute_factory import ExecuteFactory + +ENABLE_MULTI_NODE = False +SAMPLE_NUM = 5 +MAX_INDEX_COLUMN_NUM = 4 +MAX_INDEX_NUM = None +MAX_INDEX_STORAGE = None +FULL_ARRANGEMENT_THRESHOLD = 20 +NEGATIVE_RATIO_THRESHOLD = 0.2 +SHARP = '#' +JSON_TYPE = False +DRIVER = None +BLANK = ' ' +SQL_TYPE = ['select', 'delete', 'insert', 'update'] +SQL_PATTERN = [r'\((\s*(\d+(\.\d+)?\s*)[,]?)+\)', # match integer set in the IN collection + r'([^\\])\'((\')|(.*?([^\\])\'))', # match all content in single quotes + r'(([^<>]\s*=\s*)|([^<>]\s+))(\d+)(\.\d+)?'] # match single integer +SQL_DISPLAY_PATTERN = [r'\((\s*(\d+(\.\d+)?\s*)[,]?)+\)', # match integer set in the IN collection + r'\'((\')|(.*?\'))', # match all content in single quotes + r'([^\_\d])\d+(\.\d+)?'] # match single integer +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + + +class CheckValid(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + ill_character = [" ", "|", ";", "&", "$", "<", ">", "`", "\\", "'", "\"", + "{", "}", "(", ")", "[", "]", "~", "*", "?", "!", "\n"] + if not values.strip(): + return + if any(ill_char in values for ill_char in ill_character): + raise Exception( + "There are illegal characters in the %s." % self.dest) + setattr(namespace, self.dest, values) + + +def read_input_from_pipe(): + """ + Read stdin input if there is "echo 'str1 str2' | python xx.py", + return the input string + """ + input_str = "" + r_handle, _, _ = select.select([sys.stdin], [], [], 0) + if not r_handle: + return "" + + for item in r_handle: + if item == sys.stdin: + input_str = sys.stdin.read().strip() + return input_str + + +def get_password(): + password = read_input_from_pipe() + if password: + logging.warning("Read password from pipe.") + else: + password = getpass.getpass("Password for database user:") + if not password: + raise ValueError('Please input the password') + return password + + +class QueryItem: + def __init__(self, sql, freq): + self.statement = sql + self.frequency = freq + self.valid_index_list = [] + self.cost_list = [] + + +class IndexItem: + def __init__(self, tbl, cols, index_type=None): + self.table = tbl + self.columns = cols + self.atomic_pos = 0 + self.benefit = 0 + self.storage = 0 + self.positive_pos = [] + self.ineffective_pos = [] + self.negative_pos = [] + self.total_sql_num = 0 + self.insert_sql_num = 0 + self.update_sql_num = 0 + self.delete_sql_num = 0 + self.select_sql_num = 0 + self.index_type = index_type + self.is_candidate = False + + def __str__(self): + return f'{self.table} {self.columns} {self.index_type}' + + +def singleton(cls): + instances = {} + def _singleton(*args, **kwargs): + if not cls in instances: + instances[cls] = cls(*args, **kwargs) + return instances[cls] + return _singleton + + +@singleton +class IndexItemFactory: + def __init__(self): + self.indexes = {} + + def get_index(self, tbl, cols, index_type): + if not (tbl, tuple(cols), index_type) in self.indexes: + self.indexes[(tbl, tuple(cols), index_type)] = IndexItem(tbl, cols, index_type=index_type) + return self.indexes[(tbl, tuple(cols), index_type)] + + +class IndexAdvisor: + def __init__(self, db, workload_info, multi_iter_mode): + self.db = db + self.workload_info = workload_info + self.workload_used_index = set() + self.multi_iter_mode = multi_iter_mode + + self.determine_indexes = [] + self.integrate_indexes = {} + self.index_cost_total = [] + + self.display_detail_info = {} + + def retain_lower_cost_index(self, candidate_indexes): + remove_indexes = [] + for i in range(len(candidate_indexes)-1): + if candidate_indexes[i].table != candidate_indexes[i+1].table: + continue + if candidate_indexes[i].columns == candidate_indexes[i+1].columns: + if self.index_cost_total[candidate_indexes[i].atomic_pos] <= \ + self.index_cost_total[candidate_indexes[i+1].atomic_pos]: + remove_indexes.append(i+1) + else: + remove_indexes.append(i) + for index in remove_indexes[::-1]: + candidate_indexes.pop(index) + + def complex_index_advisor(self): + self.display_detail_info['workloadCount'] = self.workload_info[1] + print_header_boundary(" Generate candidate indexes ") + candidate_indexes = generate_candidate_indexes( + self.workload_info[0], self.db) + if DRIVER: + self.db.init_conn_handle() + if len(candidate_indexes) == 0: + print("No candidate indexes generated!") + self.db.estimate_workload_cost_file(self.workload_info[0], + ori_indexes_name=self.workload_used_index) + if DRIVER: + self.db.close_conn() + return None + + print_header_boundary(" Determine optimal indexes ") + atomic_config_total = generate_atomic_config(self.workload_info[0]) + if atomic_config_total and len(atomic_config_total[0]) != 0: + raise ValueError("The empty atomic config isn't generated!") + for atomic_config in atomic_config_total: + self.index_cost_total.append( + self.db.estimate_workload_cost_file(self.workload_info[0], atomic_config, + self.workload_used_index)) + if DRIVER: + self.db.close_conn() + + opt_config = greedy_determine_opt_config(self.workload_info[0], atomic_config_total, + candidate_indexes, self.index_cost_total[0]) + self.retain_lower_cost_index(candidate_indexes) + if len(opt_config) == 0: + print("No optimal indexes generated!") + return None + return opt_config + + def retain_high_benefit_index(self, candidate_indexes): + remove_indexes = [] + for i in range(len(candidate_indexes)-1): + candidate_indexes[i].cost_pos = i + 1 + if candidate_indexes[i].table != candidate_indexes[i+1].table: + continue + if candidate_indexes[i].columns == candidate_indexes[i+1].columns: + if candidate_indexes[i].benefit >= candidate_indexes[i+1].benefit: + remove_indexes.append(i+1) + else: + remove_indexes.append(i) + candidate_indexes[len(candidate_indexes)-1].cost_pos = len(candidate_indexes) + for index in remove_indexes[::-1]: + candidate_indexes.pop(index) + + def simple_index_advisor(self): + self.display_detail_info['workloadCount'] = self.workload_info[1] + print_header_boundary(" Generate candidate indexes ") + candidate_indexes = generate_candidate_indexes( + self.workload_info[0], self.db) + if DRIVER: + self.db.init_conn_handle() + if len(candidate_indexes) == 0: + print("No candidate indexes generated!") + self.db.estimate_workload_cost_file(self.workload_info[0], + ori_indexes_name=self.workload_used_index) + if DRIVER: + self.db.close_conn() + return None + + print_header_boundary(" Determine optimal indexes ") + ori_total_cost = \ + self.db.estimate_workload_cost_file(self.workload_info[0], + ori_indexes_name=self.workload_used_index) + self.index_cost_total.append(ori_total_cost) + for obj in candidate_indexes: + new_total_cost = self.db.estimate_workload_cost_file( + self.workload_info[0], [obj]) + obj.benefit = ori_total_cost - new_total_cost + self.index_cost_total.append(new_total_cost) + self.retain_high_benefit_index(candidate_indexes) + if DRIVER: + self.db.close_conn() + if len(self.index_cost_total) == 1: + print("No optimal indexes generated!") + return None + global MAX_INDEX_NUM + MAX_INDEX_NUM = MAX_INDEX_NUM or 10 + return candidate_indexes + + def filter_low_benefit_index(self, opt_indexes): + for key, index in enumerate(opt_indexes): + sql_optimzed = 0 + if self.multi_iter_mode: + cost_list_pos = index.atomic_pos + else: + cost_list_pos = index.cost_pos + # calculate the average benefit of each positive SQL + for pos in index.positive_pos: + sql_optimzed += 1 - self.workload_info[0][pos].cost_list[cost_list_pos] / \ + self.workload_info[0][pos].cost_list[0] + negative_sql_ratio = 0 + if index.total_sql_num: + negative_sql_ratio = (index.insert_sql_num + index.delete_sql_num + + index.update_sql_num) / index.total_sql_num + # filter the candidate indexes that do not meet the conditions of optimization + if not index.positive_pos: + continue + if sql_optimzed / len(index.positive_pos) < 0.1: + continue + if sql_optimzed / len(index.positive_pos) < \ + NEGATIVE_RATIO_THRESHOLD < negative_sql_ratio: + continue + self.determine_indexes.append(index) + + def record_info(self, index, sql_info, cost_list_pos, table_name, statement): + workload_optimized = (1 - self.index_cost_total[cost_list_pos] / + self.index_cost_total[0]) * 100 + sql_info['workloadOptimized'] = '%.2f' % \ + (workload_optimized if workload_optimized > 1 else 1) + sql_info['schemaName'] = index.table.split('.')[0] + sql_info['tbName'] = table_name + sql_info['columns'] = index.columns + sql_info['index_type'] = index.index_type + sql_info['statement'] = statement + sql_info['dmlCount'] = round(index.total_sql_num) + sql_info['selectRatio'] = 1 + sql_info['insertRatio'] = sql_info['deleteRatio'] = sql_info['updateRatio'] = 0 + if index.total_sql_num: + sql_info['selectRatio'] = round( + index.select_sql_num * 100 / index.total_sql_num, 2) + sql_info['insertRatio'] = round( + index.insert_sql_num * 100 / index.total_sql_num, 2) + sql_info['deleteRatio'] = round( + index.delete_sql_num * 100 / index.total_sql_num, 2) + sql_info['updateRatio'] = round(100 - sql_info['selectRatio'] - sql_info['insertRatio'] + - sql_info['deleteRatio'], 2) + self.display_detail_info['recommendIndexes'].append(sql_info) + + def computer_index_optimization_info(self, index, table_name, statement, opt_indexes): + if self.multi_iter_mode: + cost_list_pos = index.atomic_pos + else: + cost_list_pos = index.cost_pos + sql_info = {'sqlDetails': []} + benefit_types = [index.ineffective_pos, + index.positive_pos, index.negative_pos] + for category, sql_pos in enumerate(benefit_types): + sql_count = 0 + for item in sql_pos: + sql_count += self.workload_info[0][item].frequency + for pos in sql_pos: + sql_detail = {} + sql_template = self.workload_info[0][pos].statement + for pattern in SQL_DISPLAY_PATTERN: + sql_template = re.sub(pattern, '?', sql_template) + + sql_detail['sqlTemplate'] = sql_template + sql_detail['sql'] = self.workload_info[0][pos].statement + sql_detail['sqlCount'] = int(round(sql_count)) + if category == 1: + sql_optimzed = (self.workload_info[0][pos].cost_list[0] - + self.workload_info[0][pos].cost_list[cost_list_pos]) / \ + self.workload_info[0][pos].cost_list[cost_list_pos] + sql_detail['optimized'] = '%.3f' % sql_optimzed + sql_detail['correlationType'] = category + sql_info['sqlDetails'].append(sql_detail) + self.record_info(index, sql_info, cost_list_pos, table_name, statement) + + def display_advise_indexes_info(self, opt_indexes, show_detail): + index_current_storage = 0 + cnt = 0 + self.display_detail_info['recommendIndexes'] = [] + for key, index in enumerate(self.determine_indexes): + # constraints for Top-N algorithm + if MAX_INDEX_STORAGE and (index_current_storage + index.storage) > MAX_INDEX_STORAGE: + continue + if MAX_INDEX_NUM and cnt == MAX_INDEX_NUM: + break + if not self.multi_iter_mode and index.benefit <= 0: + continue + index_current_storage += index.storage + cnt += 1 + # display determine indexes + table_name = index.table.split('.')[-1] + index_name = 'idx_%s_%s%s' % (table_name, (index.index_type + + '_' if index.index_type else '') \ + ,'_'.join(index.columns.split(', '))) + statement = 'CREATE INDEX %s ON %s%s%s;' % (index_name, index.table, + '(' + index.columns + ')', + (' '+index.index_type if index.index_type else '')) + print(statement) + if show_detail: + # record detailed SQL optimization information for each index + self.computer_index_optimization_info( + index, table_name, statement, opt_indexes) + + def generate_incremental_index(self, history_advise_indexes): + self.integrate_indexes = copy.copy(history_advise_indexes) + self.integrate_indexes['currentIndexes'] = {} + for key, index in enumerate(self.determine_indexes): + self.integrate_indexes['currentIndexes'][index.table] = \ + self.integrate_indexes['currentIndexes'].get(index.table, []) + self.integrate_indexes['currentIndexes'][index.table].append( + (index.columns, index.index_type)) + + def generate_redundant_useless_indexes(self, history_invalid_indexes): + whole_indexes, redundant_indexes = get_whole_index(self.db, + self.integrate_indexes['historyIndexes'], + history_invalid_indexes, + self.display_detail_info) + display_useless_redundant_indexes(whole_indexes, redundant_indexes, + self.workload_used_index, self.display_detail_info) + + def display_incremental_index(self, history_invalid_indexes, + workload_file_path): + def rm_schema(table_name): + return table_name.split('.')[-1] + # display historical effective indexes + if self.integrate_indexes['historyIndexes']: + print_header_boundary(" Historical effective indexes ") + for table_name, index_list in self.integrate_indexes['historyIndexes'].items(): + for column in index_list: + index_name = 'idx_%s_%s%s' % (rm_schema(table_name), + (column[1] + '_' if column[1] else ''), + '_'.join(column[0].split(', '))) + statement = 'CREATE INDEX %s ON %s%s%s;' % (index_name, table_name, + '(' + column[0] + ')', (' ' + column[1] if column[1] else '')) + print(statement) + # display historical invalid indexes + if history_invalid_indexes: + print_header_boundary(" Historical invalid indexes ") + for table_name, index_list in history_invalid_indexes.items(): + for column in index_list: + index_name = 'idx_%s_%s%s' % (rm_schema(table_name), + (column[1] + '_' if column[1] else ''), + '_'.join(column[0].split(', '))) + statement = 'CREATE INDEX %s ON %s%s%s;' % (index_name, table_name, + '(' + column[0] + ')', (' ' + column[1] if column[1] else '')) + print(statement) + # save integrate indexes result + integrate_indexes_file = os.path.join(os.path.realpath(os.path.dirname(workload_file_path)), + 'index_result.json') + for table, indexes in self.integrate_indexes['currentIndexes'].items(): + self.integrate_indexes['historyIndexes'][table] = \ + self.integrate_indexes['historyIndexes'].get(table, []) + self.integrate_indexes['historyIndexes'][table].extend(indexes) + self.integrate_indexes['historyIndexes'][table] = \ + list( + set(map(tuple, (self.integrate_indexes['historyIndexes'][table])))) + with open(integrate_indexes_file, 'w') as file: + json.dump(self.integrate_indexes['historyIndexes'], file) + + +def green(text): + return '\033[32m%s\033[0m' % text + + +def print_header_boundary(header): + # Output a header first, which looks more beautiful. + try: + term_width = os.get_terminal_size().columns + # The width of each of the two sides of the terminal. + side_width = (term_width - len(header)) // 2 + except (AttributeError, OSError): + side_width = 0 + title = SHARP * side_width + header + SHARP * side_width + print(green(title)) + + +def load_workload(file_path): + wd_dict = {} + workload = [] + global BLANK + with open(file_path, 'r') as file: + raw_text = ''.join(file.readlines()) + sqls = raw_text.split(';') + for sql in sqls: + if any(re.search(r'((\A|[\s\(,])%s[\s*\(])' % tp, sql.lower()) for tp in SQL_TYPE): + TWO_BLANKS = BLANK * 2 + while TWO_BLANKS in sql: + sql = sql.replace(TWO_BLANKS, BLANK) + if sql.strip() not in wd_dict.keys(): + wd_dict[sql.strip()] = 1 + else: + wd_dict[sql.strip()] += 1 + for sql, freq in wd_dict.items(): + workload.append(QueryItem(sql, freq)) + + return workload + + +def get_workload_template(workload): + templates = {} + placeholder = r'@@@' + + for item in workload: + sql_template = item.statement + for pattern in SQL_PATTERN: + sql_template = re.sub(pattern, placeholder, sql_template) + if sql_template not in templates: + templates[sql_template] = {} + templates[sql_template]['cnt'] = 0 + templates[sql_template]['samples'] = [] + templates[sql_template]['cnt'] += item.frequency + # reservoir sampling + if len(templates[sql_template]['samples']) < SAMPLE_NUM: + templates[sql_template]['samples'].append(item.statement) + else: + if random.randint(0, templates[sql_template]['cnt']) < SAMPLE_NUM: + templates[sql_template]['samples'][random.randint(0, SAMPLE_NUM - 1)] = \ + item.statement + + return templates + + +def workload_compression(input_path): + compressed_workload = [] + total_num = 0 + if JSON_TYPE: + with open(input_path, 'r') as file: + templates = json.load(file) + else: + workload = load_workload(input_path) + templates = get_workload_template(workload) + + for _, elem in templates.items(): + for sql in elem['samples']: + compressed_workload.append( + QueryItem(sql.strip(), elem['cnt'] / len(elem['samples']))) + total_num += elem['cnt'] + return compressed_workload, total_num + + +# enumerate the column combinations for a suggested index +def get_indexable_columns(table_index_dict): + query_indexable_columns = {} + if len(table_index_dict) == 0: + return query_indexable_columns + + for table in table_index_dict.keys(): + query_indexable_columns[table] = [] + for columns_tuple in table_index_dict[table]: + indexable_columns = columns_tuple[0].split(',') + for column in indexable_columns: + for ind, item in enumerate(query_indexable_columns[table]): + if column != item[0]: + continue + if columns_tuple[1] == item[1]: + query_indexable_columns[table].pop(ind) + break + query_indexable_columns[table].append( + (column, columns_tuple[1])) + + return query_indexable_columns + + +def add_column(valid_index_dict, table, columns_info, single_col_info): + columns, columns_index_type = columns_info + single_column, single_index_type = single_col_info + if columns_index_type.strip('"') != single_index_type.strip('"'): + add_column(valid_index_dict, table, (columns, 'local'), + (single_column, 'local')) + add_column(valid_index_dict, table, (columns, 'global'), + (single_column, 'global')) + else: + current_columns_tuple = ( + columns + ',' + single_column, columns_index_type) + if current_columns_tuple in valid_index_dict[table]: + return + if single_index_type == 'local': + global_columns_tuple = (columns + ',' + single_column, 'global') + if global_columns_tuple in valid_index_dict[table]: + global_pos = valid_index_dict[table].index( + global_columns_tuple) + valid_index_dict[table][global_pos] = current_columns_tuple + current_columns_tuple = global_columns_tuple + valid_index_dict[table].append(current_columns_tuple) + + +def get_valid_index_dict(table_index_dict, query, db): + need_check = False + query_indexable_columns = get_indexable_columns(table_index_dict) + valid_index_dict = db.query_index_check(query.statement, query_indexable_columns, + ENABLE_MULTI_NODE) + + for i in range(MAX_INDEX_COLUMN_NUM): + for table in valid_index_dict.keys(): + for columns, index_type in valid_index_dict[table]: + if columns.count(',') != i: + continue + need_check = True + for single_column, single_index_type in query_indexable_columns[table]: + if single_column not in columns: + add_column(valid_index_dict, table, (columns, index_type), + (single_column, single_index_type)) + if need_check: + valid_index_dict = db.query_index_check(query.statement, valid_index_dict, + ENABLE_MULTI_NODE) + need_check = False + else: + break + return valid_index_dict + + +def print_candidate_indexes(column_sqls, table, candidate_indexes): + if column_sqls[0][1]: + print("table: ", table, "columns: ",column_sqls[0][0], + "type: ", column_sqls[0][1]) + else: + print("table: ", table, "columns: ",column_sqls[0][0]) + if (table, tuple(column_sqls[0][0]), column_sqls[0][1]) not in IndexItemFactory().indexes: + index = IndexItemFactory().get_index(table, column_sqls[0][0], 'local') + index.index_type = 'global' + else: + index = IndexItemFactory().get_index(table, column_sqls[0][0], column_sqls[0][1]) + index.is_candidate = True + candidate_indexes.append(index) + + +def filter_redundant_indexes(index_dict): + candidate_indexes = [] + for table, column_sqls in index_dict.items(): + # sorted using index_type and columns + sorted_column_sqls = sorted( + column_sqls.items(), key=lambda item: (item[0][1], item[0][0])) + merged_column_sqls = [] + # merge sqls + for i in range(len(sorted_column_sqls) - 1): + if re.match(sorted_column_sqls[i][0][0] + ',', sorted_column_sqls[i+1][0][0]) and \ + sorted_column_sqls[i][0][1] == sorted_column_sqls[i+1][0][1]: + sorted_column_sqls[i+1][1].extend(sorted_column_sqls[i][1]) + else: + merged_column_sqls.append(sorted_column_sqls[i]) + else: + merged_column_sqls.append(sorted_column_sqls[-1]) + # sort using columns + merged_column_sqls.sort(key=lambda item: item[0][0]) + for i in range(len(merged_column_sqls)-1): + # same columns + if merged_column_sqls[i][0][0] == \ + merged_column_sqls[i+1][0][0]: + print_candidate_indexes(merged_column_sqls[i], + table, + candidate_indexes) + continue + # left match for the partation table + if re.match(merged_column_sqls[i][0][0] + ',', + merged_column_sqls[i+1][0][0]): + merged_column_sqls[i+1][1].extend( + merged_column_sqls[i][1]) + merged_column_sqls[i+1] = ((merged_column_sqls[i+1][0][0], 'global'), + merged_column_sqls[i+1][1]) + continue + print_candidate_indexes(merged_column_sqls[i], table, candidate_indexes) + else: + print_candidate_indexes(merged_column_sqls[-1], table, candidate_indexes) + return candidate_indexes + + +def filter_duplicate_indexes(valid_index_dict, index_dict, workload, pos): + for table in valid_index_dict.keys(): + if table not in index_dict.keys(): + index_dict[table] = {} + valid_index_dict[table].sort(key=lambda x: -len(x[0])) + for columns, index_type in valid_index_dict[table]: + if len(workload[pos].valid_index_list) >= FULL_ARRANGEMENT_THRESHOLD: + break + if (columns, index_type) in index_dict[table]: + index_dict[table][(columns, index_type)].append(pos) + else: + column_sql = {(columns, index_type): [pos]} + index_dict[table].update(column_sql) + workload[pos].valid_index_list.append( + IndexItemFactory().get_index(table, columns, index_type=index_type)) + + +def generate_candidate_indexes(workload, db): + index_dict = {} + if DRIVER: + db.init_conn_handle() + for k, query in enumerate(workload): + if not re.search(r'(\A|\s)select\s', query.statement.lower()): + continue + table_index_dict = db.query_index_advisor(query.statement) + valid_index_dict = get_valid_index_dict(table_index_dict, query, db) + # filter duplicate indexes + filter_duplicate_indexes(valid_index_dict, index_dict, workload, k) + + # filter redundant indexes + candidate_indexes = filter_redundant_indexes(index_dict) + if DRIVER: + db.close_conn() + return candidate_indexes + + +def get_atomic_config_for_query(indexes, config, ind, atomic_configs): + if ind == len(indexes): + table_count = {} + for index in config: + if index.table not in table_count.keys(): + table_count[index.table] = 1 + else: + table_count[index.table] += 1 + if len(table_count) > 2 or table_count[index.table] > 2: + return + atomic_configs.append(config) + + return + + get_atomic_config_for_query( + indexes, copy.copy(config), ind + 1, atomic_configs) + config.append(indexes[ind]) + get_atomic_config_for_query( + indexes, copy.copy(config), ind + 1, atomic_configs) + + +def is_same_config(config1, config2): + if len(config1) != len(config2): + return False + + for index1 in config1: + is_found = False + for index2 in config2: + if index1.table == index2.table and index1.columns == index2.columns \ + and index1.index_type == index2.index_type: + is_found = True + if not is_found: + return False + + return True + + +def generate_atomic_config(workload): + atomic_config_total = [] + + for query in workload: + if len(query.valid_index_list) == 0: + continue + + atomic_configs = [] + atomic_config = [] + get_atomic_config_for_query( + query.valid_index_list, atomic_config, 0, atomic_configs) + + is_found = False + for new_config in atomic_configs: + for exist_config in atomic_config_total: + if is_same_config(new_config, exist_config): + is_found = True + break + if not is_found: + atomic_config_total.append(new_config) + is_found = False + + return atomic_config_total + + +def atomic_config_is_valid(atomic_config, config): + is_exist = False + is_same = False + if len(atomic_config) == 1: + is_same = (config[-1] is atomic_config[0]) + for atomic_index in atomic_config: + is_exist = False + for index in config: + if index is atomic_index: + index.storage = atomic_index.storage + is_exist = True + break + if not is_exist: + break + return is_exist, is_same + + +# find the subsets of a given config in the atomic configs +def find_subsets_num(config, atomic_config_total): + atomic_subsets_num = [] + cur_index_atomic_pos = -1 + for i, atomic_config in enumerate(atomic_config_total): + if len(atomic_config) > len(config): + continue + is_exist, is_same = atomic_config_is_valid(atomic_config, config) + if is_same: + cur_index_atomic_pos = i + if is_exist: + atomic_subsets_num.append(i) + if cur_index_atomic_pos == -1: + raise ValueError("No atomic configs found for current config!") + return atomic_subsets_num, cur_index_atomic_pos + + +# infer the total cost of workload for a config according to the cost of atomic configs +def infer_workload_cost(workload, config, atomic_config_total): + total_cost = 0 + is_computed = False + atomic_subsets_num, cur_index_atomic_pos = find_subsets_num( + config, atomic_config_total) + if len(atomic_subsets_num) == 0: + raise ValueError("No atomic configs found for current config!") + if not config[-1].total_sql_num: + is_computed = True + for ind, obj in enumerate(workload): + if max(atomic_subsets_num) >= len(obj.cost_list): + raise ValueError("Wrong atomic config for current query!") + # compute the cost for selection + min_cost = obj.cost_list[0] + for num in atomic_subsets_num: + if num < len(obj.cost_list) and obj.cost_list[num] < min_cost: + min_cost = obj.cost_list[num] + total_cost += min_cost + + # record ineffective sql and negative sql for candidate indexes + if is_computed: + ExecuteFactory.record_ineffective_negative_sql( + config[-1], obj, ind) + return total_cost, cur_index_atomic_pos + + +def get_whole_index(db, history_indexes, history_invalid_index, detail_info): + if DRIVER: + db.init_conn_handle() + whole_index, redundant_indexes = \ + db.check_useless_index(history_indexes, history_invalid_index) + if DRIVER: + db.close_conn() + print_header_boundary(" Created indexes ") + detail_info['createdIndexes'] = [] + if not whole_index: + print("No created index!") + else: + for index in whole_index: + index_info = {'schemaName': index.schema, 'tbName': index.table, + 'columns': index.columns, 'statement': index.indexdef + ';'} + detail_info['createdIndexes'].append(index_info) + print("%s: %s;" % (index.schema, index.indexdef)) + return whole_index, redundant_indexes + + +def display_redundant_indexes(redundant_indexes, unused_index_columns, remove_list, detail_info): + for pos, index in enumerate(redundant_indexes): + is_redundant = False + for redundant_obj in index.redundant_obj: + # redundant objects are not in the useless index set or + # both redundant objects and redundant index in useless index must be redundant index + index_exist = redundant_obj.indexname not in unused_index_columns.keys() or \ + (unused_index_columns.get(redundant_obj.indexname) and + unused_index_columns.get(index.indexname)) + if index_exist: + is_redundant = True + if not is_redundant: + remove_list.append(pos) + for item in sorted(remove_list, reverse=True): + redundant_indexes.pop(item) + + if not redundant_indexes: + print("No redundant index!") + # redundant index + for index in redundant_indexes: + statement = "DROP INDEX %s.%s;" % \ + (index.schema, index.indexname) + print(statement) + existing_index = [item.indexname + ':' + + item.columns for item in index.redundant_obj] + redundant_index = {"schemaName": index.schema, "tbName": index.table, "type": 2, + "columns": index.columns, "statement": statement, + "existingIndex": existing_index} + detail_info['uselessIndexes'].append(redundant_index) + + +def display_useless_redundant_indexes(whole_indexes, redundant_indexes, + workload_indexes, detail_info): + indexes_name = set(index.indexname for index in whole_indexes) + unused_index = list(indexes_name.difference(workload_indexes)) + remove_list = [] + print_header_boundary(" Current workload useless indexes ") + detail_info['uselessIndexes'] = [] + # useless index not contain unique index + unused_index_columns = {} + has_unused_index = False + for cur_index in unused_index: + for index in whole_indexes: + if cur_index != index.indexname: + continue + # get useless index details from whole index + unused_index_columns[cur_index] = index.columns + if 'UNIQUE INDEX' not in index.indexdef: + has_unused_index = True + statement = "DROP INDEX %s;" % index.indexname + print(statement) + useless_index = {"schemaName": index.schema, "tbName": index.table, "type": 3, + "columns": index.columns, "statement": statement} + detail_info['uselessIndexes'].append(useless_index) + if not has_unused_index: + print("No useless index!") + print_header_boundary(" Redundant indexes ") + # filter redundant index + display_redundant_indexes( + redundant_indexes, unused_index_columns, remove_list, detail_info) + + +def greedy_determine_opt_config(workload, atomic_config_total, candidate_indexes, origin_sum_cost): + opt_config = [] + index_num_record = set() + min_cost = origin_sum_cost + for i in range(len(candidate_indexes)): + if i == 1 and min_cost == origin_sum_cost: + break + cur_min_cost = origin_sum_cost + cur_index = None + cur_index_num = -1 + for k, index in enumerate(candidate_indexes): + if k in index_num_record: + continue + cur_config = copy.copy(opt_config) + cur_config.append(index) + cur_estimated_cost, cur_index_atomic_pos = \ + infer_workload_cost(workload, cur_config, atomic_config_total) + if cur_estimated_cost < cur_min_cost: + cur_min_cost = cur_estimated_cost + cur_index = index + cur_index.atomic_pos = cur_index_atomic_pos + cur_index_num = k + if cur_index and cur_min_cost < min_cost: + if MAX_INDEX_STORAGE and sum([obj.storage for obj in opt_config]) + \ + cur_index.storage > MAX_INDEX_STORAGE: + candidate_indexes.remove(cur_index) + continue + if len(opt_config) == MAX_INDEX_NUM: + break + min_cost = cur_min_cost + opt_config.append(cur_index) + index_num_record.add(cur_index_num) + else: + break + + return opt_config + + +def get_last_indexes_result(input_path): + last_indexes_result_file = os.path.join(os.path.realpath( + os.path.dirname(input_path)), 'index_result.json') + integrate_indexes = {'historyIndexes': {}} + if os.path.exists(last_indexes_result_file): + try: + with open(last_indexes_result_file, 'r') as file: + integrate_indexes['historyIndexes'] = json.load(file) + except json.JSONDecodeError: + return integrate_indexes + return integrate_indexes + + +def index_advisor_workload(history_advise_indexes, db, workload_file_path, + multi_iter_mode, show_detail): + workload_info = workload_compression(workload_file_path) + index_advisor = IndexAdvisor(db, workload_info, multi_iter_mode) + if multi_iter_mode: + opt_indexes = index_advisor.complex_index_advisor() + else: + opt_indexes = index_advisor.simple_index_advisor() + if opt_indexes: + index_advisor.filter_low_benefit_index(opt_indexes) + index_advisor.display_advise_indexes_info(opt_indexes, show_detail) + + index_advisor.generate_incremental_index(history_advise_indexes) + history_invalid_indexes = {} + index_advisor.generate_redundant_useless_indexes(history_invalid_indexes) + index_advisor.display_incremental_index( + history_invalid_indexes, workload_file_path) + if show_detail: + print_header_boundary(" Display detail information ") + sql_info = json.dumps( + index_advisor.display_detail_info, indent=4, separators=(',', ':')) + print(sql_info) + + +def check_parameter(args): + global MAX_INDEX_NUM, ENABLE_MULTI_NODE, MAX_INDEX_STORAGE, JSON_TYPE, DRIVER + if args.max_index_num is not None and args.max_index_num <= 0: + raise argparse.ArgumentTypeError("%s is an invalid positive int value" % + args.max_index_num) + if args.max_index_storage is not None and args.max_index_storage <= 0: + raise argparse.ArgumentTypeError("%s is an invalid positive int value" % + args.max_index_storage) + JSON_TYPE = args.json + MAX_INDEX_NUM = args.max_index_num + ENABLE_MULTI_NODE = args.multi_node + MAX_INDEX_STORAGE = args.max_index_storage + # check if the password contains illegal characters + is_legal = re.search(r'^[A-Za-z0-9~!@#%^*\-_=+?,.]+$', args.W) + if not is_legal: + raise ValueError("The password contains illegal characters.") + + +def main(argv): + arg_parser = argparse.ArgumentParser( + description='Generate index set for workload.') + arg_parser.add_argument("p", help="Port of database", type=int) + arg_parser.add_argument("d", help="Name of database", action=CheckValid) + arg_parser.add_argument( + "--h", help="Host for database", action=CheckValid) + arg_parser.add_argument( + "-U", help="Username for database log-in", action=CheckValid) + arg_parser.add_argument( + "f", help="File containing workload queries (One query per line)", action=CheckValid) + arg_parser.add_argument("--schema", help="Schema name for the current business data", + required=True, action=CheckValid) + arg_parser.add_argument( + "--max_index_num", help="Maximum number of suggested indexes", type=int) + arg_parser.add_argument("--max_index_storage", + help="Maximum storage of suggested indexes/MB", type=int) + arg_parser.add_argument("--multi_iter_mode", action='store_true', + help="Whether to use multi-iteration algorithm", default=False) + arg_parser.add_argument("--multi_node", action='store_true', + help="Whether to support distributed scenarios", default=False) + arg_parser.add_argument("--json", action='store_true', + help="Whether the workload file format is json", default=False) + arg_parser.add_argument("--driver", action='store_true', + help="Whether to employ python-driver", default=False) + arg_parser.add_argument("--show_detail", action='store_true', + help="Whether to show detailed sql information", default=False) + args = arg_parser.parse_args(argv) + + args.W = get_password() + check_parameter(args) + # Initialize the connection + global DRIVER + if args.driver: + try: + import psycopg2 + try: + from .dao.driver_execute import DriverExecute + except ImportError: + from dao.driver_execute import DriverExecute + db = DriverExecute(args.d, args.U, args.W, args.h, args.p, args.schema, + args.multi_node, args.max_index_storage) + except ImportError: + logging.warning('Python driver import failed, ' + 'the gsql mode will be selected to connect to the database.') + db = GSqlExecute(args.d, args.U, args.W, args.h, args.p, args.schema, + args.multi_node, args.max_index_storage) + db.init_conn_handle() + args.driver = None + else: + db = GSqlExecute(args.d, args.U, args.W, args.h, args.p, args.schema, + args.multi_node, args.max_index_storage) + db.init_conn_handle() + DRIVER = args.driver + if args.multi_node and not db.is_multi_node(): + raise argparse.ArgumentTypeError('--multi_node is only support for distributed database') + index_advisor_workload(get_last_indexes_result(args.f), db, args.f, + args.multi_iter_mode, args.show_detail) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/src/gausskernel/dbmind/tools/components/opengauss_exporter/__main__.py b/src/gausskernel/dbmind/tools/components/opengauss_exporter/__main__.py index 6894a1570..70f1abf1d 100644 --- a/src/gausskernel/dbmind/tools/components/opengauss_exporter/__main__.py +++ b/src/gausskernel/dbmind/tools/components/opengauss_exporter/__main__.py @@ -11,11 +11,13 @@ # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. # See the Mulan PSL v2 for more details. import sys +import os try: from dbmind.components.opengauss_exporter import main except ImportError: - sys.path.append('..') + libpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') + sys.path.append(libpath) from opengauss_exporter import main main(sys.argv[1:]) diff --git a/src/gausskernel/dbmind/tools/components/opengauss_exporter/core/main.py b/src/gausskernel/dbmind/tools/components/opengauss_exporter/core/main.py index 4b22b3aec..145cda9cd 100644 --- a/src/gausskernel/dbmind/tools/components/opengauss_exporter/core/main.py +++ b/src/gausskernel/dbmind/tools/components/opengauss_exporter/core/main.py @@ -29,7 +29,7 @@ from . import controller from . import service from .. import __version__ -ROOT_DIR_PATH = os.path.abspath( +ROOT_DIR_PATH = os.path.realpath( os.path.join(os.path.dirname(__file__), '..') ) @@ -70,7 +70,7 @@ def wipe_off_password(dsn): def path_type(path): if os.path.exists(path): - return os.path.abspath(path) + return os.path.realpath(path) else: raise argparse.ArgumentTypeError('%s is not a valid path.' % path) @@ -102,7 +102,7 @@ def parse_argv(argv): parser.add_argument('--ssl-certfile', type=path_type, help='set the path of ssl certificate file') parser.add_argument('--parallel', default=5, type=int, help='not collect pg_settings.yml metrics.') - parser.add_argument('--log.filepath', type=os.path.abspath, default=os.path.join(os.getcwd(), DEFAULT_LOGFILE), + parser.add_argument('--log.filepath', type=os.path.realpath, default=os.path.join(os.getcwd(), DEFAULT_LOGFILE), help='the path to log') parser.add_argument('--log.level', default='info', choices=('debug', 'info', 'warn', 'error', 'fatal'), help='only log messages with the given severity or above.' diff --git a/src/gausskernel/dbmind/tools/components/reprocessing_exporter/__main__.py b/src/gausskernel/dbmind/tools/components/reprocessing_exporter/__main__.py index 1bb75bd2b..65c30e293 100644 --- a/src/gausskernel/dbmind/tools/components/reprocessing_exporter/__main__.py +++ b/src/gausskernel/dbmind/tools/components/reprocessing_exporter/__main__.py @@ -11,11 +11,13 @@ # MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. # See the Mulan PSL v2 for more details. import sys +import os try: from dbmind.components.reprocessing_exporter import main except ImportError: - sys.path.append('..') + libpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') + sys.path.append(libpath) from reprocessing_exporter import main main(sys.argv[1:]) diff --git a/src/gausskernel/dbmind/tools/components/reprocessing_exporter/core/main.py b/src/gausskernel/dbmind/tools/components/reprocessing_exporter/core/main.py index fcea36139..e5a658cec 100644 --- a/src/gausskernel/dbmind/tools/components/reprocessing_exporter/core/main.py +++ b/src/gausskernel/dbmind/tools/components/reprocessing_exporter/core/main.py @@ -24,7 +24,7 @@ from . import dao from . import service from .. import __version__ -CURR_DIR = os.path.abspath( +CURR_DIR = os.path.realpath( os.path.join(os.path.dirname(__file__), '..') ) DEFAULT_YAML = 'reprocessing_exporter.yml' @@ -35,7 +35,7 @@ with tempfile.NamedTemporaryFile(suffix='.pid') as fp: def path_type(path): if os.path.exists(path): - return os.path.abspath(path) + return os.path.realpath(path) else: raise argparse.ArgumentTypeError('%s is not a valid path.' % path) @@ -56,7 +56,7 @@ def parse_argv(argv): help='listen port to expose metrics and web interface') parser.add_argument('--collector.config', type=path_type, default=os.path.join(CURR_DIR, DEFAULT_YAML), help='according to the content of the yaml file for metric collection') - parser.add_argument('--log.filepath', type=os.path.abspath, + parser.add_argument('--log.filepath', type=os.path.realpath, default=os.path.join(os.getcwd(), DEFAULT_LOGFILE), help='the path to log') parser.add_argument('--log.level', default='info', choices=('debug', 'info', 'warn', 'error', 'fatal'), diff --git a/src/gausskernel/dbmind/tools/components/slow_query_diagnosis.py b/src/gausskernel/dbmind/tools/components/slow_query_diagnosis.py index 3045ba963..91fd61c0d 100644 --- a/src/gausskernel/dbmind/tools/components/slow_query_diagnosis.py +++ b/src/gausskernel/dbmind/tools/components/slow_query_diagnosis.py @@ -51,7 +51,7 @@ def show(query, start_time, end_time): dump_file_name = 'slow_queries_%s.txt' % int(time.time()) with open(dump_file_name, 'w+') as fp: fp.write(str(output_table)) - write_to_terminal('Dumped file is %s.' % os.path.abspath(dump_file_name)) + write_to_terminal('Dumped file is %s.' % os.path.realpath(dump_file_name)) elif char == 'N': print(output_table) print('(%d rows)' % nb_rows) diff --git a/src/gausskernel/dbmind/tools/components/sqldiag/__main__.py b/src/gausskernel/dbmind/tools/components/sqldiag/__main__.py index 113c50f5f..ba5bd70b1 100644 --- a/src/gausskernel/dbmind/tools/components/sqldiag/__main__.py +++ b/src/gausskernel/dbmind/tools/components/sqldiag/__main__.py @@ -12,11 +12,13 @@ # See the Mulan PSL v2 for more details. import sys +import os try: from dbmind.components.sqldiag import main except ImportError: - sys.path.append('..') + libpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') + sys.path.append(libpath) from sqldiag import main main(sys.argv[1:]) diff --git a/src/gausskernel/dbmind/tools/components/sqldiag/utils.py b/src/gausskernel/dbmind/tools/components/sqldiag/utils.py index db13aca4b..d7a00afad 100644 --- a/src/gausskernel/dbmind/tools/components/sqldiag/utils.py +++ b/src/gausskernel/dbmind/tools/components/sqldiag/utils.py @@ -132,8 +132,8 @@ def check_time_legality(time_string): def is_valid_conf(filepath): if os.path.exists(filepath): file_abs_path = filepath - elif os.path.exists(os.path.abspath(os.path.join(os.getcwd(), filepath))): - file_abs_path = os.path.abspath(os.path.join(os.getcwd(), filepath)) + elif os.path.exists(os.path.realpath(os.path.join(os.getcwd(), filepath))): + file_abs_path = os.path.realpath(os.path.join(os.getcwd(), filepath)) else: print("FATAL: Not found the configuration file %s." % filepath, file=sys.stderr) return False diff --git a/src/gausskernel/dbmind/tools/components/xtuner/__main__.py b/src/gausskernel/dbmind/tools/components/xtuner/__main__.py index 3e2c206be..7a109ff06 100644 --- a/src/gausskernel/dbmind/tools/components/xtuner/__main__.py +++ b/src/gausskernel/dbmind/tools/components/xtuner/__main__.py @@ -12,11 +12,14 @@ # See the Mulan PSL v2 for more details. import sys +import os try: from dbmind.components.xtuner.tuner.main import main except ImportError: - sys.path.append('..') + libpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..') + sys.path.append(libpath) + from xtuner.tuner.main import main main(sys.argv[1:]) diff --git a/src/gausskernel/dbmind/tools/components/xtuner/tuner/knob.py b/src/gausskernel/dbmind/tools/components/xtuner/tuner/knob.py index 1f0ce0e70..71b6cba9d 100644 --- a/src/gausskernel/dbmind/tools/components/xtuner/tuner/knob.py +++ b/src/gausskernel/dbmind/tools/components/xtuner/tuner/knob.py @@ -166,10 +166,10 @@ class Knob: return self._scale = self._max - self._min - if self._scale < 0: + if self._scale <= 0: raise ValueError('Knob %s is incorrectly configured. ' 'The max value must be greater than ' - 'or equal to the min value.' % self.name) + 'the min value.' % self.name) if type(self.user_set) is str: self.current = self.to_numeric(self.user_set) elif type(self.user_set) in (int, float): diff --git a/src/gausskernel/dbmind/tools/constants.py b/src/gausskernel/dbmind/tools/constants.py index 7aaebad51..6227f5f59 100644 --- a/src/gausskernel/dbmind/tools/constants.py +++ b/src/gausskernel/dbmind/tools/constants.py @@ -15,11 +15,10 @@ import os __version__ = '1.0.0' __description__ = 'openGauss DBMind: An autonomous platform for openGauss' -DBMIND_PATH = os.path.dirname(os.path.abspath(__file__)) +DBMIND_PATH = os.path.dirname(os.path.realpath(__file__)) MISC_PATH = os.path.join(DBMIND_PATH, 'misc') CONFILE_NAME = 'dbmind.conf' # the name of configuration file -CONFILE_HEADER_NAME = 'dbmind.conf.header' PIDFILE_NAME = 'dbmind.pid' LOGFILE_NAME = 'dbmind.log' METRIC_MAP_CONFIG = 'metric_map.conf' diff --git a/src/gausskernel/dbmind/tools/misc/dbmind.conf.header b/src/gausskernel/dbmind/tools/misc/dbmind.conf.header deleted file mode 100644 index 09a79d435..000000000 --- a/src/gausskernel/dbmind/tools/misc/dbmind.conf.header +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) 2020 Huawei Technologies Co.,Ltd. -# -# openGauss is licensed under Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# -# http://license.coscl.org.cn/MulanPSL2 -# -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, -# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, -# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. -# See the Mulan PSL v2 for more details. - -# Notice: -# 1. (null) explicitly represents empty or null. Meanwhile blank represents undefined. -# 2. DBMind encrypts password parameters. Hence, there is no plain-text password after initialization. -# 3. Users can only configure the plain-text password in this file before initializing -# (that is, using the --initialize option), -# and then if users want to modify the password-related information, -# users need to use the 'set' sub-command to achieve. -# 4. If users use relative path in this file, the current working directory is the directory where this file is located. - diff --git a/src/gausskernel/dbmind/tools/misc/metric_map.conf b/src/gausskernel/dbmind/tools/misc/metric_map.conf index 1080fc74c..9394aaebe 100644 --- a/src/gausskernel/dbmind/tools/misc/metric_map.conf +++ b/src/gausskernel/dbmind/tools/misc/metric_map.conf @@ -1,19 +1,439 @@ +# Copyright (c) 2022 Huawei Technologies Co.,Ltd. +# +# openGauss is licensed under Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# +# http://license.coscl.org.cn/MulanPSL2 +# +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. + + # The name of different metric collectors may be different, # so we converte the different metric name here. +# For example, users can get the full list of metrics from Prometheus via the following URI: +# /api/v1/label/__name__/values + # Format: # metric name in the DBMind = metric name in the collector (e.g., Prometheus-exporter, Agent) # -os_disk_io_cnt = os_disk_io_cnt -os_disk_io_load = os_disk_io_load -os_disk_read_bytes = os_disk_read_bytes -os_disk_read_time = os_disk_read_time -os_disk_write_bytes = os_disk_write_bytes -os_disk_write_time = os_disk_write_time -os_cpu_usage = os_cpu_usage -os_system_cpu_usage = os_system_cpu_usage -os_user_cpu_usage = os_user_cpu_usage -opengauss_blocks_read_time = opengauss_blocks_read_time -opengauss_blocks_write_time = opengauss_blocks_write_time -opengauss_sql_cpu_time_rate = opengauss_sql_cpu_time_rate -disk_usage = disk_usage +gaussdb_qps_by_instance = gaussdb_qps_by_instance +io_queue_number = io_queue_number +io_read_bytes = io_read_bytes +io_read_delay_time = io_read_delay_time +io_read_total = io_read_total +io_write_bytes = io_write_bytes +io_write_delay_time = io_write_delay_time +io_write_total = io_write_total +node_arp_entries = node_arp_entries +node_boot_time_seconds = node_boot_time_seconds +node_context_switches_total = node_context_switches_total +node_cooling_device_cur_state = node_cooling_device_cur_state +node_cooling_device_max_state = node_cooling_device_max_state +node_cpu_core_throttles_total = node_cpu_core_throttles_total +node_cpu_frequency_max_hertz = node_cpu_frequency_max_hertz +node_cpu_frequency_min_hertz = node_cpu_frequency_min_hertz +node_cpu_guest_seconds_total = node_cpu_guest_seconds_total +node_cpu_package_throttles_total = node_cpu_package_throttles_total +node_cpu_scaling_frequency_hertz = node_cpu_scaling_frequency_hertz +node_cpu_scaling_frequency_max_hertz = node_cpu_scaling_frequency_max_hertz +node_cpu_scaling_frequency_min_hertz = node_cpu_scaling_frequency_min_hertz +node_cpu_seconds_total = node_cpu_seconds_total +node_disk_io_now = node_disk_io_now +node_disk_io_time_seconds_total = node_disk_io_time_seconds_total +node_disk_io_time_weighted_seconds_total = node_disk_io_time_weighted_seconds_total +node_disk_read_bytes_total = node_disk_read_bytes_total +node_disk_read_time_seconds_total = node_disk_read_time_seconds_total +node_disk_reads_completed_total = node_disk_reads_completed_total +node_disk_reads_merged_total = node_disk_reads_merged_total +node_disk_write_time_seconds_total = node_disk_write_time_seconds_total +node_disk_writes_completed_total = node_disk_writes_completed_total +node_disk_writes_merged_total = node_disk_writes_merged_total +node_disk_written_bytes_total = node_disk_written_bytes_total +node_edac_correctable_errors_total = node_edac_correctable_errors_total +node_edac_csrow_correctable_errors_total = node_edac_csrow_correctable_errors_total +node_edac_csrow_uncorrectable_errors_total = node_edac_csrow_uncorrectable_errors_total +node_edac_uncorrectable_errors_total = node_edac_uncorrectable_errors_total +node_entropy_available_bits = node_entropy_available_bits +node_entropy_pool_size_bits = node_entropy_pool_size_bits +node_exporter_build_info = node_exporter_build_info +node_filefd_allocated = node_filefd_allocated +node_filefd_maximum = node_filefd_maximum +node_filesystem_avail_bytes = node_filesystem_avail_bytes +node_filesystem_device_error = node_filesystem_device_error +node_filesystem_files = node_filesystem_files +node_filesystem_files_free = node_filesystem_files_free +node_filesystem_free_bytes = node_filesystem_free_bytes +node_filesystem_readonly = node_filesystem_readonly +node_filesystem_size_bytes = node_filesystem_size_bytes +node_forks_total = node_forks_total +node_hwmon_chip_names = node_hwmon_chip_names +node_hwmon_power_average_interval_max_seconds = node_hwmon_power_average_interval_max_seconds +node_hwmon_power_average_interval_min_seconds = node_hwmon_power_average_interval_min_seconds +node_hwmon_power_average_interval_seconds = node_hwmon_power_average_interval_seconds +node_hwmon_power_average_watt = node_hwmon_power_average_watt +node_hwmon_power_is_battery_watt = node_hwmon_power_is_battery_watt +node_hwmon_sensor_label = node_hwmon_sensor_label +node_hwmon_temp_celsius = node_hwmon_temp_celsius +node_hwmon_temp_crit_alar_celsius = node_hwmon_temp_crit_alar_celsius +node_hwmon_temp_crit_alarm_celsius = node_hwmon_temp_crit_alarm_celsius +node_hwmon_temp_crit_celsius = node_hwmon_temp_crit_celsius +node_hwmon_temp_max_celsius = node_hwmon_temp_max_celsius +node_intr_total = node_intr_total +node_load1 = node_load1 +node_load15 = node_load15 +node_load5 = node_load5 +node_memory_Active_anon_bytes = node_memory_Active_anon_bytes +node_memory_Active_bytes = node_memory_Active_bytes +node_memory_Active_file_bytes = node_memory_Active_file_bytes +node_memory_AnonHugePages_bytes = node_memory_AnonHugePages_bytes +node_memory_AnonPages_bytes = node_memory_AnonPages_bytes +node_memory_Bounce_bytes = node_memory_Bounce_bytes +node_memory_Buffers_bytes = node_memory_Buffers_bytes +node_memory_Cached_bytes = node_memory_Cached_bytes +node_memory_CmaFree_bytes = node_memory_CmaFree_bytes +node_memory_CmaTotal_bytes = node_memory_CmaTotal_bytes +node_memory_CommitLimit_bytes = node_memory_CommitLimit_bytes +node_memory_Committed_AS_bytes = node_memory_Committed_AS_bytes +node_memory_DirectMap1G_bytes = node_memory_DirectMap1G_bytes +node_memory_DirectMap2M_bytes = node_memory_DirectMap2M_bytes +node_memory_DirectMap4k_bytes = node_memory_DirectMap4k_bytes +node_memory_Dirty_bytes = node_memory_Dirty_bytes +node_memory_HardwareCorrupted_bytes = node_memory_HardwareCorrupted_bytes +node_memory_HugePages_Free = node_memory_HugePages_Free +node_memory_HugePages_Rsvd = node_memory_HugePages_Rsvd +node_memory_HugePages_Surp = node_memory_HugePages_Surp +node_memory_HugePages_Total = node_memory_HugePages_Total +node_memory_Hugepagesize_bytes = node_memory_Hugepagesize_bytes +node_memory_Inactive_anon_bytes = node_memory_Inactive_anon_bytes +node_memory_Inactive_bytes = node_memory_Inactive_bytes +node_memory_Inactive_file_bytes = node_memory_Inactive_file_bytes +node_memory_KernelStack_bytes = node_memory_KernelStack_bytes +node_memory_Mapped_bytes = node_memory_Mapped_bytes +node_memory_MemAvailable_bytes = node_memory_MemAvailable_bytes +node_memory_MemFree_bytes = node_memory_MemFree_bytes +node_memory_MemTotal_bytes = node_memory_MemTotal_bytes +node_memory_Mlocked_bytes = node_memory_Mlocked_bytes +node_memory_NFS_Unstable_bytes = node_memory_NFS_Unstable_bytes +node_memory_PageTables_bytes = node_memory_PageTables_bytes +node_memory_SReclaimable_bytes = node_memory_SReclaimable_bytes +node_memory_SUnreclaim_bytes = node_memory_SUnreclaim_bytes +node_memory_Shmem_bytes = node_memory_Shmem_bytes +node_memory_Slab_bytes = node_memory_Slab_bytes +node_memory_SwapCached_bytes = node_memory_SwapCached_bytes +node_memory_SwapFree_bytes = node_memory_SwapFree_bytes +node_memory_SwapTotal_bytes = node_memory_SwapTotal_bytes +node_memory_Unevictable_bytes = node_memory_Unevictable_bytes +node_memory_VmallocChunk_bytes = node_memory_VmallocChunk_bytes +node_memory_VmallocTotal_bytes = node_memory_VmallocTotal_bytes +node_memory_VmallocUsed_bytes = node_memory_VmallocUsed_bytes +node_memory_WritebackTmp_bytes = node_memory_WritebackTmp_bytes +node_memory_Writeback_bytes = node_memory_Writeback_bytes +node_netstat_Icmp6_InErrors = node_netstat_Icmp6_InErrors +node_netstat_Icmp6_InMsgs = node_netstat_Icmp6_InMsgs +node_netstat_Icmp6_OutMsgs = node_netstat_Icmp6_OutMsgs +node_netstat_Icmp_InErrors = node_netstat_Icmp_InErrors +node_netstat_Icmp_InMsgs = node_netstat_Icmp_InMsgs +node_netstat_Icmp_OutMsgs = node_netstat_Icmp_OutMsgs +node_netstat_Ip6_InOctets = node_netstat_Ip6_InOctets +node_netstat_Ip6_OutOctets = node_netstat_Ip6_OutOctets +node_netstat_IpExt_InOctets = node_netstat_IpExt_InOctets +node_netstat_IpExt_OutOctets = node_netstat_IpExt_OutOctets +node_netstat_Ip_Forwarding = node_netstat_Ip_Forwarding +node_netstat_TcpExt_ListenDrops = node_netstat_TcpExt_ListenDrops +node_netstat_TcpExt_ListenOverflows = node_netstat_TcpExt_ListenOverflows +node_netstat_TcpExt_SyncookiesFailed = node_netstat_TcpExt_SyncookiesFailed +node_netstat_TcpExt_SyncookiesRecv = node_netstat_TcpExt_SyncookiesRecv +node_netstat_TcpExt_SyncookiesSent = node_netstat_TcpExt_SyncookiesSent +node_netstat_TcpExt_TCPSynRetrans = node_netstat_TcpExt_TCPSynRetrans +node_netstat_Tcp_ActiveOpens = node_netstat_Tcp_ActiveOpens +node_netstat_Tcp_CurrEstab = node_netstat_Tcp_CurrEstab +node_netstat_Tcp_InErrs = node_netstat_Tcp_InErrs +node_netstat_Tcp_InSegs = node_netstat_Tcp_InSegs +node_netstat_Tcp_OutRsts = node_netstat_Tcp_OutRsts +node_netstat_Tcp_OutSegs = node_netstat_Tcp_OutSegs +node_netstat_Tcp_PassiveOpens = node_netstat_Tcp_PassiveOpens +node_netstat_Tcp_RetransSegs = node_netstat_Tcp_RetransSegs +node_netstat_Udp6_InDatagrams = node_netstat_Udp6_InDatagrams +node_netstat_Udp6_InErrors = node_netstat_Udp6_InErrors +node_netstat_Udp6_NoPorts = node_netstat_Udp6_NoPorts +node_netstat_Udp6_OutDatagrams = node_netstat_Udp6_OutDatagrams +node_netstat_Udp6_RcvbufErrors = node_netstat_Udp6_RcvbufErrors +node_netstat_Udp6_SndbufErrors = node_netstat_Udp6_SndbufErrors +node_netstat_UdpLite6_InErrors = node_netstat_UdpLite6_InErrors +node_netstat_UdpLite_InErrors = node_netstat_UdpLite_InErrors +node_netstat_Udp_InDatagrams = node_netstat_Udp_InDatagrams +node_netstat_Udp_InErrors = node_netstat_Udp_InErrors +node_netstat_Udp_NoPorts = node_netstat_Udp_NoPorts +node_netstat_Udp_OutDatagrams = node_netstat_Udp_OutDatagrams +node_netstat_Udp_RcvbufErrors = node_netstat_Udp_RcvbufErrors +node_netstat_Udp_SndbufErrors = node_netstat_Udp_SndbufErrors +node_network_address_assign_type = node_network_address_assign_type +node_network_carrier = node_network_carrier +node_network_carrier_changes_total = node_network_carrier_changes_total +node_network_device_id = node_network_device_id +node_network_dormant = node_network_dormant +node_network_flags = node_network_flags +node_network_iface_id = node_network_iface_id +node_network_iface_link = node_network_iface_link +node_network_iface_link_mode = node_network_iface_link_mode +node_network_info = node_network_info +node_network_mtu_bytes = node_network_mtu_bytes +node_network_net_dev_group = node_network_net_dev_group +node_network_protocol_type = node_network_protocol_type +node_network_receive_bytes_total = node_network_receive_bytes_total +node_network_receive_compressed_total = node_network_receive_compressed_total +node_network_receive_drop_total = node_network_receive_drop_total +node_network_receive_errs_total = node_network_receive_errs_total +node_network_receive_fifo_total = node_network_receive_fifo_total +node_network_receive_frame_total = node_network_receive_frame_total +node_network_receive_multicast_total = node_network_receive_multicast_total +node_network_receive_packets_total = node_network_receive_packets_total +node_network_speed_bytes = node_network_speed_bytes +node_network_transmit_bytes_total = node_network_transmit_bytes_total +node_network_transmit_carrier_total = node_network_transmit_carrier_total +node_network_transmit_colls_total = node_network_transmit_colls_total +node_network_transmit_compressed_total = node_network_transmit_compressed_total +node_network_transmit_drop_total = node_network_transmit_drop_total +node_network_transmit_errs_total = node_network_transmit_errs_total +node_network_transmit_fifo_total = node_network_transmit_fifo_total +node_network_transmit_packets_total = node_network_transmit_packets_total +node_network_transmit_queue_length = node_network_transmit_queue_length +node_network_up = node_network_up +node_nf_conntrack_entries = node_nf_conntrack_entries +node_nf_conntrack_entries_limit = node_nf_conntrack_entries_limit +node_procs_blocked = node_procs_blocked +node_procs_running = node_procs_running +node_rapl_dram_joules_total = node_rapl_dram_joules_total +node_rapl_package_joules_total = node_rapl_package_joules_total +node_schedstat_running_seconds_total = node_schedstat_running_seconds_total +node_schedstat_timeslices_total = node_schedstat_timeslices_total +node_schedstat_waiting_seconds_total = node_schedstat_waiting_seconds_total +node_scrape_collector_duration_seconds = node_scrape_collector_duration_seconds +node_scrape_collector_success = node_scrape_collector_success +node_sockstat_FRAG6_inuse = node_sockstat_FRAG6_inuse +node_sockstat_FRAG6_memory = node_sockstat_FRAG6_memory +node_sockstat_FRAG_inuse = node_sockstat_FRAG_inuse +node_sockstat_FRAG_memory = node_sockstat_FRAG_memory +node_sockstat_RAW6_inuse = node_sockstat_RAW6_inuse +node_sockstat_RAW_inuse = node_sockstat_RAW_inuse +node_sockstat_TCP6_inuse = node_sockstat_TCP6_inuse +node_sockstat_TCP_alloc = node_sockstat_TCP_alloc +node_sockstat_TCP_inuse = node_sockstat_TCP_inuse +node_sockstat_TCP_mem = node_sockstat_TCP_mem +node_sockstat_TCP_mem_bytes = node_sockstat_TCP_mem_bytes +node_sockstat_TCP_orphan = node_sockstat_TCP_orphan +node_sockstat_TCP_tw = node_sockstat_TCP_tw +node_sockstat_UDP6_inuse = node_sockstat_UDP6_inuse +node_sockstat_UDPLITE6_inuse = node_sockstat_UDPLITE6_inuse +node_sockstat_UDPLITE_inuse = node_sockstat_UDPLITE_inuse +node_sockstat_UDP_inuse = node_sockstat_UDP_inuse +node_sockstat_UDP_mem = node_sockstat_UDP_mem +node_sockstat_UDP_mem_bytes = node_sockstat_UDP_mem_bytes +node_sockstat_sockets_used = node_sockstat_sockets_used +node_softnet_dropped_total = node_softnet_dropped_total +node_softnet_processed_total = node_softnet_processed_total +node_softnet_times_squeezed_total = node_softnet_times_squeezed_total +node_textfile_scrape_error = node_textfile_scrape_error +node_time_seconds = node_time_seconds +node_timex_estimated_error_seconds = node_timex_estimated_error_seconds +node_timex_frequency_adjustment_ratio = node_timex_frequency_adjustment_ratio +node_timex_loop_time_constant = node_timex_loop_time_constant +node_timex_maxerror_seconds = node_timex_maxerror_seconds +node_timex_offset_seconds = node_timex_offset_seconds +node_timex_pps_calibration_total = node_timex_pps_calibration_total +node_timex_pps_error_total = node_timex_pps_error_total +node_timex_pps_frequency_hertz = node_timex_pps_frequency_hertz +node_timex_pps_jitter_seconds = node_timex_pps_jitter_seconds +node_timex_pps_jitter_total = node_timex_pps_jitter_total +node_timex_pps_shift_seconds = node_timex_pps_shift_seconds +node_timex_pps_stability_exceeded_total = node_timex_pps_stability_exceeded_total +node_timex_pps_stability_hertz = node_timex_pps_stability_hertz +node_timex_status = node_timex_status +node_timex_sync_status = node_timex_sync_status +node_timex_tai_offset_seconds = node_timex_tai_offset_seconds +node_timex_tick_seconds = node_timex_tick_seconds +node_udp_queues = node_udp_queues +node_uname_info = node_uname_info +node_vmstat_pgfault = node_vmstat_pgfault +node_vmstat_pgmajfault = node_vmstat_pgmajfault +node_vmstat_pgpgin = node_vmstat_pgpgin +node_vmstat_pgpgout = node_vmstat_pgpgout +node_vmstat_pswpin = node_vmstat_pswpin +node_vmstat_pswpout = node_vmstat_pswpout +og_context_memory_totalsize = og_context_memory_totalsize +og_context_memory_usedsize = og_context_memory_usedsize +og_cpu_load_total_cpu = og_cpu_load_total_cpu +og_memory_info_memorymbytes = og_memory_info_memorymbytes +og_session_memory_totalsize = og_session_memory_totalsize +og_session_memory_usedsize = og_session_memory_usedsize +og_state_memory_totalsize = og_state_memory_totalsize +os_cpu_iowait = os_cpu_iowait +os_cpu_processor_number = os_cpu_processor_number +os_cpu_usage = os_cpu_usage +os_disk_iocapacity = os_disk_iocapacity +os_disk_iops = os_disk_iops +os_disk_ioutils = os_disk_ioutils +os_disk_usage = os_disk_usage +os_mem_usage = os_mem_usage +pg_active_slowsql_query_runtime = pg_active_slowsql_query_runtime +pg_activity_count = pg_activity_count +pg_activity_max_conn_duration = pg_activity_max_conn_duration +pg_activity_max_duration = pg_activity_max_duration +pg_activity_max_tx_duration = pg_activity_max_tx_duration +pg_boot_time = pg_boot_time +pg_checkpoint_checkpoint_lsn = pg_checkpoint_checkpoint_lsn +pg_checkpoint_elapse = pg_checkpoint_elapse +pg_checkpoint_full_page_writes = pg_checkpoint_full_page_writes +pg_checkpoint_newest_commit_ts_xid = pg_checkpoint_newest_commit_ts_xid +pg_checkpoint_next_multi_offset = pg_checkpoint_next_multi_offset +pg_checkpoint_next_multixact_id = pg_checkpoint_next_multixact_id +pg_checkpoint_next_oid = pg_checkpoint_next_oid +pg_checkpoint_next_xid = pg_checkpoint_next_xid +pg_checkpoint_next_xid_epoch = pg_checkpoint_next_xid_epoch +pg_checkpoint_oldest_active_xid = pg_checkpoint_oldest_active_xid +pg_checkpoint_oldest_commit_ts_xid = pg_checkpoint_oldest_commit_ts_xid +pg_checkpoint_oldest_multi_dbid = pg_checkpoint_oldest_multi_dbid +pg_checkpoint_oldest_multi_xid = pg_checkpoint_oldest_multi_xid +pg_checkpoint_oldest_xid = pg_checkpoint_oldest_xid +pg_checkpoint_oldest_xid_dbid = pg_checkpoint_oldest_xid_dbid +pg_checkpoint_prev_tli = pg_checkpoint_prev_tli +pg_checkpoint_redo_lsn = pg_checkpoint_redo_lsn +pg_checkpoint_time = pg_checkpoint_time +pg_checkpoint_tli = pg_checkpoint_tli +pg_class_relage = pg_class_relage +pg_class_relpages = pg_class_relpages +pg_class_relsize = pg_class_relsize +pg_class_reltuples = pg_class_reltuples +pg_conf_reload_time = pg_conf_reload_time +pg_connections_max_conn = pg_connections_max_conn +pg_connections_res_for_normal = pg_connections_res_for_normal +pg_connections_used_conn = pg_connections_used_conn +pg_database_age = pg_database_age +pg_database_allow_conn = pg_database_allow_conn +pg_database_conn_limit = pg_database_conn_limit +pg_database_frozen_xid = pg_database_frozen_xid +pg_database_is_template = pg_database_is_template +pg_database_size_bytes = pg_database_size_bytes +pg_db_blk_read_time = pg_db_blk_read_time +pg_db_blk_write_time = pg_db_blk_write_time +pg_db_blks_access = pg_db_blks_access +pg_db_blks_hit = pg_db_blks_hit +pg_db_blks_read = pg_db_blks_read +pg_db_confl_bufferpin = pg_db_confl_bufferpin +pg_db_confl_deadlock = pg_db_confl_deadlock +pg_db_confl_lock = pg_db_confl_lock +pg_db_confl_snapshot = pg_db_confl_snapshot +pg_db_confl_tablespace = pg_db_confl_tablespace +pg_db_conflicts = pg_db_conflicts +pg_db_deadlocks = pg_db_deadlocks +pg_db_numbackends = pg_db_numbackends +pg_db_stats_reset = pg_db_stats_reset +pg_db_temp_bytes = pg_db_temp_bytes +pg_db_temp_files = pg_db_temp_files +pg_db_tup_deleted = pg_db_tup_deleted +pg_db_tup_fetched = pg_db_tup_fetched +pg_db_tup_inserted = pg_db_tup_inserted +pg_db_tup_returned = pg_db_tup_returned +pg_db_tup_updated = pg_db_tup_updated +pg_db_xact_commit = pg_db_xact_commit +pg_db_xact_rollback = pg_db_xact_rollback +pg_downstream_count = pg_downstream_count +pg_flush_lsn = pg_flush_lsn +pg_index_idx_blks_hit = pg_index_idx_blks_hit +pg_index_idx_blks_read = pg_index_idx_blks_read +pg_index_idx_scan = pg_index_idx_scan +pg_index_idx_tup_fetch = pg_index_idx_tup_fetch +pg_index_idx_tup_read = pg_index_idx_tup_read +pg_insert_lsn = pg_insert_lsn +pg_is_in_recovery = pg_is_in_recovery +pg_is_wal_replay_paused = pg_is_wal_replay_paused +pg_lag = pg_lag +pg_last_replay_time = pg_last_replay_time +pg_lock_count = pg_lock_count +pg_lock_sql_locked_times = pg_lock_sql_locked_times +pg_locker_count = pg_locker_count +pg_lsn = pg_lsn +pg_meta_info = pg_meta_info +pg_need_indexes_idx_scan = pg_need_indexes_idx_scan +pg_need_indexes_idx_tup_fetch = pg_need_indexes_idx_tup_fetch +pg_need_indexes_rate = pg_need_indexes_rate +pg_need_indexes_seq_scan = pg_need_indexes_seq_scan +pg_need_indexes_seq_tup_read = pg_need_indexes_seq_tup_read +pg_never_used_indexes_index_size = pg_never_used_indexes_index_size +pg_node_info_uptime = pg_node_info_uptime +pg_receive_lsn = pg_receive_lsn +pg_replay_lsn = pg_replay_lsn +pg_replication_slots_active = pg_replication_slots_active +pg_replication_slots_delay_lsn = pg_replication_slots_delay_lsn +pg_run_times_db_role = pg_run_times_db_role +pg_run_times_run_time = pg_run_times_run_time +pg_session_connection_count = pg_session_connection_count +pg_setting_block_size = pg_setting_block_size +pg_setting_max_connections = pg_setting_max_connections +pg_setting_max_locks_per_transaction = pg_setting_max_locks_per_transaction +pg_setting_max_prepared_transactions = pg_setting_max_prepared_transactions +pg_setting_max_replication_slots = pg_setting_max_replication_slots +pg_setting_max_wal_senders = pg_setting_max_wal_senders +pg_setting_wal_log_hints = pg_setting_wal_log_hints +pg_settings_setting = pg_settings_setting +pg_sql_statement_full_count = pg_sql_statement_full_count +pg_sql_statement_history_exc_time = pg_sql_statement_history_exc_time +pg_table_analyze_count = pg_table_analyze_count +pg_table_analyze_delay = pg_table_analyze_delay +pg_table_autoanalyze_count = pg_table_autoanalyze_count +pg_table_autovacuum_count = pg_table_autovacuum_count +pg_table_heap_blks_hit = pg_table_heap_blks_hit +pg_table_heap_blks_read = pg_table_heap_blks_read +pg_table_idx_blks_hit = pg_table_idx_blks_hit +pg_table_idx_blks_read = pg_table_idx_blks_read +pg_table_idx_scan = pg_table_idx_scan +pg_table_idx_tup_fetch = pg_table_idx_tup_fetch +pg_table_n_dead_tup = pg_table_n_dead_tup +pg_table_n_live_tup = pg_table_n_live_tup +pg_table_n_mod_since_analyze = pg_table_n_mod_since_analyze +pg_table_n_tup_del = pg_table_n_tup_del +pg_table_n_tup_hot_upd = pg_table_n_tup_hot_upd +pg_table_n_tup_ins = pg_table_n_tup_ins +pg_table_n_tup_mod = pg_table_n_tup_mod +pg_table_n_tup_upd = pg_table_n_tup_upd +pg_table_seq_scan = pg_table_seq_scan +pg_table_seq_tup_read = pg_table_seq_tup_read +pg_table_tbl_scan = pg_table_tbl_scan +pg_table_tidx_blks_hit = pg_table_tidx_blks_hit +pg_table_tidx_blks_read = pg_table_tidx_blks_read +pg_table_toast_blks_hit = pg_table_toast_blks_hit +pg_table_toast_blks_read = pg_table_toast_blks_read +pg_table_tup_read = pg_table_tup_read +pg_table_vacuum_count = pg_table_vacuum_count +pg_table_vacuum_delay = pg_table_vacuum_delay +pg_tables_expansion_rate_analyze_count = pg_tables_expansion_rate_analyze_count +pg_tables_expansion_rate_autoanalyze_count = pg_tables_expansion_rate_autoanalyze_count +pg_tables_expansion_rate_autovacuum_count = pg_tables_expansion_rate_autovacuum_count +pg_tables_expansion_rate_dead_rate = pg_tables_expansion_rate_dead_rate +pg_tables_expansion_rate_vacuum_count = pg_tables_expansion_rate_vacuum_count +pg_tables_size_bytes = pg_tables_size_bytes +pg_tables_size_indexsize = pg_tables_size_indexsize +pg_tables_size_relsize = pg_tables_size_relsize +pg_tables_size_toastsize = pg_tables_size_toastsize +pg_thread_pool_listener = pg_thread_pool_listener +pg_timestamp = pg_timestamp +pg_uptime = pg_uptime +pg_wait_events_total_wait_time = pg_wait_events_total_wait_time +pg_wait_events_wait = pg_wait_events_wait +pg_write_lsn = pg_write_lsn +process_cpu_seconds_total = process_cpu_seconds_total +process_max_fds = process_max_fds +process_open_fds = process_open_fds +process_resident_memory_bytes = process_resident_memory_bytes +process_start_time_seconds = process_start_time_seconds +process_virtual_memory_bytes = process_virtual_memory_bytes +process_virtual_memory_max_bytes = process_virtual_memory_max_bytes +statement_responsetime_percentile_p80 = statement_responsetime_percentile_p80 +statement_responsetime_percentile_p95 = statement_responsetime_percentile_p95 + + diff --git a/src/gausskernel/dbmind/tools/service/dai.py b/src/gausskernel/dbmind/tools/service/dai.py index 48a3617cf..78df0e0f2 100644 --- a/src/gausskernel/dbmind/tools/service/dai.py +++ b/src/gausskernel/dbmind/tools/service/dai.py @@ -28,6 +28,11 @@ from dbmind.common.types import Sequence from dbmind.common.types.misc import SlowQuery from dbmind.metadatabase import dao + +# Singleton pattern with starving formula +# will capture exception in the main thread. +TsdbClientFactory.get_tsdb_client() + # Notice: 'DISTINGUISHING_INSTANCE_LABEL' is a magic string, i.e., our own name. # Thus, not all collection agents (such as Prometheus's openGauss-exporter) # distinguish different instance addresses through this one. diff --git a/src/gausskernel/dbmind/tools/tests/test_daemon.py b/src/gausskernel/dbmind/tools/tests/test_daemon.py index 21005f2ce..9419a317e 100644 --- a/src/gausskernel/dbmind/tools/tests/test_daemon.py +++ b/src/gausskernel/dbmind/tools/tests/test_daemon.py @@ -20,7 +20,7 @@ import dbmind.common.process from dbmind.common.daemon import Daemon from dbmind.common.platform import WIN32 -BASEPATH = os.path.abspath(os.path.dirname(__file__)) +BASEPATH = os.path.realpath(os.path.dirname(__file__)) PID_NAME = 'tester.pid' diff --git a/src/gausskernel/dbmind/tools/tests/test_utils.py b/src/gausskernel/dbmind/tools/tests/test_utils.py index 7cae1e569..d889d64a0 100644 --- a/src/gausskernel/dbmind/tools/tests/test_utils.py +++ b/src/gausskernel/dbmind/tools/tests/test_utils.py @@ -16,7 +16,7 @@ import os from dbmind.constants import METRIC_MAP_CONFIG, MISC_PATH from dbmind.common import utils -CURR_DIR = os.path.abspath(os.path.dirname(__file__)) +CURR_DIR = os.path.realpath(os.path.dirname(__file__)) def test_read_simple_conf_file(): diff --git a/src/gausskernel/optimizer/commands/cluster.cpp b/src/gausskernel/optimizer/commands/cluster.cpp index 7d66f8509..57801a4e4 100755 --- a/src/gausskernel/optimizer/commands/cluster.cpp +++ b/src/gausskernel/optimizer/commands/cluster.cpp @@ -1920,6 +1920,15 @@ double CopyUHeapDataInternal(Relation oldHeap, Relation oldIndex, Relation newHe return tups_vacuumed; } +static inline bool tuple_invisible_not_hotupdate(HeapTuple tuple, Relation relation) +{ + if (HeapKeepInvisibleTuple(tuple, RelationGetDescr(relation)) && !HeapTupleIsHotUpdated(tuple)) { + return false; + } else { + return true; + } +} + double copy_heap_data_internal(Relation OldHeap, Relation OldIndex, Relation NewHeap, TransactionId OldestXmin, TransactionId FreezeXid, bool verbose, bool use_sort, AdaptMem* memUsage) { @@ -2071,7 +2080,7 @@ double copy_heap_data_internal(Relation OldHeap, Relation OldIndex, Relation New switch (HeapTupleSatisfiesVacuum(tuple, OldestXmin, buf)) { case HEAPTUPLE_DEAD: /* Definitely dead */ - isdead = true; + isdead = tuple_invisible_not_hotupdate(tuple, OldHeap); break; case HEAPTUPLE_RECENTLY_DEAD: tups_recently_dead += 1; diff --git a/src/gausskernel/optimizer/commands/explain.cpp b/src/gausskernel/optimizer/commands/explain.cpp index f9686f447..f884f6ef4 100755 --- a/src/gausskernel/optimizer/commands/explain.cpp +++ b/src/gausskernel/optimizer/commands/explain.cpp @@ -454,6 +454,15 @@ void ExplainQuery( */ if (es.format == EXPLAIN_FORMAT_TEXT) appendStringInfoString(es.str, "Query rewrites to nothing\n"); + + /* + * In centralized mode, non-stream plans only support EXPLAIN_NORMAL. + * So set explain_perf_mode to EXPLAIN_NORMAL here. + */ + if (t_thrd.explain_cxt.explain_perf_mode != EXPLAIN_NORMAL && + !(IS_STREAM_PLAN && u_sess->exec_cxt.under_stream_runtime)) { + t_thrd.explain_cxt.explain_perf_mode = EXPLAIN_NORMAL; + } } else { ListCell* l = NULL; diff --git a/src/gausskernel/optimizer/commands/tablecmds.cpp b/src/gausskernel/optimizer/commands/tablecmds.cpp index e303a4d1b..4686ea5fb 100644 --- a/src/gausskernel/optimizer/commands/tablecmds.cpp +++ b/src/gausskernel/optimizer/commands/tablecmds.cpp @@ -6159,6 +6159,11 @@ void renamePartition(RenameStmt* stmt) /* Do the work */ renamePartitionInternal(partitionedTableOid, partitionOid, stmt->newname); + + Relation parentRel = heap_openrv(stmt->relation, AccessExclusiveLock); + Oid relid = RelationGetRelid(parentRel); + UpdatePgObjectChangecsn(relid, parentRel->rd_rel->relkind); + heap_close(parentRel, NoLock); } /* @@ -6232,6 +6237,11 @@ void renamePartitionIndex(RenameStmt* stmt) /* Do the work */ renamePartitionInternal(partitionedTableIndexOid, partitionIndexOid, stmt->newname); + + Oid parRelOid = IndexGetRelation(partitionedTableIndexOid, false); + Relation partRel = RelationIdGetRelation(parRelOid); + UpdatePgObjectChangecsn(parRelOid, partRel->rd_rel->relkind); + RelationClose(partRel); } /* @@ -6993,7 +7003,7 @@ static LOCKMODE GetPartitionLockLevel(AlterTableType subType) case AT_DropSubPartition: case AT_ExchangePartition: case AT_TruncatePartition: - cmdLockMode = RowExclusiveLock; + cmdLockMode = ShareUpdateExclusiveLock; break; default: cmdLockMode = AccessExclusiveLock; @@ -20816,29 +20826,36 @@ static void renamePartitionIndexes(Oid partitionedTableOid, Oid partitionOid, ch * Description : * Notes : */ -static void heap_truncate_one_part_new(const AlterTableCmd* cmd, Relation rel, Oid srcPartOid) { +static void heap_truncate_one_part_new(const AlterTableCmd* cmd, Relation partRel, Oid srcPartOid, + Relation rel = InvalidRelation) +{ Partition srcPart = NULL; bool renameTargetPart = false; char* destPartitionName = NULL; - Oid destPartOid = AddTemporaryPartitionForAlterPartitions(cmd, rel, srcPartOid, &renameTargetPart); + Oid destPartOid = AddTemporaryPartitionForAlterPartitions(cmd, partRel, srcPartOid, &renameTargetPart); - List* indexList = RelationGetSpecificKindIndexList(rel, false); + List* indexList = NULL; + if (RelationIsPartitionOfSubPartitionTable(partRel) && RelationIsValid(rel)) { + indexList = RelationGetSpecificKindIndexList(rel, false); + } else { + indexList = RelationGetSpecificKindIndexList(partRel, false); + } char** partitionIndexNames = getPartitionIndexesName(srcPartOid, indexList); - srcPart = partitionOpen(rel, srcPartOid, AccessExclusiveLock); + srcPart = partitionOpen(partRel, srcPartOid, AccessExclusiveLock); destPartitionName = pstrdup(PartitionGetPartitionName(srcPart)); - partitionClose(rel, srcPart, NoLock); + partitionClose(partRel, srcPart, NoLock); CommandCounterIncrement(); - fastDropPartition(rel, srcPartOid, "TRUNCATE PARTITION"); + fastDropPartition(partRel, srcPartOid, "TRUNCATE PARTITION"); CommandCounterIncrement(); - renamePartitionIndexes(rel->rd_id, destPartOid, partitionIndexNames, indexList); + renamePartitionIndexes(partRel->rd_id, destPartOid, partitionIndexNames, indexList); if (renameTargetPart) { CommandCounterIncrement(); - renamePartitionInternal(rel->rd_id, destPartOid, destPartitionName); + renamePartitionInternal(partRel->rd_id, destPartOid, destPartitionName); } list_free_ext(indexList); @@ -20861,13 +20878,20 @@ static void ATExecTruncatePartitionForSubpartitionTable(Relation rel, Oid partOi } foreach (subPartOidCell, subPartOidList) { Oid subPartOid = lfirst_oid(subPartOidCell); + bool all_ubtree = true; - AlterSubPartitionedSetWaitCleanGPI(cmd->alterGPI, rel, partOid, subPartOid); if (cmd->alterGPI) { /* Delete subpartition tuples in GPI and add parent to pending vacuum list */ - DeleteGPITuplesForSubPartition(RelationGetRelid(rel), partOid, subPartOid); + all_ubtree = DeleteGPITuplesForSubPartition(RelationGetRelid(rel), partOid, subPartOid); + AlterSubPartitionedSetWaitCleanGPI(cmd->alterGPI, rel, partOid, subPartOid); + } + + if (all_ubtree) { + /* If no nbtree global index exists */ + heap_truncate_one_part(partRel, subPartOid); + } else { + heap_truncate_one_part_new(cmd, partRel, subPartOid, rel); } - heap_truncate_one_part(partRel, subPartOid); pgstat_report_truncate(subPartOid, partRel->rd_id, partRel->rd_rel->relisshared); } @@ -21029,6 +21053,7 @@ static void ATExecTruncateSubPartition(Relation rel, AlterTableCmd* cmd) List* oidList = NULL; List* relid = lappend_oid(NULL, rel->rd_id); Oid subPartOid = InvalidOid; + bool all_ubtree = true; oidList = heap_truncate_find_FKs(relid); if (PointerIsValid(oidList)) { @@ -21066,17 +21091,21 @@ static void ATExecTruncateSubPartition(Relation rel, AlterTableCmd* cmd) Partition part = partitionOpen(rel, partOid, AccessExclusiveLock); Relation partRel = partitionGetRelation(rel, part); - AlterSubPartitionedSetWaitCleanGPI(cmd->alterGPI, rel, partOid, subPartOid); - if (!cmd->alterGPI) { // Unusable Global Index ATUnusableGlobalIndex(rel); } else { /* Delete subpartition tuples in GPI and add parent to pending vacuum list */ - DeleteGPITuplesForSubPartition(RelationGetRelid(rel), partOid, subPartOid); + all_ubtree = DeleteGPITuplesForSubPartition(RelationGetRelid(rel), partOid, subPartOid); + AlterSubPartitionedSetWaitCleanGPI(cmd->alterGPI, rel, partOid, subPartOid); } - heap_truncate_one_part(partRel, subPartOid); + if (all_ubtree) { + /* If no nbtree global index exists */ + heap_truncate_one_part(partRel, subPartOid); + } else { + heap_truncate_one_part_new(cmd, partRel, subPartOid, rel); + } pgstat_report_truncate(subPartOid, partRel->rd_id, partRel->rd_rel->relisshared); releaseDummyRelation(&partRel); @@ -24406,11 +24435,11 @@ static char* GenTemporaryPartitionName(Relation partTableRel, int sequence) #ifndef ENABLE_MULTIPLE_NODES static Oid GetNewPartitionOid(Relation pgPartRel, Relation partTableRel, Node *partDef, Oid bucketOid, - bool *isTimestamptz, StorageType stype, Datum new_reloptions) + bool *isTimestamptz, StorageType stype, Datum new_reloptions, bool isSubpartition) { #else static Oid GetNewPartitionOid(Relation pgPartRel, Relation partTableRel, Node *partDef, - Oid bucketOid, bool *isTimestamptz, StorageType stype) + Oid bucketOid, bool *isTimestamptz, StorageType stype, bool isSubpartition) { Datum new_reloptions = (Datum)0; #endif @@ -24426,7 +24455,9 @@ static Oid GetNewPartitionOid(Relation pgPartRel, Relation partTableRel, Node *p (Datum)new_reloptions, isTimestamptz, stype, - AccessExclusiveLock); + AccessExclusiveLock, + NULL, + isSubpartition); break; case T_ListPartitionDefState: newPartOid = HeapAddListPartition(pgPartRel, @@ -24437,7 +24468,9 @@ static Oid GetNewPartitionOid(Relation pgPartRel, Relation partTableRel, Node *p partTableRel->rd_rel->relowner, (Datum)new_reloptions, isTimestamptz, - stype); + stype, + NULL, + isSubpartition); break; case T_HashPartitionDefState: newPartOid = HeapAddHashPartition(pgPartRel, @@ -24448,7 +24481,9 @@ static Oid GetNewPartitionOid(Relation pgPartRel, Relation partTableRel, Node *p partTableRel->rd_rel->relowner, (Datum)new_reloptions, isTimestamptz, - stype); + stype, + NULL, + isSubpartition); break; default: ereport(ERROR, @@ -24480,17 +24515,20 @@ static Oid AddTemporaryPartition(Relation partTableRel, Node* partDef) Datum rel_reloptions; Datum new_reloptions; List* old_reloptions = NIL; + bool isPartitionOfSubPartition = RelationIsPartitionOfSubPartitionTable(partTableRel); bool* isTimestamptz = CheckPartkeyHasTimestampwithzone(partTableRel); bucketOid = RelationGetBucketOid(partTableRel); pgPartRel = relation_open(PartitionRelationId, RowExclusiveLock); /* add new partition entry in pg_partition */ - tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(partTableRel->rd_id)); + Oid relOid = isPartitionOfSubPartition ? + ObjectIdGetDatum(partTableRel->parentId) : ObjectIdGetDatum(partTableRel->rd_id); + tuple = SearchSysCache1(RELOID, relOid); if (!HeapTupleIsValid(tuple)) { ereport(ERROR, (errcode(ERRCODE_CACHE_LOOKUP_FAILED), - (errmsg("cache lookup failed"), errdetail("cache lookup failed for relation %u", partTableRel->rd_id), + (errmsg("cache lookup failed"), errdetail("cache lookup failed for relation %u", relOid), errcause("The oid of the target relation is invalid."), erraction("Check whether the target relation is correct.")))); } @@ -24508,18 +24546,25 @@ static Oid AddTemporaryPartition(Relation partTableRel, Node* partDef) /* Temporary tables do not use segment-page */ #ifndef ENABLE_MULTIPLE_NODES newPartOid = GetNewPartitionOid(pgPartRel, partTableRel, partDef, bucketOid, - isTimestamptz, RelationGetStorageType(partTableRel), new_reloptions); + isTimestamptz, RelationGetStorageType(partTableRel), new_reloptions, isPartitionOfSubPartition); #else - newPartOid = GetNewPartitionOid( - pgPartRel, partTableRel, partDef, bucketOid, isTimestamptz, RelationGetStorageType(partTableRel)); + newPartOid = GetNewPartitionOid(pgPartRel, partTableRel, partDef, bucketOid, isTimestamptz, + RelationGetStorageType(partTableRel), isPartitionOfSubPartition); + #endif // We must bump the command counter to make the newly-created // partition tuple visible for opening. CommandCounterIncrement(); - addIndexForPartition(partTableRel, newPartOid); + if (isPartitionOfSubPartition) { + Relation rel = heap_open(partTableRel->parentId, AccessShareLock); + addIndexForPartition(rel, newPartOid); + heap_close(rel, NoLock); + } else { + addIndexForPartition(partTableRel, newPartOid); + } - addToastTableForNewPartition(partTableRel, newPartOid); + addToastTableForNewPartition(partTableRel, newPartOid, isPartitionOfSubPartition); // invalidate relation CacheInvalidateRelcache(partTableRel); diff --git a/src/gausskernel/optimizer/commands/user.cpp b/src/gausskernel/optimizer/commands/user.cpp index 28ca1f82b..634d0976d 100755 --- a/src/gausskernel/optimizer/commands/user.cpp +++ b/src/gausskernel/optimizer/commands/user.cpp @@ -6052,6 +6052,9 @@ static Datum gs_calculate_encrypted_sm3_password(const char* password, const cha Datum calculate_encrypted_password(bool is_encrypted, const char* password, const char* rolname, const char* salt_string) { + if (password == NULL || password[0] == '\0') { + ereport(ERROR, (errcode(ERRCODE_INVALID_PASSWORD), errmsg("The password could not be NULL."))); + } errno_t rc = EOK; char encrypted_md5_password[MD5_PASSWD_LEN + 1] = {0}; Datum datum_value; diff --git a/src/gausskernel/optimizer/commands/vacuum.cpp b/src/gausskernel/optimizer/commands/vacuum.cpp index 6c78fbdda..023fb1578 100644 --- a/src/gausskernel/optimizer/commands/vacuum.cpp +++ b/src/gausskernel/optimizer/commands/vacuum.cpp @@ -130,7 +130,8 @@ static void DropEmptyPartitionDirectories(Oid relid); static THR_LOCAL BufferAccessStrategy vac_strategy; static THR_LOCAL int elevel = -1; -static void UstoreVacuumGPIPartition(Oid relid, Relation rel); +static void UstoreVacuumMainPartitionGPIs(Relation onerel, const VacuumStmt* vacstmt, + LOCKMODE lockmode, BufferAccessStrategy bstrategy); static void vac_truncate_clog(TransactionId frozenXID, MultiXactId frozenMulti); static bool vacuum_rel(Oid relid, VacuumStmt* vacstmt, bool do_toast); @@ -2436,13 +2437,13 @@ static bool vacuum_rel(Oid relid, VacuumStmt* vacstmt, bool do_toast) * relation. */ - bool isUstoreGPI = RelationIsUstoreIndex(onerel) && RelationIsGlobalIndex(onerel); + bool isPartitionedUHeap = RelationIsUstoreFormat(onerel) && RelationIsPartitioned(onerel); if (onerel->rd_rel->relkind != RELKIND_RELATION && #ifdef ENABLE_MOT !(RelationIsForeignTable(onerel) && isMOTFromTblOid(onerel->rd_id)) && #endif onerel->rd_rel->relkind != RELKIND_MATVIEW && - onerel->rd_rel->relkind != RELKIND_TOASTVALUE && !isUstoreGPI) { + onerel->rd_rel->relkind != RELKIND_TOASTVALUE && !isPartitionedUHeap) { if (vacstmt->options & VACOPT_VERBOSE) messageLevel = VERBOSEMESSAGE; @@ -2755,16 +2756,16 @@ static bool vacuum_rel(Oid relid, VacuumStmt* vacstmt, bool do_toast) } } else if (vacuumMainPartition((uint32)(vacstmt->flags))) { pgstat_report_waitstatus_relname(STATE_VACUUM, get_nsp_relname(relid)); - GPIVacuumMainPartition(onerel, vacstmt, lmode, vac_strategy); - CBIVacuumMainPartition(onerel, vacstmt, lmode, vac_strategy); + if (isPartitionedUHeap) { + UstoreVacuumMainPartitionGPIs(onerel, vacstmt, lmode, vac_strategy); + } else { + GPIVacuumMainPartition(onerel, vacstmt, lmode, vac_strategy); + CBIVacuumMainPartition(onerel, vacstmt, lmode, vac_strategy); + } pgstat_report_vacuum(relid, InvalidOid, false, 0); } else { pgstat_report_waitstatus_relname(STATE_VACUUM, get_nsp_relname(relid)); - if (isUstoreGPI) { - UstoreVacuumGPIPartition(relid, onerel); - } else { - TableRelationVacuum(onerel, vacstmt, vac_strategy); - } + TableRelationVacuum(onerel, vacstmt, vac_strategy); } } (void)pgstat_report_waitstatus(oldStatus); @@ -4406,99 +4407,73 @@ static void GPIOpenGlobalIndexes(Relation onerel, LOCKMODE lockmode, int* nindex } -static void UstoreVacuumGPIPartition(Oid relid, Relation rel) +static void UstoreVacuumMainPartitionGPIs(Relation onerel, const VacuumStmt* vacstmt, + LOCKMODE lockmode, BufferAccessStrategy bstrategy) { - ScanKeyData skey[2]; - /* Open pg_index to find out the relation owning current GPI. */ - Relation pgIndex = heap_open(IndexRelationId, AccessShareLock); - ScanKeyInit(&skey[0], Anum_pg_index_indexrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); - SysScanDesc scanIndexes = systable_beginscan(pgIndex, IndexRelidIndexId, true, NULL, 1, skey); - HeapTuple indexTuple = systable_getnext(scanIndexes); - if (!HeapTupleIsValid(indexTuple)) { - systable_endscan(scanIndexes); - heap_close(pgIndex, NoLock); - return; + OidRBTree* invisibleParts = CreateOidRBTree(); + Oid parentOid = RelationGetRelid(onerel); + + if (vacstmt->options & VACOPT_VERBOSE) { + elevel = VERBOSEMESSAGE; + } else { + elevel = DEBUG2; } - /* Get the Oid of relation. */ - Form_pg_index indexTupleForm = (Form_pg_index)GETSTRUCT(indexTuple); - Oid indexOfRelOid = indexTupleForm->indrelid; - systable_endscan(scanIndexes); - heap_close(pgIndex, NoLock); - /* Here we first get the main partition of the relation, and then check its - * wait_cleanup_gpi flag is 'y' or 'n': - * 1. If wait_cleanup_gpi=y, which means we need to vacuum the GPI. - * 2. If wait_cleanup_gpi=n, which means we do not need to vacuum the GPI. - */ - Relation pgPartition = heap_open(PartitionRelationId, RowExclusiveLock); - ScanKeyInit(&skey[0], Anum_pg_partition_parttype, BTEqualStrategyNumber, - F_CHAREQ, CharGetDatum(PART_OBJ_TYPE_PARTED_TABLE)); - ScanKeyInit(&skey[1], Anum_pg_partition_parentid, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(indexOfRelOid)); - SysScanDesc scanParts = systable_beginscan(pgPartition, PartitionParentOidIndexId, - true, NULL, 2, skey); - HeapTuple partTuple = systable_getnext(scanParts); - if (!HeapTupleIsValid(partTuple)) { - systable_endscan(scanParts); - heap_close(pgPartition, NoLock); - return; - } - Form_pg_partition partTupleForm = (Form_pg_partition)GETSTRUCT(partTuple); - partTuple = SearchSysCache3(PARTPARTOID, PointerGetDatum(partTupleForm->relname.data), - CharGetDatum(PART_OBJ_TYPE_PARTED_TABLE), ObjectIdGetDatum(indexOfRelOid)); - systable_endscan(scanParts); - if (!HeapTupleIsValid(partTuple)) { - ereport(ERROR, - (errcode(ERRCODE_CACHE_LOOKUP_FAILED), - errmsg("cache lookup failed for partition %u", indexOfRelOid))); - } - /* Use PartitionInvisibleMetadataKeep to judge the wait_cleanup_gpi flag. */ - bool isNull = false; - Datum partOptions = fastgetattr(partTuple, Anum_pg_partition_reloptions, - RelationGetDescr(pgPartition), &isNull); - if (isNull || !PartitionInvisibleMetadataKeep(partOptions)) { - ReleaseSysCache(partTuple); - heap_close(pgPartition, NoLock); - return; - } - /* Find out the invisible parts of the relation. */ - OidRBTree *invisibleParts = CreateOidRBTree(); - if (ConditionalLockPartition(indexOfRelOid, ADD_PARTITION_ACTION, AccessShareLock, PARTITION_SEQUENCE_LOCK)) { - PartitionGetAllInvisibleParts(indexOfRelOid, &invisibleParts); - UnlockPartition(indexOfRelOid, ADD_PARTITION_ACTION, AccessShareLock, PARTITION_SEQUENCE_LOCK); + + /* Get invisable parts */ + if (ConditionalLockPartition(parentOid, ADD_PARTITION_ACTION, AccessShareLock, PARTITION_SEQUENCE_LOCK)) { + PartitionGetAllInvisibleParts(parentOid, &invisibleParts); + UnlockPartition(parentOid, ADD_PARTITION_ACTION, AccessShareLock, PARTITION_SEQUENCE_LOCK); } + /* In rbtree, rb_leftmost will return NULL if rbtree is empty. */ if (rb_leftmost(invisibleParts) == NULL) { DestroyOidRBTree(&invisibleParts); - ReleaseSysCache(partTuple); - heap_close(pgPartition, NoLock); return; } - /* Start the cleanup process and collect the info needed */ - IndexVacuumInfo ivinfo; - ivinfo.index = rel; - ivinfo.analyze_only = false; - ivinfo.estimated_count = false; - ivinfo.message_level = elevel; - ivinfo.num_heap_tuples = -1; - ivinfo.strategy = vac_strategy; - ivinfo.invisibleParts = invisibleParts; - /* Cleanup process of index */ - index_vacuum_cleanup(&ivinfo, NULL); - OidRBTree *cleanedParts = CreateOidRBTree(); + + vac_strategy = bstrategy; + + /* Open all global indexes of the main partition */ + Relation* iRel = NULL; + int nIndexes; + GPIOpenGlobalIndexes(onerel, lockmode, &nIndexes, &iRel); + Relation classRel = heap_open(RelationRelationId, RowExclusiveLock); + for (int i = 0; i < nIndexes; i++) { + /* Start the cleanup process and collect the info needed */ + IndexVacuumInfo ivinfo; + ivinfo.index = iRel[i]; + ivinfo.analyze_only = false; + ivinfo.estimated_count = false; + ivinfo.message_level = elevel; + ivinfo.num_heap_tuples = -1; + ivinfo.strategy = vac_strategy; + ivinfo.invisibleParts = invisibleParts; + + /* Cleanup process of index */ + index_bulk_delete(&ivinfo, NULL, NULL, NULL); + + index_close(iRel[i], lockmode); + } + heap_close(classRel, RowExclusiveLock); + + /* + * Before clearing the global partition index of a partition table, + * acquire a AccessShareLock on ADD_PARTITION_ACTION, and make sure that the interval partition + * creation process will not be performed concurrently. + */ + OidRBTree* cleanedParts = CreateOidRBTree(); OidRBTreeUnionOids(cleanedParts, invisibleParts); - if (ConditionalLockPartition(indexOfRelOid, ADD_PARTITION_ACTION, AccessShareLock, PARTITION_SEQUENCE_LOCK)) { - PartitionSetEnabledClean(indexOfRelOid, cleanedParts, invisibleParts, true); - UnlockPartition(indexOfRelOid, ADD_PARTITION_ACTION, AccessShareLock, PARTITION_SEQUENCE_LOCK); + if (ConditionalLockPartition(parentOid, ADD_PARTITION_ACTION, AccessShareLock, PARTITION_SEQUENCE_LOCK)) { + PartitionSetEnabledClean(parentOid, cleanedParts, invisibleParts, true); + UnlockPartition(parentOid, ADD_PARTITION_ACTION, AccessShareLock, PARTITION_SEQUENCE_LOCK); } else { /* Updates reloptions of cleanedParts in pg_partition after GPI vacuum is executed */ - PartitionSetEnabledClean(indexOfRelOid, cleanedParts, invisibleParts, false); + PartitionSetEnabledClean(parentOid, cleanedParts, invisibleParts, false); } + DestroyOidRBTree(&invisibleParts); DestroyOidRBTree(&cleanedParts); - /* Set wait_cleanup_gpi=n after we finish the vacuum cleanup. */ - UpdateWaitCleanGpiRelOptions(pgPartition, partTuple, false, true); - ReleaseSysCache(partTuple); - heap_close(pgPartition, NoLock); + pfree_ext(iRel); } // vacuum main partition table to delete invisible tuple in global partition index diff --git a/src/gausskernel/optimizer/commands/vacuumlazy.cpp b/src/gausskernel/optimizer/commands/vacuumlazy.cpp index ebef225da..842b30741 100644 --- a/src/gausskernel/optimizer/commands/vacuumlazy.cpp +++ b/src/gausskernel/optimizer/commands/vacuumlazy.cpp @@ -1051,9 +1051,11 @@ static IndexBulkDeleteResult** lazy_scan_heap( OffsetNumber offnum, maxoff; bool tupgone = false; bool hastup = false; - bool keepThisInvisbleTuple = false; + bool keepThisInvisibleTuple = false; int prev_dead_count; + OffsetNumber invalid[MaxOffsetNumber]; OffsetNumber frozen[MaxOffsetNumber]; + int ninvalid = 0; int nfrozen; Size freespace; bool all_visible_according_to_vm = false; @@ -1335,7 +1337,7 @@ static IndexBulkDeleteResult** lazy_scan_heap( tuple.t_bucketId = RelationGetBktid(onerel); HeapTupleCopyBaseFromPage(&tuple, page); tupgone = false; - keepThisInvisbleTuple = false; + keepThisInvisibleTuple = false; if (u_sess->attr.attr_storage.enable_debug_vacuum) t_thrd.utils_cxt.pRelatedRel = onerel; @@ -1358,8 +1360,8 @@ static IndexBulkDeleteResult** lazy_scan_heap( * cheaper to get rid of it in the next pruning pass than * to treat it like an indexed tuple. */ - keepThisInvisbleTuple = HeapKeepInvisbleTuple(&tuple, RelationGetDescr(onerel)); - if (HeapTupleIsHotUpdated(&tuple) || HeapTupleIsHeapOnly(&tuple) || keepThisInvisbleTuple) { + keepThisInvisibleTuple = HeapKeepInvisibleTuple(&tuple, RelationGetDescr(onerel)); + if (HeapTupleIsHotUpdated(&tuple) || HeapTupleIsHeapOnly(&tuple) || keepThisInvisibleTuple) { nkeep += 1; } else { tupgone = true; /* we can delete the tuple */ @@ -1437,8 +1439,15 @@ static IndexBulkDeleteResult** lazy_scan_heap( tups_vacuumed += 1; has_dead_tuples = true; - } else if (keepThisInvisbleTuple) { - vacrelstats->hasKeepInvisbleTuples = true; + } else if (keepThisInvisibleTuple) { + if (t_thrd.proc->workingVersionNum >= INVALID_INVISIBLE_TUPLE_VERSION + && !HeapTupleIsHotUpdated(&tuple)) { + heap_invalid_invisible_tuple(&tuple); + Assert(tuple.t_tableOid == PartitionRelationId); + invalid[ninvalid++] = offnum; + } else { + vacrelstats->hasKeepInvisbleTuples = true; + } } else { num_tuples += 1; hastup = true; @@ -1478,6 +1487,23 @@ static IndexBulkDeleteResult** lazy_scan_heap( } } + if (ninvalid > 0) { + START_CRIT_SECTION(); + MarkBufferDirty(buf); + if (RelationNeedsWAL(onerel)) { + XLogRecPtr recptr; + + recptr = log_heap_invalid(onerel, buf, u_sess->cmd_cxt.FreezeLimit, + invalid, ninvalid); + PageSetLSN(page, recptr); + } + END_CRIT_SECTION(); + if (TransactionIdPrecedes(((HeapPageHeader)page)->pd_xid_base, u_sess->utils_cxt.RecentXmin)) { + if (u_sess->utils_cxt.RecentXmin - ((HeapPageHeader)page)->pd_xid_base > CHANGE_XID_BASE) + (void)heap_change_xidbase_after_freeze(onerel, buf); + } + } + /* * If there are no indexes then we can vacuum the page right now * instead of doing a second scan. diff --git a/src/gausskernel/optimizer/path/allpaths.cpp b/src/gausskernel/optimizer/path/allpaths.cpp index 51dc7d2ab..6c5435d35 100755 --- a/src/gausskernel/optimizer/path/allpaths.cpp +++ b/src/gausskernel/optimizer/path/allpaths.cpp @@ -873,6 +873,31 @@ static void SetPlainReSizeWithPruningRatio(RelOptInfo *rel, double pruningRatio) } } +/* + * This function applies only to single partition key of range partitioned tables in PBE mode. + */ +static bool IsPbeSinglePartition(Relation rel, RelOptInfo* relInfo) +{ + if (relInfo->pruning_result->paramArg == NULL) { + return false; + } + if (RelationIsSubPartitioned(rel)) { + return false; + } + if (rel->partMap->type != PART_TYPE_RANGE) { + return false; + } + RangePartitionMap* partMap = (RangePartitionMap*)rel->partMap; + int partKeyNum = partMap->partitionKey->dim1; + if (partKeyNum > 1) { + return false; + } + if (relInfo->pruning_result->isPbeSinlePartition) { + return true; + } + return false; +} + /* * set_plain_rel_size * Set size estimates for a plain relation (no subquery, no inheritance) @@ -896,8 +921,7 @@ static void set_plain_rel_size(PlannerInfo* root, RelOptInfo* rel, RangeTblEntry Assert(rel->pruning_result); - - if (rel->pruning_result->expr != NULL) { + if (IsPbeSinglePartition(relation, rel)) { rel->partItrs = 1; } else { /* set flag for dealing with partintioned table */ diff --git a/src/gausskernel/optimizer/plan/createplan.cpp b/src/gausskernel/optimizer/plan/createplan.cpp index a55d76cd5..9ada54aa5 100755 --- a/src/gausskernel/optimizer/plan/createplan.cpp +++ b/src/gausskernel/optimizer/plan/createplan.cpp @@ -222,6 +222,8 @@ static bool relIsDeltaNode(PlannerInfo* root, RelOptInfo* relOptInfo); static void ModifyWorktableWtParam(Node* planNode, int oldWtParam, int newWtParam); +static bool ScanQualsViolateNotNullConstr(PlannerInfo* root, RelOptInfo* rel, Path* best_path); + #define SATISFY_INFORMATIONAL_CONSTRAINT(joinPlan, joinType) \ (u_sess->attr.attr_sql.enable_constraint_optimization && true == innerPlan((joinPlan))->hasUniqueResults && \ JOIN_SEMI != (joinType) && JOIN_ANTI != (joinType)) @@ -752,11 +754,49 @@ static Plan* create_scan_plan(PlannerInfo* root, Path* best_path) */ if (root->hasPseudoConstantQuals) { plan = create_gating_plan(root, plan, scan_clauses); + } else if (ScanQualsViolateNotNullConstr(root, rel, best_path)) { + /* + * If there is IS NULL qual on a known NOT-NULL attribute, insert a Result node to prevent needless execution. + */ + plan = (Plan*)make_result(root, plan->targetlist, (Node*)list_make1(makeBoolConst(false, false)), plan); } return plan; } +static bool IsScanPath(NodeTag type) +{ + return ( + type == T_CStoreScan || type == T_CStoreIndexScan || type == T_CStoreIndexHeapScan || type == T_SeqScan || + type == T_DfsScan || type == T_IndexScan || type == T_IndexOnlyScan || type == T_BitmapHeapScan + ); +} + +static bool ScanQualsViolateNotNullConstr(PlannerInfo* root, RelOptInfo* rel, Path* best_path) +{ + /* For now, we only support table scan optimization */ + if (rel->rtekind != RTE_RELATION || !IsScanPath(best_path->pathtype)) { + return false; + } + + List* scan_clauses = rel->baserestrictinfo; + ListCell* lc = NULL; + foreach (lc, scan_clauses) { + Node* clause = (Node*)lfirst(lc); + if (IsA(clause, RestrictInfo) && IsA(((RestrictInfo*)clause)->clause, NullTest)) { + NullTest* expr = (NullTest*)((RestrictInfo*)clause)->clause; + /* For attribute with NOT-NULL constraint, IS-NULL expression can be short-circuited */ + if (expr->nulltesttype != IS_NULL || !IsA(expr->arg, Var)) { + continue; + } + if (check_var_nonnullable(root->parse, (Node*)expr->arg)) { + return true; + } + } + } + return false; +} + /* * Build a target list (ie, a list of TargetEntry) for a relation. */ diff --git a/src/gausskernel/optimizer/plan/pgxcplan_single.cpp b/src/gausskernel/optimizer/plan/pgxcplan_single.cpp index 7e8d8a0b5..1f454fc17 100755 --- a/src/gausskernel/optimizer/plan/pgxcplan_single.cpp +++ b/src/gausskernel/optimizer/plan/pgxcplan_single.cpp @@ -67,6 +67,43 @@ #include "utils/fmgroids.h" #include "access/heapam.h" +/* + * Check whether the current statement supports Stream based on the status of 'context' and 'query'. + * If Stream is supported, a copy of the 'query' is returned as a backup in case generating a plan + * with Stream fails. + */ +static Query* check_shippable(bool *stream_unsupport, Query* query, shipping_context* context) +{ + if (u_sess->attr.attr_sql.rewrite_rule & PARTIAL_PUSH) { + *stream_unsupport = !context->query_shippable; + } else { + *stream_unsupport = !context->global_shippable; + } + + if (u_sess->attr.attr_sql.enable_dngather) { + u_sess->opt_cxt.is_dngather_support = !context->disable_dn_gather; + } else { + u_sess->opt_cxt.is_dngather_support = false; + } + + /* single node do not support parallel query in cursor */ + if (query->utilityStmt && IsA(query->utilityStmt, DeclareCursorStmt)) { + *stream_unsupport = true; + } + + if (*stream_unsupport || !IS_STREAM) { + output_unshipped_log(); + set_stream_off(); + } else { + /* + * make a copy of query, so we can retry to create an unshippable plan + * when we fail to generate a stream plan + */ + return (Query*)copyObject(query); + } + return NULL; +} + PlannedStmt* pgxc_planner(Query* query, int cursorOptions, ParamListInfo boundParams) { PlannedStmt* result = NULL; @@ -91,33 +128,7 @@ PlannedStmt* pgxc_planner(Query* query, int cursorOptions, ParamListInfo boundPa (void)stream_walker((Node*)query, (void*)(&context)); disable_unshipped_log(query, &context); - if (u_sess->attr.attr_sql.rewrite_rule & PARTIAL_PUSH) { - stream_unsupport = !context.query_shippable; - } else { - stream_unsupport = !context.global_shippable; - } - - if (u_sess->attr.attr_sql.enable_dngather) { - u_sess->opt_cxt.is_dngather_support = !context.disable_dn_gather; - } else { - u_sess->opt_cxt.is_dngather_support = false; - } - - /* single node do not support parallel query in cursor */ - if (query->utilityStmt && IsA(query->utilityStmt, DeclareCursorStmt)) { - stream_unsupport = true; - } - - if (stream_unsupport || !IS_STREAM) { - output_unshipped_log(); - set_stream_off(); - } else { - /* - * make a copy of query, so we can retry to create an unshippable plan - * when we fail to generate a stream plan - */ - re_query = (Query*)copyObject(query); - } + re_query = check_shippable(&stream_unsupport, query, &context); } else { errno_t sprintf_rc = sprintf_s(u_sess->opt_cxt.not_shipping_info->not_shipping_reason, NOTPLANSHIPPING_LENGTH, @@ -141,9 +152,26 @@ PlannedStmt* pgxc_planner(Query* query, int cursorOptions, ParamListInfo boundPa */ MemoryContext current_context = CurrentMemoryContext; ResourceOwner currentOwner = t_thrd.utils_cxt.CurrentResourceOwner; - ResourceOwner tempOwner = ResourceOwnerCreate(t_thrd.utils_cxt.CurrentResourceOwner, "pgxc_planner", - SESS_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_OPTIMIZER)); - t_thrd.utils_cxt.CurrentResourceOwner = tempOwner; + ResourceOwner tempOwner = NULL; + /* + * If the stream-plan is not used, the currentOwner is used to trace resources instead of + * applying for a temporary owner. This prevents the "memory temporarily unavailable" error + * caused by memory stacking. + */ + if (IS_STREAM_PLAN) { + /* + * If the stream-plan is used, a temporary owner is used to trace resources. This helps release + * resources in a unified manner when a stream-plan fails to be generated, preventing resource + * leakage. + */ + tempOwner = ResourceOwnerCreate(t_thrd.utils_cxt.CurrentResourceOwner, "pgxc_planner", + /* + * The memory context of the temporary owner must be the same as the currentOwner to ensure + * that they have the same lifecycle + */ + ResourceOwnerGetMemCxt(currentOwner)); + t_thrd.utils_cxt.CurrentResourceOwner = tempOwner; + } /* we need Coordinator for evaluation, invoke standard planner */ PG_TRY(); @@ -175,8 +203,19 @@ PlannedStmt* pgxc_planner(Query* query, int cursorOptions, ParamListInfo boundPa result->ng_use_planA = use_planA; ReSetNgQueryMem(result); } - /* release resource applied in standard_planner */ - t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; + /* If tempOwner is not NULL, the current plan is a stream-plan using the SMP technology. */ + if (tempOwner != NULL) { + /* + * When the stream-plan is successfully generated, the temporary owner tracks the + * resources opened during the plan generation. Now we put the resources of the + * stream-plan into the currentOwner for tracking, and release the tempOwner to + * further reduce the memory. This greatly avoid the "memory temporarily unavailable" + * error, caused by a large amount of SQLs being executed in a transaction/procedure. + */ + ResourceOwnerConcat(currentOwner, tempOwner); + t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; + ResourceOwnerDelete(tempOwner); + } } PG_CATCH(); { @@ -198,11 +237,13 @@ PlannedStmt* pgxc_planner(Query* query, int cursorOptions, ParamListInfo boundPa * Release resources applied in standard_planner, release the tempOwner and reinstate the currentOwner * before PG_RE_THROW(). */ - ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_BEFORE_LOCKS, false, false); - ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_LOCKS, false, false); - ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_AFTER_LOCKS, false, false); - t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; - ResourceOwnerDelete(tempOwner); + if (tempOwner != NULL) { + ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_BEFORE_LOCKS, false, false); + ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_LOCKS, false, false); + ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_AFTER_LOCKS, false, false); + t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; + ResourceOwnerDelete(tempOwner); + } MemoryContextSwitchTo(ecxt); PG_RE_THROW(); } @@ -216,11 +257,13 @@ PlannedStmt* pgxc_planner(Query* query, int cursorOptions, ParamListInfo boundPa } /* release resource applied in standard_planner of the PG_TRY. */ - ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_BEFORE_LOCKS, false, false); - ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_LOCKS, false, false); - ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_AFTER_LOCKS, false, false); - t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; - ResourceOwnerDelete(tempOwner); + if (tempOwner != NULL) { + ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_BEFORE_LOCKS, false, false); + ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_LOCKS, false, false); + ResourceOwnerRelease(tempOwner, RESOURCE_RELEASE_AFTER_LOCKS, false, false); + t_thrd.utils_cxt.CurrentResourceOwner = currentOwner; + ResourceOwnerDelete(tempOwner); + } #ifdef STREAMPLAN if (OidIsValid(lc_replan_nodegroup)) { diff --git a/src/gausskernel/optimizer/prep/prepjointree.cpp b/src/gausskernel/optimizer/prep/prepjointree.cpp index 01993edef..ff6b31939 100755 --- a/src/gausskernel/optimizer/prep/prepjointree.cpp +++ b/src/gausskernel/optimizer/prep/prepjointree.cpp @@ -107,8 +107,10 @@ static Node *pull_up_sublinks_targetlist(PlannerInfo *root, Node *node, Node *jtnode, Relids *relids, Node **newTargetList, Node *whereQuals); + #ifndef ENABLE_MULTIPLE_NODES static bool find_rownum_in_quals(PlannerInfo *root); +static bool contains_swctes(const PlannerInfo *root); #endif /* @@ -154,6 +156,34 @@ void replace_empty_jointree(Query *parse) parse->jointree->fromlist = list_make1(rtr); } +#ifndef ENABLE_MULTIPLE_NODES +/* + * helper function to check if SWCB ctes contaisn in current SubQuery, normally help us to + * idenfity if it is OK to appy SWCB related optimization steps + */ +static bool contains_swctes(const PlannerInfo *root) +{ + if (root->parse == NULL || root->parse->cteList == NIL) { + return false; + } + + List *cteList = root->parse->cteList; + ListCell *lc = NULL; + bool found = false; + foreach(lc, cteList) { + CommonTableExpr *cte = (CommonTableExpr *)lfirst(lc); + + /* check if cte from parse->ctelist is a swcb converted */ + if (cte->swoptions != NULL) { + found = true; + break; + } + } + + return found; +} +#endif + /* * pull_up_sublinks * Attempt to pull up ANY and EXISTS SubLinks to be treated as @@ -192,6 +222,11 @@ void pull_up_sublinks(PlannerInfo* root) if (find_rownum_in_quals(root)) { return; } + + /* check existance of SWCB converted */ + if (contains_swctes(root)) { + return; + } #endif /* Begin recursion through the jointree */ diff --git a/src/gausskernel/optimizer/util/learn/comm.cpp b/src/gausskernel/optimizer/util/learn/comm.cpp index 680299eeb..56c6e0fc1 100644 --- a/src/gausskernel/optimizer/util/learn/comm.cpp +++ b/src/gausskernel/optimizer/util/learn/comm.cpp @@ -409,6 +409,7 @@ bool TryConnectRemoteServer(AiEngineConnInfo* conninfo, char** buf) if (!CheckConnParams(conninfo)) { return false; } + bool exceptionCaught = false; PG_TRY(); { @@ -444,14 +445,19 @@ bool TryConnectRemoteServer(AiEngineConnInfo* conninfo, char** buf) } PG_CATCH(); { + exceptionCaught = true; t_thrd.int_cxt.ImmediateInterruptOK = immediateInterruptOKOld; DestoryAiHandle(connHandle); if (buf != NULL) { *buf = NULL; } - return false; + FlushErrorState(); } PG_END_TRY(); + if (exceptionCaught) { + return false; + } + if (buf != NULL) { *buf = pstrdup(connHandle->rec_buf); } diff --git a/src/gausskernel/optimizer/util/plancat.cpp b/src/gausskernel/optimizer/util/plancat.cpp index f04251166..1fe7d0151 100755 --- a/src/gausskernel/optimizer/util/plancat.cpp +++ b/src/gausskernel/optimizer/util/plancat.cpp @@ -316,7 +316,7 @@ static void acquireSamplesForPartitionedRelation( continue; } - part = partitionOpen(relation, partitionOid, lmode); + part = partitionOpen(relation, partitionOid, NoLock); currentPartPages = PartitionGetNumberOfBlocksInFork(relation, part, MAIN_FORKNUM, true); partitionClose(relation, part, lmode); diff --git a/src/gausskernel/optimizer/util/pruning.cpp b/src/gausskernel/optimizer/util/pruning.cpp index 2b1f51165..753b176b8 100644 --- a/src/gausskernel/optimizer/util/pruning.cpp +++ b/src/gausskernel/optimizer/util/pruning.cpp @@ -460,17 +460,21 @@ PruningResult* partitionPruningForExpr(PlannerInfo* root, RangeTblEntry* rte, Re context->pruningType = PruningPartition; if (rel->partMap != NULL && (rel->partMap->type == PART_TYPE_LIST || rel->partMap->type == PART_TYPE_HASH)) { + // for List/Hash partitioned table result = partitionEqualPruningWalker(rel->partMap->type, expr, context); } else { + // for Range/Interval partitioned table result = partitionPruningWalker(expr, context); } if (result->exprPart != NULL || result->paramArg != NULL) { Param* paramArg = (Param *)copyObject(result->paramArg); + bool isPbeSinlePartition = result->isPbeSinlePartition; destroyPruningResult(result); result = getFullPruningResult(rel); result->expr = expr; result->paramArg = paramArg; + result->isPbeSinlePartition = isPbeSinlePartition; return result; } /* Never happen, just to be self-contained */ @@ -535,10 +539,12 @@ PruningResult* partitionPruningWalker(Expr* expr, PruningContext* pruningCtx) result = makeNode(PruningResult); result->state = PRUNING_RESULT_FULL; } + result->isPbeSinlePartition = false; } break; default: { result = makeNode(PruningResult); result->state = PRUNING_RESULT_FULL; + result->isPbeSinlePartition = false; } break; } @@ -619,6 +625,7 @@ static PruningResult* partitionPruningFromBoolExpr(const BoolExpr* expr, Pruning if (expr->boolop == NOT_EXPR) { result = makeNode(PruningResult); result->state = PRUNING_RESULT_FULL; + result->isPbeSinlePartition = false; return result; } @@ -638,6 +645,7 @@ static PruningResult* partitionPruningFromBoolExpr(const BoolExpr* expr, Pruning break; case OR_EXPR: result = unionChildPruningResult(resultList, context); + result->isPbeSinlePartition = false; break; case NOT_EXPR: default: @@ -750,6 +758,7 @@ static PruningResult* partitionPruningFromNullTest(NullTest* expr, PruningContex } result->state = PRUNING_RESULT_SUBSET; + result->isPbeSinlePartition = true; result->bm_rangeSelectedPartitions = bms_make_singleton(partMap->rangeElementsNum - 1); @@ -832,6 +841,7 @@ static PruningResult* intersectChildPruningResult(const List* resultList, Prunin AssertEreport(iteratorResult, MOD_OPT, "iteratorResult context is NNULL."); if (iteratorResult->state == PRUNING_RESULT_EMPTY) { result->state = PRUNING_RESULT_EMPTY; + result->isPbeSinlePartition = false; return result; } else if (iteratorResult->state == PRUNING_RESULT_FULL) { continue; @@ -875,17 +885,22 @@ static PruningResult* intersectChildPruningResult(const List* resultList, Prunin if (BoundaryIsEmpty(result->boundary)) { result->state = PRUNING_RESULT_EMPTY; + result->isPbeSinlePartition = false; break; } result->state = PRUNING_RESULT_SUBSET; } + if (result->state != PRUNING_RESULT_EMPTY && iteratorResult->isPbeSinlePartition) { + result->isPbeSinlePartition = true; + } } if (PruningResultIsEmpty(result)) { destroyPruningResult(result); result = makeNode(PruningResult); result->state = PRUNING_RESULT_EMPTY; + result->isPbeSinlePartition = false; result->intervalOffset = -1; } @@ -981,6 +996,7 @@ static PruningResult* partitionPruningFromScalarArrayOpExpr if (T_Var != nodeTag(larg) || (T_ArrayExpr != nodeTag(rarg) && T_Const != nodeTag(rarg))) { result = makeNode(PruningResult); result->state = PRUNING_RESULT_FULL; + result->isPbeSinlePartition = false; return result; } @@ -1079,6 +1095,7 @@ static PruningResult* partitionPruningFromScalarArrayOpExpr } else { result = makeNode(PruningResult); result->state = PRUNING_RESULT_FULL; + result->isPbeSinlePartition = false; return result; } } @@ -1258,6 +1275,7 @@ static PruningResult* recordBoundaryFromOpExpr(const OpExpr* expr, PruningContex /* length of args MUST be 2 */ if (!PointerIsValid(expr) || list_length(expr->args) != 2 || !PointerIsValid(opName = get_opname(expr->opno))) { result->state = PRUNING_RESULT_FULL; + result->isPbeSinlePartition = false; return result; } @@ -1296,6 +1314,7 @@ static PruningResult* recordBoundaryFromOpExpr(const OpExpr* expr, PruningContex ((T_Const == nodeTag(rightArg) || T_Param == nodeTag(rightArg) || T_OpExpr == nodeTag(rightArg)) && T_Var == nodeTag(leftArg)))) { result->state = PRUNING_RESULT_FULL; + result->isPbeSinlePartition = false; return result; } @@ -1325,6 +1344,7 @@ static PruningResult* recordBoundaryFromOpExpr(const OpExpr* expr, PruningContex if (context->rte != NULL && context->rte->relid != context->relation->rd_id) { result->state = PRUNING_RESULT_FULL; + result->isPbeSinlePartition = false; return result; } } else { @@ -1334,6 +1354,7 @@ static PruningResult* recordBoundaryFromOpExpr(const OpExpr* expr, PruningContex paramArg != NULL || exprPart != NULL) { result->state = PRUNING_RESULT_FULL; + result->isPbeSinlePartition = false; return result; } } @@ -1351,25 +1372,32 @@ static PruningResult* recordBoundaryFromOpExpr(const OpExpr* expr, PruningContex if (exprPart != NULL) { if (!PartitionMapIsRange(partMap)) { result->state = PRUNING_RESULT_FULL; + result->isPbeSinlePartition = false; return result; } else { result->exprPart = exprPart; result->state = PRUNING_RESULT_SUBSET; + result->isPbeSinlePartition = false; return result; } } else if (paramArg != NULL) { if (paramArg->paramkind != PARAM_EXTERN || !PartitionMapIsRange(partMap)) { result->state = PRUNING_RESULT_FULL; + result->isPbeSinlePartition = false; return result; } else { result->paramArg = paramArg; result->state = PRUNING_RESULT_SUBSET; + if (0 == strcmp("=", opName)) { + result->isPbeSinlePartition = true; + } return result; } } if (constArg->constisnull) { result->state = PRUNING_RESULT_EMPTY; + result->isPbeSinlePartition = false; return result; } @@ -1377,6 +1405,7 @@ static PruningResult* recordBoundaryFromOpExpr(const OpExpr* expr, PruningContex result->boundary = makePruningBoundary(partKeyNum); boundary = result->boundary; + result->isPbeSinlePartition = false; /* decide the const is the top or bottom of boundary */ if ((0 == strcmp(">", opName) && rightArgIsConst) || (0 == strcmp("<", opName) && !rightArgIsConst)) { @@ -1409,6 +1438,7 @@ static PruningResult* recordBoundaryFromOpExpr(const OpExpr* expr, PruningContex boundary->state = PRUNING_RESULT_SUBSET; result->state = PRUNING_RESULT_SUBSET; + result->isPbeSinlePartition = true; } else if ((0 == strcmp("<=", opName) && rightArgIsConst) || (0 == strcmp(">=", opName) && !rightArgIsConst)) { boundary->maxClose[attrOffset] = true; boundary->max[attrOffset] = PointerGetDatum(constArg); diff --git a/src/gausskernel/process/postmaster/autovacuum.cpp b/src/gausskernel/process/postmaster/autovacuum.cpp index 3a3273ae9..a1764d695 100755 --- a/src/gausskernel/process/postmaster/autovacuum.cpp +++ b/src/gausskernel/process/postmaster/autovacuum.cpp @@ -2070,6 +2070,25 @@ static void do_autovacuum(void) continue; } + /* Here we skipped relation_support_autoavac() and relation_needs_vacanalyze() checks + * for Ustore partitioned tables + */ + bytea *rawRelopts = extractRelOptions(tuple, pg_class_desc, InvalidOid); + if (rawRelopts != NULL && RelationIsTableAccessMethodUStoreType(rawRelopts) && + isPartitionedRelation(classForm)) { + vacObj = (vacuum_object*)palloc(sizeof(vacuum_object)); + vacObj->tab_oid = relid; + vacObj->parent_oid = InvalidOid; + vacObj->dovacuum = true; + vacObj->dovacuum_toast = false; + vacObj->doanalyze = false; + vacObj->need_freeze = false; + vacObj->is_internal_relation = false; + vacObj->flags = VACFLG_MAIN_PARTITION; + table_oids = lappend(table_oids, vacObj); + continue; + } + /* Fetch reloptions for this table */ relopts = extract_autovac_opts(tuple, pg_class_desc); @@ -2176,7 +2195,6 @@ static void do_autovacuum(void) pfree_ext(relopts); } } - tableam_scan_end(relScan); DEBUG_MOD_STOP_TIMER(MOD_AUTOVAC, "AUTOVAC TIMER: Scan pg_class to determine which tables to vacuum"); @@ -2277,7 +2295,6 @@ static void do_autovacuum(void) } } } - /* Close the pg_partition */ tableam_scan_end(partScan); heap_close(partRel, AccessShareLock); @@ -2376,51 +2393,9 @@ static void do_autovacuum(void) pfree_ext(relopts); } } - - tableam_scan_end(relScan); - DEBUG_MOD_STOP_TIMER(MOD_AUTOVAC, "AUTOVAC TIMER: Scan pg_class to determine which toast tables to vacuum"); - - /* On the fourth pass: check USTORE GPI tables */ - ScanKeyInit(&key[0], Anum_pg_class_relkind, BTEqualStrategyNumber, F_CHAREQ, CharGetDatum(RELKIND_GLOBAL_INDEX)); - relScan = tableam_scan_begin(classRel, SnapshotNow, 1, &key[0]); - while ((tuple = (HeapTuple)tableam_scan_getnexttuple(relScan, ForwardScanDirection)) != NULL) { - Form_pg_class classForm = (Form_pg_class) GETSTRUCT(tuple); - Oid relid = HeapTupleGetOid(tuple); - AutoVacOpts *relopts = NULL; - /* - * We cannot safely process other backends' temp tables, so skip them. - */ - if (classForm->relpersistence == RELPERSISTENCE_TEMP || - classForm->relpersistence == RELPERSISTENCE_GLOBAL_TEMP) - continue; - /* only UBTree supports vacuum independently */ - if (classForm->relam != UBTREE_AM_OID) { - continue; - } - /* fetch reloptions */ - relopts = extract_autovac_opts(tuple, pg_class_desc); - /* only UBTree supports vacuum, and enabled will be set true */ - if (relopts == NULL || !relopts->enabled) { - continue; - } - /* we skipped relation_support_autoavac() and relation_needs_vacanalyze() checks here */ - vacObj = (vacuum_object*)palloc(sizeof(vacuum_object)); - vacObj->tab_oid = relid; - vacObj->parent_oid = InvalidOid; - vacObj->dovacuum = true; - vacObj->dovacuum_toast = false; - vacObj->doanalyze = false; - vacObj->need_freeze = false; - vacObj->is_internal_relation = false; - /* VACFLG_MAIN_PARTITION makes no sense when vacuuming UBTree */ - vacObj->flags = VACFLG_SIMPLE_HEAP; /* ignore this flag as we will not use it, - * and to be safe we can use VACFLG_SIMPLE_HEAP. - */ - table_oids = lappend(table_oids, vacObj); - } tableam_scan_end(relScan); heap_close(classRel, AccessShareLock); - DEBUG_MOD_STOP_TIMER(MOD_AUTOVAC, "AUTOVAC TIMER: Scan pg_class to determine which UBTree tables to vacuum"); + DEBUG_MOD_STOP_TIMER(MOD_AUTOVAC, "AUTOVAC TIMER: Scan pg_class to determine which toast tables to vacuum"); /* * Create one buffer access strategy object per buffer pool for VACUUM to use. @@ -2754,8 +2729,7 @@ AutoVacOpts* extract_autovac_opts(HeapTuple tup, TupleDesc pg_class_desc) Assert(((Form_pg_class)GETSTRUCT(tup))->relkind == RELKIND_RELATION || ((Form_pg_class) GETSTRUCT(tup))->relkind == RELKIND_MATVIEW || - ((Form_pg_class)GETSTRUCT(tup))->relkind == RELKIND_TOASTVALUE || - ((Form_pg_class)GETSTRUCT(tup))->relkind == RELKIND_GLOBAL_INDEX); + ((Form_pg_class)GETSTRUCT(tup))->relkind == RELKIND_TOASTVALUE); relopts = extractRelOptions(tup, pg_class_desc, InvalidOid); if (relopts == NULL) @@ -2765,15 +2739,9 @@ AutoVacOpts* extract_autovac_opts(HeapTuple tup, TupleDesc pg_class_desc) rc = memcpy_s(av, sizeof(AutoVacOpts), &(((StdRdOptions*)relopts)->autovacuum), sizeof(AutoVacOpts)); securec_check(rc, "\0", "\0"); - /* autovacuum for ustore table is disabled */ + /* autovacuum for ustore unpartitioned table is disabled */ if (RelationIsTableAccessMethodUStoreType(relopts)) { - av->enabled = false; - } - - /* force set av->enabled in ustore's GPI */ - if ((((Form_pg_class)GETSTRUCT(tup))->relkind == RELKIND_GLOBAL_INDEX) && - RelationIsTableAccessMethodUStoreType(relopts)) { - av->enabled = true; + av->enabled = isPartitionedRelation((Form_pg_class)GETSTRUCT(tup)); } pfree_ext(relopts); @@ -2898,8 +2866,10 @@ static autovac_table* table_recheck_autovac( if (!HeapTupleIsValid(classTup)) return NULL; classForm = (Form_pg_class)GETSTRUCT(classTup); - /* this is UBTree, use another bypass */ - if (classForm->relam == UBTREE_AM_OID) { + bytea *rawRelopts = extractRelOptions(classTup, pg_class_desc, InvalidOid); + /* this is Ustore partitioned table, use another bypass */ + if (rawRelopts != NULL && RelationIsTableAccessMethodUStoreType(rawRelopts) && + isPartitionedRelation(classForm)) { avopts = extract_autovac_opts(classTup, pg_class_desc); tab = calculate_vacuum_cost_and_freezeages(avopts, false, false); if (tab != NULL) { diff --git a/src/gausskernel/process/postmaster/barrier_creator.cpp b/src/gausskernel/process/postmaster/barrier_creator.cpp index 7c800cf4e..3715ee3c8 100755 --- a/src/gausskernel/process/postmaster/barrier_creator.cpp +++ b/src/gausskernel/process/postmaster/barrier_creator.cpp @@ -57,13 +57,14 @@ void GetCsnBarrierName(char* barrierRet, bool isSwitchoverBarrier) struct timeval tv; int rc; CommitSeqNo csn; - - gettimeofday(&tv, NULL); + if (GTM_MODE) csn = GetCSNGTM(); else csn = CommitCSNGTM(false); + gettimeofday(&tv, NULL); + if (isSwitchoverBarrier) { rc = snprintf_s(barrierRet, BARRIER_NAME_LEN, BARRIER_NAME_LEN - 1, CSN_SWITCHOVER_BARRIER_PATTREN_STR, csn); } else { @@ -206,6 +207,7 @@ uint64 GetObsFirstCNBarrierTimeline(const List *archiveSlotNames) return timeline; } +#ifdef ENABLE_MULTIPLE_NODES static void AllocBarrierLsnInfo(int nodeSize) { int rc; @@ -216,6 +218,7 @@ static void AllocBarrierLsnInfo(int nodeSize) sizeof(ArchiveBarrierLsnInfo) * nodeSize); securec_check(rc, "", ""); } +#endif #ifdef ENABLE_MULTIPLE_NODES static void BarrierCreatorPoolerReload(void) @@ -402,6 +405,7 @@ void barrier_creator_main(void) long time_diff = last_barrier_time - current_time; ereport(LOG, (errmsg("[BarrierCreator] current time %ld is smaller than barrier time %ld, and sleep %ld ms", current_time, last_barrier_time, time_diff))); + CHECK_FOR_INTERRUPTS(); pg_usleep(time_diff * 1000L); } while (1); if (t_thrd.barrier_creator_cxt.is_first_barrier) { @@ -410,6 +414,7 @@ void barrier_creator_main(void) } #ifdef ENABLE_MULTIPLE_NODES while (!START_AUTO_CSN_BARRIER) { + CHECK_FOR_INTERRUPTS(); pg_usleep(1000000L); } #endif @@ -436,6 +441,7 @@ void barrier_creator_main(void) t_thrd.barrier_creator_cxt.barrier_update_last_time_info = (BarrierUpdateLastTimeInfo*)palloc0( sizeof(BarrierUpdateLastTimeInfo) * g_instance.attr.attr_storage.max_replication_slots); } +#ifdef ENABLE_MULTIPLE_NODES if (g_instance.archive_obs_cxt.barrier_lsn_info == NULL) { int nodeSize = *t_thrd.pgxc_cxt.shmemNumCoords + *t_thrd.pgxc_cxt.shmemNumDataNodes; AllocBarrierLsnInfo(nodeSize); @@ -443,6 +449,7 @@ void barrier_creator_main(void) g_instance.archive_obs_cxt.max_node_cnt = nodeSize; SpinLockRelease(&g_instance.archive_obs_cxt.barrier_lock); } +#endif archiveSlotNames = GetAllArchiveSlotsName(); if (archiveSlotNames == NIL || archiveSlotNames->length == 0) { ereport(WARNING, (errmsg("[BarrierCreator] could not get archive slot name when barrier start"))); diff --git a/src/gausskernel/process/postmaster/barrier_preparse.cpp b/src/gausskernel/process/postmaster/barrier_preparse.cpp index b06e024fe..7ab25c93a 100644 --- a/src/gausskernel/process/postmaster/barrier_preparse.cpp +++ b/src/gausskernel/process/postmaster/barrier_preparse.cpp @@ -173,10 +173,11 @@ void BarrierPreParseMain(void) XLogReaderState *xlogreader = NULL; char *errormsg = NULL; XLogPageReadPrivate readprivate; - XLogRecPtr startLSN; - XLogRecPtr preStartLSN; + XLogRecPtr startLSN = InvalidXLogRecPtr; + XLogRecPtr preStartLSN = InvalidXLogRecPtr; + XLogRecPtr lastReadLSN = InvalidXLogRecPtr; bool found = false; - XLogRecPtr barrierLSN; + XLogRecPtr barrierLSN = InvalidXLogRecPtr; char *xLogBarrierId = NULL; char barrierId[MAX_BARRIER_ID_LENGTH] = {0}; const uint32 shiftSize = 32; @@ -258,10 +259,13 @@ void BarrierPreParseMain(void) found = false; preStartLSN = startLSN; + ereport(DEBUG1, (errmsg("[BarrierPreParse] start to preparse at: %08X/%08X", + (uint32)(startLSN >> shiftSize), (uint32)startLSN))); startLSN = XLogFindNextRecord(xlogreader, startLSN); if (XLogRecPtrIsInvalid(startLSN)) { startLSN = preStartLSN; - if (!XLByteEQ(walrcv->receiver_flush_location, startLSN)) { + if (!XLByteEQ(walrcv->receiver_flush_location, startLSN) && + !XLByteEQ(walrcv->lastRecoveredBarrierLSN, startLSN)) { /* reset startLSN */ startLSN = walrcv->lastRecoveredBarrierLSN; ereport(LOG, (errmsg("[BarrierPreParse] reset startLSN with lastRecoveredBarrierLSN: %08X/%08X", @@ -275,6 +279,7 @@ void BarrierPreParseMain(void) if (record == NULL) { break; } + lastReadLSN = xlogreader->EndRecPtr; uint8 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK; if (NEED_INSERT_INTO_HASH) { xLogBarrierId = XLogRecGetData(xlogreader); @@ -304,7 +309,7 @@ void BarrierPreParseMain(void) SetBarrieID(barrierId, barrierLSN); } - startLSN = XLogRecPtrIsInvalid(xlogreader->ReadRecPtr) ? preStartLSN : xlogreader->ReadRecPtr; + startLSN = XLogRecPtrIsInvalid(lastReadLSN) ? preStartLSN : lastReadLSN; if (XLogRecPtrIsInvalid(xlogreader->ReadRecPtr) && errormsg) { ereport(LOG, (errmsg("[BarrierPreParse] preparse thread get an error info %s", errormsg))); diff --git a/src/gausskernel/process/postmaster/pagewriter.cpp b/src/gausskernel/process/postmaster/pagewriter.cpp index 5c2cd512d..10db3afbb 100755 --- a/src/gausskernel/process/postmaster/pagewriter.cpp +++ b/src/gausskernel/process/postmaster/pagewriter.cpp @@ -1216,9 +1216,12 @@ static void HandlePageWriterMainInterrupts() ProcessConfigFile(PGC_SIGHUP); } - if (t_thrd.pagewriter_cxt.sync_requested) { + if (t_thrd.pagewriter_cxt.sync_requested || t_thrd.pagewriter_cxt.sync_retry) { t_thrd.pagewriter_cxt.sync_requested = false; + + t_thrd.pagewriter_cxt.sync_retry = true; PageWriterSyncWithAbsorption(); + t_thrd.pagewriter_cxt.sync_retry = false; } /* main thread should finally exit. */ diff --git a/src/gausskernel/process/postmaster/pgarch.cpp b/src/gausskernel/process/postmaster/pgarch.cpp index 285c81e19..1926735fc 100755 --- a/src/gausskernel/process/postmaster/pgarch.cpp +++ b/src/gausskernel/process/postmaster/pgarch.cpp @@ -425,6 +425,7 @@ static void pgarch_MainLoop(void) XLogRecPtr replayPtr; bool got_recptr = false; bool amSync = false; + int retryTimes = 3; /* FlushPtr <= ConsensusPtr on DCF mode */ if (IS_PGXC_COORDINATOR || g_instance.attr.attr_storage.dcf_attr.enable_dcf) { flushPtr = GetFlushRecPtr(); @@ -441,8 +442,16 @@ static void pgarch_MainLoop(void) "and init last lsn is %08X%08X", (uint32)(t_thrd.arch.pitr_task_last_lsn >> 32), (uint32)t_thrd.arch.pitr_task_last_lsn))); } - got_recptr = SyncRepGetSyncRecPtr(&receivePtr, &writePtr, &flushPtr, &replayPtr, &amSync, false); - if (got_recptr != true) { + while (retryTimes--) { + got_recptr = + SyncRepGetSyncRecPtr(&receivePtr, &writePtr, &flushPtr, &replayPtr, &amSync, false); + if (got_recptr == true) { + break; + } else { + pg_usleep(1000000L); + } + } + if (got_recptr == false) { ereport(ERROR, (errmsg("pgarch_ArchiverObsCopyLoop failed when call SyncRepGetSyncRecPtr"))); } @@ -739,6 +748,10 @@ static void pgarch_ArchiverObsCopyLoop(XLogRecPtr flushPtr, doArchive fun) (uint32)(targetLsn >> 32), (uint32)(targetLsn)))); pg_usleep(1000000L); /* wait a bit before retrying */ } else { + if (g_instance.roach_cxt.isXLogForceRecycled && !g_instance.roach_cxt.forceAdvanceSlotTigger) { + g_instance.roach_cxt.isXLogForceRecycled = false; + ereport(LOG, (errmsg("PgArch force advance slot success"))); + } gettimeofday(&tv, NULL); currTimestamp = TIME_GET_MILLISEC(tv); t_thrd.arch.pitr_task_last_lsn = targetLsn; @@ -1026,6 +1039,11 @@ static void pgarch_archiveRoachForPitrStandby() (uint32)(archive_task_status->archive_task.targetLsn), archive_task_status->archive_task.term, archive_task_status->archive_task.sub_term))); + if (archive_task_status->archive_task.targetLsn == InvalidXLogSegPtr) { + volatile unsigned int *pitr_task_status = &archive_task_status->pitr_task_status; + pg_atomic_write_u32(pitr_task_status, PITR_TASK_NONE); + ereport(LOG, (errmsg("PgArch standby receive invalid lsn for slot force advance"))); + } if (ArchiveReplicationAchiver(&archive_task_status->archive_task) == 0) { archive_task_status->pitr_finish_result = true; } else { @@ -1057,6 +1075,11 @@ static bool pgarch_archiveRoachForPitrMaster(XLogRecPtr targetLsn) archive_task_status->archive_task.tli = get_controlfile_timeline(); archive_task_status->archive_task.term = Max(g_instance.comm_cxt.localinfo_cxt.term_from_file, g_instance.comm_cxt.localinfo_cxt.term_from_xlog); + if (g_instance.roach_cxt.forceAdvanceSlotTigger) { + archive_task_status->archive_task.targetLsn = InvalidXLogRecPtr; + g_instance.roach_cxt.forceAdvanceSlotTigger = false; + ereport(LOG, (errmsg("PgArch need force advance this time in primary"))); + } /* subterm update when walsender changed */ int rc = strcpy_s(archive_task_status->archive_task.slot_name, NAMEDATALEN, t_thrd.arch.slot_name); securec_check(rc, "\0", "\0"); @@ -1085,8 +1108,8 @@ static bool pgarch_archiveRoachForPitrMaster(XLogRecPtr targetLsn) return false; } /* - * check targetLsn and g_instance.archive_obs_cxt.archive_task.targetLsn for deal message with wrong order - */ + * check targetLsn and g_instance.archive_obs_cxt.archive_task.targetLsn for deal message with wrong order + */ if (archive_task_status->pitr_finish_result == true && XLByteEQ(archive_task_status->archive_task.targetLsn, targetLsn)) { archive_task_status->pitr_finish_result = false; @@ -1191,6 +1214,26 @@ static WalSnd* pgarch_chooseWalsnd(XLogRecPtr targetLsn) return NULL; } +static XLogRecPtr GetLastTaskLsnFromServer(ArchiveSlotConfig* obs_archive_slot) +{ + ArchiveXlogMessage obs_archive_info; + XLogRecPtr pitr_task_last_lsn; + + if (archive_replication_get_last_xlog(&obs_archive_info, &obs_archive_slot->archive_config) == 0) { + pitr_task_last_lsn = obs_archive_info.targetLsn; + ereport(LOG, + (errmsg("initLastTaskLsn update lsn to %X/%X from server", (uint32)(pitr_task_last_lsn >> 32), + (uint32)(pitr_task_last_lsn)))); + } else { + XLogRecPtr targetLsn = GetFlushRecPtr(); + pitr_task_last_lsn = targetLsn - (targetLsn % XLogSegSize); + ereport(LOG, + (errmsg("initLastTaskLsn update lsn to %X/%X from local", (uint32)(pitr_task_last_lsn >> 32), + (uint32)(pitr_task_last_lsn)))); + } + return pitr_task_last_lsn; +} + static void InitArchiverLastTaskLsn(ArchiveSlotConfig* obs_archive_slot) { struct timeval tv; @@ -1207,8 +1250,19 @@ static void InitArchiverLastTaskLsn(ArchiveSlotConfig* obs_archive_slot) ReplicationSlot *slot = &t_thrd.slot_cxt.ReplicationSlotCtl->replication_slots[*slot_idx]; SpinLockAcquire(&slot->mutex); if (slot->in_use == true && slot->archive_config != NULL) { - t_thrd.arch.pitr_task_last_lsn = slot->data.restart_lsn; - SpinLockRelease(&slot->mutex); + /* + * In old version(<92599), the last task lsn is initialized from archive server or current flush + * position, but in new version is initialized from local slot. + * During the upgrade, the local restart lsn may be 0, so initialize it with old version way. + */ + if (slot->data.restart_lsn == InvalidXLogRecPtr && + t_thrd.proc->workingVersionNum < PITR_INIT_VERSION_NUM) { + SpinLockRelease(&slot->mutex); + t_thrd.arch.pitr_task_last_lsn = GetLastTaskLsnFromServer(obs_archive_slot); + } else { + t_thrd.arch.pitr_task_last_lsn = slot->data.restart_lsn; + SpinLockRelease(&slot->mutex); + } } else { SpinLockRelease(&slot->mutex); ereport(ERROR, (errcode_for_file_access(), errmsg("slot idx not valid, obs slot %X/%X not advance ", diff --git a/src/gausskernel/process/postmaster/pgaudit.cpp b/src/gausskernel/process/postmaster/pgaudit.cpp index 16f62980b..87a733645 100755 --- a/src/gausskernel/process/postmaster/pgaudit.cpp +++ b/src/gausskernel/process/postmaster/pgaudit.cpp @@ -203,7 +203,7 @@ typedef struct AuditIndexItem { * Brief : audit index table * Description : */ -typedef struct AuditIndexTable { +typedef struct AuditIndexTableNew { uint32 maxnum; /* max count of the audit index item */ uint32 begidx; /* the position of the first audit index item */ uint32 thread_num; /* the running audit thread num */ @@ -212,25 +212,25 @@ typedef struct AuditIndexTable { uint32 count; /* the count of the audit index item */ pg_time_t last_audit_time; /* the audit time of the latest audit record */ AuditIndexItem data[1]; -} AuditIndexTable; +} AuditIndexTableNew; /* * Brief : old audit index table * Description : */ -typedef struct AuditIndexTableOld { +typedef struct AuditIndexTable { uint32 maxnum; /* max count of the audit index item */ uint32 begidx; /* the position of the first audit index item */ uint32 curidx; /* the position of the current audit index item */ uint32 count; /* the count of the audit index item */ pg_time_t last_audit_time; /* the audit time of the latest audit record */ AuditIndexItem data[1]; -} AuditIndexTableOld; +} AuditIndexTable; static const char audit_indextbl_file[] = "index_table_new"; static const char audit_indextbl_old_file[] = "index_table"; -static const int indextbl_header_size = offsetof(AuditIndexTable, data); -static const int old_indextbl_header_size = offsetof(AuditIndexTableOld, data); +static const int indextbl_header_size = offsetof(AuditIndexTableNew, data); +static const int old_indextbl_header_size = offsetof(AuditIndexTable, data); static const char* AuditTypeDescs[] = {"unknown", "login_success", @@ -401,7 +401,6 @@ static void pgaudit_indexfile_upgrade(void); static void pgaudit_indexfile_sync(const char* mode, bool allow_errors); static void pgaudit_rewrite_indexfile(void); static void pgaudit_indextbl_init_new(void); -void extracted836(errno_t &errorno, int thread_num); static void pgaudit_reset_indexfile(); static const char* pgaudit_string_field(AuditData* adata, int num); static void deserialization_to_tuple(Datum (&values)[PGAUDIT_QUERY_COLS], @@ -808,11 +807,11 @@ void pgaudit_stop_all(void) { for (int i = 0; i < g_instance.audit_cxt.thread_num; ++i) { if (g_instance.pid_cxt.PgAuditPID[i] != 0) { + Assert(!dummyStandbyMode); signal_child(g_instance.pid_cxt.PgAuditPID[i], SIGQUIT, -1); } } audit_process_cxt_exit(); - ereport(LOG, (errmsg("parameter audit_enabled is set to false, terminate auditor process."))); } /* @@ -1471,7 +1470,6 @@ static void pgaudit_cleanup(void) if (g_instance.audit_cxt.audit_indextbl->count > 0) { --g_instance.audit_cxt.audit_indextbl->count; } - g_instance.audit_cxt.audit_indextbl->begidx = (index + 1) % g_instance.audit_cxt.audit_indextbl->maxnum; errorno = memset_s(item, sizeof(AuditIndexItem), 0, sizeof(AuditIndexItem)); securec_check(errorno, "\0", "\0"); @@ -1487,7 +1485,9 @@ static void pgaudit_cleanup(void) if (index == earliest_idx) { break; } + /* udpate audit index for next loop */ + g_instance.audit_cxt.audit_indextbl->begidx = (index + 1) % g_instance.audit_cxt.audit_indextbl->maxnum; index = g_instance.audit_cxt.audit_indextbl->begidx; } LWLockRelease(g_instance.audit_cxt.index_file_lock); @@ -2018,7 +2018,7 @@ static void pgaudit_read_indexfile(const char* audit_directory) struct stat statbuf; char tblfile_path[MAXPGPATH] = {0}; size_t nread = 0; - AuditIndexTable indextbl; + AuditIndexTableNew indextbl; int rc = snprintf_s(tblfile_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s", audit_directory, audit_indextbl_file); securec_check_intval(rc,,); @@ -2057,7 +2057,7 @@ static void pgaudit_read_indexfile(const char* audit_directory) pfree_ext(g_instance.audit_cxt.audit_indextbl); /* read the whole audit index table */ - g_instance.audit_cxt.audit_indextbl = (AuditIndexTable *)MemoryContextAllocZero( + g_instance.audit_cxt.audit_indextbl = (AuditIndexTableNew *)MemoryContextAllocZero( g_instance.audit_cxt.global_audit_context, (indextbl.maxnum * sizeof(AuditIndexItem) + indextbl_header_size)); errorno = @@ -2258,7 +2258,7 @@ static void pgaudit_rewrite_indexfile(void) FILE* fp = NULL; char tblfile_path[MAXPGPATH] = {0}; size_t nread = 0; - AuditIndexTableOld old_index_tbl; + AuditIndexTable old_index_tbl; int rc = snprintf_s(tblfile_path, MAXPGPATH, @@ -2288,7 +2288,7 @@ static void pgaudit_rewrite_indexfile(void) pfree_ext(g_instance.audit_cxt.audit_indextbl); pfree_ext(g_instance.audit_cxt.audit_indextbl_old); /* read the whole audit index table */ - g_instance.audit_cxt.audit_indextbl_old = (AuditIndexTableOld *)MemoryContextAllocZero( + g_instance.audit_cxt.audit_indextbl_old = (AuditIndexTable *)MemoryContextAllocZero( g_instance.audit_cxt.global_audit_context, (old_index_tbl.maxnum * sizeof(AuditIndexItem) + old_indextbl_header_size)); errorno = memcpy_s(g_instance.audit_cxt.audit_indextbl_old, @@ -2296,7 +2296,7 @@ static void pgaudit_rewrite_indexfile(void) &old_index_tbl, old_indextbl_header_size); securec_check(errorno, "\0", "\0"); /* rewrite old index table to new index table */ - g_instance.audit_cxt.audit_indextbl = (AuditIndexTable *)MemoryContextAllocZero( + g_instance.audit_cxt.audit_indextbl = (AuditIndexTableNew *)MemoryContextAllocZero( g_instance.audit_cxt.global_audit_context, (old_index_tbl.maxnum * sizeof(AuditIndexItem) + indextbl_header_size)); g_instance.audit_cxt.audit_indextbl->maxnum = old_index_tbl.maxnum; @@ -2334,7 +2334,7 @@ static void pgaudit_indextbl_init_new(void) if (g_instance.audit_cxt.audit_indextbl == NULL) { ereport(LOG, (errmsg("pgaudit_indextbl_init_new first init"))); g_instance.audit_cxt.audit_indextbl = - (AuditIndexTable *)MemoryContextAllocZero(g_instance.audit_cxt.global_audit_context, + (AuditIndexTableNew *)MemoryContextAllocZero(g_instance.audit_cxt.global_audit_context, (u_sess->attr.attr_security.Audit_RemainThreshold + 1) * sizeof(AuditIndexItem) + indextbl_header_size); g_instance.audit_cxt.audit_indextbl->maxnum = u_sess->attr.attr_security.Audit_RemainThreshold + 1; g_instance.audit_cxt.audit_indextbl->count = 0; /* audit files count will be updated by auditfile_open */ @@ -2417,27 +2417,31 @@ static void pgaudit_indextbl_init_new(void) return; } -static void pgaudit_udpate_maxnum() +static void pgaudit_update_maxnum() { errno_t errorno = EOK; int thread_num = g_instance.attr.attr_security.audit_thread_num; int new_indextbl_data_lenth = (u_sess->attr.attr_security.Audit_RemainThreshold + 1) * sizeof(AuditIndexItem); - AuditIndexTable *new_indextbl = (AuditIndexTable *)MemoryContextAllocZero( + AuditIndexTableNew *new_indextbl = (AuditIndexTableNew *)MemoryContextAllocZero( g_instance.audit_cxt.global_audit_context, new_indextbl_data_lenth + indextbl_header_size); /* curidx and latest_idx should be updated later from old index table file */ new_indextbl->begidx = 0; new_indextbl->maxnum = u_sess->attr.attr_security.Audit_RemainThreshold + 1; + new_indextbl->last_audit_time = g_instance.audit_cxt.audit_indextbl->last_audit_time; + new_indextbl->thread_num = g_instance.audit_cxt.audit_indextbl->thread_num; if (g_instance.audit_cxt.audit_indextbl->count > 0) { AuditIndexItem *item = NULL; uint32 latest_idx = g_instance.audit_cxt.audit_indextbl->latest_idx; + uint32 last_idx = (latest_idx == 0) ? (g_instance.audit_cxt.audit_indextbl->maxnum - 1): (latest_idx - 1); uint32 index = g_instance.audit_cxt.audit_indextbl->begidx; uint32 pos = new_indextbl->begidx; do { item = g_instance.audit_cxt.audit_indextbl->data + index; - errorno = memcpy_s(new_indextbl->data + pos, (new_indextbl_data_lenth - pos), item, sizeof(AuditIndexItem)); + errorno = memcpy_s(new_indextbl->data + pos, (new_indextbl_data_lenth - (pos * sizeof(AuditIndexItem))), + item, sizeof(AuditIndexItem)); securec_check(errorno, "\0", "\0"); new_indextbl->count++; @@ -2445,11 +2449,11 @@ static void pgaudit_udpate_maxnum() * finished copy old index table file from range [begin, latest_idx) * then update new index table file curidxes */ - if (index == (latest_idx - 1)) { + if (index == last_idx) { for (int i = 0; i < thread_num; ++i) { - new_indextbl->curidx[i] = pos - thread_num + i; + new_indextbl->curidx[i] = pos - thread_num + 1 + i; } - new_indextbl->latest_idx = pos; + new_indextbl->latest_idx = pos + 1; break; } @@ -2487,7 +2491,7 @@ static void pgaudit_reset_indexfile() /* If file remain threshold parameter changed, than copy the old audit index table to the new table */ if (old_maxnum != (uint32)u_sess->attr.attr_security.Audit_RemainThreshold + 1) { LWLockAcquire(g_instance.audit_cxt.index_file_lock, LW_EXCLUSIVE); - pgaudit_udpate_maxnum(); + pgaudit_update_maxnum(); LWLockRelease(g_instance.audit_cxt.index_file_lock); } @@ -2996,17 +3000,19 @@ Datum pg_query_audit(PG_FUNCTION_ARGS) * load the index audit table from global index audit table instance * then use the local thread one when iterate all audit files */ + pgaudit_read_indexfile(audit_dir); LWLockAcquire(g_instance.audit_cxt.index_file_lock, LW_SHARED); - t_thrd.audit.audit_indextbl = NULL; - int indextbl_len = - (u_sess->attr.attr_security.Audit_RemainThreshold + 1) * sizeof(AuditIndexItem) + indextbl_header_size; - t_thrd.audit.audit_indextbl = (AuditIndexTable *)palloc0(indextbl_len); - error_t errorno = - memcpy_s(t_thrd.audit.audit_indextbl, indextbl_len, g_instance.audit_cxt.audit_indextbl, indextbl_len); - securec_check(errorno, "\0", "\0"); - + pfree_ext(t_thrd.audit.audit_indextbl); + if (g_instance.audit_cxt.audit_indextbl != NULL) { + int indextbl_len = + (u_sess->attr.attr_security.Audit_RemainThreshold + 1) * sizeof(AuditIndexItem) + indextbl_header_size; + t_thrd.audit.audit_indextbl = (AuditIndexTableNew *)palloc0(indextbl_len); + error_t errorno = + memcpy_s(t_thrd.audit.audit_indextbl, indextbl_len, g_instance.audit_cxt.audit_indextbl, indextbl_len); + securec_check(errorno, "\0", "\0"); + } LWLockRelease(g_instance.audit_cxt.index_file_lock); - + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; oldcontext = MemoryContextSwitchTo(per_query_ctx); @@ -3017,8 +3023,6 @@ Datum pg_query_audit(PG_FUNCTION_ARGS) MemoryContextSwitchTo(oldcontext); - ereport(DEBUG1, - (errmsg("pg_query_audit count: %d indextbl_len: %d", t_thrd.audit.audit_indextbl->count, indextbl_len))); if (begtime < endtime && t_thrd.audit.audit_indextbl != NULL && t_thrd.audit.audit_indextbl->count > 0) { bool satisfied = false; uint32 index = 0; @@ -3073,14 +3077,17 @@ Datum pg_delete_audit(PG_FUNCTION_ARGS) * load the index audit table from global index audit table instance * then use the local thread one when iterate all audit files */ + pgaudit_read_indexfile(g_instance.attr.attr_security.Audit_directory); LWLockAcquire(g_instance.audit_cxt.index_file_lock, LW_SHARED); pfree_ext(t_thrd.audit.audit_indextbl); - int indextbl_len = - (u_sess->attr.attr_security.Audit_RemainThreshold + 1) * sizeof(AuditIndexItem) + indextbl_header_size; - t_thrd.audit.audit_indextbl = (AuditIndexTable *)palloc0(indextbl_len); - error_t errorno = - memcpy_s(t_thrd.audit.audit_indextbl, indextbl_len, g_instance.audit_cxt.audit_indextbl, indextbl_len); - securec_check(errorno, "\0", "\0"); + if (g_instance.audit_cxt.audit_indextbl != NULL) { + int indextbl_len = + (u_sess->attr.attr_security.Audit_RemainThreshold + 1) * sizeof(AuditIndexItem) + indextbl_header_size; + t_thrd.audit.audit_indextbl = (AuditIndexTableNew *)palloc0(indextbl_len); + error_t errorno = + memcpy_s(t_thrd.audit.audit_indextbl, indextbl_len, g_instance.audit_cxt.audit_indextbl, indextbl_len); + securec_check(errorno, "\0", "\0"); + } LWLockRelease(g_instance.audit_cxt.index_file_lock); int thread_num = g_instance.audit_cxt.thread_num; diff --git a/src/gausskernel/process/postmaster/pgstat.cpp b/src/gausskernel/process/postmaster/pgstat.cpp index cc8ed453b..2fd975b7b 100644 --- a/src/gausskernel/process/postmaster/pgstat.cpp +++ b/src/gausskernel/process/postmaster/pgstat.cpp @@ -3532,9 +3532,29 @@ void pgstat_report_activity(BackendState state, const char* cmd_str) beentry->st_state_start_timestamp = current_timestamp; if (cmd_str != NULL) { - rc = memcpy_s( - (char*)beentry->st_activity, g_instance.attr.attr_common.pgstat_track_activity_query_size, cmd_str, len); - securec_check(rc, "\0", "\0"); + char *mask_string = NULL; + if (len == g_instance.attr.attr_common.pgstat_track_activity_query_size - 1 && + t_thrd.mem_cxt.mask_password_mem_cxt != NULL) { + /* mask the cmd_str when the cmd_str is truncated. */ + mask_string = maskPassword(cmd_str); + } + + /* If mask successfully, store the mask_string. Otherwise, the cmd_str is recorded. */ + if (mask_string == NULL) { + rc = memcpy_s((char*)beentry->st_activity, g_instance.attr.attr_common.pgstat_track_activity_query_size, + cmd_str, len); + securec_check(rc, "\0", "\0"); + } else { + int copy_len = strlen(mask_string); + if (len < copy_len) { + copy_len = len; + } + rc = memcpy_s((char*)beentry->st_activity, g_instance.attr.attr_common.pgstat_track_activity_query_size, + mask_string, copy_len); + securec_check(rc, "\0", "\0"); + pfree(mask_string); + } + beentry->st_activity[len] = '\0'; beentry->st_activity_start_timestamp = start_timestamp; } diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index c5223c7b6..834ba9631 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -4020,7 +4020,8 @@ int ProcessStartupPacket(Port* port, bool SSLdone) errmsg("can not accept connection in pending mode."))); } else { #ifdef ENABLE_MULTIPLE_NODES - if (STANDBY_MODE == hashmdata->current_mode && (!IS_DISASTER_RECOVER_MODE || GTM_FREE_MODE)) { + if (STANDBY_MODE == hashmdata->current_mode && (!IS_DISASTER_RECOVER_MODE || GTM_FREE_MODE || + g_instance.attr.attr_storage.recovery_parse_workers > 1)) { ereport(ERROR, (errcode(ERRCODE_CANNOT_CONNECT_NOW), errmsg("can not accept connection in standby mode."))); } @@ -4669,9 +4670,9 @@ static void SIGHUP_handler(SIGNAL_ARGS) signal_child(g_instance.pid_cxt.SysLoggerPID, SIGHUP); /* signal the auditor process */ if (g_instance.pid_cxt.PgAuditPID != NULL) { - Assert(!dummyStandbyMode); for (int i = 0; i < g_instance.audit_cxt.thread_num; ++i) { if (g_instance.pid_cxt.PgAuditPID[i] != 0) { + Assert(!dummyStandbyMode); signal_child(g_instance.pid_cxt.PgAuditPID[i], SIGHUP); } } @@ -6073,7 +6074,6 @@ static void reaper(SIGNAL_ARGS) * nothing left for it to do. */ if (g_instance.pid_cxt.PgAuditPID != NULL) { - Assert(!dummyStandbyMode); pgaudit_stop_all(); } } else { @@ -6354,8 +6354,8 @@ static void reaper(SIGNAL_ARGS) if (g_instance.pid_cxt.PgAuditPID != NULL) { bool is_audit_thread = false; for (int i = 0; i < g_instance.audit_cxt.thread_num; ++i) { - Assert(!dummyStandbyMode); if (pid == g_instance.pid_cxt.PgAuditPID[i]) { + Assert(!dummyStandbyMode); g_instance.pid_cxt.PgAuditPID[i] = 0; is_audit_thread = true; if (!EXIT_STATUS_0(exitstatus)) @@ -7211,7 +7211,6 @@ static void PostmasterStateMachine(void) /* signal the auditor process */ if (g_instance.pid_cxt.PgAuditPID != NULL) { - Assert(!dummyStandbyMode); pgaudit_stop_all(); } diff --git a/src/gausskernel/process/tcop/postgres.cpp b/src/gausskernel/process/tcop/postgres.cpp index 22d14f8b4..7480dfdcf 100755 --- a/src/gausskernel/process/tcop/postgres.cpp +++ b/src/gausskernel/process/tcop/postgres.cpp @@ -7082,6 +7082,7 @@ void RemoveTempNamespace() u_sess->catalog_cxt.myTempToastNamespace = InvalidOid; } } + pfree_ext(str.data); } PG_CATCH(); { @@ -9236,6 +9237,7 @@ int PostgresMain(int argc, char* argv[], const char* dbname, const char* usernam InvalidTransactionTimeline, ss_need_sync_wait_all); /* quickly set my recent global xmin */ u_sess->utils_cxt.RecentGlobalXmin = GetOldestXmin(NULL, true); + u_sess->utils_cxt.RecentGlobalCatalogXmin = GetOldestCatalogXmin(); } } /* check gtm mode, remote should be false, local cannot be true */ diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index 89a6995d3..a2d5768fd 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -685,6 +685,7 @@ static void knl_g_roach_init(knl_g_roach_context* roach_cxt) roach_cxt->targetTimeInPITR = NULL; roach_cxt->globalBarrierRecordForPITR = NULL; roach_cxt->isXLogForceRecycled = false; + roach_cxt->forceAdvanceSlotTigger = false; roach_cxt->isGtmFreeCsn = false; roach_cxt->targetRestoreTimeFromMedia = NULL; } diff --git a/src/gausskernel/process/threadpool/knl_session.cpp b/src/gausskernel/process/threadpool/knl_session.cpp index e21ba8497..6c3225a37 100755 --- a/src/gausskernel/process/threadpool/knl_session.cpp +++ b/src/gausskernel/process/threadpool/knl_session.cpp @@ -280,6 +280,7 @@ static void knl_u_parser_init(knl_u_parser_context* parser_cxt) parser_cxt->ddl_pbe_context = NULL; parser_cxt->in_package_function_compile = false; parser_cxt->isCreateFuncOrProc = false; + parser_cxt->isTimeCapsule = false; } static void knl_u_advisor_init(knl_u_advisor_context* adv_cxt) @@ -441,6 +442,7 @@ static void knl_u_utils_init(knl_u_utils_context* utils_cxt) utils_cxt->RecentDataXmin = FirstNormalTransactionId; utils_cxt->RecentGlobalXmin = InvalidTransactionId; utils_cxt->RecentGlobalDataXmin = InvalidTransactionId; + utils_cxt->RecentGlobalCatalogXmin = InvalidTransactionId; utils_cxt->cn_xc_maintain_mode = false; utils_cxt->snapshot_source = SNAPSHOT_UNDEFINED; @@ -812,6 +814,7 @@ static void knl_u_plpgsql_init(knl_u_plpgsql_context* plsql_cxt) plsql_cxt->is_delete_function = false; plsql_cxt->have_error = false; plsql_cxt->client_info = NULL; + pthread_mutex_init(&plsql_cxt->client_info_lock, NULL); plsql_cxt->sess_cxt_htab = NULL; plsql_cxt->have_error = false; plsql_cxt->stp_savepoint_cnt = 0; @@ -841,6 +844,7 @@ static void knl_u_plpgsql_init(knl_u_plpgsql_context* plsql_cxt) plsql_cxt->cur_exception_cxt = NULL; plsql_cxt->pragma_autonomous = false; plsql_cxt->ActiveLobToastOid = InvalidOid; + plsql_cxt->is_insert_gs_source = false; } static void knl_u_stat_init(knl_u_stat_context* stat_cxt) @@ -981,7 +985,7 @@ static void knl_u_relcache_init(knl_u_relcache_context* relcache_cxt) relcache_cxt->criticalSharedRelcachesBuilt = false; relcache_cxt->relcacheInvalsReceived = 0L; relcache_cxt->initFileRelationIds = NIL; - relcache_cxt->need_eoxact_work = false; + relcache_cxt->RelCacheNeedEOXActWork = false; relcache_cxt->OpClassCache = NULL; relcache_cxt->pgclassdesc = NULL; relcache_cxt->pgindexdesc = NULL; @@ -1170,7 +1174,7 @@ static void knl_u_cache_init(knl_u_cache_context* cache_cxt) cache_cxt->PartitionIdCache = NULL; cache_cxt->BucketIdCache = NULL; - cache_cxt->part_cache_need_eoxact_work = false; + cache_cxt->PartCacheNeedEOXActWork = false; cache_cxt->bucket_cache_need_eoxact_work = false; cache_cxt->dn_hash_table = NULL; } diff --git a/src/gausskernel/process/threadpool/knl_thread.cpp b/src/gausskernel/process/threadpool/knl_thread.cpp old mode 100644 new mode 100755 index 673d25104..4ba914e2d --- a/src/gausskernel/process/threadpool/knl_thread.cpp +++ b/src/gausskernel/process/threadpool/knl_thread.cpp @@ -1151,6 +1151,8 @@ static void knl_t_bgwriter_init(knl_t_bgwriter_context* bgwriter_cxt) static void knl_t_pagewriter_init(knl_t_pagewriter_context* pagewriter_cxt) { pagewriter_cxt->got_SIGHUP = false; + pagewriter_cxt->sync_requested = false; + pagewriter_cxt->sync_retry = false; pagewriter_cxt->shutdown_requested = false; pagewriter_cxt->page_writer_after = WRITEBACK_MAX_PENDING_FLUSHES; pagewriter_cxt->pagewriter_id = -1; @@ -1495,7 +1497,7 @@ static void knl_t_postmaster_init(knl_t_postmaster_context* postmaster_cxt) rc = memset_s(postmaster_cxt->CrossClusterReplConnArray, MAX_REPLNODE_NUM * sizeof(replconninfo*), 0, MAX_REPLNODE_NUM * sizeof(replconninfo*)); securec_check(rc, "\0", "\0"); - rc = memset_s(postmaster_cxt->CrossClusterReplConnChanged, MAX_REPLNODE_NUM * sizeof(bool), 0, + rc = memset_s(postmaster_cxt->CrossClusterReplConnChanged, MAX_REPLNODE_NUM * sizeof(bool), 0, MAX_REPLNODE_NUM * sizeof(bool)); securec_check(rc, "\0", "\0"); @@ -1678,7 +1680,7 @@ knl_t_uheap_stats_init(knl_t_uheap_stats_context* uheap_stats_cxt) } #endif -void KnlDcfContextInit(knl_t_dcf_context* dcfContext) +void KnlDcfContextInit(knl_t_dcf_context* dcfContext) { dcfContext->isDcfShmemInited = false; dcfContext->is_dcf_thread = false; @@ -1709,7 +1711,7 @@ void knl_thread_init(knl_thread_role role) t_thrd.mcxt_group->Init(t_thrd.top_mem_cxt); MemoryContextSeal(t_thrd.top_mem_cxt); MemoryContextSwitchTo(THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_DEFAULT)); - + KnlLscContextInit(&t_thrd.lsc_cxt); /* CommProxy Support */ t_thrd.comm_sock_option = g_default_invalid_sock_opt; diff --git a/src/gausskernel/process/threadpool/threadpool_sessctl.cpp b/src/gausskernel/process/threadpool/threadpool_sessctl.cpp index e7e5090b8..1267f0ad1 100755 --- a/src/gausskernel/process/threadpool/threadpool_sessctl.cpp +++ b/src/gausskernel/process/threadpool/threadpool_sessctl.cpp @@ -568,6 +568,65 @@ void ThreadPoolSessControl::getSessionMemoryDetail(Tuplestorestate* tupStore, PG_END_TRY(); } +void ThreadPoolSessControl::calculateClientInfo( + knl_session_context* sess, Tuplestorestate* tupStore, TupleDesc tupDesc) +{ + /* build one tuple and save it in tuplestore. */ + const int COLUMN_NUM = 2; + Datum values[COLUMN_NUM] = {0}; + bool nulls[COLUMN_NUM] = {false}; + + values[0] = Int64GetDatum(sess->session_id); + if (sess->plsql_cxt.client_info != NULL) { + values[1] = CStringGetTextDatum(sess->plsql_cxt.client_info); + } else { + nulls[1] = true; + } + tuplestore_putvalues(tupStore, tupDesc, values, nulls); +} + +void ThreadPoolSessControl::getSessionClientInfo(Tuplestorestate* tupStore, TupleDesc tupDesc) +{ + AutoMutexLock alock(&m_sessCtrlock); + knl_sess_control* ctrl = NULL; + Dlelem* elem = NULL; + knl_sess_control* sess = NULL; + + PG_TRY(); + { + HOLD_INTERRUPTS(); + alock.lock(); + + /* collect all the Memory Context status, put in data */ + elem = DLGetHead(&m_activelist); + + while (elem != NULL) { + ctrl = (knl_sess_control*)DLE_VAL(elem); + sess = ctrl; + if (ctrl->sess) { + (void)syscalllockAcquire(&ctrl->sess->plsql_cxt.client_info_lock); + calculateClientInfo(ctrl->sess, tupStore, tupDesc); + (void)syscalllockRelease(&ctrl->sess->plsql_cxt.client_info_lock); + } + elem = DLGetSucc(elem); + } + alock.unLock(); + sess = NULL; + + RESUME_INTERRUPTS(); + } + PG_CATCH(); + { + if (sess != NULL) { + ctrl = sess; + (void)syscalllockRelease(&ctrl->sess->plsql_cxt.client_info_lock); + } + alock.unLock(); + PG_RE_THROW(); + } + PG_END_TRY(); +} + void ThreadPoolSessControl::getSessionMemoryContextInfo(const char* ctx_name, StringInfoData* buf, knl_sess_control** sess) { diff --git a/src/gausskernel/process/threadpool/threadpool_worker.cpp b/src/gausskernel/process/threadpool/threadpool_worker.cpp index bb7ab3aa9..5741b2c54 100644 --- a/src/gausskernel/process/threadpool/threadpool_worker.cpp +++ b/src/gausskernel/process/threadpool/threadpool_worker.cpp @@ -185,6 +185,8 @@ void ThreadPoolWorker::WaitMission() * before we serve next session we must keep us clean. */ PreventSignal(); + + MemoryContext old = CurrentMemoryContext; while (true) { /* we should keep the thread clean for next Session. */ CleanThread(); @@ -222,7 +224,7 @@ void ThreadPoolWorker::WaitMission() break; } } - + MemoryContextSwitchTo(old); (void)disable_session_sig_alarm(); /* now we can accept signal. out of this, we rely on signal handle. */ AllowSignal(); diff --git a/src/gausskernel/runtime/codegen/vecexecutor/vechashaggcodegen.cpp b/src/gausskernel/runtime/codegen/vecexecutor/vechashaggcodegen.cpp index 1eeb94a9d..2f0ca6535 100644 --- a/src/gausskernel/runtime/codegen/vecexecutor/vechashaggcodegen.cpp +++ b/src/gausskernel/runtime/codegen/vecexecutor/vechashaggcodegen.cpp @@ -81,9 +81,7 @@ int VecHashAggCodeGen::GetAlignedScale(Expr* node) } else if (opexpr->opno == NUMERICMULOID) resscale = lscale + rscale; else - ereport(ERROR, - (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), - errmodule(MOD_LLVM), + ereport(ERROR, (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), errmodule(MOD_LLVM), errmsg("Unsupported operation %u in FastAgg.", opexpr->opno))); } break; case T_Var: { @@ -478,6 +476,17 @@ void VecHashAggCodeGen::SonicHashAggCodeGen(VecAggState* node) llvmCodeGen->addFunctionToMCJit(jitted_sonicbatchagg, reinterpret_cast(&(node->jitted_sonicbatchagg))); } +/* Check if existing tuple is unique */ +static void VecAggCheckUnique(VecAgg* vecagg, GsCodeGen::LlvmBuilder* builder) +{ + const char* errorstr = "more than one row returned by a subquery used as an expression"; + + if (vecagg->unique_check) { + /* If uniqueness is not checked, the result is incorrect. */ + DebugerCodeGen::CodeGenElogInfo(builder, ERROR, errorstr); + } +} + template llvm::Function* VecHashAggCodeGen::AgghashingCodeGenorSglTbl(VecAggState* node) { @@ -656,9 +665,7 @@ llvm::Function* VecHashAggCodeGen::AgghashingCodeGenorSglTbl(VecAggState* node) llvm::Function* func_hashbatch = HashBatchCodeGen(node, i, rehash); if (func_hashbatch == NULL) { - ereport(ERROR, - (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), - errmodule(MOD_LLVM), + ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmodule(MOD_LLVM), errmsg("Failed on generating HashBatchCodeGen!\n"))); } @@ -777,9 +784,7 @@ llvm::Function* VecHashAggCodeGen::AgghashingCodeGenorSglTbl(VecAggState* node) for (i = 0; i < numkeys; i++) { llvm::Function* func_matchonekey = MatchOneKeyCodeGen(node, i); if (NULL == func_matchonekey) { - ereport(ERROR, - (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), - errmodule(MOD_LLVM), + ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmodule(MOD_LLVM), errmsg("Failed on generating MatchOneKey Function!\n"))); } @@ -806,6 +811,7 @@ llvm::Function* VecHashAggCodeGen::AgghashingCodeGenorSglTbl(VecAggState* node) * hash cell and go to next tuple. */ builder.SetInsertPoint(key_match); + VecAggCheckUnique(vecagg, &builder); /* get hashAggRunner.BaseAggRunner.m_Loc[phi_idx] */ Vals4[0] = Datum_0; Vals4[1] = int32_0; @@ -1041,9 +1047,7 @@ llvm::Function* VecHashAggCodeGen::AgghashingWithPrefetchCodeGenorSglTbl(VecAggS llvm::Function* func_hashbatch = HashBatchCodeGen(node, i, rehash); if (func_hashbatch == NULL) { - ereport(ERROR, - (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), - errmodule(MOD_LLVM), + ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmodule(MOD_LLVM), errmsg("Failed on generating HashBatchCodeGen!\n"))); } @@ -1204,8 +1208,7 @@ llvm::Function* VecHashAggCodeGen::AgghashingWithPrefetchCodeGenorSglTbl(VecAggS for (i = 0; i < numkeys; i++) { llvm::Function* func_matchonekey = MatchOneKeyCodeGen(node, i); if (NULL == func_matchonekey) { - ereport(ERROR, - (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), + ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_NULL_VALUE), errmodule(MOD_LLVM), errmsg("Failed on generating MatchOneKey Function!\n"))); } @@ -1233,6 +1236,7 @@ llvm::Function* VecHashAggCodeGen::AgghashingWithPrefetchCodeGenorSglTbl(VecAggS * hash cell and go to next tuple. */ builder.SetInsertPoint(key_match); + VecAggCheckUnique(vecagg, &builder); /* get hashAggRunner.BaseAggRunner.m_Loc[phi_idx] */ Vals4[0] = Datum_0; Vals4[1] = int32_0; diff --git a/src/gausskernel/runtime/executor/execClusterResize.cpp b/src/gausskernel/runtime/executor/execClusterResize.cpp index 0dc5b3f2b..4eb0444a5 100644 --- a/src/gausskernel/runtime/executor/execClusterResize.cpp +++ b/src/gausskernel/runtime/executor/execClusterResize.cpp @@ -596,6 +596,9 @@ void BlockUnsupportedDDL(const Node* parsetree) if (stmt->relation) { relid = RangeVarGetRelid(stmt->relation, AccessShareLock, true); if (OidIsValid(relid)) { + /* release index lock before lock table to avoid deadlock */ + UnlockRelationOid(relid, AccessShareLock); + Relation relation = relation_open(relid, NoLock); bool inRedis = false; @@ -608,8 +611,6 @@ void BlockUnsupportedDDL(const Node* parsetree) relation_close(heapRelation, AccessShareLock); } relation_close(relation, NoLock); - UnlockRelationOid(relid, AccessShareLock); - if (inRedis) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), diff --git a/src/gausskernel/runtime/executor/execQual.cpp b/src/gausskernel/runtime/executor/execQual.cpp index ccd880bda..4bbfd3156 100644 --- a/src/gausskernel/runtime/executor/execQual.cpp +++ b/src/gausskernel/runtime/executor/execQual.cpp @@ -152,6 +152,7 @@ static Datum ExecEvalGroupingIdExpr( GroupingIdExprState* gstate, ExprContext* econtext, bool* isNull, ExprDoneCond* isDone); static bool func_has_refcursor_args(Oid Funcid, FunctionCallInfoData* fcinfo); extern struct varlena *heap_tuple_fetch_and_copy(Relation rel, struct varlena *attr, bool needcheck); +static void check_huge_clob_paramter(FunctionCallInfoData* fcinfo, bool is_have_huge_clob); THR_LOCAL PLpgSQL_execstate* plpgsql_estate = NULL; @@ -1692,6 +1693,7 @@ static ExprDoneCond ExecEvalFuncArgs( i = 0; econtext->is_cursor = false; u_sess->plsql_cxt.func_tableof_index = NIL; + bool is_have_huge_clob = false; foreach (arg, argList) { ExprState* argstate = (ExprState*)lfirst(arg); ExprDoneCond thisArgIsDone; @@ -1719,6 +1721,9 @@ static ExprDoneCond ExecEvalFuncArgs( } fcinfo->argTypes[i] = argstate->resultType; econtext->is_cursor = false; + if (is_huge_clob(fcinfo->argTypes[i], fcinfo->argnull[i], fcinfo->arg[i])) { + is_have_huge_clob = true; + } if (thisArgIsDone != ExprSingleResult) { /* @@ -1734,6 +1739,7 @@ static ExprDoneCond ExecEvalFuncArgs( } i++; } + check_huge_clob_paramter(fcinfo, is_have_huge_clob); Assert(i == fcinfo->nargs); @@ -2401,10 +2407,8 @@ static Datum ExecMakeFunctionResultNoSets( if (has_refcursor && fcinfo->argTypes[i] == REFCURSOROID) econtext->is_cursor = true; fcinfo->arg[i] = ExecEvalExpr(argstate, econtext, &fcinfo->argnull[i], NULL); - if (is_external_clob(fcinfo->argTypes[i], fcinfo->argnull[i], fcinfo->arg[i])) { - bool is_null = false; - struct varatt_lob_pointer* lob_pointer = (varatt_lob_pointer*)(VARDATA_EXTERNAL(fcinfo->arg[i])); - fcinfo->arg[i] = fetch_lob_value_from_tuple(lob_pointer, InvalidOid, &is_null, &is_have_huge_clob); + if (is_huge_clob(fcinfo->argTypes[i], fcinfo->argnull[i], fcinfo->arg[i])) { + is_have_huge_clob = true; } ExecTableOfIndexInfo execTableOfIndexInfo; initExecTableOfIndexInfo(&execTableOfIndexInfo, econtext); @@ -2457,18 +2461,25 @@ static Datum ExecMakeFunctionResultNoSets( pgstat_init_function_usage(fcinfo, &fcusage); - if (fcinfo->flinfo->fn_addr != textcat && is_have_huge_clob) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("huge clob do not support as function in parameter"))); - } fcinfo->isnull = false; + check_huge_clob_paramter(fcinfo, is_have_huge_clob); if (u_sess->instr_cxt.global_instr != NULL && fcinfo->flinfo->fn_addr == plpgsql_call_handler) { StreamInstrumentation* save_global_instr = u_sess->instr_cxt.global_instr; u_sess->instr_cxt.global_instr = NULL; result = FunctionCallInvoke(fcinfo); // node will be free at here or else; u_sess->instr_cxt.global_instr = save_global_instr; } else { + if (fcinfo->argTypes[0] == CLOBOID && fcinfo->argTypes[1] == CLOBOID && fcinfo->flinfo->fn_addr == textcat) { + bool is_null = false; + if (fcinfo->arg[0] != 0 && VARATT_IS_EXTERNAL_LOB(fcinfo->arg[0])) { + struct varatt_lob_pointer* lob_pointer = (varatt_lob_pointer*)(VARDATA_EXTERNAL(fcinfo->arg[0])); + fcinfo->arg[0] = fetch_lob_value_from_tuple(lob_pointer, InvalidOid, &is_null); + } + if (fcinfo->arg[1] != 0 && VARATT_IS_EXTERNAL_LOB(fcinfo->arg[1])) { + struct varatt_lob_pointer* lob_pointer = (varatt_lob_pointer*)(VARDATA_EXTERNAL(fcinfo->arg[1])); + fcinfo->arg[1] = fetch_lob_value_from_tuple(lob_pointer, InvalidOid, &is_null); + } + } result = FunctionCallInvoke(fcinfo); } *isNull = fcinfo->isnull; @@ -5967,12 +5978,11 @@ int ExecCleanTargetListLength(List* targetlist) return len; } -HeapTuple get_tuple(Relation relation, ItemPointer tid) +static HeapTuple get_tuple(Relation relation, ItemPointer tid) { Buffer user_buf = InvalidBuffer; HeapTuple tuple = NULL; HeapTuple new_tuple = NULL; - TM_Result result; /* alloc mem for old tuple and set tuple id */ tuple = (HeapTupleData *)heaptup_alloc(BLCKSZ); @@ -5981,11 +5991,6 @@ HeapTuple get_tuple(Relation relation, ItemPointer tid) tuple->t_self = *tid; if (heap_fetch(relation, SnapshotAny, tuple, &user_buf, false, NULL)) { - result = HeapTupleSatisfiesUpdate(tuple, GetCurrentCommandId(true), user_buf, false); - if (result != TM_Ok) { - ereport(ERROR, (errcode(ERRCODE_SYSTEM_ERROR), errmsg("Failed to get tuple"))); - } - new_tuple = heapCopyTuple((HeapTuple)tuple, relation->rd_att, NULL); ReleaseBuffer(user_buf); } else { @@ -5997,6 +6002,21 @@ HeapTuple get_tuple(Relation relation, ItemPointer tid) return new_tuple; } +static void check_huge_clob_paramter(FunctionCallInfoData* fcinfo, bool is_have_huge_clob) +{ + if (!is_have_huge_clob || IsSystemObjOid(fcinfo->flinfo->fn_oid)) { + return; + } + Oid schema_oid = get_func_namespace(fcinfo->flinfo->fn_oid); + if (IsPackageSchemaOid(schema_oid)) { + return; + } + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("huge clob do not support as function in parameter"))); +} + + bool is_external_clob(Oid type_oid, bool is_null, Datum value) { if (type_oid == CLOBOID && !is_null && VARATT_IS_EXTERNAL_LOB(value)) { @@ -6005,7 +6025,37 @@ bool is_external_clob(Oid type_oid, bool is_null, Datum value) return false; } -Datum fetch_lob_value_from_tuple(varatt_lob_pointer* lob_pointer, Oid update_oid, bool* is_null, bool* is_huge_clob) +bool is_huge_clob(Oid type_oid, bool is_null, Datum value) +{ + if (!is_external_clob(type_oid, is_null, value)) { + return false; + } + + struct varatt_lob_pointer* lob_pointer = (varatt_lob_pointer*)(VARDATA_EXTERNAL(value)); + bool is_huge_clob = false; + /* get relation by relid */ + ItemPointerData tuple_ctid; + tuple_ctid.ip_blkid.bi_hi = lob_pointer->bi_hi; + tuple_ctid.ip_blkid.bi_lo = lob_pointer->bi_lo; + tuple_ctid.ip_posid = lob_pointer->ip_posid; + Relation relation = heap_open(lob_pointer->relid, RowExclusiveLock); + HeapTuple origin_tuple = get_tuple(relation, &tuple_ctid); + if (!HeapTupleIsValid(origin_tuple)) { + ereport(ERROR, + (errcode(ERRCODE_CACHE_LOOKUP_FAILED), + errmsg("cache lookup failed for tuple from relation %u", lob_pointer->relid))); + } + bool attr_is_null = false; + Datum attr = fastgetattr(origin_tuple, lob_pointer->columid, relation->rd_att, &attr_is_null); + if (!attr_is_null && VARATT_IS_HUGE_TOAST_POINTER(attr)) { + is_huge_clob = true; + } + heap_close(relation, NoLock); + heap_freetuple(origin_tuple); + return is_huge_clob; +} + +Datum fetch_lob_value_from_tuple(varatt_lob_pointer* lob_pointer, Oid update_oid, bool* is_null) { /* get relation by relid */ ItemPointerData tuple_ctid; @@ -6021,9 +6071,7 @@ Datum fetch_lob_value_from_tuple(varatt_lob_pointer* lob_pointer, Oid update_oid } Datum attr = fastgetattr(origin_tuple, lob_pointer->columid, relation->rd_att, is_null); - if (!*is_null && VARATT_IS_HUGE_TOAST_POINTER(attr) && is_huge_clob != NULL) { - *is_huge_clob = true; - } + if (!OidIsValid(update_oid)) { heap_close(relation, NoLock); @@ -6033,9 +6081,7 @@ Datum fetch_lob_value_from_tuple(varatt_lob_pointer* lob_pointer, Oid update_oid if (*is_null) { new_attr = (Datum)0; } else { - if (VARATT_IS_SHORT(attr) || VARATT_IS_EXTERNAL(attr)) { - new_attr = PointerGetDatum(attr); - } else if (VARATT_IS_HUGE_TOAST_POINTER(attr)) { + if (VARATT_IS_HUGE_TOAST_POINTER(attr)) { if (unlikely(origin_tuple->tupTableType == UHEAP_TUPLE)) { ereport(ERROR, (errcode(ERRCODE_INVALID_NAME), @@ -6046,6 +6092,8 @@ Datum fetch_lob_value_from_tuple(varatt_lob_pointer* lob_pointer, Oid update_oid struct varlena *new_value = heap_tuple_fetch_and_copy(update_rel, old_value, false); new_attr = PointerGetDatum(new_value); heap_close(update_rel, NoLock); + } else if (VARATT_IS_SHORT(attr) || VARATT_IS_EXTERNAL(attr) || VARATT_IS_4B(attr)) { + new_attr = PointerGetDatum(attr); } else { ereport(ERROR, (errcode(ERRCODE_SYSTEM_ERROR), errmsg("lob value which fetch from tuple type is not recognized."), @@ -6113,20 +6161,9 @@ static bool ExecTargetList(List* targetlist, ExprContext* econtext, Datum* value if (VARATT_IS_EXTERNAL_LOB(values[resind])) { struct varatt_lob_pointer* lob_pointer = (varatt_lob_pointer*)(VARDATA_EXTERNAL(values[resind])); bool is_null = false; - if (econtext->ecxt_scantuple != NULL) { - Oid update_oid = ((HeapTuple)(econtext->ecxt_scantuple->tts_tuple))->t_tableOid; - values[resind] = fetch_lob_value_from_tuple(lob_pointer, update_oid, &is_null, NULL); - } else { - ItemPointerData tuple_ctid; - tuple_ctid.ip_blkid.bi_hi = lob_pointer->bi_hi; - tuple_ctid.ip_blkid.bi_lo = lob_pointer->bi_lo; - tuple_ctid.ip_posid = lob_pointer->ip_posid; - Relation relation = heap_open(lob_pointer->relid, RowExclusiveLock); - HeapTuple origin_tuple = get_tuple(relation, &tuple_ctid); - Datum attr = fastgetattr(origin_tuple, lob_pointer->columid, relation->rd_att, &is_null); - values[resind] = attr; - heap_close(relation, NoLock); - } + Oid update_oid = econtext->ecxt_scantuple != NULL ? + ((HeapTuple)(econtext->ecxt_scantuple->tts_tuple))->t_tableOid : InvalidOid; + values[resind] = fetch_lob_value_from_tuple(lob_pointer, update_oid, &is_null); isnull[resind] = is_null; } } diff --git a/src/gausskernel/runtime/executor/nodeAgg.cpp b/src/gausskernel/runtime/executor/nodeAgg.cpp index 4e7d88075..d23e39f7f 100644 --- a/src/gausskernel/runtime/executor/nodeAgg.cpp +++ b/src/gausskernel/runtime/executor/nodeAgg.cpp @@ -474,6 +474,7 @@ static void advance_transition_function( *fcinfo, &(peraggstate->transfn), numTransInputs + 1, peraggstate->aggCollation, (Node*)aggstate, NULL); fcinfo->arg[0] = pergroupstate->transValue; fcinfo->argnull[0] = pergroupstate->transValueIsNull; + fcinfo->argTypes[0] = InvalidOid; fcinfo->isnull = false; /* just in case transfn doesn't set it */ Node *origin_fcxt = fcinfo->context; @@ -577,6 +578,7 @@ static void advance_collection_function( InitFunctionCallInfoData(*fcinfo, &(peraggstate->collectfn), 2, peraggstate->aggCollation, (Node*)aggstate, NULL); fcinfo->arg[0] = pergroupstate->collectValue; fcinfo->argnull[0] = pergroupstate->collectValueIsNull; + fcinfo->argTypes[0] = InvalidOid; newVal = FunctionCallInvoke(fcinfo); /* @@ -665,6 +667,7 @@ static void advance_aggregates(AggState* aggstate, AggStatePerGroup pergroup) for (i = 0; i < numTransInputs; i++) { fcinfo.arg[i + 1] = slot->tts_values[i]; fcinfo.argnull[i + 1] = slot->tts_isnull[i]; + fcinfo.argTypes[i + 1] = InvalidOid; } for (setno = 0; setno < numGroupingSets; setno++) { AggStatePerGroup pergroupstate = &pergroup[aggno + (setno * numAggs)]; @@ -896,6 +899,7 @@ static void finalize_aggregate(AggState* aggstate, AggStatePerAgg peraggstate, A foreach (lc, peraggstate->aggrefstate->aggdirectargs) { fcinfo.arg[args_pos] = ExecEvalExpr((ExprState*)lfirst(lc), aggstate->ss.ps.ps_ExprContext, &fcinfo.argnull[args_pos], NULL); + fcinfo.argTypes[args_pos] = ((ExprState*)lfirst(lc))->resultType; if (anynull == true || fcinfo.argnull[args_pos] == true) anynull = true; else @@ -928,6 +932,7 @@ static void finalize_aggregate(AggState* aggstate, AggStatePerAgg peraggstate, A fcinfo, &(peraggstate->finalfn), numFinalArgs, peraggstate->aggCollation, (Node*)aggstate, NULL); fcinfo.arg[0] = pergroupstate->transValue; fcinfo.argnull[0] = pergroupstate->transValueIsNull; + fcinfo.argTypes[0] = InvalidOid; if (anynull == true || pergroupstate->transValueIsNull == true) anynull = true; else @@ -936,6 +941,7 @@ static void finalize_aggregate(AggState* aggstate, AggStatePerAgg peraggstate, A while (args_pos < numFinalArgs) { fcinfo.arg[args_pos] = (Datum)0; fcinfo.argnull[args_pos] = true; + fcinfo.argTypes[args_pos] = InvalidOid; args_pos++; anynull = true; } diff --git a/src/gausskernel/runtime/executor/nodeIndexonlyscan.cpp b/src/gausskernel/runtime/executor/nodeIndexonlyscan.cpp index 79fefa14b..fb7bf002a 100644 --- a/src/gausskernel/runtime/executor/nodeIndexonlyscan.cpp +++ b/src/gausskernel/runtime/executor/nodeIndexonlyscan.cpp @@ -138,7 +138,7 @@ static TupleTableSlot* IndexOnlyNext(IndexOnlyScanState* node) slot = node->ss.ss_ScanTupleSlot; isUHeap = RelationIsUstoreFormat(node->ss.ss_currentRelation); tmpslot = MakeSingleTupleTableSlot(RelationGetDescr(scandesc->heapRelation), - false, scandesc->indexRelation->rd_tam_type); + false, scandesc->heapRelation->rd_tam_type); /* * OK, now that we have what we need, fetch the next tuple. diff --git a/src/gausskernel/runtime/executor/nodeStartWithOp.cpp b/src/gausskernel/runtime/executor/nodeStartWithOp.cpp index 8d0ce77e1..e556d3608 100644 --- a/src/gausskernel/runtime/executor/nodeStartWithOp.cpp +++ b/src/gausskernel/runtime/executor/nodeStartWithOp.cpp @@ -383,7 +383,7 @@ bool CheckCycleExeception(StartWithOpState *node, TupleTableSlot *slot) Assert (IsA(rustate, RecursiveUnionState)); - if (IsConnectByLevelStartWithPlan(swplan)) { + if (IsConnectByLevelStartWithPlan(swplan) || node->sw_keyAttnum == 0) { /* for connect by level case, we do not do cycle check */ return false; } diff --git a/src/gausskernel/runtime/executor/nodeTidscan.cpp b/src/gausskernel/runtime/executor/nodeTidscan.cpp index 2c3f48b73..c238155f0 100644 --- a/src/gausskernel/runtime/executor/nodeTidscan.cpp +++ b/src/gausskernel/runtime/executor/nodeTidscan.cpp @@ -768,7 +768,11 @@ static void ExecInitPartitionForTidScan(TidScanState* tidstate, EState* estate) lock = (relistarget ? RowExclusiveLock : AccessShareLock); tidstate->ss.lockMode = lock; - Assert(plan->scan.itrs == plan->scan.pruningInfo->ls_rangeSelectedPartitions->length); + if (plan->scan.pruningInfo->ls_rangeSelectedPartitions != NULL) { + plan->scan.itrs = plan->scan.pruningInfo->ls_rangeSelectedPartitions->length; + } else { + plan->scan.itrs = 0; + } foreach (cell, part_seqs) { Oid table_partitionid = InvalidOid; diff --git a/src/gausskernel/runtime/executor/nodeWindowAgg.cpp b/src/gausskernel/runtime/executor/nodeWindowAgg.cpp index 8fb83c323..d40c7879c 100644 --- a/src/gausskernel/runtime/executor/nodeWindowAgg.cpp +++ b/src/gausskernel/runtime/executor/nodeWindowAgg.cpp @@ -126,6 +126,7 @@ static void advance_windowaggregate( ExprState* arg_state = (ExprState*)lfirst(arg); fcinfo->arg[i] = ExecEvalExpr(arg_state, econtext, &fcinfo->argnull[i], NULL); + fcinfo->argTypes[i] = arg_state->resultType; i++; } @@ -176,6 +177,7 @@ static void advance_windowaggregate( *fcinfo, &(peraggstate->transfn), num_arguments + 1, perfuncstate->winCollation, (Node*)winstate, NULL); fcinfo->arg[0] = peraggstate->transValue; fcinfo->argnull[0] = peraggstate->transValueIsNull; + fcinfo->argTypes[0] = InvalidOid; new_val = FunctionCallInvoke(fcinfo); /* * If pass-by-ref datatype, must copy the new value into aggcontext and @@ -216,6 +218,7 @@ static void finalize_windowaggregate(WindowAggState* winstate, WindowStatePerFun InitFunctionCallInfoData(fcinfo, &(peraggstate->finalfn), 1, perfuncstate->winCollation, (Node*)winstate, NULL); fcinfo.arg[0] = peraggstate->transValue; fcinfo.argnull[0] = peraggstate->transValueIsNull; + fcinfo.argTypes[0] = InvalidOid; if (fcinfo.flinfo->fn_strict && peraggstate->transValueIsNull) { /* don't call a strict function with NULL inputs */ *result = (Datum)0; diff --git a/src/gausskernel/runtime/opfusion/opfusion_indexonlyscan.cpp b/src/gausskernel/runtime/opfusion/opfusion_indexonlyscan.cpp index 53cf1a61c..796d35366 100644 --- a/src/gausskernel/runtime/opfusion/opfusion_indexonlyscan.cpp +++ b/src/gausskernel/runtime/opfusion/opfusion_indexonlyscan.cpp @@ -221,7 +221,8 @@ TupleTableSlot *IndexOnlyScanFusion::getTupleSlotInternal() bool isUStore = RelationIsUstoreFormat(m_rel); bool bucket_changed = false; TupleTableSlot* tmpreslot = NULL; - tmpreslot = MakeSingleTupleTableSlot(RelationGetDescr(m_scandesc->heapRelation), false, rel->rd_tam_type); + tmpreslot = MakeSingleTupleTableSlot(RelationGetDescr(m_scandesc->heapRelation), + false, m_scandesc->heapRelation->rd_tam_type); while ((tid = scan_handler_idx_getnext_tid(m_scandesc, *m_direction, &bucket_changed)) != NULL) { HeapTuple tuple = NULL; diff --git a/src/gausskernel/storage/access/common/heaptuple.cpp b/src/gausskernel/storage/access/common/heaptuple.cpp index ba232aa0d..ccaa5203c 100644 --- a/src/gausskernel/storage/access/common/heaptuple.cpp +++ b/src/gausskernel/storage/access/common/heaptuple.cpp @@ -3401,7 +3401,7 @@ void heap_slot_store_heap_tuple(HeapTuple tuple, TupleTableSlot* slot, Buffer bu * * Note: Only the dead tuple of pg_partition needs to be verified in the current code. */ -bool HeapKeepInvisbleTuple(HeapTuple tuple, TupleDesc tupleDesc, KeepInvisbleTupleFunc checkKeepFunc) +bool HeapKeepInvisibleTuple(HeapTuple tuple, TupleDesc tupleDesc, KeepInvisbleTupleFunc checkKeepFunc) { static KeepInvisbleOpt keepInvisibleArray[] = { {PartitionRelationId, Anum_pg_partition_parttype, PartitionLocalIndexSkipping}, diff --git a/src/gausskernel/storage/access/heap/heapam.cpp b/src/gausskernel/storage/access/heap/heapam.cpp index d94a09803..adaa506ab 100755 --- a/src/gausskernel/storage/access/heap/heapam.cpp +++ b/src/gausskernel/storage/access/heap/heapam.cpp @@ -2373,6 +2373,15 @@ bool heap_fetch( return false; } +static TransactionId inline GetOldestXminForHot(Relation relation) +{ + if (IsCatalogRelation(relation) || RelationIsAccessibleInLogicalDecoding(relation)) { + return u_sess->utils_cxt.RecentGlobalCatalogXmin; + } else { + return u_sess->utils_cxt.RecentGlobalXmin; + } +} + /* * heap_hot_search_buffer - search HOT chain for tuple satisfying snapshot * @@ -2403,6 +2412,7 @@ bool heap_hot_search_buffer(ItemPointer tid, Relation relation, Buffer buffer, S bool at_chain_start = false; bool valid = false; bool skip = false; + TransactionId oldestXmin; /* If this is not the first call, previous call returned a (live!) tuple */ if (all_dead != NULL) { @@ -2411,6 +2421,8 @@ bool heap_hot_search_buffer(ItemPointer tid, Relation relation, Buffer buffer, S Assert(TransactionIdIsValid(u_sess->utils_cxt.RecentGlobalXmin)); + oldestXmin = GetOldestXminForHot(relation); + Assert(ItemPointerGetBlockNumber(tid) == BufferGetBlockNumber(buffer)); offnum = ItemPointerGetOffsetNumber(tid); at_chain_start = first_call; @@ -2533,7 +2545,7 @@ bool heap_hot_search_buffer(ItemPointer tid, Relation relation, Buffer buffer, S * request, check whether all chain members are dead to all * transactions. */ - if (all_dead && *all_dead && !HeapTupleIsSurelyDead(heap_tuple, u_sess->utils_cxt.RecentGlobalXmin)) { + if (all_dead && *all_dead && !HeapTupleIsSurelyDead(heap_tuple, oldestXmin)) { *all_dead = false; } @@ -3454,6 +3466,34 @@ static void HeapPageShiftBase(Buffer buffer, Page page, bool multi, int64 delta) } } +void heap_invalid_invisible_tuple(HeapTuple tuple) +{ + HeapTupleSetXmin(tuple, InvalidTransactionId); + HeapTupleSetXmax(tuple, InvalidTransactionId); + tuple->t_data->t_infomask &= ~HEAP_XMAX_BITS; + tuple->t_data->t_infomask &= ~HEAP_XMIN_COMMITTED; + tuple->t_data->t_infomask |= HEAP_XMIN_INVALID; + + Assert(!HeapTupleIsHotUpdated(tuple)); + + ereport(LOG, (errmsg("Dead and invisible tuple: t_ctid = { ip_blkid = { bi_hi = %hu, bi_lo = %hu }, " + "ip_posid = %hu }, t_xmin = %u, xmax = %u, infomask = %hu", + tuple->t_data->t_ctid.ip_blkid.bi_hi, + tuple->t_data->t_ctid.ip_blkid.bi_lo, + tuple->t_data->t_ctid.ip_posid, + tuple->t_data->t_choice.t_heap.t_xmin, + tuple->t_data->t_choice.t_heap.t_xmin, + tuple->t_data->t_infomask))); +} + +static inline bool heap_check_invalid_invisible_tuple(HeapTuple tuple, TupleDesc tupleDesc, + TransactionId cutoff_xid, Buffer buffer) +{ + return (t_thrd.proc->workingVersionNum >= INVALID_INVISIBLE_TUPLE_VERSION) + && !HeapTupleIsHotUpdated(tuple) && HeapKeepInvisibleTuple(tuple, tupleDesc) + && (HeapTupleSatisfiesVacuum(tuple, cutoff_xid, buffer) == HEAPTUPLE_DEAD); +} + /* * Freeze xids in the single heap page. Useful when we can't fit new xid even * with base shift. @@ -3466,7 +3506,9 @@ static int freeze_single_heap_page(Relation relation, Buffer buffer) OffsetNumber maxoff = InvalidOffsetNumber; HeapTupleData tuple; int nfrozen = 0; + int ninvalid = 0; OffsetNumber frozen[MaxOffsetNumber]; + OffsetNumber invalid[MaxOffsetNumber]; TransactionId latest_removed_xid = InvalidTransactionId; TransactionId oldest_xmin = InvalidTransactionId; TransactionId freeze_xid = InvalidTransactionId; @@ -3534,13 +3576,18 @@ static int freeze_single_heap_page(Relation relation, Buffer buffer) tuple.t_len = ItemIdGetLength(itemid); tuple.t_tableOid = RelationGetRelid(relation); tuple.t_bucketId = RelationGetBktid(relation); + ItemPointerSet(&(tuple.t_self), BufferGetBlockNumber(buffer), offnum); HeapTupleCopyBaseFromPage(&tuple, page); /* * Each non-removable tuple must be checked to see if it needs * freezing. Note we already have exclusive buffer lock. */ - if (heap_freeze_tuple(&tuple, freeze_xid, freeze_mxid, &changedMultiXid)) { + if (heap_check_invalid_invisible_tuple(&tuple, RelationGetDescr(relation), freeze_xid, buffer)) { + heap_invalid_invisible_tuple(&tuple); + Assert(tuple.t_tableOid == PartitionRelationId); + invalid[ninvalid++] = offnum; + } else if (heap_freeze_tuple(&tuple, freeze_xid, freeze_mxid, &changedMultiXid)) { frozen[nfrozen++] = offnum; } } /* scan along page */ @@ -3565,6 +3612,20 @@ static int freeze_single_heap_page(Relation relation, Buffer buffer) END_CRIT_SECTION(); } + if (ninvalid > 0) { + START_CRIT_SECTION(); + + MarkBufferDirty(buffer); + /* Now WAL-log freezing if necessary */ + if (RelationNeedsWAL(relation)) { + XLogRecPtr recptr = log_heap_invalid(relation, buffer, freeze_xid, + invalid, ninvalid); + PageSetLSN(page, recptr); + } + + END_CRIT_SECTION(); + } + gstrace_exit(GS_TRC_ID_freeze_single_heap_page); return nfrozen; } @@ -7976,7 +8037,7 @@ XLogRecPtr log_heap_freeze(Relation reln, Buffer buffer, TransactionId cutoff_xi OffsetNumber* offsets, int offcnt) { xl_heap_freeze xlrec; - XLogRecPtr recptr; + XLogRecPtr recptr = InvalidXLogRecPtr; bool useOldXlog = t_thrd.proc->workingVersionNum < ENHANCED_TUPLE_LOCK_VERSION_NUM || !MultiXactIdIsValid(cutoff_multi); #ifdef ENABLE_MULTIPLE_NODES @@ -8002,7 +8063,41 @@ XLogRecPtr log_heap_freeze(Relation reln, Buffer buffer, TransactionId cutoff_xi XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); XLogRegisterBufData(0, (char*)offsets, offcnt * sizeof(OffsetNumber)); - recptr = XLogInsert(RM_HEAP2_ID, useOldXlog ? XLOG_HEAP2_FREEZE : XLOG_HEAP2_FREEZE | XLOG_TUPLE_LOCK_UPGRADE_FLAG); + recptr = XLogInsert(RM_HEAP2_ID, + useOldXlog ? XLOG_HEAP2_FREEZE : XLOG_HEAP2_FREEZE | XLOG_TUPLE_LOCK_UPGRADE_FLAG); + + return recptr; +} + +/* + * Perform XLogInsert for a heap-invalid operation. Caller must already + * have modified the buffer and marked it dirty. + */ +XLogRecPtr log_heap_invalid(Relation reln, Buffer buffer, TransactionId cutoff_xid, OffsetNumber* offsets, + int offcnt) +{ + xl_heap_invalid xlrecInvalid; + XLogRecPtr recptr = InvalidXLogRecPtr; + + /* Caller should not call me on a non-WAL-logged relation */ + Assert(RelationNeedsWAL(reln)); + /* nor when there are no tuples to invalid */ + Assert(offcnt > 0); + + xlrecInvalid.cutoff_xid = cutoff_xid; + + XLogBeginInsert(); + XLogRegisterData((char*)&xlrecInvalid, SizeOfHeapInvalid); + + /* + * The tuple-offsets array is not actually in the buffer, but pretend that + * it is. When XLogInsert stores the whole buffer, the offsets array need + * not be stored too. + */ + XLogRegisterBuffer(0, buffer, REGBUF_STANDARD); + XLogRegisterBufData(0, (char*)offsets, offcnt * sizeof(OffsetNumber)); + + recptr = XLogInsert(RM_HEAP3_ID, XLOG_HEAP3_INVALID); return recptr; } @@ -8665,6 +8760,37 @@ static void heap_xlog_freeze(XLogReaderState* record) } } +static void heap_xlog_invalid(XLogReaderState* record) +{ + xl_heap_invalid* xlrecInvalid = (xl_heap_invalid*)XLogRecGetData(record); + TransactionId cutoff_xid = xlrecInvalid->cutoff_xid; + RedoBufferInfo buffer; + + /* + * In Hot Standby mode, ensure that there's no queries running which still + * consider the frozen xids as running. + */ + if (InHotStandby && g_supportHotStandby) { + RelFileNode rnode; + + (void)XLogRecGetBlockTag(record, HEAP_FREEZE_ORIG_BLOCK_NUM, &rnode, NULL, NULL); + XLogRecPtr lsn = record->EndRecPtr; + ResolveRecoveryConflictWithSnapshot(cutoff_xid, rnode, lsn); + } + + if (XLogReadBufferForRedo(record, HEAP_FREEZE_ORIG_BLOCK_NUM, &buffer) == BLK_NEEDS_REDO) { + Size blkdatalen; + char* blkdata = XLogRecGetBlockData(record, HEAP_FREEZE_ORIG_BLOCK_NUM, &blkdatalen); + + HeapXlogInvalidOperatorPage(&buffer, (void*)blkdata, blkdatalen); + MarkBufferDirty(buffer.buf); + + } + if (BufferIsValid(buffer.buf)) { + UnlockReleaseBuffer(buffer.buf); + } +} + /* * Replay XLOG_HEAP2_VISIBLE record. * The critical integrity requirement here is that we must never end up with @@ -9289,6 +9415,9 @@ void heap3_redo(XLogReaderState* record) break; case XLOG_HEAP3_REWRITE: break; + case XLOG_HEAP3_INVALID: + heap_xlog_invalid(record); + break; default: ereport(PANIC, (errmsg("heap3_redo: unknown op code %hhu", info))); } diff --git a/src/gausskernel/storage/access/heap/hio.cpp b/src/gausskernel/storage/access/heap/hio.cpp index f0531a906..0061ab0b4 100644 --- a/src/gausskernel/storage/access/heap/hio.cpp +++ b/src/gausskernel/storage/access/heap/hio.cpp @@ -379,10 +379,18 @@ Buffer RelationGetBufferForTuple(Relation relation, Size len, Buffer other_buffe Size save_free_space = 0; BlockNumber target_block, other_block; bool need_lock = false; - bool last_page_tested = false; Size extralen = 0; HeapPageHeader phdr; + /* + * Blocks that extended one by one are different from bulk-extend blocks, and + * are not recorded into FSM. As its creator session close this realtion, they + * can not be used by any other body. It is especially obvious for partition + * bulk insert. Here, if no avaiable found in FSM, we check the last block to + * reuse the 'leaked free space' mentioned earlier. + */ + bool test_last_block = false; + len = MAXALIGN(len); /* be conservative */ /* Bulk insert is not supported for updates, only inserts. */ @@ -454,7 +462,6 @@ Buffer RelationGetBufferForTuple(Relation relation, Size len, Buffer other_buffe if (nblocks > 0) { target_block = nblocks - 1; } - last_page_tested = true; } } /* When in append mode, cannot use cached block which smaller than rel end block */ @@ -484,7 +491,14 @@ loop: if (PageIsAllVisible(BufferGetPage(buffer))) { visibilitymap_pin(relation, target_block, vmbuffer); } - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + + if (!TryLockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE, !test_last_block)) { + Assert(test_last_block); + ReleaseBuffer(buffer); + + /* someone is using this block, give up and extend a new one. */ + break; + } } else if (other_block == target_block) { /* also easy case */ buffer = other_buffer; @@ -584,12 +598,12 @@ loop: * * The best is to record all pages into FSM using bulk-extend in later. */ - if (target_block == InvalidBlockNumber && !last_page_tested) { + if (target_block == InvalidBlockNumber && !test_last_block && other_buffer == InvalidBuffer) { BlockNumber nblocks = RelationGetNumberOfBlocks(relation); if (nblocks > 0) { target_block = nblocks - 1; } - last_page_tested = true; + test_last_block = true; } } diff --git a/src/gausskernel/storage/access/heap/pruneheap.cpp b/src/gausskernel/storage/access/heap/pruneheap.cpp index 964274ea0..be7da5975 100644 --- a/src/gausskernel/storage/access/heap/pruneheap.cpp +++ b/src/gausskernel/storage/access/heap/pruneheap.cpp @@ -344,6 +344,7 @@ static int heap_prune_chain(Relation relation, Buffer buffer, OffsetNumber rooto HeapTupleData tup; tup.t_tableOid = RelationGetRelid(relation); tup.t_bucketId = RelationGetBktid(relation); + bool keepInvisible = false; gstrace_entry(GS_TRC_ID_heap_prune_chain); @@ -389,7 +390,7 @@ static int heap_prune_chain(Relation relation, Buffer buffer, OffsetNumber rooto if (HeapTupleSatisfiesVacuum(&tup, oldest_xmin, buffer) == HEAPTUPLE_DEAD && !HeapTupleHeaderIsHotUpdated(htup)) { - if (HeapKeepInvisbleTuple(&tup, RelationGetDescr(relation))) { + if (HeapKeepInvisibleTuple(&tup, RelationGetDescr(relation))) { return ndeleted; } @@ -485,9 +486,8 @@ static int heap_prune_chain(Relation relation, Buffer buffer, OffsetNumber rooto } switch (HeapTupleSatisfiesVacuum(&tup, oldest_xmin, buffer)) { case HEAPTUPLE_DEAD: - if (!HeapKeepInvisbleTuple(&tup, RelationGetDescr(relation))) { - tupdead = true; - } + keepInvisible = HeapKeepInvisibleTuple(&tup, RelationGetDescr(relation)); + tupdead = true; break; case HEAPTUPLE_RECENTLY_DEAD: @@ -557,6 +557,11 @@ static int heap_prune_chain(Relation relation, Buffer buffer, OffsetNumber rooto prior_xmax = HeapTupleGetUpdateXid(&tup); } + /* There is only one dead tuple that needs to be retained and no processing is performed */ + if (keepInvisible && (nchain == 1)) { + latestdead = InvalidOffsetNumber; + } + /* * If we found a DEAD tuple in the chain, adjust the HOT chain so that all * the DEAD tuples at the start of the chain are removed and the root line @@ -571,6 +576,10 @@ static int heap_prune_chain(Relation relation, Buffer buffer, OffsetNumber rooto * right candidate for redirection. */ for (i = 1; (i < nchain) && (chainitems[i - 1] != latestdead); i++) { + // The entire chain is dead, but need to keep invisble tuple + if (keepInvisible && (i == nchain - 1)) { + break; + } heap_prune_record_unused(prstate, chainitems[i]); ndeleted++; } diff --git a/src/gausskernel/storage/access/heap/tuptoaster.cpp b/src/gausskernel/storage/access/heap/tuptoaster.cpp index 4411f8af3..82ec3fd95 100644 --- a/src/gausskernel/storage/access/heap/tuptoaster.cpp +++ b/src/gausskernel/storage/access/heap/tuptoaster.cpp @@ -44,6 +44,7 @@ #include "commands/vacuum.h" #include "utils/snapmgr.h" #include "mb/pg_wchar.h" +#include "access/tableam.h" #undef TOAST_DEBUG @@ -361,38 +362,49 @@ int64 calculate_huge_length(text *t) ScanKeyData toastkey; SysScanDesc toastscan; HeapTuple ttup; - Pointer chunk; bool isnull; - int2 bucketid; int64 len = 0; - struct varatt_external toast_pointer; + int offset = 0; + MemoryContext fetchLobContext = AllocSetContextCreate(CurrentMemoryContext, "lob calculate length context", + ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); struct varatt_lob_external large_toast_pointer; VARATT_EXTERNAL_GET_HUGE_POINTER(large_toast_pointer, t); - Relation toastrel = heap_open(large_toast_pointer.va_toastrelid, AccessShareLock); Relation toastidx = index_open(toastrel->rd_rel->reltoastidxid, AccessShareLock); TupleDesc toast_tup_desc = toastrel->rd_att; ScanKeyInit(&toastkey, (AttrNumber)1, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(large_toast_pointer.va_valueid)); toastscan = systable_beginscan_ordered(toastrel, toastidx, SnapshotToast, 1, &toastkey); - - int offset = 0; while ((ttup = systable_getnext_ordered(toastscan, ForwardScanDirection)) != NULL) { - chunk = DatumGetPointer(fastgetattr(ttup, CHUNK_DATA_ATTR, toast_tup_desc, &isnull)); - VARATT_EXTERNAL_GET_POINTER_B(toast_pointer, chunk, bucketid); + Pointer chunk = DatumGetPointer(fastgetattr(ttup, CHUNK_DATA_ATTR, toast_tup_desc, &isnull)); + MemoryContext oldCxt = MemoryContextSwitchTo(fetchLobContext); text *result = heap_tuple_untoast_attr((varlena *)chunk); const char* str = VARDATA_ANY(result); int limit = VARSIZE_ANY_EXHDR(result); + /* including character encoding, avoiding truncate */ + if (offset > 0) { + if (limit <= offset) { + offset -= limit; + continue; + } + str += offset; + limit -= offset; + offset = 0; + } - /* multibyte string may be truncated! */ - limit += offset; - len += (int64)pg_mbstrlen_with_len_toast(str - offset, &limit); - offset = limit; + len += (int64)pg_mbstrlen_with_len_toast(str, &limit); + if (limit != 0) { + Assert(limit < 0); + offset = -limit; + } + MemoryContextSwitchTo(oldCxt); + MemoryContextReset(fetchLobContext); } Assert(offset == 0); systable_endscan_ordered(toastscan); index_close(toastidx, AccessShareLock); heap_close(toastrel, AccessShareLock); + MemoryContextDelete(fetchLobContext); return len; } @@ -2733,6 +2745,16 @@ static struct varlena* toast_fetch_datum_slice(struct varlena* attr, int64 slice return result; } +/* ---------- + * toast_huge_fetch_datum_slice - + * + * return a Datum from the chunks saved in the toast relation + * attr: large_toast_pointer in user table + * sliceoffset: byte offset, from 0 + * length: byte length + * + * ---------- + */ static struct varlena *toast_huge_fetch_datum_slice(struct varlena *attr, int64 sliceoffset, int32 length) { ScanKeyData toastkey; @@ -2766,17 +2788,17 @@ static struct varlena *toast_huge_fetch_datum_slice(struct varlena *attr, int64 sliceoffset -= (toast_pointer.va_rawsize - VARHDRSZ); continue; } else { - if (length < (toast_pointer.va_rawsize - sliceoffset + 1)) { + if (length <= ((toast_pointer.va_rawsize - VARHDRSZ) - sliceoffset)) { curlength = length; } else { - curlength = toast_pointer.va_rawsize - sliceoffset + 1; + curlength = (toast_pointer.va_rawsize - VARHDRSZ) - sliceoffset; } first_chunk = heap_tuple_untoast_attr_slice((varlena *)chunk, sliceoffset, curlength); rc = memcpy_s(VARDATA(result) + totallength, length, VARDATA(first_chunk), curlength); securec_check(rc, "", ""); length -= curlength; totallength += curlength; - sliceoffset = 1; + sliceoffset = 0; } if (length == 0) { break; @@ -2790,6 +2812,17 @@ static struct varlena *toast_huge_fetch_datum_slice(struct varlena *attr, int64 return result; } +/* ---------- + * toast_huge_write_datum_slice - + * + * return a Datum from the chunks saved in the toast relation + * attr1: large_toast_pointer in user table + * attr2: less than 1G data + * sliceoffset: byte offset, from 0 + * length: byte length + * + * ---------- + */ struct varlena *toast_huge_write_datum_slice(struct varlena *attr1, struct varlena *attr2, int64 sliceoffset, int32 length) { @@ -2807,7 +2840,7 @@ struct varlena *toast_huge_write_datum_slice(struct varlena *attr1, struct varle VARATT_EXTERNAL_GET_HUGE_POINTER(large_toast_pointer, attr1); - if (sliceoffset < 1 || sliceoffset > large_toast_pointer.va_rawsize) { + if (sliceoffset < 0 || sliceoffset > large_toast_pointer.va_rawsize) { ereport(ERROR, (errcode(ERRCODE_UNEXPECTED_CHUNK_VALUE), errmsg("offset(%lu) is invalid.", sliceoffset))); } @@ -2836,16 +2869,16 @@ struct varlena *toast_huge_write_datum_slice(struct varlena *attr1, struct varle continue; } first_chunk = heap_tuple_untoast_attr((varlena *)chunk); - if (length < (toast_pointer.va_rawsize - VARHDRSZ - sliceoffset + 1)) { + if (length <= (toast_pointer.va_rawsize - VARHDRSZ - sliceoffset)) { curlength = length; } else { - curlength = toast_pointer.va_rawsize - VARHDRSZ - sliceoffset + 1; + curlength = toast_pointer.va_rawsize - VARHDRSZ - sliceoffset; } - rc = memcpy_s(VARDATA(first_chunk) + sliceoffset - 1, curlength, VARDATA(attr2) + totallength, curlength); + rc = memcpy_s(VARDATA(first_chunk) + sliceoffset, curlength, VARDATA(attr2) + totallength, curlength); securec_check(rc, "", ""); length -= curlength; totallength += curlength; - sliceoffset = 1; + sliceoffset = 0; toast_huge_fetch_and_append_datum(toastrel, toastidx, first_chunk, &firstchunkid, residx); } systable_endscan_ordered(toastscan); @@ -2900,3 +2933,32 @@ struct varlena * toast_pointer_fetch_data(TupleTableSlot* varSlot, Form_pg_attri return toast_pointer_lob; } +HeapTuple ctid_get_tuple(Relation relation, ItemPointer tid) +{ + Buffer user_buf = InvalidBuffer; + HeapTuple tuple = NULL; + HeapTuple new_tuple = NULL; + TM_Result result; + + /* alloc memory for old tuple and set tuple id */ + tuple = (HeapTupleData *)heaptup_alloc(BLCKSZ); + tuple->t_data = (HeapTupleHeader)((char *)tuple + HEAPTUPLESIZE); + Assert(tid != NULL); + tuple->t_self = *tid; + + if (tableam_tuple_fetch(relation, SnapshotAny, tuple, &user_buf, false, NULL)) { + result = HeapTupleSatisfiesUpdate(tuple, GetCurrentCommandId(true), user_buf, false); + if (result != TM_Ok) { + ereport(ERROR, (errcode(ERRCODE_SYSTEM_ERROR), errmsg("The tuple is updated, please use 'for update'."))); + } + + new_tuple = heapCopyTuple((HeapTuple)tuple, relation->rd_att, NULL); + ReleaseBuffer(user_buf); + } else { + heap_close(relation, NoLock); + ereport(ERROR, (errcode(ERRCODE_SYSTEM_ERROR), errmsg("The tuple is not found"))); + } + heap_freetuple(tuple); + + return new_tuple; +} diff --git a/src/gausskernel/storage/access/nbtree/nbtinsert.cpp b/src/gausskernel/storage/access/nbtree/nbtinsert.cpp index 65088796e..5fd5b0f5c 100644 --- a/src/gausskernel/storage/access/nbtree/nbtinsert.cpp +++ b/src/gausskernel/storage/access/nbtree/nbtinsert.cpp @@ -82,7 +82,6 @@ static bool _bt_isequal(Relation idxrel, Page page, OffsetNumber offnum, int key static void _bt_vacuum_one_page(Relation rel, Buffer buffer, Relation heapRel); static bool CheckItemIsAlive(ItemPointer tid, Relation relation, Snapshot snapshot, bool* all_dead, CUDescScan* cudescScan); -static bool CheckPartitionIsInvisible(GPIScanDesc gpiScan); /* * _bt_doinsert() -- Handle insertion of a single index tuple in the tree. @@ -2400,7 +2399,7 @@ static bool CheckItemIsAlive(ItemPointer tid, Relation relation, Snapshot snapsh } } -static bool CheckPartitionIsInvisible(GPIScanDesc gpiScan) +bool CheckPartitionIsInvisible(GPIScanDesc gpiScan) { if (OidRBTreeMemberOid(gpiScan->invisiblePartTreeForVacuum, gpiScan->currPartOid)) { return true; diff --git a/src/gausskernel/storage/access/redo/redo_heapam.cpp b/src/gausskernel/storage/access/redo/redo_heapam.cpp index b76ed0ae9..23fe139c0 100755 --- a/src/gausskernel/storage/access/redo/redo_heapam.cpp +++ b/src/gausskernel/storage/access/redo/redo_heapam.cpp @@ -151,6 +151,32 @@ void HeapXlogFreezeOperatorPage(RedoBufferInfo *buffer, void *recorddata, void * PageSetLSN(page, buffer->lsn); } +void HeapXlogInvalidOperatorPage(RedoBufferInfo *buffer, void *blkdata, Size datalen) +{ + Page page = buffer->pageinfo.page; + if (datalen > 0) { + OffsetNumber *offsets = (OffsetNumber *)blkdata; + OffsetNumber *offsets_end = (OffsetNumber *)((char *)offsets + datalen); + HeapTupleData tuple; + + while (offsets < offsets_end) { + /* offsets[] entries are one-based */ + ItemId lp = PageGetItemId(page, *offsets); + + tuple.t_data = (HeapTupleHeader)PageGetItem(page, lp); + tuple.t_len = ItemIdGetLength(lp); + HeapTupleCopyBaseFromPage(&tuple, page); + ItemPointerSet(&(tuple.t_self), buffer->blockinfo.blkno, *offsets); + + heap_invalid_invisible_tuple(&tuple); + + offsets++; + } + } + + PageSetLSN(page, buffer->lsn); +} + void HeapXlogVisibleOperatorPage(RedoBufferInfo *buffer, void *recorddata) { xl_heap_visible *xlrec = (xl_heap_visible *)recorddata; @@ -1083,6 +1109,45 @@ static XLogRecParseState *HeapXlogFreezeParseBlock(XLogReaderState *record, uint return recordstatehead; } +static XLogRecParseState *HeapXlogInvalidParseBlock(XLogReaderState *record, uint32 *blocknum) +{ + *blocknum = 1; + XLogRecParseState *blockstate = NULL; + XLogRecParseState *recordstatehead = NULL; + + XLogParseBufferAllocListFunc(record, &recordstatehead, NULL); + if (recordstatehead == NULL) { + return NULL; + } + XLogRecSetBlockDataState(record, HEAP_FREEZE_ORIG_BLOCK_NUM, recordstatehead); + + /* + * In Hot Standby mode, ensure that there's no queries running which still consider the + * invalid xids as running. + */ + if (g_supportHotStandby) { + (*blocknum)++; + /* need notify hot standby */ + XLogParseBufferAllocListFunc(record, &blockstate, recordstatehead); + if (blockstate == NULL) { + return NULL; + } + /* get cutoff xid */ + xl_heap_invalid *xlrecInvalid = (xl_heap_invalid *)XLogRecGetData(record); + TransactionId cutoff_xid = xlrecInvalid->cutoff_xid; + RelFileNode rnode; + + XLogRecGetBlockTag(record, HEAP_FREEZE_ORIG_BLOCK_NUM, &rnode, NULL, NULL); + + RelFileNodeForkNum filenode = + RelFileNodeForkNumFill(&rnode, InvalidBackendId, InvalidForkNumber, InvalidBlockNumber); + XLogRecSetBlockCommonState(record, BLOCK_DATA_INVALIDMSG_TYPE, filenode, blockstate); + XLogRecSetInvalidMsgState(&blockstate->blockparse.extra_rec.blockinvalidmsg, cutoff_xid); + } + + return recordstatehead; +} + static XLogRecParseState *HeapXlogCleanParseBlock(XLogReaderState *record, uint32 *blocknum) { xl_heap_clean *xlrec = (xl_heap_clean *)XLogRecGetData(record); @@ -1526,6 +1591,19 @@ static void HeapXlogFreezeBlock(XLogBlockHead *blockhead, XLogBlockDataParse *bl } } +static void HeapXlogInvalidBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatarec, RedoBufferInfo *bufferinfo) +{ + XLogBlockDataParse *datadecode = blockdatarec; + XLogRedoAction action = XLogCheckBlockDataRedoAction(datadecode, bufferinfo); + if (action == BLK_NEEDS_REDO) { + Size blkdatalen; + char *blkdata = XLogBlockDataGetBlockData(datadecode, &blkdatalen); + Assert(blkdata != NULL); + HeapXlogInvalidOperatorPage(bufferinfo, (void *)blkdata, blkdatalen); + MakeRedoBufferDirty(bufferinfo); + } +} + static void HeapXlogCleanBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatarec, RedoBufferInfo *bufferinfo) { XLogBlockDataParse *datadecode = blockdatarec; @@ -1638,6 +1716,9 @@ XLogRecParseState *Heap3RedoParseToBlock(XLogReaderState *record, uint32 *blockn break; case XLOG_HEAP3_REWRITE: break; + case XLOG_HEAP3_INVALID: + recordblockstate = HeapXlogInvalidParseBlock(record, blocknum); + break; default: ereport(PANIC, (errmsg("Heap3RedoParseToBlock: unknown op code %u", info))); } @@ -1654,6 +1735,9 @@ void Heap3RedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatar break; case XLOG_HEAP3_REWRITE: break; + case XLOG_HEAP3_INVALID: + HeapXlogInvalidBlock(blockhead, blockdatarec, bufferinfo); + break; default: ereport(PANIC, (errmsg("heap3_redo_block: unknown op code %u", info))); } diff --git a/src/gausskernel/storage/access/rmgrdesc/heapdesc.cpp b/src/gausskernel/storage/access/rmgrdesc/heapdesc.cpp index 00126e95e..f72e292e5 100644 --- a/src/gausskernel/storage/access/rmgrdesc/heapdesc.cpp +++ b/src/gausskernel/storage/access/rmgrdesc/heapdesc.cpp @@ -337,6 +337,8 @@ const char* heap3_type_name(uint8 subtype) return "heap3_new_cid"; } else if (info == XLOG_HEAP3_REWRITE) { return "heap3_rewrite"; + } else if (info == XLOG_HEAP3_INVALID) { + return "heap3_invalid"; } else { return "unkown_type"; } @@ -350,6 +352,26 @@ void heap3_desc(StringInfo buf, XLogReaderState *record) appendStringInfo(buf, "XLOG_HEAP_NEW_CID"); } else if (info == XLOG_HEAP3_REWRITE) { appendStringInfo(buf, "XLOG_HEAP2_REWRITE"); - } else + } else if (info == XLOG_HEAP3_INVALID) { + xl_heap_invalid *xlrecInvalid = (xl_heap_invalid *)XLogRecGetData(record); + + appendStringInfo(buf, "invalid: cutoff xid %lu", xlrecInvalid->cutoff_xid); + + if (!XLogRecHasBlockImage(record, 0)) { + Size datalen; + OffsetNumber *offsets = (OffsetNumber *)XLogRecGetBlockData(record, 0, &datalen); + if (datalen > 0) { + OffsetNumber *offsets_end = (OffsetNumber *)((char *)offsets + datalen); + + appendStringInfo(buf, " offsets: ["); + while (offsets < offsets_end) { + appendStringInfo(buf, " %d ", *offsets); + offsets++; + } + appendStringInfo(buf, "]"); + } + } + } else { appendStringInfo(buf, "UNKNOWN"); + } } diff --git a/src/gausskernel/storage/access/transam/double_write.cpp b/src/gausskernel/storage/access/transam/double_write.cpp index 83726fbbf..e90a026d8 100644 --- a/src/gausskernel/storage/access/transam/double_write.cpp +++ b/src/gausskernel/storage/access/transam/double_write.cpp @@ -1176,7 +1176,8 @@ void dw_file_check_rebuild() * Probably the gaussdb was killed during the first time startup after build, resulting in a half-written * DW file. So, log a warning message and remove the residual DW file. */ - ereport(WARNING, (errcode_for_file_access(), errmodule(MOD_DW), "batch flush DW file exists, deleting it")); + ereport(WARNING, (errcode_for_file_access(), errmodule(MOD_DW), + errmsg("batch flush DW file exists, deleting it"))); if (unlink(OLD_DW_FILE_NAME) != 0) { ereport(PANIC, (errcode_for_file_access(), errmodule(MOD_DW), @@ -1189,7 +1190,8 @@ void dw_file_check_rebuild() * Probably the gaussdb was killed during the first time startup after build, resulting in a half-written * DW file. So, log a warning message and remove the residual DW file. */ - ereport(WARNING, (errcode_for_file_access(), errmodule(MOD_DW), "single flush DW file exists, deleting it")); + ereport(WARNING, (errcode_for_file_access(), errmodule(MOD_DW), + errmsg("single flush DW file exists, deleting it"))); if (unlink(SINGLE_DW_FILE_NAME) != 0) { ereport(PANIC, (errcode_for_file_access(), errmodule(MOD_DW), @@ -1199,7 +1201,7 @@ void dw_file_check_rebuild() /* read meta file and remove batch file then remove meta file */ if (file_exists(DW_META_FILE)) { - ereport(WARNING, (errcode_for_file_access(), errmodule(MOD_DW), "batch meta file exists, deleting it")); + ereport(WARNING, (errcode_for_file_access(), errmodule(MOD_DW), errmsg("batch meta file exists, deleting it"))); fd = dw_open_file(DW_META_FILE); dw_recover_batch_meta_file(fd, &batch_meta_file); close(fd); @@ -1390,7 +1392,7 @@ static void dw_file_cxt_init_batch(int id, dw_batch_file_context *batch_file_cxt if (batch_file_cxt->fd == -1) { ereport(PANIC, (errcode_for_file_access(), errmodule(MOD_DW), - errmsg("Could not open file \"%s\"", batch_file_cxt->file_name))); + errmsg("Could not open file \"%s\"", batch_file_cxt->file_name))); } buf_size = DW_MEM_CTX_MAX_BLOCK_SIZE_FOR_NOHBK; @@ -1768,22 +1770,22 @@ void dw_init(bool shut_down) if (file_exists(OLD_DW_FILE_NAME)) { /* If the double write is disabled, log a warning message and remove the residual DW file. */ ereport(WARNING, (errcode_for_file_access(), errmodule(MOD_DW), - "batch flush DW file exists, deleting it when the double write is disabled")); + errmsg("batch flush DW file exists, deleting it when the double write is disabled"))); if (unlink(OLD_DW_FILE_NAME) != 0) { ereport(PANIC, (errcode_for_file_access(), errmodule(MOD_DW), - errmsg("Could not remove the residual batch flush DW single flush file"))); + errmsg("Could not remove the residual batch flush DW single flush file"))); } } if (file_exists(SINGLE_DW_FILE_NAME)) { /* If the double write is disabled, log a warning message and remove the single DW file. */ ereport(WARNING, (errcode_for_file_access(), errmodule(MOD_DW), - "single flush DW file exists, deleting it when the double write is disabled")); + errmsg("single flush DW file exists, deleting it when the double write is disabled"))); if (unlink(SINGLE_DW_FILE_NAME) != 0) { ereport(PANIC, (errcode_for_file_access(), errmodule(MOD_DW), - errmsg("Could not remove the residual single flush DW single flush file"))); + errmsg("Could not remove the residual single flush DW single flush file"))); } } diff --git a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp index 5e9ef1415..a8adcb3ae 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp @@ -198,7 +198,7 @@ static const RmgrDispatchData g_dispatchTable[RM_MAX_ID + 1] = { { DispatchSeqRecord, RmgrRecordInfoValid, RM_SEQ_ID, XLOG_SEQ_LOG, XLOG_SEQ_LOG }, { DispatchSpgistRecord, RmgrRecordInfoValid, RM_SPGIST_ID, XLOG_SPGIST_CREATE_INDEX, XLOG_SPGIST_VACUUM_REDIRECT }, { DispatchRepSlotRecord, RmgrRecordInfoValid, RM_SLOT_ID, XLOG_SLOT_CREATE, XLOG_TERM_LOG }, - { DispatchHeap3Record, RmgrRecordInfoValid, RM_HEAP3_ID, XLOG_HEAP3_NEW_CID, XLOG_HEAP3_REWRITE }, + { DispatchHeap3Record, RmgrRecordInfoValid, RM_HEAP3_ID, XLOG_HEAP3_NEW_CID, XLOG_HEAP3_INVALID }, { DispatchBarrierRecord, RmgrRecordInfoValid, RM_BARRIER_ID, XLOG_BARRIER_CREATE, XLOG_BARRIER_SWITCHOVER }, #ifdef ENABLE_MOT {DispatchMotRecord, NULL, RM_MOT_ID, 0, 0}, @@ -867,7 +867,13 @@ static bool DispatchRepSlotRecord(XLogReaderState *record, List *expectedTLIs, T /* Run from the dispatcher thread. */ static bool DispatchHeap3Record(XLogReaderState *record, List *expectedTLIs, TimestampTz recordXTime) { - DispatchTxnRecord(record, expectedTLIs); + uint8 info = (XLogRecGetInfo(record) & (~XLR_INFO_MASK)); + + if (info == XLOG_HEAP3_INVALID) { + DispatchRecordWithPages(record, expectedTLIs); + } else { + DispatchTxnRecord(record, expectedTLIs); + } return false; } diff --git a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp index a91f72893..b72db3d56 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp @@ -1,2990 +1,3002 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * page_redo.cpp - * PageRedoWorker is a thread of execution that replays data page logs. - * It provides a synchronization mechanism for replaying logs touching - * multiple pages. - * - * In the current implementation, logs modifying the same page must - * always be replayed by the same worker. There is no mechanism for - * an idle worker to "steal" work from a busy worker. - * - * IDENTIFICATION - * src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp - * - * ------------------------------------------------------------------------- - */ - -#include -#include - -#include "postgres.h" -#include "knl/knl_variable.h" -#include "gs_thread.h" -#include "miscadmin.h" -#include "access/xact.h" -#include "access/xlog.h" -#include "access/xlog_internal.h" -#include "access/xlogutils.h" -#include "access/xlogproc.h" -#include "access/nbtree.h" -#include "catalog/storage_xlog.h" -#include "gssignal/gs_signal.h" -#include "libpq/pqsignal.h" -#include "postmaster/postmaster.h" -#include "storage/ipc.h" -#include "storage/freespace.h" -#include "storage/smgr/smgr.h" -#include "storage/standby.h" -#include "storage/pmsignal.h" -#include "utils/guc.h" -#include "utils/palloc.h" -#include "portability/instr_time.h" -#include "postmaster/startup.h" -#include "postmaster/pagerepair.h" -#include "catalog/storage.h" -#include -#include -#include "commands/dbcommands.h" -#include "commands/tablespace.h" -#include "access/extreme_rto/page_redo.h" -#include "access/extreme_rto/dispatcher.h" -#include "access/extreme_rto/txn_redo.h" -#include "pgstat.h" -#include "access/extreme_rto/batch_redo.h" -#include "access/multi_redo_api.h" -#include "replication/walreceiver.h" -#include "replication/datareceiver.h" -#include "pgxc/barrier.h" -#ifdef ENABLE_MOT -#include "storage/mot/mot_fdw.h" -#endif - -#ifdef EXTREME_RTO_DEBUG -#include -#include - -#include - -#include - -#endif - -#ifdef ENABLE_UT -#include "utils/utesteventutil.h" -#define STATIC -#else -#define STATIC static -#endif - -namespace extreme_rto { -static const int MAX_PARSE_BUFF_NUM = PAGE_WORK_QUEUE_SIZE * 10 * 3; -static const int MAX_LOCAL_BUFF_NUM = PAGE_WORK_QUEUE_SIZE * 10 * 3; - -static const char *const PROCESS_TYPE_CMD_ARG = "--forkpageredo"; -static char g_AUXILIARY_TYPE_CMD_ARG[16] = {0}; - -THR_LOCAL PageRedoWorker *g_redoWorker = NULL; -THR_LOCAL RecordBufferState *g_recordbuffer = NULL; -RedoItem g_redoEndMark = { false, false, NULL, 0, NULL, 0 }; -RedoItem g_terminateMark = { false, false, NULL, 0, NULL, 0 }; -RedoItem g_GlobalLsnForwarder; -RedoItem g_cleanupMark; -RedoItem g_closefdMark; -RedoItem g_cleanInvalidPageMark; - -static const int PAGE_REDO_WORKER_ARG = 3; -static const int REDO_SLEEP_50US = 50; -static const int REDO_SLEEP_100US = 100; - -static void ApplySinglePageRecord(RedoItem *); -static void InitGlobals(); -static void LastMarkReached(); -static void SetupSignalHandlers(); -static void SigHupHandler(SIGNAL_ARGS); -static ThreadId StartWorkerThread(PageRedoWorker *); - -void RedoThrdWaitForExit(const PageRedoWorker *wk); -void AddRefRecord(void *rec); -void SubRefRecord(void *rec); -void GlobalLsnUpdate(); -static void TrxnMangerQueueCallBack(); -#ifdef USE_ASSERT_CHECKING -void RecordBlockCheck(void *rec, XLogRecPtr curPageLsn, uint32 blockId, bool replayed); -#endif -void AddRecordReadBlocks(void *rec, uint32 readblocks); - -RefOperate recordRefOperate = { - AddRefRecord, - SubRefRecord, -#ifdef USE_ASSERT_CHECKING - RecordBlockCheck, -#endif - AddRecordReadBlocks, -}; - -void UpdateRecordGlobals(RedoItem *item, HotStandbyState standbyState) -{ - t_thrd.xlog_cxt.ReadRecPtr = item->record.ReadRecPtr; - t_thrd.xlog_cxt.EndRecPtr = item->record.EndRecPtr; - t_thrd.xlog_cxt.expectedTLIs = item->expectedTLIs; - /* apply recoveryinfo will change standbystate see UpdateRecordGlobals */ - t_thrd.xlog_cxt.standbyState = standbyState; - t_thrd.xlog_cxt.XLogReceiptTime = item->syncXLogReceiptTime; - t_thrd.xlog_cxt.XLogReceiptSource = item->syncXLogReceiptSource; - u_sess->utils_cxt.RecentXmin = item->RecentXmin; - t_thrd.xlog_cxt.server_mode = item->syncServerMode; -} - -/* Run from the dispatcher thread. */ -PageRedoWorker *StartPageRedoWorker(PageRedoWorker *worker) -{ - Assert(worker); - uint32 id = worker->id; - ThreadId threadId = StartWorkerThread(worker); - if (threadId == 0) { - ereport(WARNING, (errmsg("Cannot create page-redo-worker thread: %u, %m.", id))); - DestroyPageRedoWorker(worker); - return NULL; - } else { - ereport(LOG, (errmsg("StartPageRedoWorker successfully create page-redo-worker id: %u, threadId:%lu.", id, - worker->tid.thid))); - } - g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[id].threadId = threadId; - SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.rwlock)); - uint32 state = pg_atomic_read_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[id].threadState)); - if (state != PAGE_REDO_WORKER_READY) { - g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[id].threadState = PAGE_REDO_WORKER_START; - } - SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); - return worker; -} - -void RedoWorkerQueueCallBack() -{ - RedoInterruptCallBack(); -} - -bool RedoWorkerIsUndoSpaceWorker() -{ - return g_redoWorker->isUndoSpaceWorker; -} - -/* Run from the dispatcher thread. */ -PageRedoWorker *CreateWorker(uint32 id) -{ - PageRedoWorker *tmp = (PageRedoWorker *)palloc0(sizeof(PageRedoWorker) + EXTREME_RTO_ALIGN_LEN); - PageRedoWorker *worker; - worker = (PageRedoWorker *)TYPEALIGN(EXTREME_RTO_ALIGN_LEN, tmp); - worker->selfOrinAddr = tmp; - worker->id = id; - worker->index = 0; - worker->tid.thid = InvalidTid; - worker->proc = NULL; - worker->initialServerMode = (ServerMode)t_thrd.xlog_cxt.server_mode; - worker->initialTimeLineID = t_thrd.xlog_cxt.ThisTimeLineID; - worker->expectedTLIs = t_thrd.xlog_cxt.expectedTLIs; - worker->recoveryTargetTLI = t_thrd.xlog_cxt.recoveryTargetTLI; - worker->recoveryRestoreCommand = t_thrd.xlog_cxt.recoveryRestoreCommand; - worker->ArchiveRecoveryRequested = t_thrd.xlog_cxt.ArchiveRecoveryRequested; - worker->StandbyModeRequested = t_thrd.xlog_cxt.StandbyModeRequested; - worker->InArchiveRecovery = t_thrd.xlog_cxt.InArchiveRecovery; - worker->InRecovery = t_thrd.xlog_cxt.InRecovery; - worker->ArchiveRestoreRequested = t_thrd.xlog_cxt.ArchiveRestoreRequested; - worker->minRecoveryPoint = t_thrd.xlog_cxt.minRecoveryPoint; - - worker->pendingHead = NULL; - worker->pendingTail = NULL; - worker->queue = SPSCBlockingQueueCreate(PAGE_WORK_QUEUE_SIZE, RedoWorkerQueueCallBack); - worker->lastCheckedRestartPoint = InvalidXLogRecPtr; - worker->lastReplayedEndRecPtr = InvalidXLogRecPtr; - worker->standbyState = (HotStandbyState)t_thrd.xlog_cxt.standbyState; - worker->StandbyMode = t_thrd.xlog_cxt.StandbyMode; - worker->latestObservedXid = t_thrd.storage_cxt.latestObservedXid; - worker->DataDir = t_thrd.proc_cxt.DataDir; - worker->RecentXmin = u_sess->utils_cxt.RecentXmin; - worker->xlogInvalidPages = NULL; - PosixSemaphoreInit(&worker->phaseMarker, 0); - worker->oldCtx = NULL; - worker->fullSyncFlag = 0; -#if (!defined __x86_64__) && (!defined __aarch64__) - SpinLockInit(&worker->ptrLck); -#endif - worker->parseManager.memctl.isInit = false; - worker->parseManager.parsebuffers = NULL; - worker->remoteReadPageNum = 0; - worker->badPageHashTbl = BadBlockHashTblCreate(); - return worker; -} - -/* Run from the dispatcher thread. */ -static ThreadId StartWorkerThread(PageRedoWorker *worker) -{ - worker->tid.thid = initialize_util_thread(PAGEREDO, worker); - return worker->tid.thid; -} - -/* Run from the dispatcher thread. */ -void DestroyPageRedoWorker(PageRedoWorker *worker) -{ - PosixSemaphoreDestroy(&worker->phaseMarker); - SPSCBlockingQueueDestroy(worker->queue); - XLogRedoBufferDestoryFunc(&(worker->bufferManager)); - XLogParseBufferDestoryFunc(&(worker->parseManager)); - pfree(worker->selfOrinAddr); -} - -/* automic write for lastReplayedReadRecPtr and lastReplayedEndRecPtr */ -void SetCompletedReadEndPtr(PageRedoWorker *worker, XLogRecPtr readPtr, XLogRecPtr endPtr) -{ - volatile PageRedoWorker *tmpWk = worker; -#if defined(__x86_64__) || defined(__aarch64__) - uint128_u exchange; - uint128_u current; - uint128_u compare = atomic_compare_and_swap_u128((uint128_u *)&tmpWk->lastReplayedReadRecPtr); - - Assert(sizeof(tmpWk->lastReplayedReadRecPtr) == 8); - Assert(sizeof(tmpWk->lastReplayedEndRecPtr) == 8); - - exchange.u64[0] = (uint64)readPtr; - exchange.u64[1] = (uint64)endPtr; - -loop: - current = atomic_compare_and_swap_u128((uint128_u *)&tmpWk->lastReplayedReadRecPtr, compare, exchange); - if (!UINT128_IS_EQUAL(compare, current)) { - UINT128_COPY(compare, current); - goto loop; - } -#else - SpinLockAcquire(&tmpWk->ptrLck); - tmpWk->lastReplayedReadRecPtr = readPtr; - tmpWk->lastReplayedEndRecPtr = endPtr; - SpinLockRelease(&tmpWk->ptrLck); -#endif /* __x86_64__ || __aarch64__ */ -} - -/* automic write for lastReplayedReadRecPtr and lastReplayedEndRecPtr */ -void GetCompletedReadEndPtr(PageRedoWorker *worker, XLogRecPtr *readPtr, XLogRecPtr *endPtr) -{ - volatile PageRedoWorker *tmpWk = worker; -#if defined(__x86_64__) || defined(__aarch64__) - uint128_u compare = atomic_compare_and_swap_u128((uint128_u *)&tmpWk->lastReplayedReadRecPtr); - Assert(sizeof(tmpWk->lastReplayedReadRecPtr) == 8); - Assert(sizeof(tmpWk->lastReplayedEndRecPtr) == 8); - - *readPtr = (XLogRecPtr)compare.u64[0]; - *endPtr = (XLogRecPtr)compare.u64[1]; -#else - SpinLockAcquire(&tmpWk->ptrLck); - *readPtr = tmpWk->lastReplayedReadRecPtr; - *endPtr = tmpWk->lastReplayedEndRecPtr; - SpinLockRelease(&tmpWk->ptrLck); -#endif /* __x86_64__ || __aarch64__ */ -} - -/* Run from both the dispatcher and the worker thread. */ -bool IsPageRedoWorkerProcess(int argc, char *argv[]) -{ - return strcmp(argv[1], PROCESS_TYPE_CMD_ARG) == 0; -} - -/* Run from the worker thread. */ -void AdaptArgvForPageRedoWorker(char *argv[]) -{ - if (g_AUXILIARY_TYPE_CMD_ARG[0] == 0) - sprintf_s(g_AUXILIARY_TYPE_CMD_ARG, sizeof(g_AUXILIARY_TYPE_CMD_ARG), "-x%d", PageRedoProcess); - argv[3] = g_AUXILIARY_TYPE_CMD_ARG; -} - -/* Run from the worker thread. */ -void GetThreadNameIfPageRedoWorker(int argc, char *argv[], char **threadNamePtr) -{ - if (*threadNamePtr == NULL && IsPageRedoWorkerProcess(argc, argv)) - *threadNamePtr = "PageRedoWorker"; -} - -/* Run from the worker thread. */ -uint32 GetMyPageRedoWorkerIdWithLock() -{ - bool isWorkerStarting = false; - SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.rwlock)); - isWorkerStarting = ((g_instance.comm_cxt.predo_cxt.state == REDO_STARTING_BEGIN) ? true : false); - SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); - if (!isWorkerStarting) { - ereport(WARNING, (errmsg("GetMyPageRedoWorkerIdWithLock Page-redo-worker exit."))); - proc_exit(0); - } - - return g_redoWorker->id; -} - -/* Run from any worker thread. */ -PGPROC *GetPageRedoWorkerProc(PageRedoWorker *worker) -{ - return worker->proc; -} - -void HandlePageRedoPageRepair(RepairBlockKey key, XLogPhyBlock pblk) -{ - RecordBadBlockAndPushToRemote(g_redoWorker->curRedoBlockState, CRC_CHECK_FAIL, InvalidXLogRecPtr, pblk); -} - -void HandlePageRedoInterrupts() -{ - if (t_thrd.page_redo_cxt.got_SIGHUP) { - t_thrd.page_redo_cxt.got_SIGHUP = false; - ProcessConfigFile(PGC_SIGHUP); - } - - if (t_thrd.page_redo_cxt.check_repair) { - SeqCheckRemoteReadAndRepairPage(); - t_thrd.page_redo_cxt.check_repair = false; - } - - if (t_thrd.page_redo_cxt.shutdown_requested) { - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("page worker id %u exit for request", g_redoWorker->id))); - - pg_atomic_write_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[g_redoWorker->id].threadState), - PAGE_REDO_WORKER_EXIT); - - proc_exit(1); - } -} - -void ReferenceRedoItem(void *item) -{ - RedoItem *redoItem = (RedoItem *)item; - AddRefRecord(&redoItem->record); -} - -void DereferenceRedoItem(void *item) -{ - RedoItem *redoItem = (RedoItem *)item; - SubRefRecord(&redoItem->record); -} - -#define STRUCT_CONTAINER(type, membername, ptr) ((type *)((char *)(ptr)-offsetof(type, membername))) - -#ifdef USE_ASSERT_CHECKING -void RecordBlockCheck(void *rec, XLogRecPtr curPageLsn, uint32 blockId, bool replayed) -{ - XLogReaderState *record = (XLogReaderState *)rec; - if (record->blocks[blockId].forknum != MAIN_FORKNUM) { - return; - } - - if (replayed) { - uint32 rmid = XLogRecGetRmid(record); - uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; - - if (curPageLsn == InvalidXLogRecPtr && (rmid == RM_HEAP2_ID || rmid == RM_HEAP_ID || rmid == RM_HEAP3_ID)) { - uint32 shiftSize = 32; - ereport(LOG, (errmsg("pass checked, record lsn:%X/%X, type: %u %u", - static_cast(record->EndRecPtr >> shiftSize), static_cast(record->EndRecPtr), - record->decoded_record->xl_rmid, record->decoded_record->xl_info))); - } else if (!(rmid == RM_HEAP2_ID && info == XLOG_HEAP2_VISIBLE) && - !(rmid == RM_HEAP_ID && info == XLOG_HEAP_NEWPAGE)) { - Assert(XLByteLE(record->EndRecPtr, curPageLsn)); - } - } - - Assert(blockId < (XLR_MAX_BLOCK_ID + 1)); - record->blocks[blockId].replayed = 1; -} - -#endif - -void AddRecordReadBlocks(void *rec, uint32 readblocks) -{ - XLogReaderState *record = (XLogReaderState *)rec; - record->readblocks += readblocks; -} - -void AddRefRecord(void *rec) -{ - pg_memory_barrier(); -#ifndef EXTREME_RTO_DEBUG - (void)pg_atomic_fetch_add_u32(&((XLogReaderState *)rec)->refcount, 1); -#else - uint32 relCount = pg_atomic_fetch_add_u32(&((XLogReaderState *)rec)->refcount, 1); - - const int stack_size = 5; - const int max_out_put_buf = 4096; - void *buffer[stack_size]; - int nptrs; - char output[max_out_put_buf]; - char **strings; - nptrs = backtrace(buffer, stack_size); - strings = backtrace_symbols(buffer, nptrs); - - int ret = sprintf_s(output, sizeof(output), "before add relcount %u lsn %X/%X call back trace: \n", relCount, - (uint32)(((XLogReaderState *)rec)->EndRecPtr >> 32), - (uint32)(((XLogReaderState *)rec)->EndRecPtr)); - securec_check_ss_c(ret, "\0", "\0"); - for (int i = 0; i < nptrs; ++i) { - ret = strcat_s(output, max_out_put_buf - strlen(output), strings[i]); - securec_check_ss_c(ret, "\0", "\0"); - ret = strcat_s(output, max_out_put_buf - strlen(output), "\n"); - securec_check_ss_c(ret, "\0", "\0"); - } - - free(strings); - ereport(LOG, (errcode(ERRCODE_DATA_CORRUPTED), errmsg(" AddRefRecord print: %s", output))); - -#endif -} - -void SubRefRecord(void *rec) -{ - pg_memory_barrier(); - Assert(((XLogReaderState *)rec)->refcount != 0); - uint32 relCount = pg_atomic_sub_fetch_u32(&((XLogReaderState *)rec)->refcount, 1); -#ifdef EXTREME_RTO_DEBUG - const int stack_size = 5; - const int max_out_put_buf = 4096; - void *buffer[stack_size]; - int nptrs; - char output[max_out_put_buf]; - char **strings; - nptrs = backtrace(buffer, stack_size); - strings = backtrace_symbols(buffer, nptrs); - - int ret = sprintf_s(output, sizeof(output), "after sub relcount %u lsn %X/%X call back trace:\n", relCount, - (uint32)(((XLogReaderState *)rec)->EndRecPtr >> 32), - (uint32)(((XLogReaderState *)rec)->EndRecPtr)); - securec_check_ss_c(ret, "\0", "\0"); - for (int i = 0; i < nptrs; ++i) { - ret = strcat_s(output, max_out_put_buf - strlen(output), strings[i]); - securec_check_ss_c(ret, "\0", "\0"); - ret = strcat_s(output, max_out_put_buf - strlen(output), "\n"); - securec_check_ss_c(ret, "\0", "\0"); - } - free(strings); - ereport(LOG, (errcode(ERRCODE_DATA_CORRUPTED), errmsg(" SubRefRecord print: %s", output))); - -#endif - - if (relCount == 0) { - RedoItem *item = STRUCT_CONTAINER(RedoItem, record, rec); - FreeRedoItem(item); - } -} - -bool BatchRedoParseItemAndDispatch(RedoItem *item) -{ - uint32 blockNum = 0; - XLogRecParseState *recordblockstate = XLogParseToBlockForExtermeRTO(&item->record, &blockNum); - if (recordblockstate == NULL) { - if (blockNum == 0) { - return false; - } - return true; /* out of mem */ - } - - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - AddPageRedoItem(myRedoLine->managerThd, recordblockstate); - return false; -} - -void BatchRedoDistributeEndMark(void) -{ - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - SendPageRedoEndMark(myRedoLine->managerThd); -} - -void BatchRedoProcLsnForwarder(RedoItem *lsnForwarder) -{ - SetCompletedReadEndPtr(g_redoWorker, lsnForwarder->record.ReadRecPtr, lsnForwarder->record.EndRecPtr); - (void)pg_atomic_sub_fetch_u32(&lsnForwarder->record.refcount, 1); - - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - AddPageRedoItem(myRedoLine->managerThd, lsnForwarder); -} - -void BatchRedoProcCleanupMark(RedoItem *cleanupMark) -{ - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); - AddPageRedoItem(myRedoLine->managerThd, cleanupMark); - ereport(LOG, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]BatchRedoProcCleanupMark has cleaned InvalidPages"))); -} - -#ifdef ENABLE_DISTRIBUTE_TEST -// inject delay to slow the process and also can be used as UT mock stub -void InjectDelayWaitRedoPageManagerQueueEmpty() -{ - const uint32 sleepTime = 1000000; - ereport(LOG, (errmsg("ProcessRedoPageManagerQueueEmpty sleep"))); - pg_usleep(sleepTime); -} -#endif - -void WaitAllRedoWorkerQueueEmpty() -{ - if ((get_real_recovery_parallelism() > 1) && (GetBatchCount() > 0)) { - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; - - for (uint32 i = 0; i < WorkerNumPerMng; ++i) { - while (!SPSCBlockingQueueIsEmpty(myRedoLine->redoThd[i]->queue)) { - RedoInterruptCallBack(); - } - } - } -} - -bool BatchRedoDistributeItems(void **eleArry, uint32 eleNum) -{ - bool parsecomplete = false; - for (uint32 i = 0; i < eleNum; i++) { - if (eleArry[i] == (void *)&g_redoEndMark) { - return true; - } else if (eleArry[i] == (void *)&g_GlobalLsnForwarder) { - BatchRedoProcLsnForwarder((RedoItem *)eleArry[i]); - } else if (eleArry[i] == (void *)&g_cleanupMark) { - BatchRedoProcCleanupMark((RedoItem *)eleArry[i]); - } else if (eleArry[i] == (void *)&g_closefdMark) { - smgrcloseall(); - } else if (eleArry[i] == (void *)&g_cleanInvalidPageMark) { - forget_range_invalid_pages((void *)eleArry[i]); - } else { - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - RedoItem *item = (RedoItem *)eleArry[i]; - UpdateRecordGlobals(item, g_redoWorker->standbyState); - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3], - g_redoWorker->timeCostList[TIME_COST_STEP_6]); - do { - parsecomplete = BatchRedoParseItemAndDispatch(item); - RedoInterruptCallBack(); - } while (parsecomplete); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - DereferenceRedoItem(item); - } - } - - return false; -} - -void BatchRedoMain() -{ - void **eleArry; - uint32 eleNum; - - (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); - XLogParseBufferInitFunc(&(g_redoWorker->parseManager), MAX_PARSE_BUFF_NUM, &recordRefOperate, - RedoInterruptCallBack); - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); - while (SPSCBlockingQueueGetAll(g_redoWorker->queue, &eleArry, &eleNum)) { - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); - bool isEnd = BatchRedoDistributeItems(eleArry, eleNum); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); - SPSCBlockingQueuePopN(g_redoWorker->queue, eleNum); - if (isEnd) - break; - - RedoInterruptCallBack(); - ADD_ABNORMAL_POSITION(1); - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); - } - - RedoThrdWaitForExit(g_redoWorker); - XLogParseBufferDestoryFunc(&(g_redoWorker->parseManager)); -} - -uint32 GetWorkerId(const RedoItemTag *redoItemTag, uint32 workerCount) -{ - if (workerCount != 0) { - return tag_hash(redoItemTag, sizeof(RedoItemTag)) % workerCount; - } - return 0; -} - -uint32 GetWorkerId(const uint32 attId, const uint32 workerCount) -{ - if (workerCount != 0) { - return attId % workerCount; - } - return 0; -} - -void RedoPageManagerDistributeToAllOneBlock(XLogRecParseState *ddlParseState) -{ - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; - - ddlParseState->nextrecord = NULL; - - for (uint32 i = 0; i < WorkerNumPerMng; ++i) { - XLogRecParseState *newState = XLogParseBufferCopy(ddlParseState); - AddPageRedoItem(myRedoLine->redoThd[i], newState); - } -} - -void RedoPageManagerDistributeBlockRecord(HTAB *redoItemHash, XLogRecParseState *parsestate) -{ - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; - HASH_SEQ_STATUS status; - RedoItemHashEntry *redoItemEntry = NULL; - HTAB *curMap = redoItemHash; - hash_seq_init(&status, curMap); - - while ((redoItemEntry = (RedoItemHashEntry *)hash_seq_search(&status)) != NULL) { - uint32 workId = GetWorkerId(&redoItemEntry->redoItemTag, WorkerNumPerMng); - AddPageRedoItem(myRedoLine->redoThd[workId], redoItemEntry->head); - - if (hash_search(curMap, (void *)&redoItemEntry->redoItemTag, HASH_REMOVE, NULL) == NULL) - ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("hash table corrupted"))); - } - - if (parsestate != NULL) { - RedoPageManagerDistributeToAllOneBlock(parsestate); - } -} - -void WaitCurrentPipeLineRedoWorkersQueueEmpty() -{ - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; - - for (uint32 i = 0; i < WorkerNumPerMng; ++i) { - while (!SPSCBlockingQueueIsEmpty(myRedoLine->redoThd[i]->queue)) { - RedoInterruptCallBack(); - } - } -} - -void DispatchEndMarkToRedoWorkerAndWait() -{ - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - const uint32 WorkerNumPerMng = get_page_redo_worker_num_per_manager(); - for (uint32 i = 0; i < WorkerNumPerMng; ++i) - SendPageRedoEndMark(myRedoLine->redoThd[i]); - - for (uint32 i = 0; i < myRedoLine->redoThdNum; i++) { - WaitPageRedoWorkerReachLastMark(myRedoLine->redoThd[i]); - } -} - -void RedoPageManagerDdlAction(XLogRecParseState *parsestate) -{ - switch (parsestate->blockparse.blockhead.block_valid) { - case BLOCK_DATA_DROP_DATABASE_TYPE: - xlog_db_drop(parsestate->blockparse.blockhead.end_ptr, parsestate->blockparse.blockhead.dbNode, - parsestate->blockparse.blockhead.spcNode); - break; - case BLOCK_DATA_CREATE_DATABASE_TYPE: - xlog_db_create(parsestate->blockparse.blockhead.dbNode, parsestate->blockparse.blockhead.spcNode, - parsestate->blockparse.extra_rec.blockdatabase.src_db_id, - parsestate->blockparse.extra_rec.blockdatabase.src_tablespace_id); - break; - case BLOCK_DATA_DROP_TBLSPC_TYPE: - xlog_drop_tblspc(parsestate->blockparse.blockhead.spcNode); - break; - case BLOCK_DATA_SEG_FILE_EXTEND_TYPE: - { - Assert(0); - } - break; - case BLOCK_DATA_SEG_SPACE_DROP: - case BLOCK_DATA_SEG_FULL_SYNC_TYPE: - case BLOCK_DATA_SEG_EXTEND: - ProcSegPageCommonRedo(parsestate); - break; - default: - break; - } -} - -void RedoPageManagerSmgrClose(XLogRecParseState *parsestate) -{ - switch (parsestate->blockparse.blockhead.block_valid) { - case BLOCK_DATA_DROP_DATABASE_TYPE: - smgrcloseall(); - break; - case BLOCK_DATA_SEG_FULL_SYNC_TYPE: - ProcSegPageJustFreeChildState(parsestate); - break; - default: - break; - } -} - -void RedoPageManagerSyncDdlAction(XLogRecParseState *parsestate) -{ - /* at this monent, all worker queue is empty ,just find out which one will do it */ - uint32 expected = 0; - const uint32 pipelineNum = g_dispatcher->pageLineNum; - pg_atomic_compare_exchange_u32(&g_dispatcher->syncEnterCount, &expected, pipelineNum); - uint32 entershareCount = pg_atomic_sub_fetch_u32(&g_dispatcher->syncEnterCount, 1); - - MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); - if (entershareCount == 0) { - /* do actual work */ - RedoPageManagerDdlAction(parsestate); - } else { - RedoPageManagerSmgrClose(parsestate); - do { - RedoInterruptCallBack(); - entershareCount = pg_atomic_read_u32(&g_dispatcher->syncEnterCount); - } while (entershareCount != 0); - } - (void)MemoryContextSwitchTo(oldCtx); - - expected = 0; - pg_atomic_compare_exchange_u32(&g_dispatcher->syncExitCount, &expected, pipelineNum); - uint32 exitShareCount = pg_atomic_sub_fetch_u32(&g_dispatcher->syncExitCount, 1); - while (exitShareCount != 0) { - RedoInterruptCallBack(); - exitShareCount = pg_atomic_read_u32(&g_dispatcher->syncExitCount); - } - - parsestate->nextrecord = NULL; - XLogBlockParseStateRelease(parsestate); -} - -void RedoPageManagerDoDropAction(XLogRecParseState *parsestate, HTAB *hashMap) -{ - XLogRecParseState *newState = XLogParseBufferCopy(parsestate); - PRTrackClearBlock(newState, hashMap); - RedoPageManagerDistributeBlockRecord(hashMap, parsestate); - WaitCurrentPipeLineRedoWorkersQueueEmpty(); - RedoPageManagerSyncDdlAction(parsestate); -} - -void RedoPageManagerDoSmgrAction(XLogRecParseState *recordblockstate) -{ - RedoBufferInfo bufferinfo = {0}; - void *blockrecbody; - XLogBlockHead *blockhead; - - blockhead = &recordblockstate->blockparse.blockhead; - blockrecbody = &recordblockstate->blockparse.extra_rec; - - XLogBlockInitRedoBlockInfo(blockhead, &bufferinfo.blockinfo); - - MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); - XLogBlockDdlDoSmgrAction(blockhead, blockrecbody, &bufferinfo); - (void)MemoryContextSwitchTo(oldCtx); - - recordblockstate->nextrecord = NULL; - XLogBlockParseStateRelease(recordblockstate); -} - -void RedoPageManagerDoDataTypeAction(XLogRecParseState *parsestate, HTAB *hashMap) -{ - XLogBlockDdlParse *ddlrecparse = NULL; - XLogBlockParseGetDdlParse(parsestate, ddlrecparse); - - if (ddlrecparse->blockddltype == BLOCK_DDL_DROP_RELNODE || - ddlrecparse->blockddltype == BLOCK_DDL_TRUNCATE_RELNODE) { - XLogRecParseState *newState = XLogParseBufferCopy(parsestate); - PRTrackClearBlock(newState, hashMap); - RedoPageManagerDistributeBlockRecord(hashMap, parsestate); - WaitCurrentPipeLineRedoWorkersQueueEmpty(); - } - - RedoPageManagerDoSmgrAction(parsestate); - -} - -void PageManagerProcLsnForwarder(RedoItem *lsnForwarder) -{ - SetCompletedReadEndPtr(g_redoWorker, lsnForwarder->record.ReadRecPtr, lsnForwarder->record.EndRecPtr); - (void)pg_atomic_sub_fetch_u32(&lsnForwarder->record.refcount, 1); - - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; - - for (uint32 i = 0; i < WorkerNumPerMng; ++i) { - AddPageRedoItem(myRedoLine->redoThd[i], lsnForwarder); - } -} - -void PageManagerDistributeBcmBlock(XLogRecParseState *preState) -{ - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; - uint32 workId = GetWorkerId((uint32)preState->blockparse.blockhead.forknum, WorkerNumPerMng); - AddPageRedoItem(myRedoLine->redoThd[workId], preState); -} - -void PageManagerProcCleanupMark(RedoItem *cleanupMark) -{ - PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; - const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; - g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); - for (uint32 i = 0; i < WorkerNumPerMng; ++i) { - AddPageRedoItem(myRedoLine->redoThd[i], cleanupMark); - } - ereport(LOG, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]PageManagerProcCleanupMark has cleaned InvalidPages"))); -} - -void PageManagerProcCheckPoint(HTAB *hashMap, XLogRecParseState *parseState) -{ - Assert(IsCheckPoint(parseState)); - RedoPageManagerDistributeBlockRecord(hashMap, parseState); - bool needWait = parseState->isFullSync; - if (needWait) { - pg_atomic_write_u32(&g_redoWorker->fullSyncFlag, 1); - } - - XLogBlockParseStateRelease(parseState); - uint32 val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); - while (val != 0) { - RedoInterruptCallBack(); - val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); - } - -#ifdef USE_ASSERT_CHECKING - int printLevel = WARNING; -#else - int printLevel = DEBUG1; -#endif - if (log_min_messages <= printLevel) { - GetThreadBufferLeakNum(); - } -} - -void PageManagerProcCreateTableSpace(HTAB *hashMap, XLogRecParseState *parseState) -{ - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - bool needWait = parseState->isFullSync; - if (needWait) { - pg_atomic_write_u32(&g_redoWorker->fullSyncFlag, 1); - } - - XLogBlockParseStateRelease(parseState); - uint32 val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); - while (val != 0) { - RedoInterruptCallBack(); - val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); - } -} - -void PageManagerProcSegFullSyncState(HTAB *hashMap, XLogRecParseState *parseState) -{ - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - WaitCurrentPipeLineRedoWorkersQueueEmpty(); - RedoPageManagerSyncDdlAction(parseState); -} - -void PageManagerProcSegPipeLineSyncState(HTAB *hashMap, XLogRecParseState *parseState) -{ - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - WaitCurrentPipeLineRedoWorkersQueueEmpty(); - MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); - - RedoPageManagerDdlAction(parseState); - - (void)MemoryContextSwitchTo(oldCtx); - XLogBlockParseStateRelease(parseState); -} - -static void WaitNextBarrier(XLogRecParseState *parseState) -{ - char* barrier = parseState->blockparse.extra_rec.blockbarrier.maindata; - uint8 info = parseState->blockparse.blockhead.xl_info & ~XLR_INFO_MASK; - volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; - - if (info == XLOG_BARRIER_COMMIT || !IS_DISASTER_RECOVER_MODE || !is_barrier_pausable(barrier)) - return; - - while (true) { - SpinLockAcquire(&walrcv->mutex); - if (BARRIER_LT((char *)barrier, (char *)walrcv->recoveryTargetBarrierId) || - BARRIER_LE((char *)barrier, (char *)walrcv->recoveryStopBarrierId)|| - BARRIER_EQ((char *)barrier, (char *)walrcv->recoverySwitchoverBarrierId)) { - SpinLockRelease(&walrcv->mutex); - break; - } else { - SpinLockRelease(&walrcv->mutex); - pg_usleep(1000L); - RedoInterruptCallBack(); - } - } -} - -void PageManagerRedoParseState(XLogRecParseState *preState) -{ - HTAB *hashMap = g_dispatcher->pageLines[g_redoWorker->slotId].managerThd->redoItemHash; - - switch (preState->blockparse.blockhead.block_valid) { - case BLOCK_DATA_MAIN_DATA_TYPE: - case BLOCK_DATA_UNDO_TYPE: - case BLOCK_DATA_VM_TYPE: - case BLOCK_DATA_FSM_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - PRTrackAddBlock(preState, hashMap); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - break; - case BLOCK_DATA_DDL_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); - RedoPageManagerDoDataTypeAction(preState, hashMap); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); - break; - case BLOCK_DATA_SEG_EXTEND: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); - PageManagerProcSegPipeLineSyncState(hashMap, preState); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); - break; - case BLOCK_DATA_DROP_DATABASE_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_5]); - RedoPageManagerDoDropAction(preState, hashMap); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_5]); - break; - case BLOCK_DATA_DROP_TBLSPC_TYPE: - /* just make sure any other ddl before drop tblspc is done */ - XLogBlockParseStateRelease(preState); - break; - case BLOCK_DATA_CREATE_DATABASE_TYPE: - case BLOCK_DATA_SEG_FILE_EXTEND_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - /* wait until queue empty */ - WaitCurrentPipeLineRedoWorkersQueueEmpty(); - /* do atcual action */ - RedoPageManagerSyncDdlAction(preState); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - break; - case BLOCK_DATA_SEG_FULL_SYNC_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); - PageManagerProcSegFullSyncState(hashMap, preState); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); - break; - case BLOCK_DATA_CREATE_TBLSPC_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); - PageManagerProcCreateTableSpace(hashMap, preState); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); - break; - case BLOCK_DATA_XLOG_COMMON_TYPE: - PageManagerProcCheckPoint(hashMap, preState); - break; - case BLOCK_DATA_NEWCU_TYPE: - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - PageManagerDistributeBcmBlock(preState); - break; - case BLOCK_DATA_SEG_SPACE_DROP: - case BLOCK_DATA_SEG_SPACE_SHRINK: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); - RedoPageManagerDistributeBlockRecord(hashMap, preState); - WaitCurrentPipeLineRedoWorkersQueueEmpty(); - RedoPageManagerSyncDdlAction(preState); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); - break; - case BLOCK_DATA_BARRIER_TYPE: - RedoPageManagerDistributeBlockRecord(hashMap, preState); - WaitNextBarrier(preState); - XLogBlockParseStateRelease(preState); - break; - default: - XLogBlockParseStateRelease(preState); - break; - } -} - -bool PageManagerRedoDistributeItems(void **eleArry, uint32 eleNum) -{ - HTAB *hashMap = g_dispatcher->pageLines[g_redoWorker->slotId].managerThd->redoItemHash; - - for (uint32 i = 0; i < eleNum; i++) { - if (eleArry[i] == (void *)&g_redoEndMark) { - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - return true; - } else if (eleArry[i] == (void *)&g_GlobalLsnForwarder) { - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - PageManagerProcLsnForwarder((RedoItem *)eleArry[i]); - continue; - } else if (eleArry[i] == (void *)&g_cleanupMark) { - PageManagerProcCleanupMark((RedoItem *)eleArry[i]); - continue; - } else if (eleArry[i] == (void *)&g_closefdMark) { - smgrcloseall(); - continue; - } else if (eleArry[i] == (void *)&g_cleanInvalidPageMark) { - forget_range_invalid_pages((void *)eleArry[i]); - continue; - } - XLogRecParseState *recordblockstate = (XLogRecParseState *)eleArry[i]; - XLogRecParseState *nextState = recordblockstate; - do { - XLogRecParseState *preState = nextState; - nextState = (XLogRecParseState *)nextState->nextrecord; - preState->nextrecord = NULL; -#ifdef ENABLE_UT - TestXLogRecParseStateEventProbe(UTEST_EVENT_RTO_PAGEMGR_REDO_BEFORE_DISTRIBUTE_ITEMS, - __FUNCTION__, preState); -#endif - - PageManagerRedoParseState(preState); -#ifdef ENABLE_UT - TestXLogRecParseStateEventProbe(UTEST_EVENT_RTO_PAGEMGR_REDO_AFTER_DISTRIBUTE_ITEMS, - __FUNCTION__, preState); -#endif - } while (nextState != NULL); - } - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_9]); - RedoPageManagerDistributeBlockRecord(hashMap, NULL); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_9]); - return false; -} - -void RedoPageManagerMain() -{ - void **eleArry; - uint32 eleNum; - - (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); - g_redoWorker->redoItemHash = PRRedoItemHashInitialize(g_redoWorker->oldCtx); - XLogParseBufferInitFunc(&(g_redoWorker->parseManager), MAX_PARSE_BUFF_NUM, &recordRefOperate, - RedoInterruptCallBack); - - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); - while (SPSCBlockingQueueGetAll(g_redoWorker->queue, &eleArry, &eleNum)) { - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); - bool isEnd = PageManagerRedoDistributeItems(eleArry, eleNum); - SPSCBlockingQueuePopN(g_redoWorker->queue, eleNum); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); - if (isEnd) - break; - - RedoInterruptCallBack(); - ADD_ABNORMAL_POSITION(5); - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); - } - - RedoThrdWaitForExit(g_redoWorker); - XLogParseBufferDestoryFunc(&(g_redoWorker->parseManager)); -} - -bool IsXactXlog(const XLogReaderState *record) -{ - if (XLogRecGetRmid(record) != RM_XACT_ID) { - return false; - } - return true; -} - -void TrxnManagerProcLsnForwarder(RedoItem *lsnForwarder) -{ - SetCompletedReadEndPtr(g_redoWorker, lsnForwarder->record.ReadRecPtr, lsnForwarder->record.EndRecPtr); - (void)pg_atomic_sub_fetch_u32(&lsnForwarder->record.refcount, 1); - - AddPageRedoItem(g_dispatcher->trxnLine.redoThd, lsnForwarder); -} - -void TrxnManagerProcCleanupMark(RedoItem *cleanupMark) -{ - g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); - AddPageRedoItem(g_dispatcher->trxnLine.redoThd, cleanupMark); - ereport(LOG, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]TrxnManagerProcCleanupMark has cleaned InvalidPages"))); -} - -bool TrxnManagerDistributeItemsBeforeEnd(RedoItem *item) -{ - bool exitFlag = false; - if (item == &g_redoEndMark) { - exitFlag = true; - } else if (item == (RedoItem *)&g_GlobalLsnForwarder) { - TrxnManagerProcLsnForwarder(item); - } else if (item == (RedoItem *)&g_cleanupMark) { - TrxnManagerProcCleanupMark(item); - } else if (item == (void *)&g_closefdMark) { - smgrcloseall(); - } else if (item == (void *)&g_cleanInvalidPageMark) { - forget_range_invalid_pages((void *)item); - } else { - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); - if (IsCheckPoint(&item->record) || IsTableSpaceDrop(&item->record) || IsTableSpaceCreate(&item->record) || - (IsXactXlog(&item->record) && XactWillRemoveRelFiles(&item->record)) || IsBarrierRelated(&item->record)) { - uint32 relCount; - do { - RedoInterruptCallBack(); - relCount = pg_atomic_read_u32(&item->record.refcount); - } while (relCount != 1); - } - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4], g_redoWorker->timeCostList[TIME_COST_STEP_5]); -#ifdef ENABLE_UT - TestXLogReaderProbe(UTEST_EVENT_RTO_TRXNMGR_DISTRIBUTE_ITEMS, - __FUNCTION__, &item->record); -#endif - AddPageRedoItem(g_dispatcher->trxnLine.redoThd, item); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_5]); - } - return exitFlag; -} - -void GlobalLsnUpdate() -{ - t_thrd.xlog_cxt.standbyState = g_redoWorker->standbyState; - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - if (LsnUpdate()) { - ExtremRtoUpdateMinCheckpoint(); - CheckRecoveryConsistency(); - } - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); -} - -bool LsnUpdate() -{ - XLogRecPtr minStart = MAX_XLOG_REC_PTR; - XLogRecPtr minEnd = MAX_XLOG_REC_PTR; - GetReplayedRecPtr(&minStart, &minEnd); - if ((minEnd != MAX_XLOG_REC_PTR) && (minStart != MAX_XLOG_REC_PTR)) { - SetXLogReplayRecPtr(minStart, minEnd); - return true; - } - return false; -} - -static void TrxnMangerQueueCallBack() -{ - GlobalLsnUpdate(); - HandlePageRedoInterrupts(); -} - -void TrxnManagerMain() -{ - (void)RegisterRedoInterruptCallBack(TrxnMangerQueueCallBack); - t_thrd.xlog_cxt.max_page_flush_lsn = get_global_max_page_flush_lsn(); - ereport(LOG, - (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("TrxnManagerMain: first get_global_max_page_flush_lsn %08X/%08X", - (uint32)(t_thrd.xlog_cxt.max_page_flush_lsn >> 32), (uint32)(t_thrd.xlog_cxt.max_page_flush_lsn)))); - while (true) { - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - if (FORCE_FINISH_ENABLED && t_thrd.xlog_cxt.max_page_flush_lsn == MAX_XLOG_REC_PTR) { - t_thrd.xlog_cxt.max_page_flush_lsn = get_global_max_page_flush_lsn(); - if (t_thrd.xlog_cxt.max_page_flush_lsn != MAX_XLOG_REC_PTR) { - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("TrxnManagerMain: second get_global_max_page_flush_lsn %08X/%08X", - (uint32)(t_thrd.xlog_cxt.max_page_flush_lsn >> 32), - (uint32)(t_thrd.xlog_cxt.max_page_flush_lsn)))); - } - } - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - if (!SPSCBlockingQueueIsEmpty(g_redoWorker->queue)) { - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); - RedoItem *item = (RedoItem *)SPSCBlockingQueueTop(g_redoWorker->queue); - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], - g_redoWorker->timeCostList[TIME_COST_STEP_2]); - bool isEnd = TrxnManagerDistributeItemsBeforeEnd(item); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); - SPSCBlockingQueuePop(g_redoWorker->queue); - - if (isEnd) { - break; - } - } else { - long sleeptime = 80 * 1000; - pg_usleep(sleeptime); - } - - ADD_ABNORMAL_POSITION(2); - RedoInterruptCallBack(); - } - - RedoThrdWaitForExit(g_redoWorker); - GlobalLsnUpdate(); -} - -void TrxnWorkerProcLsnForwarder(RedoItem *lsnForwarder) -{ - SetCompletedReadEndPtr(g_redoWorker, lsnForwarder->record.ReadRecPtr, lsnForwarder->record.EndRecPtr); - (void)pg_atomic_sub_fetch_u32(&lsnForwarder->record.refcount, 1); -} - -void TrxnWorkNotifyRedoWorker() -{ - for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; ++i) { - if (g_dispatcher->allWorkers[i]->role == REDO_PAGE_WORKER || - g_dispatcher->allWorkers[i]->role == REDO_PAGE_MNG) { - pg_atomic_write_u32(&(g_dispatcher->allWorkers[i]->fullSyncFlag), 0); - } - } -} - -void TrxnWorkrProcCleanupMark(RedoItem *cleanupMark) -{ - g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); - ereport(LOG, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]TrxnWorkrProcCleanupMark has cleaned InvalidPages"))); -} - -bool CheckFullSyncCheckpoint(RedoItem *item) -{ - if (!IsCheckPoint(&(item->record))) { - return true; - } - - if (XLByteLE(item->record.ReadRecPtr, t_thrd.shemem_ptr_cxt.ControlFile->checkPoint)) { - return true; - } - - return false; -} - -void TrxnWorkMain() -{ -#ifdef ENABLE_MOT - MOTBeginRedoRecovery(); -#endif - (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); - if (ParseStateWithoutCache()) { - XLogRedoBufferInitFunc(&(g_redoWorker->bufferManager), MAX_LOCAL_BUFF_NUM, &recordRefOperate, - RedoInterruptCallBack); - } - - RedoItem *item = NULL; - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); - while ((item = (RedoItem *)SPSCBlockingQueueTop(g_redoWorker->queue)) != &g_redoEndMark) { - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); - if ((void *)item == (void *)&g_GlobalLsnForwarder) { - TrxnWorkerProcLsnForwarder((RedoItem *)item); - SPSCBlockingQueuePop(g_redoWorker->queue); - } else if ((void *)item == (void *)&g_cleanupMark) { - TrxnWorkrProcCleanupMark((RedoItem *)item); - SPSCBlockingQueuePop(g_redoWorker->queue); - } else if ((void *)item == (void *)&g_closefdMark) { - smgrcloseall(); - SPSCBlockingQueuePop(g_redoWorker->queue); - } else if ((void *)item == (void *)&g_cleanInvalidPageMark) { - forget_range_invalid_pages((void *)item); - SPSCBlockingQueuePop(g_redoWorker->queue); - } else { - t_thrd.xlog_cxt.needImmediateCkp = item->needImmediateCheckpoint; - bool fullSync = item->record.isFullSync; - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - ApplySinglePageRecord(item); - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3], - g_redoWorker->timeCostList[TIME_COST_STEP_4]); - SPSCBlockingQueuePop(g_redoWorker->queue); - SetCompletedReadEndPtr(g_redoWorker, item->record.ReadRecPtr, item->record.EndRecPtr); - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4], - g_redoWorker->timeCostList[TIME_COST_STEP_5]); - if (fullSync) { - Assert(CheckFullSyncCheckpoint(item)); - TrxnWorkNotifyRedoWorker(); - } - - if (XactHasSegpageRelFiles(&item->record)) { - uint32 expected = 1; - pg_atomic_compare_exchange_u32((volatile uint32 *)&(g_dispatcher->segpageXactDoneFlag), &expected, 0); - } - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_5]); - DereferenceRedoItem(item); - RedoInterruptCallBack(); - } - ADD_ABNORMAL_POSITION(3); - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2], g_redoWorker->timeCostList[TIME_COST_STEP_1]); - } - - SPSCBlockingQueuePop(g_redoWorker->queue); - if (ParseStateWithoutCache()) - XLogRedoBufferDestoryFunc(&(g_redoWorker->bufferManager)); -#ifdef ENABLE_MOT - MOTEndRedoRecovery(); -#endif -} - -void RedoPageWorkerCheckPoint(const XLogRecParseState *redoblockstate) -{ - CheckPoint checkPoint; - Assert(IsCheckPoint(redoblockstate)); - XLogSynAllBuffer(); - Assert(redoblockstate->blockparse.extra_rec.blockxlogcommon.maindatalen >= sizeof(checkPoint)); - errno_t rc = memcpy_s(&checkPoint, sizeof(checkPoint), - redoblockstate->blockparse.extra_rec.blockxlogcommon.maindata, sizeof(checkPoint)); - securec_check(rc, "\0", "\0"); - if (IsRestartPointSafe(checkPoint.redo)) { - pg_atomic_write_u64(&g_redoWorker->lastCheckedRestartPoint, redoblockstate->blockparse.blockhead.start_ptr); - } - - UpdateTimeline(&checkPoint); - -#ifdef USE_ASSERT_CHECKING - int printLevel = WARNING; -#else - int printLevel = DEBUG1; -#endif - if (log_min_messages <= printLevel) { - GetThreadBufferLeakNum(); - } -} - -void PageWorkerProcLsnForwarder(RedoItem *lsnForwarder) -{ - SetCompletedReadEndPtr(g_redoWorker, lsnForwarder->record.ReadRecPtr, lsnForwarder->record.EndRecPtr); - (void)pg_atomic_sub_fetch_u32(&lsnForwarder->record.refcount, 1); -} - -bool XlogNeedUpdateFsm(XLogRecParseState *procState, RedoBufferInfo *bufferinfo) -{ - XLogBlockHead *blockhead = &procState->blockparse.blockhead; - if (bufferinfo->pageinfo.page == NULL || !(bufferinfo->dirtyflag) || blockhead->forknum != MAIN_FORKNUM || - XLogBlockHeadGetValidInfo(blockhead) != BLOCK_DATA_MAIN_DATA_TYPE || blockhead->bucketNode != InvalidBktId) { - return false; - } - - Size freespace = PageGetHeapFreeSpace(bufferinfo->pageinfo.page); - - RmgrId rmid = XLogBlockHeadGetRmid(blockhead); - if (rmid == RM_HEAP2_ID) { - uint8 info = XLogBlockHeadGetInfo(blockhead) & ~XLR_INFO_MASK; - if (info == XLOG_HEAP2_CLEAN) { - return true; - } else if ((info == XLOG_HEAP2_MULTI_INSERT) && (freespace < BLCKSZ / 5)) { - return true; - } - - } else if (rmid == RM_HEAP_ID) { - uint8 info = XLogBlockHeadGetInfo(blockhead) & ~XLR_INFO_MASK; - if ((info == XLOG_HEAP_INSERT || info == XLOG_HEAP_UPDATE) && (freespace < BLCKSZ / 5)) { - return true; - } - } - - return false; -} - -void RedoPageWorkerRedoBcmBlock(XLogRecParseState *procState) -{ - RmgrId rmid = XLogBlockHeadGetRmid(&procState->blockparse.blockhead); - if (rmid == RM_HEAP2_ID) { - RelFileNode node; - node.spcNode = procState->blockparse.blockhead.spcNode; - node.dbNode = procState->blockparse.blockhead.dbNode; - node.relNode = procState->blockparse.blockhead.relNode; - node.bucketNode = procState->blockparse.blockhead.bucketNode; - node.opt = procState->blockparse.blockhead.opt; - XLogBlockNewCuParse *newCuParse = &(procState->blockparse.extra_rec.blocknewcu); - uint8 info = XLogBlockHeadGetInfo(&procState->blockparse.blockhead) & ~XLR_INFO_MASK; - switch (info & XLOG_HEAP_OPMASK) { - case XLOG_HEAP2_BCM: { - xl_heap_bcm *xlrec = (xl_heap_bcm *)(newCuParse->main_data); - heap_bcm_redo(xlrec, node, procState->blockparse.blockhead.end_ptr); - break; - } - case XLOG_HEAP2_LOGICAL_NEWPAGE: { - Assert(IsHeapFileNode(node)); - xl_heap_logical_newpage *xlrec = (xl_heap_logical_newpage *)(newCuParse->main_data); - char *cuData = newCuParse->main_data + SizeOfHeapLogicalNewPage; - heap_xlog_bcm_new_page(xlrec, node, cuData); - break; - } - default: - break; - } - } -} - -void RedoPageWorkerMain() -{ - (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); - - if (ParseStateWithoutCache()) { - XLogRedoBufferInitFunc(&(g_redoWorker->bufferManager), MAX_LOCAL_BUFF_NUM, &recordRefOperate, - RedoInterruptCallBack); - } - - XLogRecParseState *redoblockstateHead = NULL; - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); - while ((redoblockstateHead = (XLogRecParseState *)SPSCBlockingQueueTop(g_redoWorker->queue)) != - (XLogRecParseState *)&g_redoEndMark) { - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); - if ((void *)redoblockstateHead == (void *)&g_cleanupMark) { - g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); - SPSCBlockingQueuePop(g_redoWorker->queue); - ereport(LOG, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]RedoPageWorkerMain has cleaned InvalidPages"))); - continue; - } - - if ((void *)redoblockstateHead == (void *)&g_closefdMark) { - smgrcloseall(); - SPSCBlockingQueuePop(g_redoWorker->queue); - continue; - } - - if ((void *)redoblockstateHead == (void *)&g_cleanInvalidPageMark) { - forget_range_invalid_pages((void *)redoblockstateHead); - SPSCBlockingQueuePop(g_redoWorker->queue); - continue; - } - if ((void *)redoblockstateHead == (void *)&g_GlobalLsnForwarder) { - PageWorkerProcLsnForwarder((RedoItem *)redoblockstateHead); - SPSCBlockingQueuePop(g_redoWorker->queue); - continue; - } - RedoBufferInfo bufferinfo = {0}; - bool notfound = false; - bool updateFsm = false; - - XLogRecParseState *procState = redoblockstateHead; - - MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); - while (procState != NULL) { - XLogRecParseState *redoblockstate = procState; - g_redoWorker->curRedoBlockState = (XLogBlockDataParse*)(&redoblockstate->blockparse.extra_rec); - procState = (XLogRecParseState *)procState->nextrecord; - - switch (XLogBlockHeadGetValidInfo(&redoblockstate->blockparse.blockhead)) { - case BLOCK_DATA_MAIN_DATA_TYPE: - case BLOCK_DATA_UNDO_TYPE: - case BLOCK_DATA_VM_TYPE: - case BLOCK_DATA_FSM_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - notfound = XLogBlockRedoForExtremeRTO(redoblockstate, &bufferinfo, notfound, - g_redoWorker->timeCostList[TIME_COST_STEP_4], g_redoWorker->timeCostList[TIME_COST_STEP_5]); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - break; - case BLOCK_DATA_XLOG_COMMON_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - RedoPageWorkerCheckPoint(redoblockstate); - SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, - redoblockstate->blockparse.blockhead.end_ptr); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - break; - case BLOCK_DATA_DDL_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - XLogForgetDDLRedo(redoblockstate); - SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, - redoblockstate->blockparse.blockhead.end_ptr); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - break; - case BLOCK_DATA_DROP_DATABASE_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - XLogDropDatabase(redoblockstate->blockparse.blockhead.dbNode); - SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, - redoblockstate->blockparse.blockhead.end_ptr); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - break; - case BLOCK_DATA_NEWCU_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - RedoPageWorkerRedoBcmBlock(redoblockstate); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - break; - case BLOCK_DATA_SEG_SPACE_DROP: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - XLogDropSegmentSpace(redoblockstate->blockparse.blockhead.spcNode, - redoblockstate->blockparse.blockhead.dbNode); - SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, - redoblockstate->blockparse.blockhead.end_ptr); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - break; - case BLOCK_DATA_SEG_SPACE_SHRINK: - XLogDropSpaceShrink(redoblockstate); - SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, - redoblockstate->blockparse.blockhead.end_ptr); - break; - case BLOCK_DATA_BARRIER_TYPE: - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, - redoblockstate->blockparse.blockhead.end_ptr); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); - break; - default: - break; - } - } - (void)MemoryContextSwitchTo(oldCtx); - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); - updateFsm = XlogNeedUpdateFsm(redoblockstateHead, &bufferinfo); - bool needWait = redoblockstateHead->isFullSync; - if (needWait) { - pg_atomic_write_u32(&g_redoWorker->fullSyncFlag, 1); - } - XLogBlockParseStateRelease(redoblockstateHead); - /* the same page */ - ExtremeRtoFlushBuffer(&bufferinfo, updateFsm); - SPSCBlockingQueuePop(g_redoWorker->queue); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); - pg_memory_barrier(); - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); - uint32 val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); - while (val != 0) { - RedoInterruptCallBack(); - val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); - } - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); - RedoInterruptCallBack(); - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2], g_redoWorker->timeCostList[TIME_COST_STEP_1]); - ADD_ABNORMAL_POSITION(4); - } - - SPSCBlockingQueuePop(g_redoWorker->queue); - if (ParseStateWithoutCache()) - XLogRedoBufferDestoryFunc(&(g_redoWorker->bufferManager)); -} - -void PutRecordToReadQueue(XLogReaderState *recordreader) -{ - SPSCBlockingQueuePut(g_dispatcher->readLine.readPageThd->queue, recordreader); -} - -inline void InitXLogRecordReadBuffer(XLogReaderState **initreader) -{ - XLogReaderState *newxlogreader; - XLogReaderState *readstate = g_dispatcher->rtoXlogBufState.initreader; - newxlogreader = NewReaderState(readstate); - g_dispatcher->rtoXlogBufState.initreader = NULL; - PutRecordToReadQueue(readstate); - SetCompletedReadEndPtr(g_redoWorker, readstate->ReadRecPtr, readstate->EndRecPtr); - *initreader = newxlogreader; -} - -void StartupSendFowarder(RedoItem *item) -{ - for (uint32 i = 0; i < g_dispatcher->pageLineNum; ++i) { - AddPageRedoItem(g_dispatcher->pageLines[i].batchThd, item); - } - - AddPageRedoItem(g_dispatcher->trxnLine.managerThd, item); -} - -void SendLsnFowarder() -{ - // update and read in the same thread, so no need atomic operation - g_GlobalLsnForwarder.record.ReadRecPtr = g_redoWorker->lastReplayedReadRecPtr; - g_GlobalLsnForwarder.record.EndRecPtr = g_redoWorker->lastReplayedEndRecPtr; - g_GlobalLsnForwarder.record.refcount = get_real_recovery_parallelism() - XLOG_READER_NUM; - g_GlobalLsnForwarder.record.isDecode = true; - PutRecordToReadQueue(&g_GlobalLsnForwarder.record); -} - -static inline bool ReadPageWorkerStop() -{ - return g_dispatcher->recoveryStop; -} - -void PushToWorkerLsn(bool force) -{ - const uint32 max_record_count = PAGE_WORK_QUEUE_SIZE; - static uint32 cur_recor_count = 0; - - cur_recor_count++; - - if (!IsExtremeRtoRunning()) { - return; - } - - if (force) { - uint32 refCount; - do { - refCount = pg_atomic_read_u32(&g_GlobalLsnForwarder.record.refcount); - RedoInterruptCallBack(); - } while (refCount != 0 && !ReadPageWorkerStop()); - cur_recor_count = 0; - SendLsnFowarder(); - } else { - uint32 refCount = pg_atomic_read_u32(&g_GlobalLsnForwarder.record.refcount); - - if (refCount != 0 || cur_recor_count < max_record_count) { - return; - } - - SendLsnFowarder(); - cur_recor_count = 0; - } -} - -void ResetRtoXlogReadBuf(XLogRecPtr targetPagePtr) -{ - uint32 startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); - if (startreadworker == WORKER_STATE_STOP) { - WalRcvCtlAcquireExitLock(); - WalRcvCtlBlock *walrcb = getCurrentWalRcvCtlBlock(); - - if (walrcb == NULL) { - WalRcvCtlReleaseExitLock(); - return; - } - - int64 walwriteoffset; - XLogRecPtr startptr; - SpinLockAcquire(&walrcb->mutex); - walwriteoffset = walrcb->walWriteOffset; - startptr = walrcb->walStart; - SpinLockRelease(&walrcb->mutex); - - if (XLByteLT(startptr, targetPagePtr)) { - WalRcvCtlReleaseExitLock(); - return; - } - - - for (uint32 i = 0; i < MAX_ALLOC_SEGNUM; ++i) { - pg_atomic_write_u32(&(g_recordbuffer->xlogsegarray[i].bufState), NONE); - } - - XLogSegNo segno; - XLByteToSeg(targetPagePtr, segno); - g_recordbuffer->xlogsegarray[g_recordbuffer->applyindex].segno = segno; - g_recordbuffer->xlogsegarray[g_recordbuffer->applyindex].readlen = targetPagePtr % XLOG_SEG_SIZE; - - pg_atomic_write_u32(&(g_recordbuffer->readindex), g_recordbuffer->applyindex); - pg_atomic_write_u32(&(g_recordbuffer->xlogsegarray[g_recordbuffer->readindex].bufState), APPLYING); - - pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_RUN); - WalRcvCtlReleaseExitLock(); - } -} - -RecordBufferAarray *GetCurrentSegmentBuf(XLogRecPtr targetPagePtr) -{ - Assert(g_recordbuffer->applyindex < MAX_ALLOC_SEGNUM); - uint32 applyindex = g_recordbuffer->applyindex; - RecordBufferAarray *cursegbuffer = &g_recordbuffer->xlogsegarray[applyindex]; - uint32 bufState = pg_atomic_read_u32(&(cursegbuffer->bufState)); - - if (bufState != APPLYING) { - return NULL; - } - uint32 targetPageOff = (targetPagePtr % XLOG_SEG_SIZE); - XLogSegNo targetSegNo; - XLByteToSeg(targetPagePtr, targetSegNo); - if (cursegbuffer->segno == targetSegNo) { - cursegbuffer->segoffset = targetPageOff; - return cursegbuffer; - } else if (cursegbuffer->segno + 1 == targetSegNo) { - Assert(targetPageOff == 0); - pg_atomic_write_u32(&(cursegbuffer->bufState), APPLIED); - if ((applyindex + 1) == MAX_ALLOC_SEGNUM) { - applyindex = 0; - } else { - applyindex++; - } - - pg_atomic_write_u32(&(g_recordbuffer->applyindex), applyindex); - cursegbuffer = &g_recordbuffer->xlogsegarray[applyindex]; - bufState = pg_atomic_read_u32(&(cursegbuffer->bufState)); - if (bufState != APPLYING) { - return NULL; - } - - Assert(cursegbuffer->segno == targetSegNo); - cursegbuffer->segoffset = targetPageOff; - return cursegbuffer; - } else { - ereport(WARNING, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("SetReadBufferForExtRto targetPagePtr:%lu", targetPagePtr))); - DumpExtremeRtoReadBuf(); - t_thrd.xlog_cxt.failedSources |= XLOG_FROM_STREAM; - return NULL; - } -} - -static const int MAX_WAIT_TIMS = 512; - -bool XLogPageReadForExtRto(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen) -{ - uint32 startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); - if (startreadworker == WORKER_STATE_RUN) { - RecordBufferAarray *cursegbuffer = GetCurrentSegmentBuf(targetPagePtr); - if (cursegbuffer == NULL) { - return false; - } - - uint32 readlen = pg_atomic_read_u32(&(cursegbuffer->readlen)); - - uint32 waitcount = 0; - while (readlen < (cursegbuffer->segoffset + reqLen)) { - readlen = pg_atomic_read_u32(&(cursegbuffer->readlen)); - if (waitcount >= MAX_WAIT_TIMS) { - return false; - } - waitcount++; - } - - Assert(cursegbuffer->segoffset == (targetPagePtr % XLogSegSize)); - xlogreader->readBuf = cursegbuffer->readsegbuf + cursegbuffer->segoffset; - return true; - } - - return false; -} - -void XLogReadWorkerSegFallback(XLogSegNo lastRplSegNo) -{ - errno_t errorno = EOK; - uint32 readindex = pg_atomic_read_u32(&(g_recordbuffer->readindex)); - uint32 applyindex = pg_atomic_read_u32(&(g_recordbuffer->applyindex)); - RecordBufferAarray *readseg = &g_recordbuffer->xlogsegarray[readindex]; - RecordBufferAarray *applyseg = &g_recordbuffer->xlogsegarray[applyindex]; - - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("XLogReadWorkerSegFallback: readindex: %u, readseg[%lu,%lu,%u,%u], applyindex: %u," - "applyseg[%lu,%lu,%u,%u]", - readindex, readseg->segno, readseg->segoffset, readseg->readlen, readseg->bufState, applyindex, - applyseg->segno, applyseg->segoffset, applyseg->readlen, applyseg->bufState))); - - pg_atomic_write_u32(&(g_recordbuffer->readindex), applyindex); - pg_atomic_write_u32(&(readseg->bufState), APPLIED); - applyseg->segno = lastRplSegNo; - applyseg->readlen = applyseg->segoffset; - errorno = memset_s(applyseg->readsegbuf, XLOG_SEG_SIZE, 0, XLOG_SEG_SIZE); - securec_check(errorno, "", ""); -} - -bool CloseReadFile() -{ - if (t_thrd.xlog_cxt.readFile >= 0) { - close(t_thrd.xlog_cxt.readFile); - t_thrd.xlog_cxt.readFile = -1; - return true; - } - return false; -} - -void DispatchCleanupMarkToAllRedoWorker() -{ - for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { - PageRedoWorker *worker = g_dispatcher->allWorkers[i]; - if (worker->role == REDO_PAGE_WORKER) { - SPSCBlockingQueuePut(worker->queue, &g_cleanupMark); - } - } -} - -void DispatchClosefdMarkToAllRedoWorker() -{ - for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { - PageRedoWorker *worker = g_dispatcher->allWorkers[i]; - if (worker->role == REDO_PAGE_WORKER || worker->role == REDO_PAGE_MNG || - worker->role == REDO_TRXN_MNG || worker->role == REDO_TRXN_WORKER) { - SPSCBlockingQueuePut(worker->queue, &g_closefdMark); - } - } -} - -void DispatchCleanInvalidPageMarkToAllRedoWorker(RepairFileKey key) -{ - for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { - PageRedoWorker *worker = g_dispatcher->allWorkers[i]; - if (worker->role == REDO_PAGE_WORKER) { - errno_t rc = memcpy_s((char*)&g_cleanInvalidPageMark, - sizeof(RepairFileKey), (char*)&key, sizeof(RepairFileKey)); - securec_check(rc, "", ""); - SPSCBlockingQueuePut(worker->queue, &g_cleanInvalidPageMark); - } - } -} - -void WaitAllRedoWorkerIdle() -{ - instr_time startTime; - instr_time endTime; - bool allIdle = false; - INSTR_TIME_SET_CURRENT(startTime); - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("WaitAllRedoWorkerIdle begin, startTime: %lu us", INSTR_TIME_GET_MICROSEC(startTime)))); - while (!allIdle) { - allIdle = true; - for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { - PageRedoWorker *worker = g_dispatcher->allWorkers[i]; - if (worker->role == REDO_READ_WORKER || worker->role == REDO_READ_MNG) { - continue; - } - if (!RedoWorkerIsIdle(worker)) { - allIdle = false; - break; - } - } - RedoInterruptCallBack(); - } - INSTR_TIME_SET_CURRENT(endTime); - INSTR_TIME_SUBTRACT(endTime, startTime); - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("WaitAllRedoWorkerIdle end, cost time: %lu us", INSTR_TIME_GET_MICROSEC(endTime)))); -} - -void WaitAllReplayWorkerIdle() -{ - instr_time startTime; - instr_time endTime; - bool allIdle = false; - INSTR_TIME_SET_CURRENT(startTime); - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("WaitAllReplayWorkerIdle begin, startTime: %lu us", INSTR_TIME_GET_MICROSEC(startTime)))); - while (!allIdle) { - allIdle = true; - for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { - PageRedoWorker *worker = g_dispatcher->allWorkers[i]; - if (worker->role == REDO_READ_WORKER || worker->role == REDO_READ_MNG || - worker->role == REDO_READ_PAGE_WORKER) { - continue; - } - if (!RedoWorkerIsIdle(worker)) { - allIdle = false; - break; - } - } - RedoInterruptCallBack(); - } - INSTR_TIME_SET_CURRENT(endTime); - INSTR_TIME_SUBTRACT(endTime, startTime); - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("WaitAllReplayWorkerIdle end, cost time: %lu us", INSTR_TIME_GET_MICROSEC(endTime)))); -} - - -void XLogForceFinish(XLogReaderState *xlogreader, TermFileData *term_file) -{ - bool closed = false; - uint32 termId = term_file->term; - XLogSegNo lastRplSegNo; - - pg_atomic_write_u32(&(extreme_rto::g_recordbuffer->readWorkerState), extreme_rto::WORKER_STATE_STOPPING); - while (pg_atomic_read_u32(&(extreme_rto::g_recordbuffer->readWorkerState)) != WORKER_STATE_STOP) { - RedoInterruptCallBack(); - }; - ShutdownWalRcv(); - ShutdownDataRcv(); - pg_atomic_write_u32(&(g_recordbuffer->readSource), XLOG_FROM_PG_XLOG); - - PushToWorkerLsn(true); - g_cleanupMark.record.isDecode = true; - PutRecordToReadQueue(&g_cleanupMark.record); - WaitAllRedoWorkerIdle(); - - XLogRecPtr lastRplReadLsn; - XLogRecPtr lastRplEndLsn = GetXLogReplayRecPtr(NULL, &lastRplReadLsn); - XLogRecPtr receivedUpto = GetWalRcvWriteRecPtr(NULL); - XLogRecPtr endRecPtr = xlogreader->EndRecPtr; - ereport(WARNING, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]ArchiveXlogForForceFinishRedo in extremeRTO " - "lastRplReadLsn:%08X/%08X, lastRplEndLsn:%08X/%08X, receivedUpto:%08X/%08X, ReadRecPtr:%08X/%08X, " - "EndRecPtr:%08X/%08X, readOff:%u, latestValidRecord:%08X/%08X", - (uint32)(lastRplReadLsn >> 32), (uint32)lastRplReadLsn,(uint32)(lastRplEndLsn >> 32), (uint32)lastRplEndLsn, - (uint32)(receivedUpto >> 32), (uint32)receivedUpto,(uint32)(xlogreader->ReadRecPtr >> 32), - (uint32)xlogreader->ReadRecPtr, (uint32)(xlogreader->EndRecPtr >> 32), (uint32)xlogreader->EndRecPtr, - xlogreader->readOff, (uint32)(latestValidRecord >> 32), (uint32)latestValidRecord))); - DumpExtremeRtoReadBuf(); - xlogreader->readOff = INVALID_READ_OFF; - XLByteToSeg(endRecPtr, lastRplSegNo); - XLogReadWorkerSegFallback(lastRplSegNo); - - closed = CloseReadFile(); - CopyXlogForForceFinishRedo(lastRplSegNo, termId, xlogreader, endRecPtr); - RenameXlogForForceFinishRedo(lastRplSegNo, xlogreader->readPageTLI, termId); - if (closed) { - ReOpenXlog(xlogreader); - } - t_thrd.xlog_cxt.invaildPageCnt = 0; - XLogCheckInvalidPages(); - SetSwitchHistoryFile(endRecPtr, receivedUpto, termId); - t_thrd.xlog_cxt.invaildPageCnt = 0; - set_wal_rcv_write_rec_ptr(endRecPtr); - t_thrd.xlog_cxt.receivedUpto = endRecPtr; - pg_atomic_write_u32(&(g_instance.comm_cxt.localinfo_cxt.is_finish_redo), 0); - ereport(WARNING, - (errcode(ERRCODE_LOG), errmsg("[ForceFinish]ArchiveXlogForForceFinishRedo in extremeRTO is over"))); -} - -void CleanUpReadPageWorkerQueue() -{ - SPSCBlockingQueue *queue = g_dispatcher->readLine.readPageThd->queue; - uint32 state; - do { - while (!SPSCBlockingQueueIsEmpty(queue)) { - XLogReaderState *xlogreader = reinterpret_cast(SPSCBlockingQueueTake(queue)); - if (xlogreader == reinterpret_cast(&(g_redoEndMark.record)) || - xlogreader == reinterpret_cast(&(g_GlobalLsnForwarder.record)) || - xlogreader == reinterpret_cast(&(g_cleanupMark.record))) { - if (xlogreader == reinterpret_cast(&(g_GlobalLsnForwarder.record))) { - pg_atomic_write_u32(&g_GlobalLsnForwarder.record.refcount, 0); - } - continue; - } - - RedoItem *item = GetRedoItemPtr(xlogreader); - FreeRedoItem(item); - } - - RedoInterruptCallBack(); - state = pg_atomic_read_u32(&extreme_rto::g_dispatcher->rtoXlogBufState.readPageWorkerState); - } while (state != WORKER_STATE_EXIT); -} - -void ExtremeRtoStopHere() -{ - if ((get_real_recovery_parallelism() > 1) && (GetBatchCount() > 0)) { - g_dispatcher->recoveryStop = true; - CleanUpReadPageWorkerQueue(); - } -} - -static void CheckAndDoForceFinish(XLogReaderState *xlogreader) -{ - TermFileData term_file; - if (CheckForForceFinishRedoTrigger(&term_file)) { - ereport(WARNING, - (errmsg("[ForceFinish] force finish triggered in XLogReadPageWorkerMain, ReadRecPtr:%08X/%08X, " - "EndRecPtr:%08X/%08X, StandbyMode:%u, startup_processing:%u, dummyStandbyMode:%u", - (uint32)(t_thrd.xlog_cxt.ReadRecPtr >> 32), (uint32)t_thrd.xlog_cxt.ReadRecPtr, - (uint32)(t_thrd.xlog_cxt.EndRecPtr >> 32), (uint32)t_thrd.xlog_cxt.EndRecPtr, - t_thrd.xlog_cxt.StandbyMode, t_thrd.xlog_cxt.startup_processing, dummyStandbyMode))); - XLogForceFinish(xlogreader, &term_file); - } -} - -/* read xlog for parellel */ -void XLogReadPageWorkerMain() -{ - XLogReaderState *xlogreader = NULL; - - (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); - - g_recordbuffer = &g_dispatcher->rtoXlogBufState; - GetRecoveryLatch(); - /* init readstate */ - InitXLogRecordReadBuffer(&xlogreader); - - pg_atomic_write_u32(&(g_recordbuffer->readPageWorkerState), WORKER_STATE_RUN); - if (IsRecoveryDone()) { - t_thrd.xlog_cxt.readSource = XLOG_FROM_STREAM; - t_thrd.xlog_cxt.XLogReceiptSource = XLOG_FROM_STREAM; - pg_atomic_write_u32(&(g_recordbuffer->readSource), XLOG_FROM_STREAM); - } - - XLogRecord *record = XLogParallelReadNextRecord(xlogreader); - while (record != NULL) { - if (ReadPageWorkerStop()) { - break; - } - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - XLogReaderState *newxlogreader = NewReaderState(xlogreader); - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3], g_redoWorker->timeCostList[TIME_COST_STEP_4]); - PutRecordToReadQueue(xlogreader); - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4], g_redoWorker->timeCostList[TIME_COST_STEP_5]); - xlogreader = newxlogreader; - - g_redoWorker->lastReplayedReadRecPtr = xlogreader->ReadRecPtr; - g_redoWorker->lastReplayedEndRecPtr = xlogreader->EndRecPtr; - - if (FORCE_FINISH_ENABLED) { - CheckAndDoForceFinish(xlogreader); - } - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_5], g_redoWorker->timeCostList[TIME_COST_STEP_1]); - record = XLogParallelReadNextRecord(xlogreader); - CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); - PushToWorkerLsn(false); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); - RedoInterruptCallBack(); - ADD_ABNORMAL_POSITION(8); - } - - uint32 workState = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); - while (workState == WORKER_STATE_STOPPING) { - workState = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); - } - - if (workState != WORKER_STATE_EXITING && workState != WORKER_STATE_EXIT) { - pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_EXITING); - } - - if (!ReadPageWorkerStop()) { - /* notify exit */ - PushToWorkerLsn(true); - g_redoEndMark.record = *xlogreader; - g_redoEndMark.record.isDecode = true; - PutRecordToReadQueue((XLogReaderState *)&g_redoEndMark.record); - } - - ReLeaseRecoveryLatch(); - pg_atomic_write_u32(&(g_recordbuffer->readPageWorkerState), WORKER_STATE_EXIT); -} - -void HandleReadWorkerRunInterrupts() -{ - if (t_thrd.page_redo_cxt.got_SIGHUP) { - t_thrd.page_redo_cxt.got_SIGHUP = false; - ProcessConfigFile(PGC_SIGHUP); - } - - if (t_thrd.page_redo_cxt.shutdown_requested) { - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("page worker id %u exit for request", g_redoWorker->id))); - - pg_atomic_write_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[g_redoWorker->id].threadState), - PAGE_REDO_WORKER_EXIT); - - proc_exit(1); - } -} - -static void InitReadBuf(uint32 bufIndex, XLogSegNo segno) -{ - if (bufIndex == MAX_ALLOC_SEGNUM) { - bufIndex = 0; - } - const uint32 sleepTime = 50; /* 50 us */ - RecordBufferAarray *nextreadseg = &g_recordbuffer->xlogsegarray[bufIndex]; - pg_memory_barrier(); - - uint32 bufState = pg_atomic_read_u32(&(nextreadseg->bufState)); - uint32 startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); - while (bufState == APPLYING && startreadworker == WORKER_STATE_RUN) { - pg_usleep(sleepTime); - RedoInterruptCallBack(); - bufState = pg_atomic_read_u32(&(nextreadseg->bufState)); - startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); - } - - nextreadseg->readlen = 0; - nextreadseg->segno = segno; - nextreadseg->segoffset = 0; - pg_atomic_write_u32(&(nextreadseg->bufState), APPLYING); - pg_atomic_write_u32(&(g_recordbuffer->readindex), bufIndex); -} - -static void XLogReadWorkRun() -{ - static uint32 waitcount = 0; - const uint32 sleepTime = 100; /* 50 us */ - XLogSegNo targetSegNo; - uint32 writeoffset; - uint32 reqlen; - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); - uint32 readindex = pg_atomic_read_u32(&(g_recordbuffer->readindex)); - Assert(readindex < MAX_ALLOC_SEGNUM); - pg_memory_barrier(); - RecordBufferAarray *readseg = &g_recordbuffer->xlogsegarray[readindex]; - - XLogRecPtr receivedUpto = GetWalRcvWriteRecPtr(NULL); - XLByteToSeg(receivedUpto, targetSegNo); - - if (targetSegNo < readseg->segno) { - pg_usleep(sleepTime); - return; - } - - writeoffset = readseg->readlen; - if (targetSegNo != readseg->segno) { - reqlen = XLOG_SEG_SIZE - writeoffset; - } else { - uint32 targetPageOff = receivedUpto % XLOG_SEG_SIZE; - if (targetPageOff <= writeoffset) { - pg_usleep(sleepTime); - return; - } - reqlen = targetPageOff - writeoffset; - if (reqlen < XLOG_BLCKSZ) { - waitcount++; - uint32 flag = pg_atomic_read_u32(&g_readManagerTriggerFlag); - if (waitcount < MAX_WAIT_TIMS && flag == TRIGGER_NORMAL) { - pg_usleep(sleepTime); - return; - } - } - } - - waitcount = 0; - char *readBuf = readseg->readsegbuf + writeoffset; - XLogRecPtr targetSartPtr = readseg->segno * XLOG_SEG_SIZE + writeoffset; - uint32 readlen = 0; - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); - bool result = XLogReadFromWriteBuffer(targetSartPtr, reqlen, readBuf, &readlen); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); - if (!result) { - return; - } - - pg_atomic_write_u32(&(readseg->readlen), (writeoffset + readlen)); - if (readseg->readlen == XLOG_SEG_SIZE) { - GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - InitReadBuf(readindex + 1, readseg->segno + 1); - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); - } - - CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); -} - -void XLogReadManagerResponseSignal(uint32 tgigger) -{ - switch (tgigger) { - case TRIGGER_PRIMARY: - break; - case TRIGGER_FAILOVER: - if (t_thrd.xlog_cxt.is_cascade_standby) { - SendPostmasterSignal(PMSIGNAL_UPDATE_PROMOTING); - t_thrd.xlog_cxt.is_cascade_standby = false; - if (t_thrd.postmaster_cxt.HaShmData->is_cross_region) { - t_thrd.xlog_cxt.is_hadr_main_standby = true; - SpinLockAcquire(&t_thrd.postmaster_cxt.HaShmData->mutex); - t_thrd.postmaster_cxt.HaShmData->is_hadr_main_standby = true; - SpinLockRelease(&t_thrd.postmaster_cxt.HaShmData->mutex); - } - t_thrd.xlog_cxt.failover_triggered = false; - SendNotifySignal(NOTIFY_STANDBY, g_instance.pid_cxt.StartupPID); - SendPostmasterSignal(PMSIGNAL_UPDATE_NORMAL); - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("failover standby ready, notify postmaster to change state."))); - break; - } - t_thrd.xlog_cxt.failover_triggered = true; - SendPostmasterSignal(PMSIGNAL_UPDATE_PROMOTING); - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("failover ready, notify postmaster to change state."))); - break; - case TRIGGER_SWITCHOVER: - t_thrd.xlog_cxt.switchover_triggered = true; - SendPostmasterSignal(PMSIGNAL_UPDATE_PROMOTING); - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("switchover ready, notify postmaster to change state."))); - break; - default: - break; - } -} - -void XLogReadManagerProcInterrupt() -{ - if (t_thrd.page_redo_cxt.shutdown_requested) { - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("page worker id %u exit for request", g_redoWorker->id))); - - pg_atomic_write_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[g_redoWorker->id].threadState), - PAGE_REDO_WORKER_EXIT); - - proc_exit(1); - } - - if (t_thrd.page_redo_cxt.got_SIGHUP) { - t_thrd.page_redo_cxt.got_SIGHUP = false; - ProcessConfigFile(PGC_SIGHUP); - } -} - -void WaitPageReadWorkerExit() -{ - uint32 state; - do { - state = pg_atomic_read_u32(&extreme_rto::g_dispatcher->rtoXlogBufState.readPageWorkerState); - RedoInterruptCallBack(); - } while (state != WORKER_STATE_EXIT); -} - -static void HandleExtremeRtoCascadeStandbyPromote(uint32 trigger) -{ - if (!t_thrd.xlog_cxt.is_cascade_standby || t_thrd.xlog_cxt.server_mode != STANDBY_MODE || - !IS_DN_MULTI_STANDYS_MODE()) { - return; - } - - ShutdownWalRcv(); - pg_atomic_write_u32(&g_dispatcher->rtoXlogBufState.waitRedoDone, 1); - WakeupRecovery(); - XLogReadManagerResponseSignal(trigger); - pg_atomic_write_u32(&(extreme_rto::g_startupTriggerState), TRIGGER_NORMAL); -} - -bool XLogReadManagerCheckSignal() -{ - uint32 trigger = pg_atomic_read_u32(&(extreme_rto::g_startupTriggerState)); - load_server_mode(); - if (g_dispatcher->smartShutdown || trigger == TRIGGER_PRIMARY || trigger == TRIGGER_SWITCHOVER || - (trigger == TRIGGER_FAILOVER && t_thrd.xlog_cxt.server_mode == STANDBY_MODE) || - t_thrd.xlog_cxt.server_mode == PRIMARY_MODE) { - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("XLogReadManagerCheckSignal: smartShutdown:%u, trigger:%u, server_mode:%u", - g_dispatcher->smartShutdown, trigger, t_thrd.xlog_cxt.server_mode))); - if (t_thrd.xlog_cxt.is_cascade_standby && t_thrd.xlog_cxt.server_mode == STANDBY_MODE && - IS_DN_MULTI_STANDYS_MODE() && (trigger == TRIGGER_SWITCHOVER || trigger == TRIGGER_FAILOVER)) { - HandleExtremeRtoCascadeStandbyPromote(trigger); - return false; - } - ShutdownWalRcv(); - if (g_dispatcher->smartShutdown) { - pg_atomic_write_u32(&g_readManagerTriggerFlag, TRIGGER_SMARTSHUTDOWN); - } else { - pg_atomic_write_u32(&g_readManagerTriggerFlag, trigger); - } - WakeupRecovery(); - WaitPageReadWorkerExit(); - XLogReadManagerResponseSignal(trigger); - return true; - } - return false; -} - -void StartRequestXLogFromStream() -{ - volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; - XLogRecPtr expectLsn = pg_atomic_read_u64(&g_dispatcher->rtoXlogBufState.expectLsn); - if (walrcv->receivedUpto == InvalidXLogRecPtr || - (expectLsn != InvalidXLogRecPtr && XLByteLE(walrcv->receivedUpto, expectLsn))) { - uint32 readWorkerstate = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); - if (readWorkerstate == WORKER_STATE_RUN) { - pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_STOPPING); - } - SpinLockAcquire(&walrcv->mutex); - walrcv->receivedUpto = 0; - SpinLockRelease(&walrcv->mutex); - XLogRecPtr targetRecPtr = pg_atomic_read_u64(&g_dispatcher->rtoXlogBufState.targetRecPtr); - CheckMaxPageFlushLSN(targetRecPtr); - - uint32 shiftSize = 32; - if (IS_OBS_DISASTER_RECOVER_MODE && !IsRoachRestore()) { - ereport(LOG, (errmsg("request xlog stream from obs at %X/%X.", (uint32)(targetRecPtr >> shiftSize), - (uint32)targetRecPtr))); - RequestXLogStreaming(&targetRecPtr, 0, REPCONNTARGET_OBS, 0); - } else if (IS_SHARED_STORAGE_STANBY_MODE && !IS_SHARED_STORAGE_MAIN_STANDBY_MODE) { -#ifndef ENABLE_MULTIPLE_NODES - rename_recovery_conf_for_roach(); -#endif - ereport(LOG, (errmsg("request xlog stream from shared storage at %X/%X.", - (uint32)(targetRecPtr >> shiftSize), - (uint32)targetRecPtr))); - RequestXLogStreaming(&targetRecPtr, 0, REPCONNTARGET_SHARED_STORAGE, 0); - } else { -#ifndef ENABLE_MULTIPLE_NODES - rename_recovery_conf_for_roach(); -#endif - ereport(LOG, (errmsg("request xlog stream at %X/%X.", (uint32)(targetRecPtr >> shiftSize), - (uint32)targetRecPtr))); - RequestXLogStreaming(&targetRecPtr, t_thrd.xlog_cxt.PrimaryConnInfo, REPCONNTARGET_PRIMARY, - u_sess->attr.attr_storage.PrimarySlotName); - } - } -} - - -void XLogReadManagerMain() -{ - const long sleepShortTime = 100000L; - const long sleepLongTime = 1000000L; - g_recordbuffer = &g_dispatcher->rtoXlogBufState; - uint32 xlogReadManagerState = READ_MANAGER_RUN; - - (void)RegisterRedoInterruptCallBack(XLogReadManagerProcInterrupt); - - while (xlogReadManagerState == READ_MANAGER_RUN) { - RedoInterruptCallBack(); - XLogRecPtr replay = InvalidXLogRecPtr; - bool exitStatus = XLogReadManagerCheckSignal(); - if (exitStatus) { - break; - } - - replay = GetXLogReplayRecPtr(NULL, NULL); - handleRecoverySusPend(replay); - - xlogReadManagerState = pg_atomic_read_u32(&g_dispatcher->rtoXlogBufState.xlogReadManagerState); - ADD_ABNORMAL_POSITION(7); - if (t_thrd.xlog_cxt.server_mode == STANDBY_MODE) { - uint32 readSource = pg_atomic_read_u32(&g_dispatcher->rtoXlogBufState.readSource); - uint32 failSource = pg_atomic_read_u32(&g_dispatcher->rtoXlogBufState.failSource); - if (readSource & XLOG_FROM_STREAM) { - uint32 disableConnectionNode = - pg_atomic_read_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node); - bool retryConnect = ((!disableConnectionNode) || (IS_SHARED_STORAGE_MODE && disableConnectionNode && - !knl_g_get_redo_finish_status() && - !pg_atomic_read_u32(&t_thrd.walreceiverfuncs_cxt.WalRcv->rcvDoneFromShareStorage))); - if (!WalRcvInProgress() && g_instance.pid_cxt.WalReceiverPID == 0 && retryConnect) { - StartRequestXLogFromStream(); - } else { - if (disableConnectionNode) { - if (IS_SHARED_STORAGE_MODE && WalRcvIsRunning()) { - ShutdownWalRcv(); - } - - if (!WalRcvInProgress() && !knl_g_get_redo_finish_status()) { - pg_atomic_write_u32(&g_dispatcher->rtoXlogBufState.waitRedoDone, 1); - WakeupRecovery(); - pg_usleep(sleepLongTime); - } else if (knl_g_get_redo_finish_status()) { - pg_atomic_write_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node, false); - pg_usleep(sleepLongTime); - } - - } - } - } - - if (failSource & XLOG_FROM_STREAM) { - ShutdownWalRcv(); - pg_atomic_write_u32(&(extreme_rto::g_dispatcher->rtoXlogBufState.failSource), 0); - } - } - pg_usleep(sleepShortTime); - } -} - -static void ReadWorkerStopCallBack(int code, Datum arg) -{ - pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_EXIT); - if (t_thrd.xlog_cxt.readFile >= 0) { - close(t_thrd.xlog_cxt.readFile); - t_thrd.xlog_cxt.readFile = -1; - } -} - -void XLogReadWorkerMain() -{ - uint32 startreadworker; - const uint32 sleepTime = 50; /* 50 us */ - - on_shmem_exit(ReadWorkerStopCallBack, 0); - (void)RegisterRedoInterruptCallBack(HandleReadWorkerRunInterrupts); - - g_recordbuffer = &g_dispatcher->rtoXlogBufState; - startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); - while (startreadworker != WORKER_STATE_EXITING) { - if (startreadworker == WORKER_STATE_RUN) { - XLogReadWorkRun(); - } else { - pg_usleep(sleepTime); - } - - RedoInterruptCallBack(); - startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); - if (startreadworker == WORKER_STATE_STOPPING) { - pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_STOP); - } - ADD_ABNORMAL_POSITION(6); - }; - /* notify manger to exit */ - pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_EXIT); -} - -int RedoMainLoop() -{ - g_redoWorker->oldCtx = MemoryContextSwitchTo(g_instance.comm_cxt.predo_cxt.parallelRedoCtx); - - instr_time startTime; - instr_time endTime; - - (void)RegisterRedoPageRepairCallBack(HandlePageRedoPageRepair); - - INSTR_TIME_SET_CURRENT(startTime); - switch (g_redoWorker->role) { - case REDO_BATCH: - BatchRedoMain(); - break; - case REDO_PAGE_MNG: - RedoPageManagerMain(); - break; - case REDO_PAGE_WORKER: - RedoPageWorkerMain(); - break; - case REDO_TRXN_MNG: - TrxnManagerMain(); - break; - case REDO_TRXN_WORKER: - TrxnWorkMain(); - break; - case REDO_READ_WORKER: - XLogReadWorkerMain(); - break; - case REDO_READ_PAGE_WORKER: - XLogReadPageWorkerMain(); - break; - case REDO_READ_MNG: - XLogReadManagerMain(); - break; - default: - break; - } - - INSTR_TIME_SET_CURRENT(endTime); - INSTR_TIME_SUBTRACT(endTime, startTime); - - /* - * We need to get the exit code here before we allow the dispatcher - * to proceed and change the exit code. - */ - int exitCode = GetDispatcherExitCode(); - g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); - g_redoWorker->committingCsnList = XLogReleaseAndGetCommittingCsnList(); - - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("worker[%d]: exitcode = %d, total elapsed = %ld", g_redoWorker->id, exitCode, - INSTR_TIME_GET_MICROSEC(endTime)))); - - (void)MemoryContextSwitchTo(g_redoWorker->oldCtx); - - return exitCode; -} - -void ParallelRedoThreadRegister() -{ - bool isWorkerStarting = false; - SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.rwlock)); - isWorkerStarting = ((g_instance.comm_cxt.predo_cxt.state == REDO_STARTING_BEGIN) ? true : false); - if (isWorkerStarting) { - pg_atomic_write_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[g_redoWorker->id].threadState), - PAGE_REDO_WORKER_READY); - } - SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); - if (!isWorkerStarting) { - ereport(LOG, (errmsg("ParallelRedoThreadRegister Page-redo-worker %u exit.", (uint32)isWorkerStarting))); - SetPageWorkStateByThreadId(PAGE_REDO_WORKER_EXIT); - proc_exit(0); - } -} - -const char *RedoWokerRole2Str(RedoRole role) -{ - switch (role) { - case REDO_BATCH: - return "redo_batch"; - break; - case REDO_PAGE_MNG: - return "redo_manager"; - break; - case REDO_PAGE_WORKER: - return "redo_woker"; - break; - case REDO_TRXN_MNG: - return "trxn_manager"; - break; - case REDO_TRXN_WORKER: - return "trxn_worker"; - break; - case REDO_READ_WORKER: - return "read_worker"; - break; - case REDO_READ_PAGE_WORKER: - return "read_page_woker"; - break; - case REDO_READ_MNG: - return "read_manager"; - break; - default: - return "unkown"; - break; - } -} - -void WaitStateNormal() -{ - do { - RedoInterruptCallBack(); - } while (g_instance.comm_cxt.predo_cxt.state < REDO_IN_PROGRESS); -} - -/* Run from the worker thread. */ -void ParallelRedoThreadMain() -{ - ParallelRedoThreadRegister(); - ereport(LOG, (errmsg("Page-redo-worker thread %u started, role:%u, slotId:%u.", g_redoWorker->id, - g_redoWorker->role, g_redoWorker->slotId))); - // regitster default interrupt call back - (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); - SetupSignalHandlers(); - InitGlobals(); - - ResourceManagerStartup(); - WaitStateNormal(); - EnableSyncRequestForwarding(); - - int retCode = RedoMainLoop(); - ResourceManagerStop(); - ereport(LOG, (errmsg("Page-redo-worker thread %u terminated, role:%u, slotId:%u, retcode %u.", g_redoWorker->id, - g_redoWorker->role, g_redoWorker->slotId, retCode))); - LastMarkReached(); - - pg_atomic_write_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[g_redoWorker->id].threadState), - PAGE_REDO_WORKER_EXIT); - proc_exit(0); -} - -static void PageRedoShutdownHandler(SIGNAL_ARGS) -{ - t_thrd.page_redo_cxt.shutdown_requested = 1; -} - -static void PageRedoQuickDie(SIGNAL_ARGS) -{ - int status = 2; - gs_signal_setmask(&t_thrd.libpq_cxt.BlockSig, NULL); - on_exit_reset(); - exit(status); -} - -static void PageRedoUser1Handler(SIGNAL_ARGS) -{ - t_thrd.page_redo_cxt.check_repair = true; -} - -static void PageRedoUser2Handler(SIGNAL_ARGS) -{ - t_thrd.page_redo_cxt.sleep_long = 1; -} - -/* Run from the worker thread. */ -static void SetupSignalHandlers() -{ - (void)gspqsignal(SIGHUP, SigHupHandler); - (void)gspqsignal(SIGINT, SIG_IGN); - (void)gspqsignal(SIGTERM, PageRedoShutdownHandler); - (void)gspqsignal(SIGQUIT, PageRedoQuickDie); - (void)gspqsignal(SIGALRM, SIG_IGN); - (void)gspqsignal(SIGPIPE, SIG_IGN); - (void)gspqsignal(SIGUSR1, PageRedoUser1Handler); - (void)gspqsignal(SIGUSR2, PageRedoUser2Handler); - (void)gspqsignal(SIGCHLD, SIG_IGN); - (void)gspqsignal(SIGTTIN, SIG_IGN); - (void)gspqsignal(SIGTTOU, SIG_IGN); - (void)gspqsignal(SIGCONT, SIG_IGN); - (void)gspqsignal(SIGWINCH, SIG_IGN); - - gs_signal_setmask(&t_thrd.libpq_cxt.UnBlockSig, NULL); - (void)gs_signal_unblock_sigusr2(); -} - -/* Run from the worker thread. */ -static void SigHupHandler(SIGNAL_ARGS) -{ - t_thrd.page_redo_cxt.got_SIGHUP = true; -} - -/* Run from the worker thread. */ -static void InitGlobals() -{ - t_thrd.utils_cxt.CurrentResourceOwner = ResourceOwnerCreate(NULL, "ExtremeRtoParallelRedoThread", - THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE)); - - t_thrd.xlog_cxt.server_mode = g_redoWorker->initialServerMode; - t_thrd.xlog_cxt.ThisTimeLineID = g_redoWorker->initialTimeLineID; - t_thrd.xlog_cxt.expectedTLIs = g_redoWorker->expectedTLIs; - /* apply recoveryinfo will change standbystate see UpdateRecordGlobals */ - t_thrd.xlog_cxt.standbyState = g_redoWorker->standbyState; - t_thrd.xlog_cxt.StandbyMode = g_redoWorker->StandbyMode; - t_thrd.xlog_cxt.InRecovery = true; - t_thrd.xlog_cxt.startup_processing = true; - t_thrd.proc_cxt.DataDir = g_redoWorker->DataDir; - u_sess->utils_cxt.RecentXmin = g_redoWorker->RecentXmin; - g_redoWorker->proc = t_thrd.proc; - t_thrd.storage_cxt.latestObservedXid = g_redoWorker->latestObservedXid; - t_thrd.xlog_cxt.recoveryTargetTLI = g_redoWorker->recoveryTargetTLI; - t_thrd.xlog_cxt.recoveryRestoreCommand= g_redoWorker->recoveryRestoreCommand; - t_thrd.xlog_cxt.ArchiveRecoveryRequested = g_redoWorker->ArchiveRecoveryRequested; - t_thrd.xlog_cxt.StandbyModeRequested = g_redoWorker->StandbyModeRequested; - t_thrd.xlog_cxt.InArchiveRecovery = g_redoWorker->InArchiveRecovery; - t_thrd.xlog_cxt.InRecovery = g_redoWorker->InRecovery; - t_thrd.xlog_cxt.ArchiveRestoreRequested = g_redoWorker->ArchiveRestoreRequested; - t_thrd.xlog_cxt.minRecoveryPoint = g_redoWorker->minRecoveryPoint; -} - -void WaitRedoWorkersQueueEmpty() -{ - bool queueIsEmpty = false; - while (!queueIsEmpty) { - queueIsEmpty = true; - for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { - PageRedoWorker *worker = g_dispatcher->allWorkers[i]; - if (worker->role == REDO_TRXN_WORKER || worker->role == REDO_PAGE_WORKER) { - if (!RedoWorkerIsIdle(worker)) { - queueIsEmpty = false; - break; - } - } - } - RedoInterruptCallBack(); - } -} - -void RedoThrdWaitForExit(const PageRedoWorker *wk) -{ - uint32 sd = wk->slotId; - switch (wk->role) { - case REDO_BATCH: - SendPageRedoEndMark(g_dispatcher->pageLines[sd].managerThd); - WaitPageRedoWorkerReachLastMark(g_dispatcher->pageLines[sd].managerThd); - break; - case REDO_PAGE_MNG: - DispatchEndMarkToRedoWorkerAndWait(); - break; - case REDO_PAGE_WORKER: - break; /* Don't need to wait for anyone */ - case REDO_TRXN_MNG: - SendPageRedoEndMark(g_dispatcher->trxnLine.redoThd); - WaitRedoWorkersQueueEmpty(); - WaitPageRedoWorkerReachLastMark(g_dispatcher->trxnLine.redoThd); - break; - case REDO_TRXN_WORKER: - break; /* Don't need to wait for anyone */ - default: - break; - } -} - -/* Run from the worker thread. */ -static void ApplySinglePageRecord(RedoItem *item) -{ - XLogReaderState *record = &item->record; - - MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); - ApplyRedoRecord(record); - (void)MemoryContextSwitchTo(oldCtx); -} - -/* Run from the worker thread. */ -static void LastMarkReached() -{ - PosixSemaphorePost(&g_redoWorker->phaseMarker); -} - -/* Run from the dispatcher thread. */ -void WaitPageRedoWorkerReachLastMark(PageRedoWorker *worker) -{ - PosixSemaphoreWait(&worker->phaseMarker); -} - -/* Run from the dispatcher thread. */ -void AddPageRedoItem(PageRedoWorker *worker, void *item) -{ - SPSCBlockingQueuePut(worker->queue, item); -} - -/* Run from the dispatcher thread. */ -bool SendPageRedoEndMark(PageRedoWorker *worker) -{ - return SPSCBlockingQueuePut(worker->queue, &g_redoEndMark); -} - -/* Run from the dispatcher thread. */ -bool SendPageRedoWorkerTerminateMark(PageRedoWorker *worker) -{ - return SPSCBlockingQueuePut(worker->queue, &g_terminateMark); -} - -/* Run from the txn worker thread. */ -void UpdatePageRedoWorkerStandbyState(PageRedoWorker *worker, HotStandbyState newState) -{ - /* - * Here we only save the new state into the worker struct. - * The actual update of the worker thread's state occurs inside - * the apply loop. - */ - worker->standbyState = newState; -} - -/* Run from the dispatcher thread. */ -void *GetXLogInvalidPages(PageRedoWorker *worker) -{ - return worker->xlogInvalidPages; -} - -bool RedoWorkerIsIdle(PageRedoWorker *worker) -{ - return SPSCBlockingQueueIsEmpty(worker->queue); -} - -void DumpPageRedoWorker(PageRedoWorker *worker) -{ - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("[REDO_LOG_TRACE]RedoWorker common info: id %u, tid %lu, " - "lastCheckedRestartPoint %lu, lastReplayedEndRecPtr %lu standbyState %u", - worker->id, worker->tid.thid, worker->lastCheckedRestartPoint, worker->lastReplayedEndRecPtr, - (uint32)worker->standbyState))); - DumpQueue(worker->queue); -} - -void DumpExtremeRtoReadBuf() -{ - if (g_dispatcher == NULL) { - return; - } - - ereport(LOG, - (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("DumpExtremeRtoReadBuf: startworker %u, readindex %u, applyindex %u, readSource %u, failSource %u", - g_dispatcher->rtoXlogBufState.readWorkerState, g_dispatcher->rtoXlogBufState.readindex, - g_dispatcher->rtoXlogBufState.applyindex, g_dispatcher->rtoXlogBufState.readSource, - g_dispatcher->rtoXlogBufState.failSource))); - - for (uint32 i = 0; i < MAX_ALLOC_SEGNUM; ++i) { - ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), - errmsg("DumpExtremeRtoReadBuf: buf %u, state %u, readlen %u, segno %lu, segoffset %lu", i, - g_dispatcher->rtoXlogBufState.xlogsegarray[i].bufState, - g_dispatcher->rtoXlogBufState.xlogsegarray[i].readlen, - g_dispatcher->rtoXlogBufState.xlogsegarray[i].segno, - g_dispatcher->rtoXlogBufState.xlogsegarray[i].segoffset))); - } -} - -bool XactHasSegpageRelFiles(XLogReaderState *record) -{ - int nrels = 0; - ColFileNodeRel *xnodes = NULL; - - if (XLogRecGetRmid(record) != RM_XACT_ID) { - return false; - } - - XactGetRelFiles(record, &xnodes, &nrels); - - for (int32 idx = 0; idx < nrels; idx++) { - ColFileNode colFileNode; - ColFileNodeRel *colFileNodeRel = xnodes + idx; - - ColFileNodeCopy(&colFileNode, colFileNodeRel); - - if (!IsValidColForkNum(colFileNode.forknum) && IsSegmentFileNode(colFileNode.filenode)) { - return true; - } - } - - return false; -} - - -void RepairPageAndRecoveryXLog(BadBlockRecEnt* page_info, const char *page) -{ - RedoBufferInfo buffer; - RedoBufferTag blockinfo; - bool updateFsm = false; - bool notfound = false; - errno_t rc; - BufferDesc *bufDesc = NULL; - RedoTimeCost timeCost1; - RedoTimeCost timeCost2; - - blockinfo.rnode = page_info->key.relfilenode; - blockinfo.forknum = page_info->key.forknum; - blockinfo.blkno = page_info->key.blocknum; - blockinfo.pblk = page_info->pblk; - - /* read page to buffer pool by RBM_ZERO_AND_LOCK mode and get buffer lock */ - (void)XLogReadBufferForRedoBlockExtend(&blockinfo, RBM_ZERO_AND_LOCK, false, &buffer, - page_info->rec_max_lsn, InvalidXLogRecPtr, false, WITH_NORMAL_CACHE); - - rc = memcpy_s(buffer.pageinfo.page, BLCKSZ, page, BLCKSZ); - securec_check(rc, "", ""); - - MarkBufferDirty(buffer.buf); - bufDesc = GetBufferDescriptor(buffer.buf - 1); - bufDesc->lsn_on_disk = PageGetLSN(buffer.pageinfo.page); - UnlockReleaseBuffer(buffer.buf); - - /* recovery the page xlog */ - rc = memset_s(&buffer, sizeof(RedoBufferInfo), 0, sizeof(RedoBufferInfo)); - securec_check(rc, "", ""); - - XLogRecParseState *procState = page_info->head; - MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); - while (procState != NULL) { - XLogRecParseState *redoblockstate = procState; - procState = (XLogRecParseState *)procState->nextrecord; - (void)XLogBlockRedoForExtremeRTO(redoblockstate, &buffer, notfound, timeCost1, timeCost2); - } - (void)MemoryContextSwitchTo(oldCtx); - updateFsm = XlogNeedUpdateFsm(page_info->head, &buffer); - XLogBlockParseStateRelease(page_info->head); - ExtremeRtoFlushBuffer(&buffer, updateFsm); -} - -HTAB* BadBlockHashTblCreate() -{ - HASHCTL ctl; - errno_t rc; - - rc = memset_s(&ctl, sizeof(ctl), 0, sizeof(ctl)); - securec_check(rc, "", ""); - - ctl.keysize = sizeof(RepairBlockKey); - ctl.entrysize = sizeof(BadBlockRecEnt); - ctl.hash = tag_hash; - - return hash_create("recovery thread bad block hashtbl", MAX_REMOTE_READ_INFO_NUM, &ctl, HASH_ELEM | HASH_FUNCTION); -} - - -/* ClearPageRepairHashTbl - * drop table, or truncate table, need clear the page repair hashTbl, if the - * repair page Filenode match need remove. - */ -void ClearRecoveryThreadHashTbl(const RelFileNode &node, ForkNumber forknum, BlockNumber minblkno, - bool segment_shrink) -{ - HTAB *bad_hash = g_redoWorker->badPageHashTbl; - bool found = false; - BadBlockRecEnt *entry = NULL; - HASH_SEQ_STATUS status; - - hash_seq_init(&status, bad_hash); - while ((entry = (BadBlockRecEnt *)hash_seq_search(&status)) != NULL) { - if (BlockNodeMatch(entry->key, entry->pblk, node, forknum, minblkno, segment_shrink)) { - if (hash_search(bad_hash, &(entry->key), HASH_REMOVE, &found) == NULL) { - ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("recovery thread bad page hash table corrupted"))); - } - g_redoWorker->remoteReadPageNum--; - } - } - - return; -} - -/* BatchClearPageRepairHashTbl - * drop database, or drop segmentspace, need clear the page repair hashTbl, - * if the repair page key dbNode match and spcNode match, need remove. - */ -void BatchClearRecoveryThreadHashTbl(Oid spcNode, Oid dbNode) -{ - HTAB *bad_hash = g_redoWorker->badPageHashTbl; - bool found = false; - BadBlockRecEnt *entry = NULL; - HASH_SEQ_STATUS status; - - hash_seq_init(&status, bad_hash); - while ((entry = (BadBlockRecEnt *)hash_seq_search(&status)) != NULL) { - if (dbNodeandSpcNodeMatch(&(entry->key.relfilenode), spcNode, dbNode)) { - if (hash_search(bad_hash, &(entry->key), HASH_REMOVE, &found) == NULL) { - ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("page repair hash table corrupted"))); - } - g_redoWorker->remoteReadPageNum--; - } - } - - return; -} - - -/* ClearSpecificsPageEntryAndMem - * If the page has been repair, need remove entry of bad page hashtable, - * and release the xlog record mem. - */ -void ClearSpecificsPageEntryAndMem(BadBlockRecEnt *entry) -{ - HTAB *bad_hash = g_redoWorker->badPageHashTbl; - bool found = false; - uint32 need_repair_num = 0; - HASH_SEQ_STATUS status; - BadBlockRecEnt *temp_entry = NULL; - - if ((BadBlockRecEnt*)hash_search(bad_hash, &(entry->key), HASH_REMOVE, &found) == NULL) { - ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("recovery thread bad block hash table corrupted"))); - } - - hash_seq_init(&status, bad_hash); - while ((temp_entry = (BadBlockRecEnt *)hash_seq_search(&status)) != NULL) { - need_repair_num++; - } - - if (need_repair_num == 0) { - XLogParseBufferDestoryFunc(&(g_redoWorker->parseManager)); - } -} - -/* RecordBadBlockAndPushToRemote - * If the bad page has been stored, record the xlog. If the bad page - * has not been stored, need push to page repair thread hash table and record to - * recovery thread hash table. - */ -void RecordBadBlockAndPushToRemote(XLogBlockDataParse *datadecode, PageErrorType error_type, - XLogRecPtr old_lsn, XLogPhyBlock pblk) -{ - bool found = false; - RepairBlockKey key; - gs_thread_t tid; - XLogBlockParse *block = STRUCT_CONTAINER(XLogBlockParse, extra_rec, datadecode); - XLogRecParseState *state = STRUCT_CONTAINER(XLogRecParseState, blockparse, block); - - key.relfilenode.spcNode = state->blockparse.blockhead.spcNode; - key.relfilenode.dbNode = state->blockparse.blockhead.dbNode; - key.relfilenode.relNode = state->blockparse.blockhead.relNode; - key.relfilenode.bucketNode = state->blockparse.blockhead.bucketNode; - key.forknum = state->blockparse.blockhead.forknum; - key.blocknum = state->blockparse.blockhead.blkno; - - tid = gs_thread_get_cur_thread(); - found = PushBadPageToRemoteHashTbl(key, error_type, old_lsn, pblk, tid.thid); - - if (found) { - /* store the record for recovery */ - HTAB *bad_hash = g_redoWorker->badPageHashTbl; - bool thread_found = false; - BadBlockRecEnt *remoteReadInfo = (BadBlockRecEnt*)hash_search(bad_hash, &(key), HASH_FIND, &thread_found); - Assert(thread_found); - if (!thread_found) { - ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), - errmsg("recovery thread bad block hash table corrupted"))); - } - XLogRecParseState *newState = XLogParseBufferCopy(state); - newState->nextrecord = NULL; - remoteReadInfo->tail->nextrecord = newState; - remoteReadInfo->tail = newState; - remoteReadInfo->rec_max_lsn = newState->blockparse.blockhead.end_ptr; - } else { - HTAB *bad_hash = g_redoWorker->badPageHashTbl; - bool thread_found = false; - BadBlockRecEnt *remoteReadInfo = (BadBlockRecEnt*)hash_search(bad_hash, &(key), HASH_ENTER, &thread_found); - - Assert(!thread_found); - if (g_parseManager == NULL) { - XLogParseBufferInitFunc(&(g_redoWorker->parseManager), - MAX_PARSE_BUFF_NUM, &recordRefOperate, RedoInterruptCallBack); - } - XLogRecParseState *newState = XLogParseBufferCopy(state); - newState->nextrecord = NULL; - - remoteReadInfo->key = key; - remoteReadInfo->pblk = pblk; - remoteReadInfo->rec_min_lsn = newState->blockparse.blockhead.end_ptr; - remoteReadInfo->rec_max_lsn = newState->blockparse.blockhead.end_ptr; - remoteReadInfo->head = newState; - remoteReadInfo->tail = newState; - g_redoWorker->remoteReadPageNum++; - - if (g_redoWorker->remoteReadPageNum >= MAX_REMOTE_READ_INFO_NUM) { - ereport(WARNING, (errmsg("recovery thread found %d error block.", g_redoWorker->remoteReadPageNum))); - } - } - - return; -} - -void CheckRemoteReadAndRepairPage(BadBlockRecEnt *entry) -{ - XLogRecPtr rec_min_lsn = InvalidXLogRecPtr; - XLogRecPtr rec_max_lsn = InvalidXLogRecPtr; - bool check = false; - RepairBlockKey key; - - key = entry->key; - rec_min_lsn = entry->rec_min_lsn; - rec_max_lsn = entry->rec_max_lsn; - check = CheckRepairPage(key, rec_min_lsn, rec_max_lsn, g_redoWorker->page); - if (check) { - /* copy page to buffer pool, and recovery the stored xlog */ - RepairPageAndRecoveryXLog(entry, g_redoWorker->page); - /* clear page repair thread hash table */ - ClearSpecificsPageRepairHashTbl(key); - /* clear this thread invalid page hash table */ - forget_specified_invalid_pages(key); - /* clear thread bad block hash entry */ - ClearSpecificsPageEntryAndMem(entry); - g_redoWorker->remoteReadPageNum--; - } -} - -void SeqCheckRemoteReadAndRepairPage() -{ - BadBlockRecEnt *entry = NULL; - HASH_SEQ_STATUS status; - - HTAB *bad_hash = g_redoWorker->badPageHashTbl; - - hash_seq_init(&status, bad_hash); - while ((entry = (BadBlockRecEnt *)hash_seq_search(&status)) != NULL) { - CheckRemoteReadAndRepairPage(entry); - } -} - -} // namespace extreme_rto +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * page_redo.cpp + * PageRedoWorker is a thread of execution that replays data page logs. + * It provides a synchronization mechanism for replaying logs touching + * multiple pages. + * + * In the current implementation, logs modifying the same page must + * always be replayed by the same worker. There is no mechanism for + * an idle worker to "steal" work from a busy worker. + * + * IDENTIFICATION + * src/gausskernel/storage/access/transam/parallel_recovery/page_redo.cpp + * + * ------------------------------------------------------------------------- + */ + +#include +#include + +#include "postgres.h" +#include "knl/knl_variable.h" +#include "gs_thread.h" +#include "miscadmin.h" +#include "access/xact.h" +#include "access/xlog.h" +#include "access/xlog_internal.h" +#include "access/xlogutils.h" +#include "access/xlogproc.h" +#include "access/nbtree.h" +#include "catalog/storage_xlog.h" +#include "gssignal/gs_signal.h" +#include "libpq/pqsignal.h" +#include "postmaster/postmaster.h" +#include "storage/ipc.h" +#include "storage/freespace.h" +#include "storage/smgr/smgr.h" +#include "storage/standby.h" +#include "storage/pmsignal.h" +#include "utils/guc.h" +#include "utils/palloc.h" +#include "portability/instr_time.h" +#include "postmaster/startup.h" +#include "postmaster/pagerepair.h" +#include "catalog/storage.h" +#include +#include +#include "commands/dbcommands.h" +#include "commands/tablespace.h" +#include "access/extreme_rto/page_redo.h" +#include "access/extreme_rto/dispatcher.h" +#include "access/extreme_rto/txn_redo.h" +#include "pgstat.h" +#include "access/extreme_rto/batch_redo.h" +#include "access/multi_redo_api.h" +#include "replication/walreceiver.h" +#include "replication/datareceiver.h" +#include "pgxc/barrier.h" +#ifdef ENABLE_MOT +#include "storage/mot/mot_fdw.h" +#endif + +#ifdef EXTREME_RTO_DEBUG +#include +#include + +#include + +#include + +#endif + +#ifdef ENABLE_UT +#include "utils/utesteventutil.h" +#define STATIC +#else +#define STATIC static +#endif + +namespace extreme_rto { +static const int MAX_PARSE_BUFF_NUM = PAGE_WORK_QUEUE_SIZE * 10 * 3; +static const int MAX_LOCAL_BUFF_NUM = PAGE_WORK_QUEUE_SIZE * 10 * 3; + +static const char *const PROCESS_TYPE_CMD_ARG = "--forkpageredo"; +static char g_AUXILIARY_TYPE_CMD_ARG[16] = {0}; + +THR_LOCAL PageRedoWorker *g_redoWorker = NULL; +THR_LOCAL RecordBufferState *g_recordbuffer = NULL; +RedoItem g_redoEndMark = { false, false, NULL, 0, NULL, 0 }; +RedoItem g_terminateMark = { false, false, NULL, 0, NULL, 0 }; +RedoItem g_GlobalLsnForwarder; +RedoItem g_cleanupMark; +RedoItem g_closefdMark; +RedoItem g_cleanInvalidPageMark; + +static const int PAGE_REDO_WORKER_ARG = 3; +static const int REDO_SLEEP_50US = 50; +static const int REDO_SLEEP_100US = 100; + +static void ApplySinglePageRecord(RedoItem *); +static void InitGlobals(); +static void LastMarkReached(); +static void SetupSignalHandlers(); +static void SigHupHandler(SIGNAL_ARGS); +static ThreadId StartWorkerThread(PageRedoWorker *); + +void RedoThrdWaitForExit(const PageRedoWorker *wk); +void AddRefRecord(void *rec); +void SubRefRecord(void *rec); +void GlobalLsnUpdate(); +static void TrxnMangerQueueCallBack(); +#ifdef USE_ASSERT_CHECKING +void RecordBlockCheck(void *rec, XLogRecPtr curPageLsn, uint32 blockId, bool replayed); +#endif +void AddRecordReadBlocks(void *rec, uint32 readblocks); + +RefOperate recordRefOperate = { + AddRefRecord, + SubRefRecord, +#ifdef USE_ASSERT_CHECKING + RecordBlockCheck, +#endif + AddRecordReadBlocks, +}; + +void UpdateRecordGlobals(RedoItem *item, HotStandbyState standbyState) +{ + t_thrd.xlog_cxt.ReadRecPtr = item->record.ReadRecPtr; + t_thrd.xlog_cxt.EndRecPtr = item->record.EndRecPtr; + t_thrd.xlog_cxt.expectedTLIs = item->expectedTLIs; + /* apply recoveryinfo will change standbystate see UpdateRecordGlobals */ + t_thrd.xlog_cxt.standbyState = standbyState; + t_thrd.xlog_cxt.XLogReceiptTime = item->syncXLogReceiptTime; + t_thrd.xlog_cxt.XLogReceiptSource = item->syncXLogReceiptSource; + u_sess->utils_cxt.RecentXmin = item->RecentXmin; + t_thrd.xlog_cxt.server_mode = item->syncServerMode; +} + +/* Run from the dispatcher thread. */ +PageRedoWorker *StartPageRedoWorker(PageRedoWorker *worker) +{ + Assert(worker); + uint32 id = worker->id; + ThreadId threadId = StartWorkerThread(worker); + if (threadId == 0) { + ereport(WARNING, (errmsg("Cannot create page-redo-worker thread: %u, %m.", id))); + DestroyPageRedoWorker(worker); + return NULL; + } else { + ereport(LOG, (errmsg("StartPageRedoWorker successfully create page-redo-worker id: %u, threadId:%lu.", id, + worker->tid.thid))); + } + g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[id].threadId = threadId; + SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.rwlock)); + uint32 state = pg_atomic_read_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[id].threadState)); + if (state != PAGE_REDO_WORKER_READY) { + g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[id].threadState = PAGE_REDO_WORKER_START; + } + SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); + return worker; +} + +void RedoWorkerQueueCallBack() +{ + RedoInterruptCallBack(); +} + +bool RedoWorkerIsUndoSpaceWorker() +{ + return g_redoWorker->isUndoSpaceWorker; +} + +/* Run from the dispatcher thread. */ +PageRedoWorker *CreateWorker(uint32 id) +{ + PageRedoWorker *tmp = (PageRedoWorker *)palloc0(sizeof(PageRedoWorker) + EXTREME_RTO_ALIGN_LEN); + PageRedoWorker *worker; + worker = (PageRedoWorker *)TYPEALIGN(EXTREME_RTO_ALIGN_LEN, tmp); + worker->selfOrinAddr = tmp; + worker->id = id; + worker->index = 0; + worker->tid.thid = InvalidTid; + worker->proc = NULL; + worker->initialServerMode = (ServerMode)t_thrd.xlog_cxt.server_mode; + worker->initialTimeLineID = t_thrd.xlog_cxt.ThisTimeLineID; + worker->expectedTLIs = t_thrd.xlog_cxt.expectedTLIs; + worker->recoveryTargetTLI = t_thrd.xlog_cxt.recoveryTargetTLI; + worker->recoveryRestoreCommand = t_thrd.xlog_cxt.recoveryRestoreCommand; + worker->ArchiveRecoveryRequested = t_thrd.xlog_cxt.ArchiveRecoveryRequested; + worker->StandbyModeRequested = t_thrd.xlog_cxt.StandbyModeRequested; + worker->InArchiveRecovery = t_thrd.xlog_cxt.InArchiveRecovery; + worker->InRecovery = t_thrd.xlog_cxt.InRecovery; + worker->ArchiveRestoreRequested = t_thrd.xlog_cxt.ArchiveRestoreRequested; + worker->minRecoveryPoint = t_thrd.xlog_cxt.minRecoveryPoint; + + worker->pendingHead = NULL; + worker->pendingTail = NULL; + worker->queue = SPSCBlockingQueueCreate(PAGE_WORK_QUEUE_SIZE, RedoWorkerQueueCallBack); + worker->lastCheckedRestartPoint = InvalidXLogRecPtr; + worker->lastReplayedEndRecPtr = InvalidXLogRecPtr; + worker->standbyState = (HotStandbyState)t_thrd.xlog_cxt.standbyState; + worker->StandbyMode = t_thrd.xlog_cxt.StandbyMode; + worker->latestObservedXid = t_thrd.storage_cxt.latestObservedXid; + worker->DataDir = t_thrd.proc_cxt.DataDir; + worker->RecentXmin = u_sess->utils_cxt.RecentXmin; + worker->xlogInvalidPages = NULL; + PosixSemaphoreInit(&worker->phaseMarker, 0); + worker->oldCtx = NULL; + worker->fullSyncFlag = 0; +#if (!defined __x86_64__) && (!defined __aarch64__) + SpinLockInit(&worker->ptrLck); +#endif + worker->parseManager.memctl.isInit = false; + worker->parseManager.parsebuffers = NULL; + worker->remoteReadPageNum = 0; + worker->badPageHashTbl = BadBlockHashTblCreate(); + return worker; +} + +/* Run from the dispatcher thread. */ +static ThreadId StartWorkerThread(PageRedoWorker *worker) +{ + worker->tid.thid = initialize_util_thread(PAGEREDO, worker); + return worker->tid.thid; +} + +/* Run from the dispatcher thread. */ +void DestroyPageRedoWorker(PageRedoWorker *worker) +{ + PosixSemaphoreDestroy(&worker->phaseMarker); + SPSCBlockingQueueDestroy(worker->queue); + XLogRedoBufferDestoryFunc(&(worker->bufferManager)); + XLogParseBufferDestoryFunc(&(worker->parseManager)); + pfree(worker->selfOrinAddr); +} + +/* automic write for lastReplayedReadRecPtr and lastReplayedEndRecPtr */ +void SetCompletedReadEndPtr(PageRedoWorker *worker, XLogRecPtr readPtr, XLogRecPtr endPtr) +{ + volatile PageRedoWorker *tmpWk = worker; +#if defined(__x86_64__) || defined(__aarch64__) + uint128_u exchange; + uint128_u current; + uint128_u compare = atomic_compare_and_swap_u128((uint128_u *)&tmpWk->lastReplayedReadRecPtr); + + Assert(sizeof(tmpWk->lastReplayedReadRecPtr) == 8); + Assert(sizeof(tmpWk->lastReplayedEndRecPtr) == 8); + + exchange.u64[0] = (uint64)readPtr; + exchange.u64[1] = (uint64)endPtr; + +loop: + current = atomic_compare_and_swap_u128((uint128_u *)&tmpWk->lastReplayedReadRecPtr, compare, exchange); + if (!UINT128_IS_EQUAL(compare, current)) { + UINT128_COPY(compare, current); + goto loop; + } +#else + SpinLockAcquire(&tmpWk->ptrLck); + tmpWk->lastReplayedReadRecPtr = readPtr; + tmpWk->lastReplayedEndRecPtr = endPtr; + SpinLockRelease(&tmpWk->ptrLck); +#endif /* __x86_64__ || __aarch64__ */ +} + +/* automic write for lastReplayedReadRecPtr and lastReplayedEndRecPtr */ +void GetCompletedReadEndPtr(PageRedoWorker *worker, XLogRecPtr *readPtr, XLogRecPtr *endPtr) +{ + volatile PageRedoWorker *tmpWk = worker; +#if defined(__x86_64__) || defined(__aarch64__) + uint128_u compare = atomic_compare_and_swap_u128((uint128_u *)&tmpWk->lastReplayedReadRecPtr); + Assert(sizeof(tmpWk->lastReplayedReadRecPtr) == 8); + Assert(sizeof(tmpWk->lastReplayedEndRecPtr) == 8); + + *readPtr = (XLogRecPtr)compare.u64[0]; + *endPtr = (XLogRecPtr)compare.u64[1]; +#else + SpinLockAcquire(&tmpWk->ptrLck); + *readPtr = tmpWk->lastReplayedReadRecPtr; + *endPtr = tmpWk->lastReplayedEndRecPtr; + SpinLockRelease(&tmpWk->ptrLck); +#endif /* __x86_64__ || __aarch64__ */ +} + +/* Run from both the dispatcher and the worker thread. */ +bool IsPageRedoWorkerProcess(int argc, char *argv[]) +{ + return strcmp(argv[1], PROCESS_TYPE_CMD_ARG) == 0; +} + +/* Run from the worker thread. */ +void AdaptArgvForPageRedoWorker(char *argv[]) +{ + if (g_AUXILIARY_TYPE_CMD_ARG[0] == 0) + sprintf_s(g_AUXILIARY_TYPE_CMD_ARG, sizeof(g_AUXILIARY_TYPE_CMD_ARG), "-x%d", PageRedoProcess); + argv[3] = g_AUXILIARY_TYPE_CMD_ARG; +} + +/* Run from the worker thread. */ +void GetThreadNameIfPageRedoWorker(int argc, char *argv[], char **threadNamePtr) +{ + if (*threadNamePtr == NULL && IsPageRedoWorkerProcess(argc, argv)) + *threadNamePtr = "PageRedoWorker"; +} + +/* Run from the worker thread. */ +uint32 GetMyPageRedoWorkerIdWithLock() +{ + bool isWorkerStarting = false; + SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.rwlock)); + isWorkerStarting = ((g_instance.comm_cxt.predo_cxt.state == REDO_STARTING_BEGIN) ? true : false); + SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); + if (!isWorkerStarting) { + ereport(WARNING, (errmsg("GetMyPageRedoWorkerIdWithLock Page-redo-worker exit."))); + proc_exit(0); + } + + return g_redoWorker->id; +} + +/* Run from any worker thread. */ +PGPROC *GetPageRedoWorkerProc(PageRedoWorker *worker) +{ + return worker->proc; +} + +void HandlePageRedoPageRepair(RepairBlockKey key, XLogPhyBlock pblk) +{ + RecordBadBlockAndPushToRemote(g_redoWorker->curRedoBlockState, CRC_CHECK_FAIL, InvalidXLogRecPtr, pblk); +} + +void HandlePageRedoInterrupts() +{ + if (t_thrd.page_redo_cxt.got_SIGHUP) { + t_thrd.page_redo_cxt.got_SIGHUP = false; + ProcessConfigFile(PGC_SIGHUP); + } + + if (t_thrd.page_redo_cxt.check_repair) { + SeqCheckRemoteReadAndRepairPage(); + t_thrd.page_redo_cxt.check_repair = false; + } + + if (t_thrd.page_redo_cxt.shutdown_requested) { + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("page worker id %u exit for request", g_redoWorker->id))); + + pg_atomic_write_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[g_redoWorker->id].threadState), + PAGE_REDO_WORKER_EXIT); + + proc_exit(1); + } +} + +void ReferenceRedoItem(void *item) +{ + RedoItem *redoItem = (RedoItem *)item; + AddRefRecord(&redoItem->record); +} + +void DereferenceRedoItem(void *item) +{ + RedoItem *redoItem = (RedoItem *)item; + SubRefRecord(&redoItem->record); +} + +#define STRUCT_CONTAINER(type, membername, ptr) ((type *)((char *)(ptr)-offsetof(type, membername))) + +#ifdef USE_ASSERT_CHECKING +void RecordBlockCheck(void *rec, XLogRecPtr curPageLsn, uint32 blockId, bool replayed) +{ + XLogReaderState *record = (XLogReaderState *)rec; + if (record->blocks[blockId].forknum != MAIN_FORKNUM) { + return; + } + + if (replayed) { + uint32 rmid = XLogRecGetRmid(record); + uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; + + if (curPageLsn == InvalidXLogRecPtr && (rmid == RM_HEAP2_ID || rmid == RM_HEAP_ID || rmid == RM_HEAP3_ID)) { + uint32 shiftSize = 32; + ereport(LOG, (errmsg("pass checked, record lsn:%X/%X, type: %u %u", + static_cast(record->EndRecPtr >> shiftSize), static_cast(record->EndRecPtr), + record->decoded_record->xl_rmid, record->decoded_record->xl_info))); + } else if (!(rmid == RM_HEAP2_ID && info == XLOG_HEAP2_VISIBLE) && + !(rmid == RM_HEAP_ID && info == XLOG_HEAP_NEWPAGE)) { + Assert(XLByteLE(record->EndRecPtr, curPageLsn)); + } + } + + Assert(blockId < (XLR_MAX_BLOCK_ID + 1)); + record->blocks[blockId].replayed = 1; +} + +#endif + +void AddRecordReadBlocks(void *rec, uint32 readblocks) +{ + XLogReaderState *record = (XLogReaderState *)rec; + record->readblocks += readblocks; +} + +void AddRefRecord(void *rec) +{ + pg_memory_barrier(); +#ifndef EXTREME_RTO_DEBUG + (void)pg_atomic_fetch_add_u32(&((XLogReaderState *)rec)->refcount, 1); +#else + uint32 relCount = pg_atomic_fetch_add_u32(&((XLogReaderState *)rec)->refcount, 1); + + const int stack_size = 5; + const int max_out_put_buf = 4096; + void *buffer[stack_size]; + int nptrs; + char output[max_out_put_buf]; + char **strings; + nptrs = backtrace(buffer, stack_size); + strings = backtrace_symbols(buffer, nptrs); + + int ret = sprintf_s(output, sizeof(output), "before add relcount %u lsn %X/%X call back trace: \n", relCount, + (uint32)(((XLogReaderState *)rec)->EndRecPtr >> 32), + (uint32)(((XLogReaderState *)rec)->EndRecPtr)); + securec_check_ss_c(ret, "\0", "\0"); + for (int i = 0; i < nptrs; ++i) { + ret = strcat_s(output, max_out_put_buf - strlen(output), strings[i]); + securec_check_ss_c(ret, "\0", "\0"); + ret = strcat_s(output, max_out_put_buf - strlen(output), "\n"); + securec_check_ss_c(ret, "\0", "\0"); + } + + free(strings); + ereport(LOG, (errcode(ERRCODE_DATA_CORRUPTED), errmsg(" AddRefRecord print: %s", output))); + +#endif +} + +void SubRefRecord(void *rec) +{ + pg_memory_barrier(); + Assert(((XLogReaderState *)rec)->refcount != 0); + uint32 relCount = pg_atomic_sub_fetch_u32(&((XLogReaderState *)rec)->refcount, 1); +#ifdef EXTREME_RTO_DEBUG + const int stack_size = 5; + const int max_out_put_buf = 4096; + void *buffer[stack_size]; + int nptrs; + char output[max_out_put_buf]; + char **strings; + nptrs = backtrace(buffer, stack_size); + strings = backtrace_symbols(buffer, nptrs); + + int ret = sprintf_s(output, sizeof(output), "after sub relcount %u lsn %X/%X call back trace:\n", relCount, + (uint32)(((XLogReaderState *)rec)->EndRecPtr >> 32), + (uint32)(((XLogReaderState *)rec)->EndRecPtr)); + securec_check_ss_c(ret, "\0", "\0"); + for (int i = 0; i < nptrs; ++i) { + ret = strcat_s(output, max_out_put_buf - strlen(output), strings[i]); + securec_check_ss_c(ret, "\0", "\0"); + ret = strcat_s(output, max_out_put_buf - strlen(output), "\n"); + securec_check_ss_c(ret, "\0", "\0"); + } + free(strings); + ereport(LOG, (errcode(ERRCODE_DATA_CORRUPTED), errmsg(" SubRefRecord print: %s", output))); + +#endif + + if (relCount == 0) { + RedoItem *item = STRUCT_CONTAINER(RedoItem, record, rec); + FreeRedoItem(item); + } +} + +bool BatchRedoParseItemAndDispatch(RedoItem *item) +{ + uint32 blockNum = 0; + XLogRecParseState *recordblockstate = XLogParseToBlockForExtermeRTO(&item->record, &blockNum); + if (recordblockstate == NULL) { + if (blockNum == 0) { + return false; + } + return true; /* out of mem */ + } + + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + AddPageRedoItem(myRedoLine->managerThd, recordblockstate); + return false; +} + +void BatchRedoDistributeEndMark(void) +{ + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + SendPageRedoEndMark(myRedoLine->managerThd); +} + +void BatchRedoProcLsnForwarder(RedoItem *lsnForwarder) +{ + SetCompletedReadEndPtr(g_redoWorker, lsnForwarder->record.ReadRecPtr, lsnForwarder->record.EndRecPtr); + (void)pg_atomic_sub_fetch_u32(&lsnForwarder->record.refcount, 1); + + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + AddPageRedoItem(myRedoLine->managerThd, lsnForwarder); +} + +void BatchRedoProcCleanupMark(RedoItem *cleanupMark) +{ + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); + AddPageRedoItem(myRedoLine->managerThd, cleanupMark); + ereport(LOG, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]BatchRedoProcCleanupMark has cleaned InvalidPages"))); +} + +#ifdef ENABLE_DISTRIBUTE_TEST +// inject delay to slow the process and also can be used as UT mock stub +void InjectDelayWaitRedoPageManagerQueueEmpty() +{ + const uint32 sleepTime = 1000000; + ereport(LOG, (errmsg("ProcessRedoPageManagerQueueEmpty sleep"))); + pg_usleep(sleepTime); +} +#endif + +void WaitAllRedoWorkerQueueEmpty() +{ + if ((get_real_recovery_parallelism() > 1) && (GetBatchCount() > 0)) { + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; + + for (uint32 i = 0; i < WorkerNumPerMng; ++i) { + while (!SPSCBlockingQueueIsEmpty(myRedoLine->redoThd[i]->queue)) { + RedoInterruptCallBack(); + } + } + } +} + +bool BatchRedoDistributeItems(void **eleArry, uint32 eleNum) +{ + bool parsecomplete = false; + for (uint32 i = 0; i < eleNum; i++) { + if (eleArry[i] == (void *)&g_redoEndMark) { + return true; + } else if (eleArry[i] == (void *)&g_GlobalLsnForwarder) { + BatchRedoProcLsnForwarder((RedoItem *)eleArry[i]); + } else if (eleArry[i] == (void *)&g_cleanupMark) { + BatchRedoProcCleanupMark((RedoItem *)eleArry[i]); + } else if (eleArry[i] == (void *)&g_closefdMark) { + smgrcloseall(); + } else if (eleArry[i] == (void *)&g_cleanInvalidPageMark) { + forget_range_invalid_pages((void *)eleArry[i]); + } else { + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); + RedoItem *item = (RedoItem *)eleArry[i]; + UpdateRecordGlobals(item, g_redoWorker->standbyState); + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3], + g_redoWorker->timeCostList[TIME_COST_STEP_6]); + do { + parsecomplete = BatchRedoParseItemAndDispatch(item); + RedoInterruptCallBack(); + } while (parsecomplete); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + DereferenceRedoItem(item); + } + } + + return false; +} + +void BatchRedoMain() +{ + void **eleArry; + uint32 eleNum; + + (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); + XLogParseBufferInitFunc(&(g_redoWorker->parseManager), MAX_PARSE_BUFF_NUM, &recordRefOperate, + RedoInterruptCallBack); + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); + while (SPSCBlockingQueueGetAll(g_redoWorker->queue, &eleArry, &eleNum)) { + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); + bool isEnd = BatchRedoDistributeItems(eleArry, eleNum); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); + SPSCBlockingQueuePopN(g_redoWorker->queue, eleNum); + if (isEnd) + break; + + RedoInterruptCallBack(); + ADD_ABNORMAL_POSITION(1); + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); + } + + RedoThrdWaitForExit(g_redoWorker); + XLogParseBufferDestoryFunc(&(g_redoWorker->parseManager)); +} + +uint32 GetWorkerId(const RedoItemTag *redoItemTag, uint32 workerCount) +{ + if (workerCount != 0) { + return tag_hash(redoItemTag, sizeof(RedoItemTag)) % workerCount; + } + return 0; +} + +uint32 GetWorkerId(const uint32 attId, const uint32 workerCount) +{ + if (workerCount != 0) { + return attId % workerCount; + } + return 0; +} + +void RedoPageManagerDistributeToAllOneBlock(XLogRecParseState *ddlParseState) +{ + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; + + ddlParseState->nextrecord = NULL; + + for (uint32 i = 0; i < WorkerNumPerMng; ++i) { + XLogRecParseState *newState = XLogParseBufferCopy(ddlParseState); + AddPageRedoItem(myRedoLine->redoThd[i], newState); + } +} + +void RedoPageManagerDistributeBlockRecord(HTAB *redoItemHash, XLogRecParseState *parsestate) +{ + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; + HASH_SEQ_STATUS status; + RedoItemHashEntry *redoItemEntry = NULL; + HTAB *curMap = redoItemHash; + hash_seq_init(&status, curMap); + + while ((redoItemEntry = (RedoItemHashEntry *)hash_seq_search(&status)) != NULL) { + uint32 workId = GetWorkerId(&redoItemEntry->redoItemTag, WorkerNumPerMng); + AddPageRedoItem(myRedoLine->redoThd[workId], redoItemEntry->head); + + if (hash_search(curMap, (void *)&redoItemEntry->redoItemTag, HASH_REMOVE, NULL) == NULL) + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("hash table corrupted"))); + } + + if (parsestate != NULL) { + RedoPageManagerDistributeToAllOneBlock(parsestate); + } +} + +void WaitCurrentPipeLineRedoWorkersQueueEmpty() +{ + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; + + for (uint32 i = 0; i < WorkerNumPerMng; ++i) { + while (!SPSCBlockingQueueIsEmpty(myRedoLine->redoThd[i]->queue)) { + RedoInterruptCallBack(); + } + } +} + +void DispatchEndMarkToRedoWorkerAndWait() +{ + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + const uint32 WorkerNumPerMng = get_page_redo_worker_num_per_manager(); + for (uint32 i = 0; i < WorkerNumPerMng; ++i) + SendPageRedoEndMark(myRedoLine->redoThd[i]); + + for (uint32 i = 0; i < myRedoLine->redoThdNum; i++) { + WaitPageRedoWorkerReachLastMark(myRedoLine->redoThd[i]); + } +} + +void RedoPageManagerDdlAction(XLogRecParseState *parsestate) +{ + switch (parsestate->blockparse.blockhead.block_valid) { + case BLOCK_DATA_DROP_DATABASE_TYPE: + xlog_db_drop(parsestate->blockparse.blockhead.end_ptr, parsestate->blockparse.blockhead.dbNode, + parsestate->blockparse.blockhead.spcNode); + break; + case BLOCK_DATA_CREATE_DATABASE_TYPE: + xlog_db_create(parsestate->blockparse.blockhead.dbNode, parsestate->blockparse.blockhead.spcNode, + parsestate->blockparse.extra_rec.blockdatabase.src_db_id, + parsestate->blockparse.extra_rec.blockdatabase.src_tablespace_id); + break; + case BLOCK_DATA_DROP_TBLSPC_TYPE: + xlog_drop_tblspc(parsestate->blockparse.blockhead.spcNode); + break; + case BLOCK_DATA_SEG_FILE_EXTEND_TYPE: + { + Assert(0); + } + break; + case BLOCK_DATA_SEG_SPACE_DROP: + case BLOCK_DATA_SEG_FULL_SYNC_TYPE: + case BLOCK_DATA_SEG_EXTEND: + ProcSegPageCommonRedo(parsestate); + break; + default: + break; + } +} + +void RedoPageManagerSmgrClose(XLogRecParseState *parsestate) +{ + switch (parsestate->blockparse.blockhead.block_valid) { + case BLOCK_DATA_DROP_DATABASE_TYPE: + smgrcloseall(); + break; + case BLOCK_DATA_SEG_FULL_SYNC_TYPE: + ProcSegPageJustFreeChildState(parsestate); + break; + default: + break; + } +} + +void RedoPageManagerSyncDdlAction(XLogRecParseState *parsestate) +{ + /* at this monent, all worker queue is empty ,just find out which one will do it */ + uint32 expected = 0; + const uint32 pipelineNum = g_dispatcher->pageLineNum; + pg_atomic_compare_exchange_u32(&g_dispatcher->syncEnterCount, &expected, pipelineNum); + uint32 entershareCount = pg_atomic_sub_fetch_u32(&g_dispatcher->syncEnterCount, 1); + + MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); + if (entershareCount == 0) { + /* do actual work */ + RedoPageManagerDdlAction(parsestate); + } else { + RedoPageManagerSmgrClose(parsestate); + do { + RedoInterruptCallBack(); + entershareCount = pg_atomic_read_u32(&g_dispatcher->syncEnterCount); + } while (entershareCount != 0); + } + (void)MemoryContextSwitchTo(oldCtx); + + expected = 0; + pg_atomic_compare_exchange_u32(&g_dispatcher->syncExitCount, &expected, pipelineNum); + uint32 exitShareCount = pg_atomic_sub_fetch_u32(&g_dispatcher->syncExitCount, 1); + while (exitShareCount != 0) { + RedoInterruptCallBack(); + exitShareCount = pg_atomic_read_u32(&g_dispatcher->syncExitCount); + } + + parsestate->nextrecord = NULL; + XLogBlockParseStateRelease(parsestate); +} + +void RedoPageManagerDoDropAction(XLogRecParseState *parsestate, HTAB *hashMap) +{ + XLogRecParseState *newState = XLogParseBufferCopy(parsestate); + PRTrackClearBlock(newState, hashMap); + RedoPageManagerDistributeBlockRecord(hashMap, parsestate); + WaitCurrentPipeLineRedoWorkersQueueEmpty(); + RedoPageManagerSyncDdlAction(parsestate); +} + +void RedoPageManagerDoSmgrAction(XLogRecParseState *recordblockstate) +{ + RedoBufferInfo bufferinfo = {0}; + void *blockrecbody; + XLogBlockHead *blockhead; + + blockhead = &recordblockstate->blockparse.blockhead; + blockrecbody = &recordblockstate->blockparse.extra_rec; + + XLogBlockInitRedoBlockInfo(blockhead, &bufferinfo.blockinfo); + + MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); + XLogBlockDdlDoSmgrAction(blockhead, blockrecbody, &bufferinfo); + (void)MemoryContextSwitchTo(oldCtx); + + recordblockstate->nextrecord = NULL; + XLogBlockParseStateRelease(recordblockstate); +} + +void RedoPageManagerDoDataTypeAction(XLogRecParseState *parsestate, HTAB *hashMap) +{ + XLogBlockDdlParse *ddlrecparse = NULL; + XLogBlockParseGetDdlParse(parsestate, ddlrecparse); + + if (ddlrecparse->blockddltype == BLOCK_DDL_DROP_RELNODE || + ddlrecparse->blockddltype == BLOCK_DDL_TRUNCATE_RELNODE) { + XLogRecParseState *newState = XLogParseBufferCopy(parsestate); + PRTrackClearBlock(newState, hashMap); + RedoPageManagerDistributeBlockRecord(hashMap, parsestate); + WaitCurrentPipeLineRedoWorkersQueueEmpty(); + } + + RedoPageManagerDoSmgrAction(parsestate); + +} + +void PageManagerProcLsnForwarder(RedoItem *lsnForwarder) +{ + SetCompletedReadEndPtr(g_redoWorker, lsnForwarder->record.ReadRecPtr, lsnForwarder->record.EndRecPtr); + (void)pg_atomic_sub_fetch_u32(&lsnForwarder->record.refcount, 1); + + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; + + for (uint32 i = 0; i < WorkerNumPerMng; ++i) { + AddPageRedoItem(myRedoLine->redoThd[i], lsnForwarder); + } +} + +void PageManagerDistributeBcmBlock(XLogRecParseState *preState) +{ + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; + uint32 workId = GetWorkerId((uint32)preState->blockparse.blockhead.forknum, WorkerNumPerMng); + AddPageRedoItem(myRedoLine->redoThd[workId], preState); +} + +void PageManagerProcCleanupMark(RedoItem *cleanupMark) +{ + PageRedoPipeline *myRedoLine = &g_dispatcher->pageLines[g_redoWorker->slotId]; + const uint32 WorkerNumPerMng = myRedoLine->redoThdNum; + g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); + for (uint32 i = 0; i < WorkerNumPerMng; ++i) { + AddPageRedoItem(myRedoLine->redoThd[i], cleanupMark); + } + ereport(LOG, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]PageManagerProcCleanupMark has cleaned InvalidPages"))); +} + +void PageManagerProcCheckPoint(HTAB *hashMap, XLogRecParseState *parseState) +{ + Assert(IsCheckPoint(parseState)); + RedoPageManagerDistributeBlockRecord(hashMap, parseState); + bool needWait = parseState->isFullSync; + if (needWait) { + pg_atomic_write_u32(&g_redoWorker->fullSyncFlag, 1); + } + + XLogBlockParseStateRelease(parseState); + uint32 val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); + while (val != 0) { + RedoInterruptCallBack(); + val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); + } + +#ifdef USE_ASSERT_CHECKING + int printLevel = WARNING; +#else + int printLevel = DEBUG1; +#endif + if (log_min_messages <= printLevel) { + GetThreadBufferLeakNum(); + } +} + +void PageManagerProcCreateTableSpace(HTAB *hashMap, XLogRecParseState *parseState) +{ + RedoPageManagerDistributeBlockRecord(hashMap, NULL); + bool needWait = parseState->isFullSync; + if (needWait) { + pg_atomic_write_u32(&g_redoWorker->fullSyncFlag, 1); + } + + XLogBlockParseStateRelease(parseState); + uint32 val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); + while (val != 0) { + RedoInterruptCallBack(); + val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); + } +} + +void PageManagerProcSegFullSyncState(HTAB *hashMap, XLogRecParseState *parseState) +{ + RedoPageManagerDistributeBlockRecord(hashMap, NULL); + WaitCurrentPipeLineRedoWorkersQueueEmpty(); + RedoPageManagerSyncDdlAction(parseState); +} + +void PageManagerProcSegPipeLineSyncState(HTAB *hashMap, XLogRecParseState *parseState) +{ + RedoPageManagerDistributeBlockRecord(hashMap, NULL); + WaitCurrentPipeLineRedoWorkersQueueEmpty(); + MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); + + RedoPageManagerDdlAction(parseState); + + (void)MemoryContextSwitchTo(oldCtx); + XLogBlockParseStateRelease(parseState); +} + +static void WaitNextBarrier(XLogRecParseState *parseState) +{ + const int MAINDATALEN = (int)parseState->blockparse.extra_rec.blockbarrier.maindatalen; + XLogRecPtr barrierLSN = parseState->blockparse.blockhead.end_ptr; + if (!IS_DISASTER_RECOVER_MODE || XLogRecPtrIsInvalid(t_thrd.xlog_cxt.minRecoveryPoint) || + XLByteLT(barrierLSN, t_thrd.xlog_cxt.minRecoveryPoint) || + t_thrd.shemem_ptr_cxt.ControlFile->backupEndRequired) { + XLogBlockParseStateRelease(parseState); + return; + } + + char barrier[MAINDATALEN + 1]; + errno_t rc = memcpy_s(barrier, MAINDATALEN, parseState->blockparse.extra_rec.blockbarrier.maindata, MAINDATALEN); + securec_check(rc, "\0", "\0"); + + uint8 info = parseState->blockparse.blockhead.xl_info & ~XLR_INFO_MASK; + XLogBlockParseStateRelease(parseState); + volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; + + if (info == XLOG_BARRIER_COMMIT || !IS_DISASTER_RECOVER_MODE || !is_barrier_pausable(barrier)) + return; + + while (true) { + SpinLockAcquire(&walrcv->mutex); + if (BARRIER_LT((char *)barrier, (char *)walrcv->recoveryTargetBarrierId) || + BARRIER_LE((char *)barrier, (char *)walrcv->recoveryStopBarrierId)|| + BARRIER_EQ((char *)barrier, (char *)walrcv->recoverySwitchoverBarrierId)) { + SpinLockRelease(&walrcv->mutex); + break; + } else { + SpinLockRelease(&walrcv->mutex); + pg_usleep(1000L); + RedoInterruptCallBack(); + } + } +} + +void PageManagerRedoParseState(XLogRecParseState *preState) +{ + HTAB *hashMap = g_dispatcher->pageLines[g_redoWorker->slotId].managerThd->redoItemHash; + + switch (preState->blockparse.blockhead.block_valid) { + case BLOCK_DATA_MAIN_DATA_TYPE: + case BLOCK_DATA_UNDO_TYPE: + case BLOCK_DATA_VM_TYPE: + case BLOCK_DATA_FSM_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); + PRTrackAddBlock(preState, hashMap); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); + break; + case BLOCK_DATA_DDL_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); + RedoPageManagerDoDataTypeAction(preState, hashMap); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); + break; + case BLOCK_DATA_SEG_EXTEND: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); + PageManagerProcSegPipeLineSyncState(hashMap, preState); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); + break; + case BLOCK_DATA_DROP_DATABASE_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_5]); + RedoPageManagerDoDropAction(preState, hashMap); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_5]); + break; + case BLOCK_DATA_DROP_TBLSPC_TYPE: + /* just make sure any other ddl before drop tblspc is done */ + XLogBlockParseStateRelease(preState); + break; + case BLOCK_DATA_CREATE_DATABASE_TYPE: + case BLOCK_DATA_SEG_FILE_EXTEND_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + RedoPageManagerDistributeBlockRecord(hashMap, NULL); + /* wait until queue empty */ + WaitCurrentPipeLineRedoWorkersQueueEmpty(); + /* do atcual action */ + RedoPageManagerSyncDdlAction(preState); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + break; + case BLOCK_DATA_SEG_FULL_SYNC_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); + PageManagerProcSegFullSyncState(hashMap, preState); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); + break; + case BLOCK_DATA_CREATE_TBLSPC_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); + PageManagerProcCreateTableSpace(hashMap, preState); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); + break; + case BLOCK_DATA_XLOG_COMMON_TYPE: + PageManagerProcCheckPoint(hashMap, preState); + break; + case BLOCK_DATA_NEWCU_TYPE: + RedoPageManagerDistributeBlockRecord(hashMap, NULL); + PageManagerDistributeBcmBlock(preState); + break; + case BLOCK_DATA_SEG_SPACE_DROP: + case BLOCK_DATA_SEG_SPACE_SHRINK: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); + RedoPageManagerDistributeBlockRecord(hashMap, preState); + WaitCurrentPipeLineRedoWorkersQueueEmpty(); + RedoPageManagerSyncDdlAction(preState); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); + break; + case BLOCK_DATA_BARRIER_TYPE: + RedoPageManagerDistributeBlockRecord(hashMap, preState); + WaitNextBarrier(preState); + break; + default: + XLogBlockParseStateRelease(preState); + break; + } +} + +bool PageManagerRedoDistributeItems(void **eleArry, uint32 eleNum) +{ + HTAB *hashMap = g_dispatcher->pageLines[g_redoWorker->slotId].managerThd->redoItemHash; + + for (uint32 i = 0; i < eleNum; i++) { + if (eleArry[i] == (void *)&g_redoEndMark) { + RedoPageManagerDistributeBlockRecord(hashMap, NULL); + return true; + } else if (eleArry[i] == (void *)&g_GlobalLsnForwarder) { + RedoPageManagerDistributeBlockRecord(hashMap, NULL); + PageManagerProcLsnForwarder((RedoItem *)eleArry[i]); + continue; + } else if (eleArry[i] == (void *)&g_cleanupMark) { + PageManagerProcCleanupMark((RedoItem *)eleArry[i]); + continue; + } else if (eleArry[i] == (void *)&g_closefdMark) { + smgrcloseall(); + continue; + } else if (eleArry[i] == (void *)&g_cleanInvalidPageMark) { + forget_range_invalid_pages((void *)eleArry[i]); + continue; + } + XLogRecParseState *recordblockstate = (XLogRecParseState *)eleArry[i]; + XLogRecParseState *nextState = recordblockstate; + do { + XLogRecParseState *preState = nextState; + nextState = (XLogRecParseState *)nextState->nextrecord; + preState->nextrecord = NULL; +#ifdef ENABLE_UT + TestXLogRecParseStateEventProbe(UTEST_EVENT_RTO_PAGEMGR_REDO_BEFORE_DISTRIBUTE_ITEMS, + __FUNCTION__, preState); +#endif + + PageManagerRedoParseState(preState); +#ifdef ENABLE_UT + TestXLogRecParseStateEventProbe(UTEST_EVENT_RTO_PAGEMGR_REDO_AFTER_DISTRIBUTE_ITEMS, + __FUNCTION__, preState); +#endif + } while (nextState != NULL); + } + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_9]); + RedoPageManagerDistributeBlockRecord(hashMap, NULL); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_9]); + return false; +} + +void RedoPageManagerMain() +{ + void **eleArry; + uint32 eleNum; + + (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); + g_redoWorker->redoItemHash = PRRedoItemHashInitialize(g_redoWorker->oldCtx); + XLogParseBufferInitFunc(&(g_redoWorker->parseManager), MAX_PARSE_BUFF_NUM, &recordRefOperate, + RedoInterruptCallBack); + + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); + while (SPSCBlockingQueueGetAll(g_redoWorker->queue, &eleArry, &eleNum)) { + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); + bool isEnd = PageManagerRedoDistributeItems(eleArry, eleNum); + SPSCBlockingQueuePopN(g_redoWorker->queue, eleNum); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); + if (isEnd) + break; + + RedoInterruptCallBack(); + ADD_ABNORMAL_POSITION(5); + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); + } + + RedoThrdWaitForExit(g_redoWorker); + XLogParseBufferDestoryFunc(&(g_redoWorker->parseManager)); +} + +bool IsXactXlog(const XLogReaderState *record) +{ + if (XLogRecGetRmid(record) != RM_XACT_ID) { + return false; + } + return true; +} + +void TrxnManagerProcLsnForwarder(RedoItem *lsnForwarder) +{ + SetCompletedReadEndPtr(g_redoWorker, lsnForwarder->record.ReadRecPtr, lsnForwarder->record.EndRecPtr); + (void)pg_atomic_sub_fetch_u32(&lsnForwarder->record.refcount, 1); + + AddPageRedoItem(g_dispatcher->trxnLine.redoThd, lsnForwarder); +} + +void TrxnManagerProcCleanupMark(RedoItem *cleanupMark) +{ + g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); + AddPageRedoItem(g_dispatcher->trxnLine.redoThd, cleanupMark); + ereport(LOG, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]TrxnManagerProcCleanupMark has cleaned InvalidPages"))); +} + +bool TrxnManagerDistributeItemsBeforeEnd(RedoItem *item) +{ + bool exitFlag = false; + if (item == &g_redoEndMark) { + exitFlag = true; + } else if (item == (RedoItem *)&g_GlobalLsnForwarder) { + TrxnManagerProcLsnForwarder(item); + } else if (item == (RedoItem *)&g_cleanupMark) { + TrxnManagerProcCleanupMark(item); + } else if (item == (void *)&g_closefdMark) { + smgrcloseall(); + } else if (item == (void *)&g_cleanInvalidPageMark) { + forget_range_invalid_pages((void *)item); + } else { + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_4]); + if (IsCheckPoint(&item->record) || IsTableSpaceDrop(&item->record) || IsTableSpaceCreate(&item->record) || + (IsXactXlog(&item->record) && XactWillRemoveRelFiles(&item->record)) || IsBarrierRelated(&item->record)) { + uint32 relCount; + do { + RedoInterruptCallBack(); + relCount = pg_atomic_read_u32(&item->record.refcount); + } while (relCount != 1); + } + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4], g_redoWorker->timeCostList[TIME_COST_STEP_5]); +#ifdef ENABLE_UT + TestXLogReaderProbe(UTEST_EVENT_RTO_TRXNMGR_DISTRIBUTE_ITEMS, + __FUNCTION__, &item->record); +#endif + AddPageRedoItem(g_dispatcher->trxnLine.redoThd, item); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_5]); + } + return exitFlag; +} + +void GlobalLsnUpdate() +{ + t_thrd.xlog_cxt.standbyState = g_redoWorker->standbyState; + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + if (LsnUpdate()) { + ExtremRtoUpdateMinCheckpoint(); + CheckRecoveryConsistency(); + } + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); +} + +bool LsnUpdate() +{ + XLogRecPtr minStart = MAX_XLOG_REC_PTR; + XLogRecPtr minEnd = MAX_XLOG_REC_PTR; + GetReplayedRecPtr(&minStart, &minEnd); + if ((minEnd != MAX_XLOG_REC_PTR) && (minStart != MAX_XLOG_REC_PTR)) { + SetXLogReplayRecPtr(minStart, minEnd); + return true; + } + return false; +} + +static void TrxnMangerQueueCallBack() +{ + GlobalLsnUpdate(); + HandlePageRedoInterrupts(); +} + +void TrxnManagerMain() +{ + (void)RegisterRedoInterruptCallBack(TrxnMangerQueueCallBack); + t_thrd.xlog_cxt.max_page_flush_lsn = get_global_max_page_flush_lsn(); + ereport(LOG, + (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("TrxnManagerMain: first get_global_max_page_flush_lsn %08X/%08X", + (uint32)(t_thrd.xlog_cxt.max_page_flush_lsn >> 32), (uint32)(t_thrd.xlog_cxt.max_page_flush_lsn)))); + while (true) { + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); + if (FORCE_FINISH_ENABLED && t_thrd.xlog_cxt.max_page_flush_lsn == MAX_XLOG_REC_PTR) { + t_thrd.xlog_cxt.max_page_flush_lsn = get_global_max_page_flush_lsn(); + if (t_thrd.xlog_cxt.max_page_flush_lsn != MAX_XLOG_REC_PTR) { + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("TrxnManagerMain: second get_global_max_page_flush_lsn %08X/%08X", + (uint32)(t_thrd.xlog_cxt.max_page_flush_lsn >> 32), + (uint32)(t_thrd.xlog_cxt.max_page_flush_lsn)))); + } + } + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); + if (!SPSCBlockingQueueIsEmpty(g_redoWorker->queue)) { + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); + RedoItem *item = (RedoItem *)SPSCBlockingQueueTop(g_redoWorker->queue); + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], + g_redoWorker->timeCostList[TIME_COST_STEP_2]); + bool isEnd = TrxnManagerDistributeItemsBeforeEnd(item); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); + SPSCBlockingQueuePop(g_redoWorker->queue); + + if (isEnd) { + break; + } + } else { + long sleeptime = 80 * 1000; + pg_usleep(sleeptime); + } + + ADD_ABNORMAL_POSITION(2); + RedoInterruptCallBack(); + } + + RedoThrdWaitForExit(g_redoWorker); + GlobalLsnUpdate(); +} + +void TrxnWorkerProcLsnForwarder(RedoItem *lsnForwarder) +{ + SetCompletedReadEndPtr(g_redoWorker, lsnForwarder->record.ReadRecPtr, lsnForwarder->record.EndRecPtr); + (void)pg_atomic_sub_fetch_u32(&lsnForwarder->record.refcount, 1); +} + +void TrxnWorkNotifyRedoWorker() +{ + for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; ++i) { + if (g_dispatcher->allWorkers[i]->role == REDO_PAGE_WORKER || + g_dispatcher->allWorkers[i]->role == REDO_PAGE_MNG) { + pg_atomic_write_u32(&(g_dispatcher->allWorkers[i]->fullSyncFlag), 0); + } + } +} + +void TrxnWorkrProcCleanupMark(RedoItem *cleanupMark) +{ + g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); + ereport(LOG, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]TrxnWorkrProcCleanupMark has cleaned InvalidPages"))); +} + +bool CheckFullSyncCheckpoint(RedoItem *item) +{ + if (!IsCheckPoint(&(item->record))) { + return true; + } + + if (XLByteLE(item->record.ReadRecPtr, t_thrd.shemem_ptr_cxt.ControlFile->checkPoint)) { + return true; + } + + return false; +} + +void TrxnWorkMain() +{ +#ifdef ENABLE_MOT + MOTBeginRedoRecovery(); +#endif + (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); + if (ParseStateWithoutCache()) { + XLogRedoBufferInitFunc(&(g_redoWorker->bufferManager), MAX_LOCAL_BUFF_NUM, &recordRefOperate, + RedoInterruptCallBack); + } + + RedoItem *item = NULL; + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); + while ((item = (RedoItem *)SPSCBlockingQueueTop(g_redoWorker->queue)) != &g_redoEndMark) { + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); + if ((void *)item == (void *)&g_GlobalLsnForwarder) { + TrxnWorkerProcLsnForwarder((RedoItem *)item); + SPSCBlockingQueuePop(g_redoWorker->queue); + } else if ((void *)item == (void *)&g_cleanupMark) { + TrxnWorkrProcCleanupMark((RedoItem *)item); + SPSCBlockingQueuePop(g_redoWorker->queue); + } else if ((void *)item == (void *)&g_closefdMark) { + smgrcloseall(); + SPSCBlockingQueuePop(g_redoWorker->queue); + } else if ((void *)item == (void *)&g_cleanInvalidPageMark) { + forget_range_invalid_pages((void *)item); + SPSCBlockingQueuePop(g_redoWorker->queue); + } else { + t_thrd.xlog_cxt.needImmediateCkp = item->needImmediateCheckpoint; + bool fullSync = item->record.isFullSync; + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); + ApplySinglePageRecord(item); + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3], + g_redoWorker->timeCostList[TIME_COST_STEP_4]); + SPSCBlockingQueuePop(g_redoWorker->queue); + SetCompletedReadEndPtr(g_redoWorker, item->record.ReadRecPtr, item->record.EndRecPtr); + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4], + g_redoWorker->timeCostList[TIME_COST_STEP_5]); + if (fullSync) { + Assert(CheckFullSyncCheckpoint(item)); + TrxnWorkNotifyRedoWorker(); + } + + if (XactHasSegpageRelFiles(&item->record)) { + uint32 expected = 1; + pg_atomic_compare_exchange_u32((volatile uint32 *)&(g_dispatcher->segpageXactDoneFlag), &expected, 0); + } + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_5]); + DereferenceRedoItem(item); + RedoInterruptCallBack(); + } + ADD_ABNORMAL_POSITION(3); + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2], g_redoWorker->timeCostList[TIME_COST_STEP_1]); + } + + SPSCBlockingQueuePop(g_redoWorker->queue); + if (ParseStateWithoutCache()) + XLogRedoBufferDestoryFunc(&(g_redoWorker->bufferManager)); +#ifdef ENABLE_MOT + MOTEndRedoRecovery(); +#endif +} + +void RedoPageWorkerCheckPoint(const XLogRecParseState *redoblockstate) +{ + CheckPoint checkPoint; + Assert(IsCheckPoint(redoblockstate)); + XLogSynAllBuffer(); + Assert(redoblockstate->blockparse.extra_rec.blockxlogcommon.maindatalen >= sizeof(checkPoint)); + errno_t rc = memcpy_s(&checkPoint, sizeof(checkPoint), + redoblockstate->blockparse.extra_rec.blockxlogcommon.maindata, sizeof(checkPoint)); + securec_check(rc, "\0", "\0"); + if (IsRestartPointSafe(checkPoint.redo)) { + pg_atomic_write_u64(&g_redoWorker->lastCheckedRestartPoint, redoblockstate->blockparse.blockhead.start_ptr); + } + + UpdateTimeline(&checkPoint); + +#ifdef USE_ASSERT_CHECKING + int printLevel = WARNING; +#else + int printLevel = DEBUG1; +#endif + if (log_min_messages <= printLevel) { + GetThreadBufferLeakNum(); + } +} + +void PageWorkerProcLsnForwarder(RedoItem *lsnForwarder) +{ + SetCompletedReadEndPtr(g_redoWorker, lsnForwarder->record.ReadRecPtr, lsnForwarder->record.EndRecPtr); + (void)pg_atomic_sub_fetch_u32(&lsnForwarder->record.refcount, 1); +} + +bool XlogNeedUpdateFsm(XLogRecParseState *procState, RedoBufferInfo *bufferinfo) +{ + XLogBlockHead *blockhead = &procState->blockparse.blockhead; + if (bufferinfo->pageinfo.page == NULL || !(bufferinfo->dirtyflag) || blockhead->forknum != MAIN_FORKNUM || + XLogBlockHeadGetValidInfo(blockhead) != BLOCK_DATA_MAIN_DATA_TYPE || blockhead->bucketNode != InvalidBktId) { + return false; + } + + Size freespace = PageGetHeapFreeSpace(bufferinfo->pageinfo.page); + + RmgrId rmid = XLogBlockHeadGetRmid(blockhead); + if (rmid == RM_HEAP2_ID) { + uint8 info = XLogBlockHeadGetInfo(blockhead) & ~XLR_INFO_MASK; + if (info == XLOG_HEAP2_CLEAN) { + return true; + } else if ((info == XLOG_HEAP2_MULTI_INSERT) && (freespace < BLCKSZ / 5)) { + return true; + } + + } else if (rmid == RM_HEAP_ID) { + uint8 info = XLogBlockHeadGetInfo(blockhead) & ~XLR_INFO_MASK; + if ((info == XLOG_HEAP_INSERT || info == XLOG_HEAP_UPDATE) && (freespace < BLCKSZ / 5)) { + return true; + } + } + + return false; +} + +void RedoPageWorkerRedoBcmBlock(XLogRecParseState *procState) +{ + RmgrId rmid = XLogBlockHeadGetRmid(&procState->blockparse.blockhead); + if (rmid == RM_HEAP2_ID) { + RelFileNode node; + node.spcNode = procState->blockparse.blockhead.spcNode; + node.dbNode = procState->blockparse.blockhead.dbNode; + node.relNode = procState->blockparse.blockhead.relNode; + node.bucketNode = procState->blockparse.blockhead.bucketNode; + node.opt = procState->blockparse.blockhead.opt; + XLogBlockNewCuParse *newCuParse = &(procState->blockparse.extra_rec.blocknewcu); + uint8 info = XLogBlockHeadGetInfo(&procState->blockparse.blockhead) & ~XLR_INFO_MASK; + switch (info & XLOG_HEAP_OPMASK) { + case XLOG_HEAP2_BCM: { + xl_heap_bcm *xlrec = (xl_heap_bcm *)(newCuParse->main_data); + heap_bcm_redo(xlrec, node, procState->blockparse.blockhead.end_ptr); + break; + } + case XLOG_HEAP2_LOGICAL_NEWPAGE: { + Assert(IsHeapFileNode(node)); + xl_heap_logical_newpage *xlrec = (xl_heap_logical_newpage *)(newCuParse->main_data); + char *cuData = newCuParse->main_data + SizeOfHeapLogicalNewPage; + heap_xlog_bcm_new_page(xlrec, node, cuData); + break; + } + default: + break; + } + } +} + +void RedoPageWorkerMain() +{ + (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); + + if (ParseStateWithoutCache()) { + XLogRedoBufferInitFunc(&(g_redoWorker->bufferManager), MAX_LOCAL_BUFF_NUM, &recordRefOperate, + RedoInterruptCallBack); + } + + XLogRecParseState *redoblockstateHead = NULL; + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); + while ((redoblockstateHead = (XLogRecParseState *)SPSCBlockingQueueTop(g_redoWorker->queue)) != + (XLogRecParseState *)&g_redoEndMark) { + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); + if ((void *)redoblockstateHead == (void *)&g_cleanupMark) { + g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); + SPSCBlockingQueuePop(g_redoWorker->queue); + ereport(LOG, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]RedoPageWorkerMain has cleaned InvalidPages"))); + continue; + } + + if ((void *)redoblockstateHead == (void *)&g_closefdMark) { + smgrcloseall(); + SPSCBlockingQueuePop(g_redoWorker->queue); + continue; + } + + if ((void *)redoblockstateHead == (void *)&g_cleanInvalidPageMark) { + forget_range_invalid_pages((void *)redoblockstateHead); + SPSCBlockingQueuePop(g_redoWorker->queue); + continue; + } + if ((void *)redoblockstateHead == (void *)&g_GlobalLsnForwarder) { + PageWorkerProcLsnForwarder((RedoItem *)redoblockstateHead); + SPSCBlockingQueuePop(g_redoWorker->queue); + continue; + } + RedoBufferInfo bufferinfo = {0}; + bool notfound = false; + bool updateFsm = false; + + XLogRecParseState *procState = redoblockstateHead; + + MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); + while (procState != NULL) { + XLogRecParseState *redoblockstate = procState; + g_redoWorker->curRedoBlockState = (XLogBlockDataParse*)(&redoblockstate->blockparse.extra_rec); + procState = (XLogRecParseState *)procState->nextrecord; + + switch (XLogBlockHeadGetValidInfo(&redoblockstate->blockparse.blockhead)) { + case BLOCK_DATA_MAIN_DATA_TYPE: + case BLOCK_DATA_UNDO_TYPE: + case BLOCK_DATA_VM_TYPE: + case BLOCK_DATA_FSM_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); + notfound = XLogBlockRedoForExtremeRTO(redoblockstate, &bufferinfo, notfound, + g_redoWorker->timeCostList[TIME_COST_STEP_4], g_redoWorker->timeCostList[TIME_COST_STEP_5]); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); + break; + case BLOCK_DATA_XLOG_COMMON_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + RedoPageWorkerCheckPoint(redoblockstate); + SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, + redoblockstate->blockparse.blockhead.end_ptr); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + break; + case BLOCK_DATA_DDL_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + XLogForgetDDLRedo(redoblockstate); + SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, + redoblockstate->blockparse.blockhead.end_ptr); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + break; + case BLOCK_DATA_DROP_DATABASE_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + XLogDropDatabase(redoblockstate->blockparse.blockhead.dbNode); + SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, + redoblockstate->blockparse.blockhead.end_ptr); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + break; + case BLOCK_DATA_NEWCU_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + RedoPageWorkerRedoBcmBlock(redoblockstate); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + break; + case BLOCK_DATA_SEG_SPACE_DROP: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + XLogDropSegmentSpace(redoblockstate->blockparse.blockhead.spcNode, + redoblockstate->blockparse.blockhead.dbNode); + SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, + redoblockstate->blockparse.blockhead.end_ptr); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + break; + case BLOCK_DATA_SEG_SPACE_SHRINK: + XLogDropSpaceShrink(redoblockstate); + SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, + redoblockstate->blockparse.blockhead.end_ptr); + break; + case BLOCK_DATA_BARRIER_TYPE: + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + SetCompletedReadEndPtr(g_redoWorker, redoblockstate->blockparse.blockhead.start_ptr, + redoblockstate->blockparse.blockhead.end_ptr); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_6]); + break; + default: + break; + } + } + (void)MemoryContextSwitchTo(oldCtx); + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); + updateFsm = XlogNeedUpdateFsm(redoblockstateHead, &bufferinfo); + bool needWait = redoblockstateHead->isFullSync; + if (needWait) { + pg_atomic_write_u32(&g_redoWorker->fullSyncFlag, 1); + } + XLogBlockParseStateRelease(redoblockstateHead); + /* the same page */ + ExtremeRtoFlushBuffer(&bufferinfo, updateFsm); + SPSCBlockingQueuePop(g_redoWorker->queue); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_7]); + pg_memory_barrier(); + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); + uint32 val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); + while (val != 0) { + RedoInterruptCallBack(); + val = pg_atomic_read_u32(&g_redoWorker->fullSyncFlag); + } + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_8]); + RedoInterruptCallBack(); + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2], g_redoWorker->timeCostList[TIME_COST_STEP_1]); + ADD_ABNORMAL_POSITION(4); + } + + SPSCBlockingQueuePop(g_redoWorker->queue); + if (ParseStateWithoutCache()) + XLogRedoBufferDestoryFunc(&(g_redoWorker->bufferManager)); +} + +void PutRecordToReadQueue(XLogReaderState *recordreader) +{ + SPSCBlockingQueuePut(g_dispatcher->readLine.readPageThd->queue, recordreader); +} + +inline void InitXLogRecordReadBuffer(XLogReaderState **initreader) +{ + XLogReaderState *newxlogreader; + XLogReaderState *readstate = g_dispatcher->rtoXlogBufState.initreader; + newxlogreader = NewReaderState(readstate); + g_dispatcher->rtoXlogBufState.initreader = NULL; + PutRecordToReadQueue(readstate); + SetCompletedReadEndPtr(g_redoWorker, readstate->ReadRecPtr, readstate->EndRecPtr); + *initreader = newxlogreader; +} + +void StartupSendFowarder(RedoItem *item) +{ + for (uint32 i = 0; i < g_dispatcher->pageLineNum; ++i) { + AddPageRedoItem(g_dispatcher->pageLines[i].batchThd, item); + } + + AddPageRedoItem(g_dispatcher->trxnLine.managerThd, item); +} + +void SendLsnFowarder() +{ + // update and read in the same thread, so no need atomic operation + g_GlobalLsnForwarder.record.ReadRecPtr = g_redoWorker->lastReplayedReadRecPtr; + g_GlobalLsnForwarder.record.EndRecPtr = g_redoWorker->lastReplayedEndRecPtr; + g_GlobalLsnForwarder.record.refcount = get_real_recovery_parallelism() - XLOG_READER_NUM; + g_GlobalLsnForwarder.record.isDecode = true; + PutRecordToReadQueue(&g_GlobalLsnForwarder.record); +} + +static inline bool ReadPageWorkerStop() +{ + return g_dispatcher->recoveryStop; +} + +void PushToWorkerLsn(bool force) +{ + const uint32 max_record_count = PAGE_WORK_QUEUE_SIZE; + static uint32 cur_recor_count = 0; + + cur_recor_count++; + + if (!IsExtremeRtoRunning()) { + return; + } + + if (force) { + uint32 refCount; + do { + refCount = pg_atomic_read_u32(&g_GlobalLsnForwarder.record.refcount); + RedoInterruptCallBack(); + } while (refCount != 0 && !ReadPageWorkerStop()); + cur_recor_count = 0; + SendLsnFowarder(); + } else { + uint32 refCount = pg_atomic_read_u32(&g_GlobalLsnForwarder.record.refcount); + + if (refCount != 0 || cur_recor_count < max_record_count) { + return; + } + + SendLsnFowarder(); + cur_recor_count = 0; + } +} + +void ResetRtoXlogReadBuf(XLogRecPtr targetPagePtr) +{ + uint32 startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); + if (startreadworker == WORKER_STATE_STOP) { + WalRcvCtlAcquireExitLock(); + WalRcvCtlBlock *walrcb = getCurrentWalRcvCtlBlock(); + + if (walrcb == NULL) { + WalRcvCtlReleaseExitLock(); + return; + } + + int64 walwriteoffset; + XLogRecPtr startptr; + SpinLockAcquire(&walrcb->mutex); + walwriteoffset = walrcb->walWriteOffset; + startptr = walrcb->walStart; + SpinLockRelease(&walrcb->mutex); + + if (XLByteLT(startptr, targetPagePtr)) { + WalRcvCtlReleaseExitLock(); + return; + } + + + for (uint32 i = 0; i < MAX_ALLOC_SEGNUM; ++i) { + pg_atomic_write_u32(&(g_recordbuffer->xlogsegarray[i].bufState), NONE); + } + + XLogSegNo segno; + XLByteToSeg(targetPagePtr, segno); + g_recordbuffer->xlogsegarray[g_recordbuffer->applyindex].segno = segno; + g_recordbuffer->xlogsegarray[g_recordbuffer->applyindex].readlen = targetPagePtr % XLOG_SEG_SIZE; + + pg_atomic_write_u32(&(g_recordbuffer->readindex), g_recordbuffer->applyindex); + pg_atomic_write_u32(&(g_recordbuffer->xlogsegarray[g_recordbuffer->readindex].bufState), APPLYING); + + pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_RUN); + WalRcvCtlReleaseExitLock(); + } +} + +RecordBufferAarray *GetCurrentSegmentBuf(XLogRecPtr targetPagePtr) +{ + Assert(g_recordbuffer->applyindex < MAX_ALLOC_SEGNUM); + uint32 applyindex = g_recordbuffer->applyindex; + RecordBufferAarray *cursegbuffer = &g_recordbuffer->xlogsegarray[applyindex]; + uint32 bufState = pg_atomic_read_u32(&(cursegbuffer->bufState)); + + if (bufState != APPLYING) { + return NULL; + } + uint32 targetPageOff = (targetPagePtr % XLOG_SEG_SIZE); + XLogSegNo targetSegNo; + XLByteToSeg(targetPagePtr, targetSegNo); + if (cursegbuffer->segno == targetSegNo) { + cursegbuffer->segoffset = targetPageOff; + return cursegbuffer; + } else if (cursegbuffer->segno + 1 == targetSegNo) { + Assert(targetPageOff == 0); + pg_atomic_write_u32(&(cursegbuffer->bufState), APPLIED); + if ((applyindex + 1) == MAX_ALLOC_SEGNUM) { + applyindex = 0; + } else { + applyindex++; + } + + pg_atomic_write_u32(&(g_recordbuffer->applyindex), applyindex); + cursegbuffer = &g_recordbuffer->xlogsegarray[applyindex]; + bufState = pg_atomic_read_u32(&(cursegbuffer->bufState)); + if (bufState != APPLYING) { + return NULL; + } + + Assert(cursegbuffer->segno == targetSegNo); + cursegbuffer->segoffset = targetPageOff; + return cursegbuffer; + } else { + ereport(WARNING, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("SetReadBufferForExtRto targetPagePtr:%lu", targetPagePtr))); + DumpExtremeRtoReadBuf(); + t_thrd.xlog_cxt.failedSources |= XLOG_FROM_STREAM; + return NULL; + } +} + +static const int MAX_WAIT_TIMS = 512; + +bool XLogPageReadForExtRto(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, int reqLen) +{ + uint32 startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); + if (startreadworker == WORKER_STATE_RUN) { + RecordBufferAarray *cursegbuffer = GetCurrentSegmentBuf(targetPagePtr); + if (cursegbuffer == NULL) { + return false; + } + + uint32 readlen = pg_atomic_read_u32(&(cursegbuffer->readlen)); + + uint32 waitcount = 0; + while (readlen < (cursegbuffer->segoffset + reqLen)) { + readlen = pg_atomic_read_u32(&(cursegbuffer->readlen)); + if (waitcount >= MAX_WAIT_TIMS) { + return false; + } + waitcount++; + } + + Assert(cursegbuffer->segoffset == (targetPagePtr % XLogSegSize)); + xlogreader->readBuf = cursegbuffer->readsegbuf + cursegbuffer->segoffset; + return true; + } + + return false; +} + +void XLogReadWorkerSegFallback(XLogSegNo lastRplSegNo) +{ + errno_t errorno = EOK; + uint32 readindex = pg_atomic_read_u32(&(g_recordbuffer->readindex)); + uint32 applyindex = pg_atomic_read_u32(&(g_recordbuffer->applyindex)); + RecordBufferAarray *readseg = &g_recordbuffer->xlogsegarray[readindex]; + RecordBufferAarray *applyseg = &g_recordbuffer->xlogsegarray[applyindex]; + + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("XLogReadWorkerSegFallback: readindex: %u, readseg[%lu,%lu,%u,%u], applyindex: %u," + "applyseg[%lu,%lu,%u,%u]", + readindex, readseg->segno, readseg->segoffset, readseg->readlen, readseg->bufState, applyindex, + applyseg->segno, applyseg->segoffset, applyseg->readlen, applyseg->bufState))); + + pg_atomic_write_u32(&(g_recordbuffer->readindex), applyindex); + pg_atomic_write_u32(&(readseg->bufState), APPLIED); + applyseg->segno = lastRplSegNo; + applyseg->readlen = applyseg->segoffset; + errorno = memset_s(applyseg->readsegbuf, XLOG_SEG_SIZE, 0, XLOG_SEG_SIZE); + securec_check(errorno, "", ""); +} + +bool CloseReadFile() +{ + if (t_thrd.xlog_cxt.readFile >= 0) { + close(t_thrd.xlog_cxt.readFile); + t_thrd.xlog_cxt.readFile = -1; + return true; + } + return false; +} + +void DispatchCleanupMarkToAllRedoWorker() +{ + for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { + PageRedoWorker *worker = g_dispatcher->allWorkers[i]; + if (worker->role == REDO_PAGE_WORKER) { + SPSCBlockingQueuePut(worker->queue, &g_cleanupMark); + } + } +} + +void DispatchClosefdMarkToAllRedoWorker() +{ + for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { + PageRedoWorker *worker = g_dispatcher->allWorkers[i]; + if (worker->role == REDO_PAGE_WORKER || worker->role == REDO_PAGE_MNG || + worker->role == REDO_TRXN_MNG || worker->role == REDO_TRXN_WORKER) { + SPSCBlockingQueuePut(worker->queue, &g_closefdMark); + } + } +} + +void DispatchCleanInvalidPageMarkToAllRedoWorker(RepairFileKey key) +{ + for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { + PageRedoWorker *worker = g_dispatcher->allWorkers[i]; + if (worker->role == REDO_PAGE_WORKER) { + errno_t rc = memcpy_s((char*)&g_cleanInvalidPageMark, + sizeof(RepairFileKey), (char*)&key, sizeof(RepairFileKey)); + securec_check(rc, "", ""); + SPSCBlockingQueuePut(worker->queue, &g_cleanInvalidPageMark); + } + } +} + +void WaitAllRedoWorkerIdle() +{ + instr_time startTime; + instr_time endTime; + bool allIdle = false; + INSTR_TIME_SET_CURRENT(startTime); + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("WaitAllRedoWorkerIdle begin, startTime: %lu us", INSTR_TIME_GET_MICROSEC(startTime)))); + while (!allIdle) { + allIdle = true; + for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { + PageRedoWorker *worker = g_dispatcher->allWorkers[i]; + if (worker->role == REDO_READ_WORKER || worker->role == REDO_READ_MNG) { + continue; + } + if (!RedoWorkerIsIdle(worker)) { + allIdle = false; + break; + } + } + RedoInterruptCallBack(); + } + INSTR_TIME_SET_CURRENT(endTime); + INSTR_TIME_SUBTRACT(endTime, startTime); + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("WaitAllRedoWorkerIdle end, cost time: %lu us", INSTR_TIME_GET_MICROSEC(endTime)))); +} + +void WaitAllReplayWorkerIdle() +{ + instr_time startTime; + instr_time endTime; + bool allIdle = false; + INSTR_TIME_SET_CURRENT(startTime); + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("WaitAllReplayWorkerIdle begin, startTime: %lu us", INSTR_TIME_GET_MICROSEC(startTime)))); + while (!allIdle) { + allIdle = true; + for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { + PageRedoWorker *worker = g_dispatcher->allWorkers[i]; + if (worker->role == REDO_READ_WORKER || worker->role == REDO_READ_MNG || + worker->role == REDO_READ_PAGE_WORKER) { + continue; + } + if (!RedoWorkerIsIdle(worker)) { + allIdle = false; + break; + } + } + RedoInterruptCallBack(); + } + INSTR_TIME_SET_CURRENT(endTime); + INSTR_TIME_SUBTRACT(endTime, startTime); + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("WaitAllReplayWorkerIdle end, cost time: %lu us", INSTR_TIME_GET_MICROSEC(endTime)))); +} + + +void XLogForceFinish(XLogReaderState *xlogreader, TermFileData *term_file) +{ + bool closed = false; + uint32 termId = term_file->term; + XLogSegNo lastRplSegNo; + + pg_atomic_write_u32(&(extreme_rto::g_recordbuffer->readWorkerState), extreme_rto::WORKER_STATE_STOPPING); + while (pg_atomic_read_u32(&(extreme_rto::g_recordbuffer->readWorkerState)) != WORKER_STATE_STOP) { + RedoInterruptCallBack(); + }; + ShutdownWalRcv(); + ShutdownDataRcv(); + pg_atomic_write_u32(&(g_recordbuffer->readSource), XLOG_FROM_PG_XLOG); + + PushToWorkerLsn(true); + g_cleanupMark.record.isDecode = true; + PutRecordToReadQueue(&g_cleanupMark.record); + WaitAllRedoWorkerIdle(); + + XLogRecPtr lastRplReadLsn; + XLogRecPtr lastRplEndLsn = GetXLogReplayRecPtr(NULL, &lastRplReadLsn); + XLogRecPtr receivedUpto = GetWalRcvWriteRecPtr(NULL); + XLogRecPtr endRecPtr = xlogreader->EndRecPtr; + ereport(WARNING, (errcode(ERRCODE_LOG), errmsg("[ForceFinish]ArchiveXlogForForceFinishRedo in extremeRTO " + "lastRplReadLsn:%08X/%08X, lastRplEndLsn:%08X/%08X, receivedUpto:%08X/%08X, ReadRecPtr:%08X/%08X, " + "EndRecPtr:%08X/%08X, readOff:%u, latestValidRecord:%08X/%08X", + (uint32)(lastRplReadLsn >> 32), (uint32)lastRplReadLsn,(uint32)(lastRplEndLsn >> 32), (uint32)lastRplEndLsn, + (uint32)(receivedUpto >> 32), (uint32)receivedUpto,(uint32)(xlogreader->ReadRecPtr >> 32), + (uint32)xlogreader->ReadRecPtr, (uint32)(xlogreader->EndRecPtr >> 32), (uint32)xlogreader->EndRecPtr, + xlogreader->readOff, (uint32)(latestValidRecord >> 32), (uint32)latestValidRecord))); + DumpExtremeRtoReadBuf(); + xlogreader->readOff = INVALID_READ_OFF; + XLByteToSeg(endRecPtr, lastRplSegNo); + XLogReadWorkerSegFallback(lastRplSegNo); + + closed = CloseReadFile(); + CopyXlogForForceFinishRedo(lastRplSegNo, termId, xlogreader, endRecPtr); + RenameXlogForForceFinishRedo(lastRplSegNo, xlogreader->readPageTLI, termId); + if (closed) { + ReOpenXlog(xlogreader); + } + t_thrd.xlog_cxt.invaildPageCnt = 0; + XLogCheckInvalidPages(); + SetSwitchHistoryFile(endRecPtr, receivedUpto, termId); + t_thrd.xlog_cxt.invaildPageCnt = 0; + set_wal_rcv_write_rec_ptr(endRecPtr); + t_thrd.xlog_cxt.receivedUpto = endRecPtr; + pg_atomic_write_u32(&(g_instance.comm_cxt.localinfo_cxt.is_finish_redo), 0); + ereport(WARNING, + (errcode(ERRCODE_LOG), errmsg("[ForceFinish]ArchiveXlogForForceFinishRedo in extremeRTO is over"))); +} + +void CleanUpReadPageWorkerQueue() +{ + SPSCBlockingQueue *queue = g_dispatcher->readLine.readPageThd->queue; + uint32 state; + do { + while (!SPSCBlockingQueueIsEmpty(queue)) { + XLogReaderState *xlogreader = reinterpret_cast(SPSCBlockingQueueTake(queue)); + if (xlogreader == reinterpret_cast(&(g_redoEndMark.record)) || + xlogreader == reinterpret_cast(&(g_GlobalLsnForwarder.record)) || + xlogreader == reinterpret_cast(&(g_cleanupMark.record))) { + if (xlogreader == reinterpret_cast(&(g_GlobalLsnForwarder.record))) { + pg_atomic_write_u32(&g_GlobalLsnForwarder.record.refcount, 0); + } + continue; + } + + RedoItem *item = GetRedoItemPtr(xlogreader); + FreeRedoItem(item); + } + + RedoInterruptCallBack(); + state = pg_atomic_read_u32(&extreme_rto::g_dispatcher->rtoXlogBufState.readPageWorkerState); + } while (state != WORKER_STATE_EXIT); +} + +void ExtremeRtoStopHere() +{ + if ((get_real_recovery_parallelism() > 1) && (GetBatchCount() > 0)) { + g_dispatcher->recoveryStop = true; + CleanUpReadPageWorkerQueue(); + } +} + +static void CheckAndDoForceFinish(XLogReaderState *xlogreader) +{ + TermFileData term_file; + if (CheckForForceFinishRedoTrigger(&term_file)) { + ereport(WARNING, + (errmsg("[ForceFinish] force finish triggered in XLogReadPageWorkerMain, ReadRecPtr:%08X/%08X, " + "EndRecPtr:%08X/%08X, StandbyMode:%u, startup_processing:%u, dummyStandbyMode:%u", + (uint32)(t_thrd.xlog_cxt.ReadRecPtr >> 32), (uint32)t_thrd.xlog_cxt.ReadRecPtr, + (uint32)(t_thrd.xlog_cxt.EndRecPtr >> 32), (uint32)t_thrd.xlog_cxt.EndRecPtr, + t_thrd.xlog_cxt.StandbyMode, t_thrd.xlog_cxt.startup_processing, dummyStandbyMode))); + XLogForceFinish(xlogreader, &term_file); + } +} + +/* read xlog for parellel */ +void XLogReadPageWorkerMain() +{ + XLogReaderState *xlogreader = NULL; + + (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); + + g_recordbuffer = &g_dispatcher->rtoXlogBufState; + GetRecoveryLatch(); + /* init readstate */ + InitXLogRecordReadBuffer(&xlogreader); + + pg_atomic_write_u32(&(g_recordbuffer->readPageWorkerState), WORKER_STATE_RUN); + if (IsRecoveryDone()) { + t_thrd.xlog_cxt.readSource = XLOG_FROM_STREAM; + t_thrd.xlog_cxt.XLogReceiptSource = XLOG_FROM_STREAM; + pg_atomic_write_u32(&(g_recordbuffer->readSource), XLOG_FROM_STREAM); + } + + XLogRecord *record = XLogParallelReadNextRecord(xlogreader); + while (record != NULL) { + if (ReadPageWorkerStop()) { + break; + } + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); + XLogReaderState *newxlogreader = NewReaderState(xlogreader); + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3], g_redoWorker->timeCostList[TIME_COST_STEP_4]); + PutRecordToReadQueue(xlogreader); + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_4], g_redoWorker->timeCostList[TIME_COST_STEP_5]); + xlogreader = newxlogreader; + + g_redoWorker->lastReplayedReadRecPtr = xlogreader->ReadRecPtr; + g_redoWorker->lastReplayedEndRecPtr = xlogreader->EndRecPtr; + + if (FORCE_FINISH_ENABLED) { + CheckAndDoForceFinish(xlogreader); + } + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_5], g_redoWorker->timeCostList[TIME_COST_STEP_1]); + record = XLogParallelReadNextRecord(xlogreader); + CountAndGetRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1], g_redoWorker->timeCostList[TIME_COST_STEP_2]); + PushToWorkerLsn(false); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); + RedoInterruptCallBack(); + ADD_ABNORMAL_POSITION(8); + } + + uint32 workState = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); + while (workState == WORKER_STATE_STOPPING) { + workState = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); + } + + if (workState != WORKER_STATE_EXITING && workState != WORKER_STATE_EXIT) { + pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_EXITING); + } + + if (!ReadPageWorkerStop()) { + /* notify exit */ + PushToWorkerLsn(true); + g_redoEndMark.record = *xlogreader; + g_redoEndMark.record.isDecode = true; + PutRecordToReadQueue((XLogReaderState *)&g_redoEndMark.record); + } + + ReLeaseRecoveryLatch(); + pg_atomic_write_u32(&(g_recordbuffer->readPageWorkerState), WORKER_STATE_EXIT); +} + +void HandleReadWorkerRunInterrupts() +{ + if (t_thrd.page_redo_cxt.got_SIGHUP) { + t_thrd.page_redo_cxt.got_SIGHUP = false; + ProcessConfigFile(PGC_SIGHUP); + } + + if (t_thrd.page_redo_cxt.shutdown_requested) { + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("page worker id %u exit for request", g_redoWorker->id))); + + pg_atomic_write_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[g_redoWorker->id].threadState), + PAGE_REDO_WORKER_EXIT); + + proc_exit(1); + } +} + +static void InitReadBuf(uint32 bufIndex, XLogSegNo segno) +{ + if (bufIndex == MAX_ALLOC_SEGNUM) { + bufIndex = 0; + } + const uint32 sleepTime = 50; /* 50 us */ + RecordBufferAarray *nextreadseg = &g_recordbuffer->xlogsegarray[bufIndex]; + pg_memory_barrier(); + + uint32 bufState = pg_atomic_read_u32(&(nextreadseg->bufState)); + uint32 startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); + while (bufState == APPLYING && startreadworker == WORKER_STATE_RUN) { + pg_usleep(sleepTime); + RedoInterruptCallBack(); + bufState = pg_atomic_read_u32(&(nextreadseg->bufState)); + startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); + } + + nextreadseg->readlen = 0; + nextreadseg->segno = segno; + nextreadseg->segoffset = 0; + pg_atomic_write_u32(&(nextreadseg->bufState), APPLYING); + pg_atomic_write_u32(&(g_recordbuffer->readindex), bufIndex); +} + +static void XLogReadWorkRun() +{ + static uint32 waitcount = 0; + const uint32 sleepTime = 100; /* 50 us */ + XLogSegNo targetSegNo; + uint32 writeoffset; + uint32 reqlen; + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); + uint32 readindex = pg_atomic_read_u32(&(g_recordbuffer->readindex)); + Assert(readindex < MAX_ALLOC_SEGNUM); + pg_memory_barrier(); + RecordBufferAarray *readseg = &g_recordbuffer->xlogsegarray[readindex]; + + XLogRecPtr receivedUpto = GetWalRcvWriteRecPtr(NULL); + XLByteToSeg(receivedUpto, targetSegNo); + + if (targetSegNo < readseg->segno) { + pg_usleep(sleepTime); + return; + } + + writeoffset = readseg->readlen; + if (targetSegNo != readseg->segno) { + reqlen = XLOG_SEG_SIZE - writeoffset; + } else { + uint32 targetPageOff = receivedUpto % XLOG_SEG_SIZE; + if (targetPageOff <= writeoffset) { + pg_usleep(sleepTime); + return; + } + reqlen = targetPageOff - writeoffset; + if (reqlen < XLOG_BLCKSZ) { + waitcount++; + uint32 flag = pg_atomic_read_u32(&g_readManagerTriggerFlag); + if (waitcount < MAX_WAIT_TIMS && flag == TRIGGER_NORMAL) { + pg_usleep(sleepTime); + return; + } + } + } + + waitcount = 0; + char *readBuf = readseg->readsegbuf + writeoffset; + XLogRecPtr targetSartPtr = readseg->segno * XLOG_SEG_SIZE + writeoffset; + uint32 readlen = 0; + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); + bool result = XLogReadFromWriteBuffer(targetSartPtr, reqlen, readBuf, &readlen); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_2]); + if (!result) { + return; + } + + pg_atomic_write_u32(&(readseg->readlen), (writeoffset + readlen)); + if (readseg->readlen == XLOG_SEG_SIZE) { + GetRedoStartTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); + InitReadBuf(readindex + 1, readseg->segno + 1); + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_3]); + } + + CountRedoTime(g_redoWorker->timeCostList[TIME_COST_STEP_1]); +} + +void XLogReadManagerResponseSignal(uint32 tgigger) +{ + switch (tgigger) { + case TRIGGER_PRIMARY: + break; + case TRIGGER_FAILOVER: + if (t_thrd.xlog_cxt.is_cascade_standby) { + SendPostmasterSignal(PMSIGNAL_UPDATE_PROMOTING); + t_thrd.xlog_cxt.is_cascade_standby = false; + if (t_thrd.postmaster_cxt.HaShmData->is_cross_region) { + t_thrd.xlog_cxt.is_hadr_main_standby = true; + SpinLockAcquire(&t_thrd.postmaster_cxt.HaShmData->mutex); + t_thrd.postmaster_cxt.HaShmData->is_hadr_main_standby = true; + SpinLockRelease(&t_thrd.postmaster_cxt.HaShmData->mutex); + } + t_thrd.xlog_cxt.failover_triggered = false; + SendNotifySignal(NOTIFY_STANDBY, g_instance.pid_cxt.StartupPID); + SendPostmasterSignal(PMSIGNAL_UPDATE_NORMAL); + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("failover standby ready, notify postmaster to change state."))); + break; + } + t_thrd.xlog_cxt.failover_triggered = true; + SendPostmasterSignal(PMSIGNAL_UPDATE_PROMOTING); + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("failover ready, notify postmaster to change state."))); + break; + case TRIGGER_SWITCHOVER: + t_thrd.xlog_cxt.switchover_triggered = true; + SendPostmasterSignal(PMSIGNAL_UPDATE_PROMOTING); + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("switchover ready, notify postmaster to change state."))); + break; + default: + break; + } +} + +void XLogReadManagerProcInterrupt() +{ + if (t_thrd.page_redo_cxt.shutdown_requested) { + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("page worker id %u exit for request", g_redoWorker->id))); + + pg_atomic_write_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[g_redoWorker->id].threadState), + PAGE_REDO_WORKER_EXIT); + + proc_exit(1); + } + + if (t_thrd.page_redo_cxt.got_SIGHUP) { + t_thrd.page_redo_cxt.got_SIGHUP = false; + ProcessConfigFile(PGC_SIGHUP); + } +} + +void WaitPageReadWorkerExit() +{ + uint32 state; + do { + state = pg_atomic_read_u32(&extreme_rto::g_dispatcher->rtoXlogBufState.readPageWorkerState); + RedoInterruptCallBack(); + } while (state != WORKER_STATE_EXIT); +} + +static void HandleExtremeRtoCascadeStandbyPromote(uint32 trigger) +{ + if (!t_thrd.xlog_cxt.is_cascade_standby || t_thrd.xlog_cxt.server_mode != STANDBY_MODE || + !IS_DN_MULTI_STANDYS_MODE()) { + return; + } + + ShutdownWalRcv(); + pg_atomic_write_u32(&g_dispatcher->rtoXlogBufState.waitRedoDone, 1); + WakeupRecovery(); + XLogReadManagerResponseSignal(trigger); + pg_atomic_write_u32(&(extreme_rto::g_startupTriggerState), TRIGGER_NORMAL); +} + +bool XLogReadManagerCheckSignal() +{ + uint32 trigger = pg_atomic_read_u32(&(extreme_rto::g_startupTriggerState)); + load_server_mode(); + if (g_dispatcher->smartShutdown || trigger == TRIGGER_PRIMARY || trigger == TRIGGER_SWITCHOVER || + (trigger == TRIGGER_FAILOVER && t_thrd.xlog_cxt.server_mode == STANDBY_MODE) || + t_thrd.xlog_cxt.server_mode == PRIMARY_MODE) { + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("XLogReadManagerCheckSignal: smartShutdown:%u, trigger:%u, server_mode:%u", + g_dispatcher->smartShutdown, trigger, t_thrd.xlog_cxt.server_mode))); + if (t_thrd.xlog_cxt.is_cascade_standby && t_thrd.xlog_cxt.server_mode == STANDBY_MODE && + IS_DN_MULTI_STANDYS_MODE() && (trigger == TRIGGER_SWITCHOVER || trigger == TRIGGER_FAILOVER)) { + HandleExtremeRtoCascadeStandbyPromote(trigger); + return false; + } + ShutdownWalRcv(); + if (g_dispatcher->smartShutdown) { + pg_atomic_write_u32(&g_readManagerTriggerFlag, TRIGGER_SMARTSHUTDOWN); + } else { + pg_atomic_write_u32(&g_readManagerTriggerFlag, trigger); + } + WakeupRecovery(); + WaitPageReadWorkerExit(); + XLogReadManagerResponseSignal(trigger); + return true; + } + return false; +} + +void StartRequestXLogFromStream() +{ + volatile WalRcvData *walrcv = t_thrd.walreceiverfuncs_cxt.WalRcv; + XLogRecPtr expectLsn = pg_atomic_read_u64(&g_dispatcher->rtoXlogBufState.expectLsn); + if (walrcv->receivedUpto == InvalidXLogRecPtr || + (expectLsn != InvalidXLogRecPtr && XLByteLE(walrcv->receivedUpto, expectLsn))) { + uint32 readWorkerstate = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); + if (readWorkerstate == WORKER_STATE_RUN) { + pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_STOPPING); + } + SpinLockAcquire(&walrcv->mutex); + walrcv->receivedUpto = 0; + SpinLockRelease(&walrcv->mutex); + XLogRecPtr targetRecPtr = pg_atomic_read_u64(&g_dispatcher->rtoXlogBufState.targetRecPtr); + CheckMaxPageFlushLSN(targetRecPtr); + + uint32 shiftSize = 32; + if (IS_OBS_DISASTER_RECOVER_MODE && !IsRoachRestore()) { + ereport(LOG, (errmsg("request xlog stream from obs at %X/%X.", (uint32)(targetRecPtr >> shiftSize), + (uint32)targetRecPtr))); + RequestXLogStreaming(&targetRecPtr, 0, REPCONNTARGET_OBS, 0); + } else if (IS_SHARED_STORAGE_STANBY_MODE && !IS_SHARED_STORAGE_MAIN_STANDBY_MODE) { +#ifndef ENABLE_MULTIPLE_NODES + rename_recovery_conf_for_roach(); +#endif + ereport(LOG, (errmsg("request xlog stream from shared storage at %X/%X.", + (uint32)(targetRecPtr >> shiftSize), + (uint32)targetRecPtr))); + RequestXLogStreaming(&targetRecPtr, 0, REPCONNTARGET_SHARED_STORAGE, 0); + } else { +#ifndef ENABLE_MULTIPLE_NODES + rename_recovery_conf_for_roach(); +#endif + ereport(LOG, (errmsg("request xlog stream at %X/%X.", (uint32)(targetRecPtr >> shiftSize), + (uint32)targetRecPtr))); + RequestXLogStreaming(&targetRecPtr, t_thrd.xlog_cxt.PrimaryConnInfo, REPCONNTARGET_PRIMARY, + u_sess->attr.attr_storage.PrimarySlotName); + } + } +} + + +void XLogReadManagerMain() +{ + const long sleepShortTime = 100000L; + const long sleepLongTime = 1000000L; + g_recordbuffer = &g_dispatcher->rtoXlogBufState; + uint32 xlogReadManagerState = READ_MANAGER_RUN; + + (void)RegisterRedoInterruptCallBack(XLogReadManagerProcInterrupt); + + while (xlogReadManagerState == READ_MANAGER_RUN) { + RedoInterruptCallBack(); + XLogRecPtr replay = InvalidXLogRecPtr; + bool exitStatus = XLogReadManagerCheckSignal(); + if (exitStatus) { + break; + } + + replay = GetXLogReplayRecPtr(NULL, NULL); + handleRecoverySusPend(replay); + + xlogReadManagerState = pg_atomic_read_u32(&g_dispatcher->rtoXlogBufState.xlogReadManagerState); + ADD_ABNORMAL_POSITION(7); + if (t_thrd.xlog_cxt.server_mode == STANDBY_MODE) { + uint32 readSource = pg_atomic_read_u32(&g_dispatcher->rtoXlogBufState.readSource); + uint32 failSource = pg_atomic_read_u32(&g_dispatcher->rtoXlogBufState.failSource); + if (readSource & XLOG_FROM_STREAM) { + uint32 disableConnectionNode = + pg_atomic_read_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node); + bool retryConnect = ((!disableConnectionNode) || (IS_SHARED_STORAGE_MODE && disableConnectionNode && + !knl_g_get_redo_finish_status() && + !pg_atomic_read_u32(&t_thrd.walreceiverfuncs_cxt.WalRcv->rcvDoneFromShareStorage))); + if (!WalRcvInProgress() && g_instance.pid_cxt.WalReceiverPID == 0 && retryConnect) { + StartRequestXLogFromStream(); + } else { + if (disableConnectionNode) { + if (IS_SHARED_STORAGE_MODE && WalRcvIsRunning()) { + ShutdownWalRcv(); + } + + if (!WalRcvInProgress() && !knl_g_get_redo_finish_status()) { + pg_atomic_write_u32(&g_dispatcher->rtoXlogBufState.waitRedoDone, 1); + WakeupRecovery(); + pg_usleep(sleepLongTime); + } else if (knl_g_get_redo_finish_status()) { + pg_atomic_write_u32(&g_instance.comm_cxt.localinfo_cxt.need_disable_connection_node, false); + pg_usleep(sleepLongTime); + } + + } + } + } + + if (failSource & XLOG_FROM_STREAM) { + ShutdownWalRcv(); + pg_atomic_write_u32(&(extreme_rto::g_dispatcher->rtoXlogBufState.failSource), 0); + } + } + pg_usleep(sleepShortTime); + } +} + +static void ReadWorkerStopCallBack(int code, Datum arg) +{ + pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_EXIT); + if (t_thrd.xlog_cxt.readFile >= 0) { + close(t_thrd.xlog_cxt.readFile); + t_thrd.xlog_cxt.readFile = -1; + } +} + +void XLogReadWorkerMain() +{ + uint32 startreadworker; + const uint32 sleepTime = 50; /* 50 us */ + + on_shmem_exit(ReadWorkerStopCallBack, 0); + (void)RegisterRedoInterruptCallBack(HandleReadWorkerRunInterrupts); + + g_recordbuffer = &g_dispatcher->rtoXlogBufState; + startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); + while (startreadworker != WORKER_STATE_EXITING) { + if (startreadworker == WORKER_STATE_RUN) { + XLogReadWorkRun(); + } else { + pg_usleep(sleepTime); + } + + RedoInterruptCallBack(); + startreadworker = pg_atomic_read_u32(&(g_recordbuffer->readWorkerState)); + if (startreadworker == WORKER_STATE_STOPPING) { + pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_STOP); + } + ADD_ABNORMAL_POSITION(6); + }; + /* notify manger to exit */ + pg_atomic_write_u32(&(g_recordbuffer->readWorkerState), WORKER_STATE_EXIT); +} + +int RedoMainLoop() +{ + g_redoWorker->oldCtx = MemoryContextSwitchTo(g_instance.comm_cxt.predo_cxt.parallelRedoCtx); + + instr_time startTime; + instr_time endTime; + + (void)RegisterRedoPageRepairCallBack(HandlePageRedoPageRepair); + + INSTR_TIME_SET_CURRENT(startTime); + switch (g_redoWorker->role) { + case REDO_BATCH: + BatchRedoMain(); + break; + case REDO_PAGE_MNG: + RedoPageManagerMain(); + break; + case REDO_PAGE_WORKER: + RedoPageWorkerMain(); + break; + case REDO_TRXN_MNG: + TrxnManagerMain(); + break; + case REDO_TRXN_WORKER: + TrxnWorkMain(); + break; + case REDO_READ_WORKER: + XLogReadWorkerMain(); + break; + case REDO_READ_PAGE_WORKER: + XLogReadPageWorkerMain(); + break; + case REDO_READ_MNG: + XLogReadManagerMain(); + break; + default: + break; + } + + INSTR_TIME_SET_CURRENT(endTime); + INSTR_TIME_SUBTRACT(endTime, startTime); + + /* + * We need to get the exit code here before we allow the dispatcher + * to proceed and change the exit code. + */ + int exitCode = GetDispatcherExitCode(); + g_redoWorker->xlogInvalidPages = XLogGetInvalidPages(); + g_redoWorker->committingCsnList = XLogReleaseAndGetCommittingCsnList(); + + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("worker[%d]: exitcode = %d, total elapsed = %ld", g_redoWorker->id, exitCode, + INSTR_TIME_GET_MICROSEC(endTime)))); + + (void)MemoryContextSwitchTo(g_redoWorker->oldCtx); + + return exitCode; +} + +void ParallelRedoThreadRegister() +{ + bool isWorkerStarting = false; + SpinLockAcquire(&(g_instance.comm_cxt.predo_cxt.rwlock)); + isWorkerStarting = ((g_instance.comm_cxt.predo_cxt.state == REDO_STARTING_BEGIN) ? true : false); + if (isWorkerStarting) { + pg_atomic_write_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[g_redoWorker->id].threadState), + PAGE_REDO_WORKER_READY); + } + SpinLockRelease(&(g_instance.comm_cxt.predo_cxt.rwlock)); + if (!isWorkerStarting) { + ereport(LOG, (errmsg("ParallelRedoThreadRegister Page-redo-worker %u exit.", (uint32)isWorkerStarting))); + SetPageWorkStateByThreadId(PAGE_REDO_WORKER_EXIT); + proc_exit(0); + } +} + +const char *RedoWokerRole2Str(RedoRole role) +{ + switch (role) { + case REDO_BATCH: + return "redo_batch"; + break; + case REDO_PAGE_MNG: + return "redo_manager"; + break; + case REDO_PAGE_WORKER: + return "redo_woker"; + break; + case REDO_TRXN_MNG: + return "trxn_manager"; + break; + case REDO_TRXN_WORKER: + return "trxn_worker"; + break; + case REDO_READ_WORKER: + return "read_worker"; + break; + case REDO_READ_PAGE_WORKER: + return "read_page_woker"; + break; + case REDO_READ_MNG: + return "read_manager"; + break; + default: + return "unkown"; + break; + } +} + +void WaitStateNormal() +{ + do { + RedoInterruptCallBack(); + } while (g_instance.comm_cxt.predo_cxt.state < REDO_IN_PROGRESS); +} + +/* Run from the worker thread. */ +void ParallelRedoThreadMain() +{ + ParallelRedoThreadRegister(); + ereport(LOG, (errmsg("Page-redo-worker thread %u started, role:%u, slotId:%u.", g_redoWorker->id, + g_redoWorker->role, g_redoWorker->slotId))); + // regitster default interrupt call back + (void)RegisterRedoInterruptCallBack(HandlePageRedoInterrupts); + SetupSignalHandlers(); + InitGlobals(); + + ResourceManagerStartup(); + WaitStateNormal(); + EnableSyncRequestForwarding(); + + int retCode = RedoMainLoop(); + ResourceManagerStop(); + ereport(LOG, (errmsg("Page-redo-worker thread %u terminated, role:%u, slotId:%u, retcode %u.", g_redoWorker->id, + g_redoWorker->role, g_redoWorker->slotId, retCode))); + LastMarkReached(); + + pg_atomic_write_u32(&(g_instance.comm_cxt.predo_cxt.pageRedoThreadStatusList[g_redoWorker->id].threadState), + PAGE_REDO_WORKER_EXIT); + proc_exit(0); +} + +static void PageRedoShutdownHandler(SIGNAL_ARGS) +{ + t_thrd.page_redo_cxt.shutdown_requested = 1; +} + +static void PageRedoQuickDie(SIGNAL_ARGS) +{ + int status = 2; + gs_signal_setmask(&t_thrd.libpq_cxt.BlockSig, NULL); + on_exit_reset(); + exit(status); +} + +static void PageRedoUser1Handler(SIGNAL_ARGS) +{ + t_thrd.page_redo_cxt.check_repair = true; +} + +static void PageRedoUser2Handler(SIGNAL_ARGS) +{ + t_thrd.page_redo_cxt.sleep_long = 1; +} + +/* Run from the worker thread. */ +static void SetupSignalHandlers() +{ + (void)gspqsignal(SIGHUP, SigHupHandler); + (void)gspqsignal(SIGINT, SIG_IGN); + (void)gspqsignal(SIGTERM, PageRedoShutdownHandler); + (void)gspqsignal(SIGQUIT, PageRedoQuickDie); + (void)gspqsignal(SIGALRM, SIG_IGN); + (void)gspqsignal(SIGPIPE, SIG_IGN); + (void)gspqsignal(SIGUSR1, PageRedoUser1Handler); + (void)gspqsignal(SIGUSR2, PageRedoUser2Handler); + (void)gspqsignal(SIGCHLD, SIG_IGN); + (void)gspqsignal(SIGTTIN, SIG_IGN); + (void)gspqsignal(SIGTTOU, SIG_IGN); + (void)gspqsignal(SIGCONT, SIG_IGN); + (void)gspqsignal(SIGWINCH, SIG_IGN); + + gs_signal_setmask(&t_thrd.libpq_cxt.UnBlockSig, NULL); + (void)gs_signal_unblock_sigusr2(); +} + +/* Run from the worker thread. */ +static void SigHupHandler(SIGNAL_ARGS) +{ + t_thrd.page_redo_cxt.got_SIGHUP = true; +} + +/* Run from the worker thread. */ +static void InitGlobals() +{ + t_thrd.utils_cxt.CurrentResourceOwner = ResourceOwnerCreate(NULL, "ExtremeRtoParallelRedoThread", + THREAD_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_STORAGE)); + + t_thrd.xlog_cxt.server_mode = g_redoWorker->initialServerMode; + t_thrd.xlog_cxt.ThisTimeLineID = g_redoWorker->initialTimeLineID; + t_thrd.xlog_cxt.expectedTLIs = g_redoWorker->expectedTLIs; + /* apply recoveryinfo will change standbystate see UpdateRecordGlobals */ + t_thrd.xlog_cxt.standbyState = g_redoWorker->standbyState; + t_thrd.xlog_cxt.StandbyMode = g_redoWorker->StandbyMode; + t_thrd.xlog_cxt.InRecovery = true; + t_thrd.xlog_cxt.startup_processing = true; + t_thrd.proc_cxt.DataDir = g_redoWorker->DataDir; + u_sess->utils_cxt.RecentXmin = g_redoWorker->RecentXmin; + g_redoWorker->proc = t_thrd.proc; + t_thrd.storage_cxt.latestObservedXid = g_redoWorker->latestObservedXid; + t_thrd.xlog_cxt.recoveryTargetTLI = g_redoWorker->recoveryTargetTLI; + t_thrd.xlog_cxt.recoveryRestoreCommand= g_redoWorker->recoveryRestoreCommand; + t_thrd.xlog_cxt.ArchiveRecoveryRequested = g_redoWorker->ArchiveRecoveryRequested; + t_thrd.xlog_cxt.StandbyModeRequested = g_redoWorker->StandbyModeRequested; + t_thrd.xlog_cxt.InArchiveRecovery = g_redoWorker->InArchiveRecovery; + t_thrd.xlog_cxt.InRecovery = g_redoWorker->InRecovery; + t_thrd.xlog_cxt.ArchiveRestoreRequested = g_redoWorker->ArchiveRestoreRequested; + t_thrd.xlog_cxt.minRecoveryPoint = g_redoWorker->minRecoveryPoint; +} + +void WaitRedoWorkersQueueEmpty() +{ + bool queueIsEmpty = false; + while (!queueIsEmpty) { + queueIsEmpty = true; + for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { + PageRedoWorker *worker = g_dispatcher->allWorkers[i]; + if (worker->role == REDO_TRXN_WORKER || worker->role == REDO_PAGE_WORKER) { + if (!RedoWorkerIsIdle(worker)) { + queueIsEmpty = false; + break; + } + } + } + RedoInterruptCallBack(); + } +} + +void RedoThrdWaitForExit(const PageRedoWorker *wk) +{ + uint32 sd = wk->slotId; + switch (wk->role) { + case REDO_BATCH: + SendPageRedoEndMark(g_dispatcher->pageLines[sd].managerThd); + WaitPageRedoWorkerReachLastMark(g_dispatcher->pageLines[sd].managerThd); + break; + case REDO_PAGE_MNG: + DispatchEndMarkToRedoWorkerAndWait(); + break; + case REDO_PAGE_WORKER: + break; /* Don't need to wait for anyone */ + case REDO_TRXN_MNG: + SendPageRedoEndMark(g_dispatcher->trxnLine.redoThd); + WaitRedoWorkersQueueEmpty(); + WaitPageRedoWorkerReachLastMark(g_dispatcher->trxnLine.redoThd); + break; + case REDO_TRXN_WORKER: + break; /* Don't need to wait for anyone */ + default: + break; + } +} + +/* Run from the worker thread. */ +static void ApplySinglePageRecord(RedoItem *item) +{ + XLogReaderState *record = &item->record; + + MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); + ApplyRedoRecord(record); + (void)MemoryContextSwitchTo(oldCtx); +} + +/* Run from the worker thread. */ +static void LastMarkReached() +{ + PosixSemaphorePost(&g_redoWorker->phaseMarker); +} + +/* Run from the dispatcher thread. */ +void WaitPageRedoWorkerReachLastMark(PageRedoWorker *worker) +{ + PosixSemaphoreWait(&worker->phaseMarker); +} + +/* Run from the dispatcher thread. */ +void AddPageRedoItem(PageRedoWorker *worker, void *item) +{ + SPSCBlockingQueuePut(worker->queue, item); +} + +/* Run from the dispatcher thread. */ +bool SendPageRedoEndMark(PageRedoWorker *worker) +{ + return SPSCBlockingQueuePut(worker->queue, &g_redoEndMark); +} + +/* Run from the dispatcher thread. */ +bool SendPageRedoWorkerTerminateMark(PageRedoWorker *worker) +{ + return SPSCBlockingQueuePut(worker->queue, &g_terminateMark); +} + +/* Run from the txn worker thread. */ +void UpdatePageRedoWorkerStandbyState(PageRedoWorker *worker, HotStandbyState newState) +{ + /* + * Here we only save the new state into the worker struct. + * The actual update of the worker thread's state occurs inside + * the apply loop. + */ + worker->standbyState = newState; +} + +/* Run from the dispatcher thread. */ +void *GetXLogInvalidPages(PageRedoWorker *worker) +{ + return worker->xlogInvalidPages; +} + +bool RedoWorkerIsIdle(PageRedoWorker *worker) +{ + return SPSCBlockingQueueIsEmpty(worker->queue); +} + +void DumpPageRedoWorker(PageRedoWorker *worker) +{ + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("[REDO_LOG_TRACE]RedoWorker common info: id %u, tid %lu, " + "lastCheckedRestartPoint %lu, lastReplayedEndRecPtr %lu standbyState %u", + worker->id, worker->tid.thid, worker->lastCheckedRestartPoint, worker->lastReplayedEndRecPtr, + (uint32)worker->standbyState))); + DumpQueue(worker->queue); +} + +void DumpExtremeRtoReadBuf() +{ + if (g_dispatcher == NULL) { + return; + } + + ereport(LOG, + (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("DumpExtremeRtoReadBuf: startworker %u, readindex %u, applyindex %u, readSource %u, failSource %u", + g_dispatcher->rtoXlogBufState.readWorkerState, g_dispatcher->rtoXlogBufState.readindex, + g_dispatcher->rtoXlogBufState.applyindex, g_dispatcher->rtoXlogBufState.readSource, + g_dispatcher->rtoXlogBufState.failSource))); + + for (uint32 i = 0; i < MAX_ALLOC_SEGNUM; ++i) { + ereport(LOG, (errmodule(MOD_REDO), errcode(ERRCODE_LOG), + errmsg("DumpExtremeRtoReadBuf: buf %u, state %u, readlen %u, segno %lu, segoffset %lu", i, + g_dispatcher->rtoXlogBufState.xlogsegarray[i].bufState, + g_dispatcher->rtoXlogBufState.xlogsegarray[i].readlen, + g_dispatcher->rtoXlogBufState.xlogsegarray[i].segno, + g_dispatcher->rtoXlogBufState.xlogsegarray[i].segoffset))); + } +} + +bool XactHasSegpageRelFiles(XLogReaderState *record) +{ + int nrels = 0; + ColFileNodeRel *xnodes = NULL; + + if (XLogRecGetRmid(record) != RM_XACT_ID) { + return false; + } + + XactGetRelFiles(record, &xnodes, &nrels); + + for (int32 idx = 0; idx < nrels; idx++) { + ColFileNode colFileNode; + ColFileNodeRel *colFileNodeRel = xnodes + idx; + + ColFileNodeCopy(&colFileNode, colFileNodeRel); + + if (!IsValidColForkNum(colFileNode.forknum) && IsSegmentFileNode(colFileNode.filenode)) { + return true; + } + } + + return false; +} + + +void RepairPageAndRecoveryXLog(BadBlockRecEnt* page_info, const char *page) +{ + RedoBufferInfo buffer; + RedoBufferTag blockinfo; + bool updateFsm = false; + bool notfound = false; + errno_t rc; + BufferDesc *bufDesc = NULL; + RedoTimeCost timeCost1; + RedoTimeCost timeCost2; + + blockinfo.rnode = page_info->key.relfilenode; + blockinfo.forknum = page_info->key.forknum; + blockinfo.blkno = page_info->key.blocknum; + blockinfo.pblk = page_info->pblk; + + /* read page to buffer pool by RBM_ZERO_AND_LOCK mode and get buffer lock */ + (void)XLogReadBufferForRedoBlockExtend(&blockinfo, RBM_ZERO_AND_LOCK, false, &buffer, + page_info->rec_max_lsn, InvalidXLogRecPtr, false, WITH_NORMAL_CACHE); + + rc = memcpy_s(buffer.pageinfo.page, BLCKSZ, page, BLCKSZ); + securec_check(rc, "", ""); + + MarkBufferDirty(buffer.buf); + bufDesc = GetBufferDescriptor(buffer.buf - 1); + bufDesc->lsn_on_disk = PageGetLSN(buffer.pageinfo.page); + UnlockReleaseBuffer(buffer.buf); + + /* recovery the page xlog */ + rc = memset_s(&buffer, sizeof(RedoBufferInfo), 0, sizeof(RedoBufferInfo)); + securec_check(rc, "", ""); + + XLogRecParseState *procState = page_info->head; + MemoryContext oldCtx = MemoryContextSwitchTo(g_redoWorker->oldCtx); + while (procState != NULL) { + XLogRecParseState *redoblockstate = procState; + procState = (XLogRecParseState *)procState->nextrecord; + (void)XLogBlockRedoForExtremeRTO(redoblockstate, &buffer, notfound, timeCost1, timeCost2); + } + (void)MemoryContextSwitchTo(oldCtx); + updateFsm = XlogNeedUpdateFsm(page_info->head, &buffer); + XLogBlockParseStateRelease(page_info->head); + ExtremeRtoFlushBuffer(&buffer, updateFsm); +} + +HTAB* BadBlockHashTblCreate() +{ + HASHCTL ctl; + errno_t rc; + + rc = memset_s(&ctl, sizeof(ctl), 0, sizeof(ctl)); + securec_check(rc, "", ""); + + ctl.keysize = sizeof(RepairBlockKey); + ctl.entrysize = sizeof(BadBlockRecEnt); + ctl.hash = tag_hash; + + return hash_create("recovery thread bad block hashtbl", MAX_REMOTE_READ_INFO_NUM, &ctl, HASH_ELEM | HASH_FUNCTION); +} + + +/* ClearPageRepairHashTbl + * drop table, or truncate table, need clear the page repair hashTbl, if the + * repair page Filenode match need remove. + */ +void ClearRecoveryThreadHashTbl(const RelFileNode &node, ForkNumber forknum, BlockNumber minblkno, + bool segment_shrink) +{ + HTAB *bad_hash = g_redoWorker->badPageHashTbl; + bool found = false; + BadBlockRecEnt *entry = NULL; + HASH_SEQ_STATUS status; + + hash_seq_init(&status, bad_hash); + while ((entry = (BadBlockRecEnt *)hash_seq_search(&status)) != NULL) { + if (BlockNodeMatch(entry->key, entry->pblk, node, forknum, minblkno, segment_shrink)) { + if (hash_search(bad_hash, &(entry->key), HASH_REMOVE, &found) == NULL) { + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("recovery thread bad page hash table corrupted"))); + } + g_redoWorker->remoteReadPageNum--; + } + } + + return; +} + +/* BatchClearPageRepairHashTbl + * drop database, or drop segmentspace, need clear the page repair hashTbl, + * if the repair page key dbNode match and spcNode match, need remove. + */ +void BatchClearRecoveryThreadHashTbl(Oid spcNode, Oid dbNode) +{ + HTAB *bad_hash = g_redoWorker->badPageHashTbl; + bool found = false; + BadBlockRecEnt *entry = NULL; + HASH_SEQ_STATUS status; + + hash_seq_init(&status, bad_hash); + while ((entry = (BadBlockRecEnt *)hash_seq_search(&status)) != NULL) { + if (dbNodeandSpcNodeMatch(&(entry->key.relfilenode), spcNode, dbNode)) { + if (hash_search(bad_hash, &(entry->key), HASH_REMOVE, &found) == NULL) { + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("page repair hash table corrupted"))); + } + g_redoWorker->remoteReadPageNum--; + } + } + + return; +} + + +/* ClearSpecificsPageEntryAndMem + * If the page has been repair, need remove entry of bad page hashtable, + * and release the xlog record mem. + */ +void ClearSpecificsPageEntryAndMem(BadBlockRecEnt *entry) +{ + HTAB *bad_hash = g_redoWorker->badPageHashTbl; + bool found = false; + uint32 need_repair_num = 0; + HASH_SEQ_STATUS status; + BadBlockRecEnt *temp_entry = NULL; + + if ((BadBlockRecEnt*)hash_search(bad_hash, &(entry->key), HASH_REMOVE, &found) == NULL) { + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("recovery thread bad block hash table corrupted"))); + } + + hash_seq_init(&status, bad_hash); + while ((temp_entry = (BadBlockRecEnt *)hash_seq_search(&status)) != NULL) { + need_repair_num++; + } + + if (need_repair_num == 0) { + XLogParseBufferDestoryFunc(&(g_redoWorker->parseManager)); + } +} + +/* RecordBadBlockAndPushToRemote + * If the bad page has been stored, record the xlog. If the bad page + * has not been stored, need push to page repair thread hash table and record to + * recovery thread hash table. + */ +void RecordBadBlockAndPushToRemote(XLogBlockDataParse *datadecode, PageErrorType error_type, + XLogRecPtr old_lsn, XLogPhyBlock pblk) +{ + bool found = false; + RepairBlockKey key; + gs_thread_t tid; + XLogBlockParse *block = STRUCT_CONTAINER(XLogBlockParse, extra_rec, datadecode); + XLogRecParseState *state = STRUCT_CONTAINER(XLogRecParseState, blockparse, block); + + key.relfilenode.spcNode = state->blockparse.blockhead.spcNode; + key.relfilenode.dbNode = state->blockparse.blockhead.dbNode; + key.relfilenode.relNode = state->blockparse.blockhead.relNode; + key.relfilenode.bucketNode = state->blockparse.blockhead.bucketNode; + key.forknum = state->blockparse.blockhead.forknum; + key.blocknum = state->blockparse.blockhead.blkno; + + tid = gs_thread_get_cur_thread(); + found = PushBadPageToRemoteHashTbl(key, error_type, old_lsn, pblk, tid.thid); + + if (found) { + /* store the record for recovery */ + HTAB *bad_hash = g_redoWorker->badPageHashTbl; + bool thread_found = false; + BadBlockRecEnt *remoteReadInfo = (BadBlockRecEnt*)hash_search(bad_hash, &(key), HASH_FIND, &thread_found); + Assert(thread_found); + if (!thread_found) { + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("recovery thread bad block hash table corrupted"))); + } + XLogRecParseState *newState = XLogParseBufferCopy(state); + newState->nextrecord = NULL; + remoteReadInfo->tail->nextrecord = newState; + remoteReadInfo->tail = newState; + remoteReadInfo->rec_max_lsn = newState->blockparse.blockhead.end_ptr; + } else { + HTAB *bad_hash = g_redoWorker->badPageHashTbl; + bool thread_found = false; + BadBlockRecEnt *remoteReadInfo = (BadBlockRecEnt*)hash_search(bad_hash, &(key), HASH_ENTER, &thread_found); + + Assert(!thread_found); + if (g_parseManager == NULL) { + XLogParseBufferInitFunc(&(g_redoWorker->parseManager), + MAX_PARSE_BUFF_NUM, &recordRefOperate, RedoInterruptCallBack); + } + XLogRecParseState *newState = XLogParseBufferCopy(state); + newState->nextrecord = NULL; + + remoteReadInfo->key = key; + remoteReadInfo->pblk = pblk; + remoteReadInfo->rec_min_lsn = newState->blockparse.blockhead.end_ptr; + remoteReadInfo->rec_max_lsn = newState->blockparse.blockhead.end_ptr; + remoteReadInfo->head = newState; + remoteReadInfo->tail = newState; + g_redoWorker->remoteReadPageNum++; + + if (g_redoWorker->remoteReadPageNum >= MAX_REMOTE_READ_INFO_NUM) { + ereport(WARNING, (errmsg("recovery thread found %d error block.", g_redoWorker->remoteReadPageNum))); + } + } + + return; +} + +void CheckRemoteReadAndRepairPage(BadBlockRecEnt *entry) +{ + XLogRecPtr rec_min_lsn = InvalidXLogRecPtr; + XLogRecPtr rec_max_lsn = InvalidXLogRecPtr; + bool check = false; + RepairBlockKey key; + + key = entry->key; + rec_min_lsn = entry->rec_min_lsn; + rec_max_lsn = entry->rec_max_lsn; + check = CheckRepairPage(key, rec_min_lsn, rec_max_lsn, g_redoWorker->page); + if (check) { + /* copy page to buffer pool, and recovery the stored xlog */ + RepairPageAndRecoveryXLog(entry, g_redoWorker->page); + /* clear page repair thread hash table */ + ClearSpecificsPageRepairHashTbl(key); + /* clear this thread invalid page hash table */ + forget_specified_invalid_pages(key); + /* clear thread bad block hash entry */ + ClearSpecificsPageEntryAndMem(entry); + g_redoWorker->remoteReadPageNum--; + } +} + +void SeqCheckRemoteReadAndRepairPage() +{ + BadBlockRecEnt *entry = NULL; + HASH_SEQ_STATUS status; + + HTAB *bad_hash = g_redoWorker->badPageHashTbl; + + hash_seq_init(&status, bad_hash); + while ((entry = (BadBlockRecEnt *)hash_seq_search(&status)) != NULL) { + CheckRemoteReadAndRepairPage(entry); + } +} + +} // namespace extreme_rto diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp index 1224d926e..dda9d91b0 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp @@ -189,7 +189,7 @@ static const RmgrDispatchData g_dispatchTable[RM_MAX_ID + 1] = { { DispatchSeqRecord, RmgrRecordInfoValid, RM_SEQ_ID, XLOG_SEQ_LOG, XLOG_SEQ_LOG }, { DispatchSpgistRecord, RmgrRecordInfoValid, RM_SPGIST_ID, XLOG_SPGIST_CREATE_INDEX, XLOG_SPGIST_VACUUM_REDIRECT }, { DispatchRepSlotRecord, RmgrRecordInfoValid, RM_SLOT_ID, XLOG_SLOT_CREATE, XLOG_TERM_LOG }, - { DispatchHeap3Record, RmgrRecordInfoValid, RM_HEAP3_ID, XLOG_HEAP3_NEW_CID, XLOG_HEAP3_REWRITE }, + { DispatchHeap3Record, RmgrRecordInfoValid, RM_HEAP3_ID, XLOG_HEAP3_NEW_CID, XLOG_HEAP3_INVALID }, { DispatchBarrierRecord, RmgrRecordInfoValid, RM_BARRIER_ID, XLOG_BARRIER_CREATE, XLOG_BARRIER_SWITCHOVER }, #ifdef ENABLE_MOT @@ -720,7 +720,13 @@ static bool DispatchRepSlotRecord(XLogReaderState *record, List *expectedTLIs, T /* Run from the dispatcher thread. */ static bool DispatchHeap3Record(XLogReaderState *record, List *expectedTLIs, TimestampTz recordXTime) { - DispatchTxnRecord(record, expectedTLIs, recordXTime, false); + uint8 info = ((XLogRecGetInfo(record) & (~XLR_INFO_MASK)) & XLOG_HEAP_OPMASK); + + if (info == XLOG_HEAP3_INVALID) { + DispatchRecordWithPages(record, expectedTLIs, SUPPORT_FPAGE_DISPATCH); + } else { + DispatchTxnRecord(record, expectedTLIs, recordXTime, false); + } return false; } diff --git a/src/gausskernel/storage/access/transam/single_double_write.cpp b/src/gausskernel/storage/access/transam/single_double_write.cpp index 602b86f21..ab63cec88 100644 --- a/src/gausskernel/storage/access/transam/single_double_write.cpp +++ b/src/gausskernel/storage/access/transam/single_double_write.cpp @@ -1,965 +1,966 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * single_double_write.cpp - * Before flush dirty pages to data file, flush them to double write file, - * in case of half-flushed pages. Recover those half-flushed data file pages - * before replaying xlog when starting. This file is for single flush double - * write. - * - * IDENTIFICATION - * src/gausskernel/storage/access/transam/single_double_write.cpp - * - * ------------------------------------------------------------------------- - */ -#include "access/double_write.h" -#include "storage/smgr/segment.h" -#include "utils/builtins.h" - -const uint16 RESULT_LEN = 256; -Datum dw_get_single_flush_dwn() -{ - if (dw_enabled()) { - char dwn[RESULT_LEN] = {0}; - errno_t rc; - if (g_instance.dw_single_cxt.dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { - uint32 dwn_first = (uint32)g_instance.dw_single_cxt.file_head->head.dwn; - uint32 dwn_second = (uint32)g_instance.dw_single_cxt.second_file_head->head.dwn; - rc = snprintf_s(dwn, RESULT_LEN, RESULT_LEN - 1, "%u/%u", dwn_first, dwn_second); - securec_check_ss(rc, "\0", "\0"); - } else { - uint32 dwn_old = (uint32)g_instance.dw_single_cxt.recovery_buf.file_head->head.dwn; - rc = snprintf_s(dwn, RESULT_LEN, RESULT_LEN - 1, "%u/0", dwn_old); - securec_check_ss(rc, "\0", "\0"); - } - return CStringGetTextDatum(dwn); - } - return CStringGetTextDatum("0/0"); -} - -Datum dw_get_single_flush_start() -{ - if (dw_enabled()) { - char start[RESULT_LEN] = {0}; - errno_t rc; - if (g_instance.dw_single_cxt.dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { - uint32 start_first = (uint32)g_instance.dw_single_cxt.file_head->start; - uint32 start_second = (uint32)g_instance.dw_single_cxt.second_file_head->start; - rc = snprintf_s(start, RESULT_LEN, RESULT_LEN - 1, "%u/%u", start_first, start_second); - securec_check_ss(rc, "\0", "\0"); - } else { - uint32 start_old = (uint32)g_instance.dw_single_cxt.recovery_buf.file_head->start; - rc = snprintf_s(start, RESULT_LEN, RESULT_LEN - 1, "%u/0", start_old); - securec_check_ss(rc, "\0", "\0"); - } - return CStringGetTextDatum(start); - } - - return CStringGetTextDatum("0/0"); -} - -Datum dw_get_single_flush_trunc_num() -{ - char trunc_num[RESULT_LEN] = {0}; - errno_t rc; - if (g_instance.dw_single_cxt.dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { - uint32 trunc_num_first = (uint32)g_instance.dw_single_cxt.single_stat_info.file_trunc_num; - uint32 trunc_num_second = (uint32)g_instance.dw_single_cxt.single_stat_info.second_file_trunc_num; - rc = snprintf_s(trunc_num, RESULT_LEN, RESULT_LEN - 1, "%u/%u", trunc_num_first, trunc_num_second); - securec_check_ss(rc, "\0", "\0"); - } else { - uint32 trunc_num_old = (uint32)g_instance.dw_single_cxt.single_stat_info.file_trunc_num; - rc = snprintf_s(trunc_num, RESULT_LEN, RESULT_LEN - 1, "%u/0", trunc_num_old); - securec_check_ss(rc, "\0", "\0"); - } - return CStringGetTextDatum(trunc_num); -} - -Datum dw_get_single_flush_reset_num() -{ - char reset_num[RESULT_LEN] = {0}; - errno_t rc; - if (g_instance.dw_single_cxt.dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { - uint32 reset_num_first = (uint32)g_instance.dw_single_cxt.single_stat_info.file_reset_num; - uint32 reset_num_second = (uint32)g_instance.dw_single_cxt.single_stat_info.second_file_reset_num; - rc = snprintf_s(reset_num, RESULT_LEN, RESULT_LEN - 1, "%u/%u", reset_num_first, reset_num_second); - securec_check_ss(rc, "\0", "\0"); - } else { - uint32 reset_num_old = (uint32)g_instance.dw_single_cxt.single_stat_info.file_reset_num; - rc = snprintf_s(reset_num, RESULT_LEN, RESULT_LEN - 1, "%u/0", reset_num_old); - securec_check_ss(rc, "\0", "\0"); - } - return CStringGetTextDatum(reset_num); -} - -Datum dw_get_single_flush_total_writes() -{ - char total_writes[RESULT_LEN] = {0}; - errno_t rc; - if (g_instance.dw_single_cxt.dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { - uint32 total_writes_first = (uint32)g_instance.dw_single_cxt.single_stat_info.total_writes; - uint32 total_writes_second = (uint32)g_instance.dw_single_cxt.single_stat_info.second_total_writes; - rc = snprintf_s(total_writes, RESULT_LEN, RESULT_LEN - 1, "%u/%u", total_writes_first, total_writes_second); - securec_check_ss(rc, "\0", "\0"); - } else { - uint32 total_writes_old = (uint32)g_instance.dw_single_cxt.single_stat_info.total_writes; - rc = snprintf_s(total_writes, RESULT_LEN, RESULT_LEN - 1, "%u/0", total_writes_old); - securec_check_ss(rc, "\0", "\0"); - } - return CStringGetTextDatum(total_writes); -} - -const dw_view_col_t g_dw_single_view[DW_SINGLE_VIEW_COL_NUM] = { - {"node_name", TEXTOID, dw_get_node_name}, - {"curr_dwn", TEXTOID, dw_get_single_flush_dwn}, - {"curr_start_page", TEXTOID, dw_get_single_flush_start}, - {"total_writes", TEXTOID, dw_get_single_flush_total_writes}, - {"file_trunc_num", TEXTOID, dw_get_single_flush_trunc_num}, - {"file_reset_num", TEXTOID, dw_get_single_flush_reset_num} -}; - -static bool dw_verify_item(const dw_single_flush_item* item, uint16 dwn); -static uint16 get_max_single_write_pos(bool is_first); -static uint16 atomic_get_dw_write_pos(bool is_first); - -static void dw_recovery_first_version_page(); -static void dw_recovery_old_single_dw_page(); -static void dw_recovery_second_version_page(); -static void dw_recovery_single_page(const dw_single_flush_item *item, uint16 item_num); - -void dw_generate_single_file() -{ - char *file_head = NULL; - int64 remain_size; - int fd = -1; - errno_t rc; - char *unaligned_buf = NULL; - - if (file_exists(SINGLE_DW_FILE_NAME)) { - ereport(PANIC, (errcode_for_file_access(), errmodule(MOD_DW), "DW single flush file already exists")); - } - - ereport(LOG, (errmodule(MOD_DW), errmsg("DW bootstrap single flush file"))); - - fd = open(SINGLE_DW_FILE_NAME, (DW_FILE_FLAG | O_CREAT), DW_FILE_PERM); - if (fd == -1) { - ereport(PANIC, - (errcode_for_file_access(), errmodule(MOD_DW), errmsg("Could not create file \"%s\"", - SINGLE_DW_FILE_NAME))); - } - - unaligned_buf = (char *)palloc0(DW_FILE_EXTEND_SIZE + BLCKSZ); /* one more BLCKSZ for alignment */ - - file_head = (char *)TYPEALIGN(BLCKSZ, unaligned_buf); - - /* file head and first batch head will be writen */ - remain_size = (DW_SINGLE_DIRTY_PAGE_NUM + DW_SINGLE_BUFTAG_PAGE_NUM) * BLCKSZ; - dw_prepare_file_head(file_head, 0, 0, 0); - dw_pwrite_file(fd, file_head, BLCKSZ, 0, SINGLE_DW_FILE_NAME); - rc = memset_s(file_head, BLCKSZ, 0, BLCKSZ); - securec_check(rc, "\0", "\0"); - dw_extend_file(fd, file_head, DW_FILE_EXTEND_SIZE, remain_size, DW_SINGLE_FILE_SIZE, true, NULL); - ereport(LOG, (errmodule(MOD_DW), errmsg("Double write single flush file created successfully"))); - - (void)close(fd); - fd = -1; - pfree(unaligned_buf); - return; -} - -static void dw_recovery_first_version_page() -{ - knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; - dw_file_head_t *file_head = single_cxt->file_head; - errno_t rc = 0; - uint64 offset = 0; - PageHeader pghr = NULL; - SMgrRelation reln = NULL; - char *unaligned_buf = (char *)palloc0(BLCKSZ + BLCKSZ); /* one more BLCKSZ for alignment */ - char *dw_block = (char *)TYPEALIGN(BLCKSZ, unaligned_buf); - char *data_block = (char *)palloc0(BLCKSZ); - dw_first_flush_item flush_item; - - for (uint16 i = file_head->start; i < DW_FIRST_DATA_PAGE_NUM; i++) { - offset = (i + 1) * BLCKSZ; /* need skip the file head */ - dw_pread_file(single_cxt->fd, dw_block, BLCKSZ, offset); - pghr = (PageHeader)dw_block; - - rc = memcpy_s(&flush_item, sizeof(dw_first_flush_item), dw_block + pghr->pd_lower, sizeof(dw_first_flush_item)); - securec_check(rc, "\0", "\0"); - - if (!dw_verify_pg_checksum((PageHeader)dw_block, flush_item.buf_tag.blockNum, true)) { - if (PageIsNew(dw_block)) { - Assert(flush_item.buf_tag.rnode.relNode == 0); - dw_log_recovery_page(LOG, "[first version] dw page is new, break this recovery", flush_item.buf_tag); - break; - } - dw_log_recovery_page(WARNING, "DW single page broken", flush_item.buf_tag); - dw_log_page_header((PageHeader)dw_block); - continue; - } - dw_log_recovery_page(DW_LOG_LEVEL, "DW page fine", flush_item.buf_tag); - - reln = smgropen(flush_item.buf_tag.rnode, InvalidBackendId, GetColumnNum(flush_item.buf_tag.forkNum)); - /* read data page */ - if (!dw_read_data_page(flush_item.buf_tag, reln, data_block)) { - continue; - } - dw_log_page_header((PageHeader)data_block); - if (!dw_verify_pg_checksum((PageHeader)data_block, flush_item.buf_tag.blockNum, false) || - XLByteLT(PageGetLSN(data_block), PageGetLSN(dw_block))) { - memset_s(dw_block + pghr->pd_lower, sizeof(dw_first_flush_item), 0, sizeof(dw_first_flush_item)); - securec_check(rc, "\0", "\0"); - dw_set_pg_checksum(dw_block, flush_item.buf_tag.blockNum); - - if (IsSegmentPhysicalRelNode(flush_item.buf_tag.rnode)) { - // seg_space must be initialized before. - seg_physical_write(reln->seg_space, flush_item.buf_tag.rnode, flush_item.buf_tag.forkNum, - flush_item.buf_tag.blockNum, (const char *)dw_block, false); - } else { - smgrwrite(reln, flush_item.buf_tag.forkNum, flush_item.buf_tag.blockNum, - (const char *)dw_block, false); - } - dw_log_recovery_page(LOG, "Date page recovered", flush_item.buf_tag); - dw_log_page_header((PageHeader)data_block); - } - } - - pfree(unaligned_buf); - pfree(data_block); -} - -static bool dw_verify_item(const dw_single_flush_item* item, uint16 dwn) -{ - if (item->dwn != dwn) { - return false; - } - - if (item->buf_tag.forkNum == InvalidForkNumber || item->buf_tag.blockNum == InvalidBlockNumber || - item->buf_tag.rnode.relNode == InvalidOid) { - if (item->dwn != 0 || item->data_page_idx != 0) { - ereport(WARNING, - (errmsg("dw recovery, find invalid item [page_idx %hu dwn %hu] skip this item," - "buf_tag[rel %u/%u/%u blk %u fork %d]", item->data_page_idx, item->dwn, - item->buf_tag.rnode.spcNode, item->buf_tag.rnode.dbNode, item->buf_tag.rnode.relNode, - item->buf_tag.blockNum, item->buf_tag.forkNum))); - } - return false; - } - pg_crc32c crc; - /* Contents are protected with a CRC */ - INIT_CRC32C(crc); - COMP_CRC32C(crc, (char*)item, offsetof(dw_single_flush_item, crc)); - FIN_CRC32C(crc); - - if (EQ_CRC32C(crc, item->crc)) { - return true; - } else { - return false; - } -} - -static void dw_recovery_single_page(const dw_single_flush_item *item, uint16 item_num) -{ - SMgrRelation reln; - uint32 offset = 0; - BufferTag buf_tag; - knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; - char *unaligned_buf = (char *)palloc0(BLCKSZ + BLCKSZ); /* one more BLCKSZ for alignment */ - char *dw_block = (char *)TYPEALIGN(BLCKSZ, unaligned_buf); - char *data_block = (char *)palloc0(BLCKSZ); - uint64 base_offset = 0; - - if (single_cxt->file_head->dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { - base_offset = 1 + DW_FIRST_DATA_PAGE_NUM + 1 + DW_SECOND_BUFTAG_PAGE_NUM; - } else { - base_offset = DW_SINGLE_BUFTAG_PAGE_NUM + 1; - } - - for (uint16 i = 0; i < item_num; i++) { - buf_tag = item[i].buf_tag; - - /* read dw file page */ - offset = (base_offset + item[i].data_page_idx) * BLCKSZ; - dw_pread_file(single_cxt->fd, dw_block, BLCKSZ, offset); - - if (!dw_verify_pg_checksum((PageHeader)dw_block, buf_tag.blockNum, true)) { - dw_log_recovery_page(WARNING, "DW single page broken", buf_tag); - dw_log_page_header((PageHeader)dw_block); - continue; - } - dw_log_recovery_page(DW_LOG_LEVEL, "DW page fine", buf_tag); - - reln = smgropen(buf_tag.rnode, InvalidBackendId, GetColumnNum(buf_tag.forkNum)); - /* read data page */ - if (!dw_read_data_page(buf_tag, reln, data_block)) { - continue; - } - dw_log_page_header((PageHeader)data_block); - if (!dw_verify_pg_checksum((PageHeader)data_block, buf_tag.blockNum, false) || - XLByteLT(PageGetLSN(data_block), PageGetLSN(dw_block))) { - if (IsSegmentPhysicalRelNode(buf_tag.rnode)) { - // seg_space must be initialized before. - seg_physical_write(reln->seg_space, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum, - (const char *)dw_block, false); - } else { - smgrwrite(reln, buf_tag.forkNum, buf_tag.blockNum, (const char *)dw_block, false); - } - dw_log_recovery_page(LOG, "Date page recovered", buf_tag); - dw_log_page_header((PageHeader)data_block); - } - } - - pfree(unaligned_buf); - pfree(data_block); - return; -} - -static void dw_recovery_second_version_page() -{ - knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; - dw_single_flush_item *item = NULL; - uint16 rec_num = 0; - dw_file_head_t *file_head = single_cxt->second_file_head; - char *buf = single_cxt->buf; - uint64 second_offset = DW_SECOND_BUFTAG_START_IDX * BLCKSZ; - - item = (dw_single_flush_item*)palloc0(sizeof(dw_single_flush_item) * DW_SECOND_DATA_PAGE_NUM); - - /* read all buffer tag item, need skip head page */ - dw_pread_file(single_cxt->fd, buf, DW_SECOND_BUFTAG_PAGE_NUM * BLCKSZ, second_offset); - - uint64 offset = 0; - dw_single_flush_item *temp = NULL; - for (uint16 i = single_cxt->second_file_head->start; i < DW_SECOND_DATA_PAGE_NUM; i++) { - offset = i * sizeof(dw_single_flush_item); - temp = (dw_single_flush_item*)((char*)buf + offset); - if (dw_verify_item(temp, file_head->head.dwn)) { - item[rec_num].data_page_idx = temp->data_page_idx; - item[rec_num].dwn = temp->dwn; - item[rec_num].buf_tag = temp->buf_tag; - item[rec_num].crc = temp->crc; - rec_num++; - } - } - ereport(LOG, (errmodule(MOD_DW), errmsg("[second version] DW single flush file valid item num is %d.", rec_num))); - if (rec_num > 0) { - qsort(item, rec_num, sizeof(dw_single_flush_item), buftag_compare); - ereport(LOG, (errmodule(MOD_DW), - errmsg("[second version] DW single flush file valid buftag item qsort finish."))); - dw_recovery_single_page(item, rec_num); - } - - pfree(item); -} - -static void dw_recovery_old_single_dw_page() -{ - knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; - dw_single_flush_item *item = NULL; - dw_file_head_t *file_head = single_cxt->recovery_buf.file_head; - uint16 blk_num = DW_SINGLE_DIRTY_PAGE_NUM / SINGLE_BLOCK_TAG_NUM; - char *buf = single_cxt->recovery_buf.buf; - uint16 rec_num = 0; - - item = (dw_single_flush_item*)palloc0(sizeof(dw_single_flush_item) * DW_SINGLE_DIRTY_PAGE_NUM); - - /* read all buffer tag item, need skip head page */ - dw_pread_file(single_cxt->fd, buf, blk_num * BLCKSZ, BLCKSZ); - int offset = 0; - dw_single_flush_item *temp = NULL; - for (int i = 0; i < DW_SINGLE_DIRTY_PAGE_NUM; i++) { - offset = i * sizeof(dw_single_flush_item); - temp = (dw_single_flush_item*)((char*)buf + offset); - if (dw_verify_item(temp, file_head->head.dwn)) { - item[rec_num].data_page_idx = temp->data_page_idx; - item[rec_num].buf_tag = temp->buf_tag; - item[rec_num].dwn = temp->dwn; - item[rec_num].crc = temp->crc; - rec_num++; - } - } - - ereport(LOG, (errmodule(MOD_DW), errmsg("[old version] DW single flush file valid item num is %d.", rec_num))); - if (rec_num > 0) { - qsort(item, rec_num, sizeof(dw_single_flush_item), buftag_compare); - ereport(LOG, (errmodule(MOD_DW), errmsg("DW single flush file valid buftag item qsort finish."))); - dw_recovery_single_page(item, rec_num); - } - - pfree(item); -} - -void dw_force_reset_single_file(uint32 dw_version) -{ - knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; - dw_file_head_t *file_head = single_cxt->file_head; - - if (USE_CKPT_THREAD_SYNC) { - ProcessSyncRequests(); - } else { - PageWriterSync(); - } - - dw_prepare_file_head((char *)file_head, 0, file_head->head.dwn + 1); - dw_pwrite_file(single_cxt->fd, file_head, BLCKSZ, 0, SINGLE_DW_FILE_NAME); - (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.file_reset_num, 1); - - ereport(LOG, (errmodule(MOD_DW), - errmsg("DW single flush finish recovery, reset the file head[dwn %hu, start %hu].", - file_head->head.dwn, file_head->start))); - - if (dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { - file_head = single_cxt->second_file_head; - dw_prepare_file_head((char *)file_head, 0, file_head->head.dwn + 1); - dw_pwrite_file(single_cxt->fd, file_head, BLCKSZ, (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ, SINGLE_DW_FILE_NAME); - (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.second_file_reset_num, 1); - ereport(LOG, (errmodule(MOD_DW), - errmsg("DW single flush finish recovery [second version], reset the file head[dwn %hu, start %hu].", - file_head->head.dwn, file_head->start))); - } - - return; -} - -void dw_recovery_partial_write_single() -{ - knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; - - if (single_cxt->file_head->dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { - dw_recovery_first_version_page(); - dw_recovery_second_version_page(); - } else { - dw_recovery_old_single_dw_page(); - } - - ereport(LOG, (errmodule(MOD_DW), errmsg("DW single flush file recovery finish."))); - - /* reset the file after the recovery is complete */ - dw_force_reset_single_file(single_cxt->file_head->dw_version); - return; -} - -void dw_single_file_truncate(bool is_first) -{ - uint16 max_idx = 0; - knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; - dw_file_head_t *file_head = NULL; - volatile uint16 org_start = 0; - volatile uint16 org_dwn = 0; - LWLock* flush_lock = NULL; - uint64 head_offset = 0; - - if (is_first) { - file_head = single_cxt->file_head; - flush_lock = single_cxt->flush_lock; - head_offset = 0; - } else { - file_head = single_cxt->second_file_head; - flush_lock = single_cxt->second_flush_lock; - head_offset = (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ; - } - - if (!LWLockConditionalAcquire(flush_lock, LW_EXCLUSIVE)) { - ereport(LOG, (errmodule(MOD_DW), - errmsg("[single flush] can not get dw flush lock and skip dw truncate for this time"))); - return; - } - - org_start = file_head->start; - org_dwn = file_head->head.dwn; - max_idx = get_max_single_write_pos(is_first); - if (max_idx == file_head->start) { - LWLockRelease(flush_lock); - return; - } - LWLockRelease(flush_lock); - - PageWriterSync(); - - if (!LWLockConditionalAcquire(flush_lock, LW_EXCLUSIVE)) { - ereport(LOG, (errmodule(MOD_DW), - errmsg("[single flush] can not get dw flush lock and skip dw truncate after sync for this time"))); - return; - } else if (org_start != file_head->start || org_dwn != file_head->head.dwn) { - LWLockRelease(flush_lock); - return; - } - - file_head->start = max_idx; - dw_prepare_file_head((char *)file_head, file_head->start, file_head->head.dwn); - - Assert(file_head->head.dwn == file_head->tail.dwn); - dw_pwrite_file(single_cxt->fd, file_head, BLCKSZ, head_offset, SINGLE_DW_FILE_NAME); - LWLockRelease(flush_lock); - - ereport(LOG, (errmodule(MOD_DW), - errmsg("[single flush][%s] DW truncate end: file_head[dwn %hu, start %hu], write_pos %hu", - is_first ? "first version" : "second_version", - file_head->head.dwn, file_head->start, is_first ? single_cxt->write_pos : single_cxt->second_write_pos))); - - if (is_first) { - (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.file_trunc_num, 1); - } else { - (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.second_file_trunc_num, 1); - } - return; -} - -static uint16 get_max_single_write_pos(bool is_first) -{ - uint16 max_idx = 0; - uint16 i = 0; - uint16 start = 0; - knl_g_dw_context* dw_single_cxt = &g_instance.dw_single_cxt; - uint16 end = 0; - dw_file_head_t *file_head = NULL; - - /* single_flush_state, first */ - if (!is_first) { - file_head = dw_single_cxt->second_file_head; - start = file_head->start + DW_FIRST_DATA_PAGE_NUM; - end = pg_atomic_read_u32(&dw_single_cxt->second_write_pos) + DW_FIRST_DATA_PAGE_NUM; - } else { - file_head = dw_single_cxt->file_head; - start = file_head->start; - end = pg_atomic_read_u32(&dw_single_cxt->write_pos); - } - - for (i = start; i < end; i++) { - if (dw_single_cxt->single_flush_state[i] == false) { - break; - } - } - max_idx = i; - - if (!is_first) { - max_idx = max_idx - DW_FIRST_DATA_PAGE_NUM; - } - - return max_idx; -} - -void dw_generate_new_single_file() -{ - char *file_head = NULL; - int64 extend_size; - int fd = -1; - errno_t rc; - char *unaligned_buf = NULL; - - if (file_exists(SINGLE_DW_FILE_NAME)) { - ereport(PANIC, (errcode_for_file_access(), errmodule(MOD_DW), "DW single flush file already exists")); - } - - ereport(LOG, (errmodule(MOD_DW), errmsg("DW bootstrap new single flush file"))); - - fd = open(SINGLE_DW_FILE_NAME, (DW_FILE_FLAG | O_CREAT), DW_FILE_PERM); - if (fd == -1) { - ereport(PANIC, - (errcode_for_file_access(), errmodule(MOD_DW), errmsg("Could not create file \"%s\"", - SINGLE_DW_FILE_NAME))); - } - - /* NO EREPORT(ERROR) from here till changes are logged */ - START_CRIT_SECTION(); - unaligned_buf = (char *)palloc0(DW_FILE_EXTEND_SIZE + BLCKSZ); /* one more BLCKSZ for alignment */ - - file_head = (char *)TYPEALIGN(BLCKSZ, unaligned_buf); - - /* first version page int */ - extend_size = DW_FIRST_DATA_PAGE_NUM * BLCKSZ; - dw_prepare_file_head(file_head, 0, 0, DW_SUPPORT_NEW_SINGLE_FLUSH); - dw_pwrite_file(fd, file_head, BLCKSZ, 0, SINGLE_DW_FILE_NAME); - - rc = memset_s(file_head, BLCKSZ, 0, BLCKSZ); - securec_check(rc, "\0", "\0"); - dw_extend_file(fd, file_head, DW_FILE_EXTEND_SIZE, extend_size, DW_NEW_SINGLE_FILE_SIZE, true, NULL); - - /* second version page init */ - extend_size = (DW_SECOND_BUFTAG_PAGE_NUM + DW_SECOND_DATA_PAGE_NUM) * BLCKSZ; - dw_prepare_file_head(file_head, 0, 0, DW_SUPPORT_NEW_SINGLE_FLUSH); - dw_pwrite_file(fd, file_head, BLCKSZ, (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ, SINGLE_DW_FILE_NAME); - - rc = memset_s(file_head, BLCKSZ, 0, BLCKSZ); - securec_check(rc, "\0", "\0"); - dw_extend_file(fd, file_head, DW_FILE_EXTEND_SIZE, extend_size, DW_NEW_SINGLE_FILE_SIZE, true, NULL); - END_CRIT_SECTION(); - - ereport(LOG, (errmodule(MOD_DW), errmsg("Double write single flush file created successfully"))); - - (void)close(fd); - fd = -1; - pfree(unaligned_buf); - return; -} - -uint16 first_version_dw_single_flush(BufferDesc *buf_desc) -{ - errno_t rc; - char *buf = t_thrd.proc->dw_buf; - knl_g_dw_context* dw_single_cxt = &g_instance.dw_single_cxt; - dw_file_head_t *file_head = dw_single_cxt->file_head; - uint16 actual_pos; - uint64 page_write_offset; - dw_first_flush_item item; - PageHeader pghr = NULL; - BufferTag phy_tag; - - uint32 buf_state = LockBufHdr(buf_desc); - Block block = BufHdrGetBlock(buf_desc); - XLogRecPtr page_lsn = BufferGetLSN(buf_desc); - UnlockBufHdr(buf_desc, buf_state); - - phy_tag = buf_desc->tag; - dw_transfer_phybuffer_addr(buf_desc, &phy_tag); - - Assert(buf_desc->buf_id < SegmentBufferStartID); - Assert(free_space_enough(buf_desc->buf_id)); - - /* first step, copy buffer to dw buf, than flush page lsn, the buffer content lock is already held */ - rc = memcpy_s(buf, BLCKSZ, block, BLCKSZ); - securec_check(rc, "\0", "\0"); - - XLogWaitFlush(page_lsn); - if (buf_desc->encrypt) { - dw_encrypt_page(buf_desc->tag, buf); - } - - actual_pos = atomic_get_dw_write_pos(true); - - item.dwn = file_head->head.dwn; - item.buf_tag = phy_tag; - pghr = (PageHeader)buf; - - rc = memcpy_s(buf + pghr->pd_lower, sizeof(dw_first_flush_item), &item, sizeof(dw_first_flush_item)); - securec_check(rc, "\0", "\0"); - - dw_set_pg_checksum(buf, item.buf_tag.blockNum); - page_write_offset = (1 + actual_pos) * BLCKSZ; - Assert(actual_pos < DW_FIRST_DATA_PAGE_NUM); - dw_pwrite_file(dw_single_cxt->fd, buf, BLCKSZ, page_write_offset, SINGLE_DW_FILE_NAME); - - (void)pg_atomic_add_fetch_u64(&dw_single_cxt->single_stat_info.total_writes, 1); - - return actual_pos; -} - -uint16 second_version_dw_single_flush(BufferTag tag, Block block, XLogRecPtr page_lsn, - bool encrypt, BufferTag phy_tag) -{ - errno_t rc; - uint16 actual_pos; - uint64 page_write_offset; - uint64 tag_write_offset; - uint16 block_offset; - dw_single_flush_item item; - knl_g_dw_context* dw_single_cxt = &g_instance.dw_single_cxt; - dw_file_head_t *file_head = dw_single_cxt->second_file_head; - char *buf = t_thrd.proc->dw_buf; - - /* first step, copy buffer to dw buf, than flush page lsn, the buffer content lock is already held */ - rc = memcpy_s(buf, BLCKSZ, block, BLCKSZ); - securec_check(rc, "\0", "\0"); - - XLogWaitFlush(page_lsn); - if (encrypt) { - dw_encrypt_page(tag, buf); - } - dw_set_pg_checksum(buf, phy_tag.blockNum); - - actual_pos = atomic_get_dw_write_pos(false); - - /* data page need skip head page and bufferTag page, bufferTag page need skip head page and first version page */ - page_write_offset = (actual_pos + DW_SECOND_DATA_START_IDX) * BLCKSZ; - tag_write_offset = DW_SECOND_BUFTAG_START_IDX * BLCKSZ + (actual_pos / SINGLE_BLOCK_TAG_NUM) * BLCKSZ; - block_offset = (actual_pos % SINGLE_BLOCK_TAG_NUM) * sizeof(dw_single_flush_item); - Assert(block_offset <= BLCKSZ - sizeof(dw_single_flush_item)); - Assert(actual_pos < DW_SECOND_DATA_PAGE_NUM); - Assert(page_write_offset < DW_NEW_SINGLE_FILE_SIZE && tag_write_offset < DW_SECOND_DATA_START_IDX * BLCKSZ); - - /* write the data page to dw file */ - dw_pwrite_file(dw_single_cxt->fd, buf, BLCKSZ, page_write_offset, SINGLE_DW_FILE_NAME); - - item.data_page_idx = actual_pos; - item.dwn = file_head->head.dwn; - item.buf_tag = phy_tag; - - /* Contents are protected with a CRC */ - INIT_CRC32C(item.crc); - COMP_CRC32C(item.crc, (char*)&item, offsetof(dw_single_flush_item, crc)); - FIN_CRC32C(item.crc); - - /* write the buffer tag item to dw file */ - (void)LWLockAcquire(dw_single_cxt->second_buftag_lock, LW_EXCLUSIVE); - dw_pread_file(dw_single_cxt->fd, buf, BLCKSZ, tag_write_offset); - rc = memcpy_s(buf + block_offset, BLCKSZ - block_offset, &item, sizeof(dw_single_flush_item)); - securec_check(rc, "\0", "\0"); - dw_pwrite_file(dw_single_cxt->fd, buf, BLCKSZ, tag_write_offset, SINGLE_DW_FILE_NAME); - - LWLockRelease(dw_single_cxt->second_buftag_lock); - (void)pg_atomic_add_fetch_u64(&dw_single_cxt->single_stat_info.second_total_writes, 1); - - return (actual_pos + DW_FIRST_DATA_PAGE_NUM); -} - -void wait_all_single_dw_finish_flush(bool is_first) -{ - uint16 start = 0; - uint16 end = 0; - dw_file_head_t *file_head = NULL; - knl_g_dw_context* dw_single_cxt = &g_instance.dw_single_cxt; - - /* single_flush_state, first */ - if (is_first) { - file_head = dw_single_cxt->file_head; - start = file_head->start; - end = pg_atomic_read_u32(&dw_single_cxt->write_pos); - } else { - file_head = dw_single_cxt->second_file_head; - start = file_head->start + DW_FIRST_DATA_PAGE_NUM; - end = pg_atomic_read_u32(&dw_single_cxt->second_write_pos) + DW_FIRST_DATA_PAGE_NUM; - } - - for (uint i = start; i < end;) { - if (dw_single_cxt->single_flush_state[i] != false) { - i++; - continue; - } else { - (void)sched_yield(); - } - } - return; -} - -void dw_single_file_recycle(bool is_first) -{ - bool file_full = false; - knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; - dw_file_head_t *file_head = NULL; - uint16 end = 0; - errno_t rc; - uint64 head_offset = 0; - uint16 flush_state_start = 0; - uint16 page_num = 0; - - if (is_first) { - file_head = single_cxt->file_head; - end = single_cxt->write_pos; - flush_state_start = 0; - page_num = DW_FIRST_DATA_PAGE_NUM; - head_offset = 0; - } else { - file_head = single_cxt->second_file_head; - end = single_cxt->second_write_pos; - flush_state_start = DW_FIRST_DATA_PAGE_NUM; - page_num = DW_SECOND_DATA_PAGE_NUM; - head_offset = (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ; - } - - file_full = end + 1 >= page_num; - - if (!file_full) { - return; - } - - /* reset start position and flush page num for full recycle */ - wait_all_single_dw_finish_flush(is_first); - - PageWriterSync(); - - rc = memset_s(single_cxt->single_flush_state + (flush_state_start * sizeof(bool)), - sizeof(bool) * page_num, 0, sizeof(bool) * page_num); - securec_check(rc, "\0", "\0"); - - dw_prepare_file_head((char *)file_head, 0, file_head->head.dwn + 1); - dw_pwrite_file(single_cxt->fd, file_head, BLCKSZ, head_offset, SINGLE_DW_FILE_NAME); - - /* The start and write_pos must be reset at the end. */ - file_head->start = 0; - if (is_first) { - single_cxt->write_pos = 0; - } else { - single_cxt->second_write_pos = 0; - } - - if (is_first) { - (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.file_reset_num, 1); - } else { - (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.second_file_reset_num, 1); - } - ereport(LOG, (errmodule(MOD_DW), errmsg("[single flush] [%s] Reset DW file: file_head[dwn %hu, start %hu], " - "writer pos is %hu", is_first ? "first version" : "second_version", file_head->head.dwn, file_head->start, - is_first ? single_cxt->write_pos : single_cxt->second_write_pos))); - return; -} - -static uint16 atomic_get_dw_write_pos(bool is_first) -{ - knl_g_dw_context* dw_single_cxt = &g_instance.dw_single_cxt; - uint16 page_num = is_first ? DW_FIRST_DATA_PAGE_NUM : DW_SECOND_DATA_PAGE_NUM; - uint32 write_pos; - LWLock *lock = is_first ? dw_single_cxt->flush_lock : dw_single_cxt->second_flush_lock; - - pg_memory_barrier(); - write_pos = is_first ? pg_atomic_read_u32(&dw_single_cxt->write_pos) : - pg_atomic_read_u32(&dw_single_cxt->second_write_pos); - - while (true) { - if ((write_pos + 1 >= page_num)) { - (void)LWLockAcquire(lock, LW_EXCLUSIVE); - dw_single_file_recycle(is_first); - LWLockRelease(lock); - write_pos = is_first ? pg_atomic_read_u32(&dw_single_cxt->write_pos) : - pg_atomic_read_u32(&dw_single_cxt->second_write_pos); - /* fetch write_pos, we need to check write_pos + 1 again */ - continue; - } - - if (is_first) { - if (pg_atomic_compare_exchange_u32(&dw_single_cxt->write_pos, &write_pos, write_pos + 1)) { - return write_pos; - } - } else { - if (pg_atomic_compare_exchange_u32(&dw_single_cxt->second_write_pos, &write_pos, write_pos + 1)) { - return write_pos; - } - } - } - - return write_pos; -} - -static uint32 dw_recover_file_head(knl_g_dw_context *cxt, bool single, bool first) -{ - uint32 i; - uint16 id; - errno_t rc; - int64 file_size; - dw_file_head_t *curr_head = NULL; - dw_file_head_t *working_head = NULL; - char *file_head = (char *)cxt->file_head; - uint32 dw_version = 0; - uint64 head_offset = 0; - int64 offset; - - if (single && !first) { - file_head = (char *)cxt->second_file_head; - head_offset = (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ; - } - dw_pread_file(cxt->fd, file_head, BLCKSZ, head_offset); - - for (i = 0; i < DW_FILE_HEAD_ID_NUM; i++) { - id = g_dw_file_head_ids[i]; - curr_head = (dw_file_head_t *)(file_head + sizeof(dw_file_head_t) * id); - if (dw_verify_file_head(curr_head)) { - working_head = curr_head; - break; - } - } - - if (working_head == NULL) { - ereport(FATAL, (errcode_for_file_access(), errmodule(MOD_DW), errmsg("Single file header is broken"))); - /* we should not get here, since FATAL will do abort. But for ut, return is needed */ - return dw_version; - } - - ereport(LOG, (errmodule(MOD_DW), errmsg("Found a valid single file header: id %hu, file_head[dwn %hu, start %hu]", - id, working_head->head.dwn, working_head->start))); - - for (i = 0; i < DW_FILE_HEAD_ID_NUM; i++) { - id = g_dw_file_head_ids[i]; - curr_head = (dw_file_head_t *)(file_head + sizeof(dw_file_head_t) * id); - if (curr_head != working_head) { - rc = memcpy_s(curr_head, sizeof(dw_file_head_t), working_head, sizeof(dw_file_head_t)); - securec_check(rc, "\0", "\0"); - } - } - - offset = dw_seek_file(cxt->fd, 0, SEEK_END); - if (single) { - dw_version = ((dw_file_head_t *)file_head)->dw_version; - file_size = (dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH ? DW_NEW_SINGLE_FILE_SIZE : DW_SINGLE_FILE_SIZE); - } else { - file_size = DW_FILE_SIZE; - } - - if (offset != file_size) { - ereport(PANIC, (errmodule(MOD_DW), - errmsg("DW check file size failed, expected_size %ld, actual_size %ld", DW_FILE_SIZE, offset))); - } - - dw_pwrite_file(cxt->fd, file_head, BLCKSZ, head_offset, single ? SINGLE_DW_FILE_NAME : OLD_DW_FILE_NAME); - return dw_version; -} - -void dw_cxt_init_single() -{ - char *buf = NULL; - knl_g_dw_context *single_cxt = &g_instance.dw_single_cxt; - uint32 dw_version = 0; - uint16 data_page_num = 0; - uint64 second_start_offset = 0; - - Assert(single_cxt->flush_lock == NULL); - single_cxt->flush_lock = LWLockAssign(LWTRANCHE_DW_SINGLE_FIRST); - single_cxt->second_flush_lock = LWLockAssign(LWTRANCHE_DW_SINGLE_SECOND); - single_cxt->second_buftag_lock = LWLockAssign(LWTRANCHE_DW_SINGLE_SECOND_BUFTAG); - - single_cxt->fd = open(SINGLE_DW_FILE_NAME, DW_FILE_FLAG, DW_FILE_PERM); - if (single_cxt->fd == -1) { - ereport(PANIC, - (errcode_for_file_access(), errmodule(MOD_DW), errmsg("Could not open file \"%s\"", SINGLE_DW_FILE_NAME))); - } - - data_page_num = DW_FIRST_DATA_PAGE_NUM + DW_SECOND_DATA_PAGE_NUM; - - /* two file head plus one for alignment */ - single_cxt->unaligned_buf = (char *)palloc0((DW_SECOND_BUFTAG_PAGE_NUM + 1 + 1 + 1) * BLCKSZ); - buf = (char *)TYPEALIGN(BLCKSZ, single_cxt->unaligned_buf); - single_cxt->file_head = (dw_file_head_t *)buf; - buf += BLCKSZ; - single_cxt->second_file_head = (dw_file_head_t *)buf; - buf += BLCKSZ; - single_cxt->buf = buf; - single_cxt->single_flush_state = (bool*)palloc0(sizeof(bool) * data_page_num); - - dw_version = dw_recover_file_head(single_cxt, true, true); - if (dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { - dw_pread_file(single_cxt->fd, single_cxt->file_head, BLCKSZ, 0); - second_start_offset = (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ; - dw_recover_file_head(single_cxt, true, false); - dw_pread_file(single_cxt->fd, single_cxt->second_file_head, BLCKSZ, second_start_offset); - } else { - Assert(dw_version == 0); - /* one file head plus one for alignment */ - single_cxt->recovery_buf.unaligned_buf = - (char *)palloc0((DW_SINGLE_DIRTY_PAGE_NUM / SINGLE_BLOCK_TAG_NUM + 1 + 1) * BLCKSZ); - buf = (char *)TYPEALIGN(BLCKSZ, single_cxt->recovery_buf.unaligned_buf); - single_cxt->recovery_buf.file_head = (dw_file_head_t *)buf; - buf += BLCKSZ; - single_cxt->recovery_buf.buf = buf; - dw_pread_file(single_cxt->fd, single_cxt->recovery_buf.file_head, BLCKSZ, 0); - single_cxt->recovery_buf.single_flush_state = - (bool*)palloc0(sizeof(bool) * DW_SINGLE_DIRTY_PAGE_NUM); - single_cxt->recovery_buf.write_pos = 0; - } - - pg_atomic_write_u32(&g_instance.dw_single_cxt.dw_version, dw_version); - single_cxt->closed = 0; - single_cxt->write_pos = 0; - single_cxt->second_write_pos = 0; - single_cxt->flush_page = 0; +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * single_double_write.cpp + * Before flush dirty pages to data file, flush them to double write file, + * in case of half-flushed pages. Recover those half-flushed data file pages + * before replaying xlog when starting. This file is for single flush double + * write. + * + * IDENTIFICATION + * src/gausskernel/storage/access/transam/single_double_write.cpp + * + * ------------------------------------------------------------------------- + */ +#include "access/double_write.h" +#include "storage/smgr/segment.h" +#include "utils/builtins.h" + +const uint16 RESULT_LEN = 256; +Datum dw_get_single_flush_dwn() +{ + if (dw_enabled()) { + char dwn[RESULT_LEN] = {0}; + errno_t rc; + if (g_instance.dw_single_cxt.dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { + uint32 dwn_first = (uint32)g_instance.dw_single_cxt.file_head->head.dwn; + uint32 dwn_second = (uint32)g_instance.dw_single_cxt.second_file_head->head.dwn; + rc = snprintf_s(dwn, RESULT_LEN, RESULT_LEN - 1, "%u/%u", dwn_first, dwn_second); + securec_check_ss(rc, "\0", "\0"); + } else { + uint32 dwn_old = (uint32)g_instance.dw_single_cxt.recovery_buf.file_head->head.dwn; + rc = snprintf_s(dwn, RESULT_LEN, RESULT_LEN - 1, "%u/0", dwn_old); + securec_check_ss(rc, "\0", "\0"); + } + return CStringGetTextDatum(dwn); + } + return CStringGetTextDatum("0/0"); +} + +Datum dw_get_single_flush_start() +{ + if (dw_enabled()) { + char start[RESULT_LEN] = {0}; + errno_t rc; + if (g_instance.dw_single_cxt.dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { + uint32 start_first = (uint32)g_instance.dw_single_cxt.file_head->start; + uint32 start_second = (uint32)g_instance.dw_single_cxt.second_file_head->start; + rc = snprintf_s(start, RESULT_LEN, RESULT_LEN - 1, "%u/%u", start_first, start_second); + securec_check_ss(rc, "\0", "\0"); + } else { + uint32 start_old = (uint32)g_instance.dw_single_cxt.recovery_buf.file_head->start; + rc = snprintf_s(start, RESULT_LEN, RESULT_LEN - 1, "%u/0", start_old); + securec_check_ss(rc, "\0", "\0"); + } + return CStringGetTextDatum(start); + } + + return CStringGetTextDatum("0/0"); +} + +Datum dw_get_single_flush_trunc_num() +{ + char trunc_num[RESULT_LEN] = {0}; + errno_t rc; + if (g_instance.dw_single_cxt.dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { + uint32 trunc_num_first = (uint32)g_instance.dw_single_cxt.single_stat_info.file_trunc_num; + uint32 trunc_num_second = (uint32)g_instance.dw_single_cxt.single_stat_info.second_file_trunc_num; + rc = snprintf_s(trunc_num, RESULT_LEN, RESULT_LEN - 1, "%u/%u", trunc_num_first, trunc_num_second); + securec_check_ss(rc, "\0", "\0"); + } else { + uint32 trunc_num_old = (uint32)g_instance.dw_single_cxt.single_stat_info.file_trunc_num; + rc = snprintf_s(trunc_num, RESULT_LEN, RESULT_LEN - 1, "%u/0", trunc_num_old); + securec_check_ss(rc, "\0", "\0"); + } + return CStringGetTextDatum(trunc_num); +} + +Datum dw_get_single_flush_reset_num() +{ + char reset_num[RESULT_LEN] = {0}; + errno_t rc; + if (g_instance.dw_single_cxt.dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { + uint32 reset_num_first = (uint32)g_instance.dw_single_cxt.single_stat_info.file_reset_num; + uint32 reset_num_second = (uint32)g_instance.dw_single_cxt.single_stat_info.second_file_reset_num; + rc = snprintf_s(reset_num, RESULT_LEN, RESULT_LEN - 1, "%u/%u", reset_num_first, reset_num_second); + securec_check_ss(rc, "\0", "\0"); + } else { + uint32 reset_num_old = (uint32)g_instance.dw_single_cxt.single_stat_info.file_reset_num; + rc = snprintf_s(reset_num, RESULT_LEN, RESULT_LEN - 1, "%u/0", reset_num_old); + securec_check_ss(rc, "\0", "\0"); + } + return CStringGetTextDatum(reset_num); +} + +Datum dw_get_single_flush_total_writes() +{ + char total_writes[RESULT_LEN] = {0}; + errno_t rc; + if (g_instance.dw_single_cxt.dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { + uint32 total_writes_first = (uint32)g_instance.dw_single_cxt.single_stat_info.total_writes; + uint32 total_writes_second = (uint32)g_instance.dw_single_cxt.single_stat_info.second_total_writes; + rc = snprintf_s(total_writes, RESULT_LEN, RESULT_LEN - 1, "%u/%u", total_writes_first, total_writes_second); + securec_check_ss(rc, "\0", "\0"); + } else { + uint32 total_writes_old = (uint32)g_instance.dw_single_cxt.single_stat_info.total_writes; + rc = snprintf_s(total_writes, RESULT_LEN, RESULT_LEN - 1, "%u/0", total_writes_old); + securec_check_ss(rc, "\0", "\0"); + } + return CStringGetTextDatum(total_writes); +} + +const dw_view_col_t g_dw_single_view[DW_SINGLE_VIEW_COL_NUM] = { + {"node_name", TEXTOID, dw_get_node_name}, + {"curr_dwn", TEXTOID, dw_get_single_flush_dwn}, + {"curr_start_page", TEXTOID, dw_get_single_flush_start}, + {"total_writes", TEXTOID, dw_get_single_flush_total_writes}, + {"file_trunc_num", TEXTOID, dw_get_single_flush_trunc_num}, + {"file_reset_num", TEXTOID, dw_get_single_flush_reset_num} +}; + +static bool dw_verify_item(const dw_single_flush_item* item, uint16 dwn); +static uint16 get_max_single_write_pos(bool is_first); +static uint16 atomic_get_dw_write_pos(bool is_first); + +static void dw_recovery_first_version_page(); +static void dw_recovery_old_single_dw_page(); +static void dw_recovery_second_version_page(); +static void dw_recovery_single_page(const dw_single_flush_item *item, uint16 item_num); + +void dw_generate_single_file() +{ + char *file_head = NULL; + int64 remain_size; + int fd = -1; + errno_t rc; + char *unaligned_buf = NULL; + + if (file_exists(SINGLE_DW_FILE_NAME)) { + ereport(PANIC, (errcode_for_file_access(), errmodule(MOD_DW), errmsg("DW single flush file already exists"))); + } + + ereport(LOG, (errmodule(MOD_DW), errmsg("DW bootstrap single flush file"))); + + fd = open(SINGLE_DW_FILE_NAME, (DW_FILE_FLAG | O_CREAT), DW_FILE_PERM); + if (fd == -1) { + ereport(PANIC, + (errcode_for_file_access(), errmodule(MOD_DW), errmsg("Could not create file \"%s\"", + SINGLE_DW_FILE_NAME))); + } + + unaligned_buf = (char *)palloc0(DW_FILE_EXTEND_SIZE + BLCKSZ); /* one more BLCKSZ for alignment */ + + file_head = (char *)TYPEALIGN(BLCKSZ, unaligned_buf); + + /* file head and first batch head will be writen */ + remain_size = (DW_SINGLE_DIRTY_PAGE_NUM + DW_SINGLE_BUFTAG_PAGE_NUM) * BLCKSZ; + dw_prepare_file_head(file_head, 0, 0, 0); + dw_pwrite_file(fd, file_head, BLCKSZ, 0, SINGLE_DW_FILE_NAME); + rc = memset_s(file_head, BLCKSZ, 0, BLCKSZ); + securec_check(rc, "\0", "\0"); + dw_extend_file(fd, file_head, DW_FILE_EXTEND_SIZE, remain_size, DW_SINGLE_FILE_SIZE, true, NULL); + ereport(LOG, (errmodule(MOD_DW), errmsg("Double write single flush file created successfully"))); + + (void)close(fd); + fd = -1; + pfree(unaligned_buf); + return; +} + +static void dw_recovery_first_version_page() +{ + knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; + dw_file_head_t *file_head = single_cxt->file_head; + errno_t rc = 0; + uint64 offset = 0; + PageHeader pghr = NULL; + SMgrRelation reln = NULL; + char *unaligned_buf = (char *)palloc0(BLCKSZ + BLCKSZ); /* one more BLCKSZ for alignment */ + char *dw_block = (char *)TYPEALIGN(BLCKSZ, unaligned_buf); + char *data_block = (char *)palloc0(BLCKSZ); + dw_first_flush_item flush_item; + + for (uint16 i = file_head->start; i < DW_FIRST_DATA_PAGE_NUM; i++) { + offset = (i + 1) * BLCKSZ; /* need skip the file head */ + dw_pread_file(single_cxt->fd, dw_block, BLCKSZ, offset); + pghr = (PageHeader)dw_block; + + rc = memcpy_s(&flush_item, sizeof(dw_first_flush_item), dw_block + pghr->pd_lower, sizeof(dw_first_flush_item)); + securec_check(rc, "\0", "\0"); + + if (!dw_verify_pg_checksum((PageHeader)dw_block, flush_item.buf_tag.blockNum, true)) { + if (PageIsNew(dw_block)) { + Assert(flush_item.buf_tag.rnode.relNode == 0); + dw_log_recovery_page(LOG, "[first version] dw page is new, break this recovery", flush_item.buf_tag); + break; + } + dw_log_recovery_page(WARNING, "DW single page broken", flush_item.buf_tag); + dw_log_page_header((PageHeader)dw_block); + continue; + } + dw_log_recovery_page(DW_LOG_LEVEL, "DW page fine", flush_item.buf_tag); + + reln = smgropen(flush_item.buf_tag.rnode, InvalidBackendId, GetColumnNum(flush_item.buf_tag.forkNum)); + /* read data page */ + if (!dw_read_data_page(flush_item.buf_tag, reln, data_block)) { + continue; + } + dw_log_page_header((PageHeader)data_block); + if (!dw_verify_pg_checksum((PageHeader)data_block, flush_item.buf_tag.blockNum, false) || + XLByteLT(PageGetLSN(data_block), PageGetLSN(dw_block))) { + memset_s(dw_block + pghr->pd_lower, sizeof(dw_first_flush_item), 0, sizeof(dw_first_flush_item)); + securec_check(rc, "\0", "\0"); + dw_set_pg_checksum(dw_block, flush_item.buf_tag.blockNum); + + if (IsSegmentPhysicalRelNode(flush_item.buf_tag.rnode)) { + // seg_space must be initialized before. + seg_physical_write(reln->seg_space, flush_item.buf_tag.rnode, flush_item.buf_tag.forkNum, + flush_item.buf_tag.blockNum, (const char *)dw_block, false); + } else { + smgrwrite(reln, flush_item.buf_tag.forkNum, flush_item.buf_tag.blockNum, + (const char *)dw_block, false); + } + dw_log_recovery_page(LOG, "Date page recovered", flush_item.buf_tag); + dw_log_page_header((PageHeader)data_block); + } + } + + pfree(unaligned_buf); + pfree(data_block); +} + +static bool dw_verify_item(const dw_single_flush_item* item, uint16 dwn) +{ + if (item->dwn != dwn) { + return false; + } + + if (item->buf_tag.forkNum == InvalidForkNumber || item->buf_tag.blockNum == InvalidBlockNumber || + item->buf_tag.rnode.relNode == InvalidOid) { + if (item->dwn != 0 || item->data_page_idx != 0) { + ereport(WARNING, + (errmsg("dw recovery, find invalid item [page_idx %hu dwn %hu] skip this item," + "buf_tag[rel %u/%u/%u blk %u fork %d]", item->data_page_idx, item->dwn, + item->buf_tag.rnode.spcNode, item->buf_tag.rnode.dbNode, item->buf_tag.rnode.relNode, + item->buf_tag.blockNum, item->buf_tag.forkNum))); + } + return false; + } + pg_crc32c crc; + /* Contents are protected with a CRC */ + INIT_CRC32C(crc); + COMP_CRC32C(crc, (char*)item, offsetof(dw_single_flush_item, crc)); + FIN_CRC32C(crc); + + if (EQ_CRC32C(crc, item->crc)) { + return true; + } else { + return false; + } +} + +static void dw_recovery_single_page(const dw_single_flush_item *item, uint16 item_num) +{ + SMgrRelation reln; + uint32 offset = 0; + BufferTag buf_tag; + knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; + char *unaligned_buf = (char *)palloc0(BLCKSZ + BLCKSZ); /* one more BLCKSZ for alignment */ + char *dw_block = (char *)TYPEALIGN(BLCKSZ, unaligned_buf); + char *data_block = (char *)palloc0(BLCKSZ); + uint64 base_offset = 0; + + if (single_cxt->file_head->dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { + base_offset = 1 + DW_FIRST_DATA_PAGE_NUM + 1 + DW_SECOND_BUFTAG_PAGE_NUM; + } else { + base_offset = DW_SINGLE_BUFTAG_PAGE_NUM + 1; + } + + for (uint16 i = 0; i < item_num; i++) { + buf_tag = item[i].buf_tag; + + /* read dw file page */ + offset = (base_offset + item[i].data_page_idx) * BLCKSZ; + dw_pread_file(single_cxt->fd, dw_block, BLCKSZ, offset); + + if (!dw_verify_pg_checksum((PageHeader)dw_block, buf_tag.blockNum, true)) { + dw_log_recovery_page(WARNING, "DW single page broken", buf_tag); + dw_log_page_header((PageHeader)dw_block); + continue; + } + dw_log_recovery_page(DW_LOG_LEVEL, "DW page fine", buf_tag); + + reln = smgropen(buf_tag.rnode, InvalidBackendId, GetColumnNum(buf_tag.forkNum)); + /* read data page */ + if (!dw_read_data_page(buf_tag, reln, data_block)) { + continue; + } + dw_log_page_header((PageHeader)data_block); + if (!dw_verify_pg_checksum((PageHeader)data_block, buf_tag.blockNum, false) || + XLByteLT(PageGetLSN(data_block), PageGetLSN(dw_block))) { + if (IsSegmentPhysicalRelNode(buf_tag.rnode)) { + // seg_space must be initialized before. + seg_physical_write(reln->seg_space, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum, + (const char *)dw_block, false); + } else { + smgrwrite(reln, buf_tag.forkNum, buf_tag.blockNum, (const char *)dw_block, false); + } + dw_log_recovery_page(LOG, "Date page recovered", buf_tag); + dw_log_page_header((PageHeader)data_block); + } + } + + pfree(unaligned_buf); + pfree(data_block); + return; +} + +static void dw_recovery_second_version_page() +{ + knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; + dw_single_flush_item *item = NULL; + uint16 rec_num = 0; + dw_file_head_t *file_head = single_cxt->second_file_head; + char *buf = single_cxt->buf; + uint64 second_offset = DW_SECOND_BUFTAG_START_IDX * BLCKSZ; + + item = (dw_single_flush_item*)palloc0(sizeof(dw_single_flush_item) * DW_SECOND_DATA_PAGE_NUM); + + /* read all buffer tag item, need skip head page */ + dw_pread_file(single_cxt->fd, buf, DW_SECOND_BUFTAG_PAGE_NUM * BLCKSZ, second_offset); + + uint64 offset = 0; + dw_single_flush_item *temp = NULL; + for (uint16 i = single_cxt->second_file_head->start; i < DW_SECOND_DATA_PAGE_NUM; i++) { + offset = i * sizeof(dw_single_flush_item); + temp = (dw_single_flush_item*)((char*)buf + offset); + if (dw_verify_item(temp, file_head->head.dwn)) { + item[rec_num].data_page_idx = temp->data_page_idx; + item[rec_num].dwn = temp->dwn; + item[rec_num].buf_tag = temp->buf_tag; + item[rec_num].crc = temp->crc; + rec_num++; + } + } + ereport(LOG, (errmodule(MOD_DW), errmsg("[second version] DW single flush file valid item num is %d.", rec_num))); + if (rec_num > 0) { + qsort(item, rec_num, sizeof(dw_single_flush_item), buftag_compare); + ereport(LOG, (errmodule(MOD_DW), + errmsg("[second version] DW single flush file valid buftag item qsort finish."))); + dw_recovery_single_page(item, rec_num); + } + + pfree(item); +} + +static void dw_recovery_old_single_dw_page() +{ + knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; + dw_single_flush_item *item = NULL; + dw_file_head_t *file_head = single_cxt->recovery_buf.file_head; + uint16 blk_num = DW_SINGLE_DIRTY_PAGE_NUM / SINGLE_BLOCK_TAG_NUM; + char *buf = single_cxt->recovery_buf.buf; + uint16 rec_num = 0; + + item = (dw_single_flush_item*)palloc0(sizeof(dw_single_flush_item) * DW_SINGLE_DIRTY_PAGE_NUM); + + /* read all buffer tag item, need skip head page */ + dw_pread_file(single_cxt->fd, buf, blk_num * BLCKSZ, BLCKSZ); + int offset = 0; + dw_single_flush_item *temp = NULL; + for (int i = 0; i < DW_SINGLE_DIRTY_PAGE_NUM; i++) { + offset = i * sizeof(dw_single_flush_item); + temp = (dw_single_flush_item*)((char*)buf + offset); + if (dw_verify_item(temp, file_head->head.dwn)) { + item[rec_num].data_page_idx = temp->data_page_idx; + item[rec_num].buf_tag = temp->buf_tag; + item[rec_num].dwn = temp->dwn; + item[rec_num].crc = temp->crc; + rec_num++; + } + } + + ereport(LOG, (errmodule(MOD_DW), errmsg("[old version] DW single flush file valid item num is %d.", rec_num))); + if (rec_num > 0) { + qsort(item, rec_num, sizeof(dw_single_flush_item), buftag_compare); + ereport(LOG, (errmodule(MOD_DW), errmsg("DW single flush file valid buftag item qsort finish."))); + dw_recovery_single_page(item, rec_num); + } + + pfree(item); +} + +void dw_force_reset_single_file(uint32 dw_version) +{ + knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; + dw_file_head_t *file_head = single_cxt->file_head; + + if (USE_CKPT_THREAD_SYNC) { + ProcessSyncRequests(); + } else { + PageWriterSync(); + } + + dw_prepare_file_head((char *)file_head, 0, file_head->head.dwn + 1); + dw_pwrite_file(single_cxt->fd, file_head, BLCKSZ, 0, SINGLE_DW_FILE_NAME); + (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.file_reset_num, 1); + + ereport(LOG, (errmodule(MOD_DW), + errmsg("DW single flush finish recovery, reset the file head[dwn %hu, start %hu].", + file_head->head.dwn, file_head->start))); + + if (dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { + file_head = single_cxt->second_file_head; + dw_prepare_file_head((char *)file_head, 0, file_head->head.dwn + 1); + dw_pwrite_file(single_cxt->fd, file_head, BLCKSZ, (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ, SINGLE_DW_FILE_NAME); + (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.second_file_reset_num, 1); + ereport(LOG, (errmodule(MOD_DW), + errmsg("DW single flush finish recovery [second version], reset the file head[dwn %hu, start %hu].", + file_head->head.dwn, file_head->start))); + } + + return; +} + +void dw_recovery_partial_write_single() +{ + knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; + + if (single_cxt->file_head->dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { + dw_recovery_first_version_page(); + dw_recovery_second_version_page(); + } else { + dw_recovery_old_single_dw_page(); + } + + ereport(LOG, (errmodule(MOD_DW), errmsg("DW single flush file recovery finish."))); + + /* reset the file after the recovery is complete */ + dw_force_reset_single_file(single_cxt->file_head->dw_version); + return; +} + +void dw_single_file_truncate(bool is_first) +{ + uint16 max_idx = 0; + knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; + dw_file_head_t *file_head = NULL; + volatile uint16 org_start = 0; + volatile uint16 org_dwn = 0; + LWLock* flush_lock = NULL; + uint64 head_offset = 0; + + if (is_first) { + file_head = single_cxt->file_head; + flush_lock = single_cxt->flush_lock; + head_offset = 0; + } else { + file_head = single_cxt->second_file_head; + flush_lock = single_cxt->second_flush_lock; + head_offset = (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ; + } + + if (!LWLockConditionalAcquire(flush_lock, LW_EXCLUSIVE)) { + ereport(LOG, (errmodule(MOD_DW), + errmsg("[single flush] can not get dw flush lock and skip dw truncate for this time"))); + return; + } + + org_start = file_head->start; + org_dwn = file_head->head.dwn; + max_idx = get_max_single_write_pos(is_first); + if (max_idx == file_head->start) { + LWLockRelease(flush_lock); + return; + } + LWLockRelease(flush_lock); + + PageWriterSync(); + + if (!LWLockConditionalAcquire(flush_lock, LW_EXCLUSIVE)) { + ereport(LOG, (errmodule(MOD_DW), + errmsg("[single flush] can not get dw flush lock and skip dw truncate after sync for this time"))); + return; + } else if (org_start != file_head->start || org_dwn != file_head->head.dwn) { + LWLockRelease(flush_lock); + return; + } + + file_head->start = max_idx; + dw_prepare_file_head((char *)file_head, file_head->start, file_head->head.dwn); + + Assert(file_head->head.dwn == file_head->tail.dwn); + dw_pwrite_file(single_cxt->fd, file_head, BLCKSZ, head_offset, SINGLE_DW_FILE_NAME); + LWLockRelease(flush_lock); + + ereport(LOG, (errmodule(MOD_DW), + errmsg("[single flush][%s] DW truncate end: file_head[dwn %hu, start %hu], write_pos %hu", + is_first ? "first version" : "second_version", + file_head->head.dwn, file_head->start, is_first ? single_cxt->write_pos : single_cxt->second_write_pos))); + + if (is_first) { + (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.file_trunc_num, 1); + } else { + (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.second_file_trunc_num, 1); + } + return; +} + +static uint16 get_max_single_write_pos(bool is_first) +{ + uint16 max_idx = 0; + uint16 i = 0; + uint16 start = 0; + knl_g_dw_context* dw_single_cxt = &g_instance.dw_single_cxt; + uint16 end = 0; + dw_file_head_t *file_head = NULL; + + /* single_flush_state, first */ + if (!is_first) { + file_head = dw_single_cxt->second_file_head; + start = file_head->start + DW_FIRST_DATA_PAGE_NUM; + end = pg_atomic_read_u32(&dw_single_cxt->second_write_pos) + DW_FIRST_DATA_PAGE_NUM; + } else { + file_head = dw_single_cxt->file_head; + start = file_head->start; + end = pg_atomic_read_u32(&dw_single_cxt->write_pos); + } + + for (i = start; i < end; i++) { + if (dw_single_cxt->single_flush_state[i] == false) { + break; + } + } + max_idx = i; + + if (!is_first) { + max_idx = max_idx - DW_FIRST_DATA_PAGE_NUM; + } + + return max_idx; +} + +void dw_generate_new_single_file() +{ + char *file_head = NULL; + int64 extend_size; + int fd = -1; + errno_t rc; + char *unaligned_buf = NULL; + + if (file_exists(SINGLE_DW_FILE_NAME)) { + ereport(PANIC, (errcode_for_file_access(), errmodule(MOD_DW), + errmsg("DW single flush file already exists"))); + } + + ereport(LOG, (errmodule(MOD_DW), errmsg("DW bootstrap new single flush file"))); + + fd = open(SINGLE_DW_FILE_NAME, (DW_FILE_FLAG | O_CREAT), DW_FILE_PERM); + if (fd == -1) { + ereport(PANIC, + (errcode_for_file_access(), errmodule(MOD_DW), errmsg("Could not create file \"%s\"", + SINGLE_DW_FILE_NAME))); + } + + /* NO EREPORT(ERROR) from here till changes are logged */ + START_CRIT_SECTION(); + unaligned_buf = (char *)palloc0(DW_FILE_EXTEND_SIZE + BLCKSZ); /* one more BLCKSZ for alignment */ + + file_head = (char *)TYPEALIGN(BLCKSZ, unaligned_buf); + + /* first version page int */ + extend_size = DW_FIRST_DATA_PAGE_NUM * BLCKSZ; + dw_prepare_file_head(file_head, 0, 0, DW_SUPPORT_NEW_SINGLE_FLUSH); + dw_pwrite_file(fd, file_head, BLCKSZ, 0, SINGLE_DW_FILE_NAME); + + rc = memset_s(file_head, BLCKSZ, 0, BLCKSZ); + securec_check(rc, "\0", "\0"); + dw_extend_file(fd, file_head, DW_FILE_EXTEND_SIZE, extend_size, DW_NEW_SINGLE_FILE_SIZE, true, NULL); + + /* second version page init */ + extend_size = (DW_SECOND_BUFTAG_PAGE_NUM + DW_SECOND_DATA_PAGE_NUM) * BLCKSZ; + dw_prepare_file_head(file_head, 0, 0, DW_SUPPORT_NEW_SINGLE_FLUSH); + dw_pwrite_file(fd, file_head, BLCKSZ, (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ, SINGLE_DW_FILE_NAME); + + rc = memset_s(file_head, BLCKSZ, 0, BLCKSZ); + securec_check(rc, "\0", "\0"); + dw_extend_file(fd, file_head, DW_FILE_EXTEND_SIZE, extend_size, DW_NEW_SINGLE_FILE_SIZE, true, NULL); + END_CRIT_SECTION(); + + ereport(LOG, (errmodule(MOD_DW), errmsg("Double write single flush file created successfully"))); + + (void)close(fd); + fd = -1; + pfree(unaligned_buf); + return; +} + +uint16 first_version_dw_single_flush(BufferDesc *buf_desc) +{ + errno_t rc; + char *buf = t_thrd.proc->dw_buf; + knl_g_dw_context* dw_single_cxt = &g_instance.dw_single_cxt; + dw_file_head_t *file_head = dw_single_cxt->file_head; + uint16 actual_pos; + uint64 page_write_offset; + dw_first_flush_item item; + PageHeader pghr = NULL; + BufferTag phy_tag; + + uint32 buf_state = LockBufHdr(buf_desc); + Block block = BufHdrGetBlock(buf_desc); + XLogRecPtr page_lsn = BufferGetLSN(buf_desc); + UnlockBufHdr(buf_desc, buf_state); + + phy_tag = buf_desc->tag; + dw_transfer_phybuffer_addr(buf_desc, &phy_tag); + + Assert(buf_desc->buf_id < SegmentBufferStartID); + Assert(free_space_enough(buf_desc->buf_id)); + + /* first step, copy buffer to dw buf, than flush page lsn, the buffer content lock is already held */ + rc = memcpy_s(buf, BLCKSZ, block, BLCKSZ); + securec_check(rc, "\0", "\0"); + + XLogWaitFlush(page_lsn); + if (buf_desc->encrypt) { + dw_encrypt_page(buf_desc->tag, buf); + } + + actual_pos = atomic_get_dw_write_pos(true); + + item.dwn = file_head->head.dwn; + item.buf_tag = phy_tag; + pghr = (PageHeader)buf; + + rc = memcpy_s(buf + pghr->pd_lower, sizeof(dw_first_flush_item), &item, sizeof(dw_first_flush_item)); + securec_check(rc, "\0", "\0"); + + dw_set_pg_checksum(buf, item.buf_tag.blockNum); + page_write_offset = (1 + actual_pos) * BLCKSZ; + Assert(actual_pos < DW_FIRST_DATA_PAGE_NUM); + dw_pwrite_file(dw_single_cxt->fd, buf, BLCKSZ, page_write_offset, SINGLE_DW_FILE_NAME); + + (void)pg_atomic_add_fetch_u64(&dw_single_cxt->single_stat_info.total_writes, 1); + + return actual_pos; +} + +uint16 second_version_dw_single_flush(BufferTag tag, Block block, XLogRecPtr page_lsn, + bool encrypt, BufferTag phy_tag) +{ + errno_t rc; + uint16 actual_pos; + uint64 page_write_offset; + uint64 tag_write_offset; + uint16 block_offset; + dw_single_flush_item item; + knl_g_dw_context* dw_single_cxt = &g_instance.dw_single_cxt; + dw_file_head_t *file_head = dw_single_cxt->second_file_head; + char *buf = t_thrd.proc->dw_buf; + + /* first step, copy buffer to dw buf, than flush page lsn, the buffer content lock is already held */ + rc = memcpy_s(buf, BLCKSZ, block, BLCKSZ); + securec_check(rc, "\0", "\0"); + + XLogWaitFlush(page_lsn); + if (encrypt) { + dw_encrypt_page(tag, buf); + } + dw_set_pg_checksum(buf, phy_tag.blockNum); + + actual_pos = atomic_get_dw_write_pos(false); + + /* data page need skip head page and bufferTag page, bufferTag page need skip head page and first version page */ + page_write_offset = (actual_pos + DW_SECOND_DATA_START_IDX) * BLCKSZ; + tag_write_offset = DW_SECOND_BUFTAG_START_IDX * BLCKSZ + (actual_pos / SINGLE_BLOCK_TAG_NUM) * BLCKSZ; + block_offset = (actual_pos % SINGLE_BLOCK_TAG_NUM) * sizeof(dw_single_flush_item); + Assert(block_offset <= BLCKSZ - sizeof(dw_single_flush_item)); + Assert(actual_pos < DW_SECOND_DATA_PAGE_NUM); + Assert(page_write_offset < DW_NEW_SINGLE_FILE_SIZE && tag_write_offset < DW_SECOND_DATA_START_IDX * BLCKSZ); + + /* write the data page to dw file */ + dw_pwrite_file(dw_single_cxt->fd, buf, BLCKSZ, page_write_offset, SINGLE_DW_FILE_NAME); + + item.data_page_idx = actual_pos; + item.dwn = file_head->head.dwn; + item.buf_tag = phy_tag; + + /* Contents are protected with a CRC */ + INIT_CRC32C(item.crc); + COMP_CRC32C(item.crc, (char*)&item, offsetof(dw_single_flush_item, crc)); + FIN_CRC32C(item.crc); + + /* write the buffer tag item to dw file */ + (void)LWLockAcquire(dw_single_cxt->second_buftag_lock, LW_EXCLUSIVE); + dw_pread_file(dw_single_cxt->fd, buf, BLCKSZ, tag_write_offset); + rc = memcpy_s(buf + block_offset, BLCKSZ - block_offset, &item, sizeof(dw_single_flush_item)); + securec_check(rc, "\0", "\0"); + dw_pwrite_file(dw_single_cxt->fd, buf, BLCKSZ, tag_write_offset, SINGLE_DW_FILE_NAME); + + LWLockRelease(dw_single_cxt->second_buftag_lock); + (void)pg_atomic_add_fetch_u64(&dw_single_cxt->single_stat_info.second_total_writes, 1); + + return (actual_pos + DW_FIRST_DATA_PAGE_NUM); +} + +void wait_all_single_dw_finish_flush(bool is_first) +{ + uint16 start = 0; + uint16 end = 0; + dw_file_head_t *file_head = NULL; + knl_g_dw_context* dw_single_cxt = &g_instance.dw_single_cxt; + + /* single_flush_state, first */ + if (is_first) { + file_head = dw_single_cxt->file_head; + start = file_head->start; + end = pg_atomic_read_u32(&dw_single_cxt->write_pos); + } else { + file_head = dw_single_cxt->second_file_head; + start = file_head->start + DW_FIRST_DATA_PAGE_NUM; + end = pg_atomic_read_u32(&dw_single_cxt->second_write_pos) + DW_FIRST_DATA_PAGE_NUM; + } + + for (uint i = start; i < end;) { + if (dw_single_cxt->single_flush_state[i] != false) { + i++; + continue; + } else { + (void)sched_yield(); + } + } + return; +} + +void dw_single_file_recycle(bool is_first) +{ + bool file_full = false; + knl_g_dw_context* single_cxt = &g_instance.dw_single_cxt; + dw_file_head_t *file_head = NULL; + uint16 end = 0; + errno_t rc; + uint64 head_offset = 0; + uint16 flush_state_start = 0; + uint16 page_num = 0; + + if (is_first) { + file_head = single_cxt->file_head; + end = single_cxt->write_pos; + flush_state_start = 0; + page_num = DW_FIRST_DATA_PAGE_NUM; + head_offset = 0; + } else { + file_head = single_cxt->second_file_head; + end = single_cxt->second_write_pos; + flush_state_start = DW_FIRST_DATA_PAGE_NUM; + page_num = DW_SECOND_DATA_PAGE_NUM; + head_offset = (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ; + } + + file_full = end + 1 >= page_num; + + if (!file_full) { + return; + } + + /* reset start position and flush page num for full recycle */ + wait_all_single_dw_finish_flush(is_first); + + PageWriterSync(); + + rc = memset_s(single_cxt->single_flush_state + (flush_state_start * sizeof(bool)), + sizeof(bool) * page_num, 0, sizeof(bool) * page_num); + securec_check(rc, "\0", "\0"); + + dw_prepare_file_head((char *)file_head, 0, file_head->head.dwn + 1); + dw_pwrite_file(single_cxt->fd, file_head, BLCKSZ, head_offset, SINGLE_DW_FILE_NAME); + + /* The start and write_pos must be reset at the end. */ + file_head->start = 0; + if (is_first) { + single_cxt->write_pos = 0; + } else { + single_cxt->second_write_pos = 0; + } + + if (is_first) { + (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.file_reset_num, 1); + } else { + (void)pg_atomic_add_fetch_u64(&single_cxt->single_stat_info.second_file_reset_num, 1); + } + ereport(LOG, (errmodule(MOD_DW), errmsg("[single flush] [%s] Reset DW file: file_head[dwn %hu, start %hu], " + "writer pos is %hu", is_first ? "first version" : "second_version", file_head->head.dwn, file_head->start, + is_first ? single_cxt->write_pos : single_cxt->second_write_pos))); + return; +} + +static uint16 atomic_get_dw_write_pos(bool is_first) +{ + knl_g_dw_context* dw_single_cxt = &g_instance.dw_single_cxt; + uint16 page_num = is_first ? DW_FIRST_DATA_PAGE_NUM : DW_SECOND_DATA_PAGE_NUM; + uint32 write_pos; + LWLock *lock = is_first ? dw_single_cxt->flush_lock : dw_single_cxt->second_flush_lock; + + pg_memory_barrier(); + write_pos = is_first ? pg_atomic_read_u32(&dw_single_cxt->write_pos) : + pg_atomic_read_u32(&dw_single_cxt->second_write_pos); + + while (true) { + if ((write_pos + 1 >= page_num)) { + (void)LWLockAcquire(lock, LW_EXCLUSIVE); + dw_single_file_recycle(is_first); + LWLockRelease(lock); + write_pos = is_first ? pg_atomic_read_u32(&dw_single_cxt->write_pos) : + pg_atomic_read_u32(&dw_single_cxt->second_write_pos); + /* fetch write_pos, we need to check write_pos + 1 again */ + continue; + } + + if (is_first) { + if (pg_atomic_compare_exchange_u32(&dw_single_cxt->write_pos, &write_pos, write_pos + 1)) { + return write_pos; + } + } else { + if (pg_atomic_compare_exchange_u32(&dw_single_cxt->second_write_pos, &write_pos, write_pos + 1)) { + return write_pos; + } + } + } + + return write_pos; +} + +static uint32 dw_recover_file_head(knl_g_dw_context *cxt, bool single, bool first) +{ + uint32 i; + uint16 id; + errno_t rc; + int64 file_size; + dw_file_head_t *curr_head = NULL; + dw_file_head_t *working_head = NULL; + char *file_head = (char *)cxt->file_head; + uint32 dw_version = 0; + uint64 head_offset = 0; + int64 offset; + + if (single && !first) { + file_head = (char *)cxt->second_file_head; + head_offset = (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ; + } + dw_pread_file(cxt->fd, file_head, BLCKSZ, head_offset); + + for (i = 0; i < DW_FILE_HEAD_ID_NUM; i++) { + id = g_dw_file_head_ids[i]; + curr_head = (dw_file_head_t *)(file_head + sizeof(dw_file_head_t) * id); + if (dw_verify_file_head(curr_head)) { + working_head = curr_head; + break; + } + } + + if (working_head == NULL) { + ereport(FATAL, (errcode_for_file_access(), errmodule(MOD_DW), errmsg("Single file header is broken"))); + /* we should not get here, since FATAL will do abort. But for ut, return is needed */ + return dw_version; + } + + ereport(LOG, (errmodule(MOD_DW), errmsg("Found a valid single file header: id %hu, file_head[dwn %hu, start %hu]", + id, working_head->head.dwn, working_head->start))); + + for (i = 0; i < DW_FILE_HEAD_ID_NUM; i++) { + id = g_dw_file_head_ids[i]; + curr_head = (dw_file_head_t *)(file_head + sizeof(dw_file_head_t) * id); + if (curr_head != working_head) { + rc = memcpy_s(curr_head, sizeof(dw_file_head_t), working_head, sizeof(dw_file_head_t)); + securec_check(rc, "\0", "\0"); + } + } + + offset = dw_seek_file(cxt->fd, 0, SEEK_END); + if (single) { + dw_version = ((dw_file_head_t *)file_head)->dw_version; + file_size = (dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH ? DW_NEW_SINGLE_FILE_SIZE : DW_SINGLE_FILE_SIZE); + } else { + file_size = DW_FILE_SIZE; + } + + if (offset != file_size) { + ereport(PANIC, (errmodule(MOD_DW), + errmsg("DW check file size failed, expected_size %ld, actual_size %ld", DW_FILE_SIZE, offset))); + } + + dw_pwrite_file(cxt->fd, file_head, BLCKSZ, head_offset, single ? SINGLE_DW_FILE_NAME : OLD_DW_FILE_NAME); + return dw_version; +} + +void dw_cxt_init_single() +{ + char *buf = NULL; + knl_g_dw_context *single_cxt = &g_instance.dw_single_cxt; + uint32 dw_version = 0; + uint16 data_page_num = 0; + uint64 second_start_offset = 0; + + Assert(single_cxt->flush_lock == NULL); + single_cxt->flush_lock = LWLockAssign(LWTRANCHE_DW_SINGLE_FIRST); + single_cxt->second_flush_lock = LWLockAssign(LWTRANCHE_DW_SINGLE_SECOND); + single_cxt->second_buftag_lock = LWLockAssign(LWTRANCHE_DW_SINGLE_SECOND_BUFTAG); + + single_cxt->fd = open(SINGLE_DW_FILE_NAME, DW_FILE_FLAG, DW_FILE_PERM); + if (single_cxt->fd == -1) { + ereport(PANIC, + (errcode_for_file_access(), errmodule(MOD_DW), errmsg("Could not open file \"%s\"", SINGLE_DW_FILE_NAME))); + } + + data_page_num = DW_FIRST_DATA_PAGE_NUM + DW_SECOND_DATA_PAGE_NUM; + + /* two file head plus one for alignment */ + single_cxt->unaligned_buf = (char *)palloc0((DW_SECOND_BUFTAG_PAGE_NUM + 1 + 1 + 1) * BLCKSZ); + buf = (char *)TYPEALIGN(BLCKSZ, single_cxt->unaligned_buf); + single_cxt->file_head = (dw_file_head_t *)buf; + buf += BLCKSZ; + single_cxt->second_file_head = (dw_file_head_t *)buf; + buf += BLCKSZ; + single_cxt->buf = buf; + single_cxt->single_flush_state = (bool*)palloc0(sizeof(bool) * data_page_num); + + dw_version = dw_recover_file_head(single_cxt, true, true); + if (dw_version == DW_SUPPORT_NEW_SINGLE_FLUSH) { + dw_pread_file(single_cxt->fd, single_cxt->file_head, BLCKSZ, 0); + second_start_offset = (1 + DW_FIRST_DATA_PAGE_NUM) * BLCKSZ; + dw_recover_file_head(single_cxt, true, false); + dw_pread_file(single_cxt->fd, single_cxt->second_file_head, BLCKSZ, second_start_offset); + } else { + Assert(dw_version == 0); + /* one file head plus one for alignment */ + single_cxt->recovery_buf.unaligned_buf = + (char *)palloc0((DW_SINGLE_DIRTY_PAGE_NUM / SINGLE_BLOCK_TAG_NUM + 1 + 1) * BLCKSZ); + buf = (char *)TYPEALIGN(BLCKSZ, single_cxt->recovery_buf.unaligned_buf); + single_cxt->recovery_buf.file_head = (dw_file_head_t *)buf; + buf += BLCKSZ; + single_cxt->recovery_buf.buf = buf; + dw_pread_file(single_cxt->fd, single_cxt->recovery_buf.file_head, BLCKSZ, 0); + single_cxt->recovery_buf.single_flush_state = + (bool*)palloc0(sizeof(bool) * DW_SINGLE_DIRTY_PAGE_NUM); + single_cxt->recovery_buf.write_pos = 0; + } + + pg_atomic_write_u32(&g_instance.dw_single_cxt.dw_version, dw_version); + single_cxt->closed = 0; + single_cxt->write_pos = 0; + single_cxt->second_write_pos = 0; + single_cxt->flush_page = 0; } \ No newline at end of file diff --git a/src/gausskernel/storage/access/transam/transam.cpp b/src/gausskernel/storage/access/transam/transam.cpp index d68a0d04f..cdfe36e31 100644 --- a/src/gausskernel/storage/access/transam/transam.cpp +++ b/src/gausskernel/storage/access/transam/transam.cpp @@ -74,7 +74,7 @@ CommitSeqNo TransactionIdGetCommitSeqNo(TransactionId transactionId, bool isComm * Before going to the commit log manager, check our single item cache to * see if we didn't just check the transaction status a moment ago. */ - if ((snapshot == NULL || !IsVersionMVCCSnapshot(snapshot)) && + if ((snapshot == NULL || (!IsVersionMVCCSnapshot(snapshot) && !(snapshot->satisfies == SNAPSHOT_DECODE_MVCC))) && TransactionIdEquals(transactionId, t_thrd.xact_cxt.cachedFetchCSNXid)) { t_thrd.xact_cxt.latestFetchCSNXid = t_thrd.xact_cxt.cachedFetchCSNXid; t_thrd.xact_cxt.latestFetchCSN = t_thrd.xact_cxt.cachedFetchCSN; @@ -98,15 +98,15 @@ RETRY: * If the XID is older than RecentGlobalXmin, check the clog. Otherwise * check the csnlog. */ - if (!isMvcc || GTM_LITE_MODE) { + if (snapshot != NULL && snapshot->satisfies == SNAPSHOT_DECODE_MVCC) { + xid = GetReplicationSlotCatalogXmin(); + } else if (!isMvcc || GTM_LITE_MODE) { TransactionId recentGlobalXmin = pg_atomic_read_u64(&t_thrd.xact_cxt.ShmemVariableCache->recentGlobalXmin); if (!TransactionIdIsValid(recentGlobalXmin)) { xid = t_thrd.xact_cxt.ShmemVariableCache->recentLocalXmin; } else { xid = recentGlobalXmin; } - } else if (snapshot != NULL && snapshot->satisfies == SNAPSHOT_DECODE_MVCC) { - xid = GetReplicationSlotCatalogXmin(); } else if (snapshot != NULL && IsMVCCSnapshot(snapshot)) { xid = snapshot->xmin; } else { diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index ebbe0148f..158c4ab49 100755 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -395,6 +395,7 @@ static bool CheckForStandbyTrigger(void); static void xlog_outrec(StringInfo buf, XLogReaderState *record); #endif static void pg_start_backup_callback(int code, Datum arg); +static void DoAbortBackupCallback(int code, Datum arg); static void AbortExcluseBackupCallback(int code, Datum arg); static bool read_tablespace_map(List **tablespaces); static bool read_backup_label(XLogRecPtr *checkPointLoc, bool *backupEndRequired, bool *backupFromStandby, @@ -7433,6 +7434,10 @@ void TruncateAndRemoveXLogForRoachRestore(XLogReaderState *record) XLogRecPtrIsValid(t_thrd.shemem_ptr_cxt.ControlFile->backupStartPoint)) { ereport(FATAL, (errmsg("truncate xlog LSN is before consistent recovery point"))); } + + /* wal receiver and wal receiver writer must be stopped before we truncate xlog */ + ShutdownWalRcv(); + uint32 xlogOff; XLogSegNo xlogsegno; char xlogFileName[1024] = {0}; @@ -11822,6 +11827,10 @@ void CreateCheckPoint(int flags) if (TransactionIdIsNormal(globalXmin) && TransactionIdPrecedes(globalXmin, cutoff_xid)) { cutoff_xid = globalXmin; } + TransactionId oldestXidInUndo = pg_atomic_read_u64(&g_instance.undo_cxt.oldestXidInUndo); + if (TransactionIdIsNormal(oldestXidInUndo) && TransactionIdPrecedes(oldestXidInUndo, cutoff_xid)) { + cutoff_xid = oldestXidInUndo; + } TruncateCSNLOG(cutoff_xid); t_thrd.checkpoint_cxt.last_truncate_log_time = now; } @@ -12618,6 +12627,10 @@ bool CreateRestartPoint(int flags) if (TransactionIdIsNormal(globalXmin) && TransactionIdPrecedes(globalXmin, cutoffXid)) { cutoffXid = globalXmin; } + TransactionId oldestXidInUndo = pg_atomic_read_u64(&g_instance.undo_cxt.oldestXidInUndo); + if (TransactionIdIsNormal(oldestXidInUndo) && TransactionIdPrecedes(oldestXidInUndo, cutoffXid)) { + cutoffXid = oldestXidInUndo; + } TruncateCSNLOG(cutoffXid); t_thrd.checkpoint_cxt.last_truncate_log_time = now; } @@ -12654,18 +12667,24 @@ static XLogSegNo CalcRecycleSegNo(XLogRecPtr curInsert, XLogRecPtr quorumMinRequ XLByteToSeg(curInsert - maxKeepSize, SegNoCanRecycled); } - if (!XLByteEQ(quorumMinRequired, InvalidXLogRecPtr)) { - XLogSegNo quorumMinSegNo; - XLByteToSeg(quorumMinRequired, quorumMinSegNo); - if (quorumMinSegNo < SegNoCanRecycled) { - SegNoCanRecycled = quorumMinSegNo; + /* + * For asynchronous replication, we do not care connected sync standbys, + * since standby build won't block xact commit on master. + */ + if (u_sess->attr.attr_storage.guc_synchronous_commit > SYNCHRONOUS_COMMIT_LOCAL_FLUSH) { + if (!XLByteEQ(quorumMinRequired, InvalidXLogRecPtr)) { + XLogSegNo quorumMinSegNo; + XLByteToSeg(quorumMinRequired, quorumMinSegNo); + if (quorumMinSegNo < SegNoCanRecycled) { + SegNoCanRecycled = quorumMinSegNo; + } } } if (!XLByteEQ(minToolsRequired, InvalidXLogRecPtr)) { XLogSegNo minToolsSegNo; XLByteToSeg(minToolsRequired, minToolsSegNo); - if (minToolsSegNo < SegNoCanRecycled) { + if (minToolsSegNo < SegNoCanRecycled && minToolsSegNo > 0) { SegNoCanRecycled = minToolsSegNo; } } @@ -12787,6 +12806,7 @@ static XLogSegNo CalculateCNRecycleSegNoForStreamingHadr(XLogRecPtr curInsert, X static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo, XLogRecPtr curInsert) { XLogSegNo segno; + XLogSegNo wal_keep_segno; XLogRecPtr keep; XLogRecPtr xlogcopystartptr; ReplicationSlotState repl_slot_state; @@ -12804,6 +12824,8 @@ static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo, XLogRecPtr curIns segno = segno - (uint32)u_sess->attr.attr_storage.wal_keep_segments; } + wal_keep_segno = segno; + ReplicationSlotsComputeRequiredLSN(&repl_slot_state); keep = repl_slot_state.min_required; ReplicationSlotReportRestartLSN(); @@ -12862,9 +12884,7 @@ static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo, XLogRecPtr curIns * 3.When standby is not alive in dummy standby mode, just keep log. * 4.Notice the users if slot is invalid */ - if (t_thrd.xlog_cxt.server_mode == PRIMARY_MODE && - u_sess->attr.attr_storage.guc_synchronous_commit > SYNCHRONOUS_COMMIT_LOCAL_FLUSH && - t_thrd.syncrep_cxt.SyncRepConfig != NULL) { + if (t_thrd.xlog_cxt.server_mode == PRIMARY_MODE) { if (WalSndInProgress(SNDROLE_PRIMARY_BUILDSTANDBY) || pg_atomic_read_u32(&g_instance.comm_cxt.current_gsrewind_count) > 0) { /* segno = 1 show all file should be keep */ @@ -12900,6 +12920,7 @@ static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo, XLogRecPtr curIns "is bigger than max keep size: %lu", maxKeepSize))); } } + if (XLByteEQ(keep, InvalidXLogRecPtr) && repl_slot_state.exist_in_use && !g_instance.attr.attr_storage.dcf_attr.enable_dcf) { /* there are slots and lsn is invalid, we keep it */ @@ -12907,6 +12928,11 @@ static void KeepLogSeg(XLogRecPtr recptr, XLogSegNo *logSegNo, XLogRecPtr curIns ereport(WARNING, (errmsg("invalid replication slot recptr"), errhint("Check slot configuration or setup standby/secondary"))); } + + /* take wal_keep_segments into consideration */ + if (wal_keep_segno < segno) { + segno = wal_keep_segno; + } } if (u_sess->attr.attr_storage.enable_cbm_tracking) { @@ -14626,12 +14652,23 @@ XLogRecPtr StandbyDoStopBackup(char *labelfile) } startPoint = (((uint64)hi) << 32) | lo; remaining = strchr(labelfile, '\n') + 1; /* %n is not portable enough */ + + XLogRecPtr minRecoveryPoint; LWLockAcquire(ControlFileLock, LW_SHARED); stopPoint = t_thrd.shemem_ptr_cxt.ControlFile->checkPointCopy.redo; + minRecoveryPoint = t_thrd.shemem_ptr_cxt.ControlFile->minRecoveryPoint; LWLockRelease(ControlFileLock); if (XLByteLE(stopPoint, startPoint)) { stopPoint = GetXLogReplayRecPtr(NULL); } + + if (XLByteLT(stopPoint, minRecoveryPoint)) { + stopPoint = minRecoveryPoint; + } + + if (XLByteLT(stopPoint, g_instance.comm_cxt.predo_cxt.redoPf.read_ptr)) { + stopPoint = g_instance.comm_cxt.predo_cxt.redoPf.read_ptr; + } return stopPoint; } @@ -14671,7 +14708,26 @@ void do_pg_abort_backup(void) u_sess->proc_cxt.sessionBackupState = SESSION_BACKUP_NONE; StopSuspendWalInsert(lastlrc); - u_sess->proc_cxt.sessionBackupState = SESSION_BACKUP_NONE; +} + +/* + * Error cleanup callback for do_pg_abort_backup + */ +static void DoAbortBackupCallback(int code, Datum arg) +{ + do_pg_abort_backup(); +} + +/* + * Register a handler that will warn about unterminated backups at the end of + * session, unless this has already been done. + */ +void RegisterPersistentAbortBackupHandler(void) +{ + if (u_sess->proc_cxt.registerAbortBackupHandlerdone) + return; + on_shmem_exit(DoAbortBackupCallback, BoolGetDatum(true)); + u_sess->proc_cxt.registerAbortBackupHandlerdone = true; } /* diff --git a/src/gausskernel/storage/access/transam/xlogfuncs.cpp b/src/gausskernel/storage/access/transam/xlogfuncs.cpp index bc5607d4c..b178c729e 100755 --- a/src/gausskernel/storage/access/transam/xlogfuncs.cpp +++ b/src/gausskernel/storage/access/transam/xlogfuncs.cpp @@ -170,13 +170,7 @@ Datum pg_start_backup_v2(PG_FUNCTION_ARGS) DIR *dir; char startxlogstr[MAXFNAMELEN]; errno_t errorno = EOK; - MemoryContext oldContext; - u_sess->probackup_context = AllocSetContextCreate(u_sess->top_mem_cxt, "probackup context", - ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, - ALLOCSET_DEFAULT_MAXSIZE); - oldContext = MemoryContextSwitchTo(u_sess->probackup_context); - SessionBackupState status = u_sess->proc_cxt.sessionBackupState; if (status == SESSION_BACKUP_NON_EXCLUSIVE) @@ -196,7 +190,15 @@ Datum pg_start_backup_v2(PG_FUNCTION_ARGS) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("a backup is already in progress in this session"))); + RegisterPersistentAbortBackupHandler(); + startpoint = do_pg_start_backup(backupidstr, fast, &labelfile,dir, &tblspcmapfile, NULL,false,true); + + if (u_sess->probackup_context == NULL) { + u_sess->probackup_context = AllocSetContextCreate(u_sess->top_mem_cxt, "probackup context", + ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, + ALLOCSET_DEFAULT_MAXSIZE); + } u_sess->proc_cxt.LabelFile = MemoryContextStrdup(u_sess->probackup_context, labelfile); if (tblspcmapfile != NULL) { u_sess->proc_cxt.TblspcMapFile = MemoryContextStrdup(u_sess->probackup_context, tblspcmapfile); @@ -210,8 +212,6 @@ Datum pg_start_backup_v2(PG_FUNCTION_ARGS) securec_check_ss(errorno, "", ""); PG_RETURN_TEXT_P(cstring_to_text(startxlogstr)); - - MemoryContextSwitchTo(oldContext); } /* @@ -223,7 +223,7 @@ Datum pg_stop_backup_v2(PG_FUNCTION_ARGS) ReturnSetInfo *rsinfo = (ReturnSetInfo *)fcinfo->resultinfo; TupleDesc tupdesc; Tuplestorestate *tupstore; - MemoryContext perqueryctx, oldcontext, oldcontext2; + MemoryContext perqueryctx, oldcontext; Datum values[3]; bool nulls[3]; XLogRecPtr stoppoint; @@ -247,8 +247,6 @@ Datum pg_stop_backup_v2(PG_FUNCTION_ARGS) errmsg("non-exclusive backup is not in progress"))); } - oldcontext2 = MemoryContextSwitchTo(u_sess->probackup_context); - /* check to see if caller supports us returning a tuplestore */ if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), @@ -305,8 +303,6 @@ Datum pg_stop_backup_v2(PG_FUNCTION_ARGS) tuplestore_putvalues(tupstore, tupdesc, values, nulls); tuplestore_donestoring(tupstore); - MemoryContextSwitchTo(oldcontext2); - return (Datum) 0; } @@ -1709,6 +1705,7 @@ Datum gs_hadr_has_barrier_creator(PG_FUNCTION_ARGS) */ Datum gs_hadr_in_recovery(PG_FUNCTION_ARGS) { +#ifndef ENABLE_LITE_MODE if (!superuser() && !(isOperatoradmin(GetUserId()) && u_sess->attr.attr_security.operation_mode)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("Must be system admin or operator admin in operation mode to gs_hadr_has_barrier_creator.")))); @@ -1716,7 +1713,9 @@ Datum gs_hadr_in_recovery(PG_FUNCTION_ARGS) if (knl_g_get_redo_finish_status()) { PG_RETURN_BOOL(false); } - +#else + FEATURE_ON_LITE_MODE_NOT_SUPPORTED(); +#endif PG_RETURN_BOOL(true); } @@ -2013,6 +2012,7 @@ Datum gs_get_hadr_key_cn(PG_FUNCTION_ARGS) Datum gs_streaming_dr_in_switchover(PG_FUNCTION_ARGS) { +#ifndef ENABLE_LITE_MODE if (!superuser() && !(isOperatoradmin(GetUserId()) && u_sess->attr.attr_security.operation_mode)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("Must be system admin or operator admin in operation mode to gs_streaming_dr_switchover.")))); @@ -2031,12 +2031,16 @@ Datum gs_streaming_dr_in_switchover(PG_FUNCTION_ARGS) RequestBarrier(barrier_id, NULL, true); #else CreateHadrSwitchoverBarrier(); +#endif +#else + FEATURE_ON_LITE_MODE_NOT_SUPPORTED(); #endif PG_RETURN_BOOL(true); } Datum gs_streaming_dr_service_truncation_check(PG_FUNCTION_ARGS) { +#ifndef ENABLE_LITE_MODE XLogRecPtr switchoverLsn = g_instance.streaming_dr_cxt.switchoverBarrierLsn; XLogRecPtr flushLsn = InvalidXLogRecPtr; bool isInteractionCompleted = false; @@ -2074,10 +2078,15 @@ Datum gs_streaming_dr_service_truncation_check(PG_FUNCTION_ARGS) } else { PG_RETURN_BOOL(false); } +#else + FEATURE_ON_LITE_MODE_NOT_SUPPORTED(); + PG_RETURN_BOOL(false); +#endif } Datum gs_streaming_dr_get_switchover_barrier(PG_FUNCTION_ARGS) { +#ifndef ENABLE_LITE_MODE if (!superuser() && !(isOperatoradmin(GetUserId()) && u_sess->attr.attr_security.operation_mode)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("Must be system admin or operator admin in operation mode to gs_streaming_dr_get_switchover_barrier.")))); @@ -2119,6 +2128,9 @@ Datum gs_streaming_dr_get_switchover_barrier(PG_FUNCTION_ARGS) PG_RETURN_BOOL(true); } } +#else + FEATURE_ON_LITE_MODE_NOT_SUPPORTED(); +#endif PG_RETURN_BOOL(false); } @@ -2377,7 +2389,11 @@ Datum gs_pitr_archive_slot_force_advance(PG_FUNCTION_ARGS) signal_child(g_instance.archive_thread_info.obsArchPID[i], SIGUSR2, -1); } } - g_instance.roach_cxt.isXLogForceRecycled = false; + if (IS_PGXC_COORDINATOR) { + g_instance.roach_cxt.isXLogForceRecycled = false; + } else { + g_instance.roach_cxt.forceAdvanceSlotTigger = true; + } rc = snprintf_s(location, MAXFNAMELEN, MAXFNAMELEN - 1, "%08X/%08X", (uint32)(archiveSlotLocNow >> 32), (uint32)(archiveSlotLocNow)); securec_check_ss(rc, "\0", "\0"); diff --git a/src/gausskernel/storage/access/transam/xloginsert.cpp b/src/gausskernel/storage/access/transam/xloginsert.cpp index df1f55f9d..022e9555a 100755 --- a/src/gausskernel/storage/access/transam/xloginsert.cpp +++ b/src/gausskernel/storage/access/transam/xloginsert.cpp @@ -957,8 +957,7 @@ static XLogRecData *XLogRecordAssemble(RmgrId rmid, uint8 info, XLogFPWInfo fpw_ *(scratch++) = XLR_BLOCK_ID_ORIGIN; remained_size--; if (istoast && t_thrd.proc->workingVersionNum >= PARALLEL_DECODE_VERSION_NUM && XLogLogicalInfoActive()) { - const int toastFlag = 1 << 8; - m_session_id |= toastFlag; + m_session_id = (int)((uint32)(m_session_id) | TOAST_FLAG); } XLOG_ASSEMBLE_ONE_ITEM(scratch, sizeof(m_session_id), &m_session_id, remained_size); } diff --git a/src/gausskernel/storage/access/ubtree/ubtinsert.cpp b/src/gausskernel/storage/access/ubtree/ubtinsert.cpp index ba7a823d0..00062fe6c 100644 --- a/src/gausskernel/storage/access/ubtree/ubtinsert.cpp +++ b/src/gausskernel/storage/access/ubtree/ubtinsert.cpp @@ -763,7 +763,9 @@ static TransactionId UBTreeCheckUnique(Relation rel, IndexTuple itup, Relation h if (curPartOid != gpiScan->currPartOid) { GPISetCurrPartOid(gpiScan, curPartOid); if (!GPIGetNextPartRelation(gpiScan, CurrentMemoryContext, AccessShareLock)) { - ItemIdMarkDead(curitemid); + if (CheckPartitionIsInvisible(gpiScan)) { + ItemIdMarkDead(curitemid); + } opaque->btpo_flags |= BTP_HAS_GARBAGE; if (nbuf != InvalidBuffer) { MarkBufferDirtyHint(nbuf, true); diff --git a/src/gausskernel/storage/access/ustore/knl_uhio.cpp b/src/gausskernel/storage/access/ustore/knl_uhio.cpp index 7af98f1af..586c98038 100644 --- a/src/gausskernel/storage/access/ustore/knl_uhio.cpp +++ b/src/gausskernel/storage/access/ustore/knl_uhio.cpp @@ -51,7 +51,15 @@ Buffer RelationGetBufferForUTuple(Relation relation, Size len, Buffer otherBuffe BlockNumber targetBlock; BlockNumber otherBlock; bool needLock = false; - bool last_page_tested = false; + + /* + * Blocks that extended one by one are different from bulk-extend blocks, and + * are not recorded into FSM. As its creator session close this realtion, they + * can not be used by any other body. It is especially obvious for partition + * bulk insert. Here, if no avaiable found in FSM, we check the last block to + * reuse the 'leaked free space' mentioned earlier. + */ + bool test_last_block = false; len = SHORTALIGN(len); /* @@ -125,7 +133,6 @@ Buffer RelationGetBufferForUTuple(Relation relation, Size len, Buffer otherBuffe if (nblocks > 0) { targetBlock = nblocks - 1; } - last_page_tested = true; } } @@ -147,7 +154,13 @@ loop: if (otherBuffer == InvalidBuffer) { /* easy case */ buffer = ReadBufferBI(relation, targetBlock, bistate); - LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + if (!TryLockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE, !test_last_block)) { + Assert(test_last_block); + ReleaseBuffer(buffer); + + /* someone is using this block, give up and extend. */ + break; + } } else if (otherBlock == targetBlock) { buffer = otherBuffer; /* also easy case */ @@ -220,12 +233,12 @@ loop: * * The best is to record all pages into FSM using bulk-extend in later. */ - if (targetBlock == InvalidBlockNumber && !last_page_tested) { + if (targetBlock == InvalidBlockNumber && !test_last_block && otherBuffer == InvalidBuffer) { BlockNumber nblocks = RelationGetNumberOfBlocks(relation); if (nblocks > 0) { targetBlock = nblocks - 1; } - last_page_tested = true; + test_last_block = true; } } diff --git a/src/gausskernel/storage/access/ustore/knl_undoaction.cpp b/src/gausskernel/storage/access/ustore/knl_undoaction.cpp index 0c285a90a..b82f02313 100644 --- a/src/gausskernel/storage/access/ustore/knl_undoaction.cpp +++ b/src/gausskernel/storage/access/ustore/knl_undoaction.cpp @@ -35,6 +35,14 @@ static UHeapDiskTuple CopyTupleFromUndoRecord(Relation relation, UndoRecord *und static void RestoreXactFromUndoRecord(UndoRecord *undorecord, Buffer buffer, UHeapDiskTuple tuple); static void LogUHeapUndoActions(UHeapUndoActionWALInfo *walInfo, Relation rel); +static int GetUndoApplySize() +{ + uint64 undoApplySize = (uint64)u_sess->attr.attr_memory.maintenance_work_mem * 1024L; + uint64 applySize = MAX_UNDO_APPLY_SIZE > undoApplySize ? undoApplySize : MAX_UNDO_APPLY_SIZE; + Assert(applySize <= MAX_UNDO_APPLY_SIZE); + return (int)applySize; +} + /* * execute_undo_actions - Execute the undo actions * @@ -49,10 +57,7 @@ void ExecuteUndoActions(TransactionId fullXid, UndoRecPtr fromUrecptr, UndoRecPt Assert(toUrecptr != INVALID_UNDO_REC_PTR && fromUrecptr != INVALID_UNDO_REC_PTR); Assert(slotPtr != INVALID_UNDO_REC_PTR); Assert(fullXid != InvalidTransactionId); - int undoApplySize = u_sess->attr.attr_memory.maintenance_work_mem * 1024L; - - /* setting maintenance_work_mem atleast 2GB will cause an overflow */ - undoApplySize = (undoApplySize < 0) ? INT_MAX : undoApplySize; + int undoApplySize = GetUndoApplySize(); UndoRecPtr urecPtr = fromUrecptr; UndoRecord *urec = New(CurrentMemoryContext)UndoRecord(); urec->SetUrp(toUrecptr); @@ -165,10 +170,7 @@ void ExecuteUndoActions(TransactionId fullXid, UndoRecPtr fromUrecptr, UndoRecPt void ExecuteUndoActionsPage(UndoRecPtr fromUrp, Relation rel, Buffer buffer, TransactionId xid) { UndoRecPtr urp = fromUrp; - int undoApplySize = u_sess->attr.attr_memory.maintenance_work_mem * 1024L; - - /* setting maintenance_work_mem atleast 2GB will cause an overflow */ - undoApplySize = (undoApplySize < 0) ? INT_MAX : undoApplySize; + int undoApplySize = GetUndoApplySize(); /* * Fetch the multiple undo records which can fit into undoApplySize; diff --git a/src/gausskernel/storage/access/ustore/knl_uredo.cpp b/src/gausskernel/storage/access/ustore/knl_uredo.cpp index cf0e193c5..6744a6282 100644 --- a/src/gausskernel/storage/access/ustore/knl_uredo.cpp +++ b/src/gausskernel/storage/access/ustore/knl_uredo.cpp @@ -356,7 +356,8 @@ static UndoRecPtr PrepareAndInsertUndoRecordForDeleteRedo(XLogReaderState *recor * If the WAL stream contains undo tuple, then replace it with the * explicitly stored tuple. */ - Size datalen = recordlen - SizeOfXLUndoHeader - SizeOfUHeapDelete - undoMetaSize - SizeOfUHeapHeader - readSize; + Size datalen = recordlen - SizeOfXLUndoHeader - SizeOfUHeapDelete - undoMetaSize - SizeOfUHeapHeader - + readSize - (hasCSN ? sizeof(CommitSeqNo) : 0); char *data = (char *)xlrec + SizeOfUHeapDelete + SizeOfXLUndoHeader + undoMetaSize + readSize; utup->disk_tuple = GetUHeapDiskTupleFromRedoData(data, &datalen, tbuf, false); @@ -856,8 +857,8 @@ static UndoRecPtr PrepareAndInsertUndoRecordForUpdateRedo(XLogReaderState *recor } char *data = (char *)curxlogptr; - Size datalen = recordlen - SizeOfUHeapHeader - SizeOfXLUndoHeader - SizeOfUHeapUpdate - - undoMetaSize - SizeOfXLUndoHeader - initPageXtraInfo - readSize; + Size datalen = recordlen - SizeOfUHeapHeader - SizeOfXLUndoHeader - SizeOfUHeapUpdate - + undoMetaSize - SizeOfXLUndoHeader - initPageXtraInfo - readSize - (hasCSN ? sizeof(CommitSeqNo) : 0); oldtup->disk_tuple = GetUHeapDiskTupleFromRedoData(data, &datalen, tbuf, false); oldtup->disk_tuple_size = datalen; diff --git a/src/gausskernel/storage/access/ustore/knl_uscan.cpp b/src/gausskernel/storage/access/ustore/knl_uscan.cpp index 62f0b4eba..923ef5bcd 100644 --- a/src/gausskernel/storage/access/ustore/knl_uscan.cpp +++ b/src/gausskernel/storage/access/ustore/knl_uscan.cpp @@ -1193,6 +1193,10 @@ Buffer UHeapIndexBuildNextBlock(UHeapScanDesc scan) } scan->rs_base.rs_ntuples = 0; scan->rs_cutup = NULL; + if (BufferIsValid(scan->rs_base.rs_cbuf)) { + ReleaseBuffer(scan->rs_base.rs_cbuf); + scan->rs_base.rs_cbuf = InvalidBuffer; + } if (blkno >= scan->rs_base.rs_nblocks) { return InvalidBuffer; /* we are done */ @@ -1202,6 +1206,7 @@ Buffer UHeapIndexBuildNextBlock(UHeapScanDesc scan) /* read the next page, and lock with exclusive mode */ Buffer buf = ReadBufferExtended(scan->rs_base.rs_rd, MAIN_FORKNUM, blkno, RBM_NORMAL, scan->rs_base.rs_strategy); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + scan->rs_base.rs_cbuf = buf; return buf; } @@ -1274,7 +1279,8 @@ bool UHeapIndexBuildNextPage(UHeapScanDesc scan) scan->rs_base.rs_ntuples = ntup; scan->rs_base.rs_cindex = 0; - UnlockReleaseBuffer(buf); + /* drop the lock but keep the pin */ + LockBuffer(buf, BUFFER_LOCK_UNLOCK); return true; } diff --git a/src/gausskernel/storage/access/ustore/knl_uundovec.cpp b/src/gausskernel/storage/access/ustore/knl_uundovec.cpp index 8d72ae7b5..e8e101a29 100644 --- a/src/gausskernel/storage/access/ustore/knl_uundovec.cpp +++ b/src/gausskernel/storage/access/ustore/knl_uundovec.cpp @@ -480,7 +480,7 @@ URecVector* FetchUndoRecordRange( __inout UndoRecPtr *startUrp, } totalSize += urec->MemoryRecordSize(); - if (totalSize > maxUndoApplySize) { + if (totalSize >= maxUndoApplySize) { *startUrp = currUrp; break; } diff --git a/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp b/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp index f71e4b023..4dfa6deb3 100644 --- a/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp +++ b/src/gausskernel/storage/access/ustore/undo/knl_uundoapi.cpp @@ -33,8 +33,6 @@ #include "utils/builtins.h" namespace undo { -static uint64 g_maxUndoSizePerTrans = 0; - void AllocateTransSlot(UndoSlotPtr slotPtr, UndoZone *zone, TransactionId xid, UndoPersistence upersistence) { TransactionSlot *slot = zone->AllocTransactionSlot(slotPtr, xid, u_sess->proc_cxt.MyDatabaseId); @@ -67,9 +65,10 @@ bool CheckNeedSwitch(UndoPersistence upersistence, uint64 size, UndoRecPtr undoP void RollbackIfUndoExceeds(TransactionId xid, uint64 size) { t_thrd.undo_cxt.transUndoSize += size; - if ((!t_thrd.xlog_cxt.InRecovery) && (t_thrd.undo_cxt.transUndoSize > g_maxUndoSizePerTrans)) { + uint64 transUndoThresholdSize = (uint64)u_sess->attr.attr_storage.undo_limit_size_transaction * BLCKSZ; + if ((!t_thrd.xlog_cxt.InRecovery) && (t_thrd.undo_cxt.transUndoSize > transUndoThresholdSize)) { ereport(ERROR, (errmsg(UNDOFORMAT("xid %lu, the undo size %lu of the transaction exceeds the threshold %lu."), - xid, t_thrd.undo_cxt.transUndoSize, g_maxUndoSizePerTrans))); + xid, t_thrd.undo_cxt.transUndoSize, transUndoThresholdSize))); } return; } @@ -365,7 +364,7 @@ void InitUndoCountThreshold() uint32 maxThreadNum = 0; - if (g_instance.attr.attr_common.enable_thread_pool) { + if (ENABLE_THREAD_POOL) { maxThreadNum = g_threadPoolControler->GetThreadNum(); } @@ -571,8 +570,6 @@ void RecoveryUndoSystemMeta(void) } g_instance.undo_cxt.undoTotalSize = 0; g_instance.undo_cxt.undoMetaSize = 0; - g_maxUndoSizePerTrans = - (uint64)u_sess->attr.attr_storage.undo_limit_size_transaction * BLCKSZ; /* Recover undospace meta. */ undo::UndoZone::RecoveryUndoZone(fd); /* Recover undospace meta. */ diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index 3fbd4525c..121dc1e97 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -5545,6 +5545,39 @@ void LockBuffer(Buffer buffer, int mode) } } +/* + * Try to acquire the content_lock for the buffer if must_wait is false. + * If the content lock is not available, return FALSE with no side-effects. + */ +bool TryLockBuffer(Buffer buffer, int mode, bool must_wait) +{ + Assert(BufferIsValid(buffer)); + + /* without tries, act as LockBuffer */ + if (must_wait) { + LockBuffer(buffer, mode); + return true; + } + + /* local buffers need no lock */ + if (BufferIsLocal(buffer)) { + return true; + } + + volatile BufferDesc *buf = GetBufferDescriptor(buffer - 1); + + if (mode == BUFFER_LOCK_SHARE) { + return LWLockConditionalAcquire(buf->content_lock, LW_SHARED); + } else if (mode == BUFFER_LOCK_EXCLUSIVE) { + return LWLockConditionalAcquire(buf->content_lock, LW_EXCLUSIVE); + } else { + ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), + (errmsg("unrecognized buffer lock mode for TryLockBuffer: %d", mode)))); + } + + return false; +} + /* * Acquire the content_lock for the buffer, but only if we don't have to wait. * diff --git a/src/gausskernel/storage/ipc/ipci.cpp b/src/gausskernel/storage/ipc/ipci.cpp index 832468591..0cd007b3e 100644 --- a/src/gausskernel/storage/ipc/ipci.cpp +++ b/src/gausskernel/storage/ipc/ipci.cpp @@ -463,6 +463,9 @@ void CreateSharedMemoryAndSemaphores(bool makePrivate, int port) MemoryContext oldcontext = MemoryContextSwitchTo(g_instance.audit_cxt.global_audit_context); int thread_num = g_instance.attr.attr_security.audit_thread_num; g_instance.pid_cxt.PgAuditPID = (ThreadId*)palloc0(sizeof(ThreadId) * thread_num); + if (g_instance.audit_cxt.index_file_lock == NULL) { + g_instance.audit_cxt.index_file_lock = LWLockAssign(LWTRANCHE_AUDIT_INDEX_WAIT); + } (void)MemoryContextSwitchTo(oldcontext); } diff --git a/src/gausskernel/storage/ipc/procarray.cpp b/src/gausskernel/storage/ipc/procarray.cpp index 0dc6f2557..1da108c3d 100755 --- a/src/gausskernel/storage/ipc/procarray.cpp +++ b/src/gausskernel/storage/ipc/procarray.cpp @@ -1567,7 +1567,7 @@ TransactionId GetOldestXmin(Relation rel, bool bFixRecentGlobalXmin, bool bRecen * possible. We need to do so if we're computing the global limit (rel = * NULL) or if the passed relation is a catalog relation of some kind. */ - if ((rel == NULL || RelationIsAccessibleInLogicalDecoding(rel)) && + if ((rel != NULL && RelationIsAccessibleInLogicalDecoding(rel)) && TransactionIdIsValid(replication_slot_catalog_xmin) && NormalTransactionIdPrecedes(replication_slot_catalog_xmin, result)) result = replication_slot_catalog_xmin; @@ -1627,6 +1627,21 @@ int GetMaxSnapshotSubxidCount(void) return TOTAL_MAX_CACHED_SUBXIDS; } +/* + * returns oldest transaction for catalog that was running when any current transaction was started. + * take replication slot into consideration. please make sure it's safe to read replication_slot_catalog_xmin + * before calling this func. + */ +TransactionId GetOldestCatalogXmin() +{ + TransactionId res = u_sess->utils_cxt.RecentGlobalXmin; + TransactionId repSlotCatalogXmin = g_instance.proc_array_idx->replication_slot_catalog_xmin; + if (TransactionIdIsNormal(repSlotCatalogXmin) && TransactionIdPrecedes(repSlotCatalogXmin, res)) { + return repSlotCatalogXmin; + } + return res; +} + /* * GetSnapshotData -- returns information about running transactions. * @@ -1720,7 +1735,9 @@ RETRY: goto RETRY; } u_sess->utils_cxt.RecentGlobalXmin = GetOldestXmin(NULL, true); + u_sess->utils_cxt.RecentGlobalCatalogXmin = GetOldestCatalogXmin(); } + return result; } } @@ -3683,6 +3700,7 @@ static bool GetSnapshotDataDataNode(Snapshot snapshot) gtm_snapshot->csn, snapshot->gtm_snapshot_type, SNAPSHOT_DIRECT); u_sess->utils_cxt.g_GTM_Snapshot->csn = snapshot->snapshotcsn; u_sess->utils_cxt.RecentGlobalXmin = GetOldestXmin(NULL, true); + u_sess->utils_cxt.RecentGlobalCatalogXmin = GetOldestCatalogXmin(); return true; } } @@ -3778,6 +3796,7 @@ static bool GetSnapshotDataCoordinator(Snapshot snapshot) u_sess->utils_cxt.g_GTM_Snapshot->csn = snapshot->snapshotcsn; u_sess->utils_cxt.RecentGlobalXmin = GetOldestXmin(NULL, true); + u_sess->utils_cxt.RecentGlobalCatalogXmin = GetOldestCatalogXmin(); } if (module_logging_is_on(MOD_TRANS_SNAPSHOT)) { ereport(LOG, (errmodule(MOD_TRANS_SNAPSHOT), @@ -4332,6 +4351,7 @@ Snapshot GetLocalSnapshotData(Snapshot snapshot) u_sess->utils_cxt.RecentGlobalXmin = replication_slot_xmin; } + u_sess->utils_cxt.RecentGlobalCatalogXmin = GetOldestCatalogXmin(); u_sess->utils_cxt.RecentXmin = snapxid->xmin; snapshot->xmin = snapxid->xmin; snapshot->xmax = snapxid->xmax; diff --git a/src/gausskernel/storage/lmgr/lmgr.cpp b/src/gausskernel/storage/lmgr/lmgr.cpp index 2314575b9..809db8618 100755 --- a/src/gausskernel/storage/lmgr/lmgr.cpp +++ b/src/gausskernel/storage/lmgr/lmgr.cpp @@ -1148,6 +1148,15 @@ void UnlockPartitionSeq(Oid relid, uint32 seq, LOCKMODE lockmode) (void)LockRelease(&tag, lockmode, false); } +void UnlockPartitionSeqIfHeld(Oid relid, uint32 seq, LOCKMODE lockmode) +{ + LOCKTAG tag; + + SetLocktagPartitionSeq(&tag, relid, seq); + + ReleaseLockIfHeld(&tag, lockmode, false); +} + void LockPartitionVacuum(Relation prel, Oid partId, LOCKMODE lockmode) { PartitionIdentifier* partIdentifier = NULL; diff --git a/src/gausskernel/storage/lmgr/lock.cpp b/src/gausskernel/storage/lmgr/lock.cpp index e49970366..b0dc2b148 100644 --- a/src/gausskernel/storage/lmgr/lock.cpp +++ b/src/gausskernel/storage/lmgr/lock.cpp @@ -2303,6 +2303,41 @@ void LockReleaseCurrentOwner(void) } } +/* clear all info of CurrentResourceOwner in locallock and release the lock if no other owners. */ +void ReleaseLockIfHeld(const LOCKTAG *locktag, LOCKMODE lockmode, bool sessionLock) +{ + LOCKMETHODID lockmethodid = locktag->locktag_lockmethodid; + LockMethod lockMethodTable; + LOCALLOCKTAG localtag; + LOCALLOCK *locallock = NULL; + + CHECK_LOCKMETHODID(lockmethodid); + lockMethodTable = LockMethods[lockmethodid]; + CHECK_LOCKMODE(lockmode, lockMethodTable); + +#ifdef LOCK_DEBUG + if (LOCK_DEBUG_ENABLED(locktag)) + ereport(LOG, (errmsg("ReleaseLockIfHeld: lock [%u,%u] %s", locktag->locktag_field1, locktag->locktag_field2, + lockMethodTable->lockModeNames[lockmode]))); +#endif + + /* + * Find the LOCALLOCK entry for this lock and lockmode + */ + errno_t rc = memset_s(&localtag, sizeof(localtag), 0, sizeof(localtag)); /* must clear padding */ + securec_check(rc, "", ""); + localtag.lock = *locktag; + localtag.mode = lockmode; + + locallock = (LOCALLOCK *)hash_search(t_thrd.storage_cxt.LockMethodLocalHash, (void *)&localtag, HASH_FIND, NULL); + if ((locallock == NULL) || locallock->nLocks <= 0) { + ereport(LOG, (errmsg("you don't own a lock of type %s", lockMethodTable->lockModeNames[lockmode]))); + return; + } + + ReleaseLockIfHeld(locallock, sessionLock); +} + /* * ReleaseLockIfHeld * Release any session-level locks on this lockable object if sessionLock diff --git a/src/gausskernel/storage/replication/dcf/dcf_callbackfuncs.cpp b/src/gausskernel/storage/replication/dcf/dcf_callbackfuncs.cpp index 39bde35e8..493182123 100755 --- a/src/gausskernel/storage/replication/dcf/dcf_callbackfuncs.cpp +++ b/src/gausskernel/storage/replication/dcf/dcf_callbackfuncs.cpp @@ -585,6 +585,9 @@ static void ProcessArchiveXlogMessage(uint32 srcNodeID, const char* msg, uint32 static_cast(archive_xlog_message->targetLsn)))); } volatile unsigned int *pitr_task_status = &archive_task_status->pitr_task_status; + if (archive_xlog_message->targetLsn == InvalidXLogRecPtr) { + pg_atomic_write_u32(pitr_task_status, PITR_TASK_NONE); + } unsigned int expected = PITR_TASK_NONE; int failed_times = 0; /* diff --git a/src/gausskernel/storage/replication/logical/decode.cpp b/src/gausskernel/storage/replication/logical/decode.cpp index 8d55420e3..9bca465a5 100644 --- a/src/gausskernel/storage/replication/logical/decode.cpp +++ b/src/gausskernel/storage/replication/logical/decode.cpp @@ -745,6 +745,8 @@ static void DecodeHeap3Op(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) SnapBuildProcessNewCid(builder, xid, buf->origptr, xlrec, bucket_id); break; } + case XLOG_HEAP3_INVALID: + break; case XLOG_HEAP3_REWRITE: break; default: @@ -888,7 +890,6 @@ static void AreaDecodeHeapOp(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) } } - static void DecodeUheapOp(LogicalDecodingContext *ctx, XLogRecordBuffer *buf) { uint8 info = XLogRecGetInfo(buf->record) & XLOG_UHEAP_OPMASK; @@ -972,12 +973,9 @@ static void AreaDecodeUheapOp(LogicalDecodingContext *ctx, XLogRecordBuffer *buf } } - - bool FilterByOrigin(LogicalDecodingContext *ctx, RepOriginId origin_id) { - const int toastMask = (1 << 8) - 1; - origin_id &= toastMask; + origin_id = (int)((uint32)(origin_id) & TOAST_MASK); if (ctx->callbacks.filter_by_origin_cb == NULL) return false; @@ -1041,8 +1039,6 @@ static void AreaDecodingChange(ReorderBufferChange *change, LogicalDecodingConte RelationClose(relation); } - - /* * Consolidated commit record handling between the different form of commit * records. diff --git a/src/gausskernel/storage/replication/logical/logical_parse.cpp b/src/gausskernel/storage/replication/logical/logical_parse.cpp index 7e5636477..778e37ee7 100644 --- a/src/gausskernel/storage/replication/logical/logical_parse.cpp +++ b/src/gausskernel/storage/replication/logical/logical_parse.cpp @@ -68,7 +68,6 @@ void ParseHeap3Op(ParallelLogicalDecodingContext *ctx, XLogRecordBuffer *buf, Pa void ParseUheapOp(ParallelLogicalDecodingContext *ctx, XLogRecordBuffer *buf, ParallelDecodeReaderWorker *worker); extern Pointer GetXlrec(XLogReaderState *record); -static const int toastMask = 1 << 8; /* * After initially parsing the log, put tuple change into the queue * and wait for the decoder thread to parse it. @@ -84,7 +83,7 @@ static const int toastMask = 1 << 8; static bool ParallelFilterByOrigin(ParallelLogicalDecodingContext *ctx, RepOriginId origin_id) { ParallelDecodingData* data = (ParallelDecodingData *)ctx->output_plugin_private; - origin_id &= toastMask - 1; + origin_id = (int)((uint32)(origin_id) & TOAST_MASK); if (data->pOptions.only_local && origin_id != InvalidRepOriginId) { return true; } @@ -479,6 +478,8 @@ void ParseHeap3Op(ParallelLogicalDecodingContext *ctx, XLogRecordBuffer *buf, Pa } case XLOG_HEAP3_REWRITE: break; + case XLOG_HEAP3_INVALID: + break; default: ereport(WARNING, (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), errmsg("unexpected RM_HEAP3_ID record type: %u", info))); @@ -552,10 +553,10 @@ void ParseInsertXlog(ParallelLogicalDecodingContext *ctx, XLogRecordBuffer *buf, } /* - * Reuse the origin field in the log. If bit 8 is non-zero, + * Reuse the origin field in the log. If the highest bit is non-zero, * the current table is considered to be toast table. */ - bool istoast = (((XLogRecGetOrigin(r)) & toastMask) != 0); + bool istoast = (((uint32)(XLogRecGetOrigin(r)) & TOAST_FLAG) != 0); if (istoast) { ereport(DEBUG2, (errmsg("ParallelDecodeInsert %d", istoast))); } @@ -630,7 +631,7 @@ void ParseUInsert(ParallelLogicalDecodingContext *ctx, XLogRecordBuffer *buf, Pa change->data.tp.snapshotcsn = curCSN; change->data.tp.clear_toast_afterwards = true; - bool isToast = (((XLogRecGetOrigin(r)) & toastMask) != 0); + bool isToast = (((uint32)(XLogRecGetOrigin(r)) & TOAST_FLAG) != 0); SplicingToastTuple(change, ctx, &needDecode, isToast, worker, false); if (needDecode) { PutChangeQueue(slotId, change); @@ -701,10 +702,10 @@ void ParseUpdateXlog(ParallelLogicalDecodingContext *ctx, XLogRecordBuffer *buf, return; /* - * Reuse the origin field in the log. If bit 8 is non-zero, + * Reuse the origin field in the log. If the highest bit is non-zero, * the current table is considered to be toast table. */ - bool istoast = (((XLogRecGetOrigin(r)) & toastMask) != 0); + bool istoast = (((uint32)(XLogRecGetOrigin(r)) & TOAST_FLAG) != 0); change = ParallelReorderBufferGetChange(ctx->reorder, slotId); change->action = PARALLEL_REORDER_BUFFER_CHANGE_UPDATE; @@ -767,7 +768,7 @@ void ParseUUpdate(ParallelLogicalDecodingContext *ctx, XLogRecordBuffer *buf, Pa return; } - bool isToast = (((XLogRecGetOrigin(r)) & toastMask) != 0); + bool isToast = (((uint32)(XLogRecGetOrigin(r)) & TOAST_FLAG) != 0); ParallelReorderBufferChange *change = ParallelReorderBufferGetChange(ctx->reorder, slotId); change->action = PARALLEL_REORDER_BUFFER_CHANGE_UUPDATE; change->lsn = ctx->reader->ReadRecPtr; @@ -817,7 +818,7 @@ void ParseDeleteXlog(ParallelLogicalDecodingContext *ctx, XLogRecordBuffer *buf, int rc = 0; Size datalen = 0; bool needDecode = false; - bool istoast = (((XLogRecGetOrigin(r)) & toastMask) != 0); + bool istoast = (((uint32)(XLogRecGetOrigin(r)) & TOAST_FLAG) != 0); if (istoast) { ereport(DEBUG2, (errmsg("ParallelDecodeDelete %d", istoast))); } @@ -878,7 +879,7 @@ void ParseUDelete(ParallelLogicalDecodingContext *ctx, XLogRecordBuffer *buf, Pa int slotId = worker->slotId; RelFileNode targetNode = {0, 0, 0, 0}; bool needDecode = false; - bool isToast = (((XLogRecGetOrigin(r)) & toastMask) != 0); + bool isToast = (((uint32)(XLogRecGetOrigin(r)) & TOAST_FLAG) != 0); xlrec = (XlUHeapDelete *)UGetXlrec(r); CommitSeqNo curCSN = *(CommitSeqNo *)((char *)xlrec + SizeOfUHeapDelete); diff --git a/src/gausskernel/storage/replication/logical/parallel_decode_worker.cpp b/src/gausskernel/storage/replication/logical/parallel_decode_worker.cpp index e8a98ca9f..43e774f38 100644 --- a/src/gausskernel/storage/replication/logical/parallel_decode_worker.cpp +++ b/src/gausskernel/storage/replication/logical/parallel_decode_worker.cpp @@ -983,10 +983,10 @@ int StartLogicalLogWorkers(char* dbUser, char* dbName, char* slotname, List *opt ParseDecodingOptions(&g_Logicaldispatcher[slotId].pOptions, options); errno_t rc = memcpy_s(g_Logicaldispatcher[slotId].slotName, NAMEDATALEN, slotname, strlen(slotname)); securec_check(rc, "", ""); + + StartLogicalDecodeWorkers(parallelDecodeNum, slotId, dbUser, dbName, slotname); g_Logicaldispatcher[slotId].readWorker = CreateLogicalReadWorker(slotId, dbUser, dbName, slotname, options); g_Logicaldispatcher[slotId].readWorker->tid = StartDecodeReadWorker(g_Logicaldispatcher[slotId].readWorker); - if (g_Logicaldispatcher[slotId].readWorker != NULL) - StartLogicalDecodeWorkers(parallelDecodeNum, slotId, dbUser, dbName, slotname); WaitWorkerReady(slotId); knl_g_parallel_decode_context *gDecodeCxt = g_instance.comm_cxt.pdecode_cxt; diff --git a/src/gausskernel/storage/replication/logical/parallel_reorderbuffer.cpp b/src/gausskernel/storage/replication/logical/parallel_reorderbuffer.cpp index 3991af570..9e82a7d1a 100644 --- a/src/gausskernel/storage/replication/logical/parallel_reorderbuffer.cpp +++ b/src/gausskernel/storage/replication/logical/parallel_reorderbuffer.cpp @@ -80,12 +80,14 @@ static void ParallelReorderBufferRestoreChange(ParallelReorderBuffer *rb, Parall /* Parallel decoding batch sending unit length is set to 1MB. */ static const int g_batch_unit_length = 1 * 1024 * 1024; +/* Parallel decoding caches at most 512 transactions. */ +static const Size g_max_cached_txns = 512; + void ParallelReorderBufferQueueChange(ParallelReorderBuffer *rb, logicalLog *change, int slotId) { ParallelReorderBufferTXN *txn = NULL; txn = ParallelReorderBufferTXNByXid(rb, change->xid, true, NULL, change->lsn, true); - Assert(InvalidXLogRecPtr != change->lsn); dlist_push_tail(&txn->changes, &change->node); txn->nentries++; txn->nentries_mem++; @@ -238,7 +240,13 @@ ParallelReorderBufferTXN *ParallelReorderBufferGetTXN(ParallelReorderBuffer *rb) ParallelReorderBufferTXN *txn = NULL; int rc = 0; /* check the slab cache */ - txn = (ParallelReorderBufferTXN *)palloc(sizeof(ParallelReorderBufferTXN)); + if (rb->nr_cached_transactions > 0) { + rb->nr_cached_transactions--; + txn = (ParallelReorderBufferTXN *)dlist_container(ParallelReorderBufferTXN, node, + dlist_pop_head_node(&rb->cached_transactions)); + } else { + txn = (ParallelReorderBufferTXN *)palloc(sizeof(ParallelReorderBufferTXN)); + } rc = memset_s(txn, sizeof(ParallelReorderBufferTXN), 0, sizeof(ParallelReorderBufferTXN)); securec_check(rc, "", ""); @@ -249,6 +257,7 @@ ParallelReorderBufferTXN *ParallelReorderBufferGetTXN(ParallelReorderBuffer *rb) return txn; } + void ParallelReorderBufferReturnTXN(ParallelReorderBuffer *rb, ParallelReorderBufferTXN *txn) { /* clean the lookup cache if we were cached (quite likely) */ @@ -270,6 +279,13 @@ void ParallelReorderBufferReturnTXN(ParallelReorderBuffer *rb, ParallelReorderBu txn->nentries = 0; txn->nentries_mem = 0; + /* check whether to put into the slab cache */ + if (rb->nr_cached_transactions < g_max_cached_txns) { + rb->nr_cached_transactions++; + dlist_push_head(&rb->cached_transactions, &txn->node); + } else { + pfree(txn); + } } /* --------------------------------------- * toast reassembly support @@ -1163,6 +1179,9 @@ static logicalLog *ParallelReorderBufferIterTXNNext(ParallelReorderBuffer *rb, entry = &state->entries[off]; if (!dlist_is_empty(&state->old_change)) { + change = dlist_container(logicalLog, node, dlist_pop_head_node(&state->old_change)); + FreeLogicalLog(change, slotId); + /* We should find atmost one old_change here */ Assert(dlist_is_empty(&state->old_change)); } @@ -1219,6 +1238,29 @@ static logicalLog *ParallelReorderBufferIterTXNNext(ParallelReorderBuffer *rb, return change; } +/* + * Deallocate the iterator + */ +static void ParallelReorderBufferIterTXNFinish(ParallelReorderBufferIterTXNState *state, int slotId) +{ + for (Size off = 0; off < state->nr_txns; off++) { + if (state->entries[off].fd != -1) { + (void)CloseTransientFile(state->entries[off].fd); + } + } + + /* free memory we might have "leaked" in the last *Next call */ + if (!dlist_is_empty(&state->old_change)) { + logicalLog *change = NULL; + change = dlist_container(logicalLog, node, dlist_pop_head_node(&state->old_change)); + FreeLogicalLog(change, slotId); + Assert(dlist_is_empty(&state->old_change)); + } + + binaryheap_free(state->heap); + pfree(state); +} + /* * Forget the contents of a transaction if we aren't interested in it's * contents. Needs to be first called for subtransactions and then for the @@ -1237,6 +1279,7 @@ void ParallelReorderBufferForget(ParallelReorderBuffer *rb, int slotId, Parallel FreeLogicalLog(logChange, slotId); } ParallelReorderBufferCleanupTXN(rb, txn); + ParallelReorderBufferIterTXNFinish(iterstate, slotId); } /* @@ -1440,6 +1483,7 @@ void ParallelReorderBufferCommit(ParallelReorderBuffer *rb, logicalLog *change, g_Logicaldispatcher[slotId].pOptions.sending_batch > 0); } ParallelReorderBufferCleanupTXN(rb, txn); + ParallelReorderBufferIterTXNFinish(iterstate, slotId); ParallelCheckBatch(ctx->out, pdata, slotId, &oldCtx, (pdata->pOptions.skip_empty_xacts && !pdata->pOptions.xact_wrote_changes)); @@ -1545,12 +1589,14 @@ ParallelReorderBuffer *ParallelReorderBufferAllocate(int slotId) hash_ctl.hcxt = buffer->context; const long reorderBufferNelem = 1000; - buffer->by_txn = hash_create("ReorderBufferByXid", reorderBufferNelem, + buffer->by_txn = hash_create("ParallelReorderBufferByXid", reorderBufferNelem, &hash_ctl, HASH_ELEM | HASH_FUNCTION | HASH_CONTEXT); buffer->by_txn_last_xid = InvalidTransactionId; buffer->by_txn_last_txn = NULL; + buffer->nr_cached_transactions = 0; + buffer->nr_cached_tuplebufs = 0; buffer->nr_cached_changes = 0; buffer->current_restart_decoding_lsn = InvalidXLogRecPtr; diff --git a/src/gausskernel/storage/replication/rto_statistic.cpp b/src/gausskernel/storage/replication/rto_statistic.cpp index f6b1ec3cc..b594e1de1 100644 --- a/src/gausskernel/storage/replication/rto_statistic.cpp +++ b/src/gausskernel/storage/replication/rto_statistic.cpp @@ -183,6 +183,7 @@ HadrRTOAndRPOData *HadrGetRTOStat(uint32 *num) for (i = 0; i < g_instance.attr.attr_storage.max_wal_senders; i++) { /* use volatile pointer to prevent code rearrangement */ volatile WalSnd *walsnd = &t_thrd.walsender_cxt.WalSndCtl->walsnds[i]; + SpinLockAcquire(&walsnd->mutex); if (walsnd->pid != 0 && ((strstr(g_instance.rto_cxt.rto_standby_data[i].id, "hadr_") != NULL) || (strstr(g_instance.rto_cxt.rto_standby_data[i].id, "hass") != NULL))) { char *standby_names = (char *)(result[readWalSnd].id); @@ -219,6 +220,7 @@ HadrRTOAndRPOData *HadrGetRTOStat(uint32 *num) result[readWalSnd].target_rpo = u_sess->attr.attr_storage.hadr_recovery_point_target; readWalSnd++; } + SpinLockRelease(&walsnd->mutex); } *num = readWalSnd; diff --git a/src/gausskernel/storage/replication/slot.cpp b/src/gausskernel/storage/replication/slot.cpp index 8671ac22c..e90dcf216 100755 --- a/src/gausskernel/storage/replication/slot.cpp +++ b/src/gausskernel/storage/replication/slot.cpp @@ -705,7 +705,8 @@ static void ReplicationSlotDropAcquired(void) } else { volatile ReplicationSlot *vslot = slot; - bool fail_softly = GET_SLOT_PERSISTENCY(slot->data) == RS_EPHEMERAL; + bool fail_softly = GET_SLOT_PERSISTENCY(slot->data) == RS_EPHEMERAL || + (GET_SLOT_PERSISTENCY(slot->data) == RS_PERSISTENT && slot->extra_content != NULL); SpinLockAcquire(&slot->mutex); vslot->active = false; SpinLockRelease(&slot->mutex); @@ -2171,18 +2172,22 @@ void AdvanceArchiveSlot(XLogRecPtr restart_pos) ReplicationSlot *slot = &t_thrd.slot_cxt.ReplicationSlotCtl->replication_slots[*slot_idx]; SpinLockAcquire(&slot->mutex); if (slot->in_use == true && slot->archive_config != NULL) { + t_thrd.slot_cxt.MyReplicationSlot = slot; slot->data.restart_lsn = restart_pos; extra_content = slot->extra_content; + SpinLockRelease(&slot->mutex); } else { + SpinLockRelease(&slot->mutex); ereport(WARNING, (errcode_for_file_access(), errmsg("slot idx not valid, obs slot %X/%X not advance ", (uint32)(restart_pos >> 32), (uint32)(restart_pos)))); } - SpinLockRelease(&slot->mutex); + if (extra_content == NULL) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("archive thread could not get slot extra content when advance slot."))); } + ReplicationSlotMarkDirty(); log_slot_advance(&slot->data, extra_content); } } diff --git a/src/gausskernel/storage/replication/slotfuncs.cpp b/src/gausskernel/storage/replication/slotfuncs.cpp index 0622ed650..df287d0cd 100755 --- a/src/gausskernel/storage/replication/slotfuncs.cpp +++ b/src/gausskernel/storage/replication/slotfuncs.cpp @@ -39,9 +39,11 @@ extern void *internal_load_library(const char *libname); extern bool PMstateIsRun(void); static void redo_slot_create(const ReplicationSlotPersistentData *slotInfo, char* extra_content = NULL); +#ifndef ENABLE_LITE_MODE static XLogRecPtr create_physical_replication_slot_for_backup(const char* slot_name, bool is_dummy, char* extra); static XLogRecPtr create_physical_replication_slot_for_archive(const char* slot_name, bool is_dummy, char* extra, XLogRecPtr currFlushPtr = InvalidXLogRecPtr); +#endif static void slot_advance(const char* slotname, XLogRecPtr &moveto, NameData &database, char *EndLsn, bool for_backup = false); @@ -259,6 +261,7 @@ Datum pg_create_physical_replication_slot(PG_FUNCTION_ARGS) */ Datum pg_create_physical_replication_slot_extern(PG_FUNCTION_ARGS) { +#ifndef ENABLE_LITE_MODE Name name = PG_GETARG_NAME(0); bool isDummyStandby = PG_GETARG_BOOL(1); XLogRecPtr currFlushPtr = InvalidXLogRecPtr; @@ -299,7 +302,6 @@ Datum pg_create_physical_replication_slot_extern(PG_FUNCTION_ARGS) if (for_backup) { restart_lsn = create_physical_replication_slot_for_backup(NameStr(*name), isDummyStandby, extra_content); } else { -#ifndef ENABLE_LITE_MODE if (isNeedRecycleXlog) { ArchiveConfig* archive_config = formArchiveConfigFromStr(extra_content, false); XLogRecPtr switchPtr = InvalidXLogRecPtr; @@ -318,7 +320,6 @@ Datum pg_create_physical_replication_slot_extern(PG_FUNCTION_ARGS) } currFlushPtr -= currFlushPtr % XLogSegSize; ereport(LOG, (errmsg("create archive slot, start point %lu", currFlushPtr))); -#endif restart_lsn = create_physical_replication_slot_for_archive(NameStr(*name), isDummyStandby, extra_content, currFlushPtr); } @@ -342,6 +343,10 @@ Datum pg_create_physical_replication_slot_extern(PG_FUNCTION_ARGS) pfree_ext(extra_content); PG_RETURN_DATUM(result); +#else + FEATURE_ON_LITE_MODE_NOT_SUPPORTED(); + PG_RETURN_DATUM(0); +#endif } void create_logical_replication_slot(const Name name, Name plugin, bool isDummyStandby, Oid databaseId, @@ -1103,6 +1108,7 @@ bool is_archive_slot(ReplicationSlotPersistentData data) return GET_SLOT_EXTRA_DATA_LENGTH(data) != 0; } +#ifndef ENABLE_LITE_MODE XLogRecPtr create_physical_replication_slot_for_archive(const char* slot_name, bool is_dummy, char* extra_content, XLogRecPtr currFlushPtr) { @@ -1169,6 +1175,7 @@ XLogRecPtr create_physical_replication_slot_for_backup(const char* slot_name, bo return restart_lsn; } +#endif void init_instance_slot() { diff --git a/src/gausskernel/storage/replication/walreceiver.cpp b/src/gausskernel/storage/replication/walreceiver.cpp index 9d59a2bc4..f97918018 100755 --- a/src/gausskernel/storage/replication/walreceiver.cpp +++ b/src/gausskernel/storage/replication/walreceiver.cpp @@ -1908,6 +1908,9 @@ static void ProcessArchiveXlogMessage(const ArchiveXlogMessage* archive_xlog_mes (uint32)(archive_xlog_message->targetLsn)))); } volatile unsigned int *pitr_task_status = &archive_task->pitr_task_status; + if (archive_xlog_message->targetLsn == InvalidXLogRecPtr) { + pg_atomic_write_u32(pitr_task_status, PITR_TASK_NONE); + } unsigned int expected = PITR_TASK_NONE; int failed_times = 0; /* lock for archiver get PITR_TASK_GET flag, but works on old task . diff --git a/src/gausskernel/storage/replication/walsender.cpp b/src/gausskernel/storage/replication/walsender.cpp index 2232dc1e4..87d685a58 100755 --- a/src/gausskernel/storage/replication/walsender.cpp +++ b/src/gausskernel/storage/replication/walsender.cpp @@ -3014,8 +3014,10 @@ static void LogCtrlCalculateCurrentRTO(StandbyReplyMessage *reply, bool *needRef } if ((walsnd->log_ctrl.apply_rate >> SHIFT_SPEED) > 1) { if (calculate_time_diff > CALCULATE_INTERVAL_MILLISECOND || IsRtoRpoOverTarget()) { - walsnd->log_ctrl.apply_rate = LogCtrlCountBigSpeed(walsnd->log_ctrl.apply_rate, - (uint64)(periodTotalApply / calculate_time_diff)); + if (needApply != 0) { + walsnd->log_ctrl.apply_rate = LogCtrlCountBigSpeed(walsnd->log_ctrl.apply_rate, + (uint64)(periodTotalApply / calculate_time_diff)); + } walsnd->log_ctrl.prev_calculate_time = reply->sendTime; } } else { @@ -4211,6 +4213,11 @@ static void InitWalSnd(void) rc = memset_s(g_instance.streaming_dr_cxt.targetBarrierId, MAX_BARRIER_ID_LENGTH, 0, sizeof(g_instance.streaming_dr_cxt.targetBarrierId)); securec_check(rc, "\0", "\0"); + SpinLockAcquire(&g_instance.streaming_dr_cxt.mutex); + rc = memset_s(g_instance.streaming_dr_cxt.currentBarrierId, MAX_BARRIER_ID_LENGTH, 0, + sizeof(g_instance.streaming_dr_cxt.currentBarrierId)); + SpinLockRelease(&g_instance.streaming_dr_cxt.mutex); + securec_check(rc, "\0", "\0"); } walsnd->isTermChanged = false; walsnd->peer_state = NORMAL_STATE; @@ -6875,9 +6882,11 @@ static void ProcessHadrReplyMessage() HadrReplyMessage hadrReply; pq_copymsgbytes(t_thrd.walsender_cxt.reply_message, (char*)&hadrReply, sizeof(HadrReplyMessage)); SpinLockAcquire(&g_instance.streaming_dr_cxt.mutex); - int rc = strncpy_s((char *)g_instance.streaming_dr_cxt.targetBarrierId, MAX_BARRIER_ID_LENGTH, - hadrReply.targetBarrierId, MAX_BARRIER_ID_LENGTH - 1); - securec_check(rc, "\0", "\0"); + if (BARRIER_LT((char *)g_instance.streaming_dr_cxt.targetBarrierId, (char *)hadrReply.targetBarrierId)) { + int rc = strncpy_s((char *)g_instance.streaming_dr_cxt.targetBarrierId, MAX_BARRIER_ID_LENGTH, + hadrReply.targetBarrierId, MAX_BARRIER_ID_LENGTH - 1); + securec_check(rc, "\0", "\0"); + } SpinLockRelease(&g_instance.streaming_dr_cxt.mutex); } diff --git a/src/gausskernel/storage/sync/knl_usync.cpp b/src/gausskernel/storage/sync/knl_usync.cpp index 87a627683..17f78738e 100644 --- a/src/gausskernel/storage/sync/knl_usync.cpp +++ b/src/gausskernel/storage/sync/knl_usync.cpp @@ -422,7 +422,7 @@ void ProcessSyncRequests(void) (errmsg("could not fsync file \"%s\": %m, this relation has been remove", path))); break; } - + /* * Absorb incoming requests and check to see if a cancel arrived * for this relation fork. diff --git a/src/gausskernel/storage/tcap/tcap_drop.cpp b/src/gausskernel/storage/tcap/tcap_drop.cpp index 930c7c2c7..a6cb8515c 100644 --- a/src/gausskernel/storage/tcap/tcap_drop.cpp +++ b/src/gausskernel/storage/tcap/tcap_drop.cpp @@ -1,1324 +1,1327 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * --------------------------------------------------------------------------------------- - * - * tcap_drop.cpp - * Routines to support Timecapsule `Recyclebin-based query, restore`. - * We use Tr prefix to indicate it in following coding. - * - * IDENTIFICATION - * src/gausskernel/storage/tcap/tcap_drop.cpp - * - * --------------------------------------------------------------------------------------- - */ - -#include "postgres.h" - -#include "pgstat.h" -#include "access/reloptions.h" -#include "access/sysattr.h" -#include "access/xlog.h" -#include "catalog/dependency.h" -#include "catalog/heap.h" -#include "catalog/index.h" -#include "catalog/indexing.h" -#include "catalog/objectaccess.h" -#include "catalog/pg_collation_fn.h" -#include "catalog/pg_collation.h" -#include "catalog/pg_constraint.h" -#include "catalog/pg_conversion_fn.h" -#include "catalog/pg_conversion.h" -#include "catalog/pg_depend.h" -#include "catalog/pg_extension_data_source.h" -#include "catalog/pg_extension.h" -#include "catalog/pg_foreign_data_wrapper.h" -#include "catalog/pg_foreign_server.h" -#include "catalog/pg_job.h" -#include "catalog/pg_language.h" -#include "catalog/pg_largeobject.h" -#include "catalog/pg_object.h" -#include "catalog/pg_opclass.h" -#include "catalog/pg_operator.h" -#include "catalog/pg_opfamily.h" -#include "catalog/pg_proc.h" -#include "catalog/pg_recyclebin.h" -#include "catalog/pg_rewrite.h" -#include "catalog/pg_rlspolicy.h" -#include "catalog/pg_synonym.h" -#include "catalog/pg_tablespace.h" -#include "catalog/pg_trigger.h" -#include "catalog/pg_ts_config.h" -#include "catalog/pg_ts_dict.h" -#include "catalog/pg_ts_parser.h" -#include "catalog/pg_ts_template.h" -#include "catalog/pgxc_class.h" -#include "catalog/storage.h" -#include "commands/comment.h" -#include "commands/dbcommands.h" -#include "commands/directory.h" -#include "commands/extension.h" -#include "commands/proclang.h" -#include "commands/schemacmds.h" -#include "commands/seclabel.h" -#include "commands/sec_rls_cmds.h" -#include "commands/tablecmds.h" -#include "commands/tablespace.h" -#include "commands/trigger.h" -#include "commands/typecmds.h" -#include "executor/node/nodeModifyTable.h" -#include "rewrite/rewriteRemove.h" -#include "storage/lmgr.h" -#include "storage/predicate.h" -#include "storage/smgr/relfilenode.h" -#include "utils/acl.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/inval.h" -#include "utils/lsyscache.h" -#include "utils/relcache.h" -#include "utils/snapmgr.h" -#include "utils/syscache.h" - -#include "storage/tcap.h" -#include "storage/tcap_impl.h" - -static void TrRenameClass(TrObjDesc *baseDesc, ObjectAddress *object, const char *newName) -{ - Relation rel; - HeapTuple tup; - HeapTuple newtup; - char rbname[NAMEDATALEN]; - Datum values[Natts_pg_class] = { 0 }; - bool nulls[Natts_pg_class] = { false }; - bool replaces[Natts_pg_class] = { false }; - Oid relid = object->objectId; - errno_t rc = EOK; - - if (newName) { - rc = strncpy_s(rbname, NAMEDATALEN, newName, strlen(newName)); - securec_check(rc, "\0", "\0"); - } else { - TrGenObjName(rbname, object->classId, relid); - } - - replaces[Anum_pg_class_relname - 1] = true; - values[Anum_pg_class_relname - 1] = CStringGetDatum(rbname); - - rel = heap_open(RelationRelationId, RowExclusiveLock); - - tup = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); - if (!HeapTupleIsValid(tup)) { - ereport(ERROR, (errcode(ERRCODE_CACHE_LOOKUP_FAILED), errmsg("cache lookup failed for relation %u", relid))); - } - - newtup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); - - simple_heap_update(rel, &newtup->t_self, newtup); - - CatalogUpdateIndexes(rel, newtup); - - ReleaseSysCache(tup); - - heap_freetuple_ext(newtup); - - heap_close(rel, RowExclusiveLock); -} - -static void TrRenameCommon(TrObjDesc *baseDesc, ObjectAddress *object, Oid relid, int natts, int oidAttrNum, - Oid oidIndexId, char *objTag) -{ - Relation rel; - HeapTuple tup; - HeapTuple newtup; - char rbname[NAMEDATALEN]; - Datum *values = (Datum *)palloc0(sizeof(Datum) * natts); - bool *nulls = (bool *)palloc0(sizeof(bool) * natts); - bool *replaces = (bool *)palloc0(sizeof(bool) * natts); - ScanKeyData skey[1]; - SysScanDesc sd; - - TrGenObjName(rbname, object->classId, object->objectId); - - replaces[oidAttrNum - 1] = true; - values[oidAttrNum - 1] = CStringGetDatum(rbname); - - rel = heap_open(relid, RowExclusiveLock); - - ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(object->objectId)); - sd = systable_beginscan(rel, oidIndexId, true, NULL, 1, skey); - - tup = systable_getnext(sd); - if (!HeapTupleIsValid(tup)) { - pfree(values); - pfree(nulls); - pfree(replaces); - ereport(ERROR, - (errcode(ERRCODE_NO_DATA_FOUND), errmsg("could not find tuple for %s %u", objTag, object->objectId))); - } - - newtup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); - - simple_heap_update(rel, &newtup->t_self, newtup); - - CatalogUpdateIndexes(rel, newtup); - - heap_freetuple_ext(newtup); - - systable_endscan(sd); - - heap_close(rel, RowExclusiveLock); - - pfree(values); - pfree(nulls); - pfree(replaces); -} - -static void TrDeleteBaseid(Oid baseid) -{ - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[1]; - HeapTuple tup; - - rbRel = heap_open(RecyclebinRelationId, RowExclusiveLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcybaseid, BTEqualStrategyNumber, F_INT8EQ, ObjectIdGetDatum(baseid)); - - sd = systable_beginscan(rbRel, RecyclebinBaseidIndexId, true, NULL, 1, skey); - while (HeapTupleIsValid(tup = systable_getnext(sd))) { - simple_heap_delete(rbRel, &tup->t_self); - } - - systable_endscan(sd); - heap_close(rbRel, RowExclusiveLock); - - /* - * CommandCounterIncrement here to ensure that preceding changes are all - * visible to the next deletion step. - */ - CommandCounterIncrement(); -} - -static void TrDeleteId(Oid id) -{ - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[1]; - HeapTuple tup; - - rbRel = heap_open(RecyclebinRelationId, RowExclusiveLock); - - ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(id)); - - sd = systable_beginscan(rbRel, RecyclebinIdIndexId, true, NULL, 1, skey); - if (HeapTupleIsValid(tup = systable_getnext(sd))) { - simple_heap_delete(rbRel, &tup->t_self); - } - - systable_endscan(sd); - heap_close(rbRel, RowExclusiveLock); - - /* - * CommandCounterIncrement here to ensure that preceding changes are all - * visible to the next deletion step. - */ - CommandCounterIncrement(); -} - -static inline bool TrNeedLogicDrop(const ObjectAddress *object) -{ - return object->rbDropMode == RB_DROP_MODE_LOGIC; -} - -static bool TrCanPurge(const TrObjDesc *baseDesc, const ObjectAddress *object, char relKind) -{ - Relation depRel; - SysScanDesc sd; - HeapTuple tuple; - ScanKeyData key[3]; - int nkeys; - bool found = false; - - if (relKind != RELKIND_INDEX && relKind != RELKIND_GLOBAL_INDEX && relKind != RELKIND_RELATION) { - return false; - } - - depRel = heap_open(DependRelationId, AccessShareLock); - - ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(object->classId)); - ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(object->objectId)); - nkeys = 2; - if (object->objectSubId != 0) { - ScanKeyInit(&key[2], Anum_pg_depend_objsubid, BTEqualStrategyNumber, F_INT4EQ, - Int32GetDatum(object->objectSubId)); - nkeys = 3; - } - - sd = systable_beginscan(depRel, DependDependerIndexId, true, NULL, nkeys, key); - while (HeapTupleIsValid(tuple = systable_getnext(sd))) { - Form_pg_depend depForm = (Form_pg_depend)GETSTRUCT(tuple); - if (depForm->refclassid == RelationRelationId && depForm->refobjid == baseDesc->relid) { - if (depForm->deptype != DEPENDENCY_AUTO) { - found = false; - break; - } - found = true; - } - } - - systable_endscan(sd); - heap_close(depRel, AccessShareLock); - return found; -} - -static void TrDoDropIndex(TrObjDesc *baseDesc, ObjectAddress *object) -{ - Assert(object->objectSubId == 0); - - if (TrNeedLogicDrop(object)) { - TrObjDesc desc = *baseDesc; - - if (!TR_IS_BASE_OBJ(baseDesc, object)) { - /* Deletion lock already accquired before single object drop. */ - Relation rel = relation_open(object->objectId, NoLock); - - TrDescInit(rel, &desc, RB_OPER_DROP, TrGetObjType(RelationGetNamespace(rel), RELKIND_INDEX), - TrCanPurge(baseDesc, object, RelationGetRelkind(rel))); - relation_close(rel, NoLock); - - TrDescWrite(&desc); - } - - TrRenameClass(baseDesc, object, desc.name); - } else { - index_drop(object->objectId, false); - } - - return; -} - -static void TrDoDropTable(TrObjDesc *baseDesc, ObjectAddress *object, char relKind) -{ - if (TrNeedLogicDrop(object)) { - TrObjDesc desc; - - if (object->objectSubId != 0 || relKind == RELKIND_VIEW || relKind == RELKIND_COMPOSITE_TYPE || - relKind == RELKIND_FOREIGN_TABLE) { - TrRenameClass(baseDesc, object, NULL); - return; - } - - desc = *baseDesc; - if (!TR_IS_BASE_OBJ(baseDesc, object)) { - /* Deletion lock already accquired before single object drop. */ - Relation rel = relation_open(object->objectId, NoLock); - - TrDescInit(rel, &desc, RB_OPER_DROP, TrGetObjType(InvalidOid, relKind), - TrCanPurge(baseDesc, object, relKind)); - relation_close(rel, NoLock); - - TrDescWrite(&desc); - } - - TrRenameClass(baseDesc, object, desc.name); - - return; - } - - /* - * relation_open() must be before the heap_drop_with_catalog(). If you reload - * relation after drop, it may cause other exceptions during the drop process. - */ - if (object->objectSubId != 0) - RemoveAttributeById(object->objectId, object->objectSubId); - else - heap_drop_with_catalog(object->objectId); - - /* - * IMPORANT: The relation must not be reloaded after heap_drop_with_catalog() - * is executed to drop this relation.If you reload relation after drop, it may - * cause other exceptions during the drop process - */ - - return; -} - -static void TrDoDropType(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, TypeRelationId, Natts_pg_type, Anum_pg_type_typname, TypeOidIndexId, "type"); - } else { - RemoveTypeById(object->objectId); - } - - return; -} - -static void TrDoDropConstraint(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, ConstraintRelationId, Natts_pg_constraint, Anum_pg_constraint_conname, - ConstraintOidIndexId, "constraint"); - } else { - RemoveConstraintById(object->objectId); - } - - return; -} - -static void TrDoDropTrigger(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, TriggerRelationId, Natts_pg_trigger, Anum_pg_trigger_tgname, TriggerOidIndexId, - "trigger"); - } else { - RemoveTriggerById(object->objectId); - } - - return; -} - -static void TrDoDropRewrite(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - /* nothing to do as rule-based view requires that origin rule name preserved. */ - } else { - RemoveRewriteRuleById(object->objectId); - } - - return; -} - -static void TrDoDropAttrdef(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - /* nothing to do as no name attribute in system catalog */ - } else { - RemoveAttrDefaultById(object->objectId); - } - - return; -} - -static void TrDoDropProc(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, ProcedureRelationId, Natts_pg_proc, Anum_pg_proc_proname, ProcedureOidIndexId, - "procedure"); - } else { - RemoveFunctionById(object->objectId); - } - - return; -} - -static void TrDoDropCast(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - /* nothing to do as no name attribute in system catalog */ - } else { - DropCastById(object->objectId); - } - - return; -} - -static void TrDoDropCollation(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, CollationRelationId, Natts_pg_collation, Anum_pg_collation_collname, - CollationOidIndexId, "collation"); - } else { - RemoveCollationById(object->objectId); - } - - return; -} - -static void TrDoDropConversion(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, ConversionRelationId, Natts_pg_conversion, Anum_pg_conversion_conname, - ConversionOidIndexId, "conversion"); - } else { - RemoveConversionById(object->objectId); - } - - return; -} - - -static void TrDoDropProceduralLanguage(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, LanguageRelationId, Natts_pg_language, Anum_pg_language_lanname, - LanguageOidIndexId, "language"); - } else { - DropProceduralLanguageById(object->objectId); - } - - return; -} - -static void TrDoDropLargeObject(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - /* nothing to do as no name attribute in system catalog */ - } else { - LargeObjectDrop(object->objectId); - } - - return; -} - -static void TrDoDropOperator(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, OperatorRelationId, Natts_pg_operator, Anum_pg_operator_oprname, - OperatorOidIndexId, "operator"); - } else { - RemoveOperatorById(object->objectId); - } - - return; -} - -static void TrDoDropOpClass(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, OperatorClassRelationId, Natts_pg_opclass, Anum_pg_opclass_opcname, - OpclassOidIndexId, "opclass"); - } else { - RemoveOpClassById(object->objectId); - } - - return; -} - -static void TrDoDropOpFamily(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, OperatorFamilyRelationId, Natts_pg_opfamily, Anum_pg_opfamily_opfname, - OpfamilyOidIndexId, "opfamily"); - } else { - RemoveOpFamilyById(object->objectId); - } - - return; -} - - -static void TrDoDropAmOp(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - /* nothing to do as no name attribute in system catalog */ - } else { - RemoveAmOpEntryById(object->objectId); - } - - return; -} - -static void TrDoDropAmProc(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - /* nothing to do as no name attribute in system catalog */ - } else { - RemoveAmProcEntryById(object->objectId); - } - - return; -} - -static void TrDoDropSchema(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, NamespaceRelationId, Natts_pg_namespace, Anum_pg_namespace_nspname, - NamespaceOidIndexId, "namespace"); - } else { - RemoveSchemaById(object->objectId); - } - - return; -} - -static void TrDoDropTSParser(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, TSParserRelationId, Natts_pg_ts_parser, Anum_pg_ts_parser_prsname, - TSParserOidIndexId, "ts parser"); - } else { - RemoveTSParserById(object->objectId); - } - - return; -} - -static void TrDoDropTSDictionary(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, TSDictionaryRelationId, Natts_pg_ts_dict, Anum_pg_ts_dict_dictname, - TSDictionaryOidIndexId, "ts dictionary"); - } else { - RemoveTSDictionaryById(object->objectId); - } - - return; -} - -static void TrDoDropTSTemplate(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, TSTemplateRelationId, Natts_pg_ts_template, Anum_pg_ts_template_tmplname, - TSTemplateOidIndexId, "ts template"); - } else { - RemoveTSTemplateById(object->objectId); - } - - return; -} - -static void TrDoDropTSConfiguration(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, TSConfigRelationId, Natts_pg_ts_config, Anum_pg_ts_config_cfgname, - TSConfigOidIndexId, "ts configuration"); - } else { - RemoveTSConfigurationById(object->objectId); - } - - return; -} - -static void TrDoDropForeignDataWrapper(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, ForeignDataWrapperRelationId, Natts_pg_foreign_data_wrapper, - Anum_pg_foreign_data_wrapper_fdwname, ForeignDataWrapperOidIndexId, "foreign data wrapper"); - } else { - RemoveForeignDataWrapperById(object->objectId); - } - - return; -} - -static void TrDoDropForeignServer(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, ForeignServerRelationId, Natts_pg_foreign_server, - Anum_pg_foreign_server_srvname, ForeignServerOidIndexId, "foreign server"); - } else { - RemoveForeignServerById(object->objectId); - } - - return; -} - -static void TrDoDropUserMapping(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - /* nothing to do as no name attribute in system catalog */ - } else { - RemoveUserMappingById(object->objectId); - } - - return; -} - - -static void TrDoDropDefaultACL(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - /* nothing to do as no name attribute in system catalog */ - } else { - RemoveDefaultACLById(object->objectId); - } - - return; -} - -static void TrDoDropPgxcClass(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - /* nothing to do as no name attribute in system catalog */ - } else { - RemovePgxcClass(object->objectId); - } - - return; -} - -static void TrDoDropExtension(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, ExtensionRelationId, Natts_pg_extension, Anum_pg_extension_extname, - ExtensionOidIndexId, "extension"); - } else { - RemoveExtensionById(object->objectId); - } - - return; -} - -static void TrDoDropDataSource(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, DataSourceRelationId, Natts_pg_extension_data_source, - Anum_pg_extension_data_source_srcname, DataSourceOidIndexId, "extension data source"); - } else { - RemoveDataSourceById(object->objectId); - } - - return; -} - -static void TrDoDropDirectory(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, PgDirectoryRelationId, Natts_pg_directory, Anum_pg_directory_directory_name, - PgDirectoryOidIndexId, "directory"); - } else { - RemoveDirectoryById(object->objectId); - } - - return; -} - -static void TrDoDropRlsPolicy(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, RlsPolicyRelationId, Natts_pg_rlspolicy, Anum_pg_rlspolicy_polname, - PgRlspolicyOidIndex, "rlspolicy"); - } else { - RemoveRlsPolicyById(object->objectId); - } - - return; -} - -static void TrDoDropJob(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - /* nothing to do as no name attribute in system catalog */ - } else { - RemoveJobById(object->objectId); - } - - return; -} - -static void TrDoDropSynonym(TrObjDesc *baseDesc, ObjectAddress *object) -{ - if (TrNeedLogicDrop(object)) { - TrRenameCommon(baseDesc, object, PgSynonymRelationId, Natts_pg_synonym, Anum_pg_synonym_synname, - SynonymOidIndexId, "synonym"); - } else { - RemoveSynonymById(object->objectId); - } - - return; -} - -/* - * doDeletion: delete a single object - * return false if logic deleted, - * return true if physical deleted, - */ -static void TrDoDrop(TrObjDesc *baseDesc, ObjectAddress *object) -{ - switch (getObjectClass(object)) { - case OCLASS_CLASS: { - char relKind = get_rel_relkind(object->objectId); - if (relKind == RELKIND_INDEX) { - TrDoDropIndex(baseDesc, object); - } else { - /* - * We use a unified entry for others: - * RELKIND_RELATION, RELKIND_SEQUENCE, - * RELKIND_TOASTVALUE, RELKIND_VIEW, - * RELKIND_COMPOSITE_TYPE, RELKIND_FOREIGN_TABLE - */ - TrDoDropTable(baseDesc, object, relKind); - } - break; - } - - case OCLASS_TYPE: - TrDoDropType(baseDesc, object); - break; - - case OCLASS_CONSTRAINT: - TrDoDropConstraint(baseDesc, object); - break; - - case OCLASS_TRIGGER: - TrDoDropTrigger(baseDesc, object); - break; - - case OCLASS_REWRITE: - TrDoDropRewrite(baseDesc, object); - break; - - case OCLASS_DEFAULT: - TrDoDropAttrdef(baseDesc, object); - break; - - case OCLASS_PROC: - TrDoDropProc(baseDesc, object); - break; - - case OCLASS_CAST: - TrDoDropCast(baseDesc, object); - break; - - case OCLASS_COLLATION: - TrDoDropCollation(baseDesc, object); - break; - - case OCLASS_CONVERSION: - TrDoDropConversion(baseDesc, object); - break; - - case OCLASS_LANGUAGE: - TrDoDropProceduralLanguage(baseDesc, object); - break; - - case OCLASS_LARGEOBJECT: - TrDoDropLargeObject(baseDesc, object); - break; - - case OCLASS_OPERATOR: - TrDoDropOperator(baseDesc, object); - break; - - case OCLASS_OPCLASS: - TrDoDropOpClass(baseDesc, object); - break; - - case OCLASS_OPFAMILY: - TrDoDropOpFamily(baseDesc, object); - break; - - case OCLASS_AMOP: - TrDoDropAmOp(baseDesc, object); - break; - - case OCLASS_AMPROC: - TrDoDropAmProc(baseDesc, object); - break; - - case OCLASS_SCHEMA: - TrDoDropSchema(baseDesc, object); - break; - - case OCLASS_TSPARSER: - TrDoDropTSParser(baseDesc, object); - break; - - case OCLASS_TSDICT: - TrDoDropTSDictionary(baseDesc, object); - break; - - case OCLASS_TSTEMPLATE: - TrDoDropTSTemplate(baseDesc, object); - break; - - case OCLASS_TSCONFIG: - TrDoDropTSConfiguration(baseDesc, object); - break; - - /* - * OCLASS_ROLE, OCLASS_DATABASE, OCLASS_TBLSPACE intentionally not - * handled here - */ - - case OCLASS_FDW: - TrDoDropForeignDataWrapper(baseDesc, object); - break; - - case OCLASS_FOREIGN_SERVER: - TrDoDropForeignServer(baseDesc, object); - break; - - case OCLASS_USER_MAPPING: - TrDoDropUserMapping(baseDesc, object); - break; - - case OCLASS_DEFACL: - TrDoDropDefaultACL(baseDesc, object); - break; - - case OCLASS_PGXC_CLASS: - TrDoDropPgxcClass(baseDesc, object); - break; - - case OCLASS_EXTENSION: - TrDoDropExtension(baseDesc, object); - break; - - case OCLASS_DATA_SOURCE: - TrDoDropDataSource(baseDesc, object); - break; - - case OCLASS_DIRECTORY: - TrDoDropDirectory(baseDesc, object); - break; - - case OCLASS_RLSPOLICY: - TrDoDropRlsPolicy(baseDesc, object); - break; - - case OCLASS_PG_JOB: - if ((IS_PGXC_COORDINATOR && !IsConnFromCoord()) || (g_instance.role == VSINGLENODE)) - TrDoDropJob(baseDesc, object); - break; - - case OCLASS_SYNONYM: - TrDoDropSynonym(baseDesc, object); - break; - - default: - ereport(ERROR, - (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), errmsg("unrecognized object class: %u", object->classId))); - break; - } - - return; -} - -/* - * deleteOneObject: delete a single object for TrDrop. - * - * *depRel is the already-open pg_depend relation. - */ -static void TrDropOneObject(TrObjDesc *baseDesc, ObjectAddress *object, Relation *depRel) -{ - ScanKeyData key[3]; - int nkeys; - SysScanDesc scan; - HeapTuple tup; - - /* DROP hook of the objects being removed */ - if (object_access_hook) { - ObjectAccessDrop dropArg; - - dropArg.dropflags = PERFORM_DELETION_INVALID; - InvokeObjectAccessHook(OAT_DROP, object->classId, object->objectId, object->objectSubId, &dropArg); - } - - /* - * Delete the object itself, in an object-type-dependent way. - * - * We used to do this after removing the outgoing dependency links, but it - * seems just as reasonable to do it beforehand. In the concurrent case - * we *must *do it in this order, because we can't make any transactional - * updates before calling doDeletion() --- they'd get committed right - * away, which is not cool if the deletion then fails. - */ - TrDoDrop(baseDesc, object); - - /* - * In logical drop mode, we will keep all related system entries, including - * linked entries such as pg_depend records. It is done! - */ - if (TrNeedLogicDrop(object)) { - /* - * CommandCounterIncrement here to ensure that preceding changes are all - * visible to the next deletion step. - */ - CommandCounterIncrement(); - - /* - * Logic Drop done! - */ - return; - } - - /* - * In physical drop mode, we continue to remove all related system entries. - */ - - /* - * Now remove any pg_depend records that link from this object to others. - * (Any records linking to this object should be gone already.) - * - * When dropping a whole object (subId = 0), remove all pg_depend records - * for its sub-objects too. - */ - ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(object->classId)); - ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(object->objectId)); - if (object->objectSubId != 0) { - ScanKeyInit(&key[2], Anum_pg_depend_objsubid, BTEqualStrategyNumber, F_INT4EQ, - Int32GetDatum(object->objectSubId)); - nkeys = 3; - } else - nkeys = 2; - - scan = systable_beginscan(*depRel, DependDependerIndexId, true, NULL, nkeys, key); - - while (HeapTupleIsValid(tup = systable_getnext(scan))) { - simple_heap_delete(*depRel, &tup->t_self); - } - - systable_endscan(scan); - - /* - * Delete shared dependency references related to this object. Again, if - * subId = 0, remove records for sub-objects too. - */ - deleteSharedDependencyRecordsFor(object->classId, object->objectId, object->objectSubId); - - /* - * Delete any comments or security labels associated with this object. - * (This is a convenient place to do these things, rather than having - * every object type know to do it.) - */ - DeleteComments(object->objectId, object->classId, object->objectSubId); - DeleteSecurityLabel(object); - - /* - * CommandCounterIncrement here to ensure that preceding changes are all - * visible to the next deletion step. - */ - CommandCounterIncrement(); - - /* - * Physical Drop done! - */ -} - -static bool TrObjIsInList(const ObjectAddresses *targetObjects, const ObjectAddress *thisobj) -{ - ObjectAddress *item = NULL; - - for (int i = 0; i < targetObjects->numrefs; i++) { - item = targetObjects->refs + i; - if (TrObjIsEqual(thisobj, item)) { - return true; - } - } - return false; -} - -static ObjectAddress *TrFindIdxInTarget(ObjectAddresses *targetObjects, ObjectAddress *item) -{ - ObjectAddress *thisobj = NULL; - - for (int i = 0; i < targetObjects->numrefs; i++) { - thisobj = targetObjects->refs + i; - if (TrObjIsEqual(item, thisobj)) { - return thisobj; - } - } - - return NULL; -} - -/* - * output: refthisobjs - */ -static void TrFindAllSubObjs(Relation depRel, const ObjectAddress *refobj, ObjectAddresses *refthisobjs) -{ - SysScanDesc sd; - HeapTuple tuple; - ScanKeyData key[3]; - int nkeys; - - ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(refobj->classId)); - ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(refobj->objectId)); - nkeys = 2; - if (refobj->objectSubId != 0) { - ScanKeyInit(&key[2], Anum_pg_depend_refobjsubid, BTEqualStrategyNumber, F_INT4EQ, - Int32GetDatum(refobj->objectSubId)); - nkeys = 3; - } - - sd = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, nkeys, key); - while (HeapTupleIsValid(tuple = systable_getnext(sd))) { - Form_pg_depend depForm = (Form_pg_depend)GETSTRUCT(tuple); - - /* add the refs to list */ - add_object_address_ext(depForm->classid, depForm->objid, depForm->objsubid, depForm->deptype, refthisobjs); - } - - systable_endscan(sd); - return; -} - -static void TrTagPhyDeleteSubObjs(Relation depRel, ObjectAddresses *targetObjects, ObjectAddress *thisobj) -{ - ObjectAddress *item = NULL; - - ObjectAddresses *refthisobjs = new_object_addresses(); - - /* Tag this obj RB_DROP_MODE_PHYSICAL */ - thisobj->rbDropMode = RB_DROP_MODE_PHYSICAL; - - /* Find all sub objs refered to this obj */ - TrFindAllSubObjs(depRel, thisobj, refthisobjs); - - for (int i = 0; i < refthisobjs->numrefs; i++) { - item = refthisobjs->refs + i; - - /* the item must exists in targetObjects. */ - item = TrFindIdxInTarget(targetObjects, item); - if (item == NULL || item->rbDropMode == RB_DROP_MODE_PHYSICAL) { - continue; - } - TrTagPhyDeleteSubObjs(depRel, targetObjects, item); - } - - free_object_addresses(refthisobjs); - return; -} - -static bool TrNeedPhyDelete(Relation depRel, ObjectAddresses *targetObjects, ObjectAddress *thisobj) -{ - ObjectAddress *item = NULL; - ObjectAddresses *refobjs = new_object_addresses(); - bool result = false; - - /* Find all objs this obj refered */ - TrFindAllRefObjs(depRel, thisobj, refobjs); - - /* Step 1: tag refobjs of thisobj, return directly if ALL refobjs not need physical drop. */ - for (int i = 0; i < refobjs->numrefs; i++) { - item = refobjs->refs + i; - if (!TrObjIsInList(targetObjects, item)) { - result = true; - break; - } - } - if (!result) { - free_object_addresses(refobjs); - return result; - } - - /* Step 2: tag refobjs with 'i' deptype to physical drop. */ - for (int i = 0; i < refobjs->numrefs; i++) { - item = refobjs->refs + i; - if (item->deptype == 'i') { - item = TrFindIdxInTarget(targetObjects, item); - Assert(item != NULL); - if (item->rbDropMode == RB_DROP_MODE_PHYSICAL) { - continue; - } - TrTagPhyDeleteSubObjs(depRel, targetObjects, item); - } - } - - free_object_addresses(refobjs); - return result; -} - -static void TrResetDropMode(const ObjectAddresses *targetObjects, const ObjectAddress *baseObj) -{ - ObjectAddress *thisobj = NULL; - - for (int i = 0; i < targetObjects->numrefs; i++) { - thisobj = targetObjects->refs + i; - if (TrObjIsEqual(thisobj, baseObj)) { - thisobj->rbDropMode = RB_DROP_MODE_LOGIC; - continue; - } - thisobj->rbDropMode = RB_DROP_MODE_INVALID; - } - return; -} - -static void TrTagDependentObjects(Relation depRel, ObjectAddresses *targetObjects, const ObjectAddress *baseObj) -{ - ObjectAddress *thisobj = NULL; - - TrResetDropMode(targetObjects, baseObj); - for (int i = 0; i < targetObjects->numrefs; i++) { - thisobj = targetObjects->refs + i; - if (TrDropModeIsAlreadySet(thisobj)) { - continue; - } - - if (TrNeedPhyDelete(depRel, targetObjects, thisobj)) { - TrTagPhyDeleteSubObjs(depRel, targetObjects, thisobj); - } else { - thisobj->rbDropMode = RB_DROP_MODE_LOGIC; - } - } - - return; -} - -bool TrCheckRecyclebinDrop(const DropStmt *stmt, ObjectAddresses *objects) -{ - Relation depRel; - bool rbDrop = false; - - /* No work if no objects... */ - if (objects->numrefs <= 0) - return false; - - if (/* - * Disable Recyclebin-based-Drop when target object is not OBJECT_TABLE, or - */ - stmt->removeType != OBJECT_TABLE || - /* in concurrent drop mode, or */ - stmt->concurrent || - /* with purge option, or */ - stmt->purge || - /* multi objects drop. */ - list_length(stmt->objects) != 1) { - return false; - } - - if (!NeedTrComm(objects->refs->objectId)) { - return false; - } - - depRel = heap_open(DependRelationId, AccessShareLock); - rbDrop = !TrNeedPhyDelete(depRel, objects, &objects->refs[0]); - heap_close(depRel, AccessShareLock); - - return rbDrop; -} - -void TrDrop(const DropStmt* drop, const ObjectAddresses *objects, DropBehavior behavior) -{ - Relation depRel; - Relation baseRel; - TrObjDesc baseDesc; - ObjectAddresses *targetObjects = NULL; - ObjectAddress *baseObj = objects->refs; - - /* - * We save some cycles by opening pg_depend just once and passing the - * Relation pointer down to all the recursive deletion steps. - */ - depRel = heap_open(DependRelationId, RowExclusiveLock); - - /* - * Construct a list of objects to delete (ie, the given objects plus - * everything directly or indirectly dependent on them). Note that - * because we pass the whole objects list as pendingObjects context, we - * won't get a failure from trying to delete an object that is internally - * dependent on another one in the list; we'll just skip that object and - * delete it when we reach its owner. - */ - targetObjects = new_object_addresses(); - - /* - * Acquire deletion lock on each target object. (Ideally the caller - * has done this already, but many places are sloppy about it.) - */ - AcquireDeletionLock(baseObj, PERFORM_DELETION_INVALID); - - /* - * Finds all subobjects that reference the base table recursively. - */ - findDependentObjects(baseObj, DEPFLAG_ORIGINAL, NULL, /* empty stack */ - targetObjects, objects, &depRel); - ereport(LOG, (errmsg("Delete object %u/%u/%d", baseObj->classId, baseObj->objectId, baseObj->objectSubId))); - - /* - * Check if deletion is allowed, and report about cascaded deletes. - * - * If there's exactly one object being deleted, report it the same way as - * in performDeletion(), else we have to be vaguer. - */ - reportDependentObjects(targetObjects, behavior, NOTICE, baseObj); - - /* - * Tag all subobjects' drop mode: LOGIC_DROP, PYHSICAL_DROP. - */ - TrTagDependentObjects(depRel, targetObjects, baseObj); - - /* - * Initialize the baseDesc structure so that the logic dropped subobjects - * can be correctly processed when renamed or placed in recycle bin. Notice - * that base object already locked. - */ - baseRel = relation_open(baseObj->objectId, NoLock); - TrDescInit(baseRel, &baseDesc, RB_OPER_DROP, RB_OBJ_TABLE, true, true); - baseDesc.id = baseDesc.baseid = TrDescWrite(&baseDesc); - TrUpdateBaseid(&baseDesc); - relation_close(baseRel, NoLock); - - /* - * Drop all the objects in the proper order. - */ - for (int i = 0; i < targetObjects->numrefs; i++) { - ObjectAddress *thisobj = targetObjects->refs + i; - TrDropOneObject(&baseDesc, thisobj, &depRel); - } - - /* And clean up */ - free_object_addresses(targetObjects); - heap_close(depRel, RowExclusiveLock); -} - -void TrDoPurgeObjectDrop(TrObjDesc *desc) -{ - ObjectAddresses *objects; - ObjectAddress obj; - - objects = new_object_addresses(); - - obj.classId = RelationRelationId; - obj.objectId = desc->relid; - obj.objectSubId = 0; - add_exact_object_address(&obj, objects); - - performMultipleDeletions(objects, DROP_CASCADE, PERFORM_DELETION_INVALID); - - if (desc->type == RB_OBJ_TABLE) { - TrDeleteBaseid(desc->baseid); - } else { /* RB_OBJ_INDEX */ - TrDeleteId(desc->id); - } - - free_object_addresses(objects); - return; -} - -/* TIMECAPSULE TABLE { table_name } TO BEFORE DROP [RENAME TO new_tablename] */ -void TrRestoreDrop(const TimeCapsuleStmt *stmt) -{ - TrObjDesc desc; - ObjectAddress obj; - Relation rel; - - desc.relid = 0; - TrOperFetch(stmt->relation, RB_OBJ_TABLE, &desc, RB_OPER_RESTORE_DROP); - if (desc.relid != 0 && (desc.type == RB_OBJ_TABLE)) { - stmt->relation->relname = desc.name; - rel = heap_openrv(stmt->relation, AccessExclusiveLock); - if (rel->rd_tam_type == TAM_HEAP) { - heap_close(rel, NoLock); - elog(ERROR, "timecapsule does not support astore yet"); - return; - } - heap_close(rel, NoLock); - } - - desc.authid = GetUserId(); - TrOperPrep(&desc, RB_OPER_RESTORE_DROP); - - obj.classId = RelationRelationId; - obj.objectId = desc.relid; - obj.objectSubId = 0; - - TrRenameClass(&desc, &obj, stmt->new_relname ? stmt->new_relname : desc.originname); - - TrDeleteBaseid(desc.baseid); - - return; -} +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * tcap_drop.cpp + * Routines to support Timecapsule `Recyclebin-based query, restore`. + * We use Tr prefix to indicate it in following coding. + * + * IDENTIFICATION + * src/gausskernel/storage/tcap/tcap_drop.cpp + * + * --------------------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "pgstat.h" +#include "access/reloptions.h" +#include "access/sysattr.h" +#include "access/xlog.h" +#include "catalog/dependency.h" +#include "catalog/heap.h" +#include "catalog/index.h" +#include "catalog/indexing.h" +#include "catalog/objectaccess.h" +#include "catalog/pg_collation_fn.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_constraint.h" +#include "catalog/pg_conversion_fn.h" +#include "catalog/pg_conversion.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_extension_data_source.h" +#include "catalog/pg_extension.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_job.h" +#include "catalog/pg_language.h" +#include "catalog/pg_largeobject.h" +#include "catalog/pg_object.h" +#include "catalog/pg_opclass.h" +#include "catalog/pg_operator.h" +#include "catalog/pg_opfamily.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_recyclebin.h" +#include "catalog/pg_rewrite.h" +#include "catalog/pg_rlspolicy.h" +#include "catalog/pg_synonym.h" +#include "catalog/pg_tablespace.h" +#include "catalog/pg_trigger.h" +#include "catalog/pg_ts_config.h" +#include "catalog/pg_ts_dict.h" +#include "catalog/pg_ts_parser.h" +#include "catalog/pg_ts_template.h" +#include "catalog/pgxc_class.h" +#include "catalog/storage.h" +#include "commands/comment.h" +#include "commands/dbcommands.h" +#include "commands/directory.h" +#include "commands/extension.h" +#include "commands/proclang.h" +#include "commands/schemacmds.h" +#include "commands/seclabel.h" +#include "commands/sec_rls_cmds.h" +#include "commands/tablecmds.h" +#include "commands/tablespace.h" +#include "commands/trigger.h" +#include "commands/typecmds.h" +#include "executor/node/nodeModifyTable.h" +#include "rewrite/rewriteRemove.h" +#include "storage/lmgr.h" +#include "storage/predicate.h" +#include "storage/smgr/relfilenode.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/inval.h" +#include "utils/lsyscache.h" +#include "utils/relcache.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" + +#include "storage/tcap.h" +#include "storage/tcap_impl.h" + +static void TrRenameClass(TrObjDesc *baseDesc, ObjectAddress *object, const char *newName) +{ + Relation rel; + HeapTuple tup; + HeapTuple newtup; + char rbname[NAMEDATALEN]; + Datum values[Natts_pg_class] = { 0 }; + bool nulls[Natts_pg_class] = { false }; + bool replaces[Natts_pg_class] = { false }; + Oid relid = object->objectId; + errno_t rc = EOK; + + if (newName) { + rc = strncpy_s(rbname, NAMEDATALEN, newName, strlen(newName)); + securec_check(rc, "\0", "\0"); + } else { + TrGenObjName(rbname, object->classId, relid); + } + + replaces[Anum_pg_class_relname - 1] = true; + values[Anum_pg_class_relname - 1] = CStringGetDatum(rbname); + + rel = heap_open(RelationRelationId, RowExclusiveLock); + + tup = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (!HeapTupleIsValid(tup)) { + ereport(ERROR, (errcode(ERRCODE_CACHE_LOOKUP_FAILED), errmsg("cache lookup failed for relation %u", relid))); + } + + newtup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); + + simple_heap_update(rel, &newtup->t_self, newtup); + + CatalogUpdateIndexes(rel, newtup); + + ReleaseSysCache(tup); + + heap_freetuple_ext(newtup); + + heap_close(rel, RowExclusiveLock); +} + +static void TrRenameCommon(TrObjDesc *baseDesc, ObjectAddress *object, Oid relid, int natts, int oidAttrNum, + Oid oidIndexId, char *objTag) +{ + Relation rel; + HeapTuple tup; + HeapTuple newtup; + char rbname[NAMEDATALEN]; + Datum *values = (Datum *)palloc0(sizeof(Datum) * natts); + bool *nulls = (bool *)palloc0(sizeof(bool) * natts); + bool *replaces = (bool *)palloc0(sizeof(bool) * natts); + ScanKeyData skey[1]; + SysScanDesc sd; + + TrGenObjName(rbname, object->classId, object->objectId); + + replaces[oidAttrNum - 1] = true; + values[oidAttrNum - 1] = CStringGetDatum(rbname); + + rel = heap_open(relid, RowExclusiveLock); + + ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(object->objectId)); + sd = systable_beginscan(rel, oidIndexId, true, NULL, 1, skey); + + tup = systable_getnext(sd); + if (!HeapTupleIsValid(tup)) { + pfree(values); + pfree(nulls); + pfree(replaces); + ereport(ERROR, + (errcode(ERRCODE_NO_DATA_FOUND), errmsg("could not find tuple for %s %u", objTag, object->objectId))); + } + + newtup = heap_modify_tuple(tup, RelationGetDescr(rel), values, nulls, replaces); + + simple_heap_update(rel, &newtup->t_self, newtup); + + CatalogUpdateIndexes(rel, newtup); + + heap_freetuple_ext(newtup); + + systable_endscan(sd); + + heap_close(rel, RowExclusiveLock); + + pfree(values); + pfree(nulls); + pfree(replaces); +} + +static void TrDeleteBaseid(Oid baseid) +{ + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[1]; + HeapTuple tup; + + rbRel = heap_open(RecyclebinRelationId, RowExclusiveLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcybaseid, BTEqualStrategyNumber, F_INT8EQ, ObjectIdGetDatum(baseid)); + + sd = systable_beginscan(rbRel, RecyclebinBaseidIndexId, true, NULL, 1, skey); + while (HeapTupleIsValid(tup = systable_getnext(sd))) { + simple_heap_delete(rbRel, &tup->t_self); + } + + systable_endscan(sd); + heap_close(rbRel, RowExclusiveLock); + + /* + * CommandCounterIncrement here to ensure that preceding changes are all + * visible to the next deletion step. + */ + CommandCounterIncrement(); +} + +static void TrDeleteId(Oid id) +{ + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[1]; + HeapTuple tup; + + rbRel = heap_open(RecyclebinRelationId, RowExclusiveLock); + + ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(id)); + + sd = systable_beginscan(rbRel, RecyclebinIdIndexId, true, NULL, 1, skey); + if (HeapTupleIsValid(tup = systable_getnext(sd))) { + simple_heap_delete(rbRel, &tup->t_self); + } + + systable_endscan(sd); + heap_close(rbRel, RowExclusiveLock); + + /* + * CommandCounterIncrement here to ensure that preceding changes are all + * visible to the next deletion step. + */ + CommandCounterIncrement(); +} + +static inline bool TrNeedLogicDrop(const ObjectAddress *object) +{ + return object->rbDropMode == RB_DROP_MODE_LOGIC; +} + +static bool TrCanPurge(const TrObjDesc *baseDesc, const ObjectAddress *object, char relKind) +{ + Relation depRel; + SysScanDesc sd; + HeapTuple tuple; + ScanKeyData key[3]; + int nkeys; + bool found = false; + + if (relKind != RELKIND_INDEX && relKind != RELKIND_GLOBAL_INDEX && relKind != RELKIND_RELATION) { + return false; + } + + depRel = heap_open(DependRelationId, AccessShareLock); + + ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(object->classId)); + ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(object->objectId)); + nkeys = 2; + if (object->objectSubId != 0) { + ScanKeyInit(&key[2], Anum_pg_depend_objsubid, BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum(object->objectSubId)); + nkeys = 3; + } + + sd = systable_beginscan(depRel, DependDependerIndexId, true, NULL, nkeys, key); + while (HeapTupleIsValid(tuple = systable_getnext(sd))) { + Form_pg_depend depForm = (Form_pg_depend)GETSTRUCT(tuple); + if (depForm->refclassid == RelationRelationId && depForm->refobjid == baseDesc->relid) { + if (depForm->deptype != DEPENDENCY_AUTO) { + found = false; + break; + } + found = true; + } + } + + systable_endscan(sd); + heap_close(depRel, AccessShareLock); + return found; +} + +static void TrDoDropIndex(TrObjDesc *baseDesc, ObjectAddress *object) +{ + Assert(object->objectSubId == 0); + + if (TrNeedLogicDrop(object)) { + TrObjDesc desc = *baseDesc; + + if (!TR_IS_BASE_OBJ(baseDesc, object)) { + /* Deletion lock already accquired before single object drop. */ + Relation rel = relation_open(object->objectId, NoLock); + + TrDescInit(rel, &desc, RB_OPER_DROP, TrGetObjType(RelationGetNamespace(rel), RELKIND_INDEX), + TrCanPurge(baseDesc, object, RelationGetRelkind(rel))); + relation_close(rel, NoLock); + + TrDescWrite(&desc); + } + + TrRenameClass(baseDesc, object, desc.name); + } else { + index_drop(object->objectId, false); + } + + return; +} + +static void TrDoDropTable(TrObjDesc *baseDesc, ObjectAddress *object, char relKind) +{ + if (TrNeedLogicDrop(object)) { + TrObjDesc desc; + + if (object->objectSubId != 0 || relKind == RELKIND_VIEW || relKind == RELKIND_COMPOSITE_TYPE || + relKind == RELKIND_FOREIGN_TABLE) { + TrRenameClass(baseDesc, object, NULL); + return; + } + + desc = *baseDesc; + if (!TR_IS_BASE_OBJ(baseDesc, object)) { + /* Deletion lock already accquired before single object drop. */ + Relation rel = relation_open(object->objectId, NoLock); + + TrDescInit(rel, &desc, RB_OPER_DROP, TrGetObjType(InvalidOid, relKind), + TrCanPurge(baseDesc, object, relKind)); + relation_close(rel, NoLock); + + TrDescWrite(&desc); + } + + TrRenameClass(baseDesc, object, desc.name); + + return; + } + + /* + * relation_open() must be before the heap_drop_with_catalog(). If you reload + * relation after drop, it may cause other exceptions during the drop process. + */ + if (object->objectSubId != 0) + RemoveAttributeById(object->objectId, object->objectSubId); + else + heap_drop_with_catalog(object->objectId); + + /* + * IMPORANT: The relation must not be reloaded after heap_drop_with_catalog() + * is executed to drop this relation.If you reload relation after drop, it may + * cause other exceptions during the drop process + */ + + return; +} + +static void TrDoDropType(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, TypeRelationId, Natts_pg_type, Anum_pg_type_typname, TypeOidIndexId, "type"); + } else { + RemoveTypeById(object->objectId); + } + + return; +} + +static void TrDoDropConstraint(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, ConstraintRelationId, Natts_pg_constraint, Anum_pg_constraint_conname, + ConstraintOidIndexId, "constraint"); + } else { + RemoveConstraintById(object->objectId); + } + + return; +} + +static void TrDoDropTrigger(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, TriggerRelationId, Natts_pg_trigger, Anum_pg_trigger_tgname, TriggerOidIndexId, + "trigger"); + } else { + RemoveTriggerById(object->objectId); + } + + return; +} + +static void TrDoDropRewrite(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + /* nothing to do as rule-based view requires that origin rule name preserved. */ + } else { + RemoveRewriteRuleById(object->objectId); + } + + return; +} + +static void TrDoDropAttrdef(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + /* nothing to do as no name attribute in system catalog */ + } else { + RemoveAttrDefaultById(object->objectId); + } + + return; +} + +static void TrDoDropProc(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, ProcedureRelationId, Natts_pg_proc, Anum_pg_proc_proname, ProcedureOidIndexId, + "procedure"); + } else { + RemoveFunctionById(object->objectId); + } + + return; +} + +static void TrDoDropCast(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + /* nothing to do as no name attribute in system catalog */ + } else { + DropCastById(object->objectId); + } + + return; +} + +static void TrDoDropCollation(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, CollationRelationId, Natts_pg_collation, Anum_pg_collation_collname, + CollationOidIndexId, "collation"); + } else { + RemoveCollationById(object->objectId); + } + + return; +} + +static void TrDoDropConversion(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, ConversionRelationId, Natts_pg_conversion, Anum_pg_conversion_conname, + ConversionOidIndexId, "conversion"); + } else { + RemoveConversionById(object->objectId); + } + + return; +} + + +static void TrDoDropProceduralLanguage(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, LanguageRelationId, Natts_pg_language, Anum_pg_language_lanname, + LanguageOidIndexId, "language"); + } else { + DropProceduralLanguageById(object->objectId); + } + + return; +} + +static void TrDoDropLargeObject(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + /* nothing to do as no name attribute in system catalog */ + } else { + LargeObjectDrop(object->objectId); + } + + return; +} + +static void TrDoDropOperator(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, OperatorRelationId, Natts_pg_operator, Anum_pg_operator_oprname, + OperatorOidIndexId, "operator"); + } else { + RemoveOperatorById(object->objectId); + } + + return; +} + +static void TrDoDropOpClass(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, OperatorClassRelationId, Natts_pg_opclass, Anum_pg_opclass_opcname, + OpclassOidIndexId, "opclass"); + } else { + RemoveOpClassById(object->objectId); + } + + return; +} + +static void TrDoDropOpFamily(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, OperatorFamilyRelationId, Natts_pg_opfamily, Anum_pg_opfamily_opfname, + OpfamilyOidIndexId, "opfamily"); + } else { + RemoveOpFamilyById(object->objectId); + } + + return; +} + + +static void TrDoDropAmOp(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + /* nothing to do as no name attribute in system catalog */ + } else { + RemoveAmOpEntryById(object->objectId); + } + + return; +} + +static void TrDoDropAmProc(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + /* nothing to do as no name attribute in system catalog */ + } else { + RemoveAmProcEntryById(object->objectId); + } + + return; +} + +static void TrDoDropSchema(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, NamespaceRelationId, Natts_pg_namespace, Anum_pg_namespace_nspname, + NamespaceOidIndexId, "namespace"); + } else { + RemoveSchemaById(object->objectId); + } + + return; +} + +static void TrDoDropTSParser(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, TSParserRelationId, Natts_pg_ts_parser, Anum_pg_ts_parser_prsname, + TSParserOidIndexId, "ts parser"); + } else { + RemoveTSParserById(object->objectId); + } + + return; +} + +static void TrDoDropTSDictionary(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, TSDictionaryRelationId, Natts_pg_ts_dict, Anum_pg_ts_dict_dictname, + TSDictionaryOidIndexId, "ts dictionary"); + } else { + RemoveTSDictionaryById(object->objectId); + } + + return; +} + +static void TrDoDropTSTemplate(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, TSTemplateRelationId, Natts_pg_ts_template, Anum_pg_ts_template_tmplname, + TSTemplateOidIndexId, "ts template"); + } else { + RemoveTSTemplateById(object->objectId); + } + + return; +} + +static void TrDoDropTSConfiguration(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, TSConfigRelationId, Natts_pg_ts_config, Anum_pg_ts_config_cfgname, + TSConfigOidIndexId, "ts configuration"); + } else { + RemoveTSConfigurationById(object->objectId); + } + + return; +} + +static void TrDoDropForeignDataWrapper(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, ForeignDataWrapperRelationId, Natts_pg_foreign_data_wrapper, + Anum_pg_foreign_data_wrapper_fdwname, ForeignDataWrapperOidIndexId, "foreign data wrapper"); + } else { + RemoveForeignDataWrapperById(object->objectId); + } + + return; +} + +static void TrDoDropForeignServer(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, ForeignServerRelationId, Natts_pg_foreign_server, + Anum_pg_foreign_server_srvname, ForeignServerOidIndexId, "foreign server"); + } else { + RemoveForeignServerById(object->objectId); + } + + return; +} + +static void TrDoDropUserMapping(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + /* nothing to do as no name attribute in system catalog */ + } else { + RemoveUserMappingById(object->objectId); + } + + return; +} + + +static void TrDoDropDefaultACL(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + /* nothing to do as no name attribute in system catalog */ + } else { + RemoveDefaultACLById(object->objectId); + } + + return; +} + +static void TrDoDropPgxcClass(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + /* nothing to do as no name attribute in system catalog */ + } else { + RemovePgxcClass(object->objectId); + } + + return; +} + +static void TrDoDropExtension(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, ExtensionRelationId, Natts_pg_extension, Anum_pg_extension_extname, + ExtensionOidIndexId, "extension"); + } else { + RemoveExtensionById(object->objectId); + } + + return; +} + +static void TrDoDropDataSource(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, DataSourceRelationId, Natts_pg_extension_data_source, + Anum_pg_extension_data_source_srcname, DataSourceOidIndexId, "extension data source"); + } else { + RemoveDataSourceById(object->objectId); + } + + return; +} + +static void TrDoDropDirectory(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, PgDirectoryRelationId, Natts_pg_directory, Anum_pg_directory_directory_name, + PgDirectoryOidIndexId, "directory"); + } else { + RemoveDirectoryById(object->objectId); + } + + return; +} + +static void TrDoDropRlsPolicy(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, RlsPolicyRelationId, Natts_pg_rlspolicy, Anum_pg_rlspolicy_polname, + PgRlspolicyOidIndex, "rlspolicy"); + } else { + RemoveRlsPolicyById(object->objectId); + } + + return; +} + +static void TrDoDropJob(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + /* nothing to do as no name attribute in system catalog */ + } else { + RemoveJobById(object->objectId); + } + + return; +} + +static void TrDoDropSynonym(TrObjDesc *baseDesc, ObjectAddress *object) +{ + if (TrNeedLogicDrop(object)) { + TrRenameCommon(baseDesc, object, PgSynonymRelationId, Natts_pg_synonym, Anum_pg_synonym_synname, + SynonymOidIndexId, "synonym"); + } else { + RemoveSynonymById(object->objectId); + } + + return; +} + +/* + * doDeletion: delete a single object + * return false if logic deleted, + * return true if physical deleted, + */ +static void TrDoDrop(TrObjDesc *baseDesc, ObjectAddress *object) +{ + switch (getObjectClass(object)) { + case OCLASS_CLASS: { + char relKind = get_rel_relkind(object->objectId); + if (relKind == RELKIND_INDEX) { + TrDoDropIndex(baseDesc, object); + } else { + /* + * We use a unified entry for others: + * RELKIND_RELATION, RELKIND_SEQUENCE, + * RELKIND_TOASTVALUE, RELKIND_VIEW, + * RELKIND_COMPOSITE_TYPE, RELKIND_FOREIGN_TABLE + */ + TrDoDropTable(baseDesc, object, relKind); + } + break; + } + + case OCLASS_TYPE: + TrDoDropType(baseDesc, object); + break; + + case OCLASS_CONSTRAINT: + TrDoDropConstraint(baseDesc, object); + break; + + case OCLASS_TRIGGER: + TrDoDropTrigger(baseDesc, object); + break; + + case OCLASS_REWRITE: + TrDoDropRewrite(baseDesc, object); + break; + + case OCLASS_DEFAULT: + TrDoDropAttrdef(baseDesc, object); + break; + + case OCLASS_PROC: + TrDoDropProc(baseDesc, object); + break; + + case OCLASS_CAST: + TrDoDropCast(baseDesc, object); + break; + + case OCLASS_COLLATION: + TrDoDropCollation(baseDesc, object); + break; + + case OCLASS_CONVERSION: + TrDoDropConversion(baseDesc, object); + break; + + case OCLASS_LANGUAGE: + TrDoDropProceduralLanguage(baseDesc, object); + break; + + case OCLASS_LARGEOBJECT: + TrDoDropLargeObject(baseDesc, object); + break; + + case OCLASS_OPERATOR: + TrDoDropOperator(baseDesc, object); + break; + + case OCLASS_OPCLASS: + TrDoDropOpClass(baseDesc, object); + break; + + case OCLASS_OPFAMILY: + TrDoDropOpFamily(baseDesc, object); + break; + + case OCLASS_AMOP: + TrDoDropAmOp(baseDesc, object); + break; + + case OCLASS_AMPROC: + TrDoDropAmProc(baseDesc, object); + break; + + case OCLASS_SCHEMA: + TrDoDropSchema(baseDesc, object); + break; + + case OCLASS_TSPARSER: + TrDoDropTSParser(baseDesc, object); + break; + + case OCLASS_TSDICT: + TrDoDropTSDictionary(baseDesc, object); + break; + + case OCLASS_TSTEMPLATE: + TrDoDropTSTemplate(baseDesc, object); + break; + + case OCLASS_TSCONFIG: + TrDoDropTSConfiguration(baseDesc, object); + break; + + /* + * OCLASS_ROLE, OCLASS_DATABASE, OCLASS_TBLSPACE intentionally not + * handled here + */ + + case OCLASS_FDW: + TrDoDropForeignDataWrapper(baseDesc, object); + break; + + case OCLASS_FOREIGN_SERVER: + TrDoDropForeignServer(baseDesc, object); + break; + + case OCLASS_USER_MAPPING: + TrDoDropUserMapping(baseDesc, object); + break; + + case OCLASS_DEFACL: + TrDoDropDefaultACL(baseDesc, object); + break; + + case OCLASS_PGXC_CLASS: + TrDoDropPgxcClass(baseDesc, object); + break; + + case OCLASS_EXTENSION: + TrDoDropExtension(baseDesc, object); + break; + + case OCLASS_DATA_SOURCE: + TrDoDropDataSource(baseDesc, object); + break; + + case OCLASS_DIRECTORY: + TrDoDropDirectory(baseDesc, object); + break; + + case OCLASS_RLSPOLICY: + TrDoDropRlsPolicy(baseDesc, object); + break; + + case OCLASS_PG_JOB: + if ((IS_PGXC_COORDINATOR && !IsConnFromCoord()) || (g_instance.role == VSINGLENODE)) + TrDoDropJob(baseDesc, object); + break; + + case OCLASS_SYNONYM: + TrDoDropSynonym(baseDesc, object); + break; + + default: + ereport(ERROR, + (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), errmsg("unrecognized object class: %u", object->classId))); + break; + } + + return; +} + +/* + * deleteOneObject: delete a single object for TrDrop. + * + * *depRel is the already-open pg_depend relation. + */ +static void TrDropOneObject(TrObjDesc *baseDesc, ObjectAddress *object, Relation *depRel) +{ + ScanKeyData key[3]; + int nkeys; + SysScanDesc scan; + HeapTuple tup; + + /* DROP hook of the objects being removed */ + if (object_access_hook) { + ObjectAccessDrop dropArg; + + dropArg.dropflags = PERFORM_DELETION_INVALID; + InvokeObjectAccessHook(OAT_DROP, object->classId, object->objectId, object->objectSubId, &dropArg); + } + + /* + * Delete the object itself, in an object-type-dependent way. + * + * We used to do this after removing the outgoing dependency links, but it + * seems just as reasonable to do it beforehand. In the concurrent case + * we *must *do it in this order, because we can't make any transactional + * updates before calling doDeletion() --- they'd get committed right + * away, which is not cool if the deletion then fails. + */ + TrDoDrop(baseDesc, object); + + /* + * In logical drop mode, we will keep all related system entries, including + * linked entries such as pg_depend records. It is done! + */ + if (TrNeedLogicDrop(object)) { + /* + * CommandCounterIncrement here to ensure that preceding changes are all + * visible to the next deletion step. + */ + CommandCounterIncrement(); + + /* + * Logic Drop done! + */ + return; + } + + /* + * In physical drop mode, we continue to remove all related system entries. + */ + + /* + * Now remove any pg_depend records that link from this object to others. + * (Any records linking to this object should be gone already.) + * + * When dropping a whole object (subId = 0), remove all pg_depend records + * for its sub-objects too. + */ + ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(object->classId)); + ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(object->objectId)); + if (object->objectSubId != 0) { + ScanKeyInit(&key[2], Anum_pg_depend_objsubid, BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum(object->objectSubId)); + nkeys = 3; + } else + nkeys = 2; + + scan = systable_beginscan(*depRel, DependDependerIndexId, true, NULL, nkeys, key); + + while (HeapTupleIsValid(tup = systable_getnext(scan))) { + simple_heap_delete(*depRel, &tup->t_self); + } + + systable_endscan(scan); + + /* + * Delete shared dependency references related to this object. Again, if + * subId = 0, remove records for sub-objects too. + */ + deleteSharedDependencyRecordsFor(object->classId, object->objectId, object->objectSubId); + + /* + * Delete any comments or security labels associated with this object. + * (This is a convenient place to do these things, rather than having + * every object type know to do it.) + */ + DeleteComments(object->objectId, object->classId, object->objectSubId); + DeleteSecurityLabel(object); + + /* + * CommandCounterIncrement here to ensure that preceding changes are all + * visible to the next deletion step. + */ + CommandCounterIncrement(); + + /* + * Physical Drop done! + */ +} + +static bool TrObjIsInList(const ObjectAddresses *targetObjects, const ObjectAddress *thisobj) +{ + ObjectAddress *item = NULL; + + for (int i = 0; i < targetObjects->numrefs; i++) { + item = targetObjects->refs + i; + if (TrObjIsEqual(thisobj, item)) { + return true; + } + } + return false; +} + +static ObjectAddress *TrFindIdxInTarget(ObjectAddresses *targetObjects, ObjectAddress *item) +{ + ObjectAddress *thisobj = NULL; + + for (int i = 0; i < targetObjects->numrefs; i++) { + thisobj = targetObjects->refs + i; + if (TrObjIsEqual(item, thisobj)) { + return thisobj; + } + } + + return NULL; +} + +/* + * output: refthisobjs + */ +static void TrFindAllSubObjs(Relation depRel, const ObjectAddress *refobj, ObjectAddresses *refthisobjs) +{ + SysScanDesc sd; + HeapTuple tuple; + ScanKeyData key[3]; + int nkeys; + + ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(refobj->classId)); + ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(refobj->objectId)); + nkeys = 2; + if (refobj->objectSubId != 0) { + ScanKeyInit(&key[2], Anum_pg_depend_refobjsubid, BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum(refobj->objectSubId)); + nkeys = 3; + } + + sd = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, nkeys, key); + while (HeapTupleIsValid(tuple = systable_getnext(sd))) { + Form_pg_depend depForm = (Form_pg_depend)GETSTRUCT(tuple); + + /* add the refs to list */ + add_object_address_ext(depForm->classid, depForm->objid, depForm->objsubid, depForm->deptype, refthisobjs); + } + + systable_endscan(sd); + return; +} + +static void TrTagPhyDeleteSubObjs(Relation depRel, ObjectAddresses *targetObjects, ObjectAddress *thisobj) +{ + ObjectAddress *item = NULL; + + ObjectAddresses *refthisobjs = new_object_addresses(); + + /* Tag this obj RB_DROP_MODE_PHYSICAL */ + thisobj->rbDropMode = RB_DROP_MODE_PHYSICAL; + + /* Find all sub objs refered to this obj */ + TrFindAllSubObjs(depRel, thisobj, refthisobjs); + + for (int i = 0; i < refthisobjs->numrefs; i++) { + item = refthisobjs->refs + i; + + /* the item must exists in targetObjects. */ + item = TrFindIdxInTarget(targetObjects, item); + if (item == NULL || item->rbDropMode == RB_DROP_MODE_PHYSICAL) { + continue; + } + TrTagPhyDeleteSubObjs(depRel, targetObjects, item); + } + + free_object_addresses(refthisobjs); + return; +} + +static bool TrNeedPhyDelete(Relation depRel, ObjectAddresses *targetObjects, ObjectAddress *thisobj) +{ + ObjectAddress *item = NULL; + ObjectAddresses *refobjs = new_object_addresses(); + bool result = false; + + /* Find all objs this obj refered */ + TrFindAllRefObjs(depRel, thisobj, refobjs); + + /* Step 1: tag refobjs of thisobj, return directly if ALL refobjs not need physical drop. */ + for (int i = 0; i < refobjs->numrefs; i++) { + item = refobjs->refs + i; + if (!TrObjIsInList(targetObjects, item)) { + result = true; + break; + } + } + if (!result) { + free_object_addresses(refobjs); + return result; + } + + /* Step 2: tag refobjs with 'i' deptype to physical drop. */ + for (int i = 0; i < refobjs->numrefs; i++) { + item = refobjs->refs + i; + if (item->deptype == 'i') { + item = TrFindIdxInTarget(targetObjects, item); + Assert(item != NULL); + if (item->rbDropMode == RB_DROP_MODE_PHYSICAL) { + continue; + } + TrTagPhyDeleteSubObjs(depRel, targetObjects, item); + } + } + + free_object_addresses(refobjs); + return result; +} + +static void TrResetDropMode(const ObjectAddresses *targetObjects, const ObjectAddress *baseObj) +{ + ObjectAddress *thisobj = NULL; + + for (int i = 0; i < targetObjects->numrefs; i++) { + thisobj = targetObjects->refs + i; + if (TrObjIsEqual(thisobj, baseObj)) { + thisobj->rbDropMode = RB_DROP_MODE_LOGIC; + continue; + } + thisobj->rbDropMode = RB_DROP_MODE_INVALID; + } + return; +} + +static void TrTagDependentObjects(Relation depRel, ObjectAddresses *targetObjects, const ObjectAddress *baseObj) +{ + ObjectAddress *thisobj = NULL; + + TrResetDropMode(targetObjects, baseObj); + for (int i = 0; i < targetObjects->numrefs; i++) { + thisobj = targetObjects->refs + i; + if (TrDropModeIsAlreadySet(thisobj)) { + continue; + } + + if (TrNeedPhyDelete(depRel, targetObjects, thisobj)) { + TrTagPhyDeleteSubObjs(depRel, targetObjects, thisobj); + } else { + thisobj->rbDropMode = RB_DROP_MODE_LOGIC; + } + } + + return; +} + +bool TrCheckRecyclebinDrop(const DropStmt *stmt, ObjectAddresses *objects) +{ + Relation depRel; + bool rbDrop = false; + + /* No work if no objects... */ + if (objects->numrefs <= 0) + return false; + + if (/* + * Disable Recyclebin-based-Drop when target object is not OBJECT_TABLE, or + */ + stmt->removeType != OBJECT_TABLE || + /* in concurrent drop mode, or */ + stmt->concurrent || + /* with purge option, or */ + stmt->purge || + /* multi objects drop. */ + list_length(stmt->objects) != 1) { + return false; + } + + if (!NeedTrComm(objects->refs->objectId)) { + return false; + } + + depRel = heap_open(DependRelationId, AccessShareLock); + rbDrop = !TrNeedPhyDelete(depRel, objects, &objects->refs[0]); + heap_close(depRel, AccessShareLock); + + return rbDrop; +} + +void TrDrop(const DropStmt* drop, const ObjectAddresses *objects, DropBehavior behavior) +{ + Relation depRel; + Relation baseRel; + TrObjDesc baseDesc; + ObjectAddresses *targetObjects = NULL; + ObjectAddress *baseObj = objects->refs; + + /* + * We save some cycles by opening pg_depend just once and passing the + * Relation pointer down to all the recursive deletion steps. + */ + depRel = heap_open(DependRelationId, RowExclusiveLock); + + /* + * Construct a list of objects to delete (ie, the given objects plus + * everything directly or indirectly dependent on them). Note that + * because we pass the whole objects list as pendingObjects context, we + * won't get a failure from trying to delete an object that is internally + * dependent on another one in the list; we'll just skip that object and + * delete it when we reach its owner. + */ + targetObjects = new_object_addresses(); + + /* + * Acquire deletion lock on each target object. (Ideally the caller + * has done this already, but many places are sloppy about it.) + */ + AcquireDeletionLock(baseObj, PERFORM_DELETION_INVALID); + + /* + * Finds all subobjects that reference the base table recursively. + */ + findDependentObjects(baseObj, DEPFLAG_ORIGINAL, NULL, /* empty stack */ + targetObjects, objects, &depRel); + ereport(LOG, (errmsg("Delete object %u/%u/%d", baseObj->classId, baseObj->objectId, baseObj->objectSubId))); + + /* + * Check if deletion is allowed, and report about cascaded deletes. + * + * If there's exactly one object being deleted, report it the same way as + * in performDeletion(), else we have to be vaguer. + */ + reportDependentObjects(targetObjects, behavior, NOTICE, baseObj); + + /* + * Tag all subobjects' drop mode: LOGIC_DROP, PYHSICAL_DROP. + */ + TrTagDependentObjects(depRel, targetObjects, baseObj); + + /* + * Initialize the baseDesc structure so that the logic dropped subobjects + * can be correctly processed when renamed or placed in recycle bin. Notice + * that base object already locked. + */ + baseRel = relation_open(baseObj->objectId, NoLock); + TrDescInit(baseRel, &baseDesc, RB_OPER_DROP, RB_OBJ_TABLE, true, true); + baseDesc.id = baseDesc.baseid = TrDescWrite(&baseDesc); + TrUpdateBaseid(&baseDesc); + relation_close(baseRel, NoLock); + + Oid relid = RelationGetRelid(baseRel); + UpdatePgObjectChangecsn(relid, baseRel->rd_rel->relkind); + + /* + * Drop all the objects in the proper order. + */ + for (int i = 0; i < targetObjects->numrefs; i++) { + ObjectAddress *thisobj = targetObjects->refs + i; + TrDropOneObject(&baseDesc, thisobj, &depRel); + } + + /* And clean up */ + free_object_addresses(targetObjects); + heap_close(depRel, RowExclusiveLock); +} + +void TrDoPurgeObjectDrop(TrObjDesc *desc) +{ + ObjectAddresses *objects; + ObjectAddress obj; + + objects = new_object_addresses(); + + obj.classId = RelationRelationId; + obj.objectId = desc->relid; + obj.objectSubId = 0; + add_exact_object_address(&obj, objects); + + performMultipleDeletions(objects, DROP_CASCADE, PERFORM_DELETION_INVALID); + + if (desc->type == RB_OBJ_TABLE) { + TrDeleteBaseid(desc->baseid); + } else { /* RB_OBJ_INDEX */ + TrDeleteId(desc->id); + } + + free_object_addresses(objects); + return; +} + +/* TIMECAPSULE TABLE { table_name } TO BEFORE DROP [RENAME TO new_tablename] */ +void TrRestoreDrop(const TimeCapsuleStmt *stmt) +{ + TrObjDesc desc; + ObjectAddress obj; + Relation rel; + + desc.relid = 0; + TrOperFetch(stmt->relation, RB_OBJ_TABLE, &desc, RB_OPER_RESTORE_DROP); + if (desc.relid != 0 && (desc.type == RB_OBJ_TABLE)) { + stmt->relation->relname = desc.name; + rel = heap_openrv(stmt->relation, AccessExclusiveLock); + if (rel->rd_tam_type == TAM_HEAP) { + heap_close(rel, NoLock); + elog(ERROR, "timecapsule does not support astore yet"); + return; + } + heap_close(rel, NoLock); + } + + desc.authid = GetUserId(); + TrOperPrep(&desc, RB_OPER_RESTORE_DROP); + + obj.classId = RelationRelationId; + obj.objectId = desc.relid; + obj.objectSubId = 0; + + TrRenameClass(&desc, &obj, stmt->new_relname ? stmt->new_relname : desc.originname); + + TrDeleteBaseid(desc.baseid); + + return; +} diff --git a/src/gausskernel/storage/tcap/tcap_manager.cpp b/src/gausskernel/storage/tcap/tcap_manager.cpp index 0df59d4ba..da0bee6fb 100644 --- a/src/gausskernel/storage/tcap/tcap_manager.cpp +++ b/src/gausskernel/storage/tcap/tcap_manager.cpp @@ -1,1888 +1,1896 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * --------------------------------------------------------------------------------------- - * - * tcap_manager.cpp - * Routines to support Timecapsule `Recyclebin-based query, restore`. - * We use Tr prefix to indicate it in following coding. - * - * IDENTIFICATION - * src/gausskernel/storage/tcap/tcap_manager.cpp - * - * --------------------------------------------------------------------------------------- - */ - -#include "postgres.h" - -#include "pgstat.h" -#include "access/reloptions.h" -#include "access/sysattr.h" -#include "access/xlog.h" -#include "catalog/pg_database.h" -#include "catalog/dependency.h" -#include "catalog/heap.h" -#include "catalog/index.h" -#include "catalog/indexing.h" -#include "catalog/objectaccess.h" -#include "catalog/pg_collation_fn.h" -#include "catalog/pg_collation.h" -#include "catalog/pg_constraint.h" -#include "catalog/pg_conversion_fn.h" -#include "catalog/pg_conversion.h" -#include "catalog/pg_depend.h" -#include "catalog/pg_extension_data_source.h" -#include "catalog/pg_extension.h" -#include "catalog/pg_foreign_data_wrapper.h" -#include "catalog/pg_foreign_server.h" -#include "catalog/pg_job.h" -#include "catalog/pg_language.h" -#include "catalog/pg_largeobject.h" -#include "catalog/pg_object.h" -#include "catalog/pg_opclass.h" -#include "catalog/pg_operator.h" -#include "catalog/pg_opfamily.h" -#include "catalog/pg_partition_fn.h" -#include "catalog/pg_proc.h" -#include "catalog/pg_recyclebin.h" -#include "catalog/pg_rewrite.h" -#include "catalog/pg_rlspolicy.h" -#include "catalog/pg_synonym.h" -#include "catalog/pg_tablespace.h" -#include "catalog/pg_trigger.h" -#include "catalog/pg_ts_config.h" -#include "catalog/pg_ts_dict.h" -#include "catalog/pg_ts_parser.h" -#include "catalog/pg_ts_template.h" -#include "catalog/pgxc_class.h" -#include "catalog/pg_partition.h" -#include "catalog/storage.h" -#include "commands/comment.h" -#include "commands/dbcommands.h" -#include "commands/directory.h" -#include "commands/extension.h" -#include "commands/proclang.h" -#include "commands/schemacmds.h" -#include "commands/seclabel.h" -#include "commands/sec_rls_cmds.h" -#include "commands/tablecmds.h" -#include "commands/tablespace.h" -#include "commands/trigger.h" -#include "commands/typecmds.h" -#include "executor/node/nodeModifyTable.h" -#include "rewrite/rewriteRemove.h" -#include "storage/lmgr.h" -#include "storage/predicate.h" -#include "storage/smgr/relfilenode.h" -#include "utils/acl.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/inval.h" -#include "utils/lsyscache.h" -#include "utils/relcache.h" -#include "utils/snapmgr.h" -#include "utils/syscache.h" - -#include "storage/tcap.h" -#include "storage/tcap_impl.h" - -static bool TrIsRefRbObject(const ObjectAddress *obj, Relation depRel = NULL); -void TrDoPurgeObjectDrop(TrObjDesc *desc); - -char *TrGenObjName(char *rbname, Oid classId, Oid objid) -{ - int rc = EOK; - - rc = snprintf_s(rbname, NAMEDATALEN, NAMEDATALEN - 1, "BIN$%X%X%X$%llX==$0", - u_sess->proc_cxt.MyDatabaseId, classId, objid, (uint64)GetXLogInsertRecPtr()); - securec_check_ss_c(rc, "\0", "\0"); - - return rbname; -} - -static TransactionId TrRbGetRcyfrozenxid64(HeapTuple rbtup, Relation rbRel = NULL) -{ - Datum datum; - bool isNull = false; - TransactionId rcyfrozenxid64; - bool relArgNull = rbRel == NULL; - - if (relArgNull) { - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - } - - datum = heap_getattr(rbtup, Anum_pg_recyclebin_rcyfrozenxid64, RelationGetDescr(rbRel), &isNull); - Assert(!isNull); - - rcyfrozenxid64 = DatumGetTransactionId(datum); - - if (relArgNull) { - heap_close(rbRel, AccessShareLock); - } - - return rcyfrozenxid64; -} - -void TrDescInit(Relation rel, TrObjDesc *desc, TrObjOperType operType, - TrObjType objType, bool canpurge, bool isBaseObj) -{ - errno_t rc = EOK; - - /* Notice: desc->id, desc->baseid will be assigned by invoker later. */ - desc->dbid = u_sess->proc_cxt.MyDatabaseId; - desc->relid = RelationGetRelid(rel); - - (void)TrGenObjName(desc->name, RelationRelationId, desc->relid); - - rc = strncpy_s(desc->originname, NAMEDATALEN, RelationGetRelationName(rel), - strlen(RelationGetRelationName(rel))); - securec_check(rc, "\0", "\0"); - - desc->operation = operType; - desc->type = objType; - desc->recyclecsn = t_thrd.xact_cxt.ShmemVariableCache->nextCommitSeqNo; - desc->recycletime = GetCurrentTimestamp(); - desc->createcsn = RelationGetCreatecsn(rel); - desc->changecsn = RelationGetChangecsn(rel); - desc->nspace = RelationGetNamespace(rel); - desc->owner = RelationGetOwner(rel); - desc->tablespace = RelationGetTablespace(rel); - desc->relfilenode = RelationGetRelFileNode(rel); - desc->frozenxid = RelationGetRelFrozenxid(rel); - desc->frozenxid64 = RelationGetRelFrozenxid64(rel); - desc->canrestore = objType == RB_OBJ_TABLE; - desc->canpurge = canpurge; -} - -void TrPartDescInit(Relation rel, Partition part, TrObjDesc *desc, TrObjOperType operType, - TrObjType objType, bool canpurge, bool isBaseObj) -{ - errno_t rc = EOK; - - /* Notice: desc->id, desc->baseid will be assigned by invoker later. */ - desc->dbid = u_sess->proc_cxt.MyDatabaseId; - desc->relid = part->pd_id; - - (void)TrGenObjName(desc->name, PartitionRelationId, desc->relid); - - rc = strncpy_s(desc->originname, NAMEDATALEN, RelationGetRelationName(rel), - strlen(RelationGetRelationName(rel))); - securec_check(rc, "\0", "\0"); - - int len = strlen(PartitionGetPartitionName(part)) + strlen(RelationGetRelationName(rel)) + 1; - rc = strcat_s(desc->originname, len, PartitionGetPartitionName(part)); - securec_check(rc, "\0", "\0"); - - desc->operation = operType; - desc->type = objType; - desc->recyclecsn = t_thrd.xact_cxt.ShmemVariableCache->nextCommitSeqNo; - desc->recycletime = GetCurrentTimestamp(); - desc->createcsn = RelationGetCreatecsn(rel); - desc->changecsn = RelationGetChangecsn(rel); - desc->nspace = RelationGetNamespace(rel); - desc->owner = RelationGetOwner(rel); - desc->tablespace = part->pd_part->reltablespace; - desc->relfilenode = part->pd_part->relfilenode; - desc->frozenxid = part->pd_part->relfrozenxid; - desc->frozenxid64 = PartGetRelFrozenxid64(part); - desc->canrestore = false; - desc->canpurge = canpurge; -} - -static void TrDescRead(TrObjDesc *desc, HeapTuple rbtup) -{ - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(rbtup); - - desc->id = HeapTupleGetOid(rbtup); - desc->baseid = rbForm->rcybaseid; - - desc->dbid = rbForm->rcydbid; - desc->relid = rbForm->rcyrelid; - (void)namestrcpy((Name)desc->name, NameStr(rbForm->rcyname)); - (void)namestrcpy((Name)desc->originname, NameStr(rbForm->rcyoriginname)); - desc->operation = (rbForm->rcyoperation == 'd') ? RB_OPER_DROP : RB_OPER_TRUNCATE; - desc->type = (TrObjType)rbForm->rcytype; - desc->recyclecsn = rbForm->rcyrecyclecsn; - desc->recycletime = rbForm->rcyrecycletime; - desc->createcsn = rbForm->rcycreatecsn; - desc->changecsn = rbForm->rcychangecsn; - desc->nspace = rbForm->rcynamespace; - desc->owner = rbForm->rcyowner; - desc->tablespace = rbForm->rcytablespace; - desc->relfilenode = rbForm->rcyrelfilenode; - desc->canrestore = rbForm->rcycanrestore; - desc->canpurge = rbForm->rcycanpurge; - desc->frozenxid = rbForm->rcyfrozenxid; - desc->frozenxid64 = TrRbGetRcyfrozenxid64(rbtup); -} - -Oid TrDescWrite(TrObjDesc *desc) -{ - Relation rel; - HeapTuple tup; - bool nulls[Natts_pg_recyclebin] = {0}; - Datum values[Natts_pg_recyclebin]; - NameData name; - NameData originname; - Oid rbid; - - values[Anum_pg_recyclebin_rcydbid - 1] = ObjectIdGetDatum(desc->dbid); - values[Anum_pg_recyclebin_rcybaseid - 1] = ObjectIdGetDatum(desc->baseid); - values[Anum_pg_recyclebin_rcyrelid - 1] = ObjectIdGetDatum(desc->relid); - (void)namestrcpy(&name, desc->name); - values[Anum_pg_recyclebin_rcyname - 1] = NameGetDatum(&name); - (void)namestrcpy(&originname, desc->originname); - values[Anum_pg_recyclebin_rcyoriginname - 1] = NameGetDatum(&originname); - values[Anum_pg_recyclebin_rcyoperation - 1] = (desc->operation == RB_OPER_DROP) ? 'd' : 't'; - values[Anum_pg_recyclebin_rcytype - 1] = Int32GetDatum(desc->type); - values[Anum_pg_recyclebin_rcyrecyclecsn - 1] = Int64GetDatum(desc->recyclecsn); - values[Anum_pg_recyclebin_rcyrecycletime - 1] = TimestampTzGetDatum(desc->recycletime); - values[Anum_pg_recyclebin_rcycreatecsn - 1] = Int64GetDatum(desc->createcsn); - values[Anum_pg_recyclebin_rcychangecsn - 1] = Int64GetDatum(desc->changecsn); - values[Anum_pg_recyclebin_rcynamespace - 1] = ObjectIdGetDatum(desc->nspace); - values[Anum_pg_recyclebin_rcyowner - 1] = ObjectIdGetDatum(desc->owner); - values[Anum_pg_recyclebin_rcytablespace - 1] = ObjectIdGetDatum(desc->tablespace); - values[Anum_pg_recyclebin_rcyrelfilenode - 1] = ObjectIdGetDatum(desc->relfilenode); - values[Anum_pg_recyclebin_rcycanrestore - 1] = BoolGetDatum(desc->canrestore); - values[Anum_pg_recyclebin_rcycanpurge - 1] = BoolGetDatum(desc->canpurge); - values[Anum_pg_recyclebin_rcyfrozenxid - 1] = ShortTransactionIdGetDatum(desc->frozenxid); - values[Anum_pg_recyclebin_rcyfrozenxid64 - 1] = TransactionIdGetDatum(desc->frozenxid64); - - rel = heap_open(RecyclebinRelationId, RowExclusiveLock); - - tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); - - rbid = simple_heap_insert(rel, tup); - - CatalogUpdateIndexes(rel, tup); - - heap_freetuple_ext(tup); - - heap_close(rel, RowExclusiveLock); - - CommandCounterIncrement(); - - return rbid; -} - -static bool TrFetchOrinameImpl(Oid nspId, const char *oriname, TrObjType type, - TrObjDesc *desc, TrOperMode operMode) -{ - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[3]; - HeapTuple tup; - bool found = false; - - if (!OidIsValid(nspId)) { - return false; - } - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcynamespace, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(nspId)); - ScanKeyInit(&skey[1], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); - ScanKeyInit(&skey[2], Anum_pg_recyclebin_rcyoriginname, BTEqualStrategyNumber, - F_NAMEEQ, CStringGetDatum(oriname)); - - sd = systable_beginscan(rbRel, RecyclebinDbidNspOrinameIndexId, true, NULL, 3, skey); - while ((tup = systable_getnext(sd)) != NULL) { - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); - if ((rbForm->rcytype != type && rbForm->rcytype == RB_OBJ_TABLE) || - (rbForm->rcytype != type && rbForm->rcytype == RB_OBJ_INDEX)) { - continue; - } - - found = true; - TrDescRead(desc, tup); - break; - } - - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return found; -} - -bool TrFetchName(const char *rcyname, TrObjType type, TrObjDesc *desc, TrOperMode operMode) -{ - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[1]; - HeapTuple tup; - bool found = false; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcyname, BTEqualStrategyNumber, - F_NAMEEQ, CStringGetDatum(rcyname)); - - sd = systable_beginscan(rbRel, RecyclebinNameIndexId, true, NULL, 1, skey); - if ((tup = systable_getnext(sd)) != NULL) { - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); - if ((rbForm->rcytype != type && rbForm->rcytype == RB_OBJ_TABLE) || - (rbForm->rcytype != type && rbForm->rcytype == RB_OBJ_INDEX)) { - ereport(ERROR, - (errmsg("The recycle object \"%s\" type mismatched.", rcyname))); - } - found = true; - TrDescRead(desc, tup); - } - - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return found; -} - -static bool TrFetchOriname(const char *schemaname, const char *relname, TrObjType type, - TrObjDesc *desc, TrOperMode operMode) -{ - bool found = false; - Oid nspId; - - if (schemaname) { - nspId = get_namespace_oid(schemaname, true); - found = TrFetchOrinameImpl(nspId, relname, type, desc, operMode); - } else { - List *activeSearchPath = NIL; - ListCell *l = NULL; - - recomputeNamespacePath(); - activeSearchPath = list_copy(u_sess->catalog_cxt.activeSearchPath); - foreach (l, activeSearchPath) { - nspId = lfirst_oid(l); - if (TrFetchOrinameImpl(nspId, relname, type, desc, operMode)) { - found = true; - break; - } - } - list_free_ext(activeSearchPath); - if (!found) { - nspId = PG_TOAST_NAMESPACE; - found = TrFetchOrinameImpl(nspId, relname, type, desc, operMode); - } - } - - return found; -} - -void TrUpdateBaseid(const TrObjDesc *desc) -{ - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[1]; - HeapTuple tup; - HeapTuple newtup; - Datum values[Natts_pg_recyclebin] = { 0 }; - bool nulls[Natts_pg_recyclebin] = { false }; - bool replaces[Natts_pg_recyclebin] = { false }; - - rbRel = heap_open(RecyclebinRelationId, RowExclusiveLock); - - ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(desc->id)); - - sd = systable_beginscan(rbRel, RecyclebinIdIndexId, true, NULL, 1, skey); - if ((tup = systable_getnext(sd)) == NULL) { - ereport(ERROR, (errmsg("recycle object %u does not exist", desc->id))); - } - - replaces[Anum_pg_recyclebin_rcybaseid - 1] = true; - values[Anum_pg_recyclebin_rcybaseid - 1] = ObjectIdGetDatum(desc->baseid); - - newtup = heap_modify_tuple(tup, RelationGetDescr(rbRel), values, nulls, replaces); - - simple_heap_update(rbRel, &newtup->t_self, newtup); - - CatalogUpdateIndexes(rbRel, newtup); - - heap_freetuple_ext(newtup); - - systable_endscan(sd); - heap_close(rbRel, RowExclusiveLock); - - return; -} - -static void TrLockRelationImpl(Oid relid, TrObjType type) -{ - /* - * Lock failed may due to concurrently purge/timecapsule/DQL - * on recycle object, or access on normal relation. - */ - if (!ConditionalLockRelationOid(relid, AccessExclusiveLock)) { - ereport(ERROR, - (errcode(ERRCODE_RBIN_LOCK_NOT_AVAILABLE), - errmsg("could not obtain lock on relation \"%u\"", relid))); - } - - /* - * Now that we have the lock, probe to see if the relation - * really exists or not. - */ - AcceptInvalidationMessages(); - if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(relid)) && type != RB_OBJ_PARTITION) { - /* Clean already held locks if error return. */ - UnlockRelationOid(relid, AccessExclusiveLock); - ereport(ERROR, - (errcode(ERRCODE_RBIN_UNDEFINED_OBJECT), - errmsg("relation \"%u\" does not exist", relid))); - } else if (!SearchSysCacheExists1(PARTRELID, ObjectIdGetDatum(relid)) && type == RB_OBJ_PARTITION) { - /* Clean already held locks if error return. */ - UnlockRelationOid(relid, AccessExclusiveLock); - ereport(ERROR, - (errcode(ERRCODE_RBIN_UNDEFINED_OBJECT), - errmsg("partition \"%u\" does not exist", relid))); - } -} - -static void TrLockRelation(TrObjDesc *desc) -{ - Oid heapOid = InvalidOid; - - /* Lock heap relation for index first */ - if (desc->type == RB_OBJ_INDEX) { - heapOid = IndexGetRelation(desc->relid, true); - if (!OidIsValid(heapOid)) { - ereport(ERROR, - (errcode(ERRCODE_RBIN_UNDEFINED_OBJECT), - errmsg("relation \"%u\" does not exist", desc->relid))); - } - TrLockRelationImpl(heapOid, desc->type); - } - - /* Use TRY-CATCH block to clean locks already held if error. */ - PG_TRY(); - { - /* Lock relation self */ - TrLockRelationImpl(desc->relid, desc->type); - } - PG_CATCH(); - { - if (desc->type == RB_OBJ_INDEX) { - UnlockRelationOid(heapOid, AccessExclusiveLock); - } - PG_RE_THROW(); - } - PG_END_TRY(); -} - -static void TrUnlockTrItem(TrObjDesc *desc) -{ - UnlockDatabaseObject(RecyclebinRelationId, desc->id, 0, - AccessExclusiveLock); -} - -static void TrLockTrItem(TrObjDesc *desc) -{ - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[1]; - HeapTuple tup; - - /* 1. Try to lock rb item in AccessExclusiveLock */ - if (!ConditionalLockDatabaseObject(RecyclebinRelationId, desc->id, 0, AccessExclusiveLock)) { - ereport(ERROR, - (errcode(ERRCODE_RBIN_LOCK_NOT_AVAILABLE), - errmsg("could not obtain lock on recycle object '%s'", desc->name))); - } - - /* - * 2. Now that we have the lock, probe to see if the rb item really - * exists or not. - */ - AcceptInvalidationMessages(); - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(desc->id)); - - sd = systable_beginscan(rbRel, RecyclebinIdIndexId, true, NULL, 1, skey); - if ((tup = systable_getnext(sd)) == NULL) { - UnlockDatabaseObject(RecyclebinRelationId, desc->id, 0, AccessExclusiveLock); - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - ereport(ERROR, - (errcode(ERRCODE_RBIN_UNDEFINED_OBJECT), - errmsg("recycle object \"%s\" does not exist", desc->name))); - } - - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return; -} - -static void TrOperMatch(const TrObjDesc *desc, TrOperMode operMode) -{ - switch (operMode) { - case RB_OPER_PURGE: - if (!desc->canpurge && desc->type != RB_OBJ_PARTITION) { - ereport(ERROR, - (errcode(ERRCODE_INVALID_OPERATION), - errmsg("recycle object \"%s\" cannot be purged", desc->name))); - } - break; - - case RB_OPER_RESTORE_DROP: - if ((!desc->canrestore && desc->type != RB_OBJ_PARTITION) || desc->operation != RB_OPER_DROP) { - ereport(ERROR, - (errcode(ERRCODE_INVALID_OPERATION), - errmsg("recycle object \"%s\" cannot be restored", desc->name))); - } - break; - - case RB_OPER_RESTORE_TRUNCATE: - if ((!desc->canrestore && desc->type != RB_OBJ_PARTITION) || desc->operation != RB_OPER_TRUNCATE) { - ereport(ERROR, - (errcode(ERRCODE_INVALID_OPERATION), - errmsg("recycle object \"%s\" cannot be restored", desc->name))); - } - break; - - default: - ereport(ERROR, - (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), - errmsg("unrecognized recyclebin operation: %u", operMode))); - break; - } -} - -/* - * Fetch object from recycle bin for rb operations - purge, restore : - * Prefer to fetch as original name, then recycle name. - */ -void TrOperFetch(const RangeVar *purobj, TrObjType objtype, TrObjDesc *desc, TrOperMode operMode) -{ - bool found = false; - - AcceptInvalidationMessages(); - - /* Prefer to fetch as original name */ - found = TrFetchOriname(purobj->schemaname, purobj->relname, objtype, desc, operMode); - /* if not found, then fetch as recycle name */ - if (!found) { - found = TrFetchName(purobj->relname, objtype, desc, operMode); - } - - /* not found, throw error */ - if (!found) { - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_TABLE), - errmsg("recycle object \"%s\" desired does not exist", purobj->relname))); - } - - TrOperMatch(desc, operMode); - - return; -} - -static void TrPermRestore(TrObjDesc *desc, TrOperMode operMode) -{ - AclResult aclCreateResult; - - /* Check namespace permissions. */ - aclCreateResult = pg_namespace_aclcheck(desc->nspace, desc->authid, ACL_CREATE); - if (aclCreateResult != ACLCHECK_OK) { - aclcheck_error(aclCreateResult, ACL_KIND_NAMESPACE, get_namespace_name(desc->nspace)); - } - - AclResult aclUsageResult = pg_namespace_aclcheck(desc->nspace, desc->authid, ACL_USAGE); - if (aclUsageResult != ACLCHECK_OK) { - aclcheck_error(aclUsageResult, ACL_KIND_NAMESPACE, get_namespace_name(desc->nspace)); - } - - /* Allow restore to either table owner or schema owner */ - if (!pg_class_ownercheck(desc->relid, desc->authid) && !pg_namespace_ownercheck(desc->nspace, desc->authid)) { - aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_CLASS, desc->name); - return; - } - - if (operMode == RB_OPER_RESTORE_TRUNCATE) { - AclResult aclTruncateResult = pg_class_aclcheck(desc->relid, desc->authid, ACL_TRUNCATE); - if (aclTruncateResult != ACLCHECK_OK) { - aclcheck_error(aclTruncateResult, ACL_KIND_CLASS, desc->name); - } - } -} - -static void TrPermPurge(TrObjDesc *desc, TrOperMode operMode) -{ - AclResult result; - - result = pg_namespace_aclcheck(desc->nspace, desc->authid, ACL_USAGE); - if (result != ACLCHECK_OK) { - aclcheck_error(result, ACL_KIND_NAMESPACE, get_namespace_name(desc->nspace)); - } - if (!pg_class_ownercheck(desc->relid, desc->authid) && !pg_namespace_ownercheck(desc->nspace, desc->authid)) { - aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_CLASS, desc->name); - } -} - -/* - * Check permission for rb operations - purge, restore - */ -static void TrPerm(TrObjDesc *desc, TrOperMode operMode) -{ - switch (operMode) { - case RB_OPER_RESTORE_DROP: - case RB_OPER_RESTORE_TRUNCATE: - TrPermRestore(desc, operMode); - break; - case RB_OPER_PURGE: - TrPermPurge(desc, operMode); - break; - default: - /* Never reached here. */ - Assert(0); - break; - } -} - -/* - * Prepare for rb operations - purge, restore : - * check permission, lock objects - */ -void TrOperPrep(TrObjDesc *desc, TrOperMode operMode) -{ - bool needLockRelation = false; - - /* - * 1. Check permission. - */ - TrPerm(desc, operMode); - - /* - * 2. Acquire lock on rb item, avoid concurrently purge, restore. - */ - TrLockTrItem(desc); - - /* - * 3. Acquire lock on relation, avoid concurrently DQL. - * Notice: ignore this step when we purge truncated relation - * as base relation may not exists. - */ - needLockRelation = !(operMode == RB_OPER_PURGE && desc->operation == RB_OPER_TRUNCATE); - if (needLockRelation) { - /* Use TRY-CATCH block to clean locks already held if error. */ - PG_TRY(); - { - TrLockRelation(desc); - } - PG_CATCH(); - { - TrUnlockTrItem(desc); - PG_RE_THROW(); - } - PG_END_TRY(); - } -} - -bool NeedTrComm(Oid relid) -{ - Relation rel; - Form_pg_class classForm; - - if (/* - *Disable Recyclebin-based-Drop/Truncate when - */ - /* recyclebin disabled, or */ - !u_sess->attr.attr_storage.enable_recyclebin || - /* target db is template1, or */ - u_sess->proc_cxt.MyDatabaseId == TemplateDbOid || - /* in maintenance mode, or */ - u_sess->attr.attr_common.xc_maintenance_mode || - /* in in-place upgrade mode, or */ - t_thrd.proc->workingVersionNum < 92350 || - /* in non-singlenode mode, or */ - (g_instance.role != VSINGLENODE) || - /* in bootstrap mode. */ - IsInitdb) { - return false; - } - - rel = relation_open(relid, NoLock); - classForm = rel->rd_rel; - if (/* - * Disable Recyclebin-based-Drop/Truncate if - */ - /* table is non ordinary table, or */ - classForm->relkind != RELKIND_RELATION || - /* is non heap table, or */ - rel->rd_tam_type == TAM_HEAP || - /* is non regular table, or */ - classForm->relpersistence != RELPERSISTENCE_PERMANENT || - /* is shared table across databases, or */ - classForm->relisshared || - /* has derived classes, or */ - classForm->relhassubclass || - /* has any PARTIAL CLUSTER KEY, or */ - classForm->relhasclusterkey || - /* is cstore table, or */ - (rel->rd_options && StdRelOptIsColStore(rel->rd_options)) || RelationIsColStore(rel) || - /* is hbkt table, or */ - (RELATION_HAS_BUCKET(rel) || RELATION_OWN_BUCKET(rel)) || - /* is dfs table, or */ - RelationIsPAXFormat(rel) || - /* is resizing, or */ - RelationInClusterResizing(rel) || - /* is in system namespace. */ - (IsSystemNamespace(classForm->relnamespace) || IsToastNamespace(classForm->relnamespace) || - IsCStoreNamespace(classForm->relnamespace))) { - relation_close(rel, NoLock); - return false; - } - - relation_close(rel, NoLock); - - return true; -} - -TrObjType TrGetObjType(Oid nspId, char relKind) -{ - TrObjType type = RB_OBJ_TABLE; - - switch (relKind) { - case RELKIND_INDEX: - type = IsToastNamespace(nspId) ? RB_OBJ_TOAST_INDEX : RB_OBJ_INDEX; - break; - case RELKIND_RELATION: - type = RB_OBJ_TABLE; - break; - case RELKIND_SEQUENCE: - case RELKIND_LARGE_SEQUENCE: - type = RB_OBJ_SEQUENCE; - break; - case RELKIND_TOASTVALUE: - type = RB_OBJ_TOAST; - break; - case PARTTYPE_PARTITIONED_RELATION: - type = RB_OBJ_PARTITION; - break; - case RELKIND_GLOBAL_INDEX: - type = RB_OBJ_GLOBAL_INDEX; - break; - case RELKIND_MATVIEW: - type = RB_OBJ_MATVIEW; - break; - default: - /* Never reached here. */ - Assert(0); - break; - } - - return type; -} - -static bool TrObjAddrExists(Oid classid, Oid objid, ObjectAddresses *objSet) -{ - int i; - - for (i = 0; i < objSet->numrefs; i++) { - if (TrObjIsEqualEx(classid, objid, &objSet->refs[i])) { - return true; - } - } - - return false; -} - -/* - * output: refobjs - */ -void TrFindAllRefObjs(Relation depRel, const ObjectAddress *subobj, - ObjectAddresses *refobjs, bool ignoreObjSubId) -{ - SysScanDesc sd; - HeapTuple tuple; - ScanKeyData key[3]; - int nkeys; - - ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(subobj->classId)); - ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(subobj->objectId)); - nkeys = 2; - if (!ignoreObjSubId && subobj->objectSubId != 0) { - ScanKeyInit(&key[2], Anum_pg_depend_objsubid, BTEqualStrategyNumber, F_INT4EQ, - Int32GetDatum(subobj->objectSubId)); - nkeys = 3; - } - - sd = systable_beginscan(depRel, DependDependerIndexId, true, NULL, nkeys, key); - while (HeapTupleIsValid(tuple = systable_getnext(sd))) { - Form_pg_depend depForm = (Form_pg_depend)GETSTRUCT(tuple); - /* Cascaded clean rb object in `DROP SCHEMA` command. */ - if (depForm->refclassid == NamespaceRelationId) { - continue; - } - - /* We keep `objSet` unique when `ignoreObjSubId = true` to avoid circle recursive. */ - if (!ignoreObjSubId || !TrObjAddrExists(depForm->refclassid, depForm->refobjid, refobjs)) { - add_object_address_ext(depForm->refclassid, depForm->refobjid, - depForm->refobjsubid, depForm->deptype, refobjs); - } - } - - systable_endscan(sd); - return; -} - -static void TrFindAllInternalObjs(Relation depRel, const ObjectAddress *refobj, - ObjectAddresses *objSet, bool ignoreObjSubId = false) -{ - SysScanDesc sd; - HeapTuple tuple; - ScanKeyData key[3]; - int nkeys; - - ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(refobj->classId)); - ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(refobj->objectId)); - nkeys = 2; - if (!ignoreObjSubId && refobj->objectSubId != 0) { - ScanKeyInit(&key[2], Anum_pg_depend_refobjsubid, BTEqualStrategyNumber, F_INT4EQ, - Int32GetDatum(refobj->objectSubId)); - nkeys = 3; - } - - sd = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, nkeys, key); - while (HeapTupleIsValid(tuple = systable_getnext(sd))) { - Form_pg_depend depForm = (Form_pg_depend)GETSTRUCT(tuple); - if (depForm->deptype != 'i') { - continue; - } - - /* We keep `objSet` unique when `ignoreObjSubId = true` to avoid circle recursive. */ - if (!ignoreObjSubId || !TrObjAddrExists(depForm->classid, depForm->objid, objSet)) { - add_object_address_ext(depForm->classid, depForm->objid, - depForm->objsubid, depForm->deptype, objSet); - } - } - - systable_endscan(sd); - return; -} - -static void TrDoPurgeObject(TrObjDesc *desc) -{ - if (desc->operation == RB_OPER_DROP) { - TrDoPurgeObjectDrop(desc); - } else { - TrDoPurgeObjectTruncate(desc); - } -} - -void TrPurgeObject(RangeVar *purobj, TrObjType type) -{ - TrObjDesc desc; - - TrOperFetch(purobj, type, &desc, RB_OPER_PURGE); - - desc.authid = GetUserId(); - TrOperPrep(&desc, RB_OPER_PURGE); - - TrDoPurgeObject(&desc); - - return; -} - -const int PURGE_BATCH = 64; -const int PURGE_SINGL = 64; -typedef void (*TrFetchBeginHook)(SysScanDesc *sd, Oid objId); -typedef bool (*TrFetchMatchHook)(Relation rbRel, HeapTuple rbTup, Oid objId); - -static void TrFetchBegin(TrFetchBeginHook fetchHook, SysScanDesc *sd, Oid objId) -{ - fetchHook(sd, objId); -} - -// @return: true for eof -static bool TrFetchExec(TrFetchMatchHook matchHook, Oid objId, SysScanDesc sd, TrObjDesc *desc) -{ - HeapTuple tup; - while ((tup = systable_getnext(sd)) != NULL) { - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); - if ((rbForm->rcytype == RB_OBJ_TABLE) && matchHook(sd->heap_rel, tup, objId)) { - Assert (rbForm->rcycanpurge); - TrDescRead(desc, tup); - return false; - } else if ((rbForm->rcytype == RB_OBJ_PARTITION) && matchHook(sd->heap_rel, tup, objId)) { - Assert (!rbForm->rcycanpurge); - TrDescRead(desc, tup); - return false; - } - } - return true; -} - -static void TrFetchEnd(SysScanDesc sd) -{ - Relation rbRel = sd->heap_rel; - - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); -} - -static bool TrPurgeBatch(TrFetchBeginHook beginHook, TrFetchMatchHook matchHook, - Oid objId, Oid roleid, uint32 maxBatch, PurgeMsgRes *localRes) -{ - SysScanDesc sd = NULL; - TrObjDesc desc; - uint32 count = 0; - bool eof = false; - - RbMsgResetRes(localRes); - - StartTransactionCommand(); - - TrFetchBegin(beginHook, &sd, objId); - while (!(eof = TrFetchExec(matchHook, objId, sd, &desc))) { - CHECK_FOR_INTERRUPTS(); - - PG_TRY(); - { - desc.authid = roleid; - TrOperPrep(&desc, RB_OPER_PURGE); - - TrDoPurgeObject(&desc); - localRes->purgedNum++; - } - PG_CATCH(); - { - int errcode = geterrcode(); - if (errcode == ERRCODE_RBIN_LOCK_NOT_AVAILABLE) { - errno_t rc; - rc = strncpy_s(localRes->errMsg, RB_MAX_ERRMSG_SIZE, Geterrmsg(), RB_MAX_ERRMSG_SIZE - 1); - securec_check(rc, "\0", "\0"); - localRes->skippedNum++; - } else if (errcode == ERRCODE_RBIN_UNDEFINED_OBJECT) { - localRes->undefinedNum++; - } else { - PG_RE_THROW(); - } - } - PG_END_TRY(); - - if (++count >= maxBatch) { - break; - } - } - - TrFetchEnd(sd); - - CommitTransactionCommand(); - - return eof; -} - -static void TrFetchBeginSpace(SysScanDesc *sd, Oid spcId) -{ - ScanKeyData skey[2]; - Relation rbRel; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcytablespace, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(spcId)); - ScanKeyInit(&skey[1], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); - - *sd = systable_beginscan(rbRel, RecyclebinDbidSpcidRcycsnIndexId, true, NULL, 2, skey); -} - -static bool TrFetchMatchSpace(Relation rbRel, HeapTuple rbTup, Oid objId) -{ - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(rbTup); - return rbForm->rcytablespace == objId; -} - -void TrPurgeTablespace(int64 id) -{ - PurgeMsgReq *req = &RbMsg(id)->req; - PurgeMsgRes localRes; - bool eof = false; - - do { - eof = TrPurgeBatch(TrFetchBeginSpace, TrFetchMatchSpace, req->objId, req->authId, PURGE_BATCH, &localRes); - RbMsgSetStatistics(id, &localRes); - } while (!eof && localRes.skippedNum == 0); -} - -void TrPurgeTablespaceDML(int64 id) -{ - PurgeMsgReq *req = &RbMsg(id)->req; - PurgeMsgRes localRes; - bool eof = false; - - do { - eof = TrPurgeBatch(TrFetchBeginSpace, TrFetchMatchSpace, req->objId, req->authId, PURGE_SINGL, &localRes); - RbMsgSetStatistics(id, &localRes); - } while (!eof && localRes.purgedNum == 0); -} - -static void TrFetchBeginRecyclebin(SysScanDesc *sd, Oid objId) -{ - ScanKeyData skey[2]; - Relation rbRel; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); - - *sd = systable_beginscan(rbRel, RecyclebinDbidRelidIndexId, true, NULL, 1, skey); -} - -static bool TrFetchMatchRecyclebin(Relation rbRel, HeapTuple rbTup, Oid objId) -{ - return true; -} - -void TrPurgeRecyclebin(int64 id) -{ - PurgeMsgReq *req = &RbMsg(id)->req; - PurgeMsgRes localRes; - bool eof = false; - - do { - eof = TrPurgeBatch(TrFetchBeginRecyclebin, TrFetchMatchRecyclebin, - InvalidOid, req->authId, PURGE_BATCH, &localRes); - RbMsgSetStatistics(id, &localRes); - } while (!eof && localRes.skippedNum == 0); -} - -static void TrFetchBeginSchema(SysScanDesc *sd, Oid objId) -{ - ScanKeyData skey[2]; - Relation rbRel; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcynamespace, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(objId)); - ScanKeyInit(&skey[1], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); - - *sd = systable_beginscan(rbRel, RecyclebinDbidNspOrinameIndexId, true, NULL, 2, skey); -} - -static bool TrFetchMatchSchema(Relation rbRel, HeapTuple rbTup, Oid objId) -{ - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(rbTup); - return rbForm->rcynamespace == objId; -} - -void TrPurgeSchema(int64 id) -{ - PurgeMsgReq *req = &RbMsg(id)->req; - PurgeMsgRes localRes; - bool eof = false; - - do { - eof = TrPurgeBatch(TrFetchBeginSchema, TrFetchMatchSchema, req->objId, req->authId, PURGE_BATCH, &localRes); - RbMsgSetStatistics(id, &localRes); - } while (!eof && localRes.skippedNum == 0); -} - -static void TrFetchBeginUser(SysScanDesc *sd, Oid objId) -{ - ScanKeyData skey[2]; - Relation rbRel; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); - - *sd = systable_beginscan(rbRel, RecyclebinDbidRelidIndexId, true, NULL, 1, skey); -} - -static bool TrFetchMatchUser(Relation rbRel, HeapTuple rbTup, Oid objId) -{ - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(rbTup); - return rbForm->rcyowner == objId; -} - -void TrPurgeUser(int64 id) -{ - PurgeMsgReq *req = &RbMsg(id)->req; - PurgeMsgRes localRes; - bool eof = false; - - do { - eof = TrPurgeBatch(TrFetchBeginUser, TrFetchMatchUser, req->objId, req->authId, PURGE_BATCH, &localRes); - RbMsgSetStatistics(id, &localRes); - } while (!eof && localRes.skippedNum == 0); -} - -static void TrFetchBeginAuto(SysScanDesc *sd, Oid objId) -{ - ScanKeyData skey[2]; - Relation rbRel; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); - - *sd = systable_beginscan(rbRel, RecyclebinDbidRelidIndexId, true, NULL, 1, skey); -} - -static bool TrFetchMatchAuto(Relation rbRel, HeapTuple rbTup, Oid objId) -{ - bool isNull = false; - Datum datumRcyTime = heap_getattr(rbTup, Anum_pg_recyclebin_rcyrecycletime, - RelationGetDescr(rbRel), &isNull); - - long secs; - int msecs; - TimestampDifference(isNull ? 0 : DatumGetTimestampTz(datumRcyTime), - GetCurrentTimestamp(), &secs, &msecs); - - return secs > u_sess->attr.attr_storage.recyclebin_retention_time || secs < 0; -} - -void TrPurgeAuto(int64 id) -{ - PurgeMsgReq *req = &RbMsg(id)->req; - PurgeMsgRes localRes; - bool eof = false; - do { - eof = TrPurgeBatch(TrFetchBeginAuto, TrFetchMatchAuto, InvalidOid, req->authId, PURGE_BATCH, &localRes); - RbMsgSetStatistics(id, &localRes); - } while (!eof); -} - -void TrSwapRelfilenode(Relation rbRel, HeapTuple rbTup, bool isPart) -{ - Relation relRel; - HeapTuple relTup; - HeapTuple newTup; - TrObjDesc desc; - int maxNattr = 0; - Datum *values = NULL; - bool *nulls = NULL; - bool *replaces = NULL; - NameData name; - errno_t rc = EOK; - bool isNull = false; - int relfilenoIndex = 0; - int frozenxidIndex = 0; - int frozenxid64Index = 0; - bool isPartition = false; - - TrDescRead(&desc, rbTup); - - if (desc.type == RB_OBJ_PARTITION || (desc.type == RB_OBJ_INDEX && isPart)) { - isPartition = true; - } - if (isPartition) { - maxNattr = Max(Natts_pg_partition, Natts_pg_recyclebin); - relRel = heap_open(PartitionRelationId, RowExclusiveLock); - relTup = SearchSysCacheCopy1(PARTRELID, ObjectIdGetDatum(desc.relid)); - relfilenoIndex = Anum_pg_partition_relfilenode; - frozenxidIndex = Anum_pg_partition_relfrozenxid; - frozenxid64Index = Anum_pg_partition_relfrozenxid64; - } else { - maxNattr = Max(Natts_pg_class, Natts_pg_recyclebin); - relRel = heap_open(RelationRelationId, RowExclusiveLock); - relTup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(desc.relid)); - relfilenoIndex = Anum_pg_class_relfilenode; - frozenxidIndex = Anum_pg_class_relfrozenxid; - frozenxid64Index = Anum_pg_class_relfrozenxid64; - } - - /* 1. Update pg_class or pg_partition */ - if (!HeapTupleIsValid(relTup)) { - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_TABLE), - errmsg("cache lookup failed for relation %u", desc.relid))); - } - - values = (Datum *)palloc0(sizeof(Datum) * maxNattr); - nulls = (bool *)palloc0(sizeof(bool) * maxNattr); - replaces = (bool *)palloc0(sizeof(bool) * maxNattr); - - replaces[relfilenoIndex - 1] = true; - values[relfilenoIndex - 1] = ObjectIdGetDatum(desc.relfilenode); - - replaces[frozenxidIndex - 1] = true; - values[frozenxidIndex - 1] = ShortTransactionIdGetDatum(desc.frozenxid); - - replaces[frozenxid64Index - 1] = true; - values[frozenxid64Index - 1] = TransactionIdGetDatum(desc.frozenxid64); - - newTup = heap_modify_tuple(relTup, RelationGetDescr(relRel), values, nulls, replaces); - - simple_heap_update(relRel, &newTup->t_self, newTup); - - CatalogUpdateIndexes(relRel, newTup); - - heap_freetuple_ext(newTup); - - /* 2. Update pg_recyclebin */ - rc = memset_s(values, sizeof(Datum) * maxNattr, 0, sizeof(Datum) * maxNattr); - securec_check(rc, "\0", "\0"); - rc = memset_s(nulls, sizeof(bool) * maxNattr, false, sizeof(bool) * maxNattr); - securec_check(rc, "\0", "\0"); - rc = memset_s(replaces, sizeof(bool) * maxNattr, false, sizeof(bool) * maxNattr); - securec_check(rc, "\0", "\0"); - - (void)TrGenObjName(NameStr(name), RelationRelationId, desc.relid); - replaces[Anum_pg_recyclebin_rcyname - 1] = true; - values[Anum_pg_recyclebin_rcyname - 1] = NameGetDatum(&name); - - replaces[Anum_pg_recyclebin_rcyoriginname - 1] = true; - if (isPartition) { - values[Anum_pg_recyclebin_rcyoriginname - 1] = NameGetDatum(&desc.originname); - } else { - values[Anum_pg_recyclebin_rcyoriginname - 1] = NameGetDatum(&((Form_pg_class)GETSTRUCT(relTup))->relname); - } - - replaces[Anum_pg_recyclebin_rcyrecyclecsn - 1] = true; - values[Anum_pg_recyclebin_rcyrecyclecsn - 1] = Int64GetDatum(t_thrd.xact_cxt.ShmemVariableCache->nextCommitSeqNo); - - replaces[Anum_pg_recyclebin_rcyrecycletime - 1] = true; - values[Anum_pg_recyclebin_rcyrecycletime - 1] = TimestampTzGetDatum(GetCurrentTimestamp()); - - replaces[Anum_pg_recyclebin_rcyrelfilenode - 1] = true; - if (isPartition) { - values[Anum_pg_recyclebin_rcyrelfilenode - 1] = - ObjectIdGetDatum(((Form_pg_partition)GETSTRUCT(relTup))->relfilenode); - } else { - values[Anum_pg_recyclebin_rcyrelfilenode - 1] = - ObjectIdGetDatum(((Form_pg_class)GETSTRUCT(relTup))->relfilenode); - } - - replaces[Anum_pg_recyclebin_rcyfrozenxid - 1] = true; - if (isPartition) { - values[Anum_pg_recyclebin_rcyfrozenxid - 1] = - ShortTransactionIdGetDatum(((Form_pg_partition)GETSTRUCT(relTup))->relfrozenxid); - } else { - values[Anum_pg_recyclebin_rcyfrozenxid - 1] = - ShortTransactionIdGetDatum(((Form_pg_class)GETSTRUCT(relTup))->relfrozenxid); - } - - replaces[Anum_pg_recyclebin_rcyfrozenxid64 - 1] = true; - Datum xid64datum = heap_getattr(relTup, frozenxid64Index, RelationGetDescr(relRel), &isNull); - values[Anum_pg_recyclebin_rcyfrozenxid64 - 1] = DatumGetTransactionId(xid64datum); - - newTup = heap_modify_tuple(rbTup, RelationGetDescr(rbRel), values, nulls, replaces); - - simple_heap_update(rbRel, &newTup->t_self, newTup); - - CatalogUpdateIndexes(rbRel, newTup); - - heap_freetuple_ext(newTup); - - pfree(values); - pfree(nulls); - pfree(replaces); - - heap_freetuple_ext(relTup); - heap_close(relRel, RowExclusiveLock); - return; -} - -void TrBaseRelMatched(TrObjDesc *baseDesc) -{ - ObjectAddress obj = {RelationRelationId, baseDesc->relid}; - if (TrIsRefRbObject(&obj)) { - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("relation \"%s\" does not exist", baseDesc->originname))); - } - - Relation rel = RelationIdGetRelation(baseDesc->relid); - Assert(RelationIsValid(rel)); - if (RelationGetCreatecsn(rel) != (CommitSeqNo)baseDesc->createcsn) { - ereport(ERROR, - (errmsg("The recycle object \"%s\" and relation \"%s\" mismatched.", - baseDesc->name, RelationGetRelationName(rel)))); - } - - if (RelationGetChangecsn(rel) > (CommitSeqNo)baseDesc->changecsn) { - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("The table definition of \"%s\" has been changed.", - RelationGetRelationName(rel)))); - } - - RelationClose(rel); -} - -void TrAdjustFrozenXid64(Oid dbid, TransactionId *frozenXID) -{ - Relation rbRel; - SysScanDesc sd; - HeapTuple rbtup; - - if (!TcapFeatureAvail()) { - return; - } - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - sd = systable_beginscan(rbRel, InvalidOid, false, NULL, 0, NULL); - while ((rbtup = systable_getnext(sd)) != NULL) { - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(rbtup); - TransactionId rcyfrozenxid64; - - if (rbForm->rcydbid != dbid || (rbForm->rcytype != RB_OBJ_TABLE && rbForm->rcytype != RB_OBJ_TOAST)) { - continue; - } - - rcyfrozenxid64 = TrRbGetRcyfrozenxid64(rbtup, rbRel); - Assert(TransactionIdIsNormal(rcyfrozenxid64)); - - if (TransactionIdPrecedes(rcyfrozenxid64, *frozenXID)) { - *frozenXID = rcyfrozenxid64; - } - } - - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return; -} - -bool TrRbIsEmptyDb(Oid dbid) -{ - Relation rbRel; - SysScanDesc sd; - HeapTuple tup; - ScanKeyData skey[1]; - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(dbid)); - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - sd = systable_beginscan(rbRel, RecyclebinDbidRelidIndexId, true, NULL, 1, skey); - tup = systable_getnext(sd); - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return tup == NULL; -} - -bool TrRbIsEmptySpc(Oid spcId) -{ - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[1]; - HeapTuple tup; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcytablespace, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(spcId)); - - sd = systable_beginscan(rbRel, RecyclebinDbidSpcidRcycsnIndexId, true, NULL, 1, skey); - tup = systable_getnext(sd); - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return tup == NULL; -} - -bool TrRbIsEmptySchema(Oid nspId) -{ - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[2]; - HeapTuple tup; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcynamespace, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(nspId)); - ScanKeyInit(&skey[1], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); - - sd = systable_beginscan(rbRel, RecyclebinDbidNspOrinameIndexId, true, NULL, 2, skey); - tup = systable_getnext(sd); - - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return tup == NULL; -} - -bool TrRbIsEmptyUser(Oid roleId) -{ - Relation rbRel; - SysScanDesc sd; - HeapTuple tup; - bool found = false; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - sd = systable_beginscan(rbRel, InvalidOid, false, NULL, 0, NULL); - - while ((tup = systable_getnext(sd)) != NULL) { - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); - if ((TrObjType)rbForm->rcytype != RB_OBJ_TABLE || rbForm->rcyowner != roleId) { - continue; - } - - found = true; - break; - } - - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return !found; -} - -static bool TrOidExists(const List *lOid, Oid oid) -{ - ListCell *cell = NULL; - if (lOid == NULL) { - return false; - } - - foreach (cell, lOid) { - if (oid == (*(Oid *)lfirst(cell))) { - return true; - } - } - return false; -} - -List *TrGetDbListRcy(void) -{ - Relation rbRel; - SysScanDesc sd; - HeapTuple tup; - - List *lName = NIL; - List *lOid = NIL; - char *dbname = NULL; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - sd = systable_beginscan(rbRel, InvalidOid, false, NULL, 0, NULL); - - while ((tup = systable_getnext(sd)) != NULL) { - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); - if (TrOidExists(lOid, rbForm->rcydbid)) { - continue; - } - Oid *oid = (Oid *)palloc0(sizeof(Oid)); - *oid = rbForm->rcydbid; - lOid = lappend(lOid, oid); - - dbname = get_database_name(rbForm->rcydbid); - if (dbname == NULL) { - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_DATABASE), - errmsg("database \"%u\" does not exist", rbForm->rcydbid))); - } - lName = lappend(lName, dbname); - } - - list_free_deep(lOid); - - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return lName; -} - -List *TrGetDbListSpc(Oid spcId) -{ - List *lName = NIL; - List *lOid = NIL; - char *dbname = NULL; - - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[1]; - HeapTuple tup; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcytablespace, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(spcId)); - - sd = systable_beginscan(rbRel, RecyclebinDbidSpcidRcycsnIndexId, true, NULL, 1, skey); - - while ((tup = systable_getnext(sd)) != NULL) { - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); - if (TrOidExists(lOid, rbForm->rcydbid)) { - continue; - } - Oid *oid = (Oid *)palloc0(sizeof(Oid)); - *oid = rbForm->rcydbid; - lOid = lappend(lOid, oid); - - dbname = get_database_name(rbForm->rcydbid); - if (dbname == NULL) { - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_DATABASE), - errmsg("database \"%u\" does not exist", rbForm->rcydbid))); - } - lName = lappend(lName, dbname); - } - - list_free_deep(lOid); - - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return lName; -} - -List *TrGetDbListSchema(Oid nspId) -{ - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[1]; - HeapTuple tup; - - List *lName = NIL; - List *lOid = NIL; - char *dbname = NULL; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcynamespace, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(nspId)); - - sd = systable_beginscan(rbRel, RecyclebinDbidNspOrinameIndexId, true, NULL, 1, skey); - - while ((tup = systable_getnext(sd)) != NULL) { - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); - if (TrOidExists(lOid, rbForm->rcydbid)) { - continue; - } - Oid *oid = (Oid *)palloc0(sizeof(Oid)); - *oid = rbForm->rcydbid; - lOid = lappend(lOid, oid); - - dbname = get_database_name(rbForm->rcydbid); - if (dbname == NULL) { - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_DATABASE), - errmsg("database \"%u\" does not exist", rbForm->rcydbid))); - } - lName = lappend(lName, dbname); - } - - list_free_deep(lOid); - - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return lName; -} - -List *TrGetDbListUser(Oid roleId) -{ - Relation rbRel; - SysScanDesc sd; - HeapTuple tup; - - List *lName = NIL; - List *lOid = NIL; - char *dbname = NULL; - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - - sd = systable_beginscan(rbRel, InvalidOid, false, NULL, 0, NULL); - - while ((tup = systable_getnext(sd)) != NULL) { - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); - if ((TrObjType)rbForm->rcytype != RB_OBJ_TABLE || rbForm->rcyowner != roleId) { - continue; - } - if (TrOidExists(lOid, rbForm->rcydbid)) { - continue; - } - Oid *oid = (Oid *)palloc0(sizeof(Oid)); - *oid = rbForm->rcydbid; - lOid = lappend(lOid, oid); - - dbname = get_database_name(rbForm->rcydbid); - if (dbname == NULL) { - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_DATABASE), - errmsg("database \"%u\" does not exist", rbForm->rcydbid))); - } - - lName = lappend(lName, dbname); - } - - list_free_deep(lOid); - - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return lName; -} - -/* - * TrGetDatabaseList - * Return a list of all databases found in pg_database. - */ -List *TrGetDbListAuto(void) -{ - List* dblist = NIL; - Relation rel; - SysScanDesc sd; - HeapTuple tup; - - rel = heap_open(DatabaseRelationId, AccessShareLock); - sd = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); - - while ((tup = systable_getnext(sd)) != NULL) { - Form_pg_database pgdatabase = (Form_pg_database)GETSTRUCT(tup); - if (strcmp(NameStr(pgdatabase->datname), "template0") == 0 || - strcmp(NameStr(pgdatabase->datname), "template1") == 0) { - continue; - } - dblist = lappend(dblist, pstrdup(NameStr(pgdatabase->datname))); - } - - systable_endscan(sd); - heap_close(rel, AccessShareLock); - - return dblist; -} - -static bool TrObjInRecyclebin(const ObjectAddress *obj) -{ - Relation rbRel; - SysScanDesc sd; - HeapTuple tup; - ScanKeyData skey[2]; - bool found = false; - - if (getObjectClass(obj) != OCLASS_CLASS) { - return false; - } - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); - ScanKeyInit(&skey[1], Anum_pg_recyclebin_rcyrelid, BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(obj->objectId)); - - rbRel = heap_open(RecyclebinRelationId, AccessShareLock); - sd = systable_beginscan(rbRel, RecyclebinDbidRelidIndexId, true, NULL, 2, skey); - while ((tup = systable_getnext(sd)) != NULL) { - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); - if ((TrObjType)rbForm->rcyoperation == 'd') { - found = true; - break; - } - } - systable_endscan(sd); - heap_close(rbRel, AccessShareLock); - - return found; -} - -/* - * May this object be a recyclebin object? - * true: with "BIN$" prefix, or not a Relation\Type\Trigger\Constraint\Rule - * false: without "BIN$" prefix, or not exists - */ -static bool TrMaybeRbObject(Oid classid, Oid objid, const char *objname = NULL) -{ - HeapTuple tup; - - /* Note: we preserve rule origin name when RbDrop. */ - if (classid != RewriteRelationId && objname) { - return strncmp(objname, "BIN$", 4) == 0; - } - - switch (classid) { - case RelationRelationId: - tup = SearchSysCache1(RELOID, ObjectIdGetDatum(objid)); - if (tup != NULL) { - objname = NameStr(((Form_pg_class)GETSTRUCT(tup))->relname); - ReleaseSysCache(tup); - } - break; - case TypeRelationId: - tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(objid)); - if (tup != NULL) { - objname = NameStr(((Form_pg_type)GETSTRUCT(tup))->typname); - ReleaseSysCache(tup); - } - break; - case TriggerRelationId: { - Relation relTrig; - ScanKeyData skey[1]; - SysScanDesc sd; - - relTrig = heap_open(TriggerRelationId, AccessShareLock); - ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, - F_OIDEQ, ObjectIdGetDatum(objid)); - sd = systable_beginscan(relTrig, TriggerOidIndexId, true, NULL, 1, skey); - if ((tup = systable_getnext(sd)) != NULL) { - objname = NameStr(((Form_pg_trigger)GETSTRUCT(tup))->tgname); - } - systable_endscan(sd); - heap_close(relTrig, AccessShareLock); - break; - } - case ConstraintRelationId: - tup = SearchSysCache1(CONSTROID, ObjectIdGetDatum(objid)); - if (tup != NULL) { - objname = NameStr(((Form_pg_constraint)GETSTRUCT(tup))->conname); - ReleaseSysCache(tup); - } - break; - case NamespaceRelationId: - /* Treate Namespace as non-recyclebin object. */ - return false; - default: - /* May be a recyclebin object. */ - return true; - } - - if (objname) { - return strncmp(objname, "BIN$", 4) == 0; - } - - return false; -} - -static bool TrIsRefRbObjectImpl(Relation depRel, const ObjectAddress *obj, ObjectAddresses *objSet) -{ - int startIdx; - - if (TrObjInRecyclebin(obj)) { - return true; - } - - startIdx = objSet->numrefs; - - if (!TrMaybeRbObject(obj->classId, obj->objectId)) { - return false; - } - - TrFindAllRefObjs(depRel, obj, objSet, true); - TrFindAllInternalObjs(depRel, obj, objSet, true); - - for (int i = startIdx; i < objSet->numrefs; i++) { - if (TrIsRefRbObjectImpl(depRel, &objSet->refs[i], objSet)) { - return true; - } - } - - return false; -} - -/* object is a rb object, or reference to a rb object. */ -static bool TrIsRefRbObject(const ObjectAddress *obj, Relation depRel) -{ - ObjectAddresses *objSet = new_object_addresses(); - bool relArgNull = depRel == NULL; - bool result = false; - - if (relArgNull) { - depRel = heap_open(DependRelationId, AccessShareLock); - } - - /* Note: we not care obj->deptype here. */ - add_object_address_ext1(obj, objSet); - - result = TrIsRefRbObjectImpl(depRel, obj, objSet); - - free_object_addresses(objSet); - - if (relArgNull) { - heap_close(depRel, AccessShareLock); - } - - return result; -} - -bool TrIsRefRbObjectEx(Oid classid, Oid objid, const char *objname) -{ - if (!TcapFeatureAvail()) { - return false; - } - - /* Note: we preserve rule origin name when RbDrop. */ - if (TrRbIsEmptyDb(u_sess->proc_cxt.MyDatabaseId)) { - return false; - } - - if (classid != RewriteRelationId && objname && strncmp(objname, "BIN$", 4) != 0) { - return false; - } - - ObjectAddress obj = {classid, objid}; - - return TrIsRefRbObject(&obj); -} - -void TrForbidAccessRbDependencies(Relation depRel, const ObjectAddress *depender, - const ObjectAddress *referenced, int nreferenced) -{ - if (!TcapFeatureAvail()) { - return; - } - - if (IsInitdb || TrRbIsEmptyDb(u_sess->proc_cxt.MyDatabaseId)) { - return; - } - - if (TrIsRefRbObject(depender, depRel)) { - elog (ERROR, "can not access recycle object."); - } - - for (int i = 0; i < nreferenced; i++, referenced++) { - if (TrIsRefRbObject(referenced, depRel)) { - elog (ERROR, "can not access recycle object."); - } - } - - return; -} - -void TrForbidAccessRbObject(Oid classid, Oid objid, const char *objname) -{ - if (!TcapFeatureAvail()) { - return; - } - - if (TrRbIsEmptyDb(u_sess->proc_cxt.MyDatabaseId) || !TrMaybeRbObject(classid, objid, objname)) { - return; - } - - ObjectAddress obj = {classid, objid}; - if (TrIsRefRbObject(&obj)) { - elog (ERROR, "can not access recycle object."); - } - - return; -} - -Datum gs_is_recycle_object(PG_FUNCTION_ARGS) -{ - int classid = PG_GETARG_INT32(0); - int objid = PG_GETARG_INT32(1); - Name objname = PG_GETARG_NAME(2); - bool result = false; - result = TrIsRefRbObjectEx(classid, objid, NameStr(*objname)); - PG_RETURN_BOOL(result); -} +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * tcap_manager.cpp + * Routines to support Timecapsule `Recyclebin-based query, restore`. + * We use Tr prefix to indicate it in following coding. + * + * IDENTIFICATION + * src/gausskernel/storage/tcap/tcap_manager.cpp + * + * --------------------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "pgstat.h" +#include "access/reloptions.h" +#include "access/sysattr.h" +#include "access/xlog.h" +#include "catalog/pg_database.h" +#include "catalog/dependency.h" +#include "catalog/heap.h" +#include "catalog/index.h" +#include "catalog/indexing.h" +#include "catalog/objectaccess.h" +#include "catalog/pg_collation_fn.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_constraint.h" +#include "catalog/pg_conversion_fn.h" +#include "catalog/pg_conversion.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_extension_data_source.h" +#include "catalog/pg_extension.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_job.h" +#include "catalog/pg_language.h" +#include "catalog/pg_largeobject.h" +#include "catalog/pg_object.h" +#include "catalog/pg_opclass.h" +#include "catalog/pg_operator.h" +#include "catalog/pg_opfamily.h" +#include "catalog/pg_partition_fn.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_recyclebin.h" +#include "catalog/pg_rewrite.h" +#include "catalog/pg_rlspolicy.h" +#include "catalog/pg_synonym.h" +#include "catalog/pg_tablespace.h" +#include "catalog/pg_trigger.h" +#include "catalog/pg_ts_config.h" +#include "catalog/pg_ts_dict.h" +#include "catalog/pg_ts_parser.h" +#include "catalog/pg_ts_template.h" +#include "catalog/pgxc_class.h" +#include "catalog/pg_partition.h" +#include "catalog/storage.h" +#include "commands/comment.h" +#include "commands/dbcommands.h" +#include "commands/directory.h" +#include "commands/extension.h" +#include "commands/proclang.h" +#include "commands/schemacmds.h" +#include "commands/seclabel.h" +#include "commands/sec_rls_cmds.h" +#include "commands/tablecmds.h" +#include "commands/tablespace.h" +#include "commands/trigger.h" +#include "commands/typecmds.h" +#include "executor/node/nodeModifyTable.h" +#include "rewrite/rewriteRemove.h" +#include "storage/lmgr.h" +#include "storage/predicate.h" +#include "storage/smgr/relfilenode.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/inval.h" +#include "utils/lsyscache.h" +#include "utils/relcache.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" + +#include "storage/tcap.h" +#include "storage/tcap_impl.h" + +static bool TrIsRefRbObject(const ObjectAddress *obj, Relation depRel = NULL); +void TrDoPurgeObjectDrop(TrObjDesc *desc); + +char *TrGenObjName(char *rbname, Oid classId, Oid objid) +{ + int rc = EOK; + + rc = snprintf_s(rbname, NAMEDATALEN, NAMEDATALEN - 1, "BIN$%X%X%X$%llX==$0", + u_sess->proc_cxt.MyDatabaseId, classId, objid, (uint64)GetXLogInsertRecPtr()); + securec_check_ss_c(rc, "\0", "\0"); + + return rbname; +} + +static TransactionId TrRbGetRcyfrozenxid64(HeapTuple rbtup, Relation rbRel = NULL) +{ + Datum datum; + bool isNull = false; + TransactionId rcyfrozenxid64; + bool relArgNull = rbRel == NULL; + + if (relArgNull) { + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + } + + datum = heap_getattr(rbtup, Anum_pg_recyclebin_rcyfrozenxid64, RelationGetDescr(rbRel), &isNull); + Assert(!isNull); + + rcyfrozenxid64 = DatumGetTransactionId(datum); + + if (relArgNull) { + heap_close(rbRel, AccessShareLock); + } + + return rcyfrozenxid64; +} + +void TrDescInit(Relation rel, TrObjDesc *desc, TrObjOperType operType, + TrObjType objType, bool canpurge, bool isBaseObj) +{ + errno_t rc = EOK; + + /* Notice: desc->id, desc->baseid will be assigned by invoker later. */ + desc->dbid = u_sess->proc_cxt.MyDatabaseId; + desc->relid = RelationGetRelid(rel); + + (void)TrGenObjName(desc->name, RelationRelationId, desc->relid); + + rc = strncpy_s(desc->originname, NAMEDATALEN, RelationGetRelationName(rel), + strlen(RelationGetRelationName(rel))); + securec_check(rc, "\0", "\0"); + + desc->operation = operType; + desc->type = objType; + desc->recyclecsn = t_thrd.xact_cxt.ShmemVariableCache->nextCommitSeqNo; + desc->recycletime = GetCurrentTimestamp(); + desc->createcsn = RelationGetCreatecsn(rel); + desc->changecsn = RelationGetChangecsn(rel); + desc->nspace = RelationGetNamespace(rel); + desc->owner = RelationGetOwner(rel); + desc->tablespace = RelationGetTablespace(rel); + desc->relfilenode = RelationGetRelFileNode(rel); + desc->frozenxid = RelationGetRelFrozenxid(rel); + desc->frozenxid64 = RelationGetRelFrozenxid64(rel); + desc->canrestore = objType == RB_OBJ_TABLE; + desc->canpurge = canpurge; +} + +void TrPartDescInit(Relation rel, Partition part, TrObjDesc *desc, TrObjOperType operType, + TrObjType objType, bool canpurge, bool isBaseObj) +{ + errno_t rc = EOK; + + /* Notice: desc->id, desc->baseid will be assigned by invoker later. */ + desc->dbid = u_sess->proc_cxt.MyDatabaseId; + desc->relid = part->pd_id; + + (void)TrGenObjName(desc->name, PartitionRelationId, desc->relid); + + rc = strncpy_s(desc->originname, NAMEDATALEN, RelationGetRelationName(rel), + strlen(RelationGetRelationName(rel))); + securec_check(rc, "\0", "\0"); + + int len = strlen(PartitionGetPartitionName(part)) + strlen(RelationGetRelationName(rel)) + 1; + rc = strcat_s(desc->originname, len, PartitionGetPartitionName(part)); + securec_check(rc, "\0", "\0"); + + desc->operation = operType; + desc->type = objType; + desc->recyclecsn = t_thrd.xact_cxt.ShmemVariableCache->nextCommitSeqNo; + desc->recycletime = GetCurrentTimestamp(); + desc->createcsn = RelationGetCreatecsn(rel); + desc->changecsn = RelationGetChangecsn(rel); + desc->nspace = RelationGetNamespace(rel); + desc->owner = RelationGetOwner(rel); + desc->tablespace = part->pd_part->reltablespace; + desc->relfilenode = part->pd_part->relfilenode; + desc->frozenxid = part->pd_part->relfrozenxid; + desc->frozenxid64 = PartGetRelFrozenxid64(part); + desc->canrestore = false; + desc->canpurge = canpurge; +} + +static void TrDescRead(TrObjDesc *desc, HeapTuple rbtup) +{ + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(rbtup); + + desc->id = HeapTupleGetOid(rbtup); + desc->baseid = rbForm->rcybaseid; + + desc->dbid = rbForm->rcydbid; + desc->relid = rbForm->rcyrelid; + (void)namestrcpy((Name)desc->name, NameStr(rbForm->rcyname)); + (void)namestrcpy((Name)desc->originname, NameStr(rbForm->rcyoriginname)); + desc->operation = (rbForm->rcyoperation == 'd') ? RB_OPER_DROP : RB_OPER_TRUNCATE; + desc->type = (TrObjType)rbForm->rcytype; + desc->recyclecsn = rbForm->rcyrecyclecsn; + desc->recycletime = rbForm->rcyrecycletime; + desc->createcsn = rbForm->rcycreatecsn; + desc->changecsn = rbForm->rcychangecsn; + desc->nspace = rbForm->rcynamespace; + desc->owner = rbForm->rcyowner; + desc->tablespace = rbForm->rcytablespace; + desc->relfilenode = rbForm->rcyrelfilenode; + desc->canrestore = rbForm->rcycanrestore; + desc->canpurge = rbForm->rcycanpurge; + desc->frozenxid = rbForm->rcyfrozenxid; + desc->frozenxid64 = TrRbGetRcyfrozenxid64(rbtup); +} + +Oid TrDescWrite(TrObjDesc *desc) +{ + Relation rel; + HeapTuple tup; + bool nulls[Natts_pg_recyclebin] = {0}; + Datum values[Natts_pg_recyclebin]; + NameData name; + NameData originname; + Oid rbid; + + values[Anum_pg_recyclebin_rcydbid - 1] = ObjectIdGetDatum(desc->dbid); + values[Anum_pg_recyclebin_rcybaseid - 1] = ObjectIdGetDatum(desc->baseid); + values[Anum_pg_recyclebin_rcyrelid - 1] = ObjectIdGetDatum(desc->relid); + (void)namestrcpy(&name, desc->name); + values[Anum_pg_recyclebin_rcyname - 1] = NameGetDatum(&name); + (void)namestrcpy(&originname, desc->originname); + values[Anum_pg_recyclebin_rcyoriginname - 1] = NameGetDatum(&originname); + values[Anum_pg_recyclebin_rcyoperation - 1] = (desc->operation == RB_OPER_DROP) ? 'd' : 't'; + values[Anum_pg_recyclebin_rcytype - 1] = Int32GetDatum(desc->type); + values[Anum_pg_recyclebin_rcyrecyclecsn - 1] = Int64GetDatum(desc->recyclecsn); + values[Anum_pg_recyclebin_rcyrecycletime - 1] = TimestampTzGetDatum(desc->recycletime); + values[Anum_pg_recyclebin_rcycreatecsn - 1] = Int64GetDatum(desc->createcsn); + values[Anum_pg_recyclebin_rcychangecsn - 1] = Int64GetDatum(desc->changecsn); + values[Anum_pg_recyclebin_rcynamespace - 1] = ObjectIdGetDatum(desc->nspace); + values[Anum_pg_recyclebin_rcyowner - 1] = ObjectIdGetDatum(desc->owner); + values[Anum_pg_recyclebin_rcytablespace - 1] = ObjectIdGetDatum(desc->tablespace); + values[Anum_pg_recyclebin_rcyrelfilenode - 1] = ObjectIdGetDatum(desc->relfilenode); + values[Anum_pg_recyclebin_rcycanrestore - 1] = BoolGetDatum(desc->canrestore); + values[Anum_pg_recyclebin_rcycanpurge - 1] = BoolGetDatum(desc->canpurge); + values[Anum_pg_recyclebin_rcyfrozenxid - 1] = ShortTransactionIdGetDatum(desc->frozenxid); + values[Anum_pg_recyclebin_rcyfrozenxid64 - 1] = TransactionIdGetDatum(desc->frozenxid64); + + rel = heap_open(RecyclebinRelationId, RowExclusiveLock); + + tup = heap_form_tuple(RelationGetDescr(rel), values, nulls); + + rbid = simple_heap_insert(rel, tup); + + CatalogUpdateIndexes(rel, tup); + + heap_freetuple_ext(tup); + + heap_close(rel, RowExclusiveLock); + + CommandCounterIncrement(); + + return rbid; +} + +static bool TrFetchOrinameImpl(Oid nspId, const char *oriname, TrObjType type, + TrObjDesc *desc, TrOperMode operMode) +{ + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[3]; + HeapTuple tup; + bool found = false; + + if (!OidIsValid(nspId)) { + return false; + } + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcynamespace, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(nspId)); + ScanKeyInit(&skey[1], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); + ScanKeyInit(&skey[2], Anum_pg_recyclebin_rcyoriginname, BTEqualStrategyNumber, + F_NAMEEQ, CStringGetDatum(oriname)); + + sd = systable_beginscan(rbRel, RecyclebinDbidNspOrinameIndexId, true, NULL, 3, skey); + while ((tup = systable_getnext(sd)) != NULL) { + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); + if ((rbForm->rcytype != type && rbForm->rcytype == RB_OBJ_TABLE) || + (rbForm->rcytype != type && rbForm->rcytype == RB_OBJ_INDEX) || + (operMode == RB_OPER_RESTORE_DROP && rbForm->rcyoperation != 'd') || + (operMode == RB_OPER_RESTORE_TRUNCATE && rbForm->rcyoperation != 't')) { + continue; + } + + found = true; + TrDescRead(desc, tup); + break; + } + + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return found; +} + +bool TrFetchName(const char *rcyname, TrObjType type, TrObjDesc *desc, TrOperMode operMode) +{ + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[1]; + HeapTuple tup; + bool found = false; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcyname, BTEqualStrategyNumber, + F_NAMEEQ, CStringGetDatum(rcyname)); + + sd = systable_beginscan(rbRel, RecyclebinNameIndexId, true, NULL, 1, skey); + if ((tup = systable_getnext(sd)) != NULL) { + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); + if ((rbForm->rcytype != type && rbForm->rcytype == RB_OBJ_TABLE) || + (rbForm->rcytype != type && rbForm->rcytype == RB_OBJ_INDEX)) { + ereport(ERROR, + (errmsg("The recycle object \"%s\" type mismatched.", rcyname))); + } + if ((operMode == RB_OPER_RESTORE_DROP && rbForm->rcyoperation != 'd') || + (operMode == RB_OPER_RESTORE_TRUNCATE && rbForm->rcyoperation != 't')) { + ereport(ERROR, + (errmsg("recycle object \"%s\" desired does not exist", rcyname))); + } + + found = true; + TrDescRead(desc, tup); + } + + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return found; +} + +static bool TrFetchOriname(const char *schemaname, const char *relname, TrObjType type, + TrObjDesc *desc, TrOperMode operMode) +{ + bool found = false; + Oid nspId; + + if (schemaname) { + nspId = get_namespace_oid(schemaname, true); + found = TrFetchOrinameImpl(nspId, relname, type, desc, operMode); + } else { + List *activeSearchPath = NIL; + ListCell *l = NULL; + + recomputeNamespacePath(); + activeSearchPath = list_copy(u_sess->catalog_cxt.activeSearchPath); + foreach (l, activeSearchPath) { + nspId = lfirst_oid(l); + if (TrFetchOrinameImpl(nspId, relname, type, desc, operMode)) { + found = true; + break; + } + } + list_free_ext(activeSearchPath); + if (!found) { + nspId = PG_TOAST_NAMESPACE; + found = TrFetchOrinameImpl(nspId, relname, type, desc, operMode); + } + } + + return found; +} + +void TrUpdateBaseid(const TrObjDesc *desc) +{ + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[1]; + HeapTuple tup; + HeapTuple newtup; + Datum values[Natts_pg_recyclebin] = { 0 }; + bool nulls[Natts_pg_recyclebin] = { false }; + bool replaces[Natts_pg_recyclebin] = { false }; + + rbRel = heap_open(RecyclebinRelationId, RowExclusiveLock); + + ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(desc->id)); + + sd = systable_beginscan(rbRel, RecyclebinIdIndexId, true, NULL, 1, skey); + if ((tup = systable_getnext(sd)) == NULL) { + ereport(ERROR, (errmsg("recycle object %u does not exist", desc->id))); + } + + replaces[Anum_pg_recyclebin_rcybaseid - 1] = true; + values[Anum_pg_recyclebin_rcybaseid - 1] = ObjectIdGetDatum(desc->baseid); + + newtup = heap_modify_tuple(tup, RelationGetDescr(rbRel), values, nulls, replaces); + + simple_heap_update(rbRel, &newtup->t_self, newtup); + + CatalogUpdateIndexes(rbRel, newtup); + + heap_freetuple_ext(newtup); + + systable_endscan(sd); + heap_close(rbRel, RowExclusiveLock); + + return; +} + +static void TrLockRelationImpl(Oid relid, TrObjType type) +{ + /* + * Lock failed may due to concurrently purge/timecapsule/DQL + * on recycle object, or access on normal relation. + */ + if (!ConditionalLockRelationOid(relid, AccessExclusiveLock)) { + ereport(ERROR, + (errcode(ERRCODE_RBIN_LOCK_NOT_AVAILABLE), + errmsg("could not obtain lock on relation \"%u\"", relid))); + } + + /* + * Now that we have the lock, probe to see if the relation + * really exists or not. + */ + AcceptInvalidationMessages(); + if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(relid)) && type != RB_OBJ_PARTITION) { + /* Clean already held locks if error return. */ + UnlockRelationOid(relid, AccessExclusiveLock); + ereport(ERROR, + (errcode(ERRCODE_RBIN_UNDEFINED_OBJECT), + errmsg("relation \"%u\" does not exist", relid))); + } else if (!SearchSysCacheExists1(PARTRELID, ObjectIdGetDatum(relid)) && type == RB_OBJ_PARTITION) { + /* Clean already held locks if error return. */ + UnlockRelationOid(relid, AccessExclusiveLock); + ereport(ERROR, + (errcode(ERRCODE_RBIN_UNDEFINED_OBJECT), + errmsg("partition \"%u\" does not exist", relid))); + } +} + +static void TrLockRelation(TrObjDesc *desc) +{ + Oid heapOid = InvalidOid; + + /* Lock heap relation for index first */ + if (desc->type == RB_OBJ_INDEX) { + heapOid = IndexGetRelation(desc->relid, true); + if (!OidIsValid(heapOid)) { + ereport(ERROR, + (errcode(ERRCODE_RBIN_UNDEFINED_OBJECT), + errmsg("relation \"%u\" does not exist", desc->relid))); + } + TrLockRelationImpl(heapOid, desc->type); + } + + /* Use TRY-CATCH block to clean locks already held if error. */ + PG_TRY(); + { + /* Lock relation self */ + TrLockRelationImpl(desc->relid, desc->type); + } + PG_CATCH(); + { + if (desc->type == RB_OBJ_INDEX) { + UnlockRelationOid(heapOid, AccessExclusiveLock); + } + PG_RE_THROW(); + } + PG_END_TRY(); +} + +static void TrUnlockTrItem(TrObjDesc *desc) +{ + UnlockDatabaseObject(RecyclebinRelationId, desc->id, 0, + AccessExclusiveLock); +} + +static void TrLockTrItem(TrObjDesc *desc) +{ + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[1]; + HeapTuple tup; + + /* 1. Try to lock rb item in AccessExclusiveLock */ + if (!ConditionalLockDatabaseObject(RecyclebinRelationId, desc->id, 0, AccessExclusiveLock)) { + ereport(ERROR, + (errcode(ERRCODE_RBIN_LOCK_NOT_AVAILABLE), + errmsg("could not obtain lock on recycle object '%s'", desc->name))); + } + + /* + * 2. Now that we have the lock, probe to see if the rb item really + * exists or not. + */ + AcceptInvalidationMessages(); + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(desc->id)); + + sd = systable_beginscan(rbRel, RecyclebinIdIndexId, true, NULL, 1, skey); + if ((tup = systable_getnext(sd)) == NULL) { + UnlockDatabaseObject(RecyclebinRelationId, desc->id, 0, AccessExclusiveLock); + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + ereport(ERROR, + (errcode(ERRCODE_RBIN_UNDEFINED_OBJECT), + errmsg("recycle object \"%s\" does not exist", desc->name))); + } + + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return; +} + +static void TrOperMatch(const TrObjDesc *desc, TrOperMode operMode) +{ + switch (operMode) { + case RB_OPER_PURGE: + if (!desc->canpurge && desc->type != RB_OBJ_PARTITION) { + ereport(ERROR, + (errcode(ERRCODE_INVALID_OPERATION), + errmsg("recycle object \"%s\" cannot be purged", desc->name))); + } + break; + + case RB_OPER_RESTORE_DROP: + if ((!desc->canrestore && desc->type != RB_OBJ_PARTITION) || desc->operation != RB_OPER_DROP) { + ereport(ERROR, + (errcode(ERRCODE_INVALID_OPERATION), + errmsg("recycle object \"%s\" cannot be restored", desc->name))); + } + break; + + case RB_OPER_RESTORE_TRUNCATE: + if ((!desc->canrestore && desc->type != RB_OBJ_PARTITION) || desc->operation != RB_OPER_TRUNCATE) { + ereport(ERROR, + (errcode(ERRCODE_INVALID_OPERATION), + errmsg("recycle object \"%s\" cannot be restored", desc->name))); + } + break; + + default: + ereport(ERROR, + (errcode(ERRCODE_UNRECOGNIZED_NODE_TYPE), + errmsg("unrecognized recyclebin operation: %u", operMode))); + break; + } +} + +/* + * Fetch object from recycle bin for rb operations - purge, restore : + * Prefer to fetch as original name, then recycle name. + */ +void TrOperFetch(const RangeVar *purobj, TrObjType objtype, TrObjDesc *desc, TrOperMode operMode) +{ + bool found = false; + + AcceptInvalidationMessages(); + + /* Prefer to fetch as original name */ + found = TrFetchOriname(purobj->schemaname, purobj->relname, objtype, desc, operMode); + /* if not found, then fetch as recycle name */ + if (!found) { + found = TrFetchName(purobj->relname, objtype, desc, operMode); + } + + /* not found, throw error */ + if (!found) { + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("recycle object \"%s\" desired does not exist", purobj->relname))); + } + + TrOperMatch(desc, operMode); + + return; +} + +static void TrPermRestore(TrObjDesc *desc, TrOperMode operMode) +{ + AclResult aclCreateResult; + + /* Check namespace permissions. */ + aclCreateResult = pg_namespace_aclcheck(desc->nspace, desc->authid, ACL_CREATE); + if (aclCreateResult != ACLCHECK_OK) { + aclcheck_error(aclCreateResult, ACL_KIND_NAMESPACE, get_namespace_name(desc->nspace)); + } + + AclResult aclUsageResult = pg_namespace_aclcheck(desc->nspace, desc->authid, ACL_USAGE); + if (aclUsageResult != ACLCHECK_OK) { + aclcheck_error(aclUsageResult, ACL_KIND_NAMESPACE, get_namespace_name(desc->nspace)); + } + + /* Allow restore to either table owner or schema owner */ + if (!pg_class_ownercheck(desc->relid, desc->authid) && !pg_namespace_ownercheck(desc->nspace, desc->authid)) { + aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_CLASS, desc->name); + return; + } + + if (operMode == RB_OPER_RESTORE_TRUNCATE) { + AclResult aclTruncateResult = pg_class_aclcheck(desc->relid, desc->authid, ACL_TRUNCATE); + if (aclTruncateResult != ACLCHECK_OK) { + aclcheck_error(aclTruncateResult, ACL_KIND_CLASS, desc->name); + } + } +} + +static void TrPermPurge(TrObjDesc *desc, TrOperMode operMode) +{ + AclResult result; + + result = pg_namespace_aclcheck(desc->nspace, desc->authid, ACL_USAGE); + if (result != ACLCHECK_OK) { + aclcheck_error(result, ACL_KIND_NAMESPACE, get_namespace_name(desc->nspace)); + } + if (!pg_class_ownercheck(desc->relid, desc->authid) && !pg_namespace_ownercheck(desc->nspace, desc->authid)) { + aclcheck_error(ACLCHECK_NOT_OWNER, ACL_KIND_CLASS, desc->name); + } +} + +/* + * Check permission for rb operations - purge, restore + */ +static void TrPerm(TrObjDesc *desc, TrOperMode operMode) +{ + switch (operMode) { + case RB_OPER_RESTORE_DROP: + case RB_OPER_RESTORE_TRUNCATE: + TrPermRestore(desc, operMode); + break; + case RB_OPER_PURGE: + TrPermPurge(desc, operMode); + break; + default: + /* Never reached here. */ + Assert(0); + break; + } +} + +/* + * Prepare for rb operations - purge, restore : + * check permission, lock objects + */ +void TrOperPrep(TrObjDesc *desc, TrOperMode operMode) +{ + bool needLockRelation = false; + + /* + * 1. Check permission. + */ + TrPerm(desc, operMode); + + /* + * 2. Acquire lock on rb item, avoid concurrently purge, restore. + */ + TrLockTrItem(desc); + + /* + * 3. Acquire lock on relation, avoid concurrently DQL. + * Notice: ignore this step when we purge truncated relation + * as base relation may not exists. + */ + needLockRelation = !(operMode == RB_OPER_PURGE && desc->operation == RB_OPER_TRUNCATE); + if (needLockRelation) { + /* Use TRY-CATCH block to clean locks already held if error. */ + PG_TRY(); + { + TrLockRelation(desc); + } + PG_CATCH(); + { + TrUnlockTrItem(desc); + PG_RE_THROW(); + } + PG_END_TRY(); + } +} + +bool NeedTrComm(Oid relid) +{ + Relation rel; + Form_pg_class classForm; + + if (/* + *Disable Recyclebin-based-Drop/Truncate when + */ + /* recyclebin disabled, or */ + !u_sess->attr.attr_storage.enable_recyclebin || + /* target db is template1, or */ + u_sess->proc_cxt.MyDatabaseId == TemplateDbOid || + /* in maintenance mode, or */ + u_sess->attr.attr_common.xc_maintenance_mode || + /* in in-place upgrade mode, or */ + t_thrd.proc->workingVersionNum < 92350 || + /* in non-singlenode mode, or */ + (g_instance.role != VSINGLENODE) || + /* in bootstrap mode. */ + IsInitdb) { + return false; + } + + rel = relation_open(relid, NoLock); + classForm = rel->rd_rel; + if (/* + * Disable Recyclebin-based-Drop/Truncate if + */ + /* table is non ordinary table, or */ + classForm->relkind != RELKIND_RELATION || + /* is non heap table, or */ + rel->rd_tam_type == TAM_HEAP || + /* is non regular table, or */ + classForm->relpersistence != RELPERSISTENCE_PERMANENT || + /* is shared table across databases, or */ + classForm->relisshared || + /* has derived classes, or */ + classForm->relhassubclass || + /* has any PARTIAL CLUSTER KEY, or */ + classForm->relhasclusterkey || + /* is cstore table, or */ + (rel->rd_options && StdRelOptIsColStore(rel->rd_options)) || RelationIsColStore(rel) || + /* is hbkt table, or */ + (RELATION_HAS_BUCKET(rel) || RELATION_OWN_BUCKET(rel)) || + /* is dfs table, or */ + RelationIsPAXFormat(rel) || + /* is resizing, or */ + RelationInClusterResizing(rel) || + /* is in system namespace. */ + (IsSystemNamespace(classForm->relnamespace) || IsToastNamespace(classForm->relnamespace) || + IsCStoreNamespace(classForm->relnamespace))) { + relation_close(rel, NoLock); + return false; + } + + relation_close(rel, NoLock); + + return true; +} + +TrObjType TrGetObjType(Oid nspId, char relKind) +{ + TrObjType type = RB_OBJ_TABLE; + + switch (relKind) { + case RELKIND_INDEX: + type = IsToastNamespace(nspId) ? RB_OBJ_TOAST_INDEX : RB_OBJ_INDEX; + break; + case RELKIND_RELATION: + type = RB_OBJ_TABLE; + break; + case RELKIND_SEQUENCE: + case RELKIND_LARGE_SEQUENCE: + type = RB_OBJ_SEQUENCE; + break; + case RELKIND_TOASTVALUE: + type = RB_OBJ_TOAST; + break; + case PARTTYPE_PARTITIONED_RELATION: + type = RB_OBJ_PARTITION; + break; + case RELKIND_GLOBAL_INDEX: + type = RB_OBJ_GLOBAL_INDEX; + break; + case RELKIND_MATVIEW: + type = RB_OBJ_MATVIEW; + break; + default: + /* Never reached here. */ + Assert(0); + break; + } + + return type; +} + +static bool TrObjAddrExists(Oid classid, Oid objid, ObjectAddresses *objSet) +{ + int i; + + for (i = 0; i < objSet->numrefs; i++) { + if (TrObjIsEqualEx(classid, objid, &objSet->refs[i])) { + return true; + } + } + + return false; +} + +/* + * output: refobjs + */ +void TrFindAllRefObjs(Relation depRel, const ObjectAddress *subobj, + ObjectAddresses *refobjs, bool ignoreObjSubId) +{ + SysScanDesc sd; + HeapTuple tuple; + ScanKeyData key[3]; + int nkeys; + + ScanKeyInit(&key[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(subobj->classId)); + ScanKeyInit(&key[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(subobj->objectId)); + nkeys = 2; + if (!ignoreObjSubId && subobj->objectSubId != 0) { + ScanKeyInit(&key[2], Anum_pg_depend_objsubid, BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum(subobj->objectSubId)); + nkeys = 3; + } + + sd = systable_beginscan(depRel, DependDependerIndexId, true, NULL, nkeys, key); + while (HeapTupleIsValid(tuple = systable_getnext(sd))) { + Form_pg_depend depForm = (Form_pg_depend)GETSTRUCT(tuple); + /* Cascaded clean rb object in `DROP SCHEMA` command. */ + if (depForm->refclassid == NamespaceRelationId) { + continue; + } + + /* We keep `objSet` unique when `ignoreObjSubId = true` to avoid circle recursive. */ + if (!ignoreObjSubId || !TrObjAddrExists(depForm->refclassid, depForm->refobjid, refobjs)) { + add_object_address_ext(depForm->refclassid, depForm->refobjid, + depForm->refobjsubid, depForm->deptype, refobjs); + } + } + + systable_endscan(sd); + return; +} + +static void TrFindAllInternalObjs(Relation depRel, const ObjectAddress *refobj, + ObjectAddresses *objSet, bool ignoreObjSubId = false) +{ + SysScanDesc sd; + HeapTuple tuple; + ScanKeyData key[3]; + int nkeys; + + ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(refobj->classId)); + ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(refobj->objectId)); + nkeys = 2; + if (!ignoreObjSubId && refobj->objectSubId != 0) { + ScanKeyInit(&key[2], Anum_pg_depend_refobjsubid, BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum(refobj->objectSubId)); + nkeys = 3; + } + + sd = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, nkeys, key); + while (HeapTupleIsValid(tuple = systable_getnext(sd))) { + Form_pg_depend depForm = (Form_pg_depend)GETSTRUCT(tuple); + if (depForm->deptype != 'i') { + continue; + } + + /* We keep `objSet` unique when `ignoreObjSubId = true` to avoid circle recursive. */ + if (!ignoreObjSubId || !TrObjAddrExists(depForm->classid, depForm->objid, objSet)) { + add_object_address_ext(depForm->classid, depForm->objid, + depForm->objsubid, depForm->deptype, objSet); + } + } + + systable_endscan(sd); + return; +} + +static void TrDoPurgeObject(TrObjDesc *desc) +{ + if (desc->operation == RB_OPER_DROP) { + TrDoPurgeObjectDrop(desc); + } else { + TrDoPurgeObjectTruncate(desc); + } +} + +void TrPurgeObject(RangeVar *purobj, TrObjType type) +{ + TrObjDesc desc; + + TrOperFetch(purobj, type, &desc, RB_OPER_PURGE); + + desc.authid = GetUserId(); + TrOperPrep(&desc, RB_OPER_PURGE); + + TrDoPurgeObject(&desc); + + return; +} + +const int PURGE_BATCH = 64; +const int PURGE_SINGL = 64; +typedef void (*TrFetchBeginHook)(SysScanDesc *sd, Oid objId); +typedef bool (*TrFetchMatchHook)(Relation rbRel, HeapTuple rbTup, Oid objId); + +static void TrFetchBegin(TrFetchBeginHook fetchHook, SysScanDesc *sd, Oid objId) +{ + fetchHook(sd, objId); +} + +// @return: true for eof +static bool TrFetchExec(TrFetchMatchHook matchHook, Oid objId, SysScanDesc sd, TrObjDesc *desc) +{ + HeapTuple tup; + while ((tup = systable_getnext(sd)) != NULL) { + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); + if ((rbForm->rcytype == RB_OBJ_TABLE) && matchHook(sd->heap_rel, tup, objId)) { + Assert (rbForm->rcycanpurge); + TrDescRead(desc, tup); + return false; + } else if ((rbForm->rcytype == RB_OBJ_PARTITION) && matchHook(sd->heap_rel, tup, objId)) { + Assert (!rbForm->rcycanpurge); + TrDescRead(desc, tup); + return false; + } + } + return true; +} + +static void TrFetchEnd(SysScanDesc sd) +{ + Relation rbRel = sd->heap_rel; + + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); +} + +static bool TrPurgeBatch(TrFetchBeginHook beginHook, TrFetchMatchHook matchHook, + Oid objId, Oid roleid, uint32 maxBatch, PurgeMsgRes *localRes) +{ + SysScanDesc sd = NULL; + TrObjDesc desc; + uint32 count = 0; + bool eof = false; + + RbMsgResetRes(localRes); + + StartTransactionCommand(); + + TrFetchBegin(beginHook, &sd, objId); + while (!(eof = TrFetchExec(matchHook, objId, sd, &desc))) { + CHECK_FOR_INTERRUPTS(); + + PG_TRY(); + { + desc.authid = roleid; + TrOperPrep(&desc, RB_OPER_PURGE); + + TrDoPurgeObject(&desc); + localRes->purgedNum++; + } + PG_CATCH(); + { + int errcode = geterrcode(); + if (errcode == ERRCODE_RBIN_LOCK_NOT_AVAILABLE) { + errno_t rc; + rc = strncpy_s(localRes->errMsg, RB_MAX_ERRMSG_SIZE, Geterrmsg(), RB_MAX_ERRMSG_SIZE - 1); + securec_check(rc, "\0", "\0"); + localRes->skippedNum++; + } else if (errcode == ERRCODE_RBIN_UNDEFINED_OBJECT) { + localRes->undefinedNum++; + } else { + PG_RE_THROW(); + } + } + PG_END_TRY(); + + if (++count >= maxBatch) { + break; + } + } + + TrFetchEnd(sd); + + CommitTransactionCommand(); + + return eof; +} + +static void TrFetchBeginSpace(SysScanDesc *sd, Oid spcId) +{ + ScanKeyData skey[2]; + Relation rbRel; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcytablespace, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(spcId)); + ScanKeyInit(&skey[1], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); + + *sd = systable_beginscan(rbRel, RecyclebinDbidSpcidRcycsnIndexId, true, NULL, 2, skey); +} + +static bool TrFetchMatchSpace(Relation rbRel, HeapTuple rbTup, Oid objId) +{ + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(rbTup); + return rbForm->rcytablespace == objId; +} + +void TrPurgeTablespace(int64 id) +{ + PurgeMsgReq *req = &RbMsg(id)->req; + PurgeMsgRes localRes; + bool eof = false; + + do { + eof = TrPurgeBatch(TrFetchBeginSpace, TrFetchMatchSpace, req->objId, req->authId, PURGE_BATCH, &localRes); + RbMsgSetStatistics(id, &localRes); + } while (!eof && localRes.skippedNum == 0); +} + +void TrPurgeTablespaceDML(int64 id) +{ + PurgeMsgReq *req = &RbMsg(id)->req; + PurgeMsgRes localRes; + bool eof = false; + + do { + eof = TrPurgeBatch(TrFetchBeginSpace, TrFetchMatchSpace, req->objId, req->authId, PURGE_SINGL, &localRes); + RbMsgSetStatistics(id, &localRes); + } while (!eof && localRes.purgedNum == 0); +} + +static void TrFetchBeginRecyclebin(SysScanDesc *sd, Oid objId) +{ + ScanKeyData skey[2]; + Relation rbRel; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); + + *sd = systable_beginscan(rbRel, RecyclebinDbidRelidIndexId, true, NULL, 1, skey); +} + +static bool TrFetchMatchRecyclebin(Relation rbRel, HeapTuple rbTup, Oid objId) +{ + return true; +} + +void TrPurgeRecyclebin(int64 id) +{ + PurgeMsgReq *req = &RbMsg(id)->req; + PurgeMsgRes localRes; + bool eof = false; + + do { + eof = TrPurgeBatch(TrFetchBeginRecyclebin, TrFetchMatchRecyclebin, + InvalidOid, req->authId, PURGE_BATCH, &localRes); + RbMsgSetStatistics(id, &localRes); + } while (!eof && localRes.skippedNum == 0); +} + +static void TrFetchBeginSchema(SysScanDesc *sd, Oid objId) +{ + ScanKeyData skey[2]; + Relation rbRel; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcynamespace, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(objId)); + ScanKeyInit(&skey[1], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); + + *sd = systable_beginscan(rbRel, RecyclebinDbidNspOrinameIndexId, true, NULL, 2, skey); +} + +static bool TrFetchMatchSchema(Relation rbRel, HeapTuple rbTup, Oid objId) +{ + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(rbTup); + return rbForm->rcynamespace == objId; +} + +void TrPurgeSchema(int64 id) +{ + PurgeMsgReq *req = &RbMsg(id)->req; + PurgeMsgRes localRes; + bool eof = false; + + do { + eof = TrPurgeBatch(TrFetchBeginSchema, TrFetchMatchSchema, req->objId, req->authId, PURGE_BATCH, &localRes); + RbMsgSetStatistics(id, &localRes); + } while (!eof && localRes.skippedNum == 0); +} + +static void TrFetchBeginUser(SysScanDesc *sd, Oid objId) +{ + ScanKeyData skey[2]; + Relation rbRel; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); + + *sd = systable_beginscan(rbRel, RecyclebinDbidRelidIndexId, true, NULL, 1, skey); +} + +static bool TrFetchMatchUser(Relation rbRel, HeapTuple rbTup, Oid objId) +{ + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(rbTup); + return rbForm->rcyowner == objId; +} + +void TrPurgeUser(int64 id) +{ + PurgeMsgReq *req = &RbMsg(id)->req; + PurgeMsgRes localRes; + bool eof = false; + + do { + eof = TrPurgeBatch(TrFetchBeginUser, TrFetchMatchUser, req->objId, req->authId, PURGE_BATCH, &localRes); + RbMsgSetStatistics(id, &localRes); + } while (!eof && localRes.skippedNum == 0); +} + +static void TrFetchBeginAuto(SysScanDesc *sd, Oid objId) +{ + ScanKeyData skey[2]; + Relation rbRel; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); + + *sd = systable_beginscan(rbRel, RecyclebinDbidRelidIndexId, true, NULL, 1, skey); +} + +static bool TrFetchMatchAuto(Relation rbRel, HeapTuple rbTup, Oid objId) +{ + bool isNull = false; + Datum datumRcyTime = heap_getattr(rbTup, Anum_pg_recyclebin_rcyrecycletime, + RelationGetDescr(rbRel), &isNull); + + long secs; + int msecs; + TimestampDifference(isNull ? 0 : DatumGetTimestampTz(datumRcyTime), + GetCurrentTimestamp(), &secs, &msecs); + + return secs > u_sess->attr.attr_storage.recyclebin_retention_time || secs < 0; +} + +void TrPurgeAuto(int64 id) +{ + PurgeMsgReq *req = &RbMsg(id)->req; + PurgeMsgRes localRes; + bool eof = false; + do { + eof = TrPurgeBatch(TrFetchBeginAuto, TrFetchMatchAuto, InvalidOid, req->authId, PURGE_BATCH, &localRes); + RbMsgSetStatistics(id, &localRes); + } while (!eof); +} + +void TrSwapRelfilenode(Relation rbRel, HeapTuple rbTup, bool isPart) +{ + Relation relRel; + HeapTuple relTup; + HeapTuple newTup; + TrObjDesc desc; + int maxNattr = 0; + Datum *values = NULL; + bool *nulls = NULL; + bool *replaces = NULL; + NameData name; + errno_t rc = EOK; + bool isNull = false; + int relfilenoIndex = 0; + int frozenxidIndex = 0; + int frozenxid64Index = 0; + bool isPartition = false; + + TrDescRead(&desc, rbTup); + + if (desc.type == RB_OBJ_PARTITION || (desc.type == RB_OBJ_INDEX && isPart)) { + isPartition = true; + } + if (isPartition) { + maxNattr = Max(Natts_pg_partition, Natts_pg_recyclebin); + relRel = heap_open(PartitionRelationId, RowExclusiveLock); + relTup = SearchSysCacheCopy1(PARTRELID, ObjectIdGetDatum(desc.relid)); + relfilenoIndex = Anum_pg_partition_relfilenode; + frozenxidIndex = Anum_pg_partition_relfrozenxid; + frozenxid64Index = Anum_pg_partition_relfrozenxid64; + } else { + maxNattr = Max(Natts_pg_class, Natts_pg_recyclebin); + relRel = heap_open(RelationRelationId, RowExclusiveLock); + relTup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(desc.relid)); + relfilenoIndex = Anum_pg_class_relfilenode; + frozenxidIndex = Anum_pg_class_relfrozenxid; + frozenxid64Index = Anum_pg_class_relfrozenxid64; + } + + /* 1. Update pg_class or pg_partition */ + if (!HeapTupleIsValid(relTup)) { + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("cache lookup failed for relation %u", desc.relid))); + } + + values = (Datum *)palloc0(sizeof(Datum) * maxNattr); + nulls = (bool *)palloc0(sizeof(bool) * maxNattr); + replaces = (bool *)palloc0(sizeof(bool) * maxNattr); + + replaces[relfilenoIndex - 1] = true; + values[relfilenoIndex - 1] = ObjectIdGetDatum(desc.relfilenode); + + replaces[frozenxidIndex - 1] = true; + values[frozenxidIndex - 1] = ShortTransactionIdGetDatum(desc.frozenxid); + + replaces[frozenxid64Index - 1] = true; + values[frozenxid64Index - 1] = TransactionIdGetDatum(desc.frozenxid64); + + newTup = heap_modify_tuple(relTup, RelationGetDescr(relRel), values, nulls, replaces); + + simple_heap_update(relRel, &newTup->t_self, newTup); + + CatalogUpdateIndexes(relRel, newTup); + + heap_freetuple_ext(newTup); + + /* 2. Update pg_recyclebin */ + rc = memset_s(values, sizeof(Datum) * maxNattr, 0, sizeof(Datum) * maxNattr); + securec_check(rc, "\0", "\0"); + rc = memset_s(nulls, sizeof(bool) * maxNattr, false, sizeof(bool) * maxNattr); + securec_check(rc, "\0", "\0"); + rc = memset_s(replaces, sizeof(bool) * maxNattr, false, sizeof(bool) * maxNattr); + securec_check(rc, "\0", "\0"); + + (void)TrGenObjName(NameStr(name), RelationRelationId, desc.relid); + replaces[Anum_pg_recyclebin_rcyname - 1] = true; + values[Anum_pg_recyclebin_rcyname - 1] = NameGetDatum(&name); + + replaces[Anum_pg_recyclebin_rcyoriginname - 1] = true; + if (isPartition) { + values[Anum_pg_recyclebin_rcyoriginname - 1] = NameGetDatum(&desc.originname); + } else { + values[Anum_pg_recyclebin_rcyoriginname - 1] = NameGetDatum(&((Form_pg_class)GETSTRUCT(relTup))->relname); + } + + replaces[Anum_pg_recyclebin_rcyrecyclecsn - 1] = true; + values[Anum_pg_recyclebin_rcyrecyclecsn - 1] = Int64GetDatum(t_thrd.xact_cxt.ShmemVariableCache->nextCommitSeqNo); + + replaces[Anum_pg_recyclebin_rcyrecycletime - 1] = true; + values[Anum_pg_recyclebin_rcyrecycletime - 1] = TimestampTzGetDatum(GetCurrentTimestamp()); + + replaces[Anum_pg_recyclebin_rcyrelfilenode - 1] = true; + if (isPartition) { + values[Anum_pg_recyclebin_rcyrelfilenode - 1] = + ObjectIdGetDatum(((Form_pg_partition)GETSTRUCT(relTup))->relfilenode); + } else { + values[Anum_pg_recyclebin_rcyrelfilenode - 1] = + ObjectIdGetDatum(((Form_pg_class)GETSTRUCT(relTup))->relfilenode); + } + + replaces[Anum_pg_recyclebin_rcyfrozenxid - 1] = true; + if (isPartition) { + values[Anum_pg_recyclebin_rcyfrozenxid - 1] = + ShortTransactionIdGetDatum(((Form_pg_partition)GETSTRUCT(relTup))->relfrozenxid); + } else { + values[Anum_pg_recyclebin_rcyfrozenxid - 1] = + ShortTransactionIdGetDatum(((Form_pg_class)GETSTRUCT(relTup))->relfrozenxid); + } + + replaces[Anum_pg_recyclebin_rcyfrozenxid64 - 1] = true; + Datum xid64datum = heap_getattr(relTup, frozenxid64Index, RelationGetDescr(relRel), &isNull); + values[Anum_pg_recyclebin_rcyfrozenxid64 - 1] = DatumGetTransactionId(xid64datum); + + newTup = heap_modify_tuple(rbTup, RelationGetDescr(rbRel), values, nulls, replaces); + + simple_heap_update(rbRel, &newTup->t_self, newTup); + + CatalogUpdateIndexes(rbRel, newTup); + + heap_freetuple_ext(newTup); + + pfree(values); + pfree(nulls); + pfree(replaces); + + heap_freetuple_ext(relTup); + heap_close(relRel, RowExclusiveLock); + return; +} + +void TrBaseRelMatched(TrObjDesc *baseDesc) +{ + ObjectAddress obj = {RelationRelationId, baseDesc->relid}; + if (TrIsRefRbObject(&obj)) { + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("relation \"%s\" does not exist", baseDesc->originname))); + } + + Relation rel = RelationIdGetRelation(baseDesc->relid); + Assert(RelationIsValid(rel)); + if (RelationGetCreatecsn(rel) != (CommitSeqNo)baseDesc->createcsn) { + ereport(ERROR, + (errmsg("The recycle object \"%s\" and relation \"%s\" mismatched.", + baseDesc->name, RelationGetRelationName(rel)))); + } + + if (RelationGetChangecsn(rel) > (CommitSeqNo)baseDesc->changecsn) { + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("The table definition of \"%s\" has been changed.", + RelationGetRelationName(rel)))); + } + + RelationClose(rel); +} + +void TrAdjustFrozenXid64(Oid dbid, TransactionId *frozenXID) +{ + Relation rbRel; + SysScanDesc sd; + HeapTuple rbtup; + + if (!TcapFeatureAvail()) { + return; + } + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + sd = systable_beginscan(rbRel, InvalidOid, false, NULL, 0, NULL); + while ((rbtup = systable_getnext(sd)) != NULL) { + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(rbtup); + TransactionId rcyfrozenxid64; + + if (rbForm->rcydbid != dbid || (rbForm->rcytype != RB_OBJ_TABLE && rbForm->rcytype != RB_OBJ_TOAST)) { + continue; + } + + rcyfrozenxid64 = TrRbGetRcyfrozenxid64(rbtup, rbRel); + Assert(TransactionIdIsNormal(rcyfrozenxid64)); + + if (TransactionIdPrecedes(rcyfrozenxid64, *frozenXID)) { + *frozenXID = rcyfrozenxid64; + } + } + + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return; +} + +bool TrRbIsEmptyDb(Oid dbid) +{ + Relation rbRel; + SysScanDesc sd; + HeapTuple tup; + ScanKeyData skey[1]; + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(dbid)); + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + sd = systable_beginscan(rbRel, RecyclebinDbidRelidIndexId, true, NULL, 1, skey); + tup = systable_getnext(sd); + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return tup == NULL; +} + +bool TrRbIsEmptySpc(Oid spcId) +{ + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[1]; + HeapTuple tup; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcytablespace, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(spcId)); + + sd = systable_beginscan(rbRel, RecyclebinDbidSpcidRcycsnIndexId, true, NULL, 1, skey); + tup = systable_getnext(sd); + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return tup == NULL; +} + +bool TrRbIsEmptySchema(Oid nspId) +{ + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[2]; + HeapTuple tup; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcynamespace, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(nspId)); + ScanKeyInit(&skey[1], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); + + sd = systable_beginscan(rbRel, RecyclebinDbidNspOrinameIndexId, true, NULL, 2, skey); + tup = systable_getnext(sd); + + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return tup == NULL; +} + +bool TrRbIsEmptyUser(Oid roleId) +{ + Relation rbRel; + SysScanDesc sd; + HeapTuple tup; + bool found = false; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + sd = systable_beginscan(rbRel, InvalidOid, false, NULL, 0, NULL); + + while ((tup = systable_getnext(sd)) != NULL) { + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); + if ((TrObjType)rbForm->rcytype != RB_OBJ_TABLE || rbForm->rcyowner != roleId) { + continue; + } + + found = true; + break; + } + + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return !found; +} + +static bool TrOidExists(const List *lOid, Oid oid) +{ + ListCell *cell = NULL; + if (lOid == NULL) { + return false; + } + + foreach (cell, lOid) { + if (oid == (*(Oid *)lfirst(cell))) { + return true; + } + } + return false; +} + +List *TrGetDbListRcy(void) +{ + Relation rbRel; + SysScanDesc sd; + HeapTuple tup; + + List *lName = NIL; + List *lOid = NIL; + char *dbname = NULL; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + sd = systable_beginscan(rbRel, InvalidOid, false, NULL, 0, NULL); + + while ((tup = systable_getnext(sd)) != NULL) { + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); + if (TrOidExists(lOid, rbForm->rcydbid)) { + continue; + } + Oid *oid = (Oid *)palloc0(sizeof(Oid)); + *oid = rbForm->rcydbid; + lOid = lappend(lOid, oid); + + dbname = get_database_name(rbForm->rcydbid); + if (dbname == NULL) { + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_DATABASE), + errmsg("database \"%u\" does not exist", rbForm->rcydbid))); + } + lName = lappend(lName, dbname); + } + + list_free_deep(lOid); + + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return lName; +} + +List *TrGetDbListSpc(Oid spcId) +{ + List *lName = NIL; + List *lOid = NIL; + char *dbname = NULL; + + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[1]; + HeapTuple tup; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcytablespace, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(spcId)); + + sd = systable_beginscan(rbRel, RecyclebinDbidSpcidRcycsnIndexId, true, NULL, 1, skey); + + while ((tup = systable_getnext(sd)) != NULL) { + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); + if (TrOidExists(lOid, rbForm->rcydbid)) { + continue; + } + Oid *oid = (Oid *)palloc0(sizeof(Oid)); + *oid = rbForm->rcydbid; + lOid = lappend(lOid, oid); + + dbname = get_database_name(rbForm->rcydbid); + if (dbname == NULL) { + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_DATABASE), + errmsg("database \"%u\" does not exist", rbForm->rcydbid))); + } + lName = lappend(lName, dbname); + } + + list_free_deep(lOid); + + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return lName; +} + +List *TrGetDbListSchema(Oid nspId) +{ + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[1]; + HeapTuple tup; + + List *lName = NIL; + List *lOid = NIL; + char *dbname = NULL; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcynamespace, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(nspId)); + + sd = systable_beginscan(rbRel, RecyclebinDbidNspOrinameIndexId, true, NULL, 1, skey); + + while ((tup = systable_getnext(sd)) != NULL) { + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); + if (TrOidExists(lOid, rbForm->rcydbid)) { + continue; + } + Oid *oid = (Oid *)palloc0(sizeof(Oid)); + *oid = rbForm->rcydbid; + lOid = lappend(lOid, oid); + + dbname = get_database_name(rbForm->rcydbid); + if (dbname == NULL) { + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_DATABASE), + errmsg("database \"%u\" does not exist", rbForm->rcydbid))); + } + lName = lappend(lName, dbname); + } + + list_free_deep(lOid); + + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return lName; +} + +List *TrGetDbListUser(Oid roleId) +{ + Relation rbRel; + SysScanDesc sd; + HeapTuple tup; + + List *lName = NIL; + List *lOid = NIL; + char *dbname = NULL; + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + + sd = systable_beginscan(rbRel, InvalidOid, false, NULL, 0, NULL); + + while ((tup = systable_getnext(sd)) != NULL) { + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); + if ((TrObjType)rbForm->rcytype != RB_OBJ_TABLE || rbForm->rcyowner != roleId) { + continue; + } + if (TrOidExists(lOid, rbForm->rcydbid)) { + continue; + } + Oid *oid = (Oid *)palloc0(sizeof(Oid)); + *oid = rbForm->rcydbid; + lOid = lappend(lOid, oid); + + dbname = get_database_name(rbForm->rcydbid); + if (dbname == NULL) { + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_DATABASE), + errmsg("database \"%u\" does not exist", rbForm->rcydbid))); + } + + lName = lappend(lName, dbname); + } + + list_free_deep(lOid); + + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return lName; +} + +/* + * TrGetDatabaseList + * Return a list of all databases found in pg_database. + */ +List *TrGetDbListAuto(void) +{ + List* dblist = NIL; + Relation rel; + SysScanDesc sd; + HeapTuple tup; + + rel = heap_open(DatabaseRelationId, AccessShareLock); + sd = systable_beginscan(rel, InvalidOid, false, NULL, 0, NULL); + + while ((tup = systable_getnext(sd)) != NULL) { + Form_pg_database pgdatabase = (Form_pg_database)GETSTRUCT(tup); + if (strcmp(NameStr(pgdatabase->datname), "template0") == 0 || + strcmp(NameStr(pgdatabase->datname), "template1") == 0) { + continue; + } + dblist = lappend(dblist, pstrdup(NameStr(pgdatabase->datname))); + } + + systable_endscan(sd); + heap_close(rel, AccessShareLock); + + return dblist; +} + +static bool TrObjInRecyclebin(const ObjectAddress *obj) +{ + Relation rbRel; + SysScanDesc sd; + HeapTuple tup; + ScanKeyData skey[2]; + bool found = false; + + if (getObjectClass(obj) != OCLASS_CLASS) { + return false; + } + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcydbid, BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(u_sess->proc_cxt.MyDatabaseId)); + ScanKeyInit(&skey[1], Anum_pg_recyclebin_rcyrelid, BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(obj->objectId)); + + rbRel = heap_open(RecyclebinRelationId, AccessShareLock); + sd = systable_beginscan(rbRel, RecyclebinDbidRelidIndexId, true, NULL, 2, skey); + while ((tup = systable_getnext(sd)) != NULL) { + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); + if ((TrObjType)rbForm->rcyoperation == 'd') { + found = true; + break; + } + } + systable_endscan(sd); + heap_close(rbRel, AccessShareLock); + + return found; +} + +/* + * May this object be a recyclebin object? + * true: with "BIN$" prefix, or not a Relation\Type\Trigger\Constraint\Rule + * false: without "BIN$" prefix, or not exists + */ +static bool TrMaybeRbObject(Oid classid, Oid objid, const char *objname = NULL) +{ + HeapTuple tup; + + /* Note: we preserve rule origin name when RbDrop. */ + if (classid != RewriteRelationId && objname) { + return strncmp(objname, "BIN$", 4) == 0; + } + + switch (classid) { + case RelationRelationId: + tup = SearchSysCache1(RELOID, ObjectIdGetDatum(objid)); + if (tup != NULL) { + objname = NameStr(((Form_pg_class)GETSTRUCT(tup))->relname); + ReleaseSysCache(tup); + } + break; + case TypeRelationId: + tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(objid)); + if (tup != NULL) { + objname = NameStr(((Form_pg_type)GETSTRUCT(tup))->typname); + ReleaseSysCache(tup); + } + break; + case TriggerRelationId: { + Relation relTrig; + ScanKeyData skey[1]; + SysScanDesc sd; + + relTrig = heap_open(TriggerRelationId, AccessShareLock); + ScanKeyInit(&skey[0], ObjectIdAttributeNumber, BTEqualStrategyNumber, + F_OIDEQ, ObjectIdGetDatum(objid)); + sd = systable_beginscan(relTrig, TriggerOidIndexId, true, NULL, 1, skey); + if ((tup = systable_getnext(sd)) != NULL) { + objname = NameStr(((Form_pg_trigger)GETSTRUCT(tup))->tgname); + } + systable_endscan(sd); + heap_close(relTrig, AccessShareLock); + break; + } + case ConstraintRelationId: + tup = SearchSysCache1(CONSTROID, ObjectIdGetDatum(objid)); + if (tup != NULL) { + objname = NameStr(((Form_pg_constraint)GETSTRUCT(tup))->conname); + ReleaseSysCache(tup); + } + break; + case NamespaceRelationId: + /* Treate Namespace as non-recyclebin object. */ + return false; + default: + /* May be a recyclebin object. */ + return true; + } + + if (objname) { + return strncmp(objname, "BIN$", 4) == 0; + } + + return false; +} + +static bool TrIsRefRbObjectImpl(Relation depRel, const ObjectAddress *obj, ObjectAddresses *objSet) +{ + int startIdx; + + if (TrObjInRecyclebin(obj)) { + return true; + } + + startIdx = objSet->numrefs; + + if (!TrMaybeRbObject(obj->classId, obj->objectId)) { + return false; + } + + TrFindAllRefObjs(depRel, obj, objSet, true); + TrFindAllInternalObjs(depRel, obj, objSet, true); + + for (int i = startIdx; i < objSet->numrefs; i++) { + if (TrIsRefRbObjectImpl(depRel, &objSet->refs[i], objSet)) { + return true; + } + } + + return false; +} + +/* object is a rb object, or reference to a rb object. */ +static bool TrIsRefRbObject(const ObjectAddress *obj, Relation depRel) +{ + ObjectAddresses *objSet = new_object_addresses(); + bool relArgNull = depRel == NULL; + bool result = false; + + if (relArgNull) { + depRel = heap_open(DependRelationId, AccessShareLock); + } + + /* Note: we not care obj->deptype here. */ + add_object_address_ext1(obj, objSet); + + result = TrIsRefRbObjectImpl(depRel, obj, objSet); + + free_object_addresses(objSet); + + if (relArgNull) { + heap_close(depRel, AccessShareLock); + } + + return result; +} + +bool TrIsRefRbObjectEx(Oid classid, Oid objid, const char *objname) +{ + if (!TcapFeatureAvail()) { + return false; + } + + /* Note: we preserve rule origin name when RbDrop. */ + if (TrRbIsEmptyDb(u_sess->proc_cxt.MyDatabaseId)) { + return false; + } + + if (classid != RewriteRelationId && objname && strncmp(objname, "BIN$", 4) != 0) { + return false; + } + + ObjectAddress obj = {classid, objid}; + + return TrIsRefRbObject(&obj); +} + +void TrForbidAccessRbDependencies(Relation depRel, const ObjectAddress *depender, + const ObjectAddress *referenced, int nreferenced) +{ + if (!TcapFeatureAvail()) { + return; + } + + if (IsInitdb || TrRbIsEmptyDb(u_sess->proc_cxt.MyDatabaseId)) { + return; + } + + if (TrIsRefRbObject(depender, depRel)) { + elog (ERROR, "can not access recycle object."); + } + + for (int i = 0; i < nreferenced; i++, referenced++) { + if (TrIsRefRbObject(referenced, depRel)) { + elog (ERROR, "can not access recycle object."); + } + } + + return; +} + +void TrForbidAccessRbObject(Oid classid, Oid objid, const char *objname) +{ + if (!TcapFeatureAvail()) { + return; + } + + if (TrRbIsEmptyDb(u_sess->proc_cxt.MyDatabaseId) || !TrMaybeRbObject(classid, objid, objname)) { + return; + } + + ObjectAddress obj = {classid, objid}; + if (TrIsRefRbObject(&obj)) { + elog (ERROR, "can not access recycle object."); + } + + return; +} + +Datum gs_is_recycle_object(PG_FUNCTION_ARGS) +{ + int classid = PG_GETARG_INT32(0); + int objid = PG_GETARG_INT32(1); + Name objname = PG_GETARG_NAME(2); + bool result = false; + result = TrIsRefRbObjectEx(classid, objid, NameStr(*objname)); + PG_RETURN_BOOL(result); +} diff --git a/src/gausskernel/storage/tcap/tcap_truncate.cpp b/src/gausskernel/storage/tcap/tcap_truncate.cpp index ea048f21a..9fb89f4f5 100644 --- a/src/gausskernel/storage/tcap/tcap_truncate.cpp +++ b/src/gausskernel/storage/tcap/tcap_truncate.cpp @@ -1,491 +1,493 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * --------------------------------------------------------------------------------------- - * - * tcap_truncate.cpp - * Routines to support Timecapsule `Recyclebin-based query, restore`. - * We use Tr prefix to indicate it in following coding. - * - * IDENTIFICATION - * src/gausskernel/storage/tcap/tcap_truncate.cpp - * - * --------------------------------------------------------------------------------------- - */ - -#include "postgres.h" - -#include "pgstat.h" -#include "access/reloptions.h" -#include "access/sysattr.h" -#include "access/xlog.h" -#include "catalog/dependency.h" -#include "catalog/heap.h" -#include "catalog/index.h" -#include "catalog/indexing.h" -#include "catalog/objectaccess.h" -#include "catalog/pg_collation_fn.h" -#include "catalog/pg_collation.h" -#include "catalog/pg_constraint.h" -#include "catalog/pg_conversion_fn.h" -#include "catalog/pg_conversion.h" -#include "catalog/pg_depend.h" -#include "catalog/pg_extension_data_source.h" -#include "catalog/pg_extension.h" -#include "catalog/pg_foreign_data_wrapper.h" -#include "catalog/pg_foreign_server.h" -#include "catalog/pg_job.h" -#include "catalog/pg_language.h" -#include "catalog/pg_largeobject.h" -#include "catalog/pg_object.h" -#include "catalog/pg_opclass.h" -#include "catalog/pg_operator.h" -#include "catalog/pg_opfamily.h" -#include "catalog/pg_proc.h" -#include "catalog/pg_recyclebin.h" -#include "catalog/pg_rewrite.h" -#include "catalog/pg_rlspolicy.h" -#include "catalog/pg_synonym.h" -#include "catalog/pg_tablespace.h" -#include "catalog/pg_trigger.h" -#include "catalog/pg_ts_config.h" -#include "catalog/pg_ts_dict.h" -#include "catalog/pg_ts_parser.h" -#include "catalog/pg_ts_template.h" -#include "catalog/pgxc_class.h" -#include "catalog/storage.h" -#include "catalog/pg_partition_fn.h" -#include "commands/comment.h" -#include "commands/dbcommands.h" -#include "commands/directory.h" -#include "commands/extension.h" -#include "commands/matview.h" -#include "commands/proclang.h" -#include "commands/schemacmds.h" -#include "commands/seclabel.h" -#include "commands/sec_rls_cmds.h" -#include "commands/tablecmds.h" -#include "commands/tablespace.h" -#include "commands/trigger.h" -#include "commands/typecmds.h" -#include "executor/node/nodeModifyTable.h" -#include "rewrite/rewriteRemove.h" -#include "storage/lmgr.h" -#include "storage/predicate.h" -#include "storage/smgr/relfilenode.h" -#include "utils/acl.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/inval.h" -#include "utils/lsyscache.h" -#include "utils/relcache.h" -#include "utils/snapmgr.h" -#include "utils/syscache.h" - -#include "storage/tcap.h" -#include "storage/tcap_impl.h" - -void TrRelationSetNewRelfilenode(Relation relation, TransactionId freezeXid, void *baseDesc) -{ - TrObjDesc desc; - TrObjDesc *trBaseDesc = (TrObjDesc *)baseDesc; - RelFileNodeBackend newrnode; - - /* Indexes, sequences must have Invalid frozenxid; other rels must not. */ - Assert((((relation->rd_rel->relkind == RELKIND_INDEX) || (relation->rd_rel->relkind == RELKIND_GLOBAL_INDEX) || - (RELKIND_IS_SEQUENCE(relation->rd_rel->relkind))) ? - (freezeXid == InvalidTransactionId) : - TransactionIdIsNormal(freezeXid)) || - relation->rd_rel->relkind == RELKIND_RELATION); - - /* Record the old relfilenode to recyclebin. */ - desc = *trBaseDesc; - if (!TR_IS_BASE_OBJ_EX(trBaseDesc, RelationGetRelid(relation))) { - TrDescInit(relation, &desc, RB_OPER_TRUNCATE, - TrGetObjType(RelationGetNamespace(relation), RelationGetRelkind(relation)), false); - TrDescWrite(&desc); - } - - /* Allocate a new relfilenode, create storage for the main fork. */ - newrnode = CreateNewRelfilenode(relation, freezeXid); - - /* - * NOTE: if the relation was created in this transaction, it will now be - * present in the pending-delete list twice, once with atCommit true and - * once with atCommit false. Hence, it will be physically deleted at end - * of xact in either case (and the other entry will be ignored by - * smgrDoPendingDeletes, so no error will occur). We could instead remove - * the existing list entry and delete the physical file immediately, but - * for now I'll keep the logic simple. - */ - RelationCloseSmgr(relation); - - /* - * Update pg_class entry for new relfilenode. - */ - UpdatePgclass(relation, freezeXid, &newrnode); - - /* - * Make the pg_class row change visible, as well as the relation map - * change if any. This will cause the relcache entry to get updated, too. - */ - CommandCounterIncrement(); - - /* - * Mark the rel as having been given a new relfilenode in the current - * (sub) transaction. This is a hint that can be used to optimize later - * operations on the rel in the same transaction. - */ - relation->rd_newRelfilenodeSubid = GetCurrentSubTransactionId(); - - /* ... and now we have eoxact cleanup work to do */ - u_sess->relcache_cxt.need_eoxact_work = true; -} - -void TrPartitionSetNewRelfilenode(Relation parent, Partition part, TransactionId freezeXid, void *baseDesc) -{ - RelFileNodeBackend newrnode; - TrObjDesc desc; - TrObjDesc *trBaseDesc = (TrObjDesc *)baseDesc; - - Assert((parent->rd_rel->relkind == RELKIND_INDEX || RELKIND_IS_SEQUENCE(parent->rd_rel->relkind)) - ? freezeXid == InvalidTransactionId - : TransactionIdIsNormal(freezeXid)); - - /* Record the old relfilenode to recyclebin. */ - desc = *trBaseDesc; - if (!TR_IS_BASE_OBJ_EX(trBaseDesc, part->pd_id)) { - TrPartDescInit(parent, part, &desc, RB_OPER_TRUNCATE, - TrGetObjType(RelationGetNamespace(parent), RelationGetRelkind(parent)), false); - TrDescWrite(&desc); - } - - /* Allocate a new relfilenode */ - newrnode = CreateNewRelfilenodePart(parent, part); - - UpdatePartition(parent, part, freezeXid, &newrnode); - - CommandCounterIncrement(); - - /* - * Mark the part as having been given a new relfilenode in the current - * (sub) transaction. This is a hint that can be used to optimize later - * operations on the rel in the same transaction. - */ - part->pd_newRelfilenodeSubid = GetCurrentSubTransactionId(); - - /* ... and now we have eoxact cleanup work to do */ - u_sess->cache_cxt.part_cache_need_eoxact_work = true; -} - -bool TrCheckRecyclebinTruncate(const TruncateStmt *stmt) -{ - RangeVar *rel = NULL; - Oid relid; - - if (/* - * Disable Recyclebin-based-Truncate when with purge option, or - */ - /* recyblebin disabled, or */ - !u_sess->attr.attr_storage.enable_recyclebin || - /* with purge option, or */ - stmt->purge || - /* with restart_seqs option, or */ - stmt->restart_seqs || - /* multi objects truncate. */ - list_length(stmt->relations) != 1) { - return false; - } - - rel = (RangeVar *)linitial(stmt->relations); - relid = RangeVarGetRelid(rel, NoLock, false); - - return NeedTrComm(relid); -} - -void TrTruncateOnePart(Relation rel, HeapTuple tup, Oid insertBaseid) -{ - Oid toastOid = ((Form_pg_partition)GETSTRUCT(tup))->reltoastrelid; - Relation toastRel = NULL; - Oid partOid = HeapTupleGetOid(tup); - Partition p = partitionOpen(rel, partOid, AccessExclusiveLock); - TrObjDesc baseDesc; - TrPartDescInit(rel, p, &baseDesc, RB_OPER_TRUNCATE, RB_OBJ_PARTITION, false); - baseDesc.id = baseDesc.baseid = insertBaseid; - (void)TrDescWrite(&baseDesc); - TrUpdateBaseid(&baseDesc); - - TrPartitionSetNewRelfilenode(rel, p, u_sess->utils_cxt.RecentXmin, &baseDesc); - - /* process the toast table */ - if (OidIsValid(toastOid)) { - Assert(rel->rd_rel->relpersistence != RELPERSISTENCE_UNLOGGED); - toastRel = heap_open(toastOid, AccessExclusiveLock); - TrRelationSetNewRelfilenode(toastRel, u_sess->utils_cxt.RecentXmin, &baseDesc); - heap_close(toastRel, AccessExclusiveLock); - } - partitionClose(rel, p, AccessExclusiveLock); - - /* report truncate partition to PgStatCollector */ - pgstat_report_truncate(partOid, rel->rd_id, rel->rd_rel->relisshared); -} - -void TrPartitionTableProcess(Relation rel, Oid insertBaseid) -{ - /* truncate partitioned table */ - List* partTupleList = NIL; - ListCell* partCell = NULL; - Oid heap_relid; - bool is_shared = rel->rd_rel->relisshared; - - heap_relid = RelationGetRelid(rel); - /* partitioned table unspport the unlogged table */ - Assert(rel->rd_rel->relpersistence != RELPERSISTENCE_UNLOGGED); - - /* process all partition */ - partTupleList = searchPgPartitionByParentId(PART_OBJ_TYPE_TABLE_PARTITION, rel->rd_id); - foreach (partCell, partTupleList) { - if (RelationIsSubPartitioned(rel)) { - /* the "tup" just for get partOid, UHeapTup has no HEAP_HASOID flag, so here use HeapTuple */ - HeapTuple tup = (HeapTuple)lfirst(partCell); - Oid partOid = HeapTupleGetOid(tup); - Partition p = partitionOpen(rel, partOid, AccessExclusiveLock); - Relation partRel = partitionGetRelation(rel, p); - List* subPartTupleList = searchPgPartitionByParentId(PART_OBJ_TYPE_TABLE_SUB_PARTITION, partOid); - ListCell* subPartCell = NULL; - foreach (subPartCell, subPartTupleList) { - HeapTuple tup = (HeapTuple)lfirst(subPartCell); - TrTruncateOnePart(partRel, tup, insertBaseid); - } - freePartList(subPartTupleList); - - releaseDummyRelation(&partRel); - partitionClose(rel, p, AccessExclusiveLock); - } else { - HeapTuple tup = (HeapTuple)lfirst(partCell); - TrTruncateOnePart(rel, tup, insertBaseid); - } - } - - freePartList(partTupleList); - /* report truncate partitioned table to PgStatCollector */ - pgstat_report_truncate(heap_relid, InvalidOid, is_shared); -} - -void TrTruncate(const TruncateStmt *stmt) -{ - RangeVar *rv = (RangeVar*)linitial(stmt->relations); - Relation rel; - Oid relid; - Oid toastRelid; - TrObjDesc baseDesc; - - /* - * 1. Open relation in AccessExclusiveLock, and check permission, etc. - */ - - rel = heap_openrv(rv, AccessExclusiveLock); - relid = RelationGetRelid(rel); - - /* find matview exists or not. */ - Oid mlogid = find_matview_mlog_table(relid); - if (OidIsValid(mlogid)) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("Not support truncate table under materialized view."))); - } - - TrForbidAccessRbObject(RelationRelationId, relid, rv->relname); - - truncate_check_rel(rel); - - /* - * This effectively deletes all rows in the table, and may be done - * in a serializable transaction. In that case we must record a - * rw-conflict in to this transaction from each transaction - * holding a predicate lock on the table. - */ - CheckTableForSerializableConflictIn(rel); - - /* - * 2. Create a new empty storage file for the relation, and assign it - * as the relfilenode value, and record the old relfilenode to recyclebin. - */ - - TrDescInit(rel, &baseDesc, RB_OPER_TRUNCATE, RB_OBJ_TABLE, true, true); - baseDesc.id = baseDesc.baseid = TrDescWrite(&baseDesc); - TrUpdateBaseid(&baseDesc); - - /* - * step 2.1. If rel is partition table, find all partitions, and Create some new empty - * storage file for the all partitions of the relation, and assign them as the - * relfilenodes value, and record the old relfilenodes to recyclebin. - */ - if (RELATION_IS_PARTITIONED(rel)) { - TrPartitionTableProcess(rel, baseDesc.baseid); - } - - TrRelationSetNewRelfilenode(rel, u_sess->utils_cxt.RecentXmin, &baseDesc); - - /* - * 3. The same for the toast table, if any. - */ - - toastRelid = rel->rd_rel->reltoastrelid; - if (OidIsValid(toastRelid) && !RELATION_IS_PARTITIONED(rel)) { - Relation relToast = relation_open(toastRelid, AccessExclusiveLock); - TrRelationSetNewRelfilenode(relToast, u_sess->utils_cxt.RecentXmin, &baseDesc); - heap_close(relToast, NoLock); - } - - /* - * 4. Reconstruct the indexes to match, and we're done. - */ - - (void)ReindexRelation(relid, REINDEX_REL_PROCESS_TOAST, REINDEX_ALL_INDEX, &baseDesc); - - /* - * 5. Report stat, and clean. - */ - - /* report truncate to PgStatCollector */ - pgstat_report_truncate(relid, InvalidOid, false); - - /* Record time of truancate relation. */ - recordRelationMTime(relid, rel->rd_rel->relkind); - - heap_close(rel, NoLock); -} - -/* - * RelationDropStorage - * Schedule unlinking of physical storage at transaction commit. - */ -void TrDoPurgeObjectTruncate(TrObjDesc *desc) -{ - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[1]; - HeapTuple tup; - - Assert (((desc->type == RB_OBJ_TABLE) && desc->canpurge) || ((desc->type == RB_OBJ_PARTITION) && !desc->canpurge)); - - rbRel = heap_open(RecyclebinRelationId, RowExclusiveLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcybaseid, BTEqualStrategyNumber, - F_INT8EQ, Int64GetDatum(desc->baseid)); - - sd = systable_beginscan(rbRel, RecyclebinBaseidIndexId, true, NULL, 1, skey); - while (HeapTupleIsValid(tup = systable_getnext(sd))) { - Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); - RelFileNode rnode; - - rnode.spcNode = ConvertToRelfilenodeTblspcOid(rbForm->rcytablespace); - rnode.dbNode = (rnode.spcNode == GLOBALTABLESPACE_OID) ? InvalidOid : - u_sess->proc_cxt.MyDatabaseId; - rnode.relNode = rbForm->rcyrelfilenode; - rnode.bucketNode = InvalidBktId; - - /* - * Schedule unlinking of the old storage at transaction commit. - */ - InsertStorageIntoPendingList( - &rnode, InvalidAttrNumber, InvalidBackendId, rbForm->rcyowner, true, false); - - simple_heap_delete(rbRel, &tup->t_self); - - ereport(LOG, (errmsg("Delete truncated object %u/%u/%u", rnode.spcNode, - rnode.dbNode, rnode.relNode))); - } - - systable_endscan(sd); - heap_close(rbRel, RowExclusiveLock); - - /* ... and now we have eoxact cleanup work to do */ - u_sess->relcache_cxt.need_eoxact_work = true; - - /* - * CommandCounterIncrement here to ensure that preceding changes are all - * visible to the next deletion step. - */ - CommandCounterIncrement(); -} - -/* flashback table to before truncate */ -void TrRestoreTruncate(const TimeCapsuleStmt *stmt) -{ - TrObjDesc baseDesc; - Relation rbRel; - SysScanDesc sd; - ScanKeyData skey[1]; - HeapTuple tup; - Relation rel; - Oid relid; - bool found = false; - TrObjDesc desc; - - /* process restore truncate fail: TIMECAPSULE TABLE "BIN$3C534EBE021$4930808==$0" TO BEFORE TRUNCATE; */ - found = TrFetchName(stmt->relation->relname, RB_OBJ_TABLE, &desc, RB_OPER_RESTORE_TRUNCATE); - if (found) { - stmt->relation->relname = desc.originname; - } - - rel = heap_openrv(stmt->relation, AccessExclusiveLock); - relid = RelationGetRelid(rel); - if (rel->rd_tam_type == TAM_HEAP) { - heap_close(rel, NoLock); - elog(ERROR, "timecapsule does not support astore yet"); - return; - } - - if (found) { - stmt->relation->relname = desc.name; - } - - /* 1. Fetch the latest available recycle object. */ - TrOperFetch(stmt->relation, RB_OBJ_TABLE, &baseDesc, RB_OPER_RESTORE_TRUNCATE); - - /* 2. Lock recycle object and base relation. */ - baseDesc.authid = GetUserId(); - TrOperPrep(&baseDesc, RB_OPER_RESTORE_TRUNCATE); - - /* 3. Check base relation whether normal and matched. */ - TrBaseRelMatched(&baseDesc); - - /* 4. Do restore. */ - rbRel = heap_open(RecyclebinRelationId, RowExclusiveLock); - - ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcybaseid, BTEqualStrategyNumber, - F_INT8EQ, Int64GetDatum(baseDesc.id)); - - sd = systable_beginscan(rbRel, RecyclebinBaseidIndexId, true, NULL, 1, skey); - while (HeapTupleIsValid(tup = systable_getnext(sd))) { - if (!RELATION_IS_PARTITIONED(rel)) { - TrSwapRelfilenode(rbRel, tup, false); - } else { - TrSwapRelfilenode(rbRel, tup, true); - } - } - - systable_endscan(sd); - heap_close(rbRel, RowExclusiveLock); - heap_close(rel, NoLock); - - /* - * CommandCounterIncrement here to ensure that preceding changes are all - * visible to the next deletion step. - */ - CommandCounterIncrement(); - - return; -} +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * tcap_truncate.cpp + * Routines to support Timecapsule `Recyclebin-based query, restore`. + * We use Tr prefix to indicate it in following coding. + * + * IDENTIFICATION + * src/gausskernel/storage/tcap/tcap_truncate.cpp + * + * --------------------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "pgstat.h" +#include "access/reloptions.h" +#include "access/sysattr.h" +#include "access/xlog.h" +#include "catalog/dependency.h" +#include "catalog/heap.h" +#include "catalog/index.h" +#include "catalog/indexing.h" +#include "catalog/objectaccess.h" +#include "catalog/pg_collation_fn.h" +#include "catalog/pg_collation.h" +#include "catalog/pg_constraint.h" +#include "catalog/pg_conversion_fn.h" +#include "catalog/pg_conversion.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_extension_data_source.h" +#include "catalog/pg_extension.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_job.h" +#include "catalog/pg_language.h" +#include "catalog/pg_largeobject.h" +#include "catalog/pg_object.h" +#include "catalog/pg_opclass.h" +#include "catalog/pg_operator.h" +#include "catalog/pg_opfamily.h" +#include "catalog/pg_proc.h" +#include "catalog/pg_recyclebin.h" +#include "catalog/pg_rewrite.h" +#include "catalog/pg_rlspolicy.h" +#include "catalog/pg_synonym.h" +#include "catalog/pg_tablespace.h" +#include "catalog/pg_trigger.h" +#include "catalog/pg_ts_config.h" +#include "catalog/pg_ts_dict.h" +#include "catalog/pg_ts_parser.h" +#include "catalog/pg_ts_template.h" +#include "catalog/pgxc_class.h" +#include "catalog/storage.h" +#include "catalog/pg_partition_fn.h" +#include "commands/comment.h" +#include "commands/dbcommands.h" +#include "commands/directory.h" +#include "commands/extension.h" +#include "commands/matview.h" +#include "commands/proclang.h" +#include "commands/schemacmds.h" +#include "commands/seclabel.h" +#include "commands/sec_rls_cmds.h" +#include "commands/tablecmds.h" +#include "commands/tablespace.h" +#include "commands/trigger.h" +#include "commands/typecmds.h" +#include "executor/node/nodeModifyTable.h" +#include "rewrite/rewriteRemove.h" +#include "storage/lmgr.h" +#include "storage/predicate.h" +#include "storage/smgr/relfilenode.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/inval.h" +#include "utils/knl_partcache.h" +#include "utils/knl_relcache.h" +#include "utils/lsyscache.h" +#include "utils/relcache.h" +#include "utils/snapmgr.h" +#include "utils/syscache.h" + +#include "storage/tcap.h" +#include "storage/tcap_impl.h" + +void TrRelationSetNewRelfilenode(Relation relation, TransactionId freezeXid, void *baseDesc) +{ + TrObjDesc desc; + TrObjDesc *trBaseDesc = (TrObjDesc *)baseDesc; + RelFileNodeBackend newrnode; + + /* Indexes, sequences must have Invalid frozenxid; other rels must not. */ + Assert((((relation->rd_rel->relkind == RELKIND_INDEX) || (relation->rd_rel->relkind == RELKIND_GLOBAL_INDEX) || + (RELKIND_IS_SEQUENCE(relation->rd_rel->relkind))) ? + (freezeXid == InvalidTransactionId) : + TransactionIdIsNormal(freezeXid)) || + relation->rd_rel->relkind == RELKIND_RELATION); + + /* Record the old relfilenode to recyclebin. */ + desc = *trBaseDesc; + if (!TR_IS_BASE_OBJ_EX(trBaseDesc, RelationGetRelid(relation))) { + TrDescInit(relation, &desc, RB_OPER_TRUNCATE, + TrGetObjType(RelationGetNamespace(relation), RelationGetRelkind(relation)), false); + TrDescWrite(&desc); + } + + /* Allocate a new relfilenode, create storage for the main fork. */ + newrnode = CreateNewRelfilenode(relation, freezeXid); + + /* + * NOTE: if the relation was created in this transaction, it will now be + * present in the pending-delete list twice, once with atCommit true and + * once with atCommit false. Hence, it will be physically deleted at end + * of xact in either case (and the other entry will be ignored by + * smgrDoPendingDeletes, so no error will occur). We could instead remove + * the existing list entry and delete the physical file immediately, but + * for now I'll keep the logic simple. + */ + RelationCloseSmgr(relation); + + /* + * Update pg_class entry for new relfilenode. + */ + UpdatePgclass(relation, freezeXid, &newrnode); + + /* + * Make the pg_class row change visible, as well as the relation map + * change if any. This will cause the relcache entry to get updated, too. + */ + CommandCounterIncrement(); + + /* + * Mark the rel as having been given a new relfilenode in the current + * (sub) transaction. This is a hint that can be used to optimize later + * operations on the rel in the same transaction. + */ + relation->rd_newRelfilenodeSubid = GetCurrentSubTransactionId(); + + /* ... and now we have eoxact cleanup work to do */ + SetRelCacheNeedEOXActWork(true); +} + +void TrPartitionSetNewRelfilenode(Relation parent, Partition part, TransactionId freezeXid, void *baseDesc) +{ + RelFileNodeBackend newrnode; + TrObjDesc desc; + TrObjDesc *trBaseDesc = (TrObjDesc *)baseDesc; + + Assert((parent->rd_rel->relkind == RELKIND_INDEX || RELKIND_IS_SEQUENCE(parent->rd_rel->relkind)) + ? freezeXid == InvalidTransactionId + : TransactionIdIsNormal(freezeXid)); + + /* Record the old relfilenode to recyclebin. */ + desc = *trBaseDesc; + if (!TR_IS_BASE_OBJ_EX(trBaseDesc, part->pd_id)) { + TrPartDescInit(parent, part, &desc, RB_OPER_TRUNCATE, + TrGetObjType(RelationGetNamespace(parent), RelationGetRelkind(parent)), false); + TrDescWrite(&desc); + } + + /* Allocate a new relfilenode */ + newrnode = CreateNewRelfilenodePart(parent, part); + + UpdatePartition(parent, part, freezeXid, &newrnode); + + CommandCounterIncrement(); + + /* + * Mark the part as having been given a new relfilenode in the current + * (sub) transaction. This is a hint that can be used to optimize later + * operations on the rel in the same transaction. + */ + part->pd_newRelfilenodeSubid = GetCurrentSubTransactionId(); + + /* ... and now we have eoxact cleanup work to do */ + SetPartCacheNeedEOXActWork(true); +} + +bool TrCheckRecyclebinTruncate(const TruncateStmt *stmt) +{ + RangeVar *rel = NULL; + Oid relid; + + if (/* + * Disable Recyclebin-based-Truncate when with purge option, or + */ + /* recyblebin disabled, or */ + !u_sess->attr.attr_storage.enable_recyclebin || + /* with purge option, or */ + stmt->purge || + /* with restart_seqs option, or */ + stmt->restart_seqs || + /* multi objects truncate. */ + list_length(stmt->relations) != 1) { + return false; + } + + rel = (RangeVar *)linitial(stmt->relations); + relid = RangeVarGetRelid(rel, NoLock, false); + + return NeedTrComm(relid); +} + +void TrTruncateOnePart(Relation rel, HeapTuple tup, Oid insertBaseid) +{ + Oid toastOid = ((Form_pg_partition)GETSTRUCT(tup))->reltoastrelid; + Relation toastRel = NULL; + Oid partOid = HeapTupleGetOid(tup); + Partition p = partitionOpen(rel, partOid, AccessExclusiveLock); + TrObjDesc baseDesc; + TrPartDescInit(rel, p, &baseDesc, RB_OPER_TRUNCATE, RB_OBJ_PARTITION, false); + baseDesc.id = baseDesc.baseid = insertBaseid; + (void)TrDescWrite(&baseDesc); + TrUpdateBaseid(&baseDesc); + + TrPartitionSetNewRelfilenode(rel, p, u_sess->utils_cxt.RecentXmin, &baseDesc); + + /* process the toast table */ + if (OidIsValid(toastOid)) { + Assert(rel->rd_rel->relpersistence != RELPERSISTENCE_UNLOGGED); + toastRel = heap_open(toastOid, AccessExclusiveLock); + TrRelationSetNewRelfilenode(toastRel, u_sess->utils_cxt.RecentXmin, &baseDesc); + heap_close(toastRel, AccessExclusiveLock); + } + partitionClose(rel, p, AccessExclusiveLock); + + /* report truncate partition to PgStatCollector */ + pgstat_report_truncate(partOid, rel->rd_id, rel->rd_rel->relisshared); +} + +void TrPartitionTableProcess(Relation rel, Oid insertBaseid) +{ + /* truncate partitioned table */ + List* partTupleList = NIL; + ListCell* partCell = NULL; + Oid heap_relid; + bool is_shared = rel->rd_rel->relisshared; + + heap_relid = RelationGetRelid(rel); + /* partitioned table unspport the unlogged table */ + Assert(rel->rd_rel->relpersistence != RELPERSISTENCE_UNLOGGED); + + /* process all partition */ + partTupleList = searchPgPartitionByParentId(PART_OBJ_TYPE_TABLE_PARTITION, rel->rd_id); + foreach (partCell, partTupleList) { + if (RelationIsSubPartitioned(rel)) { + /* the "tup" just for get partOid, UHeapTup has no HEAP_HASOID flag, so here use HeapTuple */ + HeapTuple tup = (HeapTuple)lfirst(partCell); + Oid partOid = HeapTupleGetOid(tup); + Partition p = partitionOpen(rel, partOid, AccessExclusiveLock); + Relation partRel = partitionGetRelation(rel, p); + List* subPartTupleList = searchPgPartitionByParentId(PART_OBJ_TYPE_TABLE_SUB_PARTITION, partOid); + ListCell* subPartCell = NULL; + foreach (subPartCell, subPartTupleList) { + HeapTuple tup = (HeapTuple)lfirst(subPartCell); + TrTruncateOnePart(partRel, tup, insertBaseid); + } + freePartList(subPartTupleList); + + releaseDummyRelation(&partRel); + partitionClose(rel, p, AccessExclusiveLock); + } else { + HeapTuple tup = (HeapTuple)lfirst(partCell); + TrTruncateOnePart(rel, tup, insertBaseid); + } + } + + freePartList(partTupleList); + /* report truncate partitioned table to PgStatCollector */ + pgstat_report_truncate(heap_relid, InvalidOid, is_shared); +} + +void TrTruncate(const TruncateStmt *stmt) +{ + RangeVar *rv = (RangeVar*)linitial(stmt->relations); + Relation rel; + Oid relid; + Oid toastRelid; + TrObjDesc baseDesc; + + /* + * 1. Open relation in AccessExclusiveLock, and check permission, etc. + */ + + rel = heap_openrv(rv, AccessExclusiveLock); + relid = RelationGetRelid(rel); + + /* find matview exists or not. */ + Oid mlogid = find_matview_mlog_table(relid); + if (OidIsValid(mlogid)) { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("Not support truncate table under materialized view."))); + } + + TrForbidAccessRbObject(RelationRelationId, relid, rv->relname); + + truncate_check_rel(rel); + + /* + * This effectively deletes all rows in the table, and may be done + * in a serializable transaction. In that case we must record a + * rw-conflict in to this transaction from each transaction + * holding a predicate lock on the table. + */ + CheckTableForSerializableConflictIn(rel); + + /* + * 2. Create a new empty storage file for the relation, and assign it + * as the relfilenode value, and record the old relfilenode to recyclebin. + */ + + TrDescInit(rel, &baseDesc, RB_OPER_TRUNCATE, RB_OBJ_TABLE, true, true); + baseDesc.id = baseDesc.baseid = TrDescWrite(&baseDesc); + TrUpdateBaseid(&baseDesc); + + /* + * step 2.1. If rel is partition table, find all partitions, and Create some new empty + * storage file for the all partitions of the relation, and assign them as the + * relfilenodes value, and record the old relfilenodes to recyclebin. + */ + if (RELATION_IS_PARTITIONED(rel)) { + TrPartitionTableProcess(rel, baseDesc.baseid); + } + + TrRelationSetNewRelfilenode(rel, u_sess->utils_cxt.RecentXmin, &baseDesc); + + /* + * 3. The same for the toast table, if any. + */ + + toastRelid = rel->rd_rel->reltoastrelid; + if (OidIsValid(toastRelid) && !RELATION_IS_PARTITIONED(rel)) { + Relation relToast = relation_open(toastRelid, AccessExclusiveLock); + TrRelationSetNewRelfilenode(relToast, u_sess->utils_cxt.RecentXmin, &baseDesc); + heap_close(relToast, NoLock); + } + + /* + * 4. Reconstruct the indexes to match, and we're done. + */ + + (void)ReindexRelation(relid, REINDEX_REL_PROCESS_TOAST, REINDEX_ALL_INDEX, &baseDesc); + + /* + * 5. Report stat, and clean. + */ + + /* report truncate to PgStatCollector */ + pgstat_report_truncate(relid, InvalidOid, false); + + /* Record time of truancate relation. */ + recordRelationMTime(relid, rel->rd_rel->relkind); + + heap_close(rel, NoLock); +} + +/* + * RelationDropStorage + * Schedule unlinking of physical storage at transaction commit. + */ +void TrDoPurgeObjectTruncate(TrObjDesc *desc) +{ + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[1]; + HeapTuple tup; + + Assert (((desc->type == RB_OBJ_TABLE) && desc->canpurge) || ((desc->type == RB_OBJ_PARTITION) && !desc->canpurge)); + + rbRel = heap_open(RecyclebinRelationId, RowExclusiveLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcybaseid, BTEqualStrategyNumber, + F_INT8EQ, Int64GetDatum(desc->baseid)); + + sd = systable_beginscan(rbRel, RecyclebinBaseidIndexId, true, NULL, 1, skey); + while (HeapTupleIsValid(tup = systable_getnext(sd))) { + Form_pg_recyclebin rbForm = (Form_pg_recyclebin)GETSTRUCT(tup); + RelFileNode rnode; + + rnode.spcNode = ConvertToRelfilenodeTblspcOid(rbForm->rcytablespace); + rnode.dbNode = (rnode.spcNode == GLOBALTABLESPACE_OID) ? InvalidOid : + u_sess->proc_cxt.MyDatabaseId; + rnode.relNode = rbForm->rcyrelfilenode; + rnode.bucketNode = InvalidBktId; + + /* + * Schedule unlinking of the old storage at transaction commit. + */ + InsertStorageIntoPendingList( + &rnode, InvalidAttrNumber, InvalidBackendId, rbForm->rcyowner, true, false); + + simple_heap_delete(rbRel, &tup->t_self); + + ereport(LOG, (errmsg("Delete truncated object %u/%u/%u", rnode.spcNode, + rnode.dbNode, rnode.relNode))); + } + + systable_endscan(sd); + heap_close(rbRel, RowExclusiveLock); + + /* ... and now we have eoxact cleanup work to do */ + SetRelCacheNeedEOXActWork(true); + + /* + * CommandCounterIncrement here to ensure that preceding changes are all + * visible to the next deletion step. + */ + CommandCounterIncrement(); +} + +/* flashback table to before truncate */ +void TrRestoreTruncate(const TimeCapsuleStmt *stmt) +{ + TrObjDesc baseDesc; + Relation rbRel; + SysScanDesc sd; + ScanKeyData skey[1]; + HeapTuple tup; + Relation rel; + Oid relid; + bool found = false; + TrObjDesc desc; + + /* process restore truncate fail: TIMECAPSULE TABLE "BIN$3C534EBE021$4930808==$0" TO BEFORE TRUNCATE; */ + found = TrFetchName(stmt->relation->relname, RB_OBJ_TABLE, &desc, RB_OPER_RESTORE_TRUNCATE); + if (found) { + stmt->relation->relname = desc.originname; + } + + rel = heap_openrv(stmt->relation, AccessExclusiveLock); + relid = RelationGetRelid(rel); + if (rel->rd_tam_type == TAM_HEAP) { + heap_close(rel, NoLock); + elog(ERROR, "timecapsule does not support astore yet"); + return; + } + + if (found) { + stmt->relation->relname = desc.name; + } + + /* 1. Fetch the latest available recycle object. */ + TrOperFetch(stmt->relation, RB_OBJ_TABLE, &baseDesc, RB_OPER_RESTORE_TRUNCATE); + + /* 2. Lock recycle object and base relation. */ + baseDesc.authid = GetUserId(); + TrOperPrep(&baseDesc, RB_OPER_RESTORE_TRUNCATE); + + /* 3. Check base relation whether normal and matched. */ + TrBaseRelMatched(&baseDesc); + + /* 4. Do restore. */ + rbRel = heap_open(RecyclebinRelationId, RowExclusiveLock); + + ScanKeyInit(&skey[0], Anum_pg_recyclebin_rcybaseid, BTEqualStrategyNumber, + F_INT8EQ, Int64GetDatum(baseDesc.id)); + + sd = systable_beginscan(rbRel, RecyclebinBaseidIndexId, true, NULL, 1, skey); + while (HeapTupleIsValid(tup = systable_getnext(sd))) { + if (!RELATION_IS_PARTITIONED(rel)) { + TrSwapRelfilenode(rbRel, tup, false); + } else { + TrSwapRelfilenode(rbRel, tup, true); + } + } + + systable_endscan(sd); + heap_close(rbRel, RowExclusiveLock); + heap_close(rel, NoLock); + + /* + * CommandCounterIncrement here to ensure that preceding changes are all + * visible to the next deletion step. + */ + CommandCounterIncrement(); + + return; +} diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index f04fdf90d..8419c9e2c 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -290,6 +290,8 @@ extern TableScanDesc heap_beginscan_sampling(Relation relation, Snapshot snapsho extern void heapgetpage(TableScanDesc scan, BlockNumber page); +extern void heap_invalid_invisible_tuple(HeapTuple tuple); + extern void heap_rescan(TableScanDesc sscan, ScanKey key); extern void heap_endscan(TableScanDesc scan); extern HeapTuple heap_getnext(TableScanDesc scan, ScanDirection direction); @@ -367,8 +369,10 @@ extern XLogRecPtr log_heap_cleanup_info(const RelFileNode* rnode, TransactionId extern XLogRecPtr log_heap_clean(Relation reln, Buffer buffer, OffsetNumber* redirected, int nredirected, OffsetNumber* nowdead, int ndead, OffsetNumber* nowunused, int nunused, TransactionId latestRemovedXid, bool repair_fragmentation); -extern XLogRecPtr log_heap_freeze( - Relation reln, Buffer buffer, TransactionId cutoff_xid, MultiXactId cutoff_multi, OffsetNumber* offsets, int offcnt); +extern XLogRecPtr log_heap_freeze(Relation reln, Buffer buffer, TransactionId cutoff_xid, MultiXactId cutoff_multi, + OffsetNumber* offsets, int offcnt); +extern XLogRecPtr log_heap_invalid(Relation reln, Buffer buffer, TransactionId cutoff_xid, OffsetNumber* offsets, + int offcnt); extern XLogRecPtr log_heap_visible(RelFileNode rnode, BlockNumber block, Buffer heap_buffer, Buffer vm_buffer, TransactionId cutoff_xid, bool free_dict); extern XLogRecPtr log_cu_bcm(const RelFileNode* rnode, int col, uint64 block, int status, int count); diff --git a/src/include/access/htup.h b/src/include/access/htup.h index c5094324b..4f175ec8a 100644 --- a/src/include/access/htup.h +++ b/src/include/access/htup.h @@ -748,6 +748,7 @@ inline HeapTuple heaptup_alloc(Size size) /* XLOG_HEAP_NEW_CID with 0x30 in heap is XLOGHEAP2_NEW_CID with 0x70 in heap2 in PG9.4 */ #define XLOG_HEAP3_NEW_CID 0x00 #define XLOG_HEAP3_REWRITE 0x10 +#define XLOG_HEAP3_INVALID 0x20 /* we used to put all xl_heap_* together, which made us run out of opcodes (quickly) * when trying to add a DELETE_IS_SUPER operation. Thus we split the codes carefully @@ -980,6 +981,12 @@ typedef struct xl_heap_freeze { #define SizeOfOldHeapFreeze (offsetof(xl_heap_freeze, cutoff_xid) + sizeof(TransactionId)) #define SizeOfHeapFreeze (offsetof(xl_heap_freeze, cutoff_multi) + sizeof(MultiXactId)) +typedef struct xl_heap_invalid { + TransactionId cutoff_xid; + /* TUPLE OFFSET NUMBERS FOLLOW AT THE END */ +} xl_heap_invalid; +#define SizeOfHeapInvalid (offsetof(xl_heap_invalid, cutoff_xid) + sizeof(TransactionId)) + typedef struct xl_heap_freeze_tuple { TransactionId xmax; OffsetNumber offset; @@ -1212,7 +1219,7 @@ typedef struct KeepInvisbleOpt { KeepInvisbleTupleFunc checkKeepFunc; } KeepInvisbleOpt; -bool HeapKeepInvisbleTuple(HeapTuple tuple, TupleDesc tupleDesc, KeepInvisbleTupleFunc checkKeepFunc = NULL); +bool HeapKeepInvisibleTuple(HeapTuple tuple, TupleDesc tupleDesc, KeepInvisbleTupleFunc checkKeepFunc = NULL); void HeapCopyTupleNoAlloc(HeapTuple dest, HeapTuple src); // for ut test diff --git a/src/include/access/nbtree.h b/src/include/access/nbtree.h index d5207a76a..c7dfe08c7 100644 --- a/src/include/access/nbtree.h +++ b/src/include/access/nbtree.h @@ -1081,6 +1081,7 @@ extern TransactionId _bt_check_unique(Relation rel, IndexTuple itup, Relation he CBIScanDesc cbiScan, CUDescScan* cudesc); extern bool SearchBufferAndCheckUnique(Relation rel, IndexTuple itup, IndexUniqueCheck checkUnique, Relation heapRel, GPIScanDesc gpiScan, CBIScanDesc cbiScan, CUDescScan* cudescScan, BTCheckElement* element); +extern bool CheckPartitionIsInvisible(GPIScanDesc gpiScan); /* * prototypes for functions in nbtpage.c diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h index 01d95f3c6..09bd4cbdd 100644 --- a/src/include/access/tuptoaster.h +++ b/src/include/access/tuptoaster.h @@ -247,10 +247,7 @@ extern HeapTuple toast_flatten_tuple(HeapTuple tup, TupleDesc tupleDesc); */ extern Datum toast_flatten_tuple_attribute(Datum value, Oid typeId, int32 typeMod); extern text* text_catenate_huge(text* t1, text* t2, Oid toastOid); -extern Oid get_toast_oid(); extern int64 calculate_huge_length(text* t); -extern Datum fetch_lob_value_from_tuple(varatt_lob_pointer* lob_pointer, Oid update_oid, - bool* is_null, bool* is_huge_clob = NULL); /* ---------- * toast_compress_datum - @@ -278,9 +275,28 @@ extern Size toast_datum_size(Datum value); extern bool toastrel_valueid_exists(Relation toastrel, Oid valueid); +extern bool create_toast_by_sid(Oid *toastOid); +extern Oid get_toast_oid(); extern varlena* toast_huge_write_datum_slice(struct varlena* attr1, struct varlena* attr2, int64 sliceoffset, int32 length); extern varlena* toast_pointer_fetch_data(TupleTableSlot* varSlot, Form_pg_attribute attr, int varNumber); -bool create_toast_by_sid(Oid *toastOid); +extern Datum fetch_lob_value_from_tuple(varatt_lob_pointer* lob_pointer, Oid update_oid, bool* is_null); + +inline Datum fetch_real_lob_if_need(Datum toast_pointer) +{ + Datum ret = toast_pointer; + if (VARATT_IS_EXTERNAL_LOB(toast_pointer)) { + bool isNull = false; + struct varatt_lob_pointer* lob_pointer = (varatt_lob_pointer*)(VARDATA_EXTERNAL(toast_pointer)); + ret = fetch_lob_value_from_tuple(lob_pointer, InvalidOid, &isNull); + if (unlikely(isNull)) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Invalid lob pointer."))); + } + } + return ret; +} + +extern HeapTuple ctid_get_tuple(Relation relation, ItemPointer tid); #endif /* TUPTOASTER_H */ diff --git a/src/include/access/ustore/knl_undorequest.h b/src/include/access/ustore/knl_undorequest.h index 6380b0249..a22f9f2c3 100644 --- a/src/include/access/ustore/knl_undorequest.h +++ b/src/include/access/ustore/knl_undorequest.h @@ -21,6 +21,9 @@ #include "access/ustore/knl_uundovec.h" #include "utils/rel.h" +/* Maximum size of the cache of undo records in a single rollback, 64M. */ +const uint64 MAX_UNDO_APPLY_SIZE = 64 * 1024 * 1024; + typedef struct RollbackRequestsHashKey { TransactionId xid; UndoRecPtr startUndoPtr; diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h index 2746498ed..577a99294 100755 --- a/src/include/access/xlog.h +++ b/src/include/access/xlog.h @@ -768,6 +768,7 @@ extern XLogRecPtr GetDDLDelayStartPtr(void); extern XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, unsigned long long* consensusPaxosIdx = NULL); extern XLogRecPtr StandbyDoStopBackup(char *labelfile); extern void do_pg_abort_backup(void); +extern void RegisterPersistentAbortBackupHandler(void); extern void RegisterAbortExclusiveBackup(); extern void enable_delay_xlog_recycle(bool isRedo = false); extern void disable_delay_xlog_recycle(bool isRedo = false); diff --git a/src/include/access/xlogproc.h b/src/include/access/xlogproc.h index 9bf89ffe0..0a1807668 100755 --- a/src/include/access/xlogproc.h +++ b/src/include/access/xlogproc.h @@ -1,1240 +1,1241 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * --------------------------------------------------------------------------------------- - * - * xlogproc.h - * - * - * IDENTIFICATION - * src/include/access/xlogproc.h - * - * --------------------------------------------------------------------------------------- - */ - -#ifndef XLOG_PROC_H -#define XLOG_PROC_H -#include "postgres.h" -#include "knl/knl_variable.h" - -#include "access/xlogreader.h" -#include "storage/buf/bufmgr.h" -#include "storage/buf/buf_internals.h" -#include "access/xlog_basic.h" -#include "access/xlogutils.h" -#include "access/clog.h" -#include "access/ustore/knl_uredo.h" -#include "access/ustore/knl_utuple.h" -#include "access/ustore/undo/knl_uundotxn.h" -#include "access/ustore/undo/knl_uundoxlog.h" - -#ifndef byte -#define byte unsigned char -#endif - -typedef void (*relasexlogreadstate)(void* record); -/* **************define for parse end******************************* */ -#define MIN(_a, _b) ((_a) > (_b) ? (_b) : (_a)) - -/* for common blockhead begin */ - -#define XLogBlockHeadGetInfo(blockhead) ((blockhead)->xl_info) -#define XLogBlockHeadGetXid(blockhead) ((blockhead)->xl_xid) -#define XLogBlockHeadGetRmid(blockhead) ((blockhead)->xl_rmid) - -#define XLogBlockHeadGetLSN(blockhead) ((blockhead)->end_ptr) -#define XLogBlockHeadGetRelNode(blockhead) ((blockhead)->relNode) -#define XLogBlockHeadGetSpcNode(blockhead) ((blockhead)->spcNode) -#define XLogBlockHeadGetDbNode(blockhead) ((blockhead)->dbNode) -#define XLogBlockHeadGetForkNum(blockhead) ((blockhead)->forknum) -#define XLogBlockHeadGetBlockNum(blockhead) ((blockhead)->blkno) -#define XLogBlockHeadGetBucketId(blockhead) ((blockhead)->bucketNode) -#define XLogBlockHeadGetCompressOpt(blockhead) ((blockhead)->opt) -#define XLogBlockHeadGetValidInfo(blockhead) ((blockhead)->block_valid) -#define XLogBlockHeadGetPhysicalBlock(blockhead) ((blockhead)->pblk) -/* for common blockhead end */ - -/* for block data beging */ -#define XLogBlockDataHasBlockImage(blockdata) ((blockdata)->blockhead.has_image) -#define XLogBlockDataHasBlockData(blockdata) ((blockdata)->blockhead.has_data) -#define XLogBlockDataGetLastBlockLSN(_blockdata) ((_blockdata)->blockdata.last_lsn) -#define XLogBlockDataGetBlockFlags(blockdata) ((blockdata)->blockhead.flags) - -#define XLogBlockDataGetBlockId(blockdata) ((blockdata)->blockhead.cur_block_id) -#define XLogBlockDataGetAuxiBlock1(blockdata) ((blockdata)->blockhead.auxiblk1) -#define XLogBlockDataGetAuxiBlock2(blockdata) ((blockdata)->blockhead.auxiblk2) -/* for block data end */ - -typedef struct { - RelFileNode rnode; - ForkNumber forknum; - BlockNumber blkno; - XLogPhyBlock pblk; -} RedoBufferTag; - -typedef struct { - Page page; // pagepointer - Size pagesize; -#ifdef USE_ASSERT_CHECKING - bool ignorecheck; -#endif -} RedoPageInfo; - -typedef struct { - XLogRecPtr lsn; /* block cur lsn */ - Buffer buf; - RedoBufferTag blockinfo; - RedoPageInfo pageinfo; - int dirtyflag; /* true if the buffer changed */ -} RedoBufferInfo; - -extern void GetFlushBufferInfo(void *buf, RedoBufferInfo *bufferinfo, uint32 *buf_state, ReadBufferMethod flushmethod); - -#define MakeRedoBufferDirty(bufferinfo) ((bufferinfo)->dirtyflag = true) -#define RedoBufferDirtyClear(bufferinfo) ((bufferinfo)->dirtyflag = false) -#define IsRedoBufferDirty(bufferinfo) ((bufferinfo)->dirtyflag == true) - -#define RedoMemIsValid(memctl, bufferid) (((bufferid) > InvalidBuffer) && ((bufferid) <= (memctl->totalblknum))) - -typedef struct { - RedoBufferTag blockinfo; - pg_atomic_uint32 state; -} RedoBufferDesc; - -typedef struct { - Buffer buff_id; - pg_atomic_uint32 state; -} ParseBufferDesc; - -#define RedoBufferSlotGetBuffer(bslot) ((bslot)->buf_id) - -#define EnalbeWalLsnCheck true - -#pragma pack(push, 1) - -#define INVALID_BLOCK_ID (XLR_MAX_BLOCK_ID + 2) - -#define LOW_BLOKNUMBER_BITS (32) -#define LOW_BLOKNUMBER_MASK (((uint64)1 << 32) - 1) - - -/* ********BLOCK COMMON HEADER BEGIN ***************** */ -typedef enum { - BLOCK_DATA_MAIN_DATA_TYPE = 0, /* BLOCK DATA */ - BLOCK_DATA_VM_TYPE, /* VM */ - BLOCK_DATA_UNDO_TYPE, /* UNDO */ - BLOCK_DATA_FSM_TYPE, /* FSM */ - BLOCK_DATA_DDL_TYPE, /* DDL */ - BLOCK_DATA_BCM_TYPE, /* bcm */ - BLOCK_DATA_NEWCU_TYPE, /* cu newlog */ - BLOCK_DATA_CLOG_TYPE, /* CLog */ - BLOCK_DATA_MULITACT_OFF_TYPE, /* MultiXact */ - BLOCK_DATA_MULITACT_MEM_TYPE, - BLOCK_DATA_CSNLOG_TYPE, /* CSNLog */ - /* *****xact don't need sent to dfv */ - BLOCK_DATA_MULITACT_UPDATEOID_TYPE, - BLOCK_DATA_XACTDATA_TYPE, /* XACT */ - BLOCK_DATA_RELMAP_TYPE, /* RELMAP */ - BLOCK_DATA_SLOT_TYPE, - BLOCK_DATA_BARRIER_TYPE, - BLOCK_DATA_PREPARE_TYPE, /* prepare */ - BLOCK_DATA_INVALIDMSG_TYPE, /* INVALIDMSG */ - BLOCK_DATA_INCOMPLETE_TYPE, - BLOCK_DATA_VACUUM_PIN_TYPE, - BLOCK_DATA_XLOG_COMMON_TYPE, - BLOCK_DATA_CREATE_DATABASE_TYPE, - BLOCK_DATA_DROP_DATABASE_TYPE, - BLOCK_DATA_CREATE_TBLSPC_TYPE, - BLOCK_DATA_DROP_TBLSPC_TYPE, - BLOCK_DATA_DROP_SLICE_TYPE, - BLOCK_DATA_SEG_FILE_EXTEND_TYPE, - BLOCK_DATA_SEG_SPACE_DROP, - BLOCK_DATA_SEG_SPACE_SHRINK, - BLOCK_DATA_SEG_FULL_SYNC_TYPE, - BLOCK_DATA_SEG_EXTEND, -} XLogBlockParseEnum; - -/* ********BLOCK COMMON HEADER END ***************** */ - -/* **************define for parse begin ******************************* */ - -/* ********BLOCK DATE BEGIN ***************** */ - -typedef struct { - uint8 cur_block_id; /* blockid */ - uint8 flags; - uint8 has_image; - uint8 has_data; - BlockNumber auxiblk1; - BlockNumber auxiblk2; -} XLogBlocDatakHead; - -#define XLOG_BLOCK_DATAHEAD_LEN sizeof(XLogBlocDatakHead) - -typedef struct { - uint16 extra_flag; - uint16 hole_offset; - uint16 hole_length; /* image position */ - uint16 data_len; /* data length */ - XLogRecPtr last_lsn; - char* bkp_image; - char* data; -} XLogBlockData; - -#define XLOG_BLOCK_DATA_LEN sizeof(XLogBlockData) - -typedef struct { - XLogBlocDatakHead blockhead; - XLogBlockData blockdata; - uint32 main_data_len; /* main data portion's length */ - char* main_data; /* point to XLogReaderState's main_data */ -} XLogBlockDataParse; -/* ********BLOCK DATE END ***************** */ -#define XLOG_BLOCK_DATA_PARSE_LEN sizeof(XLogBlockDataParse) - -/* ********BLOCK DDL BEGIN ***************** */ -typedef enum { - BLOCK_DDL_TYPE_NONE = 0, - BLOCK_DDL_CREATE_RELNODE, - BLOCK_DDL_DROP_RELNODE, - BLOCK_DDL_EXTEND_RELNODE, - BLOCK_DDL_TRUNCATE_RELNODE, - BLOCK_DDL_CLOG_ZERO, - BLOCK_DDL_CLOG_TRUNCATE, - BLOCK_DDL_MULTIXACT_OFF_ZERO, - BLOCK_DDL_MULTIXACT_MEM_ZERO, -} XLogBlockDdlInfoEnum; - -typedef struct { - uint32 blockddltype; - int rels; - char *mainData; -} XLogBlockDdlParse; - -/* ********BLOCK DDL END ***************** */ - -/* ********BLOCK CLOG BEGIN ***************** */ - -#define MAX_BLOCK_XID_NUMS (28) -typedef struct { - TransactionId topxid; - uint16 status; - uint16 xidnum; - uint16 xidsarry[MAX_BLOCK_XID_NUMS]; -} XLogBlockCLogParse; - -/* ********BLOCK CLOG END ***************** */ - -/* ********BLOCK CSNLOG BEGIN ***************** */ -typedef struct { - TransactionId topxid; - CommitSeqNo cslseq; - uint32 xidnum; - uint16 xidsarry[MAX_BLOCK_XID_NUMS]; -} XLogBlockCSNLogParse; - -/* ********BLOCK CSNLOG END ***************** */ - -/* ********BLOCK prepare BEGIN ***************** */ -struct TwoPhaseFileHeader; - -typedef struct { - TransactionId maxxid; - Size maindatalen; - char* maindata; -} XLogBlockPrepareParse; - -/* ********BLOCK prepare END ***************** */ - -/* ********BLOCK Bcm BEGIN ***************** */ -typedef struct { - uint64 startblock; - int count; - int status; -} XLogBlockBcmParse; - -/* ********BLOCK Bcm END ***************** */ - -/* ********BLOCK Vm BEGIN ***************** */ -typedef struct { - BlockNumber heapBlk; -} XLogBlockVmParse; - -#define XLOG_BLOCK_VM_PARSE_LEN sizeof(XLogBlockVmParse) -/* ********BLOCK Vm END ***************** */ - -/* ********BLOCK Undo BEGIN ***************** */ -struct insertUndoParse { - TransactionId recxid; - BlockNumber blkno; - Oid spcNode; - Oid relNode; - XLogRecPtr lsn; - XlUndoHeader xlundohdr; - XlUndoHeaderExtra xlundohdrextra; - undo::XlogUndoMeta xlundometa; - OffsetNumber offnum; -}; - -struct deleteUndoParse { - TransactionId recxid; - TransactionId oldxid; - BlockNumber blkno; - Oid spcNode; - Oid relNode; - XLogRecPtr lsn; - XlUndoHeader xlundohdr; - XlUndoHeaderExtra xlundohdrextra; - undo::XlogUndoMeta xlundometa; - UHeapTupleData utup; - OffsetNumber offnum; -}; - -struct updateUndoParse { - bool inplaceUpdate; - TransactionId recxid; - TransactionId oldxid; - Oid spcNode; - Oid relNode; - OffsetNumber new_offnum; - OffsetNumber old_offnum; - XLogRecPtr lsn; - XlUndoHeader xlundohdr; - XlUndoHeaderExtra xlundohdrextra; - XlUndoHeader xlnewundohdr; - XlUndoHeaderExtra xlnewundohdrextra; - undo::XlogUndoMeta xlundometa; - int undoXorDeltaSize; - char *xlogXorDelta; - BlockNumber newblk; - BlockNumber oldblk; -}; - -struct multiInsertUndoParse { - TransactionId recxid; - BlockNumber blkno; - Oid spcNode; - Oid relNode; - XLogRecPtr lsn; - bool isinit; - bool skipUndo; - XlUndoHeader xlundohdr; - XlUndoHeaderExtra xlundohdrextra; - UndoRecPtr last_urecptr; - undo::XlogUndoMeta xlundometa; -}; - -struct rollbackFinishParse { - UndoSlotPtr slotPtr; - XLogRecPtr lsn; -}; - -struct undoDiscardParse { - int zoneId; - UndoSlotPtr endSlot; - UndoSlotPtr startSlot; - UndoRecPtr endUndoPtr; - TransactionId recycledXid; - XLogRecPtr lsn; -}; - -struct undoUnlinkParse { - int zoneId; - UndoLogOffset headOffset; - XLogRecPtr unlinkLsn; -}; - -struct undoExtendParse { - int zoneId; - UndoLogOffset tailOffset; - XLogRecPtr extendLsn; -}; - -struct undoCleanParse { - int zoneId; - UndoLogOffset tailOffset; - XLogRecPtr cleanLsn; -}; - -typedef struct { - char *maindata; - Size recordlen; - union { - struct insertUndoParse insertUndoParse; - struct deleteUndoParse deleteUndoParse; - struct updateUndoParse updateUndoParse; - struct undoDiscardParse undoDiscardParse; - struct undoUnlinkParse undoUnlinkParse; - struct undoExtendParse undoExtendParse; - struct undoCleanParse undoCleanParse; - struct rollbackFinishParse rollbackFinishParse; - struct multiInsertUndoParse multiInsertUndoParse; - }; -} XLogBlockUndoParse; -/* ********BLOCK Undo END ***************** */ - -/* ********BLOCK NewCu BEGIN ***************** */ -typedef struct { - uint32 main_data_len; /* main data portion's length */ - char* main_data; /* point to XLogReaderState's main_data */ -} XLogBlockNewCuParse; - - -/* ********BLOCK NewCu END ***************** */ - -/* ********BLOCK InvalidMsg BEGIN ***************** */ -typedef struct { - TransactionId cutoffxid; -} XLogBlockInvalidParse; - -/* ********BLOCK InvalidMsg END ***************** */ - -/* ********BLOCK Incomplete BEGIN ***************** */ - -typedef enum { - INCOMPLETE_ACTION_LOG = 0, - INCOMPLETE_ACTION_FORGET -} XLogBlockIncompleteEnum; - -typedef struct { - uint16 action; /* split or delete */ - bool issplit; - bool isroot; - BlockNumber downblk; - BlockNumber leftblk; - BlockNumber rightblk; -} XLogBlockIncompleteParse; - -/* ********BLOCK Incomplete END ***************** */ - -/* ********BLOCK VacuumPin BEGIN ***************** */ -typedef struct { - BlockNumber lastBlockVacuumed; -} XLogBlockVacuumPinParse; - -/* ********BLOCK XLOG Common BEGIN ***************** */ -typedef struct { - XLogRecPtr readrecptr; - Size maindatalen; - char* maindata; -} XLogBlockXLogComParse; - -/* ********BLOCK XLOG Common END ***************** */ - -/* ********BLOCK DataBase BEGIN ***************** */ -typedef struct { - Oid src_db_id; - Oid src_tablespace_id; -} XLogBlockDataBaseParse; - -/* ********BLOCK DataBase Common END ***************** */ - -/* ********BLOCK table spc BEGIN ***************** */ -typedef struct { - char* tblPath; - bool isRelativePath; -} XLogBlockTblSpcParse; - -/* ********BLOCK table spc END ***************** */ - -/* ********BLOCK Multi Xact Offset BEGIN ***************** */ -typedef struct { - MultiXactId multi; - MultiXactOffset moffset; -} XLogBlockMultiXactOffParse; - -/* ********BLOCK Multi Xact Offset END ***************** */ - -/* ********BLOCK Multi Xact Mem BEGIN ***************** */ -typedef struct { - MultiXactId multi; - MultiXactOffset startoffset; - uint64 xidnum; - TransactionId xidsarry[MAX_BLOCK_XID_NUMS]; -} XLogBlockMultiXactMemParse; -/* ********BLOCK Multi Xact Mem END ***************** */ - -/* ********BLOCK Multi Xact update oid BEGIN ***************** */ -typedef struct { - MultiXactId nextmulti; - MultiXactOffset nextoffset; - TransactionId maxxid; -} XLogBlockMultiUpdateParse; -/* ********BLOCK Multi Xact update oid END ***************** */ - -/* ********BLOCK rel map BEGIN ***************** */ -typedef struct { - Size maindatalen; - char* maindata; -} XLogBlockRelMapParse; -/* ********BLOCK rel map END ***************** */ - -typedef struct { - uint32 xl_term; -} XLogBlockRedoHead; - -#define XLogRecRedoHeadEncodeSize (offsetof(XLogBlockRedoHead, refrecord)) -typedef struct { - XLogRecPtr start_ptr; - XLogRecPtr end_ptr; /* copy from XLogReaderState's EndRecPtr */ - BlockNumber blkno; - Oid relNode; /* relation */ - uint16 block_valid; /* block data validinfo see XLogBlockInfoEnum */ - uint8 xl_info; /* flag bits, see below */ - RmgrId xl_rmid; /* resource manager for this record */ - ForkNumber forknum; - TransactionId xl_xid; /* xact id */ - Oid spcNode; /* tablespace */ - Oid dbNode; /* database */ - int2 bucketNode; /* bucket */ - uint2 opt; - XLogPhyBlock pblk; -} XLogBlockHead; - -#define XLogBlockHeadEncodeSize (sizeof(XLogBlockHead)) - -#define BYTE_NUM_BITS (8) -#define BYTE_MASK (0xFF) -#define U64_BYTES_NUM (8) -#define U32_BYTES_NUM (4) -#define U16_BYTES_NUM (2) -#define U8_BYTES_NUM (1) - -#define U32_BITS_NUM (BYTE_NUM_BITS * U32_BYTES_NUM) - -extern uint64 XLog_Read_N_Bytes(char* buffer, Size buffersize, Size readbytes); - -#define XLog_Read_1_Bytes(buffer, buffersize) XLog_Read_N_Bytes(buffer, buffersize, U8_BYTES_NUM) -#define XLog_Read_2_Bytes(buffer, buffersize) XLog_Read_N_Bytes(buffer, buffersize, U16_BYTES_NUM) -#define XLog_Read_4_Bytes(buffer, buffersize) XLog_Read_N_Bytes(buffer, buffersize, U32_BYTES_NUM) -#define XLog_Read_8_Bytes(buffer, buffersize) XLog_Read_N_Bytes(buffer, buffersize, U64_BYTES_NUM) - -extern bool XLog_Write_N_bytes(uint64 values, Size writebytes, byte* buffer); - -#define XLog_Write_1_Bytes(values, buffer) XLog_Write_N_bytes(values, U8_BYTES_NUM, buffer) -#define XLog_Write_2_Bytes(values, buffer) XLog_Write_N_bytes(values, U16_BYTES_NUM, buffer) -#define XLog_Write_4_Bytes(values, buffer) XLog_Write_N_bytes(values, U32_BYTES_NUM, buffer) -#define XLog_Write_8_Bytes(values, buffer) XLog_Write_N_bytes(values, U64_BYTES_NUM, buffer) - -typedef struct XLogBlockEnCode { - bool (*xlog_encodefun)(byte* buffer, Size buffersize, Size* encodesize, void* xlogbody); - uint16 block_valid; -} XLogBlockEnCode; - -typedef struct XLogBlockRedoCode { - void (*xlog_redofun)(char* buffer, Size buffersize, XLogBlockHead* blockhead, XLogBlockRedoHead* redohead, - void* page, Size pagesize); - uint16 block_valid; -} XLogBlockRedoCode; - -#pragma pack(pop) - -/* ********BLOCK Xact BEGIN ***************** */ -typedef struct { - uint8 delayddlflag; - uint8 updateminrecovery; - uint16 committype; - int invalidmsgnum; - int nrels; /* delete rels */ - int nlibs; /* delete libs */ - uint64 xinfo; - TimestampTz xact_time; - TransactionId maxxid; - CommitSeqNo maxcommitseq; - void* invalidmsg; - void* xnodes; - void* libfilename; -} XLogBlockXactParse; - -typedef struct { - Size maindatalen; - char* maindata; -} XLogBlockSlotParse; -/* ********BLOCK slot END ***************** */ - -/* ********BLOCK barrier BEGIN ***************** */ -typedef struct { - char* maindata; - Size maindatalen; -} XLogBlockBarrierParse; - -/* ********BLOCK Xact END ***************** */ - -/* ********BLOCK VacuumPin END ***************** */ - -/* ********BLOCK Segfile Extend Begin */ -typedef struct { - BlockNumber target_blocks; -} XLogSegFileExtendParse; -/* ********BLOCK Segfile Extend END */ - -/* ********BLOCK Segment Truncate Begin */ -typedef struct { - XLogBlockDdlParse blockddlrec; - XLogBlockDataParse blockdatarec; -} XLogBlockSegDdlParse; -/* ********BLOCK Segment Truncate END */ - -typedef struct { - void *childState; -} XLogBlockSegFullSyncParse; - -typedef struct { - char *mainData; - Size dataLen; -} XLogBlockSegNewPage; - -typedef struct { - XLogBlockHead blockhead; - XLogBlockRedoHead redohead; - union { - XLogBlockDataParse blockdatarec; - XLogBlockVmParse blockvmrec; - XLogBlockUndoParse blockundorec; - XLogBlockDdlParse blockddlrec; - XLogBlockBcmParse blockbcmrec; - XLogBlockNewCuParse blocknewcu; - XLogBlockCLogParse blockclogrec; - XLogBlockCSNLogParse blockcsnlogrec; - XLogBlockXactParse blockxact; - XLogBlockPrepareParse blockprepare; - XLogBlockInvalidParse blockinvalidmsg; - // XLogBlockIncompleteParse blockincomplete; - XLogBlockVacuumPinParse blockvacuumpin; - XLogBlockXLogComParse blockxlogcommon; - XLogBlockDataBaseParse blockdatabase; - XLogBlockTblSpcParse blocktblspc; - XLogBlockMultiXactOffParse blockmultixactoff; - XLogBlockMultiXactMemParse blockmultixactmem; - XLogBlockMultiUpdateParse blockmultiupdate; - XLogBlockRelMapParse blockrelmap; - XLogBlockSlotParse blockslot; - XLogBlockBarrierParse blockbarrier; - XLogSegFileExtendParse segfileExtend; - XLogBlockSegDdlParse blocksegddlrec; - XLogBlockSegFullSyncParse blocksegfullsyncrec; - XLogBlockSegNewPage blocksegnewpageinfo; - } extra_rec; -} XLogBlockParse; - -#define XLogBlockParseGetDdlParse(blockdatarec, ddlrecparse) \ - do \ - { \ - Assert((blockdatarec)->blockparse.blockhead.block_valid == BLOCK_DATA_DDL_TYPE); \ - if (blockdatarec->blockparse.blockhead.bucketNode != InvalidBktId) { \ - ddlrecparse = &blockdatarec->blockparse.extra_rec.blocksegddlrec.blockddlrec; \ - } else { \ - ddlrecparse = &blockdatarec->blockparse.extra_rec.blockddlrec; \ - } \ - } while (0); - -typedef struct -{ - Buffer buf_id; - Buffer freeNext; -} RedoMemSlot; - -typedef void (*InterruptFunc)(); - -typedef struct -{ - int totalblknum; /* total slot */ - int usedblknum; /* used slot */ - Size itemsize; - Buffer firstfreeslot; /* first free slot */ - Buffer firstreleaseslot; /* first release slot */ - RedoMemSlot *memslot; /* slot itme */ - bool isInit; - InterruptFunc doInterrupt; -}RedoMemManager; - -typedef void (*RefOperateFunc)(void *record); -#ifdef USE_ASSERT_CHECKING -typedef void (*RecordCheckFunc)(void *record, XLogRecPtr curPageLsn, uint32 blockId, bool replayed); -#endif -typedef void (*AddReadBlockFunc)(void *record, uint32 readblocks); - -typedef struct { - RefOperateFunc refCount; - RefOperateFunc DerefCount; -#ifdef USE_ASSERT_CHECKING - RecordCheckFunc checkFunc; -#endif - AddReadBlockFunc addReadBlock; -}RefOperate; - -typedef struct -{ - void *BufferBlockPointers; /* RedoBufferDesc + block */ - RedoMemManager memctl; - RefOperate *refOperate; -}RedoBufferManager; - - - -typedef struct -{ - void *parsebuffers; /* ParseBufferDesc + XLogRecParseState */ - RedoMemManager memctl; - RefOperate *refOperate; -}RedoParseManager; - - - -typedef struct { - void* nextrecord; - XLogBlockParse blockparse; /* block data */ - RedoParseManager* manager; - void* refrecord; /* origin dataptr, for mem release */ - bool isFullSync; -} XLogRecParseState; - -typedef struct XLogBlockRedoExtreRto { - void (*xlog_redoextrto)(XLogBlockHead* blockhead, void* blockrecbody, RedoBufferInfo* bufferinfo); - uint16 block_valid; -} XLogBlockRedoExtreRto; - -typedef struct XLogParseBlock { - XLogRecParseState* (*xlog_parseblock)(XLogReaderState* record, uint32* blocknum); - RmgrId rmid; -} XLogParseBlock; - -typedef enum { - HEAP_INSERT_ORIG_BLOCK_NUM = 0 -} XLogHeapInsertBlockEnum; - -typedef enum { - HEAP_DELETE_ORIG_BLOCK_NUM = 0 -} XLogHeapDeleteBlockEnum; - -typedef enum { - HEAP_UPDATE_NEW_BLOCK_NUM = 0, - HEAP_UPDATE_OLD_BLOCK_NUM -} XLogHeapUpdateBlockEnum; - -typedef enum { - HEAP_BASESHIFT_ORIG_BLOCK_NUM = 0 -} XLogHeapBaeShiftBlockEnum; - -typedef enum { - HEAP_NEWPAGE_ORIG_BLOCK_NUM = 0 -} XLogHeapNewPageBlockEnum; - -typedef enum { - HEAP_LOCK_ORIG_BLOCK_NUM = 0 -} XLogHeapLockBlockEnum; - -typedef enum { - HEAP_INPLACE_ORIG_BLOCK_NUM = 0 -} XLogHeapInplaceBlockEnum; - -typedef enum { - HEAP_FREEZE_ORIG_BLOCK_NUM = 0 -} XLogHeapFreezeBlockEnum; - -typedef enum { - HEAP_CLEAN_ORIG_BLOCK_NUM = 0 -} XLogHeapCleanBlockEnum; - -typedef enum { - HEAP_VISIBLE_VM_BLOCK_NUM = 0, - HEAP_VISIBLE_DATA_BLOCK_NUM -} XLogHeapVisibleBlockEnum; - -typedef enum { - HEAP_MULTI_INSERT_ORIG_BLOCK_NUM = 0 -} XLogHeapMultiInsertBlockEnum; - -typedef enum { - UHEAP_INSERT_ORIG_BLOCK_NUM = 0 -} XLogUHeapInsertBlockEnum; - -typedef enum { - UHEAP_DELETE_ORIG_BLOCK_NUM = 0 -} XLogUHeapDeleteBlockEnum; - -typedef enum { - UHEAP_UPDATE_NEW_BLOCK_NUM = 0, - UHEAP_UPDATE_OLD_BLOCK_NUM -} XLogUHeapUpdateBlockEnum; - -typedef enum { - UHEAP_MULTI_INSERT_ORIG_BLOCK_NUM = 0 -} XLogUHeapMultiInsertBlockEnum; - -typedef enum { - UHEAP_FREEZE_TD_ORIG_BLOCK_NUM = 0 -} XLogUHeapFreezeTDBlockEnum; - -typedef enum { - UHEAP_INVALID_TD_ORIG_BLOCK_NUM = 0 -} XLogUHeapInvalidTDBlockEnum; - -typedef enum { - UHEAP_CLEAN_ORIG_BLOCK_NUM = 0 -} XLogUHeapCleanBlockEnum; - -typedef enum { - UHEAP2_ORIG_BLOCK_NUM = 0 -} XLogUHeap2BlockEnum; - -typedef enum { - UHEAP_UNDO_ORIG_BLOCK_NUM = 0 -} XLogUHeapUndoBlockEnum; - -typedef enum { - UHEAP_UNDOACTION_ORIG_BLOCK_NUM = 0 -} XLogUheapUndoActionBlockEnum; - -extern THR_LOCAL RedoParseManager* g_parseManager; -extern THR_LOCAL RedoBufferManager* g_bufferManager; - -extern void* XLogMemCtlInit(RedoMemManager* memctl, Size itemsize, int itemnum); -extern RedoMemSlot* XLogMemAlloc(RedoMemManager* memctl); -extern void XLogMemRelease(RedoMemManager* memctl, Buffer bufferid); - -extern void XLogRedoBufferInit(RedoBufferManager* buffermanager, int buffernum, RefOperate *refOperate, - InterruptFunc interruptOperte); -extern void XLogRedoBufferDestory(RedoBufferManager* buffermanager); -extern RedoMemSlot* XLogRedoBufferAlloc( - RedoBufferManager* buffermanager, RelFileNode relnode, ForkNumber forkNum, BlockNumber blockNum); -extern bool XLogRedoBufferIsValid(RedoBufferManager* buffermanager, Buffer bufferid); -extern void XLogRedoBufferRelease(RedoBufferManager* buffermanager, Buffer bufferid); -extern BlockNumber XLogRedoBufferGetBlkNumber(RedoBufferManager* buffermanager, Buffer bufferid); -extern Block XLogRedoBufferGetBlk(RedoBufferManager* buffermanager, RedoMemSlot* bufferslot); -extern Block XLogRedoBufferGetPage(RedoBufferManager* buffermanager, Buffer bufferid); -extern void XLogRedoBufferSetState(RedoBufferManager* buffermanager, RedoMemSlot* bufferslot, uint32 state); - -#define XLogRedoBufferInitFunc(bufferManager, buffernum, defOperate, interruptOperte) do { \ - XLogRedoBufferInit(bufferManager, buffernum, defOperate, interruptOperte); \ -} while (0) -#define XLogRedoBufferDestoryFunc(bufferManager) do { \ - XLogRedoBufferDestory(bufferManager); \ -} while (0) -#define XLogRedoBufferAllocFunc(relnode, forkNum, blockNum, bufferslot) do { \ - *bufferslot = XLogRedoBufferAlloc(g_bufferManager, relnode, forkNum, blockNum); \ -} while (0) -#define XLogRedoBufferIsValidFunc(bufferid, isvalid) do { \ - *isvalid = XLogRedoBufferIsValid(g_bufferManager, bufferid); \ -} while (0) -#define XLogRedoBufferReleaseFunc(bufferid) do { \ - XLogRedoBufferRelease(g_bufferManager, bufferid); \ -} while (0) - -#define XLogRedoBufferGetBlkNumberFunc(bufferid, blknumber) do { \ - *blknumber = XLogRedoBufferGetBlkNumber(g_bufferManager, bufferid); \ -} while (0) - -#define XLogRedoBufferGetBlkFunc(bufferslot, blockdata) do { \ - *blockdata = XLogRedoBufferGetBlk(g_bufferManager, bufferslot); \ -} while (0) - -#define XLogRedoBufferGetPageFunc(bufferid, blockdata) do { \ - *blockdata = (Page)XLogRedoBufferGetPage(g_bufferManager, bufferid); \ -} while (0) -#define XLogRedoBufferSetStateFunc(bufferslot, state) do { \ - XLogRedoBufferSetState(g_bufferManager, bufferslot, state); \ -} while (0) - -extern void XLogParseBufferInit(RedoParseManager* parsemanager, int buffernum, RefOperate *refOperate, - InterruptFunc interruptOperte); -extern void XLogParseBufferDestory(RedoParseManager* parsemanager); -extern void XLogParseBufferRelease(XLogRecParseState* recordstate); -extern XLogRecParseState* XLogParseBufferAllocList(RedoParseManager* parsemanager, XLogRecParseState* blkstatehead, void *record); -extern XLogRedoAction XLogReadBufferForRedo(XLogReaderState* record, uint8 buffer_id, RedoBufferInfo* bufferinfo); -extern void XLogInitBufferForRedo(XLogReaderState* record, uint8 block_id, RedoBufferInfo* bufferinfo); -extern XLogRedoAction XLogReadBufferForRedoExtended(XLogReaderState* record, uint8 buffer_id, ReadBufferMode mode, - bool get_cleanup_lock, RedoBufferInfo* bufferinfo, ReadBufferMethod readmethod = WITH_NORMAL_CACHE); -#define XLogParseBufferInitFunc(parseManager, buffernum, defOperate, interruptOperte) do { \ - XLogParseBufferInit(parseManager, buffernum, defOperate, interruptOperte); \ -} while (0) - -#define XLogParseBufferDestoryFunc(parseManager) do { \ - XLogParseBufferDestory(parseManager); \ -} while (0) - -#define XLogParseBufferReleaseFunc(recordstate) do { \ - XLogParseBufferRelease(recordstate); \ -} while (0) - -#define XLogParseBufferAllocListFunc(record, newblkstate, blkstatehead) do { \ - *newblkstate = XLogParseBufferAllocList(g_parseManager, blkstatehead, record); \ -} while (0) - -#define XLogParseBufferAllocListStateFunc(record, newblkstate, blkstatehead) do { \ - if (*blkstatehead == NULL) { \ - *newblkstate = XLogParseBufferAllocList(g_parseManager, NULL, record); \ - *blkstatehead = *newblkstate; \ - } else { \ - *newblkstate = XLogParseBufferAllocList(g_parseManager, *blkstatehead, record); \ - } \ -} while (0) - - - - -#ifdef EXTREME_RTO_DEBUG_AB -typedef void (*AbnormalProcFunc)(void); -typedef enum { - A_THREAD_EXIT, - ALLOC_FAIL, - OPEN_FILE_FAIL, - WAIT_LONG, - ABNORMAL_NUM, -}AbnormalType; -extern AbnormalProcFunc g_AbFunList[ABNORMAL_NUM]; - - -#define ADD_ABNORMAL_POSITION(pos) do { \ - static int __count##pos = 0; \ - __count##pos++; \ - if (g_instance.attr.attr_storage.extreme_rto_ab_pos == pos) { \ - if (g_instance.attr.attr_storage.extreme_rto_ab_count == __count##pos) { \ - ereport(LOG, (errmsg("extreme rto debug abnormal stop pos:%d, type:%d, count:%d", pos, \ - g_instance.attr.attr_storage.extreme_rto_ab_type, __count##pos))); \ - g_AbFunList[g_instance.attr.attr_storage.extreme_rto_ab_type % ABNORMAL_NUM](); \ - } \ - } \ -} while(0) -#else -#define ADD_ABNORMAL_POSITION(pos) -#endif - - - -void HeapXlogCleanOperatorPage( - RedoBufferInfo* buffer, void* recorddata, void* blkdata, Size datalen, Size* freespace, bool repairFragmentation); -void HeapXlogFreezeOperatorPage(RedoBufferInfo* buffer, void* recorddata, void* blkdata, Size datalen, - bool isTupleLockUpgrade); -void HeapXlogVisibleOperatorPage(RedoBufferInfo* buffer, void* recorddata); -void HeapXlogVisibleOperatorVmpage(RedoBufferInfo* vmbuffer, void* recorddata); -void HeapXlogDeleteOperatorPage(RedoBufferInfo* buffer, void* recorddata, TransactionId recordxid, - bool isTupleLockUpgrade); -void HeapXlogInsertOperatorPage(RedoBufferInfo* buffer, void* recorddata, bool isinit, void* blkdata, Size datalen, - TransactionId recxid, Size* freespace, bool tde = false); -void HeapXlogMultiInsertOperatorPage(RedoBufferInfo* buffer, const void* recoreddata, bool isinit, const void* blkdata, - Size len, TransactionId recordxid, Size* freespace, bool tde = false); -void HeapXlogUpdateOperatorOldpage(RedoBufferInfo* buffer, void* recoreddata, bool hot_update, bool isnewinit, - BlockNumber newblk, TransactionId recordxid, bool isTupleLockUpgrade); -void HeapXlogUpdateOperatorNewpage(RedoBufferInfo* buffer, void* recorddata, bool isinit, void* blkdata, - Size datalen, TransactionId recordxid, Size* freespace, bool isTupleLockUpgrade, bool tde = false); -void HeapXlogLockOperatorPage(RedoBufferInfo* buffer, void* recorddata, bool isTupleLockUpgrade); -void HeapXlogInplaceOperatorPage(RedoBufferInfo* buffer, void* recorddata, void* blkdata, Size newlen); -void HeapXlogBaseShiftOperatorPage(RedoBufferInfo* buffer, void* recorddata); - -void BtreeRestorePage(Page page, char* from, int len); -void BtreeXlogMarkDeleteOperatorPage(RedoBufferInfo* buffer, void* recorddata); -void BtreeXlogPrunePageOperatorPage(RedoBufferInfo* buffer, void* recorddata); -void Btree2XlogShiftBaseOperatorPage(RedoBufferInfo* buffer, void* recorddata); - -void BtreeRestoreMetaOperatorPage(RedoBufferInfo* metabuf, void* recorddata, Size datalen); -void BtreeXlogInsertOperatorPage(RedoBufferInfo* buffer, void* recorddata, void* data, Size datalen); -void BtreeXlogSplitOperatorRightpage( - RedoBufferInfo* rbuf, void* recorddata, BlockNumber leftsib, BlockNumber rnext, void* blkdata, Size datalen); -void BtreeXlogSplitOperatorNextpage(RedoBufferInfo* buffer, BlockNumber rightsib); -void BtreeXlogSplitOperatorLeftpage( - RedoBufferInfo* lbuf, void* recorddata, BlockNumber rightsib, bool onleft, void* blkdata, Size datalen); -void BtreeXlogVacuumOperatorPage(RedoBufferInfo* redobuffer, void* recorddata, void* blkdata, Size len); -void BtreeXlogDeleteOperatorPage(RedoBufferInfo* buffer, void* recorddata, Size recorddatalen); -void btreeXlogDeletePageOperatorRightpage(RedoBufferInfo* buffer, void* recorddata); - -void BtreeXlogDeletePageOperatorLeftpage(RedoBufferInfo* buffer, void* recorddata); - -void BtreeXlogDeletePageOperatorCurrentpage(RedoBufferInfo* buffer, void* recorddata); - -void BtreeXlogNewrootOperatorPage(RedoBufferInfo* buffer, void* record, void* blkdata, Size len, BlockNumber* downlink); -void BtreeXlogHalfdeadPageOperatorParentpage( - RedoBufferInfo* pbuf, void* recorddata); -void BtreeXlogHalfdeadPageOperatorLeafpage( - RedoBufferInfo* lbuf, void* recorddata); -void BtreeXlogUnlinkPageOperatorRightpage(RedoBufferInfo* rbuf, void* recorddata); -void BtreeXlogUnlinkPageOperatorLeftpage(RedoBufferInfo* lbuf, void* recorddata); -void BtreeXlogUnlinkPageOperatorCurpage(RedoBufferInfo* buf, void* recorddata); -void BtreeXlogUnlinkPageOperatorChildpage(RedoBufferInfo* cbuf, void* recorddata); - -void BtreeXlogClearIncompleteSplit(RedoBufferInfo* buffer); - -/* UBTree */ -extern void UBTreeRestorePage(Page page, char* from, int len); -extern void UBTreeXlogMarkDeleteOperatorPage(RedoBufferInfo* buffer, void* recorddata); -extern void UBTreeXlogPrunePageOperatorPage(RedoBufferInfo* buffer, void* recorddata); -extern void UBTree2XlogShiftBaseOperatorPage(RedoBufferInfo* buffer, void* recorddata); -extern void UBTree2XlogRecycleQueueInitPageOperatorCurrPage(RedoBufferInfo* buffer, void* recorddata); -extern void UBTree2XlogRecycleQueueInitPageOperatorAdjacentPage(RedoBufferInfo* buffer, void* recorddata, bool isLeft); -extern void UBTree2XlogRecycleQueueEndpointOperatorLeftPage(RedoBufferInfo* buffer, void* recorddata); -extern void UBTree2XlogRecycleQueueEndpointOperatorRightPage(RedoBufferInfo* buffer, void* recorddata); -extern void UBTree2XlogRecycleQueueModifyOperatorPage(RedoBufferInfo* buffer, void* recorddata); -extern void UBTree2XlogFreezeOperatorPage(RedoBufferInfo* buffer, void* recorddata); - -extern void UBTreeRestoreMetaOperatorPage(RedoBufferInfo* metabuf, void* recorddata, Size datalen); -extern void UBTreeXlogInsertOperatorPage(RedoBufferInfo* buffer, void* recorddata, void* data, Size datalen); -extern void UBTreeXlogSplitOperatorRightPage(RedoBufferInfo* rbuf, void* recorddata, BlockNumber leftsib, - BlockNumber rnext, void* blkdata, Size datalen, bool hasOpaque = true); -extern void UBTreeXlogSplitOperatorNextpage(RedoBufferInfo* buffer, BlockNumber rightsib); -extern void UBTreeXlogSplitOperatorLeftpage(RedoBufferInfo* lbuf, void* recorddata, BlockNumber rightsib, - bool onleft, void* blkdata, Size datalen, bool hasOpaque = true); -extern void UBTreeXlogVacuumOperatorPage(RedoBufferInfo* redobuffer, void* recorddata, void* blkdata, Size len); -extern void UBTreeXlogDeleteOperatorPage(RedoBufferInfo* buffer, void* recorddata, Size recorddatalen); -extern void UBTreeXlogDeletePageOperatorRightpage(RedoBufferInfo* buffer, void* recorddata); - -extern void UBTreeXlogDeletePageOperatorLeftpage(RedoBufferInfo* buffer, void* recorddata); - -extern void UBTreeXlogDeletePageOperatorCurrentpage(RedoBufferInfo* buffer, void* recorddata); - -extern void UBTreeXlogNewrootOperatorPage(RedoBufferInfo *buffer, void *record, void *blkdata, Size len, - BlockNumber *downlink); -extern void UBTreeXlogHalfdeadPageOperatorParentpage( - RedoBufferInfo* pbuf, void* recorddata); -extern void UBTreeXlogHalfdeadPageOperatorLeafpage( - RedoBufferInfo* lbuf, void* recorddata); -extern void UBTreeXlogUnlinkPageOperatorRightpage(RedoBufferInfo* rbuf, void* recorddata); -extern void UBTreeXlogUnlinkPageOperatorLeftpage(RedoBufferInfo* lbuf, void* recorddata); -extern void UBTreeXlogUnlinkPageOperatorCurpage(RedoBufferInfo* buf, void* recorddata); -extern void UBTreeXlogUnlinkPageOperatorChildpage(RedoBufferInfo* cbuf, void* recorddata); - -extern void UBTreeXlogClearIncompleteSplit(RedoBufferInfo* buffer); - -void HashRedoInitMetaPageOperatorPage(RedoBufferInfo *metabuf, void *recorddata); - -void HashRedoInitBitmapPageOperatorBitmapPage(RedoBufferInfo *bitmapbuf, void *recorddata); -void HashRedoInitBitmapPageOperatorMetaPage(RedoBufferInfo *metabuf); - -void HashRedoInsertOperatorPage(RedoBufferInfo *buffer, void *recorddata, void *data, Size datalen); -void HashRedoInsertOperatorMetaPage(RedoBufferInfo *metabuf); - -void HashRedoAddOvflPageOperatorOvflPage(RedoBufferInfo *ovflbuf, BlockNumber leftblk, void *data, Size datalen); -void HashRedoAddOvflPageOperatorLeftPage(RedoBufferInfo *ovflbuf, BlockNumber rightblk); -void HashRedoAddOvflPageOperatorMapPage(RedoBufferInfo *mapbuf, void *data); -void HashRedoAddOvflPageOperatorNewmapPage(RedoBufferInfo *newmapbuf, void *recorddata); -void HashRedoAddOvflPageOperatorMetaPage(RedoBufferInfo *metabuf, void *recorddata, void *data, Size datalen); - -void HashRedoSplitAllocatePageOperatorObukPage(RedoBufferInfo *oldbukbuf, void *recorddata); -void HashRedoSplitAllocatePageOperatorNbukPage(RedoBufferInfo *newbukbuf, void *recorddata); -void HashRedoSplitAllocatePageOperatorMetaPage(RedoBufferInfo *metabuf, void *recorddata, void *blkdata); - -void HashRedoSplitCompleteOperatorObukPage(RedoBufferInfo *oldbukbuf, void *recorddata); -void HashRedoSplitCompleteOperatorNbukPage(RedoBufferInfo *newbukbuf, void *recorddata); - -void HashXlogMoveAddPageOperatorPage(RedoBufferInfo *redobuffer, void *recorddata, void *blkdata, Size len); -void HashXlogMoveDeleteOvflPageOperatorPage(RedoBufferInfo *redobuffer, void *blkdata, Size len); - -void HashXlogSqueezeAddPageOperatorPage(RedoBufferInfo *redobuffer, void *recorddata, void *blkdata, Size len); -void HashXlogSqueezeInitOvflbufOperatorPage(RedoBufferInfo *redobuffer, void *recorddata); -void HashXlogSqueezeUpdatePrevPageOperatorPage(RedoBufferInfo *redobuffer, void *recorddata); -void HashXlogSqueezeUpdateNextPageOperatorPage(RedoBufferInfo *redobuffer, void *recorddata); -void HashXlogSqueezeUpdateBitmapOperatorPage(RedoBufferInfo *redobuffer, void *blkdata); -void HashXlogSqueezeUpdateMateOperatorPage(RedoBufferInfo *redobuffer, void *blkdata); - -void HashXlogDeleteBlockOperatorPage(RedoBufferInfo *redobuffer, void *recorddata, void *blkdata, Size len); - -void HashXlogSplitCleanupOperatorPage(RedoBufferInfo *redobuffer); - -void HashXlogUpdateMetaOperatorPage(RedoBufferInfo *redobuffer, void *recorddata); - -void HashXlogVacuumOnePageOperatorPage(RedoBufferInfo *redobuffer, void *recorddata, Size len); - -void HashXlogVacuumMateOperatorPage(RedoBufferInfo *redobuffer, void *recorddata); - -void XLogRecSetBlockCommonState(XLogReaderState* record, XLogBlockParseEnum blockvalid, - RelFileNodeForkNum filenode, XLogRecParseState* recordblockstate, XLogPhyBlock *pblk = NULL); - -void XLogRecSetBlockCLogState( - XLogBlockCLogParse* blockclogstate, TransactionId topxid, uint16 status, uint16 xidnum, uint16* xidsarry); - -void XLogRecSetBlockCSNLogState( - XLogBlockCSNLogParse* blockcsnlogstate, TransactionId topxid, CommitSeqNo csnseq, uint16 xidnum, uint16* xidsarry); -void XLogRecSetXactRecoveryState(XLogBlockXactParse* blockxactstate, TransactionId maxxid, CommitSeqNo maxcsnseq, - uint8 delayddlflag, uint8 updateminrecovery); -void XLogRecSetXactDdlState(XLogBlockXactParse* blockxactstate, int nrels, void* xnodes, int invalidmsgnum, - void* invalidmsg, int nlibs, void* libfilename); -void XLogRecSetXactCommonState( - XLogBlockXactParse* blockxactstate, uint16 committype, uint64 xinfo, TimestampTz xact_time); -void XLogRecSetBcmState(XLogBlockBcmParse* blockbcmrec, uint64 startblock, int count, int status); -void XLogRecSetNewCuState(XLogBlockNewCuParse* blockcudata, char* main_data, uint32 main_data_len); -void XLogRecSetInvalidMsgState(XLogBlockInvalidParse* blockinvalid, TransactionId cutoffxid); -void XLogRecSetIncompleteMsgState(XLogBlockIncompleteParse* blockincomplete, uint16 action, bool issplit, bool isroot, - BlockNumber downblk, BlockNumber leftblk, BlockNumber rightblk); -void XLogRecSetPinVacuumState(XLogBlockVacuumPinParse* blockvacuum, BlockNumber lastblknum); -void XLogRecSetSegFullSyncState(XLogBlockSegFullSyncParse *state, void *childState); -void XLogRecSetSegNewPageInfo(XLogBlockSegNewPage *state, char *mainData, Size len); -void XLogRecSetAuxiBlkNumState(XLogBlockDataParse* blockdatarec, BlockNumber auxilaryblkn1, BlockNumber auxilaryblkn2); -void XLogRecSetBlockDataStateContent(XLogReaderState *record, uint32 blockid, XLogBlockDataParse *blockdatarec); -void XLogRecSetBlockDataState(XLogReaderState* record, uint32 blockid, XLogRecParseState* recordblockstate, - XLogBlockParseEnum type = BLOCK_DATA_MAIN_DATA_TYPE); -extern char* XLogBlockDataGetBlockData(XLogBlockDataParse* datadecode, Size* len); -void Heap2RedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); -extern void HeapRedoDataBlock( - XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); -void SegPageRedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatarec, RedoBufferInfo *bufferinfo); -extern void xlog_redo_data_block( - XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); -extern void XLogRecSetBlockDdlState(XLogBlockDdlParse* blockddlstate, uint32 blockddltype, char *mainData, - int rels = 1); -XLogRedoAction XLogCheckBlockDataRedoAction(XLogBlockDataParse* datadecode, RedoBufferInfo* bufferinfo); - -void BtreeRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); -void Btree2RedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); - -/* UBTree */ -extern void UBTreeRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); -extern void UBTree2RedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatarec, - RedoBufferInfo *bufferinfo); - -extern void HashRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); -XLogRecParseState* XactXlogCsnlogParseToBlock(XLogReaderState* record, uint32* blocknum, TransactionId xid, - int nsubxids, TransactionId* subxids, CommitSeqNo csn, XLogRecParseState* recordstatehead); -extern void XLogRecSetVmBlockState(XLogReaderState* record, uint32 blockid, XLogRecParseState* recordblockstate); -extern void XLogRecSetUHeapUndoBlockState(XLogReaderState* record, uint32 blockid, XLogRecParseState* recordundostate); -extern void XLogRecSetUndoBlockState(XLogReaderState* record, uint32 blockid, XLogRecParseState* recordundostate); -extern void XLogRecSetRollbackFinishBlockState(XLogReaderState *record, uint32 blockid, - XLogRecParseState *recordundostate); -extern bool DoLsnCheck(const RedoBufferInfo* bufferinfo, bool willInit, XLogRecPtr lastLsn, - const XLogPhyBlock *pblk, bool *needRepair); -char* XLogBlockDataGetMainData(XLogBlockDataParse* datadecode, Size* len); -void HeapRedoVmBlock(XLogBlockHead* blockhead, XLogBlockVmParse* blockvmrec, RedoBufferInfo* bufferinfo); -void Heap2RedoVmBlock(XLogBlockHead* blockhead, XLogBlockVmParse* blockvmrec, RedoBufferInfo* bufferinfo); -XLogRecParseState* xlog_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); -XLogRecParseState* smgr_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); -XLogRecParseState* segpage_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); -void ProcSegPageCommonRedo(XLogRecParseState *parseState); -void ProcSegPageJustFreeChildState(XLogRecParseState *parseState); -XLogRecParseState* XactXlogClogParseToBlock(XLogReaderState* record, XLogRecParseState* recordstatehead, - uint32* blocknum, TransactionId xid, int nsubxids, TransactionId* subxids, CLogXidStatus status); -XLogRecParseState* xact_xlog_commit_parse_to_block(XLogReaderState* record, XLogRecParseState* recordstatehead, - uint32* blocknum, TransactionId maxxid, CommitSeqNo maxseqnum); -void visibilitymap_clear_buffer(RedoBufferInfo* bufferinfo, BlockNumber heapBlk); -XLogRecParseState* xact_xlog_abort_parse_to_block(XLogReaderState* record, XLogRecParseState* recordstatehead, - uint32* blocknum, TransactionId maxxid, CommitSeqNo maxseqnum); -XLogRecParseState* xact_xlog_prepare_parse_to_block( - XLogReaderState* record, XLogRecParseState* recordstatehead, uint32* blocknum, TransactionId maxxid); -XLogRecParseState* xact_xlog_parse_to_block(XLogReaderState* record, uint32* blocknum); -XLogRecParseState* ClogRedoParseToBlock(XLogReaderState* record, uint32* blocknum); - -XLogRecParseState* DbaseRedoParseToBlock(XLogReaderState* record, uint32* blocknum); - -XLogRecParseState* Heap2RedoParseIoBlock(XLogReaderState* record, uint32* blocknum); -extern XLogRecParseState* HeapRedoParseToBlock(XLogReaderState* record, uint32* blocknum); -extern XLogRecParseState* BtreeRedoParseToBlock(XLogReaderState* record, uint32* blocknum); -/* UBTree */ -extern XLogRecParseState* UBTreeRedoParseToBlock(XLogReaderState* record, uint32* blocknum); -extern XLogRecParseState* UBTree2RedoParseToBlock(XLogReaderState* record, uint32* blocknum); - -extern XLogRecParseState* Heap3RedoParseToBlock(XLogReaderState* record, uint32* blocknum); - -extern Size SalEncodeXLogBlock(void* recordblockstate, byte* buffer, void* sliceinfo); - -extern XLogRecParseState* XLogParseToBlockForDfv(XLogReaderState* record, uint32* blocknum); -extern Size getBlockSize(XLogRecParseState* recordblockstate); -extern XLogRecParseState* GistRedoParseToBlock(XLogReaderState* record, uint32* blocknum); -extern XLogRecParseState* GinRedoParseToBlock(XLogReaderState* record, uint32* blocknum); - -extern void GistRedoClearFollowRightOperatorPage(RedoBufferInfo* buffer); -extern void GistRedoPageUpdateOperatorPage(RedoBufferInfo* buffer, void* recorddata, void* blkdata, Size datalen); -extern void GistRedoPageSplitOperatorPage( - RedoBufferInfo* buffer, void* recorddata, void* data, Size datalen, bool Markflag, BlockNumber rightlink); -extern void GistRedoCreateIndexOperatorPage(RedoBufferInfo* buffer); - -extern void GinRedoCreateIndexOperatorMetaPage(RedoBufferInfo* MetaBuffer); -extern void GinRedoCreateIndexOperatorRootPage(RedoBufferInfo* RootBuffer); -extern void GinRedoCreatePTreeOperatorPage(RedoBufferInfo* buffer, void* recordData); -extern void GinRedoClearIncompleteSplitOperatorPage(RedoBufferInfo* buffer); -extern void GinRedoVacuumDataOperatorLeafPage(RedoBufferInfo* buffer, void* recorddata); -extern void GinRedoDeletePageOperatorCurPage(RedoBufferInfo* dbuffer); -extern void GinRedoDeletePageOperatorParentPage(RedoBufferInfo* pbuffer, void* recorddata); -extern void GinRedoDeletePageOperatorLeftPage(RedoBufferInfo* lbuffer, void* recorddata); -extern void GinRedoUpdateOperatorMetapage(RedoBufferInfo* metabuffer, void* recorddata); -extern void GinRedoUpdateOperatorTailPage(RedoBufferInfo* buffer, void* payload, Size totaltupsize, int32 ntuples); -extern void GinRedoInsertListPageOperatorPage( - RedoBufferInfo* buffer, void* recorddata, void* payload, Size totaltupsize); -extern void GinRedoUpdateAddNewTail(RedoBufferInfo* buffer, BlockNumber newRightlink); -extern void GinRedoInsertData(RedoBufferInfo* buffer, bool isLeaf, BlockNumber rightblkno, void* rdata); -extern void GinRedoInsertEntry(RedoBufferInfo* buffer, bool isLeaf, BlockNumber rightblkno, void* rdata); - -extern void GinRedoDeleteListPagesOperatorPage(RedoBufferInfo* metabuffer, const void* recorddata); -extern void GinRedoDeleteListPagesMarkDelete(RedoBufferInfo* buffer); - -extern void spgRedoCreateIndexOperatorMetaPage(RedoBufferInfo* buffer); -extern void spgRedoCreateIndexOperatorRootPage(RedoBufferInfo* buffer); -extern void spgRedoCreateIndexOperatorLeafPage(RedoBufferInfo* buffer); -extern void spgRedoAddLeafOperatorPage(RedoBufferInfo* bufferinfo, void* recorddata); -extern void spgRedoAddLeafOperatorParent(RedoBufferInfo* bufferinfo, void* recorddata, BlockNumber blknoLeaf); -extern void spgRedoMoveLeafsOpratorDstPage(RedoBufferInfo* buffer, void* recorddata, void* insertdata, void* tupledata); -extern void spgRedoMoveLeafsOpratorSrcPage( - RedoBufferInfo* buffer, void* recorddata, void* insertdata, void* deletedata, BlockNumber blknoDst, int nInsert); -extern void spgRedoMoveLeafsOpratorParentPage( - RedoBufferInfo* buffer, void* recorddata, void* insertdata, BlockNumber blknoDst, int nInsert); -extern void spgRedoAddNodeUpdateSrcPage(RedoBufferInfo* buffer, void* recorddata, void* tuple, void* tupleheader); -extern void spgRedoAddNodeOperatorSrcPage(RedoBufferInfo* buffer, void* recorddata, BlockNumber blknoNew); -extern void spgRedoAddNodeOperatorDestPage( - RedoBufferInfo* buffer, void* recorddata, void* tuple, void* tupleheader, BlockNumber blknoNew); -extern void spgRedoAddNodeOperatorParentPage(RedoBufferInfo* buffer, void* recorddata, BlockNumber blknoNew); -extern void spgRedoSplitTupleOperatorDestPage(RedoBufferInfo* buffer, void* recorddata, void* tuple); -extern void spgRedoSplitTupleOperatorSrcPage(RedoBufferInfo* buffer, void* recorddata, void* pretuple, void* posttuple); -extern void spgRedoPickSplitRestoreLeafTuples( - RedoBufferInfo* buffer, void* recorddata, bool destflag, void* pageselect, void* insertdata); -extern void spgRedoPickSplitOperatorSrcPage(RedoBufferInfo* srcBuffer, void* recorddata, void* deleteoffset, - BlockNumber blknoInner, void* pageselect, void* insertdata); -extern void spgRedoPickSplitOperatorDestPage( - RedoBufferInfo* destBuffer, void* recorddata, void* pageselect, void* insertdata); -extern void spgRedoPickSplitOperatorInnerPage( - RedoBufferInfo* innerBuffer, void* recorddata, void* tuple, void* tupleheader, BlockNumber blknoInner); -extern void spgRedoPickSplitOperatorParentPage(RedoBufferInfo* parentBuffer, void* recorddata, BlockNumber blknoInner); -extern void spgRedoVacuumLeafOperatorPage(RedoBufferInfo* buffer, void* recorddata); -extern void spgRedoVacuumRootOperatorPage(RedoBufferInfo* buffer, void* recorddata); -extern void spgRedoVacuumRedirectOperatorPage(RedoBufferInfo* buffer, void* recorddata); - -extern XLogRecParseState* SpgRedoParseToBlock(XLogReaderState* record, uint32* blocknum); - -extern void seqRedoOperatorPage(RedoBufferInfo* buffer, void* itmedata, Size itemsz); -extern void seq_redo_data_block(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); - -extern void Heap3RedoDataBlock( - XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); - -extern XLogRecParseState* xact_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); - -extern bool XLogBlockRedoForExtremeRTO(XLogRecParseState* redoblocktate, RedoBufferInfo *bufferinfo, - bool notfound, RedoTimeCost &readBufCost, RedoTimeCost &redoCost); -void XLogBlockParseStateRelease_debug(XLogRecParseState* recordstate, const char *func, uint32 line); -#define XLogBlockParseStateRelease(recordstate) XLogBlockParseStateRelease_debug(recordstate, __FUNCTION__, __LINE__) -#ifdef USE_ASSERT_CHECKING -extern void DoRecordCheck(XLogRecParseState *recordstate, XLogRecPtr pageLsn, bool replayed); -#endif -extern XLogRecParseState* XLogParseBufferCopy(XLogRecParseState *srcState); -extern XLogRecParseState* XLogParseToBlockForExtermeRTO(XLogReaderState* record, uint32* blocknum); -extern XLogRedoAction XLogReadBufferForRedoBlockExtend(RedoBufferTag *redoblock, ReadBufferMode mode, - bool get_cleanup_lock, RedoBufferInfo *redobufferinfo, - XLogRecPtr xloglsn, XLogRecPtr last_lsn, bool willinit, - ReadBufferMethod readmethod, bool tde = false); -extern XLogRecParseState* tblspc_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); -extern XLogRecParseState* relmap_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); -extern XLogRecParseState* HashRedoParseToBlock(XLogReaderState* record, uint32* blocknum); -extern XLogRecParseState* seq_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); -extern XLogRecParseState* slot_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); -extern XLogRecParseState* barrier_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); -extern XLogRecParseState* multixact_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); -extern void ExtremeRtoFlushBuffer(RedoBufferInfo *bufferinfo, bool updateFsm); -extern void XLogForgetDDLRedo(XLogRecParseState* redoblockstate); -void XLogDropSpaceShrink(XLogRecParseState *redoblockstate); -extern void SyncOneBufferForExtremRto(RedoBufferInfo *bufferinfo); -extern void XLogBlockInitRedoBlockInfo(XLogBlockHead* blockhead, RedoBufferTag* blockinfo); -extern void XLogBlockDdlDoSmgrAction(XLogBlockHead* blockhead, void* blockrecbody, RedoBufferInfo* bufferinfo); -extern void XLogBlockSegDdlDoRealAction(XLogBlockHead* blockhead, void* blockrecbody, RedoBufferInfo* bufferinfo); -extern void GinRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); -extern void GistRedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatarec, RedoBufferInfo *bufferinfo); -extern bool IsCheckPoint(const XLogRecParseState *parseState); - -void redo_atomic_xlog_dispatch(uint8 opCode, RedoBufferInfo *redo_buf, const char *data); -void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *data, XLogRecPtr lsn); - -#endif +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * xlogproc.h + * + * + * IDENTIFICATION + * src/include/access/xlogproc.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef XLOG_PROC_H +#define XLOG_PROC_H +#include "postgres.h" +#include "knl/knl_variable.h" + +#include "access/xlogreader.h" +#include "storage/buf/bufmgr.h" +#include "storage/buf/buf_internals.h" +#include "access/xlog_basic.h" +#include "access/xlogutils.h" +#include "access/clog.h" +#include "access/ustore/knl_uredo.h" +#include "access/ustore/knl_utuple.h" +#include "access/ustore/undo/knl_uundotxn.h" +#include "access/ustore/undo/knl_uundoxlog.h" + +#ifndef byte +#define byte unsigned char +#endif + +typedef void (*relasexlogreadstate)(void* record); +/* **************define for parse end******************************* */ +#define MIN(_a, _b) ((_a) > (_b) ? (_b) : (_a)) + +/* for common blockhead begin */ + +#define XLogBlockHeadGetInfo(blockhead) ((blockhead)->xl_info) +#define XLogBlockHeadGetXid(blockhead) ((blockhead)->xl_xid) +#define XLogBlockHeadGetRmid(blockhead) ((blockhead)->xl_rmid) + +#define XLogBlockHeadGetLSN(blockhead) ((blockhead)->end_ptr) +#define XLogBlockHeadGetRelNode(blockhead) ((blockhead)->relNode) +#define XLogBlockHeadGetSpcNode(blockhead) ((blockhead)->spcNode) +#define XLogBlockHeadGetDbNode(blockhead) ((blockhead)->dbNode) +#define XLogBlockHeadGetForkNum(blockhead) ((blockhead)->forknum) +#define XLogBlockHeadGetBlockNum(blockhead) ((blockhead)->blkno) +#define XLogBlockHeadGetBucketId(blockhead) ((blockhead)->bucketNode) +#define XLogBlockHeadGetCompressOpt(blockhead) ((blockhead)->opt) +#define XLogBlockHeadGetValidInfo(blockhead) ((blockhead)->block_valid) +#define XLogBlockHeadGetPhysicalBlock(blockhead) ((blockhead)->pblk) +/* for common blockhead end */ + +/* for block data beging */ +#define XLogBlockDataHasBlockImage(blockdata) ((blockdata)->blockhead.has_image) +#define XLogBlockDataHasBlockData(blockdata) ((blockdata)->blockhead.has_data) +#define XLogBlockDataGetLastBlockLSN(_blockdata) ((_blockdata)->blockdata.last_lsn) +#define XLogBlockDataGetBlockFlags(blockdata) ((blockdata)->blockhead.flags) + +#define XLogBlockDataGetBlockId(blockdata) ((blockdata)->blockhead.cur_block_id) +#define XLogBlockDataGetAuxiBlock1(blockdata) ((blockdata)->blockhead.auxiblk1) +#define XLogBlockDataGetAuxiBlock2(blockdata) ((blockdata)->blockhead.auxiblk2) +/* for block data end */ + +typedef struct { + RelFileNode rnode; + ForkNumber forknum; + BlockNumber blkno; + XLogPhyBlock pblk; +} RedoBufferTag; + +typedef struct { + Page page; // pagepointer + Size pagesize; +#ifdef USE_ASSERT_CHECKING + bool ignorecheck; +#endif +} RedoPageInfo; + +typedef struct { + XLogRecPtr lsn; /* block cur lsn */ + Buffer buf; + RedoBufferTag blockinfo; + RedoPageInfo pageinfo; + int dirtyflag; /* true if the buffer changed */ +} RedoBufferInfo; + +extern void GetFlushBufferInfo(void *buf, RedoBufferInfo *bufferinfo, uint32 *buf_state, ReadBufferMethod flushmethod); + +#define MakeRedoBufferDirty(bufferinfo) ((bufferinfo)->dirtyflag = true) +#define RedoBufferDirtyClear(bufferinfo) ((bufferinfo)->dirtyflag = false) +#define IsRedoBufferDirty(bufferinfo) ((bufferinfo)->dirtyflag == true) + +#define RedoMemIsValid(memctl, bufferid) (((bufferid) > InvalidBuffer) && ((bufferid) <= (memctl->totalblknum))) + +typedef struct { + RedoBufferTag blockinfo; + pg_atomic_uint32 state; +} RedoBufferDesc; + +typedef struct { + Buffer buff_id; + pg_atomic_uint32 state; +} ParseBufferDesc; + +#define RedoBufferSlotGetBuffer(bslot) ((bslot)->buf_id) + +#define EnalbeWalLsnCheck true + +#pragma pack(push, 1) + +#define INVALID_BLOCK_ID (XLR_MAX_BLOCK_ID + 2) + +#define LOW_BLOKNUMBER_BITS (32) +#define LOW_BLOKNUMBER_MASK (((uint64)1 << 32) - 1) + + +/* ********BLOCK COMMON HEADER BEGIN ***************** */ +typedef enum { + BLOCK_DATA_MAIN_DATA_TYPE = 0, /* BLOCK DATA */ + BLOCK_DATA_VM_TYPE, /* VM */ + BLOCK_DATA_UNDO_TYPE, /* UNDO */ + BLOCK_DATA_FSM_TYPE, /* FSM */ + BLOCK_DATA_DDL_TYPE, /* DDL */ + BLOCK_DATA_BCM_TYPE, /* bcm */ + BLOCK_DATA_NEWCU_TYPE, /* cu newlog */ + BLOCK_DATA_CLOG_TYPE, /* CLog */ + BLOCK_DATA_MULITACT_OFF_TYPE, /* MultiXact */ + BLOCK_DATA_MULITACT_MEM_TYPE, + BLOCK_DATA_CSNLOG_TYPE, /* CSNLog */ + /* *****xact don't need sent to dfv */ + BLOCK_DATA_MULITACT_UPDATEOID_TYPE, + BLOCK_DATA_XACTDATA_TYPE, /* XACT */ + BLOCK_DATA_RELMAP_TYPE, /* RELMAP */ + BLOCK_DATA_SLOT_TYPE, + BLOCK_DATA_BARRIER_TYPE, + BLOCK_DATA_PREPARE_TYPE, /* prepare */ + BLOCK_DATA_INVALIDMSG_TYPE, /* INVALIDMSG */ + BLOCK_DATA_INCOMPLETE_TYPE, + BLOCK_DATA_VACUUM_PIN_TYPE, + BLOCK_DATA_XLOG_COMMON_TYPE, + BLOCK_DATA_CREATE_DATABASE_TYPE, + BLOCK_DATA_DROP_DATABASE_TYPE, + BLOCK_DATA_CREATE_TBLSPC_TYPE, + BLOCK_DATA_DROP_TBLSPC_TYPE, + BLOCK_DATA_DROP_SLICE_TYPE, + BLOCK_DATA_SEG_FILE_EXTEND_TYPE, + BLOCK_DATA_SEG_SPACE_DROP, + BLOCK_DATA_SEG_SPACE_SHRINK, + BLOCK_DATA_SEG_FULL_SYNC_TYPE, + BLOCK_DATA_SEG_EXTEND, +} XLogBlockParseEnum; + +/* ********BLOCK COMMON HEADER END ***************** */ + +/* **************define for parse begin ******************************* */ + +/* ********BLOCK DATE BEGIN ***************** */ + +typedef struct { + uint8 cur_block_id; /* blockid */ + uint8 flags; + uint8 has_image; + uint8 has_data; + BlockNumber auxiblk1; + BlockNumber auxiblk2; +} XLogBlocDatakHead; + +#define XLOG_BLOCK_DATAHEAD_LEN sizeof(XLogBlocDatakHead) + +typedef struct { + uint16 extra_flag; + uint16 hole_offset; + uint16 hole_length; /* image position */ + uint16 data_len; /* data length */ + XLogRecPtr last_lsn; + char* bkp_image; + char* data; +} XLogBlockData; + +#define XLOG_BLOCK_DATA_LEN sizeof(XLogBlockData) + +typedef struct { + XLogBlocDatakHead blockhead; + XLogBlockData blockdata; + uint32 main_data_len; /* main data portion's length */ + char* main_data; /* point to XLogReaderState's main_data */ +} XLogBlockDataParse; +/* ********BLOCK DATE END ***************** */ +#define XLOG_BLOCK_DATA_PARSE_LEN sizeof(XLogBlockDataParse) + +/* ********BLOCK DDL BEGIN ***************** */ +typedef enum { + BLOCK_DDL_TYPE_NONE = 0, + BLOCK_DDL_CREATE_RELNODE, + BLOCK_DDL_DROP_RELNODE, + BLOCK_DDL_EXTEND_RELNODE, + BLOCK_DDL_TRUNCATE_RELNODE, + BLOCK_DDL_CLOG_ZERO, + BLOCK_DDL_CLOG_TRUNCATE, + BLOCK_DDL_MULTIXACT_OFF_ZERO, + BLOCK_DDL_MULTIXACT_MEM_ZERO, +} XLogBlockDdlInfoEnum; + +typedef struct { + uint32 blockddltype; + int rels; + char *mainData; +} XLogBlockDdlParse; + +/* ********BLOCK DDL END ***************** */ + +/* ********BLOCK CLOG BEGIN ***************** */ + +#define MAX_BLOCK_XID_NUMS (28) +typedef struct { + TransactionId topxid; + uint16 status; + uint16 xidnum; + uint16 xidsarry[MAX_BLOCK_XID_NUMS]; +} XLogBlockCLogParse; + +/* ********BLOCK CLOG END ***************** */ + +/* ********BLOCK CSNLOG BEGIN ***************** */ +typedef struct { + TransactionId topxid; + CommitSeqNo cslseq; + uint32 xidnum; + uint16 xidsarry[MAX_BLOCK_XID_NUMS]; +} XLogBlockCSNLogParse; + +/* ********BLOCK CSNLOG END ***************** */ + +/* ********BLOCK prepare BEGIN ***************** */ +struct TwoPhaseFileHeader; + +typedef struct { + TransactionId maxxid; + Size maindatalen; + char* maindata; +} XLogBlockPrepareParse; + +/* ********BLOCK prepare END ***************** */ + +/* ********BLOCK Bcm BEGIN ***************** */ +typedef struct { + uint64 startblock; + int count; + int status; +} XLogBlockBcmParse; + +/* ********BLOCK Bcm END ***************** */ + +/* ********BLOCK Vm BEGIN ***************** */ +typedef struct { + BlockNumber heapBlk; +} XLogBlockVmParse; + +#define XLOG_BLOCK_VM_PARSE_LEN sizeof(XLogBlockVmParse) +/* ********BLOCK Vm END ***************** */ + +/* ********BLOCK Undo BEGIN ***************** */ +struct insertUndoParse { + TransactionId recxid; + BlockNumber blkno; + Oid spcNode; + Oid relNode; + XLogRecPtr lsn; + XlUndoHeader xlundohdr; + XlUndoHeaderExtra xlundohdrextra; + undo::XlogUndoMeta xlundometa; + OffsetNumber offnum; +}; + +struct deleteUndoParse { + TransactionId recxid; + TransactionId oldxid; + BlockNumber blkno; + Oid spcNode; + Oid relNode; + XLogRecPtr lsn; + XlUndoHeader xlundohdr; + XlUndoHeaderExtra xlundohdrextra; + undo::XlogUndoMeta xlundometa; + UHeapTupleData utup; + OffsetNumber offnum; +}; + +struct updateUndoParse { + bool inplaceUpdate; + TransactionId recxid; + TransactionId oldxid; + Oid spcNode; + Oid relNode; + OffsetNumber new_offnum; + OffsetNumber old_offnum; + XLogRecPtr lsn; + XlUndoHeader xlundohdr; + XlUndoHeaderExtra xlundohdrextra; + XlUndoHeader xlnewundohdr; + XlUndoHeaderExtra xlnewundohdrextra; + undo::XlogUndoMeta xlundometa; + int undoXorDeltaSize; + char *xlogXorDelta; + BlockNumber newblk; + BlockNumber oldblk; +}; + +struct multiInsertUndoParse { + TransactionId recxid; + BlockNumber blkno; + Oid spcNode; + Oid relNode; + XLogRecPtr lsn; + bool isinit; + bool skipUndo; + XlUndoHeader xlundohdr; + XlUndoHeaderExtra xlundohdrextra; + UndoRecPtr last_urecptr; + undo::XlogUndoMeta xlundometa; +}; + +struct rollbackFinishParse { + UndoSlotPtr slotPtr; + XLogRecPtr lsn; +}; + +struct undoDiscardParse { + int zoneId; + UndoSlotPtr endSlot; + UndoSlotPtr startSlot; + UndoRecPtr endUndoPtr; + TransactionId recycledXid; + XLogRecPtr lsn; +}; + +struct undoUnlinkParse { + int zoneId; + UndoLogOffset headOffset; + XLogRecPtr unlinkLsn; +}; + +struct undoExtendParse { + int zoneId; + UndoLogOffset tailOffset; + XLogRecPtr extendLsn; +}; + +struct undoCleanParse { + int zoneId; + UndoLogOffset tailOffset; + XLogRecPtr cleanLsn; +}; + +typedef struct { + char *maindata; + Size recordlen; + union { + struct insertUndoParse insertUndoParse; + struct deleteUndoParse deleteUndoParse; + struct updateUndoParse updateUndoParse; + struct undoDiscardParse undoDiscardParse; + struct undoUnlinkParse undoUnlinkParse; + struct undoExtendParse undoExtendParse; + struct undoCleanParse undoCleanParse; + struct rollbackFinishParse rollbackFinishParse; + struct multiInsertUndoParse multiInsertUndoParse; + }; +} XLogBlockUndoParse; +/* ********BLOCK Undo END ***************** */ + +/* ********BLOCK NewCu BEGIN ***************** */ +typedef struct { + uint32 main_data_len; /* main data portion's length */ + char* main_data; /* point to XLogReaderState's main_data */ +} XLogBlockNewCuParse; + + +/* ********BLOCK NewCu END ***************** */ + +/* ********BLOCK InvalidMsg BEGIN ***************** */ +typedef struct { + TransactionId cutoffxid; +} XLogBlockInvalidParse; + +/* ********BLOCK InvalidMsg END ***************** */ + +/* ********BLOCK Incomplete BEGIN ***************** */ + +typedef enum { + INCOMPLETE_ACTION_LOG = 0, + INCOMPLETE_ACTION_FORGET +} XLogBlockIncompleteEnum; + +typedef struct { + uint16 action; /* split or delete */ + bool issplit; + bool isroot; + BlockNumber downblk; + BlockNumber leftblk; + BlockNumber rightblk; +} XLogBlockIncompleteParse; + +/* ********BLOCK Incomplete END ***************** */ + +/* ********BLOCK VacuumPin BEGIN ***************** */ +typedef struct { + BlockNumber lastBlockVacuumed; +} XLogBlockVacuumPinParse; + +/* ********BLOCK XLOG Common BEGIN ***************** */ +typedef struct { + XLogRecPtr readrecptr; + Size maindatalen; + char* maindata; +} XLogBlockXLogComParse; + +/* ********BLOCK XLOG Common END ***************** */ + +/* ********BLOCK DataBase BEGIN ***************** */ +typedef struct { + Oid src_db_id; + Oid src_tablespace_id; +} XLogBlockDataBaseParse; + +/* ********BLOCK DataBase Common END ***************** */ + +/* ********BLOCK table spc BEGIN ***************** */ +typedef struct { + char* tblPath; + bool isRelativePath; +} XLogBlockTblSpcParse; + +/* ********BLOCK table spc END ***************** */ + +/* ********BLOCK Multi Xact Offset BEGIN ***************** */ +typedef struct { + MultiXactId multi; + MultiXactOffset moffset; +} XLogBlockMultiXactOffParse; + +/* ********BLOCK Multi Xact Offset END ***************** */ + +/* ********BLOCK Multi Xact Mem BEGIN ***************** */ +typedef struct { + MultiXactId multi; + MultiXactOffset startoffset; + uint64 xidnum; + TransactionId xidsarry[MAX_BLOCK_XID_NUMS]; +} XLogBlockMultiXactMemParse; +/* ********BLOCK Multi Xact Mem END ***************** */ + +/* ********BLOCK Multi Xact update oid BEGIN ***************** */ +typedef struct { + MultiXactId nextmulti; + MultiXactOffset nextoffset; + TransactionId maxxid; +} XLogBlockMultiUpdateParse; +/* ********BLOCK Multi Xact update oid END ***************** */ + +/* ********BLOCK rel map BEGIN ***************** */ +typedef struct { + Size maindatalen; + char* maindata; +} XLogBlockRelMapParse; +/* ********BLOCK rel map END ***************** */ + +typedef struct { + uint32 xl_term; +} XLogBlockRedoHead; + +#define XLogRecRedoHeadEncodeSize (offsetof(XLogBlockRedoHead, refrecord)) +typedef struct { + XLogRecPtr start_ptr; + XLogRecPtr end_ptr; /* copy from XLogReaderState's EndRecPtr */ + BlockNumber blkno; + Oid relNode; /* relation */ + uint16 block_valid; /* block data validinfo see XLogBlockInfoEnum */ + uint8 xl_info; /* flag bits, see below */ + RmgrId xl_rmid; /* resource manager for this record */ + ForkNumber forknum; + TransactionId xl_xid; /* xact id */ + Oid spcNode; /* tablespace */ + Oid dbNode; /* database */ + int2 bucketNode; /* bucket */ + uint2 opt; + XLogPhyBlock pblk; +} XLogBlockHead; + +#define XLogBlockHeadEncodeSize (sizeof(XLogBlockHead)) + +#define BYTE_NUM_BITS (8) +#define BYTE_MASK (0xFF) +#define U64_BYTES_NUM (8) +#define U32_BYTES_NUM (4) +#define U16_BYTES_NUM (2) +#define U8_BYTES_NUM (1) + +#define U32_BITS_NUM (BYTE_NUM_BITS * U32_BYTES_NUM) + +extern uint64 XLog_Read_N_Bytes(char* buffer, Size buffersize, Size readbytes); + +#define XLog_Read_1_Bytes(buffer, buffersize) XLog_Read_N_Bytes(buffer, buffersize, U8_BYTES_NUM) +#define XLog_Read_2_Bytes(buffer, buffersize) XLog_Read_N_Bytes(buffer, buffersize, U16_BYTES_NUM) +#define XLog_Read_4_Bytes(buffer, buffersize) XLog_Read_N_Bytes(buffer, buffersize, U32_BYTES_NUM) +#define XLog_Read_8_Bytes(buffer, buffersize) XLog_Read_N_Bytes(buffer, buffersize, U64_BYTES_NUM) + +extern bool XLog_Write_N_bytes(uint64 values, Size writebytes, byte* buffer); + +#define XLog_Write_1_Bytes(values, buffer) XLog_Write_N_bytes(values, U8_BYTES_NUM, buffer) +#define XLog_Write_2_Bytes(values, buffer) XLog_Write_N_bytes(values, U16_BYTES_NUM, buffer) +#define XLog_Write_4_Bytes(values, buffer) XLog_Write_N_bytes(values, U32_BYTES_NUM, buffer) +#define XLog_Write_8_Bytes(values, buffer) XLog_Write_N_bytes(values, U64_BYTES_NUM, buffer) + +typedef struct XLogBlockEnCode { + bool (*xlog_encodefun)(byte* buffer, Size buffersize, Size* encodesize, void* xlogbody); + uint16 block_valid; +} XLogBlockEnCode; + +typedef struct XLogBlockRedoCode { + void (*xlog_redofun)(char* buffer, Size buffersize, XLogBlockHead* blockhead, XLogBlockRedoHead* redohead, + void* page, Size pagesize); + uint16 block_valid; +} XLogBlockRedoCode; + +#pragma pack(pop) + +/* ********BLOCK Xact BEGIN ***************** */ +typedef struct { + uint8 delayddlflag; + uint8 updateminrecovery; + uint16 committype; + int invalidmsgnum; + int nrels; /* delete rels */ + int nlibs; /* delete libs */ + uint64 xinfo; + TimestampTz xact_time; + TransactionId maxxid; + CommitSeqNo maxcommitseq; + void* invalidmsg; + void* xnodes; + void* libfilename; +} XLogBlockXactParse; + +typedef struct { + Size maindatalen; + char* maindata; +} XLogBlockSlotParse; +/* ********BLOCK slot END ***************** */ + +/* ********BLOCK barrier BEGIN ***************** */ +typedef struct { + char* maindata; + Size maindatalen; +} XLogBlockBarrierParse; + +/* ********BLOCK Xact END ***************** */ + +/* ********BLOCK VacuumPin END ***************** */ + +/* ********BLOCK Segfile Extend Begin */ +typedef struct { + BlockNumber target_blocks; +} XLogSegFileExtendParse; +/* ********BLOCK Segfile Extend END */ + +/* ********BLOCK Segment Truncate Begin */ +typedef struct { + XLogBlockDdlParse blockddlrec; + XLogBlockDataParse blockdatarec; +} XLogBlockSegDdlParse; +/* ********BLOCK Segment Truncate END */ + +typedef struct { + void *childState; +} XLogBlockSegFullSyncParse; + +typedef struct { + char *mainData; + Size dataLen; +} XLogBlockSegNewPage; + +typedef struct { + XLogBlockHead blockhead; + XLogBlockRedoHead redohead; + union { + XLogBlockDataParse blockdatarec; + XLogBlockVmParse blockvmrec; + XLogBlockUndoParse blockundorec; + XLogBlockDdlParse blockddlrec; + XLogBlockBcmParse blockbcmrec; + XLogBlockNewCuParse blocknewcu; + XLogBlockCLogParse blockclogrec; + XLogBlockCSNLogParse blockcsnlogrec; + XLogBlockXactParse blockxact; + XLogBlockPrepareParse blockprepare; + XLogBlockInvalidParse blockinvalidmsg; + // XLogBlockIncompleteParse blockincomplete; + XLogBlockVacuumPinParse blockvacuumpin; + XLogBlockXLogComParse blockxlogcommon; + XLogBlockDataBaseParse blockdatabase; + XLogBlockTblSpcParse blocktblspc; + XLogBlockMultiXactOffParse blockmultixactoff; + XLogBlockMultiXactMemParse blockmultixactmem; + XLogBlockMultiUpdateParse blockmultiupdate; + XLogBlockRelMapParse blockrelmap; + XLogBlockSlotParse blockslot; + XLogBlockBarrierParse blockbarrier; + XLogSegFileExtendParse segfileExtend; + XLogBlockSegDdlParse blocksegddlrec; + XLogBlockSegFullSyncParse blocksegfullsyncrec; + XLogBlockSegNewPage blocksegnewpageinfo; + } extra_rec; +} XLogBlockParse; + +#define XLogBlockParseGetDdlParse(blockdatarec, ddlrecparse) \ + do \ + { \ + Assert((blockdatarec)->blockparse.blockhead.block_valid == BLOCK_DATA_DDL_TYPE); \ + if (blockdatarec->blockparse.blockhead.bucketNode != InvalidBktId) { \ + ddlrecparse = &blockdatarec->blockparse.extra_rec.blocksegddlrec.blockddlrec; \ + } else { \ + ddlrecparse = &blockdatarec->blockparse.extra_rec.blockddlrec; \ + } \ + } while (0); + +typedef struct +{ + Buffer buf_id; + Buffer freeNext; +} RedoMemSlot; + +typedef void (*InterruptFunc)(); + +typedef struct +{ + int totalblknum; /* total slot */ + int usedblknum; /* used slot */ + Size itemsize; + Buffer firstfreeslot; /* first free slot */ + Buffer firstreleaseslot; /* first release slot */ + RedoMemSlot *memslot; /* slot itme */ + bool isInit; + InterruptFunc doInterrupt; +}RedoMemManager; + +typedef void (*RefOperateFunc)(void *record); +#ifdef USE_ASSERT_CHECKING +typedef void (*RecordCheckFunc)(void *record, XLogRecPtr curPageLsn, uint32 blockId, bool replayed); +#endif +typedef void (*AddReadBlockFunc)(void *record, uint32 readblocks); + +typedef struct { + RefOperateFunc refCount; + RefOperateFunc DerefCount; +#ifdef USE_ASSERT_CHECKING + RecordCheckFunc checkFunc; +#endif + AddReadBlockFunc addReadBlock; +}RefOperate; + +typedef struct +{ + void *BufferBlockPointers; /* RedoBufferDesc + block */ + RedoMemManager memctl; + RefOperate *refOperate; +}RedoBufferManager; + + + +typedef struct +{ + void *parsebuffers; /* ParseBufferDesc + XLogRecParseState */ + RedoMemManager memctl; + RefOperate *refOperate; +}RedoParseManager; + + + +typedef struct { + void* nextrecord; + XLogBlockParse blockparse; /* block data */ + RedoParseManager* manager; + void* refrecord; /* origin dataptr, for mem release */ + bool isFullSync; +} XLogRecParseState; + +typedef struct XLogBlockRedoExtreRto { + void (*xlog_redoextrto)(XLogBlockHead* blockhead, void* blockrecbody, RedoBufferInfo* bufferinfo); + uint16 block_valid; +} XLogBlockRedoExtreRto; + +typedef struct XLogParseBlock { + XLogRecParseState* (*xlog_parseblock)(XLogReaderState* record, uint32* blocknum); + RmgrId rmid; +} XLogParseBlock; + +typedef enum { + HEAP_INSERT_ORIG_BLOCK_NUM = 0 +} XLogHeapInsertBlockEnum; + +typedef enum { + HEAP_DELETE_ORIG_BLOCK_NUM = 0 +} XLogHeapDeleteBlockEnum; + +typedef enum { + HEAP_UPDATE_NEW_BLOCK_NUM = 0, + HEAP_UPDATE_OLD_BLOCK_NUM +} XLogHeapUpdateBlockEnum; + +typedef enum { + HEAP_BASESHIFT_ORIG_BLOCK_NUM = 0 +} XLogHeapBaeShiftBlockEnum; + +typedef enum { + HEAP_NEWPAGE_ORIG_BLOCK_NUM = 0 +} XLogHeapNewPageBlockEnum; + +typedef enum { + HEAP_LOCK_ORIG_BLOCK_NUM = 0 +} XLogHeapLockBlockEnum; + +typedef enum { + HEAP_INPLACE_ORIG_BLOCK_NUM = 0 +} XLogHeapInplaceBlockEnum; + +typedef enum { + HEAP_FREEZE_ORIG_BLOCK_NUM = 0 +} XLogHeapFreezeBlockEnum; + +typedef enum { + HEAP_CLEAN_ORIG_BLOCK_NUM = 0 +} XLogHeapCleanBlockEnum; + +typedef enum { + HEAP_VISIBLE_VM_BLOCK_NUM = 0, + HEAP_VISIBLE_DATA_BLOCK_NUM +} XLogHeapVisibleBlockEnum; + +typedef enum { + HEAP_MULTI_INSERT_ORIG_BLOCK_NUM = 0 +} XLogHeapMultiInsertBlockEnum; + +typedef enum { + UHEAP_INSERT_ORIG_BLOCK_NUM = 0 +} XLogUHeapInsertBlockEnum; + +typedef enum { + UHEAP_DELETE_ORIG_BLOCK_NUM = 0 +} XLogUHeapDeleteBlockEnum; + +typedef enum { + UHEAP_UPDATE_NEW_BLOCK_NUM = 0, + UHEAP_UPDATE_OLD_BLOCK_NUM +} XLogUHeapUpdateBlockEnum; + +typedef enum { + UHEAP_MULTI_INSERT_ORIG_BLOCK_NUM = 0 +} XLogUHeapMultiInsertBlockEnum; + +typedef enum { + UHEAP_FREEZE_TD_ORIG_BLOCK_NUM = 0 +} XLogUHeapFreezeTDBlockEnum; + +typedef enum { + UHEAP_INVALID_TD_ORIG_BLOCK_NUM = 0 +} XLogUHeapInvalidTDBlockEnum; + +typedef enum { + UHEAP_CLEAN_ORIG_BLOCK_NUM = 0 +} XLogUHeapCleanBlockEnum; + +typedef enum { + UHEAP2_ORIG_BLOCK_NUM = 0 +} XLogUHeap2BlockEnum; + +typedef enum { + UHEAP_UNDO_ORIG_BLOCK_NUM = 0 +} XLogUHeapUndoBlockEnum; + +typedef enum { + UHEAP_UNDOACTION_ORIG_BLOCK_NUM = 0 +} XLogUheapUndoActionBlockEnum; + +extern THR_LOCAL RedoParseManager* g_parseManager; +extern THR_LOCAL RedoBufferManager* g_bufferManager; + +extern void* XLogMemCtlInit(RedoMemManager* memctl, Size itemsize, int itemnum); +extern RedoMemSlot* XLogMemAlloc(RedoMemManager* memctl); +extern void XLogMemRelease(RedoMemManager* memctl, Buffer bufferid); + +extern void XLogRedoBufferInit(RedoBufferManager* buffermanager, int buffernum, RefOperate *refOperate, + InterruptFunc interruptOperte); +extern void XLogRedoBufferDestory(RedoBufferManager* buffermanager); +extern RedoMemSlot* XLogRedoBufferAlloc( + RedoBufferManager* buffermanager, RelFileNode relnode, ForkNumber forkNum, BlockNumber blockNum); +extern bool XLogRedoBufferIsValid(RedoBufferManager* buffermanager, Buffer bufferid); +extern void XLogRedoBufferRelease(RedoBufferManager* buffermanager, Buffer bufferid); +extern BlockNumber XLogRedoBufferGetBlkNumber(RedoBufferManager* buffermanager, Buffer bufferid); +extern Block XLogRedoBufferGetBlk(RedoBufferManager* buffermanager, RedoMemSlot* bufferslot); +extern Block XLogRedoBufferGetPage(RedoBufferManager* buffermanager, Buffer bufferid); +extern void XLogRedoBufferSetState(RedoBufferManager* buffermanager, RedoMemSlot* bufferslot, uint32 state); + +#define XLogRedoBufferInitFunc(bufferManager, buffernum, defOperate, interruptOperte) do { \ + XLogRedoBufferInit(bufferManager, buffernum, defOperate, interruptOperte); \ +} while (0) +#define XLogRedoBufferDestoryFunc(bufferManager) do { \ + XLogRedoBufferDestory(bufferManager); \ +} while (0) +#define XLogRedoBufferAllocFunc(relnode, forkNum, blockNum, bufferslot) do { \ + *bufferslot = XLogRedoBufferAlloc(g_bufferManager, relnode, forkNum, blockNum); \ +} while (0) +#define XLogRedoBufferIsValidFunc(bufferid, isvalid) do { \ + *isvalid = XLogRedoBufferIsValid(g_bufferManager, bufferid); \ +} while (0) +#define XLogRedoBufferReleaseFunc(bufferid) do { \ + XLogRedoBufferRelease(g_bufferManager, bufferid); \ +} while (0) + +#define XLogRedoBufferGetBlkNumberFunc(bufferid, blknumber) do { \ + *blknumber = XLogRedoBufferGetBlkNumber(g_bufferManager, bufferid); \ +} while (0) + +#define XLogRedoBufferGetBlkFunc(bufferslot, blockdata) do { \ + *blockdata = XLogRedoBufferGetBlk(g_bufferManager, bufferslot); \ +} while (0) + +#define XLogRedoBufferGetPageFunc(bufferid, blockdata) do { \ + *blockdata = (Page)XLogRedoBufferGetPage(g_bufferManager, bufferid); \ +} while (0) +#define XLogRedoBufferSetStateFunc(bufferslot, state) do { \ + XLogRedoBufferSetState(g_bufferManager, bufferslot, state); \ +} while (0) + +extern void XLogParseBufferInit(RedoParseManager* parsemanager, int buffernum, RefOperate *refOperate, + InterruptFunc interruptOperte); +extern void XLogParseBufferDestory(RedoParseManager* parsemanager); +extern void XLogParseBufferRelease(XLogRecParseState* recordstate); +extern XLogRecParseState* XLogParseBufferAllocList(RedoParseManager* parsemanager, XLogRecParseState* blkstatehead, void *record); +extern XLogRedoAction XLogReadBufferForRedo(XLogReaderState* record, uint8 buffer_id, RedoBufferInfo* bufferinfo); +extern void XLogInitBufferForRedo(XLogReaderState* record, uint8 block_id, RedoBufferInfo* bufferinfo); +extern XLogRedoAction XLogReadBufferForRedoExtended(XLogReaderState* record, uint8 buffer_id, ReadBufferMode mode, + bool get_cleanup_lock, RedoBufferInfo* bufferinfo, ReadBufferMethod readmethod = WITH_NORMAL_CACHE); +#define XLogParseBufferInitFunc(parseManager, buffernum, defOperate, interruptOperte) do { \ + XLogParseBufferInit(parseManager, buffernum, defOperate, interruptOperte); \ +} while (0) + +#define XLogParseBufferDestoryFunc(parseManager) do { \ + XLogParseBufferDestory(parseManager); \ +} while (0) + +#define XLogParseBufferReleaseFunc(recordstate) do { \ + XLogParseBufferRelease(recordstate); \ +} while (0) + +#define XLogParseBufferAllocListFunc(record, newblkstate, blkstatehead) do { \ + *newblkstate = XLogParseBufferAllocList(g_parseManager, blkstatehead, record); \ +} while (0) + +#define XLogParseBufferAllocListStateFunc(record, newblkstate, blkstatehead) do { \ + if (*blkstatehead == NULL) { \ + *newblkstate = XLogParseBufferAllocList(g_parseManager, NULL, record); \ + *blkstatehead = *newblkstate; \ + } else { \ + *newblkstate = XLogParseBufferAllocList(g_parseManager, *blkstatehead, record); \ + } \ +} while (0) + + + + +#ifdef EXTREME_RTO_DEBUG_AB +typedef void (*AbnormalProcFunc)(void); +typedef enum { + A_THREAD_EXIT, + ALLOC_FAIL, + OPEN_FILE_FAIL, + WAIT_LONG, + ABNORMAL_NUM, +}AbnormalType; +extern AbnormalProcFunc g_AbFunList[ABNORMAL_NUM]; + + +#define ADD_ABNORMAL_POSITION(pos) do { \ + static int __count##pos = 0; \ + __count##pos++; \ + if (g_instance.attr.attr_storage.extreme_rto_ab_pos == pos) { \ + if (g_instance.attr.attr_storage.extreme_rto_ab_count == __count##pos) { \ + ereport(LOG, (errmsg("extreme rto debug abnormal stop pos:%d, type:%d, count:%d", pos, \ + g_instance.attr.attr_storage.extreme_rto_ab_type, __count##pos))); \ + g_AbFunList[g_instance.attr.attr_storage.extreme_rto_ab_type % ABNORMAL_NUM](); \ + } \ + } \ +} while(0) +#else +#define ADD_ABNORMAL_POSITION(pos) +#endif + + + +void HeapXlogCleanOperatorPage( + RedoBufferInfo* buffer, void* recorddata, void* blkdata, Size datalen, Size* freespace, bool repairFragmentation); +void HeapXlogFreezeOperatorPage(RedoBufferInfo* buffer, void* recorddata, void* blkdata, Size datalen, + bool isTupleLockUpgrade); +void HeapXlogInvalidOperatorPage(RedoBufferInfo* buffer, void* blkdata, Size datalen); +void HeapXlogVisibleOperatorPage(RedoBufferInfo* buffer, void* recorddata); +void HeapXlogVisibleOperatorVmpage(RedoBufferInfo* vmbuffer, void* recorddata); +void HeapXlogDeleteOperatorPage(RedoBufferInfo* buffer, void* recorddata, TransactionId recordxid, + bool isTupleLockUpgrade); +void HeapXlogInsertOperatorPage(RedoBufferInfo* buffer, void* recorddata, bool isinit, void* blkdata, Size datalen, + TransactionId recxid, Size* freespace, bool tde = false); +void HeapXlogMultiInsertOperatorPage(RedoBufferInfo* buffer, const void* recoreddata, bool isinit, const void* blkdata, + Size len, TransactionId recordxid, Size* freespace, bool tde = false); +void HeapXlogUpdateOperatorOldpage(RedoBufferInfo* buffer, void* recoreddata, bool hot_update, bool isnewinit, + BlockNumber newblk, TransactionId recordxid, bool isTupleLockUpgrade); +void HeapXlogUpdateOperatorNewpage(RedoBufferInfo* buffer, void* recorddata, bool isinit, void* blkdata, + Size datalen, TransactionId recordxid, Size* freespace, bool isTupleLockUpgrade, bool tde = false); +void HeapXlogLockOperatorPage(RedoBufferInfo* buffer, void* recorddata, bool isTupleLockUpgrade); +void HeapXlogInplaceOperatorPage(RedoBufferInfo* buffer, void* recorddata, void* blkdata, Size newlen); +void HeapXlogBaseShiftOperatorPage(RedoBufferInfo* buffer, void* recorddata); + +void BtreeRestorePage(Page page, char* from, int len); +void BtreeXlogMarkDeleteOperatorPage(RedoBufferInfo* buffer, void* recorddata); +void BtreeXlogPrunePageOperatorPage(RedoBufferInfo* buffer, void* recorddata); +void Btree2XlogShiftBaseOperatorPage(RedoBufferInfo* buffer, void* recorddata); + +void BtreeRestoreMetaOperatorPage(RedoBufferInfo* metabuf, void* recorddata, Size datalen); +void BtreeXlogInsertOperatorPage(RedoBufferInfo* buffer, void* recorddata, void* data, Size datalen); +void BtreeXlogSplitOperatorRightpage( + RedoBufferInfo* rbuf, void* recorddata, BlockNumber leftsib, BlockNumber rnext, void* blkdata, Size datalen); +void BtreeXlogSplitOperatorNextpage(RedoBufferInfo* buffer, BlockNumber rightsib); +void BtreeXlogSplitOperatorLeftpage( + RedoBufferInfo* lbuf, void* recorddata, BlockNumber rightsib, bool onleft, void* blkdata, Size datalen); +void BtreeXlogVacuumOperatorPage(RedoBufferInfo* redobuffer, void* recorddata, void* blkdata, Size len); +void BtreeXlogDeleteOperatorPage(RedoBufferInfo* buffer, void* recorddata, Size recorddatalen); +void btreeXlogDeletePageOperatorRightpage(RedoBufferInfo* buffer, void* recorddata); + +void BtreeXlogDeletePageOperatorLeftpage(RedoBufferInfo* buffer, void* recorddata); + +void BtreeXlogDeletePageOperatorCurrentpage(RedoBufferInfo* buffer, void* recorddata); + +void BtreeXlogNewrootOperatorPage(RedoBufferInfo* buffer, void* record, void* blkdata, Size len, BlockNumber* downlink); +void BtreeXlogHalfdeadPageOperatorParentpage( + RedoBufferInfo* pbuf, void* recorddata); +void BtreeXlogHalfdeadPageOperatorLeafpage( + RedoBufferInfo* lbuf, void* recorddata); +void BtreeXlogUnlinkPageOperatorRightpage(RedoBufferInfo* rbuf, void* recorddata); +void BtreeXlogUnlinkPageOperatorLeftpage(RedoBufferInfo* lbuf, void* recorddata); +void BtreeXlogUnlinkPageOperatorCurpage(RedoBufferInfo* buf, void* recorddata); +void BtreeXlogUnlinkPageOperatorChildpage(RedoBufferInfo* cbuf, void* recorddata); + +void BtreeXlogClearIncompleteSplit(RedoBufferInfo* buffer); + +/* UBTree */ +extern void UBTreeRestorePage(Page page, char* from, int len); +extern void UBTreeXlogMarkDeleteOperatorPage(RedoBufferInfo* buffer, void* recorddata); +extern void UBTreeXlogPrunePageOperatorPage(RedoBufferInfo* buffer, void* recorddata); +extern void UBTree2XlogShiftBaseOperatorPage(RedoBufferInfo* buffer, void* recorddata); +extern void UBTree2XlogRecycleQueueInitPageOperatorCurrPage(RedoBufferInfo* buffer, void* recorddata); +extern void UBTree2XlogRecycleQueueInitPageOperatorAdjacentPage(RedoBufferInfo* buffer, void* recorddata, bool isLeft); +extern void UBTree2XlogRecycleQueueEndpointOperatorLeftPage(RedoBufferInfo* buffer, void* recorddata); +extern void UBTree2XlogRecycleQueueEndpointOperatorRightPage(RedoBufferInfo* buffer, void* recorddata); +extern void UBTree2XlogRecycleQueueModifyOperatorPage(RedoBufferInfo* buffer, void* recorddata); +extern void UBTree2XlogFreezeOperatorPage(RedoBufferInfo* buffer, void* recorddata); + +extern void UBTreeRestoreMetaOperatorPage(RedoBufferInfo* metabuf, void* recorddata, Size datalen); +extern void UBTreeXlogInsertOperatorPage(RedoBufferInfo* buffer, void* recorddata, void* data, Size datalen); +extern void UBTreeXlogSplitOperatorRightPage(RedoBufferInfo* rbuf, void* recorddata, BlockNumber leftsib, + BlockNumber rnext, void* blkdata, Size datalen, bool hasOpaque = true); +extern void UBTreeXlogSplitOperatorNextpage(RedoBufferInfo* buffer, BlockNumber rightsib); +extern void UBTreeXlogSplitOperatorLeftpage(RedoBufferInfo* lbuf, void* recorddata, BlockNumber rightsib, + bool onleft, void* blkdata, Size datalen, bool hasOpaque = true); +extern void UBTreeXlogVacuumOperatorPage(RedoBufferInfo* redobuffer, void* recorddata, void* blkdata, Size len); +extern void UBTreeXlogDeleteOperatorPage(RedoBufferInfo* buffer, void* recorddata, Size recorddatalen); +extern void UBTreeXlogDeletePageOperatorRightpage(RedoBufferInfo* buffer, void* recorddata); + +extern void UBTreeXlogDeletePageOperatorLeftpage(RedoBufferInfo* buffer, void* recorddata); + +extern void UBTreeXlogDeletePageOperatorCurrentpage(RedoBufferInfo* buffer, void* recorddata); + +extern void UBTreeXlogNewrootOperatorPage(RedoBufferInfo *buffer, void *record, void *blkdata, Size len, + BlockNumber *downlink); +extern void UBTreeXlogHalfdeadPageOperatorParentpage( + RedoBufferInfo* pbuf, void* recorddata); +extern void UBTreeXlogHalfdeadPageOperatorLeafpage( + RedoBufferInfo* lbuf, void* recorddata); +extern void UBTreeXlogUnlinkPageOperatorRightpage(RedoBufferInfo* rbuf, void* recorddata); +extern void UBTreeXlogUnlinkPageOperatorLeftpage(RedoBufferInfo* lbuf, void* recorddata); +extern void UBTreeXlogUnlinkPageOperatorCurpage(RedoBufferInfo* buf, void* recorddata); +extern void UBTreeXlogUnlinkPageOperatorChildpage(RedoBufferInfo* cbuf, void* recorddata); + +extern void UBTreeXlogClearIncompleteSplit(RedoBufferInfo* buffer); + +void HashRedoInitMetaPageOperatorPage(RedoBufferInfo *metabuf, void *recorddata); + +void HashRedoInitBitmapPageOperatorBitmapPage(RedoBufferInfo *bitmapbuf, void *recorddata); +void HashRedoInitBitmapPageOperatorMetaPage(RedoBufferInfo *metabuf); + +void HashRedoInsertOperatorPage(RedoBufferInfo *buffer, void *recorddata, void *data, Size datalen); +void HashRedoInsertOperatorMetaPage(RedoBufferInfo *metabuf); + +void HashRedoAddOvflPageOperatorOvflPage(RedoBufferInfo *ovflbuf, BlockNumber leftblk, void *data, Size datalen); +void HashRedoAddOvflPageOperatorLeftPage(RedoBufferInfo *ovflbuf, BlockNumber rightblk); +void HashRedoAddOvflPageOperatorMapPage(RedoBufferInfo *mapbuf, void *data); +void HashRedoAddOvflPageOperatorNewmapPage(RedoBufferInfo *newmapbuf, void *recorddata); +void HashRedoAddOvflPageOperatorMetaPage(RedoBufferInfo *metabuf, void *recorddata, void *data, Size datalen); + +void HashRedoSplitAllocatePageOperatorObukPage(RedoBufferInfo *oldbukbuf, void *recorddata); +void HashRedoSplitAllocatePageOperatorNbukPage(RedoBufferInfo *newbukbuf, void *recorddata); +void HashRedoSplitAllocatePageOperatorMetaPage(RedoBufferInfo *metabuf, void *recorddata, void *blkdata); + +void HashRedoSplitCompleteOperatorObukPage(RedoBufferInfo *oldbukbuf, void *recorddata); +void HashRedoSplitCompleteOperatorNbukPage(RedoBufferInfo *newbukbuf, void *recorddata); + +void HashXlogMoveAddPageOperatorPage(RedoBufferInfo *redobuffer, void *recorddata, void *blkdata, Size len); +void HashXlogMoveDeleteOvflPageOperatorPage(RedoBufferInfo *redobuffer, void *blkdata, Size len); + +void HashXlogSqueezeAddPageOperatorPage(RedoBufferInfo *redobuffer, void *recorddata, void *blkdata, Size len); +void HashXlogSqueezeInitOvflbufOperatorPage(RedoBufferInfo *redobuffer, void *recorddata); +void HashXlogSqueezeUpdatePrevPageOperatorPage(RedoBufferInfo *redobuffer, void *recorddata); +void HashXlogSqueezeUpdateNextPageOperatorPage(RedoBufferInfo *redobuffer, void *recorddata); +void HashXlogSqueezeUpdateBitmapOperatorPage(RedoBufferInfo *redobuffer, void *blkdata); +void HashXlogSqueezeUpdateMateOperatorPage(RedoBufferInfo *redobuffer, void *blkdata); + +void HashXlogDeleteBlockOperatorPage(RedoBufferInfo *redobuffer, void *recorddata, void *blkdata, Size len); + +void HashXlogSplitCleanupOperatorPage(RedoBufferInfo *redobuffer); + +void HashXlogUpdateMetaOperatorPage(RedoBufferInfo *redobuffer, void *recorddata); + +void HashXlogVacuumOnePageOperatorPage(RedoBufferInfo *redobuffer, void *recorddata, Size len); + +void HashXlogVacuumMateOperatorPage(RedoBufferInfo *redobuffer, void *recorddata); + +void XLogRecSetBlockCommonState(XLogReaderState* record, XLogBlockParseEnum blockvalid, + RelFileNodeForkNum filenode, XLogRecParseState* recordblockstate, XLogPhyBlock *pblk = NULL); + +void XLogRecSetBlockCLogState( + XLogBlockCLogParse* blockclogstate, TransactionId topxid, uint16 status, uint16 xidnum, uint16* xidsarry); + +void XLogRecSetBlockCSNLogState( + XLogBlockCSNLogParse* blockcsnlogstate, TransactionId topxid, CommitSeqNo csnseq, uint16 xidnum, uint16* xidsarry); +void XLogRecSetXactRecoveryState(XLogBlockXactParse* blockxactstate, TransactionId maxxid, CommitSeqNo maxcsnseq, + uint8 delayddlflag, uint8 updateminrecovery); +void XLogRecSetXactDdlState(XLogBlockXactParse* blockxactstate, int nrels, void* xnodes, int invalidmsgnum, + void* invalidmsg, int nlibs, void* libfilename); +void XLogRecSetXactCommonState( + XLogBlockXactParse* blockxactstate, uint16 committype, uint64 xinfo, TimestampTz xact_time); +void XLogRecSetBcmState(XLogBlockBcmParse* blockbcmrec, uint64 startblock, int count, int status); +void XLogRecSetNewCuState(XLogBlockNewCuParse* blockcudata, char* main_data, uint32 main_data_len); +void XLogRecSetInvalidMsgState(XLogBlockInvalidParse* blockinvalid, TransactionId cutoffxid); +void XLogRecSetIncompleteMsgState(XLogBlockIncompleteParse* blockincomplete, uint16 action, bool issplit, bool isroot, + BlockNumber downblk, BlockNumber leftblk, BlockNumber rightblk); +void XLogRecSetPinVacuumState(XLogBlockVacuumPinParse* blockvacuum, BlockNumber lastblknum); +void XLogRecSetSegFullSyncState(XLogBlockSegFullSyncParse *state, void *childState); +void XLogRecSetSegNewPageInfo(XLogBlockSegNewPage *state, char *mainData, Size len); +void XLogRecSetAuxiBlkNumState(XLogBlockDataParse* blockdatarec, BlockNumber auxilaryblkn1, BlockNumber auxilaryblkn2); +void XLogRecSetBlockDataStateContent(XLogReaderState *record, uint32 blockid, XLogBlockDataParse *blockdatarec); +void XLogRecSetBlockDataState(XLogReaderState* record, uint32 blockid, XLogRecParseState* recordblockstate, + XLogBlockParseEnum type = BLOCK_DATA_MAIN_DATA_TYPE); +extern char* XLogBlockDataGetBlockData(XLogBlockDataParse* datadecode, Size* len); +void Heap2RedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); +extern void HeapRedoDataBlock( + XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); +void SegPageRedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatarec, RedoBufferInfo *bufferinfo); +extern void xlog_redo_data_block( + XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); +extern void XLogRecSetBlockDdlState(XLogBlockDdlParse* blockddlstate, uint32 blockddltype, char *mainData, + int rels = 1); +XLogRedoAction XLogCheckBlockDataRedoAction(XLogBlockDataParse* datadecode, RedoBufferInfo* bufferinfo); + +void BtreeRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); +void Btree2RedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); + +/* UBTree */ +extern void UBTreeRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); +extern void UBTree2RedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatarec, + RedoBufferInfo *bufferinfo); + +extern void HashRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); +XLogRecParseState* XactXlogCsnlogParseToBlock(XLogReaderState* record, uint32* blocknum, TransactionId xid, + int nsubxids, TransactionId* subxids, CommitSeqNo csn, XLogRecParseState* recordstatehead); +extern void XLogRecSetVmBlockState(XLogReaderState* record, uint32 blockid, XLogRecParseState* recordblockstate); +extern void XLogRecSetUHeapUndoBlockState(XLogReaderState* record, uint32 blockid, XLogRecParseState* recordundostate); +extern void XLogRecSetUndoBlockState(XLogReaderState* record, uint32 blockid, XLogRecParseState* recordundostate); +extern void XLogRecSetRollbackFinishBlockState(XLogReaderState *record, uint32 blockid, + XLogRecParseState *recordundostate); +extern bool DoLsnCheck(const RedoBufferInfo* bufferinfo, bool willInit, XLogRecPtr lastLsn, + const XLogPhyBlock *pblk, bool *needRepair); +char* XLogBlockDataGetMainData(XLogBlockDataParse* datadecode, Size* len); +void HeapRedoVmBlock(XLogBlockHead* blockhead, XLogBlockVmParse* blockvmrec, RedoBufferInfo* bufferinfo); +void Heap2RedoVmBlock(XLogBlockHead* blockhead, XLogBlockVmParse* blockvmrec, RedoBufferInfo* bufferinfo); +XLogRecParseState* xlog_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); +XLogRecParseState* smgr_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); +XLogRecParseState* segpage_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); +void ProcSegPageCommonRedo(XLogRecParseState *parseState); +void ProcSegPageJustFreeChildState(XLogRecParseState *parseState); +XLogRecParseState* XactXlogClogParseToBlock(XLogReaderState* record, XLogRecParseState* recordstatehead, + uint32* blocknum, TransactionId xid, int nsubxids, TransactionId* subxids, CLogXidStatus status); +XLogRecParseState* xact_xlog_commit_parse_to_block(XLogReaderState* record, XLogRecParseState* recordstatehead, + uint32* blocknum, TransactionId maxxid, CommitSeqNo maxseqnum); +void visibilitymap_clear_buffer(RedoBufferInfo* bufferinfo, BlockNumber heapBlk); +XLogRecParseState* xact_xlog_abort_parse_to_block(XLogReaderState* record, XLogRecParseState* recordstatehead, + uint32* blocknum, TransactionId maxxid, CommitSeqNo maxseqnum); +XLogRecParseState* xact_xlog_prepare_parse_to_block( + XLogReaderState* record, XLogRecParseState* recordstatehead, uint32* blocknum, TransactionId maxxid); +XLogRecParseState* xact_xlog_parse_to_block(XLogReaderState* record, uint32* blocknum); +XLogRecParseState* ClogRedoParseToBlock(XLogReaderState* record, uint32* blocknum); + +XLogRecParseState* DbaseRedoParseToBlock(XLogReaderState* record, uint32* blocknum); + +XLogRecParseState* Heap2RedoParseIoBlock(XLogReaderState* record, uint32* blocknum); +extern XLogRecParseState* HeapRedoParseToBlock(XLogReaderState* record, uint32* blocknum); +extern XLogRecParseState* BtreeRedoParseToBlock(XLogReaderState* record, uint32* blocknum); +/* UBTree */ +extern XLogRecParseState* UBTreeRedoParseToBlock(XLogReaderState* record, uint32* blocknum); +extern XLogRecParseState* UBTree2RedoParseToBlock(XLogReaderState* record, uint32* blocknum); + +extern XLogRecParseState* Heap3RedoParseToBlock(XLogReaderState* record, uint32* blocknum); + +extern Size SalEncodeXLogBlock(void* recordblockstate, byte* buffer, void* sliceinfo); + +extern XLogRecParseState* XLogParseToBlockForDfv(XLogReaderState* record, uint32* blocknum); +extern Size getBlockSize(XLogRecParseState* recordblockstate); +extern XLogRecParseState* GistRedoParseToBlock(XLogReaderState* record, uint32* blocknum); +extern XLogRecParseState* GinRedoParseToBlock(XLogReaderState* record, uint32* blocknum); + +extern void GistRedoClearFollowRightOperatorPage(RedoBufferInfo* buffer); +extern void GistRedoPageUpdateOperatorPage(RedoBufferInfo* buffer, void* recorddata, void* blkdata, Size datalen); +extern void GistRedoPageSplitOperatorPage( + RedoBufferInfo* buffer, void* recorddata, void* data, Size datalen, bool Markflag, BlockNumber rightlink); +extern void GistRedoCreateIndexOperatorPage(RedoBufferInfo* buffer); + +extern void GinRedoCreateIndexOperatorMetaPage(RedoBufferInfo* MetaBuffer); +extern void GinRedoCreateIndexOperatorRootPage(RedoBufferInfo* RootBuffer); +extern void GinRedoCreatePTreeOperatorPage(RedoBufferInfo* buffer, void* recordData); +extern void GinRedoClearIncompleteSplitOperatorPage(RedoBufferInfo* buffer); +extern void GinRedoVacuumDataOperatorLeafPage(RedoBufferInfo* buffer, void* recorddata); +extern void GinRedoDeletePageOperatorCurPage(RedoBufferInfo* dbuffer); +extern void GinRedoDeletePageOperatorParentPage(RedoBufferInfo* pbuffer, void* recorddata); +extern void GinRedoDeletePageOperatorLeftPage(RedoBufferInfo* lbuffer, void* recorddata); +extern void GinRedoUpdateOperatorMetapage(RedoBufferInfo* metabuffer, void* recorddata); +extern void GinRedoUpdateOperatorTailPage(RedoBufferInfo* buffer, void* payload, Size totaltupsize, int32 ntuples); +extern void GinRedoInsertListPageOperatorPage( + RedoBufferInfo* buffer, void* recorddata, void* payload, Size totaltupsize); +extern void GinRedoUpdateAddNewTail(RedoBufferInfo* buffer, BlockNumber newRightlink); +extern void GinRedoInsertData(RedoBufferInfo* buffer, bool isLeaf, BlockNumber rightblkno, void* rdata); +extern void GinRedoInsertEntry(RedoBufferInfo* buffer, bool isLeaf, BlockNumber rightblkno, void* rdata); + +extern void GinRedoDeleteListPagesOperatorPage(RedoBufferInfo* metabuffer, const void* recorddata); +extern void GinRedoDeleteListPagesMarkDelete(RedoBufferInfo* buffer); + +extern void spgRedoCreateIndexOperatorMetaPage(RedoBufferInfo* buffer); +extern void spgRedoCreateIndexOperatorRootPage(RedoBufferInfo* buffer); +extern void spgRedoCreateIndexOperatorLeafPage(RedoBufferInfo* buffer); +extern void spgRedoAddLeafOperatorPage(RedoBufferInfo* bufferinfo, void* recorddata); +extern void spgRedoAddLeafOperatorParent(RedoBufferInfo* bufferinfo, void* recorddata, BlockNumber blknoLeaf); +extern void spgRedoMoveLeafsOpratorDstPage(RedoBufferInfo* buffer, void* recorddata, void* insertdata, void* tupledata); +extern void spgRedoMoveLeafsOpratorSrcPage( + RedoBufferInfo* buffer, void* recorddata, void* insertdata, void* deletedata, BlockNumber blknoDst, int nInsert); +extern void spgRedoMoveLeafsOpratorParentPage( + RedoBufferInfo* buffer, void* recorddata, void* insertdata, BlockNumber blknoDst, int nInsert); +extern void spgRedoAddNodeUpdateSrcPage(RedoBufferInfo* buffer, void* recorddata, void* tuple, void* tupleheader); +extern void spgRedoAddNodeOperatorSrcPage(RedoBufferInfo* buffer, void* recorddata, BlockNumber blknoNew); +extern void spgRedoAddNodeOperatorDestPage( + RedoBufferInfo* buffer, void* recorddata, void* tuple, void* tupleheader, BlockNumber blknoNew); +extern void spgRedoAddNodeOperatorParentPage(RedoBufferInfo* buffer, void* recorddata, BlockNumber blknoNew); +extern void spgRedoSplitTupleOperatorDestPage(RedoBufferInfo* buffer, void* recorddata, void* tuple); +extern void spgRedoSplitTupleOperatorSrcPage(RedoBufferInfo* buffer, void* recorddata, void* pretuple, void* posttuple); +extern void spgRedoPickSplitRestoreLeafTuples( + RedoBufferInfo* buffer, void* recorddata, bool destflag, void* pageselect, void* insertdata); +extern void spgRedoPickSplitOperatorSrcPage(RedoBufferInfo* srcBuffer, void* recorddata, void* deleteoffset, + BlockNumber blknoInner, void* pageselect, void* insertdata); +extern void spgRedoPickSplitOperatorDestPage( + RedoBufferInfo* destBuffer, void* recorddata, void* pageselect, void* insertdata); +extern void spgRedoPickSplitOperatorInnerPage( + RedoBufferInfo* innerBuffer, void* recorddata, void* tuple, void* tupleheader, BlockNumber blknoInner); +extern void spgRedoPickSplitOperatorParentPage(RedoBufferInfo* parentBuffer, void* recorddata, BlockNumber blknoInner); +extern void spgRedoVacuumLeafOperatorPage(RedoBufferInfo* buffer, void* recorddata); +extern void spgRedoVacuumRootOperatorPage(RedoBufferInfo* buffer, void* recorddata); +extern void spgRedoVacuumRedirectOperatorPage(RedoBufferInfo* buffer, void* recorddata); + +extern XLogRecParseState* SpgRedoParseToBlock(XLogReaderState* record, uint32* blocknum); + +extern void seqRedoOperatorPage(RedoBufferInfo* buffer, void* itmedata, Size itemsz); +extern void seq_redo_data_block(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); + +extern void Heap3RedoDataBlock( + XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); + +extern XLogRecParseState* xact_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); + +extern bool XLogBlockRedoForExtremeRTO(XLogRecParseState* redoblocktate, RedoBufferInfo *bufferinfo, + bool notfound, RedoTimeCost &readBufCost, RedoTimeCost &redoCost); +void XLogBlockParseStateRelease_debug(XLogRecParseState* recordstate, const char *func, uint32 line); +#define XLogBlockParseStateRelease(recordstate) XLogBlockParseStateRelease_debug(recordstate, __FUNCTION__, __LINE__) +#ifdef USE_ASSERT_CHECKING +extern void DoRecordCheck(XLogRecParseState *recordstate, XLogRecPtr pageLsn, bool replayed); +#endif +extern XLogRecParseState* XLogParseBufferCopy(XLogRecParseState *srcState); +extern XLogRecParseState* XLogParseToBlockForExtermeRTO(XLogReaderState* record, uint32* blocknum); +extern XLogRedoAction XLogReadBufferForRedoBlockExtend(RedoBufferTag *redoblock, ReadBufferMode mode, + bool get_cleanup_lock, RedoBufferInfo *redobufferinfo, + XLogRecPtr xloglsn, XLogRecPtr last_lsn, bool willinit, + ReadBufferMethod readmethod, bool tde = false); +extern XLogRecParseState* tblspc_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); +extern XLogRecParseState* relmap_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); +extern XLogRecParseState* HashRedoParseToBlock(XLogReaderState* record, uint32* blocknum); +extern XLogRecParseState* seq_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); +extern XLogRecParseState* slot_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); +extern XLogRecParseState* barrier_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); +extern XLogRecParseState* multixact_redo_parse_to_block(XLogReaderState* record, uint32* blocknum); +extern void ExtremeRtoFlushBuffer(RedoBufferInfo *bufferinfo, bool updateFsm); +extern void XLogForgetDDLRedo(XLogRecParseState* redoblockstate); +void XLogDropSpaceShrink(XLogRecParseState *redoblockstate); +extern void SyncOneBufferForExtremRto(RedoBufferInfo *bufferinfo); +extern void XLogBlockInitRedoBlockInfo(XLogBlockHead* blockhead, RedoBufferTag* blockinfo); +extern void XLogBlockDdlDoSmgrAction(XLogBlockHead* blockhead, void* blockrecbody, RedoBufferInfo* bufferinfo); +extern void XLogBlockSegDdlDoRealAction(XLogBlockHead* blockhead, void* blockrecbody, RedoBufferInfo* bufferinfo); +extern void GinRedoDataBlock(XLogBlockHead* blockhead, XLogBlockDataParse* blockdatarec, RedoBufferInfo* bufferinfo); +extern void GistRedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *blockdatarec, RedoBufferInfo *bufferinfo); +extern bool IsCheckPoint(const XLogRecParseState *parseState); + +void redo_atomic_xlog_dispatch(uint8 opCode, RedoBufferInfo *redo_buf, const char *data); +void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *data, XLogRecPtr lsn); + +#endif diff --git a/src/include/catalog/index.h b/src/include/catalog/index.h index 78f7b5e4d..9977af63b 100644 --- a/src/include/catalog/index.h +++ b/src/include/catalog/index.h @@ -240,7 +240,7 @@ extern void AddGPIForSubPartition(Oid partTableOid, Oid partOid, Oid subPartOid) void AddCBIForPartition(Relation partTableRel, Relation tempTableRel, const List* indexRelList, const List* indexDestOidList); extern bool DeleteGPITuplesForPartition(Oid partTableOid, Oid partOid); -extern void DeleteGPITuplesForSubPartition(Oid partTableOid, Oid partOid, Oid subPartOid); +extern bool DeleteGPITuplesForSubPartition(Oid partTableOid, Oid partOid, Oid subPartOid); extern void mergeBTreeIndexes(List* mergingBtreeIndexes, List* srcPartMergeOffset, int2 bktId); extern bool RecheckIndexTuple(IndexScanDesc scan, TupleTableSlot *slot); extern void SetIndexCreateExtraArgs(IndexCreateExtraArgs* extra, Oid psortOid, bool isPartition, bool isGlobal, diff --git a/src/include/catalog/pg_partition_fn.h b/src/include/catalog/pg_partition_fn.h index 92562ebc6..0787a7b73 100644 --- a/src/include/catalog/pg_partition_fn.h +++ b/src/include/catalog/pg_partition_fn.h @@ -40,6 +40,7 @@ #include "access/htup.h" #include "storage/lock/lock.h" #include "access/heapam.h" +#include "storage/lmgr.h" #define MAX_PARTITIONKEY_NUM 4 #define MAX_PARTITION_NUM 1048575 /* update LEN_PARTITION_PREFIX as well ! */ @@ -62,16 +63,12 @@ #define ADD_PARTITION_ACTION (MAX_PARTITION_NUM + 1) /* - * We acquire a AccessExclusiveLock on ADD_PARTITION_ACTION before we decide - * to add an interval partition to prevent parallel complaints. + * We add ADD_PARTITION_ACTION sequence lock to prevent parallel complaints. */ -#define lockRelationForAddIntervalPartition(relation) \ - LockPartition(RelationGetRelid(relation), ADD_PARTITION_ACTION, \ - AccessExclusiveLock, PARTITION_SEQUENCE_LOCK); - -#define unLockRelationForAddIntervalPartition(relation) \ - UnlockPartition(RelationGetRelid(relation), ADD_PARTITION_ACTION, \ - AccessExclusiveLock, PARTITION_SEQUENCE_LOCK); +extern void LockRelationForAddIntervalPartition(Relation rel); +extern void LockRelationForAccessIntervalPartitionTab(Relation rel); +extern void UnlockRelationForAccessIntervalPartTabIfHeld(Relation rel); +extern void UnlockRelationForAddIntervalPartition(Relation rel); typedef void (*PartitionNameGetPartidCallback) (Oid partitioned_relation, const char *partition_name, Oid partId, Oid oldPartId, char partition_type, void *callback_arg, LOCKMODE callbackobj_lockMode); diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback_catalog_maindb_92_604.sql b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback_catalog_maindb_92_604.sql new file mode 100644 index 000000000..c7adf8dbe --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_maindb/rollback_catalog_maindb_92_604.sql @@ -0,0 +1,9 @@ +DROP FUNCTION IF EXISTS pg_catalog.get_client_info; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 7732; +CREATE OR REPLACE FUNCTION pg_catalog.get_client_info() + RETURNS text + LANGUAGE internal + STABLE STRICT NOT FENCED SHIPPABLE +AS $function$get_client_info$function$; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_CATALOG, false, true, 0, 0, 0, 0; diff --git a/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback_catalog_otherdb_92_604.sql b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback_catalog_otherdb_92_604.sql new file mode 100644 index 000000000..c7adf8dbe --- /dev/null +++ b/src/include/catalog/upgrade_sql/rollback_catalog_otherdb/rollback_catalog_otherdb_92_604.sql @@ -0,0 +1,9 @@ +DROP FUNCTION IF EXISTS pg_catalog.get_client_info; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 7732; +CREATE OR REPLACE FUNCTION pg_catalog.get_client_info() + RETURNS text + LANGUAGE internal + STABLE STRICT NOT FENCED SHIPPABLE +AS $function$get_client_info$function$; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_CATALOG, false, true, 0, 0, 0, 0; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_507.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_507.sql index b8ded7d5a..03767e159 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_507.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_507.sql @@ -37,7 +37,29 @@ RETURNS text LANGUAGE INTERNAL STABLE STRICT AS 'sys_connect_by_path'; ------ -- int16 ------ +-- drop operators that depends on int16 first. +do $$ +BEGIN + for ans in select case when count(*) = 1 then true else false end as ans from (select typname from pg_type where typname = 'int16' limit 1) + LOOP + if ans.ans = true then + DROP OPERATOR IF EXISTS pg_catalog.=(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.<>(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.<(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.<=(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.>(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.>=(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.+(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.-(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.*(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog./(int16, int16) CASCADE; + end if; + exit; + END LOOP; +END$$; + DROP TYPE IF EXISTS pg_catalog.int16 CASCADE; +DROP TYPE IF EXISTS pg_catalog._int16 CASCADE; SET LOCAL inplace_upgrade_next_system_object_oids = IUO_TYPE, 34, 0, b; CREATE TYPE pg_catalog.int16; @@ -3292,4 +3314,4 @@ CREATE VIEW pg_catalog.pg_replication_origin_status AS SELECT * FROM pg_show_replication_origin_status(); -REVOKE ALL ON pg_catalog.pg_replication_origin_status FROM public; \ No newline at end of file +REVOKE ALL ON pg_catalog.pg_replication_origin_status FROM public; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_604.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_604.sql index 1de4e7ccf..2ad2cbdec 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_604.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_maindb/upgrade-post_catalog_maindb_92_604.sql @@ -13,3 +13,14 @@ BEGIN end if; END $$; + +DROP FUNCTION IF EXISTS pg_catalog.get_client_info; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 7732; +CREATE OR REPLACE FUNCTION pg_catalog.get_client_info(OUT sid bigint, OUT client_info text) + RETURNS SETOF record + LANGUAGE internal + STABLE NOT FENCED SHIPPABLE ROWS 100 +AS $function$get_client_info$function$; +comment on function PG_CATALOG.get_client_info() is 'read current client'; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_CATALOG, false, true, 0, 0, 0, 0; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_507.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_507.sql index 3665cc78b..fa2e6bb1a 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_507.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_507.sql @@ -37,7 +37,29 @@ RETURNS text LANGUAGE INTERNAL STABLE STRICT AS 'sys_connect_by_path'; ------ -- int16 ------ +-- drop operators that depends on int16 first. +do $$ +BEGIN + for ans in select case when count(*) = 1 then true else false end as ans from (select typname from pg_type where typname = 'int16' limit 1) + LOOP + if ans.ans = true then + DROP OPERATOR IF EXISTS pg_catalog.=(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.<>(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.<(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.<=(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.>(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.>=(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.+(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.-(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog.*(int16, int16) CASCADE; + DROP OPERATOR IF EXISTS pg_catalog./(int16, int16) CASCADE; + end if; + exit; + END LOOP; +END$$; + DROP TYPE IF EXISTS pg_catalog.int16 CASCADE; +DROP TYPE IF EXISTS pg_catalog._int16 CASCADE; SET LOCAL inplace_upgrade_next_system_object_oids = IUO_TYPE, 34, 0, b; CREATE TYPE pg_catalog.int16; @@ -3291,4 +3313,4 @@ CREATE VIEW pg_catalog.pg_replication_origin_status AS SELECT * FROM pg_show_replication_origin_status(); -REVOKE ALL ON pg_catalog.pg_replication_origin_status FROM public; \ No newline at end of file +REVOKE ALL ON pg_catalog.pg_replication_origin_status FROM public; diff --git a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_604.sql b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_604.sql index 1de4e7ccf..2ad2cbdec 100644 --- a/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_604.sql +++ b/src/include/catalog/upgrade_sql/upgrade_catalog_otherdb/upgrade-post_catalog_otherdb_92_604.sql @@ -13,3 +13,14 @@ BEGIN end if; END $$; + +DROP FUNCTION IF EXISTS pg_catalog.get_client_info; +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_PROC, 7732; +CREATE OR REPLACE FUNCTION pg_catalog.get_client_info(OUT sid bigint, OUT client_info text) + RETURNS SETOF record + LANGUAGE internal + STABLE NOT FENCED SHIPPABLE ROWS 100 +AS $function$get_client_info$function$; +comment on function PG_CATALOG.get_client_info() is 'read current client'; + +SET LOCAL inplace_upgrade_next_system_object_oids = IUO_CATALOG, false, true, 0, 0, 0, 0; diff --git a/src/include/client_logic/client_logic.h b/src/include/client_logic/client_logic.h index e10768e27..04c6ab629 100644 --- a/src/include/client_logic/client_logic.h +++ b/src/include/client_logic/client_logic.h @@ -60,4 +60,5 @@ void get_global_setting_description(StringInfo buffer, const ObjectAddress* obje void get_column_setting_description(StringInfo buffer, const ObjectAddress* object); void get_cached_column_description(StringInfo buffer, const ObjectAddress* object); void get_global_setting_args_description(StringInfo buffer, const ObjectAddress* object); -void get_column_setting_args_description(StringInfo buffer, const ObjectAddress* object); \ No newline at end of file +void get_column_setting_args_description(StringInfo buffer, const ObjectAddress* object); +extern Datum get_client_info(PG_FUNCTION_ARGS); \ No newline at end of file diff --git a/src/include/db4ai/db4ai_common.h b/src/include/db4ai/db4ai_common.h index aad1a8eaa..880eb7106 100644 --- a/src/include/db4ai/db4ai_common.h +++ b/src/include/db4ai/db4ai_common.h @@ -25,6 +25,8 @@ #include "utils/builtins.h" #include "utils/timestamp.h" +#define ITER_MAX 10000 +#define MAX_BATCH_SIZE 0x0fffff uint64_t time_diff(struct timespec *time_p1, struct timespec *time_p2); double interval_to_sec(double time_interval); diff --git a/src/include/db4ai/gd.h b/src/include/db4ai/gd.h index 0e80274fe..394f272f6 100644 --- a/src/include/db4ai/gd.h +++ b/src/include/db4ai/gd.h @@ -261,7 +261,7 @@ typedef struct HyperparametersGD { } HyperparametersGD; #define GD_HYPERPARAMETERS_SUPERVISED \ - HYPERPARAMETER_INT4("batch_size", 1000, 1, true, INT32_MAX, true, \ + HYPERPARAMETER_INT4("batch_size", 1000, 1, true, MAX_BATCH_SIZE, true, \ HyperparametersGD, batch_size, \ HP_AUTOML_INT(1, 10000, 4, ProbabilityDistribution::LOG_RANGE)), \ HYPERPARAMETER_FLOAT8("decay", 0.95, 0.0, false, DBL_MAX, true, \ @@ -270,7 +270,7 @@ typedef struct HyperparametersGD { HYPERPARAMETER_FLOAT8("learning_rate", 0.8, 0.0, false, DBL_MAX, true, \ HyperparametersGD, learning_rate, \ HP_AUTOML_FLOAT(1E-6, 1E3, 9, ProbabilityDistribution::LOG_RANGE)), \ - HYPERPARAMETER_INT4("max_iterations", 100, 1, true, INT32_MAX, true, \ + HYPERPARAMETER_INT4("max_iterations", 100, 1, true, ITER_MAX, true, \ HyperparametersGD, max_iterations, \ HP_AUTOML_INT(1, 100, 10, ProbabilityDistribution::LOG_RANGE)), \ HYPERPARAMETER_INT4("max_seconds", 0, 0, true, INT32_MAX, true, \ diff --git a/src/include/executor/executor.h b/src/include/executor/executor.h index 00a4eb98e..bb2f80996 100644 --- a/src/include/executor/executor.h +++ b/src/include/executor/executor.h @@ -280,6 +280,7 @@ extern void initExecTableOfIndexInfo(ExecTableOfIndexInfo* execTableOfIndexInfo, extern bool ExecEvalParamExternTableOfIndexById(ExecTableOfIndexInfo* execTableOfIndexInfo); extern void ExecEvalParamExternTableOfIndex(Node* node, ExecTableOfIndexInfo* execTableOfIndexInfo); extern bool is_external_clob(Oid type_oid, bool is_null, Datum value); +extern bool is_huge_clob(Oid type_oid, bool is_null, Datum value); /* * prototypes from functions in execTuples.c diff --git a/src/include/knl/knl_instance.h b/src/include/knl/knl_instance.h index f6688ab87..237a0e25c 100755 --- a/src/include/knl/knl_instance.h +++ b/src/include/knl/knl_instance.h @@ -458,8 +458,8 @@ typedef struct knl_g_audit_context { /* audit logs */ char index_file_path[MAXPGPATH]; LWLock *index_file_lock; - struct AuditIndexTable* audit_indextbl; - struct AuditIndexTableOld* audit_indextbl_old; + struct AuditIndexTableNew* audit_indextbl; + struct AuditIndexTable* audit_indextbl_old; uint64 pgaudit_totalspace; /* audit thread */ @@ -1061,6 +1061,7 @@ typedef struct knl_g_roach_context { char* targetTimeInPITR; char* globalBarrierRecordForPITR; bool isXLogForceRecycled; + bool forceAdvanceSlotTigger; bool isGtmFreeCsn; char* targetRestoreTimeFromMedia; } knl_g_roach_context; diff --git a/src/include/knl/knl_session.h b/src/include/knl/knl_session.h index 636706693..766c6179b 100644 --- a/src/include/knl/knl_session.h +++ b/src/include/knl/knl_session.h @@ -417,6 +417,8 @@ typedef struct knl_u_parser_context { /* this is used in parser.gram.y not in plpgsql */ bool isCreateFuncOrProc; + + bool isTimeCapsule; } knl_u_parser_context; typedef struct knl_u_trigger_context { @@ -562,7 +564,7 @@ typedef struct knl_u_utils_context { * for the convenience of TransactionIdIsInProgress: even in bootstrap * mode, we don't want it to say that BootstrapTransactionId is in progress. * - * RecentGlobalXmin and RecentGlobalDataXmin are initialized to + * RecentGlobalXmin, RecentGlobalDataXmin, RecentGlobalCatalogXmin are initialized to * InvalidTransactionId, to ensure that no one tries to use a stale * value. Readers should ensure that it has been set to something else * before using it. @@ -577,6 +579,9 @@ typedef struct knl_u_utils_context { TransactionId RecentGlobalDataXmin; + /* recent global catalog xmin, consider replication slot catalog xmin */ + TransactionId RecentGlobalCatalogXmin; + /* Global snapshot data */ bool cn_xc_maintain_mode; int snapshot_source; @@ -1537,6 +1542,7 @@ typedef struct knl_u_plpgsql_context { bool is_delete_function; bool is_package_instantiation; char* client_info; + pthread_mutex_t client_info_lock; char sess_cxt_name[128]; HTAB* sess_cxt_htab; bool have_error; @@ -1582,10 +1588,12 @@ typedef struct knl_u_plpgsql_context { uint64 parent_session_id; ThreadId parent_thread_id; MemoryContext parent_context; /* parent_context from parent session */ + Oid ActiveLobToastOid; struct ExceptionContext* cur_exception_cxt; bool pragma_autonomous; /* save autonomous flag */ char* debug_query_string; + bool is_insert_gs_source; /* is doing insert gs_source? */ } knl_u_plpgsql_context; //this is used to define functions in package @@ -1871,7 +1879,7 @@ typedef struct knl_u_relcache_context { /* * This flag lets us optimize away work in AtEO(Sub)Xact_RelationCache(). */ - bool need_eoxact_work; + bool RelCacheNeedEOXActWork; HTAB* OpClassCache; @@ -2301,7 +2309,7 @@ typedef struct knl_u_cache_context { struct HTAB* dn_hash_table; - bool part_cache_need_eoxact_work; + bool PartCacheNeedEOXActWork; bool bucket_cache_need_eoxact_work; diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h old mode 100644 new mode 100755 index 58e5847c2..3f7a0c9d8 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -187,7 +187,7 @@ typedef enum { } TimeCostPosition; /* -for extreme rto +for extreme rto thread step1 step2 step3 step4 step5 step6 step7 step8 redo batch get a record redo record(total) update stanbystate parse xlog @@ -733,7 +733,7 @@ typedef struct knl_t_xlog_context { struct HTAB* invalid_page_tab; struct HTAB* remain_segs; - + /* state maintained across calls */ uint32 sendId; int sendFile; @@ -926,7 +926,7 @@ typedef struct knl_t_shemem_ptr_context { */ union LWLockPadded *mainLWLockArray; - // for GTT table to track sessions and their usage of GTTs + // for GTT table to track sessions and their usage of GTTs struct gtt_ctl_data* gtt_shared_ctl; struct HTAB* active_gtt_shared_hash; @@ -1228,7 +1228,7 @@ typedef struct knl_t_audit_context { FILE *policyauditFile; Latch sysAuditorLatch; time_t last_pgaudit_start_time; - struct AuditIndexTable* audit_indextbl; + struct AuditIndexTableNew* audit_indextbl; char pgaudit_filepath[MAXPGPATH]; int cur_thread_idx; @@ -1567,6 +1567,7 @@ typedef struct knl_t_pagewriter_context { volatile sig_atomic_t got_SIGHUP; volatile sig_atomic_t shutdown_requested; volatile sig_atomic_t sync_requested; + volatile sig_atomic_t sync_retry; int page_writer_after; int pagewriter_id; uint64 next_flush_time; @@ -1813,7 +1814,7 @@ typedef struct knl_t_utils_context { bool gs_mp_inited; /* Memory Protecting need flag */ - bool memNeedProtect; + bool memNeedProtect; /* Track memory usage in chunks at individual thread level */ int32 trackedMemChunks; @@ -2724,7 +2725,7 @@ typedef struct knl_t_storage_context { struct HTAB* DataFileIdCache; /* Thread shared Seg Spc cache */ struct HTAB* SegSpcCache; - + struct HTAB* uidHashCache; struct HTAB* DisasterCache; /* @@ -3258,7 +3259,7 @@ typedef struct knl_thrd_context { MemoryContext top_mem_cxt; MemoryContextGroup* mcxt_group; knl_t_lsc_context lsc_cxt; - + /* variables to support comm proxy */ CommSocketOption comm_sock_option; CommEpollOption comm_epoll_option; diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 206f1abf4..6c43458e3 100644 --- a/src/include/mb/pg_wchar.h +++ b/src/include/mb/pg_wchar.h @@ -442,7 +442,7 @@ extern int pg_mic_mblen(const unsigned char* mbstr); extern int pg_mbstrlen(const char* mbstr); extern int pg_mbstrlen_with_len(const char* mbstr, int len); extern int pg_mbstrlen_with_len_eml(const char* mbstr, int len, int eml); -extern int pg_mbstrlen_with_len_toast(const char* mbstr, int* len); +extern int pg_mbstrlen_with_len_toast(const char* mbstr, int* limit); extern int pg_mbcliplen(const char* mbstr, int len, int limit); extern int pg_encoding_mbcliplen(int encoding, const char* mbstr, int len, int limit); extern int pg_mbcharcliplen(const char* mbstr, int len, int imit); diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index f2f96ee14..f02eca6b4 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h @@ -44,6 +44,7 @@ extern const uint32 PREDPUSH_SAME_LEVEL_VERSION_NUM; extern const uint32 UPSERT_WHERE_VERSION_NUM; extern const uint32 FUNC_PARAM_COL_VERSION_NUM; extern const uint32 SUBPARTITION_VERSION_NUM; +extern const uint32 PBESINGLEPARTITION_VERSION_NUM; extern const uint32 COMMENT_PROC_VERSION_NUM; extern const uint32 COMMENT_ROWTYPE_TABLEOF_VERSION_NUM; extern const uint32 COMMENT_PCT_TYPE_VERSION_NUM; @@ -79,6 +80,7 @@ extern const uint32 TWOPHASE_FILE_VERSION; extern const uint32 CLIENT_ENCRYPTION_PROC_VERSION_NUM; extern const uint32 PRIVS_DIRECTORY_VERSION_NUM; extern const uint32 COMMENT_RECORD_PARAM_VERSION_NUM; +extern const uint32 INVALID_INVISIBLE_TUPLE_VERSION; extern const uint32 ENHANCED_TUPLE_LOCK_VERSION_NUM; extern const uint32 HASUID_VERSION_NUM; extern const uint32 CREATE_INDEX_CONCURRENTLY_DIST_VERSION_NUM; @@ -89,6 +91,7 @@ extern const uint32 SCAN_BATCH_MODE_VERSION_NUM; extern const uint32 PUBLICATION_VERSION_NUM; extern const uint32 ANALYZER_HOOK_VERSION_NUM; extern const uint32 SUPPORT_HASH_XLOG_VERSION_NUM; +extern const uint32 PITR_INIT_VERSION_NUM; extern void register_backend_version(uint32 backend_version); extern bool contain_backend_version(uint32 version_number); diff --git a/src/include/replication/logical.h b/src/include/replication/logical.h index 5b0d34757..a7448d56d 100644 --- a/src/include/replication/logical.h +++ b/src/include/replication/logical.h @@ -45,6 +45,9 @@ /* Maximum number of max replication slots */ #define MAX_REPLICATION_SLOT_NUM 100 +/* FLag and mask for TOAST in parallel decoding */ +#define TOAST_FLAG ((uint32)1 << 31) +#define TOAST_MASK (((uint32)1 << 31) - 1) typedef void (*LogicalOutputPluginWriterWrite)( struct LogicalDecodingContext* lr, XLogRecPtr Ptr, TransactionId xid, bool last_write); diff --git a/src/include/replication/parallel_reorderbuffer.h b/src/include/replication/parallel_reorderbuffer.h index 3a69b3b20..866b53cf5 100644 --- a/src/include/replication/parallel_reorderbuffer.h +++ b/src/include/replication/parallel_reorderbuffer.h @@ -340,12 +340,15 @@ struct ParallelReorderBuffer { * ontop of reorderbuffer.c */ + /* cached ParallelReorderBufferTXNs */ + dlist_head cached_transactions; + Size nr_cached_transactions; - /* cached ReorderBufferChanges */ + /* cached ParallelReorderBufferChanges */ dlist_head cached_changes; Size nr_cached_changes; - /* cached ReorderBufferTupleBufs */ + /* cached ParallelReorderBufferTupleBufs */ slist_head cached_tuplebufs; Size nr_cached_tuplebufs; TransactionId lastRunningXactOldestXmin; diff --git a/src/include/replication/replicainternal.h b/src/include/replication/replicainternal.h index 46df4f0ce..bd8d70f0f 100755 --- a/src/include/replication/replicainternal.h +++ b/src/include/replication/replicainternal.h @@ -191,7 +191,9 @@ typedef enum { extern bool data_catchup; extern bool wal_catchup; extern BuildMode build_mode; +extern bool is_cross_region_build; /* for stream disaster recovery cluster */ #define IS_CROSS_CLUSTER_BUILD (build_mode == CROSS_CLUSTER_FULL_BUILD || \ build_mode == CROSS_CLUSTER_INC_BUILD || \ - build_mode == CROSS_CLUSTER_STANDBY_FULL_BUILD) + build_mode == CROSS_CLUSTER_STANDBY_FULL_BUILD || \ + is_cross_region_build) #endif /* _REPLICA_INTERNAL_H */ diff --git a/src/include/storage/buf/bufmgr.h b/src/include/storage/buf/bufmgr.h index 9898aca37..a2d1f3f3c 100644 --- a/src/include/storage/buf/bufmgr.h +++ b/src/include/storage/buf/bufmgr.h @@ -321,6 +321,7 @@ extern void MarkBufferDirtyHint(Buffer buffer, bool buffer_std); extern void FlushOneBuffer(Buffer buffer); extern void UnlockBuffers(void); extern void LockBuffer(Buffer buffer, int mode); +extern bool TryLockBuffer(Buffer buffer, int mode, bool must_wait); extern bool ConditionalLockBuffer(Buffer buffer); extern void LockBufferForCleanup(Buffer buffer); extern bool ConditionalLockBufferForCleanup(Buffer buffer); diff --git a/src/include/storage/lmgr.h b/src/include/storage/lmgr.h index 3dd4b8919..288e3a6d7 100644 --- a/src/include/storage/lmgr.h +++ b/src/include/storage/lmgr.h @@ -98,6 +98,7 @@ extern void UnlockPartitionOid(Oid relid, uint32 seq, LOCKMODE lockmode); extern void LockPartitionSeq(Oid relid, uint32 seq, LOCKMODE lockmode); extern bool ConditionalLockPartitionSeq(Oid relid, uint32 seq, LOCKMODE lockmode); extern void UnlockPartitionSeq(Oid relid, uint32 seq, LOCKMODE lockmode); +extern void UnlockPartitionSeqIfHeld(Oid relid, uint32 seq, LOCKMODE lockmode); extern bool ConditionalLockPartitionWithRetry(Relation relation, Oid partitionId, LOCKMODE lockmode); diff --git a/src/include/storage/lock/lock.h b/src/include/storage/lock/lock.h index 3b6f3510b..eac174006 100644 --- a/src/include/storage/lock/lock.h +++ b/src/include/storage/lock/lock.h @@ -660,6 +660,7 @@ extern LockAcquireResult LockAcquireExtended(const LOCKTAG *locktag, LOCKMODE lo bool report_memory_error, bool allow_con_update = false, int waitSec = 0); extern void AbortStrongLockAcquire(void); extern bool LockRelease(const LOCKTAG* locktag, LOCKMODE lockmode, bool sessionLock); +extern void ReleaseLockIfHeld(const LOCKTAG *locktag, LOCKMODE lockmode, bool sessionLock); extern void LockReleaseAll(LOCKMETHODID lockmethodid, bool allLocks); extern void Check_FastpathBit(); diff --git a/src/include/storage/procarray.h b/src/include/storage/procarray.h index 2b8c2f314..151b5b5d5 100755 --- a/src/include/storage/procarray.h +++ b/src/include/storage/procarray.h @@ -74,6 +74,7 @@ extern bool ProcArrayInstallImportedXmin(TransactionId xmin, TransactionId sourc extern RunningTransactions GetRunningTransactionData(void); extern bool TransactionIdIsActive(TransactionId xid); +extern TransactionId GetOldestCatalogXmin(); extern TransactionId GetRecentGlobalXmin(void); extern TransactionId GetOldestXmin(Relation rel, bool bFixRecentGlobalXmin = false, bool bRecentGlobalXminNoCheck = false); diff --git a/src/include/threadpool/threadpool_sessctl.h b/src/include/threadpool/threadpool_sessctl.h index 943e246f0..15c1d3eff 100644 --- a/src/include/threadpool/threadpool_sessctl.h +++ b/src/include/threadpool/threadpool_sessctl.h @@ -62,6 +62,7 @@ public: void CheckSessionTimeout(); void CheckPermissionForSendSignal(knl_session_context* sess, sig_atomic_t* lock); void getSessionMemoryDetail(Tuplestorestate* tupStore, TupleDesc tupDesc, knl_sess_control** sess); + void getSessionClientInfo(Tuplestorestate* tupStore, TupleDesc tupDesc); void getSessionMemoryContextInfo(const char* ctx_name, StringInfoData* buf, knl_sess_control** sess); knl_session_context* GetSessionByIdx(int idx); int FindCtrlIdxBySessId(uint64 id); @@ -78,6 +79,8 @@ private: knl_session_context* sess, const MemoryContext context, Tuplestorestate* tupStore, TupleDesc tupDesc); void calculateSessMemCxtStats( knl_session_context* sess, const MemoryContext context, Tuplestorestate* tupStore, TupleDesc tupDesc); + void calculateClientInfo( + knl_session_context* sess, Tuplestorestate* tupStore, TupleDesc tupDesc); inline bool IsValidCtrlIndex(int ctrl_index) { diff --git a/src/include/utils/knl_localpartdefcache.h b/src/include/utils/knl_localpartdefcache.h index a37b2f94e..990c2dc4a 100644 --- a/src/include/utils/knl_localpartdefcache.h +++ b/src/include/utils/knl_localpartdefcache.h @@ -39,7 +39,7 @@ public: m_bucket_list.ResetContent(); invalid_entries.ResetInitFlag(); m_global_partdefcache = NULL; - part_cache_need_eoxact_work = false; + PartCacheNeedEOXActWork = false; m_is_inited = false; m_db_id = InvalidOid; } @@ -64,7 +64,7 @@ public: Partition PartitionIdGetPartition(Oid part_oid, StorageType storage_type); public: - bool part_cache_need_eoxact_work; + bool PartCacheNeedEOXActWork; private: void InsertPartitionIntoGlobal(Partition part, uint32 hash_value); diff --git a/src/include/utils/knl_localsyscache_common.h b/src/include/utils/knl_localsyscache_common.h index 55f58071a..6421d3410 100644 --- a/src/include/utils/knl_localsyscache_common.h +++ b/src/include/utils/knl_localsyscache_common.h @@ -136,4 +136,14 @@ struct InvalidBaseEntry { void StreamTxnContextSaveInvalidMsg(void *stc); void StreamTxnContextRestoreInvalidMsg(void *stc); #define EnableLocalSysCache() t_thrd.lsc_cxt.enable_lsc + +/* + * if MAX_LSC_SWAPOUT_RATIO is 0.9, MAX_LSC_FREESIZE_RATIO is 0.8 + * used_space <= threshold * 0.9 < threshold ==> max_used_space = threshold * 0.8 + * total_space > threshold >= total_space * 0.9 ==> max_total_space = threshold / 0.8 + * so max_used_space = max_total_space * 0.72 + * this means 72% memory can be used at the worst sence */ +const double MAX_LSC_FREESIZE_RATIO = 0.2; +const double MAX_LSC_SWAPOUT_RATIO = 0.9; +const double MIN_LSC_SWAPOUT_RATIO = 0.7; #endif \ No newline at end of file diff --git a/src/include/utils/knl_localsysdbcache.h b/src/include/utils/knl_localsysdbcache.h index 590fa74c1..97719c775 100644 --- a/src/include/utils/knl_localsysdbcache.h +++ b/src/include/utils/knl_localsysdbcache.h @@ -214,8 +214,6 @@ public: bool is_closed; /* mark pgxcpoolreload flag, flush relcache's locatorinfo->nodelist */ bool got_pool_reload; - /* record other palloc on lsc */ - int64 other_space; /* record rel's index and rule cxt */ int64 rel_index_rule_space; /* record abort count, which may cause memory leak @@ -223,6 +221,7 @@ public: uint64 abort_count; /* record concurrent rd locks on syscache */ GSCRdLockInfo rdlock_info; + double cur_swapout_ratio; private: void Init(); void CreateCatBucket(); diff --git a/src/include/utils/knl_localsystabcache.h b/src/include/utils/knl_localsystabcache.h index 5acca8a3a..ba5eadd2c 100644 --- a/src/include/utils/knl_localsystabcache.h +++ b/src/include/utils/knl_localsystabcache.h @@ -38,7 +38,7 @@ public: { m_is_inited = false; local_systupcaches = NULL; - need_eoxact_work = false; + CatCacheNeedEOXActWork = false; } void ResetInitFlag(bool include_shared) @@ -54,13 +54,13 @@ public: void AtEOXact_CatCache(bool isCommit) { - if (!need_eoxact_work) { + if (!CatCacheNeedEOXActWork) { return; } for (int cache_id = 0; cache_id < SysCacheSize; cache_id++) { local_systupcaches[cache_id]->AtEOXact_CatCache(isCommit); } - need_eoxact_work = false; + CatCacheNeedEOXActWork = false; } /* systup may be shared table cache. not cleaned when cleardb */ @@ -166,7 +166,7 @@ public: } local_systupcaches[cache_id]->PrepareToInvalidateCacheTuple(tuple, newtuple, function); } - need_eoxact_work = true; + CatCacheNeedEOXActWork = true; } void CreateObject(); @@ -270,6 +270,6 @@ private: } bool m_is_inited; - bool need_eoxact_work; + bool CatCacheNeedEOXActWork; }; #endif \ No newline at end of file diff --git a/src/include/utils/knl_localtabdefcache.h b/src/include/utils/knl_localtabdefcache.h index ca86d33e1..4d020b851 100644 --- a/src/include/utils/knl_localtabdefcache.h +++ b/src/include/utils/knl_localtabdefcache.h @@ -124,7 +124,7 @@ public: */ List *initFileRelationIds; - bool need_eoxact_work; + bool RelCacheNeedEOXActWork; struct tupleDesc *m_pgclassdesc; struct tupleDesc *m_pgindexdesc; diff --git a/src/include/utils/knl_partcache.h b/src/include/utils/knl_partcache.h index a8324ca77..04fcd4acc 100644 --- a/src/include/utils/knl_partcache.h +++ b/src/include/utils/knl_partcache.h @@ -105,21 +105,21 @@ extern Partition PartitionBuildDesc(Oid targetPartId, StorageType storage_type, } \ } while (0) -inline bool PartCacheNeedEoxactWork() +inline bool GetPartCacheNeedEOXActWork() { if (EnableLocalSysCache()) { - return t_thrd.lsc_cxt.lsc->partdefcache.part_cache_need_eoxact_work; + return t_thrd.lsc_cxt.lsc->partdefcache.PartCacheNeedEOXActWork; } else { - return u_sess->cache_cxt.part_cache_need_eoxact_work; + return u_sess->cache_cxt.PartCacheNeedEOXActWork; } } -inline void SetPartCacheNeedEoxactWork(bool value) +inline void SetPartCacheNeedEOXActWork(bool value) { if (EnableLocalSysCache()) { - t_thrd.lsc_cxt.lsc->partdefcache.part_cache_need_eoxact_work = value; + t_thrd.lsc_cxt.lsc->partdefcache.PartCacheNeedEOXActWork = value; } else { - u_sess->cache_cxt.part_cache_need_eoxact_work = value; + u_sess->cache_cxt.PartCacheNeedEOXActWork = value; } } #endif \ No newline at end of file diff --git a/src/include/utils/knl_relcache.h b/src/include/utils/knl_relcache.h index cf269e9ed..1a88f2509 100644 --- a/src/include/utils/knl_relcache.h +++ b/src/include/utils/knl_relcache.h @@ -194,21 +194,21 @@ inline void AddLocalRelCacheInvalsReceived(int value) } } -inline bool LocalRelCacheNeedEOXactWork() +inline bool GetRelCacheNeedEOXActWork() { if (EnableLocalSysCache()) { - return t_thrd.lsc_cxt.lsc->tabdefcache.need_eoxact_work; + return t_thrd.lsc_cxt.lsc->tabdefcache.RelCacheNeedEOXActWork; } else { - return u_sess->relcache_cxt.need_eoxact_work; + return u_sess->relcache_cxt.RelCacheNeedEOXActWork; } } -inline void SetLocalRelCacheNeedEOXactWork(bool value) +inline void SetRelCacheNeedEOXActWork(bool value) { if (EnableLocalSysCache()) { - t_thrd.lsc_cxt.lsc->tabdefcache.need_eoxact_work = value; + t_thrd.lsc_cxt.lsc->tabdefcache.RelCacheNeedEOXActWork = value; } else { - u_sess->relcache_cxt.need_eoxact_work = value; + u_sess->relcache_cxt.RelCacheNeedEOXActWork = value; } } diff --git a/src/include/utils/partitionmap_gs.h b/src/include/utils/partitionmap_gs.h index 8bad10886..cc59c7d3a 100644 --- a/src/include/utils/partitionmap_gs.h +++ b/src/include/utils/partitionmap_gs.h @@ -470,6 +470,8 @@ typedef struct PruningResult { Param* paramArg; OpExpr* exprPart; Expr* expr; + /* This variable applies only to single-partition key range partition tables in PBE mode. */ + bool isPbeSinlePartition = false; } PruningResult; extern Oid partIDGetPartOid(Relation relation, PartitionIdentifier* partID); diff --git a/src/include/utils/pl_global_package_runtime_cache.h b/src/include/utils/pl_global_package_runtime_cache.h index 1610faf45..c36370ceb 100644 --- a/src/include/utils/pl_global_package_runtime_cache.h +++ b/src/include/utils/pl_global_package_runtime_cache.h @@ -42,6 +42,7 @@ typedef struct SessionPackageRuntime List* portalContext; List* portalData; List* funcValInfo; + bool is_insert_gs_source; } SessionPackageRuntime; typedef struct GPRCValue diff --git a/src/include/utils/plpgsql.h b/src/include/utils/plpgsql.h index 0c9b3b429..98b20c126 100644 --- a/src/include/utils/plpgsql.h +++ b/src/include/utils/plpgsql.h @@ -1053,6 +1053,7 @@ typedef struct PLpgSQL_function { /* Complete compiled function */ bool is_autonomous; bool is_plpgsql_func_with_outparam; + bool is_insert_gs_source; } PLpgSQL_function; class AutonomousSession; diff --git a/src/include/utils/resowner.h b/src/include/utils/resowner.h index 14c8fa26e..306d82485 100755 --- a/src/include/utils/resowner.h +++ b/src/include/utils/resowner.h @@ -64,10 +64,12 @@ typedef void (*ResourceReleaseCallback)(ResourceReleasePhase phase, bool isCommi extern ResourceOwner ResourceOwnerCreate(ResourceOwner parent, const char* name, MemoryContext memCxt); extern void ResourceOwnerRelease(ResourceOwner owner, ResourceReleasePhase phase, bool isCommit, bool isTopLevel); extern void ResourceOwnerDelete(ResourceOwner owner); +extern void ResourceOwnerConcat(ResourceOwner target, ResourceOwner source); extern ResourceOwner ResourceOwnerGetParent(ResourceOwner owner); extern ResourceOwner ResourceOwnerGetNextChild(ResourceOwner owner); extern const char * ResourceOwnerGetName(ResourceOwner owner); extern ResourceOwner ResourceOwnerGetFirstChild(ResourceOwner owner); +extern MemoryContext ResourceOwnerGetMemCxt(ResourceOwner owner); extern void ResourceOwnerNewParent(ResourceOwner owner, ResourceOwner newparent); /* support for buffer refcount management */ diff --git a/src/test/regress/expected/autonomous_test.out b/src/test/regress/expected/autonomous_test.out index 47e5c0d60..a9855c741 100644 --- a/src/test/regress/expected/autonomous_test.out +++ b/src/test/regress/expected/autonomous_test.out @@ -2007,3 +2007,82 @@ CONTEXT: referenced column: p1 drop package pck1; NOTICE: drop cascades to function public.p1(integer,integer) +-- anoymous block with autonomous +create or replace package pck1 as +type r1 is record(a int, b int); +va int; +vb r1; +vc varchar2(20); +end pck1; +/ +declare +begin +pck1.va := 1; +pck1.vb := (2,3); +pck1.vc := 'before auto'; +end; +/ +-- (a) autonomous anoymous block +declare +PRAGMA AUTONOMOUS_TRANSACTION; +begin +raise info 'pck1.va: %',pck1.va; +raise info 'pck1.vb: %',pck1.vb; +raise info 'pck1.vc: %',pck1.vc; +pck1.va := 11; +pck1.vb := (22,33); +pck1.vc := 'after auto'; +end; +/ +INFO: pck1.va: 1 + +INFO: pck1.vb: (2,3) + +INFO: pck1.vc: before auto + +declare +begin +raise info 'pck1.va: %',pck1.va; +raise info 'pck1.vb: %',pck1.vb; +raise info 'pck1.vc: %',pck1.vc; +pck1.va := 111; +pck1.vb := (222,333); +pck1.vc := 'before after auto'; +end; +/ +INFO: pck1.va: 11 +INFO: pck1.vb: (22,33) +INFO: pck1.vc: after auto +-- (b) autonomous anoymous block with error +declare +PRAGMA AUTONOMOUS_TRANSACTION; +begin +raise info 'pck1.va: %',pck1.va; +raise info 'pck1.vb: %',pck1.vb; +raise info 'pck1.vc: %',pck1.vc; +pck1.va := 1111; +pck1.vb := (2222,3333); +pck1.vc := 'after after auto'; +pck1.va := 3/0; +end; +/ +INFO: pck1.va: 111 + +INFO: pck1.vb: (222,333) + +INFO: pck1.vc: before after auto + +ERROR: ERROR: division by zero +CONTEXT: SQL statement "SELECT 3/0" +PL/pgSQL function inline_code_block line 10 at assignment + +declare +begin +raise info 'pck1.va: %',pck1.va; +raise info 'pck1.vb: %',pck1.vb; +raise info 'pck1.vc: %',pck1.vc; +end; +/ +INFO: pck1.va: 1111 +INFO: pck1.vb: (2222,3333) +INFO: pck1.vc: after after auto diff --git a/src/test/regress/expected/bypass_simplequery_support.out b/src/test/regress/expected/bypass_simplequery_support.out index f5bd4a9b9..7527bc401 100755 --- a/src/test/regress/expected/bypass_simplequery_support.out +++ b/src/test/regress/expected/bypass_simplequery_support.out @@ -1911,6 +1911,14 @@ create table test(a int); create view v_test as select * from test; CREATE OR REPLACE RULE v_delete as ON DELETE TO v_test DO INSTEAD NOTHING; delete from v_test; +set explain_perf_mode=pretty; +explain delete from v_test; + QUERY PLAN +--------------------------- + Query rewrites to nothing +(1 row) + +reset explain_perf_mode; drop table test cascade; NOTICE: drop cascades to view v_test -- end diff --git a/src/test/regress/expected/cstore_unique_index.out b/src/test/regress/expected/cstore_unique_index.out index 25ad06768..fdaa96f4a 100644 --- a/src/test/regress/expected/cstore_unique_index.out +++ b/src/test/regress/expected/cstore_unique_index.out @@ -105,8 +105,8 @@ create unique index on part_t1 using cbtree (a, c) local; c | integer | d | integer | Indexes: - "part_t1_pkey" PRIMARY KEY, cbtree (a, b) LOCAL(PARTITION p1_a_b_idx, PARTITION p2_a_b_idx, PARTITION p3_a_b_idx, PARTITION p4_a_b_idx) TABLESPACE pg_default - "part_t1_a_c_idx" UNIQUE, cbtree (a, c) LOCAL(PARTITION p1_a_c_idx, PARTITION p2_a_c_idx, PARTITION p3_a_c_idx, PARTITION p4_a_c_idx) TABLESPACE pg_default + "part_t1_pkey" PRIMARY KEY, cbtree (a, b) LOCAL TABLESPACE pg_default + "part_t1_a_c_idx" UNIQUE, cbtree (a, c) LOCAL TABLESPACE pg_default Partition By RANGE(a) Number of partitions: 4 (View pg_partition to check each partition range.) diff --git a/src/test/regress/expected/forall_save_exceptions.out b/src/test/regress/expected/forall_save_exceptions.out index 78870560c..bef50e3ab 100644 --- a/src/test/regress/expected/forall_save_exceptions.out +++ b/src/test/regress/expected/forall_save_exceptions.out @@ -785,7 +785,7 @@ exception end; / select p2(); -ERROR: transaction statement in store procedure used as sql to get value is not supported +ERROR: can not use commit rollback in Complex SQL CONTEXT: PL/pgSQL function p1() line 12 at FOR with integer loop variable referenced column: p1 referenced column: c1 @@ -830,14 +830,14 @@ end; / select p2(); number of failures: 8 -error: 1 array index: 2 messagecode: 1282 errormessage: transaction statement in store procedure used as sql to get value is not supported -error: 2 array index: 3 messagecode: 1282 errormessage: transaction statement in store procedure used as sql to get value is not supported -error: 3 array index: 4 messagecode: 1282 errormessage: transaction statement in store procedure used as sql to get value is not supported -error: 4 array index: 5 messagecode: 1282 errormessage: transaction statement in store procedure used as sql to get value is not supported -error: 5 array index: 6 messagecode: 1282 errormessage: transaction statement in store procedure used as sql to get value is not supported -error: 6 array index: 7 messagecode: 1282 errormessage: transaction statement in store procedure used as sql to get value is not supported -error: 7 array index: 8 messagecode: 1282 errormessage: transaction statement in store procedure used as sql to get value is not supported -error: 8 array index: 9 messagecode: 1282 errormessage: transaction statement in store procedure used as sql to get value is not supported +error: 1 array index: 2 messagecode: 1282 errormessage: can not use commit rollback in Complex SQL +error: 2 array index: 3 messagecode: 1282 errormessage: can not use commit rollback in Complex SQL +error: 3 array index: 4 messagecode: 1282 errormessage: can not use commit rollback in Complex SQL +error: 4 array index: 5 messagecode: 1282 errormessage: can not use commit rollback in Complex SQL +error: 5 array index: 6 messagecode: 1282 errormessage: can not use commit rollback in Complex SQL +error: 6 array index: 7 messagecode: 1282 errormessage: can not use commit rollback in Complex SQL +error: 7 array index: 8 messagecode: 1282 errormessage: can not use commit rollback in Complex SQL +error: 8 array index: 9 messagecode: 1282 errormessage: can not use commit rollback in Complex SQL successfully inserted: 2rows p2 ---- diff --git a/src/test/regress/expected/gpi_cluster_03.out b/src/test/regress/expected/gpi_cluster_03.out index 6c1db1fc0..a5d322457 100644 --- a/src/test/regress/expected/gpi_cluster_03.out +++ b/src/test/regress/expected/gpi_cluster_03.out @@ -51,8 +51,8 @@ create index global_inventory_table_index2 on inventory_table(INV_ITEM_SK) globa Indexes: "global_inventory_table_index1" btree (inv_item_sk) TABLESPACE pg_default "global_inventory_table_index2" btree (inv_item_sk) TABLESPACE pg_default - "inventory_table_index1" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default - "inventory_table_index2" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default + "inventory_table_index1" btree (inv_date_sk) LOCAL TABLESPACE pg_default + "inventory_table_index2" btree (inv_date_sk) LOCAL TABLESPACE pg_default Partition By RANGE(inv_date_sk) Number of partitions: 7 (View pg_partition to check each partition range.) Has OIDs: no @@ -70,8 +70,8 @@ cluster inventory_table using global_inventory_table_index1; Indexes: "global_inventory_table_index1" btree (inv_item_sk) TABLESPACE pg_default CLUSTER "global_inventory_table_index2" btree (inv_item_sk) TABLESPACE pg_default - "inventory_table_index1" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default - "inventory_table_index2" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default + "inventory_table_index1" btree (inv_date_sk) LOCAL TABLESPACE pg_default + "inventory_table_index2" btree (inv_date_sk) LOCAL TABLESPACE pg_default Partition By RANGE(inv_date_sk) Number of partitions: 7 (View pg_partition to check each partition range.) Has OIDs: no @@ -89,8 +89,8 @@ cluster inventory_table PARTITION(P3) USING inventory_table_index1; Indexes: "global_inventory_table_index1" btree (inv_item_sk) TABLESPACE pg_default "global_inventory_table_index2" btree (inv_item_sk) TABLESPACE pg_default - "inventory_table_index1" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default CLUSTER - "inventory_table_index2" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default + "inventory_table_index1" btree (inv_date_sk) LOCAL TABLESPACE pg_default CLUSTER + "inventory_table_index2" btree (inv_date_sk) LOCAL TABLESPACE pg_default Partition By RANGE(inv_date_sk) Number of partitions: 7 (View pg_partition to check each partition range.) Has OIDs: no @@ -108,8 +108,8 @@ cluster inventory_table using global_inventory_table_index1; Indexes: "global_inventory_table_index1" btree (inv_item_sk) TABLESPACE pg_default CLUSTER "global_inventory_table_index2" btree (inv_item_sk) TABLESPACE pg_default - "inventory_table_index1" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default - "inventory_table_index2" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default + "inventory_table_index1" btree (inv_date_sk) LOCAL TABLESPACE pg_default + "inventory_table_index2" btree (inv_date_sk) LOCAL TABLESPACE pg_default Partition By RANGE(inv_date_sk) Number of partitions: 7 (View pg_partition to check each partition range.) Has OIDs: no @@ -127,8 +127,8 @@ alter table inventory_table cluster on inventory_table_index1; Indexes: "global_inventory_table_index1" btree (inv_item_sk) TABLESPACE pg_default "global_inventory_table_index2" btree (inv_item_sk) TABLESPACE pg_default - "inventory_table_index1" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default CLUSTER - "inventory_table_index2" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default + "inventory_table_index1" btree (inv_date_sk) LOCAL TABLESPACE pg_default CLUSTER + "inventory_table_index2" btree (inv_date_sk) LOCAL TABLESPACE pg_default Partition By RANGE(inv_date_sk) Number of partitions: 7 (View pg_partition to check each partition range.) Has OIDs: no @@ -146,8 +146,8 @@ alter table inventory_table cluster on global_inventory_table_index1; Indexes: "global_inventory_table_index1" btree (inv_item_sk) TABLESPACE pg_default CLUSTER "global_inventory_table_index2" btree (inv_item_sk) TABLESPACE pg_default - "inventory_table_index1" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default - "inventory_table_index2" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default + "inventory_table_index1" btree (inv_date_sk) LOCAL TABLESPACE pg_default + "inventory_table_index2" btree (inv_date_sk) LOCAL TABLESPACE pg_default Partition By RANGE(inv_date_sk) Number of partitions: 7 (View pg_partition to check each partition range.) Has OIDs: no @@ -165,8 +165,8 @@ alter table inventory_table cluster on global_inventory_table_index2; Indexes: "global_inventory_table_index1" btree (inv_item_sk) TABLESPACE pg_default "global_inventory_table_index2" btree (inv_item_sk) TABLESPACE pg_default CLUSTER - "inventory_table_index1" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default - "inventory_table_index2" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default + "inventory_table_index1" btree (inv_date_sk) LOCAL TABLESPACE pg_default + "inventory_table_index2" btree (inv_date_sk) LOCAL TABLESPACE pg_default Partition By RANGE(inv_date_sk) Number of partitions: 7 (View pg_partition to check each partition range.) Has OIDs: no @@ -184,8 +184,8 @@ cluster inventory_table; Indexes: "global_inventory_table_index1" btree (inv_item_sk) TABLESPACE pg_default "global_inventory_table_index2" btree (inv_item_sk) TABLESPACE pg_default CLUSTER - "inventory_table_index1" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default - "inventory_table_index2" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default + "inventory_table_index1" btree (inv_date_sk) LOCAL TABLESPACE pg_default + "inventory_table_index2" btree (inv_date_sk) LOCAL TABLESPACE pg_default Partition By RANGE(inv_date_sk) Number of partitions: 7 (View pg_partition to check each partition range.) Has OIDs: no @@ -203,8 +203,8 @@ cluster inventory_table using global_inventory_table_index1; Indexes: "global_inventory_table_index1" btree (inv_item_sk) TABLESPACE pg_default CLUSTER "global_inventory_table_index2" btree (inv_item_sk) TABLESPACE pg_default - "inventory_table_index1" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default - "inventory_table_index2" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default + "inventory_table_index1" btree (inv_date_sk) LOCAL TABLESPACE pg_default + "inventory_table_index2" btree (inv_date_sk) LOCAL TABLESPACE pg_default Partition By RANGE(inv_date_sk) Number of partitions: 7 (View pg_partition to check each partition range.) Has OIDs: no @@ -222,8 +222,8 @@ cluster inventory_table; Indexes: "global_inventory_table_index1" btree (inv_item_sk) TABLESPACE pg_default CLUSTER "global_inventory_table_index2" btree (inv_item_sk) TABLESPACE pg_default - "inventory_table_index1" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default - "inventory_table_index2" btree (inv_date_sk) LOCAL(PARTITION p1_inv_date_sk_idx, PARTITION p2_inv_date_sk_idx, PARTITION p3_inv_date_sk_idx, PARTITION p4_inv_date_sk_idx, PARTITION p5_inv_date_sk_idx, PARTITION p6_inv_date_sk_idx, PARTITION p7_inv_date_sk_idx) TABLESPACE pg_default + "inventory_table_index1" btree (inv_date_sk) LOCAL TABLESPACE pg_default + "inventory_table_index2" btree (inv_date_sk) LOCAL TABLESPACE pg_default Partition By RANGE(inv_date_sk) Number of partitions: 7 (View pg_partition to check each partition range.) Has OIDs: no diff --git a/src/test/regress/expected/gpi_create_constraint.out b/src/test/regress/expected/gpi_create_constraint.out index 3ae71f89f..befb7bde8 100644 --- a/src/test/regress/expected/gpi_create_constraint.out +++ b/src/test/regress/expected/gpi_create_constraint.out @@ -56,9 +56,9 @@ Indexes: "gpi_table1_c1_tableoid_idx" btree (c1) TABLESPACE pg_default "idx10_gpi_table1" btree (c1) TABLESPACE pg_default "idx1_gpi_table1" btree (c1) TABLESPACE pg_default - "idx3_gpi_table1" btree (c2) LOCAL(PARTITION p0_gpi_table1_c2_idx, PARTITION p1_gpi_table1_c2_idx, PARTITION p2_gpi_table1_c2_idx, PARTITION p3_gpi_table1_c2_idx) TABLESPACE pg_default - "idx4_gpi_table1" btree ((c1 + 10)) LOCAL(PARTITION p0_gpi_table1_expr_idx, PARTITION p1_gpi_table1_expr_idx, PARTITION p2_gpi_table1_expr_idx, PARTITION p3_gpi_table1_expr_idx) TABLESPACE pg_default - "idx5_gpi_table1" btree (c1, c2) LOCAL(PARTITION p0_gpi_table1_c1_c2_idx, PARTITION p1_gpi_table1_c1_c2_idx, PARTITION p2_gpi_table1_c1_c2_idx, PARTITION p3_gpi_table1_c1_c2_idx) TABLESPACE pg_default + "idx3_gpi_table1" btree (c2) LOCAL TABLESPACE pg_default + "idx4_gpi_table1" btree ((c1 + 10)) LOCAL TABLESPACE pg_default + "idx5_gpi_table1" btree (c1, c2) LOCAL TABLESPACE pg_default Partition By RANGE(c1) Number of partitions: 4 (View pg_partition to check each partition range.) diff --git a/src/test/regress/expected/gsc_func.out b/src/test/regress/expected/gsc_func.out index 63a77c650..2e9de64c2 100644 --- a/src/test/regress/expected/gsc_func.out +++ b/src/test/regress/expected/gsc_func.out @@ -1,24 +1,24 @@ select * from gs_gsc_dbstat_info() limit 1; - database_id | database_name | tup_searches | tup_hits | tup_miss | tup_count | tup_dead | tup_memory | rel_searches | rel_hits | rel_miss | rel_count | rel_dead | rel_memory | part_searches | part_hits | part_miss | part_count | part_dead | part_memory | total_memory | swapout_count | refcount --------------+---------------+--------------+----------+----------+-----------+----------+------------+--------------+----------+----------+-----------+----------+------------+---------------+-----------+-----------+------------+-----------+-------------+--------------+---------------+---------- +--?.* database_id | database_name | tup_searches | tup_hits | tup_miss | tup_count | tup_dead | tup_memory | rel_searches | rel_hits | rel_miss | rel_count | rel_dead | rel_memory | part_searches | part_hits | part_miss | part_count | part_dead | part_memory | total_memory | swapout_count | refcount +--?.*-------------+---------------+--------------+----------+----------+-----------+----------+------------+--------------+----------+----------+-----------+----------+------------+---------------+-----------+-----------+------------+-----------+-------------+--------------+---------------+---------- --?.* 0 | | 40 | 23 | 9 | 7 | 0 | 3760 | 143 | 2 | 25 | 25 | 0 | 106656 | 0 | 0 | 0 | 0 | 0 | 0 | 405848 | 0 | 0 (1 row) select * from gs_gsc_dbstat_info(-1) limit 1; - database_id | database_name | tup_searches | tup_hits | tup_miss | tup_count | tup_dead | tup_memory | rel_searches | rel_hits | rel_miss | rel_count | rel_dead | rel_memory | part_searches | part_hits | part_miss | part_count | part_dead | part_memory | total_memory | swapout_count | refcount --------------+---------------+--------------+----------+----------+-----------+----------+------------+--------------+----------+----------+-----------+----------+------------+---------------+-----------+-----------+------------+-----------+-------------+--------------+---------------+---------- +--?.* database_id | database_name | tup_searches | tup_hits | tup_miss | tup_count | tup_dead | tup_memory | rel_searches | rel_hits | rel_miss | rel_count | rel_dead | rel_memory | part_searches | part_hits | part_miss | part_count | part_dead | part_memory | total_memory | swapout_count | refcount +--?.*-------------+---------------+--------------+----------+----------+-----------+----------+------------+--------------+----------+----------+-----------+----------+------------+---------------+-----------+-----------+------------+-----------+-------------+--------------+---------------+---------- --?.* 0 | | 40 | 23 | 9 | 7 | 0 | 3760 | 143 | 2 | 25 | 25 | 0 | 106656 | 0 | 0 | 0 | 0 | 0 | 0 | 405848 | 0 | 0 (1 row) select * from gs_gsc_dbstat_info(0) limit 1; - database_id | database_name | tup_searches | tup_hits | tup_miss | tup_count | tup_dead | tup_memory | rel_searches | rel_hits | rel_miss | rel_count | rel_dead | rel_memory | part_searches | part_hits | part_miss | part_count | part_dead | part_memory | total_memory | swapout_count | refcount --------------+---------------+--------------+----------+----------+-----------+----------+------------+--------------+----------+----------+-----------+----------+------------+---------------+-----------+-----------+------------+-----------+-------------+--------------+---------------+---------- +--?.* database_id | database_name | tup_searches | tup_hits | tup_miss | tup_count | tup_dead | tup_memory | rel_searches | rel_hits | rel_miss | rel_count | rel_dead | rel_memory | part_searches | part_hits | part_miss | part_count | part_dead | part_memory | total_memory | swapout_count | refcount +--?.*-------------+---------------+--------------+----------+----------+-----------+----------+------------+--------------+----------+----------+-----------+----------+------------+---------------+-----------+-----------+------------+-----------+-------------+--------------+---------------+---------- --?.* 0 | | 40 | 23 | 9 | 7 | 0 | 3760 | 143 | 2 | 25 | 25 | 0 | 106656 | 0 | 0 | 0 | 0 | 0 | 0 | 405848 | 0 | 0 (1 row) select * from gs_gsc_dbstat_info(1) limit 1; - database_id | database_name | tup_searches | tup_hits | tup_miss | tup_count | tup_dead | tup_memory | rel_searches | rel_hits | rel_miss | rel_count | rel_dead | rel_memory | part_searches | part_hits | part_miss | part_count | part_dead | part_memory | total_memory | swapout_count | refcount --------------+---------------+--------------+----------+----------+-----------+----------+------------+--------------+----------+----------+-----------+----------+------------+---------------+-----------+-----------+------------+-----------+-------------+--------------+---------------+---------- +--?.* database_id | database_name | tup_searches | tup_hits | tup_miss | tup_count | tup_dead | tup_memory | rel_searches | rel_hits | rel_miss | rel_count | rel_dead | rel_memory | part_searches | part_hits | part_miss | part_count | part_dead | part_memory | total_memory | swapout_count | refcount +--?.*-------------+---------------+--------------+----------+----------+-----------+----------+------------+--------------+----------+----------+-----------+----------+------------+---------------+-----------+-----------+------------+-----------+-------------+--------------+---------------+---------- --?.* 0 | | 40 | 23 | 9 | 7 | 0 | 3760 | 143 | 2 | 25 | 25 | 0 | 106656 | 0 | 0 | 0 | 0 | 0 | 0 | 405848 | 0 | 0 (1 row) @@ -26,121 +26,121 @@ select * from gs_gsc_dbstat_info(2) limit 1; ERROR: dbOid doesn't exist. DETAIL: dbOid is invalid, please pass valid dbOid. select * from gs_gsc_catalog_detail() limit 1; - database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount --------------+---------------+--------+-----------+----------+--------+--------+----------+-----------+------------+---------- +--?.* database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount +--?.*-------------+---------------+--------+-----------+----------+--------+--------+----------+-----------+------------+---------- --?.* 0 | | 1260 | pg_authid | 10 | (0, 9) | (0, 9) | 10507 | 26 | 531311568 | 9 (1 row) select * from gs_gsc_catalog_detail(-1) limit 1; - database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount --------------+---------------+--------+-----------+----------+--------+--------+----------+-----------+------------+---------- +--?.* database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount +--?.*-------------+---------------+--------+-----------+----------+--------+--------+----------+-----------+------------+---------- --?.* 0 | | 1260 | pg_authid | 10 | (0, 9) | (0, 9) | 10507 | 26 | 531311568 | 9 (1 row) select * from gs_gsc_catalog_detail(0) limit 1; - database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount --------------+---------------+--------+-----------+----------+--------+--------+----------+-----------+------------+---------- +--?.* database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount +--?.*-------------+---------------+--------+-----------+----------+--------+--------+----------+-----------+------------+---------- --?.* 0 | | 1260 | pg_authid | 10 | (0, 9) | (0, 9) | 10507 | 26 | 531311568 | 9 (1 row) select * from gs_gsc_catalog_detail(1) limit 1; - database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount --------------+---------------+--------+-----------+----------+--------+--------+----------+-----------+------------+---------- +--?.* database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount +--?.*-------------+---------------+--------+-----------+----------+--------+--------+----------+-----------+------------+---------- --?.* 0 | | 1260 | pg_authid | 10 | (0, 9) | (0, 9) | 10507 | 26 | 531311568 | 9 (1 row) select * from gs_gsc_catalog_detail(-1, 1262) limit 1; - database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount --------------+---------------+--------+-------------+----------+--------+--------+----------+-----------+------------+---------- +--?.* database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount +--?.*-------------+---------------+--------+-------------+----------+--------+--------+----------+-----------+------------+---------- --?.* 0 | | 1262 | pg_database | 27 | (0, 4) | (0, 4) | 10506 | 15 | 361410604 | 1 (1 row) select * from gs_gsc_catalog_detail(0, 1262) limit 1; - database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount --------------+---------------+--------+-------------+----------+--------+--------+----------+-----------+------------+---------- +--?.* database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount +--?.*-------------+---------------+--------+-------------+----------+--------+--------+----------+-----------+------------+---------- --?.* 0 | | 1262 | pg_database | 27 | (0, 4) | (0, 4) | 10506 | 15 | 361410604 | 1 (1 row) select * from gs_gsc_catalog_detail(1, 1262) limit 1; - database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount --------------+---------------+--------+-------------+----------+--------+--------+----------+-----------+------------+---------- +--?.* database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount +--?.*-------------+---------------+--------+-------------+----------+--------+--------+----------+-----------+------------+---------- --?.* 0 | | 1262 | pg_database | 27 | (0, 4) | (0, 4) | 10506 | 15 | 361410604 | 1 (1 row) select * from gs_gsc_catalog_detail(-1, 1259) limit 1; ---?.* database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount ---?.*-------------+---------------+--------+----------+----------+---------+---------+----------+-----------+------------+---------- +--?.* database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount +--?.*-------------+---------------+--------+----------+----------+--------+--------+----------+-----------+------------+---------- --?.* 16299 | postgres | 1259 | pg_class | 84 | (7, 3) | (7, 3) | 10507 | 40 | 1520565747 | 0 (1 row) select * from gs_gsc_catalog_detail(0, 1259) limit 1; - database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount --------------+---------------+--------+----------+----------+------+------+----------+-----------+------------+---------- +--?.* database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount +--?.*-------------+---------------+--------+----------+----------+------+------+----------+-----------+------------+---------- (0 rows) select * from gs_gsc_catalog_detail(1, 1259) limit 1; - database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount --------------+---------------+--------+----------+----------+------+------+----------+-----------+------------+---------- +--?.* database_id | database_name | rel_id | rel_name | cache_id | self | ctid | infomask | infomask2 | hash_value | refcount +--?.*-------------+---------------+--------+----------+----------+------+------+----------+-----------+------------+---------- (0 rows) select * from gs_gsc_catalog_detail(2, 1259) limit 1; ERROR: dbOid doesn't exist. DETAIL: dbOid is invalid, please pass valid dbOid. select * from gs_gsc_table_detail() limit 1; - database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo ---------------+---------------+--------+-------------------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+-----------+--------- +--?.* database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo +--?.*--------------+---------------+--------+-------------------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+-----------+--------- --?.* 0 | | 2676 | pg_authid_rolname_index | 11 | 0 | 0 | 10 | 403 | 0 | 1664 | f | t | i | 1 | f | f | n | f | 'rolname' | (1 row) select * from gs_gsc_table_detail(-1) limit 1; - database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo ---------------+---------------+--------+-------------------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+-----------+--------- +--?.* database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo +--?.*--------------+---------------+--------+-------------------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+-----------+--------- --?.* 0 | | 2676 | pg_authid_rolname_index | 11 | 0 | 0 | 10 | 403 | 0 | 1664 | f | t | i | 1 | f | f | n | f | 'rolname' | (1 row) select * from gs_gsc_table_detail(0) limit 1; - database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo ---------------+---------------+--------+-------------------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+-----------+--------- +--?.* database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo +--?.*--------------+---------------+--------+-------------------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+-----------+--------- --?.* 0 | | 2676 | pg_authid_rolname_index | 11 | 0 | 0 | 10 | 403 | 0 | 1664 | f | t | i | 1 | f | f | n | f | 'rolname' | (1 row) select * from gs_gsc_table_detail(1) limit 1; - database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo ---------------+---------------+--------+-------------------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+-----------+--------- +--?.* database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo +--?.*--------------+---------------+--------+-------------------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+-----------+--------- --?.* 0 | | 2676 | pg_authid_rolname_index | 11 | 0 | 0 | 10 | 403 | 0 | 1664 | f | t | i | 1 | f | f | n | f | 'rolname' | (1 row) select * from gs_gsc_table_detail(-1, 1262) limit 1; - database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo ---------------+---------------+--------+-------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------- +--?.* database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo +--?.*--------------+---------------+--------+-------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------- --?.* 0 | | 1262 | pg_database | 11 | 1248 | 0 | 10 | 0 | 0 | 1664 | t | t | r | 15 | t | f | n | f | 'datname','datdba','encoding','datcollate','datctype','datistemplate','datallowconn','datconnlimit','datlastsysoid','datfrozenxid','dattablespace','datcompatibility','datacl','datfrozenxid64','datminmxid' | (1 row) select * from gs_gsc_table_detail(0, 1262) limit 1; - database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo ---------------+---------------+--------+-------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------- +--?.* database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo +--?.*--------------+---------------+--------+-------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------- --?.* 0 | | 1262 | pg_database | 11 | 1248 | 0 | 10 | 0 | 0 | 1664 | t | t | r | 15 | t | f | n | f | 'datname','datdba','encoding','datcollate','datctype','datistemplate','datallowconn','datconnlimit','datlastsysoid','datfrozenxid','dattablespace','datcompatibility','datacl','datfrozenxid64','datminmxid' | (1 row) select * from gs_gsc_table_detail(1, 1262) limit 1; - database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo ---------------+---------------+--------+-------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------- +--?.* database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo +--?.*--------------+---------------+--------+-------------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------- --?.* 0 | | 1262 | pg_database | 11 | 1248 | 0 | 10 | 0 | 0 | 1664 | t | t | r | 15 | t | f | n | f | 'datname','datdba','encoding','datcollate','datctype','datistemplate','datallowconn','datconnlimit','datlastsysoid','datfrozenxid','dattablespace','datcompatibility','datacl','datfrozenxid64','datminmxid' | (1 row) select * from gs_gsc_table_detail(-1, 1259) limit 1; - database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo ---------------+---------------+--------+----------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------- +--?.* database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo +--?.*--------------+---------------+--------+----------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------- --?.* 16384 | regression | 1259 | pg_class | 11 | 83 | 0 | 10 | 0 | 0 | 0 | t | f | r | 40 | t | f | n | f | 'relname','relnamespace','reltype','reloftype','relowner','relam','relfilenode','reltablespace','relpages','reltuples','relallvisible','reltoastrelid','reltoastidxid','reldeltarelid','reldeltaidx','relcudescrelid','relcudescidx','relhasindex','relisshared','relpersistence','relkind','relnatts','relchecks','relhasoids','relhaspkey','relhasrules','relhastriggers','relhassubclass','relcmprs','relhasclusterkey','relrowmovement','parttype','relfrozenxid','relacl','reloptions','relreplident','relfrozenxid64','relbucket','relbucketkey','relminmxid' | (1 row) select * from gs_gsc_table_detail(0, 1259) limit 1; - database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo +--?.* database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo --------------+---------------+--------+---------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+----------+--------- (0 rows) select * from gs_gsc_table_detail(1, 1259) limit 1; - database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo +--?.* database_oid | database_name | reloid | relname | relnamespace | reltype | reloftype | relowner | relam | relfilenode | reltablespace | relhasindex | relisshared | relkind | relnatts | relhasoids | relhaspkey | parttype | tdhasuids | attnames | extinfo --------------+---------------+--------+---------+--------------+---------+-----------+----------+-------+-------------+---------------+-------------+-------------+---------+----------+------------+------------+----------+-----------+----------+--------- (0 rows) diff --git a/src/test/regress/expected/huge_clob.out b/src/test/regress/expected/huge_clob.out index 8c9dd1bcf..89182368a 100644 --- a/src/test/regress/expected/huge_clob.out +++ b/src/test/regress/expected/huge_clob.out @@ -11,12 +11,36 @@ drop schema if exists huge_clob; NOTICE: schema "huge_clob" does not exist, skipping create schema huge_clob; set current_schema = huge_clob; -create table bigclobtbl031(c1 int,c2 clob,c3 clob,c4 blob,c5 date,c6 timestamp,c7 varchar2); -insert into bigclobtbl031 values(generate_series(1,5),repeat('AAAA11111aaaaaaaaaaaaaaaaa',1000000),repeat('abdededfj12345679ujik',1000000),hextoraw(repeat('12345678990abcdef',1000)),sysdate,to_timestamp('','yyyy-mm-dd hh24:mi:ss.ff6'),7000); -update bigclobtbl031 set c2=c2||c2||c2||c2||c2; -update bigclobtbl031 set c2=c2||c2||c2||c2||c2; -update bigclobtbl031 set c2=c2||c2; -update bigclobtbl031 set c3='clobclob3'; +drop table if exists cloblongtbl; +NOTICE: table "cloblongtbl" does not exist, skipping +create table cloblongtbl (a int, b clob, c clob); +-- insert data less than 1G +insert into cloblongtbl values (generate_series(1,4),repeat('唐李白床前明月光,疑是地上霜,举头望明月,低头思故乡',5000000),repeat('唐李白床前明月光,疑是地上霜,举头望明月,低头思故乡',5000000)); +update cloblongtbl set b = b||b; +update cloblongtbl set c = c||c; +-- b > 1G && c < 1G when a = 2 +update cloblongtbl set b = b||b where a = 2; +-- b < 1G && c > 1G when a = 3 +update cloblongtbl set c = c||c where a = 3; +-- b > 1G && c > 1G when a = 4 +update cloblongtbl set b = b||b where a = 4; +update cloblongtbl set c = c||c where a = 4; +select a, length(b || c) from cloblongtbl order by 1; + a | length +---+------------ + 1 | 520000000 + 2 | 780000000 + 3 | 780000000 + 4 | 1040000000 +(4 rows) + +-- reset data for other test +update cloblongtbl set b = b || b where a = 1; +update cloblongtbl set b = b || b where a = 3; +update cloblongtbl set c='cloblessthan1G' where a = 1; +update cloblongtbl set c='cloblessthan1G' where a = 2; +update cloblongtbl set c='cloblessthan1G' where a = 3; +update cloblongtbl set c='cloblessthan1G' where a = 4; --I1.clob in create or replace procedure pro_cb4_031(c1 clob,c2 clob) is @@ -33,8 +57,8 @@ create or replace procedure pro_cb4_031_1 is v1 clob; v2 clob; begin -execute immediate 'select c2 from bigclobtbl031 where c1=1' into v1; -execute immediate 'select c3 from bigclobtbl031 where c1=1' into v2; +execute immediate 'select b from cloblongtbl where a=1' into v1; +execute immediate 'select c from cloblongtbl where a=1' into v2; pro_cb4_031(v1,v2); end; / @@ -43,14 +67,30 @@ ERROR: huge clob do not support as function in parameter CONTEXT: PL/pgSQL function pro_cb4_031(clob,clob) line 5 at assignment SQL statement "CALL pro_cb4_031(v1,v2)" PL/pgSQL function pro_cb4_031_1() line 7 at PERFORM +create or replace procedure pro_cb4_005 is +v1 clob; +v2 clob; +v3 clob; +v4 integer; +begin +execute immediate 'select b from cloblongtbl where a=1' into v1; +dbe_lob.read(v1,10,2,v2); +end; +/ +call pro_cb4_005(); + pro_cb4_005 +------------- + +(1 row) + --I2.clob > 1G out create or replace procedure pro_cb4_031(c1 out clob,c2 out clob) is v1 clob; v2 clob; begin -execute immediate 'select c2 from bigclobtbl031 where c1=1' into v1; -execute immediate 'select c3 from bigclobtbl031 where c1=1' into v2; +execute immediate 'select b from cloblongtbl where a=1' into v1; +execute immediate 'select c from cloblongtbl where a=1' into v2; c1:=v1; c2:=v2; end; @@ -79,8 +119,8 @@ is v1 clob; v2 clob; begin -execute immediate 'select c3 from bigclobtbl031 where c1=1' into v1; -execute immediate 'select c3 from bigclobtbl031 where c1=2' into v2; +execute immediate 'select c from cloblongtbl where a=1' into v1; +execute immediate 'select c from cloblongtbl where a=2' into v2; c1:=v1; c2:=v2; end; @@ -96,8 +136,8 @@ is v1 clob; v2 clob; begin -execute immediate 'select c2 from bigclobtbl031 where c1=1' into v1; -execute immediate 'select c3 from bigclobtbl031 where c1=1' into v2; +execute immediate 'select b from cloblongtbl where a=1' into v1; +execute immediate 'select c from cloblongtbl where a=1' into v2; c1:=v1; c2:=v2; end; @@ -127,12 +167,12 @@ v1 clob; v2 clob; v3 clob; begin -execute immediate 'select c3 from bigclobtbl031 where c1=1' into v1; -execute immediate 'select c3 from bigclobtbl031 where c1=2' into v2; -execute immediate 'select c3 from bigclobtbl031 where c1=3' into v3; +execute immediate 'select c from cloblongtbl where a=1' into v1; +execute immediate 'select c from cloblongtbl where a=2' into v2; +execute immediate 'select c from cloblongtbl where a=3' into v3; c1:=v1; c2:=v2; -c3:=v3||'clob3clob3clob3clob3'; +c3:=v3||'clobclobclobclob'; end; / create or replace procedure pro_cb4_031_1 is @@ -163,8 +203,8 @@ type ty1 is table of clob; v1 ty1; begin for i in 1..10 loop -execute immediate 'select c2 from bigclobtbl031 where c1='||i into v1(i); -update bigclobtbl030 set c3=v1(i)||v1(i) where c1=i; +execute immediate 'select b from cloblongtbl where a='||i into v1(i); +update cloblongtbl set c=v1(i)||v1(i) where a=i; end loop; end; / @@ -177,27 +217,26 @@ type ty1 is varray(10) of clob; v1 ty1; begin for i in 1..10 loop -execute immediate 'select c2 from bigclobtbl031 where c1='||i into v1(i); -update bigclobtbl030 set c3=v1(i)||v1(i) where c1=i; +execute immediate 'select b from cloblongtbl where a='||i into v1(i); +update cloblongtbl set c=v1(i)||v1(i) where a=i; end loop; end; / call pro_cb4_031(); ERROR: huge clob do not support as array element. CONTEXT: PL/pgSQL function pro_cb4_031() line 6 at EXECUTE statement -select c1,c2,length(c2),c3,length(c3) from bigclobtbl031 where c1>5 and c1<10 order by 1,2,3,4,5; - c1 | c2 | length | c3 | length -----+----+--------+----+-------- +select a,b,length(b),c,length(c) from cloblongtbl where a>5 and a<10 order by 1,2,3,4,5; + a | b | length | c | length +---+---+--------+---+-------- (0 rows) -update bigclobtbl031 set c3='clob3clob3'; -ERROR: tuple concurrently updated +update cloblongtbl set c='cloblessthan1G'; --I6.record create or replace procedure pro_cb4_031 is type ty1 is record(c1 int,c2 clob); v1 ty1; begin -execute immediate 'select c2 from bigclobtbl031 where c1=1' into v1.c2; +execute immediate 'select b from cloblongtbl where a=1' into v1.c2; end; / call pro_cb4_031(); @@ -212,7 +251,7 @@ v1 clob; v2 clob; v3 clob; v4 int; -cursor cor1 is select c2 from bigclobtbl031 where c1=1; +cursor cor1 is select b from cloblongtbl where a=1; begin open cor1; loop @@ -227,32 +266,58 @@ close cor1; end; / call pro_cb4_037(); -ERROR: huge clob do not support as record element. -CONTEXT: PL/pgSQL function pro_cb4_037() line 10 at FETCH -drop table if exists cloblongtbl; -NOTICE: table "cloblongtbl" does not exist, skipping -create table cloblongtbl (a int, b clob, c clob); -insert into cloblongtbl values (generate_series(1,4),repeat('唐李白床前明月光,疑是地上霜,举头望明月,低头思故乡',5000000),repeat('唐李白床前明月光,疑是地上霜,举头望明月,低头思故乡',5000000)); -update cloblongtbl set b = b||b; -update cloblongtbl set c = c||c; -update cloblongtbl set b = b||b where a = 2; -update cloblongtbl set c = c||c where a = 3; -update cloblongtbl set b = b||b where a = 4; -update cloblongtbl set c = c||c where a = 4; -select a, length(b || c) from cloblongtbl order by 1; - a | length ----+------------ - 1 | 520000000 - 2 | 780000000 - 3 | 780000000 - 4 | 1040000000 -(4 rows) + pro_cb4_037 +------------- + +(1 row) +create or replace procedure test_self_update is +v1 clob; +begin +execute immediate 'select b from cloblongtbl where a=1' into v1; +update cloblongtbl set b=v1 where a=1; +savepoint aaa; +update cloblongtbl set b=v1 where a=2; +rollback to aaa; +commit; +end; +/ +call test_self_update(); + test_self_update +------------------ + +(1 row) + +create or replace procedure test_update_delete is +v1 clob; +begin +execute immediate 'select b from cloblongtbl where a=1' into v1; +update cloblongtbl set b=v1 where a=1; +rollback; +update cloblongtbl set b=v1 where a=2; +commit; +end; +/ +call test_update_delete(); + test_update_delete +-------------------- + +(1 row) + +begin; +delete from cloblongtbl where a < 3; +rollback; +begin; +delete from cloblongtbl where a = 1; +delete from cloblongtbl where a = 2; +rollback; drop table if exists cloblongtbl; -- clean drop schema if exists huge_clob cascade; -NOTICE: drop cascades to 4 other objects -DETAIL: drop cascades to table bigclobtbl031 +NOTICE: drop cascades to 6 other objects +DETAIL: drop cascades to function pro_cb4_005() drop cascades to function pro_cb4_031_1() drop cascades to function pro_cb4_031() drop cascades to function pro_cb4_037() +drop cascades to function test_self_update() +drop cascades to function test_update_delete() diff --git a/src/test/regress/expected/hw_audit_multi_thread_rotation_interval.out b/src/test/regress/expected/hw_audit_multi_thread_rotation_interval.out new file mode 100644 index 000000000..e69de29bb diff --git a/src/test/regress/expected/hw_package_single.out b/src/test/regress/expected/hw_package_single.out index f99807a79..c7712b709 100644 --- a/src/test/regress/expected/hw_package_single.out +++ b/src/test/regress/expected/hw_package_single.out @@ -304,6 +304,7 @@ BEGIN END; / call test_substr(); +23 test_substr ------------- @@ -538,5 +539,156 @@ LINE 1: select proc_test1('1',''); HINT: No function matches the given name and argument types. You might need to add explicit type casts. CONTEXT: referenced column: proc_test1 drop procedure proc_test1; +--test write+update +create table test_clob(a int, b clob); +create table test_blob(a int, b blob); +insert into test_clob values(1, 'abc'); +insert into test_clob values(2, 'zzz'); +insert into test_blob values(1, 'abc'); +insert into test_blob values(2, 'fffffff'); +create or replace procedure test_clob_write +as + dest_clob clob; + PSV_SQL varchar(100); +begin + PSV_SQL := 'select b from test_clob where a = 1'; + EXECUTE IMMEDIATE PSV_SQL into dest_clob; + dbe_lob.write(dest_clob, 1, 1, 'ddd'); + return; +end; +/ +call test_clob_write(); + test_clob_write +----------------- + +(1 row) + +select * from test_clob; + a | b +---+----- + 2 | zzz + 1 | dbc +(2 rows) + +create or replace procedure test_lob_write_nodyna +as +declare + dest_clob clob; +begin + select b from test_clob where a = 2 into dest_clob; + dbe_lob.write(dest_clob, 1, 1, 'eeee'); + DBE_OUTPUT.print_line(dest_clob); + return; +end; +/ +call test_lob_write_nodyna(); +ezz + test_lob_write_nodyna +----------------------- + +(1 row) + +select * from test_clob; + a | b +---+----- + 2 | zzz + 1 | dbc +(2 rows) + +create or replace procedure test_blob_write +as + dest_blob blob; + PSV_SQL varchar(100); +begin + PSV_SQL := 'select b from test_blob where a = 1'; + EXECUTE IMMEDIATE PSV_SQL into dest_blob; + dbe_lob.write(dest_blob, 1, 1, 'ddd'); + return; +end; +/ +call test_blob_write(); + test_blob_write +----------------- + +(1 row) + +select * from test_blob; + a | b +---+---------- + 2 | 0FFFFFFF + 1 | 0DBC +(2 rows) + +--test append+update +create or replace procedure test_clob_append +as + dest_clob clob; + PSV_SQL varchar(100); +begin + PSV_SQL := 'select b from test_clob where a = 1'; + EXECUTE IMMEDIATE PSV_SQL into dest_clob; + dbe_lob.append(dest_clob, 'ddd'); + return; +end; +/ +call test_clob_append(); + test_clob_append +------------------ + +(1 row) + +select * from test_clob; + a | b +---+-------- + 2 | zzz + 1 | dbcddd +(2 rows) + +create or replace procedure test_blob_append +as + dest_blob blob; + PSV_SQL varchar(100); +begin + PSV_SQL := 'select b from test_blob where a = 1'; + EXECUTE IMMEDIATE PSV_SQL into dest_blob; + dbe_lob.append(dest_blob, 'ddd'); + return; +end; +/ +call test_blob_append(); + test_blob_append +------------------ + +(1 row) + +select * from test_blob; + a | b +---+---------- + 2 | 0FFFFFFF + 1 | 0DBC0DDD +(2 rows) + +--test read +create or replace procedure test_clob_read +as + dest_clob clob; + buf clob; + PSV_SQL varchar(100); +begin + PSV_SQL := 'select b from test_clob where a = 1'; + EXECUTE IMMEDIATE PSV_SQL into dest_clob; + dbe_lob.read(dest_clob, 1, 1, buf); + DBE_OUTPUT.print_line(buf); + return; +end; +/ +call test_clob_read(); +d + test_clob_read +---------------- + +(1 row) + +drop table if exists test_clob; \c regression; drop database IF EXISTS pl_test_pkg_single; diff --git a/src/test/regress/expected/hw_package_variable.out b/src/test/regress/expected/hw_package_variable.out index 2120467d6..983680c97 100644 --- a/src/test/regress/expected/hw_package_variable.out +++ b/src/test/regress/expected/hw_package_variable.out @@ -2732,6 +2732,61 @@ NOTICE: drop cascades to function pkg_val_2.p1() DROP PACKAGE pck2; NOTICE: drop cascades to function pkg_val_2.p1() DROP TABLE t1; +-- test check sql in for rec in query loop +create or replace function check_f04 return int as +declare +v_2 number(10,2):=-213.989; +v_3 float:=22.34; +v_4 char(1):='q'; +v_6 date:='1999-09-09'; +v_7 clob:='dddd'; +v_8 blob:='ad'; +begin +raise info '1-0 %', v_6; +for v_6 in select c3 from check_tab1 order by c3 limit 6 +loop +raise info '1-1 %', v_6; +end loop; +raise info '1-2 %', v_6; + +return 1; +exception +when others then +raise info 'error is %',sqlerrm; +return 0; +end; +/ +ERROR: compile failed when parse the query: select c3 from check_tab1 order by c3 limit 6 +DETAIL: relation "check_tab1" does not exist on datanode1 +CONTEXT: compilation of PL/pgSQL function "check_f04" near line 11 +create table check_tab1(a int); +create or replace function check_f04 return int as +declare +v_2 number(10,2):=-213.989; +v_3 float:=22.34; +v_4 char(1):='q'; +v_6 date:='1999-09-09'; +v_7 clob:='dddd'; +v_8 blob:='ad'; +begin +raise info '1-0 %', v_6; +for v_6 in select c3 from check_tab1 order by c3 limit 6 +loop +raise info '1-1 %', v_6; +end loop; +raise info '1-2 %', v_6; + +return 1; +exception +when others then +raise info 'error is %',sqlerrm; +return 0; +end; +/ +ERROR: compile failed when parse the query: select c3 from check_tab1 order by c3 limit 6 +DETAIL: column "c3" does not exist +CONTEXT: compilation of PL/pgSQL function "check_f04" near line 11 +drop table check_tab1; reset behavior_compat_options; -- ref package cursor attr at first create table t1(a int, b int); @@ -2873,6 +2928,68 @@ drop procedure pp1; drop package pck1; NOTICE: drop cascades to function pkg_val_2.p1() drop table t1; +-- test package variable as in param +create type o1 as (a int, b varchar2); +create or replace package pck1 as +va varchar2; +vb varchar2 := 'vb'; +vc o1; +procedure p1; +end pck1; +/ +create or replace package body pck1 as +procedure p1 as +begin +va := 'hahahah'; +vc := (1,'cccc'); +end; +end pck1; +/ +create table testlog(a int); +create or replace procedure insertlog() is +PRAGMA AUTONOMOUS_TRANSACTION; +begin +insert into testlog values(1); +pck1.va := 'bbbbbbb'; +pck1.vc := (123,'ayayayayay'); +exception +when others then +raise notice 'sqlerrm:%',sqlerrm; +raise; +end; +/ +create or replace procedure p1(va in varchar2, vb in varchar2, vc in o1) is +begin +raise info 'before auto: %,%', vb,vc; +insertlog(); +raise info 'after auto: %,%', vb,vc; +exception +when others then +raise notice 'sqlerrm:%',sqlerrm; +raise; +end; +/ +declare +begin +pck1.p1(); +p1(150,pck1.va,pck1.vc); +raise info 'after p1: %,%', pck1.va,pck1.vc; +end; +/ +INFO: before auto: hahahah,(1,cccc) +CONTEXT: SQL statement "CALL p1(150,pck1.va,pck1.vc)" +PL/pgSQL function inline_code_block line 4 at PERFORM +INFO: after auto: hahahah,(1,cccc) +CONTEXT: SQL statement "CALL p1(150,pck1.va,pck1.vc)" +PL/pgSQL function inline_code_block line 4 at PERFORM +INFO: after p1: bbbbbbb,(123,ayayayayay) + +drop procedure p1; +drop procedure insertlog; +drop package pck1; +NOTICE: drop cascades to function pkg_val_2.p1() +drop table testlog; +drop type o1; -- clean DROP SCHEMA IF EXISTS pkg_val_1 CASCADE; DROP SCHEMA IF EXISTS pkg_val_2 CASCADE; diff --git a/src/test/regress/expected/hw_partition_add_drop_partition.out b/src/test/regress/expected/hw_partition_add_drop_partition.out index 879aa757e..c896f7235 100644 --- a/src/test/regress/expected/hw_partition_add_drop_partition.out +++ b/src/test/regress/expected/hw_partition_add_drop_partition.out @@ -105,7 +105,7 @@ SELECT p1.relname, p1.parttype, p1.partstrategy, p1.relfilenode!=0 hasfilenode, amount_sold | numeric(10,2) | | main | | Indexes: "range_sales_pkey" PRIMARY KEY, btree (customer_id) TABLESPACE pg_default - "range_sales_idx" btree (product_id) LOCAL(PARTITION time_2008_product_id_idx, PARTITION time_2009_product_id_idx, PARTITION time_2010_product_id_idx, PARTITION time_2011_product_id_idx, PARTITION time_2012_product_id_idx, PARTITION time_end_product_id_idx) TABLESPACE pg_default + "range_sales_idx" btree (product_id) LOCAL TABLESPACE pg_default Partition By RANGE(time_id) Number of partitions: 6 (View pg_partition to check each partition range.) Has OIDs: no @@ -177,7 +177,7 @@ SELECT p1.relname, p1.parttype, p1.partstrategy, p1.relfilenode!=0 hasfilenode, amount_sold | numeric(10,2) | | main | | Indexes: "range_sales_pkey" PRIMARY KEY, btree (customer_id) TABLESPACE pg_default UNUSABLE - "range_sales_idx" btree (product_id) LOCAL(PARTITION time_2008_product_id_idx, PARTITION time_2010_product_id_idx, PARTITION time_end_product_id_idx) TABLESPACE pg_default + "range_sales_idx" btree (product_id) LOCAL TABLESPACE pg_default Partition By RANGE(time_id) Number of partitions: 3 (View pg_partition to check each partition range.) Has OIDs: no @@ -286,7 +286,7 @@ SELECT p1.relname, p1.parttype, p1.partstrategy, p1.relfilenode!=0 hasfilenode, amount_sold | numeric(10,2) | | main | | Indexes: "range2_sales_pkey" PRIMARY KEY, btree (customer_id) TABLESPACE pg_default - "range2_sales_idx" btree (product_id) LOCAL(PARTITION time_2008_product_id_idx, PARTITION time_2009_product_id_idx, PARTITION time_2010_product_id_idx, PARTITION time_2011_product_id_idx, PARTITION time_2012_product_id_idx, PARTITION time_end_product_id_idx) TABLESPACE pg_default + "range2_sales_idx" btree (product_id) LOCAL TABLESPACE pg_default Partition By RANGE(time_id, product_id) Number of partitions: 6 (View pg_partition to check each partition range.) Has OIDs: no @@ -358,7 +358,7 @@ SELECT p1.relname, p1.parttype, p1.partstrategy, p1.relfilenode!=0 hasfilenode, amount_sold | numeric(10,2) | | main | | Indexes: "range2_sales_pkey" PRIMARY KEY, btree (customer_id) TABLESPACE pg_default UNUSABLE - "range2_sales_idx" btree (product_id) LOCAL(PARTITION time_2008_product_id_idx, PARTITION time_2010_product_id_idx, PARTITION time_end_product_id_idx) TABLESPACE pg_default + "range2_sales_idx" btree (product_id) LOCAL TABLESPACE pg_default Partition By RANGE(time_id, product_id) Number of partitions: 3 (View pg_partition to check each partition range.) Has OIDs: no @@ -451,7 +451,7 @@ SELECT p1.relname, p1.parttype, p1.partstrategy, p1.relfilenode!=0 hasfilenode, amount_sold | numeric(10,2) | | main | | Indexes: "interval_sales_pkey" PRIMARY KEY, btree (customer_id) TABLESPACE pg_default - "interval_sales_idx" btree (product_id) LOCAL(PARTITION time_2008_product_id_idx, PARTITION time_2009_product_id_idx, PARTITION time_2010_product_id_idx, PARTITION sys_p1_product_id_idx) TABLESPACE pg_default + "interval_sales_idx" btree (product_id) LOCAL TABLESPACE pg_default Partition By RANGE(time_id) INTERVAL('1 year') Number of partitions: 4 (View pg_partition to check each partition range.) Has OIDs: no @@ -513,7 +513,7 @@ SELECT p1.relname, p1.parttype, p1.partstrategy, p1.relfilenode!=0 hasfilenode, amount_sold | numeric(10,2) | | main | | Indexes: "interval_sales_pkey" PRIMARY KEY, btree (customer_id) TABLESPACE pg_default UNUSABLE - "interval_sales_idx" btree (product_id) LOCAL(PARTITION time_2008_product_id_idx, PARTITION time_2010_product_id_idx) TABLESPACE pg_default + "interval_sales_idx" btree (product_id) LOCAL TABLESPACE pg_default Partition By RANGE(time_id) INTERVAL('1 year') Number of partitions: 2 (View pg_partition to check each partition range.) Has OIDs: no @@ -625,7 +625,7 @@ SELECT p1.relname, p1.parttype, p1.partstrategy, p1.relfilenode!=0 hasfilenode, amount_sold | numeric(10,2) | | main | | Indexes: "list_sales_pkey" PRIMARY KEY, btree (customer_id) TABLESPACE pg_default - "list_sales_idx" btree (product_id) LOCAL(PARTITION channel_default_product_id_idx, PARTITION channel5_product_id_idx, PARTITION channel4_product_id_idx, PARTITION channel3_product_id_idx, PARTITION channel2_product_id_idx, PARTITION channel1_product_id_idx) TABLESPACE pg_default + "list_sales_idx" btree (product_id) LOCAL TABLESPACE pg_default Partition By LIST(channel_id) Number of partitions: 6 (View pg_partition to check each partition range.) Has OIDs: no @@ -697,7 +697,7 @@ SELECT p1.relname, p1.parttype, p1.partstrategy, p1.relfilenode!=0 hasfilenode, amount_sold | numeric(10,2) | | main | | Indexes: "list_sales_pkey" PRIMARY KEY, btree (customer_id) TABLESPACE pg_default UNUSABLE - "list_sales_idx" btree (product_id) LOCAL(PARTITION channel5_product_id_idx, PARTITION channel4_product_id_idx, PARTITION channel1_product_id_idx) TABLESPACE pg_default + "list_sales_idx" btree (product_id) LOCAL TABLESPACE pg_default Partition By LIST(channel_id) Number of partitions: 3 (View pg_partition to check each partition range.) Has OIDs: no @@ -798,7 +798,7 @@ SELECT p1.relname, p1.parttype, p1.partstrategy, p1.relfilenode!=0 hasfilenode, amount_sold | numeric(10,2) | | main | | Indexes: "hash_sales_pkey" PRIMARY KEY, btree (customer_id) TABLESPACE pg_default - "hash_sales_idx" btree (product_id) LOCAL(PARTITION product4_product_id_idx, PARTITION product3_product_id_idx, PARTITION product2_product_id_idx, PARTITION product1_product_id_idx) TABLESPACE pg_default + "hash_sales_idx" btree (product_id) LOCAL TABLESPACE pg_default Partition By HASH(product_id) Number of partitions: 4 (View pg_partition to check each partition range.) Has OIDs: no @@ -873,7 +873,7 @@ SELECT p1.relname, p1.parttype, p1.partstrategy, p1.relfilenode!=0 hasfilenode, amount_sold | numeric(10,2) | | main | | Indexes: "hash_sales_pkey" PRIMARY KEY, btree (customer_id) TABLESPACE pg_default - "hash_sales_idx" btree (product_id) LOCAL(PARTITION product4_product_id_idx, PARTITION product3_product_id_idx, PARTITION product2_product_id_idx, PARTITION product1_product_id_idx) TABLESPACE pg_default + "hash_sales_idx" btree (product_id) LOCAL TABLESPACE pg_default Partition By HASH(product_id) Number of partitions: 4 (View pg_partition to check each partition range.) Has OIDs: no diff --git a/src/test/regress/expected/null_test_opt.out b/src/test/regress/expected/null_test_opt.out new file mode 100755 index 000000000..7ffa097f8 --- /dev/null +++ b/src/test/regress/expected/null_test_opt.out @@ -0,0 +1,100 @@ +create schema null_test_opt_nsp; +set search_path = null_test_opt_nsp; +create table t(a int, b int not null); +insert into t values(1, 1); +explain (costs off) select * from t where b is null order by 1, 2; + QUERY PLAN +----------------------------------- + Sort + Sort Key: a, b + -> Result + One-Time Filter: false + -> Seq Scan on t + Filter: (b IS NULL) +(6 rows) + +select * from t where b is null order by 1, 2; + a | b +---+--- +(0 rows) + +explain (costs off) select * from t where b is not null order by 1, 2; + QUERY PLAN +--------------------- + Sort + Sort Key: a, b + -> Seq Scan on t +(3 rows) + +select * from t where b is not null order by 1, 2; + a | b +---+--- + 1 | 1 +(1 row) + +explain (costs off) select * from t where b is null or b is not null order by 1, 2; + QUERY PLAN +--------------------- + Sort + Sort Key: a, b + -> Seq Scan on t +(3 rows) + +select * from t where b is null or b is not null order by 1, 2; + a | b +---+--- + 1 | 1 +(1 row) + +explain (costs off) select * from t where b is null or a = 1 order by 1, 2; + QUERY PLAN +------------------------------------------ + Sort + Sort Key: a, b + -> Seq Scan on t + Filter: ((b IS NULL) OR (a = 1)) +(4 rows) + +select * from t where b is null or a = 1 order by 1, 2; + a | b +---+--- + 1 | 1 +(1 row) + +explain (costs off) select * from t where b is null and a = 1 order by 1, 2; + QUERY PLAN +------------------------------------------------- + Sort + Sort Key: b + -> Result + One-Time Filter: false + -> Seq Scan on t + Filter: ((b IS NULL) AND (a = 1)) +(6 rows) + +select * from t where b is null and a = 1 order by 1, 2; + a | b +---+--- +(0 rows) + +explain (costs off) select * from t tt1 join t tt2 on tt1.a = tt2.b where tt1.a is null; + QUERY PLAN +----------------------------------------- + Hash Join + Hash Cond: (tt1.a = tt2.b) + -> Seq Scan on t tt1 + Filter: (a IS NULL) + -> Hash + -> Result + One-Time Filter: false + -> Seq Scan on t tt2 + Filter: (b IS NULL) +(9 rows) + +select * from t tt1 join t tt2 on tt1.a = tt2.b where tt1.a is null; + a | b | a | b +---+---+---+--- +(0 rows) + +drop schema null_test_opt_nsp cascade; +NOTICE: drop cascades to table t diff --git a/src/test/regress/expected/row_compression/normal_test.out b/src/test/regress/expected/row_compression/normal_test.out index 51b8d5ce6..d15e31e87 100644 --- a/src/test/regress/expected/row_compression/normal_test.out +++ b/src/test/regress/expected/row_compression/normal_test.out @@ -125,7 +125,7 @@ create index on normal_test.tbl_partition(id) local WITH (compresstype=2,compres --------+---------+-----------+---------+--------------+------------- id | integer | | plain | | Indexes: - "tbl_partition_id_idx" btree (id) LOCAL(PARTITION p0_id_idx, PARTITION p10_id_idx, PARTITION p11_id_idx, PARTITION p2_id_idx, PARTITION p3_id_idx, PARTITION p4_id_idx, PARTITION p5_id_idx, PARTITION p6_id_idx, PARTITION p7_id_idx) WITH (compresstype=2, compress_chunk_size=1024) TABLESPACE pg_default + "tbl_partition_id_idx" btree (id) LOCAL WITH (compresstype=2, compress_chunk_size=1024) TABLESPACE pg_default --?.* --?.* Has OIDs: no diff --git a/src/test/regress/expected/sqlbypass_partition.out b/src/test/regress/expected/sqlbypass_partition.out index 07a405197..e2ef975c5 100755 --- a/src/test/regress/expected/sqlbypass_partition.out +++ b/src/test/regress/expected/sqlbypass_partition.out @@ -2641,5 +2641,95 @@ delete from test_bypass_sql_partition where col1 <= 11 and col1 >= 15; delete from test_bypass_sql_partition where col1 > 10; delete from test_bypass_sql_partition where col1 < 10; delete from test_bypass_sql_partition where col1 >= 10 and col1 <= 30; +create table t1( +crcrd_acg_setl_dt char(8) not null, +cst_id char(18), +multi_tenancy_id char(5) +) +partition by range (multi_tenancy_id, crcrd_acg_setl_dt) +( + partition p1 values less than ('CN000', '20191201'), + partition p2 values less than ('CN000', '20200201'), + partition p3 values less than ('CN000', '20200202'), + partition p4 values less than ('CN000', '20200203'), + partition p5 values less than ('CN000', '20200204'), + partition p6 values less than ('ZZZZZ', '21000101') +) +enable row movement +; + +create index on t1(crcrd_acg_setl_dt, cst_id, multi_tenancy_id) local; + +insert into t1 values('20200201', '107190000103394943', 'CN000'); +insert into t1 values('20200225', '107190000103394943', 'CN000'); +insert into t1 values('20200228', '107190000103394943', 'CN000'); +insert into t1 values('20200301', '107190000103394943', 'CN000'); +insert into t1 values('20200310', '107190000103394943', 'CN000'); + +set enable_seqscan = off; + +select max(crcrd_acg_setl_dt) from t1 where cst_id='107190000103394943' and multi_tenancy_id = 'CN000'; + max +---------- + 20200310 +(1 row) + + +prepare p1 as select max(crcrd_acg_setl_dt) from t1 where cst_id=$1 and multi_tenancy_id = $2; + +execute p1 ('107190000103394943','CN000'); + max +---------- + 20200310 +(1 row) + +deallocate p1; +drop table t1; +drop table test_range_pt; +ERROR: table "test_range_pt" does not exist +create table test_range_pt (a int, b int, c int) +partition by range(a) +( + partition p1 values less than (2000), + partition p2 values less than (3000), + partition p3 values less than (4000), + partition p4 values less than (5000), + partition p5 values less than (maxvalue) +)ENABLE ROW MOVEMENT; +insert into test_range_pt values(1,1),(2001,2),(3001,3),(4001,4),(5001,5); + +create index idx1 on test_range_pt(a) local; +prepare p1 as select max(a) from test_range_pt where a>$1; +execute p1 (1); + max +------ + 5001 +(1 row) + +deallocate p1; +select max(a) from test_range_pt where a>b+1; + max +------ + 5001 +(1 row) + +drop table test_range_pt; +drop table test_list_lt1; +ERROR: table "test_list_lt1" does not exist +create table test_list_lt1 (a int, b int ) +partition by list(a) +( + partition p1 values (2000), + partition p2 values (3000), + partition p3 values (4000) +) ; +prepare p1 as select * from test_list_lt1 where a = $1 and ctid = '(0,1)'; +execute p1 (1); + a | b +---+--- +(0 rows) + +deallocate p1; +drop table test_list_lt1; reset enable_partition_opfusion; drop table test_bypass_sql_partition; diff --git a/src/test/regress/expected/sw_bugfix-1.out b/src/test/regress/expected/sw_bugfix-1.out index 328ba3c16..8d72667db 100644 --- a/src/test/regress/expected/sw_bugfix-1.out +++ b/src/test/regress/expected/sw_bugfix-1.out @@ -430,6 +430,7 @@ explain (costs off) select t1.ID,t1.VCH,pid,NAME,PTEX from TEST_HCB_FQB t1,TEST_ QUERY PLAN ---------------------------------------------------------------------------- CTE Scan on tmp_reuslt + Filter: ("t1@id" = "t2@id") CTE tmp_reuslt -> StartWith Operator Start With pseudo atts: RUITR, array_key_1 @@ -448,7 +449,7 @@ explain (costs off) select t1.ID,t1.VCH,pid,NAME,PTEX from TEST_HCB_FQB t1,TEST_ -> WorkTable Scan on tmp_reuslt -> Hash -> Seq Scan on test_hcb_fqb t1 -(19 rows) +(20 rows) CREATE OR REPLACE FUNCTION test_hcb_pro1(i_id in int) return int AS @@ -1470,3 +1471,22 @@ DROP TABLE IF EXISTS start; DROP TABLE IF EXISTS connect; DROP TABLE IF EXISTS siblings; DROP TABLE IF EXISTS prior; +-- test where clause pushdown result correctness +create table xt1(id int, lid int, name text); +create table xt2(idd int, lidd int, name text); +insert into xt1 values(1,null,'A'),(2,1,'B'),(3,2,'C'); +insert into xt2 values(1,null,'A'),(2,1,'B'),(3,2,'C'), (4,3,'D'); +select * from xt2,xt1 where xt1.id=xt2.idd and xt1.id!=2 start with id=2 connect by prior id=lid; + idd | lidd | name | id | lid | name +-----+------+------+----+-----+------ + 3 | 2 | C | 3 | 2 | C +(1 row) + +select * from xt2,xt1 where xt1.id=xt2.idd and xt1.id=3 start with id=2 connect by prior id=lid; + idd | lidd | name | id | lid | name +-----+------+------+----+-----+------ + 3 | 2 | C | 3 | 2 | C +(1 row) + +drop table if exists xt1; +drop table if exists xt2; diff --git a/src/test/regress/expected/sw_bugfix-2.out b/src/test/regress/expected/sw_bugfix-2.out old mode 100644 new mode 100755 index 473e29f60..fc9981c42 --- a/src/test/regress/expected/sw_bugfix-2.out +++ b/src/test/regress/expected/sw_bugfix-2.out @@ -484,17 +484,18 @@ select t1.id, t1.pid, t1.name from t1 start with not id=1 connect by prior pid=i explain select trim(t1.name) from test_hcb_ptb t1 connect by trim(t1.name) is not null; QUERY PLAN -------------------------------------------------------------------------------------------------- - CTE Scan on tmp_reuslt (cost=13471.92..22689.36 rows=409664 width=178) + CTE Scan on tmp_reuslt (cost=13473.52..22690.96 rows=409664 width=178) CTE tmp_reuslt - -> StartWith Operator (cost=0.00..13471.92 rows=409664 width=102) + -> StartWith Operator (cost=0.00..13473.52 rows=409664 width=102) Start With pseudo atts: RUITR - -> Recursive Union (cost=0.00..13471.92 rows=409664 width=102) + -> Recursive Union (cost=0.00..13473.52 rows=409664 width=102) -> Seq Scan on test_hcb_ptb t1 (cost=0.00..2.64 rows=64 width=102) - -> Nested Loop (cost=0.00..527.60 rows=40960 width=102) + -> Nested Loop (cost=0.00..527.76 rows=40960 width=102) -> WorkTable Scan on tmp_reuslt (cost=0.00..12.80 rows=640 width=0) - -> Materialize (cost=0.00..2.96 rows=64 width=102) - -> Seq Scan on test_hcb_ptb t1 (cost=0.00..2.64 rows=64 width=102) -(10 rows) + -> Materialize (cost=0.00..3.12 rows=64 width=102) + -> Seq Scan on test_hcb_ptb t1 (cost=0.00..2.80 rows=64 width=102) + Filter: (btrim((name)::text) IS NOT NULL) +(11 rows) /* fix create table as with start with */ create table ct as select t1.id,t1.pid,t1.name,level from test_hcb_ptb t1 start with id=141 connect by prior id=pid; @@ -1083,6 +1084,57 @@ select t1.id,t1.pid,t1.name from test_hcb_ptb t1 start with not exists(select * 1 | 0 | 中国 (6 rows) +-- test sublibk pull is no allowed in swcb converted cases +explain (costs off) +select id,pid,level +from test_hcb_ptb +where exists ( + select id + from test_place t + where t.id=test_hcb_ptb.id +) +start with id=151 connect by prior pid=id; + QUERY PLAN +------------------------------------------------------------------------------------------- + CTE Scan on tmp_reuslt + Filter: (alternatives: SubPlan 2 or hashed SubPlan 3) + CTE tmp_reuslt + -> StartWith Operator + Start With pseudo atts: RUITR, array_key_9 + -> Recursive Union + -> Seq Scan on test_hcb_ptb + Filter: (id = 151) + -> Hash Join + Hash Cond: (swtest.test_hcb_ptb.id = tmp_reuslt."test_hcb_ptb@pid") + -> Seq Scan on test_hcb_ptb + -> Hash + -> WorkTable Scan on tmp_reuslt + SubPlan 2 + -> Seq Scan on test_place t + Filter: (id = tmp_reuslt."test_hcb_ptb@id") + SubPlan 3 + -> Seq Scan on test_place t +(18 rows) + +select id,pid,level +from test_hcb_ptb +where exists ( + select id + from test_place t + where t.id=test_hcb_ptb.id +) +start with id=151 connect by prior pid=id; + id | pid | level +-----+-----+------- + 151 | 141 | 1 + 141 | 131 | 2 + 131 | 121 | 3 + 121 | 111 | 4 + 111 | 11 | 5 + 11 | 1 | 6 + 1 | 0 | 7 +(7 rows) + drop table test_place; -- test where quals pushdown drop table if exists brand_sw3 cascade; @@ -1252,21 +1304,23 @@ explain performance select * from sw_dummy connect by level < 50; drop table sw_dummy; --test null pointers in connect by walker explain select * from t1 connect by exists(select distinct (select id from t1)); - QUERY PLAN -------------------------------------------------------------------------------------------- - CTE Scan on tmp_reuslt (cost=293.65..455.83 rows=8109 width=40) + QUERY PLAN +------------------------------------------------------------------------------------------------- + CTE Scan on tmp_reuslt (cost=293.75..455.93 rows=8109 width=40) CTE tmp_reuslt - -> StartWith Operator (cost=0.00..293.64 rows=8109 width=10) + -> StartWith Operator (cost=0.00..293.75 rows=8109 width=10) Start With pseudo atts: RUITR - -> Recursive Union (cost=0.00..293.64 rows=8109 width=10) + -> Recursive Union (cost=0.00..293.75 rows=8109 width=10) -> Seq Scan on t1 (cost=0.00..1.09 rows=9 width=10) - -> Nested Loop (cost=0.00..13.04 rows=810 width=10) - -> WorkTable Scan on tmp_reuslt (cost=0.00..1.80 rows=90 width=0) - -> Materialize (cost=0.00..1.14 rows=9 width=10) - -> Seq Scan on t1 (cost=0.00..1.09 rows=9 width=10) - InitPlan 2 (returns $2) - -> Result (cost=0.00..0.01 rows=1 width=0) -(12 rows) + -> Result (cost=0.01..13.05 rows=810 width=10) + One-Time Filter: $1 + InitPlan 1 (returns $1) + -> Result (cost=0.00..0.01 rows=1 width=0) + -> Nested Loop (cost=0.00..13.04 rows=810 width=10) + -> WorkTable Scan on tmp_reuslt (cost=0.00..1.80 rows=90 width=0) + -> Materialize (cost=0.00..1.14 rows=9 width=10) + -> Seq Scan on t1 (cost=0.00..1.09 rows=9 width=10) +(14 rows) --test join + where for start with .. connect by select t1.id,t1.pid,t2.id from test_hcb_ptb t1 join test_hcb_ptb t2 on t1.id=t2.id where t1.id>1 start with t1.id=141 connect by prior t2.id=t1.pid; @@ -1357,4 +1411,7 @@ WHERE true CONNECT BY EXISTS ( test_hcb_ptb as ref_7 ) LIMIT 169; -ERROR: Unsupported subquery found in connect by clause. + c0 +---- +(0 rows) + diff --git a/src/test/regress/expected/sw_icbc.out b/src/test/regress/expected/sw_icbc.out old mode 100644 new mode 100755 index d20a601a8..82fe61450 --- a/src/test/regress/expected/sw_icbc.out +++ b/src/test/regress/expected/sw_icbc.out @@ -452,8 +452,8 @@ where not exists (select 1 from t1 where test.id = t1.id) start with test.id = 1 connect by prior test.id = test.pid; QUERY PLAN ------------------------------------------------------------------------------------------------- - Hash Anti Join (cost=19.42..22.29 rows=82 width=40) - Hash Cond: (tmp_reuslt."test@id" = t1.id) + CTE Scan on tmp_reuslt (cost=18.22..121.28 rows=46 width=40) + Filter: (NOT (alternatives: SubPlan 2 or hashed SubPlan 3)) CTE tmp_reuslt -> StartWith Operator (cost=0.00..18.22 rows=91 width=10) Start With pseudo atts: RUITR, array_key_1 @@ -465,16 +465,19 @@ start with test.id = 1 connect by prior test.id = test.pid; -> Seq Scan on t1 test (cost=0.00..1.09 rows=9 width=10) -> Hash (cost=0.20..0.20 rows=10 width=4) -> WorkTable Scan on tmp_reuslt (cost=0.00..0.20 rows=10 width=4) - -> CTE Scan on tmp_reuslt (cost=0.00..1.82 rows=91 width=40) - -> Hash (cost=1.09..1.09 rows=9 width=4) - -> Seq Scan on t1 (cost=0.00..1.09 rows=9 width=4) -(16 rows) + SubPlan 2 + -> Seq Scan on t1 (cost=0.00..1.11 rows=1 width=0) + Filter: (tmp_reuslt."test@id" = id) + SubPlan 3 + -> Seq Scan on t1 (cost=0.00..1.09 rows=9 width=4) +(18 rows) --multiple tables case explain (costs off) select * from t1, t2 where t1.id = t2.id start with t1.id = t2.id and t1.id = 1 connect by prior t1.id = t1.pid; QUERY PLAN ----------------------------------------------------------------------------- CTE Scan on tmp_reuslt + Filter: ("t1@id" = "t2@id") CTE tmp_reuslt -> StartWith Operator Start With pseudo atts: RUITR, array_key_1 @@ -493,7 +496,7 @@ explain (costs off) select * from t1, t2 where t1.id = t2.id start with t1.id = -> WorkTable Scan on tmp_reuslt -> Hash -> Seq Scan on t2 -(19 rows) +(20 rows) explain (costs off) select * from t1 join t2 on t1.id = t2.id start with t1.id = t2.id and t1.id = 1 connect by prior t1.id = t1.pid; QUERY PLAN @@ -523,6 +526,7 @@ explain (costs off) select * from t1, (select * from t2) as test where t1.id = t QUERY PLAN ----------------------------------------------------------------------------- CTE Scan on tmp_reuslt + Filter: ("t1@id" = "test@id") CTE tmp_reuslt -> StartWith Operator Start With pseudo atts: RUITR, array_key_1 @@ -541,7 +545,7 @@ explain (costs off) select * from t1, (select * from t2) as test where t1.id = t -> WorkTable Scan on tmp_reuslt -> Hash -> Seq Scan on t2 -(19 rows) +(20 rows) explain (costs off) select id, (select id from t2 start with t2.id = t1.id connect by t2.id = t1.id limit 1) from t1 where id = 1; ERROR: START WITH CONNECT BY clauses must have at least one prior key. diff --git a/src/test/regress/input/db4ai_kmeans_train_predict.source b/src/test/regress/input/db4ai_kmeans_train_predict.source index 05e38f476..9d0ab298e 100644 --- a/src/test/regress/input/db4ai_kmeans_train_predict.source +++ b/src/test/regress/input/db4ai_kmeans_train_predict.source @@ -1201,7 +1201,7 @@ CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivar -- Batch size CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivariate_7_1000_10) WITH max_iterations = 50, num_centroids = 10, tolerance = 0.00001, batch_size = 0, num_features = 7, distance_function = 'L2_Squared', seeding_function = 'Random++', verbose = 1, seed = 1255025990; -CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivariate_7_1000_10) WITH max_iterations = 50, num_centroids = 10, tolerance = 0.00001, batch_size = 1000001, num_features = 7, distance_function = 'L2_Squared', seeding_function = 'Random++', verbose = 1, seed = 1255025990; +CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivariate_7_1000_10) WITH max_iterations = 50, num_centroids = 10, tolerance = 0.00001, batch_size = 1048576, num_features = 7, distance_function = 'L2_Squared', seeding_function = 'Random++', verbose = 1, seed = 1255025990; -- Num of features (not matching the data) CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivariate_7_1000_10) WITH max_iterations = 50, num_centroids = 10, tolerance = 0.00001, batch_size = 1000, num_features = 9, distance_function = 'L2_Squared', seeding_function = 'Random++', verbose = 1, seed = 1255025990; diff --git a/src/test/regress/input/db4ai_xgboost_train_predict.source b/src/test/regress/input/db4ai_xgboost_train_predict.source index cb3d7075e..74eeac421 100644 --- a/src/test/regress/input/db4ai_xgboost_train_predict.source +++ b/src/test/regress/input/db4ai_xgboost_train_predict.source @@ -22,6 +22,7 @@ CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET r CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH eta=-0.1; CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH seed=-1; CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH nthread=-1; +CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH nthread=101; CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH booster=10; CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH tree_method=10; CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH verbosity=10; diff --git a/src/test/regress/input/hw_audit_multi_thread_rotation_interval.source b/src/test/regress/input/hw_audit_multi_thread_rotation_interval.source new file mode 100644 index 000000000..1b2b638c3 --- /dev/null +++ b/src/test/regress/input/hw_audit_multi_thread_rotation_interval.source @@ -0,0 +1,28 @@ +-- 设置guc参数 时间轮询 线程数为3 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_rotation_interval=1" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num=3" > /dev/null 2>&1 + +-- 重启数据库 等待1s +select pg_sleep(1); +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 + +-- 查看线程数是否改为 3 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num"; + +-- 间隔60s 获取pg_audit 中最新adt文件编号 查看是否生成了6个adt文件 +\! num1=$(expr $(ls @abs_srcdir@/tmp_check/datanode1/pg_audit -l | grep "^-" | wc -l) - 2) && sleep 1m && num2=$(expr $(ls @abs_srcdir@/tmp_check/datanode1/pg_audit -l | grep "^-" | wc -l) - 2) && diff=$(expr $num2 - $num1) && echo $diff && [[ $diff == 3 ]] && echo 'add a new log after interval-- 60 seconds' || echo 'fail to add new logs' + +-- 恢复guc参数 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_rotation_interval" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num" > /dev/null 2>&1 + +-- 重启数据库 等待1s +\! sleep 1 +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 + +-- 查看线程数是否恢复 1 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num"; diff --git a/src/test/regress/input/hw_audit_multi_thread_rotation_size.source b/src/test/regress/input/hw_audit_multi_thread_rotation_size.source new file mode 100644 index 000000000..265181091 --- /dev/null +++ b/src/test/regress/input/hw_audit_multi_thread_rotation_size.source @@ -0,0 +1,36 @@ +-- 设置guc 参数 记录DML审计日志 线程数为3 轮询空间大小2.1M +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_rotation_size=2048" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_dml_state=1" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num=3" > /dev/null 2>&1 + +-- 重启数据库 等待1s +select pg_sleep(1); +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 + +-- 查看线程数是否改为 3 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num"; + + +-- 产生dml_action 审计日志 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c " CREATE TABLE T_TEST_MULTI_THREAD_ROTATION_SIZE(COL1 int4 DEFAULT 1,COL2 VARCHAR(1024) DEFAULT 'test_rotation_size')"; +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "CREATE OR REPLACE PROCEDURE TRANSACTION_MULTI_THREAD_ROTATION_SIZE() AS BEGIN FOR i IN 0..1000000 LOOP INSERT INTO T_TEST_MULTI_THREAD_ROTATION_SIZE(COL1, COL2) VALUES (i, 'test_multi_thread_rotation_size'); COMMIT; END LOOP; END;"; +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "CALL TRANSACTION_MULTI_THREAD_ROTATION_SIZE()"; + +-- 提取新生成的文件大小 与2.1M比较 全部小于2.1M 为执行成功 +\! flag=0 && for i in $(find @abs_srcdir@/tmp_check/datanode1/pg_audit -newermt $(date -d "-75 seconds" +%H:%M:%S) -name "*_adt"); do size=$(du -h --exclude=index_table_new $i | grep -oP '\d*\.\d+M'); if [[ $size > '2.1M' ]]; then flag=1; echo $size; echo $i; fi; done && [[ "$flag" == 0 ]] && echo 'all the logs are less than 2.1M' || echo 'error -- some logs exceed limit' + +-- 恢复guc参数 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_rotation_size" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_dml_state" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num" > /dev/null 2>&1 + +-- 重启数据库 等待1s +\! sleep 1 +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 + +-- 查看线程数是否恢复 1 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num"; diff --git a/src/test/regress/input/hw_audit_multi_thread_space.source b/src/test/regress/input/hw_audit_multi_thread_space.source new file mode 100644 index 000000000..4a588a742 --- /dev/null +++ b/src/test/regress/input/hw_audit_multi_thread_space.source @@ -0,0 +1,47 @@ +-- 设置 guc参数 记录 dml 审计日志 & 审计日志上限 512M & 基于空间 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_resource_policy = 1" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_space_limit = 512MB" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_dml_state= 1" > /dev/null 2>&1 + +-- 修改 guc参数 线程从1 增加到 3 & 审计SCHEMA USER 对象的CREATE、DROP、ALTER操作 & 不记录set操作审计日志 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num=3" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_system_object=6" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_set_parameter=0" > /dev/null 2>&1 + +-- 重启数据库 等待1s +\! sleep 1 +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 + +-- 查看是否线程数修改成功 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num;"; + +-- 产生dml_action 审计日志 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "CREATE TABLE T_TEST_MULTI_THREAD_SPACE(COL1 int4 DEFAULT 1, COL2 VARCHAR(1024) DEFAULT 'test_multi_thread_space', COL3 int4 DEFAULT 1, COL4 text DEFAULT 'null')"; +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "CREATE OR REPLACE PROCEDURE TRANSACTION_TEST_MULTI_THREAD_SPACE() AS BEGIN FOR i IN 0..3000000 LOOP INSERT INTO T_TEST_MULTI_THREAD_SPACE(COL1, COL2,COL3) VALUES (i, 'test_multi_thread_space',2); COMMIT; END LOOP; END;"; +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c " CALL TRANSACTION_TEST_MULTI_THREAD_SPACE();"; + +-- 查看 pg_audit 文件总大小 是否超过设置的512M +\! (( $(du -h --exclude=done @abs_srcdir@/tmp_check/datanode1/pg_audit | grep -oP '\d*M' | grep -oP '\d*') > 530 )) && echo 'size of total logs exceeds upper limit' || echo 'size of total logs not exceeds upper limit --512M' + +-- 获取 pg_audit 中最早建立的adt文件编号 与原编号 0_adt 比较 观察是否删除了最早的日志文件 +\! [[ $(echo $(ls @abs_srcdir@/tmp_check/datanode1/pg_audit -tr | head -2 | xargs) | tr -cd "[0-9]") > 1 ]] && echo 'delete oldest files' || echo 'fail to delete oldest files' + +-- 恢复guc参数 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_resource_policy" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_space_limit" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_dml_state" > /dev/null 2>&1 + +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_system_object" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_set_parameter" > /dev/null 2>&1 + +-- 重启数据库 等待1s +\! sleep 1 +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 + +-- 再次查看是否线程数修改成功 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num;"; diff --git a/src/test/regress/input/hw_audit_pg_query.source b/src/test/regress/input/hw_audit_pg_query.source new file mode 100644 index 000000000..79fa7e73d --- /dev/null +++ b/src/test/regress/input/hw_audit_pg_query.source @@ -0,0 +1,30 @@ +-- 显示允许审计日志记录状态 +show audit_enabled; + +-- 修改guc参数 允许审计日志记录状态设为off 重启 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1 -c "audit_enabled=off" > /dev/null 2>&1 + +-- 重启数据库 等待1s +select pg_sleep(1); +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1 > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1 > /dev/null 2>&1 +\! sleep 5 + +-- 显示允许审计日志记录状态 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_enabled"; + +-- 在审计日志记录关闭状态下 新建用户 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "CREATE USER test_pg_query PASSWORD 'test_pwd@123'"; + +-- 查询审计日志 create user +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "select object_name,detail_info from pg_query_audit('2002-03-09 00:00:00','2032-03-09 24:00:00') where detail_info like 'CREATE USER%'"; + +-- 恢复guc参数 审计日志记录状态 重启 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1 -c "audit_enabled" > /dev/null 2>&1 +\! sleep 1 +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1 > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1 > /dev/null 2>&1 +\! sleep 5 + +-- 显示允许审计日志记录状态 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_enabled"; diff --git a/src/test/regress/make_fastcheck_audit_postgresql.conf b/src/test/regress/make_fastcheck_audit_postgresql.conf new file mode 100644 index 000000000..64c900d85 --- /dev/null +++ b/src/test/regress/make_fastcheck_audit_postgresql.conf @@ -0,0 +1,40 @@ +fsync = off +synchronous_commit = off +archive_mode = off +audit_user_violation = 1 +audit_system_object = 511 +audit_dml_state = 1 +audit_function_exec = 1 +audit_copy_exec = 1 +full_page_writes = off +wal_keep_segments = 50 +checkpoint_segments = 16 +checkpoint_timeout = 30min +enable_bbox_dump = off +bbox_dump_count = 4 +bbox_dump_path = '/tmp/invalidpath' +comm_tcp_mode = on +#comm_cn_dn_logic_conn = false +enable_absolute_tablespace = true +#enable_dynamic_workload = false +max_connections = 1000 +auth_iteration_count=2048 +enable_sonic_hashagg=on +enable_sonic_hashjoin=on +enable_cbm_tracking = on +enable_opfusion=on +uncontrolled_memory_context='HashCacheContext,TupleHashTable,TupleSort,AggContext,SRF multi-call context,CteScan*,FunctionScan*,RemoteQuery*,VecAgg*,HashContext,TopTransactionContext' +#enable_tsdb = on +enable_thread_pool = on +enable_default_cfunc_libpath = off +enable_stateless_pooler_reuse = on +enable_ustore = on +wal_level = logical +sql_beta_feature = 'a_style_coerce' +enable_global_syscache = on + +# for audit log management, multi thread cases +max_process_memory = 64GB +shared_buffers = 500MB +work_mem = 128MB +query_mem='500MB' diff --git a/src/test/regress/output/db4ai_gd_train_predict.source b/src/test/regress/output/db4ai_gd_train_predict.source index 65ef18577..4d4501e35 100644 --- a/src/test/regress/output/db4ai_gd_train_predict.source +++ b/src/test/regress/output/db4ai_gd_train_predict.source @@ -14,13 +14,13 @@ CREATE MODEL m using logistic_regression FEATURES size,lot FROM db4ai_houses; ERROR: Supervised ML algorithms require TARGET clause -- Errors with semantic validation of hyperparameters CREATE MODEL m USING logistic_regression FEATURES size, lot TARGET price <100000 FROM db4ai_houses with batch_size = 0, seed=1; -ERROR: Hyperparameter batch_size must be in the range [1,2147483647] +ERROR: Hyperparameter batch_size must be in the range [1,1048575] CREATE MODEL m USING logistic_regression FEATURES size, lot TARGET price <100000 FROM db4ai_houses with decay = 0.0, seed=1; ERROR: Hyperparameter decay must be in the range (0,1.7976931e+308] CREATE MODEL m USING logistic_regression FEATURES size, lot TARGET price <100000 FROM db4ai_houses with learning_rate = 0.0, seed=1; ERROR: Hyperparameter learning_rate must be in the range (0,1.7976931e+308] CREATE MODEL m USING logistic_regression FEATURES size, lot TARGET price <100000 FROM db4ai_houses with max_iterations = 0, seed=1; -ERROR: Hyperparameter max_iterations must be in the range [1,2147483647] +ERROR: Hyperparameter max_iterations must be in the range [1,10000] CREATE MODEL m USING logistic_regression FEATURES size, lot TARGET price <100000 FROM db4ai_houses with max_seconds = -1, seed=1; ERROR: Hyperparameter max_seconds must be in the range [0,2147483647] CREATE MODEL m USING logistic_regression FEATURES size, lot TARGET price <100000 FROM db4ai_houses with optimizer = nogd, seed=1; diff --git a/src/test/regress/output/db4ai_kmeans_train_predict.source b/src/test/regress/output/db4ai_kmeans_train_predict.source index 4aa618da7..64e5bee71 100644 --- a/src/test/regress/output/db4ai_kmeans_train_predict.source +++ b/src/test/regress/output/db4ai_kmeans_train_predict.source @@ -742,7 +742,7 @@ CONTEXT: referenced column: centroid_id -- Wrong parameters -- Number of iterations CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivariate_7_1000_10) WITH max_iterations = 0, num_centroids = 10, tolerance = 0.00001, batch_size = 1000, num_features = 7, distance_function = 'L2_Squared', seeding_function = 'Random++', verbose = 1, seed = 1255025990; -ERROR: Hyperparameter max_iterations must be in the range [1,2147483647] +ERROR: Hyperparameter max_iterations must be in the range [1,10000] -- Number of centroids CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivariate_7_1000_10) WITH max_iterations = 50, num_centroids = 0, tolerance = 0.00001, batch_size = 1000, num_features = 7, distance_function = 'L2_Squared', seeding_function = 'Random++', verbose = 1, seed = 1255025990; ERROR: Hyperparameter num_centroids must be in the range [1,1000000] @@ -755,9 +755,9 @@ CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivar ERROR: Hyperparameter tolerance must be in the range (0,1] -- Batch size CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivariate_7_1000_10) WITH max_iterations = 50, num_centroids = 10, tolerance = 0.00001, batch_size = 0, num_features = 7, distance_function = 'L2_Squared', seeding_function = 'Random++', verbose = 1, seed = 1255025990; -ERROR: Hyperparameter batch_size must be in the range [1,1000000] -CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivariate_7_1000_10) WITH max_iterations = 50, num_centroids = 10, tolerance = 0.00001, batch_size = 1000001, num_features = 7, distance_function = 'L2_Squared', seeding_function = 'Random++', verbose = 1, seed = 1255025990; -ERROR: Hyperparameter batch_size must be in the range [1,1000000] +ERROR: Hyperparameter batch_size must be in the range [1,1048575] +CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivariate_7_1000_10) WITH max_iterations = 50, num_centroids = 10, tolerance = 0.00001, batch_size = 1048576, num_features = 7, distance_function = 'L2_Squared', seeding_function = 'Random++', verbose = 1, seed = 1255025990; +ERROR: Hyperparameter batch_size must be in the range [1,1048575] -- Num of features (not matching the data) CREATE MODEL my_kmeans_pp_empty USING kmeans FROM (SELECT position FROM multivariate_7_1000_10) WITH max_iterations = 50, num_centroids = 10, tolerance = 0.00001, batch_size = 1000, num_features = 9, distance_function = 'L2_Squared', seeding_function = 'Random++', verbose = 1, seed = 1255025990; NOTICE: *** Initial statistics gathered: diff --git a/src/test/regress/output/db4ai_xgboost_train_predict.source b/src/test/regress/output/db4ai_xgboost_train_predict.source index b0124609c..8c245b27a 100644 --- a/src/test/regress/output/db4ai_xgboost_train_predict.source +++ b/src/test/regress/output/db4ai_xgboost_train_predict.source @@ -12,9 +12,9 @@ CREATE MODEL m using xgboost_binary_logistic FROM db4ai_rain; ERROR: Supervised ML algorithms require FEATURES clause -- Errors with semantic validation of hyperparameters CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH n_iter=-1; -ERROR: Hyperparameter n_iter must be in the range [1,2147483647] +ERROR: Hyperparameter n_iter must be in the range [1,10000] CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH batch_size=0; -ERROR: Hyperparameter batch_size must be in the range [1,2147483647] +ERROR: Hyperparameter batch_size must be in the range [1,1048575] CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH max_depth=-1; ERROR: Hyperparameter max_depth must be in the range [0,2147483647] CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH min_child_weight=-1; @@ -24,7 +24,9 @@ ERROR: Hyperparameter eta must be in the range [0,1] CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH seed=-1; ERROR: Hyperparameter seed must be in the range [0,2147483647] CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH nthread=-1; -ERROR: Hyperparameter nthread must be in the range [0,2147483647] +ERROR: Hyperparameter nthread must be in the range [0,100] +CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH nthread=101; +ERROR: Hyperparameter nthread must be in the range [0,100] CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH booster=10; ERROR: Hyperparameter booster must be a string CREATE MODEL m USING xgboost_binary_logistic FEATURES rainfall, temp9am TARGET raintoday FROM db4ai_rain WITH tree_method=10; diff --git a/src/test/regress/output/hw_audit_multi_thread_rotation_interval.source b/src/test/regress/output/hw_audit_multi_thread_rotation_interval.source new file mode 100644 index 000000000..bc9d230b5 --- /dev/null +++ b/src/test/regress/output/hw_audit_multi_thread_rotation_interval.source @@ -0,0 +1,39 @@ +-- 设置guc参数 时间轮询 线程数为3 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_rotation_interval=1" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num=3" > /dev/null 2>&1 +-- 重启数据库 等待1s +select pg_sleep(1); + pg_sleep +---------- + +(1 row) + +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +-- 查看线程数是否改为 3 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num"; + audit_thread_num +------------------ + 3 +(1 row) + +-- 间隔60s 获取pg_audit 中最新adt文件编号 查看是否生成了6个adt文件 +\! num1=$(expr $(ls @abs_srcdir@/tmp_check/datanode1/pg_audit -l | grep "^-" | wc -l) - 2) && sleep 1m && num2=$(expr $(ls @abs_srcdir@/tmp_check/datanode1/pg_audit -l | grep "^-" | wc -l) - 2) && diff=$(expr $num2 - $num1) && echo $diff && [[ $diff == 3 ]] && echo 'add a new log after interval-- 60 seconds' || echo 'fail to add new logs' +3 +add a new log after interval-- 60 seconds +-- 恢复guc参数 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_rotation_interval" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num" > /dev/null 2>&1 +-- 重启数据库 等待1s +\! sleep 1 +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +-- 查看线程数是否恢复 1 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num"; + audit_thread_num +------------------ + 1 +(1 row) + diff --git a/src/test/regress/output/hw_audit_multi_thread_rotation_size.source b/src/test/regress/output/hw_audit_multi_thread_rotation_size.source new file mode 100644 index 000000000..34baf60ca --- /dev/null +++ b/src/test/regress/output/hw_audit_multi_thread_rotation_size.source @@ -0,0 +1,51 @@ +-- 设置guc 参数 记录DML审计日志 线程数为3 轮询空间大小2.1M +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_rotation_size=2048" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_dml_state=1" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num=3" > /dev/null 2>&1 +-- 重启数据库 等待1s +select pg_sleep(1); + pg_sleep +---------- + +(1 row) + +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +-- 查看线程数是否改为 3 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num"; + audit_thread_num +------------------ + 3 +(1 row) + +-- 产生dml_action 审计日志 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c " CREATE TABLE T_TEST_MULTI_THREAD_ROTATION_SIZE(COL1 int4 DEFAULT 1,COL2 VARCHAR(1024) DEFAULT 'test_rotation_size')"; +CREATE TABLE +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "CREATE OR REPLACE PROCEDURE TRANSACTION_MULTI_THREAD_ROTATION_SIZE() AS BEGIN FOR i IN 0..1000000 LOOP INSERT INTO T_TEST_MULTI_THREAD_ROTATION_SIZE(COL1, COL2) VALUES (i, 'test_multi_thread_rotation_size'); COMMIT; END LOOP; END;"; +CREATE PROCEDURE +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "CALL TRANSACTION_MULTI_THREAD_ROTATION_SIZE()"; + transaction_multi_thread_rotation_size +---------------------------------------- + +(1 row) + +-- 提取新生成的文件大小 与2.1M比较 全部小于2.1M 为执行成功 +\! flag=0 && for i in $(find @abs_srcdir@/tmp_check/datanode1/pg_audit -newermt $(date -d "-75 seconds" +%H:%M:%S) -name "*_adt"); do size=$(du -h --exclude=index_table_new $i | grep -oP '\d*\.\d+M'); if [[ $size > '2.1M' ]]; then flag=1; echo $size; echo $i; fi; done && [[ "$flag" == 0 ]] && echo 'all the logs are less than 2.1M' || echo 'error -- some logs exceed limit' +all the logs are less than 2.1M +-- 恢复guc参数 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_rotation_size" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_dml_state" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num" > /dev/null 2>&1 +-- 重启数据库 等待1s +\! sleep 1 +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +-- 查看线程数是否恢复 1 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num"; + audit_thread_num +------------------ + 1 +(1 row) + diff --git a/src/test/regress/output/hw_audit_multi_thread_space.source b/src/test/regress/output/hw_audit_multi_thread_space.source new file mode 100644 index 000000000..36a011521 --- /dev/null +++ b/src/test/regress/output/hw_audit_multi_thread_space.source @@ -0,0 +1,56 @@ +-- 设置 guc参数 记录 dml 审计日志 & 审计日志上限 512M & 基于空间 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_resource_policy = 1" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_space_limit = 512MB" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_dml_state= 1" > /dev/null 2>&1 +-- 修改 guc参数 线程从1 增加到 3 & 审计SCHEMA USER 对象的CREATE、DROP、ALTER操作 & 不记录set操作审计日志 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num=3" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_system_object=6" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_set_parameter=0" > /dev/null 2>&1 +-- 重启数据库 等待1s +\! sleep 1 +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +-- 查看是否线程数修改成功 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num;"; + audit_thread_num +------------------ + 3 +(1 row) + +-- 产生dml_action 审计日志 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "CREATE TABLE T_TEST_MULTI_THREAD_SPACE(COL1 int4 DEFAULT 1, COL2 VARCHAR(1024) DEFAULT 'test_multi_thread_space', COL3 int4 DEFAULT 1, COL4 text DEFAULT 'null')"; +CREATE TABLE +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "CREATE OR REPLACE PROCEDURE TRANSACTION_TEST_MULTI_THREAD_SPACE() AS BEGIN FOR i IN 0..3000000 LOOP INSERT INTO T_TEST_MULTI_THREAD_SPACE(COL1, COL2,COL3) VALUES (i, 'test_multi_thread_space',2); COMMIT; END LOOP; END;"; +CREATE PROCEDURE +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c " CALL TRANSACTION_TEST_MULTI_THREAD_SPACE();"; + transaction_test_multi_thread_space +------------------------------------- + +(1 row) + +-- 查看 pg_audit 文件总大小 是否超过设置的512M +\! (( $(du -h --exclude=done @abs_srcdir@/tmp_check/datanode1/pg_audit | grep -oP '\d*M' | grep -oP '\d*') > 530 )) && echo 'size of total logs exceeds upper limit' || echo 'size of total logs not exceeds upper limit --512M' +size of total logs not exceeds upper limit --512M +-- 获取 pg_audit 中最早建立的adt文件编号 与原编号 0_adt 比较 观察是否删除了最早的日志文件 +\! [[ $(echo $(ls @abs_srcdir@/tmp_check/datanode1/pg_audit -tr | head -2 | xargs) | tr -cd "[0-9]") > 1 ]] && echo 'delete oldest files' || echo 'fail to delete oldest files' +delete oldest files +-- 恢复guc参数 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_resource_policy" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_space_limit" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_dml_state" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_thread_num" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_system_object" > /dev/null 2>&1 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1/ -c "audit_set_parameter" > /dev/null 2>&1 +-- 重启数据库 等待1s +\! sleep 1 +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1/ > /dev/null 2>&1 +-- 再次查看是否线程数修改成功 +\! sleep 5 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_thread_num;"; + audit_thread_num +------------------ + 1 +(1 row) + diff --git a/src/test/regress/output/hw_audit_pg_query.source b/src/test/regress/output/hw_audit_pg_query.source new file mode 100644 index 000000000..2f61f9361 --- /dev/null +++ b/src/test/regress/output/hw_audit_pg_query.source @@ -0,0 +1,48 @@ +-- 显示允许审计日志记录状态 +show audit_enabled; + audit_enabled +--------------- + on +(1 row) + +-- 修改guc参数 允许审计日志记录状态设为off 重启 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1 -c "audit_enabled=off" > /dev/null 2>&1 +-- 重启数据库 等待1s +select pg_sleep(1); + pg_sleep +---------- + +(1 row) + +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1 > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1 > /dev/null 2>&1 +\! sleep 5 +-- 显示允许审计日志记录状态 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_enabled"; + audit_enabled +--------------- + off +(1 row) + +-- 在审计日志记录关闭状态下 新建用户 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "CREATE USER test_pg_query PASSWORD 'test_pwd@123'"; +CREATE ROLE +-- 查询审计日志 create user +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "select object_name,detail_info from pg_query_audit('2002-03-09 00:00:00','2032-03-09 24:00:00') where detail_info like 'CREATE USER%'"; + object_name | detail_info +-------------+------------- +(0 rows) + +-- 恢复guc参数 审计日志记录状态 重启 +\! @abs_bindir@/gs_guc reload -Z datanode -D @abs_srcdir@/tmp_check/datanode1 -c "audit_enabled" > /dev/null 2>&1 +\! sleep 1 +\! @abs_bindir@/gs_ctl stop -D @abs_srcdir@/tmp_check/datanode1 > /dev/null 2>&1 +\! @abs_bindir@/gs_ctl start -D @abs_srcdir@/tmp_check/datanode1 > /dev/null 2>&1 +\! sleep 5 +-- 显示允许审计日志记录状态 +\! @abs_bindir@/gsql -r -p @portstring@ -d postgres -c "show audit_enabled"; + audit_enabled +--------------- + on +(1 row) + diff --git a/src/test/regress/output/hw_partition_interval_dump_restore.source b/src/test/regress/output/hw_partition_interval_dump_restore.source index 78e45ec66..f1ff322ba 100644 --- a/src/test/regress/output/hw_partition_interval_dump_restore.source +++ b/src/test/regress/output/hw_partition_interval_dump_restore.source @@ -38,7 +38,7 @@ end; birthday | timestamp without time zone | not null age | integer | Indexes: - "index_part_auto" btree (id) LOCAL(PARTITION idx_p1, PARTITION idx_p2, PARTITION sys_p1_id_idx, PARTITION sys_p2_id_idx, PARTITION sys_p3_id_idx, PARTITION sys_p4_id_idx, PARTITION sys_p5_id_idx, PARTITION sys_p6_id_idx, PARTITION sys_p7_id_idx, PARTITION sys_p8_id_idx, PARTITION sys_p9_id_idx, PARTITION sys_p10_id_idx, PARTITION sys_p11_id_idx, PARTITION sys_p12_id_idx, PARTITION sys_p13_id_idx, PARTITION sys_p14_id_idx, PARTITION sys_p15_id_idx, PARTITION sys_p16_id_idx, PARTITION sys_p17_id_idx, PARTITION sys_p18_id_idx, PARTITION sys_p19_id_idx, PARTITION sys_p20_id_idx, PARTITION sys_p21_id_idx, PARTITION sys_p22_id_idx, PARTITION sys_p23_id_idx, PARTITION sys_p24_id_idx, PARTITION sys_p25_id_idx, PARTITION sys_p26_id_idx, PARTITION sys_p27_id_idx, PARTITION sys_p28_id_idx, PARTITION sys_p29_id_idx, PARTITION sys_p30_id_idx, PARTITION sys_p31_id_idx, PARTITION sys_p32_id_idx, PARTITION sys_p33_id_idx, PARTITION sys_p34_id_idx, PARTITION sys_p35_id_idx, PARTITION sys_p36_id_idx, PARTITION sys_p37_id_idx, PARTITION sys_p38_id_idx, PARTITION sys_p39_id_idx, PARTITION sys_p40_id_idx, PARTITION sys_p41_id_idx, PARTITION sys_p42_id_idx, PARTITION sys_p43_id_idx, PARTITION sys_p44_id_idx, PARTITION sys_p45_id_idx, PARTITION sys_p46_id_idx, PARTITION sys_p47_id_idx, PARTITION sys_p48_id_idx, PARTITION sys_p49_id_idx, PARTITION sys_p50_id_idx, PARTITION sys_p51_id_idx, PARTITION sys_p52_id_idx, PARTITION sys_p53_id_idx, PARTITION sys_p54_id_idx, PARTITION sys_p55_id_idx, PARTITION sys_p56_id_idx, PARTITION sys_p57_id_idx, PARTITION sys_p58_id_idx, PARTITION sys_p59_id_idx, PARTITION sys_p60_id_idx, PARTITION sys_p61_id_idx, PARTITION sys_p62_id_idx, PARTITION sys_p63_id_idx, PARTITION sys_p64_id_idx, PARTITION sys_p65_id_idx, PARTITION sys_p66_id_idx, PARTITION sys_p67_id_idx, PARTITION sys_p68_id_idx, PARTITION sys_p69_id_idx, PARTITION sys_p70_id_idx, PARTITION sys_p71_id_idx, PARTITION sys_p72_id_idx, PARTITION sys_p73_id_idx, PARTITION sys_p74_id_idx, PARTITION sys_p75_id_idx, PARTITION sys_p76_id_idx, PARTITION sys_p77_id_idx, PARTITION sys_p78_id_idx, PARTITION sys_p79_id_idx, PARTITION sys_p80_id_idx, PARTITION sys_p81_id_idx, PARTITION sys_p82_id_idx, PARTITION sys_p83_id_idx, PARTITION sys_p84_id_idx, PARTITION sys_p85_id_idx, PARTITION sys_p86_id_idx, PARTITION sys_p87_id_idx, PARTITION sys_p88_id_idx, PARTITION sys_p89_id_idx, PARTITION sys_p90_id_idx, PARTITION sys_p91_id_idx, PARTITION sys_p92_id_idx, PARTITION sys_p93_id_idx, PARTITION sys_p94_id_idx, PARTITION sys_p95_id_idx, PARTITION sys_p96_id_idx, PARTITION sys_p97_id_idx, PARTITION sys_p98_id_idx, PARTITION sys_p99_id_idx, PARTITION sys_p100_id_idx, PARTITION sys_p101_id_idx, PARTITION sys_p102_id_idx, PARTITION sys_p103_id_idx, PARTITION sys_p104_id_idx, PARTITION sys_p105_id_idx, PARTITION sys_p106_id_idx, PARTITION sys_p107_id_idx, PARTITION sys_p108_id_idx, PARTITION sys_p109_id_idx, PARTITION sys_p110_id_idx, PARTITION sys_p111_id_idx, PARTITION sys_p112_id_idx, PARTITION sys_p113_id_idx, PARTITION sys_p114_id_idx, PARTITION sys_p115_id_idx, PARTITION sys_p116_id_idx, PARTITION sys_p117_id_idx, PARTITION sys_p118_id_idx, PARTITION sys_p119_id_idx, PARTITION sys_p120_id_idx, PARTITION sys_p121_id_idx, PARTITION sys_p122_id_idx, PARTITION sys_p123_id_idx, PARTITION sys_p124_id_idx, PARTITION sys_p125_id_idx, PARTITION sys_p126_id_idx, PARTITION sys_p127_id_idx, PARTITION sys_p128_id_idx, PARTITION sys_p129_id_idx, PARTITION sys_p130_id_idx, PARTITION sys_p131_id_idx, PARTITION sys_p132_id_idx, PARTITION sys_p133_id_idx, PARTITION sys_p134_id_idx, PARTITION sys_p135_id_idx, PARTITION sys_p136_id_idx, PARTITION sys_p137_id_idx, PARTITION sys_p138_id_idx, PARTITION sys_p139_id_idx, PARTITION sys_p140_id_idx, PARTITION sys_p141_id_idx, PARTITION sys_p142_id_idx, PARTITION sys_p143_id_idx, PARTITION sys_p144_id_idx, PARTITION sys_p145_id_idx, PARTITION sys_p146_id_idx, PARTITION sys_p147_id_idx, PARTITION sys_p148_id_idx, PARTITION sys_p149_id_idx, PARTITION sys_p150_id_idx, PARTITION sys_p151_id_idx, PARTITION sys_p152_id_idx, PARTITION sys_p153_id_idx, PARTITION sys_p154_id_idx, PARTITION sys_p155_id_idx, PARTITION sys_p156_id_idx, PARTITION sys_p157_id_idx, PARTITION sys_p158_id_idx, PARTITION sys_p159_id_idx, PARTITION sys_p160_id_idx, PARTITION sys_p161_id_idx, PARTITION sys_p162_id_idx, PARTITION sys_p163_id_idx, PARTITION sys_p164_id_idx, PARTITION sys_p165_id_idx, PARTITION sys_p166_id_idx, PARTITION sys_p167_id_idx, PARTITION sys_p168_id_idx, PARTITION sys_p169_id_idx, PARTITION sys_p170_id_idx, PARTITION sys_p171_id_idx, PARTITION sys_p172_id_idx, PARTITION sys_p173_id_idx, PARTITION sys_p174_id_idx, PARTITION sys_p175_id_idx, PARTITION sys_p176_id_idx, PARTITION sys_p177_id_idx, PARTITION sys_p178_id_idx, PARTITION sys_p179_id_idx, PARTITION sys_p180_id_idx, PARTITION sys_p181_id_idx, PARTITION sys_p182_id_idx, PARTITION sys_p183_id_idx, PARTITION sys_p184_id_idx, PARTITION sys_p185_id_idx, PARTITION sys_p186_id_idx, PARTITION sys_p187_id_idx, PARTITION sys_p188_id_idx, PARTITION sys_p189_id_idx, PARTITION sys_p190_id_idx, PARTITION sys_p191_id_idx, PARTITION sys_p192_id_idx, PARTITION sys_p193_id_idx) TABLESPACE pg_default + "index_part_auto" btree (id) LOCAL TABLESPACE pg_default Partition By RANGE(birthday) INTERVAL('1 month') Number of partitions: 195 (View pg_partition to check each partition range.) @@ -79,7 +79,7 @@ CREATE INDEX birthday | timestamp without time zone | not null age | integer | Indexes: - "index_part_auto" btree (id) LOCAL(PARTITION idx_p1, PARTITION idx_p2, PARTITION sys_p1_id_idx, PARTITION sys_p2_id_idx, PARTITION sys_p3_id_idx, PARTITION sys_p4_id_idx, PARTITION sys_p5_id_idx, PARTITION sys_p6_id_idx, PARTITION sys_p7_id_idx, PARTITION sys_p8_id_idx, PARTITION sys_p9_id_idx, PARTITION sys_p10_id_idx, PARTITION sys_p11_id_idx, PARTITION sys_p12_id_idx, PARTITION sys_p13_id_idx, PARTITION sys_p14_id_idx, PARTITION sys_p15_id_idx, PARTITION sys_p16_id_idx, PARTITION sys_p17_id_idx, PARTITION sys_p18_id_idx, PARTITION sys_p19_id_idx, PARTITION sys_p20_id_idx, PARTITION sys_p21_id_idx, PARTITION sys_p22_id_idx, PARTITION sys_p23_id_idx, PARTITION sys_p24_id_idx, PARTITION sys_p25_id_idx, PARTITION sys_p26_id_idx, PARTITION sys_p27_id_idx, PARTITION sys_p28_id_idx, PARTITION sys_p29_id_idx, PARTITION sys_p30_id_idx, PARTITION sys_p31_id_idx, PARTITION sys_p32_id_idx, PARTITION sys_p33_id_idx, PARTITION sys_p34_id_idx, PARTITION sys_p35_id_idx, PARTITION sys_p36_id_idx, PARTITION sys_p37_id_idx, PARTITION sys_p38_id_idx, PARTITION sys_p39_id_idx, PARTITION sys_p40_id_idx, PARTITION sys_p41_id_idx, PARTITION sys_p42_id_idx, PARTITION sys_p43_id_idx, PARTITION sys_p44_id_idx, PARTITION sys_p45_id_idx, PARTITION sys_p46_id_idx, PARTITION sys_p47_id_idx, PARTITION sys_p48_id_idx, PARTITION sys_p49_id_idx, PARTITION sys_p50_id_idx, PARTITION sys_p51_id_idx, PARTITION sys_p52_id_idx, PARTITION sys_p53_id_idx, PARTITION sys_p54_id_idx, PARTITION sys_p55_id_idx, PARTITION sys_p56_id_idx, PARTITION sys_p57_id_idx, PARTITION sys_p58_id_idx, PARTITION sys_p59_id_idx, PARTITION sys_p60_id_idx, PARTITION sys_p61_id_idx, PARTITION sys_p62_id_idx, PARTITION sys_p63_id_idx, PARTITION sys_p64_id_idx, PARTITION sys_p65_id_idx, PARTITION sys_p66_id_idx, PARTITION sys_p67_id_idx, PARTITION sys_p68_id_idx, PARTITION sys_p69_id_idx, PARTITION sys_p70_id_idx, PARTITION sys_p71_id_idx, PARTITION sys_p72_id_idx, PARTITION sys_p73_id_idx, PARTITION sys_p74_id_idx, PARTITION sys_p75_id_idx, PARTITION sys_p76_id_idx, PARTITION sys_p77_id_idx, PARTITION sys_p78_id_idx, PARTITION sys_p79_id_idx, PARTITION sys_p80_id_idx, PARTITION sys_p81_id_idx, PARTITION sys_p82_id_idx, PARTITION sys_p83_id_idx, PARTITION sys_p84_id_idx, PARTITION sys_p85_id_idx, PARTITION sys_p86_id_idx, PARTITION sys_p87_id_idx, PARTITION sys_p88_id_idx, PARTITION sys_p89_id_idx, PARTITION sys_p90_id_idx, PARTITION sys_p91_id_idx, PARTITION sys_p92_id_idx, PARTITION sys_p93_id_idx, PARTITION sys_p94_id_idx, PARTITION sys_p95_id_idx, PARTITION sys_p96_id_idx, PARTITION sys_p97_id_idx, PARTITION sys_p98_id_idx, PARTITION sys_p99_id_idx, PARTITION sys_p100_id_idx, PARTITION sys_p101_id_idx, PARTITION sys_p102_id_idx, PARTITION sys_p103_id_idx, PARTITION sys_p104_id_idx, PARTITION sys_p105_id_idx, PARTITION sys_p106_id_idx, PARTITION sys_p107_id_idx, PARTITION sys_p108_id_idx, PARTITION sys_p109_id_idx, PARTITION sys_p110_id_idx, PARTITION sys_p111_id_idx, PARTITION sys_p112_id_idx, PARTITION sys_p113_id_idx, PARTITION sys_p114_id_idx, PARTITION sys_p115_id_idx, PARTITION sys_p116_id_idx, PARTITION sys_p117_id_idx, PARTITION sys_p118_id_idx, PARTITION sys_p119_id_idx, PARTITION sys_p120_id_idx, PARTITION sys_p121_id_idx, PARTITION sys_p122_id_idx, PARTITION sys_p123_id_idx, PARTITION sys_p124_id_idx, PARTITION sys_p125_id_idx, PARTITION sys_p126_id_idx, PARTITION sys_p127_id_idx, PARTITION sys_p128_id_idx, PARTITION sys_p129_id_idx, PARTITION sys_p130_id_idx, PARTITION sys_p131_id_idx, PARTITION sys_p132_id_idx, PARTITION sys_p133_id_idx, PARTITION sys_p134_id_idx, PARTITION sys_p135_id_idx, PARTITION sys_p136_id_idx, PARTITION sys_p137_id_idx, PARTITION sys_p138_id_idx, PARTITION sys_p139_id_idx, PARTITION sys_p140_id_idx, PARTITION sys_p141_id_idx, PARTITION sys_p142_id_idx, PARTITION sys_p143_id_idx, PARTITION sys_p144_id_idx, PARTITION sys_p145_id_idx, PARTITION sys_p146_id_idx, PARTITION sys_p147_id_idx, PARTITION sys_p148_id_idx, PARTITION sys_p149_id_idx, PARTITION sys_p150_id_idx, PARTITION sys_p151_id_idx, PARTITION sys_p152_id_idx, PARTITION sys_p153_id_idx, PARTITION sys_p154_id_idx, PARTITION sys_p155_id_idx, PARTITION sys_p156_id_idx, PARTITION sys_p157_id_idx, PARTITION sys_p158_id_idx, PARTITION sys_p159_id_idx, PARTITION sys_p160_id_idx, PARTITION sys_p161_id_idx, PARTITION sys_p162_id_idx, PARTITION sys_p163_id_idx, PARTITION sys_p164_id_idx, PARTITION sys_p165_id_idx, PARTITION sys_p166_id_idx, PARTITION sys_p167_id_idx, PARTITION sys_p168_id_idx, PARTITION sys_p169_id_idx, PARTITION sys_p170_id_idx, PARTITION sys_p171_id_idx, PARTITION sys_p172_id_idx, PARTITION sys_p173_id_idx, PARTITION sys_p174_id_idx, PARTITION sys_p175_id_idx, PARTITION sys_p176_id_idx, PARTITION sys_p177_id_idx, PARTITION sys_p178_id_idx, PARTITION sys_p179_id_idx, PARTITION sys_p180_id_idx, PARTITION sys_p181_id_idx, PARTITION sys_p182_id_idx, PARTITION sys_p183_id_idx, PARTITION sys_p184_id_idx, PARTITION sys_p185_id_idx, PARTITION sys_p186_id_idx, PARTITION sys_p187_id_idx, PARTITION sys_p188_id_idx, PARTITION sys_p189_id_idx, PARTITION sys_p190_id_idx, PARTITION sys_p191_id_idx, PARTITION sys_p192_id_idx, PARTITION sys_p193_id_idx) TABLESPACE pg_default + "index_part_auto" btree (id) LOCAL TABLESPACE pg_default Partition By RANGE(birthday) INTERVAL('1 month') Number of partitions: 195 (View pg_partition to check each partition range.) @@ -155,7 +155,7 @@ CREATE INDEX birthday | timestamp without time zone | not null age | integer | Indexes: - "index_part_auto" btree (id) LOCAL(PARTITION idx_p1, PARTITION idx_p2, PARTITION sys_p1_id_idx, PARTITION sys_p2_id_idx, PARTITION sys_p3_id_idx, PARTITION sys_p4_id_idx, PARTITION sys_p5_id_idx, PARTITION sys_p6_id_idx, PARTITION sys_p7_id_idx, PARTITION sys_p8_id_idx, PARTITION sys_p9_id_idx, PARTITION sys_p10_id_idx, PARTITION sys_p11_id_idx, PARTITION sys_p12_id_idx, PARTITION sys_p13_id_idx, PARTITION sys_p14_id_idx, PARTITION sys_p15_id_idx, PARTITION sys_p16_id_idx, PARTITION sys_p17_id_idx, PARTITION sys_p18_id_idx, PARTITION sys_p19_id_idx, PARTITION sys_p20_id_idx, PARTITION sys_p21_id_idx, PARTITION sys_p22_id_idx, PARTITION sys_p23_id_idx, PARTITION sys_p24_id_idx, PARTITION sys_p25_id_idx, PARTITION sys_p26_id_idx, PARTITION sys_p27_id_idx, PARTITION sys_p28_id_idx, PARTITION sys_p29_id_idx, PARTITION sys_p30_id_idx, PARTITION sys_p31_id_idx, PARTITION sys_p32_id_idx, PARTITION sys_p33_id_idx, PARTITION sys_p34_id_idx, PARTITION sys_p35_id_idx, PARTITION sys_p36_id_idx, PARTITION sys_p37_id_idx, PARTITION sys_p38_id_idx, PARTITION sys_p39_id_idx, PARTITION sys_p40_id_idx, PARTITION sys_p41_id_idx, PARTITION sys_p42_id_idx, PARTITION sys_p43_id_idx, PARTITION sys_p44_id_idx, PARTITION sys_p45_id_idx, PARTITION sys_p46_id_idx, PARTITION sys_p47_id_idx, PARTITION sys_p48_id_idx, PARTITION sys_p49_id_idx, PARTITION sys_p50_id_idx, PARTITION sys_p51_id_idx, PARTITION sys_p52_id_idx, PARTITION sys_p53_id_idx, PARTITION sys_p54_id_idx, PARTITION sys_p55_id_idx, PARTITION sys_p56_id_idx, PARTITION sys_p57_id_idx, PARTITION sys_p58_id_idx, PARTITION sys_p59_id_idx, PARTITION sys_p60_id_idx, PARTITION sys_p61_id_idx, PARTITION sys_p62_id_idx, PARTITION sys_p63_id_idx, PARTITION sys_p64_id_idx, PARTITION sys_p65_id_idx, PARTITION sys_p66_id_idx, PARTITION sys_p67_id_idx, PARTITION sys_p68_id_idx, PARTITION sys_p69_id_idx, PARTITION sys_p70_id_idx, PARTITION sys_p71_id_idx, PARTITION sys_p72_id_idx, PARTITION sys_p73_id_idx, PARTITION sys_p74_id_idx, PARTITION sys_p75_id_idx, PARTITION sys_p76_id_idx, PARTITION sys_p77_id_idx, PARTITION sys_p78_id_idx, PARTITION sys_p79_id_idx, PARTITION sys_p80_id_idx, PARTITION sys_p81_id_idx, PARTITION sys_p82_id_idx, PARTITION sys_p83_id_idx, PARTITION sys_p84_id_idx, PARTITION sys_p85_id_idx, PARTITION sys_p86_id_idx, PARTITION sys_p87_id_idx, PARTITION sys_p88_id_idx, PARTITION sys_p89_id_idx, PARTITION sys_p90_id_idx, PARTITION sys_p91_id_idx, PARTITION sys_p92_id_idx, PARTITION sys_p93_id_idx, PARTITION sys_p94_id_idx, PARTITION sys_p95_id_idx, PARTITION sys_p96_id_idx, PARTITION sys_p97_id_idx, PARTITION sys_p98_id_idx, PARTITION sys_p99_id_idx, PARTITION sys_p100_id_idx, PARTITION sys_p101_id_idx, PARTITION sys_p102_id_idx, PARTITION sys_p103_id_idx, PARTITION sys_p104_id_idx, PARTITION sys_p105_id_idx, PARTITION sys_p106_id_idx, PARTITION sys_p107_id_idx, PARTITION sys_p108_id_idx, PARTITION sys_p109_id_idx, PARTITION sys_p110_id_idx, PARTITION sys_p111_id_idx, PARTITION sys_p112_id_idx, PARTITION sys_p113_id_idx, PARTITION sys_p114_id_idx, PARTITION sys_p115_id_idx, PARTITION sys_p116_id_idx, PARTITION sys_p117_id_idx, PARTITION sys_p118_id_idx, PARTITION sys_p119_id_idx, PARTITION sys_p120_id_idx, PARTITION sys_p121_id_idx, PARTITION sys_p122_id_idx, PARTITION sys_p123_id_idx, PARTITION sys_p124_id_idx, PARTITION sys_p125_id_idx, PARTITION sys_p126_id_idx, PARTITION sys_p127_id_idx, PARTITION sys_p128_id_idx, PARTITION sys_p129_id_idx, PARTITION sys_p130_id_idx, PARTITION sys_p131_id_idx, PARTITION sys_p132_id_idx, PARTITION sys_p133_id_idx, PARTITION sys_p134_id_idx, PARTITION sys_p135_id_idx, PARTITION sys_p136_id_idx, PARTITION sys_p137_id_idx, PARTITION sys_p138_id_idx, PARTITION sys_p139_id_idx, PARTITION sys_p140_id_idx, PARTITION sys_p141_id_idx, PARTITION sys_p142_id_idx, PARTITION sys_p143_id_idx, PARTITION sys_p144_id_idx, PARTITION sys_p145_id_idx, PARTITION sys_p146_id_idx, PARTITION sys_p147_id_idx, PARTITION sys_p148_id_idx, PARTITION sys_p149_id_idx, PARTITION sys_p150_id_idx, PARTITION sys_p151_id_idx, PARTITION sys_p152_id_idx, PARTITION sys_p153_id_idx, PARTITION sys_p154_id_idx, PARTITION sys_p155_id_idx, PARTITION sys_p156_id_idx, PARTITION sys_p157_id_idx, PARTITION sys_p158_id_idx, PARTITION sys_p159_id_idx, PARTITION sys_p160_id_idx, PARTITION sys_p161_id_idx, PARTITION sys_p162_id_idx, PARTITION sys_p163_id_idx, PARTITION sys_p164_id_idx, PARTITION sys_p165_id_idx, PARTITION sys_p166_id_idx, PARTITION sys_p167_id_idx, PARTITION sys_p168_id_idx, PARTITION sys_p169_id_idx, PARTITION sys_p170_id_idx, PARTITION sys_p171_id_idx, PARTITION sys_p172_id_idx, PARTITION sys_p173_id_idx, PARTITION sys_p174_id_idx, PARTITION sys_p175_id_idx, PARTITION sys_p176_id_idx, PARTITION sys_p177_id_idx, PARTITION sys_p178_id_idx, PARTITION sys_p179_id_idx, PARTITION sys_p180_id_idx, PARTITION sys_p181_id_idx, PARTITION sys_p182_id_idx, PARTITION sys_p183_id_idx, PARTITION sys_p184_id_idx, PARTITION sys_p185_id_idx, PARTITION sys_p186_id_idx, PARTITION sys_p187_id_idx, PARTITION sys_p188_id_idx, PARTITION sys_p189_id_idx, PARTITION sys_p190_id_idx, PARTITION sys_p191_id_idx, PARTITION sys_p192_id_idx, PARTITION sys_p193_id_idx) TABLESPACE pg_default + "index_part_auto" btree (id) LOCAL TABLESPACE pg_default Partition By RANGE(birthday) INTERVAL('1 month') Number of partitions: 195 (View pg_partition to check each partition range.) @@ -215,7 +215,7 @@ INSERT INTO sales VALUES(1, 12, '2019-02-03 00:00:00', 'a', 1, 1, 1);  quantity_sold | numeric(3,0) |  amount_sold | numeric(10,2) | Indexes: - "index_sales" btree (prod_id) LOCAL(PARTITION idx_p1, PARTITION idx_p2, PARTITION sys_p2_prod_id_idx, PARTITION sys_p1_prod_id_idx) TABLESPACE pg_default + "index_sales" btree (prod_id) LOCAL TABLESPACE pg_default Partition By RANGE( time_id) INTERVAL('1 day') Number of partitions: 4 (View pg_partition to check each partition range.) @@ -253,7 +253,7 @@ CREATE INDEX  quantity_sold | numeric(3,0) |  amount_sold | numeric(10,2) | Indexes: - "index_sales" btree (prod_id) LOCAL(PARTITION idx_p1, PARTITION idx_p2) TABLESPACE pg_default + "index_sales" btree (prod_id) LOCAL TABLESPACE pg_default Partition By RANGE( time_id) INTERVAL('1 day') Number of partitions: 2 (View pg_partition to check each partition range.) diff --git a/src/test/regress/parallel_schedule0 b/src/test/regress/parallel_schedule0 index f96fce2de..1bad619ad 100644 --- a/src/test/regress/parallel_schedule0 +++ b/src/test/regress/parallel_schedule0 @@ -25,7 +25,7 @@ test: pldeveloper_gs_source test: index_advisor test: pl_debugger_server pl_debugger_client test: update_for_wait_s1 update_for_wait_s2 -test: plan_hint plan_hint_set plan_hint_no_expand plan_hint_iud +test: plan_hint plan_hint_set plan_hint_no_expand plan_hint_iud null_test_opt test: large_sequence int16 gs_dump_sequence test: gs_dump_tableof test: analyze_commands diff --git a/src/test/regress/security_audit_schedule0 b/src/test/regress/security_audit_schedule0 index 7b157854b..1c0fe39ba 100644 --- a/src/test/regress/security_audit_schedule0 +++ b/src/test/regress/security_audit_schedule0 @@ -1,7 +1,14 @@ -# ---------- +# ------------------ # # Database security audit -# # ---------- - +# # --------------- +test: hw_audit_pg_query test: hw_audit_space test: hw_audit_rotation_interval test: hw_audit_rotation_size +#-------------------- +# # Multi thread +#------------------- +test: hw_audit_multi_thread_rotation_interval +test: hw_audit_multi_thread_rotation_size +test: hw_audit_multi_thread_space + diff --git a/src/test/regress/single_check.sh b/src/test/regress/single_check.sh index 75c686a94..f796f8c88 100755 --- a/src/test/regress/single_check.sh +++ b/src/test/regress/single_check.sh @@ -370,9 +370,9 @@ case $DO_CMD in --fastcheck_single|fastcheck_single) args_val="-d 1 -c 0 -p $p -r 1 " real_regresscheck_single parallel_schedule0$part make_fastcheck_postgresql.conf "${args_val}" ;; - --fastcheck_single_audit|fastcheck_single_audit) + --fastcheck_single_audit|fastcheck_single_audit) args_val="-d 1 -c 0 -p $p -r 1 " - real_regresscheck_single_audit security_audit_schedule0$part make_fastcheck_postgresql.conf "${args_val}" ;; + real_regresscheck_single_audit security_audit_schedule0$part make_fastcheck_audit_postgresql.conf "${args_val}" ;; --fastcheck_lite|fastcheck_lite) args_val="-d 1 -c 0 -p $p -r 1 " real_regresscheck_single parallel_schedule.lite$part make_fastcheck_postgresql.conf "${args_val}" ;; diff --git a/src/test/regress/sql/autonomous_test.sql b/src/test/regress/sql/autonomous_test.sql index c561ae27a..47356acdc 100644 --- a/src/test/regress/sql/autonomous_test.sql +++ b/src/test/regress/sql/autonomous_test.sql @@ -1478,3 +1478,66 @@ call pck1.p1(10,20); call pck1.p1(10,20); drop package pck1; + +-- anoymous block with autonomous +create or replace package pck1 as +type r1 is record(a int, b int); +va int; +vb r1; +vc varchar2(20); +end pck1; +/ + +declare +begin +pck1.va := 1; +pck1.vb := (2,3); +pck1.vc := 'before auto'; +end; +/ + +-- (a) autonomous anoymous block +declare +PRAGMA AUTONOMOUS_TRANSACTION; +begin +raise info 'pck1.va: %',pck1.va; +raise info 'pck1.vb: %',pck1.vb; +raise info 'pck1.vc: %',pck1.vc; +pck1.va := 11; +pck1.vb := (22,33); +pck1.vc := 'after auto'; +end; +/ + +declare +begin +raise info 'pck1.va: %',pck1.va; +raise info 'pck1.vb: %',pck1.vb; +raise info 'pck1.vc: %',pck1.vc; +pck1.va := 111; +pck1.vb := (222,333); +pck1.vc := 'before after auto'; +end; +/ + +-- (b) autonomous anoymous block with error +declare +PRAGMA AUTONOMOUS_TRANSACTION; +begin +raise info 'pck1.va: %',pck1.va; +raise info 'pck1.vb: %',pck1.vb; +raise info 'pck1.vc: %',pck1.vc; +pck1.va := 1111; +pck1.vb := (2222,3333); +pck1.vc := 'after after auto'; +pck1.va := 3/0; +end; +/ + +declare +begin +raise info 'pck1.va: %',pck1.va; +raise info 'pck1.vb: %',pck1.vb; +raise info 'pck1.vc: %',pck1.vc; +end; +/ \ No newline at end of file diff --git a/src/test/regress/sql/bypass_simplequery_support.sql b/src/test/regress/sql/bypass_simplequery_support.sql index d7c644c7f..ada3ec28a 100755 --- a/src/test/regress/sql/bypass_simplequery_support.sql +++ b/src/test/regress/sql/bypass_simplequery_support.sql @@ -424,6 +424,10 @@ create view v_test as select * from test; CREATE OR REPLACE RULE v_delete as ON DELETE TO v_test DO INSTEAD NOTHING; delete from v_test; +set explain_perf_mode=pretty; +explain delete from v_test; +reset explain_perf_mode; + drop table test cascade; -- end diff --git a/src/test/regress/sql/huge_clob.sql b/src/test/regress/sql/huge_clob.sql index 34b9de6f9..2fa34734f 100644 --- a/src/test/regress/sql/huge_clob.sql +++ b/src/test/regress/sql/huge_clob.sql @@ -7,12 +7,29 @@ drop schema if exists huge_clob; create schema huge_clob; set current_schema = huge_clob; -create table bigclobtbl031(c1 int,c2 clob,c3 clob,c4 blob,c5 date,c6 timestamp,c7 varchar2); -insert into bigclobtbl031 values(generate_series(1,5),repeat('AAAA11111aaaaaaaaaaaaaaaaa',1000000),repeat('abdededfj12345679ujik',1000000),hextoraw(repeat('12345678990abcdef',1000)),sysdate,to_timestamp('','yyyy-mm-dd hh24:mi:ss.ff6'),7000); -update bigclobtbl031 set c2=c2||c2||c2||c2||c2; -update bigclobtbl031 set c2=c2||c2||c2||c2||c2; -update bigclobtbl031 set c2=c2||c2; -update bigclobtbl031 set c3='clobclob3'; +drop table if exists cloblongtbl; +create table cloblongtbl (a int, b clob, c clob); +-- insert data less than 1G +insert into cloblongtbl values (generate_series(1,4),repeat('唐李白床前明月光,疑是地上霜,举头望明月,低头思故乡',5000000),repeat('唐李白床前明月光,疑是地上霜,举头望明月,低头思故乡',5000000)); +update cloblongtbl set b = b||b; +update cloblongtbl set c = c||c; +-- b > 1G && c < 1G when a = 2 +update cloblongtbl set b = b||b where a = 2; +-- b < 1G && c > 1G when a = 3 +update cloblongtbl set c = c||c where a = 3; +-- b > 1G && c > 1G when a = 4 +update cloblongtbl set b = b||b where a = 4; +update cloblongtbl set c = c||c where a = 4; +select a, length(b || c) from cloblongtbl order by 1; + +-- reset data for other test +update cloblongtbl set b = b || b where a = 1; +update cloblongtbl set b = b || b where a = 3; +update cloblongtbl set c='cloblessthan1G' where a = 1; +update cloblongtbl set c='cloblessthan1G' where a = 2; +update cloblongtbl set c='cloblessthan1G' where a = 3; +update cloblongtbl set c='cloblessthan1G' where a = 4; + --I1.clob in create or replace procedure pro_cb4_031(c1 clob,c2 clob) @@ -31,22 +48,35 @@ create or replace procedure pro_cb4_031_1 is v1 clob; v2 clob; begin -execute immediate 'select c2 from bigclobtbl031 where c1=1' into v1; -execute immediate 'select c3 from bigclobtbl031 where c1=1' into v2; +execute immediate 'select b from cloblongtbl where a=1' into v1; +execute immediate 'select c from cloblongtbl where a=1' into v2; pro_cb4_031(v1,v2); end; / call pro_cb4_031_1(); +create or replace procedure pro_cb4_005 is +v1 clob; +v2 clob; +v3 clob; +v4 integer; +begin +execute immediate 'select b from cloblongtbl where a=1' into v1; +dbe_lob.read(v1,10,2,v2); +end; +/ + +call pro_cb4_005(); + --I2.clob > 1G out create or replace procedure pro_cb4_031(c1 out clob,c2 out clob) is v1 clob; v2 clob; begin -execute immediate 'select c2 from bigclobtbl031 where c1=1' into v1; -execute immediate 'select c3 from bigclobtbl031 where c1=1' into v2; +execute immediate 'select b from cloblongtbl where a=1' into v1; +execute immediate 'select c from cloblongtbl where a=1' into v2; c1:=v1; c2:=v2; end; @@ -74,8 +104,8 @@ is v1 clob; v2 clob; begin -execute immediate 'select c3 from bigclobtbl031 where c1=1' into v1; -execute immediate 'select c3 from bigclobtbl031 where c1=2' into v2; +execute immediate 'select c from cloblongtbl where a=1' into v1; +execute immediate 'select c from cloblongtbl where a=2' into v2; c1:=v1; c2:=v2; end; @@ -89,8 +119,8 @@ is v1 clob; v2 clob; begin -execute immediate 'select c2 from bigclobtbl031 where c1=1' into v1; -execute immediate 'select c3 from bigclobtbl031 where c1=1' into v2; +execute immediate 'select b from cloblongtbl where a=1' into v1; +execute immediate 'select c from cloblongtbl where a=1' into v2; c1:=v1; c2:=v2; end; @@ -119,12 +149,12 @@ v1 clob; v2 clob; v3 clob; begin -execute immediate 'select c3 from bigclobtbl031 where c1=1' into v1; -execute immediate 'select c3 from bigclobtbl031 where c1=2' into v2; -execute immediate 'select c3 from bigclobtbl031 where c1=3' into v3; +execute immediate 'select c from cloblongtbl where a=1' into v1; +execute immediate 'select c from cloblongtbl where a=2' into v2; +execute immediate 'select c from cloblongtbl where a=3' into v3; c1:=v1; c2:=v2; -c3:=v3||'clob3clob3clob3clob3'; +c3:=v3||'clobclobclobclob'; end; / @@ -153,8 +183,8 @@ type ty1 is table of clob; v1 ty1; begin for i in 1..10 loop -execute immediate 'select c2 from bigclobtbl031 where c1='||i into v1(i); -update bigclobtbl030 set c3=v1(i)||v1(i) where c1=i; +execute immediate 'select b from cloblongtbl where a='||i into v1(i); +update cloblongtbl set c=v1(i)||v1(i) where a=i; end loop; end; / @@ -167,21 +197,21 @@ type ty1 is varray(10) of clob; v1 ty1; begin for i in 1..10 loop -execute immediate 'select c2 from bigclobtbl031 where c1='||i into v1(i); -update bigclobtbl030 set c3=v1(i)||v1(i) where c1=i; +execute immediate 'select b from cloblongtbl where a='||i into v1(i); +update cloblongtbl set c=v1(i)||v1(i) where a=i; end loop; end; / call pro_cb4_031(); -select c1,c2,length(c2),c3,length(c3) from bigclobtbl031 where c1>5 and c1<10 order by 1,2,3,4,5; -update bigclobtbl031 set c3='clob3clob3'; +select a,b,length(b),c,length(c) from cloblongtbl where a>5 and a<10 order by 1,2,3,4,5; +update cloblongtbl set c='cloblessthan1G'; --I6.record create or replace procedure pro_cb4_031 is type ty1 is record(c1 int,c2 clob); v1 ty1; begin -execute immediate 'select c2 from bigclobtbl031 where c1=1' into v1.c2; +execute immediate 'select b from cloblongtbl where a=1' into v1.c2; end; / @@ -193,7 +223,7 @@ v1 clob; v2 clob; v3 clob; v4 int; -cursor cor1 is select c2 from bigclobtbl031 where c1=1; +cursor cor1 is select b from cloblongtbl where a=1; begin open cor1; loop @@ -210,16 +240,42 @@ end; call pro_cb4_037(); -drop table if exists cloblongtbl; -create table cloblongtbl (a int, b clob, c clob); -insert into cloblongtbl values (generate_series(1,4),repeat('唐李白床前明月光,疑是地上霜,举头望明月,低头思故乡',5000000),repeat('唐李白床前明月光,疑是地上霜,举头望明月,低头思故乡',5000000)); -update cloblongtbl set b = b||b; -update cloblongtbl set c = c||c; -update cloblongtbl set b = b||b where a = 2; -update cloblongtbl set c = c||c where a = 3; -update cloblongtbl set b = b||b where a = 4; -update cloblongtbl set c = c||c where a = 4; -select a, length(b || c) from cloblongtbl order by 1; +create or replace procedure test_self_update is +v1 clob; +begin +execute immediate 'select b from cloblongtbl where a=1' into v1; +update cloblongtbl set b=v1 where a=1; +savepoint aaa; +update cloblongtbl set b=v1 where a=2; +rollback to aaa; +commit; +end; +/ + +call test_self_update(); + +create or replace procedure test_update_delete is +v1 clob; +begin +execute immediate 'select b from cloblongtbl where a=1' into v1; +update cloblongtbl set b=v1 where a=1; +rollback; +update cloblongtbl set b=v1 where a=2; +commit; +end; +/ + +call test_update_delete(); + +begin; +delete from cloblongtbl where a < 3; +rollback; + +begin; +delete from cloblongtbl where a = 1; +delete from cloblongtbl where a = 2; +rollback; + drop table if exists cloblongtbl; -- clean -drop schema if exists huge_clob cascade; \ No newline at end of file +drop schema if exists huge_clob cascade; diff --git a/src/test/regress/sql/hw_package_single.sql b/src/test/regress/sql/hw_package_single.sql index ced0cb9f0..4a8acede2 100644 --- a/src/test/regress/sql/hw_package_single.sql +++ b/src/test/regress/sql/hw_package_single.sql @@ -345,6 +345,110 @@ end; select proc_test1('1',''); drop procedure proc_test1; + +--test write+update +create table test_clob(a int, b clob); +create table test_blob(a int, b blob); + +insert into test_clob values(1, 'abc'); +insert into test_clob values(2, 'zzz'); + +insert into test_blob values(1, 'abc'); +insert into test_blob values(2, 'fffffff'); + +create or replace procedure test_clob_write +as + dest_clob clob; + PSV_SQL varchar(100); +begin + PSV_SQL := 'select b from test_clob where a = 1'; + EXECUTE IMMEDIATE PSV_SQL into dest_clob; + dbe_lob.write(dest_clob, 1, 1, 'ddd'); + return; +end; +/ +call test_clob_write(); +select * from test_clob; + +create or replace procedure test_lob_write_nodyna +as +declare + dest_clob clob; +begin + select b from test_clob where a = 2 into dest_clob; + dbe_lob.write(dest_clob, 1, 1, 'eeee'); + DBE_OUTPUT.print_line(dest_clob); + return; +end; +/ + +call test_lob_write_nodyna(); +select * from test_clob; + +create or replace procedure test_blob_write +as + dest_blob blob; + PSV_SQL varchar(100); +begin + PSV_SQL := 'select b from test_blob where a = 1'; + EXECUTE IMMEDIATE PSV_SQL into dest_blob; + dbe_lob.write(dest_blob, 1, 1, 'ddd'); + return; +end; +/ +call test_blob_write(); +select * from test_blob; + +--test append+update +create or replace procedure test_clob_append +as + dest_clob clob; + PSV_SQL varchar(100); +begin + PSV_SQL := 'select b from test_clob where a = 1'; + EXECUTE IMMEDIATE PSV_SQL into dest_clob; + dbe_lob.append(dest_clob, 'ddd'); + return; +end; +/ + +call test_clob_append(); +select * from test_clob; + +create or replace procedure test_blob_append +as + dest_blob blob; + PSV_SQL varchar(100); +begin + PSV_SQL := 'select b from test_blob where a = 1'; + EXECUTE IMMEDIATE PSV_SQL into dest_blob; + dbe_lob.append(dest_blob, 'ddd'); + return; +end; +/ + +call test_blob_append(); +select * from test_blob; + +--test read +create or replace procedure test_clob_read +as + dest_clob clob; + buf clob; + PSV_SQL varchar(100); +begin + PSV_SQL := 'select b from test_clob where a = 1'; + EXECUTE IMMEDIATE PSV_SQL into dest_clob; + dbe_lob.read(dest_clob, 1, 1, buf); + DBE_OUTPUT.print_line(buf); + return; +end; +/ + +call test_clob_read(); + +drop table if exists test_clob; + \c regression; drop database IF EXISTS pl_test_pkg_single; diff --git a/src/test/regress/sql/hw_package_variable.sql b/src/test/regress/sql/hw_package_variable.sql index 09d7980e5..2316610bd 100644 --- a/src/test/regress/sql/hw_package_variable.sql +++ b/src/test/regress/sql/hw_package_variable.sql @@ -2139,6 +2139,60 @@ end pck1; DROP PACKAGE pck1; DROP PACKAGE pck2; DROP TABLE t1; + +-- test check sql in for rec in query loop +create or replace function check_f04 return int as +declare +v_2 number(10,2):=-213.989; +v_3 float:=22.34; +v_4 char(1):='q'; +v_6 date:='1999-09-09'; +v_7 clob:='dddd'; +v_8 blob:='ad'; +begin +raise info '1-0 %', v_6; +for v_6 in select c3 from check_tab1 order by c3 limit 6 +loop +raise info '1-1 %', v_6; +end loop; +raise info '1-2 %', v_6; + +return 1; +exception +when others then +raise info 'error is %',sqlerrm; +return 0; +end; +/ + +create table check_tab1(a int); + +create or replace function check_f04 return int as +declare +v_2 number(10,2):=-213.989; +v_3 float:=22.34; +v_4 char(1):='q'; +v_6 date:='1999-09-09'; +v_7 clob:='dddd'; +v_8 blob:='ad'; +begin +raise info '1-0 %', v_6; +for v_6 in select c3 from check_tab1 order by c3 limit 6 +loop +raise info '1-1 %', v_6; +end loop; +raise info '1-2 %', v_6; + +return 1; +exception +when others then +raise info 'error is %',sqlerrm; +return 0; +end; +/ + +drop table check_tab1; + reset behavior_compat_options; -- ref package cursor attr at first @@ -2238,6 +2292,61 @@ drop procedure pp1; drop package pck1; drop table t1; +-- test package variable as in param +create type o1 as (a int, b varchar2); +create or replace package pck1 as +va varchar2; +vb varchar2 := 'vb'; +vc o1; +procedure p1; +end pck1; +/ +create or replace package body pck1 as +procedure p1 as +begin +va := 'hahahah'; +vc := (1,'cccc'); +end; +end pck1; +/ +create table testlog(a int); +create or replace procedure insertlog() is +PRAGMA AUTONOMOUS_TRANSACTION; +begin +insert into testlog values(1); +pck1.va := 'bbbbbbb'; +pck1.vc := (123,'ayayayayay'); +exception +when others then +raise notice 'sqlerrm:%',sqlerrm; +raise; +end; +/ +create or replace procedure p1(va in varchar2, vb in varchar2, vc in o1) is +begin +raise info 'before auto: %,%', vb,vc; +insertlog(); +raise info 'after auto: %,%', vb,vc; +exception +when others then +raise notice 'sqlerrm:%',sqlerrm; +raise; +end; +/ +declare +begin +pck1.p1(); +p1(150,pck1.va,pck1.vc); +raise info 'after p1: %,%', pck1.va,pck1.vc; +end; +/ + +drop procedure p1; +drop procedure insertlog; +drop package pck1; +drop table testlog; +drop type o1; + -- clean DROP SCHEMA IF EXISTS pkg_val_1 CASCADE; DROP SCHEMA IF EXISTS pkg_val_2 CASCADE; diff --git a/src/test/regress/sql/null_test_opt.sql b/src/test/regress/sql/null_test_opt.sql new file mode 100644 index 000000000..2c242cc23 --- /dev/null +++ b/src/test/regress/sql/null_test_opt.sql @@ -0,0 +1,33 @@ +create schema null_test_opt_nsp; + +set search_path = null_test_opt_nsp; + +create table t(a int, b int not null); + +insert into t values(1, 1); + +explain (costs off) select * from t where b is null order by 1, 2; + +select * from t where b is null order by 1, 2; + +explain (costs off) select * from t where b is not null order by 1, 2; + +select * from t where b is not null order by 1, 2; + +explain (costs off) select * from t where b is null or b is not null order by 1, 2; + +select * from t where b is null or b is not null order by 1, 2; + +explain (costs off) select * from t where b is null or a = 1 order by 1, 2; + +select * from t where b is null or a = 1 order by 1, 2; + +explain (costs off) select * from t where b is null and a = 1 order by 1, 2; + +select * from t where b is null and a = 1 order by 1, 2; + +explain (costs off) select * from t tt1 join t tt2 on tt1.a = tt2.b where tt1.a is null; + +select * from t tt1 join t tt2 on tt1.a = tt2.b where tt1.a is null; + +drop schema null_test_opt_nsp cascade; \ No newline at end of file diff --git a/src/test/regress/sql/sqlbypass_partition.sql b/src/test/regress/sql/sqlbypass_partition.sql index d929e76b3..cfbafc6e6 100755 --- a/src/test/regress/sql/sqlbypass_partition.sql +++ b/src/test/regress/sql/sqlbypass_partition.sql @@ -250,6 +250,73 @@ delete from test_bypass_sql_partition where col1 <= 11 and col1 >= 15; delete from test_bypass_sql_partition where col1 > 10; delete from test_bypass_sql_partition where col1 < 10; delete from test_bypass_sql_partition where col1 >= 10 and col1 <= 30; + +create table t1( +crcrd_acg_setl_dt char(8) not null, +cst_id char(18), +multi_tenancy_id char(5) +) +partition by range (multi_tenancy_id, crcrd_acg_setl_dt) +( + partition p1 values less than ('CN000', '20191201'), + partition p2 values less than ('CN000', '20200201'), + partition p3 values less than ('CN000', '20200202'), + partition p4 values less than ('CN000', '20200203'), + partition p5 values less than ('CN000', '20200204'), + partition p6 values less than ('ZZZZZ', '21000101') +) +enable row movement +; + +create index on t1(crcrd_acg_setl_dt, cst_id, multi_tenancy_id) local; + +insert into t1 values('20200201', '107190000103394943', 'CN000'); +insert into t1 values('20200225', '107190000103394943', 'CN000'); +insert into t1 values('20200228', '107190000103394943', 'CN000'); +insert into t1 values('20200301', '107190000103394943', 'CN000'); +insert into t1 values('20200310', '107190000103394943', 'CN000'); + +set enable_seqscan = off; + +select max(crcrd_acg_setl_dt) from t1 where cst_id='107190000103394943' and multi_tenancy_id = 'CN000'; + +prepare p1 as select max(crcrd_acg_setl_dt) from t1 where cst_id=$1 and multi_tenancy_id = $2; + +execute p1 ('107190000103394943','CN000'); +deallocate p1; + +drop table t1; +drop table test_range_pt; +create table test_range_pt (a int, b int, c int) +partition by range(a) +( + partition p1 values less than (2000), + partition p2 values less than (3000), + partition p3 values less than (4000), + partition p4 values less than (5000), + partition p5 values less than (maxvalue) +)ENABLE ROW MOVEMENT; +insert into test_range_pt values(1,1),(2001,2),(3001,3),(4001,4),(5001,5); + +create index idx1 on test_range_pt(a) local; +prepare p1 as select max(a) from test_range_pt where a>$1; +execute p1 (1); +deallocate p1; +select max(a) from test_range_pt where a>b+1; +drop table test_range_pt; +drop table test_list_lt1; +create table test_list_lt1 (a int, b int ) +partition by list(a) +( + partition p1 values (2000), + partition p2 values (3000), + partition p3 values (4000) +) ; +prepare p1 as select * from test_list_lt1 where a = $1 and ctid = '(0,1)'; +execute p1 (1); +deallocate p1; +drop table test_list_lt1; + reset enable_partition_opfusion; drop table test_bypass_sql_partition; diff --git a/src/test/regress/sql/sw_bugfix-1.sql b/src/test/regress/sql/sw_bugfix-1.sql index 0ff32a29b..cbf505b7a 100644 --- a/src/test/regress/sql/sw_bugfix-1.sql +++ b/src/test/regress/sql/sw_bugfix-1.sql @@ -431,3 +431,13 @@ DROP TABLE IF EXISTS start; DROP TABLE IF EXISTS connect; DROP TABLE IF EXISTS siblings; DROP TABLE IF EXISTS prior; + +-- test where clause pushdown result correctness +create table xt1(id int, lid int, name text); +create table xt2(idd int, lidd int, name text); +insert into xt1 values(1,null,'A'),(2,1,'B'),(3,2,'C'); +insert into xt2 values(1,null,'A'),(2,1,'B'),(3,2,'C'), (4,3,'D'); +select * from xt2,xt1 where xt1.id=xt2.idd and xt1.id!=2 start with id=2 connect by prior id=lid; +select * from xt2,xt1 where xt1.id=xt2.idd and xt1.id=3 start with id=2 connect by prior id=lid; +drop table if exists xt1; +drop table if exists xt2; diff --git a/src/test/regress/sql/sw_bugfix-2.sql b/src/test/regress/sql/sw_bugfix-2.sql index bd1d52850..c23ed8ae9 100644 --- a/src/test/regress/sql/sw_bugfix-2.sql +++ b/src/test/regress/sql/sw_bugfix-2.sql @@ -393,6 +393,27 @@ drop table t_customer; -- test correlated sublink create table test_place as select id, name, tex from test_hcb_ptb; select t1.id,t1.pid,t1.name from test_hcb_ptb t1 start with not exists(select * from test_place where id=t1.id and id !=141) connect by prior pid=id; + +-- test sublibk pull is no allowed in swcb converted cases +explain (costs off) +select id,pid,level +from test_hcb_ptb +where exists ( + select id + from test_place t + where t.id=test_hcb_ptb.id +) +start with id=151 connect by prior pid=id; + +select id,pid,level +from test_hcb_ptb +where exists ( + select id + from test_place t + where t.id=test_hcb_ptb.id +) +start with id=151 connect by prior pid=id; + drop table test_place; -- test where quals pushdown