From 9313d708ce2ac35e53f3354013ef9246ebd86efd Mon Sep 17 00:00:00 2001 From: xiong_xjun Date: Wed, 9 Mar 2022 21:03:55 +0800 Subject: [PATCH] standby full build standby --- src/bin/pg_ctl/backup.cpp | 87 +++++++- src/bin/pg_ctl/backup.h | 2 +- src/bin/pg_ctl/pg_build.cpp | 68 +++++++ src/bin/pg_ctl/pg_build.h | 1 + src/bin/pg_ctl/pg_ctl.cpp | 74 +++++-- .../storage/access/transam/xlog.cpp | 99 +++++++++ .../storage/replication/basebackup.cpp | 189 ++++++++++-------- .../storage/replication/repl_gram.y | 8 +- .../storage/replication/repl_scanner.l | 1 + src/include/access/xlog.h | 3 + src/include/replication/replicainternal.h | 2 +- 11 files changed, 424 insertions(+), 110 deletions(-) diff --git a/src/bin/pg_ctl/backup.cpp b/src/bin/pg_ctl/backup.cpp index a9a91a03..9f8d3a9b 100644 --- a/src/bin/pg_ctl/backup.cpp +++ b/src/bin/pg_ctl/backup.cpp @@ -59,6 +59,7 @@ /* Global options */ char* basedir = NULL; +bool isBuildFromStandby = false; char format = 'p'; /* p(lain)/t(ar) */ char* label = "gs_ctl full build"; bool showprogress = true; @@ -452,6 +453,7 @@ typedef struct { char* sysidentifier; int timeline; uint32 term; + bool isBuildFromStandby; } logstreamer_param; static int LogStreamerMain(logstreamer_param* param) @@ -459,7 +461,11 @@ static int LogStreamerMain(logstreamer_param* param) int ret = 0; /* get second connection info in child process for the sake of memmory leak */ - param->bgconn = check_and_conn(standby_connect_timeout, standby_recv_timeout, param->term); + if (param->isBuildFromStandby) { + param->bgconn = check_and_conn_for_standby(standby_connect_timeout, standby_recv_timeout, param->term); + } else { + param->bgconn = check_and_conn(standby_connect_timeout, standby_recv_timeout, param->term); + } if (param->bgconn == NULL) { return 1; } @@ -503,6 +509,7 @@ void StartLogStreamer( param->timeline = timeline; param->sysidentifier = sysidentifier; param->term = primaryTerm; + param->isBuildFromStandby = isBuildFromStandby; /* Convert the starting position */ if (sscanf_s(startpos, "%X/%X", &hi, &lo) != 2) { @@ -1019,6 +1026,62 @@ bool CreateBuildtagFile(const char* fulltagname) return true; } +bool ModifyControlFile(const char* dirname) +{ + ControlFileData controlFileNew; + char controlFilePath[MAXPGPATH] = {0}; + size_t size = 0; + int writelen; + int fd = -1; + int ss = 0; + char* buffer = NULL; + char writeBuf[PG_CONTROL_SIZE]; + errno_t errorNo = EOK; + + buffer = slurpFile(dirname, "global/pg_control", &size); + if (buffer == NULL) { + pg_log(PG_WARNING, _("could not read anything from file pg_control. \n")); + disconnect_and_exit(1); + return false; + } + errorNo = memcpy_s(&controlFileNew, sizeof(ControlFileData), buffer, sizeof(ControlFileData)); + securec_check_c(errorNo, "\0", "\0"); + controlFileNew.state = DB_IN_ARCHIVE_RECOVERY; + INIT_CRC32C(controlFileNew.crc); + COMP_CRC32C(controlFileNew.crc, (char*)&controlFileNew, offsetof(ControlFileData, crc)); + FIN_CRC32C(controlFileNew.crc); + errorNo = memset_s(writeBuf, PG_CONTROL_SIZE, 0, PG_CONTROL_SIZE); + securec_check_c(errorNo, "", ""); + errorNo = memcpy_s(writeBuf, PG_CONTROL_SIZE, &controlFileNew, sizeof(ControlFileData)); + securec_check_c(errorNo, "", ""); + ss = snprintf_s(controlFilePath, MAXPGPATH, MAXPGPATH - 1, "%s/global/pg_control", dirname); + securec_check_ss_c(ss, "\0", "\0"); + + int mode = O_WRONLY | O_CREAT | PG_BINARY; + int flags = 0600; + fd = open(controlFilePath, mode, flags); + if (fd < 0) { + pg_log(PG_WARNING, ("could not open pg_control file. \n")); + disconnect_and_exit(1); + return false; + } + if (lseek(fd, 0, SEEK_SET) == -1) { + (void)close(fd); + fd = -1; + pg_log(PG_WARNING, "could not seek in target file \"%s\"\n", controlFilePath); + disconnect_and_exit(1); + return false; + } + writelen = write(fd, writeBuf, PG_CONTROL_SIZE); + if (writelen != PG_CONTROL_SIZE) { + pg_log(PG_WARNING, "could not write in target file \"%s\"\n", controlFilePath); + disconnect_and_exit(1); + return false; + } + close(fd); + return true; +} + static void BaseBackup(const char* dirname, uint32 term) { PGresult* res = NULL; @@ -1060,7 +1123,11 @@ static void BaseBackup(const char* dirname, uint32 term) get_conninfo(conf_file); /* find a available conn */ - streamConn = check_and_conn(standby_connect_timeout, standby_recv_timeout, term); + if (isBuildFromStandby) { + streamConn = check_and_conn_for_standby(standby_connect_timeout, standby_recv_timeout, term); + } else { + streamConn = check_and_conn(standby_connect_timeout, standby_recv_timeout, term); + } if (streamConn == NULL) { show_full_build_process("could not connect to server."); disconnect_and_exit(1); @@ -1127,15 +1194,19 @@ static void BaseBackup(const char* dirname, uint32 term) */ (void)PQsetRwTimeout(streamConn, Max(BUILD_RW_TIMEOUT, standby_recv_timeout)); (void)PQescapeStringConn(streamConn, escaped_label, label, sizeof(escaped_label), &i); + if (isBuildFromStandby) { + fastcheckpoint = false; + } nRet = snprintf_s(current_path, MAXPGPATH, sizeof(current_path) - 1, - "BASE_BACKUP LABEL '%s' %s %s %s %s", + "BASE_BACKUP LABEL '%s' %s %s %s %s %s", escaped_label, showprogress ? "PROGRESS" : "", includewal && !streamwal ? "WAL" : "", fastcheckpoint ? "FAST" : "", - includewal ? "NOWAIT" : ""); + includewal ? "NOWAIT" : "", + isBuildFromStandby ? "BUILDSTANDBY" : ""); securec_check_ss_c(nRet, "", ""); if (PQsendQuery(streamConn, current_path) == 0) { @@ -1463,6 +1534,11 @@ static void BaseBackup(const char* dirname, uint32 term) nRet = snprintf_s(tblspcPath, MAXPGPATH, MAXPGPATH, "%s/pg_tblspc", dirname); securec_check_ss_c(nRet, "\0", "\0"); DeleteAlreadyDropedFile(tblspcPath, true); + + if (isBuildFromStandby) { + (void)ModifyControlFile(dirname); + } + } /* @@ -1471,13 +1547,14 @@ static void BaseBackup(const char* dirname, uint32 term) * Description : * Notes : */ -void backup_main(char* dir, uint32 term) +void backup_main(char* dir, uint32 term, bool isFromStandby) { if (dir == NULL) { pg_log(PG_PRINT, "%s: parameters dir is NULL.\n", progname); exit(1); } else { basedir = dir; + isBuildFromStandby = isFromStandby; } /* program name */ progname = "gs_ctl"; diff --git a/src/bin/pg_ctl/backup.h b/src/bin/pg_ctl/backup.h index 8cc0075d..c238a969 100644 --- a/src/bin/pg_ctl/backup.h +++ b/src/bin/pg_ctl/backup.h @@ -16,7 +16,7 @@ extern int bgpipe[2]; extern pid_t bgchild; extern char* formatLogTime(); -void backup_main(char* dir, uint32 term); +void backup_main(char* dir, uint32 term, bool isFromStandby); void backup_incremental_xlog(char* dir); void get_xlog_location(char (&xlog_location)[MAXPGPATH]); bool CreateBuildtagFile(const char* fulltagname); diff --git a/src/bin/pg_ctl/pg_build.cpp b/src/bin/pg_ctl/pg_build.cpp index f955f4e2..8c8686c9 100644 --- a/src/bin/pg_ctl/pg_build.cpp +++ b/src/bin/pg_ctl/pg_build.cpp @@ -875,6 +875,74 @@ PGconn* check_and_conn(int conn_timeout, int recv_timeout, uint32 term) return con_get; } +/* check connection for standby build standby */ +PGconn* check_and_conn_for_standby(int conn_timeout, int recv_timeout, uint32 term) +{ + PGconn* con_get = NULL; + char repl_conninfo_str[MAXPGPATH]; + ServerMode remote_mode = UNKNOWN_MODE; + int tnRet = 0; + int repl_arr_length; + int i = 0; + int parse_failed_num = 0; + + for (i = 1; i < MAX_REPLNODE_NUM; i++) { + ReplConnInfo* repl_conn_info = ParseReplConnInfo(conninfo_global[i - 1], &repl_arr_length); + if (repl_conn_info == NULL) { + parse_failed_num++; + continue; + } + + tnRet = memset_s(repl_conninfo_str, MAXPGPATH, 0, MAXPGPATH); + securec_check_ss_c(tnRet, "", ""); + + tnRet = snprintf_s(repl_conninfo_str, + sizeof(repl_conninfo_str), + sizeof(repl_conninfo_str) - 1, + "localhost=%s localport=%d host=%s port=%d " + "dbname=replication replication=true " + "fallback_application_name=gs_ctl " + "connect_timeout=%d rw_timeout=%d " + "options='-c remotetype=application'", + repl_conn_info->localhost, + repl_conn_info->localport, + repl_conn_info->remotehost, + repl_conn_info->remoteport, + conn_timeout, + recv_timeout); + securec_check_ss_c(tnRet, "", ""); + + free(repl_conn_info); + repl_conn_info = NULL; + con_get = PQconnectdb(repl_conninfo_str); + if (con_get != NULL && PQstatus(con_get) == CONNECTION_OK && check_remote_version(con_get, term)) { + remote_mode = get_remote_mode(con_get); + if (remote_mode == STANDBY_MODE && (g_replconn_idx == -1 || i == g_replconn_idx)) { + g_replconn_idx = i; + break; + } + } else { + if (conn_str != NULL) { + pg_log(PG_WARNING, "The given address can not been access.\n"); + if (con_get != NULL) { + PQfinish(con_get); + } + exit(1); + } + } + } + + if (parse_failed_num == MAX_REPLNODE_NUM - 1) { + pg_log(PG_WARNING, "Invalid value for parameter \"replconninfo\" in postgresql.conf or no correct standby.\n"); + if (con_get != NULL) { + PQfinish(con_get); + } + exit(1); + } + + return con_get; +} + /* * Brief : @@GaussDB@@ * Description : find the value of guc para according to name diff --git a/src/bin/pg_ctl/pg_build.h b/src/bin/pg_ctl/pg_build.h index 40a67319..10bc65e8 100644 --- a/src/bin/pg_ctl/pg_build.h +++ b/src/bin/pg_ctl/pg_build.h @@ -49,6 +49,7 @@ int find_gucoption( void get_conninfo(const char* filename); extern PGconn* check_and_conn(int conn_timeout, int recv_timeout, uint32 term = 0); +extern PGconn* check_and_conn_for_standby(int conn_timeout, int recv_timeout, uint32 term = 0); int GetLengthAndCheckReplConn(const char* ConnInfoList); extern int replconn_num; diff --git a/src/bin/pg_ctl/pg_ctl.cpp b/src/bin/pg_ctl/pg_ctl.cpp index 89771c1c..59f61584 100644 --- a/src/bin/pg_ctl/pg_ctl.cpp +++ b/src/bin/pg_ctl/pg_ctl.cpp @@ -3504,8 +3504,9 @@ static void do_help(void) (void)printf(_(" %s hotpatch [-D DATADIR] [-a ACTION] [-n NAME]\n"), progname); #endif printf(_("\nCommon options:\n")); - printf(_(" -b, --mode=MODE the mode of building the datanode.MODE can be \"full\", \"incremental\", " - "\"auto\"\n")); + printf(_(" -b, --mode=MODE the mode of building the datanode or coordinator." + "MODE can be \"full\", \"incremental\", " + "\"auto\", \"standby_full\"\n")); printf(_(" -D, --pgdata=DATADIR location of the database storage area\n")); printf(_(" -s, --silent only print errors, no informational messages\n")); printf(_(" -t, --timeout=SECS seconds to wait when using -w option\n")); @@ -3582,7 +3583,7 @@ static void do_help(void) #endif printf(_("\nBuild connection option:\n")); printf(_(" -r, --recvtimeout=INTERVAL time that receiver waits for communication from server (in seconds)\n")); - printf(_(" -C, connector CN/DN connect to CN for build\n")); + printf(_(" -C, connector CN/DN connect to specified CN/DN for build\n")); #if ((defined(ENABLE_MULTIPLE_NODES)) || (defined(ENABLE_PRIVATEGAUSS))) printf("\nReport bugs to GaussDB support.\n"); @@ -3841,7 +3842,7 @@ static void do_build_stop(pgpid_t pid) } do_wait = true; - if (build_mode == FULL_BUILD) { + if (build_mode == FULL_BUILD || build_mode == STANDBY_FULL_BUILD) { shutdown_mode = IMMEDIATE_MODE; sig = SIGQUIT; do_stop(true); @@ -3926,6 +3927,11 @@ static void do_build(uint32 term) createRewindFile(pg_data); do_incremental_build_xlog(); } + /* standby DN full build from standby DN */ + else if (build_mode == STANDBY_FULL_BUILD) { + createRewindFile(pg_data); + do_actual_build(term); + } } static void do_restore(void) @@ -4173,7 +4179,11 @@ static void do_actual_build(uint32 term) read_ssl_confval(); - backup_main(pg_data, term); + if (build_mode == STANDBY_FULL_BUILD) { + backup_main(pg_data, term, true); + } else { + backup_main(pg_data, term, false); + } pg_log(PG_WARNING, _("build completed(%s).\n"), pg_data); @@ -4182,7 +4192,7 @@ static void do_actual_build(uint32 term) * Standby DN incremental build from Primary DN auto start. * If connect string is not empty,CN/DN will be started by caller. */ - if (conn_str == NULL) { + if (conn_str == NULL || (build_mode == STANDBY_FULL_BUILD && conn_str != NULL)) { /* cascade standby will use use pgha_opt directly */ if (pgha_opt == NULL || strstr(pgha_opt, "cascade_standby") == NULL) { /* pg_ctl start -M standby */ @@ -4720,16 +4730,19 @@ int main(int argc, char** argv) argc, argv, "a:b:cD:l:m:M:N:n:o:p:P:r:sS:t:U:wWZ:dqL:T:", long_options, &option_index)) != -1) #else while ((c = getopt_long( - argc, argv, "b:cD:l:m:M:N:o:p:P:r:sS:t:U:wWZ:dqL:T:", long_options, &option_index)) != -1) + argc, argv, "b:cD:l:m:M:N:o:p:P:r:sS:t:U:wWZ:C:dqL:T:", long_options, &option_index)) != -1) #endif #endif { switch (c) { case 'b': { - if (strcmp(optarg, "full") == 0) + if (strcmp(optarg, "full") == 0) { build_mode = FULL_BUILD; - else if (strcmp(optarg, "incremental") == 0) + } else if (strcmp(optarg, "incremental") == 0) { build_mode = INC_BUILD; + } else if (strcmp(optarg, "standby_full") == 0) { + build_mode = STANDBY_FULL_BUILD; + } break; } case 'D': { @@ -5222,17 +5235,38 @@ int main(int argc, char** argv) break; #endif case BUILD_COMMAND: - if (conn_str != NULL) - pg_log(PG_PROGRESS, - _("gs_ctl %s build ,datadir is %s,conn_str is \'%s\'\n"), - build_mode == FULL_BUILD ? "full" : "incremental", - pg_data, - conn_str); - else - pg_log(PG_PROGRESS, - _("gs_ctl %s build ,datadir is %s\n"), - build_mode == FULL_BUILD ? "full" : "incremental", - pg_data); + if (conn_str != NULL) { + if (build_mode == FULL_BUILD) { + pg_log(PG_PROGRESS, + _("gs_ctl full build ,datadir is %s,conn_str is \'%s\'\n"), + pg_data, + conn_str); + } else if (build_mode == STANDBY_FULL_BUILD) { + pg_log(PG_PROGRESS, + _("gs_ctl standby full build ,datadir is %s,conn_str is \'%s\'\n"), + pg_data, + conn_str); + } else { + pg_log(PG_PROGRESS, + _("gs_ctl incremental build ,datadir is %s,conn_str is \'%s\'\n"), + pg_data, + conn_str); + } + } else { + if (build_mode == FULL_BUILD) { + pg_log(PG_PROGRESS, + _("gs_ctl full build ,datadir is %s\n"), + pg_data); + } else if (build_mode == STANDBY_FULL_BUILD) { + pg_log(PG_PROGRESS, + _("gs_ctl standby full build ,datadir is %s\n"), + pg_data); + } else { + pg_log(PG_PROGRESS, + _("gs_ctl incremental build ,datadir is %s\n"), + pg_data); + } + } if (-1 != pg_ctl_lock(pg_ctl_lockfile, &lockfile)) { do_build(term); (void)pg_ctl_unlock(lockfile); diff --git a/src/gausskernel/storage/access/transam/xlog.cpp b/src/gausskernel/storage/access/transam/xlog.cpp index b7317f69..6bd742ab 100644 --- a/src/gausskernel/storage/access/transam/xlog.cpp +++ b/src/gausskernel/storage/access/transam/xlog.cpp @@ -13268,6 +13268,63 @@ char** tblspcmapfile, List** tablespaces, bool infotbssize, bool needtblspcmapfi return startpoint; } +XLogRecPtr StandbyDoStartBackup(const char* backupidstr, char** labelFile, char** tblSpcMapFile, List** tableSpaces, + DIR* tblSpcDir, bool infoTbsSize) +{ + StringInfoData labelfbuf; + StringInfoData tblspc_mapfbuf; + pg_time_t stamp_time; + char strfbuf[128]; + char xlogFileName[MAXFNAMELEN]; + XLogSegNo _logSegNo; + XLogRecPtr checkPointLoc; + XLogRecPtr startPoint; + errno_t errorno = EOK; + + LWLockAcquire(ControlFileLock, LW_SHARED); + checkPointLoc = t_thrd.shemem_ptr_cxt.ControlFile->checkPoint; + startPoint = t_thrd.shemem_ptr_cxt.ControlFile->checkPointCopy.redo; + LWLockRelease(ControlFileLock); + + XLByteToSeg(startPoint, _logSegNo); + errorno = snprintf_s(xlogFileName, MAXFNAMELEN, MAXFNAMELEN - 1, "%08X%08X%08X", t_thrd.xlog_cxt.ThisTimeLineID, + (uint32)((_logSegNo) / XLogSegmentsPerXLogId), + (uint32)((_logSegNo) % XLogSegmentsPerXLogId)); + securec_check_ss(errorno, "", ""); + + /* + * Construct tablespace_map file + */ + initStringInfo(&tblspc_mapfbuf); + CollectTableSpace(tblSpcDir, tableSpaces, &tblspc_mapfbuf, infoTbsSize); + + /* + * Construct backup label file + */ + initStringInfo(&labelfbuf); + + /* Use the log timezone here, not the session timezone */ + stamp_time = (pg_time_t)time(NULL); + pg_strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", pg_localtime(&stamp_time, log_timezone)); + appendStringInfo(&labelfbuf, "START WAL LOCATION: %X/%X (file %s)\n", (uint32)(startPoint >> 32), + (uint32)startPoint, xlogFileName); + appendStringInfo(&labelfbuf, "CHECKPOINT LOCATION: %X/%X\n", (uint32)(checkPointLoc >> 32), + (uint32)checkPointLoc); + appendStringInfo(&labelfbuf, "BACKUP METHOD: streamed\n"); + appendStringInfo(&labelfbuf, "BACKUP FROM: standby\n"); + appendStringInfo(&labelfbuf, "START TIME: %s\n", strfbuf); + appendStringInfo(&labelfbuf, "LABEL: %s\n", backupidstr); + + /* + * Okay, write the file, or return its contents to caller. + */ + *labelFile = labelfbuf.data; + if (tblspc_mapfbuf.len > 0) { + *tblSpcMapFile = tblspc_mapfbuf.data; + } + return startPoint; +} + /* Error cleanup callback for pg_start_backup */ static void pg_start_backup_callback(int code, Datum arg) { @@ -13678,6 +13735,48 @@ XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive) return stoppoint; } +/* + * + */ +XLogRecPtr StandbyDoStopBackup(char *labelfile) +{ + XLogRecPtr stopPoint; + XLogRecPtr startPoint; + uint32 hi, lo; + char startxlogfilename[MAXFNAMELEN]; + char ch; + char *remaining = NULL; + /* + * During recovery, we don't need to check WAL level. Because, if WAL + * level is not sufficient, it's impossible to get here during recovery. + */ + if (!XLogIsNeeded()) { + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("WAL level not sufficient for making an online backup"), + errhint("wal_level must be set to \"archive\", \"hot_standby\" or \"logical\" at server start."))); + } + /* + * Read and parse the START WAL LOCATION line (this code is pretty crude, + * but we are not expecting any variability in the file format). + */ + if (sscanf_s(labelfile, "START WAL LOCATION: %X/%X (file %24s)%c", &hi, &lo, startxlogfilename, + sizeof(startxlogfilename), &ch, 1) != 4 || + ch != '\n') { + ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("invalid data in file \"%s\"", BACKUP_LABEL_FILE))); + } + startPoint = (((uint64)hi) << 32) | lo; + remaining = strchr(labelfile, '\n') + 1; /* %n is not portable enough */ + LWLockAcquire(ControlFileLock, LW_SHARED); + stopPoint = t_thrd.shemem_ptr_cxt.ControlFile->checkPointCopy.redo; + LWLockRelease(ControlFileLock); + if (XLByteLE(stopPoint, startPoint)) { + stopPoint = GetXLogReplayRecPtr(NULL); + } + return stopPoint; +} + /* * do_pg_abort_backup: abort a running backup * diff --git a/src/gausskernel/storage/replication/basebackup.cpp b/src/gausskernel/storage/replication/basebackup.cpp index 7d8175ae..61a3b5d6 100644 --- a/src/gausskernel/storage/replication/basebackup.cpp +++ b/src/gausskernel/storage/replication/basebackup.cpp @@ -55,6 +55,7 @@ typedef struct { bool nowait; bool includewal; bool sendtblspcmapfile; + bool isBuildFromStandby; } basebackup_options; #define BUILD_PATH_LEN 2560 /* (MAXPGPATH*2 + 512) */ @@ -103,6 +104,7 @@ static void SendXlogRecPtrResult(XLogRecPtr ptr); static void send_xlog_location(); static void send_xlog_header(const char *linkpath); static void save_xlogloc(const char *xloglocation); +static void SendTableSpaceForBackup(basebackup_options* opt, List* tablespaces, char* labelfile, char* tblspc_map_file); /* * save xlog location @@ -230,14 +232,21 @@ static void base_backup_cleanup(int code, Datum arg) */ static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir) { + XLogRecPtr startptr; XLogRecPtr endptr; char *labelfile = NULL; char* tblspc_map_file = NULL; List* tablespaces = NIL; - XLogRecPtr startptr = - do_pg_start_backup(opt->label, opt->fastcheckpoint, &labelfile, tblspcdir, &tblspc_map_file, &tablespaces, + if (opt->isBuildFromStandby) { + startptr = StandbyDoStartBackup(opt->label, &labelfile, &tblspc_map_file, &tablespaces, + tblspcdir, opt->progress); + } else { + startptr = + do_pg_start_backup(opt->label, opt->fastcheckpoint, &labelfile, tblspcdir, &tblspc_map_file, &tablespaces, opt->progress, opt->sendtblspcmapfile); + } + /* Get the slot minimum LSN */ ReplicationSlotsComputeRequiredXmin(false); ReplicationSlotsComputeRequiredLSN(NULL); @@ -262,88 +271,14 @@ static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir) SendXlogRecPtrResult(startptr); PG_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum)0); - { - ListCell *lc = NULL; - /* Add a node for the base directory at the end */ - tablespaceinfo *ti = (tablespaceinfo *)palloc0(sizeof(tablespaceinfo)); - ti->size = opt->progress ? sendDir(".", 1, true, tablespaces, true) : -1; - tablespaces = (List *)lappend(tablespaces, ti); - - /* Send tablespace header */ - SendBackupHeader(tablespaces); - - /* Send off our tablespaces one by one */ - foreach (lc, tablespaces) { - tablespaceinfo *iterti = (tablespaceinfo *)lfirst(lc); - StringInfoData buf; - - /* Send CopyOutResponse message */ - pq_beginmessage(&buf, 'H'); - pq_sendbyte(&buf, 0); /* overall format */ - pq_sendint16(&buf, 0); /* natts */ - pq_endmessage_noblock(&buf); - - /* In the main tar, include the backup_label first. */ - if (iterti->path == NULL) - sendFileWithContent(BACKUP_LABEL_FILE, labelfile); - - /* - * if the tblspc created in datadir , the files under tblspc do not send, - * and send them as normal under datadir, - * so we just send these tblspcs only once. - */ - if (iterti->path != NULL) { - /* Skip the tablespace if it's created in GAUSSDATA */ - sendTablespace(iterti->path, false); - } else { - /* Then the tablespace_map file, if required... */ - if (tblspc_map_file && opt->sendtblspcmapfile) { - sendFileWithContent(TABLESPACE_MAP, tblspc_map_file); - sendDir(".", 1, false, tablespaces, false); - } else - sendDir(".", 1, false, tablespaces, true); - } - - /* In the main tar, include pg_control last. */ - if (iterti->path == NULL) { - struct stat statbuf; - TimeLineID primay_tli = 0; - char path[MAXPGPATH] = {0}; - - if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0) { - LWLockAcquire(FullBuildXlogCopyStartPtrLock, LW_EXCLUSIVE); - XlogCopyStartPtr = InvalidXLogRecPtr; - LWLockRelease(FullBuildXlogCopyStartPtrLock); - ereport(ERROR, (errcode_for_file_access(), - errmsg("could not stat control file \"%s\": %m", XLOG_CONTROL_FILE))); - } - - sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, false); - /* In the main tar, include the last timeline history file at last. */ - primay_tli = t_thrd.xlog_cxt.ThisTimeLineID; - while (primay_tli > 1) { - TLHistoryFilePath(path, primay_tli); - if (lstat(path, &statbuf) == 0) - sendFile(path, path, &statbuf, false); - primay_tli--; - } - } - - /* - * If we're including WAL, and this is the main data directory we - * don't terminate the tar stream here. Instead, we will append - * the xlog files below and terminate it then. This is safe since - * the main data directory is always sent *last*. - */ - if (opt->includewal && iterti->path == NULL) { - Assert(lnext(lc) == NULL); - } else - pq_putemptymessage_noblock('c'); /* CopyDone */ - } - } + SendTableSpaceForBackup(opt, tablespaces, labelfile, tblspc_map_file); PG_END_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum)0); - endptr = do_pg_stop_backup(labelfile, !opt->nowait); + if (opt->isBuildFromStandby) { + endptr = StandbyDoStopBackup(labelfile); + } else { + endptr = do_pg_stop_backup(labelfile, !opt->nowait); + } if (opt->includewal) { /* @@ -626,6 +561,7 @@ static void parse_basebackup_options(List *options, basebackup_options *opt) bool o_fast = false; bool o_nowait = false; bool o_wal = false; + bool o_buildstandby = false; bool o_tablespace_map = false; errno_t rc = memset_s(opt, sizeof(*opt), 0, sizeof(*opt)); securec_check(rc, "", ""); @@ -656,6 +592,12 @@ static void parse_basebackup_options(List *options, basebackup_options *opt) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate option \"%s\"", defel->defname))); opt->includewal = true; o_wal = true; + } else if (strcmp(defel->defname, "buildstandby") == 0) { + if (o_buildstandby) { + ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate option \"%s\"", defel->defname))); + } + opt->isBuildFromStandby = true; + o_buildstandby = true; } else if (strcmp(defel->defname, "tablespace_map") == 0) { if (o_tablespace_map) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate option \"%s\"", defel->defname))); @@ -1578,6 +1520,89 @@ bool is_row_data_file(const char *path, int *segNo) } return false; } + + +static void SendTableSpaceForBackup(basebackup_options* opt, List* tablespaces, char* labelfile, char* tblspc_map_file) +{ + ListCell *lc = NULL; + /* Add a node for the base directory at the end */ + tablespaceinfo *ti = (tablespaceinfo *)palloc0(sizeof(tablespaceinfo)); + ti->size = opt->progress ? sendDir(".", 1, true, tablespaces, true) : -1; + tablespaces = (List *)lappend(tablespaces, ti); + + /* Send tablespace header */ + SendBackupHeader(tablespaces); + + /* Send off our tablespaces one by one */ + foreach (lc, tablespaces) { + tablespaceinfo *iterti = (tablespaceinfo *)lfirst(lc); + StringInfoData buf; + + /* Send CopyOutResponse message */ + pq_beginmessage(&buf, 'H'); + pq_sendbyte(&buf, 0); /* overall format */ + pq_sendint16(&buf, 0); /* natts */ + pq_endmessage_noblock(&buf); + + /* In the main tar, include the backup_label first. */ + if (iterti->path == NULL) + sendFileWithContent(BACKUP_LABEL_FILE, labelfile); + + /* + * if the tblspc created in datadir , the files under tblspc do not send, + * and send them as normal under datadir, + * so we just send these tblspcs only once. + */ + if (iterti->path != NULL) { + /* Skip the tablespace if it's created in GAUSSDATA */ + sendTablespace(iterti->path, false); + } else { + /* Then the tablespace_map file, if required... */ + if (tblspc_map_file && opt->sendtblspcmapfile) { + sendFileWithContent(TABLESPACE_MAP, tblspc_map_file); + sendDir(".", 1, false, tablespaces, false); + } else + sendDir(".", 1, false, tablespaces, true); + } + + /* In the main tar, include pg_control last. */ + if (iterti->path == NULL) { + struct stat statbuf; + TimeLineID primay_tli = 0; + char path[MAXPGPATH] = {0}; + + if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0) { + LWLockAcquire(FullBuildXlogCopyStartPtrLock, LW_EXCLUSIVE); + XlogCopyStartPtr = InvalidXLogRecPtr; + LWLockRelease(FullBuildXlogCopyStartPtrLock); + ereport(ERROR, (errcode_for_file_access(), + errmsg("could not stat control file \"%s\": %m", XLOG_CONTROL_FILE))); + } + + sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, false); + /* In the main tar, include the last timeline history file at last. */ + primay_tli = t_thrd.xlog_cxt.ThisTimeLineID; + while (primay_tli > 1) { + TLHistoryFilePath(path, primay_tli); + if (lstat(path, &statbuf) == 0) + sendFile(path, path, &statbuf, false); + primay_tli--; + } + } + + /* + * If we're including WAL, and this is the main data directory we + * don't terminate the tar stream here. Instead, we will append + * the xlog files below and terminate it then. This is safe since + * the main data directory is always sent *last*. + */ + if (opt->includewal && iterti->path == NULL) { + Assert(lnext(lc) == NULL); + } else + pq_putemptymessage_noblock('c'); /* CopyDone */ + } +} + /* * Given the member, write the TAR header & send the file. * diff --git a/src/gausskernel/storage/replication/repl_gram.y b/src/gausskernel/storage/replication/repl_gram.y index de6c23ef..b48152b4 100755 --- a/src/gausskernel/storage/replication/repl_gram.y +++ b/src/gausskernel/storage/replication/repl_gram.y @@ -87,6 +87,7 @@ %token K_PROGRESS %token K_FAST %token K_NOWAIT +%token K_BUILDSTANDBY %token K_WAL %token K_TABLESPACE_MAP %token K_DATA @@ -218,7 +219,7 @@ identify_az: ; /* - * BASE_BACKUP [LABEL '