diff --git a/src/gausskernel/process/postmaster/pgarch.cpp b/src/gausskernel/process/postmaster/pgarch.cpp index 123529bbd..626c6af53 100644 --- a/src/gausskernel/process/postmaster/pgarch.cpp +++ b/src/gausskernel/process/postmaster/pgarch.cpp @@ -36,6 +36,7 @@ #include "access/xlog.h" #include "access/xlog_internal.h" +#include "access/xact.h" #include "libpq/pqsignal.h" #include "miscadmin.h" #include "postmaster/fork_process.h" @@ -150,8 +151,14 @@ ThreadId pgarch_start(void) /* * Do nothing if no archiver needed */ - if (!XLogArchivingActive()) + if (!XLogArchivingActive() && !(getObsReplicationSlot())) return 0; + load_server_mode(); + if (getObsReplicationSlot() != NULL && + t_thrd.xlog_cxt.server_mode != PRIMARY_MODE && + t_thrd.xlog_cxt.server_mode != STANDBY_MODE) { + return 0; + } /* * Do nothing if too soon since last archiver start. This is a safety * valve to protect against continuous respawn attempts if the archiver is @@ -303,6 +310,27 @@ static void VerifyDestDirIsEmptyOrCreate(char* dirname) return; } +/* + * 1. When synchronous_standby_names is NULL, no error is reported. + * + * 2. If the value of synchronous_standby_names is not NULL but no standby node is connected, a Warning log is reported. + * + * 3. When the standby node is connected to the synchronization but the error persists, an error-level log is reported. + * As a result, the archiver is trapped in a loop of opening and exiting abnormally. + */ +static inline void getSyncRecPtrErrorHandler (bool* amSync) { + if (t_thrd.syncrep_cxt.SyncRepConfig == NULL) { + /* wait a bit before retrying */ + pg_usleep(1000000L); + return; + } + List* syncStandbyList = SyncRepGetSyncStandbys(amSync); + int syncStandbyNums = list_length(syncStandbyList); + list_free(syncStandbyList); + int elevel = syncStandbyNums == 0 ? WARNING : ERROR; + ereport(elevel, (errmsg("pgarch_ArchiverObsCopyLoop failed when call SyncRepGetSyncRecPtr"))); +} + /* * pgarch_MainLoop * @@ -344,7 +372,7 @@ static void pgarch_MainLoop(void) if (t_thrd.arch.got_SIGHUP) { t_thrd.arch.got_SIGHUP = false; ProcessConfigFile(PGC_SIGHUP); - if (!XLogArchivingActive()) { + if (!XLogArchivingActive() && getObsReplicationSlot() == NULL) { ereport(LOG, (errmsg("PgArchiver exit"))); return; } @@ -393,12 +421,23 @@ static void pgarch_MainLoop(void) } else { got_recptr = SyncRepGetSyncRecPtr(&receivePtr, &writePtr, &flushPtr, &replayPtr, &amSync, false); if (got_recptr != true) { - ereport(ERROR, - (errmsg("pgarch_ArchiverObsCopyLoop failed when call SyncRepGetSyncRecPtr"))); + getSyncRecPtrErrorHandler(&amSync); + continue; } } + if (t_thrd.arch.pitr_task_last_lsn == InvalidXLogRecPtr) { + initLastTaskLsn(); + } + uint32 size; + if (obs_archive_slot->archive_obs->media_type == ARCHIVE_OBS) { + size = OBS_XLOG_SLICE_BLOCK_SIZE; + } else if (obs_archive_slot->archive_obs->media_type == ARCHIVE_NAS) { + size = NAS_XLOG_FILE_SIZE; + } else { + ereport(ERROR, (errmsg("unknown media type"))); + } if (time_diff >= t_thrd.arch.task_wait_interval - || XLByteDifference(flushPtr, t_thrd.arch.pitr_task_last_lsn) >= OBS_XLOG_SLICE_BLOCK_SIZE) { + || XLByteDifference(flushPtr, t_thrd.arch.pitr_task_last_lsn) >= size) { if (IS_PGXC_COORDINATOR) { fun = &pgarch_archiveRoachForCoordinator; } else { @@ -463,7 +502,7 @@ static void pgarch_MainLoop(void) * or after completing one more archiving cycle after receiving * SIGUSR2. */ - } while (PostmasterIsAlive() && XLogArchivingActive() && !time_to_stop); + } while (PostmasterIsAlive() && !time_to_stop && (XLogArchivingActive() || getObsReplicationSlot() != NULL)); } /* @@ -612,8 +651,6 @@ static bool PgarchArchiveXlogToDest(const char* xlog) */ static void pgarch_ArchiverObsCopyLoop(XLogRecPtr flushPtr, doArchive fun) { - ereport(LOG, - (errmsg("pgarch_ArchiverObsCopyLoop"))); struct timeval tv; bool time_to_stop = false; @@ -645,18 +682,19 @@ static void pgarch_ArchiverObsCopyLoop(XLogRecPtr flushPtr, doArchive fun) */ if (t_thrd.arch.got_SIGHUP) { ProcessConfigFile(PGC_SIGHUP); - if (!XLogArchivingActive()) { + if (getObsReplicationSlot() == NULL) { return; } t_thrd.arch.got_SIGHUP = false; } - targetLsn = Min(t_thrd.arch.pitr_task_last_lsn + OBS_XLOG_SLICE_BLOCK_SIZE - - (t_thrd.arch.pitr_task_last_lsn % OBS_XLOG_SLICE_BLOCK_SIZE) - 1, + + uint32 size = isObsSlot() ? OBS_XLOG_SLICE_BLOCK_SIZE : NAS_XLOG_FILE_SIZE; + targetLsn = Min(t_thrd.arch.pitr_task_last_lsn + size - (t_thrd.arch.pitr_task_last_lsn % size) - 1, flushPtr); /* The previous slice has been archived, switch to the next. */ if (t_thrd.arch.pitr_task_last_lsn == targetLsn) { - targetLsn = Min(targetLsn + OBS_XLOG_SLICE_BLOCK_SIZE, flushPtr); + targetLsn = Min(targetLsn + size, flushPtr); } if (fun(targetLsn) == false) { @@ -942,8 +980,6 @@ static void pgarch_archiveRoachForPitrStandby() static bool pgarch_archiveRoachForPitrMaster(XLogRecPtr targetLsn) { ResetLatch(&t_thrd.arch.mainloop_latch); - ereport(LOG, - (errmsg("pgarch_archiveRoachForPitrMaster %X/%X", (uint32)(targetLsn >> 32), (uint32)(targetLsn)))); int rc; WalSnd* walsnd = pgarch_chooseWalsnd(targetLsn); if (walsnd == NULL) { diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index f5650dd8b..7a0bfbd28 100755 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -2785,14 +2785,11 @@ static int ServerLoop(void) } /* If we have lost the archiver, try to start a new one */ - if (XLogArchivingActive() && g_instance.pid_cxt.PgArchPID == 0 && !dummyStandbyMode){ - if (pmState == PM_RUN) { - g_instance.pid_cxt.PgArchPID = pgarch_start(); - } else if (pmState == PM_HOT_STANDBY) { - obs_slot = getObsReplicationSlot(); - if (obs_slot != NULL) { - g_instance.pid_cxt.PgArchPID = pgarch_start(); - } + if (g_instance.pid_cxt.PgArchPID == 0 && !dummyStandbyMode) { + obs_slot = getObsReplicationSlot(); + if ((XLogArchivingActive() && pmState == PM_RUN) || + ((pmState == PM_RUN || pmState == PM_HOT_STANDBY) && obs_slot != NULL)) { + g_instance.pid_cxt.PgArchPID = pgarch_start(); } } @@ -2859,6 +2856,7 @@ static int ServerLoop(void) g_instance.pid_cxt.CsnminSyncPID = initialize_util_thread(CSNMIN_SYNC); } +#ifdef ENABLE_MULTIPLE_NODES /* If we have lost the barrier creator thread, try to start a new one */ if (START_BARRIER_CREATOR && g_instance.pid_cxt.BarrierCreatorPID == 0 && pmState == PM_RUN && XLogArchivingActive()) { @@ -2867,7 +2865,7 @@ static int ServerLoop(void) g_instance.pid_cxt.BarrierCreatorPID = initialize_util_thread(BARRIER_CREATOR); } } - +#endif /* If we need to signal the autovacuum launcher, do so now */ if (t_thrd.postmaster_cxt.avlauncher_needs_signal) { t_thrd.postmaster_cxt.avlauncher_needs_signal = false; @@ -5130,11 +5128,12 @@ static void reaper(SIGNAL_ARGS) if (NeedHeartbeat()) g_instance.pid_cxt.HeartbeatPID = initialize_util_thread(HEARTBEAT); +#ifdef ENABLE_MULTIPLE_NODES if (START_BARRIER_CREATOR && g_instance.pid_cxt.BarrierCreatorPID == 0 && XLogArchivingActive() && getObsReplicationSlot() != NULL) { g_instance.pid_cxt.BarrierCreatorPID = initialize_util_thread(BARRIER_CREATOR); } - +#endif if (GTM_LITE_CN && g_instance.pid_cxt.CsnminSyncPID == 0) { g_instance.pid_cxt.CsnminSyncPID = initialize_util_thread(CSNMIN_SYNC); } @@ -5417,14 +5416,12 @@ static void reaper(SIGNAL_ARGS) if (!EXIT_STATUS_0(exitstatus)) LogChildExit(LOG, _("archiver process"), pid, exitstatus); - if (XLogArchivingActive()) { - if (pmState == PM_RUN) { + /* If we have lost the archiver, try to start a new one */ + if (g_instance.pid_cxt.PgArchPID == 0 && !dummyStandbyMode) { + obs_slot = getObsReplicationSlot(); + if ((XLogArchivingActive() && pmState == PM_RUN) || + ((pmState == PM_RUN || pmState == PM_HOT_STANDBY) && obs_slot != NULL)) { g_instance.pid_cxt.PgArchPID = pgarch_start(); - }else if (pmState == PM_HOT_STANDBY) { - obs_slot = getObsReplicationSlot(); - if (obs_slot != NULL) { - g_instance.pid_cxt.PgArchPID = pgarch_start(); - } } } continue; diff --git a/src/gausskernel/storage/access/obs/Makefile b/src/gausskernel/storage/access/obs/Makefile index d2110db47..693a9b646 100644 --- a/src/gausskernel/storage/access/obs/Makefile +++ b/src/gausskernel/storage/access/obs/Makefile @@ -33,6 +33,6 @@ ifneq "$(MAKECMDGOALS)" "clean" endif endif -OBJS = obs_am.o +OBJS = obs_am.o nas_am.o archive_am.o include $(top_srcdir)/src/gausskernel/common.mk diff --git a/src/gausskernel/storage/access/obs/archive_am.cpp b/src/gausskernel/storage/access/obs/archive_am.cpp new file mode 100644 index 000000000..0b62de0b2 --- /dev/null +++ b/src/gausskernel/storage/access/obs/archive_am.cpp @@ -0,0 +1,103 @@ +/* ------------------------------------------------------------------------- + * Portions Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * Portions Copyright 2008 Bryan Ischo + * + * + * obs_am.cpp + * obs access method definitions. + * + * IDENTIFICATION + * src/gausskernel/storage/access/obs/obs_am.cpp + * + * ------------------------------------------------------------------------- + */ +#include +#include +#include +#include +#include + +#include "access/obs/archive_am.h" +#include "access/obs/nas_am.h" +#include "access/obs/obs_am.h" + +#include "replication/slot.h" + + +size_t ArchiveRead(const char* fileName, const int offset, char *buffer, const int length, + ObsArchiveConfig *archive_config) +{ + ObsArchiveConfig *archive_obs = NULL; + + if (archive_config != NULL) { + archive_obs = archive_config; + } else { + archive_obs = getObsArchiveConfig(); + } + + if (archive_obs->media_type == ARCHIVE_OBS) { + return obsRead(fileName, offset, buffer, length, archive_obs); + } else if (archive_obs->media_type == ARCHIVE_NAS) { + return NasRead(fileName, offset, buffer, length, archive_obs); + } + + return 0; +} + +int ArchiveWrite(const char* fileName, const char *buffer, const int bufferLength, ObsArchiveConfig *archive_slot) +{ + int ret = -1; + if (archive_slot == NULL) { + return ret; + } + + if (archive_slot->media_type == ARCHIVE_OBS) { + ret = obsWrite(fileName, buffer, bufferLength, archive_slot); + } else if (archive_slot->media_type == ARCHIVE_NAS) { + ret = NasWrite(fileName, buffer, bufferLength, archive_slot); + } + + return ret; +} + +int ArchiveDelete(const char* fileName, ObsArchiveConfig *archive_config) +{ + int ret = -1; + + ObsArchiveConfig *archive_obs = NULL; + + if (archive_config != NULL) { + archive_obs = archive_config; + } else { + archive_obs = getObsArchiveConfig(); + } + + if (archive_obs->media_type == ARCHIVE_OBS) { + ret = obsDelete(fileName, archive_obs); + } else if (archive_obs->media_type == ARCHIVE_NAS) { + ret = NasDelete(fileName, archive_obs); + } + + return ret; +} + +List* ArchiveList(const char* prefix, ObsArchiveConfig *archive_config, bool reportError, bool shortenConnTime) +{ + List* fileNameList = NIL; + + ObsArchiveConfig *archive_obs = NULL; + + if (archive_config != NULL) { + archive_obs = archive_config; + } else { + archive_obs = getObsArchiveConfig(); + } + + if (archive_obs->media_type == ARCHIVE_OBS) { + fileNameList = obsList(prefix, archive_obs); + } else if (archive_obs->media_type == ARCHIVE_NAS) { + fileNameList = NasList(prefix, archive_obs); + } + + return fileNameList; +} diff --git a/src/gausskernel/storage/access/obs/nas_am.cpp b/src/gausskernel/storage/access/obs/nas_am.cpp new file mode 100644 index 000000000..8ae66d15d --- /dev/null +++ b/src/gausskernel/storage/access/obs/nas_am.cpp @@ -0,0 +1,299 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * nas_am.h + * nas access method definitions. + * + * IDENTIFICATION + * src/gausskernel/storage/access/archive/nas_am.cpp + * + * ------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include +#include + +#include "access/obs/nas_am.h" + +#include "lib/stringinfo.h" +#include "miscadmin.h" +#include "nodes/nodes.h" +#include "nodes/value.h" +#include "pgstat.h" +#include "pgxc/locator.h" +#include "pgxc/pgxc.h" +#include "storage/lock/lwlock.h" +#include "securec.h" +#include "utils/elog.h" +#include "utils/palloc.h" +#include "utils/plog.h" +#include "postmaster/alarmchecker.h" +#include "replication/walreceiver.h" + +#define MAX_PATH_LEN 1024 + +size_t NasRead(const char* fileName, const int offset, char *buffer, const int length, ObsArchiveConfig *nas_config) +{ + size_t readLength = 0; + char file_path[MAXPGPATH] = {0}; + int ret = 0; + FILE *fp = NULL; + struct stat statbuf; + + if ((fileName == NULL) || (buffer == NULL)) { + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("The parameter cannot be NULL"))); + } + + if (nas_config == NULL) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Cannot get archive config from replication slots"))); + } + + ret = snprintf_s(file_path, MAXPGPATH, MAXPGPATH - 1, "%s%s", nas_config->obs_prefix, fileName); + securec_check_ss(ret, "\0", "\0"); + + if (stat(file_path, &statbuf)) { + if (errno != ENOENT) { + ereport(ERROR, (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", fileName))); + } + ereport(ERROR, (errcode_for_file_access(), errmsg("The file \"%s\" not exists", fileName))); + return readLength; + } + + canonicalize_path(file_path); + fp = fopen(file_path, "rb"); + if (fp == NULL) { + ereport(ERROR, (errcode_for_file_access(), errmsg("could not read file \"%s\": %m", fileName))); + return readLength; + } + if (statbuf.st_size > length) { + fclose(fp); + ereport(ERROR, (errcode_for_file_access(), errmsg("file size is wrong, \"%s\": %m", fileName))); + return readLength; + } + + readLength = fread(buffer, 1, statbuf.st_size, fp); + + fclose(fp); + return readLength; +} + +int NasWrite(const char* fileName, const char *buffer, const int bufferLength, ObsArchiveConfig *nas_config) +{ + int ret = 0; + ObsArchiveConfig *archive_nas = nas_config; + char file_path[MAXPGPATH] = {0}; + char *origin_file_path = NULL; + char *base_path = NULL; + FILE *fp = NULL; + + if ((fileName == NULL) || (buffer == NULL)) { + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("The parameter cannot be NULL"))); + } + + if (archive_nas == NULL) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Cannot get archive config from replication slots"))); + } + + ret = snprintf_s(file_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s/%s", archive_nas->obs_prefix, XLOGDIR, fileName); + securec_check_ss(ret, "\0", "\0"); + + canonicalize_path(file_path); + + origin_file_path = pstrdup(file_path); + base_path = dirname(origin_file_path); + if (!isDirExist(base_path)) { + if (pg_mkdir_p(base_path, S_IRWXU) != 0) { + pfree_ext(origin_file_path); + ereport(LOG, (errmsg("could not create path \"%s\"", base_path))); + return -1; + } + } + + fp = fopen(file_path, "wb"); + if (fp == NULL) { + pfree_ext(origin_file_path); + ereport(LOG, (errmsg("could not create file \"%s\": %m", fileName))); + return -1; + } + + if (fwrite(buffer, bufferLength, 1, fp) != 1) { + pfree_ext(origin_file_path); + fclose(fp); + return -1; + } + + pfree_ext(origin_file_path); + fclose(fp); + return 0; +} + +int NasDelete(const char* fileName, ObsArchiveConfig *nas_config) +{ + int ret = 0; + struct stat statbuf; + char file_path[MAXPGPATH] = {0}; + + if (nas_config == NULL) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Cannot get obs bucket config from replication slots"))); + } + + ret = snprintf_s(file_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s", nas_config->obs_prefix, fileName); + securec_check_ss(ret, "\0", "\0"); + + if (lstat(file_path, &statbuf) < 0) { + return -1; + } + if (S_ISDIR(statbuf.st_mode)) { + return (rmdir(file_path)); + } + return (unlink(file_path)); +} + +/* + * Obtain files with specified prefix in archive directory, unsorted + * The prefix can be: + * 1. path + * 2. filename + * 3. the prefix of filename + */ +static List* GetNasFileList(const char* prefix, ObsArchiveConfig *nas_config) +{ + int ret = 0; + char file_path[MAXPGPATH] = {0}; + char path_buf[MAXPGPATH] = {0}; + char *origin_file_path = NULL; + char *base_path = NULL; + List* fileNameList = NIL; + struct dirent* de = NULL; + struct stat st; + bool isDir = false; + + if (prefix == NULL || nas_config == NULL || nas_config->obs_prefix == NULL) { + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("The parameter cannot be NULL"))); + } + + ret = snprintf_s(file_path, MAXPGPATH, MAXPGPATH - 1, "%s/%s", nas_config->obs_prefix, prefix); + securec_check_ss(ret, "\0", "\0"); + + canonicalize_path(file_path); + + if (stat(file_path, &st) == 0 && S_ISDIR(st.st_mode)) { // is dir, + isDir = true; + base_path = file_path; + } else { // may be the file_path is filename or the prefix of filename + struct stat st_base; + origin_file_path = pstrdup(file_path); + base_path = dirname(origin_file_path); + if (stat(base_path, &st_base) != 0) { + ereport(LOG, (errmsg("WARNING: there is no file in dir %s", file_path))); + return NIL; // the base_path not exists, return NIL + } + } + + DIR *dir = opendir(base_path); + while ((de = readdir(dir)) != NULL) { + if (isDir) { + if ((strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)) { + continue; + } + ret = snprintf_s(path_buf, MAXPGPATH, MAXPGPATH - 1, "%s/%s", base_path, de->d_name); + securec_check_ss_c(ret, "\0", "\0"); + if (lstat(path_buf, &st) != 0) { + continue; + } + /* only find file */ + if (S_ISREG(st.st_mode)) { + fileNameList = lappend(fileNameList, pstrdup(path_buf)); + } else { + continue; + } + } else { + if (strncmp(de->d_name, basename(file_path), strlen(basename(file_path))) == 0) { + ret = snprintf_s(path_buf, MAXPGPATH, MAXPGPATH - 1, "%s/%s", base_path, de->d_name); + securec_check_ss_c(ret, "\0", "\0"); + fileNameList = lappend(fileNameList, pstrdup(path_buf)); + } else { + continue; + } + } + } + + closedir(dir); + pfree_ext(origin_file_path); + return fileNameList; +} + +static int CompareFileNames(const void* a, const void* b) +{ + char* fna = *((char**)a); + char* fnb = *((char**)b); + + return strcmp(fna, fnb); +} + + +static List* SortFileList(List* file_list) +{ + int file_num; + char** files; + ListCell* lc = NULL; + List* result = NIL; + int i = 0; + + file_num = list_length(file_list); + if (file_num < 1) { + return NIL; + } + + files = (char**)palloc0(file_num * sizeof(char*)); + foreach (lc, file_list) { + files[i++] = (char*)lfirst(lc); + } + qsort(files, file_num, sizeof(char*), CompareFileNames); + for (i = 0; i < file_num; i++) { + result = lappend(result, pstrdup(files[i])); + } + + pfree_ext(files); + return result; +} + +List* NasList(const char* prefix, ObsArchiveConfig *nas_config) +{ + List* fileNameList = NIL; + List* fileNameListTmp = NIL; + + if (nas_config == NULL) { + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("Cannot get archive config from replication slots"))); + } + + fileNameListTmp = GetNasFileList(prefix, nas_config); + + fileNameList = SortFileList(fileNameListTmp); + + list_free_ext(fileNameListTmp); + return fileNameList; +} diff --git a/src/gausskernel/storage/access/transam/xlogfuncs.cpp b/src/gausskernel/storage/access/transam/xlogfuncs.cpp index 26d6a9944..2e16b7af0 100644 --- a/src/gausskernel/storage/access/transam/xlogfuncs.cpp +++ b/src/gausskernel/storage/access/transam/xlogfuncs.cpp @@ -1146,9 +1146,14 @@ Datum gs_set_obs_delete_location(PG_FUNCTION_ARGS) } XLByteToSeg(locationpoint, xlogsegno); - errorno = snprintf_s(xlogfilename, MAXFNAMELEN, MAXFNAMELEN - 1, "%08X%08X%08X_%02u", DEFAULT_TIMELINE_ID, + if (!isObsSlot()) { + errorno = snprintf_s(xlogfilename, MAXFNAMELEN, MAXFNAMELEN - 1, "%08X%08X%08X", DEFAULT_TIMELINE_ID, + (uint32)((xlogsegno) / XLogSegmentsPerXLogId), (uint32)((xlogsegno) % XLogSegmentsPerXLogId)); + } else { + errorno = snprintf_s(xlogfilename, MAXFNAMELEN, MAXFNAMELEN - 1, "%08X%08X%08X_%02u", DEFAULT_TIMELINE_ID, (uint32)((xlogsegno) / XLogSegmentsPerXLogId), (uint32)((xlogsegno) % XLogSegmentsPerXLogId), (uint32)((locationpoint / OBS_XLOG_SLICE_BLOCK_SIZE) & OBS_XLOG_SLICE_NUM_MAX)); + } securec_check_ss(errorno, "", ""); PG_RETURN_TEXT_P(cstring_to_text(xlogfilename)); diff --git a/src/gausskernel/storage/replication/obswalreceiver.cpp b/src/gausskernel/storage/replication/obswalreceiver.cpp index 399e5b8b0..487a7f0ac 100644 --- a/src/gausskernel/storage/replication/obswalreceiver.cpp +++ b/src/gausskernel/storage/replication/obswalreceiver.cpp @@ -24,6 +24,7 @@ #include "access/xlog_internal.h" #include "nodes/pg_list.h" #include "access/obs/obs_am.h" +#include "access/obs/archive_am.h" #include "utils/timestamp.h" #include "miscadmin.h" #include "replication/walreceiver.h" @@ -174,8 +175,10 @@ bool obs_receive(int timeout, unsigned char* type, char** buffer, int* len) msghdr.sender_flush_location = InvalidXLogRecPtr; msghdr.catchup = false; + uint32 size = isObsSlot() ? OBS_XLOG_SLICE_BLOCK_SIZE : NAS_XLOG_FILE_SIZE; + int headLen = sizeof(WalDataMessageHeader); - int totalLen = headLen + OBS_XLOG_SLICE_BLOCK_SIZE + 1; + int totalLen = headLen + size + 1; // copy WalDataMessageHeader rc = memcpy_s(recvBuf, totalLen, &msghdr, headLen); securec_check(rc, "", ""); @@ -230,9 +233,14 @@ static char *obs_replication_get_xlog_prefix(XLogRecPtr recptr, bool onlyPath) /* Generate directory path of pg_xlog on OBS when onlyPath is true */ if (onlyPath == false) { XLByteToSeg(recptr, xlogSegno); - rc = snprintf_s(xlogfname, MAXFNAMELEN, MAXFNAMELEN - 1, "%08X%08X%08X_%02u", timeLine, + if (isObsSlot()) { + rc = snprintf_s(xlogfname, MAXFNAMELEN, MAXFNAMELEN - 1, "%08X%08X%08X_%02u", timeLine, (uint32)((xlogSegno) / XLogSegmentsPerXLogId), (uint32)((xlogSegno) % XLogSegmentsPerXLogId), (uint32)((recptr / OBS_XLOG_SLICE_BLOCK_SIZE) & OBS_XLOG_SLICE_NUM_MAX)); + } else { + rc = snprintf_s(xlogfname, MAXFNAMELEN, MAXFNAMELEN - 1, "%08X%08X%08X", timeLine, + (uint32)((xlogSegno) / XLogSegmentsPerXLogId), (uint32)((xlogSegno) % XLogSegmentsPerXLogId)); + } securec_check_ss_c(rc, "", ""); } rc = snprintf_s(xlogfpath, MAXPGPATH, MAXPGPATH - 1, XLOGDIR "/%s", xlogfname); @@ -243,6 +251,9 @@ static char *obs_replication_get_xlog_prefix(XLogRecPtr recptr, bool onlyPath) static char *path_skip_prefix(char *path) { + if (!isObsSlot()) { + return path; + } char *key = path; /* Skip path prefix, prefix format:'xxxx/cn/' */ for (int i = 0; i <= 1; i++) { @@ -283,7 +294,7 @@ static char *obs_replication_get_last_xlog_slice(XLogRecPtr startPtr, bool onlyP fileNamePrefix = obs_replication_get_xlog_prefix(startPtr, onlyPath); - object_list = obsList(fileNamePrefix); + object_list = ArchiveList(fileNamePrefix); if (object_list == NIL || object_list->length <= 0) { ereport(LOG, (errmsg("The OBS objects with the prefix %s cannot be found.", fileNamePrefix))); @@ -401,9 +412,13 @@ int obs_replication_archive(const ArchiveXlogMessage *xlogInfo) uint offset = 0; XLogSegNo xlogSegno = 0; - - xlogBuff = (char *)palloc(OBS_XLOG_SLICE_FILE_SIZE); - rc = memset_s(xlogBuff, OBS_XLOG_SLICE_FILE_SIZE, 0, OBS_XLOG_SLICE_FILE_SIZE); + ReplicationSlot *archive_slot = NULL; + archive_slot = getObsReplicationSlot(); + bool isObs = isObsSlot(); + uint32 fileSize = isObs ? OBS_XLOG_SLICE_FILE_SIZE : NAS_XLOG_FILE_SIZE; + uint32 blockSize = isObs ? OBS_XLOG_SLICE_BLOCK_SIZE : NAS_XLOG_FILE_SIZE; + xlogBuff = (char *)palloc(fileSize); + rc = memset_s(xlogBuff, fileSize, 0, fileSize); securec_check(rc, "", ""); /* generate xlog path */ @@ -423,24 +438,32 @@ int obs_replication_archive(const ArchiveXlogMessage *xlogInfo) errmsg("Can not open file \"%s\": %s", xlogfpath, strerror(errno)))); } - /* Align down to 2M */ - offset = TYPEALIGN_DOWN(OBS_XLOG_SLICE_BLOCK_SIZE, ((xlogInfo->targetLsn) % XLogSegSize)); + /* Align down to blockSize */ + offset = TYPEALIGN_DOWN(blockSize, ((xlogInfo->targetLsn) % XLogSegSize)); if (lseek(xlogreadfd, (off_t)offset, SEEK_SET) < 0) { ereport(ERROR, (errcode(ERRCODE_FILE_READ_FAILED), errmsg("Can not locate to offset[%u] of xlog file \"%s\": %s", offset, xlogfpath, strerror(errno)))); } - if (read(xlogreadfd, xlogBuff + OBS_XLOG_SLICE_HEADER_SIZE, OBS_XLOG_SLICE_BLOCK_SIZE) - != OBS_XLOG_SLICE_BLOCK_SIZE) { + uint64 readResult = 0; + if (isObs) { + readResult = read(xlogreadfd, xlogBuff + OBS_XLOG_SLICE_HEADER_SIZE, OBS_XLOG_SLICE_BLOCK_SIZE); + } else { + readResult = read(xlogreadfd, xlogBuff, NAS_XLOG_FILE_SIZE); + } + if (readResult != blockSize) { ereport(ERROR, (errcode(ERRCODE_FILE_READ_FAILED), errmsg("Can not read local xlog file \"%s\": %s", xlogfpath, strerror(errno)))); } /* Add xlog slice header for recording the actual xlog length */ - actualXlogLen = (((uint32)((xlogInfo->targetLsn) % XLogSegSize)) & (OBS_XLOG_SLICE_BLOCK_SIZE - 1)) + 1; - - *(uint32*)xlogBuff = htonl(actualXlogLen); + actualXlogLen = (((uint32)((xlogInfo->targetLsn) % XLogSegSize)) & (blockSize - 1)); + if (isObs) { + /* Add xlog slice header for recording the actual xlog length */ + actualXlogLen += 1; + *(uint32*)xlogBuff = htonl(actualXlogLen); + } close(xlogreadfd); @@ -449,12 +472,16 @@ int obs_replication_archive(const ArchiveXlogMessage *xlogInfo) fileName = (char*)palloc0(MAX_PATH_LEN); /* {xlog_name}_{sliece_num}_01(version_num)_00000001{tli}_00000001{subTerm} */ - rc = sprintf_s(fileName, MAX_PATH_LEN, "%s_%02d_%08u_%08u_%08d", fileNamePrefix, - CUR_OBS_FILE_VERSION, xlogInfo->term, xlogInfo->tli, xlogInfo->sub_term); + if (isObs) { + rc = sprintf_s(fileName, MAX_PATH_LEN, "%s_%02d_%08u_%08u_%08d", fileNamePrefix, + CUR_OBS_FILE_VERSION, xlogInfo->term, xlogInfo->tli, xlogInfo->sub_term); + } else { + rc = sprintf_s(fileName, MAX_PATH_LEN, "%s", xlogfname); + } securec_check_ss(rc, "\0", "\0"); /* Upload xlog slice file to OBS */ - ret = obsWrite(fileName, xlogBuff, OBS_XLOG_SLICE_FILE_SIZE); + ret = ArchiveWrite(fileName, xlogBuff, fileSize, archive_slot->archive_obs); pfree(xlogBuff); pfree(fileNamePrefix); @@ -486,7 +513,7 @@ int obs_replication_cleanup(XLogRecPtr recptr) fileNamePrefix = obs_replication_get_xlog_prefix(recptr, true); - object_list = obsList(fileNamePrefix); + object_list = ArchiveList(fileNamePrefix); if (object_list == NIL || object_list->length <= 0) { ereport(LOG, (errmsg("The OBS objects with the prefix %s cannot be found.", fileNamePrefix))); @@ -510,7 +537,7 @@ int obs_replication_cleanup(XLogRecPtr recptr) if (strncmp(basename(key), xlogfname, len) < 0) { /* Ahead of the target lsn, need to delete */ - ret = obsDelete(key); + ret = ArchiveDelete(key); if (ret != 0) { ereport(WARNING, (errcode(ERRCODE_UNDEFINED_FILE), errmsg("The OBS objects delete fail, ret=%d, key=%s", ret, key))); @@ -556,12 +583,20 @@ int obs_replication_get_last_xlog(ArchiveXlogMessage *xlogInfo) fileBaseName = basename(filePath); ereport(DEBUG1, (errmsg("The last xlog on OBS: %s", filePath))); - rc = sscanf_s(fileBaseName, "%8X%8X%8X_%2u_%02d_%08u_%08u_%08d", &timeLine, &xlogSegId, - &xlogSegOffset, &xlogInfo->slice, &version, &xlogInfo->term, &xlogInfo->tli, &xlogInfo->sub_term); - securec_check_for_sscanf_s(rc, 6, "\0", "\0"); + if (isObsSlot()) { + rc = sscanf_s(fileBaseName, "%8X%8X%8X_%2u_%02d_%08u_%08u_%08d", &timeLine, &xlogSegId, + &xlogSegOffset, &xlogInfo->slice, &version, &xlogInfo->term, &xlogInfo->tli, &xlogInfo->sub_term); + securec_check_for_sscanf_s(rc, 6, "\0", "\0"); - ereport(DEBUG1, (errmsg("Parse xlog filename is %8X%8X%8X_%2u_%02d_%08u_%08u_%08d", timeLine, xlogSegId, - xlogSegOffset, xlogInfo->slice, version, xlogInfo->term, xlogInfo->tli, xlogInfo->sub_term))); + ereport(DEBUG1, (errmsg("Parse xlog filename is %8X%8X%8X_%2u_%02d_%08u_%08u_%08d", timeLine, xlogSegId, + xlogSegOffset, xlogInfo->slice, version, xlogInfo->term, xlogInfo->tli, xlogInfo->sub_term))); + } else { + rc = sscanf_s(fileBaseName, "%8X%8X%8X", &timeLine, &xlogSegId, &xlogSegOffset); + securec_check_for_sscanf_s(rc, 3, "\0", "\0"); + ereport(DEBUG1, (errmsg("Parse xlog filename is %8X%8X%8X", timeLine, xlogSegId, xlogSegOffset))); + } + + XLogSegNoOffsetToRecPtr(xlogSegId * XLogSegmentsPerXLogId + xlogSegOffset, 0, xlogInfo->targetLsn); diff --git a/src/gausskernel/storage/replication/slot.cpp b/src/gausskernel/storage/replication/slot.cpp index 133ff0f4f..ef32eaeac 100644 --- a/src/gausskernel/storage/replication/slot.cpp +++ b/src/gausskernel/storage/replication/slot.cpp @@ -1737,34 +1737,59 @@ static char *trim_str(char *str, int str_len, char sep) char *formObsConfigStringFromStruct(ObsArchiveConfig *obs_config) { - if (obs_config == NULL) { + if (obs_config == NULL || obs_config->media_type == ARCHIVE_NONE) { return NULL; } - int length = 0; + int length = 4; char *result; int rc = 0; char encryptSecretAccessKeyStr[DEST_CIPHER_LENGTH] = {'\0'}; - encryptKeyString(obs_config->obs_sk, encryptSecretAccessKeyStr, DEST_CIPHER_LENGTH); - length += strlen(obs_config->obs_address) + 1; - length += strlen(obs_config->obs_bucket) + 1; - length += strlen(obs_config->obs_ak) + 1; + + if (obs_config->media_type == ARCHIVE_OBS) { + encryptKeyString(obs_config->obs_sk, encryptSecretAccessKeyStr, DEST_CIPHER_LENGTH); + length += strlen(obs_config->obs_address) + 1; + length += strlen(obs_config->obs_bucket) + 1; + length += strlen(obs_config->obs_ak) + 1; + length += strlen(encryptSecretAccessKeyStr); + } + /* for archive_prefix */ length += strlen(obs_config->obs_prefix) + 1; - length += strlen(encryptSecretAccessKeyStr); + /* for is_recovery */ + length += 1 + 1; + /* for vote_replicate_first */ + length += 1 + 1; + result = (char *)palloc0(length + 1); - rc = snprintf_s(result, length + 1, length, "%s;%s;%s;%s;%s", obs_config->obs_address, - obs_config->obs_bucket, obs_config->obs_ak, encryptSecretAccessKeyStr, obs_config->obs_prefix); + + if (obs_config->media_type == ARCHIVE_OBS) { + rc = snprintf_s(result, length + 1, length, "OBS;%s;%s;%s;%s;%s;%s;%s", + obs_config->obs_address, + obs_config->obs_bucket, + obs_config->obs_ak, + encryptSecretAccessKeyStr, + obs_config->obs_prefix); + } else { + // By default, is_recovery and vote_replicate are set to '0' for NAS archiving + rc = snprintf_s(result, length + 1, length, "NAS;%s;%s;%s", obs_config->obs_prefix, "0", "0"); + } + securec_check_ss_c(rc, "\0", "\0"); return result; } ObsArchiveConfig* formObsConfigFromStr(char *content, bool encrypted) { - ObsArchiveConfig* obs_config = NULL; + ObsArchiveConfig* obs_config = NULL; + char* media_type = NULL; errno_t rc = EOK; /* SplitIdentifierString will change origin string */ char *content_copy = pstrdup(content); + char *tmp = NULL; List* elemlist = NIL; int param_num = 0; + size_t elem_index = 0; + bool is_recovery = false; + bool vote_replicate_first = false; obs_config = (ObsArchiveConfig *)palloc0(sizeof(ObsArchiveConfig)); char decryptSecretAccessKeyStr[DEST_CIPHER_LENGTH] = {'\0'}; /* Parse string into list of identifiers */ @@ -1772,23 +1797,51 @@ ObsArchiveConfig* formObsConfigFromStr(char *content, bool encrypted) goto FAILURE; } param_num = list_length(elemlist); - if (param_num != 5) { + /* + * The extra_content when create archive slot + * OBS: OBS;obs_server_ip;obs_bucket_name;obs_ak;obs_sk;archive_prefix;is_recovery;is_vote_replication_first + * NAS: NAS;archive_prefix;is_recovery;is_vote_replication_first + */ + if (param_num != 7 && param_num != 4 && param_num != 8) { goto FAILURE; } - - obs_config->obs_address = pstrdup((char*)list_nth(elemlist, 0)); - obs_config->obs_bucket = pstrdup((char*)list_nth(elemlist, 1)); - obs_config->obs_ak = pstrdup((char*)list_nth(elemlist, 2)); - if (encrypted == false) { - obs_config->obs_sk = pstrdup((char*)list_nth(elemlist, 3)); + + if (param_num == 7) { + obs_config->media_type = ARCHIVE_OBS; } else { - decryptKeyString((char*)list_nth(elemlist, 3), decryptSecretAccessKeyStr, DEST_CIPHER_LENGTH, NULL); - obs_config->obs_sk = pstrdup(decryptSecretAccessKeyStr); - rc = memset_s(decryptSecretAccessKeyStr, DEST_CIPHER_LENGTH, 0, DEST_CIPHER_LENGTH); - securec_check(rc, "\0", "\0"); + media_type = pstrdup((char*)list_nth(elemlist, elem_index++)); + if (strcmp(media_type, "OBS") == 0) { + obs_config->media_type = ARCHIVE_OBS; + } else if (strcmp(media_type, "NAS") == 0) { + obs_config->media_type = ARCHIVE_NAS; + } else { + goto FAILURE; + } } - obs_config->obs_prefix = pstrdup((char*)list_nth(elemlist, 4)); + + if (obs_config->media_type == ARCHIVE_OBS) { + obs_config->obs_address = pstrdup((char*)list_nth(elemlist, elem_index++)); + obs_config->obs_bucket = pstrdup((char*)list_nth(elemlist, elem_index++)); + obs_config->obs_ak = pstrdup((char*)list_nth(elemlist, elem_index++)); + if (encrypted == false) { + obs_config->obs_sk = pstrdup((char*)list_nth(elemlist, elem_index++)); + } else { + decryptKeyString((char*)list_nth(elemlist, elem_index++), decryptSecretAccessKeyStr, DEST_CIPHER_LENGTH, + NULL); + obs_config->obs_sk = pstrdup(decryptSecretAccessKeyStr); + rc = memset_s(decryptSecretAccessKeyStr, DEST_CIPHER_LENGTH, 0, DEST_CIPHER_LENGTH); + securec_check(rc, "\0", "\0"); + } + } + + obs_config->obs_prefix = pstrdup((char*)list_nth(elemlist, elem_index++)); + tmp = (char*)list_nth(elemlist, elem_index++); + is_recovery = (tmp != NULL && tmp[0] != '\0' && tmp[0] == '1') ? true : false; + tmp = (char*)list_nth(elemlist, elem_index++); + vote_replicate_first = (tmp != NULL && tmp[0] != '\0' && tmp[0] == '1') ? true : false; + pfree_ext(content_copy); + pfree_ext(media_type); list_free_ext(elemlist); return obs_config; FAILURE: @@ -1801,6 +1854,7 @@ FAILURE: } pfree_ext(obs_config); pfree_ext(content_copy); + pfree_ext(media_type); list_free_ext(elemlist); ereport(ERROR, (errcode_for_file_access(), errmsg("message is inleagel \"%s\"", content))); @@ -1848,6 +1902,7 @@ ReplicationSlot *getObsReplicationSlot() g_instance.archive_obs_cxt.archive_slot->archive_obs->obs_ak = pstrdup_ext(slot->archive_obs->obs_ak); g_instance.archive_obs_cxt.archive_slot->archive_obs->obs_sk = pstrdup_ext(slot->archive_obs->obs_sk); g_instance.archive_obs_cxt.archive_slot->archive_obs->obs_prefix = pstrdup_ext(slot->archive_obs->obs_prefix); + g_instance.archive_obs_cxt.archive_slot->archive_obs->media_type = slot->archive_obs->media_type; MemoryContextSwitchTo(curr); SpinLockRelease(&slot->mutex); *slot_idx = slotno; diff --git a/src/include/access/obs/archive_am.h b/src/include/access/obs/archive_am.h new file mode 100644 index 000000000..08932ae64 --- /dev/null +++ b/src/include/access/obs/archive_am.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * archive_am.h + * nass access method definitions. + * + * + * IDENTIFICATION + * src/include/access/archive/archive_am.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef ARCHIVE_AM_H +#define ARCHIVE_AM_H + +#include "postgres.h" +#include "knl/knl_variable.h" +#include "nodes/pg_list.h" +#include "storage/buf/buffile.h" +#include "replication/slot.h" + +/* in archive/archive_am.cpp */ +size_t ArchiveRead(const char* fileName, int offset, char *buffer, int length, ObsArchiveConfig *archive_config = NULL); +int ArchiveWrite(const char* fileName, const char *buffer, const int bufferLength, + ObsArchiveConfig *archive_config = NULL); +int ArchiveDelete(const char* fileName, ObsArchiveConfig *archive_config = NULL); +List* ArchiveList(const char* prefix, ObsArchiveConfig *archive_config = NULL, + bool reportError = true, bool shortenConnTime = false); + +#endif /* ARCHIVE_AM_H */ + diff --git a/src/include/access/obs/nas_am.h b/src/include/access/obs/nas_am.h new file mode 100644 index 000000000..a1a1aca37 --- /dev/null +++ b/src/include/access/obs/nas_am.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * --------------------------------------------------------------------------------------- + * + * nas_am.h + * nass access method definitions. + * + * + * IDENTIFICATION + * src/include/access/archive/nas_am.h + * + * --------------------------------------------------------------------------------------- + */ + +#ifndef NAS_AM_H +#define NAS_AM_H + +#include "postgres.h" +#include "nodes/pg_list.h" +#include "replication/slot.h" + +size_t NasRead(const char* fileName, int offset, char *buffer, int length, ObsArchiveConfig *nas_config = NULL); +int NasWrite(const char* fileName, const char *buffer, const int bufferLength, ObsArchiveConfig *nas_config = NULL); +int NasDelete(const char* fileName, ObsArchiveConfig *nas_config = NULL); +List* NasList(const char* prefix, ObsArchiveConfig *nas_config = NULL); + +#endif /* NAS_AM_H */ + diff --git a/src/include/replication/obswalreceiver.h b/src/include/replication/obswalreceiver.h index dcb3fd615..0ed8a4195 100644 --- a/src/include/replication/obswalreceiver.h +++ b/src/include/replication/obswalreceiver.h @@ -45,9 +45,12 @@ extern void obs_disconnect(void); #define OBS_XLOG_SLICE_HEADER_SIZE (sizeof(uint32)) /* sizeof(uint32) + OBS_XLOG_SLICE_BLOCK_SIZE */ #define OBS_XLOG_SLICE_FILE_SIZE (OBS_XLOG_SLICE_BLOCK_SIZE + OBS_XLOG_SLICE_HEADER_SIZE) +#define NAS_XLOG_FILE_SIZE ((uint32)(16 * 1024 * 1024)) #define OBS_XLOG_SAVED_FILES_NUM 25600 /* 100G*1024*1024*1024/OBS_XLOG_SLICE_BLOCK_SIZE */ + +/* Currently, the openGauss does not support disaster recover. */ #define IS_DISASTER_RECOVER_MODE \ - (t_thrd.xlog_cxt.server_mode == STANDBY_MODE && !XLogArchivingActive() && getObsReplicationSlot()) + (false && t_thrd.xlog_cxt.server_mode == STANDBY_MODE && !XLogArchivingActive() && getObsReplicationSlot()) #define IS_CNDISASTER_RECOVER_MODE \ (IS_PGXC_COORDINATOR && !XLogArchivingActive() && getObsReplicationSlot()) diff --git a/src/include/replication/slot.h b/src/include/replication/slot.h index 07f2cf3e8..61a87f37a 100644 --- a/src/include/replication/slot.h +++ b/src/include/replication/slot.h @@ -34,6 +34,12 @@ extern const uint32 EXTRA_SLOT_VERSION_NUM; */ typedef enum ReplicationSlotPersistency { RS_PERSISTENT, RS_EPHEMERAL, RS_BACKUP } ReplicationSlotPersistency; +typedef enum ArchiveMediaType { + ARCHIVE_NONE, + ARCHIVE_OBS, + ARCHIVE_NAS +} ArchiveMediaType; + /* * On-Disk data of a replication slot, preserved across restarts. */ @@ -97,6 +103,7 @@ typedef struct ObsArchiveConfig { char *obs_ak; char *obs_sk; char *obs_prefix; + ArchiveMediaType media_type; } ObsArchiveConfig; /* @@ -152,7 +159,6 @@ typedef struct ReplicationSlot { char* extra_content; } ReplicationSlot; - #define ReplicationSlotPersistentDataConstSize sizeof(ReplicationSlotPersistentData) /* size of the part of the slot that is version independent */ #define ReplicationSlotOnDiskConstantSize offsetof(ReplicationSlotOnDisk, slotdata) @@ -259,4 +265,18 @@ extern void advanceObsSlot(XLogRecPtr restart_pos); extern void redo_slot_reset_for_backup(const ReplicationSlotPersistentData *xlrec); extern void markObsSlotOperate(int p_slot_num); +inline bool isObsSlot() { + ReplicationSlot* slot = getObsReplicationSlot(); + if (slot == NULL) + ereport(ERROR, (errmsg("cannot get the slot"))); + if (slot->archive_obs->media_type == ARCHIVE_OBS) { + return true; + } else if (slot->archive_obs->media_type == ARCHIVE_NAS) { + return false; + } else { + ereport(ERROR, (errmsg("unknown media type"))); + } + return false; +} + #endif /* SLOT_H */ diff --git a/src/test/ha/deploy_multi_single_archive.sh b/src/test/ha/deploy_multi_single_archive.sh new file mode 100644 index 000000000..ca342d6ea --- /dev/null +++ b/src/test/ha/deploy_multi_single_archive.sh @@ -0,0 +1,65 @@ +#!/bin/sh +# deploy primary-standby-dummystandby + +source ./standby_env.sh + +node_num=4 +#python $scripts_dir/pgxc_multi.py +#stop the database +python $scripts_dir/pgxc_multi_single.py -o + +sleep 2 +#init the database +python $scripts_dir/pgxc_multi_single.py -c 1 -d $node_num + +#build the standby +gs_ctl build -D $data_dir/datanode1_standby -Z single_node +gs_ctl build -D $data_dir/datanode4_standby -Z single_node + + +#stop the database +python $scripts_dir/pgxc_multi_single.py -o + +#set the primary postgresql.conf file +gs_guc set -Z datanode -D $primary_data_dir -c "most_available_sync = on" +gs_guc set -Z datanode -D $primary_data_dir -c "synchronous_commit = on" +gs_guc set -Z datanode -D $primary_data_dir -c "log_min_messages = DEBUG5" +gs_guc set -Z datanode -D $primary_data_dir -c "data_replicate_buffer_size=256MB" +gs_guc set -Z datanode -D $primary_data_dir -c "walsender_max_send_size=8MB" +gs_guc set -Z datanode -D $primary_data_dir -c "wal_receiver_buffer_size=64MB" +gs_guc set -Z datanode -D $primary_data_dir -c "shared_buffers=2GB" +gs_guc set -Z datanode -D $primary_data_dir -c "modify_initial_password = off" +gs_guc set -Z datanode -D $primary_data_dir -c "wal_sender_timeout = 120s" +gs_guc set -Z datanode -D $primary_data_dir -c "wal_receiver_timeout = 120s" +gs_guc set -Z datanode -D $primary_data_dir -c "max_replication_slots = 8" +gs_guc set -Z datanode -D $primary_data_dir -c "max_wal_senders = 8" +gs_guc set -Z datanode -D $primary_data_dir -c "replication_type = 1" +gs_guc set -Z datanode -D $primary_data_dir -c "enable_data_replicate = off" + +echo $node_num +for((i=1; i<=$node_num; i++)) +do + datanode_dir=$data_dir/datanode$i + datanode_dir=$datanode_dir"_standby" + echo $datanode_dir + gs_guc set -Z datanode -D $datanode_dir -c "most_available_sync = on" + gs_guc set -Z datanode -D $datanode_dir -c "synchronous_commit = on" + gs_guc set -Z datanode -D $datanode_dir -c "log_min_messages = DEBUG5" + gs_guc set -Z datanode -D $datanode_dir -c "data_replicate_buffer_size=256MB" + gs_guc set -Z datanode -D $datanode_dir -c "walsender_max_send_size=8MB" + gs_guc set -Z datanode -D $datanode_dir -c "wal_receiver_buffer_size=64MB" + gs_guc set -Z datanode -D $datanode_dir -c "shared_buffers=2GB" + gs_guc set -Z datanode -D $datanode_dir -c "modify_initial_password = off" + gs_guc set -Z datanode -D $datanode_dir -c "wal_sender_timeout = 120s" + gs_guc set -Z datanode -D $datanode_dir -c "wal_receiver_timeout = 120s" + gs_guc set -Z datanode -D $datanode_dir -c "max_replication_slots = 8" + gs_guc set -Z datanode -D $datanode_dir -c "max_wal_senders = 8" + gs_guc set -Z datanode -D $datanode_dir -c "replication_type = 1" + gs_guc set -Z datanode -D $datanode_dir -c "enable_data_replicate = off" +done + +#python $scripts_dir/pgxc_multi.py -o + +sleep 2 +#start the database +python $scripts_dir/pgxc_multi_single.py -s diff --git a/src/test/ha/ha_schedule_multi_single_archive b/src/test/ha/ha_schedule_multi_single_archive new file mode 100644 index 000000000..f10380c18 --- /dev/null +++ b/src/test/ha/ha_schedule_multi_single_archive @@ -0,0 +1 @@ +multi_standby_single_archive/archive_slot diff --git a/src/test/ha/run_ha_multi_single_archive.sh b/src/test/ha/run_ha_multi_single_archive.sh new file mode 100644 index 000000000..365b7c629 --- /dev/null +++ b/src/test/ha/run_ha_multi_single_archive.sh @@ -0,0 +1,58 @@ +#!/bin/sh +# run all the test case of ha + +#init some variables +loop_num=$1 +if [ -z $1 ]; then + loop_num=1 +fi +count=0 + +source ./standby_env.sh +test -f regression.diffs.hacheck && rm regression.diffs.hacheck + +total_starttime=`date +"%Y-%m-%d %H:%M:%S"` +total_startvalue=`date -d "$total_starttime" +%s` + +array=("multi_standby_single_archive") +for element in ${array[@]} +do + mkdir -vp ./results/$element +done + +#init and start the database +printf "init and start the database\n" +sh deploy_multi_single_archive.sh > ./results/deploy_standby_multi_single_archive.log 2>&1 + +for((i=1;i<=$loop_num;i++)) +do + printf "run the ha_schedule %d time\n" $i + printf "%-50s%-10s%-10s\n" "testcase" "result" "time(s)" + for line in `cat ha_schedule_multi_single_archive$2 | grep -v ^#` + do + printf "%-50s" $line + starttime=`date +"%Y-%m-%d %H:%M:%S"` + sh ./testcase/$line.sh > ./results/$line.log 2>&1 + count=`expr $count + 1` + endtime=`date +"%Y-%m-%d %H:%M:%S"` + starttime1=`date -d "$starttime" +%s` + endtime1=`date -d "$endtime" +%s` + interval=`expr $endtime1 - $starttime1` + if [ $( grep "$failed_keyword" ./results/$line.log | grep -v "the database system is shutting down" | wc -l ) -eq 0 ]; then + printf "%-10s%-10s\n" ".... ok" $interval + else + printf "%-10s%-10s\n" ".... FAILED" $interval + cp ./results/$line.log regression.diffs.hacheck + exit 1 + fi + done +done + +#stop the database +printf "stop the database\n" +python $scripts_dir/pgxc_multi_single.py -o > ./results/stop_database_multi_single_archive.log 2>&1 + +total_endtime=`date +"%Y-%m-%d %H:%M:%S"` +total_endvalue=`date -d "$total_endtime" +%s` +printf "all %d tests passed.\n" $count +printf "total time: %ss\n" $(($total_endvalue - $total_startvalue)) diff --git a/src/test/ha/testcase/multi_standby_single_archive/archive_slot.sh b/src/test/ha/testcase/multi_standby_single_archive/archive_slot.sh new file mode 100644 index 000000000..bac3c3a14 --- /dev/null +++ b/src/test/ha/testcase/multi_standby_single_archive/archive_slot.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# keep wal segment files when standby is offline using replication slot on primary + +source ./standby_env.sh + +function test_1() +{ +check_instance_multi_standby + +#create table slot1 +gsql -d $db -p $dn1_primary_port -c "DROP TABLE if exists mpp_slot1; CREATE TABLE mpp_slot1(id INT,name VARCHAR(15) NOT NULL);" + + +#set archive destination +archive_destination=$data_dir/archive_nas/dn1 +#create archive slot +gsql -d $db -p $dn1_primary_port -c "set enable_slot_log = on; select * from pg_create_physical_replication_slot_extern('archive', false, 'NAS;$archive_destination;0;1');" + +#check archive slot created +if [ $(gsql -d $db -p $dn1_primary_port -c "select * from pg_get_replication_slots();" | grep "archive" |wc -l) -eq 1 ]; then + echo "success create archive slot" +else + echo "$failed_keyword on create archive slot" + exit 1 +fi + +#produce enough wal segment files +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 VALUES(311, 'sp');" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" +gsql -d $db -p $dn1_primary_port -c "INSERT INTO mpp_slot1 SELECT * FROM mpp_slot1;" + +#wait archive +sleep 30 + +#check archive xlog +if [ $(ls $archive_destination/pg_xlog |wc -l) -gt 0 ]; then + echo "success archive xlog" +else + echo "$failed_keyword: failed archive xlog" + exit 1 +fi + +#check global barrier +gsql -d $db -p $dn1_primary_port -c "select * from gs_get_global_barriers_status();" + +#clean archive xlog +current_lsn=`gsql -d postgres -p 28891 -c "select * from pg_get_flush_lsn()" -t -A -X` +echo "current lsn is $local_lsn" +gsql -d $db -p $dn1_primary_port -c "select * from gs_set_obs_delete_location_with_slotname('$current_lsn', 'archive');" + + +sleep 5 +#drop archive slot +gsql -d $db -p $dn1_primary_port -c "set enable_slot_log = on; select * from pg_drop_replication_slot('archive');" + +#check archive slot created +if [ $(gsql -d $db -p $dn1_primary_port -c "select * from pg_get_replication_slots();" | grep "archive" |wc -l) -eq 0 ]; then + echo "success drop archive slot" +else + echo "$failed_keyword on drop archive slot" + exit 1 +fi + + +if [ $? -eq 0 ]; then + echo "all of success" +else + echo "$failed_keyword: archive slot failed." + exit 1 +fi +} + +function tear_down() +{ +sleep 1 +gsql -d $db -p $dn1_primary_port -c "DROP TABLE if exists mpp_slot1;" +rm -rf $data_dir/archive_nas +} + +test_1 +tear_down