!1606 回合relmap文件支持4K代码

Merge pull request !1606 from pengjiong/relmap4k
This commit is contained in:
opengauss-bot 2022-03-21 01:14:31 +00:00 committed by Gitee
commit c81a284cad
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
3 changed files with 256 additions and 37 deletions

View File

@ -775,11 +775,6 @@ void ExceptionalCondition(const char* conditionName, const char* errorType, cons
abort();
}
// see to relmappper.cpp
//
#define MAX_MAPPINGS 62 /* 62 * 8 + 16 = 512 */
#define RELMAPPER_FILENAME "pg_filenode.map"
#define RELMAPPER_FILEMAGIC 0x592717 /* version ID value */
#define RELCACHE_INIT_FILEMAGIC 0x573266 /* version ID value */
typedef struct PgClass_table {
@ -3179,6 +3174,35 @@ read_failed:
return result;
}
static int ReadOldVersionRelmapFile(char *buffer, FILE *fd)
{
errno_t rc;
char oldMapCache[RELMAP_SIZE_OLD];
int readByte = fread(oldMapCache, 1, RELMAP_SIZE_OLD, fd);
rc = memcpy_s(buffer, sizeof(RelMapFile), oldMapCache, MAPPING_LEN_OLDMAP_HEAD);
securec_check(rc, "\0", "\0");
rc = memcpy_s(
buffer + offsetof(RelMapFile, crc), MAPPING_LEN_TAIL, oldMapCache + MAPPING_LEN_OLDMAP_HEAD, MAPPING_LEN_TAIL);
securec_check(rc, "\0", "\0");
return readByte;
}
static void DoRelMapCrcCheck(const RelMapFile* mapfile, int32 magicNum)
{
pg_crc32 crc;
/* verify the CRC */
INIT_CRC32(crc);
if (IS_NEW_RELMAP(magicNum)) {
COMP_CRC32(crc, (char*)mapfile, offsetof(RelMapFile, crc));
} else {
COMP_CRC32(crc, (char*)mapfile, MAPPING_LEN_OLDMAP_HEAD);
}
FIN_CRC32(crc);
if (!EQ_CRC32(crc, (pg_crc32)(mapfile->crc))) {
fprintf(stdout, "WARNING: relmap crc check failed!");
}
}
static int parse_filenodemap_file(char* filename)
{
char buffer[sizeof(RelMapFile)];
@ -3189,6 +3213,9 @@ static int parse_filenodemap_file(char* filename)
char* pg_class_map[MAX_PG_CLASS_ID];
char* pg_class = NULL;
int i = 0;
int32 magicNum;
int32 relmapSize;
int32 relmapMaxMappings;
for (i = 0; i < MAX_PG_CLASS_ID; i++) {
pg_class_map[i] = NULL;
@ -3203,17 +3230,34 @@ static int parse_filenodemap_file(char* filename)
fseek(fd, 0, SEEK_END);
size = ftell(fd);
rewind(fd);
result = fread(buffer, 1, sizeof(RelMapFile), fd);
if (sizeof(RelMapFile) != result) {
result = fread(&magicNum, 1, sizeof(int32), fd);
rewind(fd);
if (result != sizeof(int32)) {
fprintf(stderr, "Reading magic error");
return false;
}
if (IS_NEW_RELMAP(magicNum)) {
result = fread(buffer, 1, sizeof(RelMapFile), fd);
relmapSize = RELMAP_SIZE_NEW;
relmapMaxMappings = MAX_MAPPINGS_4K;
} else {
result = ReadOldVersionRelmapFile(buffer, fd);
relmapSize = RELMAP_SIZE_OLD;
relmapMaxMappings = MAX_MAPPINGS;
}
if ((size_t)relmapSize != result) {
fprintf(stderr, "Reading error");
return false;
}
mapfile = (RelMapFile*)buffer;
DoRelMapCrcCheck(mapfile, magicNum);
fprintf(stdout, "Magic number: 0x%x\n", mapfile->magic);
fprintf(stdout, "Number of mappings: %u\n", mapfile->num_mappings);
fprintf(stdout, "Mapping pairs:\n");
for (i = 0; i < MAX_MAPPINGS; i++) {
for (i = 0; i < relmapMaxMappings; i++) {
if (0 == mapfile->mappings[i].mapoid && 0 == mapfile->mappings[i].mapfilenode)
break;

View File

@ -64,6 +64,14 @@ static void recover_relmap_file(bool shared, bool backupfile);
static void write_relmap_file(bool shared, RelMapFile* newmap, bool write_wal, bool send_sinval, bool preserve_files,
Oid dbid, Oid tsid, const char* dbpath);
static void perform_relmap_update(bool shared, const RelMapFile* updates);
static int WriteOldVersionRelmap(RelMapFile* map, int fd);
static int ReadOldVersionRelmap(RelMapFile* map, int fd);
static void InitRelmapVerTag(RelMapVerTag* mapTag, bool isNewMap);
static bool IsNewVersionMap(int fd, char* fileName);
static void RegistRelMapWal(RelMapFile* map);
static pg_crc32 RelmapCrcComp(RelMapFile* map);
static int32 ReadRelMapFile(RelMapFile* map, int fd, bool isNewMap);
static int32 WriteRelMapFile(RelMapFile* map, int fd);
/*
* RelationMapOidToFilenode
@ -226,6 +234,8 @@ void RelationMapUpdateMap(Oid relationId, Oid fileNode, bool shared, bool immedi
static void apply_map_update(RelMapFile* map, Oid relationId, Oid fileNode, bool add_okay)
{
int32 i;
RelMapVerTag mapTag;
InitRelmapVerTag(&mapTag, IS_MAGIC_EXIST(map->magic) ? IS_NEW_RELMAP(map->magic) : true);
/* Replace any existing mapping */
for (i = 0; i < map->num_mappings; i++) {
@ -241,8 +251,15 @@ static void apply_map_update(RelMapFile* map, Oid relationId, Oid fileNode, bool
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("attempt to apply a mapping to unmapped relation %u", relationId)));
}
if (map->num_mappings >= MAX_MAPPINGS) {
ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("ran out of space in relation map")));
if (map->num_mappings >= mapTag.relMaxMappings) {
// switch relmap to new version
if (map->magic == RELMAPPER_FILEMAGIC) {
map->magic = RELMAPPER_FILEMAGIC_4K;
ereport(LOG, (errmsg("Switch relmap to new version!")));
} else {
ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("ran out of space in relation map")));
}
}
map->mappings[map->num_mappings].mapoid = relationId;
map->mappings[map->num_mappings].mapfilenode = fileNode;
@ -302,13 +319,13 @@ void RelationMapRemoveMapping(Oid relationId)
void RelationMapInvalidate(bool shared)
{
if (shared) {
if (u_sess->relmap_cxt.shared_map->magic == RELMAPPER_FILEMAGIC) {
if (IS_MAGIC_EXIST(u_sess->relmap_cxt.shared_map->magic)) {
LWLockAcquire(RelationMappingLock, LW_SHARED);
load_relmap_file(true);
LWLockRelease(RelationMappingLock);
}
} else {
if (u_sess->relmap_cxt.local_map->magic == RELMAPPER_FILEMAGIC) {
if (IS_MAGIC_EXIST(u_sess->relmap_cxt.local_map->magic)) {
LWLockAcquire(RelationMappingLock, LW_SHARED);
load_relmap_file(false);
LWLockRelease(RelationMappingLock);
@ -326,10 +343,10 @@ void RelationMapInvalidate(bool shared)
void RelationMapInvalidateAll(void)
{
LWLockAcquire(RelationMappingLock, LW_SHARED);
if (u_sess->relmap_cxt.shared_map->magic == RELMAPPER_FILEMAGIC) {
if (IS_MAGIC_EXIST(u_sess->relmap_cxt.shared_map->magic)) {
load_relmap_file(true);
}
if (u_sess->relmap_cxt.local_map->magic == RELMAPPER_FILEMAGIC) {
if (IS_MAGIC_EXIST(u_sess->relmap_cxt.local_map->magic)) {
load_relmap_file(false);
}
LWLockRelease(RelationMappingLock);
@ -547,6 +564,8 @@ static void load_relmap_file(bool shared)
bool retry = false;
struct stat stat_buf;
errno_t rc;
RelMapVerTag mapTag;
bool isNewMap;
if (shared) {
rc = snprintf_s(
@ -606,7 +625,11 @@ loop:
* look, the sinval signaling mechanism will make us re-read it before we
* are able to access any relation that's affected by the change.
*/
if (read(fd, map, sizeof(RelMapFile)) != sizeof(RelMapFile)) {
isNewMap = IsNewVersionMap(fd, file_name);
(void)lseek(fd, 0, SEEK_SET);
InitRelmapVerTag(&mapTag, isNewMap);
int readBytes = ReadRelMapFile(map, fd, isNewMap);
if (readBytes != mapTag.relSize) {
close(fd);
ereport(
FATAL, (errcode_for_file_access(), errmsg("could not read relation mapping file \"%s\": %m", file_name)));
@ -615,10 +638,7 @@ loop:
ereport(PANIC, (errcode_for_file_access(), errmsg("could not close control file: %m")));
}
/* verify the CRC */
INIT_CRC32(crc);
COMP_CRC32(crc, (char*)map, offsetof(RelMapFile, crc));
FIN_CRC32(crc);
crc = RelmapCrcComp(map);
if (!EQ_CRC32(crc, (pg_crc32)(map->crc))) {
if (retry == false) {
ereport(WARNING,
@ -638,7 +658,7 @@ loop:
}
// check for correct magic number, etc
if (map->magic != RELMAPPER_FILEMAGIC || map->num_mappings < 0 || map->num_mappings > MAX_MAPPINGS) {
if (map->magic != mapTag.relMagic || map->num_mappings < 0 || map->num_mappings > mapTag.relMaxMappings) {
ereport(FATAL, (errmsg("relation mapping file \"%s\" contains invalid data", file_name)));
}
if (retry == true) {
@ -677,17 +697,21 @@ static void write_relmap_file(bool shared, RelMapFile* newmap, bool write_wal, b
char map_file_name[MAXPGPATH];
errno_t rc = 0;
char* fname[2];
RelMapVerTag mapTag;
if (!IS_MAGIC_EXIST(newmap->magic)) {
newmap->magic = RELMAPPER_FILEMAGIC_4K;
}
InitRelmapVerTag(&mapTag, IS_NEW_RELMAP(newmap->magic));
/*
* Fill in the overhead fields and update CRC.
*/
newmap->magic = RELMAPPER_FILEMAGIC;
if (newmap->num_mappings < 0 || newmap->num_mappings > MAX_MAPPINGS) {
if (newmap->num_mappings < 0 || newmap->num_mappings > mapTag.relMaxMappings) {
ereport(
ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("attempt to write bogus relation mapping")));
}
INIT_CRC32(newmap->crc);
COMP_CRC32(newmap->crc, (char*)newmap, offsetof(RelMapFile, crc));
FIN_CRC32(newmap->crc);
newmap->crc = RelmapCrcComp(newmap);
/*
* Open the target file. We prefer to do this before entering the
@ -727,12 +751,14 @@ static void write_relmap_file(bool shared, RelMapFile* newmap, bool write_wal, b
xlrec.dbid = dbid;
xlrec.tsid = tsid;
xlrec.nbytes = sizeof(RelMapFile);
if (IS_NEW_RELMAP(newmap->magic)) {
xlrec.nbytes = sizeof(RelMapFile);
} else {
xlrec.nbytes = RELMAP_SIZE_OLD;
}
XLogBeginInsert();
XLogRegisterData((char*)(&xlrec), MinSizeOfRelmapUpdate);
XLogRegisterData((char*)newmap, sizeof(RelMapFile));
RegistRelMapWal(newmap);
lsn = XLogInsert(RM_RELMAP_ID, XLOG_RELMAP_UPDATE);
/* As always, WAL must hit the disk before the data update does */
@ -740,7 +766,8 @@ static void write_relmap_file(bool shared, RelMapFile* newmap, bool write_wal, b
}
errno = 0;
if (write(fd, newmap, sizeof(RelMapFile)) != sizeof(RelMapFile)) {
int writeBytes = WriteRelMapFile(newmap, fd);
if (writeBytes != mapTag.relSize) {
close(fd);
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0) {
@ -884,6 +911,7 @@ static void recover_relmap_file(bool shared, bool backupfile)
char* file_name = NULL;
int level;
errno_t rc;
int32 relmapSize;
if (backupfile) {
file_name = RELMAPPER_FILENAME_BAK;
@ -915,7 +943,15 @@ static void recover_relmap_file(bool shared, bool backupfile)
map_file_name)));
}
errno = 0;
if (write(fd, real_map, sizeof(RelMapFile)) != sizeof(RelMapFile)) {
int writeBytes = 0;
if (IS_NEW_RELMAP(real_map->magic)) {
relmapSize = RELMAP_SIZE_NEW;
writeBytes = write(fd, real_map, sizeof(RelMapFile));
} else {
relmapSize = RELMAP_SIZE_OLD;
writeBytes = WriteOldVersionRelmap(real_map, fd);
}
if (writeBytes != relmapSize) {
/* if write didn't set errno, assume problem is no disk space */
if (errno == 0) {
errno = ENOSPC;
@ -952,16 +988,28 @@ void relmap_redo(XLogReaderState* record)
RelMapFile new_map;
char* dbpath = NULL;
errno_t rc;
RelMapVerTag mapTag;
int32 magicNum;
if (xlrec->nbytes != sizeof(RelMapFile)) {
rc = memcpy_s(&magicNum, sizeof(int32), xlrec->data, sizeof(int32));
securec_check(rc, "\0", "\0");
InitRelmapVerTag(&mapTag, IS_NEW_RELMAP(magicNum));
if (xlrec->nbytes != mapTag.relSize) {
ereport(PANIC,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("relmap_redo: wrong size %d in relmap update record", xlrec->nbytes)));
}
rc = memcpy_s(&new_map, sizeof(new_map), xlrec->data, sizeof(new_map));
securec_check(rc, "\0", "\0");
if (IS_NEW_RELMAP(magicNum)) {
rc = memcpy_s(&new_map, sizeof(new_map), xlrec->data, sizeof(new_map));
securec_check(rc, "\0", "\0");
} else {
rc = memcpy_s(&new_map, sizeof(new_map), xlrec->data, MAPPING_LEN_OLDMAP_HEAD);
securec_check(rc, "\0", "\0");
rc = memcpy_s(&new_map.crc, MAPPING_LEN_TAIL, xlrec->data + MAPPING_LEN_OLDMAP_HEAD, MAPPING_LEN_TAIL);
securec_check(rc, "\0", "\0");
}
/* We need to construct the pathname for this database */
dbpath = GetDatabasePath(xlrec->dbid, xlrec->tsid);
@ -983,3 +1031,114 @@ void relmap_redo(XLogReaderState* record)
ereport(PANIC, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("relmap_redo: unknown op code %u", info)));
}
}
static int WriteOldVersionRelmap(RelMapFile* map, int fd)
{
errno_t rc;
char* mapCache = (char*)palloc0(RELMAP_SIZE_OLD);
rc = memcpy_s(mapCache, RELMAP_SIZE_OLD, map, MAPPING_LEN_OLDMAP_HEAD);
securec_check(rc, "\0", "\0");
rc = memcpy_s(
mapCache + MAPPING_LEN_OLDMAP_HEAD, RELMAP_SIZE_OLD - MAPPING_LEN_OLDMAP_HEAD, &(map->crc), MAPPING_LEN_TAIL);
securec_check(rc, "\0", "\0");
int writeBytes = write(fd, mapCache, RELMAP_SIZE_OLD);
pfree_ext(mapCache);
return writeBytes;
}
static int ReadOldVersionRelmap(RelMapFile* map, int fd)
{
errno_t rc;
char* mapCache = (char*)palloc0(RELMAP_SIZE_OLD);
int readByte = read(fd, mapCache, RELMAP_SIZE_OLD);
rc = memcpy_s(map, sizeof(RelMapFile), mapCache, MAPPING_LEN_OLDMAP_HEAD);
securec_check(rc, "\0", "\0");
rc = memcpy_s(&(map->crc), MAPPING_LEN_TAIL, mapCache + MAPPING_LEN_OLDMAP_HEAD, MAPPING_LEN_TAIL);
securec_check(rc, "\0", "\0");
pfree_ext(mapCache);
return readByte;
}
static void InitRelmapVerTag(RelMapVerTag* mapTag, bool isNewMap)
{
if (isNewMap) {
mapTag->relMagic = RELMAPPER_FILEMAGIC_4K;
mapTag->relMaxMappings = MAX_MAPPINGS_4K;
mapTag->relSize = RELMAP_SIZE_NEW;
} else {
mapTag->relMagic = RELMAPPER_FILEMAGIC;
mapTag->relMaxMappings = MAX_MAPPINGS;
mapTag->relSize = RELMAP_SIZE_OLD;
}
}
static bool IsNewVersionMap(int fd, char* fileName)
{
int32 magicNum;
bool ret = false;
int readByte = read(fd, &magicNum, sizeof(int32));
if (readByte != sizeof(int32)) {
close(fd);
ereport(
FATAL, (errcode_for_file_access(), errmsg("could not read relation mapping file \"%s\": %m", fileName)));
}
switch (magicNum) {
case RELMAPPER_FILEMAGIC:
ret = false;
break;
case RELMAPPER_FILEMAGIC_4K:
ret = true;
break;
default:
close(fd);
ereport(FATAL,
(errmsg("Illegal magic number %d for relation mapping file \"%s\": %m", magicNum, fileName)));
}
return ret;
}
static void RegistRelMapWal(RelMapFile* map)
{
if (IS_NEW_RELMAP(map->magic)) {
XLogRegisterData((char*)map, sizeof(RelMapFile));
} else {
XLogRegisterData((char*)map, MAPPING_LEN_OLDMAP_HEAD);
XLogRegisterData((char*)&map->crc, MAPPING_LEN_TAIL);
}
}
static pg_crc32 RelmapCrcComp(RelMapFile* map)
{
pg_crc32 crc;
INIT_CRC32(crc);
if (IS_NEW_RELMAP(map->magic)) {
COMP_CRC32(crc, (char*)map, offsetof(RelMapFile, crc));
} else {
COMP_CRC32(crc, (char*)map, MAPPING_LEN_OLDMAP_HEAD);
}
FIN_CRC32(crc);
return crc;
}
static int32 ReadRelMapFile(RelMapFile* map, int fd, bool isNewMap)
{
int32 readBytes = 0;
if (isNewMap) {
readBytes = read(fd, map, sizeof(RelMapFile));
} else {
readBytes = ReadOldVersionRelmap(map, fd);
}
return readBytes;
}
static int32 WriteRelMapFile(RelMapFile* map, int fd)
{
int32 writeBytes = 0;
if (IS_NEW_RELMAP(map->magic)) {
writeBytes = write(fd, map, sizeof(RelMapFile));
} else {
writeBytes = WriteOldVersionRelmap(map, fd);
}
return writeBytes;
}

View File

@ -34,8 +34,19 @@
#define RELMAPPER_FILENAME_BAK "pg_filenode.map.backup"
#define RELMAPPER_FILEMAGIC 0x592717 /* version ID value */
#define RELMAPPER_FILEMAGIC_4K 0x592718 /* version ID value */
#define MAX_MAPPINGS 62 /* 62 * 8 + 16 = 512 */
#define IS_NEW_RELMAP(magic) (magic == RELMAPPER_FILEMAGIC_4K)
#define IS_MAGIC_EXIST(magic) (magic == RELMAPPER_FILEMAGIC_4K || magic == RELMAPPER_FILEMAGIC)
#define MAX_MAPPINGS 62 /* old relmap 62 * 8 + 16 = 512 */
#define MAX_MAPPINGS_4K 510 /* new relmap 510 * 8 + 16 = 4096 */
#define MAPPING_LEN_OLDMAP_HEAD (offsetof(RelMapFile, mappings) + MAX_MAPPINGS * sizeof(RelMapping))
#define MAPPING_LEN_TAIL (8)
#define RELMAP_SIZE_OLD (MAPPING_LEN_OLDMAP_HEAD + MAPPING_LEN_TAIL)
#define RELMAP_SIZE_NEW (sizeof(RelMapFile))
typedef struct RelMapping {
Oid mapoid; /* OID of a catalog */
@ -45,11 +56,16 @@ typedef struct RelMapping {
typedef struct RelMapFile {
int32 magic; /* always RELMAPPER_FILEMAGIC */
int32 num_mappings; /* number of valid RelMapping entries */
RelMapping mappings[MAX_MAPPINGS];
RelMapping mappings[MAX_MAPPINGS_4K]; /* old relmap only use MAX_MAPPINGS */
int32 crc; /* CRC of all above */
int32 pad; /* to make the struct size be 512 exactly */
} RelMapFile;
typedef struct RelMapVerTag {
int32 relMagic;
int32 relMaxMappings;
int32 relSize;
} RelMapVerTag;
/* ----------------
* relmap-related XLOG entries
* ----------------