From 969ecd45f5a1ecbde2ef2c2f94d13591a5d37932 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 17:19:10 +0800 Subject: [PATCH 01/31] 333 --- contrib/pagehack/pagehack.cpp | 4 +- .../process/threadpool/knl_thread.cpp | 1 + .../storage/access/redo/redo_segpage.cpp | 4 +- .../storage/access/redo/redo_xlogutils.cpp | 53 +- .../redo/standby_read/base_page_proc.cpp | 15 +- .../redo/standby_read/block_info_proc.cpp | 23 +- .../redo/standby_read/lsn_info_proc.cpp | 173 ++-- .../standby_read/standby_read_interface.cpp | 155 +++- .../redo/standby_read/standby_read_proc.cpp | 853 +++++++++--------- .../access/transam/extreme_rto/dispatcher.cpp | 2 + .../transam/extreme_rto/exrto_recycle.cpp | 19 +- .../access/transam/extreme_rto/page_redo.cpp | 19 +- .../storage/access/transam/xact.cpp | 6 +- .../storage/access/transam/xlogutils.cpp | 6 + src/gausskernel/storage/buffer/bufmgr.cpp | 18 +- .../storage/smgr/segment/segbuffer.cpp | 35 +- .../storage/smgr/segment/segxlog.cpp | 8 +- src/gausskernel/storage/smgr/segstore.cpp | 1 + src/gausskernel/storage/smgr/smgr.cpp | 6 +- .../storage/smgr/storage_exrto_file.cpp | 10 +- src/include/access/extreme_rto/dispatcher.h | 1 + src/include/access/extreme_rto/page_redo.h | 2 + .../standby_read/block_info_meta.h | 9 +- .../extreme_rto/standby_read/lsn_info_meta.h | 5 +- .../standby_read/standby_read_base.h | 2 + src/include/access/xlogproc.h | 2 +- src/include/knl/knl_thread.h | 1 + src/include/postgres_ext.h | 12 + src/include/storage/buf/bufmgr.h | 5 +- src/include/storage/smgr/segment.h | 4 +- src/include/storage/smgr/smgr.h | 20 +- 31 files changed, 900 insertions(+), 574 deletions(-) diff --git a/contrib/pagehack/pagehack.cpp b/contrib/pagehack/pagehack.cpp index 294d4b79bd..82bef24bef 100644 --- a/contrib/pagehack/pagehack.cpp +++ b/contrib/pagehack/pagehack.cpp @@ -5058,8 +5058,8 @@ static void parse_lsn_info_head(LsnInfoPageHeader *header) PageXLogRecPtr lsn = header->lsn; fprintf(stdout, "%slsn: xlogid %u, xrecoff %u, lsn %lu\n", indents[indentLevel], lsn.xlogid, lsn.xrecoff, ((uint64)lsn.xlogid << WAL_ID_OFFSET) | lsn.xrecoff); - fprintf(stdout, "%schecksum: %u, flags: %u, version: %u", - indents[indentLevel], header->checksum, header->flags, header->version); + fprintf(stdout, "%schecksum: %u, flags: %u, version: %u, next_lsn_info_page: %lu", + indents[indentLevel], header->checksum, header->flags, header->version, header->next_lsn_info_page); fprintf(stdout, "%sbase page map: ", indents[indentLevel]); for (uint32 loop = 0; loop < BASE_PAGE_MAP_SIZE; loop++) { parse_map_position(header->base_page_map[loop]); diff --git a/src/gausskernel/process/threadpool/knl_thread.cpp b/src/gausskernel/process/threadpool/knl_thread.cpp index 1c5c27551b..54358dbfef 100755 --- a/src/gausskernel/process/threadpool/knl_thread.cpp +++ b/src/gausskernel/process/threadpool/knl_thread.cpp @@ -517,6 +517,7 @@ static void knl_t_xlog_init(knl_t_xlog_context* xlog_cxt) xlog_cxt->xlog_atomic_op = NULL; xlog_cxt->currentRetryTimes = 0; xlog_cxt->uwalInfoHis = NIL; + xlog_cxt->last_forwarder_lsn = 0; } static void knl_t_index_init(knl_t_index_context* index_cxt) diff --git a/src/gausskernel/storage/access/redo/redo_segpage.cpp b/src/gausskernel/storage/access/redo/redo_segpage.cpp index 2add2872b5..8d46168a71 100644 --- a/src/gausskernel/storage/access/redo/redo_segpage.cpp +++ b/src/gausskernel/storage/access/redo/redo_segpage.cpp @@ -450,8 +450,8 @@ void SegPageRedoNewPage(XLogBlockHead *blockhead, XLogBlockSegNewPage *newPageIn { Assert(newPageInfo->dataLen != 0); BufferTag *tag = (BufferTag *)newPageInfo->mainData; - - seg_redo_new_page_copy_and_flush(tag, newPageInfo->mainData + sizeof(BufferTag), blockhead->end_ptr); + char page[BLCKSZ] __attribute__((__aligned__(ALIGNOF_BUFFER))) = {0}; + seg_redo_new_page_copy_and_flush(tag, page, newPageInfo->mainData + sizeof(BufferTag), blockhead->end_ptr); } void MarkSegPageRedoChildPageDirty(RedoBufferInfo *bufferinfo) diff --git a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp index 2e5567acca..43d43ebc83 100644 --- a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp +++ b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp @@ -2034,6 +2034,47 @@ void init_redo_buffer_info(RedoBufferInfo *rb_info, const BufferTag &buf_tag, Bu rb_info->dirtyflag = false; /* initial value, actually, dirtyflag is useless in extreme RTO read */ } +bool redo_target_state(XLogRecParseState *state, RedoBufferInfo *buf_info) +{ + if (!find_target_state(state, buf_info.blockinfo)) { + return false; + } + + buf_info.lsn = state->blockparse.blockhead.end_ptr; + buf_info.blockinfo.pblk = state->blockparse.blockhead.pblk; + wal_block_redo_for_extreme_rto_read(state, &buf_info); + return true; +} + +bool redo_target_seg_state(XLogRecParseState *state, RedoBufferInfo *buf_info) +{ + uint8 xl_info = XLogBlockHeadGetInfo(&state->blockparse.blockhead) & ~XLR_INFO_MASK; + bool redo_done = false; + switch (xl_info) { + case XLOG_SEG_ATOMIC_OPERATION: + case XLOG_SEG_SEGMENT_EXTEND: + case XLOG_SEG_INIT_MAPPAGE: + case XLOG_SEG_INIT_INVRSPTR_PAGE: + case XLOG_SEG_ADD_NEW_GROUP: + case XLOG_SEG_TRUNCATE: + case XLOG_SEG_SPACE_SHRINK: + case XLOG_SEG_NEW_PAGE: + XLogRecParseState* child_state_list = (XLogRecParseState*)state->blockparse.extra_rec.blocksegfullsyncrec.childState; + XLogRecParseState* next_state = child_state_list; + while (next_state != NULL) { + if (redo_target_state(next_state, buf_info)) { + redo_done = true; + } + next_state = (XLogRecParseState*)next_state->nextrecord; + } + XLogBlockParseStateRelease(state); + break; + default: + return redo_target_state(state, buf_info); + } + return redo_done; +} + void redo_target_page(const BufferTag &buf_tag, StandbyReadLsnInfoArray *lsn_info, Buffer base_page_buf) { char *error_msg = NULL; @@ -2071,13 +2112,17 @@ void redo_target_page(const BufferTag &buf_tag, StandbyReadLsnInfoArray *lsn_inf errdetail("Failed while wal parse to block."))); } XLogRecParseState *state_iter = state; - while (state_iter != NULL) { - if (find_target_state(state_iter, buf_info.blockinfo)) { - break; + bool redo_done = false; + while(!redo_done && state_iter != NULL) { + if ((XLogBlockHeadGetRmid(&state_iter->blockparse.blockhead) == RM_SEGPAGE_ID)) { + redo_done = redo_target_seg_state(state_iter, buf_info); + } else { + redo_done = redo_target_state(state_iter, buf_info); } state_iter = (XLogRecParseState *)(state_iter->nextrecord); } - if (state_iter == NULL) { + + if (!redo_done) { ereport(ERROR, (errmsg("redo_target_page: internal error, xlog in lsn %X/%X doesn't contain target block.", (uint32)(lsn_info->lsn_array[i] >> LSN_MOVE32), (uint32)(lsn_info->lsn_array[i])))); } diff --git a/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp index b7f6ee2895..87b98ab4c6 100644 --- a/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp @@ -59,11 +59,9 @@ Buffer buffer_read_base_page(uint32 batch_id, uint32 redo_id, BasePagePosition p return buffer; } -void generate_base_page(StandbyReadMetaInfo* meta_info, const Page src_page) +void generate_base_page(StandbyReadMetaInfo* meta_info, const Page src_page, BasePagePosition base_page_pos) { - BasePagePosition position = meta_info->base_page_next_position; - - Buffer dest_buf = buffer_read_base_page(meta_info->batch_id, meta_info->redo_id, position, RBM_ZERO_AND_LOCK); + Buffer dest_buf = buffer_read_base_page(meta_info->batch_id, meta_info->redo_id, base_page_pos, RBM_ZERO_AND_LOCK); #ifdef ENABLE_UT Page dest_page = get_page_from_buffer(dest_buf); @@ -74,8 +72,6 @@ void generate_base_page(StandbyReadMetaInfo* meta_info, const Page src_page) securec_check(rc, "\0", "\0"); MarkBufferDirty(dest_buf); UnlockReleaseBuffer(dest_buf); - - meta_info->base_page_next_position += BLCKSZ; } void read_base_page(const BufferTag& buf_tag, BasePagePosition position, BufferDesc* dest_buf_desc) @@ -111,12 +107,5 @@ void recycle_base_page_file(uint32 batch_id, uint32 redo_id, BasePagePosition re smgrdounlink(smgr, true, (BlockNumber)(recycle_pos / BLCKSZ)); } -#ifdef ENABLE_UT -Page get_page_from_buffer(Buffer buf) -{ - return BufferGetPage(buf); -} -#endif - } // namespace extreme_rto_standby_read diff --git a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp index a364d964d0..2b10344972 100644 --- a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp @@ -26,6 +26,8 @@ #include "access/extreme_rto/standby_read/lsn_info_meta.h" #include "access/extreme_rto/standby_read/standby_read_base.h" #include "storage/smgr/relfilenode.h" +#include "access/extreme_rto/dispatcher.h" +#include "access/extreme_rto/page_redo.h" namespace extreme_rto_standby_read { @@ -61,8 +63,13 @@ inline uint32 block_info_meta_page_offset(BlockNumber block_num) BlockMetaInfo* get_block_meta_info_by_relfilenode( const BufferTag& buf_tag, BufferAccessStrategy strategy, ReadBufferMode mode, Buffer* buffer, bool need_share_lock) { + Assert(IsSegmentFileNode(buf_tag.rnode) || IsSegmentPhysicalRelNode(buf_tag.rnode)); RelFileNode standby_read_rnode = buf_tag.rnode; - standby_read_rnode.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; + if (IsSegmentFileNode(standby_read_rnode)) { + standby_read_rnode.bucketNode -= EXRTO_STANDBY_READ_BUCKET_OFFSET; + } else { + standby_read_rnode.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; + } SMgrRelation smgr = smgropen(standby_read_rnode, InvalidBackendId); bool hit = false; @@ -218,9 +225,9 @@ void insert_lsn_to_block_info_for_opt( UnlockReleaseBuffer(block_info_buf); } -StandbyReadRecyleState recyle_block_info(const BufferTag &buf_tag, LsnInfoPosition base_page_info_pos, - XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, - XLogRecPtr *block_info_max_lsn) +StandbyReadRecyleState recycle_block_info(const StandbyReadMetaInfo &meta_info, const BufferTag &buf_tag, + LsnInfoPosition base_page_info_pos, XLogRecPtr next_base_page_lsn, + XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn) { Buffer buffer = InvalidBuffer; BlockMetaInfo* block_meta_info = get_block_meta_info_by_relfilenode(buf_tag, NULL, RBM_NORMAL, &buffer); @@ -246,7 +253,7 @@ StandbyReadRecyleState recyle_block_info(const BufferTag &buf_tag, LsnInfoPositi } else if (XLogRecPtrIsValid(next_base_page_lsn)) { LsnInfoPosition min_page_info_pos = LSN_INFO_LIST_HEAD; XLogRecPtr min_lsn = InvalidXLogRecPtr; - recycle_one_lsn_info_list(buf_tag, base_page_info_pos, recyle_lsn, &min_page_info_pos, &min_lsn); + recycle_one_lsn_info_list(meta_info, base_page_info_pos, recyle_lsn, &min_page_info_pos, &min_lsn); Assert(INFO_POSITION_IS_VALID(min_page_info_pos)); if (block_meta_info->base_page_info_list.next != min_page_info_pos) { @@ -272,8 +279,8 @@ static void reset_tmp_lsn_info_array(StandbyReadLsnInfoArray* lsn_info) } } -bool get_page_lsn_info(const BufferTag& buf_tag, BufferAccessStrategy strategy, XLogRecPtr read_lsn, - StandbyReadLsnInfoArray* lsn_info) +bool get_page_lsn_info(const BufferTag& old_buf_tag, const BufferTag& buf_tag, BufferAccessStrategy strategy, + XLogRecPtr read_lsn, StandbyReadLsnInfoArray* lsn_info) { Buffer buf; BlockMetaInfo* block_meta_info = get_block_meta_info_by_relfilenode(buf_tag, strategy, RBM_NORMAL, &buf, true); @@ -299,7 +306,7 @@ bool get_page_lsn_info(const BufferTag& buf_tag, BufferAccessStrategy strategy, block_meta_info->max_lsn, read_lsn)))); } reset_tmp_lsn_info_array(lsn_info); - get_lsn_info_for_read(buf_tag, block_meta_info->base_page_info_list.prev, lsn_info, read_lsn); + get_lsn_info_for_read(old_buf_tag, block_meta_info->base_page_info_list.prev, lsn_info, read_lsn); UnlockReleaseBuffer(buf); return true; diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp index 1eaf0d51e1..6872cb8c9e 100644 --- a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp @@ -54,6 +54,7 @@ void lsn_info_init(LsnInfo lsn_info) lsn_info->type = LSN_INFO_TYPE_LSNS; lsn_info_list_init(&lsn_info->lsn_list); } + void base_page_info_init(BasePageInfo base_page_info) { errno_t ret = memset_s(base_page_info, BASE_PAGE_INFO_NODE_SIZE, 0, BASE_PAGE_INFO_NODE_SIZE); @@ -65,11 +66,11 @@ void base_page_info_init(BasePageInfo base_page_info) lsn_info_list_init(&base_page_info->base_page_list); } -RelFileNode make_lsn_info_relfilenode(uint32 batch_id, uint32 worker_id, LsnInfoPosition position) +RelFileNode make_lsn_info_relfilenode(LsnInfoPosition position) { RelFileNode rnode = {0}; rnode.spcNode = EXRTO_LSN_INFO_SPACE_OID; - rnode.dbNode = (batch_id << LOW_WORKERID_BITS) | worker_id; + rnode.dbNode = 0; rnode.relNode = (uint32)((position / BLCKSZ) >> UINT64_HALF); rnode.bucketNode = InvalidBktId; rnode.opt = DefaultFileNodeOpt; @@ -85,7 +86,7 @@ Page get_lsn_info_page(uint32 batch_id, uint32 worker_id, LsnInfoPosition positi bool hit = false; Page page = NULL; - rnode = make_lsn_info_relfilenode(batch_id, worker_id, position); + rnode = make_lsn_info_relfilenode(position); block_num = (uint32)(position / BLCKSZ); /* high 32 bits are stored in the relNode. */ SMgrRelation smgr = smgropen(rnode, InvalidBackendId); @@ -117,21 +118,49 @@ Page get_lsn_info_page(uint32 batch_id, uint32 worker_id, LsnInfoPosition positi return page; } -LsnInfoPosition create_lsn_info_node(StandbyReadMetaInfo *meta_info, LsnInfoPosition old_tail_pos, - XLogRecPtr next_lsn, bool create_in_old_page, Page old_page) +LsnInfoPosition create_lsn_info_node( + StandbyReadMetaInfo *meta_info, LsnInfoPosition old_tail_pos, XLogRecPtr next_lsn, Page old_page) { Page page = NULL; LsnInfo lsn_info = NULL; uint32 batch_id = meta_info->batch_id; uint32 worker_id = meta_info->redo_id; - LsnInfoPosition insert_pos = meta_info->lsn_table_next_position; Buffer buffer = InvalidBuffer; - uint32 offset; - - offset = lsn_info_postion_to_offset(insert_pos); + SpinLockAcquire(&(meta_info->mutex)); + LsnInfoPosition insert_pos = meta_info->lsn_table_next_position; + bool create_in_old_page = (insert_pos / BLCKSZ) == (old_tail_pos / BLCKSZ); + + uint32 offset = lsn_info_postion_to_offset(insert_pos); if (offset == 0) { + Assert(insert_pos >= BLCKSZ); + LsnInfoPosition pipe_old_page_position = insert_pos - BLCKSZ; + insert_pos = pg_atomic_fetch_add_u64(&(extreme_rto::g_dispatcher->next_lsn_info_page), BLCKSZ); + LsnInfoPosition next_lsn_info_page = insert_pos; insert_pos += LSN_INFO_HEAD_SIZE; /* actual insert position */ - offset += LSN_INFO_HEAD_SIZE; + meta_info->lsn_table_next_position = insert_pos + LSN_INFO_NODE_SIZE; + SpinLockRelease(&(meta_info->mutex)); + + LsnInfoPageHeader *pipe_old_page_header; + Buffer pipe_old_buffer = InvalidBuffer; + bool is_old_page_held_by_me = old_tail_pos / BLCKSZ == pipe_old_page_position / BLCKSZ; + if (is_old_page_held_by_me) { + pipe_old_page_header = (LsnInfoPageHeader *)old_page; + } else { + Page pipe_old_page = get_lsn_info_page(batch_id, worker_id, pipe_old_page_position, RBM_ZERO_ON_ERROR, &pipe_old_buffer); + LockBuffer(pipe_old_buffer, BUFFER_LOCK_SHARE); + pipe_old_page_header = (LsnInfoPageHeader *)pipe_old_page; + } + Assert(is_lsn_info_page_valid(pipe_old_page_header)); + Assert(pipe_old_page_header->next_lsn_info_page == 0); + Assert(next_lsn_info_page != 0); + if (!is_old_page_held_by_me) { + MarkBufferDirty(pipe_old_buffer); + UnlockReleaseBuffer(pipe_old_buffer); + } + offset = LSN_INFO_HEAD_SIZE; + } else { + meta_info->lsn_table_next_position = insert_pos + LSN_INFO_NODE_SIZE; + SpinLockRelease(&(meta_info->mutex)); } Assert(offset % LSN_INFO_NODE_SIZE == 0); @@ -155,9 +184,7 @@ LsnInfoPosition create_lsn_info_node(StandbyReadMetaInfo *meta_info, LsnInfoPosi MarkBufferDirty(buffer); UnlockReleaseBuffer(buffer); } - /* update meta info */ - meta_info->lsn_table_next_position = insert_pos + LSN_INFO_NODE_SIZE; - + return insert_pos; } @@ -165,10 +192,7 @@ void insert_lsn_to_lsn_info(StandbyReadMetaInfo *meta_info, LsnInfoDoubleList *l { Page page = NULL; LsnInfo lsn_info = NULL; - uint32 batch_id = meta_info->batch_id; - uint32 worker_id = meta_info->redo_id; LsnInfoPosition tail_pos = lsn_head->prev; /* lsn info node tail */ - LsnInfoPosition insert_pos = meta_info->lsn_table_next_position; Buffer buffer = InvalidBuffer; uint32 offset; @@ -193,9 +217,7 @@ void insert_lsn_to_lsn_info(StandbyReadMetaInfo *meta_info, LsnInfoDoubleList *l /* * There is no free space in the old lsn info node, create a new one. */ - bool create_in_old_page = (insert_pos / BLCKSZ) == (tail_pos / BLCKSZ); - /* insert position maybe changed */ - insert_pos = create_lsn_info_node(meta_info, tail_pos, next_lsn, create_in_old_page, page); + LsnInfoPosition insert_pos = create_lsn_info_node(meta_info, tail_pos, next_lsn, page); /* modify lsn info list */ lsn_info->lsn_list.next = insert_pos; @@ -209,33 +231,59 @@ void insert_lsn_to_lsn_info(StandbyReadMetaInfo *meta_info, LsnInfoDoubleList *l LsnInfoPosition create_base_page_info_node(StandbyReadMetaInfo *meta_info, LsnInfoPosition old_lsn_tail_pos, LsnInfoPosition old_base_page_tail_pos, const BufferTag* buf_tag, - XLogRecPtr current_page_lsn, XLogRecPtr next_lsn) + XLogRecPtr current_page_lsn, XLogRecPtr next_lsn, BasePagePosition *base_page_pos) { + Assert(!IsSegmentFileNode(buf_tag->rnode) || IsSegmentPhysicalRelNode(buf_tag->rnode)); Page page = NULL; BasePageInfo base_page_info = NULL; uint32 batch_id = meta_info->batch_id; uint32 worker_id = meta_info->redo_id; - LsnInfoPosition insert_pos = meta_info->lsn_table_next_position; - BasePagePosition base_page_pos = meta_info->base_page_next_position; Buffer buffer = InvalidBuffer; - uint32 offset; - uint32 remain_size; + bool need_update_old_page_header = false; + + SpinLockAcquire(&(meta_info->mutex)); + LsnInfoPosition insert_pos = meta_info->lsn_table_next_position; + *base_page_pos = meta_info->base_page_next_position; + LsnInfoPosition next_insert_pos = LSN_INFO_LIST_HEAD; + LsnInfoPosition pipe_old_page_position = LSN_INFO_LIST_HEAD; + uint32 remain_size = BLCKSZ - insert_pos % BLCKSZ; /* * If there is not enough space in current page, we insert base page info node in next page. */ - remain_size = BLCKSZ - insert_pos % BLCKSZ; if (remain_size < BASE_PAGE_INFO_NODE_SIZE) { Assert(remain_size == LSN_INFO_NODE_SIZE); insert_pos += LSN_INFO_NODE_SIZE; /* switch to next page */ } - offset = lsn_info_postion_to_offset(insert_pos); - Assert(offset % LSN_INFO_NODE_SIZE == 0); + uint32 offset = lsn_info_postion_to_offset(insert_pos); if (offset == 0) { + if (meta_info->is_lsn_table_recycle_position_init) { + pipe_old_page_position = insert_pos - BLCKSZ; + need_update_old_page_header = true; + } + insert_pos = pg_atomic_fetch_add_u64(&(extreme_rto::g_dispatcher->next_lsn_info_page), BLCKSZ); + next_insert_pos = insert_pos; + if (!meta_info->is_lsn_table_recycle_position_init) { + meta_info->is_lsn_table_recycle_position_init = true; + meta_info->lsn_table_recycle_position = insert_pos; + } insert_pos += LSN_INFO_HEAD_SIZE; /* actual insert position */ offset += LSN_INFO_HEAD_SIZE; } + + if (need_update_old_page_header) { + Buffer pipe_old_buffer = InvalidBuffer; + Page pipe_old_page = get_lsn_info_page(batch_id, worker_id, pipe_old_page_position, RBM_ZERO_ON_ERROR, &pipe_old_buffer); + LockBuffer(pipe_old_buffer, BUFFER_LOCK_EXCLUSIVE); + Assert(pipe_old_page != NULL); + LsnInfoPageHeader *old_page_header = (LsnInfoPageHeader *)pipe_old_page; + Assert(is_lsn_info_page_valid(old_page_header)); + Assert(old_page_header->next_lsn_info_page == 0); + old_page_header->next_lsn_info_page = next_insert_pos; + MarkBufferDirty(pipe_old_buffer); + UnlockReleaseBuffer(pipe_old_buffer); + } page = get_lsn_info_page(batch_id, worker_id, insert_pos, RBM_ZERO_ON_ERROR, &buffer); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); @@ -252,31 +300,13 @@ LsnInfoPosition create_base_page_info_node(StandbyReadMetaInfo *meta_info, base_page_info->fork_num = buf_tag->forkNum; base_page_info->block_num = buf_tag->blockNum; base_page_info->next_base_page_lsn = InvalidXLogRecPtr; - base_page_info->base_page_position = base_page_pos; + base_page_info->base_page_position = *base_page_pos; set_base_page_map_bit(page, offset); - ereport(DEBUG1, - (errmsg("create_base_page_info_node, block is %u/%u/%u %d %u, batch_id: %u, redo_worker_id: %u" - "page lsn %lu, next lsn %lu, base page pos %lu, insert pos %lu", - buf_tag->rnode.spcNode, - buf_tag->rnode.dbNode, - buf_tag->rnode.relNode, - buf_tag->forkNum, - buf_tag->blockNum, - batch_id, - worker_id, - current_page_lsn, - next_lsn, - base_page_pos, - insert_pos))); - standby_read_meta_page_set_lsn(page, next_lsn); MarkBufferDirty(buffer); UnlockReleaseBuffer(buffer); - /* update meta info */ - meta_info->lsn_table_next_position = insert_pos + BASE_PAGE_INFO_NODE_SIZE; - return insert_pos; } @@ -287,10 +317,11 @@ void insert_base_page_to_lsn_info(StandbyReadMetaInfo *meta_info, LsnInfoDoubleL LsnInfoPosition old_lsn_tail_pos = lsn_head->prev; LsnInfoPosition old_base_page_tail_pos = base_page_head->prev; LsnInfoPosition insert_pos; + BasePagePosition base_page_pos; /* possibly modified meta_info */ - insert_pos = create_base_page_info_node(meta_info, old_lsn_tail_pos, old_base_page_tail_pos, &buf_tag, - current_page_lsn, next_lsn); + insert_pos = create_base_page_info_node(meta_info, old_lsn_tail_pos, old_base_page_tail_pos, &buf_tag, + current_page_lsn, next_lsn, &base_page_pos); /* modify old tail information of lsn info node and base page info node */ if (old_lsn_tail_pos != LSN_INFO_LIST_HEAD) { @@ -312,7 +343,7 @@ void insert_base_page_to_lsn_info(StandbyReadMetaInfo *meta_info, LsnInfoDoubleL } /* generate base page */ - generate_base_page(meta_info, base_page); + generate_base_page(meta_info, base_page, base_page_pos); } void get_lsn_info_for_read(const BufferTag& buf_tag, LsnInfoPosition latest_lsn_base_page_pos, @@ -473,26 +504,20 @@ bool is_base_page_map_bit_set(Page page, uint32 which_bit) return (base_page_map[which_bytes] & (((uint8)1) << bit_offset)) != 0; } -void recycle_lsn_info_file(uint32 batch_id, uint32 redo_id, BasePagePosition recycle_pos) +void recycle_lsn_info_file(BasePagePosition recycle_pos) { - RelFileNode rnode = make_lsn_info_relfilenode(batch_id, redo_id, recycle_pos); + RelFileNode rnode = make_lsn_info_relfilenode(recycle_pos); SMgrRelation smgr = smgropen(rnode, InvalidBackendId); smgrdounlink(smgr, true, (BlockNumber)(recycle_pos / BLCKSZ)); } -void recycle_one_lsn_info_list(const BufferTag& buf_tag, LsnInfoPosition page_info_pos, +void recycle_one_lsn_info_list(const StandbyReadMetaInfo *meta_info, LsnInfoPosition page_info_pos, XLogRecPtr recycle_lsn, LsnInfoPosition *min_page_info_pos, XLogRecPtr *min_lsn) { - /* get batch id and page redo worker id */ - extreme_rto::RedoItemTag redo_item_tag; - INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); + uint32 batch_id = meta_info->batch_id; + uint32 worker_id = meta_info->redo_id; - const uint32 worker_num_per_mng = extreme_rto::get_page_redo_worker_num_per_manager(); - /* batch id and worker id start from 1 when reading a page */ - uint32 batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, (uint32)extreme_rto::get_batch_redo_num()) + 1; - uint32 worker_id = extreme_rto::GetWorkerId(&redo_item_tag, worker_num_per_mng) + 1; - while (INFO_POSITION_IS_VALID(page_info_pos)) { Buffer buffer = InvalidBuffer; Page page = get_lsn_info_page(batch_id, worker_id, page_info_pos, RBM_NORMAL, &buffer); @@ -590,7 +615,7 @@ inline void update_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info, XLogRe } bool recycle_one_lsn_info_page(StandbyReadMetaInfo *meta_info, XLogRecPtr recycle_lsn, - BasePagePosition *base_page_position) + BasePagePosition *base_page_position, uint64 *next_page_pos) { uint32 batch_id = meta_info->batch_id; uint32 worker_id = meta_info->redo_id; @@ -654,7 +679,7 @@ bool recycle_one_lsn_info_page(StandbyReadMetaInfo *meta_info, XLogRecPtr recycl XLogRecPtr block_info_max_lsn = InvalidXLogRecPtr; StandbyReadRecyleState stat = - recyle_block_info(buf_tag, cur_base_page_info_pos, next_base_page_lsn, recycle_lsn, &block_info_max_lsn); + recycle_block_info(meta_info, buf_tag, cur_base_page_info_pos, next_base_page_lsn, recycle_lsn, &block_info_max_lsn); if (stat == STANDBY_READ_RECLYE_ALL) { invalid_base_page_list(meta_info, buffer, offset); } else if (stat == STANDBY_READ_RECLYE_NONE) { @@ -663,10 +688,14 @@ bool recycle_one_lsn_info_page(StandbyReadMetaInfo *meta_info, XLogRecPtr recycl return false; } } - - if (buffer_is_locked) { - LockBuffer(buffer, BUFFER_LOCK_UNLOCK); + + if (!buffer_is_locked) { + LockBuffer(buffer, BUFFER_LOCK_SHARE); } + LsnInfoPageHeader *page_header = (LsnInfoPageHeader *)page; + Assert(page_header != NULL); + *next_page_pos = page_header->next_lsn_info_pos; + LockBuffer(buffer, BUFFER_LOCK_UNLOCK); ReleaseBuffer(buffer); return true; } @@ -683,13 +712,17 @@ void standby_read_recyle_per_workers(StandbyReadMetaInfo *meta_info, XLogRecPtr uint64 recyled_page_len = 0; const uint32 recyle_ratio = 32; // no need recyle so fast - while (meta_info->lsn_table_recyle_position + BLCKSZ * recyle_ratio < meta_info->lsn_table_next_position) { - recycle_next_page = recycle_one_lsn_info_page(meta_info, recycle_lsn, &base_page_position); + uint64 next_page_pos = meta_info->lsn_table_recyle_position; + if (!meta_info->is_lsn_table_recyle_position_init) { + return; + } + while (meta_info->lsn_table_recyle_position + BLCKSZ * recyle_ratio < next_page_pos) { + recycle_next_page = recycle_one_lsn_info_page(meta_info, recycle_lsn, &base_page_position, &next_page_pos); if (!recycle_next_page) { break; } /* update recycle position */ - meta_info->lsn_table_recyle_position += BLCKSZ; + meta_info->lsn_table_recyle_position = next_page_pos; recyled_page_len += BLCKSZ; Assert(meta_info->lsn_table_recyle_position % BLCKSZ == 0); if (recyled_page_len >= EXRTO_LSN_INFO_FILE_MAXSIZE) { @@ -709,9 +742,7 @@ void standby_read_recyle_per_workers(StandbyReadMetaInfo *meta_info, XLogRecPtr cur_lsn_table_recyle_segno > last_lsn_table_recyle_segno) { buffer_drop_exrto_standby_read_buffers(meta_info); } - if (cur_lsn_table_recyle_segno > last_lsn_table_recyle_segno) { - recycle_lsn_info_file(meta_info->batch_id, meta_info->redo_id, meta_info->lsn_table_recyle_position); - } + if (cur_base_page_recyle_segno > last_base_page_recyle_segno) { recycle_base_page_file(meta_info->batch_id, meta_info->redo_id, meta_info->base_page_recyle_position); } diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp index ffcbfb9f49..e8fe901bea 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -29,6 +29,7 @@ #include "access/extreme_rto/standby_read/standby_read_delay_ddl.h" #include "access/multi_redo_api.h" #include "pgstat.h" +#include "postgres_ext.h" #include "storage/smgr/relfilenode.h" #include "storage/buf/buf_internals.h" #include "storage/buf/bufmgr.h" @@ -48,28 +49,47 @@ const char* EXRTO_FILE_SUB_DIR[] = { const uint32 EXRTO_FILE_PATH_LEN = 1024; const uint32 XID_THIRTY_TWO = 32; -void make_standby_read_node(XLogRecPtr read_lsn, RelFileNode &read_node, bool is_start_lsn, Oid relnode) +void make_standby_read_node(XLogRecPtr read_lsn, RelFileNode &read_node, bool is_start_lsn, const RelFileNode& orig_node) { read_node.spcNode = (Oid)(read_lsn >> 32); read_node.dbNode = (Oid)(read_lsn); - read_node.relNode = relnode; + read_node.relNode = orig_node.relNode; read_node.opt = 0; if (is_start_lsn) { /* means read_lsn is the start ptr of xlog */ - read_node.bucketNode = ExrtoReadStartLSNBktId; + read_node.bucketNode = EXRTO_READ_STANDBY_START_LSN_OPT; } else { /* means read_lsn is the end ptr of xlog */ - read_node.bucketNode = ExrtoReadEndLSNBktId; + read_node.bucketNode = EXRTO_READ_STANDBY_END_LSN_OPT; } + if (IsSegmentFileNode(orig_node)) { + read_node.bucketNode = orig_node.bucketNode - EXRTO_STANDBY_READ_BUCKET_OFFSET; + } else { + read_node.bucketNode = EXRTO_READ_SPECIAL_LSN; + } +} + +BufferDesc *alloc_standby_seg_read_buf(const BufferTag &buf_tag, bool &found, XLogRecPtr read_lsn, bool is_start_lsn) +{ + RelFileNode read_node; + make_standby_read_node(read_lsn, read_node, is_start_lsn, buf_tag.rnode); + BufferDesc *buf_desc = SegBufferAlloc(read_node, buf_tag.forkNum, buf_tag.blockNum, &found); + return buf_desc; } -BufferDesc *alloc_standby_read_buf(const BufferTag &buf_tag, BufferAccessStrategy strategy, bool &found, - XLogRecPtr read_lsn, bool is_start_lsn) +BufferDesc *alloc_standby_read_buf(const BufferTag &buf_tag, BufferAccessStrategy strategy, bool &found, XLogRecPtr read_lsn, bool is_start_lsn) { RelFileNode read_node; - make_standby_read_node(read_lsn, read_node, is_start_lsn, buf_tag.rnode.relNode); + make_standby_read_node(read_lsn, read_node, is_start_lsn, buf_tag.rnode); BufferDesc *buf_desc = BufferAlloc(read_node, 0, buf_tag.forkNum, buf_tag.blockNum, strategy, &found, NULL); + return buf_desc; +} +BufferDesc *find_standby_seg_read_buf(const BufferTag &buf_tag, bool &found, XLogRecPtr read_lsn, bool is_start_lsn) +{ + RelFileNode read_node; + make_standby_read_node(read_lsn, read_node, is_start_lsn, buf_tag.rnode); + BufferDesc *buf_desc = SegBufferFind(read_node, buf_tag.forkNum, buf_tag.blockNum, &found); return buf_desc; } @@ -120,6 +140,50 @@ Buffer get_newest_page_for_read(Relation reln, ForkNumber fork_num, BlockNumber return BufferDescriptorGetBuffer(buf_desc); } +Buffer get_newest_seg_page_for_read(SegSpace *spc, const RelFileNode& rnode, ForkNumber forknum, + BlockNumber blocknum, ReadBufferMode mode, XLogRecPtr read_lsn) +{ + Buffer newest_seg_buf = ReadBufferFastNormal(spc, rnode, forknum, blocknum, mode); + if (BufferIsInvalid(newest_seg_buf)) { + return InvalidBuffer; + } + + LockBuffer(newest_seg_buf, BUFFER_LOCK_SHARE); + Page newest_page = BufferGetPage(newest_seg_buf); + XLogRecPtr page_lsn = PageGetLSN(newest_page); + if (XLByteLT(read_lsn, page_lsn)) { + UnlockReleaseBuffer(newest_seg_buf); + return InvalidBuffer; + } + + BufferTag buf_tag = { + .rnode = rnode, + .forkNum = forknum, + .blockNum = blocknum, + }; + + ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); + bool hit = false; + BufferDesc *buf_desc = alloc_standby_seg_read_buf(buf_tag, hit, page_lsn, false); + if (hit) { + UnlockReleaseBuffer(newest_seg_buf); + return BufferDescriptorGetBuffer(buf_desc); + } + Page read_page = (Page)BufHdrGetBlock(buf_desc); + + errno_t rc = memcpy_s(read_page, BLCKSZ, newest_page, BLCKSZ); + securec_check(rc, "\0", "\0"); + + UnlockReleaseBuffer(newest_seg_buf); + buf_desc->extra->lsn_on_disk = PageGetLSN(read_page); +#ifdef USE_ASSERT_CHECKING + buf_desc->lsn_dirty = InvalidXLogRecPtr; +#endif + + TerminateBufferIO(buf_desc, false, (BM_VALID | BM_IS_TMP_BUF)); + return BufferDescriptorGetBuffer(buf_desc); +} + Buffer get_newest_page_for_read_new( Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, BufferAccessStrategy strategy) { @@ -163,6 +227,83 @@ Buffer get_newest_page_for_read_new( return BufferDescriptorGetBuffer(buf_desc); } +Buffer standby_read_seg_buffer( + SegSpace *spc, const RelFileNode &rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode) +{ + Assert(IsSegmentPhysicalRelNode(rnode)); + XLogRecPtr read_lsn = MAX_XLOG_REC_PTR; + + if (u_sess->utils_cxt.CurrentSnapshot != NULL && XLogRecPtrIsValid(u_sess->utils_cxt.CurrentSnapshot->read_lsn)) { + read_lsn = u_sess->utils_cxt.CurrentSnapshot->read_lsn; + } else if (XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)) { + read_lsn = t_thrd.proc->exrto_read_lsn; + } + + Buffer read_buf = get_newest_seg_page_for_read(spc, rnode, forkNum, blockNum, mode, read_lsn); + if (read_buf != InvalidBuffer) { + // newest page's lsn smaller than read lsn + return read_buf; + } + + ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); + // read lsn info + StandbyReadLsnInfoArray *lsn_info = &t_thrd.exrto_recycle_cxt.lsn_info; + BufferTag buf_tag = { + .rnode = rnode, + .forkNum = forkNum, + .blockNum = blockNum, + }; + buffer_in_progress_pop(); + bool result = extreme_rto_standby_read::get_page_lsn_info(buf_tag, buf_tag, NULL, read_lsn, lsn_info); + if (!result) { + ereport( + ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + (errmsg("standby_read_buf couldnot found buf %u/%u/%u %d %u read lsn %08X/%08X current_time: %ld " + "gen_snaptime:%ld thread_read_lsn:%08X/%08X", + buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, + buf_tag.blockNum, (uint32)(read_lsn >> XID_THIRTY_TWO), (uint32)read_lsn, GetCurrentTimestamp(), + g_instance.comm_cxt.predo_cxt.exrto_snapshot->gen_snap_time, + (uint32)(t_thrd.proc->exrto_read_lsn >> XID_THIRTY_TWO), (uint32)t_thrd.proc->exrto_read_lsn)))); + return InvalidBuffer; + } + buffer_in_progress_push(); + + // read lsn info + XLogRecPtr expected_lsn = InvalidXLogRecPtr; + bool is_start_lsn = true; + if (lsn_info->lsn_num == 0) { + expected_lsn = lsn_info->base_page_lsn; + is_start_lsn = false; + } else { + Assert(lsn_info->lsn_array[lsn_info->lsn_num - 1] > 0); + Assert(lsn_info->lsn_array[lsn_info->lsn_num - 1] < read_lsn); + Assert(lsn_info->lsn_array[lsn_info->lsn_num - 1] >= lsn_info->base_page_lsn); + expected_lsn = lsn_info->lsn_array[lsn_info->lsn_num - 1]; + } + + bool hit = false; + BufferDesc* buf_desc = alloc_standby_seg_read_buf(buf_tag, hit, expected_lsn, is_start_lsn); + if (hit) { + return BufferDescriptorGetBuffer(buf_desc); + } + buffer_in_progress_pop(); + // read_base_page + extreme_rto_standby_read::read_base_page(buf_tag, lsn_info->base_page_pos, buf_desc); + if (lsn_info->lsn_num > 0) { + redo_target_page(buf_tag, lsn_info, BufferDescriptorGetBuffer(buf_desc)); + } + Page page = BufferGetPage(BufferDescriptorGetBuffer(buf_desc)); + buf_desc->extra->lsn_on_disk = PageGetLSN(page); +#ifdef USE_ASSERT_CHECKING + buf_desc->lsn_dirty = InvalidXLogRecPtr; +#endif + buffer_in_progress_push(); + TerminateBufferIO(buf_desc, false, (BM_VALID | BM_IS_TMP_BUF)); + + return BufferDescriptorGetBuffer(buf_desc); +} + Buffer standby_read_buf( Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, BufferAccessStrategy strategy) { diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp index f102065656..13289e2b2e 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp @@ -1,427 +1,426 @@ -/* - * Copyright (c) 2020 Huawei Technologies Co.,Ltd. - * - * openGauss is licensed under Mulan PSL v2. - * You can use this software according to the terms and conditions of the Mulan PSL v2. - * You may obtain a copy of Mulan PSL v2 at: - * - * http://license.coscl.org.cn/MulanPSL2 - * - * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, - * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, - * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. - * See the Mulan PSL v2 for more details. - * ------------------------------------------------------------------------- - * - * standby_read_proc.cpp - * - * IDENTIFICATION - * src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp - * - * ------------------------------------------------------------------------- - */ - -#include "access/extreme_rto/page_redo.h" -#include "access/extreme_rto/standby_read/block_info_meta.h" -#include "access/extreme_rto/standby_read/lsn_info_meta.h" -#include "access/extreme_rto/standby_read/standby_read_base.h" -#include "access/multi_redo_api.h" -#include "access/extreme_rto/dispatcher.h" -#include "storage/procarray.h" -#include "replication/walreceiver.h" - -inline void invalid_msg_leak_warning(XLogRecPtr trxn_lsn) -{ - if (t_thrd.page_redo_cxt.invalid_msg.valid) { - ereport(WARNING, - (errmsg(EXRTOFORMAT("[exrto_generate_snapshot] not send invalid msg: %08X/%08X"), - (uint32)(trxn_lsn >> UINT64_HALF), - (uint32)trxn_lsn))); - } -} - -void exrto_generate_snapshot(XLogRecPtr trxn_lsn) -{ - if (!g_instance.attr.attr_storage.EnableHotStandby) { - return; - } - - ExrtoSnapshot exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; - /* - * do not generate the same snapshot repeatedly. - */ - if (XLByteLE(trxn_lsn, exrto_snapshot->read_lsn)) { - invalid_msg_leak_warning(trxn_lsn); - return; - } - - TransactionId xmin; - TransactionId xmax; - CommitSeqNo snapshot_csn; - - exrto_get_snapshot_data(xmin, xmax, snapshot_csn); - (void)LWLockAcquire(ExrtoSnapshotLock, LW_EXCLUSIVE); - exrto_snapshot->snapshot_csn = snapshot_csn; - exrto_snapshot->xmin = xmin; - exrto_snapshot->xmax = xmax; - exrto_snapshot->read_lsn = trxn_lsn; - send_delay_invalid_message(); - LWLockRelease(ExrtoSnapshotLock); -} - -void exrto_read_snapshot(Snapshot snapshot) -{ - if ((!is_exrto_standby_read_worker()) || u_sess->proc_cxt.clientIsCMAgent || dummyStandbyMode) { - return; - } - - ExrtoSnapshot exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; - bool retry_get = false; - const static uint64 WAIT_COUNT = 0x7FFFF; - uint64 retry_count = 0; - t_thrd.pgxact->xmin = InvalidTransactionId; - t_thrd.proc->exrto_min = InvalidXLogRecPtr; -RETRY_GET: - if (retry_get) { - CHECK_FOR_INTERRUPTS(); - pg_usleep(100L); - } - retry_count++; - if ((retry_count & WAIT_COUNT) == WAIT_COUNT) { - ereport(LOG, - (errmsg("retry to get exrto-standby-read snapshot, standby_redo_cleanup_xmin = %lu, " - "standby_redo_cleanup_xmin_lsn = %08X/%08X, " - "exrto_snapshot->xmin = %lu, read_lsn = %08X/%08X", - t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXmin, - (uint32)(t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXminLsn >> UINT64_HALF), - (uint32)t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXminLsn, exrto_snapshot->xmin, - (uint32)(exrto_snapshot->read_lsn >> UINT64_HALF), (uint32)exrto_snapshot->read_lsn))); - } - (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); - if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { - LWLockRelease(ExrtoSnapshotLock); - ereport(ERROR, (errmsg("could not get a valid snapshot with extreme rto"))); - } - - /* In exrto_standby_read_opt mode, getting a snapshot needs to wait for the cleanup-info xlog to be processed. */ - if (g_instance.attr.attr_storage.enable_exrto_standby_read_opt) { - LWLockAcquire(ProcArrayLock, LW_SHARED); - bool condition = - (exrto_snapshot->xmin <= - t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXmin) && - (t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXminLsn > exrto_snapshot->read_lsn); - LWLockRelease(ProcArrayLock); - if (condition) { - retry_get = true; - LWLockRelease(ExrtoSnapshotLock); - goto RETRY_GET; - } - } - - snapshot->snapshotcsn = exrto_snapshot->snapshot_csn; - snapshot->xmin = exrto_snapshot->xmin; - snapshot->xmax = exrto_snapshot->xmax; - snapshot->read_lsn = exrto_snapshot->read_lsn; - - t_thrd.pgxact->xmin = snapshot->xmin; - u_sess->utils_cxt.TransactionXmin = snapshot->xmin; - - t_thrd.proc->exrto_read_lsn = snapshot->read_lsn; - t_thrd.proc->exrto_min = snapshot->read_lsn; - LWLockRelease(ExrtoSnapshotLock); - - if (t_thrd.proc->exrto_gen_snap_time == 0) { - t_thrd.proc->exrto_gen_snap_time = GetCurrentTimestamp(); - } - Assert(XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)); -} - -static inline uint64 get_force_recycle_pos(uint64 recycle_pos, uint64 insert_pos) -{ - const double force_recyle_ratio = 0.3; /* to be adjusted */ - Assert(recycle_pos <= insert_pos); - return recycle_pos + (uint64)((insert_pos - recycle_pos) * force_recyle_ratio); -} - -XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info) -{ - uint64 base_page_recycle_pos; - uint64 lsn_info_recycle_pos; - XLogRecPtr base_page_recycle_lsn = InvalidXLogRecPtr; - XLogRecPtr lsn_info_recycle_lsn = InvalidXLogRecPtr; - Buffer buffer; - Page page; - - /* for base page */ - if (meta_info->base_page_recyle_position < meta_info->base_page_next_position) { - base_page_recycle_pos = - get_force_recycle_pos(meta_info->base_page_recyle_position, meta_info->base_page_next_position); - buffer = extreme_rto_standby_read::buffer_read_base_page( - meta_info->batch_id, meta_info->redo_id, base_page_recycle_pos, RBM_NORMAL); - LockBuffer(buffer, BUFFER_LOCK_SHARE); - base_page_recycle_lsn = PageGetLSN(BufferGetPage(buffer)); - UnlockReleaseBuffer(buffer); - } - - /* for lsn info */ - if (meta_info->lsn_table_recyle_position < meta_info->lsn_table_next_position) { - lsn_info_recycle_pos = - get_force_recycle_pos(meta_info->lsn_table_recyle_position, meta_info->lsn_table_next_position); - page = extreme_rto_standby_read::get_lsn_info_page( - meta_info->batch_id, meta_info->redo_id, lsn_info_recycle_pos, RBM_NORMAL, &buffer); - if (unlikely(page == NULL || buffer == InvalidBuffer)) { - ereport(PANIC, - (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), - meta_info->batch_id, - meta_info->redo_id, - lsn_info_recycle_pos))); - } - LockBuffer(buffer, BUFFER_LOCK_SHARE); - extreme_rto_standby_read::LsnInfo lsn_info = - (extreme_rto_standby_read::LsnInfo)(page + extreme_rto_standby_read::LSN_INFO_HEAD_SIZE); - lsn_info_recycle_lsn = lsn_info->lsn[0]; - UnlockReleaseBuffer(buffer); - } - - return rtl::max(base_page_recycle_lsn, lsn_info_recycle_lsn); -} - -void calculate_force_recycle_lsn(XLogRecPtr &recycle_lsn) -{ - XLogRecPtr recycle_lsn_per_worker; - uint32 worker_nums = extreme_rto::g_dispatcher->allWorkersCnt; - extreme_rto::PageRedoWorker **workers = extreme_rto::g_dispatcher->allWorkers; - - for (uint32 i = 0; i < worker_nums; ++i) { - extreme_rto::PageRedoWorker *page_redo_worker = workers[i]; - if (page_redo_worker->role != extreme_rto::REDO_PAGE_WORKER || (page_redo_worker->isUndoSpaceWorker)) { - continue; - } - recycle_lsn_per_worker = calculate_force_recycle_lsn_per_worker(&page_redo_worker->standby_read_meta_info); - if (XLByteLT(recycle_lsn, recycle_lsn_per_worker)) { - recycle_lsn = recycle_lsn_per_worker; - } - } - ereport(LOG, - (errmsg(EXRTOFORMAT("[exrto_recycle] try force recycle, recycle lsn: %08X/%08X"), - (uint32)(recycle_lsn >> UINT64_HALF), - (uint32)recycle_lsn))); -} - -static inline bool exceed_standby_max_query_time(TimestampTz start_time) -{ - if (start_time == 0) { - return false; - } - return TimestampDifferenceExceeds( - start_time, GetCurrentTimestamp(), g_instance.attr.attr_storage.standby_max_query_time * MSECS_PER_SEC); -} - -/* 1. resolve recycle conflict with backends - * 2. get oldest xmin and oldest readlsn of backends. */ -void proc_array_get_oldeset_readlsn( - XLogRecPtr recycle_lsn, XLogRecPtr &oldest_lsn, TransactionId &oldest_xmin, bool &conflict) -{ - ProcArrayStruct *proc_array = g_instance.proc_array_idx; - conflict = false; - - (void)LWLockAcquire(ProcArrayLock, LW_SHARED); - for (int index = 0; index < proc_array->numProcs; index++) { - int pg_proc_no = proc_array->pgprocnos[index]; - PGPROC *pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; - PGXACT *pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; - TransactionId pxmin = pg_xact->xmin; - XLogRecPtr read_lsn = pg_proc->exrto_min; - ereport(DEBUG1, - (errmsg(EXRTOFORMAT("proc_array_get_oldeset_readlsn info, read_lsn: %08X/%08X ,xmin: %lu ,vacuum_flags: " - "%hhu ,pid: %lu"), - (uint32)(read_lsn >> UINT64_HALF), - (uint32)read_lsn, - pxmin, - pg_xact->vacuumFlags, - pg_proc->pid))); - - if (pg_proc->pid == 0 || XLogRecPtrIsInvalid(read_lsn)) { - continue; - } - - Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); - /* - * Backend is doing logical decoding which manages xmin - * separately, check below. - */ - if (pg_xact->vacuumFlags & PROC_IN_LOGICAL_DECODING) { - continue; - } - - /* cancel query when its read_lsn < recycle_lsn or its runtime > standby_max_query_time */ - if (XLByteLT(read_lsn, recycle_lsn) || exceed_standby_max_query_time(pg_proc->exrto_gen_snap_time)) { - pg_proc->recoveryConflictPending = true; - conflict = true; - if (pg_proc->pid != 0) { - /* - * Kill the pid if it's still here. If not, that's what we - * wanted so ignore any errors. - */ - (void)SendProcSignal(pg_proc->pid, PROCSIG_RECOVERY_CONFLICT_SNAPSHOT, pg_proc->backendId); - ereport(LOG, - (errmsg( - EXRTOFORMAT("read_lsn is less than recycle_lsn or query time exceed max_query_time while " - "get_oldeset_readlsn, read_lsn %lu, " - "recycle_lsn: %lu, exrto_gen_snap_time: %ld, current_time: %ld, thread id = %lu\n"), - read_lsn, - recycle_lsn, - pg_proc->exrto_gen_snap_time, - GetCurrentTimestamp(), - pg_proc->pid))); - /* - * Wait a little bit for it to die so that we avoid flooding - * an unresponsive backend when system is heavily loaded. - */ - pg_usleep(5000L); - } - continue; - } - - if (XLogRecPtrIsInvalid(oldest_lsn) || (XLogRecPtrIsValid(read_lsn) && XLByteLT(read_lsn, oldest_lsn))) { - oldest_lsn = read_lsn; - } - - if (!TransactionIdIsValid(oldest_xmin) || - (TransactionIdIsValid(pxmin) && TransactionIdFollows(oldest_xmin, pxmin))) { - oldest_xmin = pxmin; - } - } - LWLockRelease(ProcArrayLock); -} - -void proc_array_get_oldeset_xmin_for_undo(TransactionId &oldest_xmin) -{ - ProcArrayStruct *proc_array = g_instance.proc_array_idx; - - (void)LWLockAcquire(ProcArrayLock, LW_SHARED); - for (int index = 0; index < proc_array->numProcs; index++) { - int pg_proc_no = proc_array->pgprocnos[index]; - PGPROC *pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; - PGXACT *pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; - TransactionId pxmin = pg_xact->xmin; - - if (pg_proc->pid == 0 || !TransactionIdIsValid(pxmin)) { - continue; - } - - Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); - /* - * Backend is doing logical decoding which manages xmin - * separately, check below. - */ - if (pg_xact->vacuumFlags & PROC_IN_LOGICAL_DECODING) { - continue; - } - if (!TransactionIdIsValid(oldest_xmin) || - (TransactionIdIsValid(pxmin) && TransactionIdFollows(oldest_xmin, pxmin))) { - oldest_xmin = pxmin; - } - } - LWLockRelease(ProcArrayLock); -} - -XLogRecPtr exrto_calculate_recycle_position(bool force_recyle) -{ - Assert(t_thrd.role != PAGEREDO); - Assert(IS_EXRTO_READ); - - XLogRecPtr recycle_lsn = pg_atomic_read_u64(&g_instance.comm_cxt.predo_cxt.global_recycle_lsn); - XLogRecPtr oldest_lsn = InvalidXLogRecPtr; - TransactionId oldest_xmin = InvalidTransactionId; - bool conflict = false; - const int max_check_times = 1000; - int check_times = 0; - - if (force_recyle) { - calculate_force_recycle_lsn(recycle_lsn); - } - ereport(DEBUG1, - (errmsg(EXRTOFORMAT("time information of calculate recycle position, current_time: %ld, snapshot " - "read_lsn: %08X/%08X, gen_snaptime:%ld"), - GetCurrentTimestamp(), - (uint32)(g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn >> UINT64_HALF), - (uint32)g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn, - g_instance.comm_cxt.predo_cxt.exrto_snapshot->gen_snap_time))); - - /* - * If there is no backend read threads, set read oldest lsn to snapshot lsn. - */ - ExrtoSnapshot exrto_snapshot = NULL; - exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; - (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); - if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { - ereport(WARNING, (errmsg("could not get a valid snapshot with extreme rto"))); - } else { - oldest_lsn = exrto_snapshot->read_lsn; - oldest_xmin = exrto_snapshot->xmin; - } - LWLockRelease(ExrtoSnapshotLock); - /* Loop checks to avoid conflicting queries that were not successfully canceled. */ - do { - RedoInterruptCallBack(); - proc_array_get_oldeset_readlsn(recycle_lsn, oldest_lsn, oldest_xmin, conflict); - check_times++; - } while (conflict && check_times < max_check_times); - - recycle_lsn = rtl::max(recycle_lsn, oldest_lsn); - - ereport(LOG, - (errmsg( - EXRTOFORMAT( - "[exrto_recycle] calculate recycle position, oldestlsn: %08X/%08X, snapshot read_lsn: %08X/%08X, try " - "recycle lsn: %08X/%08X, xmin: %lu"), - (uint32)(oldest_lsn >> UINT64_HALF), - (uint32)oldest_lsn, - (uint32)(g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn >> UINT64_HALF), - (uint32)g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn, - (uint32)(recycle_lsn >> UINT64_HALF), - (uint32)recycle_lsn, oldest_xmin))); - pg_atomic_write_u64(&g_instance.comm_cxt.predo_cxt.exrto_recyle_xmin, oldest_xmin); - return recycle_lsn; -} - -TransactionId exrto_calculate_recycle_xmin_for_undo() -{ - Assert(t_thrd.role != PAGEREDO); - Assert(IS_EXRTO_READ); - TransactionId oldest_xmin = InvalidTransactionId; - TransactionId snapshot_xmin = InvalidTransactionId; - proc_array_get_oldeset_xmin_for_undo(oldest_xmin); - - /* - * If there is no backend read threads, set read oldest lsn to snapshot lsn. - */ - if (extreme_rto::g_dispatcher != NULL) { - ExrtoSnapshot exrto_snapshot = NULL; - exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; - (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); - if (XLByteEQ(exrto_snapshot->xmin, InvalidTransactionId)) { - ereport(WARNING, - (errmsg("exrto_calculate_recycle_xmin_for_undo: could not get a valid snapshot in exrto_snapshot"))); - } else { - snapshot_xmin = exrto_snapshot->xmin; - } - - LWLockRelease(ExrtoSnapshotLock); - } - ereport(DEBUG1, - (errmodule(MOD_UNDO), - errmsg(UNDOFORMAT("exrto_calculate_recycle_xmin_for_undo: oldest_xmin: %lu, snapshot_xmin: %lu."), - oldest_xmin, - snapshot_xmin))); - - if (oldest_xmin == InvalidTransactionId) { - return snapshot_xmin; - } - - if (TransactionIdPrecedes(snapshot_xmin, oldest_xmin)) { - return snapshot_xmin; - } - return oldest_xmin; -} +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * standby_read_proc.cpp + * + * IDENTIFICATION + * src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp + * + * ------------------------------------------------------------------------- + */ + +#include "access/extreme_rto/page_redo.h" +#include "access/extreme_rto/standby_read/block_info_meta.h" +#include "access/extreme_rto/standby_read/lsn_info_meta.h" +#include "access/extreme_rto/standby_read/standby_read_base.h" +#include "access/multi_redo_api.h" +#include "access/extreme_rto/dispatcher.h" +#include "storage/procarray.h" +#include "replication/walreceiver.h" + +inline void invalid_msg_leak_warning(XLogRecPtr trxn_lsn) +{ + if (t_thrd.page_redo_cxt.invalid_msg.valid) { + ereport(WARNING, + (errmsg(EXRTOFORMAT("[exrto_generate_snapshot] not send invalid msg: %08X/%08X"), + (uint32)(trxn_lsn >> UINT64_HALF), + (uint32)trxn_lsn))); + } +} + +void exrto_generate_snapshot(XLogRecPtr trxn_lsn) +{ + if (!g_instance.attr.attr_storage.EnableHotStandby) { + return; + } + + ExrtoSnapshot exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; + /* + * do not generate the same snapshot repeatedly. + */ + if (XLByteLE(trxn_lsn, exrto_snapshot->read_lsn)) { + invalid_msg_leak_warning(trxn_lsn); + return; + } + + TransactionId xmin; + TransactionId xmax; + CommitSeqNo snapshot_csn; + + exrto_get_snapshot_data(xmin, xmax, snapshot_csn); + (void)LWLockAcquire(ExrtoSnapshotLock, LW_EXCLUSIVE); + exrto_snapshot->snapshot_csn = snapshot_csn; + exrto_snapshot->xmin = xmin; + exrto_snapshot->xmax = xmax; + exrto_snapshot->read_lsn = trxn_lsn; + send_delay_invalid_message(); + LWLockRelease(ExrtoSnapshotLock); +} + +void exrto_read_snapshot(Snapshot snapshot) +{ + if ((!is_exrto_standby_read_worker()) || u_sess->proc_cxt.clientIsCMAgent || dummyStandbyMode) { + return; + } + + ExrtoSnapshot exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; + bool retry_get = false; + const static uint64 WAIT_COUNT = 0x7FFFF; + uint64 retry_count = 0; + t_thrd.pgxact->xmin = InvalidTransactionId; + t_thrd.proc->exrto_min = InvalidXLogRecPtr; +RETRY_GET: + if (retry_get) { + CHECK_FOR_INTERRUPTS(); + pg_usleep(100L); + } + retry_count++; + if ((retry_count & WAIT_COUNT) == WAIT_COUNT) { + ereport(LOG, + (errmsg("retry to get exrto-standby-read snapshot, standby_redo_cleanup_xmin = %lu, " + "standby_redo_cleanup_xmin_lsn = %08X/%08X, " + "exrto_snapshot->xmin = %lu, read_lsn = %08X/%08X", + t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXmin, + (uint32)(t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXminLsn >> UINT64_HALF), + (uint32)t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXminLsn, exrto_snapshot->xmin, + (uint32)(exrto_snapshot->read_lsn >> UINT64_HALF), (uint32)exrto_snapshot->read_lsn))); + } + (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); + if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { + LWLockRelease(ExrtoSnapshotLock); + ereport(ERROR, (errmsg("could not get a valid snapshot with extreme rto"))); + } + + /* In exrto_standby_read_opt mode, getting a snapshot needs to wait for the cleanup-info xlog to be processed. */ + if (g_instance.attr.attr_storage.enable_exrto_standby_read_opt) { + LWLockAcquire(ProcArrayLock, LW_SHARED); + bool condition = + (exrto_snapshot->xmin <= + t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXmin) && + (t_thrd.xact_cxt.ShmemVariableCache->standbyRedoCleanupXminLsn > exrto_snapshot->read_lsn); + LWLockRelease(ProcArrayLock); + if (condition) { + retry_get = true; + LWLockRelease(ExrtoSnapshotLock); + goto RETRY_GET; + } + } + + snapshot->snapshotcsn = exrto_snapshot->snapshot_csn; + snapshot->xmin = exrto_snapshot->xmin; + snapshot->xmax = exrto_snapshot->xmax; + snapshot->read_lsn = exrto_snapshot->read_lsn; + + t_thrd.pgxact->xmin = snapshot->xmin; + u_sess->utils_cxt.TransactionXmin = snapshot->xmin; + + t_thrd.proc->exrto_read_lsn = snapshot->read_lsn; + t_thrd.proc->exrto_min = snapshot->read_lsn; + LWLockRelease(ExrtoSnapshotLock); + + if (t_thrd.proc->exrto_gen_snap_time == 0) { + t_thrd.proc->exrto_gen_snap_time = GetCurrentTimestamp(); + } + Assert(XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)); +} + +static inline uint64 get_force_recycle_pos(uint64 recycle_pos, uint64 insert_pos) +{ + const double force_recyle_ratio = 0.3; /* to be adjusted */ + Assert(recycle_pos <= insert_pos); + return recycle_pos + (uint64)((insert_pos - recycle_pos) * force_recyle_ratio); +} + +XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info) +{ + uint64 lsn_info_recycle_pos; + XLogRecPtr base_page_recycle_lsn = InvalidXLogRecPtr; + XLogRecPtr lsn_info_recycle_lsn = InvalidXLogRecPtr; + Buffer buffer; + Page page; + + /* for base page */ + if (meta_info->base_page_recyle_position < meta_info->base_page_next_position) { + base_page_recycle_pos = + get_force_recycle_pos(meta_info->base_page_recyle_position, meta_info->base_page_next_position); + buffer = extreme_rto_standby_read::buffer_read_base_page( + meta_info->batch_id, meta_info->redo_id, base_page_recycle_pos, RBM_NORMAL); + LockBuffer(buffer, BUFFER_LOCK_SHARE); + base_page_recycle_lsn = PageGetLSN(BufferGetPage(buffer)); + UnlockReleaseBuffer(buffer); + } + + /* for lsn info */ + if (meta_info->lsn_table_recyle_position < meta_info->lsn_table_next_position) { + lsn_info_recycle_pos = + get_force_recycle_pos(meta_info->lsn_table_recyle_position, meta_info->lsn_table_next_position); + page = extreme_rto_standby_read::get_lsn_info_page( + meta_info->batch_id, meta_info->redo_id, lsn_info_recycle_pos, RBM_NORMAL, &buffer); + if (unlikely(page == NULL || buffer == InvalidBuffer)) { + ereport(PANIC, + (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), + meta_info->batch_id, + meta_info->redo_id, + lsn_info_recycle_pos))); + } + LockBuffer(buffer, BUFFER_LOCK_SHARE); + extreme_rto_standby_read::LsnInfo lsn_info = + (extreme_rto_standby_read::LsnInfo)(page + extreme_rto_standby_read::LSN_INFO_HEAD_SIZE); + lsn_info_recycle_lsn = lsn_info->lsn[0]; + UnlockReleaseBuffer(buffer); + } + + return lsn_info_recycle_lsn; +} + +void calculate_force_recycle_lsn(XLogRecPtr &recycle_lsn) +{ + XLogRecPtr recycle_lsn_per_worker; + uint32 worker_nums = extreme_rto::g_dispatcher->allWorkersCnt; + extreme_rto::PageRedoWorker **workers = extreme_rto::g_dispatcher->allWorkers; + + for (uint32 i = 0; i < worker_nums; ++i) { + extreme_rto::PageRedoWorker *page_redo_worker = workers[i]; + if (page_redo_worker->role != extreme_rto::REDO_PAGE_WORKER || (page_redo_worker->isUndoSpaceWorker)) { + continue; + } + recycle_lsn_per_worker = calculate_force_recycle_lsn_per_worker(&page_redo_worker->standby_read_meta_info); + if (XLByteLT(recycle_lsn, recycle_lsn_per_worker)) { + recycle_lsn = recycle_lsn_per_worker; + } + } + ereport(LOG, + (errmsg(EXRTOFORMAT("[exrto_recycle] try force recycle, recycle lsn: %08X/%08X"), + (uint32)(recycle_lsn >> UINT64_HALF), + (uint32)recycle_lsn))); +} + +static inline bool exceed_standby_max_query_time(TimestampTz start_time) +{ + if (start_time == 0) { + return false; + } + return TimestampDifferenceExceeds( + start_time, GetCurrentTimestamp(), g_instance.attr.attr_storage.standby_max_query_time * MSECS_PER_SEC); +} + +/* 1. resolve recycle conflict with backends + * 2. get oldest xmin and oldest readlsn of backends. */ +void proc_array_get_oldeset_readlsn( + XLogRecPtr recycle_lsn, XLogRecPtr &oldest_lsn, TransactionId &oldest_xmin, bool &conflict) +{ + ProcArrayStruct *proc_array = g_instance.proc_array_idx; + conflict = false; + + (void)LWLockAcquire(ProcArrayLock, LW_SHARED); + for (int index = 0; index < proc_array->numProcs; index++) { + int pg_proc_no = proc_array->pgprocnos[index]; + PGPROC *pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; + PGXACT *pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; + TransactionId pxmin = pg_xact->xmin; + XLogRecPtr read_lsn = pg_proc->exrto_min; + ereport(DEBUG1, + (errmsg(EXRTOFORMAT("proc_array_get_oldeset_readlsn info, read_lsn: %08X/%08X ,xmin: %lu ,vacuum_flags: " + "%hhu ,pid: %lu"), + (uint32)(read_lsn >> UINT64_HALF), + (uint32)read_lsn, + pxmin, + pg_xact->vacuumFlags, + pg_proc->pid))); + + if (pg_proc->pid == 0 || XLogRecPtrIsInvalid(read_lsn)) { + continue; + } + + Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); + /* + * Backend is doing logical decoding which manages xmin + * separately, check below. + */ + if (pg_xact->vacuumFlags & PROC_IN_LOGICAL_DECODING) { + continue; + } + + /* cancel query when its read_lsn < recycle_lsn or its runtime > standby_max_query_time */ + if (XLByteLT(read_lsn, recycle_lsn) || exceed_standby_max_query_time(pg_proc->exrto_gen_snap_time)) { + pg_proc->recoveryConflictPending = true; + conflict = true; + if (pg_proc->pid != 0) { + /* + * Kill the pid if it's still here. If not, that's what we + * wanted so ignore any errors. + */ + (void)SendProcSignal(pg_proc->pid, PROCSIG_RECOVERY_CONFLICT_SNAPSHOT, pg_proc->backendId); + ereport(LOG, + (errmsg( + EXRTOFORMAT("read_lsn is less than recycle_lsn or query time exceed max_query_time while " + "get_oldeset_readlsn, read_lsn %lu, " + "recycle_lsn: %lu, exrto_gen_snap_time: %ld, current_time: %ld, thread id = %lu\n"), + read_lsn, + recycle_lsn, + pg_proc->exrto_gen_snap_time, + GetCurrentTimestamp(), + pg_proc->pid))); + /* + * Wait a little bit for it to die so that we avoid flooding + * an unresponsive backend when system is heavily loaded. + */ + pg_usleep(5000L); + } + continue; + } + + if (XLogRecPtrIsInvalid(oldest_lsn) || (XLogRecPtrIsValid(read_lsn) && XLByteLT(read_lsn, oldest_lsn))) { + oldest_lsn = read_lsn; + } + + if (!TransactionIdIsValid(oldest_xmin) || + (TransactionIdIsValid(pxmin) && TransactionIdFollows(oldest_xmin, pxmin))) { + oldest_xmin = pxmin; + } + } + LWLockRelease(ProcArrayLock); +} + +void proc_array_get_oldeset_xmin_for_undo(TransactionId &oldest_xmin) +{ + ProcArrayStruct *proc_array = g_instance.proc_array_idx; + + (void)LWLockAcquire(ProcArrayLock, LW_SHARED); + for (int index = 0; index < proc_array->numProcs; index++) { + int pg_proc_no = proc_array->pgprocnos[index]; + PGPROC *pg_proc = g_instance.proc_base_all_procs[pg_proc_no]; + PGXACT *pg_xact = &g_instance.proc_base_all_xacts[pg_proc_no]; + TransactionId pxmin = pg_xact->xmin; + + if (pg_proc->pid == 0 || !TransactionIdIsValid(pxmin)) { + continue; + } + + Assert(!(pg_xact->vacuumFlags & PROC_IN_VACUUM)); + /* + * Backend is doing logical decoding which manages xmin + * separately, check below. + */ + if (pg_xact->vacuumFlags & PROC_IN_LOGICAL_DECODING) { + continue; + } + if (!TransactionIdIsValid(oldest_xmin) || + (TransactionIdIsValid(pxmin) && TransactionIdFollows(oldest_xmin, pxmin))) { + oldest_xmin = pxmin; + } + } + LWLockRelease(ProcArrayLock); +} + +XLogRecPtr exrto_calculate_recycle_position(bool force_recyle) +{ + Assert(t_thrd.role != PAGEREDO); + Assert(IS_EXRTO_READ); + + XLogRecPtr recycle_lsn = pg_atomic_read_u64(&g_instance.comm_cxt.predo_cxt.global_recycle_lsn); + XLogRecPtr oldest_lsn = InvalidXLogRecPtr; + TransactionId oldest_xmin = InvalidTransactionId; + bool conflict = false; + const int max_check_times = 1000; + int check_times = 0; + + if (force_recyle) { + calculate_force_recycle_lsn(recycle_lsn); + } + ereport(DEBUG1, + (errmsg(EXRTOFORMAT("time information of calculate recycle position, current_time: %ld, snapshot " + "read_lsn: %08X/%08X, gen_snaptime:%ld"), + GetCurrentTimestamp(), + (uint32)(g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn >> UINT64_HALF), + (uint32)g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn, + g_instance.comm_cxt.predo_cxt.exrto_snapshot->gen_snap_time))); + + /* + * If there is no backend read threads, set read oldest lsn to snapshot lsn. + */ + ExrtoSnapshot exrto_snapshot = NULL; + exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; + (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); + if (XLByteEQ(exrto_snapshot->read_lsn, 0)) { + ereport(WARNING, (errmsg("could not get a valid snapshot with extreme rto"))); + } else { + oldest_lsn = exrto_snapshot->read_lsn; + oldest_xmin = exrto_snapshot->xmin; + } + LWLockRelease(ExrtoSnapshotLock); + /* Loop checks to avoid conflicting queries that were not successfully canceled. */ + do { + RedoInterruptCallBack(); + proc_array_get_oldeset_readlsn(recycle_lsn, oldest_lsn, oldest_xmin, conflict); + check_times++; + } while (conflict && check_times < max_check_times); + + recycle_lsn = rtl::max(recycle_lsn, oldest_lsn); + + ereport(LOG, + (errmsg( + EXRTOFORMAT( + "[exrto_recycle] calculate recycle position, oldestlsn: %08X/%08X, snapshot read_lsn: %08X/%08X, try " + "recycle lsn: %08X/%08X, xmin: %lu"), + (uint32)(oldest_lsn >> UINT64_HALF), + (uint32)oldest_lsn, + (uint32)(g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn >> UINT64_HALF), + (uint32)g_instance.comm_cxt.predo_cxt.exrto_snapshot->read_lsn, + (uint32)(recycle_lsn >> UINT64_HALF), + (uint32)recycle_lsn, oldest_xmin))); + pg_atomic_write_u64(&g_instance.comm_cxt.predo_cxt.exrto_recyle_xmin, oldest_xmin); + return recycle_lsn; +} + +TransactionId exrto_calculate_recycle_xmin_for_undo() +{ + Assert(t_thrd.role != PAGEREDO); + Assert(IS_EXRTO_READ); + TransactionId oldest_xmin = InvalidTransactionId; + TransactionId snapshot_xmin = InvalidTransactionId; + proc_array_get_oldeset_xmin_for_undo(oldest_xmin); + + /* + * If there is no backend read threads, set read oldest lsn to snapshot lsn. + */ + if (extreme_rto::g_dispatcher != NULL) { + ExrtoSnapshot exrto_snapshot = NULL; + exrto_snapshot = g_instance.comm_cxt.predo_cxt.exrto_snapshot; + (void)LWLockAcquire(ExrtoSnapshotLock, LW_SHARED); + if (XLByteEQ(exrto_snapshot->xmin, InvalidTransactionId)) { + ereport(WARNING, + (errmsg("exrto_calculate_recycle_xmin_for_undo: could not get a valid snapshot in exrto_snapshot"))); + } else { + snapshot_xmin = exrto_snapshot->xmin; + } + + LWLockRelease(ExrtoSnapshotLock); + } + ereport(DEBUG1, + (errmodule(MOD_UNDO), + errmsg(UNDOFORMAT("exrto_calculate_recycle_xmin_for_undo: oldest_xmin: %lu, snapshot_xmin: %lu."), + oldest_xmin, + snapshot_xmin))); + + if (oldest_xmin == InvalidTransactionId) { + return snapshot_xmin; + } + + if (TransactionIdPrecedes(snapshot_xmin, oldest_xmin)) { + return snapshot_xmin; + } + return oldest_xmin; +} diff --git a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp index b10e835362..5db1f6b877 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp @@ -554,6 +554,7 @@ static void StartPageRedoWorkers(uint32 totalThrdNum) g_dispatcher->allWorkers = (PageRedoWorker **)palloc0(sizeof(PageRedoWorker *) * totalThrdNum); g_dispatcher->allWorkersCnt = totalThrdNum; g_dispatcher->pageLines = (PageRedoPipeline *)palloc(sizeof(PageRedoPipeline) * batchNum); + g_dispatcher->next_lsn_info_page = 0; for (started = 0; started < totalThrdNum; started++) { g_dispatcher->allWorkers[started] = CreateWorker(started); @@ -580,6 +581,7 @@ static void StartPageRedoWorkers(uint32 totalThrdNum) // start from 1 not 0 g_dispatcher->pageLines[i].redoThd[j]->standby_read_meta_info.batch_id = i + 1; g_dispatcher->pageLines[i].redoThd[j]->standby_read_meta_info.redo_id = j + 1; + SpinLockInit(&(g_dispatcher->pageLines[i].redoThd[j]->standby_read_meta_info.mutex)); } g_dispatcher->pageLines[i].redoThdNum = batchWorkerPerMng; } diff --git a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp index a7d1389eb2..36308579bc 100644 --- a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp @@ -138,23 +138,36 @@ void do_standby_read_recyle(XLogRecPtr recycle_lsn) uint32 worker_nums = g_dispatcher->allWorkersCnt; PageRedoWorker** workers = g_dispatcher->allWorkers; XLogRecPtr min_recycle_lsn = InvalidXLogRecPtr; + XLogRecPtr before_min_lsn_position = UINT64_MAX; + XLogRecPtr after_min_lsn_position = UINT64_MAX; for (uint32 i = 0; i < worker_nums; ++i) { PageRedoWorker* page_redo_worker = workers[i]; if (page_redo_worker->role != REDO_PAGE_WORKER || (page_redo_worker->isUndoSpaceWorker)) { continue; } + + uint64 before_lsn_position = page_redo_worker->standby_read_meta_info.lsn_table_recyle_position; + before_min_lsn_position = before_min_lsn_position < before_lsn_position ? before_min_lsn_position : before_lsn_position; extreme_rto_standby_read::standby_read_recyle_per_workers(&page_redo_worker->standby_read_meta_info, recycle_lsn); + uint64 after_lsn_position = page_redo_worker->standby_read_meta_info.lsn_table_recyle_position; + after_min_lsn_position = after_min_lsn_position < after_lsn_position ? after_min_lsn_position : after_lsn_position; + if (XLogRecPtrIsInvalid(min_recycle_lsn) || XLByteLT(page_redo_worker->standby_read_meta_info.recycle_lsn_per_worker, min_recycle_lsn)) { min_recycle_lsn = page_redo_worker->standby_read_meta_info.recycle_lsn_per_worker; } pg_usleep(1000); // sleep 1ms } - if (XLByteLT(g_instance.comm_cxt.predo_cxt.global_recycle_lsn, min_recycle_lsn)) { - pg_atomic_write_u64(&g_instance.comm_cxt.predo_cxt.global_recycle_lsn, min_recycle_lsn); + if (after_min_lsn_position / EXRTO_LSN_INFO_FILE_MAXSIZE > before_min_lsn_position / EXRTO_LSN_INFO_FILE_MAXSIZE) { + extreme_rto_standby_read::recycle_lsn_info_file(after_min_lsn_position); + } + + if (XLogRecPtrIsInvalid(min_recycle_lsn) && min_recycle_lsn > MAX_LSN_SIZE_PER_FORWARDER && + XLByteLT(g_instance.comm_cxt.predo_cxt.global_recycle_lsn, min_recycle_lsn - MAX_LSN_SIZE_PER_FORWARDER)) { + pg_atomic_write_u64(&g_instance.comm_cxt.predo_cxt.global_recycle_lsn, min_recycle_lsn - MAX_LSN_SIZE_PER_FORWARDER); ereport(LOG, (errmsg(EXRTOFORMAT("[exrto_recycle] update global recycle lsn: %08X/%08X"), - (uint32)(min_recycle_lsn >> UINT64_HALF), (uint32)min_recycle_lsn))); + (uint32)(min_recycle_lsn >> UINT64_HALF), (uint32)(min_recycle_lsn - MAX_LSN_SIZE_PER_FORWARDER)))); } delete_by_lsn(recycle_lsn); } diff --git a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp index 4b14847609..e337b018bf 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/page_redo.cpp @@ -1697,6 +1697,7 @@ void StartupSendFowarder(RedoItem *item) void SendLsnFowarder() { + t_thrd.xlog_cxt.last_forwarder_lsn = g_redoWorker->lastReplayedReadRecPtr; // update and read in the same thread, so no need atomic operation g_GlobalLsnForwarder.record.ReadRecPtr = g_redoWorker->lastReplayedReadRecPtr; g_GlobalLsnForwarder.record.EndRecPtr = g_redoWorker->lastReplayedEndRecPtr; @@ -1747,10 +1748,18 @@ void PushToWorkerLsn(bool force) inline bool send_lsn_forwarder_for_check_to_hot_standby(XLogRecPtr lsn) { + if (!g_instance.attr.attr_storage.EnableHotStandby) { + return false; + } + if (t_thrd.xlog_cxt.reachedConsistency) { - // means has send lsn forwarder for consistenstcy check + if (lsn - t_thrd.xlog_cxt.last_forwarder_lsn > MAX_LSN_SIZE_PER_FORWARDER) { + return true; + } + /* means has send lsn forwarder for consistenstcy check */ return false; } + if (XLogRecPtrIsInvalid(t_thrd.xlog_cxt.minRecoveryPoint)) { return false; } @@ -1934,6 +1943,14 @@ void DispatchCleanInvalidPageMarkToAllRedoWorker(RepairFileKey key) } } +void wait_trxn_worker_queue_empty_or_exceed(XLogRecPtr end_lsn) +{ + PageRedoWorker *worker = g_dispatcher->trxnLine.redoThd; + while (!SPSCBlockingQueueIsEmpty(worker->queue) && worker->lastReplayedEndRecPtr < end_lsn) { + RedoInterruptCallBack(); + } +} + void DispatchClosefdMarkToAllRedoWorker() { for (uint32 i = 0; i < g_dispatcher->allWorkersCnt; i++) { diff --git a/src/gausskernel/storage/access/transam/xact.cpp b/src/gausskernel/storage/access/transam/xact.cpp index c001299d35..314cffe267 100755 --- a/src/gausskernel/storage/access/transam/xact.cpp +++ b/src/gausskernel/storage/access/transam/xact.cpp @@ -7225,7 +7225,11 @@ void unlink_relfiles(_in_ ColFileNode *xnodes, _in_ int nrels, bool is_old_delay */ if (!is_old_delay_ddl && RecoveryInProgress() && IS_EXRTO_READ) { RelFileNode block_meta_file = relFileNode; - block_meta_file.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; + if (IsSegmentFileNode(block_meta_file)) { + block_meta_file.bucketNode -= EXRTO_STANDBY_READ_BUCKET_OFFSET; + } else { + block_meta_file.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; + } extreme_rto_standby_read::remove_one_block_info_file(block_meta_file); } diff --git a/src/gausskernel/storage/access/transam/xlogutils.cpp b/src/gausskernel/storage/access/transam/xlogutils.cpp index aac7634ebf..04c1ea6478 100644 --- a/src/gausskernel/storage/access/transam/xlogutils.cpp +++ b/src/gausskernel/storage/access/transam/xlogutils.cpp @@ -1396,6 +1396,12 @@ void XlogDropRowReation(RelFileNode rnode) RelFileNodeBackend standbyReadRnode; standbyReadRnode.node = rnode; standbyReadRnode.node.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; + if (IsSegmentFileNode(rnode)) { + standbyReadRnode.node.spcNode = EXRTO_BLOCK_INFO_SPACE_OID; + } else { + standbyReadRnode.node.bucketNode -= EXRTO_STANDBY_READ_BUCKET_OFFSET; + } + standbyReadRnode.backend = InvalidBackendId; smgrclosenode(standbyReadRnode); } diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index c6a4c2c2f3..172a310836 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -2542,14 +2542,13 @@ Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber fork ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); isExtend = (blockNum == P_NEW); - - if (!ENABLE_DMS && IsSegmentFileNode(smgr->smgr_rnode.node) && RecoveryInProgress() && - !t_thrd.xlog_cxt.InRecovery) { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("bucket and undo segment standby read is not yet supported."))); + if (!ENABLE_DMS && IsSegmentFileNode(smgr->smgr_rnode.node) && RecoveryInProgress() && !t_thrd.xlog_cxt.InRecovery) { + if (IsBucketFileNode(smgr->smgr_rnode.node) || IS_UNDO_RELFILENODE(smgr->smgr_rnode.node)) { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("bucket and undo segment standby read is not yet supported."))); + } } - TRACE_POSTGRESQL_BUFFER_READ_START(forkNum, blockNum, smgr->smgr_rnode.node.spcNode, smgr->smgr_rnode.node.dbNode, smgr->smgr_rnode.node.relNode, smgr->smgr_rnode.backend, isExtend); @@ -6119,8 +6118,9 @@ void MarkBufferDirtyHint(Buffer buffer, bool buffer_std) uint64 old_buf_state; buf_state = pg_atomic_read_u64(&buf_desc->state); // temp buf just for old page version, could not write to disk - if (IS_EXRTO_READ && (buf_state & BM_IS_TMP_BUF)) { - return; + if (BUCKET_NODE_IS_EXRTO_READ(buf_desc->tag.rnode.bucketNode) && + buf_desc->tag.rnode.opt <= EXRTO_READ_STANDBY_INIT_LSN_OPT) { + return; } /* diff --git a/src/gausskernel/storage/smgr/segment/segbuffer.cpp b/src/gausskernel/storage/smgr/segment/segbuffer.cpp index b1a7d6880b..45a9562834 100644 --- a/src/gausskernel/storage/smgr/segment/segbuffer.cpp +++ b/src/gausskernel/storage/smgr/segment/segbuffer.cpp @@ -34,6 +34,7 @@ #include "pgstat.h" #include "ddes/dms/ss_dms_bufmgr.h" #include "replication/ss_disaster_cluster.h" +#include "access/multi_redo_api.h" /* * Segment buffer, used for segment meta data, e.g., segment head, space map head. We separate segment * meta data buffer and normal data buffer (in bufmgr.cpp) to avoid potential dead locks. @@ -577,14 +578,14 @@ Buffer ReadSegBufferForDMS(BufferDesc* bufHdr, ReadBufferMode mode, SegSpace *sp return BufferDescriptorGetBuffer(bufHdr); } -Buffer ReadBufferFast(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode) +Buffer ReadBufferFastNormal(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode) { bool found = false; /* Make sure we will have room to remember the buffer pin */ ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); - BufferDesc *bufHdr = SegBufferAlloc(spc, rnode, forkNum, blockNum, &found); + BufferDesc *bufHdr = SegBufferAlloc(rnode, forkNum, blockNum, &found); if (!found) { SegmentCheck(!(pg_atomic_read_u64(&bufHdr->state) & BM_VALID)); @@ -680,7 +681,16 @@ found_branch: return BufferDescriptorGetBuffer(bufHdr); } -BufferDesc * FoundBufferInHashTable(int buf_id, LWLock *new_partition_lock, bool *foundPtr) +Buffer ReadBufferFast(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode) +{ + if (!RecoveryInProgress() || !IS_EXRTO_STANDBY_READ || !is_exrto_standby_read_worker()) { + return ReadBufferFastNormal(spc, rnode, forkNum, blockNum, mode); + } else { + return standby_read_seg_buffer(spc, rnode, forkNum, blockNum, mode); + } +} + +BufferDesc *FoundBufferInHashTable(int buf_id, LWLock *new_partition_lock, bool *foundPtr) { BufferDesc *buf = GetBufferDescriptor(buf_id); bool valid = SegPinBuffer(buf); @@ -696,7 +706,24 @@ BufferDesc * FoundBufferInHashTable(int buf_id, LWLock *new_partition_lock, bool return buf; } -BufferDesc *SegBufferAlloc(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, +BufferDesc *SegBufferFind(RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, + bool *foundPtr) +{ + BufferTag new_tag; + INIT_BUFFERTAG(new_tag, rnode, forkNum, blockNum); + uint32 new_hash = BufTableHashCode(&new_tag); + LWLock *new_partition_lock = BufMappingPartitionLock(new_hash); + LWLockAcquire(new_partition_lock, LW_SHARED); + int buf_id = BufTableLookup(&new_tag, new_hash); + if (buf_id >= 0) { + return FoundBufferInHashTable(buf_id, new_partition_lock, foundPtr); + } + *foundPtr = false; + LWLockRelease(new_partition_lock); + return NULL; +} + +BufferDesc *SegBufferAlloc(RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, bool *foundPtr) { BufferDesc *buf; diff --git a/src/gausskernel/storage/smgr/segment/segxlog.cpp b/src/gausskernel/storage/smgr/segment/segxlog.cpp index c44bcaed59..d70c7bcf56 100644 --- a/src/gausskernel/storage/smgr/segment/segxlog.cpp +++ b/src/gausskernel/storage/smgr/segment/segxlog.cpp @@ -781,10 +781,8 @@ static void redo_space_drop(XLogReaderState *record) XLogDropSegmentSpace(spcNode, dbNode); } -void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *data, XLogRecPtr lsn) +void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *page, char *data, XLogRecPtr lsn) { - char page[BLCKSZ] __attribute__((__aligned__(ALIGNOF_BUFFER))) = {0}; - errno_t er = memcpy_s(page, BLCKSZ, data, BLCKSZ); securec_check(er, "\0", "\0"); @@ -816,7 +814,6 @@ void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *data, XLogRecPtr lsn } } - /* * This xlog only copy data to the new block, without modifying data in buffer. If the logic block being in the * buffer pool, its pblk points to the old block. The buffer descriptor can not have the logic blocknumber and the new @@ -827,7 +824,8 @@ static void redo_new_page(XLogReaderState *record) { Assert(record != NULL); BufferTag *tag = (BufferTag *)XLogRecGetData(record); - seg_redo_new_page_copy_and_flush(tag, (char *)XLogRecGetData(record) + sizeof(BufferTag), record->EndRecPtr); + char page[BLCKSZ] __attribute__((__aligned__(ALIGNOF_BUFFER))) = {0}; + seg_redo_new_page_copy_and_flush(tag, page, (char *)XLogRecGetData(record) + sizeof(BufferTag), record->EndRecPtr); } void segpage_smgr_redo(XLogReaderState *record) diff --git a/src/gausskernel/storage/smgr/segstore.cpp b/src/gausskernel/storage/smgr/segstore.cpp index 7428af20d3..a19bc58da3 100755 --- a/src/gausskernel/storage/smgr/segstore.cpp +++ b/src/gausskernel/storage/smgr/segstore.cpp @@ -1934,6 +1934,7 @@ int32 seg_physical_aio_prep_pwrite(SegSpace *spc, RelFileNode &rNode, ForkNumber return spc_aio_prep_pwrite(spc, rNode, forknum, blocknum, buffer, iocb_ptr, tempAioExtra); } + static bool check_meta_data(BlockNumber extent, uint32 extent_size, uint32* offset_block) { if (extent < DF_MAP_HEAD_PAGE + 1 || extent_size == EXTENT_1) { diff --git a/src/gausskernel/storage/smgr/smgr.cpp b/src/gausskernel/storage/smgr/smgr.cpp index eb856a1795..955f3dc58b 100755 --- a/src/gausskernel/storage/smgr/smgr.cpp +++ b/src/gausskernel/storage/smgr/smgr.cpp @@ -441,7 +441,8 @@ void smgrclose(SMgrRelation reln, BlockNumber blockNum) int forknum; int max_forknum; - if (reln->smgr_which == EXRTO_MANAGER && reln->smgr_rnode.node.spcNode == EXRTO_BLOCK_INFO_SPACE_OID) { + if (reln->smgr_which == EXRTO_MANAGER && (reln->smgr_rnode.node.spcNode == EXRTO_BLOCK_INFO_SPACE_OID || + is_standby_read_seg_relnode(reln->smgr_rnode.node))) { max_forknum = EXRTO_FORK_NUM; } else { max_forknum = reln->md_fdarray_size; @@ -607,7 +608,8 @@ void smgrdounlink(SMgrRelation reln, bool isRedo, BlockNumber blockNum) DelFileTag *entry = NULL; bool found = false; - if (which == EXRTO_MANAGER && reln->smgr_rnode.node.spcNode == EXRTO_BLOCK_INFO_SPACE_OID) { + if (which == EXRTO_MANAGER && (reln->smgr_rnode.node.spcNode == EXRTO_BLOCK_INFO_SPACE_OID || + is_standby_read_seg_relnode(reln->smgr_rnode.node))) { max_forknum = EXRTO_FORK_NUM; } else { max_forknum = reln->md_fdarray_size; diff --git a/src/gausskernel/storage/smgr/storage_exrto_file.cpp b/src/gausskernel/storage/smgr/storage_exrto_file.cpp index 20f433d8a2..437feb6192 100644 --- a/src/gausskernel/storage/smgr/storage_exrto_file.cpp +++ b/src/gausskernel/storage/smgr/storage_exrto_file.cpp @@ -83,8 +83,14 @@ static void exrto_get_file_path(const RelFileNode node, ForkNumber forknum, uint rc = snprintf_s(filename, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%02x%02x%016lX", batch_id, worker_id, segno); } else { - rc = snprintf_s(filename, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%u_%u_%s.%u", - node.dbNode, node.relNode, forkNames[forknum], (uint32)segno); + if (is_standby_read_seg_relnode(node)) { + uint32 bucketid = node.bucketNode + EXRTO_STANDBY_READ_BUCKET_OFFSET; + rc = snprintf_s(filename, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%u_%u_%u_%d_%s.%u", + node.spcNode, node.dbNode, node.relNode, bucketid, forkNames[forknum], (uint32)segno); + } else { + rc = snprintf_s(filename, EXRTO_FILE_PATH_LEN, EXRTO_FILE_PATH_LEN - 1, "%u_%u_%s.%u", + node.dbNode, node.relNode, forkNames[forknum], (uint32)segno); + } } securec_check_ss(rc, "\0", "\0"); diff --git a/src/include/access/extreme_rto/dispatcher.h b/src/include/access/extreme_rto/dispatcher.h index 60b2d65b0b..b707c637ea 100644 --- a/src/include/access/extreme_rto/dispatcher.h +++ b/src/include/access/extreme_rto/dispatcher.h @@ -133,6 +133,7 @@ typedef struct { RecordBufferState rtoXlogBufState; PageRedoWorker **allWorkers; /* Array of page redo workers. */ uint32 allWorkersCnt; + pg_atomic_uint64 next_lsn_info_page; RedoItem *freeHead; /* Head of freed-item list. */ RedoItem *freeStateHead; RedoItem *allocatedRedoItem; diff --git a/src/include/access/extreme_rto/page_redo.h b/src/include/access/extreme_rto/page_redo.h index e05b6a606c..baac3cbe8f 100644 --- a/src/include/access/extreme_rto/page_redo.h +++ b/src/include/access/extreme_rto/page_redo.h @@ -46,6 +46,7 @@ static const uint32 PAGE_WORK_QUEUE_SIZE = 8192; static const uint32 EXTREME_RTO_ALIGN_LEN = 16; /* need 128-bit aligned */ static const uint32 MAX_REMOTE_READ_INFO_NUM = 100; +static const uint32 MAX_LSN_SIZE_PER_FORWARDER = 1024 * 1024 * 10; typedef enum { REDO_BATCH, @@ -242,6 +243,7 @@ void WaitAllRedoWorkerQueueEmpty(); void WaitAllReplayWorkerIdle(); void DispatchClosefdMarkToAllRedoWorker(); void DispatchCleanInvalidPageMarkToAllRedoWorker(RepairFileKey key); +void wait_trxn_worker_queue_empty_or_exceed(XLogRecPtr end_lsn); const char *RedoWokerRole2Str(RedoRole role); uint32 GetWorkerId(const RedoItemTag *redo_item_tag, uint32 worker_count); diff --git a/src/include/access/extreme_rto/standby_read/block_info_meta.h b/src/include/access/extreme_rto/standby_read/block_info_meta.h index 9a104db693..11935d1d61 100644 --- a/src/include/access/extreme_rto/standby_read/block_info_meta.h +++ b/src/include/access/extreme_rto/standby_read/block_info_meta.h @@ -84,11 +84,10 @@ void insert_lsn_to_block_info( void insert_lsn_to_block_info_for_opt( StandbyReadMetaInfo *mete_info, const BufferTag &buf_tag, const Page base_page, XLogRecPtr next_lsn); -StandbyReadRecyleState recyle_block_info(const BufferTag &buf_tag, LsnInfoPosition base_page_info_pos, - XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, - XLogRecPtr *block_info_max_lsn); -bool get_page_lsn_info(const BufferTag& buf_tag, BufferAccessStrategy strategy, XLogRecPtr read_lsn, - StandbyReadLsnInfoArray* lsn_info); +StandbyReadRecyleState recycle_block_info(StandbyReadMetaInfo *meta_info, const BufferTag &buf_tag, const BufferTag &seg_buf_tag, + XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn); +bool get_page_lsn_info(const BufferTag& old_buf_tag, const BufferTag& buf_tag, BufferAccessStrategy strategy, + XLogRecPtr read_lsn, StandbyReadLsnInfoArray* lsn_info); static inline bool is_block_info_page_valid(BlockInfoPageHeader* header) { return ((header->flags & BLOCK_INFO_PAGE_VALID_FLAG) == BLOCK_INFO_PAGE_VALID_FLAG); diff --git a/src/include/access/extreme_rto/standby_read/lsn_info_meta.h b/src/include/access/extreme_rto/standby_read/lsn_info_meta.h index 8e641b5ab3..90edd66d84 100644 --- a/src/include/access/extreme_rto/standby_read/lsn_info_meta.h +++ b/src/include/access/extreme_rto/standby_read/lsn_info_meta.h @@ -137,13 +137,14 @@ void get_lsn_info_for_read(const BufferTag& buf_tag, LsnInfoPosition latest_lsn_ StandbyReadLsnInfoArray* lsn_info_list, XLogRecPtr read_lsn); Buffer buffer_read_base_page(uint32 batch_id, uint32 redo_id, BasePagePosition position, ReadBufferMode mode); -void generate_base_page(StandbyReadMetaInfo* meta_info, const Page src_page); +void generate_base_page(StandbyReadMetaInfo* meta_info, const Page src_page, BasePagePosition base_page_pos); void read_base_page(const BufferTag& buf_tag, BasePagePosition position, BufferDesc* dest_buf_desc); void recycle_base_page_file(uint32 batch_id, uint32 redo_id, BasePagePosition recycle_pos); void set_base_page_map_bit(Page page, uint32 base_page_loc); bool is_base_page_map_bit_set(Page page, uint32 which_bit); -void recycle_one_lsn_info_list(const BufferTag& buf_tag, LsnInfoPosition page_info_pos, +void recycle_lsn_info_file(BasePagePosition recycle_pos); +void recycle_one_lsn_info_list(const StandbyReadMetaInfo *meta_info, LsnInfoPosition page_info_pos, XLogRecPtr recycle_lsn, LsnInfoPosition *min_page_info_pos, XLogRecPtr *min_lsn); void standby_read_recyle_per_workers(StandbyReadMetaInfo *standby_read_meta_info, XLogRecPtr recycle_lsn); LsnInfoPosition get_nearest_base_page_pos( diff --git a/src/include/access/extreme_rto/standby_read/standby_read_base.h b/src/include/access/extreme_rto/standby_read/standby_read_base.h index efea1de98b..523332434c 100644 --- a/src/include/access/extreme_rto/standby_read/standby_read_base.h +++ b/src/include/access/extreme_rto/standby_read/standby_read_base.h @@ -66,6 +66,8 @@ typedef struct _StandbyReadMetaInfo { BasePagePosition base_page_recyle_position; BasePagePosition base_page_next_position; // next position can insert page XLogRecPtr recycle_lsn_per_worker; + bool is_lsn_table_recyle_position_init; + slock_t mutex; } StandbyReadMetaInfo; typedef struct WalFilter { diff --git a/src/include/access/xlogproc.h b/src/include/access/xlogproc.h index 0b1c3a4ed7..527058fa7a 100755 --- a/src/include/access/xlogproc.h +++ b/src/include/access/xlogproc.h @@ -1302,7 +1302,7 @@ extern void GistRedoDataBlock(XLogBlockHead *blockhead, XLogBlockDataParse *bloc extern bool IsCheckPoint(const XLogRecParseState *parseState); bool is_backup_end(const XLogRecParseState *parse_state); void redo_atomic_xlog_dispatch(uint8 opCode, RedoBufferInfo *redo_buf, const char *data); -void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *data, XLogRecPtr lsn); +void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *page, char *data, XLogRecPtr lsn); void redo_target_page(const BufferTag& buf_tag, StandbyReadLsnInfoArray* lsn_info, Buffer base_page_buf); void MarkSegPageRedoChildPageDirty(RedoBufferInfo *bufferinfo); diff --git a/src/include/knl/knl_thread.h b/src/include/knl/knl_thread.h index 362361f33c..3441893910 100755 --- a/src/include/knl/knl_thread.h +++ b/src/include/knl/knl_thread.h @@ -802,6 +802,7 @@ typedef struct knl_t_xlog_context { UwalInfo uwalInfo; List* uwalInfoHis; XLogRecPtr uwalFileRenamePtr; + XLogRecPtr last_forwarder_lsn; } knl_t_xlog_context; typedef struct knl_t_dfs_context { diff --git a/src/include/postgres_ext.h b/src/include/postgres_ext.h index d1d3f3cf53..13cdb7ba80 100644 --- a/src/include/postgres_ext.h +++ b/src/include/postgres_ext.h @@ -40,8 +40,20 @@ typedef unsigned int Oid; #define InvalidBktId (-1) #define SegmentBktId (16384) +#define EXRTO_READ_SPECIAL_LSN (-7) +#define MAX_SEGMENT_BUCKETID (2000) +#define MIN_SEGMENT_BUCKETID (0) +#define EXRTO_STANDBY_READ_BUCKET_OFFSET (2200) +#define MIN_EXRTO_STANDBY_READ_BUCKETID (-(EXRTO_STANDBY_READ_BUCKET_OFFSET)) +#define MAX_EXRTO_STANDBY_READ_BUCKETID (MAX_SEGMENT_BUCKETID - EXRTO_STANDBY_READ_BUCKET_OFFSET) +#define EXRTO_READ_STANDBY_START_LSN_OPT (1) +#define EXRTO_READ_STANDBY_END_LSN_OPT (2) +#define EXRTO_READ_STANDBY_INIT_LSN_OPT (3) #define BUCKET_NODE_IS_VALID(bucket_node) ((bucket_node) > InvalidBktId && (bucket_node) < SegmentBktId) #define BUCKET_OID_IS_VALID(bucketOid) ((bucketOid) >= FirstNormalObjectId) +#define BUCKET_NODE_IS_EXRTO_READ(bucket_node) \ + ((bucket_node) >= MIN_EXRTO_STANDBY_READ_BUCKETID && (bucket_node) <= MAX_EXRTO_STANDBY_READ_BUCKETID) || \ + (bucket_node == EXRTO_READ_SPECIAL_LSN) #define OID_MAX UINT_MAX diff --git a/src/include/storage/buf/bufmgr.h b/src/include/storage/buf/bufmgr.h index 2c0f6da631..0007c8d40a 100644 --- a/src/include/storage/buf/bufmgr.h +++ b/src/include/storage/buf/bufmgr.h @@ -435,12 +435,15 @@ extern long RedoDatabaseForOndemandExtremeRTO(Oid dbId); extern BufferDesc *RedoForOndemandExtremeRTOQuery(BufferDesc *bufHdr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); extern Buffer standby_read_buf(Relation reln, ForkNumber fork_num, BlockNumber block_num, ReadBufferMode mode, - BufferAccessStrategy strategy); + BufferAccessStrategy strategy); + typedef struct SMgrRelationData *SMgrRelation; BufferDesc *BufferAlloc(const RelFileNode &rel_file_node, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, BufferAccessStrategy strategy, bool *foundPtr, const XLogPhyBlock *pblk); Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, const XLogPhyBlock *pblk); +struct SegSpace; +extern Buffer standby_read_seg_buffer(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); Buffer MultiBulkReadBufferCommon(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber firstBlockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, int maxBulkCount, const XLogPhyBlock *pblk, int paramNum, char* bufRead); void buffer_in_progress_pop(); diff --git a/src/include/storage/smgr/segment.h b/src/include/storage/smgr/segment.h index 6457574a2a..3353bc6926 100644 --- a/src/include/storage/smgr/segment.h +++ b/src/include/storage/smgr/segment.h @@ -134,9 +134,11 @@ DecodedXLogBlockOp XLogAtomicDecodeBlockData(char *data, int len); /* * APIs used for segment store metadata. */ -BufferDesc *SegBufferAlloc(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, bool *foundPtr); +BufferDesc *SegBufferFind(RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, bool *foundPtr); +BufferDesc *SegBufferAlloc(RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, bool *foundPtr); Buffer ReadSegBufferForDMS(BufferDesc* bufHdr, ReadBufferMode mode, SegSpace *spc = NULL); void ReadSegBufferForCheck(BufferDesc* bufHdr, ReadBufferMode mode, SegSpace *spc, Block bufBlock); +Buffer ReadBufferFastNormal(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); Buffer ReadBufferFast(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); void SegReleaseBuffer(Buffer buffer); void SegUnlockReleaseBuffer(Buffer buffer); diff --git a/src/include/storage/smgr/smgr.h b/src/include/storage/smgr/smgr.h index 2b94e2cd0f..879e4de7d4 100644 --- a/src/include/storage/smgr/smgr.h +++ b/src/include/storage/smgr/smgr.h @@ -136,9 +136,23 @@ enum SMGR_READ_STATUS { #define EXRTO_MANAGER (3) #define IS_UNDO_RELFILENODE(rnode) ((rnode).dbNode == UNDO_DB_OID || (rnode).dbNode == UNDO_SLOT_DB_OID) -#define IS_EXRTO_RELFILENODE(rnode) ((rnode).spcNode == EXRTO_BASE_PAGE_SPACE_OID || \ - (rnode).spcNode == EXRTO_LSN_INFO_SPACE_OID || \ - (rnode).spcNode == EXRTO_BLOCK_INFO_SPACE_OID) +#define IS_EXRTO_RELFILENODE(rnode) ((rnode).spcNode == EXRTO_BASE_PAGE_SPACE_OID || \ + (rnode).spcNode == EXRTO_LSN_INFO_SPACE_OID || \ + (rnode).spcNode == EXRTO_BLOCK_INFO_SPACE_OID || \ + is_standby_read_seg_relnode(rnode)) + +inline bool is_standby_read_seg_relnode(const RelFileNode &rnode) +{ + return (rnode.bucketNode >= MIN_EXRTO_STANDBY_READ_BUCKETID && + rnode.bucketNode <= MAX_EXRTO_STANDBY_READ_BUCKETID); +} + +inline bool is_segment_logical_relnode(const RelFileNode &rnode) +{ + const int extent_8192 = 5; + return (IsSegmentFileNode(rnode) && rnode.bucketNode > extent_8192); +} + /* * On Windows, we have to interpret EACCES as possibly meaning the same as * ENOENT, because if a file is unlinked-but-not-yet-gone on that platform, -- Gitee From 2089aee437811d11d9dfab2bad63f6016f5ee8ef Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 17:28:05 +0800 Subject: [PATCH 02/31] 2 --- src/include/access/extreme_rto/standby_read/lsn_info_meta.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/include/access/extreme_rto/standby_read/lsn_info_meta.h b/src/include/access/extreme_rto/standby_read/lsn_info_meta.h index 90edd66d84..3eb0afe895 100644 --- a/src/include/access/extreme_rto/standby_read/lsn_info_meta.h +++ b/src/include/access/extreme_rto/standby_read/lsn_info_meta.h @@ -35,7 +35,7 @@ namespace extreme_rto_standby_read { const static uint32 BASE_PAGE_MAP_SIZE = 16; -const static uint32 LSN_INFO_PAGE_HEAD_PAD_SIZE = 32; +const static uint32 LSN_INFO_PAGE_HEAD_PAD_SIZE = 24; const static uint32 LSN_INFO_PAGE_VERSION = 1; /* currently the first version of extreme rto standby read */ const static uint32 LSN_NUM_PER_NODE = 5; const static uint32 BYTE_BITS = 8; @@ -46,6 +46,7 @@ typedef struct _LsnInfoPageHeader { uint16 flags; uint32 version; uint8 base_page_map[BASE_PAGE_MAP_SIZE]; + uint64 next_lsn_info_page; uint8 pad[LSN_INFO_PAGE_HEAD_PAD_SIZE]; } LsnInfoPageHeader; -- Gitee From ea1e838c64d33fd8c038ce9ed998a0e86b26af84 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 17:37:40 +0800 Subject: [PATCH 03/31] 444 --- src/gausskernel/storage/access/redo/redo_xlogutils.cpp | 8 ++++---- .../storage/access/redo/standby_read/lsn_info_proc.cpp | 4 ++-- .../access/extreme_rto/standby_read/standby_read_base.h | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp index 43d43ebc83..7d827beb0b 100644 --- a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp +++ b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp @@ -2036,13 +2036,13 @@ void init_redo_buffer_info(RedoBufferInfo *rb_info, const BufferTag &buf_tag, Bu bool redo_target_state(XLogRecParseState *state, RedoBufferInfo *buf_info) { - if (!find_target_state(state, buf_info.blockinfo)) { + if (!find_target_state(state, buf_info->blockinfo)) { return false; } - buf_info.lsn = state->blockparse.blockhead.end_ptr; - buf_info.blockinfo.pblk = state->blockparse.blockhead.pblk; - wal_block_redo_for_extreme_rto_read(state, &buf_info); + buf_info->lsn = state->blockparse.blockhead.end_ptr; + buf_info->blockinfo.pblk = state->blockparse.blockhead.pblk; + wal_block_redo_for_extreme_rto_read(state, buf_info); return true; } diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp index 6872cb8c9e..2e2221ddd0 100644 --- a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp @@ -197,7 +197,7 @@ void insert_lsn_to_lsn_info(StandbyReadMetaInfo *meta_info, LsnInfoDoubleList *l uint32 offset; Assert(!INFO_POSITION_IS_INVALID(tail_pos)); - page = get_lsn_info_page(batch_id, worker_id, tail_pos, RBM_ZERO_ON_ERROR, &buffer); + page = get_lsn_info_page(meta_info->batch_id, meta_info->redo_id, tail_pos, RBM_ZERO_ON_ERROR, &buffer); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); offset = lsn_info_postion_to_offset(tail_pos); @@ -713,7 +713,7 @@ void standby_read_recyle_per_workers(StandbyReadMetaInfo *meta_info, XLogRecPtr uint64 recyled_page_len = 0; const uint32 recyle_ratio = 32; // no need recyle so fast uint64 next_page_pos = meta_info->lsn_table_recyle_position; - if (!meta_info->is_lsn_table_recyle_position_init) { + if (!meta_info->is_lsn_table_recycle_position_init) { return; } while (meta_info->lsn_table_recyle_position + BLCKSZ * recyle_ratio < next_page_pos) { diff --git a/src/include/access/extreme_rto/standby_read/standby_read_base.h b/src/include/access/extreme_rto/standby_read/standby_read_base.h index 523332434c..7563fb113c 100644 --- a/src/include/access/extreme_rto/standby_read/standby_read_base.h +++ b/src/include/access/extreme_rto/standby_read/standby_read_base.h @@ -66,7 +66,7 @@ typedef struct _StandbyReadMetaInfo { BasePagePosition base_page_recyle_position; BasePagePosition base_page_next_position; // next position can insert page XLogRecPtr recycle_lsn_per_worker; - bool is_lsn_table_recyle_position_init; + bool is_lsn_table_recycle_position_init; slock_t mutex; } StandbyReadMetaInfo; -- Gitee From 17b0a51ae05852ab4d6fc57071f94dca2851a918 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 17:38:32 +0800 Subject: [PATCH 04/31] 555 --- .../access/redo/standby_read/block_info_proc.cpp | 2 +- .../access/redo/standby_read/lsn_info_proc.cpp | 16 ++++++++-------- .../redo/standby_read/standby_read_interface.cpp | 12 ++++++------ .../redo/standby_read/standby_read_proc.cpp | 4 ++-- .../access/transam/extreme_rto/exrto_recycle.cpp | 6 +++--- .../extreme_rto/standby_read/standby_read_base.h | 2 +- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp index 2b10344972..c800b5848b 100644 --- a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp @@ -201,7 +201,7 @@ void insert_lsn_to_block_info_for_opt( * files belongs to this block), we reset this block */ if (!is_block_meta_info_valid(block_info) || - block_info->lsn_info_list.prev < meta_info->lsn_table_recyle_position) { + block_info->lsn_info_list.prev < meta_info->lsn_table_recycle_position) { if (!is_block_info_page_valid((BlockInfoPageHeader *)page)) { block_info_page_init(page); } diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp index 2e2221ddd0..dfabd0004f 100644 --- a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp @@ -602,7 +602,7 @@ inline void update_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info, XLogRe meta_info->recycle_lsn_per_worker = base_page_lsn; } uint64 cur_base_page_recyle_segno = meta_info->base_page_recyle_position / EXRTO_BASE_PAGE_FILE_MAXSIZE; - uint64 cur_lsn_table_recyle_segno = meta_info->lsn_table_recyle_position / EXRTO_LSN_INFO_FILE_MAXSIZE; + uint64 cur_lsn_table_recyle_segno = meta_info->lsn_table_recycle_position / EXRTO_LSN_INFO_FILE_MAXSIZE; ereport(LOG, (errmsg(EXRTOFORMAT("[exrto_recycle] update recycle lsn per worker , batch_id: %u, redo_id: %u, recycle " "base_page_lsn: %08X/%08X, next_base_page_lsn: %08X/%08X, block_info_max_lsn: " @@ -620,7 +620,7 @@ bool recycle_one_lsn_info_page(StandbyReadMetaInfo *meta_info, XLogRecPtr recycl uint32 batch_id = meta_info->batch_id; uint32 worker_id = meta_info->redo_id; Buffer buffer = InvalidBuffer; - LsnInfoPosition recycle_pos = meta_info->lsn_table_recyle_position; + LsnInfoPosition recycle_pos = meta_info->lsn_table_recycle_position; Page page = get_lsn_info_page(batch_id, worker_id, recycle_pos, RBM_NORMAL, &buffer); if (unlikely(page == NULL || buffer == InvalidBuffer)) { ereport(PANIC, (errmsg(EXRTOFORMAT("get_lsn_info_page failed, batch_id: %u, redo_id: %u, pos: %lu"), batch_id, @@ -707,24 +707,24 @@ void standby_read_recyle_per_workers(StandbyReadMetaInfo *meta_info, XLogRecPtr bool recycle_next_page = true; BasePagePosition base_page_position = meta_info->base_page_recyle_position; uint64 last_base_page_recyle_segno = meta_info->base_page_recyle_position / EXRTO_BASE_PAGE_FILE_MAXSIZE; - uint64 last_lsn_table_recyle_segno = meta_info->lsn_table_recyle_position / EXRTO_LSN_INFO_FILE_MAXSIZE; + uint64 last_lsn_table_recyle_segno = meta_info->lsn_table_recycle_position / EXRTO_LSN_INFO_FILE_MAXSIZE; uint64 cur_base_page_recyle_segno, cur_lsn_table_recyle_segno; uint64 recyled_page_len = 0; const uint32 recyle_ratio = 32; // no need recyle so fast - uint64 next_page_pos = meta_info->lsn_table_recyle_position; + uint64 next_page_pos = meta_info->lsn_table_recycle_position; if (!meta_info->is_lsn_table_recycle_position_init) { return; } - while (meta_info->lsn_table_recyle_position + BLCKSZ * recyle_ratio < next_page_pos) { + while (meta_info->lsn_table_recycle_position + BLCKSZ * recyle_ratio < next_page_pos) { recycle_next_page = recycle_one_lsn_info_page(meta_info, recycle_lsn, &base_page_position, &next_page_pos); if (!recycle_next_page) { break; } /* update recycle position */ - meta_info->lsn_table_recyle_position = next_page_pos; + meta_info->lsn_table_recycle_position = next_page_pos; recyled_page_len += BLCKSZ; - Assert(meta_info->lsn_table_recyle_position % BLCKSZ == 0); + Assert(meta_info->lsn_table_recycle_position % BLCKSZ == 0); if (recyled_page_len >= EXRTO_LSN_INFO_FILE_MAXSIZE) { RedoInterruptCallBack(); pg_usleep(100); // sleep 0.1ms @@ -737,7 +737,7 @@ void standby_read_recyle_per_workers(StandbyReadMetaInfo *meta_info, XLogRecPtr Assert(meta_info->base_page_recyle_position <= meta_info->base_page_next_position); cur_base_page_recyle_segno = meta_info->base_page_recyle_position / EXRTO_BASE_PAGE_FILE_MAXSIZE; - cur_lsn_table_recyle_segno = meta_info->lsn_table_recyle_position / EXRTO_LSN_INFO_FILE_MAXSIZE; + cur_lsn_table_recyle_segno = meta_info->lsn_table_recycle_position / EXRTO_LSN_INFO_FILE_MAXSIZE; if (cur_base_page_recyle_segno > last_base_page_recyle_segno || cur_lsn_table_recyle_segno > last_lsn_table_recyle_segno) { buffer_drop_exrto_standby_read_buffers(meta_info); diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp index e8fe901bea..09bafa2167 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -473,7 +473,7 @@ bool check_need_drop_buffer(StandbyReadMetaInfo *meta_info, const BufferTag tag) uint64 total_block_num = get_total_block_num(type, tag.rnode.relNode, tag.blockNum); uint64 recycle_pos = ((type == BASE_PAGE) ? meta_info->base_page_recyle_position - : meta_info->lsn_table_recyle_position); + : meta_info->lsn_table_recycle_position); return (total_block_num < (recycle_pos / BLCKSZ)); } @@ -564,14 +564,14 @@ Datum gs_hot_standby_space_info(PG_FUNCTION_ARGS) StandbyReadMetaInfo meta_info = page_redo_worker->standby_read_meta_info; uint64 lsn_file_size_per_thread = 0; - if (meta_info.lsn_table_next_position > meta_info.lsn_table_recyle_position) { - lsn_file_size_per_thread = meta_info.lsn_table_next_position - meta_info.lsn_table_recyle_position; - /* in 0~lsn_table_recyle_position No data is stored, + if (meta_info.lsn_table_next_position > meta_info.lsn_table_recycle_position) { + lsn_file_size_per_thread = meta_info.lsn_table_next_position - meta_info.lsn_table_recycle_position; + /* in 0~lsn_table_recycle_position No data is stored, means the size of one lsn info file does not reach maxsize - eg:0~100KB(lsn_table_recyle_position), 100KB~(16M+100KB)(lsn_table_next_position), filenum:2, size:16M */ + eg:0~100KB(lsn_table_recycle_position), 100KB~(16M+100KB)(lsn_table_next_position), filenum:2, size:16M */ lsn_file_num += meta_info.lsn_table_next_position / EXRTO_LSN_INFO_FILE_MAXSIZE + ((meta_info.lsn_table_next_position % EXRTO_LSN_INFO_FILE_MAXSIZE) > 0 ? 1 : 0) - - (meta_info.lsn_table_recyle_position / EXRTO_LSN_INFO_FILE_MAXSIZE); + (meta_info.lsn_table_recycle_position / EXRTO_LSN_INFO_FILE_MAXSIZE); } lsn_file_size += lsn_file_size_per_thread; diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp index 13289e2b2e..6cb8c23be0 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp @@ -163,9 +163,9 @@ XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info } /* for lsn info */ - if (meta_info->lsn_table_recyle_position < meta_info->lsn_table_next_position) { + if (meta_info->lsn_table_recycle_position < meta_info->lsn_table_next_position) { lsn_info_recycle_pos = - get_force_recycle_pos(meta_info->lsn_table_recyle_position, meta_info->lsn_table_next_position); + get_force_recycle_pos(meta_info->lsn_table_recycle_position, meta_info->lsn_table_next_position); page = extreme_rto_standby_read::get_lsn_info_page( meta_info->batch_id, meta_info->redo_id, lsn_info_recycle_pos, RBM_NORMAL, &buffer); if (unlikely(page == NULL || buffer == InvalidBuffer)) { diff --git a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp index 36308579bc..e4ceeb929c 100644 --- a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp @@ -119,7 +119,7 @@ bool check_if_need_force_recycle() continue; } total_base_page_size += (meta_info.base_page_next_position - meta_info.base_page_recyle_position); - total_lsn_info_size += (meta_info.lsn_table_next_position - meta_info.lsn_table_recyle_position); + total_lsn_info_size += (meta_info.lsn_table_next_position - meta_info.lsn_table_recycle_position); } /* the unit of max_standby_base_page_size and max_standby_lsn_info_size is KB */ @@ -146,10 +146,10 @@ void do_standby_read_recyle(XLogRecPtr recycle_lsn) continue; } - uint64 before_lsn_position = page_redo_worker->standby_read_meta_info.lsn_table_recyle_position; + uint64 before_lsn_position = page_redo_worker->standby_read_meta_info.lsn_table_recycle_position; before_min_lsn_position = before_min_lsn_position < before_lsn_position ? before_min_lsn_position : before_lsn_position; extreme_rto_standby_read::standby_read_recyle_per_workers(&page_redo_worker->standby_read_meta_info, recycle_lsn); - uint64 after_lsn_position = page_redo_worker->standby_read_meta_info.lsn_table_recyle_position; + uint64 after_lsn_position = page_redo_worker->standby_read_meta_info.lsn_table_recycle_position; after_min_lsn_position = after_min_lsn_position < after_lsn_position ? after_min_lsn_position : after_lsn_position; if (XLogRecPtrIsInvalid(min_recycle_lsn) || diff --git a/src/include/access/extreme_rto/standby_read/standby_read_base.h b/src/include/access/extreme_rto/standby_read/standby_read_base.h index 7563fb113c..df1fa74832 100644 --- a/src/include/access/extreme_rto/standby_read/standby_read_base.h +++ b/src/include/access/extreme_rto/standby_read/standby_read_base.h @@ -61,7 +61,7 @@ typedef uint64 BasePagePosition; typedef struct _StandbyReadMetaInfo { uint32 batch_id; uint32 redo_id; - uint64 lsn_table_recyle_position; + uint64 lsn_table_recycle_position; uint64 lsn_table_next_position; // next position can insert node, shoud jump page header before use BasePagePosition base_page_recyle_position; BasePagePosition base_page_next_position; // next position can insert page -- Gitee From aac6e10e656725711c561ed3e0ac0d4d18abab52 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 17:41:28 +0800 Subject: [PATCH 05/31] 222 --- .../storage/access/redo/standby_read/standby_read_proc.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp index 6cb8c23be0..3204e24833 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp @@ -145,6 +145,7 @@ static inline uint64 get_force_recycle_pos(uint64 recycle_pos, uint64 insert_pos XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info) { + uint64 base_page_recycle_pos; uint64 lsn_info_recycle_pos; XLogRecPtr base_page_recycle_lsn = InvalidXLogRecPtr; XLogRecPtr lsn_info_recycle_lsn = InvalidXLogRecPtr; @@ -163,9 +164,9 @@ XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info } /* for lsn info */ - if (meta_info->lsn_table_recycle_position < meta_info->lsn_table_next_position) { + if (meta_info->lsn_table_recyle_position < meta_info->lsn_table_next_position) { lsn_info_recycle_pos = - get_force_recycle_pos(meta_info->lsn_table_recycle_position, meta_info->lsn_table_next_position); + get_force_recycle_pos(meta_info->lsn_table_recyle_position, meta_info->lsn_table_next_position); page = extreme_rto_standby_read::get_lsn_info_page( meta_info->batch_id, meta_info->redo_id, lsn_info_recycle_pos, RBM_NORMAL, &buffer); if (unlikely(page == NULL || buffer == InvalidBuffer)) { @@ -182,7 +183,7 @@ XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info UnlockReleaseBuffer(buffer); } - return lsn_info_recycle_lsn; + return rtl::max(base_page_recycle_lsn, lsn_info_recycle_lsn); } void calculate_force_recycle_lsn(XLogRecPtr &recycle_lsn) -- Gitee From 0cd4f6cc6ff9faa8e5006b4b02bdc87bc456ef9b Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 17:46:11 +0800 Subject: [PATCH 06/31] 555 --- src/gausskernel/storage/access/redo/redo_xlogutils.cpp | 8 +++++--- .../storage/access/redo/standby_read/lsn_info_proc.cpp | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp index 7d827beb0b..12204afa94 100644 --- a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp +++ b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp @@ -2058,7 +2058,7 @@ bool redo_target_seg_state(XLogRecParseState *state, RedoBufferInfo *buf_info) case XLOG_SEG_ADD_NEW_GROUP: case XLOG_SEG_TRUNCATE: case XLOG_SEG_SPACE_SHRINK: - case XLOG_SEG_NEW_PAGE: + case XLOG_SEG_NEW_PAGE: { XLogRecParseState* child_state_list = (XLogRecParseState*)state->blockparse.extra_rec.blocksegfullsyncrec.childState; XLogRecParseState* next_state = child_state_list; while (next_state != NULL) { @@ -2069,6 +2069,8 @@ bool redo_target_seg_state(XLogRecParseState *state, RedoBufferInfo *buf_info) } XLogBlockParseStateRelease(state); break; + } + default: return redo_target_state(state, buf_info); } @@ -2115,9 +2117,9 @@ void redo_target_page(const BufferTag &buf_tag, StandbyReadLsnInfoArray *lsn_inf bool redo_done = false; while(!redo_done && state_iter != NULL) { if ((XLogBlockHeadGetRmid(&state_iter->blockparse.blockhead) == RM_SEGPAGE_ID)) { - redo_done = redo_target_seg_state(state_iter, buf_info); + redo_done = redo_target_seg_state(state_iter, &buf_info); } else { - redo_done = redo_target_state(state_iter, buf_info); + redo_done = redo_target_state(state_iter, &buf_info); } state_iter = (XLogRecParseState *)(state_iter->nextrecord); } diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp index dfabd0004f..daa407cd14 100644 --- a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp @@ -694,7 +694,7 @@ bool recycle_one_lsn_info_page(StandbyReadMetaInfo *meta_info, XLogRecPtr recycl } LsnInfoPageHeader *page_header = (LsnInfoPageHeader *)page; Assert(page_header != NULL); - *next_page_pos = page_header->next_lsn_info_pos; + *next_page_pos = page_header->next_lsn_info_page; LockBuffer(buffer, BUFFER_LOCK_UNLOCK); ReleaseBuffer(buffer); return true; -- Gitee From 166bacc616fd1d8a3d57dd5645f1d152d1f53275 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 17:47:22 +0800 Subject: [PATCH 07/31] 111111 --- .../storage/access/redo/standby_read/standby_read_proc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp index 3204e24833..46a96f4ea2 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp @@ -164,9 +164,9 @@ XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info } /* for lsn info */ - if (meta_info->lsn_table_recyle_position < meta_info->lsn_table_next_position) { + if (meta_info->lsn_table_recycle_position < meta_info->lsn_table_next_position) { lsn_info_recycle_pos = - get_force_recycle_pos(meta_info->lsn_table_recyle_position, meta_info->lsn_table_next_position); + get_force_recycle_pos(meta_info->lsn_table_recycle_position, meta_info->lsn_table_next_position); page = extreme_rto_standby_read::get_lsn_info_page( meta_info->batch_id, meta_info->redo_id, lsn_info_recycle_pos, RBM_NORMAL, &buffer); if (unlikely(page == NULL || buffer == InvalidBuffer)) { -- Gitee From 5155bc70db4268dda22bbdb0dab08ad21f01896c Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 17:52:34 +0800 Subject: [PATCH 08/31] 8888 --- .../storage/access/redo/standby_read/block_info_proc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp index c800b5848b..2f7738d76a 100644 --- a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp @@ -225,7 +225,7 @@ void insert_lsn_to_block_info_for_opt( UnlockReleaseBuffer(block_info_buf); } -StandbyReadRecyleState recycle_block_info(const StandbyReadMetaInfo &meta_info, const BufferTag &buf_tag, +StandbyReadRecyleState recycle_block_info(StandbyReadMetaInfo *meta_info, const BufferTag &buf_tag, LsnInfoPosition base_page_info_pos, XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn) { -- Gitee From 2166e2664e534fba3b4a41934ba773146aa7f729 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 17:55:31 +0800 Subject: [PATCH 09/31] xxxxx --- .../storage/access/redo/standby_read/standby_read_interface.cpp | 2 +- src/include/access/extreme_rto/standby_read/block_info_meta.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp index 09bafa2167..a0607a3502 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -342,7 +342,7 @@ Buffer standby_read_buf( ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); // read lsn info StandbyReadLsnInfoArray *lsn_info = &t_thrd.exrto_recycle_cxt.lsn_info; - bool result = extreme_rto_standby_read::get_page_lsn_info(buf_tag, strategy, read_lsn, lsn_info); + bool result = extreme_rto_standby_read::get_page_lsn_info(buf_tag, buf_tag, strategy, read_lsn, lsn_info); if (!result) { ereport( ERROR, diff --git a/src/include/access/extreme_rto/standby_read/block_info_meta.h b/src/include/access/extreme_rto/standby_read/block_info_meta.h index 11935d1d61..414359817a 100644 --- a/src/include/access/extreme_rto/standby_read/block_info_meta.h +++ b/src/include/access/extreme_rto/standby_read/block_info_meta.h @@ -84,7 +84,7 @@ void insert_lsn_to_block_info( void insert_lsn_to_block_info_for_opt( StandbyReadMetaInfo *mete_info, const BufferTag &buf_tag, const Page base_page, XLogRecPtr next_lsn); -StandbyReadRecyleState recycle_block_info(StandbyReadMetaInfo *meta_info, const BufferTag &buf_tag, const BufferTag &seg_buf_tag, +StandbyReadRecyleState recycle_block_info(StandbyReadMetaInfo *meta_info, const BufferTag &buf_tag, XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn); bool get_page_lsn_info(const BufferTag& old_buf_tag, const BufferTag& buf_tag, BufferAccessStrategy strategy, XLogRecPtr read_lsn, StandbyReadLsnInfoArray* lsn_info); -- Gitee From 191f2436d665f7e839adb78af187dac6b08d7b57 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 17:59:16 +0800 Subject: [PATCH 10/31] 1 --- src/include/access/extreme_rto/standby_read/block_info_meta.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/include/access/extreme_rto/standby_read/block_info_meta.h b/src/include/access/extreme_rto/standby_read/block_info_meta.h index 414359817a..ef2c63a2bc 100644 --- a/src/include/access/extreme_rto/standby_read/block_info_meta.h +++ b/src/include/access/extreme_rto/standby_read/block_info_meta.h @@ -85,7 +85,8 @@ void insert_lsn_to_block_info_for_opt( StandbyReadMetaInfo *mete_info, const BufferTag &buf_tag, const Page base_page, XLogRecPtr next_lsn); StandbyReadRecyleState recycle_block_info(StandbyReadMetaInfo *meta_info, const BufferTag &buf_tag, - XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn); + LsnInfoPosition base_page_info_pos, XLogRecPtr next_base_page_lsn, + XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn); bool get_page_lsn_info(const BufferTag& old_buf_tag, const BufferTag& buf_tag, BufferAccessStrategy strategy, XLogRecPtr read_lsn, StandbyReadLsnInfoArray* lsn_info); static inline bool is_block_info_page_valid(BlockInfoPageHeader* header) -- Gitee From 796e2918cb18dcf1776055224d0aeba39b1db108 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 18:01:10 +0800 Subject: [PATCH 11/31] xxxx --- src/include/storage/buf/bufmgr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/storage/buf/bufmgr.h b/src/include/storage/buf/bufmgr.h index 0007c8d40a..9713298ef2 100644 --- a/src/include/storage/buf/bufmgr.h +++ b/src/include/storage/buf/bufmgr.h @@ -443,7 +443,7 @@ BufferDesc *BufferAlloc(const RelFileNode &rel_file_node, char relpersistence, F Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, const XLogPhyBlock *pblk); struct SegSpace; -extern Buffer standby_read_seg_buffer(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); +Buffer standby_read_seg_buffer(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); Buffer MultiBulkReadBufferCommon(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber firstBlockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, int maxBulkCount, const XLogPhyBlock *pblk, int paramNum, char* bufRead); void buffer_in_progress_pop(); -- Gitee From 41fb20cb4df15ee7ea6c5e34b78bf50b830c842d Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 18:08:50 +0800 Subject: [PATCH 12/31] 33333 --- src/include/storage/buf/bufmgr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/storage/buf/bufmgr.h b/src/include/storage/buf/bufmgr.h index 9713298ef2..0007c8d40a 100644 --- a/src/include/storage/buf/bufmgr.h +++ b/src/include/storage/buf/bufmgr.h @@ -443,7 +443,7 @@ BufferDesc *BufferAlloc(const RelFileNode &rel_file_node, char relpersistence, F Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, const XLogPhyBlock *pblk); struct SegSpace; -Buffer standby_read_seg_buffer(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); +extern Buffer standby_read_seg_buffer(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); Buffer MultiBulkReadBufferCommon(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber firstBlockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, int maxBulkCount, const XLogPhyBlock *pblk, int paramNum, char* bufRead); void buffer_in_progress_pop(); -- Gitee From 52a165fcc1ba01b7b3d2e820f605d850b265655a Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 18:11:38 +0800 Subject: [PATCH 13/31] 999 --- src/gausskernel/storage/buffer/bufmgr.cpp | 9 +++++++++ src/gausskernel/storage/smgr/segment/segbuffer.cpp | 10 +--------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index 172a310836..7205e293a0 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -1797,6 +1797,15 @@ Buffer ReadBufferExtended(Relation reln, ForkNumber fork_num, BlockNumber block_ } } +Buffer ReadBufferFast(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode) +{ + if (!RecoveryInProgress() || !IS_EXRTO_STANDBY_READ || !is_exrto_standby_read_worker()) { + return ReadBufferFastNormal(spc, rnode, forkNum, blockNum, mode); + } else { + return standby_read_seg_buffer(spc, rnode, forkNum, blockNum, mode); + } +} + /* * ReadBufferWithoutRelcache -- like ReadBufferExtended, but doesn't require * a relcache entry for the relation. diff --git a/src/gausskernel/storage/smgr/segment/segbuffer.cpp b/src/gausskernel/storage/smgr/segment/segbuffer.cpp index 45a9562834..9181737ff7 100644 --- a/src/gausskernel/storage/smgr/segment/segbuffer.cpp +++ b/src/gausskernel/storage/smgr/segment/segbuffer.cpp @@ -35,6 +35,7 @@ #include "ddes/dms/ss_dms_bufmgr.h" #include "replication/ss_disaster_cluster.h" #include "access/multi_redo_api.h" + /* * Segment buffer, used for segment meta data, e.g., segment head, space map head. We separate segment * meta data buffer and normal data buffer (in bufmgr.cpp) to avoid potential dead locks. @@ -681,15 +682,6 @@ found_branch: return BufferDescriptorGetBuffer(bufHdr); } -Buffer ReadBufferFast(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode) -{ - if (!RecoveryInProgress() || !IS_EXRTO_STANDBY_READ || !is_exrto_standby_read_worker()) { - return ReadBufferFastNormal(spc, rnode, forkNum, blockNum, mode); - } else { - return standby_read_seg_buffer(spc, rnode, forkNum, blockNum, mode); - } -} - BufferDesc *FoundBufferInHashTable(int buf_id, LWLock *new_partition_lock, bool *foundPtr) { BufferDesc *buf = GetBufferDescriptor(buf_id); -- Gitee From e1bcb4807993d616c376493ae40cca7a47728b9a Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Sat, 4 Jan 2025 18:14:32 +0800 Subject: [PATCH 14/31] 0000 --- src/include/storage/buf/bufmgr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/storage/buf/bufmgr.h b/src/include/storage/buf/bufmgr.h index 0007c8d40a..b445fb1ab6 100644 --- a/src/include/storage/buf/bufmgr.h +++ b/src/include/storage/buf/bufmgr.h @@ -443,7 +443,7 @@ BufferDesc *BufferAlloc(const RelFileNode &rel_file_node, char relpersistence, F Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, const XLogPhyBlock *pblk); struct SegSpace; -extern Buffer standby_read_seg_buffer(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); +extern Buffer standby_read_seg_buffer(SegSpace *spc, RelFileNode &rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); Buffer MultiBulkReadBufferCommon(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber firstBlockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, int maxBulkCount, const XLogPhyBlock *pblk, int paramNum, char* bufRead); void buffer_in_progress_pop(); -- Gitee From 04a5cbc85d316f4e15a79355d03f893c72321faa Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 09:28:31 +0800 Subject: [PATCH 15/31] xxxx --- src/include/storage/buf/bufmgr.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/storage/buf/bufmgr.h b/src/include/storage/buf/bufmgr.h index b445fb1ab6..406ad6f561 100644 --- a/src/include/storage/buf/bufmgr.h +++ b/src/include/storage/buf/bufmgr.h @@ -443,7 +443,7 @@ BufferDesc *BufferAlloc(const RelFileNode &rel_file_node, char relpersistence, F Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, const XLogPhyBlock *pblk); struct SegSpace; -extern Buffer standby_read_seg_buffer(SegSpace *spc, RelFileNode &rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); +Buffer standby_read_seg_buffer(SegSpace *spc, RelFileNode &rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); Buffer MultiBulkReadBufferCommon(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber firstBlockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, int maxBulkCount, const XLogPhyBlock *pblk, int paramNum, char* bufRead); void buffer_in_progress_pop(); -- Gitee From 1a8916c5772d05c455fc43a214f5631328e3cc30 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 09:46:29 +0800 Subject: [PATCH 16/31] 1 --- src/include/storage/buf/bufmgr.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/include/storage/buf/bufmgr.h b/src/include/storage/buf/bufmgr.h index 406ad6f561..d4c0834413 100644 --- a/src/include/storage/buf/bufmgr.h +++ b/src/include/storage/buf/bufmgr.h @@ -442,8 +442,10 @@ BufferDesc *BufferAlloc(const RelFileNode &rel_file_node, char relpersistence, F BufferAccessStrategy strategy, bool *foundPtr, const XLogPhyBlock *pblk); Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, const XLogPhyBlock *pblk); -struct SegSpace; -Buffer standby_read_seg_buffer(SegSpace *spc, RelFileNode &rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); + +extern Buffer standby_read_seg_buffer( + SegSpace *spc, const RelFileNode &rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); + Buffer MultiBulkReadBufferCommon(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber firstBlockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, int maxBulkCount, const XLogPhyBlock *pblk, int paramNum, char* bufRead); void buffer_in_progress_pop(); -- Gitee From 6e2291f70504c65389b7fe6d5cd7c8e37b460f20 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 10:09:24 +0800 Subject: [PATCH 17/31] 2 --- src/include/storage/buf/bufmgr.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/include/storage/buf/bufmgr.h b/src/include/storage/buf/bufmgr.h index d4c0834413..256b2ea6f2 100644 --- a/src/include/storage/buf/bufmgr.h +++ b/src/include/storage/buf/bufmgr.h @@ -21,6 +21,7 @@ #include "storage/smgr/relfilenode.h" #include "utils/relcache.h" #include "postmaster/pagerepair.h" +#include "storage/smgr/segment.h" /* [ dram buffer | nvm buffer | segment buffer] */ #define NVM_BUFFER_NUM (g_instance.attr.attr_storage.NNvmBuffers) -- Gitee From d0c8ce1d7ad3f920949da4e84709d275b3773514 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 10:14:38 +0800 Subject: [PATCH 18/31] xx --- src/include/storage/buf/bufmgr.h | 4 ---- src/include/storage/smgr/segment.h | 2 ++ 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/include/storage/buf/bufmgr.h b/src/include/storage/buf/bufmgr.h index 256b2ea6f2..6abfa92b87 100644 --- a/src/include/storage/buf/bufmgr.h +++ b/src/include/storage/buf/bufmgr.h @@ -21,7 +21,6 @@ #include "storage/smgr/relfilenode.h" #include "utils/relcache.h" #include "postmaster/pagerepair.h" -#include "storage/smgr/segment.h" /* [ dram buffer | nvm buffer | segment buffer] */ #define NVM_BUFFER_NUM (g_instance.attr.attr_storage.NNvmBuffers) @@ -444,9 +443,6 @@ BufferDesc *BufferAlloc(const RelFileNode &rel_file_node, char relpersistence, F Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, const XLogPhyBlock *pblk); -extern Buffer standby_read_seg_buffer( - SegSpace *spc, const RelFileNode &rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); - Buffer MultiBulkReadBufferCommon(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber firstBlockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool *hit, int maxBulkCount, const XLogPhyBlock *pblk, int paramNum, char* bufRead); void buffer_in_progress_pop(); diff --git a/src/include/storage/smgr/segment.h b/src/include/storage/smgr/segment.h index 3353bc6926..825fba31a1 100644 --- a/src/include/storage/smgr/segment.h +++ b/src/include/storage/smgr/segment.h @@ -139,6 +139,8 @@ BufferDesc *SegBufferAlloc(RelFileNode rnode, ForkNumber forkNum, BlockNumber bl Buffer ReadSegBufferForDMS(BufferDesc* bufHdr, ReadBufferMode mode, SegSpace *spc = NULL); void ReadSegBufferForCheck(BufferDesc* bufHdr, ReadBufferMode mode, SegSpace *spc, Block bufBlock); Buffer ReadBufferFastNormal(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); +Buffer standby_read_seg_buffer( + SegSpace *spc, const RelFileNode &rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); Buffer ReadBufferFast(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode); void SegReleaseBuffer(Buffer buffer); void SegUnlockReleaseBuffer(Buffer buffer); -- Gitee From da5786941eca4fd0469a39edd5880161a886451b Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 11:05:52 +0800 Subject: [PATCH 19/31] xxx --- src/gausskernel/storage/buffer/bufmgr.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index 7205e293a0..021935a9c2 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -2552,7 +2552,7 @@ Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber fork isExtend = (blockNum == P_NEW); if (!ENABLE_DMS && IsSegmentFileNode(smgr->smgr_rnode.node) && RecoveryInProgress() && !t_thrd.xlog_cxt.InRecovery) { - if (IsBucketFileNode(smgr->smgr_rnode.node) || IS_UNDO_RELFILENODE(smgr->smgr_rnode.node)) { + if (IsBucketFileNode(smgr->smgr_rnode.node)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("bucket and undo segment standby read is not yet supported."))); -- Gitee From b881a9fbce5bbe04e01595a84bf9ab1421e30d06 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 11:12:09 +0800 Subject: [PATCH 20/31] 7777 --- src/gausskernel/storage/buffer/bufmgr.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index 021935a9c2..45b7400106 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -2552,10 +2552,10 @@ Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber fork isExtend = (blockNum == P_NEW); if (!ENABLE_DMS && IsSegmentFileNode(smgr->smgr_rnode.node) && RecoveryInProgress() && !t_thrd.xlog_cxt.InRecovery) { - if (IsBucketFileNode(smgr->smgr_rnode.node)) { + if (IS_UNDO_RELFILENODE(smgr->smgr_rnode.node)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("bucket and undo segment standby read is not yet supported."))); + errmsg("undo segment standby read is not yet supported."))); } } TRACE_POSTGRESQL_BUFFER_READ_START(forkNum, blockNum, smgr->smgr_rnode.node.spcNode, smgr->smgr_rnode.node.dbNode, -- Gitee From 4752cb95e4e9a8263d7eebe81b8d48b448f2c3b6 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 11:16:47 +0800 Subject: [PATCH 21/31] 3 --- src/gausskernel/storage/buffer/bufmgr.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index 45b7400106..186f5ab382 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -2551,6 +2551,7 @@ Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber fork ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); isExtend = (blockNum == P_NEW); + /* if (!ENABLE_DMS && IsSegmentFileNode(smgr->smgr_rnode.node) && RecoveryInProgress() && !t_thrd.xlog_cxt.InRecovery) { if (IS_UNDO_RELFILENODE(smgr->smgr_rnode.node)) { ereport(ERROR, @@ -2558,6 +2559,7 @@ Buffer ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber fork errmsg("undo segment standby read is not yet supported."))); } } + */ TRACE_POSTGRESQL_BUFFER_READ_START(forkNum, blockNum, smgr->smgr_rnode.node.spcNode, smgr->smgr_rnode.node.dbNode, smgr->smgr_rnode.node.relNode, smgr->smgr_rnode.backend, isExtend); -- Gitee From 8fad79d2a9b17b3994f8b651783aa522d678f499 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 11:42:22 +0800 Subject: [PATCH 22/31] xxxx --- .../access/redo/standby_read/standby_read_interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp index a0607a3502..163f9a8e7a 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -180,7 +180,7 @@ Buffer get_newest_seg_page_for_read(SegSpace *spc, const RelFileNode& rnode, For buf_desc->lsn_dirty = InvalidXLogRecPtr; #endif - TerminateBufferIO(buf_desc, false, (BM_VALID | BM_IS_TMP_BUF)); + SegTerminateBufferIO(buf_desc, false, BM_VALID); return BufferDescriptorGetBuffer(buf_desc); } @@ -299,7 +299,7 @@ Buffer standby_read_seg_buffer( buf_desc->lsn_dirty = InvalidXLogRecPtr; #endif buffer_in_progress_push(); - TerminateBufferIO(buf_desc, false, (BM_VALID | BM_IS_TMP_BUF)); + SegTerminateBufferIO(buf_desc, false, BM_VALID); return BufferDescriptorGetBuffer(buf_desc); } -- Gitee From cdebf4780e2d084549f6b4e0f8a3254a503cf1c8 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 14:02:18 +0800 Subject: [PATCH 23/31] 333 --- .../storage/access/redo/standby_read/block_info_proc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp index 2f7738d76a..ed831c5caa 100644 --- a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp @@ -63,7 +63,7 @@ inline uint32 block_info_meta_page_offset(BlockNumber block_num) BlockMetaInfo* get_block_meta_info_by_relfilenode( const BufferTag& buf_tag, BufferAccessStrategy strategy, ReadBufferMode mode, Buffer* buffer, bool need_share_lock) { - Assert(IsSegmentFileNode(buf_tag.rnode) || IsSegmentPhysicalRelNode(buf_tag.rnode)); + Assert(!IsSegmentFileNode(buf_tag.rnode) || IsSegmentPhysicalRelNode(buf_tag.rnode)); RelFileNode standby_read_rnode = buf_tag.rnode; if (IsSegmentFileNode(standby_read_rnode)) { standby_read_rnode.bucketNode -= EXRTO_STANDBY_READ_BUCKET_OFFSET; -- Gitee From 8f806950c0d15c64b276100b3c77378875e5ee79 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 14:18:35 +0800 Subject: [PATCH 24/31] xxxxx --- .../storage/access/redo/standby_read/lsn_info_proc.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp index daa407cd14..085c67f045 100644 --- a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp @@ -271,6 +271,11 @@ LsnInfoPosition create_base_page_info_node(StandbyReadMetaInfo *meta_info, insert_pos += LSN_INFO_HEAD_SIZE; /* actual insert position */ offset += LSN_INFO_HEAD_SIZE; } + Assert(offset % LSN_INFO_NODE_SIZE == 0); + /* update meta info */ + meta_info->lsn_table_next_position = insert_pos + LSN_INFO_NODE_SIZE; + meta_info->base_page_next_position += BLCKSZ; + SpinLockRelease(&(meta_info->mutex)); if (need_update_old_page_header) { Buffer pipe_old_buffer = InvalidBuffer; -- Gitee From 57af30532f6c5d6555e65002f2bf10940131380d Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 15:21:29 +0800 Subject: [PATCH 25/31] xxxxx --- .../redo/standby_read/base_page_proc.cpp | 36 +++++++------------ .../redo/standby_read/lsn_info_proc.cpp | 4 --- .../standby_read/standby_read_interface.cpp | 15 +++----- .../redo/standby_read/standby_read_proc.cpp | 13 ------- .../transam/extreme_rto/exrto_recycle.cpp | 16 ++++++--- .../extreme_rto/standby_read/lsn_info_meta.h | 8 ++--- 6 files changed, 32 insertions(+), 60 deletions(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp index 87b98ab4c6..ad67bf66a8 100644 --- a/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/base_page_proc.cpp @@ -30,11 +30,11 @@ namespace extreme_rto_standby_read { -inline RelFileNode make_base_page_relfilenode(uint32 batch_id, uint32 redo_worker_id, BasePagePosition position) +inline RelFileNode make_base_page_relfilenode(BasePagePosition position) { RelFileNode rnode; rnode.spcNode = EXRTO_BASE_PAGE_SPACE_OID; - rnode.dbNode = (batch_id << LOW_WORKERID_BITS) | redo_worker_id; + rnode.dbNode = 0; rnode.relNode = (uint32)((position / BLCKSZ) >> UINT64_HALF); rnode.bucketNode = InvalidBktId; rnode.opt = DefaultFileNodeOpt; @@ -42,12 +42,14 @@ inline RelFileNode make_base_page_relfilenode(uint32 batch_id, uint32 redo_worke return rnode; } -Buffer buffer_read_base_page(uint32 batch_id, uint32 redo_id, BasePagePosition position, ReadBufferMode mode) +Buffer buffer_read_base_page(BasePagePosition position, ReadBufferMode mode) { - RelFileNode rnode = make_base_page_relfilenode(batch_id, redo_id, position); + RelFileNode rnode = make_base_page_relfilenode(position); BlockNumber blocknum = (BlockNumber)(position / BLCKSZ); bool hit = false; SMgrRelation smgr = smgropen(rnode, InvalidBackendId); + uint32 batch_id = 0; + uint32 redo_id = 0; Buffer buffer = ReadBuffer_common(smgr, RELPERSISTENCE_PERMANENT, MAIN_FORKNUM, blocknum, mode, NULL, &hit, NULL); if (buffer == InvalidBuffer) { @@ -59,33 +61,20 @@ Buffer buffer_read_base_page(uint32 batch_id, uint32 redo_id, BasePagePosition p return buffer; } -void generate_base_page(StandbyReadMetaInfo* meta_info, const Page src_page, BasePagePosition base_page_pos) +void generate_base_page(const Page src_page, BasePagePosition base_page_pos) { - Buffer dest_buf = buffer_read_base_page(meta_info->batch_id, meta_info->redo_id, base_page_pos, RBM_ZERO_AND_LOCK); + Buffer dest_buf = buffer_read_base_page(base_page_pos, RBM_ZERO_AND_LOCK); -#ifdef ENABLE_UT - Page dest_page = get_page_from_buffer(dest_buf); -#else Page dest_page = BufferGetPage(dest_buf); -#endif errno_t rc = memcpy_s(dest_page, BLCKSZ, src_page, BLCKSZ); securec_check(rc, "\0", "\0"); MarkBufferDirty(dest_buf); UnlockReleaseBuffer(dest_buf); } -void read_base_page(const BufferTag& buf_tag, BasePagePosition position, BufferDesc* dest_buf_desc) +void read_base_page(BasePagePosition position, BufferDesc* dest_buf_desc) { - extreme_rto::RedoItemTag redo_item_tag; - INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); - - const uint32 worker_num_per_mng = extreme_rto::get_page_redo_worker_num_per_manager(); - /* batch id and worker id start from 1 when reading a page */ - uint32 batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, (uint32)extreme_rto::get_batch_redo_num()) + 1; - uint32 redo_worker_id = extreme_rto::GetWorkerId(&redo_item_tag, worker_num_per_mng) + 1; - - Buffer buffer = buffer_read_base_page(batch_id, redo_worker_id, position, RBM_NORMAL); - + Buffer buffer = buffer_read_base_page(position, RBM_NORMAL); LockBuffer(buffer, BUFFER_LOCK_SHARE); #ifdef ENABLE_UT Page src_page = get_page_from_buffer(buffer); @@ -99,13 +88,12 @@ void read_base_page(const BufferTag& buf_tag, BasePagePosition position, BufferD UnlockReleaseBuffer(buffer); } -void recycle_base_page_file(uint32 batch_id, uint32 redo_id, BasePagePosition recycle_pos) +void recycle_base_page_file( BasePagePosition recycle_pos) { - RelFileNode rnode = make_base_page_relfilenode(batch_id, redo_id, recycle_pos); + RelFileNode rnode = make_base_page_relfilenode(recycle_pos); SMgrRelation smgr = smgropen(rnode, InvalidBackendId); smgrdounlink(smgr, true, (BlockNumber)(recycle_pos / BLCKSZ)); } } // namespace extreme_rto_standby_read - diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp index 085c67f045..594c00b564 100644 --- a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp @@ -747,10 +747,6 @@ void standby_read_recyle_per_workers(StandbyReadMetaInfo *meta_info, XLogRecPtr cur_lsn_table_recyle_segno > last_lsn_table_recyle_segno) { buffer_drop_exrto_standby_read_buffers(meta_info); } - - if (cur_base_page_recyle_segno > last_base_page_recyle_segno) { - recycle_base_page_file(meta_info->batch_id, meta_info->redo_id, meta_info->base_page_recyle_position); - } } LsnInfoPosition get_nearest_base_page_pos( diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp index 163f9a8e7a..0ca22792cb 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -289,7 +289,7 @@ Buffer standby_read_seg_buffer( } buffer_in_progress_pop(); // read_base_page - extreme_rto_standby_read::read_base_page(buf_tag, lsn_info->base_page_pos, buf_desc); + extreme_rto_standby_read::read_base_page(lsn_info->base_page_pos, buf_desc); if (lsn_info->lsn_num > 0) { redo_target_page(buf_tag, lsn_info, BufferDescriptorGetBuffer(buf_desc)); } @@ -376,7 +376,7 @@ Buffer standby_read_buf( } buffer_in_progress_pop(); // read_base_page - extreme_rto_standby_read::read_base_page(buf_tag, lsn_info->base_page_pos, buf_desc); + extreme_rto_standby_read::read_base_page(lsn_info->base_page_pos, buf_desc); if (lsn_info->lsn_num > 0) { redo_target_page(buf_tag, lsn_info, BufferDescriptorGetBuffer(buf_desc)); } @@ -890,8 +890,7 @@ void dump_base_page_info_lsn_info(const BufferTag &buf_tag, LsnInfoPosition head break; } - Buffer base_page_datum_buffer = - buffer_read_base_page(batch_id, worker_id, base_page_info->base_page_position, RBM_NORMAL); + Buffer base_page_datum_buffer = buffer_read_base_page(base_page_info->base_page_position, RBM_NORMAL); LockBuffer(base_page_datum_buffer, BUFFER_LOCK_SHARE); UnlockReleaseBuffer(base_page_datum_buffer); @@ -1078,14 +1077,8 @@ Buffer standby_read_buf_new( } ReleaseBuffer(read_buf); - extreme_rto::RedoItemTag redo_item_tag; - INIT_REDO_ITEM_TAG(redo_item_tag, buf_tag.rnode, buf_tag.forkNum, buf_tag.blockNum); - const uint32 worker_num_per_mng = (uint32)extreme_rto::get_page_redo_worker_num_per_manager(); - /* batch id and worker id start from 1 when reading a page */ - uint32 batch_id = extreme_rto::GetSlotId(buf_tag.rnode, 0, 0, (uint32)extreme_rto::get_batch_redo_num()) + 1; - uint32 redo_worker_id = extreme_rto::GetWorkerId(&redo_item_tag, worker_num_per_mng) + 1; + Buffer base_page_buffer = buffer_read_base_page(base_page_pos, RBM_NORMAL); - Buffer base_page_buffer = buffer_read_base_page(batch_id, redo_worker_id, base_page_pos, RBM_NORMAL); bool hit = false; LockBuffer(base_page_buffer, BUFFER_LOCK_SHARE); ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp index 46a96f4ea2..8f63e1c6f2 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp @@ -145,24 +145,11 @@ static inline uint64 get_force_recycle_pos(uint64 recycle_pos, uint64 insert_pos XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info) { - uint64 base_page_recycle_pos; uint64 lsn_info_recycle_pos; - XLogRecPtr base_page_recycle_lsn = InvalidXLogRecPtr; XLogRecPtr lsn_info_recycle_lsn = InvalidXLogRecPtr; Buffer buffer; Page page; - /* for base page */ - if (meta_info->base_page_recyle_position < meta_info->base_page_next_position) { - base_page_recycle_pos = - get_force_recycle_pos(meta_info->base_page_recyle_position, meta_info->base_page_next_position); - buffer = extreme_rto_standby_read::buffer_read_base_page( - meta_info->batch_id, meta_info->redo_id, base_page_recycle_pos, RBM_NORMAL); - LockBuffer(buffer, BUFFER_LOCK_SHARE); - base_page_recycle_lsn = PageGetLSN(BufferGetPage(buffer)); - UnlockReleaseBuffer(buffer); - } - /* for lsn info */ if (meta_info->lsn_table_recycle_position < meta_info->lsn_table_next_position) { lsn_info_recycle_pos = diff --git a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp index e4ceeb929c..d203d333d5 100644 --- a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp @@ -139,18 +139,22 @@ void do_standby_read_recyle(XLogRecPtr recycle_lsn) PageRedoWorker** workers = g_dispatcher->allWorkers; XLogRecPtr min_recycle_lsn = InvalidXLogRecPtr; XLogRecPtr before_min_lsn_position = UINT64_MAX; + XLogRecPtr before_min_base_page_position = UINT64_MAX; XLogRecPtr after_min_lsn_position = UINT64_MAX; + XLogRecPtr after_min_base_page_position = UINT64_MAX; for (uint32 i = 0; i < worker_nums; ++i) { PageRedoWorker* page_redo_worker = workers[i]; if (page_redo_worker->role != REDO_PAGE_WORKER || (page_redo_worker->isUndoSpaceWorker)) { continue; } - uint64 before_lsn_position = page_redo_worker->standby_read_meta_info.lsn_table_recycle_position; - before_min_lsn_position = before_min_lsn_position < before_lsn_position ? before_min_lsn_position : before_lsn_position; + before_min_lsn_position = rtl::min(before_min_lsn_position, page_redo_worker->standby_read_meta_info.lsn_table_recycle_position); + before_min_base_page_position = rtl::min(before_min_base_page_position, page_redo_worker->standby_read_meta_info.base_page_recyle_position); + extreme_rto_standby_read::standby_read_recyle_per_workers(&page_redo_worker->standby_read_meta_info, recycle_lsn); - uint64 after_lsn_position = page_redo_worker->standby_read_meta_info.lsn_table_recycle_position; - after_min_lsn_position = after_min_lsn_position < after_lsn_position ? after_min_lsn_position : after_lsn_position; + + after_min_lsn_position = rtl::min(after_min_lsn_position, page_redo_worker->standby_read_meta_info.lsn_table_recycle_position); + after_min_base_page_position = rtl::min(after_min_base_page_position, page_redo_worker->standby_read_meta_info.base_page_recyle_position); if (XLogRecPtrIsInvalid(min_recycle_lsn) || XLByteLT(page_redo_worker->standby_read_meta_info.recycle_lsn_per_worker, min_recycle_lsn)) { @@ -159,8 +163,12 @@ void do_standby_read_recyle(XLogRecPtr recycle_lsn) pg_usleep(1000); // sleep 1ms } if (after_min_lsn_position / EXRTO_LSN_INFO_FILE_MAXSIZE > before_min_lsn_position / EXRTO_LSN_INFO_FILE_MAXSIZE) { + g_dispatcher->global_recycle_lsn = after_min_lsn_position; extreme_rto_standby_read::recycle_lsn_info_file(after_min_lsn_position); } + if (after_min_base_page_position / EXRTO_BASE_PAGE_FILE_MAXSIZE > before_min_base_page_position / EXRTO_BASE_PAGE_FILE_MAXSIZE) { + extreme_rto_standby_read::recycle_base_page_file(after_min_base_page_position); + } if (XLogRecPtrIsInvalid(min_recycle_lsn) && min_recycle_lsn > MAX_LSN_SIZE_PER_FORWARDER && XLByteLT(g_instance.comm_cxt.predo_cxt.global_recycle_lsn, min_recycle_lsn - MAX_LSN_SIZE_PER_FORWARDER)) { diff --git a/src/include/access/extreme_rto/standby_read/lsn_info_meta.h b/src/include/access/extreme_rto/standby_read/lsn_info_meta.h index 3eb0afe895..76f7cd76b6 100644 --- a/src/include/access/extreme_rto/standby_read/lsn_info_meta.h +++ b/src/include/access/extreme_rto/standby_read/lsn_info_meta.h @@ -137,10 +137,10 @@ void insert_base_page_to_lsn_info(StandbyReadMetaInfo* meta_info, LsnInfoDoubleL void get_lsn_info_for_read(const BufferTag& buf_tag, LsnInfoPosition latest_lsn_base_page_pos, StandbyReadLsnInfoArray* lsn_info_list, XLogRecPtr read_lsn); -Buffer buffer_read_base_page(uint32 batch_id, uint32 redo_id, BasePagePosition position, ReadBufferMode mode); -void generate_base_page(StandbyReadMetaInfo* meta_info, const Page src_page, BasePagePosition base_page_pos); -void read_base_page(const BufferTag& buf_tag, BasePagePosition position, BufferDesc* dest_buf_desc); -void recycle_base_page_file(uint32 batch_id, uint32 redo_id, BasePagePosition recycle_pos); +Buffer buffer_read_base_page(BasePagePosition position, ReadBufferMode mode); +void generate_base_page(const Page src_page, BasePagePosition base_page_pos); +void read_base_page(BasePagePosition position, BufferDesc* dest_buf_desc); +void recycle_base_page_file(BasePagePosition recycle_pos); void set_base_page_map_bit(Page page, uint32 base_page_loc); bool is_base_page_map_bit_set(Page page, uint32 which_bit); -- Gitee From 32e12f5948c35926e78393274175c7867836a6d3 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 15:33:51 +0800 Subject: [PATCH 26/31] xxxxx --- .../storage/access/redo/standby_read/block_info_proc.cpp | 4 ++-- .../storage/access/redo/standby_read/lsn_info_proc.cpp | 2 +- .../storage/access/redo/standby_read/standby_read_proc.cpp | 2 +- .../storage/access/transam/extreme_rto/dispatcher.cpp | 2 ++ .../storage/access/transam/extreme_rto/exrto_recycle.cpp | 2 +- src/include/access/extreme_rto/dispatcher.h | 2 ++ 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp index ed831c5caa..e47a604ab9 100644 --- a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp @@ -200,8 +200,8 @@ void insert_lsn_to_block_info_for_opt( /* if block is invalid or block is valid but all the lsn object of this block has been recycled(no data in lsn info * files belongs to this block), we reset this block */ - if (!is_block_meta_info_valid(block_info) || - block_info->lsn_info_list.prev < meta_info->lsn_table_recycle_position) { + uint64 global_recycle_lsn_info_page = pg_atomic_read_u64(&g_dispatcher->global_recycle_lsn_info_page); + if (!is_block_meta_info_valid(block_info) || block_info->lsn_info_list.prev < global_recycle_lsn_info_page) { if (!is_block_info_page_valid((BlockInfoPageHeader *)page)) { block_info_page_init(page); } diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp index 594c00b564..0d9588cabe 100644 --- a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp @@ -348,7 +348,7 @@ void insert_base_page_to_lsn_info(StandbyReadMetaInfo *meta_info, LsnInfoDoubleL } /* generate base page */ - generate_base_page(meta_info, base_page, base_page_pos); + generate_base_page(base_page, base_page_pos); } void get_lsn_info_for_read(const BufferTag& buf_tag, LsnInfoPosition latest_lsn_base_page_pos, diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp index 8f63e1c6f2..40bbf8a026 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_proc.cpp @@ -170,7 +170,7 @@ XLogRecPtr calculate_force_recycle_lsn_per_worker(StandbyReadMetaInfo *meta_info UnlockReleaseBuffer(buffer); } - return rtl::max(base_page_recycle_lsn, lsn_info_recycle_lsn); + return lsn_info_recycle_lsn; } void calculate_force_recycle_lsn(XLogRecPtr &recycle_lsn) diff --git a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp index 5db1f6b877..796ce67921 100755 --- a/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/dispatcher.cpp @@ -555,6 +555,8 @@ static void StartPageRedoWorkers(uint32 totalThrdNum) g_dispatcher->allWorkersCnt = totalThrdNum; g_dispatcher->pageLines = (PageRedoPipeline *)palloc(sizeof(PageRedoPipeline) * batchNum); g_dispatcher->next_lsn_info_page = 0; + g_dispatcher->next_base_page = 0; + g_dispatcher->global_recycle_lsn_info_page = 0; for (started = 0; started < totalThrdNum; started++) { g_dispatcher->allWorkers[started] = CreateWorker(started); diff --git a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp index d203d333d5..6ad769c6b9 100644 --- a/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp +++ b/src/gausskernel/storage/access/transam/extreme_rto/exrto_recycle.cpp @@ -163,7 +163,7 @@ void do_standby_read_recyle(XLogRecPtr recycle_lsn) pg_usleep(1000); // sleep 1ms } if (after_min_lsn_position / EXRTO_LSN_INFO_FILE_MAXSIZE > before_min_lsn_position / EXRTO_LSN_INFO_FILE_MAXSIZE) { - g_dispatcher->global_recycle_lsn = after_min_lsn_position; + g_dispatcher->global_recycle_lsn_info_page = after_min_lsn_position; extreme_rto_standby_read::recycle_lsn_info_file(after_min_lsn_position); } if (after_min_base_page_position / EXRTO_BASE_PAGE_FILE_MAXSIZE > before_min_base_page_position / EXRTO_BASE_PAGE_FILE_MAXSIZE) { diff --git a/src/include/access/extreme_rto/dispatcher.h b/src/include/access/extreme_rto/dispatcher.h index b707c637ea..5eec260fec 100644 --- a/src/include/access/extreme_rto/dispatcher.h +++ b/src/include/access/extreme_rto/dispatcher.h @@ -134,6 +134,8 @@ typedef struct { PageRedoWorker **allWorkers; /* Array of page redo workers. */ uint32 allWorkersCnt; pg_atomic_uint64 next_lsn_info_page; + pg_atomic_uint64 next_base_page; + pg_atomic_uint64 global_recycle_lsn_info_page; RedoItem *freeHead; /* Head of freed-item list. */ RedoItem *freeStateHead; RedoItem *allocatedRedoItem; -- Gitee From 47fba2d7c9dceb2959a00e1812c813d3f0f78850 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 15:38:43 +0800 Subject: [PATCH 27/31] 222 --- .../storage/access/redo/standby_read/block_info_proc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp index e47a604ab9..74f406ec7e 100644 --- a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp @@ -200,7 +200,7 @@ void insert_lsn_to_block_info_for_opt( /* if block is invalid or block is valid but all the lsn object of this block has been recycled(no data in lsn info * files belongs to this block), we reset this block */ - uint64 global_recycle_lsn_info_page = pg_atomic_read_u64(&g_dispatcher->global_recycle_lsn_info_page); + uint64 global_recycle_lsn_info_page = pg_atomic_read_u64(&extreme_rto::g_dispatcher->global_recycle_lsn_info_page); if (!is_block_meta_info_valid(block_info) || block_info->lsn_info_list.prev < global_recycle_lsn_info_page) { if (!is_block_info_page_valid((BlockInfoPageHeader *)page)) { block_info_page_init(page); -- Gitee From 4f2a4f03974067edef3aa189fd38717c887447b3 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 16:58:27 +0800 Subject: [PATCH 28/31] xxxxxxxxx --- .../storage/access/redo/redo_segpage.cpp | 9 ++++++ .../storage/access/redo/redo_xlogutils.cpp | 25 ++++++++++++----- .../redo/standby_read/block_info_proc.cpp | 9 ++++-- .../redo/standby_read/lsn_info_proc.cpp | 6 ++-- .../standby_read/standby_read_interface.cpp | 28 +++++++++++++------ .../storage/smgr/segment/segbuffer.cpp | 9 +++++- .../standby_read/block_info_meta.h | 4 +-- src/include/access/xlogproc.h | 1 + 8 files changed, 65 insertions(+), 26 deletions(-) diff --git a/src/gausskernel/storage/access/redo/redo_segpage.cpp b/src/gausskernel/storage/access/redo/redo_segpage.cpp index 8d46168a71..1c199dfcec 100644 --- a/src/gausskernel/storage/access/redo/redo_segpage.cpp +++ b/src/gausskernel/storage/access/redo/redo_segpage.cpp @@ -228,6 +228,15 @@ static XLogRecParseState *segpage_parse_new_page(XLogReaderState *record, uint32 return recordstatehead; } +void segpage_redo_new_page_for_standby_read(XLogBlockSegNewPage *block_data_rec, RedoBufferInfo *buf_info) +{ + Page src_page = block_data_rec->mainData + sizeof(BufferTag); + errno_t rc = memcpy_s(buf_info->pageinfo.page, BLCKSZ, src_page, BLCKSZ); + securec_check(rc, "\0", "\0"); + PageSetLSN(buf_info->pageinfo.page, buf_info->lsn); + MakeRedoBufferDirty(buf_info); +} + XLogRecParseState *segpage_redo_parse_to_block(XLogReaderState *record, uint32 *blocknum) { uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; diff --git a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp index 12204afa94..901d432392 100644 --- a/src/gausskernel/storage/access/redo/redo_xlogutils.cpp +++ b/src/gausskernel/storage/access/redo/redo_xlogutils.cpp @@ -2046,7 +2046,7 @@ bool redo_target_state(XLogRecParseState *state, RedoBufferInfo *buf_info) return true; } -bool redo_target_seg_state(XLogRecParseState *state, RedoBufferInfo *buf_info) +bool redo_target_seg_state(const BufferTag &new_buf_tag, XLogRecParseState *state, RedoBufferInfo *buf_info) { uint8 xl_info = XLogBlockHeadGetInfo(&state->blockparse.blockhead) & ~XLR_INFO_MASK; bool redo_done = false; @@ -2056,8 +2056,7 @@ bool redo_target_seg_state(XLogRecParseState *state, RedoBufferInfo *buf_info) case XLOG_SEG_INIT_MAPPAGE: case XLOG_SEG_INIT_INVRSPTR_PAGE: case XLOG_SEG_ADD_NEW_GROUP: - case XLOG_SEG_TRUNCATE: - case XLOG_SEG_SPACE_SHRINK: + case XLOG_SEG_TRUNCATE: case XLOG_SEG_NEW_PAGE: { XLogRecParseState* child_state_list = (XLogRecParseState*)state->blockparse.extra_rec.blocksegfullsyncrec.childState; XLogRecParseState* next_state = child_state_list; @@ -2070,14 +2069,26 @@ bool redo_target_seg_state(XLogRecParseState *state, RedoBufferInfo *buf_info) XLogBlockParseStateRelease(state); break; } - + /* + case XLOG_SEG_NEW_PAGE: { + buf_info->lsn = state->blockparse.blockhead.end_ptr; + BufferTag *buf_tag = (BufferTag*)state->blockparse.extra_rec.blocksegnewpageinfo.mainData; + if (!RelFileNodeEquals(new_buf_tag.rnode, buf_tag->rnode) || new_buf_tag.forkNum != buf_tag->forkNum || new_buf_tag.blockNum != buf_tag->blockNum) { + return false; + } + buf_info->blockinfo.pblk = state->blockparse.blockhead.pblk; + segpage_redo_new_page_for_standby_read((XLogBlockSegNewPage*)&state->blockparse.extra_rec.blocksegnewpageinfo, buf_info); + redo_done = true; + break; + } + */ default: return redo_target_state(state, buf_info); } return redo_done; } -void redo_target_page(const BufferTag &buf_tag, StandbyReadLsnInfoArray *lsn_info, Buffer base_page_buf) +void redo_target_page(const BufferTag &old_buf_tag, const BufferTag &new_buf_tag, StandbyReadLsnInfoArray *lsn_info, Buffer base_page_buf) { char *error_msg = NULL; RedoParseManager redo_pm; @@ -2091,7 +2102,7 @@ void redo_target_page(const BufferTag &buf_tag, StandbyReadLsnInfoArray *lsn_inf } RedoBufferInfo buf_info; - init_redo_buffer_info(&buf_info, buf_tag, base_page_buf); + init_redo_buffer_info(&buf_info, old_buf_tag, base_page_buf); for (uint32 i = 0; i < lsn_info->lsn_num; i++) { XLogRecord *record = XLogReadRecord(xlog_reader, lsn_info->lsn_array[i], &error_msg); if (record == NULL) { @@ -2117,7 +2128,7 @@ void redo_target_page(const BufferTag &buf_tag, StandbyReadLsnInfoArray *lsn_inf bool redo_done = false; while(!redo_done && state_iter != NULL) { if ((XLogBlockHeadGetRmid(&state_iter->blockparse.blockhead) == RM_SEGPAGE_ID)) { - redo_done = redo_target_seg_state(state_iter, &buf_info); + redo_done = redo_target_seg_state(new_buf_tag, state_iter, &buf_info); } else { redo_done = redo_target_state(state_iter, &buf_info); } diff --git a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp index 74f406ec7e..ad2be0626b 100644 --- a/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/block_info_proc.cpp @@ -226,8 +226,7 @@ void insert_lsn_to_block_info_for_opt( } StandbyReadRecyleState recycle_block_info(StandbyReadMetaInfo *meta_info, const BufferTag &buf_tag, - LsnInfoPosition base_page_info_pos, XLogRecPtr next_base_page_lsn, - XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn) + XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn) { Buffer buffer = InvalidBuffer; BlockMetaInfo* block_meta_info = get_block_meta_info_by_relfilenode(buf_tag, NULL, RBM_NORMAL, &buffer); @@ -253,7 +252,7 @@ StandbyReadRecyleState recycle_block_info(StandbyReadMetaInfo *meta_info, const } else if (XLogRecPtrIsValid(next_base_page_lsn)) { LsnInfoPosition min_page_info_pos = LSN_INFO_LIST_HEAD; XLogRecPtr min_lsn = InvalidXLogRecPtr; - recycle_one_lsn_info_list(meta_info, base_page_info_pos, recyle_lsn, &min_page_info_pos, &min_lsn); + recycle_one_lsn_info_list(meta_info, block_meta_info->base_page_info_list.next, recyle_lsn, &min_page_info_pos, &min_lsn); Assert(INFO_POSITION_IS_VALID(min_page_info_pos)); if (block_meta_info->base_page_info_list.next != min_page_info_pos) { @@ -338,6 +337,10 @@ void remove_one_block_info_file(const RelFileNode rnode) if (!found && g_instance.bgwriter_cxt.invalid_buf_proc_latch != NULL) { SetLatch(g_instance.bgwriter_cxt.invalid_buf_proc_latch); } + + if (IsSegmentFileNode(rnode)) { + return; + } SMgrRelation srel = smgropen(rnode, InvalidBackendId); smgrdounlink(srel, true); diff --git a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp index 0d9588cabe..7ff98560f0 100644 --- a/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/lsn_info_proc.cpp @@ -243,7 +243,7 @@ LsnInfoPosition create_base_page_info_node(StandbyReadMetaInfo *meta_info, SpinLockAcquire(&(meta_info->mutex)); LsnInfoPosition insert_pos = meta_info->lsn_table_next_position; - *base_page_pos = meta_info->base_page_next_position; + *base_page_pos = pg_atomic_fetch_add_u64(&(extreme_rto::g_dispatcher->next_base_page), BLCKSZ); LsnInfoPosition next_insert_pos = LSN_INFO_LIST_HEAD; LsnInfoPosition pipe_old_page_position = LSN_INFO_LIST_HEAD; uint32 remain_size = BLCKSZ - insert_pos % BLCKSZ; @@ -274,7 +274,6 @@ LsnInfoPosition create_base_page_info_node(StandbyReadMetaInfo *meta_info, Assert(offset % LSN_INFO_NODE_SIZE == 0); /* update meta info */ meta_info->lsn_table_next_position = insert_pos + LSN_INFO_NODE_SIZE; - meta_info->base_page_next_position += BLCKSZ; SpinLockRelease(&(meta_info->mutex)); if (need_update_old_page_header) { @@ -645,7 +644,6 @@ bool recycle_one_lsn_info_page(StandbyReadMetaInfo *meta_info, XLogRecPtr recycl } uint32 offset = bit_to_offset(bit); BasePageInfo base_page_info = (BasePageInfo)(page + offset); - LsnInfoPosition cur_base_page_info_pos = recycle_pos + offset; Assert(is_base_page_type(base_page_info->lsn_info_node.type)); /* block meta file may be dropped */ @@ -684,7 +682,7 @@ bool recycle_one_lsn_info_page(StandbyReadMetaInfo *meta_info, XLogRecPtr recycl XLogRecPtr block_info_max_lsn = InvalidXLogRecPtr; StandbyReadRecyleState stat = - recycle_block_info(meta_info, buf_tag, cur_base_page_info_pos, next_base_page_lsn, recycle_lsn, &block_info_max_lsn); + recycle_block_info(meta_info, buf_tag, next_base_page_lsn, recycle_lsn, &block_info_max_lsn); if (stat == STANDBY_READ_RECLYE_ALL) { invalid_base_page_list(meta_info, buffer, offset); } else if (stat == STANDBY_READ_RECLYE_NONE) { diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp index 0ca22792cb..2e4c018b56 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -291,7 +291,7 @@ Buffer standby_read_seg_buffer( // read_base_page extreme_rto_standby_read::read_base_page(lsn_info->base_page_pos, buf_desc); if (lsn_info->lsn_num > 0) { - redo_target_page(buf_tag, lsn_info, BufferDescriptorGetBuffer(buf_desc)); + redo_target_page(buf_tag, buf_tag, lsn_info, BufferDescriptorGetBuffer(buf_desc)); } Page page = BufferGetPage(BufferDescriptorGetBuffer(buf_desc)); buf_desc->extra->lsn_on_disk = PageGetLSN(page); @@ -320,13 +320,8 @@ Buffer standby_read_buf( } bool hit = false; - BufferTag buf_tag = { - .rnode = reln->rd_smgr->smgr_rnode.node, - .forkNum = fork_num, - .blockNum = block_num, - }; - XLogRecPtr read_lsn = MAX_XLOG_REC_PTR; + if (u_sess->utils_cxt.CurrentSnapshot != NULL && XLogRecPtrIsValid(u_sess->utils_cxt.CurrentSnapshot->read_lsn)) { read_lsn = u_sess->utils_cxt.CurrentSnapshot->read_lsn; } else if (XLogRecPtrIsValid(t_thrd.proc->exrto_read_lsn)) { @@ -334,7 +329,6 @@ Buffer standby_read_buf( } Buffer read_buf = get_newest_page_for_read(reln, fork_num, block_num, mode, strategy, read_lsn); - if (read_buf != InvalidBuffer) { // newest page's lsn smaller than read lsn return read_buf; @@ -342,6 +336,22 @@ Buffer standby_read_buf( ResourceOwnerEnlargeBuffers(t_thrd.utils_cxt.CurrentResourceOwner); // read lsn info StandbyReadLsnInfoArray *lsn_info = &t_thrd.exrto_recycle_cxt.lsn_info; + + BufferTag old_buf_tag = { + .rnode = reln->rd_smgr->smgr_rnode.node, + .forkNum = fork_num, + .blockNum = block_num, + }; + BufferTag new_buf_tag = old_buf_tag; + /* + if (is_segment_logical_relnode(old_buf_tag.rnode)) { + SMgrRelation segreln = smgropen(old_buf_tag.rnode, InvalidBackendId); + XLogPhyBlock pblk; + seg_get_pblk(segreln, old_buf_tag.blockNum, &pblk); + old_buf_tag.rnode.relNode = pblk.relNode; + old_buf_tag.blockNum = pblk.block; + } + */ bool result = extreme_rto_standby_read::get_page_lsn_info(buf_tag, buf_tag, strategy, read_lsn, lsn_info); if (!result) { ereport( @@ -378,7 +388,7 @@ Buffer standby_read_buf( // read_base_page extreme_rto_standby_read::read_base_page(lsn_info->base_page_pos, buf_desc); if (lsn_info->lsn_num > 0) { - redo_target_page(buf_tag, lsn_info, BufferDescriptorGetBuffer(buf_desc)); + redo_target_page(buf_tag, buf_tag, lsn_info, BufferDescriptorGetBuffer(buf_desc)); } Page page = BufferGetPage(BufferDescriptorGetBuffer(buf_desc)); buf_desc->extra->lsn_on_disk = PageGetLSN(page); diff --git a/src/gausskernel/storage/smgr/segment/segbuffer.cpp b/src/gausskernel/storage/smgr/segment/segbuffer.cpp index 9181737ff7..0cdb926a09 100644 --- a/src/gausskernel/storage/smgr/segment/segbuffer.cpp +++ b/src/gausskernel/storage/smgr/segment/segbuffer.cpp @@ -708,7 +708,14 @@ BufferDesc *SegBufferFind(RelFileNode rnode, ForkNumber forkNum, BlockNumber blo LWLockAcquire(new_partition_lock, LW_SHARED); int buf_id = BufTableLookup(&new_tag, new_hash); if (buf_id >= 0) { - return FoundBufferInHashTable(buf_id, new_partition_lock, foundPtr); + BufferDesc *buf = GetBufferDescriptor(buf_id); + bool valid = SegPinBuffer(buf); + if (valid) { + *foundPtr = true; + LWLockRelease(new_partition_lock); + return buf; + } + SegUnpinBuffer(buf); } *foundPtr = false; LWLockRelease(new_partition_lock); diff --git a/src/include/access/extreme_rto/standby_read/block_info_meta.h b/src/include/access/extreme_rto/standby_read/block_info_meta.h index ef2c63a2bc..782c2b3e4b 100644 --- a/src/include/access/extreme_rto/standby_read/block_info_meta.h +++ b/src/include/access/extreme_rto/standby_read/block_info_meta.h @@ -85,8 +85,8 @@ void insert_lsn_to_block_info_for_opt( StandbyReadMetaInfo *mete_info, const BufferTag &buf_tag, const Page base_page, XLogRecPtr next_lsn); StandbyReadRecyleState recycle_block_info(StandbyReadMetaInfo *meta_info, const BufferTag &buf_tag, - LsnInfoPosition base_page_info_pos, XLogRecPtr next_base_page_lsn, - XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn); + XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn) + bool get_page_lsn_info(const BufferTag& old_buf_tag, const BufferTag& buf_tag, BufferAccessStrategy strategy, XLogRecPtr read_lsn, StandbyReadLsnInfoArray* lsn_info); static inline bool is_block_info_page_valid(BlockInfoPageHeader* header) diff --git a/src/include/access/xlogproc.h b/src/include/access/xlogproc.h index 527058fa7a..c425059f34 100755 --- a/src/include/access/xlogproc.h +++ b/src/include/access/xlogproc.h @@ -1303,6 +1303,7 @@ extern bool IsCheckPoint(const XLogRecParseState *parseState); bool is_backup_end(const XLogRecParseState *parse_state); void redo_atomic_xlog_dispatch(uint8 opCode, RedoBufferInfo *redo_buf, const char *data); void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *page, char *data, XLogRecPtr lsn); +void segpage_redo_new_page_for_standby_read(XLogBlockSegNewPage *block_data_rec, RedoBufferInfo *buf_info); void redo_target_page(const BufferTag& buf_tag, StandbyReadLsnInfoArray* lsn_info, Buffer base_page_buf); void MarkSegPageRedoChildPageDirty(RedoBufferInfo *bufferinfo); -- Gitee From 659eecaa962420bb01591167b70b29f2cea79f1d Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 17:10:34 +0800 Subject: [PATCH 29/31] 111 --- src/include/access/extreme_rto/standby_read/block_info_meta.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/access/extreme_rto/standby_read/block_info_meta.h b/src/include/access/extreme_rto/standby_read/block_info_meta.h index 782c2b3e4b..cb934ef8fd 100644 --- a/src/include/access/extreme_rto/standby_read/block_info_meta.h +++ b/src/include/access/extreme_rto/standby_read/block_info_meta.h @@ -85,7 +85,7 @@ void insert_lsn_to_block_info_for_opt( StandbyReadMetaInfo *mete_info, const BufferTag &buf_tag, const Page base_page, XLogRecPtr next_lsn); StandbyReadRecyleState recycle_block_info(StandbyReadMetaInfo *meta_info, const BufferTag &buf_tag, - XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn) + XLogRecPtr next_base_page_lsn, XLogRecPtr recyle_lsn, XLogRecPtr *block_info_max_lsn); bool get_page_lsn_info(const BufferTag& old_buf_tag, const BufferTag& buf_tag, BufferAccessStrategy strategy, XLogRecPtr read_lsn, StandbyReadLsnInfoArray* lsn_info); -- Gitee From 1a87530ed89ab3af0f92c6699bddf6d191aab21a Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 17:14:05 +0800 Subject: [PATCH 30/31] 23123 --- .../access/redo/standby_read/standby_read_interface.cpp | 6 +++--- src/include/access/xlogproc.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp index 2e4c018b56..b4c75defc4 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -352,7 +352,7 @@ Buffer standby_read_buf( old_buf_tag.blockNum = pblk.block; } */ - bool result = extreme_rto_standby_read::get_page_lsn_info(buf_tag, buf_tag, strategy, read_lsn, lsn_info); + bool result = extreme_rto_standby_read::get_page_lsn_info(old_buf_tag, old_buf_tag, strategy, read_lsn, lsn_info); if (!result) { ereport( ERROR, @@ -379,7 +379,7 @@ Buffer standby_read_buf( expected_lsn = lsn_info->lsn_array[lsn_info->lsn_num - 1]; } - BufferDesc* buf_desc = alloc_standby_read_buf(buf_tag, strategy, hit, expected_lsn, is_start_lsn); + BufferDesc* buf_desc = alloc_standby_read_buf(old_buf_tag, strategy, hit, expected_lsn, is_start_lsn); if (hit) { return BufferDescriptorGetBuffer(buf_desc); @@ -388,7 +388,7 @@ Buffer standby_read_buf( // read_base_page extreme_rto_standby_read::read_base_page(lsn_info->base_page_pos, buf_desc); if (lsn_info->lsn_num > 0) { - redo_target_page(buf_tag, buf_tag, lsn_info, BufferDescriptorGetBuffer(buf_desc)); + redo_target_page(old_buf_tag, new_buf_tag, lsn_info, BufferDescriptorGetBuffer(buf_desc)); } Page page = BufferGetPage(BufferDescriptorGetBuffer(buf_desc)); buf_desc->extra->lsn_on_disk = PageGetLSN(page); diff --git a/src/include/access/xlogproc.h b/src/include/access/xlogproc.h index c425059f34..ff1ff98b3e 100755 --- a/src/include/access/xlogproc.h +++ b/src/include/access/xlogproc.h @@ -1304,7 +1304,7 @@ bool is_backup_end(const XLogRecParseState *parse_state); void redo_atomic_xlog_dispatch(uint8 opCode, RedoBufferInfo *redo_buf, const char *data); void seg_redo_new_page_copy_and_flush(BufferTag *tag, char *page, char *data, XLogRecPtr lsn); void segpage_redo_new_page_for_standby_read(XLogBlockSegNewPage *block_data_rec, RedoBufferInfo *buf_info); -void redo_target_page(const BufferTag& buf_tag, StandbyReadLsnInfoArray* lsn_info, Buffer base_page_buf); +void redo_target_page(const BufferTag &old_buf_tag, const BufferTag &new_buf_tag, StandbyReadLsnInfoArray *lsn_info, Buffer base_page_buf); void MarkSegPageRedoChildPageDirty(RedoBufferInfo *bufferinfo); // shared-storage -- Gitee From 7bc5e33d7ed4d42ddb69755a24cf7eb7497cf5a7 Mon Sep 17 00:00:00 2001 From: chenzhikai <895543892@qq.com> Date: Mon, 6 Jan 2025 17:14:48 +0800 Subject: [PATCH 31/31] 999 --- .../access/redo/standby_read/standby_read_interface.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp index b4c75defc4..7f5181e51c 100644 --- a/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp +++ b/src/gausskernel/storage/access/redo/standby_read/standby_read_interface.cpp @@ -359,8 +359,8 @@ Buffer standby_read_buf( (errcode(ERRCODE_INTERNAL_ERROR), (errmsg("standby_read_buf couldnot found buf %u/%u/%u %d %u read lsn %08X/%08X current_time: %ld " "gen_snaptime:%ld thread_read_lsn:%08X/%08X", - buf_tag.rnode.spcNode, buf_tag.rnode.dbNode, buf_tag.rnode.relNode, buf_tag.forkNum, - buf_tag.blockNum, (uint32)(read_lsn >> XID_THIRTY_TWO), (uint32)read_lsn, GetCurrentTimestamp(), + old_buf_tag.rnode.spcNode, old_buf_tag.rnode.dbNode, old_buf_tag.rnode.relNode, old_buf_tag.forkNum, + old_buf_tag.blockNum, (uint32)(read_lsn >> XID_THIRTY_TWO), (uint32)read_lsn, GetCurrentTimestamp(), g_instance.comm_cxt.predo_cxt.exrto_snapshot->gen_snap_time, (uint32)(t_thrd.proc->exrto_read_lsn >> XID_THIRTY_TWO), (uint32)t_thrd.proc->exrto_read_lsn)))); return InvalidBuffer; -- Gitee