From cef589b56509299205a7a943e3df0830c3c7940e Mon Sep 17 00:00:00 2001 From: fengyang Date: Thu, 6 Apr 2023 21:06:33 +0800 Subject: [PATCH 1/3] =?UTF-8?q?TableBuilder=20struct=E5=AE=9A=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/table/block_builder.rs | 6 +++--- src/table/format.rs | 34 ++++++++++++++++++---------------- src/table/table_builder.rs | 12 ++++++++++++ 3 files changed, 33 insertions(+), 19 deletions(-) diff --git a/src/table/block_builder.rs b/src/table/block_builder.rs index 63166ce..f24a995 100644 --- a/src/table/block_builder.rs +++ b/src/table/block_builder.rs @@ -39,7 +39,7 @@ impl BlockBuilder { todo!() } - /// 添加数据到block + /// 向datablock增加entry /// /// # Arguments /// @@ -68,7 +68,7 @@ impl BlockBuilder { todo!() } - /// 构造block + /// 追加Restart points /// /// /// # Examples @@ -91,7 +91,7 @@ impl BlockBuilder { todo!() } - /// 估算当前的block大小 + /// 估算当前的block大小, 超过一定大小后，写入文件 /// /// # Examples /// diff --git a/src/table/format.rs b/src/table/format.rs index 01c9610..ccfe72b 100644 --- a/src/table/format.rs +++ b/src/table/format.rs @@ -12,26 +12,28 @@ pub const k_max_encoded_length: u32 = 10 + 10; /// of two block handles and a magic number. pub const k_encoded_length: u32 = 2 * k_max_encoded_length + 8; -// // kTableMagicNumber was picked by running -// // echo http://code.google.com/p/leveldb/ | sha1sum -// // and taking the leading 64 bits. -// pub const k_table_magic_number: &str = 0xdb4775248b80fb57ull; +/// Footer 的大小为 48 字节，内容是一个 8 字节的 magic number 和两个 BlockHandle 构成 +/// 在 Footer::EncodeTo 和 Footer::DecodeFrom 中起作用 +/// kTableMagicNumber was picked by running +/// echo http://code.google.com/p/leveldb/ | sha1sum +/// and taking the leading 64 bits. +pub const k_table_magic_number: u64 = 0xdb4775248b80fb57; /// 1-byte type + 32-bit crc pub const k_block_trailer_size: usize = 5; pub struct BlockHandle { // 偏移量 - offset_: u64, + offset: u64, // - size_: u64 + size: u64 } /// Footer encapsulates the fixed information stored at the tail /// end of every table file. pub struct Footer { - metaindex_handle_: BlockHandle, - index_handle_: BlockHandle + meta_index_handle: BlockHandle, + index_handle: BlockHandle } pub struct BlockContents { @@ -102,7 +104,7 @@ trait BlockTrait { trait FootTrait { // The block handle for the metaindex block of the table - fn metaindex_handle(&self) -> BlockHandle; + fn meta_index_handle(&self) -> BlockHandle; fn set_metaindex_handle(&mut self, h: BlockHandle); @@ -153,19 +155,19 @@ trait BlockContent { impl BlockTrait for BlockHandle { fn offset(&self) -> u64 { - self.offset_ + self.offset } fn set_offset(&mut self, offset: u64) { - self.offset_ = offset; + self.offset = offset; } fn size(&self) -> u64 { - self.size_ + self.size } fn set_size(&mut self, size: u64) { - self.size_ = size; + self.size = size; } fn encode_to(&self) -> Result { @@ -190,14 +192,14 @@ impl Default for BlockHandle { #[inline] fn default() -> Self { BlockHandle { - offset_: 0, - size_: 0, + offset: 0, + size: 0, } } } impl FootTrait for Footer { - fn metaindex_handle(&self) -> BlockHandle { + fn meta_index_handle(&self) -> BlockHandle { todo!() } diff --git a/src/table/table_builder.rs b/src/table/table_builder.rs index 2d7ed0a..dc9cc91 100644 --- a/src/table/table_builder.rs +++ b/src/table/table_builder.rs @@ -10,6 +10,12 @@ use crate::util::slice::Slice; use crate::util::status::Status; use crate::util::unsafe_slice::UnsafeSlice; +/// 在一个 SSTable 中，文件末尾的 Footer 是定长的， +/// 其他数据都被划分成一个个变长的 block： +/// index block(@see Footer#index_handle)、 +/// meta_index block(@see Footer#meta_index_handle)、 +/// meta blocks、 +/// data blocks。 pub struct TableBuilder { rep: Box } @@ -41,6 +47,7 @@ struct Rep<> { pending_index_entry: bool, // Handle to add to index block // pending_handle 记录需要生成数据索引的数据块在 SSTable 中的偏移量和大小 + // 也就是说， pending_handle 主要用于表示当前块的offset及size。 pending_handle: BlockHandle, compressed_output: Slice, @@ -57,18 +64,22 @@ impl TableBuilder { todo!() } + /// 写入 entry pub fn add(&self, key: &UnsafeSlice, value: &UnsafeSlice) { todo!() } + /// flush到文件 pub fn flush(&self) { todo!() } + /// block->Finish、压缩 pub fn write_block(&self, block: &BlockBuilder, handler: &BlockHandle) { todo!() } + /// datablock写入文件，添加压缩方式、crc。 pub fn write_raw_block(&self, block_contents: &UnsafeSlice, compression_type: CompressionType, handler: &BlockHandle) { todo!() } @@ -77,6 +88,7 @@ impl TableBuilder { todo!() } + /// 剩余datablock写入文件，并生成管理区。 pub fn finish(&self) -> Status { todo!() } -- Gitee From 8f92c247c340c890698ba0d2daea82922c251cb9 Mon Sep 17 00:00:00 2001 From: fengyang Date: Fri, 7 Apr 2023 12:33:24 +0800 Subject: [PATCH 2/3] =?UTF-8?q?format=20api=20=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/table/format.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/table/format.rs b/src/table/format.rs index ccfe72b..e082810 100644 --- a/src/table/format.rs +++ b/src/table/format.rs @@ -47,7 +47,7 @@ pub struct BlockContents { heap_allocated:bool, } -trait BlockTrait { +trait BlockHandleTrait { /// /// The offset of the block in the file. /// @@ -153,7 +153,7 @@ trait BlockContent { } -impl BlockTrait for BlockHandle { +impl BlockHandleTrait for BlockHandle { fn offset(&self) -> u64 { self.offset } @@ -199,6 +199,7 @@ impl Default for BlockHandle { } impl FootTrait for Footer { + /// The block handle for the metaindex block of the table fn meta_index_handle(&self) -> BlockHandle { todo!() } -- Gitee From 1ac1d7a7dfbd795984c85610fb7d238563be5f7b Mon Sep 17 00:00:00 2001 From: fengyang Date: Fri, 7 Apr 2023 13:57:21 +0800 Subject: [PATCH 3/3] =?UTF-8?q?TableBuilder=20=E5=A2=9E=E5=8A=A0=E6=B3=A8?= =?UTF-8?q?=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/table/filter_block.rs | 1 + src/table/table_builder.rs | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/table/filter_block.rs b/src/table/filter_block.rs index c24694e..0013382 100644 --- a/src/table/filter_block.rs +++ b/src/table/filter_block.rs @@ -13,6 +13,7 @@ const FILTER_BASE: usize = 1 << FILTER_BASE_LG; /// /// meta block 构建器 +/// FilterBlock，实质上就是SST文件里面的 meta block /// pub trait FilterBlock { fn new_with_policy(policy: FilterPolicyPtr) -> Self; diff --git a/src/table/table_builder.rs b/src/table/table_builder.rs index dc9cc91..c2d4492 100644 --- a/src/table/table_builder.rs +++ b/src/table/table_builder.rs @@ -12,9 +12,9 @@ use crate::util::unsafe_slice::UnsafeSlice; /// 在一个 SSTable 中，文件末尾的 Footer 是定长的， /// 其他数据都被划分成一个个变长的 block： -/// index block(@see Footer#index_handle)、 -/// meta_index block(@see Footer#meta_index_handle)、 -/// meta blocks、 +/// index block(@see format.BlockHandle、Footer#index_handle)、 +/// meta_index block(@see format.BlockHandle、Footer#meta_index_handle)、 +/// meta blocks(@see table.FilterBlock)、 /// data blocks。 pub struct TableBuilder { rep: Box -- Gitee