From 11f4c7c73b42cb6689b65816ba61ce8e1e20b71f Mon Sep 17 00:00:00 2001 From: fengyang Date: Tue, 17 Jan 2023 18:39:25 +0800 Subject: [PATCH 1/6] =?UTF-8?q?FileMetaData=20=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/db/file_meta_data.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/db/file_meta_data.rs b/src/db/file_meta_data.rs index 31d4390..1c3e8cd 100644 --- a/src/db/file_meta_data.rs +++ b/src/db/file_meta_data.rs @@ -1,5 +1,6 @@ use crate::db::db_format::InternalKey; +/// @see version_edit FileMetaData pub struct FileMetaData { refs: u32, // Seeks allowed until compaction @@ -7,6 +8,22 @@ pub struct FileMetaData { number: u64, // File size in bytes file_size: u64, + // Smallest internal key served by table smallest: InternalKey, + // Largest internal key served by table + largest: InternalKey +} +impl Default for FileMetaData { + #[inline] + fn default() -> Self { + Self { + refs: 0, + allowed_seeks: 1 << 30, + number: 0, + file_size: 0, + smallest: InternalKey::default(), + largest: InternalKey::default() + } + } } \ No newline at end of file -- Gitee From 6961efe55549f4ad2b7cacf7eb27ad8703492277 Mon Sep 17 00:00:00 2001 From: fengyang Date: Tue, 17 Jan 2023 19:34:12 +0800 Subject: [PATCH 2/6] =?UTF-8?q?FileMetaData=20=E5=AE=9E=E7=8E=B0=E6=8E=92?= =?UTF-8?q?=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/db/db_format.rs | 9 ++++++ src/db/file_meta_data.rs | 60 +++++++++++++++++++++++++++++++++++ src/db/file_meta_data_test.rs | 33 +++++++++++++++++-- src/db/mod.rs | 4 +++ src/db/version_edit.rs | 0 src/db/version_edit_test.rs | 7 ++++ src/db/version_set.rs | 32 +++++++++++++++++++ src/db/version_set_test.rs | 7 ++++ 8 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 src/db/version_edit.rs create mode 100644 src/db/version_edit_test.rs create mode 100644 src/db/version_set.rs create mode 100644 src/db/version_set_test.rs diff --git a/src/db/db_format.rs b/src/db/db_format.rs index 87c7701..1692569 100644 --- a/src/db/db_format.rs +++ b/src/db/db_format.rs @@ -18,6 +18,7 @@ pub struct ParsedInternalKey { value_type: ValueType } +#[derive(Debug)] pub struct InternalKey { rep_: Slice } @@ -143,6 +144,14 @@ impl Default for InternalKey { } } +impl PartialEq for InternalKey { + /// 判断两个 InternalKey 是否相同 + #[inline] + fn eq(&self, other: &Self) -> bool { + self.rep_.eq(&other.rep_) + } +} + impl InternalKey { fn new(user_key: Slice, sequence: u64, value_type: ValueType) -> Self { // line 145 diff --git a/src/db/file_meta_data.rs b/src/db/file_meta_data.rs index 1c3e8cd..a78acac 100644 --- a/src/db/file_meta_data.rs +++ b/src/db/file_meta_data.rs @@ -1,6 +1,10 @@ +use std::cmp; +use std::cmp::Ordering; +use std::collections::HashMap; use crate::db::db_format::InternalKey; /// @see version_edit FileMetaData +#[derive(Debug)] pub struct FileMetaData { refs: u32, // Seeks allowed until compaction @@ -14,6 +18,11 @@ pub struct FileMetaData { largest: InternalKey } +#[allow(improper_ctypes)] +extern { + fn memcmp(s1: *const i32, s2: *const i32) -> i32; +} + impl Default for FileMetaData { #[inline] fn default() -> Self { @@ -26,4 +35,55 @@ impl Default for FileMetaData { largest: InternalKey::default() } } +} + +impl FileMetaData { + pub fn create_refs(refs: u32) -> Self { + FileMetaData::create_refs_allowed_seeks(refs, 1 << 30) + } + + pub fn create_refs_allowed_seeks(refs: u32, allowed_seeks: u32) -> Self { + FileMetaData::create_refs_allowed_seeks_file_size(refs, allowed_seeks, 0) + } + + pub fn create_refs_allowed_seeks_file_size(refs: u32, allowed_seeks: u32, file_size: u64) -> Self { + FileMetaData::create_refs_allowed_seeks_file_size_internal_key(refs, allowed_seeks, file_size, InternalKey::default(), InternalKey::default()) + } + + pub fn create_refs_allowed_seeks_file_size_internal_key(refs: u32, allowed_seeks: u32, file_size: u64, + smallest: InternalKey, largest: InternalKey) -> Self { + FileMetaData::create(refs, allowed_seeks, 0, file_size, smallest, largest) + } + + pub fn create(refs: u32, allowed_seeks: u32, number: u64, file_size: u64, smallest: InternalKey, largest: InternalKey) -> Self { + Self { + refs, + allowed_seeks, + number, + file_size, + smallest, + largest + } + } +} + +impl PartialEq for FileMetaData { + /// 判断两个 FileMetaData 是否相同 + #[inline] + fn eq(&self, other: &Self) -> bool { + self.refs == other.refs + && self.allowed_seeks == other.allowed_seeks + && self.number == other.number + && self.file_size == other.file_size + && self.smallest.eq(&other.smallest) + && self.largest.eq(&other.largest) + } +} + +impl PartialOrd for FileMetaData { + /// 判断两个 FileMetaData 的大小关系 + fn partial_cmp(&self, other: &Self) -> Option { + // todo + Option::Some(Ordering::Equal) + } } \ No newline at end of file diff --git a/src/db/file_meta_data_test.rs b/src/db/file_meta_data_test.rs index b40ab66..3cf9bff 100644 --- a/src/db/file_meta_data_test.rs +++ b/src/db/file_meta_data_test.rs @@ -1,5 +1,32 @@ -#[test] -fn test_file_meta_data() { - todo!() +mod test { + use std::cmp::Ordering; + use crate::db::file_meta_data::FileMetaData; + + #[test] + fn test_eq() { + let meta1: FileMetaData = FileMetaData::default(); + let meta2: FileMetaData = FileMetaData::default(); + let meta3: FileMetaData = FileMetaData::create_refs(6); + let meta3_1: FileMetaData = FileMetaData::create_refs(6); + + assert!(meta1.eq(&meta2)); + assert!(!meta1.eq(&meta3)); + assert!(meta3.eq(&meta3_1)); + } + + #[test] + fn test_partial_ord() { + let meta0: FileMetaData = FileMetaData::default(); + let meta0_1: FileMetaData = FileMetaData::default(); + let meta3: FileMetaData = FileMetaData::create_refs(3); + let meta6: FileMetaData = FileMetaData::create_refs(6); + let meta7: FileMetaData = FileMetaData::create_refs(7); + + assert!(meta0.partial_cmp(&meta0_1).is_some()); + assert!(meta0.partial_cmp(&meta0_1).is_some_and(Ordering::Equal)); + assert!(!meta1.eq(&meta3)); + assert!(meta3.eq(&meta3_1)); + } + } \ No newline at end of file diff --git a/src/db/mod.rs b/src/db/mod.rs index d9f915b..0acd557 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -15,6 +15,10 @@ pub mod db_format; mod db_format_test; pub mod file_meta_data; mod file_meta_data_test; +pub mod version_set; +mod version_set_test; +pub mod version_edit; +mod version_edit_test; /// 默认调表 pub type DefaultSkipList = SkipList; diff --git a/src/db/version_edit.rs b/src/db/version_edit.rs new file mode 100644 index 0000000..e69de29 diff --git a/src/db/version_edit_test.rs b/src/db/version_edit_test.rs new file mode 100644 index 0000000..71f136d --- /dev/null +++ b/src/db/version_edit_test.rs @@ -0,0 +1,7 @@ + +mod test { + #[test] + fn test_() { + println!("get_name: {}", "a"); + } +} \ No newline at end of file diff --git a/src/db/version_set.rs b/src/db/version_set.rs new file mode 100644 index 0000000..4bbf973 --- /dev/null +++ b/src/db/version_set.rs @@ -0,0 +1,32 @@ +use std::collections::{BTreeMap, BTreeSet}; +use crate::db::db_format::InternalKeyComparator; +use crate::db::file_meta_data::FileMetaData; + +// line 164 +pub struct VersionSet { +} + +// line 323 +pub struct Compaction { +} + +// line 604 +pub struct Builder { + a : BTreeSet, +} + +struct BySmallestKey { + internal_comparator: InternalKeyComparator +} + +struct LevelState { + deleted_files: Vec, + added_files: BTreeSet +} + +impl BySmallestKey { + fn operator(f1: FileMetaData, f2: FileMetaData) -> bool { + // line 607 + true + } +} \ No newline at end of file diff --git a/src/db/version_set_test.rs b/src/db/version_set_test.rs new file mode 100644 index 0000000..71f136d --- /dev/null +++ b/src/db/version_set_test.rs @@ -0,0 +1,7 @@ + +mod test { + #[test] + fn test_() { + println!("get_name: {}", "a"); + } +} \ No newline at end of file -- Gitee From 01a68aaff79a11f324be578637ed44d633fb3a56 Mon Sep 17 00:00:00 2001 From: fengyang Date: Tue, 17 Jan 2023 20:12:28 +0800 Subject: [PATCH 3/6] =?UTF-8?q?VersionEdit=20=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/db/file_meta_data.rs | 4 +- src/db/file_meta_data_test.rs | 3 - src/db/version_edit.rs | 102 ++++++++++++++++++++++++++++++++++ src/db/version_set.rs | 75 ++++++++++++++++++++++++- src/util/filter_policy.rs | 30 +++++----- src/util/options.rs | 7 ++- 6 files changed, 197 insertions(+), 24 deletions(-) diff --git a/src/db/file_meta_data.rs b/src/db/file_meta_data.rs index a78acac..8cef576 100644 --- a/src/db/file_meta_data.rs +++ b/src/db/file_meta_data.rs @@ -82,8 +82,8 @@ impl PartialEq for FileMetaData { impl PartialOrd for FileMetaData { /// 判断两个 FileMetaData 的大小关系 - fn partial_cmp(&self, other: &Self) -> Option { + fn partial_cmp(&self, other: &Self) -> Option { // todo Option::Some(Ordering::Equal) - } + } } \ No newline at end of file diff --git a/src/db/file_meta_data_test.rs b/src/db/file_meta_data_test.rs index 3cf9bff..2889a1a 100644 --- a/src/db/file_meta_data_test.rs +++ b/src/db/file_meta_data_test.rs @@ -24,9 +24,6 @@ mod test { let meta7: FileMetaData = FileMetaData::create_refs(7); assert!(meta0.partial_cmp(&meta0_1).is_some()); - assert!(meta0.partial_cmp(&meta0_1).is_some_and(Ordering::Equal)); - assert!(!meta1.eq(&meta3)); - assert!(meta3.eq(&meta3_1)); } } \ No newline at end of file diff --git a/src/db/version_edit.rs b/src/db/version_edit.rs index e69de29..aaa1c4d 100644 --- a/src/db/version_edit.rs +++ b/src/db/version_edit.rs @@ -0,0 +1,102 @@ +use std::iter::Map; +use crate::db::db_format::InternalKey; +use crate::db::file_meta_data::FileMetaData; +use crate::util::slice::Slice; + +pub struct VersionEdit { + comparator_: Slice, + log_number_: u64, + prev_log_number_: u64, + next_file_number_: u64, + last_sequence_: u64, + has_comparator_: bool, + has_log_number_: bool, + has_prev_log_number_: bool, + has_next_file_number_: bool, + has_last_sequence_: bool, + + compact_pointers_: Vec>, + deleted_files_: Vec>, + new_files_: Vec>, +} + +pub struct Pair { + left: LEFT, + right: RIGHT +} + +impl Pair { + fn make_pair(l: LEFT, r: RIGHT) -> Self{ + Self{ + left: l, + right: r, + } + } +} + +enum Tag { + // kComparator = 1, + // kLogNumber = 2, + // kNextFileNumber = 3, + // kLastSequence = 4, + // kCompactPointer = 5, + // kDeletedFile = 6, + // kNewFile = 7, + // // 8 was used for large value refs + // kPrevLogNumber = 9 + + kComparator, + kLogNumber, + kNextFileNumber, + kLastSequence, + kCompactPointer, + kDeletedFile, + kNewFile, + // 8 was used for large value refs + kPrevLogNumber +} + +impl VersionEdit { + /// 清空 + fn clear(&mut self) { + // todo + // self.comparator_.clear(); + self.log_number_ = 0; + self.prev_log_number_ = 0; + self.last_sequence_ = 0; + self.next_file_number_ = 0; + self.has_comparator_ = false; + self.has_log_number_ = false; + self.has_prev_log_number_ = false; + self.has_next_file_number_ = false; + self.has_last_sequence_ = false; + self.deleted_files_.clear(); + self.new_files_.clear(); + + // compact_pointers_ don't clear + } + + fn set_comparator_name(&mut self, name: Slice){ + self.has_comparator_ = true; + self.comparator_ = name; + } + + fn set_prev_log_number(&mut self, num: u64){ + self.has_prev_log_number_ = true; + self.prev_log_number_ = num; + } + + fn set_next_file(&mut self, num: u64){ + self.has_next_file_number_ = true; + self.next_file_number_ = num; + } + + fn set_last_sequence(&mut self, seq: u64){ + self.has_last_sequence_ = true; + self.last_sequence_ = seq; + } + + fn set_compact_pointer(&mut self, level: u32, key: InternalKey) { + self.compact_pointers_.push(Pair::make_pair(level, key)) + } +} \ No newline at end of file diff --git a/src/db/version_set.rs b/src/db/version_set.rs index 4bbf973..00c35d8 100644 --- a/src/db/version_set.rs +++ b/src/db/version_set.rs @@ -1,13 +1,69 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::db::db_format::InternalKeyComparator; use crate::db::file_meta_data::FileMetaData; +use crate::db::version_edit::VersionEdit; +use crate::util::cache::Cache; +use crate::util::options::{Env, Options}; +use crate::util::slice::Slice; -// line 164 +// .h line 58 - 162 +pub struct Version { + + // VersionSet* vset_; // VersionSet to which this Version belongs + // Version* next_; // Next version in linked list + // Version* prev_; // Previous version in linked list + // int refs_; // Number of live refs to this version + // + // // List of files per level + // std::vector files_[config::kNumLevels]; + // + // // Next file to compact based on seek stats. + // FileMetaData* file_to_compact_; + // int file_to_compact_level_; + // + // // Level that should be compacted next and its compaction score. + // // Score < 1 means compaction is not strictly needed. These fields + // // are initialized by Finalize(). + // double compaction_score_; + // int compaction_level_; +} + +// .h line 164 - 320 pub struct VersionSet { + // TableCache } -// line 323 +// .h line 323 - 393 pub struct Compaction { + level_: u32, + max_output_file_size_: u64, + input_version_: Version, + edit_: VersionEdit + + // // Each compaction reads inputs from "level_" and "level_+1" + // std::vector inputs_[2]; // The two sets of inputs + // + // // State used to check for number of overlapping grandparent files + // // (parent == level_ + 1, grandparent == level_ + 2) + // std::vector grandparents_; + // size_t grandparent_index_; // Index in grandparent_starts_ + // bool seen_key_; // Some output key has been seen + // int64_t overlapped_bytes_; // Bytes of overlap between current output + // // and grandparent files + // + // // State for implementing IsBaseLevelForKey + // + // // level_ptrs_ holds indices into input_version_->levels_: our state + // // is that we are positioned at one of the file ranges for each + // // higher level than the ones involved in this compaction (i.e. for + // // all L >= level_ + 2). + // size_t level_ptrs_[config::kNumLevels]; +} + +// .h line 68 - 71 +struct GetStats { + seek_file: FileMetaData, + seek_file_level: u32 } // line 604 @@ -24,9 +80,24 @@ struct LevelState { added_files: BTreeSet } +impl Compaction { + fn create(options: Options, level: u32) -> Self { + Self { + + } + } +} + impl BySmallestKey { fn operator(f1: FileMetaData, f2: FileMetaData) -> bool { // line 607 true } +} + +// todo 临时使用。等待提供方 +struct TableCache { + // todo Env 临时使用。等待提供方 + env_: Env, + dbname_: Slice } \ No newline at end of file diff --git a/src/util/filter_policy.rs b/src/util/filter_policy.rs index 7a98a78..3ba1ac8 100644 --- a/src/util/filter_policy.rs +++ b/src/util/filter_policy.rs @@ -16,6 +16,21 @@ pub trait AsBloomHash { fn bloom_hash(&self) -> u32; } +/// 实现了 Slice 转 bloom_hash 的特质 +/// Sample: +/// ``` +/// let val = "aabbccd"; +/// let slice: Slice = Slice::from_buf(val.as_bytes()); +/// let hash_val = slice.bloom_hash(); +/// ``` +impl AsBloomHash for Slice { + #[inline] + fn bloom_hash(&self) -> u32 { + BloomFilterPolicy::bloom_hash(self) + } +} + +// ######################### BloomFilterPolicy pub struct BloomFilterPolicy { bits_per_key: usize, k: usize @@ -140,6 +155,7 @@ impl FilterPolicy for BloomFilterPolicy { } } +// ######################### InternalFilterPolicy pub struct InternalFilterPolicy { user_policy_: dyn FilterPolicy } @@ -163,18 +179,4 @@ impl FilterPolicy for InternalFilterPolicy { fn key_may_match(&self, key: &Slice, bloom_filter: &Slice) -> bool { todo!() } -} - -/// 实现了 Slice 转 bloom_hash 的特质 -/// Sample: -/// ``` -/// let val = "aabbccd"; -/// let slice: Slice = Slice::from_buf(val.as_bytes()); -/// let hash_val = slice.bloom_hash(); -/// ``` -impl AsBloomHash for Slice { - #[inline] - fn bloom_hash(&self) -> u32 { - BloomFilterPolicy::bloom_hash(self) - } } \ No newline at end of file diff --git a/src/util/options.rs b/src/util/options.rs index 3c3060b..d103472 100644 --- a/src/util/options.rs +++ b/src/util/options.rs @@ -1,5 +1,6 @@ use crate::db::db::Snapshot; use crate::traits::comparator_trait::Comparator; +use crate::traits::filter_policy_trait::FilterPolicy; use crate::util::comparator::BytewiseComparatorImpl; pub enum CompressionType { @@ -11,8 +12,8 @@ pub enum CompressionType { pub struct Env {} pub struct Cache {} - -pub struct FilterPolicy {} +// +// pub struct FilterPolicy {} pub struct Options { @@ -95,7 +96,7 @@ pub struct Options { /// If non-null, use the specified filter policy to reduce disk reads. /// Many applications will benefit from passing the result of /// NewBloomFilterPolicy() here. - pub filter_policy: Option, + pub filter_policy: Option, } /// Options that control read operations pub struct ReadOptions { -- Gitee From 8b28e046c8af848e33fa7d99876a7a3b4c7245e5 Mon Sep 17 00:00:00 2001 From: fengyang Date: Tue, 17 Jan 2023 20:24:26 +0800 Subject: [PATCH 4/6] =?UTF-8?q?=E7=BC=96=E8=AF=91=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/db/version_set.rs | 10 +++++----- src/util/options.rs | 9 ++++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/db/version_set.rs b/src/db/version_set.rs index 00c35d8..8ac23ea 100644 --- a/src/db/version_set.rs +++ b/src/db/version_set.rs @@ -81,11 +81,11 @@ struct LevelState { } impl Compaction { - fn create(options: Options, level: u32) -> Self { - Self { - - } - } + // fn create(options: Options, level: u32) -> Self { + // Self { + // + // } + // } } impl BySmallestKey { diff --git a/src/util/options.rs b/src/util/options.rs index d103472..a63c9f6 100644 --- a/src/util/options.rs +++ b/src/util/options.rs @@ -1,6 +1,5 @@ use crate::db::db::Snapshot; use crate::traits::comparator_trait::Comparator; -use crate::traits::filter_policy_trait::FilterPolicy; use crate::util::comparator::BytewiseComparatorImpl; pub enum CompressionType { @@ -12,8 +11,8 @@ pub enum CompressionType { pub struct Env {} pub struct Cache {} -// -// pub struct FilterPolicy {} + +pub struct FilterPolicy {} pub struct Options { @@ -96,9 +95,9 @@ pub struct Options { /// If non-null, use the specified filter policy to reduce disk reads. /// Many applications will benefit from passing the result of /// NewBloomFilterPolicy() here. - pub filter_policy: Option, + pub filter_policy: Option, } - /// Options that control read operations +/// Options that control read operations pub struct ReadOptions { /// If true, all data read from underlying storage will be /// verified against corresponding checksums. -- Gitee From 4d85264123617bc40fd88e45173b0b9038fe56fa Mon Sep 17 00:00:00 2001 From: fengyang Date: Wed, 18 Jan 2023 13:12:27 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E5=8F=8C=E5=90=91=E9=93=BE=E8=A1=A8=20?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E5=AE=9A=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 + src/db/db_format.rs | 7 +- src/db/file_meta_data.rs | 37 +++- src/db/version_edit.rs | 123 ++++++++++-- src/db/version_edit_test.rs | 3 + src/db/version_set.rs | 378 ++++++++++++++++++++++++++++++++--- src/db/version_set_test.rs | 2 + src/util/linked_list.rs | 58 ++++++ src/util/linked_list_test.rs | 12 ++ src/util/mod.rs | 2 + 10 files changed, 559 insertions(+), 64 deletions(-) create mode 100644 src/util/linked_list.rs create mode 100644 src/util/linked_list_test.rs diff --git a/README.md b/README.md index b4cd64f..373c322 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ LSM tree 是许多 KV型或日志型数据库所依赖的核心实现，例如Bi | db.LogReader | wangboo | 90% | | db.LogWriter | wangboo | 90% | | db.TableCache | colagy | | +| LinkedList | fengyang | | | db.VersionEdit(Tag, VersionEdit, FileMetaData) | fengyang | | | db.VersionSet(Version, LevelFileNumIterator, SaverState) | fengyang | | | WriteBatch | peach | | diff --git a/src/db/db_format.rs b/src/db/db_format.rs index 1692569..891b906 100644 --- a/src/db/db_format.rs +++ b/src/db/db_format.rs @@ -1,6 +1,7 @@ use std::cmp::Ordering; use std::ops::Deref; use crate::db::db_format::ValueType::{K_TYPE_DELETION, K_TYPE_VALUE}; +use crate::db::file_meta_data::FileMetaData; use crate::traits::comparator_trait::Comparator; use crate::util::slice::Slice; @@ -209,15 +210,15 @@ impl InternalKey { } impl InternalKeyComparator { - fn new(c: Box) -> Box { + pub fn create(c: Box) -> Box { todo!() } - fn user_comparator(&self) -> Box { + pub fn user_comparator(&self) -> Box { todo!() } - fn compare(&self, a: InternalKey, b: InternalKey) -> u32 { + pub fn compare_internal_key(&self, key1: &InternalKey, key2: &InternalKey) -> u32 { // line 122, 167 todo!() } diff --git a/src/db/file_meta_data.rs b/src/db/file_meta_data.rs index 8cef576..7747ac3 100644 --- a/src/db/file_meta_data.rs +++ b/src/db/file_meta_data.rs @@ -1,14 +1,14 @@ use std::cmp; use std::cmp::Ordering; use std::collections::HashMap; -use crate::db::db_format::InternalKey; +use crate::db::db_format::{InternalKey, InternalKeyComparator}; /// @see version_edit FileMetaData #[derive(Debug)] pub struct FileMetaData { - refs: u32, + refs: i32, // Seeks allowed until compaction - allowed_seeks: u32, + allowed_seeks: i32, number: u64, // File size in bytes file_size: u64, @@ -38,24 +38,28 @@ impl Default for FileMetaData { } impl FileMetaData { - pub fn create_refs(refs: u32) -> Self { + pub fn create_refs(refs: i32) -> Self { FileMetaData::create_refs_allowed_seeks(refs, 1 << 30) } - pub fn create_refs_allowed_seeks(refs: u32, allowed_seeks: u32) -> Self { + pub fn create_refs_allowed_seeks(refs: i32, allowed_seeks: i32) -> Self { FileMetaData::create_refs_allowed_seeks_file_size(refs, allowed_seeks, 0) } - pub fn create_refs_allowed_seeks_file_size(refs: u32, allowed_seeks: u32, file_size: u64) -> Self { + pub fn create_refs_allowed_seeks_file_size(refs: i32, allowed_seeks: i32, file_size: u64) -> Self { FileMetaData::create_refs_allowed_seeks_file_size_internal_key(refs, allowed_seeks, file_size, InternalKey::default(), InternalKey::default()) } - pub fn create_refs_allowed_seeks_file_size_internal_key(refs: u32, allowed_seeks: u32, file_size: u64, + pub fn create_refs_allowed_seeks_file_size_internal_key(refs: i32, allowed_seeks: i32, file_size: u64, smallest: InternalKey, largest: InternalKey) -> Self { FileMetaData::create(refs, allowed_seeks, 0, file_size, smallest, largest) } - pub fn create(refs: u32, allowed_seeks: u32, number: u64, file_size: u64, smallest: InternalKey, largest: InternalKey) -> Self { + pub fn create_number_file_size_internal_key(number: u64, file_size: u64, smallest: InternalKey, largest: InternalKey) -> Self { + FileMetaData::create(0, 1 << 30, number, file_size, smallest, largest) + } + + pub fn create(refs: i32, allowed_seeks: i32, number: u64, file_size: u64, smallest: InternalKey, largest: InternalKey) -> Self { Self { refs, allowed_seeks, @@ -65,6 +69,22 @@ impl FileMetaData { largest } } + + pub fn get_smallest(&self) -> &InternalKey { + &self.smallest + } + + pub fn get_number(&self) -> u64 { + self.number + } + + pub fn get_refs(&self) -> i32 { + self.refs + } + + pub fn add_refs(&mut self, num: i32) { + self.refs += num; + } } impl PartialEq for FileMetaData { @@ -84,6 +104,7 @@ impl PartialOrd for FileMetaData { /// 判断两个 FileMetaData 的大小关系 fn partial_cmp(&self, other: &Self) -> Option { // todo + // InternalKeyComparator::compare() Option::Some(Ordering::Equal) } } \ No newline at end of file diff --git a/src/db/version_edit.rs b/src/db/version_edit.rs index aaa1c4d..14be86f 100644 --- a/src/db/version_edit.rs +++ b/src/db/version_edit.rs @@ -2,9 +2,10 @@ use std::iter::Map; use crate::db::db_format::InternalKey; use crate::db::file_meta_data::FileMetaData; use crate::util::slice::Slice; +use crate::util::Result; pub struct VersionEdit { - comparator_: Slice, + comparator_: String, log_number_: u64, prev_log_number_: u64, next_file_number_: u64, @@ -15,23 +16,9 @@ pub struct VersionEdit { has_next_file_number_: bool, has_last_sequence_: bool, - compact_pointers_: Vec>, - deleted_files_: Vec>, - new_files_: Vec>, -} - -pub struct Pair { - left: LEFT, - right: RIGHT -} - -impl Pair { - fn make_pair(l: LEFT, r: RIGHT) -> Self{ - Self{ - left: l, - right: r, - } - } + compact_pointers_: Vec<(u32, InternalKey)>, + deleted_files_: Vec<(u32, u64)>, + new_files_: Vec<(u32, FileMetaData)>, } enum Tag { @@ -59,8 +46,7 @@ enum Tag { impl VersionEdit { /// 清空 fn clear(&mut self) { - // todo - // self.comparator_.clear(); + self.comparator_.clear(); self.log_number_ = 0; self.prev_log_number_ = 0; self.last_sequence_ = 0; @@ -78,7 +64,7 @@ impl VersionEdit { fn set_comparator_name(&mut self, name: Slice){ self.has_comparator_ = true; - self.comparator_ = name; + self.comparator_ = name.into(); } fn set_prev_log_number(&mut self, num: u64){ @@ -97,6 +83,99 @@ impl VersionEdit { } fn set_compact_pointer(&mut self, level: u32, key: InternalKey) { - self.compact_pointers_.push(Pair::make_pair(level, key)) + self.compact_pointers_.push((level, key)) + } + + /// Add the specified file at the specified number. + /// REQUIRES: This version has not been saved (see VersionSet::SaveTo) + /// REQUIRES: "smallest" and "largest" are smallest and largest keys in file + /// + /// # Arguments + /// + /// * `level`: + /// * `file`: + /// * `file_size`: + /// * `smallest`: 移交所有权 + /// * `largest`: 移交所有权 + /// + /// returns: () + /// + /// # Examples + /// + /// ``` + /// + /// ``` + fn add_file(&mut self, level: u32, file: u64, file_size: u64, smallest: InternalKey, largest: InternalKey) { + let file_meta_data = FileMetaData::create_number_file_size_internal_key(file, file_size, smallest, largest); + + self.new_files_.push((level, file_meta_data)); + } + + fn delete_file(&mut self, level: u32, file: u64) { + self.deleted_files_.push((level, file)); + } + + /// 将 VersionEdit 对象编码至 target 中 + /// + /// # Arguments + /// + /// * `target`: + /// + /// returns: () + /// + /// # Examples + /// + /// ``` + /// + /// ``` + fn encode_to(&self, target: Vec) { + todo!() + } + + /// 将 source 中的数据解码至 self VersionEdit 中 + /// + /// # Arguments + /// + /// * `source`: + /// + /// returns: () + /// + /// # Examples + /// + /// ``` + /// + /// ``` + fn decode_from(&mut self, source: Slice) { + self.clear(); + + todo!() + } + + /// VersionEdit 输出调试信息 + fn debug_string(&self) -> Slice { + todo!() + } +} + +impl<'a> VersionEdit { + pub fn get_internal_key(inout: Slice) -> Result { + todo!() + } + + /// 从 Slice 中解出 level 值 + /// + /// # Arguments + /// + /// * `input`: + /// + /// returns: bool + /// + /// # Examples + /// + /// ``` + /// + /// ``` + pub fn get_level(input: Slice) -> Result { + todo!() } } \ No newline at end of file diff --git a/src/db/version_edit_test.rs b/src/db/version_edit_test.rs index 71f136d..3024e55 100644 --- a/src/db/version_edit_test.rs +++ b/src/db/version_edit_test.rs @@ -2,6 +2,9 @@ mod test { #[test] fn test_() { + println!("get_name: {}", "a"); + + } } \ No newline at end of file diff --git a/src/db/version_set.rs b/src/db/version_set.rs index 8ac23ea..9c7a708 100644 --- a/src/db/version_set.rs +++ b/src/db/version_set.rs @@ -1,36 +1,69 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::db::db_format::InternalKeyComparator; +use std::rc::Rc; +use crate::db::db_format::{Config, InternalKeyComparator, LookupKey}; use crate::db::file_meta_data::FileMetaData; +use crate::db::table_cache::TableCache; use crate::db::version_edit::VersionEdit; +use crate::traits::comparator_trait::Comparator; use crate::util::cache::Cache; -use crate::util::options::{Env, Options}; +use crate::util::options::{Env, Options, ReadOptions}; use crate::util::slice::Slice; +use crate::util::Result; // .h line 58 - 162 pub struct Version { + // todo 链表实现，前驱后继 + // VersionSet to which this Version belongs + vset_: Rc, + // Next version in linked list + next_: Rc, + // Previous version in linked list + prev_: Rc, - // VersionSet* vset_; // VersionSet to which this Version belongs - // Version* next_; // Next version in linked list - // Version* prev_; // Previous version in linked list - // int refs_; // Number of live refs to this version - // - // // List of files per level - // std::vector files_[config::kNumLevels]; - // - // // Next file to compact based on seek stats. - // FileMetaData* file_to_compact_; - // int file_to_compact_level_; - // - // // Level that should be compacted next and its compaction score. - // // Score < 1 means compaction is not strictly needed. These fields - // // are initialized by Finalize(). - // double compaction_score_; - // int compaction_level_; + // Number of live refs to this version + refs_: i32, + + // List of files per level, 内部vec 初始化长度 config::kNumLevels + files_: Vec>, + + // Next file to compact based on seek stats. + file_to_compact_: FileMetaData, + file_to_compact_level_: i32, + + // Level that should be compacted next and its compaction score. + // Score < 1 means compaction is not strictly needed. These fields are initialized by Finalize(). + compaction_score_: f64, + compaction_level_: i32 } // .h line 164 - 320 pub struct VersionSet { - // TableCache + env_: Env, + dbname_: Slice, + options_: Options, + table_cache_: TableCache, + icmp_: Box, + next_file_number_: u64, + manifest_file_number_: u64, + last_sequence_: u64, + log_number_: u64, + // 0 or backing store for memtable being compacted + prev_log_number_: u64, + + // Opened lazily + // todo WritableFile + // descriptor_file_: tokio.File, + // todo log::Writer + // descriptor_log_: log::Writer, + + // Head of circular doubly-linked list of versions. + dummy_versions_: Version, + // dummy_versions_.prev_ + current_: Version, + + // Per-level key at which the next compaction at that level should start. + // Either an empty string, or a valid InternalKey. + compact_pointer_: [String; Config::K_NUM_LEVELS] } // .h line 323 - 393 @@ -63,12 +96,29 @@ pub struct Compaction { // .h line 68 - 71 struct GetStats { seek_file: FileMetaData, - seek_file_level: u32 + seek_file_level: i32 +} + +// ,cc line 163 +struct LevelFileNumIterator { + icmp_: Rc, + flist_: Vec, + index_: u32, + + // // Backing store for value(). Holds the file number and size. + // mutable char value_buf_[16]; + value_buf_: [u8] } // line 604 pub struct Builder { - a : BTreeSet, + vset_: VersionSet, + + // VersionSet* ; + // Version* base_; + // LevelState levels_[config::kNumLevels]; + + // a : BTreeSet, } struct BySmallestKey { @@ -77,9 +127,248 @@ struct BySmallestKey { struct LevelState { deleted_files: Vec, + + // replace std::set FileSet -> added_files added_files: BTreeSet } +impl Version { + fn create(vset: VersionSet) -> Self { + todo!() + } + + fn set_next(&mut self, data: &Rc) -> bool { + self.next_ = (*data).clone().into(); + + true + } + + fn set_prev(&mut self, data: &Rc) -> bool { + self.prev_ = (*data).clone().into(); + + true + } + + fn get_refs(&self) -> i32 { + self.refs_ + } + + /// todo 等待 Iterator 接口 + /// + /// 通过self.versionSet中的TableCache.NewIterator, 将 self 对象数据追加到 iters 中 + /// + /// Append to *iters a sequence of iterators that will yield the contents of this Version when merged together. + /// + /// REQUIRES: This version has been saved (see VersionSet::SaveTo) + /// + /// # Arguments + /// + /// * `options`: + /// * `iters`: + /// + /// returns: () + /// + /// # Examples + /// + /// ``` + /// + /// ``` + fn add_iterators(&self, options: ReadOptions, /* iters: Vec */) { + todo!() + } + + /// 数据搜索 + /// + /// 一级一级地搜索，因为条目不会跨越级别。如果在较小的级别上发现数据，则后面的级别是不相关的。 + /// + /// # Arguments + /// + /// * `options`: + /// * `key`: + /// * `value`: + /// + /// returns: Result + /// + /// # Examples + /// + /// ``` + /// + /// ``` + fn get(&self, options: ReadOptions, key: LookupKey, value: Slice) -> Result { + todo!() + } + + /// Adds "stats" into the current state. + /// Returns true if a new compaction may need to be triggered, false otherwise. + /// + /// REQUIRES: lock is held + /// + /// # Arguments + /// + /// * `stats`: + /// + /// returns: bool + /// + /// # Examples + /// + /// ``` + /// + /// ``` + fn update_stats(&self, stats: GetStats) -> bool{ + todo!() + } + + /// 记录在指定内部键处读取的字节样本。 + /// 大约每 config:：K_READ_BYTES_PERIOD 字节采样一次。如果可能需要触发新的压缩，则返回true + /// + /// Record a sample of bytes read at the specified internal key. + /// Samples are taken approximately once every config::kReadBytesPeriod + /// bytes. Returns true if a new compaction may need to be triggered. + /// + /// REQUIRES: lock is held 要求：锁定 + /// + /// # Arguments + /// + /// * `key`: + /// + /// returns: bool + /// + /// # Examples + /// + /// ``` + /// + /// ``` + fn record_read_sample(&self, key: Slice) -> bool { + todo!() + } + + // // + // // Reference count management (so Versions do not disappear out from + // // under live iterators) + // void Ref(); + // void Unref(); + // + // void GetOverlappingInputs( + // int level, + // const InternalKey* begin, // nullptr means before all keys + // const InternalKey* end, // nullptr means after all keys + // std::vector* inputs); + // + // // Returns true iff some file in the specified level overlaps + // // some part of [*smallest_user_key,*largest_user_key]. + // // smallest_user_key==nullptr represents a key smaller than all the DB's keys. + // // largest_user_key==nullptr represents a key largest than all the DB's keys. + // bool OverlapInLevel(int level, + // const Slice* smallest_user_key, + // const Slice* largest_user_key); + // + // // Return the level at which we should place a new memtable compaction + // // result that covers the range [smallest_user_key,largest_user_key]. + // int PickLevelForMemTableOutput(const Slice& smallest_user_key, + // const Slice& largest_user_key); + // + // int NumFiles(int level) const { return files_[level].size(); } + // + // // Return a human readable string that describes this version's contents. + // std::string DebugString() const; + + // Iterator* NewConcatenatingIterator(const ReadOptions&, int level) const; + // + // // Call func(arg, level, f) for every file that overlaps user_key in + // // order from newest to oldest. If an invocation of func returns + // // false, makes no more calls. + // // + // // REQUIRES: user portion of internal_key == user_key. + // void ForEachOverlapping(Slice user_key, Slice internal_key, + // void* arg, + // bool (*func)(void*, int, FileMetaData*)); +} + +impl Drop for Version { + /// version_set.cc 中析构方法 Version::~Version() 的对应实现 + fn drop(&mut self) { + assert_eq!(self.get_refs(), 0); + + // // Remove from linked list + // self.prev_.set_next(&self.next_); + // self.next_.set_prev(&self.prev_); + + // // Drop references to files + // for level in 0..Config::K_NUM_LEVELS { + // for i in 0..self.files_[level].len() { + // let mut meta: FileMetaData = files_[level][i]; + // assert!(meta.get_refs() > 0); + // + // meta.add_refs(-1); + // if meta.get_refs() <= 0 { + // // todo C 语义 delete f + // // delete f; + // } + // } + // } + } +} + +// impl VersionSet { +// //. +// } + +impl<'a> VersionSet { + /// 返回文件源数据中最小的索引。 如果文件不存在，则返回文件数量 + /// + /// Return the smallest index i such that files[i]->largest >= key. + /// Return files.size() if there is no such file. + /// + /// REQUIRES: "files" contains a sorted list of non-overlapping files. + /// # Arguments + /// + /// * `icmp`: + /// * `files`: + /// * `key`: + /// + /// returns: u32 + /// + /// # Examples + /// + /// ``` + /// + /// ``` + fn find_file(icmp: Box, files:Vec, key:Slice) -> u32 { + todo!() + } + + /// 如果 user key 范围[smallest_user_key, largest_user_key] 与 “files”中的 [smallest.user_key(), largest.user_key()] 重叠，则返回true + /// minimal==nullptr表示比DB中的所有键都小的键。 + /// maximum==nullptr表示比DB中的所有键都大的键。 + /// + /// Returns true iff some file in "files" overlaps the user key range [smallest_user_key, largest_user_key]. + /// smallest==nullptr represents a key smaller than all keys in the DB. + /// largest==nullptr represents a key largest than all keys in the DB. + /// + /// REQUIRES: If disjoint_sorted_files, files[] contains disjoint ranges + /// in sorted order. + /// + /// # Arguments + /// + /// * `icmp`: + /// * `disjoint_sorted_files`: + /// * `files`: + /// * `smallest_user_key`: + /// * `largest_user_key`: + /// + /// returns: bool + /// + /// # Examples + /// + /// ``` + /// + /// ``` + fn some_file_overlaps_range(icmp: Box, disjoint_sorted_files:bool, + files:Vec, smallest_user_key:Slice,largest_user_key:Slice) -> bool { + todo!() + } +} + impl Compaction { // fn create(options: Options, level: u32) -> Self { // Self { @@ -88,16 +377,43 @@ impl Compaction { // } } +// todo 等到 Iterator 接口 +// impl Iterator for LevelFileNumIterator { +// +// } + +impl Builder { + +} + impl BySmallestKey { - fn operator(f1: FileMetaData, f2: FileMetaData) -> bool { + + /// FileMetaData 比较 + /// + /// # Arguments + /// + /// * `f1`: + /// * `f2`: + /// + /// returns: bool + /// + /// # Examples + /// + /// ``` + /// + /// ``` + pub fn operator(&self, f1: &FileMetaData, f2: &FileMetaData) -> bool { // line 607 - true - } -} + let r: u32 = self.internal_comparator.compare_internal_key( + f1.get_smallest(), + f2.get_smallest() + ); -// todo 临时使用。等待提供方 -struct TableCache { - // todo Env 临时使用。等待提供方 - env_: Env, - dbname_: Slice + if r != 0 { + return r < 0; + } + + // Break ties by file number + return f1.get_number() < f2.get_number(); + } } \ No newline at end of file diff --git a/src/db/version_set_test.rs b/src/db/version_set_test.rs index 71f136d..44c68c4 100644 --- a/src/db/version_set_test.rs +++ b/src/db/version_set_test.rs @@ -2,6 +2,8 @@ mod test { #[test] fn test_() { + println!("get_name: {}", "a"); + } } \ No newline at end of file diff --git a/src/util/linked_list.rs b/src/util/linked_list.rs new file mode 100644 index 0000000..b976997 --- /dev/null +++ b/src/util/linked_list.rs @@ -0,0 +1,58 @@ +use std::fmt::{Display, Formatter}; +use std::marker::PhantomData; +use std::ptr::NonNull; + +/// 节点 +#[derive(Debug)] +struct Node { + // 节点值. 如果存在这个 Node，则该Node中 val 必定是有值的; 如果 val 的值为空，则这个 Node 就应该为 None + val: T, + // 前驱 + prev: Option>> + // 后继. Option 表示该节点为空，即不存在 prev 前置节点（整个链表为空时）、或不存在next 后置节点（链表的尾节点） + next: Option>>, +} + +/// 双向链表 +#[derive(Debug)] +pub struct LinkedList { + // 链表长度 + length: usize, + // 头 + head: Option>>, + // 尾 + tail: Option>>, +} + +impl Node { + fn new(val: T) -> Node { + Node { + val, + prev: None, + next: None, + } + } + + /// 用于将 Box 中的 Node 转为含有所有权的 T 类型 + fn into_val(self: Box) -> T { + self.val + } +} + +impl Default for LinkedList { + /// 构造空的双向链表 + fn default() -> Self { + Self::new() + } +} + +impl LinkedList { + /// 构造空的双向链表 + pub fn new() -> Self { + Self { + length: 0, + head: None, + tail: None + } + } +} diff --git a/src/util/linked_list_test.rs b/src/util/linked_list_test.rs new file mode 100644 index 0000000..45493c1 --- /dev/null +++ b/src/util/linked_list_test.rs @@ -0,0 +1,12 @@ + +mod test { + use crate::util::linked_list::LinkedList; + use crate::util::slice::Slice; + + #[test] + fn test_() { + let mut list:LinkedList = LinkedList::new(); + + println!("linked list: {:#?}", list); + } +} \ No newline at end of file diff --git a/src/util/mod.rs b/src/util/mod.rs index bd12ce0..3291ada 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -38,6 +38,8 @@ mod mutex_lock_test; pub mod random; mod random_test; pub mod options; +pub mod linked_list; +mod linked_list_test; /// 定义别名 pub type Result = result::Result; -- Gitee From f59fc5957ee8b19da0748ce3071bc4f168b209de Mon Sep 17 00:00:00 2001 From: fengyang Date: Wed, 18 Jan 2023 13:12:50 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E5=8F=8C=E5=90=91=E9=93=BE=E8=A1=A8=20?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E5=AE=9A=E4=B9=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/linked_list.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/linked_list.rs b/src/util/linked_list.rs index b976997..f28ddd7 100644 --- a/src/util/linked_list.rs +++ b/src/util/linked_list.rs @@ -8,7 +8,7 @@ struct Node { // 节点值. 如果存在这个 Node，则该Node中 val 必定是有值的; 如果 val 的值为空，则这个 Node 就应该为 None val: T, // 前驱 - prev: Option>> + prev: Option>>, // 后继. Option 表示该节点为空，即不存在 prev 前置节点（整个链表为空时）、或不存在next 后置节点（链表的尾节点） next: Option>>, } -- Gitee