diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000000000000000000000000000000000000..0d0b4dad7ce90ef4d73db0ab149378d8d24533ef --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +rustflags='--cfg CORE_DEBUG="true"' \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 6f688ef24e926e517e32af2d66ae4932fa50edca..b2f54eca1a27bc895e61c5fd24c482ba0cbb4126 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,11 +7,18 @@ edition = "2021" name = "level_db_rust" path = "src/lib.rs" +[features] +default = ["debug-macro"] +release=[] + +debug-macro=[] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] - +rand = "0.8.5" [profile.dev] + [profile.release] + diff --git a/README.md b/README.md index 07c70b6cadd721ba046746edcf30779bde802bc8..f3c18eeaa4e94387ca4b1e3dd3e933e53deac7da 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,14 @@ LSM tree 是许多 KV型或日志型数据库所依赖的核心实现，例如Bi #### 使用说明 -1. xxxx +1. 编译参数 + CORE_DEBUG 默认开启，打印调试信息 + +在构建正式版本时，用户可以用 RUSTFLAGS 环境变量覆盖以上编译参数。 +eg: +```bash +RUSTFLAGS='--cfg CORE_DEBUG="false"' cargo build --release +``` #### 参与贡献 @@ -59,7 +66,7 @@ LSM tree 是许多 KV型或日志型数据库所依赖的核心实现，例如Bi | Slice | wangboo | 100% | 2. 1.1.0 版本, 完成基础零部件 - + | 功能模块 | 完成人 | 进度 | |------------------|---------|-----| | skiplist | 未认领 | | diff --git a/src/db/log_wr_test.rs b/src/db/log_wr_test.rs index 98b16481a9dfbeafe726b4da4788d456b99319f4..d333556c9563abd6bb769eaa67bb2f22674acaf0 100644 --- a/src/db/log_wr_test.rs +++ b/src/db/log_wr_test.rs @@ -1,12 +1,12 @@ mod test { - use std::fs::File; - use crate::db::log_reader::LogReader; - use crate::db::log_writer::LogWriter; - use crate::traits::coding_trait::CodingTrait; - use crate::util::coding::Coding; - use crate::util::crc::{AsCrc, ToMask}; - use crate::util::Result; + + + + + + + use crate::util::slice::Slice; #[test] @@ -28,7 +28,7 @@ mod test { let sample: Vec = ('0'..='9').map(|a|a as u8).collect(); for i in 0..100 { let slice = reader.read_next().expect("not error").expect("must have record"); - let mut expect = generate_slice(i, &sample); + let expect = generate_slice(i, &sample); assert_eq!(expect.len(), slice.len()); assert_eq!(expect.as_ref(), slice.as_ref()) } diff --git a/src/db/mem_table.rs b/src/db/mem_table.rs index 9bc002cc10fae2d69fbf6c4418c9cffbddfb5872..a9dcf7160e76fa59efcf0dfed9c77fd0fcdfb1dd 100644 --- a/src/db/mem_table.rs +++ b/src/db/mem_table.rs @@ -1,7 +1,7 @@ use std::rc::Rc; use crate::traits::comparator_trait::ComparatorTrait; use crate::traits::DataIterator; -use crate::util::comparator::InternalKeyComparator; + use crate::util::slice::Slice; use crate::util::Result; diff --git a/src/db/mod.rs b/src/db/mod.rs index 5a34cf6ff1473e78a6cc2b7ce8c884ce44c5318f..5b5ff72b8eeb31cbcfb3b84637b48b7641e20411 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,7 +1,7 @@ -use crate::db::skip_list::SkipList; use crate::db::mem_table::MemTable; +use crate::db::skip_list::SkipList; use crate::util::comparator::{BytewiseComparatorImpl, InternalKeyComparator}; -use crate::util::slice::Slice; + pub mod log_writer; pub mod log_reader; @@ -9,8 +9,9 @@ mod log_wr_test; pub mod skip_list; pub mod mem_table; pub mod db; +mod skip_list_test; /// 默认调表 -pub type DefaultSkipList = SkipList; +pub type DefaultSkipList = SkipList; /// 默认内存表 pub type DefaultMemTable = MemTable; \ No newline at end of file diff --git a/src/db/skip_list.rs b/src/db/skip_list.rs index 144c0b60f26f215d4c7065a134c631bdf964f433..ddef2421df16174ac9848bf6985676f79dbafa17 100644 --- a/src/db/skip_list.rs +++ b/src/db/skip_list.rs @@ -1,39 +1,401 @@ -use std::rc::Rc; +use std::cmp::Ordering; +use std::mem; +use std::mem::size_of; +use std::ptr::null_mut; +use std::sync::Arc; + +use rand::prelude::*; +use crate::debug; use crate::traits::comparator_trait::ComparatorTrait; -use crate::util::Arena; -use crate::util::comparator::BytewiseComparatorImpl; +use crate::util::arena::{ArenaAllocLike, ArenaRef}; use crate::util::Result; use crate::util::slice::Slice; +use crate::util::status::{LevelError, Status}; + +type RawNode = *mut Node; + +const MAX_LEVEL: usize = 8; // todo -struct Node { - value: T, +struct Node { + /// 存储的值, 如果为空，则是头指针或者尾指针 + key: Option, + /// 数组元素首地址，代表一个数组，指向每层的下一个节点。 + next_elems: *mut RawNode, + /// 当前节点高度 + level: usize, } -pub struct SkipList { - node: Option>, - comp: Rc, +pub struct SkipList { + /// 最高层数 + height: usize, + /// 存储数据数量 + num: usize, + /// 头部指针 + head: RawNode, + /// 尾指针 + tail: RawNode, + /// 比较器 + cmp: Arc, + /// 内存分配器 + arena: ArenaRef, } -impl SkipList { +pub struct Iter<'a, Cmp: ComparatorTrait> { + list: &'a SkipList, + node: RawNode, +} - pub fn create(comparator: Rc, _arena: Rc) -> Self { +impl SkipList { + pub fn create(comparator: Arc, arena: ArenaRef) -> Self { Self { - node: None, - comp: comparator, + height: 0, + num: 0, + head: Node::create_head(arena.clone()), + tail: Node::create_tail(), + cmp: comparator, + arena, + } + } + + pub fn insert(&mut self, key: Slice) -> Result<()> { + // TODO 这里是否可以优化 + if self.contains(&key) { + return Ok(()); + } + if self.num == 0 { + self.insert_ele0(key) + } else { + unsafe { + self.insert_elen(key) + } + } + } + + #[inline] + fn insert_ele0(&mut self, key: Slice) -> Result<()> { + let level = rand_level(); + debug!("insert {}, level: {}", &key, level); + let node = unsafe { Node::create(key, level, self.arena.clone()) }; + // head bind node + // TODO, use macro to expand for-loop + unsafe { + (&mut *node).level = level; + (&mut *self.head).level = level; + for l in 0..level { + (&mut *self.head).set_node(l, node); + (&mut *node).set_node(l, self.tail); + } } + self.height = level; + self.num = 1; + return Ok(()); } - pub fn insert(&mut self, _seq_no: usize, _key: &Slice) -> Result<()> { - todo!() + unsafe fn insert_elen(&mut self, key: Slice) -> Result<()> { + let mut current = self.head; + let node_height = rand_level(); + let node_top_level = node_height - 1; + debug!("insert {}, level: {}", &key, node_height); + let node_ptr = unsafe { + Node::create(key, node_height, self.arena.clone()) + }; + let node = unsafe { &mut *node_ptr }; + // loop from highest level to 0 + for l in (0..self.height).rev() { + 'inner_loop: loop { + let ele_ptr = unsafe { (&*current).get_node(l) }; + let ele = unsafe { &mut *ele_ptr }; + if ele.is_tail() { + if l <= node_top_level { + // ele is tail node, add node to last + (&mut *current).set_node(l, node_ptr); + node.set_node(l, self.tail); + debug!("bind: {} before: {}, after: , at level: {}", + node.key.as_ref().unwrap(), + (&*current).key.as_ref().unwrap(), + l); + } + break 'inner_loop; + } else { + match self.cmp.compare(node.key.as_ref().unwrap(), ele.key.as_ref().unwrap()) { + Some(Ordering::Less) => { + // node higher than current level at ele + if node_top_level >= l { + (&mut *current).set_node(l, node_ptr); + node.set_node(l, ele_ptr); + if (&*current).is_head() { + debug!("bind: {} before: , after: {}, at level: {}", + node.key.as_ref().unwrap(), + ele.key.as_ref().unwrap(), + l); + } else { + debug!("bind: {} before: {}, after: {}, at level: {}", + node.key.as_ref().unwrap(), + (&*current).key.as_ref().unwrap(), + ele.key.as_ref().unwrap(), + l); + } + } + break 'inner_loop; + } + Some(Ordering::Greater) => { + current = ele; + } + Some(Ordering::Equal) => { + // ignore equals + return Ok(()); + } + None => { + return Err(Status::wrapper(LevelError::KInvalidArgument, "key not comparable".into())); + } + } + } + } + } + // if head level is less than new node, then fix head node height + if self.height < node_height { + for l in (self.height()..node_height).rev() { + (&mut *self.head).set_node(l, node_ptr); + node.set_node(l, self.tail); + } + self.height = node_height; + } + self.num += 1; + Ok(()) } - pub fn contains(&self, _key: &Slice) -> bool { - todo!() + #[macro_use] + pub fn contains(&self, key: &Slice) -> bool { + debug!("================== begin contains, key: {} ==================", key); + if self.num == 0 { + return false; + } + unsafe { + let mut current = unsafe { &*self.head }; + for level in (0..self.height).rev() { + 'a_loop: loop { + let ele_ptr = current.get_node(level); + let ele = &*ele_ptr; + if ele.is_tail() { + // tail node + if level == 0 { + debug!("next is tail, return false"); + return false; + } else { + debug!("next is tail, continue"); + break 'a_loop; + } + } + { + debug!("node: {} at level: {}", ele.key.as_ref().unwrap(), level) + } + match self.cmp.compare(key, ele.key.as_ref().unwrap()) { + None => return false, + Some(Ordering::Equal) => return true, + Some(Ordering::Less) => { + // break loop, decrease the level + break; + } + Some(Ordering::Greater) => { + if current.level() == 0 { + return false; + } + current = ele; + } + }; + } + } + } + // can not found in all level + false + } + + #[inline] + pub fn max_height(&self) -> usize { + MAX_LEVEL } - pub fn get_max_height(&self) -> usize { - todo!() + #[inline] + pub fn height(&self) -> usize { + self.height + } + + #[inline] + pub fn len(&self) -> usize { + self.num + } + + #[inline] + pub fn iter(&self) -> Iter { + Iter::create(&self) + } + + fn rnd_level(&self) -> usize { + let mut level = 1; + for _ in 1..MAX_LEVEL { + if random() { + level += 1; + } + } + level + } +} + +impl ToString for SkipList { + fn to_string(&self) -> String { + let mut tree = String::with_capacity(1024); + // calculate each item width + let mut widths = Vec::with_capacity(tree.len()); + self.iter().for_each(|s| { + widths.push(s.size()); + }); + // print value list + if self.num > 0 { + unsafe { + let mut node = &*((&*self.head).get_node(0)); + tree.push_str("[head]"); + while !node.is_head_or_tail() { + tree.push_str(" -> "); + tree.push_str(node.key.as_ref().unwrap().as_str()); + let level_str = format!("({})", node.level); + tree.push_str(level_str.as_str()); + node = &*node.get_node(0); + } + } + } + tree.push_str("-> [tail]"); + format!("height: {}, num: {}\n {}", self.height, self.num, tree) + } +} + + +impl Node { + #[inline] + fn create(src: Slice, level: usize, arena: ArenaRef) -> RawNode { + let key = src.copy_with_arena(arena.clone()); + let node = box Self { + key: Some(key), + next_elems: allocate_next_elems(arena), + level, + }; + Box::into_raw(node) + } + + #[inline] + fn create_head(arena: ArenaRef) -> RawNode { + let node = box Self { + key: None, + next_elems: allocate_next_elems(arena), + level: MAX_LEVEL, + }; + Box::into_raw(node) + } + + #[inline] + fn create_tail() -> RawNode { + let node = box Self { + key: None, + next_elems: null_mut(), + level: 0, + }; + Box::into_raw(node) + } + + #[inline] + #[must_use] + fn is_head_or_tail(&self) -> bool { + self.key.is_none() + } + + #[inline] + #[must_use] + fn is_tail(&self) -> bool { + self.key.is_none() && self.level == 0 + } + + #[inline] + #[must_use] + fn is_head(&self) -> bool { + self.key.is_none() && self.level > 0 + } + + + #[inline] + fn level(&self) -> usize { + self.level + } + + #[inline] + #[must_use] + unsafe fn get_node(&self, level: usize) -> RawNode { + assert!(level < MAX_LEVEL); + self.next_elems.offset(level as isize).read() + } + + #[inline] + unsafe fn set_node(&mut self, level: usize, node: RawNode) { + assert!(level < MAX_LEVEL); + self.next_elems.offset(level as isize).write(node); + } +} + +fn rand_level() -> usize { + let mut level = 1_usize; + while random::() { + level += 1; + if level >= MAX_LEVEL { + break; + } + } + level +} + +fn allocate_next_elems(arena: ArenaRef) -> *mut RawNode { + // RawNode is a raw ptr + assert_eq!(size_of::(), size_of::()); + // allocate next_elems to 8 capacity array + let elems_size = size_of::() * MAX_LEVEL; + let mut lock = arena.lock().expect("lock arena"); + let elems_ptr = lock.allocate(elems_size); + // transmute raw ptr to RawNode ptr + unsafe { + mem::transmute(elems_ptr.as_ptr()) + } +} + +#[inline] +fn min_max(a: usize, b: usize) -> (usize, usize) { + if a < b { + (a, b) + } else { + (b, a) + } +} + +// 'b lifetime is bigger than 'a +impl<'a, Cmp: ComparatorTrait> Iter<'a, Cmp> { + fn create(list: &'a SkipList) -> Self { + Self { + list, + node: list.head, + } + } +} + +impl<'a, Cmp: ComparatorTrait> Iterator for Iter<'a, Cmp> { + type Item = &'a Slice; + + #[inline] + fn next(&mut self) -> Option { + unsafe { + if (&*self.node).is_tail() { + return None; + } else { + self.node = (&*self.node).get_node(0); + } + (&*self.node).key.as_ref() + } } } \ No newline at end of file diff --git a/src/db/skip_list_test.rs b/src/db/skip_list_test.rs new file mode 100644 index 0000000000000000000000000000000000000000..287d18c0afee0507905443f37dcbb61781fd3cd9 --- /dev/null +++ b/src/db/skip_list_test.rs @@ -0,0 +1,59 @@ +mod test { + + + + + + + + + + + + + #[test] + fn test_add() -> Result<()> { + let cmp = Arc::new(BytewiseComparatorImpl::default()); + let arena = Arc::new(Mutex::new(Arena::default())); + let mut list = DefaultSkipList::create(cmp, arena); + let len = 10; + for i in 0..len { + list.insert(format!("key_{}", i).into()).expect("insert ok"); + } + assert_eq!(10, list.len(), "expect 10, but actually is: {}", list.len()); + debug!("{}", list.to_string()); + for i in 0..len { + let key: Slice = format!("key_{}", i).into(); + debug!("contains key: {}", key); + assert!(list.contains(&key), "contains key: {}", key); + } + list.iter().for_each(|slice| { + debug!("slice: {}", slice.as_str()) + }); + Ok(()) + } + + #[test] + fn test_rnd_add() -> Result<()> { + let cmp = Arc::new(BytewiseComparatorImpl::default()); + let arena = Arc::new(Mutex::new(Arena::default())); + let mut list = DefaultSkipList::create(cmp, arena); + let len = 10; + let mut rnd = rand::thread_rng(); + let mut set = HashSet::new(); + for _i in 0..10 { + let j = rnd.gen_range(0..len); + let key = format!("key_{}", j); + set.insert(key.clone()); + list.insert(key.into())?; + debug!("skiplist: {}", list.to_string()); + } + assert_eq!(set.len(), list.len(), "list length must eq: {}", list.len()); + set.iter().for_each(|key| { + let c = list.contains(&key.clone().into()); + assert!(c, "must contains key: {}", key) + }); + + Ok(()) + } +} \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 064907a4348b12aece079868fa6b01bcf2a3552a..b3fbd97594f29468b6844bbf353f39810fe5a00c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,8 @@ #![feature(box_syntax)] +#![feature(let_else)] +#![feature(generic_associated_types)] + +extern crate core; mod db; mod table; diff --git a/src/traits/mod.rs b/src/traits/mod.rs index 47c662e6d0774ea213fb7838379a7ae1e0073d24..a1332ef7548ed05a5d266799f00c677e3720015f 100644 --- a/src/traits/mod.rs +++ b/src/traits/mod.rs @@ -4,5 +4,5 @@ pub mod comparator_trait; pub mod coding_trait; pub mod filter_policy_trait; -use std::rc::Rc; + pub use iterator::DataIterator; diff --git a/src/util/arena.rs b/src/util/arena.rs index 3f46dcb401207e891698b81eb28388c407d88531..d239b485d6d0fa0bd97e646cd1615bec8000a49d 100644 --- a/src/util/arena.rs +++ b/src/util/arena.rs @@ -1,10 +1,21 @@ +use std::slice; use std::alloc::{alloc, dealloc, Layout}; use std::ptr::NonNull; -use std::slice; +use std::sync::{Arc, Mutex}; + +use crate::util::slice::Slice; // Arena block size const ARENA_BLOCK_SIZE: usize = 4096; +pub type ArenaRef = Arc>; + + +/// +pub trait ArenaAllocLike { + fn copy_with_arena(&self, arena: ArenaRef) -> Self; +} + pub struct Arena { alloc_ptr: Option>, alloc_bytes_remaining: usize, @@ -24,7 +35,6 @@ impl Default for Arena { } impl Arena { - /// 申请一块内存 /// /// # Arguments @@ -102,4 +112,16 @@ impl Drop for Arena { } } } +} + +impl ArenaAllocLike for Slice { + fn copy_with_arena(&self, arena: ArenaRef) -> Self { + unsafe { + let mut lock_guard = arena.lock().unwrap(); + let dst = lock_guard.allocate(self.len()); + let src = &**self; + dst.copy_from_slice(src); + Slice::from_raw_parts(dst.as_mut_ptr(), self.len()) + } + } } \ No newline at end of file diff --git a/src/util/arena_test.rs b/src/util/arena_test.rs index dbc8cf12ac94c5b61572f5b0f410a2c09f9a3fa3..fc800e197b6813b812c3e9e9f2cf0c9fd5ca7a3b 100644 --- a/src/util/arena_test.rs +++ b/src/util/arena_test.rs @@ -1,4 +1,4 @@ -use crate::util::Arena; + #[test] fn test_memory_usage() { diff --git a/src/util/coding.rs b/src/util/coding.rs index 125c042b5335a3bb6b265816d97a2dd2b17492df..f27bc7e55a4f207fd262cfaad88d74cc0f14ba63 100644 --- a/src/util/coding.rs +++ b/src/util/coding.rs @@ -28,7 +28,7 @@ macro_rules! varint { pub struct Coding {} impl CodingTrait for Coding { - fn put_fixed32(mut dst: &mut String, value: u32) { + fn put_fixed32(dst: &mut String, value: u32) { let mut buf: [u8; 4] = [0, 0, 0, 0]; Self::encode_fixed32(value, &mut buf, 0); for b in buf.iter() { @@ -36,7 +36,7 @@ impl CodingTrait for Coding { } } - fn put_fixed64(mut dst: &mut String, value: u64) { + fn put_fixed64(dst: &mut String, value: u64) { let mut buf: [u8; 8] = [0, 0, 0, 0, 0, 0, 0, 0]; Self::encode_fixed64(value, &mut buf, 0); for b in buf.iter() { @@ -48,7 +48,7 @@ impl CodingTrait for Coding { varint!(u64,encode_varint64); - fn put_varint32(mut dst: &mut String, value: u32) { + fn put_varint32(dst: &mut String, value: u32) { let mut buf: [u8; 4] = [0, 0, 0, 0]; Self::encode_fixed32(value, &mut buf, 0); for b in buf.iter() { @@ -56,7 +56,7 @@ impl CodingTrait for Coding { } } - fn put_varint64(mut dst: &mut String, value: u64) { + fn put_varint64(dst: &mut String, value: u64) { let mut buf: [u8; 8] = [0, 0, 0, 0, 0, 0, 0, 0]; Self::encode_fixed64(value, &mut buf, 0); for b in buf.iter() { @@ -125,7 +125,7 @@ impl CodingTrait for Coding { len } - fn encode_fixed32(mut value: u32, buf: &mut [u8], mut offset: usize) -> usize { + fn encode_fixed32(value: u32, buf: &mut [u8], mut offset: usize) -> usize { buf[offset] = value as u8; offset += 1; buf[offset] = (value >> 8) as u8; @@ -137,7 +137,7 @@ impl CodingTrait for Coding { offset } - fn encode_fixed64(mut value: u64, buf: &mut [u8], mut offset: usize) -> usize { + fn encode_fixed64(value: u64, buf: &mut [u8], mut offset: usize) -> usize { buf[offset] = value as u8; offset += 1; buf[offset] = (value >> 8) as u8; @@ -196,7 +196,7 @@ macro_rules! encoding_impl { /// let value: u32 = 65534; /// let offset = value.varint(&mut buf, 0); /// ``` - fn varint(self, buf: &mut [u8], mut offset: usize) -> usize { + fn varint(self, buf: &mut [u8], offset: usize) -> usize { Coding::$VAR_NAME (self, buf, offset) } /// 定长正整数编码 @@ -215,7 +215,7 @@ macro_rules! encoding_impl { /// let value: u32 = 65534; /// let offset = value.fixedint(&mut buf, 0); /// ``` - fn fixedint(self, buf: &mut [u8], mut offset: usize) -> usize { + fn fixedint(self, buf: &mut [u8], offset: usize) -> usize { Coding::$FIXED_NAME (self, buf, offset) } } diff --git a/src/util/coding_test.rs b/src/util/coding_test.rs index 69ed74443b2944a53945762995ab444fc426a9a6..fd56327dc27451f7ae8f543221dc3f5035f463d4 100644 --- a/src/util/coding_test.rs +++ b/src/util/coding_test.rs @@ -1,7 +1,7 @@ mod test { - use crate::traits::coding_trait::{Coding32, Coding64, CodingTrait}; - use crate::util::slice::Slice; - use crate::util::coding::{Coding}; + + + #[test] fn test_put_fixed32() { diff --git a/src/util/comparator.rs b/src/util/comparator.rs index 71560febc43ae7dcc5fa88e950541c95b55666a6..0417c1085eef02b4b98ca8bc09d02d4e7cba00bb 100644 --- a/src/util/comparator.rs +++ b/src/util/comparator.rs @@ -62,7 +62,6 @@ impl ComparatorTrait for BytewiseComparatorImpl { } let shortest_separator: &[u8] = &start_char_vec[0..diff_index+1]; - let shortest_separator_val: String= Slice::from_buf(shortest_separator).into(); shortest_separator_val } diff --git a/src/util/comparator_test.rs b/src/util/comparator_test.rs index fcfb0347d7ebc468869edacd7f4f8abd3a42660a..234ccb8f50d8362f1d9ab68967c59e7882d1b116 100644 --- a/src/util/comparator_test.rs +++ b/src/util/comparator_test.rs @@ -1,10 +1,10 @@ mod test { - use std::cmp::Ordering; - use std::io::Write; - use crate::traits::comparator_trait::ComparatorTrait; - use crate::util::comparator::{BytewiseComparatorImpl, InternalKeyComparator}; - use crate::util::slice::Slice; + + + + + #[test] fn test_bytewise_comparator_impl_get_name() { diff --git a/src/util/const.rs b/src/util/const.rs index a8ffcf84196499637a80e70873f174ae079929b0..4ae23686e47641f2aeab78fe093ba0b9195fa2d2 100644 --- a/src/util/const.rs +++ b/src/util/const.rs @@ -5,3 +5,5 @@ pub const COLON_WHITE_SPACE: &'static str = ": "; /// hash 的默认seed: 0xbc9f1d34 pub const HASH_DEFAULT_SEED: u32 = 0xbc9f1d34; + +pub const DEBUG_ENABLE: bool = true; diff --git a/src/util/crc_test.rs b/src/util/crc_test.rs index c67c0dbee1d29b15dac956fdb1d043c3ac248d54..e304450dfccb49391a1336031c92324d1b0e0d27 100644 --- a/src/util/crc_test.rs +++ b/src/util/crc_test.rs @@ -1,5 +1,5 @@ -use crate::util::crc::{AsCrc, CRC, ToMask}; -use crate::util::slice::Slice; + + #[test] fn test_crc() { diff --git a/src/util/debug.rs b/src/util/debug.rs new file mode 100644 index 0000000000000000000000000000000000000000..464919b011763959d8fec31668f7ebd04115d0a6 --- /dev/null +++ b/src/util/debug.rs @@ -0,0 +1,26 @@ + + + +// #[cfg(feature = "debug-macro")] +#[cfg(CORE_DEBUG = "true")] +#[macro_export] +macro_rules! debug { + () => { + std::io::stdout().write("\n".as_bytes()).unwrap(); + }; + ($($arg:tt)*) => {{ + use std::io::Write; + std::io::stdout().write(format!($($arg)*).as_bytes()); + debug!(); + }}; +} + +// #[cfg(not(feature = "debug-macro"))] +#[cfg(not(CORE_DEBUG = "true"))] +#[macro_export] +macro_rules! debug { + () => { + }; + ($($arg:tt)*) => {{ + }}; +} \ No newline at end of file diff --git a/src/util/filter_policy.rs b/src/util/filter_policy.rs index 5a87f2e8163ed800895668f56bb693336abdc961..8ddd6514dd7dda1f118e4c92913c1ca80f0f791a 100644 --- a/src/util/filter_policy.rs +++ b/src/util/filter_policy.rs @@ -1,6 +1,6 @@ use std::ops::Mul; use crate::traits::filter_policy_trait::{FilterPolicy}; -use crate::util::hash::{Hash, ToHash}; +use crate::util::hash::{ToHash}; use crate::util::slice::Slice; pub struct BloomFilterPolicy { @@ -43,7 +43,7 @@ impl FilterPolicy for BloomFilterPolicy { String::from("leveldb.BuiltinBloomFilter2") } - fn create_filter(&self, keys: Slice, n: u32, dst: String) -> String { + fn create_filter(&self, _keys: Slice, _n: u32, _dst: String) -> String { // 根据指定的参数创建过滤器，并返回结果， 结果为dst的原始内容 + append结果。 // 参数keys[0,n-1]包含依据用户提供的comparator排序的key列表--可重复， // 并把根据这些key创建的filter追加到 dst中。 @@ -51,7 +51,7 @@ impl FilterPolicy for BloomFilterPolicy { todo!() } - fn key_may_match(key: &Slice, filter: &Slice) -> bool { + fn key_may_match(_key: &Slice, _filter: &Slice) -> bool { todo!() } } \ No newline at end of file diff --git a/src/util/filter_policy_test.rs b/src/util/filter_policy_test.rs index b49448ff809bba65c38afa91d2405faf52bcf1e0..c5175de3dd759413ff075b045b6e131930420870 100644 --- a/src/util/filter_policy_test.rs +++ b/src/util/filter_policy_test.rs @@ -1,14 +1,14 @@ -use std::ptr::null; -use crate::util::bloom_filter; -use crate::util::filter_policy::BloomFilterPolicy; + + + #[test] fn test_new() { - let bloom_filter = BloomFilterPolicy::new(8); + let _bloom_filter = BloomFilterPolicy::new(8); println!("hash:{}", "a"); // assert_eq!(bloom_filter, null()); - let bloom_filter = BloomFilterPolicy::new(800); + let _bloom_filter = BloomFilterPolicy::new(800); println!("hash:{}", "a"); } \ No newline at end of file diff --git a/src/util/hash.rs b/src/util/hash.rs index c9fce802b0d7eada4f3ad1334d8ce72d6da1881c..3d7f25756a5584dd4414f44ce73aeb6514b666fe 100644 --- a/src/util/hash.rs +++ b/src/util/hash.rs @@ -4,9 +4,9 @@ use std::slice as stds; use crate::traits::coding_trait::CodingTrait; use crate::util::coding::Coding; -use crate::util::crc::AsCrc; + use crate::util::r#const::HASH_DEFAULT_SEED; -use crate::util::slice; + use crate::util::slice::Slice; /// 一种可以计算 hash 的特质 diff --git a/src/util/hash_test.rs b/src/util/hash_test.rs index 81dce89e612992852a6e784b4a32915532de6bfb..e0961d0a79bea34ab6e58ee5b867a60ad4a6fcce 100644 --- a/src/util/hash_test.rs +++ b/src/util/hash_test.rs @@ -1,7 +1,7 @@ -use crate::util::hash::{Hash, ToHash}; -use crate::util::r#const::HASH_DEFAULT_SEED; -use crate::util::slice::Slice; -use std::slice; + + + + #[test] fn test_hash() { diff --git a/src/util/histogram_test.rs b/src/util/histogram_test.rs index a4f01e76a5778a4a0688ec0b954567892c2e9d57..74e9033c7cf72a8e2a7e093a70e11875835abc03 100644 --- a/src/util/histogram_test.rs +++ b/src/util/histogram_test.rs @@ -1,6 +1,6 @@ mod test{ - use crate::util::histogram::Histogram; + #[test] fn test_add() { diff --git a/src/util/mod.rs b/src/util/mod.rs index b3e53d7a409d35c4e4aa0f68e21bf58ae1bc91bc..19c30979b26e275135e19ff3e4ceb13344a313e2 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,9 +1,7 @@ -use std::rc::Rc; use std::result; pub use arena::Arena; -use crate::util::comparator::{BytewiseComparatorImpl, InternalKeyComparator}; use crate::util::status::Status; /// 常量定义 @@ -34,6 +32,7 @@ mod hash_test; mod mutex_lock; mod mutex_lock_test; pub mod options; +pub mod debug; /// 定义别名 pub type Result = result::Result; diff --git a/src/util/mutex_lock.rs b/src/util/mutex_lock.rs index 84e6d65acfc23d92adb8510dbf052674c53f70e1..c6c4d77632e91041d8981a2e469435f0c4b5523a 100644 --- a/src/util/mutex_lock.rs +++ b/src/util/mutex_lock.rs @@ -1,4 +1,4 @@ -use std::ops::Deref; + use std::sync::{Arc, LockResult, Mutex, MutexGuard, TryLockResult}; pub struct Lock { diff --git a/src/util/mutex_lock_test.rs b/src/util/mutex_lock_test.rs index 7273a71d7ae2bd90dbf01f4a186800a4cc51a2b4..548871fd396d9856bb5b1d04346973db0d5d84f8 100644 --- a/src/util/mutex_lock_test.rs +++ b/src/util/mutex_lock_test.rs @@ -1,7 +1,7 @@ mod test { - use std::thread; + - use crate::util::mutex_lock::MutexLock; + #[test] fn test() { diff --git a/src/util/slice.rs b/src/util/slice.rs index 26ea8b17359ad73bd5484abfe492ba504dc74da5..7a14a22e84baa86a51d692bef75c04f33fe07735 100644 --- a/src/util/slice.rs +++ b/src/util/slice.rs @@ -1,6 +1,8 @@ use std::mem; use std::borrow::Cow; use std::cmp::Ordering; +use std::fmt::{Display, Formatter}; +use std::mem::ManuallyDrop; use std::ops::Deref; #[derive(Debug)] @@ -40,6 +42,12 @@ impl Slice { } } + #[inline] + pub unsafe fn from_raw_parts(ptr: *mut u8, len: usize) -> Self { + let data = Vec::from_raw_parts(ptr, len, len); + Self { data } + } + /// 获取 slice 长度 #[inline] pub fn size(&self) -> usize { @@ -96,6 +104,10 @@ impl Slice { } } + pub fn as_str(&self) -> &str { + let s = self.as_ref(); + std::str::from_utf8(s).unwrap() + } } impl<'a> Slice { @@ -195,3 +207,16 @@ impl Deref for Slice { } } +impl Display for Slice { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + unsafe { + let string = ManuallyDrop::new( + String::from_raw_parts( + self.as_ptr() as *mut u8, + self.data.len(), + self.data.capacity()) + ); + f.write_str(string.as_str()) + } + } +} diff --git a/src/util/slice_test.rs b/src/util/slice_test.rs index a2e0ec5780b3916957f2eeb2538eeb116ff81c7f..75ae7a7068883ba9c485a1b4adfb3d56fbff321f 100644 --- a/src/util/slice_test.rs +++ b/src/util/slice_test.rs @@ -1,5 +1,5 @@ mod test { - use std::cmp::Ordering; + use crate::util::slice::Slice; #[test] @@ -88,7 +88,7 @@ mod test { #[test] fn test_merge2() { let mut a0 = Slice::from("123"); - let mut a2 = Slice::from("456"); + let a2 = Slice::from("456"); a0.merge(a2, None); assert_eq!(String::from("123456"), String::from(a0)); } diff --git a/src/util/status.rs b/src/util/status.rs index 8f5365d9405667b8c617185fd7f88b44041e84f6..afd40407648bab793e8c2953961a739ecb7470c3 100644 --- a/src/util/status.rs +++ b/src/util/status.rs @@ -2,7 +2,7 @@ use std::fmt::{Display, Formatter}; use std::io; use crate::util::r#const::COLON_WHITE_SPACE; use crate::util::slice::Slice; -use crate::util::status::LevelError::{KCorruption, KIOError, KInvalidArgument, KNotSupported, KNotFound, KOk, KBadRecord}; +use crate::util::status::LevelError::{KCorruption, KIOError, KInvalidArgument, KNotSupported, KNotFound, KOk, KBadRecord, KRepeatedRecord}; /// db 中的返回状态，将错误号和错误信息封装成Status类，统一进行处理。 /// 在 leveldb的实现里， 为了节省空间Status将返回码(code), 错误信息message及长度打包存储于一个字符串数组中， 来存储错误信息。 @@ -134,6 +134,7 @@ impl Status { KInvalidArgument => "Invalid argument: ", KIOError => "IO error: ", KBadRecord=> "wal bad record", + KRepeatedRecord => "repeated record" }; if self.err.is_ok() { @@ -180,6 +181,7 @@ pub enum LevelError { KInvalidArgument, KIOError, KBadRecord, + KRepeatedRecord, } impl LevelError { @@ -225,6 +227,10 @@ impl LevelError { } } + pub fn is_repeated_record(&self) -> bool { + matches!(self, KRepeatedRecord) + } + pub fn ok() -> Status { Status{ err: Default::default(), @@ -282,6 +288,14 @@ impl LevelError { } } + #[inline] + pub fn repeated_record(msg: Slice) -> Status { + Status { + err: KRepeatedRecord, + msg + } + } + /// 生成 LevelError.KIOError /// /// # Arguments @@ -339,6 +353,7 @@ impl TryFrom for LevelError { 4 => Ok(KInvalidArgument), 5 => Ok(KIOError), 6 => Ok(KBadRecord), + 7 => Ok(KRepeatedRecord), // all other numbers _ => Err(String::from(format!("Unknown code: {}", value))) } @@ -363,6 +378,7 @@ impl Display for LevelError { KInvalidArgument => "Invalid argument: ", KIOError => "IO error: ", KBadRecord => "wal bad record: ", + KRepeatedRecord => "repeated record: ", }; print.push_str(msg_type); diff --git a/src/util/status_test.rs b/src/util/status_test.rs index 2ac1030cd30292af50470c762a3b33658f52d5cd..90f66ec6cd8d8370d5ae120ceaad2860557e3279 100644 --- a/src/util/status_test.rs +++ b/src/util/status_test.rs @@ -1,8 +1,8 @@ mod test { - use crate::util::r#const::COLON_WHITE_SPACE; - use crate::util::slice::Slice; - use crate::util::status::{LevelError, Status}; + + + #[test] fn test_wraper() {