diff --git a/Cargo.toml b/Cargo.toml index b60d1d1ad18006583417c87958aab5397a240e70..8b4c9c3adce6834ada5f9fcad08271867a1107fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ jemallocator = "0.5" jemalloc-sys = {version = "0.5", features = ["stats"]} [dev-dependencies] -criterion = "0.3.0" +criterion = {version = "0.4.0", features = ["html_reports"]} crc32fast = "1.3.2" skiplist = "0.4.0" @@ -34,5 +34,5 @@ name = "skiplist_bench" harness = false [[bench]] -name = "skiplist_memory_useage" +name = "u32_shift" harness = false \ No newline at end of file diff --git a/README.md b/README.md index 97f5fa6da73f8a89f63107963f7dd8a07545b882..54ba069ccedc24e004b2b349bfc2dcb50a1abef9 100644 --- a/README.md +++ b/README.md @@ -71,32 +71,32 @@ RUSTFLAGS='--cfg CORE_DEBUG="false"' cargo build --release ### 1.1.0 版本, 完成基础零部件 -| 功能模块 | 完成人 | 进度 | -|----------------------------------------------------------------------------------|----------------------|------| -| util.Options(ReadOptions, WriteOptions) | kazeseiriou,wangboo | 0% | -| util.ENV(WritableFile, SequentialFile, RandomAccessFile, FileLock) | lxd5866 | 0% | -| util.Logger/Log日志库 | peach | 50% | -| table.Block, BlockBuilder, FilterBlockBuilder | colagy | 0% | -| FilterBlock, FilterBlockReader | colagy | 0% | -| table.format(Footer, BlockHandle) | 半支烟 | 20% | -| db.dbformat(InternalKeyComparator, InternalFilterPolicy, LookupKey, InternalKey) | 半支烟 | 20% | -| db.SkipList | wangboo | 100% | -| table.Iterator(DBIter, MergingIterator, TwoLevelIterator...) | kazeseiriou | 0% | -| IteratorWrapper | kazeseiriou | 0% | -| db.MemTable(MemTable, MemTableIterator) | wangboo,tzcyujunyong | 20% | -| SSTable | fengyang | 0% | -| table.Table | peach,tzcyujunyong | | -| db.leveldb_util | wangboo | 0% | -| db.log_format | wangboo | 90% | -| db.LogReader | wangboo | 90% | -| db.LogWriter | wangboo | 90% | -| db.TableCache | colagy | 10% | -| LinkedList | fengyang | 60% | -| db.VersionEdit(Tag, VersionEdit, FileMetaData) | fengyang | 20% | -| db.VersionSet(Version, LevelFileNumIterator, SaverState) | fengyang | 20% | -| WriteBatch | tzcyujunyong,wangboo | 50% | -| | 半支烟 | 40% | - +| 功能模块 | 完成人 | 进度 | +|------------------------------------------------------------------------------------|------------------------|--------| +| util.Options(ReadOptions, WriteOptions) | kazeseiriou,wangboo | 0% | +| util.ENV(WritableFile, SequentialFile, RandomAccessFile, FileLock) | lxd5866 | 0% | +| util.Logger/Log日志库 | peach | 50% | +| table.Block, BlockBuilder, FilterBlockBuilder | colagy | 0% | +| FilterBlock, FilterBlockReader | colagy | 0% | +| table.format(Footer, BlockHandle) | 半支烟 | 20% | +| db.dbformat(InternalKeyComparator, InternalFilterPolicy, LookupKey, InternalKey) | 半支烟 | 20% | +| db.SkipList | wangboo | 100% | +| table.Iterator(DBIter, MergingIterator, TwoLevelIterator...) | kazeseiriou | 0% | +| IteratorWrapper | kazeseiriou | 0% | +| db.MemTable(MemTable, MemTableIterator) | wangboo,tzcyujunyong | 20% | +| SSTable | fengyang | 0% | +| table.Table | peach,tzcyujunyong | | +| db.leveldb_util | wangboo | 0% | +| db.log_format | wangboo | 90% | +| db.LogReader | wangboo | 90% | +| db.LogWriter | wangboo | 90% | +| db.TableCache | colagy | 10% | +| LinkedList | fengyang | 60% | +| db.VersionEdit(Tag, VersionEdit, FileMetaData) | fengyang | 20% | +| db.VersionSet(Version, LevelFileNumIterator, SaverState) | fengyang | 20% | +| WriteBatch | tzcyujunyong,wangboo | 50% | +| | 半支烟 | 90% | +| ---------------------------------------------------------------------------------- | ---------------------- | ------ | diff --git a/benches/skiplist_memory_useage.rs b/benches/skiplist_memory_useage.rs deleted file mode 100644 index deaee476e88cf183e8a7d47a9f36ab55c30dfecf..0000000000000000000000000000000000000000 --- a/benches/skiplist_memory_useage.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::ffi::{c_char, c_void}; -use std::ptr::{null, null_mut}; -use std::sync::{Arc, Mutex}; -use skiplist::OrderedSkipList; -use level_db_rust::db::skip_list::SkipList; -use level_db_rust::util::Arena; -use level_db_rust::util::arena::ArenaRef; -use level_db_rust::util::comparator::BytewiseComparatorImpl; -use level_db_rust::util::unsafe_slice::TryIntoUnsafeSlice; - -#[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; - -extern "C" fn write_cb(_: *mut c_void, message: *const c_char) { - print!("{}", String::from_utf8_lossy(unsafe { - std::ffi::CStr::from_ptr(message as *const i8).to_bytes() - })); -} - -fn mem_print() { - unsafe { jemalloc_sys::malloc_stats_print(Some(write_cb), null_mut(), null()) } -} - -fn bench_default_skiplist(mut list: SkipList, arena: ArenaRef, record_count: usize) { - for j in 0..record_count { - let value = format!("key_{}", j); - list.insert(value.try_into_unsafe_slice(arena.clone()).unwrap()).unwrap(); - } - println!("bench_default_skiplist: "); - mem_print(); -} - -fn bench_skiplist_v_0_4_0(mut list: OrderedSkipList, record_count: usize) { - for j in 0..record_count { - let value = format!("key_{}", j); - list.insert(value.clone()); - } - println!("bench_skiplist_v_0_4_0: "); - mem_print(); -} - -fn main() { - let record_count = 100 * 1024; - // let cmp = Arc::new(BytewiseComparatorImpl::default()); - // let arena = Arc::new(Mutex::new(Arena::default())); - // let list = SkipList::create(cmp, arena.clone()); - // bench_default_skiplist(list, arena, record_count); - - let list: OrderedSkipList = unsafe { - OrderedSkipList::with_comp(|a: &String, b: &String| { - a.cmp(b) - }) - }; - bench_skiplist_v_0_4_0(list, record_count); -} \ No newline at end of file diff --git a/benches/u32_shift.rs b/benches/u32_shift.rs new file mode 100644 index 0000000000000000000000000000000000000000..f9a2eb5835896e2db9ea149425cea5e0c91efe6e --- /dev/null +++ b/benches/u32_shift.rs @@ -0,0 +1,46 @@ +use std::{mem, slice}; +use std::alloc::{alloc, Layout}; +use std::io::Write; + +use criterion::{Criterion, criterion_group, criterion_main}; +use level_db_rust::debug; + +pub fn u32_shift_bench(c: &mut Criterion) { + let mut data = [0_u8; 4]; + let mut buf = data.as_mut_slice(); + let value = 12345678_u32; + let mut g = c.benchmark_group("u32_shift"); + + g.bench_function("to_ne_bytes", |g| { + g.iter(|| { + buf.write(&value.to_be_bytes()).unwrap(); + }); + }); + buf = data.as_mut_slice(); + buf.fill(0); // reset + debug!("is big endian: {}", cfg!(target_endian = "big")); + g.bench_function("raw_write", |g| { + g.iter(|| { + unsafe { + if cfg!(target_endian = "big") { + (buf.as_mut_ptr() as *mut u32).write(value); + } else { + (buf.as_mut_ptr() as *mut u32).write(value.swap_bytes()); + } + } + }); + }); + buf = data.as_mut_slice(); + buf.fill(0); // reset + g.bench_function("shift_bytes", |g| { + g.iter(|| { + buf[0] = ((value >> 0) & 0xff) as u8; + buf[1] = ((value >> 1) & 0xff) as u8; + buf[2] = ((value >> 2) & 0xff) as u8; + buf[3] = ((value >> 3) & 0xff) as u8; + }); + }); +} + +criterion_group!(benches, u32_shift_bench); +criterion_main!(benches); \ No newline at end of file diff --git a/src/db/skip_list.rs b/src/db/skip_list.rs index a043adc40b514eb04cba38e9c502878538f17354..4d13c1212798c675392934b67f8a6930f27153eb 100644 --- a/src/db/skip_list.rs +++ b/src/db/skip_list.rs @@ -5,15 +5,15 @@ use std::ptr::null_mut; use std::sync::{Arc, RwLock}; use rand::prelude::*; + use crate::debug; use crate::traits::comparator_trait::Comparator; use crate::traits::DataIterator; - -use crate::util::arena::ArenaRef; use crate::util::{Arena, Result}; +use crate::util::arena::ArenaRef; use crate::util::slice::Slice; -use crate::util::unsafe_slice::UnsafeSlice; use crate::util::status::{LevelError, Status}; +use crate::util::unsafe_slice::UnsafeSlice; type RawNode = *mut Node; @@ -213,6 +213,10 @@ impl SkipList { false } + unsafe fn find_eq_or_greater>(&self, key: &R) -> Option { + todo!() + } + #[inline] pub fn max_height(&self) -> usize { MAX_LEVEL @@ -346,6 +350,28 @@ impl Node { assert!(level < MAX_LEVEL); self.next_elems.offset(level as isize).write(node); } + + /// 找到最后一个数据元素 + unsafe fn seek_to_last(&self) -> Option { + if self.is_tail() { + return None; + } + let mut pre = self; + let mut cur = &*self.next_top_node(); + loop { + if cur.is_tail() { + return Some(pre as *const Node as *mut Node); + } + pre = cur; + cur = &*cur.next_top_node(); + } + } + + /// 找到最上层的下一个元素 + #[inline] + unsafe fn next_top_node(&self) -> RawNode { + self.get_node(self.level - 1) + } } fn rand_level() -> usize { @@ -410,7 +436,6 @@ impl Iterator for Iter { } impl DataIterator for Iter { - #[inline] fn valid(&self) -> bool { unsafe { @@ -420,12 +445,16 @@ impl DataIterator for Iter { #[inline] fn seek_to_first(&mut self) { - self.current = self.head + self.current = unsafe { + (&*self.head).get_node(0) + } } #[inline] fn seek_to_last(&mut self) { - self.current = self.tail + unsafe { + self.current = (&*self.current).seek_to_last().unwrap_or(self.tail) + } } fn seek(&mut self, key: &Slice) { diff --git a/src/db/skip_list_test.rs b/src/db/skip_list_test.rs index 11ef4bdb36bd4808978971b25be809dd4d24ede7..11b4573a426568d7a099c9c5a1eb676eb10403f2 100644 --- a/src/db/skip_list_test.rs +++ b/src/db/skip_list_test.rs @@ -1,11 +1,21 @@ mod test { use std::collections::HashSet; + use std::env::args; + use std::ffi::{c_char, c_void}; + use std::ptr::{null, null_mut}; use std::sync::{Arc, Mutex}; + + use criterion::{Criterion, criterion_group, criterion_main}; use rand::Rng; + use skiplist::OrderedSkipList; + use crate::db::DefaultSkipList; + use crate::db::skip_list::SkipList; use crate::debug; use crate::util::Arena; + use crate::util::arena::ArenaRef; use crate::util::comparator::BytewiseComparatorImpl; + use crate::util::mem_debug::mem_print; use crate::util::Result; use crate::util::slice::Slice; use crate::util::unsafe_slice::TryIntoUnsafeSlice; @@ -55,4 +65,45 @@ mod test { Ok(()) } + + + fn default_skiplist(mut list: SkipList, arena: ArenaRef, record_count: usize) { + for j in 0..record_count { + let value = format!("key_{}", j); + list.insert(value.try_into_unsafe_slice(arena.clone()).unwrap()).unwrap(); + } + println!("bench_default_skiplist: "); + mem_print(); + } + + fn bench_skiplist_v_0_4_0(mut list: OrderedSkipList, record_count: usize) { + for j in 0..record_count { + let value = format!("key_{}", j); + list.insert(value.clone()); + } + println!("bench_skiplist_v_0_4_0: "); + mem_print(); + } + + #[test] + fn bench_default_skiplist() { + let record_count = 100 * 1024; + println!("bench default skiplist"); + let cmp = Arc::new(BytewiseComparatorImpl::default()); + let arena = Arc::new(Mutex::new(Arena::default())); + let list = SkipList::create(cmp, arena.clone()); + default_skiplist(list, arena, record_count); + } + + #[test] + fn bench_crate_skiplist() { + let record_count = 100 * 1024; + println!("bench crate skiplist"); + let list: OrderedSkipList = unsafe { + OrderedSkipList::with_comp(|a: &String, b: &String| { + a.cmp(b) + }) + }; + bench_skiplist_v_0_4_0(list, record_count); + } } \ No newline at end of file diff --git a/src/db/version_edit_test.rs b/src/db/version_edit_test.rs index 2c2efec824a1ce11feaae552d53f68ba7a402da2..c38442deb0740fad94b0b843fa51477add0d0980 100644 --- a/src/db/version_edit_test.rs +++ b/src/db/version_edit_test.rs @@ -5,7 +5,7 @@ mod test { use crate::util::slice::Slice; #[test] - fn test_Tag() { + fn test_tag() { let tag = Tag::kCompactPointer; assert_eq!(tag.get_value(), 5); diff --git a/src/lib.rs b/src/lib.rs index c8b17d305c244fcad61f59965d8b2277006ce518..3b67f8e10739053e8589706b3c4a62eac55e4717 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,4 @@ #![feature(box_syntax)] -#![feature(let_else)] -#![feature(generic_associated_types)] extern crate core; @@ -9,6 +7,9 @@ mod table; pub mod util; mod traits; +#[global_allocator] +static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; + mod test { #[test] diff --git a/src/util/comparator_test.rs b/src/util/comparator_test.rs index d6d311b8822e7fd7b289a6cabf13e5c2a2e2358e..b018bf6588bb960fb610a30288ec466f7bb815eb 100644 --- a/src/util/comparator_test.rs +++ b/src/util/comparator_test.rs @@ -93,7 +93,7 @@ mod test { // u8max 结尾 let mut u8_vec: Vec = vec![]; - u8_vec.write(&String::from("helloWorld").as_bytes().to_vec()); + u8_vec.write(&String::from("helloWorld").as_bytes().to_vec()).unwrap(); u8_vec.push(u8::MAX); let u8_array_str = String::from(Slice::from_buf(u8_vec.as_slice())); @@ -106,7 +106,7 @@ mod test { // u8max 开头 let mut u8_vec: Vec = vec![]; u8_vec.push(u8::MAX); - u8_vec.write(&String::from("helloWorld").as_bytes().to_vec()); + u8_vec.write(&String::from("helloWorld").as_bytes().to_vec()).unwrap(); let u8_max_str = String::from(Slice::from_buf(u8_vec.as_slice())); let comp = BytewiseComparatorImpl::default(); diff --git a/src/util/hash.rs b/src/util/hash.rs index 8aa51e2871d243dbe04b8dbda49d69ab763d2766..15a1a0328f2639f07d7d9da34d64e3e822cf8328 100644 --- a/src/util/hash.rs +++ b/src/util/hash.rs @@ -11,10 +11,9 @@ use crate::util::slice::Slice; /// 一种可以计算 hash 的特质 pub trait ToHash { - #[inline] + fn to_hash(&self) -> u32; - #[inline] fn to_hash_with_seed(&self, seed: u32) -> u32; } diff --git a/src/util/mem_debug.rs b/src/util/mem_debug.rs new file mode 100644 index 0000000000000000000000000000000000000000..bef8a837f271a8644bd1d2365d2e17c3dba126a4 --- /dev/null +++ b/src/util/mem_debug.rs @@ -0,0 +1,12 @@ +use std::ffi::{c_char, c_void}; +use std::ptr::{null, null_mut}; + +extern "C" fn write_cb(_: *mut c_void, message: *const c_char) { + print!("{}", String::from_utf8_lossy(unsafe { + std::ffi::CStr::from_ptr(message as *const i8).to_bytes() + })); +} + +pub fn mem_print() { + unsafe { jemalloc_sys::malloc_stats_print(Some(write_cb), null_mut(), null()) } +} \ No newline at end of file diff --git a/src/util/mod.rs b/src/util/mod.rs index ade5ddcd886631e5f833e69cd0ff137219873ded..d777f71c2cae50b6b95a7b45b3e83fa10a4f6c1f 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -41,6 +41,7 @@ pub mod debug; pub mod linked_list; mod linked_list_test; pub mod unsafe_slice; +pub mod mem_debug; /// 定义别名 pub type Result = result::Result; diff --git a/src/util/status_test.rs b/src/util/status_test.rs index a7dce285ea754dff0fae290c40055bb90acb86f2..c186ceea0ce27595744696872608d4a07997b029 100644 --- a/src/util/status_test.rs +++ b/src/util/status_test.rs @@ -100,7 +100,7 @@ mod test { } #[test] - fn test_level_error_toString() { + fn test_level_error_to_string() { // ok let status: Status = LevelError::ok(); assert_eq!("OK", status.to_string());