From f48b7b658f7fb1db09b423faaca09a9c6dee1c27 Mon Sep 17 00:00:00 2001 From: colagy Date: Wed, 29 Mar 2023 18:31:11 +0800 Subject: [PATCH 1/5] coding rewrite --- src/traits/coding_trait.rs | 4 +- src/util/coding.rs | 96 +++++++++++++++++--------------------- src/util/coding_test.rs | 5 ++ 3 files changed, 50 insertions(+), 55 deletions(-) diff --git a/src/traits/coding_trait.rs b/src/traits/coding_trait.rs index 3a0a49e..940e564 100644 --- a/src/traits/coding_trait.rs +++ b/src/traits/coding_trait.rs @@ -93,7 +93,7 @@ pub trait CodingTrait { /// ``` /// /// ``` - fn get_varint32(input: &mut Slice) -> u32; + fn get_varint32(input: &mut Slice) -> Option; /// 从slice的开头解码一个64位的变长整数, 并将slice的索引置于解码后的位置 /// /// # Arguments @@ -121,7 +121,7 @@ pub trait CodingTrait { /// ``` /// /// ``` - fn get_length_prefixed_slice(input: &mut Slice) -> Slice; + fn get_length_prefixed_slice(input: &mut Slice) -> Option; /// 32位变长正整数编码 /// /// # Arguments diff --git a/src/util/coding.rs b/src/util/coding.rs index 3a13b7d..9419aa2 100644 --- a/src/util/coding.rs +++ b/src/util/coding.rs @@ -28,34 +28,26 @@ impl CodingTrait for Coding { fn put_fixed32(dst: &mut [u8], mut offset: usize, value: u32) -> usize { let mut buf: [u8; 4] = [0, 0, 0, 0]; Self::encode_fixed32(value, &mut buf, 0); - dst[offset] = buf[0]; - offset += 1; - dst[offset] = buf[1]; - offset += 1; - dst[offset] = buf[2]; - offset += 1; - dst[offset] = buf[3]; + dst[0] = buf[0]; + dst[1] = buf[1]; + dst[2] = buf[2]; + dst[3] = buf[3]; + offset += 4; offset } fn put_fixed64(dst: &mut [u8], mut offset: usize, value: u64) -> usize { let mut buf: [u8; 8] = [0, 0, 0, 0, 0, 0, 0, 0]; Self::encode_fixed64(value, &mut buf, 0); - dst[offset] = buf[0]; - offset += 1; - dst[offset] = buf[1]; - offset += 1; - dst[offset] = buf[2]; - offset += 1; - dst[offset] = buf[3]; - offset += 1; - dst[offset] = buf[4]; - offset += 1; - dst[offset] = buf[5]; - offset += 1; - dst[offset] = buf[6]; - offset += 1; - dst[offset] = buf[7]; + dst[0] = buf[0]; + dst[1] = buf[1]; + dst[2] = buf[2]; + dst[3] = buf[3]; + dst[4] = buf[4]; + dst[5] = buf[5]; + dst[6] = buf[6]; + dst[7] = buf[7]; + offset += 8; offset } @@ -88,24 +80,25 @@ impl CodingTrait for Coding { offset } - fn get_varint32(input: &mut Slice) -> u32 { + fn get_varint32(input: &mut Slice) -> Option { let cow = input.borrow_data(); let bytes = cow.as_bytes(); - let mut result = 0_u32; + let mut result: Option = None; let mut shift = 0_u32; let limit = input.size(); let mut i = 0; + let mut value = 0_u32; while shift <= 28 && i < limit { let b = bytes[i]; i += 1; if (b & 128) != 0 { - result |= ((b & 127) << shift) as u32; + value |= ((b & 127) << shift) as u32; } else { - result |= (b << shift) as u32; + value |= (b << shift) as u32; } shift += 7; } - result + Some(value) } fn get_varint64(input: &mut Slice) -> u64 { @@ -128,9 +121,16 @@ impl CodingTrait for Coding { result } - fn get_length_prefixed_slice(input: &mut Slice) -> Slice { + fn get_length_prefixed_slice(input: &mut Slice) -> Option { let decode = Coding::get_varint32(input); - Slice::from_buf(decode.to_le_bytes().as_mut_slice()) + match decode { + None => { + None + } + Some(v) => { + Some(Slice::from_buf(v.to_le_bytes().as_mut_slice())) + } + } } fn varint_length(mut value: u64) -> i32 { @@ -143,34 +143,24 @@ impl CodingTrait for Coding { } fn encode_fixed32(value: u32, buf: &mut [u8], mut offset: usize) -> usize { - buf[offset] = value as u8; - offset += 1; - buf[offset] = (value >> 8) as u8; - offset += 1; - buf[offset] = (value >> 16) as u8; - offset += 1; - buf[offset] = (value >> 24) as u8; - offset += 1; + buf[0] = value as u8; + buf[1] = (value >> 8) as u8; + buf[2] = (value >> 16) as u8; + buf[3] = (value >> 24) as u8; + offset += 4; offset } fn encode_fixed64(value: u64, buf: &mut [u8], mut offset: usize) -> usize { - buf[offset] = value as u8; - offset += 1; - buf[offset] = (value >> 8) as u8; - offset += 1; - buf[offset] = (value >> 16) as u8; - offset += 1; - buf[offset] = (value >> 24) as u8; - offset += 1; - buf[offset] = (value >> 32) as u8; - offset += 1; - buf[offset] = (value >> 40) as u8; - offset += 1; - buf[offset] = (value >> 48) as u8; - offset += 1; - buf[offset] = (value >> 56) as u8; - offset += 1; + buf[0] = value as u8; + buf[1] = (value >> 8) as u8; + buf[2] = (value >> 16) as u8; + buf[3] = (value >> 24) as u8; + buf[4] = (value >> 32) as u8; + buf[5] = (value >> 40) as u8; + buf[6] = (value >> 48) as u8; + buf[7] = (value >> 56) as u8; + offset += 8; offset } diff --git a/src/util/coding_test.rs b/src/util/coding_test.rs index c924acb..2f284f2 100644 --- a/src/util/coding_test.rs +++ b/src/util/coding_test.rs @@ -40,6 +40,11 @@ mod test { assert_eq!([0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 12], dst); } + #[test] + fn test_get_varint32(){ + let mut value = 65535; + } + #[test] fn test_encode_varint32() { let mut buf: [u8; 4] = [0, 0, 0, 0]; -- Gitee From aaf53ffce910f3ca5b999358d993a4011c8dce81 Mon Sep 17 00:00:00 2001 From: colagy Date: Thu, 6 Apr 2023 18:41:18 +0800 Subject: [PATCH 2/5] coding rewrite --- src/db/table_cache.rs | 8 +- src/db/version_edit.rs | 2 +- src/lib.rs | 1 + src/table/filter_block_test.rs | 8 +- src/traits/coding_trait.rs | 35 ++++++--- src/util/coding.rs | 94 ++++++++++------------- src/util/coding_test.rs | 135 ++++++++++++++++++++++++++++----- src/util/mem_debug.rs | 2 +- src/util/slice.rs | 19 ++++- 9 files changed, 205 insertions(+), 99 deletions(-) diff --git a/src/db/table_cache.rs b/src/db/table_cache.rs index 5824617..57806af 100644 --- a/src/db/table_cache.rs +++ b/src/db/table_cache.rs @@ -4,15 +4,13 @@ use crate::util::options::ReadOptions; use crate::util::slice::Slice; use crate::util::Result; -struct Saver {} +pub struct Saver {} pub struct TableCache {} impl TableCache { pub fn new() -> Self { - Self { - - } + Self {} } /// 从缓存中获取Table /// @@ -33,7 +31,7 @@ impl TableCache { /// ``` pub fn get(&self, _options: &ReadOptions, _file_number: u64, _file_size: usize, _k: &Slice, _arg: &mut Saver, _handle_result: F) where F: FnMut(&mut Saver, &Slice, &Slice) -> Result<()> { - () + todo!() } /// 根据文件号消除缓存 /// diff --git a/src/db/version_edit.rs b/src/db/version_edit.rs index 792d00a..1d2b20b 100644 --- a/src/db/version_edit.rs +++ b/src/db/version_edit.rs @@ -81,7 +81,7 @@ impl VersionEdit { #[inline] pub fn new() -> Self { Self { - comparator_ : String::new(), + comparator_: String::new(), log_number_: 0, prev_log_number_: 0, next_file_number_: 0, diff --git a/src/lib.rs b/src/lib.rs index c0dce00..5a22977 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +#![feature(core_ffi_c)] extern crate core; pub mod db; diff --git a/src/table/filter_block_test.rs b/src/table/filter_block_test.rs index 9582b2a..a5f7855 100644 --- a/src/table/filter_block_test.rs +++ b/src/table/filter_block_test.rs @@ -115,7 +115,7 @@ mod test { fn test_filter_block_new_with_policy() { let policy = Box::new(TestHashFilter::new()); - let filter_block: FilterBlockBuilder = FilterBlockBuilder::new_with_policy(policy, 10); + let filter_block: FilterBlockBuilder = FilterBlockBuilder::new_with_policy(policy); let fp = filter_block.get_policy(); let filter_policy_name = fp.name(); @@ -132,7 +132,7 @@ mod test { let policy = Box::new(TestHashFilter::new()); let contents = Slice::default(); - let filter_block_reader: FilterBlockReader = FilterBlockReader::new_with_policy(policy, contents); + let filter_block_reader: FilterBlockReader = FilterBlockReader::new_with_policy(policy, contents); let fp_reader = filter_block_reader.get_policy(); let _reader_filter_policy_name = fp_reader.name(); @@ -146,8 +146,8 @@ mod test { #[test] fn test_filter_block_new_with_policy_and_addkey() { let policy = Box::new(TestHashFilter::new()); - let mut filter_block_builder: FilterBlockBuilder = FilterBlockBuilder::new_with_policy( - policy, 10); + let mut filter_block_builder: FilterBlockBuilder = FilterBlockBuilder::new_with_policy( + policy); filter_block_builder.start_block(100); filter_block_builder.add_key_from_str("foo"); diff --git a/src/traits/coding_trait.rs b/src/traits/coding_trait.rs index 2e5c257..c09efe2 100644 --- a/src/traits/coding_trait.rs +++ b/src/traits/coding_trait.rs @@ -6,15 +6,18 @@ pub trait CodingTrait { /// # Arguments /// /// * `dst`: 目标字符串 + /// * `offset` 偏移量 /// * `value`: 编码值 /// /// returns: () /// /// # Examples /// - /// ``` - /// let mut string = String::from("encode:"); - /// put_fixed32(&mut string, 65535); + ///``` + /// let mut dst = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + /// let value = 65535; + /// let mut offset = 2; + /// offset = put_fixed32(&mut dst, offset, value); /// ``` fn put_fixed32(dst: &mut [u8], offset: usize, value: u32) -> usize; ///64位定长编码写入字符串 @@ -22,6 +25,7 @@ pub trait CodingTrait { /// # Arguments /// /// * `dst`: 目标字符串 + /// * `offset` 偏移量 /// * `value`: 编码值 /// /// returns: () @@ -29,8 +33,10 @@ pub trait CodingTrait { /// # Examples /// /// ``` - /// let mut string = String::from("encode:"); - /// put_fixed64(&mut string, 65535); + /// let mut dst = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + /// let value = 65535; + /// let mut offset = 2; + /// offset = put_fixed64(&mut dst, offset, value); /// ``` fn put_fixed64(dst: &mut [u8], offset: usize, value: u64) -> usize; /// 32位变长编码写入字符串 @@ -38,6 +44,7 @@ pub trait CodingTrait { /// # Arguments /// /// * `dst`: 目标字符串 + /// * `offset` 偏移量 /// * `value`: 编码值 /// /// returns: () @@ -45,8 +52,10 @@ pub trait CodingTrait { /// # Examples /// /// ``` - /// let mut string = String::from("encode:"); - /// put_varint32(&mut string, 65535); + /// let mut dst = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + /// let value = 65535; + /// let mut offset = 2; + /// offset = put_varint32(&mut dst, offset, value); /// ``` fn put_varint32(dst: &mut [u8], offset: usize, value: u32) -> usize; /// 64位变长编码写入字符串 @@ -61,8 +70,10 @@ pub trait CodingTrait { /// # Examples /// /// ``` - /// let mut string = String::from("encode:"); - /// put_varint64(&mut string, 65535); + /// let mut dst = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + /// let value = 65535; + /// let mut offset = 2; + /// offset = put_varint64(&mut dst, offset, value); /// ``` fn put_varint64(dst: &mut [u8], offset: usize, value: u64) -> usize; /// 将slice的长度写入目标字符串 @@ -70,6 +81,7 @@ pub trait CodingTrait { /// # Arguments /// /// * `dst`: 目标字符串 + /// * `offset`: 偏移量 /// * `value_len`: Slice类型的编码值长度 /// /// returns: () @@ -86,6 +98,7 @@ pub trait CodingTrait { /// # Arguments /// /// * `input`: slice + /// * `offset`: 偏移量 /// /// returns: u32 /// @@ -94,7 +107,7 @@ pub trait CodingTrait { /// ``` /// /// ``` - fn get_varint32(input: & Slice) -> Option; + fn get_varint32(input: &Slice, offset: usize) -> Option<(u32, usize)>; /// 从slice的开头解码一个64位的变长整数, 并将slice的索引置于解码后的位置 /// /// # Arguments @@ -108,7 +121,7 @@ pub trait CodingTrait { /// ``` /// /// ``` - fn get_varint64(input: &Slice) -> u64; + fn get_varint64(input: &Slice, offset: usize) -> Option<(u64, usize)>; /// 从slice数据中读取长度 返回长度的Slice /// /// # Arguments diff --git a/src/util/coding.rs b/src/util/coding.rs index 59e2efc..d39699a 100644 --- a/src/util/coding.rs +++ b/src/util/coding.rs @@ -1,70 +1,50 @@ +use std::io::Read; use crate::traits::coding_trait::CodingTrait; use crate::traits::coding_trait::Coding32; use crate::traits::coding_trait::Coding64; use crate::util::slice::Slice; -macro_rules! varint { - ($TYPE: ty, $NAME: ident, $SNAME: expr) => { - fn $NAME(mut value: $TYPE, buf: &mut [u8], mut offset: usize) -> usize { - while value >= 128 { - buf[offset] = (value | 128) as u8; - offset += 1; - value >>= 7; - } - buf[offset] = value as u8; - - offset - } - }; - - ($TYPE: ty, $NAME: ident) => { - varint!( $TYPE, $NAME, stringify!($NAME)); - } -} - pub struct Coding {} impl CodingTrait for Coding { fn put_fixed32(dst: &mut [u8], mut offset: usize, value: u32) -> usize { - let mut buf: [u8; 4] = [0, 0, 0, 0]; - Self::encode_fixed32(value, &mut buf, 0); - dst[0] = buf[0]; - dst[1] = buf[1]; - dst[2] = buf[2]; - dst[3] = buf[3]; + Self::encode_fixed32(value, dst, offset); offset += 4; offset } fn put_fixed64(dst: &mut [u8], mut offset: usize, value: u64) -> usize { - let mut buf: [u8; 8] = [0, 0, 0, 0, 0, 0, 0, 0]; - Self::encode_fixed64(value, &mut buf, 0); - dst[0] = buf[0]; - dst[1] = buf[1]; - dst[2] = buf[2]; - dst[3] = buf[3]; - dst[4] = buf[4]; - dst[5] = buf[5]; - dst[6] = buf[6]; - dst[7] = buf[7]; + Self::encode_fixed64(value, dst, offset); offset += 8; offset } - varint!(u32,encode_varint32); - - varint!(u64,encode_varint64); + fn encode_varint32(mut value: u32, buf: &mut [u8], mut offset: usize) -> usize { + while value >= 128 { + buf[offset] = (value | 128) as u8; + value >>= 7; + offset += 1; + } + buf[offset] = value as u8; + offset += 1; + offset + } - fn put_varint32(dst: &mut [u8], mut offset: usize, value: u32) -> usize { - let mut buf: [u8; 4] = [0, 0, 0, 0]; - let var_offset = Self::encode_varint32(value, &mut buf, 0); - for i in 0..var_offset { - dst[offset] = buf[i]; + fn encode_varint64(mut value: u64, buf: &mut [u8], mut offset: usize) -> usize { + while value >= 128 { + buf[offset] = (value | 128) as u8; + value >>= 7; offset += 1; } + buf[offset] = value as u8; + offset += 1; offset } + fn put_varint32(dst: &mut [u8], mut offset: usize, value: u32) -> usize { + Self::encode_varint32(value, dst, offset) + } + fn put_varint64(dst: &mut [u8], mut offset: usize, value: u64) -> usize { let mut buf: [u8; 8] = [0, 0, 0, 0, 0, 0, 0, 0]; let var_offset = Self::encode_varint64(value, &mut buf, 0); @@ -81,28 +61,30 @@ impl CodingTrait for Coding { offset } - fn get_varint32(input: & Slice) -> Option { - let cow = input.borrow_data(); - let bytes = cow.as_bytes(); - let mut result: Option = None; + fn get_varint32(input: &Slice, mut offset: usize) -> Option<(u32, usize)> { + let bytes = &input[offset..input.size()]; let mut shift = 0_u32; let limit = input.size(); let mut i = 0; let mut value = 0_u32; while shift <= 28 && i < limit { - let b = bytes[i]; + let byte = bytes[i]; i += 1; - if (b & 128) != 0 { - value |= ((b & 127) << shift) as u32; + if (byte & 128) != 0 { + value |= ((byte & 127) << shift) as u32; + offset += 1; } else { - value |= (b << shift) as u32; + // 溢出左移 + value |= (byte as u32) << shift; + offset += 1; + return Some((value, offset)); } shift += 7; } - Some(value) + None } - fn get_varint64(input: &Slice) -> u64 { + fn get_varint64(input: &Slice, mut offset: usize) -> Option<(u64, usize)> { let cow = input.borrow_data(); let bytes = cow.as_bytes(); let mut result = 0_u64; @@ -119,17 +101,17 @@ impl CodingTrait for Coding { } shift += 7; } - result + None } fn get_length_prefixed_slice(input: &mut Slice) -> Option { - let decode = Coding::get_varint32(input); + let decode = Coding::get_varint32(input, 0); match decode { None => { None } Some(v) => { - Some(Slice::from_buf(v.to_le_bytes().as_mut_slice())) + Some(Slice::from_buf(v.0.to_le_bytes().as_mut_slice())) } } } diff --git a/src/util/coding_test.rs b/src/util/coding_test.rs index 400d9c6..ac39741 100644 --- a/src/util/coding_test.rs +++ b/src/util/coding_test.rs @@ -1,59 +1,160 @@ mod test { use crate::traits::coding_trait::{Coding32, Coding64, CodingTrait}; use crate::util::coding::{Coding}; + use crate::util::slice::Slice; #[test] fn test_put_fixed32() { let mut dst = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; let value = 65535; - Coding::put_fixed32(&mut dst, 2, value); + let mut offset = 2; + println!("offset:{:?}", offset); + println!("dst:{:?}", dst); + + offset = Coding::put_fixed32(&mut dst, offset, value); assert_eq!([0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 12], dst); + println!("offset:{:?}", offset); + println!("dst:{:?}", dst); + + offset = Coding::put_fixed32(&mut dst, offset, value); + assert_eq!([0, 0, 255, 255, 0, 0, 255, 255, 0, 0, 0, 0] as [u8; 12], dst); + println!("offset:{:?}", offset); + println!("dst:{:?}", dst); } #[test] fn test_put_fixed64() { - let mut dst = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut dst = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; let value = 65535; - Coding::put_fixed64(&mut dst, 2, value); - assert_eq!([0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 12], dst); + + let mut offset = 2; + println!("offset:{:?}", offset); + println!("dst:{:?}", dst); + offset = Coding::put_fixed64(&mut dst, offset, value); + assert_eq!([0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 20], dst); + + println!("offset:{:?}", offset); + println!("dst:{:?}", dst); + + offset = Coding::put_fixed64(&mut dst, offset, value); + + assert_eq!([0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 20], dst); + println!("offset:{:?}", offset); + println!("dst:{:?}", dst); } #[test] fn test_put_varint32() { - let mut value = 65535; - let mut dst = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; - let offset = Coding::put_varint32(&mut dst, 2, value); + let mut value = 255; + let mut value1 = 512; + let mut value2 = 65534; + let mut value3 = 65535; + let mut dst = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut offset = 2; + + offset = Coding::put_varint32(&mut dst, offset, value); + println!("dst:{:?}", dst); + assert_eq!([0, 0, 255, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 20], dst); println!("offset:{:?}", offset); assert_eq!(offset, 4); + + offset = Coding::put_varint32(&mut dst, offset, value1); println!("dst:{:?}", dst); - assert_eq!([0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 12], dst); + assert_eq!([0, 0, 255, 1, 128, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 20], dst); + println!("offset:{:?}", offset); + assert_eq!(offset, 6); + + offset = Coding::put_varint32(&mut dst, offset, value2); + println!("dst:{:?}", dst); + assert_eq!([0, 0, 255, 1, 128, 4, 254, 255, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 20], dst); + println!("offset:{:?}", offset); + assert_eq!(offset, 9); + + offset = Coding::put_varint32(&mut dst, offset, value3); + println!("dst:{:?}", dst); + assert_eq!([0, 0, 255, 1, 128, 4, 254, 255, 3, 255, 255, 3, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 20], dst); + println!("offset:{:?}", offset); + assert_eq!(offset, 12); } #[test] fn test_put_varint64() { - let mut value = 65535; - let mut dst = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; - let offset = Coding::put_varint64(&mut dst, 2, value); + let mut value = 255; + let mut value1 = 512; + let mut value2 = 65534; + let mut value3 = 65535; + let mut dst = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut offset = 2; + + offset = Coding::put_varint64(&mut dst, offset, value); + println!("dst:{:?}", dst); + assert_eq!([0, 0, 255, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 20], dst); println!("offset:{:?}", offset); assert_eq!(offset, 4); + + offset = Coding::put_varint64(&mut dst, offset, value1); println!("dst:{:?}", dst); - assert_eq!([0, 0, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 12], dst); + assert_eq!([0, 0, 255, 1, 128, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 20], dst); + println!("offset:{:?}", offset); + assert_eq!(offset, 6); + + offset = Coding::put_varint64(&mut dst, offset, value2); + println!("dst:{:?}", dst); + assert_eq!([0, 0, 255, 1, 128, 4, 254, 255, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 20], dst); + println!("offset:{:?}", offset); + assert_eq!(offset, 9); + + offset = Coding::put_varint64(&mut dst, offset, value3); + println!("dst:{:?}", dst); + assert_eq!([0, 0, 255, 1, 128, 4, 254, 255, 3, 255, 255, 3, 0, 0, 0, 0, 0, 0, 0, 0] as [u8; 20], dst); + println!("offset:{:?}", offset); + assert_eq!(offset, 12); } #[test] - fn test_get_varint32(){ - let mut value = 65535; + fn test_get_varint32() { + let value = [255, 512]; + let data = [0, 0, 255, 1, 128, 4, 254, 255, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]; + let mut slice = Slice::from_buf(&data); + let mut offset = 2; + + let mut i = 0; + while offset < slice.len() { + let got = Coding::get_varint32(&mut slice, offset); + match got { + Some(v) => { + offset = v.1; + println!("value:{:?}", v.0); + assert_eq!(value[i], v.0); + i += 1; + } + None => { + println!("value is none"); + } + } + } } #[test] fn test_encode_varint32() { let mut buf: [u8; 4] = [0, 0, 0, 0]; - let mut value: u32 = 65534; + let mut value: u32 = 65535; + let offset = Coding::encode_varint32(value, &mut buf, 0); + println!("offset:{:?}", offset); + assert_eq!(offset, 3); + println!("buf:{:?}", buf); + assert_eq!(buf, [255, 255, 3, 0]); + } + + #[test] + fn test_decode_varint32() { + let mut buf: [u8; 4] = [255, 255, 3, 0]; + let mut value: u32 = 65535; let offset = Coding::encode_varint32(value, &mut buf, 0); println!("offset:{:?}", offset); assert_eq!(offset, 2); println!("buf:{:?}", buf); - assert_eq!(buf, [254, 255, 3, 0]); + assert_eq!(buf, [255, 255, 3, 0]); } #[test] @@ -62,7 +163,7 @@ mod test { let mut value: u64 = 65535; let offset = Coding::encode_varint64(value, &mut buf, 0); println!("offset:{:?}", offset); - assert_eq!(offset, 2); + assert_eq!(offset, 3); println!("buf:{:?}", buf); assert_eq!(buf, [255, 255, 3, 0]); } diff --git a/src/util/mem_debug.rs b/src/util/mem_debug.rs index bef8a83..93a79a2 100644 --- a/src/util/mem_debug.rs +++ b/src/util/mem_debug.rs @@ -1,4 +1,4 @@ -use std::ffi::{c_char, c_void}; +use core::ffi::{c_char, c_void}; use std::ptr::{null, null_mut}; extern "C" fn write_cb(_: *mut c_void, message: *const c_char) { diff --git a/src/util/slice.rs b/src/util/slice.rs index 1f923b8..fae6aa5 100644 --- a/src/util/slice.rs +++ b/src/util/slice.rs @@ -26,7 +26,6 @@ impl Default for Slice { } impl Slice { - /// 从 &mut [u8] 转到 Slice, 这里存在内存拷贝开销 #[inline] pub fn from_buf(buf: &[u8]) -> Self { @@ -62,7 +61,7 @@ impl Slice { #[inline] pub fn as_sub_ref(&self, start: usize, length: usize) -> &[u8] { - &(**self)[start..(start+length)] + &(**self)[start..(start + length)] } /// 移除头部 n 个元素 @@ -145,7 +144,7 @@ impl From for Vec { } } -impl > From for Slice { +impl> From for Slice { #[inline] fn from(r: R) -> Self { Self { @@ -207,13 +206,25 @@ impl core::ops::Index for Slice { } } +impl core::ops::Index> for Slice { + type Output = [u8]; + + /// 获取指定下标范围的数据 + #[inline] + fn index(&self, range: core::ops::Range) -> &Self::Output { + assert!(range.end <= self.size()); + assert!(range.start >= 0); + &(**self)[range.start..range.end] + } +} + impl Deref for Slice { type Target = [u8]; /// Slice 解引用到 &[u8] #[inline] fn deref(&self) -> &Self::Target { - &*self.data + &*self.data } } -- Gitee From 30795a377adba257dd5129e2391972114b11b852 Mon Sep 17 00:00:00 2001 From: colagy Date: Fri, 14 Apr 2023 13:40:09 +0800 Subject: [PATCH 3/5] Add more tests and examples --- src/util/coding.rs | 193 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 145 insertions(+), 48 deletions(-) diff --git a/src/util/coding.rs b/src/util/coding.rs index 0c7ddbf..e5c19a2 100644 --- a/src/util/coding.rs +++ b/src/util/coding.rs @@ -20,7 +20,9 @@ use crate::util::status::LevelError; /// # Examples /// /// ``` -/// +/// use level_db_rust::util::coding::varint_length; +/// // length == 2 +/// let length = varint_length(255); /// ``` pub fn varint_length(mut value: u64) -> usize { let mut len = 1; @@ -143,7 +145,11 @@ macro_rules! encode_fixed { /// # Examples /// /// ``` - /// + /// let mut vec = vec![]; + /// // [0, 0, 4, 210] + /// unsafe { + /// uncheck_encode_fixed32(&mut MutVector(&mut vec), 0, 1234); + /// } /// ``` #[inline] unsafe fn $name(data: &mut MutEncodeData, offset: usize, value: $type) { @@ -178,7 +184,10 @@ encode_fixed!(uncheck_encode_fixed64, u64, u64); /// # Examples /// /// ``` -/// +/// let mut vec = vec![]; +/// let mut offset = 0; +/// // [255, 255, 3] +/// unsafe { offset = uncheck_encode_varint32(&mut MutVector(&mut vec), offset, 65535); } /// ``` unsafe fn uncheck_encode_varint32(data: &mut MutEncodeData, offset: usize, value: u32) -> usize { let length = varint_length(value as u64); @@ -236,7 +245,10 @@ unsafe fn uncheck_encode_varint32(data: &mut MutEncodeData, offset: usize, value /// # Examples /// /// ``` -/// +/// let mut vec = vec![]; +/// let mut offset = 0; +/// // offset = 7, vec = [255, 255, 208, 148, 181, 244, 1] +/// unsafe { offset = uncheck_encode_varint64(&mut MutVector(&mut vec), offset, 8_3980_4651_1103) }; /// ``` unsafe fn uncheck_encode_varint64(data: &mut MutEncodeData, mut offset: usize, mut value: u64) -> usize { let length = varint_length(value); @@ -269,7 +281,9 @@ macro_rules! decode_fixed { /// # Examples /// /// ``` - /// + /// let vec = vec![0, 0, 255, 255]; + /// // 65535 + /// let result = unsafe { uncheck_decode_fixed32(&Vector(&vec), 0) }; /// ``` #[inline] unsafe fn $name(data: &EncodeData, offset: usize) -> $type { @@ -301,7 +315,11 @@ macro_rules! decode_varint { /// # Examples /// /// ``` - /// + /// let vec = vec![255, 255, 3]; + /// println!("{:?}", vec); + /// let mut offset = 0; + /// // 65535 + /// let res = unsafe { uncheck_decode_varint32(&Vector(&vec), offset, vec.len()) }; /// ``` unsafe fn $name(data: &EncodeData, mut offset: usize, limit: usize) -> ($type, usize) { let ptr = get_ptr!(data); @@ -347,7 +365,11 @@ decode_varint!(uncheck_decode_varint64, u64, 63); /// # Examples /// /// ``` +/// let mut vec = vec![]; /// +/// let buf = [1, 2, 3, 4, 5]; +/// // vec = [1, 2, 3, 4, 5] +/// unsafe { uncheck_write_buf(&mut MutVector(&mut vec), 0, &buf); } /// ``` unsafe fn uncheck_write_buf(data: &mut MutEncodeData, offset: usize, buf: &[u8]) { let mut_ptr = get_mut_ptr!(data, buf.len(), offset).add(offset); @@ -371,7 +393,9 @@ unsafe fn uncheck_write_buf(data: &mut MutEncodeData, offset: usize, buf: &[u8]) /// # Examples /// /// ``` -/// +/// let vec = vec![1, 2, 3, 4, 5, 1, 2, 3, 4]; +/// // [1, 2, 3, 4, 5] +/// let buf = unsafe { uncheck_read_buf(&Vector(&vec), 0, 5) }; /// ``` unsafe fn uncheck_read_buf(data: &EncodeData, offset: usize, len: usize) -> Slice { let ptr: *const u8 = get_ptr!(data).add(offset); @@ -395,7 +419,10 @@ unsafe fn uncheck_read_buf(data: &EncodeData, offset: usize, len: usize) -> Slic /// # Examples /// /// ``` -/// +/// let vec = vec![1, 2, 3, 4, 5, 1, 2, 3, 4]; +/// let mut dst = [0; 5]; +/// // dst = [1, 2, 3, 4, 5] +/// unsafe { uncheck_read_into_buf(&Vector(&vec), 0, &mut dst) }; /// ``` unsafe fn uncheck_read_into_buf(data: &EncodeData, offset: usize, dst: &mut [u8]) { let ptr: *const u8 = get_ptr!(data).add(offset); @@ -480,7 +507,9 @@ impl<'a> Encoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Encoder; + /// let mut vec = vec![]; + /// let mut encoder = Encoder::with_vec(&mut vec); /// ``` pub fn with_vec(vec: &'a mut Vec) -> Self { Self { @@ -554,34 +583,6 @@ impl<'a> Encoder<'a> { put_varint!(put_varint32, uncheck_encode_varint32, u32, check); put_varint!(put_varint64, uncheck_encode_varint64, u64, check); - /// 写入slice时先写入slice的长度做为前缀 - /// slice(data:[1,2,3],size:3), 写入后[3,1,2,3] - /// - /// # Safety - /// * u32的字节数(4) + slice的字节数(slice.size()) < self.data.len(), 否则溢出(vec除外) - /// - /// # Arguments - /// - /// * `slice`: slice - /// - /// returns: () - /// - /// # Examples - /// - /// ``` - /// - /// ``` - pub unsafe fn uncheck_put_length_prefixed_slice(&mut self, slice: &Slice) { - self.uncheck_put_varint32(slice.size() as u32); - self.uncheck_put_buf(slice); - } - - pub fn put_length_prefixed_slice(&mut self, slice: &Slice) -> Result<()> { - self.put_varint32(slice.size() as u32)?; - self.put_buf(slice)?; - Ok(()) - } - /// 向encoder中直接写入数据不用进行编码 /// 向vec中写入时会自动扩容 /// # Safety @@ -626,6 +627,34 @@ impl<'a> Encoder<'a> { Ok(()) } + /// 写入slice时先写入slice的长度做为前缀 + /// slice(data:[1,2,3],size:3), 写入后[3,1,2,3] + /// + /// # Safety + /// * u32的字节数(4) + slice的字节数(slice.size()) < self.data.len(), 否则溢出(vec除外) + /// + /// # Arguments + /// + /// * `slice`: slice + /// + /// returns: () + /// + /// # Examples + /// + /// ``` + /// + /// ``` + pub unsafe fn uncheck_put_length_prefixed_slice(&mut self, slice: &Slice) { + self.uncheck_put_varint32(slice.size() as u32); + self.uncheck_put_buf(slice); + } + + pub fn put_length_prefixed_slice(&mut self, slice: &Slice) -> Result<()> { + self.put_varint32(slice.size() as u32)?; + self.put_buf(slice)?; + Ok(()) + } + /// 获取当前编码到的位置 /// /// returns: usize @@ -859,8 +888,10 @@ impl<'a> Decoder<'a> { /// ``` /// /// ``` - unsafe fn uncheck_get_buf(&self, len: usize) -> Slice { - uncheck_read_buf(&self.data, self.offset, len) + unsafe fn uncheck_get_buf(&mut self, len: usize) -> Slice { + let slice = uncheck_read_buf(&self.data, self.offset, len); + self.offset += len; + slice } /// 读取buf @@ -1277,10 +1308,11 @@ fn test_put_fixed() -> Result<()> { encoder.uncheck_put_fixed64(655535); encoder.uncheck_put_fixed64(8_3980_4651_1103); encoder.uncheck_put_fixed64(900_3372_0368_5477_5808); - println!("{:?}", &encoder.data); println!("{:?}", &encoder); if let MutVector(data) = encoder.data { - assert_eq!(&mut vec![0, 0, 0, 2, 0, 0, 0, 128, 0, 0, 0, 255, 0, 0, 255, 255, 0, 152, 150, 128, 0, 0, 0, 0, 0, 10, 0, 175, 0, 0, 7, 163, 82, 148, 63, 255, 124, 242, 103, 42, 101, 106, 0, 0], + assert_eq!(&mut vec![0, 0, 0, 2, 0, 0, 0, 128, 0, 0, 0, 255, 0, 0, 255, 255, 0, 152, 150, + 128, 0, 0, 0, 0, 0, 10, 0, 175, 0, 0, 7, 163, 82, 148, 63, 255, 124, + 242, 103, 42, 101, 106, 0, 0], data); } } @@ -1295,13 +1327,43 @@ fn test_put_fixed() -> Result<()> { encoder.put_fixed64(655535)?; encoder.put_fixed64(8_3980_4651_1103)?; encoder.put_fixed64(900_3372_0368_5477_5808)?; - println!("{:?}", &encoder.data); println!("{:?}", &encoder); if let MutVector(data) = encoder.data { - assert_eq!(&mut vec![0, 0, 0, 2, 0, 0, 0, 128, 0, 0, 0, 255, 0, 0, 255, 255, 0, 152, 150, 128, 0, 0, 0, 0, 0, 10, 0, 175, 0, 0, 7, 163, 82, 148, 63, 255, 124, 242, 103, 42, 101, 106, 0, 0], + assert_eq!(&mut vec![0, 0, 0, 2, 0, 0, 0, 128, 0, 0, 0, 255, 0, 0, 255, 255, 0, 152, 150, + 128, 0, 0, 0, 0, 0, 10, 0, 175, 0, 0, 7, 163, 82, 148, 63, 255, 124, + 242, 103, 42, 101, 106, 0, 0], data); } + let mut buf = [0; 20]; + unsafe { + let mut encoder = Encoder::with_buf(&mut buf); + println!("{:?}", encoder); + encoder.uncheck_put_fixed32(2); + encoder.uncheck_put_fixed64(655535); + encoder.uncheck_put_fixed64(8_3980_4651_1103); + println!("{:?}", &encoder); + if let MutVector(data) = encoder.data { + assert_eq!(&mut vec![0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 175, 0, 0, 7, 163, 82, 148, 63, 255], + data); + } + } + + + let mut slice = Slice::from_vec(vec![0; 20]); + unsafe { + let mut encoder = Encoder::with_slice(&mut slice); + println!("{:?}", encoder); + encoder.uncheck_put_fixed32(2); + encoder.uncheck_put_fixed64(655535); + encoder.uncheck_put_fixed64(8_3980_4651_1103); + println!("{:?}", &encoder); + if let MutVector(data) = encoder.data { + assert_eq!(&mut vec![0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 175, 0, 0, 7, 163, 82, 148, 63, 255], + data); + } + } + Ok(()) } @@ -1454,19 +1516,24 @@ fn test_get_buf() { println!("{:?}", buf); assert_eq!(&[1_u8, 2, 3], vec.clone().as_slice()); } - let decoder = Decoder::with_vec(&vec); + let mut decoder = Decoder::with_vec(&vec); let buf = unsafe { decoder.uncheck_get_buf(3) }; println!("{:?}", buf); + assert_eq!(Slice::from_vec(vec![1, 2, 3]), buf); + assert_eq!(3, decoder.offset) } #[test] fn test_put_length_prefixed_slice() { let mut vec = vec![]; - let mut encoder = Encoder::with_vec(&mut vec); - let slice = Slice::from_vec(vec![1, 2, 3]); - unsafe { encoder.uncheck_put_length_prefixed_slice(&slice); } + { + let mut encoder = Encoder::with_vec(&mut vec); + let slice = Slice::from_vec(vec![1, 2, 3]); + unsafe { encoder.uncheck_put_length_prefixed_slice(&slice); } + assert_eq!(4, encoder.offset) + } println!("{:?}", vec); - assert_eq!(vec![3, 1, 2, 3], vec); + assert_eq!(&vec![3, 1, 2, 3], &vec); } #[test] @@ -1484,6 +1551,36 @@ fn test_get_length_prefixed_slice() { let slice = unsafe { decoder.uncheck_get_length_prefixed_slice() }; println!("{:?}", slice); assert_eq!(&[1_u8, 2, 3], &*slice); + assert_eq!(4, decoder.offset) +} + +#[test] +fn test_mixed_put_get() { + let mut vec = vec![]; + let mut encoder = Encoder::with_vec(&mut vec); + + unsafe { + encoder.uncheck_put_fixed32(3); + encoder.uncheck_put_varint32(65535); + encoder.uncheck_put_fixed64(7); + encoder.uncheck_put_varint64(8_3980_4651_1103); + let buf = [1, 2, 3]; + encoder.uncheck_put_buf(&buf); + let slice = Slice::from_vec(vec![1, 2, 3]); + encoder.uncheck_put_length_prefixed_slice(&slice); + } + + let mut decoder = Decoder::with_vec(&vec); + unsafe { + assert_eq!(3, decoder.uncheck_get_fixed32()); + assert_eq!(65535, decoder.uncheck_get_varint32()); + assert_eq!(7, decoder.uncheck_get_fixed64()); + assert_eq!(8_3980_4651_1103, decoder.uncheck_get_varint64()); + let buf = [1_u8, 2, 3]; + assert_eq!(&buf, &*decoder.uncheck_get_buf(3)); + let slice = Slice::from_vec(vec![1, 2, 3]); + assert_eq!(slice, decoder.uncheck_get_length_prefixed_slice()) + } } #[test] -- Gitee From 555dcbf70a57ab17add6efd52f24a17dfa342321 Mon Sep 17 00:00:00 2001 From: colagy Date: Fri, 14 Apr 2023 19:35:14 +0800 Subject: [PATCH 4/5] Add more tests/examples and skip() method; --- src/util/coding.rs | 353 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 332 insertions(+), 21 deletions(-) diff --git a/src/util/coding.rs b/src/util/coding.rs index e5c19a2..e8fccd6 100644 --- a/src/util/coding.rs +++ b/src/util/coding.rs @@ -459,14 +459,53 @@ pub struct Decoder<'a> { /// 实现put_fixed macro_rules! put_fixed { ($name:ident, $var_name:ident, $type:ty, $capacity: ident, uncheck) => { + /// 编码定长整数 不检查长度 + /// + /// # Safety + /// * offset + type_size < data.len() , 否则溢出 + /// + /// # Arguments + /// + /// * `value`: 待编码的数据 + /// + /// # Examples + /// + /// ``` + /// use level_db_rust::util::coding::Encoder; + /// let mut vec = vec![]; + /// let mut encoder = Encoder::with_vec(&mut vec); + /// unsafe { + /// // [0, 0, 255, 255] + /// encoder.uncheck_put_fixed32(65535); + /// // [0, 0, 255, 255, 0, 0, 255, 255] + /// encoder.uncheck_put_fixed32(65535); + /// } + /// ``` pub unsafe fn $name(&mut self, value: $type) { $var_name(&mut self.data, self.offset, value); self.offset += type_capacity!($capacity); } }; ($name:ident, $var_name:ident, $type:ty, $capacity: ident, check) => { + /// 编码定长整数 会检查长度 + /// + /// # Arguments + /// + /// * `value`: 待编码的数据 + /// + /// returns: Result<()> + /// + /// # Examples + /// + /// ``` + /// use level_db_rust::util::coding::Encoder; + /// let mut vec = vec![]; + /// let mut encoder = Encoder::with_vec(&mut vec); + /// // [0, 0, 255, 255] + /// encoder.put_fixed32(65535)?; + /// ``` pub fn $name(&mut self, value: $type) -> Result<()> { - // vec类型自动扩容 buf 和 slice类型检查长度 + // vec类型自动扩容, buf 和 slice类型检查长度 if let MutVector(_) = self.data {} else { check_length!(self.offset, type_capacity!($capacity), self.len()) }; unsafe {$var_name(&mut self.data, self.offset, value);} self.offset += type_capacity!($capacity); @@ -478,13 +517,52 @@ macro_rules! put_fixed { /// 实现put_varint macro_rules! put_varint { ($name:ident, $var_name:ident, $type:ty, uncheck) => { + /// 编码变长整数 不检查长度 + /// + /// # Safety + /// * offset + varint_length < data.len() , 否则溢出 + /// + /// # Arguments + /// + /// * `value`: 待编码的数据 + /// + /// # Examples + /// + /// ``` + /// use level_db_rust::util::coding::Encoder; + /// let mut vec = vec![]; + /// let mut encoder = Encoder::with_vec(&mut vec); + /// unsafe { + /// // [255, 255, 3] + /// encoder.uncheck_put_varint32(65535); + /// // [255, 255, 3, 255, 255, 3] + /// encoder.uncheck_put_varint64(65535); + /// } + /// ``` pub unsafe fn $name(&mut self, value: $type) { self.offset = $var_name(&mut self.data, self.offset, value); } }; ($name:ident, $var_name:ident, $type:ty, check) => { + /// 编码变长整数 会检查长度 + /// + /// # Arguments + /// + /// * `value`: 待编码的数据 + /// + /// # Examples + /// + /// ``` + /// use level_db_rust::util::coding::Encoder; + /// let mut vec = vec![]; + /// let mut encoder = Encoder::with_vec(&mut vec); + /// // [255, 255, 3] + /// encoder.put_varint32(65535)?; + /// // [255, 255, 3, 255, 255, 3] + /// encoder.put_varint64(65535)?; + /// ``` pub fn $name(&mut self, value: $type) -> Result<()> { - // vec类型自动扩容 buf 和 slice类型检查长度 + // vec类型自动扩容, buf 和 slice类型检查长度 if let MutVector(_) = self.data {} else { check_length!(self.offset, varint_length(value as u64), self.len()) }; unsafe { self.offset = $var_name(&mut self.data, self.offset, value) } Ok(()) @@ -530,7 +608,11 @@ impl<'a> Encoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Encoder; + /// let mut buf = [0; 20]; + /// unsafe { + /// let mut encoder = Encoder::with_buf(&mut buf); + /// } /// ``` pub fn with_buf(buf: &'a mut [u8]) -> Self { Self { @@ -551,7 +633,12 @@ impl<'a> Encoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Encoder; + /// use level_db_rust::util::slice::Slice; + /// let mut slice = Slice::from_vec(vec![0; 20]); + /// unsafe { + /// let mut encoder = Encoder::with_slice(&mut slice); + /// } /// ``` pub fn with_slice(slice: &'a mut Slice) -> Self { Self { @@ -567,7 +654,10 @@ impl<'a> Encoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Encoder; + /// let mut data = vec![1, 2, 3]; + /// let encoder = Encoder::with_vec(&mut data); + /// let decoder = encoder.create_decoder(); /// ``` pub fn create_decoder(&'a self) -> Decoder<'a> { Decoder::from_encoder(self) @@ -597,7 +687,12 @@ impl<'a> Encoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Encoder; + /// let mut vec = vec![]; + /// let mut encoder = Encoder::with_vec(&mut vec); + /// let buf = [1, 2, 3]; + /// // vec: [1, 2, 3] + /// unsafe { encoder.uncheck_put_buf(&buf) } /// ``` pub unsafe fn uncheck_put_buf(&mut self, buf: &[u8]) { uncheck_write_buf(&mut self.data, self.offset, buf); @@ -617,7 +712,12 @@ impl<'a> Encoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Encoder; + /// let mut vec = vec![]; + /// let mut encoder = Encoder::with_vec(&mut vec); + /// let buf = [1, 2, 3]; + /// // vec: [1, 2, 3] + /// encoder.put_buf(&buf)? /// ``` pub fn put_buf(&mut self, buf: &[u8]) -> Result<()> { // vec类型自动扩容 buf 和 slice类型检查长度 @@ -635,20 +735,49 @@ impl<'a> Encoder<'a> { /// /// # Arguments /// - /// * `slice`: slice + /// * `slice`: 待写入的slice /// /// returns: () /// /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Encoder; + /// use level_db_rust::util::slice::Slice; + /// let mut vec = vec![]; + /// let mut encoder = Encoder::with_vec(&mut vec); + /// let slice = Slice::from_vec(vec![1, 2, 3]); + /// // vec: [3, 1, 2, 3] + /// // The first '3' of the vec is the length of the slice, + /// // and the following '1,2,3' is the data of the slice + /// unsafe { encoder.uncheck_put_length_prefixed_slice(&slice); } /// ``` pub unsafe fn uncheck_put_length_prefixed_slice(&mut self, slice: &Slice) { self.uncheck_put_varint32(slice.size() as u32); self.uncheck_put_buf(slice); } + /// 写入slice时先写入slice的长度做为前缀 + /// + /// # Arguments + /// + /// * `slice`: 待写入的slice + /// + /// returns: Result<(), Status> + /// + /// # Examples + /// + /// ``` + /// use level_db_rust::util::coding::Encoder; + /// use level_db_rust::util::slice::Slice; + /// let mut vec = vec![]; + /// let mut encoder = Encoder::with_vec(&mut vec); + /// let slice = Slice::from_vec(vec![1, 2, 3]); + /// // vec: [3, 1, 2, 3] + /// // The first '3' of the vec is the length of the slice, + /// // and the following '1,2,3' is the data of the slice + /// encoder.put_length_prefixed_slice(&slice)?; + /// ``` pub fn put_length_prefixed_slice(&mut self, slice: &Slice) -> Result<()> { self.put_varint32(slice.size() as u32)?; self.put_buf(slice)?; @@ -662,8 +791,16 @@ impl<'a> Encoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Encoder; + /// let mut vec = vec![]; + /// let mut encoder = Encoder::with_vec(&mut vec); + /// // offset: 0 + /// let offset = encoder.offset(); + /// encoder.put_varint32(65535)?; + /// // offset: 3 + /// let offset = encoder.offset(); /// ``` + #[inline] pub fn offset(&self) -> usize { self.offset } @@ -675,9 +812,15 @@ impl<'a> Encoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Encoder; + /// let mut vec = vec![]; + /// let mut encoder = Encoder::with_vec(&mut vec); + /// // len: 0 + /// let len = encoder.len(); + /// encoder.put_varint32(65535)?; + /// // len: 3 + /// let len = encoder.len(); /// ``` - #[inline] pub fn len(&self) -> usize { match self.data { MutVector(ref vec) => { @@ -704,7 +847,11 @@ macro_rules! get_fixed { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Decoder; + /// let mut vec = vec![0, 0, 255, 255]; + /// let mut decoder = Decoder::with_vec(&mut vec); + /// // 65535 + /// let value = unsafe { decoder.uncheck_get_fixed32() }; /// ``` #[inline] pub unsafe fn $name(&mut self) -> $type { @@ -723,7 +870,11 @@ macro_rules! get_fixed { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Decoder; + /// let mut vec = vec![0, 0, 255, 255]; + /// let mut decoder = Decoder::with_vec(&mut vec); + /// // 65535 + /// let value = decoder.get_fixed32()?; /// ``` #[inline] pub fn $name(&mut self) -> Result<$type> { @@ -744,7 +895,11 @@ macro_rules! get_varint { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Decoder; + /// let mut vec = vec![255, 255, 3]; + /// let mut decoder = Decoder::with_vec(&mut vec); + /// // 65535 + /// let value = unsafe { decoder.uncheck_get_varint32() }; /// ``` #[inline] pub unsafe fn $name(&mut self) -> $type { @@ -761,7 +916,11 @@ macro_rules! get_varint { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Decoder; + /// let mut vec = vec![255, 255, 3]; + /// let mut decoder = Decoder::with_vec(&mut vec); + /// // 65535 + /// let value = decoder.get_varint32()?; /// ``` #[inline] pub fn $name(&mut self) -> Result<$type> { @@ -815,6 +974,7 @@ impl<'a> Decoder<'a> { } /// 判断是否有数据可以读取 + /// 数据读取到末尾 不满足 offset < limit 时为false /// 如果使用了uncheck的方法 需要调用这个方法判断是否可以读取 否则可能会溢出 /// /// returns: bool @@ -822,7 +982,14 @@ impl<'a> Decoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Decoder; + /// let mut vec = vec![255, 255, 3]; + /// let mut decoder = Decoder::with_vec(&mut vec); + /// // true + /// let can_get = decoder.can_get(); + /// decoder.get_varint32()?; + /// // false + /// let can_get = decoder.can_get(); /// ``` #[inline] pub fn can_get(&self) -> bool { @@ -888,7 +1055,7 @@ impl<'a> Decoder<'a> { /// ``` /// /// ``` - unsafe fn uncheck_get_buf(&mut self, len: usize) -> Slice { + pub unsafe fn uncheck_get_buf(&mut self, len: usize) -> Slice { let slice = uncheck_read_buf(&self.data, self.offset, len); self.offset += len; slice @@ -908,7 +1075,7 @@ impl<'a> Decoder<'a> { /// ``` /// /// ``` - fn get_buf(&self, len: usize) -> Result { + pub fn get_buf(&self, len: usize) -> Result { check_length!(self.offset, len, self.limit); unsafe { Ok(uncheck_read_buf(&self.data, self.offset, len)) @@ -932,7 +1099,7 @@ impl<'a> Decoder<'a> { /// ``` /// /// ``` - unsafe fn uncheck_get_into_buf(&self, dst: &mut [u8]) { + pub unsafe fn uncheck_get_into_buf(&self, dst: &mut [u8]) { // todo 增加长度字段, 以写入到dst的任意位置 uncheck_read_into_buf(&self.data, self.offset, dst) } @@ -951,13 +1118,104 @@ impl<'a> Decoder<'a> { /// ``` /// /// ``` - fn get_into_buf(&self, dst: &mut [u8]) -> Result<()> { + pub fn get_into_buf(&self, dst: &mut [u8]) -> Result<()> { check_length!(self.offset, dst.len(), self.limit); unsafe { uncheck_read_into_buf(&self.data, self.offset, dst); } Ok(()) } + + /// 跳过一段长度 偏移量会移动到跳过后的位置继续读取 未检查偏移量 + /// + /// # Safety + /// * offset + skip < self.limit, 否则会出现未定义行为, 读取将溢出 + /// + /// # Arguments + /// + /// * `skip`: 需要跳过的长度 + /// + /// returns: usize + /// + /// # Examples + /// + /// ``` + /// use level_db_rust::util::coding::Decoder; + /// let vec = vec![255, 1, 255, 255, 3]; + /// // offset: 0 + /// let mut decoder = Decoder::with_vec(&vec); + /// // offset: 2 + /// unsafe { decoder.uncheck_skip(2) }; + /// // value: 65535 + /// let value = decoder.get_varint32()?; + /// ``` + pub unsafe fn uncheck_skip(&mut self, skip: usize) -> usize { + self.offset += skip; + self.offset + } + + /// 跳过一段长度 偏移量会移动到跳过后的位置继续读取 + /// + /// # Arguments + /// + /// * `skip`: 需要跳过的长度 + /// + /// returns: Result + /// + /// # Examples + /// + /// ``` + /// use level_db_rust::util::coding::Decoder; + /// let vec = vec![255, 1, 255, 255, 3]; + /// // offset: 0 + /// let mut decoder = Decoder::with_vec(&vec); + /// // offset: 2 + /// decoder.skip(2)?; + /// // value: 65535 + /// let value = decoder.get_varint32()?; + /// ``` + pub fn skip(&mut self, skip: usize) -> Result { + check_length!(self.offset, self.limit); + self.offset += skip; + Ok(self.offset) + } + + /// 获取当前编码到的位置 + /// + /// returns: usize + /// + /// # Examples + /// + /// ``` + /// use level_db_rust::util::coding::Decoder; + /// let vec = vec![255, 1, 255, 255, 3]; + /// let mut decoder = Decoder::with_vec(&vec); + /// // offset: 0 + /// let value = decoder.get_varint32()?; + /// // offset: 2 + /// let offset = decoder.offset(); + /// ``` + pub fn offset(&self) -> usize { + self.offset + } + + /// 获取编码数据的可解码限制 + /// offset < limit + /// + /// returns: usize + /// + /// # Examples + /// + /// ``` + /// use level_db_rust::util::coding::Decoder; + /// let vec = vec![255, 1, 255, 255, 3]; + /// let mut decoder = Decoder::with_vec(&vec); + /// // limit: 5 + /// let limit = decoder.limit(); + /// ``` + pub fn limit(&self) -> usize { + self.limit + } } #[test] @@ -1583,6 +1841,59 @@ fn test_mixed_put_get() { } } +#[test] +fn test_offset_len_skip() -> Result<()> { + let mut vec = vec![]; + let mut encoder = Encoder::with_vec(&mut vec); + assert_eq!(0, encoder.offset()); + assert_eq!(0, encoder.len()); + encoder.put_varint32(65535)?; + assert_eq!(3, encoder.offset()); + assert_eq!(3, encoder.len()); + + encoder.put_varint32(65535)?; + assert_eq!(6, encoder.offset()); + assert_eq!(6, encoder.len()); + + encoder.put_varint32(65535)?; + assert_eq!(9, encoder.offset()); + assert_eq!(9, encoder.len()); + + let mut decoder = Decoder::with_vec(&vec); + assert_eq!(0, decoder.offset()); + assert_eq!(9, decoder.limit()); + + let value = decoder.get_varint32()?; + assert_eq!(3, decoder.offset()); + assert_eq!(9, decoder.limit()); + assert_eq!(65535, value); + + decoder.skip(3)?; + + let value = decoder.get_varint32()?; + assert_eq!(9, decoder.offset()); + assert_eq!(9, decoder.limit()); + assert_eq!(65535, value); + + let mut decoder = Decoder::with_vec(&vec); + assert_eq!(0, decoder.offset()); + assert_eq!(9, decoder.limit()); + + let value = decoder.get_varint32()?; + assert_eq!(3, decoder.offset()); + assert_eq!(9, decoder.limit()); + assert_eq!(65535, value); + + unsafe { decoder.uncheck_skip(3); } + + let value = decoder.get_varint32()?; + assert_eq!(9, decoder.offset()); + assert_eq!(9, decoder.limit()); + assert_eq!(65535, value); + + Ok(()) +} + #[test] fn test_from_into() { let mut data = vec![1, 2, 3]; -- Gitee From d9bd3cf6e6c67d6be2793f5f0229ec13c6d03551 Mon Sep 17 00:00:00 2001 From: colagy Date: Fri, 14 Apr 2023 23:16:45 +0800 Subject: [PATCH 5/5] Add more examples; --- src/util/coding.rs | 124 ++++++++++++--------------------------------- 1 file changed, 31 insertions(+), 93 deletions(-) diff --git a/src/util/coding.rs b/src/util/coding.rs index e8fccd6..4fe6fd1 100644 --- a/src/util/coding.rs +++ b/src/util/coding.rs @@ -404,31 +404,6 @@ unsafe fn uncheck_read_buf(data: &EncodeData, offset: usize, len: usize) -> Slic Slice::from_raw_parts(dst, len) } -/// 读取buf 读取时需要知道需要读取的长度 传入的dst需要具有长度信息 -/// -/// # Safety -/// * offset + dst.len() < data.len() , 否则溢出 -/// -/// # Arguments -/// -/// * `data`: 存储编码的数据 -/// * `offset`: 解码的偏移量 -/// -/// returns: &[u8] -/// -/// # Examples -/// -/// ``` -/// let vec = vec![1, 2, 3, 4, 5, 1, 2, 3, 4]; -/// let mut dst = [0; 5]; -/// // dst = [1, 2, 3, 4, 5] -/// unsafe { uncheck_read_into_buf(&Vector(&vec), 0, &mut dst) }; -/// ``` -unsafe fn uncheck_read_into_buf(data: &EncodeData, offset: usize, dst: &mut [u8]) { - let ptr: *const u8 = get_ptr!(data).add(offset); - intrinsics::copy_nonoverlapping(ptr, dst.as_mut_ptr(), dst.len()); -} - #[derive(Debug)] enum EncodeData<'a> { Vector(&'a Vec), @@ -1016,7 +991,11 @@ impl<'a> Decoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Decoder; + /// let vec = vec![3, 1, 2, 3]; + /// let mut decoder = Decoder::with_vec(&vec); + /// // [1, 2, 3] + /// let slice = unsafe { decoder.uncheck_get_length_prefixed_slice() }; /// ``` pub unsafe fn uncheck_get_length_prefixed_slice(&mut self) -> Slice { let size = self.uncheck_get_varint32() as usize; @@ -1030,7 +1009,11 @@ impl<'a> Decoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Decoder; + /// let vec = vec![3, 1, 2, 3]; + /// let mut decoder = Decoder::with_vec(&vec); + /// // [1, 2, 3] + /// let slice = decoder.get_length_prefixed_slice()?; /// ``` pub fn get_length_prefixed_slice(&mut self) -> Result { check_length!(self.offset, self.limit); @@ -1053,7 +1036,11 @@ impl<'a> Decoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Decoder; + /// let vec = vec![1, 2, 3]; + /// let mut decoder = Decoder::with_vec(&vec); + /// // [1, 2, 3] + /// let buf = unsafe { decoder.uncheck_get_buf(3) }; /// ``` pub unsafe fn uncheck_get_buf(&mut self, len: usize) -> Slice { let slice = uncheck_read_buf(&self.data, self.offset, len); @@ -1073,7 +1060,11 @@ impl<'a> Decoder<'a> { /// # Examples /// /// ``` - /// + /// use level_db_rust::util::coding::Decoder; + /// let vec = vec![1, 2, 3]; + /// let mut decoder = Decoder::with_vec(&vec); + /// // [1, 2, 3] + /// let buf = decoder.get_buf(3)?; /// ``` pub fn get_buf(&self, len: usize) -> Result { check_length!(self.offset, len, self.limit); @@ -1082,50 +1073,6 @@ impl<'a> Decoder<'a> { } } - /// 获取buf写入到dst 不检查长度 - /// - /// # Safety - /// * self.offset + dst.len() < self.limit, 否则溢出 - /// - /// # Arguments - /// - /// * `data`: 待解码数据 - /// * `dst`: 目标数组, 需要指定长度的 - /// - /// returns: () - /// - /// # Examples - /// - /// ``` - /// - /// ``` - pub unsafe fn uncheck_get_into_buf(&self, dst: &mut [u8]) { - // todo 增加长度字段, 以写入到dst的任意位置 - uncheck_read_into_buf(&self.data, self.offset, dst) - } - - /// 获取buf写入到dst - /// - /// # Arguments - /// - /// * `data`: 待解码数据 - /// * `dst`: 目标数组, 需要指定长度的 - /// - /// returns: Result<(), Status> - /// - /// # Examples - /// - /// ``` - /// - /// ``` - pub fn get_into_buf(&self, dst: &mut [u8]) -> Result<()> { - check_length!(self.offset, dst.len(), self.limit); - unsafe { - uncheck_read_into_buf(&self.data, self.offset, dst); - } - Ok(()) - } - /// 跳过一段长度 偏移量会移动到跳过后的位置继续读取 未检查偏移量 /// /// # Safety @@ -1491,15 +1438,6 @@ fn test_read_buf() { println!("{:?}", buf); assert_eq!(&[1_u8, 2, 3, 4] as &[u8; 4], buf.deref()); - let mut dst = [0; 5]; - unsafe { uncheck_read_into_buf(&Vector(&vec), 0, &mut dst) }; - println!("{:?}", dst); - assert_eq!(&[1_u8, 2, 3, 4, 5] as &[u8; 5], &dst); - - let mut dst = [0; 4]; - unsafe { uncheck_read_into_buf(&Vector(&vec), 5, &mut dst) }; - println!("{:?}", dst); - assert_eq!(&[1_u8, 2, 3, 4] as &[u8; 4], &dst); } #[test] @@ -1541,14 +1479,8 @@ fn test_mixed_encode_decode() { println!("{:?}", buf); assert_eq!(&[1_u8, 2, 3, 4] as &[u8; 4], buf.deref()); - let mut dst = [0; 4]; - unsafe { uncheck_read_into_buf(&Vector(&vec), offset, &mut dst) }; - offset += 4; - println!("{:?}", dst); - assert_eq!(&[1_u8, 2, 3, 4] as &[u8; 4], &dst as &[u8; 4]); - println!("offset: {}", offset); - assert_eq!(26, offset); + assert_eq!(22, offset); } #[test] @@ -1755,17 +1687,21 @@ fn test_get_varint() -> Result<()> { } #[test] -fn test_put_buf() { +fn test_put_buf() -> Result<()> { let mut vec = vec![]; let mut encoder = Encoder::with_vec(&mut vec); let buf = [1, 2, 3]; unsafe { encoder.uncheck_put_buf(&buf) } println!("{:?}", buf); - assert_eq!(&[1_u8, 2, 3], vec.as_slice()) + encoder.put_buf(&buf)?; + assert_eq!(&[1_u8, 2, 3, 1, 2, 3], vec.as_slice()); + println!("{:?}", vec); + + Ok(()) } #[test] -fn test_get_buf() { +fn test_get_buf() -> Result<()> { let mut vec = vec![]; { let mut encoder = Encoder::with_vec(&mut vec); @@ -1778,7 +1714,9 @@ fn test_get_buf() { let buf = unsafe { decoder.uncheck_get_buf(3) }; println!("{:?}", buf); assert_eq!(Slice::from_vec(vec![1, 2, 3]), buf); - assert_eq!(3, decoder.offset) + assert_eq!(3, decoder.offset); + + Ok(()) } #[test] -- Gitee