From dad8f0cfb20addbe1252658752be0c12ef5cd21d Mon Sep 17 00:00:00 2001 From: wangboo <5417808+wangboa@user.noreply.gitee.com> Date: Tue, 13 Dec 2022 21:56:12 +0800 Subject: [PATCH 01/15] add crc code --- src/util/crc.rs | 152 ++++++++++++++++++++++++++++++++++++++++++++++++ src/util/mod.rs | 1 + 2 files changed, 153 insertions(+) create mode 100644 src/util/crc.rs diff --git a/src/util/crc.rs b/src/util/crc.rs new file mode 100644 index 0000000..85e8f33 --- /dev/null +++ b/src/util/crc.rs @@ -0,0 +1,152 @@ +static K_MASK_DELTA: u32 = 0xa282ead8; +// CRCs are pre- and post- conditioned by xoring with all ones. +static K_CRC32_XOR: u32 = 0xffffffff; + +const K_BYTE_EXTENSION_TABLE: [u32; 256] = [ +0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, +0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, +0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, +0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, +0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, +0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, +0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, +0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, +0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, +0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, +0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, 0x417b1dbc, 0xb3109ebf, +0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, +0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, +0xed03a29b, 0x1f682198, 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, +0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f, +0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, +0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, +0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, +0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, +0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, +0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, +0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, +0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x082f63b7, 0xfa44e0b4, +0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, +0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, +0xb4091bff, 0x466298fc, 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, +0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5, +0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, +0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, +0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, +0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, +0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, +0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, +0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, +0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, +0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, +0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, +0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, +0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, +0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, +0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, +0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, +0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351]; + +pub struct CRC {} + +// step1, Process one byte at a time. +macro_rules! step1 { + () => { + let c = (l & 0xff) ^ data[s]; + s+=1; + l = K_BYTE_EXTENSION_TABLE[c] ^ (l >> 8); + } +} + +macro_rules! step4 { + ($name: ident) => { + crc$name = Coding + } +} + +impl CRC { + + + /// 基于初始的CRC码和数据生成新的CRC码 + /// + /// # Arguments + /// + /// * `init_crc`: 初始的CRC码 + /// * `data`: 数据 + /// + /// returns: u32 新的CRC码 + /// + /// # Examples + /// + /// ``` + ///let init_crc = CRC::value([1_u8,2,3,4,5].as_ref()); + ///let crc = CRC::extend(init_crc, [1_u8,2,3,4,5].as_ref()); + /// ``` + pub fn extend(init_crc: u32, data: &[u8]) -> u32 { + let mut s = 0_usize; + let n = data.len(); + let mut l = init_crc ^ K_CRC32_XOR; + + crc0 + + todo!() + } + + + /// 生成CRC码 + /// + /// # Arguments + /// + /// * `data`: 数据 + /// + /// returns: u32 CRC码 + /// + /// # Examples + /// + /// ``` + ///let crc = CRC::value([1_u8,2,3,4,5].as_ref()); + /// ``` + #[inline] + pub fn value(data: &[u8]) -> u32 { + CRC::extend(0, data) + } + + /// 将CRC码转为CRC掩码 + /// + /// # Arguments + /// + /// * `crc`: CRC码 + /// + /// returns: u32 CRC掩码 + /// + /// # Examples + /// + /// ``` + /// let mask = CRC::mask(123_u32); + /// ``` + #[inline] + pub fn mask(crc: u32) -> u32 { + // Rotate right by 15 bits and add a constant. + ((crc >> 15) | (crc << 17)) + kMaskDelta + } + + /// 将CRC掩码转为CRC码 + /// + /// # Arguments + /// + /// * `masked_crc`: 掩码 + /// + /// returns: u32 CRC码 + /// + /// # Examples + /// + /// ``` + ///let crc = CRC::unmask(32_u32); + /// ``` + #[inline] + pub fn unmask(masked_crc: u32) -> u32 { + let rot = masked_crc - kMaskDelta; + (rot >> 17) | (rot << 15) + } + +} \ No newline at end of file diff --git a/src/util/mod.rs b/src/util/mod.rs index b696650..a4092d5 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -17,6 +17,7 @@ pub mod status; mod status_test; mod comparator; mod comparator_test; +mod crc; /// 定义别名 pub type Result = result::Result; -- Gitee From d465a811ddf78da12c82430c7c100232fd69bbf4 Mon Sep 17 00:00:00 2001 From: wangboo <5417808+wangboa@user.noreply.gitee.com> Date: Tue, 13 Dec 2022 22:57:32 +0800 Subject: [PATCH 02/15] add crc code --- src/lib.rs | 2 + src/util/crc.rs | 160 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 157 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3768322..db17daa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,5 @@ +#![feature(concat_idents)] + mod db; mod table; mod util; diff --git a/src/util/crc.rs b/src/util/crc.rs index 85e8f33..57c9c3b 100644 --- a/src/util/crc.rs +++ b/src/util/crc.rs @@ -1,3 +1,6 @@ +use crate::traits::coding_trait::CodingTrait; +use crate::util::coding::Coding; + static K_MASK_DELTA: u32 = 0xa282ead8; // CRCs are pre- and post- conditioned by xoring with all ones. static K_CRC32_XOR: u32 = 0xffffffff; @@ -47,6 +50,146 @@ const K_BYTE_EXTENSION_TABLE: [u32; 256] = [ 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351]; +const K_STRIDE_EXTENSION_TABLE1: [u32; 256] = [ + 0x00000000, 0x54075546, 0xa80eaa8c, 0xfc09ffca, 0x55f123e9, 0x01f676af, + 0xfdff8965, 0xa9f8dc23, 0xabe247d2, 0xffe51294, 0x03eced5e, 0x57ebb818, + 0xfe13643b, 0xaa14317d, 0x561dceb7, 0x021a9bf1, 0x5228f955, 0x062fac13, + 0xfa2653d9, 0xae21069f, 0x07d9dabc, 0x53de8ffa, 0xafd77030, 0xfbd02576, + 0xf9cabe87, 0xadcdebc1, 0x51c4140b, 0x05c3414d, 0xac3b9d6e, 0xf83cc828, + 0x043537e2, 0x503262a4, 0xa451f2aa, 0xf056a7ec, 0x0c5f5826, 0x58580d60, + 0xf1a0d143, 0xa5a78405, 0x59ae7bcf, 0x0da92e89, 0x0fb3b578, 0x5bb4e03e, + 0xa7bd1ff4, 0xf3ba4ab2, 0x5a429691, 0x0e45c3d7, 0xf24c3c1d, 0xa64b695b, + 0xf6790bff, 0xa27e5eb9, 0x5e77a173, 0x0a70f435, 0xa3882816, 0xf78f7d50, + 0x0b86829a, 0x5f81d7dc, 0x5d9b4c2d, 0x099c196b, 0xf595e6a1, 0xa192b3e7, + 0x086a6fc4, 0x5c6d3a82, 0xa064c548, 0xf463900e, 0x4d4f93a5, 0x1948c6e3, + 0xe5413929, 0xb1466c6f, 0x18beb04c, 0x4cb9e50a, 0xb0b01ac0, 0xe4b74f86, + 0xe6add477, 0xb2aa8131, 0x4ea37efb, 0x1aa42bbd, 0xb35cf79e, 0xe75ba2d8, + 0x1b525d12, 0x4f550854, 0x1f676af0, 0x4b603fb6, 0xb769c07c, 0xe36e953a, + 0x4a964919, 0x1e911c5f, 0xe298e395, 0xb69fb6d3, 0xb4852d22, 0xe0827864, + 0x1c8b87ae, 0x488cd2e8, 0xe1740ecb, 0xb5735b8d, 0x497aa447, 0x1d7df101, + 0xe91e610f, 0xbd193449, 0x4110cb83, 0x15179ec5, 0xbcef42e6, 0xe8e817a0, + 0x14e1e86a, 0x40e6bd2c, 0x42fc26dd, 0x16fb739b, 0xeaf28c51, 0xbef5d917, + 0x170d0534, 0x430a5072, 0xbf03afb8, 0xeb04fafe, 0xbb36985a, 0xef31cd1c, + 0x133832d6, 0x473f6790, 0xeec7bbb3, 0xbac0eef5, 0x46c9113f, 0x12ce4479, + 0x10d4df88, 0x44d38ace, 0xb8da7504, 0xecdd2042, 0x4525fc61, 0x1122a927, + 0xed2b56ed, 0xb92c03ab, 0x9a9f274a, 0xce98720c, 0x32918dc6, 0x6696d880, + 0xcf6e04a3, 0x9b6951e5, 0x6760ae2f, 0x3367fb69, 0x317d6098, 0x657a35de, + 0x9973ca14, 0xcd749f52, 0x648c4371, 0x308b1637, 0xcc82e9fd, 0x9885bcbb, + 0xc8b7de1f, 0x9cb08b59, 0x60b97493, 0x34be21d5, 0x9d46fdf6, 0xc941a8b0, + 0x3548577a, 0x614f023c, 0x635599cd, 0x3752cc8b, 0xcb5b3341, 0x9f5c6607, + 0x36a4ba24, 0x62a3ef62, 0x9eaa10a8, 0xcaad45ee, 0x3eced5e0, 0x6ac980a6, + 0x96c07f6c, 0xc2c72a2a, 0x6b3ff609, 0x3f38a34f, 0xc3315c85, 0x973609c3, + 0x952c9232, 0xc12bc774, 0x3d2238be, 0x69256df8, 0xc0ddb1db, 0x94dae49d, + 0x68d31b57, 0x3cd44e11, 0x6ce62cb5, 0x38e179f3, 0xc4e88639, 0x90efd37f, + 0x39170f5c, 0x6d105a1a, 0x9119a5d0, 0xc51ef096, 0xc7046b67, 0x93033e21, + 0x6f0ac1eb, 0x3b0d94ad, 0x92f5488e, 0xc6f21dc8, 0x3afbe202, 0x6efcb744, + 0xd7d0b4ef, 0x83d7e1a9, 0x7fde1e63, 0x2bd94b25, 0x82219706, 0xd626c240, + 0x2a2f3d8a, 0x7e2868cc, 0x7c32f33d, 0x2835a67b, 0xd43c59b1, 0x803b0cf7, + 0x29c3d0d4, 0x7dc48592, 0x81cd7a58, 0xd5ca2f1e, 0x85f84dba, 0xd1ff18fc, + 0x2df6e736, 0x79f1b270, 0xd0096e53, 0x840e3b15, 0x7807c4df, 0x2c009199, + 0x2e1a0a68, 0x7a1d5f2e, 0x8614a0e4, 0xd213f5a2, 0x7beb2981, 0x2fec7cc7, + 0xd3e5830d, 0x87e2d64b, 0x73814645, 0x27861303, 0xdb8fecc9, 0x8f88b98f, + 0x267065ac, 0x727730ea, 0x8e7ecf20, 0xda799a66, 0xd8630197, 0x8c6454d1, + 0x706dab1b, 0x246afe5d, 0x8d92227e, 0xd9957738, 0x259c88f2, 0x719bddb4, + 0x21a9bf10, 0x75aeea56, 0x89a7159c, 0xdda040da, 0x74589cf9, 0x205fc9bf, + 0xdc563675, 0x88516333, 0x8a4bf8c2, 0xde4cad84, 0x2245524e, 0x76420708, + 0xdfbadb2b, 0x8bbd8e6d, 0x77b471a7, 0x23b324e1 +]; + +const K_STRIDE_EXTENSION_TABLE2: [u32; 256] = [ + 0x00000000, 0x678efd01, 0xcf1dfa02, 0xa8930703, 0x9bd782f5, 0xfc597ff4, + 0x54ca78f7, 0x334485f6, 0x3243731b, 0x55cd8e1a, 0xfd5e8919, 0x9ad07418, + 0xa994f1ee, 0xce1a0cef, 0x66890bec, 0x0107f6ed, 0x6486e636, 0x03081b37, + 0xab9b1c34, 0xcc15e135, 0xff5164c3, 0x98df99c2, 0x304c9ec1, 0x57c263c0, + 0x56c5952d, 0x314b682c, 0x99d86f2f, 0xfe56922e, 0xcd1217d8, 0xaa9cead9, + 0x020fedda, 0x658110db, 0xc90dcc6c, 0xae83316d, 0x0610366e, 0x619ecb6f, + 0x52da4e99, 0x3554b398, 0x9dc7b49b, 0xfa49499a, 0xfb4ebf77, 0x9cc04276, + 0x34534575, 0x53ddb874, 0x60993d82, 0x0717c083, 0xaf84c780, 0xc80a3a81, + 0xad8b2a5a, 0xca05d75b, 0x6296d058, 0x05182d59, 0x365ca8af, 0x51d255ae, + 0xf94152ad, 0x9ecfafac, 0x9fc85941, 0xf846a440, 0x50d5a343, 0x375b5e42, + 0x041fdbb4, 0x639126b5, 0xcb0221b6, 0xac8cdcb7, 0x97f7ee29, 0xf0791328, + 0x58ea142b, 0x3f64e92a, 0x0c206cdc, 0x6bae91dd, 0xc33d96de, 0xa4b36bdf, + 0xa5b49d32, 0xc23a6033, 0x6aa96730, 0x0d279a31, 0x3e631fc7, 0x59ede2c6, + 0xf17ee5c5, 0x96f018c4, 0xf371081f, 0x94fff51e, 0x3c6cf21d, 0x5be20f1c, + 0x68a68aea, 0x0f2877eb, 0xa7bb70e8, 0xc0358de9, 0xc1327b04, 0xa6bc8605, + 0x0e2f8106, 0x69a17c07, 0x5ae5f9f1, 0x3d6b04f0, 0x95f803f3, 0xf276fef2, + 0x5efa2245, 0x3974df44, 0x91e7d847, 0xf6692546, 0xc52da0b0, 0xa2a35db1, + 0x0a305ab2, 0x6dbea7b3, 0x6cb9515e, 0x0b37ac5f, 0xa3a4ab5c, 0xc42a565d, + 0xf76ed3ab, 0x90e02eaa, 0x387329a9, 0x5ffdd4a8, 0x3a7cc473, 0x5df23972, + 0xf5613e71, 0x92efc370, 0xa1ab4686, 0xc625bb87, 0x6eb6bc84, 0x09384185, + 0x083fb768, 0x6fb14a69, 0xc7224d6a, 0xa0acb06b, 0x93e8359d, 0xf466c89c, + 0x5cf5cf9f, 0x3b7b329e, 0x2a03aaa3, 0x4d8d57a2, 0xe51e50a1, 0x8290ada0, + 0xb1d42856, 0xd65ad557, 0x7ec9d254, 0x19472f55, 0x1840d9b8, 0x7fce24b9, + 0xd75d23ba, 0xb0d3debb, 0x83975b4d, 0xe419a64c, 0x4c8aa14f, 0x2b045c4e, + 0x4e854c95, 0x290bb194, 0x8198b697, 0xe6164b96, 0xd552ce60, 0xb2dc3361, + 0x1a4f3462, 0x7dc1c963, 0x7cc63f8e, 0x1b48c28f, 0xb3dbc58c, 0xd455388d, + 0xe711bd7b, 0x809f407a, 0x280c4779, 0x4f82ba78, 0xe30e66cf, 0x84809bce, + 0x2c139ccd, 0x4b9d61cc, 0x78d9e43a, 0x1f57193b, 0xb7c41e38, 0xd04ae339, + 0xd14d15d4, 0xb6c3e8d5, 0x1e50efd6, 0x79de12d7, 0x4a9a9721, 0x2d146a20, + 0x85876d23, 0xe2099022, 0x878880f9, 0xe0067df8, 0x48957afb, 0x2f1b87fa, + 0x1c5f020c, 0x7bd1ff0d, 0xd342f80e, 0xb4cc050f, 0xb5cbf3e2, 0xd2450ee3, + 0x7ad609e0, 0x1d58f4e1, 0x2e1c7117, 0x49928c16, 0xe1018b15, 0x868f7614, + 0xbdf4448a, 0xda7ab98b, 0x72e9be88, 0x15674389, 0x2623c67f, 0x41ad3b7e, + 0xe93e3c7d, 0x8eb0c17c, 0x8fb73791, 0xe839ca90, 0x40aacd93, 0x27243092, + 0x1460b564, 0x73ee4865, 0xdb7d4f66, 0xbcf3b267, 0xd972a2bc, 0xbefc5fbd, + 0x166f58be, 0x71e1a5bf, 0x42a52049, 0x252bdd48, 0x8db8da4b, 0xea36274a, + 0xeb31d1a7, 0x8cbf2ca6, 0x242c2ba5, 0x43a2d6a4, 0x70e65352, 0x1768ae53, + 0xbffba950, 0xd8755451, 0x74f988e6, 0x137775e7, 0xbbe472e4, 0xdc6a8fe5, + 0xef2e0a13, 0x88a0f712, 0x2033f011, 0x47bd0d10, 0x46bafbfd, 0x213406fc, + 0x89a701ff, 0xee29fcfe, 0xdd6d7908, 0xbae38409, 0x1270830a, 0x75fe7e0b, + 0x107f6ed0, 0x77f193d1, 0xdf6294d2, 0xb8ec69d3, 0x8ba8ec25, 0xec261124, + 0x44b51627, 0x233beb26, 0x223c1dcb, 0x45b2e0ca, 0xed21e7c9, 0x8aaf1ac8, + 0xb9eb9f3e, 0xde65623f, 0x76f6653c, 0x1178983d +]; + +const K_STRIDE_EXTENSION_TABLE3: [u32; 256] = [ + 0x00000000, 0xf20c0dfe, 0xe1f46d0d, 0x13f860f3, 0xc604aceb, 0x3408a115, + 0x27f0c1e6, 0xd5fccc18, 0x89e52f27, 0x7be922d9, 0x6811422a, 0x9a1d4fd4, + 0x4fe183cc, 0xbded8e32, 0xae15eec1, 0x5c19e33f, 0x162628bf, 0xe42a2541, + 0xf7d245b2, 0x05de484c, 0xd0228454, 0x222e89aa, 0x31d6e959, 0xc3dae4a7, + 0x9fc30798, 0x6dcf0a66, 0x7e376a95, 0x8c3b676b, 0x59c7ab73, 0xabcba68d, + 0xb833c67e, 0x4a3fcb80, 0x2c4c517e, 0xde405c80, 0xcdb83c73, 0x3fb4318d, + 0xea48fd95, 0x1844f06b, 0x0bbc9098, 0xf9b09d66, 0xa5a97e59, 0x57a573a7, + 0x445d1354, 0xb6511eaa, 0x63add2b2, 0x91a1df4c, 0x8259bfbf, 0x7055b241, + 0x3a6a79c1, 0xc866743f, 0xdb9e14cc, 0x29921932, 0xfc6ed52a, 0x0e62d8d4, + 0x1d9ab827, 0xef96b5d9, 0xb38f56e6, 0x41835b18, 0x527b3beb, 0xa0773615, + 0x758bfa0d, 0x8787f7f3, 0x947f9700, 0x66739afe, 0x5898a2fc, 0xaa94af02, + 0xb96ccff1, 0x4b60c20f, 0x9e9c0e17, 0x6c9003e9, 0x7f68631a, 0x8d646ee4, + 0xd17d8ddb, 0x23718025, 0x3089e0d6, 0xc285ed28, 0x17792130, 0xe5752cce, + 0xf68d4c3d, 0x048141c3, 0x4ebe8a43, 0xbcb287bd, 0xaf4ae74e, 0x5d46eab0, + 0x88ba26a8, 0x7ab62b56, 0x694e4ba5, 0x9b42465b, 0xc75ba564, 0x3557a89a, + 0x26afc869, 0xd4a3c597, 0x015f098f, 0xf3530471, 0xe0ab6482, 0x12a7697c, + 0x74d4f382, 0x86d8fe7c, 0x95209e8f, 0x672c9371, 0xb2d05f69, 0x40dc5297, + 0x53243264, 0xa1283f9a, 0xfd31dca5, 0x0f3dd15b, 0x1cc5b1a8, 0xeec9bc56, + 0x3b35704e, 0xc9397db0, 0xdac11d43, 0x28cd10bd, 0x62f2db3d, 0x90fed6c3, + 0x8306b630, 0x710abbce, 0xa4f677d6, 0x56fa7a28, 0x45021adb, 0xb70e1725, + 0xeb17f41a, 0x191bf9e4, 0x0ae39917, 0xf8ef94e9, 0x2d1358f1, 0xdf1f550f, + 0xcce735fc, 0x3eeb3802, 0xb13145f8, 0x433d4806, 0x50c528f5, 0xa2c9250b, + 0x7735e913, 0x8539e4ed, 0x96c1841e, 0x64cd89e0, 0x38d46adf, 0xcad86721, + 0xd92007d2, 0x2b2c0a2c, 0xfed0c634, 0x0cdccbca, 0x1f24ab39, 0xed28a6c7, + 0xa7176d47, 0x551b60b9, 0x46e3004a, 0xb4ef0db4, 0x6113c1ac, 0x931fcc52, + 0x80e7aca1, 0x72eba15f, 0x2ef24260, 0xdcfe4f9e, 0xcf062f6d, 0x3d0a2293, + 0xe8f6ee8b, 0x1afae375, 0x09028386, 0xfb0e8e78, 0x9d7d1486, 0x6f711978, + 0x7c89798b, 0x8e857475, 0x5b79b86d, 0xa975b593, 0xba8dd560, 0x4881d89e, + 0x14983ba1, 0xe694365f, 0xf56c56ac, 0x07605b52, 0xd29c974a, 0x20909ab4, + 0x3368fa47, 0xc164f7b9, 0x8b5b3c39, 0x795731c7, 0x6aaf5134, 0x98a35cca, + 0x4d5f90d2, 0xbf539d2c, 0xacabfddf, 0x5ea7f021, 0x02be131e, 0xf0b21ee0, + 0xe34a7e13, 0x114673ed, 0xc4babff5, 0x36b6b20b, 0x254ed2f8, 0xd742df06, + 0xe9a9e704, 0x1ba5eafa, 0x085d8a09, 0xfa5187f7, 0x2fad4bef, 0xdda14611, + 0xce5926e2, 0x3c552b1c, 0x604cc823, 0x9240c5dd, 0x81b8a52e, 0x73b4a8d0, + 0xa64864c8, 0x54446936, 0x47bc09c5, 0xb5b0043b, 0xff8fcfbb, 0x0d83c245, + 0x1e7ba2b6, 0xec77af48, 0x398b6350, 0xcb876eae, 0xd87f0e5d, 0x2a7303a3, + 0x766ae09c, 0x8466ed62, 0x979e8d91, 0x6592806f, 0xb06e4c77, 0x42624189, + 0x519a217a, 0xa3962c84, 0xc5e5b67a, 0x37e9bb84, 0x2411db77, 0xd61dd689, + 0x03e11a91, 0xf1ed176f, 0xe215779c, 0x10197a62, 0x4c00995d, 0xbe0c94a3, + 0xadf4f450, 0x5ff8f9ae, 0x8a0435b6, 0x78083848, 0x6bf058bb, 0x99fc5545, + 0xd3c39ec5, 0x21cf933b, 0x3237f3c8, 0xc03bfe36, 0x15c7322e, 0xe7cb3fd0, + 0xf4335f23, 0x063f52dd, 0x5a26b1e2, 0xa82abc1c, 0xbbd2dcef, 0x49ded111, + 0x9c221d09, 0x6e2e10f7, 0x7dd67004, 0x8fda7dfa +]; + + + pub struct CRC {} // step1, Process one byte at a time. @@ -58,9 +201,14 @@ macro_rules! step1 { } } +// Process one of the 4 strides of 4-byte data. macro_rules! step4 { - ($name: ident) => { - crc$name = Coding + ($name: ident, $s: expr) => { + $ident = Coding::decode_fixed32(&data[$s*4..]) ^ + K_STRIDE_EXTENSION_TABLE3[$ident as u8] ^ + K_STRIDE_EXTENSION_TABLE2[($ident >> 8) as u8] ^ + K_STRIDE_EXTENSION_TABLE2[($ident >> 16) as u8] ^ + K_STRIDE_EXTENSION_TABLE2[($ident >> 24) as u8]; } } @@ -87,7 +235,9 @@ impl CRC { let n = data.len(); let mut l = init_crc ^ K_CRC32_XOR; - crc0 + // step1!(); + + todo!() } @@ -127,7 +277,7 @@ impl CRC { #[inline] pub fn mask(crc: u32) -> u32 { // Rotate right by 15 bits and add a constant. - ((crc >> 15) | (crc << 17)) + kMaskDelta + ((crc >> 15) | (crc << 17)) + K_MASK_DELTA } /// 将CRC掩码转为CRC码 @@ -145,7 +295,7 @@ impl CRC { /// ``` #[inline] pub fn unmask(masked_crc: u32) -> u32 { - let rot = masked_crc - kMaskDelta; + let rot = masked_crc - K_MASK_DELTA; (rot >> 17) | (rot << 15) } -- Gitee From db77bd9fccba656ec6ab2bcff15f50f176b8574a Mon Sep 17 00:00:00 2001 From: wangboo <5417808+wangboa@user.noreply.gitee.com> Date: Wed, 14 Dec 2022 20:01:39 +0800 Subject: [PATCH 03/15] add code --- src/util/crc.rs | 69 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/src/util/crc.rs b/src/util/crc.rs index 57c9c3b..d009a10 100644 --- a/src/util/crc.rs +++ b/src/util/crc.rs @@ -1,3 +1,4 @@ +use std::alloc::Layout; use crate::traits::coding_trait::CodingTrait; use crate::util::coding::Coding; @@ -192,23 +193,45 @@ const K_STRIDE_EXTENSION_TABLE3: [u32; 256] = [ pub struct CRC {} -// step1, Process one byte at a time. +/// step1, Process one byte at a time. macro_rules! step1 { - () => { - let c = (l & 0xff) ^ data[s]; - s+=1; - l = K_BYTE_EXTENSION_TABLE[c] ^ (l >> 8); + ($data: tt, $s: tt, $l: tt) => { + let c = ($l & 0xff) ^ ($data[0] as u32); + $s+=1; + $l = K_BYTE_EXTENSION_TABLE[c as usize] ^ ($l >> 8); } } -// Process one of the 4 strides of 4-byte data. +/// Process one of the 4 strides of 4-byte data. macro_rules! step4 { - ($name: ident, $s: expr) => { - $ident = Coding::decode_fixed32(&data[$s*4..]) ^ - K_STRIDE_EXTENSION_TABLE3[$ident as u8] ^ - K_STRIDE_EXTENSION_TABLE2[($ident >> 8) as u8] ^ - K_STRIDE_EXTENSION_TABLE2[($ident >> 16) as u8] ^ - K_STRIDE_EXTENSION_TABLE2[($ident >> 24) as u8]; + ($name: ident, $data: tt, $i: tt) => { + $name = Coding::decode_fixed32(&$data[$i*4..]) ^ + K_STRIDE_EXTENSION_TABLE3[$name as u8 as usize] ^ + K_STRIDE_EXTENSION_TABLE2[($name >> 8) as u8 as usize] ^ + K_STRIDE_EXTENSION_TABLE2[($name >> 16) as u8 as usize] ^ + K_STRIDE_EXTENSION_TABLE2[($name >> 24) as u8 as usize]; + } +} + +/// Process a 16-byte swath of 4 strides, each of which has 4 bytes of data. +macro_rules! step16 { + ($name: ident, $data: tt, $s: tt) => { + step4!($name, $data, 0); + step4!($name, $data, 1); + step4!($name, $data, 2); + step4!($name, $data, 3); + $s += 16; + } +} + +/// Process 4 bytes that were already loaded into a word. +macro_rules! stepw { + ($w: tt, $l: tt) => { + $w ^= $l; + for _ in 0..4 { + $w = ($w >> 8) ^ K_BYTE_EXTENSION_TABLE[$w as u8 as usize]; + } + $l = $w; } } @@ -235,9 +258,22 @@ impl CRC { let n = data.len(); let mut l = init_crc ^ K_CRC32_XOR; - // step1!(); + step1!(data, s, l); + let mut crc0 = 0_u32; + let mut crc1 = 0_u32; + let mut crc2 = 0_u32; + let mut crc3 = 0_u32; + step4!(crc0, data, 0); + + step16!(crc0, data, s); + step16!(crc1, data, s); + step16!(crc2, data, s); + step16!(crc3, data, s); + + let mut w = 12_u32; + stepw!(w, l); todo!() } @@ -299,4 +335,11 @@ impl CRC { (rot >> 17) | (rot << 15) } + /// 指针对齐到 4byte 需要的偏移量 + fn ptr_align_by4_offset(ptr: *const u8) -> usize { + let addr = ptr as usize; + // eg: addr = 10, output = 8 + (addr+3) & !(addr-3) + } + } \ No newline at end of file -- Gitee From 343820274c3d511ed796d9ab333504a43356ba4d Mon Sep 17 00:00:00 2001 From: wangboo Date: Wed, 14 Dec 2022 21:11:40 +0800 Subject: [PATCH 04/15] add code --- src/lib.rs | 2 -- src/util/slice_test.rs | 7 ++----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index db17daa..3768322 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,3 @@ -#![feature(concat_idents)] - mod db; mod table; mod util; diff --git a/src/util/slice_test.rs b/src/util/slice_test.rs index 06591c9..a2e0ec5 100644 --- a/src/util/slice_test.rs +++ b/src/util/slice_test.rs @@ -1,6 +1,5 @@ mod test { use std::cmp::Ordering; - use std::mem::ManuallyDrop; use crate::util::slice::Slice; #[test] @@ -12,10 +11,8 @@ mod test { let a1 = Slice::from(String::from("123")); assert_eq!(String::from("123"), String::from(a1)); // from buf - // let mut data = ManuallyDrop::new([48_u8, 49, 50]); - // let slice = data.as_mut_slice(); - // let a2 = Slice::from_buf(slice); - // assert_eq!(String::from("012"), String::from(a2)); + let a2 = Slice::from_buf([48_u8, 49, 50].as_mut_slice()); + assert_eq!(String::from("012"), String::from(a2)); } #[test] -- Gitee From f8fb7f696f296d212519b653618857221717b5ac Mon Sep 17 00:00:00 2001 From: fengyang Date: Wed, 14 Dec 2022 21:29:08 +0800 Subject: [PATCH 05/15] level_error_toString --- src/traits/comparator_trait.rs | 13 +++--- src/util/comparator.rs | 77 ++++++++++++++++++++-------------- src/util/comparator_test.rs | 2 +- src/util/status.rs | 75 ++++++++++++++++++++------------- src/util/status_test.rs | 59 +++++++++++++++++--------- 5 files changed, 140 insertions(+), 86 deletions(-) diff --git a/src/traits/comparator_trait.rs b/src/traits/comparator_trait.rs index 2d82ba6..aaa0138 100644 --- a/src/traits/comparator_trait.rs +++ b/src/traits/comparator_trait.rs @@ -31,11 +31,14 @@ pub trait ComparatorTrait { /// 返回comparator的名字 fn get_name() -> String; - /// 函数：用于减少像index blocks这样的内部数据结构占用的空间, 其中的*start和*key参数都是IN OUT的。 - /// - /// 这个函数的作用就是： - /// 如果*start < limit，就在[startlimit,)中找到一个短字符串，并赋给*start返回. 如“helloworld”和”hellozoomer”之间最短的key可以是”hellox”。 - /// 简单的comparator实现可能不改变*start(start==limit)，这也是正确的 + /// 函数：找到start、limit之间最短的字符串，如“helloworld”和”hellozoomer”之间最短的key可以是”hellox” + /// + /// 作用是： + /// 作用是，如果start < limit,就把start修改为*start和limit的共同前缀后面多一个字符加1 + /// 例如： + /// start: helloWorld + /// limit: helloZookeeper + /// 由于 *start < limit, 所以调用 FindShortSuccessor(start, limit)之后，start变成： helloX (保留前缀，第一个不相同的字符+1) /// /// # Arguments /// diff --git a/src/util/comparator.rs b/src/util/comparator.rs index 51d0123..1e455bc 100644 --- a/src/util/comparator.rs +++ b/src/util/comparator.rs @@ -1,5 +1,5 @@ -use std::cmp::Ordering; +use std::cmp::{min, Ordering}; use crate::traits::comparator_trait::{ComparatorTrait}; use crate::util::slice::Slice; @@ -18,7 +18,20 @@ impl ComparatorTrait for BytewiseComparatorImpl { } fn find_shortest_separator(&self, start: &String, limit: &Slice) -> String { - todo!() + // 首先计算共同前缀字符串的长度 + let min_lengrh: usize = min(start.len(), limit.len()); + + // let mut diff_index: usize = 0; + // let start_char_vec: Vec = start.chars().collect::>(); + // let limit_char_vec: Vec = limit.chars().collect::>(); + // while ( + // (diff_index < min_lengt) && + // (start_char_vec[diff_index] == limit_char_vec[diff_index]) + // ){ + // diff_index = diff_index + 1; + // } + + String::from("") } fn find_short_successor(&self, key: &String) -> String { @@ -32,33 +45,33 @@ impl Default for BytewiseComparatorImpl { } } -/// InternalKeyComparator -pub struct InternalKeyComparator { - // fn user_comparator(&self) -> Box { - // todo!() - // } - - // fn Compare(InternalKey, InternalKey) -} - -/// InternalKeyComparator 比较器: 用来比较内部键（Internal Key）。 -/// 内部键值是为了方便处理，将原普通键、序列号和值类型组成的新键。 -impl ComparatorTrait for InternalKeyComparator { - // todo InternalKeyComparator 的构造方法 - - fn compare(&self, a: &Slice, b: &Slice) -> Option { - todo!() - } - - fn get_name() -> String { - String::from("leveldb.InternalKeyComparator") - } - - fn find_shortest_separator(&self, start: &String, limit: &Slice) -> String { - todo!() - } - - fn find_short_successor(&self, key: &String) -> String { - todo!() - } -} +// /// InternalKeyComparator +// pub struct InternalKeyComparator { +// // fn user_comparator(&self) -> Box { +// // todo!() +// // } +// +// // fn Compare(InternalKey, InternalKey) +// } +// +// /// InternalKeyComparator 比较器: 用来比较内部键（Internal Key）。 +// /// 内部键值是为了方便处理，将原普通键、序列号和值类型组成的新键。 +// impl ComparatorTrait for InternalKeyComparator { +// // todo InternalKeyComparator 的构造方法 +// +// fn compare(&self, a: &Slice, b: &Slice) -> Option { +// todo!() +// } +// +// fn get_name() -> String { +// String::from("leveldb.InternalKeyComparator") +// } +// +// fn find_shortest_separator(&self, start: &String, limit: &Slice) -> String { +// todo!() +// } +// +// fn find_short_successor(&self, key: &String) -> String { +// todo!() +// } +// } diff --git a/src/util/comparator_test.rs b/src/util/comparator_test.rs index b7f3af3..420581f 100644 --- a/src/util/comparator_test.rs +++ b/src/util/comparator_test.rs @@ -2,7 +2,7 @@ mod test { use std::cmp::Ordering; use crate::traits::comparator_trait::ComparatorTrait; - use crate::util::comparator::{BytewiseComparatorImpl, InternalKeyComparator}; + use crate::util::comparator::{BytewiseComparatorImpl}; use crate::util::slice::Slice; #[test] diff --git a/src/util/status.rs b/src/util/status.rs index 9b534bd..9bb6662 100644 --- a/src/util/status.rs +++ b/src/util/status.rs @@ -1,5 +1,4 @@ use std::fmt::{Display, Error, Formatter}; -use std::ops::Deref; use crate::util::r#const::COLON_WHITE_SPACE; use crate::util::ResultT; use crate::util::slice::Slice; @@ -78,6 +77,10 @@ impl Status { self.err.is_invalid_argument() } + pub fn get_error_string(&self) -> String { + self.err.to_string() + } + /// 请注意， err 的所有权会发生转移！！！ pub fn get_error(self) -> LevelError { self.err @@ -102,8 +105,8 @@ impl Status { /// assert_eq!("abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc: 456456456456456456456456456456456456456456456456", /// String::from(slice.unwrap())); /// ``` - pub fn into_msg(self) -> Option { - Some(self.msg) + pub fn into_msg(self) -> Slice { + self.msg } @@ -144,6 +147,29 @@ impl Status { } } +// impl Display for Status { +// fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { +// let mut print = String::new(); +// +// if self.is_ok() { +// print.push_str("OK"); +// write!(f, "{}", print); +// +// return Ok(()); +// } +// +// print.push_str(&self.get_error_string()); +// +// let slice: &Slice = &self.msg; +// let errMsg = String::from(slice); +// print.push_str(errMsg.as_str()); +// +// write!(f, "{}", print); +// +// Ok(()) +// } +// } + /// Status 的状态 pub enum LevelError { KOk, @@ -316,32 +342,23 @@ impl TryFrom for LevelError { } } -// /// 实现Display，自动提供to_string -// impl Display for LevelError { -// fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { -// let mut print = String::new(); -// -// if self.is_default() { -// print.push_str("OK"); -// return write!(f, "{}", print); -// } -// -// let msg_type = match self { -// KOk => "OK", -// KNotFound(_) => "NotFound: ", -// KCorruption(_) => "Corruption: ", -// KNotSupported(_) => "Not implemented: ", -// KInvalidArgument(_) => "Invalid argument: ", -// KIOError(_) => "IO error: " -// }; -// print.push_str(msg_type); -// -// let mut msg_err: Slice = self.into_msg().unwrap(); -// print.push_str(msg_err.borrow_data().as_str()); -// -// write!(f, "{}", print) -// } -// } +impl Display for LevelError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut print = String::new(); + + let msg_type = match &self { + KOk => "OK", + KNotFound => "NotFound: ", + KCorruption => "Corruption: ", + KNotSupported => "Not implemented: ", + KInvalidArgument => "Invalid argument: ", + KIOError => "IO error: " + }; + print.push_str(msg_type); + + write!(f, "{}", print) + } +} // impl Deref for LevelError { // type Target = i32; diff --git a/src/util/status_test.rs b/src/util/status_test.rs index af745d0..eadccd3 100644 --- a/src/util/status_test.rs +++ b/src/util/status_test.rs @@ -1,5 +1,6 @@ mod test { + use std::borrow::Cow; use crate::util::r#const::COLON_WHITE_SPACE; use crate::util::slice::Slice; use crate::util::status::{LevelError, Status}; @@ -10,9 +11,9 @@ mod test { let status = Status::wrapper(LevelError::KIOError, String::from(msg1).into()); assert!(&status.is_io_error()); - let slice: Option = status.into_msg(); + let slice: Slice = status.into_msg(); assert_eq!("abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc", - String::from(slice.unwrap())); + String::from(slice)); let ss = Status::wrapper(LevelError::KOk, String::from(msg1).into()); assert!(&ss.is_ok()); @@ -25,27 +26,27 @@ mod test { let msg2 = "456456456456456456456456456456456456456456456456"; let status = Status::wrappers(LevelError::KIOError, String::from(msg1).into(), String::from(msg2).into()); - let slice: Option = status.into_msg(); + let slice: Slice = status.into_msg(); assert_eq!("abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc: 456456456456456456456456456456456456456456456456", - String::from(slice.unwrap())); + String::from(slice)); let err: Status = LevelError::invalid_argument(String::from(msg1).into(), String::from(msg2).into()); - assert!(&err.get_error().is_invalid_argument()); + assert!(&err.is_invalid_argument()); // assert_eq!(err.into_code(), 4); let err: Status = LevelError::corruption(String::from(msg1).into(), String::from(msg2).into()); - assert!(&err.get_error().is_corruption()); + assert!(&err.is_corruption()); // assert_eq!(err.into_code(), 2); let err: Status = LevelError::not_found(String::from(msg1).into(), String::from(msg2).into()); - assert!(&err.get_error().is_not_found()); + assert!(&err.is_not_found()); let err: Status = LevelError::not_supported(String::from(msg1).into(), String::from(msg2).into()); - assert!(&err.get_error().is_not_supported_error()); + assert!(&err.is_not_supported_error()); let err: LevelError = LevelError::KOk; assert!(&err.is_ok()); @@ -55,7 +56,22 @@ mod test { } #[test] - fn test_toString() { + fn test_is_default() { + let err: Status = LevelError::ok(); + assert!(err.is_ok()); + + let err: Status = LevelError::io_error(String::from("a").into(), + String::from("b").into()); + assert!(!err.is_ok()); + + let status: Status = LevelError::not_found(String::from("a").into(), + String::from("b").into()); + assert!(status.is_not_found()); + assert!(status.get_error().is_not_found()); + } + + #[test] + fn test_status_to_string() { // ok let status: Status = LevelError::ok(); assert_eq!("OK", status.to_string()); @@ -84,21 +100,26 @@ mod test { } #[test] - fn test_is_default() { - let err: Status = LevelError::ok(); - assert!(err.get_error().is_ok()); + fn test_level_error_toString() { + // ok + let status: Status = LevelError::ok(); + assert_eq!("OK", status.to_string()); - let err: Status = LevelError::io_error(String::from("a").into(), - String::from("b").into()); - assert!(!err.get_error().is_ok()); + // err invalid_argument + let msg1 = "bcabcabcabcabcabcbc"; + let msg2 = "56"; + let error: Status = LevelError::invalid_argument(String::from(msg1).into(), + String::from(msg2).into()); + + let le_err: LevelError = error.get_error(); + println!("{}", &le_err); - // let err: LevelError = LevelError::ok(); - // let a = err.into(); - // print!("{}", a); + // Display + assert_eq!(String::from("Invalid argument: "), le_err.to_string()); } #[test] - fn test_try_from() -> Result<(), String> { + fn test_level_error_try_from() -> Result<(), String> { let rs = LevelError::try_from(1)?; assert!(&rs.is_not_found()); let rs: Result = 1.try_into(); -- Gitee From faf5d310283c984b5b723b1bd88c70e68fbf653b Mon Sep 17 00:00:00 2001 From: wangboo Date: Wed, 14 Dec 2022 22:03:11 +0800 Subject: [PATCH 06/15] add code --- src/util/crc.rs | 62 ++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/src/util/crc.rs b/src/util/crc.rs index d009a10..14e9094 100644 --- a/src/util/crc.rs +++ b/src/util/crc.rs @@ -215,11 +215,11 @@ macro_rules! step4 { /// Process a 16-byte swath of 4 strides, each of which has 4 bytes of data. macro_rules! step16 { - ($name: ident, $data: tt, $s: tt) => { - step4!($name, $data, 0); - step4!($name, $data, 1); - step4!($name, $data, 2); - step4!($name, $data, 3); + ($c0: tt, $c1: tt, $c2: tt, $c3: tt, $data: tt, $s: tt) => { + step4!($c0, $data, 0); + step4!($c1, $data, 1); + step4!($c2, $data, 2); + step4!($c3, $data, 3); $s += 16; } } @@ -258,23 +258,28 @@ impl CRC { let n = data.len(); let mut l = init_crc ^ K_CRC32_XOR; + // 4 byte align offset + let x = ptr_align_by4_offset(data.as_ptr()); + while s < x { + step1!(data, s, l); + } - - step1!(data, s, l); - let mut crc0 = 0_u32; - let mut crc1 = 0_u32; - let mut crc2 = 0_u32; - let mut crc3 = 0_u32; - step4!(crc0, data, 0); - - step16!(crc0, data, s); - step16!(crc1, data, s); - step16!(crc2, data, s); - step16!(crc3, data, s); - - let mut w = 12_u32; - stepw!(w, l); - + if n - s >= 16 { + let mut crc0 = Coding::decode_fixed32(&data[s..]) ^ l; + let mut crc1 = Coding::decode_fixed32(&data[(s+4)..]); + let mut crc2 = Coding::decode_fixed32(&data[(s+8)..]); + let mut crc3 = Coding::decode_fixed32(&data[(s+12)..]); + s += 16; + while (n - s) >= 16 { + step16!(crc0, crc1, crc2, crc3, data, s); + } + while (n-s) >= 4 { + step4!(crc0, data, 0); + // swap variables + (crc1, crc2, crc3) = (crc0, crc1, crc2); + s += 4; + } + } todo!() } @@ -335,11 +340,14 @@ impl CRC { (rot >> 17) | (rot << 15) } - /// 指针对齐到 4byte 需要的偏移量 - fn ptr_align_by4_offset(ptr: *const u8) -> usize { - let addr = ptr as usize; - // eg: addr = 10, output = 8 - (addr+3) & !(addr-3) - } + +} + +/// 指针对齐到 4byte 需要的偏移量 +fn ptr_align_by4_offset(ptr: *const u8) -> usize { + let addr = ptr as usize; + // eg: addr = 10, output = 2 + // (addr+3) & !(addr-3) + addr % 4 } \ No newline at end of file -- Gitee From 5723668fa893154a3ae8456810034f5cb0d80985 Mon Sep 17 00:00:00 2001 From: wangboo Date: Wed, 14 Dec 2022 22:03:37 +0800 Subject: [PATCH 07/15] add to vec func --- src/util/slice.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/util/slice.rs b/src/util/slice.rs index b767f8d..c9ddf7f 100644 --- a/src/util/slice.rs +++ b/src/util/slice.rs @@ -102,6 +102,12 @@ impl From for String { } } +impl From for Vec { + fn from(s: Slice) -> Self { + s.data + } +} + impl > From for Slice { fn from(r: R) -> Self { Self { -- Gitee From 608682e4f818d98f768590a756ae845686882b87 Mon Sep 17 00:00:00 2001 From: wangboo Date: Wed, 14 Dec 2022 23:43:24 +0800 Subject: [PATCH 08/15] finish crc and test --- src/util/crc.rs | 90 +++++++++++++++++++++++++++++++++++++------- src/util/crc_test.rs | 40 ++++++++++++++++++++ src/util/mod.rs | 1 + 3 files changed, 117 insertions(+), 14 deletions(-) create mode 100644 src/util/crc_test.rs diff --git a/src/util/crc.rs b/src/util/crc.rs index 14e9094..ed6244b 100644 --- a/src/util/crc.rs +++ b/src/util/crc.rs @@ -1,4 +1,3 @@ -use std::alloc::Layout; use crate::traits::coding_trait::CodingTrait; use crate::util::coding::Coding; @@ -51,6 +50,52 @@ const K_BYTE_EXTENSION_TABLE: [u32; 256] = [ 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351]; +const K_STRIDE_EXTENSION_TABLE0: [u32; 256] = [ + 0x00000000, 0x30d23865, 0x61a470ca, 0x517648af, 0xc348e194, 0xf39ad9f1, + 0xa2ec915e, 0x923ea93b, 0x837db5d9, 0xb3af8dbc, 0xe2d9c513, 0xd20bfd76, + 0x4035544d, 0x70e76c28, 0x21912487, 0x11431ce2, 0x03171d43, 0x33c52526, + 0x62b36d89, 0x526155ec, 0xc05ffcd7, 0xf08dc4b2, 0xa1fb8c1d, 0x9129b478, + 0x806aa89a, 0xb0b890ff, 0xe1ced850, 0xd11ce035, 0x4322490e, 0x73f0716b, + 0x228639c4, 0x125401a1, 0x062e3a86, 0x36fc02e3, 0x678a4a4c, 0x57587229, + 0xc566db12, 0xf5b4e377, 0xa4c2abd8, 0x941093bd, 0x85538f5f, 0xb581b73a, + 0xe4f7ff95, 0xd425c7f0, 0x461b6ecb, 0x76c956ae, 0x27bf1e01, 0x176d2664, + 0x053927c5, 0x35eb1fa0, 0x649d570f, 0x544f6f6a, 0xc671c651, 0xf6a3fe34, + 0xa7d5b69b, 0x97078efe, 0x8644921c, 0xb696aa79, 0xe7e0e2d6, 0xd732dab3, + 0x450c7388, 0x75de4bed, 0x24a80342, 0x147a3b27, 0x0c5c750c, 0x3c8e4d69, + 0x6df805c6, 0x5d2a3da3, 0xcf149498, 0xffc6acfd, 0xaeb0e452, 0x9e62dc37, + 0x8f21c0d5, 0xbff3f8b0, 0xee85b01f, 0xde57887a, 0x4c692141, 0x7cbb1924, + 0x2dcd518b, 0x1d1f69ee, 0x0f4b684f, 0x3f99502a, 0x6eef1885, 0x5e3d20e0, + 0xcc0389db, 0xfcd1b1be, 0xada7f911, 0x9d75c174, 0x8c36dd96, 0xbce4e5f3, + 0xed92ad5c, 0xdd409539, 0x4f7e3c02, 0x7fac0467, 0x2eda4cc8, 0x1e0874ad, + 0x0a724f8a, 0x3aa077ef, 0x6bd63f40, 0x5b040725, 0xc93aae1e, 0xf9e8967b, + 0xa89eded4, 0x984ce6b1, 0x890ffa53, 0xb9ddc236, 0xe8ab8a99, 0xd879b2fc, + 0x4a471bc7, 0x7a9523a2, 0x2be36b0d, 0x1b315368, 0x096552c9, 0x39b76aac, + 0x68c12203, 0x58131a66, 0xca2db35d, 0xfaff8b38, 0xab89c397, 0x9b5bfbf2, + 0x8a18e710, 0xbacadf75, 0xebbc97da, 0xdb6eafbf, 0x49500684, 0x79823ee1, + 0x28f4764e, 0x18264e2b, 0x18b8ea18, 0x286ad27d, 0x791c9ad2, 0x49cea2b7, + 0xdbf00b8c, 0xeb2233e9, 0xba547b46, 0x8a864323, 0x9bc55fc1, 0xab1767a4, + 0xfa612f0b, 0xcab3176e, 0x588dbe55, 0x685f8630, 0x3929ce9f, 0x09fbf6fa, + 0x1baff75b, 0x2b7dcf3e, 0x7a0b8791, 0x4ad9bff4, 0xd8e716cf, 0xe8352eaa, + 0xb9436605, 0x89915e60, 0x98d24282, 0xa8007ae7, 0xf9763248, 0xc9a40a2d, + 0x5b9aa316, 0x6b489b73, 0x3a3ed3dc, 0x0aecebb9, 0x1e96d09e, 0x2e44e8fb, + 0x7f32a054, 0x4fe09831, 0xddde310a, 0xed0c096f, 0xbc7a41c0, 0x8ca879a5, + 0x9deb6547, 0xad395d22, 0xfc4f158d, 0xcc9d2de8, 0x5ea384d3, 0x6e71bcb6, + 0x3f07f419, 0x0fd5cc7c, 0x1d81cddd, 0x2d53f5b8, 0x7c25bd17, 0x4cf78572, + 0xdec92c49, 0xee1b142c, 0xbf6d5c83, 0x8fbf64e6, 0x9efc7804, 0xae2e4061, + 0xff5808ce, 0xcf8a30ab, 0x5db49990, 0x6d66a1f5, 0x3c10e95a, 0x0cc2d13f, + 0x14e49f14, 0x2436a771, 0x7540efde, 0x4592d7bb, 0xd7ac7e80, 0xe77e46e5, + 0xb6080e4a, 0x86da362f, 0x97992acd, 0xa74b12a8, 0xf63d5a07, 0xc6ef6262, + 0x54d1cb59, 0x6403f33c, 0x3575bb93, 0x05a783f6, 0x17f38257, 0x2721ba32, + 0x7657f29d, 0x4685caf8, 0xd4bb63c3, 0xe4695ba6, 0xb51f1309, 0x85cd2b6c, + 0x948e378e, 0xa45c0feb, 0xf52a4744, 0xc5f87f21, 0x57c6d61a, 0x6714ee7f, + 0x3662a6d0, 0x06b09eb5, 0x12caa592, 0x22189df7, 0x736ed558, 0x43bced3d, + 0xd1824406, 0xe1507c63, 0xb02634cc, 0x80f40ca9, 0x91b7104b, 0xa165282e, + 0xf0136081, 0xc0c158e4, 0x52fff1df, 0x622dc9ba, 0x335b8115, 0x0389b970, + 0x11ddb8d1, 0x210f80b4, 0x7079c81b, 0x40abf07e, 0xd2955945, 0xe2476120, + 0xb331298f, 0x83e311ea, 0x92a00d08, 0xa272356d, 0xf3047dc2, 0xc3d645a7, + 0x51e8ec9c, 0x613ad4f9, 0x304c9c56, 0x009ea433 +]; + const K_STRIDE_EXTENSION_TABLE1: [u32; 256] = [ 0x00000000, 0x54075546, 0xa80eaa8c, 0xfc09ffca, 0x55f123e9, 0x01f676af, 0xfdff8965, 0xa9f8dc23, 0xabe247d2, 0xffe51294, 0x03eced5e, 0x57ebb818, @@ -196,7 +241,7 @@ pub struct CRC {} /// step1, Process one byte at a time. macro_rules! step1 { ($data: tt, $s: tt, $l: tt) => { - let c = ($l & 0xff) ^ ($data[0] as u32); + let c = ($l & 0xff) ^ ($data[$s] as u32); $s+=1; $l = K_BYTE_EXTENSION_TABLE[c as usize] ^ ($l >> 8); } @@ -204,22 +249,22 @@ macro_rules! step1 { /// Process one of the 4 strides of 4-byte data. macro_rules! step4 { - ($name: ident, $data: tt, $i: tt) => { - $name = Coding::decode_fixed32(&$data[$i*4..]) ^ + ($name: ident, $data: tt, $s: tt, $i: tt) => { + $name = Coding::decode_fixed32(&$data[$s+$i*4..]) ^ K_STRIDE_EXTENSION_TABLE3[$name as u8 as usize] ^ K_STRIDE_EXTENSION_TABLE2[($name >> 8) as u8 as usize] ^ - K_STRIDE_EXTENSION_TABLE2[($name >> 16) as u8 as usize] ^ - K_STRIDE_EXTENSION_TABLE2[($name >> 24) as u8 as usize]; + K_STRIDE_EXTENSION_TABLE1[($name >> 16) as u8 as usize] ^ + K_STRIDE_EXTENSION_TABLE0[($name >> 24) as u8 as usize]; } } /// Process a 16-byte swath of 4 strides, each of which has 4 bytes of data. macro_rules! step16 { ($c0: tt, $c1: tt, $c2: tt, $c3: tt, $data: tt, $s: tt) => { - step4!($c0, $data, 0); - step4!($c1, $data, 1); - step4!($c2, $data, 2); - step4!($c3, $data, 3); + step4!($c0, $data, $s, 0); + step4!($c1, $data, $s, 1); + step4!($c2, $data, $s, 2); + step4!($c3, $data, $s, 3); $s += 16; } } @@ -250,16 +295,18 @@ impl CRC { /// # Examples /// /// ``` - ///let init_crc = CRC::value([1_u8,2,3,4,5].as_ref()); - ///let crc = CRC::extend(init_crc, [1_u8,2,3,4,5].as_ref()); + ///let init_crc = CRC::value(&[1_u8,2,3,4,5]); + ///let crc = CRC::extend(init_crc, &[1_u8,2,3,4,5]); /// ``` pub fn extend(init_crc: u32, data: &[u8]) -> u32 { + // s: start_offset, n: data length, l: result crc let mut s = 0_usize; let n = data.len(); let mut l = init_crc ^ K_CRC32_XOR; // 4 byte align offset let x = ptr_align_by4_offset(data.as_ptr()); + // println!("x: {}, l: {:x}, n: {}", x, l, n); while s < x { step1!(data, s, l); } @@ -270,17 +317,32 @@ impl CRC { let mut crc2 = Coding::decode_fixed32(&data[(s+8)..]); let mut crc3 = Coding::decode_fixed32(&data[(s+12)..]); s += 16; + // println!("c0: {:x}, c1: {:x}, c2: {:x}, c3: {:x}, s: {}", crc0, crc1, crc2, crc3, s); while (n - s) >= 16 { step16!(crc0, crc1, crc2, crc3, data, s); + // println!("step16, c0: {:x}, c1: {:x}, c2: {:x}, c3: {:x}, s: {}", crc0, crc1, crc2, crc3, s); } while (n-s) >= 4 { - step4!(crc0, data, 0); + step4!(crc0, data, s, 0); // swap variables (crc1, crc2, crc3) = (crc0, crc1, crc2); s += 4; + // println!("step4, c0: {:x}, c1: {:x}, c2: {:x}, c3: {:x}, s: {}", crc0, crc1, crc2, crc3, s); } + // Combine the 4 partial stride results. + l = 0; + stepw!(crc0, l); + stepw!(crc1, l); + stepw!(crc2, l); + stepw!(crc3, l); + } + + while s != n { + step1!(data, s, l); + // println!("step1, l: {:x}, s: {}", l, s); } - todo!() + + l ^ K_CRC32_XOR } diff --git a/src/util/crc_test.rs b/src/util/crc_test.rs new file mode 100644 index 0000000..0167006 --- /dev/null +++ b/src/util/crc_test.rs @@ -0,0 +1,40 @@ +use crate::util::crc::CRC; + +#[test] +fn test_crc() { + // From rfc3720 section B.4. + let mut buf = [0_u8; 32]; + let crc0 = CRC::value(&buf); + assert_eq!(0x8a9136aa, crc0); + // println!("crc0: {:x}, eq: {:x}", crc0, 0x8a9136aa_u32); + buf.fill(0xff); + assert_eq!(0x62a8ab43, CRC::value(&buf)); + (0..32).for_each(|idx| buf[idx] = idx as u8); + assert_eq!(0x46dd794e, CRC::value(&buf)); + (0..32).for_each(|idx| buf[idx] = 31-(idx as u8)); + assert_eq!(0x113fdb5c, CRC::value(&buf)); + let data = [ + 0x01_u8, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, + 0x00, 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x18, 0x28, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + ]; + assert_eq!(0xd9963a56, CRC::value(&data)); +} + +#[test] +fn test_extend() { + let init_crc = CRC::value("hello ".as_bytes()); + let crc = CRC::extend(init_crc, "world".as_bytes()); + let value = CRC::value("hello world".as_bytes()); + assert_eq!(value, crc); +} + +#[test] +fn test_mask() { + let crc = CRC::value("foo".as_bytes()); + assert_ne!(crc, CRC::mask(crc)); + assert_ne!(crc, CRC::mask(CRC::mask(crc))); + assert_eq!(crc, CRC::unmask(CRC::mask(crc))); + assert_eq!(crc, CRC::unmask(CRC::unmask(CRC::mask(CRC::mask(crc))))); +} \ No newline at end of file diff --git a/src/util/mod.rs b/src/util/mod.rs index cfb31d6..b56627f 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -27,3 +27,4 @@ pub type ResultT = result::Result; pub mod histogram; mod histogram_test; +mod crc_test; -- Gitee From 546b053eab4b83c5065ef3dbd36571b8b89cd53f Mon Sep 17 00:00:00 2001 From: wangboo Date: Thu, 15 Dec 2022 01:12:20 +0800 Subject: [PATCH 09/15] add as_crc, as_crc_extend functions --- src/util/coding.rs | 1 - src/util/crc.rs | 149 ++++++++++++++++++++++++++++--------------- src/util/crc_test.rs | 20 +++++- 3 files changed, 116 insertions(+), 54 deletions(-) diff --git a/src/util/coding.rs b/src/util/coding.rs index 5397dd9..33c8769 100644 --- a/src/util/coding.rs +++ b/src/util/coding.rs @@ -113,7 +113,6 @@ impl CodingTrait for Coding { } fn get_length_prefixed_slice(input: &mut Slice) -> Slice { - let len = 0_u32; let decode = Coding::get_varint32(input); Slice::from_buf(decode.to_le_bytes().as_mut_slice()) } diff --git a/src/util/crc.rs b/src/util/crc.rs index ed6244b..c0f8b34 100644 --- a/src/util/crc.rs +++ b/src/util/crc.rs @@ -1,54 +1,57 @@ +use std::mem::size_of; +use std::slice; use crate::traits::coding_trait::CodingTrait; use crate::util::coding::Coding; +use crate::util::slice::Slice; static K_MASK_DELTA: u32 = 0xa282ead8; // CRCs are pre- and post- conditioned by xoring with all ones. static K_CRC32_XOR: u32 = 0xffffffff; const K_BYTE_EXTENSION_TABLE: [u32; 256] = [ -0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, -0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, -0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, -0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, -0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, -0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, -0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, -0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, -0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, -0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, -0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, 0x417b1dbc, 0xb3109ebf, -0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, -0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, -0xed03a29b, 0x1f682198, 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, -0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f, -0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, -0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, -0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, -0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, -0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, -0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, -0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, -0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x082f63b7, 0xfa44e0b4, -0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, -0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, -0xb4091bff, 0x466298fc, 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, -0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5, -0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, -0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, -0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, -0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, -0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, -0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, -0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, -0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, -0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, -0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, -0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, -0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, -0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, -0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, -0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, -0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351]; + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, 0xc79a971f, 0x35f1141c, + 0x26a1e7e8, 0xd4ca64eb, 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, 0x105ec76f, 0xe235446c, + 0xf165b798, 0x030e349b, 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, 0x5d1d08bf, 0xaf768bbc, + 0xbc267848, 0x4e4dfb4b, 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, 0xaa64d611, 0x580f5512, + 0x4b5fa6e6, 0xb93425e5, 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, 0xf779deae, 0x05125dad, + 0x1642ae59, 0xe4292d5a, 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, 0x417b1dbc, 0xb3109ebf, + 0xa0406d4b, 0x522bee48, 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, 0x0c38d26c, 0xfe53516f, + 0xed03a29b, 0x1f682198, 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, 0xdbfc821c, 0x2997011f, + 0x3ac7f2eb, 0xc8ac71e8, 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, 0xa65c047d, 0x5437877e, + 0x4767748a, 0xb50cf789, 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, 0x7198540d, 0x83f3d70e, + 0x90a324fa, 0x62c8a7f9, 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, 0x3cdb9bdd, 0xceb018de, + 0xdde0eb2a, 0x2f8b6829, 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, 0x082f63b7, 0xfa44e0b4, + 0xe9141340, 0x1b7f9043, 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, 0x55326b08, 0xa759e80b, + 0xb4091bff, 0x466298fc, 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, 0xa24bb5a6, 0x502036a5, + 0x4370c551, 0xb11b4652, 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, 0xef087a76, 0x1d63f975, + 0x0e330a81, 0xfc588982, 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, 0x38cc2a06, 0xcaa7a905, + 0xd9f75af1, 0x2b9cd9f2, 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, 0x0417b1db, 0xf67c32d8, + 0xe52cc12c, 0x1747422f, 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, 0xd3d3e1ab, 0x21b862a8, + 0x32e8915c, 0xc083125f, 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, 0x9e902e7b, 0x6cfbad78, + 0x7fab5e8c, 0x8dc0dd8f, 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, 0x69e9f0d5, 0x9b8273d6, + 0x88d28022, 0x7ab90321, 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69, + 0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351]; const K_STRIDE_EXTENSION_TABLE0: [u32; 256] = [ 0x00000000, 0x30d23865, 0x61a470ca, 0x517648af, 0xc348e194, 0xf39ad9f1, @@ -234,7 +237,54 @@ const K_STRIDE_EXTENSION_TABLE3: [u32; 256] = [ 0x9c221d09, 0x6e2e10f7, 0x7dd67004, 0x8fda7dfa ]; +/// 可以被计算 crc 值的特质 +/// 默认实现了 &[T], Vec[T], Slice, &str, String +pub trait AsCrc { + fn as_crc(&self) -> u32 { + self.as_crc_extend(0) + } + + fn as_crc_extend(&self, crc: u32) -> u32; +} + +impl AsCrc for &[T] { + #[inline] + fn as_crc_extend(&self, crc: u32) -> u32 { + let ptr = self.as_ptr() as *const _ as *const u8; + let buf = unsafe { + slice::from_raw_parts(ptr, size_of::() * self.len()) + }; + CRC::extend(crc, buf) + } +} +impl AsCrc for &str { + #[inline] + fn as_crc_extend(&self, crc: u32) -> u32 { + CRC::extend(crc, self.as_bytes()) + } +} + +impl AsCrc for Slice { + #[inline] + fn as_crc_extend(&self, crc: u32) -> u32 { + CRC::extend(crc, &**self) + } +} + +impl AsCrc for Vec { + #[inline] + fn as_crc_extend(&self, crc: u32) -> u32 { + self.as_slice().as_crc_extend(crc) + } +} + +impl AsCrc for String { + #[inline] + fn as_crc_extend(&self, crc: u32) -> u32 { + CRC::extend(crc, self.as_bytes()) + } +} pub struct CRC {} @@ -281,8 +331,6 @@ macro_rules! stepw { } impl CRC { - - /// 基于初始的CRC码和数据生成新的CRC码 /// /// # Arguments @@ -313,16 +361,16 @@ impl CRC { if n - s >= 16 { let mut crc0 = Coding::decode_fixed32(&data[s..]) ^ l; - let mut crc1 = Coding::decode_fixed32(&data[(s+4)..]); - let mut crc2 = Coding::decode_fixed32(&data[(s+8)..]); - let mut crc3 = Coding::decode_fixed32(&data[(s+12)..]); + let mut crc1 = Coding::decode_fixed32(&data[(s + 4)..]); + let mut crc2 = Coding::decode_fixed32(&data[(s + 8)..]); + let mut crc3 = Coding::decode_fixed32(&data[(s + 12)..]); s += 16; // println!("c0: {:x}, c1: {:x}, c2: {:x}, c3: {:x}, s: {}", crc0, crc1, crc2, crc3, s); while (n - s) >= 16 { step16!(crc0, crc1, crc2, crc3, data, s); // println!("step16, c0: {:x}, c1: {:x}, c2: {:x}, c3: {:x}, s: {}", crc0, crc1, crc2, crc3, s); } - while (n-s) >= 4 { + while (n - s) >= 4 { step4!(crc0, data, s, 0); // swap variables (crc1, crc2, crc3) = (crc0, crc1, crc2); @@ -401,9 +449,6 @@ impl CRC { let rot = masked_crc - K_MASK_DELTA; (rot >> 17) | (rot << 15) } - - - } /// 指针对齐到 4byte 需要的偏移量 diff --git a/src/util/crc_test.rs b/src/util/crc_test.rs index 0167006..8d34ff9 100644 --- a/src/util/crc_test.rs +++ b/src/util/crc_test.rs @@ -1,4 +1,5 @@ -use crate::util::crc::CRC; +use crate::util::crc::{AsCrc, CRC}; +use crate::util::slice::Slice; #[test] fn test_crc() { @@ -30,6 +31,23 @@ fn test_extend() { assert_eq!(value, crc); } +#[test] +fn test_as_crc() { + let expect = CRC::value("123".as_bytes()); + let crc0 = "123".as_crc(); + assert_eq!(expect, crc0); + let crc1 = String::from("123").as_crc(); + assert_eq!(expect, crc1); + let crc2 = Slice::from("123").as_crc(); + assert_eq!(expect, crc2); + let buf = "123".as_bytes(); + let crc3 = buf.as_crc(); + assert_eq!(expect, crc3); + let crc4 = Vec::from(buf).as_crc(); + assert_eq!(expect, crc4); + +} + #[test] fn test_mask() { let crc = CRC::value("foo".as_bytes()); -- Gitee From 0bc18d82911e30655ad2bd2c02545fa87066c89b Mon Sep 17 00:00:00 2001 From: fengyang Date: Thu, 15 Dec 2022 20:38:24 +0800 Subject: [PATCH 10/15] =?UTF-8?q?=E6=9C=80=E7=9F=AD=E7=9A=84=E5=AD=97?= =?UTF-8?q?=E7=AC=A6=E4=B8=B2=20=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/traits/comparator_trait.rs | 31 +++++++++++----- src/util/comparator.rs | 64 ++++++++++++++++++++++++---------- src/util/comparator_test.rs | 49 +++++++++++++++++++++++++- 3 files changed, 116 insertions(+), 28 deletions(-) diff --git a/src/traits/comparator_trait.rs b/src/traits/comparator_trait.rs index aaa0138..a78e054 100644 --- a/src/traits/comparator_trait.rs +++ b/src/traits/comparator_trait.rs @@ -33,29 +33,44 @@ pub trait ComparatorTrait { /// 函数：找到start、limit之间最短的字符串，如“helloworld”和”hellozoomer”之间最短的key可以是”hellox” /// - /// 作用是： - /// 作用是，如果start < limit,就把start修改为*start和limit的共同前缀后面多一个字符加1 + /// 作用是， + /// 如果start < limit,就把start修改为 start和limit的共同前缀, 后面多一个字符加1 + /// /// 例如： /// start: helloWorld /// limit: helloZookeeper - /// 由于 *start < limit, 所以调用 FindShortSuccessor(start, limit)之后，start变成： helloX (保留前缀，第一个不相同的字符+1) + /// 由于 start < limit, 所以调用 FindShortSuccessor(start, limit)之后， + /// start变成： helloX (保留前缀，第一个不相同的字符+1) /// /// # Arguments /// - /// * `start`: - /// * `limit`: + /// * `start`: String + /// * `limit`: Slice /// /// returns: String /// /// # Examples /// /// ``` - /// + /// let comp = BytewiseComparatorImpl::default(); + /// let find_shortest_separator_val = comp.find_shortest_separator( + /// &String::from("abcdefghijklimA"), + /// &Slice::from("abcdefghijklimNy")); + /// /// A < N + /// assert_eq!(find_shortest_separator_val, "abcdefghijklimB"); + /// + /// let comp = BytewiseComparatorImpl::default(); + /// let find_shortest_separator_val = comp.find_shortest_separator( + /// &String::from("abcdefghijklima"), + /// &Slice::from("abcdefghijklimNy")); + /// /// a > N + /// assert_eq!(find_shortest_separator_val, "abcdefghijklima"); /// ``` fn find_shortest_separator(&self, start: &String, limit:&Slice) -> String; - /// 找一个 >= key的短字符串, key变成一个比原*key大的短字符串，并返回。 - /// 简单的comparator实现可能不改变 key，这也是正确的 + /// 用于找到比key大的最短字符串，如传入“helloworld”，返回的key可能是“i”而已。 + /// + /// 找一个 >= key的短字符串, key变成一个比原key大的短字符串，并返回。 /// /// # Arguments /// diff --git a/src/util/comparator.rs b/src/util/comparator.rs index 1e455bc..a5f4442 100644 --- a/src/util/comparator.rs +++ b/src/util/comparator.rs @@ -3,10 +3,18 @@ use std::cmp::{min, Ordering}; use crate::traits::comparator_trait::{ComparatorTrait}; use crate::util::slice::Slice; -/// BytewiseComparatorImpl是按字典逐字节序进行比较 -/// 也就是说 i>helloworld，因为先比较i和h，i>h，比较直接结束 pub struct BytewiseComparatorImpl {} +/// +/// compare: +/// 按字典逐字节序进行比较 +/// 也就是说 i>helloworld，因为先比较i和h，i>h，比较直接结束 +impl Default for BytewiseComparatorImpl { + fn default() -> Self { + Self{} + } +} + impl ComparatorTrait for BytewiseComparatorImpl { fn compare(&self, a: &Slice, b: &Slice) -> Option { @@ -19,29 +27,47 @@ impl ComparatorTrait for BytewiseComparatorImpl { fn find_shortest_separator(&self, start: &String, limit: &Slice) -> String { // 首先计算共同前缀字符串的长度 - let min_lengrh: usize = min(start.len(), limit.len()); + let min_length: usize = min(start.len(), limit.len()); - // let mut diff_index: usize = 0; + let mut diff_index: usize = 0; + let mut start_char_vec: Vec = start.as_bytes().to_vec(); + let limit_char_vec: &Vec = &limit.to_vec(); + // or use // let start_char_vec: Vec = start.chars().collect::>(); - // let limit_char_vec: Vec = limit.chars().collect::>(); - // while ( - // (diff_index < min_lengt) && - // (start_char_vec[diff_index] == limit_char_vec[diff_index]) - // ){ - // diff_index = diff_index + 1; - // } + // let limit_char_vec: Vec = limit.to_vec().iter().map(|b| *b as char).collect::>(); - String::from("") - } + while diff_index < min_length && + start_char_vec[diff_index] == limit_char_vec[diff_index] + { + // Increment counter + diff_index += 1; + } - fn find_short_successor(&self, key: &String) -> String { - todo!() + // 如果一个字符串是另个一字符串的前缀，无需做截短操作，否则进入 else。 + if diff_index >= min_length { + // 说明 start是limit的前缀，或者反之，此时不作修改，直接返回 + } else{ + // 尝试执行字符start[diff_index]++， 设置start长度为diff_index+1，并返回 + // ++条件：字符 < oxff 并且字符+1 < limit上该index的字符 + let diff_byte = start_char_vec[diff_index]; + // let diff_char = diff_byte as char; + + if diff_byte < 0xff && + // 且 start 中的差异字符的next 小于 limit中的diff_index的字符， + // 则将 start 差异字符位置+1的元素变更为 差异字符的next + (diff_byte + 1) < limit_char_vec[diff_index] { + start_char_vec[diff_index] = diff_byte + 1; + } + } + + let shortest_separator: &[u8] = &start_char_vec[0..diff_index+1]; + + let shortest_separator_val: String= Slice::from_buf(shortest_separator).into(); + shortest_separator_val } -} -impl Default for BytewiseComparatorImpl { - fn default() -> Self { - Self{} + fn find_short_successor(&self, key: &String) -> String { + String::from("a") } } diff --git a/src/util/comparator_test.rs b/src/util/comparator_test.rs index 420581f..b0688dc 100644 --- a/src/util/comparator_test.rs +++ b/src/util/comparator_test.rs @@ -8,7 +8,7 @@ mod test { #[test] fn test_bytewise_comparator_impl_get_name() { let name = BytewiseComparatorImpl::get_name(); - println!("{}", &name); + println!("get_name: {}", &name); assert_eq!("leveldb.BytewiseComparator", name); } @@ -18,6 +18,7 @@ mod test { let option_val = comp.compare(&Slice::from("a"), &Slice::from("ab")); assert_eq!(option_val.unwrap(), Ordering::Less); + // todo Slice 存在 bug 未修复 // let comp = BytewiseComparatorImpl::default(); // let option_val = comp.compare(&Slice::from("b"), &Slice::from("abcd")); // assert_eq!(option_val.unwrap(), Ordering::Greater); @@ -27,4 +28,50 @@ mod test { assert_eq!(option_val.unwrap(), Ordering::Equal); } + #[test] + fn test_bytewise_comparator_impl_find_shortest_separator() { + let comp = BytewiseComparatorImpl::default(); + let find_shortest_separator_val = comp.find_shortest_separator( + &String::from("helloWorld"), + &Slice::from("helloZookeeper")); + // W < Z + assert_eq!(find_shortest_separator_val, "helloX"); + + let comp = BytewiseComparatorImpl::default(); + let find_shortest_separator_val = comp.find_shortest_separator( + &String::from("abcdefghijklimx"), + &Slice::from("abcdefghijklimNy")); + // x(!X) > N + assert_eq!(find_shortest_separator_val, "abcdefghijklimx"); + + let comp = BytewiseComparatorImpl::default(); + let find_shortest_separator_val = comp.find_shortest_separator( + &String::from("abcdefghijklimA"), + &Slice::from("abcdefghijklimNy")); + // A < N + assert_eq!(find_shortest_separator_val, "abcdefghijklimB"); + + let comp = BytewiseComparatorImpl::default(); + let find_shortest_separator_val = comp.find_shortest_separator( + &String::from("abcdefghijklima"), + &Slice::from("abcdefghijklimNy")); + // a > N + assert_eq!(find_shortest_separator_val, "abcdefghijklima"); + + let comp = BytewiseComparatorImpl::default(); + let find_shortest_separator_val = comp.find_shortest_separator( + &String::from("abcdefghijklima"), + &Slice::from("abcdefghijklimny")); + // a < n + assert_eq!(find_shortest_separator_val, "abcdefghijklimb"); + } + + #[test] + fn test_bytewise_comparator_impl_find_short_successor() { + let comp = BytewiseComparatorImpl::default(); + let find_short_successor_val = comp.find_short_successor(&String::from("helloWorld")); + println!("find_short_successor_val: {}", &find_short_successor_val); + assert_eq!(find_short_successor_val, "a"); + } + } \ No newline at end of file -- Gitee From 970cc6949a5788e2a03f22917cb050ed3025c2ac Mon Sep 17 00:00:00 2001 From: fengyang Date: Thu, 15 Dec 2022 21:18:56 +0800 Subject: [PATCH 11/15] =?UTF-8?q?=E6=AF=94key=E5=A4=A7=E7=9A=84=E6=9C=80?= =?UTF-8?q?=E7=9F=AD=E5=AD=97=E7=AC=A6=E4=B8=B2=20=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/traits/comparator_trait.rs | 2 +- src/util/comparator.rs | 25 ++++++++++++++-- src/util/comparator_test.rs | 52 ++++++++++++++++++++++++++++++++-- 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/src/traits/comparator_trait.rs b/src/traits/comparator_trait.rs index a78e054..47388a2 100644 --- a/src/traits/comparator_trait.rs +++ b/src/traits/comparator_trait.rs @@ -68,7 +68,7 @@ pub trait ComparatorTrait { /// ``` fn find_shortest_separator(&self, start: &String, limit:&Slice) -> String; - /// 用于找到比key大的最短字符串，如传入“helloworld”，返回的key可能是“i”而已。 + /// 用于找到比key大的最短字符串，如传入“helloworld”，返回的key可能是“i” /// /// 找一个 >= key的短字符串, key变成一个比原key大的短字符串，并返回。 /// diff --git a/src/util/comparator.rs b/src/util/comparator.rs index a5f4442..219539c 100644 --- a/src/util/comparator.rs +++ b/src/util/comparator.rs @@ -36,6 +36,7 @@ impl ComparatorTrait for BytewiseComparatorImpl { // let start_char_vec: Vec = start.chars().collect::>(); // let limit_char_vec: Vec = limit.to_vec().iter().map(|b| *b as char).collect::>(); + assert_eq!(u8::MAX, 255); while diff_index < min_length && start_char_vec[diff_index] == limit_char_vec[diff_index] { @@ -49,10 +50,10 @@ impl ComparatorTrait for BytewiseComparatorImpl { } else{ // 尝试执行字符start[diff_index]++， 设置start长度为diff_index+1，并返回 // ++条件：字符 < oxff 并且字符+1 < limit上该index的字符 - let diff_byte = start_char_vec[diff_index]; + let diff_byte: u8 = start_char_vec[diff_index]; // let diff_char = diff_byte as char; - if diff_byte < 0xff && + if diff_byte < u8::MAX && // 且 start 中的差异字符的next 小于 limit中的diff_index的字符， // 则将 start 差异字符位置+1的元素变更为 差异字符的next (diff_byte + 1) < limit_char_vec[diff_index] { @@ -67,7 +68,25 @@ impl ComparatorTrait for BytewiseComparatorImpl { } fn find_short_successor(&self, key: &String) -> String { - String::from("a") + /// 找到第一个可以++的字符，执行++后，截断字符串； + /// 如果找不到说明 key的字符都是 u8::MAX，直接返回 + let key_len = key.len(); + + let mut key_char_vec: Vec = key.as_bytes().to_vec(); + for i in 0..key_len { + let byte_val: u8 = key_char_vec[i]; + if byte_val != u8::MAX { + key_char_vec[i] = byte_val + 1; + + let short_successor: &[u8] = &key_char_vec[0..i+1]; + + let short_successor_val: String= Slice::from_buf(short_successor).into(); + return short_successor_val; + } + } + /// key is a run of u8::MAX. Leave it alone. + + Slice::from_buf(key.as_bytes()).into() } } diff --git a/src/util/comparator_test.rs b/src/util/comparator_test.rs index b0688dc..11b72ec 100644 --- a/src/util/comparator_test.rs +++ b/src/util/comparator_test.rs @@ -1,6 +1,7 @@ mod test { use std::cmp::Ordering; + use std::io::Write; use crate::traits::comparator_trait::ComparatorTrait; use crate::util::comparator::{BytewiseComparatorImpl}; use crate::util::slice::Slice; @@ -70,8 +71,55 @@ mod test { fn test_bytewise_comparator_impl_find_short_successor() { let comp = BytewiseComparatorImpl::default(); let find_short_successor_val = comp.find_short_successor(&String::from("helloWorld")); - println!("find_short_successor_val: {}", &find_short_successor_val); - assert_eq!(find_short_successor_val, "a"); + assert_eq!(find_short_successor_val, "i"); + + + let comp = BytewiseComparatorImpl::default(); + let find_short_successor_val = comp.find_short_successor(&String::from("a")); + assert_eq!(find_short_successor_val, "b"); + + + let comp = BytewiseComparatorImpl::default(); + let find_short_successor_val = comp.find_short_successor(&String::from("123")); + assert_eq!(find_short_successor_val, "2"); + + + // 只有 u8::MAX + let u8_max_vec: Vec = vec![u8::MAX]; + let u8_max_str = String::from(Slice::from_buf(u8_max_vec.as_slice())); + + let comp = BytewiseComparatorImpl::default(); + let find_short_successor_val = comp.find_short_successor(&u8_max_str); + assert_eq!(u8_max_str, find_short_successor_val); + + + // u8max 结尾 + let mut u8_vec: Vec = vec![]; + u8_vec.write(&String::from("helloWorld").as_bytes().to_vec()); + u8_vec.push(u8::MAX); + + let u8_array_str = String::from(Slice::from_buf(u8_vec.as_slice())); + + let comp = BytewiseComparatorImpl::default(); + let find_short_successor_val = comp.find_short_successor(&u8_array_str); + assert_eq!(find_short_successor_val, "i"); + + + // u8max 开头 + let mut u8_vec: Vec = vec![]; + u8_vec.push(u8::MAX); + u8_vec.write(&String::from("helloWorld").as_bytes().to_vec()); + let u8_max_str = String::from(Slice::from_buf(u8_vec.as_slice())); + + let comp = BytewiseComparatorImpl::default(); + let find_short_successor_val = comp.find_short_successor(&u8_max_str); + + // 只有 u8::MAX + let mut expect_u8_max_vec: Vec = vec![]; + expect_u8_max_vec.push(u8::MAX); + expect_u8_max_vec.write("i".as_bytes()).expect("panic message"); + assert_eq!(find_short_successor_val, + String::from(Slice::from_buf(expect_u8_max_vec.as_slice()))); } } \ No newline at end of file -- Gitee From 0782d1b555cc4178592e3a582bc8fed8ec3d4cfa Mon Sep 17 00:00:00 2001 From: fengyang Date: Thu, 15 Dec 2022 21:20:27 +0800 Subject: [PATCH 12/15] =?UTF-8?q?=E6=AF=94key=E5=A4=A7=E7=9A=84=E6=9C=80?= =?UTF-8?q?=E7=9F=AD=E5=AD=97=E7=AC=A6=E4=B8=B2=20=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/util/comparator.rs | 60 ++++++++++++++++++------------------- src/util/comparator_test.rs | 8 ++++- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/src/util/comparator.rs b/src/util/comparator.rs index 219539c..3c8c01c 100644 --- a/src/util/comparator.rs +++ b/src/util/comparator.rs @@ -90,33 +90,33 @@ impl ComparatorTrait for BytewiseComparatorImpl { } } -// /// InternalKeyComparator -// pub struct InternalKeyComparator { -// // fn user_comparator(&self) -> Box { -// // todo!() -// // } -// -// // fn Compare(InternalKey, InternalKey) -// } -// -// /// InternalKeyComparator 比较器: 用来比较内部键（Internal Key）。 -// /// 内部键值是为了方便处理，将原普通键、序列号和值类型组成的新键。 -// impl ComparatorTrait for InternalKeyComparator { -// // todo InternalKeyComparator 的构造方法 -// -// fn compare(&self, a: &Slice, b: &Slice) -> Option { -// todo!() -// } -// -// fn get_name() -> String { -// String::from("leveldb.InternalKeyComparator") -// } -// -// fn find_shortest_separator(&self, start: &String, limit: &Slice) -> String { -// todo!() -// } -// -// fn find_short_successor(&self, key: &String) -> String { -// todo!() -// } -// } +/// InternalKeyComparator +pub struct InternalKeyComparator { + // fn user_comparator(&self) -> Box { + // todo!() + // } + + // fn Compare(InternalKey, InternalKey) +} + +/// InternalKeyComparator 比较器: 用来比较内部键（Internal Key）。 +/// 内部键值是为了方便处理，将原普通键、序列号和值类型组成的新键。 +impl ComparatorTrait for InternalKeyComparator { + // todo InternalKeyComparator 的构造方法 + + fn compare(&self, a: &Slice, b: &Slice) -> Option { + todo!() + } + + fn get_name() -> String { + String::from("leveldb.InternalKeyComparator") + } + + fn find_shortest_separator(&self, start: &String, limit: &Slice) -> String { + todo!() + } + + fn find_short_successor(&self, key: &String) -> String { + todo!() + } +} diff --git a/src/util/comparator_test.rs b/src/util/comparator_test.rs index 11b72ec..a9b2807 100644 --- a/src/util/comparator_test.rs +++ b/src/util/comparator_test.rs @@ -3,7 +3,7 @@ mod test { use std::cmp::Ordering; use std::io::Write; use crate::traits::comparator_trait::ComparatorTrait; - use crate::util::comparator::{BytewiseComparatorImpl}; + use crate::util::comparator::{BytewiseComparatorImpl, InternalKeyComparator}; use crate::util::slice::Slice; #[test] @@ -122,4 +122,10 @@ mod test { String::from(Slice::from_buf(expect_u8_max_vec.as_slice()))); } + #[test] + fn test_internal_key_comparator_get_name() { + let name = InternalKeyComparator::get_name(); + assert_eq!("leveldb.InternalKeyComparator", name); + } + } \ No newline at end of file -- Gitee From 7fdd44295f301e3eabe7dcfa8f613550ab25e780 Mon Sep 17 00:00:00 2001 From: fengyang Date: Thu, 15 Dec 2022 21:30:17 +0800 Subject: [PATCH 13/15] test --- src/util/comparator.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/util/comparator.rs b/src/util/comparator.rs index 3c8c01c..173c68a 100644 --- a/src/util/comparator.rs +++ b/src/util/comparator.rs @@ -68,8 +68,8 @@ impl ComparatorTrait for BytewiseComparatorImpl { } fn find_short_successor(&self, key: &String) -> String { - /// 找到第一个可以++的字符，执行++后，截断字符串； - /// 如果找不到说明 key的字符都是 u8::MAX，直接返回 + // 找到第一个可以++的字符，执行++后，截断字符串； + // 如果找不到说明 key的字符都是 u8::MAX，直接返回 let key_len = key.len(); let mut key_char_vec: Vec = key.as_bytes().to_vec(); @@ -84,7 +84,6 @@ impl ComparatorTrait for BytewiseComparatorImpl { return short_successor_val; } } - /// key is a run of u8::MAX. Leave it alone. Slice::from_buf(key.as_bytes()).into() } @@ -104,7 +103,7 @@ pub struct InternalKeyComparator { impl ComparatorTrait for InternalKeyComparator { // todo InternalKeyComparator 的构造方法 - fn compare(&self, a: &Slice, b: &Slice) -> Option { + fn compare(&self, _a: &Slice, _b: &Slice) -> Option { todo!() } @@ -112,11 +111,11 @@ impl ComparatorTrait for InternalKeyComparator { String::from("leveldb.InternalKeyComparator") } - fn find_shortest_separator(&self, start: &String, limit: &Slice) -> String { + fn find_shortest_separator(&self, _start: &String, _limit: &Slice) -> String { todo!() } - fn find_short_successor(&self, key: &String) -> String { + fn find_short_successor(&self, _key: &String) -> String { todo!() } } -- Gitee From 077cd5107584d9c22547e8c42a8e7de7fcfafd5c Mon Sep 17 00:00:00 2001 From: fengyang Date: Thu, 15 Dec 2022 22:22:28 +0800 Subject: [PATCH 14/15] readme.md --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 725b374..e323364 100644 --- a/README.md +++ b/README.md @@ -37,18 +37,18 @@ LevelDB for rust 1. 1.0.0 版本, 完成 util 相关的内容 -| 功能模块 | 完成人 | -|-------------------------------|-------------| -| Arena (Memory Management) | wangboo | -| Slice | wangboo | -| Random | colagy | -| Cache | colagy | -| Coding (Primitive Type SerDe) | colagy | -| Comparator | fengyang | -| Status | fengyang | -| BloomFilter | fengyang | -| CRC | lxd5866 | -| Env | lxd5866 | -| Hash | lxd5866 | -| MutexLock | kazeseiriou | -| Histgram | kazeseiriou | \ No newline at end of file +| 功能模块 | 完成人 | 进度 | +|-------------------------------|-----------------|------| +| Arena (Memory Management) | wangboo | | +| Slice | wangboo | | +| Random | colagy | | +| Cache | colagy | | +| Coding (Primitive Type SerDe) | colagy | | +| Comparator | fengyang | 85% | +| Status | fengyang | 100% | +| BloomFilter | fengyang | | +| CRC | wangboo、lxd5866 | | +| Env | lxd5866 | | +| Hash | lxd5866 | | +| MutexLock | kazeseiriou | | +| Histgram | kazeseiriou | | \ No newline at end of file -- Gitee From 159ac905c3d93349c2e881dfe70bb5f6133c9a0f Mon Sep 17 00:00:00 2001 From: fengyang Date: Thu, 15 Dec 2022 22:55:38 +0800 Subject: [PATCH 15/15] =?UTF-8?q?hash=20=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- src/util/hash.rs | 21 +++++++++++++++++++++ src/util/hash_test.rs | 6 ++++++ src/util/mod.rs | 7 ++++--- 4 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 src/util/hash.rs create mode 100644 src/util/hash_test.rs diff --git a/README.md b/README.md index e323364..b7d8eb0 100644 --- a/README.md +++ b/README.md @@ -46,9 +46,9 @@ LevelDB for rust | Coding (Primitive Type SerDe) | colagy | | | Comparator | fengyang | 85% | | Status | fengyang | 100% | -| BloomFilter | fengyang | | +| BloomFilter | fengyang | 0% | | CRC | wangboo、lxd5866 | | | Env | lxd5866 | | -| Hash | lxd5866 | | +| Hash | fengyang | 30% | | MutexLock | kazeseiriou | | | Histgram | kazeseiriou | | \ No newline at end of file diff --git a/src/util/hash.rs b/src/util/hash.rs new file mode 100644 index 0000000..d383f2b --- /dev/null +++ b/src/util/hash.rs @@ -0,0 +1,21 @@ + +pub trait AsHash { + /// + /// + /// # Arguments + /// + /// * `data`: + /// * `n`: data 的长度 + /// * `seed`: 随机数种子 + /// + /// returns: u32 + /// + /// # Examples + /// + /// ``` + /// + /// ``` + fn hash(data: String, n: usize, seed: u32) -> u32; +} + +pub struct Hash {} \ No newline at end of file diff --git a/src/util/hash_test.rs b/src/util/hash_test.rs new file mode 100644 index 0000000..0384579 --- /dev/null +++ b/src/util/hash_test.rs @@ -0,0 +1,6 @@ + +#[test] +fn test_hash() { + + +} \ No newline at end of file diff --git a/src/util/mod.rs b/src/util/mod.rs index b56627f..dab1d24 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,5 +1,6 @@ use crate::util::status::LevelError; use std::result; +pub use arena::Arena; /// 常量定义 pub mod r#const; @@ -11,13 +12,12 @@ mod coding_test; pub mod arena; mod arena_test; -pub use arena::Arena; - pub mod status; mod status_test; pub mod comparator; mod comparator_test; mod crc; +mod crc_test; pub mod bloom_filter; mod bloom_filter_test; pub mod filter_policy; @@ -27,4 +27,5 @@ pub type ResultT = result::Result; pub mod histogram; mod histogram_test; -mod crc_test; +mod hash; +mod hash_test; -- Gitee