use super::*; use std::ptr; use std::cmp; // FastCDC // Paper: "FastCDC: a Fast and Efficient Content-Defined Chunking Approach for Data Deduplication" // Paper-URL: https://www.usenix.org/system/files/conference/atc16/atc16-paper-xia.pdf // Presentation: https://www.usenix.org/sites/default/files/conference/protected-files/atc16_slides_xia.pdf // Creating 256 pseudo-random values (based on Knuth's MMIX) fn create_gear(seed: u64) -> [u64; 256] { let mut table = [0u64; 256]; let a = 6_364_136_223_846_793_005; let c = 1_442_695_040_888_963_407; let mut v = seed; for t in &mut table.iter_mut() { v = v.wrapping_mul(a).wrapping_add(c); *t = v; } table } fn get_masks(avg_size: usize, nc_level: usize, seed: u64) -> (u64, u64) { let bits = (avg_size.next_power_of_two() - 1).count_ones(); if bits == 13 { // From the paper return (0x0003_5907_0353_0000, 0x0000_d900_0353_0000); } let mut mask = 0u64; let mut v = seed; let a = 6_364_136_223_846_793_005; let c = 1_442_695_040_888_963_407; while mask.count_ones() < bits - nc_level as u32 { v = v.wrapping_mul(a).wrapping_add(c); mask = (mask | 1).rotate_left(v as u32 & 0x3f); } let mask_long = mask; while mask.count_ones() < bits + nc_level as u32 { v = v.wrapping_mul(a).wrapping_add(c); mask = (mask | 1).rotate_left(v as u32 & 0x3f); } let mask_short = mask; (mask_short, mask_long) } pub struct FastCdcChunker { buffer: [u8; 0x1000], buffered: usize, gear: [u64; 256], min_size: usize, max_size: usize, avg_size: usize, mask_long: u64, mask_short: u64, } impl FastCdcChunker { pub fn new(avg_size: usize, seed: u64) -> Self { let (mask_short, mask_long) = get_masks(avg_size, 2, seed); FastCdcChunker { buffer: [0; 0x1000], buffered: 0, gear: create_gear(seed), min_size: avg_size/4, max_size: avg_size*8, avg_size: avg_size, mask_long: mask_long, mask_short: mask_short, } } } impl FastCdcChunker { fn write_output(&mut self, w: &mut Write, pos: usize, max: usize) -> Result { debug_assert!(max <= self.buffer.len()); debug_assert!(pos <= self.buffer.len()); try!(w.write_all(&self.buffer[..pos]).map_err(ChunkerError::Write)); unsafe { ptr::copy(self.buffer[pos..].as_ptr(), self.buffer.as_mut_ptr(), max-pos) }; self.buffered = max-pos; Ok(ChunkerStatus::Continue) } } impl Chunker for FastCdcChunker { #[allow(unknown_lints,explicit_counter_loop,needless_range_loop)] fn chunk(&mut self, r: &mut Read, w: &mut Write) -> Result { let mut max; let mut hash = 0u64; let mut pos = 0; loop { // Fill the buffer, there might be some bytes still in there from last chunk max = try!(r.read(&mut self.buffer[self.buffered..]).map_err(ChunkerError::Read)) + self.buffered; // If nothing to do, finish if max == 0 { return Ok(ChunkerStatus::Finished) } let min_size_p = cmp::min(max, cmp::max(self.min_size as isize - pos as isize, 0) as usize); let avg_size_p = cmp::min(max, cmp::max(self.avg_size as isize - pos as isize, 0) as usize); let max_size_p = cmp::min(max, cmp::max(self.max_size as isize - pos as isize, 0) as usize); // Skipping first min_size bytes. This is ok as same data still results in same hash. if self.avg_size > pos { for i in min_size_p..avg_size_p { hash = (hash << 1).wrapping_add(self.gear[self.buffer[i] as usize]); if hash & self.mask_short == 0 { return self.write_output(w, i + 1, max); } } } if self.max_size > pos { for i in avg_size_p..max_size_p { hash = (hash << 1).wrapping_add(self.gear[self.buffer[i] as usize]); if hash & self.mask_long == 0 { return self.write_output(w, i+1, max); } } } if max + pos >= self.max_size { return self.write_output(w, max_size_p, max); } pos += max; try!(w.write_all(&self.buffer[..max]).map_err(ChunkerError::Write)); self.buffered = 0; } } }