Configurable cut-point-skipping in fastcdc

This commit is contained in:
Dennis Schwerdel 2017-08-02 23:36:01 +02:00
parent cbfcc255af
commit b4e6b34bbe
4 changed files with 9 additions and 7 deletions

View File

@ -99,7 +99,7 @@ fn test_rabin_8192(b: &mut Bencher) {
#[bench]
fn test_fastcdc_init(b: &mut Bencher) {
b.iter(|| {
FastCdcChunker::new(8*1024, 0);
FastCdcChunker::new(8*1024, 0, true);
})
}
@ -108,7 +108,7 @@ fn test_fastcdc_8192(b: &mut Bencher) {
let data = random_data(0, 1024*1024);
b.bytes = data.len() as u64;
b.iter(|| {
let mut chunker = FastCdcChunker::new(8*1024, 0);
let mut chunker = FastCdcChunker::new(8*1024, 0, true);
let mut cursor = Cursor::new(&data);
while chunker.chunk(&mut cursor, &mut DevNull).unwrap() == ChunkerStatus::Continue {}
})

View File

@ -54,10 +54,11 @@ pub struct FastCdcChunker {
avg_size: usize,
mask_long: u64,
mask_short: u64,
cut_point_skip: bool,
}
impl FastCdcChunker {
pub fn new(avg_size: usize, seed: u64) -> Self {
pub fn new(avg_size: usize, seed: u64, cut_point_skip: bool) -> Self {
let (mask_short, mask_long) = get_masks(avg_size, 2, seed);
FastCdcChunker {
buffer: [0; 4096],
@ -68,6 +69,7 @@ impl FastCdcChunker {
avg_size: avg_size,
mask_long: mask_long,
mask_short: mask_short,
cut_point_skip: cut_point_skip,
}
}
}
@ -99,7 +101,7 @@ impl Chunker for FastCdcChunker {
let min_size_p = cmp::min(max, cmp::max(self.min_size as isize - pos as isize, 0) as usize);
let avg_size_p = cmp::min(max, cmp::max(self.avg_size as isize - pos as isize, 0) as usize);
let max_size_p = cmp::min(max, cmp::max(self.max_size as isize - pos as isize, 0) as usize);
if self.min_size > pos {
if !self.cut_point_skip && self.min_size > pos {
for i in 0..min_size_p {
hash = (hash << 1).wrapping_add(self.gear[self.buffer[i] as usize]);
}

View File

@ -84,12 +84,12 @@ fn test_rabin() {
fn test_fastcdc() {
let data = random_data(0, 10*1024*1024);
for n in &[1usize,2,4,8,16,32,64,128,256,512,1024] {
let mut chunker = FastCdcChunker::new(1024*n, 0);
let mut chunker = FastCdcChunker::new(1024*n, 0, true);
let len = test_chunking(&mut chunker, &data, None);
assert!(len >= data.len()/n/1024/4);
assert!(len <= data.len()/n/1024*4);
}
test_chunking(&mut FastCdcChunker::new(8192, 0), &random_data(0, 128*1024),
test_chunking(&mut FastCdcChunker::new(8192, 0, true), &random_data(0, 128*1024),
Some(&[8712, 8018, 2847, 9157, 8997, 8581, 8867, 5422, 5412, 9478,
11553, 9206, 4606, 8529, 3821, 11342, 6524]));
}

View File

@ -48,7 +48,7 @@ impl ChunkerType {
match *self {
ChunkerType::Ae(size) => Box::new(AeChunker::new(size)),
ChunkerType::Rabin((size, seed)) => Box::new(RabinChunker::new(size, seed)),
ChunkerType::FastCdc((size, seed)) => Box::new(FastCdcChunker::new(size, seed)),
ChunkerType::FastCdc((size, seed)) => Box::new(FastCdcChunker::new(size, seed, true)),
ChunkerType::Fixed(size) => Box::new(FixedChunker::new(size)),
}
}