Added fixed chunker

This commit is contained in:
Dennis Schwerdel 2017-04-19 19:54:53 +02:00
parent 304dfe16e8
commit bf00a2b156
4 changed files with 55 additions and 23 deletions

17
chunking/Cargo.lock generated
View File

@ -3,8 +3,6 @@ name = "chunking"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.9.14 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]] [[package]]
@ -12,20 +10,5 @@ name = "quick-error"
version = "1.1.0" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "serde"
version = "0.9.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "serde_utils"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde 0.9.14 (registry+https://github.com/rust-lang/crates.io-index)",
]
[metadata] [metadata]
"checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c" "checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c"
"checksum serde 0.9.14 (registry+https://github.com/rust-lang/crates.io-index)" = "a4c9a40d556f8431394def53446db659f796dc87a53ef67b7541f21057fbdd91"
"checksum serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b34a52969c7fc0254e214b82518c9a95dc88c84fc84cd847add314996a031be6"

40
chunking/src/fixed.rs Normal file
View File

@ -0,0 +1,40 @@
use super::*;
use std::cmp::min;
pub struct FixedChunker {
buffer: [u8; 4096],
size: usize
}
impl FixedChunker {
pub fn new(avg_size: usize) -> FixedChunker {
FixedChunker{
buffer: [0; 4096],
size: avg_size,
}
}
}
impl Chunker for FixedChunker {
#[allow(unknown_lints,explicit_counter_loop)]
fn chunk(&mut self, r: &mut Read, mut w: &mut Write) -> Result<ChunkerStatus, ChunkerError> {
let mut todo = self.size;
loop {
// Fill the buffer, there might be some bytes still in there from last chunk
let max_read = min(todo, self.buffer.len());
let read = try!(r.read(&mut self.buffer[..max_read]).map_err(ChunkerError::Read));
// If nothing to do, finish
if read == 0 {
return Ok(ChunkerStatus::Finished)
}
// Write all bytes from this chunk out to sink and store rest for next chunk
try!(w.write_all(&self.buffer[..read]).map_err(ChunkerError::Write));
todo -= read;
if todo == 0 {
return Ok(ChunkerStatus::Continue)
}
}
}
}

View File

@ -2,10 +2,12 @@
use std::io::{self, Write, Read}; use std::io::{self, Write, Read};
mod fixed;
mod ae; mod ae;
mod rabin; mod rabin;
mod fastcdc; mod fastcdc;
pub use self::fixed::FixedChunker;
pub use self::ae::AeChunker; pub use self::ae::AeChunker;
pub use self::rabin::RabinChunker; pub use self::rabin::RabinChunker;
pub use self::fastcdc::FastCdcChunker; pub use self::fastcdc::FastCdcChunker;

View File

@ -7,12 +7,14 @@ use std::str::FromStr;
pub enum ChunkerType { pub enum ChunkerType {
Ae(usize), Ae(usize),
Rabin((usize, u32)), Rabin((usize, u32)),
FastCdc((usize, u64)) FastCdc((usize, u64)),
Fixed(usize)
} }
serde_impl!(ChunkerType(u64) { serde_impl!(ChunkerType(u64) {
Ae(usize) => 1, Ae(usize) => 1,
Rabin((usize, u32)) => 2, Rabin((usize, u32)) => 2,
FastCdc((usize, u64)) => 3 FastCdc((usize, u64)) => 3,
Fixed(usize) => 4
}); });
@ -22,6 +24,7 @@ impl ChunkerType {
"ae" => Ok(ChunkerType::Ae(avg_size)), "ae" => Ok(ChunkerType::Ae(avg_size)),
"rabin" => Ok(ChunkerType::Rabin((avg_size, seed as u32))), "rabin" => Ok(ChunkerType::Rabin((avg_size, seed as u32))),
"fastcdc" => Ok(ChunkerType::FastCdc((avg_size, seed))), "fastcdc" => Ok(ChunkerType::FastCdc((avg_size, seed))),
"fixed" => Ok(ChunkerType::Fixed(avg_size)),
_ => Err("Unsupported chunker type") _ => Err("Unsupported chunker type")
} }
} }
@ -43,7 +46,8 @@ impl ChunkerType {
match *self { match *self {
ChunkerType::Ae(size) => Box::new(AeChunker::new(size)), ChunkerType::Ae(size) => Box::new(AeChunker::new(size)),
ChunkerType::Rabin((size, seed)) => Box::new(RabinChunker::new(size, seed)), ChunkerType::Rabin((size, seed)) => Box::new(RabinChunker::new(size, seed)),
ChunkerType::FastCdc((size, seed)) => Box::new(FastCdcChunker::new(size, seed)) ChunkerType::FastCdc((size, seed)) => Box::new(FastCdcChunker::new(size, seed)),
ChunkerType::Fixed(size) => Box::new(FixedChunker::new(size)),
} }
} }
@ -51,7 +55,8 @@ impl ChunkerType {
match *self { match *self {
ChunkerType::Ae(_size) => "ae", ChunkerType::Ae(_size) => "ae",
ChunkerType::Rabin((_size, _seed)) => "rabin", ChunkerType::Rabin((_size, _seed)) => "rabin",
ChunkerType::FastCdc((_size, _seed)) => "fastcdc" ChunkerType::FastCdc((_size, _seed)) => "fastcdc",
ChunkerType::Fixed(_size) => "fixed",
} }
} }
@ -59,7 +64,8 @@ impl ChunkerType {
match *self { match *self {
ChunkerType::Ae(size) => size, ChunkerType::Ae(size) => size,
ChunkerType::Rabin((size, _seed)) => size, ChunkerType::Rabin((size, _seed)) => size,
ChunkerType::FastCdc((size, _seed)) => size ChunkerType::FastCdc((size, _seed)) => size,
ChunkerType::Fixed(size) => size
} }
} }
@ -71,7 +77,8 @@ impl ChunkerType {
match *self { match *self {
ChunkerType::Ae(_size) => 0, ChunkerType::Ae(_size) => 0,
ChunkerType::Rabin((_size, seed)) => seed as u64, ChunkerType::Rabin((_size, seed)) => seed as u64,
ChunkerType::FastCdc((_size, seed)) => seed ChunkerType::FastCdc((_size, seed)) => seed,
ChunkerType::Fixed(_size) => 0,
} }
} }
} }