From fb73e29a20c37c7f8fc6a150ae7f6af47294dfc2 Mon Sep 17 00:00:00 2001 From: Dennis Schwerdel Date: Mon, 19 Feb 2018 21:18:47 +0100 Subject: [PATCH] Some minor changes --- .gitignore | 1 + Cargo.lock | 74 ++++++++++++++++++------------------- chunking/src/fastcdc.rs | 3 ++ src/cli/algotest.rs | 2 +- src/repository/integrity.rs | 2 +- src/repository/vacuum.rs | 7 +++- 6 files changed, 48 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index 8c0b983..0150c7d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ excludes ._* .~* docs/logo +.idea \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 191fc77..b4814c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,40 +1,3 @@ -[root] -name = "zvault" -version = "0.4.0" -dependencies = [ - "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "chunking 0.1.0", - "clap 2.26.0 (registry+https://github.com/rust-lang/crates.io-index)", - "crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "fuse 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "index 0.1.0", - "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.28 (registry+https://github.com/rust-lang/crates.io-index)", - "libsodium-sys 0.0.15 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", - "pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "quick-error 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rmp-serde 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_bytes 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_utils 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_yaml 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "sodiumoxide 0.0.15 (registry+https://github.com/rust-lang/crates.io-index)", - "squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tar 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", - "users 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "xattr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "aho-corasick" version = "0.6.3" @@ -549,6 +512,43 @@ dependencies = [ "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "zvault" +version = "0.4.0" +dependencies = [ + "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "chunking 0.1.0", + "clap 2.26.0 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "fuse 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "index 0.1.0", + "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.28 (registry+https://github.com/rust-lang/crates.io-index)", + "libsodium-sys 0.0.15 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "quick-error 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp-serde 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_bytes 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_utils 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_yaml 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sodiumoxide 0.0.15 (registry+https://github.com/rust-lang/crates.io-index)", + "squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", + "users 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "xattr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [metadata] "checksum aho-corasick 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "500909c4f87a9e52355b26626d890833e9e1d53ac566db76c36faa984b889699" "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" diff --git a/chunking/src/fastcdc.rs b/chunking/src/fastcdc.rs index 5788889..a02e5ab 100644 --- a/chunking/src/fastcdc.rs +++ b/chunking/src/fastcdc.rs @@ -9,6 +9,7 @@ use std::cmp; // Presentation: https://www.usenix.org/sites/default/files/conference/protected-files/atc16_slides_xia.pdf + // Creating 256 pseudo-random values (based on Knuth's MMIX) fn create_gear(seed: u64) -> [u64; 256] { let mut table = [0u64; 256]; @@ -75,6 +76,8 @@ impl FastCdcChunker { impl FastCdcChunker { fn write_output(&mut self, w: &mut Write, pos: usize, max: usize) -> Result { + debug_assert!(max <= self.buffer.len()); + debug_assert!(pos <= self.buffer.len()); try!(w.write_all(&self.buffer[..pos]).map_err(ChunkerError::Write)); unsafe { ptr::copy(self.buffer[pos..].as_ptr(), self.buffer.as_mut_ptr(), max-pos) }; self.buffered = max-pos; diff --git a/src/cli/algotest.rs b/src/cli/algotest.rs index 0ede62a..dfeb980 100644 --- a/src/cli/algotest.rs +++ b/src/cli/algotest.rs @@ -129,7 +129,7 @@ pub fn run( dup_size += len; } println!( - "- {} duplicate chunks, {}, {:.1}% saved", + "- {} duplicate chunks, {}, {:.1}% saved by internal deduplication", dups.len(), to_file_size(dup_size as u64), dup_size as f32 / size as f32 * 100.0 diff --git a/src/repository/integrity.rs b/src/repository/integrity.rs index b0ddb83..7338af3 100644 --- a/src/repository/integrity.rs +++ b/src/repository/integrity.rs @@ -447,7 +447,7 @@ impl Repository { self.index.clear(); let mut bundles = self.bundle_map.bundles(); bundles.sort_by_key(|&(_, ref v)| v.clone()); - for (num, id) in bundles { + for (num, id) in ProgressIter::new("Rebuilding index from bundles", bundles.len(), bundles.into_iter()) { let chunks = try!(self.bundles.get_chunk_list(&id)); for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() { try!(self.index.set( diff --git a/src/repository/vacuum.rs b/src/repository/vacuum.rs index 15250d4..d008846 100644 --- a/src/repository/vacuum.rs +++ b/src/repository/vacuum.rs @@ -40,10 +40,12 @@ impl Repository { ); let mut rewrite_bundles = HashSet::new(); let mut reclaim_space = 0; + let mut rewrite_data = 0; for (id, bundle) in &usage { if bundle.get_usage_ratio() <= ratio { rewrite_bundles.insert(*id); reclaim_space += bundle.get_unused_size(); + rewrite_data += bundle.get_used_size(); } } if combine { @@ -69,9 +71,10 @@ impl Repository { } } info!( - "Reclaiming {} by rewriting {} bundles", + "Reclaiming about {} by rewriting {} bundles ({})", to_file_size(reclaim_space as u64), - rewrite_bundles.len() + rewrite_bundles.len(), + to_file_size(rewrite_data as u64) ); if !force { self.dirty = false;