dswd
/
zvault
mirror of https://github.com/dswd/zvault
1
0
Fork 0
Code Issues Releases Wiki Activity

Some minor changes

This commit is contained in:
Dennis Schwerdel 2018-02-19 21:18:47 +01:00
parent b2331c61fd
commit fb73e29a20
6 changed files with 48 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -7,3 +7,4 @@ excludes
._* ._*
.~* .~*
docs/logo docs/logo
.idea

74
Cargo.lock generated
View File

@ -1,40 +1,3 @@
[root]
name = "zvault"
version = "0.4.0"
dependencies = [
"ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"chunking 0.1.0",
"clap 2.26.0 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"fuse 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"index 0.1.0",
"lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.28 (registry+https://github.com/rust-lang/crates.io-index)",
"libsodium-sys 0.0.15 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rmp-serde 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_bytes 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_utils 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_yaml 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"sodiumoxide 0.0.15 (registry+https://github.com/rust-lang/crates.io-index)",
"squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tar 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"users 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"xattr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
version = "0.6.3" version = "0.6.3"
@ -549,6 +512,43 @@ dependencies = [
"linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "zvault"
version = "0.4.0"
dependencies = [
"ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"chunking 0.1.0",
"clap 2.26.0 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"fuse 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"index 0.1.0",
"lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.28 (registry+https://github.com/rust-lang/crates.io-index)",
"libsodium-sys 0.0.15 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.16 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rmp-serde 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_bytes 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_utils 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_yaml 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
"sodiumoxide 0.0.15 (registry+https://github.com/rust-lang/crates.io-index)",
"squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tar 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"users 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"xattr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[metadata] [metadata]
"checksum aho-corasick 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "500909c4f87a9e52355b26626d890833e9e1d53ac566db76c36faa984b889699" "checksum aho-corasick 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "500909c4f87a9e52355b26626d890833e9e1d53ac566db76c36faa984b889699"
"checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"

3
chunking/src/fastcdc.rs
View File

@ -9,6 +9,7 @@ use std::cmp;
// Presentation: https://www.usenix.org/sites/default/files/conference/protected-files/atc16_slides_xia.pdf // Presentation: https://www.usenix.org/sites/default/files/conference/protected-files/atc16_slides_xia.pdf
// Creating 256 pseudo-random values (based on Knuth's MMIX) // Creating 256 pseudo-random values (based on Knuth's MMIX)
fn create_gear(seed: u64) -> [u64; 256] { fn create_gear(seed: u64) -> [u64; 256] {
let mut table = [0u64; 256]; let mut table = [0u64; 256];
@ -75,6 +76,8 @@ impl FastCdcChunker {
impl FastCdcChunker { impl FastCdcChunker {
fn write_output(&mut self, w: &mut Write, pos: usize, max: usize) -> Result<ChunkerStatus, ChunkerError> { fn write_output(&mut self, w: &mut Write, pos: usize, max: usize) -> Result<ChunkerStatus, ChunkerError> {
debug_assert!(max <= self.buffer.len());
debug_assert!(pos <= self.buffer.len());
try!(w.write_all(&self.buffer[..pos]).map_err(ChunkerError::Write)); try!(w.write_all(&self.buffer[..pos]).map_err(ChunkerError::Write));
unsafe { ptr::copy(self.buffer[pos..].as_ptr(), self.buffer.as_mut_ptr(), max-pos) }; unsafe { ptr::copy(self.buffer[pos..].as_ptr(), self.buffer.as_mut_ptr(), max-pos) };
self.buffered = max-pos; self.buffered = max-pos;

2
src/cli/algotest.rs
View File

@ -129,7 +129,7 @@ pub fn run(
dup_size += len; dup_size += len;
} }
println!( println!(
"- {} duplicate chunks, {}, {:.1}% saved", "- {} duplicate chunks, {}, {:.1}% saved by internal deduplication",
dups.len(), dups.len(),
to_file_size(dup_size as u64), to_file_size(dup_size as u64),
dup_size as f32 / size as f32 * 100.0 dup_size as f32 / size as f32 * 100.0

2
src/repository/integrity.rs
View File

@ -447,7 +447,7 @@ impl Repository {
self.index.clear(); self.index.clear();
let mut bundles = self.bundle_map.bundles(); let mut bundles = self.bundle_map.bundles();
bundles.sort_by_key(|&(_, ref v)| v.clone()); bundles.sort_by_key(|&(_, ref v)| v.clone());
for (num, id) in bundles { for (num, id) in ProgressIter::new("Rebuilding index from bundles", bundles.len(), bundles.into_iter()) {
let chunks = try!(self.bundles.get_chunk_list(&id)); let chunks = try!(self.bundles.get_chunk_list(&id));
for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() { for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() {
try!(self.index.set( try!(self.index.set(

7
src/repository/vacuum.rs
View File

@ -40,10 +40,12 @@ impl Repository {
); );
let mut rewrite_bundles = HashSet::new(); let mut rewrite_bundles = HashSet::new();
let mut reclaim_space = 0; let mut reclaim_space = 0;
let mut rewrite_data = 0;
for (id, bundle) in &usage { for (id, bundle) in &usage {
if bundle.get_usage_ratio() <= ratio { if bundle.get_usage_ratio() <= ratio {
rewrite_bundles.insert(*id); rewrite_bundles.insert(*id);
reclaim_space += bundle.get_unused_size(); reclaim_space += bundle.get_unused_size();
rewrite_data += bundle.get_used_size();
} }
} }
if combine { if combine {
@ -69,9 +71,10 @@ impl Repository {
} }
} }
info!( info!(
"Reclaiming {} by rewriting {} bundles", "Reclaiming about {} by rewriting {} bundles ({})",
to_file_size(reclaim_space as u64), to_file_size(reclaim_space as u64),
rewrite_bundles.len() rewrite_bundles.len(),
to_file_size(rewrite_data as u64)
); );
if !force { if !force {
self.dirty = false; self.dirty = false;