dswd
/
zvault
mirror of https://github.com/dswd/zvault
1
0
Fork 0
Code Issues Releases Wiki Activity

Fixes and progress bars

pull/10/head
Dennis Schwerdel 2017-04-11 08:49:45 +02:00
parent fcbc2e131f
commit 221e3dff38
10 changed files with 101 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Cargo.lock generated
View File

@ -15,6 +15,7 @@ dependencies = [
"log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"pbr 1.0.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
@ -228,6 +229,17 @@ name = "num-traits"
version = "0.1.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "pbr"
version = "1.0.0-alpha.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "pkg-config"
version = "0.3.9"
@ -489,6 +501,7 @@ dependencies = [
"checksum num-integer 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "21e4df1098d1d797d27ef0c69c178c3fab64941559b290fcae198e0825c9c8b5"
"checksum num-iter 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "f7d1891bd7b936f12349b7d1403761c8a0b85a18b148e9da4429d5d102c1a41e"
"checksum num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "e1cbfa3781f3fe73dc05321bed52a06d2d491eaa764c52335cf4399f046ece99"
"checksum pbr 1.0.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2c0434823e05f3f0031a1d9f4323c954543ed66f2cc2bb4700ff539fdd06ce0e"
"checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903"
"checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c"
"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"

1
Cargo.toml
View File

@ -29,6 +29,7 @@ rand = "0.3"
tar = "0.4"
xattr = "0.1"
crossbeam = "0.2"
pbr = "1.0.0-alpha.3"
time = "*"
libc = "*"

14
src/bundledb/db.rs
View File

@ -8,7 +8,6 @@ use std::sync::{Arc, Mutex};
use std::io;
use std::mem;
quick_error!{
#[derive(Debug)]
pub enum BundleDbError {
@ -78,10 +77,15 @@ pub fn load_bundles(path: &Path, base: &Path, bundles: &mut HashMap<BundleId, St
}
let mut new = vec![];
for path in bundle_paths {
let bundle = StoredBundle {
info: try!(BundleReader::load_info(base.join(&path), crypto.clone())),
path: path
let info = match BundleReader::load_info(base.join(&path), crypto.clone()) {
Ok(info) => info,
Err(BundleReaderError::TruncatedBundle(path)) => {
warn!("Ignoring truncated bundle {:?}", path);
continue
},
Err(err) => return Err(err.into())
};
let bundle = StoredBundle { info: info, path: path };
let id = bundle.info.id.clone();
if !bundles.contains_key(&id) {
new.push(bundle.clone());
@ -304,7 +308,7 @@ impl BundleDb {
}
pub fn check(&mut self, full: bool) -> Result<(), BundleDbError> {
for stored in self.remote_bundles.values() {
for stored in ProgressIter::new("checking bundles", self.remote_bundles.len(), self.remote_bundles.values()) {
let mut bundle = try!(self.get_bundle(stored));
try!(bundle.check(full))
}

8
src/bundledb/reader.rs
View File

@ -12,6 +12,10 @@ use std::sync::{Arc, Mutex};
quick_error!{
#[derive(Debug)]
pub enum BundleReaderError {
TruncatedBundle(path: PathBuf) {
description("Bundle file is truncated")
display("Bundle reader error: bundle file is truncated {:?}", path)
}
Read(err: io::Error, path: PathBuf) {
cause(err)
context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf())
@ -107,6 +111,10 @@ impl BundleReader {
info.encryption = header.encryption;
debug!("Load bundle {}", info.id);
let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize + info.chunk_list_size;
let actual_size = try!(fs::metadata(path).context(path)).len();
if content_start + info.encoded_size != actual_size as usize {
return Err(BundleReaderError::TruncatedBundle(path.to_path_buf()));
}
Ok((info, version, content_start))
}

2
src/cli/args.rs
View File

@ -340,7 +340,7 @@ pub fn parse() -> Result<(LogLevel, Arguments), ErrorCode> {
.validator(|val| validate_repo_path(val, true, Some(false), Some(false)))))
.subcommand(SubCommand::with_name("check").about("Check the repository, a backup or a backup subtree")
.arg(Arg::from_usage("-b --bundles 'Check the bundles'"))
.arg(Arg::from_usage("[bundle_data] --bundle-data 'Check bundle contents (slow)'").requires("bundles").alias("--data"))
.arg(Arg::from_usage("[bundle_data] --bundle-data 'Check bundle contents (slow)'").requires("bundles").alias("data"))
.arg(Arg::from_usage("-i --index 'Check the chunk index'"))
.arg(Arg::from_usage("<PATH> 'Path of the repository/backup/subtree, [repository][::backup[::subtree]]'")
.validator(|val| validate_repo_path(val, true, None, None))))

1
src/main.rs
View File

@ -24,6 +24,7 @@ extern crate rand;
extern crate time;
extern crate xattr;
extern crate crossbeam;
extern crate pbr;
extern crate libc;
extern crate tar;

19
src/repository/integrity.rs
View File

@ -2,6 +2,9 @@ use ::prelude::*;
use std::collections::VecDeque;
use std::path::{Path, PathBuf};
use std::time::Duration;
use pbr::ProgressBar;
quick_error!{
@ -47,7 +50,11 @@ quick_error!{
impl Repository {
fn check_index_chunks(&self) -> Result<(), RepositoryError> {
self.index.walk(|_hash, location| {
let mut count = 0;
let mut progress = ProgressBar::new(self.index.len() as u64);
progress.message("checking index: ");
progress.set_max_refresh_rate(Some(Duration::from_millis(100)));
let res = self.index.walk(|_hash, location| {
// Lookup bundle id from map
let bundle_id = try!(self.get_bundle_id(location.bundle));
// Get bundle object from bundledb
@ -60,8 +67,14 @@ impl Repository {
if bundle.info.chunk_count <= location.chunk as usize {
return Err(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into())
}
count += 1;
if count % 1000 == 0 {
progress.set(count);
}
Ok(())
})
});
progress.finish_print("checking index: done.");
res
}
fn check_chunks(&self, checked: &mut Bitmap, chunks: &[Chunk]) -> Result<bool, RepositoryError> {
@ -148,7 +161,7 @@ impl Repository {
},
Err(err) => return Err(err)
};
for (name, backup) in backup_map {
for (name, backup) in ProgressIter::new("ckecking backups", backup_map.len(), backup_map.into_iter()) {
let path = name+"::";
try!(self.check_subtree(Path::new(&path).to_path_buf(), &backup.root, &mut checked));
}

14
src/repository/mod.rs
View File

@ -111,15 +111,15 @@ impl Repository {
};
if !new.is_empty() {
info!("Adding {} new bundles to index", new.len());
}
for bundle in new {
try!(repo.add_new_remote_bundle(bundle))
for bundle in ProgressIter::new("adding bundles to index", new.len(), new.into_iter()) {
try!(repo.add_new_remote_bundle(bundle))
}
}
if !gone.is_empty() {
info!("Removig {} old bundles from index", gone.len());
}
for bundle in gone {
try!(repo.remove_gone_remote_bundle(bundle))
for bundle in gone {
try!(repo.remove_gone_remote_bundle(bundle))
}
}
try!(repo.save_bundle_map());
repo.next_meta_bundle = repo.next_free_bundle_id();
@ -227,10 +227,10 @@ impl Repository {
}
fn add_new_remote_bundle(&mut self, bundle: BundleInfo) -> Result<(), RepositoryError> {
debug!("Adding new bundle to index: {}", bundle.id);
if self.bundle_map.find(&bundle.id).is_some() {
return Ok(())
}
debug!("Adding new bundle to index: {}", bundle.id);
let bundle_id = match bundle.mode {
BundleMode::Data => self.next_data_bundle,
BundleMode::Meta => self.next_meta_bundle

2
src/repository/vacuum.rs
View File

@ -43,7 +43,7 @@ impl Repository {
self.dirty = false;
return Ok(())
}
for id in &rewrite_bundles {
for id in ProgressIter::new("rewriting bundles", rewrite_bundles.len(), rewrite_bundles.iter()) {
let bundle = &usage[id];
let bundle_id = self.bundle_map.get(*id).unwrap();
let chunks = try!(self.bundles.get_chunk_list(&bundle_id));

44
src/util/cli.rs
View File

@ -1,3 +1,7 @@
use pbr;
use std::io::Stdout;
use std::time::Duration;
pub fn to_file_size(size: u64) -> String {
let mut size = size as f32;
if size >= 512.0 {
@ -36,3 +40,43 @@ pub fn to_duration(dur: f32) -> String {
let secs = (secs % 60) as f32 + subsecs;
format!("{}:{:02}:{:04.1}", hours, mins, secs)
}
pub struct ProgressIter<T> {
inner: T,
msg: String,
bar: pbr::ProgressBar<Stdout>
}
impl<T> ProgressIter<T> {
#[allow(blacklisted_name)]
pub fn new(msg: &str, max: usize, inner: T) -> Self {
let mut bar = pbr::ProgressBar::new(max as u64);
let msg = format!("{}: ", msg);
bar.message(&msg);
bar.set_max_refresh_rate(Some(Duration::from_millis(100)));
ProgressIter { inner: inner, bar: bar, msg: msg }
}
}
impl<T: Iterator> Iterator for ProgressIter<T> {
type Item = T::Item;
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
fn next(&mut self) -> Option<Self::Item> {
match self.inner.next() {
None => {
let msg = self.msg.clone() + "done.";
self.bar.finish_print(&msg);
None
},
Some(item) => {
self.bar.inc();
Some(item)
}
}
}
}