Fixes and progress bars

pull/10/head
Dennis Schwerdel 2017-04-11 08:49:45 +02:00
parent fcbc2e131f
commit 221e3dff38
10 changed files with 101 additions and 17 deletions

13
Cargo.lock generated
View File

@ -15,6 +15,7 @@ dependencies = [
"log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"pbr 1.0.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
@ -228,6 +229,17 @@ name = "num-traits"
version = "0.1.37" version = "0.1.37"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "pbr"
version = "1.0.0-alpha.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "pkg-config" name = "pkg-config"
version = "0.3.9" version = "0.3.9"
@ -489,6 +501,7 @@ dependencies = [
"checksum num-integer 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "21e4df1098d1d797d27ef0c69c178c3fab64941559b290fcae198e0825c9c8b5" "checksum num-integer 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "21e4df1098d1d797d27ef0c69c178c3fab64941559b290fcae198e0825c9c8b5"
"checksum num-iter 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "f7d1891bd7b936f12349b7d1403761c8a0b85a18b148e9da4429d5d102c1a41e" "checksum num-iter 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "f7d1891bd7b936f12349b7d1403761c8a0b85a18b148e9da4429d5d102c1a41e"
"checksum num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "e1cbfa3781f3fe73dc05321bed52a06d2d491eaa764c52335cf4399f046ece99" "checksum num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "e1cbfa3781f3fe73dc05321bed52a06d2d491eaa764c52335cf4399f046ece99"
"checksum pbr 1.0.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2c0434823e05f3f0031a1d9f4323c954543ed66f2cc2bb4700ff539fdd06ce0e"
"checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903" "checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903"
"checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c" "checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c"
"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"

View File

@ -29,6 +29,7 @@ rand = "0.3"
tar = "0.4" tar = "0.4"
xattr = "0.1" xattr = "0.1"
crossbeam = "0.2" crossbeam = "0.2"
pbr = "1.0.0-alpha.3"
time = "*" time = "*"
libc = "*" libc = "*"

View File

@ -8,7 +8,6 @@ use std::sync::{Arc, Mutex};
use std::io; use std::io;
use std::mem; use std::mem;
quick_error!{ quick_error!{
#[derive(Debug)] #[derive(Debug)]
pub enum BundleDbError { pub enum BundleDbError {
@ -78,10 +77,15 @@ pub fn load_bundles(path: &Path, base: &Path, bundles: &mut HashMap<BundleId, St
} }
let mut new = vec![]; let mut new = vec![];
for path in bundle_paths { for path in bundle_paths {
let bundle = StoredBundle { let info = match BundleReader::load_info(base.join(&path), crypto.clone()) {
info: try!(BundleReader::load_info(base.join(&path), crypto.clone())), Ok(info) => info,
path: path Err(BundleReaderError::TruncatedBundle(path)) => {
warn!("Ignoring truncated bundle {:?}", path);
continue
},
Err(err) => return Err(err.into())
}; };
let bundle = StoredBundle { info: info, path: path };
let id = bundle.info.id.clone(); let id = bundle.info.id.clone();
if !bundles.contains_key(&id) { if !bundles.contains_key(&id) {
new.push(bundle.clone()); new.push(bundle.clone());
@ -304,7 +308,7 @@ impl BundleDb {
} }
pub fn check(&mut self, full: bool) -> Result<(), BundleDbError> { pub fn check(&mut self, full: bool) -> Result<(), BundleDbError> {
for stored in self.remote_bundles.values() { for stored in ProgressIter::new("checking bundles", self.remote_bundles.len(), self.remote_bundles.values()) {
let mut bundle = try!(self.get_bundle(stored)); let mut bundle = try!(self.get_bundle(stored));
try!(bundle.check(full)) try!(bundle.check(full))
} }

View File

@ -12,6 +12,10 @@ use std::sync::{Arc, Mutex};
quick_error!{ quick_error!{
#[derive(Debug)] #[derive(Debug)]
pub enum BundleReaderError { pub enum BundleReaderError {
TruncatedBundle(path: PathBuf) {
description("Bundle file is truncated")
display("Bundle reader error: bundle file is truncated {:?}", path)
}
Read(err: io::Error, path: PathBuf) { Read(err: io::Error, path: PathBuf) {
cause(err) cause(err)
context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf()) context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf())
@ -107,6 +111,10 @@ impl BundleReader {
info.encryption = header.encryption; info.encryption = header.encryption;
debug!("Load bundle {}", info.id); debug!("Load bundle {}", info.id);
let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize + info.chunk_list_size; let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize + info.chunk_list_size;
let actual_size = try!(fs::metadata(path).context(path)).len();
if content_start + info.encoded_size != actual_size as usize {
return Err(BundleReaderError::TruncatedBundle(path.to_path_buf()));
}
Ok((info, version, content_start)) Ok((info, version, content_start))
} }

View File

@ -340,7 +340,7 @@ pub fn parse() -> Result<(LogLevel, Arguments), ErrorCode> {
.validator(|val| validate_repo_path(val, true, Some(false), Some(false))))) .validator(|val| validate_repo_path(val, true, Some(false), Some(false)))))
.subcommand(SubCommand::with_name("check").about("Check the repository, a backup or a backup subtree") .subcommand(SubCommand::with_name("check").about("Check the repository, a backup or a backup subtree")
.arg(Arg::from_usage("-b --bundles 'Check the bundles'")) .arg(Arg::from_usage("-b --bundles 'Check the bundles'"))
.arg(Arg::from_usage("[bundle_data] --bundle-data 'Check bundle contents (slow)'").requires("bundles").alias("--data")) .arg(Arg::from_usage("[bundle_data] --bundle-data 'Check bundle contents (slow)'").requires("bundles").alias("data"))
.arg(Arg::from_usage("-i --index 'Check the chunk index'")) .arg(Arg::from_usage("-i --index 'Check the chunk index'"))
.arg(Arg::from_usage("<PATH> 'Path of the repository/backup/subtree, [repository][::backup[::subtree]]'") .arg(Arg::from_usage("<PATH> 'Path of the repository/backup/subtree, [repository][::backup[::subtree]]'")
.validator(|val| validate_repo_path(val, true, None, None)))) .validator(|val| validate_repo_path(val, true, None, None))))

View File

@ -24,6 +24,7 @@ extern crate rand;
extern crate time; extern crate time;
extern crate xattr; extern crate xattr;
extern crate crossbeam; extern crate crossbeam;
extern crate pbr;
extern crate libc; extern crate libc;
extern crate tar; extern crate tar;

View File

@ -2,6 +2,9 @@ use ::prelude::*;
use std::collections::VecDeque; use std::collections::VecDeque;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::time::Duration;
use pbr::ProgressBar;
quick_error!{ quick_error!{
@ -47,7 +50,11 @@ quick_error!{
impl Repository { impl Repository {
fn check_index_chunks(&self) -> Result<(), RepositoryError> { fn check_index_chunks(&self) -> Result<(), RepositoryError> {
self.index.walk(|_hash, location| { let mut count = 0;
let mut progress = ProgressBar::new(self.index.len() as u64);
progress.message("checking index: ");
progress.set_max_refresh_rate(Some(Duration::from_millis(100)));
let res = self.index.walk(|_hash, location| {
// Lookup bundle id from map // Lookup bundle id from map
let bundle_id = try!(self.get_bundle_id(location.bundle)); let bundle_id = try!(self.get_bundle_id(location.bundle));
// Get bundle object from bundledb // Get bundle object from bundledb
@ -60,8 +67,14 @@ impl Repository {
if bundle.info.chunk_count <= location.chunk as usize { if bundle.info.chunk_count <= location.chunk as usize {
return Err(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into()) return Err(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into())
} }
count += 1;
if count % 1000 == 0 {
progress.set(count);
}
Ok(()) Ok(())
}) });
progress.finish_print("checking index: done.");
res
} }
fn check_chunks(&self, checked: &mut Bitmap, chunks: &[Chunk]) -> Result<bool, RepositoryError> { fn check_chunks(&self, checked: &mut Bitmap, chunks: &[Chunk]) -> Result<bool, RepositoryError> {
@ -148,7 +161,7 @@ impl Repository {
}, },
Err(err) => return Err(err) Err(err) => return Err(err)
}; };
for (name, backup) in backup_map { for (name, backup) in ProgressIter::new("ckecking backups", backup_map.len(), backup_map.into_iter()) {
let path = name+"::"; let path = name+"::";
try!(self.check_subtree(Path::new(&path).to_path_buf(), &backup.root, &mut checked)); try!(self.check_subtree(Path::new(&path).to_path_buf(), &backup.root, &mut checked));
} }

View File

@ -111,15 +111,15 @@ impl Repository {
}; };
if !new.is_empty() { if !new.is_empty() {
info!("Adding {} new bundles to index", new.len()); info!("Adding {} new bundles to index", new.len());
} for bundle in ProgressIter::new("adding bundles to index", new.len(), new.into_iter()) {
for bundle in new { try!(repo.add_new_remote_bundle(bundle))
try!(repo.add_new_remote_bundle(bundle)) }
} }
if !gone.is_empty() { if !gone.is_empty() {
info!("Removig {} old bundles from index", gone.len()); info!("Removig {} old bundles from index", gone.len());
} for bundle in gone {
for bundle in gone { try!(repo.remove_gone_remote_bundle(bundle))
try!(repo.remove_gone_remote_bundle(bundle)) }
} }
try!(repo.save_bundle_map()); try!(repo.save_bundle_map());
repo.next_meta_bundle = repo.next_free_bundle_id(); repo.next_meta_bundle = repo.next_free_bundle_id();
@ -227,10 +227,10 @@ impl Repository {
} }
fn add_new_remote_bundle(&mut self, bundle: BundleInfo) -> Result<(), RepositoryError> { fn add_new_remote_bundle(&mut self, bundle: BundleInfo) -> Result<(), RepositoryError> {
debug!("Adding new bundle to index: {}", bundle.id);
if self.bundle_map.find(&bundle.id).is_some() { if self.bundle_map.find(&bundle.id).is_some() {
return Ok(()) return Ok(())
} }
debug!("Adding new bundle to index: {}", bundle.id);
let bundle_id = match bundle.mode { let bundle_id = match bundle.mode {
BundleMode::Data => self.next_data_bundle, BundleMode::Data => self.next_data_bundle,
BundleMode::Meta => self.next_meta_bundle BundleMode::Meta => self.next_meta_bundle

View File

@ -43,7 +43,7 @@ impl Repository {
self.dirty = false; self.dirty = false;
return Ok(()) return Ok(())
} }
for id in &rewrite_bundles { for id in ProgressIter::new("rewriting bundles", rewrite_bundles.len(), rewrite_bundles.iter()) {
let bundle = &usage[id]; let bundle = &usage[id];
let bundle_id = self.bundle_map.get(*id).unwrap(); let bundle_id = self.bundle_map.get(*id).unwrap();
let chunks = try!(self.bundles.get_chunk_list(&bundle_id)); let chunks = try!(self.bundles.get_chunk_list(&bundle_id));

View File

@ -1,3 +1,7 @@
use pbr;
use std::io::Stdout;
use std::time::Duration;
pub fn to_file_size(size: u64) -> String { pub fn to_file_size(size: u64) -> String {
let mut size = size as f32; let mut size = size as f32;
if size >= 512.0 { if size >= 512.0 {
@ -36,3 +40,43 @@ pub fn to_duration(dur: f32) -> String {
let secs = (secs % 60) as f32 + subsecs; let secs = (secs % 60) as f32 + subsecs;
format!("{}:{:02}:{:04.1}", hours, mins, secs) format!("{}:{:02}:{:04.1}", hours, mins, secs)
} }
pub struct ProgressIter<T> {
inner: T,
msg: String,
bar: pbr::ProgressBar<Stdout>
}
impl<T> ProgressIter<T> {
#[allow(blacklisted_name)]
pub fn new(msg: &str, max: usize, inner: T) -> Self {
let mut bar = pbr::ProgressBar::new(max as u64);
let msg = format!("{}: ", msg);
bar.message(&msg);
bar.set_max_refresh_rate(Some(Duration::from_millis(100)));
ProgressIter { inner: inner, bar: bar, msg: msg }
}
}
impl<T: Iterator> Iterator for ProgressIter<T> {
type Item = T::Item;
fn size_hint(&self) -> (usize, Option<usize>) {
self.inner.size_hint()
}
fn next(&mut self) -> Option<Self::Item> {
match self.inner.next() {
None => {
let msg = self.msg.clone() + "done.";
self.bar.finish_print(&msg);
None
},
Some(item) => {
self.bar.inc();
Some(item)
}
}
}
}