diff --git a/Cargo.lock b/Cargo.lock index 966d251..0f4c8f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,6 +15,7 @@ dependencies = [ "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "pbr 1.0.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", @@ -228,6 +229,17 @@ name = "num-traits" version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "pbr" +version = "1.0.0-alpha.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "pkg-config" version = "0.3.9" @@ -489,6 +501,7 @@ dependencies = [ "checksum num-integer 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "21e4df1098d1d797d27ef0c69c178c3fab64941559b290fcae198e0825c9c8b5" "checksum num-iter 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "f7d1891bd7b936f12349b7d1403761c8a0b85a18b148e9da4429d5d102c1a41e" "checksum num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "e1cbfa3781f3fe73dc05321bed52a06d2d491eaa764c52335cf4399f046ece99" +"checksum pbr 1.0.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2c0434823e05f3f0031a1d9f4323c954543ed66f2cc2bb4700ff539fdd06ce0e" "checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903" "checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c" "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" diff --git a/Cargo.toml b/Cargo.toml index d74c456..c13728b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ rand = "0.3" tar = "0.4" xattr = "0.1" crossbeam = "0.2" +pbr = "1.0.0-alpha.3" time = "*" libc = "*" diff --git a/src/bundledb/db.rs b/src/bundledb/db.rs index 434c1d8..55a3bc9 100644 --- a/src/bundledb/db.rs +++ b/src/bundledb/db.rs @@ -8,7 +8,6 @@ use std::sync::{Arc, Mutex}; use std::io; use std::mem; - quick_error!{ #[derive(Debug)] pub enum BundleDbError { @@ -78,10 +77,15 @@ pub fn load_bundles(path: &Path, base: &Path, bundles: &mut HashMap info, + Err(BundleReaderError::TruncatedBundle(path)) => { + warn!("Ignoring truncated bundle {:?}", path); + continue + }, + Err(err) => return Err(err.into()) }; + let bundle = StoredBundle { info: info, path: path }; let id = bundle.info.id.clone(); if !bundles.contains_key(&id) { new.push(bundle.clone()); @@ -304,7 +308,7 @@ impl BundleDb { } pub fn check(&mut self, full: bool) -> Result<(), BundleDbError> { - for stored in self.remote_bundles.values() { + for stored in ProgressIter::new("checking bundles", self.remote_bundles.len(), self.remote_bundles.values()) { let mut bundle = try!(self.get_bundle(stored)); try!(bundle.check(full)) } diff --git a/src/bundledb/reader.rs b/src/bundledb/reader.rs index cef97e4..9658a75 100644 --- a/src/bundledb/reader.rs +++ b/src/bundledb/reader.rs @@ -12,6 +12,10 @@ use std::sync::{Arc, Mutex}; quick_error!{ #[derive(Debug)] pub enum BundleReaderError { + TruncatedBundle(path: PathBuf) { + description("Bundle file is truncated") + display("Bundle reader error: bundle file is truncated {:?}", path) + } Read(err: io::Error, path: PathBuf) { cause(err) context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf()) @@ -107,6 +111,10 @@ impl BundleReader { info.encryption = header.encryption; debug!("Load bundle {}", info.id); let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize + info.chunk_list_size; + let actual_size = try!(fs::metadata(path).context(path)).len(); + if content_start + info.encoded_size != actual_size as usize { + return Err(BundleReaderError::TruncatedBundle(path.to_path_buf())); + } Ok((info, version, content_start)) } diff --git a/src/cli/args.rs b/src/cli/args.rs index 5e33765..fb121cf 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -340,7 +340,7 @@ pub fn parse() -> Result<(LogLevel, Arguments), ErrorCode> { .validator(|val| validate_repo_path(val, true, Some(false), Some(false))))) .subcommand(SubCommand::with_name("check").about("Check the repository, a backup or a backup subtree") .arg(Arg::from_usage("-b --bundles 'Check the bundles'")) - .arg(Arg::from_usage("[bundle_data] --bundle-data 'Check bundle contents (slow)'").requires("bundles").alias("--data")) + .arg(Arg::from_usage("[bundle_data] --bundle-data 'Check bundle contents (slow)'").requires("bundles").alias("data")) .arg(Arg::from_usage("-i --index 'Check the chunk index'")) .arg(Arg::from_usage(" 'Path of the repository/backup/subtree, [repository][::backup[::subtree]]'") .validator(|val| validate_repo_path(val, true, None, None)))) diff --git a/src/main.rs b/src/main.rs index ed08f5b..8cbb50c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,6 +24,7 @@ extern crate rand; extern crate time; extern crate xattr; extern crate crossbeam; +extern crate pbr; extern crate libc; extern crate tar; diff --git a/src/repository/integrity.rs b/src/repository/integrity.rs index 3161991..6f318cc 100644 --- a/src/repository/integrity.rs +++ b/src/repository/integrity.rs @@ -2,6 +2,9 @@ use ::prelude::*; use std::collections::VecDeque; use std::path::{Path, PathBuf}; +use std::time::Duration; + +use pbr::ProgressBar; quick_error!{ @@ -47,7 +50,11 @@ quick_error!{ impl Repository { fn check_index_chunks(&self) -> Result<(), RepositoryError> { - self.index.walk(|_hash, location| { + let mut count = 0; + let mut progress = ProgressBar::new(self.index.len() as u64); + progress.message("checking index: "); + progress.set_max_refresh_rate(Some(Duration::from_millis(100))); + let res = self.index.walk(|_hash, location| { // Lookup bundle id from map let bundle_id = try!(self.get_bundle_id(location.bundle)); // Get bundle object from bundledb @@ -60,8 +67,14 @@ impl Repository { if bundle.info.chunk_count <= location.chunk as usize { return Err(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into()) } + count += 1; + if count % 1000 == 0 { + progress.set(count); + } Ok(()) - }) + }); + progress.finish_print("checking index: done."); + res } fn check_chunks(&self, checked: &mut Bitmap, chunks: &[Chunk]) -> Result { @@ -148,7 +161,7 @@ impl Repository { }, Err(err) => return Err(err) }; - for (name, backup) in backup_map { + for (name, backup) in ProgressIter::new("ckecking backups", backup_map.len(), backup_map.into_iter()) { let path = name+"::"; try!(self.check_subtree(Path::new(&path).to_path_buf(), &backup.root, &mut checked)); } diff --git a/src/repository/mod.rs b/src/repository/mod.rs index 53ae60c..b973db0 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -111,15 +111,15 @@ impl Repository { }; if !new.is_empty() { info!("Adding {} new bundles to index", new.len()); - } - for bundle in new { - try!(repo.add_new_remote_bundle(bundle)) + for bundle in ProgressIter::new("adding bundles to index", new.len(), new.into_iter()) { + try!(repo.add_new_remote_bundle(bundle)) + } } if !gone.is_empty() { info!("Removig {} old bundles from index", gone.len()); - } - for bundle in gone { - try!(repo.remove_gone_remote_bundle(bundle)) + for bundle in gone { + try!(repo.remove_gone_remote_bundle(bundle)) + } } try!(repo.save_bundle_map()); repo.next_meta_bundle = repo.next_free_bundle_id(); @@ -227,10 +227,10 @@ impl Repository { } fn add_new_remote_bundle(&mut self, bundle: BundleInfo) -> Result<(), RepositoryError> { - debug!("Adding new bundle to index: {}", bundle.id); if self.bundle_map.find(&bundle.id).is_some() { return Ok(()) } + debug!("Adding new bundle to index: {}", bundle.id); let bundle_id = match bundle.mode { BundleMode::Data => self.next_data_bundle, BundleMode::Meta => self.next_meta_bundle diff --git a/src/repository/vacuum.rs b/src/repository/vacuum.rs index 4ec0abe..0d50c4f 100644 --- a/src/repository/vacuum.rs +++ b/src/repository/vacuum.rs @@ -43,7 +43,7 @@ impl Repository { self.dirty = false; return Ok(()) } - for id in &rewrite_bundles { + for id in ProgressIter::new("rewriting bundles", rewrite_bundles.len(), rewrite_bundles.iter()) { let bundle = &usage[id]; let bundle_id = self.bundle_map.get(*id).unwrap(); let chunks = try!(self.bundles.get_chunk_list(&bundle_id)); diff --git a/src/util/cli.rs b/src/util/cli.rs index f4c1ca5..d6e7685 100644 --- a/src/util/cli.rs +++ b/src/util/cli.rs @@ -1,3 +1,7 @@ +use pbr; +use std::io::Stdout; +use std::time::Duration; + pub fn to_file_size(size: u64) -> String { let mut size = size as f32; if size >= 512.0 { @@ -36,3 +40,43 @@ pub fn to_duration(dur: f32) -> String { let secs = (secs % 60) as f32 + subsecs; format!("{}:{:02}:{:04.1}", hours, mins, secs) } + + +pub struct ProgressIter { + inner: T, + msg: String, + bar: pbr::ProgressBar +} + +impl ProgressIter { + #[allow(blacklisted_name)] + pub fn new(msg: &str, max: usize, inner: T) -> Self { + let mut bar = pbr::ProgressBar::new(max as u64); + let msg = format!("{}: ", msg); + bar.message(&msg); + bar.set_max_refresh_rate(Some(Duration::from_millis(100))); + ProgressIter { inner: inner, bar: bar, msg: msg } + } +} + +impl Iterator for ProgressIter { + type Item = T::Item; + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } + + fn next(&mut self) -> Option { + match self.inner.next() { + None => { + let msg = self.msg.clone() + "done."; + self.bar.finish_print(&msg); + None + }, + Some(item) => { + self.bar.inc(); + Some(item) + } + } + } +}