Fixes and progress bars

Cargo.lock

@@ -15,6 +15,7 @@ dependencies = [
  "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
  "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pbr 1.0.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -228,6 +229,17 @@ name = "num-traits"
 version = "0.1.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "pbr"
+version = "1.0.0-alpha.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "pkg-config"
 version = "0.3.9"
@@ -489,6 +501,7 @@ dependencies = [
 "checksum num-integer 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "21e4df1098d1d797d27ef0c69c178c3fab64941559b290fcae198e0825c9c8b5"
 "checksum num-iter 0.1.33 (registry+https://github.com/rust-lang/crates.io-index)" = "f7d1891bd7b936f12349b7d1403761c8a0b85a18b148e9da4429d5d102c1a41e"
 "checksum num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "e1cbfa3781f3fe73dc05321bed52a06d2d491eaa764c52335cf4399f046ece99"
+"checksum pbr 1.0.0-alpha.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2c0434823e05f3f0031a1d9f4323c954543ed66f2cc2bb4700ff539fdd06ce0e"
 "checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903"
 "checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c"
 "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"

Cargo.toml

@@ -29,6 +29,7 @@ rand = "0.3"
 tar = "0.4"
 xattr = "0.1"
 crossbeam = "0.2"
+pbr = "1.0.0-alpha.3"
 time = "*"
 libc = "*"
 

src/bundledb/db.rs

@@ -8,7 +8,6 @@ use std::sync::{Arc, Mutex};
 use std::io;
 use std::mem;
 
-
 quick_error!{
     #[derive(Debug)]
     pub enum BundleDbError {
@@ -78,10 +77,15 @@ pub fn load_bundles(path: &Path, base: &Path, bundles: &mut HashMap<BundleId, St
     }
     let mut new = vec![];
     for path in bundle_paths {
-        let bundle = StoredBundle {
+        let info = match BundleReader::load_info(base.join(&path), crypto.clone()) {
-            info: try!(BundleReader::load_info(base.join(&path), crypto.clone())),
+            Ok(info) => info,
-            path: path
+            Err(BundleReaderError::TruncatedBundle(path)) => {
+                warn!("Ignoring truncated bundle {:?}", path);
+                continue
+            },
+            Err(err) => return Err(err.into())
         };
+        let bundle = StoredBundle { info: info, path: path };
         let id = bundle.info.id.clone();
         if !bundles.contains_key(&id) {
             new.push(bundle.clone());
@@ -304,7 +308,7 @@ impl BundleDb {
     }
 
     pub fn check(&mut self, full: bool) -> Result<(), BundleDbError> {
-        for stored in self.remote_bundles.values() {
+        for stored in ProgressIter::new("checking bundles", self.remote_bundles.len(), self.remote_bundles.values()) {
             let mut bundle = try!(self.get_bundle(stored));
             try!(bundle.check(full))
         }

src/bundledb/reader.rs

@@ -12,6 +12,10 @@ use std::sync::{Arc, Mutex};
 quick_error!{
     #[derive(Debug)]
     pub enum BundleReaderError {
+        TruncatedBundle(path: PathBuf) {
+            description("Bundle file is truncated")
+            display("Bundle reader error: bundle file is truncated {:?}", path)
+        }
         Read(err: io::Error, path: PathBuf) {
             cause(err)
             context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf())
@@ -107,6 +111,10 @@ impl BundleReader {
         info.encryption = header.encryption;
         debug!("Load bundle {}", info.id);
         let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize + info.chunk_list_size;
+        let actual_size = try!(fs::metadata(path).context(path)).len();
+        if content_start + info.encoded_size != actual_size as usize {
+            return Err(BundleReaderError::TruncatedBundle(path.to_path_buf()));
+        }
         Ok((info, version, content_start))
     }
 

src/cli/args.rs

@@ -340,7 +340,7 @@ pub fn parse() -> Result<(LogLevel, Arguments), ErrorCode> {
                 .validator(|val| validate_repo_path(val, true, Some(false), Some(false)))))
         .subcommand(SubCommand::with_name("check").about("Check the repository, a backup or a backup subtree")
             .arg(Arg::from_usage("-b --bundles 'Check the bundles'"))
-            .arg(Arg::from_usage("[bundle_data] --bundle-data 'Check bundle contents (slow)'").requires("bundles").alias("--data"))
+            .arg(Arg::from_usage("[bundle_data] --bundle-data 'Check bundle contents (slow)'").requires("bundles").alias("data"))
             .arg(Arg::from_usage("-i --index 'Check the chunk index'"))
             .arg(Arg::from_usage("<PATH> 'Path of the repository/backup/subtree, [repository][::backup[::subtree]]'")
                 .validator(|val| validate_repo_path(val, true, None, None))))

src/main.rs

@@ -24,6 +24,7 @@ extern crate rand;
 extern crate time;
 extern crate xattr;
 extern crate crossbeam;
+extern crate pbr;
 extern crate libc;
 extern crate tar;
 

src/repository/integrity.rs

@@ -2,6 +2,9 @@ use ::prelude::*;
 
 use std::collections::VecDeque;
 use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use pbr::ProgressBar;
 
 
 quick_error!{
@@ -47,7 +50,11 @@ quick_error!{
 
 impl Repository {
     fn check_index_chunks(&self) -> Result<(), RepositoryError> {
-        self.index.walk(|_hash, location| {
+        let mut count = 0;
+        let mut progress = ProgressBar::new(self.index.len() as u64);
+        progress.message("checking index: ");
+        progress.set_max_refresh_rate(Some(Duration::from_millis(100)));
+        let res = self.index.walk(|_hash, location| {
             // Lookup bundle id from map
             let bundle_id = try!(self.get_bundle_id(location.bundle));
             // Get bundle object from bundledb
@@ -60,8 +67,14 @@ impl Repository {
             if bundle.info.chunk_count <= location.chunk as usize {
                 return Err(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into())
             }
+            count += 1;
+            if count % 1000 == 0 {
+                progress.set(count);
+            }
             Ok(())
-        })
+        });
+        progress.finish_print("checking index: done.");
+        res
     }
 
     fn check_chunks(&self, checked: &mut Bitmap, chunks: &[Chunk]) -> Result<bool, RepositoryError> {
@@ -148,7 +161,7 @@ impl Repository {
             },
             Err(err) => return Err(err)
         };
-        for (name, backup) in backup_map {
+        for (name, backup) in ProgressIter::new("ckecking backups", backup_map.len(), backup_map.into_iter()) {
             let path = name+"::";
             try!(self.check_subtree(Path::new(&path).to_path_buf(), &backup.root, &mut checked));
         }

src/repository/mod.rs

@@ -111,15 +111,15 @@ impl Repository {
         };
         if !new.is_empty() {
             info!("Adding {} new bundles to index", new.len());
-        }
+            for bundle in ProgressIter::new("adding bundles to index", new.len(), new.into_iter()) {
-        for bundle in new {
+                try!(repo.add_new_remote_bundle(bundle))
-            try!(repo.add_new_remote_bundle(bundle))
+            }
         }
         if !gone.is_empty() {
             info!("Removig {} old bundles from index", gone.len());
-        }
+            for bundle in gone {
-        for bundle in gone {
+                try!(repo.remove_gone_remote_bundle(bundle))
-            try!(repo.remove_gone_remote_bundle(bundle))
+            }
         }
         try!(repo.save_bundle_map());
         repo.next_meta_bundle = repo.next_free_bundle_id();
@@ -227,10 +227,10 @@ impl Repository {
     }
 
     fn add_new_remote_bundle(&mut self, bundle: BundleInfo) -> Result<(), RepositoryError> {
-        debug!("Adding new bundle to index: {}", bundle.id);
         if self.bundle_map.find(&bundle.id).is_some() {
             return Ok(())
         }
+        debug!("Adding new bundle to index: {}", bundle.id);
         let bundle_id = match bundle.mode {
             BundleMode::Data => self.next_data_bundle,
             BundleMode::Meta => self.next_meta_bundle

src/repository/vacuum.rs

@@ -43,7 +43,7 @@ impl Repository {
             self.dirty = false;
             return Ok(())
         }
-        for id in &rewrite_bundles {
+        for id in ProgressIter::new("rewriting bundles", rewrite_bundles.len(), rewrite_bundles.iter()) {
             let bundle = &usage[id];
             let bundle_id = self.bundle_map.get(*id).unwrap();
             let chunks = try!(self.bundles.get_chunk_list(&bundle_id));

src/util/cli.rs

@@ -1,3 +1,7 @@
+use pbr;
+use std::io::Stdout;
+use std::time::Duration;
+
 pub fn to_file_size(size: u64) -> String {
     let mut size = size as f32;
     if size >= 512.0 {
@@ -36,3 +40,43 @@ pub fn to_duration(dur: f32) -> String {
     let secs = (secs % 60) as f32 + subsecs;
     format!("{}:{:02}:{:04.1}", hours, mins, secs)
 }
+
+
+pub struct ProgressIter<T> {
+    inner: T,
+    msg: String,
+    bar: pbr::ProgressBar<Stdout>
+}
+
+impl<T> ProgressIter<T> {
+    #[allow(blacklisted_name)]
+    pub fn new(msg: &str, max: usize, inner: T) -> Self {
+        let mut bar = pbr::ProgressBar::new(max as u64);
+        let msg = format!("{}: ", msg);
+        bar.message(&msg);
+        bar.set_max_refresh_rate(Some(Duration::from_millis(100)));
+        ProgressIter { inner: inner, bar: bar, msg: msg }
+    }
+}
+
+impl<T: Iterator> Iterator for ProgressIter<T> {
+    type Item = T::Item;
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.inner.size_hint()
+    }
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.inner.next() {
+            None => {
+                let msg = self.msg.clone() + "done.";
+                self.bar.finish_print(&msg);
+                None
+            },
+            Some(item) => {
+                self.bar.inc();
+                Some(item)
+            }
+        }
+    }
+}