Repairing backups (re #3)

CHANGELOG.md

@@ -10,6 +10,8 @@ This project follows [semantic versioning](http://semver.org).
 - [modified] No longer trying to upload by rename
 - [modified] No longer failing restore if setting file attributes fails
 - [modified] Backup files must end with `.backup` (**conversion needed**)
+- [modified] Bundle files must end with `.bundle`
+- [modified] Ingnoring corrupt bundles instead of failing
 - [fixed] Creating empty bundle cache on init to avoid warnings
 - [fixed] Calling sodiumoxide::init for faster algorithms and thread safety (not needed)
 - [fixed] Fixed a deadlock in the bundle upload code

src/bundledb/db.rs

@@ -376,6 +376,7 @@ impl BundleDb {
                 return self.evacuate_broken_bundle(stored);
             }
         };
+        warn!("Problem detected: bundle data was truncated: {}", id);
         info!("Copying readable data into new bundle");
         let info = stored.info.clone();
         let mut new_bundle = try!(self.create_bundle(info.mode, info.hash_method, info.compression, info.encryption));

src/cli/mod.rs

@@ -430,15 +430,14 @@ pub fn run() -> Result<(), ErrorCode> {
                 checked!(repo.check_index(repair), "check index", ErrorCode::CheckRun);
             }
             if let Some(backup_name) = backup_name {
-                let backup = try!(get_backup(&repo, &backup_name));
+                let mut backup = try!(get_backup(&repo, &backup_name));
                 if let Some(path) = inode {
-                    let inode = checked!(repo.get_backup_inode(&backup, &path), "load subpath inode", ErrorCode::LoadInode);
+                    checked!(repo.check_backup_inode(&backup_name, &mut backup, Path::new(&path), repair), "check inode", ErrorCode::CheckRun)
-                    checked!(repo.check_inode(&inode, Path::new(&path)), "check inode", ErrorCode::CheckRun)
                 } else {
-                    checked!(repo.check_backup(&backup), "check backup", ErrorCode::CheckRun)
+                    checked!(repo.check_backup(&backup_name, &mut backup, repair), "check backup", ErrorCode::CheckRun)
                 }
             } else {
-                checked!(repo.check_backups(), "check repository", ErrorCode::CheckRun)
+                checked!(repo.check_backups(repair), "check repository", ErrorCode::CheckRun)
             }
             repo.set_clean();
             info!("Integrity verified")

src/repository/integrity.rs

@@ -2,7 +2,6 @@ use ::prelude::*;
 
 use super::*;
 
-use std::collections::VecDeque;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
 
@@ -106,50 +105,154 @@ impl Repository {
         Ok(())
     }
 
-    fn check_subtree(&mut self, path: PathBuf, chunks: &[Chunk], checked: &mut Bitmap) -> Result<(), RepositoryError> {
+    fn check_subtree(&mut self, path: PathBuf, chunks: &[Chunk], checked: &mut Bitmap, repair: bool) -> Result<Option<ChunkList>, RepositoryError> {
-        let mut todo = VecDeque::new();
+        let mut modified = false;
-        todo.push_back((path, ChunkList::from(chunks.to_vec())));
+        match self.check_chunks(checked, chunks) {
-        while let Some((path, chunks)) = todo.pop_front() {
+            Ok(false) => return Ok(None),
-            match self.check_chunks(checked, &chunks) {
-                Ok(false) => continue, // checked this chunk list before
             Ok(true) => (),
             Err(err) => return Err(IntegrityError::BrokenInode(path, Box::new(err)).into())
         }
-            let inode = try!(self.get_inode(&chunks));
+        let mut inode = try!(self.get_inode(chunks));
         // Mark the content chunks as used
         if let Err(err) = self.check_inode_contents(&inode, checked) {
+            if repair {
+                warn!("Problem detected: data of {:?} is corrupt\n\tcaused by: {}", path, err);
+                info!("Removing inode data");
+                inode.data = Some(FileData::Inline(vec![].into()));
+                inode.size = 0;
+                modified = true;
+            } else {
                 return Err(IntegrityError::MissingInodeData(path, Box::new(err)).into())
             }
+        }
         // Put children in todo
-            if let Some(children) = inode.children {
+        if let Some(ref mut children) = inode.children {
-                for (name, chunks) in children {
+            let mut removed = vec![];
-                    todo.push_back((path.join(name), chunks));
+            for (name, chunks) in children.iter_mut() {
+                match self.check_subtree(path.join(name), chunks, checked, repair) {
+                    Ok(None) => (),
+                    Ok(Some(c)) => {
+                        *chunks = c;
+                        modified = true;
+                    },
+                    Err(err) => {
+                        warn!("Problem detected: inode {:?} is corrupt\n\tcaused by: {}", path.join(name), err);
+                        info!("Removing broken inode from backup");
+                        removed.push(name.to_string());
+                        modified = true;
                     }
                 }
             }
+            for name in removed {
+                children.remove(&name);
+            }
+        }
+        if modified {
+            Ok(Some(try!(self.put_inode(&inode))))
+        } else {
+            Ok(None)
+        }
+    }
+
+    fn evacuate_broken_backup(&self, name: &str) -> Result<(), RepositoryError> {
+        warn!("The backup {} was corrupted and needed to be modified.", name);
+        let src = self.layout.backup_path(name);
+        let dst = src.with_extension("backup.broken");
+        if fs::rename(&src, &dst).is_err() {
+            try!(fs::copy(&src, &dst));
+            try!(fs::remove_file(&src));
+        }
+        info!("The original backup was renamed to {:?}", dst);
         Ok(())
     }
 
     #[inline]
-    pub fn check_backup(&mut self, backup: &Backup) -> Result<(), RepositoryError> {
+    pub fn check_backup(&mut self, name: &str, backup: &mut Backup, repair: bool) -> Result<(), RepositoryError> {
+        let _lock = if repair {
+            try!(self.write_mode());
+            Some(self.lock(false))
+        } else {
+            None
+        };
         info!("Checking backup...");
         let mut checked = Bitmap::new(self.index.capacity());
-        self.check_subtree(Path::new("").to_path_buf(), &backup.root, &mut checked)
+        if let Some(chunks) = try!(self.check_subtree(Path::new("").to_path_buf(), &backup.root, &mut checked, repair)) {
-    }
+            try!(self.flush());
-
+            backup.root = chunks;
-    pub fn check_inode(&mut self, inode: &Inode, path: &Path) -> Result<(), RepositoryError> {
+            backup.modified = true;
-        info!("Checking inode...");
+            try!(self.evacuate_broken_backup(&name));
-        let mut checked = Bitmap::new(self.index.capacity());
+            try!(self.save_backup(&backup, &name));
-        try!(self.check_inode_contents(inode, &mut checked));
-        if let Some(ref children) = inode.children {
-            for chunks in children.values() {
-                try!(self.check_subtree(path.to_path_buf(), chunks, &mut checked))
-            }
         }
         Ok(())
     }
 
-    pub fn check_backups(&mut self) -> Result<(), RepositoryError> {
+    pub fn check_backup_inode(&mut self, name: &str, backup: &mut Backup, path: &Path, repair: bool) -> Result<(), RepositoryError> {
+        let _lock = if repair {
+            try!(self.write_mode());
+            Some(self.lock(false))
+        } else {
+            None
+        };
+        info!("Checking inode...");
+        let mut checked = Bitmap::new(self.index.capacity());
+        let mut inodes = try!(self.get_backup_path(&backup, path));
+        let mut inode = inodes.pop().unwrap();
+        let mut modified = false;
+        if let Err(err) = self.check_inode_contents(&inode, &mut checked) {
+            if repair {
+                warn!("Problem detected: data of {:?} is corrupt\n\tcaused by: {}", path, err);
+                info!("Removing inode data");
+                inode.data = Some(FileData::Inline(vec![].into()));
+                inode.size = 0;
+                modified = true;
+            } else {
+                return Err(IntegrityError::MissingInodeData(path.to_path_buf(), Box::new(err)).into())
+            }
+        }
+        if let Some(ref mut children) = inode.children {
+            let mut removed = vec![];
+            for (name, chunks) in children.iter_mut() {
+                match self.check_subtree(path.join(name), chunks, &mut checked, repair) {
+                    Ok(None) => (),
+                    Ok(Some(c)) => {
+                        *chunks = c;
+                        modified = true;
+                    },
+                    Err(err) => {
+                        warn!("Problem detected: inode {:?} is corrupt\n\tcaused by: {}", path.join(name), err);
+                        info!("Removing broken inode from backup");
+                        removed.push(name.to_string());
+                        modified = true;
+                    }
+                }
+            }
+            for name in removed {
+                children.remove(&name);
+            }
+        }
+        let mut chunks = try!(self.put_inode(&inode));
+        while let Some(mut parent) = inodes.pop() {
+            parent.children.as_mut().unwrap().insert(inode.name, chunks);
+            inode = parent;
+            chunks = try!(self.put_inode(&inode));
+        }
+        if modified {
+            try!(self.flush());
+            backup.root = chunks;
+            backup.modified = true;
+            try!(self.evacuate_broken_backup(&name));
+            try!(self.save_backup(&backup, &name));
+        }
+        Ok(())
+    }
+
+    pub fn check_backups(&mut self, repair: bool) -> Result<(), RepositoryError> {
+        let _lock = if repair {
+            try!(self.write_mode());
+            Some(self.lock(false))
+        } else {
+            None
+        };
         info!("Checking backups...");
         let mut checked = Bitmap::new(self.index.capacity());
         let backup_map = match self.get_backups() {
@@ -160,9 +263,15 @@ impl Repository {
             },
             Err(err) => return Err(err)
         };
-        for (name, backup) in ProgressIter::new("ckecking backups", backup_map.len(), backup_map.into_iter()) {
+        for (name, mut backup) in ProgressIter::new("ckecking backups", backup_map.len(), backup_map.into_iter()) {
-            let path = name+"::";
+            let path = format!("{}::", name);
-            try!(self.check_subtree(Path::new(&path).to_path_buf(), &backup.root, &mut checked));
+            if let Some(chunks) = try!(self.check_subtree(Path::new(&path).to_path_buf(), &backup.root, &mut checked, repair)) {
+                try!(self.flush());
+                backup.root = chunks;
+                backup.modified = true;
+                try!(self.evacuate_broken_backup(&name));
+                try!(self.save_backup(&backup, &name));
+            }
         }
         Ok(())
     }
@@ -248,6 +357,7 @@ impl Repository {
         info!("Checking bundle integrity...");
         if try!(self.bundles.check(full, repair)) {
             // Some bundles got repaired
+            warn!("Some bundles have been rewritten, please remove the broken bundles manually.");
             try!(self.bundles.finish_uploads());
             try!(self.rebuild_bundle_map());
             try!(self.rebuild_index());