Repairing backups (re #3)

pull/10/head
Dennis Schwerdel 2017-04-13 07:41:18 +02:00
parent 489a442821
commit df30f7a8fe
4 changed files with 150 additions and 38 deletions

View File

@ -10,6 +10,8 @@ This project follows [semantic versioning](http://semver.org).
- [modified] No longer trying to upload by rename - [modified] No longer trying to upload by rename
- [modified] No longer failing restore if setting file attributes fails - [modified] No longer failing restore if setting file attributes fails
- [modified] Backup files must end with `.backup` (**conversion needed**) - [modified] Backup files must end with `.backup` (**conversion needed**)
- [modified] Bundle files must end with `.bundle`
- [modified] Ingnoring corrupt bundles instead of failing
- [fixed] Creating empty bundle cache on init to avoid warnings - [fixed] Creating empty bundle cache on init to avoid warnings
- [fixed] Calling sodiumoxide::init for faster algorithms and thread safety (not needed) - [fixed] Calling sodiumoxide::init for faster algorithms and thread safety (not needed)
- [fixed] Fixed a deadlock in the bundle upload code - [fixed] Fixed a deadlock in the bundle upload code

View File

@ -376,6 +376,7 @@ impl BundleDb {
return self.evacuate_broken_bundle(stored); return self.evacuate_broken_bundle(stored);
} }
}; };
warn!("Problem detected: bundle data was truncated: {}", id);
info!("Copying readable data into new bundle"); info!("Copying readable data into new bundle");
let info = stored.info.clone(); let info = stored.info.clone();
let mut new_bundle = try!(self.create_bundle(info.mode, info.hash_method, info.compression, info.encryption)); let mut new_bundle = try!(self.create_bundle(info.mode, info.hash_method, info.compression, info.encryption));

View File

@ -430,15 +430,14 @@ pub fn run() -> Result<(), ErrorCode> {
checked!(repo.check_index(repair), "check index", ErrorCode::CheckRun); checked!(repo.check_index(repair), "check index", ErrorCode::CheckRun);
} }
if let Some(backup_name) = backup_name { if let Some(backup_name) = backup_name {
let backup = try!(get_backup(&repo, &backup_name)); let mut backup = try!(get_backup(&repo, &backup_name));
if let Some(path) = inode { if let Some(path) = inode {
let inode = checked!(repo.get_backup_inode(&backup, &path), "load subpath inode", ErrorCode::LoadInode); checked!(repo.check_backup_inode(&backup_name, &mut backup, Path::new(&path), repair), "check inode", ErrorCode::CheckRun)
checked!(repo.check_inode(&inode, Path::new(&path)), "check inode", ErrorCode::CheckRun)
} else { } else {
checked!(repo.check_backup(&backup), "check backup", ErrorCode::CheckRun) checked!(repo.check_backup(&backup_name, &mut backup, repair), "check backup", ErrorCode::CheckRun)
} }
} else { } else {
checked!(repo.check_backups(), "check repository", ErrorCode::CheckRun) checked!(repo.check_backups(repair), "check repository", ErrorCode::CheckRun)
} }
repo.set_clean(); repo.set_clean();
info!("Integrity verified") info!("Integrity verified")

View File

@ -2,7 +2,6 @@ use ::prelude::*;
use super::*; use super::*;
use std::collections::VecDeque;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::time::Duration; use std::time::Duration;
@ -106,50 +105,154 @@ impl Repository {
Ok(()) Ok(())
} }
fn check_subtree(&mut self, path: PathBuf, chunks: &[Chunk], checked: &mut Bitmap) -> Result<(), RepositoryError> { fn check_subtree(&mut self, path: PathBuf, chunks: &[Chunk], checked: &mut Bitmap, repair: bool) -> Result<Option<ChunkList>, RepositoryError> {
let mut todo = VecDeque::new(); let mut modified = false;
todo.push_back((path, ChunkList::from(chunks.to_vec()))); match self.check_chunks(checked, chunks) {
while let Some((path, chunks)) = todo.pop_front() { Ok(false) => return Ok(None),
match self.check_chunks(checked, &chunks) { Ok(true) => (),
Ok(false) => continue, // checked this chunk list before Err(err) => return Err(IntegrityError::BrokenInode(path, Box::new(err)).into())
Ok(true) => (), }
Err(err) => return Err(IntegrityError::BrokenInode(path, Box::new(err)).into()) let mut inode = try!(self.get_inode(chunks));
} // Mark the content chunks as used
let inode = try!(self.get_inode(&chunks)); if let Err(err) = self.check_inode_contents(&inode, checked) {
// Mark the content chunks as used if repair {
if let Err(err) = self.check_inode_contents(&inode, checked) { warn!("Problem detected: data of {:?} is corrupt\n\tcaused by: {}", path, err);
info!("Removing inode data");
inode.data = Some(FileData::Inline(vec![].into()));
inode.size = 0;
modified = true;
} else {
return Err(IntegrityError::MissingInodeData(path, Box::new(err)).into()) return Err(IntegrityError::MissingInodeData(path, Box::new(err)).into())
} }
// Put children in todo }
if let Some(children) = inode.children { // Put children in todo
for (name, chunks) in children { if let Some(ref mut children) = inode.children {
todo.push_back((path.join(name), chunks)); let mut removed = vec![];
for (name, chunks) in children.iter_mut() {
match self.check_subtree(path.join(name), chunks, checked, repair) {
Ok(None) => (),
Ok(Some(c)) => {
*chunks = c;
modified = true;
},
Err(err) => {
warn!("Problem detected: inode {:?} is corrupt\n\tcaused by: {}", path.join(name), err);
info!("Removing broken inode from backup");
removed.push(name.to_string());
modified = true;
}
} }
} }
for name in removed {
children.remove(&name);
}
} }
if modified {
Ok(Some(try!(self.put_inode(&inode))))
} else {
Ok(None)
}
}
fn evacuate_broken_backup(&self, name: &str) -> Result<(), RepositoryError> {
warn!("The backup {} was corrupted and needed to be modified.", name);
let src = self.layout.backup_path(name);
let dst = src.with_extension("backup.broken");
if fs::rename(&src, &dst).is_err() {
try!(fs::copy(&src, &dst));
try!(fs::remove_file(&src));
}
info!("The original backup was renamed to {:?}", dst);
Ok(()) Ok(())
} }
#[inline] #[inline]
pub fn check_backup(&mut self, backup: &Backup) -> Result<(), RepositoryError> { pub fn check_backup(&mut self, name: &str, backup: &mut Backup, repair: bool) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.write_mode());
Some(self.lock(false))
} else {
None
};
info!("Checking backup..."); info!("Checking backup...");
let mut checked = Bitmap::new(self.index.capacity()); let mut checked = Bitmap::new(self.index.capacity());
self.check_subtree(Path::new("").to_path_buf(), &backup.root, &mut checked) if let Some(chunks) = try!(self.check_subtree(Path::new("").to_path_buf(), &backup.root, &mut checked, repair)) {
} try!(self.flush());
backup.root = chunks;
pub fn check_inode(&mut self, inode: &Inode, path: &Path) -> Result<(), RepositoryError> { backup.modified = true;
info!("Checking inode..."); try!(self.evacuate_broken_backup(&name));
let mut checked = Bitmap::new(self.index.capacity()); try!(self.save_backup(&backup, &name));
try!(self.check_inode_contents(inode, &mut checked));
if let Some(ref children) = inode.children {
for chunks in children.values() {
try!(self.check_subtree(path.to_path_buf(), chunks, &mut checked))
}
} }
Ok(()) Ok(())
} }
pub fn check_backups(&mut self) -> Result<(), RepositoryError> { pub fn check_backup_inode(&mut self, name: &str, backup: &mut Backup, path: &Path, repair: bool) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.write_mode());
Some(self.lock(false))
} else {
None
};
info!("Checking inode...");
let mut checked = Bitmap::new(self.index.capacity());
let mut inodes = try!(self.get_backup_path(&backup, path));
let mut inode = inodes.pop().unwrap();
let mut modified = false;
if let Err(err) = self.check_inode_contents(&inode, &mut checked) {
if repair {
warn!("Problem detected: data of {:?} is corrupt\n\tcaused by: {}", path, err);
info!("Removing inode data");
inode.data = Some(FileData::Inline(vec![].into()));
inode.size = 0;
modified = true;
} else {
return Err(IntegrityError::MissingInodeData(path.to_path_buf(), Box::new(err)).into())
}
}
if let Some(ref mut children) = inode.children {
let mut removed = vec![];
for (name, chunks) in children.iter_mut() {
match self.check_subtree(path.join(name), chunks, &mut checked, repair) {
Ok(None) => (),
Ok(Some(c)) => {
*chunks = c;
modified = true;
},
Err(err) => {
warn!("Problem detected: inode {:?} is corrupt\n\tcaused by: {}", path.join(name), err);
info!("Removing broken inode from backup");
removed.push(name.to_string());
modified = true;
}
}
}
for name in removed {
children.remove(&name);
}
}
let mut chunks = try!(self.put_inode(&inode));
while let Some(mut parent) = inodes.pop() {
parent.children.as_mut().unwrap().insert(inode.name, chunks);
inode = parent;
chunks = try!(self.put_inode(&inode));
}
if modified {
try!(self.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(&name));
try!(self.save_backup(&backup, &name));
}
Ok(())
}
pub fn check_backups(&mut self, repair: bool) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.write_mode());
Some(self.lock(false))
} else {
None
};
info!("Checking backups..."); info!("Checking backups...");
let mut checked = Bitmap::new(self.index.capacity()); let mut checked = Bitmap::new(self.index.capacity());
let backup_map = match self.get_backups() { let backup_map = match self.get_backups() {
@ -160,9 +263,15 @@ impl Repository {
}, },
Err(err) => return Err(err) Err(err) => return Err(err)
}; };
for (name, backup) in ProgressIter::new("ckecking backups", backup_map.len(), backup_map.into_iter()) { for (name, mut backup) in ProgressIter::new("ckecking backups", backup_map.len(), backup_map.into_iter()) {
let path = name+"::"; let path = format!("{}::", name);
try!(self.check_subtree(Path::new(&path).to_path_buf(), &backup.root, &mut checked)); if let Some(chunks) = try!(self.check_subtree(Path::new(&path).to_path_buf(), &backup.root, &mut checked, repair)) {
try!(self.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(&name));
try!(self.save_backup(&backup, &name));
}
} }
Ok(()) Ok(())
} }
@ -248,6 +357,7 @@ impl Repository {
info!("Checking bundle integrity..."); info!("Checking bundle integrity...");
if try!(self.bundles.check(full, repair)) { if try!(self.bundles.check(full, repair)) {
// Some bundles got repaired // Some bundles got repaired
warn!("Some bundles have been rewritten, please remove the broken bundles manually.");
try!(self.bundles.finish_uploads()); try!(self.bundles.finish_uploads());
try!(self.rebuild_bundle_map()); try!(self.rebuild_bundle_map());
try!(self.rebuild_index()); try!(self.rebuild_index());