zvault/src/repository/integrity.rs

420 lines
16 KiB
Rust
Raw Normal View History

2017-03-21 10:28:11 +00:00
use ::prelude::*;
2017-03-10 11:43:32 +00:00
2017-04-12 18:19:21 +00:00
use super::*;
use std::path::{Path, PathBuf};
2017-04-11 06:49:45 +00:00
use std::time::Duration;
use pbr::ProgressBar;
2017-03-10 11:43:32 +00:00
2017-03-16 08:42:30 +00:00
quick_error!{
#[derive(Debug)]
pub enum IntegrityError {
2017-03-16 08:42:30 +00:00
MissingChunk(hash: Hash) {
description("Missing chunk")
display("Missing chunk: {}", hash)
}
MissingBundleId(id: u32) {
description("Missing bundle")
display("Missing bundle: {}", id)
}
MissingBundle(id: BundleId) {
description("Missing bundle")
display("Missing bundle: {}", id)
}
NoSuchChunk(bundle: BundleId, chunk: u32) {
description("No such chunk")
2017-04-07 16:57:49 +00:00
display("Bundle {} does not contain the chunk {}", bundle, chunk)
2017-03-16 08:42:30 +00:00
}
2017-04-10 16:21:26 +00:00
RemoteBundlesNotInMap {
description("Remote bundles missing from map")
}
MapContainsDuplicates {
description("Map contains duplicates")
}
BrokenInode(path: PathBuf, err: Box<RepositoryError>) {
cause(err)
description("Broken inode")
display("Broken inode: {:?}\n\tcaused by: {}", path, err)
}
MissingInodeData(path: PathBuf, err: Box<RepositoryError>) {
cause(err)
description("Missing inode data")
display("Missing inode data in: {:?}\n\tcaused by: {}", path, err)
}
2017-03-16 08:42:30 +00:00
}
}
2017-03-10 11:43:32 +00:00
impl Repository {
2017-03-20 13:03:29 +00:00
fn check_index_chunks(&self) -> Result<(), RepositoryError> {
2017-04-11 06:49:45 +00:00
let mut count = 0;
let mut progress = ProgressBar::new(self.index.len() as u64);
progress.message("checking index: ");
progress.set_max_refresh_rate(Some(Duration::from_millis(100)));
let res = self.index.walk(|_hash, location| {
2017-03-20 13:03:29 +00:00
// Lookup bundle id from map
2017-03-23 08:31:23 +00:00
let bundle_id = try!(self.get_bundle_id(location.bundle));
2017-03-20 13:03:29 +00:00
// Get bundle object from bundledb
2017-03-21 14:38:42 +00:00
let bundle = if let Some(bundle) = self.bundles.get_bundle_info(&bundle_id) {
2017-03-20 13:03:29 +00:00
bundle
} else {
return Err(IntegrityError::MissingBundle(bundle_id.clone()).into())
2017-03-20 13:03:29 +00:00
};
// Get chunk from bundle
2017-04-07 16:57:49 +00:00
if bundle.info.chunk_count <= location.chunk as usize {
return Err(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into())
2017-03-20 13:03:29 +00:00
}
2017-04-11 06:49:45 +00:00
count += 1;
if count % 1000 == 0 {
progress.set(count);
}
2017-03-23 08:31:23 +00:00
Ok(())
2017-04-11 06:49:45 +00:00
});
progress.finish_print("checking index: done.");
res
2017-03-20 13:03:29 +00:00
}
fn check_chunks(&self, checked: &mut Bitmap, chunks: &[Chunk], mark: bool) -> Result<bool, RepositoryError> {
2017-03-20 13:03:29 +00:00
let mut new = false;
for &(hash, _len) in chunks {
if let Some(pos) = self.index.pos(&hash) {
2017-03-20 21:24:53 +00:00
new |= !checked.get(pos);
if mark {
checked.set(pos);
}
2017-03-20 13:03:29 +00:00
} else {
return Err(IntegrityError::MissingChunk(hash).into())
2017-03-20 13:03:29 +00:00
}
}
Ok(new)
}
2017-03-22 10:10:13 +00:00
fn check_inode_contents(&mut self, inode: &Inode, checked: &mut Bitmap) -> Result<(), RepositoryError> {
2017-04-03 05:35:00 +00:00
match inode.data {
None | Some(FileData::Inline(_)) => (),
Some(FileData::ChunkedDirect(ref chunks)) => {
try!(self.check_chunks(checked, chunks, true));
2017-03-22 10:10:13 +00:00
},
2017-04-03 05:35:00 +00:00
Some(FileData::ChunkedIndirect(ref chunks)) => {
if try!(self.check_chunks(checked, chunks, true)) {
2017-03-22 10:10:13 +00:00
let chunk_data = try!(self.get_data(&chunks));
let chunks = ChunkList::read_from(&chunk_data);
try!(self.check_chunks(checked, &chunks, true));
2017-03-22 10:10:13 +00:00
}
}
}
Ok(())
}
2017-04-13 05:41:18 +00:00
fn check_subtree(&mut self, path: PathBuf, chunks: &[Chunk], checked: &mut Bitmap, repair: bool) -> Result<Option<ChunkList>, RepositoryError> {
let mut modified = false;
match self.check_chunks(checked, chunks, false) {
2017-04-13 05:41:18 +00:00
Ok(false) => return Ok(None),
Ok(true) => (),
Err(err) => return Err(IntegrityError::BrokenInode(path, Box::new(err)).into())
}
let mut inode = try!(self.get_inode(chunks));
// Mark the content chunks as used
if let Err(err) = self.check_inode_contents(&inode, checked) {
if repair {
warn!("Problem detected: data of {:?} is corrupt\n\tcaused by: {}", path, err);
info!("Removing inode data");
inode.data = Some(FileData::Inline(vec![].into()));
inode.size = 0;
modified = true;
} else {
return Err(IntegrityError::MissingInodeData(path, Box::new(err)).into())
}
2017-04-13 05:41:18 +00:00
}
// Put children in todo
if let Some(ref mut children) = inode.children {
let mut removed = vec![];
for (name, chunks) in children.iter_mut() {
match self.check_subtree(path.join(name), chunks, checked, repair) {
Ok(None) => (),
Ok(Some(c)) => {
*chunks = c;
modified = true;
},
2017-04-16 21:18:25 +00:00
Err(err) => if repair {
2017-04-13 05:41:18 +00:00
warn!("Problem detected: inode {:?} is corrupt\n\tcaused by: {}", path.join(name), err);
info!("Removing broken inode from backup");
removed.push(name.to_string());
modified = true;
2017-04-16 21:18:25 +00:00
} else {
return Err(err)
2017-04-13 05:41:18 +00:00
}
2017-03-22 10:10:13 +00:00
}
}
2017-04-13 05:41:18 +00:00
for name in removed {
children.remove(&name);
}
}
if modified {
Ok(Some(try!(self.put_inode(&inode))))
} else {
try!(self.check_chunks(checked, chunks, true));
2017-04-13 05:41:18 +00:00
Ok(None)
2017-03-22 10:10:13 +00:00
}
2017-04-13 05:41:18 +00:00
}
fn evacuate_broken_backup(&self, name: &str) -> Result<(), RepositoryError> {
warn!("The backup {} was corrupted and needed to be modified.", name);
let src = self.layout.backup_path(name);
let mut dst = src.with_extension("backup.broken");
let mut num = 1;
while dst.exists() {
dst = src.with_extension(&format!("backup.{}.broken", num));
num += 1;
}
2017-04-13 05:41:18 +00:00
if fs::rename(&src, &dst).is_err() {
try!(fs::copy(&src, &dst));
try!(fs::remove_file(&src));
}
info!("The original backup was renamed to {:?}", dst);
2017-03-22 10:10:13 +00:00
Ok(())
}
2017-04-10 18:35:28 +00:00
#[inline]
2017-04-13 05:41:18 +00:00
pub fn check_backup(&mut self, name: &str, backup: &mut Backup, repair: bool) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.write_mode());
Some(self.lock(false))
} else {
None
};
info!("Checking backup...");
2017-03-22 10:10:13 +00:00
let mut checked = Bitmap::new(self.index.capacity());
match self.check_subtree(Path::new("").to_path_buf(), &backup.root, &mut checked, repair) {
Ok(None) => (),
Ok(Some(chunks)) => {
try!(self.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(&name));
try!(self.save_backup(&backup, &name));
},
2017-04-16 21:18:25 +00:00
Err(err) => if repair {
warn!("The root of the backup {} has been corrupted\n\tcaused by: {}", name, err);
try!(self.evacuate_broken_backup(&name));
2017-04-16 21:18:25 +00:00
} else {
return Err(err)
}
2017-04-13 05:41:18 +00:00
}
Ok(())
2017-03-22 10:10:13 +00:00
}
2017-04-13 05:41:18 +00:00
pub fn check_backup_inode(&mut self, name: &str, backup: &mut Backup, path: &Path, repair: bool) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.write_mode());
Some(self.lock(false))
} else {
None
};
info!("Checking inode...");
2017-03-22 10:10:13 +00:00
let mut checked = Bitmap::new(self.index.capacity());
2017-04-13 05:41:18 +00:00
let mut inodes = try!(self.get_backup_path(&backup, path));
let mut inode = inodes.pop().unwrap();
let mut modified = false;
if let Err(err) = self.check_inode_contents(&inode, &mut checked) {
if repair {
warn!("Problem detected: data of {:?} is corrupt\n\tcaused by: {}", path, err);
info!("Removing inode data");
inode.data = Some(FileData::Inline(vec![].into()));
inode.size = 0;
modified = true;
} else {
return Err(IntegrityError::MissingInodeData(path.to_path_buf(), Box::new(err)).into())
2017-03-22 10:10:13 +00:00
}
}
2017-04-13 05:41:18 +00:00
if let Some(ref mut children) = inode.children {
let mut removed = vec![];
for (name, chunks) in children.iter_mut() {
match self.check_subtree(path.join(name), chunks, &mut checked, repair) {
Ok(None) => (),
Ok(Some(c)) => {
*chunks = c;
modified = true;
},
Err(err) => if repair {
warn!("Problem detected: inode {:?} is corrupt\n\tcaused by: {}", path.join(name), err);
info!("Removing broken inode from backup");
removed.push(name.to_string());
modified = true;
} else {
return Err(err)
2017-04-13 05:41:18 +00:00
}
}
}
for name in removed {
children.remove(&name);
}
}
let mut chunks = try!(self.put_inode(&inode));
while let Some(mut parent) = inodes.pop() {
parent.children.as_mut().unwrap().insert(inode.name, chunks);
inode = parent;
chunks = try!(self.put_inode(&inode));
}
if modified {
try!(self.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(&name));
try!(self.save_backup(&backup, &name));
}
2017-03-22 10:10:13 +00:00
Ok(())
}
2017-04-13 05:41:18 +00:00
pub fn check_backups(&mut self, repair: bool) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.write_mode());
Some(self.lock(false))
} else {
None
};
info!("Checking backups...");
2017-03-20 13:03:29 +00:00
let mut checked = Bitmap::new(self.index.capacity());
2017-04-13 11:32:59 +00:00
let backup_map = match self.get_all_backups() {
2017-03-22 08:19:16 +00:00
Ok(backup_map) => backup_map,
Err(RepositoryError::BackupFile(BackupFileError::PartialBackupsList(backup_map, _failed))) => {
warn!("Some backups could not be read, ignoring them");
backup_map
},
Err(err) => return Err(err)
};
2017-04-13 05:41:18 +00:00
for (name, mut backup) in ProgressIter::new("ckecking backups", backup_map.len(), backup_map.into_iter()) {
let path = format!("{}::", name);
match self.check_subtree(Path::new(&path).to_path_buf(), &backup.root, &mut checked, repair) {
Ok(None) => (),
Ok(Some(chunks)) => {
try!(self.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(&name));
try!(self.save_backup(&backup, &name));
},
2017-04-16 21:18:25 +00:00
Err(err) => if repair {
warn!("The root of the backup {} has been corrupted\n\tcaused by: {}", name, err);
try!(self.evacuate_broken_backup(&name));
2017-04-16 21:18:25 +00:00
} else {
return Err(err)
}
2017-04-13 05:41:18 +00:00
}
2017-03-20 13:03:29 +00:00
}
Ok(())
}
2017-04-13 05:46:16 +00:00
pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> {
info!("Checking repository integrity...");
2017-04-13 05:46:16 +00:00
let mut rebuild = false;
2017-04-10 16:21:26 +00:00
for (_id, bundle_id) in self.bundle_map.bundles() {
if self.bundles.get_bundle_info(&bundle_id).is_none() {
2017-04-13 05:46:16 +00:00
if repair {
warn!("Problem detected: bundle map contains unknown bundle {}", bundle_id);
rebuild = true;
} else {
return Err(IntegrityError::MissingBundle(bundle_id).into())
}
2017-04-10 16:21:26 +00:00
}
}
if self.bundle_map.len() < self.bundles.len() {
2017-04-13 05:46:16 +00:00
if repair {
warn!("Problem detected: bundle map does not contain all remote bundles");
rebuild = true;
} else {
return Err(IntegrityError::RemoteBundlesNotInMap.into())
}
2017-04-10 16:21:26 +00:00
}
if self.bundle_map.len() > self.bundles.len() {
2017-04-13 05:46:16 +00:00
if repair {
warn!("Problem detected: bundle map contains bundles multiple times");
rebuild = true;
} else {
return Err(IntegrityError::MapContainsDuplicates.into())
}
}
if rebuild {
try!(self.rebuild_bundle_map());
try!(self.rebuild_index());
2017-04-10 16:21:26 +00:00
}
Ok(())
}
2017-04-12 18:19:21 +00:00
pub fn rebuild_bundle_map(&mut self) -> Result<(), RepositoryError> {
info!("Rebuilding bundle map from bundles");
self.bundle_map = BundleMap::create();
for bundle in self.bundles.list_bundles() {
let bundle_id = match bundle.mode {
BundleMode::Data => self.next_data_bundle,
BundleMode::Meta => self.next_meta_bundle
};
self.bundle_map.set(bundle_id, bundle.id.clone());
if self.next_meta_bundle == bundle_id {
self.next_meta_bundle = self.next_free_bundle_id()
}
if self.next_data_bundle == bundle_id {
self.next_data_bundle = self.next_free_bundle_id()
}
}
self.save_bundle_map()
}
pub fn rebuild_index(&mut self) -> Result<(), RepositoryError> {
info!("Rebuilding index from bundles");
self.index.clear();
let mut bundles = self.bundle_map.bundles();
bundles.sort_by_key(|&(_, ref v)| v.clone());
for (num, id) in bundles {
2017-04-12 18:19:21 +00:00
let chunks = try!(self.bundles.get_chunk_list(&id));
for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() {
try!(self.index.set(&hash, &Location{bundle: num as u32, chunk: i as u32}));
}
}
Ok(())
}
2017-04-10 18:35:28 +00:00
#[inline]
2017-04-12 18:19:21 +00:00
pub fn check_index(&mut self, repair: bool) -> Result<(), RepositoryError> {
if repair {
try!(self.write_mode());
}
2017-03-20 13:03:29 +00:00
info!("Checking index integrity...");
2017-04-12 18:19:21 +00:00
if let Err(err) = self.index.check() {
if repair {
warn!("Problem detected: index was corrupted\n\tcaused by: {}", err);
return self.rebuild_index();
} else {
return Err(err.into())
}
}
2017-04-10 16:21:26 +00:00
info!("Checking index entries...");
2017-04-12 18:19:21 +00:00
if let Err(err) = self.check_index_chunks() {
if repair {
warn!("Problem detected: index entries were inconsistent\n\tcaused by: {}", err);
return self.rebuild_index();
} else {
return Err(err.into())
}
}
Ok(())
}
2017-04-10 18:35:28 +00:00
#[inline]
2017-04-12 18:19:21 +00:00
pub fn check_bundles(&mut self, full: bool, repair: bool) -> Result<(), RepositoryError> {
if repair {
try!(self.write_mode());
}
info!("Checking bundle integrity...");
2017-04-12 18:19:21 +00:00
if try!(self.bundles.check(full, repair)) {
// Some bundles got repaired
2017-04-13 05:41:18 +00:00
warn!("Some bundles have been rewritten, please remove the broken bundles manually.");
2017-04-12 18:19:21 +00:00
try!(self.bundles.finish_uploads());
try!(self.rebuild_bundle_map());
try!(self.rebuild_index());
}
Ok(())
2017-03-20 13:03:29 +00:00
}
2017-03-10 11:43:32 +00:00
}