dswd
/
zvault
mirror of https://github.com/dswd/zvault
1
0
Fork 0
Code Issues Releases Wiki Activity

First stage of refactoring complete

This commit is contained in:
Dennis Schwerdel 2018-08-07 11:31:50 +02:00
parent 31c6650374
commit 46e94bc7a6
13 changed files with 834 additions and 645 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

241
src/backups/backup.rs
View File

@ -39,55 +39,83 @@ pub enum DiffType {
} }
impl BackupRepository { pub trait RepositoryBackupIO {
pub fn get_all_backups(&self) -> Result<HashMap<String, BackupFile>, RepositoryError> {
Ok(try!(BackupFile::get_all_from( fn get_all_backups(&self) -> Result<HashMap<String, BackupFile>, RepositoryError>;
&self.crypto, fn get_backups<P: AsRef<Path>>(&self, path: P
self.layout.backups_path() ) -> Result<HashMap<String, BackupFile>, RepositoryError>;
))) fn has_backup(&self, name: &str) -> bool;
fn get_backup(&self, name: &str) -> Result<BackupFile, RepositoryError>;
fn save_backup(&mut self, backup: &BackupFile, name: &str, lock: &BackupMode
) -> Result<(), RepositoryError>;
fn delete_backup(&mut self, name: &str, lock: &BackupMode) -> Result<(), RepositoryError>;
fn prune_backups(&mut self, prefix: &str, daily: usize, weekly: usize, monthly: usize,
yearly: usize, force: bool, lock: &BackupMode) -> Result<(), RepositoryError>;
fn restore_inode_tree<P: AsRef<Path>>(&mut self, backup: &BackupFile, inode: Inode, path: P,
lock: &OnlineMode) -> Result<(), RepositoryError>;
fn create_backup_recurse<P: AsRef<Path>>(&mut self, path: P, reference: Option<&Inode>,
options: &BackupOptions, backup: &mut BackupFile, failed_paths: &mut Vec<PathBuf>,
lock: &BackupMode) -> Result<Inode, RepositoryError>;
fn create_backup<P: AsRef<Path>>(&mut self, path: P, name: &str,
reference: Option<&BackupFile>, options: &BackupOptions, lock: &BackupMode
) -> Result<BackupFile, RepositoryError>;
fn remove_backup_path<P: AsRef<Path>>(&mut self, backup: &mut BackupFile, path: P,
lock: &BackupMode) -> Result<(), RepositoryError>;
fn get_backup_path<P: AsRef<Path>>(&mut self, backup: &BackupFile, path: P, lock: &OnlineMode
) -> Result<Vec<Inode>, RepositoryError>;
fn get_backup_inode<P: AsRef<Path>>(&mut self, backup: &BackupFile, path: P, lock: &OnlineMode
) -> Result<Inode, RepositoryError>;
fn find_versions<P: AsRef<Path>>(&mut self, path: P, lock: &OnlineMode
) -> Result<Vec<(String, Inode)>, RepositoryError>;
fn find_differences_recurse(&mut self, inode1: &Inode, inode2: &Inode, path: PathBuf,
diffs: &mut Vec<(DiffType, PathBuf)>, lock: &OnlineMode) -> Result<(), RepositoryError>;
fn find_differences(&mut self, inode1: &Inode, inode2: &Inode, lock: &OnlineMode
) -> Result<Vec<(DiffType, PathBuf)>, RepositoryError>;
fn count_sizes_recursive(&mut self, inode: &Inode, sizes: &mut HashMap<u64, usize>,
min_size: u64, lock: &OnlineMode) -> Result<(), RepositoryError>;
fn find_duplicates_recursive(&mut self, inode: &Inode, path: &Path, sizes: &HashMap<u64, usize>,
hashes: &mut HashMap<Hash, (Vec<PathBuf>, u64)>, lock: &OnlineMode
) -> Result<(), RepositoryError>;
fn find_duplicates(&mut self, inode: &Inode, min_size: u64, lock: &OnlineMode
) -> Result<Vec<(Vec<PathBuf>, u64)>, RepositoryError>;
}
impl RepositoryBackupIO for Repository {
fn get_all_backups(&self) -> Result<HashMap<String, BackupFile>, RepositoryError> {
Ok(try!(BackupFile::get_all_from(&self.get_crypto(),self.get_layout().backups_path())))
} }
pub fn get_backups<P: AsRef<Path>>( fn get_backups<P: AsRef<Path>>(&self, path: P) -> Result<HashMap<String, BackupFile>, RepositoryError> {
&self, Ok(try!(BackupFile::get_all_from(&self.get_crypto(), self.get_layout().backups_path().join(path))))
path: P,
) -> Result<HashMap<String, BackupFile>, RepositoryError> {
Ok(try!(BackupFile::get_all_from(
&self.crypto,
self.layout.backups_path().join(path)
)))
} }
#[inline] #[inline]
pub fn has_backup(&self, name: &str) -> bool { fn has_backup(&self, name: &str) -> bool {
self.layout.backup_path(name).exists() self.get_layout().backup_path(name).exists()
} }
pub fn get_backup(&self, name: &str) -> Result<BackupFile, RepositoryError> { fn get_backup(&self, name: &str) -> Result<BackupFile, RepositoryError> {
Ok(try!(BackupFile::read_from( Ok(try!(BackupFile::read_from(&self.get_crypto(), self.get_layout().backup_path(name))))
&self.crypto,
self.layout.backup_path(name)
)))
} }
pub fn save_backup(&mut self, backup: &BackupFile, name: &str) -> Result<(), RepositoryError> { fn save_backup(&mut self, backup: &BackupFile, name: &str, lock: &BackupMode) -> Result<(), RepositoryError> {
try!(self.repo.write_mode()); let path = self.get_layout().backup_path(name);
let path = self.layout.backup_path(name);
try!(fs::create_dir_all(path.parent().unwrap())); try!(fs::create_dir_all(path.parent().unwrap()));
try!(backup.save_to( try!(backup.save_to(
&self.crypto, &self.get_crypto(),
self.get_config().encryption.clone(), self.get_config().encryption.clone(),
path path
)); ));
Ok(()) Ok(())
} }
pub fn delete_backup(&mut self, name: &str) -> Result<(), RepositoryError> { fn delete_backup(&mut self, name: &str, lock: &BackupMode) -> Result<(), RepositoryError> {
try!(self.repo.write_mode()); let mut path = self.get_layout().backup_path(name);
let mut path = self.layout.backup_path(name);
try!(fs::remove_file(&path)); try!(fs::remove_file(&path));
loop { loop {
path = path.parent().unwrap().to_owned(); path = path.parent().unwrap().to_owned();
if path == self.layout.backups_path() || fs::remove_dir(&path).is_err() { if path == self.get_layout().backups_path() || fs::remove_dir(&path).is_err() {
break; break;
} }
} }
@ -95,16 +123,9 @@ impl BackupRepository {
} }
pub fn prune_backups( fn prune_backups(&mut self, prefix: &str, daily: usize, weekly: usize, monthly: usize,
&mut self, yearly: usize, force: bool, lock: &BackupMode) -> Result<(), RepositoryError>
prefix: &str, {
daily: usize,
weekly: usize,
monthly: usize,
yearly: usize,
force: bool,
) -> Result<(), RepositoryError> {
try!(self.repo.write_mode());
let mut backups = Vec::new(); let mut backups = Vec::new();
let backup_map = match self.get_all_backups() { let backup_map = match self.get_all_backups() {
Ok(backup_map) => backup_map, Ok(backup_map) => backup_map,
@ -175,19 +196,13 @@ impl BackupRepository {
} }
if force { if force {
for name in remove { for name in remove {
try!(self.delete_backup(&name)); try!(self.delete_backup(&name, lock));
} }
} }
Ok(()) Ok(())
} }
pub fn restore_inode_tree<P: AsRef<Path>>( fn restore_inode_tree<P: AsRef<Path>>(&mut self, backup: &BackupFile, inode: Inode, path: P, lock: &OnlineMode) -> Result<(), RepositoryError> {
&mut self,
backup: &BackupFile,
inode: Inode,
path: P,
) -> Result<(), RepositoryError> {
let _lock = try!(self.repo.lock(false));
let mut queue = VecDeque::new(); let mut queue = VecDeque::new();
queue.push_back((path.as_ref().to_owned(), inode)); queue.push_back((path.as_ref().to_owned(), inode));
let cache = users::UsersCache::new(); let cache = users::UsersCache::new();
@ -204,7 +219,7 @@ impl BackupRepository {
inode.group = group.gid(); inode.group = group.gid();
} }
} }
try!(self.save_inode_at(&inode, &path)); try!(self.save_inode_at(&inode, &path, lock));
} }
if inode.file_type == FileType::Directory { if inode.file_type == FileType::Directory {
let path = if is_root { let path = if is_root {
@ -213,7 +228,7 @@ impl BackupRepository {
path.join(inode.name) path.join(inode.name)
}; };
for chunks in inode.children.unwrap().values() { for chunks in inode.children.unwrap().values() {
let inode = try!(self.get_inode(chunks)); let inode = try!(self.get_inode(chunks, lock));
queue.push_back((path.clone(), inode)); queue.push_back((path.clone(), inode));
} }
} }
@ -222,16 +237,12 @@ impl BackupRepository {
Ok(()) Ok(())
} }
pub fn create_backup_recurse<P: AsRef<Path>>( fn create_backup_recurse<P: AsRef<Path>>(&mut self, path: P, reference: Option<&Inode>,
&mut self, options: &BackupOptions, backup: &mut BackupFile, failed_paths: &mut Vec<PathBuf>,
path: P, lock: &BackupMode
reference: Option<&Inode>,
options: &BackupOptions,
backup: &mut BackupFile,
failed_paths: &mut Vec<PathBuf>,
) -> Result<Inode, RepositoryError> { ) -> Result<Inode, RepositoryError> {
let path = path.as_ref(); let path = path.as_ref();
let mut inode = try!(self.create_inode(path, reference)); let mut inode = try!(self.create_inode(path, reference, lock));
if !backup.user_names.contains_key(&inode.user) { if !backup.user_names.contains_key(&inode.user) {
if let Some(user) = users::get_user_by_uid(inode.user) { if let Some(user) = users::get_user_by_uid(inode.user) {
backup.user_names.insert( backup.user_names.insert(
@ -278,13 +289,14 @@ impl BackupRepository {
.as_ref() .as_ref()
.and_then(|inode| inode.children.as_ref()) .and_then(|inode| inode.children.as_ref())
.and_then(|map| map.get(&name)) .and_then(|map| map.get(&name))
.and_then(|chunks| self.get_inode(chunks).ok()); .and_then(|chunks| self.get_inode(chunks, lock.as_online()).ok());
let child_inode = match self.create_backup_recurse( let child_inode = match self.create_backup_recurse(
&child_path, &child_path,
ref_child.as_ref(), ref_child.as_ref(),
options, options,
backup, backup,
failed_paths failed_paths,
lock
) { ) {
Ok(inode) => inode, Ok(inode) => inode,
Err(RepositoryError::Inode(_)) | Err(RepositoryError::Inode(_)) |
@ -296,7 +308,7 @@ impl BackupRepository {
} }
Err(err) => return Err(err), Err(err) => return Err(err),
}; };
let chunks = try!(self.put_inode(&child_inode)); let chunks = try!(self.put_inode(&child_inode, lock));
inode.cum_size += child_inode.cum_size; inode.cum_size += child_inode.cum_size;
for &(_, len) in chunks.iter() { for &(_, len) in chunks.iter() {
meta_size += u64::from(len); meta_size += u64::from(len);
@ -325,19 +337,10 @@ impl BackupRepository {
Ok(inode) Ok(inode)
} }
pub fn create_backup_recursively<P: AsRef<Path>>( fn create_backup<P: AsRef<Path>>(&mut self, path: P, name: &str,
&mut self, reference: Option<&BackupFile>, options: &BackupOptions, lock: &BackupMode
path: P,
reference: Option<&BackupFile>,
options: &BackupOptions,
) -> Result<BackupFile, RepositoryError> { ) -> Result<BackupFile, RepositoryError> {
try!(self.repo.write_mode()); let reference_inode = reference.and_then(|b| self.get_inode(&b.root, lock.as_online()).ok());
let _lock = try!(self.repo.lock(false));
if self.repo.is_dirty() {
return Err(RepositoryError::Dirty);
}
try!(self.repo.set_dirty());
let reference_inode = reference.and_then(|b| self.get_inode(&b.root).ok());
let mut backup = BackupFile::default(); let mut backup = BackupFile::default();
backup.config = self.get_config().clone(); backup.config = self.get_config().clone();
backup.host = get_hostname().unwrap_or_else(|_| "".to_string()); backup.host = get_hostname().unwrap_or_else(|_| "".to_string());
@ -350,10 +353,11 @@ impl BackupRepository {
reference_inode.as_ref(), reference_inode.as_ref(),
options, options,
&mut backup, &mut backup,
&mut failed_paths &mut failed_paths,
lock
)); ));
backup.root = try!(self.put_inode(&root_inode)); backup.root = try!(self.put_inode(&root_inode, lock));
try!(self.repo.flush()); try!(self.flush(lock));
let elapsed = Local::now().signed_duration_since(start); let elapsed = Local::now().signed_duration_since(start);
backup.timestamp = start.timestamp(); backup.timestamp = start.timestamp();
backup.total_data_size = root_inode.cum_size; backup.total_data_size = root_inode.cum_size;
@ -369,7 +373,7 @@ impl BackupRepository {
backup.bundle_count = info_after.bundle_count - info_before.bundle_count; backup.bundle_count = info_after.bundle_count - info_before.bundle_count;
backup.chunk_count = info_after.chunk_count - info_before.chunk_count; backup.chunk_count = info_after.chunk_count - info_before.chunk_count;
backup.avg_chunk_size = backup.deduplicated_data_size as f32 / backup.chunk_count as f32; backup.avg_chunk_size = backup.deduplicated_data_size as f32 / backup.chunk_count as f32;
self.repo.set_clean(); try!(self.save_backup(&backup, name, lock));
if failed_paths.is_empty() { if failed_paths.is_empty() {
Ok(backup) Ok(backup)
} else { } else {
@ -377,14 +381,10 @@ impl BackupRepository {
} }
} }
pub fn remove_backup_path<P: AsRef<Path>>( fn remove_backup_path<P: AsRef<Path>>(&mut self, backup: &mut BackupFile, path: P,
&mut self, lock: &BackupMode
backup: &mut BackupFile,
path: P,
) -> Result<(), RepositoryError> { ) -> Result<(), RepositoryError> {
try!(self.repo.write_mode()); let mut inodes = try!(self.get_backup_path(backup, path, lock.as_online()));
let _lock = try!(self.repo.lock(false));
let mut inodes = try!(self.get_backup_path(backup, path));
let to_remove = inodes.pop().unwrap(); let to_remove = inodes.pop().unwrap();
let mut remove_from = match inodes.pop() { let mut remove_from = match inodes.pop() {
Some(inode) => inode, Some(inode) => inode,
@ -393,28 +393,26 @@ impl BackupRepository {
remove_from.children.as_mut().unwrap().remove( remove_from.children.as_mut().unwrap().remove(
&to_remove.name &to_remove.name
); );
let mut last_inode_chunks = try!(self.put_inode(&remove_from)); let mut last_inode_chunks = try!(self.put_inode(&remove_from, lock));
let mut last_inode_name = remove_from.name; let mut last_inode_name = remove_from.name;
while let Some(mut inode) = inodes.pop() { while let Some(mut inode) = inodes.pop() {
inode.children.as_mut().unwrap().insert( inode.children.as_mut().unwrap().insert(
last_inode_name, last_inode_name,
last_inode_chunks last_inode_chunks
); );
last_inode_chunks = try!(self.put_inode(&inode)); last_inode_chunks = try!(self.put_inode(&inode, lock));
last_inode_name = inode.name; last_inode_name = inode.name;
} }
backup.root = last_inode_chunks; backup.root = last_inode_chunks;
backup.modified = true; backup.modified = true;
//TODO: save
Ok(()) Ok(())
} }
pub fn get_backup_path<P: AsRef<Path>>( fn get_backup_path<P: AsRef<Path>>(&mut self, backup: &BackupFile, path: P, lock: &OnlineMode
&mut self,
backup: &BackupFile,
path: P,
) -> Result<Vec<Inode>, RepositoryError> { ) -> Result<Vec<Inode>, RepositoryError> {
let mut inodes = vec![]; let mut inodes = vec![];
let mut inode = try!(self.get_inode(&backup.root)); let mut inode = try!(self.get_inode(&backup.root, lock));
for c in path.as_ref().components() { for c in path.as_ref().components() {
if let path::Component::Normal(name) = c { if let path::Component::Normal(name) = c {
let name = name.to_string_lossy(); let name = name.to_string_lossy();
@ -428,7 +426,7 @@ impl BackupRepository {
) )
{ {
inodes.push(inode); inodes.push(inode);
inode = try!(self.get_inode(&chunks)); inode = try!(self.get_inode(&chunks, lock));
} else { } else {
return Err(RepositoryError::NoSuchFileInBackup( return Err(RepositoryError::NoSuchFileInBackup(
backup.clone(), backup.clone(),
@ -442,24 +440,19 @@ impl BackupRepository {
} }
#[inline] #[inline]
pub fn get_backup_inode<P: AsRef<Path>>( fn get_backup_inode<P: AsRef<Path>>(&mut self, backup: &BackupFile, path: P, lock: &OnlineMode
&mut self,
backup: &BackupFile,
path: P,
) -> Result<Inode, RepositoryError> { ) -> Result<Inode, RepositoryError> {
self.get_backup_path(backup, path).map(|mut inodes| { self.get_backup_path(backup, path, lock).map(|mut inodes| {
inodes.pop().unwrap() inodes.pop().unwrap()
}) })
} }
pub fn find_versions<P: AsRef<Path>>( fn find_versions<P: AsRef<Path>>(&mut self, path: P, lock: &OnlineMode
&mut self,
path: P,
) -> Result<Vec<(String, Inode)>, RepositoryError> { ) -> Result<Vec<(String, Inode)>, RepositoryError> {
let path = path.as_ref(); let path = path.as_ref();
let mut versions = HashMap::new(); let mut versions = HashMap::new();
for (name, backup) in try!(self.get_all_backups()) { for (name, backup) in try!(self.get_all_backups()) {
match self.get_backup_inode(&backup, path) { match self.get_backup_inode(&backup, path, lock) {
Ok(inode) => { Ok(inode) => {
versions.insert( versions.insert(
(inode.file_type, inode.timestamp, inode.size), (inode.file_type, inode.timestamp, inode.size),
@ -476,12 +469,8 @@ impl BackupRepository {
} }
#[allow(needless_pass_by_value)] #[allow(needless_pass_by_value)]
fn find_differences_recurse( fn find_differences_recurse(&mut self, inode1: &Inode, inode2: &Inode, path: PathBuf,
&mut self, diffs: &mut Vec<(DiffType, PathBuf)>, lock: &OnlineMode
inode1: &Inode,
inode2: &Inode,
path: PathBuf,
diffs: &mut Vec<(DiffType, PathBuf)>,
) -> Result<(), RepositoryError> { ) -> Result<(), RepositoryError> {
if !inode1.is_same_meta(inode2) || inode1.data != inode2.data { if !inode1.is_same_meta(inode2) || inode1.data != inode2.data {
diffs.push((DiffType::Mod, path.clone())); diffs.push((DiffType::Mod, path.clone()));
@ -504,13 +493,14 @@ impl BackupRepository {
for (name, chunks2) in children2 { for (name, chunks2) in children2 {
if let Some(chunks1) = children1.get(name) { if let Some(chunks1) = children1.get(name) {
if chunks1 != chunks2 { if chunks1 != chunks2 {
let inode1 = try!(self.get_inode(chunks1)); let inode1 = try!(self.get_inode(chunks1, lock));
let inode2 = try!(self.get_inode(chunks2)); let inode2 = try!(self.get_inode(chunks2, lock));
try!(self.find_differences_recurse( try!(self.find_differences_recurse(
&inode1, &inode1,
&inode2, &inode2,
path.join(name), path.join(name),
diffs diffs,
lock
)); ));
} }
} else { } else {
@ -527,10 +517,7 @@ impl BackupRepository {
} }
#[inline] #[inline]
pub fn find_differences( fn find_differences(&mut self, inode1: &Inode, inode2: &Inode, lock: &OnlineMode
&mut self,
inode1: &Inode,
inode2: &Inode,
) -> Result<Vec<(DiffType, PathBuf)>, RepositoryError> { ) -> Result<Vec<(DiffType, PathBuf)>, RepositoryError> {
let mut diffs = vec![]; let mut diffs = vec![];
let path = PathBuf::from("/"); let path = PathBuf::from("/");
@ -538,25 +525,30 @@ impl BackupRepository {
inode1, inode1,
inode2, inode2,
path, path,
&mut diffs &mut diffs,
lock
)); ));
Ok(diffs) Ok(diffs)
} }
fn count_sizes_recursive(&mut self, inode: &Inode, sizes: &mut HashMap<u64, usize>, min_size: u64) -> Result<(), RepositoryError> { fn count_sizes_recursive(&mut self, inode: &Inode, sizes: &mut HashMap<u64, usize>,
min_size: u64, lock: &OnlineMode
) -> Result<(), RepositoryError> {
if inode.size >= min_size { if inode.size >= min_size {
*sizes.entry(inode.size).or_insert(0) += 1; *sizes.entry(inode.size).or_insert(0) += 1;
} }
if let Some(ref children) = inode.children { if let Some(ref children) = inode.children {
for chunks in children.values() { for chunks in children.values() {
let ch = try!(self.get_inode(chunks)); let ch = try!(self.get_inode(chunks, lock));
try!(self.count_sizes_recursive(&ch, sizes, min_size)); try!(self.count_sizes_recursive(&ch, sizes, min_size, lock));
} }
} }
Ok(()) Ok(())
} }
fn find_duplicates_recursive(&mut self, inode: &Inode, path: &Path, sizes: &HashMap<u64, usize>, hashes: &mut HashMap<Hash, (Vec<PathBuf>, u64)>) -> Result<(), RepositoryError> { fn find_duplicates_recursive(&mut self, inode: &Inode, path: &Path, sizes: &HashMap<u64, usize>,
hashes: &mut HashMap<Hash, (Vec<PathBuf>, u64)>, lock: &OnlineMode
) -> Result<(), RepositoryError> {
let path = path.join(&inode.name); let path = path.join(&inode.name);
if sizes.get(&inode.size).cloned().unwrap_or(0) > 1 { if sizes.get(&inode.size).cloned().unwrap_or(0) > 1 {
if let Some(ref data) = inode.data { if let Some(ref data) = inode.data {
@ -567,21 +559,22 @@ impl BackupRepository {
} }
if let Some(ref children) = inode.children { if let Some(ref children) = inode.children {
for chunks in children.values() { for chunks in children.values() {
let ch = try!(self.get_inode(chunks)); let ch = try!(self.get_inode(chunks, lock));
try!(self.find_duplicates_recursive(&ch, &path, sizes, hashes)); try!(self.find_duplicates_recursive(&ch, &path, sizes, hashes, lock));
} }
} }
Ok(()) Ok(())
} }
pub fn find_duplicates(&mut self, inode: &Inode, min_size: u64) -> Result<Vec<(Vec<PathBuf>, u64)>, RepositoryError> { fn find_duplicates(&mut self, inode: &Inode, min_size: u64, lock: &OnlineMode
) -> Result<Vec<(Vec<PathBuf>, u64)>, RepositoryError> {
let mut sizes = HashMap::new(); let mut sizes = HashMap::new();
try!(self.count_sizes_recursive(inode, &mut sizes, min_size)); try!(self.count_sizes_recursive(inode, &mut sizes, min_size, lock));
let mut hashes = HashMap::new(); let mut hashes = HashMap::new();
if let Some(ref children) = inode.children { if let Some(ref children) = inode.children {
for chunks in children.values() { for chunks in children.values() {
let ch = try!(self.get_inode(chunks)); let ch = try!(self.get_inode(chunks, lock));
try!(self.find_duplicates_recursive(&ch, Path::new(""), &sizes, &mut hashes)); try!(self.find_duplicates_recursive(&ch, Path::new(""), &sizes, &mut hashes, lock));
} }
} }
let dups = hashes.into_iter().map(|(_,v)| v).filter(|&(ref v, _)| v.len() > 1).collect(); let dups = hashes.into_iter().map(|(_,v)| v).filter(|&(ref v, _)| v.len() > 1).collect();

325
src/backups/integrity.rs
View File

@ -21,47 +21,98 @@ quick_error!{
} }
} }
impl BackupRepository {
fn check_inode_contents( pub trait RepositoryIntegrityIO {
&mut self, fn check_inode_contents(&mut self, inode: &Inode, checked: &mut Bitmap, lock: &OnlineMode
inode: &Inode, ) -> Result<(), RepositoryError>;
checked: &mut Bitmap,
fn check_subtree(&mut self, path: PathBuf, chunks: &[Chunk], checked: &mut Bitmap,
lock: &OnlineMode
) -> Result<(), RepositoryError>;
fn check_and_repair_subtree(&mut self, path: PathBuf, chunks: &[Chunk], checked: &mut Bitmap,
lock: &BackupMode
) -> Result<Option<ChunkList>, RepositoryError>;
fn evacuate_broken_backup(&self, name: &str, lock: &BackupMode) -> Result<(), RepositoryError>;
fn check_backup_inode(&mut self, name: &str, backup: &mut BackupFile, path: &Path,
lock: &OnlineMode
) -> Result<(), RepositoryError>;
fn check_and_repair_backup_inode(&mut self, name: &str, backup: &mut BackupFile, path: &Path,
lock: &BackupMode,
) -> Result<(), RepositoryError>;
fn check_backup(&mut self, name: &str, backup: &mut BackupFile, lock: &OnlineMode
) -> Result<(), RepositoryError>;
fn check_and_repair_backup(&mut self, name: &str, backup: &mut BackupFile, lock: &BackupMode
) -> Result<(), RepositoryError>;
fn check_backups(&mut self, lock: &OnlineMode) -> Result<(), RepositoryError>;
fn check_and_repair_backups(&mut self, lock: &BackupMode) -> Result<(), RepositoryError>;
}
impl RepositoryIntegrityIO for Repository {
fn check_inode_contents(&mut self, inode: &Inode, checked: &mut Bitmap, lock: &OnlineMode
) -> Result<(), RepositoryError> { ) -> Result<(), RepositoryError> {
match inode.data { match inode.data {
None | None |
Some(FileData::Inline(_)) => (), Some(FileData::Inline(_)) => (),
Some(FileData::ChunkedDirect(ref chunks)) => { Some(FileData::ChunkedDirect(ref chunks)) => {
try!(self.repo.mark_chunks(checked, chunks, true)); try!(self.mark_chunks(checked, chunks, true));
} }
Some(FileData::ChunkedIndirect(ref chunks)) => { Some(FileData::ChunkedIndirect(ref chunks)) => {
if try!(self.repo.mark_chunks(checked, chunks, false)) { if try!(self.mark_chunks(checked, chunks, false)) {
let chunk_data = try!(self.get_data(chunks)); let chunk_data = try!(self.get_data(chunks, lock));
let chunks2 = ChunkList::read_from(&chunk_data); let chunks2 = ChunkList::read_from(&chunk_data);
try!(self.repo.mark_chunks(checked, &chunks2, true)); try!(self.mark_chunks(checked, &chunks2, true));
try!(self.repo.mark_chunks(checked, chunks, true)); try!(self.mark_chunks(checked, chunks, true));
} }
} }
} }
Ok(()) Ok(())
} }
fn check_subtree( fn check_subtree(&mut self, path: PathBuf, chunks: &[Chunk], checked: &mut Bitmap,
&mut self, lock: &OnlineMode,
path: PathBuf, ) -> Result<(), RepositoryError> {
chunks: &[Chunk], let mut modified = false;
checked: &mut Bitmap, match self.mark_chunks(checked, chunks, false) {
repair: bool, Ok(false) => return Ok(()),
Ok(true) => (),
Err(err) => return Err(InodeIntegrityError::BrokenInode(path, Box::new(err)).into()),
}
let mut inode = try!(self.get_inode(chunks, lock));
// Mark the content chunks as used
if let Err(err) = self.check_inode_contents(&inode, checked, lock) {
return Err(InodeIntegrityError::MissingInodeData(path, Box::new(err)).into());
}
// Put children in to do
if let Some(ref mut children) = inode.children {
for (name, chunks) in children.iter_mut() {
try!(self.check_subtree(path.join(name), chunks, checked, lock));
}
}
try!(self.mark_chunks(checked, chunks, true));
Ok(())
}
fn check_and_repair_subtree(&mut self, path: PathBuf, chunks: &[Chunk], checked: &mut Bitmap,
lock: &BackupMode,
) -> Result<Option<ChunkList>, RepositoryError> { ) -> Result<Option<ChunkList>, RepositoryError> {
let mut modified = false; let mut modified = false;
match self.repo.mark_chunks(checked, chunks, false) { match self.mark_chunks(checked, chunks, false) {
Ok(false) => return Ok(None), Ok(false) => return Ok(None),
Ok(true) => (), Ok(true) => (),
Err(err) => return Err(InodeIntegrityError::BrokenInode(path, Box::new(err)).into()), Err(err) => return Err(InodeIntegrityError::BrokenInode(path, Box::new(err)).into()),
} }
let mut inode = try!(self.get_inode(chunks)); let mut inode = try!(self.get_inode(chunks, lock.as_online()));
// Mark the content chunks as used // Mark the content chunks as used
if let Err(err) = self.check_inode_contents(&inode, checked) { if let Err(err) = self.check_inode_contents(&inode, checked, lock.as_online()) {
if repair {
tr_warn!( tr_warn!(
"Problem detected: data of {:?} is corrupt\n\tcaused by: {}", "Problem detected: data of {:?} is corrupt\n\tcaused by: {}",
path, path,
@ -71,22 +122,18 @@ impl BackupRepository {
inode.data = Some(FileData::Inline(vec![].into())); inode.data = Some(FileData::Inline(vec![].into()));
inode.size = 0; inode.size = 0;
modified = true; modified = true;
} else {
return Err(InodeIntegrityError::MissingInodeData(path, Box::new(err)).into());
}
} }
// Put children in to do // Put children in to do
if let Some(ref mut children) = inode.children { if let Some(ref mut children) = inode.children {
let mut removed = vec![]; let mut removed = vec![];
for (name, chunks) in children.iter_mut() { for (name, chunks) in children.iter_mut() {
match self.check_subtree(path.join(name), chunks, checked, repair) { match self.check_and_repair_subtree(path.join(name), chunks, checked, lock) {
Ok(None) => (), Ok(None) => (),
Ok(Some(c)) => { Ok(Some(c)) => {
*chunks = c; *chunks = c;
modified = true; modified = true;
} }
Err(err) => { Err(err) => {
if repair {
tr_warn!( tr_warn!(
"Problem detected: inode {:?} is corrupt\n\tcaused by: {}", "Problem detected: inode {:?} is corrupt\n\tcaused by: {}",
path.join(name), path.join(name),
@ -95,9 +142,6 @@ impl BackupRepository {
tr_info!("Removing broken inode from backup"); tr_info!("Removing broken inode from backup");
removed.push(name.to_string()); removed.push(name.to_string());
modified = true; modified = true;
} else {
return Err(err);
}
} }
} }
} }
@ -106,19 +150,20 @@ impl BackupRepository {
} }
} }
if modified { if modified {
Ok(Some(try!(self.put_inode(&inode)))) Ok(Some(try!(self.put_inode(&inode, lock))))
} else { } else {
try!(self.repo.mark_chunks(checked, chunks, true)); try!(self.mark_chunks(checked, chunks, true));
Ok(None) Ok(None)
} }
} }
fn evacuate_broken_backup(&self, name: &str) -> Result<(), RepositoryError> {
fn evacuate_broken_backup(&self, name: &str, lock: &BackupMode) -> Result<(), RepositoryError> {
tr_warn!( tr_warn!(
"The backup {} was corrupted and needed to be modified.", "The backup {} was corrupted and needed to be modified.",
name name
); );
let src = self.layout.backup_path(name); let src = self.get_layout().backup_path(name);
let mut dst = src.with_extension("backup.broken"); let mut dst = src.with_extension("backup.broken");
let mut num = 1; let mut num = 1;
while dst.exists() { while dst.exists() {
@ -133,71 +178,36 @@ impl BackupRepository {
Ok(()) Ok(())
} }
#[inline] fn check_backup_inode(&mut self, name: &str, backup: &mut BackupFile, path: &Path,
pub fn check_backup( lock: &OnlineMode
&mut self,
name: &str,
backup: &mut BackupFile,
repair: bool,
) -> Result<(), RepositoryError> { ) -> Result<(), RepositoryError> {
let _lock = if repair { tr_info!("Checking inode...");
try!(self.repo.write_mode()); let mut checked = self.get_chunk_marker();
Some(self.repo.lock(false)) let mut inodes = try!(self.get_backup_path(backup, path, lock));
} else { let mut inode = inodes.pop().unwrap();
None let mut modified = false;
}; if let Err(err) = self.check_inode_contents(&inode, &mut checked, lock) {
tr_info!("Checking backup..."); return Err(
let mut checked = self.repo.get_chunk_marker(); InodeIntegrityError::MissingInodeData(path.to_path_buf(), Box::new(err)).into()
match self.check_subtree(
Path::new("").to_path_buf(),
&backup.root,
&mut checked,
repair
) {
Ok(None) => (),
Ok(Some(chunks)) => {
try!(self.repo.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(name));
try!(self.save_backup(backup, name));
}
Err(err) => {
if repair {
tr_warn!(
"The root of the backup {} has been corrupted\n\tcaused by: {}",
name,
err
); );
try!(self.evacuate_broken_backup(name));
} else {
return Err(err);
} }
if let Some(ref mut children) = inode.children {
for (name, chunks) in children.iter_mut() {
try!(self.check_subtree(path.join(name), chunks, &mut checked, lock));
} }
} }
Ok(()) Ok(())
} }
pub fn check_backup_inode( fn check_and_repair_backup_inode(&mut self, name: &str, backup: &mut BackupFile, path: &Path,
&mut self, lock: &BackupMode
name: &str,
backup: &mut BackupFile,
path: &Path,
repair: bool,
) -> Result<(), RepositoryError> { ) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.repo.write_mode());
Some(self.repo.lock(false))
} else {
None
};
tr_info!("Checking inode..."); tr_info!("Checking inode...");
let mut checked = self.repo.get_chunk_marker(); let mut checked = self.get_chunk_marker();
let mut inodes = try!(self.get_backup_path(backup, path)); let mut inodes = try!(self.get_backup_path(backup, path, lock.as_online()));
let mut inode = inodes.pop().unwrap(); let mut inode = inodes.pop().unwrap();
let mut modified = false; let mut modified = false;
if let Err(err) = self.check_inode_contents(&inode, &mut checked) { if let Err(err) = self.check_inode_contents(&inode, &mut checked, lock.as_online()) {
if repair {
tr_warn!( tr_warn!(
"Problem detected: data of {:?} is corrupt\n\tcaused by: {}", "Problem detected: data of {:?} is corrupt\n\tcaused by: {}",
path, path,
@ -207,23 +217,17 @@ impl BackupRepository {
inode.data = Some(FileData::Inline(vec![].into())); inode.data = Some(FileData::Inline(vec![].into()));
inode.size = 0; inode.size = 0;
modified = true; modified = true;
} else {
return Err(
InodeIntegrityError::MissingInodeData(path.to_path_buf(), Box::new(err)).into()
);
}
} }
if let Some(ref mut children) = inode.children { if let Some(ref mut children) = inode.children {
let mut removed = vec![]; let mut removed = vec![];
for (name, chunks) in children.iter_mut() { for (name, chunks) in children.iter_mut() {
match self.check_subtree(path.join(name), chunks, &mut checked, repair) { match self.check_and_repair_subtree(path.join(name), chunks, &mut checked, lock) {
Ok(None) => (), Ok(None) => (),
Ok(Some(c)) => { Ok(Some(c)) => {
*chunks = c; *chunks = c;
modified = true; modified = true;
} }
Err(err) => { Err(err) => {
if repair {
tr_warn!( tr_warn!(
"Problem detected: inode {:?} is corrupt\n\tcaused by: {}", "Problem detected: inode {:?} is corrupt\n\tcaused by: {}",
path.join(name), path.join(name),
@ -232,9 +236,6 @@ impl BackupRepository {
tr_info!("Removing broken inode from backup"); tr_info!("Removing broken inode from backup");
removed.push(name.to_string()); removed.push(name.to_string());
modified = true; modified = true;
} else {
return Err(err);
}
} }
} }
} }
@ -242,31 +243,62 @@ impl BackupRepository {
children.remove(&name); children.remove(&name);
} }
} }
let mut chunks = try!(self.put_inode(&inode)); if modified {
let mut chunks = try!(self.put_inode(&inode, lock));
while let Some(mut parent) = inodes.pop() { while let Some(mut parent) = inodes.pop() {
parent.children.as_mut().unwrap().insert(inode.name, chunks); parent.children.as_mut().unwrap().insert(inode.name, chunks);
inode = parent; inode = parent;
chunks = try!(self.put_inode(&inode)); chunks = try!(self.put_inode(&inode, lock));
} }
if modified { try!(self.flush(lock));
try!(self.repo.flush());
backup.root = chunks; backup.root = chunks;
backup.modified = true; backup.modified = true;
try!(self.evacuate_broken_backup(name)); try!(self.evacuate_broken_backup(name, lock));
try!(self.save_backup(backup, name)); try!(self.save_backup(backup, name, lock));
} }
Ok(()) Ok(())
} }
pub fn check_backups(&mut self, repair: bool) -> Result<(), RepositoryError> { #[inline]
let _lock = if repair { fn check_backup(&mut self, name: &str, backup: &mut BackupFile, lock: &OnlineMode,
try!(self.repo.write_mode()); ) -> Result<(), RepositoryError> {
Some(self.repo.lock(false)) tr_info!("Checking backup...");
} else { let mut checked = self.get_chunk_marker();
None try!(self.check_subtree(Path::new("").to_path_buf(), &backup.root, &mut checked, lock));
}; Ok(())
}
#[inline]
fn check_and_repair_backup(&mut self, name: &str, backup: &mut BackupFile, lock: &BackupMode,
) -> Result<(), RepositoryError> {
tr_info!("Checking backup...");
let mut checked = self.get_chunk_marker();
match self.check_and_repair_subtree(Path::new("").to_path_buf(),
&backup.root, &mut checked, lock
) {
Ok(None) => (),
Ok(Some(chunks)) => {
try!(self.flush(lock));
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(name, lock));
try!(self.save_backup(backup, name, lock));
}
Err(err) => {
tr_warn!(
"The root of the backup {} has been corrupted\n\tcaused by: {}",
name,
err
);
try!(self.evacuate_broken_backup(name, lock));
}
}
Ok(())
}
fn check_backups(&mut self, lock: &OnlineMode) -> Result<(), RepositoryError> {
tr_info!("Checking backups..."); tr_info!("Checking backups...");
let mut checked = self.repo.get_chunk_marker(); let mut checked = self.get_chunk_marker();
let backup_map = match self.get_all_backups() { let backup_map = match self.get_all_backups() {
Ok(backup_map) => backup_map, Ok(backup_map) => backup_map,
Err(RepositoryError::BackupFile(BackupFileError::PartialBackupsList(backup_map, Err(RepositoryError::BackupFile(BackupFileError::PartialBackupsList(backup_map,
@ -280,44 +312,53 @@ impl BackupRepository {
ProgressIter::new(tr!("checking backups"), backup_map.len(), backup_map.into_iter()) ProgressIter::new(tr!("checking backups"), backup_map.len(), backup_map.into_iter())
{ {
let path = format!("{}::", name); let path = format!("{}::", name);
match self.check_subtree( try!(self.check_subtree(Path::new(&path).to_path_buf(), &backup.root,
Path::new(&path).to_path_buf(), &mut checked, lock));
&backup.root,
&mut checked,
repair
) {
Ok(None) => (),
Ok(Some(chunks)) => {
try!(self.repo.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(&name));
try!(self.save_backup(&backup, &name));
}
Err(err) => {
if repair {
tr_warn!(
"The root of the backup {} has been corrupted\n\tcaused by: {}",
name,
err
);
try!(self.evacuate_broken_backup(&name));
} else {
return Err(err);
}
}
}
} }
Ok(()) Ok(())
} }
#[inline] fn check_and_repair_backups(&mut self, lock: &BackupMode) -> Result<(), RepositoryError> {
pub fn check_bundles(&mut self, full: bool, repair: bool) -> Result<(), RepositoryError> { tr_info!("Checking backups...");
self.repo.check_bundles(full, repair) let mut checked = self.get_chunk_marker();
let backup_map = match self.get_all_backups() {
Ok(backup_map) => backup_map,
Err(RepositoryError::BackupFile(BackupFileError::PartialBackupsList(backup_map,
_failed))) => {
tr_warn!("Some backups could not be read, ignoring them");
backup_map
} }
Err(err) => return Err(err),
pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> { };
self.repo.check_repository(repair) for (name, mut backup) in
ProgressIter::new(tr!("checking backups"), backup_map.len(), backup_map.into_iter())
{
let path = format!("{}::", name);
match self.check_and_repair_subtree(
Path::new(&path).to_path_buf(),
&backup.root,
&mut checked,
lock
) {
Ok(None) => (),
Ok(Some(chunks)) => {
try!(self.flush(lock));
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(&name, lock));
try!(self.save_backup(&backup, &name, lock));
}
Err(err) => {
tr_warn!(
"The root of the backup {} has been corrupted\n\tcaused by: {}",
name,
err
);
try!(self.evacuate_broken_backup(&name, lock));
}
}
}
Ok(())
} }
} }

45
src/backups/layout.rs Normal file
View File

@ -0,0 +1,45 @@
use ::repository::ChunkRepositoryLayout;
use std::path::PathBuf;
pub trait BackupRepositoryLayout {
fn config_path(&self) -> PathBuf;
fn keys_path(&self) -> PathBuf;
fn excludes_path(&self) -> PathBuf;
fn backups_path(&self) -> PathBuf;
fn backup_path(&self, name: &str) -> PathBuf;
fn remote_exists(&self) -> bool;
fn remote_readme_path(&self) -> PathBuf;
}
impl<P: AsRef<ChunkRepositoryLayout>> BackupRepositoryLayout for P {
fn config_path(&self) -> PathBuf {
self.as_ref().base_path().join("config.yaml")
}
fn keys_path(&self) -> PathBuf {
self.as_ref().base_path().join("keys")
}
fn excludes_path(&self) -> PathBuf {
self.as_ref().base_path().join("excludes")
}
fn backups_path(&self) -> PathBuf {
self.as_ref().base_path().join("remote/backups")
}
fn backup_path(&self, name: &str) -> PathBuf {
self.backups_path().join(format!("{}.backup", name))
}
fn remote_exists(&self) -> bool {
self.as_ref().remote_bundles_path().exists() && self.backups_path().exists() &&
self.as_ref().remote_locks_path().exists()
}
fn remote_readme_path(&self) -> PathBuf {
self.as_ref().base_path().join("remote/README.md")
}
}

51
src/backups/metadata.rs
View File

@ -7,12 +7,16 @@ use std::io::{Read, Write};
use super::*; use super::*;
impl BackupRepository { pub trait RepositoryMetadataIO {
pub fn create_inode<P: AsRef<Path>>( fn create_inode<P: AsRef<Path>>(&mut self, path: P, reference: Option<&Inode>, lock: &BackupMode) -> Result<Inode, RepositoryError>;
&mut self, fn put_inode(&mut self, inode: &Inode, lock: &BackupMode) -> Result<ChunkList, RepositoryError>;
path: P, fn get_inode(&mut self, chunks: &[Chunk], lock: &OnlineMode) -> Result<Inode, RepositoryError>;
reference: Option<&Inode>, fn save_inode_at<P: AsRef<Path>>(&mut self, inode: &Inode, path: P, lock: &OnlineMode) -> Result<(), RepositoryError>;
) -> Result<Inode, RepositoryError> { fn get_inode_children(&mut self, inode: &Inode, lock: &OnlineMode) -> Result<Vec<Inode>, RepositoryError>;
}
impl RepositoryMetadataIO for Repository {
fn create_inode<P: AsRef<Path>>(&mut self, path: P, reference: Option<&Inode>, lock: &BackupMode) -> Result<Inode, RepositoryError> {
let mut inode = try!(Inode::get_from(path.as_ref())); let mut inode = try!(Inode::get_from(path.as_ref()));
if inode.file_type == FileType::File && inode.size > 0 { if inode.file_type == FileType::File && inode.size > 0 {
if let Some(reference) = reference { if let Some(reference) = reference {
@ -27,13 +31,13 @@ impl BackupRepository {
try!(file.read_to_end(&mut data)); try!(file.read_to_end(&mut data));
inode.data = Some(FileData::Inline(data.into())); inode.data = Some(FileData::Inline(data.into()));
} else { } else {
let mut chunks = try!(self.repo.put_stream(BundleMode::Data, &mut file)); let mut chunks = try!(self.put_stream(BundleMode::Data, &mut file, lock));
if chunks.len() < 10 { if chunks.len() < 10 {
inode.data = Some(FileData::ChunkedDirect(chunks)); inode.data = Some(FileData::ChunkedDirect(chunks));
} else { } else {
let mut chunk_data = Vec::with_capacity(chunks.encoded_size()); let mut chunk_data = Vec::with_capacity(chunks.encoded_size());
chunks.write_to(&mut chunk_data).unwrap(); chunks.write_to(&mut chunk_data).unwrap();
chunks = try!(self.repo.put_data(BundleMode::Meta, &chunk_data)); chunks = try!(self.put_data(BundleMode::Meta, &chunk_data, lock));
inode.data = Some(FileData::ChunkedIndirect(chunks)); inode.data = Some(FileData::ChunkedIndirect(chunks));
} }
} }
@ -42,20 +46,27 @@ impl BackupRepository {
} }
#[inline] #[inline]
pub fn put_inode(&mut self, inode: &Inode) -> Result<ChunkList, RepositoryError> { fn put_inode(&mut self, inode: &Inode, lock: &BackupMode) -> Result<ChunkList, RepositoryError> {
self.repo.put_data(BundleMode::Meta, &try!(inode.encode())) self.put_data(BundleMode::Meta, &try!(inode.encode()), lock)
} }
#[inline] #[inline]
pub fn get_inode(&mut self, chunks: &[Chunk]) -> Result<Inode, RepositoryError> { fn get_inode(&mut self, chunks: &[Chunk], lock: &OnlineMode) -> Result<Inode, RepositoryError> {
Ok(try!(Inode::decode(&try!(self.get_data(chunks))))) Ok(try!(Inode::decode(&try!(self.get_data(chunks, lock)))))
} }
pub fn save_inode_at<P: AsRef<Path>>( #[inline]
&mut self, fn get_inode_children(&mut self, inode: &Inode, lock: &OnlineMode) -> Result<Vec<Inode>, RepositoryError> {
inode: &Inode, let mut res = vec![];
path: P, if let Some(ref children) = inode.children {
) -> Result<(), RepositoryError> { for chunks in children.values() {
res.push(try!(self.get_inode(chunks, lock)))
}
}
Ok(res)
}
fn save_inode_at<P: AsRef<Path>>(&mut self, inode: &Inode, path: P, lock: &OnlineMode) -> Result<(), RepositoryError> {
if let Some(mut file) = try!(inode.create_at(path.as_ref())) { if let Some(mut file) = try!(inode.create_at(path.as_ref())) {
if let Some(ref contents) = inode.data { if let Some(ref contents) = inode.data {
match *contents { match *contents {
@ -63,12 +74,12 @@ impl BackupRepository {
try!(file.write_all(data)); try!(file.write_all(data));
} }
FileData::ChunkedDirect(ref chunks) => { FileData::ChunkedDirect(ref chunks) => {
try!(self.repo.get_stream(chunks, &mut file)); try!(self.get_stream(chunks, &mut file, lock));
} }
FileData::ChunkedIndirect(ref chunks) => { FileData::ChunkedIndirect(ref chunks) => {
let chunk_data = try!(self.get_data(chunks)); let chunk_data = try!(self.get_data(chunks, lock));
let chunks = ChunkList::read_from(&chunk_data); let chunks = ChunkList::read_from(&chunk_data);
try!(self.repo.get_stream(&chunks, &mut file)); try!(self.get_stream(&chunks, &mut file, lock));
} }
} }
} }

278
src/backups/mod.rs
View File

@ -6,15 +6,20 @@ mod backup;
mod integrity; mod integrity;
mod vacuum; mod vacuum;
mod metadata; mod metadata;
mod layout;
pub use self::backup::{BackupOptions, BackupError, DiffType}; pub use self::backup::{BackupOptions, BackupError, DiffType, RepositoryBackupIO};
pub use self::backup_file::{BackupFile, BackupFileError}; pub use self::backup_file::{BackupFile, BackupFileError};
pub use self::inode::{Inode, FileData, FileType, InodeError}; pub use self::inode::{Inode, FileData, FileType, InodeError};
pub use self::integrity::InodeIntegrityError; pub use self::integrity::{InodeIntegrityError, RepositoryIntegrityIO};
pub use self::layout::BackupRepositoryLayout;
pub use self::metadata::RepositoryMetadataIO;
pub use self::vacuum::RepositoryVacuumIO;
pub use self::tarfile::RepositoryTarfileIO;
use ::prelude::*; use ::prelude::*;
use std::path::Path; use std::path::{Path, PathBuf};
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use std::fs::{self, File}; use std::fs::{self, File};
@ -24,16 +29,76 @@ use std::io::Write;
const DEFAULT_EXCLUDES: &[u8] = include_bytes!("../../docs/excludes.default"); const DEFAULT_EXCLUDES: &[u8] = include_bytes!("../../docs/excludes.default");
pub struct CheckOptions {
pub struct BackupRepository { all_backups: bool,
layout: Arc<RepositoryLayout>, single_backup: Option<String>,
crypto: Arc<Crypto>, subpath: Option<PathBuf>,
repo: Repository index: bool,
bundles: bool,
bundle_data: bool,
repair: bool
} }
impl CheckOptions {
pub fn new() -> CheckOptions {
CheckOptions {
all_backups: false,
single_backup: None,
subpath: None,
index: false,
bundles: false,
bundle_data: false,
repair: false
}
}
pub fn all_backups(&mut self) -> &mut Self {
self.all_backups = true;
self.single_backup = None;
self.subpath = None;
self
}
pub fn single_backup(&mut self, backup: &str) -> &mut Self {
self.all_backups = false;
self.single_backup = Some(backup.to_string());
self
}
pub fn subpath(&mut self, subpath: &Path) -> &mut Self {
self.subpath = Some(subpath.to_path_buf());
self
}
pub fn index(&mut self, index: bool) -> &mut Self {
self.index = index;
self
}
pub fn bundles(&mut self, bundles: bool) -> &mut Self {
self.bundles = bundles;
self.bundle_data &= bundles;
self
}
pub fn bundle_data(&mut self, bundle_data: bool) -> &mut Self {
self.bundle_data = bundle_data;
self.bundles |= bundle_data;
self
}
pub fn repair(&mut self, repair: bool) -> &mut Self {
self.repair = repair;
self
}
}
pub struct BackupRepository(Repository);
impl BackupRepository { impl BackupRepository {
pub fn create<P: AsRef<Path>, R: AsRef<Path>>(path: P, config: &Config, remote: R) -> Result<Self, RepositoryError> { pub fn create<P: AsRef<Path>, R: AsRef<Path>>(path: P, config: &Config, remote: R) -> Result<Self, RepositoryError> {
let layout = Arc::new(RepositoryLayout::new(path.as_ref())); let layout: Arc<ChunkRepositoryLayout> = Arc::new(path.as_ref().to_owned());
try!(fs::create_dir(layout.base_path())); try!(fs::create_dir(layout.base_path()));
try!(File::create(layout.excludes_path()).and_then(|mut f| { try!(File::create(layout.excludes_path()).and_then(|mut f| {
f.write_all(DEFAULT_EXCLUDES) f.write_all(DEFAULT_EXCLUDES)
@ -41,36 +106,28 @@ impl BackupRepository {
try!(fs::create_dir_all(layout.backups_path())); try!(fs::create_dir_all(layout.backups_path()));
try!(fs::create_dir(layout.keys_path())); try!(fs::create_dir(layout.keys_path()));
let crypto = Arc::new(try!(Crypto::open(layout.keys_path()))); let crypto = Arc::new(try!(Crypto::open(layout.keys_path())));
Ok(BackupRepository { Ok(BackupRepository(try!(Repository::create(layout, config, crypto, remote))))
crypto: crypto.clone(),
layout: layout.clone(),
repo: try!(Repository::create(layout, config, crypto, remote))
})
} }
#[allow(unknown_lints, useless_let_if_seq)] #[allow(unknown_lints, useless_let_if_seq)]
pub fn open<P: AsRef<Path>>(path: P, online: bool) -> Result<Self, RepositoryError> { pub fn open<P: AsRef<Path>>(path: P, online: bool) -> Result<Self, RepositoryError> {
let layout = Arc::new(RepositoryLayout::new(path.as_ref())); let layout: Arc<ChunkRepositoryLayout> = Arc::new(path.as_ref().to_owned());
let crypto = Arc::new(try!(Crypto::open(layout.keys_path()))); let crypto = Arc::new(try!(Crypto::open(layout.keys_path())));
Ok(BackupRepository { Ok(BackupRepository(try!(Repository::open(layout, crypto, online))))
crypto: crypto.clone(),
layout: layout.clone(),
repo: try!(Repository::open(layout, crypto, online))
})
} }
pub fn import<P: AsRef<Path>, R: AsRef<Path>>(path: P, remote: R, key_files: Vec<String>) -> Result<Self, RepositoryError> { pub fn import<P: AsRef<Path>, R: AsRef<Path>>(path: P, remote: R, key_files: Vec<String>) -> Result<Self, RepositoryError> {
let config = Config::default(); let config = Config::default();
let mut repo = try!(Self::create(&path, &config, remote)); let mut repo = try!(Self::create(&path, &config, remote));
for file in key_files { for file in key_files {
try!(repo.crypto.register_keyfile(file)); try!(repo.0.get_crypto().register_keyfile(file));
} }
repo = try!(Self::open(&path, true)); repo = try!(Self::open(&path, true));
let mut backups: Vec<(String, BackupFile)> = try!(repo.get_all_backups()).into_iter().collect(); let mut backups: Vec<(String, BackupFile)> = try!(repo.0.get_all_backups()).into_iter().collect();
backups.sort_by_key(|&(_, ref b)| b.timestamp); backups.sort_by_key(|&(_, ref b)| b.timestamp);
if let Some((name, backup)) = backups.pop() { if let Some((name, backup)) = backups.pop() {
tr_info!("Taking configuration from the last backup '{}'", name); tr_info!("Taking configuration from the last backup '{}'", name);
repo.repo.set_config(backup.config); repo.0.set_config(backup.config);
try!(repo.save_config()) try!(repo.save_config())
} else { } else {
tr_warn!( tr_warn!(
@ -80,69 +137,204 @@ impl BackupRepository {
Ok(repo) Ok(repo)
} }
#[inline]
pub fn has_backup(&self, name: &str) -> bool {
self.0.has_backup(name)
}
#[inline]
pub fn get_backup(&self, name: &str) -> Result<BackupFile, RepositoryError> {
self.0.get_backup(name)
}
#[inline] #[inline]
pub fn register_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), RepositoryError> { pub fn register_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), RepositoryError> {
try!(self.repo.write_mode()); try!(self.0.get_crypto().register_secret_key(public, secret));
try!(self.crypto.register_secret_key(public, secret));
Ok(()) Ok(())
} }
#[inline] #[inline]
pub fn save_config(&mut self) -> Result<(), RepositoryError> { pub fn save_config(&mut self) -> Result<(), RepositoryError> {
self.repo.save_config() self.0.localwrite_mode(|r, l| r.save_config(l))
} }
#[inline] #[inline]
pub fn set_encryption(&mut self, public: Option<&PublicKey>) { pub fn set_encryption(&mut self, public: Option<&PublicKey>) {
self.repo.set_encryption(public) self.0.set_encryption(public)
} }
#[inline]
pub fn get_config(&self) -> &Config { pub fn get_config(&self) -> &Config {
self.repo.get_config() self.0.get_config()
} }
#[inline]
pub fn set_config(&mut self, config: Config) { pub fn set_config(&mut self, config: Config) {
self.repo.set_config(config); self.0.set_config(config);
} }
pub fn get_layout(&self) -> &RepositoryLayout { #[inline]
&self.layout pub fn get_layout(&self) -> Arc<ChunkRepositoryLayout> {
self.0.get_layout()
} }
#[inline]
pub fn info(&self) -> RepositoryInfo { pub fn info(&self) -> RepositoryInfo {
self.repo.info() self.0.info()
} }
#[inline] #[inline]
pub fn check_index(&mut self, repair: bool) -> Result<(), RepositoryError> { pub fn check_repo(&mut self, index: bool, bundles: bool, bundle_data: bool) -> Result<IntegrityReport, RepositoryError> {
self.repo.check_index(repair) self.0.online_mode(|r, l| Ok(r.check(index, bundles, bundle_data, l)))
} }
#[inline] #[inline]
pub fn set_clean(&mut self) { pub fn check_and_repair_repo(&mut self, index: bool, bundles: bool, bundle_data: bool) -> Result<IntegrityReport, RepositoryError> {
self.repo.set_clean() self.0.vacuum_mode(|r, l| r.check_and_repair(index, bundles, bundle_data, l))
} }
#[inline]
pub fn statistics(&self) -> RepositoryStatistics { pub fn statistics(&self) -> RepositoryStatistics {
self.repo.statistics() self.0.statistics()
} }
#[inline] #[inline]
pub fn list_bundles(&self) -> Vec<&BundleInfo> { pub fn list_bundles(&self) -> Vec<&BundleInfo> {
self.repo.list_bundles() self.0.list_bundles()
} }
#[inline] #[inline]
pub fn get_bundle(&self, bundle: &BundleId) -> Option<&StoredBundle> { pub fn get_bundle(&self, bundle: &BundleId) -> Option<&StoredBundle> {
self.repo.get_bundle(bundle) self.0.get_bundle(bundle)
} }
pub fn get_chunk(&mut self, hash: Hash) -> Result<Option<Vec<u8>>, RepositoryError> { #[inline]
self.repo.get_chunk(hash) pub fn get_all_backups(&self) -> Result<HashMap<String, BackupFile>, RepositoryError> {
self.0.get_all_backups()
} }
pub fn get_data(&mut self, chunks: &[Chunk]) -> Result<Vec<u8>, RepositoryError> { #[inline]
self.repo.get_data(chunks) pub fn get_backups<P: AsRef<Path>>(&self, path: P) -> Result<HashMap<String, BackupFile>, RepositoryError> {
self.0.get_backups(path)
} }
#[inline]
pub fn delete_backup(&mut self, name: &str) -> Result<(), RepositoryError> {
self.0.backup_mode(|r, l| r.delete_backup(name, l))
}
#[inline]
pub fn prune_backups(&mut self, prefix: &str, daily: usize, weekly: usize, monthly: usize,
yearly: usize, force: bool) -> Result<(), RepositoryError>
{
self.0.backup_mode(|r, l| r.prune_backups(prefix, daily, weekly, monthly, yearly, force, l))
}
#[inline]
pub fn get_root_inode(&mut self, backup: &BackupFile) -> Result<Inode, RepositoryError> {
self.0.online_mode(|r, l| r.get_inode(&backup.root, l))
}
#[inline]
pub fn get_inode_children(&mut self, inode: &Inode) -> Result<Vec<Inode>, RepositoryError> {
self.0.online_mode(|r, l| r.get_inode_children(inode, l))
}
#[inline]
pub fn restore_inode_tree<P: AsRef<Path>>(&mut self, backup: &BackupFile, inode: Inode, path: P) -> Result<(), RepositoryError> {
self.0.online_mode(|r, l| r.restore_inode_tree(backup, inode, path, l))
}
#[inline]
pub fn create_backup<P: AsRef<Path>>(&mut self, path: P, name: &str, reference: Option<&BackupFile>,
options: &BackupOptions) -> Result<BackupFile, RepositoryError>
{
self.0.backup_mode(|r, l| r.create_backup(path, name, reference, options,l))
}
#[inline]
pub fn remove_backup_path<P: AsRef<Path>>(&mut self, backup: &mut BackupFile, path: P
) -> Result<(), RepositoryError> {
self.0.backup_mode(|r, l| r.remove_backup_path(backup, path, l))
}
#[inline]
pub fn get_backup_path<P: AsRef<Path>>(&mut self, backup: &BackupFile, path: P) -> Result<Vec<Inode>, RepositoryError> {
self.0.online_mode(|r, l| r.get_backup_path(backup, path, l))
}
#[inline]
pub fn get_backup_inode<P: AsRef<Path>>(&mut self, backup: &BackupFile, path: P) -> Result<Inode, RepositoryError> {
self.0.online_mode(|r, l| r.get_backup_inode(backup, path, l))
}
#[inline]
pub fn find_differences(&mut self, inode1: &Inode, inode2: &Inode
) -> Result<Vec<(DiffType, PathBuf)>, RepositoryError> {
self.0.online_mode(|r, l| r.find_differences(inode1, inode2, l))
}
#[inline]
pub fn find_versions<P: AsRef<Path>>(&mut self, path: P
) -> Result<Vec<(String, Inode)>, RepositoryError> {
self.0.online_mode(|r, l| r.find_versions(path, l))
}
#[inline]
pub fn find_duplicates(&mut self, inode: &Inode, min_size: u64
) -> Result<Vec<(Vec<PathBuf>, u64)>, RepositoryError> {
self.0.online_mode(|r, l| r.find_duplicates(inode, min_size, l))
}
#[inline]
pub fn analyze_usage(&mut self) -> Result<HashMap<u32, BundleAnalysis>, RepositoryError> {
self.0.online_mode(|r, l| r.analyze_usage(l))
}
#[inline]
pub fn vacuum(&mut self, ratio: f32, combine: bool, force: bool) -> Result<(), RepositoryError> {
self.0.vacuum_mode(|r, l| r.vacuum(ratio, combine, force, l))
}
pub fn mount_repository<P: AsRef<Path>>(&mut self, path: Option<&str>,
mountpoint: P) -> Result<(), RepositoryError> {
self.0.online_mode(|r, l| {
let fs = try!(FuseFilesystem::from_repository(r, l, path));
fs.mount(mountpoint)
})
}
pub fn mount_backup<P: AsRef<Path>>(&mut self, backup: BackupFile,
mountpoint: P) -> Result<(), RepositoryError> {
self.0.online_mode(|r, l| {
let fs = try!(FuseFilesystem::from_backup(r, l, backup));
fs.mount(mountpoint)
})
}
pub fn mount_inode<P: AsRef<Path>>(&mut self, backup: BackupFile, inode: Inode,
mountpoint: P) -> Result<(), RepositoryError> {
self.0.online_mode(|r, l| {
let fs = try!(FuseFilesystem::from_inode(r, l, backup, inode));
fs.mount(mountpoint)
})
}
pub fn check(&mut self, options: CheckOptions) -> Result<(), RepositoryError> {
unimplemented!()
//TODO: implement
}
#[inline]
pub fn import_tarfile<P: AsRef<Path>>(&mut self, tarfile: P) -> Result<BackupFile, RepositoryError> {
self.0.backup_mode(|r, l| r.import_tarfile(tarfile, l))
}
#[inline]
pub fn export_tarfile<P: AsRef<Path>>(&mut self, backup: &BackupFile, inode: Inode, tarfile: P
) -> Result<(), RepositoryError> {
self.0.online_mode(|r, l| r.export_tarfile(backup, inode, tarfile, l))
}
} }

35
src/backups/mount.rs
View File

@ -150,21 +150,24 @@ impl FuseInode {
pub struct FuseFilesystem<'a> { pub struct FuseFilesystem<'a> {
next_id: u64, next_id: u64,
repository: &'a mut BackupRepository, lock: &'a OnlineMode,
inodes: HashMap<u64, FuseInodeRef> repository: &'a mut Repository,
inodes: HashMap<u64, FuseInodeRef>,
} }
impl<'a> FuseFilesystem<'a> { impl<'a> FuseFilesystem<'a> {
pub fn new(repository: &'a mut BackupRepository) -> Result<Self, RepositoryError> { pub fn new(repository: &'a mut Repository, lock: &'a OnlineMode) -> Result<Self, RepositoryError> {
Ok(FuseFilesystem { Ok(FuseFilesystem {
next_id: 1, next_id: 1,
lock,
repository, repository,
inodes: HashMap::new() inodes: HashMap::new()
}) })
} }
pub fn from_repository( pub fn from_repository(
repository: &'a mut BackupRepository, repository: &'a mut Repository,
lock: &'a OnlineMode,
path: Option<&str>, path: Option<&str>,
) -> Result<Self, RepositoryError> { ) -> Result<Self, RepositoryError> {
let mut backups = vec![]; let mut backups = vec![];
@ -173,10 +176,10 @@ impl<'a> FuseFilesystem<'a> {
None => try!(repository.get_all_backups()), None => try!(repository.get_all_backups()),
}; };
for (name, backup) in backup_map { for (name, backup) in backup_map {
let inode = try!(repository.get_inode(&backup.root)); let inode = try!(repository.get_inode(&backup.root, lock));
backups.push((name, backup, inode)); backups.push((name, backup, inode));
} }
let mut fs = try!(FuseFilesystem::new(repository)); let mut fs = try!(FuseFilesystem::new(repository, lock));
let root = fs.add_virtual_directory("".to_string(), None); let root = fs.add_virtual_directory("".to_string(), None);
for (name, backup, mut inode) in backups { for (name, backup, mut inode) in backups {
let mut parent = root.clone(); let mut parent = root.clone();
@ -196,21 +199,23 @@ impl<'a> FuseFilesystem<'a> {
} }
pub fn from_backup( pub fn from_backup(
repository: &'a mut BackupRepository, repository: &'a mut Repository,
lock: &'a OnlineMode,
backup: BackupFile, backup: BackupFile,
) -> Result<Self, RepositoryError> { ) -> Result<Self, RepositoryError> {
let inode = try!(repository.get_inode(&backup.root)); let inode = try!(repository.get_inode(&backup.root, lock));
let mut fs = try!(FuseFilesystem::new(repository)); let mut fs = try!(FuseFilesystem::new(repository, lock));
fs.add_inode(inode, None, backup.user_names, backup.group_names); fs.add_inode(inode, None, backup.user_names, backup.group_names);
Ok(fs) Ok(fs)
} }
pub fn from_inode( pub fn from_inode(
repository: &'a mut BackupRepository, repository: &'a mut Repository,
lock: &'a OnlineMode,
backup: BackupFile, backup: BackupFile,
inode: Inode, inode: Inode,
) -> Result<Self, RepositoryError> { ) -> Result<Self, RepositoryError> {
let mut fs = try!(FuseFilesystem::new(repository)); let mut fs = try!(FuseFilesystem::new(repository, lock));
fs.add_inode(inode, None, backup.user_names, backup.group_names); fs.add_inode(inode, None, backup.user_names, backup.group_names);
Ok(fs) Ok(fs)
} }
@ -290,7 +295,7 @@ impl<'a> FuseFilesystem<'a> {
if let Some(chunks) = parent_mut.inode.children.as_ref().and_then(|c| c.get(name)) { if let Some(chunks) = parent_mut.inode.children.as_ref().and_then(|c| c.get(name)) {
child = Rc::new(RefCell::new(FuseInode { child = Rc::new(RefCell::new(FuseInode {
num: self.next_id, num: self.next_id,
inode: try!(self.repository.get_inode(chunks)), inode: try!(self.repository.get_inode(chunks, self.lock)),
parent: Some(parent.clone()), parent: Some(parent.clone()),
children: HashMap::new(), children: HashMap::new(),
chunks: None, chunks: None,
@ -316,7 +321,7 @@ impl<'a> FuseFilesystem<'a> {
if !parent_mut.children.contains_key(name) { if !parent_mut.children.contains_key(name) {
let child = Rc::new(RefCell::new(FuseInode { let child = Rc::new(RefCell::new(FuseInode {
num: self.next_id, num: self.next_id,
inode: try!(self.repository.get_inode(chunks)), inode: try!(self.repository.get_inode(chunks, self.lock)),
parent: Some(parent.clone()), parent: Some(parent.clone()),
children: HashMap::new(), children: HashMap::new(),
chunks: None, chunks: None,
@ -344,7 +349,7 @@ impl<'a> FuseFilesystem<'a> {
chunks = Some(c.clone()); chunks = Some(c.clone());
} }
Some(FileData::ChunkedIndirect(ref c)) => { Some(FileData::ChunkedIndirect(ref c)) => {
let chunk_data = try!(self.repository.get_data(c)); let chunk_data = try!(self.repository.get_data(c, self.lock));
chunks = Some(ChunkList::read_from(&chunk_data)); chunks = Some(ChunkList::read_from(&chunk_data));
} }
} }
@ -556,7 +561,7 @@ impl<'a> fuse::Filesystem for FuseFilesystem<'a> {
offset -= i64::from(len); offset -= i64::from(len);
continue; continue;
} }
let chunk = match fuse_try!(self.repository.get_chunk(hash), reply) { let chunk = match fuse_try!(self.repository.get_chunk(hash, self.lock), reply) {
Some(chunk) => chunk, Some(chunk) => chunk,
None => return reply.error(libc::EIO), None => return reply.error(libc::EIO),
}; };

104
src/backups/tarfile.rs
View File

@ -134,10 +134,27 @@ fn inode_from_entry<R: Read>(entry: &mut tar::Entry<R>) -> Result<Inode, Reposit
} }
impl BackupRepository { pub trait RepositoryTarfileIO {
fn import_tar_entry<R: Read>( fn import_tar_entry<R: Read>(&mut self, entry: &mut tar::Entry<R>, lock: &BackupMode
&mut self, ) -> Result<Inode, RepositoryError>;
entry: &mut tar::Entry<R>, fn import_tarfile_as_inode<R: Read>(&mut self, backup: &mut BackupFile, input: R,
failed_paths: &mut Vec<PathBuf>, lock: &BackupMode
) -> Result<(Inode, ChunkList), RepositoryError>;
fn import_tarfile<P: AsRef<Path>>(&mut self, tarfile: P, lock: &BackupMode
) -> Result<BackupFile, RepositoryError>;
fn export_xattrs<W: Write>(&mut self, inode: &Inode, tarfile: &mut tar::Builder<W>,
lock: &OnlineMode
) -> Result<(), RepositoryError>;
fn export_tarfile_recurse<W: Write>(&mut self, backup: &BackupFile, path: &Path,
inode: Inode, tarfile: &mut tar::Builder<W>, skip_root: bool, lock: &OnlineMode
) -> Result<(), RepositoryError>;
fn export_tarfile<P: AsRef<Path>>(&mut self, backup: &BackupFile, inode: Inode,
tarfile: P, lock: &OnlineMode
) -> Result<(), RepositoryError>;
}
impl RepositoryTarfileIO for Repository {
fn import_tar_entry<R: Read>(&mut self, entry: &mut tar::Entry<R>, lock: &BackupMode
) -> Result<Inode, RepositoryError> { ) -> Result<Inode, RepositoryError> {
let mut inode = try!(inode_from_entry(entry)); let mut inode = try!(inode_from_entry(entry));
if inode.size < 100 { if inode.size < 100 {
@ -145,24 +162,21 @@ impl BackupRepository {
try!(entry.read_to_end(&mut data)); try!(entry.read_to_end(&mut data));
inode.data = Some(FileData::Inline(data.into())); inode.data = Some(FileData::Inline(data.into()));
} else { } else {
let mut chunks = try!(self.repo.put_stream(BundleMode::Data, entry)); let mut chunks = try!(self.put_stream(BundleMode::Data, entry, lock));
if chunks.len() < 10 { if chunks.len() < 10 {
inode.data = Some(FileData::ChunkedDirect(chunks)); inode.data = Some(FileData::ChunkedDirect(chunks));
} else { } else {
let mut chunk_data = Vec::with_capacity(chunks.encoded_size()); let mut chunk_data = Vec::with_capacity(chunks.encoded_size());
chunks.write_to(&mut chunk_data).unwrap(); chunks.write_to(&mut chunk_data).unwrap();
chunks = try!(self.repo.put_data(BundleMode::Meta, &chunk_data)); chunks = try!(self.put_data(BundleMode::Meta, &chunk_data, lock));
inode.data = Some(FileData::ChunkedIndirect(chunks)); inode.data = Some(FileData::ChunkedIndirect(chunks));
} }
} }
Ok(inode) Ok(inode)
} }
fn import_tarfile_as_inode<R: Read>( fn import_tarfile_as_inode<R: Read>(&mut self, backup: &mut BackupFile, input: R,
&mut self, failed_paths: &mut Vec<PathBuf>, lock: &BackupMode
backup: &mut BackupFile,
input: R,
failed_paths: &mut Vec<PathBuf>,
) -> Result<(Inode, ChunkList), RepositoryError> { ) -> Result<(Inode, ChunkList), RepositoryError> {
let mut tarfile = tar::Archive::new(input); let mut tarfile = tar::Archive::new(input);
// Step 1: create inodes for all entries // Step 1: create inodes for all entries
@ -170,7 +184,7 @@ impl BackupRepository {
for entry in try!(tarfile.entries()) { for entry in try!(tarfile.entries()) {
let mut entry = try!(entry); let mut entry = try!(entry);
let path = try!(entry.path()).to_path_buf(); let path = try!(entry.path()).to_path_buf();
match self.import_tar_entry(&mut entry) { match self.import_tar_entry(&mut entry, lock) {
Ok(mut inode) => { Ok(mut inode) => {
inode.cum_size = inode.size; inode.cum_size = inode.size;
if inode.file_type == FileType::Directory { if inode.file_type == FileType::Directory {
@ -219,7 +233,7 @@ impl BackupRepository {
} }
for path in childless { for path in childless {
let (inode, _) = inodes.remove(&path).unwrap(); let (inode, _) = inodes.remove(&path).unwrap();
let chunks = try!(self.put_inode(&inode)); let chunks = try!(self.put_inode(&inode, lock));
if let Some(parent_path) = path.parent() { if let Some(parent_path) = path.parent() {
if let Some(&mut (ref mut parent_inode, ref mut children)) = if let Some(&mut (ref mut parent_inode, ref mut children)) =
inodes.get_mut(parent_path) inodes.get_mut(parent_path)
@ -265,23 +279,15 @@ impl BackupRepository {
children.insert(inode.name, chunks); children.insert(inode.name, chunks);
} }
root_inode.children = Some(children); root_inode.children = Some(children);
let chunks = try!(self.put_inode(&root_inode)); let chunks = try!(self.put_inode(&root_inode, lock));
Ok((root_inode, chunks)) Ok((root_inode, chunks))
} }
} }
pub fn import_tarfile<P: AsRef<Path>>( fn import_tarfile<P: AsRef<Path>>(&mut self, tarfile: P, lock: &BackupMode
&mut self,
tarfile: P,
) -> Result<BackupFile, RepositoryError> { ) -> Result<BackupFile, RepositoryError> {
try!(self.repo.write_mode());
let _lock = try!(self.repo.lock(false));
if self.repo.is_dirty() {
return Err(RepositoryError::Dirty);
}
try!(self.repo.set_dirty());
let mut backup = BackupFile::default(); let mut backup = BackupFile::default();
backup.config = self.repo.get_config().clone(); backup.config = self.get_config().clone();
backup.host = get_hostname().unwrap_or_else(|_| "".to_string()); backup.host = get_hostname().unwrap_or_else(|_| "".to_string());
backup.path = tarfile.as_ref().to_string_lossy().to_string(); backup.path = tarfile.as_ref().to_string_lossy().to_string();
let info_before = self.info(); let info_before = self.info();
@ -292,17 +298,19 @@ impl BackupRepository {
try!(self.import_tarfile_as_inode( try!(self.import_tarfile_as_inode(
&mut backup, &mut backup,
io::stdin(), io::stdin(),
&mut failed_paths &mut failed_paths,
lock
)) ))
} else { } else {
try!(self.import_tarfile_as_inode( try!(self.import_tarfile_as_inode(
&mut backup, &mut backup,
try!(File::open(tarfile)), try!(File::open(tarfile)),
&mut failed_paths &mut failed_paths,
lock
)) ))
}; };
backup.root = chunks; backup.root = chunks;
try!(self.repo.flush()); try!(self.flush(lock));
let elapsed = Local::now().signed_duration_since(start); let elapsed = Local::now().signed_duration_since(start);
backup.timestamp = start.timestamp(); backup.timestamp = start.timestamp();
backup.total_data_size = root_inode.cum_size; backup.total_data_size = root_inode.cum_size;
@ -315,7 +323,6 @@ impl BackupRepository {
backup.bundle_count = info_after.bundle_count - info_before.bundle_count; backup.bundle_count = info_after.bundle_count - info_before.bundle_count;
backup.chunk_count = info_after.chunk_count - info_before.chunk_count; backup.chunk_count = info_after.chunk_count - info_before.chunk_count;
backup.avg_chunk_size = backup.deduplicated_data_size as f32 / backup.chunk_count as f32; backup.avg_chunk_size = backup.deduplicated_data_size as f32 / backup.chunk_count as f32;
self.repo.set_clean();
if failed_paths.is_empty() { if failed_paths.is_empty() {
Ok(backup) Ok(backup)
} else { } else {
@ -323,10 +330,8 @@ impl BackupRepository {
} }
} }
fn export_xattrs<W: Write>( fn export_xattrs<W: Write>(&mut self, inode: &Inode, tarfile: &mut tar::Builder<W>,
&mut self, lock: &OnlineMode
inode: &Inode,
tarfile: &mut tar::Builder<W>,
) -> Result<(), RepositoryError> { ) -> Result<(), RepositoryError> {
let mut pax = PaxBuilder::new(); let mut pax = PaxBuilder::new();
for (key, value) in &inode.xattrs { for (key, value) in &inode.xattrs {
@ -339,13 +344,8 @@ impl BackupRepository {
Ok(()) Ok(())
} }
fn export_tarfile_recurse<W: Write>( fn export_tarfile_recurse<W: Write>(&mut self, backup: &BackupFile, path: &Path, inode: Inode,
&mut self, tarfile: &mut tar::Builder<W>, skip_root: bool, lock: &OnlineMode
backup: &BackupFile,
path: &Path,
inode: Inode,
tarfile: &mut tar::Builder<W>,
skip_root: bool,
) -> Result<(), RepositoryError> { ) -> Result<(), RepositoryError> {
let path = if skip_root { let path = if skip_root {
path.to_path_buf() path.to_path_buf()
@ -354,7 +354,7 @@ impl BackupRepository {
}; };
if inode.file_type != FileType::Directory || !skip_root { if inode.file_type != FileType::Directory || !skip_root {
if !inode.xattrs.is_empty() { if !inode.xattrs.is_empty() {
try!(self.export_xattrs(&inode, tarfile)); try!(self.export_xattrs(&inode, tarfile, lock));
} }
let mut header = tar::Header::new_gnu(); let mut header = tar::Header::new_gnu();
header.set_size(inode.size); header.set_size(inode.size);
@ -397,34 +397,32 @@ impl BackupRepository {
None => try!(tarfile.append(&header, Cursor::new(&[]))), None => try!(tarfile.append(&header, Cursor::new(&[]))),
Some(FileData::Inline(data)) => try!(tarfile.append(&header, Cursor::new(data))), Some(FileData::Inline(data)) => try!(tarfile.append(&header, Cursor::new(data))),
Some(FileData::ChunkedDirect(chunks)) => { Some(FileData::ChunkedDirect(chunks)) => {
try!(tarfile.append(&header, self.repo.get_reader(chunks))) try!(tarfile.append(&header, self.get_reader(chunks, lock)))
} }
Some(FileData::ChunkedIndirect(chunks)) => { Some(FileData::ChunkedIndirect(chunks)) => {
let chunks = ChunkList::read_from(&try!(self.get_data(&chunks))); let chunks = ChunkList::read_from(&try!(self.get_data(&chunks, lock)));
try!(tarfile.append(&header, self.repo.get_reader(chunks))) try!(tarfile.append(&header, self.get_reader(chunks, lock)))
} }
} }
} }
if let Some(children) = inode.children { if let Some(children) = inode.children {
for chunks in children.values() { for chunks in children.values() {
let inode = try!(self.get_inode(chunks)); let inode = try!(self.get_inode(chunks, lock));
try!(self.export_tarfile_recurse( try!(self.export_tarfile_recurse(
backup, backup,
&path, &path,
inode, inode,
tarfile, tarfile,
false false,
lock
)); ));
} }
} }
Ok(()) Ok(())
} }
pub fn export_tarfile<P: AsRef<Path>>( fn export_tarfile<P: AsRef<Path>>(&mut self, backup: &BackupFile, inode: Inode, tarfile: P,
&mut self, lock: &OnlineMode
backup: &BackupFile,
inode: Inode,
tarfile: P,
) -> Result<(), RepositoryError> { ) -> Result<(), RepositoryError> {
let tarfile = tarfile.as_ref(); let tarfile = tarfile.as_ref();
if tarfile == Path::new("-") { if tarfile == Path::new("-") {
@ -434,7 +432,8 @@ impl BackupRepository {
Path::new(""), Path::new(""),
inode, inode,
&mut tarfile, &mut tarfile,
true true,
lock
)); ));
try!(tarfile.finish()); try!(tarfile.finish());
} else { } else {
@ -444,7 +443,8 @@ impl BackupRepository {
Path::new(""), Path::new(""),
inode, inode,
&mut tarfile, &mut tarfile,
true true,
lock
)); ));
try!(tarfile.finish()); try!(tarfile.finish());
} }

62
src/backups/vacuum.rs
View File

@ -5,15 +5,23 @@ use super::*;
use std::collections::{VecDeque, HashSet}; use std::collections::{VecDeque, HashSet};
impl BackupRepository { pub trait RepositoryVacuumIO {
fn mark_used( fn mark_used(&self, bundles: &mut HashMap<u32, BundleAnalysis>, chunks: &[Chunk],
&self, lock: &OnlineMode
bundles: &mut HashMap<u32, BundleAnalysis>, ) -> Result<bool, RepositoryError>;
chunks: &[Chunk], fn analyze_usage(&mut self, lock: &OnlineMode
) -> Result<HashMap<u32, BundleAnalysis>, RepositoryError>;
fn vacuum(&mut self, ratio: f32, combine: bool, force: bool, lock: &VacuumMode
) -> Result<(), RepositoryError>;
}
impl RepositoryVacuumIO for Repository {
fn mark_used(&self, bundles: &mut HashMap<u32, BundleAnalysis>, chunks: &[Chunk],
lock: &OnlineMode
) -> Result<bool, RepositoryError> { ) -> Result<bool, RepositoryError> {
let mut new = false; let mut new = false;
for &(hash, len) in chunks { for &(hash, len) in chunks {
if let Some(pos) = self.repo.get_chunk_location(hash) { if let Some(pos) = self.get_chunk_location(hash) {
let bundle = pos.bundle; let bundle = pos.bundle;
if let Some(bundle) = bundles.get_mut(&bundle) { if let Some(bundle) = bundles.get_mut(&bundle) {
if !bundle.chunk_usage.get(pos.chunk as usize) { if !bundle.chunk_usage.get(pos.chunk as usize) {
@ -31,13 +39,10 @@ impl BackupRepository {
Ok(new) Ok(new)
} }
pub fn analyze_usage(&mut self) -> Result<HashMap<u32, BundleAnalysis>, RepositoryError> { fn analyze_usage(&mut self, lock: &OnlineMode
if self.repo.is_dirty() { ) -> Result<HashMap<u32, BundleAnalysis>, RepositoryError> {
return Err(RepositoryError::Dirty);
}
try!(self.repo.set_dirty());
let mut usage = HashMap::new(); let mut usage = HashMap::new();
for (id, bundle) in try!(self.repo.get_bundle_map()) { for (id, bundle) in try!(self.get_bundle_map()) {
usage.insert( usage.insert(
id, id,
BundleAnalysis { BundleAnalysis {
@ -53,22 +58,22 @@ impl BackupRepository {
todo.push_back(backup.root); todo.push_back(backup.root);
} }
while let Some(chunks) = todo.pop_back() { while let Some(chunks) = todo.pop_back() {
if !try!(self.mark_used(&mut usage, &chunks)) { if !try!(self.mark_used(&mut usage, &chunks, lock)) {
continue; continue;
} }
let inode = try!(self.get_inode(&chunks)); let inode = try!(self.get_inode(&chunks, lock));
// Mark the content chunks as used // Mark the content chunks as used
match inode.data { match inode.data {
None | None |
Some(FileData::Inline(_)) => (), Some(FileData::Inline(_)) => (),
Some(FileData::ChunkedDirect(chunks)) => { Some(FileData::ChunkedDirect(chunks)) => {
try!(self.mark_used(&mut usage, &chunks)); try!(self.mark_used(&mut usage, &chunks, lock));
} }
Some(FileData::ChunkedIndirect(chunks)) => { Some(FileData::ChunkedIndirect(chunks)) => {
if try!(self.mark_used(&mut usage, &chunks)) { if try!(self.mark_used(&mut usage, &chunks, lock)) {
let chunk_data = try!(self.get_data(&chunks)); let chunk_data = try!(self.get_data(&chunks, lock));
let chunks = ChunkList::read_from(&chunk_data); let chunks = ChunkList::read_from(&chunk_data);
try!(self.mark_used(&mut usage, &chunks)); try!(self.mark_used(&mut usage, &chunks, lock));
} }
} }
} }
@ -79,23 +84,14 @@ impl BackupRepository {
} }
} }
} }
self.repo.set_clean();
Ok(usage) Ok(usage)
} }
pub fn vacuum( fn vacuum(&mut self, ratio: f32, combine: bool, force: bool, lock: &VacuumMode
&mut self,
ratio: f32,
combine: bool,
force: bool,
) -> Result<(), RepositoryError> { ) -> Result<(), RepositoryError> {
try!(self.repo.flush()); try!(self.flush(lock.as_backup()));
tr_info!("Locking repository");
try!(self.repo.write_mode());
let _lock = try!(self.repo.lock(true));
// analyze_usage will set the dirty flag
tr_info!("Analyzing chunk usage"); tr_info!("Analyzing chunk usage");
let usage = try!(self.analyze_usage()); let usage = try!(self.analyze_usage(lock.as_online()));
let mut data_total = 0; let mut data_total = 0;
let mut data_used = 0; let mut data_used = 0;
for bundle in usage.values() { for bundle in usage.values() {
@ -122,7 +118,7 @@ impl BackupRepository {
let mut small_meta = vec![]; let mut small_meta = vec![];
let mut small_data = vec![]; let mut small_data = vec![];
for (id, bundle) in &usage { for (id, bundle) in &usage {
if bundle.info.encoded_size * 4 < self.repo.get_config().bundle_size { if bundle.info.encoded_size * 4 < self.get_config().bundle_size {
match bundle.info.mode { match bundle.info.mode {
BundleMode::Meta => small_meta.push(*id), BundleMode::Meta => small_meta.push(*id),
BundleMode::Data => small_data.push(*id), BundleMode::Data => small_data.push(*id),
@ -147,12 +143,10 @@ impl BackupRepository {
to_file_size(rewrite_data as u64) to_file_size(rewrite_data as u64)
); );
if !force { if !force {
self.repo.set_clean();
return Ok(()); return Ok(());
} }
let rewrite_bundles: Vec<_> = rewrite_bundles.into_iter().collect(); let rewrite_bundles: Vec<_> = rewrite_bundles.into_iter().collect();
try!(self.repo.rewrite_bundles(&rewrite_bundles, &usage)); try!(self.rewrite_bundles(&rewrite_bundles, &usage, lock));
self.repo.set_clean();
Ok(()) Ok(())
} }
} }

99
src/cli/mod.rs
View File

@ -144,7 +144,7 @@ fn get_inode(repo: &mut BackupRepository, backup: &BackupFile, inode: Option<&St
) )
} else { } else {
checked!( checked!(
repo.get_inode(&backup.root), repo.get_root_inode(&backup),
"load root inode", "load root inode",
ErrorCode::LoadInode ErrorCode::LoadInode
) )
@ -655,7 +655,7 @@ pub fn run() -> Result<(), ErrorCode> {
let result = if tar { let result = if tar {
repo.import_tarfile(&src_path) repo.import_tarfile(&src_path)
} else { } else {
repo.create_backup_recursively(&src_path, reference_backup.as_ref(), &options) repo.create_backup(&src_path, &backup_name, reference_backup.as_ref(), &options)
}; };
let backup = match result { let backup = match result {
Ok(backup) => { Ok(backup) => {
@ -671,11 +671,6 @@ pub fn run() -> Result<(), ErrorCode> {
return Err(ErrorCode::BackupRun); return Err(ErrorCode::BackupRun);
} }
}; };
checked!(
repo.save_backup(&backup, &backup_name),
"save backup file",
ErrorCode::SaveBackup
);
print_backup(&backup); print_backup(&backup);
} }
Arguments::Restore { Arguments::Restore {
@ -719,11 +714,7 @@ pub fn run() -> Result<(), ErrorCode> {
return Err(ErrorCode::BackupAlreadyExists); return Err(ErrorCode::BackupAlreadyExists);
} }
let backup = try!(get_backup(&repo, &backup_name_src)); let backup = try!(get_backup(&repo, &backup_name_src));
checked!( //TODO: implement
repo.save_backup(&backup, &backup_name_dst),
"save backup file",
ErrorCode::SaveBackup
);
} }
Arguments::Remove { Arguments::Remove {
repo_path, repo_path,
@ -739,11 +730,6 @@ pub fn run() -> Result<(), ErrorCode> {
"remove backup subpath", "remove backup subpath",
ErrorCode::RemoveRun ErrorCode::RemoveRun
); );
checked!(
repo.save_backup(&backup, &backup_name),
"save backup file",
ErrorCode::SaveBackup
);
tr_info!("The backup subpath has been deleted, run vacuum to reclaim space"); tr_info!("The backup subpath has been deleted, run vacuum to reclaim space");
} else if repo.get_layout().backups_path().join(&backup_name).is_dir() { } else if repo.get_layout().backups_path().join(&backup_name).is_dir() {
let backups = checked!( let backups = checked!(
@ -830,44 +816,21 @@ pub fn run() -> Result<(), ErrorCode> {
repair repair
} => { } => {
let mut repo = try!(open_repository(&repo_path, true)); let mut repo = try!(open_repository(&repo_path, true));
checked!( let mut options = CheckOptions::new();
repo.check_repository(repair), options.index(index).bundle_data(bundle_data).bundles(bundles).repair(repair);
"check repository",
ErrorCode::CheckRun
);
if bundles {
checked!(
repo.check_bundles(bundle_data, repair),
"check bundles",
ErrorCode::CheckRun
);
}
if index {
checked!(repo.check_index(repair), "check index", ErrorCode::CheckRun);
}
if let Some(backup_name) = backup_name { if let Some(backup_name) = backup_name {
let mut backup = try!(get_backup(&repo, &backup_name)); options.single_backup(&backup_name);
if let Some(path) = inode { if let Some(inode) = inode {
checked!( options.subpath(Path::new(&inode));
repo.check_backup_inode(&backup_name, &mut backup, Path::new(&path), repair),
"check inode",
ErrorCode::CheckRun
)
} else {
checked!(
repo.check_backup(&backup_name, &mut backup, repair),
"check backup",
ErrorCode::CheckRun
)
} }
} else { } else {
options.all_backups();
}
checked!( checked!(
repo.check_backups(repair), repo.check(options),
"check repository", "check repository",
ErrorCode::CheckRun ErrorCode::CheckRun
) );
}
repo.set_clean();
tr_info!("Integrity verified") tr_info!("Integrity verified")
} }
Arguments::List { Arguments::List {
@ -890,15 +853,8 @@ pub fn run() -> Result<(), ErrorCode> {
ErrorCode::LoadInode ErrorCode::LoadInode
); );
println!("{}", format_inode_one_line(&inode)); println!("{}", format_inode_one_line(&inode));
if let Some(children) = inode.children { for ch in checked!(repo.get_inode_children(&inode), "load inodes", ErrorCode::LoadInode) {
for chunks in children.values() { println!("- {}", format_inode_one_line(&ch));
let inode = checked!(
repo.get_inode(chunks),
"load child inode",
ErrorCode::LoadInode
);
println!("- {}", format_inode_one_line(&inode));
}
} }
return Ok(()); return Ok(());
} }
@ -969,10 +925,15 @@ pub fn run() -> Result<(), ErrorCode> {
mount_point mount_point
} => { } => {
let mut repo = try!(open_repository(&repo_path, true)); let mut repo = try!(open_repository(&repo_path, true));
let fs = if let Some(backup_name) = backup_name { tr_info!("Mounting the filesystem...");
tr_info!(
"Please unmount the filesystem via 'fusermount -u {}' when done.",
mount_point
);
if let Some(backup_name) = backup_name {
if repo.get_layout().backups_path().join(&backup_name).is_dir() { if repo.get_layout().backups_path().join(&backup_name).is_dir() {
checked!( checked!(
FuseFilesystem::from_repository(&mut repo, Some(&backup_name)), repo.mount_repository(Some(&backup_name), mount_point),
"create fuse filesystem", "create fuse filesystem",
ErrorCode::FuseMount ErrorCode::FuseMount
) )
@ -985,13 +946,13 @@ pub fn run() -> Result<(), ErrorCode> {
ErrorCode::LoadInode ErrorCode::LoadInode
); );
checked!( checked!(
FuseFilesystem::from_inode(&mut repo, backup, inode), repo.mount_inode(backup, inode, mount_point),
"create fuse filesystem", "create fuse filesystem",
ErrorCode::FuseMount ErrorCode::FuseMount
) )
} else { } else {
checked!( checked!(
FuseFilesystem::from_backup(&mut repo, backup), repo.mount_backup(backup, mount_point),
"create fuse filesystem", "create fuse filesystem",
ErrorCode::FuseMount ErrorCode::FuseMount
) )
@ -999,21 +960,11 @@ pub fn run() -> Result<(), ErrorCode> {
} }
} else { } else {
checked!( checked!(
FuseFilesystem::from_repository(&mut repo, None), repo.mount_repository(None, mount_point),
"create fuse filesystem", "create fuse filesystem",
ErrorCode::FuseMount ErrorCode::FuseMount
) )
}; }
tr_info!("Mounting the filesystem...");
tr_info!(
"Please unmount the filesystem via 'fusermount -u {}' when done.",
mount_point
);
checked!(
fs.mount(&mount_point),
"mount filesystem",
ErrorCode::FuseMount
);
} }
Arguments::Analyze { repo_path } => { Arguments::Analyze { repo_path } => {
let mut repo = try!(open_repository(&repo_path, true)); let mut repo = try!(open_repository(&repo_path, true));

5
src/prelude.rs
View File

@ -3,13 +3,14 @@ pub use repository::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInf
BundleDb, BundleWriterError, StoredBundle, BundleStatistics}; BundleDb, BundleWriterError, StoredBundle, BundleStatistics};
pub use repository::chunking::{ChunkerType, Chunker, ChunkerStatus, ChunkerError}; pub use repository::chunking::{ChunkerType, Chunker, ChunkerStatus, ChunkerError};
pub use repository::{Repository, Config, RepositoryError, RepositoryInfo, pub use repository::{Repository, Config, RepositoryError, RepositoryInfo,
IntegrityError, BundleAnalysis, RepositoryLayout, Location, IntegrityError, BundleAnalysis, Location,
RepositoryStatistics, ChunkRepositoryLayout}; RepositoryStatistics, ChunkRepositoryLayout};
pub use repository::*; pub use repository::*;
pub use repository::index::{Index, IndexError, IndexStatistics}; pub use repository::index::{Index, IndexError, IndexStatistics};
pub use backups::mount::FuseFilesystem; pub use backups::mount::FuseFilesystem;
pub use backups::{BackupFile, BackupFileError, Inode, FileType, FileData, InodeError, BackupError, pub use backups::{BackupFile, BackupFileError, Inode, FileType, FileData, InodeError, BackupError,
BackupOptions, DiffType, InodeIntegrityError}; BackupOptions, DiffType, InodeIntegrityError, BackupRepositoryLayout,
RepositoryBackupIO, RepositoryMetadataIO, CheckOptions};
pub use translation::CowStr; pub use translation::CowStr;
pub use backups::BackupRepository; pub use backups::BackupRepository;

26
src/repository/integrity.rs
View File

@ -63,14 +63,18 @@ pub struct ChunkMarker<'a> {
repo: &'a Repository repo: &'a Repository
} }
impl<'a> ChunkMarker<'a> { impl Repository {
pub fn mark_chunks(&mut self, chunks: &[Chunk], set_marked: bool) -> Result<bool, RepositoryError> { pub fn get_chunk_marker(&self) -> Bitmap {
Bitmap::new(self.index.capacity())
}
pub fn mark_chunks(&mut self, bitmap: &mut Bitmap, chunks: &[Chunk], set_marked: bool) -> Result<bool, RepositoryError> {
let mut new = false; let mut new = false;
for &(hash, _len) in chunks { for &(hash, _len) in chunks {
if let Some(pos) = self.repo.index.pos(&hash) { if let Some(pos) = self.index.pos(&hash) {
new |= !self.marked.get(pos); new |= !bitmap.get(pos);
if set_marked { if set_marked {
self.marked.set(pos); bitmap.set(pos);
} }
} else { } else {
return Err(IntegrityError::MissingChunk(hash).into()); return Err(IntegrityError::MissingChunk(hash).into());
@ -78,16 +82,6 @@ impl<'a> ChunkMarker<'a> {
} }
Ok(new) Ok(new)
} }
}
impl Repository {
pub fn get_chunk_marker(&self) -> ChunkMarker {
ChunkMarker {
marked: Bitmap::new( self.index.capacity()),
repo: self
}
}
pub fn check_bundle_map(&mut self) -> ModuleIntegrityReport { pub fn check_bundle_map(&mut self) -> ModuleIntegrityReport {
tr_info!("Checking bundle map..."); tr_info!("Checking bundle map...");
@ -212,7 +206,7 @@ impl Repository {
let mut errors = vec![]; let mut errors = vec![];
let mut bundles = vec![]; let mut bundles = vec![];
for (id, err) in self.bundles.check(full, lock) { for (id, err) in self.bundles.check(full, lock) {
bundles.push(id); bundles.push(id.clone());
errors.push(IntegrityError::BundleIntegrity(id, err)); errors.push(IntegrityError::BundleIntegrity(id, err));
} }
(ModuleIntegrityReport { errors_fixed: vec![], errors_unfixed: errors }, bundles) (ModuleIntegrityReport { errors_fixed: vec![], errors_unfixed: errors }, bundles)

82
src/repository/layout.rs
View File

@ -27,51 +27,7 @@ pub trait ChunkRepositoryLayout {
} }
#[derive(Clone)] fn bundle_path(bundle: &BundleId, mut folder: PathBuf, mut count: usize) -> PathBuf {
pub struct RepositoryLayout(PathBuf);
impl RepositoryLayout {
pub fn new<P: AsRef<Path>>(path: P) -> Self {
RepositoryLayout(path.as_ref().to_path_buf())
}
#[inline]
pub fn config_path(&self) -> PathBuf {
self.0.join("config.yaml")
}
#[inline]
pub fn keys_path(&self) -> PathBuf {
self.0.join("keys")
}
#[inline]
pub fn excludes_path(&self) -> PathBuf {
self.0.join("excludes")
}
#[inline]
pub fn backups_path(&self) -> PathBuf {
self.0.join("remote/backups")
}
#[inline]
pub fn backup_path(&self, name: &str) -> PathBuf {
self.backups_path().join(format!("{}.backup", name))
}
#[inline]
pub fn remote_exists(&self) -> bool {
self.remote_bundles_path().exists() && self.backups_path().exists() &&
self.remote_locks_path().exists()
}
#[inline]
pub fn remote_readme_path(&self) -> PathBuf {
self.0.join("remote/README.md")
}
fn bundle_path(&self, bundle: &BundleId, mut folder: PathBuf, mut count: usize) -> PathBuf {
let file = bundle.to_string().to_owned() + ".bundle"; let file = bundle.to_string().to_owned() + ".bundle";
{ {
let mut rest = &file as &str; let mut rest = &file as &str;
@ -85,15 +41,13 @@ impl RepositoryLayout {
} }
} }
folder.join(Path::new(&file)) folder.join(Path::new(&file))
}
} }
impl ChunkRepositoryLayout for RepositoryLayout { impl ChunkRepositoryLayout for PathBuf {
#[inline] #[inline]
fn base_path(&self) -> &Path { fn base_path(&self) -> &Path {
&self.0 &self
} }
#[inline] #[inline]
@ -103,52 +57,52 @@ impl ChunkRepositoryLayout for RepositoryLayout {
#[inline] #[inline]
fn index_path(&self) -> PathBuf { fn index_path(&self) -> PathBuf {
self.0.join("index") self.join("index")
} }
#[inline] #[inline]
fn bundle_map_path(&self) -> PathBuf { fn bundle_map_path(&self) -> PathBuf {
self.0.join("bundles.map") self.join("bundles.map")
} }
#[inline] #[inline]
fn local_locks_path(&self) -> PathBuf { fn local_locks_path(&self) -> PathBuf {
self.0.join("locks") self.join("locks")
} }
#[inline] #[inline]
fn remote_path(&self) -> PathBuf { fn remote_path(&self) -> PathBuf {
self.0.join("remote") self.join("remote")
} }
#[inline] #[inline]
fn remote_locks_path(&self) -> PathBuf { fn remote_locks_path(&self) -> PathBuf {
self.0.join("remote/locks") self.join("remote/locks")
} }
#[inline] #[inline]
fn remote_bundles_path(&self) -> PathBuf { fn remote_bundles_path(&self) -> PathBuf {
self.0.join("remote/bundles") self.join("remote/bundles")
} }
#[inline] #[inline]
fn local_bundles_path(&self) -> PathBuf { fn local_bundles_path(&self) -> PathBuf {
self.0.join("bundles/cached") self.join("bundles/cached")
} }
#[inline] #[inline]
fn remote_bundle_path(&self, _bundle: &BundleId, count: usize) -> PathBuf { fn remote_bundle_path(&self, _bundle: &BundleId, count: usize) -> PathBuf {
self.bundle_path(&BundleId::random(), self.remote_bundles_path(), count) bundle_path(&BundleId::random(), self.remote_bundles_path(), count)
} }
#[inline] #[inline]
fn local_bundle_path(&self, bundle: &BundleId, count: usize) -> PathBuf { fn local_bundle_path(&self, bundle: &BundleId, count: usize) -> PathBuf {
self.bundle_path(bundle, self.local_bundles_path(), count) bundle_path(bundle, self.local_bundles_path(), count)
} }
#[inline] #[inline]
fn temp_bundles_path(&self) -> PathBuf { fn temp_bundles_path(&self) -> PathBuf {
self.0.join("bundles/temp") self.join("bundles/temp")
} }
#[inline] #[inline]
@ -158,29 +112,29 @@ impl ChunkRepositoryLayout for RepositoryLayout {
#[inline] #[inline]
fn local_bundle_cache_path(&self) -> PathBuf { fn local_bundle_cache_path(&self) -> PathBuf {
self.0.join("bundles/local.cache") self.join("bundles/local.cache")
} }
#[inline] #[inline]
fn remote_bundle_cache_path(&self) -> PathBuf { fn remote_bundle_cache_path(&self) -> PathBuf {
self.0.join("bundles/remote.cache") self.join("bundles/remote.cache")
} }
#[inline] #[inline]
fn dirtyfile_path(&self) -> PathBuf { fn dirtyfile_path(&self) -> PathBuf {
self.0.join("dirty") self.join("dirty")
} }
#[inline] #[inline]
fn config_path(&self) -> PathBuf { fn config_path(&self) -> PathBuf {
self.0.join("config.yaml") self.join("config.yaml")
} }
#[inline] #[inline]
fn remote_readme_path(&self) -> PathBuf { fn remote_readme_path(&self) -> PathBuf {
self.0.join("remote/README.md") self.join("remote/README.md")
} }
} }

12
src/repository/mod.rs
View File

@ -22,9 +22,9 @@ use std::io::Write;
pub use self::error::RepositoryError; pub use self::error::RepositoryError;
pub use self::config::Config; pub use self::config::Config;
pub use self::layout::{RepositoryLayout, ChunkRepositoryLayout}; pub use self::layout::ChunkRepositoryLayout;
use self::bundle_map::BundleMap; use self::bundle_map::BundleMap;
pub use self::integrity::IntegrityError; pub use self::integrity::{IntegrityError, ModuleIntegrityReport, IntegrityReport};
pub use self::info::{BundleAnalysis, RepositoryInfo, RepositoryStatistics}; pub use self::info::{BundleAnalysis, RepositoryInfo, RepositoryStatistics};
const REPOSITORY_README: &[u8] = include_bytes!("../../docs/repository_readme.md"); const REPOSITORY_README: &[u8] = include_bytes!("../../docs/repository_readme.md");
@ -319,6 +319,14 @@ impl Repository {
pub fn set_config(&mut self, config: Config) { pub fn set_config(&mut self, config: Config) {
self.config = config; self.config = config;
} }
pub fn get_crypto(&self) -> Arc<Crypto> {
self.crypto.clone()
}
pub fn get_layout(&self) -> Arc<ChunkRepositoryLayout> {
self.layout.clone()
}
} }