2017-07-21 09:21:59 +00:00
|
|
|
use prelude::*;
|
2017-03-15 20:53:05 +00:00
|
|
|
|
2017-03-24 08:26:55 +00:00
|
|
|
use std::fs;
|
2017-03-21 09:52:48 +00:00
|
|
|
use std::path::{self, Path, PathBuf};
|
2017-03-22 20:42:43 +00:00
|
|
|
use std::collections::{HashMap, BTreeMap, VecDeque};
|
2017-03-24 08:26:55 +00:00
|
|
|
use std::os::linux::fs::MetadataExt;
|
2017-03-16 11:33:10 +00:00
|
|
|
|
|
|
|
use chrono::prelude::*;
|
2017-03-24 10:00:20 +00:00
|
|
|
use regex::RegexSet;
|
2017-04-12 12:08:21 +00:00
|
|
|
use users::{self, Users, Groups};
|
2017-03-15 20:53:05 +00:00
|
|
|
|
|
|
|
|
2017-03-22 08:19:16 +00:00
|
|
|
quick_error!{
|
|
|
|
#[derive(Debug)]
|
2017-03-23 07:24:27 +00:00
|
|
|
#[allow(unknown_lints,large_enum_variant)]
|
2017-03-22 08:19:16 +00:00
|
|
|
pub enum BackupError {
|
2018-03-10 15:35:40 +00:00
|
|
|
FailedPaths(backup: BackupFile, failed: Vec<PathBuf>) {
|
2018-02-24 12:19:51 +00:00
|
|
|
description(tr!("Some paths could not be backed up"))
|
|
|
|
display("{}", tr_format!("Backup error: some paths could not be backed up"))
|
2017-03-22 10:10:13 +00:00
|
|
|
}
|
2017-03-23 07:24:27 +00:00
|
|
|
RemoveRoot {
|
2018-02-24 12:19:51 +00:00
|
|
|
description(tr!("The root of a backup can not be removed"))
|
|
|
|
display("{}", tr_format!("Backup error: the root of a backup can not be removed"))
|
2017-03-23 07:24:27 +00:00
|
|
|
}
|
2017-03-22 08:19:16 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-03-24 08:26:55 +00:00
|
|
|
pub struct BackupOptions {
|
2017-03-24 10:00:20 +00:00
|
|
|
pub same_device: bool,
|
|
|
|
pub excludes: Option<RegexSet>
|
2017-03-24 08:26:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-03-29 21:24:26 +00:00
|
|
|
pub enum DiffType {
|
2017-07-21 09:21:59 +00:00
|
|
|
Add,
|
|
|
|
Mod,
|
|
|
|
Del
|
2017-03-29 21:24:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
pub trait RepositoryBackupIO {
|
|
|
|
|
|
|
|
fn get_all_backups(&self) -> Result<HashMap<String, BackupFile>, RepositoryError>;
|
|
|
|
fn get_backups<P: AsRef<Path>>(&self, path: P
|
|
|
|
) -> Result<HashMap<String, BackupFile>, RepositoryError>;
|
|
|
|
fn has_backup(&self, name: &str) -> bool;
|
|
|
|
fn get_backup(&self, name: &str) -> Result<BackupFile, RepositoryError>;
|
|
|
|
fn save_backup(&mut self, backup: &BackupFile, name: &str, lock: &BackupMode
|
|
|
|
) -> Result<(), RepositoryError>;
|
|
|
|
fn delete_backup(&mut self, name: &str, lock: &BackupMode) -> Result<(), RepositoryError>;
|
|
|
|
fn prune_backups(&mut self, prefix: &str, daily: usize, weekly: usize, monthly: usize,
|
|
|
|
yearly: usize, force: bool, lock: &BackupMode) -> Result<(), RepositoryError>;
|
|
|
|
fn restore_inode_tree<P: AsRef<Path>>(&mut self, backup: &BackupFile, inode: Inode, path: P,
|
|
|
|
lock: &OnlineMode) -> Result<(), RepositoryError>;
|
|
|
|
fn create_backup_recurse<P: AsRef<Path>>(&mut self, path: P, reference: Option<&Inode>,
|
|
|
|
options: &BackupOptions, backup: &mut BackupFile, failed_paths: &mut Vec<PathBuf>,
|
|
|
|
lock: &BackupMode) -> Result<Inode, RepositoryError>;
|
|
|
|
fn create_backup<P: AsRef<Path>>(&mut self, path: P, name: &str,
|
|
|
|
reference: Option<&BackupFile>, options: &BackupOptions, lock: &BackupMode
|
|
|
|
) -> Result<BackupFile, RepositoryError>;
|
2018-10-26 12:05:28 +00:00
|
|
|
fn remove_backup_path<P: AsRef<Path>>(&mut self, backup: &mut BackupFile, name: &str, path: P,
|
2018-08-07 09:31:50 +00:00
|
|
|
lock: &BackupMode) -> Result<(), RepositoryError>;
|
|
|
|
fn get_backup_path<P: AsRef<Path>>(&mut self, backup: &BackupFile, path: P, lock: &OnlineMode
|
|
|
|
) -> Result<Vec<Inode>, RepositoryError>;
|
|
|
|
fn get_backup_inode<P: AsRef<Path>>(&mut self, backup: &BackupFile, path: P, lock: &OnlineMode
|
|
|
|
) -> Result<Inode, RepositoryError>;
|
|
|
|
fn find_versions<P: AsRef<Path>>(&mut self, path: P, lock: &OnlineMode
|
|
|
|
) -> Result<Vec<(String, Inode)>, RepositoryError>;
|
|
|
|
fn find_differences_recurse(&mut self, inode1: &Inode, inode2: &Inode, path: PathBuf,
|
|
|
|
diffs: &mut Vec<(DiffType, PathBuf)>, lock: &OnlineMode) -> Result<(), RepositoryError>;
|
|
|
|
fn find_differences(&mut self, inode1: &Inode, inode2: &Inode, lock: &OnlineMode
|
|
|
|
) -> Result<Vec<(DiffType, PathBuf)>, RepositoryError>;
|
|
|
|
fn count_sizes_recursive(&mut self, inode: &Inode, sizes: &mut HashMap<u64, usize>,
|
|
|
|
min_size: u64, lock: &OnlineMode) -> Result<(), RepositoryError>;
|
|
|
|
fn find_duplicates_recursive(&mut self, inode: &Inode, path: &Path, sizes: &HashMap<u64, usize>,
|
|
|
|
hashes: &mut HashMap<Hash, (Vec<PathBuf>, u64)>, lock: &OnlineMode
|
|
|
|
) -> Result<(), RepositoryError>;
|
|
|
|
fn find_duplicates(&mut self, inode: &Inode, min_size: u64, lock: &OnlineMode
|
|
|
|
) -> Result<Vec<(Vec<PathBuf>, u64)>, RepositoryError>;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl RepositoryBackupIO for Repository {
|
|
|
|
fn get_all_backups(&self) -> Result<HashMap<String, BackupFile>, RepositoryError> {
|
|
|
|
Ok(try!(BackupFile::get_all_from(&self.get_crypto(),self.get_layout().backups_path())))
|
2017-03-15 20:53:05 +00:00
|
|
|
}
|
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
fn get_backups<P: AsRef<Path>>(&self, path: P) -> Result<HashMap<String, BackupFile>, RepositoryError> {
|
|
|
|
Ok(try!(BackupFile::get_all_from(&self.get_crypto(), self.get_layout().backups_path().join(path))))
|
2017-04-13 11:32:59 +00:00
|
|
|
}
|
|
|
|
|
2017-04-10 18:35:28 +00:00
|
|
|
#[inline]
|
2018-08-07 09:31:50 +00:00
|
|
|
fn has_backup(&self, name: &str) -> bool {
|
|
|
|
self.get_layout().backup_path(name).exists()
|
2017-04-09 16:48:38 +00:00
|
|
|
}
|
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
fn get_backup(&self, name: &str) -> Result<BackupFile, RepositoryError> {
|
|
|
|
Ok(try!(BackupFile::read_from(&self.get_crypto(), self.get_layout().backup_path(name))))
|
2017-03-15 20:53:05 +00:00
|
|
|
}
|
|
|
|
|
2018-10-26 12:05:28 +00:00
|
|
|
fn save_backup(&mut self, backup: &BackupFile, name: &str, _lock: &BackupMode) -> Result<(), RepositoryError> {
|
2018-08-07 09:31:50 +00:00
|
|
|
let path = self.get_layout().backup_path(name);
|
2017-03-18 15:54:43 +00:00
|
|
|
try!(fs::create_dir_all(path.parent().unwrap()));
|
2018-02-19 21:30:59 +00:00
|
|
|
try!(backup.save_to(
|
2018-08-07 09:31:50 +00:00
|
|
|
&self.get_crypto(),
|
2018-03-10 15:35:40 +00:00
|
|
|
self.get_config().encryption.clone(),
|
2017-07-21 09:21:59 +00:00
|
|
|
path
|
2018-02-19 21:30:59 +00:00
|
|
|
));
|
|
|
|
Ok(())
|
2017-03-15 20:53:05 +00:00
|
|
|
}
|
|
|
|
|
2018-10-26 12:05:28 +00:00
|
|
|
fn delete_backup(&mut self, name: &str, _lock: &BackupMode) -> Result<(), RepositoryError> {
|
2018-08-07 09:31:50 +00:00
|
|
|
let mut path = self.get_layout().backup_path(name);
|
2017-03-18 15:54:43 +00:00
|
|
|
try!(fs::remove_file(&path));
|
|
|
|
loop {
|
|
|
|
path = path.parent().unwrap().to_owned();
|
2018-08-07 09:31:50 +00:00
|
|
|
if path == self.get_layout().backups_path() || fs::remove_dir(&path).is_err() {
|
2017-07-21 09:21:59 +00:00
|
|
|
break;
|
2017-03-18 15:54:43 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2017-03-20 17:11:03 +00:00
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
fn prune_backups(&mut self, prefix: &str, daily: usize, weekly: usize, monthly: usize,
|
|
|
|
yearly: usize, force: bool, lock: &BackupMode) -> Result<(), RepositoryError>
|
|
|
|
{
|
2017-03-20 14:38:33 +00:00
|
|
|
let mut backups = Vec::new();
|
2017-04-13 11:32:59 +00:00
|
|
|
let backup_map = match self.get_all_backups() {
|
2017-03-22 08:19:16 +00:00
|
|
|
Ok(backup_map) => backup_map,
|
2017-07-21 09:21:59 +00:00
|
|
|
Err(RepositoryError::BackupFile(BackupFileError::PartialBackupsList(backup_map,
|
|
|
|
_failed))) => {
|
2018-02-24 12:19:51 +00:00
|
|
|
tr_warn!("Some backups could not be read, ignoring them");
|
2017-03-22 08:19:16 +00:00
|
|
|
backup_map
|
2017-07-21 09:21:59 +00:00
|
|
|
}
|
|
|
|
Err(err) => return Err(err),
|
2017-03-22 08:19:16 +00:00
|
|
|
};
|
2017-03-21 09:52:48 +00:00
|
|
|
for (name, backup) in backup_map {
|
2017-03-20 14:38:33 +00:00
|
|
|
if name.starts_with(prefix) {
|
2017-04-14 20:46:55 +00:00
|
|
|
let date = Local.timestamp(backup.timestamp, 0);
|
2017-03-20 14:38:33 +00:00
|
|
|
backups.push((name, date, backup));
|
|
|
|
}
|
|
|
|
}
|
2017-04-14 20:46:55 +00:00
|
|
|
backups.sort_by_key(|backup| -backup.2.timestamp);
|
2017-03-20 14:38:33 +00:00
|
|
|
let mut keep = Bitmap::new(backups.len());
|
2017-03-20 17:11:03 +00:00
|
|
|
|
2017-07-21 09:21:59 +00:00
|
|
|
fn mark_needed<K: Eq, F: Fn(&DateTime<Local>) -> K>(
|
2018-03-10 15:35:40 +00:00
|
|
|
backups: &[(String, DateTime<Local>, BackupFile)],
|
2017-07-21 09:21:59 +00:00
|
|
|
keep: &mut Bitmap,
|
|
|
|
max: usize,
|
|
|
|
keyfn: F,
|
|
|
|
) {
|
2017-03-25 12:09:45 +00:00
|
|
|
let mut kept = 0;
|
2017-03-20 14:38:33 +00:00
|
|
|
let mut last = None;
|
|
|
|
for (i, backup) in backups.iter().enumerate() {
|
2017-03-20 17:11:03 +00:00
|
|
|
let val = keyfn(&backup.1);
|
|
|
|
let cur = Some(val);
|
|
|
|
if cur != last {
|
2017-03-25 12:09:45 +00:00
|
|
|
if kept >= max {
|
2017-07-21 09:21:59 +00:00
|
|
|
break;
|
2017-03-20 14:38:33 +00:00
|
|
|
}
|
2017-03-25 12:09:45 +00:00
|
|
|
last = cur;
|
|
|
|
keep.set(i);
|
|
|
|
kept += 1;
|
2017-03-20 14:38:33 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-04-07 16:57:49 +00:00
|
|
|
if yearly > 0 {
|
|
|
|
mark_needed(&backups, &mut keep, yearly, |d| d.year());
|
2017-03-20 17:11:03 +00:00
|
|
|
}
|
2017-04-07 16:57:49 +00:00
|
|
|
if monthly > 0 {
|
|
|
|
mark_needed(&backups, &mut keep, monthly, |d| (d.year(), d.month()));
|
2017-03-20 14:38:33 +00:00
|
|
|
}
|
2017-04-07 16:57:49 +00:00
|
|
|
if weekly > 0 {
|
2017-06-27 12:06:19 +00:00
|
|
|
mark_needed(&backups, &mut keep, weekly, |d| {
|
|
|
|
let week = d.iso_week();
|
|
|
|
(week.year(), week.week())
|
|
|
|
});
|
2017-03-20 14:38:33 +00:00
|
|
|
}
|
2017-04-07 16:57:49 +00:00
|
|
|
if daily > 0 {
|
2017-07-21 09:21:59 +00:00
|
|
|
mark_needed(
|
|
|
|
&backups,
|
|
|
|
&mut keep,
|
|
|
|
daily,
|
|
|
|
|d| (d.year(), d.month(), d.day())
|
|
|
|
);
|
2017-03-20 14:38:33 +00:00
|
|
|
}
|
|
|
|
let mut remove = Vec::new();
|
2017-04-09 10:04:28 +00:00
|
|
|
println!("Removing the following backups");
|
2017-03-20 14:38:33 +00:00
|
|
|
for (i, backup) in backups.into_iter().enumerate() {
|
|
|
|
if !keep.get(i) {
|
2017-04-07 09:05:28 +00:00
|
|
|
println!(" - {}", backup.0);
|
2017-03-20 14:38:33 +00:00
|
|
|
remove.push(backup.0);
|
|
|
|
}
|
|
|
|
}
|
2017-03-20 17:11:03 +00:00
|
|
|
if force {
|
2017-03-20 14:38:33 +00:00
|
|
|
for name in remove {
|
2018-08-07 09:31:50 +00:00
|
|
|
try!(self.delete_backup(&name, lock));
|
2017-03-20 14:38:33 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
fn restore_inode_tree<P: AsRef<Path>>(&mut self, backup: &BackupFile, inode: Inode, path: P, lock: &OnlineMode) -> Result<(), RepositoryError> {
|
2017-03-16 12:59:57 +00:00
|
|
|
let mut queue = VecDeque::new();
|
2017-03-16 13:14:35 +00:00
|
|
|
queue.push_back((path.as_ref().to_owned(), inode));
|
2017-04-12 12:08:21 +00:00
|
|
|
let cache = users::UsersCache::new();
|
2017-05-11 08:47:21 +00:00
|
|
|
let mut is_root = true;
|
2017-04-12 12:08:21 +00:00
|
|
|
while let Some((path, mut inode)) = queue.pop_front() {
|
2017-05-11 08:47:21 +00:00
|
|
|
if inode.file_type != FileType::Directory || !is_root {
|
|
|
|
if let Some(name) = backup.user_names.get(&inode.user) {
|
|
|
|
if let Some(user) = cache.get_user_by_name(name) {
|
|
|
|
inode.user = user.uid();
|
|
|
|
}
|
2017-04-12 12:08:21 +00:00
|
|
|
}
|
2017-05-11 08:47:21 +00:00
|
|
|
if let Some(name) = backup.group_names.get(&inode.group) {
|
|
|
|
if let Some(group) = cache.get_group_by_name(name) {
|
|
|
|
inode.group = group.gid();
|
|
|
|
}
|
2017-04-12 12:08:21 +00:00
|
|
|
}
|
2018-08-07 09:31:50 +00:00
|
|
|
try!(self.save_inode_at(&inode, &path, lock));
|
2017-04-12 12:08:21 +00:00
|
|
|
}
|
2017-03-16 12:59:57 +00:00
|
|
|
if inode.file_type == FileType::Directory {
|
2017-07-21 09:21:59 +00:00
|
|
|
let path = if is_root {
|
|
|
|
path.to_path_buf()
|
|
|
|
} else {
|
|
|
|
path.join(inode.name)
|
|
|
|
};
|
2017-03-16 12:59:57 +00:00
|
|
|
for chunks in inode.children.unwrap().values() {
|
2018-08-07 09:31:50 +00:00
|
|
|
let inode = try!(self.get_inode(chunks, lock));
|
2017-03-16 12:59:57 +00:00
|
|
|
queue.push_back((path.clone(), inode));
|
|
|
|
}
|
|
|
|
}
|
2017-05-11 08:47:21 +00:00
|
|
|
is_root = false;
|
2017-03-16 12:59:57 +00:00
|
|
|
}
|
2017-03-15 20:53:05 +00:00
|
|
|
Ok(())
|
|
|
|
}
|
2017-03-15 21:14:50 +00:00
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
fn create_backup_recurse<P: AsRef<Path>>(&mut self, path: P, reference: Option<&Inode>,
|
|
|
|
options: &BackupOptions, backup: &mut BackupFile, failed_paths: &mut Vec<PathBuf>,
|
|
|
|
lock: &BackupMode
|
2017-04-02 18:37:34 +00:00
|
|
|
) -> Result<Inode, RepositoryError> {
|
2017-03-23 08:31:23 +00:00
|
|
|
let path = path.as_ref();
|
2018-08-07 09:31:50 +00:00
|
|
|
let mut inode = try!(self.create_inode(path, reference, lock));
|
2017-04-12 12:08:21 +00:00
|
|
|
if !backup.user_names.contains_key(&inode.user) {
|
|
|
|
if let Some(user) = users::get_user_by_uid(inode.user) {
|
2017-07-21 09:21:59 +00:00
|
|
|
backup.user_names.insert(
|
|
|
|
inode.user,
|
|
|
|
user.name().to_string()
|
|
|
|
);
|
2017-04-12 12:08:21 +00:00
|
|
|
} else {
|
2018-02-24 12:19:51 +00:00
|
|
|
tr_warn!("Failed to retrieve name of user {}", inode.user);
|
2017-04-12 12:08:21 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
if !backup.group_names.contains_key(&inode.group) {
|
|
|
|
if let Some(group) = users::get_group_by_gid(inode.group) {
|
2017-07-21 09:21:59 +00:00
|
|
|
backup.group_names.insert(
|
|
|
|
inode.group,
|
|
|
|
group.name().to_string()
|
|
|
|
);
|
2017-04-12 12:08:21 +00:00
|
|
|
} else {
|
2018-02-24 12:19:51 +00:00
|
|
|
tr_warn!("Failed to retrieve name of group {}", inode.group);
|
2017-04-12 12:08:21 +00:00
|
|
|
}
|
|
|
|
}
|
2017-04-08 08:18:46 +00:00
|
|
|
let mut meta_size = 0;
|
|
|
|
inode.cum_size = inode.size;
|
2017-03-23 08:31:23 +00:00
|
|
|
if inode.file_type == FileType::Directory {
|
2017-04-02 18:37:34 +00:00
|
|
|
inode.cum_dirs = 1;
|
2017-03-23 08:31:23 +00:00
|
|
|
let mut children = BTreeMap::new();
|
2017-03-24 08:26:55 +00:00
|
|
|
let parent_dev = try!(path.metadata()).st_dev();
|
2017-03-23 08:31:23 +00:00
|
|
|
for ch in try!(fs::read_dir(path)) {
|
|
|
|
let child = try!(ch);
|
2017-03-24 10:00:20 +00:00
|
|
|
let child_path = child.path();
|
2017-03-24 08:26:55 +00:00
|
|
|
if options.same_device {
|
|
|
|
let child_dev = try!(child.metadata()).st_dev();
|
|
|
|
if child_dev != parent_dev {
|
2017-07-21 09:21:59 +00:00
|
|
|
continue;
|
2017-03-24 08:26:55 +00:00
|
|
|
}
|
|
|
|
}
|
2017-03-24 10:00:20 +00:00
|
|
|
if let Some(ref excludes) = options.excludes {
|
|
|
|
let child_path_str = child_path.to_string_lossy();
|
|
|
|
if excludes.is_match(&child_path_str) {
|
2017-07-21 09:21:59 +00:00
|
|
|
continue;
|
2017-03-24 10:00:20 +00:00
|
|
|
}
|
|
|
|
}
|
2017-03-23 08:31:23 +00:00
|
|
|
let name = child.file_name().to_string_lossy().to_string();
|
2017-07-21 09:21:59 +00:00
|
|
|
let ref_child = reference
|
|
|
|
.as_ref()
|
2017-03-23 08:31:23 +00:00
|
|
|
.and_then(|inode| inode.children.as_ref())
|
|
|
|
.and_then(|map| map.get(&name))
|
2018-08-07 09:31:50 +00:00
|
|
|
.and_then(|chunks| self.get_inode(chunks, lock.as_online()).ok());
|
2017-07-21 09:21:59 +00:00
|
|
|
let child_inode = match self.create_backup_recurse(
|
|
|
|
&child_path,
|
|
|
|
ref_child.as_ref(),
|
|
|
|
options,
|
|
|
|
backup,
|
2018-08-07 09:31:50 +00:00
|
|
|
failed_paths,
|
|
|
|
lock
|
2017-07-21 09:21:59 +00:00
|
|
|
) {
|
2017-04-02 18:37:34 +00:00
|
|
|
Ok(inode) => inode,
|
2017-07-21 09:21:59 +00:00
|
|
|
Err(RepositoryError::Inode(_)) |
|
|
|
|
Err(RepositoryError::Chunker(_)) |
|
|
|
|
Err(RepositoryError::Io(_)) => {
|
2017-04-09 10:04:28 +00:00
|
|
|
info!("Failed to backup {:?}", child_path);
|
2017-03-23 08:31:23 +00:00
|
|
|
failed_paths.push(child_path);
|
2017-07-21 09:21:59 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
Err(err) => return Err(err),
|
2017-03-23 08:31:23 +00:00
|
|
|
};
|
2018-08-07 09:31:50 +00:00
|
|
|
let chunks = try!(self.put_inode(&child_inode, lock));
|
2017-04-02 18:37:34 +00:00
|
|
|
inode.cum_size += child_inode.cum_size;
|
2017-04-08 08:18:46 +00:00
|
|
|
for &(_, len) in chunks.iter() {
|
2018-02-19 21:30:59 +00:00
|
|
|
meta_size += u64::from(len);
|
2017-04-08 08:18:46 +00:00
|
|
|
}
|
2017-04-02 18:37:34 +00:00
|
|
|
inode.cum_dirs += child_inode.cum_dirs;
|
|
|
|
inode.cum_files += child_inode.cum_files;
|
2017-04-08 08:18:46 +00:00
|
|
|
children.insert(name, chunks);
|
2017-03-23 08:31:23 +00:00
|
|
|
}
|
|
|
|
inode.children = Some(children);
|
|
|
|
} else {
|
2017-04-02 18:37:34 +00:00
|
|
|
inode.cum_files = 1;
|
2017-04-08 08:18:46 +00:00
|
|
|
if let Some(FileData::ChunkedIndirect(ref chunks)) = inode.data {
|
|
|
|
for &(_, len) in chunks.iter() {
|
2018-02-19 21:30:59 +00:00
|
|
|
meta_size += u64::from(len);
|
2017-04-08 08:18:46 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
inode.cum_size += meta_size;
|
|
|
|
if let Some(ref_inode) = reference {
|
|
|
|
if !ref_inode.is_same_meta_quick(&inode) {
|
|
|
|
backup.changed_data_size += inode.size + meta_size;
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
backup.changed_data_size += inode.size + meta_size;
|
2017-03-23 08:31:23 +00:00
|
|
|
}
|
2017-04-02 18:37:34 +00:00
|
|
|
Ok(inode)
|
2017-03-23 08:31:23 +00:00
|
|
|
}
|
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
fn create_backup<P: AsRef<Path>>(&mut self, path: P, name: &str,
|
|
|
|
reference: Option<&BackupFile>, options: &BackupOptions, lock: &BackupMode
|
2018-03-10 15:35:40 +00:00
|
|
|
) -> Result<BackupFile, RepositoryError> {
|
2018-08-07 09:31:50 +00:00
|
|
|
let reference_inode = reference.and_then(|b| self.get_inode(&b.root, lock.as_online()).ok());
|
2018-03-10 15:35:40 +00:00
|
|
|
let mut backup = BackupFile::default();
|
|
|
|
backup.config = self.get_config().clone();
|
2017-03-20 21:24:53 +00:00
|
|
|
backup.host = get_hostname().unwrap_or_else(|_| "".to_string());
|
|
|
|
backup.path = path.as_ref().to_string_lossy().to_string();
|
2017-03-16 11:33:10 +00:00
|
|
|
let info_before = self.info();
|
|
|
|
let start = Local::now();
|
2017-03-22 10:10:13 +00:00
|
|
|
let mut failed_paths = vec![];
|
2017-07-21 09:21:59 +00:00
|
|
|
let root_inode = try!(self.create_backup_recurse(
|
|
|
|
path,
|
|
|
|
reference_inode.as_ref(),
|
|
|
|
options,
|
|
|
|
&mut backup,
|
2018-08-07 09:31:50 +00:00
|
|
|
&mut failed_paths,
|
|
|
|
lock
|
2017-07-21 09:21:59 +00:00
|
|
|
));
|
2018-08-07 09:31:50 +00:00
|
|
|
backup.root = try!(self.put_inode(&root_inode, lock));
|
|
|
|
try!(self.flush(lock));
|
2017-03-16 11:33:10 +00:00
|
|
|
let elapsed = Local::now().signed_duration_since(start);
|
2017-04-14 20:46:55 +00:00
|
|
|
backup.timestamp = start.timestamp();
|
2017-04-02 18:37:34 +00:00
|
|
|
backup.total_data_size = root_inode.cum_size;
|
2017-04-08 08:18:46 +00:00
|
|
|
for &(_, len) in backup.root.iter() {
|
2018-02-19 21:30:59 +00:00
|
|
|
backup.total_data_size += u64::from(len);
|
2017-04-08 08:18:46 +00:00
|
|
|
}
|
2017-04-02 18:37:34 +00:00
|
|
|
backup.file_count = root_inode.cum_files;
|
|
|
|
backup.dir_count = root_inode.cum_dirs;
|
2017-03-16 11:33:10 +00:00
|
|
|
backup.duration = elapsed.num_milliseconds() as f32 / 1_000.0;
|
|
|
|
let info_after = self.info();
|
|
|
|
backup.deduplicated_data_size = info_after.raw_data_size - info_before.raw_data_size;
|
|
|
|
backup.encoded_data_size = info_after.encoded_data_size - info_before.encoded_data_size;
|
|
|
|
backup.bundle_count = info_after.bundle_count - info_before.bundle_count;
|
|
|
|
backup.chunk_count = info_after.chunk_count - info_before.chunk_count;
|
|
|
|
backup.avg_chunk_size = backup.deduplicated_data_size as f32 / backup.chunk_count as f32;
|
2018-08-07 09:31:50 +00:00
|
|
|
try!(self.save_backup(&backup, name, lock));
|
2017-03-22 10:10:13 +00:00
|
|
|
if failed_paths.is_empty() {
|
|
|
|
Ok(backup)
|
|
|
|
} else {
|
|
|
|
Err(BackupError::FailedPaths(backup, failed_paths).into())
|
|
|
|
}
|
2017-03-15 21:14:50 +00:00
|
|
|
}
|
2017-03-16 12:59:57 +00:00
|
|
|
|
2018-10-26 12:05:28 +00:00
|
|
|
fn remove_backup_path<P: AsRef<Path>>(&mut self, backup: &mut BackupFile, name: &str, path: P,
|
2018-08-07 09:31:50 +00:00
|
|
|
lock: &BackupMode
|
2017-07-21 09:21:59 +00:00
|
|
|
) -> Result<(), RepositoryError> {
|
2018-08-07 09:31:50 +00:00
|
|
|
let mut inodes = try!(self.get_backup_path(backup, path, lock.as_online()));
|
2017-03-23 07:24:27 +00:00
|
|
|
let to_remove = inodes.pop().unwrap();
|
|
|
|
let mut remove_from = match inodes.pop() {
|
|
|
|
Some(inode) => inode,
|
2017-07-21 09:21:59 +00:00
|
|
|
None => return Err(BackupError::RemoveRoot.into()),
|
2017-03-23 07:24:27 +00:00
|
|
|
};
|
2017-07-21 09:21:59 +00:00
|
|
|
remove_from.children.as_mut().unwrap().remove(
|
|
|
|
&to_remove.name
|
|
|
|
);
|
2018-08-07 09:31:50 +00:00
|
|
|
let mut last_inode_chunks = try!(self.put_inode(&remove_from, lock));
|
2017-03-23 07:24:27 +00:00
|
|
|
let mut last_inode_name = remove_from.name;
|
|
|
|
while let Some(mut inode) = inodes.pop() {
|
2017-07-21 09:21:59 +00:00
|
|
|
inode.children.as_mut().unwrap().insert(
|
|
|
|
last_inode_name,
|
|
|
|
last_inode_chunks
|
|
|
|
);
|
2018-08-07 09:31:50 +00:00
|
|
|
last_inode_chunks = try!(self.put_inode(&inode, lock));
|
2017-03-23 07:24:27 +00:00
|
|
|
last_inode_name = inode.name;
|
|
|
|
}
|
|
|
|
backup.root = last_inode_chunks;
|
2017-04-03 13:18:06 +00:00
|
|
|
backup.modified = true;
|
2018-10-26 12:05:28 +00:00
|
|
|
self.save_backup(backup, name, lock)
|
2017-03-23 07:24:27 +00:00
|
|
|
}
|
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
fn get_backup_path<P: AsRef<Path>>(&mut self, backup: &BackupFile, path: P, lock: &OnlineMode
|
2017-07-21 09:21:59 +00:00
|
|
|
) -> Result<Vec<Inode>, RepositoryError> {
|
2017-03-23 07:24:27 +00:00
|
|
|
let mut inodes = vec![];
|
2018-08-07 09:31:50 +00:00
|
|
|
let mut inode = try!(self.get_inode(&backup.root, lock));
|
2017-03-16 12:59:57 +00:00
|
|
|
for c in path.as_ref().components() {
|
|
|
|
if let path::Component::Normal(name) = c {
|
|
|
|
let name = name.to_string_lossy();
|
2017-07-21 09:21:59 +00:00
|
|
|
if inodes.is_empty() && inode.file_type != FileType::Directory &&
|
|
|
|
inode.name == name
|
|
|
|
{
|
2017-04-13 12:42:56 +00:00
|
|
|
return Ok(vec![inode]);
|
|
|
|
}
|
2017-07-21 09:21:59 +00:00
|
|
|
if let Some(chunks) = inode.children.as_mut().and_then(
|
|
|
|
|c| c.remove(&name as &str)
|
|
|
|
)
|
|
|
|
{
|
2017-03-23 07:24:27 +00:00
|
|
|
inodes.push(inode);
|
2018-08-07 09:31:50 +00:00
|
|
|
inode = try!(self.get_inode(&chunks, lock));
|
2017-03-16 12:59:57 +00:00
|
|
|
} else {
|
2017-07-21 09:21:59 +00:00
|
|
|
return Err(RepositoryError::NoSuchFileInBackup(
|
|
|
|
backup.clone(),
|
|
|
|
path.as_ref().to_owned()
|
|
|
|
));
|
2017-03-16 12:59:57 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-03-23 07:24:27 +00:00
|
|
|
inodes.push(inode);
|
|
|
|
Ok(inodes)
|
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
2018-08-07 09:31:50 +00:00
|
|
|
fn get_backup_inode<P: AsRef<Path>>(&mut self, backup: &BackupFile, path: P, lock: &OnlineMode
|
2017-07-21 09:21:59 +00:00
|
|
|
) -> Result<Inode, RepositoryError> {
|
2018-08-07 09:31:50 +00:00
|
|
|
self.get_backup_path(backup, path, lock).map(|mut inodes| {
|
2017-07-21 09:21:59 +00:00
|
|
|
inodes.pop().unwrap()
|
|
|
|
})
|
2017-03-16 12:59:57 +00:00
|
|
|
}
|
2017-03-27 20:31:24 +00:00
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
fn find_versions<P: AsRef<Path>>(&mut self, path: P, lock: &OnlineMode
|
2017-07-21 09:21:59 +00:00
|
|
|
) -> Result<Vec<(String, Inode)>, RepositoryError> {
|
2017-03-27 20:31:24 +00:00
|
|
|
let path = path.as_ref();
|
|
|
|
let mut versions = HashMap::new();
|
2017-04-13 11:32:59 +00:00
|
|
|
for (name, backup) in try!(self.get_all_backups()) {
|
2018-08-07 09:31:50 +00:00
|
|
|
match self.get_backup_inode(&backup, path, lock) {
|
2017-03-27 20:31:24 +00:00
|
|
|
Ok(inode) => {
|
2017-07-21 09:21:59 +00:00
|
|
|
versions.insert(
|
|
|
|
(inode.file_type, inode.timestamp, inode.size),
|
|
|
|
(name, inode)
|
|
|
|
);
|
|
|
|
}
|
2017-03-27 20:31:24 +00:00
|
|
|
Err(RepositoryError::NoSuchFileInBackup(..)) => continue,
|
2017-07-21 09:21:59 +00:00
|
|
|
Err(err) => return Err(err),
|
2017-03-27 20:31:24 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
let mut versions: Vec<_> = versions.into_iter().map(|(_, v)| v).collect();
|
2017-04-02 16:55:53 +00:00
|
|
|
versions.sort_by_key(|v| v.1.timestamp);
|
2017-03-27 20:31:24 +00:00
|
|
|
Ok(versions)
|
|
|
|
}
|
2017-03-29 21:24:26 +00:00
|
|
|
|
2018-02-19 21:30:59 +00:00
|
|
|
#[allow(needless_pass_by_value)]
|
2018-08-07 09:31:50 +00:00
|
|
|
fn find_differences_recurse(&mut self, inode1: &Inode, inode2: &Inode, path: PathBuf,
|
|
|
|
diffs: &mut Vec<(DiffType, PathBuf)>, lock: &OnlineMode
|
2017-07-21 09:21:59 +00:00
|
|
|
) -> Result<(), RepositoryError> {
|
2017-04-03 05:35:00 +00:00
|
|
|
if !inode1.is_same_meta(inode2) || inode1.data != inode2.data {
|
2017-03-29 21:24:26 +00:00
|
|
|
diffs.push((DiffType::Mod, path.clone()));
|
|
|
|
}
|
|
|
|
if let Some(ref children1) = inode1.children {
|
|
|
|
if let Some(ref children2) = inode2.children {
|
|
|
|
for name in children1.keys() {
|
|
|
|
if !children2.contains_key(name) {
|
|
|
|
diffs.push((DiffType::Del, path.join(name)));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
for name in children1.keys() {
|
|
|
|
diffs.push((DiffType::Del, path.join(name)));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if let Some(ref children2) = inode2.children {
|
|
|
|
if let Some(ref children1) = inode1.children {
|
|
|
|
for (name, chunks2) in children2 {
|
|
|
|
if let Some(chunks1) = children1.get(name) {
|
|
|
|
if chunks1 != chunks2 {
|
2018-08-07 09:31:50 +00:00
|
|
|
let inode1 = try!(self.get_inode(chunks1, lock));
|
|
|
|
let inode2 = try!(self.get_inode(chunks2, lock));
|
2017-07-21 09:21:59 +00:00
|
|
|
try!(self.find_differences_recurse(
|
|
|
|
&inode1,
|
|
|
|
&inode2,
|
|
|
|
path.join(name),
|
2018-08-07 09:31:50 +00:00
|
|
|
diffs,
|
|
|
|
lock
|
2017-07-21 09:21:59 +00:00
|
|
|
));
|
2017-03-29 21:24:26 +00:00
|
|
|
}
|
|
|
|
} else {
|
|
|
|
diffs.push((DiffType::Add, path.join(name)));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
for name in children2.keys() {
|
|
|
|
diffs.push((DiffType::Add, path.join(name)));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
2018-08-07 09:31:50 +00:00
|
|
|
fn find_differences(&mut self, inode1: &Inode, inode2: &Inode, lock: &OnlineMode
|
2017-07-21 09:21:59 +00:00
|
|
|
) -> Result<Vec<(DiffType, PathBuf)>, RepositoryError> {
|
2017-03-29 21:24:26 +00:00
|
|
|
let mut diffs = vec![];
|
|
|
|
let path = PathBuf::from("/");
|
2017-07-21 09:21:59 +00:00
|
|
|
try!(self.find_differences_recurse(
|
|
|
|
inode1,
|
|
|
|
inode2,
|
|
|
|
path,
|
2018-08-07 09:31:50 +00:00
|
|
|
&mut diffs,
|
|
|
|
lock
|
2017-07-21 09:21:59 +00:00
|
|
|
));
|
2017-03-29 21:24:26 +00:00
|
|
|
Ok(diffs)
|
|
|
|
}
|
2018-03-08 14:20:20 +00:00
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
fn count_sizes_recursive(&mut self, inode: &Inode, sizes: &mut HashMap<u64, usize>,
|
|
|
|
min_size: u64, lock: &OnlineMode
|
|
|
|
) -> Result<(), RepositoryError> {
|
2018-03-08 14:20:20 +00:00
|
|
|
if inode.size >= min_size {
|
|
|
|
*sizes.entry(inode.size).or_insert(0) += 1;
|
|
|
|
}
|
|
|
|
if let Some(ref children) = inode.children {
|
|
|
|
for chunks in children.values() {
|
2018-08-07 09:31:50 +00:00
|
|
|
let ch = try!(self.get_inode(chunks, lock));
|
|
|
|
try!(self.count_sizes_recursive(&ch, sizes, min_size, lock));
|
2018-03-08 14:20:20 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
fn find_duplicates_recursive(&mut self, inode: &Inode, path: &Path, sizes: &HashMap<u64, usize>,
|
|
|
|
hashes: &mut HashMap<Hash, (Vec<PathBuf>, u64)>, lock: &OnlineMode
|
|
|
|
) -> Result<(), RepositoryError> {
|
2018-03-08 14:20:20 +00:00
|
|
|
let path = path.join(&inode.name);
|
|
|
|
if sizes.get(&inode.size).cloned().unwrap_or(0) > 1 {
|
|
|
|
if let Some(ref data) = inode.data {
|
|
|
|
let chunk_data = try!(msgpack::encode(data).map_err(InodeError::from));
|
|
|
|
let hash = HashMethod::Blake2.hash(&chunk_data);
|
|
|
|
hashes.entry(hash).or_insert((Vec::new(), inode.size)).0.push(path.clone());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if let Some(ref children) = inode.children {
|
|
|
|
for chunks in children.values() {
|
2018-08-07 09:31:50 +00:00
|
|
|
let ch = try!(self.get_inode(chunks, lock));
|
|
|
|
try!(self.find_duplicates_recursive(&ch, &path, sizes, hashes, lock));
|
2018-03-08 14:20:20 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2018-08-07 09:31:50 +00:00
|
|
|
fn find_duplicates(&mut self, inode: &Inode, min_size: u64, lock: &OnlineMode
|
|
|
|
) -> Result<Vec<(Vec<PathBuf>, u64)>, RepositoryError> {
|
2018-03-08 14:20:20 +00:00
|
|
|
let mut sizes = HashMap::new();
|
2018-08-07 09:31:50 +00:00
|
|
|
try!(self.count_sizes_recursive(inode, &mut sizes, min_size, lock));
|
2018-03-08 14:20:20 +00:00
|
|
|
let mut hashes = HashMap::new();
|
|
|
|
if let Some(ref children) = inode.children {
|
|
|
|
for chunks in children.values() {
|
2018-08-07 09:31:50 +00:00
|
|
|
let ch = try!(self.get_inode(chunks, lock));
|
|
|
|
try!(self.find_duplicates_recursive(&ch, Path::new(""), &sizes, &mut hashes, lock));
|
2018-03-08 14:20:20 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
let dups = hashes.into_iter().map(|(_,v)| v).filter(|&(ref v, _)| v.len() > 1).collect();
|
|
|
|
Ok(dups)
|
|
|
|
}
|
2017-03-15 20:53:05 +00:00
|
|
|
}
|