dswd
/
zvault
mirror of https://github.com/dswd/zvault
1
0
Fork 0
Code Issues Releases Wiki Activity

Refactored backups, vacuum, etc.

refactor
Dennis Schwerdel 2018-03-10 16:35:40 +01:00
parent 4287896945
commit f3f6a3cf49
17 changed files with 1082 additions and 1121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
src/repository/backup.rs → src/backups/backup.rs
View File

@ -14,7 +14,7 @@ quick_error!{
#[derive(Debug)]
#[allow(unknown_lints,large_enum_variant)]
pub enum BackupError {
FailedPaths(backup: Backup, failed: Vec<PathBuf>) {
FailedPaths(backup: BackupFile, failed: Vec<PathBuf>) {
description(tr!("Some paths could not be backed up"))
display("{}", tr_format!("Backup error: some paths could not be backed up"))
}
@ -39,9 +39,9 @@ pub enum DiffType {
}
impl Repository {
pub fn get_all_backups(&self) -> Result<HashMap<String, Backup>, RepositoryError> {
Ok(try!(Backup::get_all_from(
impl BackupRepository {
pub fn get_all_backups(&self) -> Result<HashMap<String, BackupFile>, RepositoryError> {
Ok(try!(BackupFile::get_all_from(
&self.crypto,
self.layout.backups_path()
)))
@ -50,8 +50,8 @@ impl Repository {
pub fn get_backups<P: AsRef<Path>>(
&self,
path: P,
) -> Result<HashMap<String, Backup>, RepositoryError> {
Ok(try!(Backup::get_all_from(
) -> Result<HashMap<String, BackupFile>, RepositoryError> {
Ok(try!(BackupFile::get_all_from(
&self.crypto,
self.layout.backups_path().join(path)
)))
@ -62,27 +62,27 @@ impl Repository {
self.layout.backup_path(name).exists()
}
pub fn get_backup(&self, name: &str) -> Result<Backup, RepositoryError> {
Ok(try!(Backup::read_from(
pub fn get_backup(&self, name: &str) -> Result<BackupFile, RepositoryError> {
Ok(try!(BackupFile::read_from(
&self.crypto,
self.layout.backup_path(name)
)))
}
pub fn save_backup(&mut self, backup: &Backup, name: &str) -> Result<(), RepositoryError> {
try!(self.write_mode());
pub fn save_backup(&mut self, backup: &BackupFile, name: &str) -> Result<(), RepositoryError> {
try!(self.repo.write_mode());
let path = self.layout.backup_path(name);
try!(fs::create_dir_all(path.parent().unwrap()));
try!(backup.save_to(
&self.crypto,
self.config.encryption.clone(),
self.get_config().encryption.clone(),
path
));
Ok(())
}
pub fn delete_backup(&mut self, name: &str) -> Result<(), RepositoryError> {
try!(self.write_mode());
try!(self.repo.write_mode());
let mut path = self.layout.backup_path(name);
try!(fs::remove_file(&path));
loop {
@ -104,7 +104,7 @@ impl Repository {
yearly: usize,
force: bool,
) -> Result<(), RepositoryError> {
try!(self.write_mode());
try!(self.repo.write_mode());
let mut backups = Vec::new();
let backup_map = match self.get_all_backups() {
Ok(backup_map) => backup_map,
@ -125,7 +125,7 @@ impl Repository {
let mut keep = Bitmap::new(backups.len());
fn mark_needed<K: Eq, F: Fn(&DateTime<Local>) -> K>(
backups: &[(String, DateTime<Local>, Backup)],
backups: &[(String, DateTime<Local>, BackupFile)],
keep: &mut Bitmap,
max: usize,
keyfn: F,
@ -183,11 +183,11 @@ impl Repository {
pub fn restore_inode_tree<P: AsRef<Path>>(
&mut self,
backup: &Backup,
backup: &BackupFile,
inode: Inode,
path: P,
) -> Result<(), RepositoryError> {
let _lock = try!(self.lock(false));
let _lock = try!(self.repo.lock(false));
let mut queue = VecDeque::new();
queue.push_back((path.as_ref().to_owned(), inode));
let cache = users::UsersCache::new();
@ -204,7 +204,7 @@ impl Repository {
inode.group = group.gid();
}
}
try!(self.save_inode_at(&inode, &path));
try!(self.repo.save_inode_at(&inode, &path));
}
if inode.file_type == FileType::Directory {
let path = if is_root {
@ -227,11 +227,11 @@ impl Repository {
path: P,
reference: Option<&Inode>,
options: &BackupOptions,
backup: &mut Backup,
backup: &mut BackupFile,
failed_paths: &mut Vec<PathBuf>,
) -> Result<Inode, RepositoryError> {
let path = path.as_ref();
let mut inode = try!(self.create_inode(path, reference));
let mut inode = try!(self.repo.create_inode(path, reference));
if !backup.user_names.contains_key(&inode.user) {
if let Some(user) = users::get_user_by_uid(inode.user) {
backup.user_names.insert(
@ -296,7 +296,7 @@ impl Repository {
}
Err(err) => return Err(err),
};
let chunks = try!(self.put_inode(&child_inode));
let chunks = try!(self.repo.put_inode(&child_inode));
inode.cum_size += child_inode.cum_size;
for &(_, len) in chunks.iter() {
meta_size += u64::from(len);
@ -328,18 +328,18 @@ impl Repository {
pub fn create_backup_recursively<P: AsRef<Path>>(
&mut self,
path: P,
reference: Option<&Backup>,
reference: Option<&BackupFile>,
options: &BackupOptions,
) -> Result<Backup, RepositoryError> {
try!(self.write_mode());
let _lock = try!(self.lock(false));
if self.dirty {
) -> Result<BackupFile, RepositoryError> {
try!(self.repo.write_mode());
let _lock = try!(self.repo.lock(false));
if self.repo.is_dirty() {
return Err(RepositoryError::Dirty);
}
try!(self.set_dirty());
try!(self.repo.set_dirty());
let reference_inode = reference.and_then(|b| self.get_inode(&b.root).ok());
let mut backup = Backup::default();
backup.config = self.config.clone();
let mut backup = BackupFile::default();
backup.config = self.get_config().clone();
backup.host = get_hostname().unwrap_or_else(|_| "".to_string());
backup.path = path.as_ref().to_string_lossy().to_string();
let info_before = self.info();
@ -352,8 +352,8 @@ impl Repository {
&mut backup,
&mut failed_paths
));
backup.root = try!(self.put_inode(&root_inode));
try!(self.flush());
backup.root = try!(self.repo.put_inode(&root_inode));
try!(self.repo.flush());
let elapsed = Local::now().signed_duration_since(start);
backup.timestamp = start.timestamp();
backup.total_data_size = root_inode.cum_size;
@ -369,7 +369,7 @@ impl Repository {
backup.bundle_count = info_after.bundle_count - info_before.bundle_count;
backup.chunk_count = info_after.chunk_count - info_before.chunk_count;
backup.avg_chunk_size = backup.deduplicated_data_size as f32 / backup.chunk_count as f32;
self.dirty = false;
self.repo.set_clean();
if failed_paths.is_empty() {
Ok(backup)
} else {
@ -379,11 +379,11 @@ impl Repository {
pub fn remove_backup_path<P: AsRef<Path>>(
&mut self,
backup: &mut Backup,
backup: &mut BackupFile,
path: P,
) -> Result<(), RepositoryError> {
try!(self.write_mode());
let _lock = try!(self.lock(false));
try!(self.repo.write_mode());
let _lock = try!(self.repo.lock(false));
let mut inodes = try!(self.get_backup_path(backup, path));
let to_remove = inodes.pop().unwrap();
let mut remove_from = match inodes.pop() {
@ -393,14 +393,14 @@ impl Repository {
remove_from.children.as_mut().unwrap().remove(
&to_remove.name
);
let mut last_inode_chunks = try!(self.put_inode(&remove_from));
let mut last_inode_chunks = try!(self.repo.put_inode(&remove_from));
let mut last_inode_name = remove_from.name;
while let Some(mut inode) = inodes.pop() {
inode.children.as_mut().unwrap().insert(
last_inode_name,
last_inode_chunks
);
last_inode_chunks = try!(self.put_inode(&inode));
last_inode_chunks = try!(self.repo.put_inode(&inode));
last_inode_name = inode.name;
}
backup.root = last_inode_chunks;
@ -410,7 +410,7 @@ impl Repository {
pub fn get_backup_path<P: AsRef<Path>>(
&mut self,
backup: &Backup,
backup: &BackupFile,
path: P,
) -> Result<Vec<Inode>, RepositoryError> {
let mut inodes = vec![];
@ -444,7 +444,7 @@ impl Repository {
#[inline]
pub fn get_backup_inode<P: AsRef<Path>>(
&mut self,
backup: &Backup,
backup: &BackupFile,
path: P,
) -> Result<Inode, RepositoryError> {
self.get_backup_path(backup, path).map(|mut inodes| {

21
src/repository/backup_file.rs → src/backups/backup_file.rs
View File

@ -55,24 +55,25 @@ quick_error!{
description(tr!("Encryption failed"))
display("{}", tr_format!("Backup file error: encryption failed\n\tcaused by: {}", err))
}
PartialBackupsList(partial: HashMap<String, Backup>, failed: Vec<PathBuf>) {
PartialBackupsList(partial: HashMap<String, BackupFile>, failed: Vec<PathBuf>) {
description(tr!("Some backups could not be loaded"))
display("{}", tr_format!("Backup file error: some backups could not be loaded: {:?}", failed))
}
}
}
#[derive(Default, Debug, Clone)]
struct BackupHeader {
struct BackupFileHeader {
pub encryption: Option<Encryption>
}
serde_impl!(BackupHeader(u8) {
serde_impl!(BackupFileHeader(u8) {
encryption: Option<Encryption> => 0
});
#[derive(Default, Debug, Clone)]
pub struct Backup {
pub struct BackupFile {
pub root: ChunkList,
pub total_data_size: u64, // Sum of all raw sizes of all entities
pub changed_data_size: u64, // Sum of all raw sizes of all entities actively stored
@ -92,7 +93,7 @@ pub struct Backup {
pub user_names: HashMap<u32, String>,
pub group_names: HashMap<u32, String>
}
serde_impl!(Backup(u8?) {
serde_impl!(BackupFile(u8?) {
root: ChunkList => 0,
total_data_size: u64 => 1,
changed_data_size: u64 => 2,
@ -113,7 +114,7 @@ serde_impl!(Backup(u8?) {
group_names: HashMap<u32, String> => 17
});
impl Backup {
impl BackupFile {
pub fn read_from<P: AsRef<Path>>(crypto: &Crypto, path: P) -> Result<Self, BackupFileError> {
let path = path.as_ref();
let mut file = BufReader::new(try!(File::open(path).map_err(|err| {
@ -133,7 +134,7 @@ impl Backup {
version
));
}
let header: BackupHeader = try!(msgpack::decode_from_stream(&mut file).context(path));
let header: BackupFileHeader = try!(msgpack::decode_from_stream(&mut file).context(path));
let mut data = Vec::new();
try!(file.read_to_end(&mut data).map_err(|err| {
BackupFileError::Read(err, path.to_path_buf())
@ -164,7 +165,7 @@ impl Backup {
try!(file.write_all(&[HEADER_VERSION]).map_err(|err| {
BackupFileError::Write(err, path.to_path_buf())
}));
let header = BackupHeader { encryption };
let header = BackupFileHeader { encryption };
try!(msgpack::encode_to_stream(&header, &mut file).context(path));
try!(file.write_all(&data).map_err(|err| {
BackupFileError::Write(err, path.to_path_buf())
@ -175,7 +176,7 @@ impl Backup {
pub fn get_all_from<P: AsRef<Path>>(
crypto: &Crypto,
path: P,
) -> Result<HashMap<String, Backup>, BackupFileError> {
) -> Result<HashMap<String, BackupFile>, BackupFileError> {
let mut backups = HashMap::new();
let base_path = path.as_ref();
let path = path.as_ref();
@ -203,7 +204,7 @@ impl Backup {
.with_file_name(relpath.file_stem().unwrap())
.to_string_lossy()
.to_string();
if let Ok(backup) = Backup::read_from(crypto, &path) {
if let Ok(backup) = BackupFile::read_from(crypto, &path) {
backups.insert(name, backup);
} else {
failed_paths.push(path.clone());

350
src/backups/inode.rs Normal file
View File

@ -0,0 +1,350 @@
use prelude::*;
use filetime::{self, FileTime};
use xattr;
use libc;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::fs::{self, File, Permissions};
use std::os::linux::fs::MetadataExt;
use std::os::unix::fs::{FileTypeExt, PermissionsExt, MetadataExt as UnixMetadataExt, symlink};
use std::io;
use std::os::unix::ffi::OsStrExt;
use std::fmt;
use std::ffi;
quick_error!{
#[derive(Debug)]
pub enum InodeError {
UnsupportedFiletype(path: PathBuf) {
description(tr!("Unsupported file type"))
display("{}", tr_format!("Inode error: file {:?} has an unsupported type", path))
}
ReadMetadata(err: io::Error, path: PathBuf) {
cause(err)
description(tr!("Failed to obtain metadata for file"))
display("{}", tr_format!("Inode error: failed to obtain metadata for file {:?}\n\tcaused by: {}", path, err))
}
ReadXattr(err: io::Error, path: PathBuf) {
cause(err)
description(tr!("Failed to obtain xattr for file"))
display("{}", tr_format!("Inode error: failed to obtain xattr for file {:?}\n\tcaused by: {}", path, err))
}
ReadLinkTarget(err: io::Error, path: PathBuf) {
cause(err)
description(tr!("Failed to obtain link target for file"))
display("{}", tr_format!("Inode error: failed to obtain link target for file {:?}\n\tcaused by: {}", path, err))
}
Create(err: io::Error, path: PathBuf) {
cause(err)
description(tr!("Failed to create entity"))
display("{}", tr_format!("Inode error: failed to create entity {:?}\n\tcaused by: {}", path, err))
}
Integrity(reason: &'static str) {
description(tr!("Integrity error"))
display("{}", tr_format!("Inode error: inode integrity error: {}", reason))
}
Decode(err: msgpack::DecodeError) {
from()
cause(err)
description(tr!("Failed to decode metadata"))
display("{}", tr_format!("Inode error: failed to decode metadata\n\tcaused by: {}", err))
}
Encode(err: msgpack::EncodeError) {
from()
cause(err)
description(tr!("Failed to encode metadata"))
display("{}", tr_format!("Inode error: failed to encode metadata\n\tcaused by: {}", err))
}
}
}
#[derive(Debug, Eq, PartialEq, Clone, Copy, Hash)]
pub enum FileType {
File,
Directory,
Symlink,
BlockDevice,
CharDevice,
NamedPipe
}
serde_impl!(FileType(u8) {
File => 0,
Directory => 1,
Symlink => 2,
BlockDevice => 3,
CharDevice => 4,
NamedPipe => 5
});
impl fmt::Display for FileType {
fn fmt(&self, format: &mut fmt::Formatter) -> Result<(), fmt::Error> {
match *self {
FileType::File => write!(format, "{}", tr!("file")),
FileType::Directory => write!(format, "{}", tr!("directory")),
FileType::Symlink => write!(format, "{}", tr!("symlink")),
FileType::BlockDevice => write!(format, "{}", tr!("block device")),
FileType::CharDevice => write!(format, "{}", tr!("char device")),
FileType::NamedPipe => write!(format, "{}", tr!("named pipe")),
}
}
}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub enum FileData {
Inline(msgpack::Bytes),
ChunkedDirect(ChunkList),
ChunkedIndirect(ChunkList)
}
serde_impl!(FileData(u8) {
Inline(ByteBuf) => 0,
ChunkedDirect(ChunkList) => 1,
ChunkedIndirect(ChunkList) => 2
});
#[derive(Debug, Hash, Eq, PartialEq)]
pub struct Inode {
pub name: String,
pub size: u64,
pub file_type: FileType,
pub mode: u32,
pub user: u32,
pub group: u32,
pub timestamp: i64,
pub symlink_target: Option<String>,
pub data: Option<FileData>,
pub children: Option<BTreeMap<String, ChunkList>>,
pub cum_size: u64,
pub cum_dirs: usize,
pub cum_files: usize,
pub xattrs: BTreeMap<String, msgpack::Bytes>,
pub device: Option<(u32, u32)>
}
impl Default for Inode {
fn default() -> Self {
Inode {
name: "".to_string(),
size: 0,
file_type: FileType::File,
mode: 0o644,
user: 1000,
group: 1000,
timestamp: 0,
symlink_target: None,
data: None,
children: None,
cum_size: 0,
cum_dirs: 0,
cum_files: 0,
xattrs: BTreeMap::new(),
device: None
}
}
}
serde_impl!(Inode(u8?) {
name: String => 0,
size: u64 => 1,
file_type: FileType => 2,
mode: u32 => 3,
user: u32 => 4,
group: u32 => 5,
timestamp: i64 => 7,
symlink_target: Option<String> => 9,
data: Option<FileData> => 10,
children: Option<BTreeMap<String, ChunkList>> => 11,
cum_size: u64 => 12,
cum_dirs: usize => 13,
cum_files: usize => 14,
xattrs: BTreeMap<String, msgpack::Bytes> => 15,
device: Option<(u32, u32)> => 16
});
impl Inode {
pub fn get_from<P: AsRef<Path>>(path: P) -> Result<Self, InodeError> {
let path = path.as_ref();
let name = path.file_name()
.map(|s| s.to_string_lossy().to_string())
.unwrap_or_else(|| "_".to_string());
let meta = try!(fs::symlink_metadata(path).map_err(|e| {
InodeError::ReadMetadata(e, path.to_owned())
}));
let mut inode = Inode::default();
inode.name = name;
if meta.is_file() {
inode.size = meta.len();
}
inode.file_type = if meta.is_file() {
FileType::File
} else if meta.is_dir() {
FileType::Directory
} else if meta.file_type().is_symlink() {
FileType::Symlink
} else if meta.file_type().is_block_device() {
FileType::BlockDevice
} else if meta.file_type().is_char_device() {
FileType::CharDevice
} else if meta.file_type().is_fifo() {
FileType::NamedPipe
} else {
return Err(InodeError::UnsupportedFiletype(path.to_owned()));
};
if meta.file_type().is_symlink() {
inode.symlink_target = Some(
try!(fs::read_link(path).map_err(|e| {
InodeError::ReadLinkTarget(e, path.to_owned())
})).to_string_lossy()
.to_string()
);
}
if meta.file_type().is_block_device() || meta.file_type().is_char_device() {
let rdev = meta.rdev();
let major = (rdev >> 8) as u32;
let minor = (rdev & 0xff) as u32;
inode.device = Some((major, minor));
}
inode.mode = meta.permissions().mode();
inode.user = meta.st_uid();
inode.group = meta.st_gid();
inode.timestamp = meta.st_mtime();
if xattr::SUPPORTED_PLATFORM {
if let Ok(attrs) = xattr::list(path) {
for name in attrs {
if let Some(data) = try!(xattr::get(path, &name).map_err(|e| {
InodeError::ReadXattr(e, path.to_owned())
}))
{
inode.xattrs.insert(
name.to_string_lossy().to_string(),
data.into()
);
}
}
}
}
Ok(inode)
}
pub fn create_at<P: AsRef<Path>>(&self, path: P) -> Result<Option<File>, InodeError> {
let full_path = path.as_ref().join(&self.name);
let mut file = None;
match self.file_type {
FileType::File => {
file = Some(try!(File::create(&full_path).map_err(|e| {
InodeError::Create(e, full_path.clone())
})));
}
FileType::Directory => {
try!(fs::create_dir(&full_path).map_err(|e| {
InodeError::Create(e, full_path.clone())
}));
}
FileType::Symlink => {
if let Some(ref src) = self.symlink_target {
try!(symlink(src, &full_path).map_err(|e| {
InodeError::Create(e, full_path.clone())
}));
} else {
return Err(InodeError::Integrity(tr!("Symlink without target")));
}
}
FileType::NamedPipe => {
let name = try!(
ffi::CString::new(full_path.as_os_str().as_bytes())
.map_err(|_| InodeError::Integrity(tr!("Name contains nulls")))
);
let mode = self.mode | libc::S_IFIFO;
if unsafe { libc::mkfifo(name.as_ptr(), mode) } != 0 {
return Err(InodeError::Create(
io::Error::last_os_error(),
full_path.clone()
));
}
}
FileType::BlockDevice | FileType::CharDevice => {
let name = try!(
ffi::CString::new(full_path.as_os_str().as_bytes())
.map_err(|_| InodeError::Integrity(tr!("Name contains nulls")))
);
let mode = self.mode |
match self.file_type {
FileType::BlockDevice => libc::S_IFBLK,
FileType::CharDevice => libc::S_IFCHR,
_ => unreachable!(),
};
let device = if let Some((major, minor)) = self.device {
unsafe { libc::makedev(major, minor) }
} else {
return Err(InodeError::Integrity(tr!("Device without id")));
};
if unsafe { libc::mknod(name.as_ptr(), mode, device) } != 0 {
return Err(InodeError::Create(
io::Error::last_os_error(),
full_path.clone()
));
}
}
}
let time = FileTime::from_seconds_since_1970(self.timestamp as u64, 0);
if let Err(err) = filetime::set_file_times(&full_path, time, time) {
tr_warn!("Failed to set file time on {:?}: {}", full_path, err);
}
if !self.xattrs.is_empty() {
if xattr::SUPPORTED_PLATFORM {
for (name, data) in &self.xattrs {
if let Err(err) = xattr::set(&full_path, name, data) {
tr_warn!("Failed to set xattr {} on {:?}: {}", name, full_path, err);
}
}
} else {
tr_warn!("Not setting xattr on {:?}", full_path);
}
}
if let Err(err) = fs::set_permissions(&full_path, Permissions::from_mode(self.mode)) {
tr_warn!(
"Failed to set permissions {:o} on {:?}: {}",
self.mode,
full_path,
err
);
}
if let Err(err) = chown(&full_path, self.user, self.group) {
tr_warn!(
"Failed to set user {} and group {} on {:?}: {}",
self.user,
self.group,
full_path,
err
);
}
Ok(file)
}
#[inline]
pub fn is_same_meta(&self, other: &Inode) -> bool {
self.file_type == other.file_type && self.size == other.size &&
self.mode == other.mode && self.user == other.user &&
self.group == other.group && self.name == other.name &&
self.timestamp == other.timestamp && self.symlink_target == other.symlink_target
}
#[inline]
pub fn is_same_meta_quick(&self, other: &Inode) -> bool {
self.timestamp == other.timestamp && self.file_type == other.file_type &&
self.size == other.size
}
#[inline]
pub fn encode(&self) -> Result<Vec<u8>, InodeError> {
Ok(try!(msgpack::encode(&self)))
}
#[inline]
pub fn decode(data: &[u8]) -> Result<Self, InodeError> {
Ok(try!(msgpack::decode(data)))
}
}

323
src/backups/integrity.rs Normal file
View File

@ -0,0 +1,323 @@
use prelude::*;
use super::*;
use std::path::{Path, PathBuf};
quick_error!{
#[derive(Debug)]
pub enum InodeIntegrityError {
BrokenInode(path: PathBuf, err: Box<RepositoryError>) {
cause(err)
description(tr!("Broken inode"))
display("{}", tr_format!("Broken inode: {:?}\n\tcaused by: {}", path, err))
}
MissingInodeData(path: PathBuf, err: Box<RepositoryError>) {
cause(err)
description(tr!("Missing inode data"))
display("{}", tr_format!("Missing inode data in: {:?}\n\tcaused by: {}", path, err))
}
}
}
impl BackupRepository {
fn check_inode_contents(
&mut self,
inode: &Inode,
checked: &mut Bitmap,
) -> Result<(), RepositoryError> {
match inode.data {
None |
Some(FileData::Inline(_)) => (),
Some(FileData::ChunkedDirect(ref chunks)) => {
try!(self.repo.check_chunks(checked, chunks, true));
}
Some(FileData::ChunkedIndirect(ref chunks)) => {
if try!(self.repo.check_chunks(checked, chunks, false)) {
let chunk_data = try!(self.get_data(chunks));
let chunks2 = ChunkList::read_from(&chunk_data);
try!(self.repo.check_chunks(checked, &chunks2, true));
try!(self.repo.check_chunks(checked, chunks, true));
}
}
}
Ok(())
}
fn check_subtree(
&mut self,
path: PathBuf,
chunks: &[Chunk],
checked: &mut Bitmap,
repair: bool,
) -> Result<Option<ChunkList>, RepositoryError> {
let mut modified = false;
match self.repo.check_chunks(checked, chunks, false) {
Ok(false) => return Ok(None),
Ok(true) => (),
Err(err) => return Err(InodeIntegrityError::BrokenInode(path, Box::new(err)).into()),
}
let mut inode = try!(self.get_inode(chunks));
// Mark the content chunks as used
if let Err(err) = self.check_inode_contents(&inode, checked) {
if repair {
tr_warn!(
"Problem detected: data of {:?} is corrupt\n\tcaused by: {}",
path,
err
);
tr_info!("Removing inode data");
inode.data = Some(FileData::Inline(vec![].into()));
inode.size = 0;
modified = true;
} else {
return Err(InodeIntegrityError::MissingInodeData(path, Box::new(err)).into());
}
}
// Put children in to do
if let Some(ref mut children) = inode.children {
let mut removed = vec![];
for (name, chunks) in children.iter_mut() {
match self.check_subtree(path.join(name), chunks, checked, repair) {
Ok(None) => (),
Ok(Some(c)) => {
*chunks = c;
modified = true;
}
Err(err) => {
if repair {
tr_warn!(
"Problem detected: inode {:?} is corrupt\n\tcaused by: {}",
path.join(name),
err
);
tr_info!("Removing broken inode from backup");
removed.push(name.to_string());
modified = true;
} else {
return Err(err);
}
}
}
}
for name in removed {
children.remove(&name);
}
}
if modified {
Ok(Some(try!(self.repo.put_inode(&inode))))
} else {
try!(self.repo.check_chunks(checked, chunks, true));
Ok(None)
}
}
fn evacuate_broken_backup(&self, name: &str) -> Result<(), RepositoryError> {
tr_warn!(
"The backup {} was corrupted and needed to be modified.",
name
);
let src = self.layout.backup_path(name);
let mut dst = src.with_extension("backup.broken");
let mut num = 1;
while dst.exists() {
dst = src.with_extension(&format!("backup.{}.broken", num));
num += 1;
}
if fs::rename(&src, &dst).is_err() {
try!(fs::copy(&src, &dst));
try!(fs::remove_file(&src));
}
tr_info!("The original backup was renamed to {:?}", dst);
Ok(())
}
#[inline]
pub fn check_backup(
&mut self,
name: &str,
backup: &mut BackupFile,
repair: bool,
) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.repo.write_mode());
Some(self.repo.lock(false))
} else {
None
};
tr_info!("Checking backup...");
let mut checked = self.repo.get_chunk_marker();
match self.check_subtree(
Path::new("").to_path_buf(),
&backup.root,
&mut checked,
repair
) {
Ok(None) => (),
Ok(Some(chunks)) => {
try!(self.repo.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(name));
try!(self.save_backup(backup, name));
}
Err(err) => {
if repair {
tr_warn!(
"The root of the backup {} has been corrupted\n\tcaused by: {}",
name,
err
);
try!(self.evacuate_broken_backup(name));
} else {
return Err(err);
}
}
}
Ok(())
}
pub fn check_backup_inode(
&mut self,
name: &str,
backup: &mut BackupFile,
path: &Path,
repair: bool,
) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.repo.write_mode());
Some(self.repo.lock(false))
} else {
None
};
tr_info!("Checking inode...");
let mut checked = self.repo.get_chunk_marker();
let mut inodes = try!(self.get_backup_path(backup, path));
let mut inode = inodes.pop().unwrap();
let mut modified = false;
if let Err(err) = self.check_inode_contents(&inode, &mut checked) {
if repair {
tr_warn!(
"Problem detected: data of {:?} is corrupt\n\tcaused by: {}",
path,
err
);
tr_info!("Removing inode data");
inode.data = Some(FileData::Inline(vec![].into()));
inode.size = 0;
modified = true;
} else {
return Err(
InodeIntegrityError::MissingInodeData(path.to_path_buf(), Box::new(err)).into()
);
}
}
if let Some(ref mut children) = inode.children {
let mut removed = vec![];
for (name, chunks) in children.iter_mut() {
match self.check_subtree(path.join(name), chunks, &mut checked, repair) {
Ok(None) => (),
Ok(Some(c)) => {
*chunks = c;
modified = true;
}
Err(err) => {
if repair {
tr_warn!(
"Problem detected: inode {:?} is corrupt\n\tcaused by: {}",
path.join(name),
err
);
tr_info!("Removing broken inode from backup");
removed.push(name.to_string());
modified = true;
} else {
return Err(err);
}
}
}
}
for name in removed {
children.remove(&name);
}
}
let mut chunks = try!(self.repo.put_inode(&inode));
while let Some(mut parent) = inodes.pop() {
parent.children.as_mut().unwrap().insert(inode.name, chunks);
inode = parent;
chunks = try!(self.repo.put_inode(&inode));
}
if modified {
try!(self.repo.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(name));
try!(self.save_backup(backup, name));
}
Ok(())
}
pub fn check_backups(&mut self, repair: bool) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.repo.write_mode());
Some(self.repo.lock(false))
} else {
None
};
tr_info!("Checking backups...");
let mut checked = self.repo.get_chunk_marker();
let backup_map = match self.get_all_backups() {
Ok(backup_map) => backup_map,
Err(RepositoryError::BackupFile(BackupFileError::PartialBackupsList(backup_map,
_failed))) => {
tr_warn!("Some backups could not be read, ignoring them");
backup_map
}
Err(err) => return Err(err),
};
for (name, mut backup) in
ProgressIter::new(tr!("checking backups"), backup_map.len(), backup_map.into_iter())
{
let path = format!("{}::", name);
match self.check_subtree(
Path::new(&path).to_path_buf(),
&backup.root,
&mut checked,
repair
) {
Ok(None) => (),
Ok(Some(chunks)) => {
try!(self.repo.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(&name));
try!(self.save_backup(&backup, &name));
}
Err(err) => {
if repair {
tr_warn!(
"The root of the backup {} has been corrupted\n\tcaused by: {}",
name,
err
);
try!(self.evacuate_broken_backup(&name));
} else {
return Err(err);
}
}
}
}
Ok(())
}
#[inline]
pub fn check_bundles(&mut self, full: bool, repair: bool) -> Result<(), RepositoryError> {
self.repo.check_bundles(full, repair)
}
pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> {
self.repo.check_repository(repair)
}
}

158
src/backups/mod.rs
View File

@ -1,48 +1,91 @@
pub mod mount;
mod backup_file;
mod inode;
mod tarfile;
mod backup;
mod integrity;
mod vacuum;
pub use self::backup::{BackupOptions, BackupError, DiffType};
pub use self::backup_file::{BackupFile, BackupFileError};
pub use self::inode::{Inode, FileData, FileType, InodeError};
pub use self::integrity::InodeIntegrityError;
use ::prelude::*;
use std::path::{Path, PathBuf};
use std::path::Path;
use std::collections::HashMap;
use std::sync::Arc;
use std::fs::{self, File};
use std::io::Write;
const DEFAULT_EXCLUDES: &[u8] = include_bytes!("../../docs/excludes.default");
pub struct BackupRepository {
layout: Arc<RepositoryLayout>,
crypto: Arc<Crypto>,
repo: Repository
}
impl BackupRepository {
pub fn create<P: AsRef<Path>, R: AsRef<Path>>(path: P, config: &Config, remote: R) -> Result<Self, RepositoryError> {
let layout = Arc::new(RepositoryLayout::new(path.as_ref()));
try!(fs::create_dir(layout.base_path()));
try!(File::create(layout.excludes_path()).and_then(|mut f| {
f.write_all(DEFAULT_EXCLUDES)
}));
try!(fs::create_dir(layout.keys_path()));
let crypto = Arc::new(try!(Crypto::open(layout.keys_path())));
Ok(BackupRepository {
crypto: crypto.clone(),
layout: layout.clone(),
repo: try!(Repository::create(layout, config, remote))
repo: try!(Repository::create(layout, config, crypto, remote))
})
}
#[allow(unknown_lints, useless_let_if_seq)]
pub fn open<P: AsRef<Path>>(path: P, online: bool) -> Result<Self, RepositoryError> {
let layout = Arc::new(RepositoryLayout::new(path.as_ref()));
let crypto = Arc::new(try!(Crypto::open(layout.keys_path())));
Ok(BackupRepository {
crypto: crypto.clone(),
layout: layout.clone(),
repo: try!(Repository::open(layout, online))
repo: try!(Repository::open(layout, crypto, online))
})
}
pub fn import<P: AsRef<Path>, R: AsRef<Path>>(path: P, remote: R, key_files: Vec<String>) -> Result<Self, RepositoryError> {
let layout = Arc::new(RepositoryLayout::new(path.as_ref()));
Ok(BackupRepository {
layout: layout.clone(),
repo: try!(Repository::import(layout, remote, key_files))
})
let config = Config::default();
let mut repo = try!(Self::create(&path, &config, remote));
for file in key_files {
try!(repo.crypto.register_keyfile(file));
}
repo = try!(Self::open(&path, true));
let mut backups: Vec<(String, BackupFile)> = try!(repo.get_all_backups()).into_iter().collect();
backups.sort_by_key(|&(_, ref b)| b.timestamp);
if let Some((name, backup)) = backups.pop() {
tr_info!("Taking configuration from the last backup '{}'", name);
repo.repo.set_config(backup.config);
try!(repo.save_config())
} else {
tr_warn!(
"No backup found in the repository to take configuration from, please set the configuration manually."
);
}
Ok(repo)
}
#[inline]
pub fn register_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), RepositoryError> {
self.repo.register_key(public, secret)
try!(self.repo.write_mode());
try!(self.crypto.register_secret_key(public, secret));
Ok(())
}
#[inline]
pub fn save_config(&mut self) -> Result<(), RepositoryError> {
self.repo.save_config()
@ -53,29 +96,11 @@ impl BackupRepository {
self.repo.set_encryption(public)
}
#[inline]
pub fn has_backup(&self, name: &str) -> bool {
self.repo.has_backup(name)
}
pub fn get_backup(&self, name: &str) -> Result<Backup, RepositoryError> {
self.repo.get_backup(name)
}
#[inline]
pub fn get_backup_inode<P: AsRef<Path>>(&mut self, backup: &Backup, path: P) -> Result<Inode, RepositoryError> {
self.repo.get_backup_inode(backup, path)
}
#[inline]
pub fn get_inode(&mut self, chunks: &[Chunk]) -> Result<Inode, RepositoryError> {
self.repo.get_inode(chunks)
}
pub fn get_all_backups(&self) -> Result<HashMap<String, Backup>, RepositoryError> {
self.repo.get_all_backups()
}
pub fn get_config(&self) -> &Config {
self.repo.get_config()
}
@ -88,77 +113,15 @@ impl BackupRepository {
&self.layout
}
pub fn create_backup_recursively<P: AsRef<Path>>(&mut self, path: P, reference: Option<&Backup>, options: &BackupOptions) -> Result<Backup, RepositoryError> {
self.repo.create_backup_recursively(path, reference, options)
}
pub fn import_tarfile<P: AsRef<Path>>(&mut self, tarfile: P) -> Result<Backup, RepositoryError> {
self.repo.import_tarfile(tarfile)
}
pub fn save_backup(&mut self, backup: &Backup, name: &str) -> Result<(), RepositoryError> {
self.repo.save_backup(backup, name)
}
pub fn export_tarfile<P: AsRef<Path>>(&mut self, backup: &Backup, inode: Inode, tarfile: P) -> Result<(), RepositoryError> {
self.repo.export_tarfile(backup, inode, tarfile)
}
pub fn restore_inode_tree<P: AsRef<Path>>(&mut self, backup: &Backup, inode: Inode, path: P) -> Result<(), RepositoryError> {
self.repo.restore_inode_tree(backup, inode, path)
}
pub fn remove_backup_path<P: AsRef<Path>>(&mut self, backup: &mut Backup, path: P) -> Result<(), RepositoryError> {
self.repo.remove_backup_path(backup, path)
}
pub fn get_backups<P: AsRef<Path>>(&self, path: P) -> Result<HashMap<String, Backup>, RepositoryError> {
self.repo.get_backups(path)
}
pub fn delete_backup(&mut self, name: &str) -> Result<(), RepositoryError> {
self.repo.delete_backup(name)
}
pub fn prune_backups(&mut self, prefix: &str, daily: usize, weekly: usize, monthly: usize, yearly: usize, force: bool) -> Result<(), RepositoryError> {
self.repo.prune_backups(prefix, daily, weekly, monthly, yearly, force)
}
pub fn info(&self) -> RepositoryInfo {
self.repo.info()
}
pub fn vacuum(&mut self, ratio: f32, combine: bool, force: bool) -> Result<(), RepositoryError> {
self.repo.vacuum(ratio, combine, force)
}
pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> {
self.repo.check_repository(repair)
}
#[inline]
pub fn check_bundles(&mut self, full: bool, repair: bool) -> Result<(), RepositoryError> {
self.repo.check_bundles(full, repair)
}
#[inline]
pub fn check_index(&mut self, repair: bool) -> Result<(), RepositoryError> {
self.repo.check_index(repair)
}
pub fn check_backup_inode(&mut self, name: &str, backup: &mut Backup, path: &Path, repair: bool) -> Result<(), RepositoryError> {
self.repo.check_backup_inode(name, backup, path, repair)
}
#[inline]
pub fn check_backup(&mut self, name: &str, backup: &mut Backup, repair: bool) -> Result<(), RepositoryError> {
self.repo.check_backup(name, backup, repair)
}
pub fn check_backups(&mut self, repair: bool) -> Result<(), RepositoryError> {
self.repo.check_backups(repair)
}
#[inline]
pub fn set_clean(&mut self) {
self.repo.set_clean()
@ -168,14 +131,6 @@ impl BackupRepository {
self.repo.statistics()
}
pub fn find_duplicates(&mut self, inode: &Inode, min_size: u64) -> Result<Vec<(Vec<PathBuf>, u64)>, RepositoryError> {
self.repo.find_duplicates(inode, min_size)
}
pub fn analyze_usage(&mut self) -> Result<HashMap<u32, BundleAnalysis>, RepositoryError> {
self.repo.analyze_usage()
}
#[inline]
pub fn list_bundles(&self) -> Vec<&BundleInfo> {
self.repo.list_bundles()
@ -186,15 +141,6 @@ impl BackupRepository {
self.repo.get_bundle(bundle)
}
pub fn find_versions<P: AsRef<Path>>(&mut self, path: P) -> Result<Vec<(String, Inode)>, RepositoryError> {
self.repo.find_versions(path)
}
#[inline]
pub fn find_differences(&mut self, inode1: &Inode, inode2: &Inode) -> Result<Vec<(DiffType, PathBuf)>, RepositoryError> {
self.repo.find_differences(inode1, inode2)
}
pub fn get_chunk(&mut self, hash: Hash) -> Result<Option<Vec<u8>>, RepositoryError> {
self.repo.get_chunk(hash)
}

4
src/backups/mount.rs
View File

@ -197,7 +197,7 @@ impl<'a> FuseFilesystem<'a> {
pub fn from_backup(
repository: &'a mut BackupRepository,
backup: Backup,
backup: BackupFile,
) -> Result<Self, RepositoryError> {
let inode = try!(repository.get_inode(&backup.root));
let mut fs = try!(FuseFilesystem::new(repository));
@ -207,7 +207,7 @@ impl<'a> FuseFilesystem<'a> {
pub fn from_inode(
repository: &'a mut BackupRepository,
backup: Backup,
backup: BackupFile,
inode: Inode,
) -> Result<Self, RepositoryError> {
let mut fs = try!(FuseFilesystem::new(repository));

39
src/repository/tarfile.rs → src/backups/tarfile.rs
View File

@ -133,7 +133,8 @@ fn inode_from_entry<R: Read>(entry: &mut tar::Entry<R>) -> Result<Inode, Reposit
Ok(inode)
}
impl Repository {
impl BackupRepository {
fn import_tar_entry<R: Read>(
&mut self,
entry: &mut tar::Entry<R>,
@ -144,13 +145,13 @@ impl Repository {
try!(entry.read_to_end(&mut data));
inode.data = Some(FileData::Inline(data.into()));
} else {
let mut chunks = try!(self.put_stream(BundleMode::Data, entry));
let mut chunks = try!(self.repo.put_stream(BundleMode::Data, entry));
if chunks.len() < 10 {
inode.data = Some(FileData::ChunkedDirect(chunks));
} else {
let mut chunk_data = Vec::with_capacity(chunks.encoded_size());
chunks.write_to(&mut chunk_data).unwrap();
chunks = try!(self.put_data(BundleMode::Meta, &chunk_data));
chunks = try!(self.repo.put_data(BundleMode::Meta, &chunk_data));
inode.data = Some(FileData::ChunkedIndirect(chunks));
}
}
@ -159,7 +160,7 @@ impl Repository {
fn import_tarfile_as_inode<R: Read>(
&mut self,
backup: &mut Backup,
backup: &mut BackupFile,
input: R,
failed_paths: &mut Vec<PathBuf>,
) -> Result<(Inode, ChunkList), RepositoryError> {
@ -218,7 +219,7 @@ impl Repository {
}
for path in childless {
let (inode, _) = inodes.remove(&path).unwrap();
let chunks = try!(self.put_inode(&inode));
let chunks = try!(self.repo.put_inode(&inode));
if let Some(parent_path) = path.parent() {
if let Some(&mut (ref mut parent_inode, ref mut children)) =
inodes.get_mut(parent_path)
@ -264,7 +265,7 @@ impl Repository {
children.insert(inode.name, chunks);
}
root_inode.children = Some(children);
let chunks = try!(self.put_inode(&root_inode));
let chunks = try!(self.repo.put_inode(&root_inode));
Ok((root_inode, chunks))
}
}
@ -272,15 +273,15 @@ impl Repository {
pub fn import_tarfile<P: AsRef<Path>>(
&mut self,
tarfile: P,
) -> Result<Backup, RepositoryError> {
try!(self.write_mode());
let _lock = try!(self.lock(false));
if self.dirty {
) -> Result<BackupFile, RepositoryError> {
try!(self.repo.write_mode());
let _lock = try!(self.repo.lock(false));
if self.repo.is_dirty() {
return Err(RepositoryError::Dirty);
}
try!(self.set_dirty());
let mut backup = Backup::default();
backup.config = self.config.clone();
try!(self.repo.set_dirty());
let mut backup = BackupFile::default();
backup.config = self.repo.get_config().clone();
backup.host = get_hostname().unwrap_or_else(|_| "".to_string());
backup.path = tarfile.as_ref().to_string_lossy().to_string();
let info_before = self.info();
@ -301,7 +302,7 @@ impl Repository {
))
};
backup.root = chunks;
try!(self.flush());
try!(self.repo.flush());
let elapsed = Local::now().signed_duration_since(start);
backup.timestamp = start.timestamp();
backup.total_data_size = root_inode.cum_size;
@ -314,7 +315,7 @@ impl Repository {
backup.bundle_count = info_after.bundle_count - info_before.bundle_count;
backup.chunk_count = info_after.chunk_count - info_before.chunk_count;
backup.avg_chunk_size = backup.deduplicated_data_size as f32 / backup.chunk_count as f32;
self.dirty = false;
self.repo.set_clean();
if failed_paths.is_empty() {
Ok(backup)
} else {
@ -340,7 +341,7 @@ impl Repository {
fn export_tarfile_recurse<W: Write>(
&mut self,
backup: &Backup,
backup: &BackupFile,
path: &Path,
inode: Inode,
tarfile: &mut tar::Builder<W>,
@ -396,11 +397,11 @@ impl Repository {
None => try!(tarfile.append(&header, Cursor::new(&[]))),
Some(FileData::Inline(data)) => try!(tarfile.append(&header, Cursor::new(data))),
Some(FileData::ChunkedDirect(chunks)) => {
try!(tarfile.append(&header, self.get_reader(chunks)))
try!(tarfile.append(&header, self.repo.get_reader(chunks)))
}
Some(FileData::ChunkedIndirect(chunks)) => {
let chunks = ChunkList::read_from(&try!(self.get_data(&chunks)));
try!(tarfile.append(&header, self.get_reader(chunks)))
try!(tarfile.append(&header, self.repo.get_reader(chunks)))
}
}
}
@ -421,7 +422,7 @@ impl Repository {
pub fn export_tarfile<P: AsRef<Path>>(
&mut self,
backup: &Backup,
backup: &BackupFile,
inode: Inode,
tarfile: P,
) -> Result<(), RepositoryError> {

158
src/backups/vacuum.rs Normal file
View File

@ -0,0 +1,158 @@
use ::prelude::*;
use super::*;
use std::collections::{VecDeque, HashSet};
impl BackupRepository {
fn mark_used(
&self,
bundles: &mut HashMap<u32, BundleAnalysis>,
chunks: &[Chunk],
) -> Result<bool, RepositoryError> {
let mut new = false;
for &(hash, len) in chunks {
if let Some(pos) = self.repo.get_chunk_location(hash) {
let bundle = pos.bundle;
if let Some(bundle) = bundles.get_mut(&bundle) {
if !bundle.chunk_usage.get(pos.chunk as usize) {
new = true;
bundle.chunk_usage.set(pos.chunk as usize);
bundle.used_raw_size += len as usize;
}
} else {
return Err(IntegrityError::MissingBundleId(pos.bundle).into());
}
} else {
return Err(IntegrityError::MissingChunk(hash).into());
}
}
Ok(new)
}
pub fn analyze_usage(&mut self) -> Result<HashMap<u32, BundleAnalysis>, RepositoryError> {
if self.repo.is_dirty() {
return Err(RepositoryError::Dirty);
}
try!(self.repo.set_dirty());
let mut usage = HashMap::new();
for (id, bundle) in try!(self.repo.get_bundle_map()) {
usage.insert(
id,
BundleAnalysis {
chunk_usage: Bitmap::new(bundle.chunk_count),
info: bundle.clone(),
used_raw_size: 0
}
);
}
let backups = try!(self.get_all_backups());
let mut todo = VecDeque::new();
for (_name, backup) in backups {
todo.push_back(backup.root);
}
while let Some(chunks) = todo.pop_back() {
if !try!(self.mark_used(&mut usage, &chunks)) {
continue;
}
let inode = try!(self.get_inode(&chunks));
// Mark the content chunks as used
match inode.data {
None |
Some(FileData::Inline(_)) => (),
Some(FileData::ChunkedDirect(chunks)) => {
try!(self.mark_used(&mut usage, &chunks));
}
Some(FileData::ChunkedIndirect(chunks)) => {
if try!(self.mark_used(&mut usage, &chunks)) {
let chunk_data = try!(self.get_data(&chunks));
let chunks = ChunkList::read_from(&chunk_data);
try!(self.mark_used(&mut usage, &chunks));
}
}
}
// Put children in to do
if let Some(children) = inode.children {
for (_name, chunks) in children {
todo.push_back(chunks);
}
}
}
self.repo.set_clean();
Ok(usage)
}
pub fn vacuum(
&mut self,
ratio: f32,
combine: bool,
force: bool,
) -> Result<(), RepositoryError> {
try!(self.repo.flush());
tr_info!("Locking repository");
try!(self.repo.write_mode());
let _lock = try!(self.repo.lock(true));
// analyze_usage will set the dirty flag
tr_info!("Analyzing chunk usage");
let usage = try!(self.analyze_usage());
let mut data_total = 0;
let mut data_used = 0;
for bundle in usage.values() {
data_total += bundle.info.encoded_size;
data_used += bundle.get_used_size();
}
tr_info!(
"Usage: {} of {}, {:.1}%",
to_file_size(data_used as u64),
to_file_size(data_total as u64),
data_used as f32 / data_total as f32 * 100.0
);
let mut rewrite_bundles = HashSet::new();
let mut reclaim_space = 0;
let mut rewrite_data = 0;
for (id, bundle) in &usage {
if bundle.get_usage_ratio() <= ratio {
rewrite_bundles.insert(*id);
reclaim_space += bundle.get_unused_size();
rewrite_data += bundle.get_used_size();
}
}
if combine {
let mut small_meta = vec![];
let mut small_data = vec![];
for (id, bundle) in &usage {
if bundle.info.encoded_size * 4 < self.repo.get_config().bundle_size {
match bundle.info.mode {
BundleMode::Meta => small_meta.push(*id),
BundleMode::Data => small_data.push(*id),
}
}
}
if small_meta.len() >= 2 {
for bundle in small_meta {
rewrite_bundles.insert(bundle);
}
}
if small_data.len() >= 2 {
for bundle in small_data {
rewrite_bundles.insert(bundle);
}
}
}
tr_info!(
"Reclaiming about {} by rewriting {} bundles ({})",
to_file_size(reclaim_space as u64),
rewrite_bundles.len(),
to_file_size(rewrite_data as u64)
);
if !force {
self.repo.set_clean();
return Ok(());
}
let rewrite_bundles: Vec<_> = rewrite_bundles.into_iter().collect();
try!(self.repo.rewrite_bundles(&rewrite_bundles, &usage));
self.repo.set_clean();
Ok(())
}
}

10
src/cli/mod.rs
View File

@ -123,7 +123,7 @@ fn open_repository(path: &Path, online: bool) -> Result<BackupRepository, ErrorC
))
}
fn get_backup(repo: &BackupRepository, backup_name: &str) -> Result<Backup, ErrorCode> {
fn get_backup(repo: &BackupRepository, backup_name: &str) -> Result<BackupFile, ErrorCode> {
if !repo.has_backup(backup_name) {
tr_error!("A backup with that name does not exist");
return Err(ErrorCode::NoSuchBackup);
@ -135,7 +135,7 @@ fn get_backup(repo: &BackupRepository, backup_name: &str) -> Result<Backup, Erro
))
}
fn get_inode(repo: &mut BackupRepository, backup: &Backup, inode: Option<&String>) -> Result<Inode, ErrorCode> {
fn get_inode(repo: &mut BackupRepository, backup: &BackupFile, inode: Option<&String>) -> Result<Inode, ErrorCode> {
Ok(if let Some(inode) = inode {
checked!(
repo.get_backup_inode(backup, &inode),
@ -154,7 +154,7 @@ fn get_inode(repo: &mut BackupRepository, backup: &Backup, inode: Option<&String
fn find_reference_backup(
repo: &BackupRepository,
path: &str,
) -> Result<Option<(String, Backup)>, ErrorCode> {
) -> Result<Option<(String, BackupFile)>, ErrorCode> {
let mut matching = Vec::new();
let hostname = match get_hostname() {
Ok(hostname) => hostname,
@ -181,7 +181,7 @@ fn find_reference_backup(
Ok(matching.pop())
}
fn print_backup(backup: &Backup) {
fn print_backup(backup: &BackupFile) {
if backup.modified {
tr_warn!("This backup has been modified");
}
@ -299,7 +299,7 @@ fn print_inode(inode: &Inode) {
}
}
fn print_backups(backup_map: &HashMap<String, Backup>) {
fn print_backups(backup_map: &HashMap<String, BackupFile>) {
let mut backups: Vec<_> = backup_map.into_iter().collect();
backups.sort_by_key(|b| b.0);
for (name, backup) in backups {

8
src/prelude.rs
View File

@ -2,12 +2,14 @@ pub use util::*;
pub use repository::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError,
BundleDb, BundleWriterError, StoredBundle, BundleStatistics};
pub use repository::chunking::{ChunkerType, Chunker, ChunkerStatus, ChunkerError};
pub use repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType,
IntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis,
FileData, DiffType, InodeError, RepositoryLayout, Location,
pub use repository::{Repository, Config, RepositoryError, RepositoryInfo,
IntegrityError, BundleAnalysis,
RepositoryLayout, Location,
RepositoryStatistics, ChunkRepositoryLayout};
pub use repository::index::{Index, IndexError, IndexStatistics};
pub use backups::mount::FuseFilesystem;
pub use backups::{BackupFile, BackupFileError, Inode, FileType, FileData, InodeError, BackupError,
BackupOptions, DiffType, InodeIntegrityError};
pub use translation::CowStr;
pub use backups::BackupRepository;

11
src/repository/error.rs
View File

@ -3,11 +3,8 @@ use prelude::*;
use std::io;
use std::path::PathBuf;
use super::backup_file::BackupFileError;
use super::backup::BackupError;
use super::bundle_map::BundleMapError;
use super::config::ConfigError;
use super::metadata::InodeError;
quick_error!{
@ -72,6 +69,12 @@ quick_error!{
description(tr!("Bundle map error"))
display("{}", tr_format!("Repository error: bundle map error\n\tcaused by: {}", err))
}
InodeIntegrity(err: InodeIntegrityError) {
from()
cause(err)
description(tr!("Integrity error"))
display("{}", tr_format!("Repository error: integrity error\n\tcaused by: {}", err))
}
Integrity(err: IntegrityError) {
from()
cause(err)
@ -101,7 +104,7 @@ quick_error!{
description(tr!("IO error"))
display("{}", tr_format!("IO error: {}", err))
}
NoSuchFileInBackup(backup: Backup, path: PathBuf) {
NoSuchFileInBackup(backup: BackupFile, path: PathBuf) {
description(tr!("No such file in backup"))
display("{}", tr_format!("The backup does not contain the file {:?}", path))
}

93
src/repository/info.rs
View File

@ -1,6 +1,6 @@
use prelude::*;
use std::collections::{HashMap, VecDeque};
use std::collections::HashMap;
pub struct BundleAnalysis {
@ -47,91 +47,22 @@ pub struct RepositoryStatistics {
impl Repository {
fn mark_used(
&self,
bundles: &mut HashMap<u32, BundleAnalysis>,
chunks: &[Chunk],
) -> Result<bool, RepositoryError> {
let mut new = false;
for &(hash, len) in chunks {
if let Some(pos) = self.index.get(&hash) {
let bundle = pos.bundle;
if let Some(bundle) = bundles.get_mut(&bundle) {
if !bundle.chunk_usage.get(pos.chunk as usize) {
new = true;
bundle.chunk_usage.set(pos.chunk as usize);
bundle.used_raw_size += len as usize;
}
} else {
return Err(IntegrityError::MissingBundleId(pos.bundle).into());
}
} else {
return Err(IntegrityError::MissingChunk(hash).into());
}
}
Ok(new)
}
pub fn analyze_usage(&mut self) -> Result<HashMap<u32, BundleAnalysis>, RepositoryError> {
if self.dirty {
return Err(RepositoryError::Dirty);
}
try!(self.set_dirty());
let mut usage = HashMap::new();
for (id, bundle) in self.bundle_map.bundles() {
let bundle = try!(self.bundles.get_bundle_info(&bundle).ok_or_else(|| {
IntegrityError::MissingBundle(bundle)
}));
usage.insert(
id,
BundleAnalysis {
chunk_usage: Bitmap::new(bundle.info.chunk_count),
info: bundle.info.clone(),
used_raw_size: 0
}
);
}
let backups = try!(self.get_all_backups());
let mut todo = VecDeque::new();
for (_name, backup) in backups {
todo.push_back(backup.root);
}
while let Some(chunks) = todo.pop_back() {
if !try!(self.mark_used(&mut usage, &chunks)) {
continue;
}
let inode = try!(self.get_inode(&chunks));
// Mark the content chunks as used
match inode.data {
None |
Some(FileData::Inline(_)) => (),
Some(FileData::ChunkedDirect(chunks)) => {
try!(self.mark_used(&mut usage, &chunks));
}
Some(FileData::ChunkedIndirect(chunks)) => {
if try!(self.mark_used(&mut usage, &chunks)) {
let chunk_data = try!(self.get_data(&chunks));
let chunks = ChunkList::read_from(&chunk_data);
try!(self.mark_used(&mut usage, &chunks));
}
}
}
// Put children in to do
if let Some(children) = inode.children {
for (_name, chunks) in children {
todo.push_back(chunks);
}
}
}
self.dirty = false;
Ok(usage)
}
#[inline]
pub fn list_bundles(&self) -> Vec<&BundleInfo> {
self.bundles.list_bundles()
}
pub fn get_bundle_map(&self) -> Result<HashMap<u32, &BundleInfo>, RepositoryError> {
let mut map = HashMap::with_capacity(self.bundle_map.len());
for (id, bundle_id) in self.bundle_map.bundles() {
let info = try!(self.bundles.get_bundle_info(&bundle_id).ok_or_else(|| {
IntegrityError::MissingBundle(bundle_id)
}));
map.insert(id, &info.info);
}
Ok(map)
}
#[inline]
pub fn get_bundle(&self, bundle: &BundleId) -> Option<&StoredBundle> {
self.bundles.get_bundle_info(bundle)

424
src/repository/integrity.rs
View File

@ -2,7 +2,6 @@ use prelude::*;
use super::*;
use std::path::{Path, PathBuf};
use std::time::Duration;
use pbr::ProgressBar;
@ -33,20 +32,35 @@ quick_error!{
MapContainsDuplicates {
description(tr!("Map contains duplicates"))
}
BrokenInode(path: PathBuf, err: Box<RepositoryError>) {
cause(err)
description(tr!("Broken inode"))
display("{}", tr_format!("Broken inode: {:?}\n\tcaused by: {}", path, err))
}
MissingInodeData(path: PathBuf, err: Box<RepositoryError>) {
cause(err)
description(tr!("Missing inode data"))
display("{}", tr_format!("Missing inode data in: {:?}\n\tcaused by: {}", path, err))
}
}
}
impl Repository {
pub fn get_chunk_marker(&self) -> Bitmap {
Bitmap::new(self.index.capacity())
}
pub fn check_chunks(
&self,
checked: &mut Bitmap,
chunks: &[Chunk],
mark: bool,
) -> Result<bool, RepositoryError> {
let mut new = false;
for &(hash, _len) in chunks {
if let Some(pos) = self.index.pos(&hash) {
new |= !checked.get(pos);
if mark {
checked.set(pos);
}
} else {
return Err(IntegrityError::MissingChunk(hash).into());
}
}
Ok(new)
}
fn check_index_chunks(&self) -> Result<(), RepositoryError> {
let mut progress = ProgressBar::new(self.index.len() as u64);
progress.message(tr!("checking index: "));
@ -76,354 +90,6 @@ impl Repository {
Ok(())
}
fn check_chunks(
&self,
checked: &mut Bitmap,
chunks: &[Chunk],
mark: bool,
) -> Result<bool, RepositoryError> {
let mut new = false;
for &(hash, _len) in chunks {
if let Some(pos) = self.index.pos(&hash) {
new |= !checked.get(pos);
if mark {
checked.set(pos);
}
} else {
return Err(IntegrityError::MissingChunk(hash).into());
}
}
Ok(new)
}
fn check_inode_contents(
&mut self,
inode: &Inode,
checked: &mut Bitmap,
) -> Result<(), RepositoryError> {
match inode.data {
None |
Some(FileData::Inline(_)) => (),
Some(FileData::ChunkedDirect(ref chunks)) => {
try!(self.check_chunks(checked, chunks, true));
}
Some(FileData::ChunkedIndirect(ref chunks)) => {
if try!(self.check_chunks(checked, chunks, false)) {
let chunk_data = try!(self.get_data(chunks));
let chunks2 = ChunkList::read_from(&chunk_data);
try!(self.check_chunks(checked, &chunks2, true));
try!(self.check_chunks(checked, chunks, true));
}
}
}
Ok(())
}
fn check_subtree(
&mut self,
path: PathBuf,
chunks: &[Chunk],
checked: &mut Bitmap,
repair: bool,
) -> Result<Option<ChunkList>, RepositoryError> {
let mut modified = false;
match self.check_chunks(checked, chunks, false) {
Ok(false) => return Ok(None),
Ok(true) => (),
Err(err) => return Err(IntegrityError::BrokenInode(path, Box::new(err)).into()),
}
let mut inode = try!(self.get_inode(chunks));
// Mark the content chunks as used
if let Err(err) = self.check_inode_contents(&inode, checked) {
if repair {
tr_warn!(
"Problem detected: data of {:?} is corrupt\n\tcaused by: {}",
path,
err
);
tr_info!("Removing inode data");
inode.data = Some(FileData::Inline(vec![].into()));
inode.size = 0;
modified = true;
} else {
return Err(IntegrityError::MissingInodeData(path, Box::new(err)).into());
}
}
// Put children in to do
if let Some(ref mut children) = inode.children {
let mut removed = vec![];
for (name, chunks) in children.iter_mut() {
match self.check_subtree(path.join(name), chunks, checked, repair) {
Ok(None) => (),
Ok(Some(c)) => {
*chunks = c;
modified = true;
}
Err(err) => {
if repair {
tr_warn!(
"Problem detected: inode {:?} is corrupt\n\tcaused by: {}",
path.join(name),
err
);
tr_info!("Removing broken inode from backup");
removed.push(name.to_string());
modified = true;
} else {
return Err(err);
}
}
}
}
for name in removed {
children.remove(&name);
}
}
if modified {
Ok(Some(try!(self.put_inode(&inode))))
} else {
try!(self.check_chunks(checked, chunks, true));
Ok(None)
}
}
fn evacuate_broken_backup(&self, name: &str) -> Result<(), RepositoryError> {
tr_warn!(
"The backup {} was corrupted and needed to be modified.",
name
);
let src = self.layout.backup_path(name);
let mut dst = src.with_extension("backup.broken");
let mut num = 1;
while dst.exists() {
dst = src.with_extension(&format!("backup.{}.broken", num));
num += 1;
}
if fs::rename(&src, &dst).is_err() {
try!(fs::copy(&src, &dst));
try!(fs::remove_file(&src));
}
tr_info!("The original backup was renamed to {:?}", dst);
Ok(())
}
#[inline]
pub fn check_backup(
&mut self,
name: &str,
backup: &mut Backup,
repair: bool,
) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.write_mode());
Some(self.lock(false))
} else {
None
};
tr_info!("Checking backup...");
let mut checked = Bitmap::new(self.index.capacity());
match self.check_subtree(
Path::new("").to_path_buf(),
&backup.root,
&mut checked,
repair
) {
Ok(None) => (),
Ok(Some(chunks)) => {
try!(self.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(name));
try!(self.save_backup(backup, name));
}
Err(err) => {
if repair {
tr_warn!(
"The root of the backup {} has been corrupted\n\tcaused by: {}",
name,
err
);
try!(self.evacuate_broken_backup(name));
} else {
return Err(err);
}
}
}
Ok(())
}
pub fn check_backup_inode(
&mut self,
name: &str,
backup: &mut Backup,
path: &Path,
repair: bool,
) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.write_mode());
Some(self.lock(false))
} else {
None
};
tr_info!("Checking inode...");
let mut checked = Bitmap::new(self.index.capacity());
let mut inodes = try!(self.get_backup_path(backup, path));
let mut inode = inodes.pop().unwrap();
let mut modified = false;
if let Err(err) = self.check_inode_contents(&inode, &mut checked) {
if repair {
tr_warn!(
"Problem detected: data of {:?} is corrupt\n\tcaused by: {}",
path,
err
);
tr_info!("Removing inode data");
inode.data = Some(FileData::Inline(vec![].into()));
inode.size = 0;
modified = true;
} else {
return Err(
IntegrityError::MissingInodeData(path.to_path_buf(), Box::new(err)).into()
);
}
}
if let Some(ref mut children) = inode.children {
let mut removed = vec![];
for (name, chunks) in children.iter_mut() {
match self.check_subtree(path.join(name), chunks, &mut checked, repair) {
Ok(None) => (),
Ok(Some(c)) => {
*chunks = c;
modified = true;
}
Err(err) => {
if repair {
tr_warn!(
"Problem detected: inode {:?} is corrupt\n\tcaused by: {}",
path.join(name),
err
);
tr_info!("Removing broken inode from backup");
removed.push(name.to_string());
modified = true;
} else {
return Err(err);
}
}
}
}
for name in removed {
children.remove(&name);
}
}
let mut chunks = try!(self.put_inode(&inode));
while let Some(mut parent) = inodes.pop() {
parent.children.as_mut().unwrap().insert(inode.name, chunks);
inode = parent;
chunks = try!(self.put_inode(&inode));
}
if modified {
try!(self.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(name));
try!(self.save_backup(backup, name));
}
Ok(())
}
pub fn check_backups(&mut self, repair: bool) -> Result<(), RepositoryError> {
let _lock = if repair {
try!(self.write_mode());
Some(self.lock(false))
} else {
None
};
tr_info!("Checking backups...");
let mut checked = Bitmap::new(self.index.capacity());
let backup_map = match self.get_all_backups() {
Ok(backup_map) => backup_map,
Err(RepositoryError::BackupFile(BackupFileError::PartialBackupsList(backup_map,
_failed))) => {
tr_warn!("Some backups could not be read, ignoring them");
backup_map
}
Err(err) => return Err(err),
};
for (name, mut backup) in
ProgressIter::new(tr!("checking backups"), backup_map.len(), backup_map.into_iter())
{
let path = format!("{}::", name);
match self.check_subtree(
Path::new(&path).to_path_buf(),
&backup.root,
&mut checked,
repair
) {
Ok(None) => (),
Ok(Some(chunks)) => {
try!(self.flush());
backup.root = chunks;
backup.modified = true;
try!(self.evacuate_broken_backup(&name));
try!(self.save_backup(&backup, &name));
}
Err(err) => {
if repair {
tr_warn!(
"The root of the backup {} has been corrupted\n\tcaused by: {}",
name,
err
);
try!(self.evacuate_broken_backup(&name));
} else {
return Err(err);
}
}
}
}
Ok(())
}
pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> {
tr_info!("Checking repository integrity...");
let mut rebuild = false;
for (_id, bundle_id) in self.bundle_map.bundles() {
if self.bundles.get_bundle_info(&bundle_id).is_none() {
if repair {
tr_warn!(
"Problem detected: bundle map contains unknown bundle {}",
bundle_id
);
rebuild = true;
} else {
return Err(IntegrityError::MissingBundle(bundle_id).into());
}
}
}
if self.bundle_map.len() < self.bundles.len() {
if repair {
tr_warn!("Problem detected: bundle map does not contain all remote bundles");
rebuild = true;
} else {
return Err(IntegrityError::RemoteBundlesNotInMap.into());
}
}
if self.bundle_map.len() > self.bundles.len() {
if repair {
tr_warn!("Problem detected: bundle map contains bundles multiple times");
rebuild = true;
} else {
return Err(IntegrityError::MapContainsDuplicates.into());
}
}
if rebuild {
try!(self.rebuild_bundle_map());
try!(self.rebuild_index());
}
Ok(())
}
pub fn rebuild_bundle_map(&mut self) -> Result<(), RepositoryError> {
tr_info!("Rebuilding bundle map from bundles");
self.bundle_map = BundleMap::create();
@ -509,4 +175,44 @@ impl Repository {
}
Ok(())
}
pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> {
tr_info!("Checking repository integrity...");
let mut rebuild = false;
for (_id, bundle_id) in self.bundle_map.bundles() {
if self.bundles.get_bundle_info(&bundle_id).is_none() {
if repair {
tr_warn!(
"Problem detected: bundle map contains unknown bundle {}",
bundle_id
);
rebuild = true;
} else {
return Err(IntegrityError::MissingBundle(bundle_id).into());
}
}
}
if self.bundle_map.len() < self.bundles.len() {
if repair {
tr_warn!("Problem detected: bundle map does not contain all remote bundles");
rebuild = true;
} else {
return Err(IntegrityError::RemoteBundlesNotInMap.into());
}
}
if self.bundle_map.len() > self.bundles.len() {
if repair {
tr_warn!("Problem detected: bundle map contains bundles multiple times");
rebuild = true;
} else {
return Err(IntegrityError::MapContainsDuplicates.into());
}
}
if rebuild {
try!(self.rebuild_bundle_map());
try!(self.rebuild_index());
}
Ok(())
}
}

6
src/repository/layout.rs
View File

@ -23,7 +23,6 @@ pub trait ChunkRepositoryLayout {
fn config_path(&self) -> PathBuf;
fn keys_path(&self) -> PathBuf;
fn excludes_path(&self) -> PathBuf;
fn backups_path(&self) -> PathBuf;
fn backup_path(&self, name: &str) -> PathBuf;
@ -182,11 +181,6 @@ impl ChunkRepositoryLayout for RepositoryLayout {
self.0.join("config.yaml")
}
#[inline]
fn keys_path(&self) -> PathBuf {
self.0.join("keys")
}
#[inline]
fn excludes_path(&self) -> PathBuf {
self.0.join("excludes")

351
src/repository/metadata.rs
View File

@ -1,353 +1,8 @@
use prelude::*;
use filetime::{self, FileTime};
use xattr;
use libc;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::fs::{self, File, Permissions};
use std::os::linux::fs::MetadataExt;
use std::os::unix::fs::{FileTypeExt, PermissionsExt, MetadataExt as UnixMetadataExt, symlink};
use std::io::{self, Read, Write};
use std::os::unix::ffi::OsStrExt;
use std::fmt;
use std::ffi;
quick_error!{
#[derive(Debug)]
pub enum InodeError {
UnsupportedFiletype(path: PathBuf) {
description(tr!("Unsupported file type"))
display("{}", tr_format!("Inode error: file {:?} has an unsupported type", path))
}
ReadMetadata(err: io::Error, path: PathBuf) {
cause(err)
description(tr!("Failed to obtain metadata for file"))
display("{}", tr_format!("Inode error: failed to obtain metadata for file {:?}\n\tcaused by: {}", path, err))
}
ReadXattr(err: io::Error, path: PathBuf) {
cause(err)
description(tr!("Failed to obtain xattr for file"))
display("{}", tr_format!("Inode error: failed to obtain xattr for file {:?}\n\tcaused by: {}", path, err))
}
ReadLinkTarget(err: io::Error, path: PathBuf) {
cause(err)
description(tr!("Failed to obtain link target for file"))
display("{}", tr_format!("Inode error: failed to obtain link target for file {:?}\n\tcaused by: {}", path, err))
}
Create(err: io::Error, path: PathBuf) {
cause(err)
description(tr!("Failed to create entity"))
display("{}", tr_format!("Inode error: failed to create entity {:?}\n\tcaused by: {}", path, err))
}
Integrity(reason: &'static str) {
description(tr!("Integrity error"))
display("{}", tr_format!("Inode error: inode integrity error: {}", reason))
}
Decode(err: msgpack::DecodeError) {
from()
cause(err)
description(tr!("Failed to decode metadata"))
display("{}", tr_format!("Inode error: failed to decode metadata\n\tcaused by: {}", err))
}
Encode(err: msgpack::EncodeError) {
from()
cause(err)
description(tr!("Failed to encode metadata"))
display("{}", tr_format!("Inode error: failed to encode metadata\n\tcaused by: {}", err))
}
}
}
#[derive(Debug, Eq, PartialEq, Clone, Copy, Hash)]
pub enum FileType {
File,
Directory,
Symlink,
BlockDevice,
CharDevice,
NamedPipe
}
serde_impl!(FileType(u8) {
File => 0,
Directory => 1,
Symlink => 2,
BlockDevice => 3,
CharDevice => 4,
NamedPipe => 5
});
impl fmt::Display for FileType {
fn fmt(&self, format: &mut fmt::Formatter) -> Result<(), fmt::Error> {
match *self {
FileType::File => write!(format, "{}", tr!("file")),
FileType::Directory => write!(format, "{}", tr!("directory")),
FileType::Symlink => write!(format, "{}", tr!("symlink")),
FileType::BlockDevice => write!(format, "{}", tr!("block device")),
FileType::CharDevice => write!(format, "{}", tr!("char device")),
FileType::NamedPipe => write!(format, "{}", tr!("named pipe")),
}
}
}
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub enum FileData {
Inline(msgpack::Bytes),
ChunkedDirect(ChunkList),
ChunkedIndirect(ChunkList)
}
serde_impl!(FileData(u8) {
Inline(ByteBuf) => 0,
ChunkedDirect(ChunkList) => 1,
ChunkedIndirect(ChunkList) => 2
});
#[derive(Debug, Hash, Eq, PartialEq)]
pub struct Inode {
pub name: String,
pub size: u64,
pub file_type: FileType,
pub mode: u32,
pub user: u32,
pub group: u32,
pub timestamp: i64,
pub symlink_target: Option<String>,
pub data: Option<FileData>,
pub children: Option<BTreeMap<String, ChunkList>>,
pub cum_size: u64,
pub cum_dirs: usize,
pub cum_files: usize,
pub xattrs: BTreeMap<String, msgpack::Bytes>,
pub device: Option<(u32, u32)>
}
impl Default for Inode {
fn default() -> Self {
Inode {
name: "".to_string(),
size: 0,
file_type: FileType::File,
mode: 0o644,
user: 1000,
group: 1000,
timestamp: 0,
symlink_target: None,
data: None,
children: None,
cum_size: 0,
cum_dirs: 0,
cum_files: 0,
xattrs: BTreeMap::new(),
device: None
}
}
}
serde_impl!(Inode(u8?) {
name: String => 0,
size: u64 => 1,
file_type: FileType => 2,
mode: u32 => 3,
user: u32 => 4,
group: u32 => 5,
timestamp: i64 => 7,
symlink_target: Option<String> => 9,
data: Option<FileData> => 10,
children: Option<BTreeMap<String, ChunkList>> => 11,
cum_size: u64 => 12,
cum_dirs: usize => 13,
cum_files: usize => 14,
xattrs: BTreeMap<String, msgpack::Bytes> => 15,
device: Option<(u32, u32)> => 16
});
impl Inode {
pub fn get_from<P: AsRef<Path>>(path: P) -> Result<Self, InodeError> {
let path = path.as_ref();
let name = path.file_name()
.map(|s| s.to_string_lossy().to_string())
.unwrap_or_else(|| "_".to_string());
let meta = try!(fs::symlink_metadata(path).map_err(|e| {
InodeError::ReadMetadata(e, path.to_owned())
}));
let mut inode = Inode::default();
inode.name = name;
if meta.is_file() {
inode.size = meta.len();
}
inode.file_type = if meta.is_file() {
FileType::File
} else if meta.is_dir() {
FileType::Directory
} else if meta.file_type().is_symlink() {
FileType::Symlink
} else if meta.file_type().is_block_device() {
FileType::BlockDevice
} else if meta.file_type().is_char_device() {
FileType::CharDevice
} else if meta.file_type().is_fifo() {
FileType::NamedPipe
} else {
return Err(InodeError::UnsupportedFiletype(path.to_owned()));
};
if meta.file_type().is_symlink() {
inode.symlink_target = Some(
try!(fs::read_link(path).map_err(|e| {
InodeError::ReadLinkTarget(e, path.to_owned())
})).to_string_lossy()
.to_string()
);
}
if meta.file_type().is_block_device() || meta.file_type().is_char_device() {
let rdev = meta.rdev();
let major = (rdev >> 8) as u32;
let minor = (rdev & 0xff) as u32;
inode.device = Some((major, minor));
}
inode.mode = meta.permissions().mode();
inode.user = meta.st_uid();
inode.group = meta.st_gid();
inode.timestamp = meta.st_mtime();
if xattr::SUPPORTED_PLATFORM {
if let Ok(attrs) = xattr::list(path) {
for name in attrs {
if let Some(data) = try!(xattr::get(path, &name).map_err(|e| {
InodeError::ReadXattr(e, path.to_owned())
}))
{
inode.xattrs.insert(
name.to_string_lossy().to_string(),
data.into()
);
}
}
}
}
Ok(inode)
}
pub fn create_at<P: AsRef<Path>>(&self, path: P) -> Result<Option<File>, InodeError> {
let full_path = path.as_ref().join(&self.name);
let mut file = None;
match self.file_type {
FileType::File => {
file = Some(try!(File::create(&full_path).map_err(|e| {
InodeError::Create(e, full_path.clone())
})));
}
FileType::Directory => {
try!(fs::create_dir(&full_path).map_err(|e| {
InodeError::Create(e, full_path.clone())
}));
}
FileType::Symlink => {
if let Some(ref src) = self.symlink_target {
try!(symlink(src, &full_path).map_err(|e| {
InodeError::Create(e, full_path.clone())
}));
} else {
return Err(InodeError::Integrity(tr!("Symlink without target")));
}
}
FileType::NamedPipe => {
let name = try!(
ffi::CString::new(full_path.as_os_str().as_bytes())
.map_err(|_| InodeError::Integrity(tr!("Name contains nulls")))
);
let mode = self.mode | libc::S_IFIFO;
if unsafe { libc::mkfifo(name.as_ptr(), mode) } != 0 {
return Err(InodeError::Create(
io::Error::last_os_error(),
full_path.clone()
));
}
}
FileType::BlockDevice | FileType::CharDevice => {
let name = try!(
ffi::CString::new(full_path.as_os_str().as_bytes())
.map_err(|_| InodeError::Integrity(tr!("Name contains nulls")))
);
let mode = self.mode |
match self.file_type {
FileType::BlockDevice => libc::S_IFBLK,
FileType::CharDevice => libc::S_IFCHR,
_ => unreachable!(),
};
let device = if let Some((major, minor)) = self.device {
unsafe { libc::makedev(major, minor) }
} else {
return Err(InodeError::Integrity(tr!("Device without id")));
};
if unsafe { libc::mknod(name.as_ptr(), mode, device) } != 0 {
return Err(InodeError::Create(
io::Error::last_os_error(),
full_path.clone()
));
}
}
}
let time = FileTime::from_seconds_since_1970(self.timestamp as u64, 0);
if let Err(err) = filetime::set_file_times(&full_path, time, time) {
tr_warn!("Failed to set file time on {:?}: {}", full_path, err);
}
if !self.xattrs.is_empty() {
if xattr::SUPPORTED_PLATFORM {
for (name, data) in &self.xattrs {
if let Err(err) = xattr::set(&full_path, name, data) {
tr_warn!("Failed to set xattr {} on {:?}: {}", name, full_path, err);
}
}
} else {
tr_warn!("Not setting xattr on {:?}", full_path);
}
}
if let Err(err) = fs::set_permissions(&full_path, Permissions::from_mode(self.mode)) {
tr_warn!(
"Failed to set permissions {:o} on {:?}: {}",
self.mode,
full_path,
err
);
}
if let Err(err) = chown(&full_path, self.user, self.group) {
tr_warn!(
"Failed to set user {} and group {} on {:?}: {}",
self.user,
self.group,
full_path,
err
);
}
Ok(file)
}
#[inline]
pub fn is_same_meta(&self, other: &Inode) -> bool {
self.file_type == other.file_type && self.size == other.size &&
self.mode == other.mode && self.user == other.user &&
self.group == other.group && self.name == other.name &&
self.timestamp == other.timestamp && self.symlink_target == other.symlink_target
}
#[inline]
pub fn is_same_meta_quick(&self, other: &Inode) -> bool {
self.timestamp == other.timestamp && self.file_type == other.file_type &&
self.size == other.size
}
#[inline]
pub fn encode(&self) -> Result<Vec<u8>, InodeError> {
Ok(try!(msgpack::encode(&self)))
}
#[inline]
pub fn decode(data: &[u8]) -> Result<Self, InodeError> {
Ok(try!(msgpack::decode(data)))
}
}
use std::path::Path;
use std::fs::File;
use std::io::{Read, Write};
impl Repository {

70
src/repository/mod.rs
View File

@ -4,11 +4,8 @@ mod integrity;
mod basic_io;
mod info;
mod metadata;
mod backup;
mod error;
mod vacuum;
mod backup_file;
mod tarfile;
mod layout;
pub mod bundledb;
pub mod index;
@ -26,9 +23,6 @@ use std::io::Write;
pub use self::error::RepositoryError;
pub use self::config::Config;
pub use self::metadata::{Inode, FileType, FileData, InodeError};
pub use self::backup::{BackupError, BackupOptions, DiffType};
pub use self::backup_file::{Backup, BackupFileError};
pub use self::integrity::IntegrityError;
pub use self::info::{RepositoryInfo, BundleAnalysis, RepositoryStatistics};
pub use self::layout::{RepositoryLayout, ChunkRepositoryLayout};
@ -36,7 +30,6 @@ use self::bundle_map::BundleMap;
const REPOSITORY_README: &[u8] = include_bytes!("../../docs/repository_readme.md");
const DEFAULT_EXCLUDES: &[u8] = include_bytes!("../../docs/excludes.default");
const INDEX_MAGIC: [u8; 7] = *b"zvault\x02";
const INDEX_VERSION: u8 = 1;
@ -96,13 +89,9 @@ impl Repository {
pub fn create<R: AsRef<Path>>(
layout: Arc<ChunkRepositoryLayout>,
config: &Config,
crypto: Arc<Crypto>,
remote: R,
) -> Result<Self, RepositoryError> {
try!(fs::create_dir(layout.base_path()));
try!(File::create(layout.excludes_path()).and_then(|mut f| {
f.write_all(DEFAULT_EXCLUDES)
}));
try!(fs::create_dir(layout.keys_path()));
try!(fs::create_dir(layout.local_locks_path()));
try!(symlink(remote, layout.remote_path()));
try!(File::create(layout.remote_readme_path()).and_then(
@ -120,11 +109,11 @@ impl Repository {
));
try!(BundleMap::create().save(layout.bundle_map_path()));
try!(fs::create_dir_all(layout.backups_path()));
Self::open(layout, true)
Self::open(layout, crypto, true)
}
#[allow(unknown_lints, useless_let_if_seq)]
pub fn open(layout: Arc<ChunkRepositoryLayout>, online: bool) -> Result<Self, RepositoryError> {
pub fn open(layout: Arc<ChunkRepositoryLayout>, crypto: Arc<Crypto>, online: bool) -> Result<Self, RepositoryError> {
if !layout.remote_exists() {
return Err(RepositoryError::NoRemote);
}
@ -133,7 +122,6 @@ impl Repository {
try!(fs::create_dir_all(layout.local_locks_path())); // Added after v0.1.0
let local_locks = LockFolder::new(layout.local_locks_path());
let lock = try!(local_locks.lock(false));
let crypto = Arc::new(try!(Crypto::open(layout.keys_path())));
let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone(), online));
let (index, mut rebuild_index) =
match unsafe { Index::open(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION) } {
@ -218,44 +206,6 @@ impl Repository {
Ok(repo)
}
pub fn import<R: AsRef<Path>>(
layout: Arc<ChunkRepositoryLayout>,
remote: R,
key_files: Vec<String>,
) -> Result<Self, RepositoryError> {
let mut repo = try!(Repository::create(layout.clone(), &Config::default(), remote));
for file in key_files {
try!(repo.crypto.register_keyfile(file));
}
repo = try!(Repository::open(layout, true));
let mut backups: Vec<(String, Backup)> = try!(repo.get_all_backups()).into_iter().collect();
backups.sort_by_key(|&(_, ref b)| b.timestamp);
if let Some((name, backup)) = backups.pop() {
tr_info!("Taking configuration from the last backup '{}'", name);
repo.config = backup.config;
try!(repo.save_config())
} else {
tr_warn!(
"No backup found in the repository to take configuration from, please set the configuration manually."
);
}
Ok(repo)
}
#[inline]
pub fn register_key(
&mut self,
public: PublicKey,
secret: SecretKey,
) -> Result<(), RepositoryError> {
try!(self.write_mode());
try!(self.crypto.register_secret_key(
public,
secret
));
Ok(())
}
#[inline]
pub fn save_config(&mut self) -> Result<(), RepositoryError> {
try!(self.write_mode());
@ -278,7 +228,7 @@ impl Repository {
}
#[inline]
fn save_bundle_map(&self) -> Result<(), RepositoryError> {
pub fn save_bundle_map(&self) -> Result<(), RepositoryError> {
try!(self.bundle_map.save(self.layout.bundle_map_path()));
Ok(())
}
@ -385,16 +335,24 @@ impl Repository {
}
#[inline]
fn write_mode(&mut self) -> Result<(), RepositoryError> {
pub fn write_mode(&mut self) -> Result<(), RepositoryError> {
try!(self.local_locks.upgrade(&mut self.lock));
Ok(())
}
#[inline]
fn lock(&self, exclusive: bool) -> Result<LockHandle, RepositoryError> {
pub fn lock(&self, exclusive: bool) -> Result<LockHandle, RepositoryError> {
Ok(try!(self.remote_locks.lock(exclusive)))
}
pub fn is_dirty(&self) -> bool {
self.dirty
}
pub fn get_chunk_location(&self, chunk: Hash) -> Option<Location> {
self.index.get(&chunk)
}
#[inline]
pub fn set_clean(&mut self) {
self.dirty = false;

101
src/repository/vacuum.rs
View File

@ -1,10 +1,10 @@
use prelude::*;
use std::collections::HashSet;
use std::collections::HashMap;
impl Repository {
fn delete_bundle(&mut self, id: u32) -> Result<(), RepositoryError> {
pub fn delete_bundle(&mut self, id: u32) -> Result<(), RepositoryError> {
if let Some(bundle) = self.bundle_map.remove(id) {
try!(self.bundles.delete_bundle(&bundle));
Ok(())
@ -13,92 +13,26 @@ impl Repository {
}
}
pub fn vacuum(
&mut self,
ratio: f32,
combine: bool,
force: bool,
) -> Result<(), RepositoryError> {
try!(self.flush());
tr_info!("Locking repository");
try!(self.write_mode());
let _lock = try!(self.lock(true));
// analyze_usage will set the dirty flag
tr_info!("Analyzing chunk usage");
let usage = try!(self.analyze_usage());
let mut data_total = 0;
let mut data_used = 0;
for bundle in usage.values() {
data_total += bundle.info.encoded_size;
data_used += bundle.get_used_size();
}
tr_info!(
"Usage: {} of {}, {:.1}%",
to_file_size(data_used as u64),
to_file_size(data_total as u64),
data_used as f32 / data_total as f32 * 100.0
);
let mut rewrite_bundles = HashSet::new();
let mut reclaim_space = 0;
let mut rewrite_data = 0;
for (id, bundle) in &usage {
if bundle.get_usage_ratio() <= ratio {
rewrite_bundles.insert(*id);
reclaim_space += bundle.get_unused_size();
rewrite_data += bundle.get_used_size();
}
}
if combine {
let mut small_meta = vec![];
let mut small_data = vec![];
for (id, bundle) in &usage {
if bundle.info.encoded_size * 4 < self.config.bundle_size {
match bundle.info.mode {
BundleMode::Meta => small_meta.push(*id),
BundleMode::Data => small_data.push(*id),
}
}
}
if small_meta.len() >= 2 {
for bundle in small_meta {
rewrite_bundles.insert(bundle);
}
}
if small_data.len() >= 2 {
for bundle in small_data {
rewrite_bundles.insert(bundle);
}
}
}
tr_info!(
"Reclaiming about {} by rewriting {} bundles ({})",
to_file_size(reclaim_space as u64),
rewrite_bundles.len(),
to_file_size(rewrite_data as u64)
);
if !force {
self.dirty = false;
return Ok(());
}
for id in ProgressIter::new(
pub fn rewrite_bundles(&mut self, rewrite_bundles: &[u32], usage: &HashMap<u32, BundleAnalysis>) -> Result<(), RepositoryError> {
for &id in ProgressIter::new(
tr!("rewriting bundles"),
rewrite_bundles.len(),
rewrite_bundles.iter()
)
{
let bundle = &usage[id];
let bundle_id = self.bundle_map.get(*id).unwrap();
let chunks = try!(self.bundles.get_chunk_list(&bundle_id));
let mode = usage[id].info.mode;
for (chunk, &(hash, _len)) in chunks.into_iter().enumerate() {
if !bundle.chunk_usage.get(chunk) {
try!(self.index.delete(&hash));
continue;
{
let bundle = &usage[&id];
let bundle_id = self.bundle_map.get(id).unwrap();
let chunks = try!(self.bundles.get_chunk_list(&bundle_id));
let mode = usage[&id].info.mode;
for (chunk, &(hash, _len)) in chunks.into_iter().enumerate() {
if !bundle.chunk_usage.get(chunk) {
try!(self.index.delete(&hash));
continue;
}
let data = try!(self.bundles.get_chunk(&bundle_id, chunk));
try!(self.put_chunk_override(mode, hash, &data));
}
let data = try!(self.bundles.get_chunk(&bundle_id, chunk));
try!(self.put_chunk_override(mode, hash, &data));
}
}
try!(self.flush());
tr_info!("Checking index");
for (hash, location) in self.index.iter() {
@ -114,11 +48,10 @@ impl Repository {
}
}
tr_info!("Deleting {} bundles", rewrite_bundles.len());
for id in rewrite_bundles {
for &id in rewrite_bundles {
try!(self.delete_bundle(id));
}
try!(self.save_bundle_map());
self.dirty = false;
Ok(())
}
}