diff --git a/src/repository/backup.rs b/src/backups/backup.rs similarity index 91% rename from src/repository/backup.rs rename to src/backups/backup.rs index 264e817..7224c76 100644 --- a/src/repository/backup.rs +++ b/src/backups/backup.rs @@ -14,7 +14,7 @@ quick_error!{ #[derive(Debug)] #[allow(unknown_lints,large_enum_variant)] pub enum BackupError { - FailedPaths(backup: Backup, failed: Vec) { + FailedPaths(backup: BackupFile, failed: Vec) { description(tr!("Some paths could not be backed up")) display("{}", tr_format!("Backup error: some paths could not be backed up")) } @@ -39,9 +39,9 @@ pub enum DiffType { } -impl Repository { - pub fn get_all_backups(&self) -> Result, RepositoryError> { - Ok(try!(Backup::get_all_from( +impl BackupRepository { + pub fn get_all_backups(&self) -> Result, RepositoryError> { + Ok(try!(BackupFile::get_all_from( &self.crypto, self.layout.backups_path() ))) @@ -50,8 +50,8 @@ impl Repository { pub fn get_backups>( &self, path: P, - ) -> Result, RepositoryError> { - Ok(try!(Backup::get_all_from( + ) -> Result, RepositoryError> { + Ok(try!(BackupFile::get_all_from( &self.crypto, self.layout.backups_path().join(path) ))) @@ -62,27 +62,27 @@ impl Repository { self.layout.backup_path(name).exists() } - pub fn get_backup(&self, name: &str) -> Result { - Ok(try!(Backup::read_from( + pub fn get_backup(&self, name: &str) -> Result { + Ok(try!(BackupFile::read_from( &self.crypto, self.layout.backup_path(name) ))) } - pub fn save_backup(&mut self, backup: &Backup, name: &str) -> Result<(), RepositoryError> { - try!(self.write_mode()); + pub fn save_backup(&mut self, backup: &BackupFile, name: &str) -> Result<(), RepositoryError> { + try!(self.repo.write_mode()); let path = self.layout.backup_path(name); try!(fs::create_dir_all(path.parent().unwrap())); try!(backup.save_to( &self.crypto, - self.config.encryption.clone(), + self.get_config().encryption.clone(), path )); Ok(()) } pub fn delete_backup(&mut self, name: &str) -> Result<(), RepositoryError> { - try!(self.write_mode()); + try!(self.repo.write_mode()); let mut path = self.layout.backup_path(name); try!(fs::remove_file(&path)); loop { @@ -104,7 +104,7 @@ impl Repository { yearly: usize, force: bool, ) -> Result<(), RepositoryError> { - try!(self.write_mode()); + try!(self.repo.write_mode()); let mut backups = Vec::new(); let backup_map = match self.get_all_backups() { Ok(backup_map) => backup_map, @@ -125,7 +125,7 @@ impl Repository { let mut keep = Bitmap::new(backups.len()); fn mark_needed) -> K>( - backups: &[(String, DateTime, Backup)], + backups: &[(String, DateTime, BackupFile)], keep: &mut Bitmap, max: usize, keyfn: F, @@ -183,11 +183,11 @@ impl Repository { pub fn restore_inode_tree>( &mut self, - backup: &Backup, + backup: &BackupFile, inode: Inode, path: P, ) -> Result<(), RepositoryError> { - let _lock = try!(self.lock(false)); + let _lock = try!(self.repo.lock(false)); let mut queue = VecDeque::new(); queue.push_back((path.as_ref().to_owned(), inode)); let cache = users::UsersCache::new(); @@ -204,7 +204,7 @@ impl Repository { inode.group = group.gid(); } } - try!(self.save_inode_at(&inode, &path)); + try!(self.repo.save_inode_at(&inode, &path)); } if inode.file_type == FileType::Directory { let path = if is_root { @@ -227,11 +227,11 @@ impl Repository { path: P, reference: Option<&Inode>, options: &BackupOptions, - backup: &mut Backup, + backup: &mut BackupFile, failed_paths: &mut Vec, ) -> Result { let path = path.as_ref(); - let mut inode = try!(self.create_inode(path, reference)); + let mut inode = try!(self.repo.create_inode(path, reference)); if !backup.user_names.contains_key(&inode.user) { if let Some(user) = users::get_user_by_uid(inode.user) { backup.user_names.insert( @@ -296,7 +296,7 @@ impl Repository { } Err(err) => return Err(err), }; - let chunks = try!(self.put_inode(&child_inode)); + let chunks = try!(self.repo.put_inode(&child_inode)); inode.cum_size += child_inode.cum_size; for &(_, len) in chunks.iter() { meta_size += u64::from(len); @@ -328,18 +328,18 @@ impl Repository { pub fn create_backup_recursively>( &mut self, path: P, - reference: Option<&Backup>, + reference: Option<&BackupFile>, options: &BackupOptions, - ) -> Result { - try!(self.write_mode()); - let _lock = try!(self.lock(false)); - if self.dirty { + ) -> Result { + try!(self.repo.write_mode()); + let _lock = try!(self.repo.lock(false)); + if self.repo.is_dirty() { return Err(RepositoryError::Dirty); } - try!(self.set_dirty()); + try!(self.repo.set_dirty()); let reference_inode = reference.and_then(|b| self.get_inode(&b.root).ok()); - let mut backup = Backup::default(); - backup.config = self.config.clone(); + let mut backup = BackupFile::default(); + backup.config = self.get_config().clone(); backup.host = get_hostname().unwrap_or_else(|_| "".to_string()); backup.path = path.as_ref().to_string_lossy().to_string(); let info_before = self.info(); @@ -352,8 +352,8 @@ impl Repository { &mut backup, &mut failed_paths )); - backup.root = try!(self.put_inode(&root_inode)); - try!(self.flush()); + backup.root = try!(self.repo.put_inode(&root_inode)); + try!(self.repo.flush()); let elapsed = Local::now().signed_duration_since(start); backup.timestamp = start.timestamp(); backup.total_data_size = root_inode.cum_size; @@ -369,7 +369,7 @@ impl Repository { backup.bundle_count = info_after.bundle_count - info_before.bundle_count; backup.chunk_count = info_after.chunk_count - info_before.chunk_count; backup.avg_chunk_size = backup.deduplicated_data_size as f32 / backup.chunk_count as f32; - self.dirty = false; + self.repo.set_clean(); if failed_paths.is_empty() { Ok(backup) } else { @@ -379,11 +379,11 @@ impl Repository { pub fn remove_backup_path>( &mut self, - backup: &mut Backup, + backup: &mut BackupFile, path: P, ) -> Result<(), RepositoryError> { - try!(self.write_mode()); - let _lock = try!(self.lock(false)); + try!(self.repo.write_mode()); + let _lock = try!(self.repo.lock(false)); let mut inodes = try!(self.get_backup_path(backup, path)); let to_remove = inodes.pop().unwrap(); let mut remove_from = match inodes.pop() { @@ -393,14 +393,14 @@ impl Repository { remove_from.children.as_mut().unwrap().remove( &to_remove.name ); - let mut last_inode_chunks = try!(self.put_inode(&remove_from)); + let mut last_inode_chunks = try!(self.repo.put_inode(&remove_from)); let mut last_inode_name = remove_from.name; while let Some(mut inode) = inodes.pop() { inode.children.as_mut().unwrap().insert( last_inode_name, last_inode_chunks ); - last_inode_chunks = try!(self.put_inode(&inode)); + last_inode_chunks = try!(self.repo.put_inode(&inode)); last_inode_name = inode.name; } backup.root = last_inode_chunks; @@ -410,7 +410,7 @@ impl Repository { pub fn get_backup_path>( &mut self, - backup: &Backup, + backup: &BackupFile, path: P, ) -> Result, RepositoryError> { let mut inodes = vec![]; @@ -444,7 +444,7 @@ impl Repository { #[inline] pub fn get_backup_inode>( &mut self, - backup: &Backup, + backup: &BackupFile, path: P, ) -> Result { self.get_backup_path(backup, path).map(|mut inodes| { diff --git a/src/repository/backup_file.rs b/src/backups/backup_file.rs similarity index 93% rename from src/repository/backup_file.rs rename to src/backups/backup_file.rs index 767c8f2..eca9648 100644 --- a/src/repository/backup_file.rs +++ b/src/backups/backup_file.rs @@ -55,24 +55,25 @@ quick_error!{ description(tr!("Encryption failed")) display("{}", tr_format!("Backup file error: encryption failed\n\tcaused by: {}", err)) } - PartialBackupsList(partial: HashMap, failed: Vec) { + PartialBackupsList(partial: HashMap, failed: Vec) { description(tr!("Some backups could not be loaded")) display("{}", tr_format!("Backup file error: some backups could not be loaded: {:?}", failed)) } } } + #[derive(Default, Debug, Clone)] -struct BackupHeader { +struct BackupFileHeader { pub encryption: Option } -serde_impl!(BackupHeader(u8) { +serde_impl!(BackupFileHeader(u8) { encryption: Option => 0 }); #[derive(Default, Debug, Clone)] -pub struct Backup { +pub struct BackupFile { pub root: ChunkList, pub total_data_size: u64, // Sum of all raw sizes of all entities pub changed_data_size: u64, // Sum of all raw sizes of all entities actively stored @@ -92,7 +93,7 @@ pub struct Backup { pub user_names: HashMap, pub group_names: HashMap } -serde_impl!(Backup(u8?) { +serde_impl!(BackupFile(u8?) { root: ChunkList => 0, total_data_size: u64 => 1, changed_data_size: u64 => 2, @@ -113,7 +114,7 @@ serde_impl!(Backup(u8?) { group_names: HashMap => 17 }); -impl Backup { +impl BackupFile { pub fn read_from>(crypto: &Crypto, path: P) -> Result { let path = path.as_ref(); let mut file = BufReader::new(try!(File::open(path).map_err(|err| { @@ -133,7 +134,7 @@ impl Backup { version )); } - let header: BackupHeader = try!(msgpack::decode_from_stream(&mut file).context(path)); + let header: BackupFileHeader = try!(msgpack::decode_from_stream(&mut file).context(path)); let mut data = Vec::new(); try!(file.read_to_end(&mut data).map_err(|err| { BackupFileError::Read(err, path.to_path_buf()) @@ -164,7 +165,7 @@ impl Backup { try!(file.write_all(&[HEADER_VERSION]).map_err(|err| { BackupFileError::Write(err, path.to_path_buf()) })); - let header = BackupHeader { encryption }; + let header = BackupFileHeader { encryption }; try!(msgpack::encode_to_stream(&header, &mut file).context(path)); try!(file.write_all(&data).map_err(|err| { BackupFileError::Write(err, path.to_path_buf()) @@ -175,7 +176,7 @@ impl Backup { pub fn get_all_from>( crypto: &Crypto, path: P, - ) -> Result, BackupFileError> { + ) -> Result, BackupFileError> { let mut backups = HashMap::new(); let base_path = path.as_ref(); let path = path.as_ref(); @@ -203,7 +204,7 @@ impl Backup { .with_file_name(relpath.file_stem().unwrap()) .to_string_lossy() .to_string(); - if let Ok(backup) = Backup::read_from(crypto, &path) { + if let Ok(backup) = BackupFile::read_from(crypto, &path) { backups.insert(name, backup); } else { failed_paths.push(path.clone()); diff --git a/src/backups/inode.rs b/src/backups/inode.rs new file mode 100644 index 0000000..9ac887a --- /dev/null +++ b/src/backups/inode.rs @@ -0,0 +1,350 @@ +use prelude::*; + +use filetime::{self, FileTime}; +use xattr; +use libc; + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::fs::{self, File, Permissions}; +use std::os::linux::fs::MetadataExt; +use std::os::unix::fs::{FileTypeExt, PermissionsExt, MetadataExt as UnixMetadataExt, symlink}; +use std::io; +use std::os::unix::ffi::OsStrExt; +use std::fmt; +use std::ffi; + + +quick_error!{ + #[derive(Debug)] + pub enum InodeError { + UnsupportedFiletype(path: PathBuf) { + description(tr!("Unsupported file type")) + display("{}", tr_format!("Inode error: file {:?} has an unsupported type", path)) + } + ReadMetadata(err: io::Error, path: PathBuf) { + cause(err) + description(tr!("Failed to obtain metadata for file")) + display("{}", tr_format!("Inode error: failed to obtain metadata for file {:?}\n\tcaused by: {}", path, err)) + } + ReadXattr(err: io::Error, path: PathBuf) { + cause(err) + description(tr!("Failed to obtain xattr for file")) + display("{}", tr_format!("Inode error: failed to obtain xattr for file {:?}\n\tcaused by: {}", path, err)) + } + ReadLinkTarget(err: io::Error, path: PathBuf) { + cause(err) + description(tr!("Failed to obtain link target for file")) + display("{}", tr_format!("Inode error: failed to obtain link target for file {:?}\n\tcaused by: {}", path, err)) + } + Create(err: io::Error, path: PathBuf) { + cause(err) + description(tr!("Failed to create entity")) + display("{}", tr_format!("Inode error: failed to create entity {:?}\n\tcaused by: {}", path, err)) + } + Integrity(reason: &'static str) { + description(tr!("Integrity error")) + display("{}", tr_format!("Inode error: inode integrity error: {}", reason)) + } + Decode(err: msgpack::DecodeError) { + from() + cause(err) + description(tr!("Failed to decode metadata")) + display("{}", tr_format!("Inode error: failed to decode metadata\n\tcaused by: {}", err)) + } + Encode(err: msgpack::EncodeError) { + from() + cause(err) + description(tr!("Failed to encode metadata")) + display("{}", tr_format!("Inode error: failed to encode metadata\n\tcaused by: {}", err)) + } + } +} + + +#[derive(Debug, Eq, PartialEq, Clone, Copy, Hash)] +pub enum FileType { + File, + Directory, + Symlink, + BlockDevice, + CharDevice, + NamedPipe +} +serde_impl!(FileType(u8) { + File => 0, + Directory => 1, + Symlink => 2, + BlockDevice => 3, + CharDevice => 4, + NamedPipe => 5 +}); +impl fmt::Display for FileType { + fn fmt(&self, format: &mut fmt::Formatter) -> Result<(), fmt::Error> { + match *self { + FileType::File => write!(format, "{}", tr!("file")), + FileType::Directory => write!(format, "{}", tr!("directory")), + FileType::Symlink => write!(format, "{}", tr!("symlink")), + FileType::BlockDevice => write!(format, "{}", tr!("block device")), + FileType::CharDevice => write!(format, "{}", tr!("char device")), + FileType::NamedPipe => write!(format, "{}", tr!("named pipe")), + } + } +} + + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum FileData { + Inline(msgpack::Bytes), + ChunkedDirect(ChunkList), + ChunkedIndirect(ChunkList) +} +serde_impl!(FileData(u8) { + Inline(ByteBuf) => 0, + ChunkedDirect(ChunkList) => 1, + ChunkedIndirect(ChunkList) => 2 +}); + + +#[derive(Debug, Hash, Eq, PartialEq)] +pub struct Inode { + pub name: String, + pub size: u64, + pub file_type: FileType, + pub mode: u32, + pub user: u32, + pub group: u32, + pub timestamp: i64, + pub symlink_target: Option, + pub data: Option, + pub children: Option>, + pub cum_size: u64, + pub cum_dirs: usize, + pub cum_files: usize, + pub xattrs: BTreeMap, + pub device: Option<(u32, u32)> +} +impl Default for Inode { + fn default() -> Self { + Inode { + name: "".to_string(), + size: 0, + file_type: FileType::File, + mode: 0o644, + user: 1000, + group: 1000, + timestamp: 0, + symlink_target: None, + data: None, + children: None, + cum_size: 0, + cum_dirs: 0, + cum_files: 0, + xattrs: BTreeMap::new(), + device: None + } + } +} +serde_impl!(Inode(u8?) { + name: String => 0, + size: u64 => 1, + file_type: FileType => 2, + mode: u32 => 3, + user: u32 => 4, + group: u32 => 5, + timestamp: i64 => 7, + symlink_target: Option => 9, + data: Option => 10, + children: Option> => 11, + cum_size: u64 => 12, + cum_dirs: usize => 13, + cum_files: usize => 14, + xattrs: BTreeMap => 15, + device: Option<(u32, u32)> => 16 +}); + + +impl Inode { + pub fn get_from>(path: P) -> Result { + let path = path.as_ref(); + let name = path.file_name() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_else(|| "_".to_string()); + let meta = try!(fs::symlink_metadata(path).map_err(|e| { + InodeError::ReadMetadata(e, path.to_owned()) + })); + let mut inode = Inode::default(); + inode.name = name; + if meta.is_file() { + inode.size = meta.len(); + } + inode.file_type = if meta.is_file() { + FileType::File + } else if meta.is_dir() { + FileType::Directory + } else if meta.file_type().is_symlink() { + FileType::Symlink + } else if meta.file_type().is_block_device() { + FileType::BlockDevice + } else if meta.file_type().is_char_device() { + FileType::CharDevice + } else if meta.file_type().is_fifo() { + FileType::NamedPipe + } else { + return Err(InodeError::UnsupportedFiletype(path.to_owned())); + }; + if meta.file_type().is_symlink() { + inode.symlink_target = Some( + try!(fs::read_link(path).map_err(|e| { + InodeError::ReadLinkTarget(e, path.to_owned()) + })).to_string_lossy() + .to_string() + ); + } + if meta.file_type().is_block_device() || meta.file_type().is_char_device() { + let rdev = meta.rdev(); + let major = (rdev >> 8) as u32; + let minor = (rdev & 0xff) as u32; + inode.device = Some((major, minor)); + } + inode.mode = meta.permissions().mode(); + inode.user = meta.st_uid(); + inode.group = meta.st_gid(); + inode.timestamp = meta.st_mtime(); + if xattr::SUPPORTED_PLATFORM { + if let Ok(attrs) = xattr::list(path) { + for name in attrs { + if let Some(data) = try!(xattr::get(path, &name).map_err(|e| { + InodeError::ReadXattr(e, path.to_owned()) + })) + { + inode.xattrs.insert( + name.to_string_lossy().to_string(), + data.into() + ); + } + } + } + } + Ok(inode) + } + + pub fn create_at>(&self, path: P) -> Result, InodeError> { + let full_path = path.as_ref().join(&self.name); + let mut file = None; + match self.file_type { + FileType::File => { + file = Some(try!(File::create(&full_path).map_err(|e| { + InodeError::Create(e, full_path.clone()) + }))); + } + FileType::Directory => { + try!(fs::create_dir(&full_path).map_err(|e| { + InodeError::Create(e, full_path.clone()) + })); + } + FileType::Symlink => { + if let Some(ref src) = self.symlink_target { + try!(symlink(src, &full_path).map_err(|e| { + InodeError::Create(e, full_path.clone()) + })); + } else { + return Err(InodeError::Integrity(tr!("Symlink without target"))); + } + } + FileType::NamedPipe => { + let name = try!( + ffi::CString::new(full_path.as_os_str().as_bytes()) + .map_err(|_| InodeError::Integrity(tr!("Name contains nulls"))) + ); + let mode = self.mode | libc::S_IFIFO; + if unsafe { libc::mkfifo(name.as_ptr(), mode) } != 0 { + return Err(InodeError::Create( + io::Error::last_os_error(), + full_path.clone() + )); + } + } + FileType::BlockDevice | FileType::CharDevice => { + let name = try!( + ffi::CString::new(full_path.as_os_str().as_bytes()) + .map_err(|_| InodeError::Integrity(tr!("Name contains nulls"))) + ); + let mode = self.mode | + match self.file_type { + FileType::BlockDevice => libc::S_IFBLK, + FileType::CharDevice => libc::S_IFCHR, + _ => unreachable!(), + }; + let device = if let Some((major, minor)) = self.device { + unsafe { libc::makedev(major, minor) } + } else { + return Err(InodeError::Integrity(tr!("Device without id"))); + }; + if unsafe { libc::mknod(name.as_ptr(), mode, device) } != 0 { + return Err(InodeError::Create( + io::Error::last_os_error(), + full_path.clone() + )); + } + } + } + let time = FileTime::from_seconds_since_1970(self.timestamp as u64, 0); + if let Err(err) = filetime::set_file_times(&full_path, time, time) { + tr_warn!("Failed to set file time on {:?}: {}", full_path, err); + } + if !self.xattrs.is_empty() { + if xattr::SUPPORTED_PLATFORM { + for (name, data) in &self.xattrs { + if let Err(err) = xattr::set(&full_path, name, data) { + tr_warn!("Failed to set xattr {} on {:?}: {}", name, full_path, err); + } + } + } else { + tr_warn!("Not setting xattr on {:?}", full_path); + } + } + if let Err(err) = fs::set_permissions(&full_path, Permissions::from_mode(self.mode)) { + tr_warn!( + "Failed to set permissions {:o} on {:?}: {}", + self.mode, + full_path, + err + ); + } + if let Err(err) = chown(&full_path, self.user, self.group) { + tr_warn!( + "Failed to set user {} and group {} on {:?}: {}", + self.user, + self.group, + full_path, + err + ); + } + Ok(file) + } + + #[inline] + pub fn is_same_meta(&self, other: &Inode) -> bool { + self.file_type == other.file_type && self.size == other.size && + self.mode == other.mode && self.user == other.user && + self.group == other.group && self.name == other.name && + self.timestamp == other.timestamp && self.symlink_target == other.symlink_target + } + + #[inline] + pub fn is_same_meta_quick(&self, other: &Inode) -> bool { + self.timestamp == other.timestamp && self.file_type == other.file_type && + self.size == other.size + } + + #[inline] + pub fn encode(&self) -> Result, InodeError> { + Ok(try!(msgpack::encode(&self))) + } + + #[inline] + pub fn decode(data: &[u8]) -> Result { + Ok(try!(msgpack::decode(data))) + } +} \ No newline at end of file diff --git a/src/backups/integrity.rs b/src/backups/integrity.rs new file mode 100644 index 0000000..d51aeef --- /dev/null +++ b/src/backups/integrity.rs @@ -0,0 +1,323 @@ +use prelude::*; + +use super::*; + +use std::path::{Path, PathBuf}; + + +quick_error!{ + #[derive(Debug)] + pub enum InodeIntegrityError { + BrokenInode(path: PathBuf, err: Box) { + cause(err) + description(tr!("Broken inode")) + display("{}", tr_format!("Broken inode: {:?}\n\tcaused by: {}", path, err)) + } + MissingInodeData(path: PathBuf, err: Box) { + cause(err) + description(tr!("Missing inode data")) + display("{}", tr_format!("Missing inode data in: {:?}\n\tcaused by: {}", path, err)) + } + } +} + +impl BackupRepository { + fn check_inode_contents( + &mut self, + inode: &Inode, + checked: &mut Bitmap, + ) -> Result<(), RepositoryError> { + match inode.data { + None | + Some(FileData::Inline(_)) => (), + Some(FileData::ChunkedDirect(ref chunks)) => { + try!(self.repo.check_chunks(checked, chunks, true)); + } + Some(FileData::ChunkedIndirect(ref chunks)) => { + if try!(self.repo.check_chunks(checked, chunks, false)) { + let chunk_data = try!(self.get_data(chunks)); + let chunks2 = ChunkList::read_from(&chunk_data); + try!(self.repo.check_chunks(checked, &chunks2, true)); + try!(self.repo.check_chunks(checked, chunks, true)); + } + } + } + Ok(()) + } + + fn check_subtree( + &mut self, + path: PathBuf, + chunks: &[Chunk], + checked: &mut Bitmap, + repair: bool, + ) -> Result, RepositoryError> { + let mut modified = false; + match self.repo.check_chunks(checked, chunks, false) { + Ok(false) => return Ok(None), + Ok(true) => (), + Err(err) => return Err(InodeIntegrityError::BrokenInode(path, Box::new(err)).into()), + } + let mut inode = try!(self.get_inode(chunks)); + // Mark the content chunks as used + if let Err(err) = self.check_inode_contents(&inode, checked) { + if repair { + tr_warn!( + "Problem detected: data of {:?} is corrupt\n\tcaused by: {}", + path, + err + ); + tr_info!("Removing inode data"); + inode.data = Some(FileData::Inline(vec![].into())); + inode.size = 0; + modified = true; + } else { + return Err(InodeIntegrityError::MissingInodeData(path, Box::new(err)).into()); + } + } + // Put children in to do + if let Some(ref mut children) = inode.children { + let mut removed = vec![]; + for (name, chunks) in children.iter_mut() { + match self.check_subtree(path.join(name), chunks, checked, repair) { + Ok(None) => (), + Ok(Some(c)) => { + *chunks = c; + modified = true; + } + Err(err) => { + if repair { + tr_warn!( + "Problem detected: inode {:?} is corrupt\n\tcaused by: {}", + path.join(name), + err + ); + tr_info!("Removing broken inode from backup"); + removed.push(name.to_string()); + modified = true; + } else { + return Err(err); + } + } + } + } + for name in removed { + children.remove(&name); + } + } + if modified { + Ok(Some(try!(self.repo.put_inode(&inode)))) + } else { + try!(self.repo.check_chunks(checked, chunks, true)); + Ok(None) + } + } + + fn evacuate_broken_backup(&self, name: &str) -> Result<(), RepositoryError> { + tr_warn!( + "The backup {} was corrupted and needed to be modified.", + name + ); + let src = self.layout.backup_path(name); + let mut dst = src.with_extension("backup.broken"); + let mut num = 1; + while dst.exists() { + dst = src.with_extension(&format!("backup.{}.broken", num)); + num += 1; + } + if fs::rename(&src, &dst).is_err() { + try!(fs::copy(&src, &dst)); + try!(fs::remove_file(&src)); + } + tr_info!("The original backup was renamed to {:?}", dst); + Ok(()) + } + + #[inline] + pub fn check_backup( + &mut self, + name: &str, + backup: &mut BackupFile, + repair: bool, + ) -> Result<(), RepositoryError> { + let _lock = if repair { + try!(self.repo.write_mode()); + Some(self.repo.lock(false)) + } else { + None + }; + tr_info!("Checking backup..."); + let mut checked = self.repo.get_chunk_marker(); + match self.check_subtree( + Path::new("").to_path_buf(), + &backup.root, + &mut checked, + repair + ) { + Ok(None) => (), + Ok(Some(chunks)) => { + try!(self.repo.flush()); + backup.root = chunks; + backup.modified = true; + try!(self.evacuate_broken_backup(name)); + try!(self.save_backup(backup, name)); + } + Err(err) => { + if repair { + tr_warn!( + "The root of the backup {} has been corrupted\n\tcaused by: {}", + name, + err + ); + try!(self.evacuate_broken_backup(name)); + } else { + return Err(err); + } + } + } + Ok(()) + } + + pub fn check_backup_inode( + &mut self, + name: &str, + backup: &mut BackupFile, + path: &Path, + repair: bool, + ) -> Result<(), RepositoryError> { + let _lock = if repair { + try!(self.repo.write_mode()); + Some(self.repo.lock(false)) + } else { + None + }; + tr_info!("Checking inode..."); + let mut checked = self.repo.get_chunk_marker(); + let mut inodes = try!(self.get_backup_path(backup, path)); + let mut inode = inodes.pop().unwrap(); + let mut modified = false; + if let Err(err) = self.check_inode_contents(&inode, &mut checked) { + if repair { + tr_warn!( + "Problem detected: data of {:?} is corrupt\n\tcaused by: {}", + path, + err + ); + tr_info!("Removing inode data"); + inode.data = Some(FileData::Inline(vec![].into())); + inode.size = 0; + modified = true; + } else { + return Err( + InodeIntegrityError::MissingInodeData(path.to_path_buf(), Box::new(err)).into() + ); + } + } + if let Some(ref mut children) = inode.children { + let mut removed = vec![]; + for (name, chunks) in children.iter_mut() { + match self.check_subtree(path.join(name), chunks, &mut checked, repair) { + Ok(None) => (), + Ok(Some(c)) => { + *chunks = c; + modified = true; + } + Err(err) => { + if repair { + tr_warn!( + "Problem detected: inode {:?} is corrupt\n\tcaused by: {}", + path.join(name), + err + ); + tr_info!("Removing broken inode from backup"); + removed.push(name.to_string()); + modified = true; + } else { + return Err(err); + } + } + } + } + for name in removed { + children.remove(&name); + } + } + let mut chunks = try!(self.repo.put_inode(&inode)); + while let Some(mut parent) = inodes.pop() { + parent.children.as_mut().unwrap().insert(inode.name, chunks); + inode = parent; + chunks = try!(self.repo.put_inode(&inode)); + } + if modified { + try!(self.repo.flush()); + backup.root = chunks; + backup.modified = true; + try!(self.evacuate_broken_backup(name)); + try!(self.save_backup(backup, name)); + } + Ok(()) + } + + pub fn check_backups(&mut self, repair: bool) -> Result<(), RepositoryError> { + let _lock = if repair { + try!(self.repo.write_mode()); + Some(self.repo.lock(false)) + } else { + None + }; + tr_info!("Checking backups..."); + let mut checked = self.repo.get_chunk_marker(); + let backup_map = match self.get_all_backups() { + Ok(backup_map) => backup_map, + Err(RepositoryError::BackupFile(BackupFileError::PartialBackupsList(backup_map, + _failed))) => { + tr_warn!("Some backups could not be read, ignoring them"); + backup_map + } + Err(err) => return Err(err), + }; + for (name, mut backup) in + ProgressIter::new(tr!("checking backups"), backup_map.len(), backup_map.into_iter()) + { + let path = format!("{}::", name); + match self.check_subtree( + Path::new(&path).to_path_buf(), + &backup.root, + &mut checked, + repair + ) { + Ok(None) => (), + Ok(Some(chunks)) => { + try!(self.repo.flush()); + backup.root = chunks; + backup.modified = true; + try!(self.evacuate_broken_backup(&name)); + try!(self.save_backup(&backup, &name)); + } + Err(err) => { + if repair { + tr_warn!( + "The root of the backup {} has been corrupted\n\tcaused by: {}", + name, + err + ); + try!(self.evacuate_broken_backup(&name)); + } else { + return Err(err); + } + } + } + } + Ok(()) + } + + + #[inline] + pub fn check_bundles(&mut self, full: bool, repair: bool) -> Result<(), RepositoryError> { + self.repo.check_bundles(full, repair) + } + + pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> { + self.repo.check_repository(repair) + } +} \ No newline at end of file diff --git a/src/backups/mod.rs b/src/backups/mod.rs index 0c8e0e6..fba391e 100644 --- a/src/backups/mod.rs +++ b/src/backups/mod.rs @@ -1,48 +1,91 @@ pub mod mount; +mod backup_file; +mod inode; +mod tarfile; +mod backup; +mod integrity; +mod vacuum; + +pub use self::backup::{BackupOptions, BackupError, DiffType}; +pub use self::backup_file::{BackupFile, BackupFileError}; +pub use self::inode::{Inode, FileData, FileType, InodeError}; +pub use self::integrity::InodeIntegrityError; use ::prelude::*; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::collections::HashMap; use std::sync::Arc; +use std::fs::{self, File}; +use std::io::Write; + + +const DEFAULT_EXCLUDES: &[u8] = include_bytes!("../../docs/excludes.default"); + pub struct BackupRepository { layout: Arc, + crypto: Arc, repo: Repository } impl BackupRepository { pub fn create, R: AsRef>(path: P, config: &Config, remote: R) -> Result { let layout = Arc::new(RepositoryLayout::new(path.as_ref())); + try!(fs::create_dir(layout.base_path())); + try!(File::create(layout.excludes_path()).and_then(|mut f| { + f.write_all(DEFAULT_EXCLUDES) + })); + try!(fs::create_dir(layout.keys_path())); + let crypto = Arc::new(try!(Crypto::open(layout.keys_path()))); Ok(BackupRepository { + crypto: crypto.clone(), layout: layout.clone(), - repo: try!(Repository::create(layout, config, remote)) + repo: try!(Repository::create(layout, config, crypto, remote)) }) } #[allow(unknown_lints, useless_let_if_seq)] pub fn open>(path: P, online: bool) -> Result { let layout = Arc::new(RepositoryLayout::new(path.as_ref())); + let crypto = Arc::new(try!(Crypto::open(layout.keys_path()))); Ok(BackupRepository { + crypto: crypto.clone(), layout: layout.clone(), - repo: try!(Repository::open(layout, online)) + repo: try!(Repository::open(layout, crypto, online)) }) } pub fn import, R: AsRef>(path: P, remote: R, key_files: Vec) -> Result { - let layout = Arc::new(RepositoryLayout::new(path.as_ref())); - Ok(BackupRepository { - layout: layout.clone(), - repo: try!(Repository::import(layout, remote, key_files)) - }) + let config = Config::default(); + let mut repo = try!(Self::create(&path, &config, remote)); + for file in key_files { + try!(repo.crypto.register_keyfile(file)); + } + repo = try!(Self::open(&path, true)); + let mut backups: Vec<(String, BackupFile)> = try!(repo.get_all_backups()).into_iter().collect(); + backups.sort_by_key(|&(_, ref b)| b.timestamp); + if let Some((name, backup)) = backups.pop() { + tr_info!("Taking configuration from the last backup '{}'", name); + repo.repo.set_config(backup.config); + try!(repo.save_config()) + } else { + tr_warn!( + "No backup found in the repository to take configuration from, please set the configuration manually." + ); + } + Ok(repo) } #[inline] pub fn register_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), RepositoryError> { - self.repo.register_key(public, secret) + try!(self.repo.write_mode()); + try!(self.crypto.register_secret_key(public, secret)); + Ok(()) } + #[inline] pub fn save_config(&mut self) -> Result<(), RepositoryError> { self.repo.save_config() @@ -53,29 +96,11 @@ impl BackupRepository { self.repo.set_encryption(public) } - #[inline] - pub fn has_backup(&self, name: &str) -> bool { - self.repo.has_backup(name) - } - - pub fn get_backup(&self, name: &str) -> Result { - self.repo.get_backup(name) - } - - #[inline] - pub fn get_backup_inode>(&mut self, backup: &Backup, path: P) -> Result { - self.repo.get_backup_inode(backup, path) - } - #[inline] pub fn get_inode(&mut self, chunks: &[Chunk]) -> Result { self.repo.get_inode(chunks) } - pub fn get_all_backups(&self) -> Result, RepositoryError> { - self.repo.get_all_backups() - } - pub fn get_config(&self) -> &Config { self.repo.get_config() } @@ -88,77 +113,15 @@ impl BackupRepository { &self.layout } - pub fn create_backup_recursively>(&mut self, path: P, reference: Option<&Backup>, options: &BackupOptions) -> Result { - self.repo.create_backup_recursively(path, reference, options) - } - - pub fn import_tarfile>(&mut self, tarfile: P) -> Result { - self.repo.import_tarfile(tarfile) - } - - pub fn save_backup(&mut self, backup: &Backup, name: &str) -> Result<(), RepositoryError> { - self.repo.save_backup(backup, name) - } - - pub fn export_tarfile>(&mut self, backup: &Backup, inode: Inode, tarfile: P) -> Result<(), RepositoryError> { - self.repo.export_tarfile(backup, inode, tarfile) - } - - pub fn restore_inode_tree>(&mut self, backup: &Backup, inode: Inode, path: P) -> Result<(), RepositoryError> { - self.repo.restore_inode_tree(backup, inode, path) - } - - pub fn remove_backup_path>(&mut self, backup: &mut Backup, path: P) -> Result<(), RepositoryError> { - self.repo.remove_backup_path(backup, path) - } - - pub fn get_backups>(&self, path: P) -> Result, RepositoryError> { - self.repo.get_backups(path) - } - - pub fn delete_backup(&mut self, name: &str) -> Result<(), RepositoryError> { - self.repo.delete_backup(name) - } - - pub fn prune_backups(&mut self, prefix: &str, daily: usize, weekly: usize, monthly: usize, yearly: usize, force: bool) -> Result<(), RepositoryError> { - self.repo.prune_backups(prefix, daily, weekly, monthly, yearly, force) - } - pub fn info(&self) -> RepositoryInfo { self.repo.info() } - pub fn vacuum(&mut self, ratio: f32, combine: bool, force: bool) -> Result<(), RepositoryError> { - self.repo.vacuum(ratio, combine, force) - } - - pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> { - self.repo.check_repository(repair) - } - - #[inline] - pub fn check_bundles(&mut self, full: bool, repair: bool) -> Result<(), RepositoryError> { - self.repo.check_bundles(full, repair) - } - #[inline] pub fn check_index(&mut self, repair: bool) -> Result<(), RepositoryError> { self.repo.check_index(repair) } - pub fn check_backup_inode(&mut self, name: &str, backup: &mut Backup, path: &Path, repair: bool) -> Result<(), RepositoryError> { - self.repo.check_backup_inode(name, backup, path, repair) - } - - #[inline] - pub fn check_backup(&mut self, name: &str, backup: &mut Backup, repair: bool) -> Result<(), RepositoryError> { - self.repo.check_backup(name, backup, repair) - } - - pub fn check_backups(&mut self, repair: bool) -> Result<(), RepositoryError> { - self.repo.check_backups(repair) - } - #[inline] pub fn set_clean(&mut self) { self.repo.set_clean() @@ -168,14 +131,6 @@ impl BackupRepository { self.repo.statistics() } - pub fn find_duplicates(&mut self, inode: &Inode, min_size: u64) -> Result, u64)>, RepositoryError> { - self.repo.find_duplicates(inode, min_size) - } - - pub fn analyze_usage(&mut self) -> Result, RepositoryError> { - self.repo.analyze_usage() - } - #[inline] pub fn list_bundles(&self) -> Vec<&BundleInfo> { self.repo.list_bundles() @@ -186,15 +141,6 @@ impl BackupRepository { self.repo.get_bundle(bundle) } - pub fn find_versions>(&mut self, path: P) -> Result, RepositoryError> { - self.repo.find_versions(path) - } - - #[inline] - pub fn find_differences(&mut self, inode1: &Inode, inode2: &Inode) -> Result, RepositoryError> { - self.repo.find_differences(inode1, inode2) - } - pub fn get_chunk(&mut self, hash: Hash) -> Result>, RepositoryError> { self.repo.get_chunk(hash) } diff --git a/src/backups/mount.rs b/src/backups/mount.rs index b71b39c..e574033 100644 --- a/src/backups/mount.rs +++ b/src/backups/mount.rs @@ -197,7 +197,7 @@ impl<'a> FuseFilesystem<'a> { pub fn from_backup( repository: &'a mut BackupRepository, - backup: Backup, + backup: BackupFile, ) -> Result { let inode = try!(repository.get_inode(&backup.root)); let mut fs = try!(FuseFilesystem::new(repository)); @@ -207,7 +207,7 @@ impl<'a> FuseFilesystem<'a> { pub fn from_inode( repository: &'a mut BackupRepository, - backup: Backup, + backup: BackupFile, inode: Inode, ) -> Result { let mut fs = try!(FuseFilesystem::new(repository)); diff --git a/src/repository/tarfile.rs b/src/backups/tarfile.rs similarity index 94% rename from src/repository/tarfile.rs rename to src/backups/tarfile.rs index ccb1586..c522d38 100644 --- a/src/repository/tarfile.rs +++ b/src/backups/tarfile.rs @@ -133,7 +133,8 @@ fn inode_from_entry(entry: &mut tar::Entry) -> Result( &mut self, entry: &mut tar::Entry, @@ -144,13 +145,13 @@ impl Repository { try!(entry.read_to_end(&mut data)); inode.data = Some(FileData::Inline(data.into())); } else { - let mut chunks = try!(self.put_stream(BundleMode::Data, entry)); + let mut chunks = try!(self.repo.put_stream(BundleMode::Data, entry)); if chunks.len() < 10 { inode.data = Some(FileData::ChunkedDirect(chunks)); } else { let mut chunk_data = Vec::with_capacity(chunks.encoded_size()); chunks.write_to(&mut chunk_data).unwrap(); - chunks = try!(self.put_data(BundleMode::Meta, &chunk_data)); + chunks = try!(self.repo.put_data(BundleMode::Meta, &chunk_data)); inode.data = Some(FileData::ChunkedIndirect(chunks)); } } @@ -159,7 +160,7 @@ impl Repository { fn import_tarfile_as_inode( &mut self, - backup: &mut Backup, + backup: &mut BackupFile, input: R, failed_paths: &mut Vec, ) -> Result<(Inode, ChunkList), RepositoryError> { @@ -218,7 +219,7 @@ impl Repository { } for path in childless { let (inode, _) = inodes.remove(&path).unwrap(); - let chunks = try!(self.put_inode(&inode)); + let chunks = try!(self.repo.put_inode(&inode)); if let Some(parent_path) = path.parent() { if let Some(&mut (ref mut parent_inode, ref mut children)) = inodes.get_mut(parent_path) @@ -264,7 +265,7 @@ impl Repository { children.insert(inode.name, chunks); } root_inode.children = Some(children); - let chunks = try!(self.put_inode(&root_inode)); + let chunks = try!(self.repo.put_inode(&root_inode)); Ok((root_inode, chunks)) } } @@ -272,15 +273,15 @@ impl Repository { pub fn import_tarfile>( &mut self, tarfile: P, - ) -> Result { - try!(self.write_mode()); - let _lock = try!(self.lock(false)); - if self.dirty { + ) -> Result { + try!(self.repo.write_mode()); + let _lock = try!(self.repo.lock(false)); + if self.repo.is_dirty() { return Err(RepositoryError::Dirty); } - try!(self.set_dirty()); - let mut backup = Backup::default(); - backup.config = self.config.clone(); + try!(self.repo.set_dirty()); + let mut backup = BackupFile::default(); + backup.config = self.repo.get_config().clone(); backup.host = get_hostname().unwrap_or_else(|_| "".to_string()); backup.path = tarfile.as_ref().to_string_lossy().to_string(); let info_before = self.info(); @@ -301,7 +302,7 @@ impl Repository { )) }; backup.root = chunks; - try!(self.flush()); + try!(self.repo.flush()); let elapsed = Local::now().signed_duration_since(start); backup.timestamp = start.timestamp(); backup.total_data_size = root_inode.cum_size; @@ -314,7 +315,7 @@ impl Repository { backup.bundle_count = info_after.bundle_count - info_before.bundle_count; backup.chunk_count = info_after.chunk_count - info_before.chunk_count; backup.avg_chunk_size = backup.deduplicated_data_size as f32 / backup.chunk_count as f32; - self.dirty = false; + self.repo.set_clean(); if failed_paths.is_empty() { Ok(backup) } else { @@ -340,7 +341,7 @@ impl Repository { fn export_tarfile_recurse( &mut self, - backup: &Backup, + backup: &BackupFile, path: &Path, inode: Inode, tarfile: &mut tar::Builder, @@ -396,11 +397,11 @@ impl Repository { None => try!(tarfile.append(&header, Cursor::new(&[]))), Some(FileData::Inline(data)) => try!(tarfile.append(&header, Cursor::new(data))), Some(FileData::ChunkedDirect(chunks)) => { - try!(tarfile.append(&header, self.get_reader(chunks))) + try!(tarfile.append(&header, self.repo.get_reader(chunks))) } Some(FileData::ChunkedIndirect(chunks)) => { let chunks = ChunkList::read_from(&try!(self.get_data(&chunks))); - try!(tarfile.append(&header, self.get_reader(chunks))) + try!(tarfile.append(&header, self.repo.get_reader(chunks))) } } } @@ -421,7 +422,7 @@ impl Repository { pub fn export_tarfile>( &mut self, - backup: &Backup, + backup: &BackupFile, inode: Inode, tarfile: P, ) -> Result<(), RepositoryError> { diff --git a/src/backups/vacuum.rs b/src/backups/vacuum.rs new file mode 100644 index 0000000..bbea6f9 --- /dev/null +++ b/src/backups/vacuum.rs @@ -0,0 +1,158 @@ +use ::prelude::*; + +use super::*; + +use std::collections::{VecDeque, HashSet}; + + +impl BackupRepository { + fn mark_used( + &self, + bundles: &mut HashMap, + chunks: &[Chunk], + ) -> Result { + let mut new = false; + for &(hash, len) in chunks { + if let Some(pos) = self.repo.get_chunk_location(hash) { + let bundle = pos.bundle; + if let Some(bundle) = bundles.get_mut(&bundle) { + if !bundle.chunk_usage.get(pos.chunk as usize) { + new = true; + bundle.chunk_usage.set(pos.chunk as usize); + bundle.used_raw_size += len as usize; + } + } else { + return Err(IntegrityError::MissingBundleId(pos.bundle).into()); + } + } else { + return Err(IntegrityError::MissingChunk(hash).into()); + } + } + Ok(new) + } + + pub fn analyze_usage(&mut self) -> Result, RepositoryError> { + if self.repo.is_dirty() { + return Err(RepositoryError::Dirty); + } + try!(self.repo.set_dirty()); + let mut usage = HashMap::new(); + for (id, bundle) in try!(self.repo.get_bundle_map()) { + usage.insert( + id, + BundleAnalysis { + chunk_usage: Bitmap::new(bundle.chunk_count), + info: bundle.clone(), + used_raw_size: 0 + } + ); + } + let backups = try!(self.get_all_backups()); + let mut todo = VecDeque::new(); + for (_name, backup) in backups { + todo.push_back(backup.root); + } + while let Some(chunks) = todo.pop_back() { + if !try!(self.mark_used(&mut usage, &chunks)) { + continue; + } + let inode = try!(self.get_inode(&chunks)); + // Mark the content chunks as used + match inode.data { + None | + Some(FileData::Inline(_)) => (), + Some(FileData::ChunkedDirect(chunks)) => { + try!(self.mark_used(&mut usage, &chunks)); + } + Some(FileData::ChunkedIndirect(chunks)) => { + if try!(self.mark_used(&mut usage, &chunks)) { + let chunk_data = try!(self.get_data(&chunks)); + let chunks = ChunkList::read_from(&chunk_data); + try!(self.mark_used(&mut usage, &chunks)); + } + } + } + // Put children in to do + if let Some(children) = inode.children { + for (_name, chunks) in children { + todo.push_back(chunks); + } + } + } + self.repo.set_clean(); + Ok(usage) + } + + pub fn vacuum( + &mut self, + ratio: f32, + combine: bool, + force: bool, + ) -> Result<(), RepositoryError> { + try!(self.repo.flush()); + tr_info!("Locking repository"); + try!(self.repo.write_mode()); + let _lock = try!(self.repo.lock(true)); + // analyze_usage will set the dirty flag + tr_info!("Analyzing chunk usage"); + let usage = try!(self.analyze_usage()); + let mut data_total = 0; + let mut data_used = 0; + for bundle in usage.values() { + data_total += bundle.info.encoded_size; + data_used += bundle.get_used_size(); + } + tr_info!( + "Usage: {} of {}, {:.1}%", + to_file_size(data_used as u64), + to_file_size(data_total as u64), + data_used as f32 / data_total as f32 * 100.0 + ); + let mut rewrite_bundles = HashSet::new(); + let mut reclaim_space = 0; + let mut rewrite_data = 0; + for (id, bundle) in &usage { + if bundle.get_usage_ratio() <= ratio { + rewrite_bundles.insert(*id); + reclaim_space += bundle.get_unused_size(); + rewrite_data += bundle.get_used_size(); + } + } + if combine { + let mut small_meta = vec![]; + let mut small_data = vec![]; + for (id, bundle) in &usage { + if bundle.info.encoded_size * 4 < self.repo.get_config().bundle_size { + match bundle.info.mode { + BundleMode::Meta => small_meta.push(*id), + BundleMode::Data => small_data.push(*id), + } + } + } + if small_meta.len() >= 2 { + for bundle in small_meta { + rewrite_bundles.insert(bundle); + } + } + if small_data.len() >= 2 { + for bundle in small_data { + rewrite_bundles.insert(bundle); + } + } + } + tr_info!( + "Reclaiming about {} by rewriting {} bundles ({})", + to_file_size(reclaim_space as u64), + rewrite_bundles.len(), + to_file_size(rewrite_data as u64) + ); + if !force { + self.repo.set_clean(); + return Ok(()); + } + let rewrite_bundles: Vec<_> = rewrite_bundles.into_iter().collect(); + try!(self.repo.rewrite_bundles(&rewrite_bundles, &usage)); + self.repo.set_clean(); + Ok(()) + } +} \ No newline at end of file diff --git a/src/cli/mod.rs b/src/cli/mod.rs index b5f6bbf..2152faa 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -123,7 +123,7 @@ fn open_repository(path: &Path, online: bool) -> Result Result { +fn get_backup(repo: &BackupRepository, backup_name: &str) -> Result { if !repo.has_backup(backup_name) { tr_error!("A backup with that name does not exist"); return Err(ErrorCode::NoSuchBackup); @@ -135,7 +135,7 @@ fn get_backup(repo: &BackupRepository, backup_name: &str) -> Result) -> Result { +fn get_inode(repo: &mut BackupRepository, backup: &BackupFile, inode: Option<&String>) -> Result { Ok(if let Some(inode) = inode { checked!( repo.get_backup_inode(backup, &inode), @@ -154,7 +154,7 @@ fn get_inode(repo: &mut BackupRepository, backup: &Backup, inode: Option<&String fn find_reference_backup( repo: &BackupRepository, path: &str, -) -> Result, ErrorCode> { +) -> Result, ErrorCode> { let mut matching = Vec::new(); let hostname = match get_hostname() { Ok(hostname) => hostname, @@ -181,7 +181,7 @@ fn find_reference_backup( Ok(matching.pop()) } -fn print_backup(backup: &Backup) { +fn print_backup(backup: &BackupFile) { if backup.modified { tr_warn!("This backup has been modified"); } @@ -299,7 +299,7 @@ fn print_inode(inode: &Inode) { } } -fn print_backups(backup_map: &HashMap) { +fn print_backups(backup_map: &HashMap) { let mut backups: Vec<_> = backup_map.into_iter().collect(); backups.sort_by_key(|b| b.0); for (name, backup) in backups { diff --git a/src/prelude.rs b/src/prelude.rs index 324fac2..142ee20 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -2,12 +2,14 @@ pub use util::*; pub use repository::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError, BundleDb, BundleWriterError, StoredBundle, BundleStatistics}; pub use repository::chunking::{ChunkerType, Chunker, ChunkerStatus, ChunkerError}; -pub use repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, - IntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, - FileData, DiffType, InodeError, RepositoryLayout, Location, +pub use repository::{Repository, Config, RepositoryError, RepositoryInfo, + IntegrityError, BundleAnalysis, + RepositoryLayout, Location, RepositoryStatistics, ChunkRepositoryLayout}; pub use repository::index::{Index, IndexError, IndexStatistics}; pub use backups::mount::FuseFilesystem; +pub use backups::{BackupFile, BackupFileError, Inode, FileType, FileData, InodeError, BackupError, + BackupOptions, DiffType, InodeIntegrityError}; pub use translation::CowStr; pub use backups::BackupRepository; diff --git a/src/repository/error.rs b/src/repository/error.rs index 2740e2d..640d3d0 100644 --- a/src/repository/error.rs +++ b/src/repository/error.rs @@ -3,11 +3,8 @@ use prelude::*; use std::io; use std::path::PathBuf; -use super::backup_file::BackupFileError; -use super::backup::BackupError; use super::bundle_map::BundleMapError; use super::config::ConfigError; -use super::metadata::InodeError; quick_error!{ @@ -72,6 +69,12 @@ quick_error!{ description(tr!("Bundle map error")) display("{}", tr_format!("Repository error: bundle map error\n\tcaused by: {}", err)) } + InodeIntegrity(err: InodeIntegrityError) { + from() + cause(err) + description(tr!("Integrity error")) + display("{}", tr_format!("Repository error: integrity error\n\tcaused by: {}", err)) + } Integrity(err: IntegrityError) { from() cause(err) @@ -101,7 +104,7 @@ quick_error!{ description(tr!("IO error")) display("{}", tr_format!("IO error: {}", err)) } - NoSuchFileInBackup(backup: Backup, path: PathBuf) { + NoSuchFileInBackup(backup: BackupFile, path: PathBuf) { description(tr!("No such file in backup")) display("{}", tr_format!("The backup does not contain the file {:?}", path)) } diff --git a/src/repository/info.rs b/src/repository/info.rs index 6b3e582..c247b23 100644 --- a/src/repository/info.rs +++ b/src/repository/info.rs @@ -1,6 +1,6 @@ use prelude::*; -use std::collections::{HashMap, VecDeque}; +use std::collections::HashMap; pub struct BundleAnalysis { @@ -47,91 +47,22 @@ pub struct RepositoryStatistics { impl Repository { - fn mark_used( - &self, - bundles: &mut HashMap, - chunks: &[Chunk], - ) -> Result { - let mut new = false; - for &(hash, len) in chunks { - if let Some(pos) = self.index.get(&hash) { - let bundle = pos.bundle; - if let Some(bundle) = bundles.get_mut(&bundle) { - if !bundle.chunk_usage.get(pos.chunk as usize) { - new = true; - bundle.chunk_usage.set(pos.chunk as usize); - bundle.used_raw_size += len as usize; - } - } else { - return Err(IntegrityError::MissingBundleId(pos.bundle).into()); - } - } else { - return Err(IntegrityError::MissingChunk(hash).into()); - } - } - Ok(new) - } - - pub fn analyze_usage(&mut self) -> Result, RepositoryError> { - if self.dirty { - return Err(RepositoryError::Dirty); - } - try!(self.set_dirty()); - let mut usage = HashMap::new(); - for (id, bundle) in self.bundle_map.bundles() { - let bundle = try!(self.bundles.get_bundle_info(&bundle).ok_or_else(|| { - IntegrityError::MissingBundle(bundle) - })); - usage.insert( - id, - BundleAnalysis { - chunk_usage: Bitmap::new(bundle.info.chunk_count), - info: bundle.info.clone(), - used_raw_size: 0 - } - ); - } - let backups = try!(self.get_all_backups()); - let mut todo = VecDeque::new(); - for (_name, backup) in backups { - todo.push_back(backup.root); - } - while let Some(chunks) = todo.pop_back() { - if !try!(self.mark_used(&mut usage, &chunks)) { - continue; - } - let inode = try!(self.get_inode(&chunks)); - // Mark the content chunks as used - match inode.data { - None | - Some(FileData::Inline(_)) => (), - Some(FileData::ChunkedDirect(chunks)) => { - try!(self.mark_used(&mut usage, &chunks)); - } - Some(FileData::ChunkedIndirect(chunks)) => { - if try!(self.mark_used(&mut usage, &chunks)) { - let chunk_data = try!(self.get_data(&chunks)); - let chunks = ChunkList::read_from(&chunk_data); - try!(self.mark_used(&mut usage, &chunks)); - } - } - } - // Put children in to do - if let Some(children) = inode.children { - for (_name, chunks) in children { - todo.push_back(chunks); - } - } - } - self.dirty = false; - Ok(usage) - } - #[inline] pub fn list_bundles(&self) -> Vec<&BundleInfo> { self.bundles.list_bundles() } + pub fn get_bundle_map(&self) -> Result, RepositoryError> { + let mut map = HashMap::with_capacity(self.bundle_map.len()); + for (id, bundle_id) in self.bundle_map.bundles() { + let info = try!(self.bundles.get_bundle_info(&bundle_id).ok_or_else(|| { + IntegrityError::MissingBundle(bundle_id) + })); + map.insert(id, &info.info); + } + Ok(map) + } + #[inline] pub fn get_bundle(&self, bundle: &BundleId) -> Option<&StoredBundle> { self.bundles.get_bundle_info(bundle) diff --git a/src/repository/integrity.rs b/src/repository/integrity.rs index 003d8d9..3632914 100644 --- a/src/repository/integrity.rs +++ b/src/repository/integrity.rs @@ -2,7 +2,6 @@ use prelude::*; use super::*; -use std::path::{Path, PathBuf}; use std::time::Duration; use pbr::ProgressBar; @@ -33,20 +32,35 @@ quick_error!{ MapContainsDuplicates { description(tr!("Map contains duplicates")) } - BrokenInode(path: PathBuf, err: Box) { - cause(err) - description(tr!("Broken inode")) - display("{}", tr_format!("Broken inode: {:?}\n\tcaused by: {}", path, err)) - } - MissingInodeData(path: PathBuf, err: Box) { - cause(err) - description(tr!("Missing inode data")) - display("{}", tr_format!("Missing inode data in: {:?}\n\tcaused by: {}", path, err)) - } } } + impl Repository { + pub fn get_chunk_marker(&self) -> Bitmap { + Bitmap::new(self.index.capacity()) + } + + pub fn check_chunks( + &self, + checked: &mut Bitmap, + chunks: &[Chunk], + mark: bool, + ) -> Result { + let mut new = false; + for &(hash, _len) in chunks { + if let Some(pos) = self.index.pos(&hash) { + new |= !checked.get(pos); + if mark { + checked.set(pos); + } + } else { + return Err(IntegrityError::MissingChunk(hash).into()); + } + } + Ok(new) + } + fn check_index_chunks(&self) -> Result<(), RepositoryError> { let mut progress = ProgressBar::new(self.index.len() as u64); progress.message(tr!("checking index: ")); @@ -76,354 +90,6 @@ impl Repository { Ok(()) } - fn check_chunks( - &self, - checked: &mut Bitmap, - chunks: &[Chunk], - mark: bool, - ) -> Result { - let mut new = false; - for &(hash, _len) in chunks { - if let Some(pos) = self.index.pos(&hash) { - new |= !checked.get(pos); - if mark { - checked.set(pos); - } - } else { - return Err(IntegrityError::MissingChunk(hash).into()); - } - } - Ok(new) - } - - fn check_inode_contents( - &mut self, - inode: &Inode, - checked: &mut Bitmap, - ) -> Result<(), RepositoryError> { - match inode.data { - None | - Some(FileData::Inline(_)) => (), - Some(FileData::ChunkedDirect(ref chunks)) => { - try!(self.check_chunks(checked, chunks, true)); - } - Some(FileData::ChunkedIndirect(ref chunks)) => { - if try!(self.check_chunks(checked, chunks, false)) { - let chunk_data = try!(self.get_data(chunks)); - let chunks2 = ChunkList::read_from(&chunk_data); - try!(self.check_chunks(checked, &chunks2, true)); - try!(self.check_chunks(checked, chunks, true)); - } - } - } - Ok(()) - } - - fn check_subtree( - &mut self, - path: PathBuf, - chunks: &[Chunk], - checked: &mut Bitmap, - repair: bool, - ) -> Result, RepositoryError> { - let mut modified = false; - match self.check_chunks(checked, chunks, false) { - Ok(false) => return Ok(None), - Ok(true) => (), - Err(err) => return Err(IntegrityError::BrokenInode(path, Box::new(err)).into()), - } - let mut inode = try!(self.get_inode(chunks)); - // Mark the content chunks as used - if let Err(err) = self.check_inode_contents(&inode, checked) { - if repair { - tr_warn!( - "Problem detected: data of {:?} is corrupt\n\tcaused by: {}", - path, - err - ); - tr_info!("Removing inode data"); - inode.data = Some(FileData::Inline(vec![].into())); - inode.size = 0; - modified = true; - } else { - return Err(IntegrityError::MissingInodeData(path, Box::new(err)).into()); - } - } - // Put children in to do - if let Some(ref mut children) = inode.children { - let mut removed = vec![]; - for (name, chunks) in children.iter_mut() { - match self.check_subtree(path.join(name), chunks, checked, repair) { - Ok(None) => (), - Ok(Some(c)) => { - *chunks = c; - modified = true; - } - Err(err) => { - if repair { - tr_warn!( - "Problem detected: inode {:?} is corrupt\n\tcaused by: {}", - path.join(name), - err - ); - tr_info!("Removing broken inode from backup"); - removed.push(name.to_string()); - modified = true; - } else { - return Err(err); - } - } - } - } - for name in removed { - children.remove(&name); - } - } - if modified { - Ok(Some(try!(self.put_inode(&inode)))) - } else { - try!(self.check_chunks(checked, chunks, true)); - Ok(None) - } - } - - fn evacuate_broken_backup(&self, name: &str) -> Result<(), RepositoryError> { - tr_warn!( - "The backup {} was corrupted and needed to be modified.", - name - ); - let src = self.layout.backup_path(name); - let mut dst = src.with_extension("backup.broken"); - let mut num = 1; - while dst.exists() { - dst = src.with_extension(&format!("backup.{}.broken", num)); - num += 1; - } - if fs::rename(&src, &dst).is_err() { - try!(fs::copy(&src, &dst)); - try!(fs::remove_file(&src)); - } - tr_info!("The original backup was renamed to {:?}", dst); - Ok(()) - } - - #[inline] - pub fn check_backup( - &mut self, - name: &str, - backup: &mut Backup, - repair: bool, - ) -> Result<(), RepositoryError> { - let _lock = if repair { - try!(self.write_mode()); - Some(self.lock(false)) - } else { - None - }; - tr_info!("Checking backup..."); - let mut checked = Bitmap::new(self.index.capacity()); - match self.check_subtree( - Path::new("").to_path_buf(), - &backup.root, - &mut checked, - repair - ) { - Ok(None) => (), - Ok(Some(chunks)) => { - try!(self.flush()); - backup.root = chunks; - backup.modified = true; - try!(self.evacuate_broken_backup(name)); - try!(self.save_backup(backup, name)); - } - Err(err) => { - if repair { - tr_warn!( - "The root of the backup {} has been corrupted\n\tcaused by: {}", - name, - err - ); - try!(self.evacuate_broken_backup(name)); - } else { - return Err(err); - } - } - } - Ok(()) - } - - pub fn check_backup_inode( - &mut self, - name: &str, - backup: &mut Backup, - path: &Path, - repair: bool, - ) -> Result<(), RepositoryError> { - let _lock = if repair { - try!(self.write_mode()); - Some(self.lock(false)) - } else { - None - }; - tr_info!("Checking inode..."); - let mut checked = Bitmap::new(self.index.capacity()); - let mut inodes = try!(self.get_backup_path(backup, path)); - let mut inode = inodes.pop().unwrap(); - let mut modified = false; - if let Err(err) = self.check_inode_contents(&inode, &mut checked) { - if repair { - tr_warn!( - "Problem detected: data of {:?} is corrupt\n\tcaused by: {}", - path, - err - ); - tr_info!("Removing inode data"); - inode.data = Some(FileData::Inline(vec![].into())); - inode.size = 0; - modified = true; - } else { - return Err( - IntegrityError::MissingInodeData(path.to_path_buf(), Box::new(err)).into() - ); - } - } - if let Some(ref mut children) = inode.children { - let mut removed = vec![]; - for (name, chunks) in children.iter_mut() { - match self.check_subtree(path.join(name), chunks, &mut checked, repair) { - Ok(None) => (), - Ok(Some(c)) => { - *chunks = c; - modified = true; - } - Err(err) => { - if repair { - tr_warn!( - "Problem detected: inode {:?} is corrupt\n\tcaused by: {}", - path.join(name), - err - ); - tr_info!("Removing broken inode from backup"); - removed.push(name.to_string()); - modified = true; - } else { - return Err(err); - } - } - } - } - for name in removed { - children.remove(&name); - } - } - let mut chunks = try!(self.put_inode(&inode)); - while let Some(mut parent) = inodes.pop() { - parent.children.as_mut().unwrap().insert(inode.name, chunks); - inode = parent; - chunks = try!(self.put_inode(&inode)); - } - if modified { - try!(self.flush()); - backup.root = chunks; - backup.modified = true; - try!(self.evacuate_broken_backup(name)); - try!(self.save_backup(backup, name)); - } - Ok(()) - } - - pub fn check_backups(&mut self, repair: bool) -> Result<(), RepositoryError> { - let _lock = if repair { - try!(self.write_mode()); - Some(self.lock(false)) - } else { - None - }; - tr_info!("Checking backups..."); - let mut checked = Bitmap::new(self.index.capacity()); - let backup_map = match self.get_all_backups() { - Ok(backup_map) => backup_map, - Err(RepositoryError::BackupFile(BackupFileError::PartialBackupsList(backup_map, - _failed))) => { - tr_warn!("Some backups could not be read, ignoring them"); - backup_map - } - Err(err) => return Err(err), - }; - for (name, mut backup) in - ProgressIter::new(tr!("checking backups"), backup_map.len(), backup_map.into_iter()) - { - let path = format!("{}::", name); - match self.check_subtree( - Path::new(&path).to_path_buf(), - &backup.root, - &mut checked, - repair - ) { - Ok(None) => (), - Ok(Some(chunks)) => { - try!(self.flush()); - backup.root = chunks; - backup.modified = true; - try!(self.evacuate_broken_backup(&name)); - try!(self.save_backup(&backup, &name)); - } - Err(err) => { - if repair { - tr_warn!( - "The root of the backup {} has been corrupted\n\tcaused by: {}", - name, - err - ); - try!(self.evacuate_broken_backup(&name)); - } else { - return Err(err); - } - } - } - } - Ok(()) - } - - pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> { - tr_info!("Checking repository integrity..."); - let mut rebuild = false; - for (_id, bundle_id) in self.bundle_map.bundles() { - if self.bundles.get_bundle_info(&bundle_id).is_none() { - if repair { - tr_warn!( - "Problem detected: bundle map contains unknown bundle {}", - bundle_id - ); - rebuild = true; - } else { - return Err(IntegrityError::MissingBundle(bundle_id).into()); - } - } - } - if self.bundle_map.len() < self.bundles.len() { - if repair { - tr_warn!("Problem detected: bundle map does not contain all remote bundles"); - rebuild = true; - } else { - return Err(IntegrityError::RemoteBundlesNotInMap.into()); - } - } - if self.bundle_map.len() > self.bundles.len() { - if repair { - tr_warn!("Problem detected: bundle map contains bundles multiple times"); - rebuild = true; - } else { - return Err(IntegrityError::MapContainsDuplicates.into()); - } - } - if rebuild { - try!(self.rebuild_bundle_map()); - try!(self.rebuild_index()); - } - Ok(()) - } - pub fn rebuild_bundle_map(&mut self) -> Result<(), RepositoryError> { tr_info!("Rebuilding bundle map from bundles"); self.bundle_map = BundleMap::create(); @@ -509,4 +175,44 @@ impl Repository { } Ok(()) } + + pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> { + tr_info!("Checking repository integrity..."); + let mut rebuild = false; + for (_id, bundle_id) in self.bundle_map.bundles() { + if self.bundles.get_bundle_info(&bundle_id).is_none() { + if repair { + tr_warn!( + "Problem detected: bundle map contains unknown bundle {}", + bundle_id + ); + rebuild = true; + } else { + return Err(IntegrityError::MissingBundle(bundle_id).into()); + } + } + } + if self.bundle_map.len() < self.bundles.len() { + if repair { + tr_warn!("Problem detected: bundle map does not contain all remote bundles"); + rebuild = true; + } else { + return Err(IntegrityError::RemoteBundlesNotInMap.into()); + } + } + if self.bundle_map.len() > self.bundles.len() { + if repair { + tr_warn!("Problem detected: bundle map contains bundles multiple times"); + rebuild = true; + } else { + return Err(IntegrityError::MapContainsDuplicates.into()); + } + } + if rebuild { + try!(self.rebuild_bundle_map()); + try!(self.rebuild_index()); + } + Ok(()) + } + } diff --git a/src/repository/layout.rs b/src/repository/layout.rs index a2d3692..489b7c6 100644 --- a/src/repository/layout.rs +++ b/src/repository/layout.rs @@ -23,7 +23,6 @@ pub trait ChunkRepositoryLayout { fn config_path(&self) -> PathBuf; - fn keys_path(&self) -> PathBuf; fn excludes_path(&self) -> PathBuf; fn backups_path(&self) -> PathBuf; fn backup_path(&self, name: &str) -> PathBuf; @@ -182,11 +181,6 @@ impl ChunkRepositoryLayout for RepositoryLayout { self.0.join("config.yaml") } - #[inline] - fn keys_path(&self) -> PathBuf { - self.0.join("keys") - } - #[inline] fn excludes_path(&self) -> PathBuf { self.0.join("excludes") diff --git a/src/repository/metadata.rs b/src/repository/metadata.rs index 8ea1956..e23de94 100644 --- a/src/repository/metadata.rs +++ b/src/repository/metadata.rs @@ -1,353 +1,8 @@ use prelude::*; -use filetime::{self, FileTime}; -use xattr; -use libc; - -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; -use std::fs::{self, File, Permissions}; -use std::os::linux::fs::MetadataExt; -use std::os::unix::fs::{FileTypeExt, PermissionsExt, MetadataExt as UnixMetadataExt, symlink}; -use std::io::{self, Read, Write}; -use std::os::unix::ffi::OsStrExt; -use std::fmt; -use std::ffi; - - -quick_error!{ - #[derive(Debug)] - pub enum InodeError { - UnsupportedFiletype(path: PathBuf) { - description(tr!("Unsupported file type")) - display("{}", tr_format!("Inode error: file {:?} has an unsupported type", path)) - } - ReadMetadata(err: io::Error, path: PathBuf) { - cause(err) - description(tr!("Failed to obtain metadata for file")) - display("{}", tr_format!("Inode error: failed to obtain metadata for file {:?}\n\tcaused by: {}", path, err)) - } - ReadXattr(err: io::Error, path: PathBuf) { - cause(err) - description(tr!("Failed to obtain xattr for file")) - display("{}", tr_format!("Inode error: failed to obtain xattr for file {:?}\n\tcaused by: {}", path, err)) - } - ReadLinkTarget(err: io::Error, path: PathBuf) { - cause(err) - description(tr!("Failed to obtain link target for file")) - display("{}", tr_format!("Inode error: failed to obtain link target for file {:?}\n\tcaused by: {}", path, err)) - } - Create(err: io::Error, path: PathBuf) { - cause(err) - description(tr!("Failed to create entity")) - display("{}", tr_format!("Inode error: failed to create entity {:?}\n\tcaused by: {}", path, err)) - } - Integrity(reason: &'static str) { - description(tr!("Integrity error")) - display("{}", tr_format!("Inode error: inode integrity error: {}", reason)) - } - Decode(err: msgpack::DecodeError) { - from() - cause(err) - description(tr!("Failed to decode metadata")) - display("{}", tr_format!("Inode error: failed to decode metadata\n\tcaused by: {}", err)) - } - Encode(err: msgpack::EncodeError) { - from() - cause(err) - description(tr!("Failed to encode metadata")) - display("{}", tr_format!("Inode error: failed to encode metadata\n\tcaused by: {}", err)) - } - } -} - - -#[derive(Debug, Eq, PartialEq, Clone, Copy, Hash)] -pub enum FileType { - File, - Directory, - Symlink, - BlockDevice, - CharDevice, - NamedPipe -} -serde_impl!(FileType(u8) { - File => 0, - Directory => 1, - Symlink => 2, - BlockDevice => 3, - CharDevice => 4, - NamedPipe => 5 -}); -impl fmt::Display for FileType { - fn fmt(&self, format: &mut fmt::Formatter) -> Result<(), fmt::Error> { - match *self { - FileType::File => write!(format, "{}", tr!("file")), - FileType::Directory => write!(format, "{}", tr!("directory")), - FileType::Symlink => write!(format, "{}", tr!("symlink")), - FileType::BlockDevice => write!(format, "{}", tr!("block device")), - FileType::CharDevice => write!(format, "{}", tr!("char device")), - FileType::NamedPipe => write!(format, "{}", tr!("named pipe")), - } - } -} - - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub enum FileData { - Inline(msgpack::Bytes), - ChunkedDirect(ChunkList), - ChunkedIndirect(ChunkList) -} -serde_impl!(FileData(u8) { - Inline(ByteBuf) => 0, - ChunkedDirect(ChunkList) => 1, - ChunkedIndirect(ChunkList) => 2 -}); - - -#[derive(Debug, Hash, Eq, PartialEq)] -pub struct Inode { - pub name: String, - pub size: u64, - pub file_type: FileType, - pub mode: u32, - pub user: u32, - pub group: u32, - pub timestamp: i64, - pub symlink_target: Option, - pub data: Option, - pub children: Option>, - pub cum_size: u64, - pub cum_dirs: usize, - pub cum_files: usize, - pub xattrs: BTreeMap, - pub device: Option<(u32, u32)> -} -impl Default for Inode { - fn default() -> Self { - Inode { - name: "".to_string(), - size: 0, - file_type: FileType::File, - mode: 0o644, - user: 1000, - group: 1000, - timestamp: 0, - symlink_target: None, - data: None, - children: None, - cum_size: 0, - cum_dirs: 0, - cum_files: 0, - xattrs: BTreeMap::new(), - device: None - } - } -} -serde_impl!(Inode(u8?) { - name: String => 0, - size: u64 => 1, - file_type: FileType => 2, - mode: u32 => 3, - user: u32 => 4, - group: u32 => 5, - timestamp: i64 => 7, - symlink_target: Option => 9, - data: Option => 10, - children: Option> => 11, - cum_size: u64 => 12, - cum_dirs: usize => 13, - cum_files: usize => 14, - xattrs: BTreeMap => 15, - device: Option<(u32, u32)> => 16 -}); - - -impl Inode { - pub fn get_from>(path: P) -> Result { - let path = path.as_ref(); - let name = path.file_name() - .map(|s| s.to_string_lossy().to_string()) - .unwrap_or_else(|| "_".to_string()); - let meta = try!(fs::symlink_metadata(path).map_err(|e| { - InodeError::ReadMetadata(e, path.to_owned()) - })); - let mut inode = Inode::default(); - inode.name = name; - if meta.is_file() { - inode.size = meta.len(); - } - inode.file_type = if meta.is_file() { - FileType::File - } else if meta.is_dir() { - FileType::Directory - } else if meta.file_type().is_symlink() { - FileType::Symlink - } else if meta.file_type().is_block_device() { - FileType::BlockDevice - } else if meta.file_type().is_char_device() { - FileType::CharDevice - } else if meta.file_type().is_fifo() { - FileType::NamedPipe - } else { - return Err(InodeError::UnsupportedFiletype(path.to_owned())); - }; - if meta.file_type().is_symlink() { - inode.symlink_target = Some( - try!(fs::read_link(path).map_err(|e| { - InodeError::ReadLinkTarget(e, path.to_owned()) - })).to_string_lossy() - .to_string() - ); - } - if meta.file_type().is_block_device() || meta.file_type().is_char_device() { - let rdev = meta.rdev(); - let major = (rdev >> 8) as u32; - let minor = (rdev & 0xff) as u32; - inode.device = Some((major, minor)); - } - inode.mode = meta.permissions().mode(); - inode.user = meta.st_uid(); - inode.group = meta.st_gid(); - inode.timestamp = meta.st_mtime(); - if xattr::SUPPORTED_PLATFORM { - if let Ok(attrs) = xattr::list(path) { - for name in attrs { - if let Some(data) = try!(xattr::get(path, &name).map_err(|e| { - InodeError::ReadXattr(e, path.to_owned()) - })) - { - inode.xattrs.insert( - name.to_string_lossy().to_string(), - data.into() - ); - } - } - } - } - Ok(inode) - } - - pub fn create_at>(&self, path: P) -> Result, InodeError> { - let full_path = path.as_ref().join(&self.name); - let mut file = None; - match self.file_type { - FileType::File => { - file = Some(try!(File::create(&full_path).map_err(|e| { - InodeError::Create(e, full_path.clone()) - }))); - } - FileType::Directory => { - try!(fs::create_dir(&full_path).map_err(|e| { - InodeError::Create(e, full_path.clone()) - })); - } - FileType::Symlink => { - if let Some(ref src) = self.symlink_target { - try!(symlink(src, &full_path).map_err(|e| { - InodeError::Create(e, full_path.clone()) - })); - } else { - return Err(InodeError::Integrity(tr!("Symlink without target"))); - } - } - FileType::NamedPipe => { - let name = try!( - ffi::CString::new(full_path.as_os_str().as_bytes()) - .map_err(|_| InodeError::Integrity(tr!("Name contains nulls"))) - ); - let mode = self.mode | libc::S_IFIFO; - if unsafe { libc::mkfifo(name.as_ptr(), mode) } != 0 { - return Err(InodeError::Create( - io::Error::last_os_error(), - full_path.clone() - )); - } - } - FileType::BlockDevice | FileType::CharDevice => { - let name = try!( - ffi::CString::new(full_path.as_os_str().as_bytes()) - .map_err(|_| InodeError::Integrity(tr!("Name contains nulls"))) - ); - let mode = self.mode | - match self.file_type { - FileType::BlockDevice => libc::S_IFBLK, - FileType::CharDevice => libc::S_IFCHR, - _ => unreachable!(), - }; - let device = if let Some((major, minor)) = self.device { - unsafe { libc::makedev(major, minor) } - } else { - return Err(InodeError::Integrity(tr!("Device without id"))); - }; - if unsafe { libc::mknod(name.as_ptr(), mode, device) } != 0 { - return Err(InodeError::Create( - io::Error::last_os_error(), - full_path.clone() - )); - } - } - } - let time = FileTime::from_seconds_since_1970(self.timestamp as u64, 0); - if let Err(err) = filetime::set_file_times(&full_path, time, time) { - tr_warn!("Failed to set file time on {:?}: {}", full_path, err); - } - if !self.xattrs.is_empty() { - if xattr::SUPPORTED_PLATFORM { - for (name, data) in &self.xattrs { - if let Err(err) = xattr::set(&full_path, name, data) { - tr_warn!("Failed to set xattr {} on {:?}: {}", name, full_path, err); - } - } - } else { - tr_warn!("Not setting xattr on {:?}", full_path); - } - } - if let Err(err) = fs::set_permissions(&full_path, Permissions::from_mode(self.mode)) { - tr_warn!( - "Failed to set permissions {:o} on {:?}: {}", - self.mode, - full_path, - err - ); - } - if let Err(err) = chown(&full_path, self.user, self.group) { - tr_warn!( - "Failed to set user {} and group {} on {:?}: {}", - self.user, - self.group, - full_path, - err - ); - } - Ok(file) - } - - #[inline] - pub fn is_same_meta(&self, other: &Inode) -> bool { - self.file_type == other.file_type && self.size == other.size && - self.mode == other.mode && self.user == other.user && - self.group == other.group && self.name == other.name && - self.timestamp == other.timestamp && self.symlink_target == other.symlink_target - } - - #[inline] - pub fn is_same_meta_quick(&self, other: &Inode) -> bool { - self.timestamp == other.timestamp && self.file_type == other.file_type && - self.size == other.size - } - - #[inline] - pub fn encode(&self) -> Result, InodeError> { - Ok(try!(msgpack::encode(&self))) - } - - #[inline] - pub fn decode(data: &[u8]) -> Result { - Ok(try!(msgpack::decode(data))) - } -} +use std::path::Path; +use std::fs::File; +use std::io::{Read, Write}; impl Repository { diff --git a/src/repository/mod.rs b/src/repository/mod.rs index 4eb6620..79b35e6 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -4,11 +4,8 @@ mod integrity; mod basic_io; mod info; mod metadata; -mod backup; mod error; mod vacuum; -mod backup_file; -mod tarfile; mod layout; pub mod bundledb; pub mod index; @@ -26,9 +23,6 @@ use std::io::Write; pub use self::error::RepositoryError; pub use self::config::Config; -pub use self::metadata::{Inode, FileType, FileData, InodeError}; -pub use self::backup::{BackupError, BackupOptions, DiffType}; -pub use self::backup_file::{Backup, BackupFileError}; pub use self::integrity::IntegrityError; pub use self::info::{RepositoryInfo, BundleAnalysis, RepositoryStatistics}; pub use self::layout::{RepositoryLayout, ChunkRepositoryLayout}; @@ -36,7 +30,6 @@ use self::bundle_map::BundleMap; const REPOSITORY_README: &[u8] = include_bytes!("../../docs/repository_readme.md"); -const DEFAULT_EXCLUDES: &[u8] = include_bytes!("../../docs/excludes.default"); const INDEX_MAGIC: [u8; 7] = *b"zvault\x02"; const INDEX_VERSION: u8 = 1; @@ -96,13 +89,9 @@ impl Repository { pub fn create>( layout: Arc, config: &Config, + crypto: Arc, remote: R, ) -> Result { - try!(fs::create_dir(layout.base_path())); - try!(File::create(layout.excludes_path()).and_then(|mut f| { - f.write_all(DEFAULT_EXCLUDES) - })); - try!(fs::create_dir(layout.keys_path())); try!(fs::create_dir(layout.local_locks_path())); try!(symlink(remote, layout.remote_path())); try!(File::create(layout.remote_readme_path()).and_then( @@ -120,11 +109,11 @@ impl Repository { )); try!(BundleMap::create().save(layout.bundle_map_path())); try!(fs::create_dir_all(layout.backups_path())); - Self::open(layout, true) + Self::open(layout, crypto, true) } #[allow(unknown_lints, useless_let_if_seq)] - pub fn open(layout: Arc, online: bool) -> Result { + pub fn open(layout: Arc, crypto: Arc, online: bool) -> Result { if !layout.remote_exists() { return Err(RepositoryError::NoRemote); } @@ -133,7 +122,6 @@ impl Repository { try!(fs::create_dir_all(layout.local_locks_path())); // Added after v0.1.0 let local_locks = LockFolder::new(layout.local_locks_path()); let lock = try!(local_locks.lock(false)); - let crypto = Arc::new(try!(Crypto::open(layout.keys_path()))); let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone(), online)); let (index, mut rebuild_index) = match unsafe { Index::open(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION) } { @@ -218,44 +206,6 @@ impl Repository { Ok(repo) } - pub fn import>( - layout: Arc, - remote: R, - key_files: Vec, - ) -> Result { - let mut repo = try!(Repository::create(layout.clone(), &Config::default(), remote)); - for file in key_files { - try!(repo.crypto.register_keyfile(file)); - } - repo = try!(Repository::open(layout, true)); - let mut backups: Vec<(String, Backup)> = try!(repo.get_all_backups()).into_iter().collect(); - backups.sort_by_key(|&(_, ref b)| b.timestamp); - if let Some((name, backup)) = backups.pop() { - tr_info!("Taking configuration from the last backup '{}'", name); - repo.config = backup.config; - try!(repo.save_config()) - } else { - tr_warn!( - "No backup found in the repository to take configuration from, please set the configuration manually." - ); - } - Ok(repo) - } - - #[inline] - pub fn register_key( - &mut self, - public: PublicKey, - secret: SecretKey, - ) -> Result<(), RepositoryError> { - try!(self.write_mode()); - try!(self.crypto.register_secret_key( - public, - secret - )); - Ok(()) - } - #[inline] pub fn save_config(&mut self) -> Result<(), RepositoryError> { try!(self.write_mode()); @@ -278,7 +228,7 @@ impl Repository { } #[inline] - fn save_bundle_map(&self) -> Result<(), RepositoryError> { + pub fn save_bundle_map(&self) -> Result<(), RepositoryError> { try!(self.bundle_map.save(self.layout.bundle_map_path())); Ok(()) } @@ -385,16 +335,24 @@ impl Repository { } #[inline] - fn write_mode(&mut self) -> Result<(), RepositoryError> { + pub fn write_mode(&mut self) -> Result<(), RepositoryError> { try!(self.local_locks.upgrade(&mut self.lock)); Ok(()) } #[inline] - fn lock(&self, exclusive: bool) -> Result { + pub fn lock(&self, exclusive: bool) -> Result { Ok(try!(self.remote_locks.lock(exclusive))) } + pub fn is_dirty(&self) -> bool { + self.dirty + } + + pub fn get_chunk_location(&self, chunk: Hash) -> Option { + self.index.get(&chunk) + } + #[inline] pub fn set_clean(&mut self) { self.dirty = false; diff --git a/src/repository/vacuum.rs b/src/repository/vacuum.rs index d11b667..ad64042 100644 --- a/src/repository/vacuum.rs +++ b/src/repository/vacuum.rs @@ -1,10 +1,10 @@ use prelude::*; -use std::collections::HashSet; +use std::collections::HashMap; impl Repository { - fn delete_bundle(&mut self, id: u32) -> Result<(), RepositoryError> { + pub fn delete_bundle(&mut self, id: u32) -> Result<(), RepositoryError> { if let Some(bundle) = self.bundle_map.remove(id) { try!(self.bundles.delete_bundle(&bundle)); Ok(()) @@ -13,92 +13,26 @@ impl Repository { } } - pub fn vacuum( - &mut self, - ratio: f32, - combine: bool, - force: bool, - ) -> Result<(), RepositoryError> { - try!(self.flush()); - tr_info!("Locking repository"); - try!(self.write_mode()); - let _lock = try!(self.lock(true)); - // analyze_usage will set the dirty flag - tr_info!("Analyzing chunk usage"); - let usage = try!(self.analyze_usage()); - let mut data_total = 0; - let mut data_used = 0; - for bundle in usage.values() { - data_total += bundle.info.encoded_size; - data_used += bundle.get_used_size(); - } - tr_info!( - "Usage: {} of {}, {:.1}%", - to_file_size(data_used as u64), - to_file_size(data_total as u64), - data_used as f32 / data_total as f32 * 100.0 - ); - let mut rewrite_bundles = HashSet::new(); - let mut reclaim_space = 0; - let mut rewrite_data = 0; - for (id, bundle) in &usage { - if bundle.get_usage_ratio() <= ratio { - rewrite_bundles.insert(*id); - reclaim_space += bundle.get_unused_size(); - rewrite_data += bundle.get_used_size(); - } - } - if combine { - let mut small_meta = vec![]; - let mut small_data = vec![]; - for (id, bundle) in &usage { - if bundle.info.encoded_size * 4 < self.config.bundle_size { - match bundle.info.mode { - BundleMode::Meta => small_meta.push(*id), - BundleMode::Data => small_data.push(*id), - } - } - } - if small_meta.len() >= 2 { - for bundle in small_meta { - rewrite_bundles.insert(bundle); - } - } - if small_data.len() >= 2 { - for bundle in small_data { - rewrite_bundles.insert(bundle); - } - } - } - tr_info!( - "Reclaiming about {} by rewriting {} bundles ({})", - to_file_size(reclaim_space as u64), - rewrite_bundles.len(), - to_file_size(rewrite_data as u64) - ); - if !force { - self.dirty = false; - return Ok(()); - } - for id in ProgressIter::new( + pub fn rewrite_bundles(&mut self, rewrite_bundles: &[u32], usage: &HashMap) -> Result<(), RepositoryError> { + for &id in ProgressIter::new( tr!("rewriting bundles"), rewrite_bundles.len(), rewrite_bundles.iter() ) - { - let bundle = &usage[id]; - let bundle_id = self.bundle_map.get(*id).unwrap(); - let chunks = try!(self.bundles.get_chunk_list(&bundle_id)); - let mode = usage[id].info.mode; - for (chunk, &(hash, _len)) in chunks.into_iter().enumerate() { - if !bundle.chunk_usage.get(chunk) { - try!(self.index.delete(&hash)); - continue; + { + let bundle = &usage[&id]; + let bundle_id = self.bundle_map.get(id).unwrap(); + let chunks = try!(self.bundles.get_chunk_list(&bundle_id)); + let mode = usage[&id].info.mode; + for (chunk, &(hash, _len)) in chunks.into_iter().enumerate() { + if !bundle.chunk_usage.get(chunk) { + try!(self.index.delete(&hash)); + continue; + } + let data = try!(self.bundles.get_chunk(&bundle_id, chunk)); + try!(self.put_chunk_override(mode, hash, &data)); } - let data = try!(self.bundles.get_chunk(&bundle_id, chunk)); - try!(self.put_chunk_override(mode, hash, &data)); } - } try!(self.flush()); tr_info!("Checking index"); for (hash, location) in self.index.iter() { @@ -114,11 +48,10 @@ impl Repository { } } tr_info!("Deleting {} bundles", rewrite_bundles.len()); - for id in rewrite_bundles { + for &id in rewrite_bundles { try!(self.delete_bundle(id)); } try!(self.save_bundle_map()); - self.dirty = false; Ok(()) } }