Partial backups

This commit is contained in:
Dennis Schwerdel 2017-03-20 22:24:53 +01:00 committed by Dennis Schwerdel
parent a8ff046c04
commit 657952d4d7
11 changed files with 101 additions and 19 deletions

1
Cargo.lock generated
View File

@ -7,6 +7,7 @@ dependencies = [
"byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",

View File

@ -21,6 +21,7 @@ log = "0.3"
byteorder = "1.0"
ansi_term = "0.9"
sodiumoxide = "*"
libc = "*"
[build-dependencies]
pkg-config = "0.3"

View File

@ -254,7 +254,7 @@ impl Bundle {
#[inline]
fn load_encoded_contents(&self) -> Result<Vec<u8>, BundleError> {
debug!("Load bundle data {}", self.info.id);
debug!("Load bundle data {} ({:?})", self.info.id, self.info.mode);
let mut file = BufReader::new(try!(File::open(&self.path).context(&self.path as &Path)));
try!(file.seek(SeekFrom::Start(self.content_start as u64)).context(&self.path as &Path));
let mut data = Vec::with_capacity(max(self.info.encoded_size, self.info.raw_size)+1024);

View File

@ -17,7 +17,8 @@ pub enum Arguments {
repo_path: String,
backup_name: String,
src_path: String,
full: bool
full: bool,
reference: Option<String>
},
Restore {
repo_path: String,
@ -204,6 +205,7 @@ pub fn parse() -> Arguments {
(@subcommand backup =>
(about: "creates a new backup")
(@arg full: --full "create a full backup")
(@arg reference: --ref +takes_value "the reference backup to use for partial backup")
(@arg SRC: +required "source path to backup")
(@arg BACKUP: +required "repository::backup path")
)
@ -313,7 +315,8 @@ pub fn parse() -> Arguments {
repo_path: repository.to_string(),
backup_name: backup.unwrap().to_string(),
full: args.is_present("full"),
src_path: args.value_of("SRC").unwrap().to_string()
src_path: args.value_of("SRC").unwrap().to_string(),
reference: args.value_of("reference").map(|v| v.to_string())
}
}
if let Some(args) = args.subcommand_matches("restore") {

View File

@ -30,6 +30,22 @@ fn get_backup(repo: &Repository, backup_name: &str) -> Backup {
}
}
fn find_reference_backup(repo: &Repository, path: &str) -> Option<Backup> {
let mut matching = Vec::new();
let hostname = match get_hostname() {
Ok(hostname) => hostname,
Err(_) => return None
};
for (_, backup) in repo.list_backups().unwrap() {
if backup.host == hostname && backup.path == path {
matching.push(backup);
}
}
matching.sort_by_key(|b| b.date);
matching.pop()
}
#[allow(unknown_lints,cyclomatic_complexity)]
pub fn run() {
if let Err(err) = logger::init() {
@ -54,12 +70,21 @@ pub fn run() {
repo.save_config().unwrap();
}
},
Arguments::Backup{repo_path, backup_name, src_path, full} => {
Arguments::Backup{repo_path, backup_name, src_path, full, reference} => {
let mut repo = open_repository(&repo_path);
let mut reference_backup = None;
if !full {
warn!("Partial backups are not implemented yet, creating full backup");
reference_backup = reference.map(|r| get_backup(&repo, &r));
if reference_backup.is_none() {
reference_backup = find_reference_backup(&repo, &src_path);
}
if let Some(ref backup) = reference_backup {
info!("Using backup from {} as reference", Local.timestamp(backup.date, 0).to_rfc2822());
} else {
info!("No reference backup found, doing a full scan instead");
}
}
let backup = repo.create_full_backup(&src_path).unwrap();
let backup = repo.create_backup(&src_path, reference_backup.as_ref()).unwrap();
repo.save_backup(&backup, &backup_name).unwrap();
},
Arguments::Restore{repo_path, backup_name, inode, dst_path} => {

View File

@ -14,6 +14,7 @@ extern crate chrono;
extern crate byteorder;
extern crate sodiumoxide;
extern crate ansi_term;
extern crate libc;
pub mod util;
@ -33,7 +34,6 @@ mod cli;
// TODO: Recompress & combine bundles
// TODO: Encrypt backup files too
// TODO: list --tree
// TODO: Partial backups via reference inode
// TODO: Import repository from remote folder
// TODO: Continue on errors

View File

@ -23,7 +23,9 @@ pub struct Backup {
pub date: i64,
pub duration: f32,
pub file_count: usize,
pub dir_count: usize
pub dir_count: usize,
pub host: String,
pub path: String
}
serde_impl!(Backup(u8) {
root: Vec<Chunk> => 0,
@ -37,7 +39,9 @@ serde_impl!(Backup(u8) {
date: i64 => 8,
duration: f32 => 9,
file_count: usize => 10,
dir_count: usize => 11
dir_count: usize => 11,
host: String => 12,
path: String => 13
});
@ -168,19 +172,28 @@ impl Repository {
}
#[allow(dead_code)]
pub fn create_full_backup<P: AsRef<Path>>(&mut self, path: P) -> Result<Backup, RepositoryError> {
let mut scan_stack = vec![path.as_ref().to_owned()];
pub fn create_backup<P: AsRef<Path>>(&mut self, path: P, reference: Option<&Backup>) -> Result<Backup, RepositoryError> {
let reference_inode = reference.and_then(|b| self.get_inode(&b.root).ok());
let mut scan_stack = vec![(path.as_ref().to_owned(), reference_inode)];
let mut save_stack = vec![];
let mut directories = HashMap::new();
let mut backup = Backup::default();
backup.host = get_hostname().unwrap_or_else(|_| "".to_string());
backup.path = path.as_ref().to_string_lossy().to_string();
let info_before = self.info();
let start = Local::now();
while let Some(path) = scan_stack.pop() {
while let Some((path, reference_inode)) = scan_stack.pop() {
// Create an inode for this path containing all attributes and contents
// (for files) but no children (for directories)
let mut inode = try!(self.create_inode(&path));
let mut inode = try!(self.create_inode(&path, reference_inode.as_ref()));
backup.total_data_size += inode.size;
backup.changed_data_size += inode.size;
if let Some(ref ref_inode) = reference_inode {
if !ref_inode.is_unchanged(&inode) {
backup.changed_data_size += inode.size;
}
} else {
backup.changed_data_size += inode.size;
}
if inode.file_type == FileType::Directory {
backup.dir_count +=1;
// For directories we need to put all children on the stack too, so there will be inodes created for them
@ -189,7 +202,13 @@ impl Repository {
inode.children = Some(HashMap::new());
directories.insert(path.clone(), inode);
for ch in try!(fs::read_dir(&path)) {
scan_stack.push(try!(ch).path());
let child = try!(ch);
let name = child.file_name().to_string_lossy().to_string();
let ref_child = reference_inode.as_ref()
.and_then(|inode| inode.children.as_ref())
.and_then(|map| map.get(&name))
.and_then(|chunks| self.get_inode(chunks).ok());
scan_stack.push((child.path(), ref_child));
}
} else {
backup.file_count +=1;

View File

@ -79,7 +79,7 @@ impl Repository {
let mut new = false;
for &(hash, _len) in chunks {
if let Some(pos) = self.index.pos(&hash) {
new |= checked.get(pos);
new |= !checked.get(pos);
checked.set(pos);
} else {
return Err(RepositoryIntegrityError::MissingChunk(hash).into())

View File

@ -24,7 +24,7 @@ serde_impl!(FileType(u8) {
});
#[derive(Debug)]
#[derive(Debug, Clone)]
pub enum FileContents {
Inline(msgpack::Bytes),
ChunkedDirect(ChunkList),
@ -85,6 +85,7 @@ serde_impl!(Inode(u8) {
children: HashMap<String, ChunkList> => 11
});
impl Inode {
#[inline]
fn get_extended_attrs_from(&mut self, meta: &Metadata) -> Result<(), RepositoryError> {
@ -145,13 +146,25 @@ impl Inode {
// https://crates.io/crates/filetime
Ok(file)
}
pub fn is_unchanged(&self, other: &Inode) -> bool {
self.modify_time == other.modify_time
&& self.create_time == other.create_time
&& self.file_type == other.file_type
}
}
impl Repository {
pub fn create_inode<P: AsRef<Path>>(&mut self, path: P) -> Result<Inode, RepositoryError> {
pub fn create_inode<P: AsRef<Path>>(&mut self, path: P, reference: Option<&Inode>) -> Result<Inode, RepositoryError> {
let mut inode = try!(Inode::get_from(path.as_ref()));
if inode.file_type == FileType::File && inode.size > 0 {
if let Some(reference) = reference {
if reference.is_unchanged(&inode) {
inode.contents = reference.contents.clone();
return Ok(inode)
}
}
let mut file = try!(File::open(path));
if inode.size < 100 {
let mut data = Vec::with_capacity(inode.size as usize);
@ -164,7 +177,7 @@ impl Repository {
} else {
let mut chunk_data = Vec::with_capacity(chunks.encoded_size());
chunks.write_to(&mut chunk_data).unwrap();
chunks = try!(self.put_data(BundleMode::Content, &chunk_data));
chunks = try!(self.put_data(BundleMode::Meta, &chunk_data));
inode.contents = Some(FileContents::ChunkedIndirect(chunks));
}
}

18
src/util/hostname.rs Normal file
View File

@ -0,0 +1,18 @@
use libc;
use std::ffi;
extern {
fn gethostname(name: *mut libc::c_char, size: libc::size_t) -> libc::c_int;
}
pub fn get_hostname() -> Result<String, ()> {
let mut buf = Vec::with_capacity(255);
buf.resize(255, 0u8);
if unsafe { gethostname(buf.as_mut_ptr() as *mut libc::c_char, buf.len() as libc::size_t) } == 0 {
buf[254] = 0; //enforce null-termination
let name = unsafe { ffi::CStr::from_ptr(buf.as_ptr() as *const libc::c_char) };
name.to_str().map(|s| s.to_string()).map_err(|_| ())
} else {
Err(())
}
}

View File

@ -7,6 +7,7 @@ mod chunk;
mod bitmap;
mod hex;
mod cli;
mod hostname;
pub mod msgpack;
pub use self::chunk::*;
@ -17,3 +18,4 @@ pub use self::lru_cache::*;
pub use self::bitmap::*;
pub use self::hex::*;
pub use self::cli::*;
pub use self::hostname::*;