dswd
/
zvault
mirror of https://github.com/dswd/zvault
1
0
Fork 0
Code Issues Releases Wiki Activity

Partial backups

This commit is contained in:
Dennis Schwerdel 2017-03-20 22:24:53 +01:00 committed by Dennis Schwerdel
parent a8ff046c04
commit 657952d4d7
11 changed files with 101 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -7,6 +7,7 @@ dependencies = [
"byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",

1
Cargo.toml
View File

@ -21,6 +21,7 @@ log = "0.3"
byteorder = "1.0"
ansi_term = "0.9"
sodiumoxide = "*"
libc = "*"
[build-dependencies]
pkg-config = "0.3"

2
src/bundle.rs
View File

@ -254,7 +254,7 @@ impl Bundle {
#[inline]
fn load_encoded_contents(&self) -> Result<Vec<u8>, BundleError> {
debug!("Load bundle data {}", self.info.id);
debug!("Load bundle data {} ({:?})", self.info.id, self.info.mode);
let mut file = BufReader::new(try!(File::open(&self.path).context(&self.path as &Path)));
try!(file.seek(SeekFrom::Start(self.content_start as u64)).context(&self.path as &Path));
let mut data = Vec::with_capacity(max(self.info.encoded_size, self.info.raw_size)+1024);

7
src/cli/args.rs
View File

@ -17,7 +17,8 @@ pub enum Arguments {
repo_path: String,
backup_name: String,
src_path: String,
full: bool
full: bool,
reference: Option<String>
},
Restore {
repo_path: String,
@ -204,6 +205,7 @@ pub fn parse() -> Arguments {
(@subcommand backup =>
(about: "creates a new backup")
(@arg full: --full "create a full backup")
(@arg reference: --ref +takes_value "the reference backup to use for partial backup")
(@arg SRC: +required "source path to backup")
(@arg BACKUP: +required "repository::backup path")
)
@ -313,7 +315,8 @@ pub fn parse() -> Arguments {
repo_path: repository.to_string(),
backup_name: backup.unwrap().to_string(),
full: args.is_present("full"),
src_path: args.value_of("SRC").unwrap().to_string()
src_path: args.value_of("SRC").unwrap().to_string(),
reference: args.value_of("reference").map(|v| v.to_string())
}
}
if let Some(args) = args.subcommand_matches("restore") {

31
src/cli/mod.rs
View File

@ -30,6 +30,22 @@ fn get_backup(repo: &Repository, backup_name: &str) -> Backup {
}
}
fn find_reference_backup(repo: &Repository, path: &str) -> Option<Backup> {
let mut matching = Vec::new();
let hostname = match get_hostname() {
Ok(hostname) => hostname,
Err(_) => return None
};
for (_, backup) in repo.list_backups().unwrap() {
if backup.host == hostname && backup.path == path {
matching.push(backup);
}
}
matching.sort_by_key(|b| b.date);
matching.pop()
}
#[allow(unknown_lints,cyclomatic_complexity)]
pub fn run() {
if let Err(err) = logger::init() {
@ -54,12 +70,21 @@ pub fn run() {
repo.save_config().unwrap();
}
},
Arguments::Backup{repo_path, backup_name, src_path, full} => {
Arguments::Backup{repo_path, backup_name, src_path, full, reference} => {
let mut repo = open_repository(&repo_path);
let mut reference_backup = None;
if !full {
warn!("Partial backups are not implemented yet, creating full backup");
reference_backup = reference.map(|r| get_backup(&repo, &r));
if reference_backup.is_none() {
reference_backup = find_reference_backup(&repo, &src_path);
}
if let Some(ref backup) = reference_backup {
info!("Using backup from {} as reference", Local.timestamp(backup.date, 0).to_rfc2822());
} else {
info!("No reference backup found, doing a full scan instead");
}
}
let backup = repo.create_full_backup(&src_path).unwrap();
let backup = repo.create_backup(&src_path, reference_backup.as_ref()).unwrap();
repo.save_backup(&backup, &backup_name).unwrap();
},
Arguments::Restore{repo_path, backup_name, inode, dst_path} => {

2
src/main.rs
View File

@ -14,6 +14,7 @@ extern crate chrono;
extern crate byteorder;
extern crate sodiumoxide;
extern crate ansi_term;
extern crate libc;
pub mod util;
@ -33,7 +34,6 @@ mod cli;
// TODO: Recompress & combine bundles
// TODO: Encrypt backup files too
// TODO: list --tree
// TODO: Partial backups via reference inode
// TODO: Import repository from remote folder
// TODO: Continue on errors

35
src/repository/backup.rs
View File

@ -23,7 +23,9 @@ pub struct Backup {
pub date: i64,
pub duration: f32,
pub file_count: usize,
pub dir_count: usize
pub dir_count: usize,
pub host: String,
pub path: String
}
serde_impl!(Backup(u8) {
root: Vec<Chunk> => 0,
@ -37,7 +39,9 @@ serde_impl!(Backup(u8) {
date: i64 => 8,
duration: f32 => 9,
file_count: usize => 10,
dir_count: usize => 11
dir_count: usize => 11,
host: String => 12,
path: String => 13
});
@ -168,19 +172,28 @@ impl Repository {
}
#[allow(dead_code)]
pub fn create_full_backup<P: AsRef<Path>>(&mut self, path: P) -> Result<Backup, RepositoryError> {
let mut scan_stack = vec![path.as_ref().to_owned()];
pub fn create_backup<P: AsRef<Path>>(&mut self, path: P, reference: Option<&Backup>) -> Result<Backup, RepositoryError> {
let reference_inode = reference.and_then(|b| self.get_inode(&b.root).ok());
let mut scan_stack = vec![(path.as_ref().to_owned(), reference_inode)];
let mut save_stack = vec![];
let mut directories = HashMap::new();
let mut backup = Backup::default();
backup.host = get_hostname().unwrap_or_else(|_| "".to_string());
backup.path = path.as_ref().to_string_lossy().to_string();
let info_before = self.info();
let start = Local::now();
while let Some(path) = scan_stack.pop() {
while let Some((path, reference_inode)) = scan_stack.pop() {
// Create an inode for this path containing all attributes and contents
// (for files) but no children (for directories)
let mut inode = try!(self.create_inode(&path));
let mut inode = try!(self.create_inode(&path, reference_inode.as_ref()));
backup.total_data_size += inode.size;
backup.changed_data_size += inode.size;
if let Some(ref ref_inode) = reference_inode {
if !ref_inode.is_unchanged(&inode) {
backup.changed_data_size += inode.size;
}
} else {
backup.changed_data_size += inode.size;
}
if inode.file_type == FileType::Directory {
backup.dir_count +=1;
// For directories we need to put all children on the stack too, so there will be inodes created for them
@ -189,7 +202,13 @@ impl Repository {
inode.children = Some(HashMap::new());
directories.insert(path.clone(), inode);
for ch in try!(fs::read_dir(&path)) {
scan_stack.push(try!(ch).path());
let child = try!(ch);
let name = child.file_name().to_string_lossy().to_string();
let ref_child = reference_inode.as_ref()
.and_then(|inode| inode.children.as_ref())
.and_then(|map| map.get(&name))
.and_then(|chunks| self.get_inode(chunks).ok());
scan_stack.push((child.path(), ref_child));
}
} else {
backup.file_count +=1;

2
src/repository/integrity.rs
View File

@ -79,7 +79,7 @@ impl Repository {
let mut new = false;
for &(hash, _len) in chunks {
if let Some(pos) = self.index.pos(&hash) {
new |= checked.get(pos);
new |= !checked.get(pos);
checked.set(pos);
} else {
return Err(RepositoryIntegrityError::MissingChunk(hash).into())

19
src/repository/metadata.rs
View File

@ -24,7 +24,7 @@ serde_impl!(FileType(u8) {
});
#[derive(Debug)]
#[derive(Debug, Clone)]
pub enum FileContents {
Inline(msgpack::Bytes),
ChunkedDirect(ChunkList),
@ -85,6 +85,7 @@ serde_impl!(Inode(u8) {
children: HashMap<String, ChunkList> => 11
});
impl Inode {
#[inline]
fn get_extended_attrs_from(&mut self, meta: &Metadata) -> Result<(), RepositoryError> {
@ -145,13 +146,25 @@ impl Inode {
// https://crates.io/crates/filetime
Ok(file)
}
pub fn is_unchanged(&self, other: &Inode) -> bool {
self.modify_time == other.modify_time
&& self.create_time == other.create_time
&& self.file_type == other.file_type
}
}
impl Repository {
pub fn create_inode<P: AsRef<Path>>(&mut self, path: P) -> Result<Inode, RepositoryError> {
pub fn create_inode<P: AsRef<Path>>(&mut self, path: P, reference: Option<&Inode>) -> Result<Inode, RepositoryError> {
let mut inode = try!(Inode::get_from(path.as_ref()));
if inode.file_type == FileType::File && inode.size > 0 {
if let Some(reference) = reference {
if reference.is_unchanged(&inode) {
inode.contents = reference.contents.clone();
return Ok(inode)
}
}
let mut file = try!(File::open(path));
if inode.size < 100 {
let mut data = Vec::with_capacity(inode.size as usize);
@ -164,7 +177,7 @@ impl Repository {
} else {
let mut chunk_data = Vec::with_capacity(chunks.encoded_size());
chunks.write_to(&mut chunk_data).unwrap();
chunks = try!(self.put_data(BundleMode::Content, &chunk_data));
chunks = try!(self.put_data(BundleMode::Meta, &chunk_data));
inode.contents = Some(FileContents::ChunkedIndirect(chunks));
}
}

18
src/util/hostname.rs Normal file
View File

@ -0,0 +1,18 @@
use libc;
use std::ffi;
extern {
fn gethostname(name: *mut libc::c_char, size: libc::size_t) -> libc::c_int;
}
pub fn get_hostname() -> Result<String, ()> {
let mut buf = Vec::with_capacity(255);
buf.resize(255, 0u8);
if unsafe { gethostname(buf.as_mut_ptr() as *mut libc::c_char, buf.len() as libc::size_t) } == 0 {
buf[254] = 0; //enforce null-termination
let name = unsafe { ffi::CStr::from_ptr(buf.as_ptr() as *const libc::c_char) };
name.to_str().map(|s| s.to_string()).map_err(|_| ())
} else {
Err(())
}
}

2
src/util/mod.rs
View File

@ -7,6 +7,7 @@ mod chunk;
mod bitmap;
mod hex;
mod cli;
mod hostname;
pub mod msgpack;
pub use self::chunk::*;
@ -17,3 +18,4 @@ pub use self::lru_cache::*;
pub use self::bitmap::*;
pub use self::hex::*;
pub use self::cli::*;
pub use self::hostname::*;