dswd
/
zvault
mirror of https://github.com/dswd/zvault
1
0
Fork 0
Code Issues Releases Wiki Activity

Some improvements

This commit is contained in:
Dennis Schwerdel 2017-03-24 12:52:01 +01:00
parent 226107c112
commit d80c8ffb69
9 changed files with 90 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -98,7 +98,6 @@ Recommended: Brotli/2-7
## TODO
### Core functionality
- Options for creating backups (same filesystem, exclude/include patterns)
- Recompress & combine bundles
- Allow to use tar files for backup and restore (--tar, http://alexcrichton.com/tar-rs/tar/index.html)
- File attributes

6
src/bundledb/reader.rs
View File

@ -161,7 +161,11 @@ impl BundleReader {
data = try!(self.crypto.lock().unwrap().decrypt(&encryption, &data).context(&self.path as &Path));
}
if let Some(ref compression) = self.info.compression {
data = try!(compression.decompress(&data).context(&self.path as &Path));
let mut stream = try!(compression.decompress_stream().context(&self.path as &Path));
let mut buffer = Vec::with_capacity(self.info.raw_size);
try!(stream.process(&data, &mut buffer).context(&self.path as &Path));
try!(stream.finish(&mut buffer).context(&self.path as &Path));
data = buffer;
}
Ok(data)
}

51
src/cli/args.rs
View File

@ -65,9 +65,13 @@ pub enum Arguments {
backup_name: Option<String>,
inode: Option<String>
},
ListBundles {
BundleList {
repo_path: String
},
BundleInfo {
repo_path: String,
bundle_id: BundleId
},
Import {
repo_path: String,
remote_path: String,
@ -117,16 +121,6 @@ fn parse_num(num: &str, name: &str) -> u64 {
}
}
fn parse_float(num: &str, name: &str) -> f64 {
if let Ok(num) = num.parse::<f64>() {
num
} else {
error!("{} must be a floating-point number, was '{}'", name, num);
exit(1);
}
}
fn parse_chunker(val: &str) -> ChunkerType {
if let Ok(chunker) = ChunkerType::from_string(val) {
chunker
@ -173,6 +167,15 @@ fn parse_hash(val: &str) -> HashMethod {
}
}
fn parse_bundle_id(val: &str) -> BundleId {
if let Ok(hash) = Hash::from_string(val) {
BundleId(hash)
} else {
error!("Invalid bundle id: {}", val);
exit(1);
}
}
pub fn parse() -> Arguments {
let args = clap_app!(zvault =>
@ -224,7 +227,7 @@ pub fn parse() -> Arguments {
)
(@subcommand vacuum =>
(about: "saves space by combining and recompressing bundles")
(@arg ratio: --ratio -r +takes_value "ratio of unused chunks in a bundle to rewrite that bundle")
(@arg ratio: --ratio -r +takes_value "ratio in % of unused space in a bundle to rewrite that bundle")
(@arg force: --force -f "actually run the vacuum instead of simulating it")
(@arg REPO: +required "path of the repository")
)
@ -237,10 +240,15 @@ pub fn parse() -> Arguments {
(about: "lists backups or backup contents")
(@arg PATH: +required "repository[::backup[::subpath]] path")
)
(@subcommand listbundles =>
(@subcommand bundlelist =>
(about: "lists bundles in a repository")
(@arg REPO: +required "path of the repository")
)
(@subcommand bundleinfo =>
(about: "lists bundles in a repository")
(@arg REPO: +required "path of the repository")
(@arg BUNDLE: +required "the bundle id")
)
(@subcommand import =>
(about: "reconstruct a repository from the remote files")
(@arg key: --key -k ... +takes_value "a file with a needed to read the bundles")
@ -368,7 +376,7 @@ pub fn parse() -> Arguments {
return Arguments::Vacuum {
repo_path: repository.to_string(),
force: args.is_present("force"),
ratio: parse_float(args.value_of("ratio").unwrap_or(&DEFAULT_VACUUM_RATIO.to_string()), "ratio") as f32
ratio: parse_num(args.value_of("ratio").unwrap_or(&DEFAULT_VACUUM_RATIO.to_string()), "ratio") as f32 / 100.0
}
}
if let Some(args) = args.subcommand_matches("check") {
@ -388,16 +396,27 @@ pub fn parse() -> Arguments {
inode: inode.map(|v| v.to_string())
}
}
if let Some(args) = args.subcommand_matches("listbundles") {
if let Some(args) = args.subcommand_matches("bundlelist") {
let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap());
if backup.is_some() || inode.is_some() {
println!("No backups or subpaths may be given here");
exit(1);
}
return Arguments::ListBundles {
return Arguments::BundleList {
repo_path: repository.to_string(),
}
}
if let Some(args) = args.subcommand_matches("bundleinfo") {
let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap());
if backup.is_some() || inode.is_some() {
println!("No backups or subpaths may be given here");
exit(1);
}
return Arguments::BundleInfo {
repo_path: repository.to_string(),
bundle_id: parse_bundle_id(args.value_of("BUNDLE").unwrap())
}
}
if let Some(args) = args.subcommand_matches("info") {
let (repository, backup, inode) = split_repo_path(args.value_of("PATH").unwrap());
return Arguments::Info {

38
src/cli/mod.rs
View File

@ -20,7 +20,7 @@ pub const DEFAULT_CHUNKER: &'static str = "fastcdc/16";
pub const DEFAULT_HASH: &'static str = "blake2";
pub const DEFAULT_COMPRESSION: &'static str = "brotli/3";
pub const DEFAULT_BUNDLE_SIZE: usize = 25;
pub const DEFAULT_VACUUM_RATIO: f32 = 0.5;
pub const DEFAULT_VACUUM_RATIO: usize = 50;
fn checked<T, E: Display>(result: Result<T, E>, msg: &'static str) -> T {
@ -80,6 +80,14 @@ fn print_backup(backup: &Backup) {
println!("Chunk count: {}, avg size: {}", backup.chunk_count, to_file_size(backup.avg_chunk_size as u64));
}
pub fn format_inode_one_line(inode: &Inode) -> String {
match inode.file_type {
FileType::Directory => format!("{:25}\t{} entries", format!("{}/", inode.name), inode.children.as_ref().unwrap().len()),
FileType::File => format!("{:25}\t{}", inode.name, to_file_size(inode.size)),
FileType::Symlink => format!("{:25}\t -> {}", inode.name, inode.symlink_target.as_ref().unwrap()),
}
}
fn print_inode(inode: &Inode) {
println!("Name: {}", inode.name);
println!("Type: {}", inode.file_type);
@ -123,6 +131,12 @@ fn print_bundle(bundle: &BundleInfo) {
println!("Bundle {}", bundle.id);
println!(" - Mode: {:?}", bundle.mode);
println!(" - Hash method: {:?}", bundle.hash_method);
let encryption = if let Some((_, ref key)) = bundle.encryption {
to_hex(key)
} else {
"none".to_string()
};
println!(" - Encryption: {}", encryption);
println!(" - Chunks: {}", bundle.chunk_count);
println!(" - Size: {}", to_file_size(bundle.encoded_size as u64));
println!(" - Data size: {}", to_file_size(bundle.raw_size as u64));
@ -135,6 +149,10 @@ fn print_bundle(bundle: &BundleInfo) {
println!(" - Compression: {}, ratio: {:.1}%", compression, ratio * 100.0);
}
fn print_bundle_one_line(bundle: &BundleInfo) {
println!("{}: {:8?}, {:5} chunks, {:8}", bundle.id, bundle.mode, bundle.chunk_count, to_file_size(bundle.encoded_size as u64))
}
fn print_config(config: &Config) {
println!("Bundle size: {}", to_file_size(config.bundle_size as u64));
println!("Chunker: {}", config.chunker.to_string());
@ -198,7 +216,7 @@ pub fn run() {
}
}
let excludes: Vec<String> = excludes.into_iter().map(|mut exclude| {
exclude = regex::escape(&exclude).replace('?', ".").replace(r"\*\", ".*").replace(r"\*", "[^/]*");
exclude = regex::escape(&exclude).replace('?', ".").replace(r"\*\*", ".*").replace(r"\*", "[^/]*");
if exclude.starts_with('/') {
format!(r"^{}($|/)", exclude)
} else {
@ -263,9 +281,13 @@ pub fn run() {
},
Arguments::Vacuum{repo_path, ratio, force} => {
let mut repo = open_repository(&repo_path);
let info_before = repo.info();
checked(repo.vacuum(ratio, force), "vacuum");
if !force {
info!("Run with --force to actually execute this command");
} else {
let info_after = repo.info();
info!("Reclaimed {}", to_file_size(info_before.encoded_data_size - info_after.encoded_data_size));
}
},
Arguments::Check{repo_path, backup_name, inode, full} => {
@ -324,11 +346,19 @@ pub fn run() {
print_repoinfo(&repo.info());
}
},
Arguments::ListBundles{repo_path} => {
Arguments::BundleList{repo_path} => {
let repo = open_repository(&repo_path);
for bundle in repo.list_bundles() {
print_bundle_one_line(bundle);
}
},
Arguments::BundleInfo{repo_path, bundle_id} => {
let repo = open_repository(&repo_path);
if let Some(bundle) = repo.get_bundle(&bundle_id) {
print_bundle(bundle);
println!();
} else {
error!("No such bundle");
exit(3);
}
},
Arguments::Import{repo_path, remote_path, key_files} => {

2
src/repository/backup.rs
View File

@ -51,7 +51,7 @@ impl Repository {
try!(fs::remove_file(&path));
loop {
path = path.parent().unwrap().to_owned();
if fs::remove_dir(&path).is_err() {
if path == self.backups_path || fs::remove_dir(&path).is_err() {
break
}
}

5
src/repository/info.rs
View File

@ -20,6 +20,11 @@ impl Repository {
self.bundles.list_bundles()
}
#[inline]
pub fn get_bundle(&self, bundle: &BundleId) -> Option<&BundleInfo> {
self.bundles.get_bundle_info(bundle)
}
pub fn info(&self) -> RepositoryInfo {
let bundles = self.list_bundles();
let encoded_data_size = bundles.iter().map(|b| b.encoded_size as u64).sum();

6
src/repository/vacuum.rs
View File

@ -114,13 +114,13 @@ impl Repository {
for id in &rewrite_bundles {
let bundle = &usage[id];
let bundle_id = self.bundle_map.get(*id).unwrap();
for chunk in 0..bundle.chunk_count {
let data = try!(self.bundles.get_chunk(&bundle_id, chunk));
let hash = self.config.hash.hash(&data);
let chunks = try!(self.bundles.get_chunk_list(&bundle_id));
for (chunk, &(hash, _len)) in chunks.into_iter().enumerate() {
if !bundle.used.get(chunk) {
try!(self.index.delete(&hash));
continue
}
let data = try!(self.bundles.get_chunk(&bundle_id, chunk));
let mode = if bundle.mode.get(chunk) {
BundleMode::Meta
} else {

10
src/util/cli.rs
View File

@ -1,5 +1,3 @@
use ::repository::{Inode, FileType};
pub fn to_file_size(size: u64) -> String {
let mut size = size as f32;
if size >= 512.0 {
@ -38,11 +36,3 @@ pub fn to_duration(dur: f32) -> String {
let secs = (secs % 60) as f32 + subsecs;
format!("{}:{:02}:{:04.1}", hours, mins, secs)
}
pub fn format_inode_one_line(inode: &Inode) -> String {
match inode.file_type {
FileType::Directory => format!("{:25}\t{} entries", format!("{}/", inode.name), inode.children.as_ref().unwrap().len()),
FileType::File => format!("{:25}\t{}", inode.name, to_file_size(inode.size)),
FileType::Symlink => format!("{:25}\t -> {}", inode.name, inode.symlink_target.as_ref().unwrap()),
}
}

8
src/util/hash.rs
View File

@ -12,7 +12,6 @@ use std::u64;
use std::io::{self, Read, Write};
#[repr(packed)]
#[derive(Clone, Copy, PartialEq, Hash, Eq, Default)]
pub struct Hash {
@ -48,6 +47,13 @@ impl Hash {
let low = try!(src.read_u64::<LittleEndian>());
Ok(Hash { high: high, low: low })
}
#[inline]
pub fn from_string(val: &str) -> Result<Self, ()> {
let high = try!(u64::from_str_radix(&val[..16], 16).map_err(|_| ()));
let low = try!(u64::from_str_radix(&val[16..], 16).map_err(|_| ()));
Ok(Self { high: high, low: low })
}
}
impl fmt::Display for Hash {