From d80c8ffb6947eddb4ef005cb57e6cca513ff5e56 Mon Sep 17 00:00:00 2001 From: Dennis Schwerdel Date: Fri, 24 Mar 2017 12:52:01 +0100 Subject: [PATCH] Some improvements --- README.md | 1 - src/bundledb/reader.rs | 6 ++++- src/cli/args.rs | 51 +++++++++++++++++++++++++++------------- src/cli/mod.rs | 38 ++++++++++++++++++++++++++---- src/repository/backup.rs | 2 +- src/repository/info.rs | 5 ++++ src/repository/vacuum.rs | 6 ++--- src/util/cli.rs | 10 -------- src/util/hash.rs | 8 ++++++- 9 files changed, 90 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index dcfc1ef..8d3a494 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,6 @@ Recommended: Brotli/2-7 ## TODO ### Core functionality -- Options for creating backups (same filesystem, exclude/include patterns) - Recompress & combine bundles - Allow to use tar files for backup and restore (--tar, http://alexcrichton.com/tar-rs/tar/index.html) - File attributes diff --git a/src/bundledb/reader.rs b/src/bundledb/reader.rs index e6fa629..de2162d 100644 --- a/src/bundledb/reader.rs +++ b/src/bundledb/reader.rs @@ -161,7 +161,11 @@ impl BundleReader { data = try!(self.crypto.lock().unwrap().decrypt(&encryption, &data).context(&self.path as &Path)); } if let Some(ref compression) = self.info.compression { - data = try!(compression.decompress(&data).context(&self.path as &Path)); + let mut stream = try!(compression.decompress_stream().context(&self.path as &Path)); + let mut buffer = Vec::with_capacity(self.info.raw_size); + try!(stream.process(&data, &mut buffer).context(&self.path as &Path)); + try!(stream.finish(&mut buffer).context(&self.path as &Path)); + data = buffer; } Ok(data) } diff --git a/src/cli/args.rs b/src/cli/args.rs index 34fe74d..5f02a58 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -65,9 +65,13 @@ pub enum Arguments { backup_name: Option, inode: Option }, - ListBundles { + BundleList { repo_path: String }, + BundleInfo { + repo_path: String, + bundle_id: BundleId + }, Import { repo_path: String, remote_path: String, @@ -117,16 +121,6 @@ fn parse_num(num: &str, name: &str) -> u64 { } } -fn parse_float(num: &str, name: &str) -> f64 { - if let Ok(num) = num.parse::() { - num - } else { - error!("{} must be a floating-point number, was '{}'", name, num); - exit(1); - } -} - - fn parse_chunker(val: &str) -> ChunkerType { if let Ok(chunker) = ChunkerType::from_string(val) { chunker @@ -173,6 +167,15 @@ fn parse_hash(val: &str) -> HashMethod { } } +fn parse_bundle_id(val: &str) -> BundleId { + if let Ok(hash) = Hash::from_string(val) { + BundleId(hash) + } else { + error!("Invalid bundle id: {}", val); + exit(1); + } +} + pub fn parse() -> Arguments { let args = clap_app!(zvault => @@ -224,7 +227,7 @@ pub fn parse() -> Arguments { ) (@subcommand vacuum => (about: "saves space by combining and recompressing bundles") - (@arg ratio: --ratio -r +takes_value "ratio of unused chunks in a bundle to rewrite that bundle") + (@arg ratio: --ratio -r +takes_value "ratio in % of unused space in a bundle to rewrite that bundle") (@arg force: --force -f "actually run the vacuum instead of simulating it") (@arg REPO: +required "path of the repository") ) @@ -237,10 +240,15 @@ pub fn parse() -> Arguments { (about: "lists backups or backup contents") (@arg PATH: +required "repository[::backup[::subpath]] path") ) - (@subcommand listbundles => + (@subcommand bundlelist => (about: "lists bundles in a repository") (@arg REPO: +required "path of the repository") ) + (@subcommand bundleinfo => + (about: "lists bundles in a repository") + (@arg REPO: +required "path of the repository") + (@arg BUNDLE: +required "the bundle id") + ) (@subcommand import => (about: "reconstruct a repository from the remote files") (@arg key: --key -k ... +takes_value "a file with a needed to read the bundles") @@ -368,7 +376,7 @@ pub fn parse() -> Arguments { return Arguments::Vacuum { repo_path: repository.to_string(), force: args.is_present("force"), - ratio: parse_float(args.value_of("ratio").unwrap_or(&DEFAULT_VACUUM_RATIO.to_string()), "ratio") as f32 + ratio: parse_num(args.value_of("ratio").unwrap_or(&DEFAULT_VACUUM_RATIO.to_string()), "ratio") as f32 / 100.0 } } if let Some(args) = args.subcommand_matches("check") { @@ -388,16 +396,27 @@ pub fn parse() -> Arguments { inode: inode.map(|v| v.to_string()) } } - if let Some(args) = args.subcommand_matches("listbundles") { + if let Some(args) = args.subcommand_matches("bundlelist") { let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap()); if backup.is_some() || inode.is_some() { println!("No backups or subpaths may be given here"); exit(1); } - return Arguments::ListBundles { + return Arguments::BundleList { repo_path: repository.to_string(), } } + if let Some(args) = args.subcommand_matches("bundleinfo") { + let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap()); + if backup.is_some() || inode.is_some() { + println!("No backups or subpaths may be given here"); + exit(1); + } + return Arguments::BundleInfo { + repo_path: repository.to_string(), + bundle_id: parse_bundle_id(args.value_of("BUNDLE").unwrap()) + } + } if let Some(args) = args.subcommand_matches("info") { let (repository, backup, inode) = split_repo_path(args.value_of("PATH").unwrap()); return Arguments::Info { diff --git a/src/cli/mod.rs b/src/cli/mod.rs index a3deb42..2a29ab0 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -20,7 +20,7 @@ pub const DEFAULT_CHUNKER: &'static str = "fastcdc/16"; pub const DEFAULT_HASH: &'static str = "blake2"; pub const DEFAULT_COMPRESSION: &'static str = "brotli/3"; pub const DEFAULT_BUNDLE_SIZE: usize = 25; -pub const DEFAULT_VACUUM_RATIO: f32 = 0.5; +pub const DEFAULT_VACUUM_RATIO: usize = 50; fn checked(result: Result, msg: &'static str) -> T { @@ -80,6 +80,14 @@ fn print_backup(backup: &Backup) { println!("Chunk count: {}, avg size: {}", backup.chunk_count, to_file_size(backup.avg_chunk_size as u64)); } +pub fn format_inode_one_line(inode: &Inode) -> String { + match inode.file_type { + FileType::Directory => format!("{:25}\t{} entries", format!("{}/", inode.name), inode.children.as_ref().unwrap().len()), + FileType::File => format!("{:25}\t{}", inode.name, to_file_size(inode.size)), + FileType::Symlink => format!("{:25}\t -> {}", inode.name, inode.symlink_target.as_ref().unwrap()), + } +} + fn print_inode(inode: &Inode) { println!("Name: {}", inode.name); println!("Type: {}", inode.file_type); @@ -123,6 +131,12 @@ fn print_bundle(bundle: &BundleInfo) { println!("Bundle {}", bundle.id); println!(" - Mode: {:?}", bundle.mode); println!(" - Hash method: {:?}", bundle.hash_method); + let encryption = if let Some((_, ref key)) = bundle.encryption { + to_hex(key) + } else { + "none".to_string() + }; + println!(" - Encryption: {}", encryption); println!(" - Chunks: {}", bundle.chunk_count); println!(" - Size: {}", to_file_size(bundle.encoded_size as u64)); println!(" - Data size: {}", to_file_size(bundle.raw_size as u64)); @@ -135,6 +149,10 @@ fn print_bundle(bundle: &BundleInfo) { println!(" - Compression: {}, ratio: {:.1}%", compression, ratio * 100.0); } +fn print_bundle_one_line(bundle: &BundleInfo) { + println!("{}: {:8?}, {:5} chunks, {:8}", bundle.id, bundle.mode, bundle.chunk_count, to_file_size(bundle.encoded_size as u64)) +} + fn print_config(config: &Config) { println!("Bundle size: {}", to_file_size(config.bundle_size as u64)); println!("Chunker: {}", config.chunker.to_string()); @@ -198,7 +216,7 @@ pub fn run() { } } let excludes: Vec = excludes.into_iter().map(|mut exclude| { - exclude = regex::escape(&exclude).replace('?', ".").replace(r"\*\", ".*").replace(r"\*", "[^/]*"); + exclude = regex::escape(&exclude).replace('?', ".").replace(r"\*\*", ".*").replace(r"\*", "[^/]*"); if exclude.starts_with('/') { format!(r"^{}($|/)", exclude) } else { @@ -263,9 +281,13 @@ pub fn run() { }, Arguments::Vacuum{repo_path, ratio, force} => { let mut repo = open_repository(&repo_path); + let info_before = repo.info(); checked(repo.vacuum(ratio, force), "vacuum"); if !force { info!("Run with --force to actually execute this command"); + } else { + let info_after = repo.info(); + info!("Reclaimed {}", to_file_size(info_before.encoded_data_size - info_after.encoded_data_size)); } }, Arguments::Check{repo_path, backup_name, inode, full} => { @@ -324,11 +346,19 @@ pub fn run() { print_repoinfo(&repo.info()); } }, - Arguments::ListBundles{repo_path} => { + Arguments::BundleList{repo_path} => { let repo = open_repository(&repo_path); for bundle in repo.list_bundles() { + print_bundle_one_line(bundle); + } + }, + Arguments::BundleInfo{repo_path, bundle_id} => { + let repo = open_repository(&repo_path); + if let Some(bundle) = repo.get_bundle(&bundle_id) { print_bundle(bundle); - println!(); + } else { + error!("No such bundle"); + exit(3); } }, Arguments::Import{repo_path, remote_path, key_files} => { diff --git a/src/repository/backup.rs b/src/repository/backup.rs index 5fd8144..88105c6 100644 --- a/src/repository/backup.rs +++ b/src/repository/backup.rs @@ -51,7 +51,7 @@ impl Repository { try!(fs::remove_file(&path)); loop { path = path.parent().unwrap().to_owned(); - if fs::remove_dir(&path).is_err() { + if path == self.backups_path || fs::remove_dir(&path).is_err() { break } } diff --git a/src/repository/info.rs b/src/repository/info.rs index 4a9d03e..bdd46bb 100644 --- a/src/repository/info.rs +++ b/src/repository/info.rs @@ -20,6 +20,11 @@ impl Repository { self.bundles.list_bundles() } + #[inline] + pub fn get_bundle(&self, bundle: &BundleId) -> Option<&BundleInfo> { + self.bundles.get_bundle_info(bundle) + } + pub fn info(&self) -> RepositoryInfo { let bundles = self.list_bundles(); let encoded_data_size = bundles.iter().map(|b| b.encoded_size as u64).sum(); diff --git a/src/repository/vacuum.rs b/src/repository/vacuum.rs index 1c62c26..811a3d1 100644 --- a/src/repository/vacuum.rs +++ b/src/repository/vacuum.rs @@ -114,13 +114,13 @@ impl Repository { for id in &rewrite_bundles { let bundle = &usage[id]; let bundle_id = self.bundle_map.get(*id).unwrap(); - for chunk in 0..bundle.chunk_count { - let data = try!(self.bundles.get_chunk(&bundle_id, chunk)); - let hash = self.config.hash.hash(&data); + let chunks = try!(self.bundles.get_chunk_list(&bundle_id)); + for (chunk, &(hash, _len)) in chunks.into_iter().enumerate() { if !bundle.used.get(chunk) { try!(self.index.delete(&hash)); continue } + let data = try!(self.bundles.get_chunk(&bundle_id, chunk)); let mode = if bundle.mode.get(chunk) { BundleMode::Meta } else { diff --git a/src/util/cli.rs b/src/util/cli.rs index c7ab241..f4c1ca5 100644 --- a/src/util/cli.rs +++ b/src/util/cli.rs @@ -1,5 +1,3 @@ -use ::repository::{Inode, FileType}; - pub fn to_file_size(size: u64) -> String { let mut size = size as f32; if size >= 512.0 { @@ -38,11 +36,3 @@ pub fn to_duration(dur: f32) -> String { let secs = (secs % 60) as f32 + subsecs; format!("{}:{:02}:{:04.1}", hours, mins, secs) } - -pub fn format_inode_one_line(inode: &Inode) -> String { - match inode.file_type { - FileType::Directory => format!("{:25}\t{} entries", format!("{}/", inode.name), inode.children.as_ref().unwrap().len()), - FileType::File => format!("{:25}\t{}", inode.name, to_file_size(inode.size)), - FileType::Symlink => format!("{:25}\t -> {}", inode.name, inode.symlink_target.as_ref().unwrap()), - } -} diff --git a/src/util/hash.rs b/src/util/hash.rs index b21a65f..526332c 100644 --- a/src/util/hash.rs +++ b/src/util/hash.rs @@ -12,7 +12,6 @@ use std::u64; use std::io::{self, Read, Write}; - #[repr(packed)] #[derive(Clone, Copy, PartialEq, Hash, Eq, Default)] pub struct Hash { @@ -48,6 +47,13 @@ impl Hash { let low = try!(src.read_u64::()); Ok(Hash { high: high, low: low }) } + + #[inline] + pub fn from_string(val: &str) -> Result { + let high = try!(u64::from_str_radix(&val[..16], 16).map_err(|_| ())); + let low = try!(u64::from_str_radix(&val[16..], 16).map_err(|_| ())); + Ok(Self { high: high, low: low }) + } } impl fmt::Display for Hash {