diff --git a/Cargo.lock b/Cargo.lock index d8e3566..f5afab2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,6 +4,7 @@ version = "0.1.0" dependencies = [ "blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)", "docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -24,11 +25,31 @@ dependencies = [ "memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ansi_term" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "atty" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bitflags" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bitflags" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "blake2-rfc" version = "0.2.17" @@ -51,6 +72,21 @@ dependencies = [ "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "clap" +version = "2.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "term_size 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-segmentation 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "vec_map 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "constant_time_eq" version = "0.1.2" @@ -260,6 +296,16 @@ dependencies = [ "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "term_size" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "thread-id" version = "3.0.0" @@ -289,6 +335,16 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "unicode-segmentation" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-width" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unreachable" version = "0.1.1" @@ -302,6 +358,11 @@ name = "utf8-ranges" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "vec_map" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "void" version = "1.0.2" @@ -327,10 +388,14 @@ dependencies = [ [metadata] "checksum aho-corasick 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0638fd549427caa90c499814196d1b9e3725eb4d15d7339d6de073a680ed0ca2" +"checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" +"checksum atty 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d912da0db7fa85514874458ca3651fe2cddace8d0b0505571dbdcd41ab490159" "checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" +"checksum bitflags 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e1ab483fc81a8143faa7203c4a3c02888ebd1a782e37e41fa34753ba9a162" "checksum blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "0c6a476f32fef3402f1161f89d0d39822809627754a126f8441ff2a9d45e2d59" "checksum byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c40977b0ee6b9885c9013cd41d9feffdd22deb3bb4dc3a71d901cc7a77de18c8" "checksum chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "158b0bd7d75cbb6bf9c25967a48a2e9f77da95876b858eadfabaa99cd069de6e" +"checksum clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)" = "74a80f603221c9cd9aa27a28f52af452850051598537bb6b359c38a7d61e5cda" "checksum constant_time_eq 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "07dcb7959f0f6f1cf662f9a7ff389bcb919924d99ac41cf31f10d611d8721323" "checksum docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab32ea6e284d87987066f21a9e809a73c14720571ef34516f0890b3d355ccfd8" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" @@ -360,11 +425,15 @@ dependencies = [ "checksum squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db1f9dde91d819b7746e153bc32489fa19e6a106c3d7f2b92187a4efbdc88b40" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" +"checksum term_size 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "07b6c1ac5b3fffd75073276bca1ceed01f67a28537097a2a9539e116e50fb21a" "checksum thread-id 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4437c97558c70d129e40629a5b385b3fb1ffac301e63941335e4d354081ec14a" "checksum thread_local 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c85048c6260d17cf486ceae3282d9fb6b90be220bf5b28c400f5485ffc29f0c7" "checksum time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "211b63c112206356ef1ff9b19355f43740fc3f85960c598a93d3a3d3ba7beade" +"checksum unicode-segmentation 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18127285758f0e2c6cf325bb3f3d138a12fee27de4f23e146cd6a179f26c2cf3" +"checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" "checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91" "checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" +"checksum vec_map 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8cdc8b93bd0198ed872357fb2e667f7125646b1762f16d60b2c96350d361897" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" diff --git a/Cargo.toml b/Cargo.toml index db4024e..5194a9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,3 +16,4 @@ murmurhash3 = "*" docopt = "0.7" rustc-serialize = "0.3" chrono = "0.3" +clap = "2.19" diff --git a/src/bundle.rs b/src/bundle.rs index 6a32f3d..284570e 100644 --- a/src/bundle.rs +++ b/src/bundle.rs @@ -116,10 +116,20 @@ impl fmt::Debug for BundleId { } +#[derive(Eq, Debug, PartialEq, Clone, Copy)] +pub enum BundleMode { + Content, Meta +} +serde_impl!(BundleMode(u8) { + Content => 0, + Meta => 1 +}); + #[derive(Clone)] pub struct BundleInfo { pub id: BundleId, + pub mode: BundleMode, pub compression: Option, pub encryption: Option, pub checksum: Checksum, @@ -130,6 +140,7 @@ pub struct BundleInfo { } serde_impl!(BundleInfo(u64) { id: BundleId => 0, + mode: BundleMode => 8, compression: Option => 1, encryption: Option => 2, checksum: Checksum => 3, @@ -149,7 +160,8 @@ impl Default for BundleInfo { raw_size: 0, encoded_size: 0, chunk_count: 0, - chunk_sizes: vec![] + chunk_sizes: vec![], + mode: BundleMode::Content } } } @@ -281,6 +293,7 @@ impl Debug for Bundle { pub struct BundleWriter { + mode: BundleMode, data: Vec, compression: Option, compression_stream: Option, @@ -293,12 +306,13 @@ pub struct BundleWriter { } impl BundleWriter { - fn new(compression: Option, encryption: Option, crypto: Arc>, checksum: ChecksumType) -> Result { + fn new(mode: BundleMode, compression: Option, encryption: Option, crypto: Arc>, checksum: ChecksumType) -> Result { let compression_stream = match compression { Some(ref compression) => Some(try!(compression.compress_stream())), None => None }; Ok(BundleWriter { + mode: mode, data: vec![], compression: compression, compression_stream: compression_stream, @@ -341,6 +355,7 @@ impl BundleWriter { try!(file.write_all(&HEADER_STRING).map_err(|e| BundleError::Write(e, path.clone()))); try!(file.write_all(&[HEADER_VERSION]).map_err(|e| BundleError::Write(e, path.clone()))); let header = BundleInfo { + mode: self.mode, checksum: checksum, compression: self.compression, encryption: self.encryption, @@ -454,8 +469,8 @@ impl BundleDb { } #[inline] - pub fn create_bundle(&self) -> Result { - BundleWriter::new(self.compression.clone(), self.encryption.clone(), self.crypto.clone(), self.checksum) + pub fn create_bundle(&self, mode: BundleMode) -> Result { + BundleWriter::new(mode, self.compression.clone(), self.encryption.clone(), self.crypto.clone(), self.checksum) } pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result, BundleError> { diff --git a/src/cli/args.rs b/src/cli/args.rs index 03a0741..4de3731 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -1,8 +1,13 @@ +use clap::{Arg, App, SubCommand}; + use docopt::Docopt; use ::chunker::ChunkerType; use ::util::{ChecksumType, Compression, HashMethod}; +use std::process::exit; +use std::path::Path; + static USAGE: &'static str = " Usage: @@ -66,7 +71,7 @@ pub enum Arguments { bundle_size: usize, chunker: ChunkerType, chunk_size: usize, - compresion: Compression + compression: Compression }, Backup { repo_path: String, @@ -108,3 +113,58 @@ pub enum Arguments { pub fn parse() -> DocoptArgs { Docopt::new(USAGE).and_then(|d| d.decode()).unwrap_or_else(|e| e.exit()) } + +pub fn parse2() -> Arguments { + let args = clap_app!(zvault => + (version: "0.1") + (author: "Dennis Schwerdel ") + (about: "Deduplicating backup tool") + (@subcommand init => + (about: "initializes a new repository") + (@arg bundle_size: --bundle-size +takes_value "maximal bundle size") + (@arg chunker: --chunker +takes_value "chunker algorithm") + (@arg chunk_size: --chunk-size +takes_value "average chunk size") + (@arg compression: --compression -c +takes_value "compression to use") + (@arg REPO: +required "path of the repository") + ) + (@subcommand backup => + (about: "creates a new backup") + (@arg full: --full "create a full backup") + (@arg BACKUP: +required "repository::backup path") + (@arg SRC: +required "source path to backup") + ) + (@subcommand restore => + (about: "restores a backup") + (@arg BACKUP: +required "repository::backup[::subpath] path") + (@arg DST: +required "destination path for backup") + ) + (@subcommand check => + (about: "checks the repository") + (@arg full: --full "also check file contents") + (@arg PATH: +required "repository[::backup] path") + ) + (@subcommand list => + (about: "lists backups or backup contents") + (@arg PATH: +required "repository[::backup[::subpath]] path") + ) + (@subcommand listbundles => + (about: "lists bundles in a repository") + (@arg PATH: +required "repository path") + ) + (@subcommand info => + (about: "displays information on a repository, a backup or a path in a backup") + (@arg PATH: +required "repository[::backup[::subpath]] path") + ) + (@subcommand algotest => + (about: "test a specific algorithm combination") + (@arg bundle_size: --bundle-size +takes_value "maximal bundle size") + (@arg chunker: --chunker +takes_value "chunker algorithm") + (@arg chunk_size: --chunk-size +takes_value "average chunk size") + (@arg compression: --compression -c +takes_value "compression to use") + (@arg FILE: +required "the file to test the algorithms with") + ) + ).get_matches(); + if let Some(args) = args.subcommand_matches("init") { + } + unimplemented!() +} diff --git a/src/cli/mod.rs b/src/cli/mod.rs index c79e450..86208c4 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -79,6 +79,7 @@ pub fn run() { if args.cmd_bundles { for bundle in repo.list_bundles() { println!("Bundle {}", bundle.id); + println!(" - Mode: {:?}", bundle.mode); println!(" - Chunks: {}", bundle.chunk_count); println!(" - Size: {}", to_file_size(bundle.encoded_size as u64)); println!(" - Data size: {}", to_file_size(bundle.raw_size as u64)); diff --git a/src/main.rs b/src/main.rs index 0f892ca..ea6f3bd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,6 +10,7 @@ extern crate serde_yaml; extern crate docopt; extern crate rustc_serialize; extern crate chrono; +#[macro_use] extern crate clap; pub mod util; pub mod bundle; @@ -21,7 +22,6 @@ mod cli; // TODO: Seperate remote folder // TODO: Copy backup files to remote folder // TODO: Keep meta bundles also locally -// TODO: Store bundle type in bundle // TODO: Remove backups (based on age like attic) // TODO: Backup files tree structure // TODO: Recompress & combine bundles @@ -31,6 +31,8 @@ mod cli; // TODO: Partial backups // TODO: Load and compare remote bundles to bundle map // TODO: Nice errors / checks for CLI +// TODO: Import remote backup +// TODO: Continue on errors fn main() { cli::run(); diff --git a/src/repository/basic_io.rs b/src/repository/basic_io.rs index 9603b24..b67d3c3 100644 --- a/src/repository/basic_io.rs +++ b/src/repository/basic_io.rs @@ -1,9 +1,9 @@ use std::mem; use std::io::{Read, Write, Cursor}; -use super::{Repository, Mode, RepositoryError}; +use super::{Repository, RepositoryError}; use ::index::Location; -use ::bundle::BundleId; +use ::bundle::{BundleId, BundleMode}; use super::integrity::RepositoryIntegrityError; use ::util::Hash; @@ -35,7 +35,7 @@ impl Repository { Ok(Some(try!(self.bundles.get_chunk(&bundle_id, found.chunk as usize)))) } - pub fn put_chunk(&mut self, mode: Mode, hash: Hash, data: &[u8]) -> Result<(), RepositoryError> { + pub fn put_chunk(&mut self, mode: BundleMode, hash: Hash, data: &[u8]) -> Result<(), RepositoryError> { // If this chunk is in the index, ignore it if self.index.contains(&hash) { return Ok(()) @@ -44,12 +44,12 @@ impl Repository { let next_free_bundle_id = self.next_free_bundle_id(); // Select a bundle writer according to the mode and... let writer = match mode { - Mode::Content => &mut self.content_bundle, - Mode::Meta => &mut self.meta_bundle + BundleMode::Content => &mut self.content_bundle, + BundleMode::Meta => &mut self.meta_bundle }; // ...alocate one if needed if writer.is_none() { - *writer = Some(try!(self.bundles.create_bundle())); + *writer = Some(try!(self.bundles.create_bundle(mode))); } debug_assert!(writer.is_some()); let chunk_id; @@ -63,8 +63,8 @@ impl Repository { raw_size = writer_obj.raw_size(); } let bundle_id = match mode { - Mode::Content => self.next_content_bundle, - Mode::Meta => self.next_meta_bundle + BundleMode::Content => self.next_content_bundle, + BundleMode::Meta => self.next_meta_bundle }; // Finish bundle if over maximum size if size >= self.config.bundle_size || raw_size >= 4 * self.config.bundle_size { @@ -86,12 +86,12 @@ impl Repository { } #[inline] - pub fn put_data(&mut self, mode: Mode, data: &[u8]) -> Result, RepositoryError> { + pub fn put_data(&mut self, mode: BundleMode, data: &[u8]) -> Result, RepositoryError> { let mut input = Cursor::new(data); self.put_stream(mode, &mut input) } - pub fn put_stream(&mut self, mode: Mode, data: &mut R) -> Result, RepositoryError> { + pub fn put_stream(&mut self, mode: BundleMode, data: &mut R) -> Result, RepositoryError> { let avg_size = self.config.chunker.avg_size(); let mut chunks = Vec::new(); let mut chunk = Vec::with_capacity(avg_size * 2); diff --git a/src/repository/metadata.rs b/src/repository/metadata.rs index e868e08..50a5882 100644 --- a/src/repository/metadata.rs +++ b/src/repository/metadata.rs @@ -6,8 +6,9 @@ use std::os::unix::fs::{PermissionsExt, symlink}; use std::io::{Read, Write}; use ::util::*; -use super::{Repository, RepositoryError, Mode, Chunk}; +use super::{Repository, RepositoryError, Chunk}; use super::integrity::RepositoryIntegrityError; +use ::bundle::BundleMode; #[derive(Debug, Eq, PartialEq)] @@ -85,6 +86,7 @@ serde_impl!(Inode(u8) { }); impl Inode { + #[inline] fn get_extended_attrs_from(&mut self, meta: &Metadata) -> Result<(), RepositoryError> { self.mode = meta.st_mode(); self.user = meta.st_uid(); @@ -156,12 +158,12 @@ impl Repository { try!(file.read_to_end(&mut data)); inode.contents = Some(FileContents::Inline(data.into())); } else { - let mut chunks = try!(self.put_stream(Mode::Content, &mut file)); + let mut chunks = try!(self.put_stream(BundleMode::Content, &mut file)); if chunks.len() < 10 { inode.contents = Some(FileContents::ChunkedDirect(chunks)); } else { let chunks_data = try!(msgpack::encode(&chunks)); - chunks = try!(self.put_data(Mode::Content, &chunks_data)); + chunks = try!(self.put_data(BundleMode::Content, &chunks_data)); inode.contents = Some(FileContents::ChunkedIndirect(chunks)); } } @@ -169,8 +171,9 @@ impl Repository { Ok(inode) } + #[inline] pub fn put_inode(&mut self, inode: &Inode) -> Result, RepositoryError> { - self.put_data(Mode::Meta, &try!(msgpack::encode(inode))) + self.put_data(BundleMode::Meta, &try!(msgpack::encode(inode))) } #[inline] diff --git a/src/repository/mod.rs b/src/repository/mod.rs index fbeee89..7654265 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -24,11 +24,6 @@ pub use self::backup::Backup; use self::bundle_map::BundleMap; -#[derive(Eq, Debug, PartialEq, Clone, Copy)] -pub enum Mode { - Content, Meta -} - pub struct Repository { path: PathBuf, config: Config,