Store bundle type in bundle

5 years ago · 47d316ace3
9 changed files with 171 additions and 25 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -4,6 +4,7 @@ version = "0.1.0"
 dependencies = [
 "blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
 "chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)",
 "docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
 "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
 "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
@ -24,11 +25,31 @@ dependencies = [
 "memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

+[[package]]
+name = "ansi_term"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "atty"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "bitflags"
 version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"

+[[package]]
+name = "bitflags"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "blake2-rfc"
 version = "0.2.17"
@ -51,6 +72,21 @@ dependencies = [
 "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

+[[package]]
+name = "clap"
+version = "2.21.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "atty 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bitflags 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "term_size 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-segmentation 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "vec_map 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "constant_time_eq"
 version = "0.1.2"
@ -260,6 +296,16 @@ dependencies = [
 "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

+[[package]]
+name = "term_size"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "thread-id"
 version = "3.0.0"
@ -289,6 +335,16 @@ dependencies = [
 "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]

+[[package]]
+name = "unicode-segmentation"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "unreachable"
 version = "0.1.1"
@ -302,6 +358,11 @@ name = "utf8-ranges"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"

+[[package]]
+name = "vec_map"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "void"
 version = "1.0.2"
@ -327,10 +388,14 @@ dependencies = [

 [metadata]
 "checksum aho-corasick 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0638fd549427caa90c499814196d1b9e3725eb4d15d7339d6de073a680ed0ca2"
+"checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"
+"checksum atty 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d912da0db7fa85514874458ca3651fe2cddace8d0b0505571dbdcd41ab490159"
 "checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d"
+"checksum bitflags 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e1ab483fc81a8143faa7203c4a3c02888ebd1a782e37e41fa34753ba9a162"
 "checksum blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "0c6a476f32fef3402f1161f89d0d39822809627754a126f8441ff2a9d45e2d59"
 "checksum byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c40977b0ee6b9885c9013cd41d9feffdd22deb3bb4dc3a71d901cc7a77de18c8"
 "checksum chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "158b0bd7d75cbb6bf9c25967a48a2e9f77da95876b858eadfabaa99cd069de6e"
+"checksum clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)" = "74a80f603221c9cd9aa27a28f52af452850051598537bb6b359c38a7d61e5cda"
 "checksum constant_time_eq 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "07dcb7959f0f6f1cf662f9a7ff389bcb919924d99ac41cf31f10d611d8721323"
 "checksum docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab32ea6e284d87987066f21a9e809a73c14720571ef34516f0890b3d355ccfd8"
 "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
@ -360,11 +425,15 @@ dependencies = [
 "checksum squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db1f9dde91d819b7746e153bc32489fa19e6a106c3d7f2b92187a4efbdc88b40"
 "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694"
 "checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"
+"checksum term_size 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "07b6c1ac5b3fffd75073276bca1ceed01f67a28537097a2a9539e116e50fb21a"
 "checksum thread-id 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4437c97558c70d129e40629a5b385b3fb1ffac301e63941335e4d354081ec14a"
 "checksum thread_local 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c85048c6260d17cf486ceae3282d9fb6b90be220bf5b28c400f5485ffc29f0c7"
 "checksum time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "211b63c112206356ef1ff9b19355f43740fc3f85960c598a93d3a3d3ba7beade"
+"checksum unicode-segmentation 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18127285758f0e2c6cf325bb3f3d138a12fee27de4f23e146cd6a179f26c2cf3"
+"checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f"
 "checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91"
 "checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122"
+"checksum vec_map 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8cdc8b93bd0198ed872357fb2e667f7125646b1762f16d60b2c96350d361897"
 "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
 "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
 "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
--- a/Cargo.toml
+++ b/Cargo.toml
@ -16,3 +16,4 @@ murmurhash3 = "*"
 docopt = "0.7"
 rustc-serialize = "0.3"
 chrono = "0.3"
+clap = "2.19"
--- a/src/bundle.rs
+++ b/src/bundle.rs
@ -116,10 +116,20 @@ impl fmt::Debug for BundleId {
 }


+#[derive(Eq, Debug, PartialEq, Clone, Copy)]
+pub enum BundleMode {
+    Content, Meta
+}
+serde_impl!(BundleMode(u8) {
+    Content => 0,
+    Meta => 1
+});
+

 #[derive(Clone)]
 pub struct BundleInfo {
    pub id: BundleId,
+    pub mode: BundleMode,
    pub compression: Option<Compression>,
    pub encryption: Option<Encryption>,
    pub checksum: Checksum,
@ -130,6 +140,7 @@ pub struct BundleInfo {
 }
 serde_impl!(BundleInfo(u64) {
    id: BundleId => 0,
+    mode: BundleMode => 8,
    compression: Option<Compression> => 1,
    encryption: Option<Encryption> => 2,
    checksum: Checksum => 3,
@ -149,7 +160,8 @@ impl Default for BundleInfo {
            raw_size: 0,
            encoded_size: 0,
            chunk_count: 0,
-            chunk_sizes: vec![]
+            chunk_sizes: vec![],
+            mode: BundleMode::Content
        }
    }
 }
@ -281,6 +293,7 @@ impl Debug for Bundle {


 pub struct BundleWriter {
+    mode: BundleMode,
    data: Vec<u8>,
    compression: Option<Compression>,
    compression_stream: Option<CompressionStream>,
@ -293,12 +306,13 @@ pub struct BundleWriter {
 }

 impl BundleWriter {
-    fn new(compression: Option<Compression>, encryption: Option<Encryption>, crypto: Arc<Mutex<Crypto>>, checksum: ChecksumType) -> Result<Self, BundleError> {
+    fn new(mode: BundleMode, compression: Option<Compression>, encryption: Option<Encryption>, crypto: Arc<Mutex<Crypto>>, checksum: ChecksumType) -> Result<Self, BundleError> {
        let compression_stream = match compression {
            Some(ref compression) => Some(try!(compression.compress_stream())),
            None => None
        };
        Ok(BundleWriter {
+            mode: mode,
            data: vec![],
            compression: compression,
            compression_stream: compression_stream,
@ -341,6 +355,7 @@ impl BundleWriter {
        try!(file.write_all(&HEADER_STRING).map_err(|e| BundleError::Write(e, path.clone())));
        try!(file.write_all(&[HEADER_VERSION]).map_err(|e| BundleError::Write(e, path.clone())));
        let header = BundleInfo {
+            mode: self.mode,
            checksum: checksum,
            compression: self.compression,
            encryption: self.encryption,
@ -454,8 +469,8 @@ impl BundleDb {
    }

    #[inline]
-    pub fn create_bundle(&self) -> Result<BundleWriter, BundleError> {
-        BundleWriter::new(self.compression.clone(), self.encryption.clone(), self.crypto.clone(), self.checksum)
+    pub fn create_bundle(&self, mode: BundleMode) -> Result<BundleWriter, BundleError> {
+        BundleWriter::new(mode, self.compression.clone(), self.encryption.clone(), self.crypto.clone(), self.checksum)
    }

    pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleError> {
--- a/src/cli/args.rs
+++ b/src/cli/args.rs
@ -1,8 +1,13 @@
+use clap::{Arg, App, SubCommand};
+
 use docopt::Docopt;

 use ::chunker::ChunkerType;
 use ::util::{ChecksumType, Compression, HashMethod};

+use std::process::exit;
+use std::path::Path;
+

 static USAGE: &'static str = "
 Usage:
@ -66,7 +71,7 @@ pub enum Arguments {
        bundle_size: usize,
        chunker: ChunkerType,
        chunk_size: usize,
-        compresion: Compression
+        compression: Compression
    },
    Backup {
        repo_path: String,
@ -108,3 +113,58 @@ pub enum Arguments {
 pub fn parse() -> DocoptArgs {
    Docopt::new(USAGE).and_then(|d| d.decode()).unwrap_or_else(|e| e.exit())
 }
+
+pub fn parse2() -> Arguments {
+    let args = clap_app!(zvault =>
+        (version: "0.1")
+        (author: "Dennis Schwerdel <schwerdel@googlemail.com>")
+        (about: "Deduplicating backup tool")
+        (@subcommand init =>
+            (about: "initializes a new repository")
+            (@arg bundle_size: --bundle-size +takes_value "maximal bundle size")
+            (@arg chunker: --chunker +takes_value "chunker algorithm")
+            (@arg chunk_size: --chunk-size +takes_value "average chunk size")
+            (@arg compression: --compression -c +takes_value "compression to use")
+            (@arg REPO: +required "path of the repository")
+        )
+        (@subcommand backup =>
+            (about: "creates a new backup")
+            (@arg full: --full "create a full backup")
+            (@arg BACKUP: +required "repository::backup path")
+            (@arg SRC: +required "source path to backup")
+        )
+        (@subcommand restore =>
+            (about: "restores a backup")
+            (@arg BACKUP: +required "repository::backup[::subpath] path")
+            (@arg DST: +required "destination path for backup")
+        )
+        (@subcommand check =>
+            (about: "checks the repository")
+            (@arg full: --full "also check file contents")
+            (@arg PATH: +required "repository[::backup] path")
+        )
+        (@subcommand list =>
+            (about: "lists backups or backup contents")
+            (@arg PATH: +required "repository[::backup[::subpath]] path")
+        )
+        (@subcommand listbundles =>
+            (about: "lists bundles in a repository")
+            (@arg PATH: +required "repository path")
+        )
+        (@subcommand info =>
+            (about: "displays information on a repository, a backup or a path in a backup")
+            (@arg PATH: +required "repository[::backup[::subpath]] path")
+        )
+        (@subcommand algotest =>
+            (about: "test a specific algorithm combination")
+            (@arg bundle_size: --bundle-size +takes_value "maximal bundle size")
+            (@arg chunker: --chunker +takes_value "chunker algorithm")
+            (@arg chunk_size: --chunk-size +takes_value "average chunk size")
+            (@arg compression: --compression -c +takes_value "compression to use")
+            (@arg FILE: +required "the file to test the algorithms with")
+        )
+    ).get_matches();
+    if let Some(args) = args.subcommand_matches("init") {
+    }
+    unimplemented!()
+}
--- a/src/cli/mod.rs
+++ b/src/cli/mod.rs
@ -79,6 +79,7 @@ pub fn run() {
    if args.cmd_bundles {
        for bundle in repo.list_bundles() {
            println!("Bundle {}", bundle.id);
+            println!("  - Mode: {:?}", bundle.mode);
            println!("  - Chunks: {}", bundle.chunk_count);
            println!("  - Size: {}", to_file_size(bundle.encoded_size as u64));
            println!("  - Data size: {}", to_file_size(bundle.raw_size as u64));
--- a/src/main.rs
+++ b/src/main.rs
@ -10,6 +10,7 @@ extern crate serde_yaml;
 extern crate docopt;
 extern crate rustc_serialize;
 extern crate chrono;
+#[macro_use] extern crate clap;

 pub mod util;
 pub mod bundle;
@ -21,7 +22,6 @@ mod cli;
 // TODO: Seperate remote folder
 // TODO: Copy backup files to remote folder
 // TODO: Keep meta bundles also locally
-// TODO: Store bundle type in bundle
 // TODO: Remove backups (based on age like attic)
 // TODO: Backup files tree structure
 // TODO: Recompress & combine bundles
@ -31,6 +31,8 @@ mod cli;
 // TODO: Partial backups
 // TODO: Load and compare remote bundles to bundle map
 // TODO: Nice errors / checks for CLI
+// TODO: Import remote backup
+// TODO: Continue on errors

 fn main() {
    cli::run();
--- a/src/repository/basic_io.rs
+++ b/src/repository/basic_io.rs
@ -1,9 +1,9 @@
 use std::mem;
 use std::io::{Read, Write, Cursor};

-use super::{Repository, Mode, RepositoryError};
+use super::{Repository, RepositoryError};
 use ::index::Location;
-use ::bundle::BundleId;
+use ::bundle::{BundleId, BundleMode};
 use super::integrity::RepositoryIntegrityError;

 use ::util::Hash;
@ -35,7 +35,7 @@ impl Repository {
        Ok(Some(try!(self.bundles.get_chunk(&bundle_id, found.chunk as usize))))
    }

-    pub fn put_chunk(&mut self, mode: Mode, hash: Hash, data: &[u8]) -> Result<(), RepositoryError> {
+    pub fn put_chunk(&mut self, mode: BundleMode, hash: Hash, data: &[u8]) -> Result<(), RepositoryError> {
        // If this chunk is in the index, ignore it
        if self.index.contains(&hash) {
            return Ok(())
@ -44,12 +44,12 @@ impl Repository {
        let next_free_bundle_id = self.next_free_bundle_id();
        // Select a bundle writer according to the mode and...
        let writer = match mode {
-            Mode::Content => &mut self.content_bundle,
-            Mode::Meta => &mut self.meta_bundle
+            BundleMode::Content => &mut self.content_bundle,
+            BundleMode::Meta => &mut self.meta_bundle
        };
        // ...alocate one if needed
        if writer.is_none() {
-            *writer = Some(try!(self.bundles.create_bundle()));
+            *writer = Some(try!(self.bundles.create_bundle(mode)));
        }
        debug_assert!(writer.is_some());
        let chunk_id;
@ -63,8 +63,8 @@ impl Repository {
            raw_size = writer_obj.raw_size();
        }
        let bundle_id = match mode {
-            Mode::Content => self.next_content_bundle,
-            Mode::Meta => self.next_meta_bundle
+            BundleMode::Content => self.next_content_bundle,
+            BundleMode::Meta => self.next_meta_bundle
        };
        // Finish bundle if over maximum size
        if size >= self.config.bundle_size || raw_size >= 4 * self.config.bundle_size {
@ -86,12 +86,12 @@ impl Repository {
    }

    #[inline]
-    pub fn put_data(&mut self, mode: Mode, data: &[u8]) -> Result<Vec<Chunk>, RepositoryError> {
+    pub fn put_data(&mut self, mode: BundleMode, data: &[u8]) -> Result<Vec<Chunk>, RepositoryError> {
        let mut input = Cursor::new(data);
        self.put_stream(mode, &mut input)
    }

-    pub fn put_stream<R: Read>(&mut self, mode: Mode, data: &mut R) -> Result<Vec<Chunk>, RepositoryError> {
+    pub fn put_stream<R: Read>(&mut self, mode: BundleMode, data: &mut R) -> Result<Vec<Chunk>, RepositoryError> {
        let avg_size = self.config.chunker.avg_size();
        let mut chunks = Vec::new();
        let mut chunk = Vec::with_capacity(avg_size * 2);
--- a/src/repository/metadata.rs
+++ b/src/repository/metadata.rs
@ -6,8 +6,9 @@ use std::os::unix::fs::{PermissionsExt, symlink};
 use std::io::{Read, Write};

 use ::util::*;
-use super::{Repository, RepositoryError, Mode, Chunk};
+use super::{Repository, RepositoryError, Chunk};
 use super::integrity::RepositoryIntegrityError;
+use ::bundle::BundleMode;


 #[derive(Debug, Eq, PartialEq)]
@ -85,6 +86,7 @@ serde_impl!(Inode(u8) {
 });

 impl Inode {
+    #[inline]
    fn get_extended_attrs_from(&mut self, meta: &Metadata) -> Result<(), RepositoryError> {
        self.mode = meta.st_mode();
        self.user = meta.st_uid();
@ -156,12 +158,12 @@ impl Repository {
                try!(file.read_to_end(&mut data));
                inode.contents = Some(FileContents::Inline(data.into()));
            } else {
-                let mut chunks = try!(self.put_stream(Mode::Content, &mut file));
+                let mut chunks = try!(self.put_stream(BundleMode::Content, &mut file));
                if chunks.len() < 10 {
                    inode.contents = Some(FileContents::ChunkedDirect(chunks));
                } else {
                    let chunks_data = try!(msgpack::encode(&chunks));
-                    chunks = try!(self.put_data(Mode::Content, &chunks_data));
+                    chunks = try!(self.put_data(BundleMode::Content, &chunks_data));
                    inode.contents = Some(FileContents::ChunkedIndirect(chunks));
                }
            }
@ -169,8 +171,9 @@ impl Repository {
        Ok(inode)
    }

+    #[inline]
    pub fn put_inode(&mut self, inode: &Inode) -> Result<Vec<Chunk>, RepositoryError> {
-        self.put_data(Mode::Meta, &try!(msgpack::encode(inode)))
+        self.put_data(BundleMode::Meta, &try!(msgpack::encode(inode)))
    }

    #[inline]
--- a/src/repository/mod.rs
+++ b/src/repository/mod.rs
@ -24,11 +24,6 @@ pub use self::backup::Backup;
 use self::bundle_map::BundleMap;


-#[derive(Eq, Debug, PartialEq, Clone, Copy)]
-pub enum Mode {
-    Content, Meta
-}
-
 pub struct Repository {
    path: PathBuf,
    config: Config,