Store bundle type in bundle

This commit is contained in:
Dennis Schwerdel 2017-03-17 07:15:19 +01:00
parent 5bca245643
commit 47d316ace3
9 changed files with 171 additions and 25 deletions

69
Cargo.lock generated
View File

@ -4,6 +4,7 @@ version = "0.1.0"
dependencies = [
"blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)",
"docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
@ -24,11 +25,31 @@ dependencies = [
"memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "ansi_term"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "atty"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "bitflags"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "bitflags"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "blake2-rfc"
version = "0.2.17"
@ -51,6 +72,21 @@ dependencies = [
"time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "clap"
version = "2.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"atty 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"term_size 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-segmentation 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"vec_map 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "constant_time_eq"
version = "0.1.2"
@ -260,6 +296,16 @@ dependencies = [
"rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "term_size"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "thread-id"
version = "3.0.0"
@ -289,6 +335,16 @@ dependencies = [
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "unicode-segmentation"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "unicode-width"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "unreachable"
version = "0.1.1"
@ -302,6 +358,11 @@ name = "utf8-ranges"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "vec_map"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "void"
version = "1.0.2"
@ -327,10 +388,14 @@ dependencies = [
[metadata]
"checksum aho-corasick 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0638fd549427caa90c499814196d1b9e3725eb4d15d7339d6de073a680ed0ca2"
"checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"
"checksum atty 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d912da0db7fa85514874458ca3651fe2cddace8d0b0505571dbdcd41ab490159"
"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d"
"checksum bitflags 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e1ab483fc81a8143faa7203c4a3c02888ebd1a782e37e41fa34753ba9a162"
"checksum blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "0c6a476f32fef3402f1161f89d0d39822809627754a126f8441ff2a9d45e2d59"
"checksum byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c40977b0ee6b9885c9013cd41d9feffdd22deb3bb4dc3a71d901cc7a77de18c8"
"checksum chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "158b0bd7d75cbb6bf9c25967a48a2e9f77da95876b858eadfabaa99cd069de6e"
"checksum clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)" = "74a80f603221c9cd9aa27a28f52af452850051598537bb6b359c38a7d61e5cda"
"checksum constant_time_eq 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "07dcb7959f0f6f1cf662f9a7ff389bcb919924d99ac41cf31f10d611d8721323"
"checksum docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab32ea6e284d87987066f21a9e809a73c14720571ef34516f0890b3d355ccfd8"
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
@ -360,11 +425,15 @@ dependencies = [
"checksum squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db1f9dde91d819b7746e153bc32489fa19e6a106c3d7f2b92187a4efbdc88b40"
"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694"
"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"
"checksum term_size 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "07b6c1ac5b3fffd75073276bca1ceed01f67a28537097a2a9539e116e50fb21a"
"checksum thread-id 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4437c97558c70d129e40629a5b385b3fb1ffac301e63941335e4d354081ec14a"
"checksum thread_local 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c85048c6260d17cf486ceae3282d9fb6b90be220bf5b28c400f5485ffc29f0c7"
"checksum time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "211b63c112206356ef1ff9b19355f43740fc3f85960c598a93d3a3d3ba7beade"
"checksum unicode-segmentation 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18127285758f0e2c6cf325bb3f3d138a12fee27de4f23e146cd6a179f26c2cf3"
"checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f"
"checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91"
"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122"
"checksum vec_map 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8cdc8b93bd0198ed872357fb2e667f7125646b1762f16d60b2c96350d361897"
"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"

View File

@ -16,3 +16,4 @@ murmurhash3 = "*"
docopt = "0.7"
rustc-serialize = "0.3"
chrono = "0.3"
clap = "2.19"

View File

@ -116,10 +116,20 @@ impl fmt::Debug for BundleId {
}
#[derive(Eq, Debug, PartialEq, Clone, Copy)]
pub enum BundleMode {
Content, Meta
}
serde_impl!(BundleMode(u8) {
Content => 0,
Meta => 1
});
#[derive(Clone)]
pub struct BundleInfo {
pub id: BundleId,
pub mode: BundleMode,
pub compression: Option<Compression>,
pub encryption: Option<Encryption>,
pub checksum: Checksum,
@ -130,6 +140,7 @@ pub struct BundleInfo {
}
serde_impl!(BundleInfo(u64) {
id: BundleId => 0,
mode: BundleMode => 8,
compression: Option<Compression> => 1,
encryption: Option<Encryption> => 2,
checksum: Checksum => 3,
@ -149,7 +160,8 @@ impl Default for BundleInfo {
raw_size: 0,
encoded_size: 0,
chunk_count: 0,
chunk_sizes: vec![]
chunk_sizes: vec![],
mode: BundleMode::Content
}
}
}
@ -281,6 +293,7 @@ impl Debug for Bundle {
pub struct BundleWriter {
mode: BundleMode,
data: Vec<u8>,
compression: Option<Compression>,
compression_stream: Option<CompressionStream>,
@ -293,12 +306,13 @@ pub struct BundleWriter {
}
impl BundleWriter {
fn new(compression: Option<Compression>, encryption: Option<Encryption>, crypto: Arc<Mutex<Crypto>>, checksum: ChecksumType) -> Result<Self, BundleError> {
fn new(mode: BundleMode, compression: Option<Compression>, encryption: Option<Encryption>, crypto: Arc<Mutex<Crypto>>, checksum: ChecksumType) -> Result<Self, BundleError> {
let compression_stream = match compression {
Some(ref compression) => Some(try!(compression.compress_stream())),
None => None
};
Ok(BundleWriter {
mode: mode,
data: vec![],
compression: compression,
compression_stream: compression_stream,
@ -341,6 +355,7 @@ impl BundleWriter {
try!(file.write_all(&HEADER_STRING).map_err(|e| BundleError::Write(e, path.clone())));
try!(file.write_all(&[HEADER_VERSION]).map_err(|e| BundleError::Write(e, path.clone())));
let header = BundleInfo {
mode: self.mode,
checksum: checksum,
compression: self.compression,
encryption: self.encryption,
@ -454,8 +469,8 @@ impl BundleDb {
}
#[inline]
pub fn create_bundle(&self) -> Result<BundleWriter, BundleError> {
BundleWriter::new(self.compression.clone(), self.encryption.clone(), self.crypto.clone(), self.checksum)
pub fn create_bundle(&self, mode: BundleMode) -> Result<BundleWriter, BundleError> {
BundleWriter::new(mode, self.compression.clone(), self.encryption.clone(), self.crypto.clone(), self.checksum)
}
pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleError> {

View File

@ -1,8 +1,13 @@
use clap::{Arg, App, SubCommand};
use docopt::Docopt;
use ::chunker::ChunkerType;
use ::util::{ChecksumType, Compression, HashMethod};
use std::process::exit;
use std::path::Path;
static USAGE: &'static str = "
Usage:
@ -66,7 +71,7 @@ pub enum Arguments {
bundle_size: usize,
chunker: ChunkerType,
chunk_size: usize,
compresion: Compression
compression: Compression
},
Backup {
repo_path: String,
@ -108,3 +113,58 @@ pub enum Arguments {
pub fn parse() -> DocoptArgs {
Docopt::new(USAGE).and_then(|d| d.decode()).unwrap_or_else(|e| e.exit())
}
pub fn parse2() -> Arguments {
let args = clap_app!(zvault =>
(version: "0.1")
(author: "Dennis Schwerdel <schwerdel@googlemail.com>")
(about: "Deduplicating backup tool")
(@subcommand init =>
(about: "initializes a new repository")
(@arg bundle_size: --bundle-size +takes_value "maximal bundle size")
(@arg chunker: --chunker +takes_value "chunker algorithm")
(@arg chunk_size: --chunk-size +takes_value "average chunk size")
(@arg compression: --compression -c +takes_value "compression to use")
(@arg REPO: +required "path of the repository")
)
(@subcommand backup =>
(about: "creates a new backup")
(@arg full: --full "create a full backup")
(@arg BACKUP: +required "repository::backup path")
(@arg SRC: +required "source path to backup")
)
(@subcommand restore =>
(about: "restores a backup")
(@arg BACKUP: +required "repository::backup[::subpath] path")
(@arg DST: +required "destination path for backup")
)
(@subcommand check =>
(about: "checks the repository")
(@arg full: --full "also check file contents")
(@arg PATH: +required "repository[::backup] path")
)
(@subcommand list =>
(about: "lists backups or backup contents")
(@arg PATH: +required "repository[::backup[::subpath]] path")
)
(@subcommand listbundles =>
(about: "lists bundles in a repository")
(@arg PATH: +required "repository path")
)
(@subcommand info =>
(about: "displays information on a repository, a backup or a path in a backup")
(@arg PATH: +required "repository[::backup[::subpath]] path")
)
(@subcommand algotest =>
(about: "test a specific algorithm combination")
(@arg bundle_size: --bundle-size +takes_value "maximal bundle size")
(@arg chunker: --chunker +takes_value "chunker algorithm")
(@arg chunk_size: --chunk-size +takes_value "average chunk size")
(@arg compression: --compression -c +takes_value "compression to use")
(@arg FILE: +required "the file to test the algorithms with")
)
).get_matches();
if let Some(args) = args.subcommand_matches("init") {
}
unimplemented!()
}

View File

@ -79,6 +79,7 @@ pub fn run() {
if args.cmd_bundles {
for bundle in repo.list_bundles() {
println!("Bundle {}", bundle.id);
println!(" - Mode: {:?}", bundle.mode);
println!(" - Chunks: {}", bundle.chunk_count);
println!(" - Size: {}", to_file_size(bundle.encoded_size as u64));
println!(" - Data size: {}", to_file_size(bundle.raw_size as u64));

View File

@ -10,6 +10,7 @@ extern crate serde_yaml;
extern crate docopt;
extern crate rustc_serialize;
extern crate chrono;
#[macro_use] extern crate clap;
pub mod util;
pub mod bundle;
@ -21,7 +22,6 @@ mod cli;
// TODO: Seperate remote folder
// TODO: Copy backup files to remote folder
// TODO: Keep meta bundles also locally
// TODO: Store bundle type in bundle
// TODO: Remove backups (based on age like attic)
// TODO: Backup files tree structure
// TODO: Recompress & combine bundles
@ -31,6 +31,8 @@ mod cli;
// TODO: Partial backups
// TODO: Load and compare remote bundles to bundle map
// TODO: Nice errors / checks for CLI
// TODO: Import remote backup
// TODO: Continue on errors
fn main() {
cli::run();

View File

@ -1,9 +1,9 @@
use std::mem;
use std::io::{Read, Write, Cursor};
use super::{Repository, Mode, RepositoryError};
use super::{Repository, RepositoryError};
use ::index::Location;
use ::bundle::BundleId;
use ::bundle::{BundleId, BundleMode};
use super::integrity::RepositoryIntegrityError;
use ::util::Hash;
@ -35,7 +35,7 @@ impl Repository {
Ok(Some(try!(self.bundles.get_chunk(&bundle_id, found.chunk as usize))))
}
pub fn put_chunk(&mut self, mode: Mode, hash: Hash, data: &[u8]) -> Result<(), RepositoryError> {
pub fn put_chunk(&mut self, mode: BundleMode, hash: Hash, data: &[u8]) -> Result<(), RepositoryError> {
// If this chunk is in the index, ignore it
if self.index.contains(&hash) {
return Ok(())
@ -44,12 +44,12 @@ impl Repository {
let next_free_bundle_id = self.next_free_bundle_id();
// Select a bundle writer according to the mode and...
let writer = match mode {
Mode::Content => &mut self.content_bundle,
Mode::Meta => &mut self.meta_bundle
BundleMode::Content => &mut self.content_bundle,
BundleMode::Meta => &mut self.meta_bundle
};
// ...alocate one if needed
if writer.is_none() {
*writer = Some(try!(self.bundles.create_bundle()));
*writer = Some(try!(self.bundles.create_bundle(mode)));
}
debug_assert!(writer.is_some());
let chunk_id;
@ -63,8 +63,8 @@ impl Repository {
raw_size = writer_obj.raw_size();
}
let bundle_id = match mode {
Mode::Content => self.next_content_bundle,
Mode::Meta => self.next_meta_bundle
BundleMode::Content => self.next_content_bundle,
BundleMode::Meta => self.next_meta_bundle
};
// Finish bundle if over maximum size
if size >= self.config.bundle_size || raw_size >= 4 * self.config.bundle_size {
@ -86,12 +86,12 @@ impl Repository {
}
#[inline]
pub fn put_data(&mut self, mode: Mode, data: &[u8]) -> Result<Vec<Chunk>, RepositoryError> {
pub fn put_data(&mut self, mode: BundleMode, data: &[u8]) -> Result<Vec<Chunk>, RepositoryError> {
let mut input = Cursor::new(data);
self.put_stream(mode, &mut input)
}
pub fn put_stream<R: Read>(&mut self, mode: Mode, data: &mut R) -> Result<Vec<Chunk>, RepositoryError> {
pub fn put_stream<R: Read>(&mut self, mode: BundleMode, data: &mut R) -> Result<Vec<Chunk>, RepositoryError> {
let avg_size = self.config.chunker.avg_size();
let mut chunks = Vec::new();
let mut chunk = Vec::with_capacity(avg_size * 2);

View File

@ -6,8 +6,9 @@ use std::os::unix::fs::{PermissionsExt, symlink};
use std::io::{Read, Write};
use ::util::*;
use super::{Repository, RepositoryError, Mode, Chunk};
use super::{Repository, RepositoryError, Chunk};
use super::integrity::RepositoryIntegrityError;
use ::bundle::BundleMode;
#[derive(Debug, Eq, PartialEq)]
@ -85,6 +86,7 @@ serde_impl!(Inode(u8) {
});
impl Inode {
#[inline]
fn get_extended_attrs_from(&mut self, meta: &Metadata) -> Result<(), RepositoryError> {
self.mode = meta.st_mode();
self.user = meta.st_uid();
@ -156,12 +158,12 @@ impl Repository {
try!(file.read_to_end(&mut data));
inode.contents = Some(FileContents::Inline(data.into()));
} else {
let mut chunks = try!(self.put_stream(Mode::Content, &mut file));
let mut chunks = try!(self.put_stream(BundleMode::Content, &mut file));
if chunks.len() < 10 {
inode.contents = Some(FileContents::ChunkedDirect(chunks));
} else {
let chunks_data = try!(msgpack::encode(&chunks));
chunks = try!(self.put_data(Mode::Content, &chunks_data));
chunks = try!(self.put_data(BundleMode::Content, &chunks_data));
inode.contents = Some(FileContents::ChunkedIndirect(chunks));
}
}
@ -169,8 +171,9 @@ impl Repository {
Ok(inode)
}
#[inline]
pub fn put_inode(&mut self, inode: &Inode) -> Result<Vec<Chunk>, RepositoryError> {
self.put_data(Mode::Meta, &try!(msgpack::encode(inode)))
self.put_data(BundleMode::Meta, &try!(msgpack::encode(inode)))
}
#[inline]

View File

@ -24,11 +24,6 @@ pub use self::backup::Backup;
use self::bundle_map::BundleMap;
#[derive(Eq, Debug, PartialEq, Clone, Copy)]
pub enum Mode {
Content, Meta
}
pub struct Repository {
path: PathBuf,
config: Config,