Browse Source

Bundle encryption

pull/10/head
Dennis Schwerdel 5 years ago committed by Dennis Schwerdel
parent
commit
c8b69ebe25
  1. 24
      Cargo.lock
  2. 6
      Cargo.toml
  3. 110
      src/bundle.rs
  4. 7
      src/chunker/mod.rs
  5. 68
      src/cli/algotest.rs
  6. 162
      src/cli/args.rs
  7. 12
      src/cli/logger.rs
  8. 73
      src/cli/mod.rs
  9. 5
      src/main.rs
  10. 7
      src/repository/basic_io.rs
  11. 45
      src/repository/config.rs
  12. 17
      src/repository/error.rs
  13. 37
      src/repository/mod.rs
  14. 152
      src/util/encryption.rs
  15. 32
      src/util/mod.rs
  16. 2
      src/util/msgpack.rs

24
Cargo.lock generated

@ -2,6 +2,7 @@
name = "zvault"
version = "0.1.0"
dependencies = [
"ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -9,12 +10,14 @@ dependencies = [
"log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rmp-serde 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
"sodiumoxide 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
"squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -104,6 +107,15 @@ name = "libc"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "libsodium-sys"
version = "0.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "linked-hash-map"
version = "0.3.0"
@ -230,6 +242,16 @@ dependencies = [
"yaml-rust 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "sodiumoxide"
version = "0.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"libsodium-sys 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "squash-sys"
version = "0.9.0"
@ -320,6 +342,7 @@ dependencies = [
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
"checksum libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122"
"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135"
"checksum libsodium-sys 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "cbbc6e46017815abf8698de0ed4847fad45fd8cad2909ac38ac6de79673c1ad1"
"checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd"
"checksum log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "5141eca02775a762cc6cd564d8d2c50f67c0ea3a372cbf1c51592b3e029e10ad"
"checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823"
@ -338,6 +361,7 @@ dependencies = [
"checksum serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)" = "a702319c807c016e51f672e5c77d6f0b46afddd744b5e437d6b8436b888b458f"
"checksum serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b34a52969c7fc0254e214b82518c9a95dc88c84fc84cd847add314996a031be6"
"checksum serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f8bd3f24ad8c7bcd34a6d70ba676dc11302b96f4f166aa5f947762e01098844d"
"checksum sodiumoxide 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "bc02c0bc77ffed8e8eaef004399b825cf4fd8aa02d0af6e473225affd583ff4d"
"checksum squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db1f9dde91d819b7746e153bc32489fa19e6a106c3d7f2b92187a4efbdc88b40"
"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694"
"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"

6
Cargo.toml

@ -2,6 +2,7 @@
name = "zvault"
version = "0.1.0"
authors = ["Dennis Schwerdel <schwerdel@informatik.uni-kl.de>"]
description = "Deduplicating backup tool"
[dependencies]
serde = "0.9"
@ -18,3 +19,8 @@ chrono = "0.3"
clap = "2.19"
log = "0.3"
byteorder = "1.0"
ansi_term = "0.9"
sodiumoxide = "*"
[build-dependencies]
pkg-config = "0.3"

110
src/bundle.rs

@ -7,6 +7,7 @@ use std::fmt::{self, Debug};
use std::sync::{Arc, Mutex};
use serde::{self, Serialize, Deserialize};
use quick_error::ResultExt;
use util::*;
@ -18,7 +19,7 @@ static HEADER_VERSION: u8 = 1;
Bundle format
- Magic header + version
- Encoded header structure (contains size of next structure)
- Encoded contents structure (with chunk sizes and hashes)
- Encoded chunk list (with chunk hashes and sizes)
- Chunk data
*/
@ -31,22 +32,25 @@ quick_error!{
List(err: io::Error) {
cause(err)
description("Failed to list bundles")
display("Failed to list bundles: {}", err)
}
Read(err: io::Error, path: PathBuf) {
Io(err: io::Error, path: PathBuf) {
cause(err)
context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf())
description("Failed to read bundle")
display("Failed to read bundle {:?}: {}", path, err)
}
Decode(err: msgpack::DecodeError, path: PathBuf) {
cause(err)
context(path: &'a Path, err: msgpack::DecodeError) -> (err, path.to_path_buf())
description("Failed to decode bundle header")
}
Write(err: io::Error, path: PathBuf) {
cause(err)
description("Failed to write bundle")
display("Failed to decode bundle header of {:?}: {}", path, err)
}
Encode(err: msgpack::EncodeError, path: PathBuf) {
cause(err)
context(path: &'a Path, err: msgpack::EncodeError) -> (err, path.to_path_buf())
description("Failed to encode bundle header")
display("Failed to encode bundle header of {:?}: {}", path, err)
}
WrongHeader(path: PathBuf) {
description("Wrong header")
@ -68,13 +72,29 @@ quick_error!{
description("Bundle has no such chunk")
display("Bundle {:?} has no chunk with that id: {}", bundle, id)
}
Decompression(err: CompressionError, path: PathBuf) {
cause(err)
context(path: &'a Path, err: CompressionError) -> (err, path.to_path_buf())
description("Decompression failed")
display("Decompression failed on bundle {:?}: {}", path, err)
}
Compression(err: CompressionError) {
from()
cause(err)
description("Compression failed")
display("Compression failed: {}", err)
}
Decryption(err: EncryptionError, path: PathBuf) {
cause(err)
context(path: &'a Path, err: EncryptionError) -> (err, path.to_path_buf())
description("Decryption failed")
display("Decryption failed on bundle {:?}: {}", path, err)
}
Encryption(err: EncryptionError) {
from()
cause(err)
description("Encryption failed")
display("Encryption failed: {}", err)
}
Remove(err: io::Error, bundle: BundleId) {
cause(err)
@ -209,9 +229,9 @@ impl Bundle {
}
pub fn load(path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
let mut file = BufReader::new(try!(File::open(&path).map_err(|e| BundleError::Read(e, path.clone()))));
let mut file = BufReader::new(try!(File::open(&path).context(&path as &Path)));
let mut header = [0u8; 8];
try!(file.read_exact(&mut header).map_err(|e| BundleError::Read(e, path.clone())));
try!(file.read_exact(&mut header).context(&path as &Path));
if header[..HEADER_STRING.len()] != HEADER_STRING {
return Err(BundleError::WrongHeader(path.clone()))
}
@ -223,9 +243,9 @@ impl Bundle {
.map_err(|e| BundleError::Decode(e, path.clone())));
let mut chunk_data = Vec::with_capacity(header.chunk_info_size);
chunk_data.resize(header.chunk_info_size, 0);
try!(file.read_exact(&mut chunk_data).map_err(|e| BundleError::Read(e, path.clone())));
try!(file.read_exact(&mut chunk_data).context(&path as &Path));
if let Some(ref encryption) = header.encryption {
chunk_data = try!(crypto.lock().unwrap().decrypt(encryption.clone(), &chunk_data));
chunk_data = try!(crypto.lock().unwrap().decrypt(&encryption, &chunk_data).context(&path as &Path));
}
let chunks = ChunkList::read_from(&chunk_data);
let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize;
@ -234,20 +254,20 @@ impl Bundle {
#[inline]
fn load_encoded_contents(&self) -> Result<Vec<u8>, BundleError> {
let mut file = BufReader::new(try!(File::open(&self.path).map_err(|e| BundleError::Read(e, self.path.clone()))));
try!(file.seek(SeekFrom::Start(self.content_start as u64)).map_err(|e| BundleError::Read(e, self.path.clone())));
let mut file = BufReader::new(try!(File::open(&self.path).context(&self.path as &Path)));
try!(file.seek(SeekFrom::Start(self.content_start as u64)).context(&self.path as &Path));
let mut data = Vec::with_capacity(max(self.info.encoded_size, self.info.raw_size)+1024);
try!(file.read_to_end(&mut data).map_err(|e| BundleError::Read(e, self.path.clone())));
try!(file.read_to_end(&mut data).context(&self.path as &Path));
Ok(data)
}
#[inline]
fn decode_contents(&self, mut data: Vec<u8>) -> Result<Vec<u8>, BundleError> {
if let Some(ref encryption) = self.info.encryption {
data = try!(self.crypto.lock().unwrap().decrypt(encryption.clone(), &data));
data = try!(self.crypto.lock().unwrap().decrypt(&encryption, &data).context(&self.path as &Path));
}
if let Some(ref compression) = self.info.compression {
data = try!(compression.decompress(&data));
data = try!(compression.decompress(&data).context(&self.path as &Path));
}
Ok(data)
}
@ -276,8 +296,7 @@ impl Bundle {
"Individual chunk sizes do not add up to total size"))
}
if !full {
let size = try!(fs::metadata(&self.path).map_err(|e| BundleError::Read(e, self.path.clone()))
).len();
let size = try!(fs::metadata(&self.path).context(&self.path as &Path)).len();
if size as usize != self.info.encoded_size + self.content_start {
return Err(BundleError::Integrity(self.id(),
"File size does not match size in header, truncated file"))
@ -365,21 +384,21 @@ impl BundleWriter {
try!(stream.finish(&mut self.data))
}
if let Some(ref encryption) = self.encryption {
self.data = try!(self.crypto.lock().unwrap().encrypt(encryption.clone(), &self.data));
self.data = try!(self.crypto.lock().unwrap().encrypt(&encryption, &self.data));
}
let encoded_size = self.data.len();
let mut chunk_data = Vec::with_capacity(self.chunks.encoded_size());
self.chunks.write_to(&mut chunk_data).unwrap();
let id = BundleId(self.hash_method.hash(&chunk_data));
if let Some(ref encryption) = self.encryption {
chunk_data = try!(self.crypto.lock().unwrap().encrypt(encryption.clone(), &chunk_data));
chunk_data = try!(self.crypto.lock().unwrap().encrypt(&encryption, &chunk_data));
}
let (folder, file) = db.bundle_path(&id);
let path = folder.join(file);
try!(fs::create_dir_all(&folder).map_err(|e| BundleError::Write(e, path.clone())));
let mut file = BufWriter::new(try!(File::create(&path).map_err(|e| BundleError::Write(e, path.clone()))));
try!(file.write_all(&HEADER_STRING).map_err(|e| BundleError::Write(e, path.clone())));
try!(file.write_all(&[HEADER_VERSION]).map_err(|e| BundleError::Write(e, path.clone())));
try!(fs::create_dir_all(&folder).context(&path as &Path));
let mut file = BufWriter::new(try!(File::create(&path).context(&path as &Path)));
try!(file.write_all(&HEADER_STRING).context(&path as &Path));
try!(file.write_all(&[HEADER_VERSION]).context(&path as &Path));
let header = BundleInfo {
mode: self.mode,
hash_method: self.hash_method,
@ -393,9 +412,9 @@ impl BundleWriter {
};
try!(msgpack::encode_to_stream(&header, &mut file)
.map_err(|e| BundleError::Encode(e, path.clone())));
try!(file.write_all(&chunk_data).map_err(|e| BundleError::Write(e, path.clone())));
try!(file.write_all(&chunk_data).context(&path as &Path));
let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize;
try!(file.write_all(&self.data).map_err(|e| BundleError::Write(e, path.clone())));
try!(file.write_all(&self.data).context(&path as &Path));
Ok(Bundle::new(path, HEADER_VERSION, content_start, self.crypto, header, self.chunks))
}
@ -413,8 +432,6 @@ impl BundleWriter {
pub struct BundleDb {
path: PathBuf,
compression: Option<Compression>,
encryption: Option<Encryption>,
crypto: Arc<Mutex<Crypto>>,
bundles: HashMap<BundleId, Bundle>,
bundle_cache: LruCache<BundleId, Vec<u8>>
@ -422,13 +439,10 @@ pub struct BundleDb {
impl BundleDb {
fn new(path: PathBuf, compression: Option<Compression>, encryption: Option<Encryption>) -> Self {
fn new(path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Self {
BundleDb {
path: path,
compression:
compression,
crypto: Arc::new(Mutex::new(Crypto::new())),
encryption: encryption,
crypto: crypto,
bundles: HashMap::new(),
bundle_cache: LruCache::new(5, 10)
}
@ -436,7 +450,7 @@ impl BundleDb {
fn bundle_path(&self, bundle: &BundleId) -> (PathBuf, PathBuf) {
let mut folder = self.path.clone();
let mut file = bundle.to_string()[0..32].to_owned() + ".bundle";
let mut file = bundle.to_string().to_owned() + ".bundle";
let mut count = self.bundles.len();
while count >= 100 {
if file.len() < 10 {
@ -469,33 +483,29 @@ impl BundleDb {
}
#[inline]
pub fn open<P: AsRef<Path>>(path: P, compression: Option<Compression>, encryption: Option<Encryption>) -> Result<Self, BundleError> {
pub fn open<P: AsRef<Path>>(path: P, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
let path = path.as_ref().to_owned();
let mut self_ = Self::new(path, compression, encryption);
let mut self_ = Self::new(path, crypto);
try!(self_.load_bundle_list());
Ok(self_)
}
#[inline]
pub fn create<P: AsRef<Path>>(path: P, compression: Option<Compression>, encryption: Option<Encryption>) -> Result<Self, BundleError> {
pub fn create<P: AsRef<Path>>(path: P, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
let path = path.as_ref().to_owned();
try!(fs::create_dir_all(&path)
.map_err(|e| BundleError::Write(e, path.clone())));
Ok(Self::new(path, compression, encryption))
try!(fs::create_dir_all(&path).context(&path as &Path));
Ok(Self::new(path, crypto))
}
#[inline]
pub fn open_or_create<P: AsRef<Path>>(path: P, compression: Option<Compression>, encryption: Option<Encryption>) -> Result<Self, BundleError> {
if path.as_ref().exists() {
Self::open(path, compression, encryption)
} else {
Self::create(path, compression, encryption)
}
}
#[inline]
pub fn create_bundle(&self, mode: BundleMode, hash_method: HashMethod) -> Result<BundleWriter, BundleError> {
BundleWriter::new(mode, hash_method, self.compression.clone(), self.encryption.clone(), self.crypto.clone())
pub fn create_bundle(
&self,
mode: BundleMode,
hash_method: HashMethod,
compression: Option<Compression>,
encryption: Option<Encryption>
) -> Result<BundleWriter, BundleError> {
BundleWriter::new(mode, hash_method, compression, encryption, self.crypto.clone())
}
pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleError> {

7
src/chunker/mod.rs

@ -77,7 +77,7 @@ impl IChunker for Chunker {
}
#[derive(Debug)]
#[derive(Debug, Clone, Copy)]
pub enum ChunkerType {
Ae(usize),
Rabin((usize, u32)),
@ -141,6 +141,11 @@ impl ChunkerType {
}
}
#[inline]
pub fn to_string(&self) -> String {
format!("{}/{}", self.name(), self.avg_size()/1024)
}
#[inline]
pub fn seed(&self) -> u64 {
match *self {

68
src/cli/algotest.rs

@ -43,8 +43,9 @@ fn chunk(data: &[u8], mut chunker: Chunker, sink: &mut ChunkSink) {
}
#[allow(dead_code)]
pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Option<Compression>, hash: HashMethod) {
let mut total_time = 0.0;
pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Option<Compression>, encrypt: bool,hash: HashMethod) {
let mut total_write_time = 0.0;
let mut total_read_time = 0.0;
println!("Reading input file ...");
let mut file = File::open(path).unwrap();
@ -68,7 +69,7 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
let chunk_time = Duration::span(|| {
chunk(&data, chunker, &mut chunk_sink)
}).num_milliseconds() as f32 / 1_000.0;
total_time += chunk_time;
total_write_time += chunk_time;
println!("- {}, {}", to_duration(chunk_time), to_speed(size, chunk_time));
let mut chunks = chunk_sink.chunks;
assert_eq!(chunks.iter().map(|c| c.1).sum::<usize>(), size as usize);
@ -85,7 +86,7 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
hashes.push(hash.hash(&data[pos..pos+len]))
}
}).num_milliseconds() as f32 / 1_000.0;
total_time += hash_time;
total_write_time += hash_time;
println!("- {}, {}", to_duration(hash_time), to_speed(size, hash_time));
let mut seen_hashes = HashSet::with_capacity(hashes.len());
let mut dups = Vec::new();
@ -103,11 +104,12 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
println!("- {} duplicate chunks, {}, {:.1}% saved", dups.len(), to_file_size(dup_size as u64), dup_size as f32 / size as f32*100.0);
size -= dup_size as u64;
if let Some(compression) = compression {
let mut bundles = Vec::new();
if let Some(compression) = compression.clone() {
println!();
println!("Compressing chunks with {} ...", compression.to_string());
let mut bundles = Vec::new();
let compress_time = Duration::span(|| {
let mut bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize);
let mut c = compression.compress_stream().unwrap();
@ -123,12 +125,56 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
c.finish(&mut bundle).unwrap();
bundles.push(bundle);
}).num_milliseconds() as f32 / 1_000.0;
total_time += compress_time;
total_write_time += compress_time;
println!("- {}, {}", to_duration(compress_time), to_speed(size, compress_time));
let compressed_size = bundles.iter().map(|b| b.len()).sum::<usize>();
println!("- {} bundles, {}, {:.1}% saved", bundles.len(), to_file_size(compressed_size as u64), (size as f32 - compressed_size as f32)/size as f32*100.0);
size = compressed_size as u64;
} else {
let mut bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize);
for &(pos, len) in &chunks {
bundle.extend_from_slice(&data[pos..pos+len]);
if bundle.len() >= bundle_size {
bundles.push(bundle);
bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize);
}
}
bundles.push(bundle);
}
if encrypt {
println!();
let (public, secret) = gen_keypair();
let mut crypto = Crypto::new();
crypto.add_secret_key(public, secret);
let encryption = (EncryptionMethod::Sodium, public[..].iter().cloned().collect::<Vec<u8>>().into());
println!("Encrypting bundles...");
let mut encrypted_bundles = Vec::with_capacity(bundles.len());
let encrypt_time = Duration::span(|| {
for bundle in bundles {
encrypted_bundles.push(crypto.encrypt(&encryption, &bundle).unwrap());
}
}).num_milliseconds() as f32 / 1_000.0;
println!("- {}, {}", to_duration(encrypt_time), to_speed(size, encrypt_time));
total_write_time += encrypt_time;
println!();
println!("Decrypting bundles...");
bundles = Vec::with_capacity(encrypted_bundles.len());
let decrypt_time = Duration::span(|| {
for bundle in encrypted_bundles {
bundles.push(crypto.decrypt(&encryption, &bundle).unwrap());
}
}).num_milliseconds() as f32 / 1_000.0;
println!("- {}, {}", to_duration(decrypt_time), to_speed(size, decrypt_time));
total_read_time += decrypt_time;
}
if let Some(compression) = compression {
println!();
println!("Decompressing bundles with {} ...", compression.to_string());
@ -141,12 +187,12 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
}
}).num_milliseconds() as f32 / 1_000.0;
println!("- {}, {}", to_duration(decompress_time), to_speed(size, decompress_time));
total_read_time += decompress_time;
}
println!();
let total_saved = total_size - size;
println!("Total space saved: {}, {:.1}%", to_file_size(total_saved as u64), total_saved as f32/total_size as f32*100.0);
println!("Total processing speed: {}", to_speed(total_size, total_time));
println!("Total storage size: {} / {}, ratio: {:.1}%", to_file_size(size as u64), to_file_size(total_size as u64), size as f32/total_size as f32*100.0);
println!("Total processing speed: {}", to_speed(total_size, total_write_time));
println!("Total read speed: {}", to_speed(total_size, total_read_time));
}

162
src/cli/args.rs

@ -1,5 +1,5 @@
use ::chunker::ChunkerType;
use ::util::{Compression, HashMethod};
use ::util::*;
use std::process::exit;
@ -10,6 +10,7 @@ pub enum Arguments {
bundle_size: usize,
chunker: ChunkerType,
compression: Option<Compression>,
encryption: bool,
hash: HashMethod
},
Backup {
@ -56,11 +57,27 @@ pub enum Arguments {
repo_path: String,
remote_path: String
},
Configure {
repo_path: String,
bundle_size: Option<usize>,
chunker: Option<ChunkerType>,
compression: Option<Option<Compression>>,
encryption: Option<Option<PublicKey>>,
hash: Option<HashMethod>
},
GenKey {
},
AddKey {
repo_path: String,
key_pair: Option<(PublicKey, SecretKey)>,
set_default: bool
},
AlgoTest {
file: String,
bundle_size: usize,
chunker: ChunkerType,
compression: Option<Compression>,
encrypt: bool,
hash: HashMethod
}
}
@ -93,17 +110,16 @@ fn parse_float(num: &str, name: &str) -> f64 {
}
fn parse_chunker(val: Option<&str>) -> ChunkerType {
if let Ok(chunker) = ChunkerType::from_string(val.unwrap_or("fastcdc/8")) {
fn parse_chunker(val: &str) -> ChunkerType {
if let Ok(chunker) = ChunkerType::from_string(val) {
chunker
} else {
error!("Invalid chunker method/size: {}", val.unwrap());
error!("Invalid chunker method/size: {}", val);
exit(1);
}
}
fn parse_compression(val: Option<&str>) -> Option<Compression> {
let val = val.unwrap_or("brotli/3");
fn parse_compression(val: &str) -> Option<Compression> {
if val == "none" {
return None
}
@ -115,11 +131,43 @@ fn parse_compression(val: Option<&str>) -> Option<Compression> {
}
}
fn parse_hash(val: Option<&str>) -> HashMethod {
if let Ok(hash) = HashMethod::from(val.unwrap_or("blake2")) {
fn parse_public_key(val: &str) -> PublicKey {
let bytes = match parse_hex(val) {
Ok(bytes) => bytes,
Err(_) => {
error!("Invalid key: {}", val);
exit(1);
}
};
if let Some(key) = PublicKey::from_slice(&bytes) {
key
} else {
error!("Invalid key: {}", val);
exit(1);
}
}
fn parse_secret_key(val: &str) -> SecretKey {
let bytes = match parse_hex(val) {
Ok(bytes) => bytes,
Err(_) => {
error!("Invalid key: {}", val);
exit(1);
}
};
if let Some(key) = SecretKey::from_slice(&bytes) {
key
} else {
error!("Invalid key: {}", val);
exit(1);
}
}
fn parse_hash(val: &str) -> HashMethod {
if let Ok(hash) = HashMethod::from(val) {
hash
} else {
error!("Invalid hash method: {}", val.unwrap());
error!("Invalid hash method: {}", val);
exit(1);
}
}
@ -127,18 +175,19 @@ fn parse_hash(val: Option<&str>) -> HashMethod {
pub fn parse() -> Arguments {
let args = clap_app!(zvault =>
(version: env!("CARGO_PKG_VERSION"))
(author: "Dennis Schwerdel <schwerdel@googlemail.com>")
(about: "Deduplicating backup tool")
(version: crate_version!())
(author: crate_authors!(",\n"))
(about: crate_description!())
(@setting SubcommandRequiredElseHelp)
(@setting GlobalVersion)
(@setting VersionlessSubcommands)
(@setting UnifiedHelpMessage)
(@subcommand init =>
(about: "initializes a new repository")
(@arg bundle_size: --bundle-size +takes_value "maximal bundle size in MiB [default: 25]")
(@arg bundle_size: --bundlesize +takes_value "maximal bundle size in MiB [default: 25]")
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]")
(@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]")
(@arg encryption: --encryption -e "generate a keypair and enable encryption")
(@arg hash: --hash +takes_value "hash method to use [default: blake2]")
(@arg REPO: +required "path of the repository")
)
@ -184,11 +233,32 @@ pub fn parse() -> Arguments {
(about: "displays information on a repository, a backup or a path in a backup")
(@arg PATH: +required "repository[::backup[::subpath]] path")
)
(@subcommand configure =>
(about: "changes the configuration")
(@arg REPO: +required "path of the repository")
(@arg bundle_size: --bundlesize +takes_value "maximal bundle size in MiB [default: 25]")
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]")
(@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]")
(@arg encryption: --encryption -e +takes_value "the public key to use for encryption")
(@arg hash: --hash +takes_value "hash method to use [default: blake2]")
)
(@subcommand genkey =>
(about: "generates a new key pair")
)
(@subcommand addkey =>
(about: "adds a key to the respository")
(@arg REPO: +required "path of the repository")
(@arg generate: --generate "generate a new key")
(@arg set_default: --default "set this key as default")
(@arg PUBLIC: +takes_value "the public key")
(@arg SECRET: +takes_value "the secret key")
)
(@subcommand algotest =>
(about: "test a specific algorithm combination")
(@arg bundle_size: --bundle-size +takes_value "maximal bundle size in MiB [default: 25]")
(@arg bundle_size: --bundlesize +takes_value "maximal bundle size in MiB [default: 25]")
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]")
(@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]")
(@arg encrypt: --encrypt -e "enable encryption")
(@arg hash: --hash +takes_value "hash method to use [default: blake2]")
(@arg FILE: +required "the file to test the algorithms with")
)
@ -201,9 +271,10 @@ pub fn parse() -> Arguments {
}
return Arguments::Init {
bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize,
chunker: parse_chunker(args.value_of("chunker")),
compression: parse_compression(args.value_of("compression")),
hash: parse_hash(args.value_of("hash")),
chunker: parse_chunker(args.value_of("chunker").unwrap_or("fastcdc/8")),
compression: parse_compression(args.value_of("compression").unwrap_or("brotli/3")),
encryption: args.is_present("encryption"),
hash: parse_hash(args.value_of("hash").unwrap_or("blake2")),
repo_path: repository.to_string(),
}
}
@ -306,12 +377,63 @@ pub fn parse() -> Arguments {
remote_path: args.value_of("REMOTE").unwrap().to_string()
}
}
if let Some(args) = args.subcommand_matches("configure") {
let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap());
if backup.is_some() || inode.is_some() {
println!("No backups or subpaths may be given here");
exit(1);
}
return Arguments::Configure {
bundle_size: args.value_of("bundle_size").map(|v| (parse_num(v, "Bundle size") * 1024 * 1024) as usize),
chunker: args.value_of("chunker").map(|v| parse_chunker(v)),
compression: args.value_of("compression").map(|v| parse_compression(v)),
encryption: args.value_of("encryption").map(|v| {
if v == "none" {
None
} else {
Some(parse_public_key(v))
}
}),
hash: args.value_of("hash").map(|v| parse_hash(v)),
repo_path: repository.to_string(),
}
}
if let Some(_args) = args.subcommand_matches("genkey") {
return Arguments::GenKey {}
}
if let Some(args) = args.subcommand_matches("addkey") {
let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap());
if backup.is_some() || inode.is_some() {
println!("No backups or subpaths may be given here");
exit(1);
}
let generate = args.is_present("generate");
if !generate && (!args.is_present("PUBLIC") || !args.is_present("SECRET")) {
println!("Without --generate, a public and secret key must be given");
exit(1);
}
if generate && (args.is_present("PUBLIC") || args.is_present("SECRET")) {
println!("With --generate, no public or secret key may be given");
exit(1);
}
let key_pair = if generate {
None
} else {
Some((parse_public_key(args.value_of("PUBLIC").unwrap()), parse_secret_key(args.value_of("SECRET").unwrap())))
};
return Arguments::AddKey {
repo_path: repository.to_string(),
set_default: args.is_present("set_default"),
key_pair: key_pair
}
}
if let Some(args) = args.subcommand_matches("algotest") {
return Arguments::AlgoTest {
bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize,
chunker: parse_chunker(args.value_of("chunker")),
compression: parse_compression(args.value_of("compression")),
hash: parse_hash(args.value_of("hash")),
chunker: parse_chunker(args.value_of("chunker").unwrap_or("fastcdc/8")),
compression: parse_compression(args.value_of("compression").unwrap_or("brotli/3")),
encrypt: args.is_present("encrypt"),
hash: parse_hash(args.value_of("hash").unwrap_or("blake2")),
file: args.value_of("FILE").unwrap().to_string(),
}
}

12
src/cli/logger.rs

@ -1,6 +1,9 @@
use log::{self, LogRecord, LogLevel, LogMetadata, LogLevelFilter};
pub use log::SetLoggerError;
use ansi_term::{Color, Style};
struct Logger;
impl log::Log for Logger {
@ -10,7 +13,14 @@ impl log::Log for Logger {
fn log(&self, record: &LogRecord) {
if self.enabled(record.metadata()) {
println!("{} - {}", record.level(), record.args());
let lvl = record.level();
match lvl {
LogLevel::Error => println!("{} - {}", Color::Red.bold().paint("error"), record.args()),
LogLevel::Warn => println!("{} - {}", Color::Yellow.bold().paint("warning"), record.args()),
LogLevel::Info => println!("{} - {}", Color::Green.bold().paint("info"), record.args()),
LogLevel::Debug => println!("{} - {}", Style::new().bold().paint("debug"), record.args()),
LogLevel::Trace => println!("{} - {}", "trace", record.args())
}
}
}
}

73
src/cli/mod.rs

@ -7,6 +7,7 @@ use std::process::exit;
use ::repository::{Repository, Config, Backup};
use ::util::cli::*;
use ::util::*;
use self::args::Arguments;
@ -36,13 +37,22 @@ pub fn run() {
exit(-1)
}
match args::parse() {
Arguments::Init{repo_path, bundle_size, chunker, compression, hash} => {
Repository::create(repo_path, Config {
Arguments::Init{repo_path, bundle_size, chunker, compression, encryption, hash} => {
let mut repo = Repository::create(repo_path, Config {
bundle_size: bundle_size,
chunker: chunker,
compression: compression,
encryption: None,
hash: hash
}).unwrap();
if encryption {
let (public, secret) = gen_keypair();
println!("Public key: {}", to_hex(&public[..]));
println!("Secret key: {}", to_hex(&secret[..]));
repo.set_encryption(Some(&public));
repo.register_key(public, secret).unwrap();
repo.save_config().unwrap();
}
},
Arguments::Backup{repo_path, backup_name, src_path, full} => {
let mut repo = open_repository(&repo_path);
@ -165,8 +175,63 @@ pub fn run() {
error!("Import is not implemented yet");
return
},
Arguments::AlgoTest{bundle_size, chunker, compression, hash, file} => {
algotest::run(&file, bundle_size, chunker, compression, hash);
Arguments::Configure{repo_path, bundle_size, chunker, compression, encryption, hash} => {
let mut repo = open_repository(&repo_path);
if let Some(bundle_size) = bundle_size {
repo.config.bundle_size = bundle_size
}
if let Some(chunker) = chunker {
warn!("Changing the chunker makes it impossible to use existing data for deduplication");
repo.config.chunker = chunker
}
if let Some(compression) = compression {
repo.config.compression = compression
}
if let Some(encryption) = encryption {
repo.set_encryption(encryption.as_ref())
}
if let Some(hash) = hash {
warn!("Changing the hash makes it impossible to use existing data for deduplication");
repo.config.hash = hash
}
repo.save_config().unwrap();
println!("Bundle size: {}", to_file_size(repo.config.bundle_size as u64));
println!("Chunker: {}", repo.config.chunker.to_string());
if let Some(ref compression) = repo.config.compression {
println!("Compression: {}", compression.to_string());
} else {
println!("Compression: none");
}
if let Some(ref encryption) = repo.config.encryption {
println!("Encryption: {}", to_hex(&encryption.1[..]));
} else {
println!("Encryption: none");
}
println!("Hash method: {}", repo.config.hash.name());
},
Arguments::GenKey{} => {
let (public, secret) = gen_keypair();
println!("Public key: {}", to_hex(&public[..]));
println!("Secret key: {}", to_hex(&secret[..]));
},
Arguments::AddKey{repo_path, set_default, key_pair} => {
let mut repo = open_repository(&repo_path);
let (public, secret) = if let Some(key_pair) = key_pair {
key_pair
} else {
let (public, secret) = gen_keypair();
println!("Public key: {}", to_hex(&public[..]));
println!("Secret key: {}", to_hex(&secret[..]));
(public, secret)
};
if set_default {
repo.set_encryption(Some(&public));
repo.save_config().unwrap();
}
repo.register_key(public, secret).unwrap();
},
Arguments::AlgoTest{bundle_size, chunker, compression, encrypt, hash, file} => {
algotest::run(&file, bundle_size, chunker, compression, encrypt, hash);
}
}
}

5
src/main.rs

@ -12,6 +12,8 @@ extern crate chrono;
#[macro_use] extern crate clap;
#[macro_use] extern crate log;
extern crate byteorder;
extern crate sodiumoxide;
extern crate ansi_term;
pub mod util;
@ -26,12 +28,11 @@ mod cli;
// TODO: - Keep meta bundles also locally
// TODO: - Load and compare remote bundles to bundle map
// TODO: - Write backup files there as well
// TODO: Store list of hashes and hash method in bundle
// TODO: Remove backup subtrees
// TODO: Recompress & combine bundles
// TODO: Prune backups (based on age like attic)
// TODO: Check backup integrity too
// TODO: Encryption
// TODO: Encrypt backup files too
// TODO: list --tree
// TODO: Partial backups
// TODO: Import repository from remote folder

7
src/repository/basic_io.rs

@ -46,7 +46,12 @@ impl Repository {
};
// ...alocate one if needed
if writer.is_none() {
*writer = Some(try!(self.bundles.create_bundle(mode, self.config.hash)));
*writer = Some(try!(self.bundles.create_bundle(
mode,
self.config.hash,
self.config.compression.clone(),
self.config.encryption.clone()
)));
}
debug_assert!(writer.is_some());
let chunk_id;

45
src/repository/config.rs

@ -24,6 +24,7 @@ quick_error!{
from()
cause(err)
description("Yaml format error")
display("Yaml format error: {}", err)
}
}
}
@ -89,9 +90,41 @@ impl Compression {
}
impl EncryptionMethod {
#[inline]
fn from_yaml(yaml: String) -> Result<Self, ConfigError> {
EncryptionMethod::from_string(&yaml).map_err(|_| ConfigError::Parse("Invalid codec"))
}
#[inline]
fn to_yaml(&self) -> String {
self.to_string()
}
}
struct EncryptionYaml {
method: String,
key: String
}
impl Default for EncryptionYaml {
fn default() -> Self {
EncryptionYaml {
method: "sodium".to_string(),
key: "".to_string()
}
}
}
serde_impl!(EncryptionYaml(String) {
method: String => "method",
key: String => "key"
});
struct ConfigYaml {
compression: Option<String>,
encryption: Option<EncryptionYaml>,
bundle_size: usize,
chunker: ChunkerYaml,
hash: String,
@ -100,6 +133,7 @@ impl Default for ConfigYaml {
fn default() -> Self {
ConfigYaml {
compression: Some("brotli/5".to_string()),
encryption: None,
bundle_size: 25*1024*1024,
chunker: ChunkerYaml::default(),
hash: "blake2".to_string()
@ -108,6 +142,7 @@ impl Default for ConfigYaml {
}
serde_impl!(ConfigYaml(String) {
compression: Option<String> => "compression",
encryption: Option<EncryptionYaml> => "encryption",
bundle_size: usize => "bundle_size",
chunker: ChunkerYaml => "chunker",
hash: String => "hash"
@ -118,6 +153,7 @@ serde_impl!(ConfigYaml(String) {
#[derive(Debug)]
pub struct Config {
pub compression: Option<Compression>,
pub encryption: Option<Encryption>,
pub bundle_size: usize,
pub chunker: ChunkerType,
pub hash: HashMethod
@ -129,8 +165,16 @@ impl Config {
} else {
None
};
let encryption = if let Some(e) = yaml.encryption {
let method = try!(EncryptionMethod::from_yaml(e.method));
let key = try!(parse_hex(&e.key).map_err(|_| ConfigError::Parse("Invalid public key")));
Some((method, key.into()))
} else {
None
};
Ok(Config{
compression: compression,
encryption: encryption,
bundle_size: yaml.bundle_size,
chunker: try!(ChunkerType::from_yaml(yaml.chunker)),
hash: try!(HashMethod::from_yaml(yaml.hash))
@ -140,6 +184,7 @@ impl Config {
fn to_yaml(&self) -> ConfigYaml {
ConfigYaml {
compression: self.compression.as_ref().map(|c| c.to_yaml()),
encryption: self.encryption.as_ref().map(|e| EncryptionYaml{method: e.0.to_yaml(), key: to_hex(&e.1[..])}),
bundle_size: self.bundle_size,
chunker: self.chunker.to_yaml(),
hash: self.hash.to_yaml()

17
src/repository/error.rs

@ -17,47 +17,62 @@ quick_error!{
Io(err: io::Error) {
from()
cause(err)
description("IO Error")
description("IO error")
display("IO error: {}", err)
}
Config(err: ConfigError) {
from()
cause(err)
description("Configuration error")
display("Configuration error: {}", err)
}
BundleMap(err: BundleMapError) {
from()
cause(err)
description("Bundle map error")
display("Bundle map error: {}", err)
}
Index(err: IndexError) {
from()
cause(err)
description("Index error")
display("Index error: {}", err)
}
Bundle(err: BundleError) {
from()
cause(err)
description("Bundle error")
display("Bundle error: {}", err)
}
Chunker(err: ChunkerError) {
from()
cause(err)
description("Chunker error")
display("Chunker error: {}", err)
}
Decode(err: msgpack::DecodeError) {
from()
cause(err)
description("Failed to decode metadata")
display("Failed to decode metadata: {}", err)
}
Encode(err: msgpack::EncodeError) {
from()
cause(err)
description("Failed to encode metadata")
display("Failed to encode metadata: {}", err)
}
Integrity(err: RepositoryIntegrityError) {
from()
cause(err)
description("Integrity error")
display("Integrity error: {}", err)
}
Encryption(err: EncryptionError) {
from()
cause(err)
description("Failed to load keys")
display("Failed to load keys: {}", err)
}
InvalidFileType(path: PathBuf) {
description("Invalid file type")

37
src/repository/mod.rs

@ -11,10 +11,12 @@ use std::mem;
use std::cmp::max;
use std::path::{PathBuf, Path};
use std::fs;
use std::sync::{Arc, Mutex};
use super::index::Index;
use super::bundle::{BundleDb, BundleWriter};
use super::chunker::Chunker;
use ::util::*;
pub use self::error::RepositoryError;
pub use self::config::Config;
@ -25,8 +27,9 @@ use self::bundle_map::BundleMap;
pub struct Repository {
path: PathBuf,
config: Config,
pub config: Config,
index: Index,
crypto: Arc<Mutex<Crypto>>,
bundle_map: BundleMap,
next_content_bundle: u32,
next_meta_bundle: u32,
@ -41,10 +44,11 @@ impl Repository {
pub fn create<P: AsRef<Path>>(path: P, config: Config) -> Result<Self, RepositoryError> {
let path = path.as_ref().to_owned();
try!(fs::create_dir(&path));
try!(fs::create_dir(path.join("keys")));
let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys")))));
let bundles = try!(BundleDb::create(
path.join("bundles"),
config.compression.clone(),
None, //FIXME: store encryption in config
crypto.clone()
));
let index = try!(Index::create(&path.join("index")));
try!(config.save(path.join("config.yaml")));
@ -62,16 +66,17 @@ impl Repository {
bundles: b