Bundle encryption

pull/10/head
Dennis Schwerdel 2017-03-18 17:22:11 +01:00 committed by Dennis Schwerdel
parent 0a807b16ab
commit c8b69ebe25
16 changed files with 646 additions and 123 deletions

24
Cargo.lock generated
View File

@ -2,6 +2,7 @@
name = "zvault" name = "zvault"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", "blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -9,12 +10,14 @@ dependencies = [
"log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rmp-serde 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", "rmp-serde 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
"sodiumoxide 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
"squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
@ -104,6 +107,15 @@ name = "libc"
version = "0.2.21" version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "libsodium-sys"
version = "0.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "linked-hash-map" name = "linked-hash-map"
version = "0.3.0" version = "0.3.0"
@ -230,6 +242,16 @@ dependencies = [
"yaml-rust 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "yaml-rust 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "sodiumoxide"
version = "0.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"libsodium-sys 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "squash-sys" name = "squash-sys"
version = "0.9.0" version = "0.9.0"
@ -320,6 +342,7 @@ dependencies = [
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
"checksum libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122" "checksum libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122"
"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135" "checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135"
"checksum libsodium-sys 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "cbbc6e46017815abf8698de0ed4847fad45fd8cad2909ac38ac6de79673c1ad1"
"checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" "checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd"
"checksum log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "5141eca02775a762cc6cd564d8d2c50f67c0ea3a372cbf1c51592b3e029e10ad" "checksum log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "5141eca02775a762cc6cd564d8d2c50f67c0ea3a372cbf1c51592b3e029e10ad"
"checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823" "checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823"
@ -338,6 +361,7 @@ dependencies = [
"checksum serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)" = "a702319c807c016e51f672e5c77d6f0b46afddd744b5e437d6b8436b888b458f" "checksum serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)" = "a702319c807c016e51f672e5c77d6f0b46afddd744b5e437d6b8436b888b458f"
"checksum serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b34a52969c7fc0254e214b82518c9a95dc88c84fc84cd847add314996a031be6" "checksum serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b34a52969c7fc0254e214b82518c9a95dc88c84fc84cd847add314996a031be6"
"checksum serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f8bd3f24ad8c7bcd34a6d70ba676dc11302b96f4f166aa5f947762e01098844d" "checksum serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f8bd3f24ad8c7bcd34a6d70ba676dc11302b96f4f166aa5f947762e01098844d"
"checksum sodiumoxide 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "bc02c0bc77ffed8e8eaef004399b825cf4fd8aa02d0af6e473225affd583ff4d"
"checksum squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db1f9dde91d819b7746e153bc32489fa19e6a106c3d7f2b92187a4efbdc88b40" "checksum squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db1f9dde91d819b7746e153bc32489fa19e6a106c3d7f2b92187a4efbdc88b40"
"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694"
"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" "checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"

View File

@ -2,6 +2,7 @@
name = "zvault" name = "zvault"
version = "0.1.0" version = "0.1.0"
authors = ["Dennis Schwerdel <schwerdel@informatik.uni-kl.de>"] authors = ["Dennis Schwerdel <schwerdel@informatik.uni-kl.de>"]
description = "Deduplicating backup tool"
[dependencies] [dependencies]
serde = "0.9" serde = "0.9"
@ -18,3 +19,8 @@ chrono = "0.3"
clap = "2.19" clap = "2.19"
log = "0.3" log = "0.3"
byteorder = "1.0" byteorder = "1.0"
ansi_term = "0.9"
sodiumoxide = "*"
[build-dependencies]
pkg-config = "0.3"

View File

@ -7,6 +7,7 @@ use std::fmt::{self, Debug};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use serde::{self, Serialize, Deserialize}; use serde::{self, Serialize, Deserialize};
use quick_error::ResultExt;
use util::*; use util::*;
@ -18,7 +19,7 @@ static HEADER_VERSION: u8 = 1;
Bundle format Bundle format
- Magic header + version - Magic header + version
- Encoded header structure (contains size of next structure) - Encoded header structure (contains size of next structure)
- Encoded contents structure (with chunk sizes and hashes) - Encoded chunk list (with chunk hashes and sizes)
- Chunk data - Chunk data
*/ */
@ -31,22 +32,25 @@ quick_error!{
List(err: io::Error) { List(err: io::Error) {
cause(err) cause(err)
description("Failed to list bundles") description("Failed to list bundles")
display("Failed to list bundles: {}", err)
} }
Read(err: io::Error, path: PathBuf) { Io(err: io::Error, path: PathBuf) {
cause(err) cause(err)
context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf())
description("Failed to read bundle") description("Failed to read bundle")
display("Failed to read bundle {:?}: {}", path, err)
} }
Decode(err: msgpack::DecodeError, path: PathBuf) { Decode(err: msgpack::DecodeError, path: PathBuf) {
cause(err) cause(err)
context(path: &'a Path, err: msgpack::DecodeError) -> (err, path.to_path_buf())
description("Failed to decode bundle header") description("Failed to decode bundle header")
} display("Failed to decode bundle header of {:?}: {}", path, err)
Write(err: io::Error, path: PathBuf) {
cause(err)
description("Failed to write bundle")
} }
Encode(err: msgpack::EncodeError, path: PathBuf) { Encode(err: msgpack::EncodeError, path: PathBuf) {
cause(err) cause(err)
context(path: &'a Path, err: msgpack::EncodeError) -> (err, path.to_path_buf())
description("Failed to encode bundle header") description("Failed to encode bundle header")
display("Failed to encode bundle header of {:?}: {}", path, err)
} }
WrongHeader(path: PathBuf) { WrongHeader(path: PathBuf) {
description("Wrong header") description("Wrong header")
@ -68,13 +72,29 @@ quick_error!{
description("Bundle has no such chunk") description("Bundle has no such chunk")
display("Bundle {:?} has no chunk with that id: {}", bundle, id) display("Bundle {:?} has no chunk with that id: {}", bundle, id)
} }
Decompression(err: CompressionError, path: PathBuf) {
cause(err)
context(path: &'a Path, err: CompressionError) -> (err, path.to_path_buf())
description("Decompression failed")
display("Decompression failed on bundle {:?}: {}", path, err)
}
Compression(err: CompressionError) { Compression(err: CompressionError) {
from() from()
cause(err) cause(err)
description("Compression failed")
display("Compression failed: {}", err)
}
Decryption(err: EncryptionError, path: PathBuf) {
cause(err)
context(path: &'a Path, err: EncryptionError) -> (err, path.to_path_buf())
description("Decryption failed")
display("Decryption failed on bundle {:?}: {}", path, err)
} }
Encryption(err: EncryptionError) { Encryption(err: EncryptionError) {
from() from()
cause(err) cause(err)
description("Encryption failed")
display("Encryption failed: {}", err)
} }
Remove(err: io::Error, bundle: BundleId) { Remove(err: io::Error, bundle: BundleId) {
cause(err) cause(err)
@ -209,9 +229,9 @@ impl Bundle {
} }
pub fn load(path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> { pub fn load(path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
let mut file = BufReader::new(try!(File::open(&path).map_err(|e| BundleError::Read(e, path.clone())))); let mut file = BufReader::new(try!(File::open(&path).context(&path as &Path)));
let mut header = [0u8; 8]; let mut header = [0u8; 8];
try!(file.read_exact(&mut header).map_err(|e| BundleError::Read(e, path.clone()))); try!(file.read_exact(&mut header).context(&path as &Path));
if header[..HEADER_STRING.len()] != HEADER_STRING { if header[..HEADER_STRING.len()] != HEADER_STRING {
return Err(BundleError::WrongHeader(path.clone())) return Err(BundleError::WrongHeader(path.clone()))
} }
@ -223,9 +243,9 @@ impl Bundle {
.map_err(|e| BundleError::Decode(e, path.clone()))); .map_err(|e| BundleError::Decode(e, path.clone())));
let mut chunk_data = Vec::with_capacity(header.chunk_info_size); let mut chunk_data = Vec::with_capacity(header.chunk_info_size);
chunk_data.resize(header.chunk_info_size, 0); chunk_data.resize(header.chunk_info_size, 0);
try!(file.read_exact(&mut chunk_data).map_err(|e| BundleError::Read(e, path.clone()))); try!(file.read_exact(&mut chunk_data).context(&path as &Path));
if let Some(ref encryption) = header.encryption { if let Some(ref encryption) = header.encryption {
chunk_data = try!(crypto.lock().unwrap().decrypt(encryption.clone(), &chunk_data)); chunk_data = try!(crypto.lock().unwrap().decrypt(&encryption, &chunk_data).context(&path as &Path));
} }
let chunks = ChunkList::read_from(&chunk_data); let chunks = ChunkList::read_from(&chunk_data);
let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize; let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize;
@ -234,20 +254,20 @@ impl Bundle {
#[inline] #[inline]
fn load_encoded_contents(&self) -> Result<Vec<u8>, BundleError> { fn load_encoded_contents(&self) -> Result<Vec<u8>, BundleError> {
let mut file = BufReader::new(try!(File::open(&self.path).map_err(|e| BundleError::Read(e, self.path.clone())))); let mut file = BufReader::new(try!(File::open(&self.path).context(&self.path as &Path)));
try!(file.seek(SeekFrom::Start(self.content_start as u64)).map_err(|e| BundleError::Read(e, self.path.clone()))); try!(file.seek(SeekFrom::Start(self.content_start as u64)).context(&self.path as &Path));
let mut data = Vec::with_capacity(max(self.info.encoded_size, self.info.raw_size)+1024); let mut data = Vec::with_capacity(max(self.info.encoded_size, self.info.raw_size)+1024);
try!(file.read_to_end(&mut data).map_err(|e| BundleError::Read(e, self.path.clone()))); try!(file.read_to_end(&mut data).context(&self.path as &Path));
Ok(data) Ok(data)
} }
#[inline] #[inline]
fn decode_contents(&self, mut data: Vec<u8>) -> Result<Vec<u8>, BundleError> { fn decode_contents(&self, mut data: Vec<u8>) -> Result<Vec<u8>, BundleError> {
if let Some(ref encryption) = self.info.encryption { if let Some(ref encryption) = self.info.encryption {
data = try!(self.crypto.lock().unwrap().decrypt(encryption.clone(), &data)); data = try!(self.crypto.lock().unwrap().decrypt(&encryption, &data).context(&self.path as &Path));
} }
if let Some(ref compression) = self.info.compression { if let Some(ref compression) = self.info.compression {
data = try!(compression.decompress(&data)); data = try!(compression.decompress(&data).context(&self.path as &Path));
} }
Ok(data) Ok(data)
} }
@ -276,8 +296,7 @@ impl Bundle {
"Individual chunk sizes do not add up to total size")) "Individual chunk sizes do not add up to total size"))
} }
if !full { if !full {
let size = try!(fs::metadata(&self.path).map_err(|e| BundleError::Read(e, self.path.clone())) let size = try!(fs::metadata(&self.path).context(&self.path as &Path)).len();
).len();
if size as usize != self.info.encoded_size + self.content_start { if size as usize != self.info.encoded_size + self.content_start {
return Err(BundleError::Integrity(self.id(), return Err(BundleError::Integrity(self.id(),
"File size does not match size in header, truncated file")) "File size does not match size in header, truncated file"))
@ -365,21 +384,21 @@ impl BundleWriter {
try!(stream.finish(&mut self.data)) try!(stream.finish(&mut self.data))
} }
if let Some(ref encryption) = self.encryption { if let Some(ref encryption) = self.encryption {
self.data = try!(self.crypto.lock().unwrap().encrypt(encryption.clone(), &self.data)); self.data = try!(self.crypto.lock().unwrap().encrypt(&encryption, &self.data));
} }
let encoded_size = self.data.len(); let encoded_size = self.data.len();
let mut chunk_data = Vec::with_capacity(self.chunks.encoded_size()); let mut chunk_data = Vec::with_capacity(self.chunks.encoded_size());
self.chunks.write_to(&mut chunk_data).unwrap(); self.chunks.write_to(&mut chunk_data).unwrap();
let id = BundleId(self.hash_method.hash(&chunk_data)); let id = BundleId(self.hash_method.hash(&chunk_data));
if let Some(ref encryption) = self.encryption { if let Some(ref encryption) = self.encryption {
chunk_data = try!(self.crypto.lock().unwrap().encrypt(encryption.clone(), &chunk_data)); chunk_data = try!(self.crypto.lock().unwrap().encrypt(&encryption, &chunk_data));
} }
let (folder, file) = db.bundle_path(&id); let (folder, file) = db.bundle_path(&id);
let path = folder.join(file); let path = folder.join(file);
try!(fs::create_dir_all(&folder).map_err(|e| BundleError::Write(e, path.clone()))); try!(fs::create_dir_all(&folder).context(&path as &Path));
let mut file = BufWriter::new(try!(File::create(&path).map_err(|e| BundleError::Write(e, path.clone())))); let mut file = BufWriter::new(try!(File::create(&path).context(&path as &Path)));
try!(file.write_all(&HEADER_STRING).map_err(|e| BundleError::Write(e, path.clone()))); try!(file.write_all(&HEADER_STRING).context(&path as &Path));
try!(file.write_all(&[HEADER_VERSION]).map_err(|e| BundleError::Write(e, path.clone()))); try!(file.write_all(&[HEADER_VERSION]).context(&path as &Path));
let header = BundleInfo { let header = BundleInfo {
mode: self.mode, mode: self.mode,
hash_method: self.hash_method, hash_method: self.hash_method,
@ -393,9 +412,9 @@ impl BundleWriter {
}; };
try!(msgpack::encode_to_stream(&header, &mut file) try!(msgpack::encode_to_stream(&header, &mut file)
.map_err(|e| BundleError::Encode(e, path.clone()))); .map_err(|e| BundleError::Encode(e, path.clone())));
try!(file.write_all(&chunk_data).map_err(|e| BundleError::Write(e, path.clone()))); try!(file.write_all(&chunk_data).context(&path as &Path));
let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize; let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize;
try!(file.write_all(&self.data).map_err(|e| BundleError::Write(e, path.clone()))); try!(file.write_all(&self.data).context(&path as &Path));
Ok(Bundle::new(path, HEADER_VERSION, content_start, self.crypto, header, self.chunks)) Ok(Bundle::new(path, HEADER_VERSION, content_start, self.crypto, header, self.chunks))
} }
@ -413,8 +432,6 @@ impl BundleWriter {
pub struct BundleDb { pub struct BundleDb {
path: PathBuf, path: PathBuf,
compression: Option<Compression>,
encryption: Option<Encryption>,
crypto: Arc<Mutex<Crypto>>, crypto: Arc<Mutex<Crypto>>,
bundles: HashMap<BundleId, Bundle>, bundles: HashMap<BundleId, Bundle>,
bundle_cache: LruCache<BundleId, Vec<u8>> bundle_cache: LruCache<BundleId, Vec<u8>>
@ -422,13 +439,10 @@ pub struct BundleDb {
impl BundleDb { impl BundleDb {
fn new(path: PathBuf, compression: Option<Compression>, encryption: Option<Encryption>) -> Self { fn new(path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Self {
BundleDb { BundleDb {
path: path, path: path,
compression: crypto: crypto,
compression,
crypto: Arc::new(Mutex::new(Crypto::new())),
encryption: encryption,
bundles: HashMap::new(), bundles: HashMap::new(),
bundle_cache: LruCache::new(5, 10) bundle_cache: LruCache::new(5, 10)
} }
@ -436,7 +450,7 @@ impl BundleDb {
fn bundle_path(&self, bundle: &BundleId) -> (PathBuf, PathBuf) { fn bundle_path(&self, bundle: &BundleId) -> (PathBuf, PathBuf) {
let mut folder = self.path.clone(); let mut folder = self.path.clone();
let mut file = bundle.to_string()[0..32].to_owned() + ".bundle"; let mut file = bundle.to_string().to_owned() + ".bundle";
let mut count = self.bundles.len(); let mut count = self.bundles.len();
while count >= 100 { while count >= 100 {
if file.len() < 10 { if file.len() < 10 {
@ -469,33 +483,29 @@ impl BundleDb {
} }
#[inline] #[inline]
pub fn open<P: AsRef<Path>>(path: P, compression: Option<Compression>, encryption: Option<Encryption>) -> Result<Self, BundleError> { pub fn open<P: AsRef<Path>>(path: P, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
let path = path.as_ref().to_owned(); let path = path.as_ref().to_owned();
let mut self_ = Self::new(path, compression, encryption); let mut self_ = Self::new(path, crypto);
try!(self_.load_bundle_list()); try!(self_.load_bundle_list());
Ok(self_) Ok(self_)
} }
#[inline] #[inline]
pub fn create<P: AsRef<Path>>(path: P, compression: Option<Compression>, encryption: Option<Encryption>) -> Result<Self, BundleError> { pub fn create<P: AsRef<Path>>(path: P, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
let path = path.as_ref().to_owned(); let path = path.as_ref().to_owned();
try!(fs::create_dir_all(&path) try!(fs::create_dir_all(&path).context(&path as &Path));
.map_err(|e| BundleError::Write(e, path.clone()))); Ok(Self::new(path, crypto))
Ok(Self::new(path, compression, encryption))
} }
#[inline] #[inline]
pub fn open_or_create<P: AsRef<Path>>(path: P, compression: Option<Compression>, encryption: Option<Encryption>) -> Result<Self, BundleError> { pub fn create_bundle(
if path.as_ref().exists() { &self,
Self::open(path, compression, encryption) mode: BundleMode,
} else { hash_method: HashMethod,
Self::create(path, compression, encryption) compression: Option<Compression>,
} encryption: Option<Encryption>
} ) -> Result<BundleWriter, BundleError> {
BundleWriter::new(mode, hash_method, compression, encryption, self.crypto.clone())
#[inline]
pub fn create_bundle(&self, mode: BundleMode, hash_method: HashMethod) -> Result<BundleWriter, BundleError> {
BundleWriter::new(mode, hash_method, self.compression.clone(), self.encryption.clone(), self.crypto.clone())
} }
pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleError> { pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleError> {

View File

@ -77,7 +77,7 @@ impl IChunker for Chunker {
} }
#[derive(Debug)] #[derive(Debug, Clone, Copy)]
pub enum ChunkerType { pub enum ChunkerType {
Ae(usize), Ae(usize),
Rabin((usize, u32)), Rabin((usize, u32)),
@ -141,6 +141,11 @@ impl ChunkerType {
} }
} }
#[inline]
pub fn to_string(&self) -> String {
format!("{}/{}", self.name(), self.avg_size()/1024)
}
#[inline] #[inline]
pub fn seed(&self) -> u64 { pub fn seed(&self) -> u64 {
match *self { match *self {

View File

@ -43,8 +43,9 @@ fn chunk(data: &[u8], mut chunker: Chunker, sink: &mut ChunkSink) {
} }
#[allow(dead_code)] #[allow(dead_code)]
pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Option<Compression>, hash: HashMethod) { pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Option<Compression>, encrypt: bool,hash: HashMethod) {
let mut total_time = 0.0; let mut total_write_time = 0.0;
let mut total_read_time = 0.0;
println!("Reading input file ..."); println!("Reading input file ...");
let mut file = File::open(path).unwrap(); let mut file = File::open(path).unwrap();
@ -68,7 +69,7 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
let chunk_time = Duration::span(|| { let chunk_time = Duration::span(|| {
chunk(&data, chunker, &mut chunk_sink) chunk(&data, chunker, &mut chunk_sink)
}).num_milliseconds() as f32 / 1_000.0; }).num_milliseconds() as f32 / 1_000.0;
total_time += chunk_time; total_write_time += chunk_time;
println!("- {}, {}", to_duration(chunk_time), to_speed(size, chunk_time)); println!("- {}, {}", to_duration(chunk_time), to_speed(size, chunk_time));
let mut chunks = chunk_sink.chunks; let mut chunks = chunk_sink.chunks;
assert_eq!(chunks.iter().map(|c| c.1).sum::<usize>(), size as usize); assert_eq!(chunks.iter().map(|c| c.1).sum::<usize>(), size as usize);
@ -85,7 +86,7 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
hashes.push(hash.hash(&data[pos..pos+len])) hashes.push(hash.hash(&data[pos..pos+len]))
} }
}).num_milliseconds() as f32 / 1_000.0; }).num_milliseconds() as f32 / 1_000.0;
total_time += hash_time; total_write_time += hash_time;
println!("- {}, {}", to_duration(hash_time), to_speed(size, hash_time)); println!("- {}, {}", to_duration(hash_time), to_speed(size, hash_time));
let mut seen_hashes = HashSet::with_capacity(hashes.len()); let mut seen_hashes = HashSet::with_capacity(hashes.len());
let mut dups = Vec::new(); let mut dups = Vec::new();
@ -103,11 +104,12 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
println!("- {} duplicate chunks, {}, {:.1}% saved", dups.len(), to_file_size(dup_size as u64), dup_size as f32 / size as f32*100.0); println!("- {} duplicate chunks, {}, {:.1}% saved", dups.len(), to_file_size(dup_size as u64), dup_size as f32 / size as f32*100.0);
size -= dup_size as u64; size -= dup_size as u64;
if let Some(compression) = compression { let mut bundles = Vec::new();
if let Some(compression) = compression.clone() {
println!(); println!();
println!("Compressing chunks with {} ...", compression.to_string()); println!("Compressing chunks with {} ...", compression.to_string());
let mut bundles = Vec::new();
let compress_time = Duration::span(|| { let compress_time = Duration::span(|| {
let mut bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize); let mut bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize);
let mut c = compression.compress_stream().unwrap(); let mut c = compression.compress_stream().unwrap();
@ -123,12 +125,56 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
c.finish(&mut bundle).unwrap(); c.finish(&mut bundle).unwrap();
bundles.push(bundle); bundles.push(bundle);
}).num_milliseconds() as f32 / 1_000.0; }).num_milliseconds() as f32 / 1_000.0;
total_time += compress_time; total_write_time += compress_time;
println!("- {}, {}", to_duration(compress_time), to_speed(size, compress_time)); println!("- {}, {}", to_duration(compress_time), to_speed(size, compress_time));
let compressed_size = bundles.iter().map(|b| b.len()).sum::<usize>(); let compressed_size = bundles.iter().map(|b| b.len()).sum::<usize>();
println!("- {} bundles, {}, {:.1}% saved", bundles.len(), to_file_size(compressed_size as u64), (size as f32 - compressed_size as f32)/size as f32*100.0); println!("- {} bundles, {}, {:.1}% saved", bundles.len(), to_file_size(compressed_size as u64), (size as f32 - compressed_size as f32)/size as f32*100.0);
size = compressed_size as u64; size = compressed_size as u64;
} else {
let mut bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize);
for &(pos, len) in &chunks {
bundle.extend_from_slice(&data[pos..pos+len]);
if bundle.len() >= bundle_size {
bundles.push(bundle);
bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize);
}
}
bundles.push(bundle);
}
if encrypt {
println!();
let (public, secret) = gen_keypair();
let mut crypto = Crypto::new();
crypto.add_secret_key(public, secret);
let encryption = (EncryptionMethod::Sodium, public[..].iter().cloned().collect::<Vec<u8>>().into());
println!("Encrypting bundles...");
let mut encrypted_bundles = Vec::with_capacity(bundles.len());
let encrypt_time = Duration::span(|| {
for bundle in bundles {
encrypted_bundles.push(crypto.encrypt(&encryption, &bundle).unwrap());
}
}).num_milliseconds() as f32 / 1_000.0;
println!("- {}, {}", to_duration(encrypt_time), to_speed(size, encrypt_time));
total_write_time += encrypt_time;
println!();
println!("Decrypting bundles...");
bundles = Vec::with_capacity(encrypted_bundles.len());
let decrypt_time = Duration::span(|| {
for bundle in encrypted_bundles {
bundles.push(crypto.decrypt(&encryption, &bundle).unwrap());
}
}).num_milliseconds() as f32 / 1_000.0;
println!("- {}, {}", to_duration(decrypt_time), to_speed(size, decrypt_time));
total_read_time += decrypt_time;
}
if let Some(compression) = compression {
println!(); println!();
println!("Decompressing bundles with {} ...", compression.to_string()); println!("Decompressing bundles with {} ...", compression.to_string());
@ -141,12 +187,12 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
} }
}).num_milliseconds() as f32 / 1_000.0; }).num_milliseconds() as f32 / 1_000.0;
println!("- {}, {}", to_duration(decompress_time), to_speed(size, decompress_time)); println!("- {}, {}", to_duration(decompress_time), to_speed(size, decompress_time));
total_read_time += decompress_time;
} }
println!(); println!();
let total_saved = total_size - size; println!("Total storage size: {} / {}, ratio: {:.1}%", to_file_size(size as u64), to_file_size(total_size as u64), size as f32/total_size as f32*100.0);
println!("Total space saved: {}, {:.1}%", to_file_size(total_saved as u64), total_saved as f32/total_size as f32*100.0); println!("Total processing speed: {}", to_speed(total_size, total_write_time));
println!("Total processing speed: {}", to_speed(total_size, total_time)); println!("Total read speed: {}", to_speed(total_size, total_read_time));
} }

View File

@ -1,5 +1,5 @@
use ::chunker::ChunkerType; use ::chunker::ChunkerType;
use ::util::{Compression, HashMethod}; use ::util::*;
use std::process::exit; use std::process::exit;
@ -10,6 +10,7 @@ pub enum Arguments {
bundle_size: usize, bundle_size: usize,
chunker: ChunkerType, chunker: ChunkerType,
compression: Option<Compression>, compression: Option<Compression>,
encryption: bool,
hash: HashMethod hash: HashMethod
}, },
Backup { Backup {
@ -56,11 +57,27 @@ pub enum Arguments {
repo_path: String, repo_path: String,
remote_path: String remote_path: String
}, },
Configure {
repo_path: String,
bundle_size: Option<usize>,
chunker: Option<ChunkerType>,
compression: Option<Option<Compression>>,
encryption: Option<Option<PublicKey>>,
hash: Option<HashMethod>
},
GenKey {
},
AddKey {
repo_path: String,
key_pair: Option<(PublicKey, SecretKey)>,
set_default: bool
},
AlgoTest { AlgoTest {
file: String, file: String,
bundle_size: usize, bundle_size: usize,
chunker: ChunkerType, chunker: ChunkerType,
compression: Option<Compression>, compression: Option<Compression>,
encrypt: bool,
hash: HashMethod hash: HashMethod
} }
} }
@ -93,17 +110,16 @@ fn parse_float(num: &str, name: &str) -> f64 {
} }
fn parse_chunker(val: Option<&str>) -> ChunkerType { fn parse_chunker(val: &str) -> ChunkerType {
if let Ok(chunker) = ChunkerType::from_string(val.unwrap_or("fastcdc/8")) { if let Ok(chunker) = ChunkerType::from_string(val) {
chunker chunker
} else { } else {
error!("Invalid chunker method/size: {}", val.unwrap()); error!("Invalid chunker method/size: {}", val);
exit(1); exit(1);
} }
} }
fn parse_compression(val: Option<&str>) -> Option<Compression> { fn parse_compression(val: &str) -> Option<Compression> {
let val = val.unwrap_or("brotli/3");
if val == "none" { if val == "none" {
return None return None
} }
@ -115,11 +131,43 @@ fn parse_compression(val: Option<&str>) -> Option<Compression> {
} }
} }
fn parse_hash(val: Option<&str>) -> HashMethod { fn parse_public_key(val: &str) -> PublicKey {
if let Ok(hash) = HashMethod::from(val.unwrap_or("blake2")) { let bytes = match parse_hex(val) {
Ok(bytes) => bytes,
Err(_) => {
error!("Invalid key: {}", val);
exit(1);
}
};
if let Some(key) = PublicKey::from_slice(&bytes) {
key
} else {
error!("Invalid key: {}", val);
exit(1);
}
}
fn parse_secret_key(val: &str) -> SecretKey {
let bytes = match parse_hex(val) {
Ok(bytes) => bytes,
Err(_) => {
error!("Invalid key: {}", val);
exit(1);
}
};
if let Some(key) = SecretKey::from_slice(&bytes) {
key
} else {
error!("Invalid key: {}", val);
exit(1);
}
}
fn parse_hash(val: &str) -> HashMethod {
if let Ok(hash) = HashMethod::from(val) {
hash hash
} else { } else {
error!("Invalid hash method: {}", val.unwrap()); error!("Invalid hash method: {}", val);
exit(1); exit(1);
} }
} }
@ -127,18 +175,19 @@ fn parse_hash(val: Option<&str>) -> HashMethod {
pub fn parse() -> Arguments { pub fn parse() -> Arguments {
let args = clap_app!(zvault => let args = clap_app!(zvault =>
(version: env!("CARGO_PKG_VERSION")) (version: crate_version!())
(author: "Dennis Schwerdel <schwerdel@googlemail.com>") (author: crate_authors!(",\n"))
(about: "Deduplicating backup tool") (about: crate_description!())
(@setting SubcommandRequiredElseHelp) (@setting SubcommandRequiredElseHelp)
(@setting GlobalVersion) (@setting GlobalVersion)
(@setting VersionlessSubcommands) (@setting VersionlessSubcommands)
(@setting UnifiedHelpMessage) (@setting UnifiedHelpMessage)
(@subcommand init => (@subcommand init =>
(about: "initializes a new repository") (about: "initializes a new repository")
(@arg bundle_size: --bundle-size +takes_value "maximal bundle size in MiB [default: 25]") (@arg bundle_size: --bundlesize +takes_value "maximal bundle size in MiB [default: 25]")
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]") (@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]")
(@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]") (@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]")
(@arg encryption: --encryption -e "generate a keypair and enable encryption")
(@arg hash: --hash +takes_value "hash method to use [default: blake2]") (@arg hash: --hash +takes_value "hash method to use [default: blake2]")
(@arg REPO: +required "path of the repository") (@arg REPO: +required "path of the repository")
) )
@ -184,11 +233,32 @@ pub fn parse() -> Arguments {
(about: "displays information on a repository, a backup or a path in a backup") (about: "displays information on a repository, a backup or a path in a backup")
(@arg PATH: +required "repository[::backup[::subpath]] path") (@arg PATH: +required "repository[::backup[::subpath]] path")
) )
(@subcommand algotest => (@subcommand configure =>
(about: "test a specific algorithm combination") (about: "changes the configuration")
(@arg bundle_size: --bundle-size +takes_value "maximal bundle size in MiB [default: 25]") (@arg REPO: +required "path of the repository")
(@arg bundle_size: --bundlesize +takes_value "maximal bundle size in MiB [default: 25]")
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]") (@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]")
(@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]") (@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]")
(@arg encryption: --encryption -e +takes_value "the public key to use for encryption")
(@arg hash: --hash +takes_value "hash method to use [default: blake2]")
)
(@subcommand genkey =>
(about: "generates a new key pair")
)
(@subcommand addkey =>
(about: "adds a key to the respository")
(@arg REPO: +required "path of the repository")
(@arg generate: --generate "generate a new key")
(@arg set_default: --default "set this key as default")
(@arg PUBLIC: +takes_value "the public key")
(@arg SECRET: +takes_value "the secret key")
)
(@subcommand algotest =>
(about: "test a specific algorithm combination")
(@arg bundle_size: --bundlesize +takes_value "maximal bundle size in MiB [default: 25]")
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]")
(@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]")
(@arg encrypt: --encrypt -e "enable encryption")
(@arg hash: --hash +takes_value "hash method to use [default: blake2]") (@arg hash: --hash +takes_value "hash method to use [default: blake2]")
(@arg FILE: +required "the file to test the algorithms with") (@arg FILE: +required "the file to test the algorithms with")
) )
@ -201,9 +271,10 @@ pub fn parse() -> Arguments {
} }
return Arguments::Init { return Arguments::Init {
bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize, bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize,
chunker: parse_chunker(args.value_of("chunker")), chunker: parse_chunker(args.value_of("chunker").unwrap_or("fastcdc/8")),
compression: parse_compression(args.value_of("compression")), compression: parse_compression(args.value_of("compression").unwrap_or("brotli/3")),
hash: parse_hash(args.value_of("hash")), encryption: args.is_present("encryption"),
hash: parse_hash(args.value_of("hash").unwrap_or("blake2")),
repo_path: repository.to_string(), repo_path: repository.to_string(),
} }
} }
@ -306,12 +377,63 @@ pub fn parse() -> Arguments {
remote_path: args.value_of("REMOTE").unwrap().to_string() remote_path: args.value_of("REMOTE").unwrap().to_string()
} }
} }
if let Some(args) = args.subcommand_matches("configure") {
let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap());
if backup.is_some() || inode.is_some() {
println!("No backups or subpaths may be given here");
exit(1);
}
return Arguments::Configure {
bundle_size: args.value_of("bundle_size").map(|v| (parse_num(v, "Bundle size") * 1024 * 1024) as usize),
chunker: args.value_of("chunker").map(|v| parse_chunker(v)),
compression: args.value_of("compression").map(|v| parse_compression(v)),
encryption: args.value_of("encryption").map(|v| {
if v == "none" {
None
} else {
Some(parse_public_key(v))
}
}),
hash: args.value_of("hash").map(|v| parse_hash(v)),
repo_path: repository.to_string(),
}
}
if let Some(_args) = args.subcommand_matches("genkey") {
return Arguments::GenKey {}
}
if let Some(args) = args.subcommand_matches("addkey") {
let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap());
if backup.is_some() || inode.is_some() {
println!("No backups or subpaths may be given here");
exit(1);
}
let generate = args.is_present("generate");
if !generate && (!args.is_present("PUBLIC") || !args.is_present("SECRET")) {
println!("Without --generate, a public and secret key must be given");
exit(1);
}
if generate && (args.is_present("PUBLIC") || args.is_present("SECRET")) {
println!("With --generate, no public or secret key may be given");
exit(1);
}
let key_pair = if generate {
None
} else {
Some((parse_public_key(args.value_of("PUBLIC").unwrap()), parse_secret_key(args.value_of("SECRET").unwrap())))
};
return Arguments::AddKey {
repo_path: repository.to_string(),
set_default: args.is_present("set_default"),
key_pair: key_pair
}
}
if let Some(args) = args.subcommand_matches("algotest") { if let Some(args) = args.subcommand_matches("algotest") {
return Arguments::AlgoTest { return Arguments::AlgoTest {
bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize, bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize,
chunker: parse_chunker(args.value_of("chunker")), chunker: parse_chunker(args.value_of("chunker").unwrap_or("fastcdc/8")),
compression: parse_compression(args.value_of("compression")), compression: parse_compression(args.value_of("compression").unwrap_or("brotli/3")),
hash: parse_hash(args.value_of("hash")), encrypt: args.is_present("encrypt"),
hash: parse_hash(args.value_of("hash").unwrap_or("blake2")),
file: args.value_of("FILE").unwrap().to_string(), file: args.value_of("FILE").unwrap().to_string(),
} }
} }

View File

@ -1,6 +1,9 @@
use log::{self, LogRecord, LogLevel, LogMetadata, LogLevelFilter}; use log::{self, LogRecord, LogLevel, LogMetadata, LogLevelFilter};
pub use log::SetLoggerError; pub use log::SetLoggerError;
use ansi_term::{Color, Style};
struct Logger; struct Logger;
impl log::Log for Logger { impl log::Log for Logger {
@ -10,7 +13,14 @@ impl log::Log for Logger {
fn log(&self, record: &LogRecord) { fn log(&self, record: &LogRecord) {
if self.enabled(record.metadata()) { if self.enabled(record.metadata()) {
println!("{} - {}", record.level(), record.args()); let lvl = record.level();
match lvl {
LogLevel::Error => println!("{} - {}", Color::Red.bold().paint("error"), record.args()),
LogLevel::Warn => println!("{} - {}", Color::Yellow.bold().paint("warning"), record.args()),
LogLevel::Info => println!("{} - {}", Color::Green.bold().paint("info"), record.args()),
LogLevel::Debug => println!("{} - {}", Style::new().bold().paint("debug"), record.args()),
LogLevel::Trace => println!("{} - {}", "trace", record.args())
}
} }
} }
} }

View File

@ -7,6 +7,7 @@ use std::process::exit;
use ::repository::{Repository, Config, Backup}; use ::repository::{Repository, Config, Backup};
use ::util::cli::*; use ::util::cli::*;
use ::util::*;
use self::args::Arguments; use self::args::Arguments;
@ -36,13 +37,22 @@ pub fn run() {
exit(-1) exit(-1)
} }
match args::parse() { match args::parse() {
Arguments::Init{repo_path, bundle_size, chunker, compression, hash} => { Arguments::Init{repo_path, bundle_size, chunker, compression, encryption, hash} => {
Repository::create(repo_path, Config { let mut repo = Repository::create(repo_path, Config {
bundle_size: bundle_size, bundle_size: bundle_size,
chunker: chunker, chunker: chunker,
compression: compression, compression: compression,
encryption: None,
hash: hash hash: hash
}).unwrap(); }).unwrap();
if encryption {
let (public, secret) = gen_keypair();
println!("Public key: {}", to_hex(&public[..]));
println!("Secret key: {}", to_hex(&secret[..]));
repo.set_encryption(Some(&public));
repo.register_key(public, secret).unwrap();
repo.save_config().unwrap();
}
}, },
Arguments::Backup{repo_path, backup_name, src_path, full} => { Arguments::Backup{repo_path, backup_name, src_path, full} => {
let mut repo = open_repository(&repo_path); let mut repo = open_repository(&repo_path);
@ -165,8 +175,63 @@ pub fn run() {
error!("Import is not implemented yet"); error!("Import is not implemented yet");
return return
}, },
Arguments::AlgoTest{bundle_size, chunker, compression, hash, file} => { Arguments::Configure{repo_path, bundle_size, chunker, compression, encryption, hash} => {
algotest::run(&file, bundle_size, chunker, compression, hash); let mut repo = open_repository(&repo_path);
if let Some(bundle_size) = bundle_size {
repo.config.bundle_size = bundle_size
}
if let Some(chunker) = chunker {
warn!("Changing the chunker makes it impossible to use existing data for deduplication");
repo.config.chunker = chunker
}
if let Some(compression) = compression {
repo.config.compression = compression
}
if let Some(encryption) = encryption {
repo.set_encryption(encryption.as_ref())
}
if let Some(hash) = hash {
warn!("Changing the hash makes it impossible to use existing data for deduplication");
repo.config.hash = hash
}
repo.save_config().unwrap();
println!("Bundle size: {}", to_file_size(repo.config.bundle_size as u64));
println!("Chunker: {}", repo.config.chunker.to_string());
if let Some(ref compression) = repo.config.compression {
println!("Compression: {}", compression.to_string());
} else {
println!("Compression: none");
}
if let Some(ref encryption) = repo.config.encryption {
println!("Encryption: {}", to_hex(&encryption.1[..]));
} else {
println!("Encryption: none");
}
println!("Hash method: {}", repo.config.hash.name());
},
Arguments::GenKey{} => {
let (public, secret) = gen_keypair();
println!("Public key: {}", to_hex(&public[..]));
println!("Secret key: {}", to_hex(&secret[..]));
},
Arguments::AddKey{repo_path, set_default, key_pair} => {
let mut repo = open_repository(&repo_path);
let (public, secret) = if let Some(key_pair) = key_pair {
key_pair
} else {
let (public, secret) = gen_keypair();
println!("Public key: {}", to_hex(&public[..]));
println!("Secret key: {}", to_hex(&secret[..]));
(public, secret)
};
if set_default {
repo.set_encryption(Some(&public));
repo.save_config().unwrap();
}
repo.register_key(public, secret).unwrap();
},
Arguments::AlgoTest{bundle_size, chunker, compression, encrypt, hash, file} => {
algotest::run(&file, bundle_size, chunker, compression, encrypt, hash);
} }
} }
} }

View File

@ -12,6 +12,8 @@ extern crate chrono;
#[macro_use] extern crate clap; #[macro_use] extern crate clap;
#[macro_use] extern crate log; #[macro_use] extern crate log;
extern crate byteorder; extern crate byteorder;
extern crate sodiumoxide;
extern crate ansi_term;
pub mod util; pub mod util;
@ -26,12 +28,11 @@ mod cli;
// TODO: - Keep meta bundles also locally // TODO: - Keep meta bundles also locally
// TODO: - Load and compare remote bundles to bundle map // TODO: - Load and compare remote bundles to bundle map
// TODO: - Write backup files there as well // TODO: - Write backup files there as well
// TODO: Store list of hashes and hash method in bundle
// TODO: Remove backup subtrees // TODO: Remove backup subtrees
// TODO: Recompress & combine bundles // TODO: Recompress & combine bundles
// TODO: Prune backups (based on age like attic) // TODO: Prune backups (based on age like attic)
// TODO: Check backup integrity too // TODO: Check backup integrity too
// TODO: Encryption // TODO: Encrypt backup files too
// TODO: list --tree // TODO: list --tree
// TODO: Partial backups // TODO: Partial backups
// TODO: Import repository from remote folder // TODO: Import repository from remote folder

View File

@ -46,7 +46,12 @@ impl Repository {
}; };
// ...alocate one if needed // ...alocate one if needed
if writer.is_none() { if writer.is_none() {
*writer = Some(try!(self.bundles.create_bundle(mode, self.config.hash))); *writer = Some(try!(self.bundles.create_bundle(
mode,
self.config.hash,
self.config.compression.clone(),
self.config.encryption.clone()
)));
} }
debug_assert!(writer.is_some()); debug_assert!(writer.is_some());
let chunk_id; let chunk_id;

View File

@ -24,6 +24,7 @@ quick_error!{
from() from()
cause(err) cause(err)
description("Yaml format error") description("Yaml format error")
display("Yaml format error: {}", err)
} }
} }
} }
@ -89,9 +90,41 @@ impl Compression {
} }
impl EncryptionMethod {
#[inline]
fn from_yaml(yaml: String) -> Result<Self, ConfigError> {
EncryptionMethod::from_string(&yaml).map_err(|_| ConfigError::Parse("Invalid codec"))
}
#[inline]
fn to_yaml(&self) -> String {
self.to_string()
}
}
struct EncryptionYaml {
method: String,
key: String
}
impl Default for EncryptionYaml {
fn default() -> Self {
EncryptionYaml {
method: "sodium".to_string(),
key: "".to_string()
}
}
}
serde_impl!(EncryptionYaml(String) {
method: String => "method",
key: String => "key"
});
struct ConfigYaml { struct ConfigYaml {
compression: Option<String>, compression: Option<String>,
encryption: Option<EncryptionYaml>,
bundle_size: usize, bundle_size: usize,
chunker: ChunkerYaml, chunker: ChunkerYaml,
hash: String, hash: String,
@ -100,6 +133,7 @@ impl Default for ConfigYaml {
fn default() -> Self { fn default() -> Self {
ConfigYaml { ConfigYaml {
compression: Some("brotli/5".to_string()), compression: Some("brotli/5".to_string()),
encryption: None,
bundle_size: 25*1024*1024, bundle_size: 25*1024*1024,
chunker: ChunkerYaml::default(), chunker: ChunkerYaml::default(),
hash: "blake2".to_string() hash: "blake2".to_string()
@ -108,6 +142,7 @@ impl Default for ConfigYaml {
} }
serde_impl!(ConfigYaml(String) { serde_impl!(ConfigYaml(String) {
compression: Option<String> => "compression", compression: Option<String> => "compression",
encryption: Option<EncryptionYaml> => "encryption",
bundle_size: usize => "bundle_size", bundle_size: usize => "bundle_size",
chunker: ChunkerYaml => "chunker", chunker: ChunkerYaml => "chunker",
hash: String => "hash" hash: String => "hash"
@ -118,6 +153,7 @@ serde_impl!(ConfigYaml(String) {
#[derive(Debug)] #[derive(Debug)]
pub struct Config { pub struct Config {
pub compression: Option<Compression>, pub compression: Option<Compression>,
pub encryption: Option<Encryption>,
pub bundle_size: usize, pub bundle_size: usize,
pub chunker: ChunkerType, pub chunker: ChunkerType,
pub hash: HashMethod pub hash: HashMethod
@ -129,8 +165,16 @@ impl Config {
} else { } else {
None None
}; };
let encryption = if let Some(e) = yaml.encryption {
let method = try!(EncryptionMethod::from_yaml(e.method));
let key = try!(parse_hex(&e.key).map_err(|_| ConfigError::Parse("Invalid public key")));
Some((method, key.into()))
} else {
None
};
Ok(Config{ Ok(Config{
compression: compression, compression: compression,
encryption: encryption,
bundle_size: yaml.bundle_size, bundle_size: yaml.bundle_size,
chunker: try!(ChunkerType::from_yaml(yaml.chunker)), chunker: try!(ChunkerType::from_yaml(yaml.chunker)),
hash: try!(HashMethod::from_yaml(yaml.hash)) hash: try!(HashMethod::from_yaml(yaml.hash))
@ -140,6 +184,7 @@ impl Config {
fn to_yaml(&self) -> ConfigYaml { fn to_yaml(&self) -> ConfigYaml {
ConfigYaml { ConfigYaml {
compression: self.compression.as_ref().map(|c| c.to_yaml()), compression: self.compression.as_ref().map(|c| c.to_yaml()),
encryption: self.encryption.as_ref().map(|e| EncryptionYaml{method: e.0.to_yaml(), key: to_hex(&e.1[..])}),
bundle_size: self.bundle_size, bundle_size: self.bundle_size,
chunker: self.chunker.to_yaml(), chunker: self.chunker.to_yaml(),
hash: self.hash.to_yaml() hash: self.hash.to_yaml()

View File

@ -17,47 +17,62 @@ quick_error!{
Io(err: io::Error) { Io(err: io::Error) {
from() from()
cause(err) cause(err)
description("IO Error") description("IO error")
display("IO error: {}", err)
} }
Config(err: ConfigError) { Config(err: ConfigError) {
from() from()
cause(err) cause(err)
description("Configuration error") description("Configuration error")
display("Configuration error: {}", err)
} }
BundleMap(err: BundleMapError) { BundleMap(err: BundleMapError) {
from() from()
cause(err) cause(err)
description("Bundle map error") description("Bundle map error")
display("Bundle map error: {}", err)
} }
Index(err: IndexError) { Index(err: IndexError) {
from() from()
cause(err) cause(err)
description("Index error") description("Index error")
display("Index error: {}", err)
} }
Bundle(err: BundleError) { Bundle(err: BundleError) {
from() from()
cause(err) cause(err)
description("Bundle error") description("Bundle error")
display("Bundle error: {}", err)
} }
Chunker(err: ChunkerError) { Chunker(err: ChunkerError) {
from() from()
cause(err) cause(err)
description("Chunker error") description("Chunker error")
display("Chunker error: {}", err)
} }
Decode(err: msgpack::DecodeError) { Decode(err: msgpack::DecodeError) {
from() from()
cause(err) cause(err)
description("Failed to decode metadata") description("Failed to decode metadata")
display("Failed to decode metadata: {}", err)
} }
Encode(err: msgpack::EncodeError) { Encode(err: msgpack::EncodeError) {
from() from()
cause(err) cause(err)
description("Failed to encode metadata") description("Failed to encode metadata")
display("Failed to encode metadata: {}", err)
} }
Integrity(err: RepositoryIntegrityError) { Integrity(err: RepositoryIntegrityError) {
from() from()
cause(err) cause(err)
description("Integrity error") description("Integrity error")
display("Integrity error: {}", err)
}
Encryption(err: EncryptionError) {
from()
cause(err)
description("Failed to load keys")
display("Failed to load keys: {}", err)
} }
InvalidFileType(path: PathBuf) { InvalidFileType(path: PathBuf) {
description("Invalid file type") description("Invalid file type")

View File

@ -11,10 +11,12 @@ use std::mem;
use std::cmp::max; use std::cmp::max;
use std::path::{PathBuf, Path}; use std::path::{PathBuf, Path};
use std::fs; use std::fs;
use std::sync::{Arc, Mutex};
use super::index::Index; use super::index::Index;
use super::bundle::{BundleDb, BundleWriter}; use super::bundle::{BundleDb, BundleWriter};
use super::chunker::Chunker; use super::chunker::Chunker;
use ::util::*;
pub use self::error::RepositoryError; pub use self::error::RepositoryError;
pub use self::config::Config; pub use self::config::Config;
@ -25,8 +27,9 @@ use self::bundle_map::BundleMap;
pub struct Repository { pub struct Repository {
path: PathBuf, path: PathBuf,
config: Config, pub config: Config,
index: Index, index: Index,
crypto: Arc<Mutex<Crypto>>,
bundle_map: BundleMap, bundle_map: BundleMap,
next_content_bundle: u32, next_content_bundle: u32,
next_meta_bundle: u32, next_meta_bundle: u32,
@ -41,10 +44,11 @@ impl Repository {
pub fn create<P: AsRef<Path>>(path: P, config: Config) -> Result<Self, RepositoryError> { pub fn create<P: AsRef<Path>>(path: P, config: Config) -> Result<Self, RepositoryError> {
let path = path.as_ref().to_owned(); let path = path.as_ref().to_owned();
try!(fs::create_dir(&path)); try!(fs::create_dir(&path));
try!(fs::create_dir(path.join("keys")));
let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys")))));
let bundles = try!(BundleDb::create( let bundles = try!(BundleDb::create(
path.join("bundles"), path.join("bundles"),
config.compression.clone(), crypto.clone()
None, //FIXME: store encryption in config
)); ));
let index = try!(Index::create(&path.join("index"))); let index = try!(Index::create(&path.join("index")));
try!(config.save(path.join("config.yaml"))); try!(config.save(path.join("config.yaml")));
@ -62,16 +66,17 @@ impl Repository {
bundles: bundles, bundles: bundles,
content_bundle: None, content_bundle: None,
meta_bundle: None, meta_bundle: None,
crypto: crypto
}) })
} }
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, RepositoryError> { pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, RepositoryError> {
let path = path.as_ref().to_owned(); let path = path.as_ref().to_owned();
let config = try!(Config::load(path.join("config.yaml"))); let config = try!(Config::load(path.join("config.yaml")));
let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys")))));
let bundles = try!(BundleDb::open( let bundles = try!(BundleDb::open(
path.join("bundles"), path.join("bundles"),
config.compression.clone(), crypto.clone()
None, //FIXME: load encryption from config
)); ));
let index = try!(Index::open(&path.join("index"))); let index = try!(Index::open(&path.join("index")));
let bundle_map = try!(BundleMap::load(path.join("bundles.map"))); let bundle_map = try!(BundleMap::load(path.join("bundles.map")));
@ -80,6 +85,7 @@ impl Repository {
chunker: config.chunker.create(), chunker: config.chunker.create(),
config: config, config: config,
index: index, index: index,
crypto: crypto,
bundle_map: bundle_map, bundle_map: bundle_map,
next_content_bundle: 0, next_content_bundle: 0,
next_meta_bundle: 0, next_meta_bundle: 0,
@ -92,6 +98,27 @@ impl Repository {
Ok(repo) Ok(repo)
} }
#[inline]
pub fn register_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), RepositoryError> {
Ok(try!(self.crypto.lock().unwrap().register_secret_key(public, secret)))
}
pub fn save_config(&mut self) -> Result<(), RepositoryError> {
try!(self.config.save(self.path.join("config.yaml")));
Ok(())
}
#[inline]
pub fn set_encryption(&mut self, public: Option<&PublicKey>) {
if let Some(key) = public {
let mut key_bytes = Vec::new();
key_bytes.extend_from_slice(&key[..]);
self.config.encryption = Some((EncryptionMethod::Sodium, key_bytes.into()))
} else {
self.config.encryption = None
}
}
#[inline] #[inline]
fn save_bundle_map(&self) -> Result<(), RepositoryError> { fn save_bundle_map(&self) -> Result<(), RepositoryError> {
try!(self.bundle_map.save(self.path.join("bundles.map"))); try!(self.bundle_map.save(self.path.join("bundles.map")));

View File

@ -1,65 +1,175 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::io;
use std::fs::{self, File};
use serde_yaml;
use serde::bytes::ByteBuf;
use sodiumoxide::crypto::sealedbox;
pub use sodiumoxide::crypto::box_::{SecretKey, PublicKey, gen_keypair};
use ::util::*;
quick_error!{ quick_error!{
#[derive(Debug)] #[derive(Debug)]
pub enum EncryptionError { pub enum EncryptionError {
InvalidKey {
description("Invalid key")
}
MissingKey(key: PublicKey) {
description("Missing key")
display("Missing key: {}", to_hex(&key[..]))
}
Operation(reason: &'static str) { Operation(reason: &'static str) {
description("Operation failed") description("Operation failed")
display("Operation failed: {}", reason) display("Operation failed: {}", reason)
} }
Io(err: io::Error) {
from()
cause(err)
description("IO error")
display("IO error: {}", err)
}
Yaml(err: serde_yaml::Error) {
from()
cause(err)
description("Yaml format error")
display("Yaml format error: {}", err)
}
} }
} }
#[derive(Clone)] #[derive(Clone, Debug, Eq, PartialEq, Hash)]
#[allow(unknown_lints,non_camel_case_types)]
pub enum EncryptionMethod { pub enum EncryptionMethod {
Dummy Sodium,
} }
serde_impl!(EncryptionMethod(u64) { serde_impl!(EncryptionMethod(u64) {
Dummy => 0 Sodium => 0
}); });
pub type EncryptionKey = Vec<u8>; impl EncryptionMethod {
pub fn from_string(val: &str) -> Result<Self, &'static str> {
match val {
"sodium" => Ok(EncryptionMethod::Sodium),
_ => Err("Unsupported encryption method")
}
}
pub type EncryptionKeyId = u64; pub fn to_string(&self) -> String {
match *self {
EncryptionMethod::Sodium => "sodium".to_string()
}
}
}
pub type Encryption = (EncryptionMethod, ByteBuf);
struct KeyfileYaml {
public: String,
secret: String
}
impl Default for KeyfileYaml {
fn default() -> Self {
KeyfileYaml {
public: "".to_string(),
secret: "".to_string()
}
}
}
serde_impl!(KeyfileYaml(String) {
public: String => "public",
secret: String => "secret"
});
impl KeyfileYaml {
pub fn load<P: AsRef<Path>>(path: P) -> Result<Self, EncryptionError> {
let f = try!(File::open(path));
Ok(try!(serde_yaml::from_reader(f)))
}
pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<(), EncryptionError> {
let mut f = try!(File::create(path));
Ok(try!(serde_yaml::to_writer(&mut f, &self)))
}
}
pub type Encryption = (EncryptionMethod, EncryptionKeyId);
#[derive(Clone)]
pub struct Crypto { pub struct Crypto {
keys: HashMap<EncryptionKeyId, EncryptionKey> path: PathBuf,
keys: HashMap<PublicKey, SecretKey>
} }
impl Crypto { impl Crypto {
#[inline] #[inline]
pub fn new() -> Self { pub fn new() -> Self {
Crypto { keys: Default::default() } Crypto { path: PathBuf::new(), keys: HashMap::new() }
} }
#[inline] #[inline]
pub fn register_key(&mut self, key: EncryptionKey, id: EncryptionKeyId) { pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, EncryptionError> {
self.keys.insert(id, key); let path = path.as_ref().to_owned();
let mut keys: HashMap<PublicKey, SecretKey> = HashMap::default();
for entry in try!(fs::read_dir(&path)) {
let entry = try!(entry);
let keyfile = try!(KeyfileYaml::load(entry.path()));
let public = try!(parse_hex(&keyfile.public).map_err(|_| EncryptionError::InvalidKey));
let public = try!(PublicKey::from_slice(&public).ok_or(EncryptionError::InvalidKey));
let secret = try!(parse_hex(&keyfile.secret).map_err(|_| EncryptionError::InvalidKey));
let secret = try!(SecretKey::from_slice(&secret).ok_or(EncryptionError::InvalidKey));
keys.insert(public, secret);
}
Ok(Crypto { path: path, keys: keys })
} }
#[inline] #[inline]
pub fn contains_key(&mut self, id: EncryptionKeyId) -> bool { pub fn add_secret_key(&mut self, public: PublicKey, secret: SecretKey) {
self.keys.contains_key(&id) self.keys.insert(public, secret);
} }
#[inline] #[inline]
pub fn encrypt(&self, _enc: Encryption, _data: &[u8]) -> Result<Vec<u8>, EncryptionError> { pub fn register_secret_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), EncryptionError> {
unimplemented!() let keyfile = KeyfileYaml { public: to_hex(&public[..]), secret: to_hex(&secret[..]) };
let path = self.path.join(to_hex(&public[..]) + ".yaml");
try!(keyfile.save(path));
self.keys.insert(public, secret);
Ok(())
} }
#[inline] #[inline]
pub fn decrypt(&self, _enc: Encryption, _data: &[u8]) -> Result<Vec<u8>, EncryptionError> { pub fn contains_secret_key(&mut self, public: &PublicKey) -> bool {
unimplemented!() self.keys.contains_key(public)
} }
}
fn get_secret_key(&self, public: &PublicKey) -> Result<&SecretKey, EncryptionError> {
impl Default for Crypto { self.keys.get(public).ok_or_else(|| EncryptionError::MissingKey(*public))
#[inline] }
fn default() -> Self {
Crypto::new() #[inline]
pub fn encrypt(&self, enc: &Encryption, data: &[u8]) -> Result<Vec<u8>, EncryptionError> {
let &(ref method, ref public) = enc;
let public = try!(PublicKey::from_slice(public).ok_or(EncryptionError::InvalidKey));
match *method {
EncryptionMethod::Sodium => {
Ok(sealedbox::seal(data, &public))
}
}
}
#[inline]
pub fn decrypt(&self, enc: &Encryption, data: &[u8]) -> Result<Vec<u8>, EncryptionError> {
let &(ref method, ref public) = enc;
let public = try!(PublicKey::from_slice(public).ok_or(EncryptionError::InvalidKey));
let secret = try!(self.get_secret_key(&public));
match *method {
EncryptionMethod::Sodium => {
sealedbox::open(data, &public, secret).map_err(|_| EncryptionError::Operation("Decryption failed"))
}
}
} }
} }

View File

@ -12,3 +12,35 @@ pub use self::compression::*;
pub use self::encryption::*; pub use self::encryption::*;
pub use self::hash::*; pub use self::hash::*;
pub use self::lru_cache::*; pub use self::lru_cache::*;
pub fn to_hex(data: &[u8]) -> String {
data.iter().map(|b| format!("{:02x}", b)).collect::<Vec<String>>().join("")
}
pub fn parse_hex(hex: &str) -> Result<Vec<u8>, ()> {
let mut b = Vec::with_capacity(hex.len() / 2);
let mut modulus = 0;
let mut buf = 0;
for (_, byte) in hex.bytes().enumerate() {
buf <<= 4;
match byte {
b'A'...b'F' => buf |= byte - b'A' + 10,
b'a'...b'f' => buf |= byte - b'a' + 10,
b'0'...b'9' => buf |= byte - b'0',
b' '|b'\r'|b'\n'|b'\t' => {
buf >>= 4;
continue
}
_ => return Err(()),
}
modulus += 1;
if modulus == 2 {
modulus = 0;
b.push(buf);
}
}
match modulus {
0 => Ok(b.into_iter().collect()),
_ => Err(()),
}
}

View File

@ -17,7 +17,7 @@ pub fn encode<T: Serialize>(t: &T) -> Result<Vec<u8>, EncodeError> {
Ok(data) Ok(data)
} }
pub fn encode_to_stream<T: Serialize>(t: T, w: &mut Write) -> Result<(), EncodeError> { pub fn encode_to_stream<T: Serialize>(t: &T, w: &mut Write) -> Result<(), EncodeError> {
let mut writer = rmp_serde::Serializer::new(w); let mut writer = rmp_serde::Serializer::new(w);
t.serialize(&mut writer) t.serialize(&mut writer)
} }