mirror of https://github.com/dswd/zvault
Bundle encryption
This commit is contained in:
Dennis Schwerdel
Dennis Schwerdel
parent
0a807b16ab
commit
c8b69ebe25
|
|
@ -2,6 +2,7 @@
|
|||
name = "zvault"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -9,12 +10,14 @@ dependencies = [
|
|||
"log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rmp-serde 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"sodiumoxide 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
|
|
@ -104,6 +107,15 @@ name = "libc"
|
|||
version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "libsodium-sys"
|
||||
version = "0.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linked-hash-map"
|
||||
version = "0.3.0"
|
||||
|
|
@ -230,6 +242,16 @@ dependencies = [
|
|||
"yaml-rust 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sodiumoxide"
|
||||
version = "0.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libsodium-sys 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "squash-sys"
|
||||
version = "0.9.0"
|
||||
|
|
@ -320,6 +342,7 @@ dependencies = [
|
|||
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
|
||||
"checksum libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122"
|
||||
"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135"
|
||||
"checksum libsodium-sys 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "cbbc6e46017815abf8698de0ed4847fad45fd8cad2909ac38ac6de79673c1ad1"
|
||||
"checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd"
|
||||
"checksum log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "5141eca02775a762cc6cd564d8d2c50f67c0ea3a372cbf1c51592b3e029e10ad"
|
||||
"checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823"
|
||||
|
|
@ -338,6 +361,7 @@ dependencies = [
|
|||
"checksum serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)" = "a702319c807c016e51f672e5c77d6f0b46afddd744b5e437d6b8436b888b458f"
|
||||
"checksum serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b34a52969c7fc0254e214b82518c9a95dc88c84fc84cd847add314996a031be6"
|
||||
"checksum serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f8bd3f24ad8c7bcd34a6d70ba676dc11302b96f4f166aa5f947762e01098844d"
|
||||
"checksum sodiumoxide 0.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "bc02c0bc77ffed8e8eaef004399b825cf4fd8aa02d0af6e473225affd583ff4d"
|
||||
"checksum squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db1f9dde91d819b7746e153bc32489fa19e6a106c3d7f2b92187a4efbdc88b40"
|
||||
"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694"
|
||||
"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
name = "zvault"
|
||||
version = "0.1.0"
|
||||
authors = ["Dennis Schwerdel <schwerdel@informatik.uni-kl.de>"]
|
||||
description = "Deduplicating backup tool"
|
||||
|
||||
[dependencies]
|
||||
serde = "0.9"
|
||||
|
|
@ -18,3 +19,8 @@ chrono = "0.3"
|
|||
clap = "2.19"
|
||||
log = "0.3"
|
||||
byteorder = "1.0"
|
||||
ansi_term = "0.9"
|
||||
sodiumoxide = "*"
|
||||
|
||||
[build-dependencies]
|
||||
pkg-config = "0.3"
|
||||
|
|
|
|||
110
src/bundle.rs
110
src/bundle.rs
|
|
@ -7,6 +7,7 @@ use std::fmt::{self, Debug};
|
|||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use serde::{self, Serialize, Deserialize};
|
||||
use quick_error::ResultExt;
|
||||
|
||||
use util::*;
|
||||
|
||||
|
|
@ -18,7 +19,7 @@ static HEADER_VERSION: u8 = 1;
|
|||
Bundle format
|
||||
- Magic header + version
|
||||
- Encoded header structure (contains size of next structure)
|
||||
- Encoded contents structure (with chunk sizes and hashes)
|
||||
- Encoded chunk list (with chunk hashes and sizes)
|
||||
- Chunk data
|
||||
|
||||
*/
|
||||
|
|
@ -31,22 +32,25 @@ quick_error!{
|
|||
List(err: io::Error) {
|
||||
cause(err)
|
||||
description("Failed to list bundles")
|
||||
display("Failed to list bundles: {}", err)
|
||||
}
|
||||
Read(err: io::Error, path: PathBuf) {
|
||||
Io(err: io::Error, path: PathBuf) {
|
||||
cause(err)
|
||||
context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf())
|
||||
description("Failed to read bundle")
|
||||
display("Failed to read bundle {:?}: {}", path, err)
|
||||
}
|
||||
Decode(err: msgpack::DecodeError, path: PathBuf) {
|
||||
cause(err)
|
||||
context(path: &'a Path, err: msgpack::DecodeError) -> (err, path.to_path_buf())
|
||||
description("Failed to decode bundle header")
|
||||
}
|
||||
Write(err: io::Error, path: PathBuf) {
|
||||
cause(err)
|
||||
description("Failed to write bundle")
|
||||
display("Failed to decode bundle header of {:?}: {}", path, err)
|
||||
}
|
||||
Encode(err: msgpack::EncodeError, path: PathBuf) {
|
||||
cause(err)
|
||||
context(path: &'a Path, err: msgpack::EncodeError) -> (err, path.to_path_buf())
|
||||
description("Failed to encode bundle header")
|
||||
display("Failed to encode bundle header of {:?}: {}", path, err)
|
||||
}
|
||||
WrongHeader(path: PathBuf) {
|
||||
description("Wrong header")
|
||||
|
|
@ -68,13 +72,29 @@ quick_error!{
|
|||
description("Bundle has no such chunk")
|
||||
display("Bundle {:?} has no chunk with that id: {}", bundle, id)
|
||||
}
|
||||
Decompression(err: CompressionError, path: PathBuf) {
|
||||
cause(err)
|
||||
context(path: &'a Path, err: CompressionError) -> (err, path.to_path_buf())
|
||||
description("Decompression failed")
|
||||
display("Decompression failed on bundle {:?}: {}", path, err)
|
||||
}
|
||||
Compression(err: CompressionError) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Compression failed")
|
||||
display("Compression failed: {}", err)
|
||||
}
|
||||
Decryption(err: EncryptionError, path: PathBuf) {
|
||||
cause(err)
|
||||
context(path: &'a Path, err: EncryptionError) -> (err, path.to_path_buf())
|
||||
description("Decryption failed")
|
||||
display("Decryption failed on bundle {:?}: {}", path, err)
|
||||
}
|
||||
Encryption(err: EncryptionError) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Encryption failed")
|
||||
display("Encryption failed: {}", err)
|
||||
}
|
||||
Remove(err: io::Error, bundle: BundleId) {
|
||||
cause(err)
|
||||
|
|
@ -209,9 +229,9 @@ impl Bundle {
|
|||
}
|
||||
|
||||
pub fn load(path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
|
||||
let mut file = BufReader::new(try!(File::open(&path).map_err(|e| BundleError::Read(e, path.clone()))));
|
||||
let mut file = BufReader::new(try!(File::open(&path).context(&path as &Path)));
|
||||
let mut header = [0u8; 8];
|
||||
try!(file.read_exact(&mut header).map_err(|e| BundleError::Read(e, path.clone())));
|
||||
try!(file.read_exact(&mut header).context(&path as &Path));
|
||||
if header[..HEADER_STRING.len()] != HEADER_STRING {
|
||||
return Err(BundleError::WrongHeader(path.clone()))
|
||||
}
|
||||
|
|
@ -223,9 +243,9 @@ impl Bundle {
|
|||
.map_err(|e| BundleError::Decode(e, path.clone())));
|
||||
let mut chunk_data = Vec::with_capacity(header.chunk_info_size);
|
||||
chunk_data.resize(header.chunk_info_size, 0);
|
||||
try!(file.read_exact(&mut chunk_data).map_err(|e| BundleError::Read(e, path.clone())));
|
||||
try!(file.read_exact(&mut chunk_data).context(&path as &Path));
|
||||
if let Some(ref encryption) = header.encryption {
|
||||
chunk_data = try!(crypto.lock().unwrap().decrypt(encryption.clone(), &chunk_data));
|
||||
chunk_data = try!(crypto.lock().unwrap().decrypt(&encryption, &chunk_data).context(&path as &Path));
|
||||
}
|
||||
let chunks = ChunkList::read_from(&chunk_data);
|
||||
let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize;
|
||||
|
|
@ -234,20 +254,20 @@ impl Bundle {
|
|||
|
||||
#[inline]
|
||||
fn load_encoded_contents(&self) -> Result<Vec<u8>, BundleError> {
|
||||
let mut file = BufReader::new(try!(File::open(&self.path).map_err(|e| BundleError::Read(e, self.path.clone()))));
|
||||
try!(file.seek(SeekFrom::Start(self.content_start as u64)).map_err(|e| BundleError::Read(e, self.path.clone())));
|
||||
let mut file = BufReader::new(try!(File::open(&self.path).context(&self.path as &Path)));
|
||||
try!(file.seek(SeekFrom::Start(self.content_start as u64)).context(&self.path as &Path));
|
||||
let mut data = Vec::with_capacity(max(self.info.encoded_size, self.info.raw_size)+1024);
|
||||
try!(file.read_to_end(&mut data).map_err(|e| BundleError::Read(e, self.path.clone())));
|
||||
try!(file.read_to_end(&mut data).context(&self.path as &Path));
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn decode_contents(&self, mut data: Vec<u8>) -> Result<Vec<u8>, BundleError> {
|
||||
if let Some(ref encryption) = self.info.encryption {
|
||||
data = try!(self.crypto.lock().unwrap().decrypt(encryption.clone(), &data));
|
||||
data = try!(self.crypto.lock().unwrap().decrypt(&encryption, &data).context(&self.path as &Path));
|
||||
}
|
||||
if let Some(ref compression) = self.info.compression {
|
||||
data = try!(compression.decompress(&data));
|
||||
data = try!(compression.decompress(&data).context(&self.path as &Path));
|
||||
}
|
||||
Ok(data)
|
||||
}
|
||||
|
|
@ -276,8 +296,7 @@ impl Bundle {
|
|||
"Individual chunk sizes do not add up to total size"))
|
||||
}
|
||||
if !full {
|
||||
let size = try!(fs::metadata(&self.path).map_err(|e| BundleError::Read(e, self.path.clone()))
|
||||
).len();
|
||||
let size = try!(fs::metadata(&self.path).context(&self.path as &Path)).len();
|
||||
if size as usize != self.info.encoded_size + self.content_start {
|
||||
return Err(BundleError::Integrity(self.id(),
|
||||
"File size does not match size in header, truncated file"))
|
||||
|
|
@ -365,21 +384,21 @@ impl BundleWriter {
|
|||
try!(stream.finish(&mut self.data))
|
||||
}
|
||||
if let Some(ref encryption) = self.encryption {
|
||||
self.data = try!(self.crypto.lock().unwrap().encrypt(encryption.clone(), &self.data));
|
||||
self.data = try!(self.crypto.lock().unwrap().encrypt(&encryption, &self.data));
|
||||
}
|
||||
let encoded_size = self.data.len();
|
||||
let mut chunk_data = Vec::with_capacity(self.chunks.encoded_size());
|
||||
self.chunks.write_to(&mut chunk_data).unwrap();
|
||||
let id = BundleId(self.hash_method.hash(&chunk_data));
|
||||
if let Some(ref encryption) = self.encryption {
|
||||
chunk_data = try!(self.crypto.lock().unwrap().encrypt(encryption.clone(), &chunk_data));
|
||||
chunk_data = try!(self.crypto.lock().unwrap().encrypt(&encryption, &chunk_data));
|
||||
}
|
||||
let (folder, file) = db.bundle_path(&id);
|
||||
let path = folder.join(file);
|
||||
try!(fs::create_dir_all(&folder).map_err(|e| BundleError::Write(e, path.clone())));
|
||||
let mut file = BufWriter::new(try!(File::create(&path).map_err(|e| BundleError::Write(e, path.clone()))));
|
||||
try!(file.write_all(&HEADER_STRING).map_err(|e| BundleError::Write(e, path.clone())));
|
||||
try!(file.write_all(&[HEADER_VERSION]).map_err(|e| BundleError::Write(e, path.clone())));
|
||||
try!(fs::create_dir_all(&folder).context(&path as &Path));
|
||||
let mut file = BufWriter::new(try!(File::create(&path).context(&path as &Path)));
|
||||
try!(file.write_all(&HEADER_STRING).context(&path as &Path));
|
||||
try!(file.write_all(&[HEADER_VERSION]).context(&path as &Path));
|
||||
let header = BundleInfo {
|
||||
mode: self.mode,
|
||||
hash_method: self.hash_method,
|
||||
|
|
@ -393,9 +412,9 @@ impl BundleWriter {
|
|||
};
|
||||
try!(msgpack::encode_to_stream(&header, &mut file)
|
||||
.map_err(|e| BundleError::Encode(e, path.clone())));
|
||||
try!(file.write_all(&chunk_data).map_err(|e| BundleError::Write(e, path.clone())));
|
||||
try!(file.write_all(&chunk_data).context(&path as &Path));
|
||||
let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize;
|
||||
try!(file.write_all(&self.data).map_err(|e| BundleError::Write(e, path.clone())));
|
||||
try!(file.write_all(&self.data).context(&path as &Path));
|
||||
Ok(Bundle::new(path, HEADER_VERSION, content_start, self.crypto, header, self.chunks))
|
||||
}
|
||||
|
||||
|
|
@ -413,8 +432,6 @@ impl BundleWriter {
|
|||
|
||||
pub struct BundleDb {
|
||||
path: PathBuf,
|
||||
compression: Option<Compression>,
|
||||
encryption: Option<Encryption>,
|
||||
crypto: Arc<Mutex<Crypto>>,
|
||||
bundles: HashMap<BundleId, Bundle>,
|
||||
bundle_cache: LruCache<BundleId, Vec<u8>>
|
||||
|
|
@ -422,13 +439,10 @@ pub struct BundleDb {
|
|||
|
||||
|
||||
impl BundleDb {
|
||||
fn new(path: PathBuf, compression: Option<Compression>, encryption: Option<Encryption>) -> Self {
|
||||
fn new(path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Self {
|
||||
BundleDb {
|
||||
path: path,
|
||||
compression:
|
||||
compression,
|
||||
crypto: Arc::new(Mutex::new(Crypto::new())),
|
||||
encryption: encryption,
|
||||
crypto: crypto,
|
||||
bundles: HashMap::new(),
|
||||
bundle_cache: LruCache::new(5, 10)
|
||||
}
|
||||
|
|
@ -436,7 +450,7 @@ impl BundleDb {
|
|||
|
||||
fn bundle_path(&self, bundle: &BundleId) -> (PathBuf, PathBuf) {
|
||||
let mut folder = self.path.clone();
|
||||
let mut file = bundle.to_string()[0..32].to_owned() + ".bundle";
|
||||
let mut file = bundle.to_string().to_owned() + ".bundle";
|
||||
let mut count = self.bundles.len();
|
||||
while count >= 100 {
|
||||
if file.len() < 10 {
|
||||
|
|
@ -469,33 +483,29 @@ impl BundleDb {
|
|||
}
|
||||
|
||||
#[inline]
|
||||
pub fn open<P: AsRef<Path>>(path: P, compression: Option<Compression>, encryption: Option<Encryption>) -> Result<Self, BundleError> {
|
||||
pub fn open<P: AsRef<Path>>(path: P, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
|
||||
let path = path.as_ref().to_owned();
|
||||
let mut self_ = Self::new(path, compression, encryption);
|
||||
let mut self_ = Self::new(path, crypto);
|
||||
try!(self_.load_bundle_list());
|
||||
Ok(self_)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn create<P: AsRef<Path>>(path: P, compression: Option<Compression>, encryption: Option<Encryption>) -> Result<Self, BundleError> {
|
||||
pub fn create<P: AsRef<Path>>(path: P, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
|
||||
let path = path.as_ref().to_owned();
|
||||
try!(fs::create_dir_all(&path)
|
||||
.map_err(|e| BundleError::Write(e, path.clone())));
|
||||
Ok(Self::new(path, compression, encryption))
|
||||
try!(fs::create_dir_all(&path).context(&path as &Path));
|
||||
Ok(Self::new(path, crypto))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn open_or_create<P: AsRef<Path>>(path: P, compression: Option<Compression>, encryption: Option<Encryption>) -> Result<Self, BundleError> {
|
||||
if path.as_ref().exists() {
|
||||
Self::open(path, compression, encryption)
|
||||
} else {
|
||||
Self::create(path, compression, encryption)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn create_bundle(&self, mode: BundleMode, hash_method: HashMethod) -> Result<BundleWriter, BundleError> {
|
||||
BundleWriter::new(mode, hash_method, self.compression.clone(), self.encryption.clone(), self.crypto.clone())
|
||||
pub fn create_bundle(
|
||||
&self,
|
||||
mode: BundleMode,
|
||||
hash_method: HashMethod,
|
||||
compression: Option<Compression>,
|
||||
encryption: Option<Encryption>
|
||||
) -> Result<BundleWriter, BundleError> {
|
||||
BundleWriter::new(mode, hash_method, compression, encryption, self.crypto.clone())
|
||||
}
|
||||
|
||||
pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleError> {
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ impl IChunker for Chunker {
|
|||
}
|
||||
|
||||
|
||||
#[derive(Debug)]
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum ChunkerType {
|
||||
Ae(usize),
|
||||
Rabin((usize, u32)),
|
||||
|
|
@ -141,6 +141,11 @@ impl ChunkerType {
|
|||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn to_string(&self) -> String {
|
||||
format!("{}/{}", self.name(), self.avg_size()/1024)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn seed(&self) -> u64 {
|
||||
match *self {
|
||||
|
|
|
|||
|
|
@ -43,8 +43,9 @@ fn chunk(data: &[u8], mut chunker: Chunker, sink: &mut ChunkSink) {
|
|||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Option<Compression>, hash: HashMethod) {
|
||||
let mut total_time = 0.0;
|
||||
pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Option<Compression>, encrypt: bool,hash: HashMethod) {
|
||||
let mut total_write_time = 0.0;
|
||||
let mut total_read_time = 0.0;
|
||||
|
||||
println!("Reading input file ...");
|
||||
let mut file = File::open(path).unwrap();
|
||||
|
|
@ -68,7 +69,7 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
|
|||
let chunk_time = Duration::span(|| {
|
||||
chunk(&data, chunker, &mut chunk_sink)
|
||||
}).num_milliseconds() as f32 / 1_000.0;
|
||||
total_time += chunk_time;
|
||||
total_write_time += chunk_time;
|
||||
println!("- {}, {}", to_duration(chunk_time), to_speed(size, chunk_time));
|
||||
let mut chunks = chunk_sink.chunks;
|
||||
assert_eq!(chunks.iter().map(|c| c.1).sum::<usize>(), size as usize);
|
||||
|
|
@ -85,7 +86,7 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
|
|||
hashes.push(hash.hash(&data[pos..pos+len]))
|
||||
}
|
||||
}).num_milliseconds() as f32 / 1_000.0;
|
||||
total_time += hash_time;
|
||||
total_write_time += hash_time;
|
||||
println!("- {}, {}", to_duration(hash_time), to_speed(size, hash_time));
|
||||
let mut seen_hashes = HashSet::with_capacity(hashes.len());
|
||||
let mut dups = Vec::new();
|
||||
|
|
@ -103,11 +104,12 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
|
|||
println!("- {} duplicate chunks, {}, {:.1}% saved", dups.len(), to_file_size(dup_size as u64), dup_size as f32 / size as f32*100.0);
|
||||
size -= dup_size as u64;
|
||||
|
||||
if let Some(compression) = compression {
|
||||
let mut bundles = Vec::new();
|
||||
|
||||
if let Some(compression) = compression.clone() {
|
||||
println!();
|
||||
|
||||
println!("Compressing chunks with {} ...", compression.to_string());
|
||||
let mut bundles = Vec::new();
|
||||
let compress_time = Duration::span(|| {
|
||||
let mut bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize);
|
||||
let mut c = compression.compress_stream().unwrap();
|
||||
|
|
@ -123,12 +125,56 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
|
|||
c.finish(&mut bundle).unwrap();
|
||||
bundles.push(bundle);
|
||||
}).num_milliseconds() as f32 / 1_000.0;
|
||||
total_time += compress_time;
|
||||
total_write_time += compress_time;
|
||||
println!("- {}, {}", to_duration(compress_time), to_speed(size, compress_time));
|
||||
let compressed_size = bundles.iter().map(|b| b.len()).sum::<usize>();
|
||||
println!("- {} bundles, {}, {:.1}% saved", bundles.len(), to_file_size(compressed_size as u64), (size as f32 - compressed_size as f32)/size as f32*100.0);
|
||||
size = compressed_size as u64;
|
||||
} else {
|
||||
let mut bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize);
|
||||
for &(pos, len) in &chunks {
|
||||
bundle.extend_from_slice(&data[pos..pos+len]);
|
||||
if bundle.len() >= bundle_size {
|
||||
bundles.push(bundle);
|
||||
bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize);
|
||||
}
|
||||
}
|
||||
bundles.push(bundle);
|
||||
}
|
||||
|
||||
if encrypt {
|
||||
println!();
|
||||
|
||||
let (public, secret) = gen_keypair();
|
||||
let mut crypto = Crypto::new();
|
||||
crypto.add_secret_key(public, secret);
|
||||
let encryption = (EncryptionMethod::Sodium, public[..].iter().cloned().collect::<Vec<u8>>().into());
|
||||
|
||||
println!("Encrypting bundles...");
|
||||
let mut encrypted_bundles = Vec::with_capacity(bundles.len());
|
||||
|
||||
let encrypt_time = Duration::span(|| {
|
||||
for bundle in bundles {
|
||||
encrypted_bundles.push(crypto.encrypt(&encryption, &bundle).unwrap());
|
||||
}
|
||||
}).num_milliseconds() as f32 / 1_000.0;
|
||||
println!("- {}, {}", to_duration(encrypt_time), to_speed(size, encrypt_time));
|
||||
total_write_time += encrypt_time;
|
||||
|
||||
println!();
|
||||
|
||||
println!("Decrypting bundles...");
|
||||
bundles = Vec::with_capacity(encrypted_bundles.len());
|
||||
let decrypt_time = Duration::span(|| {
|
||||
for bundle in encrypted_bundles {
|
||||
bundles.push(crypto.decrypt(&encryption, &bundle).unwrap());
|
||||
}
|
||||
}).num_milliseconds() as f32 / 1_000.0;
|
||||
println!("- {}, {}", to_duration(decrypt_time), to_speed(size, decrypt_time));
|
||||
total_read_time += decrypt_time;
|
||||
}
|
||||
|
||||
if let Some(compression) = compression {
|
||||
println!();
|
||||
|
||||
println!("Decompressing bundles with {} ...", compression.to_string());
|
||||
|
|
@ -141,12 +187,12 @@ pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Op
|
|||
}
|
||||
}).num_milliseconds() as f32 / 1_000.0;
|
||||
println!("- {}, {}", to_duration(decompress_time), to_speed(size, decompress_time));
|
||||
total_read_time += decompress_time;
|
||||
}
|
||||
|
||||
println!();
|
||||
|
||||
let total_saved = total_size - size;
|
||||
println!("Total space saved: {}, {:.1}%", to_file_size(total_saved as u64), total_saved as f32/total_size as f32*100.0);
|
||||
println!("Total processing speed: {}", to_speed(total_size, total_time));
|
||||
|
||||
println!("Total storage size: {} / {}, ratio: {:.1}%", to_file_size(size as u64), to_file_size(total_size as u64), size as f32/total_size as f32*100.0);
|
||||
println!("Total processing speed: {}", to_speed(total_size, total_write_time));
|
||||
println!("Total read speed: {}", to_speed(total_size, total_read_time));
|
||||
}
|
||||
|
|
|
|||
166
src/cli/args.rs
166
src/cli/args.rs
|
|
@ -1,5 +1,5 @@
|
|||
use ::chunker::ChunkerType;
|
||||
use ::util::{Compression, HashMethod};
|
||||
use ::util::*;
|
||||
|
||||
use std::process::exit;
|
||||
|
||||
|
|
@ -10,6 +10,7 @@ pub enum Arguments {
|
|||
bundle_size: usize,
|
||||
chunker: ChunkerType,
|
||||
compression: Option<Compression>,
|
||||
encryption: bool,
|
||||
hash: HashMethod
|
||||
},
|
||||
Backup {
|
||||
|
|
@ -56,11 +57,27 @@ pub enum Arguments {
|
|||
repo_path: String,
|
||||
remote_path: String
|
||||
},
|
||||
Configure {
|
||||
repo_path: String,
|
||||
bundle_size: Option<usize>,
|
||||
chunker: Option<ChunkerType>,
|
||||
compression: Option<Option<Compression>>,
|
||||
encryption: Option<Option<PublicKey>>,
|
||||
hash: Option<HashMethod>
|
||||
},
|
||||
GenKey {
|
||||
},
|
||||
AddKey {
|
||||
repo_path: String,
|
||||
key_pair: Option<(PublicKey, SecretKey)>,
|
||||
set_default: bool
|
||||
},
|
||||
AlgoTest {
|
||||
file: String,
|
||||
bundle_size: usize,
|
||||
chunker: ChunkerType,
|
||||
compression: Option<Compression>,
|
||||
encrypt: bool,
|
||||
hash: HashMethod
|
||||
}
|
||||
}
|
||||
|
|
@ -93,17 +110,16 @@ fn parse_float(num: &str, name: &str) -> f64 {
|
|||
}
|
||||
|
||||
|
||||
fn parse_chunker(val: Option<&str>) -> ChunkerType {
|
||||
if let Ok(chunker) = ChunkerType::from_string(val.unwrap_or("fastcdc/8")) {
|
||||
fn parse_chunker(val: &str) -> ChunkerType {
|
||||
if let Ok(chunker) = ChunkerType::from_string(val) {
|
||||
chunker
|
||||
} else {
|
||||
error!("Invalid chunker method/size: {}", val.unwrap());
|
||||
error!("Invalid chunker method/size: {}", val);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_compression(val: Option<&str>) -> Option<Compression> {
|
||||
let val = val.unwrap_or("brotli/3");
|
||||
fn parse_compression(val: &str) -> Option<Compression> {
|
||||
if val == "none" {
|
||||
return None
|
||||
}
|
||||
|
|
@ -115,11 +131,43 @@ fn parse_compression(val: Option<&str>) -> Option<Compression> {
|
|||
}
|
||||
}
|
||||
|
||||
fn parse_hash(val: Option<&str>) -> HashMethod {
|
||||
if let Ok(hash) = HashMethod::from(val.unwrap_or("blake2")) {
|
||||
fn parse_public_key(val: &str) -> PublicKey {
|
||||
let bytes = match parse_hex(val) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(_) => {
|
||||
error!("Invalid key: {}", val);
|
||||
exit(1);
|
||||
}
|
||||
};
|
||||
if let Some(key) = PublicKey::from_slice(&bytes) {
|
||||
key
|
||||
} else {
|
||||
error!("Invalid key: {}", val);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_secret_key(val: &str) -> SecretKey {
|
||||
let bytes = match parse_hex(val) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(_) => {
|
||||
error!("Invalid key: {}", val);
|
||||
exit(1);
|
||||
}
|
||||
};
|
||||
if let Some(key) = SecretKey::from_slice(&bytes) {
|
||||
key
|
||||
} else {
|
||||
error!("Invalid key: {}", val);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_hash(val: &str) -> HashMethod {
|
||||
if let Ok(hash) = HashMethod::from(val) {
|
||||
hash
|
||||
} else {
|
||||
error!("Invalid hash method: {}", val.unwrap());
|
||||
error!("Invalid hash method: {}", val);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
|
@ -127,18 +175,19 @@ fn parse_hash(val: Option<&str>) -> HashMethod {
|
|||
|
||||
pub fn parse() -> Arguments {
|
||||
let args = clap_app!(zvault =>
|
||||
(version: env!("CARGO_PKG_VERSION"))
|
||||
(author: "Dennis Schwerdel <schwerdel@googlemail.com>")
|
||||
(about: "Deduplicating backup tool")
|
||||
(version: crate_version!())
|
||||
(author: crate_authors!(",\n"))
|
||||
(about: crate_description!())
|
||||
(@setting SubcommandRequiredElseHelp)
|
||||
(@setting GlobalVersion)
|
||||
(@setting VersionlessSubcommands)
|
||||
(@setting UnifiedHelpMessage)
|
||||
(@subcommand init =>
|
||||
(about: "initializes a new repository")
|
||||
(@arg bundle_size: --bundle-size +takes_value "maximal bundle size in MiB [default: 25]")
|
||||
(@arg bundle_size: --bundlesize +takes_value "maximal bundle size in MiB [default: 25]")
|
||||
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]")
|
||||
(@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]")
|
||||
(@arg encryption: --encryption -e "generate a keypair and enable encryption")
|
||||
(@arg hash: --hash +takes_value "hash method to use [default: blake2]")
|
||||
(@arg REPO: +required "path of the repository")
|
||||
)
|
||||
|
|
@ -184,11 +233,32 @@ pub fn parse() -> Arguments {
|
|||
(about: "displays information on a repository, a backup or a path in a backup")
|
||||
(@arg PATH: +required "repository[::backup[::subpath]] path")
|
||||
)
|
||||
(@subcommand algotest =>
|
||||
(about: "test a specific algorithm combination")
|
||||
(@arg bundle_size: --bundle-size +takes_value "maximal bundle size in MiB [default: 25]")
|
||||
(@subcommand configure =>
|
||||
(about: "changes the configuration")
|
||||
(@arg REPO: +required "path of the repository")
|
||||
(@arg bundle_size: --bundlesize +takes_value "maximal bundle size in MiB [default: 25]")
|
||||
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]")
|
||||
(@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]")
|
||||
(@arg encryption: --encryption -e +takes_value "the public key to use for encryption")
|
||||
(@arg hash: --hash +takes_value "hash method to use [default: blake2]")
|
||||
)
|
||||
(@subcommand genkey =>
|
||||
(about: "generates a new key pair")
|
||||
)
|
||||
(@subcommand addkey =>
|
||||
(about: "adds a key to the respository")
|
||||
(@arg REPO: +required "path of the repository")
|
||||
(@arg generate: --generate "generate a new key")
|
||||
(@arg set_default: --default "set this key as default")
|
||||
(@arg PUBLIC: +takes_value "the public key")
|
||||
(@arg SECRET: +takes_value "the secret key")
|
||||
)
|
||||
(@subcommand algotest =>
|
||||
(about: "test a specific algorithm combination")
|
||||
(@arg bundle_size: --bundlesize +takes_value "maximal bundle size in MiB [default: 25]")
|
||||
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]")
|
||||
(@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]")
|
||||
(@arg encrypt: --encrypt -e "enable encryption")
|
||||
(@arg hash: --hash +takes_value "hash method to use [default: blake2]")
|
||||
(@arg FILE: +required "the file to test the algorithms with")
|
||||
)
|
||||
|
|
@ -201,9 +271,10 @@ pub fn parse() -> Arguments {
|
|||
}
|
||||
return Arguments::Init {
|
||||
bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize,
|
||||
chunker: parse_chunker(args.value_of("chunker")),
|
||||
compression: parse_compression(args.value_of("compression")),
|
||||
hash: parse_hash(args.value_of("hash")),
|
||||
chunker: parse_chunker(args.value_of("chunker").unwrap_or("fastcdc/8")),
|
||||
compression: parse_compression(args.value_of("compression").unwrap_or("brotli/3")),
|
||||
encryption: args.is_present("encryption"),
|
||||
hash: parse_hash(args.value_of("hash").unwrap_or("blake2")),
|
||||
repo_path: repository.to_string(),
|
||||
}
|
||||
}
|
||||
|
|
@ -306,12 +377,63 @@ pub fn parse() -> Arguments {
|
|||
remote_path: args.value_of("REMOTE").unwrap().to_string()
|
||||
}
|
||||
}
|
||||
if let Some(args) = args.subcommand_matches("configure") {
|
||||
let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap());
|
||||
if backup.is_some() || inode.is_some() {
|
||||
println!("No backups or subpaths may be given here");
|
||||
exit(1);
|
||||
}
|
||||
return Arguments::Configure {
|
||||
bundle_size: args.value_of("bundle_size").map(|v| (parse_num(v, "Bundle size") * 1024 * 1024) as usize),
|
||||
chunker: args.value_of("chunker").map(|v| parse_chunker(v)),
|
||||
compression: args.value_of("compression").map(|v| parse_compression(v)),
|
||||
encryption: args.value_of("encryption").map(|v| {
|
||||
if v == "none" {
|
||||
None
|
||||
} else {
|
||||
Some(parse_public_key(v))
|
||||
}
|
||||
}),
|
||||
hash: args.value_of("hash").map(|v| parse_hash(v)),
|
||||
repo_path: repository.to_string(),
|
||||
}
|
||||
}
|
||||
if let Some(_args) = args.subcommand_matches("genkey") {
|
||||
return Arguments::GenKey {}
|
||||
}
|
||||
if let Some(args) = args.subcommand_matches("addkey") {
|
||||
let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap());
|
||||
if backup.is_some() || inode.is_some() {
|
||||
println!("No backups or subpaths may be given here");
|
||||
exit(1);
|
||||
}
|
||||
let generate = args.is_present("generate");
|
||||
if !generate && (!args.is_present("PUBLIC") || !args.is_present("SECRET")) {
|
||||
println!("Without --generate, a public and secret key must be given");
|
||||
exit(1);
|
||||
}
|
||||
if generate && (args.is_present("PUBLIC") || args.is_present("SECRET")) {
|
||||
println!("With --generate, no public or secret key may be given");
|
||||
exit(1);
|
||||
}
|
||||
let key_pair = if generate {
|
||||
None
|
||||
} else {
|
||||
Some((parse_public_key(args.value_of("PUBLIC").unwrap()), parse_secret_key(args.value_of("SECRET").unwrap())))
|
||||
};
|
||||
return Arguments::AddKey {
|
||||
repo_path: repository.to_string(),
|
||||
set_default: args.is_present("set_default"),
|
||||
key_pair: key_pair
|
||||
}
|
||||
}
|
||||
if let Some(args) = args.subcommand_matches("algotest") {
|
||||
return Arguments::AlgoTest {
|
||||
bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize,
|
||||
chunker: parse_chunker(args.value_of("chunker")),
|
||||
compression: parse_compression(args.value_of("compression")),
|
||||
hash: parse_hash(args.value_of("hash")),
|
||||
chunker: parse_chunker(args.value_of("chunker").unwrap_or("fastcdc/8")),
|
||||
compression: parse_compression(args.value_of("compression").unwrap_or("brotli/3")),
|
||||
encrypt: args.is_present("encrypt"),
|
||||
hash: parse_hash(args.value_of("hash").unwrap_or("blake2")),
|
||||
file: args.value_of("FILE").unwrap().to_string(),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
use log::{self, LogRecord, LogLevel, LogMetadata, LogLevelFilter};
|
||||
pub use log::SetLoggerError;
|
||||
|
||||
use ansi_term::{Color, Style};
|
||||
|
||||
|
||||
struct Logger;
|
||||
|
||||
impl log::Log for Logger {
|
||||
|
|
@ -10,7 +13,14 @@ impl log::Log for Logger {
|
|||
|
||||
fn log(&self, record: &LogRecord) {
|
||||
if self.enabled(record.metadata()) {
|
||||
println!("{} - {}", record.level(), record.args());
|
||||
let lvl = record.level();
|
||||
match lvl {
|
||||
LogLevel::Error => println!("{} - {}", Color::Red.bold().paint("error"), record.args()),
|
||||
LogLevel::Warn => println!("{} - {}", Color::Yellow.bold().paint("warning"), record.args()),
|
||||
LogLevel::Info => println!("{} - {}", Color::Green.bold().paint("info"), record.args()),
|
||||
LogLevel::Debug => println!("{} - {}", Style::new().bold().paint("debug"), record.args()),
|
||||
LogLevel::Trace => println!("{} - {}", "trace", record.args())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ use std::process::exit;
|
|||
|
||||
use ::repository::{Repository, Config, Backup};
|
||||
use ::util::cli::*;
|
||||
use ::util::*;
|
||||
use self::args::Arguments;
|
||||
|
||||
|
||||
|
|
@ -36,13 +37,22 @@ pub fn run() {
|
|||
exit(-1)
|
||||
}
|
||||
match args::parse() {
|
||||
Arguments::Init{repo_path, bundle_size, chunker, compression, hash} => {
|
||||
Repository::create(repo_path, Config {
|
||||
Arguments::Init{repo_path, bundle_size, chunker, compression, encryption, hash} => {
|
||||
let mut repo = Repository::create(repo_path, Config {
|
||||
bundle_size: bundle_size,
|
||||
chunker: chunker,
|
||||
compression: compression,
|
||||
encryption: None,
|
||||
hash: hash
|
||||
}).unwrap();
|
||||
if encryption {
|
||||
let (public, secret) = gen_keypair();
|
||||
println!("Public key: {}", to_hex(&public[..]));
|
||||
println!("Secret key: {}", to_hex(&secret[..]));
|
||||
repo.set_encryption(Some(&public));
|
||||
repo.register_key(public, secret).unwrap();
|
||||
repo.save_config().unwrap();
|
||||
}
|
||||
},
|
||||
Arguments::Backup{repo_path, backup_name, src_path, full} => {
|
||||
let mut repo = open_repository(&repo_path);
|
||||
|
|
@ -165,8 +175,63 @@ pub fn run() {
|
|||
error!("Import is not implemented yet");
|
||||
return
|
||||
},
|
||||
Arguments::AlgoTest{bundle_size, chunker, compression, hash, file} => {
|
||||
algotest::run(&file, bundle_size, chunker, compression, hash);
|
||||
Arguments::Configure{repo_path, bundle_size, chunker, compression, encryption, hash} => {
|
||||
let mut repo = open_repository(&repo_path);
|
||||
if let Some(bundle_size) = bundle_size {
|
||||
repo.config.bundle_size = bundle_size
|
||||
}
|
||||
if let Some(chunker) = chunker {
|
||||
warn!("Changing the chunker makes it impossible to use existing data for deduplication");
|
||||
repo.config.chunker = chunker
|
||||
}
|
||||
if let Some(compression) = compression {
|
||||
repo.config.compression = compression
|
||||
}
|
||||
if let Some(encryption) = encryption {
|
||||
repo.set_encryption(encryption.as_ref())
|
||||
}
|
||||
if let Some(hash) = hash {
|
||||
warn!("Changing the hash makes it impossible to use existing data for deduplication");
|
||||
repo.config.hash = hash
|
||||
}
|
||||
repo.save_config().unwrap();
|
||||
println!("Bundle size: {}", to_file_size(repo.config.bundle_size as u64));
|
||||
println!("Chunker: {}", repo.config.chunker.to_string());
|
||||
if let Some(ref compression) = repo.config.compression {
|
||||
println!("Compression: {}", compression.to_string());
|
||||
} else {
|
||||
println!("Compression: none");
|
||||
}
|
||||
if let Some(ref encryption) = repo.config.encryption {
|
||||
println!("Encryption: {}", to_hex(&encryption.1[..]));
|
||||
} else {
|
||||
println!("Encryption: none");
|
||||
}
|
||||
println!("Hash method: {}", repo.config.hash.name());
|
||||
},
|
||||
Arguments::GenKey{} => {
|
||||
let (public, secret) = gen_keypair();
|
||||
println!("Public key: {}", to_hex(&public[..]));
|
||||
println!("Secret key: {}", to_hex(&secret[..]));
|
||||
},
|
||||
Arguments::AddKey{repo_path, set_default, key_pair} => {
|
||||
let mut repo = open_repository(&repo_path);
|
||||
let (public, secret) = if let Some(key_pair) = key_pair {
|
||||
key_pair
|
||||
} else {
|
||||
let (public, secret) = gen_keypair();
|
||||
println!("Public key: {}", to_hex(&public[..]));
|
||||
println!("Secret key: {}", to_hex(&secret[..]));
|
||||
(public, secret)
|
||||
};
|
||||
if set_default {
|
||||
repo.set_encryption(Some(&public));
|
||||
repo.save_config().unwrap();
|
||||
}
|
||||
repo.register_key(public, secret).unwrap();
|
||||
},
|
||||
Arguments::AlgoTest{bundle_size, chunker, compression, encrypt, hash, file} => {
|
||||
algotest::run(&file, bundle_size, chunker, compression, encrypt, hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ extern crate chrono;
|
|||
#[macro_use] extern crate clap;
|
||||
#[macro_use] extern crate log;
|
||||
extern crate byteorder;
|
||||
extern crate sodiumoxide;
|
||||
extern crate ansi_term;
|
||||
|
||||
|
||||
pub mod util;
|
||||
|
|
@ -26,12 +28,11 @@ mod cli;
|
|||
// TODO: - Keep meta bundles also locally
|
||||
// TODO: - Load and compare remote bundles to bundle map
|
||||
// TODO: - Write backup files there as well
|
||||
// TODO: Store list of hashes and hash method in bundle
|
||||
// TODO: Remove backup subtrees
|
||||
// TODO: Recompress & combine bundles
|
||||
// TODO: Prune backups (based on age like attic)
|
||||
// TODO: Check backup integrity too
|
||||
// TODO: Encryption
|
||||
// TODO: Encrypt backup files too
|
||||
// TODO: list --tree
|
||||
// TODO: Partial backups
|
||||
// TODO: Import repository from remote folder
|
||||
|
|
|
|||
|
|
@ -46,7 +46,12 @@ impl Repository {
|
|||
};
|
||||
// ...alocate one if needed
|
||||
if writer.is_none() {
|
||||
*writer = Some(try!(self.bundles.create_bundle(mode, self.config.hash)));
|
||||
*writer = Some(try!(self.bundles.create_bundle(
|
||||
mode,
|
||||
self.config.hash,
|
||||
self.config.compression.clone(),
|
||||
self.config.encryption.clone()
|
||||
)));
|
||||
}
|
||||
debug_assert!(writer.is_some());
|
||||
let chunk_id;
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ quick_error!{
|
|||
from()
|
||||
cause(err)
|
||||
description("Yaml format error")
|
||||
display("Yaml format error: {}", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -89,9 +90,41 @@ impl Compression {
|
|||
}
|
||||
|
||||
|
||||
impl EncryptionMethod {
|
||||
#[inline]
|
||||
fn from_yaml(yaml: String) -> Result<Self, ConfigError> {
|
||||
EncryptionMethod::from_string(&yaml).map_err(|_| ConfigError::Parse("Invalid codec"))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn to_yaml(&self) -> String {
|
||||
self.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
struct EncryptionYaml {
|
||||
method: String,
|
||||
key: String
|
||||
}
|
||||
impl Default for EncryptionYaml {
|
||||
fn default() -> Self {
|
||||
EncryptionYaml {
|
||||
method: "sodium".to_string(),
|
||||
key: "".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
serde_impl!(EncryptionYaml(String) {
|
||||
method: String => "method",
|
||||
key: String => "key"
|
||||
});
|
||||
|
||||
|
||||
|
||||
struct ConfigYaml {
|
||||
compression: Option<String>,
|
||||
encryption: Option<EncryptionYaml>,
|
||||
bundle_size: usize,
|
||||
chunker: ChunkerYaml,
|
||||
hash: String,
|
||||
|
|
@ -100,6 +133,7 @@ impl Default for ConfigYaml {
|
|||
fn default() -> Self {
|
||||
ConfigYaml {
|
||||
compression: Some("brotli/5".to_string()),
|
||||
encryption: None,
|
||||
bundle_size: 25*1024*1024,
|
||||
chunker: ChunkerYaml::default(),
|
||||
hash: "blake2".to_string()
|
||||
|
|
@ -108,6 +142,7 @@ impl Default for ConfigYaml {
|
|||
}
|
||||
serde_impl!(ConfigYaml(String) {
|
||||
compression: Option<String> => "compression",
|
||||
encryption: Option<EncryptionYaml> => "encryption",
|
||||
bundle_size: usize => "bundle_size",
|
||||
chunker: ChunkerYaml => "chunker",
|
||||
hash: String => "hash"
|
||||
|
|
@ -118,6 +153,7 @@ serde_impl!(ConfigYaml(String) {
|
|||
#[derive(Debug)]
|
||||
pub struct Config {
|
||||
pub compression: Option<Compression>,
|
||||
pub encryption: Option<Encryption>,
|
||||
pub bundle_size: usize,
|
||||
pub chunker: ChunkerType,
|
||||
pub hash: HashMethod
|
||||
|
|
@ -129,8 +165,16 @@ impl Config {
|
|||
} else {
|
||||
None
|
||||
};
|
||||
let encryption = if let Some(e) = yaml.encryption {
|
||||
let method = try!(EncryptionMethod::from_yaml(e.method));
|
||||
let key = try!(parse_hex(&e.key).map_err(|_| ConfigError::Parse("Invalid public key")));
|
||||
Some((method, key.into()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(Config{
|
||||
compression: compression,
|
||||
encryption: encryption,
|
||||
bundle_size: yaml.bundle_size,
|
||||
chunker: try!(ChunkerType::from_yaml(yaml.chunker)),
|
||||
hash: try!(HashMethod::from_yaml(yaml.hash))
|
||||
|
|
@ -140,6 +184,7 @@ impl Config {
|
|||
fn to_yaml(&self) -> ConfigYaml {
|
||||
ConfigYaml {
|
||||
compression: self.compression.as_ref().map(|c| c.to_yaml()),
|
||||
encryption: self.encryption.as_ref().map(|e| EncryptionYaml{method: e.0.to_yaml(), key: to_hex(&e.1[..])}),
|
||||
bundle_size: self.bundle_size,
|
||||
chunker: self.chunker.to_yaml(),
|
||||
hash: self.hash.to_yaml()
|
||||
|
|
|
|||
|
|
@ -17,47 +17,62 @@ quick_error!{
|
|||
Io(err: io::Error) {
|
||||
from()
|
||||
cause(err)
|
||||
description("IO Error")
|
||||
description("IO error")
|
||||
display("IO error: {}", err)
|
||||
}
|
||||
Config(err: ConfigError) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Configuration error")
|
||||
display("Configuration error: {}", err)
|
||||
}
|
||||
BundleMap(err: BundleMapError) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Bundle map error")
|
||||
display("Bundle map error: {}", err)
|
||||
}
|
||||
Index(err: IndexError) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Index error")
|
||||
display("Index error: {}", err)
|
||||
}
|
||||
Bundle(err: BundleError) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Bundle error")
|
||||
display("Bundle error: {}", err)
|
||||
}
|
||||
Chunker(err: ChunkerError) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Chunker error")
|
||||
display("Chunker error: {}", err)
|
||||
}
|
||||
Decode(err: msgpack::DecodeError) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Failed to decode metadata")
|
||||
display("Failed to decode metadata: {}", err)
|
||||
}
|
||||
Encode(err: msgpack::EncodeError) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Failed to encode metadata")
|
||||
display("Failed to encode metadata: {}", err)
|
||||
}
|
||||
Integrity(err: RepositoryIntegrityError) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Integrity error")
|
||||
display("Integrity error: {}", err)
|
||||
}
|
||||
Encryption(err: EncryptionError) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Failed to load keys")
|
||||
display("Failed to load keys: {}", err)
|
||||
}
|
||||
InvalidFileType(path: PathBuf) {
|
||||
description("Invalid file type")
|
||||
|
|
|
|||
|
|
@ -11,10 +11,12 @@ use std::mem;
|
|||
use std::cmp::max;
|
||||
use std::path::{PathBuf, Path};
|
||||
use std::fs;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use super::index::Index;
|
||||
use super::bundle::{BundleDb, BundleWriter};
|
||||
use super::chunker::Chunker;
|
||||
use ::util::*;
|
||||
|
||||
pub use self::error::RepositoryError;
|
||||
pub use self::config::Config;
|
||||
|
|
@ -25,8 +27,9 @@ use self::bundle_map::BundleMap;
|
|||
|
||||
pub struct Repository {
|
||||
path: PathBuf,
|
||||
config: Config,
|
||||
pub config: Config,
|
||||
index: Index,
|
||||
crypto: Arc<Mutex<Crypto>>,
|
||||
bundle_map: BundleMap,
|
||||
next_content_bundle: u32,
|
||||
next_meta_bundle: u32,
|
||||
|
|
@ -41,10 +44,11 @@ impl Repository {
|
|||
pub fn create<P: AsRef<Path>>(path: P, config: Config) -> Result<Self, RepositoryError> {
|
||||
let path = path.as_ref().to_owned();
|
||||
try!(fs::create_dir(&path));
|
||||
try!(fs::create_dir(path.join("keys")));
|
||||
let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys")))));
|
||||
let bundles = try!(BundleDb::create(
|
||||
path.join("bundles"),
|
||||
config.compression.clone(),
|
||||
None, //FIXME: store encryption in config
|
||||
crypto.clone()
|
||||
));
|
||||
let index = try!(Index::create(&path.join("index")));
|
||||
try!(config.save(path.join("config.yaml")));
|
||||
|
|
@ -62,16 +66,17 @@ impl Repository {
|
|||
bundles: bundles,
|
||||
content_bundle: None,
|
||||
meta_bundle: None,
|
||||
crypto: crypto
|
||||
})
|
||||
}
|
||||
|
||||
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, RepositoryError> {
|
||||
let path = path.as_ref().to_owned();
|
||||
let config = try!(Config::load(path.join("config.yaml")));
|
||||
let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys")))));
|
||||
let bundles = try!(BundleDb::open(
|
||||
path.join("bundles"),
|
||||
config.compression.clone(),
|
||||
None, //FIXME: load encryption from config
|
||||
crypto.clone()
|
||||
));
|
||||
let index = try!(Index::open(&path.join("index")));
|
||||
let bundle_map = try!(BundleMap::load(path.join("bundles.map")));
|
||||
|
|
@ -80,6 +85,7 @@ impl Repository {
|
|||
chunker: config.chunker.create(),
|
||||
config: config,
|
||||
index: index,
|
||||
crypto: crypto,
|
||||
bundle_map: bundle_map,
|
||||
next_content_bundle: 0,
|
||||
next_meta_bundle: 0,
|
||||
|
|
@ -92,6 +98,27 @@ impl Repository {
|
|||
Ok(repo)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn register_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), RepositoryError> {
|
||||
Ok(try!(self.crypto.lock().unwrap().register_secret_key(public, secret)))
|
||||
}
|
||||
|
||||
pub fn save_config(&mut self) -> Result<(), RepositoryError> {
|
||||
try!(self.config.save(self.path.join("config.yaml")));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn set_encryption(&mut self, public: Option<&PublicKey>) {
|
||||
if let Some(key) = public {
|
||||
let mut key_bytes = Vec::new();
|
||||
key_bytes.extend_from_slice(&key[..]);
|
||||
self.config.encryption = Some((EncryptionMethod::Sodium, key_bytes.into()))
|
||||
} else {
|
||||
self.config.encryption = None
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn save_bundle_map(&self) -> Result<(), RepositoryError> {
|
||||
try!(self.bundle_map.save(self.path.join("bundles.map")));
|
||||
|
|
|
|||
|
|
@ -1,65 +1,175 @@
|
|||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::io;
|
||||
use std::fs::{self, File};
|
||||
|
||||
use serde_yaml;
|
||||
use serde::bytes::ByteBuf;
|
||||
|
||||
use sodiumoxide::crypto::sealedbox;
|
||||
pub use sodiumoxide::crypto::box_::{SecretKey, PublicKey, gen_keypair};
|
||||
|
||||
use ::util::*;
|
||||
|
||||
|
||||
quick_error!{
|
||||
#[derive(Debug)]
|
||||
pub enum EncryptionError {
|
||||
InvalidKey {
|
||||
description("Invalid key")
|
||||
}
|
||||
MissingKey(key: PublicKey) {
|
||||
description("Missing key")
|
||||
display("Missing key: {}", to_hex(&key[..]))
|
||||
}
|
||||
Operation(reason: &'static str) {
|
||||
description("Operation failed")
|
||||
display("Operation failed: {}", reason)
|
||||
}
|
||||
Io(err: io::Error) {
|
||||
from()
|
||||
cause(err)
|
||||
description("IO error")
|
||||
display("IO error: {}", err)
|
||||
}
|
||||
Yaml(err: serde_yaml::Error) {
|
||||
from()
|
||||
cause(err)
|
||||
description("Yaml format error")
|
||||
display("Yaml format error: {}", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#[derive(Clone)]
|
||||
#[derive(Clone, Debug, Eq, PartialEq, Hash)]
|
||||
#[allow(unknown_lints,non_camel_case_types)]
|
||||
pub enum EncryptionMethod {
|
||||
Dummy
|
||||
Sodium,
|
||||
}
|
||||
serde_impl!(EncryptionMethod(u64) {
|
||||
Dummy => 0
|
||||
Sodium => 0
|
||||
});
|
||||
|
||||
pub type EncryptionKey = Vec<u8>;
|
||||
impl EncryptionMethod {
|
||||
pub fn from_string(val: &str) -> Result<Self, &'static str> {
|
||||
match val {
|
||||
"sodium" => Ok(EncryptionMethod::Sodium),
|
||||
_ => Err("Unsupported encryption method")
|
||||
}
|
||||
}
|
||||
|
||||
pub type EncryptionKeyId = u64;
|
||||
pub fn to_string(&self) -> String {
|
||||
match *self {
|
||||
EncryptionMethod::Sodium => "sodium".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
pub type Encryption = (EncryptionMethod, ByteBuf);
|
||||
|
||||
|
||||
struct KeyfileYaml {
|
||||
public: String,
|
||||
secret: String
|
||||
}
|
||||
impl Default for KeyfileYaml {
|
||||
fn default() -> Self {
|
||||
KeyfileYaml {
|
||||
public: "".to_string(),
|
||||
secret: "".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
serde_impl!(KeyfileYaml(String) {
|
||||
public: String => "public",
|
||||
secret: String => "secret"
|
||||
});
|
||||
|
||||
impl KeyfileYaml {
|
||||
pub fn load<P: AsRef<Path>>(path: P) -> Result<Self, EncryptionError> {
|
||||
let f = try!(File::open(path));
|
||||
Ok(try!(serde_yaml::from_reader(f)))
|
||||
}
|
||||
|
||||
pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<(), EncryptionError> {
|
||||
let mut f = try!(File::create(path));
|
||||
Ok(try!(serde_yaml::to_writer(&mut f, &self)))
|
||||
}
|
||||
}
|
||||
|
||||
pub type Encryption = (EncryptionMethod, EncryptionKeyId);
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Crypto {
|
||||
keys: HashMap<EncryptionKeyId, EncryptionKey>
|
||||
path: PathBuf,
|
||||
keys: HashMap<PublicKey, SecretKey>
|
||||
}
|
||||
|
||||
impl Crypto {
|
||||
#[inline]
|
||||
pub fn new() -> Self {
|
||||
Crypto { keys: Default::default() }
|
||||
Crypto { path: PathBuf::new(), keys: HashMap::new() }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn register_key(&mut self, key: EncryptionKey, id: EncryptionKeyId) {
|
||||
self.keys.insert(id, key);
|
||||
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, EncryptionError> {
|
||||
let path = path.as_ref().to_owned();
|
||||
let mut keys: HashMap<PublicKey, SecretKey> = HashMap::default();
|
||||
for entry in try!(fs::read_dir(&path)) {
|
||||
let entry = try!(entry);
|
||||
let keyfile = try!(KeyfileYaml::load(entry.path()));
|
||||
let public = try!(parse_hex(&keyfile.public).map_err(|_| EncryptionError::InvalidKey));
|
||||
let public = try!(PublicKey::from_slice(&public).ok_or(EncryptionError::InvalidKey));
|
||||
let secret = try!(parse_hex(&keyfile.secret).map_err(|_| EncryptionError::InvalidKey));
|
||||
let secret = try!(SecretKey::from_slice(&secret).ok_or(EncryptionError::InvalidKey));
|
||||
keys.insert(public, secret);
|
||||
}
|
||||
Ok(Crypto { path: path, keys: keys })
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn contains_key(&mut self, id: EncryptionKeyId) -> bool {
|
||||
self.keys.contains_key(&id)
|
||||
pub fn add_secret_key(&mut self, public: PublicKey, secret: SecretKey) {
|
||||
self.keys.insert(public, secret);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn encrypt(&self, _enc: Encryption, _data: &[u8]) -> Result<Vec<u8>, EncryptionError> {
|
||||
unimplemented!()
|
||||
pub fn register_secret_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), EncryptionError> {
|
||||
let keyfile = KeyfileYaml { public: to_hex(&public[..]), secret: to_hex(&secret[..]) };
|
||||
let path = self.path.join(to_hex(&public[..]) + ".yaml");
|
||||
try!(keyfile.save(path));
|
||||
self.keys.insert(public, secret);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn decrypt(&self, _enc: Encryption, _data: &[u8]) -> Result<Vec<u8>, EncryptionError> {
|
||||
unimplemented!()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Crypto {
|
||||
#[inline]
|
||||
fn default() -> Self {
|
||||
Crypto::new()
|
||||
pub fn contains_secret_key(&mut self, public: &PublicKey) -> bool {
|
||||
self.keys.contains_key(public)
|
||||
}
|
||||
|
||||
fn get_secret_key(&self, public: &PublicKey) -> Result<&SecretKey, EncryptionError> {
|
||||
self.keys.get(public).ok_or_else(|| EncryptionError::MissingKey(*public))
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn encrypt(&self, enc: &Encryption, data: &[u8]) -> Result<Vec<u8>, EncryptionError> {
|
||||
let &(ref method, ref public) = enc;
|
||||
let public = try!(PublicKey::from_slice(public).ok_or(EncryptionError::InvalidKey));
|
||||
match *method {
|
||||
EncryptionMethod::Sodium => {
|
||||
Ok(sealedbox::seal(data, &public))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn decrypt(&self, enc: &Encryption, data: &[u8]) -> Result<Vec<u8>, EncryptionError> {
|
||||
let &(ref method, ref public) = enc;
|
||||
let public = try!(PublicKey::from_slice(public).ok_or(EncryptionError::InvalidKey));
|
||||
let secret = try!(self.get_secret_key(&public));
|
||||
match *method {
|
||||
EncryptionMethod::Sodium => {
|
||||
sealedbox::open(data, &public, secret).map_err(|_| EncryptionError::Operation("Decryption failed"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,3 +12,35 @@ pub use self::compression::*;
|
|||
pub use self::encryption::*;
|
||||
pub use self::hash::*;
|
||||
pub use self::lru_cache::*;
|
||||
|
||||
pub fn to_hex(data: &[u8]) -> String {
|
||||
data.iter().map(|b| format!("{:02x}", b)).collect::<Vec<String>>().join("")
|
||||
}
|
||||
|
||||
pub fn parse_hex(hex: &str) -> Result<Vec<u8>, ()> {
|
||||
let mut b = Vec::with_capacity(hex.len() / 2);
|
||||
let mut modulus = 0;
|
||||
let mut buf = 0;
|
||||
for (_, byte) in hex.bytes().enumerate() {
|
||||
buf <<= 4;
|
||||
match byte {
|
||||
b'A'...b'F' => buf |= byte - b'A' + 10,
|
||||
b'a'...b'f' => buf |= byte - b'a' + 10,
|
||||
b'0'...b'9' => buf |= byte - b'0',
|
||||
b' '|b'\r'|b'\n'|b'\t' => {
|
||||
buf >>= 4;
|
||||
continue
|
||||
}
|
||||
_ => return Err(()),
|
||||
}
|
||||
modulus += 1;
|
||||
if modulus == 2 {
|
||||
modulus = 0;
|
||||
b.push(buf);
|
||||
}
|
||||
}
|
||||
match modulus {
|
||||
0 => Ok(b.into_iter().collect()),
|
||||
_ => Err(()),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ pub fn encode<T: Serialize>(t: &T) -> Result<Vec<u8>, EncodeError> {
|
|||
Ok(data)
|
||||
}
|
||||
|
||||
pub fn encode_to_stream<T: Serialize>(t: T, w: &mut Write) -> Result<(), EncodeError> {
|
||||
pub fn encode_to_stream<T: Serialize>(t: &T, w: &mut Write) -> Result<(), EncodeError> {
|
||||
let mut writer = rmp_serde::Serializer::new(w);
|
||||
t.serialize(&mut writer)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue