dswd
/
zvault
mirror of https://github.com/dswd/zvault
1
0
Fork 0
Code Issues Releases Wiki Activity

Better errors

This commit is contained in:
Dennis Schwerdel 2017-03-22 09:19:16 +01:00
parent 365121cfe1
commit 1aca00c027
18 changed files with 703 additions and 416 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
src/bundledb/cache.rs Normal file
View File

@ -0,0 +1,106 @@
use ::prelude::*;
use std::path::{Path, PathBuf};
use std::fs::{self, File};
use std::io::{self, BufReader, BufWriter, Write, Read};
pub static CACHE_FILE_STRING: [u8; 7] = *b"zvault\x04";
pub static CACHE_FILE_VERSION: u8 = 1;
quick_error!{
#[derive(Debug)]
pub enum BundleCacheError {
Read(err: io::Error) {
cause(err)
description("Failed to read bundle cache")
display("Bundle cache error: failed to read bundle cache\n\tcaused by: {}", err)
}
Write(err: io::Error) {
cause(err)
description("Failed to write bundle cache")
display("Bundle cache error: failed to write bundle cache\n\tcaused by: {}", err)
}
WrongHeader {
description("Wrong header")
display("Bundle cache error: wrong header on bundle cache")
}
UnsupportedVersion(version: u8) {
description("Wrong version")
display("Bundle cache error: unsupported version: {}", version)
}
Decode(err: msgpack::DecodeError) {
from()
cause(err)
description("Failed to decode bundle cache")
display("Bundle cache error: failed to decode bundle cache\n\tcaused by: {}", err)
}
Encode(err: msgpack::EncodeError) {
from()
cause(err)
description("Failed to encode bundle cache")
display("Bundle cache error: failed to encode bundle cache\n\tcaused by: {}", err)
}
}
}
#[derive(Clone, Default)]
pub struct StoredBundle {
pub info: BundleInfo,
pub path: PathBuf
}
serde_impl!(StoredBundle(u64) {
info: BundleInfo => 0,
path: PathBuf => 1
});
impl StoredBundle {
#[inline]
pub fn id(&self) -> BundleId {
self.info.id.clone()
}
pub fn move_to<P: AsRef<Path>>(mut self, path: P) -> Result<Self, BundleDbError> {
let path = path.as_ref();
if fs::rename(&self.path, path).is_err() {
try!(fs::copy(&self.path, path).context(path));
try!(fs::remove_file(&self.path).context(&self.path as &Path));
}
self.path = path.to_path_buf();
Ok(self)
}
pub fn copy_to<P: AsRef<Path>>(&self, path: P) -> Result<Self, BundleDbError> {
let path = path.as_ref();
try!(fs::copy(&self.path, path).context(path));
let mut bundle = self.clone();
bundle.path = path.to_path_buf();
Ok(bundle)
}
pub fn read_list_from<P: AsRef<Path>>(path: P) -> Result<Vec<Self>, BundleCacheError> {
let path = path.as_ref();
let mut file = BufReader::new(try!(File::open(path).map_err(BundleCacheError::Read)));
let mut header = [0u8; 8];
try!(file.read_exact(&mut header).map_err(BundleCacheError::Read));
if header[..CACHE_FILE_STRING.len()] != CACHE_FILE_STRING {
return Err(BundleCacheError::WrongHeader)
}
let version = header[CACHE_FILE_STRING.len()];
if version != CACHE_FILE_VERSION {
return Err(BundleCacheError::UnsupportedVersion(version))
}
Ok(try!(msgpack::decode_from_stream(&mut file)))
}
pub fn save_list_to<P: AsRef<Path>>(list: &[Self], path: P) -> Result<(), BundleCacheError> {
let path = path.as_ref();
let mut file = BufWriter::new(try!(File::create(path).map_err(BundleCacheError::Write)));
try!(file.write_all(&CACHE_FILE_STRING).map_err(BundleCacheError::Write));
try!(file.write_all(&[CACHE_FILE_VERSION]).map_err(BundleCacheError::Write));
try!(msgpack::encode_to_stream(&list, &mut file));
Ok(())
}
}

156
src/bundledb/db.rs
View File

@ -3,13 +3,54 @@ use super::*;
use std::path::{Path, PathBuf};
use std::collections::{HashMap, HashSet};
use std::fs::{self, File};
use std::fs;
use std::sync::{Arc, Mutex};
use std::io::{BufReader, BufWriter, Read, Write};
use std::io;
pub static CACHE_FILE_STRING: [u8; 7] = *b"zvault\x04";
pub static CACHE_FILE_VERSION: u8 = 1;
quick_error!{
#[derive(Debug)]
pub enum BundleDbError {
ListBundles(err: io::Error) {
cause(err)
description("Failed to list bundles")
display("Bundle db error: failed to list bundles\n\tcaused by: {}", err)
}
Reader(err: BundleReaderError) {
from()
cause(err)
description("Failed to read bundle")
display("Bundle db error: failed to read bundle\n\tcaused by: {}", err)
}
Writer(err: BundleWriterError) {
from()
cause(err)
description("Failed to write bundle")
display("Bundle db error: failed to write bundle\n\tcaused by: {}", err)
}
Cache(err: BundleCacheError) {
from()
cause(err)
description("Failed to read/write bundle cache")
display("Bundle db error: failed to read/write bundle cache\n\tcaused by: {}", err)
}
Io(err: io::Error, path: PathBuf) {
cause(err)
context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf())
description("Io error")
display("Bundle db error: io error on {:?}\n\tcaused by: {}", path, err)
}
NoSuchBundle(bundle: BundleId) {
description("No such bundle")
display("Bundle db error: no such bundle: {:?}", bundle)
}
Remove(err: io::Error, bundle: BundleId) {
cause(err)
description("Failed to remove bundle")
display("Bundle db error: failed to remove bundle {}\n\tcaused by: {}", bundle, err)
}
}
}
pub fn bundle_path(bundle: &BundleId, mut folder: PathBuf, mut count: usize) -> (PathBuf, PathBuf) {
@ -25,12 +66,12 @@ pub fn bundle_path(bundle: &BundleId, mut folder: PathBuf, mut count: usize) ->
(folder, file.into())
}
pub fn load_bundles<P: AsRef<Path>>(path: P, bundles: &mut HashMap<BundleId, StoredBundle>) -> Result<(Vec<BundleId>, Vec<BundleInfo>), BundleError> {
pub fn load_bundles<P: AsRef<Path>>(path: P, bundles: &mut HashMap<BundleId, StoredBundle>) -> Result<(Vec<BundleId>, Vec<BundleInfo>), BundleDbError> {
let mut paths = vec![path.as_ref().to_path_buf()];
let mut bundle_paths = HashSet::new();
while let Some(path) = paths.pop() {
for entry in try!(fs::read_dir(path).map_err(BundleError::List)) {
let entry = try!(entry.map_err(BundleError::List));
for entry in try!(fs::read_dir(path).map_err(BundleDbError::ListBundles)) {
let entry = try!(entry.map_err(BundleDbError::ListBundles));
let path = entry.path();
if path.is_dir() {
paths.push(path);
@ -51,7 +92,7 @@ pub fn load_bundles<P: AsRef<Path>>(path: P, bundles: &mut HashMap<BundleId, Sto
let mut new = vec![];
for path in bundle_paths {
let bundle = StoredBundle {
info: try!(Bundle::load_info(&path)),
info: try!(BundleReader::load_info(&path)),
path: path
};
new.push(bundle.info.id.clone());
@ -61,65 +102,6 @@ pub fn load_bundles<P: AsRef<Path>>(path: P, bundles: &mut HashMap<BundleId, Sto
}
#[derive(Clone, Default)]
pub struct StoredBundle {
pub info: BundleInfo,
pub path: PathBuf
}
serde_impl!(StoredBundle(u64) {
info: BundleInfo => 0,
path: PathBuf => 1
});
impl StoredBundle {
#[inline]
pub fn id(&self) -> BundleId {
self.info.id.clone()
}
pub fn move_to<P: AsRef<Path>>(mut self, path: P) -> Result<Self, BundleError> {
let path = path.as_ref();
if fs::rename(&self.path, path).is_err() {
try!(fs::copy(&self.path, path).context(path));
try!(fs::remove_file(&self.path).context(&self.path as &Path));
}
self.path = path.to_path_buf();
Ok(self)
}
pub fn copy_to<P: AsRef<Path>>(&self, path: P) -> Result<Self, BundleError> {
let path = path.as_ref();
try!(fs::copy(&self.path, path).context(path));
let mut bundle = self.clone();
bundle.path = path.to_path_buf();
Ok(bundle)
}
pub fn read_list_from<P: AsRef<Path>>(path: P) -> Result<Vec<Self>, BundleError> {
let path = path.as_ref();
let mut file = BufReader::new(try!(File::open(path).context(path)));
let mut header = [0u8; 8];
try!(file.read_exact(&mut header).context(&path as &Path));
if header[..CACHE_FILE_STRING.len()] != CACHE_FILE_STRING {
return Err(BundleError::WrongHeader(path.to_path_buf()))
}
let version = header[HEADER_STRING.len()];
if version != HEADER_VERSION {
return Err(BundleError::WrongVersion(path.to_path_buf(), version))
}
Ok(try!(msgpack::decode_from_stream(&mut file).context(path)))
}
pub fn save_list_to<P: AsRef<Path>>(list: &[Self], path: P) -> Result<(), BundleError> {
let path = path.as_ref();
let mut file = BufWriter::new(try!(File::create(path).context(path)));
try!(file.write_all(&HEADER_STRING).context(path));
try!(file.write_all(&[HEADER_VERSION]).context(path));
try!(msgpack::encode_to_stream(&list, &mut file).context(path));
Ok(())
}
}
pub struct BundleDb {
remote_path: PathBuf,
@ -129,7 +111,7 @@ pub struct BundleDb {
crypto: Arc<Mutex<Crypto>>,
local_bundles: HashMap<BundleId, StoredBundle>,
remote_bundles: HashMap<BundleId, StoredBundle>,
bundle_cache: LruCache<BundleId, (Bundle, Vec<u8>)>
bundle_cache: LruCache<BundleId, (BundleReader, Vec<u8>)>
}
@ -147,7 +129,7 @@ impl BundleDb {
}
}
fn load_bundle_list(&mut self) -> Result<(Vec<BundleId>, Vec<BundleInfo>), BundleError> {
fn load_bundle_list(&mut self) -> Result<(Vec<BundleId>, Vec<BundleInfo>), BundleDbError> {
let bundle_info_cache = &self.remote_cache_path;
if let Ok(list) = StoredBundle::read_list_from(&bundle_info_cache) {
for bundle in list {
@ -168,7 +150,7 @@ impl BundleDb {
}
#[inline]
pub fn open<R: AsRef<Path>, L: AsRef<Path>>(remote_path: R, local_path: L, crypto: Arc<Mutex<Crypto>>) -> Result<(Self, Vec<BundleId>, Vec<BundleInfo>), BundleError> {
pub fn open<R: AsRef<Path>, L: AsRef<Path>>(remote_path: R, local_path: L, crypto: Arc<Mutex<Crypto>>) -> Result<(Self, Vec<BundleId>, Vec<BundleInfo>), BundleDbError> {
let remote_path = remote_path.as_ref().to_owned();
let local_path = local_path.as_ref().to_owned();
let mut self_ = Self::new(remote_path, local_path, crypto);
@ -177,7 +159,7 @@ impl BundleDb {
}
#[inline]
pub fn create<R: AsRef<Path>, L: AsRef<Path>>(remote_path: R, local_path: L, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
pub fn create<R: AsRef<Path>, L: AsRef<Path>>(remote_path: R, local_path: L, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleDbError> {
let remote_path = remote_path.as_ref().to_owned();
let local_path = local_path.as_ref().to_owned();
let self_ = Self::new(remote_path, local_path, crypto);
@ -188,29 +170,23 @@ impl BundleDb {
}
#[inline]
pub fn create_bundle(
&self,
mode: BundleMode,
hash_method: HashMethod,
compression: Option<Compression>,
encryption: Option<Encryption>
) -> Result<BundleWriter, BundleError> {
BundleWriter::new(mode, hash_method, compression, encryption, self.crypto.clone())
pub fn create_bundle(&self, mode: BundleMode, hash_method: HashMethod, compression: Option<Compression>, encryption: Option<Encryption>) -> Result<BundleWriter, BundleDbError> {
Ok(try!(BundleWriter::new(mode, hash_method, compression, encryption, self.crypto.clone())))
}
fn get_stored_bundle(&self, bundle_id: &BundleId) -> Result<&StoredBundle, BundleError> {
fn get_stored_bundle(&self, bundle_id: &BundleId) -> Result<&StoredBundle, BundleDbError> {
if let Some(stored) = self.local_bundles.get(bundle_id).or_else(|| self.remote_bundles.get(bundle_id)) {
Ok(stored)
} else {
Err(BundleError::NoSuchBundle(bundle_id.clone()))
Err(BundleDbError::NoSuchBundle(bundle_id.clone()))
}
}
fn get_bundle(&self, stored: &StoredBundle) -> Result<Bundle, BundleError> {
Bundle::load(stored.path.clone(), self.crypto.clone())
fn get_bundle(&self, stored: &StoredBundle) -> Result<BundleReader, BundleDbError> {
Ok(try!(BundleReader::load(stored.path.clone(), self.crypto.clone())))
}
pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleError> {
pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleDbError> {
if let Some(&mut (ref mut bundle, ref data)) = self.bundle_cache.get_mut(bundle_id) {
let (pos, len) = try!(bundle.get_chunk_position(id));
let mut chunk = Vec::with_capacity(len);
@ -227,7 +203,7 @@ impl BundleDb {
}
#[inline]
pub fn add_bundle(&mut self, bundle: BundleWriter) -> Result<BundleInfo, BundleError> {
pub fn add_bundle(&mut self, bundle: BundleWriter) -> Result<BundleInfo, BundleDbError> {
let bundle = try!(bundle.finish(&self));
let id = bundle.id();
if bundle.info.mode == BundleMode::Meta {
@ -254,19 +230,19 @@ impl BundleDb {
}
#[inline]
pub fn delete_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleError> {
pub fn delete_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> {
if let Some(bundle) = self.local_bundles.remove(bundle) {
try!(fs::remove_file(&bundle.path).map_err(|e| BundleError::Remove(e, bundle.id())))
try!(fs::remove_file(&bundle.path).map_err(|e| BundleDbError::Remove(e, bundle.id())))
}
if let Some(bundle) = self.remote_bundles.remove(bundle) {
fs::remove_file(&bundle.path).map_err(|e| BundleError::Remove(e, bundle.id()))
fs::remove_file(&bundle.path).map_err(|e| BundleDbError::Remove(e, bundle.id()))
} else {
Err(BundleError::NoSuchBundle(bundle.clone()))
Err(BundleDbError::NoSuchBundle(bundle.clone()))
}
}
#[inline]
pub fn check(&mut self, full: bool) -> Result<(), BundleError> {
pub fn check(&mut self, full: bool) -> Result<(), BundleDbError> {
for stored in self.remote_bundles.values() {
let mut bundle = try!(self.get_bundle(stored));
try!(bundle.check(full))

15
src/bundledb/error.rs
View File

@ -4,9 +4,10 @@ use super::*;
use std::path::{Path, PathBuf};
use std::io;
quick_error!{
#[derive(Debug)]
pub enum BundleError {
pub enum BundleDbError {
List(err: io::Error) {
cause(err)
description("Failed to list bundles")
@ -79,5 +80,17 @@ quick_error!{
description("Failed to remove bundle")
display("Failed to remove bundle {}", bundle)
}
Writer(err: BundleWriterError) {
from()
cause(err)
description("Failed to write new bundle")
display("Bundle db error: failed to write new bundle\n\tcaused by: {}", err)
}
Reader(err: BundleReaderError) {
from()
cause(err)
description("Failed to read bundle")
display("Bundle db error: failed to read a bundle\n\tcaused by: {}", err)
}
}
}

105
src/bundledb/mod.rs
View File

@ -1,12 +1,107 @@
mod error;
mod writer;
mod bundle;
mod reader;
mod db;
mod cache;
pub use self::error::BundleError;
pub use self::writer::BundleWriter;
pub use self::bundle::*;
pub use self::cache::{StoredBundle, BundleCacheError};
pub use self::writer::{BundleWriter, BundleWriterError};
pub use self::reader::{BundleReader, BundleReaderError};
pub use self::db::*;
use ::prelude::*;
use std::fmt;
use serde;
pub static HEADER_STRING: [u8; 7] = *b"zvault\x01";
pub static HEADER_VERSION: u8 = 1;
#[derive(Hash, PartialEq, Eq, Clone, Default)]
pub struct BundleId(pub Hash);
impl Serialize for BundleId {
fn serialize<S: serde::Serializer>(&self, ser: S) -> Result<S::Ok, S::Error> {
self.0.serialize(ser)
}
}
impl Deserialize for BundleId {
fn deserialize<D: serde::Deserializer>(de: D) -> Result<Self, D::Error> {
let hash = try!(Hash::deserialize(de));
Ok(BundleId(hash))
}
}
impl BundleId {
#[inline]
fn to_string(&self) -> String {
self.0.to_string()
}
}
impl fmt::Display for BundleId {
#[inline]
fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
write!(fmt, "{}", self.to_string())
}
}
impl fmt::Debug for BundleId {
#[inline]
fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
write!(fmt, "{}", self.to_string())
}
}
#[derive(Eq, Debug, PartialEq, Clone, Copy)]
pub enum BundleMode {
Content, Meta
}
serde_impl!(BundleMode(u8) {
Content => 0,
Meta => 1
});
#[derive(Clone)]
pub struct BundleInfo {
pub id: BundleId,
pub mode: BundleMode,
pub compression: Option<Compression>,
pub encryption: Option<Encryption>,
pub hash_method: HashMethod,
pub raw_size: usize,
pub encoded_size: usize,
pub chunk_count: usize,
pub chunk_info_size: usize
}
serde_impl!(BundleInfo(u64) {
id: BundleId => 0,
mode: BundleMode => 1,
compression: Option<Compression> => 2,
encryption: Option<Encryption> => 3,
hash_method: HashMethod => 4,
raw_size: usize => 6,
encoded_size: usize => 7,
chunk_count: usize => 8,
chunk_info_size: usize => 9
});
impl Default for BundleInfo {
fn default() -> Self {
BundleInfo {
id: BundleId(Hash::empty()),
compression: None,
encryption: None,
hash_method: HashMethod::Blake2,
raw_size: 0,
encoded_size: 0,
chunk_count: 0,
mode: BundleMode::Content,
chunk_info_size: 0
}
}
}

174
src/bundledb/bundle.rs → src/bundledb/reader.rs
View File

@ -3,104 +3,60 @@ use super::*;
use std::path::{Path, PathBuf};
use std::fs::{self, File};
use std::io::{Read, Seek, SeekFrom, BufReader};
use std::io::{self, Read, Seek, SeekFrom, BufReader};
use std::cmp::max;
use std::fmt::{self, Debug};
use std::sync::{Arc, Mutex};
use serde;
#[derive(Hash, PartialEq, Eq, Clone, Default)]
pub struct BundleId(pub Hash);
impl Serialize for BundleId {
fn serialize<S: serde::Serializer>(&self, ser: S) -> Result<S::Ok, S::Error> {
self.0.serialize(ser)
}
}
impl Deserialize for BundleId {
fn deserialize<D: serde::Deserializer>(de: D) -> Result<Self, D::Error> {
let hash = try!(Hash::deserialize(de));
Ok(BundleId(hash))
}
}
impl BundleId {
#[inline]
fn to_string(&self) -> String {
self.0.to_string()
}
}
impl fmt::Display for BundleId {
#[inline]
fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
write!(fmt, "{}", self.to_string())
}
}
impl fmt::Debug for BundleId {
#[inline]
fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
write!(fmt, "{}", self.to_string())
}
}
#[derive(Eq, Debug, PartialEq, Clone, Copy)]
pub enum BundleMode {
Content, Meta
}
serde_impl!(BundleMode(u8) {
Content => 0,
Meta => 1
});
#[derive(Clone)]
pub struct BundleInfo {
pub id: BundleId,
pub mode: BundleMode,
pub compression: Option<Compression>,
pub encryption: Option<Encryption>,
pub hash_method: HashMethod,
pub raw_size: usize,
pub encoded_size: usize,
pub chunk_count: usize,
pub chunk_info_size: usize
}
serde_impl!(BundleInfo(u64) {
id: BundleId => 0,
mode: BundleMode => 1,
compression: Option<Compression> => 2,
encryption: Option<Encryption> => 3,
hash_method: HashMethod => 4,
raw_size: usize => 6,
encoded_size: usize => 7,
chunk_count: usize => 8,
chunk_info_size: usize => 9
});
impl Default for BundleInfo {
fn default() -> Self {
BundleInfo {
id: BundleId(Hash::empty()),
compression: None,
encryption: None,
hash_method: HashMethod::Blake2,
raw_size: 0,
encoded_size: 0,
chunk_count: 0,
mode: BundleMode::Content,
chunk_info_size: 0
quick_error!{
#[derive(Debug)]
pub enum BundleReaderError {
Read(err: io::Error, path: PathBuf) {
cause(err)
context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf())
description("Failed to read data from file")
display("Bundle reader error: failed to read data from file {:?}\n\tcaused by: {}", path, err)
}
WrongHeader(path: PathBuf) {
description("Wrong header")
display("Bundle reader error: wrong header on bundle {:?}", path)
}
UnsupportedVersion(path: PathBuf, version: u8) {
description("Wrong version")
display("Bundle reader error: unsupported version on bundle {:?}: {}", path, version)
}
NoSuchChunk(bundle: BundleId, id: usize) {
description("Bundle has no such chunk")
display("Bundle reader error: bundle {:?} has no chunk with id {}", bundle, id)
}
Decode(err: msgpack::DecodeError, path: PathBuf) {
cause(err)
context(path: &'a Path, err: msgpack::DecodeError) -> (err, path.to_path_buf())
description("Failed to decode bundle header")
display("Bundle reader error: failed to decode bundle header of {:?}\n\tcaused by: {}", path, err)
}
Decompression(err: CompressionError, path: PathBuf) {
cause(err)
context(path: &'a Path, err: CompressionError) -> (err, path.to_path_buf())
description("Decompression failed")
display("Bundle reader error: decompression failed on bundle {:?}\n\tcaused by: {}", path, err)
}
Decryption(err: EncryptionError, path: PathBuf) {
cause(err)
context(path: &'a Path, err: EncryptionError) -> (err, path.to_path_buf())
description("Decryption failed")
display("Bundle reader error: decryption failed on bundle {:?}\n\tcaused by: {}", path, err)
}
Integrity(bundle: BundleId, reason: &'static str) {
description("Bundle has an integrity error")
display("Bundle reader error: bundle {:?} has an integrity error: {}", bundle, reason)
}
}
}
pub struct Bundle {
pub struct BundleReader {
pub info: BundleInfo,
pub version: u8,
pub path: PathBuf,
@ -110,9 +66,9 @@ pub struct Bundle {
pub chunk_positions: Option<Vec<usize>>
}
impl Bundle {
impl BundleReader {
pub fn new(path: PathBuf, version: u8, content_start: usize, crypto: Arc<Mutex<Crypto>>, info: BundleInfo) -> Self {
Bundle {
BundleReader {
info: info,
chunks: None,
version: version,
@ -128,17 +84,17 @@ impl Bundle {
self.info.id.clone()
}
fn load_header<P: AsRef<Path>>(path: P) -> Result<(BundleInfo, u8, usize), BundleError> {
fn load_header<P: AsRef<Path>>(path: P) -> Result<(BundleInfo, u8, usize), BundleReaderError> {
let path = path.as_ref();
let mut file = BufReader::new(try!(File::open(path).context(path)));
let mut header = [0u8; 8];
try!(file.read_exact(&mut header).context(path));
if header[..HEADER_STRING.len()] != HEADER_STRING {
return Err(BundleError::WrongHeader(path.to_path_buf()))
return Err(BundleReaderError::WrongHeader(path.to_path_buf()))
}
let version = header[HEADER_STRING.len()];
if version != HEADER_VERSION {
return Err(BundleError::WrongVersion(path.to_path_buf(), version))
return Err(BundleReaderError::UnsupportedVersion(path.to_path_buf(), version))
}
let header: BundleInfo = try!(msgpack::decode_from_stream(&mut file).context(path));
debug!("Load bundle {}", header.id);
@ -147,17 +103,17 @@ impl Bundle {
}
#[inline]
pub fn load_info<P: AsRef<Path>>(path: P) -> Result<BundleInfo, BundleError> {
pub fn load_info<P: AsRef<Path>>(path: P) -> Result<BundleInfo, BundleReaderError> {
Self::load_header(path).map(|b| b.0)
}
#[inline]
pub fn load(path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
pub fn load(path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleReaderError> {
let (header, version, content_start) = try!(Self::load_header(&path));
Ok(Bundle::new(path, version, content_start, crypto, header))
Ok(BundleReader::new(path, version, content_start, crypto, header))
}
pub fn load_chunklist(&mut self) -> Result<(), BundleError> {
pub fn load_chunklist(&mut self) -> Result<(), BundleReaderError> {
debug!("Load bundle chunklist {} ({:?})", self.info.id, self.info.mode);
let mut file = BufReader::new(try!(File::open(&self.path).context(&self.path as &Path)));
let len = self.info.chunk_info_size;
@ -182,7 +138,7 @@ impl Bundle {
}
#[inline]
fn load_encoded_contents(&self) -> Result<Vec<u8>, BundleError> {
fn load_encoded_contents(&self) -> Result<Vec<u8>, BundleReaderError> {
debug!("Load bundle data {} ({:?})", self.info.id, self.info.mode);
let mut file = BufReader::new(try!(File::open(&self.path).context(&self.path as &Path)));
try!(file.seek(SeekFrom::Start(self.content_start as u64)).context(&self.path as &Path));
@ -192,7 +148,7 @@ impl Bundle {
}
#[inline]
fn decode_contents(&self, mut data: Vec<u8>) -> Result<Vec<u8>, BundleError> {
fn decode_contents(&self, mut data: Vec<u8>) -> Result<Vec<u8>, BundleReaderError> {
if let Some(ref encryption) = self.info.encryption {
data = try!(self.crypto.lock().unwrap().decrypt(&encryption, &data).context(&self.path as &Path));
}
@ -203,14 +159,14 @@ impl Bundle {
}
#[inline]
pub fn load_contents(&self) -> Result<Vec<u8>, BundleError> {
pub fn load_contents(&self) -> Result<Vec<u8>, BundleReaderError> {
self.load_encoded_contents().and_then(|data| self.decode_contents(data))
}
#[inline]
pub fn get_chunk_position(&mut self, id: usize) -> Result<(usize, usize), BundleError> {
pub fn get_chunk_position(&mut self, id: usize) -> Result<(usize, usize), BundleReaderError> {
if id >= self.info.chunk_count {
return Err(BundleError::NoSuchChunk(self.id(), id))
return Err(BundleReaderError::NoSuchChunk(self.id(), id))
}
if self.chunks.is_none() || self.chunk_positions.is_none() {
try!(self.load_chunklist());
@ -220,34 +176,34 @@ impl Bundle {
Ok((pos, len))
}
pub fn check(&mut self, full: bool) -> Result<(), BundleError> {
pub fn check(&mut self, full: bool) -> Result<(), BundleReaderError> {
if self.chunks.is_none() || self.chunk_positions.is_none() {
try!(self.load_chunklist());
}
if self.info.chunk_count != self.chunks.as_ref().unwrap().len() {
return Err(BundleError::Integrity(self.id(),
return Err(BundleReaderError::Integrity(self.id(),
"Chunk list size does not match chunk count"))
}
if self.chunks.as_ref().unwrap().iter().map(|c| c.1 as usize).sum::<usize>() != self.info.raw_size {
return Err(BundleError::Integrity(self.id(),
return Err(BundleReaderError::Integrity(self.id(),
"Individual chunk sizes do not add up to total size"))
}
if !full {
let size = try!(fs::metadata(&self.path).context(&self.path as &Path)).len();
if size as usize != self.info.encoded_size + self.content_start {
return Err(BundleError::Integrity(self.id(),
return Err(BundleReaderError::Integrity(self.id(),
"File size does not match size in header, truncated file"))
}
return Ok(())
}
let encoded_contents = try!(self.load_encoded_contents());
if self.info.encoded_size != encoded_contents.len() {
return Err(BundleError::Integrity(self.id(),
return Err(BundleReaderError::Integrity(self.id(),
"Encoded data size does not match size in header, truncated bundle"))
}
let contents = try!(self.decode_contents(encoded_contents));
if self.info.raw_size != contents.len() {
return Err(BundleError::Integrity(self.id(),
return Err(BundleReaderError::Integrity(self.id(),
"Raw data size does not match size in header, truncated bundle"))
}
//TODO: verify checksum
@ -255,7 +211,7 @@ impl Bundle {
}
}
impl Debug for Bundle {
impl Debug for BundleReader {
fn fmt(&self, fmt: &mut fmt::Formatter) -> Result<(), fmt::Error> {
write!(fmt, "Bundle(\n\tid: {}\n\tpath: {:?}\n\tchunks: {}\n\tsize: {}, encoded: {}\n\tcompression: {:?}\n)",
self.info.id.to_string(), self.path, self.info.chunk_count, self.info.raw_size,

57
src/bundledb/writer.rs
View File

@ -1,12 +1,47 @@
use ::prelude::*;
use super::*;
use std::path::Path;
use std::path::{Path, PathBuf};
use std::fs::File;
use std::io::{Write, BufWriter};
use std::io::{self, Write, BufWriter};
use std::sync::{Arc, Mutex};
quick_error!{
#[derive(Debug)]
pub enum BundleWriterError {
CompressionSetup(err: CompressionError) {
cause(err)
description("Failed to setup compression")
display("Bundle writer error: failed to setup compression\n\tcaused by: {}", err)
}
Compression(err: CompressionError) {
cause(err)
description("Failed to compress data")
display("Bundle writer error: failed to compress data\n\tcaused by: {}", err)
}
Encryption(err: EncryptionError) {
from()
cause(err)
description("Encryption failed")
display("Bundle writer error: failed to encrypt data\n\tcaused by: {}", err)
}
Encode(err: msgpack::EncodeError, path: PathBuf) {
cause(err)
context(path: &'a Path, err: msgpack::EncodeError) -> (err, path.to_path_buf())
description("Failed to encode bundle header to file")
display("Bundle writer error: failed to encode bundle header to file {:?}\n\tcaused by: {}", path, err)
}
Write(err: io::Error, path: PathBuf) {
cause(err)
context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf())
description("Failed to write data to file")
display("Bundle writer error: failed to write data to file {:?}\n\tcaused by: {}", path, err)
}
}
}
pub struct BundleWriter {
mode: BundleMode,
hash_method: HashMethod,
@ -21,15 +56,9 @@ pub struct BundleWriter {
}
impl BundleWriter {
pub fn new(
mode: BundleMode,
hash_method: HashMethod,
compression: Option<Compression>,
encryption: Option<Encryption>,
crypto: Arc<Mutex<Crypto>>
) -> Result<Self, BundleError> {
pub fn new(mode: BundleMode, hash_method: HashMethod, compression: Option<Compression>, encryption: Option<Encryption>, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleWriterError> {
let compression_stream = match compression {
Some(ref compression) => Some(try!(compression.compress_stream())),
Some(ref compression) => Some(try!(compression.compress_stream().map_err(BundleWriterError::CompressionSetup))),
None => None
};
Ok(BundleWriter {
@ -46,9 +75,9 @@ impl BundleWriter {
})
}
pub fn add(&mut self, chunk: &[u8], hash: Hash) -> Result<usize, BundleError> {
pub fn add(&mut self, chunk: &[u8], hash: Hash) -> Result<usize, BundleWriterError> {
if let Some(ref mut stream) = self.compression_stream {
try!(stream.process(chunk, &mut self.data))
try!(stream.process(chunk, &mut self.data).map_err(BundleWriterError::Compression))
} else {
self.data.extend_from_slice(chunk)
}
@ -58,9 +87,9 @@ impl BundleWriter {
Ok(self.chunk_count-1)
}
pub fn finish(mut self, db: &BundleDb) -> Result<StoredBundle, BundleError> {
pub fn finish(mut self, db: &BundleDb) -> Result<StoredBundle, BundleWriterError> {
if let Some(stream) = self.compression_stream {
try!(stream.finish(&mut self.data))
try!(stream.finish(&mut self.data).map_err(BundleWriterError::Compression))
}
if let Some(ref encryption) = self.encryption {
self.data = try!(self.crypto.lock().unwrap().encrypt(&encryption, &self.data));

13
src/chunker/mod.rs
View File

@ -21,18 +21,19 @@ quick_error!{
#[derive(Debug)]
pub enum ChunkerError {
Read(err: io::Error) {
from(err)
cause(err)
description("Failed to read")
description("Failed to read input")
display("Chunker error: failed to read input\n\tcaused by: {}", err)
}
Write(err: io::Error) {
from(err)
cause(err)
description("Failed to write")
description("Failed to write to output")
display("Chunker error: failed to write to output\n\tcaused by: {}", err)
}
Custom {
from(&'static str)
Custom(reason: &'static str) {
from()
description("Custom error")
display("Chunker error: {}", reason)
}
}
}

23
src/cli/args.rs
View File

@ -1,4 +1,5 @@
use ::prelude::*;
use super::*;
use std::process::exit;
@ -259,7 +260,7 @@ pub fn parse() -> Arguments {
(about: "changes the configuration")
(@arg REPO: +required "path of the repository")
(@arg bundle_size: --bundlesize +takes_value "maximal bundle size in MiB [default: 25]")
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]")
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/16]")
(@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]")
(@arg encryption: --encryption -e +takes_value "the public key to use for encryption")
(@arg hash: --hash +takes_value "hash method to use [default: blake2]")
@ -278,7 +279,7 @@ pub fn parse() -> Arguments {
(@subcommand algotest =>
(about: "test a specific algorithm combination")
(@arg bundle_size: --bundlesize +takes_value "maximal bundle size in MiB [default: 25]")
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]")
(@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/16]")
(@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]")
(@arg encrypt: --encrypt -e "enable encryption")
(@arg hash: --hash +takes_value "hash method to use [default: blake2]")
@ -292,11 +293,11 @@ pub fn parse() -> Arguments {
exit(1);
}
return Arguments::Init {
bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize,
chunker: parse_chunker(args.value_of("chunker").unwrap_or("fastcdc/8")),
compression: parse_compression(args.value_of("compression").unwrap_or("brotli/3")),
bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or(&DEFAULT_BUNDLE_SIZE.to_string()), "Bundle size") * 1024 * 1024) as usize,
chunker: parse_chunker(args.value_of("chunker").unwrap_or(DEFAULT_CHUNKER)),
compression: parse_compression(args.value_of("compression").unwrap_or(DEFAULT_COMPRESSION)),
encryption: args.is_present("encryption"),
hash: parse_hash(args.value_of("hash").unwrap_or("blake2")),
hash: parse_hash(args.value_of("hash").unwrap_or(DEFAULT_HASH)),
repo_path: repository.to_string(),
}
}
@ -368,7 +369,7 @@ pub fn parse() -> Arguments {
return Arguments::Vacuum {
repo_path: repository.to_string(),
force: args.is_present("force"),
ratio: parse_float(args.value_of("ratio").unwrap_or("0.5"), "ratio") as f32
ratio: parse_float(args.value_of("ratio").unwrap_or(&DEFAULT_VACUUM_RATIO.to_string()), "ratio") as f32
}
}
if let Some(args) = args.subcommand_matches("check") {
@ -469,11 +470,11 @@ pub fn parse() -> Arguments {
}
if let Some(args) = args.subcommand_matches("algotest") {
return Arguments::AlgoTest {
bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize,
chunker: parse_chunker(args.value_of("chunker").unwrap_or("fastcdc/8")),
compression: parse_compression(args.value_of("compression").unwrap_or("brotli/3")),
bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or(&DEFAULT_BUNDLE_SIZE.to_string()), "Bundle size") * 1024 * 1024) as usize,
chunker: parse_chunker(args.value_of("chunker").unwrap_or(DEFAULT_CHUNKER)),
compression: parse_compression(args.value_of("compression").unwrap_or(DEFAULT_COMPRESSION)),
encrypt: args.is_present("encrypt"),
hash: parse_hash(args.value_of("hash").unwrap_or("blake2")),
hash: parse_hash(args.value_of("hash").unwrap_or(DEFAULT_HASH)),
file: args.value_of("FILE").unwrap().to_string(),
}
}

33
src/cli/mod.rs
View File

@ -10,6 +10,13 @@ use std::process::exit;
use self::args::Arguments;
pub const DEFAULT_CHUNKER: &'static str = "fastcdc/16";
pub const DEFAULT_HASH: &'static str = "blake2";
pub const DEFAULT_COMPRESSION: &'static str = "brotli/3";
pub const DEFAULT_BUNDLE_SIZE: usize = 25;
pub const DEFAULT_VACUUM_RATIO: f32 = 0.5;
fn open_repository(path: &str) -> Repository {
match Repository::open(path) {
Ok(repo) => repo,
@ -36,7 +43,18 @@ fn find_reference_backup(repo: &Repository, path: &str) -> Option<Backup> {
Ok(hostname) => hostname,
Err(_) => return None
};
for (_name, backup) in repo.get_backups().unwrap().0 {
let backup_map = match repo.get_backups() {
Ok(backup_map) => backup_map,
Err(RepositoryError::BackupFile(BackupFileError::PartialBackupsList(backup_map, _failed))) => {
warn!("Some backups could not be read, ignoring them");
backup_map
},
Err(err) => {
error!("Failed to load backup files: {}", err);
exit(3)
}
};
for (_name, backup) in backup_map {
if backup.host == hostname && backup.path == path {
matching.push(backup);
}
@ -155,7 +173,18 @@ pub fn run() {
}
}
} else {
for (name, backup) in repo.get_backups().unwrap().0 {
let backup_map = match repo.get_backups() {
Ok(backup_map) => backup_map,
Err(RepositoryError::BackupFile(BackupFileError::PartialBackupsList(backup_map, _failed))) => {
warn!("Some backups could not be read, ignoring them");
backup_map
},
Err(err) => {
error!("Failed to load backup files: {}", err);
exit(3)
}
};
for (name, backup) in backup_map {
println!("{:25} {:>32} {:5} files, {:4} dirs, {:>10}",
name, Local.timestamp(backup.date, 0).to_rfc2822(), backup.file_count,
backup.dir_count, to_file_size(backup.total_data_size));

76
src/index.rs
View File

@ -9,12 +9,49 @@ use std::os::unix::io::AsRawFd;
use mmap::{MemoryMap, MapOption, MapError};
const MAGIC: [u8; 7] = *b"zvault\x02";
const VERSION: u8 = 1;
pub const MAX_USAGE: f64 = 0.9;
pub const MIN_USAGE: f64 = 0.25;
pub const INITIAL_SIZE: usize = 1024;
quick_error!{
#[derive(Debug)]
pub enum IndexError {
Io(err: io::Error) {
from()
cause(err)
description("Failed to open index file")
display("Index error: failed to open the index file\n\tcaused by: {}", err)
}
Mmap(err: MapError) {
from()
cause(err)
description("Failed to memory-map the index file")
display("Index error: failed to memory-map the index file\n\tcaused by: {}", err)
}
WrongMagic {
description("Wrong header")
display("Index error: file has the wrong magic header")
}
UnsupportedVersion(version: u8) {
description("Unsupported version")
display("Index error: index file has unsupported version: {}", version)
}
WrongPosition(key: Hash, should: usize, is: LocateResult) {
description("Key at wrong position")
display("Index error: key {} has wrong position, expected at: {}, but is at: {:?}", key, should, is)
}
WrongEntryCount(header: usize, actual: usize) {
description("Wrong entry count")
display("Index error: index has wrong entry count, expected {}, but is {}", header, actual)
}
}
}
#[repr(packed)]
pub struct Header {
magic: [u8; 7],
@ -65,39 +102,6 @@ pub struct Index {
data: &'static mut [Entry]
}
quick_error!{
#[derive(Debug)]
pub enum IndexError {
Io(err: io::Error) {
from()
cause(err)
description("Failed to open index file")
}
Mmap(err: MapError) {
from()
cause(err)
description("Failed to write bundle map")
}
NoHeader {
description("Index file does not contain a header")
}
WrongHeader {
description("Wrong header")
}
WrongVersion(version: u8) {
description("Wrong version")
display("Wrong version: {}", version)
}
WrongPosition(key: Hash, should: usize, is: LocateResult) {
description("Key at wrong position")
display("Key {} has wrong position, expected at: {}, but is at: {:?}", key, should, is)
}
WrongEntryCount(header: usize, actual: usize) {
description("Wrong entry count")
display("Wrong entry count, expected {}, but is {}", header, actual)
}
}
}
#[derive(Debug)]
pub enum LocateResult {
@ -114,7 +118,7 @@ impl Index {
}
let mmap = try!(Index::map_fd(&fd));
if mmap.len() < mem::size_of::<Header>() {
return Err(IndexError::NoHeader);
return Err(IndexError::WrongMagic);
}
let data = Index::mmap_as_slice(&mmap, INITIAL_SIZE as usize);
let mut index = Index{capacity: 0, max_entries: 0, min_entries: 0, entries: 0, fd: fd, mmap: mmap, data: data};
@ -130,10 +134,10 @@ impl Index {
header.capacity = INITIAL_SIZE as u64;
} else {
if header.magic != MAGIC {
return Err(IndexError::WrongHeader);
return Err(IndexError::WrongMagic);
}
if header.version != VERSION {
return Err(IndexError::WrongVersion(header.version));
return Err(IndexError::UnsupportedVersion(header.version));
}
}
capacity = header.capacity;

1
src/main.rs
View File

@ -1,3 +1,4 @@
#![recursion_limit="128"]
extern crate serde;
extern crate rmp_serde;
#[macro_use] extern crate serde_utils;

4
src/prelude.rs
View File

@ -1,7 +1,7 @@
pub use ::util::*;
pub use ::bundledb::{Bundle, BundleMode, BundleWriter, BundleInfo, BundleId, BundleError, BundleDb};
pub use ::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError, BundleDb, BundleWriterError};
pub use ::chunker::{ChunkerType, Chunker, ChunkerStatus, IChunker, ChunkerError};
pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, RepositoryIntegrityError};
pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, RepositoryIntegrityError, BackupFileError};
pub use ::index::{Index, Location, IndexError};
pub use serde::{Serialize, Deserialize};

107
src/repository/backup.rs
View File

@ -14,49 +14,56 @@ static HEADER_VERSION: u8 = 1;
quick_error!{
#[derive(Debug)]
pub enum BackupError {
Io(err: io::Error, path: PathBuf) {
pub enum BackupFileError {
Read(err: io::Error, path: PathBuf) {
cause(err)
description("Failed to write backup")
display("Backup file error: failed to write backup file {:?}\n\tcaused by: {}", path, err)
}
Write(err: io::Error, path: PathBuf) {
cause(err)
context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf())
description("Failed to read/write backup")
display("Failed to read/write backup {:?}: {}", path, err)
display("Backup file error: failed to read backup file {:?}\n\tcaused by: {}", path, err)
}
Decode(err: msgpack::DecodeError, path: PathBuf) {
cause(err)
context(path: &'a Path, err: msgpack::DecodeError) -> (err, path.to_path_buf())
description("Failed to decode backup")