2018-03-08 22:41:56 +00:00
|
|
|
use ::prelude::*;
|
2017-03-21 10:28:11 +00:00
|
|
|
use super::*;
|
|
|
|
|
2017-03-21 10:08:01 +00:00
|
|
|
use std::path::{Path, PathBuf};
|
2017-03-21 14:38:42 +00:00
|
|
|
use std::collections::{HashMap, HashSet};
|
2017-03-22 08:19:16 +00:00
|
|
|
use std::fs;
|
2018-03-09 22:56:18 +00:00
|
|
|
use std::sync::Arc;
|
2017-03-22 08:19:16 +00:00
|
|
|
use std::io;
|
2017-04-10 06:53:55 +00:00
|
|
|
use std::mem;
|
2017-04-12 18:19:21 +00:00
|
|
|
use std::cmp::min;
|
2017-03-21 18:18:18 +00:00
|
|
|
|
2017-03-22 08:19:16 +00:00
|
|
|
quick_error!{
|
|
|
|
#[derive(Debug)]
|
|
|
|
pub enum BundleDbError {
|
|
|
|
ListBundles(err: io::Error) {
|
|
|
|
cause(err)
|
2018-02-24 12:19:51 +00:00
|
|
|
description(tr!("Failed to list bundles"))
|
|
|
|
display("{}", tr_format!("Bundle db error: failed to list bundles\n\tcaused by: {}", err))
|
2017-03-22 08:19:16 +00:00
|
|
|
}
|
|
|
|
Reader(err: BundleReaderError) {
|
|
|
|
from()
|
|
|
|
cause(err)
|
2018-02-24 12:19:51 +00:00
|
|
|
description(tr!("Failed to read bundle"))
|
|
|
|
display("{}", tr_format!("Bundle db error: failed to read bundle\n\tcaused by: {}", err))
|
2017-03-22 08:19:16 +00:00
|
|
|
}
|
|
|
|
Writer(err: BundleWriterError) {
|
|
|
|
from()
|
|
|
|
cause(err)
|
2018-02-24 12:19:51 +00:00
|
|
|
description(tr!("Failed to write bundle"))
|
|
|
|
display("{}", tr_format!("Bundle db error: failed to write bundle\n\tcaused by: {}", err))
|
2017-03-22 08:19:16 +00:00
|
|
|
}
|
|
|
|
Cache(err: BundleCacheError) {
|
|
|
|
from()
|
|
|
|
cause(err)
|
2018-02-24 12:19:51 +00:00
|
|
|
description(tr!("Failed to read/write bundle cache"))
|
|
|
|
display("{}", tr_format!("Bundle db error: failed to read/write bundle cache\n\tcaused by: {}", err))
|
2017-03-22 08:19:16 +00:00
|
|
|
}
|
2017-04-12 09:34:31 +00:00
|
|
|
UploadFailed {
|
2018-02-24 12:19:51 +00:00
|
|
|
description(tr!("Uploading a bundle failed"))
|
2017-04-12 09:34:31 +00:00
|
|
|
}
|
2017-03-22 08:19:16 +00:00
|
|
|
Io(err: io::Error, path: PathBuf) {
|
|
|
|
cause(err)
|
|
|
|
context(path: &'a Path, err: io::Error) -> (err, path.to_path_buf())
|
2018-02-24 12:19:51 +00:00
|
|
|
description(tr!("Io error"))
|
|
|
|
display("{}", tr_format!("Bundle db error: io error on {:?}\n\tcaused by: {}", path, err))
|
2017-03-22 08:19:16 +00:00
|
|
|
}
|
|
|
|
NoSuchBundle(bundle: BundleId) {
|
2018-02-24 12:19:51 +00:00
|
|
|
description(tr!("No such bundle"))
|
|
|
|
display("{}", tr_format!("Bundle db error: no such bundle: {:?}", bundle))
|
2017-03-22 08:19:16 +00:00
|
|
|
}
|
|
|
|
Remove(err: io::Error, bundle: BundleId) {
|
|
|
|
cause(err)
|
2018-02-24 12:19:51 +00:00
|
|
|
description(tr!("Failed to remove bundle"))
|
|
|
|
display("{}", tr_format!("Bundle db error: failed to remove bundle {}\n\tcaused by: {}", bundle, err))
|
2017-03-22 08:19:16 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-03-21 10:08:01 +00:00
|
|
|
|
|
|
|
|
2018-02-19 21:30:59 +00:00
|
|
|
#[allow(needless_pass_by_value)]
|
2017-07-21 09:21:59 +00:00
|
|
|
fn load_bundles(
|
|
|
|
path: &Path,
|
|
|
|
base: &Path,
|
|
|
|
bundles: &mut HashMap<BundleId, StoredBundle>,
|
2018-03-09 22:56:18 +00:00
|
|
|
crypto: Arc<Crypto>,
|
2017-07-21 09:21:59 +00:00
|
|
|
) -> Result<(Vec<StoredBundle>, Vec<StoredBundle>), BundleDbError> {
|
2017-04-08 12:35:10 +00:00
|
|
|
let mut paths = vec![path.to_path_buf()];
|
2017-03-21 14:38:42 +00:00
|
|
|
let mut bundle_paths = HashSet::new();
|
|
|
|
while let Some(path) = paths.pop() {
|
2017-03-22 08:19:16 +00:00
|
|
|
for entry in try!(fs::read_dir(path).map_err(BundleDbError::ListBundles)) {
|
|
|
|
let entry = try!(entry.map_err(BundleDbError::ListBundles));
|
2017-03-21 14:38:42 +00:00
|
|
|
let path = entry.path();
|
|
|
|
if path.is_dir() {
|
|
|
|
paths.push(path);
|
|
|
|
} else {
|
2017-04-12 18:19:21 +00:00
|
|
|
if path.extension() != Some("bundle".as_ref()) {
|
2017-07-21 09:21:59 +00:00
|
|
|
continue;
|
2017-04-12 18:19:21 +00:00
|
|
|
}
|
2017-04-08 07:59:55 +00:00
|
|
|
bundle_paths.insert(path.strip_prefix(base).unwrap().to_path_buf());
|
2017-03-21 14:38:42 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-03-24 06:09:43 +00:00
|
|
|
let mut gone = HashSet::new();
|
2017-04-10 16:21:26 +00:00
|
|
|
for (id, bundle) in bundles.iter() {
|
2017-03-21 14:38:42 +00:00
|
|
|
if !bundle_paths.contains(&bundle.path) {
|
2017-03-24 06:09:43 +00:00
|
|
|
gone.insert(id.clone());
|
2017-03-21 14:38:42 +00:00
|
|
|
} else {
|
|
|
|
bundle_paths.remove(&bundle.path);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
let mut new = vec![];
|
|
|
|
for path in bundle_paths {
|
2017-04-11 06:49:45 +00:00
|
|
|
let info = match BundleReader::load_info(base.join(&path), crypto.clone()) {
|
|
|
|
Ok(info) => info,
|
2017-04-12 18:19:21 +00:00
|
|
|
Err(err) => {
|
|
|
|
warn!("Failed to read bundle {:?}\n\tcaused by: {}", path, err);
|
|
|
|
info!("Ignoring unreadable bundle");
|
2017-07-21 09:21:59 +00:00
|
|
|
continue;
|
2017-04-12 18:19:21 +00:00
|
|
|
}
|
2017-03-21 14:38:42 +00:00
|
|
|
};
|
2017-07-21 09:21:59 +00:00
|
|
|
let bundle = StoredBundle {
|
2018-03-03 16:25:05 +00:00
|
|
|
info,
|
|
|
|
path
|
2017-07-21 09:21:59 +00:00
|
|
|
};
|
2017-03-24 06:09:43 +00:00
|
|
|
let id = bundle.info.id.clone();
|
|
|
|
if !bundles.contains_key(&id) {
|
|
|
|
new.push(bundle.clone());
|
|
|
|
} else {
|
|
|
|
gone.remove(&id);
|
|
|
|
}
|
|
|
|
bundles.insert(id, bundle);
|
2017-03-21 14:38:42 +00:00
|
|
|
}
|
2017-03-24 06:09:43 +00:00
|
|
|
let gone = gone.iter().map(|id| bundles.remove(id).unwrap()).collect();
|
2017-03-21 14:38:42 +00:00
|
|
|
Ok((new, gone))
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
2017-03-21 10:08:01 +00:00
|
|
|
pub struct BundleDb {
|
2018-03-10 11:09:04 +00:00
|
|
|
layout: Arc<ChunkRepositoryLayout>,
|
2017-04-10 06:53:55 +00:00
|
|
|
uploader: Option<Arc<BundleUploader>>,
|
2018-03-09 22:56:18 +00:00
|
|
|
crypto: Arc<Crypto>,
|
2017-03-21 14:38:42 +00:00
|
|
|
local_bundles: HashMap<BundleId, StoredBundle>,
|
|
|
|
remote_bundles: HashMap<BundleId, StoredBundle>,
|
2017-03-22 08:19:16 +00:00
|
|
|
bundle_cache: LruCache<BundleId, (BundleReader, Vec<u8>)>
|
2017-03-21 10:08:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
impl BundleDb {
|
2018-03-09 22:56:18 +00:00
|
|
|
fn new(layout: Arc<ChunkRepositoryLayout>, crypto: Arc<Crypto>) -> Self {
|
2017-03-21 10:08:01 +00:00
|
|
|
BundleDb {
|
2018-03-03 16:25:05 +00:00
|
|
|
layout,
|
|
|
|
crypto,
|
2017-04-10 06:53:55 +00:00
|
|
|
uploader: None,
|
2017-03-21 14:38:42 +00:00
|
|
|
local_bundles: HashMap::new(),
|
|
|
|
remote_bundles: HashMap::new(),
|
2017-03-21 10:08:01 +00:00
|
|
|
bundle_cache: LruCache::new(5, 10)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-07-21 09:21:59 +00:00
|
|
|
fn load_bundle_list(
|
|
|
|
&mut self,
|
2018-02-25 00:03:18 +00:00
|
|
|
online: bool
|
2017-07-21 09:21:59 +00:00
|
|
|
) -> Result<(Vec<StoredBundle>, Vec<StoredBundle>), BundleDbError> {
|
2017-04-08 12:35:10 +00:00
|
|
|
if let Ok(list) = StoredBundle::read_list_from(&self.layout.local_bundle_cache_path()) {
|
2017-04-01 16:43:36 +00:00
|
|
|
for bundle in list {
|
|
|
|
self.local_bundles.insert(bundle.id(), bundle);
|
|
|
|
}
|
2017-04-09 10:04:28 +00:00
|
|
|
} else {
|
2018-02-24 12:19:51 +00:00
|
|
|
tr_warn!("Failed to read local bundle cache, rebuilding cache");
|
2017-04-01 16:43:36 +00:00
|
|
|
}
|
2017-04-08 12:35:10 +00:00
|
|
|
if let Ok(list) = StoredBundle::read_list_from(&self.layout.remote_bundle_cache_path()) {
|
2017-03-21 18:18:18 +00:00
|
|
|
for bundle in list {
|
|
|
|
self.remote_bundles.insert(bundle.id(), bundle);
|
|
|
|
}
|
2017-04-09 10:04:28 +00:00
|
|
|
} else {
|
2018-02-24 12:19:51 +00:00
|
|
|
tr_warn!("Failed to read remote bundle cache, rebuilding cache");
|
2017-03-21 18:18:18 +00:00
|
|
|
}
|
2017-04-08 12:35:10 +00:00
|
|
|
let base_path = self.layout.base_path();
|
2017-07-21 09:21:59 +00:00
|
|
|
let (new, gone) = try!(load_bundles(
|
|
|
|
&self.layout.local_bundles_path(),
|
|
|
|
base_path,
|
|
|
|
&mut self.local_bundles,
|
|
|
|
self.crypto.clone()
|
|
|
|
));
|
2017-04-01 16:43:36 +00:00
|
|
|
if !new.is_empty() || !gone.is_empty() {
|
|
|
|
let bundles: Vec<_> = self.local_bundles.values().cloned().collect();
|
2017-07-21 09:21:59 +00:00
|
|
|
try!(StoredBundle::save_list_to(
|
|
|
|
&bundles,
|
|
|
|
&self.layout.local_bundle_cache_path()
|
|
|
|
));
|
2017-04-01 16:43:36 +00:00
|
|
|
}
|
2018-02-25 00:03:18 +00:00
|
|
|
if !online {
|
|
|
|
return Ok((vec![], vec![]))
|
|
|
|
}
|
2017-07-21 09:21:59 +00:00
|
|
|
let (new, gone) = try!(load_bundles(
|
|
|
|
&self.layout.remote_bundles_path(),
|
|
|
|
base_path,
|
|
|
|
&mut self.remote_bundles,
|
|
|
|
self.crypto.clone()
|
|
|
|
));
|
2017-03-21 18:18:18 +00:00
|
|
|
if !new.is_empty() || !gone.is_empty() {
|
|
|
|
let bundles: Vec<_> = self.remote_bundles.values().cloned().collect();
|
2017-07-21 09:21:59 +00:00
|
|
|
try!(StoredBundle::save_list_to(
|
|
|
|
&bundles,
|
|
|
|
&self.layout.remote_bundle_cache_path()
|
|
|
|
));
|
2017-03-21 18:18:18 +00:00
|
|
|
}
|
|
|
|
Ok((new, gone))
|
2017-03-21 14:38:42 +00:00
|
|
|
}
|
|
|
|
|
2017-04-19 15:46:51 +00:00
|
|
|
pub fn flush(&mut self) -> Result<(), BundleDbError> {
|
|
|
|
self.finish_uploads().and_then(|()| self.save_cache())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn save_cache(&self) -> Result<(), BundleDbError> {
|
2017-04-01 16:43:36 +00:00
|
|
|
let bundles: Vec<_> = self.local_bundles.values().cloned().collect();
|
2017-07-21 09:21:59 +00:00
|
|
|
try!(StoredBundle::save_list_to(
|
|
|
|
&bundles,
|
|
|
|
&self.layout.local_bundle_cache_path()
|
|
|
|
));
|
2017-03-22 13:10:42 +00:00
|
|
|
let bundles: Vec<_> = self.remote_bundles.values().cloned().collect();
|
2018-02-19 21:30:59 +00:00
|
|
|
try!(StoredBundle::save_list_to(
|
2017-07-21 09:21:59 +00:00
|
|
|
&bundles,
|
|
|
|
&self.layout.remote_bundle_cache_path()
|
2018-02-19 21:30:59 +00:00
|
|
|
));
|
|
|
|
Ok(())
|
2017-03-22 13:10:42 +00:00
|
|
|
}
|
|
|
|
|
2017-04-19 15:46:51 +00:00
|
|
|
fn update_cache(&mut self) -> Result<(), BundleDbError> {
|
2017-04-10 17:09:50 +00:00
|
|
|
let mut meta_bundles = HashSet::new();
|
|
|
|
for (id, bundle) in &self.remote_bundles {
|
2017-03-22 11:27:17 +00:00
|
|
|
if bundle.info.mode == BundleMode::Meta {
|
2017-04-10 17:09:50 +00:00
|
|
|
meta_bundles.insert(id.clone());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
let mut remove = vec![];
|
|
|
|
for id in self.local_bundles.keys() {
|
|
|
|
if !meta_bundles.contains(id) {
|
|
|
|
remove.push(id.clone());
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for id in meta_bundles {
|
|
|
|
if !self.local_bundles.contains_key(&id) {
|
|
|
|
let bundle = self.remote_bundles[&id].clone();
|
2018-02-24 12:19:51 +00:00
|
|
|
tr_debug!("Copying new meta bundle to local cache: {}", bundle.info.id);
|
2017-04-10 17:09:50 +00:00
|
|
|
try!(self.copy_remote_bundle_to_cache(&bundle));
|
2017-03-22 11:27:17 +00:00
|
|
|
}
|
|
|
|
}
|
2017-04-08 12:35:10 +00:00
|
|
|
let base_path = self.layout.base_path();
|
2017-04-10 17:09:50 +00:00
|
|
|
for id in remove {
|
|
|
|
if let Some(bundle) = self.local_bundles.remove(&id) {
|
2017-07-21 09:21:59 +00:00
|
|
|
try!(fs::remove_file(base_path.join(&bundle.path)).map_err(|e| {
|
|
|
|
BundleDbError::Remove(e, id)
|
|
|
|
}))
|
2017-03-22 11:27:17 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2017-07-21 09:21:59 +00:00
|
|
|
pub fn open(
|
2018-03-09 22:31:20 +00:00
|
|
|
layout: Arc<ChunkRepositoryLayout>,
|
2018-03-09 22:56:18 +00:00
|
|
|
crypto: Arc<Crypto>,
|
2018-02-25 00:03:18 +00:00
|
|
|
online: bool
|
2017-07-21 09:21:59 +00:00
|
|
|
) -> Result<(Self, Vec<BundleInfo>, Vec<BundleInfo>), BundleDbError> {
|
2017-04-08 12:35:10 +00:00
|
|
|
let mut self_ = Self::new(layout, crypto);
|
2018-02-25 00:03:18 +00:00
|
|
|
let (new, gone) = try!(self_.load_bundle_list(online));
|
2017-04-10 17:09:50 +00:00
|
|
|
try!(self_.update_cache());
|
2017-03-22 11:27:17 +00:00
|
|
|
let new = new.into_iter().map(|s| s.info).collect();
|
|
|
|
let gone = gone.into_iter().map(|s| s.info).collect();
|
2017-03-21 14:38:42 +00:00
|
|
|
Ok((self_, new, gone))
|
2017-03-21 10:08:01 +00:00
|
|
|
}
|
|
|
|
|
2018-03-09 22:31:20 +00:00
|
|
|
pub fn create(layout: Arc<ChunkRepositoryLayout>) -> Result<(), BundleDbError> {
|
2017-07-21 09:21:59 +00:00
|
|
|
try!(fs::create_dir_all(layout.remote_bundles_path()).context(
|
|
|
|
&layout.remote_bundles_path() as
|
|
|
|
&Path
|
|
|
|
));
|
|
|
|
try!(fs::create_dir_all(layout.local_bundles_path()).context(
|
|
|
|
&layout.local_bundles_path() as
|
|
|
|
&Path
|
|
|
|
));
|
|
|
|
try!(fs::create_dir_all(layout.temp_bundles_path()).context(
|
|
|
|
&layout.temp_bundles_path() as
|
|
|
|
&Path
|
|
|
|
));
|
|
|
|
try!(StoredBundle::save_list_to(
|
|
|
|
&[],
|
|
|
|
layout.local_bundle_cache_path()
|
|
|
|
));
|
|
|
|
try!(StoredBundle::save_list_to(
|
|
|
|
&[],
|
|
|
|
layout.remote_bundle_cache_path()
|
|
|
|
));
|
2017-04-08 12:35:10 +00:00
|
|
|
Ok(())
|
2017-03-21 10:08:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
2017-07-21 09:21:59 +00:00
|
|
|
pub fn create_bundle(
|
|
|
|
&self,
|
|
|
|
mode: BundleMode,
|
|
|
|
hash_method: HashMethod,
|
|
|
|
compression: Option<Compression>,
|
|
|
|
encryption: Option<Encryption>,
|
|
|
|
) -> Result<BundleWriter, BundleDbError> {
|
|
|
|
Ok(try!(BundleWriter::new(
|
2018-03-10 11:09:04 +00:00
|
|
|
self.layout.clone(),
|
2017-07-21 09:21:59 +00:00
|
|
|
mode,
|
|
|
|
hash_method,
|
|
|
|
compression,
|
|
|
|
encryption,
|
|
|
|
self.crypto.clone()
|
|
|
|
)))
|
2017-03-21 10:08:01 +00:00
|
|
|
}
|
|
|
|
|
2017-03-22 08:19:16 +00:00
|
|
|
fn get_stored_bundle(&self, bundle_id: &BundleId) -> Result<&StoredBundle, BundleDbError> {
|
2017-07-21 09:21:59 +00:00
|
|
|
if let Some(stored) = self.local_bundles.get(bundle_id).or_else(|| {
|
|
|
|
self.remote_bundles.get(bundle_id)
|
|
|
|
})
|
|
|
|
{
|
2017-03-21 14:38:42 +00:00
|
|
|
Ok(stored)
|
|
|
|
} else {
|
2017-03-22 08:19:16 +00:00
|
|
|
Err(BundleDbError::NoSuchBundle(bundle_id.clone()))
|
2017-03-21 14:38:42 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-10 18:35:28 +00:00
|
|
|
#[inline]
|
2017-03-22 08:19:16 +00:00
|
|
|
fn get_bundle(&self, stored: &StoredBundle) -> Result<BundleReader, BundleDbError> {
|
2017-04-08 12:35:10 +00:00
|
|
|
let base_path = self.layout.base_path();
|
2017-07-21 09:21:59 +00:00
|
|
|
Ok(try!(BundleReader::load(
|
|
|
|
base_path.join(&stored.path),
|
|
|
|
self.crypto.clone()
|
|
|
|
)))
|
2017-03-21 14:38:42 +00:00
|
|
|
}
|
|
|
|
|
2017-03-22 08:19:16 +00:00
|
|
|
pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleDbError> {
|
2017-03-21 14:38:42 +00:00
|
|
|
if let Some(&mut (ref mut bundle, ref data)) = self.bundle_cache.get_mut(bundle_id) {
|
|
|
|
let (pos, len) = try!(bundle.get_chunk_position(id));
|
|
|
|
let mut chunk = Vec::with_capacity(len);
|
2017-07-21 09:21:59 +00:00
|
|
|
chunk.extend_from_slice(&data[pos..pos + len]);
|
2017-03-21 10:08:01 +00:00
|
|
|
return Ok(chunk);
|
|
|
|
}
|
2017-07-21 09:21:59 +00:00
|
|
|
let mut bundle = try!(self.get_stored_bundle(bundle_id).and_then(
|
|
|
|
|s| self.get_bundle(s)
|
|
|
|
));
|
2017-03-21 14:38:42 +00:00
|
|
|
let (pos, len) = try!(bundle.get_chunk_position(id));
|
|
|
|
let mut chunk = Vec::with_capacity(len);
|
2017-03-21 10:08:01 +00:00
|
|
|
let data = try!(bundle.load_contents());
|
2017-07-21 09:21:59 +00:00
|
|
|
chunk.extend_from_slice(&data[pos..pos + len]);
|
2017-03-21 14:38:42 +00:00
|
|
|
self.bundle_cache.put(bundle_id.clone(), (bundle, data));
|
2017-03-21 10:08:01 +00:00
|
|
|
Ok(chunk)
|
|
|
|
}
|
|
|
|
|
2017-03-22 11:27:17 +00:00
|
|
|
fn copy_remote_bundle_to_cache(&mut self, bundle: &StoredBundle) -> Result<(), BundleDbError> {
|
|
|
|
let id = bundle.id();
|
2018-03-09 22:31:20 +00:00
|
|
|
let dst_path = self.layout.local_bundle_path(&id, self.local_bundles.len());
|
|
|
|
{
|
|
|
|
let folder = dst_path.parent().unwrap();
|
|
|
|
try!(fs::create_dir_all(folder).context(folder as &Path));
|
|
|
|
}
|
2017-07-21 09:21:59 +00:00
|
|
|
let bundle = try!(bundle.copy_to(
|
|
|
|
self.layout.base_path(),
|
2018-03-09 22:31:20 +00:00
|
|
|
dst_path
|
2017-07-21 09:21:59 +00:00
|
|
|
));
|
2017-03-22 11:27:17 +00:00
|
|
|
self.local_bundles.insert(id, bundle);
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2017-03-22 08:19:16 +00:00
|
|
|
pub fn add_bundle(&mut self, bundle: BundleWriter) -> Result<BundleInfo, BundleDbError> {
|
2018-03-10 11:09:04 +00:00
|
|
|
let mut bundle = try!(bundle.finish());
|
2017-03-21 14:38:42 +00:00
|
|
|
if bundle.info.mode == BundleMode::Meta {
|
2017-03-22 11:27:17 +00:00
|
|
|
try!(self.copy_remote_bundle_to_cache(&bundle))
|
2017-03-21 14:38:42 +00:00
|
|
|
}
|
2018-03-09 22:31:20 +00:00
|
|
|
let dst_path = self.layout.remote_bundle_path(&bundle.id(),self.remote_bundles.len());
|
2017-04-12 11:57:28 +00:00
|
|
|
let src_path = self.layout.base_path().join(bundle.path);
|
2017-07-21 09:21:59 +00:00
|
|
|
bundle.path = dst_path
|
|
|
|
.strip_prefix(self.layout.base_path())
|
|
|
|
.unwrap()
|
|
|
|
.to_path_buf();
|
2017-04-10 06:53:55 +00:00
|
|
|
if self.uploader.is_none() {
|
|
|
|
self.uploader = Some(BundleUploader::new(5));
|
|
|
|
}
|
|
|
|
try!(self.uploader.as_ref().unwrap().queue(src_path, dst_path));
|
2017-03-21 14:38:42 +00:00
|
|
|
self.remote_bundles.insert(bundle.id(), bundle.clone());
|
|
|
|
Ok(bundle.info)
|
2017-03-21 10:08:01 +00:00
|
|
|
}
|
|
|
|
|
2017-04-19 15:46:51 +00:00
|
|
|
fn finish_uploads(&mut self) -> Result<(), BundleDbError> {
|
2017-04-10 06:53:55 +00:00
|
|
|
let mut uploader = None;
|
|
|
|
mem::swap(&mut self.uploader, &mut uploader);
|
|
|
|
if let Some(uploader) = uploader {
|
|
|
|
uploader.finish()
|
|
|
|
} else {
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-03-22 11:27:17 +00:00
|
|
|
pub fn get_chunk_list(&self, bundle: &BundleId) -> Result<ChunkList, BundleDbError> {
|
2017-07-21 09:21:59 +00:00
|
|
|
let mut bundle = try!(self.get_stored_bundle(bundle).and_then(|stored| {
|
|
|
|
self.get_bundle(stored)
|
|
|
|
}));
|
2017-03-22 11:27:17 +00:00
|
|
|
Ok(try!(bundle.get_chunk_list()).clone())
|
|
|
|
}
|
|
|
|
|
2017-03-21 10:08:01 +00:00
|
|
|
#[inline]
|
2017-04-07 16:57:49 +00:00
|
|
|
pub fn get_bundle_info(&self, bundle: &BundleId) -> Option<&StoredBundle> {
|
2017-04-10 17:28:17 +00:00
|
|
|
self.remote_bundles.get(bundle)
|
2017-03-21 10:08:01 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[inline]
|
2017-03-21 14:38:42 +00:00
|
|
|
pub fn list_bundles(&self) -> Vec<&BundleInfo> {
|
|
|
|
self.remote_bundles.values().map(|b| &b.info).collect()
|
2017-03-21 10:08:01 +00:00
|
|
|
}
|
|
|
|
|
2017-03-30 15:50:52 +00:00
|
|
|
pub fn delete_local_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> {
|
2017-03-21 14:38:42 +00:00
|
|
|
if let Some(bundle) = self.local_bundles.remove(bundle) {
|
2017-04-08 12:35:10 +00:00
|
|
|
let path = self.layout.base_path().join(&bundle.path);
|
2017-07-21 09:21:59 +00:00
|
|
|
try!(fs::remove_file(path).map_err(|e| {
|
|
|
|
BundleDbError::Remove(e, bundle.id())
|
|
|
|
}))
|
2017-03-21 14:38:42 +00:00
|
|
|
}
|
2017-03-30 15:50:52 +00:00
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn delete_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> {
|
|
|
|
try!(self.delete_local_bundle(bundle));
|
2017-03-21 14:38:42 +00:00
|
|
|
if let Some(bundle) = self.remote_bundles.remove(bundle) {
|
2017-04-08 12:35:10 +00:00
|
|
|
let path = self.layout.base_path().join(&bundle.path);
|
|
|
|
fs::remove_file(path).map_err(|e| BundleDbError::Remove(e, bundle.id()))
|
2017-03-21 10:08:01 +00:00
|
|
|
} else {
|
2017-03-22 08:19:16 +00:00
|
|
|
Err(BundleDbError::NoSuchBundle(bundle.clone()))
|
2017-03-21 10:08:01 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-12 18:19:21 +00:00
|
|
|
pub fn check(&mut self, full: bool, repair: bool) -> Result<bool, BundleDbError> {
|
|
|
|
let mut to_repair = vec![];
|
2017-07-21 09:21:59 +00:00
|
|
|
for (id, stored) in ProgressIter::new(
|
2018-02-24 12:19:51 +00:00
|
|
|
tr!("checking bundles"),
|
2017-07-21 09:21:59 +00:00
|
|
|
self.remote_bundles.len(),
|
|
|
|
self.remote_bundles.iter()
|
|
|
|
)
|
|
|
|
{
|
2017-04-12 18:19:21 +00:00
|
|
|
let mut bundle = match self.get_bundle(stored) {
|
|
|
|
Ok(bundle) => bundle,
|
|
|
|
Err(err) => {
|
|
|
|
if repair {
|
|
|
|
to_repair.push(id.clone());
|
2017-07-21 09:21:59 +00:00
|
|
|
continue;
|
2017-04-12 18:19:21 +00:00
|
|
|
} else {
|
2017-07-21 09:21:59 +00:00
|
|
|
return Err(err);
|
2017-04-12 18:19:21 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
};
|
|
|
|
if let Err(err) = bundle.check(full) {
|
|
|
|
if repair {
|
|
|
|
to_repair.push(id.clone());
|
2017-07-21 09:21:59 +00:00
|
|
|
continue;
|
2017-04-12 18:19:21 +00:00
|
|
|
} else {
|
2017-07-21 09:21:59 +00:00
|
|
|
return Err(err.into());
|
2017-04-12 18:19:21 +00:00
|
|
|
}
|
|
|
|
}
|
2017-03-21 10:08:01 +00:00
|
|
|
}
|
2017-04-14 11:13:39 +00:00
|
|
|
if !to_repair.is_empty() {
|
2018-02-24 12:19:51 +00:00
|
|
|
for id in ProgressIter::new(tr!("repairing bundles"), to_repair.len(), to_repair.iter()) {
|
2018-02-19 21:30:59 +00:00
|
|
|
try!(self.repair_bundle(id));
|
2017-04-14 11:13:39 +00:00
|
|
|
}
|
2017-04-19 15:46:51 +00:00
|
|
|
try!(self.flush());
|
2017-04-12 18:19:21 +00:00
|
|
|
}
|
|
|
|
Ok(!to_repair.is_empty())
|
|
|
|
}
|
|
|
|
|
|
|
|
fn evacuate_broken_bundle(&mut self, mut bundle: StoredBundle) -> Result<(), BundleDbError> {
|
2017-04-17 13:42:06 +00:00
|
|
|
let src = self.layout.base_path().join(&bundle.path);
|
|
|
|
let mut dst = src.with_extension("bundle.broken");
|
|
|
|
let mut num = 1;
|
|
|
|
while dst.exists() {
|
|
|
|
dst = src.with_extension(&format!("bundle.{}.broken", num));
|
|
|
|
num += 1;
|
|
|
|
}
|
|
|
|
warn!("Moving bundle to {:?}", dst);
|
|
|
|
try!(bundle.move_to(self.layout.base_path(), dst));
|
2017-04-12 18:19:21 +00:00
|
|
|
self.remote_bundles.remove(&bundle.info.id);
|
2017-03-21 10:08:01 +00:00
|
|
|
Ok(())
|
|
|
|
}
|
2017-04-10 16:21:26 +00:00
|
|
|
|
2018-02-19 21:30:59 +00:00
|
|
|
fn repair_bundle(&mut self, id: &BundleId) -> Result<(), BundleDbError> {
|
|
|
|
let stored = self.remote_bundles[id].clone();
|
2017-04-12 18:19:21 +00:00
|
|
|
let mut bundle = match self.get_bundle(&stored) {
|
|
|
|
Ok(bundle) => bundle,
|
|
|
|
Err(err) => {
|
2018-02-24 12:19:51 +00:00
|
|
|
tr_warn!(
|
2017-07-21 09:21:59 +00:00
|
|
|
"Problem detected: failed to read bundle header: {}\n\tcaused by: {}",
|
|
|
|
id,
|
|
|
|
err
|
|
|
|
);
|
2017-04-12 18:19:21 +00:00
|
|
|
return self.evacuate_broken_bundle(stored);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
let chunks = match bundle.get_chunk_list() {
|
|
|
|
Ok(chunks) => chunks.clone(),
|
|
|
|
Err(err) => {
|
2018-02-24 12:19:51 +00:00
|
|
|
tr_warn!(
|
2017-07-21 09:21:59 +00:00
|
|
|
"Problem detected: failed to read bundle chunks: {}\n\tcaused by: {}",
|
|
|
|
id,
|
|
|
|
err
|
|
|
|
);
|
2017-04-12 18:19:21 +00:00
|
|
|
return self.evacuate_broken_bundle(stored);
|
|
|
|
}
|
|
|
|
};
|
|
|
|
let data = match bundle.load_contents() {
|
|
|
|
Ok(data) => data,
|
|
|
|
Err(err) => {
|
2018-02-24 12:19:51 +00:00
|
|
|
tr_warn!(
|
2017-07-21 09:21:59 +00:00
|
|
|
"Problem detected: failed to read bundle data: {}\n\tcaused by: {}",
|
|
|
|
id,
|
|
|
|
err
|
|
|
|
);
|
2017-04-12 18:19:21 +00:00
|
|
|
return self.evacuate_broken_bundle(stored);
|
|
|
|
}
|
|
|
|
};
|
2018-02-24 12:19:51 +00:00
|
|
|
tr_warn!("Problem detected: bundle data was truncated: {}", id);
|
|
|
|
tr_info!("Copying readable data into new bundle");
|
2017-04-12 18:19:21 +00:00
|
|
|
let info = stored.info.clone();
|
2017-07-21 09:21:59 +00:00
|
|
|
let mut new_bundle = try!(self.create_bundle(
|
|
|
|
info.mode,
|
|
|
|
info.hash_method,
|
|
|
|
info.compression,
|
|
|
|
info.encryption
|
|
|
|
));
|
2017-04-12 18:19:21 +00:00
|
|
|
let mut pos = 0;
|
|
|
|
for (hash, mut len) in chunks.into_inner() {
|
|
|
|
if pos >= data.len() {
|
2017-07-21 09:21:59 +00:00
|
|
|
break;
|
2017-04-12 18:19:21 +00:00
|
|
|
}
|
|
|
|
len = min(len, (data.len() - pos) as u32);
|
2017-07-21 09:21:59 +00:00
|
|
|
try!(new_bundle.add(&data[pos..pos + len as usize], hash));
|
2017-04-12 18:19:21 +00:00
|
|
|
pos += len as usize;
|
|
|
|
}
|
|
|
|
let bundle = try!(self.add_bundle(new_bundle));
|
2018-02-24 12:19:51 +00:00
|
|
|
tr_info!("New bundle id is {}", bundle.id);
|
2017-04-12 18:19:21 +00:00
|
|
|
self.evacuate_broken_bundle(stored)
|
|
|
|
}
|
|
|
|
|
2017-04-10 16:21:26 +00:00
|
|
|
#[inline]
|
|
|
|
pub fn len(&self) -> usize {
|
|
|
|
self.remote_bundles.len()
|
|
|
|
}
|
2018-03-06 21:53:35 +00:00
|
|
|
|
|
|
|
pub fn statistics(&self) -> BundleStatistics {
|
|
|
|
let bundles = self.list_bundles();
|
2018-03-06 23:36:44 +00:00
|
|
|
let bundles_meta: Vec<_> = bundles.iter().filter(|b| b.mode == BundleMode::Meta).collect();
|
|
|
|
let bundles_data: Vec<_> = bundles.iter().filter(|b| b.mode == BundleMode::Data).collect();
|
|
|
|
let mut hash_methods = HashMap::new();
|
|
|
|
let mut compressions = HashMap::new();
|
2018-03-08 14:20:20 +00:00
|
|
|
let mut encryptions = HashMap::new();
|
2018-03-06 23:36:44 +00:00
|
|
|
for bundle in &bundles {
|
|
|
|
*hash_methods.entry(bundle.hash_method).or_insert(0) += 1;
|
|
|
|
*compressions.entry(bundle.compression.clone()).or_insert(0) += 1;
|
2018-03-08 14:20:20 +00:00
|
|
|
*encryptions.entry(bundle.encryption.clone()).or_insert(0) += 1;
|
2018-03-06 23:36:44 +00:00
|
|
|
}
|
2018-03-06 21:53:35 +00:00
|
|
|
BundleStatistics {
|
2018-03-08 14:20:20 +00:00
|
|
|
hash_methods, compressions, encryptions,
|
2018-03-06 21:53:35 +00:00
|
|
|
raw_size: ValueStats::from_iter(|| bundles.iter().map(|b| b.raw_size as f32)),
|
|
|
|
encoded_size: ValueStats::from_iter(|| bundles.iter().map(|b| b.encoded_size as f32)),
|
2018-03-06 23:36:44 +00:00
|
|
|
chunk_count: ValueStats::from_iter(|| bundles.iter().map(|b| b.chunk_count as f32)),
|
|
|
|
raw_size_meta: ValueStats::from_iter(|| bundles_meta.iter().map(|b| b.raw_size as f32)),
|
|
|
|
encoded_size_meta: ValueStats::from_iter(|| bundles_meta.iter().map(|b| b.encoded_size as f32)),
|
|
|
|
chunk_count_meta: ValueStats::from_iter(|| bundles_meta.iter().map(|b| b.chunk_count as f32)),
|
|
|
|
raw_size_data: ValueStats::from_iter(|| bundles_data.iter().map(|b| b.raw_size as f32)),
|
|
|
|
encoded_size_data: ValueStats::from_iter(|| bundles_data.iter().map(|b| b.encoded_size as f32)),
|
|
|
|
chunk_count_data: ValueStats::from_iter(|| bundles_data.iter().map(|b| b.chunk_count as f32))
|
2018-03-06 21:53:35 +00:00
|
|
|
}
|
|
|
|
}
|
2017-03-21 10:08:01 +00:00
|
|
|
}
|