zvault/src/repository/mod.rs

349 lines
12 KiB
Rust
Raw Normal View History

2017-03-10 11:43:32 +00:00
mod config;
mod bundle_map;
mod integrity;
mod basic_io;
2017-03-15 07:27:27 +00:00
mod info;
2017-03-15 11:32:44 +00:00
mod metadata;
2017-03-15 20:53:05 +00:00
mod backup;
2017-03-16 08:42:30 +00:00
mod error;
2017-03-20 13:03:29 +00:00
mod vacuum;
2017-03-24 08:26:55 +00:00
mod backup_file;
2017-04-03 13:18:06 +00:00
mod tarfile;
2017-04-08 12:35:10 +00:00
mod layout;
2017-03-10 11:43:32 +00:00
2017-03-21 10:28:11 +00:00
use ::prelude::*;
2017-03-10 11:43:32 +00:00
use std::mem;
use std::cmp::max;
2017-04-08 12:35:10 +00:00
use std::path::Path;
2017-03-26 18:33:32 +00:00
use std::fs::{self, File};
2017-03-18 16:22:11 +00:00
use std::sync::{Arc, Mutex};
2017-03-22 13:10:42 +00:00
use std::os::unix::fs::symlink;
2017-03-26 18:33:32 +00:00
use std::io::Write;
2017-03-10 11:43:32 +00:00
2017-03-16 08:42:30 +00:00
pub use self::error::RepositoryError;
2017-03-10 11:43:32 +00:00
pub use self::config::Config;
2017-04-03 13:18:06 +00:00
pub use self::metadata::{Inode, FileType, FileData, InodeError};
2017-03-29 21:24:26 +00:00
pub use self::backup::{BackupError, BackupOptions, DiffType};
2017-03-24 08:26:55 +00:00
pub use self::backup_file::{Backup, BackupFileError};
pub use self::integrity::IntegrityError;
2017-03-25 11:43:49 +00:00
pub use self::info::{RepositoryInfo, BundleAnalysis};
2017-04-08 12:35:10 +00:00
pub use self::layout::RepositoryLayout;
2017-03-15 07:27:27 +00:00
use self::bundle_map::BundleMap;
2017-03-10 11:43:32 +00:00
2017-04-02 16:55:53 +00:00
const REPOSITORY_README: &'static [u8] = include_bytes!("../../docs/repository_readme.md");
2017-04-03 13:18:06 +00:00
const DEFAULT_EXCLUDES: &'static [u8] = include_bytes!("../../docs/excludes.default");
2017-03-26 18:33:32 +00:00
2017-03-10 11:43:32 +00:00
pub struct Repository {
2017-04-08 12:35:10 +00:00
pub layout: RepositoryLayout,
2017-03-18 16:22:11 +00:00
pub config: Config,
2017-03-10 11:43:32 +00:00
index: Index,
2017-03-18 16:22:11 +00:00
crypto: Arc<Mutex<Crypto>>,
2017-03-10 11:43:32 +00:00
bundle_map: BundleMap,
2017-04-03 12:05:16 +00:00
next_data_bundle: u32,
2017-03-10 11:43:32 +00:00
next_meta_bundle: u32,
bundles: BundleDb,
2017-04-03 12:05:16 +00:00
data_bundle: Option<BundleWriter>,
2017-03-10 11:43:32 +00:00
meta_bundle: Option<BundleWriter>,
2017-03-24 07:56:57 +00:00
chunker: Chunker,
remote_locks: LockFolder,
local_locks: LockFolder,
lock: LockHandle,
2017-04-10 18:15:13 +00:00
dirty: bool
2017-03-10 11:43:32 +00:00
}
impl Repository {
2017-03-22 13:10:42 +00:00
pub fn create<P: AsRef<Path>, R: AsRef<Path>>(path: P, config: Config, remote: R) -> Result<Self, RepositoryError> {
let layout = RepositoryLayout::new(path.as_ref().to_path_buf());
2017-04-08 12:35:10 +00:00
try!(fs::create_dir(layout.base_path()));
try!(File::create(layout.excludes_path()).and_then(|mut f| f.write_all(DEFAULT_EXCLUDES)));
try!(fs::create_dir(layout.keys_path()));
try!(fs::create_dir(layout.local_locks_path()));
2017-04-08 12:35:10 +00:00
try!(symlink(remote, layout.remote_path()));
try!(File::create(layout.remote_readme_path()).and_then(|mut f| f.write_all(REPOSITORY_README)));
try!(fs::create_dir_all(layout.remote_locks_path()));
try!(config.save(layout.config_path()));
try!(BundleDb::create(layout.clone()));
try!(Index::create(layout.index_path()));
try!(BundleMap::create().save(layout.bundle_map_path()));
try!(fs::create_dir_all(layout.backups_path()));
Self::open(path)
2017-03-10 11:43:32 +00:00
}
#[allow(unknown_lints,useless_let_if_seq)]
2017-03-16 08:42:30 +00:00
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, RepositoryError> {
let layout = RepositoryLayout::new(path.as_ref().to_path_buf());
if !layout.remote_exists() {
return Err(RepositoryError::NoRemote)
}
2017-04-08 12:35:10 +00:00
let config = try!(Config::load(layout.config_path()));
let remote_locks = LockFolder::new(layout.remote_locks_path());
try!(fs::create_dir_all(layout.local_locks_path())); // Added after v0.1.0
let local_locks = LockFolder::new(layout.local_locks_path());
let lock = try!(local_locks.lock(false));
2017-04-08 12:35:10 +00:00
let crypto = Arc::new(Mutex::new(try!(Crypto::open(layout.keys_path()))));
let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone()));
let (index, mut rebuild_index) = match Index::open(layout.index_path()) {
Ok(index) => (index, false),
Err(err) => {
error!("Failed to load local index:\n\tcaused by: {}", err);
(try!(Index::create(layout.index_path())), true)
}
};
let (bundle_map, rebuild_bundle_map) = match BundleMap::load(layout.bundle_map_path()) {
Ok(bundle_map) => (bundle_map, false),
Err(err) => {
error!("Failed to load local bundle map:\n\tcaused by: {}", err);
(BundleMap::create(), true)
}
};
2017-04-10 18:15:13 +00:00
let dirty = layout.dirtyfile_path().exists();
2017-03-10 11:43:32 +00:00
let mut repo = Repository {
2017-04-08 12:35:10 +00:00
layout: layout,
2017-04-10 18:15:13 +00:00
dirty: true,
2017-03-10 11:43:32 +00:00
chunker: config.chunker.create(),
config: config,
index: index,
2017-03-18 16:22:11 +00:00
crypto: crypto,
2017-03-10 11:43:32 +00:00
bundle_map: bundle_map,
2017-04-03 12:05:16 +00:00
next_data_bundle: 0,
2017-03-10 11:43:32 +00:00
next_meta_bundle: 0,
bundles: bundles,
2017-04-03 12:05:16 +00:00
data_bundle: None,
2017-03-10 11:43:32 +00:00
meta_bundle: None,
lock: lock,
remote_locks: remote_locks,
local_locks: local_locks
2017-03-10 11:43:32 +00:00
};
if !rebuild_bundle_map {
let mut save_bundle_map = false;
if !new.is_empty() {
info!("Adding {} new bundles to index", new.len());
try!(repo.write_mode());
for bundle in ProgressIter::new("adding bundles to index", new.len(), new.into_iter()) {
try!(repo.add_new_remote_bundle(bundle))
}
save_bundle_map = true;
2017-04-11 06:49:45 +00:00
}
if !gone.is_empty() {
info!("Removig {} old bundles from index", gone.len());
try!(repo.write_mode());
for bundle in gone {
try!(repo.remove_gone_remote_bundle(bundle))
}
save_bundle_map = true;
}
if save_bundle_map {
try!(repo.write_mode());
try!(repo.save_bundle_map());
2017-04-11 06:49:45 +00:00
}
2017-03-22 11:27:17 +00:00
}
2017-03-10 11:43:32 +00:00
repo.next_meta_bundle = repo.next_free_bundle_id();
2017-04-03 12:05:16 +00:00
repo.next_data_bundle = repo.next_free_bundle_id();
if rebuild_bundle_map {
try!(repo.write_mode());
try!(repo.rebuild_bundle_map());
rebuild_index = true;
}
if rebuild_index {
try!(repo.write_mode());
try!(repo.rebuild_index());
}
2017-04-10 18:15:13 +00:00
repo.dirty = dirty;
2017-03-10 11:43:32 +00:00
Ok(repo)
}
2017-03-22 16:28:45 +00:00
pub fn import<P: AsRef<Path>, R: AsRef<Path>>(path: P, remote: R, key_files: Vec<String>) -> Result<Self, RepositoryError> {
2017-03-22 13:42:27 +00:00
let path = path.as_ref();
2017-03-22 16:28:45 +00:00
let mut repo = try!(Repository::create(path, Config::default(), remote));
for file in key_files {
try!(repo.crypto.lock().unwrap().register_keyfile(file));
}
repo = try!(Repository::open(path));
2017-04-07 09:05:28 +00:00
let mut backups: Vec<(String, Backup)> = try!(repo.get_backups()).into_iter().collect();
backups.sort_by_key(|&(_, ref b)| b.date);
if let Some((name, backup)) = backups.pop() {
info!("Taking configuration from the last backup '{}'", name);
2017-03-22 13:42:27 +00:00
repo.config = backup.config;
try!(repo.save_config())
2017-04-07 09:05:28 +00:00
} else {
warn!("No backup found in the repository to take configuration from, please set the configuration manually.");
2017-03-22 13:42:27 +00:00
}
Ok(repo)
}
2017-03-18 16:22:11 +00:00
#[inline]
pub fn register_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), RepositoryError> {
try!(self.write_mode());
2017-03-18 16:22:11 +00:00
Ok(try!(self.crypto.lock().unwrap().register_secret_key(public, secret)))
}
2017-04-10 18:35:28 +00:00
#[inline]
2017-03-18 16:22:11 +00:00
pub fn save_config(&mut self) -> Result<(), RepositoryError> {
try!(self.write_mode());
2017-04-08 12:35:10 +00:00
try!(self.config.save(self.layout.config_path()));
2017-03-18 16:22:11 +00:00
Ok(())
}
#[inline]
pub fn set_encryption(&mut self, public: Option<&PublicKey>) {
if let Some(key) = public {
if !self.crypto.lock().unwrap().contains_secret_key(key) {
warn!("The secret key for that public key is not stored in the repository.")
}
2017-03-18 16:22:11 +00:00
let mut key_bytes = Vec::new();
key_bytes.extend_from_slice(&key[..]);
self.config.encryption = Some((EncryptionMethod::Sodium, key_bytes.into()))
} else {
self.config.encryption = None
}
}
2017-03-10 11:43:32 +00:00
#[inline]
2017-03-16 08:42:30 +00:00
fn save_bundle_map(&self) -> Result<(), RepositoryError> {
2017-04-08 12:35:10 +00:00
try!(self.bundle_map.save(self.layout.bundle_map_path()));
2017-03-16 08:42:30 +00:00
Ok(())
2017-03-10 11:43:32 +00:00
}
#[inline]
fn next_free_bundle_id(&self) -> u32 {
2017-04-03 12:05:16 +00:00
let mut id = max(self.next_data_bundle, self.next_meta_bundle) + 1;
2017-03-10 11:43:32 +00:00
while self.bundle_map.get(id).is_some() {
id += 1;
}
id
}
2017-04-12 08:34:36 +00:00
pub fn set_dirty(&mut self) -> Result<(), RepositoryError> {
self.dirty = true;
let dirtyfile = self.layout.dirtyfile_path();
if !dirtyfile.exists() {
try!(File::create(&dirtyfile));
}
Ok(())
}
2017-03-16 08:42:30 +00:00
pub fn flush(&mut self) -> Result<(), RepositoryError> {
2017-04-10 18:15:13 +00:00
let dirtyfile = self.layout.dirtyfile_path();
if self.dirty && !dirtyfile.exists() {
try!(File::create(&dirtyfile));
}
2017-04-03 12:05:16 +00:00
if self.data_bundle.is_some() {
2017-03-10 11:43:32 +00:00
let mut finished = None;
2017-04-03 12:05:16 +00:00
mem::swap(&mut self.data_bundle, &mut finished);
2017-03-10 11:43:32 +00:00
{
2017-03-16 08:42:30 +00:00
let bundle = try!(self.bundles.add_bundle(finished.unwrap()));
2017-04-03 12:05:16 +00:00
self.bundle_map.set(self.next_data_bundle, bundle.id.clone());
2017-03-10 11:43:32 +00:00
}
2017-04-03 12:05:16 +00:00
self.next_data_bundle = self.next_free_bundle_id()
2017-03-10 11:43:32 +00:00
}
if self.meta_bundle.is_some() {
let mut finished = None;
mem::swap(&mut self.meta_bundle, &mut finished);
{
2017-03-16 08:42:30 +00:00
let bundle = try!(self.bundles.add_bundle(finished.unwrap()));
2017-03-22 11:27:17 +00:00
self.bundle_map.set(self.next_meta_bundle, bundle.id.clone());
2017-03-10 11:43:32 +00:00
}
self.next_meta_bundle = self.next_free_bundle_id()
}
2017-04-10 06:53:55 +00:00
try!(self.bundles.finish_uploads());
2017-03-16 08:42:30 +00:00
try!(self.save_bundle_map());
2017-03-22 13:10:42 +00:00
try!(self.bundles.save_cache());
2017-04-10 18:15:13 +00:00
if !self.dirty && dirtyfile.exists() {
try!(fs::remove_file(&dirtyfile));
}
2017-03-10 11:43:32 +00:00
Ok(())
}
2017-03-22 11:27:17 +00:00
fn add_new_remote_bundle(&mut self, bundle: BundleInfo) -> Result<(), RepositoryError> {
2017-04-10 16:21:26 +00:00
if self.bundle_map.find(&bundle.id).is_some() {
return Ok(())
}
2017-04-11 06:49:45 +00:00
debug!("Adding new bundle to index: {}", bundle.id);
2017-03-22 11:27:17 +00:00
let bundle_id = match bundle.mode {
2017-04-03 12:05:16 +00:00
BundleMode::Data => self.next_data_bundle,
2017-03-22 11:27:17 +00:00
BundleMode::Meta => self.next_meta_bundle
};
let chunks = try!(self.bundles.get_chunk_list(&bundle.id));
self.bundle_map.set(bundle_id, bundle.id);
if self.next_meta_bundle == bundle_id {
self.next_meta_bundle = self.next_free_bundle_id()
}
2017-04-03 12:05:16 +00:00
if self.next_data_bundle == bundle_id {
self.next_data_bundle = self.next_free_bundle_id()
2017-03-22 11:27:17 +00:00
}
for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() {
try!(self.index.set(&hash, &Location{bundle: bundle_id as u32, chunk: i as u32}));
}
Ok(())
}
fn rebuild_bundle_map(&mut self) -> Result<(), RepositoryError> {
info!("Rebuilding bundle map from bundles");
2017-04-10 16:21:26 +00:00
self.bundle_map = BundleMap::create();
for bundle in self.bundles.list_bundles() {
let bundle_id = match bundle.mode {
BundleMode::Data => self.next_data_bundle,
BundleMode::Meta => self.next_meta_bundle
};
self.bundle_map.set(bundle_id, bundle.id.clone());
if self.next_meta_bundle == bundle_id {
self.next_meta_bundle = self.next_free_bundle_id()
}
if self.next_data_bundle == bundle_id {
self.next_data_bundle = self.next_free_bundle_id()
}
}
self.save_bundle_map()
}
fn rebuild_index(&mut self) -> Result<(), RepositoryError> {
info!("Rebuilding index from bundles");
2017-04-07 16:57:49 +00:00
self.index.clear();
for (num, id) in self.bundle_map.bundles() {
let chunks = try!(self.bundles.get_chunk_list(&id));
for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() {
try!(self.index.set(&hash, &Location{bundle: num as u32, chunk: i as u32}));
}
}
Ok(())
}
2017-03-22 11:27:17 +00:00
fn remove_gone_remote_bundle(&mut self, bundle: BundleInfo) -> Result<(), RepositoryError> {
if let Some(id) = self.bundle_map.find(&bundle.id) {
2017-04-09 10:04:28 +00:00
debug!("Removing bundle from index: {}", bundle.id);
try!(self.bundles.delete_local_bundle(&bundle.id));
try!(self.index.filter(|_key, data| data.bundle != id));
2017-03-22 11:27:17 +00:00
self.bundle_map.remove(id);
}
Ok(())
}
2017-03-24 07:56:57 +00:00
#[inline]
fn write_mode(&mut self) -> Result<(), RepositoryError> {
Ok(try!(self.local_locks.upgrade(&mut self.lock)))
}
2017-04-10 18:35:28 +00:00
#[inline]
2017-03-24 07:56:57 +00:00
fn lock(&self, exclusive: bool) -> Result<LockHandle, RepositoryError> {
Ok(try!(self.remote_locks.lock(exclusive)))
2017-03-24 07:56:57 +00:00
}
2017-04-10 18:15:13 +00:00
2017-04-10 18:35:28 +00:00
#[inline]
2017-04-10 18:15:13 +00:00
pub fn set_clean(&mut self) {
2017-04-10 18:35:28 +00:00
self.dirty = false;
2017-04-10 18:15:13 +00:00
}
2017-03-10 11:43:32 +00:00
}
2017-04-03 13:18:06 +00:00
2017-03-10 11:43:32 +00:00
impl Drop for Repository {
fn drop(&mut self) {
2017-04-12 09:34:31 +00:00
if let Err(err) = self.flush() {
error!("Failed to flush repository: {}", err);
}
2017-03-10 11:43:32 +00:00
}
}