You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

323 lines
12 KiB

mod config;
mod bundle_map;
mod integrity;
mod basic_io;
mod info;
mod metadata;
mod backup;
mod error;
mod vacuum;
mod backup_file;
mod tarfile;
mod layout;
use ::prelude::*;
use std::mem;
use std::cmp::max;
use std::path::Path;
use std::fs::{self, File};
use std::sync::{Arc, Mutex};
use std::os::unix::fs::symlink;
use std::io::Write;
pub use self::error::RepositoryError;
pub use self::config::Config;
pub use self::metadata::{Inode, FileType, FileData, InodeError};
pub use self::backup::{BackupError, BackupOptions, DiffType};
pub use self::backup_file::{Backup, BackupFileError};
pub use self::integrity::IntegrityError;
pub use self::info::{RepositoryInfo, BundleAnalysis};
pub use self::layout::RepositoryLayout;
use self::bundle_map::BundleMap;
const REPOSITORY_README: &'static [u8] = include_bytes!("../../docs/repository_readme.md");
const DEFAULT_EXCLUDES: &'static [u8] = include_bytes!("../../docs/excludes.default");
pub struct Repository {
pub layout: RepositoryLayout,
pub config: Config,
index: Index,
crypto: Arc<Mutex<Crypto>>,
bundle_map: BundleMap,
next_data_bundle: u32,
next_meta_bundle: u32,
bundles: BundleDb,
data_bundle: Option<BundleWriter>,
meta_bundle: Option<BundleWriter>,
chunker: Chunker,
remote_locks: LockFolder,
local_locks: LockFolder,
lock: LockHandle,
dirty: bool
}
impl Repository {
pub fn create<P: AsRef<Path>, R: AsRef<Path>>(path: P, config: Config, remote: R) -> Result<Self, RepositoryError> {
let layout = RepositoryLayout::new(path.as_ref().to_path_buf());
try!(fs::create_dir(layout.base_path()));
try!(File::create(layout.excludes_path()).and_then(|mut f| f.write_all(DEFAULT_EXCLUDES)));
try!(fs::create_dir(layout.keys_path()));
try!(fs::create_dir(layout.local_locks_path()));
try!(symlink(remote, layout.remote_path()));
try!(File::create(layout.remote_readme_path()).and_then(|mut f| f.write_all(REPOSITORY_README)));
try!(fs::create_dir_all(layout.remote_locks_path()));
try!(config.save(layout.config_path()));
try!(BundleDb::create(layout.clone()));
try!(Index::create(layout.index_path()));
try!(BundleMap::create().save(layout.bundle_map_path()));
try!(fs::create_dir_all(layout.backups_path()));
Self::open(path)
}
#[allow(unknown_lints,useless_let_if_seq)]
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, RepositoryError> {
let layout = RepositoryLayout::new(path.as_ref().to_path_buf());
if !layout.remote_exists() {
return Err(RepositoryError::NoRemote)
}
let config = try!(Config::load(layout.config_path()));
let remote_locks = LockFolder::new(layout.remote_locks_path());
try!(fs::create_dir_all(layout.local_locks_path())); // Added after v0.1.0
let local_locks = LockFolder::new(layout.local_locks_path());
let lock = try!(local_locks.lock(false));
let crypto = Arc::new(Mutex::new(try!(Crypto::open(layout.keys_path()))));
let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone()));
let (index, mut rebuild_index) = match Index::open(layout.index_path()) {
Ok(index) => (index, false),
Err(err) => {
error!("Failed to load local index:\n\tcaused by: {}", err);
(try!(Index::create(layout.index_path())), true)
}
};
let (bundle_map, rebuild_bundle_map) = match BundleMap::load(layout.bundle_map_path()) {
Ok(bundle_map) => (bundle_map, false),
Err(err) => {
error!("Failed to load local bundle map:\n\tcaused by: {}", err);
(BundleMap::create(), true)
}
};
let dirty = layout.dirtyfile_path().exists();
let mut repo = Repository {
layout: layout,
dirty: true,
chunker: config.chunker.create(),
config: config,
index: index,
crypto: crypto,
bundle_map: bundle_map,
next_data_bundle: 0,
next_meta_bundle: 0,
bundles: bundles,
data_bundle: None,
meta_bundle: None,
lock: lock,
remote_locks: remote_locks,
local_locks: local_locks
};
if !rebuild_bundle_map {
let mut save_bundle_map = false;
if !gone.is_empty() {
info!("Removig {} old bundles from index", gone.len());
try!(repo.write_mode());
for bundle in gone {
try!(repo.remove_gone_remote_bundle(bundle))
}
save_bundle_map = true;
}
if !new.is_empty() {
info!("Adding {} new bundles to index", new.len());
try!(repo.write_mode());
for bundle in ProgressIter::new("adding bundles to index", new.len(), new.into_iter()) {
try!(repo.add_new_remote_bundle(bundle))
}
save_bundle_map = true;
}
if save_bundle_map {
try!(repo.write_mode());
try!(repo.save_bundle_map());
}
}
repo.next_meta_bundle = repo.next_free_bundle_id();
repo.next_data_bundle = repo.next_free_bundle_id();
if rebuild_bundle_map {
try!(repo.write_mode());
try!(repo.rebuild_bundle_map());
rebuild_index = true;
}
if rebuild_index {
try!(repo.write_mode());
try!(repo.rebuild_index());
}
repo.dirty = dirty;
Ok(repo)
}
pub fn import<P: AsRef<Path>, R: AsRef<Path>>(path: P, remote: R, key_files: Vec<String>) -> Result<Self, RepositoryError> {
let path = path.as_ref();
let mut repo = try!(Repository::create(path, Config::default(), remote));
for file in key_files {
try!(repo.crypto.lock().unwrap().register_keyfile(file));
}
repo = try!(Repository::open(path));
let mut backups: Vec<(String, Backup)> = try!(repo.get_all_backups()).into_iter().collect();
backups.sort_by_key(|&(_, ref b)| b.timestamp);
if let Some((name, backup)) = backups.pop() {
info!("Taking configuration from the last backup '{}'", name);
repo.config = backup.config;
try!(repo.save_config())
} else {
warn!("No backup found in the repository to take configuration from, please set the configuration manually.");
}
Ok(repo)
}
#[inline]
pub fn register_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), RepositoryError> {
try!(self.write_mode());
Ok(try!(self.crypto.lock().unwrap().register_secret_key(public, secret)))
}
#[inline]
pub fn save_config(&mut self) -> Result<(), RepositoryError> {
try!(self.write_mode());
try!(self.config.save(self.layout.config_path()));
Ok(())
}
#[inline]
pub fn set_encryption(&mut self, public: Option<&PublicKey>) {
if let Some(key) = public {
if !self.crypto.lock().unwrap().contains_secret_key(key) {
warn!("The secret key for that public key is not stored in the repository.")
}
let mut key_bytes = Vec::new();
key_bytes.extend_from_slice(&key[..]);
self.config.encryption = Some((EncryptionMethod::Sodium, key_bytes.into()))
} else {
self.config.encryption = None
}
}
#[inline]
fn save_bundle_map(&self) -> Result<(), RepositoryError> {
try!(self.bundle_map.save(self.layout.bundle_map_path()));
Ok(())
}
#[inline]
fn next_free_bundle_id(&self) -> u32 {
let mut id = max(self.next_data_bundle, self.next_meta_bundle) + 1;
while self.bundle_map.get(id).is_some() {
id += 1;
}
id
}
pub fn set_dirty(&mut self) -> Result<(), RepositoryError> {
self.dirty = true;
let dirtyfile = self.layout.dirtyfile_path();
if !dirtyfile.exists() {
try!(File::create(&dirtyfile));
}
Ok(())
}
pub fn flush(&mut self) -> Result<(), RepositoryError> {
let dirtyfile = self.layout.dirtyfile_path();
if self.dirty && !dirtyfile.exists() {
try!(File::create(&dirtyfile));
}
if self.data_bundle.is_some() {
let mut finished = None;
mem::swap(&mut self.data_bundle, &mut finished);
{
let bundle = try!(self.bundles.add_bundle(finished.unwrap()));
self.bundle_map.set(self.next_data_bundle, bundle.id.clone());
}
self.next_data_bundle = self.next_free_bundle_id()
}
if self.meta_bundle.is_some() {
let mut finished = None;
mem::swap(&mut self.meta_bundle, &mut finished);
{
let bundle = try!(self.bundles.add_bundle(finished.unwrap()));
self.bundle_map.set(self.next_meta_bundle, bundle.id.clone());
}
self.next_meta_bundle = self.next_free_bundle_id()
}
try!(self.bundles.finish_uploads());
try!(self.save_bundle_map());
try!(self.bundles.save_cache());
if !self.dirty && dirtyfile.exists() {
try!(fs::remove_file(&dirtyfile));
}
Ok(())
}
fn add_new_remote_bundle(&mut self, bundle: BundleInfo) -> Result<(), RepositoryError> {
if self.bundle_map.find(&bundle.id).is_some() {
return Ok(())
}
debug!("Adding new bundle to index: {}", bundle.id);
let bundle_id = match bundle.mode {
BundleMode::Data => self.next_data_bundle,
BundleMode::Meta => self.next_meta_bundle
};
let chunks = try!(self.bundles.get_chunk_list(&bundle.id));
self.bundle_map.set(bundle_id, bundle.id.clone());
if self.next_meta_bundle == bundle_id {
self.next_meta_bundle = self.next_free_bundle_id()
}
if self.next_data_bundle == bundle_id {
self.next_data_bundle = self.next_free_bundle_id()
}
for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() {
if let Some(old) = try!(self.index.set(&hash, &Location{bundle: bundle_id as u32, chunk: i as u32})) {
// Duplicate chunk, forced ordering: higher bundle id wins
let old_bundle_id = try!(self.get_bundle_id(old.bundle));
if old_bundle_id > bundle.id {
try!(self.index.set(&hash, &old));
}
}
}
Ok(())
}
fn remove_gone_remote_bundle(&mut self, bundle: BundleInfo) -> Result<(), RepositoryError> {
if let Some(id) = self.bundle_map.find(&bundle.id) {
debug!("Removing bundle from index: {}", bundle.id);
try!(self.bundles.delete_local_bundle(&bundle.id));
try!(self.index.filter(|_key, data| data.bundle != id));
self.bundle_map.remove(id);
}
Ok(())
}
#[inline]
fn write_mode(&mut self) -> Result<(), RepositoryError> {
Ok(try!(self.local_locks.upgrade(&mut self.lock)))
}
#[inline]
fn lock(&self, exclusive: bool) -> Result<LockHandle, RepositoryError> {
Ok(try!(self.remote_locks.lock(exclusive)))
}
#[inline]
pub fn set_clean(&mut self) {
self.dirty = false;
}
}
impl Drop for Repository {
fn drop(&mut self) {
if let Err(err) = self.flush() {
error!("Failed to flush repository: {}", err);
}
}
}