mod config; mod bundle_map; mod integrity; mod basic_io; mod info; mod metadata; mod backup; mod error; mod vacuum; mod backup_file; mod tarfile; mod layout; use ::prelude::*; use std::mem; use std::cmp::max; use std::path::Path; use std::fs::{self, File}; use std::sync::{Arc, Mutex}; use std::os::unix::fs::symlink; use std::io::Write; pub use self::error::RepositoryError; pub use self::config::Config; pub use self::metadata::{Inode, FileType, FileData, InodeError}; pub use self::backup::{BackupError, BackupOptions, DiffType}; pub use self::backup_file::{Backup, BackupFileError}; pub use self::integrity::IntegrityError; pub use self::info::{RepositoryInfo, BundleAnalysis}; pub use self::layout::RepositoryLayout; use self::bundle_map::BundleMap; const REPOSITORY_README: &'static [u8] = include_bytes!("../../docs/repository_readme.md"); const DEFAULT_EXCLUDES: &'static [u8] = include_bytes!("../../docs/excludes.default"); pub struct Repository { pub layout: RepositoryLayout, pub config: Config, index: Index, crypto: Arc>, bundle_map: BundleMap, next_data_bundle: u32, next_meta_bundle: u32, bundles: BundleDb, data_bundle: Option, meta_bundle: Option, chunker: Chunker, remote_locks: LockFolder, local_locks: LockFolder, lock: LockHandle, dirty: bool } impl Repository { pub fn create, R: AsRef>(path: P, config: Config, remote: R) -> Result { let layout = RepositoryLayout::new(path.as_ref().to_path_buf()); try!(fs::create_dir(layout.base_path())); try!(File::create(layout.excludes_path()).and_then(|mut f| f.write_all(DEFAULT_EXCLUDES))); try!(fs::create_dir(layout.keys_path())); try!(fs::create_dir(layout.local_locks_path())); try!(symlink(remote, layout.remote_path())); try!(File::create(layout.remote_readme_path()).and_then(|mut f| f.write_all(REPOSITORY_README))); try!(fs::create_dir_all(layout.remote_locks_path())); try!(config.save(layout.config_path())); try!(BundleDb::create(layout.clone())); try!(Index::create(layout.index_path())); try!(BundleMap::create().save(layout.bundle_map_path())); try!(fs::create_dir_all(layout.backups_path())); Self::open(path) } #[allow(unknown_lints,useless_let_if_seq)] pub fn open>(path: P) -> Result { let layout = RepositoryLayout::new(path.as_ref().to_path_buf()); if !layout.remote_exists() { return Err(RepositoryError::NoRemote) } let config = try!(Config::load(layout.config_path())); let remote_locks = LockFolder::new(layout.remote_locks_path()); try!(fs::create_dir_all(layout.local_locks_path())); // Added after v0.1.0 let local_locks = LockFolder::new(layout.local_locks_path()); let lock = try!(local_locks.lock(false)); let crypto = Arc::new(Mutex::new(try!(Crypto::open(layout.keys_path())))); let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone())); let (index, mut rebuild_index) = match Index::open(layout.index_path()) { Ok(index) => (index, false), Err(err) => { error!("Failed to load local index:\n\tcaused by: {}", err); (try!(Index::create(layout.index_path())), true) } }; let (bundle_map, rebuild_bundle_map) = match BundleMap::load(layout.bundle_map_path()) { Ok(bundle_map) => (bundle_map, false), Err(err) => { error!("Failed to load local bundle map:\n\tcaused by: {}", err); (BundleMap::create(), true) } }; let dirty = layout.dirtyfile_path().exists(); let mut repo = Repository { layout: layout, dirty: true, chunker: config.chunker.create(), config: config, index: index, crypto: crypto, bundle_map: bundle_map, next_data_bundle: 0, next_meta_bundle: 0, bundles: bundles, data_bundle: None, meta_bundle: None, lock: lock, remote_locks: remote_locks, local_locks: local_locks }; if !rebuild_bundle_map { let mut save_bundle_map = false; if !new.is_empty() { info!("Adding {} new bundles to index", new.len()); try!(repo.write_mode()); for bundle in ProgressIter::new("adding bundles to index", new.len(), new.into_iter()) { try!(repo.add_new_remote_bundle(bundle)) } save_bundle_map = true; } if !gone.is_empty() { info!("Removig {} old bundles from index", gone.len()); try!(repo.write_mode()); for bundle in gone { try!(repo.remove_gone_remote_bundle(bundle)) } save_bundle_map = true; } if save_bundle_map { try!(repo.write_mode()); try!(repo.save_bundle_map()); } } repo.next_meta_bundle = repo.next_free_bundle_id(); repo.next_data_bundle = repo.next_free_bundle_id(); if rebuild_bundle_map { try!(repo.write_mode()); try!(repo.rebuild_bundle_map()); rebuild_index = true; } if rebuild_index { try!(repo.write_mode()); try!(repo.rebuild_index()); } repo.dirty = dirty; Ok(repo) } pub fn import, R: AsRef>(path: P, remote: R, key_files: Vec) -> Result { let path = path.as_ref(); let mut repo = try!(Repository::create(path, Config::default(), remote)); for file in key_files { try!(repo.crypto.lock().unwrap().register_keyfile(file)); } repo = try!(Repository::open(path)); let mut backups: Vec<(String, Backup)> = try!(repo.get_backups()).into_iter().collect(); backups.sort_by_key(|&(_, ref b)| b.date); if let Some((name, backup)) = backups.pop() { info!("Taking configuration from the last backup '{}'", name); repo.config = backup.config; try!(repo.save_config()) } else { warn!("No backup found in the repository to take configuration from, please set the configuration manually."); } Ok(repo) } #[inline] pub fn register_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), RepositoryError> { try!(self.write_mode()); Ok(try!(self.crypto.lock().unwrap().register_secret_key(public, secret))) } #[inline] pub fn save_config(&mut self) -> Result<(), RepositoryError> { try!(self.write_mode()); try!(self.config.save(self.layout.config_path())); Ok(()) } #[inline] pub fn set_encryption(&mut self, public: Option<&PublicKey>) { if let Some(key) = public { if !self.crypto.lock().unwrap().contains_secret_key(key) { warn!("The secret key for that public key is not stored in the repository.") } let mut key_bytes = Vec::new(); key_bytes.extend_from_slice(&key[..]); self.config.encryption = Some((EncryptionMethod::Sodium, key_bytes.into())) } else { self.config.encryption = None } } #[inline] fn save_bundle_map(&self) -> Result<(), RepositoryError> { try!(self.bundle_map.save(self.layout.bundle_map_path())); Ok(()) } #[inline] fn next_free_bundle_id(&self) -> u32 { let mut id = max(self.next_data_bundle, self.next_meta_bundle) + 1; while self.bundle_map.get(id).is_some() { id += 1; } id } pub fn set_dirty(&mut self) -> Result<(), RepositoryError> { self.dirty = true; let dirtyfile = self.layout.dirtyfile_path(); if !dirtyfile.exists() { try!(File::create(&dirtyfile)); } Ok(()) } pub fn flush(&mut self) -> Result<(), RepositoryError> { let dirtyfile = self.layout.dirtyfile_path(); if self.dirty && !dirtyfile.exists() { try!(File::create(&dirtyfile)); } if self.data_bundle.is_some() { let mut finished = None; mem::swap(&mut self.data_bundle, &mut finished); { let bundle = try!(self.bundles.add_bundle(finished.unwrap())); self.bundle_map.set(self.next_data_bundle, bundle.id.clone()); } self.next_data_bundle = self.next_free_bundle_id() } if self.meta_bundle.is_some() { let mut finished = None; mem::swap(&mut self.meta_bundle, &mut finished); { let bundle = try!(self.bundles.add_bundle(finished.unwrap())); self.bundle_map.set(self.next_meta_bundle, bundle.id.clone()); } self.next_meta_bundle = self.next_free_bundle_id() } try!(self.bundles.finish_uploads()); try!(self.save_bundle_map()); try!(self.bundles.save_cache()); if !self.dirty && dirtyfile.exists() { try!(fs::remove_file(&dirtyfile)); } Ok(()) } fn add_new_remote_bundle(&mut self, bundle: BundleInfo) -> Result<(), RepositoryError> { if self.bundle_map.find(&bundle.id).is_some() { return Ok(()) } debug!("Adding new bundle to index: {}", bundle.id); let bundle_id = match bundle.mode { BundleMode::Data => self.next_data_bundle, BundleMode::Meta => self.next_meta_bundle }; let chunks = try!(self.bundles.get_chunk_list(&bundle.id)); self.bundle_map.set(bundle_id, bundle.id); if self.next_meta_bundle == bundle_id { self.next_meta_bundle = self.next_free_bundle_id() } if self.next_data_bundle == bundle_id { self.next_data_bundle = self.next_free_bundle_id() } for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() { try!(self.index.set(&hash, &Location{bundle: bundle_id as u32, chunk: i as u32})); } Ok(()) } fn rebuild_bundle_map(&mut self) -> Result<(), RepositoryError> { info!("Rebuilding bundle map from bundles"); self.bundle_map = BundleMap::create(); for bundle in self.bundles.list_bundles() { let bundle_id = match bundle.mode { BundleMode::Data => self.next_data_bundle, BundleMode::Meta => self.next_meta_bundle }; self.bundle_map.set(bundle_id, bundle.id.clone()); if self.next_meta_bundle == bundle_id { self.next_meta_bundle = self.next_free_bundle_id() } if self.next_data_bundle == bundle_id { self.next_data_bundle = self.next_free_bundle_id() } } self.save_bundle_map() } fn rebuild_index(&mut self) -> Result<(), RepositoryError> { info!("Rebuilding index from bundles"); self.index.clear(); for (num, id) in self.bundle_map.bundles() { let chunks = try!(self.bundles.get_chunk_list(&id)); for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() { try!(self.index.set(&hash, &Location{bundle: num as u32, chunk: i as u32})); } } Ok(()) } fn remove_gone_remote_bundle(&mut self, bundle: BundleInfo) -> Result<(), RepositoryError> { if let Some(id) = self.bundle_map.find(&bundle.id) { debug!("Removing bundle from index: {}", bundle.id); try!(self.bundles.delete_local_bundle(&bundle.id)); try!(self.index.filter(|_key, data| data.bundle != id)); self.bundle_map.remove(id); } Ok(()) } #[inline] fn write_mode(&mut self) -> Result<(), RepositoryError> { Ok(try!(self.local_locks.upgrade(&mut self.lock))) } #[inline] fn lock(&self, exclusive: bool) -> Result { Ok(try!(self.remote_locks.lock(exclusive))) } #[inline] pub fn set_clean(&mut self) { self.dirty = false; } } impl Drop for Repository { fn drop(&mut self) { if let Err(err) = self.flush() { error!("Failed to flush repository: {}", err); } } }