diff --git a/TODO.md b/TODO.md index ca9fc17..b66ff07 100644 --- a/TODO.md +++ b/TODO.md @@ -22,7 +22,6 @@ * Benchmarks * Full fuse method coverage * Clippy -* Central repository layout class ## Other * Homepage diff --git a/src/bundledb/db.rs b/src/bundledb/db.rs index a275c85..3cca1cb 100644 --- a/src/bundledb/db.rs +++ b/src/bundledb/db.rs @@ -53,22 +53,8 @@ quick_error!{ } -pub fn bundle_path(bundle: &BundleId, mut folder: PathBuf, mut count: usize) -> (PathBuf, PathBuf) { - let mut file = bundle.to_string().to_owned() + ".bundle"; - while count >= 100 { - if file.len() < 10 { - break - } - folder = folder.join(&file[0..2]); - file = file[2..].to_string(); - count /= 250; - } - (folder, file.into()) -} - -pub fn load_bundles>(path: P, base: P, bundles: &mut HashMap, crypto: Arc>) -> Result<(Vec, Vec), BundleDbError> { - let base = base.as_ref(); - let mut paths = vec![path.as_ref().to_path_buf()]; +pub fn load_bundles(path: &Path, base: &Path, bundles: &mut HashMap, crypto: Arc>) -> Result<(Vec, Vec), BundleDbError> { + let mut paths = vec![path.to_path_buf()]; let mut bundle_paths = HashSet::new(); while let Some(path) = paths.pop() { for entry in try!(fs::read_dir(path).map_err(BundleDbError::ListBundles)) { @@ -110,12 +96,7 @@ pub fn load_bundles>(path: P, base: P, bundles: &mut HashMap>, local_bundles: HashMap, remote_bundles: HashMap, @@ -124,14 +105,9 @@ pub struct BundleDb { impl BundleDb { - fn new(repo_path: PathBuf, remote_path: PathBuf, local_path: PathBuf, crypto: Arc>) -> Self { + fn new(layout: RepositoryLayout, crypto: Arc>) -> Self { BundleDb { - repo_path: repo_path, - remote_cache_path: local_path.join("remote.cache"), - local_cache_path: local_path.join("local.cache"), - local_bundles_path: local_path.join("cached"), - temp_path: local_path.join("temp"), - remote_path: remote_path, + layout: layout, crypto: crypto, local_bundles: HashMap::new(), remote_bundles: HashMap::new(), @@ -140,36 +116,35 @@ impl BundleDb { } fn load_bundle_list(&mut self) -> Result<(Vec, Vec), BundleDbError> { - let local_cache_path = &self.local_cache_path; - if let Ok(list) = StoredBundle::read_list_from(&local_cache_path) { + if let Ok(list) = StoredBundle::read_list_from(&self.layout.local_bundle_cache_path()) { for bundle in list { self.local_bundles.insert(bundle.id(), bundle); } } - let remote_cache_path = &self.remote_cache_path; - if let Ok(list) = StoredBundle::read_list_from(&remote_cache_path) { + if let Ok(list) = StoredBundle::read_list_from(&self.layout.remote_bundle_cache_path()) { for bundle in list { self.remote_bundles.insert(bundle.id(), bundle); } } - let (new, gone) = try!(load_bundles(&self.local_bundles_path, &self.repo_path, &mut self.local_bundles, self.crypto.clone())); + let base_path = self.layout.base_path(); + let (new, gone) = try!(load_bundles(&self.layout.local_bundles_path(), base_path, &mut self.local_bundles, self.crypto.clone())); if !new.is_empty() || !gone.is_empty() { let bundles: Vec<_> = self.local_bundles.values().cloned().collect(); - try!(StoredBundle::save_list_to(&bundles, &local_cache_path)); + try!(StoredBundle::save_list_to(&bundles, &self.layout.local_bundle_cache_path())); } - let (new, gone) = try!(load_bundles(&self.remote_path, &self.repo_path, &mut self.remote_bundles, self.crypto.clone())); + let (new, gone) = try!(load_bundles(&self.layout.remote_bundles_path(), base_path, &mut self.remote_bundles, self.crypto.clone())); if !new.is_empty() || !gone.is_empty() { let bundles: Vec<_> = self.remote_bundles.values().cloned().collect(); - try!(StoredBundle::save_list_to(&bundles, &remote_cache_path)); + try!(StoredBundle::save_list_to(&bundles, &self.layout.remote_bundle_cache_path())); } Ok((new, gone)) } pub fn save_cache(&self) -> Result<(), BundleDbError> { let bundles: Vec<_> = self.local_bundles.values().cloned().collect(); - try!(StoredBundle::save_list_to(&bundles, &self.local_cache_path)); + try!(StoredBundle::save_list_to(&bundles, &self.layout.local_bundle_cache_path())); let bundles: Vec<_> = self.remote_bundles.values().cloned().collect(); - Ok(try!(StoredBundle::save_list_to(&bundles, &self.remote_cache_path))) + Ok(try!(StoredBundle::save_list_to(&bundles, &self.layout.remote_bundle_cache_path()))) } pub fn update_cache(&mut self, new: &[StoredBundle], gone: &[StoredBundle]) -> Result<(), BundleDbError> { @@ -179,23 +154,18 @@ impl BundleDb { try!(self.copy_remote_bundle_to_cache(bundle)); } } + let base_path = self.layout.base_path(); for bundle in gone { if let Some(bundle) = self.local_bundles.remove(&bundle.id()) { - try!(fs::remove_file(self.repo_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id()))) + try!(fs::remove_file(base_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id()))) } } Ok(()) } - pub fn temp_bundle_path(&self, id: &BundleId) -> PathBuf { - self.temp_path.join(id.to_string().to_owned() + ".bundle") - } - #[inline] - pub fn open, L: AsRef>(repo_path: PathBuf, remote_path: R, local_path: L, crypto: Arc>) -> Result<(Self, Vec, Vec), BundleDbError> { - let remote_path = remote_path.as_ref().to_owned(); - let local_path = local_path.as_ref().to_owned(); - let mut self_ = Self::new(repo_path, remote_path, local_path, crypto); + pub fn open(layout: RepositoryLayout, crypto: Arc>) -> Result<(Self, Vec, Vec), BundleDbError> { + let mut self_ = Self::new(layout, crypto); let (new, gone) = try!(self_.load_bundle_list()); try!(self_.update_cache(&new, &gone)); let new = new.into_iter().map(|s| s.info).collect(); @@ -204,14 +174,11 @@ impl BundleDb { } #[inline] - pub fn create, L: AsRef>(repo_path: PathBuf, remote_path: R, local_path: L, crypto: Arc>) -> Result { - let remote_path = remote_path.as_ref().to_owned(); - let local_path = local_path.as_ref().to_owned(); - let self_ = Self::new(repo_path, remote_path, local_path, crypto); - try!(fs::create_dir_all(&self_.remote_path).context(&self_.remote_path as &Path)); - try!(fs::create_dir_all(&self_.local_bundles_path).context(&self_.local_bundles_path as &Path)); - try!(fs::create_dir_all(&self_.temp_path).context(&self_.temp_path as &Path)); - Ok(self_) + pub fn create(layout: RepositoryLayout) -> Result<(), BundleDbError> { + try!(fs::create_dir_all(layout.remote_bundles_path()).context(&layout.remote_bundles_path() as &Path)); + try!(fs::create_dir_all(layout.local_bundles_path()).context(&layout.local_bundles_path() as &Path)); + try!(fs::create_dir_all(layout.temp_bundles_path()).context(&layout.temp_bundles_path() as &Path)); + Ok(()) } #[inline] @@ -228,7 +195,8 @@ impl BundleDb { } fn get_bundle(&self, stored: &StoredBundle) -> Result { - Ok(try!(BundleReader::load(self.repo_path.join(&stored.path), self.crypto.clone()))) + let base_path = self.layout.base_path(); + Ok(try!(BundleReader::load(base_path.join(&stored.path), self.crypto.clone()))) } pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result, BundleDbError> { @@ -249,9 +217,9 @@ impl BundleDb { fn copy_remote_bundle_to_cache(&mut self, bundle: &StoredBundle) -> Result<(), BundleDbError> { let id = bundle.id(); - let (folder, filename) = bundle_path(&id, self.local_bundles_path.clone(), self.local_bundles.len()); + let (folder, filename) = self.layout.local_bundle_path(&id, self.local_bundles.len()); try!(fs::create_dir_all(&folder).context(&folder as &Path)); - let bundle = try!(bundle.copy_to(&self.repo_path, folder.join(filename))); + let bundle = try!(bundle.copy_to(&self.layout.base_path(), folder.join(filename))); self.local_bundles.insert(id, bundle); Ok(()) } @@ -259,13 +227,12 @@ impl BundleDb { #[inline] pub fn add_bundle(&mut self, bundle: BundleWriter) -> Result { let bundle = try!(bundle.finish(&self)); - let random_id = BundleId::random(); if bundle.info.mode == BundleMode::Meta { try!(self.copy_remote_bundle_to_cache(&bundle)) } - let (folder, filename) = bundle_path(&random_id, self.remote_path.clone(), self.remote_bundles.len()); + let (folder, filename) = self.layout.remote_bundle_path(self.remote_bundles.len()); try!(fs::create_dir_all(&folder).context(&folder as &Path)); - let bundle = try!(bundle.move_to(&self.repo_path, folder.join(filename))); + let bundle = try!(bundle.move_to(&self.layout.base_path(), folder.join(filename))); self.remote_bundles.insert(bundle.id(), bundle.clone()); Ok(bundle.info) } @@ -289,7 +256,8 @@ impl BundleDb { #[inline] pub fn delete_local_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> { if let Some(bundle) = self.local_bundles.remove(bundle) { - try!(fs::remove_file(self.repo_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id()))) + let path = self.layout.base_path().join(&bundle.path); + try!(fs::remove_file(path).map_err(|e| BundleDbError::Remove(e, bundle.id()))) } Ok(()) } @@ -298,7 +266,8 @@ impl BundleDb { pub fn delete_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> { try!(self.delete_local_bundle(bundle)); if let Some(bundle) = self.remote_bundles.remove(bundle) { - fs::remove_file(self.repo_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id())) + let path = self.layout.base_path().join(&bundle.path); + fs::remove_file(path).map_err(|e| BundleDbError::Remove(e, bundle.id())) } else { Err(BundleDbError::NoSuchBundle(bundle.clone())) } diff --git a/src/bundledb/writer.rs b/src/bundledb/writer.rs index b6bd134..6726d17 100644 --- a/src/bundledb/writer.rs +++ b/src/bundledb/writer.rs @@ -101,7 +101,7 @@ impl BundleWriter { if let Some(ref encryption) = self.encryption { chunk_data = try!(self.crypto.lock().unwrap().encrypt(&encryption, &chunk_data)); } - let path = db.temp_bundle_path(&id); + let path = db.layout.temp_bundle_path(); let mut file = BufWriter::new(try!(File::create(&path).context(&path as &Path))); try!(file.write_all(&HEADER_STRING).context(&path as &Path)); try!(file.write_all(&[HEADER_VERSION]).context(&path as &Path)); diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 5028e8a..e01ee8b 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -306,7 +306,7 @@ pub fn run() -> Result<(), ErrorCode> { } let reference_backup = reference_backup.map(|(_, backup)| backup); if !no_default_excludes && !tar { - for line in BufReader::new(checked!(File::open(&repo.excludes_path), "open default excludes file", ErrorCode::LoadExcludes)).lines() { + for line in BufReader::new(checked!(File::open(&repo.layout.excludes_path()), "open default excludes file", ErrorCode::LoadExcludes)).lines() { excludes.push(checked!(line, "read default excludes file", ErrorCode::LoadExcludes)); } } diff --git a/src/index.rs b/src/index.rs index fe61689..dbe1fd5 100644 --- a/src/index.rs +++ b/src/index.rs @@ -152,13 +152,13 @@ impl Index { } #[inline] - pub fn open(path: &Path) -> Result { - Index::new(path, false) + pub fn open>(path: P) -> Result { + Index::new(path.as_ref(), false) } #[inline] - pub fn create(path: &Path) -> Result { - Index::new(path, true) + pub fn create>(path: P) -> Result { + Index::new(path.as_ref(), true) } #[inline] diff --git a/src/prelude.rs b/src/prelude.rs index 04d241f..07254d3 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,7 +1,7 @@ pub use ::util::*; pub use ::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError, BundleDb, BundleWriterError, StoredBundle}; pub use ::chunker::{ChunkerType, Chunker, ChunkerStatus, IChunker, ChunkerError}; -pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, RepositoryIntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError}; +pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, RepositoryIntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError, RepositoryLayout}; pub use ::index::{Index, Location, IndexError}; pub use ::mount::FuseFilesystem; diff --git a/src/repository/backup.rs b/src/repository/backup.rs index 0407a5e..08393bf 100644 --- a/src/repository/backup.rs +++ b/src/repository/backup.rs @@ -38,25 +38,25 @@ pub enum DiffType { impl Repository { pub fn get_backups(&self) -> Result, RepositoryError> { - Ok(try!(Backup::get_all_from(&self.crypto.lock().unwrap(), &self.backups_path))) + Ok(try!(Backup::get_all_from(&self.crypto.lock().unwrap(), self.layout.backups_path()))) } pub fn get_backup(&self, name: &str) -> Result { - Ok(try!(Backup::read_from(&self.crypto.lock().unwrap(), self.backups_path.join(name)))) + Ok(try!(Backup::read_from(&self.crypto.lock().unwrap(), self.layout.backup_path(name)))) } pub fn save_backup(&mut self, backup: &Backup, name: &str) -> Result<(), RepositoryError> { - let path = &self.backups_path.join(name); + let path = self.layout.backup_path(name); try!(fs::create_dir_all(path.parent().unwrap())); Ok(try!(backup.save_to(&self.crypto.lock().unwrap(), self.config.encryption.clone(), path))) } pub fn delete_backup(&self, name: &str) -> Result<(), RepositoryError> { - let mut path = self.backups_path.join(name); + let mut path = self.layout.backup_path(name); try!(fs::remove_file(&path)); loop { path = path.parent().unwrap().to_owned(); - if path == self.backups_path || fs::remove_dir(&path).is_err() { + if path == self.layout.backups_path() || fs::remove_dir(&path).is_err() { break } } diff --git a/src/repository/layout.rs b/src/repository/layout.rs new file mode 100644 index 0000000..bd4432e --- /dev/null +++ b/src/repository/layout.rs @@ -0,0 +1,101 @@ +use ::prelude::*; + +use std::path::{Path, PathBuf}; + +#[derive(Clone)] +pub struct RepositoryLayout(PathBuf); + +impl RepositoryLayout { + pub fn new>(path: P) -> Self { + RepositoryLayout(path.as_ref().to_path_buf()) + } + + pub fn base_path(&self) -> &Path { + &self.0 + } + + pub fn config_path(&self) -> PathBuf { + self.0.join("config.yaml") + } + + pub fn excludes_path(&self) -> PathBuf { + self.0.join("excludes") + } + + pub fn index_path(&self) -> PathBuf { + self.0.join("index") + } + + pub fn keys_path(&self) -> PathBuf { + self.0.join("keys") + } + + pub fn bundle_map_path(&self) -> PathBuf { + self.0.join("bundles.map") + } + + pub fn backups_path(&self) -> PathBuf { + self.0.join("remote/backups") + } + + pub fn backup_path(&self, name: &str) -> PathBuf { + self.backups_path().join(name) + } + + pub fn remote_path(&self) -> PathBuf { + self.0.join("remote") + } + + pub fn remote_readme_path(&self) -> PathBuf { + self.0.join("remote/README.md") + } + + pub fn remote_locks_path(&self) -> PathBuf { + self.0.join("remote/locks") + } + + pub fn remote_bundles_path(&self) -> PathBuf { + self.0.join("remote/bundles") + } + + pub fn local_bundles_path(&self) -> PathBuf { + self.0.join("bundles/cached") + } + + fn bundle_path(&self, bundle: &BundleId, mut folder: PathBuf, mut count: usize) -> (PathBuf, PathBuf) { + let mut file = bundle.to_string().to_owned() + ".bundle"; + while count >= 100 { + if file.len() < 10 { + break + } + folder = folder.join(&file[0..2]); + file = file[2..].to_string(); + count /= 250; + } + (folder, file.into()) + } + + pub fn remote_bundle_path(&self, count: usize) -> (PathBuf, PathBuf) { + self.bundle_path(&BundleId::random(), self.remote_bundles_path(), count) + } + + pub fn local_bundle_path(&self, bundle: &BundleId, count: usize) -> (PathBuf, PathBuf) { + self.bundle_path(bundle, self.local_bundles_path(), count) + } + + pub fn temp_bundles_path(&self) -> PathBuf { + self.0.join("bundles/temp") + } + + pub fn temp_bundle_path(&self) -> PathBuf { + self.temp_bundles_path().join(BundleId::random().to_string().to_owned() + ".bundle") + } + + pub fn local_bundle_cache_path(&self) -> PathBuf { + self.0.join("bundles/local.cache") + } + + pub fn remote_bundle_cache_path(&self) -> PathBuf { + self.0.join("bundles/remote.cache") + } +} diff --git a/src/repository/mod.rs b/src/repository/mod.rs index f93376a..11af174 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -9,12 +9,13 @@ mod error; mod vacuum; mod backup_file; mod tarfile; +mod layout; use ::prelude::*; use std::mem; use std::cmp::max; -use std::path::{PathBuf, Path}; +use std::path::Path; use std::fs::{self, File}; use std::sync::{Arc, Mutex}; use std::os::unix::fs::symlink; @@ -27,6 +28,7 @@ pub use self::backup::{BackupError, BackupOptions, DiffType}; pub use self::backup_file::{Backup, BackupFileError}; pub use self::integrity::RepositoryIntegrityError; pub use self::info::{RepositoryInfo, BundleAnalysis}; +pub use self::layout::RepositoryLayout; use self::bundle_map::BundleMap; @@ -35,9 +37,7 @@ const DEFAULT_EXCLUDES: &'static [u8] = include_bytes!("../../docs/excludes.defa pub struct Repository { - path: PathBuf, - backups_path: PathBuf, - pub excludes_path: PathBuf, + pub layout: RepositoryLayout, pub config: Config, index: Index, crypto: Arc>, @@ -55,62 +55,32 @@ pub struct Repository { impl Repository { pub fn create, R: AsRef>(path: P, config: Config, remote: R) -> Result { let path = path.as_ref().to_owned(); - try!(fs::create_dir(&path)); - let mut excludes = try!(File::create(path.join("excludes"))); - try!(excludes.write_all(DEFAULT_EXCLUDES)); - try!(fs::create_dir(path.join("keys"))); - let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys"))))); - try!(symlink(remote, path.join("remote"))); - let mut remote_readme = try!(File::create(path.join("remote/README.md"))); - try!(remote_readme.write_all(REPOSITORY_README)); - try!(fs::create_dir_all(path.join("remote/locks"))); - let locks = LockFolder::new(path.join("remote/locks")); - let bundles = try!(BundleDb::create( - path.to_path_buf(), - path.join("remote/bundles"), - path.join("bundles"), - crypto.clone() - )); - let index = try!(Index::create(&path.join("index"))); - try!(config.save(path.join("config.yaml"))); - let bundle_map = BundleMap::create(); - try!(bundle_map.save(path.join("bundles.map"))); - try!(fs::create_dir_all(&path.join("remote/backups"))); - Ok(Repository { - backups_path: path.join("remote/backups"), - excludes_path: path.join("excludes"), - path: path, - chunker: config.chunker.create(), - config: config, - index: index, - bundle_map: bundle_map, - next_data_bundle: 1, - next_meta_bundle: 0, - bundles: bundles, - data_bundle: None, - meta_bundle: None, - crypto: crypto, - locks: locks - }) + let layout = RepositoryLayout::new(path.clone()); + try!(fs::create_dir(layout.base_path())); + try!(File::create(layout.excludes_path()).and_then(|mut f| f.write_all(DEFAULT_EXCLUDES))); + try!(fs::create_dir(layout.keys_path())); + try!(symlink(remote, layout.remote_path())); + try!(File::create(layout.remote_readme_path()).and_then(|mut f| f.write_all(REPOSITORY_README))); + try!(fs::create_dir_all(layout.remote_locks_path())); + try!(config.save(layout.config_path())); + try!(BundleDb::create(layout.clone())); + try!(Index::create(layout.index_path())); + try!(BundleMap::create().save(layout.bundle_map_path())); + try!(fs::create_dir_all(layout.backups_path())); + Self::open(path) } pub fn open>(path: P) -> Result { let path = path.as_ref().to_owned(); - let config = try!(Config::load(path.join("config.yaml"))); - let locks = LockFolder::new(path.join("remote/locks")); - let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys"))))); - let (bundles, new, gone) = try!(BundleDb::open( - path.to_path_buf(), - path.join("remote/bundles"), - path.join("bundles"), - crypto.clone() - )); - let index = try!(Index::open(&path.join("index"))); - let bundle_map = try!(BundleMap::load(path.join("bundles.map"))); + let layout = RepositoryLayout::new(path.clone()); + let config = try!(Config::load(layout.config_path())); + let locks = LockFolder::new(layout.remote_locks_path()); + let crypto = Arc::new(Mutex::new(try!(Crypto::open(layout.keys_path())))); + let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone())); + let index = try!(Index::open(layout.index_path())); + let bundle_map = try!(BundleMap::load(layout.bundle_map_path())); let mut repo = Repository { - backups_path: path.join("remote/backups"), - excludes_path: path.join("excludes"), - path: path, + layout: layout, chunker: config.chunker.create(), config: config, index: index, @@ -160,7 +130,7 @@ impl Repository { } pub fn save_config(&mut self) -> Result<(), RepositoryError> { - try!(self.config.save(self.path.join("config.yaml"))); + try!(self.config.save(self.layout.config_path())); Ok(()) } @@ -180,7 +150,7 @@ impl Repository { #[inline] fn save_bundle_map(&self) -> Result<(), RepositoryError> { - try!(self.bundle_map.save(self.path.join("bundles.map"))); + try!(self.bundle_map.save(self.layout.bundle_map_path())); Ok(()) } diff --git a/src/repository/vacuum.rs b/src/repository/vacuum.rs index 637ccdf..4862629 100644 --- a/src/repository/vacuum.rs +++ b/src/repository/vacuum.rs @@ -64,7 +64,7 @@ impl Repository { for id in rewrite_bundles { try!(self.delete_bundle(id)); } - try!(self.bundle_map.save(self.path.join("bundles.map"))); + try!(self.save_bundle_map()); Ok(()) } }