Central repository layout class

pull/10/head
Dennis Schwerdel 2017-04-08 14:35:10 +02:00 committed by Dennis Schwerdel
parent c1e4cb2bdf
commit a29e95e4f4
10 changed files with 174 additions and 135 deletions

View File

@ -22,7 +22,6 @@
* Benchmarks
* Full fuse method coverage
* Clippy
* Central repository layout class
## Other
* Homepage

View File

@ -53,22 +53,8 @@ quick_error!{
}
pub fn bundle_path(bundle: &BundleId, mut folder: PathBuf, mut count: usize) -> (PathBuf, PathBuf) {
let mut file = bundle.to_string().to_owned() + ".bundle";
while count >= 100 {
if file.len() < 10 {
break
}
folder = folder.join(&file[0..2]);
file = file[2..].to_string();
count /= 250;
}
(folder, file.into())
}
pub fn load_bundles<P: AsRef<Path>>(path: P, base: P, bundles: &mut HashMap<BundleId, StoredBundle>, crypto: Arc<Mutex<Crypto>>) -> Result<(Vec<StoredBundle>, Vec<StoredBundle>), BundleDbError> {
let base = base.as_ref();
let mut paths = vec![path.as_ref().to_path_buf()];
pub fn load_bundles(path: &Path, base: &Path, bundles: &mut HashMap<BundleId, StoredBundle>, crypto: Arc<Mutex<Crypto>>) -> Result<(Vec<StoredBundle>, Vec<StoredBundle>), BundleDbError> {
let mut paths = vec![path.to_path_buf()];
let mut bundle_paths = HashSet::new();
while let Some(path) = paths.pop() {
for entry in try!(fs::read_dir(path).map_err(BundleDbError::ListBundles)) {
@ -110,12 +96,7 @@ pub fn load_bundles<P: AsRef<Path>>(path: P, base: P, bundles: &mut HashMap<Bund
pub struct BundleDb {
repo_path: PathBuf,
remote_path: PathBuf,
local_bundles_path: PathBuf,
temp_path: PathBuf,
remote_cache_path: PathBuf,
local_cache_path: PathBuf,
pub layout: RepositoryLayout,
crypto: Arc<Mutex<Crypto>>,
local_bundles: HashMap<BundleId, StoredBundle>,
remote_bundles: HashMap<BundleId, StoredBundle>,
@ -124,14 +105,9 @@ pub struct BundleDb {
impl BundleDb {
fn new(repo_path: PathBuf, remote_path: PathBuf, local_path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Self {
fn new(layout: RepositoryLayout, crypto: Arc<Mutex<Crypto>>) -> Self {
BundleDb {
repo_path: repo_path,
remote_cache_path: local_path.join("remote.cache"),
local_cache_path: local_path.join("local.cache"),
local_bundles_path: local_path.join("cached"),
temp_path: local_path.join("temp"),
remote_path: remote_path,
layout: layout,
crypto: crypto,
local_bundles: HashMap::new(),
remote_bundles: HashMap::new(),
@ -140,36 +116,35 @@ impl BundleDb {
}
fn load_bundle_list(&mut self) -> Result<(Vec<StoredBundle>, Vec<StoredBundle>), BundleDbError> {
let local_cache_path = &self.local_cache_path;
if let Ok(list) = StoredBundle::read_list_from(&local_cache_path) {
if let Ok(list) = StoredBundle::read_list_from(&self.layout.local_bundle_cache_path()) {
for bundle in list {
self.local_bundles.insert(bundle.id(), bundle);
}
}
let remote_cache_path = &self.remote_cache_path;
if let Ok(list) = StoredBundle::read_list_from(&remote_cache_path) {
if let Ok(list) = StoredBundle::read_list_from(&self.layout.remote_bundle_cache_path()) {
for bundle in list {
self.remote_bundles.insert(bundle.id(), bundle);
}
}
let (new, gone) = try!(load_bundles(&self.local_bundles_path, &self.repo_path, &mut self.local_bundles, self.crypto.clone()));
let base_path = self.layout.base_path();
let (new, gone) = try!(load_bundles(&self.layout.local_bundles_path(), base_path, &mut self.local_bundles, self.crypto.clone()));
if !new.is_empty() || !gone.is_empty() {
let bundles: Vec<_> = self.local_bundles.values().cloned().collect();
try!(StoredBundle::save_list_to(&bundles, &local_cache_path));
try!(StoredBundle::save_list_to(&bundles, &self.layout.local_bundle_cache_path()));
}
let (new, gone) = try!(load_bundles(&self.remote_path, &self.repo_path, &mut self.remote_bundles, self.crypto.clone()));
let (new, gone) = try!(load_bundles(&self.layout.remote_bundles_path(), base_path, &mut self.remote_bundles, self.crypto.clone()));
if !new.is_empty() || !gone.is_empty() {
let bundles: Vec<_> = self.remote_bundles.values().cloned().collect();
try!(StoredBundle::save_list_to(&bundles, &remote_cache_path));
try!(StoredBundle::save_list_to(&bundles, &self.layout.remote_bundle_cache_path()));
}
Ok((new, gone))
}
pub fn save_cache(&self) -> Result<(), BundleDbError> {
let bundles: Vec<_> = self.local_bundles.values().cloned().collect();
try!(StoredBundle::save_list_to(&bundles, &self.local_cache_path));
try!(StoredBundle::save_list_to(&bundles, &self.layout.local_bundle_cache_path()));
let bundles: Vec<_> = self.remote_bundles.values().cloned().collect();
Ok(try!(StoredBundle::save_list_to(&bundles, &self.remote_cache_path)))
Ok(try!(StoredBundle::save_list_to(&bundles, &self.layout.remote_bundle_cache_path())))
}
pub fn update_cache(&mut self, new: &[StoredBundle], gone: &[StoredBundle]) -> Result<(), BundleDbError> {
@ -179,23 +154,18 @@ impl BundleDb {
try!(self.copy_remote_bundle_to_cache(bundle));
}
}
let base_path = self.layout.base_path();
for bundle in gone {
if let Some(bundle) = self.local_bundles.remove(&bundle.id()) {
try!(fs::remove_file(self.repo_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id())))
try!(fs::remove_file(base_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id())))
}
}
Ok(())
}
pub fn temp_bundle_path(&self, id: &BundleId) -> PathBuf {
self.temp_path.join(id.to_string().to_owned() + ".bundle")
}
#[inline]
pub fn open<R: AsRef<Path>, L: AsRef<Path>>(repo_path: PathBuf, remote_path: R, local_path: L, crypto: Arc<Mutex<Crypto>>) -> Result<(Self, Vec<BundleInfo>, Vec<BundleInfo>), BundleDbError> {
let remote_path = remote_path.as_ref().to_owned();
let local_path = local_path.as_ref().to_owned();
let mut self_ = Self::new(repo_path, remote_path, local_path, crypto);
pub fn open(layout: RepositoryLayout, crypto: Arc<Mutex<Crypto>>) -> Result<(Self, Vec<BundleInfo>, Vec<BundleInfo>), BundleDbError> {
let mut self_ = Self::new(layout, crypto);
let (new, gone) = try!(self_.load_bundle_list());
try!(self_.update_cache(&new, &gone));
let new = new.into_iter().map(|s| s.info).collect();
@ -204,14 +174,11 @@ impl BundleDb {
}
#[inline]
pub fn create<R: AsRef<Path>, L: AsRef<Path>>(repo_path: PathBuf, remote_path: R, local_path: L, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleDbError> {
let remote_path = remote_path.as_ref().to_owned();
let local_path = local_path.as_ref().to_owned();
let self_ = Self::new(repo_path, remote_path, local_path, crypto);
try!(fs::create_dir_all(&self_.remote_path).context(&self_.remote_path as &Path));
try!(fs::create_dir_all(&self_.local_bundles_path).context(&self_.local_bundles_path as &Path));
try!(fs::create_dir_all(&self_.temp_path).context(&self_.temp_path as &Path));
Ok(self_)
pub fn create(layout: RepositoryLayout) -> Result<(), BundleDbError> {
try!(fs::create_dir_all(layout.remote_bundles_path()).context(&layout.remote_bundles_path() as &Path));
try!(fs::create_dir_all(layout.local_bundles_path()).context(&layout.local_bundles_path() as &Path));
try!(fs::create_dir_all(layout.temp_bundles_path()).context(&layout.temp_bundles_path() as &Path));
Ok(())
}
#[inline]
@ -228,7 +195,8 @@ impl BundleDb {
}
fn get_bundle(&self, stored: &StoredBundle) -> Result<BundleReader, BundleDbError> {
Ok(try!(BundleReader::load(self.repo_path.join(&stored.path), self.crypto.clone())))
let base_path = self.layout.base_path();
Ok(try!(BundleReader::load(base_path.join(&stored.path), self.crypto.clone())))
}
pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleDbError> {
@ -249,9 +217,9 @@ impl BundleDb {
fn copy_remote_bundle_to_cache(&mut self, bundle: &StoredBundle) -> Result<(), BundleDbError> {
let id = bundle.id();
let (folder, filename) = bundle_path(&id, self.local_bundles_path.clone(), self.local_bundles.len());
let (folder, filename) = self.layout.local_bundle_path(&id, self.local_bundles.len());
try!(fs::create_dir_all(&folder).context(&folder as &Path));
let bundle = try!(bundle.copy_to(&self.repo_path, folder.join(filename)));
let bundle = try!(bundle.copy_to(&self.layout.base_path(), folder.join(filename)));
self.local_bundles.insert(id, bundle);
Ok(())
}
@ -259,13 +227,12 @@ impl BundleDb {
#[inline]
pub fn add_bundle(&mut self, bundle: BundleWriter) -> Result<BundleInfo, BundleDbError> {
let bundle = try!(bundle.finish(&self));
let random_id = BundleId::random();
if bundle.info.mode == BundleMode::Meta {
try!(self.copy_remote_bundle_to_cache(&bundle))
}
let (folder, filename) = bundle_path(&random_id, self.remote_path.clone(), self.remote_bundles.len());
let (folder, filename) = self.layout.remote_bundle_path(self.remote_bundles.len());
try!(fs::create_dir_all(&folder).context(&folder as &Path));
let bundle = try!(bundle.move_to(&self.repo_path, folder.join(filename)));
let bundle = try!(bundle.move_to(&self.layout.base_path(), folder.join(filename)));
self.remote_bundles.insert(bundle.id(), bundle.clone());
Ok(bundle.info)
}
@ -289,7 +256,8 @@ impl BundleDb {
#[inline]
pub fn delete_local_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> {
if let Some(bundle) = self.local_bundles.remove(bundle) {
try!(fs::remove_file(self.repo_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id())))
let path = self.layout.base_path().join(&bundle.path);
try!(fs::remove_file(path).map_err(|e| BundleDbError::Remove(e, bundle.id())))
}
Ok(())
}
@ -298,7 +266,8 @@ impl BundleDb {
pub fn delete_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> {
try!(self.delete_local_bundle(bundle));
if let Some(bundle) = self.remote_bundles.remove(bundle) {
fs::remove_file(self.repo_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id()))
let path = self.layout.base_path().join(&bundle.path);
fs::remove_file(path).map_err(|e| BundleDbError::Remove(e, bundle.id()))
} else {
Err(BundleDbError::NoSuchBundle(bundle.clone()))
}

View File

@ -101,7 +101,7 @@ impl BundleWriter {
if let Some(ref encryption) = self.encryption {
chunk_data = try!(self.crypto.lock().unwrap().encrypt(&encryption, &chunk_data));
}
let path = db.temp_bundle_path(&id);
let path = db.layout.temp_bundle_path();
let mut file = BufWriter::new(try!(File::create(&path).context(&path as &Path)));
try!(file.write_all(&HEADER_STRING).context(&path as &Path));
try!(file.write_all(&[HEADER_VERSION]).context(&path as &Path));

View File

@ -306,7 +306,7 @@ pub fn run() -> Result<(), ErrorCode> {
}
let reference_backup = reference_backup.map(|(_, backup)| backup);
if !no_default_excludes && !tar {
for line in BufReader::new(checked!(File::open(&repo.excludes_path), "open default excludes file", ErrorCode::LoadExcludes)).lines() {
for line in BufReader::new(checked!(File::open(&repo.layout.excludes_path()), "open default excludes file", ErrorCode::LoadExcludes)).lines() {
excludes.push(checked!(line, "read default excludes file", ErrorCode::LoadExcludes));
}
}

View File

@ -152,13 +152,13 @@ impl Index {
}
#[inline]
pub fn open(path: &Path) -> Result<Index, IndexError> {
Index::new(path, false)
pub fn open<P: AsRef<Path>>(path: P) -> Result<Index, IndexError> {
Index::new(path.as_ref(), false)
}
#[inline]
pub fn create(path: &Path) -> Result<Index, IndexError> {
Index::new(path, true)
pub fn create<P: AsRef<Path>>(path: P) -> Result<Index, IndexError> {
Index::new(path.as_ref(), true)
}
#[inline]

View File

@ -1,7 +1,7 @@
pub use ::util::*;
pub use ::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError, BundleDb, BundleWriterError, StoredBundle};
pub use ::chunker::{ChunkerType, Chunker, ChunkerStatus, IChunker, ChunkerError};
pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, RepositoryIntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError};
pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, RepositoryIntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError, RepositoryLayout};
pub use ::index::{Index, Location, IndexError};
pub use ::mount::FuseFilesystem;

View File

@ -38,25 +38,25 @@ pub enum DiffType {
impl Repository {
pub fn get_backups(&self) -> Result<HashMap<String, Backup>, RepositoryError> {
Ok(try!(Backup::get_all_from(&self.crypto.lock().unwrap(), &self.backups_path)))
Ok(try!(Backup::get_all_from(&self.crypto.lock().unwrap(), self.layout.backups_path())))
}
pub fn get_backup(&self, name: &str) -> Result<Backup, RepositoryError> {
Ok(try!(Backup::read_from(&self.crypto.lock().unwrap(), self.backups_path.join(name))))
Ok(try!(Backup::read_from(&self.crypto.lock().unwrap(), self.layout.backup_path(name))))
}
pub fn save_backup(&mut self, backup: &Backup, name: &str) -> Result<(), RepositoryError> {
let path = &self.backups_path.join(name);
let path = self.layout.backup_path(name);
try!(fs::create_dir_all(path.parent().unwrap()));
Ok(try!(backup.save_to(&self.crypto.lock().unwrap(), self.config.encryption.clone(), path)))
}
pub fn delete_backup(&self, name: &str) -> Result<(), RepositoryError> {
let mut path = self.backups_path.join(name);
let mut path = self.layout.backup_path(name);
try!(fs::remove_file(&path));
loop {
path = path.parent().unwrap().to_owned();
if path == self.backups_path || fs::remove_dir(&path).is_err() {
if path == self.layout.backups_path() || fs::remove_dir(&path).is_err() {
break
}
}

101
src/repository/layout.rs Normal file
View File

@ -0,0 +1,101 @@
use ::prelude::*;
use std::path::{Path, PathBuf};
#[derive(Clone)]
pub struct RepositoryLayout(PathBuf);
impl RepositoryLayout {
pub fn new<P: AsRef<Path>>(path: P) -> Self {
RepositoryLayout(path.as_ref().to_path_buf())
}
pub fn base_path(&self) -> &Path {
&self.0
}
pub fn config_path(&self) -> PathBuf {
self.0.join("config.yaml")
}
pub fn excludes_path(&self) -> PathBuf {
self.0.join("excludes")
}
pub fn index_path(&self) -> PathBuf {
self.0.join("index")
}
pub fn keys_path(&self) -> PathBuf {
self.0.join("keys")
}
pub fn bundle_map_path(&self) -> PathBuf {
self.0.join("bundles.map")
}
pub fn backups_path(&self) -> PathBuf {
self.0.join("remote/backups")
}
pub fn backup_path(&self, name: &str) -> PathBuf {
self.backups_path().join(name)
}
pub fn remote_path(&self) -> PathBuf {
self.0.join("remote")
}
pub fn remote_readme_path(&self) -> PathBuf {
self.0.join("remote/README.md")
}
pub fn remote_locks_path(&self) -> PathBuf {
self.0.join("remote/locks")
}
pub fn remote_bundles_path(&self) -> PathBuf {
self.0.join("remote/bundles")
}
pub fn local_bundles_path(&self) -> PathBuf {
self.0.join("bundles/cached")
}
fn bundle_path(&self, bundle: &BundleId, mut folder: PathBuf, mut count: usize) -> (PathBuf, PathBuf) {
let mut file = bundle.to_string().to_owned() + ".bundle";
while count >= 100 {
if file.len() < 10 {
break
}
folder = folder.join(&file[0..2]);
file = file[2..].to_string();
count /= 250;
}
(folder, file.into())
}
pub fn remote_bundle_path(&self, count: usize) -> (PathBuf, PathBuf) {
self.bundle_path(&BundleId::random(), self.remote_bundles_path(), count)
}
pub fn local_bundle_path(&self, bundle: &BundleId, count: usize) -> (PathBuf, PathBuf) {
self.bundle_path(bundle, self.local_bundles_path(), count)
}
pub fn temp_bundles_path(&self) -> PathBuf {
self.0.join("bundles/temp")
}
pub fn temp_bundle_path(&self) -> PathBuf {
self.temp_bundles_path().join(BundleId::random().to_string().to_owned() + ".bundle")
}
pub fn local_bundle_cache_path(&self) -> PathBuf {
self.0.join("bundles/local.cache")
}
pub fn remote_bundle_cache_path(&self) -> PathBuf {
self.0.join("bundles/remote.cache")
}
}

View File

@ -9,12 +9,13 @@ mod error;
mod vacuum;
mod backup_file;
mod tarfile;
mod layout;
use ::prelude::*;
use std::mem;
use std::cmp::max;
use std::path::{PathBuf, Path};
use std::path::Path;
use std::fs::{self, File};
use std::sync::{Arc, Mutex};
use std::os::unix::fs::symlink;
@ -27,6 +28,7 @@ pub use self::backup::{BackupError, BackupOptions, DiffType};
pub use self::backup_file::{Backup, BackupFileError};
pub use self::integrity::RepositoryIntegrityError;
pub use self::info::{RepositoryInfo, BundleAnalysis};
pub use self::layout::RepositoryLayout;
use self::bundle_map::BundleMap;
@ -35,9 +37,7 @@ const DEFAULT_EXCLUDES: &'static [u8] = include_bytes!("../../docs/excludes.defa
pub struct Repository {
path: PathBuf,
backups_path: PathBuf,
pub excludes_path: PathBuf,
pub layout: RepositoryLayout,
pub config: Config,
index: Index,
crypto: Arc<Mutex<Crypto>>,
@ -55,62 +55,32 @@ pub struct Repository {
impl Repository {
pub fn create<P: AsRef<Path>, R: AsRef<Path>>(path: P, config: Config, remote: R) -> Result<Self, RepositoryError> {
let path = path.as_ref().to_owned();
try!(fs::create_dir(&path));
let mut excludes = try!(File::create(path.join("excludes")));
try!(excludes.write_all(DEFAULT_EXCLUDES));
try!(fs::create_dir(path.join("keys")));
let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys")))));
try!(symlink(remote, path.join("remote")));
let mut remote_readme = try!(File::create(path.join("remote/README.md")));
try!(remote_readme.write_all(REPOSITORY_README));
try!(fs::create_dir_all(path.join("remote/locks")));
let locks = LockFolder::new(path.join("remote/locks"));
let bundles = try!(BundleDb::create(
path.to_path_buf(),
path.join("remote/bundles"),
path.join("bundles"),
crypto.clone()
));
let index = try!(Index::create(&path.join("index")));
try!(config.save(path.join("config.yaml")));
let bundle_map = BundleMap::create();
try!(bundle_map.save(path.join("bundles.map")));
try!(fs::create_dir_all(&path.join("remote/backups")));
Ok(Repository {
backups_path: path.join("remote/backups"),
excludes_path: path.join("excludes"),
path: path,
chunker: config.chunker.create(),
config: config,
index: index,
bundle_map: bundle_map,
next_data_bundle: 1,
next_meta_bundle: 0,
bundles: bundles,
data_bundle: None,
meta_bundle: None,
crypto: crypto,
locks: locks
})
let layout = RepositoryLayout::new(path.clone());
try!(fs::create_dir(layout.base_path()));
try!(File::create(layout.excludes_path()).and_then(|mut f| f.write_all(DEFAULT_EXCLUDES)));
try!(fs::create_dir(layout.keys_path()));
try!(symlink(remote, layout.remote_path()));
try!(File::create(layout.remote_readme_path()).and_then(|mut f| f.write_all(REPOSITORY_README)));
try!(fs::create_dir_all(layout.remote_locks_path()));
try!(config.save(layout.config_path()));
try!(BundleDb::create(layout.clone()));
try!(Index::create(layout.index_path()));
try!(BundleMap::create().save(layout.bundle_map_path()));
try!(fs::create_dir_all(layout.backups_path()));
Self::open(path)
}
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, RepositoryError> {
let path = path.as_ref().to_owned();
let config = try!(Config::load(path.join("config.yaml")));
let locks = LockFolder::new(path.join("remote/locks"));
let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys")))));
let (bundles, new, gone) = try!(BundleDb::open(
path.to_path_buf(),
path.join("remote/bundles"),
path.join("bundles"),
crypto.clone()
));
let index = try!(Index::open(&path.join("index")));
let bundle_map = try!(BundleMap::load(path.join("bundles.map")));
let layout = RepositoryLayout::new(path.clone());
let config = try!(Config::load(layout.config_path()));
let locks = LockFolder::new(layout.remote_locks_path());
let crypto = Arc::new(Mutex::new(try!(Crypto::open(layout.keys_path()))));
let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone()));
let index = try!(Index::open(layout.index_path()));
let bundle_map = try!(BundleMap::load(layout.bundle_map_path()));
let mut repo = Repository {
backups_path: path.join("remote/backups"),
excludes_path: path.join("excludes"),
path: path,
layout: layout,
chunker: config.chunker.create(),
config: config,
index: index,
@ -160,7 +130,7 @@ impl Repository {
}
pub fn save_config(&mut self) -> Result<(), RepositoryError> {
try!(self.config.save(self.path.join("config.yaml")));
try!(self.config.save(self.layout.config_path()));
Ok(())
}
@ -180,7 +150,7 @@ impl Repository {
#[inline]
fn save_bundle_map(&self) -> Result<(), RepositoryError> {
try!(self.bundle_map.save(self.path.join("bundles.map")));
try!(self.bundle_map.save(self.layout.bundle_map_path()));
Ok(())
}

View File

@ -64,7 +64,7 @@ impl Repository {
for id in rewrite_bundles {
try!(self.delete_bundle(id));
}
try!(self.bundle_map.save(self.path.join("bundles.map")));
try!(self.save_bundle_map());
Ok(())
}
}