dswd
/
zvault
mirror of https://github.com/dswd/zvault
1
0
Fork 0
Code Issues Releases Wiki Activity

Central repository layout class

This commit is contained in:
Dennis Schwerdel 2017-04-08 14:35:10 +02:00 committed by Dennis Schwerdel
parent c1e4cb2bdf
commit a29e95e4f4
10 changed files with 174 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
TODO.md
View File

@ -22,7 +22,6 @@
* Benchmarks
* Full fuse method coverage
* Clippy
* Central repository layout class
## Other
* Homepage

97
src/bundledb/db.rs
View File

@ -53,22 +53,8 @@ quick_error!{
}
pub fn bundle_path(bundle: &BundleId, mut folder: PathBuf, mut count: usize) -> (PathBuf, PathBuf) {
let mut file = bundle.to_string().to_owned() + ".bundle";
while count >= 100 {
if file.len() < 10 {
break
}
folder = folder.join(&file[0..2]);
file = file[2..].to_string();
count /= 250;
}
(folder, file.into())
}
pub fn load_bundles<P: AsRef<Path>>(path: P, base: P, bundles: &mut HashMap<BundleId, StoredBundle>, crypto: Arc<Mutex<Crypto>>) -> Result<(Vec<StoredBundle>, Vec<StoredBundle>), BundleDbError> {
let base = base.as_ref();
let mut paths = vec![path.as_ref().to_path_buf()];
pub fn load_bundles(path: &Path, base: &Path, bundles: &mut HashMap<BundleId, StoredBundle>, crypto: Arc<Mutex<Crypto>>) -> Result<(Vec<StoredBundle>, Vec<StoredBundle>), BundleDbError> {
let mut paths = vec![path.to_path_buf()];
let mut bundle_paths = HashSet::new();
while let Some(path) = paths.pop() {
for entry in try!(fs::read_dir(path).map_err(BundleDbError::ListBundles)) {
@ -110,12 +96,7 @@ pub fn load_bundles<P: AsRef<Path>>(path: P, base: P, bundles: &mut HashMap<Bund
pub struct BundleDb {
repo_path: PathBuf,
remote_path: PathBuf,
local_bundles_path: PathBuf,
temp_path: PathBuf,
remote_cache_path: PathBuf,
local_cache_path: PathBuf,
pub layout: RepositoryLayout,
crypto: Arc<Mutex<Crypto>>,
local_bundles: HashMap<BundleId, StoredBundle>,
remote_bundles: HashMap<BundleId, StoredBundle>,
@ -124,14 +105,9 @@ pub struct BundleDb {
impl BundleDb {
fn new(repo_path: PathBuf, remote_path: PathBuf, local_path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Self {
fn new(layout: RepositoryLayout, crypto: Arc<Mutex<Crypto>>) -> Self {
BundleDb {
repo_path: repo_path,
remote_cache_path: local_path.join("remote.cache"),
local_cache_path: local_path.join("local.cache"),
local_bundles_path: local_path.join("cached"),
temp_path: local_path.join("temp"),
remote_path: remote_path,
layout: layout,
crypto: crypto,
local_bundles: HashMap::new(),
remote_bundles: HashMap::new(),
@ -140,36 +116,35 @@ impl BundleDb {
}
fn load_bundle_list(&mut self) -> Result<(Vec<StoredBundle>, Vec<StoredBundle>), BundleDbError> {
let local_cache_path = &self.local_cache_path;
if let Ok(list) = StoredBundle::read_list_from(&local_cache_path) {
if let Ok(list) = StoredBundle::read_list_from(&self.layout.local_bundle_cache_path()) {
for bundle in list {
self.local_bundles.insert(bundle.id(), bundle);
}
}
let remote_cache_path = &self.remote_cache_path;
if let Ok(list) = StoredBundle::read_list_from(&remote_cache_path) {
if let Ok(list) = StoredBundle::read_list_from(&self.layout.remote_bundle_cache_path()) {
for bundle in list {
self.remote_bundles.insert(bundle.id(), bundle);
}
}
let (new, gone) = try!(load_bundles(&self.local_bundles_path, &self.repo_path, &mut self.local_bundles, self.crypto.clone()));
let base_path = self.layout.base_path();
let (new, gone) = try!(load_bundles(&self.layout.local_bundles_path(), base_path, &mut self.local_bundles, self.crypto.clone()));
if !new.is_empty() || !gone.is_empty() {
let bundles: Vec<_> = self.local_bundles.values().cloned().collect();
try!(StoredBundle::save_list_to(&bundles, &local_cache_path));
try!(StoredBundle::save_list_to(&bundles, &self.layout.local_bundle_cache_path()));
}
let (new, gone) = try!(load_bundles(&self.remote_path, &self.repo_path, &mut self.remote_bundles, self.crypto.clone()));
let (new, gone) = try!(load_bundles(&self.layout.remote_bundles_path(), base_path, &mut self.remote_bundles, self.crypto.clone()));
if !new.is_empty() || !gone.is_empty() {
let bundles: Vec<_> = self.remote_bundles.values().cloned().collect();
try!(StoredBundle::save_list_to(&bundles, &remote_cache_path));
try!(StoredBundle::save_list_to(&bundles, &self.layout.remote_bundle_cache_path()));
}
Ok((new, gone))
}
pub fn save_cache(&self) -> Result<(), BundleDbError> {
let bundles: Vec<_> = self.local_bundles.values().cloned().collect();
try!(StoredBundle::save_list_to(&bundles, &self.local_cache_path));
try!(StoredBundle::save_list_to(&bundles, &self.layout.local_bundle_cache_path()));
let bundles: Vec<_> = self.remote_bundles.values().cloned().collect();
Ok(try!(StoredBundle::save_list_to(&bundles, &self.remote_cache_path)))
Ok(try!(StoredBundle::save_list_to(&bundles, &self.layout.remote_bundle_cache_path())))
}
pub fn update_cache(&mut self, new: &[StoredBundle], gone: &[StoredBundle]) -> Result<(), BundleDbError> {
@ -179,23 +154,18 @@ impl BundleDb {
try!(self.copy_remote_bundle_to_cache(bundle));
}
}
let base_path = self.layout.base_path();
for bundle in gone {
if let Some(bundle) = self.local_bundles.remove(&bundle.id()) {
try!(fs::remove_file(self.repo_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id())))
try!(fs::remove_file(base_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id())))
}
}
Ok(())
}
pub fn temp_bundle_path(&self, id: &BundleId) -> PathBuf {
self.temp_path.join(id.to_string().to_owned() + ".bundle")
}
#[inline]
pub fn open<R: AsRef<Path>, L: AsRef<Path>>(repo_path: PathBuf, remote_path: R, local_path: L, crypto: Arc<Mutex<Crypto>>) -> Result<(Self, Vec<BundleInfo>, Vec<BundleInfo>), BundleDbError> {
let remote_path = remote_path.as_ref().to_owned();
let local_path = local_path.as_ref().to_owned();
let mut self_ = Self::new(repo_path, remote_path, local_path, crypto);
pub fn open(layout: RepositoryLayout, crypto: Arc<Mutex<Crypto>>) -> Result<(Self, Vec<BundleInfo>, Vec<BundleInfo>), BundleDbError> {
let mut self_ = Self::new(layout, crypto);
let (new, gone) = try!(self_.load_bundle_list());
try!(self_.update_cache(&new, &gone));
let new = new.into_iter().map(|s| s.info).collect();
@ -204,14 +174,11 @@ impl BundleDb {
}
#[inline]
pub fn create<R: AsRef<Path>, L: AsRef<Path>>(repo_path: PathBuf, remote_path: R, local_path: L, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleDbError> {
let remote_path = remote_path.as_ref().to_owned();
let local_path = local_path.as_ref().to_owned();
let self_ = Self::new(repo_path, remote_path, local_path, crypto);
try!(fs::create_dir_all(&self_.remote_path).context(&self_.remote_path as &Path));
try!(fs::create_dir_all(&self_.local_bundles_path).context(&self_.local_bundles_path as &Path));
try!(fs::create_dir_all(&self_.temp_path).context(&self_.temp_path as &Path));
Ok(self_)
pub fn create(layout: RepositoryLayout) -> Result<(), BundleDbError> {
try!(fs::create_dir_all(layout.remote_bundles_path()).context(&layout.remote_bundles_path() as &Path));
try!(fs::create_dir_all(layout.local_bundles_path()).context(&layout.local_bundles_path() as &Path));
try!(fs::create_dir_all(layout.temp_bundles_path()).context(&layout.temp_bundles_path() as &Path));
Ok(())
}
#[inline]
@ -228,7 +195,8 @@ impl BundleDb {
}
fn get_bundle(&self, stored: &StoredBundle) -> Result<BundleReader, BundleDbError> {
Ok(try!(BundleReader::load(self.repo_path.join(&stored.path), self.crypto.clone())))
let base_path = self.layout.base_path();
Ok(try!(BundleReader::load(base_path.join(&stored.path), self.crypto.clone())))
}
pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleDbError> {
@ -249,9 +217,9 @@ impl BundleDb {
fn copy_remote_bundle_to_cache(&mut self, bundle: &StoredBundle) -> Result<(), BundleDbError> {
let id = bundle.id();
let (folder, filename) = bundle_path(&id, self.local_bundles_path.clone(), self.local_bundles.len());
let (folder, filename) = self.layout.local_bundle_path(&id, self.local_bundles.len());
try!(fs::create_dir_all(&folder).context(&folder as &Path));
let bundle = try!(bundle.copy_to(&self.repo_path, folder.join(filename)));
let bundle = try!(bundle.copy_to(&self.layout.base_path(), folder.join(filename)));
self.local_bundles.insert(id, bundle);
Ok(())
}
@ -259,13 +227,12 @@ impl BundleDb {
#[inline]
pub fn add_bundle(&mut self, bundle: BundleWriter) -> Result<BundleInfo, BundleDbError> {
let bundle = try!(bundle.finish(&self));
let random_id = BundleId::random();
if bundle.info.mode == BundleMode::Meta {
try!(self.copy_remote_bundle_to_cache(&bundle))
}
let (folder, filename) = bundle_path(&random_id, self.remote_path.clone(), self.remote_bundles.len());
let (folder, filename) = self.layout.remote_bundle_path(self.remote_bundles.len());
try!(fs::create_dir_all(&folder).context(&folder as &Path));
let bundle = try!(bundle.move_to(&self.repo_path, folder.join(filename)));
let bundle = try!(bundle.move_to(&self.layout.base_path(), folder.join(filename)));
self.remote_bundles.insert(bundle.id(), bundle.clone());
Ok(bundle.info)
}
@ -289,7 +256,8 @@ impl BundleDb {
#[inline]
pub fn delete_local_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> {
if let Some(bundle) = self.local_bundles.remove(bundle) {
try!(fs::remove_file(self.repo_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id())))
let path = self.layout.base_path().join(&bundle.path);
try!(fs::remove_file(path).map_err(|e| BundleDbError::Remove(e, bundle.id())))
}
Ok(())
}
@ -298,7 +266,8 @@ impl BundleDb {
pub fn delete_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> {
try!(self.delete_local_bundle(bundle));
if let Some(bundle) = self.remote_bundles.remove(bundle) {
fs::remove_file(self.repo_path.join(&bundle.path)).map_err(|e| BundleDbError::Remove(e, bundle.id()))
let path = self.layout.base_path().join(&bundle.path);
fs::remove_file(path).map_err(|e| BundleDbError::Remove(e, bundle.id()))
} else {
Err(BundleDbError::NoSuchBundle(bundle.clone()))
}

2
src/bundledb/writer.rs
View File

@ -101,7 +101,7 @@ impl BundleWriter {
if let Some(ref encryption) = self.encryption {
chunk_data = try!(self.crypto.lock().unwrap().encrypt(&encryption, &chunk_data));
}
let path = db.temp_bundle_path(&id);
let path = db.layout.temp_bundle_path();
let mut file = BufWriter::new(try!(File::create(&path).context(&path as &Path)));
try!(file.write_all(&HEADER_STRING).context(&path as &Path));
try!(file.write_all(&[HEADER_VERSION]).context(&path as &Path));

2
src/cli/mod.rs
View File

@ -306,7 +306,7 @@ pub fn run() -> Result<(), ErrorCode> {
}
let reference_backup = reference_backup.map(|(_, backup)| backup);
if !no_default_excludes && !tar {
for line in BufReader::new(checked!(File::open(&repo.excludes_path), "open default excludes file", ErrorCode::LoadExcludes)).lines() {
for line in BufReader::new(checked!(File::open(&repo.layout.excludes_path()), "open default excludes file", ErrorCode::LoadExcludes)).lines() {
excludes.push(checked!(line, "read default excludes file", ErrorCode::LoadExcludes));
}
}

8
src/index.rs
View File

@ -152,13 +152,13 @@ impl Index {
}
#[inline]
pub fn open(path: &Path) -> Result<Index, IndexError> {
Index::new(path, false)
pub fn open<P: AsRef<Path>>(path: P) -> Result<Index, IndexError> {
Index::new(path.as_ref(), false)
}
#[inline]
pub fn create(path: &Path) -> Result<Index, IndexError> {
Index::new(path, true)
pub fn create<P: AsRef<Path>>(path: P) -> Result<Index, IndexError> {
Index::new(path.as_ref(), true)
}
#[inline]

2
src/prelude.rs
View File

@ -1,7 +1,7 @@
pub use ::util::*;
pub use ::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError, BundleDb, BundleWriterError, StoredBundle};
pub use ::chunker::{ChunkerType, Chunker, ChunkerStatus, IChunker, ChunkerError};
pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, RepositoryIntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError};
pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, RepositoryIntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError, RepositoryLayout};
pub use ::index::{Index, Location, IndexError};
pub use ::mount::FuseFilesystem;

10
src/repository/backup.rs
View File

@ -38,25 +38,25 @@ pub enum DiffType {
impl Repository {
pub fn get_backups(&self) -> Result<HashMap<String, Backup>, RepositoryError> {
Ok(try!(Backup::get_all_from(&self.crypto.lock().unwrap(), &self.backups_path)))
Ok(try!(Backup::get_all_from(&self.crypto.lock().unwrap(), self.layout.backups_path())))
}
pub fn get_backup(&self, name: &str) -> Result<Backup, RepositoryError> {
Ok(try!(Backup::read_from(&self.crypto.lock().unwrap(), self.backups_path.join(name))))
Ok(try!(Backup::read_from(&self.crypto.lock().unwrap(), self.layout.backup_path(name))))
}
pub fn save_backup(&mut self, backup: &Backup, name: &str) -> Result<(), RepositoryError> {
let path = &self.backups_path.join(name);
let path = self.layout.backup_path(name);
try!(fs::create_dir_all(path.parent().unwrap()));
Ok(try!(backup.save_to(&self.crypto.lock().unwrap(), self.config.encryption.clone(), path)))
}
pub fn delete_backup(&self, name: &str) -> Result<(), RepositoryError> {
let mut path = self.backups_path.join(name);
let mut path = self.layout.backup_path(name);
try!(fs::remove_file(&path));
loop {
path = path.parent().unwrap().to_owned();
if path == self.backups_path || fs::remove_dir(&path).is_err() {
if path == self.layout.backups_path() || fs::remove_dir(&path).is_err() {
break
}
}

101
src/repository/layout.rs Normal file
View File

@ -0,0 +1,101 @@
use ::prelude::*;
use std::path::{Path, PathBuf};
#[derive(Clone)]
pub struct RepositoryLayout(PathBuf);
impl RepositoryLayout {
pub fn new<P: AsRef<Path>>(path: P) -> Self {
RepositoryLayout(path.as_ref().to_path_buf())
}
pub fn base_path(&self) -> &Path {
&self.0
}
pub fn config_path(&self) -> PathBuf {
self.0.join("config.yaml")
}
pub fn excludes_path(&self) -> PathBuf {
self.0.join("excludes")
}
pub fn index_path(&self) -> PathBuf {
self.0.join("index")
}
pub fn keys_path(&self) -> PathBuf {
self.0.join("keys")
}
pub fn bundle_map_path(&self) -> PathBuf {
self.0.join("bundles.map")
}
pub fn backups_path(&self) -> PathBuf {
self.0.join("remote/backups")
}
pub fn backup_path(&self, name: &str) -> PathBuf {
self.backups_path().join(name)
}
pub fn remote_path(&self) -> PathBuf {
self.0.join("remote")
}
pub fn remote_readme_path(&self) -> PathBuf {
self.0.join("remote/README.md")
}
pub fn remote_locks_path(&self) -> PathBuf {
self.0.join("remote/locks")
}
pub fn remote_bundles_path(&self) -> PathBuf {
self.0.join("remote/bundles")
}
pub fn local_bundles_path(&self) -> PathBuf {
self.0.join("bundles/cached")
}
fn bundle_path(&self, bundle: &BundleId, mut folder: PathBuf, mut count: usize) -> (PathBuf, PathBuf) {
let mut file = bundle.to_string().to_owned() + ".bundle";
while count >= 100 {
if file.len() < 10 {
break
}
folder = folder.join(&file[0..2]);
file = file[2..].to_string();
count /= 250;
}
(folder, file.into())
}
pub fn remote_bundle_path(&self, count: usize) -> (PathBuf, PathBuf) {
self.bundle_path(&BundleId::random(), self.remote_bundles_path(), count)
}
pub fn local_bundle_path(&self, bundle: &BundleId, count: usize) -> (PathBuf, PathBuf) {
self.bundle_path(bundle, self.local_bundles_path(), count)
}
pub fn temp_bundles_path(&self) -> PathBuf {
self.0.join("bundles/temp")
}
pub fn temp_bundle_path(&self) -> PathBuf {
self.temp_bundles_path().join(BundleId::random().to_string().to_owned() + ".bundle")
}
pub fn local_bundle_cache_path(&self) -> PathBuf {
self.0.join("bundles/local.cache")
}
pub fn remote_bundle_cache_path(&self) -> PathBuf {
self.0.join("bundles/remote.cache")
}
}

84
src/repository/mod.rs
View File

@ -9,12 +9,13 @@ mod error;
mod vacuum;
mod backup_file;
mod tarfile;
mod layout;
use ::prelude::*;
use std::mem;
use std::cmp::max;
use std::path::{PathBuf, Path};
use std::path::Path;
use std::fs::{self, File};
use std::sync::{Arc, Mutex};
use std::os::unix::fs::symlink;
@ -27,6 +28,7 @@ pub use self::backup::{BackupError, BackupOptions, DiffType};
pub use self::backup_file::{Backup, BackupFileError};
pub use self::integrity::RepositoryIntegrityError;
pub use self::info::{RepositoryInfo, BundleAnalysis};
pub use self::layout::RepositoryLayout;
use self::bundle_map::BundleMap;
@ -35,9 +37,7 @@ const DEFAULT_EXCLUDES: &'static [u8] = include_bytes!("../../docs/excludes.defa
pub struct Repository {
path: PathBuf,
backups_path: PathBuf,
pub excludes_path: PathBuf,
pub layout: RepositoryLayout,
pub config: Config,
index: Index,
crypto: Arc<Mutex<Crypto>>,
@ -55,62 +55,32 @@ pub struct Repository {
impl Repository {
pub fn create<P: AsRef<Path>, R: AsRef<Path>>(path: P, config: Config, remote: R) -> Result<Self, RepositoryError> {
let path = path.as_ref().to_owned();
try!(fs::create_dir(&path));
let mut excludes = try!(File::create(path.join("excludes")));
try!(excludes.write_all(DEFAULT_EXCLUDES));
try!(fs::create_dir(path.join("keys")));
let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys")))));
try!(symlink(remote, path.join("remote")));
let mut remote_readme = try!(File::create(path.join("remote/README.md")));
try!(remote_readme.write_all(REPOSITORY_README));
try!(fs::create_dir_all(path.join("remote/locks")));
let locks = LockFolder::new(path.join("remote/locks"));
let bundles = try!(BundleDb::create(
path.to_path_buf(),
path.join("remote/bundles"),
path.join("bundles"),
crypto.clone()
));
let index = try!(Index::create(&path.join("index")));
try!(config.save(path.join("config.yaml")));
let bundle_map = BundleMap::create();
try!(bundle_map.save(path.join("bundles.map")));
try!(fs::create_dir_all(&path.join("remote/backups")));
Ok(Repository {
backups_path: path.join("remote/backups"),
excludes_path: path.join("excludes"),
path: path,
chunker: config.chunker.create(),
config: config,
index: index,
bundle_map: bundle_map,
next_data_bundle: 1,
next_meta_bundle: 0,
bundles: bundles,
data_bundle: None,
meta_bundle: None,
crypto: crypto,
locks: locks
})
let layout = RepositoryLayout::new(path.clone());
try!(fs::create_dir(layout.base_path()));
try!(File::create(layout.excludes_path()).and_then(|mut f| f.write_all(DEFAULT_EXCLUDES)));
try!(fs::create_dir(layout.keys_path()));
try!(symlink(remote, layout.remote_path()));
try!(File::create(layout.remote_readme_path()).and_then(|mut f| f.write_all(REPOSITORY_README)));
try!(fs::create_dir_all(layout.remote_locks_path()));
try!(config.save(layout.config_path()));
try!(BundleDb::create(layout.clone()));
try!(Index::create(layout.index_path()));
try!(BundleMap::create().save(layout.bundle_map_path()));
try!(fs::create_dir_all(layout.backups_path()));
Self::open(path)
}
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self, RepositoryError> {
let path = path.as_ref().to_owned();
let config = try!(Config::load(path.join("config.yaml")));
let locks = LockFolder::new(path.join("remote/locks"));
let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys")))));
let (bundles, new, gone) = try!(BundleDb::open(
path.to_path_buf(),
path.join("remote/bundles"),
path.join("bundles"),
crypto.clone()
));
let index = try!(Index::open(&path.join("index")));
let bundle_map = try!(BundleMap::load(path.join("bundles.map")));
let layout = RepositoryLayout::new(path.clone());
let config = try!(Config::load(layout.config_path()));
let locks = LockFolder::new(layout.remote_locks_path());
let crypto = Arc::new(Mutex::new(try!(Crypto::open(layout.keys_path()))));
let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone()));
let index = try!(Index::open(layout.index_path()));
let bundle_map = try!(BundleMap::load(layout.bundle_map_path()));
let mut repo = Repository {
backups_path: path.join("remote/backups"),
excludes_path: path.join("excludes"),
path: path,
layout: layout,
chunker: config.chunker.create(),
config: config,
index: index,
@ -160,7 +130,7 @@ impl Repository {
}
pub fn save_config(&mut self) -> Result<(), RepositoryError> {
try!(self.config.save(self.path.join("config.yaml")));
try!(self.config.save(self.layout.config_path()));
Ok(())
}
@ -180,7 +150,7 @@ impl Repository {
#[inline]
fn save_bundle_map(&self) -> Result<(), RepositoryError> {
try!(self.bundle_map.save(self.path.join("bundles.map")));
try!(self.bundle_map.save(self.layout.bundle_map_path()));
Ok(())
}

2
src/repository/vacuum.rs
View File

@ -64,7 +64,7 @@ impl Repository {
for id in rewrite_bundles {
try!(self.delete_bundle(id));
}
try!(self.bundle_map.save(self.path.join("bundles.map")));
try!(self.save_bundle_map());
Ok(())
}
}