From c354abac91d3c09af6c6706b0380c828c45d1945 Mon Sep 17 00:00:00 2001 From: Dennis Schwerdel Date: Tue, 21 Mar 2017 13:44:30 +0100 Subject: [PATCH] Some changes for remote path --- src/bundledb/bundle.rs | 66 +++++++++++++++++++++++++-------------- src/bundledb/db.rs | 35 ++++++++++++--------- src/bundledb/writer.rs | 2 +- src/directory_struture.md | 22 +++++++++++++ src/main.rs | 1 + src/repository/mod.rs | 2 ++ 6 files changed, 88 insertions(+), 40 deletions(-) create mode 100644 src/directory_struture.md diff --git a/src/bundledb/bundle.rs b/src/bundledb/bundle.rs index 663cfa1..bf346ad 100644 --- a/src/bundledb/bundle.rs +++ b/src/bundledb/bundle.rs @@ -102,30 +102,24 @@ impl Default for BundleInfo { pub struct Bundle { pub info: BundleInfo, - pub chunks: ChunkList, pub version: u8, pub path: PathBuf, crypto: Arc>, pub content_start: usize, - pub chunk_positions: Vec + pub chunks: Option, + pub chunk_positions: Option> } impl Bundle { - pub fn new(path: PathBuf, version: u8, content_start: usize, crypto: Arc>, info: BundleInfo, chunks: ChunkList) -> Self { - let mut chunk_positions = Vec::with_capacity(chunks.len()); - let mut pos = 0; - for &(_, len) in (&chunks).iter() { - chunk_positions.push(pos); - pos += len as usize; - } + pub fn new(path: PathBuf, version: u8, content_start: usize, crypto: Arc>, info: BundleInfo) -> Self { Bundle { info: info, - chunks: chunks, + chunks: None, version: version, path: path, crypto: crypto, content_start: content_start, - chunk_positions: chunk_positions + chunk_positions: None } } @@ -147,15 +141,32 @@ impl Bundle { } let header: BundleInfo = try!(msgpack::decode_from_stream(&mut file).context(&path as &Path)); debug!("Load bundle {}", header.id); - let mut chunk_data = Vec::with_capacity(header.chunk_info_size); - chunk_data.resize(header.chunk_info_size, 0); - try!(file.read_exact(&mut chunk_data).context(&path as &Path)); - if let Some(ref encryption) = header.encryption { - chunk_data = try!(crypto.lock().unwrap().decrypt(&encryption, &chunk_data).context(&path as &Path)); + let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize + header.chunk_info_size; + Ok(Bundle::new(path, version, content_start, crypto, header)) + } + + pub fn load_chunklist(&mut self) -> Result<(), BundleError> { + debug!("Load bundle chunklist {} ({:?})", self.info.id, self.info.mode); + let mut file = BufReader::new(try!(File::open(&self.path).context(&self.path as &Path))); + let len = self.info.chunk_info_size; + let start = self.content_start - len; + try!(file.seek(SeekFrom::Start(start as u64)).context(&self.path as &Path)); + let mut chunk_data = Vec::with_capacity(len); + chunk_data.resize(self.info.chunk_info_size, 0); + try!(file.read_exact(&mut chunk_data).context(&self.path as &Path)); + if let Some(ref encryption) = self.info.encryption { + chunk_data = try!(self.crypto.lock().unwrap().decrypt(&encryption, &chunk_data).context(&self.path as &Path)); } let chunks = ChunkList::read_from(&chunk_data); - let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize; - Ok(Bundle::new(path, version, content_start, crypto, header, chunks)) + let mut chunk_positions = Vec::with_capacity(chunks.len()); + let mut pos = 0; + for &(_, len) in (&chunks).iter() { + chunk_positions.push(pos); + pos += len as usize; + } + self.chunks = Some(chunks); + self.chunk_positions = Some(chunk_positions); + Ok(()) } #[inline] @@ -185,20 +196,27 @@ impl Bundle { } #[inline] - pub fn get_chunk_position(&self, id: usize) -> Result<(usize, usize), BundleError> { + pub fn get_chunk_position(&mut self, id: usize) -> Result<(usize, usize), BundleError> { if id >= self.info.chunk_count { return Err(BundleError::NoSuchChunk(self.id(), id)) } - Ok((self.chunk_positions[id], self.chunks[id].1 as usize)) + if self.chunks.is_none() || self.chunk_positions.is_none() { + try!(self.load_chunklist()); + } + let pos = self.chunk_positions.as_ref().unwrap()[id]; + let len = self.chunks.as_ref().unwrap()[id].1 as usize; + Ok((pos, len)) } - pub fn check(&self, full: bool) -> Result<(), BundleError> { - //FIXME: adapt to new format - if self.info.chunk_count != self.chunks.len() { + pub fn check(&mut self, full: bool) -> Result<(), BundleError> { + if self.chunks.is_none() || self.chunk_positions.is_none() { + try!(self.load_chunklist()); + } + if self.info.chunk_count != self.chunks.as_ref().unwrap().len() { return Err(BundleError::Integrity(self.id(), "Chunk list size does not match chunk count")) } - if self.chunks.iter().map(|c| c.1 as usize).sum::() != self.info.raw_size { + if self.chunks.as_ref().unwrap().iter().map(|c| c.1 as usize).sum::() != self.info.raw_size { return Err(BundleError::Integrity(self.id(), "Individual chunk sizes do not add up to total size")) } diff --git a/src/bundledb/db.rs b/src/bundledb/db.rs index ae6b8f2..293ba19 100644 --- a/src/bundledb/db.rs +++ b/src/bundledb/db.rs @@ -8,7 +8,8 @@ use std::sync::{Arc, Mutex}; pub struct BundleDb { - path: PathBuf, + remote_path: PathBuf, + local_path: PathBuf, crypto: Arc>, bundles: HashMap, bundle_cache: LruCache> @@ -16,9 +17,10 @@ pub struct BundleDb { impl BundleDb { - fn new(path: PathBuf, crypto: Arc>) -> Self { + fn new(remote_path: PathBuf, local_path: PathBuf, crypto: Arc>) -> Self { BundleDb { - path: path, + remote_path: remote_path, + local_path: local_path, crypto: crypto, bundles: HashMap::new(), bundle_cache: LruCache::new(5, 10) @@ -26,7 +28,7 @@ impl BundleDb { } pub fn bundle_path(&self, bundle: &BundleId) -> (PathBuf, PathBuf) { - let mut folder = self.path.clone(); + let mut folder = self.remote_path.clone(); let mut file = bundle.to_string().to_owned() + ".bundle"; let mut count = self.bundles.len(); while count >= 100 { @@ -43,7 +45,7 @@ impl BundleDb { fn load_bundle_list(&mut self) -> Result<(), BundleError> { self.bundles.clear(); let mut paths = Vec::new(); - paths.push(self.path.clone()); + paths.push(self.remote_path.clone()); while let Some(path) = paths.pop() { for entry in try!(fs::read_dir(path).map_err(BundleError::List)) { let entry = try!(entry.map_err(BundleError::List)); @@ -60,18 +62,21 @@ impl BundleDb { } #[inline] - pub fn open>(path: P, crypto: Arc>) -> Result { - let path = path.as_ref().to_owned(); - let mut self_ = Self::new(path, crypto); + pub fn open, L: AsRef>(remote_path: R, local_path: L, crypto: Arc>) -> Result { + let remote_path = remote_path.as_ref().to_owned(); + let local_path = local_path.as_ref().to_owned(); + let mut self_ = Self::new(remote_path, local_path, crypto); try!(self_.load_bundle_list()); Ok(self_) } #[inline] - pub fn create>(path: P, crypto: Arc>) -> Result { - let path = path.as_ref().to_owned(); - try!(fs::create_dir_all(&path).context(&path as &Path)); - Ok(Self::new(path, crypto)) + pub fn create, L: AsRef>(remote_path: R, local_path: L, crypto: Arc>) -> Result { + let remote_path = remote_path.as_ref().to_owned(); + let local_path = local_path.as_ref().to_owned(); + try!(fs::create_dir_all(&remote_path).context(&remote_path as &Path)); + try!(fs::create_dir_all(&local_path).context(&local_path as &Path)); + Ok(Self::new(remote_path, local_path, crypto)) } #[inline] @@ -86,7 +91,7 @@ impl BundleDb { } pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result, BundleError> { - let bundle = try!(self.bundles.get(bundle_id).ok_or(BundleError::NoSuchBundle(bundle_id.clone()))); + let bundle = try!(self.bundles.get_mut(bundle_id).ok_or(BundleError::NoSuchBundle(bundle_id.clone()))); let (pos, len) = try!(bundle.get_chunk_position(id)); let mut chunk = Vec::with_capacity(len); if let Some(data) = self.bundle_cache.get(bundle_id) { @@ -127,8 +132,8 @@ impl BundleDb { } #[inline] - pub fn check(&self, full: bool) -> Result<(), BundleError> { - for bundle in self.bundles.values() { + pub fn check(&mut self, full: bool) -> Result<(), BundleError> { + for bundle in self.bundles.values_mut() { try!(bundle.check(full)) } Ok(()) diff --git a/src/bundledb/writer.rs b/src/bundledb/writer.rs index 30a514b..43bf4c2 100644 --- a/src/bundledb/writer.rs +++ b/src/bundledb/writer.rs @@ -93,7 +93,7 @@ impl BundleWriter { try!(file.write_all(&chunk_data).context(&path as &Path)); let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize; try!(file.write_all(&self.data).context(&path as &Path)); - Ok(Bundle::new(path, HEADER_VERSION, content_start, self.crypto, header, self.chunks)) + Ok(Bundle::new(path, HEADER_VERSION, content_start, self.crypto, header)) } #[inline] diff --git a/src/directory_struture.md b/src/directory_struture.md new file mode 100644 index 0000000..3f8206b --- /dev/null +++ b/src/directory_struture.md @@ -0,0 +1,22 @@ +% Repository directory structure +# Repository directory structure + +# `/bundles` + +#### `/bundles/cache.dat` + +#### `/bundles/cache` + +### `/bundles.map` + +### `/index` + +### `/keys` + +### `/config.yaml` + +### `/remote` + +#### `/remote/bundles` + +#### `/remote/backups` diff --git a/src/main.rs b/src/main.rs index cd06da8..b589f33 100644 --- a/src/main.rs +++ b/src/main.rs @@ -36,6 +36,7 @@ mod prelude; // TODO: list --tree // TODO: Import repository from remote folder // TODO: Continue on errors +// TODO: Allow to use tar files for backup and restore (--tar, http://alexcrichton.com/tar-rs/tar/index.html) fn main() { cli::run(); diff --git a/src/repository/mod.rs b/src/repository/mod.rs index 17eeb0f..f6d298d 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -47,6 +47,7 @@ impl Repository { try!(fs::create_dir(path.join("keys"))); let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys"))))); let bundles = try!(BundleDb::create( + path.join("remote/bundles"), path.join("bundles"), crypto.clone() )); @@ -75,6 +76,7 @@ impl Repository { let config = try!(Config::load(path.join("config.yaml"))); let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys"))))); let bundles = try!(BundleDb::open( + path.join("remote/bundles"), path.join("bundles"), crypto.clone() ));