Some changes for remote path

src/bundledb/bundle.rs

@@ -102,30 +102,24 @@ impl Default for BundleInfo {
 
 pub struct Bundle {
     pub info: BundleInfo,
-    pub chunks: ChunkList,
     pub version: u8,
     pub path: PathBuf,
     crypto: Arc<Mutex<Crypto>>,
     pub content_start: usize,
-    pub chunk_positions: Vec<usize>
+    pub chunks: Option<ChunkList>,
+    pub chunk_positions: Option<Vec<usize>>
 }
 
 impl Bundle {
-    pub fn new(path: PathBuf, version: u8, content_start: usize, crypto: Arc<Mutex<Crypto>>, info: BundleInfo, chunks: ChunkList) -> Self {
+    pub fn new(path: PathBuf, version: u8, content_start: usize, crypto: Arc<Mutex<Crypto>>, info: BundleInfo) -> Self {
-        let mut chunk_positions = Vec::with_capacity(chunks.len());
-        let mut pos = 0;
-        for &(_, len) in (&chunks).iter() {
-            chunk_positions.push(pos);
-            pos += len as usize;
-        }
         Bundle {
             info: info,
-            chunks: chunks,
+            chunks: None,
             version: version,
             path: path,
             crypto: crypto,
             content_start: content_start,
-            chunk_positions: chunk_positions
+            chunk_positions: None
         }
     }
 
@@ -147,15 +141,32 @@ impl Bundle {
         }
         let header: BundleInfo = try!(msgpack::decode_from_stream(&mut file).context(&path as &Path));
         debug!("Load bundle {}", header.id);
-        let mut chunk_data = Vec::with_capacity(header.chunk_info_size);
+        let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize + header.chunk_info_size;
-        chunk_data.resize(header.chunk_info_size, 0);
+        Ok(Bundle::new(path, version, content_start, crypto, header))
-        try!(file.read_exact(&mut chunk_data).context(&path as &Path));
+    }
-        if let Some(ref encryption) = header.encryption {
+
-            chunk_data = try!(crypto.lock().unwrap().decrypt(&encryption, &chunk_data).context(&path as &Path));
+    pub fn load_chunklist(&mut self) -> Result<(), BundleError> {
+        debug!("Load bundle chunklist {} ({:?})", self.info.id, self.info.mode);
+        let mut file = BufReader::new(try!(File::open(&self.path).context(&self.path as &Path)));
+        let len = self.info.chunk_info_size;
+        let start = self.content_start - len;
+        try!(file.seek(SeekFrom::Start(start as u64)).context(&self.path as &Path));
+        let mut chunk_data = Vec::with_capacity(len);
+        chunk_data.resize(self.info.chunk_info_size, 0);
+        try!(file.read_exact(&mut chunk_data).context(&self.path as &Path));
+        if let Some(ref encryption) = self.info.encryption {
+            chunk_data = try!(self.crypto.lock().unwrap().decrypt(&encryption, &chunk_data).context(&self.path as &Path));
         }
         let chunks = ChunkList::read_from(&chunk_data);
-        let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize;
+        let mut chunk_positions = Vec::with_capacity(chunks.len());
-        Ok(Bundle::new(path, version, content_start, crypto, header, chunks))
+        let mut pos = 0;
+        for &(_, len) in (&chunks).iter() {
+            chunk_positions.push(pos);
+            pos += len as usize;
+        }
+        self.chunks = Some(chunks);
+        self.chunk_positions = Some(chunk_positions);
+        Ok(())
     }
 
     #[inline]
@@ -185,20 +196,27 @@ impl Bundle {
     }
 
     #[inline]
-    pub fn get_chunk_position(&self, id: usize) -> Result<(usize, usize), BundleError> {
+    pub fn get_chunk_position(&mut self, id: usize) -> Result<(usize, usize), BundleError> {
         if id >= self.info.chunk_count {
             return Err(BundleError::NoSuchChunk(self.id(), id))
         }
-        Ok((self.chunk_positions[id], self.chunks[id].1 as usize))
+        if self.chunks.is_none() || self.chunk_positions.is_none() {
+            try!(self.load_chunklist());
+        }
+        let pos = self.chunk_positions.as_ref().unwrap()[id];
+        let len = self.chunks.as_ref().unwrap()[id].1 as usize;
+        Ok((pos, len))
     }
 
-    pub fn check(&self, full: bool) -> Result<(), BundleError> {
+    pub fn check(&mut self, full: bool) -> Result<(), BundleError> {
-        //FIXME: adapt to new format
+        if self.chunks.is_none() || self.chunk_positions.is_none() {
-        if self.info.chunk_count != self.chunks.len() {
+            try!(self.load_chunklist());
+        }
+        if self.info.chunk_count != self.chunks.as_ref().unwrap().len() {
             return Err(BundleError::Integrity(self.id(),
                 "Chunk list size does not match chunk count"))
         }
-        if self.chunks.iter().map(|c| c.1 as usize).sum::<usize>() != self.info.raw_size {
+        if self.chunks.as_ref().unwrap().iter().map(|c| c.1 as usize).sum::<usize>() != self.info.raw_size {
             return Err(BundleError::Integrity(self.id(),
                 "Individual chunk sizes do not add up to total size"))
         }

src/bundledb/db.rs

@@ -8,7 +8,8 @@ use std::sync::{Arc, Mutex};
 
 
 pub struct BundleDb {
-    path: PathBuf,
+    remote_path: PathBuf,
+    local_path: PathBuf,
     crypto: Arc<Mutex<Crypto>>,
     bundles: HashMap<BundleId, Bundle>,
     bundle_cache: LruCache<BundleId, Vec<u8>>
@@ -16,9 +17,10 @@ pub struct BundleDb {
 
 
 impl BundleDb {
-    fn new(path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Self {
+    fn new(remote_path: PathBuf, local_path: PathBuf, crypto: Arc<Mutex<Crypto>>) -> Self {
         BundleDb {
-            path: path,
+            remote_path: remote_path,
+            local_path: local_path,
             crypto: crypto,
             bundles: HashMap::new(),
             bundle_cache: LruCache::new(5, 10)
@@ -26,7 +28,7 @@ impl BundleDb {
     }
 
     pub fn bundle_path(&self, bundle: &BundleId) -> (PathBuf, PathBuf) {
-        let mut folder = self.path.clone();
+        let mut folder = self.remote_path.clone();
         let mut file = bundle.to_string().to_owned() + ".bundle";
         let mut count = self.bundles.len();
         while count >= 100 {
@@ -43,7 +45,7 @@ impl BundleDb {
     fn load_bundle_list(&mut self) -> Result<(), BundleError> {
         self.bundles.clear();
         let mut paths = Vec::new();
-        paths.push(self.path.clone());
+        paths.push(self.remote_path.clone());
         while let Some(path) = paths.pop() {
             for entry in try!(fs::read_dir(path).map_err(BundleError::List)) {
                 let entry = try!(entry.map_err(BundleError::List));
@@ -60,18 +62,21 @@ impl BundleDb {
     }
 
     #[inline]
-    pub fn open<P: AsRef<Path>>(path: P, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
+    pub fn open<R: AsRef<Path>, L: AsRef<Path>>(remote_path: R, local_path: L, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
-        let path = path.as_ref().to_owned();
+        let remote_path = remote_path.as_ref().to_owned();
-        let mut self_ = Self::new(path, crypto);
+        let local_path = local_path.as_ref().to_owned();
+        let mut self_ = Self::new(remote_path, local_path, crypto);
         try!(self_.load_bundle_list());
         Ok(self_)
     }
 
     #[inline]
-    pub fn create<P: AsRef<Path>>(path: P, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
+    pub fn create<R: AsRef<Path>, L: AsRef<Path>>(remote_path: R, local_path: L, crypto: Arc<Mutex<Crypto>>) -> Result<Self, BundleError> {
-        let path = path.as_ref().to_owned();
+        let remote_path = remote_path.as_ref().to_owned();
-        try!(fs::create_dir_all(&path).context(&path as &Path));
+        let local_path = local_path.as_ref().to_owned();
-        Ok(Self::new(path, crypto))
+        try!(fs::create_dir_all(&remote_path).context(&remote_path as &Path));
+        try!(fs::create_dir_all(&local_path).context(&local_path as &Path));
+        Ok(Self::new(remote_path, local_path, crypto))
     }
 
     #[inline]
@@ -86,7 +91,7 @@ impl BundleDb {
     }
 
     pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleError> {
-        let bundle = try!(self.bundles.get(bundle_id).ok_or(BundleError::NoSuchBundle(bundle_id.clone())));
+        let bundle = try!(self.bundles.get_mut(bundle_id).ok_or(BundleError::NoSuchBundle(bundle_id.clone())));
         let (pos, len) = try!(bundle.get_chunk_position(id));
         let mut chunk = Vec::with_capacity(len);
         if let Some(data) = self.bundle_cache.get(bundle_id) {
@@ -127,8 +132,8 @@ impl BundleDb {
     }
 
     #[inline]
-    pub fn check(&self, full: bool) -> Result<(), BundleError> {
+    pub fn check(&mut self, full: bool) -> Result<(), BundleError> {
-        for bundle in self.bundles.values() {
+        for bundle in self.bundles.values_mut() {
             try!(bundle.check(full))
         }
         Ok(())

src/bundledb/writer.rs

@@ -93,7 +93,7 @@ impl BundleWriter {
         try!(file.write_all(&chunk_data).context(&path as &Path));
         let content_start = file.seek(SeekFrom::Current(0)).unwrap() as usize;
         try!(file.write_all(&self.data).context(&path as &Path));
-        Ok(Bundle::new(path, HEADER_VERSION, content_start, self.crypto, header, self.chunks))
+        Ok(Bundle::new(path, HEADER_VERSION, content_start, self.crypto, header))
     }
 
     #[inline]

src/directory_struture.md

@@ -0,0 +1,22 @@
+% Repository directory structure
+# Repository directory structure
+
+# `/bundles`
+
+#### `/bundles/cache.dat`
+
+#### `/bundles/cache`
+
+### `/bundles.map`
+
+### `/index`
+
+### `/keys`
+
+### `/config.yaml`
+
+### `/remote`
+
+#### `/remote/bundles`
+
+#### `/remote/backups`

src/main.rs

@@ -36,6 +36,7 @@ mod prelude;
 // TODO: list --tree
 // TODO: Import repository from remote folder
 // TODO: Continue on errors
+// TODO: Allow to use tar files for backup and restore (--tar, http://alexcrichton.com/tar-rs/tar/index.html)
 
 fn main() {
     cli::run();

src/repository/mod.rs

@@ -47,6 +47,7 @@ impl Repository {
         try!(fs::create_dir(path.join("keys")));
         let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys")))));
         let bundles = try!(BundleDb::create(
+            path.join("remote/bundles"),
             path.join("bundles"),
             crypto.clone()
         ));
@@ -75,6 +76,7 @@ impl Repository {
         let config = try!(Config::load(path.join("config.yaml")));
         let crypto = Arc::new(Mutex::new(try!(Crypto::open(path.join("keys")))));
         let bundles = try!(BundleDb::open(
+            path.join("remote/bundles"),
             path.join("bundles"),
             crypto.clone()
         ));