From 365121cfe157637477578a2618ca0d29acd05874 Mon Sep 17 00:00:00 2001 From: Dennis Schwerdel Date: Tue, 21 Mar 2017 19:18:18 +0100 Subject: [PATCH] Some changes --- Cargo.lock | 26 +++++++++--------- Cargo.toml | 2 +- src/bundledb/db.rs | 67 ++++++++++++++++++++++++++++++++++++++-------- src/main.rs | 9 +++---- 4 files changed, 74 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3adadc8..c5e36e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,15 +6,15 @@ dependencies = [ "blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.21.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rmp-serde 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.23 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)", "serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -71,7 +71,7 @@ dependencies = [ [[package]] name = "clap" -version = "2.21.1" +version = "2.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -193,12 +193,12 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rmp" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -211,13 +211,13 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rmp 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", "serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rustc-serialize" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -293,7 +293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -338,7 +338,7 @@ dependencies = [ "checksum blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "0c6a476f32fef3402f1161f89d0d39822809627754a126f8441ff2a9d45e2d59" "checksum byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c40977b0ee6b9885c9013cd41d9feffdd22deb3bb4dc3a71d901cc7a77de18c8" "checksum chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "158b0bd7d75cbb6bf9c25967a48a2e9f77da95876b858eadfabaa99cd069de6e" -"checksum clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)" = "74a80f603221c9cd9aa27a28f52af452850051598537bb6b359c38a7d61e5cda" +"checksum clap 2.21.2 (registry+https://github.com/rust-lang/crates.io-index)" = "58ad5e8142f3a5eab0c1cba5011aa383e009842936107fe4d94f1a8d380a1aec" "checksum constant_time_eq 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "07dcb7959f0f6f1cf662f9a7ff389bcb919924d99ac41cf31f10d611d8721323" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122" @@ -355,10 +355,10 @@ dependencies = [ "checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903" "checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c" "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" -"checksum redox_syscall 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "8dd35cc9a8bdec562c757e3d43c1526b5c6d2653e23e2315065bc25556550753" -"checksum rmp 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e59917c01f49718a59c644a621a4848aafc6577c4a47d66270d78951a807541a" +"checksum redox_syscall 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "29dbdfd4b9df8ab31dec47c6087b7b13cbf4a776f335e4de8efba8288dda075b" +"checksum rmp 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)" = "333f01365885cb192edaa22acb06d7e2f196bfd19d6969419e8b61307e0710ea" "checksum rmp-serde 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "06ec4d0cdea2645de5d0e649f90c3e654205d913e14adefa452257314a24e76e" -"checksum rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "237546c689f20bb44980270c73c3b9edd0891c1be49cc1274406134a66d3957b" +"checksum rustc-serialize 0.3.23 (registry+https://github.com/rust-lang/crates.io-index)" = "684ce48436d6465300c9ea783b6b14c4361d6b8dcbb1375b486a69cc19e2dfb0" "checksum serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)" = "a702319c807c016e51f672e5c77d6f0b46afddd744b5e437d6b8436b888b458f" "checksum serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b34a52969c7fc0254e214b82518c9a95dc88c84fc84cd847add314996a031be6" "checksum serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f8bd3f24ad8c7bcd34a6d70ba676dc11302b96f4f166aa5f947762e01098844d" diff --git a/Cargo.toml b/Cargo.toml index f1ea19a..1bf0804 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ blake2-rfc = "0.2" murmurhash3 = "0.0.5" rustc-serialize = "0.3" chrono = "0.3" -clap = "2.19" +clap = "2.21" log = "0.3" byteorder = "1.0" ansi_term = "0.9" diff --git a/src/bundledb/db.rs b/src/bundledb/db.rs index 6e5e440..b260044 100644 --- a/src/bundledb/db.rs +++ b/src/bundledb/db.rs @@ -3,8 +3,13 @@ use super::*; use std::path::{Path, PathBuf}; use std::collections::{HashMap, HashSet}; -use std::fs; +use std::fs::{self, File}; use std::sync::{Arc, Mutex}; +use std::io::{BufReader, BufWriter, Read, Write}; + + +pub static CACHE_FILE_STRING: [u8; 7] = *b"zvault\x04"; +pub static CACHE_FILE_VERSION: u8 = 1; pub fn bundle_path(bundle: &BundleId, mut folder: PathBuf, mut count: usize) -> (PathBuf, PathBuf) { @@ -89,12 +94,38 @@ impl StoredBundle { bundle.path = path.to_path_buf(); Ok(bundle) } + + pub fn read_list_from>(path: P) -> Result, BundleError> { + let path = path.as_ref(); + let mut file = BufReader::new(try!(File::open(path).context(path))); + let mut header = [0u8; 8]; + try!(file.read_exact(&mut header).context(&path as &Path)); + if header[..CACHE_FILE_STRING.len()] != CACHE_FILE_STRING { + return Err(BundleError::WrongHeader(path.to_path_buf())) + } + let version = header[HEADER_STRING.len()]; + if version != HEADER_VERSION { + return Err(BundleError::WrongVersion(path.to_path_buf(), version)) + } + Ok(try!(msgpack::decode_from_stream(&mut file).context(path))) + } + + pub fn save_list_to>(list: &[Self], path: P) -> Result<(), BundleError> { + let path = path.as_ref(); + let mut file = BufWriter::new(try!(File::create(path).context(path))); + try!(file.write_all(&HEADER_STRING).context(path)); + try!(file.write_all(&[HEADER_VERSION]).context(path)); + try!(msgpack::encode_to_stream(&list, &mut file).context(path)); + Ok(()) + } } pub struct BundleDb { remote_path: PathBuf, - local_path: PathBuf, + local_bundles_path: PathBuf, + temp_path: PathBuf, + remote_cache_path: PathBuf, crypto: Arc>, local_bundles: HashMap, remote_bundles: HashMap, @@ -105,8 +136,10 @@ pub struct BundleDb { impl BundleDb { fn new(remote_path: PathBuf, local_path: PathBuf, crypto: Arc>) -> Self { BundleDb { + remote_cache_path: local_path.join("bundle_info.cache"), + local_bundles_path: local_path.join("cached"), + temp_path: local_path.join("temp"), remote_path: remote_path, - local_path: local_path, crypto: crypto, local_bundles: HashMap::new(), remote_bundles: HashMap::new(), @@ -115,12 +148,23 @@ impl BundleDb { } fn load_bundle_list(&mut self) -> Result<(Vec, Vec), BundleError> { - try!(load_bundles(self.local_path.join("cached"), &mut self.local_bundles)); - load_bundles(&self.remote_path, &mut self.remote_bundles) + let bundle_info_cache = &self.remote_cache_path; + if let Ok(list) = StoredBundle::read_list_from(&bundle_info_cache) { + for bundle in list { + self.remote_bundles.insert(bundle.id(), bundle); + } + } + try!(load_bundles(&self.local_bundles_path, &mut self.local_bundles)); + let (new, gone) = try!(load_bundles(&self.remote_path, &mut self.remote_bundles)); + if !new.is_empty() || !gone.is_empty() { + let bundles: Vec<_> = self.remote_bundles.values().cloned().collect(); + try!(StoredBundle::save_list_to(&bundles, &bundle_info_cache)); + } + Ok((new, gone)) } pub fn temp_bundle_path(&self, id: &BundleId) -> PathBuf { - self.local_path.join("temp").join(id.to_string().to_owned() + ".bundle") + self.temp_path.join(id.to_string().to_owned() + ".bundle") } #[inline] @@ -136,10 +180,11 @@ impl BundleDb { pub fn create, L: AsRef>(remote_path: R, local_path: L, crypto: Arc>) -> Result { let remote_path = remote_path.as_ref().to_owned(); let local_path = local_path.as_ref().to_owned(); - try!(fs::create_dir_all(&remote_path).context(&remote_path as &Path)); - try!(fs::create_dir_all(local_path.join("cached")).context(&local_path as &Path)); - try!(fs::create_dir_all(local_path.join("temp")).context(&local_path as &Path)); - Ok(Self::new(remote_path, local_path, crypto)) + let self_ = Self::new(remote_path, local_path, crypto); + try!(fs::create_dir_all(&self_.remote_path).context(&self_.remote_path as &Path)); + try!(fs::create_dir_all(&self_.local_bundles_path).context(&self_.local_bundles_path as &Path)); + try!(fs::create_dir_all(&self_.temp_path).context(&self_.temp_path as &Path)); + Ok(self_) } #[inline] @@ -186,7 +231,7 @@ impl BundleDb { let bundle = try!(bundle.finish(&self)); let id = bundle.id(); if bundle.info.mode == BundleMode::Meta { - let (folder, filename) = bundle_path(&id, self.local_path.join("cached"), self.local_bundles.len()); + let (folder, filename) = bundle_path(&id, self.local_bundles_path.clone(), self.local_bundles.len()); try!(fs::create_dir_all(&folder).context(&folder as &Path)); let bundle = try!(bundle.copy_to(folder.join(filename))); self.local_bundles.insert(id.clone(), bundle); diff --git a/src/main.rs b/src/main.rs index 4164b6f..f9c6d37 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,15 +24,14 @@ mod repository; mod cli; mod prelude; -// TODO: Seperate remote folder -// TODO: - Copy/move backup files to remote folder -// TODO: - Load and compare remote bundles to bundle map -// TODO: - Lock during vacuum +// TODO: React on changes in remote bundles +// TODO: Lock during backup and vacuum // TODO: Remove backup subtrees // TODO: Recompress & combine bundles // TODO: list --tree // TODO: Import repository from remote folder -// TODO: Continue on errors +// TODO: Continue on errors (return summary as error) +// TODO: More detailed errors with nicer text // TODO: Allow to use tar files for backup and restore (--tar, http://alexcrichton.com/tar-rs/tar/index.html) fn main() {