From 229c4f7e288364b8a311127cf5c550e2de8e23e0 Mon Sep 17 00:00:00 2001 From: Dennis Schwerdel Date: Tue, 18 Apr 2017 13:52:43 +0200 Subject: [PATCH] Moved index to separate project (re #17) --- CHANGELOG.md | 1 + Cargo.lock | 10 +- Cargo.toml | 2 +- index/Cargo.lock | 55 +++++++ index/Cargo.toml | 8 + src/index.rs => index/src/lib.rs | 274 ++++++++++++++----------------- src/main.rs | 4 +- src/prelude.rs | 4 +- src/repository/integrity.rs | 13 +- src/repository/mod.rs | 38 ++++- src/repository/vacuum.rs | 5 +- 11 files changed, 240 insertions(+), 174 deletions(-) create mode 100644 index/Cargo.lock create mode 100644 index/Cargo.toml rename src/index.rs => index/src/lib.rs (66%) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4d19a7..5f53d2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ This project follows [semantic versioning](http://semver.org). ### UNRELEASED * [added] Ability to read/write tar file from/to stdin/stdout * [added] Added date to bundles +* [added] Option to combine small bundles * [modified] Logging to stderr * [modified] Enforce deterministic bundle ordering * [modified] More info in analyze subcommand diff --git a/Cargo.lock b/Cargo.lock index 97c88bc..cc6d17b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,10 +10,10 @@ dependencies = [ "crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", "filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "fuse 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "index 0.1.0", "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -133,6 +133,14 @@ dependencies = [ "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "index" +version = "0.1.0" +dependencies = [ + "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "kernel32-sys" version = "0.2.2" diff --git a/Cargo.toml b/Cargo.toml index 9a9e7da..d4f2303 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,6 @@ rmp-serde = "0.12" serde_yaml = "0.6" serde_utils = "0.5.1" squash-sys = "0.9" -mmap = "0.1" quick-error = "1.1" blake2-rfc = "0.2" murmurhash3 = "0.0.5" @@ -33,6 +32,7 @@ pbr = "1.0" users = "0.5" time = "*" libc = "*" +index = {path="index"} [build-dependencies] pkg-config = "0.3" diff --git a/index/Cargo.lock b/index/Cargo.lock new file mode 100644 index 0000000..4220f63 --- /dev/null +++ b/index/Cargo.lock @@ -0,0 +1,55 @@ +[root] +name = "index" +version = "0.1.0" +dependencies = [ + "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mmap" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quick-error" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tempdir" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122" +"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135" +"checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823" +"checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c" +"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" +"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" diff --git a/index/Cargo.toml b/index/Cargo.toml new file mode 100644 index 0000000..c2a86c8 --- /dev/null +++ b/index/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "index" +version = "0.1.0" +authors = ["Dennis Schwerdel "] + +[dependencies] +mmap = "0.1" +quick-error = "1.1" diff --git a/src/index.rs b/index/src/lib.rs similarity index 66% rename from src/index.rs rename to index/src/lib.rs index 400af0a..f1bd6c5 100644 --- a/src/index.rs +++ b/index/src/lib.rs @@ -1,4 +1,5 @@ -use super::util::Hash; +extern crate mmap; +#[macro_use] extern crate quick_error; use std::path::Path; use std::fs::{File, OpenOptions}; @@ -10,8 +11,6 @@ use std::os::unix::io::AsRawFd; use mmap::{MemoryMap, MapOption, MapError}; -const MAGIC: [u8; 7] = *b"zvault\x02"; -const VERSION: u8 = 1; pub const MAX_USAGE: f64 = 0.9; pub const MIN_USAGE: f64 = 0.35; pub const INITIAL_SIZE: usize = 1024; @@ -40,9 +39,9 @@ quick_error!{ description("Unsupported version") display("Index error: index file has unsupported version: {}", version) } - WrongPosition(key: Hash, should: usize, is: LocateResult) { + WrongPosition(should: usize, is: LocateResult) { description("Key at wrong position") - display("Index error: key {} has wrong position, expected at: {}, but is at: {:?}", key, should, is) + display("Index error: key has wrong position, expected at: {}, but is at: {:?}", should, is) } WrongEntryCount(header: usize, actual: usize) { description("Wrong entry count") @@ -60,49 +59,31 @@ pub struct Header { capacity: u64, } -#[repr(packed)] -#[derive(Clone, Copy, PartialEq, Debug)] -pub struct Location { - pub bundle: u32, - pub chunk: u32 +pub trait Key: Clone + Eq + Copy { + fn hash(&self) -> u64; + fn is_used(&self) -> bool; + fn clear(&mut self); } -impl Location { - pub fn new(bundle: u32, chunk: u32) -> Self { - Location{ bundle: bundle, chunk: chunk } - } -} - #[repr(packed)] #[derive(Clone)] -pub struct Entry { - pub key: Hash, - pub data: Location +pub struct Entry { + pub key: K, + pub data: V } -impl Entry { +impl Entry { #[inline] fn is_used(&self) -> bool { - self.key.low != 0 || self.key.high != 0 + self.key.is_used() } + #[inline] fn clear(&mut self) { - self.key.low = 0; - self.key.high = 0; + self.key.clear() } } -pub struct Index { - capacity: usize, - entries: usize, - max_entries: usize, - min_entries: usize, - fd: File, - mmap: MemoryMap, - data: &'static mut [Entry] -} - - #[derive(Debug)] pub enum LocateResult { Found(usize), // Found the key at this position @@ -110,55 +91,94 @@ pub enum LocateResult { Steal(usize) // Found a spot to steal at this position while searching for a key } -impl Index { - pub fn new(path: &Path, create: bool) -> Result { + +pub struct Iter<'a, K: 'static, V: 'static> { + items: &'a [Entry], +} + +impl<'a, K: Key, V> Iterator for Iter<'a, K, V> { + type Item = (&'a K, &'a V); + fn next(&mut self) -> Option { + while let Some((first, rest)) = self.items.split_first() { + self.items = rest; + if first.is_used() { + return Some((&first.key, &first.data)); + } + } + None + } +} + + +fn mmap_as_ref(mmap: &MemoryMap, len: usize) -> (&'static mut Header, &'static mut [Entry]) { + if mmap.len() < mem::size_of::
() + len * mem::size_of::>() { + panic!("Memory map too small"); + } + let header = unsafe { &mut *(mmap.data() as *mut Header) }; + let ptr = unsafe { mmap.data().offset(mem::size_of::
() as isize) as *mut Entry }; + let data = unsafe { slice::from_raw_parts_mut(ptr, len) }; + (header, data) +} + +pub struct Index { + capacity: usize, + mask: usize, + entries: usize, + max_entries: usize, + min_entries: usize, + fd: File, + mmap: MemoryMap, + header: &'static mut Header, + data: &'static mut [Entry] +} + +impl Index { + pub fn new(path: &Path, create: bool, magic: &[u8; 7], version: u8) -> Result { let fd = try!(OpenOptions::new().read(true).write(true).create(create).open(path)); if create { - try!(Index::resize_fd(&fd, INITIAL_SIZE)); + try!(Self::resize_fd(&fd, INITIAL_SIZE)); } - let mmap = try!(Index::map_fd(&fd)); + let mmap = try!(Self::map_fd(&fd)); if mmap.len() < mem::size_of::
() { return Err(IndexError::WrongMagic); } - let data = Index::mmap_as_slice(&mmap, INITIAL_SIZE as usize); - let mut index = Index{capacity: 0, max_entries: 0, min_entries: 0, entries: 0, fd: fd, mmap: mmap, data: data}; - { - let capacity; - let entries; - { - let header = index.header(); - if create { - header.magic = MAGIC; - header.version = VERSION; - header.entries = 0; - header.capacity = INITIAL_SIZE as u64; - } else { - if header.magic != MAGIC { - return Err(IndexError::WrongMagic); - } - if header.version != VERSION { - return Err(IndexError::UnsupportedVersion(header.version)); - } - } - capacity = header.capacity; - entries = header.entries; - } - index.data = Index::mmap_as_slice(&index.mmap, capacity as usize); - index.set_capacity(capacity as usize); - index.entries = entries as usize; + let (header, _) = mmap_as_ref::(&mmap, INITIAL_SIZE as usize); + if create { + header.magic = magic.to_owned(); + header.version = version; + header.entries = 0; + header.capacity = INITIAL_SIZE as u64; } + if &header.magic != magic { + return Err(IndexError::WrongMagic); + } + if header.version != version { + return Err(IndexError::UnsupportedVersion(header.version)); + } + let (header, data) = mmap_as_ref(&mmap, header.capacity as usize); + let index = Index{ + capacity: header.capacity as usize, + mask: header.capacity as usize -1, + max_entries: (header.capacity as f64 * MAX_USAGE) as usize, + min_entries: (header.capacity as f64 * MIN_USAGE) as usize, + entries: header.entries as usize, + fd: fd, + mmap: mmap, + data: data, + header: header + }; debug_assert!(index.check().is_ok(), "Inconsistent after creation"); Ok(index) } #[inline] - pub fn open>(path: P) -> Result { - Index::new(path.as_ref(), false) + pub fn open>(path: P, magic: &[u8; 7], version: u8) -> Result { + Index::new(path.as_ref(), false, magic, version) } #[inline] - pub fn create>(path: P) -> Result { - Index::new(path.as_ref(), true) + pub fn create>(path: P, magic: &[u8; 7], version: u8) -> Result { + Index::new(path.as_ref(), true, magic, version) } #[inline] @@ -174,29 +194,14 @@ impl Index { #[inline] fn resize_fd(fd: &File, capacity: usize) -> Result<(), IndexError> { - fd.set_len((mem::size_of::
() + capacity * mem::size_of::()) as u64).map_err(IndexError::Io) - } - - #[inline] - fn mmap_as_slice(mmap: &MemoryMap, len: usize) -> &'static mut [Entry] { - if mmap.len() < mem::size_of::
() + len * mem::size_of::() { - panic!("Memory map too small"); - } - let ptr = unsafe { mmap.data().offset(mem::size_of::
() as isize) as *mut Entry }; - unsafe { slice::from_raw_parts_mut(ptr, len) } - } - - #[inline] - fn header(&mut self) -> &mut Header { - if self.mmap.len() < mem::size_of::
() { - panic!("Failed to read beyond end"); - } - unsafe { &mut *(self.mmap.data() as *mut Header) } + fd.set_len((mem::size_of::
() + capacity * mem::size_of::>()) as u64).map_err(IndexError::Io) } #[inline] fn set_capacity(&mut self, capacity: usize) { self.capacity = capacity; + debug_assert_eq!(capacity.count_ones(), 1); + self.mask = capacity -1; self.min_entries = (capacity as f64 * MIN_USAGE) as usize; self.max_entries = (capacity as f64 * MAX_USAGE) as usize; } @@ -228,9 +233,11 @@ impl Index { let new_capacity = self.capacity / 2; self.set_capacity(new_capacity); try!(self.reinsert(new_capacity, old_capacity)); - try!(Index::resize_fd(&self.fd, new_capacity)); - self.mmap = try!(Index::map_fd(&self.fd)); - self.data = Index::mmap_as_slice(&self.mmap, new_capacity); + try!(Self::resize_fd(&self.fd, new_capacity)); + self.mmap = try!(Self::map_fd(&self.fd)); + let (header, data) = mmap_as_ref(&self.mmap, new_capacity); + self.header = header; + self.data = data; assert_eq!(self.data.len(), self.capacity); Ok(true) } @@ -240,9 +247,11 @@ impl Index { return Ok(false) } let new_capacity = 2 * self.capacity; - try!(Index::resize_fd(&self.fd, new_capacity)); - self.mmap = try!(Index::map_fd(&self.fd)); - self.data = Index::mmap_as_slice(&self.mmap, new_capacity); + try!(Self::resize_fd(&self.fd, new_capacity)); + self.mmap = try!(Self::map_fd(&self.fd)); + let (header, data) = mmap_as_ref(&self.mmap, new_capacity); + self.header = header; + self.data = data; self.set_capacity(new_capacity); assert_eq!(self.data.len(), self.capacity); try!(self.reinsert(0, new_capacity)); @@ -259,7 +268,7 @@ impl Index { entries += 1; match self.locate(&entry.key) { LocateResult::Found(p) if p == pos => true, - found => return Err(IndexError::WrongPosition(entry.key, pos, found)) + found => return Err(IndexError::WrongPosition(pos, found)) }; } if entries != self.entries { @@ -286,18 +295,15 @@ impl Index { #[inline] fn write_header(&mut self) { - let entries = self.entries; - let capacity = self.capacity; - let header = self.header(); - header.entries = entries as u64; - header.capacity = capacity as u64; + self.header.entries = self.entries as u64; + self.header.capacity = self.capacity as u64; } /// Finds the position for this key /// If the key is in the table, it will be the position of the key, /// otherwise it will be the position where this key should be inserted - fn locate(&self, key: &Hash) -> LocateResult { - let mut pos = key.hash() as usize % self.capacity; + fn locate(&self, key: &K) -> LocateResult { + let mut pos = key.hash() as usize & self.mask; let mut dist = 0; loop { let entry = &self.data[pos]; @@ -307,11 +313,11 @@ impl Index { if entry.key == *key { return LocateResult::Found(pos); } - let odist = (pos + self.capacity - entry.key.hash() as usize % self.capacity) % self.capacity; + let odist = (pos + self.capacity - entry.key.hash() as usize & self.mask) & self.mask; if dist > odist { return LocateResult::Steal(pos); } - pos = (pos + 1) % self.capacity ; + pos = (pos + 1) & self.mask; dist += 1; } } @@ -323,14 +329,14 @@ impl Index { let mut last_pos; loop { last_pos = pos; - pos = (pos + 1) % self.capacity; + pos = (pos + 1) & self.mask; { let entry = &self.data[pos]; if !entry.is_used() { // we found a hole, stop shifting here break; } - if entry.key.hash() as usize % self.capacity == pos { + if entry.key.hash() as usize & self.mask == pos { // we found an entry at the right position, stop shifting here break; } @@ -342,7 +348,7 @@ impl Index { /// Adds the key, data pair into the table. /// If the key existed the old data is returned. - pub fn set(&mut self, key: &Hash, data: &Location) -> Result, IndexError> { + pub fn set(&mut self, key: &K, data: &V) -> Result, IndexError> { match self.locate(key) { LocateResult::Found(pos) => { let mut old = *data; @@ -370,7 +376,7 @@ impl Index { entry.data = *data; } loop { - cur_pos = (cur_pos + 1) % self.capacity; + cur_pos = (cur_pos + 1) & self.mask; let entry = &mut self.data[cur_pos]; if entry.is_used() { mem::swap(&mut stolen_key, &mut entry.key); @@ -388,7 +394,7 @@ impl Index { } #[inline] - pub fn contains(&self, key: &Hash) -> bool { + pub fn contains(&self, key: &K) -> bool { debug_assert!(self.check().is_ok(), "Inconsistent before get"); match self.locate(key) { LocateResult::Found(_) => true, @@ -397,7 +403,7 @@ impl Index { } #[inline] - pub fn pos(&self, key: &Hash) -> Option { + pub fn pos(&self, key: &K) -> Option { debug_assert!(self.check().is_ok(), "Inconsistent before get"); match self.locate(key) { LocateResult::Found(pos) => Some(pos), @@ -406,7 +412,7 @@ impl Index { } #[inline] - pub fn get(&self, key: &Hash) -> Option { + pub fn get(&self, key: &K) -> Option { debug_assert!(self.check().is_ok(), "Inconsistent before get"); match self.locate(key) { LocateResult::Found(pos) => Some(self.data[pos].data), @@ -415,7 +421,7 @@ impl Index { } #[inline] - pub fn modify(&mut self, key: &Hash, mut f: F) -> bool where F: FnMut(&mut Location) { + pub fn modify(&mut self, key: &K, mut f: F) -> bool where F: FnMut(&mut V) { debug_assert!(self.check().is_ok(), "Inconsistent before get"); match self.locate(key) { LocateResult::Found(pos) => { @@ -427,7 +433,7 @@ impl Index { } #[inline] - pub fn delete(&mut self, key: &Hash) -> Result { + pub fn delete(&mut self, key: &K) -> Result { match self.locate(key) { LocateResult::Found(pos) => { self.backshift(pos); @@ -438,7 +444,7 @@ impl Index { } } - pub fn filter(&mut self, mut f: F) -> Result where F: FnMut(&Hash, &Location) -> bool { + pub fn filter(&mut self, mut f: F) -> Result where F: FnMut(&K, &V) -> bool { //TODO: is it faster to walk in reverse direction? let mut deleted = 0; let mut pos = 0; @@ -460,48 +466,8 @@ impl Index { } #[inline] - pub fn walk(&self, mut f: F) -> Result<(), E> where F: FnMut(&Hash, &Location) -> Result<(), E> { - for pos in 0..self.capacity { - let entry = &self.data[pos]; - if entry.is_used() { - try!(f(&entry.key, &entry.data)); - } - } - Ok(()) - } - - #[inline] - pub fn walk_mut(&mut self, mut f: F) -> Result<(), E> where F: FnMut(&Hash, &mut Location) -> Result<(), E> { - for pos in 0..self.capacity { - let entry = &mut self.data[pos]; - if entry.is_used() { - try!(f(&entry.key, &mut entry.data)); - } - } - Ok(()) - } - - #[inline] - pub fn next_entry(&self, index: usize) -> Option { - let mut i = index; - while i < self.capacity && !self.data[i].is_used() { - i += 1; - } - if i == self.capacity { - None - } else { - Some(i) - } - } - - #[inline] - pub fn get_entry(&self, index: usize) -> Option<&Entry> { - let entry = &self.data[index]; - if entry.is_used() { - Some(entry) - } else { - None - } + pub fn iter<'a>(&'a self) -> Iter<'a, K, V> { + Iter{items: &self.data} } #[inline] diff --git a/src/main.rs b/src/main.rs index 7a88882..d7214bd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,6 @@ extern crate serde; extern crate rmp_serde; #[macro_use] extern crate serde_utils; extern crate squash_sys as squash; -extern crate mmap; extern crate blake2_rfc as blake2; extern crate murmurhash3; extern crate serde_yaml; @@ -28,10 +27,11 @@ extern crate pbr; extern crate users; extern crate libc; extern crate tar; +extern crate index; pub mod util; mod bundledb; -pub mod index; +//pub mod index; mod chunker; mod repository; mod cli; diff --git a/src/prelude.rs b/src/prelude.rs index 962bbaa..8a6f33b 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,8 +1,8 @@ pub use ::util::*; pub use ::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError, BundleDb, BundleWriterError, StoredBundle}; pub use ::chunker::{ChunkerType, Chunker, ChunkerStatus, IChunker, ChunkerError}; -pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, IntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError, RepositoryLayout}; -pub use ::index::{Index, Location, IndexError}; +pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, IntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError, RepositoryLayout, Location}; +pub use ::index::{Index, IndexError}; pub use ::mount::FuseFilesystem; pub use serde::{Serialize, Deserialize}; diff --git a/src/repository/integrity.rs b/src/repository/integrity.rs index 7ed0c46..d95b89b 100644 --- a/src/repository/integrity.rs +++ b/src/repository/integrity.rs @@ -48,31 +48,30 @@ quick_error!{ impl Repository { fn check_index_chunks(&self) -> Result<(), RepositoryError> { - let mut count = 0; let mut progress = ProgressBar::new(self.index.len() as u64); progress.message("checking index: "); progress.set_max_refresh_rate(Some(Duration::from_millis(100))); - let res = self.index.walk(|_hash, location| { + for (count,(_hash, location)) in self.index.iter().enumerate() { // Lookup bundle id from map let bundle_id = try!(self.get_bundle_id(location.bundle)); // Get bundle object from bundledb let bundle = if let Some(bundle) = self.bundles.get_bundle_info(&bundle_id) { bundle } else { + progress.finish_print("checking index: done."); return Err(IntegrityError::MissingBundle(bundle_id.clone()).into()) }; // Get chunk from bundle if bundle.info.chunk_count <= location.chunk as usize { + progress.finish_print("checking index: done."); return Err(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into()) } - count += 1; if count % 1000 == 0 { - progress.set(count); + progress.set(count as u64); } - Ok(()) - }); + } progress.finish_print("checking index: done."); - res + Ok(()) } fn check_chunks(&self, checked: &mut Bitmap, chunks: &[Chunk], mark: bool) -> Result { diff --git a/src/repository/mod.rs b/src/repository/mod.rs index 3470398..e41dca7 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -35,11 +35,41 @@ use self::bundle_map::BundleMap; const REPOSITORY_README: &'static [u8] = include_bytes!("../../docs/repository_readme.md"); const DEFAULT_EXCLUDES: &'static [u8] = include_bytes!("../../docs/excludes.default"); +const INDEX_MAGIC: [u8; 7] = *b"zvault\x02"; +const INDEX_VERSION: u8 = 1; + + +#[repr(packed)] +#[derive(Clone, Copy, PartialEq, Debug)] +pub struct Location { + pub bundle: u32, + pub chunk: u32 +} +impl Location { + pub fn new(bundle: u32, chunk: u32) -> Self { + Location{ bundle: bundle, chunk: chunk } + } +} + +impl ::index::Key for Hash { + fn hash(&self) -> u64 { + self.low + } + + fn is_used(&self) -> bool { + self.low != 0 || self.high != 0 + } + + fn clear(&mut self) { + self.low = 0; + self.high = 0; + } +} pub struct Repository { pub layout: RepositoryLayout, pub config: Config, - index: Index, + index: Index, crypto: Arc>, bundle_map: BundleMap, next_data_bundle: u32, @@ -67,7 +97,7 @@ impl Repository { try!(fs::create_dir_all(layout.remote_locks_path())); try!(config.save(layout.config_path())); try!(BundleDb::create(layout.clone())); - try!(Index::create(layout.index_path())); + try!(Index::::create(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION)); try!(BundleMap::create().save(layout.bundle_map_path())); try!(fs::create_dir_all(layout.backups_path())); Self::open(path) @@ -86,11 +116,11 @@ impl Repository { let lock = try!(local_locks.lock(false)); let crypto = Arc::new(Mutex::new(try!(Crypto::open(layout.keys_path())))); let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone())); - let (index, mut rebuild_index) = match Index::open(layout.index_path()) { + let (index, mut rebuild_index) = match Index::open(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION) { Ok(index) => (index, false), Err(err) => { error!("Failed to load local index:\n\tcaused by: {}", err); - (try!(Index::create(layout.index_path())), true) + (try!(Index::create(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION)), true) } }; let (bundle_map, rebuild_bundle_map) = match BundleMap::load(layout.bundle_map_path()) { diff --git a/src/repository/vacuum.rs b/src/repository/vacuum.rs index 6f30de6..f2c6316 100644 --- a/src/repository/vacuum.rs +++ b/src/repository/vacuum.rs @@ -79,12 +79,11 @@ impl Repository { } try!(self.flush()); info!("Checking index"); - self.index.walk::<_, ()>(|hash, location| { + for (hash, location) in self.index.iter() { if rewrite_bundles.contains(&location.bundle) { panic!("Removed bundle is still referenced in index: hash:{}, bundle:{}, chunk:{}", hash, location.bundle, location.chunk); } - Ok(()) - }).ok(); + } info!("Deleting {} bundles", rewrite_bundles.len()); for id in rewrite_bundles { try!(self.delete_bundle(id));