Moved index to separate project (re #17)

CHANGELOG.md

@@ -6,6 +6,7 @@ This project follows [semantic versioning](http://semver.org).
 ### UNRELEASED
 * [added] Ability to read/write tar file from/to stdin/stdout
 * [added] Added date to bundles
+* [added] Option to combine small bundles
 * [modified] Logging to stderr
 * [modified] Enforce deterministic bundle ordering
 * [modified] More info in analyze subcommand

Cargo.lock

@@ -10,10 +10,10 @@ dependencies = [
  "crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "fuse 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "index 0.1.0",
  "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
- "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -133,6 +133,14 @@ dependencies = [
  "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "index"
+version = "0.1.0"
+dependencies = [
+ "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "kernel32-sys"
 version = "0.2.2"

Cargo.toml

@@ -10,7 +10,6 @@ rmp-serde = "0.12"
 serde_yaml = "0.6"
 serde_utils = "0.5.1"
 squash-sys = "0.9"
-mmap = "0.1"
 quick-error = "1.1"
 blake2-rfc = "0.2"
 murmurhash3 = "0.0.5"
@@ -33,6 +32,7 @@ pbr = "1.0"
 users = "0.5"
 time = "*"
 libc = "*"
+index = {path="index"}
 
 [build-dependencies]
 pkg-config = "0.3"

index/Cargo.lock

@@ -0,0 +1,55 @@
+[root]
+name = "index"
+version = "0.1.0"
+dependencies = [
+ "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "libc"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "libc"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "mmap"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "quick-error"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "rand"
+version = "0.3.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "tempdir"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[metadata]
+"checksum libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122"
+"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135"
+"checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823"
+"checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c"
+"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"
+"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"

index/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "index"
+version = "0.1.0"
+authors = ["Dennis Schwerdel <schwerdel@googlemail.com>"]
+
+[dependencies]
+mmap = "0.1"
+quick-error = "1.1"

index/src/lib.rs

@@ -1,4 +1,5 @@
-use super::util::Hash;
+extern crate mmap;
+#[macro_use] extern crate quick_error;
 
 use std::path::Path;
 use std::fs::{File, OpenOptions};
@@ -10,8 +11,6 @@ use std::os::unix::io::AsRawFd;
 use mmap::{MemoryMap, MapOption, MapError};
 
 
-const MAGIC: [u8; 7] = *b"zvault\x02";
-const VERSION: u8 = 1;
 pub const MAX_USAGE: f64 = 0.9;
 pub const MIN_USAGE: f64 = 0.35;
 pub const INITIAL_SIZE: usize = 1024;
@@ -40,9 +39,9 @@ quick_error!{
             description("Unsupported version")
             display("Index error: index file has unsupported version: {}", version)
         }
-        WrongPosition(key: Hash, should: usize, is: LocateResult) {
+        WrongPosition(should: usize, is: LocateResult) {
             description("Key at wrong position")
-            display("Index error: key {} has wrong position, expected at: {}, but is at: {:?}", key, should, is)
+            display("Index error: key has wrong position, expected at: {}, but is at: {:?}", should, is)
         }
         WrongEntryCount(header: usize, actual: usize) {
             description("Wrong entry count")
@@ -60,49 +59,31 @@ pub struct Header {
     capacity: u64,
 }
 
-#[repr(packed)]
+pub trait Key: Clone + Eq + Copy {
-#[derive(Clone, Copy, PartialEq, Debug)]
+    fn hash(&self) -> u64;
-pub struct Location {
+    fn is_used(&self) -> bool;
-    pub bundle: u32,
+    fn clear(&mut self);
-    pub chunk: u32
 }
-impl Location {
-    pub fn new(bundle: u32, chunk: u32) -> Self {
-        Location{ bundle: bundle, chunk: chunk }
-    }
-}
-
 
 #[repr(packed)]
 #[derive(Clone)]
-pub struct Entry {
+pub struct Entry<K, V> {
-    pub key: Hash,
+    pub key: K,
-    pub data: Location
+    pub data: V
 }
 
-impl Entry {
+impl<K: Key, V> Entry<K, V> {
     #[inline]
     fn is_used(&self) -> bool {
-        self.key.low != 0 || self.key.high != 0
+        self.key.is_used()
     }
 
+    #[inline]
     fn clear(&mut self) {
-        self.key.low = 0;
+        self.key.clear()
-        self.key.high = 0;
     }
 }
 
-pub struct Index {
-    capacity: usize,
-    entries: usize,
-    max_entries: usize,
-    min_entries: usize,
-    fd: File,
-    mmap: MemoryMap,
-    data: &'static mut [Entry]
-}
-
-
 #[derive(Debug)]
 pub enum LocateResult {
     Found(usize), // Found the key at this position
@@ -110,55 +91,94 @@ pub enum LocateResult {
     Steal(usize) // Found a spot to steal at this position while searching for a key
 }
 
-impl Index {
+
-    pub fn new(path: &Path, create: bool) -> Result<Index, IndexError> {
+pub struct Iter<'a, K: 'static, V: 'static> {
+    items: &'a [Entry<K, V>],
+}
+
+impl<'a, K: Key, V> Iterator for Iter<'a, K, V> {
+    type Item = (&'a K, &'a V);
+    fn next(&mut self) -> Option<Self::Item> {
+        while let Some((first, rest)) = self.items.split_first() {
+            self.items = rest;
+            if first.is_used() {
+                return Some((&first.key, &first.data));
+            }
+        }
+        None
+    }
+}
+
+
+fn mmap_as_ref<K, V>(mmap: &MemoryMap, len: usize) -> (&'static mut Header, &'static mut [Entry<K, V>]) {
+    if mmap.len() < mem::size_of::<Header>() + len * mem::size_of::<Entry<K, V>>() {
+        panic!("Memory map too small");
+    }
+    let header = unsafe { &mut *(mmap.data() as *mut Header) };
+    let ptr = unsafe { mmap.data().offset(mem::size_of::<Header>() as isize) as *mut Entry<K, V> };
+    let data = unsafe { slice::from_raw_parts_mut(ptr, len) };
+    (header, data)
+}
+
+pub struct Index<K: 'static, V: 'static> {
+    capacity: usize,
+    mask: usize,
+    entries: usize,
+    max_entries: usize,
+    min_entries: usize,
+    fd: File,
+    mmap: MemoryMap,
+    header: &'static mut Header,
+    data: &'static mut [Entry<K, V>]
+}
+
+impl<K: Key, V: Clone + Copy> Index<K, V> {
+    pub fn new(path: &Path, create: bool, magic: &[u8; 7], version: u8) -> Result<Self, IndexError> {
         let fd = try!(OpenOptions::new().read(true).write(true).create(create).open(path));
         if create {
-            try!(Index::resize_fd(&fd, INITIAL_SIZE));
+            try!(Self::resize_fd(&fd, INITIAL_SIZE));
         }
-        let mmap = try!(Index::map_fd(&fd));
+        let mmap = try!(Self::map_fd(&fd));
         if mmap.len() < mem::size_of::<Header>() {
             return Err(IndexError::WrongMagic);
         }
-        let data = Index::mmap_as_slice(&mmap, INITIAL_SIZE as usize);
+        let (header, _) = mmap_as_ref::<K, V>(&mmap, INITIAL_SIZE as usize);
-        let mut index = Index{capacity: 0, max_entries: 0, min_entries: 0, entries: 0, fd: fd, mmap: mmap, data: data};
+        if create {
-        {
+            header.magic = magic.to_owned();
-            let capacity;
+            header.version = version;
-            let entries;
+            header.entries = 0;
-            {
+            header.capacity = INITIAL_SIZE as u64;
-                let header = index.header();
-                if create {
-                    header.magic = MAGIC;
-                    header.version = VERSION;
-                    header.entries = 0;
-                    header.capacity = INITIAL_SIZE as u64;
-                } else {
-                    if header.magic != MAGIC {
-                        return Err(IndexError::WrongMagic);
-                    }
-                    if header.version != VERSION {
-                        return Err(IndexError::UnsupportedVersion(header.version));
-                    }
-                }
-                capacity = header.capacity;
-                entries = header.entries;
-            }
-            index.data = Index::mmap_as_slice(&index.mmap, capacity as usize);
-            index.set_capacity(capacity as usize);
-            index.entries = entries as usize;
         }
+        if &header.magic != magic {
+            return Err(IndexError::WrongMagic);
+        }
+        if header.version != version {
+            return Err(IndexError::UnsupportedVersion(header.version));
+        }
+        let (header, data) = mmap_as_ref(&mmap, header.capacity as usize);
+        let index = Index{
+            capacity: header.capacity as usize,
+            mask: header.capacity as usize -1,
+            max_entries: (header.capacity as f64 * MAX_USAGE) as usize,
+            min_entries: (header.capacity as f64 * MIN_USAGE) as usize,
+            entries: header.entries as usize,
+            fd: fd,
+            mmap: mmap,
+            data: data,
+            header: header
+        };
         debug_assert!(index.check().is_ok(), "Inconsistent after creation");
         Ok(index)
     }
 
     #[inline]
-    pub fn open<P: AsRef<Path>>(path: P) -> Result<Index, IndexError> {
+    pub fn open<P: AsRef<Path>>(path: P, magic: &[u8; 7], version: u8) -> Result<Self, IndexError> {
-        Index::new(path.as_ref(), false)
+        Index::new(path.as_ref(), false, magic, version)
     }
 
     #[inline]
-    pub fn create<P: AsRef<Path>>(path: P) -> Result<Index, IndexError> {
+    pub fn create<P: AsRef<Path>>(path: P, magic: &[u8; 7], version: u8) -> Result<Self, IndexError> {
-        Index::new(path.as_ref(), true)
+        Index::new(path.as_ref(), true, magic, version)
     }
 
     #[inline]
@@ -174,29 +194,14 @@ impl Index {
 
     #[inline]
     fn resize_fd(fd: &File, capacity: usize) -> Result<(), IndexError> {
-        fd.set_len((mem::size_of::<Header>() + capacity * mem::size_of::<Entry>()) as u64).map_err(IndexError::Io)
+        fd.set_len((mem::size_of::<Header>() + capacity * mem::size_of::<Entry<K, V>>()) as u64).map_err(IndexError::Io)
-    }
-
-    #[inline]
-    fn mmap_as_slice(mmap: &MemoryMap, len: usize) -> &'static mut [Entry] {
-        if mmap.len() < mem::size_of::<Header>() + len * mem::size_of::<Entry>() {
-            panic!("Memory map too small");
-        }
-        let ptr = unsafe { mmap.data().offset(mem::size_of::<Header>() as isize) as *mut Entry };
-        unsafe { slice::from_raw_parts_mut(ptr, len) }
-    }
-
-    #[inline]
-    fn header(&mut self) -> &mut Header {
-        if self.mmap.len() < mem::size_of::<Header>() {
-            panic!("Failed to read beyond end");
-        }
-        unsafe { &mut *(self.mmap.data() as *mut Header) }
     }
 
     #[inline]
     fn set_capacity(&mut self, capacity: usize) {
         self.capacity = capacity;
+        debug_assert_eq!(capacity.count_ones(), 1);
+        self.mask = capacity -1;
         self.min_entries = (capacity as f64 * MIN_USAGE) as usize;
         self.max_entries = (capacity as f64 * MAX_USAGE) as usize;
     }
@@ -228,9 +233,11 @@ impl Index {
         let new_capacity = self.capacity / 2;
         self.set_capacity(new_capacity);
         try!(self.reinsert(new_capacity, old_capacity));
-        try!(Index::resize_fd(&self.fd, new_capacity));
+        try!(Self::resize_fd(&self.fd, new_capacity));
-        self.mmap = try!(Index::map_fd(&self.fd));
+        self.mmap = try!(Self::map_fd(&self.fd));
-        self.data = Index::mmap_as_slice(&self.mmap, new_capacity);
+        let (header, data) = mmap_as_ref(&self.mmap, new_capacity);
+        self.header = header;
+        self.data = data;
         assert_eq!(self.data.len(), self.capacity);
         Ok(true)
     }
@@ -240,9 +247,11 @@ impl Index {
             return Ok(false)
         }
         let new_capacity = 2 * self.capacity;
-        try!(Index::resize_fd(&self.fd, new_capacity));
+        try!(Self::resize_fd(&self.fd, new_capacity));
-        self.mmap = try!(Index::map_fd(&self.fd));
+        self.mmap = try!(Self::map_fd(&self.fd));
-        self.data = Index::mmap_as_slice(&self.mmap, new_capacity);
+        let (header, data) = mmap_as_ref(&self.mmap, new_capacity);
+        self.header = header;
+        self.data = data;
         self.set_capacity(new_capacity);
         assert_eq!(self.data.len(), self.capacity);
         try!(self.reinsert(0, new_capacity));
@@ -259,7 +268,7 @@ impl Index {
             entries += 1;
             match self.locate(&entry.key) {
                 LocateResult::Found(p) if p == pos => true,
-                found => return Err(IndexError::WrongPosition(entry.key, pos, found))
+                found => return Err(IndexError::WrongPosition(pos, found))
             };
         }
         if entries != self.entries {
@@ -286,18 +295,15 @@ impl Index {
 
     #[inline]
     fn write_header(&mut self) {
-        let entries = self.entries;
+        self.header.entries = self.entries as u64;
-        let capacity = self.capacity;
+        self.header.capacity = self.capacity as u64;
-        let header = self.header();
-        header.entries = entries as u64;
-        header.capacity = capacity as u64;
     }
 
     /// Finds the position for this key
     /// If the key is in the table, it will be the position of the key,
     /// otherwise it will be the position where this key should be inserted
-    fn locate(&self, key: &Hash) -> LocateResult {
+    fn locate(&self, key: &K) -> LocateResult {
-        let mut pos = key.hash() as usize % self.capacity;
+        let mut pos = key.hash() as usize & self.mask;
         let mut dist = 0;
         loop {
             let entry = &self.data[pos];
@@ -307,11 +313,11 @@ impl Index {
             if entry.key == *key {
                 return LocateResult::Found(pos);
             }
-            let odist = (pos + self.capacity - entry.key.hash() as usize % self.capacity) % self.capacity;
+            let odist = (pos + self.capacity - entry.key.hash() as usize & self.mask) & self.mask;
             if dist > odist {
                 return LocateResult::Steal(pos);
             }
-            pos = (pos + 1) % self.capacity ;
+            pos = (pos + 1) & self.mask;
             dist += 1;
         }
     }
@@ -323,14 +329,14 @@ impl Index {
         let mut last_pos;
         loop {
             last_pos = pos;
-            pos = (pos + 1) % self.capacity;
+            pos = (pos + 1) & self.mask;
             {
                 let entry = &self.data[pos];
                 if !entry.is_used() {
                     // we found a hole, stop shifting here
                     break;
                 }
-                if entry.key.hash() as usize % self.capacity == pos {
+                if entry.key.hash() as usize & self.mask == pos {
                     // we found an entry at the right position, stop shifting here
                     break;
                 }
@@ -342,7 +348,7 @@ impl Index {
 
     /// Adds the key, data pair into the table.
     /// If the key existed the old data is returned.
-    pub fn set(&mut self, key: &Hash, data: &Location) -> Result<Option<Location>, IndexError> {
+    pub fn set(&mut self, key: &K, data: &V) -> Result<Option<V>, IndexError> {
         match self.locate(key) {
             LocateResult::Found(pos) => {
                 let mut old = *data;
@@ -370,7 +376,7 @@ impl Index {
                     entry.data = *data;
                 }
                 loop {
-                    cur_pos = (cur_pos + 1) % self.capacity;
+                    cur_pos = (cur_pos + 1) & self.mask;
                     let entry = &mut self.data[cur_pos];
                     if entry.is_used() {
                         mem::swap(&mut stolen_key, &mut entry.key);
@@ -388,7 +394,7 @@ impl Index {
     }
 
     #[inline]
-    pub fn contains(&self, key: &Hash) -> bool {
+    pub fn contains(&self, key: &K) -> bool {
         debug_assert!(self.check().is_ok(), "Inconsistent before get");
         match self.locate(key) {
             LocateResult::Found(_) => true,
@@ -397,7 +403,7 @@ impl Index {
     }
 
     #[inline]
-    pub fn pos(&self, key: &Hash) -> Option<usize> {
+    pub fn pos(&self, key: &K) -> Option<usize> {
         debug_assert!(self.check().is_ok(), "Inconsistent before get");
         match self.locate(key) {
             LocateResult::Found(pos) => Some(pos),
@@ -406,7 +412,7 @@ impl Index {
     }
 
     #[inline]
-    pub fn get(&self, key: &Hash) -> Option<Location> {
+    pub fn get(&self, key: &K) -> Option<V> {
         debug_assert!(self.check().is_ok(), "Inconsistent before get");
         match self.locate(key) {
             LocateResult::Found(pos) => Some(self.data[pos].data),
@@ -415,7 +421,7 @@ impl Index {
     }
 
     #[inline]
-    pub fn modify<F>(&mut self, key: &Hash, mut f: F) -> bool where F: FnMut(&mut Location) {
+    pub fn modify<F>(&mut self, key: &K, mut f: F) -> bool where F: FnMut(&mut V) {
         debug_assert!(self.check().is_ok(), "Inconsistent before get");
         match self.locate(key) {
             LocateResult::Found(pos) => {
@@ -427,7 +433,7 @@ impl Index {
     }
 
     #[inline]
-    pub fn delete(&mut self, key: &Hash) -> Result<bool, IndexError> {
+    pub fn delete(&mut self, key: &K) -> Result<bool, IndexError> {
         match self.locate(key) {
             LocateResult::Found(pos) => {
                 self.backshift(pos);
@@ -438,7 +444,7 @@ impl Index {
         }
     }
 
-    pub fn filter<F>(&mut self, mut f: F) -> Result<usize, IndexError> where F: FnMut(&Hash, &Location) -> bool {
+    pub fn filter<F>(&mut self, mut f: F) -> Result<usize, IndexError> where F: FnMut(&K, &V) -> bool {
         //TODO: is it faster to walk in reverse direction?
         let mut deleted = 0;
         let mut pos = 0;
@@ -460,48 +466,8 @@ impl Index {
     }
 
     #[inline]
-    pub fn walk<F, E>(&self, mut f: F) -> Result<(), E> where F: FnMut(&Hash, &Location) -> Result<(), E> {
+    pub fn iter<'a>(&'a self) -> Iter<'a, K, V> {
-        for pos in 0..self.capacity {
+        Iter{items: &self.data}
-            let entry = &self.data[pos];
-            if entry.is_used() {
-                try!(f(&entry.key, &entry.data));
-            }
-        }
-        Ok(())
-    }
-
-    #[inline]
-    pub fn walk_mut<F, E>(&mut self, mut f: F) -> Result<(), E> where F: FnMut(&Hash, &mut Location) -> Result<(), E> {
-        for pos in 0..self.capacity {
-            let entry = &mut self.data[pos];
-            if entry.is_used() {
-                try!(f(&entry.key, &mut entry.data));
-            }
-        }
-        Ok(())
-    }
-
-    #[inline]
-    pub fn next_entry(&self, index: usize) -> Option<usize> {
-        let mut i = index;
-        while i < self.capacity && !self.data[i].is_used() {
-            i += 1;
-        }
-        if i == self.capacity {
-            None
-        } else {
-            Some(i)
-        }
-    }
-
-    #[inline]
-    pub fn get_entry(&self, index: usize) -> Option<&Entry> {
-        let entry = &self.data[index];
-        if entry.is_used() {
-            Some(entry)
-        } else {
-            None
-        }
     }
 
     #[inline]

src/main.rs

@@ -4,7 +4,6 @@ extern crate serde;
 extern crate rmp_serde;
 #[macro_use] extern crate serde_utils;
 extern crate squash_sys as squash;
-extern crate mmap;
 extern crate blake2_rfc as blake2;
 extern crate murmurhash3;
 extern crate serde_yaml;
@@ -28,10 +27,11 @@ extern crate pbr;
 extern crate users;
 extern crate libc;
 extern crate tar;
+extern crate index;
 
 pub mod util;
 mod bundledb;
-pub mod index;
+//pub mod index;
 mod chunker;
 mod repository;
 mod cli;

src/prelude.rs

@@ -1,8 +1,8 @@
 pub use ::util::*;
 pub use ::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError, BundleDb, BundleWriterError, StoredBundle};
 pub use ::chunker::{ChunkerType, Chunker, ChunkerStatus, IChunker, ChunkerError};
-pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, IntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError, RepositoryLayout};
+pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, IntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError, RepositoryLayout, Location};
-pub use ::index::{Index, Location, IndexError};
+pub use ::index::{Index, IndexError};
 pub use ::mount::FuseFilesystem;
 
 pub use serde::{Serialize, Deserialize};

src/repository/integrity.rs

@@ -48,31 +48,30 @@ quick_error!{
 
 impl Repository {
     fn check_index_chunks(&self) -> Result<(), RepositoryError> {
-        let mut count = 0;
         let mut progress = ProgressBar::new(self.index.len() as u64);
         progress.message("checking index: ");
         progress.set_max_refresh_rate(Some(Duration::from_millis(100)));
-        let res = self.index.walk(|_hash, location| {
+        for (count,(_hash, location)) in self.index.iter().enumerate() {
             // Lookup bundle id from map
             let bundle_id = try!(self.get_bundle_id(location.bundle));
             // Get bundle object from bundledb
             let bundle = if let Some(bundle) = self.bundles.get_bundle_info(&bundle_id) {
                 bundle
             } else {
+                progress.finish_print("checking index: done.");
                 return Err(IntegrityError::MissingBundle(bundle_id.clone()).into())
             };
             // Get chunk from bundle
             if bundle.info.chunk_count <= location.chunk as usize {
+                progress.finish_print("checking index: done.");
                 return Err(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into())
             }
-            count += 1;
             if count % 1000 == 0 {
-                progress.set(count);
+                progress.set(count as u64);
             }
-            Ok(())
+        }
-        });
         progress.finish_print("checking index: done.");
-        res
+        Ok(())
     }
 
     fn check_chunks(&self, checked: &mut Bitmap, chunks: &[Chunk], mark: bool) -> Result<bool, RepositoryError> {

src/repository/mod.rs

@@ -35,11 +35,41 @@ use self::bundle_map::BundleMap;
 const REPOSITORY_README: &'static [u8] = include_bytes!("../../docs/repository_readme.md");
 const DEFAULT_EXCLUDES: &'static [u8] = include_bytes!("../../docs/excludes.default");
 
+const INDEX_MAGIC: [u8; 7] = *b"zvault\x02";
+const INDEX_VERSION: u8 = 1;
+
+
+#[repr(packed)]
+#[derive(Clone, Copy, PartialEq, Debug)]
+pub struct Location {
+    pub bundle: u32,
+    pub chunk: u32
+}
+impl Location {
+    pub fn new(bundle: u32, chunk: u32) -> Self {
+        Location{ bundle: bundle, chunk: chunk }
+    }
+}
+
+impl ::index::Key for Hash {
+    fn hash(&self) -> u64 {
+        self.low
+    }
+
+    fn is_used(&self) -> bool {
+        self.low != 0 || self.high != 0
+    }
+
+    fn clear(&mut self) {
+        self.low = 0;
+        self.high = 0;
+    }
+}
 
 pub struct Repository {
     pub layout: RepositoryLayout,
     pub config: Config,
-    index: Index,
+    index: Index<Hash, Location>,
     crypto: Arc<Mutex<Crypto>>,
     bundle_map: BundleMap,
     next_data_bundle: u32,
@@ -67,7 +97,7 @@ impl Repository {
         try!(fs::create_dir_all(layout.remote_locks_path()));
         try!(config.save(layout.config_path()));
         try!(BundleDb::create(layout.clone()));
-        try!(Index::create(layout.index_path()));
+        try!(Index::<Hash, Location>::create(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION));
         try!(BundleMap::create().save(layout.bundle_map_path()));
         try!(fs::create_dir_all(layout.backups_path()));
         Self::open(path)
@@ -86,11 +116,11 @@ impl Repository {
         let lock = try!(local_locks.lock(false));
         let crypto = Arc::new(Mutex::new(try!(Crypto::open(layout.keys_path()))));
         let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone()));
-        let (index, mut rebuild_index) = match Index::open(layout.index_path()) {
+        let (index, mut rebuild_index) = match Index::open(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION) {
             Ok(index) => (index, false),
             Err(err) => {
                 error!("Failed to load local index:\n\tcaused by: {}", err);
-                (try!(Index::create(layout.index_path())), true)
+                (try!(Index::create(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION)), true)
             }
         };
         let (bundle_map, rebuild_bundle_map) = match BundleMap::load(layout.bundle_map_path()) {

src/repository/vacuum.rs

@@ -79,12 +79,11 @@ impl Repository {
         }
         try!(self.flush());
         info!("Checking index");
-        self.index.walk::<_, ()>(|hash, location| {
+        for (hash, location) in self.index.iter() {
             if rewrite_bundles.contains(&location.bundle) {
                 panic!("Removed bundle is still referenced in index: hash:{}, bundle:{}, chunk:{}", hash, location.bundle, location.chunk);
             }
-            Ok(())
+        }
-        }).ok();
         info!("Deleting {} bundles", rewrite_bundles.len());
         for id in rewrite_bundles {
             try!(self.delete_bundle(id));