Moved index to separate project (re #17)

This commit is contained in:
Dennis Schwerdel 2017-04-18 13:52:43 +02:00
parent d18cb17281
commit 229c4f7e28
11 changed files with 240 additions and 174 deletions

View File

@ -6,6 +6,7 @@ This project follows [semantic versioning](http://semver.org).
### UNRELEASED ### UNRELEASED
* [added] Ability to read/write tar file from/to stdin/stdout * [added] Ability to read/write tar file from/to stdin/stdout
* [added] Added date to bundles * [added] Added date to bundles
* [added] Option to combine small bundles
* [modified] Logging to stderr * [modified] Logging to stderr
* [modified] Enforce deterministic bundle ordering * [modified] Enforce deterministic bundle ordering
* [modified] More info in analyze subcommand * [modified] More info in analyze subcommand

10
Cargo.lock generated
View File

@ -10,10 +10,10 @@ dependencies = [
"crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"fuse 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "fuse 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"index 0.1.0",
"lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
@ -133,6 +133,14 @@ dependencies = [
"time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "index"
version = "0.1.0"
dependencies = [
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "kernel32-sys" name = "kernel32-sys"
version = "0.2.2" version = "0.2.2"

View File

@ -10,7 +10,6 @@ rmp-serde = "0.12"
serde_yaml = "0.6" serde_yaml = "0.6"
serde_utils = "0.5.1" serde_utils = "0.5.1"
squash-sys = "0.9" squash-sys = "0.9"
mmap = "0.1"
quick-error = "1.1" quick-error = "1.1"
blake2-rfc = "0.2" blake2-rfc = "0.2"
murmurhash3 = "0.0.5" murmurhash3 = "0.0.5"
@ -33,6 +32,7 @@ pbr = "1.0"
users = "0.5" users = "0.5"
time = "*" time = "*"
libc = "*" libc = "*"
index = {path="index"}
[build-dependencies] [build-dependencies]
pkg-config = "0.3" pkg-config = "0.3"

55
index/Cargo.lock generated Normal file
View File

@ -0,0 +1,55 @@
[root]
name = "index"
version = "0.1.0"
dependencies = [
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "libc"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "libc"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "mmap"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "quick-error"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "rand"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tempdir"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
]
[metadata]
"checksum libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122"
"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135"
"checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823"
"checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c"
"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"
"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"

8
index/Cargo.toml Normal file
View File

@ -0,0 +1,8 @@
[package]
name = "index"
version = "0.1.0"
authors = ["Dennis Schwerdel <schwerdel@googlemail.com>"]
[dependencies]
mmap = "0.1"
quick-error = "1.1"

View File

@ -1,4 +1,5 @@
use super::util::Hash; extern crate mmap;
#[macro_use] extern crate quick_error;
use std::path::Path; use std::path::Path;
use std::fs::{File, OpenOptions}; use std::fs::{File, OpenOptions};
@ -10,8 +11,6 @@ use std::os::unix::io::AsRawFd;
use mmap::{MemoryMap, MapOption, MapError}; use mmap::{MemoryMap, MapOption, MapError};
const MAGIC: [u8; 7] = *b"zvault\x02";
const VERSION: u8 = 1;
pub const MAX_USAGE: f64 = 0.9; pub const MAX_USAGE: f64 = 0.9;
pub const MIN_USAGE: f64 = 0.35; pub const MIN_USAGE: f64 = 0.35;
pub const INITIAL_SIZE: usize = 1024; pub const INITIAL_SIZE: usize = 1024;
@ -40,9 +39,9 @@ quick_error!{
description("Unsupported version") description("Unsupported version")
display("Index error: index file has unsupported version: {}", version) display("Index error: index file has unsupported version: {}", version)
} }
WrongPosition(key: Hash, should: usize, is: LocateResult) { WrongPosition(should: usize, is: LocateResult) {
description("Key at wrong position") description("Key at wrong position")
display("Index error: key {} has wrong position, expected at: {}, but is at: {:?}", key, should, is) display("Index error: key has wrong position, expected at: {}, but is at: {:?}", should, is)
} }
WrongEntryCount(header: usize, actual: usize) { WrongEntryCount(header: usize, actual: usize) {
description("Wrong entry count") description("Wrong entry count")
@ -60,49 +59,31 @@ pub struct Header {
capacity: u64, capacity: u64,
} }
#[repr(packed)] pub trait Key: Clone + Eq + Copy {
#[derive(Clone, Copy, PartialEq, Debug)] fn hash(&self) -> u64;
pub struct Location { fn is_used(&self) -> bool;
pub bundle: u32, fn clear(&mut self);
pub chunk: u32
} }
impl Location {
pub fn new(bundle: u32, chunk: u32) -> Self {
Location{ bundle: bundle, chunk: chunk }
}
}
#[repr(packed)] #[repr(packed)]
#[derive(Clone)] #[derive(Clone)]
pub struct Entry { pub struct Entry<K, V> {
pub key: Hash, pub key: K,
pub data: Location pub data: V
} }
impl Entry { impl<K: Key, V> Entry<K, V> {
#[inline] #[inline]
fn is_used(&self) -> bool { fn is_used(&self) -> bool {
self.key.low != 0 || self.key.high != 0 self.key.is_used()
} }
#[inline]
fn clear(&mut self) { fn clear(&mut self) {
self.key.low = 0; self.key.clear()
self.key.high = 0;
} }
} }
pub struct Index {
capacity: usize,
entries: usize,
max_entries: usize,
min_entries: usize,
fd: File,
mmap: MemoryMap,
data: &'static mut [Entry]
}
#[derive(Debug)] #[derive(Debug)]
pub enum LocateResult { pub enum LocateResult {
Found(usize), // Found the key at this position Found(usize), // Found the key at this position
@ -110,55 +91,94 @@ pub enum LocateResult {
Steal(usize) // Found a spot to steal at this position while searching for a key Steal(usize) // Found a spot to steal at this position while searching for a key
} }
impl Index {
pub fn new(path: &Path, create: bool) -> Result<Index, IndexError> { pub struct Iter<'a, K: 'static, V: 'static> {
items: &'a [Entry<K, V>],
}
impl<'a, K: Key, V> Iterator for Iter<'a, K, V> {
type Item = (&'a K, &'a V);
fn next(&mut self) -> Option<Self::Item> {
while let Some((first, rest)) = self.items.split_first() {
self.items = rest;
if first.is_used() {
return Some((&first.key, &first.data));
}
}
None
}
}
fn mmap_as_ref<K, V>(mmap: &MemoryMap, len: usize) -> (&'static mut Header, &'static mut [Entry<K, V>]) {
if mmap.len() < mem::size_of::<Header>() + len * mem::size_of::<Entry<K, V>>() {
panic!("Memory map too small");
}
let header = unsafe { &mut *(mmap.data() as *mut Header) };
let ptr = unsafe { mmap.data().offset(mem::size_of::<Header>() as isize) as *mut Entry<K, V> };
let data = unsafe { slice::from_raw_parts_mut(ptr, len) };
(header, data)
}
pub struct Index<K: 'static, V: 'static> {
capacity: usize,
mask: usize,
entries: usize,
max_entries: usize,
min_entries: usize,
fd: File,
mmap: MemoryMap,
header: &'static mut Header,
data: &'static mut [Entry<K, V>]
}
impl<K: Key, V: Clone + Copy> Index<K, V> {
pub fn new(path: &Path, create: bool, magic: &[u8; 7], version: u8) -> Result<Self, IndexError> {
let fd = try!(OpenOptions::new().read(true).write(true).create(create).open(path)); let fd = try!(OpenOptions::new().read(true).write(true).create(create).open(path));
if create { if create {
try!(Index::resize_fd(&fd, INITIAL_SIZE)); try!(Self::resize_fd(&fd, INITIAL_SIZE));
} }
let mmap = try!(Index::map_fd(&fd)); let mmap = try!(Self::map_fd(&fd));
if mmap.len() < mem::size_of::<Header>() { if mmap.len() < mem::size_of::<Header>() {
return Err(IndexError::WrongMagic); return Err(IndexError::WrongMagic);
} }
let data = Index::mmap_as_slice(&mmap, INITIAL_SIZE as usize); let (header, _) = mmap_as_ref::<K, V>(&mmap, INITIAL_SIZE as usize);
let mut index = Index{capacity: 0, max_entries: 0, min_entries: 0, entries: 0, fd: fd, mmap: mmap, data: data}; if create {
{ header.magic = magic.to_owned();
let capacity; header.version = version;
let entries; header.entries = 0;
{ header.capacity = INITIAL_SIZE as u64;
let header = index.header();
if create {
header.magic = MAGIC;
header.version = VERSION;
header.entries = 0;
header.capacity = INITIAL_SIZE as u64;
} else {
if header.magic != MAGIC {
return Err(IndexError::WrongMagic);
}
if header.version != VERSION {
return Err(IndexError::UnsupportedVersion(header.version));
}
}
capacity = header.capacity;
entries = header.entries;
}
index.data = Index::mmap_as_slice(&index.mmap, capacity as usize);
index.set_capacity(capacity as usize);
index.entries = entries as usize;
} }
if &header.magic != magic {
return Err(IndexError::WrongMagic);
}
if header.version != version {
return Err(IndexError::UnsupportedVersion(header.version));
}
let (header, data) = mmap_as_ref(&mmap, header.capacity as usize);
let index = Index{
capacity: header.capacity as usize,
mask: header.capacity as usize -1,
max_entries: (header.capacity as f64 * MAX_USAGE) as usize,
min_entries: (header.capacity as f64 * MIN_USAGE) as usize,
entries: header.entries as usize,
fd: fd,
mmap: mmap,
data: data,
header: header
};
debug_assert!(index.check().is_ok(), "Inconsistent after creation"); debug_assert!(index.check().is_ok(), "Inconsistent after creation");
Ok(index) Ok(index)
} }
#[inline] #[inline]
pub fn open<P: AsRef<Path>>(path: P) -> Result<Index, IndexError> { pub fn open<P: AsRef<Path>>(path: P, magic: &[u8; 7], version: u8) -> Result<Self, IndexError> {
Index::new(path.as_ref(), false) Index::new(path.as_ref(), false, magic, version)
} }
#[inline] #[inline]
pub fn create<P: AsRef<Path>>(path: P) -> Result<Index, IndexError> { pub fn create<P: AsRef<Path>>(path: P, magic: &[u8; 7], version: u8) -> Result<Self, IndexError> {
Index::new(path.as_ref(), true) Index::new(path.as_ref(), true, magic, version)
} }
#[inline] #[inline]
@ -174,29 +194,14 @@ impl Index {
#[inline] #[inline]
fn resize_fd(fd: &File, capacity: usize) -> Result<(), IndexError> { fn resize_fd(fd: &File, capacity: usize) -> Result<(), IndexError> {
fd.set_len((mem::size_of::<Header>() + capacity * mem::size_of::<Entry>()) as u64).map_err(IndexError::Io) fd.set_len((mem::size_of::<Header>() + capacity * mem::size_of::<Entry<K, V>>()) as u64).map_err(IndexError::Io)
}
#[inline]
fn mmap_as_slice(mmap: &MemoryMap, len: usize) -> &'static mut [Entry] {
if mmap.len() < mem::size_of::<Header>() + len * mem::size_of::<Entry>() {
panic!("Memory map too small");
}
let ptr = unsafe { mmap.data().offset(mem::size_of::<Header>() as isize) as *mut Entry };
unsafe { slice::from_raw_parts_mut(ptr, len) }
}
#[inline]
fn header(&mut self) -> &mut Header {
if self.mmap.len() < mem::size_of::<Header>() {
panic!("Failed to read beyond end");
}
unsafe { &mut *(self.mmap.data() as *mut Header) }
} }
#[inline] #[inline]
fn set_capacity(&mut self, capacity: usize) { fn set_capacity(&mut self, capacity: usize) {
self.capacity = capacity; self.capacity = capacity;
debug_assert_eq!(capacity.count_ones(), 1);
self.mask = capacity -1;
self.min_entries = (capacity as f64 * MIN_USAGE) as usize; self.min_entries = (capacity as f64 * MIN_USAGE) as usize;
self.max_entries = (capacity as f64 * MAX_USAGE) as usize; self.max_entries = (capacity as f64 * MAX_USAGE) as usize;
} }
@ -228,9 +233,11 @@ impl Index {
let new_capacity = self.capacity / 2; let new_capacity = self.capacity / 2;
self.set_capacity(new_capacity); self.set_capacity(new_capacity);
try!(self.reinsert(new_capacity, old_capacity)); try!(self.reinsert(new_capacity, old_capacity));
try!(Index::resize_fd(&self.fd, new_capacity)); try!(Self::resize_fd(&self.fd, new_capacity));
self.mmap = try!(Index::map_fd(&self.fd)); self.mmap = try!(Self::map_fd(&self.fd));
self.data = Index::mmap_as_slice(&self.mmap, new_capacity); let (header, data) = mmap_as_ref(&self.mmap, new_capacity);
self.header = header;
self.data = data;
assert_eq!(self.data.len(), self.capacity); assert_eq!(self.data.len(), self.capacity);
Ok(true) Ok(true)
} }
@ -240,9 +247,11 @@ impl Index {
return Ok(false) return Ok(false)
} }
let new_capacity = 2 * self.capacity; let new_capacity = 2 * self.capacity;
try!(Index::resize_fd(&self.fd, new_capacity)); try!(Self::resize_fd(&self.fd, new_capacity));
self.mmap = try!(Index::map_fd(&self.fd)); self.mmap = try!(Self::map_fd(&self.fd));
self.data = Index::mmap_as_slice(&self.mmap, new_capacity); let (header, data) = mmap_as_ref(&self.mmap, new_capacity);
self.header = header;
self.data = data;
self.set_capacity(new_capacity); self.set_capacity(new_capacity);
assert_eq!(self.data.len(), self.capacity); assert_eq!(self.data.len(), self.capacity);
try!(self.reinsert(0, new_capacity)); try!(self.reinsert(0, new_capacity));
@ -259,7 +268,7 @@ impl Index {
entries += 1; entries += 1;
match self.locate(&entry.key) { match self.locate(&entry.key) {
LocateResult::Found(p) if p == pos => true, LocateResult::Found(p) if p == pos => true,
found => return Err(IndexError::WrongPosition(entry.key, pos, found)) found => return Err(IndexError::WrongPosition(pos, found))
}; };
} }
if entries != self.entries { if entries != self.entries {
@ -286,18 +295,15 @@ impl Index {
#[inline] #[inline]
fn write_header(&mut self) { fn write_header(&mut self) {
let entries = self.entries; self.header.entries = self.entries as u64;
let capacity = self.capacity; self.header.capacity = self.capacity as u64;
let header = self.header();
header.entries = entries as u64;
header.capacity = capacity as u64;
} }
/// Finds the position for this key /// Finds the position for this key
/// If the key is in the table, it will be the position of the key, /// If the key is in the table, it will be the position of the key,
/// otherwise it will be the position where this key should be inserted /// otherwise it will be the position where this key should be inserted
fn locate(&self, key: &Hash) -> LocateResult { fn locate(&self, key: &K) -> LocateResult {
let mut pos = key.hash() as usize % self.capacity; let mut pos = key.hash() as usize & self.mask;
let mut dist = 0; let mut dist = 0;
loop { loop {
let entry = &self.data[pos]; let entry = &self.data[pos];
@ -307,11 +313,11 @@ impl Index {
if entry.key == *key { if entry.key == *key {
return LocateResult::Found(pos); return LocateResult::Found(pos);
} }
let odist = (pos + self.capacity - entry.key.hash() as usize % self.capacity) % self.capacity; let odist = (pos + self.capacity - entry.key.hash() as usize & self.mask) & self.mask;
if dist > odist { if dist > odist {
return LocateResult::Steal(pos); return LocateResult::Steal(pos);
} }
pos = (pos + 1) % self.capacity ; pos = (pos + 1) & self.mask;
dist += 1; dist += 1;
} }
} }
@ -323,14 +329,14 @@ impl Index {
let mut last_pos; let mut last_pos;
loop { loop {
last_pos = pos; last_pos = pos;
pos = (pos + 1) % self.capacity; pos = (pos + 1) & self.mask;
{ {
let entry = &self.data[pos]; let entry = &self.data[pos];
if !entry.is_used() { if !entry.is_used() {
// we found a hole, stop shifting here // we found a hole, stop shifting here
break; break;
} }
if entry.key.hash() as usize % self.capacity == pos { if entry.key.hash() as usize & self.mask == pos {
// we found an entry at the right position, stop shifting here // we found an entry at the right position, stop shifting here
break; break;
} }
@ -342,7 +348,7 @@ impl Index {
/// Adds the key, data pair into the table. /// Adds the key, data pair into the table.
/// If the key existed the old data is returned. /// If the key existed the old data is returned.
pub fn set(&mut self, key: &Hash, data: &Location) -> Result<Option<Location>, IndexError> { pub fn set(&mut self, key: &K, data: &V) -> Result<Option<V>, IndexError> {
match self.locate(key) { match self.locate(key) {
LocateResult::Found(pos) => { LocateResult::Found(pos) => {
let mut old = *data; let mut old = *data;
@ -370,7 +376,7 @@ impl Index {
entry.data = *data; entry.data = *data;
} }
loop { loop {
cur_pos = (cur_pos + 1) % self.capacity; cur_pos = (cur_pos + 1) & self.mask;
let entry = &mut self.data[cur_pos]; let entry = &mut self.data[cur_pos];
if entry.is_used() { if entry.is_used() {
mem::swap(&mut stolen_key, &mut entry.key); mem::swap(&mut stolen_key, &mut entry.key);
@ -388,7 +394,7 @@ impl Index {
} }
#[inline] #[inline]
pub fn contains(&self, key: &Hash) -> bool { pub fn contains(&self, key: &K) -> bool {
debug_assert!(self.check().is_ok(), "Inconsistent before get"); debug_assert!(self.check().is_ok(), "Inconsistent before get");
match self.locate(key) { match self.locate(key) {
LocateResult::Found(_) => true, LocateResult::Found(_) => true,
@ -397,7 +403,7 @@ impl Index {
} }
#[inline] #[inline]
pub fn pos(&self, key: &Hash) -> Option<usize> { pub fn pos(&self, key: &K) -> Option<usize> {
debug_assert!(self.check().is_ok(), "Inconsistent before get"); debug_assert!(self.check().is_ok(), "Inconsistent before get");
match self.locate(key) { match self.locate(key) {
LocateResult::Found(pos) => Some(pos), LocateResult::Found(pos) => Some(pos),
@ -406,7 +412,7 @@ impl Index {
} }
#[inline] #[inline]
pub fn get(&self, key: &Hash) -> Option<Location> { pub fn get(&self, key: &K) -> Option<V> {
debug_assert!(self.check().is_ok(), "Inconsistent before get"); debug_assert!(self.check().is_ok(), "Inconsistent before get");
match self.locate(key) { match self.locate(key) {
LocateResult::Found(pos) => Some(self.data[pos].data), LocateResult::Found(pos) => Some(self.data[pos].data),
@ -415,7 +421,7 @@ impl Index {
} }
#[inline] #[inline]
pub fn modify<F>(&mut self, key: &Hash, mut f: F) -> bool where F: FnMut(&mut Location) { pub fn modify<F>(&mut self, key: &K, mut f: F) -> bool where F: FnMut(&mut V) {
debug_assert!(self.check().is_ok(), "Inconsistent before get"); debug_assert!(self.check().is_ok(), "Inconsistent before get");
match self.locate(key) { match self.locate(key) {
LocateResult::Found(pos) => { LocateResult::Found(pos) => {
@ -427,7 +433,7 @@ impl Index {
} }
#[inline] #[inline]
pub fn delete(&mut self, key: &Hash) -> Result<bool, IndexError> { pub fn delete(&mut self, key: &K) -> Result<bool, IndexError> {
match self.locate(key) { match self.locate(key) {
LocateResult::Found(pos) => { LocateResult::Found(pos) => {
self.backshift(pos); self.backshift(pos);
@ -438,7 +444,7 @@ impl Index {
} }
} }
pub fn filter<F>(&mut self, mut f: F) -> Result<usize, IndexError> where F: FnMut(&Hash, &Location) -> bool { pub fn filter<F>(&mut self, mut f: F) -> Result<usize, IndexError> where F: FnMut(&K, &V) -> bool {
//TODO: is it faster to walk in reverse direction? //TODO: is it faster to walk in reverse direction?
let mut deleted = 0; let mut deleted = 0;
let mut pos = 0; let mut pos = 0;
@ -460,48 +466,8 @@ impl Index {
} }
#[inline] #[inline]
pub fn walk<F, E>(&self, mut f: F) -> Result<(), E> where F: FnMut(&Hash, &Location) -> Result<(), E> { pub fn iter<'a>(&'a self) -> Iter<'a, K, V> {
for pos in 0..self.capacity { Iter{items: &self.data}
let entry = &self.data[pos];
if entry.is_used() {
try!(f(&entry.key, &entry.data));
}
}
Ok(())
}
#[inline]
pub fn walk_mut<F, E>(&mut self, mut f: F) -> Result<(), E> where F: FnMut(&Hash, &mut Location) -> Result<(), E> {
for pos in 0..self.capacity {
let entry = &mut self.data[pos];
if entry.is_used() {
try!(f(&entry.key, &mut entry.data));
}
}
Ok(())
}
#[inline]
pub fn next_entry(&self, index: usize) -> Option<usize> {
let mut i = index;
while i < self.capacity && !self.data[i].is_used() {
i += 1;
}
if i == self.capacity {
None
} else {
Some(i)
}
}
#[inline]
pub fn get_entry(&self, index: usize) -> Option<&Entry> {
let entry = &self.data[index];
if entry.is_used() {
Some(entry)
} else {
None
}
} }
#[inline] #[inline]

View File

@ -4,7 +4,6 @@ extern crate serde;
extern crate rmp_serde; extern crate rmp_serde;
#[macro_use] extern crate serde_utils; #[macro_use] extern crate serde_utils;
extern crate squash_sys as squash; extern crate squash_sys as squash;
extern crate mmap;
extern crate blake2_rfc as blake2; extern crate blake2_rfc as blake2;
extern crate murmurhash3; extern crate murmurhash3;
extern crate serde_yaml; extern crate serde_yaml;
@ -28,10 +27,11 @@ extern crate pbr;
extern crate users; extern crate users;
extern crate libc; extern crate libc;
extern crate tar; extern crate tar;
extern crate index;
pub mod util; pub mod util;
mod bundledb; mod bundledb;
pub mod index; //pub mod index;
mod chunker; mod chunker;
mod repository; mod repository;
mod cli; mod cli;

View File

@ -1,8 +1,8 @@
pub use ::util::*; pub use ::util::*;
pub use ::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError, BundleDb, BundleWriterError, StoredBundle}; pub use ::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError, BundleDb, BundleWriterError, StoredBundle};
pub use ::chunker::{ChunkerType, Chunker, ChunkerStatus, IChunker, ChunkerError}; pub use ::chunker::{ChunkerType, Chunker, ChunkerStatus, IChunker, ChunkerError};
pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, IntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError, RepositoryLayout}; pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, IntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError, RepositoryLayout, Location};
pub use ::index::{Index, Location, IndexError}; pub use ::index::{Index, IndexError};
pub use ::mount::FuseFilesystem; pub use ::mount::FuseFilesystem;
pub use serde::{Serialize, Deserialize}; pub use serde::{Serialize, Deserialize};

View File

@ -48,31 +48,30 @@ quick_error!{
impl Repository { impl Repository {
fn check_index_chunks(&self) -> Result<(), RepositoryError> { fn check_index_chunks(&self) -> Result<(), RepositoryError> {
let mut count = 0;
let mut progress = ProgressBar::new(self.index.len() as u64); let mut progress = ProgressBar::new(self.index.len() as u64);
progress.message("checking index: "); progress.message("checking index: ");
progress.set_max_refresh_rate(Some(Duration::from_millis(100))); progress.set_max_refresh_rate(Some(Duration::from_millis(100)));
let res = self.index.walk(|_hash, location| { for (count,(_hash, location)) in self.index.iter().enumerate() {
// Lookup bundle id from map // Lookup bundle id from map
let bundle_id = try!(self.get_bundle_id(location.bundle)); let bundle_id = try!(self.get_bundle_id(location.bundle));
// Get bundle object from bundledb // Get bundle object from bundledb
let bundle = if let Some(bundle) = self.bundles.get_bundle_info(&bundle_id) { let bundle = if let Some(bundle) = self.bundles.get_bundle_info(&bundle_id) {
bundle bundle
} else { } else {
progress.finish_print("checking index: done.");
return Err(IntegrityError::MissingBundle(bundle_id.clone()).into()) return Err(IntegrityError::MissingBundle(bundle_id.clone()).into())
}; };
// Get chunk from bundle // Get chunk from bundle
if bundle.info.chunk_count <= location.chunk as usize { if bundle.info.chunk_count <= location.chunk as usize {
progress.finish_print("checking index: done.");
return Err(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into()) return Err(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into())
} }
count += 1;
if count % 1000 == 0 { if count % 1000 == 0 {
progress.set(count); progress.set(count as u64);
} }
Ok(()) }
});
progress.finish_print("checking index: done."); progress.finish_print("checking index: done.");
res Ok(())
} }
fn check_chunks(&self, checked: &mut Bitmap, chunks: &[Chunk], mark: bool) -> Result<bool, RepositoryError> { fn check_chunks(&self, checked: &mut Bitmap, chunks: &[Chunk], mark: bool) -> Result<bool, RepositoryError> {

View File

@ -35,11 +35,41 @@ use self::bundle_map::BundleMap;
const REPOSITORY_README: &'static [u8] = include_bytes!("../../docs/repository_readme.md"); const REPOSITORY_README: &'static [u8] = include_bytes!("../../docs/repository_readme.md");
const DEFAULT_EXCLUDES: &'static [u8] = include_bytes!("../../docs/excludes.default"); const DEFAULT_EXCLUDES: &'static [u8] = include_bytes!("../../docs/excludes.default");
const INDEX_MAGIC: [u8; 7] = *b"zvault\x02";
const INDEX_VERSION: u8 = 1;
#[repr(packed)]
#[derive(Clone, Copy, PartialEq, Debug)]
pub struct Location {
pub bundle: u32,
pub chunk: u32
}
impl Location {
pub fn new(bundle: u32, chunk: u32) -> Self {
Location{ bundle: bundle, chunk: chunk }
}
}
impl ::index::Key for Hash {
fn hash(&self) -> u64 {
self.low
}
fn is_used(&self) -> bool {
self.low != 0 || self.high != 0
}
fn clear(&mut self) {
self.low = 0;
self.high = 0;
}
}
pub struct Repository { pub struct Repository {
pub layout: RepositoryLayout, pub layout: RepositoryLayout,
pub config: Config, pub config: Config,
index: Index, index: Index<Hash, Location>,
crypto: Arc<Mutex<Crypto>>, crypto: Arc<Mutex<Crypto>>,
bundle_map: BundleMap, bundle_map: BundleMap,
next_data_bundle: u32, next_data_bundle: u32,
@ -67,7 +97,7 @@ impl Repository {
try!(fs::create_dir_all(layout.remote_locks_path())); try!(fs::create_dir_all(layout.remote_locks_path()));
try!(config.save(layout.config_path())); try!(config.save(layout.config_path()));
try!(BundleDb::create(layout.clone())); try!(BundleDb::create(layout.clone()));
try!(Index::create(layout.index_path())); try!(Index::<Hash, Location>::create(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION));
try!(BundleMap::create().save(layout.bundle_map_path())); try!(BundleMap::create().save(layout.bundle_map_path()));
try!(fs::create_dir_all(layout.backups_path())); try!(fs::create_dir_all(layout.backups_path()));
Self::open(path) Self::open(path)
@ -86,11 +116,11 @@ impl Repository {
let lock = try!(local_locks.lock(false)); let lock = try!(local_locks.lock(false));
let crypto = Arc::new(Mutex::new(try!(Crypto::open(layout.keys_path())))); let crypto = Arc::new(Mutex::new(try!(Crypto::open(layout.keys_path()))));
let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone())); let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone()));
let (index, mut rebuild_index) = match Index::open(layout.index_path()) { let (index, mut rebuild_index) = match Index::open(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION) {
Ok(index) => (index, false), Ok(index) => (index, false),
Err(err) => { Err(err) => {
error!("Failed to load local index:\n\tcaused by: {}", err); error!("Failed to load local index:\n\tcaused by: {}", err);
(try!(Index::create(layout.index_path())), true) (try!(Index::create(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION)), true)
} }
}; };
let (bundle_map, rebuild_bundle_map) = match BundleMap::load(layout.bundle_map_path()) { let (bundle_map, rebuild_bundle_map) = match BundleMap::load(layout.bundle_map_path()) {

View File

@ -79,12 +79,11 @@ impl Repository {
} }
try!(self.flush()); try!(self.flush());
info!("Checking index"); info!("Checking index");
self.index.walk::<_, ()>(|hash, location| { for (hash, location) in self.index.iter() {
if rewrite_bundles.contains(&location.bundle) { if rewrite_bundles.contains(&location.bundle) {
panic!("Removed bundle is still referenced in index: hash:{}, bundle:{}, chunk:{}", hash, location.bundle, location.chunk); panic!("Removed bundle is still referenced in index: hash:{}, bundle:{}, chunk:{}", hash, location.bundle, location.chunk);
} }
Ok(()) }
}).ok();
info!("Deleting {} bundles", rewrite_bundles.len()); info!("Deleting {} bundles", rewrite_bundles.len());
for id in rewrite_bundles { for id in rewrite_bundles {
try!(self.delete_bundle(id)); try!(self.delete_bundle(id));