Browse Source

Moved index to separate project (re #17)

pull/10/head
Dennis Schwerdel 5 years ago
parent
commit
229c4f7e28
  1. 1
      CHANGELOG.md
  2. 10
      Cargo.lock
  3. 2
      Cargo.toml
  4. 55
      index/Cargo.lock
  5. 8
      index/Cargo.toml
  6. 272
      index/src/lib.rs
  7. 4
      src/main.rs
  8. 4
      src/prelude.rs
  9. 13
      src/repository/integrity.rs
  10. 38
      src/repository/mod.rs
  11. 5
      src/repository/vacuum.rs

1
CHANGELOG.md

@ -6,6 +6,7 @@ This project follows [semantic versioning](http://semver.org).
### UNRELEASED
* [added] Ability to read/write tar file from/to stdin/stdout
* [added] Added date to bundles
* [added] Option to combine small bundles
* [modified] Logging to stderr
* [modified] Enforce deterministic bundle ordering
* [modified] More info in analyze subcommand

10
Cargo.lock generated

@ -10,10 +10,10 @@ dependencies = [
"crossbeam 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"filetime 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"fuse 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"index 0.1.0",
"lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)",
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
@ -133,6 +133,14 @@ dependencies = [
"time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "index"
version = "0.1.0"
dependencies = [
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "kernel32-sys"
version = "0.2.2"

2
Cargo.toml

@ -10,7 +10,6 @@ rmp-serde = "0.12"
serde_yaml = "0.6"
serde_utils = "0.5.1"
squash-sys = "0.9"
mmap = "0.1"
quick-error = "1.1"
blake2-rfc = "0.2"
murmurhash3 = "0.0.5"
@ -33,6 +32,7 @@ pbr = "1.0"
users = "0.5"
time = "*"
libc = "*"
index = {path="index"}
[build-dependencies]
pkg-config = "0.3"

55
index/Cargo.lock generated

@ -0,0 +1,55 @@
[root]
name = "index"
version = "0.1.0"
dependencies = [
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "libc"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "libc"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "mmap"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
"tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "quick-error"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "rand"
version = "0.3.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "tempdir"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
]
[metadata]
"checksum libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122"
"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135"
"checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823"
"checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c"
"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"
"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"

8
index/Cargo.toml

@ -0,0 +1,8 @@
[package]
name = "index"
version = "0.1.0"
authors = ["Dennis Schwerdel <schwerdel@googlemail.com>"]
[dependencies]
mmap = "0.1"
quick-error = "1.1"

272
src/index.rs → index/src/lib.rs

@ -1,4 +1,5 @@
use super::util::Hash;
extern crate mmap;
#[macro_use] extern crate quick_error;
use std::path::Path;
use std::fs::{File, OpenOptions};
@ -10,8 +11,6 @@ use std::os::unix::io::AsRawFd;
use mmap::{MemoryMap, MapOption, MapError};
const MAGIC: [u8; 7] = *b"zvault\x02";
const VERSION: u8 = 1;
pub const MAX_USAGE: f64 = 0.9;
pub const MIN_USAGE: f64 = 0.35;
pub const INITIAL_SIZE: usize = 1024;
@ -40,9 +39,9 @@ quick_error!{
description("Unsupported version")
display("Index error: index file has unsupported version: {}", version)
}
WrongPosition(key: Hash, should: usize, is: LocateResult) {
WrongPosition(should: usize, is: LocateResult) {
description("Key at wrong position")
display("Index error: key {} has wrong position, expected at: {}, but is at: {:?}", key, should, is)
display("Index error: key has wrong position, expected at: {}, but is at: {:?}", should, is)
}
WrongEntryCount(header: usize, actual: usize) {
description("Wrong entry count")
@ -60,105 +59,126 @@ pub struct Header {
capacity: u64,
}
#[repr(packed)]
#[derive(Clone, Copy, PartialEq, Debug)]
pub struct Location {
pub bundle: u32,
pub chunk: u32
}
impl Location {
pub fn new(bundle: u32, chunk: u32) -> Self {
Location{ bundle: bundle, chunk: chunk }
}
pub trait Key: Clone + Eq + Copy {
fn hash(&self) -> u64;
fn is_used(&self) -> bool;
fn clear(&mut self);
}
#[repr(packed)]
#[derive(Clone)]
pub struct Entry {
pub key: Hash,
pub data: Location
pub struct Entry<K, V> {
pub key: K,
pub data: V
}
impl Entry {
impl<K: Key, V> Entry<K, V> {
#[inline]
fn is_used(&self) -> bool {
self.key.low != 0 || self.key.high != 0
self.key.is_used()
}
#[inline]
fn clear(&mut self) {
self.key.low = 0;
self.key.high = 0;
self.key.clear()
}
}
pub struct Index {
#[derive(Debug)]
pub enum LocateResult {
Found(usize), // Found the key at this position
Hole(usize), // Found a hole at this position while searching for a key
Steal(usize) // Found a spot to steal at this position while searching for a key
}
pub struct Iter<'a, K: 'static, V: 'static> {
items: &'a [Entry<K, V>],
}
impl<'a, K: Key, V> Iterator for Iter<'a, K, V> {
type Item = (&'a K, &'a V);
fn next(&mut self) -> Option<Self::Item> {
while let Some((first, rest)) = self.items.split_first() {
self.items = rest;
if first.is_used() {
return Some((&first.key, &first.data));
}
}
None
}
}
fn mmap_as_ref<K, V>(mmap: &MemoryMap, len: usize) -> (&'static mut Header, &'static mut [Entry<K, V>]) {
if mmap.len() < mem::size_of::<Header>() + len * mem::size_of::<Entry<K, V>>() {
panic!("Memory map too small");
}
let header = unsafe { &mut *(mmap.data() as *mut Header) };
let ptr = unsafe { mmap.data().offset(mem::size_of::<Header>() as isize) as *mut Entry<K, V> };
let data = unsafe { slice::from_raw_parts_mut(ptr, len) };
(header, data)
}
pub struct Index<K: 'static, V: 'static> {
capacity: usize,
mask: usize,
entries: usize,
max_entries: usize,
min_entries: usize,
fd: File,
mmap: MemoryMap,
data: &'static mut [Entry]
header: &'static mut Header,
data: &'static mut [Entry<K, V>]
}
#[derive(Debug)]
pub enum LocateResult {
Found(usize), // Found the key at this position
Hole(usize), // Found a hole at this position while searching for a key
Steal(usize) // Found a spot to steal at this position while searching for a key
}
impl Index {
pub fn new(path: &Path, create: bool) -> Result<Index, IndexError> {
impl<K: Key, V: Clone + Copy> Index<K, V> {
pub fn new(path: &Path, create: bool, magic: &[u8; 7], version: u8) -> Result<Self, IndexError> {
let fd = try!(OpenOptions::new().read(true).write(true).create(create).open(path));
if create {
try!(Index::resize_fd(&fd, INITIAL_SIZE));
try!(Self::resize_fd(&fd, INITIAL_SIZE));
}
let mmap = try!(Index::map_fd(&fd));
let mmap = try!(Self::map_fd(&fd));
if mmap.len() < mem::size_of::<Header>() {
return Err(IndexError::WrongMagic);
}
let data = Index::mmap_as_slice(&mmap, INITIAL_SIZE as usize);
let mut index = Index{capacity: 0, max_entries: 0, min_entries: 0, entries: 0, fd: fd, mmap: mmap, data: data};
{
let capacity;
let entries;
{
let header = index.header();
if create {
header.magic = MAGIC;
header.version = VERSION;
header.entries = 0;
header.capacity = INITIAL_SIZE as u64;
} else {
if header.magic != MAGIC {
return Err(IndexError::WrongMagic);
}
if header.version != VERSION {
return Err(IndexError::UnsupportedVersion(header.version));
}
}
capacity = header.capacity;
entries = header.entries;
}
index.data = Index::mmap_as_slice(&index.mmap, capacity as usize);
index.set_capacity(capacity as usize);
index.entries = entries as usize;
let (header, _) = mmap_as_ref::<K, V>(&mmap, INITIAL_SIZE as usize);
if create {
header.magic = magic.to_owned();
header.version = version;
header.entries = 0;
header.capacity = INITIAL_SIZE as u64;
}
if &header.magic != magic {
return Err(IndexError::WrongMagic);
}
if header.version != version {
return Err(IndexError::UnsupportedVersion(header.version));
}
let (header, data) = mmap_as_ref(&mmap, header.capacity as usize);
let index = Index{
capacity: header.capacity as usize,
mask: header.capacity as usize -1,
max_entries: (header.capacity as f64 * MAX_USAGE) as usize,
min_entries: (header.capacity as f64 * MIN_USAGE) as usize,
entries: header.entries as usize,
fd: fd,
mmap: mmap,
data: data,
header: header
};
debug_assert!(index.check().is_ok(), "Inconsistent after creation");
Ok(index)
}
#[inline]
pub fn open<P: AsRef<Path>>(path: P) -> Result<Index, IndexError> {
Index::new(path.as_ref(), false)
pub fn open<P: AsRef<Path>>(path: P, magic: &[u8; 7], version: u8) -> Result<Self, IndexError> {
Index::new(path.as_ref(), false, magic, version)
}
#[inline]
pub fn create<P: AsRef<Path>>(path: P) -> Result<Index, IndexError> {
Index::new(path.as_ref(), true)
pub fn create<P: AsRef<Path>>(path: P, magic: &[u8; 7], version: u8) -> Result<Self, IndexError> {
Index::new(path.as_ref(), true, magic, version)
}
#[inline]
@ -174,29 +194,14 @@ impl Index {
#[inline]
fn resize_fd(fd: &File, capacity: usize) -> Result<(), IndexError> {
fd.set_len((mem::size_of::<Header>() + capacity * mem::size_of::<Entry>()) as u64).map_err(IndexError::Io)
}
#[inline]
fn mmap_as_slice(mmap: &MemoryMap, len: usize) -> &'static mut [Entry] {
if mmap.len() < mem::size_of::<Header>() + len * mem::size_of::<Entry>() {
panic!("Memory map too small");
}
let ptr = unsafe { mmap.data().offset(mem::size_of::<Header>() as isize) as *mut Entry };
unsafe { slice::from_raw_parts_mut(ptr, len) }
}
#[inline]
fn header(&mut self) -> &mut Header {
if self.mmap.len() < mem::size_of::<Header>() {
panic!("Failed to read beyond end");
}
unsafe { &mut *(self.mmap.data() as *mut Header) }
fd.set_len((mem::size_of::<Header>() + capacity * mem::size_of::<Entry<K, V>>()) as u64).map_err(IndexError::Io)
}
#[inline]
fn set_capacity(&mut self, capacity: usize) {
self.capacity = capacity;
debug_assert_eq!(capacity.count_ones(), 1);
self.mask = capacity -1;
self.min_entries = (capacity as f64 * MIN_USAGE) as usize;
self.max_entries = (capacity as f64 * MAX_USAGE) as usize;
}
@ -228,9 +233,11 @@ impl Index {
let new_capacity = self.capacity / 2;
self.set_capacity(new_capacity);
try!(self.reinsert(new_capacity, old_capacity));
try!(Index::resize_fd(&self.fd, new_capacity));
self.mmap = try!(Index::map_fd(&self.fd));
self.data = Index::mmap_as_slice(&self.mmap, new_capacity);
try!(Self::resize_fd(&self.fd, new_capacity));
self.mmap = try!(Self::map_fd(&self.fd));
let (header, data) = mmap_as_ref(&self.mmap, new_capacity);
self.header = header;
self.data = data;
assert_eq!(self.data.len(), self.capacity);
Ok(true)
}
@ -240,9 +247,11 @@ impl Index {
return Ok(false)
}
let new_capacity = 2 * self.capacity;
try!(Index::resize_fd(&self.fd, new_capacity));
self.mmap = try!(Index::map_fd(&self.fd));
self.data = Index::mmap_as_slice(&self.mmap, new_capacity);
try!(Self::resize_fd(&self.fd, new_capacity));
self.mmap = try!(Self::map_fd(&self.fd));
let (header, data) = mmap_as_ref(&self.mmap, new_capacity);
self.header = header;
self.data = data;
self.set_capacity(new_capacity);
assert_eq!(self.data.len(), self.capacity);
try!(self.reinsert(0, new_capacity));
@ -259,7 +268,7 @@ impl Index {
entries += 1;
match self.locate(&entry.key) {
LocateResult::Found(p) if p == pos => true,
found => return Err(IndexError::WrongPosition(entry.key, pos, found))
found => return Err(IndexError::WrongPosition(pos, found))
};
}
if entries != self.entries {
@ -286,18 +295,15 @@ impl Index {
#[inline]
fn write_header(&mut self) {
let entries = self.entries;
let capacity = self.capacity;
let header = self.header();
header.entries = entries as u64;
header.capacity = capacity as u64;
self.header.entries = self.entries as u64;
self.header.capacity = self.capacity as u64;
}
/// Finds the position for this key
/// If the key is in the table, it will be the position of the key,
/// otherwise it will be the position where this key should be inserted
fn locate(&self, key: &Hash) -> LocateResult {
let mut pos = key.hash() as usize % self.capacity;
fn locate(&self, key: &K) -> LocateResult {
let mut pos = key.hash() as usize & self.mask;
let mut dist = 0;
loop {
let entry = &self.data[pos];
@ -307,11 +313,11 @@ impl Index {
if entry.key == *key {
return LocateResult::Found(pos);
}
let odist = (pos + self.capacity - entry.key.hash() as usize % self.capacity) % self.capacity;
let odist = (pos + self.capacity - entry.key.hash() as usize & self.mask) & self.mask;
if dist > odist {
return LocateResult::Steal(pos);
}
pos = (pos + 1) % self.capacity ;
pos = (pos + 1) & self.mask;
dist += 1;
}
}
@ -323,14 +329,14 @@ impl Index {
let mut last_pos;
loop {
last_pos = pos;
pos = (pos + 1) % self.capacity;
pos = (pos + 1) & self.mask;
{
let entry = &self.data[pos];
if !entry.is_used() {
// we found a hole, stop shifting here
break;
}
if entry.key.hash() as usize % self.capacity == pos {
if entry.key.hash() as usize & self.mask == pos {
// we found an entry at the right position, stop shifting here
break;
}
@ -342,7 +348,7 @@ impl Index {
/// Adds the key, data pair into the table.
/// If the key existed the old data is returned.
pub fn set(&mut self, key: &Hash, data: &Location) -> Result<Option<Location>, IndexError> {
pub fn set(&mut self, key: &K, data: &V) -> Result<Option<V>, IndexError> {
match self.locate(key) {
LocateResult::Found(pos) => {
let mut old = *data;
@ -370,7 +376,7 @@ impl Index {
entry.data = *data;
}
loop {
cur_pos = (cur_pos + 1) % self.capacity;
cur_pos = (cur_pos + 1) & self.mask;
let entry = &mut self.data[cur_pos];
if entry.is_used() {
mem::swap(&mut stolen_key, &mut entry.key);
@ -388,7 +394,7 @@ impl Index {
}
#[inline]
pub fn contains(&self, key: &Hash) -> bool {
pub fn contains(&self, key: &K) -> bool {
debug_assert!(self.check().is_ok(), "Inconsistent before get");
match self.locate(key) {
LocateResult::Found(_) => true,
@ -397,7 +403,7 @@ impl Index {
}
#[inline]
pub fn pos(&self, key: &Hash) -> Option<usize> {
pub fn pos(&self, key: &K) -> Option<usize> {
debug_assert!(self.check().is_ok(), "Inconsistent before get");
match self.locate(key) {
LocateResult::Found(pos) => Some(pos),
@ -406,7 +412,7 @@ impl Index {
}
#[inline]
pub fn get(&self, key: &Hash) -> Option<Location> {
pub fn get(&self, key: &K) -> Option<V> {
debug_assert!(self.check().is_ok(), "Inconsistent before get");
match self.locate(key) {
LocateResult::Found(pos) => Some(self.data[pos].data),
@ -415,7 +421,7 @@ impl Index {
}
#[inline]
pub fn modify<F>(&mut self, key: &Hash, mut f: F) -> bool where F: FnMut(&mut Location) {
pub fn modify<F>(&mut self, key: &K, mut f: F) -> bool where F: FnMut(&mut V) {
debug_assert!(self.check().is_ok(), "Inconsistent before get");
match self.locate(key) {
LocateResult::Found(pos) => {
@ -427,7 +433,7 @@ impl Index {
}
#[inline]
pub fn delete(&mut self, key: &Hash) -> Result<bool, IndexError> {
pub fn delete(&mut self, key: &K) -> Result<bool, IndexError> {
match self.locate(key) {
LocateResult::Found(pos) => {
self.backshift(pos);
@ -438,7 +444,7 @@ impl Index {
}
}
pub fn filter<F>(&mut self, mut f: F) -> Result<usize, IndexError> where F: FnMut(&Hash, &Location) -> bool {
pub fn filter<F>(&mut self, mut f: F) -> Result<usize, IndexError> where F: FnMut(&K, &V) -> bool {
//TODO: is it faster to walk in reverse direction?
let mut deleted = 0;
let mut pos = 0;
@ -460,48 +466,8 @@ impl Index {
}
#[inline]
pub fn walk<F, E>(&self, mut f: F) -> Result<(), E> where F: FnMut(&Hash, &Location) -> Result<(), E> {
for pos in 0..self.capacity {
let entry = &self.data[pos];
if entry.is_used() {
try!(f(&entry.key, &entry.data));
}
}
Ok(())
}
#[inline]
pub fn walk_mut<F, E>(&mut self, mut f: F) -> Result<(), E> where F: FnMut(&Hash, &mut Location) -> Result<(), E> {
for pos in 0..self.capacity {
let entry = &mut self.data[pos];
if entry.is_used() {
try!(f(&entry.key, &mut entry.data));
}
}
Ok(())
}
#[inline]
pub fn next_entry(&self, index: usize) -> Option<usize> {
let mut i = index;
while i < self.capacity && !self.data[i].is_used() {
i += 1;
}
if i == self.capacity {
None
} else {
Some(i)
}
}
#[inline]
pub fn get_entry(&self, index: usize) -> Option<&Entry> {
let entry = &self.data[index];
if entry.is_used() {
Some(entry)
} else {
None
}
pub fn iter<'a>(&'a self) -> Iter<'a, K, V> {
Iter{items: &self.data}
}
#[inline]

4
src/main.rs

@ -4,7 +4,6 @@ extern crate serde;
extern crate rmp_serde;
#[macro_use] extern crate serde_utils;
extern crate squash_sys as squash;
extern crate mmap;
extern crate blake2_rfc as blake2;
extern crate murmurhash3;
extern crate serde_yaml;
@ -28,10 +27,11 @@ extern crate pbr;
extern crate users;
extern crate libc;
extern crate tar;
extern crate index;
pub mod util;
mod bundledb;
pub mod index;
//pub mod index;
mod chunker;
mod repository;
mod cli;

4
src/prelude.rs

@ -1,8 +1,8 @@
pub use ::util::*;
pub use ::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError, BundleDb, BundleWriterError, StoredBundle};
pub use ::chunker::{ChunkerType, Chunker, ChunkerStatus, IChunker, ChunkerError};
pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, IntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError, RepositoryLayout};
pub use ::index::{Index, Location, IndexError};
pub use ::repository::{Repository, Backup, Config, RepositoryError, RepositoryInfo, Inode, FileType, IntegrityError, BackupFileError, BackupError, BackupOptions, BundleAnalysis, FileData, DiffType, InodeError, RepositoryLayout, Location};
pub use ::index::{Index, IndexError};
pub use ::mount::FuseFilesystem;
pub use serde::{Serialize, Deserialize};

13
src/repository/integrity.rs

@ -48,31 +48,30 @@ quick_error!{
impl Repository {
fn check_index_chunks(&self) -> Result<(), RepositoryError> {
let mut count = 0;
let mut progress = ProgressBar::new(self.index.len() as u64);
progress.message("checking index: ");
progress.set_max_refresh_rate(Some(Duration::from_millis(100)));
let res = self.index.walk(|_hash, location| {
for (count,(_hash, location)) in self.index.iter().enumerate() {
// Lookup bundle id from map
let bundle_id = try!(self.get_bundle_id(location.bundle));
// Get bundle object from bundledb
let bundle = if let Some(bundle) = self.bundles.get_bundle_info(&bundle_id) {
bundle
} else {
progress.finish_print("checking index: done.");
return Err(IntegrityError::MissingBundle(bundle_id.clone()).into())
};
// Get chunk from bundle
if bundle.info.chunk_count <= location.chunk as usize {
progress.finish_print("checking index: done.");
return Err(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into())
}
count += 1;
if count % 1000 == 0 {
progress.set(count);
progress.set(count as u64);
}
Ok(())
});
}
progress.finish_print("checking index: done.");
res
Ok(())
}
fn check_chunks(&self, checked: &mut Bitmap, chunks: &[Chunk], mark: bool) -> Result<bool, RepositoryError> {

38
src/repository/mod.rs

@ -35,11 +35,41 @@ use self::bundle_map::BundleMap;
const REPOSITORY_README: &'static [u8] = include_bytes!("../../docs/repository_readme.md");
const DEFAULT_EXCLUDES: &'static [u8] = include_bytes!("../../docs/excludes.default");
const INDEX_MAGIC: [u8; 7] = *b"zvault\x02";
const INDEX_VERSION: u8 = 1;
#[repr(packed)]
#[derive(Clone, Copy, PartialEq, Debug)]
pub struct Location {
pub bundle: u32,
pub chunk: u32
}
impl Location {
pub fn new(bundle: u32, chunk: u32) -> Self {
Location{ bundle: bundle, chunk: chunk }
}
}
impl ::index::Key for Hash {
fn hash(&self) -> u64 {
self.low
}
fn is_used(&self) -> bool {
self.low != 0 || self.high != 0
}
fn clear(&mut self) {
self.low = 0;
self.high = 0;
}
}
pub struct Repository {
pub layout: RepositoryLayout,
pub config: Config,
index: Index,
index: Index<Hash, Location>,
crypto: Arc<Mutex<Crypto>>,
bundle_map: BundleMap,
next_data_bundle: u32,
@ -67,7 +97,7 @@ impl Repository {
try!(fs::create_dir_all(layout.remote_locks_path()));
try!(config.save(layout.config_path()));
try!(BundleDb::create(layout.clone()));
try!(Index::create(layout.index_path()));
try!(Index::<Hash, Location>::create(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION));
try!(BundleMap::create().save(layout.bundle_map_path()));
try!(fs::create_dir_all(layout.backups_path()));
Self::open(path)
@ -86,11 +116,11 @@ impl Repository {
let lock = try!(local_locks.lock(false));
let crypto = Arc::new(Mutex::new(try!(Crypto::open(layout.keys_path()))));
let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone()));
let (index, mut rebuild_index) = match Index::open(layout.index_path()) {
let (index, mut rebuild_index) = match Index::open(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION) {
Ok(index) => (index, false),
Err(err) => {
error!("Failed to load local index:\n\tcaused by: {}", err);
(try!(Index::create(layout.index_path())), true)
(try!(Index::create(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION)), true)
}
};
let (bundle_map, rebuild_bundle_map) = match BundleMap::load(layout.bundle_map_path()) {

5
src/repository/vacuum.rs

@ -79,12 +79,11 @@ impl Repository {
}
try!(self.flush());
info!("Checking index");
self.index.walk::<_, ()>(|hash, location| {
for (hash, location) in self.index.iter() {
if rewrite_bundles.contains(&location.bundle) {
panic!("Removed bundle is still referenced in index: hash:{}, bundle:{}, chunk:{}", hash, location.bundle, location.chunk);
}
Ok(())
}).ok();
}
info!("Deleting {} bundles", rewrite_bundles.len());
for id in rewrite_bundles {
try!(self.delete_bundle(id));

Loading…
Cancel
Save