dswd
/
zvault
mirror of https://github.com/dswd/zvault
1
0
Fork 0
Code Issues Releases Wiki Activity

Refactored chunk repository

refactor
Dennis Schwerdel 2018-06-13 23:08:45 +02:00
parent 22279b9527
commit 31c6650374
20 changed files with 993 additions and 1136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -5,6 +5,8 @@ This project follows [semantic versioning](http://semver.org).
### UNRELEASED
* [added] Translation infrastructure (**requires nightly rust**)
* [added] Internal refactoring of lock levels
* [added] Internal refactoring of check ans repair
* [added] Checking hashes of chunks in check --bundle-data
* [added] Debian packet for libsodium23
* [modified] Updated dependencies

169
Cargo.lock generated
View File

@ -24,7 +24,7 @@ dependencies = [
[[package]]
name = "atty"
version = "0.2.8"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
@ -39,7 +39,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "bitflags"
version = "1.0.1"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -53,22 +53,22 @@ dependencies = [
[[package]]
name = "byteorder"
version = "1.2.2"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "cfg-if"
version = "0.1.2"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "chrono"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)",
"num-integer 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -77,12 +77,12 @@ version = "2.31.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -95,12 +95,27 @@ name = "crossbeam"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "dtoa"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "filetime"
version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "filetime"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -110,7 +125,7 @@ name = "fuchsia-zircon"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -126,9 +141,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)",
"thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -166,7 +181,7 @@ version = "0.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -181,7 +196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -198,7 +213,7 @@ name = "log"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -230,10 +245,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "num-integer"
version = "0.1.36"
version = "0.1.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -241,28 +256,29 @@ name = "num-traits"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "num-traits"
version = "0.2.2"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "pbr"
version = "1.0.0"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)",
"termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "pkg-config"
version = "0.3.9"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -295,19 +311,19 @@ dependencies = [
[[package]]
name = "regex"
version = "0.2.10"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"regex-syntax 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
"thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "regex-syntax"
version = "0.5.3"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -315,7 +331,7 @@ dependencies = [
[[package]]
name = "remove_dir_all"
version = "0.5.0"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
@ -326,7 +342,7 @@ name = "rmp"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -335,9 +351,9 @@ name = "rmp-serde"
version = "0.13.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"rmp 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -347,7 +363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "serde"
version = "1.0.37"
version = "1.0.55"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -355,7 +371,7 @@ name = "serde_bytes"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -363,18 +379,18 @@ name = "serde_utils"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_bytes 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "serde_yaml"
version = "0.7.3"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)",
"yaml-rust 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -385,7 +401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
"libsodium-sys 0.0.16 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -395,7 +411,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"pkg-config 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -405,13 +421,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "tar"
version = "0.4.14"
version = "0.4.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"filetime 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
"filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)",
"xattr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)",
"xattr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -420,7 +436,7 @@ version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"remove_dir_all 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -457,7 +473,7 @@ dependencies = [
[[package]]
name = "time"
version = "0.1.39"
version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
@ -498,7 +514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "vec_map"
version = "0.8.0"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -535,14 +551,6 @@ name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "xattr"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "xattr"
version = "0.2.1"
@ -565,8 +573,8 @@ version = "0.5.0"
dependencies = [
"ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)",
"crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"filetime 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
@ -578,20 +586,20 @@ dependencies = [
"log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"pbr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"rmp-serde 0.13.7 (registry+https://github.com/rust-lang/crates.io-index)",
"runtime-fmt 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_bytes 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_utils 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_yaml 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_yaml 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)",
"sodiumoxide 0.0.16 (registry+https://github.com/rust-lang/crates.io-index)",
"squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tar 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)",
"tar 0.4.15 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)",
"users 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"xattr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -600,17 +608,19 @@ dependencies = [
"checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
"checksum atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "af80143d6f7608d746df1520709e5d141c96f240b0e62b0aa41bdfb53374d9d4"
"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1"
"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d"
"checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf"
"checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789"
"checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400"
"checksum byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "73b5bdfe7ee3ad0b99c9801d58807a9dbc9e09196365b0203853b99889ab3c87"
"checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de"
"checksum chrono 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ba5f60682a4c264e7f8d77b82e7788938a76befdf949d4a98026d19099c9d873"
"checksum byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "74c0b906e9446b0a2e4f760cdb3fa4b2c48cdc6db8766a845c54b6ff063fd2e9"
"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18"
"checksum chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1cce36c92cb605414e9b824f866f5babe0a0368e39ea07393b9b63cf3844c0e6"
"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536"
"checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e"
"checksum crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "24ce9782d4d5c53674646a6a4c1863a21a8fc0cb649b3c94dfc16e45071dea19"
"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab"
"checksum filetime 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "714653f3e34871534de23771ac7b26e999651a0a228f47beb324dfdf1dd4b10f"
"checksum filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "da4b9849e77b13195302c174324b5ba73eec9b236b24c221a61000daefb95c5f"
"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
"checksum fuse 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80e57070510966bfef93662a81cb8aa2b1c7db0964354fa9921434f04b9e8660"
@ -628,47 +638,46 @@ dependencies = [
"checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823"
"checksum murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a2983372caf4480544083767bf2d27defafe32af49ab4df3a0b7fc90793a3664"
"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2"
"checksum num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f8d26da319fb45674985c78f1d1caf99aa4941f785d384a2ae36d0740bc3e2fe"
"checksum num-integer 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)" = "6ac0ea58d64a89d9d6b7688031b3be9358d6c919badcf7fbb0527ccfd891ee45"
"checksum num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31"
"checksum num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dee092fcdf725aee04dd7da1d21debff559237d49ef1cb3e69bcb8ece44c7364"
"checksum pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e048e3afebb6c454bb1c5d0fe73fda54698b4715d78ed8e7302447c37736d23a"
"checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903"
"checksum num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "775393e285254d2f5004596d69bb8bc1149754570dcc08cf30cabeba67955e28"
"checksum pbr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "deb73390ab68d81992bd994d145f697451bb0b54fd39738e72eef32458ad6907"
"checksum pkg-config 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "110d5ee3593dbb73f56294327fe5668bcc997897097cbc76b51e7aed3f52452f"
"checksum quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eda5fe9b71976e62bc81b781206aaa076401769b2143379d3eb2118388babac4"
"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5"
"checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd"
"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
"checksum regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "aec3f58d903a7d2a9dc2bf0e41a746f4530e0cab6b615494e058f67a3ef947fb"
"checksum regex-syntax 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b2550876c31dc914696a6c2e01cbce8afba79a93c8ae979d2fe051c0230b3756"
"checksum remove_dir_all 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dfc5b3ce5d5ea144bb04ebd093a9e14e9765bcfec866aecda9b6dec43b3d1e24"
"checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384"
"checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7"
"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5"
"checksum rmp 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a3d45d7afc9b132b34a2479648863aa95c5c88e98b32285326a6ebadc80ec5c9"
"checksum rmp-serde 0.13.7 (registry+https://github.com/rust-lang/crates.io-index)" = "011e1d58446e9fa3af7cdc1fb91295b10621d3ac4cb3a85cc86385ee9ca50cd3"
"checksum runtime-fmt 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "647a821d66049faccc993fc3c379d1181b81a484097495cda79ffdb17b55b87f"
"checksum serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)" = "d3bcee660dcde8f52c3765dd9ca5ee36b4bf35470a738eb0bd5a8752b0389645"
"checksum serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)" = "97f6a6c3caba0cf8f883b53331791036404ce3c1bd895961cf8bb2f8cecfd84b"
"checksum serde_bytes 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)" = "adb6e51a6b3696b301bc221d785f898b4457c619b51d7ce195a6d20baecb37b3"
"checksum serde_utils 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f6e0edb364c93646633800df969086bc7c5c25fb3f1eb57349990d1cb4cae4bc"
"checksum serde_yaml 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e0f868d400d9d13d00988da49f7f02aeac6ef00f11901a8c535bd59d777b9e19"
"checksum serde_yaml 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)" = "107bb818146aaf922e7bbcf6a940f1db2f0dcf381779b451e400331b2c6f86db"
"checksum sodiumoxide 0.0.16 (registry+https://github.com/rust-lang/crates.io-index)" = "eb5cb2f14f9a51352ad65e59257a0a9459d5a36a3615f3d53a974c82fdaaa00a"
"checksum squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db1f9dde91d819b7746e153bc32489fa19e6a106c3d7f2b92187a4efbdc88b40"
"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
"checksum tar 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)" = "1605d3388ceb50252952ffebab4b5dc43017ead7e4481b175961c283bb951195"
"checksum tar 0.4.15 (registry+https://github.com/rust-lang/crates.io-index)" = "6af6b94659f9a571bf769a5b71f54079393585ee0bfdd71b691be22d7d6b1d18"
"checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096"
"checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693"
"checksum thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bcbb6aa301e5d3b0b5ef639c9a9c7e2f1c944f177b460c04dc24c69b1fa2bd99"
"checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963"
"checksum time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "a15375f1df02096fb3317256ce2cee6a1f42fc84ea5ad5fc8c421cfe40c73098"
"checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b"
"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d"
"checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f"
"checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56"
"checksum users 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a098d836637f965bbe0df8f744088318c43b685ffd46b676ed21036b7c94bae6"
"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122"
"checksum vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "887b5b631c2ad01628bbbaa7dd4c869f80d3186688f8d0b6f58774fbe324988c"
"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
"checksum xattr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "5f04de8a1346489a2f9e9bd8526b73d135ec554227b17568456e86aa35b6f3fc"
"checksum xattr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "abb373b92de38a4301d66bec009929b4fb83120ea1c4a401be89dbe0b9777443"
"checksum yaml-rust 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57ab38ee1a4a266ed033496cf9af1828d8d6e6c1cfa5f643a2809effcae4d628"

12
src/backups/integrity.rs
View File

@ -31,14 +31,14 @@ impl BackupRepository {
None |
Some(FileData::Inline(_)) => (),
Some(FileData::ChunkedDirect(ref chunks)) => {
try!(self.repo.check_chunks(checked, chunks, true));
try!(self.repo.mark_chunks(checked, chunks, true));
}
Some(FileData::ChunkedIndirect(ref chunks)) => {
if try!(self.repo.check_chunks(checked, chunks, false)) {
if try!(self.repo.mark_chunks(checked, chunks, false)) {
let chunk_data = try!(self.get_data(chunks));
let chunks2 = ChunkList::read_from(&chunk_data);
try!(self.repo.check_chunks(checked, &chunks2, true));
try!(self.repo.check_chunks(checked, chunks, true));
try!(self.repo.mark_chunks(checked, &chunks2, true));
try!(self.repo.mark_chunks(checked, chunks, true));
}
}
}
@ -53,7 +53,7 @@ impl BackupRepository {
repair: bool,
) -> Result<Option<ChunkList>, RepositoryError> {
let mut modified = false;
match self.repo.check_chunks(checked, chunks, false) {
match self.repo.mark_chunks(checked, chunks, false) {
Ok(false) => return Ok(None),
Ok(true) => (),
Err(err) => return Err(InodeIntegrityError::BrokenInode(path, Box::new(err)).into()),
@ -108,7 +108,7 @@ impl BackupRepository {
if modified {
Ok(Some(try!(self.put_inode(&inode))))
} else {
try!(self.repo.check_chunks(checked, chunks, true));
try!(self.repo.mark_chunks(checked, chunks, true));
Ok(None)
}
}

6
src/backups/mod.rs
View File

@ -28,7 +28,7 @@ const DEFAULT_EXCLUDES: &[u8] = include_bytes!("../../docs/excludes.default");
pub struct BackupRepository {
layout: Arc<RepositoryLayout>,
crypto: Arc<Crypto>,
repo: RepositoryInner
repo: Repository
}
impl BackupRepository {
@ -44,7 +44,7 @@ impl BackupRepository {
Ok(BackupRepository {
crypto: crypto.clone(),
layout: layout.clone(),
repo: try!(RepositoryInner::create(layout, config, crypto, remote))
repo: try!(Repository::create(layout, config, crypto, remote))
})
}
@ -55,7 +55,7 @@ impl BackupRepository {
Ok(BackupRepository {
crypto: crypto.clone(),
layout: layout.clone(),
repo: try!(RepositoryInner::open(layout, crypto, online))
repo: try!(Repository::open(layout, crypto, online))
})
}

2
src/prelude.rs
View File

@ -2,7 +2,7 @@ pub use util::*;
pub use repository::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError,
BundleDb, BundleWriterError, StoredBundle, BundleStatistics};
pub use repository::chunking::{ChunkerType, Chunker, ChunkerStatus, ChunkerError};
pub use repository::{RepositoryInner, Config, RepositoryError, RepositoryInfo,
pub use repository::{Repository, Config, RepositoryError, RepositoryInfo,
IntegrityError, BundleAnalysis, RepositoryLayout, Location,
RepositoryStatistics, ChunkRepositoryLayout};
pub use repository::*;

59
src/repository/inner/basic_io.rs → src/repository/basic_io.rs
View File

@ -10,16 +10,18 @@ pub struct ChunkReader<'a> {
chunks: VecDeque<Chunk>,
data: Vec<u8>,
pos: usize,
repo: &'a mut RepositoryInner
repo: &'a mut Repository,
lock: &'a OnlineMode
}
impl<'a> ChunkReader<'a> {
pub fn new(repo: &'a mut RepositoryInner, chunks: ChunkList) -> Self {
pub fn new(repo: &'a mut Repository, chunks: ChunkList, lock: &'a OnlineMode) -> Self {
ChunkReader {
repo,
chunks: chunks.into_inner().into(),
data: vec![],
pos: 0
pos: 0,
lock
}
}
}
@ -33,7 +35,7 @@ impl<'a> Read for ChunkReader<'a> {
}
if self.data.len() == self.pos {
if let Some(chunk) = self.chunks.pop_front() {
self.data = match self.repo.get_chunk(chunk.0) {
self.data = match self.repo.get_chunk(chunk.0, self.lock) {
Ok(Some(data)) => data,
Ok(None) => {
return Err(io::Error::new(
@ -58,7 +60,7 @@ impl<'a> Read for ChunkReader<'a> {
}
impl RepositoryInner {
impl Repository {
#[inline]
pub fn get_bundle_id(&self, id: u32) -> Result<BundleId, RepositoryError> {
self.bundle_map.get(id).ok_or_else(|| {
@ -66,7 +68,7 @@ impl RepositoryInner {
})
}
pub fn get_chunk(&mut self, hash: Hash) -> Result<Option<Vec<u8>>, RepositoryError> {
pub fn get_chunk(&mut self, hash: Hash, lock: &OnlineMode) -> Result<Option<Vec<u8>>, RepositoryError> {
// Find bundle and chunk id in index
let found = if let Some(found) = self.index.get(&hash) {
found
@ -77,7 +79,7 @@ impl RepositoryInner {
let bundle_id = try!(self.get_bundle_id(found.bundle));
// Get chunk from bundle
Ok(Some(try!(
self.bundles.get_chunk(&bundle_id, found.chunk as usize)
self.bundles.get_chunk(&bundle_id, found.chunk as usize, lock)
)))
}
@ -87,12 +89,13 @@ impl RepositoryInner {
mode: BundleMode,
hash: Hash,
data: &[u8],
lock: &BackupMode
) -> Result<(), RepositoryError> {
// If this chunk is in the index, ignore it
if self.index.contains(&hash) {
return Ok(());
}
self.put_chunk_override(mode, hash, data)
self.put_chunk_override(mode, hash, data, lock)
}
fn write_chunk_to_bundle_and_index(
@ -100,6 +103,7 @@ impl RepositoryInner {
mode: BundleMode,
hash: Hash,
data: &[u8],
lock: &BackupMode
) -> Result<(), RepositoryError> {
let writer = match mode {
BundleMode::Data => &mut self.data_bundle,
@ -111,7 +115,8 @@ impl RepositoryInner {
mode,
self.config.hash,
self.config.compression.clone(),
self.config.encryption.clone()
self.config.encryption.clone(),
lock
)));
}
debug_assert!(writer.is_some());
@ -130,7 +135,7 @@ impl RepositoryInner {
Ok(())
}
fn finish_bundle(&mut self, mode: BundleMode) -> Result<(), RepositoryError> {
fn finish_bundle(&mut self, mode: BundleMode, lock: &BackupMode) -> Result<(), RepositoryError> {
// Calculate the next free bundle id now (late lifetime prevents this)
let next_free_bundle_id = self.next_free_bundle_id();
let writer = match mode {
@ -146,8 +151,8 @@ impl RepositoryInner {
};
let mut finished = None;
mem::swap(writer, &mut finished);
let bundle = try!(self.bundles.add_bundle(finished.unwrap()));
self.bundle_map.set(bundle_id, bundle.id.clone());
let bundle = try!(self.bundles.add_bundle(finished.unwrap(), lock));
self.bundle_map.set(bundle_id, bundle.id.clone(), lock.as_localwrite());
if self.next_meta_bundle == bundle_id {
self.next_meta_bundle = next_free_bundle_id
}
@ -157,7 +162,7 @@ impl RepositoryInner {
Ok(())
}
fn finish_bundle_if_needed(&mut self, mode: BundleMode) -> Result<(), RepositoryError> {
fn finish_bundle_if_needed(&mut self, mode: BundleMode, lock: &BackupMode) -> Result<(), RepositoryError> {
let (size, raw_size) = {
let writer = match mode {
BundleMode::Data => &mut self.data_bundle,
@ -171,10 +176,10 @@ impl RepositoryInner {
};
if size >= self.config.bundle_size || raw_size >= 4 * self.config.bundle_size {
if mode == BundleMode::Meta {
//First store the current data bundle as meta referrs to those chunks
try!(self.finish_bundle(BundleMode::Data))
//First store the current data bundle as meta refers to those chunks
try!(self.finish_bundle(BundleMode::Data, lock))
}
try!(self.finish_bundle(mode))
try!(self.finish_bundle(mode, lock))
}
Ok(())
}
@ -185,9 +190,10 @@ impl RepositoryInner {
mode: BundleMode,
hash: Hash,
data: &[u8],
lock: &BackupMode
) -> Result<(), RepositoryError> {
try!(self.write_chunk_to_bundle_and_index(mode, hash, data));
self.finish_bundle_if_needed(mode)
try!(self.write_chunk_to_bundle_and_index(mode, hash, data, lock));
self.finish_bundle_if_needed(mode, lock)
}
#[inline]
@ -195,15 +201,17 @@ impl RepositoryInner {
&mut self,
mode: BundleMode,
data: &[u8],
lock: &BackupMode
) -> Result<ChunkList, RepositoryError> {
let mut input = Cursor::new(data);
self.put_stream(mode, &mut input)
self.put_stream(mode, &mut input, lock)
}
pub fn put_stream<R: Read>(
&mut self,
mode: BundleMode,
data: &mut R,
lock: &BackupMode
) -> Result<ChunkList, RepositoryError> {
let avg_size = self.config.chunker.avg_size();
let mut chunks = Vec::new();
@ -214,7 +222,7 @@ impl RepositoryInner {
let res = try!(self.chunker.chunk(data, &mut output));
chunk = output.into_inner();
let hash = self.config.hash.hash(&chunk);
try!(self.put_chunk(mode, hash, &chunk));
try!(self.put_chunk(mode, hash, &chunk, lock));
chunks.push((hash, chunk.len() as u32));
if res == ChunkerStatus::Finished {
break;
@ -223,25 +231,26 @@ impl RepositoryInner {
Ok(chunks.into())
}
pub fn get_data(&mut self, chunks: &[Chunk]) -> Result<Vec<u8>, RepositoryError> {
pub fn get_data(&mut self, chunks: &[Chunk], lock: &OnlineMode) -> Result<Vec<u8>, RepositoryError> {
let mut data =
Vec::with_capacity(chunks.iter().map(|&(_, size)| size).sum::<u32>() as usize);
try!(self.get_stream(chunks, &mut data));
try!(self.get_stream(chunks, &mut data, lock));
Ok(data)
}
#[inline]
pub fn get_reader(&mut self, chunks: ChunkList) -> ChunkReader {
ChunkReader::new(self, chunks)
pub fn get_reader<'a>(&'a mut self, chunks: ChunkList, lock: &'a OnlineMode) -> ChunkReader<'a> {
ChunkReader::new(self, chunks, lock)
}
pub fn get_stream<W: Write>(
&mut self,
chunks: &[Chunk],
w: &mut W,
lock: &OnlineMode
) -> Result<(), RepositoryError> {
for &(ref hash, len) in chunks {
let data = try!(try!(self.get_chunk(*hash)).ok_or_else(|| {
let data = try!(try!(self.get_chunk(*hash, lock)).ok_or_else(|| {
IntegrityError::MissingChunk(*hash)
}));
debug_assert_eq!(data.len() as u32, len);

8
src/repository/bundle_map.rs
View File

@ -46,7 +46,7 @@ impl BundleMap {
BundleMap(Default::default())
}
pub fn load<P: AsRef<Path>>(path: P) -> Result<Self, BundleMapError> {
pub fn load<P: AsRef<Path>>(path: P, _lock: &ReadonlyMode) -> Result<Self, BundleMapError> {
let mut file = BufReader::new(try!(File::open(path.as_ref())));
let mut header = [0u8; 8];
try!(file.read_exact(&mut header));
@ -60,7 +60,7 @@ impl BundleMap {
Ok(BundleMap(try!(msgpack::decode_from_stream(&mut file))))
}
pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<(), BundleMapError> {
pub fn save<P: AsRef<Path>>(&self, path: P, _lock: &LocalWriteMode) -> Result<(), BundleMapError> {
let mut file = BufWriter::new(try!(File::create(path)));
try!(file.write_all(&HEADER_STRING));
try!(file.write_all(&[HEADER_VERSION]));
@ -73,7 +73,7 @@ impl BundleMap {
}
#[inline]
pub fn remove(&mut self, id: u32) -> Option<BundleId> {
pub fn remove(&mut self, id: u32, _lock: &LocalWriteMode) -> Option<BundleId> {
self.0.remove(&id)
}
@ -87,7 +87,7 @@ impl BundleMap {
}
#[inline]
pub fn set(&mut self, id: u32, bundle: BundleId) {
pub fn set(&mut self, id: u32, bundle: BundleId, _lock: &LocalWriteMode) {
self.0.insert(id, bundle);
}

128
src/repository/bundledb/db.rs
View File

@ -62,7 +62,7 @@ fn load_bundles(
path: &Path,
base: &Path,
bundles: &mut HashMap<BundleId, StoredBundle>,
crypto: Arc<Crypto>,
crypto: Arc<Crypto>
) -> Result<(Vec<StoredBundle>, Vec<StoredBundle>), BundleDbError> {
let mut paths = vec![path.to_path_buf()];
let mut bundle_paths = HashSet::new();
@ -138,10 +138,10 @@ impl BundleDb {
}
}
fn load_bundle_list(
fn load_local_bundle_list(
&mut self,
online: bool
) -> Result<(Vec<StoredBundle>, Vec<StoredBundle>), BundleDbError> {
_lock: &ReadonlyMode
) -> Result<(), BundleDbError> {
if let Ok(list) = StoredBundle::read_list_from(&self.layout.local_bundle_cache_path()) {
for bundle in list {
self.local_bundles.insert(bundle.id(), bundle);
@ -170,9 +170,15 @@ impl BundleDb {
&self.layout.local_bundle_cache_path()
));
}
if !online {
return Ok((vec![], vec![]))
}
Ok(())
}
fn load_remote_bundle_list(
&mut self,
lock: &OnlineMode
) -> Result<(Vec<StoredBundle>, Vec<StoredBundle>), BundleDbError> {
try!(self.load_local_bundle_list(lock.as_readonly()));
let base_path = self.layout.base_path();
let (new, gone) = try!(load_bundles(
&self.layout.remote_bundles_path(),
base_path,
@ -189,11 +195,11 @@ impl BundleDb {
Ok((new, gone))
}
pub fn flush(&mut self) -> Result<(), BundleDbError> {
self.finish_uploads().and_then(|()| self.save_cache())
pub fn flush(&mut self, lock: &BackupMode) -> Result<(), BundleDbError> {
self.finish_uploads(lock).and_then(|()| self.save_cache(lock.as_localwrite()))
}
fn save_cache(&self) -> Result<(), BundleDbError> {
fn save_cache(&self, _lock: &LocalWriteMode) -> Result<(), BundleDbError> {
let bundles: Vec<_> = self.local_bundles.values().cloned().collect();
try!(StoredBundle::save_list_to(
&bundles,
@ -207,7 +213,8 @@ impl BundleDb {
Ok(())
}
fn update_cache(&mut self) -> Result<(), BundleDbError> {
pub fn synchronize(&mut self, lock: &OnlineMode) -> Result<(Vec<BundleInfo>, Vec<BundleInfo>), BundleDbError> {
let (new, gone) = try!(self.load_remote_bundle_list(lock));
let mut meta_bundles = HashSet::new();
for (id, bundle) in &self.remote_bundles {
if bundle.info.mode == BundleMode::Meta {
@ -224,7 +231,7 @@ impl BundleDb {
if !self.local_bundles.contains_key(&id) {
let bundle = self.remote_bundles[&id].clone();
tr_debug!("Copying new meta bundle to local cache: {}", bundle.info.id);
try!(self.copy_remote_bundle_to_cache(&bundle));
try!(self.copy_remote_bundle_to_cache(&bundle, lock));
}
}
let base_path = self.layout.base_path();
@ -235,20 +242,19 @@ impl BundleDb {
}))
}
}
Ok(())
let new = new.into_iter().map(|s| s.info).collect();
let gone = gone.into_iter().map(|s| s.info).collect();
Ok((new, gone))
}
pub fn open(
layout: Arc<ChunkRepositoryLayout>,
crypto: Arc<Crypto>,
online: bool
) -> Result<(Self, Vec<BundleInfo>, Vec<BundleInfo>), BundleDbError> {
lock: &ReadonlyMode
) -> Result<Self, BundleDbError> {
let mut self_ = Self::new(layout, crypto);
let (new, gone) = try!(self_.load_bundle_list(online));
try!(self_.update_cache());
let new = new.into_iter().map(|s| s.info).collect();
let gone = gone.into_iter().map(|s| s.info).collect();
Ok((self_, new, gone))
try!(self_.load_local_bundle_list(lock));
Ok(self_)
}
pub fn create(layout: Arc<ChunkRepositoryLayout>) -> Result<(), BundleDbError> {
@ -282,6 +288,7 @@ impl BundleDb {
hash_method: HashMethod,
compression: Option<Compression>,
encryption: Option<Encryption>,
lock: &BackupMode
) -> Result<BundleWriter, BundleDbError> {
Ok(try!(BundleWriter::new(
self.layout.clone(),
@ -305,7 +312,7 @@ impl BundleDb {
}
#[inline]
fn get_bundle(&self, stored: &StoredBundle) -> Result<BundleReader, BundleDbError> {
fn get_bundle(&self, stored: &StoredBundle, _lock: &OnlineMode) -> Result<BundleReader, BundleDbError> {
let base_path = self.layout.base_path();
Ok(try!(BundleReader::load(
base_path.join(&stored.path),
@ -313,7 +320,7 @@ impl BundleDb {
)))
}
pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result<Vec<u8>, BundleDbError> {
pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize, lock: &OnlineMode) -> Result<Vec<u8>, BundleDbError> {
if let Some(&mut (ref mut bundle, ref data)) = self.bundle_cache.get_mut(bundle_id) {
let (pos, len) = try!(bundle.get_chunk_position(id));
let mut chunk = Vec::with_capacity(len);
@ -321,7 +328,7 @@ impl BundleDb {
return Ok(chunk);
}
let mut bundle = try!(self.get_stored_bundle(bundle_id).and_then(
|s| self.get_bundle(s)
|s| self.get_bundle(s, lock)
));
let (pos, len) = try!(bundle.get_chunk_position(id));
let mut chunk = Vec::with_capacity(len);
@ -331,7 +338,7 @@ impl BundleDb {
Ok(chunk)
}
fn copy_remote_bundle_to_cache(&mut self, bundle: &StoredBundle) -> Result<(), BundleDbError> {
fn copy_remote_bundle_to_cache(&mut self, bundle: &StoredBundle, _lock: &OnlineMode) -> Result<(), BundleDbError> {
let id = bundle.id();
let dst_path = self.layout.local_bundle_path(&id, self.local_bundles.len());
{
@ -346,10 +353,10 @@ impl BundleDb {
Ok(())
}
pub fn add_bundle(&mut self, bundle: BundleWriter) -> Result<BundleInfo, BundleDbError> {
pub fn add_bundle(&mut self, bundle: BundleWriter, lock: &BackupMode) -> Result<BundleInfo, BundleDbError> {
let mut bundle = try!(bundle.finish());
if bundle.info.mode == BundleMode::Meta {
try!(self.copy_remote_bundle_to_cache(&bundle))
try!(self.copy_remote_bundle_to_cache(&bundle, lock.as_online()))
}
let dst_path = self.layout.remote_bundle_path(&bundle.id(),self.remote_bundles.len());
let src_path = self.layout.base_path().join(bundle.path);
@ -365,7 +372,7 @@ impl BundleDb {
Ok(bundle.info)
}
fn finish_uploads(&mut self) -> Result<(), BundleDbError> {
fn finish_uploads(&mut self, _lock: &BackupMode) -> Result<(), BundleDbError> {
let mut uploader = None;
mem::swap(&mut self.uploader, &mut uploader);
if let Some(uploader) = uploader {
@ -375,9 +382,9 @@ impl BundleDb {
}
}
pub fn get_chunk_list(&self, bundle: &BundleId) -> Result<ChunkList, BundleDbError> {
pub fn get_chunk_list(&self, bundle: &BundleId, lock: &OnlineMode) -> Result<ChunkList, BundleDbError> {
let mut bundle = try!(self.get_stored_bundle(bundle).and_then(|stored| {
self.get_bundle(stored)
self.get_bundle(stored, lock)
}));
Ok(try!(bundle.get_chunk_list()).clone())
}
@ -392,7 +399,7 @@ impl BundleDb {
self.remote_bundles.values().map(|b| &b.info).collect()
}
pub fn delete_local_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> {
pub fn delete_local_bundle(&mut self, bundle: &BundleId, _lock: &LocalWriteMode) -> Result<(), BundleDbError> {
if let Some(bundle) = self.local_bundles.remove(bundle) {
let path = self.layout.base_path().join(&bundle.path);
try!(fs::remove_file(path).map_err(|e| {
@ -402,8 +409,8 @@ impl BundleDb {
Ok(())
}
pub fn delete_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> {
try!(self.delete_local_bundle(bundle));
pub fn delete_bundle(&mut self, bundle: &BundleId, lock: &VacuumMode) -> Result<(), BundleDbError> {
try!(self.delete_local_bundle(bundle, lock.as_localwrite()));
if let Some(bundle) = self.remote_bundles.remove(bundle) {
let path = self.layout.base_path().join(&bundle.path);
fs::remove_file(path).map_err(|e| BundleDbError::Remove(e, bundle.id()))
@ -412,44 +419,38 @@ impl BundleDb {
}
}
pub fn check(&mut self, full: bool, repair: bool) -> Result<bool, BundleDbError> {
let mut to_repair = vec![];
pub fn check(&mut self, full: bool, lock: &OnlineMode) -> HashMap<BundleId, BundleDbError> {
let mut errors = HashMap::new();
for (id, stored) in ProgressIter::new(
tr!("checking bundles"),
self.remote_bundles.len(),
self.remote_bundles.iter()
)
{
let mut bundle = match self.get_bundle(stored) {
let mut bundle = match self.get_bundle(stored, lock) {
Ok(bundle) => bundle,
Err(err) => {
if repair {
to_repair.push(id.clone());
continue;
} else {
return Err(err);
}
errors.insert(id.clone(), err);
continue;
}
};
if let Err(err) = bundle.check(full) {
if repair {
to_repair.push(id.clone());
continue;
} else {
return Err(err.into());
}
errors.insert(id.clone(), err.into());
continue;
}
}
if !to_repair.is_empty() {
for id in ProgressIter::new(tr!("repairing bundles"), to_repair.len(), to_repair.iter()) {
try!(self.repair_bundle(id));
}
try!(self.flush());
}
Ok(!to_repair.is_empty())
errors
}
fn evacuate_broken_bundle(&mut self, mut bundle: StoredBundle) -> Result<(), BundleDbError> {
pub fn repair(&mut self, lock: &VacuumMode, bundles: &[BundleId]) -> Result<(), BundleDbError> {
for id in ProgressIter::new(tr!("repairing bundles"), bundles.len(), bundles.iter()) {
try!(self.repair_bundle(id, lock));
}
try!(self.flush(lock.as_backup()));
Ok(())
}
fn evacuate_broken_bundle(&mut self, mut bundle: StoredBundle, _lock: &VacuumMode) -> Result<(), BundleDbError> {
let src = self.layout.base_path().join(&bundle.path);
let mut dst = src.with_extension("bundle.broken");
let mut num = 1;
@ -463,9 +464,9 @@ impl BundleDb {
Ok(())
}
fn repair_bundle(&mut self, id: &BundleId) -> Result<(), BundleDbError> {
fn repair_bundle(&mut self, id: &BundleId, lock: &VacuumMode) -> Result<(), BundleDbError> {
let stored = self.remote_bundles[id].clone();
let mut bundle = match self.get_bundle(&stored) {
let mut bundle = match self.get_bundle(&stored, lock.as_online()) {
Ok(bundle) => bundle,
Err(err) => {
tr_warn!(
@ -473,7 +474,7 @@ impl BundleDb {
id,
err
);
return self.evacuate_broken_bundle(stored);
return self.evacuate_broken_bundle(stored, lock);
}
};
let chunks = match bundle.get_chunk_list() {
@ -484,7 +485,7 @@ impl BundleDb {
id,
err
);
return self.evacuate_broken_bundle(stored);
return self.evacuate_broken_bundle(stored, lock);
}
};
let data = match bundle.load_contents() {
@ -495,7 +496,7 @@ impl BundleDb {
id,
err
);
return self.evacuate_broken_bundle(stored);
return self.evacuate_broken_bundle(stored, lock);
}
};
tr_warn!("Problem detected: bundle data was truncated: {}", id);
@ -505,7 +506,8 @@ impl BundleDb {
info.mode,
info.hash_method,
info.compression,
info.encryption
info.encryption,
lock.as_backup()
));
let mut pos = 0;
for (hash, mut len) in chunks.into_inner() {
@ -516,9 +518,9 @@ impl BundleDb {
try!(new_bundle.add(&data[pos..pos + len as usize], hash));
pos += len as usize;
}
let bundle = try!(self.add_bundle(new_bundle));
let bundle = try!(self.add_bundle(new_bundle, lock.as_backup()));
tr_info!("New bundle id is {}", bundle.id);
self.evacuate_broken_bundle(stored)
self.evacuate_broken_bundle(stored, lock)
}
#[inline]

8
src/repository/bundledb/reader.rs
View File

@ -253,13 +253,7 @@ impl BundleReader {
tr!("Chunk list size does not match chunk count")
));
}
if self.chunks
.as_ref()
.unwrap()
.iter()
.map(|c| c.1 as usize)
.sum::<usize>() != self.info.raw_size
{
if self.chunks.as_ref().unwrap().iter().map(|c| c.1 as usize).sum::<usize>() != self.info.raw_size {
return Err(BundleReaderError::Integrity(
self.id(),
tr!("Individual chunk sizes do not add up to total size")

2
src/repository/config.rs
View File

@ -222,7 +222,7 @@ impl Config {
Config::from_yaml(config)
}
pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<(), ConfigError> {
pub fn save<P: AsRef<Path>>(&self, path: P, _lock: &LocalWriteMode) -> Result<(), ConfigError> {
let mut f = try!(File::create(path));
try!(serde_yaml::to_writer(&mut f, &self.to_yaml()));
Ok(())

21
src/repository/index/mod.rs
View File

@ -227,7 +227,7 @@ impl<K: Key, V: Value> Index<K, V> {
data,
header
};
debug_assert!(index.check().is_ok(), tr!("Inconsistent after creation"));
debug_assert!(index.check().is_empty(), tr!("Inconsistent after creation"));
Ok(index)
}
@ -325,7 +325,8 @@ impl<K: Key, V: Value> Index<K, V> {
Ok(true)
}
pub fn check(&self) -> Result<(), IndexError> {
pub fn check(&self) -> Vec<IndexError> {
let mut errors = vec![];
let mut entries = 0;
for pos in 0..self.capacity {
let entry = &self.data[pos];
@ -334,14 +335,14 @@ impl<K: Key, V: Value> Index<K, V> {
}
entries += 1;
match self.locate(entry.get_key()) {
LocateResult::Found(p) if p == pos => true,
found => return Err(IndexError::WrongPosition(pos, found))
LocateResult::Found(p) if p == pos => (),
found => errors.push(IndexError::WrongPosition(pos, found))
};
}
if entries != self.entries {
return Err(IndexError::WrongEntryCount(self.entries, entries));
errors.push(IndexError::WrongEntryCount(self.entries, entries));
}
Ok(())
errors
}
#[inline]
@ -467,7 +468,7 @@ impl<K: Key, V: Value> Index<K, V> {
#[inline]
pub fn contains(&self, key: &K) -> bool {
debug_assert!(self.check().is_ok(), tr!("Inconsistent before get"));
debug_assert!(self.check().is_empty(), tr!("Inconsistent before get"));
match self.locate(key) {
LocateResult::Found(_) => true,
_ => false
@ -476,7 +477,7 @@ impl<K: Key, V: Value> Index<K, V> {
#[inline]
pub fn pos(&self, key: &K) -> Option<usize> {
debug_assert!(self.check().is_ok(), tr!("Inconsistent before get"));
debug_assert!(self.check().is_empty(), tr!("Inconsistent before get"));
match self.locate(key) {
LocateResult::Found(pos) => Some(pos),
_ => None
@ -485,7 +486,7 @@ impl<K: Key, V: Value> Index<K, V> {
#[inline]
pub fn get(&self, key: &K) -> Option<V> {
debug_assert!(self.check().is_ok(), tr!("Inconsistent before get"));
debug_assert!(self.check().is_empty(), tr!("Inconsistent before get"));
match self.locate(key) {
LocateResult::Found(pos) => Some(self.data[pos].data),
_ => None
@ -495,7 +496,7 @@ impl<K: Key, V: Value> Index<K, V> {
#[inline]
#[allow(dead_code)]
pub fn modify<F>(&mut self, key: &K, mut f: F) -> bool where F: FnMut(&mut V) {
debug_assert!(self.check().is_ok(), tr!("Inconsistent before get"));
debug_assert!(self.check().is_empty(), tr!("Inconsistent before get"));
match self.locate(key) {
LocateResult::Found(pos) => {
f(self.data[pos].get_mut_data());

2
src/repository/inner/info.rs → src/repository/info.rs
View File

@ -46,7 +46,7 @@ pub struct RepositoryStatistics {
}
impl RepositoryInner {
impl Repository {
#[inline]
pub fn list_bundles(&self) -> Vec<&BundleInfo> {
self.bundles.list_bundles()

178
src/repository/inner/api.rs
View File

@ -1,178 +0,0 @@
use ::prelude::*;
pub struct Repository(RepositoryInner);
pub struct LocalWriteMode<'a>(&'a mut RepositoryInner);
pub struct RestoreMode<'a>(&'a mut RepositoryInner);
pub struct BackupMode<'a>(&'a mut RepositoryInner);
pub struct VacuumMode<'a>(&'a mut RepositoryInner);
macro_rules! in_readonly_mode {
( $($f:tt)* ) => {
impl Repository {
$( $f )*
}
impl<'a> LocalWriteMode<'a> {
$( $f )*
}
impl<'a> RestoreMode<'a> {
$( $f )*
}
impl<'a> BackupMode<'a> {
$( $f )*
}
impl<'a> VacuumMode<'a> {
$( $f )*
}
};
}
macro_rules! in_local_write_mode {
( $($f:tt)* ) => {
impl<'a> LocalWriteMode<'a> {
$( $f )*
}
impl<'a> RestoreMode<'a> {
$( $f )*
}
impl<'a> BackupMode<'a> {
$( $f )*
}
impl<'a> VacuumMode<'a> {
$( $f )*
}
};
}
macro_rules! in_restore_mode {
( $($f:tt)* ) => {
impl<'a> RestoreMode<'a> {
$( $f )*
}
impl<'a> BackupMode<'a> {
$( $f )*
}
impl<'a> VacuumMode<'a> {
$( $f )*
}
};
}
macro_rules! in_backup_mode {
( $($f:tt)* ) => {
impl<'a> BackupMode<'a> {
$( $f )*
}
impl<'a> VacuumMode<'a> {
$( $f )*
}
};
}
macro_rules! in_vacuum_mode {
( $($f:tt)* ) => {
impl<'a> VacuumMode<'a> {
$( $f )*
}
};
}
impl RepositoryInner {
fn local_write_mode<R, F: FnOnce(LocalWriteMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
let ret = f(LocalWriteMode(self));
ret
}
fn restore_mode<R, F: FnOnce(RestoreMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
let ret = f(RestoreMode(self));
ret
}
fn backup_mode<R, F: FnOnce(BackupMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
let ret = f(BackupMode(self));
ret
}
fn vacuum_mode<R, F: FnOnce(VacuumMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
let ret = f(VacuumMode(self));
ret
}
}
impl Repository {
pub fn local_write_mode<R, F: FnOnce(LocalWriteMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
self.0.local_write_mode(f)
}
pub fn restore_mode<R, F: FnOnce(RestoreMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
self.0.restore_mode(f)
}
pub fn backup_mode<R, F: FnOnce(BackupMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
self.0.backup_mode(f)
}
pub fn vacuum_mode<R, F: FnOnce(VacuumMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
self.0.vacuum_mode(f)
}
}
impl<'a> LocalWriteMode<'a> {
pub fn restore_mode<R, F: FnOnce(RestoreMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
self.0.restore_mode(f)
}
pub fn backup_mode<R, F: FnOnce(BackupMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
self.0.backup_mode(f)
}
pub fn vacuum_mode<R, F: FnOnce(VacuumMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
self.0.vacuum_mode(f)
}
}
impl<'a> RestoreMode<'a> {
pub fn backup_mode<R, F: FnOnce(BackupMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
self.0.backup_mode(f)
}
pub fn vacuum_mode<R, F: FnOnce(VacuumMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
self.0.vacuum_mode(f)
}
}
impl<'a> BackupMode<'a> {
pub fn vacuum_mode<R, F: FnOnce(VacuumMode) -> Result<R, RepositoryError>>(&mut self, f: F) -> Result<R, RepositoryError> {
self.0.vacuum_mode(f)
}
}
impl Repository {
fn test(&mut self) {
self.local_write_mode(|s| {
s.dummy("aaa");
Ok(())
});
}
}
in_readonly_mode! {
pub fn get_config(&self) -> &Config {
self.0.get_config()
}
pub fn set_config(&mut self, config: Config) {
self.0.set_config(config);
}
}
in_local_write_mode! {
fn dummy<R>(&self, r: R) {
}
}

218
src/repository/inner/integrity.rs
View File

@ -1,218 +0,0 @@
use prelude::*;
use super::*;
use super::super::bundle_map::BundleMap;
use std::time::Duration;
use pbr::ProgressBar;
quick_error!{
#[derive(Debug)]
pub enum IntegrityError {
MissingChunk(hash: Hash) {
description(tr!("Missing chunk"))
display("{}", tr_format!("Missing chunk: {}", hash))
}
MissingBundleId(id: u32) {
description(tr!("Missing bundle"))
display("{}", tr_format!("Missing bundle: {}", id))
}
MissingBundle(id: BundleId) {
description(tr!("Missing bundle"))
display("{}", tr_format!("Missing bundle: {}", id))
}
NoSuchChunk(bundle: BundleId, chunk: u32) {
description(tr!("No such chunk"))
display("{}", tr_format!("Bundle {} does not contain the chunk {}", bundle, chunk))
}
RemoteBundlesNotInMap {
description(tr!("Remote bundles missing from map"))
}
MapContainsDuplicates {
description(tr!("Map contains duplicates"))
}
}
}
impl RepositoryInner {
pub fn get_chunk_marker(&self) -> Bitmap {
Bitmap::new(self.index.capacity())
}
pub fn check_chunks(
&self,
checked: &mut Bitmap,
chunks: &[Chunk],
mark: bool,
) -> Result<bool, RepositoryError> {
let mut new = false;
for &(hash, _len) in chunks {
if let Some(pos) = self.index.pos(&hash) {
new |= !checked.get(pos);
if mark {
checked.set(pos);
}
} else {
return Err(IntegrityError::MissingChunk(hash).into());
}
}
Ok(new)
}
fn check_index_chunks(&self) -> Result<(), RepositoryError> {
let mut progress = ProgressBar::new(self.index.len() as u64);
progress.message(tr!("checking index: "));
progress.set_max_refresh_rate(Some(Duration::from_millis(100)));
for (count, (_hash, location)) in self.index.iter().enumerate() {
// Lookup bundle id from map
let bundle_id = try!(self.get_bundle_id(location.bundle));
// Get bundle object from bundledb
let bundle = if let Some(bundle) = self.bundles.get_bundle_info(&bundle_id) {
bundle
} else {
progress.finish_print(tr!("checking index: done."));
return Err(IntegrityError::MissingBundle(bundle_id.clone()).into());
};
// Get chunk from bundle
if bundle.info.chunk_count <= location.chunk as usize {
progress.finish_print(tr!("checking index: done."));
return Err(
IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into()
);
}
if count % 1000 == 0 {
progress.set(count as u64);
}
}
progress.finish_print(tr!("checking index: done."));
Ok(())
}
pub fn rebuild_bundle_map(&mut self) -> Result<(), RepositoryError> {
tr_info!("Rebuilding bundle map from bundles");
self.bundle_map = BundleMap::create();
for bundle in self.bundles.list_bundles() {
let bundle_id = match bundle.mode {
BundleMode::Data => self.next_data_bundle,
BundleMode::Meta => self.next_meta_bundle,
};
self.bundle_map.set(bundle_id, bundle.id.clone());
if self.next_meta_bundle == bundle_id {
self.next_meta_bundle = self.next_free_bundle_id()
}
if self.next_data_bundle == bundle_id {
self.next_data_bundle = self.next_free_bundle_id()
}
}
self.save_bundle_map()
}
pub fn rebuild_index(&mut self) -> Result<(), RepositoryError> {
tr_info!("Rebuilding index from bundles");
self.index.clear();
let mut bundles = self.bundle_map.bundles();
bundles.sort_by_key(|&(_, ref v)| v.clone());
for (num, id) in ProgressIter::new(tr!("Rebuilding index from bundles"), bundles.len(), bundles.into_iter()) {
let chunks = try!(self.bundles.get_chunk_list(&id));
for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() {
try!(self.index.set(
&hash,
&Location {
bundle: num as u32,
chunk: i as u32
}
));
}
}
Ok(())
}
#[inline]
pub fn check_index(&mut self, repair: bool) -> Result<(), RepositoryError> {
if repair {
try!(self.write_mode());
}
tr_info!("Checking index integrity...");
if let Err(err) = self.index.check() {
if repair {
tr_warn!(
"Problem detected: index was corrupted\n\tcaused by: {}",
err
);
return self.rebuild_index();
} else {
return Err(err.into());
}
}
tr_info!("Checking index entries...");
if let Err(err) = self.check_index_chunks() {
if repair {
tr_warn!(
"Problem detected: index entries were inconsistent\n\tcaused by: {}",
err
);
return self.rebuild_index();
} else {
return Err(err);
}
}
Ok(())
}
#[inline]
pub fn check_bundles(&mut self, full: bool, repair: bool) -> Result<(), RepositoryError> {
if repair {
try!(self.write_mode());
}
tr_info!("Checking bundle integrity...");
if try!(self.bundles.check(full, repair)) {
// Some bundles got repaired
tr_warn!("Some bundles have been rewritten, please remove the broken bundles manually.");
try!(self.rebuild_bundle_map());
try!(self.rebuild_index());
}
Ok(())
}
pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> {
tr_info!("Checking repository integrity...");
let mut rebuild = false;
for (_id, bundle_id) in self.bundle_map.bundles() {
if self.bundles.get_bundle_info(&bundle_id).is_none() {
if repair {
tr_warn!(
"Problem detected: bundle map contains unknown bundle {}",
bundle_id
);
rebuild = true;
} else {
return Err(IntegrityError::MissingBundle(bundle_id).into());
}
}
}
if self.bundle_map.len() < self.bundles.len() {
if repair {
tr_warn!("Problem detected: bundle map does not contain all remote bundles");
rebuild = true;
} else {
return Err(IntegrityError::RemoteBundlesNotInMap.into());
}
}
if self.bundle_map.len() > self.bundles.len() {
if repair {
tr_warn!("Problem detected: bundle map contains bundles multiple times");
rebuild = true;
} else {
return Err(IntegrityError::MapContainsDuplicates.into());
}
}
if rebuild {
try!(self.rebuild_bundle_map());
try!(self.rebuild_index());
}
Ok(())
}
}

364
src/repository/inner/mod.rs
View File

@ -1,364 +0,0 @@
mod integrity;
mod basic_io;
mod info;
mod vacuum;
pub mod api;
use prelude::*;
use std::mem;
use std::cmp::max;
use std::path::Path;
use std::fs::{self, File};
use std::sync::Arc;
use std::os::unix::fs::symlink;
use std::io::Write;
use super::{INDEX_MAGIC, INDEX_VERSION, REPOSITORY_README};
use super::bundle_map::BundleMap;
use super::index;
pub use self::integrity::IntegrityError;
pub use self::info::{BundleAnalysis, RepositoryInfo, RepositoryStatistics};
#[derive(Clone, Copy, PartialEq, Debug, Default)]
pub struct Location {
pub bundle: u32,
pub chunk: u32
}
impl Location {
pub fn new(bundle: u32, chunk: u32) -> Self {
Location {
bundle,
chunk
}
}
}
impl index::Value for Location {}
impl index::Key for Hash {
fn hash(&self) -> u64 {
self.low
}
fn is_used(&self) -> bool {
self.low != 0 || self.high != 0
}
fn clear(&mut self) {
self.low = 0;
self.high = 0;
}
}
pub struct RepositoryInner {
layout: Arc<ChunkRepositoryLayout>,
config: Config,
index: Index<Hash, Location>,
crypto: Arc<Crypto>,
bundle_map: BundleMap,
next_data_bundle: u32,
next_meta_bundle: u32,
bundles: BundleDb,
data_bundle: Option<BundleWriter>,
meta_bundle: Option<BundleWriter>,
chunker: Box<Chunker>,
remote_locks: LockFolder,
local_locks: LockFolder,
lock: LockHandle,
dirty: bool
}
impl RepositoryInner {
pub fn create<R: AsRef<Path>>(
layout: Arc<ChunkRepositoryLayout>,
config: &Config,
crypto: Arc<Crypto>,
remote: R,
) -> Result<Self, RepositoryError> {
try!(fs::create_dir(layout.local_locks_path()));
try!(symlink(remote, layout.remote_path()));
try!(File::create(layout.remote_readme_path()).and_then(
|mut f| {
f.write_all(REPOSITORY_README)
}
));
try!(fs::create_dir_all(layout.remote_locks_path()));
try!(config.save(layout.config_path()));
try!(BundleDb::create(layout.clone()));
try!(Index::<Hash, Location>::create(
layout.index_path(),
&INDEX_MAGIC,
INDEX_VERSION
));
try!(BundleMap::create().save(layout.bundle_map_path()));
Self::open(layout, crypto, true)
}
#[allow(unknown_lints, useless_let_if_seq)]
pub fn open(layout: Arc<ChunkRepositoryLayout>, crypto: Arc<Crypto>, online: bool) -> Result<Self, RepositoryError> {
if !layout.remote_exists() {
return Err(RepositoryError::NoRemote);
}
let config = try!(Config::load(layout.config_path()));
let remote_locks = LockFolder::new(layout.remote_locks_path());
try!(fs::create_dir_all(layout.local_locks_path())); // Added after v0.1.0
let local_locks = LockFolder::new(layout.local_locks_path());
let lock = try!(local_locks.lock(false));
let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone(), online));
let (index, mut rebuild_index) =
match unsafe { Index::open(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION) } {
Ok(index) => (index, false),
Err(err) => {
tr_error!("Failed to load local index:\n\tcaused by: {}", err);
(
try!(Index::create(
layout.index_path(),
&INDEX_MAGIC,
INDEX_VERSION
)),
true
)
}
};
let (bundle_map, rebuild_bundle_map) = match BundleMap::load(layout.bundle_map_path()) {
Ok(bundle_map) => (bundle_map, false),
Err(err) => {
tr_error!("Failed to load local bundle map:\n\tcaused by: {}", err);
(BundleMap::create(), true)
}
};
let dirty = layout.dirtyfile_path().exists();
let mut repo = RepositoryInner {
layout,
dirty: true,
chunker: config.chunker.create(),
config,
index,
crypto,
bundle_map,
next_data_bundle: 0,
next_meta_bundle: 0,
bundles,
data_bundle: None,
meta_bundle: None,
lock,
remote_locks,
local_locks
};
if !rebuild_bundle_map {
let mut save_bundle_map = false;
if !gone.is_empty() {
tr_info!("Removing {} old bundles from index", gone.len());
try!(repo.write_mode());
for bundle in gone {
try!(repo.remove_gone_remote_bundle(&bundle))
}
save_bundle_map = true;
}
if !new.is_empty() {
tr_info!("Adding {} new bundles to index", new.len());
try!(repo.write_mode());
for bundle in ProgressIter::new(
tr!("adding bundles to index"),
new.len(),
new.into_iter()
)
{
try!(repo.add_new_remote_bundle(&bundle))
}
save_bundle_map = true;
}
if save_bundle_map {
try!(repo.write_mode());
try!(repo.save_bundle_map());
}
}
repo.next_meta_bundle = repo.next_free_bundle_id();
repo.next_data_bundle = repo.next_free_bundle_id();
if rebuild_bundle_map {
try!(repo.write_mode());
try!(repo.rebuild_bundle_map());
rebuild_index = true;
}
if rebuild_index {
try!(repo.write_mode());
try!(repo.rebuild_index());
}
repo.dirty = dirty;
Ok(repo)
}
#[inline]
pub fn save_config(&mut self) -> Result<(), RepositoryError> {
try!(self.write_mode());
try!(self.config.save(self.layout.config_path()));
Ok(())
}
#[inline]
pub fn set_encryption(&mut self, public: Option<&PublicKey>) {
if let Some(key) = public {
if !self.crypto.contains_secret_key(key) {
tr_warn!("The secret key for that public key is not stored in the repository.")
}
let mut key_bytes = Vec::new();
key_bytes.extend_from_slice(&key[..]);
self.config.encryption = Some((EncryptionMethod::Sodium, key_bytes.into()))
} else {
self.config.encryption = None
}
}
#[inline]
pub fn save_bundle_map(&self) -> Result<(), RepositoryError> {
try!(self.bundle_map.save(self.layout.bundle_map_path()));
Ok(())
}
#[inline]
pub fn next_free_bundle_id(&self) -> u32 {
let mut id = max(self.next_data_bundle, self.next_meta_bundle) + 1;
while self.bundle_map.get(id).is_some() {
id += 1;
}
id
}
pub fn set_dirty(&mut self) -> Result<(), RepositoryError> {
self.dirty = true;
let dirtyfile = self.layout.dirtyfile_path();
if !dirtyfile.exists() {
try!(File::create(&dirtyfile));
}
Ok(())
}
pub fn flush(&mut self) -> Result<(), RepositoryError> {
let dirtyfile = self.layout.dirtyfile_path();
if self.dirty && !dirtyfile.exists() {
try!(File::create(&dirtyfile));
}
if self.data_bundle.is_some() {
let mut finished = None;
mem::swap(&mut self.data_bundle, &mut finished);
{
let bundle = try!(self.bundles.add_bundle(finished.unwrap()));
self.bundle_map.set(
self.next_data_bundle,
bundle.id.clone()
);
}
self.next_data_bundle = self.next_free_bundle_id()
}
if self.meta_bundle.is_some() {
let mut finished = None;
mem::swap(&mut self.meta_bundle, &mut finished);
{
let bundle = try!(self.bundles.add_bundle(finished.unwrap()));
self.bundle_map.set(
self.next_meta_bundle,
bundle.id.clone()
);
}
self.next_meta_bundle = self.next_free_bundle_id()
}
try!(self.bundles.flush());
try!(self.save_bundle_map());
if !self.dirty && dirtyfile.exists() {
try!(fs::remove_file(&dirtyfile));
}
Ok(())
}
fn add_new_remote_bundle(&mut self, bundle: &BundleInfo) -> Result<(), RepositoryError> {
if self.bundle_map.find(&bundle.id).is_some() {
return Ok(());
}
tr_debug!("Adding new bundle to index: {}", bundle.id);
let bundle_id = match bundle.mode {
BundleMode::Data => self.next_data_bundle,
BundleMode::Meta => self.next_meta_bundle,
};
let chunks = try!(self.bundles.get_chunk_list(&bundle.id));
self.bundle_map.set(bundle_id, bundle.id.clone());
if self.next_meta_bundle == bundle_id {
self.next_meta_bundle = self.next_free_bundle_id()
}
if self.next_data_bundle == bundle_id {
self.next_data_bundle = self.next_free_bundle_id()
}
for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() {
if let Some(old) = try!(self.index.set(
&hash,
&Location {
bundle: bundle_id as u32,
chunk: i as u32
}
))
{
// Duplicate chunk, forced ordering: higher bundle id wins
let old_bundle_id = try!(self.get_bundle_id(old.bundle));
if old_bundle_id > bundle.id {
try!(self.index.set(&hash, &old));
}
}
}
Ok(())
}
fn remove_gone_remote_bundle(&mut self, bundle: &BundleInfo) -> Result<(), RepositoryError> {
if let Some(id) = self.bundle_map.find(&bundle.id) {
tr_debug!("Removing bundle from index: {}", bundle.id);
try!(self.bundles.delete_local_bundle(&bundle.id));
try!(self.index.filter(|_key, data| data.bundle != id));
self.bundle_map.remove(id);
}
Ok(())
}
#[inline]
pub fn write_mode(&mut self) -> Result<(), RepositoryError> {
try!(self.local_locks.upgrade(&mut self.lock));
Ok(())
}
#[inline]
pub fn lock(&self, exclusive: bool) -> Result<LockHandle, RepositoryError> {
Ok(try!(self.remote_locks.lock(exclusive)))
}
pub fn is_dirty(&self) -> bool {
self.dirty
}
pub fn get_chunk_location(&self, chunk: Hash) -> Option<Location> {
self.index.get(&chunk)
}
#[inline]
pub fn set_clean(&mut self) {
self.dirty = false;
}
pub fn get_config(&self) -> &Config {
&self.config
}
pub fn set_config(&mut self, config: Config) {
self.config = config;
}
}
impl Drop for RepositoryInner {
fn drop(&mut self) {
if let Err(err) = self.flush() {
tr_error!("Failed to flush repository: {}", err);
}
}
}

275
src/repository/integrity.rs Normal file
View File

@ -0,0 +1,275 @@
use prelude::*;
use super::*;
use super::bundle_map::BundleMap;
use super::bundledb::BundleDbError;
use super::index::IndexError;
use std::time::Duration;
use pbr::ProgressBar;
quick_error!{
#[derive(Debug)]
pub enum IntegrityError {
MissingChunk(hash: Hash) {
description(tr!("Missing chunk"))
display("{}", tr_format!("Missing chunk: {}", hash))
}
MissingBundleId(id: u32) {
description(tr!("Missing bundle"))
display("{}", tr_format!("Missing bundle: {}", id))
}
MissingBundle(id: BundleId) {
description(tr!("Missing bundle"))
display("{}", tr_format!("Missing bundle: {}", id))
}
NoSuchChunk(bundle: BundleId, chunk: u32) {
description(tr!("No such chunk"))
display("{}", tr_format!("Bundle {} does not contain the chunk {}", bundle, chunk))
}
RemoteBundlesNotInMap {
description(tr!("Remote bundles missing from map"))
}
MapContainsDuplicates {
description(tr!("Map contains duplicates"))
}
Index(err: IndexError) {
description(tr!("Index error"))
display("{}", tr_format!("Index error: {}", err))
}
BundleIntegrity(id: BundleId, err: BundleDbError) {
description(tr!("Bundle error"))
display("{}", tr_format!("Bundle {} has error: {}", id, err))
}
}
}
pub struct ModuleIntegrityReport {
pub errors_fixed: Vec<IntegrityError>,
pub errors_unfixed: Vec<IntegrityError>
}
pub struct IntegrityReport {
pub bundle_map: Option<ModuleIntegrityReport>,
pub index: Option<ModuleIntegrityReport>,
pub bundles: Option<ModuleIntegrityReport>
}
pub struct ChunkMarker<'a> {
marked: Bitmap,
repo: &'a Repository
}
impl<'a> ChunkMarker<'a> {
pub fn mark_chunks(&mut self, chunks: &[Chunk], set_marked: bool) -> Result<bool, RepositoryError> {
let mut new = false;
for &(hash, _len) in chunks {
if let Some(pos) = self.repo.index.pos(&hash) {
new |= !self.marked.get(pos);
if set_marked {
self.marked.set(pos);
}
} else {
return Err(IntegrityError::MissingChunk(hash).into());
}
}
Ok(new)
}
}
impl Repository {
pub fn get_chunk_marker(&self) -> ChunkMarker {
ChunkMarker {
marked: Bitmap::new( self.index.capacity()),
repo: self
}
}
pub fn check_bundle_map(&mut self) -> ModuleIntegrityReport {
tr_info!("Checking bundle map...");
let mut errors = vec![];
for (_id, bundle_id) in self.bundle_map.bundles() {
if self.bundles.get_bundle_info(&bundle_id).is_none() {
errors.push(IntegrityError::MissingBundle(bundle_id).into());
}
}
if self.bundle_map.len() < self.bundles.len() {
errors.push(IntegrityError::RemoteBundlesNotInMap.into());
}
if self.bundle_map.len() > self.bundles.len() {
errors.push(IntegrityError::MapContainsDuplicates.into());
}
ModuleIntegrityReport { errors_fixed: vec![], errors_unfixed: errors }
}
pub fn rebuild_bundle_map(&mut self, lock: &OnlineMode) -> Result<(), RepositoryError> {
tr_info!("Rebuilding bundle map from bundles");
try!(self.bundles.synchronize(lock));
self.bundle_map = BundleMap::create();
for bundle in self.bundles.list_bundles() {
let bundle_id = match bundle.mode {
BundleMode::Data => self.next_data_bundle,
BundleMode::Meta => self.next_meta_bundle,
};
self.bundle_map.set(bundle_id, bundle.id.clone(), lock.as_localwrite());
if self.next_meta_bundle == bundle_id {
self.next_meta_bundle = self.next_free_bundle_id()
}
if self.next_data_bundle == bundle_id {
self.next_data_bundle = self.next_free_bundle_id()
}
}
self.save_bundle_map(lock.as_localwrite())
}
pub fn check_and_repair_bundle_map(&mut self, lock: &OnlineMode) -> Result<ModuleIntegrityReport, RepositoryError> {
let mut report = self.check_bundle_map();
if !report.errors_unfixed.is_empty() {
try!(self.rebuild_bundle_map(lock));
mem::swap(&mut report.errors_unfixed, &mut report.errors_fixed);
}
Ok(report)
}
fn check_index_chunks(&self) -> Vec<IntegrityError> {
let mut errors = vec![];
let mut progress = ProgressBar::new(self.index.len() as u64);
progress.message(tr!("checking index: "));
progress.set_max_refresh_rate(Some(Duration::from_millis(100)));
for (count, (_hash, location)) in self.index.iter().enumerate() {
// Lookup bundle id from map
let bundle_id = if let Some(bundle_id) = self.bundle_map.get(location.bundle) {
bundle_id
} else {
errors.push(IntegrityError::MissingBundleId(location.bundle));
continue
};
// Get bundle object from bundledb
let bundle = if let Some(bundle) = self.bundles.get_bundle_info(&bundle_id) {
bundle
} else {
errors.push(IntegrityError::MissingBundle(bundle_id.clone()));
continue
};
// Get chunk from bundle
if bundle.info.chunk_count <= location.chunk as usize {
errors.push(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk));
continue
}
if count % 1000 == 0 {
progress.set(count as u64);
}
}
progress.finish_print(tr!("checking index: done."));
errors
}
pub fn rebuild_index(&mut self, lock: &OnlineMode) -> Result<(), RepositoryError> {
tr_info!("Rebuilding index from bundles");
self.index.clear();
let mut bundles = self.bundle_map.bundles();
bundles.sort_by_key(|&(_, ref v)| v.clone());
for (num, id) in ProgressIter::new(tr!("Rebuilding index from bundles"), bundles.len(), bundles.into_iter()) {
let chunks = try!(self.bundles.get_chunk_list(&id, lock));
for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() {
try!(self.index.set(
&hash,
&Location {
bundle: num as u32,
chunk: i as u32
}
));
}
}
Ok(())
}
#[inline]
pub fn check_index(&mut self, lock: &ReadonlyMode) -> ModuleIntegrityReport {
tr_info!("Checking index integrity...");
let mut errors: Vec<IntegrityError> = self.index.check().into_iter().map(IntegrityError::Index).collect();
tr_info!("Checking index entries...");
errors.extend(self.check_index_chunks());
ModuleIntegrityReport { errors_fixed: vec![], errors_unfixed: errors }
}
pub fn check_and_repair_index(&mut self, lock: &OnlineMode) -> Result<ModuleIntegrityReport, RepositoryError> {
let mut report = self.check_index(lock.as_readonly());
if !report.errors_unfixed.is_empty() {
try!(self.rebuild_index(lock));
mem::swap(&mut report.errors_unfixed, &mut report.errors_fixed);
}
Ok(report)
}
#[inline]
fn check_bundles_internal(&mut self, full: bool, lock: &OnlineMode) -> (ModuleIntegrityReport, Vec<BundleId>) {
tr_info!("Checking bundle integrity...");
let mut errors = vec![];
let mut bundles = vec![];
for (id, err) in self.bundles.check(full, lock) {
bundles.push(id);
errors.push(IntegrityError::BundleIntegrity(id, err));
}
(ModuleIntegrityReport { errors_fixed: vec![], errors_unfixed: errors }, bundles)
}
#[inline]
pub fn check_bundles(&mut self, full: bool, lock: &OnlineMode) -> ModuleIntegrityReport {
self.check_bundles_internal(full, lock).0
}
pub fn check_and_repair_bundles(&mut self, full: bool, lock: &VacuumMode) -> Result<ModuleIntegrityReport, RepositoryError> {
let (mut report, bundles) = self.check_bundles_internal(full, lock.as_online());
if !report.errors_unfixed.is_empty() {
try!(self.bundles.repair(lock, &bundles));
mem::swap(&mut report.errors_unfixed, &mut report.errors_fixed);
// Some bundles got repaired
tr_warn!("Some bundles have been rewritten, please remove the broken bundles manually.");
try!(self.rebuild_bundle_map(lock.as_online()));
try!(self.rebuild_index(lock.as_online()));
}
Ok(report)
}
pub fn check(&mut self, index: bool, bundles: bool, bundle_data: bool, lock: &OnlineMode) -> IntegrityReport {
let mut report = IntegrityReport {
bundle_map: None,
index: None,
bundles: None
};
report.bundle_map = Some(self.check_bundle_map());
if index {
report.index = Some(self.check_index(lock.as_readonly()));
}
if bundles {
report.bundles = Some(self.check_bundles(bundle_data, lock));
}
report
}
pub fn check_and_repair(&mut self, index: bool, bundles: bool, bundle_data: bool, lock: &VacuumMode) -> Result<IntegrityReport, RepositoryError> {
let mut report = IntegrityReport {
bundle_map: None,
index: None,
bundles: None
};
let bundle_map = try!(self.check_and_repair_bundle_map(lock.as_online()));
if !bundle_map.errors_fixed.is_empty() {
try!(self.rebuild_index(lock.as_online()));
}
report.bundle_map = Some(bundle_map);
if index {
report.index = Some(try!(self.check_and_repair_index(lock.as_online())));
}
if bundles {
report.bundles = Some(try!(self.check_and_repair_bundles(bundle_data, lock)));
}
Ok(report)
}
}

447
src/repository/mod.rs
View File

@ -2,23 +2,454 @@ mod config;
mod bundle_map;
mod layout;
mod error;
mod inner;
pub mod bundledb;
pub mod index;
pub mod chunking;
mod integrity;
mod basic_io;
mod info;
mod vacuum;
use prelude::*;
use std::mem;
use std::cmp::max;
use std::path::Path;
use std::fs::{self, File};
use std::sync::Arc;
use std::os::unix::fs::symlink;
use std::io::Write;
pub use self::error::RepositoryError;
pub use self::config::Config;
pub use self::inner::IntegrityError;
pub use self::inner::{RepositoryInfo, BundleAnalysis, RepositoryStatistics};
pub use self::layout::{RepositoryLayout, ChunkRepositoryLayout};
pub use self::inner::{Location, RepositoryInner};
pub use self::inner::api::*;
use self::bundle_map::BundleMap;
pub use self::integrity::IntegrityError;
pub use self::info::{BundleAnalysis, RepositoryInfo, RepositoryStatistics};
const REPOSITORY_README: &[u8] = include_bytes!("../../docs/repository_readme.md");
const INDEX_MAGIC: [u8; 7] = *b"zvault\x02";
const INDEX_VERSION: u8 = 1;
const INDEX_VERSION: u8 = 1;
#[derive(Clone, Copy, PartialEq, Debug, Default)]
pub struct Location {
pub bundle: u32,
pub chunk: u32
}
impl Location {
pub fn new(bundle: u32, chunk: u32) -> Self {
Location {
bundle,
chunk
}
}
}
impl index::Value for Location {}
impl index::Key for Hash {
fn hash(&self) -> u64 {
self.low
}
fn is_used(&self) -> bool {
self.low != 0 || self.high != 0
}
fn clear(&mut self) {
self.low = 0;
self.high = 0;
}
}
pub struct Repository {
layout: Arc<ChunkRepositoryLayout>,
config: Config,
index: Index<Hash, Location>,
crypto: Arc<Crypto>,
bundle_map: BundleMap,
next_data_bundle: u32,
next_meta_bundle: u32,
bundles: BundleDb,
data_bundle: Option<BundleWriter>,
meta_bundle: Option<BundleWriter>,
chunker: Box<Chunker>,
remote_locks: LockFolder,
local_locks: LockFolder,
}
impl Repository {
pub fn create<R: AsRef<Path>>(
layout: Arc<ChunkRepositoryLayout>,
config: &Config,
crypto: Arc<Crypto>,
remote: R,
) -> Result<Self, RepositoryError> {
try!(fs::create_dir(layout.local_locks_path()));
try!(symlink(remote, layout.remote_path()));
try!(File::create(layout.remote_readme_path()).and_then(
|mut f| {
f.write_all(REPOSITORY_README)
}
));
try!(fs::create_dir_all(layout.remote_locks_path()));
let mock_lock = Lock;
try!(config.save(layout.config_path(), &mock_lock));
try!(BundleDb::create(layout.clone()));
try!(Index::<Hash, Location>::create(
layout.index_path(),
&INDEX_MAGIC,
INDEX_VERSION
));
try!(BundleMap::create().save(layout.bundle_map_path(), &mock_lock));
Self::open(layout, crypto, true)
}
#[allow(unknown_lints, useless_let_if_seq)]
pub fn open(layout: Arc<ChunkRepositoryLayout>, crypto: Arc<Crypto>, read_only: bool) -> Result<Self, RepositoryError> {
if !layout.remote_exists() {
return Err(RepositoryError::NoRemote);
}
let config = try!(Config::load(layout.config_path()));
let remote_locks = LockFolder::new(layout.remote_locks_path());
try!(fs::create_dir_all(layout.local_locks_path())); // Added after v0.1.0
let local_locks = LockFolder::new(layout.local_locks_path());
let lock = try!(local_locks.lock(false));
let mock_lock = Lock;
let bundles = try!(BundleDb::open(layout.clone(), crypto.clone(), &mock_lock));
let mut rebuild_index = false;
let mut rebuild_bundle_map = false;
let index = match unsafe { Index::open(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION) } {
Ok(index) => index,
Err(err) => {
tr_error!("Failed to load local index:\n\tcaused by: {}", err);
if read_only {
return Err(err.into());
}
try!(Index::create(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION))
}
};
let bundle_map = match BundleMap::load(layout.bundle_map_path(), &mock_lock) {
Ok(bundle_map) => bundle_map,
Err(err) => {
tr_error!("Failed to load local bundle map:\n\tcaused by: {}", err);
if read_only {
return Err(err.into());
}
BundleMap::create()
}
};
let mut repo = Repository {
layout,
chunker: config.chunker.create(),
config,
index,
crypto,
bundle_map,
next_data_bundle: 0,
next_meta_bundle: 0,
bundles,
data_bundle: None,
meta_bundle: None,
remote_locks,
local_locks
};
if rebuild_bundle_map {
try!(repo.rebuild_bundle_map(&mock_lock));
rebuild_index = true;
}
if rebuild_index {
try!(repo.rebuild_index(&mock_lock));
}
repo.next_meta_bundle = repo.next_free_bundle_id();
repo.next_data_bundle = repo.next_free_bundle_id();
Ok(repo)
}
pub fn synchronize(&mut self, lock: &OnlineMode) -> Result<(), RepositoryError> {
let (new, gone) = try!(self.bundles.synchronize(lock));
let mut save_bundle_map = false;
if !gone.is_empty() {
tr_info!("Removing {} old bundles from index", gone.len());
for bundle in gone {
try!(self.remove_gone_remote_bundle(&bundle, lock.as_localwrite()))
}
save_bundle_map = true;
}
if !new.is_empty() {
tr_info!("Adding {} new bundles to index", new.len());
for bundle in ProgressIter::new(tr!("adding bundles to index"), new.len(), new.into_iter()) {
try!(self.add_new_remote_bundle(&bundle, lock))
}
save_bundle_map = true;
}
if save_bundle_map {
try!(self.save_bundle_map(lock.as_localwrite()));
}
self.next_meta_bundle = self.next_free_bundle_id();
self.next_data_bundle = self.next_free_bundle_id();
Ok(())
}
#[inline]
pub fn save_config(&mut self, lock: &LocalWriteMode) -> Result<(), RepositoryError> {
try!(self.config.save(self.layout.config_path(), lock));
Ok(())
}
#[inline]
pub fn set_encryption(&mut self, public: Option<&PublicKey>) {
if let Some(key) = public {
if !self.crypto.contains_secret_key(key) {
tr_warn!("The secret key for that public key is not stored in the repository.")
}
let mut key_bytes = Vec::new();
key_bytes.extend_from_slice(&key[..]);
self.config.encryption = Some((EncryptionMethod::Sodium, key_bytes.into()))
} else {
self.config.encryption = None
}
}
#[inline]
pub fn save_bundle_map(&self, lock: &LocalWriteMode) -> Result<(), RepositoryError> {
try!(self.bundle_map.save(self.layout.bundle_map_path(), lock));
Ok(())
}
#[inline]
pub fn next_free_bundle_id(&self) -> u32 {
let mut id = max(self.next_data_bundle, self.next_meta_bundle) + 1;
while self.bundle_map.get(id).is_some() {
id += 1;
}
id
}
pub fn flush(&mut self, lock: &BackupMode) -> Result<(), RepositoryError> {
if self.data_bundle.is_some() {
let mut finished = None;
mem::swap(&mut self.data_bundle, &mut finished);
{
let bundle = try!(self.bundles.add_bundle(finished.unwrap(), lock));
self.bundle_map.set(
self.next_data_bundle,
bundle.id.clone(),
lock.as_localwrite()
);
}
self.next_data_bundle = self.next_free_bundle_id()
}
if self.meta_bundle.is_some() {
let mut finished = None;
mem::swap(&mut self.meta_bundle, &mut finished);
{
let bundle = try!(self.bundles.add_bundle(finished.unwrap(), lock));
self.bundle_map.set(
self.next_meta_bundle,
bundle.id.clone(),
lock.as_localwrite()
);
}
self.next_meta_bundle = self.next_free_bundle_id()
}
try!(self.bundles.flush(lock));
try!(self.save_bundle_map(lock.as_localwrite()));
Ok(())
}
fn add_new_remote_bundle(&mut self, bundle: &BundleInfo, lock: &OnlineMode) -> Result<(), RepositoryError> {
if self.bundle_map.find(&bundle.id).is_some() {
return Ok(());
}
tr_debug!("Adding new bundle to index: {}", bundle.id);
let bundle_id = match bundle.mode {
BundleMode::Data => self.next_data_bundle,
BundleMode::Meta => self.next_meta_bundle,
};
let chunks = try!(self.bundles.get_chunk_list(&bundle.id, lock));
self.bundle_map.set(bundle_id, bundle.id.clone(), lock.as_localwrite());
if self.next_meta_bundle == bundle_id {
self.next_meta_bundle = self.next_free_bundle_id()
}
if self.next_data_bundle == bundle_id {
self.next_data_bundle = self.next_free_bundle_id()
}
for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() {
if let Some(old) = try!(self.index.set(
&hash,
&Location {
bundle: bundle_id as u32,
chunk: i as u32
}
))
{
// Duplicate chunk, forced ordering: higher bundle id wins
let old_bundle_id = try!(self.get_bundle_id(old.bundle));
if old_bundle_id > bundle.id {
try!(self.index.set(&hash, &old));
}
}
}
Ok(())
}
fn remove_gone_remote_bundle(&mut self, bundle: &BundleInfo, lock: &LocalWriteMode) -> Result<(), RepositoryError> {
if let Some(id) = self.bundle_map.find(&bundle.id) {
tr_debug!("Removing bundle from index: {}", bundle.id);
try!(self.bundles.delete_local_bundle(&bundle.id, lock));
try!(self.index.filter(|_key, data| data.bundle != id));
self.bundle_map.remove(id, lock);
}
Ok(())
}
pub fn get_chunk_location(&self, chunk: Hash) -> Option<Location> {
self.index.get(&chunk)
}
pub fn get_config(&self) -> &Config {
&self.config
}
pub fn set_config(&mut self, config: Config) {
self.config = config;
}
}
struct Lock;
/**
- Local: readonly, shared lock
- Remote: offline
**/
pub trait ReadonlyMode {}
impl ReadonlyMode for Lock {}
/**
- Local: writable, exclusive lock, dirty flag
- Remote: offline
**/
pub trait LocalWriteMode: ReadonlyMode {
fn as_readonly(&self) -> &ReadonlyMode;
}
impl LocalWriteMode for Lock {
fn as_readonly(&self) -> &ReadonlyMode {
self
}
}
/**
- Local: writable, exclusive lock, dirty flag
- Remote: readonly, shared lock
**/
pub trait OnlineMode: LocalWriteMode {
fn as_localwrite(&self) -> &LocalWriteMode;
}
impl OnlineMode for Lock {
fn as_localwrite(&self) -> &LocalWriteMode {
self
}
}
/**
- Local: writable, exclusive lock, dirty flag
- Remote: append-only, shared lock
**/
pub trait BackupMode: OnlineMode {
fn as_online(&self) -> &OnlineMode;
}
impl BackupMode for Lock {
fn as_online(&self) -> &OnlineMode {
self
}
}
/**
- Local: writable, exclusive lock, dirty flag
- Remote: writable, exclusive lock
**/
pub trait VacuumMode: BackupMode {
fn as_backup(&self) -> &BackupMode;
}
impl VacuumMode for Lock {
fn as_backup(&self) -> &BackupMode {
self
}
}
impl Repository {
fn create_dirty_file(&mut self) -> Result<(), RepositoryError> {
let dirtyfile = self.layout.dirtyfile_path();
if !dirtyfile.exists() {
try!(File::create(&dirtyfile));
Ok(())
} else {
Err(RepositoryError::Dirty)
}
}
fn delete_dirty_file(&mut self) -> Result<(), RepositoryError> {
let dirtyfile = self.layout.dirtyfile_path();
if dirtyfile.exists() {
try!(fs::remove_file(&dirtyfile));
}
Ok(())
}
pub fn readonly_mode<R, F: FnOnce(&mut Repository, &ReadonlyMode) -> Result<R, RepositoryError>> (&mut self, f: F) -> Result<R, RepositoryError> {
let _local_lock = try!(self.local_locks.lock(false));
f(self, &Lock)
}
pub fn localwrite_mode<R, F: FnOnce(&mut Repository, &LocalWriteMode) -> Result<R, RepositoryError>> (&mut self, f: F) -> Result<R, RepositoryError> {
let _local_lock = try!(self.local_locks.lock(true));
f(self, &Lock)
}
pub fn online_mode<R, F: FnOnce(&mut Repository, &OnlineMode) -> Result<R, RepositoryError>> (&mut self, f: F) -> Result<R, RepositoryError> {
let _local_lock = try!(self.local_locks.lock(true));
let _remote_lock = try!(self.remote_locks.lock(false));
f(self, &Lock)
}
pub fn backup_mode<R, F: FnOnce(&mut Repository, &BackupMode) -> Result<R, RepositoryError>> (&mut self, f: F) -> Result<R, RepositoryError> {
let _local_lock = try!(self.local_locks.lock(true));
let _remote_lock = try!(self.remote_locks.lock(false));
try!(self.create_dirty_file());
let res = f(self, &Lock);
try!(self.flush(&Lock));
if res.is_ok() {
try!(self.delete_dirty_file());
}
res
}
pub fn vacuum_mode<R, F: FnOnce(&mut Repository, &VacuumMode) -> Result<R, RepositoryError>> (&mut self, f: F) -> Result<R, RepositoryError> {
let _local_lock = try!(self.local_locks.lock(true));
let _remote_lock = try!(self.remote_locks.lock(true));
try!(self.create_dirty_file());
let res = f(self, &Lock);
try!(self.flush(&Lock));
if res.is_ok() {
try!(self.delete_dirty_file());
}
res
}
}

22
src/repository/inner/vacuum.rs → src/repository/vacuum.rs
View File

@ -3,17 +3,17 @@ use prelude::*;
use std::collections::HashMap;
impl RepositoryInner {
pub fn delete_bundle(&mut self, id: u32) -> Result<(), RepositoryError> {
if let Some(bundle) = self.bundle_map.remove(id) {
try!(self.bundles.delete_bundle(&bundle));
impl Repository {
pub fn delete_bundle(&mut self, id: u32, lock: &VacuumMode) -> Result<(), RepositoryError> {
if let Some(bundle) = self.bundle_map.remove(id, lock.as_localwrite()) {
try!(self.bundles.delete_bundle(&bundle, lock));
Ok(())
} else {
Err(IntegrityError::MissingBundleId(id).into())
}
}
pub fn rewrite_bundles(&mut self, rewrite_bundles: &[u32], usage: &HashMap<u32, BundleAnalysis>) -> Result<(), RepositoryError> {
pub fn rewrite_bundles(&mut self, rewrite_bundles: &[u32], usage: &HashMap<u32, BundleAnalysis>, lock: &VacuumMode) -> Result<(), RepositoryError> {
for &id in ProgressIter::new(
tr!("rewriting bundles"),
rewrite_bundles.len(),
@ -22,18 +22,18 @@ impl RepositoryInner {
{
let bundle = &usage[&id];
let bundle_id = self.bundle_map.get(id).unwrap();
let chunks = try!(self.bundles.get_chunk_list(&bundle_id));
let chunks = try!(self.bundles.get_chunk_list(&bundle_id, lock.as_online()));
let mode = usage[&id].info.mode;
for (chunk, &(hash, _len)) in chunks.into_iter().enumerate() {
if !bundle.chunk_usage.get(chunk) {
try!(self.index.delete(&hash));
continue;
}
let data = try!(self.bundles.get_chunk(&bundle_id, chunk));
try!(self.put_chunk_override(mode, hash, &data));
let data = try!(self.bundles.get_chunk(&bundle_id, chunk, lock.as_online()));
try!(self.put_chunk_override(mode, hash, &data, lock.as_backup()));
}
}
try!(self.flush());
try!(self.flush(lock.as_backup()));
tr_info!("Checking index");
for (hash, location) in self.index.iter() {
let loc_bundle = location.bundle;
@ -49,9 +49,9 @@ impl RepositoryInner {
}
tr_info!("Deleting {} bundles", rewrite_bundles.len());
for &id in rewrite_bundles {
try!(self.delete_bundle(id));
try!(self.delete_bundle(id, lock));
}
try!(self.save_bundle_map());
try!(self.save_bundle_map(lock.as_localwrite()));
Ok(())
}
}

1
src/util/mod.rs
View File

@ -10,6 +10,7 @@ mod hostname;
mod fs;
mod lock;
mod statistics;
mod mode_test;
pub mod msgpack;

205
src/util/mode_test.rs
View File

@ -23,202 +23,95 @@ VacuumMode
**/
pub enum RepositoryError {
Error
pub struct Repository {
}
enum LockMode {
None, Shared, Exclusive
}
struct RepositoryInner {
}
impl RepositoryInner {
fn set_local_lock(&mut self, mode: LockMode) -> Result<(), RepositoryError> {
Ok(())
impl Repository {
pub fn readonly_mode<R, F: FnOnce(&mut Repository, &ReadonlyMode) -> R> (&mut self, f: F) -> R {
f(self, &Lock)
}
fn set_remote_lock(&mut self, mode: LockMode) -> Result<(), RepositoryError> {
Ok(())
pub fn localwrite_mode<R, F: FnOnce(&mut Repository, &LocalWriteMode) -> R> (&mut self, f: F) -> R {
f(self, &Lock)
}
fn set_dirty(&mut self, dirty: bool) -> Result<(), RepositoryError> {
Ok(())
pub fn online_mode<R, F: FnOnce(&mut Repository, &OnlineMode) -> R> (&mut self, f: F) -> R {
f(self, &Lock)
}
}
pub fn backup_mode<R, F: FnOnce(&mut Repository, &BackupMode) -> R> (&mut self, f: F) -> R {
f(self, &Lock)
}
pub trait ReadonlyMode {
fn func1(&self) -> Result<(), RepositoryError>;
}
impl ReadonlyMode for RepositoryInner {
fn func1(&self) -> Result<(), RepositoryError> {
Ok(())
pub fn vacuum_mode<R, F: FnOnce(&mut Repository, &VacuumMode) -> R> (&mut self, f: F) -> R {
f(self, &Lock)
}
}
struct Lock;
pub trait ReadonlyMode {}
impl ReadonlyMode for Lock {}
pub trait LocalWriteMode: ReadonlyMode {
fn func2(&self) -> Result<(), RepositoryError>;
fn as_readonly(&self) -> &ReadonlyMode;
}
impl LocalWriteMode for RepositoryInner {
fn func2(&self) -> Result<(), RepositoryError> {
Ok(())
impl LocalWriteMode for Lock {
fn as_readonly(&self) -> &ReadonlyMode {
self
}
}
pub trait OnlineMode: LocalWriteMode {
fn as_localwrite(&self) -> &LocalWriteMode;
}
impl OnlineMode for RepositoryInner {
impl OnlineMode for Lock {
fn as_localwrite(&self) -> &LocalWriteMode {
self
}
}
pub trait BackupMode: OnlineMode {
fn as_online(&self) -> &OnlineMode;
}
impl BackupMode for RepositoryInner {
impl BackupMode for Lock {
fn as_online(&self) -> &OnlineMode {
self
}
}
pub trait VacuumMode: BackupMode {
fn as_backup(&self) -> &BackupMode;
}
impl VacuumMode for RepositoryInner {
}
pub trait UpgradeToLocalWriteMode {
fn in_local_write_mode<R, E: From<RepositoryError>, F: FnOnce(&mut LocalWriteMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E>;
}
impl UpgradeToLocalWriteMode for RepositoryInner {
fn in_local_write_mode<R, E: From<RepositoryError>, F: FnOnce(&mut LocalWriteMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E> {
try!(self.set_local_lock(LockMode::Exclusive));
try!(self.set_dirty(true));
let res = f(self);
if res.is_ok() {
try!(self.set_dirty(false));
}
try!(self.set_local_lock(LockMode::Shared));
res
}
}
pub trait UpgradeToOnlineMode {
fn in_online_mode<R, E: From<RepositoryError>, F: FnOnce(&mut OnlineMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E>;
}
impl UpgradeToOnlineMode for RepositoryInner {
fn in_online_mode<R, E: From<RepositoryError>, F: FnOnce(&mut OnlineMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E> {
try!(self.set_local_lock(LockMode::Exclusive));
try!(self.set_remote_lock(LockMode::Shared));
try!(self.set_dirty(true));
let res = f(self);
if res.is_ok() {
try!(self.set_dirty(false));
}
try!(self.set_remote_lock(LockMode::None));
try!(self.set_local_lock(LockMode::Shared));
res
}
}
pub trait UpgradeToBackupMode {
fn in_backup_mode<R, E: From<RepositoryError>, F: FnOnce(&mut BackupMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E>;
}
impl UpgradeToBackupMode for RepositoryInner {
fn in_backup_mode<R, E: From<RepositoryError>, F: FnOnce(&mut BackupMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E> {
try!(self.set_local_lock(LockMode::Exclusive));
try!(self.set_remote_lock(LockMode::Shared));
try!(self.set_dirty(true));
let res = f(self);
if res.is_ok() {
try!(self.set_dirty(false));
}
try!(self.set_remote_lock(LockMode::None));
try!(self.set_local_lock(LockMode::Shared));
res
}
}
pub trait UpgradeToVacuumMode {
fn in_vacuum_mode<R, E: From<RepositoryError>, F: FnOnce(&mut VacuumMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E>;
}
impl UpgradeToVacuumMode for RepositoryInner {
fn in_vacuum_mode<R, E: From<RepositoryError>, F: FnOnce(&mut VacuumMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E> {
try!(self.set_local_lock(LockMode::Exclusive));
try!(self.set_remote_lock(LockMode::Exclusive));
try!(self.set_dirty(true));
let res = f(self);
if res.is_ok() {
try!(self.set_dirty(false));
}
try!(self.set_remote_lock(LockMode::None));
try!(self.set_local_lock(LockMode::Shared));
res
}
}
pub struct Repository(RepositoryInner);
impl ReadonlyMode for Repository {
fn func1(&self) -> Result<(), RepositoryError> {
self.0.func1()
}
}
impl UpgradeToLocalWriteMode for Repository {
fn in_local_write_mode<R, E: From<RepositoryError>, F: FnOnce(&mut LocalWriteMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E> {
self.0.in_local_write_mode(f)
}
}
impl UpgradeToOnlineMode for Repository {
fn in_online_mode<R, E: From<RepositoryError>, F: FnOnce(&mut OnlineMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E> {
self.0.in_online_mode(f)
}
}
impl UpgradeToBackupMode for Repository {
fn in_backup_mode<R, E: From<RepositoryError>, F: FnOnce(&mut BackupMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E> {
self.0.in_backup_mode(f)
}
}
impl UpgradeToVacuumMode for Repository {
fn in_vacuum_mode<R, E: From<RepositoryError>, F: FnOnce(&mut VacuumMode) -> Result<R, E>>(&mut self, f: F) -> Result<R, E> {
self.0.in_vacuum_mode(f)
impl VacuumMode for Lock {
fn as_backup(&self) -> &BackupMode {
self
}
}
impl Repository {
fn write<W: ::std::io::Write>(&mut self, w: W, lock: &LocalWriteMode) {
}
}
fn test_it(mut repo: Repository) {
repo.func1();
repo.in_local_write_mode(|repo| repo.func2());
fn test(&mut self) {
self.localwrite_mode(|repo, lock| {
repo.write(&mut Vec::new(), lock)
});
self.online_mode(|repo, lock| {
repo.write(&mut Vec::new(), lock.as_localwrite())
});
}
}