diff --git a/CHANGELOG.md b/CHANGELOG.md index 20d7a1c..8c6c92c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ This project follows [semantic versioning](http://semver.org). ### UNRELEASED * [added] Translation infrastructure (**requires nightly rust**) +* [added] Internal refactoring of lock levels +* [added] Internal refactoring of check ans repair * [added] Checking hashes of chunks in check --bundle-data * [added] Debian packet for libsodium23 * [modified] Updated dependencies diff --git a/Cargo.lock b/Cargo.lock index 9991b33..e198eaa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ dependencies = [ [[package]] name = "atty" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", @@ -39,7 +39,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "bitflags" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -53,22 +53,22 @@ dependencies = [ [[package]] name = "byteorder" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "cfg-if" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "chrono" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -77,12 +77,12 @@ version = "2.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -95,12 +95,27 @@ name = "crossbeam" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "dtoa" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "filetime" version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "filetime" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -110,7 +125,7 @@ name = "fuchsia-zircon" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -126,9 +141,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)", "thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -166,7 +181,7 @@ version = "0.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -181,7 +196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -198,7 +213,7 @@ name = "log" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -230,10 +245,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "num-integer" -version = "0.1.36" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -241,28 +256,29 @@ name = "num-traits" version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "num-traits" -version = "0.2.2" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "pbr" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "pkg-config" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -295,19 +311,19 @@ dependencies = [ [[package]] name = "regex" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", "thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "regex-syntax" -version = "0.5.3" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -315,7 +331,7 @@ dependencies = [ [[package]] name = "remove_dir_all" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -326,7 +342,7 @@ name = "rmp" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -335,9 +351,9 @@ name = "rmp-serde" version = "0.13.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "rmp 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -347,7 +363,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde" -version = "1.0.37" +version = "1.0.55" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -355,7 +371,7 @@ name = "serde_bytes" version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -363,18 +379,18 @@ name = "serde_utils" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)", "serde_bytes 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_yaml" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "linked-hash-map 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)", "yaml-rust 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -385,7 +401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", "libsodium-sys 0.0.16 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -395,7 +411,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -405,13 +421,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "tar" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "filetime 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", + "filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", - "xattr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "xattr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -420,7 +436,7 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "remove_dir_all 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -457,7 +473,7 @@ dependencies = [ [[package]] name = "time" -version = "0.1.39" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", @@ -498,7 +514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "vec_map" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -535,14 +551,6 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "xattr" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "xattr" version = "0.2.1" @@ -565,8 +573,8 @@ version = "0.5.0" dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "chrono 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "filetime 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", @@ -578,20 +586,20 @@ dependencies = [ "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", - "pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pbr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "rmp-serde 0.13.7 (registry+https://github.com/rust-lang/crates.io-index)", "runtime-fmt 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)", "serde_bytes 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde_utils 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_yaml 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_yaml 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", "sodiumoxide 0.0.16 (registry+https://github.com/rust-lang/crates.io-index)", "squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tar 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.15 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", "users 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "xattr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -600,17 +608,19 @@ dependencies = [ "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" -"checksum atty 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "af80143d6f7608d746df1520709e5d141c96f240b0e62b0aa41bdfb53374d9d4" +"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1" "checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" -"checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf" +"checksum bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d0c54bb8f454c567f21197eefcdbf5679d0bd99f2ddbe52e84c77061952e6789" "checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" -"checksum byteorder 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "73b5bdfe7ee3ad0b99c9801d58807a9dbc9e09196365b0203853b99889ab3c87" -"checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" -"checksum chrono 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ba5f60682a4c264e7f8d77b82e7788938a76befdf949d4a98026d19099c9d873" +"checksum byteorder 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "74c0b906e9446b0a2e4f760cdb3fa4b2c48cdc6db8766a845c54b6ff063fd2e9" +"checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18" +"checksum chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1cce36c92cb605414e9b824f866f5babe0a0368e39ea07393b9b63cf3844c0e6" "checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" "checksum crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "24ce9782d4d5c53674646a6a4c1863a21a8fc0cb649b3c94dfc16e45071dea19" +"checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab" "checksum filetime 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "714653f3e34871534de23771ac7b26e999651a0a228f47beb324dfdf1dd4b10f" +"checksum filetime 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "da4b9849e77b13195302c174324b5ba73eec9b236b24c221a61000daefb95c5f" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" "checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum fuse 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80e57070510966bfef93662a81cb8aa2b1c7db0964354fa9921434f04b9e8660" @@ -628,47 +638,46 @@ dependencies = [ "checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823" "checksum murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a2983372caf4480544083767bf2d27defafe32af49ab4df3a0b7fc90793a3664" "checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2" -"checksum num-integer 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f8d26da319fb45674985c78f1d1caf99aa4941f785d384a2ae36d0740bc3e2fe" +"checksum num-integer 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)" = "6ac0ea58d64a89d9d6b7688031b3be9358d6c919badcf7fbb0527ccfd891ee45" "checksum num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" -"checksum num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dee092fcdf725aee04dd7da1d21debff559237d49ef1cb3e69bcb8ece44c7364" -"checksum pbr 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e048e3afebb6c454bb1c5d0fe73fda54698b4715d78ed8e7302447c37736d23a" -"checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903" +"checksum num-traits 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "775393e285254d2f5004596d69bb8bc1149754570dcc08cf30cabeba67955e28" +"checksum pbr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "deb73390ab68d81992bd994d145f697451bb0b54fd39738e72eef32458ad6907" +"checksum pkg-config 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "110d5ee3593dbb73f56294327fe5668bcc997897097cbc76b51e7aed3f52452f" "checksum quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "eda5fe9b71976e62bc81b781206aaa076401769b2143379d3eb2118388babac4" "checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" "checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" -"checksum regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "aec3f58d903a7d2a9dc2bf0e41a746f4530e0cab6b615494e058f67a3ef947fb" -"checksum regex-syntax 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b2550876c31dc914696a6c2e01cbce8afba79a93c8ae979d2fe051c0230b3756" -"checksum remove_dir_all 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dfc5b3ce5d5ea144bb04ebd093a9e14e9765bcfec866aecda9b6dec43b3d1e24" +"checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384" +"checksum regex-syntax 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7d707a4fa2637f2dca2ef9fd02225ec7661fe01a53623c1e6515b6916511f7a7" +"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" "checksum rmp 0.8.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a3d45d7afc9b132b34a2479648863aa95c5c88e98b32285326a6ebadc80ec5c9" "checksum rmp-serde 0.13.7 (registry+https://github.com/rust-lang/crates.io-index)" = "011e1d58446e9fa3af7cdc1fb91295b10621d3ac4cb3a85cc86385ee9ca50cd3" "checksum runtime-fmt 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "647a821d66049faccc993fc3c379d1181b81a484097495cda79ffdb17b55b87f" -"checksum serde 1.0.37 (registry+https://github.com/rust-lang/crates.io-index)" = "d3bcee660dcde8f52c3765dd9ca5ee36b4bf35470a738eb0bd5a8752b0389645" +"checksum serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)" = "97f6a6c3caba0cf8f883b53331791036404ce3c1bd895961cf8bb2f8cecfd84b" "checksum serde_bytes 0.10.4 (registry+https://github.com/rust-lang/crates.io-index)" = "adb6e51a6b3696b301bc221d785f898b4457c619b51d7ce195a6d20baecb37b3" "checksum serde_utils 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f6e0edb364c93646633800df969086bc7c5c25fb3f1eb57349990d1cb4cae4bc" -"checksum serde_yaml 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e0f868d400d9d13d00988da49f7f02aeac6ef00f11901a8c535bd59d777b9e19" +"checksum serde_yaml 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)" = "107bb818146aaf922e7bbcf6a940f1db2f0dcf381779b451e400331b2c6f86db" "checksum sodiumoxide 0.0.16 (registry+https://github.com/rust-lang/crates.io-index)" = "eb5cb2f14f9a51352ad65e59257a0a9459d5a36a3615f3d53a974c82fdaaa00a" "checksum squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db1f9dde91d819b7746e153bc32489fa19e6a106c3d7f2b92187a4efbdc88b40" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" -"checksum tar 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)" = "1605d3388ceb50252952ffebab4b5dc43017ead7e4481b175961c283bb951195" +"checksum tar 0.4.15 (registry+https://github.com/rust-lang/crates.io-index)" = "6af6b94659f9a571bf769a5b71f54079393585ee0bfdd71b691be22d7d6b1d18" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" "checksum thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bcbb6aa301e5d3b0b5ef639c9a9c7e2f1c944f177b460c04dc24c69b1fa2bd99" "checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" -"checksum time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "a15375f1df02096fb3317256ce2cee6a1f42fc84ea5ad5fc8c421cfe40c73098" +"checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" "checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" "checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum users 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a098d836637f965bbe0df8f744088318c43b685ffd46b676ed21036b7c94bae6" "checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" -"checksum vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "887b5b631c2ad01628bbbaa7dd4c869f80d3186688f8d0b6f58774fbe324988c" +"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" "checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "04e3bd221fcbe8a271359c04f21a76db7d0c6028862d1bb5512d85e1e2eb5bb3" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum xattr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "5f04de8a1346489a2f9e9bd8526b73d135ec554227b17568456e86aa35b6f3fc" "checksum xattr 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "abb373b92de38a4301d66bec009929b4fb83120ea1c4a401be89dbe0b9777443" "checksum yaml-rust 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57ab38ee1a4a266ed033496cf9af1828d8d6e6c1cfa5f643a2809effcae4d628" diff --git a/src/backups/integrity.rs b/src/backups/integrity.rs index a5d6b78..1211d8e 100644 --- a/src/backups/integrity.rs +++ b/src/backups/integrity.rs @@ -31,14 +31,14 @@ impl BackupRepository { None | Some(FileData::Inline(_)) => (), Some(FileData::ChunkedDirect(ref chunks)) => { - try!(self.repo.check_chunks(checked, chunks, true)); + try!(self.repo.mark_chunks(checked, chunks, true)); } Some(FileData::ChunkedIndirect(ref chunks)) => { - if try!(self.repo.check_chunks(checked, chunks, false)) { + if try!(self.repo.mark_chunks(checked, chunks, false)) { let chunk_data = try!(self.get_data(chunks)); let chunks2 = ChunkList::read_from(&chunk_data); - try!(self.repo.check_chunks(checked, &chunks2, true)); - try!(self.repo.check_chunks(checked, chunks, true)); + try!(self.repo.mark_chunks(checked, &chunks2, true)); + try!(self.repo.mark_chunks(checked, chunks, true)); } } } @@ -53,7 +53,7 @@ impl BackupRepository { repair: bool, ) -> Result, RepositoryError> { let mut modified = false; - match self.repo.check_chunks(checked, chunks, false) { + match self.repo.mark_chunks(checked, chunks, false) { Ok(false) => return Ok(None), Ok(true) => (), Err(err) => return Err(InodeIntegrityError::BrokenInode(path, Box::new(err)).into()), @@ -108,7 +108,7 @@ impl BackupRepository { if modified { Ok(Some(try!(self.put_inode(&inode)))) } else { - try!(self.repo.check_chunks(checked, chunks, true)); + try!(self.repo.mark_chunks(checked, chunks, true)); Ok(None) } } diff --git a/src/backups/mod.rs b/src/backups/mod.rs index a203969..650544f 100644 --- a/src/backups/mod.rs +++ b/src/backups/mod.rs @@ -28,7 +28,7 @@ const DEFAULT_EXCLUDES: &[u8] = include_bytes!("../../docs/excludes.default"); pub struct BackupRepository { layout: Arc, crypto: Arc, - repo: RepositoryInner + repo: Repository } impl BackupRepository { @@ -44,7 +44,7 @@ impl BackupRepository { Ok(BackupRepository { crypto: crypto.clone(), layout: layout.clone(), - repo: try!(RepositoryInner::create(layout, config, crypto, remote)) + repo: try!(Repository::create(layout, config, crypto, remote)) }) } @@ -55,7 +55,7 @@ impl BackupRepository { Ok(BackupRepository { crypto: crypto.clone(), layout: layout.clone(), - repo: try!(RepositoryInner::open(layout, crypto, online)) + repo: try!(Repository::open(layout, crypto, online)) }) } diff --git a/src/prelude.rs b/src/prelude.rs index c68742e..020e559 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -2,7 +2,7 @@ pub use util::*; pub use repository::bundledb::{BundleReader, BundleMode, BundleWriter, BundleInfo, BundleId, BundleDbError, BundleDb, BundleWriterError, StoredBundle, BundleStatistics}; pub use repository::chunking::{ChunkerType, Chunker, ChunkerStatus, ChunkerError}; -pub use repository::{RepositoryInner, Config, RepositoryError, RepositoryInfo, +pub use repository::{Repository, Config, RepositoryError, RepositoryInfo, IntegrityError, BundleAnalysis, RepositoryLayout, Location, RepositoryStatistics, ChunkRepositoryLayout}; pub use repository::*; diff --git a/src/repository/inner/basic_io.rs b/src/repository/basic_io.rs similarity index 80% rename from src/repository/inner/basic_io.rs rename to src/repository/basic_io.rs index ba88874..d26d9b9 100644 --- a/src/repository/inner/basic_io.rs +++ b/src/repository/basic_io.rs @@ -10,16 +10,18 @@ pub struct ChunkReader<'a> { chunks: VecDeque, data: Vec, pos: usize, - repo: &'a mut RepositoryInner + repo: &'a mut Repository, + lock: &'a OnlineMode } impl<'a> ChunkReader<'a> { - pub fn new(repo: &'a mut RepositoryInner, chunks: ChunkList) -> Self { + pub fn new(repo: &'a mut Repository, chunks: ChunkList, lock: &'a OnlineMode) -> Self { ChunkReader { repo, chunks: chunks.into_inner().into(), data: vec![], - pos: 0 + pos: 0, + lock } } } @@ -33,7 +35,7 @@ impl<'a> Read for ChunkReader<'a> { } if self.data.len() == self.pos { if let Some(chunk) = self.chunks.pop_front() { - self.data = match self.repo.get_chunk(chunk.0) { + self.data = match self.repo.get_chunk(chunk.0, self.lock) { Ok(Some(data)) => data, Ok(None) => { return Err(io::Error::new( @@ -58,7 +60,7 @@ impl<'a> Read for ChunkReader<'a> { } -impl RepositoryInner { +impl Repository { #[inline] pub fn get_bundle_id(&self, id: u32) -> Result { self.bundle_map.get(id).ok_or_else(|| { @@ -66,7 +68,7 @@ impl RepositoryInner { }) } - pub fn get_chunk(&mut self, hash: Hash) -> Result>, RepositoryError> { + pub fn get_chunk(&mut self, hash: Hash, lock: &OnlineMode) -> Result>, RepositoryError> { // Find bundle and chunk id in index let found = if let Some(found) = self.index.get(&hash) { found @@ -77,7 +79,7 @@ impl RepositoryInner { let bundle_id = try!(self.get_bundle_id(found.bundle)); // Get chunk from bundle Ok(Some(try!( - self.bundles.get_chunk(&bundle_id, found.chunk as usize) + self.bundles.get_chunk(&bundle_id, found.chunk as usize, lock) ))) } @@ -87,12 +89,13 @@ impl RepositoryInner { mode: BundleMode, hash: Hash, data: &[u8], + lock: &BackupMode ) -> Result<(), RepositoryError> { // If this chunk is in the index, ignore it if self.index.contains(&hash) { return Ok(()); } - self.put_chunk_override(mode, hash, data) + self.put_chunk_override(mode, hash, data, lock) } fn write_chunk_to_bundle_and_index( @@ -100,6 +103,7 @@ impl RepositoryInner { mode: BundleMode, hash: Hash, data: &[u8], + lock: &BackupMode ) -> Result<(), RepositoryError> { let writer = match mode { BundleMode::Data => &mut self.data_bundle, @@ -111,7 +115,8 @@ impl RepositoryInner { mode, self.config.hash, self.config.compression.clone(), - self.config.encryption.clone() + self.config.encryption.clone(), + lock ))); } debug_assert!(writer.is_some()); @@ -130,7 +135,7 @@ impl RepositoryInner { Ok(()) } - fn finish_bundle(&mut self, mode: BundleMode) -> Result<(), RepositoryError> { + fn finish_bundle(&mut self, mode: BundleMode, lock: &BackupMode) -> Result<(), RepositoryError> { // Calculate the next free bundle id now (late lifetime prevents this) let next_free_bundle_id = self.next_free_bundle_id(); let writer = match mode { @@ -146,8 +151,8 @@ impl RepositoryInner { }; let mut finished = None; mem::swap(writer, &mut finished); - let bundle = try!(self.bundles.add_bundle(finished.unwrap())); - self.bundle_map.set(bundle_id, bundle.id.clone()); + let bundle = try!(self.bundles.add_bundle(finished.unwrap(), lock)); + self.bundle_map.set(bundle_id, bundle.id.clone(), lock.as_localwrite()); if self.next_meta_bundle == bundle_id { self.next_meta_bundle = next_free_bundle_id } @@ -157,7 +162,7 @@ impl RepositoryInner { Ok(()) } - fn finish_bundle_if_needed(&mut self, mode: BundleMode) -> Result<(), RepositoryError> { + fn finish_bundle_if_needed(&mut self, mode: BundleMode, lock: &BackupMode) -> Result<(), RepositoryError> { let (size, raw_size) = { let writer = match mode { BundleMode::Data => &mut self.data_bundle, @@ -171,10 +176,10 @@ impl RepositoryInner { }; if size >= self.config.bundle_size || raw_size >= 4 * self.config.bundle_size { if mode == BundleMode::Meta { - //First store the current data bundle as meta referrs to those chunks - try!(self.finish_bundle(BundleMode::Data)) + //First store the current data bundle as meta refers to those chunks + try!(self.finish_bundle(BundleMode::Data, lock)) } - try!(self.finish_bundle(mode)) + try!(self.finish_bundle(mode, lock)) } Ok(()) } @@ -185,9 +190,10 @@ impl RepositoryInner { mode: BundleMode, hash: Hash, data: &[u8], + lock: &BackupMode ) -> Result<(), RepositoryError> { - try!(self.write_chunk_to_bundle_and_index(mode, hash, data)); - self.finish_bundle_if_needed(mode) + try!(self.write_chunk_to_bundle_and_index(mode, hash, data, lock)); + self.finish_bundle_if_needed(mode, lock) } #[inline] @@ -195,15 +201,17 @@ impl RepositoryInner { &mut self, mode: BundleMode, data: &[u8], + lock: &BackupMode ) -> Result { let mut input = Cursor::new(data); - self.put_stream(mode, &mut input) + self.put_stream(mode, &mut input, lock) } pub fn put_stream( &mut self, mode: BundleMode, data: &mut R, + lock: &BackupMode ) -> Result { let avg_size = self.config.chunker.avg_size(); let mut chunks = Vec::new(); @@ -214,7 +222,7 @@ impl RepositoryInner { let res = try!(self.chunker.chunk(data, &mut output)); chunk = output.into_inner(); let hash = self.config.hash.hash(&chunk); - try!(self.put_chunk(mode, hash, &chunk)); + try!(self.put_chunk(mode, hash, &chunk, lock)); chunks.push((hash, chunk.len() as u32)); if res == ChunkerStatus::Finished { break; @@ -223,25 +231,26 @@ impl RepositoryInner { Ok(chunks.into()) } - pub fn get_data(&mut self, chunks: &[Chunk]) -> Result, RepositoryError> { + pub fn get_data(&mut self, chunks: &[Chunk], lock: &OnlineMode) -> Result, RepositoryError> { let mut data = Vec::with_capacity(chunks.iter().map(|&(_, size)| size).sum::() as usize); - try!(self.get_stream(chunks, &mut data)); + try!(self.get_stream(chunks, &mut data, lock)); Ok(data) } #[inline] - pub fn get_reader(&mut self, chunks: ChunkList) -> ChunkReader { - ChunkReader::new(self, chunks) + pub fn get_reader<'a>(&'a mut self, chunks: ChunkList, lock: &'a OnlineMode) -> ChunkReader<'a> { + ChunkReader::new(self, chunks, lock) } pub fn get_stream( &mut self, chunks: &[Chunk], w: &mut W, + lock: &OnlineMode ) -> Result<(), RepositoryError> { for &(ref hash, len) in chunks { - let data = try!(try!(self.get_chunk(*hash)).ok_or_else(|| { + let data = try!(try!(self.get_chunk(*hash, lock)).ok_or_else(|| { IntegrityError::MissingChunk(*hash) })); debug_assert_eq!(data.len() as u32, len); diff --git a/src/repository/bundle_map.rs b/src/repository/bundle_map.rs index b5723f3..087c324 100644 --- a/src/repository/bundle_map.rs +++ b/src/repository/bundle_map.rs @@ -46,7 +46,7 @@ impl BundleMap { BundleMap(Default::default()) } - pub fn load>(path: P) -> Result { + pub fn load>(path: P, _lock: &ReadonlyMode) -> Result { let mut file = BufReader::new(try!(File::open(path.as_ref()))); let mut header = [0u8; 8]; try!(file.read_exact(&mut header)); @@ -60,7 +60,7 @@ impl BundleMap { Ok(BundleMap(try!(msgpack::decode_from_stream(&mut file)))) } - pub fn save>(&self, path: P) -> Result<(), BundleMapError> { + pub fn save>(&self, path: P, _lock: &LocalWriteMode) -> Result<(), BundleMapError> { let mut file = BufWriter::new(try!(File::create(path))); try!(file.write_all(&HEADER_STRING)); try!(file.write_all(&[HEADER_VERSION])); @@ -73,7 +73,7 @@ impl BundleMap { } #[inline] - pub fn remove(&mut self, id: u32) -> Option { + pub fn remove(&mut self, id: u32, _lock: &LocalWriteMode) -> Option { self.0.remove(&id) } @@ -87,7 +87,7 @@ impl BundleMap { } #[inline] - pub fn set(&mut self, id: u32, bundle: BundleId) { + pub fn set(&mut self, id: u32, bundle: BundleId, _lock: &LocalWriteMode) { self.0.insert(id, bundle); } diff --git a/src/repository/bundledb/db.rs b/src/repository/bundledb/db.rs index 61aff9f..985c4f5 100644 --- a/src/repository/bundledb/db.rs +++ b/src/repository/bundledb/db.rs @@ -62,7 +62,7 @@ fn load_bundles( path: &Path, base: &Path, bundles: &mut HashMap, - crypto: Arc, + crypto: Arc ) -> Result<(Vec, Vec), BundleDbError> { let mut paths = vec![path.to_path_buf()]; let mut bundle_paths = HashSet::new(); @@ -138,10 +138,10 @@ impl BundleDb { } } - fn load_bundle_list( + fn load_local_bundle_list( &mut self, - online: bool - ) -> Result<(Vec, Vec), BundleDbError> { + _lock: &ReadonlyMode + ) -> Result<(), BundleDbError> { if let Ok(list) = StoredBundle::read_list_from(&self.layout.local_bundle_cache_path()) { for bundle in list { self.local_bundles.insert(bundle.id(), bundle); @@ -170,9 +170,15 @@ impl BundleDb { &self.layout.local_bundle_cache_path() )); } - if !online { - return Ok((vec![], vec![])) - } + Ok(()) + } + + fn load_remote_bundle_list( + &mut self, + lock: &OnlineMode + ) -> Result<(Vec, Vec), BundleDbError> { + try!(self.load_local_bundle_list(lock.as_readonly())); + let base_path = self.layout.base_path(); let (new, gone) = try!(load_bundles( &self.layout.remote_bundles_path(), base_path, @@ -189,11 +195,11 @@ impl BundleDb { Ok((new, gone)) } - pub fn flush(&mut self) -> Result<(), BundleDbError> { - self.finish_uploads().and_then(|()| self.save_cache()) + pub fn flush(&mut self, lock: &BackupMode) -> Result<(), BundleDbError> { + self.finish_uploads(lock).and_then(|()| self.save_cache(lock.as_localwrite())) } - fn save_cache(&self) -> Result<(), BundleDbError> { + fn save_cache(&self, _lock: &LocalWriteMode) -> Result<(), BundleDbError> { let bundles: Vec<_> = self.local_bundles.values().cloned().collect(); try!(StoredBundle::save_list_to( &bundles, @@ -207,7 +213,8 @@ impl BundleDb { Ok(()) } - fn update_cache(&mut self) -> Result<(), BundleDbError> { + pub fn synchronize(&mut self, lock: &OnlineMode) -> Result<(Vec, Vec), BundleDbError> { + let (new, gone) = try!(self.load_remote_bundle_list(lock)); let mut meta_bundles = HashSet::new(); for (id, bundle) in &self.remote_bundles { if bundle.info.mode == BundleMode::Meta { @@ -224,7 +231,7 @@ impl BundleDb { if !self.local_bundles.contains_key(&id) { let bundle = self.remote_bundles[&id].clone(); tr_debug!("Copying new meta bundle to local cache: {}", bundle.info.id); - try!(self.copy_remote_bundle_to_cache(&bundle)); + try!(self.copy_remote_bundle_to_cache(&bundle, lock)); } } let base_path = self.layout.base_path(); @@ -235,20 +242,19 @@ impl BundleDb { })) } } - Ok(()) + let new = new.into_iter().map(|s| s.info).collect(); + let gone = gone.into_iter().map(|s| s.info).collect(); + Ok((new, gone)) } pub fn open( layout: Arc, crypto: Arc, - online: bool - ) -> Result<(Self, Vec, Vec), BundleDbError> { + lock: &ReadonlyMode + ) -> Result { let mut self_ = Self::new(layout, crypto); - let (new, gone) = try!(self_.load_bundle_list(online)); - try!(self_.update_cache()); - let new = new.into_iter().map(|s| s.info).collect(); - let gone = gone.into_iter().map(|s| s.info).collect(); - Ok((self_, new, gone)) + try!(self_.load_local_bundle_list(lock)); + Ok(self_) } pub fn create(layout: Arc) -> Result<(), BundleDbError> { @@ -282,6 +288,7 @@ impl BundleDb { hash_method: HashMethod, compression: Option, encryption: Option, + lock: &BackupMode ) -> Result { Ok(try!(BundleWriter::new( self.layout.clone(), @@ -305,7 +312,7 @@ impl BundleDb { } #[inline] - fn get_bundle(&self, stored: &StoredBundle) -> Result { + fn get_bundle(&self, stored: &StoredBundle, _lock: &OnlineMode) -> Result { let base_path = self.layout.base_path(); Ok(try!(BundleReader::load( base_path.join(&stored.path), @@ -313,7 +320,7 @@ impl BundleDb { ))) } - pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize) -> Result, BundleDbError> { + pub fn get_chunk(&mut self, bundle_id: &BundleId, id: usize, lock: &OnlineMode) -> Result, BundleDbError> { if let Some(&mut (ref mut bundle, ref data)) = self.bundle_cache.get_mut(bundle_id) { let (pos, len) = try!(bundle.get_chunk_position(id)); let mut chunk = Vec::with_capacity(len); @@ -321,7 +328,7 @@ impl BundleDb { return Ok(chunk); } let mut bundle = try!(self.get_stored_bundle(bundle_id).and_then( - |s| self.get_bundle(s) + |s| self.get_bundle(s, lock) )); let (pos, len) = try!(bundle.get_chunk_position(id)); let mut chunk = Vec::with_capacity(len); @@ -331,7 +338,7 @@ impl BundleDb { Ok(chunk) } - fn copy_remote_bundle_to_cache(&mut self, bundle: &StoredBundle) -> Result<(), BundleDbError> { + fn copy_remote_bundle_to_cache(&mut self, bundle: &StoredBundle, _lock: &OnlineMode) -> Result<(), BundleDbError> { let id = bundle.id(); let dst_path = self.layout.local_bundle_path(&id, self.local_bundles.len()); { @@ -346,10 +353,10 @@ impl BundleDb { Ok(()) } - pub fn add_bundle(&mut self, bundle: BundleWriter) -> Result { + pub fn add_bundle(&mut self, bundle: BundleWriter, lock: &BackupMode) -> Result { let mut bundle = try!(bundle.finish()); if bundle.info.mode == BundleMode::Meta { - try!(self.copy_remote_bundle_to_cache(&bundle)) + try!(self.copy_remote_bundle_to_cache(&bundle, lock.as_online())) } let dst_path = self.layout.remote_bundle_path(&bundle.id(),self.remote_bundles.len()); let src_path = self.layout.base_path().join(bundle.path); @@ -365,7 +372,7 @@ impl BundleDb { Ok(bundle.info) } - fn finish_uploads(&mut self) -> Result<(), BundleDbError> { + fn finish_uploads(&mut self, _lock: &BackupMode) -> Result<(), BundleDbError> { let mut uploader = None; mem::swap(&mut self.uploader, &mut uploader); if let Some(uploader) = uploader { @@ -375,9 +382,9 @@ impl BundleDb { } } - pub fn get_chunk_list(&self, bundle: &BundleId) -> Result { + pub fn get_chunk_list(&self, bundle: &BundleId, lock: &OnlineMode) -> Result { let mut bundle = try!(self.get_stored_bundle(bundle).and_then(|stored| { - self.get_bundle(stored) + self.get_bundle(stored, lock) })); Ok(try!(bundle.get_chunk_list()).clone()) } @@ -392,7 +399,7 @@ impl BundleDb { self.remote_bundles.values().map(|b| &b.info).collect() } - pub fn delete_local_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> { + pub fn delete_local_bundle(&mut self, bundle: &BundleId, _lock: &LocalWriteMode) -> Result<(), BundleDbError> { if let Some(bundle) = self.local_bundles.remove(bundle) { let path = self.layout.base_path().join(&bundle.path); try!(fs::remove_file(path).map_err(|e| { @@ -402,8 +409,8 @@ impl BundleDb { Ok(()) } - pub fn delete_bundle(&mut self, bundle: &BundleId) -> Result<(), BundleDbError> { - try!(self.delete_local_bundle(bundle)); + pub fn delete_bundle(&mut self, bundle: &BundleId, lock: &VacuumMode) -> Result<(), BundleDbError> { + try!(self.delete_local_bundle(bundle, lock.as_localwrite())); if let Some(bundle) = self.remote_bundles.remove(bundle) { let path = self.layout.base_path().join(&bundle.path); fs::remove_file(path).map_err(|e| BundleDbError::Remove(e, bundle.id())) @@ -412,44 +419,38 @@ impl BundleDb { } } - pub fn check(&mut self, full: bool, repair: bool) -> Result { - let mut to_repair = vec![]; + pub fn check(&mut self, full: bool, lock: &OnlineMode) -> HashMap { + let mut errors = HashMap::new(); for (id, stored) in ProgressIter::new( tr!("checking bundles"), self.remote_bundles.len(), self.remote_bundles.iter() ) { - let mut bundle = match self.get_bundle(stored) { + let mut bundle = match self.get_bundle(stored, lock) { Ok(bundle) => bundle, Err(err) => { - if repair { - to_repair.push(id.clone()); - continue; - } else { - return Err(err); - } + errors.insert(id.clone(), err); + continue; } }; if let Err(err) = bundle.check(full) { - if repair { - to_repair.push(id.clone()); - continue; - } else { - return Err(err.into()); - } + errors.insert(id.clone(), err.into()); + continue; } } - if !to_repair.is_empty() { - for id in ProgressIter::new(tr!("repairing bundles"), to_repair.len(), to_repair.iter()) { - try!(self.repair_bundle(id)); - } - try!(self.flush()); - } - Ok(!to_repair.is_empty()) + errors } - fn evacuate_broken_bundle(&mut self, mut bundle: StoredBundle) -> Result<(), BundleDbError> { + pub fn repair(&mut self, lock: &VacuumMode, bundles: &[BundleId]) -> Result<(), BundleDbError> { + for id in ProgressIter::new(tr!("repairing bundles"), bundles.len(), bundles.iter()) { + try!(self.repair_bundle(id, lock)); + } + try!(self.flush(lock.as_backup())); + Ok(()) + } + + fn evacuate_broken_bundle(&mut self, mut bundle: StoredBundle, _lock: &VacuumMode) -> Result<(), BundleDbError> { let src = self.layout.base_path().join(&bundle.path); let mut dst = src.with_extension("bundle.broken"); let mut num = 1; @@ -463,9 +464,9 @@ impl BundleDb { Ok(()) } - fn repair_bundle(&mut self, id: &BundleId) -> Result<(), BundleDbError> { + fn repair_bundle(&mut self, id: &BundleId, lock: &VacuumMode) -> Result<(), BundleDbError> { let stored = self.remote_bundles[id].clone(); - let mut bundle = match self.get_bundle(&stored) { + let mut bundle = match self.get_bundle(&stored, lock.as_online()) { Ok(bundle) => bundle, Err(err) => { tr_warn!( @@ -473,7 +474,7 @@ impl BundleDb { id, err ); - return self.evacuate_broken_bundle(stored); + return self.evacuate_broken_bundle(stored, lock); } }; let chunks = match bundle.get_chunk_list() { @@ -484,7 +485,7 @@ impl BundleDb { id, err ); - return self.evacuate_broken_bundle(stored); + return self.evacuate_broken_bundle(stored, lock); } }; let data = match bundle.load_contents() { @@ -495,7 +496,7 @@ impl BundleDb { id, err ); - return self.evacuate_broken_bundle(stored); + return self.evacuate_broken_bundle(stored, lock); } }; tr_warn!("Problem detected: bundle data was truncated: {}", id); @@ -505,7 +506,8 @@ impl BundleDb { info.mode, info.hash_method, info.compression, - info.encryption + info.encryption, + lock.as_backup() )); let mut pos = 0; for (hash, mut len) in chunks.into_inner() { @@ -516,9 +518,9 @@ impl BundleDb { try!(new_bundle.add(&data[pos..pos + len as usize], hash)); pos += len as usize; } - let bundle = try!(self.add_bundle(new_bundle)); + let bundle = try!(self.add_bundle(new_bundle, lock.as_backup())); tr_info!("New bundle id is {}", bundle.id); - self.evacuate_broken_bundle(stored) + self.evacuate_broken_bundle(stored, lock) } #[inline] diff --git a/src/repository/bundledb/reader.rs b/src/repository/bundledb/reader.rs index f4d33f7..67571c3 100644 --- a/src/repository/bundledb/reader.rs +++ b/src/repository/bundledb/reader.rs @@ -253,13 +253,7 @@ impl BundleReader { tr!("Chunk list size does not match chunk count") )); } - if self.chunks - .as_ref() - .unwrap() - .iter() - .map(|c| c.1 as usize) - .sum::() != self.info.raw_size - { + if self.chunks.as_ref().unwrap().iter().map(|c| c.1 as usize).sum::() != self.info.raw_size { return Err(BundleReaderError::Integrity( self.id(), tr!("Individual chunk sizes do not add up to total size") diff --git a/src/repository/config.rs b/src/repository/config.rs index a0d414e..77d8219 100644 --- a/src/repository/config.rs +++ b/src/repository/config.rs @@ -222,7 +222,7 @@ impl Config { Config::from_yaml(config) } - pub fn save>(&self, path: P) -> Result<(), ConfigError> { + pub fn save>(&self, path: P, _lock: &LocalWriteMode) -> Result<(), ConfigError> { let mut f = try!(File::create(path)); try!(serde_yaml::to_writer(&mut f, &self.to_yaml())); Ok(()) diff --git a/src/repository/index/mod.rs b/src/repository/index/mod.rs index f1da7af..2522ed8 100644 --- a/src/repository/index/mod.rs +++ b/src/repository/index/mod.rs @@ -227,7 +227,7 @@ impl Index { data, header }; - debug_assert!(index.check().is_ok(), tr!("Inconsistent after creation")); + debug_assert!(index.check().is_empty(), tr!("Inconsistent after creation")); Ok(index) } @@ -325,7 +325,8 @@ impl Index { Ok(true) } - pub fn check(&self) -> Result<(), IndexError> { + pub fn check(&self) -> Vec { + let mut errors = vec![]; let mut entries = 0; for pos in 0..self.capacity { let entry = &self.data[pos]; @@ -334,14 +335,14 @@ impl Index { } entries += 1; match self.locate(entry.get_key()) { - LocateResult::Found(p) if p == pos => true, - found => return Err(IndexError::WrongPosition(pos, found)) + LocateResult::Found(p) if p == pos => (), + found => errors.push(IndexError::WrongPosition(pos, found)) }; } if entries != self.entries { - return Err(IndexError::WrongEntryCount(self.entries, entries)); + errors.push(IndexError::WrongEntryCount(self.entries, entries)); } - Ok(()) + errors } #[inline] @@ -467,7 +468,7 @@ impl Index { #[inline] pub fn contains(&self, key: &K) -> bool { - debug_assert!(self.check().is_ok(), tr!("Inconsistent before get")); + debug_assert!(self.check().is_empty(), tr!("Inconsistent before get")); match self.locate(key) { LocateResult::Found(_) => true, _ => false @@ -476,7 +477,7 @@ impl Index { #[inline] pub fn pos(&self, key: &K) -> Option { - debug_assert!(self.check().is_ok(), tr!("Inconsistent before get")); + debug_assert!(self.check().is_empty(), tr!("Inconsistent before get")); match self.locate(key) { LocateResult::Found(pos) => Some(pos), _ => None @@ -485,7 +486,7 @@ impl Index { #[inline] pub fn get(&self, key: &K) -> Option { - debug_assert!(self.check().is_ok(), tr!("Inconsistent before get")); + debug_assert!(self.check().is_empty(), tr!("Inconsistent before get")); match self.locate(key) { LocateResult::Found(pos) => Some(self.data[pos].data), _ => None @@ -495,7 +496,7 @@ impl Index { #[inline] #[allow(dead_code)] pub fn modify(&mut self, key: &K, mut f: F) -> bool where F: FnMut(&mut V) { - debug_assert!(self.check().is_ok(), tr!("Inconsistent before get")); + debug_assert!(self.check().is_empty(), tr!("Inconsistent before get")); match self.locate(key) { LocateResult::Found(pos) => { f(self.data[pos].get_mut_data()); diff --git a/src/repository/inner/info.rs b/src/repository/info.rs similarity index 99% rename from src/repository/inner/info.rs rename to src/repository/info.rs index 8e12357..c247b23 100644 --- a/src/repository/inner/info.rs +++ b/src/repository/info.rs @@ -46,7 +46,7 @@ pub struct RepositoryStatistics { } -impl RepositoryInner { +impl Repository { #[inline] pub fn list_bundles(&self) -> Vec<&BundleInfo> { self.bundles.list_bundles() diff --git a/src/repository/inner/api.rs b/src/repository/inner/api.rs deleted file mode 100644 index 0e5db38..0000000 --- a/src/repository/inner/api.rs +++ /dev/null @@ -1,178 +0,0 @@ -use ::prelude::*; - -pub struct Repository(RepositoryInner); -pub struct LocalWriteMode<'a>(&'a mut RepositoryInner); -pub struct RestoreMode<'a>(&'a mut RepositoryInner); -pub struct BackupMode<'a>(&'a mut RepositoryInner); -pub struct VacuumMode<'a>(&'a mut RepositoryInner); - -macro_rules! in_readonly_mode { - ( $($f:tt)* ) => { - impl Repository { - $( $f )* - } - impl<'a> LocalWriteMode<'a> { - $( $f )* - } - impl<'a> RestoreMode<'a> { - $( $f )* - } - impl<'a> BackupMode<'a> { - $( $f )* - } - impl<'a> VacuumMode<'a> { - $( $f )* - } - }; -} - -macro_rules! in_local_write_mode { - ( $($f:tt)* ) => { - impl<'a> LocalWriteMode<'a> { - $( $f )* - } - impl<'a> RestoreMode<'a> { - $( $f )* - } - impl<'a> BackupMode<'a> { - $( $f )* - } - impl<'a> VacuumMode<'a> { - $( $f )* - } - }; -} - -macro_rules! in_restore_mode { - ( $($f:tt)* ) => { - impl<'a> RestoreMode<'a> { - $( $f )* - } - impl<'a> BackupMode<'a> { - $( $f )* - } - impl<'a> VacuumMode<'a> { - $( $f )* - } - }; -} - -macro_rules! in_backup_mode { - ( $($f:tt)* ) => { - impl<'a> BackupMode<'a> { - $( $f )* - } - impl<'a> VacuumMode<'a> { - $( $f )* - } - }; -} - -macro_rules! in_vacuum_mode { - ( $($f:tt)* ) => { - impl<'a> VacuumMode<'a> { - $( $f )* - } - }; -} - - -impl RepositoryInner { - fn local_write_mode Result>(&mut self, f: F) -> Result { - let ret = f(LocalWriteMode(self)); - ret - } - - fn restore_mode Result>(&mut self, f: F) -> Result { - let ret = f(RestoreMode(self)); - ret - } - - fn backup_mode Result>(&mut self, f: F) -> Result { - let ret = f(BackupMode(self)); - ret - } - - fn vacuum_mode Result>(&mut self, f: F) -> Result { - let ret = f(VacuumMode(self)); - ret - } -} - - -impl Repository { - pub fn local_write_mode Result>(&mut self, f: F) -> Result { - self.0.local_write_mode(f) - } - - pub fn restore_mode Result>(&mut self, f: F) -> Result { - self.0.restore_mode(f) - } - - pub fn backup_mode Result>(&mut self, f: F) -> Result { - self.0.backup_mode(f) - } - - pub fn vacuum_mode Result>(&mut self, f: F) -> Result { - self.0.vacuum_mode(f) - } -} - -impl<'a> LocalWriteMode<'a> { - pub fn restore_mode Result>(&mut self, f: F) -> Result { - self.0.restore_mode(f) - } - - pub fn backup_mode Result>(&mut self, f: F) -> Result { - self.0.backup_mode(f) - } - - pub fn vacuum_mode Result>(&mut self, f: F) -> Result { - self.0.vacuum_mode(f) - } -} - -impl<'a> RestoreMode<'a> { - pub fn backup_mode Result>(&mut self, f: F) -> Result { - self.0.backup_mode(f) - } - - pub fn vacuum_mode Result>(&mut self, f: F) -> Result { - self.0.vacuum_mode(f) - } -} - -impl<'a> BackupMode<'a> { - pub fn vacuum_mode Result>(&mut self, f: F) -> Result { - self.0.vacuum_mode(f) - } -} - - -impl Repository { - fn test(&mut self) { - self.local_write_mode(|s| { - s.dummy("aaa"); - Ok(()) - }); - } -} - -in_readonly_mode! { - pub fn get_config(&self) -> &Config { - self.0.get_config() - } - - pub fn set_config(&mut self, config: Config) { - self.0.set_config(config); - } -} - - -in_local_write_mode! { - fn dummy(&self, r: R) { - - } -} - - diff --git a/src/repository/inner/integrity.rs b/src/repository/inner/integrity.rs deleted file mode 100644 index 91f477c..0000000 --- a/src/repository/inner/integrity.rs +++ /dev/null @@ -1,218 +0,0 @@ -use prelude::*; - -use super::*; -use super::super::bundle_map::BundleMap; - -use std::time::Duration; -use pbr::ProgressBar; - - -quick_error!{ - #[derive(Debug)] - pub enum IntegrityError { - MissingChunk(hash: Hash) { - description(tr!("Missing chunk")) - display("{}", tr_format!("Missing chunk: {}", hash)) - } - MissingBundleId(id: u32) { - description(tr!("Missing bundle")) - display("{}", tr_format!("Missing bundle: {}", id)) - } - MissingBundle(id: BundleId) { - description(tr!("Missing bundle")) - display("{}", tr_format!("Missing bundle: {}", id)) - } - NoSuchChunk(bundle: BundleId, chunk: u32) { - description(tr!("No such chunk")) - display("{}", tr_format!("Bundle {} does not contain the chunk {}", bundle, chunk)) - } - RemoteBundlesNotInMap { - description(tr!("Remote bundles missing from map")) - } - MapContainsDuplicates { - description(tr!("Map contains duplicates")) - } - } -} - - -impl RepositoryInner { - pub fn get_chunk_marker(&self) -> Bitmap { - Bitmap::new(self.index.capacity()) - } - - pub fn check_chunks( - &self, - checked: &mut Bitmap, - chunks: &[Chunk], - mark: bool, - ) -> Result { - let mut new = false; - for &(hash, _len) in chunks { - if let Some(pos) = self.index.pos(&hash) { - new |= !checked.get(pos); - if mark { - checked.set(pos); - } - } else { - return Err(IntegrityError::MissingChunk(hash).into()); - } - } - Ok(new) - } - - fn check_index_chunks(&self) -> Result<(), RepositoryError> { - let mut progress = ProgressBar::new(self.index.len() as u64); - progress.message(tr!("checking index: ")); - progress.set_max_refresh_rate(Some(Duration::from_millis(100))); - for (count, (_hash, location)) in self.index.iter().enumerate() { - // Lookup bundle id from map - let bundle_id = try!(self.get_bundle_id(location.bundle)); - // Get bundle object from bundledb - let bundle = if let Some(bundle) = self.bundles.get_bundle_info(&bundle_id) { - bundle - } else { - progress.finish_print(tr!("checking index: done.")); - return Err(IntegrityError::MissingBundle(bundle_id.clone()).into()); - }; - // Get chunk from bundle - if bundle.info.chunk_count <= location.chunk as usize { - progress.finish_print(tr!("checking index: done.")); - return Err( - IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk).into() - ); - } - if count % 1000 == 0 { - progress.set(count as u64); - } - } - progress.finish_print(tr!("checking index: done.")); - Ok(()) - } - - pub fn rebuild_bundle_map(&mut self) -> Result<(), RepositoryError> { - tr_info!("Rebuilding bundle map from bundles"); - self.bundle_map = BundleMap::create(); - for bundle in self.bundles.list_bundles() { - let bundle_id = match bundle.mode { - BundleMode::Data => self.next_data_bundle, - BundleMode::Meta => self.next_meta_bundle, - }; - self.bundle_map.set(bundle_id, bundle.id.clone()); - if self.next_meta_bundle == bundle_id { - self.next_meta_bundle = self.next_free_bundle_id() - } - if self.next_data_bundle == bundle_id { - self.next_data_bundle = self.next_free_bundle_id() - } - } - self.save_bundle_map() - } - - pub fn rebuild_index(&mut self) -> Result<(), RepositoryError> { - tr_info!("Rebuilding index from bundles"); - self.index.clear(); - let mut bundles = self.bundle_map.bundles(); - bundles.sort_by_key(|&(_, ref v)| v.clone()); - for (num, id) in ProgressIter::new(tr!("Rebuilding index from bundles"), bundles.len(), bundles.into_iter()) { - let chunks = try!(self.bundles.get_chunk_list(&id)); - for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() { - try!(self.index.set( - &hash, - &Location { - bundle: num as u32, - chunk: i as u32 - } - )); - } - } - Ok(()) - } - - #[inline] - pub fn check_index(&mut self, repair: bool) -> Result<(), RepositoryError> { - if repair { - try!(self.write_mode()); - } - tr_info!("Checking index integrity..."); - if let Err(err) = self.index.check() { - if repair { - tr_warn!( - "Problem detected: index was corrupted\n\tcaused by: {}", - err - ); - return self.rebuild_index(); - } else { - return Err(err.into()); - } - } - tr_info!("Checking index entries..."); - if let Err(err) = self.check_index_chunks() { - if repair { - tr_warn!( - "Problem detected: index entries were inconsistent\n\tcaused by: {}", - err - ); - return self.rebuild_index(); - } else { - return Err(err); - } - } - Ok(()) - } - - #[inline] - pub fn check_bundles(&mut self, full: bool, repair: bool) -> Result<(), RepositoryError> { - if repair { - try!(self.write_mode()); - } - tr_info!("Checking bundle integrity..."); - if try!(self.bundles.check(full, repair)) { - // Some bundles got repaired - tr_warn!("Some bundles have been rewritten, please remove the broken bundles manually."); - try!(self.rebuild_bundle_map()); - try!(self.rebuild_index()); - } - Ok(()) - } - - pub fn check_repository(&mut self, repair: bool) -> Result<(), RepositoryError> { - tr_info!("Checking repository integrity..."); - let mut rebuild = false; - for (_id, bundle_id) in self.bundle_map.bundles() { - if self.bundles.get_bundle_info(&bundle_id).is_none() { - if repair { - tr_warn!( - "Problem detected: bundle map contains unknown bundle {}", - bundle_id - ); - rebuild = true; - } else { - return Err(IntegrityError::MissingBundle(bundle_id).into()); - } - } - } - if self.bundle_map.len() < self.bundles.len() { - if repair { - tr_warn!("Problem detected: bundle map does not contain all remote bundles"); - rebuild = true; - } else { - return Err(IntegrityError::RemoteBundlesNotInMap.into()); - } - } - if self.bundle_map.len() > self.bundles.len() { - if repair { - tr_warn!("Problem detected: bundle map contains bundles multiple times"); - rebuild = true; - } else { - return Err(IntegrityError::MapContainsDuplicates.into()); - } - } - if rebuild { - try!(self.rebuild_bundle_map()); - try!(self.rebuild_index()); - } - Ok(()) - } - -} diff --git a/src/repository/inner/mod.rs b/src/repository/inner/mod.rs deleted file mode 100644 index 22aeb6a..0000000 --- a/src/repository/inner/mod.rs +++ /dev/null @@ -1,364 +0,0 @@ -mod integrity; -mod basic_io; -mod info; -mod vacuum; -pub mod api; - - -use prelude::*; - -use std::mem; -use std::cmp::max; -use std::path::Path; -use std::fs::{self, File}; -use std::sync::Arc; -use std::os::unix::fs::symlink; -use std::io::Write; - -use super::{INDEX_MAGIC, INDEX_VERSION, REPOSITORY_README}; -use super::bundle_map::BundleMap; -use super::index; -pub use self::integrity::IntegrityError; -pub use self::info::{BundleAnalysis, RepositoryInfo, RepositoryStatistics}; - - -#[derive(Clone, Copy, PartialEq, Debug, Default)] -pub struct Location { - pub bundle: u32, - pub chunk: u32 -} -impl Location { - pub fn new(bundle: u32, chunk: u32) -> Self { - Location { - bundle, - chunk - } - } -} - -impl index::Value for Location {} - -impl index::Key for Hash { - fn hash(&self) -> u64 { - self.low - } - - fn is_used(&self) -> bool { - self.low != 0 || self.high != 0 - } - - fn clear(&mut self) { - self.low = 0; - self.high = 0; - } -} - - -pub struct RepositoryInner { - layout: Arc, - config: Config, - index: Index, - crypto: Arc, - bundle_map: BundleMap, - next_data_bundle: u32, - next_meta_bundle: u32, - bundles: BundleDb, - data_bundle: Option, - meta_bundle: Option, - chunker: Box, - remote_locks: LockFolder, - local_locks: LockFolder, - lock: LockHandle, - dirty: bool -} - - -impl RepositoryInner { - pub fn create>( - layout: Arc, - config: &Config, - crypto: Arc, - remote: R, - ) -> Result { - try!(fs::create_dir(layout.local_locks_path())); - try!(symlink(remote, layout.remote_path())); - try!(File::create(layout.remote_readme_path()).and_then( - |mut f| { - f.write_all(REPOSITORY_README) - } - )); - try!(fs::create_dir_all(layout.remote_locks_path())); - try!(config.save(layout.config_path())); - try!(BundleDb::create(layout.clone())); - try!(Index::::create( - layout.index_path(), - &INDEX_MAGIC, - INDEX_VERSION - )); - try!(BundleMap::create().save(layout.bundle_map_path())); - Self::open(layout, crypto, true) - } - - #[allow(unknown_lints, useless_let_if_seq)] - pub fn open(layout: Arc, crypto: Arc, online: bool) -> Result { - if !layout.remote_exists() { - return Err(RepositoryError::NoRemote); - } - let config = try!(Config::load(layout.config_path())); - let remote_locks = LockFolder::new(layout.remote_locks_path()); - try!(fs::create_dir_all(layout.local_locks_path())); // Added after v0.1.0 - let local_locks = LockFolder::new(layout.local_locks_path()); - let lock = try!(local_locks.lock(false)); - let (bundles, new, gone) = try!(BundleDb::open(layout.clone(), crypto.clone(), online)); - let (index, mut rebuild_index) = - match unsafe { Index::open(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION) } { - Ok(index) => (index, false), - Err(err) => { - tr_error!("Failed to load local index:\n\tcaused by: {}", err); - ( - try!(Index::create( - layout.index_path(), - &INDEX_MAGIC, - INDEX_VERSION - )), - true - ) - } - }; - let (bundle_map, rebuild_bundle_map) = match BundleMap::load(layout.bundle_map_path()) { - Ok(bundle_map) => (bundle_map, false), - Err(err) => { - tr_error!("Failed to load local bundle map:\n\tcaused by: {}", err); - (BundleMap::create(), true) - } - }; - let dirty = layout.dirtyfile_path().exists(); - let mut repo = RepositoryInner { - layout, - dirty: true, - chunker: config.chunker.create(), - config, - index, - crypto, - bundle_map, - next_data_bundle: 0, - next_meta_bundle: 0, - bundles, - data_bundle: None, - meta_bundle: None, - lock, - remote_locks, - local_locks - }; - if !rebuild_bundle_map { - let mut save_bundle_map = false; - if !gone.is_empty() { - tr_info!("Removing {} old bundles from index", gone.len()); - try!(repo.write_mode()); - for bundle in gone { - try!(repo.remove_gone_remote_bundle(&bundle)) - } - save_bundle_map = true; - } - if !new.is_empty() { - tr_info!("Adding {} new bundles to index", new.len()); - try!(repo.write_mode()); - for bundle in ProgressIter::new( - tr!("adding bundles to index"), - new.len(), - new.into_iter() - ) - { - try!(repo.add_new_remote_bundle(&bundle)) - } - save_bundle_map = true; - } - if save_bundle_map { - try!(repo.write_mode()); - try!(repo.save_bundle_map()); - } - } - repo.next_meta_bundle = repo.next_free_bundle_id(); - repo.next_data_bundle = repo.next_free_bundle_id(); - if rebuild_bundle_map { - try!(repo.write_mode()); - try!(repo.rebuild_bundle_map()); - rebuild_index = true; - } - if rebuild_index { - try!(repo.write_mode()); - try!(repo.rebuild_index()); - } - repo.dirty = dirty; - Ok(repo) - } - - #[inline] - pub fn save_config(&mut self) -> Result<(), RepositoryError> { - try!(self.write_mode()); - try!(self.config.save(self.layout.config_path())); - Ok(()) - } - - #[inline] - pub fn set_encryption(&mut self, public: Option<&PublicKey>) { - if let Some(key) = public { - if !self.crypto.contains_secret_key(key) { - tr_warn!("The secret key for that public key is not stored in the repository.") - } - let mut key_bytes = Vec::new(); - key_bytes.extend_from_slice(&key[..]); - self.config.encryption = Some((EncryptionMethod::Sodium, key_bytes.into())) - } else { - self.config.encryption = None - } - } - - #[inline] - pub fn save_bundle_map(&self) -> Result<(), RepositoryError> { - try!(self.bundle_map.save(self.layout.bundle_map_path())); - Ok(()) - } - - #[inline] - pub fn next_free_bundle_id(&self) -> u32 { - let mut id = max(self.next_data_bundle, self.next_meta_bundle) + 1; - while self.bundle_map.get(id).is_some() { - id += 1; - } - id - } - - pub fn set_dirty(&mut self) -> Result<(), RepositoryError> { - self.dirty = true; - let dirtyfile = self.layout.dirtyfile_path(); - if !dirtyfile.exists() { - try!(File::create(&dirtyfile)); - } - Ok(()) - } - - pub fn flush(&mut self) -> Result<(), RepositoryError> { - let dirtyfile = self.layout.dirtyfile_path(); - if self.dirty && !dirtyfile.exists() { - try!(File::create(&dirtyfile)); - } - if self.data_bundle.is_some() { - let mut finished = None; - mem::swap(&mut self.data_bundle, &mut finished); - { - let bundle = try!(self.bundles.add_bundle(finished.unwrap())); - self.bundle_map.set( - self.next_data_bundle, - bundle.id.clone() - ); - } - self.next_data_bundle = self.next_free_bundle_id() - } - if self.meta_bundle.is_some() { - let mut finished = None; - mem::swap(&mut self.meta_bundle, &mut finished); - { - let bundle = try!(self.bundles.add_bundle(finished.unwrap())); - self.bundle_map.set( - self.next_meta_bundle, - bundle.id.clone() - ); - } - self.next_meta_bundle = self.next_free_bundle_id() - } - try!(self.bundles.flush()); - try!(self.save_bundle_map()); - if !self.dirty && dirtyfile.exists() { - try!(fs::remove_file(&dirtyfile)); - } - Ok(()) - } - - fn add_new_remote_bundle(&mut self, bundle: &BundleInfo) -> Result<(), RepositoryError> { - if self.bundle_map.find(&bundle.id).is_some() { - return Ok(()); - } - tr_debug!("Adding new bundle to index: {}", bundle.id); - let bundle_id = match bundle.mode { - BundleMode::Data => self.next_data_bundle, - BundleMode::Meta => self.next_meta_bundle, - }; - let chunks = try!(self.bundles.get_chunk_list(&bundle.id)); - self.bundle_map.set(bundle_id, bundle.id.clone()); - if self.next_meta_bundle == bundle_id { - self.next_meta_bundle = self.next_free_bundle_id() - } - if self.next_data_bundle == bundle_id { - self.next_data_bundle = self.next_free_bundle_id() - } - for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() { - if let Some(old) = try!(self.index.set( - &hash, - &Location { - bundle: bundle_id as u32, - chunk: i as u32 - } - )) - { - // Duplicate chunk, forced ordering: higher bundle id wins - let old_bundle_id = try!(self.get_bundle_id(old.bundle)); - if old_bundle_id > bundle.id { - try!(self.index.set(&hash, &old)); - } - } - } - Ok(()) - } - - fn remove_gone_remote_bundle(&mut self, bundle: &BundleInfo) -> Result<(), RepositoryError> { - if let Some(id) = self.bundle_map.find(&bundle.id) { - tr_debug!("Removing bundle from index: {}", bundle.id); - try!(self.bundles.delete_local_bundle(&bundle.id)); - try!(self.index.filter(|_key, data| data.bundle != id)); - self.bundle_map.remove(id); - } - Ok(()) - } - - #[inline] - pub fn write_mode(&mut self) -> Result<(), RepositoryError> { - try!(self.local_locks.upgrade(&mut self.lock)); - Ok(()) - } - - #[inline] - pub fn lock(&self, exclusive: bool) -> Result { - Ok(try!(self.remote_locks.lock(exclusive))) - } - - pub fn is_dirty(&self) -> bool { - self.dirty - } - - pub fn get_chunk_location(&self, chunk: Hash) -> Option { - self.index.get(&chunk) - } - - #[inline] - pub fn set_clean(&mut self) { - self.dirty = false; - } - - pub fn get_config(&self) -> &Config { - &self.config - } - - pub fn set_config(&mut self, config: Config) { - self.config = config; - } -} - - -impl Drop for RepositoryInner { - fn drop(&mut self) { - if let Err(err) = self.flush() { - tr_error!("Failed to flush repository: {}", err); - } - } -} diff --git a/src/repository/integrity.rs b/src/repository/integrity.rs new file mode 100644 index 0000000..4b3b964 --- /dev/null +++ b/src/repository/integrity.rs @@ -0,0 +1,275 @@ +use prelude::*; + +use super::*; +use super::bundle_map::BundleMap; +use super::bundledb::BundleDbError; +use super::index::IndexError; + +use std::time::Duration; +use pbr::ProgressBar; + + +quick_error!{ + #[derive(Debug)] + pub enum IntegrityError { + MissingChunk(hash: Hash) { + description(tr!("Missing chunk")) + display("{}", tr_format!("Missing chunk: {}", hash)) + } + MissingBundleId(id: u32) { + description(tr!("Missing bundle")) + display("{}", tr_format!("Missing bundle: {}", id)) + } + MissingBundle(id: BundleId) { + description(tr!("Missing bundle")) + display("{}", tr_format!("Missing bundle: {}", id)) + } + NoSuchChunk(bundle: BundleId, chunk: u32) { + description(tr!("No such chunk")) + display("{}", tr_format!("Bundle {} does not contain the chunk {}", bundle, chunk)) + } + RemoteBundlesNotInMap { + description(tr!("Remote bundles missing from map")) + } + MapContainsDuplicates { + description(tr!("Map contains duplicates")) + } + Index(err: IndexError) { + description(tr!("Index error")) + display("{}", tr_format!("Index error: {}", err)) + } + BundleIntegrity(id: BundleId, err: BundleDbError) { + description(tr!("Bundle error")) + display("{}", tr_format!("Bundle {} has error: {}", id, err)) + } + } +} + + +pub struct ModuleIntegrityReport { + pub errors_fixed: Vec, + pub errors_unfixed: Vec +} + +pub struct IntegrityReport { + pub bundle_map: Option, + pub index: Option, + pub bundles: Option +} + + +pub struct ChunkMarker<'a> { + marked: Bitmap, + repo: &'a Repository +} + +impl<'a> ChunkMarker<'a> { + pub fn mark_chunks(&mut self, chunks: &[Chunk], set_marked: bool) -> Result { + let mut new = false; + for &(hash, _len) in chunks { + if let Some(pos) = self.repo.index.pos(&hash) { + new |= !self.marked.get(pos); + if set_marked { + self.marked.set(pos); + } + } else { + return Err(IntegrityError::MissingChunk(hash).into()); + } + } + Ok(new) + } +} + + +impl Repository { + pub fn get_chunk_marker(&self) -> ChunkMarker { + ChunkMarker { + marked: Bitmap::new( self.index.capacity()), + repo: self + } + } + + pub fn check_bundle_map(&mut self) -> ModuleIntegrityReport { + tr_info!("Checking bundle map..."); + let mut errors = vec![]; + for (_id, bundle_id) in self.bundle_map.bundles() { + if self.bundles.get_bundle_info(&bundle_id).is_none() { + errors.push(IntegrityError::MissingBundle(bundle_id).into()); + } + } + if self.bundle_map.len() < self.bundles.len() { + errors.push(IntegrityError::RemoteBundlesNotInMap.into()); + } + if self.bundle_map.len() > self.bundles.len() { + errors.push(IntegrityError::MapContainsDuplicates.into()); + } + ModuleIntegrityReport { errors_fixed: vec![], errors_unfixed: errors } + } + + pub fn rebuild_bundle_map(&mut self, lock: &OnlineMode) -> Result<(), RepositoryError> { + tr_info!("Rebuilding bundle map from bundles"); + try!(self.bundles.synchronize(lock)); + self.bundle_map = BundleMap::create(); + for bundle in self.bundles.list_bundles() { + let bundle_id = match bundle.mode { + BundleMode::Data => self.next_data_bundle, + BundleMode::Meta => self.next_meta_bundle, + }; + self.bundle_map.set(bundle_id, bundle.id.clone(), lock.as_localwrite()); + if self.next_meta_bundle == bundle_id { + self.next_meta_bundle = self.next_free_bundle_id() + } + if self.next_data_bundle == bundle_id { + self.next_data_bundle = self.next_free_bundle_id() + } + } + self.save_bundle_map(lock.as_localwrite()) + } + + pub fn check_and_repair_bundle_map(&mut self, lock: &OnlineMode) -> Result { + let mut report = self.check_bundle_map(); + if !report.errors_unfixed.is_empty() { + try!(self.rebuild_bundle_map(lock)); + mem::swap(&mut report.errors_unfixed, &mut report.errors_fixed); + } + Ok(report) + } + + fn check_index_chunks(&self) -> Vec { + let mut errors = vec![]; + let mut progress = ProgressBar::new(self.index.len() as u64); + progress.message(tr!("checking index: ")); + progress.set_max_refresh_rate(Some(Duration::from_millis(100))); + for (count, (_hash, location)) in self.index.iter().enumerate() { + // Lookup bundle id from map + let bundle_id = if let Some(bundle_id) = self.bundle_map.get(location.bundle) { + bundle_id + } else { + errors.push(IntegrityError::MissingBundleId(location.bundle)); + continue + }; + // Get bundle object from bundledb + let bundle = if let Some(bundle) = self.bundles.get_bundle_info(&bundle_id) { + bundle + } else { + errors.push(IntegrityError::MissingBundle(bundle_id.clone())); + continue + }; + // Get chunk from bundle + if bundle.info.chunk_count <= location.chunk as usize { + errors.push(IntegrityError::NoSuchChunk(bundle_id.clone(), location.chunk)); + continue + } + if count % 1000 == 0 { + progress.set(count as u64); + } + } + progress.finish_print(tr!("checking index: done.")); + errors + } + + pub fn rebuild_index(&mut self, lock: &OnlineMode) -> Result<(), RepositoryError> { + tr_info!("Rebuilding index from bundles"); + self.index.clear(); + let mut bundles = self.bundle_map.bundles(); + bundles.sort_by_key(|&(_, ref v)| v.clone()); + for (num, id) in ProgressIter::new(tr!("Rebuilding index from bundles"), bundles.len(), bundles.into_iter()) { + let chunks = try!(self.bundles.get_chunk_list(&id, lock)); + for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() { + try!(self.index.set( + &hash, + &Location { + bundle: num as u32, + chunk: i as u32 + } + )); + } + } + Ok(()) + } + + #[inline] + pub fn check_index(&mut self, lock: &ReadonlyMode) -> ModuleIntegrityReport { + tr_info!("Checking index integrity..."); + let mut errors: Vec = self.index.check().into_iter().map(IntegrityError::Index).collect(); + tr_info!("Checking index entries..."); + errors.extend(self.check_index_chunks()); + ModuleIntegrityReport { errors_fixed: vec![], errors_unfixed: errors } + } + + pub fn check_and_repair_index(&mut self, lock: &OnlineMode) -> Result { + let mut report = self.check_index(lock.as_readonly()); + if !report.errors_unfixed.is_empty() { + try!(self.rebuild_index(lock)); + mem::swap(&mut report.errors_unfixed, &mut report.errors_fixed); + } + Ok(report) + } + + #[inline] + fn check_bundles_internal(&mut self, full: bool, lock: &OnlineMode) -> (ModuleIntegrityReport, Vec) { + tr_info!("Checking bundle integrity..."); + let mut errors = vec![]; + let mut bundles = vec![]; + for (id, err) in self.bundles.check(full, lock) { + bundles.push(id); + errors.push(IntegrityError::BundleIntegrity(id, err)); + } + (ModuleIntegrityReport { errors_fixed: vec![], errors_unfixed: errors }, bundles) + } + + #[inline] + pub fn check_bundles(&mut self, full: bool, lock: &OnlineMode) -> ModuleIntegrityReport { + self.check_bundles_internal(full, lock).0 + } + + pub fn check_and_repair_bundles(&mut self, full: bool, lock: &VacuumMode) -> Result { + let (mut report, bundles) = self.check_bundles_internal(full, lock.as_online()); + if !report.errors_unfixed.is_empty() { + try!(self.bundles.repair(lock, &bundles)); + mem::swap(&mut report.errors_unfixed, &mut report.errors_fixed); + // Some bundles got repaired + tr_warn!("Some bundles have been rewritten, please remove the broken bundles manually."); + try!(self.rebuild_bundle_map(lock.as_online())); + try!(self.rebuild_index(lock.as_online())); + } + Ok(report) + } + + pub fn check(&mut self, index: bool, bundles: bool, bundle_data: bool, lock: &OnlineMode) -> IntegrityReport { + let mut report = IntegrityReport { + bundle_map: None, + index: None, + bundles: None + }; + report.bundle_map = Some(self.check_bundle_map()); + if index { + report.index = Some(self.check_index(lock.as_readonly())); + } + if bundles { + report.bundles = Some(self.check_bundles(bundle_data, lock)); + } + report + } + + pub fn check_and_repair(&mut self, index: bool, bundles: bool, bundle_data: bool, lock: &VacuumMode) -> Result { + let mut report = IntegrityReport { + bundle_map: None, + index: None, + bundles: None + }; + let bundle_map = try!(self.check_and_repair_bundle_map(lock.as_online())); + if !bundle_map.errors_fixed.is_empty() { + try!(self.rebuild_index(lock.as_online())); + } + report.bundle_map = Some(bundle_map); + if index { + report.index = Some(try!(self.check_and_repair_index(lock.as_online()))); + } + if bundles { + report.bundles = Some(try!(self.check_and_repair_bundles(bundle_data, lock))); + } + Ok(report) + } + +} diff --git a/src/repository/mod.rs b/src/repository/mod.rs index f7bf068..f78cc38 100644 --- a/src/repository/mod.rs +++ b/src/repository/mod.rs @@ -2,23 +2,454 @@ mod config; mod bundle_map; mod layout; mod error; -mod inner; pub mod bundledb; pub mod index; pub mod chunking; +mod integrity; +mod basic_io; +mod info; +mod vacuum; +use prelude::*; + +use std::mem; +use std::cmp::max; +use std::path::Path; +use std::fs::{self, File}; +use std::sync::Arc; +use std::os::unix::fs::symlink; +use std::io::Write; pub use self::error::RepositoryError; pub use self::config::Config; -pub use self::inner::IntegrityError; -pub use self::inner::{RepositoryInfo, BundleAnalysis, RepositoryStatistics}; pub use self::layout::{RepositoryLayout, ChunkRepositoryLayout}; -pub use self::inner::{Location, RepositoryInner}; - -pub use self::inner::api::*; - +use self::bundle_map::BundleMap; +pub use self::integrity::IntegrityError; +pub use self::info::{BundleAnalysis, RepositoryInfo, RepositoryStatistics}; const REPOSITORY_README: &[u8] = include_bytes!("../../docs/repository_readme.md"); const INDEX_MAGIC: [u8; 7] = *b"zvault\x02"; -const INDEX_VERSION: u8 = 1; \ No newline at end of file +const INDEX_VERSION: u8 = 1; + + +#[derive(Clone, Copy, PartialEq, Debug, Default)] +pub struct Location { + pub bundle: u32, + pub chunk: u32 +} +impl Location { + pub fn new(bundle: u32, chunk: u32) -> Self { + Location { + bundle, + chunk + } + } +} + +impl index::Value for Location {} + +impl index::Key for Hash { + fn hash(&self) -> u64 { + self.low + } + + fn is_used(&self) -> bool { + self.low != 0 || self.high != 0 + } + + fn clear(&mut self) { + self.low = 0; + self.high = 0; + } +} + + +pub struct Repository { + layout: Arc, + config: Config, + index: Index, + crypto: Arc, + bundle_map: BundleMap, + next_data_bundle: u32, + next_meta_bundle: u32, + bundles: BundleDb, + data_bundle: Option, + meta_bundle: Option, + chunker: Box, + remote_locks: LockFolder, + local_locks: LockFolder, +} + + +impl Repository { + pub fn create>( + layout: Arc, + config: &Config, + crypto: Arc, + remote: R, + ) -> Result { + try!(fs::create_dir(layout.local_locks_path())); + try!(symlink(remote, layout.remote_path())); + try!(File::create(layout.remote_readme_path()).and_then( + |mut f| { + f.write_all(REPOSITORY_README) + } + )); + try!(fs::create_dir_all(layout.remote_locks_path())); + let mock_lock = Lock; + try!(config.save(layout.config_path(), &mock_lock)); + try!(BundleDb::create(layout.clone())); + try!(Index::::create( + layout.index_path(), + &INDEX_MAGIC, + INDEX_VERSION + )); + try!(BundleMap::create().save(layout.bundle_map_path(), &mock_lock)); + Self::open(layout, crypto, true) + } + + #[allow(unknown_lints, useless_let_if_seq)] + pub fn open(layout: Arc, crypto: Arc, read_only: bool) -> Result { + if !layout.remote_exists() { + return Err(RepositoryError::NoRemote); + } + let config = try!(Config::load(layout.config_path())); + let remote_locks = LockFolder::new(layout.remote_locks_path()); + try!(fs::create_dir_all(layout.local_locks_path())); // Added after v0.1.0 + let local_locks = LockFolder::new(layout.local_locks_path()); + let lock = try!(local_locks.lock(false)); + let mock_lock = Lock; + let bundles = try!(BundleDb::open(layout.clone(), crypto.clone(), &mock_lock)); + let mut rebuild_index = false; + let mut rebuild_bundle_map = false; + let index = match unsafe { Index::open(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION) } { + Ok(index) => index, + Err(err) => { + tr_error!("Failed to load local index:\n\tcaused by: {}", err); + if read_only { + return Err(err.into()); + } + try!(Index::create(layout.index_path(), &INDEX_MAGIC, INDEX_VERSION)) + } + }; + let bundle_map = match BundleMap::load(layout.bundle_map_path(), &mock_lock) { + Ok(bundle_map) => bundle_map, + Err(err) => { + tr_error!("Failed to load local bundle map:\n\tcaused by: {}", err); + if read_only { + return Err(err.into()); + } + BundleMap::create() + } + }; + let mut repo = Repository { + layout, + chunker: config.chunker.create(), + config, + index, + crypto, + bundle_map, + next_data_bundle: 0, + next_meta_bundle: 0, + bundles, + data_bundle: None, + meta_bundle: None, + remote_locks, + local_locks + }; + if rebuild_bundle_map { + try!(repo.rebuild_bundle_map(&mock_lock)); + rebuild_index = true; + } + if rebuild_index { + try!(repo.rebuild_index(&mock_lock)); + } + repo.next_meta_bundle = repo.next_free_bundle_id(); + repo.next_data_bundle = repo.next_free_bundle_id(); + Ok(repo) + } + + pub fn synchronize(&mut self, lock: &OnlineMode) -> Result<(), RepositoryError> { + let (new, gone) = try!(self.bundles.synchronize(lock)); + let mut save_bundle_map = false; + if !gone.is_empty() { + tr_info!("Removing {} old bundles from index", gone.len()); + for bundle in gone { + try!(self.remove_gone_remote_bundle(&bundle, lock.as_localwrite())) + } + save_bundle_map = true; + } + if !new.is_empty() { + tr_info!("Adding {} new bundles to index", new.len()); + for bundle in ProgressIter::new(tr!("adding bundles to index"), new.len(), new.into_iter()) { + try!(self.add_new_remote_bundle(&bundle, lock)) + } + save_bundle_map = true; + } + if save_bundle_map { + try!(self.save_bundle_map(lock.as_localwrite())); + } + self.next_meta_bundle = self.next_free_bundle_id(); + self.next_data_bundle = self.next_free_bundle_id(); + Ok(()) + } + + #[inline] + pub fn save_config(&mut self, lock: &LocalWriteMode) -> Result<(), RepositoryError> { + try!(self.config.save(self.layout.config_path(), lock)); + Ok(()) + } + + #[inline] + pub fn set_encryption(&mut self, public: Option<&PublicKey>) { + if let Some(key) = public { + if !self.crypto.contains_secret_key(key) { + tr_warn!("The secret key for that public key is not stored in the repository.") + } + let mut key_bytes = Vec::new(); + key_bytes.extend_from_slice(&key[..]); + self.config.encryption = Some((EncryptionMethod::Sodium, key_bytes.into())) + } else { + self.config.encryption = None + } + } + + #[inline] + pub fn save_bundle_map(&self, lock: &LocalWriteMode) -> Result<(), RepositoryError> { + try!(self.bundle_map.save(self.layout.bundle_map_path(), lock)); + Ok(()) + } + + #[inline] + pub fn next_free_bundle_id(&self) -> u32 { + let mut id = max(self.next_data_bundle, self.next_meta_bundle) + 1; + while self.bundle_map.get(id).is_some() { + id += 1; + } + id + } + + pub fn flush(&mut self, lock: &BackupMode) -> Result<(), RepositoryError> { + if self.data_bundle.is_some() { + let mut finished = None; + mem::swap(&mut self.data_bundle, &mut finished); + { + let bundle = try!(self.bundles.add_bundle(finished.unwrap(), lock)); + self.bundle_map.set( + self.next_data_bundle, + bundle.id.clone(), + lock.as_localwrite() + ); + } + self.next_data_bundle = self.next_free_bundle_id() + } + if self.meta_bundle.is_some() { + let mut finished = None; + mem::swap(&mut self.meta_bundle, &mut finished); + { + let bundle = try!(self.bundles.add_bundle(finished.unwrap(), lock)); + self.bundle_map.set( + self.next_meta_bundle, + bundle.id.clone(), + lock.as_localwrite() + ); + } + self.next_meta_bundle = self.next_free_bundle_id() + } + try!(self.bundles.flush(lock)); + try!(self.save_bundle_map(lock.as_localwrite())); + Ok(()) + } + + fn add_new_remote_bundle(&mut self, bundle: &BundleInfo, lock: &OnlineMode) -> Result<(), RepositoryError> { + if self.bundle_map.find(&bundle.id).is_some() { + return Ok(()); + } + tr_debug!("Adding new bundle to index: {}", bundle.id); + let bundle_id = match bundle.mode { + BundleMode::Data => self.next_data_bundle, + BundleMode::Meta => self.next_meta_bundle, + }; + let chunks = try!(self.bundles.get_chunk_list(&bundle.id, lock)); + self.bundle_map.set(bundle_id, bundle.id.clone(), lock.as_localwrite()); + if self.next_meta_bundle == bundle_id { + self.next_meta_bundle = self.next_free_bundle_id() + } + if self.next_data_bundle == bundle_id { + self.next_data_bundle = self.next_free_bundle_id() + } + for (i, (hash, _len)) in chunks.into_inner().into_iter().enumerate() { + if let Some(old) = try!(self.index.set( + &hash, + &Location { + bundle: bundle_id as u32, + chunk: i as u32 + } + )) + { + // Duplicate chunk, forced ordering: higher bundle id wins + let old_bundle_id = try!(self.get_bundle_id(old.bundle)); + if old_bundle_id > bundle.id { + try!(self.index.set(&hash, &old)); + } + } + } + Ok(()) + } + + fn remove_gone_remote_bundle(&mut self, bundle: &BundleInfo, lock: &LocalWriteMode) -> Result<(), RepositoryError> { + if let Some(id) = self.bundle_map.find(&bundle.id) { + tr_debug!("Removing bundle from index: {}", bundle.id); + try!(self.bundles.delete_local_bundle(&bundle.id, lock)); + try!(self.index.filter(|_key, data| data.bundle != id)); + self.bundle_map.remove(id, lock); + } + Ok(()) + } + + pub fn get_chunk_location(&self, chunk: Hash) -> Option { + self.index.get(&chunk) + } + + pub fn get_config(&self) -> &Config { + &self.config + } + + pub fn set_config(&mut self, config: Config) { + self.config = config; + } +} + + +struct Lock; + + +/** +- Local: readonly, shared lock +- Remote: offline +**/ +pub trait ReadonlyMode {} + +impl ReadonlyMode for Lock {} + + +/** +- Local: writable, exclusive lock, dirty flag +- Remote: offline +**/ +pub trait LocalWriteMode: ReadonlyMode { + fn as_readonly(&self) -> &ReadonlyMode; +} + +impl LocalWriteMode for Lock { + fn as_readonly(&self) -> &ReadonlyMode { + self + } +} + + +/** +- Local: writable, exclusive lock, dirty flag +- Remote: readonly, shared lock +**/ +pub trait OnlineMode: LocalWriteMode { + fn as_localwrite(&self) -> &LocalWriteMode; +} + +impl OnlineMode for Lock { + fn as_localwrite(&self) -> &LocalWriteMode { + self + } +} + + +/** +- Local: writable, exclusive lock, dirty flag +- Remote: append-only, shared lock +**/ +pub trait BackupMode: OnlineMode { + fn as_online(&self) -> &OnlineMode; +} + +impl BackupMode for Lock { + fn as_online(&self) -> &OnlineMode { + self + } +} + + +/** +- Local: writable, exclusive lock, dirty flag +- Remote: writable, exclusive lock +**/ +pub trait VacuumMode: BackupMode { + fn as_backup(&self) -> &BackupMode; +} + +impl VacuumMode for Lock { + fn as_backup(&self) -> &BackupMode { + self + } +} + + +impl Repository { + fn create_dirty_file(&mut self) -> Result<(), RepositoryError> { + let dirtyfile = self.layout.dirtyfile_path(); + if !dirtyfile.exists() { + try!(File::create(&dirtyfile)); + Ok(()) + } else { + Err(RepositoryError::Dirty) + } + } + + fn delete_dirty_file(&mut self) -> Result<(), RepositoryError> { + let dirtyfile = self.layout.dirtyfile_path(); + if dirtyfile.exists() { + try!(fs::remove_file(&dirtyfile)); + } + Ok(()) + } + + pub fn readonly_mode Result> (&mut self, f: F) -> Result { + let _local_lock = try!(self.local_locks.lock(false)); + f(self, &Lock) + } + + pub fn localwrite_mode Result> (&mut self, f: F) -> Result { + let _local_lock = try!(self.local_locks.lock(true)); + f(self, &Lock) + } + + pub fn online_mode Result> (&mut self, f: F) -> Result { + let _local_lock = try!(self.local_locks.lock(true)); + let _remote_lock = try!(self.remote_locks.lock(false)); + f(self, &Lock) + } + + pub fn backup_mode Result> (&mut self, f: F) -> Result { + let _local_lock = try!(self.local_locks.lock(true)); + let _remote_lock = try!(self.remote_locks.lock(false)); + try!(self.create_dirty_file()); + let res = f(self, &Lock); + try!(self.flush(&Lock)); + if res.is_ok() { + try!(self.delete_dirty_file()); + } + res + } + + pub fn vacuum_mode Result> (&mut self, f: F) -> Result { + let _local_lock = try!(self.local_locks.lock(true)); + let _remote_lock = try!(self.remote_locks.lock(true)); + try!(self.create_dirty_file()); + let res = f(self, &Lock); + try!(self.flush(&Lock)); + if res.is_ok() { + try!(self.delete_dirty_file()); + } + res + } +} diff --git a/src/repository/inner/vacuum.rs b/src/repository/vacuum.rs similarity index 73% rename from src/repository/inner/vacuum.rs rename to src/repository/vacuum.rs index 3060cee..c71ee61 100644 --- a/src/repository/inner/vacuum.rs +++ b/src/repository/vacuum.rs @@ -3,17 +3,17 @@ use prelude::*; use std::collections::HashMap; -impl RepositoryInner { - pub fn delete_bundle(&mut self, id: u32) -> Result<(), RepositoryError> { - if let Some(bundle) = self.bundle_map.remove(id) { - try!(self.bundles.delete_bundle(&bundle)); +impl Repository { + pub fn delete_bundle(&mut self, id: u32, lock: &VacuumMode) -> Result<(), RepositoryError> { + if let Some(bundle) = self.bundle_map.remove(id, lock.as_localwrite()) { + try!(self.bundles.delete_bundle(&bundle, lock)); Ok(()) } else { Err(IntegrityError::MissingBundleId(id).into()) } } - pub fn rewrite_bundles(&mut self, rewrite_bundles: &[u32], usage: &HashMap) -> Result<(), RepositoryError> { + pub fn rewrite_bundles(&mut self, rewrite_bundles: &[u32], usage: &HashMap, lock: &VacuumMode) -> Result<(), RepositoryError> { for &id in ProgressIter::new( tr!("rewriting bundles"), rewrite_bundles.len(), @@ -22,18 +22,18 @@ impl RepositoryInner { { let bundle = &usage[&id]; let bundle_id = self.bundle_map.get(id).unwrap(); - let chunks = try!(self.bundles.get_chunk_list(&bundle_id)); + let chunks = try!(self.bundles.get_chunk_list(&bundle_id, lock.as_online())); let mode = usage[&id].info.mode; for (chunk, &(hash, _len)) in chunks.into_iter().enumerate() { if !bundle.chunk_usage.get(chunk) { try!(self.index.delete(&hash)); continue; } - let data = try!(self.bundles.get_chunk(&bundle_id, chunk)); - try!(self.put_chunk_override(mode, hash, &data)); + let data = try!(self.bundles.get_chunk(&bundle_id, chunk, lock.as_online())); + try!(self.put_chunk_override(mode, hash, &data, lock.as_backup())); } } - try!(self.flush()); + try!(self.flush(lock.as_backup())); tr_info!("Checking index"); for (hash, location) in self.index.iter() { let loc_bundle = location.bundle; @@ -49,9 +49,9 @@ impl RepositoryInner { } tr_info!("Deleting {} bundles", rewrite_bundles.len()); for &id in rewrite_bundles { - try!(self.delete_bundle(id)); + try!(self.delete_bundle(id, lock)); } - try!(self.save_bundle_map()); + try!(self.save_bundle_map(lock.as_localwrite())); Ok(()) } } diff --git a/src/util/mod.rs b/src/util/mod.rs index 1ac1075..a90e8c0 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -10,6 +10,7 @@ mod hostname; mod fs; mod lock; mod statistics; +mod mode_test; pub mod msgpack; diff --git a/src/util/mode_test.rs b/src/util/mode_test.rs index 348aace..60de71d 100644 --- a/src/util/mode_test.rs +++ b/src/util/mode_test.rs @@ -23,202 +23,95 @@ VacuumMode **/ -pub enum RepositoryError { - Error +pub struct Repository { + } -enum LockMode { - None, Shared, Exclusive -} - - - -struct RepositoryInner { -} - - -impl RepositoryInner { - fn set_local_lock(&mut self, mode: LockMode) -> Result<(), RepositoryError> { - Ok(()) +impl Repository { + pub fn readonly_mode R> (&mut self, f: F) -> R { + f(self, &Lock) } - fn set_remote_lock(&mut self, mode: LockMode) -> Result<(), RepositoryError> { - Ok(()) + pub fn localwrite_mode R> (&mut self, f: F) -> R { + f(self, &Lock) } - fn set_dirty(&mut self, dirty: bool) -> Result<(), RepositoryError> { - Ok(()) + pub fn online_mode R> (&mut self, f: F) -> R { + f(self, &Lock) } -} + pub fn backup_mode R> (&mut self, f: F) -> R { + f(self, &Lock) + } -pub trait ReadonlyMode { - fn func1(&self) -> Result<(), RepositoryError>; -} - -impl ReadonlyMode for RepositoryInner { - fn func1(&self) -> Result<(), RepositoryError> { - Ok(()) + pub fn vacuum_mode R> (&mut self, f: F) -> R { + f(self, &Lock) } } +struct Lock; + +pub trait ReadonlyMode {} + +impl ReadonlyMode for Lock {} + + pub trait LocalWriteMode: ReadonlyMode { - fn func2(&self) -> Result<(), RepositoryError>; + fn as_readonly(&self) -> &ReadonlyMode; } -impl LocalWriteMode for RepositoryInner { - fn func2(&self) -> Result<(), RepositoryError> { - Ok(()) +impl LocalWriteMode for Lock { + fn as_readonly(&self) -> &ReadonlyMode { + self } } pub trait OnlineMode: LocalWriteMode { - + fn as_localwrite(&self) -> &LocalWriteMode; } -impl OnlineMode for RepositoryInner { - +impl OnlineMode for Lock { + fn as_localwrite(&self) -> &LocalWriteMode { + self + } } pub trait BackupMode: OnlineMode { - + fn as_online(&self) -> &OnlineMode; } -impl BackupMode for RepositoryInner { - +impl BackupMode for Lock { + fn as_online(&self) -> &OnlineMode { + self + } } pub trait VacuumMode: BackupMode { - + fn as_backup(&self) -> &BackupMode; } -impl VacuumMode for RepositoryInner { - -} - - - -pub trait UpgradeToLocalWriteMode { - fn in_local_write_mode, F: FnOnce(&mut LocalWriteMode) -> Result>(&mut self, f: F) -> Result; -} - -impl UpgradeToLocalWriteMode for RepositoryInner { - fn in_local_write_mode, F: FnOnce(&mut LocalWriteMode) -> Result>(&mut self, f: F) -> Result { - try!(self.set_local_lock(LockMode::Exclusive)); - try!(self.set_dirty(true)); - let res = f(self); - if res.is_ok() { - try!(self.set_dirty(false)); - } - try!(self.set_local_lock(LockMode::Shared)); - res - } -} - - -pub trait UpgradeToOnlineMode { - fn in_online_mode, F: FnOnce(&mut OnlineMode) -> Result>(&mut self, f: F) -> Result; -} - -impl UpgradeToOnlineMode for RepositoryInner { - fn in_online_mode, F: FnOnce(&mut OnlineMode) -> Result>(&mut self, f: F) -> Result { - try!(self.set_local_lock(LockMode::Exclusive)); - try!(self.set_remote_lock(LockMode::Shared)); - try!(self.set_dirty(true)); - let res = f(self); - if res.is_ok() { - try!(self.set_dirty(false)); - } - try!(self.set_remote_lock(LockMode::None)); - try!(self.set_local_lock(LockMode::Shared)); - res - } -} - - -pub trait UpgradeToBackupMode { - fn in_backup_mode, F: FnOnce(&mut BackupMode) -> Result>(&mut self, f: F) -> Result; -} - -impl UpgradeToBackupMode for RepositoryInner { - fn in_backup_mode, F: FnOnce(&mut BackupMode) -> Result>(&mut self, f: F) -> Result { - try!(self.set_local_lock(LockMode::Exclusive)); - try!(self.set_remote_lock(LockMode::Shared)); - try!(self.set_dirty(true)); - let res = f(self); - if res.is_ok() { - try!(self.set_dirty(false)); - } - try!(self.set_remote_lock(LockMode::None)); - try!(self.set_local_lock(LockMode::Shared)); - res - } -} - - -pub trait UpgradeToVacuumMode { - fn in_vacuum_mode, F: FnOnce(&mut VacuumMode) -> Result>(&mut self, f: F) -> Result; -} - -impl UpgradeToVacuumMode for RepositoryInner { - fn in_vacuum_mode, F: FnOnce(&mut VacuumMode) -> Result>(&mut self, f: F) -> Result { - try!(self.set_local_lock(LockMode::Exclusive)); - try!(self.set_remote_lock(LockMode::Exclusive)); - try!(self.set_dirty(true)); - let res = f(self); - if res.is_ok() { - try!(self.set_dirty(false)); - } - try!(self.set_remote_lock(LockMode::None)); - try!(self.set_local_lock(LockMode::Shared)); - res - } -} - - -pub struct Repository(RepositoryInner); - -impl ReadonlyMode for Repository { - fn func1(&self) -> Result<(), RepositoryError> { - self.0.func1() - } -} - -impl UpgradeToLocalWriteMode for Repository { - fn in_local_write_mode, F: FnOnce(&mut LocalWriteMode) -> Result>(&mut self, f: F) -> Result { - self.0.in_local_write_mode(f) - } -} - -impl UpgradeToOnlineMode for Repository { - fn in_online_mode, F: FnOnce(&mut OnlineMode) -> Result>(&mut self, f: F) -> Result { - self.0.in_online_mode(f) - } -} - -impl UpgradeToBackupMode for Repository { - fn in_backup_mode, F: FnOnce(&mut BackupMode) -> Result>(&mut self, f: F) -> Result { - self.0.in_backup_mode(f) - } -} - -impl UpgradeToVacuumMode for Repository { - fn in_vacuum_mode, F: FnOnce(&mut VacuumMode) -> Result>(&mut self, f: F) -> Result { - self.0.in_vacuum_mode(f) +impl VacuumMode for Lock { + fn as_backup(&self) -> &BackupMode { + self } } impl Repository { + fn write(&mut self, w: W, lock: &LocalWriteMode) { -} + } - -fn test_it(mut repo: Repository) { - repo.func1(); - repo.in_local_write_mode(|repo| repo.func2()); + fn test(&mut self) { + self.localwrite_mode(|repo, lock| { + repo.write(&mut Vec::new(), lock) + }); + self.online_mode(|repo, lock| { + repo.write(&mut Vec::new(), lock.as_localwrite()) + }); + } } \ No newline at end of file