Browse Source

Parsing arguments with Docopt

pull/10/head
Dennis Schwerdel 5 years ago
parent
commit
6aa35bcdb5
  1. 116
      Cargo.lock
  2. 2
      Cargo.toml
  3. 145
      src/main.rs
  4. 43
      src/repository/config.rs
  5. 29
      src/util/compression.rs

116
Cargo.lock generated

@ -3,17 +3,26 @@ name = "zvault"
version = "0.1.0"
dependencies = [
"blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)",
"docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
"mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rmp-serde 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
"squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "aho-corasick"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "bitflags"
version = "0.7.0"
@ -38,9 +47,15 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "gcc"
version = "0.3.43"
name = "docopt"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)",
"strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "kernel32-sys"
@ -51,6 +66,11 @@ dependencies = [
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "lazy_static"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "libc"
version = "0.1.12"
@ -66,6 +86,14 @@ name = "linked-hash-map"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "memchr"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "mmap"
version = "0.1.1"
@ -104,8 +132,20 @@ dependencies = [
]
[[package]]
name = "redox_syscall"
version = "0.1.16"
name = "regex"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"aho-corasick 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"regex-syntax 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"thread_local 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "regex-syntax"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@ -127,18 +167,6 @@ dependencies = [
"serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rust-crypto"
version = "0.2.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"gcc 0.3.43 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
"rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "rustc-serialize"
version = "0.3.22"
@ -177,6 +205,11 @@ dependencies = [
"pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "strsim"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "tempdir"
version = "0.3.5"
@ -186,16 +219,41 @@ dependencies = [
]
[[package]]
name = "time"
version = "0.1.36"
name = "thread-id"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "thread_local"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"thread-id 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "unreachable"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "utf8-ranges"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "void"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winapi"
version = "0.2.8"
@ -215,32 +273,40 @@ dependencies = [
]
[metadata]
"checksum aho-corasick 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0638fd549427caa90c499814196d1b9e3725eb4d15d7339d6de073a680ed0ca2"
"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d"
"checksum blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "0c6a476f32fef3402f1161f89d0d39822809627754a126f8441ff2a9d45e2d59"
"checksum byteorder 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c40977b0ee6b9885c9013cd41d9feffdd22deb3bb4dc3a71d901cc7a77de18c8"
"checksum constant_time_eq 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "07dcb7959f0f6f1cf662f9a7ff389bcb919924d99ac41cf31f10d611d8721323"
"checksum gcc 0.3.43 (registry+https://github.com/rust-lang/crates.io-index)" = "c07c758b972368e703a562686adb39125707cc1ef3399da8c019fc6c2498a75d"
"checksum docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab32ea6e284d87987066f21a9e809a73c14720571ef34516f0890b3d355ccfd8"
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
"checksum lazy_static 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7291b1dd97d331f752620b02dfdbc231df7fc01bf282a00769e1cdb963c460dc"
"checksum libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122"
"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135"
"checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd"
"checksum memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1dbccc0e46f1ea47b9f17e6d67c5a96bd27030519c519c9c91327e31275a47b4"
"checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823"
"checksum murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a2983372caf4480544083767bf2d27defafe32af49ab4df3a0b7fc90793a3664"
"checksum num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "e1cbfa3781f3fe73dc05321bed52a06d2d491eaa764c52335cf4399f046ece99"
"checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903"
"checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c"
"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d"
"checksum redox_syscall 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "8dd35cc9a8bdec562c757e3d43c1526b5c6d2653e23e2315065bc25556550753"
"checksum regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4278c17d0f6d62dfef0ab00028feb45bd7d2102843f80763474eeb1be8a10c01"
"checksum regex-syntax 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9191b1f57603095f105d317e375d19b1c9c5c3185ea9633a99a6dcbed04457"
"checksum rmp 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e59917c01f49718a59c644a621a4848aafc6577c4a47d66270d78951a807541a"
"checksum rmp-serde 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "06ec4d0cdea2645de5d0e649f90c3e654205d913e14adefa452257314a24e76e"
"checksum rust-crypto 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)" = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a"
"checksum rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "237546c689f20bb44980270c73c3b9edd0891c1be49cc1274406134a66d3957b"
"checksum serde 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)" = "a702319c807c016e51f672e5c77d6f0b46afddd744b5e437d6b8436b888b458f"
"checksum serde_utils 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b34a52969c7fc0254e214b82518c9a95dc88c84fc84cd847add314996a031be6"
"checksum serde_yaml 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f8bd3f24ad8c7bcd34a6d70ba676dc11302b96f4f166aa5f947762e01098844d"
"checksum squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db1f9dde91d819b7746e153bc32489fa19e6a106c3d7f2b92187a4efbdc88b40"
"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694"
"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6"
"checksum time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "211b63c112206356ef1ff9b19355f43740fc3f85960c598a93d3a3d3ba7beade"
"checksum thread-id 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4437c97558c70d129e40629a5b385b3fb1ffac301e63941335e4d354081ec14a"
"checksum thread_local 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c85048c6260d17cf486ceae3282d9fb6b90be220bf5b28c400f5485ffc29f0c7"
"checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91"
"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122"
"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
"checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
"checksum yaml-rust 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992"

2
Cargo.toml

@ -13,3 +13,5 @@ mmap = "*"
quick-error = "1.1"
blake2-rfc = "*"
murmurhash3 = "*"
docopt = "0.7"
rustc-serialize = "0.3"

145
src/main.rs

@ -7,6 +7,8 @@ extern crate blake2_rfc as blake2;
extern crate murmurhash3;
extern crate serde_yaml;
#[macro_use] extern crate quick_error;
extern crate docopt;
extern crate rustc_serialize;
mod errors;
mod util;
@ -20,59 +22,110 @@ use chunker::ChunkerType;
use repository::{Repository, Config, Mode};
use util::{ChecksumType, Compression, HashMethod};
use std::path::Path;
use std::fs::File;
use std::env;
use std::io::Read;
use std::time;
use docopt::Docopt;
static USAGE: &'static str = "
Usage:
zvault init <repo>
zvault algotest <path>
zvault check [--full] <repo>
zvault test <repo> <path>
Options:
--full Whether to verify the repository by loading all bundles
--bundle-size SIZE The target size of a full bundle in MiB [default: 25]
--chunker METHOD The chunking algorithm to use [default: fastcdc]
--chunk-size SIZE The target average chunk size in KiB [default: 8]
--compression COMPRESSION The compression to use [default: brotli/3]
";
#[derive(RustcDecodable, Debug)]
struct Args {
cmd_init: bool,
cmd_algotest: bool,
cmd_test: bool,
cmd_check: bool,
arg_repo: Option<String>,
arg_path: Option<String>,
flag_full: bool,
flag_bundle_size: usize,
flag_chunker: String,
flag_chunk_size: usize,
flag_compression: String
}
fn main() {
let path: &Path = "test_data".as_ref();
let mut repo = if path.exists() {
Repository::open(path).unwrap()
} else {
Repository::create(path, Config {
bundle_size: 25*1024*1024,
let args: Args = Docopt::new(USAGE).and_then(|d| d.decode()).unwrap_or_else(|e| e.exit());
println!("{:?}", args);
if args.cmd_algotest {
algotest::run(&args.arg_path.unwrap());
return
}
if args.cmd_init {
let chunker = ChunkerType::from(&args.flag_chunker, args.flag_chunk_size, 0).expect("No such chunk algorithm");
let compression = if args.flag_compression == "none" {
None
} else {
Some(Compression::from_string(&args.flag_compression).expect("Failed to parse compression"))
};
Repository::create(&args.arg_repo.unwrap(), Config {
bundle_size: args.flag_bundle_size*1024*1024,
checksum: ChecksumType::Blake2_256,
chunker: ChunkerType::FastCdc((8*1024, 0)),
compression: Some(Compression::Brotli(1)),
chunker: chunker,
compression: compression,
hash: HashMethod::Blake2
}).unwrap()
};
print!("Integrity check before...");
repo.check(true).unwrap();
println!(" done.");
let file_path = env::args().nth(1).expect("Need file as argument");
print!("Reading file {}...", file_path);
let mut data = Vec::new();
let mut file = File::open(file_path).unwrap();
file.read_to_end(&mut data).unwrap();
println!(" done. {} bytes", data.len());
print!("Adding data to repository...");
let start = time::Instant::now();
let chunks = repo.put_data(Mode::Content, &data).unwrap();
repo.flush().unwrap();
let elapsed = start.elapsed();
let duration = elapsed.as_secs() as f64 * 1.0 + elapsed.subsec_nanos() as f64 / 1_000_000_000.0;
let write_speed = data.len() as f64 / duration;
println!(" done. {} chunks, {:.1} MB/s", chunks.len(), write_speed / 1_000_000.0);
println!("Integrity check after...");
repo.check(true).unwrap();
println!(" done.");
print!("Reading data from repository...");
let start = time::Instant::now();
let data2 = repo.get_data(&chunks).unwrap();
let elapsed = start.elapsed();
let duration = elapsed.as_secs() as f64 * 1.0 + elapsed.subsec_nanos() as f64 / 1_000_000_000.0;
let read_speed = data.len() as f64 / duration;
assert_eq!(data.len(), data2.len());
println!(" done. {:.1} MB/s", read_speed / 1_000_000.0);
//algotest::run("test.tar");
}).unwrap();
return
}
let mut repo = Repository::open(&args.arg_repo.unwrap()).unwrap();
if args.cmd_check {
repo.check(args.flag_full).unwrap();
return
}
if args.cmd_test {
print!("Integrity check before...");
repo.check(true).unwrap();
println!(" done.");
let file_path = args.arg_path.unwrap();
print!("Reading file {}...", file_path);
let mut data = Vec::new();
let mut file = File::open(file_path).unwrap();
file.read_to_end(&mut data).unwrap();
println!(" done. {} bytes", data.len());
print!("Adding data to repository...");
let start = time::Instant::now();
let chunks = repo.put_data(Mode::Content, &data).unwrap();
repo.flush().unwrap();
let elapsed = start.elapsed();
let duration = elapsed.as_secs() as f64 * 1.0 + elapsed.subsec_nanos() as f64 / 1_000_000_000.0;
let write_speed = data.len() as f64 / duration;
println!(" done. {} chunks, {:.1} MB/s", chunks.len(), write_speed / 1_000_000.0);
println!("Integrity check after...");
repo.check(true).unwrap();
println!(" done.");
print!("Reading data from repository...");
let start = time::Instant::now();
let data2 = repo.get_data(&chunks).unwrap();
let elapsed = start.elapsed();
let duration = elapsed.as_secs() as f64 * 1.0 + elapsed.subsec_nanos() as f64 / 1_000_000_000.0;
let read_speed = data.len() as f64 / duration;
assert_eq!(data.len(), data2.len());
println!(" done. {:.1} MB/s", read_speed / 1_000_000.0);
}
}

43
src/repository/config.rs

@ -67,47 +67,22 @@ impl ChunkerType {
struct CompressionYaml {
codec: String,
level: Option<u8>
}
impl Default for CompressionYaml {
fn default() -> Self {
CompressionYaml {
codec: "brotli".to_string(),
level: None
}
}
}
serde_impl!(CompressionYaml(String) {
codec: String => "codec",
level: Option<u8> => "level"
});
impl Compression {
fn from_yaml(yaml: CompressionYaml) -> Result<Self, &'static str> {
match &yaml.codec as &str {
"snappy" => Ok(Compression::Snappy(())),
"zstd" => Ok(Compression::ZStd(yaml.level.unwrap_or(5))),
"deflate" | "zlib" | "gzip" => Ok(Compression::Deflate(yaml.level.unwrap_or(5))),
"brotli" => Ok(Compression::Brotli(yaml.level.unwrap_or(5))),
"lzma2" => Ok(Compression::Lzma2(yaml.level.unwrap_or(5))),
_ => Err("Unsupported codec")
}
#[inline]
fn from_yaml(yaml: String) -> Result<Self, &'static str> {
Compression::from_string(&yaml)
}
fn to_yaml(&self) -> CompressionYaml {
CompressionYaml {
codec: self.name().to_string(),
level: self.level()
}
#[inline]
fn to_yaml(&self) -> String {
self.to_string()
}
}
struct ConfigYaml {
compression: Option<CompressionYaml>,
compression: Option<String>,
bundle_size: usize,
chunker: ChunkerYaml,
checksum: String,
@ -116,7 +91,7 @@ struct ConfigYaml {
impl Default for ConfigYaml {
fn default() -> Self {
ConfigYaml {
compression: Some(CompressionYaml { codec: "brotli".to_string(), level: Some(5) }),
compression: Some("brotli/5".to_string()),
bundle_size: 25*1024*1024,
chunker: ChunkerYaml::default(),
checksum: "blake2_256".to_string(),
@ -125,7 +100,7 @@ impl Default for ConfigYaml {
}
}
serde_impl!(ConfigYaml(String) {
compression: Option<CompressionYaml> => "compression",
compression: Option<String> => "compression",
bundle_size: usize => "bundle_size",
chunker: ChunkerYaml => "chunker",
checksum: String => "checksum",

29
src/util/compression.rs

@ -1,6 +1,7 @@
use std::ptr;
use std::ffi::{CStr, CString};
use std::io::Write;
use std::str::FromStr;
use squash::*;
@ -23,6 +24,34 @@ serde_impl!(Compression(u64) {
impl Compression {
#[inline]
pub fn to_string(&self) -> String {
if let Some(level) = self.level() {
format!("{}/{}", self.name(), level)
} else {
self.name().to_string()
}
}
#[inline]
pub fn from_string(name: &str) -> Result<Self, &'static str> {
let (name, level) = if let Some(pos) = name.find("/") {
let level = try!(u8::from_str(&name[pos+1..]).map_err(|_| "Level must be a number"));
let name = &name[..pos];
(name, level)
} else {
(name, 5)
};
match name {
"snappy" => Ok(Compression::Snappy(())),
"zstd" => Ok(Compression::ZStd(level)),
"deflate" | "zlib" | "gzip" => Ok(Compression::Deflate(level)),
"brotli" => Ok(Compression::Brotli(level)),
"lzma2" => Ok(Compression::Lzma2(level)),
_ => Err("Unsupported codec")
}
}
#[inline]
pub fn name(&self) -> &'static str {
match *self {

Loading…
Cancel
Save