From 69eaf4085e860d4c5f80622989c320080445b79e Mon Sep 17 00:00:00 2001 From: Dennis Schwerdel Date: Fri, 17 Mar 2017 11:03:07 +0100 Subject: [PATCH] algotest --- .gitignore | 1 + Cargo.lock | 98 +--------------- Cargo.toml | 2 +- src/chunker/mod.rs | 14 +++ src/cli/algotest.rs | 234 ++++++++++++++++++++----------------- src/cli/args.rs | 245 ++++++++++++++++++++++++++------------- src/cli/logger.rs | 23 ++++ src/cli/mod.rs | 272 ++++++++++++++++++++++---------------------- src/main.rs | 6 +- src/util/cli.rs | 13 +-- 10 files changed, 489 insertions(+), 419 deletions(-) create mode 100644 src/cli/logger.rs diff --git a/.gitignore b/.gitignore index 4b1923a..69ba049 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ target squash test.tar test_* +restored diff --git a/Cargo.lock b/Cargo.lock index f5afab2..309f16f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5,7 +5,7 @@ dependencies = [ "blake2-rfc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)", - "docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", "mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -17,14 +17,6 @@ dependencies = [ "squash-sys 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "aho-corasick" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "ansi_term" version = "0.9.0" @@ -92,17 +84,6 @@ name = "constant_time_eq" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "docopt" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "kernel32-sys" version = "0.2.2" @@ -112,11 +93,6 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "lazy_static" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "libc" version = "0.1.12" @@ -133,12 +109,9 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "memchr" -version = "1.0.1" +name = "log" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "mmap" @@ -209,23 +182,6 @@ name = "redox_syscall" version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "regex" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "aho-corasick 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex-syntax" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "rmp" version = "0.8.4" @@ -306,24 +262,6 @@ dependencies = [ "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "thread-id" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "thread_local" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "thread-id 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "time" version = "0.1.36" @@ -345,29 +283,11 @@ name = "unicode-width" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "unreachable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "utf8-ranges" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "vec_map" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "winapi" version = "0.2.8" @@ -387,7 +307,6 @@ dependencies = [ ] [metadata] -"checksum aho-corasick 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0638fd549427caa90c499814196d1b9e3725eb4d15d7339d6de073a680ed0ca2" "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" "checksum atty 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d912da0db7fa85514874458ca3651fe2cddace8d0b0505571dbdcd41ab490159" "checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" @@ -397,13 +316,11 @@ dependencies = [ "checksum chrono 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "158b0bd7d75cbb6bf9c25967a48a2e9f77da95876b858eadfabaa99cd069de6e" "checksum clap 2.21.1 (registry+https://github.com/rust-lang/crates.io-index)" = "74a80f603221c9cd9aa27a28f52af452850051598537bb6b359c38a7d61e5cda" "checksum constant_time_eq 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "07dcb7959f0f6f1cf662f9a7ff389bcb919924d99ac41cf31f10d611d8721323" -"checksum docopt 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab32ea6e284d87987066f21a9e809a73c14720571ef34516f0890b3d355ccfd8" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -"checksum lazy_static 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "7291b1dd97d331f752620b02dfdbc231df7fc01bf282a00769e1cdb963c460dc" "checksum libc 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "e32a70cf75e5846d53a673923498228bbec6a8624708a9ea5645f075d6276122" "checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135" "checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" -"checksum memchr 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1dbccc0e46f1ea47b9f17e6d67c5a96bd27030519c519c9c91327e31275a47b4" +"checksum log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "5141eca02775a762cc6cd564d8d2c50f67c0ea3a372cbf1c51592b3e029e10ad" "checksum mmap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc85448a6006dd2ba26a385a564a8a0f1f2c7e78c70f1a70b2e0f4af286b823" "checksum murmurhash3 0.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a2983372caf4480544083767bf2d27defafe32af49ab4df3a0b7fc90793a3664" "checksum num 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "98b15ba84e910ea7a1973bccd3df7b31ae282bf9d8bd2897779950c9b8303d40" @@ -414,8 +331,6 @@ dependencies = [ "checksum quick-error 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0aad603e8d7fb67da22dbdf1f4b826ce8829e406124109e73cf1b2454b93a71c" "checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" "checksum redox_syscall 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "8dd35cc9a8bdec562c757e3d43c1526b5c6d2653e23e2315065bc25556550753" -"checksum regex 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4278c17d0f6d62dfef0ab00028feb45bd7d2102843f80763474eeb1be8a10c01" -"checksum regex-syntax 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f9191b1f57603095f105d317e375d19b1c9c5c3185ea9633a99a6dcbed04457" "checksum rmp 0.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e59917c01f49718a59c644a621a4848aafc6577c4a47d66270d78951a807541a" "checksum rmp-serde 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "06ec4d0cdea2645de5d0e649f90c3e654205d913e14adefa452257314a24e76e" "checksum rustc-serialize 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "237546c689f20bb44980270c73c3b9edd0891c1be49cc1274406134a66d3957b" @@ -426,15 +341,10 @@ dependencies = [ "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" "checksum term_size 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "07b6c1ac5b3fffd75073276bca1ceed01f67a28537097a2a9539e116e50fb21a" -"checksum thread-id 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4437c97558c70d129e40629a5b385b3fb1ffac301e63941335e4d354081ec14a" -"checksum thread_local 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c85048c6260d17cf486ceae3282d9fb6b90be220bf5b28c400f5485ffc29f0c7" "checksum time 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)" = "211b63c112206356ef1ff9b19355f43740fc3f85960c598a93d3a3d3ba7beade" "checksum unicode-segmentation 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "18127285758f0e2c6cf325bb3f3d138a12fee27de4f23e146cd6a179f26c2cf3" "checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" -"checksum unreachable 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91" -"checksum utf8-ranges 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "662fab6525a98beff2921d7f61a39e7d59e0b425ebc7d0d9e66d316e55124122" "checksum vec_map 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8cdc8b93bd0198ed872357fb2e667f7125646b1762f16d60b2c96350d361897" -"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" "checksum yaml-rust 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e66366e18dc58b46801afbf2ca7661a9f59cc8c5962c29892b6039b4f86fa992" diff --git a/Cargo.toml b/Cargo.toml index 5194a9b..05a29b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ mmap = "*" quick-error = "1.1" blake2-rfc = "*" murmurhash3 = "*" -docopt = "0.7" rustc-serialize = "0.3" chrono = "0.3" clap = "2.19" +log = "0.3" diff --git a/src/chunker/mod.rs b/src/chunker/mod.rs index 98f7ada..6e6a330 100644 --- a/src/chunker/mod.rs +++ b/src/chunker/mod.rs @@ -1,4 +1,5 @@ use std::io::{self, Write, Read}; +use std::str::FromStr; mod ae; mod rabin; @@ -100,6 +101,19 @@ impl ChunkerType { } } + #[inline] + pub fn from_string(name: &str) -> Result { + let (name, size) = if let Some(pos) = name.find('/') { + let size = try!(usize::from_str(&name[pos+1..]).map_err(|_| "Chunk size must be a number")); + let name = &name[..pos]; + (name, size) + } else { + (name, 8) + }; + Self::from(name, size * 1024, 0) + } + + #[inline] pub fn create(&self) -> Chunker { match *self { diff --git a/src/cli/algotest.rs b/src/cli/algotest.rs index f53681c..6e4d8e9 100644 --- a/src/cli/algotest.rs +++ b/src/cli/algotest.rs @@ -1,122 +1,152 @@ -use std::io::{Cursor, Read}; +use std::io::{self, Cursor, Read, Write}; use std::fs::File; -use std::time; +use std::collections::HashSet; + +use chrono::Duration; use ::chunker::*; use ::util::*; +use ::util::cli::*; -fn speed_chunk(chunker: &mut C, data: &[u8]) { - let mut input = Cursor::new(data); - let mut chunk = Vec::with_capacity(1_000_000); - loop { - chunk.clear(); - let result = chunker.chunk(&mut input, &mut chunk).unwrap(); - if result == ChunkerStatus::Finished { - return - } + +struct ChunkSink { + chunks: Vec<(usize, usize)>, + pos: usize, + written: usize +} + +impl ChunkSink { + fn end_chunk(&mut self) { + self.chunks.push((self.pos, self.written)); + self.pos += self.written; + self.written = 0; } } -fn chunk(chunker: &mut C, data: &[u8]) -> Vec> { - let mut input = Cursor::new(data); - let mut chunks = Vec::with_capacity(100_000); - loop { - let mut chunk = Vec::with_capacity(100_000); - let result = chunker.chunk(&mut input, &mut chunk).unwrap(); - chunks.push(chunk); - if result == ChunkerStatus::Finished { - return chunks; - } +impl Write for ChunkSink { + fn write(&mut self, data: &[u8]) -> Result { + self.written += data.len(); + Ok(data.len()) + } + + fn flush(&mut self) -> Result<(), io::Error> { + Ok(()) } } -fn analyze_chunks(mut chunks: Vec>) -> (usize, f64, f64, f64) { - let count = chunks.len(); - let total = chunks.iter().map(|c| c.len()).sum::(); - let avg_size = total as f64 / count as f64; - let stddev = (chunks.iter().map(|c| (c.len() as f64 - avg_size).powi(2)).sum::() / (count as f64 - 1.0)).sqrt(); - chunks.sort(); - chunks.dedup(); - let non_dup: usize = chunks.iter().map(|c| c.len()).sum(); - let saved = 1.0 - non_dup as f64 / total as f64; - (count, avg_size, stddev, saved) -} - -fn compare_chunker(name: &str, mut chunker: C, data: &[u8]) { - let start = time::Instant::now(); - speed_chunk(&mut chunker, data); - let elapsed = start.elapsed(); - let chunks = chunk(&mut chunker, data); - let duration = elapsed.as_secs() as f64 * 1.0 + elapsed.subsec_nanos() as f64 / 1_000_000_000.0; - let speed = data.len() as f64 / duration; - assert_eq!(chunks.iter().map(|c| c.len()).sum::(), data.len()); - let (_count, avg_size, stddev, saved) = analyze_chunks(chunks); - println!("{}: \tavg chunk size {:.1}\t± {:.1} bytes, \t{:.1}% saved,\tspeed {:.1} MB/s", - name, avg_size, stddev, saved * 100.0, speed / 1_000_000.0); -} - -fn compare_hash(name: &str, hash: HashMethod, data: &[u8]) { - let start = time::Instant::now(); - let _ = hash.hash(data); - let elapsed = start.elapsed(); - let duration = elapsed.as_secs() as f64 * 1.0 + elapsed.subsec_nanos() as f64 / 1_000_000_000.0; - let speed = data.len() as f64 / duration; - println!("{}: {:.1} MB/s", name, speed / 1_000_000.0); -} - -fn compare_compression(name: &str, method: Compression, data: &[u8]) { - let start = time::Instant::now(); - let compressed = method.compress(data).unwrap(); - let elapsed = start.elapsed(); - let duration = elapsed.as_secs() as f64 * 1.0 + elapsed.subsec_nanos() as f64 / 1_000_000_000.0; - let cspeed = data.len() as f64 / duration; - let ratio = compressed.len() as f64 / data.len() as f64; - let start = time::Instant::now(); - let uncompressed = method.decompress(&compressed).unwrap(); - if uncompressed != data { - panic!("{} did not uncompress to the same value", name); +fn chunk(data: &[u8], mut chunker: Chunker, sink: &mut ChunkSink) { + let mut cursor = Cursor::new(data); + while chunker.chunk(&mut cursor, sink).unwrap() == ChunkerStatus::Continue { + sink.end_chunk(); } - let elapsed = start.elapsed(); - let duration = elapsed.as_secs() as f64 * 1.0 + elapsed.subsec_nanos() as f64 / 1_000_000_000.0; - let dspeed = data.len() as f64 / duration; - println!("{}:\tratio: {:.1}%,\tcompress: {:.1} MB/s,\tdecompress: {:.1} MB/s", - name, ratio * 100.0, cspeed / 1_000_000.0, dspeed / 1_000_000.0); + sink.end_chunk(); } #[allow(dead_code)] -pub fn run(path: &str) { - println!("Algorithm comparison on file {}", path); - println!(); - print!("Reading input file..."); +pub fn run(path: &str, bundle_size: usize, chunker: ChunkerType, compression: Option, hash: HashMethod) { + let mut total_time = 0.0; + + println!("Reading input file ..."); let mut file = File::open(path).unwrap(); - let mut data = Vec::new(); - file.read_to_end(&mut data).unwrap(); - println!(" done. {} bytes", data.len()); + let total_size = file.metadata().unwrap().len(); + let mut size = total_size; + let mut data = Vec::with_capacity(size as usize); + let read_time = Duration::span(|| { + file.read_to_end(&mut data).unwrap(); + }).num_milliseconds() as f32 / 1_000.0; + println!("- {}, {}", to_duration(read_time), to_speed(size, read_time)); + println!(); - println!("Chunker algorithms"); - for size in &[4usize, 8, 16, 32, 64] { - println!(" Chunk size: {} KiB", size); - compare_chunker(" AE", AeChunker::new(size*1024), &data); - compare_chunker(" Rabin", RabinChunker::new(size*1024, 0), &data); - compare_chunker(" FastCdc", FastCdcChunker::new(size*1024, 0), &data); - } + + println!("Chunking data with {}, avg chunk size {} ...", chunker.name(), to_file_size(chunker.avg_size() as u64)); + let mut chunk_sink = ChunkSink { + chunks: Vec::with_capacity(2*size as usize/chunker.avg_size()), + written: 0, + pos: 0 + }; + let chunker = chunker.create(); + let chunk_time = Duration::span(|| { + chunk(&data, chunker, &mut chunk_sink) + }).num_milliseconds() as f32 / 1_000.0; + total_time += chunk_time; + println!("- {}, {}", to_duration(chunk_time), to_speed(size, chunk_time)); + let mut chunks = chunk_sink.chunks; + assert_eq!(chunks.iter().map(|c| c.1).sum::(), size as usize); + let chunk_size_avg = size as f32 / chunks.len() as f32; + let chunk_size_stddev = (chunks.iter().map(|c| (c.1 as f32 - chunk_size_avg).powi(2)).sum::() / (chunks.len() as f32 - 1.0)).sqrt(); + println!("- {} chunks, avg size: {}, stddev: {}", chunks.len(), to_file_size(chunk_size_avg as u64), to_file_size(chunk_size_stddev as u64)); + println!(); - println!("Hash algorithms"); - compare_hash(" Blake2", HashMethod::Blake2, &data); - compare_hash(" Murmur3", HashMethod::Murmur3, &data); + + println!("Hashing chunks with {} ...", hash.name()); + let mut hashes = Vec::with_capacity(chunks.len()); + let hash_time = Duration::span(|| { + for &(pos, len) in &chunks { + hashes.push(hash.hash(&data[pos..pos+len])) + } + }).num_milliseconds() as f32 / 1_000.0; + total_time += hash_time; + println!("- {}, {}", to_duration(hash_time), to_speed(size, hash_time)); + let mut seen_hashes = HashSet::with_capacity(hashes.len()); + let mut dups = Vec::new(); + for (i, hash) in hashes.into_iter().enumerate() { + if !seen_hashes.insert(hash) { + dups.push(i); + } + } + let mut dup_size = 0; + dups.reverse(); + for i in &dups { + let (_, len) = chunks.remove(*i); + dup_size += len; + } + println!("- {} duplicate chunks, {}, {:.1}% saved", dups.len(), to_file_size(dup_size as u64), dup_size as f32 / size as f32*100.0); + size -= dup_size as u64; + + if let Some(compression) = compression { + println!(); + + println!("Compressing chunks with {} ...", compression.to_string()); + let mut bundles = Vec::new(); + let compress_time = Duration::span(|| { + let mut bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize); + let mut c = compression.compress_stream().unwrap(); + for &(pos, len) in &chunks { + c.process(&data[pos..pos+len], &mut bundle).unwrap(); + if bundle.len() >= bundle_size { + c.finish(&mut bundle).unwrap(); + bundles.push(bundle); + bundle = Vec::with_capacity(bundle_size + 2*chunk_size_avg as usize); + c = compression.compress_stream().unwrap(); + } + } + c.finish(&mut bundle).unwrap(); + bundles.push(bundle); + }).num_milliseconds() as f32 / 1_000.0; + total_time += compress_time; + println!("- {}, {}", to_duration(compress_time), to_speed(size, compress_time)); + let compressed_size = bundles.iter().map(|b| b.len()).sum::(); + println!("- {} bundles, {}, {:.1}% saved", bundles.len(), to_file_size(compressed_size as u64), (size as f32 - compressed_size as f32)/size as f32*100.0); + size = compressed_size as u64; + + println!(); + + println!("Decompressing bundles with {} ...", compression.to_string()); + let mut dummy = ChunkSink { chunks: vec![], written: 0, pos: 0 }; + let decompress_time = Duration::span(|| { + for bundle in &bundles { + let mut c = compression.decompress_stream().unwrap(); + c.process(bundle, &mut dummy).unwrap(); + c.finish(&mut dummy).unwrap(); + } + }).num_milliseconds() as f32 / 1_000.0; + println!("- {}, {}", to_duration(decompress_time), to_speed(size, decompress_time)); + } + println!(); - println!("Compression algorithms"); - compare_compression(" Snappy", Compression::Snappy(()), &data); - for level in 1..10 { - compare_compression(&format!(" ZStd/{}", level), Compression::ZStd(level), &data); - } - for level in 1..10 { - compare_compression(&format!(" Deflate/{}", level), Compression::Deflate(level), &data); - } - for level in 1..10 { - compare_compression(&format!(" Brotli/{}", level), Compression::Brotli(level), &data); - } - for level in 1..7 { - compare_compression(&format!(" Lzma2/{}", level), Compression::Lzma2(level), &data); - } + + let total_saved = total_size - size; + println!("Total space saved: {}, {:.1}%", to_file_size(total_saved as u64), total_saved as f32/total_size as f32*100.0); + println!("Total processing speed: {}", to_speed(total_size, total_time)); + } diff --git a/src/cli/args.rs b/src/cli/args.rs index 4de3731..387d3a5 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -1,68 +1,7 @@ -use clap::{Arg, App, SubCommand}; - -use docopt::Docopt; - use ::chunker::ChunkerType; -use ::util::{ChecksumType, Compression, HashMethod}; +use ::util::{Compression, HashMethod, ChecksumType}; use std::process::exit; -use std::path::Path; - - -static USAGE: &'static str = " -Usage: - zvault init [--bundle-size SIZE] [--chunker METHOD] [--chunk-size SIZE] [--compression COMPRESSION] - zvault backup [--full] - zvault restore [] - zvault check [--full] - zvault backups - zvault info - zvault list [--tree] - zvault stats - zvault bundles - zvault algotest - -Options: - --tree Print the whole (sub-)tree from the backup - --full Whether to verify the repository by loading all bundles - --bundle-size SIZE The target size of a full bundle in MiB [default: 25] - --chunker METHOD The chunking algorithm to use [default: fastcdc] - --chunk-size SIZE The target average chunk size in KiB [default: 8] - --compression COMPRESSION The compression to use [default: brotli/3] -"; - - -#[derive(RustcDecodable, Debug)] -pub struct DocoptArgs { - pub cmd_init: bool, - pub cmd_backup: bool, - pub cmd_restore: bool, - pub cmd_check: bool, - - pub cmd_backups: bool, - pub cmd_info: bool, - pub cmd_list: bool, - - pub cmd_stats: bool, - pub cmd_bundles: bool, - - pub cmd_algotest: bool, - pub cmd_stat: bool, - - pub arg_file: Option, - pub arg_repo: Option, - pub arg_path: Option, - pub arg_src: Option, - pub arg_dst: Option, - pub arg_backup: Option, - - pub flag_full: bool, - pub flag_bundle_size: usize, - pub flag_chunker: String, - pub flag_chunk_size: usize, - pub flag_compression: String, - pub flag_tree: bool -} pub enum Arguments { @@ -70,8 +9,8 @@ pub enum Arguments { repo_path: String, bundle_size: usize, chunker: ChunkerType, - chunk_size: usize, - compression: Compression + compression: Option, + hash: HashMethod }, Backup { repo_path: String, @@ -105,26 +44,89 @@ pub enum Arguments { repo_path: String }, AlgoTest { - file: String + file: String, + bundle_size: usize, + chunker: ChunkerType, + compression: Option, + hash: HashMethod } } -pub fn parse() -> DocoptArgs { - Docopt::new(USAGE).and_then(|d| d.decode()).unwrap_or_else(|e| e.exit()) +pub fn split_repo_path(repo_path: &str) -> (&str, Option<&str>, Option<&str>) { + let mut parts = repo_path.splitn(3, "::"); + let repo = parts.next().unwrap(); + let backup = parts.next(); + let inode = parts.next(); + (repo, backup, inode) } -pub fn parse2() -> Arguments { +fn parse_num(num: &str, name: &str) -> u64 { + if let Ok(num) = num.parse::() { + num + } else { + error!("{} must be a number, was '{}'", name, num); + exit(1); + } +} + +fn parse_chunker(val: Option<&str>) -> ChunkerType { + if let Ok(chunker) = ChunkerType::from_string(val.unwrap_or("fastcdc/8")) { + chunker + } else { + error!("Invalid chunker method/size: {}", val.unwrap()); + exit(1); + } +} + +fn parse_compression(val: Option<&str>) -> Option { + let val = val.unwrap_or("brotli/3"); + if val == "none" { + return None + } + if let Ok(compression) = Compression::from_string(val) { + Some(compression) + } else { + error!("Invalid compression method/level: {}", val); + exit(1); + } +} + +#[allow(dead_code)] +fn parse_checksum(val: Option<&str>) -> ChecksumType { + if let Ok(checksum) = ChecksumType::from(val.unwrap_or("blake2")) { + checksum + } else { + error!("Invalid checksum method: {}", val.unwrap()); + exit(1); + } +} + +fn parse_hash(val: Option<&str>) -> HashMethod { + if let Ok(hash) = HashMethod::from(val.unwrap_or("blake2")) { + hash + } else { + error!("Invalid hash method: {}", val.unwrap()); + exit(1); + } +} + + +pub fn parse() -> Arguments { let args = clap_app!(zvault => - (version: "0.1") + (version: env!("CARGO_PKG_VERSION")) (author: "Dennis Schwerdel ") (about: "Deduplicating backup tool") + (@setting SubcommandRequiredElseHelp) + (@setting GlobalVersion) + (@setting VersionlessSubcommands) + (@setting UnifiedHelpMessage) (@subcommand init => (about: "initializes a new repository") - (@arg bundle_size: --bundle-size +takes_value "maximal bundle size") - (@arg chunker: --chunker +takes_value "chunker algorithm") - (@arg chunk_size: --chunk-size +takes_value "average chunk size") - (@arg compression: --compression -c +takes_value "compression to use") + (@arg bundle_size: --bundle-size +takes_value "maximal bundle size in MiB [default: 25]") + (@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]") + (@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]") + (@arg hash: --hash +takes_value "hash method to use [default: blake2]") (@arg REPO: +required "path of the repository") ) (@subcommand backup => @@ -157,14 +159,101 @@ pub fn parse2() -> Arguments { ) (@subcommand algotest => (about: "test a specific algorithm combination") - (@arg bundle_size: --bundle-size +takes_value "maximal bundle size") - (@arg chunker: --chunker +takes_value "chunker algorithm") - (@arg chunk_size: --chunk-size +takes_value "average chunk size") - (@arg compression: --compression -c +takes_value "compression to use") + (@arg bundle_size: --bundle-size +takes_value "maximal bundle size in MiB [default: 25]") + (@arg chunker: --chunker +takes_value "chunker algorithm [default: fastcdc/8]") + (@arg compression: --compression -c +takes_value "compression to use [default: brotli/3]") + (@arg hash: --hash +takes_value "hash method to use [default: blake2]") (@arg FILE: +required "the file to test the algorithms with") ) ).get_matches(); if let Some(args) = args.subcommand_matches("init") { + let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap()); + if backup.is_some() || inode.is_some() { + println!("No backups or subpaths may be given here"); + exit(1); + } + return Arguments::Init { + bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize, + chunker: parse_chunker(args.value_of("chunker")), + compression: parse_compression(args.value_of("compression")), + hash: parse_hash(args.value_of("hash")), + repo_path: repository.to_string(), + } } - unimplemented!() + if let Some(args) = args.subcommand_matches("backup") { + let (repository, backup, inode) = split_repo_path(args.value_of("BACKUP").unwrap()); + if backup.is_none() { + println!("A backup must be specified"); + exit(1); + } + if inode.is_some() { + println!("No subpaths may be given here"); + exit(1); + } + return Arguments::Backup { + repo_path: repository.to_string(), + backup_name: backup.unwrap().to_string(), + full: args.is_present("full"), + src_path: args.value_of("SRC").unwrap().to_string() + } + } + if let Some(args) = args.subcommand_matches("restore") { + let (repository, backup, inode) = split_repo_path(args.value_of("BACKUP").unwrap()); + if backup.is_none() { + println!("A backup must be specified"); + exit(1); + } + return Arguments::Restore { + repo_path: repository.to_string(), + backup_name: backup.unwrap().to_string(), + inode: inode.map(|v| v.to_string()), + dst_path: args.value_of("DST").unwrap().to_string() + } + } + if let Some(args) = args.subcommand_matches("check") { + let (repository, backup, inode) = split_repo_path(args.value_of("PATH").unwrap()); + return Arguments::Check { + repo_path: repository.to_string(), + backup_name: backup.map(|v| v.to_string()), + inode: inode.map(|v| v.to_string()), + full: args.is_present("full") + } + } + if let Some(args) = args.subcommand_matches("list") { + let (repository, backup, inode) = split_repo_path(args.value_of("PATH").unwrap()); + return Arguments::List { + repo_path: repository.to_string(), + backup_name: backup.map(|v| v.to_string()), + inode: inode.map(|v| v.to_string()) + } + } + if let Some(args) = args.subcommand_matches("listbundles") { + let (repository, backup, inode) = split_repo_path(args.value_of("PATH").unwrap()); + if backup.is_some() || inode.is_some() { + println!("No backups or subpaths may be given here"); + exit(1); + } + return Arguments::ListBundles { + repo_path: repository.to_string(), + } + } + if let Some(args) = args.subcommand_matches("info") { + let (repository, backup, inode) = split_repo_path(args.value_of("PATH").unwrap()); + return Arguments::Info { + repo_path: repository.to_string(), + backup_name: backup.map(|v| v.to_string()), + inode: inode.map(|v| v.to_string()) + } + } + if let Some(args) = args.subcommand_matches("algotest") { + return Arguments::AlgoTest { + bundle_size: (parse_num(args.value_of("bundle_size").unwrap_or("25"), "Bundle size") * 1024 * 1024) as usize, + chunker: parse_chunker(args.value_of("chunker")), + compression: parse_compression(args.value_of("compression")), + hash: parse_hash(args.value_of("hash")), + file: args.value_of("FILE").unwrap().to_string(), + } + } + error!("No subcommand given"); + exit(1); } diff --git a/src/cli/logger.rs b/src/cli/logger.rs new file mode 100644 index 0000000..34841d9 --- /dev/null +++ b/src/cli/logger.rs @@ -0,0 +1,23 @@ +use log::{self, LogRecord, LogLevel, LogMetadata, LogLevelFilter}; +pub use log::SetLoggerError; + +struct Logger; + +impl log::Log for Logger { + fn enabled(&self, metadata: &LogMetadata) -> bool { + metadata.level() <= LogLevel::Info + } + + fn log(&self, record: &LogRecord) { + if self.enabled(record.metadata()) { + println!("{} - {}", record.level(), record.args()); + } + } +} + +pub fn init() -> Result<(), SetLoggerError> { + log::set_logger(|max_log_level| { + max_log_level.set(LogLevelFilter::Info); + Box::new(Logger) + }) +} diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 86208c4..5629f45 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1,144 +1,150 @@ mod args; +mod logger; mod algotest; use chrono::prelude::*; +use std::process::exit; -use ::chunker::ChunkerType; -use ::repository::{Repository, Config, Inode}; -use ::util::{ChecksumType, Compression, HashMethod}; +use ::repository::{Repository, Config, Inode, Backup}; +use ::util::ChecksumType; use ::util::cli::*; +use self::args::Arguments; -pub fn run() { - let args = args::parse(); - - if args.cmd_algotest { - let file = args.arg_file.unwrap(); - algotest::run(&file); - return - } - - if args.cmd_init { - let chunker = ChunkerType::from(&args.flag_chunker, args.flag_chunk_size*1024, 0).expect("No such chunk algorithm"); - let compression = if args.flag_compression == "none" { - None - } else { - Some(Compression::from_string(&args.flag_compression).expect("Failed to parse compression")) - }; - Repository::create(&args.arg_repo.unwrap(), Config { - bundle_size: args.flag_bundle_size*1024*1024, - checksum: ChecksumType::Blake2_256, - chunker: chunker, - compression: compression, - hash: HashMethod::Blake2 - }).unwrap(); - return - } - - if args.cmd_stat { - println!("{:?}", Inode::get_from(&args.arg_path.unwrap()).unwrap()); - return - } - - - let mut repo; - if let Some(path) = args.arg_repo { - repo = Repository::open(path).unwrap(); - } else if let Some(ref backup) = args.arg_backup { - let path = backup.splitn(2, "::").nth(0).unwrap(); - repo = Repository::open(path).unwrap(); - } else { - panic!("Repository is needed"); - } - - if args.cmd_check { - repo.check(args.flag_full).unwrap(); - return - } - - if args.cmd_stats { - let info = repo.info(); - println!("Bundles: {}", info.bundle_count); - println!("Total size: {}", to_file_size(info.encoded_data_size)); - println!("Uncompressed size: {}", to_file_size(info.raw_data_size)); - println!("Compression ratio: {:.1}%", info.compression_ratio * 100.0); - println!("Chunk count: {}", info.chunk_count); - println!("Average chunk size: {}", to_file_size(info.avg_chunk_size as u64)); - let index_usage = info.index_entries as f32 / info.index_capacity as f32; - println!("Index: {}, {:.0}% full", to_file_size(info.index_size as u64), index_usage * 100.0); - return - } - - if args.cmd_backups { - for backup in repo.list_backups().unwrap() { - println!("{}", backup); +fn open_repository(path: &str) -> Repository { + match Repository::open(path) { + Ok(repo) => repo, + Err(err) => { + error!("Failed to open repository: {}", err); + exit(2); + } + } +} + +fn get_backup(repo: &Repository, backup_name: &str) -> Backup { + match repo.get_backup(backup_name) { + Ok(backup) => backup, + Err(err) => { + error!("Failed to load backup: {}", err); + exit(3); + } + } +} + +pub fn run() { + if let Err(err) = logger::init() { + println!("Failed to initialize the logger: {}", err); + exit(-1) + } + match args::parse() { + Arguments::Init{repo_path, bundle_size, chunker, compression, hash} => { + Repository::create(repo_path, Config { + bundle_size: bundle_size, + checksum: ChecksumType::Blake2_256, + chunker: chunker, + compression: compression, + hash: hash + }).unwrap(); + }, + Arguments::Backup{repo_path, backup_name, src_path, full} => { + let mut repo = open_repository(&repo_path); + if !full { + warn!("Partial backups are not implemented yet, creating full backup"); + } + let backup = repo.create_full_backup(&src_path).unwrap(); + repo.save_backup(&backup, &backup_name).unwrap(); + }, + Arguments::Restore{repo_path, backup_name, inode, dst_path} => { + let mut repo = open_repository(&repo_path); + let backup = get_backup(&repo, &backup_name); + if let Some(inode) = inode { + let inode = repo.get_backup_inode(&backup, &inode).unwrap(); + repo.restore_inode_tree(inode, &dst_path).unwrap(); + } else { + repo.restore_backup(&backup, &dst_path).unwrap(); + } + }, + Arguments::Check{repo_path, backup_name, inode, full} => { + let mut repo = open_repository(&repo_path); + if let Some(backup_name) = backup_name { + let backup = get_backup(&repo, &backup_name); + if let Some(inode) = inode { + unimplemented!() + } else { + unimplemented!() + } + } else { + repo.check(full).unwrap() + } + }, + Arguments::List{repo_path, backup_name, inode} => { + let mut repo = open_repository(&repo_path); + if let Some(backup_name) = backup_name { + let backup = get_backup(&repo, &backup_name); + let inode = repo.get_backup_inode(&backup, inode.as_ref().map(|v| v as &str).unwrap_or("/")).unwrap(); + println!("{}", format_inode_one_line(&inode)); + if let Some(children) = inode.children { + for chunks in children.values() { + let inode = repo.get_inode(chunks).unwrap(); + println!("- {}", format_inode_one_line(&inode)); + } + } + } else { + for backup in repo.list_backups().unwrap() { + println!("{}", backup); + } + } + } + Arguments::Info{repo_path, backup_name, inode} => { + let repo = open_repository(&repo_path); + if let Some(backup_name) = backup_name { + let backup = get_backup(&repo, &backup_name); + if let Some(inode) = inode { + unimplemented!() + } else { + println!("Date: {}", Local.timestamp(backup.date, 0).to_rfc2822()); + println!("Duration: {}", to_duration(backup.duration)); + println!("Entries: {} files, {} dirs", backup.file_count, backup.dir_count); + println!("Total backup size: {}", to_file_size(backup.total_data_size)); + println!("Modified data size: {}", to_file_size(backup.changed_data_size)); + let dedup_ratio = backup.deduplicated_data_size as f32 / backup.changed_data_size as f32; + println!("Deduplicated size: {}, {:.1}% saved", to_file_size(backup.deduplicated_data_size), (1.0 - dedup_ratio)*100.0); + let compress_ratio = backup.encoded_data_size as f32 / backup.deduplicated_data_size as f32; + println!("Compressed size: {} in {} bundles, {:.1}% saved", to_file_size(backup.encoded_data_size), backup.bundle_count, (1.0 - compress_ratio)*100.0); + println!("Chunk count: {}, avg size: {}", backup.chunk_count, to_file_size(backup.avg_chunk_size as u64)); + } + } else { + let info = repo.info(); + println!("Bundles: {}", info.bundle_count); + println!("Total size: {}", to_file_size(info.encoded_data_size)); + println!("Uncompressed size: {}", to_file_size(info.raw_data_size)); + println!("Compression ratio: {:.1}%", info.compression_ratio * 100.0); + println!("Chunk count: {}", info.chunk_count); + println!("Average chunk size: {}", to_file_size(info.avg_chunk_size as u64)); + let index_usage = info.index_entries as f32 / info.index_capacity as f32; + println!("Index: {}, {:.0}% full", to_file_size(info.index_size as u64), index_usage * 100.0); + } + } + Arguments::ListBundles{repo_path} => { + let repo = open_repository(&repo_path); + for bundle in repo.list_bundles() { + println!("Bundle {}", bundle.id); + println!(" - Mode: {:?}", bundle.mode); + println!(" - Chunks: {}", bundle.chunk_count); + println!(" - Size: {}", to_file_size(bundle.encoded_size as u64)); + println!(" - Data size: {}", to_file_size(bundle.raw_size as u64)); + let ratio = bundle.encoded_size as f32 / bundle.raw_size as f32; + let compression = if let Some(ref c) = bundle.compression { + c.to_string() + } else { + "none".to_string() + }; + println!(" - Compression: {}, ratio: {:.1}%", compression, ratio * 100.0); + println!(); + } + } + Arguments::AlgoTest{bundle_size, chunker, compression, hash, file} => { + algotest::run(&file, bundle_size, chunker, compression, hash); } - return - } - - if args.cmd_bundles { - for bundle in repo.list_bundles() { - println!("Bundle {}", bundle.id); - println!(" - Mode: {:?}", bundle.mode); - println!(" - Chunks: {}", bundle.chunk_count); - println!(" - Size: {}", to_file_size(bundle.encoded_size as u64)); - println!(" - Data size: {}", to_file_size(bundle.raw_size as u64)); - let ratio = bundle.encoded_size as f32 / bundle.raw_size as f32; - let compression = if let Some(ref c) = bundle.compression { - c.to_string() - } else { - "none".to_string() - }; - println!(" - Compression: {}, ratio: {:.1}%", compression, ratio * 100.0); - println!(); - } - return - } - - let backup_name = args.arg_backup.unwrap().splitn(2, "::").nth(1).unwrap().to_string(); - - if args.cmd_backup { - let backup = repo.create_full_backup(&args.arg_path.unwrap()).unwrap(); - repo.save_backup(&backup, &backup_name).unwrap(); - return - } - - let backup = repo.get_backup(&backup_name).unwrap(); - - if args.cmd_info { - println!("Date: {}", Local.timestamp(backup.date, 0).to_rfc2822()); - println!("Duration: {}", to_duration(backup.duration)); - println!("Entries: {} files, {} dirs", backup.file_count, backup.dir_count); - println!("Total backup size: {}", to_file_size(backup.total_data_size)); - println!("Modified data size: {}", to_file_size(backup.changed_data_size)); - let dedup_ratio = backup.deduplicated_data_size as f32 / backup.changed_data_size as f32; - println!("Deduplicated size: {}, {:.1}% saved", to_file_size(backup.deduplicated_data_size), (1.0 - dedup_ratio)*100.0); - let compress_ratio = backup.encoded_data_size as f32 / backup.deduplicated_data_size as f32; - println!("Compressed size: {} in {} bundles, {:.1}% saved", to_file_size(backup.encoded_data_size), backup.bundle_count, (1.0 - compress_ratio)*100.0); - println!("Chunk count: {}, avg size: {}", backup.chunk_count, to_file_size(backup.avg_chunk_size as u64)); - return - } - - if args.cmd_restore { - let dst = args.arg_dst.unwrap(); - if let Some(src) = args.arg_src { - let inode = repo.get_backup_inode(&backup, src).unwrap(); - repo.restore_inode_tree(inode, &dst).unwrap(); - } else { - repo.restore_backup(&backup, &dst).unwrap(); - } - return - } - - if args.cmd_list { - let inode = repo.get_backup_inode(&backup, &args.arg_path.unwrap()).unwrap(); - println!("{}", format_inode_one_line(&inode)); - if let Some(children) = inode.children { - for chunks in children.values() { - let inode = repo.get_inode(chunks).unwrap(); - println!("- {}", format_inode_one_line(&inode)); - } - } - return } } diff --git a/src/main.rs b/src/main.rs index ea6f3bd..31def03 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,10 +7,10 @@ extern crate blake2_rfc as blake2; extern crate murmurhash3; extern crate serde_yaml; #[macro_use] extern crate quick_error; -extern crate docopt; extern crate rustc_serialize; extern crate chrono; #[macro_use] extern crate clap; +#[macro_use] extern crate log; pub mod util; pub mod bundle; @@ -22,7 +22,8 @@ mod cli; // TODO: Seperate remote folder // TODO: Copy backup files to remote folder // TODO: Keep meta bundles also locally -// TODO: Remove backups (based on age like attic) +// TODO: Remove backups/subtrees +// TODO: Prune backups (based on age like attic) // TODO: Backup files tree structure // TODO: Recompress & combine bundles // TODO: Check backup integrity @@ -30,7 +31,6 @@ mod cli; // TODO: list --tree // TODO: Partial backups // TODO: Load and compare remote bundles to bundle map -// TODO: Nice errors / checks for CLI // TODO: Import remote backup // TODO: Continue on errors diff --git a/src/util/cli.rs b/src/util/cli.rs index 48662a9..c7ab241 100644 --- a/src/util/cli.rs +++ b/src/util/cli.rs @@ -1,13 +1,5 @@ use ::repository::{Inode, FileType}; -pub fn split_repo_path(repo_path: &str) -> (&str, Option<&str>, Option<&str>) { - let mut parts = repo_path.splitn(3, "::"); - let repo = parts.next().unwrap(); - let backup = parts.next(); - let inode = parts.next(); - (repo, backup, inode) -} - pub fn to_file_size(size: u64) -> String { let mut size = size as f32; if size >= 512.0 { @@ -33,6 +25,11 @@ pub fn to_file_size(size: u64) -> String { format!("{:.1} TiB", size) } +pub fn to_speed(size: u64, dur: f32) -> String { + let speed = (size as f32 / dur) as u64; + to_file_size(speed) + "/s" +} + pub fn to_duration(dur: f32) -> String { let secs = dur.floor() as u64; let subsecs = dur - dur.floor();