mirror of https://github.com/dswd/zvault
Key commands use files
This commit is contained in:
parent
b43be07ed0
commit
205c193108
106
README.md
106
README.md
|
@ -16,7 +16,7 @@ data size and is compressed as a whole to save space ("solid archive").
|
||||||
|
|
||||||
### Independent backups
|
### Independent backups
|
||||||
All backups share common data in form of chunks but are independent on a higher
|
All backups share common data in form of chunks but are independent on a higher
|
||||||
level. Backups can be delete and chunks that are not used by any backup can be
|
level. Backups can be deleted and chunks that are not used by any backup can be
|
||||||
removed.
|
removed.
|
||||||
|
|
||||||
Other backup solutions use differential backups organized in chains. This makes
|
Other backup solutions use differential backups organized in chains. This makes
|
||||||
|
@ -94,87 +94,33 @@ Recommended: Brotli/2-7
|
||||||
|
|
||||||
## Design
|
## Design
|
||||||
|
|
||||||
- Use rolling checksum to create content-dependent chunks
|
|
||||||
- Use sha3-shake128 to hash chunks
|
|
||||||
- Use mmapped hashtable to find duplicate chunks
|
|
||||||
- Serialize metadata into chunks
|
|
||||||
- Store small file data within metadata
|
|
||||||
- Store directory metadata to avoid calculating checksums of unchanged files (same mtime and size)
|
|
||||||
- Store full directory tree in each backup (use cached metadata and checksums for unchanged entries)
|
|
||||||
- Compress data chunks in blocks of ~10MB to improve compression ("solid archive")
|
|
||||||
- Store metadata in separate data chunks to enable metadata caching on client
|
|
||||||
- Encrypt archive
|
|
||||||
- Sort new files by file extension to improve compression
|
|
||||||
|
|
||||||
## Configurable parameters
|
|
||||||
|
|
||||||
- Rolling chunker algorithm
|
|
||||||
- Minimal chunk size [default: 1 KiB]
|
|
||||||
- Maximal chunk size [default: 64 KiB]
|
|
||||||
- Maximal file size for inlining [default: 128 Bytes]
|
|
||||||
- Block size [default: 10 MiB]
|
|
||||||
- Block compression algorithm [default: Brotli 6]
|
|
||||||
- Encryption algorithm [default: chacha20+poly1305]
|
|
||||||
|
|
||||||
## TODO
|
## TODO
|
||||||
|
|
||||||
- Remove old data
|
### Core functionality
|
||||||
- Locking / Multiple clients
|
- Keep backup files also remotely and sync them
|
||||||
|
- Lock during backup and vacuum
|
||||||
|
- Options for creating backups (same filesystem, exclude/include patterns)
|
||||||
|
- Recompress & combine bundles
|
||||||
|
- Allow to use tar files for backup and restore (--tar, http://alexcrichton.com/tar-rs/tar/index.html)
|
||||||
|
|
||||||
## Modules
|
### CLI functionality
|
||||||
|
- Remove backup subtrees
|
||||||
|
- list --tree
|
||||||
|
- More detailed errors with nicer text
|
||||||
|
|
||||||
- Rolling checksum chunker
|
### Other
|
||||||
- Also creates hashes
|
- Stability
|
||||||
- Mmapped hashtable that stores existing chunks hashes
|
- Tests & benchmarks
|
||||||
- Remote block writing and compression/encryption
|
- Chunker
|
||||||
- Inode data serialization
|
- Index
|
||||||
- Recursive directory scanning, difference calculation, new entry sorting
|
- BundleDB
|
||||||
|
- Bundle map
|
||||||
|
- Config files
|
||||||
### ChunkDB
|
- Backup files
|
||||||
|
- Backup
|
||||||
- Stores data in chunks
|
- Prune
|
||||||
- A chunk is a file
|
- Vacuum
|
||||||
- Per Chunk properties
|
- Documentation
|
||||||
- Format version
|
- All file formats
|
||||||
- Encryption method
|
- Design
|
||||||
- Encryption key
|
|
||||||
- Compression method / level
|
|
||||||
- Chunk ID is the hash of the contents
|
|
||||||
- No locks needed on shared chunk repository !!!
|
|
||||||
- Chunk ID is calculated after compression and encryption
|
|
||||||
- Chunk header
|
|
||||||
- "zvault01"
|
|
||||||
- Chunk size compressed / raw
|
|
||||||
- Content hash method / value
|
|
||||||
- Encryption method / options / key hash
|
|
||||||
- Compression method / options
|
|
||||||
- Chunks are write-once read-often
|
|
||||||
- Chunks are prepared outside the repository
|
|
||||||
- Only one chunk is being prepared at a time
|
|
||||||
- Adding data to the chunk returns starting position in raw data
|
|
||||||
- Operations:
|
|
||||||
- List available chunks
|
|
||||||
- Add data
|
|
||||||
- Flush chunk
|
|
||||||
- Delete chunk
|
|
||||||
- Get data
|
|
||||||
- Check chunk
|
|
||||||
- Chunk path is `checksum.chunk` or `chec/ksum.chunk`
|
|
||||||
- Data is added to current chunk and compressed in memory
|
|
||||||
- Operations on chunk files are just sequencial read/write and delete
|
|
||||||
- Ability to recompress chunks
|
|
||||||
|
|
||||||
|
|
||||||
### Index
|
|
||||||
|
|
||||||
16 Bytes per hash key
|
|
||||||
8 Bytes data per entry (4 bytes bundle id, 4 bytes chunk id)
|
|
||||||
=> 24 Bytes per entry
|
|
||||||
|
|
||||||
Average chunk sizes
|
|
||||||
8 Kib => 3 MiB / 1 GiB
|
|
||||||
16 Kib => 1.5 MiB / 1 GiB
|
|
||||||
24 Kib => 1.0 MiB / 1 GiB
|
|
||||||
32 Kib => 750 Kib / 1 GiB
|
|
||||||
64 Kib => 375 Kib / 1 GiB
|
|
||||||
|
|
|
@ -67,7 +67,8 @@ pub enum Arguments {
|
||||||
},
|
},
|
||||||
Import {
|
Import {
|
||||||
repo_path: String,
|
repo_path: String,
|
||||||
remote_path: String
|
remote_path: String,
|
||||||
|
key_files: Vec<String>
|
||||||
},
|
},
|
||||||
Configure {
|
Configure {
|
||||||
repo_path: String,
|
repo_path: String,
|
||||||
|
@ -78,10 +79,11 @@ pub enum Arguments {
|
||||||
hash: Option<HashMethod>
|
hash: Option<HashMethod>
|
||||||
},
|
},
|
||||||
GenKey {
|
GenKey {
|
||||||
|
file: Option<String>
|
||||||
},
|
},
|
||||||
AddKey {
|
AddKey {
|
||||||
repo_path: String,
|
repo_path: String,
|
||||||
key_pair: Option<(PublicKey, SecretKey)>,
|
file: Option<String>,
|
||||||
set_default: bool
|
set_default: bool
|
||||||
},
|
},
|
||||||
AlgoTest {
|
AlgoTest {
|
||||||
|
@ -159,22 +161,6 @@ fn parse_public_key(val: &str) -> PublicKey {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn parse_secret_key(val: &str) -> SecretKey {
|
|
||||||
let bytes = match parse_hex(val) {
|
|
||||||
Ok(bytes) => bytes,
|
|
||||||
Err(_) => {
|
|
||||||
error!("Invalid key: {}", val);
|
|
||||||
exit(1);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if let Some(key) = SecretKey::from_slice(&bytes) {
|
|
||||||
key
|
|
||||||
} else {
|
|
||||||
error!("Invalid key: {}", val);
|
|
||||||
exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn parse_hash(val: &str) -> HashMethod {
|
fn parse_hash(val: &str) -> HashMethod {
|
||||||
if let Ok(hash) = HashMethod::from(val) {
|
if let Ok(hash) = HashMethod::from(val) {
|
||||||
hash
|
hash
|
||||||
|
@ -251,6 +237,7 @@ pub fn parse() -> Arguments {
|
||||||
)
|
)
|
||||||
(@subcommand import =>
|
(@subcommand import =>
|
||||||
(about: "reconstruct a repository from the remote files")
|
(about: "reconstruct a repository from the remote files")
|
||||||
|
(@arg key: --key -k ... +takes_value "a file with a needed to read the bundles")
|
||||||
(@arg REMOTE: +required "remote repository path")
|
(@arg REMOTE: +required "remote repository path")
|
||||||
(@arg REPO: +required "path of the local repository to create")
|
(@arg REPO: +required "path of the local repository to create")
|
||||||
)
|
)
|
||||||
|
@ -269,14 +256,14 @@ pub fn parse() -> Arguments {
|
||||||
)
|
)
|
||||||
(@subcommand genkey =>
|
(@subcommand genkey =>
|
||||||
(about: "generates a new key pair")
|
(about: "generates a new key pair")
|
||||||
|
(@arg FILE: +takes_value "the destination file for the keypair")
|
||||||
)
|
)
|
||||||
(@subcommand addkey =>
|
(@subcommand addkey =>
|
||||||
(about: "adds a key to the respository")
|
(about: "adds a key to the respository")
|
||||||
(@arg REPO: +required "path of the repository")
|
(@arg REPO: +required "path of the repository")
|
||||||
(@arg generate: --generate "generate a new key")
|
(@arg generate: --generate "generate a new key")
|
||||||
(@arg set_default: --default "set this key as default")
|
(@arg set_default: --default "set this key as default")
|
||||||
(@arg PUBLIC: +takes_value "the public key")
|
(@arg FILE: +takes_value "the file containing the keypair")
|
||||||
(@arg SECRET: +takes_value "the secret key")
|
|
||||||
)
|
)
|
||||||
(@subcommand algotest =>
|
(@subcommand algotest =>
|
||||||
(about: "test a specific algorithm combination")
|
(about: "test a specific algorithm combination")
|
||||||
|
@ -418,7 +405,8 @@ pub fn parse() -> Arguments {
|
||||||
}
|
}
|
||||||
return Arguments::Import {
|
return Arguments::Import {
|
||||||
repo_path: repository.to_string(),
|
repo_path: repository.to_string(),
|
||||||
remote_path: args.value_of("REMOTE").unwrap().to_string()
|
remote_path: args.value_of("REMOTE").unwrap().to_string(),
|
||||||
|
key_files: args.values_of("key").map(|v| v.map(|k| k.to_string()).collect()).unwrap_or_else(|| vec![])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(args) = args.subcommand_matches("configure") {
|
if let Some(args) = args.subcommand_matches("configure") {
|
||||||
|
@ -442,8 +430,10 @@ pub fn parse() -> Arguments {
|
||||||
repo_path: repository.to_string(),
|
repo_path: repository.to_string(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(_args) = args.subcommand_matches("genkey") {
|
if let Some(args) = args.subcommand_matches("genkey") {
|
||||||
return Arguments::GenKey {}
|
return Arguments::GenKey {
|
||||||
|
file: args.value_of("FILE").map(|v| v.to_string())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if let Some(args) = args.subcommand_matches("addkey") {
|
if let Some(args) = args.subcommand_matches("addkey") {
|
||||||
let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap());
|
let (repository, backup, inode) = split_repo_path(args.value_of("REPO").unwrap());
|
||||||
|
@ -452,23 +442,18 @@ pub fn parse() -> Arguments {
|
||||||
exit(1);
|
exit(1);
|
||||||
}
|
}
|
||||||
let generate = args.is_present("generate");
|
let generate = args.is_present("generate");
|
||||||
if !generate && (!args.is_present("PUBLIC") || !args.is_present("SECRET")) {
|
if !generate && !args.is_present("FILE") {
|
||||||
println!("Without --generate, a public and secret key must be given");
|
println!("Without --generate, a file containing the key pair must be given");
|
||||||
exit(1);
|
exit(1);
|
||||||
}
|
}
|
||||||
if generate && (args.is_present("PUBLIC") || args.is_present("SECRET")) {
|
if generate && args.is_present("FILE") {
|
||||||
println!("With --generate, no public or secret key may be given");
|
println!("With --generate, no file may be given");
|
||||||
exit(1);
|
exit(1);
|
||||||
}
|
}
|
||||||
let key_pair = if generate {
|
|
||||||
None
|
|
||||||
} else {
|
|
||||||
Some((parse_public_key(args.value_of("PUBLIC").unwrap()), parse_secret_key(args.value_of("SECRET").unwrap())))
|
|
||||||
};
|
|
||||||
return Arguments::AddKey {
|
return Arguments::AddKey {
|
||||||
repo_path: repository.to_string(),
|
repo_path: repository.to_string(),
|
||||||
set_default: args.is_present("set_default"),
|
set_default: args.is_present("set_default"),
|
||||||
key_pair: key_pair
|
file: args.value_of("FILE").map(|v| v.to_string())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if let Some(args) = args.subcommand_matches("algotest") {
|
if let Some(args) = args.subcommand_matches("algotest") {
|
||||||
|
|
|
@ -284,8 +284,8 @@ pub fn run() {
|
||||||
println!();
|
println!();
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
Arguments::Import{repo_path, remote_path} => {
|
Arguments::Import{repo_path, remote_path, key_files} => {
|
||||||
Repository::import(repo_path, remote_path).unwrap();
|
Repository::import(repo_path, remote_path, key_files).unwrap();
|
||||||
},
|
},
|
||||||
Arguments::Configure{repo_path, bundle_size, chunker, compression, encryption, hash} => {
|
Arguments::Configure{repo_path, bundle_size, chunker, compression, encryption, hash} => {
|
||||||
let mut repo = open_repository(&repo_path);
|
let mut repo = open_repository(&repo_path);
|
||||||
|
@ -309,26 +309,29 @@ pub fn run() {
|
||||||
repo.save_config().unwrap();
|
repo.save_config().unwrap();
|
||||||
print_config(&repo.config);
|
print_config(&repo.config);
|
||||||
},
|
},
|
||||||
Arguments::GenKey{} => {
|
Arguments::GenKey{file} => {
|
||||||
let (public, secret) = gen_keypair();
|
let (public, secret) = gen_keypair();
|
||||||
println!("Public key: {}", to_hex(&public[..]));
|
println!("public: {}", to_hex(&public[..]));
|
||||||
println!("Secret key: {}", to_hex(&secret[..]));
|
println!("secret: {}", to_hex(&secret[..]));
|
||||||
|
if let Some(file) = file {
|
||||||
|
Crypto::save_keypair_to_file(&public, &secret, file).unwrap();
|
||||||
|
}
|
||||||
},
|
},
|
||||||
Arguments::AddKey{repo_path, set_default, key_pair} => {
|
Arguments::AddKey{repo_path, set_default, file} => {
|
||||||
let mut repo = open_repository(&repo_path);
|
let mut repo = open_repository(&repo_path);
|
||||||
let (public, secret) = if let Some(key_pair) = key_pair {
|
let (public, secret) = if let Some(file) = file {
|
||||||
key_pair
|
Crypto::load_keypair_from_file(file).unwrap()
|
||||||
} else {
|
} else {
|
||||||
let (public, secret) = gen_keypair();
|
let (public, secret) = gen_keypair();
|
||||||
println!("Public key: {}", to_hex(&public[..]));
|
println!("public: {}", to_hex(&public[..]));
|
||||||
println!("Secret key: {}", to_hex(&secret[..]));
|
println!("secret: {}", to_hex(&secret[..]));
|
||||||
(public, secret)
|
(public, secret)
|
||||||
};
|
};
|
||||||
|
repo.register_key(public, secret).unwrap();
|
||||||
if set_default {
|
if set_default {
|
||||||
repo.set_encryption(Some(&public));
|
repo.set_encryption(Some(&public));
|
||||||
repo.save_config().unwrap();
|
repo.save_config().unwrap();
|
||||||
}
|
}
|
||||||
repo.register_key(public, secret).unwrap();
|
|
||||||
},
|
},
|
||||||
Arguments::AlgoTest{bundle_size, chunker, compression, encrypt, hash, file} => {
|
Arguments::AlgoTest{bundle_size, chunker, compression, encrypt, hash, file} => {
|
||||||
algotest::run(&file, bundle_size, chunker, compression, encrypt, hash);
|
algotest::run(&file, bundle_size, chunker, compression, encrypt, hash);
|
||||||
|
|
|
@ -25,14 +25,6 @@ mod repository;
|
||||||
mod cli;
|
mod cli;
|
||||||
mod prelude;
|
mod prelude;
|
||||||
|
|
||||||
// TODO: Keep backup files also remotely and sync them
|
|
||||||
// TODO: Lock during backup and vacuum
|
|
||||||
// TODO: Remove backup subtrees
|
|
||||||
// TODO: Recompress & combine bundles
|
|
||||||
// TODO: list --tree
|
|
||||||
// TODO: Give crypto keys for import
|
|
||||||
// TODO: More detailed errors with nicer text
|
|
||||||
// TODO: Allow to use tar files for backup and restore (--tar, http://alexcrichton.com/tar-rs/tar/index.html)
|
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
cli::run();
|
cli::run();
|
||||||
|
|
|
@ -109,10 +109,13 @@ impl Repository {
|
||||||
Ok(repo)
|
Ok(repo)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn import<P: AsRef<Path>, R: AsRef<Path>>(path: P, remote: R) -> Result<Self, RepositoryError> {
|
pub fn import<P: AsRef<Path>, R: AsRef<Path>>(path: P, remote: R, key_files: Vec<String>) -> Result<Self, RepositoryError> {
|
||||||
let path = path.as_ref();
|
let path = path.as_ref();
|
||||||
try!(Repository::create(path, Config::default(), remote));
|
let mut repo = try!(Repository::create(path, Config::default(), remote));
|
||||||
let mut repo = try!(Repository::open(path));
|
for file in key_files {
|
||||||
|
try!(repo.crypto.lock().unwrap().register_keyfile(file));
|
||||||
|
}
|
||||||
|
repo = try!(Repository::open(path));
|
||||||
let mut backups: Vec<Backup> = try!(repo.get_backups()).into_iter().map(|(_, v)| v).collect();
|
let mut backups: Vec<Backup> = try!(repo.get_backups()).into_iter().map(|(_, v)| v).collect();
|
||||||
backups.sort_by_key(|b| b.date);
|
backups.sort_by_key(|b| b.date);
|
||||||
if let Some(backup) = backups.pop() {
|
if let Some(backup) = backups.pop() {
|
||||||
|
|
|
@ -132,11 +132,31 @@ impl Crypto {
|
||||||
self.keys.insert(public, secret);
|
self.keys.insert(public, secret);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn register_keyfile<P: AsRef<Path>>(&mut self, path: P) -> Result<(), EncryptionError> {
|
||||||
|
let (public, secret) = try!(Self::load_keypair_from_file(path));
|
||||||
|
self.register_secret_key(public, secret)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn load_keypair_from_file<P: AsRef<Path>>(path: P) -> Result<(PublicKey, SecretKey), EncryptionError> {
|
||||||
|
let keyfile = try!(KeyfileYaml::load(path));
|
||||||
|
let public = try!(parse_hex(&keyfile.public).map_err(|_| EncryptionError::InvalidKey));
|
||||||
|
let public = try!(PublicKey::from_slice(&public).ok_or(EncryptionError::InvalidKey));
|
||||||
|
let secret = try!(parse_hex(&keyfile.secret).map_err(|_| EncryptionError::InvalidKey));
|
||||||
|
let secret = try!(SecretKey::from_slice(&secret).ok_or(EncryptionError::InvalidKey));
|
||||||
|
Ok((public, secret))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub fn save_keypair_to_file<P: AsRef<Path>>(public: &PublicKey, secret: &SecretKey, path: P) -> Result<(), EncryptionError> {
|
||||||
|
KeyfileYaml { public: to_hex(&public[..]), secret: to_hex(&secret[..]) }.save(path)
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub fn register_secret_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), EncryptionError> {
|
pub fn register_secret_key(&mut self, public: PublicKey, secret: SecretKey) -> Result<(), EncryptionError> {
|
||||||
let keyfile = KeyfileYaml { public: to_hex(&public[..]), secret: to_hex(&secret[..]) };
|
|
||||||
let path = self.path.join(to_hex(&public[..]) + ".yaml");
|
let path = self.path.join(to_hex(&public[..]) + ".yaml");
|
||||||
try!(keyfile.save(path));
|
try!(Self::save_keypair_to_file(&public, &secret, path));
|
||||||
self.keys.insert(public, secret);
|
self.keys.insert(public, secret);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue