diff --git a/README.md b/README.md index 2b10198..29567c7 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,18 @@ Recommended: Brotli/2-7 ## Design +### Semantic Versioning + +zVault sticks to the semantic versioning scheme. In its current pre-1.0 stage +this has the following implications: +- Even now the repository format is considered pretty stable. All future + versions will be able to read the current repository format. Maybe conversions + might be necessary but the backups should always be forward-compatible. +- The CLI might see breaking changes but at least it is guaranteed that calls + that are currently non-destructive will not become destructive in the future. + Running todays commands on a future version will not cause any harm. + + ## TODO ### Packaging diff --git a/deb/.gitignore b/deb/.gitignore index 68a9cd4..45ffad7 100644 --- a/deb/.gitignore +++ b/deb/.gitignore @@ -1,5 +1,6 @@ zvault/debian/zvault zvault/zvault* +zvault/man/* libsquash/src libsquash/debian/libsquash */debian/debhelper* diff --git a/deb/Makefile b/deb/Makefile index b142e7c..45c4334 100644 --- a/deb/Makefile +++ b/deb/Makefile @@ -10,7 +10,7 @@ build: libsquash_*.deb $(PACKAGE)_*.deb libsquash_*.deb: (cd libsquash; make clean; debuild -b -us -uc; cd ..) -$(PACKAGE)_*.deb: $(PACKAGE)/zvault.1.ronn $(PACKAGE)/zvault +$(PACKAGE)_*.deb: $(PACKAGE)/man/* $(PACKAGE)/zvault (cd $(PACKAGE); make clean; debuild -b -us -uc; cd ..) .PHONY: clean @@ -19,10 +19,9 @@ clean: rm -rf $(PACKAGE)_* (cd libsquash; debuild clean; cd ..) rm -rf libsqash_* - rm -f ../target/release/zvault -$(PACKAGE)/zvault.1.ronn: ../docs/manpage.md - cp ../docs/manpage.md $(PACKAGE)/zvault.1.ronn +$(PACKAGE)/man/*: ../docs/man/* + cp ../docs/man/* $(PACKAGE)/man $(PACKAGE)/zvault: ../target/release/zvault cp ../target/release/zvault $(PACKAGE)/zvault diff --git a/deb/zvault/Makefile b/deb/zvault/Makefile index 980050f..0b76d1f 100644 --- a/deb/zvault/Makefile +++ b/deb/zvault/Makefile @@ -1,7 +1,10 @@ -build: zvault.1 +build: man/zvault.1 man/zvault-backup.1 man/zvault-check.1 man/zvault-config.1 \ + man/zvault-import.1 man/zvault-info.1 man/zvault-init.1 man/zvault-list.1 \ + man/zvault-mount.1 man/zvault-prune.1 man/zvault-remove.1 \ + man/zvault-restore.1 man/zvault-vacuum.1 -zvault.1: zvault.1.ronn - ronn -r zvault.1.ronn +%.1: %.1.md + ronn -r $< install: install -d $(DESTDIR)/usr/bin diff --git a/deb/zvault/debian/manpages b/deb/zvault/debian/manpages index daddf2e..fbb5bcd 100644 --- a/deb/zvault/debian/manpages +++ b/deb/zvault/debian/manpages @@ -1 +1,13 @@ -zvault.1 +man/zvault.1 +man/zvault-init.1 +man/zvault-info.1 +man/zvault-list.1 +man/zvault-prune.1 +man/zvault-backup.1 +man/zvault-check.1 +man/zvault-config.1 +man/zvault-import.1 +man/zvault-mount.1 +man/zvault-remove.1 +man/zvault-restore.1 +man/zvault-vacuum.1 diff --git a/docs/man/zvault-backup.1.md b/docs/man/zvault-backup.1.md new file mode 100644 index 0000000..5da5868 --- /dev/null +++ b/docs/man/zvault-backup.1.md @@ -0,0 +1,148 @@ +zvault-backup(1) -- Create a new backup +======================================= + +## SYNOPSIS + +`zvault backup [OPTIONS] ` + + +## DESCRIPTION + +This subcommand creates a new backup `BACKUP` from the data located at `SRC`. + +The backup given by `BACKUP` must be in the format `[repository]::backup_name` +as described in _zvault(1)_. If `repository` is omitted, the default repository +location is used instead. + +The source data given by `SRC` can either be a filesystem path or the path of a +tar archive (with `--tar`). + +If `SRC` is a filesystem path, a reference backup is used (unless `--full` is +set) to compare the data with and only store modified data and take the +unmodified data from the reference backup. Unless a specific reference backup +is chosen via `--ref`, the latest matching backup from the same machine with the +same source path is used as reference. + +When `SRC` is a filesystem path, a set of exclude patterns can be configured. +The patterns can be given directly via `--exclude` or be read from a file via +`--excludes-from`. Unless `--no-default-excludes` is set, a set of default +exclude pattern is read from the file `excludes` in the repository folder. +All exclude pattern given via any of these ways will be combined. + +Unless `--xdev` is set, zVault will not traverse into subfolders that are on a +different filesystem, i.e. mount points will not be included. + +When zVault fails to read a source file, either because of file permissions, +filesystem errors or because the file has an unsupported type, it will print a +warning message and continue with the backup process. + +zVault will store all file attributes including extended attributes except for +creation time and access time as creation time can not be reliably set on +restore and access times change by reading files. + + +## OPTIONS + + * `-e`, `--exclude ...`: + + Exclude this path or file pattern. This option can be given multiple times. + Please see *EXCLUDE PATTERNS* for details on pattern. + + This option conflicts with `--tar`. + + + * `--excludes-from `: + + Read the list of excludes from this file. + Please see *EXCLUDE PATTERNS* for details on pattern. + + This option conflicts with `--tar`. + + + * `--full`: + + Create a full backup without using another backup as a reference. This makes + sure that all files in the source path (except excluded files) are fully + read. The file contents will still be deduplicated by using existing backups + but all files are read fully. + + This option conflicts with `--ref`. + + + * `--no-default-excludes`: + + Do not load the default `excludes` file from the repository folder. + Those excludes are pre-filled with generic pattern to exclude like pseudo + filesystems or cache folders. + + + * `--ref `: + + Base the new backup on this reference backup instead of automatically + selecting a matching one. The backup given as `REF` must be a valid backup + name as listed by zvault-list(1). + + This option conflicts with `--full`. + + + * `--tar`: + + Read the source data from a tar archive instead of the filesystem. When this + flag is set, the `SRC` path must specify a valid tar file. + The contents of the archive are then read instead of the filesystem. Note + that the tar file contents are read as files and directories and not just + as a single file (this would happen when `SRC` is a tar file and `--tar` is + not set). + + This option can be used to import a backup that has been exported using + zvault-restore(1) with the `--tar` flag. + + This flag conflicts with `--exclude` and `--excludes_from`. + + + * `-x`, `--xdev`: + + Allow to cross filesystem boundaries. By default, paths on different + filesystems than the start path will be ignored. If this flag is set, + the scan will traverse also into mounted filesystems. + **Note:** Please use this option with case. Some pseudo filesystems + contain arbitrarily deep nested directories that will send zVault into + an infinite loop. Also it should be avoided to include the remote storage + in the backup. + + + * `-h`, `--help`: + + Prints help information + + +## EXCLUDE PATTERNS + +Exclude patterns can either be absolute patterns or relative patterns. Absolute +patterns start with `/` and must match from the begin of the absolute file path. +Relative patterns start with anything but `/` and can also match any portion of +the absolute path. For example the pattern `/bin` only matches the system +directory `/bin` but not `/usr/bin` or `/usr/local/bin` while the pattern `bin` +matches them too. + +Exclude patterns must match full path components, i.e. the pattern `bin` will +match any path that contains `bin` as as component (e.g. `/bin` and `/usr/bin`) +but not paths that contain `bin` only as substring like `/sbin`. + +Wildcards can be used to match also substrings of path components: + +- `?` matches any single character. +- `*` matches any string not containing `/`, i.e. `*` only matches within a path + component but does not span components. For example `/usr/*bin` matches + `/usr/bin` and `/usr/sbin` but not `/usr/local/bin`. +- `**` matches any string, even spanning across path components. So `/usr/**bin` + will match `/usr/bin`, `/usr/sbin` and also `/usr/local/bin`. + +If a pattern matches on a filesystem entry, that entry and any child entry (in +the case of directories) will be left out of the backup. + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault-check.1.md b/docs/man/zvault-check.1.md new file mode 100644 index 0000000..026483f --- /dev/null +++ b/docs/man/zvault-check.1.md @@ -0,0 +1,56 @@ +zvault-check(1) -- Check the repository, a backup or a backup subtree +===================================================================== + +## SYNOPSIS + +`zvault check [OPTIONS] [PATH]` + + +## DESCRIPTION + +This subcommand checks the repository, a backup or a backup subtree given by +`PATH`. + +The repository, backup, of subtree given by `PATH` must be in the format +`[repository][::backup_name[::subtree]]` as described in _zvault(1)_. +If `PATH` is omitted, the default repository location is used instead. + +The command will perform the following checks in order: +- Bundle integrity +- Full bundle contents (optional) +- Index integrity +- Backup integrity +- Filesystem integrity + +If a backup is specified in `PATH`, only this backup will be check in the backup +integrity check and only the filesystem integrity of this backup will be checked +in the filesystem integrity check. + +If a subtree is specified in `PATH`, no backups will be checked and only the +given subtree will be checked in the filesystem integrity check. + +Unless `--full` is set, the bundles will only be checked without actually +fetching them fully. This means that their contents can only be read from their +header and this information is not verified. If `--full` is set, the full +bundles are fetched and their contents are compared to what their header claims. +This check takes a long time since all bundles need to fetched, decrypted and +decompressed fully to read their contents. + + +## OPTIONS + + * `--full`: + + Also check the contents of the bundles by fetching and decompressing them. + Note: This flag causes the check to be much slower. + + + * `-h`, `--help`: + + Prints help information + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault-config.1.md b/docs/man/zvault-config.1.md new file mode 100644 index 0000000..bd06a40 --- /dev/null +++ b/docs/man/zvault-config.1.md @@ -0,0 +1,78 @@ +zvault-config(1) -- Display or change the configuration +======================================================= + +## SYNOPSIS + +`zvault config [REPO]` + + +## DESCRIPTION + +This subcommand displays or changes the configuration of the repository `REPO`. +If `REPO` is omitted, the default repository location will be used. +The configuration can be changes using the options described below. If no +options are set, the current configuration is displayed. Otherwise, the +configuration is changed as specified and then displayed. + +Beware that the *chunker algorithm*, *chunk size* and *hash method* should not +be changed on existing repositories already containing many backups. If those +values are changed, new backups will not be able to use existing data for +deduplication. This can waste lots of storage space and most likely outweighs +the expected benefits. + +The values for *bundle size*, *compression* and *encryption* only affect new +data and can be changed at any time without any drawback. + + +## OPTIONS + + * `--bundle-size `: + + Set the target bundle size in MiB (default: 25). + Please see _zvault(1)_ for more information on *bundle size*. + + + * `--chunker `: + + Set the chunker algorithm and target chunk size (default: fastcdc/16). + Please see _zvault(1)_ for more information on *chunkers* and possible + values. + + + * `-c`, `--compression `: + + Set the compression method and level (default: brotli/3). + Please see _zvault(1)_ for more information on *compression* and possible + values. + + + * `-e`, `--encryption `: + + Use the given public key for encryption. The key must be a valid public key + encoded as hexadecimal. Please use _zvault-genkey(1)_ to generate keys and + _zvault-addkey(1)_ to add keys to the repository. + + If `none` is given as public key, encryption is deactivated. + + **Warning:** ZVault does not verify that the matching secret key which is + needed for decryption is known. + + Please see _zvault(1)_ for more information on *encryption*. + + + * `--hash `: + + Set the hash method (default: blake2). + Please see _zvault(1)_ for more information on *hash methods* and possible + values. + + + * `-h`, `--help`: + + Prints help information + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault-import.1.md b/docs/man/zvault-import.1.md new file mode 100644 index 0000000..218cd33 --- /dev/null +++ b/docs/man/zvault-import.1.md @@ -0,0 +1,45 @@ +zvault-import(1) -- Reconstruct a repository from the remote storage +==================================================================== + +## SYNOPSIS + +`zvault import [REPO]` + + +## DESCRIPTION + +This subcommand imports a repository from remote storage. First, an empty +repository will be created and then the remote bundles will be imported and +added to the local index. + +The repository will be created at the location `REPO`. If `REPO` is omitted, +the default repository location will be used. It is important that the path +given as `REPO` does not yet exist, so that it can be created. + +The remote storage path `REMOTE` must be an existing remote storage folder +initialized by _zvault-init(1)_. + +Note that this command is not intended to import single backups exported as tar +files via _zvault-restore(1)_ with the `--tar` flag. Those archives can be +imported via _zvault-backup(1)_ also with the `--tar` flag. + + +## OPTIONS + + * `-k`, `--key ...`: + + Add the key pair in the given file to the repository before importing the + remote bundles. This option can be used to add keys that are needed to read + the bundles. If multiple keys are needed, this options can be given multiple + times. + + + * `-h`, `--help`: + + Prints help information + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault-info.1.md b/docs/man/zvault-info.1.md new file mode 100644 index 0000000..435798c --- /dev/null +++ b/docs/man/zvault-info.1.md @@ -0,0 +1,29 @@ +zvault-info(1) -- Display information on a repository, a backup or a subtree +============================================================================ + +## SYNOPSIS + +`zvault info [PATH]` + + +## DESCRIPTION + +This subcommand displays information on the repository, backup or backup subtree +specified by `PATH`. + +The repository, backup or backup subtree given by `PATH` must be in the format +`[repository][::backup_name[::subtree]]` as described in _zvault(1)_. +If `PATH` is omitted, the default repository location is used instead. + + +## OPTIONS + + * `-h`, `--help`: + + Prints help information + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault-init.1.md b/docs/man/zvault-init.1.md new file mode 100644 index 0000000..9c6b88b --- /dev/null +++ b/docs/man/zvault-init.1.md @@ -0,0 +1,75 @@ +zvault-init(1) -- Initialize a new repository +============================================= + +## SYNOPSIS + +`zvault init [OPTIONS] --remote [REPO]` + + +## DESCRIPTION + +This subcommand initializes a new repository at the location `REPO`. If `REPO` +is omitted, the default repository location will be used. It is important that +the path given as `REPO` does not yet exist, so that it can be created. + +The remote storage path `REMOTE` must be an existing empty folder. ZVault +supports mounted remote filesystems, so it is a good idea to use such a folder +to keep the backups on a remote location. + +This subcommand should **NOT** be used to import existing remote backup +locations. Please use _zvault-import(1)_ for this purpose. + +The rest of the options sets configuration options for the new repository. The +configuration can be changed by _zvault-config(1)_ later. + + +## OPTIONS + + * `--bundle-size `: + + Set the target bundle size in MiB (default: 25). + Please see zvault(1) for more information on *bundle size*. + + + * `--chunker `: + + Set the chunker algorithm and target chunk size (default: fastcdc/16). + Please see _zvault(1)_ for more information on *chunkers* and possible + values. + + + * `-c`, `--compression `: + + Set the compression method and level (default: brotli/3). + Please see _zvault(1)_ for more information on *compression* and possible + values. + + + * `-e`, `--encryption`: + + Generate a keypair and enable encryption. + Please see _zvault(1)_ for more information on *encryption*. + + + * `--hash `: + + Set the hash method (default: blake2). + Please see _zvault(1)_ for more information on *hash methods* and possible + values. + + + * `-h`, `--help`: + + Prints help information + + + * `-r`, `--remote `: + + Set the path to the mounted remote storage. There should be an empty folder + at this location. + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault-list.1.md b/docs/man/zvault-list.1.md new file mode 100644 index 0000000..8288539 --- /dev/null +++ b/docs/man/zvault-list.1.md @@ -0,0 +1,40 @@ +zvault-list(1) -- List backups or backup contents +================================================= + +## SYNOPSIS + +`zvault list [PATH]` + + +## DESCRIPTION + +This subcommand lists all backups or backup contents of the repository or backup +specified by `PATH`. + +The repository, backup or backup subtree given by `PATH` must be in the format +`[repository][::backup_name[::subtree]]` as described in _zvault(1)_. +If `PATH` is omitted, the default repository location is used instead. + +If `PATH` specifies a repository, all backups of this repository are listed. + +If `PATH` specifies a backup or a backup subtree, all contents of this folder +are displayed. In the case of a backup, the contents of its root folder are +displayed. + +_zvault-info(1)_ can be used to display more information on single entities. + +Note that _zvault-mount(1)_ can be used to make backups accessible as a +filesystem which is faster than _zvault-list(1)_ for multiple listings. + + +## OPTIONS + + * `-h`, `--help`: + + Prints help information + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault-mount.1.md b/docs/man/zvault-mount.1.md new file mode 100644 index 0000000..b1d6310 --- /dev/null +++ b/docs/man/zvault-mount.1.md @@ -0,0 +1,42 @@ +zvault-mount(1) -- Mount the repository, a backup or a subtree +============================================================== + +## SYNOPSIS + +`zvault mount [PATH] ` + + +## DESCRIPTION + +This subcommand mounts a repository, backup or backup subtree specified by +`PATH` on the location given by `MOUNTPOINT` making it accessible as a +filesystem. + +The repository, backup or backup subtree given by `PATH` must be in the format +`[repository][::backup_name[::subtree]]` as described in _zvault(1)_. +If `PATH` is omitted, the default repository location is used instead. + +If `PATH` specifies a backup or backup subtree, the root of that backup or the +respective subtree is mounted onto the given location. +If `PATH` specifies a whole repository, all backups of that repository will be +accessible in separate folders below the given mount point. + +The provided file system is mounted read-only, i.e. it can only be used to +inspect and restore backups but not to create new backups or modify exiting +ones. + +Please note that since the filesystem is mounted via fuse, restoring huge data +this way is slower than using _zvault-restore(1)_. + + +## OPTIONS + + * `-h`, `--help`: + + Prints help information + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault-prune.1.md b/docs/man/zvault-prune.1.md new file mode 100644 index 0000000..a61bec3 --- /dev/null +++ b/docs/man/zvault-prune.1.md @@ -0,0 +1,95 @@ +zvault-prune(1) -- Remove backups based on age +============================================== + +## SYNOPSIS + +`zvault prune [OPTIONS] [REPO]` + + +## DESCRIPTION + +This subcommand removes backups in the repository `REPO` based on their age. + +If `REPO` is omitted, the default repository location is used instead. + +If a prefix is specified via `--prefix`, only backups which start with this +string are considered for removal. + +The prune logic will preserve a certain number of backups for different time +periods and discard the rest. The available periods are `daily`, `weekly`, +`monthly` and `yearly`. For each of those periods, a number `N` can be specified +that defines that for each of the last `N` of these periods, a single backup +(the newest one in that period) will be kept. + +For example, `--daily 3` will keep backups of the last 3 days, i.e. one backup +for today, yesterday and the day before yesterday (if a backup has been saved +today). If several backups have been saved on a single day, only the newest is +kept. + +The different periods can also be combined to preserve backups using multiple +different time periods. Backups are only removed if they are not preserved by +any of the time periods. + +For example, `--daily 3 --weekly 4 --monthly 3` will keep one backup for each of +the last 3 days, for each of the last 4 weeks and for each of the last 3 months. +As time progresses, the daily backups will be removed as new ones are created so +that only 3 of them are kept but each week one of them will be preserved as a +weekly backup and an old weekly backup will be removed unless that backup +happens to be the last backup of last month... + +If one period is not set, no backups for that time period will be preserved. +This command will refuse to remove all backups if called without options. + +Unless the option `--force` is set, this command only displays the backups that +would be removed but does not remove them. + +This command renders certain chunks unused, but reclaiming their space is a +complicated task as chunks are combined into bundles together with other chunks +which are potentially still used. Please use _zvault-vacuum(1)_ to reclaim +unused space. + +**Important note: Although this command does not actually remove any data, the +data of the deleted backups becomes inaccessible and can not be restored.** + + +## OPTIONS + + * `-p`, `--prefix `: + + Only consider backups starting with this prefix. + + + * `-d`, `--daily `: + + Keep the newest backup for each of the last `NUM` days. + + + * `-w`, `--weekly `: + + Keep the newest backup for each of the last `NUM` weeks. + + + * `-m`, `--monthly `: + + Keep the newest backup for each of the last `NUM` months. + + + * `-y`, `--yearly `: + + Keep the newest backup for each of the last `NUM` years. + + + * `-f`, `--force`: + + Actually remove backups instead of displaying what would be removed. + + + * `-h`, `--help`: + + Prints help information + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault-remove.1.md b/docs/man/zvault-remove.1.md new file mode 100644 index 0000000..93a7c87 --- /dev/null +++ b/docs/man/zvault-remove.1.md @@ -0,0 +1,43 @@ +zvault-remove(1) -- Remove a backup or a subtree +================================================ + +## SYNOPSIS + +`zvault remove ` + + +## DESCRIPTION + +This subcommand removes a backup or a backup subtree `BACKUP`. + +The backup or backup subtree given by `BACKUP` must be in the format +`[repository]::backup_name[::subtree]` as described in _zvault(1)_. +If `repository` is omitted, the default repository location is used instead. + +If a backup is referenced, this backup will be deleted. If a subtree is given, +the backup is instead rewritten to not include that subtree anymore. + +Note: When removing backup subtrees, the meta information of that backup is left +unchanged and still contains the data (e.g. duration and size) of the original +backup run. + +This command renders certain chunks unused, but reclaiming their space is a +complicated task as chunks are combined into bundles together with other chunks +which are potentially still used. Please use _zvault-vacuum(1)_ to reclaim +unused space. + +**Important note: Although this command does not actually remove any data, the +data of the deleted backups becomes inaccessible and can not be restored.** + + +## OPTIONS + + * `-h`, `--help`: + + Prints help information + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault-restore.1.md b/docs/man/zvault-restore.1.md new file mode 100644 index 0000000..37fb1f9 --- /dev/null +++ b/docs/man/zvault-restore.1.md @@ -0,0 +1,43 @@ +zvault-restore(1) -- Restore a backup or subtree +================================================ + +## SYNOPSIS + +`zvault restore [OPTIONS] ` + + +## DESCRIPTION + +This subcommand restores a backup or a backup subtree `BACKUP` into the folder +`DST`. + +The backup or backup subtree given by `BACKUP` must be in the format +`[repository]::backup_name[::subtree]` as described in _zvault(1)_. +If `repository` is omitted, the default repository location is used instead. + +If `--tar` is set, the data is written to a tar file named `DST`. In this case +`DST` must not exist. + +If `--tar` is not set, the data will be written into the existing folder `DST`. + + +## OPTIONS + + * `--tar`: + + Write the backup to a tar archive named `DST` instead of creating files and + folders at this location. + + This option can be used to export a backup that can be imported again using + zvault-backup(1) with the `--tar` flag. + + + * `-h`, `--help`: + + Prints help information + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault-vacuum.1.md b/docs/man/zvault-vacuum.1.md new file mode 100644 index 0000000..be32468 --- /dev/null +++ b/docs/man/zvault-vacuum.1.md @@ -0,0 +1,66 @@ +zvault-vacuum(1) -- Reclaim space by rewriting bundles +====================================================== + +## SYNOPSIS + +`zvault vacuum [OPTIONS] [REPO]` + + +## DESCRIPTION + +This subcommand reclaims space by rewriting bundles in the repository `REPO`. +If `REPO` is omitted, the default repository location is used instead. + +This command rewrites bundles to remove unused chunks of backups that have been +removed by _zvault-remove(1)_ or _zvault-prune(1)_. +To accomplish this, it will scan all backups and track all used chunks to +identify chunks that are not used by any backup. Those chunks are then grouped +by bundle and bundles with many unused chunks will be rewritten with those +chunks left out. + +The option `--ratio` configures the minimal ratio of used chunks in a bundle +required to remove it. Since all chunks that are still used must be read from +the bundle and written to a new one and only the storage space of the unused +chunks can be reclaimed, rewriting a bundle is more economical the lower the +ratio. At a ratio of 0% will only rewrite bundles with no used chunks at all +(in this case the bundle is just removed). At a ratio of 100%, all bundles will +be rewritten regardless of unused chunks. + +Please note that the bundles will be rewritten with the current settings for +encryption and compression, disregarding the original settings during bundle +creation. + +Unless `--force` is set, this command will only simulate the process but not +actually rewrite any bundle. + +As this is a critical operation, zVault takes many precaution measures to avoid +any damaging the integrity to the repository or other backups. The whole process +is performed with an exclusive lock on the repository which prevents any backup +runs. Also the chunk index is double checked before removing bundles to make +sure that they are unused. Nevertheless, this is a critical operation which +should be avoided when the storage space permits it. + + + +## OPTIONS + + * `-r`, `--ratio `: + + Do not rewrite bundles with more than `NUM`% of used chunks. + The ratio must be given in whole percentage, e.g. 50 mean 50%. + + + * `-f`, `--force`: + + Actually run the vacuum instead of simulating it. + + + * `-h`, `--help`: + + Prints help information + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/man/zvault.1.md b/docs/man/zvault.1.md new file mode 100644 index 0000000..cf40101 --- /dev/null +++ b/docs/man/zvault.1.md @@ -0,0 +1,324 @@ +zvault(1) -- Deduplicating backup solution +========================================== + +## SYNOPSIS + +`zvault ` + + + +## DESCRIPTION + +ZVault is a deduplicating backup solution. It creates backups from data read +from the filesystem or a tar file, deduplicates it, optionally compresses and +encrypts the data and stores the data in bundles at a potentially remote storage +location. + + + +## OPTIONS + + * `-h`, `--help`: + + Prints help information + + + * `-V`, `--version`: + + Prints version information + + + +## SUBCOMMANDS + + +### Main Commands + + * `init` Initialize a new repository, _zvault-init(1)_ + * `import` Reconstruct a repository from the remote storage, _zvault-import(1)_ + * `backup` Create a new backup, _zvault-backup(1)_ + * `restore` Restore a backup or subtree, _zvault-restore(1)_ + * `check` Check the repository, a backup or a backup subtree, _zvault-check(1)_ + * `list` List backups or backup contents, _zvault-list(1)_ + * `info` Display information on a repository, a backup or a subtree, _zvault-info(1)_ + * `mount` Mount the repository, a backup or a subtree, _zvault-mount(1)_ + * `remove` Remove a backup or a subtree, _zvault-remove(1)_ + * `prune` Remove backups based on age, _zvault-prune(1)_ + * `vacuum` Reclaim space by rewriting bundles, _zvault-vacuum(1)_ + + +### Other Commands + + * `addkey` Add a key pair to the repository, _zvault-addkey(1)_ + * `algotest` Test a specific algorithm combination, _zvault-algotest(1)_ + * `analyze` Analyze the used and reclaimable space of bundles, _zvault-analyze(1)_ + * `bundleinfo` Display information on a bundle, _zvault-bundleinfo(1)_ + * `bundlelist` List bundles in a repository, _zvault-bundlelist(1)_ + * `config` Display or change the configuration, _zvault-config(1)_ + * `diff` Display differences between two backup versions, _zvault-diff(1)_ + * `genkey` Generate a new key pair, _zvault-genkey(1)_ + * `versions` Find different versions of a file in all backups, _zvault-versions(1)_ + + +## USAGE + +### Path syntax + +Most subcommands work with a repository that has to be specified as a parameter. +If this repository is not specified, the default repository in `~/.zvault` will +be used instead. + +Some subcommands need to reference a specific backup in the repository. This is +done via the syntax `repository::backup_name` where `repository` is the path to +the repository and `backup_name` is the name of the backup in that repository +as listed by `zvault list`. In this case, `repository` can be omitted, +shortening the syntax to `::backup_name`. In this case, the default repository +is used. + +Some subcommands need to reference a specific subtree inside a backup. This is +done via the syntax `repository::backup_name::subtree` where +`repository::backup_name` specifies a backup as described before and `subtree` +is the path to the subtree of the backup. Again, `repository` can be omitted, +yielding the shortened syntax `::backup_name::subtree`. + +Some subcommands can take either a repository, a backup or a backup subtree. In +this case it is important to note that if a path component is empty, it is +regarded as not set at all. + +Examples: +- `~/.zvault` references the repository in `~/.zvault` and is identical with + `::` (as well as not setting the path at all). +- `::backup1` references the backup `backup1` in the default repository +- `::backup1::/` references the root folder of the backup `backup1` in the + default repository + + +## CONFIGURATION OPTIONS +ZVault offers some configuration options that affect the backup speed, storage +space, security and RAM usage. Users should select them carefully for their +scenario. The performance of different combinations can be compared using +_zvault-algotest(1)_. + + +### Bundle size +The target bundle size affects how big bundles written to the remote storage +will become. The configured size is not a hard maximum, as headers and some +effects of compression can cause bundles to become slightly larger than this +size. Also since bundles will be closed at the end of a backup run, some bundles +can also be smaller than this size. However most bundles will end up with +approximately the specified size. + +The configured value for the bundle size has some practical consequences. +Since the whole bundle is compressed as a whole (a so-called *solid archive*), +the compression ratio is impacted negatively if bundles are small. Also the +remote storage could become inefficient if too many small bundle files are +stored. On the other side, since the whole bundle has to be fetched and +decompressed to read a single chunk from that bundle, bigger bundles increase +the overhead of reading the data. + +The recommended bundle size is 25 MiB, but values between 5 MiB and 100 MiB +should also be feasable. + + +### Chunker +The chunker is the component that splits the input data into so-called *chunks*. +The main goal of the chunker is to produce as many identical chunks as possible +when only small parts of the data changed since the last backup. The identical +chunks do not have to be stored again, thus the input data is deduplicated. +To achieve this goal, the chunker splits the input data based on the data +itself, so that identical parts can be detected even when their position +changed. + +ZVault offers different chunker algorithms with different properties to choose +from: +- The **rabin** chunker is a very common algorithm with a good quality but a + mediocre speed (about 350 MB/s). +- The **ae** chunker is a novel approach that can reach very high speeds + (over 750 MB/s) at a cost of deduplication rate. +- The **fastcdc** algorithm reaches a similar deduplication rate as the rabin + chunker but is faster (about 550 MB/s). +The recommended chunker is **fastcdc**. + +Besides the chunker algorithm, an important setting is the target chunk size, +i.e. the planned average chunk size. Since the chunker splits the data on +data-dependent criteria, it will not achieve the configured size exactly. +The chunk size has a number of practical implications. Since deduplication works +by identifying identical chunks, smaller chunk sizes will be able to find more +identical chunks and thereby reduce the overall storage space. + +On the other side, the index needs to store 24 bytes per chunk, so many small +chunks will take more space than few big chunks. Since the index of all chunks +in the repository needs to be loaded into memory during the backup, huge +repositories can get a problem with memory usage. Since the index could be only +40% filled and the chunker could yield smaller chunks than configured, 100 bytes +per chunk should be a safe value to calculate with. + +The configured value for chunk size needs to be a power of 2. Here is a +selection of chunk sizes and their estimated RAM usage: +- Chunk size 4 KiB => ~40 GiB data stored in 1 GiB RAM +- Chunk size 8 KiB => ~80 GiB data stored in 1 GiB RAM +- Chunk size 16 KiB => ~160 GiB data stored in 1 GiB RAM +- Chunk size 32 KiB => ~325 GiB data stored in 1 GiB RAM +- Chunk size 64 KiB => ~650 GiB data stored in 1 GiB RAM +- Chunk size 128 KiB => ~1.3 TiB data stored in 1 GiB RAM +- Chunk size 256 KiB => ~2.5 TiB data stored in 1 GiB RAM +- Chunk size 512 KiB => ~5 TiB data stored in 1 GiB RAM +- Chunk size 1024 KiB => ~10 TiB data stored in 1 GiB RAM +The recommended chunk size for normal computers is 16 KiB. Servers with lots of +data might want to use 128 KiB or 1024 KiB instead. + +The chunker algortihm and chunk size are configured together in the format +`algorithm/size` where algorithm is one of `rabin`, `ae` and `fastcdc` and size +is the size in KiB e.g. `16`. So the recommended configuration is `fastcdc/16`. + +Please not that since the chunker algorithm and chunk size affect the chunks +created from the input data, any change to those values will make existing +chunks inaccessible for deduplication purposes. The old data is still readable +but new backups will have to store all data again. + + +### Compression +ZVault offers different compression algorithms that can be used to compress the +stored data after deduplication. The compression ratio that can be achieved +mostly depends on the input data (test data can be compressed well and media +data like music and videos are already compressed and can not be compressed +significantly). + +Using a compression algorithm is a trade-off between backup speed and storage +space. Higher compression takes longer and saves more space while low +compression is faster but needs more space. + +ZVault supports the following compression methods: +- **deflate** (also called *zlib* and *gzip*) is the most common algorithm today + and guarantees that backups can be decompressed in future centuries. Its + speed and compression ratio are acceptable but other algorithms are better. + This is a rather conservative choice. This algorithm supports the levels 1 + (fastest) to 9 (best). +- **lz4** is a very fast compression algorithm that does not impact backup speed + very much. It does not compress as good as other algorithms but is much faster + than all other algorithms. This algorithm supports levels 1 (fastest) to 14 + (best) but levels above 7 are significantly slower and not recommended. +- **brotli** is a modern compression algorithm that is both faster and + compresses better than deflate. It offers a big range of compression ratios + and speeds via its levels. This algorithm supports levels 1 (fastest) to 10 + (best). +- **lzma** is about the algorithm with the best compression today. That comes + at the cost of speed. LZMA is rather slow at all levels so it can slow down + the backup speed significantly. This algorithm supports levels 1 (fastest) to + 9 (best). + +The recommended combinations are: +- Focusing speed: lz4 with level between 1 and 7 +- Balanced focus: brotli with levels between 1 and 10 +- Focusing storage space: lzma with levels between 1 and 9 + +The compression algorithm and level are configured together via the syntax +`algorithm/level` where `algorithm` is either `deflate`, `lz4`, `brotli` or +`lzma` and `level` is a number. +The default compression setting is **brotli/3**. + +Since the compression ratio and speed hugely depend on the input data, +_zvault-algotest(1)_ should be used to compare algorithms with actual input +data. + + + +### Encryption +When enabled, zVault uses modern encryption provided by *libsodium* to encrypt +the bundles that are stored remotely. This makes it impossible for anyone with +access to the remote bundles to read their contents or to modify them. + +zVault uses asymmetric encryption, which means that encryption uses a so called +*public key* and decryption uses a different *secret key*. This makes it +possible to setup a backup configuration where the machine can only create +backups but not read them. Since lots of subcommands need to read the backups, +this setup is not recommended in general. + +The key pairs used by zVault can be created by _zvault-genkey(1)_ and added to a +repository via _zvault-addkey(1)_ or upon creation via the `--encryption` flag +in _zvault-init(1)_. + +**Important: The key pair is needed to read and restore any encrypted backup. +Loosing the secret key means that all data in the backups is lost forever. +There is no backdoor, even the developers of zVault can not recover a lost key +pair. So it is important to store the key pair in a safe location. The key pair +is small enough to be printed on paper for example.** + + +### Hash method +ZVault uses hash fingerprints to identify chunks. It is critically important +that no two chunks have the same hash value (a so-called hash collision) as this +would cause one chunk to overwrite the other chunk. For this purpose zVault uses +128 bit hashes, that have a collision probability of less than 1.5e-15 even for +1 trillion stored chunks (about 15.000 TiB stored data in 16 KiB chunks). + +ZVault offers two different hash algorithms: **blake2** and **murmur3**. + +Murmur3 is blazingly fast but is not cryptographically secure. That means that +while random hash collisions are negligible, an attacker with access to files +could manipulate a file so that it will cause a hash collision and affects other +data in the repository. **This hash should only be used when the security +implications of this are fully understood.** + +Blake2 is slower than murmur3 but also pretty fast and this hash algorithm is +cryptographically secure, i.e. even an attacker can not cause hash collisions. + +The recommended hash algorithm is **blake2**. + + + +## EXAMPLES + +This command will initialize a repository in the default location with +encryption enabled: + + $> zvault init -e --remote /mnt/remote/backups + +Before using this repository, the key pair located at `~/.zvault/keys` should be +backed up in a safe location (e.g. printed to paper). + +This command will create a backup of the whole system tagged by date: + + $> zvault backup / ::system/$(date +%F) + +If the home folders are mounted on /home, the following command can be used to +backup them separatly (zVault will not backup mounted folders by default): + + $> zvault backup /home ::homes/$(date +%F) + +The backups can be listed by this command: + + $> zvault list + +and inspected by this command (the date needs to be adapted): + + $> zvault info ::homes/2017-04-06 + +To restore some files from a backup, the following command can be used: + + $> zvault restore ::homes/2017-04-06::bob/file.dat /tmp + +Alternatively the repository can be mounted with this command: + + $> zvault mount ::homes/2017-04-06 /mnt/tmp + +A single backup can be removed with this command: + + $> zvault remove ::homes/2017-04-06 + +Multiple backups can be removed based on their date with the following command +(add `-f` to actually remove backups): + + $> zvault prune --prefix system --daily 7 --weekly 5 --monthly 12 + +To reclaim storage space after removing some backups vacuum needs to be run +(add `-f` to actually remove bundles): + + $> zvault vacuum + + + +## COPYRIGHT + +Copyright (C) 2017 Dennis Schwerdel +This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/docs/manpage.md b/docs/manpage.md deleted file mode 100644 index eb7dd04..0000000 --- a/docs/manpage.md +++ /dev/null @@ -1,37 +0,0 @@ -zvault(1) -- Deduplicating backup solution -========================================== - -## SYNOPSIS - -`zvault ...` - - -## DESCRIPTION - - -## OPTIONS - - * `-h`, `--help`: - - Display the help. - - -## USAGE - - -## EXIT STATUS - - -## EXAMPLES - - -## FILES - - -## SEE ALSO - - -## COPYRIGHT - -Copyright (C) 2017 Dennis Schwerdel -This software is licensed under GPL-3 or newer (see LICENSE.md) diff --git a/src/cli/args.rs b/src/cli/args.rs index 0916105..d947c1c 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -235,92 +235,92 @@ fn parse_bundle_id(val: &str) -> Result { #[allow(unknown_lints,cyclomatic_complexity)] pub fn parse() -> Result { let args = App::new("zvault").version(crate_version!()).author(crate_authors!(",\n")).about(crate_description!()) - .settings(&[AppSettings::GlobalVersion, AppSettings::VersionlessSubcommands, AppSettings::SubcommandRequiredElseHelp]) + .settings(&[AppSettings::AllowMissingPositional, AppSettings::VersionlessSubcommands, AppSettings::SubcommandRequiredElseHelp]) .global_settings(&[AppSettings::UnifiedHelpMessage, AppSettings::ColoredHelp, AppSettings::ColorAuto]) - .subcommand(SubCommand::with_name("init").about("initializes a new repository") - .arg(Arg::from_usage("bundle_size --bundle-size [SIZE] 'maximal bundle size in MiB [default: 25]'")) - .arg(Arg::from_usage("--chunker [CHUNKER] 'chunker algorithm [default: fastcdc/8]'")) - .arg(Arg::from_usage("-c --compression [COMPRESSION] 'compression to use [default: brotli/3]'")) - .arg(Arg::from_usage("-e --encryption [ENCRYPTION] 'generate a keypair and enable encryption'")) - .arg(Arg::from_usage("--hash [HASH] 'hash method to use [default: blake2]'")) - .arg(Arg::from_usage("-r --remote 'path to the mounted remote storage'")) - .arg(Arg::from_usage("[REPO] 'path of the repository'"))) - .subcommand(SubCommand::with_name("backup").about("creates a new backup") - .arg(Arg::from_usage("--full 'create a full backup'")) - .arg(Arg::from_usage("reference --ref [REF] 'the reference backup to use for partial backup'")) - .arg(Arg::from_usage("cross_device -x --xdev 'allow to cross filesystem boundaries'")) - .arg(Arg::from_usage("-e --exclude [PATTERN]... 'exclude this path or file'")) - .arg(Arg::from_usage("excludes_from --excludes-from [FILE] 'read the list of exludes from this file'")) - .arg(Arg::from_usage("no_default_excludes --no-default-excludes 'do not load the default excludes file'")) - .arg(Arg::from_usage("--tar 'the source is a tar file'")) - .arg(Arg::from_usage(" 'source path to backup'")) - .arg(Arg::from_usage(" 'repository::backup path'"))) - .subcommand(SubCommand::with_name("restore").about("restores a backup (or subpath)") - .arg(Arg::from_usage("--tar 'restore in form of a tar file'")) - .arg(Arg::from_usage(" 'repository::backup[::subpath] path'")) - .arg(Arg::from_usage(" 'destination path for backup'"))) - .subcommand(SubCommand::with_name("remove").aliases(&["rm", "delete", "del"]).about("removes a backup or a subpath") - .arg(Arg::from_usage(" 'repository::backup[::subpath] path'"))) - .subcommand(SubCommand::with_name("prune").about("removes backups based on age") - .arg(Arg::from_usage("--prefix [PREFIX] 'only consider backups starting with this prefix'")) - .arg(Arg::from_usage("--daily [NUM] 'keep this number of daily backups'")) - .arg(Arg::from_usage("--weekly [NUM] 'keep this number of weekly backups'")) - .arg(Arg::from_usage("--monthly [NUM] 'keep this number of monthly backups'")) - .arg(Arg::from_usage("--yearly [NUM] 'keep this number of yearly backups'")) - .arg(Arg::from_usage("-f --force 'actually run the prunce instead of simulating it'")) - .arg(Arg::from_usage("[REPO] 'path of the repository'"))) - .subcommand(SubCommand::with_name("vacuum").about("saves space by combining and recompressing bundles") - .arg(Arg::from_usage("-r --ratio [NUM] 'ratio in % of unused space in a bundle to rewrite that bundle'")) - .arg(Arg::from_usage("-f --force 'actually run the vacuum instead of simulating it'")) - .arg(Arg::from_usage("[REPO] 'path of the repository'"))) - .subcommand(SubCommand::with_name("check").about("checks the repository, a backup or a backup subpath") - .arg(Arg::from_usage("--full 'also check file contents'")) - .arg(Arg::from_usage("[PATH] 'repository[::backup] path'"))) - .subcommand(SubCommand::with_name("list").alias("ls").about("lists backups or backup contents") - .arg(Arg::from_usage("[PATH] 'repository[::backup[::subpath]] path'"))) - .subcommand(SubCommand::with_name("mount").about("mount a backup for inspection") - .arg(Arg::from_usage("[PATH] 'repository[::backup[::subpath]] path'")) - .arg(Arg::from_usage(" 'where to mount to backup'"))) - .subcommand(SubCommand::with_name("bundlelist").about("lists bundles in a repository") - .arg(Arg::from_usage("[REPO] 'path of the repository'"))) - .subcommand(SubCommand::with_name("bundleinfo").about("lists bundles in a repository") - .arg(Arg::from_usage("[REPO] 'path of the repository'")) - .arg(Arg::from_usage(" 'the bundle id'"))) - .subcommand(SubCommand::with_name("import").about("reconstruct a repository from the remote files") - .arg(Arg::from_usage("-k --key [FILE]... 'a file with a needed to read the bundles'")) - .arg(Arg::from_usage(" 'remote repository path'")) - .arg(Arg::from_usage("[REPO] 'path of the local repository to create'"))) - .subcommand(SubCommand::with_name("info").about("displays information on a repository, a backup or a path in a backup") - .arg(Arg::from_usage("[PATH] 'repository[::backup[::subpath]] path'"))) - .subcommand(SubCommand::with_name("analyze").about("analyze the used and reclaimable space of bundles") - .arg(Arg::from_usage("[REPO] 'repository path'"))) - .subcommand(SubCommand::with_name("versions").about("display different versions of a file in all backups") - .arg(Arg::from_usage("[REPO] 'repository path'")) - .arg(Arg::from_usage(" 'the file path'"))) - .subcommand(SubCommand::with_name("diff").about("display difference between two backup versions") - .arg(Arg::from_usage(" 'old repository::backup[::subpath] path'")) - .arg(Arg::from_usage(" 'new repository::backup[::subpath] path'"))) - .subcommand(SubCommand::with_name("config").about("changes the configuration") - .arg(Arg::from_usage("[REPO] 'path of the repository'")) - .arg(Arg::from_usage("bundle_size --bundle-size [SIZE] 'maximal bundle size in MiB [default: 25]'")) - .arg(Arg::from_usage("--chunker [CHUNKER] 'chunker algorithm [default: fastcdc/16]'")) - .arg(Arg::from_usage("-c --compression [COMPRESSION] 'compression to use [default: brotli/3]'")) - .arg(Arg::from_usage("-e --encryption [ENCRYPTION] 'the public key to use for encryption'")) - .arg(Arg::from_usage("--hash [HASH] 'hash method to use [default: blake2]'"))) - .subcommand(SubCommand::with_name("genkey").about("generates a new key pair") - .arg(Arg::from_usage("[FILE] 'the destination file for the keypair'"))) - .subcommand(SubCommand::with_name("addkey").about("adds a key to the respository") - .arg(Arg::from_usage("[REPO] 'path of the repository'")) - .arg(Arg::from_usage("-g --generate 'generate a new key'").conflicts_with("FILE")) - .arg(Arg::from_usage("set_default --default -d 'set this key as default'")) - .arg(Arg::from_usage("[FILE] 'the file containing the keypair'").conflicts_with("generate"))) - .subcommand(SubCommand::with_name("algotest").about("test a specific algorithm combination") - .arg(Arg::from_usage("bundle_size --bundle-size [SIZE] 'maximal bundle size in MiB [default: 25]'")) - .arg(Arg::from_usage("--chunker [CHUNKER] 'chunker algorithm [default: fastcdc/16]'")) - .arg(Arg::from_usage("-c --compression [COMPRESSION] 'compression to use [default: brotli/3]'")) - .arg(Arg::from_usage("-e --encrypt 'enable encryption'")) - .arg(Arg::from_usage("--hash [HASH] 'hash method to use [default: blake2]'")) - .arg(Arg::from_usage(" 'the file to test the algorithms with'"))).get_matches(); + .subcommand(SubCommand::with_name("init").about("Initialize a new repository") + .arg(Arg::from_usage("bundle_size --bundle-size [SIZE] 'Set the target bundle size in MiB (default: 25)'")) + .arg(Arg::from_usage("--chunker [CHUNKER] 'Set the chunker algorithm and target chunk size (default: fastcdc/16)'")) + .arg(Arg::from_usage("-c --compression [COMPRESSION] 'Set the compression method and level (default: brotli/3)'")) + .arg(Arg::from_usage("-e --encryption 'Generate a keypair and enable encryption'")) + .arg(Arg::from_usage("--hash [HASH] 'Set the hash method (default: blake2)'")) + .arg(Arg::from_usage("-r --remote 'Set the path to the mounted remote storage'")) + .arg(Arg::from_usage("[REPO] 'The path for the new repository'"))) + .subcommand(SubCommand::with_name("backup").about("Create a new backup") + .arg(Arg::from_usage("--full 'Create a full backup without using a reference'")) + .arg(Arg::from_usage("reference --ref [REF] 'Base the new backup on this reference'").conflicts_with("full")) + .arg(Arg::from_usage("cross_device -x --xdev 'Allow to cross filesystem boundaries'")) + .arg(Arg::from_usage("-e --exclude [PATTERN]... 'Exclude this path or file pattern'")) + .arg(Arg::from_usage("excludes_from --excludes-from [FILE] 'Read the list of excludes from this file'")) + .arg(Arg::from_usage("no_default_excludes --no-default-excludes 'Do not load the default excludes file'")) + .arg(Arg::from_usage("--tar 'Read the source data from a tar file'").conflicts_with_all(&["reference", "exclude", "excludes_from"])) + .arg(Arg::from_usage(" 'Source path to backup'")) + .arg(Arg::from_usage(" 'Backup path, [repository]::backup'"))) + .subcommand(SubCommand::with_name("restore").about("Restore a backup or subtree") + .arg(Arg::from_usage("--tar 'Restore in form of a tar file'")) + .arg(Arg::from_usage(" 'The backup/subtree path, [repository]::backup[::subtree]'")) + .arg(Arg::from_usage(" 'Destination path for backup'"))) + .subcommand(SubCommand::with_name("remove").aliases(&["rm", "delete", "del"]).about("Remove a backup or a subtree") + .arg(Arg::from_usage(" 'The backup/subtree path, [repository]::backup[::subtree]'"))) + .subcommand(SubCommand::with_name("prune").about("Remove backups based on age") + .arg(Arg::from_usage("-p --prefix [PREFIX] 'Only consider backups starting with this prefix'")) + .arg(Arg::from_usage("-d --daily [NUM] 'Keep this number of daily backups'")) + .arg(Arg::from_usage("-w --weekly [NUM] 'Keep this number of weekly backups'")) + .arg(Arg::from_usage("-m --monthly [NUM] 'Keep this number of monthly backups'")) + .arg(Arg::from_usage("-y --yearly [NUM] 'Keep this number of yearly backups'")) + .arg(Arg::from_usage("-f --force 'Actually run the prune instead of simulating it'")) + .arg(Arg::from_usage("[REPO] 'Path of the repository'"))) + .subcommand(SubCommand::with_name("vacuum").about("Reclaim space by rewriting bundles") + .arg(Arg::from_usage("-r --ratio [NUM] 'Ratio in % of unused space in a bundle to rewrite that bundle'")) + .arg(Arg::from_usage("-f --force 'Actually run the vacuum instead of simulating it'")) + .arg(Arg::from_usage("[REPO] 'Path of the repository'"))) + .subcommand(SubCommand::with_name("check").about("Check the repository, a backup or a backup subtree") + .arg(Arg::from_usage("--full 'Also check file contents (slow)'")) + .arg(Arg::from_usage("[PATH] 'Path of the repository/backup/subtree, [repository][::backup[::subtree]]'"))) + .subcommand(SubCommand::with_name("list").alias("ls").about("List backups or backup contents") + .arg(Arg::from_usage("[PATH] 'Path of the repository/backup/subtree, [repository][::backup[::subtree]]'"))) + .subcommand(SubCommand::with_name("mount").about("Mount the repository, a backup or a subtree") + .arg(Arg::from_usage("[PATH] 'Path of the repository/backup/subtree, [repository][::backup[::subtree]]'")) + .arg(Arg::from_usage(" 'Existing mount point'"))) + .subcommand(SubCommand::with_name("bundlelist").about("List bundles in a repository") + .arg(Arg::from_usage("[REPO] 'Path of the repository'"))) + .subcommand(SubCommand::with_name("bundleinfo").about("Display information on a bundle") + .arg(Arg::from_usage("[REPO] 'Path of the repository'")) + .arg(Arg::from_usage(" 'Id of the bundle'"))) + .subcommand(SubCommand::with_name("import").about("Reconstruct a repository from the remote storage") + .arg(Arg::from_usage("-k --key [FILE]... 'Key file needed to read the bundles'")) + .arg(Arg::from_usage(" 'Remote repository path'")) + .arg(Arg::from_usage("[REPO] 'The path for the new repository'"))) + .subcommand(SubCommand::with_name("info").about("Display information on a repository, a backup or a subtree") + .arg(Arg::from_usage("[PATH] 'Path of the repository/backup/subtree, [repository][::backup[::subtree]]'"))) + .subcommand(SubCommand::with_name("analyze").about("Analyze the used and reclaimable space of bundles") + .arg(Arg::from_usage("[REPO] 'Path of the repository'"))) + .subcommand(SubCommand::with_name("versions").about("Find different versions of a file in all backups") + .arg(Arg::from_usage("[REPO] 'Path of the repository'")) + .arg(Arg::from_usage(" 'Path of the file'"))) + .subcommand(SubCommand::with_name("diff").about("Display differences between two backup versions") + .arg(Arg::from_usage(" 'Old version, [repository]::backup[::subpath]'")) + .arg(Arg::from_usage(" 'New version, [repository]::backup[::subpath]'"))) + .subcommand(SubCommand::with_name("config").about("Display or change the configuration") + .arg(Arg::from_usage("bundle_size --bundle-size [SIZE] 'Set the target bundle size in MiB (default: 25)'")) + .arg(Arg::from_usage("--chunker [CHUNKER] 'Set the chunker algorithm and target chunk size (default: fastcdc/16)'")) + .arg(Arg::from_usage("-c --compression [COMPRESSION] 'Set the compression method and level (default: brotli/3)'")) + .arg(Arg::from_usage("-e --encryption [PUBLIC_KEY] 'The public key to use for encryption'")) + .arg(Arg::from_usage("--hash [HASH] 'Set the hash method (default: blake2)'")) + .arg(Arg::from_usage("[REPO] 'Path of the repository'"))) + .subcommand(SubCommand::with_name("genkey").about("Generate a new key pair") + .arg(Arg::from_usage("[FILE] 'Destination file for the keypair'"))) + .subcommand(SubCommand::with_name("addkey").about("Add a key pair to the repository") + .arg(Arg::from_usage("-g --generate 'Generate a new key pair'").conflicts_with("FILE")) + .arg(Arg::from_usage("set_default --default -d 'Set the key pair as default'")) + .arg(Arg::from_usage("[FILE] 'File containing the keypair'").conflicts_with("generate")) + .arg(Arg::from_usage("[REPO] 'Path of the repository'"))) + .subcommand(SubCommand::with_name("algotest").about("Test a specific algorithm combination") + .arg(Arg::from_usage("bundle_size --bundle-size [SIZE] 'Set the target bundle size in MiB (default: 25)'")) + .arg(Arg::from_usage("--chunker [CHUNKER] 'Set the chunker algorithm and target chunk size (default: fastcdc/16)'")) + .arg(Arg::from_usage("-c --compression [COMPRESSION] 'Set the compression method and level (default: brotli/3)'")) + .arg(Arg::from_usage("-e --encryption 'Generate a keypair and enable encryption'")) + .arg(Arg::from_usage("--hash [HASH] 'Set the hash method (default: blake2)'")) + .arg(Arg::from_usage(" 'File with test data'"))).get_matches(); if let Some(args) = args.subcommand_matches("init") { let (repository, _backup, _inode) = try!(parse_repo_path(args.value_of("REPO").unwrap_or(""), Some(false), Some(false))); return Ok(Arguments::Init { diff --git a/src/cli/mod.rs b/src/cli/mod.rs index f81bec5..87bf607 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -68,7 +68,7 @@ pub const DEFAULT_CHUNKER: &'static str = "fastcdc/16"; pub const DEFAULT_HASH: &'static str = "blake2"; pub const DEFAULT_COMPRESSION: &'static str = "brotli/3"; pub const DEFAULT_BUNDLE_SIZE: usize = 25; -pub const DEFAULT_VACUUM_RATIO: usize = 50; +pub const DEFAULT_VACUUM_RATIO: usize = 0; lazy_static! { pub static ref DEFAULT_REPOSITORY: String = { env::home_dir().unwrap().join(".zvault").to_string_lossy().to_string()