New Upstream Release - restic
Ready changes
Summary
Merged new upstream version: 0.15.1+ds (was: 0.14.0).
Diff
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 000000000..d608a8244
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,13 @@
+version: 2
+updates:
+ # Dependencies listed in go.mod
+ - package-ecosystem: "gomod"
+ directory: "/" # Location of package manifests
+ schedule:
+ interval: "weekly"
+
+ # Dependencies listed in .github/workflows/*.yml
+ - package-ecosystem: "github-actions"
+ directory: "/"
+ schedule:
+ interval: "weekly"
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 2e628b5b8..4ef827843 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -21,13 +21,11 @@ jobs:
- job_name: Windows
go: 1.19.x
os: windows-latest
- install_verb: install
- job_name: macOS
go: 1.19.x
os: macOS-latest
test_fuse: false
- install_verb: install
- job_name: Linux
go: 1.19.x
@@ -35,31 +33,17 @@ jobs:
test_cloud_backends: true
test_fuse: true
check_changelog: true
- install_verb: install
- - job_name: Linux
- go: 1.18.x
- os: ubuntu-latest
- test_fuse: true
- install_verb: install
-
- - job_name: Linux
- go: 1.17.x
- os: ubuntu-latest
- test_fuse: true
- install_verb: install
-
- - job_name: Linux
- go: 1.16.x
+ - job_name: Linux (race)
+ go: 1.19.x
os: ubuntu-latest
test_fuse: true
- install_verb: get
+ test_opts: "-race"
- job_name: Linux
- go: 1.15.x
+ go: 1.18.x
os: ubuntu-latest
test_fuse: true
- install_verb: get
name: ${{ matrix.job_name }} Go ${{ matrix.go }}
runs-on: ${{ matrix.os }}
@@ -69,14 +53,14 @@ jobs:
steps:
- name: Set up Go ${{ matrix.go }}
- uses: actions/setup-go@v2
+ uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go }}
- name: Get programs (Linux/macOS)
run: |
echo "build Go tools"
- go ${{ matrix.install_verb }} github.com/restic/rest-server/cmd/rest-server@latest
+ go install github.com/restic/rest-server/cmd/rest-server@latest
echo "install minio server"
mkdir $HOME/bin
@@ -98,7 +82,7 @@ jobs:
chmod 755 $HOME/bin/rclone
rm -rf rclone*
- # add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v2)
+ # add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v3)
echo $HOME/bin >> $GITHUB_PATH
if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
@@ -108,7 +92,7 @@ jobs:
$ProgressPreference = 'SilentlyContinue'
echo "build Go tools"
- go ${{ matrix.install_verb }} github.com/restic/rest-server/...
+ go install github.com/restic/rest-server/...
echo "install minio server"
mkdir $Env:USERPROFILE/bin
@@ -120,7 +104,7 @@ jobs:
unzip rclone.zip
copy rclone*/rclone.exe $Env:USERPROFILE/bin
- # add $USERPROFILE/bin to path ($GOBIN was already added to the path by setup-go@v2)
+ # add $USERPROFILE/bin to path ($GOBIN was already added to the path by setup-go@v3)
echo $Env:USERPROFILE\bin >> $Env:GITHUB_PATH
echo "install tar"
@@ -142,7 +126,7 @@ jobs:
if: matrix.os == 'windows-latest'
- name: Check out code
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
- name: Build with build.go
run: |
@@ -152,7 +136,7 @@ jobs:
env:
RESTIC_TEST_FUSE: ${{ matrix.test_fuse }}
run: |
- go test -cover ./...
+ go test -cover ${{matrix.test_opts}} ./...
- name: Test cloud backends
env:
@@ -193,7 +177,9 @@ jobs:
# only run cloud backend tests for pull requests from and pushes to our
# own repo, otherwise the secrets are not available
- if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && matrix.test_cloud_backends
+ # Skip for Dependabot pull requests as these are run without secrets
+ # https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#responding-to-events
+ if: (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && (github.actor != 'dependabot[bot]') && matrix.test_cloud_backends
- name: Check changelog files with calens
run: |
@@ -209,15 +195,16 @@ jobs:
# ATTENTION: the list of architectures must be in sync with helpers/build-release-binaries/main.go!
matrix:
- # run cross-compile in two batches parallel so the overall tests run faster
+ # run cross-compile in three batches parallel so the overall tests run faster
targets:
- - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x \
- openbsd/386 openbsd/amd64"
+ - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/s390x"
- - "freebsd/386 freebsd/amd64 freebsd/arm \
+ - "openbsd/386 openbsd/amd64 \
+ freebsd/386 freebsd/amd64 freebsd/arm \
aix/ppc64 \
- darwin/amd64 darwin/arm64 \
- netbsd/386 netbsd/amd64 \
+ darwin/amd64 darwin/arm64"
+
+ - "netbsd/386 netbsd/amd64 \
windows/386 windows/amd64 \
solaris/amd64"
@@ -230,7 +217,7 @@ jobs:
steps:
- name: Set up Go ${{ env.latest_go }}
- uses: actions/setup-go@v2
+ uses: actions/setup-go@v3
with:
go-version: ${{ env.latest_go }}
@@ -239,7 +226,7 @@ jobs:
go install github.com/mitchellh/gox@latest
- name: Check out code
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
- name: Cross-compile with gox for ${{ matrix.targets }}
env:
@@ -255,22 +242,21 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Set up Go ${{ env.latest_go }}
- uses: actions/setup-go@v2
+ uses: actions/setup-go@v3
with:
go-version: ${{ env.latest_go }}
- name: Check out code
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
- name: golangci-lint
- uses: golangci/golangci-lint-action@v2
+ uses: golangci/golangci-lint-action@v3
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
- version: v1.48
+ version: v1.49
# Optional: show only new issues if it's a pull request. The default value is `false`.
only-new-issues: true
args: --verbose --timeout 5m
- skip-go-installation: true
# only run golangci-lint for pull requests, otherwise ALL hints get
# reported. We need to slowly address all issues until we can enable
@@ -288,11 +274,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out code
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
- name: Docker meta
id: meta
- uses: docker/metadata-action@v3
+ uses: docker/metadata-action@v4
with:
# list of Docker images to use as base name for tags
images: |
@@ -308,14 +294,14 @@ jobs:
type=sha
- name: Set up QEMU
- uses: docker/setup-qemu-action@v1
+ uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v1
+ uses: docker/setup-buildx-action@v2
- name: Build and push
id: docker_build
- uses: docker/build-push-action@v2
+ uses: docker/build-push-action@v3
with:
push: false
context: .
diff --git a/.golangci.yml b/.golangci.yml
index 44dfcdc7c..d97b3bd9b 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -55,3 +55,5 @@ issues:
- exported (function|method|var|type|const) .* should have comment or be unexported
# revive: ignore constants in all caps
- don't use ALL_CAPS in Go names; use CamelCase
+ # revive: lots of packages don't have such a comment
+ - "package-comments: should have a package comment"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 18ab202e6..8e5635404 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,559 @@
+Changelog for restic 0.15.1 (2023-01-30)
+=======================================
+
+The following sections list the changes in restic 0.15.1 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #3750: Remove `b2_download_file_by_name: 404` warning from B2 backend
+ * Fix #4147: Make `prune --quiet` not print progress bar
+ * Fix #4163: Make `self-update --output` work with new filename on Windows
+ * Fix #4167: Add missing ETA in `backup` progress bar
+ * Enh #4143: Ignore empty lock files
+
+Details
+-------
+
+ * Bugfix #3750: Remove `b2_download_file_by_name: 404` warning from B2 backend
+
+ In some cases the B2 backend could print `b2_download_file_by_name: 404: : b2.b2err`
+ warnings. These are only debug messages and can be safely ignored.
+
+ Restic now uses an updated library for accessing B2, which removes the warning.
+
+ https://github.com/restic/restic/issues/3750
+ https://github.com/restic/restic/issues/4144
+ https://github.com/restic/restic/pull/4146
+
+ * Bugfix #4147: Make `prune --quiet` not print progress bar
+
+ A regression in restic 0.15.0 caused `prune --quiet` to show a progress bar while deciding how
+ to process each pack files. This has now been fixed.
+
+ https://github.com/restic/restic/issues/4147
+ https://github.com/restic/restic/pull/4153
+
+ * Bugfix #4163: Make `self-update --output` work with new filename on Windows
+
+ Since restic 0.14.0 the `self-update` command did not work when a custom output filename was
+ specified via the `--output` option. This has now been fixed.
+
+ As a workaround, either use an older restic version to run the self-update or create an empty
+ file with the output filename before updating e.g. using CMD:
+
+ `type nul > new-file.exe` `restic self-update --output new-file.exe`
+
+ https://github.com/restic/restic/pull/4163
+ https://forum.restic.net/t/self-update-windows-started-failing-after-release-of-0-15/5836
+
+ * Bugfix #4167: Add missing ETA in `backup` progress bar
+
+ A regression in restic 0.15.0 caused the ETA to be missing from the progress bar displayed by the
+ `backup` command. This has now been fixed.
+
+ https://github.com/restic/restic/pull/4167
+
+ * Enhancement #4143: Ignore empty lock files
+
+ With restic 0.15.0 the checks for stale locks became much stricter than before. In particular,
+ empty or unreadable locks were no longer silently ignored. This made restic to complain with
+ `Load(<lock/1234567812>, 0, 0) returned error, retrying after 552.330144ms:
+ load(<lock/1234567812>): invalid data returned` and fail in the end.
+
+ The error message is now clarified and the implementation changed to ignore empty lock files
+ which are sometimes created as the result of a failed uploads on some backends.
+
+ Please note that unreadable lock files still have to cleaned up manually. To do so, you can run
+ `restic unlock --remove-all` which removes all existing lock files. But first make sure that
+ no other restic process is currently using the repository.
+
+ https://github.com/restic/restic/issues/4143
+ https://github.com/restic/restic/pull/4152
+
+
+Changelog for restic 0.15.0 (2023-01-12)
+=======================================
+
+The following sections list the changes in restic 0.15.0 relevant to
+restic users. The changes are ordered by importance.
+
+Summary
+-------
+
+ * Fix #2015: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT
+ * Fix #2578: Make `restore` replace existing symlinks
+ * Fix #2591: Don't read password from stdin for `backup --stdin`
+ * Fix #3161: Delete files on Backblaze B2 more reliably
+ * Fix #3336: Make SFTP backend report no space left on device
+ * Fix #3567: Improve handling of interrupted syscalls in `mount` command
+ * Fix #3897: Fix stuck `copy` command when `-o <backend>.connections=1`
+ * Fix #3918: Correct prune statistics for partially compressed repositories
+ * Fix #3951: Make `ls` return exit code 1 if snapshot cannot be loaded
+ * Fix #4003: Make `backup` no longer hang on Solaris when seeing a FIFO file
+ * Fix #4016: Support ExFAT-formatted local backends on macOS Ventura
+ * Fix #4085: Make `init` ignore "Access Denied" errors when creating S3 buckets
+ * Fix #4100: Make `self-update` enabled by default only in release builds
+ * Fix #4103: Don't generate negative UIDs and GIDs in tar files from `dump`
+ * Chg #2724: Include full snapshot ID in JSON output of `backup`
+ * Chg #3929: Make `unlock` display message only when locks were actually removed
+ * Chg #4033: Don't print skipped snapshots by default in `copy` command
+ * Chg #4041: Update dependencies and require Go 1.18 or newer
+ * Enh #14: Implement `rewrite` command
+ * Enh #79: Restore files with long runs of zeros as sparse files
+ * Enh #1078: Support restoring symbolic links on Windows
+ * Enh #1734: Inform about successful retries after errors
+ * Enh #1866: Improve handling of directories with duplicate entries
+ * Enh #2134: Support B2 API keys restricted to hiding but not deleting files
+ * Enh #2152: Make `init` open only one connection for the SFTP backend
+ * Enh #2533: Handle cache corruption on disk and in downloads
+ * Enh #2715: Stricter repository lock handling
+ * Enh #2750: Make backup file read concurrency configurable
+ * Enh #3029: Add support for `credential_process` to S3 backend
+ * Enh #3096: Make `mount` command support macOS using macFUSE 4.x
+ * Enh #3124: Support JSON output for the `init` command
+ * Enh #3899: Optimize prune memory usage
+ * Enh #3905: Improve speed of parent snapshot detection in `backup` command
+ * Enh #3915: Add compression statistics to the `stats` command
+ * Enh #3925: Provide command completion for PowerShell
+ * Enh #3931: Allow `backup` file tree scanner to be disabled
+ * Enh #3932: Improve handling of ErrDot errors in rclone and sftp backends
+ * Enh #3943: Ignore additional/unknown files in repository
+ * Enh #3955: Improve `backup` performance for small files
+
+Details
+-------
+
+ * Bugfix #2015: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT
+
+ To stop the `mount` command, a user has to press Ctrl-C or send a SIGINT signal to restic. This
+ used to cause restic to exit with a non-zero exit code.
+
+ The exit code has now been changed to zero as the above is the expected way to stop the `mount`
+ command and should therefore be considered successful.
+
+ https://github.com/restic/restic/issues/2015
+ https://github.com/restic/restic/pull/3894
+
+ * Bugfix #2578: Make `restore` replace existing symlinks
+
+ When restoring a symlink, restic used to report an error if the target path already existed.
+ This has now been fixed such that the potentially existing target path is first removed before
+ the symlink is restored.
+
+ https://github.com/restic/restic/issues/2578
+ https://github.com/restic/restic/pull/3780
+
+ * Bugfix #2591: Don't read password from stdin for `backup --stdin`
+
+ The `backup` command when used with `--stdin` previously tried to read first the password,
+ then the data to be backed up from standard input. This meant it would often confuse part of the
+ data for the password.
+
+ From now on, it will instead exit with the message `Fatal: cannot read both password and data
+ from stdin` unless the password is passed in some other way (such as
+ `--restic-password-file`, `RESTIC_PASSWORD`, etc).
+
+ To enter the password interactively a password command has to be used. For example on Linux,
+ `mysqldump somedatabase | restic backup --stdin --password-command='sh -c
+ "systemd-ask-password < /dev/tty"'` securely reads the password from the terminal.
+
+ https://github.com/restic/restic/issues/2591
+ https://github.com/restic/restic/pull/4011
+
+ * Bugfix #3161: Delete files on Backblaze B2 more reliably
+
+ Restic used to only delete the latest version of files stored in B2. In most cases this worked
+ well as there was only a single version of the file. However, due to retries while uploading it is
+ possible for multiple file versions to be stored at B2. This could lead to various problems for
+ files that should have been deleted but still existed.
+
+ The implementation has now been changed to delete all versions of files, which doubles the
+ amount of Class B transactions necessary to delete files, but assures that no file versions are
+ left behind.
+
+ https://github.com/restic/restic/issues/3161
+ https://github.com/restic/restic/pull/3885
+
+ * Bugfix #3336: Make SFTP backend report no space left on device
+
+ Backing up to an SFTP backend would spew repeated SSH_FX_FAILURE messages when the remote disk
+ was full. Restic now reports "sftp: no space left on device" and exits immediately when it
+ detects this condition.
+
+ A fix for this issue was implemented in restic 0.12.1, but unfortunately the fix itself
+ contained a bug that prevented it from taking effect.
+
+ https://github.com/restic/restic/issues/3336
+ https://github.com/restic/restic/pull/3345
+ https://github.com/restic/restic/pull/4075
+
+ * Bugfix #3567: Improve handling of interrupted syscalls in `mount` command
+
+ Accessing restic's FUSE mount could result in "input/output" errors when using programs in
+ which syscalls can be interrupted. This is for example the case for Go programs. This has now
+ been fixed by improved error handling of interrupted syscalls.
+
+ https://github.com/restic/restic/issues/3567
+ https://github.com/restic/restic/issues/3694
+ https://github.com/restic/restic/pull/3875
+
+ * Bugfix #3897: Fix stuck `copy` command when `-o <backend>.connections=1`
+
+ When running the `copy` command with `-o <backend>.connections=1` the command would be
+ infinitely stuck. This has now been fixed.
+
+ https://github.com/restic/restic/issues/3897
+ https://github.com/restic/restic/pull/3898
+
+ * Bugfix #3918: Correct prune statistics for partially compressed repositories
+
+ In a partially compressed repository, one data blob can exist both in an uncompressed and a
+ compressed version. This caused the `prune` statistics to become inaccurate and e.g. report a
+ too high value for the unused size, such as "unused size after prune: 16777215.991 TiB". This
+ has now been fixed.
+
+ https://github.com/restic/restic/issues/3918
+ https://github.com/restic/restic/pull/3980
+
+ * Bugfix #3951: Make `ls` return exit code 1 if snapshot cannot be loaded
+
+ The `ls` command used to show a warning and return exit code 0 when failing to load a snapshot.
+ This has now been fixed such that it instead returns exit code 1 (still showing a warning).
+
+ https://github.com/restic/restic/pull/3951
+
+ * Bugfix #4003: Make `backup` no longer hang on Solaris when seeing a FIFO file
+
+ The `backup` command used to hang on Solaris whenever it encountered a FIFO file (named pipe),
+ due to a bug in the handling of extended attributes. This bug has now been fixed.
+
+ https://github.com/restic/restic/issues/4003
+ https://github.com/restic/restic/pull/4053
+
+ * Bugfix #4016: Support ExFAT-formatted local backends on macOS Ventura
+
+ ExFAT-formatted disks could not be used as local backends starting from macOS Ventura. Restic
+ commands would fail with an "inappropriate ioctl for device" error. This has now been fixed.
+
+ https://github.com/restic/restic/issues/4016
+ https://github.com/restic/restic/pull/4021
+
+ * Bugfix #4085: Make `init` ignore "Access Denied" errors when creating S3 buckets
+
+ In restic 0.9.0 through 0.13.0, the `init` command ignored some permission errors from S3
+ backends when trying to check for bucket existence, so that manually created buckets with
+ custom permissions could be used for backups.
+
+ This feature became broken in 0.14.0, but has now been restored again.
+
+ https://github.com/restic/restic/issues/4085
+ https://github.com/restic/restic/pull/4086
+
+ * Bugfix #4100: Make `self-update` enabled by default only in release builds
+
+ The `self-update` command was previously included by default in all builds of restic as
+ opposed to only in official release builds, even if the `selfupdate` tag was not explicitly
+ enabled when building.
+
+ This has now been corrected, and the `self-update` command is only available if restic was
+ built with `-tags selfupdate` (as done for official release builds by `build.go`).
+
+ https://github.com/restic/restic/pull/4100
+
+ * Bugfix #4103: Don't generate negative UIDs and GIDs in tar files from `dump`
+
+ When using a 32-bit build of restic, the `dump` command could in some cases create tar files
+ containing negative UIDs and GIDs, which cannot be read by GNU tar. This corner case especially
+ applies to backups from stdin on Windows.
+
+ This is now fixed such that `dump` creates valid tar files in these cases too.
+
+ https://github.com/restic/restic/issues/4103
+ https://github.com/restic/restic/pull/4104
+
+ * Change #2724: Include full snapshot ID in JSON output of `backup`
+
+ We have changed the JSON output of the backup command to include the full snapshot ID instead of
+ just a shortened version, as the latter can be ambiguous in some rare cases. To derive the short
+ ID, please truncate the full ID down to eight characters.
+
+ https://github.com/restic/restic/issues/2724
+ https://github.com/restic/restic/pull/3993
+
+ * Change #3929: Make `unlock` display message only when locks were actually removed
+
+ The `unlock` command used to print the "successfully removed locks" message whenever it was
+ run, regardless of lock files having being removed or not.
+
+ This has now been changed such that it only prints the message if any lock files were actually
+ removed. In addition, it also reports the number of removed lock files.
+
+ https://github.com/restic/restic/issues/3929
+ https://github.com/restic/restic/pull/3935
+
+ * Change #4033: Don't print skipped snapshots by default in `copy` command
+
+ The `copy` command used to print each snapshot that was skipped because it already existed in
+ the target repository. The amount of this output could practically bury the list of snapshots
+ that were actually copied.
+
+ From now on, the skipped snapshots are by default not printed at all, but this can be re-enabled
+ by increasing the verbosity level of the command.
+
+ https://github.com/restic/restic/issues/4033
+ https://github.com/restic/restic/pull/4066
+
+ * Change #4041: Update dependencies and require Go 1.18 or newer
+
+ Most dependencies have been updated. Since some libraries require newer language features,
+ support for Go 1.15-1.17 has been dropped, which means that restic now requires at least Go 1.18
+ to build.
+
+ https://github.com/restic/restic/pull/4041
+
+ * Enhancement #14: Implement `rewrite` command
+
+ Restic now has a `rewrite` command which allows to rewrite existing snapshots to remove
+ unwanted files.
+
+ https://github.com/restic/restic/issues/14
+ https://github.com/restic/restic/pull/2731
+ https://github.com/restic/restic/pull/4079
+
+ * Enhancement #79: Restore files with long runs of zeros as sparse files
+
+ When using `restore --sparse`, the restorer may now write files containing long runs of zeros
+ as sparse files (also called files with holes), where the zeros are not actually written to
+ disk.
+
+ How much space is saved by writing sparse files depends on the operating system, file system and
+ the distribution of zeros in the file.
+
+ During backup restic still reads the whole file including sparse regions, but with optimized
+ processing speed of sparse regions.
+
+ https://github.com/restic/restic/issues/79
+ https://github.com/restic/restic/issues/3903
+ https://github.com/restic/restic/pull/2601
+ https://github.com/restic/restic/pull/3854
+ https://forum.restic.net/t/sparse-file-support/1264
+
+ * Enhancement #1078: Support restoring symbolic links on Windows
+
+ The `restore` command now supports restoring symbolic links on Windows. Because of Windows
+ specific restrictions this is only possible when running restic with the
+ `SeCreateSymbolicLinkPrivilege` privilege or as an administrator.
+
+ https://github.com/restic/restic/issues/1078
+ https://github.com/restic/restic/issues/2699
+ https://github.com/restic/restic/pull/2875
+
+ * Enhancement #1734: Inform about successful retries after errors
+
+ When a recoverable error is encountered, restic shows a warning message saying that it's
+ retrying, e.g.:
+
+ `Save(<data/956b9ced99>) returned error, retrying after 357.131936ms: ...`
+
+ This message can be confusing in that it never clearly states whether the retry is successful or
+ not. This has now been fixed such that restic follows up with a message confirming a successful
+ retry, e.g.:
+
+ `Save(<data/956b9ced99>) operation successful after 1 retries`
+
+ https://github.com/restic/restic/issues/1734
+ https://github.com/restic/restic/pull/2661
+
+ * Enhancement #1866: Improve handling of directories with duplicate entries
+
+ If for some reason a directory contains a duplicate entry, the `backup` command would
+ previously fail with a `node "path/to/file" already present` or `nodes are not ordered got
+ "path/to/file", last "path/to/file"` error.
+
+ The error handling has been improved to only report a warning in this case. Make sure to check
+ that the filesystem in question is not damaged if you see this!
+
+ https://github.com/restic/restic/issues/1866
+ https://github.com/restic/restic/issues/3937
+ https://github.com/restic/restic/pull/3880
+
+ * Enhancement #2134: Support B2 API keys restricted to hiding but not deleting files
+
+ When the B2 backend does not have the necessary permissions to permanently delete files, it now
+ automatically falls back to hiding files. This allows using restic with an application key
+ which is not allowed to delete files. This can prevent an attacker from deleting backups with
+ such an API key.
+
+ To use this feature create an application key without the `deleteFiles` capability. It is
+ recommended to restrict the key to just one bucket. For example using the `b2` command line
+ tool:
+
+ `b2 create-key --bucket <bucketName> <keyName>
+ listBuckets,readFiles,writeFiles,listFiles`
+
+ Alternatively, you can use the S3 backend to access B2, as described in the documentation. In
+ this mode, files are also only hidden instead of being deleted permanently.
+
+ https://github.com/restic/restic/issues/2134
+ https://github.com/restic/restic/pull/2398
+
+ * Enhancement #2152: Make `init` open only one connection for the SFTP backend
+
+ The `init` command using the SFTP backend used to connect twice to the repository. This could be
+ inconvenient if the user must enter a password, or cause `init` to fail if the server does not
+ correctly close the first SFTP connection.
+
+ This has now been fixed by reusing the first/initial SFTP connection opened.
+
+ https://github.com/restic/restic/issues/2152
+ https://github.com/restic/restic/pull/3882
+
+ * Enhancement #2533: Handle cache corruption on disk and in downloads
+
+ In rare situations, like for example after a system crash, the data stored in the cache might be
+ corrupted. This could cause restic to fail and required manually deleting the cache.
+
+ Restic now automatically removes broken data from the cache, allowing it to recover from such a
+ situation without user intervention. In addition, restic retries downloads which return
+ corrupt data in order to also handle temporary download problems.
+
+ https://github.com/restic/restic/issues/2533
+ https://github.com/restic/restic/pull/3521
+
+ * Enhancement #2715: Stricter repository lock handling
+
+ Previously, restic commands kept running even if they failed to refresh their locks in time.
+ This could be a problem e.g. in case the client system running a backup entered the standby power
+ mode while the backup was still in progress (which would prevent the client from refreshing its
+ lock), and after a short delay another host successfully runs `unlock` and `prune` on the
+ repository, which would remove all data added by the in-progress backup. If the backup client
+ later continues its backup, even though its lock had expired in the meantime, this would lead to
+ an incomplete snapshot.
+
+ To address this, lock handling is now much stricter. Commands requiring a lock are canceled if
+ the lock is not refreshed successfully in time. In addition, if a lock file is not readable
+ restic will not allow starting a command. It may be necessary to remove invalid lock files
+ manually or use `unlock --remove-all`. Please make sure that no other restic processes are
+ running concurrently before doing this, however.
+
+ https://github.com/restic/restic/issues/2715
+ https://github.com/restic/restic/pull/3569
+
+ * Enhancement #2750: Make backup file read concurrency configurable
+
+ The `backup` command now supports a `--read-concurrency` option which allows tuning restic
+ for very fast storage like NVMe disks by controlling the number of concurrent file reads during
+ the backup process.
+
+ https://github.com/restic/restic/pull/2750
+
+ * Enhancement #3029: Add support for `credential_process` to S3 backend
+
+ Restic now uses a newer library for the S3 backend, which adds support for the
+ `credential_process` option in the AWS credential configuration.
+
+ https://github.com/restic/restic/issues/3029
+ https://github.com/restic/restic/issues/4034
+ https://github.com/restic/restic/pull/4025
+
+ * Enhancement #3096: Make `mount` command support macOS using macFUSE 4.x
+
+ Restic now uses a different FUSE library for mounting snapshots and making them available as a
+ FUSE filesystem using the `mount` command. This adds support for macFUSE 4.x which can be used
+ to make this work on recent macOS versions.
+
+ https://github.com/restic/restic/issues/3096
+ https://github.com/restic/restic/pull/4024
+
+ * Enhancement #3124: Support JSON output for the `init` command
+
+ The `init` command used to ignore the `--json` option, but now outputs a JSON message if the
+ repository was created successfully.
+
+ https://github.com/restic/restic/issues/3124
+ https://github.com/restic/restic/pull/3132
+
+ * Enhancement #3899: Optimize prune memory usage
+
+ The `prune` command needs large amounts of memory in order to determine what to keep and what to
+ remove. This is now optimized to use up to 30% less memory.
+
+ https://github.com/restic/restic/pull/3899
+
+ * Enhancement #3905: Improve speed of parent snapshot detection in `backup` command
+
+ Backing up a large number of files using `--files-from-verbatim` or `--files-from-raw`
+ options could require a long time to find the parent snapshot. This has been improved.
+
+ https://github.com/restic/restic/pull/3905
+
+ * Enhancement #3915: Add compression statistics to the `stats` command
+
+ When executed with `--mode raw-data` on a repository that supports compression, the `stats`
+ command now calculates and displays, for the selected repository or snapshots: the
+ uncompressed size of the data; the compression progress (percentage of data that has been
+ compressed); the compression ratio of the compressed data; the total space saving.
+
+ It also takes into account both the compressed and uncompressed data if the repository is only
+ partially compressed.
+
+ https://github.com/restic/restic/pull/3915
+
+ * Enhancement #3925: Provide command completion for PowerShell
+
+ Restic already provided generation of completion files for bash, fish and zsh. Now powershell
+ is supported, too.
+
+ https://github.com/restic/restic/pull/3925/files
+
+ * Enhancement #3931: Allow `backup` file tree scanner to be disabled
+
+ The `backup` command walks the file tree in a separate scanner process to find the total size and
+ file/directory count, and uses this to provide an ETA. This can slow down backups, especially
+ of network filesystems.
+
+ The command now has a new option `--no-scan` which can be used to disable this scanning in order
+ to speed up backups when needed.
+
+ https://github.com/restic/restic/pull/3931
+
+ * Enhancement #3932: Improve handling of ErrDot errors in rclone and sftp backends
+
+ Since Go 1.19, restic can no longer implicitly run relative executables which are found in the
+ current directory (e.g. `rclone` if found in `.`). This is a security feature of Go to prevent
+ against running unintended and possibly harmful executables.
+
+ The error message for this was just "cannot run executable found relative to current
+ directory". This has now been improved to yield a more specific error message, informing the
+ user how to explicitly allow running the executable using the `-o rclone.program` and `-o
+ sftp.command` extended options with `./`.
+
+ https://github.com/restic/restic/issues/3932
+ https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory
+ https://go.dev/blog/path-security
+
+ * Enhancement #3943: Ignore additional/unknown files in repository
+
+ If a restic repository had additional files in it (not created by restic), commands like `find`
+ and `restore` could become confused and fail with an `multiple IDs with prefix "12345678"
+ found` error. These commands now ignore such additional files.
+
+ https://github.com/restic/restic/pull/3943
+ https://forum.restic.net/t/which-protocol-should-i-choose-for-remote-linux-backups/5446/17
+
+ * Enhancement #3955: Improve `backup` performance for small files
+
+ When backing up small files restic was slower than it could be. In particular this affected
+ backups using maximum compression.
+
+ This has been fixed by reworking the internal parallelism of the backup command, making it back
+ up small files around two times faster.
+
+ https://github.com/restic/restic/pull/3955
+
+
Changelog for restic 0.14.0 (2022-08-25)
=======================================
@@ -10,36 +566,36 @@ Summary
* Fix #2248: Support `self-update` on Windows
* Fix #3428: List snapshots in backend at most once to resolve snapshot IDs
* Fix #3432: Fix rare 'not found in repository' error for `copy` command
- * Fix #3685: The `diff` command incorrectly listed some files as added
* Fix #3681: Fix rclone (shimmed by Scoop) and sftp not working on Windows
+ * Fix #3685: The `diff` command incorrectly listed some files as added
+ * Fix #3716: Print "wrong password" to stderr instead of stdout
* Fix #3720: Directory sync errors for repositories accessed via SMB
* Fix #3736: The `stats` command miscalculated restore size for multiple snapshots
- * Fix #3861: Yield error on invalid policy to `forget`
- * Fix #3716: Print "wrong password" to stderr instead of stdout
* Fix #3772: Correctly rebuild index for legacy repositories
* Fix #3776: Limit number of key files tested while opening a repository
+ * Fix #3861: Yield error on invalid policy to `forget`
* Chg #1842: Support debug log creation in release builds
* Chg #3295: Deprecate `check --check-unused` and add further checks
* Chg #3680: Update dependencies and require Go 1.15 or newer
* Chg #3742: Replace `--repo2` option used by `init`/`copy` with `--from-repo`
- * Enh #1153: Support pruning even when the disk is full
* Enh #21: Add compression support
+ * Enh #1153: Support pruning even when the disk is full
* Enh #2162: Adaptive IO concurrency based on backend connections
* Enh #2291: Allow pack size customization
* Enh #2295: Allow use of SAS token to authenticate to Azure
+ * Enh #2351: Use config file permissions to control file group access
* Enh #2696: Improve backup speed with many small files
* Enh #2907: Make snapshot directory structure of `mount` command customizable
+ * Enh #2923: Improve speed of `copy` command
* Enh #3114: Optimize handling of duplicate blobs in `prune`
* Enh #3465: Improve handling of temporary files on Windows
- * Enh #3709: Validate exclude patterns before backing up
- * Enh #3837: Improve SFTP repository initialization over slow links
- * Enh #2351: Use config file permissions to control file group access
* Enh #3475: Allow limiting IO concurrency for local and SFTP backend
* Enh #3484: Stream data in `check` and `prune` commands
- * Enh #2923: Improve speed of `copy` command
+ * Enh #3709: Validate exclude patterns before backing up
* Enh #3729: Display full IDs in `check` warnings
* Enh #3773: Optimize memory usage for directories with many files
* Enh #3819: Validate include/exclude patterns before restoring
+ * Enh #3837: Improve SFTP repository initialization over slow links
Details
-------
@@ -81,14 +637,6 @@ Details
https://github.com/restic/restic/issues/3432
https://github.com/restic/restic/pull/3570
- * Bugfix #3685: The `diff` command incorrectly listed some files as added
-
- There was a bug in the `diff` command, causing it to always show files in a removed directory as
- added. This has now been fixed.
-
- https://github.com/restic/restic/issues/3685
- https://github.com/restic/restic/pull/3686
-
* Bugfix #3681: Fix rclone (shimmed by Scoop) and sftp not working on Windows
In #3602 a fix was introduced to address the problem of `rclone` prematurely exiting when
@@ -103,6 +651,22 @@ Details
https://github.com/restic/restic/issues/3692
https://github.com/restic/restic/pull/3696
+ * Bugfix #3685: The `diff` command incorrectly listed some files as added
+
+ There was a bug in the `diff` command, causing it to always show files in a removed directory as
+ added. This has now been fixed.
+
+ https://github.com/restic/restic/issues/3685
+ https://github.com/restic/restic/pull/3686
+
+ * Bugfix #3716: Print "wrong password" to stderr instead of stdout
+
+ If an invalid password was entered, the error message was printed on stdout and not on stderr as
+ intended. This has now been fixed.
+
+ https://github.com/restic/restic/pull/3716
+ https://forum.restic.net/t/4965
+
* Bugfix #3720: Directory sync errors for repositories accessed via SMB
On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in restic failing to
@@ -127,24 +691,6 @@ Details
https://github.com/restic/restic/issues/3736
https://github.com/restic/restic/pull/3740
- * Bugfix #3861: Yield error on invalid policy to `forget`
-
- The `forget` command previously silently ignored invalid/unsupported units in the duration
- options, such as e.g. `--keep-within-daily 2w`.
-
- Specifying an invalid/unsupported duration unit now results in an error.
-
- https://github.com/restic/restic/issues/3861
- https://github.com/restic/restic/pull/3862
-
- * Bugfix #3716: Print "wrong password" to stderr instead of stdout
-
- If an invalid password was entered, the error message was printed on stdout and not on stderr as
- intended. This has now been fixed.
-
- https://github.com/restic/restic/pull/3716
- https://forum.restic.net/t/4965
-
* Bugfix #3772: Correctly rebuild index for legacy repositories
After running `rebuild-index` on a legacy repository containing mixed pack files (that is,
@@ -169,6 +715,16 @@ Details
https://github.com/restic/restic/pull/3776
+ * Bugfix #3861: Yield error on invalid policy to `forget`
+
+ The `forget` command previously silently ignored invalid/unsupported units in the duration
+ options, such as e.g. `--keep-within-daily 2w`.
+
+ Specifying an invalid/unsupported duration unit now results in an error.
+
+ https://github.com/restic/restic/issues/3861
+ https://github.com/restic/restic/pull/3862
+
* Change #1842: Support debug log creation in release builds
Creating a debug log was only possible in debug builds which required users to manually build
@@ -215,16 +771,6 @@ Details
https://github.com/restic/restic/pull/3742
https://forum.restic.net/t/5017
- * Enhancement #1153: Support pruning even when the disk is full
-
- When running out of disk space it was no longer possible to add or remove data from a repository.
- To help with recovering from such a deadlock, the prune command now supports an
- `--unsafe-recover-no-free-space` option to recover from these situations. Make sure to
- read the documentation first!
-
- https://github.com/restic/restic/issues/1153
- https://github.com/restic/restic/pull/3481
-
* Enhancement #21: Add compression support
We've added compression support to the restic repository format. To create a repository using
@@ -253,6 +799,16 @@ Details
https://github.com/restic/restic/pull/3704
https://github.com/restic/restic/pull/3733
+ * Enhancement #1153: Support pruning even when the disk is full
+
+ When running out of disk space it was no longer possible to add or remove data from a repository.
+ To help with recovering from such a deadlock, the prune command now supports an
+ `--unsafe-recover-no-free-space` option to recover from these situations. Make sure to
+ read the documentation first!
+
+ https://github.com/restic/restic/issues/1153
+ https://github.com/restic/restic/pull/3481
+
* Enhancement #2162: Adaptive IO concurrency based on backend connections
Many commands used hard-coded limits for the number of concurrent operations. This prevented
@@ -294,6 +850,27 @@ Details
https://github.com/restic/restic/issues/2295
https://github.com/restic/restic/pull/3661
+ * Enhancement #2351: Use config file permissions to control file group access
+
+ Previously files in a local/SFTP repository would always end up with very restrictive access
+ permissions, allowing access only to the owner. This prevented a number of valid use-cases
+ involving groups and ACLs.
+
+ We now use the permissions of the config file in the repository to decide whether group access
+ should be given to newly created repository files or not. We arrange for repository files to be
+ created group readable exactly when the repository config file is group readable.
+
+ To opt-in to group readable repositories, a simple `chmod -R g+r` or equivalent on the config
+ file can be used. For repositories that should be writable by group members a tad more setup is
+ required, see the docs.
+
+ Posix ACLs can also be used now that the group permissions being forced to zero no longer masks
+ the effect of ACL entries.
+
+ https://github.com/restic/restic/issues/2351
+ https://github.com/restic/restic/pull/3419
+ https://forum.restic.net/t/1391
+
* Enhancement #2696: Improve backup speed with many small files
We have restructured the backup pipeline to continue reading files while all upload
@@ -322,6 +899,16 @@ Details
https://github.com/restic/restic/pull/2913
https://github.com/restic/restic/pull/3691
+ * Enhancement #2923: Improve speed of `copy` command
+
+ The `copy` command could require a long time to copy snapshots for non-local backends. This has
+ been improved to provide a throughput comparable to the `restore` command.
+
+ Additionally, `copy` now displays a progress bar.
+
+ https://github.com/restic/restic/issues/2923
+ https://github.com/restic/restic/pull/3513
+
* Enhancement #3114: Optimize handling of duplicate blobs in `prune`
Restic `prune` always used to repack all data files containing duplicate blobs. This
@@ -344,48 +931,6 @@ Details
https://github.com/restic/restic/issues/1551
https://github.com/restic/restic/pull/3610
- * Enhancement #3709: Validate exclude patterns before backing up
-
- Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or
- `--iexclude-file` previously weren't validated. As a consequence, invalid patterns
- resulted in files that were meant to be excluded being backed up.
-
- Restic now validates all patterns before running the backup and aborts with a fatal error if an
- invalid pattern is detected.
-
- https://github.com/restic/restic/issues/3709
- https://github.com/restic/restic/pull/3734
-
- * Enhancement #3837: Improve SFTP repository initialization over slow links
-
- The `init` command, when used on an SFTP backend, now sends multiple `mkdir` commands to the
- backend concurrently. This reduces the waiting times when creating a repository over a very
- slow connection.
-
- https://github.com/restic/restic/issues/3837
- https://github.com/restic/restic/pull/3840
-
- * Enhancement #2351: Use config file permissions to control file group access
-
- Previously files in a local/SFTP repository would always end up with very restrictive access
- permissions, allowing access only to the owner. This prevented a number of valid use-cases
- involving groups and ACLs.
-
- We now use the permissions of the config file in the repository to decide whether group access
- should be given to newly created repository files or not. We arrange for repository files to be
- created group readable exactly when the repository config file is group readable.
-
- To opt-in to group readable repositories, a simple `chmod -R g+r` or equivalent on the config
- file can be used. For repositories that should be writable by group members a tad more setup is
- required, see the docs.
-
- Posix ACLs can also be used now that the group permissions being forced to zero no longer masks
- the effect of ACL entries.
-
- https://github.com/restic/restic/issues/2351
- https://github.com/restic/restic/pull/3419
- https://forum.restic.net/t/1391
-
* Enhancement #3475: Allow limiting IO concurrency for local and SFTP backend
Restic did not support limiting the IO concurrency / number of connections for accessing
@@ -409,15 +954,17 @@ Details
https://github.com/restic/restic/pull/3484
https://github.com/restic/restic/pull/3717
- * Enhancement #2923: Improve speed of `copy` command
+ * Enhancement #3709: Validate exclude patterns before backing up
- The `copy` command could require a long time to copy snapshots for non-local backends. This has
- been improved to provide a throughput comparable to the `restore` command.
+ Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or
+ `--iexclude-file` previously weren't validated. As a consequence, invalid patterns
+ resulted in files that were meant to be excluded being backed up.
- Additionally, `copy` now displays a progress bar.
+ Restic now validates all patterns before running the backup and aborts with a fatal error if an
+ invalid pattern is detected.
- https://github.com/restic/restic/issues/2923
- https://github.com/restic/restic/pull/3513
+ https://github.com/restic/restic/issues/3709
+ https://github.com/restic/restic/pull/3734
* Enhancement #3729: Display full IDs in `check` warnings
@@ -447,6 +994,15 @@ Details
https://github.com/restic/restic/pull/3819
+ * Enhancement #3837: Improve SFTP repository initialization over slow links
+
+ The `init` command, when used on an SFTP backend, now sends multiple `mkdir` commands to the
+ backend concurrently. This reduces the waiting times when creating a repository over a very
+ slow connection.
+
+ https://github.com/restic/restic/issues/3837
+ https://github.com/restic/restic/pull/3840
+
Changelog for restic 0.13.0 (2022-03-26)
=======================================
@@ -462,34 +1018,34 @@ Summary
* Fix #2452: Improve error handling of repository locking
* Fix #2738: Don't print progress for `backup --json --quiet`
* Fix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable
- * Fix #3518: Make `copy` command honor `--no-lock` for source repository
- * Fix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
- * Fix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
- * Fix #3667: The `mount` command now reports symlinks sizes
* Fix #3488: `rebuild-index` failed if an index file was damaged
+ * Fix #3518: Make `copy` command honor `--no-lock` for source repository
+ * Fix #3556: Fix hang with Backblaze B2 on SSL certificate authority error
* Fix #3591: Fix handling of `prune --max-repack-size=0`
+ * Fix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
* Fix #3619: Avoid choosing parent snapshots newer than time of new snapshot
- * Chg #3641: Ignore parent snapshot for `backup --stdin`
+ * Fix #3667: The `mount` command now reports symlinks sizes
* Chg #3519: Require Go 1.14 or newer
+ * Chg #3641: Ignore parent snapshot for `backup --stdin`
+ * Enh #233: Support negative include/exclude patterns
* Enh #1542: Add `--dry-run`/`-n` option to `backup` command
* Enh #2202: Add upload checksum for Azure, GS, S3 and Swift backends
- * Enh #233: Support negative include/exclude patterns
* Enh #2388: Add warning for S3 if partial credentials are provided
* Enh #2508: Support JSON output and quiet mode for the `diff` command
- * Enh #2656: Add flag to disable TLS verification for self-signed certificates
- * Enh #3003: Atomic uploads for the SFTP backend
- * Enh #3127: Add xattr (extended attributes) support for Solaris
- * Enh #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
- * Enh #3490: Support random subset by size in `check --read-data-subset`
- * Enh #3541: Improve handling of temporary B2 delete errors
- * Enh #3542: Add file mode in symbolic notation to `ls --json`
* Enh #2594: Speed up the `restore --verify` command
+ * Enh #2656: Add flag to disable TLS verification for self-signed certificates
* Enh #2816: The `backup` command no longer updates file access times on Linux
* Enh #2880: Make `recover` collect only unreferenced trees
+ * Enh #3003: Atomic uploads for the SFTP backend
+ * Enh #3127: Add xattr (extended attributes) support for Solaris
* Enh #3429: Verify that new or modified keys are stored correctly
* Enh #3436: Improve local backend's resilience to (system) crashes
+ * Enh #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
+ * Enh #3490: Support random subset by size in `check --read-data-subset`
* Enh #3508: Cache blobs read by the `dump` command
* Enh #3511: Support configurable timeout for the rclone backend
+ * Enh #3541: Improve handling of temporary B2 delete errors
+ * Enh #3542: Add file mode in symbolic notation to `ls --json`
* Enh #3593: Improve `copy` performance by parallelizing IO
Details
@@ -546,6 +1102,16 @@ Details
https://github.com/restic/restic/issues/3382
https://github.com/restic/restic/pull/3474
+ * Bugfix #3488: `rebuild-index` failed if an index file was damaged
+
+ Previously, the `rebuild-index` command would fail with an error if an index file was damaged
+ or truncated. This has now been fixed.
+
+ On older restic versions, a (slow) workaround is to use `rebuild-index --read-all-packs` or
+ to manually delete the damaged index.
+
+ https://github.com/restic/restic/pull/3488
+
* Bugfix #3518: Make `copy` command honor `--no-lock` for source repository
The `copy` command previously did not respect the `--no-lock` option for the source
@@ -566,6 +1132,15 @@ Details
https://github.com/restic/restic/issues/2355
https://github.com/restic/restic/pull/3571
+ * Bugfix #3591: Fix handling of `prune --max-repack-size=0`
+
+ Restic ignored the `--max-repack-size` option when passing a value of 0. This has now been
+ fixed.
+
+ As a workaround, `--max-repack-size=1` can be used with older versions of restic.
+
+ https://github.com/restic/restic/pull/3591
+
* Bugfix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows
Previously, pressing Ctrl+C in a Windows console where restic was running with rclone as the
@@ -580,33 +1155,6 @@ Details
https://github.com/restic/restic/issues/3601
https://github.com/restic/restic/pull/3602
- * Bugfix #3667: The `mount` command now reports symlinks sizes
-
- Symlinks used to have size zero in restic mountpoints, confusing some third-party tools. They
- now have a size equal to the byte length of their target path, as required by POSIX.
-
- https://github.com/restic/restic/issues/3667
- https://github.com/restic/restic/pull/3668
-
- * Bugfix #3488: `rebuild-index` failed if an index file was damaged
-
- Previously, the `rebuild-index` command would fail with an error if an index file was damaged
- or truncated. This has now been fixed.
-
- On older restic versions, a (slow) workaround is to use `rebuild-index --read-all-packs` or
- to manually delete the damaged index.
-
- https://github.com/restic/restic/pull/3488
-
- * Bugfix #3591: Fix handling of `prune --max-repack-size=0`
-
- Restic ignored the `--max-repack-size` option when passing a value of 0. This has now been
- fixed.
-
- As a workaround, `--max-repack-size=1` can be used with older versions of restic.
-
- https://github.com/restic/restic/pull/3591
-
* Bugfix #3619: Avoid choosing parent snapshots newer than time of new snapshot
The `backup` command, when a `--parent` was not provided, previously chose the most recent
@@ -618,6 +1166,21 @@ Details
https://github.com/restic/restic/pull/3619
+ * Bugfix #3667: The `mount` command now reports symlinks sizes
+
+ Symlinks used to have size zero in restic mountpoints, confusing some third-party tools. They
+ now have a size equal to the byte length of their target path, as required by POSIX.
+
+ https://github.com/restic/restic/issues/3667
+ https://github.com/restic/restic/pull/3668
+
+ * Change #3519: Require Go 1.14 or newer
+
+ Restic now requires Go 1.14 to build. This allows it to use new standard library features
+ instead of an external dependency.
+
+ https://github.com/restic/restic/issues/3519
+
* Change #3641: Ignore parent snapshot for `backup --stdin`
Restic uses a parent snapshot to speed up directory scanning when performing backups, but this
@@ -632,12 +1195,22 @@ Details
https://github.com/restic/restic/issues/3641
https://github.com/restic/restic/pull/3645
- * Change #3519: Require Go 1.14 or newer
+ * Enhancement #233: Support negative include/exclude patterns
- Restic now requires Go 1.14 to build. This allows it to use new standard library features
- instead of an external dependency.
+ If a pattern starts with an exclamation mark and it matches a file that was previously matched by
+ a regular pattern, the match is cancelled. Notably, this can be used with `--exclude-file` to
+ cancel the exclusion of some files.
- https://github.com/restic/restic/issues/3519
+ It works similarly to `.gitignore`, with the same limitation; Once a directory is excluded, it
+ is not possible to include files inside the directory.
+
+ Example of use as an exclude pattern for the `backup` command:
+
+ $HOME/**/* !$HOME/Documents !$HOME/code !$HOME/.emacs.d !$HOME/games # [...]
+ node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git # [...]
+
+ https://github.com/restic/restic/issues/233
+ https://github.com/restic/restic/pull/2311
* Enhancement #1542: Add `--dry-run`/`-n` option to `backup` command
@@ -672,23 +1245,6 @@ Details
https://github.com/restic/restic/issues/3023
https://github.com/restic/restic/pull/3246
- * Enhancement #233: Support negative include/exclude patterns
-
- If a pattern starts with an exclamation mark and it matches a file that was previously matched by
- a regular pattern, the match is cancelled. Notably, this can be used with `--exclude-file` to
- cancel the exclusion of some files.
-
- It works similarly to `.gitignore`, with the same limitation; Once a directory is excluded, it
- is not possible to include files inside the directory.
-
- Example of use as an exclude pattern for the `backup` command:
-
- $HOME/**/* !$HOME/Documents !$HOME/code !$HOME/.emacs.d !$HOME/games # [...]
- node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git # [...]
-
- https://github.com/restic/restic/issues/233
- https://github.com/restic/restic/pull/2311
-
* Enhancement #2388: Add warning for S3 if partial credentials are provided
Previously restic did not notify about incomplete credentials when using the S3 backend,
@@ -709,6 +1265,13 @@ Details
https://github.com/restic/restic/issues/2508
https://github.com/restic/restic/pull/3592
+ * Enhancement #2594: Speed up the `restore --verify` command
+
+ The `--verify` option lets the `restore` command verify the file content after it has restored
+ a snapshot. The performance of this operation has now been improved by up to a factor of two.
+
+ https://github.com/restic/restic/pull/2594
+
* Enhancement #2656: Add flag to disable TLS verification for self-signed certificates
There is now an `--insecure-tls` global option in restic, which disables TLS verification for
@@ -717,67 +1280,6 @@ Details
https://github.com/restic/restic/issues/2656
https://github.com/restic/restic/pull/2657
- * Enhancement #3003: Atomic uploads for the SFTP backend
-
- The SFTP backend did not upload files atomically. An interrupted upload could leave an
- incomplete file behind which could prevent restic from accessing the repository. This has now
- been fixed and uploads in the SFTP backend are done atomically.
-
- https://github.com/restic/restic/issues/3003
- https://github.com/restic/restic/pull/3524
-
- * Enhancement #3127: Add xattr (extended attributes) support for Solaris
-
- Restic now supports xattr for the Solaris operating system.
-
- https://github.com/restic/restic/issues/3127
- https://github.com/restic/restic/pull/3628
-
- * Enhancement #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
-
- Restic used to silently ignore the `--no-lock` option of the `forget` command.
-
- It now skips creation of lock file in case both `--dry-run` and `--no-lock` are specified. If
- `--no-lock` option is specified without `--dry-run`, restic prints a warning message to
- stderr.
-
- https://github.com/restic/restic/issues/3464
- https://github.com/restic/restic/pull/3623
-
- * Enhancement #3490: Support random subset by size in `check --read-data-subset`
-
- The `--read-data-subset` option of the `check` command now supports a third way of specifying
- the subset to check, namely `nS` where `n` is a size in bytes with suffix `S` as k/K, m/M, g/G or
- t/T.
-
- https://github.com/restic/restic/issues/3490
- https://github.com/restic/restic/pull/3548
-
- * Enhancement #3541: Improve handling of temporary B2 delete errors
-
- Deleting files on B2 could sometimes fail temporarily, which required restic to retry the
- delete operation. In some cases the file was deleted nevertheless, causing the retries and
- ultimately the restic command to fail. This has now been fixed.
-
- https://github.com/restic/restic/issues/3541
- https://github.com/restic/restic/pull/3544
-
- * Enhancement #3542: Add file mode in symbolic notation to `ls --json`
-
- The `ls --json` command now provides the file mode in symbolic notation (using the
- `permissions` key), aligned with `find --json`.
-
- https://github.com/restic/restic/issues/3542
- https://github.com/restic/restic/pull/3573
- https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371
-
- * Enhancement #2594: Speed up the `restore --verify` command
-
- The `--verify` option lets the `restore` command verify the file content after it has restored
- a snapshot. The performance of this operation has now been improved by up to a factor of two.
-
- https://github.com/restic/restic/pull/2594
-
* Enhancement #2816: The `backup` command no longer updates file access times on Linux
When reading files during backup, restic used to cause the operating system to update the
@@ -799,6 +1301,22 @@ Details
https://github.com/restic/restic/pull/2880
+ * Enhancement #3003: Atomic uploads for the SFTP backend
+
+ The SFTP backend did not upload files atomically. An interrupted upload could leave an
+ incomplete file behind which could prevent restic from accessing the repository. This has now
+ been fixed and uploads in the SFTP backend are done atomically.
+
+ https://github.com/restic/restic/issues/3003
+ https://github.com/restic/restic/pull/3524
+
+ * Enhancement #3127: Add xattr (extended attributes) support for Solaris
+
+ Restic now supports xattr for the Solaris operating system.
+
+ https://github.com/restic/restic/issues/3127
+ https://github.com/restic/restic/pull/3628
+
* Enhancement #3429: Verify that new or modified keys are stored correctly
When adding a new key or changing the password of a key, restic used to just create the new key (and
@@ -823,6 +1341,26 @@ Details
https://github.com/restic/restic/pull/3436
+ * Enhancement #3464: Skip lock creation on `forget` if `--no-lock` and `--dry-run`
+
+ Restic used to silently ignore the `--no-lock` option of the `forget` command.
+
+ It now skips creation of lock file in case both `--dry-run` and `--no-lock` are specified. If
+ `--no-lock` option is specified without `--dry-run`, restic prints a warning message to
+ stderr.
+
+ https://github.com/restic/restic/issues/3464
+ https://github.com/restic/restic/pull/3623
+
+ * Enhancement #3490: Support random subset by size in `check --read-data-subset`
+
+ The `--read-data-subset` option of the `check` command now supports a third way of specifying
+ the subset to check, namely `nS` where `n` is a size in bytes with suffix `S` as k/K, m/M, g/G or
+ t/T.
+
+ https://github.com/restic/restic/issues/3490
+ https://github.com/restic/restic/pull/3548
+
* Enhancement #3508: Cache blobs read by the `dump` command
When dumping a file using the `dump` command, restic did not cache blobs in any way, so even
@@ -842,6 +1380,24 @@ Details
https://github.com/restic/restic/issues/3511
https://github.com/restic/restic/pull/3514
+ * Enhancement #3541: Improve handling of temporary B2 delete errors
+
+ Deleting files on B2 could sometimes fail temporarily, which required restic to retry the
+ delete operation. In some cases the file was deleted nevertheless, causing the retries and
+ ultimately the restic command to fail. This has now been fixed.
+
+ https://github.com/restic/restic/issues/3541
+ https://github.com/restic/restic/pull/3544
+
+ * Enhancement #3542: Add file mode in symbolic notation to `ls --json`
+
+ The `ls --json` command now provides the file mode in symbolic notation (using the
+ `permissions` key), aligned with `find --json`.
+
+ https://github.com/restic/restic/issues/3542
+ https://github.com/restic/restic/pull/3573
+ https://forum.restic.net/t/restic-ls-understanding-file-mode-with-json/4371
+
* Enhancement #3593: Improve `copy` performance by parallelizing IO
Restic copy previously only used a single thread for copying blobs between repositories,
@@ -864,26 +1420,26 @@ Summary
* Fix #2742: Improve error handling for rclone and REST backend over HTTP2
* Fix #3111: Fix terminal output redirection for PowerShell
+ * Fix #3184: `backup --quiet` no longer prints status information
* Fix #3214: Treat an empty password as a fatal error for repository init
* Fix #3267: `copy` failed to copy snapshots in rare cases
- * Fix #3184: `backup --quiet` no longer prints status information
* Fix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
* Fix #3302: Fix `fdopendir: not a directory` error for local backend
+ * Fix #3305: Fix possibly missing backup summary of JSON output in case of error
* Fix #3334: Print `created new cache` message only on a terminal
* Fix #3380: Fix crash of `backup --exclude='**'`
- * Fix #3305: Fix possibly missing backup summary of JSON output in case of error
* Fix #3439: Correctly handle download errors during `restore`
* Chg #3247: Empty files now have size of 0 in `ls --json` output
* Enh #2780: Add release binaries for s390x architecture on Linux
+ * Enh #3167: Allow specifying limit of `snapshots` list
* Enh #3293: Add `--repository-file2` option to `init` and `copy` command
* Enh #3312: Add auto-completion support for fish
* Enh #3336: SFTP backend now checks for disk space
* Enh #3377: Add release binaries for Apple Silicon
* Enh #3414: Add `--keep-within-hourly` option to restic forget
- * Enh #3456: Support filtering and specifying untagged snapshots
- * Enh #3167: Allow specifying limit of `snapshots` list
* Enh #3426: Optimize read performance of mount command
* Enh #3427: `find --pack` fallback to index if data file is missing
+ * Enh #3456: Support filtering and specifying untagged snapshots
Details
-------
@@ -907,10 +1463,21 @@ Details
When redirecting the output of restic using PowerShell on Windows, the output contained
terminal escape characters. This has been fixed by properly detecting the terminal type.
- In addition, the mintty terminal now shows progress output for the backup command.
+ In addition, the mintty terminal now shows progress output for the backup command.
+
+ https://github.com/restic/restic/issues/3111
+ https://github.com/restic/restic/pull/3325
+
+ * Bugfix #3184: `backup --quiet` no longer prints status information
+
+ A regression in the latest restic version caused the output of `backup --quiet` to contain
+ large amounts of backup progress information when run using an interactive terminal. This is
+ fixed now.
+
+ A workaround for this bug is to run restic as follows: `restic backup --quiet [..] | cat -`.
- https://github.com/restic/restic/issues/3111
- https://github.com/restic/restic/pull/3325
+ https://github.com/restic/restic/issues/3184
+ https://github.com/restic/restic/pull/3186
* Bugfix #3214: Treat an empty password as a fatal error for repository init
@@ -933,17 +1500,6 @@ Details
https://github.com/restic/restic/issues/3267
https://github.com/restic/restic/pull/3310
- * Bugfix #3184: `backup --quiet` no longer prints status information
-
- A regression in the latest restic version caused the output of `backup --quiet` to contain
- large amounts of backup progress information when run using an interactive terminal. This is
- fixed now.
-
- A workaround for this bug is to run restic as follows: `restic backup --quiet [..] | cat -`.
-
- https://github.com/restic/restic/issues/3184
- https://github.com/restic/restic/pull/3186
-
* Bugfix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository
The command `restic check --read-data-subset=x%` crashed when run for an empty repository.
@@ -960,6 +1516,13 @@ Details
https://github.com/restic/restic/issues/3302
https://github.com/restic/restic/pull/3308
+ * Bugfix #3305: Fix possibly missing backup summary of JSON output in case of error
+
+ When using `--json` output it happened from time to time that the summary output was missing in
+ case an error occurred. This has been fixed.
+
+ https://github.com/restic/restic/pull/3305
+
* Bugfix #3334: Print `created new cache` message only on a terminal
The message `created new cache` was printed even when the output wasn't a terminal. That broke
@@ -977,13 +1540,6 @@ Details
https://github.com/restic/restic/issues/3380
https://github.com/restic/restic/pull/3393
- * Bugfix #3305: Fix possibly missing backup summary of JSON output in case of error
-
- When using `--json` output it happened from time to time that the summary output was missing in
- case an error occurred. This has been fixed.
-
- https://github.com/restic/restic/pull/3305
-
* Bugfix #3439: Correctly handle download errors during `restore`
Due to a regression in restic 0.12.0, the `restore` command in some cases did not retry download
@@ -1007,6 +1563,16 @@ Details
https://github.com/restic/restic/issues/2780
https://github.com/restic/restic/pull/3452
+ * Enhancement #3167: Allow specifying limit of `snapshots` list
+
+ The `--last` option allowed limiting the output of the `snapshots` command to the latest
+ snapshot for each host. The new `--latest n` option allows limiting the output to the latest `n`
+ snapshots.
+
+ This change deprecates the option `--last` in favour of `--latest 1`.
+
+ https://github.com/restic/restic/pull/3167
+
* Enhancement #3293: Add `--repository-file2` option to `init` and `copy` command
The `init` and `copy` command can now be used with the `--repository-file2` option or the
@@ -1058,26 +1624,6 @@ Details
https://github.com/restic/restic/pull/3416
https://forum.restic.net/t/forget-policy/4014/11
- * Enhancement #3456: Support filtering and specifying untagged snapshots
-
- It was previously not possible to specify an empty tag with the `--tag` and `--keep-tag`
- options. This has now been fixed, such that `--tag ''` and `--keep-tag ''` now matches
- snapshots without tags. This allows e.g. the `snapshots` and `forget` commands to only
- operate on untagged snapshots.
-
- https://github.com/restic/restic/issues/3456
- https://github.com/restic/restic/pull/3457
-
- * Enhancement #3167: Allow specifying limit of `snapshots` list
-
- The `--last` option allowed limiting the output of the `snapshots` command to the latest
- snapshot for each host. The new `--latest n` option allows limiting the output to the latest `n`
- snapshots.
-
- This change deprecates the option `--last` in favour of `--latest 1`.
-
- https://github.com/restic/restic/pull/3167
-
* Enhancement #3426: Optimize read performance of mount command
Reading large files in a mounted repository may be up to five times faster. This improvement
@@ -1098,6 +1644,16 @@ Details
https://github.com/restic/restic/pull/3427
https://forum.restic.net/t/missing-packs-not-found/2600
+ * Enhancement #3456: Support filtering and specifying untagged snapshots
+
+ It was previously not possible to specify an empty tag with the `--tag` and `--keep-tag`
+ options. This has now been fixed, such that `--tag ''` and `--keep-tag ''` now matches
+ snapshots without tags. This allows e.g. the `snapshots` and `forget` commands to only
+ operate on untagged snapshots.
+
+ https://github.com/restic/restic/issues/3456
+ https://github.com/restic/restic/pull/3457
+
Changelog for restic 0.12.0 (2021-02-14)
=======================================
@@ -1113,35 +1669,35 @@ Summary
* Fix #2563: Report the correct owner of directories in FUSE mounts
* Fix #2688: Make `backup` and `tag` commands separate tags by comma
* Fix #2739: Make the `cat` command respect the `--no-lock` option
+ * Fix #3014: Fix sporadic stream reset between rclone and restic
* Fix #3087: The `--use-fs-snapshot` option now works on windows/386
* Fix #3100: Do not require gs bucket permissions when running `init`
* Fix #3111: Correctly detect output redirection for `backup` command on Windows
* Fix #3151: Don't create invalid snapshots when `backup` is interrupted
+ * Fix #3152: Do not hang until foregrounded when completed in background
* Fix #3166: Improve error handling in the `restore` command
* Fix #3232: Correct statistics for overlapping targets
- * Fix #3014: Fix sporadic stream reset between rclone and restic
- * Fix #3152: Do not hang until foregrounded when completed in background
* Fix #3249: Improve error handling in `gs` backend
* Chg #3095: Deleting files on Google Drive now moves them to the trash
+ * Enh #909: Back up mountpoints as empty directories
* Enh #2186: Allow specifying percentage in `check --read-data-subset`
+ * Enh #2433: Make the `dump` command support `zip` format
* Enh #2453: Report permanent/fatal backend errors earlier
+ * Enh #2495: Add option to let `backup` trust mtime without checking ctime
* Enh #2528: Add Alibaba/Aliyun OSS support in the `s3` backend
* Enh #2706: Configurable progress reports for non-interactive terminals
- * Enh #2944: Add `backup` options `--files-from-{verbatim,raw}`
- * Enh #3083: Allow usage of deprecated S3 `ListObjects` API
- * Enh #3147: Support additional environment variables for Swift authentication
- * Enh #3191: Add release binaries for MIPS architectures
- * Enh #909: Back up mountpoints as empty directories
- * Enh #3250: Add several more error checks
* Enh #2718: Improve `prune` performance and make it more customizable
- * Enh #2495: Add option to let `backup` trust mtime without checking ctime
* Enh #2941: Speed up the repacking step of the `prune` command
+ * Enh #2944: Add `backup` options `--files-from-{verbatim,raw}`
* Enh #3006: Speed up the `rebuild-index` command
* Enh #3048: Add more checks for index and pack files in the `check` command
- * Enh #2433: Make the `dump` command support `zip` format
+ * Enh #3083: Allow usage of deprecated S3 `ListObjects` API
* Enh #3099: Reduce memory usage of `check` command
* Enh #3106: Parallelize scan of snapshot content in `copy` and `prune`
* Enh #3130: Parallelize reading of locks and snapshots
+ * Enh #3147: Support additional environment variables for Swift authentication
+ * Enh #3191: Add release binaries for MIPS architectures
+ * Enh #3250: Add several more error checks
* Enh #3254: Enable HTTP/2 for backend connections
Details
@@ -1214,6 +1770,20 @@ Details
https://github.com/restic/restic/issues/2739
+ * Bugfix #3014: Fix sporadic stream reset between rclone and restic
+
+ Sometimes when using restic with the `rclone` backend, an error message similar to the
+ following would be printed:
+
+ Didn't finish writing GET request (wrote 0/xxx): http2: stream closed
+
+ It was found that this was caused by restic closing the connection to rclone to soon when
+ downloading data. A workaround has been added which waits for the end of the download before
+ closing the connection.
+
+ https://github.com/rclone/rclone/issues/2598
+ https://github.com/restic/restic/pull/3014
+
* Bugfix #3087: The `--use-fs-snapshot` option now works on windows/386
Restic failed to create VSS snapshots on windows/386 with the following error:
@@ -1253,6 +1823,15 @@ Details
https://github.com/restic/restic/issues/3151
https://github.com/restic/restic/pull/3164
+ * Bugfix #3152: Do not hang until foregrounded when completed in background
+
+ On Linux, when running in the background restic failed to stop the terminal output of the
+ `backup` command after it had completed. This caused restic to hang until moved to the
+ foreground. This has now been fixed.
+
+ https://github.com/restic/restic/pull/3152
+ https://forum.restic.net/t/restic-alpine-container-cron-hangs-epoll-pwait/3334
+
* Bugfix #3166: Improve error handling in the `restore` command
The `restore` command used to not print errors while downloading file contents from the
@@ -1274,29 +1853,6 @@ Details
https://github.com/restic/restic/issues/3232
https://github.com/restic/restic/pull/3243
- * Bugfix #3014: Fix sporadic stream reset between rclone and restic
-
- Sometimes when using restic with the `rclone` backend, an error message similar to the
- following would be printed:
-
- Didn't finish writing GET request (wrote 0/xxx): http2: stream closed
-
- It was found that this was caused by restic closing the connection to rclone to soon when
- downloading data. A workaround has been added which waits for the end of the download before
- closing the connection.
-
- https://github.com/rclone/rclone/issues/2598
- https://github.com/restic/restic/pull/3014
-
- * Bugfix #3152: Do not hang until foregrounded when completed in background
-
- On Linux, when running in the background restic failed to stop the terminal output of the
- `backup` command after it had completed. This caused restic to hang until moved to the
- foreground. This has now been fixed.
-
- https://github.com/restic/restic/pull/3152
- https://forum.restic.net/t/restic-alpine-container-cron-hangs-epoll-pwait/3334
-
* Bugfix #3249: Improve error handling in `gs` backend
The `gs` backend did not notice when the last step of completing a file upload failed. Under rare
@@ -1321,6 +1877,18 @@ Details
https://github.com/restic/restic/issues/3095
https://github.com/restic/restic/pull/3102
+ * Enhancement #909: Back up mountpoints as empty directories
+
+ When the `--one-file-system` option is specified to `restic backup`, it ignores all file
+ systems mounted below one of the target directories. This means that when a snapshot is
+ restored, users needed to manually recreate the mountpoint directories.
+
+ Restic now backs up mountpoints as empty directories and therefore implements the same
+ approach as `tar`.
+
+ https://github.com/restic/restic/issues/909
+ https://github.com/restic/restic/pull/3119
+
* Enhancement #2186: Allow specifying percentage in `check --read-data-subset`
We've enhanced the `check` command's `--read-data-subset` option to also accept a
@@ -1330,6 +1898,15 @@ Details
https://github.com/restic/restic/issues/2186
https://github.com/restic/restic/pull/3038
+ * Enhancement #2433: Make the `dump` command support `zip` format
+
+ Previously, restic could dump the contents of a whole folder structure only in the `tar`
+ format. The `dump` command now has a new flag to change output format to `zip`. Just pass
+ `--archive zip` as an option to `restic dump`.
+
+ https://github.com/restic/restic/pull/2433
+ https://github.com/restic/restic/pull/3081
+
* Enhancement #2453: Report permanent/fatal backend errors earlier
When encountering errors in reading from or writing to storage backends, restic retries the
@@ -1345,6 +1922,31 @@ Details
https://github.com/restic/restic/pull/3170
https://github.com/restic/restic/pull/3181
+ * Enhancement #2495: Add option to let `backup` trust mtime without checking ctime
+
+ The `backup` command used to require that both `ctime` and `mtime` of a file matched with a
+ previously backed up version to determine that the file was unchanged. In other words, if
+ either `ctime` or `mtime` of the file had changed, it would be considered changed and restic
+ would read the file's content again to back up the relevant (changed) parts of it.
+
+ The new option `--ignore-ctime` makes restic look at `mtime` only, such that `ctime` changes
+ for a file does not cause restic to read the file's contents again.
+
+ The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make backups more
+ reliable in the face of programs that reset `mtime` (some Unix archivers do that), but it turned
+ out to often be expensive because it made restic read file contents even if only the metadata
+ (owner, permissions) of a file had changed. The new `--ignore-ctime` option lets the user
+ restore the 0.9.5 behavior when needed. The existing `--ignore-inode` option already turned
+ off this behavior, but also removed a different check.
+
+ Please note that changes in files' metadata are still recorded, regardless of the command line
+ options provided to the backup command.
+
+ https://github.com/restic/restic/issues/2495
+ https://github.com/restic/restic/issues/2558
+ https://github.com/restic/restic/issues/2819
+ https://github.com/restic/restic/pull/2823
+
* Enhancement #2528: Add Alibaba/Aliyun OSS support in the `s3` backend
A new extended option `s3.bucket-lookup` has been added to support Alibaba/Aliyun OSS in the
@@ -1382,76 +1984,6 @@ Details
https://github.com/restic/restic/issues/3194
https://github.com/restic/restic/pull/3199
- * Enhancement #2944: Add `backup` options `--files-from-{verbatim,raw}`
-
- The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a list of
- files to back up from a file. Unlike the existing `--files-from` option, these options do not
- interpret the listed filenames as glob patterns; instead, whitespace in filenames is
- preserved as-is and no pattern expansion is done. Please see the documentation for specifics.
-
- These new options are highly recommended over `--files-from`, when using a script to generate
- the list of files to back up.
-
- https://github.com/restic/restic/issues/2944
- https://github.com/restic/restic/issues/3013
-
- * Enhancement #3083: Allow usage of deprecated S3 `ListObjects` API
-
- Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken `ListObjectsV2`
- implementation which causes problems for restic when using their API endpoints. When a broken
- server implementation is used, restic prints errors similar to the following:
-
- List() returned error: Truncated response should have continuation token set
-
- As a temporary workaround, restic now allows using the older `ListObjects` endpoint by
- setting the `s3.list-objects-v1` extended option, for instance:
-
- Restic -o s3.list-objects-v1=true snapshots
-
- Please note that this option may be removed in future versions of restic.
-
- https://github.com/restic/restic/issues/3083
- https://github.com/restic/restic/pull/3085
-
- * Enhancement #3147: Support additional environment variables for Swift authentication
-
- The `swift` backend now supports the following additional environment variables for passing
- authentication details to restic: `OS_USER_ID`, `OS_USER_DOMAIN_ID`,
- `OS_PROJECT_DOMAIN_ID` and `OS_TRUST_ID`
-
- Depending on the `openrc` configuration file these might be required when the user and project
- domains differ from one another.
-
- https://github.com/restic/restic/issues/3147
- https://github.com/restic/restic/pull/3158
-
- * Enhancement #3191: Add release binaries for MIPS architectures
-
- We've added a few new architectures for Linux to the release binaries: `mips`, `mipsle`,
- `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded systems.
-
- https://github.com/restic/restic/issues/3191
- https://github.com/restic/restic/pull/3208
-
- * Enhancement #909: Back up mountpoints as empty directories
-
- When the `--one-file-system` option is specified to `restic backup`, it ignores all file
- systems mounted below one of the target directories. This means that when a snapshot is
- restored, users needed to manually recreate the mountpoint directories.
-
- Restic now backs up mountpoints as empty directories and therefore implements the same
- approach as `tar`.
-
- https://github.com/restic/restic/issues/909
- https://github.com/restic/restic/pull/3119
-
- * Enhancement #3250: Add several more error checks
-
- We've added a lot more error checks in places where errors were previously ignored (as hinted by
- the static analysis program `errcheck` via `golangci-lint`).
-
- https://github.com/restic/restic/pull/3250
-
* Enhancement #2718: Improve `prune` performance and make it more customizable
The `prune` command is now much faster. This is especially the case for remote repositories or
@@ -1481,31 +2013,6 @@ Details
https://github.com/restic/restic/pull/2718
https://github.com/restic/restic/pull/2842
- * Enhancement #2495: Add option to let `backup` trust mtime without checking ctime
-
- The `backup` command used to require that both `ctime` and `mtime` of a file matched with a
- previously backed up version to determine that the file was unchanged. In other words, if
- either `ctime` or `mtime` of the file had changed, it would be considered changed and restic
- would read the file's content again to back up the relevant (changed) parts of it.
-
- The new option `--ignore-ctime` makes restic look at `mtime` only, such that `ctime` changes
- for a file does not cause restic to read the file's contents again.
-
- The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make backups more
- reliable in the face of programs that reset `mtime` (some Unix archivers do that), but it turned
- out to often be expensive because it made restic read file contents even if only the metadata
- (owner, permissions) of a file had changed. The new `--ignore-ctime` option lets the user
- restore the 0.9.5 behavior when needed. The existing `--ignore-inode` option already turned
- off this behavior, but also removed a different check.
-
- Please note that changes in files' metadata are still recorded, regardless of the command line
- options provided to the backup command.
-
- https://github.com/restic/restic/issues/2495
- https://github.com/restic/restic/issues/2558
- https://github.com/restic/restic/issues/2819
- https://github.com/restic/restic/pull/2823
-
* Enhancement #2941: Speed up the repacking step of the `prune` command
The repack step of the `prune` command, which moves still used file parts into new pack files
@@ -1515,6 +2022,19 @@ Details
https://github.com/restic/restic/pull/2941
+ * Enhancement #2944: Add `backup` options `--files-from-{verbatim,raw}`
+
+ The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a list of
+ files to back up from a file. Unlike the existing `--files-from` option, these options do not
+ interpret the listed filenames as glob patterns; instead, whitespace in filenames is
+ preserved as-is and no pattern expansion is done. Please see the documentation for specifics.
+
+ These new options are highly recommended over `--files-from`, when using a script to generate
+ the list of files to back up.
+
+ https://github.com/restic/restic/issues/2944
+ https://github.com/restic/restic/issues/3013
+
* Enhancement #3006: Speed up the `rebuild-index` command
We've optimized the `rebuild-index` command. Now, existing index entries are used to
@@ -1547,14 +2067,23 @@ Details
https://github.com/restic/restic/pull/3048
https://github.com/restic/restic/pull/3082
- * Enhancement #2433: Make the `dump` command support `zip` format
+ * Enhancement #3083: Allow usage of deprecated S3 `ListObjects` API
- Previously, restic could dump the contents of a whole folder structure only in the `tar`
- format. The `dump` command now has a new flag to change output format to `zip`. Just pass
- `--archive zip` as an option to `restic dump`.
+ Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken `ListObjectsV2`
+ implementation which causes problems for restic when using their API endpoints. When a broken
+ server implementation is used, restic prints errors similar to the following:
- https://github.com/restic/restic/pull/2433
- https://github.com/restic/restic/pull/3081
+ List() returned error: Truncated response should have continuation token set
+
+ As a temporary workaround, restic now allows using the older `ListObjects` endpoint by
+ setting the `s3.list-objects-v1` extended option, for instance:
+
+ Restic -o s3.list-objects-v1=true snapshots
+
+ Please note that this option may be removed in future versions of restic.
+
+ https://github.com/restic/restic/issues/3083
+ https://github.com/restic/restic/pull/3085
* Enhancement #3099: Reduce memory usage of `check` command
@@ -1586,6 +2115,33 @@ Details
https://github.com/restic/restic/pull/3130
https://github.com/restic/restic/pull/3174
+ * Enhancement #3147: Support additional environment variables for Swift authentication
+
+ The `swift` backend now supports the following additional environment variables for passing
+ authentication details to restic: `OS_USER_ID`, `OS_USER_DOMAIN_ID`,
+ `OS_PROJECT_DOMAIN_ID` and `OS_TRUST_ID`
+
+ Depending on the `openrc` configuration file these might be required when the user and project
+ domains differ from one another.
+
+ https://github.com/restic/restic/issues/3147
+ https://github.com/restic/restic/pull/3158
+
+ * Enhancement #3191: Add release binaries for MIPS architectures
+
+ We've added a few new architectures for Linux to the release binaries: `mips`, `mipsle`,
+ `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded systems.
+
+ https://github.com/restic/restic/issues/3191
+ https://github.com/restic/restic/pull/3208
+
+ * Enhancement #3250: Add several more error checks
+
+ We've added a lot more error checks in places where errors were previously ignored (as hinted by
+ the static analysis program `errcheck` via `golangci-lint`).
+
+ https://github.com/restic/restic/pull/3250
+
* Enhancement #3254: Enable HTTP/2 for backend connections
Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2 depending on
@@ -1615,10 +2171,10 @@ Summary
* Fix #2942: Make --exclude-larger-than handle disappearing files
* Fix #2951: Restic generate, help and self-update no longer check passwords
* Fix #2979: Make snapshots --json output [] instead of null when no snapshots
- * Enh #2969: Optimize check for unchanged files during backup
* Enh #340: Add support for Volume Shadow Copy Service (VSS) on Windows
- * Enh #2849: Authenticate to Google Cloud Storage with access token
* Enh #1458: New option --repository-file
+ * Enh #2849: Authenticate to Google Cloud Storage with access token
+ * Enh #2969: Optimize check for unchanged files during backup
* Enh #2978: Warn if parent snapshot cannot be loaded during backup
Details
@@ -1722,16 +2278,6 @@ Details
https://github.com/restic/restic/issues/2979
https://github.com/restic/restic/pull/2984
- * Enhancement #2969: Optimize check for unchanged files during backup
-
- During a backup restic skips processing files which have not changed since the last backup run.
- Previously this required opening each file once which can be slow on network filesystems. The
- backup command now checks for file changes before opening a file. This considerably reduces
- the time to create a backup on network filesystems.
-
- https://github.com/restic/restic/issues/2969
- https://github.com/restic/restic/pull/2970
-
* Enhancement #340: Add support for Volume Shadow Copy Service (VSS) on Windows
Volume Shadow Copy Service allows read access to files that are locked by another process using
@@ -1744,13 +2290,6 @@ Details
https://github.com/restic/restic/issues/340
https://github.com/restic/restic/pull/2274
- * Enhancement #2849: Authenticate to Google Cloud Storage with access token
-
- When using the GCS backend, it is now possible to authenticate with OAuth2 access tokens
- instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN environment variable.
-
- https://github.com/restic/restic/pull/2849
-
* Enhancement #1458: New option --repository-file
We've added a new command-line option --repository-file as an alternative to -r. This allows
@@ -1761,6 +2300,23 @@ Details
https://github.com/restic/restic/issues/2900
https://github.com/restic/restic/pull/2910
+ * Enhancement #2849: Authenticate to Google Cloud Storage with access token
+
+ When using the GCS backend, it is now possible to authenticate with OAuth2 access tokens
+ instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN environment variable.
+
+ https://github.com/restic/restic/pull/2849
+
+ * Enhancement #2969: Optimize check for unchanged files during backup
+
+ During a backup restic skips processing files which have not changed since the last backup run.
+ Previously this required opening each file once which can be slow on network filesystems. The
+ backup command now checks for file changes before opening a file. This considerably reduces
+ the time to create a backup on network filesystems.
+
+ https://github.com/restic/restic/issues/2969
+ https://github.com/restic/restic/pull/2970
+
* Enhancement #2978: Warn if parent snapshot cannot be loaded during backup
During a backup restic uses the parent snapshot to check whether a file was changed and has to be
@@ -1796,25 +2352,24 @@ Summary
* Fix #2668: Don't abort the stats command when data blobs are missing
* Fix #2674: Add stricter prune error checks
* Fix #2899: Fix possible crash in the progress bar of check --read-data
+ * Chg #1597: Honor the --no-lock flag in the mount command
* Chg #2482: Remove vendored dependencies
* Chg #2546: Return exit code 3 when failing to backup all source data
* Chg #2600: Update dependencies, require Go >= 1.13
- * Chg #1597: Honor the --no-lock flag in the mount command
+ * Enh #323: Add command for copying snapshots between repositories
+ * Enh #551: Use optimized library for hash calculation of file chunks
* Enh #1570: Support specifying multiple host flags for various commands
* Enh #1680: Optimize `restic mount`
* Enh #2072: Display snapshot date when using `restic find`
* Enh #2175: Allow specifying user and host when creating keys
+ * Enh #2195: Simplify and improve restore performance
* Enh #2277: Add support for ppc64le
+ * Enh #2328: Improve speed of check command
* Enh #2395: Ignore sync errors when operation not supported by local filesystem
+ * Enh #2423: Support user@domain parsing as user
* Enh #2427: Add flag `--iexclude-file` to backup command
* Enh #2569: Support excluding files by their size
* Enh #2571: Self-heal missing file parts during backup of unchanged files
- * Enh #2858: Support filtering snapshots by tag and path in the stats command
- * Enh #323: Add command for copying snapshots between repositories
- * Enh #551: Use optimized library for hash calculation of file chunks
- * Enh #2195: Simplify and improve restore performance
- * Enh #2328: Improve speed of check command
- * Enh #2423: Support user@domain parsing as user
* Enh #2576: Improve the chunking algorithm
* Enh #2598: Improve speed of diff command
* Enh #2599: Slightly reduce memory usage of prune and stats commands
@@ -1824,6 +2379,7 @@ Summary
* Enh #2786: Optimize `list blobs` command
* Enh #2790: Optimized file access in restic mount
* Enh #2840: Speed-up file deletion in forget, prune and rebuild-index
+ * Enh #2858: Support filtering snapshots by tag and path in the stats command
Details
-------
@@ -1955,6 +2511,14 @@ Details
https://github.com/restic/restic/pull/2899
https://forum.restic.net/t/restic-rclone-pcloud-connection-issues/2963/15
+ * Change #1597: Honor the --no-lock flag in the mount command
+
+ The mount command now does not lock the repository if given the --no-lock flag. This allows to
+ mount repositories which are archived on a read only backend/filesystem.
+
+ https://github.com/restic/restic/issues/1597
+ https://github.com/restic/restic/pull/2821
+
* Change #2482: Remove vendored dependencies
We've removed the vendored dependencies (in the subdir `vendor/`). When building restic, the
@@ -1978,128 +2542,25 @@ Details
some source data could not be read (incomplete snapshot created)
https://github.com/restic/restic/issues/956
- https://github.com/restic/restic/issues/2064
- https://github.com/restic/restic/issues/2526
- https://github.com/restic/restic/issues/2364
- https://github.com/restic/restic/pull/2546
-
- * Change #2600: Update dependencies, require Go >= 1.13
-
- Restic now requires Go to be at least 1.13. This allows simplifications in the build process and
- removing workarounds.
-
- This is also probably the last version of restic still supporting mounting repositories via
- fuse on macOS. The library we're using for fuse does not support macOS any more and osxfuse is not
- open source any more.
-
- https://github.com/bazil/fuse/issues/224
- https://github.com/osxfuse/osxfuse/issues/590
- https://github.com/restic/restic/pull/2600
- https://github.com/restic/restic/pull/2852
- https://github.com/restic/restic/pull/2927
-
- * Change #1597: Honor the --no-lock flag in the mount command
-
- The mount command now does not lock the repository if given the --no-lock flag. This allows to
- mount repositories which are archived on a read only backend/filesystem.
-
- https://github.com/restic/restic/issues/1597
- https://github.com/restic/restic/pull/2821
-
- * Enhancement #1570: Support specifying multiple host flags for various commands
-
- Previously commands didn't take more than one `--host` or `-H` argument into account, which
- could be limiting with e.g. the `forget` command.
-
- The `dump`, `find`, `forget`, `ls`, `mount`, `restore`, `snapshots`, `stats` and `tag`
- commands will now take into account multiple `--host` and `-H` flags.
-
- https://github.com/restic/restic/issues/1570
-
- * Enhancement #1680: Optimize `restic mount`
-
- We've optimized the FUSE implementation used within restic. `restic mount` is now more
- responsive and uses less memory.
-
- https://github.com/restic/restic/issues/1680
- https://github.com/restic/restic/pull/2587
- https://github.com/restic/restic/pull/2787
-
- * Enhancement #2072: Display snapshot date when using `restic find`
-
- Added the respective snapshot date to the output of `restic find`.
-
- https://github.com/restic/restic/issues/2072
-
- * Enhancement #2175: Allow specifying user and host when creating keys
-
- When adding a new key to the repository, the username and hostname for the new key can be
- specified on the command line. This allows overriding the defaults, for example if you would
- prefer to use the FQDN to identify the host or if you want to add keys for several different hosts
- without having to run the key add command on those hosts.
-
- https://github.com/restic/restic/issues/2175
-
- * Enhancement #2277: Add support for ppc64le
-
- Adds support for ppc64le, the processor architecture from IBM.
-
- https://github.com/restic/restic/issues/2277
-
- * Enhancement #2395: Ignore sync errors when operation not supported by local filesystem
-
- The local backend has been modified to work with filesystems which doesn't support the `sync`
- operation. This operation is normally used by restic to ensure that data files are fully
- written to disk before continuing.
-
- For these limited filesystems, saving a file in the backend would previously fail with an
- "operation not supported" error. This error is now ignored, which means that e.g. an SMB mount
- on macOS can now be used as storage location for a repository.
-
- https://github.com/restic/restic/issues/2395
- https://forum.restic.net/t/sync-errors-on-mac-over-smb/1859
-
- * Enhancement #2427: Add flag `--iexclude-file` to backup command
-
- The backup command now supports the flag `--iexclude-file` which is a case-insensitive
- version of `--exclude-file`.
-
- https://github.com/restic/restic/issues/2427
- https://github.com/restic/restic/pull/2898
-
- * Enhancement #2569: Support excluding files by their size
-
- The `backup` command now supports the `--exclude-larger-than` option to exclude files which
- are larger than the specified maximum size. This can for example be useful to exclude
- unimportant files with a large file size.
-
- https://github.com/restic/restic/issues/2569
- https://github.com/restic/restic/pull/2914
-
- * Enhancement #2571: Self-heal missing file parts during backup of unchanged files
-
- We've improved the resilience of restic to certain types of repository corruption.
-
- For files that are unchanged since the parent snapshot, the backup command now verifies that
- all parts of the files still exist in the repository. Parts that are missing, e.g. from a damaged
- repository, are backed up again. This verification was already run for files that were
- modified since the parent snapshot, but is now also done for unchanged files.
-
- Note that restic will not backup file parts that are referenced in the index but where the actual
- data is not present on disk, as this situation can only be detected by restic check. Please
- ensure that you run `restic check` regularly.
+ https://github.com/restic/restic/issues/2064
+ https://github.com/restic/restic/issues/2526
+ https://github.com/restic/restic/issues/2364
+ https://github.com/restic/restic/pull/2546
- https://github.com/restic/restic/issues/2571
- https://github.com/restic/restic/pull/2827
+ * Change #2600: Update dependencies, require Go >= 1.13
- * Enhancement #2858: Support filtering snapshots by tag and path in the stats command
+ Restic now requires Go to be at least 1.13. This allows simplifications in the build process and
+ removing workarounds.
- We've added filtering snapshots by `--tag tagList` and by `--path path` to the `stats`
- command. This includes filtering of only 'latest' snapshots or all snapshots in a repository.
+ This is also probably the last version of restic still supporting mounting repositories via
+ fuse on macOS. The library we're using for fuse does not support macOS any more and osxfuse is not
+ open source any more.
- https://github.com/restic/restic/issues/2858
- https://github.com/restic/restic/pull/2859
- https://forum.restic.net/t/stats-for-a-host-and-filtered-snapshots/3020
+ https://github.com/bazil/fuse/issues/224
+ https://github.com/osxfuse/osxfuse/issues/590
+ https://github.com/restic/restic/pull/2600
+ https://github.com/restic/restic/pull/2852
+ https://github.com/restic/restic/pull/2927
* Enhancement #323: Add command for copying snapshots between repositories
@@ -2130,6 +2591,40 @@ Details
https://github.com/restic/restic/issues/551
https://github.com/restic/restic/pull/2709
+ * Enhancement #1570: Support specifying multiple host flags for various commands
+
+ Previously commands didn't take more than one `--host` or `-H` argument into account, which
+ could be limiting with e.g. the `forget` command.
+
+ The `dump`, `find`, `forget`, `ls`, `mount`, `restore`, `snapshots`, `stats` and `tag`
+ commands will now take into account multiple `--host` and `-H` flags.
+
+ https://github.com/restic/restic/issues/1570
+
+ * Enhancement #1680: Optimize `restic mount`
+
+ We've optimized the FUSE implementation used within restic. `restic mount` is now more
+ responsive and uses less memory.
+
+ https://github.com/restic/restic/issues/1680
+ https://github.com/restic/restic/pull/2587
+ https://github.com/restic/restic/pull/2787
+
+ * Enhancement #2072: Display snapshot date when using `restic find`
+
+ Added the respective snapshot date to the output of `restic find`.
+
+ https://github.com/restic/restic/issues/2072
+
+ * Enhancement #2175: Allow specifying user and host when creating keys
+
+ When adding a new key to the repository, the username and hostname for the new key can be
+ specified on the command line. This allows overriding the defaults, for example if you would
+ prefer to use the FQDN to identify the host or if you want to add keys for several different hosts
+ without having to run the key add command on those hosts.
+
+ https://github.com/restic/restic/issues/2175
+
* Enhancement #2195: Simplify and improve restore performance
Significantly improves restore performance of large files (i.e. 50M+):
@@ -2153,6 +2648,12 @@ Details
https://github.com/restic/restic/pull/2195
https://github.com/restic/restic/pull/2893
+ * Enhancement #2277: Add support for ppc64le
+
+ Adds support for ppc64le, the processor architecture from IBM.
+
+ https://github.com/restic/restic/issues/2277
+
* Enhancement #2328: Improve speed of check command
We've improved the check command to traverse trees only once independent of whether they are
@@ -2162,12 +2663,58 @@ Details
https://github.com/restic/restic/issues/2284
https://github.com/restic/restic/pull/2328
+ * Enhancement #2395: Ignore sync errors when operation not supported by local filesystem
+
+ The local backend has been modified to work with filesystems which doesn't support the `sync`
+ operation. This operation is normally used by restic to ensure that data files are fully
+ written to disk before continuing.
+
+ For these limited filesystems, saving a file in the backend would previously fail with an
+ "operation not supported" error. This error is now ignored, which means that e.g. an SMB mount
+ on macOS can now be used as storage location for a repository.
+
+ https://github.com/restic/restic/issues/2395
+ https://forum.restic.net/t/sync-errors-on-mac-over-smb/1859
+
* Enhancement #2423: Support user@domain parsing as user
Added the ability for user@domain-like users to be authenticated over SFTP servers.
https://github.com/restic/restic/pull/2423
+ * Enhancement #2427: Add flag `--iexclude-file` to backup command
+
+ The backup command now supports the flag `--iexclude-file` which is a case-insensitive
+ version of `--exclude-file`.
+
+ https://github.com/restic/restic/issues/2427
+ https://github.com/restic/restic/pull/2898
+
+ * Enhancement #2569: Support excluding files by their size
+
+ The `backup` command now supports the `--exclude-larger-than` option to exclude files which
+ are larger than the specified maximum size. This can for example be useful to exclude
+ unimportant files with a large file size.
+
+ https://github.com/restic/restic/issues/2569
+ https://github.com/restic/restic/pull/2914
+
+ * Enhancement #2571: Self-heal missing file parts during backup of unchanged files
+
+ We've improved the resilience of restic to certain types of repository corruption.
+
+ For files that are unchanged since the parent snapshot, the backup command now verifies that
+ all parts of the files still exist in the repository. Parts that are missing, e.g. from a damaged
+ repository, are backed up again. This verification was already run for files that were
+ modified since the parent snapshot, but is now also done for unchanged files.
+
+ Note that restic will not backup file parts that are referenced in the index but where the actual
+ data is not present on disk, as this situation can only be detected by restic check. Please
+ ensure that you run `restic check` regularly.
+
+ https://github.com/restic/restic/issues/2571
+ https://github.com/restic/restic/pull/2827
+
* Enhancement #2576: Improve the chunking algorithm
We've updated the chunker library responsible for splitting files into smaller blocks. It
@@ -2234,6 +2781,15 @@ Details
https://github.com/restic/restic/pull/2840
+ * Enhancement #2858: Support filtering snapshots by tag and path in the stats command
+
+ We've added filtering snapshots by `--tag tagList` and by `--path path` to the `stats`
+ command. This includes filtering of only 'latest' snapshots or all snapshots in a repository.
+
+ https://github.com/restic/restic/issues/2858
+ https://github.com/restic/restic/pull/2859
+ https://forum.restic.net/t/stats-for-a-host-and-filtered-snapshots/3020
+
Changelog for restic 0.9.6 (2019-11-22)
=======================================
@@ -2353,11 +2909,11 @@ Summary
* Fix #2224: Don't abort the find command when a tree can't be loaded
* Enh #1895: Add case insensitive include & exclude options
* Enh #1937: Support streaming JSON output for backup
- * Enh #2155: Add Openstack application credential auth for Swift
- * Enh #2184: Add --json support to forget command
* Enh #2037: Add group-by option to snapshots command
* Enh #2124: Ability to dump folders to tar via stdout
* Enh #2139: Return error if no bytes could be read for `backup --stdin`
+ * Enh #2155: Add Openstack application credential auth for Swift
+ * Enh #2184: Add --json support to forget command
* Enh #2205: Add --ignore-inode option to backup cmd
* Enh #2220: Add config option to set S3 storage class
@@ -2411,22 +2967,6 @@ Details
https://github.com/restic/restic/issues/1937
https://github.com/restic/restic/pull/1944
- * Enhancement #2155: Add Openstack application credential auth for Swift
-
- Since Openstack Queens Identity (auth V3) service supports an application credential auth
- method. It allows to create a technical account with the limited roles. This commit adds an
- application credential authentication method for the Swift backend.
-
- https://github.com/restic/restic/issues/2155
-
- * Enhancement #2184: Add --json support to forget command
-
- The forget command now supports the --json argument, outputting the information about what is
- (or would-be) kept and removed from the repository.
-
- https://github.com/restic/restic/issues/2184
- https://github.com/restic/restic/pull/2185
-
* Enhancement #2037: Add group-by option to snapshots command
We have added an option to group the output of the snapshots command, similar to the output of the
@@ -2453,6 +2993,22 @@ Details
https://github.com/restic/restic/pull/2139
+ * Enhancement #2155: Add Openstack application credential auth for Swift
+
+ Since Openstack Queens Identity (auth V3) service supports an application credential auth
+ method. It allows to create a technical account with the limited roles. This commit adds an
+ application credential authentication method for the Swift backend.
+
+ https://github.com/restic/restic/issues/2155
+
+ * Enhancement #2184: Add --json support to forget command
+
+ The forget command now supports the --json argument, outputting the information about what is
+ (or would-be) kept and removed from the repository.
+
+ https://github.com/restic/restic/issues/2184
+ https://github.com/restic/restic/pull/2185
+
* Enhancement #2205: Add --ignore-inode option to backup cmd
This option handles backup of virtual filesystems that do not keep fixed inodes for files, like
@@ -2494,12 +3050,12 @@ Summary
* Fix #2068: Correctly return error loading data
* Fix #2095: Consistently use local time for snapshots times
* Enh #1605: Concurrent restore
- * Enh #2089: Increase granularity of the "keep within" retention policy
- * Enh #2097: Add key hinting
* Enh #2017: Mount: Enforce FUSE Unix permissions with allow-other
* Enh #2070: Make all commands display timestamps in local time
* Enh #2085: Allow --files-from to be specified multiple times
+ * Enh #2089: Increase granularity of the "keep within" retention policy
* Enh #2094: Run command to get password
+ * Enh #2097: Add key hinting
Details
-------
@@ -2552,26 +3108,6 @@ Details
https://github.com/restic/restic/issues/1605
https://github.com/restic/restic/pull/1719
- * Enhancement #2089: Increase granularity of the "keep within" retention policy
-
- The `keep-within` option of the `forget` command now accepts time ranges with an hourly
- granularity. For example, running `restic forget --keep-within 3d12h` will keep all the
- snapshots made within three days and twelve hours from the time of the latest snapshot.
-
- https://github.com/restic/restic/issues/2089
- https://github.com/restic/restic/pull/2090
-
- * Enhancement #2097: Add key hinting
-
- Added a new option `--key-hint` and corresponding environment variable `RESTIC_KEY_HINT`.
- The key hint is a key ID to try decrypting first, before other keys in the repository.
-
- This change will benefit repositories with many keys; if the correct key hint is supplied then
- restic only needs to check one key. If the key hint is incorrect (the key does not exist, or the
- password is incorrect) then restic will check all keys, as usual.
-
- https://github.com/restic/restic/issues/2097
-
* Enhancement #2017: Mount: Enforce FUSE Unix permissions with allow-other
The fuse mount (`restic mount`) now lets the kernel check the permissions of the files within
@@ -2598,6 +3134,15 @@ Details
https://github.com/restic/restic/issues/2085
https://github.com/restic/restic/pull/2086
+ * Enhancement #2089: Increase granularity of the "keep within" retention policy
+
+ The `keep-within` option of the `forget` command now accepts time ranges with an hourly
+ granularity. For example, running `restic forget --keep-within 3d12h` will keep all the
+ snapshots made within three days and twelve hours from the time of the latest snapshot.
+
+ https://github.com/restic/restic/issues/2089
+ https://github.com/restic/restic/pull/2090
+
* Enhancement #2094: Run command to get password
We've added the `--password-command` option which allows specifying a command that restic
@@ -2607,6 +3152,17 @@ Details
https://github.com/restic/restic/pull/2094
+ * Enhancement #2097: Add key hinting
+
+ Added a new option `--key-hint` and corresponding environment variable `RESTIC_KEY_HINT`.
+ The key hint is a key ID to try decrypting first, before other keys in the repository.
+
+ This change will benefit repositories with many keys; if the correct key hint is supplied then
+ restic only needs to check one key. If the key hint is incorrect (the key does not exist, or the
+ password is incorrect) then restic will check all keys, as usual.
+
+ https://github.com/restic/restic/issues/2097
+
Changelog for restic 0.9.3 (2018-10-13)
=======================================
@@ -2620,92 +3176,48 @@ Summary
* Fix #1935: Remove truncated files from cache
* Fix #1978: Do not return an error when the scanner is slower than backup
* Enh #1766: Restore: suppress lchown errors when not running as root
- * Enh #1909: Reject files/dirs by name first
- * Enh #1940: Add directory filter to ls command
- * Enh #1967: Use `--host` everywhere
- * Enh #2028: Display size of cache directories
* Enh #1777: Improve the `find` command
* Enh #1876: Display reason why forget keeps snapshots
* Enh #1891: Accept glob in paths loaded via --files-from
+ * Enh #1909: Reject files/dirs by name first
* Enh #1920: Vendor dependencies with Go 1.11 Modules
+ * Enh #1940: Add directory filter to ls command
* Enh #1949: Add new command `self-update`
* Enh #1953: Ls: Add JSON output support for restic ls cmd
* Enh #1962: Stream JSON output for ls command
+ * Enh #1967: Use `--host` everywhere
+ * Enh #2028: Display size of cache directories
-Details
--------
-
- * Bugfix #1935: Remove truncated files from cache
-
- When a file in the local cache is truncated, and restic tries to access data beyond the end of the
- (cached) file, it used to return an error "EOF". This is now fixed, such truncated files are
- removed and the data is fetched directly from the backend.
-
- https://github.com/restic/restic/issues/1935
-
- * Bugfix #1978: Do not return an error when the scanner is slower than backup
-
- When restic makes a backup, there's a background task called "scanner" which collects
- information on how many files and directories are to be saved, in order to display progress
- information to the user. When the backup finishes faster than the scanner, it is aborted
- because the result is not needed any more. This logic contained a bug, where quitting the
- scanner process was treated as an error, and caused restic to print an unhelpful error message
- ("context canceled").
-
- https://github.com/restic/restic/issues/1978
- https://github.com/restic/restic/pull/1991
-
- * Enhancement #1766: Restore: suppress lchown errors when not running as root
-
- Like "cp" and "rsync" do, restic now only reports errors for changing the ownership of files
- during restore if it is run as root, on non-Windows operating systems. On Windows, the error
- is reported as usual.
-
- https://github.com/restic/restic/issues/1766
-
- * Enhancement #1909: Reject files/dirs by name first
-
- The current scanner/archiver code had an architectural limitation: it always ran the
- `lstat()` system call on all files and directories before a decision to include/exclude the
- file/dir was made. This lead to a lot of unnecessary system calls for items that could have been
- rejected by their name or path only.
-
- We've changed the archiver/scanner implementation so that it now first rejects by name/path,
- and only runs the system call on the remaining items. This reduces the number of `lstat()`
- system calls a lot (depending on the exclude settings).
-
- https://github.com/restic/restic/issues/1909
- https://github.com/restic/restic/pull/1912
-
- * Enhancement #1940: Add directory filter to ls command
-
- The ls command can now be filtered by directories, so that only files in the given directories
- will be shown. If the --recursive flag is specified, then ls will traverse subfolders and list
- their files as well.
+Details
+-------
- It used to be possible to specify multiple snapshots, but that has been replaced by only one
- snapshot and the possibility of specifying multiple directories.
+ * Bugfix #1935: Remove truncated files from cache
- Specifying directories constrains the walk, which can significantly speed up the listing.
+ When a file in the local cache is truncated, and restic tries to access data beyond the end of the
+ (cached) file, it used to return an error "EOF". This is now fixed, such truncated files are
+ removed and the data is fetched directly from the backend.
- https://github.com/restic/restic/issues/1940
- https://github.com/restic/restic/pull/1941
+ https://github.com/restic/restic/issues/1935
- * Enhancement #1967: Use `--host` everywhere
+ * Bugfix #1978: Do not return an error when the scanner is slower than backup
- We now use the flag `--host` for all commands which need a host name, using `--hostname` (e.g.
- for `restic backup`) still works, but will print a deprecation warning. Also, add the short
- option `-H` where possible.
+ When restic makes a backup, there's a background task called "scanner" which collects
+ information on how many files and directories are to be saved, in order to display progress
+ information to the user. When the backup finishes faster than the scanner, it is aborted
+ because the result is not needed any more. This logic contained a bug, where quitting the
+ scanner process was treated as an error, and caused restic to print an unhelpful error message
+ ("context canceled").
- https://github.com/restic/restic/issues/1967
+ https://github.com/restic/restic/issues/1978
+ https://github.com/restic/restic/pull/1991
- * Enhancement #2028: Display size of cache directories
+ * Enhancement #1766: Restore: suppress lchown errors when not running as root
- The `cache` command now by default shows the size of the individual cache directories. It can be
- disabled with `--no-size`.
+ Like "cp" and "rsync" do, restic now only reports errors for changing the ownership of files
+ during restore if it is run as root, on non-Windows operating systems. On Windows, the error
+ is reported as usual.
- https://github.com/restic/restic/issues/2028
- https://github.com/restic/restic/pull/2033
+ https://github.com/restic/restic/issues/1766
* Enhancement #1777: Improve the `find` command
@@ -2737,6 +3249,20 @@ Details
https://github.com/restic/restic/issues/1891
+ * Enhancement #1909: Reject files/dirs by name first
+
+ The current scanner/archiver code had an architectural limitation: it always ran the
+ `lstat()` system call on all files and directories before a decision to include/exclude the
+ file/dir was made. This lead to a lot of unnecessary system calls for items that could have been
+ rejected by their name or path only.
+
+ We've changed the archiver/scanner implementation so that it now first rejects by name/path,
+ and only runs the system call on the remaining items. This reduces the number of `lstat()`
+ system calls a lot (depending on the exclude settings).
+
+ https://github.com/restic/restic/issues/1909
+ https://github.com/restic/restic/pull/1912
+
* Enhancement #1920: Vendor dependencies with Go 1.11 Modules
Until now, we've used `dep` for managing dependencies, we've now switch to using Go modules.
@@ -2745,6 +3271,20 @@ Details
https://github.com/restic/restic/pull/1920
+ * Enhancement #1940: Add directory filter to ls command
+
+ The ls command can now be filtered by directories, so that only files in the given directories
+ will be shown. If the --recursive flag is specified, then ls will traverse subfolders and list
+ their files as well.
+
+ It used to be possible to specify multiple snapshots, but that has been replaced by only one
+ snapshot and the possibility of specifying multiple directories.
+
+ Specifying directories constrains the walk, which can significantly speed up the listing.
+
+ https://github.com/restic/restic/issues/1940
+ https://github.com/restic/restic/pull/1941
+
* Enhancement #1949: Add new command `self-update`
We have added a new command called `self-update` which downloads the latest released version
@@ -2778,6 +3318,22 @@ Details
https://github.com/restic/restic/pull/1962
+ * Enhancement #1967: Use `--host` everywhere
+
+ We now use the flag `--host` for all commands which need a host name, using `--hostname` (e.g.
+ for `restic backup`) still works, but will print a deprecation warning. Also, add the short
+ option `-H` where possible.
+
+ https://github.com/restic/restic/issues/1967
+
+ * Enhancement #2028: Display size of cache directories
+
+ The `cache` command now by default shows the size of the individual cache directories. It can be
+ disabled with `--no-size`.
+
+ https://github.com/restic/restic/issues/2028
+ https://github.com/restic/restic/pull/2033
+
Changelog for restic 0.9.2 (2018-08-06)
=======================================
@@ -2789,16 +3345,16 @@ Summary
-------
* Fix #1854: Allow saving files/dirs on different fs with `--one-file-system`
+ * Fix #1861: Fix case-insensitive search with restic find
* Fix #1870: Fix restore with --include
* Fix #1880: Use `--cache-dir` argument for `check` command
* Fix #1893: Return error when exclude file cannot be read
- * Fix #1861: Fix case-insensitive search with restic find
- * Enh #1906: Add support for B2 application keys
* Enh #874: Add stats command to get information about a repository
+ * Enh #1477: S3 backend: accept AWS_SESSION_TOKEN
* Enh #1772: Add restore --verify to verify restored file content
* Enh #1853: Add JSON output support to `restic key list`
- * Enh #1477: S3 backend: accept AWS_SESSION_TOKEN
* Enh #1901: Update the Backblaze B2 library
+ * Enh #1906: Add support for B2 application keys
Details
-------
@@ -2819,6 +3375,12 @@ Details
https://github.com/restic/restic/issues/1854
https://github.com/restic/restic/pull/1855
+ * Bugfix #1861: Fix case-insensitive search with restic find
+
+ We've fixed the behavior for `restic find -i PATTERN`, which was broken in v0.9.1.
+
+ https://github.com/restic/restic/pull/1861
+
* Bugfix #1870: Fix restore with --include
We fixed a bug which prevented restic to restore files with an include filter.
@@ -2846,26 +3408,21 @@ Details
https://github.com/restic/restic/issues/1893
- * Bugfix #1861: Fix case-insensitive search with restic find
-
- We've fixed the behavior for `restic find -i PATTERN`, which was broken in v0.9.1.
-
- https://github.com/restic/restic/pull/1861
-
- * Enhancement #1906: Add support for B2 application keys
-
- Restic can now use so-called "application keys" which can be created in the B2 dashboard and
- were only introduced recently. In contrast to the "master key", such keys can be restricted to a
- specific bucket and/or path.
-
- https://github.com/restic/restic/issues/1906
- https://github.com/restic/restic/pull/1914
-
* Enhancement #874: Add stats command to get information about a repository
https://github.com/restic/restic/issues/874
https://github.com/restic/restic/pull/1729
+ * Enhancement #1477: S3 backend: accept AWS_SESSION_TOKEN
+
+ Before, it was not possible to use s3 backend with AWS temporary security credentials(with
+ AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials
+ provider.
+
+ https://github.com/restic/restic/issues/1477
+ https://github.com/restic/restic/pull/1479
+ https://github.com/restic/restic/pull/1647
+
* Enhancement #1772: Add restore --verify to verify restored file content
Restore will print error message if restored file content does not match expected SHA256
@@ -2880,16 +3437,6 @@ Details
https://github.com/restic/restic/pull/1853
- * Enhancement #1477: S3 backend: accept AWS_SESSION_TOKEN
-
- Before, it was not possible to use s3 backend with AWS temporary security credentials(with
- AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials
- provider.
-
- https://github.com/restic/restic/issues/1477
- https://github.com/restic/restic/pull/1479
- https://github.com/restic/restic/pull/1647
-
* Enhancement #1901: Update the Backblaze B2 library
We've updated the library we're using for accessing the Backblaze B2 service to 0.5.0 to
@@ -2900,6 +3447,15 @@ Details
https://github.com/restic/restic/pull/1901
https://github.com/kurin/blazer
+ * Enhancement #1906: Add support for B2 application keys
+
+ Restic can now use so-called "application keys" which can be created in the B2 dashboard and
+ were only introduced recently. In contrast to the "master key", such keys can be restricted to a
+ specific bucket and/or path.
+
+ https://github.com/restic/restic/issues/1906
+ https://github.com/restic/restic/pull/1914
+
Changelog for restic 0.9.1 (2018-06-10)
=======================================
@@ -2979,22 +3535,22 @@ Summary
* Fix #1608: Respect time stamp for new backup when reading from stdin
* Fix #1652: Ignore/remove invalid lock files
- * Fix #1730: Ignore sockets for restore
* Fix #1684: Fix backend tests for rest-server
+ * Fix #1730: Ignore sockets for restore
* Fix #1745: Correctly parse the argument to --tls-client-cert
- * Enh #1433: Support UTF-16 encoding and process Byte Order Mark
- * Enh #1561: Allow using rclone to access other services
- * Enh #1665: Improve cache handling for `restic check`
- * Enh #1721: Add `cache` command to list cache dirs
- * Enh #1758: Allow saving OneDrive folders in Windows
* Enh #549: Rework archiver code
- * Enh #1552: Use Google Application Default credentials
+ * Enh #827: Add --new-password-file flag for non-interactive password changes
+ * Enh #1433: Support UTF-16 encoding and process Byte Order Mark
* Enh #1477: Accept AWS_SESSION_TOKEN for the s3 backend
+ * Enh #1552: Use Google Application Default credentials
+ * Enh #1561: Allow using rclone to access other services
* Enh #1648: Ignore AWS permission denied error when creating a repository
* Enh #1649: Add illumos/Solaris support
+ * Enh #1665: Improve cache handling for `restic check`
* Enh #1709: Improve messages `restic check` prints
- * Enh #827: Add --new-password-file flag for non-interactive password changes
+ * Enh #1721: Add `cache` command to list cache dirs
* Enh #1735: Allow keeping a time range of snaphots
+ * Enh #1758: Allow saving OneDrive folders in Windows
* Enh #1782: Use default AWS credentials chain for S3 backend
Details
@@ -3018,6 +3574,13 @@ Details
https://github.com/restic/restic/issues/1652
https://github.com/restic/restic/pull/1653
+ * Bugfix #1684: Fix backend tests for rest-server
+
+ The REST server for restic now requires an explicit parameter (`--no-auth`) if no
+ authentication should be allowed. This is fixed in the tests.
+
+ https://github.com/restic/restic/pull/1684
+
* Bugfix #1730: Ignore sockets for restore
We've received a report and correct the behavior in which the restore code aborted restoring a
@@ -3028,13 +3591,6 @@ Details
https://github.com/restic/restic/issues/1730
https://github.com/restic/restic/pull/1731
- * Bugfix #1684: Fix backend tests for rest-server
-
- The REST server for restic now requires an explicit parameter (`--no-auth`) if no
- authentication should be allowed. This is fixed in the tests.
-
- https://github.com/restic/restic/pull/1684
-
* Bugfix #1745: Correctly parse the argument to --tls-client-cert
Previously, the --tls-client-cert method attempt to read ARGV[1] (hardcoded) instead of the
@@ -3043,73 +3599,6 @@ Details
https://github.com/restic/restic/issues/1745
https://github.com/restic/restic/pull/1746
- * Enhancement #1433: Support UTF-16 encoding and process Byte Order Mark
-
- On Windows, text editors commonly leave a Byte Order Mark at the beginning of the file to define
- which encoding is used (oftentimes UTF-16). We've added code to support processing the BOMs in
- text files, like the exclude files, the password file and the file passed via `--files-from`.
- This does not apply to any file being saved in a backup, those are not touched and archived as they
- are.
-
- https://github.com/restic/restic/issues/1433
- https://github.com/restic/restic/issues/1738
- https://github.com/restic/restic/pull/1748
-
- * Enhancement #1561: Allow using rclone to access other services
-
- We've added the ability to use rclone to store backup data on all backends that it supports. This
- was done in collaboration with Nick, the author of rclone. You can now use it to first configure a
- service, then restic manages the rest (starting and stopping rclone). For details, please see
- the manual.
-
- https://github.com/restic/restic/issues/1561
- https://github.com/restic/restic/pull/1657
- https://rclone.org
-
- * Enhancement #1665: Improve cache handling for `restic check`
-
- For safety reasons, restic does not use a local metadata cache for the `restic check` command,
- so that data is loaded from the repository and restic can check it's in good condition. When the
- cache is disabled, restic will fetch each tiny blob needed for checking the integrity using a
- separate backend request. For non-local backends, that will take a long time, and depending on
- the backend (e.g. B2) may also be much more expensive.
-
- This PR adds a few commits which will change the behavior as follows:
-
- * When `restic check` is called without any additional parameters, it will build a new cache in a
- temporary directory, which is removed at the end of the check. This way, we'll get readahead for
- metadata files (so restic will fetch the whole file when the first blob from the file is
- requested), but all data is freshly fetched from the storage backend. This is the default
- behavior and will work for almost all users.
-
- * When `restic check` is called with `--with-cache`, the default on-disc cache is used. This
- behavior hasn't changed since the cache was introduced.
-
- * When `--no-cache` is specified, restic falls back to the old behavior, and read all tiny blobs
- in separate requests.
-
- https://github.com/restic/restic/issues/1665
- https://github.com/restic/restic/issues/1694
- https://github.com/restic/restic/pull/1696
-
- * Enhancement #1721: Add `cache` command to list cache dirs
-
- The command `cache` was added, it allows listing restic's cache directoriers together with
- the last usage. It also allows removing old cache dirs without having to access a repo, via
- `restic cache --cleanup`
-
- https://github.com/restic/restic/issues/1721
- https://github.com/restic/restic/pull/1749
-
- * Enhancement #1758: Allow saving OneDrive folders in Windows
-
- Restic now contains a bugfix to two libraries, which allows saving OneDrive folders in
- Windows. In order to use the newer versions of the libraries, the minimal version required to
- compile restic is now Go 1.9.
-
- https://github.com/restic/restic/issues/1758
- https://github.com/restic/restic/pull/1765
-
* Enhancement #549: Rework archiver code
The core archiver code and the complementary code for the `backup` command was rewritten
@@ -3151,6 +3640,36 @@ Details
https://github.com/restic/restic/issues/1160
https://github.com/restic/restic/pull/1494
+ * Enhancement #827: Add --new-password-file flag for non-interactive password changes
+
+ This makes it possible to change a repository password without being prompted.
+
+ https://github.com/restic/restic/issues/827
+ https://github.com/restic/restic/pull/1720
+ https://forum.restic.net/t/changing-repo-password-without-prompt/591
+
+ * Enhancement #1433: Support UTF-16 encoding and process Byte Order Mark
+
+ On Windows, text editors commonly leave a Byte Order Mark at the beginning of the file to define
+ which encoding is used (oftentimes UTF-16). We've added code to support processing the BOMs in
+ text files, like the exclude files, the password file and the file passed via `--files-from`.
+ This does not apply to any file being saved in a backup, those are not touched and archived as they
+ are.
+
+ https://github.com/restic/restic/issues/1433
+ https://github.com/restic/restic/issues/1738
+ https://github.com/restic/restic/pull/1748
+
+ * Enhancement #1477: Accept AWS_SESSION_TOKEN for the s3 backend
+
+ Before, it was not possible to use s3 backend with AWS temporary security credentials(with
+ AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials
+ provider.
+
+ https://github.com/restic/restic/issues/1477
+ https://github.com/restic/restic/pull/1479
+ https://github.com/restic/restic/pull/1647
+
* Enhancement #1552: Use Google Application Default credentials
Google provide libraries to generate appropriate credentials with various fallback
@@ -3164,15 +3683,16 @@ Details
https://github.com/restic/restic/pull/1552
https://developers.google.com/identity/protocols/application-default-credentials
- * Enhancement #1477: Accept AWS_SESSION_TOKEN for the s3 backend
+ * Enhancement #1561: Allow using rclone to access other services
- Before, it was not possible to use s3 backend with AWS temporary security credentials(with
- AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials
- provider.
+ We've added the ability to use rclone to store backup data on all backends that it supports. This
+ was done in collaboration with Nick, the author of rclone. You can now use it to first configure a
+ service, then restic manages the rest (starting and stopping rclone). For details, please see
+ the manual.
- https://github.com/restic/restic/issues/1477
- https://github.com/restic/restic/pull/1479
- https://github.com/restic/restic/pull/1647
+ https://github.com/restic/restic/issues/1561
+ https://github.com/restic/restic/pull/1657
+ https://rclone.org
* Enhancement #1648: Ignore AWS permission denied error when creating a repository
@@ -3186,6 +3706,32 @@ Details
https://github.com/restic/restic/pull/1649
+ * Enhancement #1665: Improve cache handling for `restic check`
+
+ For safety reasons, restic does not use a local metadata cache for the `restic check` command,
+ so that data is loaded from the repository and restic can check it's in good condition. When the
+ cache is disabled, restic will fetch each tiny blob needed for checking the integrity using a
+ separate backend request. For non-local backends, that will take a long time, and depending on
+ the backend (e.g. B2) may also be much more expensive.
+
+ This PR adds a few commits which will change the behavior as follows:
+
+ * When `restic check` is called without any additional parameters, it will build a new cache in a
+ temporary directory, which is removed at the end of the check. This way, we'll get readahead for
+ metadata files (so restic will fetch the whole file when the first blob from the file is
+ requested), but all data is freshly fetched from the storage backend. This is the default
+ behavior and will work for almost all users.
+
+ * When `restic check` is called with `--with-cache`, the default on-disc cache is used. This
+ behavior hasn't changed since the cache was introduced.
+
+ * When `--no-cache` is specified, restic falls back to the old behavior, and read all tiny blobs
+ in separate requests.
+
+ https://github.com/restic/restic/issues/1665
+ https://github.com/restic/restic/issues/1694
+ https://github.com/restic/restic/pull/1696
+
* Enhancement #1709: Improve messages `restic check` prints
Some messages `restic check` prints are not really errors, so from now on restic does not treat
@@ -3194,13 +3740,14 @@ Details
https://github.com/restic/restic/pull/1709
https://forum.restic.net/t/what-is-the-standard-procedure-to-follow-if-a-backup-or-restore-is-interrupted/571/2
- * Enhancement #827: Add --new-password-file flag for non-interactive password changes
+ * Enhancement #1721: Add `cache` command to list cache dirs
- This makes it possible to change a repository password without being prompted.
+ The command `cache` was added, it allows listing restic's cache directoriers together with
+ the last usage. It also allows removing old cache dirs without having to access a repo, via
+ `restic cache --cleanup`
- https://github.com/restic/restic/issues/827
- https://github.com/restic/restic/pull/1720
- https://forum.restic.net/t/changing-repo-password-without-prompt/591
+ https://github.com/restic/restic/issues/1721
+ https://github.com/restic/restic/pull/1749
* Enhancement #1735: Allow keeping a time range of snaphots
@@ -3211,6 +3758,15 @@ Details
https://github.com/restic/restic/pull/1735
+ * Enhancement #1758: Allow saving OneDrive folders in Windows
+
+ Restic now contains a bugfix to two libraries, which allows saving OneDrive folders in
+ Windows. In order to use the newer versions of the libraries, the minimal version required to
+ compile restic is now Go 1.9.
+
+ https://github.com/restic/restic/issues/1758
+ https://github.com/restic/restic/pull/1765
+
* Enhancement #1782: Use default AWS credentials chain for S3 backend
Adds support for file credentials to the S3 backend (e.g. ~/.aws/credentials), and reorders
@@ -3230,8 +3786,8 @@ Summary
-------
* Fix #1633: Fixed unexpected 'pack file cannot be listed' error
- * Fix #1641: Ignore files with invalid names in the repo
* Fix #1638: Handle errors listing files in the backend
+ * Fix #1641: Ignore files with invalid names in the repo
* Enh #1497: Add --read-data-subset flag to check command
* Enh #1560: Retry all repository file download errors
* Enh #1623: Don't check for presence of files in the backend before writing
@@ -3248,16 +3804,6 @@ Details
https://github.com/restic/restic/issues/1633
https://github.com/restic/restic/pull/1635
- * Bugfix #1641: Ignore files with invalid names in the repo
-
- The release 0.8.2 introduced a bug: when restic encounters files in the repo which do not have a
- valid name, it tries to load a file with a name of lots of zeroes instead of ignoring it. This is now
- resolved, invalid file names are just ignored.
-
- https://github.com/restic/restic/issues/1641
- https://github.com/restic/restic/pull/1643
- https://forum.restic.net/t/help-fixing-repo-no-such-file/485/3
-
* Bugfix #1638: Handle errors listing files in the backend
A user reported in the forum that restic completes a backup although a concurrent `prune`
@@ -3273,6 +3819,16 @@ Details
https://github.com/restic/restic/pull/1638
https://forum.restic.net/t/restic-backup-returns-0-exit-code-when-already-locked/484
+ * Bugfix #1641: Ignore files with invalid names in the repo
+
+ The release 0.8.2 introduced a bug: when restic encounters files in the repo which do not have a
+ valid name, it tries to load a file with a name of lots of zeroes instead of ignoring it. This is now
+ resolved, invalid file names are just ignored.
+
+ https://github.com/restic/restic/issues/1641
+ https://github.com/restic/restic/pull/1643
+ https://forum.restic.net/t/help-fixing-repo-no-such-file/485/3
+
* Enhancement #1497: Add --read-data-subset flag to check command
This change introduces ability to check integrity of a subset of repository data packs. This
@@ -3320,18 +3876,18 @@ Summary
* Fix #1506: Limit bandwith at the http.RoundTripper for HTTP based backends
* Fix #1512: Restore directory permissions as the last step
* Fix #1528: Correctly create missing subdirs in data/
- * Fix #1590: Strip spaces for lines read via --files-from
* Fix #1589: Complete intermediate index upload
+ * Fix #1590: Strip spaces for lines read via --files-from
* Fix #1594: Google Cloud Storage: Use generic HTTP transport
* Fix #1595: Backup: Remove bandwidth display
- * Enh #1522: Add support for TLS client certificate authentication
- * Enh #1541: Reduce number of remote requests during repository check
- * Enh #1567: Reduce number of backend requests for rebuild-index and prune
* Enh #1507: Only reload snapshots once per minute for fuse mount
+ * Enh #1522: Add support for TLS client certificate authentication
* Enh #1538: Reduce memory allocations for querying the index
+ * Enh #1541: Reduce number of remote requests during repository check
* Enh #1549: Speed up querying across indices and scanning existing files
* Enh #1554: Fuse/mount: Correctly handle EOF, add template option
* Enh #1564: Don't terminate ssh on SIGINT
+ * Enh #1567: Reduce number of backend requests for rebuild-index and prune
* Enh #1579: Retry Backend.List() in case of errors
* Enh #1584: Limit index file size
@@ -3358,14 +3914,6 @@ Details
https://github.com/restic/restic/issues/1528
https://github.com/restic/restic/pull/1529
- * Bugfix #1590: Strip spaces for lines read via --files-from
-
- Leading and trailing spaces in lines read via `--files-from` are now stripped, so it behaves
- the same as with lines read via `--exclude-file`.
-
- https://github.com/restic/restic/issues/1590
- https://github.com/restic/restic/pull/1613
-
* Bugfix #1589: Complete intermediate index upload
After a user posted a comprehensive report of what he observed, we were able to find a bug and
@@ -3381,6 +3929,14 @@ Details
https://github.com/restic/restic/pull/1589
https://forum.restic.net/t/error-loading-tree-check-prune-and-forget-gives-error-b2-backend/406
+ * Bugfix #1590: Strip spaces for lines read via --files-from
+
+ Leading and trailing spaces in lines read via `--files-from` are now stripped, so it behaves
+ the same as with lines read via `--exclude-file`.
+
+ https://github.com/restic/restic/issues/1590
+ https://github.com/restic/restic/pull/1613
+
* Bugfix #1594: Google Cloud Storage: Use generic HTTP transport
It was discovered that the Google Cloud Storage backend did not use the generic HTTP transport,
@@ -3400,6 +3956,10 @@ Details
https://github.com/restic/restic/pull/1595
+ * Enhancement #1507: Only reload snapshots once per minute for fuse mount
+
+ https://github.com/restic/restic/pull/1507
+
* Enhancement #1522: Add support for TLS client certificate authentication
Support has been added for using a TLS client certificate for authentication to HTTP based
@@ -3409,27 +3969,6 @@ Details
https://github.com/restic/restic/issues/1522
https://github.com/restic/restic/pull/1524
- * Enhancement #1541: Reduce number of remote requests during repository check
-
- This change eliminates redundant remote repository calls and significantly improves
- repository check time.
-
- https://github.com/restic/restic/issues/1541
- https://github.com/restic/restic/pull/1548
-
- * Enhancement #1567: Reduce number of backend requests for rebuild-index and prune
-
- We've found a way to reduce then number of backend requests for the `rebuild-index` and `prune`
- operations. This significantly speeds up the operations for high-latency backends.
-
- https://github.com/restic/restic/issues/1567
- https://github.com/restic/restic/pull/1574
- https://github.com/restic/restic/pull/1575
-
- * Enhancement #1507: Only reload snapshots once per minute for fuse mount
-
- https://github.com/restic/restic/pull/1507
-
* Enhancement #1538: Reduce memory allocations for querying the index
This change reduces the internal memory allocations when the index data structures in memory
@@ -3438,6 +3977,14 @@ Details
https://github.com/restic/restic/pull/1538
+ * Enhancement #1541: Reduce number of remote requests during repository check
+
+ This change eliminates redundant remote repository calls and significantly improves
+ repository check time.
+
+ https://github.com/restic/restic/issues/1541
+ https://github.com/restic/restic/pull/1548
+
* Enhancement #1549: Speed up querying across indices and scanning existing files
This change increases the whenever a blob (part of a file) is searched for in a restic
@@ -3464,6 +4011,15 @@ Details
https://github.com/restic/restic/pull/1564
https://github.com/restic/restic/pull/1588
+ * Enhancement #1567: Reduce number of backend requests for rebuild-index and prune
+
+ We've found a way to reduce then number of backend requests for the `rebuild-index` and `prune`
+ operations. This significantly speeds up the operations for high-latency backends.
+
+ https://github.com/restic/restic/issues/1567
+ https://github.com/restic/restic/pull/1574
+ https://github.com/restic/restic/pull/1575
+
* Enhancement #1579: Retry Backend.List() in case of errors
https://github.com/restic/restic/pull/1579
@@ -3490,22 +4046,17 @@ restic users. The changes are ordered by importance.
Summary
-------
- * Fix #1457: Improve s3 backend with DigitalOcean Spaces
* Fix #1454: Correct cache dir location for Windows and Darwin
+ * Fix #1457: Improve s3 backend with DigitalOcean Spaces
* Fix #1459: Disable handling SIGPIPE
* Chg #1452: Do not save atime by default
+ * Enh #11: Add the `diff` command
* Enh #1436: Add code to detect old cache directories
* Enh #1439: Improve cancellation logic
- * Enh #11: Add the `diff` command
Details
-------
- * Bugfix #1457: Improve s3 backend with DigitalOcean Spaces
-
- https://github.com/restic/restic/issues/1457
- https://github.com/restic/restic/pull/1459
-
* Bugfix #1454: Correct cache dir location for Windows and Darwin
The cache directory on Windows and Darwin was not correct, instead the directory `.cache` was
@@ -3513,6 +4064,11 @@ Details
https://github.com/restic/restic/pull/1454
+ * Bugfix #1457: Improve s3 backend with DigitalOcean Spaces
+
+ https://github.com/restic/restic/issues/1457
+ https://github.com/restic/restic/pull/1459
+
* Bugfix #1459: Disable handling SIGPIPE
We've disabled handling SIGPIPE again. Turns out, writing to broken TCP connections also
@@ -3532,6 +4088,14 @@ Details
https://github.com/restic/restic/pull/1452
+ * Enhancement #11: Add the `diff` command
+
+ The command `diff` was added, it allows comparing two snapshots and listing all differences.
+
+ https://github.com/restic/restic/issues/11
+ https://github.com/restic/restic/issues/1460
+ https://github.com/restic/restic/pull/1462
+
* Enhancement #1436: Add code to detect old cache directories
We've added code to detect old cache directories of repositories that haven't been used in a
@@ -3548,14 +4112,6 @@ Details
https://github.com/restic/restic/pull/1439
- * Enhancement #11: Add the `diff` command
-
- The command `diff` was added, it allows comparing two snapshots and listing all differences.
-
- https://github.com/restic/restic/issues/11
- https://github.com/restic/restic/issues/1460
- https://github.com/restic/restic/pull/1462
-
Changelog for restic 0.8.0 (2017-11-26)
=======================================
@@ -3571,20 +4127,20 @@ Summary
* Fix #1291: Reuse backend TCP connections to BackBlaze B2
* Fix #1317: Run prune when `forget --prune` is called with just snapshot IDs
* Fix #1437: Remove implicit path `/restic` for the s3 backend
- * Enh #1102: Add subdirectory `ids` to fuse mount
- * Enh #1114: Add `--cacert` to specify TLS certificates to check against
- * Enh #1216: Add upload/download limiting
- * Enh #1271: Cache results for excludes for `backup`
- * Enh #1274: Add `generate` command, replaces `manpage` and `autocomplete`
- * Enh #1367: Allow comments in files read from via `--file-from`
* Enh #448: Sftp backend prompts for password
* Enh #510: Add `dump` command
* Enh #1040: Add local metadata cache
+ * Enh #1102: Add subdirectory `ids` to fuse mount
+ * Enh #1114: Add `--cacert` to specify TLS certificates to check against
+ * Enh #1216: Add upload/download limiting
* Enh #1249: Add `latest` symlink in fuse mount
* Enh #1269: Add `--compact` to `forget` command
+ * Enh #1271: Cache results for excludes for `backup`
+ * Enh #1274: Add `generate` command, replaces `manpage` and `autocomplete`
* Enh #1281: Google Cloud Storage backend needs less permissions
* Enh #1319: Make `check` print `no errors found` explicitly
* Enh #1353: Retry failed backend requests
+ * Enh #1367: Allow comments in files read from via `--file-from`
Details
-------
@@ -3624,75 +4180,23 @@ Details
https://github.com/restic/restic/issues/1291
https://github.com/restic/restic/pull/1301
- * Bugfix #1317: Run prune when `forget --prune` is called with just snapshot IDs
-
- A bug in the `forget` command caused `prune` not to be run when `--prune` was specified without a
- policy, e.g. when only snapshot IDs that should be forgotten are listed manually.
-
- https://github.com/restic/restic/pull/1317
-
- * Bugfix #1437: Remove implicit path `/restic` for the s3 backend
-
- The s3 backend used the subdir `restic` within a bucket if no explicit path after the bucket name
- was specified. Since this version, restic does not use this default path any more. If you
- created a repo on s3 in a bucket without specifying a path within the bucket, you need to add
- `/restic` at the end of the repository specification to access your repo:
- `s3:s3.amazonaws.com/bucket/restic`
-
- https://github.com/restic/restic/issues/1292
- https://github.com/restic/restic/pull/1437
-
- * Enhancement #1102: Add subdirectory `ids` to fuse mount
-
- The fuse mount now has an `ids` subdirectory which contains the snapshots below their (short)
- IDs.
-
- https://github.com/restic/restic/issues/1102
- https://github.com/restic/restic/pull/1299
- https://github.com/restic/restic/pull/1320
-
- * Enhancement #1114: Add `--cacert` to specify TLS certificates to check against
-
- We've added the `--cacert` option which can be used to pass one (or more) CA certificates to
- restic. These are used in addition to the system CA certificates to verify HTTPS certificates
- (e.g. for the REST backend).
-
- https://github.com/restic/restic/issues/1114
- https://github.com/restic/restic/pull/1276
-
- * Enhancement #1216: Add upload/download limiting
-
- We've added support for rate limiting through `--limit-upload` and `--limit-download`
- flags.
-
- https://github.com/restic/restic/issues/1216
- https://github.com/restic/restic/pull/1336
- https://github.com/restic/restic/pull/1358
-
- * Enhancement #1271: Cache results for excludes for `backup`
-
- The `backup` command now caches the result of excludes for a directory.
-
- https://github.com/restic/restic/issues/1271
- https://github.com/restic/restic/pull/1326
-
- * Enhancement #1274: Add `generate` command, replaces `manpage` and `autocomplete`
-
- The `generate` command has been added, which replaces the now removed commands `manpage` and
- `autocomplete`. This release of restic contains the most recent manpages in `doc/man` and the
- auto-completion files for bash and zsh in `doc/bash-completion.sh` and
- `doc/zsh-completion.zsh`
+ * Bugfix #1317: Run prune when `forget --prune` is called with just snapshot IDs
- https://github.com/restic/restic/issues/1274
- https://github.com/restic/restic/pull/1282
+ A bug in the `forget` command caused `prune` not to be run when `--prune` was specified without a
+ policy, e.g. when only snapshot IDs that should be forgotten are listed manually.
- * Enhancement #1367: Allow comments in files read from via `--file-from`
+ https://github.com/restic/restic/pull/1317
- When the list of files/dirs to be saved is read from a file with `--files-from`, comment lines
- (starting with `#`) are now ignored.
+ * Bugfix #1437: Remove implicit path `/restic` for the s3 backend
- https://github.com/restic/restic/issues/1367
- https://github.com/restic/restic/pull/1368
+ The s3 backend used the subdir `restic` within a bucket if no explicit path after the bucket name
+ was specified. Since this version, restic does not use this default path any more. If you
+ created a repo on s3 in a bucket without specifying a path within the bucket, you need to add
+ `/restic` at the end of the repository specification to access your repo:
+ `s3:s3.amazonaws.com/bucket/restic`
+
+ https://github.com/restic/restic/issues/1292
+ https://github.com/restic/restic/pull/1437
* Enhancement #448: Sftp backend prompts for password
@@ -3731,6 +4235,33 @@ Details
https://github.com/restic/restic/pull/1436
https://github.com/restic/restic/pull/1265
+ * Enhancement #1102: Add subdirectory `ids` to fuse mount
+
+ The fuse mount now has an `ids` subdirectory which contains the snapshots below their (short)
+ IDs.
+
+ https://github.com/restic/restic/issues/1102
+ https://github.com/restic/restic/pull/1299
+ https://github.com/restic/restic/pull/1320
+
+ * Enhancement #1114: Add `--cacert` to specify TLS certificates to check against
+
+ We've added the `--cacert` option which can be used to pass one (or more) CA certificates to
+ restic. These are used in addition to the system CA certificates to verify HTTPS certificates
+ (e.g. for the REST backend).
+
+ https://github.com/restic/restic/issues/1114
+ https://github.com/restic/restic/pull/1276
+
+ * Enhancement #1216: Add upload/download limiting
+
+ We've added support for rate limiting through `--limit-upload` and `--limit-download`
+ flags.
+
+ https://github.com/restic/restic/issues/1216
+ https://github.com/restic/restic/pull/1336
+ https://github.com/restic/restic/pull/1358
+
* Enhancement #1249: Add `latest` symlink in fuse mount
The directory structure in the fuse mount now exposes a symlink `latest` which points to the
@@ -3745,6 +4276,23 @@ Details
https://github.com/restic/restic/pull/1269
+ * Enhancement #1271: Cache results for excludes for `backup`
+
+ The `backup` command now caches the result of excludes for a directory.
+
+ https://github.com/restic/restic/issues/1271
+ https://github.com/restic/restic/pull/1326
+
+ * Enhancement #1274: Add `generate` command, replaces `manpage` and `autocomplete`
+
+ The `generate` command has been added, which replaces the now removed commands `manpage` and
+ `autocomplete`. This release of restic contains the most recent manpages in `doc/man` and the
+ auto-completion files for bash and zsh in `doc/bash-completion.sh` and
+ `doc/zsh-completion.zsh`
+
+ https://github.com/restic/restic/issues/1274
+ https://github.com/restic/restic/pull/1282
+
* Enhancement #1281: Google Cloud Storage backend needs less permissions
The Google Cloud Storage backend no longer requires the service account to have the
@@ -3764,6 +4312,14 @@ Details
https://github.com/restic/restic/pull/1353
+ * Enhancement #1367: Allow comments in files read from via `--file-from`
+
+ When the list of files/dirs to be saved is read from a file with `--files-from`, comment lines
+ (starting with `#`) are now ignored.
+
+ https://github.com/restic/restic/issues/1367
+ https://github.com/restic/restic/pull/1368
+
Changelog for restic 0.7.3 (2017-09-20)
=======================================
@@ -3798,27 +4354,31 @@ restic users. The changes are ordered by importance.
Summary
-------
- * Fix #1167: Do not create a local repo unless `init` is used
* Fix #1164: Make the `key remove` command behave as documented
+ * Fix #1167: Do not create a local repo unless `init` is used
* Fix #1191: Make sure to write profiling files on interrupt
- * Enh #1132: Make `key` command always prompt for a password
- * Enh #1179: Resolve name conflicts, append a counter
- * Enh #1218: Add `--compact` to `snapshots` command
* Enh #317: Add `--exclude-caches` and `--exclude-if-present`
* Enh #697: Automatically generate man pages for all restic commands
* Enh #1044: Improve `restore`, do not traverse/load excluded directories
* Enh #1061: Add Dockerfile and official Docker image
* Enh #1126: Use the standard Go git repository layout, use `dep` for vendoring
+ * Enh #1132: Make `key` command always prompt for a password
* Enh #1134: Add support for storing backups on Google Cloud Storage
* Enh #1144: Properly report errors when reading files with exclude patterns
* Enh #1149: Add support for storing backups on Microsoft Azure Blob Storage
+ * Enh #1179: Resolve name conflicts, append a counter
* Enh #1196: Add `--group-by` to `forget` command for flexible grouping
* Enh #1203: Print stats on all BSD systems when SIGINFO (ctrl+t) is received
* Enh #1205: Allow specifying time/date for a backup with `--time`
+ * Enh #1218: Add `--compact` to `snapshots` command
Details
-------
+ * Bugfix #1164: Make the `key remove` command behave as documented
+
+ https://github.com/restic/restic/pull/1164
+
* Bugfix #1167: Do not create a local repo unless `init` is used
When a restic command other than `init` is used with a local repository and the repository
@@ -3828,10 +4388,6 @@ Details
https://github.com/restic/restic/issues/1167
https://github.com/restic/restic/pull/1182
- * Bugfix #1164: Make the `key remove` command behave as documented
-
- https://github.com/restic/restic/pull/1164
-
* Bugfix #1191: Make sure to write profiling files on interrupt
Since a few releases restic had the ability to write profiling files for memory and CPU usage
@@ -3840,27 +4396,6 @@ Details
https://github.com/restic/restic/pull/1191
- * Enhancement #1132: Make `key` command always prompt for a password
-
- The `key` command now prompts for a password even if the original password to access a repo has
- been specified via the `RESTIC_PASSWORD` environment variable or a password file.
-
- https://github.com/restic/restic/issues/1132
- https://github.com/restic/restic/pull/1133
-
- * Enhancement #1179: Resolve name conflicts, append a counter
-
- https://github.com/restic/restic/issues/1179
- https://github.com/restic/restic/pull/1209
-
- * Enhancement #1218: Add `--compact` to `snapshots` command
-
- The option `--compact` was added to the `snapshots` command to get a better overview of the
- snapshots in a repo. It limits each snapshot to a single line.
-
- https://github.com/restic/restic/issues/1218
- https://github.com/restic/restic/pull/1223
-
* Enhancement #317: Add `--exclude-caches` and `--exclude-if-present`
A new option `--exclude-caches` was added that allows excluding cache directories (that are
@@ -3892,6 +4427,14 @@ Details
https://github.com/restic/restic/pull/1126
+ * Enhancement #1132: Make `key` command always prompt for a password
+
+ The `key` command now prompts for a password even if the original password to access a repo has
+ been specified via the `RESTIC_PASSWORD` environment variable or a password file.
+
+ https://github.com/restic/restic/issues/1132
+ https://github.com/restic/restic/pull/1133
+
* Enhancement #1134: Add support for storing backups on Google Cloud Storage
https://github.com/restic/restic/issues/211
@@ -3911,6 +4454,11 @@ Details
https://github.com/restic/restic/pull/1149
https://github.com/restic/restic/pull/1059
+ * Enhancement #1179: Resolve name conflicts, append a counter
+
+ https://github.com/restic/restic/issues/1179
+ https://github.com/restic/restic/pull/1209
+
* Enhancement #1196: Add `--group-by` to `forget` command for flexible grouping
https://github.com/restic/restic/pull/1196
@@ -3924,6 +4472,14 @@ Details
https://github.com/restic/restic/pull/1205
+ * Enhancement #1218: Add `--compact` to `snapshots` command
+
+ The option `--compact` was added to the `snapshots` command to get a better overview of the
+ snapshots in a repo. It limits each snapshot to a single line.
+
+ https://github.com/restic/restic/issues/1218
+ https://github.com/restic/restic/pull/1223
+
Changelog for restic 0.7.1 (2017-07-22)
=======================================
@@ -3938,8 +4494,8 @@ Summary
* Enh #1055: Create subdirs below `data/` for local/sftp backends
* Enh #1067: Allow loading credentials for s3 from IAM
* Enh #1073: Add `migrate` cmd to migrate from `s3legacy` to `default` layout
- * Enh #1081: Clarify semantic for `--tag` for the `forget` command
* Enh #1080: Ignore chmod() errors on filesystems which do not support it
+ * Enh #1081: Clarify semantic for `--tag` for the `forget` command
* Enh #1082: Print stats on SIGINFO on Darwin and FreeBSD (ctrl+t)
Details
@@ -3982,16 +4538,16 @@ Details
https://github.com/restic/restic/issues/1073
https://github.com/restic/restic/pull/1075
- * Enhancement #1081: Clarify semantic for `--tag` for the `forget` command
-
- https://github.com/restic/restic/issues/1081
- https://github.com/restic/restic/pull/1090
-
* Enhancement #1080: Ignore chmod() errors on filesystems which do not support it
https://github.com/restic/restic/pull/1080
https://github.com/restic/restic/pull/1112
+ * Enhancement #1081: Clarify semantic for `--tag` for the `forget` command
+
+ https://github.com/restic/restic/issues/1081
+ https://github.com/restic/restic/pull/1090
+
* Enhancement #1082: Print stats on SIGINFO on Darwin and FreeBSD (ctrl+t)
https://github.com/restic/restic/pull/1082
@@ -4006,29 +4562,19 @@ restic users. The changes are ordered by importance.
Summary
-------
- * Fix #1013: Switch back to using the high-level minio-go API for s3
* Fix #965: Switch to `default` repo layout for the s3 backend
- * Enh #1021: Detect invalid backend name and print error
- * Enh #1029: Remove invalid pack files when `prune` is run
+ * Fix #1013: Switch back to using the high-level minio-go API for s3
* Enh #512: Add Backblaze B2 backend
* Enh #636: Add dirs `tags` and `hosts` to fuse mount
- * Enh #989: Improve performance of the `find` command
* Enh #975: Add new backend for OpenStack Swift
+ * Enh #989: Improve performance of the `find` command
* Enh #998: Improve performance of the fuse mount
+ * Enh #1021: Detect invalid backend name and print error
+ * Enh #1029: Remove invalid pack files when `prune` is run
Details
-------
- * Bugfix #1013: Switch back to using the high-level minio-go API for s3
-
- For the s3 backend we're back to using the high-level API the s3 client library for uploading
- data, a few users reported dropped connections (which the library will automatically retry
- now).
-
- https://github.com/restic/restic/issues/1013
- https://github.com/restic/restic/issues/1023
- https://github.com/restic/restic/pull/1025
-
* Bugfix #965: Switch to `default` repo layout for the s3 backend
The default layout for the s3 backend is now `default` (instead of `s3legacy`). Also, there's a
@@ -4038,21 +4584,15 @@ Details
https://github.com/restic/restic/issues/965
https://github.com/restic/restic/pull/1004
- * Enhancement #1021: Detect invalid backend name and print error
-
- Restic now tries to detect when an invalid/unknown backend is used and returns an error
- message.
-
- https://github.com/restic/restic/issues/1021
- https://github.com/restic/restic/pull/1070
-
- * Enhancement #1029: Remove invalid pack files when `prune` is run
+ * Bugfix #1013: Switch back to using the high-level minio-go API for s3
- The `prune` command has been improved and will now remove invalid pack files, for example files
- that have not been uploaded completely because a backup was interrupted.
+ For the s3 backend we're back to using the high-level API the s3 client library for uploading
+ data, a few users reported dropped connections (which the library will automatically retry
+ now).
- https://github.com/restic/restic/issues/1029
- https://github.com/restic/restic/pull/1036
+ https://github.com/restic/restic/issues/1013
+ https://github.com/restic/restic/issues/1023
+ https://github.com/restic/restic/pull/1025
* Enhancement #512: Add Backblaze B2 backend
@@ -4068,6 +4608,11 @@ Details
https://github.com/restic/restic/issues/636
https://github.com/restic/restic/pull/1050
+ * Enhancement #975: Add new backend for OpenStack Swift
+
+ https://github.com/restic/restic/pull/975
+ https://github.com/restic/restic/pull/648
+
* Enhancement #989: Improve performance of the `find` command
Improved performance for the `find` command: Restic recognizes paths it has already checked
@@ -4076,17 +4621,28 @@ Details
https://github.com/restic/restic/issues/989
https://github.com/restic/restic/pull/993
- * Enhancement #975: Add new backend for OpenStack Swift
-
- https://github.com/restic/restic/pull/975
- https://github.com/restic/restic/pull/648
-
* Enhancement #998: Improve performance of the fuse mount
Listing directories which contain large files now is significantly faster.
https://github.com/restic/restic/pull/998
+ * Enhancement #1021: Detect invalid backend name and print error
+
+ Restic now tries to detect when an invalid/unknown backend is used and returns an error
+ message.
+
+ https://github.com/restic/restic/issues/1021
+ https://github.com/restic/restic/pull/1070
+
+ * Enhancement #1029: Remove invalid pack files when `prune` is run
+
+ The `prune` command has been improved and will now remove invalid pack files, for example files
+ that have not been uploaded completely because a backup was interrupted.
+
+ https://github.com/restic/restic/issues/1029
+ https://github.com/restic/restic/pull/1036
+
Changelog for restic 0.6.1 (2017-06-01)
=======================================
@@ -4097,21 +4653,20 @@ restic users. The changes are ordered by importance.
Summary
-------
- * Enh #985: Allow multiple parallel idle HTTP connections
- * Enh #981: Remove temporary path from binary in `build.go`
* Enh #974: Remove regular status reports
+ * Enh #981: Remove temporary path from binary in `build.go`
+ * Enh #985: Allow multiple parallel idle HTTP connections
Details
-------
- * Enhancement #985: Allow multiple parallel idle HTTP connections
+ * Enhancement #974: Remove regular status reports
- Backends based on HTTP now allow several idle connections in parallel. This is especially
- important for the REST backend, which (when used with a local server) may create a lot
- connections and exhaust available ports quickly.
+ Regular status report: We've removed the status report that was printed every 10 seconds when
+ restic is run non-interactively. You can still force reporting the current status by sending a
+ `USR1` signal to the process.
- https://github.com/restic/restic/issues/985
- https://github.com/restic/restic/pull/986
+ https://github.com/restic/restic/pull/974
* Enhancement #981: Remove temporary path from binary in `build.go`
@@ -4120,13 +4675,14 @@ Details
https://github.com/restic/restic/pull/981
- * Enhancement #974: Remove regular status reports
+ * Enhancement #985: Allow multiple parallel idle HTTP connections
- Regular status report: We've removed the status report that was printed every 10 seconds when
- restic is run non-interactively. You can still force reporting the current status by sending a
- `USR1` signal to the process.
+ Backends based on HTTP now allow several idle connections in parallel. This is especially
+ important for the REST backend, which (when used with a local server) may create a lot
+ connections and exhaust available ports quickly.
- https://github.com/restic/restic/pull/974
+ https://github.com/restic/restic/issues/985
+ https://github.com/restic/restic/pull/986
Changelog for restic 0.6.0 (2017-05-29)
@@ -4139,8 +4695,8 @@ Summary
-------
* Enh #957: Make `forget` consistent
- * Enh #966: Unify repository layout for all backends
* Enh #962: Improve memory and runtime for the s3 backend
+ * Enh #966: Unify repository layout for all backends
Details
-------
@@ -4154,17 +4710,6 @@ Details
https://github.com/restic/restic/issues/953
https://github.com/restic/restic/pull/957
- * Enhancement #966: Unify repository layout for all backends
-
- Up to now the s3 backend used a special repository layout. We've decided to unify the repository
- layout and implemented the default layout also for the s3 backend. For creating a new
- repository on s3 with the default layout, use `restic -o s3.layout=default init`. For further
- commands the option is not necessary any more, restic will automatically detect the correct
- layout to use. A future version will switch to the default layout for new repositories.
-
- https://github.com/restic/restic/issues/965
- https://github.com/restic/restic/pull/966
-
* Enhancement #962: Improve memory and runtime for the s3 backend
We've updated the library used for accessing s3, switched to using a lower level API and added
@@ -4180,4 +4725,15 @@ Details
https://github.com/restic/restic/pull/938
https://github.com/restic/restic/pull/883
+ * Enhancement #966: Unify repository layout for all backends
+
+ Up to now the s3 backend used a special repository layout. We've decided to unify the repository
+ layout and implemented the default layout also for the s3 backend. For creating a new
+ repository on s3 with the default layout, use `restic -o s3.layout=default init`. For further
+ commands the option is not necessary any more, restic will automatically detect the correct
+ layout to use. A future version will switch to the default layout for new repositories.
+
+ https://github.com/restic/restic/issues/965
+ https://github.com/restic/restic/pull/966
+
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index cf1f1e739..4b4be0757 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -76,7 +76,7 @@ Then use the `go` tool to build restic:
$ go build ./cmd/restic
$ ./restic version
- restic 0.10.0-dev (compiled manually) compiled with go1.15.2 on linux/amd64
+ restic 0.14.0-dev (compiled manually) compiled with go1.19 on linux/amd64
You can run all tests with the following command:
diff --git a/VERSION b/VERSION
index a803cc227..e815b861f 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.14.0
+0.15.1
diff --git a/build.go b/build.go
index 18454454e..dddc3b964 100644
--- a/build.go
+++ b/build.go
@@ -3,8 +3,8 @@
// This program aims to make building Go programs for end users easier by just
// calling it with `go run`, without having to setup a GOPATH.
//
-// This program needs Go >= 1.12. It'll use Go modules for compilation. It
-// builds the package configured as Main in the Config struct.
+// This program checks for a minimum Go version. It will use Go modules for
+// compilation. It builds the package configured as Main in the Config struct.
// BSD 2-Clause License
//
@@ -43,7 +43,6 @@ package main
import (
"fmt"
"io"
- "io/ioutil"
"os"
"os/exec"
"path/filepath"
@@ -59,7 +58,7 @@ var config = Config{
Main: "./cmd/restic", // package name for the main package
DefaultBuildTags: []string{"selfupdate"}, // specify build tags which are always used
Tests: []string{"./..."}, // tests to run
- MinVersion: GoVersion{Major: 1, Minor: 14, Patch: 0}, // minimum Go version supported
+ MinVersion: GoVersion{Major: 1, Minor: 18, Patch: 0}, // minimum Go version supported
}
// Config configures the build.
@@ -179,7 +178,7 @@ func test(cwd string, env map[string]string, args ...string) error {
// getVersion returns the version string from the file VERSION in the current
// directory.
func getVersionFromFile() string {
- buf, err := ioutil.ReadFile("VERSION")
+ buf, err := os.ReadFile("VERSION")
if err != nil {
verbosePrintf("error reading file VERSION: %v\n", err)
return ""
@@ -319,12 +318,8 @@ func (v GoVersion) String() string {
}
func main() {
- if !goVersion.AtLeast(GoVersion{1, 12, 0}) {
- die("Go version (%v) is too old, restic requires Go >= 1.12\n", goVersion)
- }
-
if !goVersion.AtLeast(config.MinVersion) {
- fmt.Fprintf(os.Stderr, "%s detected, this program requires at least %s\n", goVersion, config.MinVersion)
+ fmt.Fprintf(os.Stderr, "Detected version %s is too old, restic requires at least %s\n", goVersion, config.MinVersion)
os.Exit(1)
}
diff --git a/changelog/0.15.0_2023-01-12/issue-14 b/changelog/0.15.0_2023-01-12/issue-14
new file mode 100644
index 000000000..b1f9e225b
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-14
@@ -0,0 +1,8 @@
+Enhancement: Implement `rewrite` command
+
+Restic now has a `rewrite` command which allows to rewrite existing snapshots
+to remove unwanted files.
+
+https://github.com/restic/restic/issues/14
+https://github.com/restic/restic/pull/2731
+https://github.com/restic/restic/pull/4079
diff --git a/changelog/0.15.0_2023-01-12/issue-1734 b/changelog/0.15.0_2023-01-12/issue-1734
new file mode 100644
index 000000000..8d4a16c13
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-1734
@@ -0,0 +1,15 @@
+Enhancement: Inform about successful retries after errors
+
+When a recoverable error is encountered, restic shows a warning message saying
+that it's retrying, e.g.:
+
+`Save(<data/956b9ced99>) returned error, retrying after 357.131936ms: ...`
+
+This message can be confusing in that it never clearly states whether the retry
+is successful or not. This has now been fixed such that restic follows up with
+a message confirming a successful retry, e.g.:
+
+`Save(<data/956b9ced99>) operation successful after 1 retries`
+
+https://github.com/restic/restic/issues/1734
+https://github.com/restic/restic/pull/2661
diff --git a/changelog/0.15.0_2023-01-12/issue-1866 b/changelog/0.15.0_2023-01-12/issue-1866
new file mode 100644
index 000000000..1ff5b9ad5
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-1866
@@ -0,0 +1,12 @@
+Enhancement: Improve handling of directories with duplicate entries
+
+If for some reason a directory contains a duplicate entry, the `backup` command
+would previously fail with a `node "path/to/file" already present` or `nodes
+are not ordered got "path/to/file", last "path/to/file"` error.
+
+The error handling has been improved to only report a warning in this case. Make
+sure to check that the filesystem in question is not damaged if you see this!
+
+https://github.com/restic/restic/issues/1866
+https://github.com/restic/restic/issues/3937
+https://github.com/restic/restic/pull/3880
diff --git a/changelog/0.15.0_2023-01-12/issue-2015 b/changelog/0.15.0_2023-01-12/issue-2015
new file mode 100644
index 000000000..cb40f1868
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-2015
@@ -0,0 +1,10 @@
+Bugfix: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT
+
+To stop the `mount` command, a user has to press Ctrl-C or send a SIGINT
+signal to restic. This used to cause restic to exit with a non-zero exit code.
+
+The exit code has now been changed to zero as the above is the expected way
+to stop the `mount` command and should therefore be considered successful.
+
+https://github.com/restic/restic/issues/2015
+https://github.com/restic/restic/pull/3894
diff --git a/changelog/0.15.0_2023-01-12/issue-2134 b/changelog/0.15.0_2023-01-12/issue-2134
new file mode 100644
index 000000000..527b9dc2e
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-2134
@@ -0,0 +1,19 @@
+Enhancement: Support B2 API keys restricted to hiding but not deleting files
+
+When the B2 backend does not have the necessary permissions to permanently
+delete files, it now automatically falls back to hiding files. This allows
+using restic with an application key which is not allowed to delete files.
+This can prevent an attacker from deleting backups with such an API key.
+
+To use this feature create an application key without the `deleteFiles`
+capability. It is recommended to restrict the key to just one bucket.
+For example using the `b2` command line tool:
+
+`b2 create-key --bucket <bucketName> <keyName> listBuckets,readFiles,writeFiles,listFiles`
+
+Alternatively, you can use the S3 backend to access B2, as described
+in the documentation. In this mode, files are also only hidden instead
+of being deleted permanently.
+
+https://github.com/restic/restic/issues/2134
+https://github.com/restic/restic/pull/2398
diff --git a/changelog/0.15.0_2023-01-12/issue-2152 b/changelog/0.15.0_2023-01-12/issue-2152
new file mode 100644
index 000000000..7eafe2e14
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-2152
@@ -0,0 +1,11 @@
+Enhancement: Make `init` open only one connection for the SFTP backend
+
+The `init` command using the SFTP backend used to connect twice to the
+repository. This could be inconvenient if the user must enter a password,
+or cause `init` to fail if the server does not correctly close the first SFTP
+connection.
+
+This has now been fixed by reusing the first/initial SFTP connection opened.
+
+https://github.com/restic/restic/issues/2152
+https://github.com/restic/restic/pull/3882
diff --git a/changelog/0.15.0_2023-01-12/issue-2533 b/changelog/0.15.0_2023-01-12/issue-2533
new file mode 100644
index 000000000..e53ab1179
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-2533
@@ -0,0 +1,13 @@
+Enhancement: Handle cache corruption on disk and in downloads
+
+In rare situations, like for example after a system crash, the data stored
+in the cache might be corrupted. This could cause restic to fail and required
+manually deleting the cache.
+
+Restic now automatically removes broken data from the cache, allowing it
+to recover from such a situation without user intervention. In addition,
+restic retries downloads which return corrupt data in order to also handle
+temporary download problems.
+
+https://github.com/restic/restic/issues/2533
+https://github.com/restic/restic/pull/3521
diff --git a/changelog/0.15.0_2023-01-12/issue-2591 b/changelog/0.15.0_2023-01-12/issue-2591
new file mode 100644
index 000000000..5d74bd4dc
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-2591
@@ -0,0 +1,17 @@
+Bugfix: Don't read password from stdin for `backup --stdin`
+
+The `backup` command when used with `--stdin` previously tried to read first
+the password, then the data to be backed up from standard input. This meant
+it would often confuse part of the data for the password.
+
+From now on, it will instead exit with the message `Fatal: cannot read both
+password and data from stdin` unless the password is passed in some other
+way (such as `--restic-password-file`, `RESTIC_PASSWORD`, etc).
+
+To enter the password interactively a password command has to be used. For
+example on Linux, `mysqldump somedatabase | restic backup --stdin
+--password-command='sh -c "systemd-ask-password < /dev/tty"'` securely reads
+the password from the terminal.
+
+https://github.com/restic/restic/issues/2591
+https://github.com/restic/restic/pull/4011
diff --git a/changelog/0.15.0_2023-01-12/issue-2699 b/changelog/0.15.0_2023-01-12/issue-2699
new file mode 100644
index 000000000..1d35259ee
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-2699
@@ -0,0 +1,9 @@
+Enhancement: Support restoring symbolic links on Windows
+
+The `restore` command now supports restoring symbolic links on Windows. Because
+of Windows specific restrictions this is only possible when running restic with
+the `SeCreateSymbolicLinkPrivilege` privilege or as an administrator.
+
+https://github.com/restic/restic/issues/1078
+https://github.com/restic/restic/issues/2699
+https://github.com/restic/restic/pull/2875
diff --git a/changelog/0.15.0_2023-01-12/issue-2715 b/changelog/0.15.0_2023-01-12/issue-2715
new file mode 100644
index 000000000..be086384e
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-2715
@@ -0,0 +1,20 @@
+Enhancement: Stricter repository lock handling
+
+Previously, restic commands kept running even if they failed to refresh their
+locks in time. This could be a problem e.g. in case the client system running
+a backup entered the standby power mode while the backup was still in progress
+(which would prevent the client from refreshing its lock), and after a short
+delay another host successfully runs `unlock` and `prune` on the repository,
+which would remove all data added by the in-progress backup. If the backup
+client later continues its backup, even though its lock had expired in the
+meantime, this would lead to an incomplete snapshot.
+
+To address this, lock handling is now much stricter. Commands requiring a lock
+are canceled if the lock is not refreshed successfully in time. In addition,
+if a lock file is not readable restic will not allow starting a command. It may
+be necessary to remove invalid lock files manually or use `unlock --remove-all`.
+Please make sure that no other restic processes are running concurrently before
+doing this, however.
+
+https://github.com/restic/restic/issues/2715
+https://github.com/restic/restic/pull/3569
diff --git a/changelog/0.15.0_2023-01-12/issue-2724 b/changelog/0.15.0_2023-01-12/issue-2724
new file mode 100644
index 000000000..ba91ccb08
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-2724
@@ -0,0 +1,9 @@
+Change: Include full snapshot ID in JSON output of `backup`
+
+We have changed the JSON output of the backup command to include the full
+snapshot ID instead of just a shortened version, as the latter can be ambiguous
+in some rare cases. To derive the short ID, please truncate the full ID down to
+eight characters.
+
+https://github.com/restic/restic/issues/2724
+https://github.com/restic/restic/pull/3993
diff --git a/changelog/0.15.0_2023-01-12/issue-3029 b/changelog/0.15.0_2023-01-12/issue-3029
new file mode 100644
index 000000000..e66179bf2
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-3029
@@ -0,0 +1,8 @@
+Enhancement: Add support for `credential_process` to S3 backend
+
+Restic now uses a newer library for the S3 backend, which adds support for the
+`credential_process` option in the AWS credential configuration.
+
+https://github.com/restic/restic/issues/3029
+https://github.com/restic/restic/issues/4034
+https://github.com/restic/restic/pull/4025
diff --git a/changelog/0.15.0_2023-01-12/issue-3096 b/changelog/0.15.0_2023-01-12/issue-3096
new file mode 100644
index 000000000..e16d68684
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-3096
@@ -0,0 +1,8 @@
+Enhancement: Make `mount` command support macOS using macFUSE 4.x
+
+Restic now uses a different FUSE library for mounting snapshots and making them
+available as a FUSE filesystem using the `mount` command. This adds support for
+macFUSE 4.x which can be used to make this work on recent macOS versions.
+
+https://github.com/restic/restic/issues/3096
+https://github.com/restic/restic/pull/4024
diff --git a/changelog/0.15.0_2023-01-12/issue-3124 b/changelog/0.15.0_2023-01-12/issue-3124
new file mode 100644
index 000000000..5b7751071
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-3124
@@ -0,0 +1,7 @@
+Enhancement: Support JSON output for the `init` command
+
+The `init` command used to ignore the `--json` option, but now outputs a JSON
+message if the repository was created successfully.
+
+https://github.com/restic/restic/issues/3124
+https://github.com/restic/restic/pull/3132
diff --git a/changelog/0.15.0_2023-01-12/issue-3161 b/changelog/0.15.0_2023-01-12/issue-3161
new file mode 100644
index 000000000..4bb109339
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-3161
@@ -0,0 +1,14 @@
+Bugfix: Delete files on Backblaze B2 more reliably
+
+Restic used to only delete the latest version of files stored in B2. In most
+cases this worked well as there was only a single version of the file. However,
+due to retries while uploading it is possible for multiple file versions to be
+stored at B2. This could lead to various problems for files that should have
+been deleted but still existed.
+
+The implementation has now been changed to delete all versions of files, which
+doubles the amount of Class B transactions necessary to delete files, but
+assures that no file versions are left behind.
+
+https://github.com/restic/restic/issues/3161
+https://github.com/restic/restic/pull/3885
diff --git a/changelog/0.15.0_2023-01-12/issue-3336 b/changelog/0.15.0_2023-01-12/issue-3336
new file mode 100644
index 000000000..e222ef052
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-3336
@@ -0,0 +1,12 @@
+Bugfix: Make SFTP backend report no space left on device
+
+Backing up to an SFTP backend would spew repeated SSH_FX_FAILURE messages when
+the remote disk was full. Restic now reports "sftp: no space left on device"
+and exits immediately when it detects this condition.
+
+A fix for this issue was implemented in restic 0.12.1, but unfortunately the
+fix itself contained a bug that prevented it from taking effect.
+
+https://github.com/restic/restic/issues/3336
+https://github.com/restic/restic/pull/3345
+https://github.com/restic/restic/pull/4075
diff --git a/changelog/0.15.0_2023-01-12/issue-3567 b/changelog/0.15.0_2023-01-12/issue-3567
new file mode 100644
index 000000000..ae6bc53ed
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-3567
@@ -0,0 +1,10 @@
+Bugfix: Improve handling of interrupted syscalls in `mount` command
+
+Accessing restic's FUSE mount could result in "input/output" errors when using
+programs in which syscalls can be interrupted. This is for example the case for
+Go programs. This has now been fixed by improved error handling of interrupted
+syscalls.
+
+https://github.com/restic/restic/issues/3567
+https://github.com/restic/restic/issues/3694
+https://github.com/restic/restic/pull/3875
diff --git a/changelog/0.15.0_2023-01-12/issue-3897 b/changelog/0.15.0_2023-01-12/issue-3897
new file mode 100644
index 000000000..42422db78
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-3897
@@ -0,0 +1,7 @@
+Bugfix: Fix stuck `copy` command when `-o <backend>.connections=1`
+
+When running the `copy` command with `-o <backend>.connections=1` the
+command would be infinitely stuck. This has now been fixed.
+
+https://github.com/restic/restic/issues/3897
+https://github.com/restic/restic/pull/3898
diff --git a/changelog/0.15.0_2023-01-12/issue-3918 b/changelog/0.15.0_2023-01-12/issue-3918
new file mode 100644
index 000000000..1f5604c83
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-3918
@@ -0,0 +1,9 @@
+Bugfix: Correct prune statistics for partially compressed repositories
+
+In a partially compressed repository, one data blob can exist both in an
+uncompressed and a compressed version. This caused the `prune` statistics to
+become inaccurate and e.g. report a too high value for the unused size, such
+as "unused size after prune: 16777215.991 TiB". This has now been fixed.
+
+https://github.com/restic/restic/issues/3918
+https://github.com/restic/restic/pull/3980
diff --git a/changelog/0.15.0_2023-01-12/issue-3929 b/changelog/0.15.0_2023-01-12/issue-3929
new file mode 100644
index 000000000..128d85194
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-3929
@@ -0,0 +1,11 @@
+Change: Make `unlock` display message only when locks were actually removed
+
+The `unlock` command used to print the "successfully removed locks" message
+whenever it was run, regardless of lock files having being removed or not.
+
+This has now been changed such that it only prints the message if any lock
+files were actually removed. In addition, it also reports the number of
+removed lock files.
+
+https://github.com/restic/restic/issues/3929
+https://github.com/restic/restic/pull/3935
diff --git a/changelog/0.15.0_2023-01-12/issue-3932 b/changelog/0.15.0_2023-01-12/issue-3932
new file mode 100644
index 000000000..132684009
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-3932
@@ -0,0 +1,15 @@
+Enhancement: Improve handling of ErrDot errors in rclone and sftp backends
+
+Since Go 1.19, restic can no longer implicitly run relative executables which
+are found in the current directory (e.g. `rclone` if found in `.`). This is a
+security feature of Go to prevent against running unintended and possibly
+harmful executables.
+
+The error message for this was just "cannot run executable found relative to
+current directory". This has now been improved to yield a more specific error
+message, informing the user how to explicitly allow running the executable
+using the `-o rclone.program` and `-o sftp.command` extended options with `./`.
+
+https://github.com/restic/restic/issues/3932
+https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory
+https://go.dev/blog/path-security
diff --git a/changelog/0.15.0_2023-01-12/issue-4003 b/changelog/0.15.0_2023-01-12/issue-4003
new file mode 100644
index 000000000..37bcf4da6
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-4003
@@ -0,0 +1,8 @@
+Bugfix: Make `backup` no longer hang on Solaris when seeing a FIFO file
+
+The `backup` command used to hang on Solaris whenever it encountered a FIFO
+file (named pipe), due to a bug in the handling of extended attributes. This
+bug has now been fixed.
+
+https://github.com/restic/restic/issues/4003
+https://github.com/restic/restic/pull/4053
diff --git a/changelog/0.15.0_2023-01-12/issue-4016 b/changelog/0.15.0_2023-01-12/issue-4016
new file mode 100644
index 000000000..11dd32332
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-4016
@@ -0,0 +1,8 @@
+Bugfix: Support ExFAT-formatted local backends on macOS Ventura
+
+ExFAT-formatted disks could not be used as local backends starting from macOS
+Ventura. Restic commands would fail with an "inappropriate ioctl for device"
+error. This has now been fixed.
+
+https://github.com/restic/restic/issues/4016
+https://github.com/restic/restic/pull/4021
diff --git a/changelog/0.15.0_2023-01-12/issue-4033 b/changelog/0.15.0_2023-01-12/issue-4033
new file mode 100644
index 000000000..66093042c
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-4033
@@ -0,0 +1,11 @@
+Change: Don't print skipped snapshots by default in `copy` command
+
+The `copy` command used to print each snapshot that was skipped because it
+already existed in the target repository. The amount of this output could
+practically bury the list of snapshots that were actually copied.
+
+From now on, the skipped snapshots are by default not printed at all, but
+this can be re-enabled by increasing the verbosity level of the command.
+
+https://github.com/restic/restic/issues/4033
+https://github.com/restic/restic/pull/4066
diff --git a/changelog/0.15.0_2023-01-12/issue-4085 b/changelog/0.15.0_2023-01-12/issue-4085
new file mode 100644
index 000000000..0ff53362d
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-4085
@@ -0,0 +1,10 @@
+Bugfix: Make `init` ignore "Access Denied" errors when creating S3 buckets
+
+In restic 0.9.0 through 0.13.0, the `init` command ignored some permission
+errors from S3 backends when trying to check for bucket existence, so that
+manually created buckets with custom permissions could be used for backups.
+
+This feature became broken in 0.14.0, but has now been restored again.
+
+https://github.com/restic/restic/issues/4085
+https://github.com/restic/restic/pull/4086
diff --git a/changelog/0.15.0_2023-01-12/issue-4103 b/changelog/0.15.0_2023-01-12/issue-4103
new file mode 100644
index 000000000..c2752f372
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-4103
@@ -0,0 +1,10 @@
+Bugfix: Don't generate negative UIDs and GIDs in tar files from `dump`
+
+When using a 32-bit build of restic, the `dump` command could in some cases
+create tar files containing negative UIDs and GIDs, which cannot be read by
+GNU tar. This corner case especially applies to backups from stdin on Windows.
+
+This is now fixed such that `dump` creates valid tar files in these cases too.
+
+https://github.com/restic/restic/issues/4103
+https://github.com/restic/restic/pull/4104
diff --git a/changelog/0.15.0_2023-01-12/issue-79 b/changelog/0.15.0_2023-01-12/issue-79
new file mode 100644
index 000000000..b3ba70ee8
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/issue-79
@@ -0,0 +1,17 @@
+Enhancement: Restore files with long runs of zeros as sparse files
+
+When using `restore --sparse`, the restorer may now write files containing long
+runs of zeros as sparse files (also called files with holes), where the zeros
+are not actually written to disk.
+
+How much space is saved by writing sparse files depends on the operating
+system, file system and the distribution of zeros in the file.
+
+During backup restic still reads the whole file including sparse regions, but
+with optimized processing speed of sparse regions.
+
+https://github.com/restic/restic/issues/79
+https://github.com/restic/restic/issues/3903
+https://github.com/restic/restic/pull/2601
+https://github.com/restic/restic/pull/3854
+https://forum.restic.net/t/sparse-file-support/1264
diff --git a/changelog/0.15.0_2023-01-12/pull-2750 b/changelog/0.15.0_2023-01-12/pull-2750
new file mode 100644
index 000000000..47eb2b877
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-2750
@@ -0,0 +1,7 @@
+Enhancement: Make backup file read concurrency configurable
+
+The `backup` command now supports a `--read-concurrency` option which allows
+tuning restic for very fast storage like NVMe disks by controlling the number
+of concurrent file reads during the backup process.
+
+https://github.com/restic/restic/pull/2750
diff --git a/changelog/0.15.0_2023-01-12/pull-3780 b/changelog/0.15.0_2023-01-12/pull-3780
new file mode 100644
index 000000000..1451a54d9
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-3780
@@ -0,0 +1,8 @@
+Bugfix: Make `restore` replace existing symlinks
+
+When restoring a symlink, restic used to report an error if the target path
+already existed. This has now been fixed such that the potentially existing
+target path is first removed before the symlink is restored.
+
+https://github.com/restic/restic/issues/2578
+https://github.com/restic/restic/pull/3780
diff --git a/changelog/0.15.0_2023-01-12/pull-3899 b/changelog/0.15.0_2023-01-12/pull-3899
new file mode 100644
index 000000000..d1f1678b6
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-3899
@@ -0,0 +1,6 @@
+Enhancement: Optimize prune memory usage
+
+The `prune` command needs large amounts of memory in order to determine what to
+keep and what to remove. This is now optimized to use up to 30% less memory.
+
+https://github.com/restic/restic/pull/3899
diff --git a/changelog/0.15.0_2023-01-12/pull-3905 b/changelog/0.15.0_2023-01-12/pull-3905
new file mode 100644
index 000000000..6a3189e84
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-3905
@@ -0,0 +1,6 @@
+Enhancement: Improve speed of parent snapshot detection in `backup` command
+
+Backing up a large number of files using `--files-from-verbatim` or `--files-from-raw`
+options could require a long time to find the parent snapshot. This has been improved.
+
+https://github.com/restic/restic/pull/3905
diff --git a/changelog/0.15.0_2023-01-12/pull-3915 b/changelog/0.15.0_2023-01-12/pull-3915
new file mode 100644
index 000000000..54e34ac5e
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-3915
@@ -0,0 +1,12 @@
+Enhancement: Add compression statistics to the `stats` command
+
+When executed with `--mode raw-data` on a repository that supports compression,
+the `stats` command now calculates and displays, for the selected repository or
+snapshots: the uncompressed size of the data; the compression progress
+(percentage of data that has been compressed); the compression ratio of the
+compressed data; the total space saving.
+
+It also takes into account both the compressed and uncompressed data if the
+repository is only partially compressed.
+
+https://github.com/restic/restic/pull/3915
diff --git a/changelog/0.15.0_2023-01-12/pull-3925 b/changelog/0.15.0_2023-01-12/pull-3925
new file mode 100644
index 000000000..853e7916e
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-3925
@@ -0,0 +1,6 @@
+Enhancement: Provide command completion for PowerShell
+
+Restic already provided generation of completion files for bash, fish and zsh.
+Now powershell is supported, too.
+
+https://github.com/restic/restic/pull/3925/files
diff --git a/changelog/0.15.0_2023-01-12/pull-3931 b/changelog/0.15.0_2023-01-12/pull-3931
new file mode 100644
index 000000000..2c38e066a
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-3931
@@ -0,0 +1,10 @@
+Enhancement: Allow `backup` file tree scanner to be disabled
+
+The `backup` command walks the file tree in a separate scanner process to find
+the total size and file/directory count, and uses this to provide an ETA. This
+can slow down backups, especially of network filesystems.
+
+The command now has a new option `--no-scan` which can be used to disable this
+scanning in order to speed up backups when needed.
+
+https://github.com/restic/restic/pull/3931
diff --git a/changelog/0.15.0_2023-01-12/pull-3943 b/changelog/0.15.0_2023-01-12/pull-3943
new file mode 100644
index 000000000..c2e813d2f
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-3943
@@ -0,0 +1,9 @@
+Enhancement: Ignore additional/unknown files in repository
+
+If a restic repository had additional files in it (not created by restic),
+commands like `find` and `restore` could become confused and fail with an
+`multiple IDs with prefix "12345678" found` error. These commands now
+ignore such additional files.
+
+https://github.com/restic/restic/pull/3943
+https://forum.restic.net/t/which-protocol-should-i-choose-for-remote-linux-backups/5446/17
diff --git a/changelog/0.15.0_2023-01-12/pull-3951 b/changelog/0.15.0_2023-01-12/pull-3951
new file mode 100644
index 000000000..aa913c2f8
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-3951
@@ -0,0 +1,7 @@
+Bugfix: Make `ls` return exit code 1 if snapshot cannot be loaded
+
+The `ls` command used to show a warning and return exit code 0 when failing
+to load a snapshot. This has now been fixed such that it instead returns exit
+code 1 (still showing a warning).
+
+https://github.com/restic/restic/pull/3951
diff --git a/changelog/0.15.0_2023-01-12/pull-3955 b/changelog/0.15.0_2023-01-12/pull-3955
new file mode 100644
index 000000000..b8eba1246
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-3955
@@ -0,0 +1,9 @@
+Enhancement: Improve `backup` performance for small files
+
+When backing up small files restic was slower than it could be. In particular
+this affected backups using maximum compression.
+
+This has been fixed by reworking the internal parallelism of the backup
+command, making it back up small files around two times faster.
+
+https://github.com/restic/restic/pull/3955
diff --git a/changelog/0.15.0_2023-01-12/pull-4041 b/changelog/0.15.0_2023-01-12/pull-4041
new file mode 100644
index 000000000..9ce2b4858
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-4041
@@ -0,0 +1,7 @@
+Change: Update dependencies and require Go 1.18 or newer
+
+Most dependencies have been updated. Since some libraries require newer language
+features, support for Go 1.15-1.17 has been dropped, which means that restic now
+requires at least Go 1.18 to build.
+
+https://github.com/restic/restic/pull/4041
diff --git a/changelog/0.15.0_2023-01-12/pull-4100 b/changelog/0.15.0_2023-01-12/pull-4100
new file mode 100644
index 000000000..e214cbb17
--- /dev/null
+++ b/changelog/0.15.0_2023-01-12/pull-4100
@@ -0,0 +1,11 @@
+Bugfix: Make `self-update` enabled by default only in release builds
+
+The `self-update` command was previously included by default in all builds of
+restic as opposed to only in official release builds, even if the `selfupdate`
+tag was not explicitly enabled when building.
+
+This has now been corrected, and the `self-update` command is only available
+if restic was built with `-tags selfupdate` (as done for official release
+builds by `build.go`).
+
+https://github.com/restic/restic/pull/4100
diff --git a/changelog/0.15.1_2023-01-30/issue-3750 b/changelog/0.15.1_2023-01-30/issue-3750
new file mode 100644
index 000000000..bcb0feaf2
--- /dev/null
+++ b/changelog/0.15.1_2023-01-30/issue-3750
@@ -0,0 +1,10 @@
+Bugfix: Remove `b2_download_file_by_name: 404` warning from B2 backend
+
+In some cases the B2 backend could print `b2_download_file_by_name: 404: :
+b2.b2err` warnings. These are only debug messages and can be safely ignored.
+
+Restic now uses an updated library for accessing B2, which removes the warning.
+
+https://github.com/restic/restic/issues/3750
+https://github.com/restic/restic/issues/4144
+https://github.com/restic/restic/pull/4146
diff --git a/changelog/0.15.1_2023-01-30/issue-4147 b/changelog/0.15.1_2023-01-30/issue-4147
new file mode 100644
index 000000000..cc3a39bee
--- /dev/null
+++ b/changelog/0.15.1_2023-01-30/issue-4147
@@ -0,0 +1,7 @@
+Bugfix: Make `prune --quiet` not print progress bar
+
+A regression in restic 0.15.0 caused `prune --quiet` to show a progress bar
+while deciding how to process each pack files. This has now been fixed.
+
+https://github.com/restic/restic/issues/4147
+https://github.com/restic/restic/pull/4153
diff --git a/changelog/0.15.1_2023-01-30/pull-4152 b/changelog/0.15.1_2023-01-30/pull-4152
new file mode 100644
index 000000000..1a067f2ef
--- /dev/null
+++ b/changelog/0.15.1_2023-01-30/pull-4152
@@ -0,0 +1,19 @@
+Enhancement: Ignore empty lock files
+
+With restic 0.15.0 the checks for stale locks became much stricter than before.
+In particular, empty or unreadable locks were no longer silently ignored. This
+made restic to complain with `Load(<lock/1234567812>, 0, 0) returned error,
+retrying after 552.330144ms: load(<lock/1234567812>): invalid data returned`
+and fail in the end.
+
+The error message is now clarified and the implementation changed to ignore
+empty lock files which are sometimes created as the result of a failed uploads
+on some backends.
+
+Please note that unreadable lock files still have to cleaned up manually. To do
+so, you can run `restic unlock --remove-all` which removes all existing lock
+files. But first make sure that no other restic process is currently using the
+repository.
+
+https://github.com/restic/restic/issues/4143
+https://github.com/restic/restic/pull/4152
diff --git a/changelog/0.15.1_2023-01-30/pull-4163 b/changelog/0.15.1_2023-01-30/pull-4163
new file mode 100644
index 000000000..ca32ab554
--- /dev/null
+++ b/changelog/0.15.1_2023-01-30/pull-4163
@@ -0,0 +1,13 @@
+Bugfix: Make `self-update --output` work with new filename on Windows
+
+Since restic 0.14.0 the `self-update` command did not work when a custom output
+filename was specified via the `--output` option. This has now been fixed.
+
+As a workaround, either use an older restic version to run the self-update or
+create an empty file with the output filename before updating e.g. using CMD:
+
+`type nul > new-file.exe`
+`restic self-update --output new-file.exe`
+
+https://github.com/restic/restic/pull/4163
+https://forum.restic.net/t/self-update-windows-started-failing-after-release-of-0-15/5836
diff --git a/changelog/0.15.1_2023-01-30/pull-4167 b/changelog/0.15.1_2023-01-30/pull-4167
new file mode 100644
index 000000000..490327170
--- /dev/null
+++ b/changelog/0.15.1_2023-01-30/pull-4167
@@ -0,0 +1,6 @@
+Bugfix: Add missing ETA in `backup` progress bar
+
+A regression in restic 0.15.0 caused the ETA to be missing from the progress
+bar displayed by the `backup` command. This has now been fixed.
+
+https://github.com/restic/restic/pull/4167
diff --git a/cmd/restic/cleanup.go b/cmd/restic/cleanup.go
index 67a007d59..61af72802 100644
--- a/cmd/restic/cleanup.go
+++ b/cmd/restic/cleanup.go
@@ -11,7 +11,7 @@ import (
var cleanupHandlers struct {
sync.Mutex
- list []func() error
+ list []func(code int) (int, error)
done bool
ch chan os.Signal
}
@@ -25,7 +25,7 @@ func init() {
// AddCleanupHandler adds the function f to the list of cleanup handlers so
// that it is executed when all the cleanup handlers are run, e.g. when SIGINT
// is received.
-func AddCleanupHandler(f func() error) {
+func AddCleanupHandler(f func(code int) (int, error)) {
cleanupHandlers.Lock()
defer cleanupHandlers.Unlock()
@@ -36,22 +36,24 @@ func AddCleanupHandler(f func() error) {
}
// RunCleanupHandlers runs all registered cleanup handlers
-func RunCleanupHandlers() {
+func RunCleanupHandlers(code int) int {
cleanupHandlers.Lock()
defer cleanupHandlers.Unlock()
if cleanupHandlers.done {
- return
+ return code
}
cleanupHandlers.done = true
for _, f := range cleanupHandlers.list {
- err := f()
+ var err error
+ code, err = f(code)
if err != nil {
Warnf("error in cleanup handler: %v\n", err)
}
}
cleanupHandlers.list = nil
+ return code
}
// CleanupHandler handles the SIGINT signals.
@@ -75,6 +77,6 @@ func CleanupHandler(c <-chan os.Signal) {
// Exit runs the cleanup handlers and then terminates the process with the
// given exit code.
func Exit(code int) {
- RunCleanupHandlers()
+ code = RunCleanupHandlers(code)
os.Exit(code)
}
diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go
index 0b33f2263..e59f503db 100644
--- a/cmd/restic/cmd_backup.go
+++ b/cmd/restic/cmd_backup.go
@@ -6,11 +6,11 @@ import (
"context"
"fmt"
"io"
- "io/ioutil"
"os"
"path"
"path/filepath"
"runtime"
+ "strconv"
"strings"
"sync"
"time"
@@ -21,11 +21,11 @@ import (
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
- "github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/textfile"
+ "github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/backup"
"github.com/restic/restic/internal/ui/termstatus"
)
@@ -56,8 +56,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
+ ctx := cmd.Context()
var wg sync.WaitGroup
- cancelCtx, cancel := context.WithCancel(globalOptions.ctx)
+ cancelCtx, cancel := context.WithCancel(ctx)
defer func() {
// shutdown termstatus
cancel()
@@ -71,35 +72,43 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
term.Run(cancelCtx)
}()
- return runBackup(backupOptions, globalOptions, term, args)
+ // use the terminal for stdout/stderr
+ prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr
+ defer func() {
+ globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr
+ }()
+ stdioWrapper := ui.NewStdioWrapper(term)
+ globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr()
+
+ return runBackup(ctx, backupOptions, globalOptions, term, args)
},
}
// BackupOptions bundles all options for the backup command.
type BackupOptions struct {
- Parent string
- Force bool
- Excludes []string
- InsensitiveExcludes []string
- ExcludeFiles []string
- InsensitiveExcludeFiles []string
- ExcludeOtherFS bool
- ExcludeIfPresent []string
- ExcludeCaches bool
- ExcludeLargerThan string
- Stdin bool
- StdinFilename string
- Tags restic.TagLists
- Host string
- FilesFrom []string
- FilesFromVerbatim []string
- FilesFromRaw []string
- TimeStamp string
- WithAtime bool
- IgnoreInode bool
- IgnoreCtime bool
- UseFsSnapshot bool
- DryRun bool
+ excludePatternOptions
+
+ Parent string
+ Force bool
+ ExcludeOtherFS bool
+ ExcludeIfPresent []string
+ ExcludeCaches bool
+ ExcludeLargerThan string
+ Stdin bool
+ StdinFilename string
+ Tags restic.TagLists
+ Host string
+ FilesFrom []string
+ FilesFromVerbatim []string
+ FilesFromRaw []string
+ TimeStamp string
+ WithAtime bool
+ IgnoreInode bool
+ IgnoreCtime bool
+ UseFsSnapshot bool
+ DryRun bool
+ ReadConcurrency uint
+ NoScan bool
}
var backupOptions BackupOptions
@@ -113,10 +122,9 @@ func init() {
f := cmdBackup.Flags()
f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)")
f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
- f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
- f.StringArrayVar(&backupOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
- f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
- f.StringArrayVar(&backupOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
+
+ initExcludePatternOptions(f, &backupOptions.excludePatternOptions)
+
f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file. See https://bford.info/cachedir/ for the Cache Directory Tagging Standard`)
@@ -124,7 +132,7 @@ func init() {
f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
-
+ f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
err := f.MarkDeprecated("hostname", "use --host")
@@ -132,7 +140,6 @@ func init() {
// MarkDeprecated only returns an error when the flag could not be found
panic(err)
}
-
f.StringArrayVar(&backupOptions.FilesFrom, "files-from", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
f.StringArrayVar(&backupOptions.FilesFromVerbatim, "files-from-verbatim", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
@@ -141,9 +148,14 @@ func init() {
f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
+ f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
if runtime.GOOS == "windows" {
f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
}
+
+ // parse read concurrency from env, on error the default value will be used
+ readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
+ backupOptions.ReadConcurrency = uint(readConcurrency)
}
// filterExisting returns a slice of all existing items, or an error if no
@@ -183,7 +195,7 @@ func readLines(filename string) ([]string, error) {
)
if filename == "-" {
- data, err = ioutil.ReadAll(os.Stdin)
+ data, err = io.ReadAll(os.Stdin)
} else {
data, err = textfile.Read(filename)
}
@@ -260,6 +272,10 @@ func readFilenamesRaw(r io.Reader) (names []string, err error) {
// Check returns an error when an invalid combination of options was set.
func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
if gopts.password == "" {
+ if opts.Stdin {
+ return errors.Fatal("cannot read both password and data from stdin")
+ }
+
filesFrom := append(append(opts.FilesFrom, opts.FilesFromVerbatim...), opts.FilesFromRaw...)
for _, filename := range filesFrom {
if filename == "-" {
@@ -300,48 +316,11 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
fs = append(fs, f)
}
- // add patterns from file
- if len(opts.ExcludeFiles) > 0 {
- excludes, err := readExcludePatternsFromFiles(opts.ExcludeFiles)
- if err != nil {
- return nil, err
- }
-
- if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid {
- return nil, errors.Fatalf("--exclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
- }
-
- opts.Excludes = append(opts.Excludes, excludes...)
- }
-
- if len(opts.InsensitiveExcludeFiles) > 0 {
- excludes, err := readExcludePatternsFromFiles(opts.InsensitiveExcludeFiles)
- if err != nil {
- return nil, err
- }
-
- if valid, invalidPatterns := filter.ValidatePatterns(excludes); !valid {
- return nil, errors.Fatalf("--iexclude-file: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
- }
-
- opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
- }
-
- if len(opts.InsensitiveExcludes) > 0 {
- if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveExcludes); !valid {
- return nil, errors.Fatalf("--iexclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
- }
-
- fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
- }
-
- if len(opts.Excludes) > 0 {
- if valid, invalidPatterns := filter.ValidatePatterns(opts.Excludes); !valid {
- return nil, errors.Fatalf("--exclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
- }
-
- fs = append(fs, rejectByPattern(opts.Excludes))
+ fsPatterns, err := opts.excludePatternOptions.CollectPatterns()
+ if err != nil {
+ return nil, err
}
+ fs = append(fs, fsPatterns...)
if opts.ExcludeCaches {
opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55")
@@ -382,53 +361,6 @@ func collectRejectFuncs(opts BackupOptions, repo *repository.Repository, targets
return fs, nil
}
-// readExcludePatternsFromFiles reads all exclude files and returns the list of
-// exclude patterns. For each line, leading and trailing white space is removed
-// and comment lines are ignored. For each remaining pattern, environment
-// variables are resolved. For adding a literal dollar sign ($), write $$ to
-// the file.
-func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) {
- getenvOrDollar := func(s string) string {
- if s == "$" {
- return "$"
- }
- return os.Getenv(s)
- }
-
- var excludes []string
- for _, filename := range excludeFiles {
- err := func() (err error) {
- data, err := textfile.Read(filename)
- if err != nil {
- return err
- }
-
- scanner := bufio.NewScanner(bytes.NewReader(data))
- for scanner.Scan() {
- line := strings.TrimSpace(scanner.Text())
-
- // ignore empty lines
- if line == "" {
- continue
- }
-
- // strip comments
- if strings.HasPrefix(line, "#") {
- continue
- }
-
- line = os.Expand(line, getenvOrDollar)
- excludes = append(excludes, line)
- }
- return scanner.Err()
- }()
- if err != nil {
- return nil, err
- }
- }
- return excludes, nil
-}
-
// collectTargets returns a list of target files/dirs from several sources.
func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
if opts.Stdin {
@@ -451,7 +383,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
var expanded []string
expanded, err := filepath.Glob(line)
if err != nil {
- return nil, errors.WithMessage(err, fmt.Sprintf("pattern: %s", line))
+ return nil, fmt.Errorf("pattern: %s: %w", line, err)
}
if len(expanded) == 0 {
Warnf("pattern %q does not match any files, skipping\n", line)
@@ -498,31 +430,24 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
// parent returns the ID of the parent snapshot. If there is none, nil is
// returned.
-func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (parentID *restic.ID, err error) {
- // Force using a parent
- if !opts.Force && opts.Parent != "" {
- id, err := restic.FindSnapshot(ctx, repo.Backend(), opts.Parent)
- if err != nil {
- return nil, errors.Fatalf("invalid id %q: %v", opts.Parent, err)
- }
-
- parentID = &id
+func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
+ if opts.Force {
+ return nil, nil
}
- // Find last snapshot to set it as parent, if not already set
- if !opts.Force && parentID == nil {
- id, err := restic.FindLatestSnapshot(ctx, repo.Backend(), repo, targets, []restic.TagList{}, []string{opts.Host}, &timeStampLimit)
- if err == nil {
- parentID = &id
- } else if err != restic.ErrNoSnapshotFound {
- return nil, err
- }
+ snName := opts.Parent
+ if snName == "" {
+ snName = "latest"
}
-
- return parentID, nil
+ sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, []string{opts.Host}, []restic.TagList{}, targets, &timeStampLimit, snName)
+ // Snapshot not found is ok if no explicit parent was set
+ if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
+ err = nil
+ }
+ return sn, err
}
-func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
+func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
err := opts.Check(gopts, args)
if err != nil {
return err
@@ -545,7 +470,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
Verbosef("open repository\n")
}
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@@ -556,31 +481,18 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
} else {
progressPrinter = backup.NewTextProgress(term, gopts.verbosity)
}
- progressReporter := backup.NewProgress(progressPrinter)
+ progressReporter := backup.NewProgress(progressPrinter,
+ calculateProgressInterval(!gopts.Quiet, gopts.JSON))
+ defer progressReporter.Done()
if opts.DryRun {
repo.SetDryRun()
- progressReporter.SetDryRun()
}
- // use the terminal for stdout/stderr
- prevStdout, prevStderr := gopts.stdout, gopts.stderr
- defer func() {
- gopts.stdout, gopts.stderr = prevStdout, prevStderr
- }()
- gopts.stdout, gopts.stderr = progressPrinter.Stdout(), progressPrinter.Stderr()
-
- progressReporter.SetMinUpdatePause(calculateProgressInterval(!gopts.Quiet, gopts.JSON))
-
- wg, wgCtx := errgroup.WithContext(gopts.ctx)
- cancelCtx, cancel := context.WithCancel(wgCtx)
- defer cancel()
- wg.Go(func() error { return progressReporter.Run(cancelCtx) })
-
if !gopts.JSON {
progressPrinter.V("lock repository")
}
- lock, err := lockRepo(gopts.ctx, repo)
+ lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -598,16 +510,16 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
return err
}
- var parentSnapshotID *restic.ID
+ var parentSnapshot *restic.Snapshot
if !opts.Stdin {
- parentSnapshotID, err = findParentSnapshot(gopts.ctx, repo, opts, targets, timeStamp)
+ parentSnapshot, err = findParentSnapshot(ctx, repo, opts, targets, timeStamp)
if err != nil {
return err
}
if !gopts.JSON {
- if parentSnapshotID != nil {
- progressPrinter.P("using parent snapshot %v\n", parentSnapshotID.Str())
+ if parentSnapshot != nil {
+ progressPrinter.P("using parent snapshot %v\n", parentSnapshot.ID().Str())
} else {
progressPrinter.P("no parent snapshot found, will read all files\n")
}
@@ -617,7 +529,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
if !gopts.JSON {
progressPrinter.V("load index files")
}
- err = repo.LoadIndex(gopts.ctx)
+ err = repo.LoadIndex(ctx)
if err != nil {
return err
}
@@ -674,18 +586,24 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
targets = []string{filename}
}
- sc := archiver.NewScanner(targetFS)
- sc.SelectByName = selectByNameFilter
- sc.Select = selectFilter
- sc.Error = progressReporter.ScannerError
- sc.Result = progressReporter.ReportTotal
+ wg, wgCtx := errgroup.WithContext(ctx)
+ cancelCtx, cancel := context.WithCancel(wgCtx)
+ defer cancel()
- if !gopts.JSON {
- progressPrinter.V("start scan on %v", targets)
+ if !opts.NoScan {
+ sc := archiver.NewScanner(targetFS)
+ sc.SelectByName = selectByNameFilter
+ sc.Select = selectFilter
+ sc.Error = progressPrinter.ScannerError
+ sc.Result = progressReporter.ReportTotal
+
+ if !gopts.JSON {
+ progressPrinter.V("start scan on %v", targets)
+ }
+ wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
}
- wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
- arch := archiver.New(repo, targetFS, archiver.Options{})
+ arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency})
arch.SelectByName = selectByNameFilter
arch.Select = selectFilter
arch.WithAtime = opts.WithAtime
@@ -707,22 +625,18 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
arch.ChangeIgnoreFlags |= archiver.ChangeIgnoreCtime
}
- if parentSnapshotID == nil {
- parentSnapshotID = &restic.ID{}
- }
-
snapshotOpts := archiver.SnapshotOptions{
Excludes: opts.Excludes,
Tags: opts.Tags.Flatten(),
Time: timeStamp,
Hostname: opts.Host,
- ParentSnapshot: *parentSnapshotID,
+ ParentSnapshot: parentSnapshot,
}
if !gopts.JSON {
progressPrinter.V("start backup on %v", targets)
}
- _, id, err := arch.Snapshot(gopts.ctx, targets, snapshotOpts)
+ _, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
// cleanly shutdown all running goroutines
cancel()
@@ -736,7 +650,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
}
// Report finished execution
- progressReporter.Finish(id)
+ progressReporter.Finish(id, opts.DryRun)
if !gopts.JSON && !opts.DryRun {
progressPrinter.P("snapshot %s saved\n", id.Str())
}
diff --git a/cmd/restic/cmd_backup_test.go b/cmd/restic/cmd_backup_test.go
index 49642b872..5cbc42436 100644
--- a/cmd/restic/cmd_backup_test.go
+++ b/cmd/restic/cmd_backup_test.go
@@ -14,8 +14,7 @@ import (
)
func TestCollectTargets(t *testing.T) {
- dir, cleanup := rtest.TempDir(t)
- defer cleanup()
+ dir := rtest.TempDir(t)
fooSpace := "foo "
barStar := "bar*" // Must sort before the others, below.
diff --git a/cmd/restic/cmd_cache.go b/cmd/restic/cmd_cache.go
index 50a738d5c..334063fdc 100644
--- a/cmd/restic/cmd_cache.go
+++ b/cmd/restic/cmd_cache.go
@@ -11,6 +11,7 @@ import (
"github.com/restic/restic/internal/cache"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
+ "github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/table"
"github.com/spf13/cobra"
)
@@ -138,7 +139,7 @@ func runCache(opts CacheOptions, gopts GlobalOptions, args []string) error {
if err != nil {
return err
}
- size = fmt.Sprintf("%11s", formatBytes(uint64(bytes)))
+ size = fmt.Sprintf("%11s", ui.FormatBytes(uint64(bytes)))
}
name := entry.Name()
diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go
index 991df86a2..f46502d5a 100644
--- a/cmd/restic/cmd_cat.go
+++ b/cmd/restic/cmd_cat.go
@@ -1,6 +1,7 @@
package main
import (
+ "context"
"encoding/json"
"github.com/spf13/cobra"
@@ -24,7 +25,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runCat(globalOptions, args)
+ return runCat(cmd.Context(), globalOptions, args)
},
}
@@ -32,40 +33,32 @@ func init() {
cmdRoot.AddCommand(cmdCat)
}
-func runCat(gopts GlobalOptions, args []string) error {
+func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
return errors.Fatal("type or ID not specified")
}
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
- lock, err := lockRepo(gopts.ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepo(ctx, repo)
+ defer unlockRepo(lock)
if err != nil {
return err
}
-
- defer unlockRepo(lock)
}
tpe := args[0]
var id restic.ID
- if tpe != "masterkey" && tpe != "config" {
+ if tpe != "masterkey" && tpe != "config" && tpe != "snapshot" {
id, err = restic.ParseID(args[1])
if err != nil {
- if tpe != "snapshot" {
- return errors.Fatalf("unable to parse ID: %v\n", err)
- }
-
- // find snapshot id with prefix
- id, err = restic.FindSnapshot(gopts.ctx, repo.Backend(), args[1])
- if err != nil {
- return errors.Fatalf("could not find snapshot: %v\n", err)
- }
+ return errors.Fatalf("unable to parse ID: %v\n", err)
}
}
@@ -79,7 +72,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "index":
- buf, err := repo.LoadUnpacked(gopts.ctx, restic.IndexFile, id, nil)
+ buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
if err != nil {
return err
}
@@ -87,9 +80,9 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "snapshot":
- sn, err := restic.LoadSnapshot(gopts.ctx, repo, id)
+ sn, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
if err != nil {
- return err
+ return errors.Fatalf("could not find snapshot: %v\n", err)
}
buf, err := json.MarshalIndent(sn, "", " ")
@@ -100,19 +93,12 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "key":
- h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
- buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
- if err != nil {
- return err
- }
-
- key := &repository.Key{}
- err = json.Unmarshal(buf, key)
+ key, err := repository.LoadKey(ctx, repo, id)
if err != nil {
return err
}
- buf, err = json.MarshalIndent(&key, "", " ")
+ buf, err := json.MarshalIndent(&key, "", " ")
if err != nil {
return err
}
@@ -128,7 +114,7 @@ func runCat(gopts GlobalOptions, args []string) error {
Println(string(buf))
return nil
case "lock":
- lock, err := restic.LoadLock(gopts.ctx, repo, id)
+ lock, err := restic.LoadLock(ctx, repo, id)
if err != nil {
return err
}
@@ -143,7 +129,7 @@ func runCat(gopts GlobalOptions, args []string) error {
case "pack":
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
- buf, err := backend.LoadAll(gopts.ctx, nil, repo.Backend(), h)
+ buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
if err != nil {
return err
}
@@ -157,7 +143,7 @@ func runCat(gopts GlobalOptions, args []string) error {
return err
case "blob":
- err = repo.LoadIndex(gopts.ctx)
+ err = repo.LoadIndex(ctx)
if err != nil {
return err
}
@@ -168,7 +154,7 @@ func runCat(gopts GlobalOptions, args []string) error {
continue
}
- buf, err := repo.LoadBlob(gopts.ctx, t, id, nil)
+ buf, err := repo.LoadBlob(ctx, t, id, nil)
if err != nil {
return err
}
diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go
index 80b92862d..be9dd5130 100644
--- a/cmd/restic/cmd_check.go
+++ b/cmd/restic/cmd_check.go
@@ -1,8 +1,9 @@
package main
import (
- "io/ioutil"
+ "context"
"math/rand"
+ "os"
"strconv"
"strings"
"sync"
@@ -34,7 +35,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runCheck(checkOptions, globalOptions, args)
+ return runCheck(cmd.Context(), checkOptions, globalOptions, args)
},
PreRunE: func(cmd *cobra.Command, args []string) error {
return checkFlags(checkOptions)
@@ -170,7 +171,7 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
}
// use a cache in a temporary directory
- tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
+ tempdir, err := os.MkdirTemp(cachedir, "restic-check-cache-")
if err != nil {
// if an error occurs, don't use any cache
Warnf("unable to create temporary directory for cache during check, disabling cache: %v\n", err)
@@ -191,25 +192,26 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
return cleanup
}
-func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
+func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string) error {
if len(args) != 0 {
return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
}
cleanup := prepareCheckCache(opts, &gopts)
- AddCleanupHandler(func() error {
+ AddCleanupHandler(func(code int) (int, error) {
cleanup()
- return nil
+ return code, nil
})
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
Verbosef("create exclusive lock for repository\n")
- lock, err := lockRepoExclusive(gopts.ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -217,13 +219,13 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
}
chkr := checker.New(repo, opts.CheckUnused)
- err = chkr.LoadSnapshots(gopts.ctx)
+ err = chkr.LoadSnapshots(ctx)
if err != nil {
return err
}
Verbosef("load indexes\n")
- hints, errs := chkr.LoadIndex(gopts.ctx)
+ hints, errs := chkr.LoadIndex(ctx)
errorsFound := false
suggestIndexRebuild := false
@@ -243,7 +245,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
}
if suggestIndexRebuild {
- Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n")
+ Printf("Duplicate packs/old indexes are non-critical, you can run `restic rebuild-index' to correct this.\n")
}
if mixedFound {
Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
@@ -260,13 +262,13 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
errChan := make(chan error)
Verbosef("check all packs\n")
- go chkr.Packs(gopts.ctx, errChan)
+ go chkr.Packs(ctx, errChan)
for err := range errChan {
if checker.IsOrphanedPack(err) {
orphanedPacks++
Verbosef("%v\n", err)
- } else if _, ok := err.(*checker.ErrLegacyLayout); ok {
+ } else if err == checker.ErrLegacyLayout {
Verbosef("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n")
} else {
errorsFound = true
@@ -287,13 +289,17 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
defer wg.Done()
bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
defer bar.Done()
- chkr.Structure(gopts.ctx, bar, errChan)
+ chkr.Structure(ctx, bar, errChan)
}()
for err := range errChan {
errorsFound = true
if e, ok := err.(*checker.TreeError); ok {
- Warnf("error for tree %v:\n", e.ID.Str())
+ var clean string
+ if stdoutCanUpdateStatus() {
+ clean = clearLine(0)
+ }
+ Warnf(clean+"error for tree %v:\n", e.ID.Str())
for _, treeErr := range e.Errors {
Warnf(" %v\n", treeErr)
}
@@ -308,7 +314,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
wg.Wait()
if opts.CheckUnused {
- for _, id := range chkr.UnusedBlobs(gopts.ctx) {
+ for _, id := range chkr.UnusedBlobs(ctx) {
Verbosef("unused blob %v\n", id)
errorsFound = true
}
@@ -320,7 +326,7 @@ func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
p := newProgressMax(!gopts.Quiet, packCount, "packs")
errChan := make(chan error)
- go chkr.ReadPacks(gopts.ctx, packs, p, errChan)
+ go chkr.ReadPacks(ctx, packs, p, errChan)
for err := range errChan {
errorsFound = true
diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go
index 98007c8d2..14ab1917a 100644
--- a/cmd/restic/cmd_copy.go
+++ b/cmd/restic/cmd_copy.go
@@ -32,16 +32,14 @@ This can be mitigated by the "--copy-chunker-params" option when initializing a
new destination repository using the "init" command.
`,
RunE: func(cmd *cobra.Command, args []string) error {
- return runCopy(copyOptions, globalOptions, args)
+ return runCopy(cmd.Context(), copyOptions, globalOptions, args)
},
}
// CopyOptions bundles all options for the copy command.
type CopyOptions struct {
secondaryRepoOptions
- Hosts []string
- Tags restic.TagLists
- Paths []string
+ snapshotFilterOptions
}
var copyOptions CopyOptions
@@ -51,12 +49,10 @@ func init() {
f := cmdCopy.Flags()
initSecondaryRepoOptions(f, ©Options.secondaryRepoOptions, "destination", "to copy snapshots from")
- f.StringArrayVarP(©Options.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
- f.Var(©Options.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
- f.StringArrayVar(©Options.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
+ initMultiSnapshotFilterOptions(f, ©Options.snapshotFilterOptions, true)
}
-func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
+func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
if err != nil {
return err
@@ -66,28 +62,26 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
gopts, secondaryGopts = secondaryGopts, gopts
}
- ctx, cancel := context.WithCancel(gopts.ctx)
- defer cancel()
-
- srcRepo, err := OpenRepository(gopts)
+ srcRepo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
- dstRepo, err := OpenRepository(secondaryGopts)
+ dstRepo, err := OpenRepository(ctx, secondaryGopts)
if err != nil {
return err
}
if !gopts.NoLock {
- srcLock, err := lockRepo(ctx, srcRepo)
+ var srcLock *restic.Lock
+ srcLock, ctx, err = lockRepo(ctx, srcRepo)
defer unlockRepo(srcLock)
if err != nil {
return err
}
}
- dstLock, err := lockRepo(ctx, dstRepo)
+ dstLock, ctx, err := lockRepo(ctx, dstRepo)
defer unlockRepo(dstLock)
if err != nil {
return err
@@ -126,7 +120,6 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
visitedTrees := restic.NewIDSet()
for sn := range FindFilteredSnapshots(ctx, srcSnapshotLister, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
- Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
// check whether the destination has a snapshot with the same persistent ID which has similar snapshot fields
srcOriginal := *sn.ID()
@@ -137,7 +130,8 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
isCopy := false
for _, originalSn := range originalSns {
if similarSnapshots(originalSn, sn) {
- Verbosef("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
+ Verboseff("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+ Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
isCopy = true
break
}
@@ -146,6 +140,7 @@ func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
continue
}
}
+ Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
Verbosef(" copy started, this may take a while...\n")
if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
return err
diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go
index ac4996b7c..c8626d46c 100644
--- a/cmd/restic/cmd_debug.go
+++ b/cmd/restic/cmd_debug.go
@@ -13,6 +13,7 @@ import (
"os"
"runtime"
"sort"
+ "sync"
"time"
"github.com/klauspost/compress/zstd"
@@ -22,6 +23,7 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/errors"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@@ -46,7 +48,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runDebugDump(globalOptions, args)
+ return runDebugDump(cmd.Context(), globalOptions, args)
},
}
@@ -104,10 +106,9 @@ type Blob struct {
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
- return repo.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
- h := restic.Handle{Type: restic.PackFile, Name: id.String()}
-
- blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), size)
+ var m sync.Mutex
+ return restic.ParallelList(ctx, repo.Backend(), restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
+ blobs, _, err := repo.ListPack(ctx, id, size)
if err != nil {
Warnf("error for pack %v: %v\n", id.Str(), err)
return nil
@@ -126,12 +127,14 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
}
}
+ m.Lock()
+ defer m.Unlock()
return prettyPrintJSON(wr, p)
})
}
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
- return repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
+ return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
Printf("index_id: %v\n", id)
if err != nil {
return err
@@ -141,18 +144,19 @@ func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) erro
})
}
-func runDebugDump(gopts GlobalOptions, args []string) error {
+func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) != 1 {
return errors.Fatal("type not specified")
}
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
- lock, err := lockRepo(gopts.ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -163,20 +167,20 @@ func runDebugDump(gopts GlobalOptions, args []string) error {
switch tpe {
case "indexes":
- return dumpIndexes(gopts.ctx, repo, gopts.stdout)
+ return dumpIndexes(ctx, repo, gopts.stdout)
case "snapshots":
- return debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
+ return debugPrintSnapshots(ctx, repo, gopts.stdout)
case "packs":
- return printPacks(gopts.ctx, repo, gopts.stdout)
+ return printPacks(ctx, repo, gopts.stdout)
case "all":
Printf("snapshots:\n")
- err := debugPrintSnapshots(gopts.ctx, repo, gopts.stdout)
+ err := debugPrintSnapshots(ctx, repo, gopts.stdout)
if err != nil {
return err
}
Printf("\nindexes:\n")
- err = dumpIndexes(gopts.ctx, repo, gopts.stdout)
+ err = dumpIndexes(ctx, repo, gopts.stdout)
if err != nil {
return err
}
@@ -192,7 +196,7 @@ var cmdDebugExamine = &cobra.Command{
Short: "Examine a pack file",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runDebugExamine(globalOptions, args)
+ return runDebugExamine(cmd.Context(), globalOptions, args)
},
}
@@ -311,97 +315,104 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
return out
}
-func loadBlobs(ctx context.Context, repo restic.Repository, pack restic.ID, list []restic.Blob) error {
+func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
dec, err := zstd.NewReader(nil)
if err != nil {
panic(err)
}
be := repo.Backend()
h := restic.Handle{
- Name: pack.String(),
+ Name: packID.String(),
Type: restic.PackFile,
}
- for _, blob := range list {
- Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
- buf := make([]byte, blob.Length)
- err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
- n, err := io.ReadFull(rd, buf)
- if err != nil {
- return fmt.Errorf("read error after %d bytes: %v", n, err)
- }
- return nil
- })
- if err != nil {
- Warnf("error read: %v\n", err)
- continue
- }
- key := repo.Key()
+ wg, ctx := errgroup.WithContext(ctx)
- nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
- plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
- outputPrefix := ""
- filePrefix := ""
- if err != nil {
- Warnf("error decrypting blob: %v\n", err)
- if tryRepair || repairByte {
- plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte)
- }
- if plaintext != nil {
- outputPrefix = "repaired "
- filePrefix = "repaired-"
- } else {
- plaintext = decryptUnsigned(ctx, key, buf)
- err = storePlainBlob(blob.ID, "damaged-", plaintext)
+ if reuploadBlobs {
+ repo.StartPackUploader(ctx, wg)
+ }
+
+ wg.Go(func() error {
+ for _, blob := range list {
+ Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
+ buf := make([]byte, blob.Length)
+ err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
+ n, err := io.ReadFull(rd, buf)
if err != nil {
- return err
+ return fmt.Errorf("read error after %d bytes: %v", n, err)
}
+ return nil
+ })
+ if err != nil {
+ Warnf("error read: %v\n", err)
continue
}
- }
- if blob.IsCompressed() {
- decompressed, err := dec.DecodeAll(plaintext, nil)
+ key := repo.Key()
+
+ nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
+ plaintext, err = key.Open(plaintext[:0], nonce, plaintext, nil)
+ outputPrefix := ""
+ filePrefix := ""
if err != nil {
- Printf(" failed to decompress blob %v\n", blob.ID)
+ Warnf("error decrypting blob: %v\n", err)
+ if tryRepair || repairByte {
+ plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte)
+ }
+ if plaintext != nil {
+ outputPrefix = "repaired "
+ filePrefix = "repaired-"
+ } else {
+ plaintext = decryptUnsigned(ctx, key, buf)
+ err = storePlainBlob(blob.ID, "damaged-", plaintext)
+ if err != nil {
+ return err
+ }
+ continue
+ }
}
- if decompressed != nil {
- plaintext = decompressed
+
+ if blob.IsCompressed() {
+ decompressed, err := dec.DecodeAll(plaintext, nil)
+ if err != nil {
+ Printf(" failed to decompress blob %v\n", blob.ID)
+ }
+ if decompressed != nil {
+ plaintext = decompressed
+ }
}
- }
- id := restic.Hash(plaintext)
- var prefix string
- if !id.Equal(blob.ID) {
- Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", outputPrefix, len(plaintext), id, blob.ID)
- prefix = "wrong-hash-"
- } else {
- Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
- prefix = "correct-"
- }
- if extractPack {
- err = storePlainBlob(id, filePrefix+prefix, plaintext)
- if err != nil {
- return err
+ id := restic.Hash(plaintext)
+ var prefix string
+ if !id.Equal(blob.ID) {
+ Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID does not match, wanted %v\n", outputPrefix, len(plaintext), id, blob.ID)
+ prefix = "wrong-hash-"
+ } else {
+ Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
+ prefix = "correct-"
}
- }
- if reuploadBlobs {
- _, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
- if err != nil {
- return err
+ if extractPack {
+ err = storePlainBlob(id, filePrefix+prefix, plaintext)
+ if err != nil {
+ return err
+ }
+ }
+ if reuploadBlobs {
+ _, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
+ if err != nil {
+ return err
+ }
+ Printf(" uploaded %v %v\n", blob.Type, id)
}
- Printf(" uploaded %v %v\n", blob.Type, id)
}
- }
- if reuploadBlobs {
- err := repo.Flush(ctx)
- if err != nil {
- return err
+ if reuploadBlobs {
+ return repo.Flush(ctx)
}
- }
+ return nil
+ })
- return nil
+ return wg.Wait()
}
func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
@@ -426,8 +437,8 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
return nil
}
-func runDebugExamine(gopts GlobalOptions, args []string) error {
- repo, err := OpenRepository(gopts)
+func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error {
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@@ -436,10 +447,7 @@ func runDebugExamine(gopts GlobalOptions, args []string) error {
for _, name := range args {
id, err := restic.ParseID(name)
if err != nil {
- name, err = restic.Find(gopts.ctx, repo.Backend(), restic.PackFile, name)
- if err == nil {
- id, err = restic.ParseID(name)
- }
+ id, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name)
if err != nil {
Warnf("error: %v\n", err)
continue
@@ -453,20 +461,21 @@ func runDebugExamine(gopts GlobalOptions, args []string) error {
}
if !gopts.NoLock {
- lock, err := lockRepo(gopts.ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
- err = repo.LoadIndex(gopts.ctx)
+ err = repo.LoadIndex(ctx)
if err != nil {
return err
}
for _, id := range ids {
- err := examinePack(gopts.ctx, repo, id)
+ err := examinePack(ctx, repo, id)
if err != nil {
Warnf("error: %v\n", err)
}
@@ -525,7 +534,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
Printf(" ========================================\n")
Printf(" inspect the pack itself\n")
- blobs, _, err := pack.List(repo.Key(), backend.ReaderAt(ctx, repo.Backend(), h), fi.Size)
+ blobs, _, err := repo.ListPack(ctx, id, fi.Size)
if err != nil {
return fmt.Errorf("pack %v: %v", id.Str(), err)
}
@@ -556,7 +565,7 @@ func checkPackSize(blobs []restic.Blob, fileSize int64) {
size += uint64(pack.CalculateHeaderSize(blobs))
if uint64(fileSize) != size {
- Printf(" file sizes do not match: computed %v from index, file size is %v\n", size, fileSize)
+ Printf(" file sizes do not match: computed %v, file size is %v\n", size, fileSize)
} else {
Printf(" file sizes match\n")
}
diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go
index 5fdd28d97..0000fd18a 100644
--- a/cmd/restic/cmd_diff.go
+++ b/cmd/restic/cmd_diff.go
@@ -11,6 +11,7 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
+ "github.com/restic/restic/internal/ui"
"github.com/spf13/cobra"
)
@@ -35,7 +36,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runDiff(diffOptions, globalOptions, args)
+ return runDiff(cmd.Context(), diffOptions, globalOptions, args)
},
}
@@ -54,11 +55,11 @@ func init() {
}
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, error) {
- id, err := restic.FindSnapshot(ctx, be, desc)
+ sn, err := restic.FindSnapshot(ctx, be, repo, desc)
if err != nil {
return nil, errors.Fatal(err.Error())
}
- return restic.LoadSnapshot(ctx, repo, id)
+ return sn, err
}
// Comparer collects all things needed to compare two snapshots.
@@ -321,21 +322,19 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
return nil
}
-func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
+func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []string) error {
if len(args) != 2 {
return errors.Fatalf("specify two snapshot IDs")
}
- ctx, cancel := context.WithCancel(gopts.ctx)
- defer cancel()
-
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
- lock, err := lockRepo(ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -427,8 +426,8 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
Printf("Others: %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
Printf("Data Blobs: %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
Printf("Tree Blobs: %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
- Printf(" Added: %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
- Printf(" Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
+ Printf(" Added: %-5s\n", ui.FormatBytes(uint64(stats.Added.Bytes)))
+ Printf(" Removed: %-5s\n", ui.FormatBytes(uint64(stats.Removed.Bytes)))
}
return nil
diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go
index 993072f9c..a480b12f4 100644
--- a/cmd/restic/cmd_dump.go
+++ b/cmd/restic/cmd_dump.go
@@ -34,15 +34,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runDump(dumpOptions, globalOptions, args)
+ return runDump(cmd.Context(), dumpOptions, globalOptions, args)
},
}
// DumpOptions collects all options for the dump command.
type DumpOptions struct {
- Hosts []string
- Paths []string
- Tags restic.TagLists
+ snapshotFilterOptions
Archive string
}
@@ -52,9 +50,7 @@ func init() {
cmdRoot.AddCommand(cmdDump)
flags := cmdDump.Flags()
- flags.StringArrayVarP(&dumpOptions.Hosts, "host", "H", nil, `only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)`)
- flags.Var(&dumpOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
- flags.StringArrayVar(&dumpOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
+ initSingleSnapshotFilterOptions(flags, &dumpOptions.snapshotFilterOptions)
flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
}
@@ -111,9 +107,7 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
return fmt.Errorf("path %q not found in snapshot", item)
}
-func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
- ctx := gopts.ctx
-
+func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []string) error {
if len(args) != 2 {
return errors.Fatal("no file and no snapshot ID specified")
}
@@ -131,36 +125,23 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
splittedPath := splitPath(path.Clean(pathToPrint))
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
- lock, err := lockRepo(ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
- var id restic.ID
-
- if snapshotIDString == "latest" {
- id, err = restic.FindLatestSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil)
- if err != nil {
- Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
- }
- } else {
- id, err = restic.FindSnapshot(ctx, repo.Backend(), snapshotIDString)
- if err != nil {
- Exitf(1, "invalid id %q: %v", snapshotIDString, err)
- }
- }
-
- sn, err := restic.LoadSnapshot(gopts.ctx, repo, id)
+ sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil, snapshotIDString)
if err != nil {
- Exitf(2, "loading snapshot %q failed: %v", snapshotIDString, err)
+ return errors.Fatalf("failed to find snapshot: %v", err)
}
err = repo.LoadIndex(ctx)
@@ -170,13 +151,13 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error {
tree, err := restic.LoadTree(ctx, repo, *sn.Tree)
if err != nil {
- Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err)
+ return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)
}
d := dump.New(opts.Archive, repo, os.Stdout)
err = printFromTree(ctx, tree, repo, "/", splittedPath, d)
if err != nil {
- Exitf(2, "cannot dump file: %v", err)
+ return errors.Fatalf("cannot dump file: %v", err)
}
return nil
diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go
index 7171314e2..8e5f9b604 100644
--- a/cmd/restic/cmd_find.go
+++ b/cmd/restic/cmd_find.go
@@ -38,7 +38,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runFind(findOptions, globalOptions, args)
+ return runFind(cmd.Context(), findOptions, globalOptions, args)
},
}
@@ -51,9 +51,7 @@ type FindOptions struct {
PackID, ShowPackID bool
CaseInsensitive bool
ListLong bool
- Hosts []string
- Paths []string
- Tags restic.TagLists
+ snapshotFilterOptions
}
var findOptions FindOptions
@@ -72,9 +70,7 @@ func init() {
f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
- f.StringArrayVarP(&findOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
- f.Var(&findOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
- f.StringArrayVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+ initMultiSnapshotFilterOptions(f, &findOptions.snapshotFilterOptions, true)
}
type findPattern struct {
@@ -471,7 +467,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
// remember which packs were found in the index
indexPackIDs := make(map[string]struct{})
- for pb := range f.repo.Index().Each(wctx) {
+ f.repo.Index().Each(wctx, func(pb restic.PackedBlob) {
idStr := pb.PackID.String()
// keep entry in packIDs as Each() returns individual index entries
matchingID := false
@@ -489,7 +485,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
f.blobIDs[pb.ID.String()] = struct{}{}
indexPackIDs[idStr] = struct{}{}
}
- }
+ })
for id := range indexPackIDs {
delete(packIDs, id)
@@ -538,7 +534,7 @@ func (f *Finder) findObjectsPacks(ctx context.Context) {
}
}
-func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
+func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
return errors.Fatal("wrong number of arguments")
}
@@ -572,31 +568,29 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
return errors.Fatal("cannot have several ID types")
}
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
- lock, err := lockRepo(gopts.ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
- snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+ snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
- if err = repo.LoadIndex(gopts.ctx); err != nil {
+ if err = repo.LoadIndex(ctx); err != nil {
return err
}
- ctx, cancel := context.WithCancel(gopts.ctx)
- defer cancel()
-
f := &Finder{
repo: repo,
pat: pat,
diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go
index 46f3fee0f..472b22b79 100644
--- a/cmd/restic/cmd_forget.go
+++ b/cmd/restic/cmd_forget.go
@@ -32,7 +32,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runForget(forgetOptions, globalOptions, args)
+ return runForget(cmd.Context(), forgetOptions, globalOptions, args)
},
}
@@ -52,9 +52,7 @@ type ForgetOptions struct {
WithinYearly restic.Duration
KeepTags restic.TagLists
- Hosts []string
- Tags restic.TagLists
- Paths []string
+ snapshotFilterOptions
Compact bool
// Grouping
@@ -81,9 +79,9 @@ func init() {
f.VarP(&forgetOptions.WithinWeekly, "keep-within-weekly", "", "keep weekly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
-
f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
- f.StringArrayVar(&forgetOptions.Hosts, "host", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
+
+ initMultiSnapshotFilterOptions(f, &forgetOptions.snapshotFilterOptions, false)
f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
err := f.MarkDeprecated("hostname", "use --host")
if err != nil {
@@ -91,9 +89,6 @@ func init() {
panic(err)
}
- f.Var(&forgetOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
-
- f.StringArrayVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
f.BoolVarP(&forgetOptions.Compact, "compact", "c", false, "use compact output format")
f.StringVarP(&forgetOptions.GroupBy, "group-by", "g", "host,paths", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
@@ -104,13 +99,13 @@ func init() {
addPruneOptions(cmdForget)
}
-func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
+func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error {
err := verifyPruneOptions(&pruneOptions)
if err != nil {
return err
}
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@@ -120,16 +115,14 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
}
if !opts.DryRun || !gopts.NoLock {
- lock, err := lockRepoExclusive(gopts.ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
- ctx, cancel := context.WithCancel(gopts.ctx)
- defer cancel()
-
var snapshots restic.Snapshots
removeSnIDs := restic.NewIDSet()
@@ -224,7 +217,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
if len(removeSnIDs) > 0 {
if !opts.DryRun {
- err := DeleteFilesChecked(gopts, repo, removeSnIDs, restic.SnapshotFile)
+ err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile)
if err != nil {
return err
}
@@ -244,10 +237,14 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
if len(removeSnIDs) > 0 && opts.Prune {
if !gopts.JSON {
- Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
+ if opts.DryRun {
+ Verbosef("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
+ } else {
+ Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
+ }
}
pruneOptions.DryRun = opts.DryRun
- return runPruneWithRepo(pruneOptions, gopts, repo, removeSnIDs)
+ return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs)
}
return nil
diff --git a/cmd/restic/cmd_generate.go b/cmd/restic/cmd_generate.go
index 710c5c721..959a9d518 100644
--- a/cmd/restic/cmd_generate.go
+++ b/cmd/restic/cmd_generate.go
@@ -10,7 +10,7 @@ import (
var cmdGenerate = &cobra.Command{
Use: "generate [flags]",
- Short: "Generate manual pages and auto-completion files (bash, fish, zsh)",
+ Short: "Generate manual pages and auto-completion files (bash, fish, zsh, powershell)",
Long: `
The "generate" command writes automatically generated files (like the man pages
and the auto-completion files for bash, fish and zsh).
@@ -25,10 +25,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
}
type generateOptions struct {
- ManDir string
- BashCompletionFile string
- FishCompletionFile string
- ZSHCompletionFile string
+ ManDir string
+ BashCompletionFile string
+ FishCompletionFile string
+ ZSHCompletionFile string
+ PowerShellCompletionFile string
}
var genOpts generateOptions
@@ -40,6 +41,7 @@ func init() {
fs.StringVar(&genOpts.BashCompletionFile, "bash-completion", "", "write bash completion `file`")
fs.StringVar(&genOpts.FishCompletionFile, "fish-completion", "", "write fish completion `file`")
fs.StringVar(&genOpts.ZSHCompletionFile, "zsh-completion", "", "write zsh completion `file`")
+ fs.StringVar(&genOpts.PowerShellCompletionFile, "powershell-completion", "", "write powershell completion `file`")
}
func writeManpages(dir string) error {
@@ -75,6 +77,11 @@ func writeZSHCompletion(file string) error {
return cmdRoot.GenZshCompletionFile(file)
}
+func writePowerShellCompletion(file string) error {
+ Verbosef("writing powershell completion file to %v\n", file)
+ return cmdRoot.GenPowerShellCompletionFile(file)
+}
+
func runGenerate(cmd *cobra.Command, args []string) error {
if genOpts.ManDir != "" {
err := writeManpages(genOpts.ManDir)
@@ -104,6 +111,13 @@ func runGenerate(cmd *cobra.Command, args []string) error {
}
}
+ if genOpts.PowerShellCompletionFile != "" {
+ err := writePowerShellCompletion(genOpts.PowerShellCompletionFile)
+ if err != nil {
+ return err
+ }
+ }
+
var empty generateOptions
if genOpts == empty {
return errors.Fatal("nothing to do, please specify at least one output file/dir")
diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go
index 4c0392ff1..2932870e8 100644
--- a/cmd/restic/cmd_init.go
+++ b/cmd/restic/cmd_init.go
@@ -1,6 +1,8 @@
package main
import (
+ "context"
+ "encoding/json"
"strconv"
"github.com/restic/chunker"
@@ -25,7 +27,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runInit(initOptions, globalOptions, args)
+ return runInit(cmd.Context(), initOptions, globalOptions, args)
},
}
@@ -47,7 +49,7 @@ func init() {
f.StringVar(&initOptions.RepositoryVersion, "repository-version", "stable", "repository format version to use, allowed values are a format version, 'latest' and 'stable'")
}
-func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
+func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []string) error {
var version uint
if opts.RepositoryVersion == "latest" || opts.RepositoryVersion == "" {
version = restic.MaxRepoVersion
@@ -64,7 +66,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return errors.Fatalf("only repository versions between %v and %v are allowed", restic.MinRepoVersion, restic.MaxRepoVersion)
}
- chunkerPolynomial, err := maybeReadChunkerPolynomial(opts, gopts)
+ chunkerPolynomial, err := maybeReadChunkerPolynomial(ctx, opts, gopts)
if err != nil {
return err
}
@@ -81,7 +83,7 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return err
}
- be, err := create(repo, gopts.extended)
+ be, err := create(ctx, repo, gopts.extended)
if err != nil {
return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
}
@@ -94,28 +96,38 @@ func runInit(opts InitOptions, gopts GlobalOptions, args []string) error {
return err
}
- err = s.Init(gopts.ctx, version, gopts.password, chunkerPolynomial)
+ err = s.Init(ctx, version, gopts.password, chunkerPolynomial)
if err != nil {
return errors.Fatalf("create key in repository at %s failed: %v\n", location.StripPassword(gopts.Repo), err)
}
- Verbosef("created restic repository %v at %s\n", s.Config().ID[:10], location.StripPassword(gopts.Repo))
- Verbosef("\n")
- Verbosef("Please note that knowledge of your password is required to access\n")
- Verbosef("the repository. Losing your password means that your data is\n")
- Verbosef("irrecoverably lost.\n")
+ if !gopts.JSON {
+ Verbosef("created restic repository %v at %s\n", s.Config().ID[:10], location.StripPassword(gopts.Repo))
+ Verbosef("\n")
+ Verbosef("Please note that knowledge of your password is required to access\n")
+ Verbosef("the repository. Losing your password means that your data is\n")
+ Verbosef("irrecoverably lost.\n")
+
+ } else {
+ status := initSuccess{
+ MessageType: "initialized",
+ ID: s.Config().ID,
+ Repository: location.StripPassword(gopts.Repo),
+ }
+ return json.NewEncoder(gopts.stdout).Encode(status)
+ }
return nil
}
-func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
+func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
if opts.CopyChunkerParameters {
otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
if err != nil {
return nil, err
}
- otherRepo, err := OpenRepository(otherGopts)
+ otherRepo, err := OpenRepository(ctx, otherGopts)
if err != nil {
return nil, err
}
@@ -129,3 +141,9 @@ func maybeReadChunkerPolynomial(opts InitOptions, gopts GlobalOptions) (*chunker
}
return nil, nil
}
+
+type initSuccess struct {
+ MessageType string `json:"message_type"` // "initialized"
+ ID string `json:"id"`
+ Repository string `json:"repository"`
+}
diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go
index 69c2542b7..88b6d5c0c 100644
--- a/cmd/restic/cmd_key.go
+++ b/cmd/restic/cmd_key.go
@@ -3,9 +3,9 @@ package main
import (
"context"
"encoding/json"
- "io/ioutil"
"os"
"strings"
+ "sync"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
@@ -28,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runKey(globalOptions, args)
+ return runKey(cmd.Context(), globalOptions, args)
},
}
@@ -56,23 +56,26 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
Created string `json:"created"`
}
+ var m sync.Mutex
var keys []keyInfo
- err := s.List(ctx, restic.KeyFile, func(id restic.ID, size int64) error {
- k, err := repository.LoadKey(ctx, s, id.String())
+ err := restic.ParallelList(ctx, s.Backend(), restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
+ k, err := repository.LoadKey(ctx, s, id)
if err != nil {
Warnf("LoadKey() failed: %v\n", err)
return nil
}
key := keyInfo{
- Current: id.String() == s.KeyName(),
+ Current: id == s.KeyID(),
ID: id.Str(),
UserName: k.Username,
HostName: k.Hostname,
Created: k.Created.Local().Format(TimeFormat),
}
+ m.Lock()
+ defer m.Unlock()
keys = append(keys, key)
return nil
})
@@ -120,18 +123,18 @@ func getNewPassword(gopts GlobalOptions) (string, error) {
"enter password again: ")
}
-func addKey(gopts GlobalOptions, repo *repository.Repository) error {
+func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
pw, err := getNewPassword(gopts)
if err != nil {
return err
}
- id, err := repository.AddKey(gopts.ctx, repo, pw, keyUsername, keyHostname, repo.Key())
+ id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key())
if err != nil {
return errors.Fatalf("creating new key failed: %v\n", err)
}
- err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
+ err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
if err != nil {
return err
}
@@ -141,40 +144,40 @@ func addKey(gopts GlobalOptions, repo *repository.Repository) error {
return nil
}
-func deleteKey(ctx context.Context, repo *repository.Repository, name string) error {
- if name == repo.KeyName() {
+func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) error {
+ if id == repo.KeyID() {
return errors.Fatal("refusing to remove key currently used to access repository")
}
- h := restic.Handle{Type: restic.KeyFile, Name: name}
+ h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
err := repo.Backend().Remove(ctx, h)
if err != nil {
return err
}
- Verbosef("removed key %v\n", name)
+ Verbosef("removed key %v\n", id)
return nil
}
-func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
+func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
pw, err := getNewPassword(gopts)
if err != nil {
return err
}
- id, err := repository.AddKey(gopts.ctx, repo, pw, "", "", repo.Key())
+ id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
if err != nil {
return errors.Fatalf("creating new key failed: %v\n", err)
}
- oldID := repo.KeyName()
+ oldID := repo.KeyID()
- err = switchToNewKeyAndRemoveIfBroken(gopts.ctx, repo, id, pw)
+ err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
if err != nil {
return err
}
- h := restic.Handle{Type: restic.KeyFile, Name: oldID}
- err = repo.Backend().Remove(gopts.ctx, h)
+ h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()}
+ err = repo.Backend().Remove(ctx, h)
if err != nil {
return err
}
@@ -187,32 +190,29 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
// Verify new key to make sure it really works. A broken key can render the
// whole repository inaccessible
- err := repo.SearchKey(ctx, pw, 0, key.Name())
+ err := repo.SearchKey(ctx, pw, 0, key.ID().String())
if err != nil {
// the key is invalid, try to remove it
- h := restic.Handle{Type: restic.KeyFile, Name: key.Name()}
+ h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()}
_ = repo.Backend().Remove(ctx, h)
return errors.Fatalf("failed to access repository with new key: %v", err)
}
return nil
}
-func runKey(gopts GlobalOptions, args []string) error {
+func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
return errors.Fatal("wrong number of arguments")
}
- ctx, cancel := context.WithCancel(gopts.ctx)
- defer cancel()
-
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
switch args[0] {
case "list":
- lock, err := lockRepo(ctx, repo)
+ lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -220,15 +220,15 @@ func runKey(gopts GlobalOptions, args []string) error {
return listKeys(ctx, repo, gopts)
case "add":
- lock, err := lockRepo(ctx, repo)
+ lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
- return addKey(gopts, repo)
+ return addKey(ctx, repo, gopts)
case "remove":
- lock, err := lockRepoExclusive(ctx, repo)
+ lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -239,22 +239,22 @@ func runKey(gopts GlobalOptions, args []string) error {
return err
}
- return deleteKey(gopts.ctx, repo, id)
+ return deleteKey(ctx, repo, id)
case "passwd":
- lock, err := lockRepoExclusive(ctx, repo)
+ lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
- return changePassword(gopts, repo)
+ return changePassword(ctx, repo, gopts)
}
return nil
}
func loadPasswordFromFile(pwdFile string) (string, error) {
- s, err := ioutil.ReadFile(pwdFile)
+ s, err := os.ReadFile(pwdFile)
if os.IsNotExist(err) {
return "", errors.Fatalf("%s does not exist", pwdFile)
}
diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go
index 811b17e41..4809092c0 100644
--- a/cmd/restic/cmd_list.go
+++ b/cmd/restic/cmd_list.go
@@ -1,8 +1,10 @@
package main
import (
+ "context"
+
"github.com/restic/restic/internal/errors"
- "github.com/restic/restic/internal/repository"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/restic"
"github.com/spf13/cobra"
@@ -21,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runList(cmd, globalOptions, args)
+ return runList(cmd.Context(), cmd, globalOptions, args)
},
}
@@ -29,18 +31,19 @@ func init() {
cmdRoot.AddCommand(cmdList)
}
-func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
+func runList(ctx context.Context, cmd *cobra.Command, opts GlobalOptions, args []string) error {
if len(args) != 1 {
return errors.Fatal("type not specified, usage: " + cmd.Use)
}
- repo, err := OpenRepository(opts)
+ repo, err := OpenRepository(ctx, opts)
if err != nil {
return err
}
if !opts.NoLock && args[0] != "locks" {
- lock, err := lockRepo(opts.ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -60,20 +63,20 @@ func runList(cmd *cobra.Command, opts GlobalOptions, args []string) error {
case "locks":
t = restic.LockFile
case "blobs":
- return repository.ForAllIndexes(opts.ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
+ return index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
return err
}
- for blobs := range idx.Each(opts.ctx) {
+ idx.Each(ctx, func(blobs restic.PackedBlob) {
Printf("%v %v\n", blobs.Type, blobs.ID)
- }
+ })
return nil
})
default:
return errors.Fatal("invalid type")
}
- return repo.List(opts.ctx, t, func(id restic.ID, size int64) error {
+ return repo.List(ctx, t, func(id restic.ID, size int64) error {
Printf("%s\n", id)
return nil
})
diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go
index ec6695714..7dd41ab21 100644
--- a/cmd/restic/cmd_ls.go
+++ b/cmd/restic/cmd_ls.go
@@ -42,16 +42,14 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runLs(lsOptions, globalOptions, args)
+ return runLs(cmd.Context(), lsOptions, globalOptions, args)
},
}
// LsOptions collects all options for the ls command.
type LsOptions struct {
- ListLong bool
- Hosts []string
- Tags restic.TagLists
- Paths []string
+ ListLong bool
+ snapshotFilterOptions
Recursive bool
}
@@ -61,10 +59,8 @@ func init() {
cmdRoot.AddCommand(cmdLs)
flags := cmdLs.Flags()
+ initSingleSnapshotFilterOptions(flags, &lsOptions.snapshotFilterOptions)
flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
- flags.StringArrayVarP(&lsOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
- flags.Var(&lsOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when snapshot ID \"latest\" is given (can be specified multiple times)")
- flags.StringArrayVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
}
@@ -115,7 +111,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
return enc.Encode(n)
}
-func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
+func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
}
@@ -165,23 +161,20 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
return false
}
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
- snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+ snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
- if err = repo.LoadIndex(gopts.ctx); err != nil {
+ if err = repo.LoadIndex(ctx); err != nil {
return err
}
- ctx, cancel := context.WithCancel(gopts.ctx)
- defer cancel()
-
var (
printSnapshot func(sn *restic.Snapshot)
printNode func(path string, node *restic.Node)
@@ -217,45 +210,48 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
}
}
- for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args[:1]) {
- printSnapshot(sn)
+ sn, err := restic.FindFilteredSnapshot(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, nil, args[0])
+ if err != nil {
+ return err
+ }
- err := walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
- if err != nil {
- return false, err
- }
- if node == nil {
- return false, nil
- }
+ printSnapshot(sn)
- if withinDir(nodepath) {
- // if we're within a dir, print the node
- printNode(nodepath, node)
+ err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
+ if err != nil {
+ return false, err
+ }
+ if node == nil {
+ return false, nil
+ }
- // if recursive listing is requested, signal the walker that it
- // should continue walking recursively
- if opts.Recursive {
- return false, nil
- }
- }
+ if withinDir(nodepath) {
+ // if we're within a dir, print the node
+ printNode(nodepath, node)
- // if there's an upcoming match deeper in the tree (but we're not
- // there yet), signal the walker to descend into any subdirs
- if approachingMatchingTree(nodepath) {
+ // if recursive listing is requested, signal the walker that it
+ // should continue walking recursively
+ if opts.Recursive {
return false, nil
}
+ }
- // otherwise, signal the walker to not walk recursively into any
- // subdirs
- if node.Type == "dir" {
- return false, walker.ErrSkipNode
- }
+ // if there's an upcoming match deeper in the tree (but we're not
+ // there yet), signal the walker to descend into any subdirs
+ if approachingMatchingTree(nodepath) {
return false, nil
- })
+ }
- if err != nil {
- return err
+ // otherwise, signal the walker to not walk recursively into any
+ // subdirs
+ if node.Type == "dir" {
+ return false, walker.ErrSkipNode
}
+ return false, nil
+ })
+
+ if err != nil {
+ return err
}
return nil
diff --git a/cmd/restic/cmd_migrate.go b/cmd/restic/cmd_migrate.go
index c8f0e9478..6d614be39 100644
--- a/cmd/restic/cmd_migrate.go
+++ b/cmd/restic/cmd_migrate.go
@@ -1,6 +1,8 @@
package main
import (
+ "context"
+
"github.com/restic/restic/internal/migrations"
"github.com/restic/restic/internal/restic"
@@ -22,7 +24,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runMigrate(migrateOptions, globalOptions, args)
+ return runMigrate(cmd.Context(), migrateOptions, globalOptions, args)
},
}
@@ -39,13 +41,12 @@ func init() {
f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`)
}
-func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
- ctx := gopts.ctx
+func checkMigrations(ctx context.Context, repo restic.Repository) error {
Printf("available migrations:\n")
found := false
for _, m := range migrations.All {
- ok, err := m.Check(ctx, repo)
+ ok, _, err := m.Check(ctx, repo)
if err != nil {
return err
}
@@ -57,27 +58,28 @@ func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
}
if !found {
- Printf("no migrations found")
+ Printf("no migrations found\n")
}
return nil
}
-func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
- ctx := gopts.ctx
-
+func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
var firsterr error
for _, name := range args {
for _, m := range migrations.All {
if m.Name() == name {
- ok, err := m.Check(ctx, repo)
+ ok, reason, err := m.Check(ctx, repo)
if err != nil {
return err
}
if !ok {
if !opts.Force {
- Warnf("migration %v cannot be applied: check failed\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name())
+ if reason == "" {
+ reason = "check failed"
+ }
+ Warnf("migration %v cannot be applied: %v\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name(), reason)
continue
}
@@ -91,7 +93,7 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
checkGopts := gopts
// the repository is already locked
checkGopts.NoLock = true
- err = runCheck(checkOptions, checkGopts, []string{})
+ err = runCheck(ctx, checkOptions, checkGopts, []string{})
if err != nil {
return err
}
@@ -114,21 +116,21 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos
return firsterr
}
-func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error {
- repo, err := OpenRepository(gopts)
+func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string) error {
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
- lock, err := lockRepoExclusive(gopts.ctx, repo)
+ lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
if len(args) == 0 {
- return checkMigrations(opts, gopts, repo)
+ return checkMigrations(ctx, repo)
}
- return applyMigrations(opts, gopts, repo, args)
+ return applyMigrations(ctx, opts, gopts, repo, args)
}
diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go
index 747316f9f..7afb30f7c 100644
--- a/cmd/restic/cmd_mount.go
+++ b/cmd/restic/cmd_mount.go
@@ -4,6 +4,7 @@
package main
import (
+ "context"
"os"
"strings"
"time"
@@ -17,8 +18,8 @@ import (
resticfs "github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/fuse"
- systemFuse "bazil.org/fuse"
- "bazil.org/fuse/fs"
+ systemFuse "github.com/anacrolix/fuse"
+ "github.com/anacrolix/fuse/fs"
)
var cmdMount = &cobra.Command{
@@ -67,7 +68,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runMount(mountOptions, globalOptions, args)
+ return runMount(cmd.Context(), mountOptions, globalOptions, args)
},
}
@@ -76,11 +77,9 @@ type MountOptions struct {
OwnerRoot bool
AllowOther bool
NoDefaultPermissions bool
- Hosts []string
- Tags restic.TagLists
- Paths []string
- TimeTemplate string
- PathTemplates []string
+ snapshotFilterOptions
+ TimeTemplate string
+ PathTemplates []string
}
var mountOptions MountOptions
@@ -93,9 +92,7 @@ func init() {
mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
mountFlags.BoolVar(&mountOptions.NoDefaultPermissions, "no-default-permissions", false, "for 'allow-other', ignore Unix permissions and allow users to read all snapshot files")
- mountFlags.StringArrayVarP(&mountOptions.Hosts, "host", "H", nil, `only consider snapshots for this host (can be specified multiple times)`)
- mountFlags.Var(&mountOptions.Tags, "tag", "only consider snapshots which include this `taglist`")
- mountFlags.StringArrayVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
+ initMultiSnapshotFilterOptions(mountFlags, &mountOptions.snapshotFilterOptions, true)
mountFlags.StringArrayVar(&mountOptions.PathTemplates, "path-template", nil, "set `template` for path names (can be specified multiple times)")
mountFlags.StringVar(&mountOptions.TimeTemplate, "snapshot-template", time.RFC3339, "set `template` to use for snapshot dirs")
@@ -103,7 +100,7 @@ func init() {
_ = mountFlags.MarkDeprecated("snapshot-template", "use --time-template")
}
-func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
+func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args []string) error {
if opts.TimeTemplate == "" {
return errors.Fatal("time template string cannot be empty")
}
@@ -119,20 +116,21 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
debug.Log("start mount")
defer debug.Log("finish mount")
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
- lock, err := lockRepo(gopts.ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
- err = repo.LoadIndex(gopts.ctx)
+ err = repo.LoadIndex(ctx)
if err != nil {
return err
}
@@ -158,13 +156,17 @@ func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
}
}
- AddCleanupHandler(func() error {
+ AddCleanupHandler(func(code int) (int, error) {
debug.Log("running umount cleanup handler for mount at %v", mountpoint)
err := umount(mountpoint)
if err != nil {
Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
}
- return nil
+ // replace error code of sigint
+ if code == 130 {
+ code = 0
+ }
+ return code, nil
})
c, err := systemFuse.Mount(mountpoint, mountOptions...)
diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go
index 7421bc0cc..f59be2967 100644
--- a/cmd/restic/cmd_prune.go
+++ b/cmd/restic/cmd_prune.go
@@ -9,9 +9,11 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
+ "github.com/restic/restic/internal/ui"
"github.com/spf13/cobra"
)
@@ -34,7 +36,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runPrune(pruneOptions, globalOptions)
+ return runPrune(cmd.Context(), pruneOptions, globalOptions)
},
}
@@ -134,7 +136,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
return nil
}
-func runPrune(opts PruneOptions, gopts GlobalOptions) error {
+func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error {
err := verifyPruneOptions(&opts)
if err != nil {
return err
@@ -144,7 +146,7 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error {
return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
}
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@@ -165,16 +167,16 @@ func runPrune(opts PruneOptions, gopts GlobalOptions) error {
opts.unsafeRecovery = true
}
- lock, err := lockRepoExclusive(gopts.ctx, repo)
+ lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
- return runPruneWithRepo(opts, gopts, repo, restic.NewIDSet())
+ return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet())
}
-func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
+func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
// we do not need index updates while pruning!
repo.DisableAutoIndexUpdate()
@@ -184,22 +186,26 @@ func runPruneWithRepo(opts PruneOptions, gopts GlobalOptions, repo *repository.R
Verbosef("loading indexes...\n")
// loading the index before the snapshots is ok, as we use an exclusive lock here
- err := repo.LoadIndex(gopts.ctx)
+ err := repo.LoadIndex(ctx)
if err != nil {
return err
}
- plan, stats, err := planPrune(opts, gopts, repo, ignoreSnapshots)
+ plan, stats, err := planPrune(ctx, opts, repo, ignoreSnapshots, gopts.Quiet)
if err != nil {
return err
}
- err = printPruneStats(gopts, stats)
+ if opts.DryRun {
+ Verbosef("\nWould have made the following changes:")
+ }
+
+ err = printPruneStats(stats)
if err != nil {
return err
}
- return doPrune(opts, gopts, repo, plan)
+ return doPrune(ctx, opts, gopts, repo, plan)
}
type pruneStats struct {
@@ -212,13 +218,14 @@ type pruneStats struct {
repackrm uint
}
size struct {
- used uint64
- duplicate uint64
- unused uint64
- remove uint64
- repack uint64
- repackrm uint64
- unref uint64
+ used uint64
+ duplicate uint64
+ unused uint64
+ remove uint64
+ repack uint64
+ repackrm uint64
+ unref uint64
+ uncompressed uint64
}
packs struct {
used uint
@@ -232,11 +239,11 @@ type pruneStats struct {
}
type prunePlan struct {
- removePacksFirst restic.IDSet // packs to remove first (unreferenced packs)
- repackPacks restic.IDSet // packs to repack
- keepBlobs restic.BlobSet // blobs to keep during repacking
- removePacks restic.IDSet // packs to remove
- ignorePacks restic.IDSet // packs to ignore when rebuilding the index
+ removePacksFirst restic.IDSet // packs to remove first (unreferenced packs)
+ repackPacks restic.IDSet // packs to repack
+ keepBlobs restic.CountedBlobSet // blobs to keep during repacking
+ removePacks restic.IDSet // packs to remove
+ ignorePacks restic.IDSet // packs to ignore when rebuilding the index
}
type packInfo struct {
@@ -251,15 +258,15 @@ type packInfo struct {
type packInfoWithID struct {
ID restic.ID
packInfo
+ mustCompress bool
}
// planPrune selects which files to rewrite and which to delete and which blobs to keep.
// Also some summary statistics are returned.
-func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (prunePlan, pruneStats, error) {
- ctx := gopts.ctx
+func planPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (prunePlan, pruneStats, error) {
var stats pruneStats
- usedBlobs, err := getUsedBlobs(gopts, repo, ignoreSnapshots)
+ usedBlobs, err := getUsedBlobs(ctx, repo, ignoreSnapshots, quiet)
if err != nil {
return prunePlan{}, stats, err
}
@@ -271,19 +278,25 @@ func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, i
}
Verbosef("collecting packs for deletion and repacking\n")
- plan, err := decidePackAction(ctx, opts, gopts, repo, indexPack, &stats)
+ plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, quiet)
if err != nil {
return prunePlan{}, stats, err
}
if len(plan.repackPacks) != 0 {
+ blobCount := keepBlobs.Len()
// when repacking, we do not want to keep blobs which are
// already contained in kept packs, so delete them from keepBlobs
- for blob := range repo.Index().Each(ctx) {
+ repo.Index().Each(ctx, func(blob restic.PackedBlob) {
if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
- continue
+ return
}
keepBlobs.Delete(blob.BlobHandle)
+ })
+
+ if keepBlobs.Len() < blobCount/2 {
+ // replace with copy to shrink map to necessary size if there's a chance to benefit
+ keepBlobs = keepBlobs.Copy()
}
} else {
// keepBlobs is only needed if packs are repacked
@@ -294,46 +307,40 @@ func planPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, i
return plan, stats, nil
}
-func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.BlobSet, stats *pruneStats) (restic.BlobSet, map[restic.ID]packInfo, error) {
- keepBlobs := restic.NewBlobSet()
- duplicateBlobs := make(map[restic.BlobHandle]uint8)
-
+func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *pruneStats) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
// iterate over all blobs in index to find out which blobs are duplicates
- for blob := range idx.Each(ctx) {
+ // The counter in usedBlobs describes how many instances of the blob exist in the repository index
+ // Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
+ idx.Each(ctx, func(blob restic.PackedBlob) {
bh := blob.BlobHandle
- size := uint64(blob.Length)
- switch {
- case usedBlobs.Has(bh): // used blob, move to keepBlobs
- usedBlobs.Delete(bh)
- keepBlobs.Insert(bh)
- stats.size.used += size
- stats.blobs.used++
- case keepBlobs.Has(bh): // duplicate blob
- count, ok := duplicateBlobs[bh]
- if !ok {
- count = 2 // this one is already the second blob!
- } else if count < math.MaxUint8 {
+ count, ok := usedBlobs[bh]
+ if ok {
+ if count < math.MaxUint8 {
// don't overflow, but saturate count at 255
// this can lead to a non-optimal pack selection, but won't cause
// problems otherwise
count++
}
- duplicateBlobs[bh] = count
- stats.size.duplicate += size
- stats.blobs.duplicate++
- default:
- stats.size.unused += size
- stats.blobs.unused++
+
+ usedBlobs[bh] = count
}
- }
+ })
// Check if all used blobs have been found in index
- if len(usedBlobs) != 0 {
+ missingBlobs := restic.NewBlobSet()
+ for bh, count := range usedBlobs {
+ if count == 0 {
+ // blob does not exist in any pack files
+ missingBlobs.Insert(bh)
+ }
+ }
+
+ if len(missingBlobs) != 0 {
Warnf("%v not found in the index\n\n"+
"Integrity check failed: Data seems to be missing.\n"+
"Will not start prune to prevent (additional) data loss!\n"+
"Please report this error (along with the output of the 'prune' run) at\n"+
- "https://github.com/restic/restic/issues/new/choose\n", usedBlobs)
+ "https://github.com/restic/restic/issues/new/choose\n", missingBlobs)
return nil, nil, errorIndexIncomplete
}
@@ -345,8 +352,9 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
}
+ hasDuplicates := false
// iterate over all blobs in index to generate packInfo
- for blob := range idx.Each(ctx) {
+ idx.Each(ctx, func(blob restic.PackedBlob) {
ip := indexPack[blob.PackID]
// Set blob type if not yet set
@@ -361,64 +369,95 @@ func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs re
bh := blob.BlobHandle
size := uint64(blob.Length)
- _, isDuplicate := duplicateBlobs[bh]
+ dupCount := usedBlobs[bh]
switch {
- case isDuplicate: // duplicate blobs will be handled later
- case keepBlobs.Has(bh): // used blob, not duplicate
+ case dupCount >= 2:
+ hasDuplicates = true
+ // mark as unused for now, we will later on select one copy
+ ip.unusedSize += size
+ ip.unusedBlobs++
+
+ // count as duplicate, will later on change one copy to be counted as used
+ stats.size.duplicate += size
+ stats.blobs.duplicate++
+ case dupCount == 1: // used blob, not duplicate
ip.usedSize += size
ip.usedBlobs++
+
+ stats.size.used += size
+ stats.blobs.used++
default: // unused blob
ip.unusedSize += size
ip.unusedBlobs++
+
+ stats.size.unused += size
+ stats.blobs.unused++
}
if !blob.IsCompressed() {
ip.uncompressed = true
}
// update indexPack
indexPack[blob.PackID] = ip
- }
+ })
// if duplicate blobs exist, those will be set to either "used" or "unused":
// - mark only one occurence of duplicate blobs as used
// - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used"
// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
- if len(duplicateBlobs) > 0 {
+ if hasDuplicates {
// iterate again over all blobs in index (this is pretty cheap, all in-mem)
- for blob := range idx.Each(ctx) {
+ idx.Each(ctx, func(blob restic.PackedBlob) {
bh := blob.BlobHandle
- count, isDuplicate := duplicateBlobs[bh]
- if !isDuplicate {
- continue
+ count, ok := usedBlobs[bh]
+ // skip non-duplicate, aka. normal blobs
+ // count == 0 is used to mark that this was a duplicate blob with only a single occurence remaining
+ if !ok || count == 1 {
+ return
}
ip := indexPack[blob.PackID]
size := uint64(blob.Length)
switch {
- case count == 0:
- // used duplicate exists -> mark as unused
- ip.unusedSize += size
- ip.unusedBlobs++
- case ip.usedBlobs > 0, count == 1:
- // other used blobs in pack or "last" occurency -> mark as used
+ case ip.usedBlobs > 0, count == 0:
+ // other used blobs in pack or "last" occurence -> transition to used
ip.usedSize += size
ip.usedBlobs++
- // let other occurences be marked as unused
- duplicateBlobs[bh] = 0
+ ip.unusedSize -= size
+ ip.unusedBlobs--
+ // same for the global statistics
+ stats.size.used += size
+ stats.blobs.used++
+ stats.size.duplicate -= size
+ stats.blobs.duplicate--
+ // let other occurences remain marked as unused
+ usedBlobs[bh] = 1
default:
- // mark as unused and decrease counter
- ip.unusedSize += size
- ip.unusedBlobs++
- duplicateBlobs[bh] = count - 1
+ // remain unused and decrease counter
+ count--
+ if count == 1 {
+ // setting count to 1 would lead to forgetting that this blob had duplicates
+ // thus use the special value zero. This will select the last instance of the blob for keeping.
+ count = 0
+ }
+ usedBlobs[bh] = count
}
// update indexPack
indexPack[blob.PackID] = ip
+ })
+ }
+
+ // Sanity check. If no duplicates exist, all blobs have value 1. After handling
+ // duplicates, this also applies to duplicates.
+ for _, count := range usedBlobs {
+ if count != 1 {
+ panic("internal error during blob selection")
}
}
- return keepBlobs, indexPack, nil
+ return usedBlobs, indexPack, nil
}
-func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats) (prunePlan, error) {
+func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats, quiet bool) (prunePlan, error) {
removePacksFirst := restic.NewIDSet()
removePacks := restic.NewIDSet()
repackPacks := restic.NewIDSet()
@@ -434,7 +473,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
}
// loop over all packs and decide what to do
- bar := newProgressMax(!gopts.Quiet, uint64(len(indexPack)), "packs processed")
+ bar := newProgressMax(!quiet, uint64(len(indexPack)), "packs processed")
err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
p, ok := indexPack[id]
if !ok {
@@ -464,14 +503,15 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
stats.packs.partlyUsed++
}
+ if p.uncompressed {
+ stats.size.uncompressed += p.unusedSize + p.usedSize
+ }
mustCompress := false
if repoVersion >= 2 {
// repo v2: always repack tree blobs if uncompressed
// compress data blobs if requested
mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed
}
- // use a flag that pack must be compressed
- p.uncompressed = mustCompress
// decide what to do
switch {
@@ -490,12 +530,12 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
// All blobs in pack are used and not mixed => keep pack!
stats.packs.keep++
} else {
- repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p})
+ repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
}
default:
// all other packs are candidates for repacking
- repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p})
+ repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
}
delete(indexPack, id)
@@ -569,6 +609,9 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
stats.size.repack += p.unusedSize + p.usedSize
stats.blobs.repackrm += p.unusedBlobs
stats.size.repackrm += p.unusedSize
+ if p.uncompressed {
+ stats.size.uncompressed -= p.unusedSize + p.usedSize
+ }
}
// calculate limit for number of unused bytes in the repo after repacking
@@ -583,7 +626,7 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
case reachedRepackSize:
stats.packs.keep++
- case p.tpe != restic.DataBlob, p.uncompressed:
+ case p.tpe != restic.DataBlob, p.mustCompress:
// repacking non-data packs / uncompressed-trees is only limited by repackSize
repack(p.ID, p.packInfo)
@@ -600,6 +643,11 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
stats.packs.repack = uint(len(repackPacks))
stats.packs.remove = uint(len(removePacks))
+ if repo.Config().Version < 2 {
+ // compression not supported for repository format version 1
+ stats.size.uncompressed = 0
+ }
+
return prunePlan{removePacksFirst: removePacksFirst,
removePacks: removePacks,
repackPacks: repackPacks,
@@ -608,30 +656,33 @@ func decidePackAction(ctx context.Context, opts PruneOptions, gopts GlobalOption
}
// printPruneStats prints out the statistics
-func printPruneStats(gopts GlobalOptions, stats pruneStats) error {
- Verboseff("\nused: %10d blobs / %s\n", stats.blobs.used, formatBytes(stats.size.used))
+func printPruneStats(stats pruneStats) error {
+ Verboseff("\nused: %10d blobs / %s\n", stats.blobs.used, ui.FormatBytes(stats.size.used))
if stats.blobs.duplicate > 0 {
- Verboseff("duplicates: %10d blobs / %s\n", stats.blobs.duplicate, formatBytes(stats.size.duplicate))
+ Verboseff("duplicates: %10d blobs / %s\n", stats.blobs.duplicate, ui.FormatBytes(stats.size.duplicate))
}
- Verboseff("unused: %10d blobs / %s\n", stats.blobs.unused, formatBytes(stats.size.unused))
+ Verboseff("unused: %10d blobs / %s\n", stats.blobs.unused, ui.FormatBytes(stats.size.unused))
if stats.size.unref > 0 {
- Verboseff("unreferenced: %s\n", formatBytes(stats.size.unref))
+ Verboseff("unreferenced: %s\n", ui.FormatBytes(stats.size.unref))
}
totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
unusedSize := stats.size.duplicate + stats.size.unused
- Verboseff("total: %10d blobs / %s\n", totalBlobs, formatBytes(totalSize))
- Verboseff("unused size: %s of total size\n", formatPercent(unusedSize, totalSize))
+ Verboseff("total: %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
+ Verboseff("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
- Verbosef("\nto repack: %10d blobs / %s\n", stats.blobs.repack, formatBytes(stats.size.repack))
- Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, formatBytes(stats.size.repackrm))
- Verbosef("to delete: %10d blobs / %s\n", stats.blobs.remove, formatBytes(stats.size.remove+stats.size.unref))
+ Verbosef("\nto repack: %10d blobs / %s\n", stats.blobs.repack, ui.FormatBytes(stats.size.repack))
+ Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, ui.FormatBytes(stats.size.repackrm))
+ Verbosef("to delete: %10d blobs / %s\n", stats.blobs.remove, ui.FormatBytes(stats.size.remove+stats.size.unref))
totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
- Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, formatBytes(totalPruneSize))
- Verbosef("remaining: %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), formatBytes(totalSize-totalPruneSize))
+ Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, ui.FormatBytes(totalPruneSize))
+ if stats.size.uncompressed > 0 {
+ Verbosef("not yet compressed: %s\n", ui.FormatBytes(stats.size.uncompressed))
+ }
+ Verbosef("remaining: %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), ui.FormatBytes(totalSize-totalPruneSize))
unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
Verbosef("unused size after prune: %s (%s of remaining size)\n",
- formatBytes(unusedAfter), formatPercent(unusedAfter, totalSize-totalPruneSize))
+ ui.FormatBytes(unusedAfter), ui.FormatPercent(unusedAfter, totalSize-totalPruneSize))
Verbosef("\n")
Verboseff("totally used packs: %10d\n", stats.packs.used)
Verboseff("partly used packs: %10d\n", stats.packs.partlyUsed)
@@ -652,11 +703,10 @@ func printPruneStats(gopts GlobalOptions, stats pruneStats) error {
// - rebuild the index while ignoring all files that will be deleted
// - delete the files
// plan.removePacks and plan.ignorePacks are modified in this function.
-func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
- ctx := gopts.ctx
-
+func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
if opts.DryRun {
if !gopts.JSON && gopts.verbosity >= 2 {
+ Printf("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n")
if len(plan.removePacksFirst) > 0 {
Printf("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
}
@@ -670,7 +720,7 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
// unreferenced packs can be safely deleted first
if len(plan.removePacksFirst) != 0 {
Verbosef("deleting unreferenced packs\n")
- DeleteFiles(gopts, repo, plan.removePacksFirst, restic.PackFile)
+ DeleteFiles(ctx, gopts, repo, plan.removePacksFirst, restic.PackFile)
}
if len(plan.repackPacks) != 0 {
@@ -692,6 +742,9 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
"https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs)
return errors.Fatal("internal error: blobs were not repacked")
}
+
+ // allow GC of the blob set
+ plan.keepBlobs = nil
}
if len(plan.ignorePacks) == 0 {
@@ -702,13 +755,13 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
if opts.unsafeRecovery {
Verbosef("deleting index files\n")
- indexFiles := repo.Index().(*repository.MasterIndex).IDs()
- err = DeleteFilesChecked(gopts, repo, indexFiles, restic.IndexFile)
+ indexFiles := repo.Index().(*index.MasterIndex).IDs()
+ err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
if err != nil {
return errors.Fatalf("%s", err)
}
} else if len(plan.ignorePacks) != 0 {
- err = rebuildIndexFiles(gopts, repo, plan.ignorePacks, nil)
+ err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
if err != nil {
return errors.Fatalf("%s", err)
}
@@ -716,11 +769,11 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
if len(plan.removePacks) != 0 {
Verbosef("removing %d old packs\n", len(plan.removePacks))
- DeleteFiles(gopts, repo, plan.removePacks, restic.PackFile)
+ DeleteFiles(ctx, gopts, repo, plan.removePacks, restic.PackFile)
}
if opts.unsafeRecovery {
- _, err = writeIndexFiles(gopts, repo, plan.ignorePacks, nil)
+ _, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
if err != nil {
return errors.Fatalf("%s", err)
}
@@ -730,31 +783,29 @@ func doPrune(opts PruneOptions, gopts GlobalOptions, repo restic.Repository, pla
return nil
}
-func writeIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
+func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
Verbosef("rebuilding index\n")
bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
- obsoleteIndexes, err := repo.Index().Save(gopts.ctx, repo, removePacks, extraObsolete, bar)
+ obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar)
bar.Done()
return obsoleteIndexes, err
}
-func rebuildIndexFiles(gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
- obsoleteIndexes, err := writeIndexFiles(gopts, repo, removePacks, extraObsolete)
+func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
+ obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete)
if err != nil {
return err
}
Verbosef("deleting obsolete index files\n")
- return DeleteFilesChecked(gopts, repo, obsoleteIndexes, restic.IndexFile)
+ return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile)
}
-func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots restic.IDSet) (usedBlobs restic.BlobSet, err error) {
- ctx := gopts.ctx
-
+func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) {
var snapshotTrees restic.IDs
Verbosef("loading all snapshots...\n")
- err = restic.ForAllSnapshots(gopts.ctx, repo.Backend(), repo, ignoreSnapshots,
+ err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots,
func(id restic.ID, sn *restic.Snapshot, err error) error {
if err != nil {
debug.Log("failed to load snapshot %v (error %v)", id, err)
@@ -770,9 +821,9 @@ func getUsedBlobs(gopts GlobalOptions, repo restic.Repository, ignoreSnapshots r
Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
- usedBlobs = restic.NewBlobSet()
+ usedBlobs = restic.NewCountedBlobSet()
- bar := newProgressMax(!gopts.Quiet, uint64(len(snapshotTrees)), "snapshots")
+ bar := newProgressMax(!quiet, uint64(len(snapshotTrees)), "snapshots")
defer bar.Done()
err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)
diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go
index 0b3274ec4..6d49cb917 100644
--- a/cmd/restic/cmd_rebuild_index.go
+++ b/cmd/restic/cmd_rebuild_index.go
@@ -1,6 +1,9 @@
package main
import (
+ "context"
+
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@@ -22,7 +25,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runRebuildIndex(rebuildIndexOptions, globalOptions)
+ return runRebuildIndex(cmd.Context(), rebuildIndexOptions, globalOptions)
},
}
@@ -40,24 +43,22 @@ func init() {
}
-func runRebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions) error {
- repo, err := OpenRepository(gopts)
+func runRebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions) error {
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
- lock, err := lockRepoExclusive(gopts.ctx, repo)
+ lock, ctx, err := lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
- return rebuildIndex(opts, gopts, repo, restic.NewIDSet())
+ return rebuildIndex(ctx, opts, gopts, repo, restic.NewIDSet())
}
-func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
- ctx := gopts.ctx
-
+func rebuildIndex(ctx context.Context, opts RebuildIndexOptions, gopts GlobalOptions, repo *repository.Repository, ignorePacks restic.IDSet) error {
var obsoleteIndexes restic.IDs
packSizeFromList := make(map[restic.ID]int64)
packSizeFromIndex := make(map[restic.ID]int64)
@@ -74,8 +75,8 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
}
} else {
Verbosef("loading indexes...\n")
- mi := repository.NewMasterIndex()
- err := repository.ForAllIndexes(ctx, repo, func(id restic.ID, idx *repository.Index, oldFormat bool, err error) error {
+ mi := index.NewMasterIndex()
+ err := index.ForAllIndexes(ctx, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
Warnf("removing invalid index %v: %v\n", id, err)
obsoleteIndexes = append(obsoleteIndexes, id)
@@ -141,7 +142,7 @@ func rebuildIndex(opts RebuildIndexOptions, gopts GlobalOptions, repo *repositor
}
}
- err = rebuildIndexFiles(gopts, repo, removePacks, obsoleteIndexes)
+ err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes)
if err != nil {
return err
}
diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go
index 9f6d2061d..65f4c8750 100644
--- a/cmd/restic/cmd_recover.go
+++ b/cmd/restic/cmd_recover.go
@@ -27,7 +27,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runRecover(globalOptions)
+ return runRecover(cmd.Context(), globalOptions)
},
}
@@ -35,30 +35,30 @@ func init() {
cmdRoot.AddCommand(cmdRecover)
}
-func runRecover(gopts GlobalOptions) error {
+func runRecover(ctx context.Context, gopts GlobalOptions) error {
hostname, err := os.Hostname()
if err != nil {
return err
}
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
- lock, err := lockRepo(gopts.ctx, repo)
+ lock, ctx, err := lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
- snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+ snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
Verbosef("load index files\n")
- if err = repo.LoadIndex(gopts.ctx); err != nil {
+ if err = repo.LoadIndex(ctx); err != nil {
return err
}
@@ -66,16 +66,16 @@ func runRecover(gopts GlobalOptions) error {
// tree. If it is not referenced, we have a root tree.
trees := make(map[restic.ID]bool)
- for blob := range repo.Index().Each(gopts.ctx) {
+ repo.Index().Each(ctx, func(blob restic.PackedBlob) {
if blob.Type == restic.TreeBlob {
trees[blob.Blob.ID] = false
}
- }
+ })
Verbosef("load %d trees\n", len(trees))
bar := newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
for id := range trees {
- tree, err := restic.LoadTree(gopts.ctx, repo, id)
+ tree, err := restic.LoadTree(ctx, repo, id)
if err != nil {
Warnf("unable to load tree %v: %v\n", id.Str(), err)
continue
@@ -91,7 +91,7 @@ func runRecover(gopts GlobalOptions) error {
bar.Done()
Verbosef("load snapshots\n")
- err = restic.ForAllSnapshots(gopts.ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
+ err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
trees[*sn.Tree] = true
return nil
})
@@ -132,18 +132,18 @@ func runRecover(gopts GlobalOptions) error {
}
}
- wg, ctx := errgroup.WithContext(gopts.ctx)
- repo.StartPackUploader(ctx, wg)
+ wg, wgCtx := errgroup.WithContext(ctx)
+ repo.StartPackUploader(wgCtx, wg)
var treeID restic.ID
wg.Go(func() error {
var err error
- treeID, err = restic.SaveTree(ctx, repo, tree)
+ treeID, err = restic.SaveTree(wgCtx, repo, tree)
if err != nil {
return errors.Fatalf("unable to save new tree to the repository: %v", err)
}
- err = repo.Flush(ctx)
+ err = repo.Flush(wgCtx)
if err != nil {
return errors.Fatalf("unable to save blobs to the repository: %v", err)
}
@@ -154,7 +154,7 @@ func runRecover(gopts GlobalOptions) error {
return err
}
- return createSnapshot(gopts.ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
+ return createSnapshot(ctx, "/recover", hostname, []string{"recovered"}, repo, &treeID)
}
diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go
index addd36661..b70cb52ff 100644
--- a/cmd/restic/cmd_restore.go
+++ b/cmd/restic/cmd_restore.go
@@ -1,6 +1,7 @@
package main
import (
+ "context"
"strings"
"time"
@@ -30,7 +31,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runRestore(restoreOptions, globalOptions, args)
+ return runRestore(cmd.Context(), restoreOptions, globalOptions, args)
},
}
@@ -41,10 +42,9 @@ type RestoreOptions struct {
Include []string
InsensitiveInclude []string
Target string
- Hosts []string
- Paths []string
- Tags restic.TagLists
- Verify bool
+ snapshotFilterOptions
+ Sparse bool
+ Verify bool
}
var restoreOptions RestoreOptions
@@ -59,36 +59,34 @@ func init() {
flags.StringArrayVar(&restoreOptions.InsensitiveInclude, "iinclude", nil, "same as `--include` but ignores the casing of filenames")
flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
- flags.StringArrayVarP(&restoreOptions.Hosts, "host", "H", nil, `only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)`)
- flags.Var(&restoreOptions.Tags, "tag", "only consider snapshots which include this `taglist` for snapshot ID \"latest\"")
- flags.StringArrayVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
+ initSingleSnapshotFilterOptions(flags, &restoreOptions.snapshotFilterOptions)
+ flags.BoolVar(&restoreOptions.Sparse, "sparse", false, "restore files as sparse")
flags.BoolVar(&restoreOptions.Verify, "verify", false, "verify restored files content")
}
-func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
- ctx := gopts.ctx
+func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, args []string) error {
hasExcludes := len(opts.Exclude) > 0 || len(opts.InsensitiveExclude) > 0
hasIncludes := len(opts.Include) > 0 || len(opts.InsensitiveInclude) > 0
// Validate provided patterns
if len(opts.Exclude) > 0 {
- if valid, invalidPatterns := filter.ValidatePatterns(opts.Exclude); !valid {
- return errors.Fatalf("--exclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+ if err := filter.ValidatePatterns(opts.Exclude); err != nil {
+ return errors.Fatalf("--exclude: %s", err)
}
}
if len(opts.InsensitiveExclude) > 0 {
- if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveExclude); !valid {
- return errors.Fatalf("--iexclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+ if err := filter.ValidatePatterns(opts.InsensitiveExclude); err != nil {
+ return errors.Fatalf("--iexclude: %s", err)
}
}
if len(opts.Include) > 0 {
- if valid, invalidPatterns := filter.ValidatePatterns(opts.Include); !valid {
- return errors.Fatalf("--include: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+ if err := filter.ValidatePatterns(opts.Include); err != nil {
+ return errors.Fatalf("--include: %s", err)
}
}
if len(opts.InsensitiveInclude) > 0 {
- if valid, invalidPatterns := filter.ValidatePatterns(opts.InsensitiveInclude); !valid {
- return errors.Fatalf("--iinclude: invalid pattern(s) provided:\n%s", strings.Join(invalidPatterns, "\n"))
+ if err := filter.ValidatePatterns(opts.InsensitiveInclude); err != nil {
+ return errors.Fatalf("--iinclude: %s", err)
}
}
@@ -119,31 +117,23 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
debug.Log("restore %v to %v", snapshotIDString, opts.Target)
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
- lock, err := lockRepo(ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
- var id restic.ID
-
- if snapshotIDString == "latest" {
- id, err = restic.FindLatestSnapshot(ctx, repo.Backend(), repo, opts.Paths, opts.Tags, opts.Hosts, nil)
- if err != nil {
- Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Hosts:%v", err, opts.Paths, opts.Hosts)
- }
- } else {
- id, err = restic.FindSnapshot(ctx, repo.Backend(), snapshotIDString)
- if err != nil {
- Exitf(1, "invalid id %q: %v", snapshotIDString, err)
- }
+ sn, err := restic.FindFilteredSnapshot(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, nil, snapshotIDString)
+ if err != nil {
+ return errors.Fatalf("failed to find snapshot: %v", err)
}
err = repo.LoadIndex(ctx)
@@ -151,10 +141,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
return err
}
- res, err := restorer.NewRestorer(ctx, repo, id)
- if err != nil {
- Exitf(2, "creating restorer failed: %v\n", err)
- }
+ res := restorer.NewRestorer(ctx, repo, sn, opts.Sparse)
totalErrors := 0
res.Error = func(location string, err error) error {
diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
new file mode 100644
index 000000000..cfe56db87
--- /dev/null
+++ b/cmd/restic/cmd_rewrite.go
@@ -0,0 +1,216 @@
+package main
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/spf13/cobra"
+ "golang.org/x/sync/errgroup"
+
+ "github.com/restic/restic/internal/backend"
+ "github.com/restic/restic/internal/debug"
+ "github.com/restic/restic/internal/errors"
+ "github.com/restic/restic/internal/repository"
+ "github.com/restic/restic/internal/restic"
+ "github.com/restic/restic/internal/walker"
+)
+
+var cmdRewrite = &cobra.Command{
+ Use: "rewrite [flags] [snapshotID ...]",
+ Short: "Rewrite snapshots to exclude unwanted files",
+ Long: `
+The "rewrite" command excludes files from existing snapshots. It creates new
+snapshots containing the same data as the original ones, but without the files
+you specify to exclude. All metadata (time, host, tags) will be preserved.
+
+The snapshots to rewrite are specified using the --host, --tag and --path options,
+or by providing a list of snapshot IDs. Please note that specifying neither any of
+these options nor a snapshot ID will cause the command to rewrite all snapshots.
+
+The special tag 'rewrite' will be added to the new snapshots to distinguish
+them from the original ones, unless --forget is used. If the --forget option is
+used, the original snapshots will instead be directly removed from the repository.
+
+Please note that the --forget option only removes the snapshots and not the actual
+data stored in the repository. In order to delete the no longer referenced data,
+use the "prune" command.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command was successful, and non-zero if there was any error.
+`,
+ DisableAutoGenTag: true,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return runRewrite(cmd.Context(), rewriteOptions, globalOptions, args)
+ },
+}
+
+// RewriteOptions collects all options for the rewrite command.
+type RewriteOptions struct {
+ Forget bool
+ DryRun bool
+
+ snapshotFilterOptions
+ excludePatternOptions
+}
+
+var rewriteOptions RewriteOptions
+
+func init() {
+ cmdRoot.AddCommand(cmdRewrite)
+
+ f := cmdRewrite.Flags()
+ f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones")
+ f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
+
+ initMultiSnapshotFilterOptions(f, &rewriteOptions.snapshotFilterOptions, true)
+ initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions)
+}
+
+func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) {
+ if sn.Tree == nil {
+ return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
+ }
+
+ rejectByNameFuncs, err := opts.excludePatternOptions.CollectPatterns()
+ if err != nil {
+ return false, err
+ }
+
+ selectByName := func(nodepath string) bool {
+ for _, reject := range rejectByNameFuncs {
+ if reject(nodepath) {
+ return false
+ }
+ }
+ return true
+ }
+
+ wg, wgCtx := errgroup.WithContext(ctx)
+ repo.StartPackUploader(wgCtx, wg)
+
+ var filteredTree restic.ID
+ wg.Go(func() error {
+ filteredTree, err = walker.FilterTree(wgCtx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{
+ SelectByName: selectByName,
+ PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) },
+ })
+ if err != nil {
+ return err
+ }
+
+ return repo.Flush(wgCtx)
+ })
+ err = wg.Wait()
+ if err != nil {
+ return false, err
+ }
+
+ if filteredTree == *sn.Tree {
+ debug.Log("Snapshot %v not modified", sn)
+ return false, nil
+ }
+
+ debug.Log("Snapshot %v modified", sn)
+ if opts.DryRun {
+ Verbosef("would save new snapshot\n")
+
+ if opts.Forget {
+ Verbosef("would remove old snapshot\n")
+ }
+
+ return true, nil
+ }
+
+ // Always set the original snapshot id as this essentially a new snapshot.
+ sn.Original = sn.ID()
+ *sn.Tree = filteredTree
+
+ if !opts.Forget {
+ sn.AddTags([]string{"rewrite"})
+ }
+
+ // Save the new snapshot.
+ id, err := restic.SaveSnapshot(ctx, repo, sn)
+ if err != nil {
+ return false, err
+ }
+
+ if opts.Forget {
+ h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+ if err = repo.Backend().Remove(ctx, h); err != nil {
+ return false, err
+ }
+ debug.Log("removed old snapshot %v", sn.ID())
+ Verbosef("removed old snapshot %v\n", sn.ID().Str())
+ }
+ Verbosef("saved new snapshot %v\n", id.Str())
+ return true, nil
+}
+
+func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
+ if opts.excludePatternOptions.Empty() {
+ return errors.Fatal("Nothing to do: no excludes provided")
+ }
+
+ repo, err := OpenRepository(ctx, gopts)
+ if err != nil {
+ return err
+ }
+
+ if !opts.DryRun {
+ var lock *restic.Lock
+ var err error
+ if opts.Forget {
+ Verbosef("create exclusive lock for repository\n")
+ lock, ctx, err = lockRepoExclusive(ctx, repo)
+ } else {
+ lock, ctx, err = lockRepo(ctx, repo)
+ }
+ defer unlockRepo(lock)
+ if err != nil {
+ return err
+ }
+ } else {
+ repo.SetDryRun()
+ }
+
+ snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+ if err != nil {
+ return err
+ }
+
+ if err = repo.LoadIndex(ctx); err != nil {
+ return err
+ }
+
+ changedCount := 0
+ for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
+ Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+ changed, err := rewriteSnapshot(ctx, repo, sn, opts)
+ if err != nil {
+ return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
+ }
+ if changed {
+ changedCount++
+ }
+ }
+
+ Verbosef("\n")
+ if changedCount == 0 {
+ if !opts.DryRun {
+ Verbosef("no snapshots were modified\n")
+ } else {
+ Verbosef("no snapshots would be modified\n")
+ }
+ } else {
+ if !opts.DryRun {
+ Verbosef("modified %v snapshots\n", changedCount)
+ } else {
+ Verbosef("would modify %v snapshots\n", changedCount)
+ }
+ }
+
+ return nil
+}
diff --git a/cmd/restic/cmd_self_update.go b/cmd/restic/cmd_self_update.go
index 6d604c792..4b86c416f 100644
--- a/cmd/restic/cmd_self_update.go
+++ b/cmd/restic/cmd_self_update.go
@@ -1,8 +1,9 @@
-// xbuild selfupdate
+//go:build selfupdate
package main
import (
+ "context"
"os"
"path/filepath"
@@ -27,7 +28,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runSelfUpdate(selfUpdateOptions, globalOptions, args)
+ return runSelfUpdate(cmd.Context(), selfUpdateOptions, globalOptions, args)
},
}
@@ -45,7 +46,7 @@ func init() {
flags.StringVar(&selfUpdateOptions.Output, "output", "", "Save the downloaded file as `filename` (default: running binary itself)")
}
-func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) error {
+func runSelfUpdate(ctx context.Context, opts SelfUpdateOptions, gopts GlobalOptions, args []string) error {
if opts.Output == "" {
file, err := os.Executable()
if err != nil {
@@ -73,7 +74,7 @@ func runSelfUpdate(opts SelfUpdateOptions, gopts GlobalOptions, args []string) e
Verbosef("writing restic to %v\n", opts.Output)
- v, err := selfupdate.DownloadLatestStableRelease(gopts.ctx, opts.Output, version, Verbosef)
+ v, err := selfupdate.DownloadLatestStableRelease(ctx, opts.Output, version, Verbosef)
if err != nil {
return errors.Fatalf("unable to update restic: %v", err)
}
diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go
index ed201bf65..0bfa4d110 100644
--- a/cmd/restic/cmd_snapshots.go
+++ b/cmd/restic/cmd_snapshots.go
@@ -26,15 +26,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runSnapshots(snapshotOptions, globalOptions, args)
+ return runSnapshots(cmd.Context(), snapshotOptions, globalOptions, args)
},
}
// SnapshotOptions bundles all options for the snapshots command.
type SnapshotOptions struct {
- Hosts []string
- Tags restic.TagLists
- Paths []string
+ snapshotFilterOptions
Compact bool
Last bool // This option should be removed in favour of Latest.
Latest int
@@ -47,9 +45,7 @@ func init() {
cmdRoot.AddCommand(cmdSnapshots)
f := cmdSnapshots.Flags()
- f.StringArrayVarP(&snapshotOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host` (can be specified multiple times)")
- f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
- f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
+ initMultiSnapshotFilterOptions(f, &snapshotOptions.snapshotFilterOptions, true)
f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact output format")
f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
err := f.MarkDeprecated("last", "use --latest 1")
@@ -61,23 +57,21 @@ func init() {
f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "`group` snapshots by host, paths and/or tags, separated by comma")
}
-func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
- repo, err := OpenRepository(gopts)
+func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions, args []string) error {
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
- lock, err := lockRepo(gopts.ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
- ctx, cancel := context.WithCancel(gopts.ctx)
- defer cancel()
-
var snapshots restic.Snapshots
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
snapshots = append(snapshots, sn)
diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go
index a8bcb2b85..99d16b932 100644
--- a/cmd/restic/cmd_stats.go
+++ b/cmd/restic/cmd_stats.go
@@ -7,7 +7,9 @@ import (
"path/filepath"
"github.com/restic/restic/internal/backend"
+ "github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/restic"
+ "github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/walker"
"github.com/minio/sha256-simd"
@@ -47,7 +49,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runStats(globalOptions, args)
+ return runStats(cmd.Context(), globalOptions, args)
},
}
@@ -56,10 +58,7 @@ type StatsOptions struct {
// the mode of counting to perform (see consts for available modes)
countMode string
- // filter snapshots by, if given by user
- Hosts []string
- Tags restic.TagLists
- Paths []string
+ snapshotFilterOptions
}
var statsOptions StatsOptions
@@ -68,34 +67,30 @@ func init() {
cmdRoot.AddCommand(cmdStats)
f := cmdStats.Flags()
f.StringVar(&statsOptions.countMode, "mode", countModeRestoreSize, "counting mode: restore-size (default), files-by-contents, blobs-per-file or raw-data")
- f.StringArrayVarP(&statsOptions.Hosts, "host", "H", nil, "only consider snapshots with the given `host` (can be specified multiple times)")
- f.Var(&statsOptions.Tags, "tag", "only consider snapshots which include this `taglist` in the format `tag[,tag,...]` (can be specified multiple times)")
- f.StringArrayVar(&statsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
+ initMultiSnapshotFilterOptions(f, &statsOptions.snapshotFilterOptions, true)
}
-func runStats(gopts GlobalOptions, args []string) error {
+func runStats(ctx context.Context, gopts GlobalOptions, args []string) error {
err := verifyStatsInput(gopts, args)
if err != nil {
return err
}
- ctx, cancel := context.WithCancel(gopts.ctx)
- defer cancel()
-
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
- lock, err := lockRepo(ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepo(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
}
}
- snapshotLister, err := backend.MemorizeList(gopts.ctx, repo.Backend(), restic.SnapshotFile)
+ snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
if err != nil {
return err
}
@@ -135,8 +130,22 @@ func runStats(gopts GlobalOptions, args []string) error {
return fmt.Errorf("blob %v not found", blobHandle)
}
stats.TotalSize += uint64(pbs[0].Length)
+ if repo.Config().Version >= 2 {
+ stats.TotalUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength())))
+ if pbs[0].IsCompressed() {
+ stats.TotalCompressedBlobsSize += uint64(pbs[0].Length)
+ stats.TotalCompressedBlobsUncompressedSize += uint64(crypto.CiphertextLength(int(pbs[0].DataLength())))
+ }
+ }
stats.TotalBlobCount++
}
+ if stats.TotalCompressedBlobsSize > 0 {
+ stats.CompressionRatio = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalCompressedBlobsSize)
+ }
+ if stats.TotalUncompressedSize > 0 {
+ stats.CompressionProgress = float64(stats.TotalCompressedBlobsUncompressedSize) / float64(stats.TotalUncompressedSize) * 100
+ stats.CompressionSpaceSaving = (1 - float64(stats.TotalSize)/float64(stats.TotalUncompressedSize)) * 100
+ }
}
if gopts.JSON {
@@ -148,15 +157,26 @@ func runStats(gopts GlobalOptions, args []string) error {
}
Printf("Stats in %s mode:\n", statsOptions.countMode)
- Printf("Snapshots processed: %d\n", stats.SnapshotsCount)
-
+ Printf(" Snapshots processed: %d\n", stats.SnapshotsCount)
if stats.TotalBlobCount > 0 {
- Printf(" Total Blob Count: %d\n", stats.TotalBlobCount)
+ Printf(" Total Blob Count: %d\n", stats.TotalBlobCount)
}
if stats.TotalFileCount > 0 {
- Printf(" Total File Count: %d\n", stats.TotalFileCount)
+ Printf(" Total File Count: %d\n", stats.TotalFileCount)
+ }
+ if stats.TotalUncompressedSize > 0 {
+ Printf(" Total Uncompressed Size: %-5s\n", ui.FormatBytes(stats.TotalUncompressedSize))
+ }
+ Printf(" Total Size: %-5s\n", ui.FormatBytes(stats.TotalSize))
+ if stats.CompressionProgress > 0 {
+ Printf(" Compression Progress: %.2f%%\n", stats.CompressionProgress)
+ }
+ if stats.CompressionRatio > 0 {
+ Printf(" Compression Ratio: %.2fx\n", stats.CompressionRatio)
+ }
+ if stats.CompressionSpaceSaving > 0 {
+ Printf("Compression Space Saving: %.2f%%\n", stats.CompressionSpaceSaving)
}
- Printf(" Total Size: %-5s\n", formatBytes(stats.TotalSize))
return nil
}
@@ -282,9 +302,15 @@ func verifyStatsInput(gopts GlobalOptions, args []string) error {
// to collect information about it, as well as state needed
// for a successful and efficient walk.
type statsContainer struct {
- TotalSize uint64 `json:"total_size"`
- TotalFileCount uint64 `json:"total_file_count"`
- TotalBlobCount uint64 `json:"total_blob_count,omitempty"`
+ TotalSize uint64 `json:"total_size"`
+ TotalUncompressedSize uint64 `json:"total_uncompressed_size,omitempty"`
+ TotalCompressedBlobsSize uint64 `json:"-"`
+ TotalCompressedBlobsUncompressedSize uint64 `json:"-"`
+ CompressionRatio float64 `json:"compression_ratio,omitempty"`
+ CompressionProgress float64 `json:"compression_progress,omitempty"`
+ CompressionSpaceSaving float64 `json:"compression_space_saving,omitempty"`
+ TotalFileCount uint64 `json:"total_file_count,omitempty"`
+ TotalBlobCount uint64 `json:"total_blob_count,omitempty"`
// holds count of all considered snapshots
SnapshotsCount int `json:"snapshots_count"`
diff --git a/cmd/restic/cmd_tag.go b/cmd/restic/cmd_tag.go
index 1b99a4d56..222ddd04a 100644
--- a/cmd/restic/cmd_tag.go
+++ b/cmd/restic/cmd_tag.go
@@ -29,15 +29,13 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runTag(tagOptions, globalOptions, args)
+ return runTag(cmd.Context(), tagOptions, globalOptions, args)
},
}
// TagOptions bundles all options for the 'tag' command.
type TagOptions struct {
- Hosts []string
- Paths []string
- Tags restic.TagLists
+ snapshotFilterOptions
SetTags restic.TagLists
AddTags restic.TagLists
RemoveTags restic.TagLists
@@ -52,10 +50,7 @@ func init() {
tagFlags.Var(&tagOptions.SetTags, "set", "`tags` which will replace the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
tagFlags.Var(&tagOptions.AddTags, "add", "`tags` which will be added to the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
tagFlags.Var(&tagOptions.RemoveTags, "remove", "`tags` which will be removed from the existing tags in the format `tag[,tag,...]` (can be given multiple times)")
-
- tagFlags.StringArrayVarP(&tagOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
- tagFlags.Var(&tagOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
- tagFlags.StringArrayVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+ initMultiSnapshotFilterOptions(tagFlags, &tagOptions.snapshotFilterOptions, true)
}
func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) {
@@ -100,7 +95,7 @@ func changeTags(ctx context.Context, repo *repository.Repository, sn *restic.Sna
return changed, nil
}
-func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
+func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []string) error {
if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 {
return errors.Fatal("nothing to do!")
}
@@ -108,14 +103,15 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
return errors.Fatal("--set and --add/--remove cannot be given at the same time")
}
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
if !gopts.NoLock {
Verbosef("create exclusive lock for repository\n")
- lock, err := lockRepoExclusive(gopts.ctx, repo)
+ var lock *restic.Lock
+ lock, ctx, err = lockRepoExclusive(ctx, repo)
defer unlockRepo(lock)
if err != nil {
return err
@@ -123,8 +119,6 @@ func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
}
changeCnt := 0
- ctx, cancel := context.WithCancel(gopts.ctx)
- defer cancel()
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, opts.Hosts, opts.Tags, opts.Paths, args) {
changed, err := changeTags(ctx, repo, sn, opts.SetTags.Flatten(), opts.AddTags.Flatten(), opts.RemoveTags.Flatten())
if err != nil {
diff --git a/cmd/restic/cmd_unlock.go b/cmd/restic/cmd_unlock.go
index 7f5d44ada..7b449d949 100644
--- a/cmd/restic/cmd_unlock.go
+++ b/cmd/restic/cmd_unlock.go
@@ -1,6 +1,8 @@
package main
import (
+ "context"
+
"github.com/restic/restic/internal/restic"
"github.com/spf13/cobra"
)
@@ -18,7 +20,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
- return runUnlock(unlockOptions, globalOptions)
+ return runUnlock(cmd.Context(), unlockOptions, globalOptions)
},
}
@@ -35,8 +37,8 @@ func init() {
unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "remove all locks, even non-stale ones")
}
-func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {
- repo, err := OpenRepository(gopts)
+func runUnlock(ctx context.Context, opts UnlockOptions, gopts GlobalOptions) error {
+ repo, err := OpenRepository(ctx, gopts)
if err != nil {
return err
}
@@ -46,11 +48,13 @@ func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {
fn = restic.RemoveAllLocks
}
- err = fn(gopts.ctx, repo)
+ processed, err := fn(ctx, repo)
if err != nil {
return err
}
- Verbosef("successfully removed locks\n")
+ if processed > 0 {
+ Verbosef("successfully removed %d locks\n", processed)
+ }
return nil
}
diff --git a/cmd/restic/delete.go b/cmd/restic/delete.go
index d97b9e617..2046ccfde 100644
--- a/cmd/restic/delete.go
+++ b/cmd/restic/delete.go
@@ -1,6 +1,8 @@
package main
import (
+ "context"
+
"golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/restic"
@@ -8,22 +10,22 @@ import (
// DeleteFiles deletes the given fileList of fileType in parallel
// it will print a warning if there is an error, but continue deleting the remaining files
-func DeleteFiles(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
- _ = deleteFiles(gopts, true, repo, fileList, fileType)
+func DeleteFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
+ _ = deleteFiles(ctx, gopts, true, repo, fileList, fileType)
}
// DeleteFilesChecked deletes the given fileList of fileType in parallel
// if an error occurs, it will cancel and return this error
-func DeleteFilesChecked(gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
- return deleteFiles(gopts, false, repo, fileList, fileType)
+func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
+ return deleteFiles(ctx, gopts, false, repo, fileList, fileType)
}
// deleteFiles deletes the given fileList of fileType in parallel
// if ignoreError=true, it will print a warning if there was an error, else it will abort.
-func deleteFiles(gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
+func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
totalCount := len(fileList)
fileChan := make(chan restic.ID)
- wg, ctx := errgroup.WithContext(gopts.ctx)
+ wg, ctx := errgroup.WithContext(ctx)
wg.Go(func() error {
defer close(fileChan)
for id := range fileList {
diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go
index 1ddd8932c..efe6f41e4 100644
--- a/cmd/restic/exclude.go
+++ b/cmd/restic/exclude.go
@@ -1,6 +1,7 @@
package main
import (
+ "bufio"
"bytes"
"fmt"
"io"
@@ -15,6 +16,8 @@ import (
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
+ "github.com/restic/restic/internal/textfile"
+ "github.com/spf13/pflag"
)
type rejectionCache struct {
@@ -410,3 +413,115 @@ func parseSizeStr(sizeStr string) (int64, error) {
}
return value * unit, nil
}
+
+// readExcludePatternsFromFiles reads all exclude files and returns the list of
+// exclude patterns. For each line, leading and trailing white space is removed
+// and comment lines are ignored. For each remaining pattern, environment
+// variables are resolved. For adding a literal dollar sign ($), write $$ to
+// the file.
+func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) {
+ getenvOrDollar := func(s string) string {
+ if s == "$" {
+ return "$"
+ }
+ return os.Getenv(s)
+ }
+
+ var excludes []string
+ for _, filename := range excludeFiles {
+ err := func() (err error) {
+ data, err := textfile.Read(filename)
+ if err != nil {
+ return err
+ }
+
+ scanner := bufio.NewScanner(bytes.NewReader(data))
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+
+ // ignore empty lines
+ if line == "" {
+ continue
+ }
+
+ // strip comments
+ if strings.HasPrefix(line, "#") {
+ continue
+ }
+
+ line = os.Expand(line, getenvOrDollar)
+ excludes = append(excludes, line)
+ }
+ return scanner.Err()
+ }()
+ if err != nil {
+ return nil, err
+ }
+ }
+ return excludes, nil
+}
+
+type excludePatternOptions struct {
+ Excludes []string
+ InsensitiveExcludes []string
+ ExcludeFiles []string
+ InsensitiveExcludeFiles []string
+}
+
+func initExcludePatternOptions(f *pflag.FlagSet, opts *excludePatternOptions) {
+ f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
+ f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
+ f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
+ f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
+}
+
+func (opts *excludePatternOptions) Empty() bool {
+ return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
+}
+
+func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error) {
+ var fs []RejectByNameFunc
+ // add patterns from file
+ if len(opts.ExcludeFiles) > 0 {
+ excludePatterns, err := readExcludePatternsFromFiles(opts.ExcludeFiles)
+ if err != nil {
+ return nil, err
+ }
+
+ if err := filter.ValidatePatterns(excludePatterns); err != nil {
+ return nil, errors.Fatalf("--exclude-file: %s", err)
+ }
+
+ opts.Excludes = append(opts.Excludes, excludePatterns...)
+ }
+
+ if len(opts.InsensitiveExcludeFiles) > 0 {
+ excludes, err := readExcludePatternsFromFiles(opts.InsensitiveExcludeFiles)
+ if err != nil {
+ return nil, err
+ }
+
+ if err := filter.ValidatePatterns(excludes); err != nil {
+ return nil, errors.Fatalf("--iexclude-file: %s", err)
+ }
+
+ opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
+ }
+
+ if len(opts.InsensitiveExcludes) > 0 {
+ if err := filter.ValidatePatterns(opts.InsensitiveExcludes); err != nil {
+ return nil, errors.Fatalf("--iexclude: %s", err)
+ }
+
+ fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
+ }
+
+ if len(opts.Excludes) > 0 {
+ if err := filter.ValidatePatterns(opts.Excludes); err != nil {
+ return nil, errors.Fatalf("--exclude: %s", err)
+ }
+
+ fs = append(fs, rejectByPattern(opts.Excludes))
+ }
+ return fs, nil
+}
diff --git a/cmd/restic/exclude_test.go b/cmd/restic/exclude_test.go
index c7bec4352..050a083e4 100644
--- a/cmd/restic/exclude_test.go
+++ b/cmd/restic/exclude_test.go
@@ -1,7 +1,6 @@
package main
import (
- "io/ioutil"
"os"
"path/filepath"
"testing"
@@ -85,17 +84,16 @@ func TestIsExcludedByFile(t *testing.T) {
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
- tempDir, cleanup := test.TempDir(t)
- defer cleanup()
+ tempDir := test.TempDir(t)
foo := filepath.Join(tempDir, "foo")
- err := ioutil.WriteFile(foo, []byte("foo"), 0666)
+ err := os.WriteFile(foo, []byte("foo"), 0666)
if err != nil {
t.Fatalf("could not write file: %v", err)
}
if tc.tagFile != "" {
tagFile := filepath.Join(tempDir, tc.tagFile)
- err = ioutil.WriteFile(tagFile, []byte(tc.content), 0666)
+ err = os.WriteFile(tagFile, []byte(tc.content), 0666)
if err != nil {
t.Fatalf("could not write tagfile: %v", err)
}
@@ -116,8 +114,7 @@ func TestIsExcludedByFile(t *testing.T) {
// cancel each other out. It was initially written to demonstrate a bug in
// rejectIfPresent.
func TestMultipleIsExcludedByFile(t *testing.T) {
- tempDir, cleanup := test.TempDir(t)
- defer cleanup()
+ tempDir := test.TempDir(t)
// Create some files in a temporary directory.
// Files in UPPERCASE will be used as exclusion triggers later on.
@@ -150,7 +147,7 @@ func TestMultipleIsExcludedByFile(t *testing.T) {
// create directories first, then the file
p := filepath.Join(tempDir, filepath.FromSlash(f.path))
errs = append(errs, os.MkdirAll(filepath.Dir(p), 0700))
- errs = append(errs, ioutil.WriteFile(p, []byte(f.path), 0600))
+ errs = append(errs, os.WriteFile(p, []byte(f.path), 0600))
}
test.OKs(t, errs) // see if anything went wrong during the creation
@@ -241,8 +238,7 @@ func TestParseInvalidSizeStr(t *testing.T) {
// TestIsExcludedByFileSize is for testing the instance of
// --exclude-larger-than parameters
func TestIsExcludedByFileSize(t *testing.T) {
- tempDir, cleanup := test.TempDir(t)
- defer cleanup()
+ tempDir := test.TempDir(t)
// Max size of file is set to be 1k
maxSizeStr := "1k"
diff --git a/cmd/restic/find.go b/cmd/restic/find.go
index 5107ef599..7b488c7aa 100644
--- a/cmd/restic/find.go
+++ b/cmd/restic/find.go
@@ -5,77 +5,60 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/restic"
+ "github.com/spf13/pflag"
)
+type snapshotFilterOptions struct {
+ Hosts []string
+ Tags restic.TagLists
+ Paths []string
+}
+
+// initMultiSnapshotFilterOptions is used for commands that work on multiple snapshots
+// MUST be combined with restic.FindFilteredSnapshots or FindFilteredSnapshots
+func initMultiSnapshotFilterOptions(flags *pflag.FlagSet, options *snapshotFilterOptions, addHostShorthand bool) {
+ hostShorthand := "H"
+ if !addHostShorthand {
+ hostShorthand = ""
+ }
+ flags.StringArrayVarP(&options.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times)")
+ flags.Var(&options.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)")
+ flags.StringArrayVar(&options.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times)")
+}
+
+// initSingleSnapshotFilterOptions is used for commands that work on a single snapshot
+// MUST be combined with restic.FindFilteredSnapshot
+func initSingleSnapshotFilterOptions(flags *pflag.FlagSet, options *snapshotFilterOptions) {
+ flags.StringArrayVarP(&options.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+ flags.Var(&options.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+ flags.StringArrayVar(&options.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+}
+
// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
func FindFilteredSnapshots(ctx context.Context, be restic.Lister, loader restic.LoaderUnpacked, hosts []string, tags []restic.TagList, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
out := make(chan *restic.Snapshot)
go func() {
defer close(out)
- if len(snapshotIDs) != 0 {
- // memorize snapshots list to prevent repeated backend listings
- be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile)
- if err != nil {
- Warnf("could not load snapshots: %v\n", err)
- return
- }
-
- var (
- id restic.ID
- usedFilter bool
- )
- ids := make(restic.IDs, 0, len(snapshotIDs))
- // Process all snapshot IDs given as arguments.
- for _, s := range snapshotIDs {
- if s == "latest" {
- usedFilter = true
- id, err = restic.FindLatestSnapshot(ctx, be, loader, paths, tags, hosts, nil)
- if err != nil {
- Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)\n", s, paths, tags, hosts)
- continue
- }
- } else {
- id, err = restic.FindSnapshot(ctx, be, s)
- if err != nil {
- Warnf("Ignoring %q: %v\n", s, err)
- continue
- }
- }
- ids = append(ids, id)
- }
-
- // Give the user some indication their filters are not used.
- if !usedFilter && (len(hosts) != 0 || len(tags) != 0 || len(paths) != 0) {
- Warnf("Ignoring filters as there are explicit snapshot ids given\n")
- }
+ be, err := backend.MemorizeList(ctx, be, restic.SnapshotFile)
+ if err != nil {
+ Warnf("could not load snapshots: %v\n", err)
+ return
+ }
- for _, id := range ids.Uniq() {
- sn, err := restic.LoadSnapshot(ctx, loader, id)
- if err != nil {
- Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
- continue
- }
+ err = restic.FindFilteredSnapshots(ctx, be, loader, hosts, tags, paths, snapshotIDs, func(id string, sn *restic.Snapshot, err error) error {
+ if err != nil {
+ Warnf("Ignoring %q: %v\n", id, err)
+ } else {
select {
case <-ctx.Done():
- return
+ return ctx.Err()
case out <- sn:
}
}
- return
- }
-
- snapshots, err := restic.FindFilteredSnapshots(ctx, be, loader, hosts, tags, paths)
+ return nil
+ })
if err != nil {
Warnf("could not load snapshots: %v\n", err)
- return
- }
-
- for _, sn := range snapshots {
- select {
- case <-ctx.Done():
- return
- case out <- sn:
- }
}
}()
return out
diff --git a/cmd/restic/flags_test.go b/cmd/restic/flags_test.go
index b7f88e906..1ae20119c 100644
--- a/cmd/restic/flags_test.go
+++ b/cmd/restic/flags_test.go
@@ -1,7 +1,7 @@
package main
import (
- "io/ioutil"
+ "io"
"testing"
)
@@ -10,7 +10,7 @@ import (
func TestFlags(t *testing.T) {
for _, cmd := range cmdRoot.Commands() {
t.Run(cmd.Name(), func(t *testing.T) {
- cmd.Flags().SetOutput(ioutil.Discard)
+ cmd.Flags().SetOutput(io.Discard)
err := cmd.ParseFlags([]string{"--help"})
if err.Error() == "pflag: help requested" {
err = nil
diff --git a/cmd/restic/format.go b/cmd/restic/format.go
index 40918c897..2f14a4575 100644
--- a/cmd/restic/format.go
+++ b/cmd/restic/format.go
@@ -3,59 +3,10 @@ package main
import (
"fmt"
"os"
- "time"
"github.com/restic/restic/internal/restic"
)
-func formatBytes(c uint64) string {
- b := float64(c)
-
- switch {
- case c > 1<<40:
- return fmt.Sprintf("%.3f TiB", b/(1<<40))
- case c > 1<<30:
- return fmt.Sprintf("%.3f GiB", b/(1<<30))
- case c > 1<<20:
- return fmt.Sprintf("%.3f MiB", b/(1<<20))
- case c > 1<<10:
- return fmt.Sprintf("%.3f KiB", b/(1<<10))
- default:
- return fmt.Sprintf("%d B", c)
- }
-}
-
-func formatSeconds(sec uint64) string {
- hours := sec / 3600
- sec -= hours * 3600
- min := sec / 60
- sec -= min * 60
- if hours > 0 {
- return fmt.Sprintf("%d:%02d:%02d", hours, min, sec)
- }
-
- return fmt.Sprintf("%d:%02d", min, sec)
-}
-
-func formatPercent(numerator uint64, denominator uint64) string {
- if denominator == 0 {
- return ""
- }
-
- percent := 100.0 * float64(numerator) / float64(denominator)
-
- if percent > 100 {
- percent = 100
- }
-
- return fmt.Sprintf("%3.2f%%", percent)
-}
-
-func formatDuration(d time.Duration) string {
- sec := uint64(d / time.Second)
- return formatSeconds(sec)
-}
-
func formatNode(path string, n *restic.Node, long bool) string {
if !long {
return path
diff --git a/cmd/restic/global.go b/cmd/restic/global.go
index 2e7580aa7..517388e8d 100644
--- a/cmd/restic/global.go
+++ b/cmd/restic/global.go
@@ -22,6 +22,7 @@ import (
"github.com/restic/restic/internal/backend/location"
"github.com/restic/restic/internal/backend/rclone"
"github.com/restic/restic/internal/backend/rest"
+ "github.com/restic/restic/internal/backend/retry"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/sftp"
"github.com/restic/restic/internal/backend/swift"
@@ -41,7 +42,7 @@ import (
"golang.org/x/term"
)
-var version = "0.14.0"
+var version = "0.15.1"
// TimeFormat is the format used for all timestamps printed by restic.
const TimeFormat = "2006-01-02 15:04:05"
@@ -68,7 +69,6 @@ type GlobalOptions struct {
backend.TransportOptions
limiter.Limits
- ctx context.Context
password string
stdout io.Writer
stderr io.Writer
@@ -93,28 +93,26 @@ var globalOptions = GlobalOptions{
}
var isReadingPassword bool
+var internalGlobalCtx context.Context
func init() {
var cancel context.CancelFunc
- globalOptions.ctx, cancel = context.WithCancel(context.Background())
- AddCleanupHandler(func() error {
+ internalGlobalCtx, cancel = context.WithCancel(context.Background())
+ AddCleanupHandler(func(code int) (int, error) {
// Must be called before the unlock cleanup handler to ensure that the latter is
// not blocked due to limited number of backend connections, see #1434
cancel()
- return nil
+ return code, nil
})
- // parse target pack size from env, on error the default value will be used
- targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32)
-
f := cmdRoot.PersistentFlags()
- f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
- f.StringVarP(&globalOptions.RepositoryFile, "repository-file", "", os.Getenv("RESTIC_REPOSITORY_FILE"), "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)")
- f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", os.Getenv("RESTIC_PASSWORD_FILE"), "`file` to read the repository password from (default: $RESTIC_PASSWORD_FILE)")
- f.StringVarP(&globalOptions.KeyHint, "key-hint", "", os.Getenv("RESTIC_KEY_HINT"), "`key` ID of key to try decrypting first (default: $RESTIC_KEY_HINT)")
- f.StringVarP(&globalOptions.PasswordCommand, "password-command", "", os.Getenv("RESTIC_PASSWORD_COMMAND"), "shell `command` to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)")
+ f.StringVarP(&globalOptions.Repo, "repo", "r", "", "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
+ f.StringVarP(&globalOptions.RepositoryFile, "repository-file", "", "", "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)")
+ f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "`file` to read the repository password from (default: $RESTIC_PASSWORD_FILE)")
+ f.StringVarP(&globalOptions.KeyHint, "key-hint", "", "", "`key` ID of key to try decrypting first (default: $RESTIC_KEY_HINT)")
+ f.StringVarP(&globalOptions.PasswordCommand, "password-command", "", "", "shell `command` to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)")
f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
- f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=`n`, max level/times is 3)")
+ f.CountVarP(&globalOptions.Verbose, "verbose", "v", "be verbose (specify multiple times or a level using --verbose=`n`, max level/times is 2)")
f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repository, this allows some operations on read-only repositories")
f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
f.StringVar(&globalOptions.CacheDir, "cache-dir", "", "set the cache `directory`. (default: use system default cache directory)")
@@ -124,18 +122,26 @@ func init() {
f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)")
f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories")
f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max)")
- f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum rate in KiB/s. (default: unlimited)")
- f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum rate in KiB/s. (default: unlimited)")
- f.UintVar(&globalOptions.PackSize, "pack-size", uint(targetPackSize), "set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
+ f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)")
+ f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)")
+ f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)")
f.StringSliceVarP(&globalOptions.Options, "option", "o", []string{}, "set extended option (`key=value`, can be specified multiple times)")
// Use our "generate" command instead of the cobra provided "completion" command
cmdRoot.CompletionOptions.DisableDefaultCmd = true
+ globalOptions.Repo = os.Getenv("RESTIC_REPOSITORY")
+ globalOptions.RepositoryFile = os.Getenv("RESTIC_REPOSITORY_FILE")
+ globalOptions.PasswordFile = os.Getenv("RESTIC_PASSWORD_FILE")
+ globalOptions.KeyHint = os.Getenv("RESTIC_KEY_HINT")
+ globalOptions.PasswordCommand = os.Getenv("RESTIC_PASSWORD_COMMAND")
comp := os.Getenv("RESTIC_COMPRESSION")
if comp != "" {
// ignore error as there's no good way to handle it
_ = globalOptions.Compression.Set(comp)
}
+ // parse target pack size from env, on error the default value will be used
+ targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32)
+ globalOptions.PackSize = uint(targetPackSize)
restoreTerminal()
}
@@ -194,20 +200,20 @@ func restoreTerminal() {
return
}
- AddCleanupHandler(func() error {
+ AddCleanupHandler(func(code int) (int, error) {
// Restoring the terminal configuration while restic runs in the
// background, causes restic to get stopped on unix systems with
// a SIGTTOU signal. Thus only restore the terminal settings if
// they might have been modified, which is the case while reading
// a password.
if !isReadingPassword {
- return nil
+ return code, nil
}
err := checkErrno(term.Restore(fd, state))
if err != nil {
fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err)
}
- return err
+ return code, err
})
}
@@ -275,17 +281,6 @@ func Warnf(format string, args ...interface{}) {
}
}
-// Exitf uses Warnf to write the message and then terminates the process with
-// the given exit code.
-func Exitf(exitcode int, format string, args ...interface{}) {
- if !(strings.HasSuffix(format, "\n")) {
- format += "\n"
- }
-
- Warnf(format, args...)
- Exit(exitcode)
-}
-
// resolvePassword determines the password to be used for opening the repository.
func resolvePassword(opts GlobalOptions, envStr string) (string, error) {
if opts.PasswordFile != "" && opts.PasswordCommand != "" {
@@ -423,20 +418,24 @@ func ReadRepo(opts GlobalOptions) (string, error) {
const maxKeys = 20
// OpenRepository reads the password and opens the repository.
-func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
+func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Repository, error) {
repo, err := ReadRepo(opts)
if err != nil {
return nil, err
}
- be, err := open(repo, opts, opts.extended)
+ be, err := open(ctx, repo, opts, opts.extended)
if err != nil {
return nil, err
}
- be = backend.NewRetryBackend(be, 10, func(msg string, err error, d time.Duration) {
+ report := func(msg string, err error, d time.Duration) {
Warnf("%v returned error, retrying after %v: %v\n", msg, d, err)
- })
+ }
+ success := func(msg string, retries int) {
+ Warnf("%v operation successful after %d retries\n", msg, retries)
+ }
+ be = retry.New(be, 10, report, success)
// wrap backend if a test specified a hook
if opts.backendTestHook != nil {
@@ -469,7 +468,7 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
continue
}
- err = s.SearchKey(opts.ctx, opts.password, maxKeys, opts.KeyHint)
+ err = s.SearchKey(ctx, opts.password, maxKeys, opts.KeyHint)
if err != nil && passwordTriesLeft > 1 {
opts.password = ""
fmt.Fprintf(os.Stderr, "%s. Try again\n", err)
@@ -488,7 +487,11 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
id = id[:8]
}
if !opts.JSON {
- Verbosef("repository %v opened (repository version %v) successfully, password is correct\n", id, s.Config().Version)
+ extra := ""
+ if s.Config().Version >= 2 {
+ extra = ", compression level " + opts.Compression.String()
+ }
+ Verbosef("repository %v opened (version %v%s)\n", id, s.Config().Version, extra)
}
}
@@ -686,7 +689,7 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
}
// Open the backend specified by a location config.
-func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
+func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (restic.Backend, error) {
debug.Log("parsing location %v", location.StripPassword(s))
loc, err := location.Parse(s)
if err != nil {
@@ -711,19 +714,19 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
switch loc.Scheme {
case "local":
- be, err = local.Open(globalOptions.ctx, cfg.(local.Config))
+ be, err = local.Open(ctx, cfg.(local.Config))
case "sftp":
- be, err = sftp.Open(globalOptions.ctx, cfg.(sftp.Config))
+ be, err = sftp.Open(ctx, cfg.(sftp.Config))
case "s3":
- be, err = s3.Open(globalOptions.ctx, cfg.(s3.Config), rt)
+ be, err = s3.Open(ctx, cfg.(s3.Config), rt)
case "gs":
be, err = gs.Open(cfg.(gs.Config), rt)
case "azure":
- be, err = azure.Open(cfg.(azure.Config), rt)
+ be, err = azure.Open(ctx, cfg.(azure.Config), rt)
case "swift":
- be, err = swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
+ be, err = swift.Open(ctx, cfg.(swift.Config), rt)
case "b2":
- be, err = b2.Open(globalOptions.ctx, cfg.(b2.Config), rt)
+ be, err = b2.Open(ctx, cfg.(b2.Config), rt)
case "rest":
be, err = rest.Open(cfg.(rest.Config), rt)
case "rclone":
@@ -751,7 +754,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
}
// check if config is there
- fi, err := be.Stat(globalOptions.ctx, restic.Handle{Type: restic.ConfigFile})
+ fi, err := be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
if err != nil {
return nil, errors.Fatalf("unable to open config file: %v\nIs there a repository at the following location?\n%v", err, location.StripPassword(s))
}
@@ -764,7 +767,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
}
// Create the backend specified by URI.
-func create(s string, opts options.Options) (restic.Backend, error) {
+func create(ctx context.Context, s string, opts options.Options) (restic.Backend, error) {
debug.Log("parsing location %v", s)
loc, err := location.Parse(s)
if err != nil {
@@ -783,23 +786,23 @@ func create(s string, opts options.Options) (restic.Backend, error) {
switch loc.Scheme {
case "local":
- return local.Create(globalOptions.ctx, cfg.(local.Config))
+ return local.Create(ctx, cfg.(local.Config))
case "sftp":
- return sftp.Create(globalOptions.ctx, cfg.(sftp.Config))
+ return sftp.Create(ctx, cfg.(sftp.Config))
case "s3":
- return s3.Create(globalOptions.ctx, cfg.(s3.Config), rt)
+ return s3.Create(ctx, cfg.(s3.Config), rt)
case "gs":
return gs.Create(cfg.(gs.Config), rt)
case "azure":
- return azure.Create(cfg.(azure.Config), rt)
+ return azure.Create(ctx, cfg.(azure.Config), rt)
case "swift":
- return swift.Open(globalOptions.ctx, cfg.(swift.Config), rt)
+ return swift.Open(ctx, cfg.(swift.Config), rt)
case "b2":
- return b2.Create(globalOptions.ctx, cfg.(b2.Config), rt)
+ return b2.Create(ctx, cfg.(b2.Config), rt)
case "rest":
- return rest.Create(globalOptions.ctx, cfg.(rest.Config), rt)
+ return rest.Create(ctx, cfg.(rest.Config), rt)
case "rclone":
- return rclone.Create(globalOptions.ctx, cfg.(rclone.Config))
+ return rclone.Create(ctx, cfg.(rclone.Config))
}
debug.Log("invalid repository scheme: %v", s)
diff --git a/cmd/restic/global_debug.go b/cmd/restic/global_debug.go
index 172f3451b..b798074d1 100644
--- a/cmd/restic/global_debug.go
+++ b/cmd/restic/global_debug.go
@@ -84,9 +84,9 @@ func runDebug() error {
}
if prof != nil {
- AddCleanupHandler(func() error {
+ AddCleanupHandler(func(code int) (int, error) {
prof.Stop()
- return nil
+ return code, nil
})
}
diff --git a/cmd/restic/global_test.go b/cmd/restic/global_test.go
index fee5294b5..85a9514b9 100644
--- a/cmd/restic/global_test.go
+++ b/cmd/restic/global_test.go
@@ -2,7 +2,7 @@ package main
import (
"bytes"
- "io/ioutil"
+ "os"
"path/filepath"
"testing"
@@ -31,8 +31,7 @@ func Test_PrintFunctionsRespectsGlobalStdout(t *testing.T) {
}
func TestReadRepo(t *testing.T) {
- tempDir, cleanup := test.TempDir(t)
- defer cleanup()
+ tempDir := test.TempDir(t)
// test --repo option
var opts GlobalOptions
@@ -43,7 +42,7 @@ func TestReadRepo(t *testing.T) {
// test --repository-file option
foo := filepath.Join(tempDir, "foo")
- err = ioutil.WriteFile(foo, []byte(tempDir+"\n"), 0666)
+ err = os.WriteFile(foo, []byte(tempDir+"\n"), 0666)
rtest.OK(t, err)
var opts2 GlobalOptions
diff --git a/cmd/restic/integration_filter_pattern_test.go b/cmd/restic/integration_filter_pattern_test.go
index c0c1d932f..ea5753d20 100644
--- a/cmd/restic/integration_filter_pattern_test.go
+++ b/cmd/restic/integration_filter_pattern_test.go
@@ -1,14 +1,7 @@
-//go:build go1.16
-// +build go1.16
-
-// Before Go 1.16 filepath.Match returned early on a failed match,
-// and thus did not report any later syntax error in the pattern.
-// https://go.dev/doc/go1.16#path/filepath
-
package main
import (
- "io/ioutil"
+ "os"
"path/filepath"
"testing"
@@ -24,14 +17,14 @@ func TestBackupFailsWhenUsingInvalidPatterns(t *testing.T) {
var err error
// Test --exclude
- err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
+ err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --iexclude
- err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}, env.gopts)
+ err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
*[._]log[.-][0-9]
@@ -46,7 +39,7 @@ func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
// Create an exclude file with some invalid patterns
excludeFile := env.base + "/excludefile"
- fileErr := ioutil.WriteFile(excludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
+ fileErr := os.WriteFile(excludeFile, []byte("*.go\n*[._]log[.-][0-9]\n!*[._]log[.-][0-9]"), 0644)
if fileErr != nil {
t.Fatalf("Could not write exclude file: %v", fileErr)
}
@@ -54,14 +47,14 @@ func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
var err error
// Test --exclude-file:
- err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludeFiles: []string{excludeFile}}, env.gopts)
+ err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
!*[._]log[.-][0-9]`, err.Error())
// Test --iexclude-file
- err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{InsensitiveExcludeFiles: []string{excludeFile}}, env.gopts)
+ err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts)
rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
*[._]log[.-][0-9]
diff --git a/cmd/restic/integration_fuse_test.go b/cmd/restic/integration_fuse_test.go
index 6a95ac87d..a99064b8f 100644
--- a/cmd/restic/integration_fuse_test.go
+++ b/cmd/restic/integration_fuse_test.go
@@ -4,9 +4,11 @@
package main
import (
+ "context"
"fmt"
"os"
"path/filepath"
+ "sync"
"testing"
"time"
@@ -53,11 +55,12 @@ func waitForMount(t testing.TB, dir string) {
t.Errorf("subdir %q of dir %s never appeared", mountTestSubdir, dir)
}
-func testRunMount(t testing.TB, gopts GlobalOptions, dir string) {
+func testRunMount(t testing.TB, gopts GlobalOptions, dir string, wg *sync.WaitGroup) {
+ defer wg.Done()
opts := MountOptions{
TimeTemplate: time.RFC3339,
}
- rtest.OK(t, runMount(opts, gopts, []string{dir}))
+ rtest.OK(t, runMount(context.TODO(), opts, gopts, []string{dir}))
}
func testRunUmount(t testing.TB, gopts GlobalOptions, dir string) {
@@ -86,8 +89,11 @@ func listSnapshots(t testing.TB, dir string) []string {
func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Repository, mountpoint, repodir string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) {
t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs)
- go testRunMount(t, global, mountpoint)
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go testRunMount(t, global, mountpoint, &wg)
waitForMount(t, mountpoint)
+ defer wg.Wait()
defer testRunUmount(t, global, mountpoint)
if !snapshotsDirExists(t, mountpoint) {
@@ -119,7 +125,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
}
for _, id := range snapshotIDs {
- snapshot, err := restic.LoadSnapshot(global.ctx, repo, id)
+ snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
rtest.OK(t, err)
ts := snapshot.Time.Format(time.RFC3339)
@@ -160,7 +166,7 @@ func TestMount(t *testing.T) {
testRunInit(t, env.gopts)
- repo, err := OpenRepository(env.gopts)
+ repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{}, 0)
@@ -205,7 +211,7 @@ func TestMountSameTimestamps(t *testing.T) {
rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))
- repo, err := OpenRepository(env.gopts)
+ repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
ids := []restic.ID{
diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go
index e87baddca..655aa9335 100644
--- a/cmd/restic/integration_helpers_test.go
+++ b/cmd/restic/integration_helpers_test.go
@@ -2,14 +2,13 @@ package main
import (
"bytes"
- "context"
"fmt"
- "io/ioutil"
"os"
"path/filepath"
"runtime"
"testing"
+ "github.com/restic/restic/internal/backend/retry"
"github.com/restic/restic/internal/options"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@@ -172,8 +171,9 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
repository.TestUseLowSecurityKDFParameters(t)
restic.TestDisableCheckPolynomial(t)
+ retry.TestFastRetries(t)
- tempdir, err := ioutil.TempDir(rtest.TestTempDir, "restic-test-")
+ tempdir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-")
rtest.OK(t, err)
env = &testEnvironment{
@@ -193,7 +193,6 @@ func withTestEnvironment(t testing.TB) (env *testEnvironment, cleanup func()) {
Repo: env.repo,
Quiet: true,
CacheDir: env.cache,
- ctx: context.Background(),
password: rtest.TestPassword,
stdout: os.Stdout,
stderr: os.Stderr,
diff --git a/cmd/restic/integration_helpers_unix_test.go b/cmd/restic/integration_helpers_unix_test.go
index 830d41b3d..df0c4fe63 100644
--- a/cmd/restic/integration_helpers_unix_test.go
+++ b/cmd/restic/integration_helpers_unix_test.go
@@ -6,7 +6,6 @@ package main
import (
"fmt"
"io"
- "io/ioutil"
"os"
"path/filepath"
"syscall"
@@ -57,7 +56,7 @@ func nlink(info os.FileInfo) uint64 {
func createFileSetPerHardlink(dir string) map[uint64][]string {
var stat syscall.Stat_t
linkTests := make(map[uint64][]string)
- files, err := ioutil.ReadDir(dir)
+ files, err := os.ReadDir(dir)
if err != nil {
return nil
}
diff --git a/cmd/restic/integration_helpers_windows_test.go b/cmd/restic/integration_helpers_windows_test.go
index a46d1e5cd..4f2c8b54f 100644
--- a/cmd/restic/integration_helpers_windows_test.go
+++ b/cmd/restic/integration_helpers_windows_test.go
@@ -6,7 +6,6 @@ package main
import (
"fmt"
"io"
- "io/ioutil"
"os"
)
@@ -39,7 +38,7 @@ func inode(info os.FileInfo) uint64 {
func createFileSetPerHardlink(dir string) map[uint64][]string {
linkTests := make(map[uint64][]string)
- files, err := ioutil.ReadDir(dir)
+ files, err := os.ReadDir(dir)
if err != nil {
return nil
}
diff --git a/cmd/restic/integration_rewrite_test.go b/cmd/restic/integration_rewrite_test.go
new file mode 100644
index 000000000..e6007973b
--- /dev/null
+++ b/cmd/restic/integration_rewrite_test.go
@@ -0,0 +1,73 @@
+package main
+
+import (
+ "context"
+ "path/filepath"
+ "testing"
+
+ "github.com/restic/restic/internal/restic"
+ rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool) {
+ opts := RewriteOptions{
+ excludePatternOptions: excludePatternOptions{
+ Excludes: excludes,
+ },
+ Forget: forget,
+ }
+
+ rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil))
+}
+
+func createBasicRewriteRepo(t testing.TB, env *testEnvironment) restic.ID {
+ testSetupBackupData(t, env)
+
+ // create backup
+ testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts)
+ snapshotIDs := testRunList(t, "snapshots", env.gopts)
+ rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs)
+ testRunCheck(t, env.gopts)
+
+ return snapshotIDs[0]
+}
+
+func TestRewrite(t *testing.T) {
+ env, cleanup := withTestEnvironment(t)
+ defer cleanup()
+ createBasicRewriteRepo(t, env)
+
+ // exclude some data
+ testRunRewriteExclude(t, env.gopts, []string{"3"}, false)
+ snapshotIDs := testRunList(t, "snapshots", env.gopts)
+ rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs)
+ testRunCheck(t, env.gopts)
+}
+
+func TestRewriteUnchanged(t *testing.T) {
+ env, cleanup := withTestEnvironment(t)
+ defer cleanup()
+ snapshotID := createBasicRewriteRepo(t, env)
+
+ // use an exclude that will not exclude anything
+ testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false)
+ newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
+ rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
+ rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly")
+ testRunCheck(t, env.gopts)
+}
+
+func TestRewriteReplace(t *testing.T) {
+ env, cleanup := withTestEnvironment(t)
+ defer cleanup()
+ snapshotID := createBasicRewriteRepo(t, env)
+
+ // exclude some data
+ testRunRewriteExclude(t, env.gopts, []string{"3"}, true)
+ newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
+ rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
+ rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed")
+ // check forbids unused blobs, thus remove them first
+ testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"})
+ testRunCheck(t, env.gopts)
+}
diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go
index c04a5a2fb..062a5954c 100644
--- a/cmd/restic/integration_test.go
+++ b/cmd/restic/integration_test.go
@@ -8,7 +8,6 @@ import (
"encoding/json"
"fmt"
"io"
- "io/ioutil"
mrand "math/rand"
"os"
"path/filepath"
@@ -23,6 +22,7 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/filter"
"github.com/restic/restic/internal/fs"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
@@ -52,26 +52,26 @@ func testRunInit(t testing.TB, opts GlobalOptions) {
restic.TestDisableCheckPolynomial(t)
restic.TestSetLockTimeout(t, 0)
- rtest.OK(t, runInit(InitOptions{}, opts, nil))
+ rtest.OK(t, runInit(context.TODO(), InitOptions{}, opts, nil))
t.Logf("repository initialized at %v", opts.Repo)
}
func testRunBackupAssumeFailure(t testing.TB, dir string, target []string, opts BackupOptions, gopts GlobalOptions) error {
- ctx, cancel := context.WithCancel(gopts.ctx)
+ ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
var wg errgroup.Group
term := termstatus.New(gopts.stdout, gopts.stderr, gopts.Quiet)
wg.Go(func() error { term.Run(ctx); return nil })
- gopts.stdout = ioutil.Discard
+ gopts.stdout = io.Discard
t.Logf("backing up %v in %v", target, dir)
if dir != "" {
cleanup := rtest.Chdir(t, dir)
defer cleanup()
}
- backupErr := runBackup(opts, gopts, term, target)
+ backupErr := runBackup(ctx, opts, gopts, term, target)
cancel()
@@ -95,7 +95,7 @@ func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {
globalOptions.stdout = os.Stdout
}()
- rtest.OK(t, runList(cmdList, opts, []string{tpe}))
+ rtest.OK(t, runList(context.TODO(), cmdList, opts, []string{tpe}))
return parseIDsFromReader(t, buf)
}
@@ -106,11 +106,13 @@ func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID res
func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) {
opts := RestoreOptions{
Target: dir,
- Hosts: hosts,
- Paths: paths,
+ snapshotFilterOptions: snapshotFilterOptions{
+ Hosts: hosts,
+ Paths: paths,
+ },
}
- rtest.OK(t, runRestore(opts, gopts, []string{"latest"}))
+ rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{"latest"}))
}
func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) {
@@ -119,7 +121,7 @@ func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snaps
Exclude: excludes,
}
- rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()}))
+ rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()}))
}
func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) {
@@ -128,11 +130,11 @@ func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snaps
Include: includes,
}
- rtest.OK(t, runRestore(opts, gopts, []string{snapshotID.String()}))
+ rtest.OK(t, runRestore(context.TODO(), opts, gopts, []string{snapshotID.String()}))
}
func testRunRestoreAssumeFailure(t testing.TB, snapshotID string, opts RestoreOptions, gopts GlobalOptions) error {
- err := runRestore(opts, gopts, []string{snapshotID})
+ err := runRestore(context.TODO(), opts, gopts, []string{snapshotID})
return err
}
@@ -142,7 +144,7 @@ func testRunCheck(t testing.TB, gopts GlobalOptions) {
ReadData: true,
CheckUnused: true,
}
- rtest.OK(t, runCheck(opts, gopts, nil))
+ rtest.OK(t, runCheck(context.TODO(), opts, gopts, nil))
}
func testRunCheckOutput(gopts GlobalOptions) (string, error) {
@@ -157,7 +159,7 @@ func testRunCheckOutput(gopts GlobalOptions) (string, error) {
ReadData: true,
}
- err := runCheck(opts, gopts, nil)
+ err := runCheck(context.TODO(), opts, gopts, nil)
return buf.String(), err
}
@@ -175,17 +177,17 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh
opts := DiffOptions{
ShowMetadata: false,
}
- err := runDiff(opts, gopts, []string{firstSnapshotID, secondSnapshotID})
+ err := runDiff(context.TODO(), opts, gopts, []string{firstSnapshotID, secondSnapshotID})
return buf.String(), err
}
func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) {
- globalOptions.stdout = ioutil.Discard
+ globalOptions.stdout = io.Discard
defer func() {
globalOptions.stdout = os.Stdout
}()
- rtest.OK(t, runRebuildIndex(RebuildIndexOptions{}, gopts))
+ rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, gopts))
}
func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
@@ -200,7 +202,7 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
opts := LsOptions{}
- rtest.OK(t, runLs(opts, gopts, []string{snapshotID}))
+ rtest.OK(t, runLs(context.TODO(), opts, gopts, []string{snapshotID}))
return strings.Split(buf.String(), "\n")
}
@@ -216,7 +218,7 @@ func testRunFind(t testing.TB, wantJSON bool, gopts GlobalOptions, pattern strin
opts := FindOptions{}
- rtest.OK(t, runFind(opts, gopts, []string{pattern}))
+ rtest.OK(t, runFind(context.TODO(), opts, gopts, []string{pattern}))
return buf.Bytes()
}
@@ -232,7 +234,7 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snap
opts := SnapshotOptions{}
- rtest.OK(t, runSnapshots(opts, globalOptions, []string{}))
+ rtest.OK(t, runSnapshots(context.TODO(), opts, globalOptions, []string{}))
snapshots := []Snapshot{}
rtest.OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
@@ -249,7 +251,7 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snap
func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
opts := ForgetOptions{}
- rtest.OK(t, runForget(opts, gopts, args))
+ rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
}
func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
@@ -267,7 +269,7 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
Last: 1,
}
- rtest.OK(t, runForget(opts, gopts, args))
+ rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
var forgets []*ForgetGroup
rtest.OK(t, json.Unmarshal(buf.Bytes(), &forgets))
@@ -286,7 +288,7 @@ func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
defer func() {
gopts.backendTestHook = oldHook
}()
- rtest.OK(t, runPrune(opts, gopts))
+ rtest.OK(t, runPrune(context.TODO(), opts, gopts))
}
func testSetupBackupData(t testing.TB, env *testEnvironment) string {
@@ -416,7 +418,7 @@ func TestBackupNonExistingFile(t *testing.T) {
defer cleanup()
testSetupBackupData(t, env)
- globalOptions.stderr = ioutil.Discard
+ globalOptions.stderr = io.Discard
defer func() {
globalOptions.stderr = os.Stderr
}()
@@ -435,25 +437,25 @@ func TestBackupNonExistingFile(t *testing.T) {
}
func removePacksExcept(gopts GlobalOptions, t *testing.T, keep restic.IDSet, removeTreePacks bool) {
- r, err := OpenRepository(gopts)
+ r, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err)
// Get all tree packs
- rtest.OK(t, r.LoadIndex(gopts.ctx))
+ rtest.OK(t, r.LoadIndex(context.TODO()))
treePacks := restic.NewIDSet()
- for pb := range r.Index().Each(context.TODO()) {
+ r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
if pb.Type == restic.TreeBlob {
treePacks.Insert(pb.PackID)
}
- }
+ })
// remove all packs containing data blobs
- rtest.OK(t, r.List(gopts.ctx, restic.PackFile, func(id restic.ID, size int64) error {
+ rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
if treePacks.Has(id) != removeTreePacks || keep.Has(id) {
return nil
}
- return r.Backend().Remove(gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()})
+ return r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()})
}))
}
@@ -477,7 +479,7 @@ func TestBackupSelfHealing(t *testing.T) {
testRunRebuildIndex(t, env.gopts)
// now the repo is also missing the data blob in the index; check should report this
- rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil,
+ rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil,
"check should have reported an error")
// second backup should report an error but "heal" this situation
@@ -500,26 +502,26 @@ func TestBackupTreeLoadError(t *testing.T) {
// Backup a subdirectory first, such that we can remove the tree pack for the subdirectory
testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts)
- r, err := OpenRepository(env.gopts)
+ r, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
- rtest.OK(t, r.LoadIndex(env.gopts.ctx))
+ rtest.OK(t, r.LoadIndex(context.TODO()))
treePacks := restic.NewIDSet()
- for pb := range r.Index().Each(context.TODO()) {
+ r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
if pb.Type == restic.TreeBlob {
treePacks.Insert(pb.PackID)
}
- }
+ })
testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
testRunCheck(t, env.gopts)
// delete the subdirectory pack first
for id := range treePacks {
- rtest.OK(t, r.Backend().Remove(env.gopts.ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}))
+ rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
}
testRunRebuildIndex(t, env.gopts)
// now the repo is missing the tree blob in the index; check should report this
- rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
+ rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
// second backup should report an error but "heal" this situation
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory")
@@ -529,7 +531,7 @@ func TestBackupTreeLoadError(t *testing.T) {
removePacksExcept(env.gopts, t, restic.NewIDSet(), true)
testRunRebuildIndex(t, env.gopts)
// now the repo is also missing the data blob in the index; check should report this
- rtest.Assert(t, runCheck(CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
+ rtest.Assert(t, runCheck(context.TODO(), CheckOptions{}, env.gopts, nil) != nil, "check should have reported an error")
// second backup should report an error but "heal" this situation
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
rtest.Assert(t, err != nil, "backup should have reported an error")
@@ -638,7 +640,7 @@ func TestBackupErrors(t *testing.T) {
}()
opts := BackupOptions{}
gopts := env.gopts
- gopts.stderr = ioutil.Discard
+ gopts.stderr = io.Discard
err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, gopts)
rtest.Assert(t, err != nil, "Assumed failure, but no error occurred.")
rtest.Assert(t, err == ErrInvalidSourceData, "Wrong error returned")
@@ -759,7 +761,7 @@ func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) {
},
}
- rtest.OK(t, runCopy(copyOpts, gopts, nil))
+ rtest.OK(t, runCopy(context.TODO(), copyOpts, gopts, nil))
}
func TestCopy(t *testing.T) {
@@ -901,15 +903,15 @@ func TestInitCopyChunkerParams(t *testing.T) {
password: env2.gopts.password,
},
}
- rtest.Assert(t, runInit(initOpts, env.gopts, nil) != nil, "expected invalid init options to fail")
+ rtest.Assert(t, runInit(context.TODO(), initOpts, env.gopts, nil) != nil, "expected invalid init options to fail")
initOpts.CopyChunkerParameters = true
- rtest.OK(t, runInit(initOpts, env.gopts, nil))
+ rtest.OK(t, runInit(context.TODO(), initOpts, env.gopts, nil))
- repo, err := OpenRepository(env.gopts)
+ repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
- otherRepo, err := OpenRepository(env2.gopts)
+ otherRepo, err := OpenRepository(context.TODO(), env2.gopts)
rtest.OK(t, err)
rtest.Assert(t, repo.Config().ChunkerPolynomial == otherRepo.Config().ChunkerPolynomial,
@@ -918,7 +920,7 @@ func TestInitCopyChunkerParams(t *testing.T) {
}
func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
- rtest.OK(t, runTag(opts, gopts, []string{}))
+ rtest.OK(t, runTag(context.TODO(), opts, gopts, []string{}))
}
func TestTag(t *testing.T) {
@@ -1010,7 +1012,7 @@ func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
globalOptions.stdout = os.Stdout
}()
- rtest.OK(t, runKey(gopts, []string{"list"}))
+ rtest.OK(t, runKey(context.TODO(), gopts, []string{"list"}))
scanner := bufio.NewScanner(buf)
exp := regexp.MustCompile(`^ ([a-f0-9]+) `)
@@ -1031,7 +1033,7 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions)
testKeyNewPassword = ""
}()
- rtest.OK(t, runKey(gopts, []string{"add"}))
+ rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
}
func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
@@ -1045,11 +1047,11 @@ func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"}))
t.Log("adding key for john@example.com")
- rtest.OK(t, runKey(gopts, []string{"add"}))
+ rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
- repo, err := OpenRepository(gopts)
+ repo, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err)
- key, err := repository.SearchKey(gopts.ctx, repo, testKeyNewPassword, 2, "")
+ key, err := repository.SearchKey(context.TODO(), repo, testKeyNewPassword, 2, "")
rtest.OK(t, err)
rtest.Equals(t, "john", key.Username)
@@ -1062,13 +1064,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) {
testKeyNewPassword = ""
}()
- rtest.OK(t, runKey(gopts, []string{"passwd"}))
+ rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"}))
}
func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) {
t.Logf("remove %d keys: %q\n", len(IDs), IDs)
for _, id := range IDs {
- rtest.OK(t, runKey(gopts, []string{"remove", id}))
+ rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id}))
}
}
@@ -1098,7 +1100,7 @@ func TestKeyAddRemove(t *testing.T) {
env.gopts.password = passwordList[len(passwordList)-1]
t.Logf("testing access with last password %q\n", env.gopts.password)
- rtest.OK(t, runKey(env.gopts, []string{"list"}))
+ rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
testRunCheck(t, env.gopts)
testRunKeyAddNewKeyUserHost(t, env.gopts)
@@ -1126,16 +1128,16 @@ func TestKeyProblems(t *testing.T) {
testKeyNewPassword = ""
}()
- err := runKey(env.gopts, []string{"passwd"})
+ err := runKey(context.TODO(), env.gopts, []string{"passwd"})
t.Log(err)
rtest.Assert(t, err != nil, "expected passwd change to fail")
- err = runKey(env.gopts, []string{"add"})
+ err = runKey(context.TODO(), env.gopts, []string{"add"})
t.Log(err)
rtest.Assert(t, err != nil, "expected key adding to fail")
t.Logf("testing access with initial password %q\n", env.gopts.password)
- rtest.OK(t, runKey(env.gopts, []string{"list"}))
+ rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
testRunCheck(t, env.gopts)
}
@@ -1195,7 +1197,7 @@ func TestRestoreFilter(t *testing.T) {
if ok, _ := filter.Match(pat, filepath.Base(testFile.name)); !ok {
rtest.OK(t, err)
} else {
- rtest.Assert(t, os.IsNotExist(errors.Cause(err)),
+ rtest.Assert(t, os.IsNotExist(err),
"expected %v to not exist in restore step %v, but it exists, err %v", testFile.name, i+1, err)
}
}
@@ -1240,7 +1242,7 @@ func TestRestoreLatest(t *testing.T) {
opts := BackupOptions{}
// chdir manually here so we can get the current directory. This is not the
- // same as the temp dir returned by ioutil.TempDir() on darwin.
+ // same as the temp dir returned by os.MkdirTemp() on darwin.
back := rtest.Chdir(t, filepath.Dir(env.testdata))
defer back()
@@ -1281,15 +1283,15 @@ func TestRestoreLatest(t *testing.T) {
testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore1"), []string{filepath.Dir(p1)}, nil)
rtest.OK(t, testFileSize(p1rAbs, int64(102)))
- if _, err := os.Stat(p2rAbs); os.IsNotExist(errors.Cause(err)) {
- rtest.Assert(t, os.IsNotExist(errors.Cause(err)),
+ if _, err := os.Stat(p2rAbs); os.IsNotExist(err) {
+ rtest.Assert(t, os.IsNotExist(err),
"expected %v to not exist in restore, but it exists, err %v", p2rAbs, err)
}
testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore2"), []string{filepath.Dir(p2)}, nil)
rtest.OK(t, testFileSize(p2rAbs, int64(103)))
- if _, err := os.Stat(p1rAbs); os.IsNotExist(errors.Cause(err)) {
- rtest.Assert(t, os.IsNotExist(errors.Cause(err)),
+ if _, err := os.Stat(p1rAbs); os.IsNotExist(err) {
+ rtest.Assert(t, os.IsNotExist(err),
"expected %v to not exist in restore, but it exists, err %v", p1rAbs, err)
}
}
@@ -1305,7 +1307,7 @@ func TestRestoreWithPermissionFailure(t *testing.T) {
rtest.Assert(t, len(snapshots) > 0,
"no snapshots found in repo (%v)", datafile)
- globalOptions.stderr = ioutil.Discard
+ globalOptions.stderr = io.Discard
defer func() {
globalOptions.stderr = os.Stderr
}()
@@ -1475,11 +1477,11 @@ func TestRebuildIndex(t *testing.T) {
}
func TestRebuildIndexAlwaysFull(t *testing.T) {
- indexFull := repository.IndexFull
+ indexFull := index.IndexFull
defer func() {
- repository.IndexFull = indexFull
+ index.IndexFull = indexFull
}()
- repository.IndexFull = func(*repository.Index, bool) bool { return true }
+ index.IndexFull = func(*index.Index, bool) bool { return true }
testRebuildIndex(t, nil)
}
@@ -1539,7 +1541,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
datafile := filepath.Join("..", "..", "internal", "checker", "testdata", "duplicate-packs-in-index-test-repo.tar.gz")
rtest.SetupTarTestFixture(t, env.base, datafile)
- globalOptions.stdout = ioutil.Discard
+ globalOptions.stdout = io.Discard
defer func() {
globalOptions.stdout = os.Stdout
}()
@@ -1547,7 +1549,7 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
return &appendOnlyBackend{r}, nil
}
- err := runRebuildIndex(RebuildIndexOptions{}, env.gopts)
+ err := runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts)
if err == nil {
t.Error("expected rebuildIndex to fail")
}
@@ -1621,10 +1623,7 @@ func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) {
})
}
-func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
- env, cleanup := withTestEnvironment(t)
- defer cleanup()
-
+func createPrunableRepo(t *testing.T, env *testEnvironment) {
testSetupBackupData(t, env)
opts := BackupOptions{}
@@ -1642,19 +1641,26 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
testRunForgetJSON(t, env.gopts)
testRunForget(t, env.gopts, firstSnapshot[0].String())
+}
+
+func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
+ env, cleanup := withTestEnvironment(t)
+ defer cleanup()
+
+ createPrunableRepo(t, env)
testRunPrune(t, env.gopts, pruneOpts)
- rtest.OK(t, runCheck(checkOpts, env.gopts, nil))
+ rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
}
var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"}
func listPacks(gopts GlobalOptions, t *testing.T) restic.IDSet {
- r, err := OpenRepository(gopts)
+ r, err := OpenRepository(context.TODO(), gopts)
rtest.OK(t, err)
packs := restic.NewIDSet()
- rtest.OK(t, r.List(gopts.ctx, restic.PackFile, func(id restic.ID, size int64) error {
+ rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
packs.Insert(id)
return nil
}))
@@ -1695,7 +1701,7 @@ func TestPruneWithDamagedRepository(t *testing.T) {
env.gopts.backendTestHook = oldHook
}()
// prune should fail
- rtest.Assert(t, runPrune(pruneDefaultOptions, env.gopts) == errorPacksMissing,
+ rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing,
"prune should have reported index not complete error")
}
@@ -1767,7 +1773,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
if checkOK {
testRunCheck(t, env.gopts)
} else {
- rtest.Assert(t, runCheck(optionsCheck, env.gopts, nil) != nil,
+ rtest.Assert(t, runCheck(context.TODO(), optionsCheck, env.gopts, nil) != nil,
"check should have reported an error")
}
@@ -1775,7 +1781,7 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
testRunPrune(t, env.gopts, optionsPrune)
testRunCheck(t, env.gopts)
} else {
- rtest.Assert(t, runPrune(optionsPrune, env.gopts) != nil,
+ rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil,
"prune should have reported an error")
}
}
@@ -1824,32 +1830,15 @@ func TestListOnce(t *testing.T) {
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
return newListOnceBackend(r), nil
}
-
pruneOpts := PruneOptions{MaxUnused: "0"}
checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
- testSetupBackupData(t, env)
- opts := BackupOptions{}
-
- testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
- firstSnapshot := testRunList(t, "snapshots", env.gopts)
- rtest.Assert(t, len(firstSnapshot) == 1,
- "expected one snapshot, got %v", firstSnapshot)
-
- testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts)
- testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
-
- snapshotIDs := testRunList(t, "snapshots", env.gopts)
- rtest.Assert(t, len(snapshotIDs) == 3,
- "expected 3 snapshot, got %v", snapshotIDs)
-
- testRunForgetJSON(t, env.gopts)
- testRunForget(t, env.gopts, firstSnapshot[0].String())
+ createPrunableRepo(t, env)
testRunPrune(t, env.gopts, pruneOpts)
- rtest.OK(t, runCheck(checkOpts, env.gopts, nil))
+ rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
- rtest.OK(t, runRebuildIndex(RebuildIndexOptions{}, env.gopts))
- rtest.OK(t, runRebuildIndex(RebuildIndexOptions{ReadAllPacks: true}, env.gopts))
+ rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{}, env.gopts))
+ rtest.OK(t, runRebuildIndex(context.TODO(), RebuildIndexOptions{ReadAllPacks: true}, env.gopts))
}
func TestHardLink(t *testing.T) {
@@ -1859,7 +1848,7 @@ func TestHardLink(t *testing.T) {
datafile := filepath.Join("testdata", "test.hl.tar.gz")
fd, err := os.Open(datafile)
- if os.IsNotExist(errors.Cause(err)) {
+ if os.IsNotExist(err) {
t.Skipf("unable to find data file %q, skipping", datafile)
return
}
@@ -2202,7 +2191,7 @@ func TestFindListOnce(t *testing.T) {
testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
thirdSnapshot := restic.NewIDSet(testRunList(t, "snapshots", env.gopts)...)
- repo, err := OpenRepository(env.gopts)
+ repo, err := OpenRepository(context.TODO(), env.gopts)
rtest.OK(t, err)
snapshotIDs := restic.NewIDSet()
diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go
index 64f82cf52..f39a08db6 100644
--- a/cmd/restic/lock.go
+++ b/cmd/restic/lock.go
@@ -7,27 +7,31 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
- "github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
)
+type lockContext struct {
+ cancel context.CancelFunc
+ refreshWG sync.WaitGroup
+}
+
var globalLocks struct {
- locks []*restic.Lock
- cancelRefresh chan struct{}
- refreshWG sync.WaitGroup
+ locks map[*restic.Lock]*lockContext
sync.Mutex
sync.Once
}
-func lockRepo(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) {
+func lockRepo(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) {
return lockRepository(ctx, repo, false)
}
-func lockRepoExclusive(ctx context.Context, repo *repository.Repository) (*restic.Lock, error) {
+func lockRepoExclusive(ctx context.Context, repo restic.Repository) (*restic.Lock, context.Context, error) {
return lockRepository(ctx, repo, true)
}
-func lockRepository(ctx context.Context, repo *repository.Repository, exclusive bool) (*restic.Lock, error) {
+// lockRepository wraps the ctx such that it is cancelled when the repository is unlocked
+// cancelling the original context also stops the lock refresh
+func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool) (*restic.Lock, context.Context, error) {
// make sure that a repository is unlocked properly and after cancel() was
// called by the cleanup handler in global.go
globalLocks.Do(func() {
@@ -40,98 +44,157 @@ func lockRepository(ctx context.Context, repo *repository.Repository, exclusive
}
lock, err := lockFn(ctx, repo)
+ if restic.IsInvalidLock(err) {
+ return nil, ctx, errors.Fatalf("%v\n\nthe `unlock --remove-all` command can be used to remove invalid locks. Make sure that no other restic process is accessing the repository when running the command", err)
+ }
if err != nil {
- return nil, errors.WithMessage(err, "unable to create lock in backend")
+ return nil, ctx, errors.Fatalf("unable to create lock in backend: %v", err)
}
debug.Log("create lock %p (exclusive %v)", lock, exclusive)
- globalLocks.Lock()
- if globalLocks.cancelRefresh == nil {
- debug.Log("start goroutine for lock refresh")
- globalLocks.cancelRefresh = make(chan struct{})
- globalLocks.refreshWG = sync.WaitGroup{}
- globalLocks.refreshWG.Add(1)
- go refreshLocks(&globalLocks.refreshWG, globalLocks.cancelRefresh)
+ ctx, cancel := context.WithCancel(ctx)
+ lockInfo := &lockContext{
+ cancel: cancel,
}
+ lockInfo.refreshWG.Add(2)
+ refreshChan := make(chan struct{})
- globalLocks.locks = append(globalLocks.locks, lock)
+ globalLocks.Lock()
+ globalLocks.locks[lock] = lockInfo
+ go refreshLocks(ctx, lock, lockInfo, refreshChan)
+ go monitorLockRefresh(ctx, lock, lockInfo, refreshChan)
globalLocks.Unlock()
- return lock, err
+ return lock, ctx, err
}
var refreshInterval = 5 * time.Minute
-func refreshLocks(wg *sync.WaitGroup, done <-chan struct{}) {
+// consider a lock refresh failed a bit before the lock actually becomes stale
+// the difference allows to compensate for a small time drift between clients.
+var refreshabilityTimeout = restic.StaleLockTimeout - refreshInterval*3/2
+
+func refreshLocks(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed chan<- struct{}) {
debug.Log("start")
+ ticker := time.NewTicker(refreshInterval)
+ lastRefresh := lock.Time
+
defer func() {
- wg.Done()
- globalLocks.Lock()
- globalLocks.cancelRefresh = nil
- globalLocks.Unlock()
- }()
+ ticker.Stop()
+ // ensure that the context was cancelled before removing the lock
+ lockInfo.cancel()
- ticker := time.NewTicker(refreshInterval)
+ // remove the lock from the repo
+ debug.Log("unlocking repository with lock %v", lock)
+ if err := lock.Unlock(); err != nil {
+ debug.Log("error while unlocking: %v", err)
+ Warnf("error while unlocking: %v", err)
+ }
+
+ lockInfo.refreshWG.Done()
+ }()
for {
select {
- case <-done:
+ case <-ctx.Done():
debug.Log("terminate")
return
case <-ticker.C:
+ if time.Since(lastRefresh) > refreshabilityTimeout {
+ // the lock is too old, wait until the expiry monitor cancels the context
+ continue
+ }
+
debug.Log("refreshing locks")
- globalLocks.Lock()
- for _, lock := range globalLocks.locks {
- err := lock.Refresh(context.TODO())
- if err != nil {
- Warnf("unable to refresh lock: %v\n", err)
+ err := lock.Refresh(context.TODO())
+ if err != nil {
+ Warnf("unable to refresh lock: %v\n", err)
+ } else {
+ lastRefresh = lock.Time
+ // inform monitor gorountine about successful refresh
+ select {
+ case <-ctx.Done():
+ case refreshed <- struct{}{}:
}
}
- globalLocks.Unlock()
}
}
}
-func unlockRepo(lock *restic.Lock) {
- if lock == nil {
- return
+func monitorLockRefresh(ctx context.Context, lock *restic.Lock, lockInfo *lockContext, refreshed <-chan struct{}) {
+ // time.Now() might use a monotonic timer which is paused during standby
+ // convert to unix time to ensure we compare real time values
+ lastRefresh := time.Now().UnixNano()
+ pollDuration := 1 * time.Second
+ if refreshInterval < pollDuration {
+ // require for TestLockFailedRefresh
+ pollDuration = refreshInterval / 5
}
+ // timers are paused during standby, which is a problem as the refresh timeout
+ // _must_ expire if the host was too long in standby. Thus fall back to periodic checks
+ // https://github.com/golang/go/issues/35012
+ timer := time.NewTimer(pollDuration)
+ defer func() {
+ timer.Stop()
+ lockInfo.cancel()
+ lockInfo.refreshWG.Done()
+ }()
- globalLocks.Lock()
- defer globalLocks.Unlock()
-
- for i := 0; i < len(globalLocks.locks); i++ {
- if lock == globalLocks.locks[i] {
- // remove the lock from the repo
- debug.Log("unlocking repository with lock %v", lock)
- if err := lock.Unlock(); err != nil {
- debug.Log("error while unlocking: %v", err)
- Warnf("error while unlocking: %v", err)
- return
+ for {
+ select {
+ case <-ctx.Done():
+ debug.Log("terminate expiry monitoring")
+ return
+ case <-refreshed:
+ lastRefresh = time.Now().UnixNano()
+ case <-timer.C:
+ if time.Now().UnixNano()-lastRefresh < refreshabilityTimeout.Nanoseconds() {
+ // restart timer
+ timer.Reset(pollDuration)
+ continue
}
- // remove the lock from the list of locks
- globalLocks.locks = append(globalLocks.locks[:i], globalLocks.locks[i+1:]...)
+ Warnf("Fatal: failed to refresh lock in time\n")
return
}
}
-
- debug.Log("unable to find lock %v in the global list of locks, ignoring", lock)
}
-func unlockAll() error {
+func unlockRepo(lock *restic.Lock) {
+ if lock == nil {
+ return
+ }
+
globalLocks.Lock()
- defer globalLocks.Unlock()
+ lockInfo, exists := globalLocks.locks[lock]
+ delete(globalLocks.locks, lock)
+ globalLocks.Unlock()
+ if !exists {
+ debug.Log("unable to find lock %v in the global list of locks, ignoring", lock)
+ return
+ }
+ lockInfo.cancel()
+ lockInfo.refreshWG.Wait()
+}
+
+func unlockAll(code int) (int, error) {
+ globalLocks.Lock()
+ locks := globalLocks.locks
debug.Log("unlocking %d locks", len(globalLocks.locks))
- for _, lock := range globalLocks.locks {
- if err := lock.Unlock(); err != nil {
- debug.Log("error while unlocking: %v", err)
- return err
- }
- debug.Log("successfully removed lock")
+ for _, lockInfo := range globalLocks.locks {
+ lockInfo.cancel()
}
- globalLocks.locks = globalLocks.locks[:0]
+ globalLocks.locks = make(map[*restic.Lock]*lockContext)
+ globalLocks.Unlock()
+
+ for _, lockInfo := range locks {
+ lockInfo.refreshWG.Wait()
+ }
+
+ return code, nil
+}
- return nil
+func init() {
+ globalLocks.locks = make(map[*restic.Lock]*lockContext)
}
diff --git a/cmd/restic/lock_test.go b/cmd/restic/lock_test.go
new file mode 100644
index 000000000..c074f15a6
--- /dev/null
+++ b/cmd/restic/lock_test.go
@@ -0,0 +1,170 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/restic/restic/internal/repository"
+ "github.com/restic/restic/internal/restic"
+ rtest "github.com/restic/restic/internal/test"
+)
+
+func openTestRepo(t *testing.T, wrapper backendWrapper) (*repository.Repository, func(), *testEnvironment) {
+ env, cleanup := withTestEnvironment(t)
+ if wrapper != nil {
+ env.gopts.backendTestHook = wrapper
+ }
+ testRunInit(t, env.gopts)
+
+ repo, err := OpenRepository(context.TODO(), env.gopts)
+ rtest.OK(t, err)
+ return repo, cleanup, env
+}
+
+func checkedLockRepo(ctx context.Context, t *testing.T, repo restic.Repository) (*restic.Lock, context.Context) {
+ lock, wrappedCtx, err := lockRepo(ctx, repo)
+ rtest.OK(t, err)
+ rtest.OK(t, wrappedCtx.Err())
+ if lock.Stale() {
+ t.Fatal("lock returned stale lock")
+ }
+ return lock, wrappedCtx
+}
+
+func TestLock(t *testing.T) {
+ repo, cleanup, _ := openTestRepo(t, nil)
+ defer cleanup()
+
+ lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
+ unlockRepo(lock)
+ if wrappedCtx.Err() == nil {
+ t.Fatal("unlock did not cancel context")
+ }
+}
+
+func TestLockCancel(t *testing.T) {
+ repo, cleanup, _ := openTestRepo(t, nil)
+ defer cleanup()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ lock, wrappedCtx := checkedLockRepo(ctx, t, repo)
+ cancel()
+ if wrappedCtx.Err() == nil {
+ t.Fatal("canceled parent context did not cancel context")
+ }
+
+ // unlockRepo should not crash
+ unlockRepo(lock)
+}
+
+func TestLockUnlockAll(t *testing.T) {
+ repo, cleanup, _ := openTestRepo(t, nil)
+ defer cleanup()
+
+ lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
+ _, err := unlockAll(0)
+ rtest.OK(t, err)
+ if wrappedCtx.Err() == nil {
+ t.Fatal("canceled parent context did not cancel context")
+ }
+
+ // unlockRepo should not crash
+ unlockRepo(lock)
+}
+
+func TestLockConflict(t *testing.T) {
+ repo, cleanup, env := openTestRepo(t, nil)
+ defer cleanup()
+ repo2, err := OpenRepository(context.TODO(), env.gopts)
+ rtest.OK(t, err)
+
+ lock, _, err := lockRepoExclusive(context.Background(), repo)
+ rtest.OK(t, err)
+ defer unlockRepo(lock)
+ _, _, err = lockRepo(context.Background(), repo2)
+ if err == nil {
+ t.Fatal("second lock should have failed")
+ }
+}
+
+type writeOnceBackend struct {
+ restic.Backend
+ written bool
+}
+
+func (b *writeOnceBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+ if b.written {
+ return fmt.Errorf("fail after first write")
+ }
+ b.written = true
+ return b.Backend.Save(ctx, h, rd)
+}
+
+func TestLockFailedRefresh(t *testing.T) {
+ repo, cleanup, _ := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
+ return &writeOnceBackend{Backend: r}, nil
+ })
+ defer cleanup()
+
+ // reduce locking intervals to be suitable for testing
+ ri, rt := refreshInterval, refreshabilityTimeout
+ refreshInterval = 20 * time.Millisecond
+ refreshabilityTimeout = 100 * time.Millisecond
+ defer func() {
+ refreshInterval, refreshabilityTimeout = ri, rt
+ }()
+
+ lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
+
+ select {
+ case <-wrappedCtx.Done():
+ // expected lock refresh failure
+ case <-time.After(time.Second):
+ t.Fatal("failed lock refresh did not cause context cancellation")
+ }
+ // unlockRepo should not crash
+ unlockRepo(lock)
+}
+
+type loggingBackend struct {
+ restic.Backend
+ t *testing.T
+}
+
+func (b *loggingBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+ b.t.Logf("save %v @ %v", h, time.Now())
+ return b.Backend.Save(ctx, h, rd)
+}
+
+func TestLockSuccessfulRefresh(t *testing.T) {
+ repo, cleanup, _ := openTestRepo(t, func(r restic.Backend) (restic.Backend, error) {
+ return &loggingBackend{
+ Backend: r,
+ t: t,
+ }, nil
+ })
+ defer cleanup()
+
+ t.Logf("test for successful lock refresh %v", time.Now())
+ // reduce locking intervals to be suitable for testing
+ ri, rt := refreshInterval, refreshabilityTimeout
+ refreshInterval = 40 * time.Millisecond
+ refreshabilityTimeout = 200 * time.Millisecond
+ defer func() {
+ refreshInterval, refreshabilityTimeout = ri, rt
+ }()
+
+ lock, wrappedCtx := checkedLockRepo(context.Background(), t, repo)
+
+ select {
+ case <-wrappedCtx.Done():
+ t.Fatal("lock refresh failed")
+ case <-time.After(2 * refreshabilityTimeout):
+ // expected lock refresh to work
+ }
+ // unlockRepo should not crash
+ unlockRepo(lock)
+}
diff --git a/cmd/restic/main.go b/cmd/restic/main.go
index ad3ef89d4..cfef7c885 100644
--- a/cmd/restic/main.go
+++ b/cmd/restic/main.go
@@ -85,17 +85,15 @@ func needsPassword(cmd string) bool {
var logBuffer = bytes.NewBuffer(nil)
-func init() {
+func main() {
// install custom global logger into a buffer, if an error occurs
// we can show the logs
log.SetOutput(logBuffer)
-}
-func main() {
debug.Log("main %#v", os.Args)
debug.Log("restic %s compiled with %v on %v/%v",
version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
- err := cmdRoot.Execute()
+ err := cmdRoot.ExecuteContext(internalGlobalCtx)
switch {
case restic.IsAlreadyLocked(err):
diff --git a/cmd/restic/progress.go b/cmd/restic/progress.go
index 4f33e2072..4b6025a54 100644
--- a/cmd/restic/progress.go
+++ b/cmd/restic/progress.go
@@ -7,6 +7,7 @@ import (
"strings"
"time"
+ "github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/progress"
"github.com/restic/restic/internal/ui/termstatus"
)
@@ -36,13 +37,14 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter
interval := calculateProgressInterval(show, false)
canUpdateStatus := stdoutCanUpdateStatus()
- return progress.New(interval, max, func(v uint64, max uint64, d time.Duration, final bool) {
+ return progress.NewCounter(interval, max, func(v uint64, max uint64, d time.Duration, final bool) {
var status string
if max == 0 {
- status = fmt.Sprintf("[%s] %d %s", formatDuration(d), v, description)
+ status = fmt.Sprintf("[%s] %d %s",
+ ui.FormatDuration(d), v, description)
} else {
status = fmt.Sprintf("[%s] %s %d / %d %s",
- formatDuration(d), formatPercent(v, max), v, max, description)
+ ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description)
}
printProgress(status, canUpdateStatus)
diff --git a/cmd/restic/secondary_repo.go b/cmd/restic/secondary_repo.go
index 7b08004a7..4c46b60df 100644
--- a/cmd/restic/secondary_repo.go
+++ b/cmd/restic/secondary_repo.go
@@ -24,11 +24,11 @@ type secondaryRepoOptions struct {
}
func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repoPrefix string, repoUsage string) {
- f.StringVarP(&opts.LegacyRepo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)")
- f.StringVarP(&opts.LegacyRepositoryFile, "repository-file2", "", os.Getenv("RESTIC_REPOSITORY_FILE2"), "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)")
- f.StringVarP(&opts.LegacyPasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)")
- f.StringVarP(&opts.LegacyKeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)")
- f.StringVarP(&opts.LegacyPasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)")
+ f.StringVarP(&opts.LegacyRepo, "repo2", "", "", repoPrefix+" `repository` "+repoUsage+" (default: $RESTIC_REPOSITORY2)")
+ f.StringVarP(&opts.LegacyRepositoryFile, "repository-file2", "", "", "`file` from which to read the "+repoPrefix+" repository location "+repoUsage+" (default: $RESTIC_REPOSITORY_FILE2)")
+ f.StringVarP(&opts.LegacyPasswordFile, "password-file2", "", "", "`file` to read the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_FILE2)")
+ f.StringVarP(&opts.LegacyKeyHint, "key-hint2", "", "", "key ID of key to try decrypting the "+repoPrefix+" repository first (default: $RESTIC_KEY_HINT2)")
+ f.StringVarP(&opts.LegacyPasswordCommand, "password-command2", "", "", "shell `command` to obtain the "+repoPrefix+" repository password from (default: $RESTIC_PASSWORD_COMMAND2)")
// hide repo2 options
_ = f.MarkDeprecated("repo2", "use --repo or --from-repo instead")
@@ -37,11 +37,23 @@ func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repo
_ = f.MarkHidden("key-hint2")
_ = f.MarkHidden("password-command2")
- f.StringVarP(&opts.Repo, "from-repo", "", os.Getenv("RESTIC_FROM_REPOSITORY"), "source `repository` "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY)")
- f.StringVarP(&opts.RepositoryFile, "from-repository-file", "", os.Getenv("RESTIC_FROM_REPOSITORY_FILE"), "`file` from which to read the source repository location "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY_FILE)")
- f.StringVarP(&opts.PasswordFile, "from-password-file", "", os.Getenv("RESTIC_FROM_PASSWORD_FILE"), "`file` to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)")
- f.StringVarP(&opts.KeyHint, "from-key-hint", "", os.Getenv("RESTIC_FROM_KEY_HINT"), "key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)")
- f.StringVarP(&opts.PasswordCommand, "from-password-command", "", os.Getenv("RESTIC_FROM_PASSWORD_COMMAND"), "shell `command` to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)")
+ opts.LegacyRepo = os.Getenv("RESTIC_REPOSITORY2")
+ opts.LegacyRepositoryFile = os.Getenv("RESTIC_REPOSITORY_FILE2")
+ opts.LegacyPasswordFile = os.Getenv("RESTIC_PASSWORD_FILE2")
+ opts.LegacyKeyHint = os.Getenv("RESTIC_KEY_HINT2")
+ opts.LegacyPasswordCommand = os.Getenv("RESTIC_PASSWORD_COMMAND2")
+
+ f.StringVarP(&opts.Repo, "from-repo", "", "", "source `repository` "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY)")
+ f.StringVarP(&opts.RepositoryFile, "from-repository-file", "", "", "`file` from which to read the source repository location "+repoUsage+" (default: $RESTIC_FROM_REPOSITORY_FILE)")
+ f.StringVarP(&opts.PasswordFile, "from-password-file", "", "", "`file` to read the source repository password from (default: $RESTIC_FROM_PASSWORD_FILE)")
+ f.StringVarP(&opts.KeyHint, "from-key-hint", "", "", "key ID of key to try decrypting the source repository first (default: $RESTIC_FROM_KEY_HINT)")
+ f.StringVarP(&opts.PasswordCommand, "from-password-command", "", "", "shell `command` to obtain the source repository password from (default: $RESTIC_FROM_PASSWORD_COMMAND)")
+
+ opts.Repo = os.Getenv("RESTIC_FROM_REPOSITORY")
+ opts.RepositoryFile = os.Getenv("RESTIC_FROM_REPOSITORY_FILE")
+ opts.PasswordFile = os.Getenv("RESTIC_FROM_PASSWORD_FILE")
+ opts.KeyHint = os.Getenv("RESTIC_FROM_KEY_HINT")
+ opts.PasswordCommand = os.Getenv("RESTIC_FROM_PASSWORD_COMMAND")
}
func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, bool, error) {
diff --git a/cmd/restic/secondary_repo_test.go b/cmd/restic/secondary_repo_test.go
index cb410f1b9..ff1a10b03 100644
--- a/cmd/restic/secondary_repo_test.go
+++ b/cmd/restic/secondary_repo_test.go
@@ -1,7 +1,7 @@
package main
import (
- "io/ioutil"
+ "os"
"path/filepath"
"testing"
@@ -160,14 +160,12 @@ func TestFillSecondaryGlobalOpts(t *testing.T) {
}
//Create temp dir to create password file.
- dir, cleanup := rtest.TempDir(t)
- defer cleanup()
-
- cleanup = rtest.Chdir(t, dir)
+ dir := rtest.TempDir(t)
+ cleanup := rtest.Chdir(t, dir)
defer cleanup()
//Create temporary password file
- err := ioutil.WriteFile(filepath.Join(dir, "passwordFileDst"), []byte("secretDst"), 0666)
+ err := os.WriteFile(filepath.Join(dir, "passwordFileDst"), []byte("secretDst"), 0666)
rtest.OK(t, err)
// Test all valid cases
diff --git a/debian/changelog b/debian/changelog
index 74c6957b0..c9420bd2f 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+restic (0.15.1+ds-1) UNRELEASED; urgency=low
+
+ * New upstream release.
+
+ -- Debian Janitor <janitor@jelmer.uk> Wed, 08 Feb 2023 00:08:46 -0000
+
restic (0.14.0-1) unstable; urgency=medium
* New upstream version 0.14.0 (Closes: #1018154)
diff --git a/debian/patches/0001-privacy-breach.patch b/debian/patches/0001-privacy-breach.patch
index 975f3c1a2..82b184490 100644
--- a/debian/patches/0001-privacy-breach.patch
+++ b/debian/patches/0001-privacy-breach.patch
@@ -6,10 +6,10 @@ Subject: privacy breach
README.md | 6 ------
1 file changed, 6 deletions(-)
-diff --git a/README.md b/README.md
-index 8f72a02..dfbd58e 100644
---- a/README.md
-+++ b/README.md
+Index: restic.git/README.md
+===================================================================
+--- restic.git.orig/README.md
++++ restic.git/README.md
@@ -1,7 +1,3 @@
-[](https://restic.readthedocs.io/en/latest/?badge=latest)
-[](https://github.com/restic/restic/actions?query=workflow%3Atest)
diff --git a/doc/020_installation.rst b/doc/020_installation.rst
index 9f6ffa141..5ae93c94d 100644
--- a/doc/020_installation.rst
+++ b/doc/020_installation.rst
@@ -265,16 +265,11 @@ binary, you can get it with `docker pull` like this:
$ docker pull restic/restic
-.. note::
- | Another docker container which offers more configuration options is
- | available as a contribution (Thank you!). You can find it at
- | https://github.com/Lobaro/restic-backup-docker
-
From Source
***********
restic is written in the Go programming language and you need at least
-Go version 1.15. Building restic may also work with older versions of Go,
+Go version 1.18. Building restic may also work with older versions of Go,
but that's not supported. See the `Getting
started <https://golang.org/doc/install>`__ guide of the Go project for
instructions how to install Go.
@@ -313,14 +308,14 @@ compiler. Building restic with gccgo may work, but is not supported.
Autocompletion
**************
-Restic can write out man pages and bash/fish/zsh compatible autocompletion scripts:
+Restic can write out man pages and bash/fish/zsh/powershell compatible autocompletion scripts:
.. code-block:: console
$ ./restic generate --help
The "generate" command writes automatically generated files (like the man pages
- and the auto-completion files for bash, fish and zsh).
+ and the auto-completion files for bash, fish, zsh and powershell).
Usage:
restic generate [flags] [command]
@@ -330,6 +325,7 @@ Restic can write out man pages and bash/fish/zsh compatible autocompletion scrip
--fish-completion file write fish completion file
-h, --help help for generate
--man directory write man pages to directory
+ --powershell-completion write powershell completion file
--zsh-completion file write zsh completion file
Example for using sudo to write a bash completion script directly to the system-wide location:
diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst
index dbb641746..39a3a0744 100644
--- a/doc/030_preparing_a_new_repo.rst
+++ b/doc/030_preparing_a_new_repo.rst
@@ -58,9 +58,9 @@ versions.
+--------------------+-------------------------+---------------------+------------------+
| Repository version | Required restic version | Major new features | Comment |
+====================+=========================+=====================+==================+
-| ``1`` | Any | | Current default |
+| ``1`` | Any | | |
+--------------------+-------------------------+---------------------+------------------+
-| ``2`` | 0.14.0 or newer | Compression support | |
+| ``2`` | 0.14.0 or newer | Compression support | Current default |
+--------------------+-------------------------+---------------------+------------------+
@@ -86,10 +86,11 @@ command and enter the same password twice:
.. warning::
- On Linux, storing the backup repository on a CIFS (SMB) share is not
- recommended due to compatibility issues. Either use another backend
- or set the environment variable `GODEBUG` to `asyncpreemptoff=1`.
- Refer to GitHub issue `#2659 <https://github.com/restic/restic/issues/2659>`_ for further explanations.
+ On Linux, storing the backup repository on a CIFS (SMB) share or backing up
+ data from a CIFS share is not recommended due to compatibility issues in
+ older Linux kernels. Either use another backend or set the environment
+ variable `GODEBUG` to `asyncpreemptoff=1`. Refer to GitHub issue
+ `#2659 <https://github.com/restic/restic/issues/2659>`_ for further explanations.
SFTP
****
@@ -221,6 +222,8 @@ REST server uses exactly the same directory structure as local backend,
so you should be able to access it both locally and via HTTP, even
simultaneously.
+.. _Amazon S3:
+
Amazon S3
*********
@@ -301,7 +304,7 @@ credentials of your Minio Server.
.. code-block:: console
$ export AWS_ACCESS_KEY_ID=<YOUR-MINIO-ACCESS-KEY-ID>
- $ export AWS_SECRET_ACCESS_KEY= <YOUR-MINIO-SECRET-ACCESS-KEY>
+ $ export AWS_SECRET_ACCESS_KEY=<YOUR-MINIO-SECRET-ACCESS-KEY>
Now you can easily initialize restic to use Minio server as a backend with
this command.
@@ -464,6 +467,19 @@ The policy of the new container created by restic can be changed using environme
Backblaze B2
************
+.. warning::
+
+ Due to issues with error handling in the current B2 library that restic uses,
+ the recommended way to utilize Backblaze B2 is by using its S3-compatible API.
+
+ Follow the documentation to `generate S3-compatible access keys`_ and then
+ setup restic as described at :ref:`Amazon S3`. This is expected to work better
+ than using the Backblaze B2 backend directly.
+
+ Different from the B2 backend, restic's S3 backend will only hide no longer
+ necessary files. Thus, make sure to setup lifecycle rules to eventually
+ delete hidden files.
+
Restic can backup data to any Backblaze B2 bucket. You need to first setup the
following environment variables with the credentials you can find in the
dashboard on the "Buckets" page when signed into your B2 account:
@@ -502,11 +518,13 @@ The number of concurrent connections to the B2 service can be set with the ``-o
b2.connections=10`` switch. By default, at most five parallel connections are
established.
+.. _generate S3-compatible access keys: https://help.backblaze.com/hc/en-us/articles/360047425453-Getting-Started-with-the-S3-Compatible-API
+
Microsoft Azure Blob Storage
****************************
You can also store backups on Microsoft Azure Blob Storage. Export the Azure
-account name and key as follows:
+Blob Storage account name and key as follows:
.. code-block:: console
@@ -618,6 +636,13 @@ initiate a new repository in the path ``bar`` in the remote ``foo``:
Restic takes care of starting and stopping rclone.
+.. note:: If you get an error message saying "cannot implicitly run relative
+ executable rclone found in current directory", this means that an
+ rclone executable was found in the current directory. For security
+ reasons restic will not run this implicitly, instead you have to
+ use the ``-o rclone.program=./rclone`` extended option to override
+ this security check and explicitly tell restic to use the executable.
+
As a more concrete example, suppose you have configured a remote named
``b2prod`` for Backblaze B2 with rclone, with a bucket called ``yggdrasil``.
You can then use rclone to list files in the bucket like this:
diff --git a/doc/040_backup.rst b/doc/040_backup.rst
index 891c6820d..b9996311d 100644
--- a/doc/040_backup.rst
+++ b/doc/040_backup.rst
@@ -204,6 +204,7 @@ Combined with ``--verbose``, you can see a list of changes:
modified /archive.tar.gz, saved in 0.140s (25.542 MiB added)
Would be added to the repository: 25.551 MiB
+.. _backup-excluding-files:
Excluding Files
***************
@@ -299,7 +300,7 @@ directory, then selectively add back some of them.
::
- $HOME/**/*
+ $HOME/*
!$HOME/Documents
!$HOME/code
!$HOME/.emacs.d
@@ -555,6 +556,7 @@ environment variables. The following lists these environment variables:
RESTIC_COMPRESSION Compression mode (only available for repository format version 2)
RESTIC_PROGRESS_FPS Frames per second by which the progress bar is updated
RESTIC_PACK_SIZE Target size for pack files
+ RESTIC_READ_CONCURRENCY Concurrency for file reads
TMPDIR Location for temporary files
diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst
index 8f702bc6c..00d87a450 100644
--- a/doc/045_working_with_repos.rst
+++ b/doc/045_working_with_repos.rst
@@ -136,22 +136,24 @@ or the environment variable ``$RESTIC_FROM_KEY_HINT``.
repository. You can avoid this limitation by using the rclone backend
along with remotes which are configured in rclone.
+.. _copy-filtering-snapshots:
+
Filtering snapshots to copy
---------------------------
The list of snapshots to copy can be filtered by host, path in the backup
-and / or a comma-separated tag list:
+and/or a comma-separated tag list:
.. code-block:: console
- $ restic -r /srv/restic-repo copy --repo2 /srv/restic-repo-copy --host luigi --path /srv --tag foo,bar
+ $ restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo --host luigi --path /srv --tag foo,bar
It is also possible to explicitly specify the list of snapshots to copy, in
which case only these instead of all snapshots will be copied:
.. code-block:: console
- $ restic -r /srv/restic-repo copy --repo2 /srv/restic-repo-copy 410b18a2 4e5d5487 latest
+ $ restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo 410b18a2 4e5d5487 latest
Ensuring deduplication for copied snapshots
-------------------------------------------
@@ -170,11 +172,66 @@ using the same chunker parameters as the source repository:
.. code-block:: console
- $ restic -r /srv/restic-repo-copy init --repo2 /srv/restic-repo --copy-chunker-params
+ $ restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo --copy-chunker-params
Note that it is not possible to change the chunker parameters of an existing repository.
+Removing files from snapshots
+=============================
+
+Snapshots sometimes turn out to include more files that intended. Instead of
+removing the snapshots entirely and running the corresponding backup commands
+again (which is not always practical after the fact) it is possible to remove
+the unwanted files from affected snapshots by rewriting them using the
+``rewrite`` command:
+
+.. code-block:: console
+
+ $ restic -r /srv/restic-repo rewrite --exclude secret-file
+ repository c881945a opened (repository version 2) successfully, password is correct
+
+ snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST)
+ excluding /home/user/work/secret-file
+ saved new snapshot b6aee1ff
+
+ snapshot 4fbaf325 of [/home/user/work] at 2022-05-01 11:22:26.500093107 +0200 CEST)
+
+ modified 1 snapshots
+
+ $ restic -r /srv/restic-repo rewrite --exclude secret-file 6160ddb2
+ repository c881945a opened (repository version 2) successfully, password is correct
+
+ snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST)
+ excluding /home/user/work/secret-file
+ new snapshot saved as b6aee1ff
+
+ modified 1 snapshots
+
+The options ``--exclude``, ``--exclude-file``, ``--iexclude`` and
+``--iexclude-file`` are supported. They behave the same way as for the backup
+command, see :ref:`backup-excluding-files` for details.
+
+It is possible to rewrite only a subset of snapshots by filtering them the same
+way as for the ``copy`` command, see :ref:`copy-filtering-snapshots`.
+
+By default, the ``rewrite`` command will keep the original snapshots and create
+new ones for every snapshot which was modified during rewriting. The new
+snapshots are marked with the tag ``rewrite`` to differentiate them from the
+original, rewritten snapshots.
+
+Alternatively, you can use the ``--forget`` option to immediately remove the
+original snapshots. In this case, no tag is added to the new snapshots. Please
+note that this only removes the snapshots and not the actual data stored in the
+repository. Run the ``prune`` command afterwards to remove the now unreferenced
+data (just like when having used the ``forget`` command).
+
+In order to preview the changes which ``rewrite`` would make, you can use the
+``--dry-run`` option. This will simulate the rewriting process without actually
+modifying the repository. Instead restic will only print the actions it would
+perform.
+
+
Checking integrity and consistency
==================================
diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst
index ecf2bebbb..6ea39dc75 100644
--- a/doc/047_tuning_backup_parameters.rst
+++ b/doc/047_tuning_backup_parameters.rst
@@ -19,6 +19,15 @@ values. As the restic commands evolve over time, the optimal value for each para
can also change across restic versions.
+Disabling Backup Progress Estimation
+====================================
+
+When you start a backup, restic will concurrently count the number of files and
+their total size, which is used to estimate how long it will take. This will
+cause some extra I/O, which can slow down backups of network file systems or
+FUSE mounts. To avoid this overhead at the cost of not seeing a progress
+estimate, use the ``--no-scan`` option which disables this file scanning.
+
Backend Connections
===================
@@ -51,6 +60,16 @@ only applied for the single run of restic. The option can also be set via the en
variable ``RESTIC_COMPRESSION``.
+File Read Concurrency
+=====================
+
+When backing up files from fast storage like NVMe disks, it can be beneficial to increase
+the read concurrency. This can increase the overall performance of the backup operation
+by reading more files in parallel. You can specify the concurrency of file reads with the
+``RESTIC_READ_CONCURRENCY`` environment variable or the ``--read-concurrency`` option of
+the ``backup`` command.
+
+
Pack Size
=========
diff --git a/doc/050_restore.rst b/doc/050_restore.rst
index c7f6c0f28..b0ea021cc 100644
--- a/doc/050_restore.rst
+++ b/doc/050_restore.rst
@@ -56,6 +56,18 @@ There are case insensitive variants of ``--exclude`` and ``--include`` called
``--iexclude`` and ``--iinclude``. These options will behave the same way but
ignore the casing of paths.
+Restoring symbolic links on windows is only possible when the user has
+``SeCreateSymbolicLinkPrivilege`` privilege or is running as admin. This is a
+restriction of windows not restic.
+
+By default, restic does not restore files as sparse. Use ``restore --sparse`` to
+enable the creation of sparse files if supported by the filesystem. Then restic
+will restore long runs of zero bytes as holes in the corresponding files.
+Reading from a hole returns the original zero bytes, but it does not consume
+disk space. Note that the exact location of the holes can differ from those in
+the original file, as their location is determined while restoring and is not
+stored explicitly.
+
Restore using mount
===================
diff --git a/doc/bash-completion.sh b/doc/bash-completion.sh
index f76ae5a7c..42f459f65 100644
--- a/doc/bash-completion.sh
+++ b/doc/bash-completion.sh
@@ -436,6 +436,8 @@ _restic_backup()
local_nonpersistent_flags+=("--ignore-ctime")
flags+=("--ignore-inode")
local_nonpersistent_flags+=("--ignore-inode")
+ flags+=("--no-scan")
+ local_nonpersistent_flags+=("--no-scan")
flags+=("--one-file-system")
flags+=("-x")
local_nonpersistent_flags+=("--one-file-system")
@@ -444,6 +446,10 @@ _restic_backup()
two_word_flags+=("--parent")
local_nonpersistent_flags+=("--parent")
local_nonpersistent_flags+=("--parent=")
+ flags+=("--read-concurrency=")
+ two_word_flags+=("--read-concurrency")
+ local_nonpersistent_flags+=("--read-concurrency")
+ local_nonpersistent_flags+=("--read-concurrency=")
flags+=("--stdin")
local_nonpersistent_flags+=("--stdin")
flags+=("--stdin-filename=")
@@ -1256,6 +1262,10 @@ _restic_generate()
two_word_flags+=("--man")
local_nonpersistent_flags+=("--man")
local_nonpersistent_flags+=("--man=")
+ flags+=("--powershell-completion=")
+ two_word_flags+=("--powershell-completion")
+ local_nonpersistent_flags+=("--powershell-completion")
+ local_nonpersistent_flags+=("--powershell-completion=")
flags+=("--zsh-completion=")
two_word_flags+=("--zsh-completion")
local_nonpersistent_flags+=("--zsh-completion")
@@ -2083,6 +2093,8 @@ _restic_restore()
two_word_flags+=("--path")
local_nonpersistent_flags+=("--path")
local_nonpersistent_flags+=("--path=")
+ flags+=("--sparse")
+ local_nonpersistent_flags+=("--sparse")
flags+=("--tag=")
two_word_flags+=("--tag")
local_nonpersistent_flags+=("--tag")
@@ -2139,6 +2151,106 @@ _restic_restore()
noun_aliases=()
}
+_restic_rewrite()
+{
+ last_command="restic_rewrite"
+
+ command_aliases=()
+
+ commands=()
+
+ flags=()
+ two_word_flags=()
+ local_nonpersistent_flags=()
+ flags_with_completion=()
+ flags_completion=()
+
+ flags+=("--dry-run")
+ flags+=("-n")
+ local_nonpersistent_flags+=("--dry-run")
+ local_nonpersistent_flags+=("-n")
+ flags+=("--exclude=")
+ two_word_flags+=("--exclude")
+ two_word_flags+=("-e")
+ local_nonpersistent_flags+=("--exclude")
+ local_nonpersistent_flags+=("--exclude=")
+ local_nonpersistent_flags+=("-e")
+ flags+=("--exclude-file=")
+ two_word_flags+=("--exclude-file")
+ local_nonpersistent_flags+=("--exclude-file")
+ local_nonpersistent_flags+=("--exclude-file=")
+ flags+=("--forget")
+ local_nonpersistent_flags+=("--forget")
+ flags+=("--help")
+ flags+=("-h")
+ local_nonpersistent_flags+=("--help")
+ local_nonpersistent_flags+=("-h")
+ flags+=("--host=")
+ two_word_flags+=("--host")
+ two_word_flags+=("-H")
+ local_nonpersistent_flags+=("--host")
+ local_nonpersistent_flags+=("--host=")
+ local_nonpersistent_flags+=("-H")
+ flags+=("--iexclude=")
+ two_word_flags+=("--iexclude")
+ local_nonpersistent_flags+=("--iexclude")
+ local_nonpersistent_flags+=("--iexclude=")
+ flags+=("--iexclude-file=")
+ two_word_flags+=("--iexclude-file")
+ local_nonpersistent_flags+=("--iexclude-file")
+ local_nonpersistent_flags+=("--iexclude-file=")
+ flags+=("--path=")
+ two_word_flags+=("--path")
+ local_nonpersistent_flags+=("--path")
+ local_nonpersistent_flags+=("--path=")
+ flags+=("--tag=")
+ two_word_flags+=("--tag")
+ local_nonpersistent_flags+=("--tag")
+ local_nonpersistent_flags+=("--tag=")
+ flags+=("--cacert=")
+ two_word_flags+=("--cacert")
+ flags+=("--cache-dir=")
+ two_word_flags+=("--cache-dir")
+ flags+=("--cleanup-cache")
+ flags+=("--compression=")
+ two_word_flags+=("--compression")
+ flags+=("--insecure-tls")
+ flags+=("--json")
+ flags+=("--key-hint=")
+ two_word_flags+=("--key-hint")
+ flags+=("--limit-download=")
+ two_word_flags+=("--limit-download")
+ flags+=("--limit-upload=")
+ two_word_flags+=("--limit-upload")
+ flags+=("--no-cache")
+ flags+=("--no-lock")
+ flags+=("--option=")
+ two_word_flags+=("--option")
+ two_word_flags+=("-o")
+ flags+=("--pack-size=")
+ two_word_flags+=("--pack-size")
+ flags+=("--password-command=")
+ two_word_flags+=("--password-command")
+ flags+=("--password-file=")
+ two_word_flags+=("--password-file")
+ two_word_flags+=("-p")
+ flags+=("--quiet")
+ flags+=("-q")
+ flags+=("--repo=")
+ two_word_flags+=("--repo")
+ two_word_flags+=("-r")
+ flags+=("--repository-file=")
+ two_word_flags+=("--repository-file")
+ flags+=("--tls-client-cert=")
+ two_word_flags+=("--tls-client-cert")
+ flags+=("--verbose")
+ flags+=("-v")
+
+ must_have_one_flag=()
+ must_have_one_noun=()
+ noun_aliases=()
+}
+
_restic_self-update()
{
last_command="restic_self-update"
@@ -2617,6 +2729,7 @@ _restic_root_command()
commands+=("rebuild-index")
commands+=("recover")
commands+=("restore")
+ commands+=("rewrite")
commands+=("self-update")
commands+=("snapshots")
commands+=("stats")
diff --git a/doc/design.rst b/doc/design.rst
index 17ab4c1b5..3e25a0852 100644
--- a/doc/design.rst
+++ b/doc/design.rst
@@ -257,7 +257,7 @@ be downloaded and used to reconstruct the index. The file encoding is
described in the "Unpacked Data Format" section. The plaintext consists
of a JSON document like the following:
-.. code:: json
+.. code:: javascript
{
"supersedes": [
diff --git a/doc/developer_information.rst b/doc/developer_information.rst
index f2d410577..c05edc9d2 100644
--- a/doc/developer_information.rst
+++ b/doc/developer_information.rst
@@ -23,17 +23,17 @@ timestamp and filename of the binary contained in it. In order to reproduce the
exact same ZIP file every time, we update the timestamp of the file ``VERSION``
in the source code archive and set the timezone to Europe/Berlin.
-In the following example, we'll use the file ``restic-0.12.1.tar.gz`` and Go
-1.16.6 to reproduce the released binaries.
+In the following example, we'll use the file ``restic-0.14.0.tar.gz`` and Go
+1.19 to reproduce the released binaries.
1. Determine the Go compiler version used to build the released binaries, then download and extract the Go compiler into ``/usr/local/go``:
.. code::
$ restic version
- restic 0.12.1 compiled with go1.16.6 on linux/amd64
+ restic 0.14.0 compiled with go1.19 on linux/amd64
$ cd /usr/local
- $ curl -L https://dl.google.com/go/go1.16.6.linux-amd64.tar.gz | tar xz
+ $ curl -L https://dl.google.com/go/go1.19.linux-amd64.tar.gz | tar xz
2. Extract the restic source code into ``/restic``
@@ -41,7 +41,7 @@ In the following example, we'll use the file ``restic-0.12.1.tar.gz`` and Go
$ mkdir /restic
$ cd /restic
- $ TZ=Europe/Berlin curl -L https://github.com/restic/restic/releases/download/v0.12.1/restic-0.12.1.tar.gz | tar xz --strip-components=1
+ $ TZ=Europe/Berlin curl -L https://github.com/restic/restic/releases/download/v0.14.0/restic-0.14.0.tar.gz | tar xz --strip-components=1
3. Build the binaries for Windows and Linux:
@@ -50,14 +50,14 @@ In the following example, we'll use the file ``restic-0.12.1.tar.gz`` and Go
$ export PATH=/usr/local/go/bin:$PATH
$ export GOPATH=/home/build/go
$ go version
- go version go1.16.6 linux/amd64
+ go version go1.19 linux/amd64
$ GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s -w" -tags selfupdate -o restic_linux_amd64 ./cmd/restic
$ bzip2 restic_linux_amd64
- $ GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s -w" -tags selfupdate -o restic_0.12.1_windows_amd64.exe ./cmd/restic
- $ touch --reference VERSION restic_0.12.1_windows_amd64.exe
- $ TZ=Europe/Berlin zip -q -X restic_0.12.1_windows_amd64.zip restic_0.12.1_windows_amd64.exe
+ $ GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-s -w" -tags selfupdate -o restic_0.14.0_windows_amd64.exe ./cmd/restic
+ $ touch --reference VERSION restic_0.14.0_windows_amd64.exe
+ $ TZ=Europe/Berlin zip -q -X restic_0.14.0_windows_amd64.zip restic_0.14.0_windows_amd64.exe
Building the Official Binaries
******************************
@@ -85,7 +85,7 @@ The following steps are necessary to build the binaries:
.. code::
- tar xvzf restic-0.12.1.tar.gz
+ tar xvzf restic-0.14.0.tar.gz
3. Create a directory to place the resulting binaries in:
@@ -98,20 +98,20 @@ The following steps are necessary to build the binaries:
.. code::
docker run --rm \
- --volume "$PWD/restic-0.12.1:/restic" \
+ --volume "$PWD/restic-0.14.0:/restic" \
--volume "$PWD/output:/output" \
restic/builder \
- go run helpers/build-release-binaries/main.go --version 0.12.1
+ go run helpers/build-release-binaries/main.go --version 0.14.0
4. If anything goes wrong, you can enable debug output like this:
.. code::
docker run --rm \
- --volume "$PWD/restic-0.12.1:/restic" \
+ --volume "$PWD/restic-0.14.0:/restic" \
--volume "$PWD/output:/output" \
restic/builder \
- go run helpers/build-release-binaries/main.go --version 0.12.1 --verbose
+ go run helpers/build-release-binaries/main.go --version 0.14.0 --verbose
Prepare a New Release
*********************
@@ -124,6 +124,6 @@ required argument is the new version number (in `Semantic Versioning
.. code::
- go run helpers/prepare-release/main.go 0.12.1
+ go run helpers/prepare-release/main.go 0.14.0
Checks can be skipped on demand via flags, please see ``--help`` for details.
diff --git a/doc/faq.rst b/doc/faq.rst
index 6292f2de8..8e56b5d9e 100644
--- a/doc/faq.rst
+++ b/doc/faq.rst
@@ -179,7 +179,7 @@ with an error similar to the following:
::
- $ restic init -r sftp:user@nas:/volume1/restic-repo init
+ $ restic -r sftp:user@nas:/volume1/restic-repo init
create backend at sftp:user@nas:/volume1/restic-repo/ failed:
mkdirAll(/volume1/restic-repo/index): unable to create directories: [...]
@@ -199,7 +199,7 @@ The following may work:
::
- $ restic init -r sftp:user@nas:/restic-repo init
+ $ restic -r sftp:user@nas:/restic-repo init
Why does restic perform so poorly on Windows?
---------------------------------------------
diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1
index 8a7bfc1ce..2598678d0 100644
--- a/doc/man/restic-backup.1
+++ b/doc/man/restic-backup.1
@@ -89,6 +89,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
\fB--ignore-inode\fP[=false]
ignore inode number changes when checking for modified files
+.PP
+\fB--no-scan\fP[=false]
+ do not run scanner to estimate size of backup
+
.PP
\fB-x\fP, \fB--one-file-system\fP[=false]
exclude other file systems, don't cross filesystem boundaries and subvolumes
@@ -97,6 +101,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
\fB--parent\fP=""
use this parent \fB\fCsnapshot\fR (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)
+.PP
+\fB--read-concurrency\fP=0
+ read \fB\fCn\fR files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)
+
.PP
\fB--stdin\fP[=false]
read backup from stdin
@@ -149,11 +157,11 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -169,7 +177,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -197,7 +205,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-cache.1 b/doc/man/restic-cache.1
index bedf699a8..302bcb01e 100644
--- a/doc/man/restic-cache.1
+++ b/doc/man/restic-cache.1
@@ -70,11 +70,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -90,7 +90,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -118,7 +118,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-cat.1 b/doc/man/restic-cat.1
index 4fcc373c3..1fb7dd45f 100644
--- a/doc/man/restic-cat.1
+++ b/doc/man/restic-cat.1
@@ -58,11 +58,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -78,7 +78,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -106,7 +106,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-check.1 b/doc/man/restic-check.1
index fd617b524..e08b83d46 100644
--- a/doc/man/restic-check.1
+++ b/doc/man/restic-check.1
@@ -75,11 +75,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -95,7 +95,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -123,7 +123,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-copy.1 b/doc/man/restic-copy.1
index d87f05d6e..07dcfe957 100644
--- a/doc/man/restic-copy.1
+++ b/doc/man/restic-copy.1
@@ -57,15 +57,15 @@ new destination repository using the "init" command.
.PP
\fB-H\fP, \fB--host\fP=[]
- only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given (can be specified multiple times)
+ only consider snapshots for this \fB\fChost\fR (can be specified multiple times)
.PP
\fB--path\fP=[]
- only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot ID is given
+ only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times)
.PP
\fB--tag\fP=[]
- only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot ID is given
+ only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times)
.SH OPTIONS INHERITED FROM PARENT COMMANDS
@@ -99,11 +99,11 @@ new destination repository using the "init" command.
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -119,7 +119,7 @@ new destination repository using the "init" command.
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -147,7 +147,7 @@ new destination repository using the "init" command.
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-diff.1 b/doc/man/restic-diff.1
index bb1f8af97..f0707a257 100644
--- a/doc/man/restic-diff.1
+++ b/doc/man/restic-diff.1
@@ -78,11 +78,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -98,7 +98,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -126,7 +126,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-dump.1 b/doc/man/restic-dump.1
index f0475cbe3..f9a2368bc 100644
--- a/doc/man/restic-dump.1
+++ b/doc/man/restic-dump.1
@@ -39,15 +39,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-H\fP, \fB--host\fP=[]
- only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)
+ only consider snapshots for this \fB\fChost\fR, when snapshot ID "latest" is given (can be specified multiple times)
.PP
\fB--path\fP=[]
- only consider snapshots which include this (absolute) \fB\fCpath\fR for snapshot ID "latest"
+ only consider snapshots including this (absolute) \fB\fCpath\fR, when snapshot ID "latest" is given (can be specified multiple times)
.PP
\fB--tag\fP=[]
- only consider snapshots which include this \fB\fCtaglist\fR for snapshot ID "latest"
+ only consider snapshots including \fB\fCtag[,tag,...]\fR, when snapshot ID "latest" is given (can be specified multiple times)
.SH OPTIONS INHERITED FROM PARENT COMMANDS
@@ -81,11 +81,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -101,7 +101,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -129,7 +129,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-find.1 b/doc/man/restic-find.1
index 51185f53b..4f5bdd4e3 100644
--- a/doc/man/restic-find.1
+++ b/doc/man/restic-find.1
@@ -29,7 +29,7 @@ It can also be used to search for restic blobs or trees for troubleshooting.
.PP
\fB-H\fP, \fB--host\fP=[]
- only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given (can be specified multiple times)
+ only consider snapshots for this \fB\fChost\fR (can be specified multiple times)
.PP
\fB-i\fP, \fB--ignore-case\fP[=false]
@@ -53,7 +53,7 @@ It can also be used to search for restic blobs or trees for troubleshooting.
.PP
\fB--path\fP=[]
- only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot-ID is given
+ only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times)
.PP
\fB--show-pack-id\fP[=false]
@@ -65,7 +65,7 @@ It can also be used to search for restic blobs or trees for troubleshooting.
.PP
\fB--tag\fP=[]
- only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot-ID is given
+ only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times)
.PP
\fB--tree\fP[=false]
@@ -103,11 +103,11 @@ It can also be used to search for restic blobs or trees for troubleshooting.
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -123,7 +123,7 @@ It can also be used to search for restic blobs or trees for troubleshooting.
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -151,7 +151,7 @@ It can also be used to search for restic blobs or trees for troubleshooting.
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH EXAMPLE
diff --git a/doc/man/restic-forget.1 b/doc/man/restic-forget.1
index 0be653216..f46d05736 100644
--- a/doc/man/restic-forget.1
+++ b/doc/man/restic-forget.1
@@ -87,15 +87,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--host\fP=[]
- only consider snapshots with the given \fB\fChost\fR (can be specified multiple times)
+ only consider snapshots for this \fB\fChost\fR (can be specified multiple times)
.PP
\fB--tag\fP=[]
- only consider snapshots which include this \fB\fCtaglist\fR in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times)
+ only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times)
.PP
\fB--path\fP=[]
- only consider snapshots which include this (absolute) \fB\fCpath\fR (can be specified multiple times)
+ only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times)
.PP
\fB-c\fP, \fB--compact\fP[=false]
@@ -169,11 +169,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -189,7 +189,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -217,7 +217,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-generate.1 b/doc/man/restic-generate.1
index 6bda99f7c..e3733ce60 100644
--- a/doc/man/restic-generate.1
+++ b/doc/man/restic-generate.1
@@ -3,7 +3,7 @@
.SH NAME
.PP
-restic-generate - Generate manual pages and auto-completion files (bash, fish, zsh)
+restic-generate - Generate manual pages and auto-completion files (bash, fish, zsh, powershell)
.SH SYNOPSIS
@@ -39,6 +39,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
\fB--man\fP=""
write man pages to \fB\fCdirectory\fR
+.PP
+\fB--powershell-completion\fP=""
+ write powershell completion \fB\fCfile\fR
+
.PP
\fB--zsh-completion\fP=""
write zsh completion \fB\fCfile\fR
@@ -75,11 +79,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -95,7 +99,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -123,7 +127,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-init.1 b/doc/man/restic-init.1
index 87ba79a36..80edf5362 100644
--- a/doc/man/restic-init.1
+++ b/doc/man/restic-init.1
@@ -86,11 +86,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -106,7 +106,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -134,7 +134,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-key.1 b/doc/man/restic-key.1
index 4afa3d116..ff6ab4fd0 100644
--- a/doc/man/restic-key.1
+++ b/doc/man/restic-key.1
@@ -70,11 +70,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -90,7 +90,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -118,7 +118,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-list.1 b/doc/man/restic-list.1
index c87f1d0a6..e2f878c76 100644
--- a/doc/man/restic-list.1
+++ b/doc/man/restic-list.1
@@ -58,11 +58,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -78,7 +78,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -106,7 +106,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-ls.1 b/doc/man/restic-ls.1
index 9d77ee62d..afd72ff71 100644
--- a/doc/man/restic-ls.1
+++ b/doc/man/restic-ls.1
@@ -51,7 +51,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--path\fP=[]
- only consider snapshots which include this (absolute) \fB\fCpath\fR, when snapshot ID "latest" is given (can be specified multiple times)
+ only consider snapshots including this (absolute) \fB\fCpath\fR, when snapshot ID "latest" is given (can be specified multiple times)
.PP
\fB--recursive\fP[=false]
@@ -59,7 +59,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--tag\fP=[]
- only consider snapshots which include this \fB\fCtaglist\fR, when snapshot ID "latest" is given (can be specified multiple times)
+ only consider snapshots including \fB\fCtag[,tag,...]\fR, when snapshot ID "latest" is given (can be specified multiple times)
.SH OPTIONS INHERITED FROM PARENT COMMANDS
@@ -93,11 +93,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -113,7 +113,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -141,7 +141,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-migrate.1 b/doc/man/restic-migrate.1
index 578c85d69..ee4d44e71 100644
--- a/doc/man/restic-migrate.1
+++ b/doc/man/restic-migrate.1
@@ -64,11 +64,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -84,7 +84,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -112,7 +112,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-mount.1 b/doc/man/restic-mount.1
index b2feee01c..da38ae451 100644
--- a/doc/man/restic-mount.1
+++ b/doc/man/restic-mount.1
@@ -84,7 +84,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-H\fP, \fB--host\fP=[]
- only consider snapshots for this host (can be specified multiple times)
+ only consider snapshots for this \fB\fChost\fR (can be specified multiple times)
.PP
\fB--no-default-permissions\fP[=false]
@@ -96,7 +96,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--path\fP=[]
- only consider snapshots which include this (absolute) \fB\fCpath\fR
+ only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times)
.PP
\fB--path-template\fP=[]
@@ -104,7 +104,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--tag\fP=[]
- only consider snapshots which include this \fB\fCtaglist\fR
+ only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times)
.PP
\fB--time-template\fP="2006-01-02T15:04:05Z07:00"
@@ -142,11 +142,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -162,7 +162,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -190,7 +190,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-prune.1 b/doc/man/restic-prune.1
index 77a712039..88c03f72a 100644
--- a/doc/man/restic-prune.1
+++ b/doc/man/restic-prune.1
@@ -87,11 +87,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -107,7 +107,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -135,7 +135,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-rebuild-index.1 b/doc/man/restic-rebuild-index.1
index c37f55a18..3be67e79e 100644
--- a/doc/man/restic-rebuild-index.1
+++ b/doc/man/restic-rebuild-index.1
@@ -63,11 +63,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -83,7 +83,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -111,7 +111,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-recover.1 b/doc/man/restic-recover.1
index cc45eec9a..7415a1113 100644
--- a/doc/man/restic-recover.1
+++ b/doc/man/restic-recover.1
@@ -60,11 +60,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -80,7 +80,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -108,7 +108,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-restore.1 b/doc/man/restic-restore.1
index e96337e7d..2348f7478 100644
--- a/doc/man/restic-restore.1
+++ b/doc/man/restic-restore.1
@@ -37,7 +37,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-H\fP, \fB--host\fP=[]
- only consider snapshots for this host when the snapshot ID is "latest" (can be specified multiple times)
+ only consider snapshots for this \fB\fChost\fR, when snapshot ID "latest" is given (can be specified multiple times)
.PP
\fB--iexclude\fP=[]
@@ -53,11 +53,15 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--path\fP=[]
- only consider snapshots which include this (absolute) \fB\fCpath\fR for snapshot ID "latest"
+ only consider snapshots including this (absolute) \fB\fCpath\fR, when snapshot ID "latest" is given (can be specified multiple times)
+
+.PP
+\fB--sparse\fP[=false]
+ restore files as sparse
.PP
\fB--tag\fP=[]
- only consider snapshots which include this \fB\fCtaglist\fR for snapshot ID "latest"
+ only consider snapshots including \fB\fCtag[,tag,...]\fR, when snapshot ID "latest" is given (can be specified multiple times)
.PP
\fB-t\fP, \fB--target\fP=""
@@ -99,11 +103,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -119,7 +123,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -147,7 +151,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-rewrite.1 b/doc/man/restic-rewrite.1
new file mode 100644
index 000000000..9f33bcb64
--- /dev/null
+++ b/doc/man/restic-rewrite.1
@@ -0,0 +1,167 @@
+.nh
+.TH "restic backup" "1" "Jan 2017" "generated by \fB\fCrestic generate\fR" ""
+
+.SH NAME
+.PP
+restic-rewrite - Rewrite snapshots to exclude unwanted files
+
+
+.SH SYNOPSIS
+.PP
+\fBrestic rewrite [flags] [snapshotID ...]\fP
+
+
+.SH DESCRIPTION
+.PP
+The "rewrite" command excludes files from existing snapshots. It creates new
+snapshots containing the same data as the original ones, but without the files
+you specify to exclude. All metadata (time, host, tags) will be preserved.
+
+.PP
+The snapshots to rewrite are specified using the --host, --tag and --path options,
+or by providing a list of snapshot IDs. Please note that specifying neither any of
+these options nor a snapshot ID will cause the command to rewrite all snapshots.
+
+.PP
+The special tag 'rewrite' will be added to the new snapshots to distinguish
+them from the original ones, unless --forget is used. If the --forget option is
+used, the original snapshots will instead be directly removed from the repository.
+
+.PP
+Please note that the --forget option only removes the snapshots and not the actual
+data stored in the repository. In order to delete the no longer referenced data,
+use the "prune" command.
+
+
+.SH EXIT STATUS
+.PP
+Exit status is 0 if the command was successful, and non-zero if there was any error.
+
+
+.SH OPTIONS
+.PP
+\fB-n\fP, \fB--dry-run\fP[=false]
+ do not do anything, just print what would be done
+
+.PP
+\fB-e\fP, \fB--exclude\fP=[]
+ exclude a \fB\fCpattern\fR (can be specified multiple times)
+
+.PP
+\fB--exclude-file\fP=[]
+ read exclude patterns from a \fB\fCfile\fR (can be specified multiple times)
+
+.PP
+\fB--forget\fP[=false]
+ remove original snapshots after creating new ones
+
+.PP
+\fB-h\fP, \fB--help\fP[=false]
+ help for rewrite
+
+.PP
+\fB-H\fP, \fB--host\fP=[]
+ only consider snapshots for this \fB\fChost\fR (can be specified multiple times)
+
+.PP
+\fB--iexclude\fP=[]
+ same as --exclude \fB\fCpattern\fR but ignores the casing of filenames
+
+.PP
+\fB--iexclude-file\fP=[]
+ same as --exclude-file but ignores casing of \fB\fCfile\fRnames in patterns
+
+.PP
+\fB--path\fP=[]
+ only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times)
+
+.PP
+\fB--tag\fP=[]
+ only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times)
+
+
+.SH OPTIONS INHERITED FROM PARENT COMMANDS
+.PP
+\fB--cacert\fP=[]
+ \fB\fCfile\fR to load root certificates from (default: use system certificates)
+
+.PP
+\fB--cache-dir\fP=""
+ set the cache \fB\fCdirectory\fR\&. (default: use system default cache directory)
+
+.PP
+\fB--cleanup-cache\fP[=false]
+ auto remove old cache directories
+
+.PP
+\fB--compression\fP=auto
+ compression mode (only available for repository format version 2), one of (auto|off|max)
+
+.PP
+\fB--insecure-tls\fP[=false]
+ skip TLS certificate verification when connecting to the repository (insecure)
+
+.PP
+\fB--json\fP[=false]
+ set output mode to JSON for commands that support it
+
+.PP
+\fB--key-hint\fP=""
+ \fB\fCkey\fR ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
+
+.PP
+\fB--limit-download\fP=0
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
+
+.PP
+\fB--limit-upload\fP=0
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
+
+.PP
+\fB--no-cache\fP[=false]
+ do not use a local cache
+
+.PP
+\fB--no-lock\fP[=false]
+ do not lock the repository, this allows some operations on read-only repositories
+
+.PP
+\fB-o\fP, \fB--option\fP=[]
+ set extended option (\fB\fCkey=value\fR, can be specified multiple times)
+
+.PP
+\fB--pack-size\fP=0
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+
+.PP
+\fB--password-command\fP=""
+ shell \fB\fCcommand\fR to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
+
+.PP
+\fB-p\fP, \fB--password-file\fP=""
+ \fB\fCfile\fR to read the repository password from (default: $RESTIC_PASSWORD_FILE)
+
+.PP
+\fB-q\fP, \fB--quiet\fP[=false]
+ do not output comprehensive progress report
+
+.PP
+\fB-r\fP, \fB--repo\fP=""
+ \fB\fCrepository\fR to backup to or restore from (default: $RESTIC_REPOSITORY)
+
+.PP
+\fB--repository-file\fP=""
+ \fB\fCfile\fR to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
+
+.PP
+\fB--tls-client-cert\fP=""
+ path to a \fB\fCfile\fR containing PEM encoded TLS client certificate and private key
+
+.PP
+\fB-v\fP, \fB--verbose\fP[=0]
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
+
+
+.SH SEE ALSO
+.PP
+\fBrestic(1)\fP
diff --git a/doc/man/restic-self-update.1 b/doc/man/restic-self-update.1
index 9ac97d8c7..25f863396 100644
--- a/doc/man/restic-self-update.1
+++ b/doc/man/restic-self-update.1
@@ -65,11 +65,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -85,7 +85,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -113,7 +113,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-snapshots.1 b/doc/man/restic-snapshots.1
index b99ec93d0..78cd664e3 100644
--- a/doc/man/restic-snapshots.1
+++ b/doc/man/restic-snapshots.1
@@ -44,11 +44,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--path\fP=[]
- only consider snapshots for this \fB\fCpath\fR (can be specified multiple times)
+ only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times)
.PP
\fB--tag\fP=[]
- only consider snapshots which include this \fB\fCtaglist\fR in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times)
+ only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times)
.SH OPTIONS INHERITED FROM PARENT COMMANDS
@@ -82,11 +82,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -102,7 +102,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -130,7 +130,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-stats.1 b/doc/man/restic-stats.1
index 08d058beb..6e3b9838b 100644
--- a/doc/man/restic-stats.1
+++ b/doc/man/restic-stats.1
@@ -58,7 +58,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-H\fP, \fB--host\fP=[]
- only consider snapshots with the given \fB\fChost\fR (can be specified multiple times)
+ only consider snapshots for this \fB\fChost\fR (can be specified multiple times)
.PP
\fB--mode\fP="restore-size"
@@ -66,11 +66,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--path\fP=[]
- only consider snapshots which include this (absolute) \fB\fCpath\fR (can be specified multiple times)
+ only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times)
.PP
\fB--tag\fP=[]
- only consider snapshots which include this \fB\fCtaglist\fR in the format \fB\fCtag[,tag,...]\fR (can be specified multiple times)
+ only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times)
.SH OPTIONS INHERITED FROM PARENT COMMANDS
@@ -104,11 +104,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -124,7 +124,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -152,7 +152,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-tag.1 b/doc/man/restic-tag.1
index bb72612f1..06bf25495 100644
--- a/doc/man/restic-tag.1
+++ b/doc/man/restic-tag.1
@@ -39,11 +39,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-H\fP, \fB--host\fP=[]
- only consider snapshots for this \fB\fChost\fR, when no snapshot ID is given (can be specified multiple times)
+ only consider snapshots for this \fB\fChost\fR (can be specified multiple times)
.PP
\fB--path\fP=[]
- only consider snapshots which include this (absolute) \fB\fCpath\fR, when no snapshot-ID is given
+ only consider snapshots including this (absolute) \fB\fCpath\fR (can be specified multiple times)
.PP
\fB--remove\fP=[]
@@ -55,7 +55,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--tag\fP=[]
- only consider snapshots which include this \fB\fCtaglist\fR, when no snapshot-ID is given
+ only consider snapshots including \fB\fCtag[,tag,...]\fR (can be specified multiple times)
.SH OPTIONS INHERITED FROM PARENT COMMANDS
@@ -89,11 +89,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -109,7 +109,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -137,7 +137,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-unlock.1 b/doc/man/restic-unlock.1
index 99a969498..c4ad7f050 100644
--- a/doc/man/restic-unlock.1
+++ b/doc/man/restic-unlock.1
@@ -62,11 +62,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -82,7 +82,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -110,7 +110,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic-version.1 b/doc/man/restic-version.1
index a803cd491..b410d1231 100644
--- a/doc/man/restic-version.1
+++ b/doc/man/restic-version.1
@@ -59,11 +59,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -79,7 +79,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -107,7 +107,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
diff --git a/doc/man/restic.1 b/doc/man/restic.1
index 9577e012a..76602d02d 100644
--- a/doc/man/restic.1
+++ b/doc/man/restic.1
@@ -52,11 +52,11 @@ directories in an encrypted repository stored on different backends.
.PP
\fB--limit-download\fP=0
- limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ limits downloads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--limit-upload\fP=0
- limits uploads to a maximum rate in KiB/s. (default: unlimited)
+ limits uploads to a maximum \fB\fCrate\fR in KiB/s. (default: unlimited)
.PP
\fB--no-cache\fP[=false]
@@ -72,7 +72,7 @@ directories in an encrypted repository stored on different backends.
.PP
\fB--pack-size\fP=0
- set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
+ set target pack \fB\fCsize\fR in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
.PP
\fB--password-command\fP=""
@@ -100,9 +100,9 @@ directories in an encrypted repository stored on different backends.
.PP
\fB-v\fP, \fB--verbose\fP[=0]
- be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 3)
+ be verbose (specify multiple times or a level using --verbose=\fB\fCn\fR, max level/times is 2)
.SH SEE ALSO
.PP
-\fBrestic-backup(1)\fP, \fBrestic-cache(1)\fP, \fBrestic-cat(1)\fP, \fBrestic-check(1)\fP, \fBrestic-copy(1)\fP, \fBrestic-diff(1)\fP, \fBrestic-dump(1)\fP, \fBrestic-find(1)\fP, \fBrestic-forget(1)\fP, \fBrestic-generate(1)\fP, \fBrestic-init(1)\fP, \fBrestic-key(1)\fP, \fBrestic-list(1)\fP, \fBrestic-ls(1)\fP, \fBrestic-migrate(1)\fP, \fBrestic-mount(1)\fP, \fBrestic-prune(1)\fP, \fBrestic-rebuild-index(1)\fP, \fBrestic-recover(1)\fP, \fBrestic-restore(1)\fP, \fBrestic-self-update(1)\fP, \fBrestic-snapshots(1)\fP, \fBrestic-stats(1)\fP, \fBrestic-tag(1)\fP, \fBrestic-unlock(1)\fP, \fBrestic-version(1)\fP
+\fBrestic-backup(1)\fP, \fBrestic-cache(1)\fP, \fBrestic-cat(1)\fP, \fBrestic-check(1)\fP, \fBrestic-copy(1)\fP, \fBrestic-diff(1)\fP, \fBrestic-dump(1)\fP, \fBrestic-find(1)\fP, \fBrestic-forget(1)\fP, \fBrestic-generate(1)\fP, \fBrestic-init(1)\fP, \fBrestic-key(1)\fP, \fBrestic-list(1)\fP, \fBrestic-ls(1)\fP, \fBrestic-migrate(1)\fP, \fBrestic-mount(1)\fP, \fBrestic-prune(1)\fP, \fBrestic-rebuild-index(1)\fP, \fBrestic-recover(1)\fP, \fBrestic-restore(1)\fP, \fBrestic-rewrite(1)\fP, \fBrestic-self-update(1)\fP, \fBrestic-snapshots(1)\fP, \fBrestic-stats(1)\fP, \fBrestic-tag(1)\fP, \fBrestic-unlock(1)\fP, \fBrestic-version(1)\fP
diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst
index e17e5cd8e..899a45688 100644
--- a/doc/manual_rest.rst
+++ b/doc/manual_rest.rst
@@ -38,6 +38,7 @@ Usage help is available:
rebuild-index Build a new index
recover Recover data from the repository not referenced by snapshots
restore Extract the data from a snapshot
+ rewrite Rewrite snapshots to exclude unwanted files
self-update Update the restic binary
snapshots List all snapshots
stats Scan the repository and show basic statistics
@@ -54,19 +55,19 @@ Usage help is available:
--insecure-tls skip TLS certificate verification when connecting to the repository (insecure)
--json set output mode to JSON for commands that support it
--key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
- --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited)
- --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited)
- --pack-size uint set target pack size in MiB. (default: $RESTIC_PACK_SIZE)
+ --limit-download rate limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ --limit-upload rate limits uploads to a maximum rate in KiB/s. (default: unlimited)
--no-cache do not use a local cache
--no-lock do not lock the repository, this allows some operations on read-only repositories
-o, --option key=value set extended option (key=value, can be specified multiple times)
+ --pack-size size set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
--password-command command shell command to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
-p, --password-file file file to read the repository password from (default: $RESTIC_PASSWORD_FILE)
-q, --quiet do not output comprehensive progress report
-r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY)
--repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
--tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key
- -v, --verbose n be verbose (specify multiple times or a level using --verbose=n, max level/times is 3)
+ -v, --verbose n be verbose (specify multiple times or a level using --verbose=n, max level/times is 2)
Use "restic [command] --help" for more information about a command.
@@ -91,7 +92,7 @@ command:
Exit status is 3 if some source data could not be read (incomplete snapshot created).
Usage:
- restic backup [flags] FILE/DIR [FILE/DIR] ...
+ restic backup [flags] [FILE/DIR] ...
Flags:
-n, --dry-run do not upload or write any data, just show what would be done
@@ -110,8 +111,10 @@ command:
--iexclude-file file same as --exclude-file but ignores casing of filenames in patterns
--ignore-ctime ignore ctime changes when checking for modified files
--ignore-inode ignore inode number changes when checking for modified files
+ --no-scan do not run scanner to estimate size of backup
-x, --one-file-system exclude other file systems, don't cross filesystem boundaries and subvolumes
--parent snapshot use this parent snapshot (default: last snapshot in the repository that has the same target files/directories, and is not newer than the snapshot time)
+ --read-concurrency n read n file concurrently (default: $RESTIC_READ_CONCURRENCY or 2)
--stdin read backup from stdin
--stdin-filename filename filename to use when reading from stdin (default "stdin")
--tag tags add tags for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times) (default [])
@@ -127,19 +130,19 @@ command:
--insecure-tls skip TLS certificate verification when connecting to the repository (insecure)
--json set output mode to JSON for commands that support it
--key-hint key key ID of key to try decrypting first (default: $RESTIC_KEY_HINT)
- --limit-download int limits downloads to a maximum rate in KiB/s. (default: unlimited)
- --limit-upload int limits uploads to a maximum rate in KiB/s. (default: unlimited)
- --pack-size uint set target pack size in MiB. (default: $RESTIC_PACK_SIZE)
+ --limit-download rate limits downloads to a maximum rate in KiB/s. (default: unlimited)
+ --limit-upload rate limits uploads to a maximum rate in KiB/s. (default: unlimited)
--no-cache do not use a local cache
--no-lock do not lock the repository, this allows some operations on read-only repositories
-o, --option key=value set extended option (key=value, can be specified multiple times)
+ --pack-size size set target pack size in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)
--password-command command shell command to obtain the repository password from (default: $RESTIC_PASSWORD_COMMAND)
-p, --password-file file file to read the repository password from (default: $RESTIC_PASSWORD_FILE)
-q, --quiet do not output comprehensive progress report
-r, --repo repository repository to backup to or restore from (default: $RESTIC_REPOSITORY)
--repository-file file file to read the repository location from (default: $RESTIC_REPOSITORY_FILE)
--tls-client-cert file path to a file containing PEM encoded TLS client certificate and private key
- -v, --verbose n be verbose (specify multiple times or a level using --verbose=n, max level/times is 3)
+ -v, --verbose n be verbose (specify multiple times or a level using --verbose=n, max level/times is 2)
Subcommands that support showing progress information such as ``backup``,
``check`` and ``prune`` will do so unless the quiet flag ``-q`` or
@@ -256,7 +259,10 @@ Metadata handling
~~~~~~~~~~~~~~~~~
Restic saves and restores most default attributes, including extended attributes like ACLs.
-Sparse files are not handled in a special way yet, and aren't restored.
+Information about holes in a sparse file is not stored explicitly, that is during a backup
+the zero bytes in a hole are deduplicated and compressed like any other data backed up.
+Instead, the restore command optionally creates holes in files by detecting and replacing
+long runs of zeros, in filesystems that support sparse files.
The following metadata is handled by restic:
@@ -442,3 +448,4 @@ time it is used, so by looking at the timestamps of the sub directories of the
cache directory it can decide which sub directories are old and probably not
needed any more. You can either remove these directories manually, or run a
restic command with the ``--cleanup-cache`` flag.
+
diff --git a/doc/powershell-completion.ps1 b/doc/powershell-completion.ps1
new file mode 100644
index 000000000..271809161
--- /dev/null
+++ b/doc/powershell-completion.ps1
@@ -0,0 +1,230 @@
+# powershell completion for restic -*- shell-script -*-
+
+function __restic_debug {
+ if ($env:BASH_COMP_DEBUG_FILE) {
+ "$args" | Out-File -Append -FilePath "$env:BASH_COMP_DEBUG_FILE"
+ }
+}
+
+filter __restic_escapeStringWithSpecialChars {
+ $_ -replace '\s|#|@|\$|;|,|''|\{|\}|\(|\)|"|`|\||<|>|&','`$&'
+}
+
+[scriptblock]$__resticCompleterBlock = {
+ param(
+ $WordToComplete,
+ $CommandAst,
+ $CursorPosition
+ )
+
+ # Get the current command line and convert into a string
+ $Command = $CommandAst.CommandElements
+ $Command = "$Command"
+
+ __restic_debug ""
+ __restic_debug "========= starting completion logic =========="
+ __restic_debug "WordToComplete: $WordToComplete Command: $Command CursorPosition: $CursorPosition"
+
+ # The user could have moved the cursor backwards on the command-line.
+ # We need to trigger completion from the $CursorPosition location, so we need
+ # to truncate the command-line ($Command) up to the $CursorPosition location.
+ # Make sure the $Command is longer then the $CursorPosition before we truncate.
+ # This happens because the $Command does not include the last space.
+ if ($Command.Length -gt $CursorPosition) {
+ $Command=$Command.Substring(0,$CursorPosition)
+ }
+ __restic_debug "Truncated command: $Command"
+
+ $ShellCompDirectiveError=1
+ $ShellCompDirectiveNoSpace=2
+ $ShellCompDirectiveNoFileComp=4
+ $ShellCompDirectiveFilterFileExt=8
+ $ShellCompDirectiveFilterDirs=16
+
+ # Prepare the command to request completions for the program.
+ # Split the command at the first space to separate the program and arguments.
+ $Program,$Arguments = $Command.Split(" ",2)
+
+ $RequestComp="$Program __completeNoDesc $Arguments"
+ __restic_debug "RequestComp: $RequestComp"
+
+ # we cannot use $WordToComplete because it
+ # has the wrong values if the cursor was moved
+ # so use the last argument
+ if ($WordToComplete -ne "" ) {
+ $WordToComplete = $Arguments.Split(" ")[-1]
+ }
+ __restic_debug "New WordToComplete: $WordToComplete"
+
+
+ # Check for flag with equal sign
+ $IsEqualFlag = ($WordToComplete -Like "--*=*" )
+ if ( $IsEqualFlag ) {
+ __restic_debug "Completing equal sign flag"
+ # Remove the flag part
+ $Flag,$WordToComplete = $WordToComplete.Split("=",2)
+ }
+
+ if ( $WordToComplete -eq "" -And ( -Not $IsEqualFlag )) {
+ # If the last parameter is complete (there is a space following it)
+ # We add an extra empty parameter so we can indicate this to the go method.
+ __restic_debug "Adding extra empty parameter"
+ # We need to use `"`" to pass an empty argument a "" or '' does not work!!!
+ $RequestComp="$RequestComp" + ' `"`"'
+ }
+
+ __restic_debug "Calling $RequestComp"
+ # First disable ActiveHelp which is not supported for Powershell
+ $env:RESTIC_ACTIVE_HELP=0
+
+ #call the command store the output in $out and redirect stderr and stdout to null
+ # $Out is an array contains each line per element
+ Invoke-Expression -OutVariable out "$RequestComp" 2>&1 | Out-Null
+
+ # get directive from last line
+ [int]$Directive = $Out[-1].TrimStart(':')
+ if ($Directive -eq "") {
+ # There is no directive specified
+ $Directive = 0
+ }
+ __restic_debug "The completion directive is: $Directive"
+
+ # remove directive (last element) from out
+ $Out = $Out | Where-Object { $_ -ne $Out[-1] }
+ __restic_debug "The completions are: $Out"
+
+ if (($Directive -band $ShellCompDirectiveError) -ne 0 ) {
+ # Error code. No completion.
+ __restic_debug "Received error from custom completion go code"
+ return
+ }
+
+ $Longest = 0
+ $Values = $Out | ForEach-Object {
+ #Split the output in name and description
+ $Name, $Description = $_.Split("`t",2)
+ __restic_debug "Name: $Name Description: $Description"
+
+ # Look for the longest completion so that we can format things nicely
+ if ($Longest -lt $Name.Length) {
+ $Longest = $Name.Length
+ }
+
+ # Set the description to a one space string if there is none set.
+ # This is needed because the CompletionResult does not accept an empty string as argument
+ if (-Not $Description) {
+ $Description = " "
+ }
+ @{Name="$Name";Description="$Description"}
+ }
+
+
+ $Space = " "
+ if (($Directive -band $ShellCompDirectiveNoSpace) -ne 0 ) {
+ # remove the space here
+ __restic_debug "ShellCompDirectiveNoSpace is called"
+ $Space = ""
+ }
+
+ if ((($Directive -band $ShellCompDirectiveFilterFileExt) -ne 0 ) -or
+ (($Directive -band $ShellCompDirectiveFilterDirs) -ne 0 )) {
+ __restic_debug "ShellCompDirectiveFilterFileExt ShellCompDirectiveFilterDirs are not supported"
+
+ # return here to prevent the completion of the extensions
+ return
+ }
+
+ $Values = $Values | Where-Object {
+ # filter the result
+ $_.Name -like "$WordToComplete*"
+
+ # Join the flag back if we have an equal sign flag
+ if ( $IsEqualFlag ) {
+ __restic_debug "Join the equal sign flag back to the completion value"
+ $_.Name = $Flag + "=" + $_.Name
+ }
+ }
+
+ if (($Directive -band $ShellCompDirectiveNoFileComp) -ne 0 ) {
+ __restic_debug "ShellCompDirectiveNoFileComp is called"
+
+ if ($Values.Length -eq 0) {
+ # Just print an empty string here so the
+ # shell does not start to complete paths.
+ # We cannot use CompletionResult here because
+ # it does not accept an empty string as argument.
+ ""
+ return
+ }
+ }
+
+ # Get the current mode
+ $Mode = (Get-PSReadLineKeyHandler | Where-Object {$_.Key -eq "Tab" }).Function
+ __restic_debug "Mode: $Mode"
+
+ $Values | ForEach-Object {
+
+ # store temporary because switch will overwrite $_
+ $comp = $_
+
+ # PowerShell supports three different completion modes
+ # - TabCompleteNext (default windows style - on each key press the next option is displayed)
+ # - Complete (works like bash)
+ # - MenuComplete (works like zsh)
+ # You set the mode with Set-PSReadLineKeyHandler -Key Tab -Function <mode>
+
+ # CompletionResult Arguments:
+ # 1) CompletionText text to be used as the auto completion result
+ # 2) ListItemText text to be displayed in the suggestion list
+ # 3) ResultType type of completion result
+ # 4) ToolTip text for the tooltip with details about the object
+
+ switch ($Mode) {
+
+ # bash like
+ "Complete" {
+
+ if ($Values.Length -eq 1) {
+ __restic_debug "Only one completion left"
+
+ # insert space after value
+ [System.Management.Automation.CompletionResult]::new($($comp.Name | __restic_escapeStringWithSpecialChars) + $Space, "$($comp.Name)", 'ParameterValue', "$($comp.Description)")
+
+ } else {
+ # Add the proper number of spaces to align the descriptions
+ while($comp.Name.Length -lt $Longest) {
+ $comp.Name = $comp.Name + " "
+ }
+
+ # Check for empty description and only add parentheses if needed
+ if ($($comp.Description) -eq " " ) {
+ $Description = ""
+ } else {
+ $Description = " ($($comp.Description))"
+ }
+
+ [System.Management.Automation.CompletionResult]::new("$($comp.Name)$Description", "$($comp.Name)$Description", 'ParameterValue', "$($comp.Description)")
+ }
+ }
+
+ # zsh like
+ "MenuComplete" {
+ # insert space after value
+ # MenuComplete will automatically show the ToolTip of
+ # the highlighted value at the bottom of the suggestions.
+ [System.Management.Automation.CompletionResult]::new($($comp.Name | __restic_escapeStringWithSpecialChars) + $Space, "$($comp.Name)", 'ParameterValue', "$($comp.Description)")
+ }
+
+ # TabCompleteNext and in case we get something unknown
+ Default {
+ # Like MenuComplete but we don't want to add a space here because
+ # the user need to press space anyway to get the completion.
+ # Description will not be shown because that's not possible with TabCompleteNext
+ [System.Management.Automation.CompletionResult]::new($($comp.Name | __restic_escapeStringWithSpecialChars), "$($comp.Name)", 'ParameterValue', "$($comp.Description)")
+ }
+ }
+
+ }
+}
+
+Register-ArgumentCompleter -CommandName 'restic' -ScriptBlock $__resticCompleterBlock
diff --git a/docker/Dockerfile b/docker/Dockerfile
index dacb02542..9f47fa10f 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.16-alpine AS builder
+FROM golang:1.19-alpine AS builder
WORKDIR /go/src/github.com/restic/restic
diff --git a/go.mod b/go.mod
index d819a6be7..a172c6992 100644
--- a/go.mod
+++ b/go.mod
@@ -1,49 +1,73 @@
module github.com/restic/restic
require (
- bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512
- cloud.google.com/go v0.103.0 // indirect
- cloud.google.com/go/compute v1.9.0 // indirect
- cloud.google.com/go/storage v1.25.0
- github.com/Azure/azure-sdk-for-go v66.0.0+incompatible
- github.com/Azure/go-autorest/autorest v0.11.28 // indirect
- github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect
- github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
- github.com/cenkalti/backoff/v4 v4.1.3
- github.com/cespare/xxhash/v2 v2.1.2
- github.com/dnaeon/go-vcr v1.2.0 // indirect
+ cloud.google.com/go/storage v1.29.0
+ github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0
+ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1
+ github.com/anacrolix/fuse v0.2.0
+ github.com/cenkalti/backoff/v4 v4.2.0
+ github.com/cespare/xxhash/v2 v2.2.0
github.com/elithrar/simple-scrypt v1.3.0
github.com/go-ole/go-ole v1.2.6
- github.com/gofrs/uuid v4.2.0+incompatible // indirect
- github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
- github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
- github.com/google/go-cmp v0.5.8
- github.com/googleapis/gax-go/v2 v2.5.1 // indirect
- github.com/hashicorp/golang-lru v0.5.4
- github.com/inconshreveable/mousetrap v1.0.1 // indirect
+ github.com/google/go-cmp v0.5.9
+ github.com/hashicorp/golang-lru/v2 v2.0.1
github.com/juju/ratelimit v1.0.2
- github.com/klauspost/compress v1.15.9
- github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6
- github.com/minio/minio-go/v7 v7.0.34
+ github.com/klauspost/compress v1.15.15
+ github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5
+ github.com/minio/minio-go/v7 v7.0.47
github.com/minio/sha256-simd v1.0.0
github.com/ncw/swift/v2 v2.0.1
github.com/pkg/errors v0.9.1
- github.com/pkg/profile v1.6.0
+ github.com/pkg/profile v1.7.0
github.com/pkg/sftp v1.13.5
- github.com/pkg/xattr v0.4.8
+ github.com/pkg/xattr v0.4.10-0.20221120235825-35026bbbd013
github.com/restic/chunker v0.4.0
- github.com/spf13/cobra v1.5.0
+ github.com/spf13/cobra v1.6.1
github.com/spf13/pflag v1.0.5
- golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8
- golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c
- golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2
- golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
- golang.org/x/sys v0.0.0-20220818161305-2296e01440c6
- golang.org/x/term v0.0.0-20220722155259-a9ba230a4035
- golang.org/x/text v0.3.7
- google.golang.org/api v0.93.0
- google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
+ golang.org/x/crypto v0.5.0
+ golang.org/x/net v0.5.0
+ golang.org/x/oauth2 v0.4.0
+ golang.org/x/sync v0.1.0
+ golang.org/x/sys v0.4.0
+ golang.org/x/term v0.4.0
+ golang.org/x/text v0.6.0
+ google.golang.org/api v0.108.0
+)
+
+require (
+ cloud.google.com/go v0.108.0 // indirect
+ cloud.google.com/go/compute v1.15.1 // indirect
+ cloud.google.com/go/compute/metadata v0.2.3 // indirect
+ cloud.google.com/go/iam v0.10.0 // indirect
+ github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect
+ github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+ github.com/dnaeon/go-vcr v1.2.0 // indirect
+ github.com/dustin/go-humanize v1.0.0 // indirect
+ github.com/felixge/fgprof v0.9.3 // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+ github.com/golang/protobuf v1.5.2 // indirect
+ github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b // indirect
+ github.com/google/uuid v1.3.0 // indirect
+ github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
+ github.com/googleapis/gax-go/v2 v2.7.0 // indirect
+ github.com/inconshreveable/mousetrap v1.1.0 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
+ github.com/klauspost/cpuid/v2 v2.2.3 // indirect
+ github.com/kr/fs v0.1.0 // indirect
+ github.com/minio/md5-simd v1.1.2 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
+ github.com/rs/xid v1.4.0 // indirect
+ github.com/russross/blackfriday/v2 v2.1.0 // indirect
+ github.com/sirupsen/logrus v1.9.0 // indirect
+ go.opencensus.io v0.24.0 // indirect
+ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+ google.golang.org/appengine v1.6.7 // indirect
+ google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
+ google.golang.org/grpc v1.52.0 // indirect
+ google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
)
-go 1.15
+go 1.18
diff --git a/go.sum b/go.sum
index 959651048..08069a411 100644
--- a/go.sum
+++ b/go.sum
@@ -1,115 +1,37 @@
-bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512 h1:SRsZGA7aFnCZETmov57jwPrWuTmaZK6+4R4v5FUe1/c=
-bazil.org/fuse v0.0.0-20200407214033-5883e5a4b512/go.mod h1:FbcW6z/2VytnFDhZfumh8Ss8zxHE6qpMP5sHTRe0EaM=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
-cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
-cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
-cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
-cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=
-cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=
-cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=
-cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=
-cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI=
-cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
-cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
-cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
-cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
-cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY=
-cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM=
-cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY=
-cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ=
-cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI=
-cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4=
-cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc=
-cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA=
-cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
-cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc=
-cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU=
-cloud.google.com/go v0.103.0 h1:YXtxp9ymmZjlGzxV7VrYQ8aaQuAgcqxSy6YhDX4I458=
-cloud.google.com/go v0.103.0/go.mod h1:vwLx1nqLrzLX/fpwSMOXmFIqBOyHsvHbnAdbGSJ+mKk=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
-cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
-cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
-cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
-cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow=
-cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
-cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
-cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s=
-cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU=
-cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U=
-cloud.google.com/go/compute v1.9.0 h1:ED/FP4xv8GJw63v556/ASNc1CeeLUO2Bs8nzaHchkHg=
-cloud.google.com/go/compute v1.9.0/go.mod h1:lWv1h/zUWTm/LozzfTJhBSkd6ShQq8la8VeeuOEGxfY=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
-cloud.google.com/go/iam v0.3.0 h1:exkAomrVUuzx9kWFI1wm3KI0uoDeUFPB4kKGzx6x+Gc=
-cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
-cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
-cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
-cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
-cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
-cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y=
-cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc=
-cloud.google.com/go/storage v1.25.0 h1:D2Dn0PslpK7Z3B2AvuUHyIC762bDbGJdlmQlCBR71os=
-cloud.google.com/go/storage v1.25.0/go.mod h1:Qys4JU+jeup3QnuKKAosWuxrD95C4MSqxfVDnSirDsI=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Azure/azure-sdk-for-go v66.0.0+incompatible h1:bmmC38SlE8/E81nNADlgmVGurPWMHDX2YNXVQMrBpEE=
-github.com/Azure/azure-sdk-for-go v66.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
-github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
-github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM=
-github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA=
-github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ=
-github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk=
-github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U=
-github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
-github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
-github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
-github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw=
-github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU=
-github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk=
-github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE=
-github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
-github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
-github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
-github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+cloud.google.com/go v0.108.0 h1:xntQwnfn8oHGX0crLVinvHM+AhXvi3QHQIEcX/2hiWk=
+cloud.google.com/go v0.108.0/go.mod h1:lNUfQqusBJp0bgAg6qrHgYFYbTB+dOiob1itwnlD33Q=
+cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE=
+cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA=
+cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
+cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
+cloud.google.com/go/iam v0.10.0 h1:fpP/gByFs6US1ma53v7VxhvbJpO2Aapng6wabJ99MuI=
+cloud.google.com/go/iam v0.10.0/go.mod h1:nXAECrMt2qHpF6RZUZseteD6QyanL68reN4OXPw0UWM=
+cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs=
+cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI=
+cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0 h1:VuHAcMq8pU1IWNT/m5yRaGqbK0BiQKHT8X4DTp9CHdI=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.3.0/go.mod h1:tZoQYdDZNOiIjdSn0dVWVfl0NEPGOJqVLzSrcFk4Is0=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8=
+github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU=
+github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4=
-github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
+github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
+github.com/anacrolix/fuse v0.2.0 h1:pc+To78kI2d/WUjIyrsdqeJQAesuwpGxlI3h1nAv3Do=
+github.com/anacrolix/fuse v0.2.0/go.mod h1:Kfu02xBwnySDpH3N23BmrP3MDfwAQGRLUCj6XyeOvBQ=
+github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4=
+github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
-github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
-github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
-github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -119,156 +41,83 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
+github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
github.com/elithrar/simple-scrypt v1.3.0 h1:KIlOlxdoQf9JWKl5lMAJ28SY2URB0XTRDn2TckyzAZg=
github.com/elithrar/simple-scrypt v1.3.0/go.mod h1:U2XQRI95XHY0St410VE3UjT7vuKb1qPwrl/EJwEqnZo=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
-github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
-github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
-github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
-github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
+github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
-github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0=
-github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
-github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
-github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
-github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
-github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
-github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
-github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
-github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ=
-github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
-github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
+github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b h1:8htHrh2bw9c7Idkb7YNac+ZpTqLMjRpI+FWu51ltaQc=
+github.com/google/pprof v0.0.0-20230111200839-76d1ae5aea2b/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
-github.com/googleapis/enterprise-certificate-proxy v0.1.0 h1:zO8WHNx/MYiAKJ3d5spxZXZE6KHmIQGQcAzwUzV7qQw=
-github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
-github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
-github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
-github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
-github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
-github.com/googleapis/gax-go/v2 v2.5.1 h1:kBRZU0PSuI7PspsSb/ChWoVResUcwNVIdpB049pKTiw=
-github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo=
-github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
-github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
-github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
+github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg=
+github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
+github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ=
+github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8=
+github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
+github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI=
github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
-github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw=
+github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0=
-github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
+github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
+github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6 h1:nz7i1au+nDzgExfqW5Zl6q85XNTvYoGnM5DHiQC0yYs=
-github.com/kurin/blazer v0.5.4-0.20211030221322-ba894c124ac6/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
+github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5 h1:OUlGa6AAolmjyPtILbMJ8vHayz5wd4wBUloheGcMhfA=
+github.com/kurin/blazer v0.5.4-0.20230113224640-3887e1ec64b5/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.34 h1:JMfS5fudx1mN6V2MMNyCJ7UMrjEzZzIvMgfkWc1Vnjk=
-github.com/minio/minio-go/v7 v7.0.34/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
+github.com/minio/minio-go/v7 v7.0.47 h1:sLiuCKGSIcn/MI6lREmTzX91DX/oRau4ia0j6e6eOSs=
+github.com/minio/minio-go/v7 v7.0.47/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -279,502 +128,133 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0=
github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
+github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/profile v1.6.0 h1:hUDfIISABYI59DyeB3OTay/HxSRwTQ8rB/H83k6r5dM=
-github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
+github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
+github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
-github.com/pkg/xattr v0.4.8 h1:3QwVADT+4oUm3zg7MXO/2i/lqnKkQ9viNY8pl5egRDE=
-github.com/pkg/xattr v0.4.8/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pkg/xattr v0.4.10-0.20221120235825-35026bbbd013 h1:aqByeeNnF7NiEbXCi7nBxZ272+6f6FUBmj/dUzWCdvc=
+github.com/pkg/xattr v0.4.10-0.20221120235825-35026bbbd013/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/restic/chunker v0.4.0 h1:YUPYCUn70MYP7VO4yllypp2SjmsRhRJaad3xKu1QFRw=
github.com/restic/chunker v0.4.0/go.mod h1:z0cH2BejpW636LXw0R/BGyv+Ey8+m9QGiOanDHItzyw=
-github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s=
github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
-github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
+github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
+github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stephens2424/writerset v1.0.2/go.mod h1:aS2JhsMn6eA7e82oNmW4rfsgAOp9COBTTl8mzkwADnc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
-github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
-go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
-go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
-go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 h1:GIAS/yBem/gq2MUqgNIzUHW7cJMmx3TGZOrnyYaNQ6c=
-golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
+golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
-golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
-golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
-golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
-golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c h1:JVAXQ10yGGVbSyoer5VILysz6YKjdNT2bsvlayjqhes=
-golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
-golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
-golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
-golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2 h1:+jnHzr9VPj32ykQVai5DNahi9+NSp7yYuCsl5eAQtL0=
-golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
+golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
+golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc=
-golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 h1:Sx/u41w+OwrInGdEckYmEuU5gHoGSL4QbDz3S9s6j4U=
-golang.org/x/sys v0.0.0-20220818161305-2296e01440c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc=
-golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
+golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
-golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=
-golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
-golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
-golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
-golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
-golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
-golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
-google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
-google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
-google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg=
-google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE=
-google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8=
-google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU=
-google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94=
-google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo=
-google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4=
-google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw=
-google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU=
-google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k=
-google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
-google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE=
-google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI=
-google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I=
-google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo=
-google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g=
-google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
-google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
-google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
-google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
-google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
-google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
-google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
-google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6FO2g=
-google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
-google.golang.org/api v0.88.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
-google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
-google.golang.org/api v0.91.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
-google.golang.org/api v0.93.0 h1:T2xt9gi0gHdxdnRkVQhT8mIvPaXKNsDNWz+L696M66M=
-google.golang.org/api v0.93.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
+golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+google.golang.org/api v0.108.0 h1:WVBc/faN0DkKtR43Q/7+tPny9ZoLZdIiAyG5Q9vFClg=
+google.golang.org/api v0.108.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
-google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
-google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
-google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A=
-google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A=
-google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
-google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
-google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
-google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k=
-google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48=
-google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w=
-google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY=
-google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
-google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI=
-google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E=
-google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
-google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
-google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
-google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220628213854-d9e0b6570c03/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220720214146-176da50484ac/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE=
-google.golang.org/genproto v0.0.0-20220722212130-b98a9ff5e252/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE=
-google.golang.org/genproto v0.0.0-20220804142021-4e6b2dfa6612/go.mod h1:iHe1svFLAZg9VWz891+QbRMwUv9O/1Ww+/mngYeThbc=
-google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc h1:Nf+EdcTLHR8qDNN/KfkQL0u0ssxt9OhbaWCl5C0ucEI=
-google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc/go.mod h1:dbqgFATTzChvnt+ujMdZwITVAJHFtfyN1qUhDqEiIlk=
+google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w=
+google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
-google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
-google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
-google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
-google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
-google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
-google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
-google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
-google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE=
-google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
-google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
-google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
-google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w=
-google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
-google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
+google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk=
+google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -783,35 +263,19 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
-google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
-rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
diff --git a/helpers/prepare-release/main.go b/helpers/prepare-release/main.go
index 2d38b0dd9..03924b0d9 100644
--- a/helpers/prepare-release/main.go
+++ b/helpers/prepare-release/main.go
@@ -4,7 +4,6 @@ import (
"bufio"
"bytes"
"fmt"
- "io/ioutil"
"os"
"os/exec"
"path/filepath"
@@ -73,13 +72,13 @@ func run(cmd string, args ...string) {
func replace(filename, from, to string) {
reg := regexp.MustCompile(from)
- buf, err := ioutil.ReadFile(filename)
+ buf, err := os.ReadFile(filename)
if err != nil {
die("error reading file %v: %v", filename, err)
}
buf = reg.ReplaceAll(buf, []byte(to))
- err = ioutil.WriteFile(filename, buf, 0644)
+ err = os.WriteFile(filename, buf, 0644)
if err != nil {
die("error writing file %v: %v", filename, err)
}
@@ -290,6 +289,7 @@ func generateFiles() {
run("./restic-generate.temp", "generate",
"--man", "doc/man",
"--zsh-completion", "doc/zsh-completion.zsh",
+ "--powershell-completion", "doc/powershell-completion.ps1",
"--fish-completion", "doc/fish-completion.fish",
"--bash-completion", "doc/bash-completion.sh")
rm("restic-generate.temp")
@@ -307,7 +307,7 @@ var versionPattern = `var version = ".*"`
const versionCodeFile = "cmd/restic/global.go"
func updateVersion() {
- err := ioutil.WriteFile("VERSION", []byte(opts.Version+"\n"), 0644)
+ err := os.WriteFile("VERSION", []byte(opts.Version+"\n"), 0644)
if err != nil {
die("unable to write version to file: %v", err)
}
@@ -365,7 +365,7 @@ func runBuild(sourceDir, outputDir, version string) {
}
func readdir(dir string) []string {
- fis, err := ioutil.ReadDir(dir)
+ fis, err := os.ReadDir(dir)
if err != nil {
die("readdir %v failed: %v", dir, err)
}
@@ -419,7 +419,7 @@ func updateDocker(outputDir, version string) {
}
func tempdir(prefix string) string {
- dir, err := ioutil.TempDir(getwd(), prefix)
+ dir, err := os.MkdirTemp(getwd(), prefix)
if err != nil {
die("unable to create temp dir %q: %v", prefix, err)
}
diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go
index 4fcc8e30c..a56965d63 100644
--- a/internal/archiver/archiver.go
+++ b/internal/archiver/archiver.go
@@ -68,6 +68,9 @@ type Archiver struct {
// be in the snapshot after saving. s contains some statistics about this
// particular file/dir.
//
+ // Once reading a file has completed successfully (but not saving it yet),
+ // CompleteItem will be called with current == nil.
+ //
// CompleteItem may be called asynchronously from several different
// goroutines!
CompleteItem func(item string, previous, current *restic.Node, s ItemStats, d time.Duration)
@@ -76,7 +79,7 @@ type Archiver struct {
StartFile func(filename string)
// CompleteBlob is called for all saved blobs for files.
- CompleteBlob func(filename string, bytes uint64)
+ CompleteBlob func(bytes uint64)
// WithAtime configures if the access time for files and directories should
// be saved. Enabling it may result in much metadata, so it's off by
@@ -95,10 +98,10 @@ const (
// Options is used to configure the archiver.
type Options struct {
- // FileReadConcurrency sets how many files are read in concurrently. If
+ // ReadConcurrency sets how many files are read in concurrently. If
// it's set to zero, at most two files are read in concurrently (which
// turned out to be a good default for most situations).
- FileReadConcurrency uint
+ ReadConcurrency uint
// SaveBlobConcurrency sets how many blobs are hashed and saved
// concurrently. If it's set to zero, the default is the number of CPUs
@@ -113,11 +116,11 @@ type Options struct {
// ApplyDefaults returns a copy of o with the default options set for all unset
// fields.
func (o Options) ApplyDefaults() Options {
- if o.FileReadConcurrency == 0 {
+ if o.ReadConcurrency == 0 {
// two is a sweet spot for almost all situations. We've done some
// experiments documented here:
// https://github.com/borgbackup/borg/issues/3500
- o.FileReadConcurrency = 2
+ o.ReadConcurrency = 2
}
if o.SaveBlobConcurrency == 0 {
@@ -132,7 +135,7 @@ func (o Options) ApplyDefaults() Options {
// Also allow waiting for FileReadConcurrency files, this is the maximum of FutureFiles
// which currently can be in progress. The main backup loop blocks when trying to queue
// more files to read.
- o.SaveTreeConcurrency = uint(runtime.GOMAXPROCS(0)) + o.FileReadConcurrency
+ o.SaveTreeConcurrency = uint(runtime.GOMAXPROCS(0)) + o.ReadConcurrency
}
return o
@@ -149,7 +152,7 @@ func New(repo restic.Repository, fs fs.FS, opts Options) *Archiver {
CompleteItem: func(string, *restic.Node, *restic.Node, ItemStats, time.Duration) {},
StartFile: func(string) {},
- CompleteBlob: func(string, uint64) {},
+ CompleteBlob: func(uint64) {},
}
return arch
@@ -172,38 +175,15 @@ func (arch *Archiver) error(item string, err error) error {
return errf
}
-// saveTree stores a tree in the repo. It checks the index and the known blobs
-// before saving anything.
-func (arch *Archiver) saveTree(ctx context.Context, t *restic.TreeJSONBuilder) (restic.ID, ItemStats, error) {
- var s ItemStats
- buf, err := t.Finalize()
- if err != nil {
- return restic.ID{}, s, err
- }
-
- b := &Buffer{Data: buf}
- res := arch.blobSaver.Save(ctx, restic.TreeBlob, b)
-
- sbr := res.Take(ctx)
- if !sbr.known {
- s.TreeBlobs++
- s.TreeSize += uint64(sbr.length)
- s.TreeSizeInRepo += uint64(sbr.sizeInRepo)
- }
- // The context was canceled in the meantime, id might be invalid
- if ctx.Err() != nil {
- return restic.ID{}, s, ctx.Err()
- }
- return sbr.id, s, nil
-}
-
// nodeFromFileInfo returns the restic node from an os.FileInfo.
-func (arch *Archiver) nodeFromFileInfo(filename string, fi os.FileInfo) (*restic.Node, error) {
+func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo) (*restic.Node, error) {
node, err := restic.NodeFromFileInfo(filename, fi)
if !arch.WithAtime {
node.AccessTime = node.ModTime
}
- return node, errors.Wrap(err, "NodeFromFileInfo")
+ // overwrite name to match that within the snapshot
+ node.Name = path.Base(snPath)
+ return node, errors.WithStack(err)
}
// loadSubtree tries to load the subtree referenced by node. In case of an error, nil is returned.
@@ -237,7 +217,7 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
debug.Log("%v %v", snPath, dir)
- treeNode, err := arch.nodeFromFileInfo(dir, fi)
+ treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi)
if err != nil {
return FutureNode{}, err
}
@@ -329,7 +309,7 @@ func (fn *FutureNode) take(ctx context.Context) futureNodeResult {
}
case <-ctx.Done():
}
- return futureNodeResult{}
+ return futureNodeResult{err: errors.Errorf("no result")}
}
// allBlobsPresent checks if all blobs (contents) of the given node are
@@ -372,7 +352,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
debug.Log("lstat() for %v returned error: %v", target, err)
err = arch.error(abstarget, err)
if err != nil {
- return FutureNode{}, false, errors.Wrap(err, "Lstat")
+ return FutureNode{}, false, errors.WithStack(err)
}
return FutureNode{}, true, nil
}
@@ -384,7 +364,6 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
switch {
case fs.IsRegularFile(fi):
debug.Log(" %v regular file", target)
- start := time.Now()
// check if the file has not changed before performing a fopen operation (more expensive, specially
// in network filesystems)
@@ -392,8 +371,8 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
if arch.allBlobsPresent(previous) {
debug.Log("%v hasn't changed, using old list of blobs", target)
arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start))
- arch.CompleteBlob(snPath, previous.Size)
- node, err := arch.nodeFromFileInfo(target, fi)
+ arch.CompleteBlob(previous.Size)
+ node, err := arch.nodeFromFileInfo(snPath, target, fi)
if err != nil {
return FutureNode{}, false, err
}
@@ -425,7 +404,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
debug.Log("Openfile() for %v returned error: %v", target, err)
err = arch.error(abstarget, err)
if err != nil {
- return FutureNode{}, false, errors.Wrap(err, "Lstat")
+ return FutureNode{}, false, errors.WithStack(err)
}
return FutureNode{}, true, nil
}
@@ -436,14 +415,14 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
_ = file.Close()
err = arch.error(abstarget, err)
if err != nil {
- return FutureNode{}, false, errors.Wrap(err, "Lstat")
+ return FutureNode{}, false, errors.WithStack(err)
}
return FutureNode{}, true, nil
}
// make sure it's still a file
if !fs.IsRegularFile(fi) {
- err = errors.Errorf("file %v changed type, refusing to archive")
+ err = errors.Errorf("file %v changed type, refusing to archive", fi.Name())
_ = file.Close()
err = arch.error(abstarget, err)
if err != nil {
@@ -455,6 +434,8 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
// Save will close the file, we don't need to do that
fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
arch.StartFile(snPath)
+ }, func() {
+ arch.CompleteItem(snPath, nil, nil, ItemStats{}, 0)
}, func(node *restic.Node, stats ItemStats) {
arch.CompleteItem(snPath, previous, node, stats, time.Since(start))
})
@@ -463,7 +444,6 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
debug.Log(" %v dir", target)
snItem := snPath + "/"
- start := time.Now()
oldSubtree, err := arch.loadSubtree(ctx, previous)
if err != nil {
err = arch.error(abstarget, err)
@@ -488,7 +468,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
default:
debug.Log(" %v other", target)
- node, err := arch.nodeFromFileInfo(target, fi)
+ node, err := arch.nodeFromFileInfo(snPath, target, fi)
if err != nil {
return FutureNode{}, false, err
}
@@ -544,7 +524,7 @@ func join(elem ...string) string {
func (arch *Archiver) statDir(dir string) (os.FileInfo, error) {
fi, err := arch.FS.Stat(dir)
if err != nil {
- return nil, errors.Wrap(err, "Lstat")
+ return nil, errors.WithStack(err)
}
tpe := fi.Mode() & (os.ModeType | os.ModeCharDevice)
@@ -557,13 +537,32 @@ func (arch *Archiver) statDir(dir string) (os.FileInfo, error) {
// SaveTree stores a Tree in the repo, returned is the tree. snPath is the path
// within the current snapshot.
-func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree) (*restic.Tree, error) {
- debug.Log("%v (%v nodes), parent %v", snPath, len(atree.Nodes), previous)
+func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
- nodeNames := atree.NodeNames()
- tree := restic.NewTree(len(nodeNames))
+ var node *restic.Node
+ if snPath != "/" {
+ if atree.FileInfoPath == "" {
+ return FutureNode{}, 0, errors.Errorf("FileInfoPath for %v is empty", snPath)
+ }
- futureNodes := make(map[string]FutureNode)
+ fi, err := arch.statDir(atree.FileInfoPath)
+ if err != nil {
+ return FutureNode{}, 0, err
+ }
+
+ debug.Log("%v, dir node data loaded from %v", snPath, atree.FileInfoPath)
+ node, err = arch.nodeFromFileInfo(snPath, atree.FileInfoPath, fi)
+ if err != nil {
+ return FutureNode{}, 0, err
+ }
+ } else {
+ // fake root node
+ node = &restic.Node{}
+ }
+
+ debug.Log("%v (%v nodes), parent %v", snPath, len(atree.Nodes), previous)
+ nodeNames := atree.NodeNames()
+ nodes := make([]FutureNode, 0, len(nodeNames))
// iterate over the nodes of atree in lexicographic (=deterministic) order
for _, name := range nodeNames {
@@ -571,7 +570,7 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
// test if context has been cancelled
if ctx.Err() != nil {
- return nil, ctx.Err()
+ return FutureNode{}, 0, ctx.Err()
}
// this is a leaf node
@@ -584,15 +583,15 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
// ignore error
continue
}
- return nil, err
+ return FutureNode{}, 0, err
}
if err != nil {
- return nil, err
+ return FutureNode{}, 0, err
}
if !excluded {
- futureNodes[name] = fn
+ nodes = append(nodes, fn)
}
continue
}
@@ -606,92 +605,28 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
err = arch.error(join(snPath, name), err)
}
if err != nil {
- return nil, err
+ return FutureNode{}, 0, err
}
// not a leaf node, archive subtree
- subtree, err := arch.SaveTree(ctx, join(snPath, name), &subatree, oldSubtree)
- if err != nil {
- return nil, err
- }
-
- tb, err := restic.TreeToBuilder(subtree)
- if err != nil {
- return nil, err
- }
- id, nodeStats, err := arch.saveTree(ctx, tb)
- if err != nil {
- return nil, err
- }
-
- if subatree.FileInfoPath == "" {
- return nil, errors.Errorf("FileInfoPath for %v/%v is empty", snPath, name)
- }
-
- debug.Log("%v, saved subtree %v as %v", snPath, subtree, id.Str())
-
- fi, err := arch.statDir(subatree.FileInfoPath)
- if err != nil {
- return nil, err
- }
-
- debug.Log("%v, dir node data loaded from %v", snPath, subatree.FileInfoPath)
-
- node, err := arch.nodeFromFileInfo(subatree.FileInfoPath, fi)
- if err != nil {
- return nil, err
- }
-
- node.Name = name
- node.Subtree = &id
-
- err = tree.Insert(node)
- if err != nil {
- return nil, err
- }
-
- arch.CompleteItem(snItem, oldNode, node, nodeStats, time.Since(start))
- }
-
- debug.Log("waiting on %d nodes", len(futureNodes))
-
- // process all futures
- for name, fn := range futureNodes {
- fnr := fn.take(ctx)
-
- // return the error, or ignore it
- if fnr.err != nil {
- fnr.err = arch.error(fnr.target, fnr.err)
- if fnr.err == nil {
- // ignore error
- continue
- }
-
- return nil, fnr.err
- }
-
- // when the error is ignored, the node could not be saved, so ignore it
- if fnr.node == nil {
- debug.Log("%v excluded: %v", fnr.snPath, fnr.target)
- continue
- }
-
- fnr.node.Name = name
-
- err := tree.Insert(fnr.node)
+ fn, _, err := arch.SaveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) {
+ arch.CompleteItem(snItem, oldNode, n, is, time.Since(start))
+ })
if err != nil {
- return nil, err
+ return FutureNode{}, 0, err
}
+ nodes = append(nodes, fn)
}
- return tree, nil
+ fn := arch.treeSaver.Save(ctx, snPath, atree.FileInfoPath, node, nodes, complete)
+ return fn, len(nodes), nil
}
// flags are passed to fs.OpenFile. O_RDONLY is implied.
func readdirnames(filesystem fs.FS, dir string, flags int) ([]string, error) {
f, err := filesystem.OpenFile(dir, fs.O_RDONLY|flags, 0)
if err != nil {
- return nil, errors.Wrap(err, "Open")
+ return nil, errors.WithStack(err)
}
entries, err := f.Readdirnames(-1)
@@ -744,24 +679,17 @@ type SnapshotOptions struct {
Hostname string
Excludes []string
Time time.Time
- ParentSnapshot restic.ID
+ ParentSnapshot *restic.Snapshot
}
// loadParentTree loads a tree referenced by snapshot id. If id is null, nil is returned.
-func (arch *Archiver) loadParentTree(ctx context.Context, snapshotID restic.ID) *restic.Tree {
- if snapshotID.IsNull() {
- return nil
- }
-
- debug.Log("load parent snapshot %v", snapshotID)
- sn, err := restic.LoadSnapshot(ctx, arch.Repo, snapshotID)
- if err != nil {
- debug.Log("unable to load snapshot %v: %v", snapshotID, err)
+func (arch *Archiver) loadParentTree(ctx context.Context, sn *restic.Snapshot) *restic.Tree {
+ if sn == nil {
return nil
}
if sn.Tree == nil {
- debug.Log("snapshot %v has empty tree %v", snapshotID)
+ debug.Log("snapshot %v has empty tree %v", *sn.ID())
return nil
}
@@ -782,11 +710,11 @@ func (arch *Archiver) runWorkers(ctx context.Context, wg *errgroup.Group) {
arch.fileSaver = NewFileSaver(ctx, wg,
arch.blobSaver.Save,
arch.Repo.Config().ChunkerPolynomial,
- arch.Options.FileReadConcurrency, arch.Options.SaveBlobConcurrency)
+ arch.Options.ReadConcurrency, arch.Options.SaveBlobConcurrency)
arch.fileSaver.CompleteBlob = arch.CompleteBlob
arch.fileSaver.NodeFromFileInfo = arch.nodeFromFileInfo
- arch.treeSaver = NewTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.saveTree, arch.Error)
+ arch.treeSaver = NewTreeSaver(ctx, wg, arch.Options.SaveTreeConcurrency, arch.blobSaver.Save, arch.Error)
}
func (arch *Archiver) stopWorkers() {
@@ -819,27 +747,33 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
wg, wgCtx := errgroup.WithContext(wgUpCtx)
start := time.Now()
- var stats ItemStats
wg.Go(func() error {
arch.runWorkers(wgCtx, wg)
debug.Log("starting snapshot")
- tree, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot))
+ fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(n *restic.Node, is ItemStats) {
+ arch.CompleteItem("/", nil, nil, is, time.Since(start))
+ })
if err != nil {
return err
}
- if len(tree.Nodes) == 0 {
- return errors.New("snapshot is empty")
+ fnr := fn.take(wgCtx)
+ if fnr.err != nil {
+ return fnr.err
}
- tb, err := restic.TreeToBuilder(tree)
- if err != nil {
- return err
+ if wgCtx.Err() != nil {
+ return wgCtx.Err()
+ }
+
+ if nodeCount == 0 {
+ return errors.New("snapshot is empty")
}
- rootTreeID, stats, err = arch.saveTree(wgCtx, tb)
+
+ rootTreeID = *fnr.node.Subtree
arch.stopWorkers()
- return err
+ return nil
})
err = wg.Wait()
@@ -850,8 +784,6 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
return err
}
- arch.CompleteItem("/", nil, nil, stats, time.Since(start))
-
return arch.Repo.Flush(ctx)
})
err = wgUp.Wait()
@@ -865,9 +797,8 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
}
sn.Excludes = opts.Excludes
- if !opts.ParentSnapshot.IsNull() {
- id := opts.ParentSnapshot
- sn.Parent = &id
+ if opts.ParentSnapshot != nil {
+ sn.Parent = opts.ParentSnapshot.ID()
}
sn.Tree = &rootTreeID
diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go
index a6485234f..e3a850591 100644
--- a/internal/archiver/archiver_test.go
+++ b/internal/archiver/archiver_test.go
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"io"
- "io/ioutil"
"os"
"path/filepath"
"runtime"
@@ -26,18 +25,13 @@ import (
"golang.org/x/sync/errgroup"
)
-func prepareTempdirRepoSrc(t testing.TB, src TestDir) (tempdir string, repo restic.Repository, cleanup func()) {
- tempdir, removeTempdir := restictest.TempDir(t)
- repo, removeRepository := repository.TestRepository(t)
+func prepareTempdirRepoSrc(t testing.TB, src TestDir) (string, restic.Repository) {
+ tempdir := restictest.TempDir(t)
+ repo := repository.TestRepository(t)
TestCreateFiles(t, tempdir, src)
- cleanup = func() {
- removeRepository()
- removeTempdir()
- }
-
- return tempdir, repo, cleanup
+ return tempdir, repo
}
func saveFile(t testing.TB, repo restic.Repository, filename string, filesystem fs.FS) (*restic.Node, ItemStats) {
@@ -53,6 +47,8 @@ func saveFile(t testing.TB, repo restic.Repository, filename string, filesystem
}
var (
+ completeReadingCallback bool
+
completeCallbackNode *restic.Node
completeCallbackStats ItemStats
completeCallback bool
@@ -60,6 +56,13 @@ func saveFile(t testing.TB, repo restic.Repository, filename string, filesystem
startCallback bool
)
+ completeReading := func() {
+ completeReadingCallback = true
+ if completeCallback {
+ t.Error("callbacks called in wrong order")
+ }
+ }
+
complete := func(node *restic.Node, stats ItemStats) {
completeCallback = true
completeCallbackNode = node
@@ -80,7 +83,7 @@ func saveFile(t testing.TB, repo restic.Repository, filename string, filesystem
t.Fatal(err)
}
- res := arch.fileSaver.Save(ctx, "/", filename, file, fi, start, complete)
+ res := arch.fileSaver.Save(ctx, "/", filename, file, fi, start, completeReading, complete)
fnr := res.take(ctx)
if fnr.err != nil {
@@ -101,6 +104,10 @@ func saveFile(t testing.TB, repo restic.Repository, filename string, filesystem
t.Errorf("start callback did not happen")
}
+ if !completeReadingCallback {
+ t.Errorf("completeReading callback did not happen")
+ }
+
if !completeCallback {
t.Errorf("complete callback did not happen")
}
@@ -132,9 +139,7 @@ func TestArchiverSaveFile(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tempdir, repo, cleanup := prepareTempdirRepoSrc(t, TestDir{"file": testfile})
- defer cleanup()
-
+ tempdir, repo := prepareTempdirRepoSrc(t, TestDir{"file": testfile})
node, stats := saveFile(t, repo, filepath.Join(tempdir, "file"), fs.Track{FS: fs.Local{}})
TestEnsureFileContent(ctx, t, repo, "file", node, testfile)
@@ -167,8 +172,7 @@ func TestArchiverSaveFileReaderFS(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
ts := time.Now()
filename := "xx"
@@ -176,7 +180,7 @@ func TestArchiverSaveFileReaderFS(t *testing.T) {
ModTime: ts,
Mode: 0123,
Name: filename,
- ReadCloser: ioutil.NopCloser(strings.NewReader(test.Data)),
+ ReadCloser: io.NopCloser(strings.NewReader(test.Data)),
}
node, stats := saveFile(t, repo, filename, readerFs)
@@ -210,8 +214,7 @@ func TestArchiverSave(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tempdir, repo, cleanup := prepareTempdirRepoSrc(t, TestDir{"file": testfile})
- defer cleanup()
+ tempdir, repo := prepareTempdirRepoSrc(t, TestDir{"file": testfile})
wg, ctx := errgroup.WithContext(ctx)
repo.StartPackUploader(ctx, wg)
@@ -279,8 +282,7 @@ func TestArchiverSaveReaderFS(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
wg, ctx := errgroup.WithContext(ctx)
repo.StartPackUploader(ctx, wg)
@@ -291,7 +293,7 @@ func TestArchiverSaveReaderFS(t *testing.T) {
ModTime: ts,
Mode: 0123,
Name: filename,
- ReadCloser: ioutil.NopCloser(strings.NewReader(test.Data)),
+ ReadCloser: io.NopCloser(strings.NewReader(test.Data)),
}
arch := New(repo, readerFs, Options{})
@@ -355,7 +357,7 @@ func BenchmarkArchiverSaveFileSmall(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
- tempdir, repo, cleanup := prepareTempdirRepoSrc(b, d)
+ tempdir, repo := prepareTempdirRepoSrc(b, d)
b.StartTimer()
_, stats := saveFile(b, repo, filepath.Join(tempdir, "file"), fs.Track{FS: fs.Local{}})
@@ -373,7 +375,6 @@ func BenchmarkArchiverSaveFileSmall(b *testing.B) {
if stats.TreeBlobs != 0 {
b.Errorf("wrong stats returned in DataBlobs, want 0, got %d", stats.DataBlobs)
}
- cleanup()
b.StartTimer()
}
}
@@ -388,7 +389,7 @@ func BenchmarkArchiverSaveFileLarge(b *testing.B) {
for i := 0; i < b.N; i++ {
b.StopTimer()
- tempdir, repo, cleanup := prepareTempdirRepoSrc(b, d)
+ tempdir, repo := prepareTempdirRepoSrc(b, d)
b.StartTimer()
_, stats := saveFile(b, repo, filepath.Join(tempdir, "file"), fs.Track{FS: fs.Local{}})
@@ -406,7 +407,6 @@ func BenchmarkArchiverSaveFileLarge(b *testing.B) {
if stats.TreeBlobs != 0 {
b.Errorf("wrong stats returned in DataBlobs, want 0, got %d", stats.DataBlobs)
}
- cleanup()
b.StartTimer()
}
}
@@ -458,14 +458,10 @@ func appendToFile(t testing.TB, filename string, data []byte) {
}
func TestArchiverSaveFileIncremental(t *testing.T) {
- tempdir, removeTempdir := restictest.TempDir(t)
- defer removeTempdir()
-
- testRepo, removeRepository := repository.TestRepository(t)
- defer removeRepository()
+ tempdir := restictest.TempDir(t)
repo := &blobCountingRepo{
- Repository: testRepo,
+ Repository: repository.TestRepository(t),
saved: make(map[restic.BlobHandle]uint),
}
@@ -676,8 +672,7 @@ func TestFileChanged(t *testing.T) {
t.Skip("don't run test on Windows")
}
- tempdir, cleanup := restictest.TempDir(t)
- defer cleanup()
+ tempdir := restictest.TempDir(t)
filename := filepath.Join(tempdir, "file")
content := defaultContent
@@ -713,8 +708,7 @@ func TestFileChanged(t *testing.T) {
}
func TestFilChangedSpecialCases(t *testing.T) {
- tempdir, cleanup := restictest.TempDir(t)
- defer cleanup()
+ tempdir := restictest.TempDir(t)
filename := filepath.Join(tempdir, "file")
content := []byte("foobar")
@@ -829,8 +823,7 @@ func TestArchiverSaveDir(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
- tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src)
- defer cleanup()
+ tempdir, repo := prepareTempdirRepoSrc(t, test.src)
wg, ctx := errgroup.WithContext(context.Background())
repo.StartPackUploader(ctx, wg)
@@ -901,14 +894,10 @@ func TestArchiverSaveDir(t *testing.T) {
}
func TestArchiverSaveDirIncremental(t *testing.T) {
- tempdir, removeTempdir := restictest.TempDir(t)
- defer removeTempdir()
-
- testRepo, removeRepository := repository.TestRepository(t)
- defer removeRepository()
+ tempdir := restictest.TempDir(t)
repo := &blobCountingRepo{
- Repository: testRepo,
+ Repository: repository.TestRepository(t),
saved: make(map[restic.BlobHandle]uint),
}
@@ -1086,8 +1075,7 @@ func TestArchiverSaveTree(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
- tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src)
- defer cleanup()
+ tempdir, repo := prepareTempdirRepoSrc(t, test.src)
testFS := fs.Track{FS: fs.Local{}}
@@ -1118,16 +1106,18 @@ func TestArchiverSaveTree(t *testing.T) {
t.Fatal(err)
}
- tree, err := arch.SaveTree(ctx, "/", atree, nil)
+ fn, _, err := arch.SaveTree(ctx, "/", atree, nil, nil)
if err != nil {
t.Fatal(err)
}
- treeID, err := restic.SaveTree(ctx, repo, tree)
- if err != nil {
- t.Fatal(err)
+ fnr := fn.take(context.TODO())
+ if fnr.err != nil {
+ t.Fatal(fnr.err)
}
+ treeID := *fnr.node.Subtree
+
arch.stopWorkers()
err = repo.Flush(ctx)
if err != nil {
@@ -1386,8 +1376,7 @@ func TestArchiverSnapshot(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src)
- defer cleanup()
+ tempdir, repo := prepareTempdirRepoSrc(t, test.src)
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
@@ -1544,8 +1533,7 @@ func TestArchiverSnapshotSelect(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src)
- defer cleanup()
+ tempdir, repo := prepareTempdirRepoSrc(t, test.src)
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
arch.Select = test.selFn
@@ -1647,8 +1635,7 @@ func TestArchiverParent(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src)
- defer cleanup()
+ tempdir, repo := prepareTempdirRepoSrc(t, test.src)
testFS := &MockFS{
FS: fs.Track{FS: fs.Local{}},
@@ -1660,7 +1647,7 @@ func TestArchiverParent(t *testing.T) {
back := restictest.Chdir(t, tempdir)
defer back()
- _, firstSnapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
+ firstSnapshot, firstSnapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
if err != nil {
t.Fatal(err)
}
@@ -1688,7 +1675,7 @@ func TestArchiverParent(t *testing.T) {
opts := SnapshotOptions{
Time: time.Now(),
- ParentSnapshot: firstSnapshotID,
+ ParentSnapshot: firstSnapshot,
}
_, secondSnapshotID, err := arch.Snapshot(ctx, []string{"."}, opts)
if err != nil {
@@ -1814,8 +1801,7 @@ func TestArchiverErrorReporting(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src)
- defer cleanup()
+ tempdir, repo := prepareTempdirRepoSrc(t, test.src)
back := restictest.Chdir(t, tempdir)
defer back()
@@ -1859,10 +1845,6 @@ type noCancelBackend struct {
restic.Backend
}
-func (c *noCancelBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
- return c.Backend.Test(context.Background(), h)
-}
-
func (c *noCancelBackend) Remove(ctx context.Context, h restic.Handle) error {
return c.Backend.Remove(context.Background(), h)
}
@@ -1891,14 +1873,13 @@ func TestArchiverContextCanceled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
- tempdir, removeTempdir := restictest.TempDir(t)
+ tempdir := restictest.TempDir(t)
TestCreateFiles(t, tempdir, TestDir{
"targetfile": TestFile{Content: "foobar"},
})
- defer removeTempdir()
// Ensure that the archiver itself reports the canceled context and not just the backend
- repo, _ := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0)
+ repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0)
back := restictest.Chdir(t, tempdir)
defer back()
@@ -1954,7 +1935,7 @@ type failSaveRepo struct {
func (f *failSaveRepo) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error) {
val := atomic.AddInt32(&f.cnt, 1)
if val >= f.failAfter {
- return restic.ID{}, false, 0, f.err
+ return restic.Hash(buf), false, 0, f.err
}
return f.Repository.SaveBlob(ctx, t, buf, id, storeDuplicate)
@@ -1986,6 +1967,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
{
src: TestDir{
"dir": TestDir{
+ "file0": TestFile{Content: string(restictest.Random(0, 1024))},
"file1": TestFile{Content: string(restictest.Random(1, 1024))},
"file2": TestFile{Content: string(restictest.Random(2, 1024))},
"file3": TestFile{Content: string(restictest.Random(3, 1024))},
@@ -1998,15 +1980,15 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
},
},
wantOpen: map[string]uint{
+ filepath.FromSlash("dir/file0"): 1,
filepath.FromSlash("dir/file1"): 1,
filepath.FromSlash("dir/file2"): 1,
filepath.FromSlash("dir/file3"): 1,
- filepath.FromSlash("dir/file4"): 1,
filepath.FromSlash("dir/file8"): 0,
filepath.FromSlash("dir/file9"): 0,
},
- // fails four to six files were opened as the FileReadConcurrency allows for
- // two queued files
+ // fails after four to seven files were opened, as the ReadConcurrency allows for
+ // two queued files and SaveBlobConcurrency for one blob queued for saving.
failAfter: 4,
err: testErr,
},
@@ -2017,8 +1999,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tempdir, repo, cleanup := prepareTempdirRepoSrc(t, test.src)
- defer cleanup()
+ tempdir, repo := prepareTempdirRepoSrc(t, test.src)
back := restictest.Chdir(t, tempdir)
defer back()
@@ -2040,7 +2021,8 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
// at most two files may be queued
arch := New(testRepo, testFS, Options{
- FileReadConcurrency: 2,
+ ReadConcurrency: 2,
+ SaveBlobConcurrency: 1,
})
_, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
@@ -2061,7 +2043,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
}
}
-func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent restic.ID, filename string) (restic.ID, *restic.Node) {
+func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent *restic.Snapshot, filename string) (*restic.Snapshot, *restic.Node) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -2071,7 +2053,7 @@ func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent restic.ID,
Time: time.Now(),
ParentSnapshot: parent,
}
- snapshot, snapshotID, err := arch.Snapshot(ctx, []string{filename}, sopts)
+ snapshot, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
if err != nil {
t.Fatal(err)
}
@@ -2086,7 +2068,7 @@ func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent restic.ID,
t.Fatalf("unable to find node for testfile in snapshot")
}
- return snapshotID, node
+ return snapshot, node
}
// StatFS allows overwriting what is returned by the Lstat function.
@@ -2148,8 +2130,7 @@ func TestMetadataChanged(t *testing.T) {
},
}
- tempdir, repo, cleanup := prepareTempdirRepoSrc(t, files)
- defer cleanup()
+ tempdir, repo := prepareTempdirRepoSrc(t, files)
back := restictest.Chdir(t, tempdir)
defer back()
@@ -2168,7 +2149,7 @@ func TestMetadataChanged(t *testing.T) {
},
}
- snapshotID, node2 := snapshot(t, repo, fs, restic.ID{}, "testfile")
+ sn, node2 := snapshot(t, repo, fs, nil, "testfile")
// set some values so we can then compare the nodes
want.Content = node2.Content
@@ -2199,7 +2180,7 @@ func TestMetadataChanged(t *testing.T) {
want.Group = ""
// make another snapshot
- _, node3 := snapshot(t, repo, fs, snapshotID, "testfile")
+ _, node3 := snapshot(t, repo, fs, sn, "testfile")
// Override username and group to empty string - in case underlying system has user with UID 51234
// See https://github.com/restic/restic/issues/2372
node3.User = ""
@@ -2223,8 +2204,7 @@ func TestRacyFileSwap(t *testing.T) {
},
}
- tempdir, repo, cleanup := prepareTempdirRepoSrc(t, files)
- defer cleanup()
+ tempdir, repo := prepareTempdirRepoSrc(t, files)
back := restictest.Chdir(t, tempdir)
defer back()
diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go
index b2b5e59bb..ae4879ff4 100644
--- a/internal/archiver/blob_saver.go
+++ b/internal/archiver/blob_saver.go
@@ -43,51 +43,18 @@ func (s *BlobSaver) TriggerShutdown() {
// Save stores a blob in the repo. It checks the index and the known blobs
// before saving anything. It takes ownership of the buffer passed in.
-func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer) FutureBlob {
- ch := make(chan SaveBlobResponse, 1)
+func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)) {
select {
- case s.ch <- saveBlobJob{BlobType: t, buf: buf, ch: ch}:
+ case s.ch <- saveBlobJob{BlobType: t, buf: buf, cb: cb}:
case <-ctx.Done():
debug.Log("not sending job, context is cancelled")
- close(ch)
- return FutureBlob{ch: ch}
}
-
- return FutureBlob{ch: ch}
-}
-
-// FutureBlob is returned by SaveBlob and will return the data once it has been processed.
-type FutureBlob struct {
- ch <-chan SaveBlobResponse
-}
-
-func (s *FutureBlob) Poll() *SaveBlobResponse {
- select {
- case res, ok := <-s.ch:
- if ok {
- return &res
- }
- default:
- }
- return nil
-}
-
-// Take blocks until the result is available or the context is cancelled.
-func (s *FutureBlob) Take(ctx context.Context) SaveBlobResponse {
- select {
- case res, ok := <-s.ch:
- if ok {
- return res
- }
- case <-ctx.Done():
- }
- return SaveBlobResponse{}
}
type saveBlobJob struct {
restic.BlobType
buf *Buffer
- ch chan<- SaveBlobResponse
+ cb func(res SaveBlobResponse)
}
type SaveBlobResponse struct {
@@ -128,11 +95,9 @@ func (s *BlobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error {
res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data)
if err != nil {
debug.Log("saveBlob returned error, exiting: %v", err)
- close(job.ch)
return err
}
- job.ch <- res
- close(job.ch)
+ job.cb(res)
job.buf.Release()
}
}
diff --git a/internal/archiver/blob_saver_test.go b/internal/archiver/blob_saver_test.go
index 481139a3f..367b7be8b 100644
--- a/internal/archiver/blob_saver_test.go
+++ b/internal/archiver/blob_saver_test.go
@@ -4,11 +4,12 @@ import (
"context"
"fmt"
"runtime"
+ "sync"
"sync/atomic"
"testing"
"github.com/restic/restic/internal/errors"
- "github.com/restic/restic/internal/repository"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/restic"
"golang.org/x/sync/errgroup"
)
@@ -40,21 +41,32 @@ func TestBlobSaver(t *testing.T) {
wg, ctx := errgroup.WithContext(ctx)
saver := &saveFail{
- idx: repository.NewMasterIndex(),
+ idx: index.NewMasterIndex(),
}
b := NewBlobSaver(ctx, wg, saver, uint(runtime.NumCPU()))
- var results []FutureBlob
+ var wait sync.WaitGroup
+ var results []SaveBlobResponse
+ var lock sync.Mutex
+ wait.Add(20)
for i := 0; i < 20; i++ {
buf := &Buffer{Data: []byte(fmt.Sprintf("foo%d", i))}
- fb := b.Save(ctx, restic.DataBlob, buf)
- results = append(results, fb)
+ idx := i
+ lock.Lock()
+ results = append(results, SaveBlobResponse{})
+ lock.Unlock()
+ b.Save(ctx, restic.DataBlob, buf, func(res SaveBlobResponse) {
+ lock.Lock()
+ results[idx] = res
+ lock.Unlock()
+ wait.Done()
+ })
}
- for i, blob := range results {
- sbr := blob.Take(ctx)
+ wait.Wait()
+ for i, sbr := range results {
if sbr.known {
t.Errorf("blob %v is known, that should not be the case", i)
}
@@ -86,7 +98,7 @@ func TestBlobSaverError(t *testing.T) {
wg, ctx := errgroup.WithContext(ctx)
saver := &saveFail{
- idx: repository.NewMasterIndex(),
+ idx: index.NewMasterIndex(),
failAt: int32(test.failAt),
}
@@ -94,7 +106,7 @@ func TestBlobSaverError(t *testing.T) {
for i := 0; i < test.blobs; i++ {
buf := &Buffer{Data: []byte(fmt.Sprintf("foo%d", i))}
- b.Save(ctx, restic.DataBlob, buf)
+ b.Save(ctx, restic.DataBlob, buf, func(res SaveBlobResponse) {})
}
b.TriggerShutdown()
diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go
index 52dd59113..0742c8b57 100644
--- a/internal/archiver/file_saver.go
+++ b/internal/archiver/file_saver.go
@@ -4,6 +4,7 @@ import (
"context"
"io"
"os"
+ "sync"
"github.com/restic/chunker"
"github.com/restic/restic/internal/debug"
@@ -14,7 +15,7 @@ import (
)
// SaveBlobFn saves a blob to a repo.
-type SaveBlobFn func(context.Context, restic.BlobType, *Buffer) FutureBlob
+type SaveBlobFn func(context.Context, restic.BlobType, *Buffer, func(res SaveBlobResponse))
// FileSaver concurrently saves incoming files to the repo.
type FileSaver struct {
@@ -25,9 +26,9 @@ type FileSaver struct {
ch chan<- saveFileJob
- CompleteBlob func(filename string, bytes uint64)
+ CompleteBlob func(bytes uint64)
- NodeFromFileInfo func(filename string, fi os.FileInfo) (*restic.Node, error)
+ NodeFromFileInfo func(snPath, filename string, fi os.FileInfo) (*restic.Node, error)
}
// NewFileSaver returns a new file saver. A worker pool with fileWorkers is
@@ -45,7 +46,7 @@ func NewFileSaver(ctx context.Context, wg *errgroup.Group, save SaveBlobFn, pol
pol: pol,
ch: ch,
- CompleteBlob: func(string, uint64) {},
+ CompleteBlob: func(uint64) {},
}
for i := uint(0); i < fileWorkers; i++ {
@@ -66,17 +67,21 @@ func (s *FileSaver) TriggerShutdown() {
type CompleteFunc func(*restic.Node, ItemStats)
// Save stores the file f and returns the data once it has been completed. The
-// file is closed by Save.
-func (s *FileSaver) Save(ctx context.Context, snPath string, target string, file fs.File, fi os.FileInfo, start func(), complete CompleteFunc) FutureNode {
+// file is closed by Save. completeReading is only called if the file was read
+// successfully. complete is always called. If completeReading is called, then
+// this will always happen before calling complete.
+func (s *FileSaver) Save(ctx context.Context, snPath string, target string, file fs.File, fi os.FileInfo, start func(), completeReading func(), complete CompleteFunc) FutureNode {
fn, ch := newFutureNode()
job := saveFileJob{
- snPath: snPath,
- target: target,
- file: file,
- fi: fi,
- start: start,
- complete: complete,
- ch: ch,
+ snPath: snPath,
+ target: target,
+ file: file,
+ fi: fi,
+ ch: ch,
+
+ start: start,
+ completeReading: completeReading,
+ complete: complete,
}
select {
@@ -91,56 +96,84 @@ func (s *FileSaver) Save(ctx context.Context, snPath string, target string, file
}
type saveFileJob struct {
- snPath string
- target string
- file fs.File
- fi os.FileInfo
- ch chan<- futureNodeResult
- complete CompleteFunc
- start func()
+ snPath string
+ target string
+ file fs.File
+ fi os.FileInfo
+ ch chan<- futureNodeResult
+
+ start func()
+ completeReading func()
+ complete CompleteFunc
}
// saveFile stores the file f in the repo, then closes it.
-func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPath string, target string, f fs.File, fi os.FileInfo, start func()) futureNodeResult {
+func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPath string, target string, f fs.File, fi os.FileInfo, start func(), finishReading func(), finish func(res futureNodeResult)) {
start()
- stats := ItemStats{}
fnr := futureNodeResult{
snPath: snPath,
target: target,
}
+ var lock sync.Mutex
+ remaining := 0
+ isCompleted := false
+
+ completeBlob := func() {
+ lock.Lock()
+ defer lock.Unlock()
+
+ remaining--
+ if remaining == 0 && fnr.err == nil {
+ if isCompleted {
+ panic("completed twice")
+ }
+ for _, id := range fnr.node.Content {
+ if id.IsNull() {
+ panic("completed file with null ID")
+ }
+ }
+ isCompleted = true
+ finish(fnr)
+ }
+ }
+ completeError := func(err error) {
+ lock.Lock()
+ defer lock.Unlock()
+
+ if fnr.err == nil {
+ if isCompleted {
+ panic("completed twice")
+ }
+ isCompleted = true
+ fnr.err = err
+ fnr.node = nil
+ fnr.stats = ItemStats{}
+ finish(fnr)
+ }
+ }
debug.Log("%v", snPath)
- node, err := s.NodeFromFileInfo(f.Name(), fi)
+ node, err := s.NodeFromFileInfo(snPath, f.Name(), fi)
if err != nil {
_ = f.Close()
- fnr.err = err
- return fnr
+ completeError(err)
+ return
}
if node.Type != "file" {
_ = f.Close()
- fnr.err = errors.Errorf("node type %q is wrong", node.Type)
- return fnr
+ completeError(errors.Errorf("node type %q is wrong", node.Type))
+ return
}
// reuse the chunker
chnker.Reset(f, s.pol)
- var results []FutureBlob
- complete := func(sbr SaveBlobResponse) {
- if !sbr.known {
- stats.DataBlobs++
- stats.DataSize += uint64(sbr.length)
- stats.DataSizeInRepo += uint64(sbr.sizeInRepo)
- }
-
- node.Content = append(node.Content, sbr.id)
- }
-
node.Content = []restic.ID{}
- var size uint64
+ node.Size = 0
+ var idx int
for {
buf := s.saveFilePool.Get()
chunk, err := chnker.Next(buf.Data)
@@ -150,62 +183,66 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat
}
buf.Data = chunk.Data
-
- size += uint64(chunk.Length)
+ node.Size += uint64(chunk.Length)
if err != nil {
_ = f.Close()
- fnr.err = err
- return fnr
+ completeError(err)
+ return
}
-
// test if the context has been cancelled, return the error
if ctx.Err() != nil {
_ = f.Close()
- fnr.err = ctx.Err()
- return fnr
+ completeError(ctx.Err())
+ return
}
- res := s.saveBlob(ctx, restic.DataBlob, buf)
- results = append(results, res)
+ // add a place to store the saveBlob result
+ pos := idx
+
+ lock.Lock()
+ node.Content = append(node.Content, restic.ID{})
+ lock.Unlock()
+
+ s.saveBlob(ctx, restic.DataBlob, buf, func(sbr SaveBlobResponse) {
+ lock.Lock()
+ if !sbr.known {
+ fnr.stats.DataBlobs++
+ fnr.stats.DataSize += uint64(sbr.length)
+ fnr.stats.DataSizeInRepo += uint64(sbr.sizeInRepo)
+ }
+
+ node.Content[pos] = sbr.id
+ lock.Unlock()
+
+ completeBlob()
+ })
+ idx++
// test if the context has been cancelled, return the error
if ctx.Err() != nil {
_ = f.Close()
- fnr.err = ctx.Err()
- return fnr
+ completeError(ctx.Err())
+ return
}
- s.CompleteBlob(f.Name(), uint64(len(chunk.Data)))
-
- // collect already completed blobs
- for len(results) > 0 {
- sbr := results[0].Poll()
- if sbr == nil {
- break
- }
- results[0] = FutureBlob{}
- results = results[1:]
- complete(*sbr)
- }
+ s.CompleteBlob(uint64(len(chunk.Data)))
}
err = f.Close()
if err != nil {
- fnr.err = err
- return fnr
- }
-
- for i, res := range results {
- results[i] = FutureBlob{}
- sbr := res.Take(ctx)
- complete(sbr)
+ completeError(err)
+ return
}
- node.Size = size
fnr.node = node
- fnr.stats = stats
- return fnr
+ lock.Lock()
+ // require one additional completeFuture() call to ensure that the future only completes
+ // after reaching the end of this method
+ remaining += idx + 1
+ lock.Unlock()
+ finishReading()
+ completeBlob()
}
func (s *FileSaver) worker(ctx context.Context, jobs <-chan saveFileJob) {
@@ -224,11 +261,16 @@ func (s *FileSaver) worker(ctx context.Context, jobs <-chan saveFileJob) {
}
}
- res := s.saveFile(ctx, chnker, job.snPath, job.target, job.file, job.fi, job.start)
- if job.complete != nil {
- job.complete(res.node, res.stats)
- }
- job.ch <- res
- close(job.ch)
+ s.saveFile(ctx, chnker, job.snPath, job.target, job.file, job.fi, job.start, func() {
+ if job.completeReading != nil {
+ job.completeReading()
+ }
+ }, func(res futureNodeResult) {
+ if job.complete != nil {
+ job.complete(res.node, res.stats)
+ }
+ job.ch <- res
+ close(job.ch)
+ })
}
}
diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go
index e4d1dcdb8..5c4472c62 100644
--- a/internal/archiver/file_saver_test.go
+++ b/internal/archiver/file_saver_test.go
@@ -3,7 +3,7 @@ package archiver
import (
"context"
"fmt"
- "io/ioutil"
+ "os"
"path/filepath"
"runtime"
"testing"
@@ -15,28 +15,31 @@ import (
"golang.org/x/sync/errgroup"
)
-func createTestFiles(t testing.TB, num int) (files []string, cleanup func()) {
- tempdir, cleanup := test.TempDir(t)
+func createTestFiles(t testing.TB, num int) (files []string) {
+ tempdir := test.TempDir(t)
for i := 0; i < 15; i++ {
filename := fmt.Sprintf("testfile-%d", i)
- err := ioutil.WriteFile(filepath.Join(tempdir, filename), []byte(filename), 0600)
+ err := os.WriteFile(filepath.Join(tempdir, filename), []byte(filename), 0600)
if err != nil {
t.Fatal(err)
}
files = append(files, filepath.Join(tempdir, filename))
}
- return files, cleanup
+ return files
}
func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Context, *errgroup.Group) {
wg, ctx := errgroup.WithContext(ctx)
- saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *Buffer) FutureBlob {
- ch := make(chan SaveBlobResponse)
- close(ch)
- return FutureBlob{ch: ch}
+ saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *Buffer, cb func(SaveBlobResponse)) {
+ cb(SaveBlobResponse{
+ id: restic.Hash(buf.Data),
+ length: len(buf.Data),
+ sizeInRepo: len(buf.Data),
+ known: false,
+ })
}
workers := uint(runtime.NumCPU())
@@ -46,7 +49,9 @@ func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Cont
}
s := NewFileSaver(ctx, wg, saveBlob, pol, workers, workers)
- s.NodeFromFileInfo = restic.NodeFromFileInfo
+ s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo) (*restic.Node, error) {
+ return restic.NodeFromFileInfo(filename, fi)
+ }
return s, ctx, wg
}
@@ -55,10 +60,10 @@ func TestFileSaver(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- files, cleanup := createTestFiles(t, 15)
- defer cleanup()
+ files := createTestFiles(t, 15)
startFn := func() {}
+ completeReadingFn := func() {}
completeFn := func(*restic.Node, ItemStats) {}
testFs := fs.Local{}
@@ -77,7 +82,7 @@ func TestFileSaver(t *testing.T) {
t.Fatal(err)
}
- ff := s.Save(ctx, filename, filename, f, fi, startFn, completeFn)
+ ff := s.Save(ctx, filename, filename, f, fi, startFn, completeReadingFn, completeFn)
results = append(results, ff)
}
diff --git a/internal/archiver/scanner_test.go b/internal/archiver/scanner_test.go
index 87d8c887d..1b4cd1f7f 100644
--- a/internal/archiver/scanner_test.go
+++ b/internal/archiver/scanner_test.go
@@ -81,9 +81,7 @@ func TestScanner(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tempdir, cleanup := restictest.TempDir(t)
- defer cleanup()
-
+ tempdir := restictest.TempDir(t)
TestCreateFiles(t, tempdir, test.src)
back := restictest.Chdir(t, tempdir)
@@ -218,9 +216,7 @@ func TestScannerError(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tempdir, cleanup := restictest.TempDir(t)
- defer cleanup()
-
+ tempdir := restictest.TempDir(t)
TestCreateFiles(t, tempdir, test.src)
back := restictest.Chdir(t, tempdir)
@@ -292,9 +288,7 @@ func TestScannerCancel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tempdir, cleanup := restictest.TempDir(t)
- defer cleanup()
-
+ tempdir := restictest.TempDir(t)
TestCreateFiles(t, tempdir, src)
back := restictest.Chdir(t, tempdir)
diff --git a/internal/archiver/testing.go b/internal/archiver/testing.go
index 35a2d2933..c7482d160 100644
--- a/internal/archiver/testing.go
+++ b/internal/archiver/testing.go
@@ -2,7 +2,6 @@ package archiver
import (
"context"
- "io/ioutil"
"os"
"path"
"path/filepath"
@@ -26,7 +25,11 @@ func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *res
Tags: []string{"test"},
}
if parent != nil {
- opts.ParentSnapshot = *parent
+ sn, err := restic.LoadSnapshot(context.TODO(), arch.Repo, *parent)
+ if err != nil {
+ t.Fatal(err)
+ }
+ opts.ParentSnapshot = sn
}
sn, _, err := arch.Snapshot(context.TODO(), []string{path}, opts)
if err != nil {
@@ -69,15 +72,11 @@ func TestCreateFiles(t testing.TB, target string, dir TestDir) {
switch it := item.(type) {
case TestFile:
- err := ioutil.WriteFile(targetPath, []byte(it.Content), 0644)
+ err := os.WriteFile(targetPath, []byte(it.Content), 0644)
if err != nil {
t.Fatal(err)
}
case TestSymlink:
- if runtime.GOOS == "windows" {
- continue
- }
-
err := fs.Symlink(filepath.FromSlash(it.Target), targetPath)
if err != nil {
t.Fatal(err)
@@ -135,16 +134,6 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) {
// first, test that all items are there
TestWalkFiles(t, target, dir, func(path string, item interface{}) error {
- // ignore symlinks on Windows
- if _, ok := item.(TestSymlink); ok && runtime.GOOS == "windows" {
- // mark paths and parents as checked
- pathsChecked[path] = struct{}{}
- for parent := filepath.Dir(path); parent != target; parent = filepath.Dir(parent) {
- pathsChecked[parent] = struct{}{}
- }
- return nil
- }
-
fi, err := fs.Lstat(path)
if err != nil {
return err
@@ -162,7 +151,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) {
return nil
}
- content, err := ioutil.ReadFile(path)
+ content, err := os.ReadFile(path)
if err != nil {
return err
}
@@ -294,10 +283,6 @@ func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo resti
}
TestEnsureFileContent(ctx, t, repo, nodePrefix, node, e)
case TestSymlink:
- // skip symlinks on windows
- if runtime.GOOS == "windows" {
- continue
- }
if node.Type != "symlink" {
t.Errorf("tree node %v has wrong type %q, want %q", nodePrefix, node.Type, "file")
}
@@ -309,12 +294,6 @@ func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo resti
}
for name := range dir {
- // skip checking symlinks on Windows
- entry := dir[name]
- if _, ok := entry.(TestSymlink); ok && runtime.GOOS == "windows" {
- continue
- }
-
_, ok := checked[name]
if !ok {
t.Errorf("tree %v: expected node %q not found, has: %v", prefix, name, nodeNames)
diff --git a/internal/archiver/testing_test.go b/internal/archiver/testing_test.go
index e11b86250..ada7261f1 100644
--- a/internal/archiver/testing_test.go
+++ b/internal/archiver/testing_test.go
@@ -3,10 +3,8 @@ package archiver
import (
"context"
"fmt"
- "io/ioutil"
"os"
"path/filepath"
- "runtime"
"testing"
"time"
@@ -63,15 +61,11 @@ func createFilesAt(t testing.TB, targetdir string, files map[string]interface{})
switch it := item.(type) {
case TestFile:
- err := ioutil.WriteFile(target, []byte(it.Content), 0600)
+ err := os.WriteFile(target, []byte(it.Content), 0600)
if err != nil {
t.Fatal(err)
}
case TestSymlink:
- // ignore symlinks on windows
- if runtime.GOOS == "windows" {
- continue
- }
err := fs.Symlink(filepath.FromSlash(it.Target), target)
if err != nil {
t.Fatal(err)
@@ -93,7 +87,7 @@ func TestTestCreateFiles(t *testing.T) {
},
"sub": TestDir{
"subsub": TestDir{
- "link": TestSymlink{Target: "x/y/z"},
+ "link": TestSymlink{Target: filepath.Clean("x/y/z")},
},
},
},
@@ -101,14 +95,13 @@ func TestTestCreateFiles(t *testing.T) {
"foo": TestFile{Content: "foo"},
"subdir": TestDir{},
"subdir/subfile": TestFile{Content: "bar"},
- "sub/subsub/link": TestSymlink{Target: "x/y/z"},
+ "sub/subsub/link": TestSymlink{Target: filepath.Clean("x/y/z")},
},
},
}
for i, test := range tests {
- tempdir, cleanup := restictest.TempDir(t)
- defer cleanup()
+ tempdir := restictest.TempDir(t)
t.Run("", func(t *testing.T) {
tempdir := filepath.Join(tempdir, fmt.Sprintf("test-%d", i))
@@ -120,13 +113,6 @@ func TestTestCreateFiles(t *testing.T) {
TestCreateFiles(t, tempdir, test.dir)
for name, item := range test.files {
- // don't check symlinks on windows
- if runtime.GOOS == "windows" {
- if _, ok := item.(TestSymlink); ok {
- continue
- }
- }
-
targetPath := filepath.Join(tempdir, filepath.FromSlash(name))
fi, err := fs.Lstat(targetPath)
if err != nil {
@@ -141,7 +127,7 @@ func TestTestCreateFiles(t *testing.T) {
continue
}
- content, err := ioutil.ReadFile(targetPath)
+ content, err := os.ReadFile(targetPath)
if err != nil {
t.Error(err)
continue
@@ -205,8 +191,7 @@ func TestTestWalkFiles(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
- tempdir, cleanup := restictest.TempDir(t)
- defer cleanup()
+ tempdir := restictest.TempDir(t)
got := make(map[string]string)
@@ -217,7 +202,7 @@ func TestTestWalkFiles(t *testing.T) {
return err
}
- got[p] = fmt.Sprintf("%v", item)
+ got[p] = fmt.Sprint(item)
return nil
})
@@ -233,13 +218,12 @@ func TestTestEnsureFiles(t *testing.T) {
expectFailure bool
files map[string]interface{}
want TestDir
- unixOnly bool
}{
{
files: map[string]interface{}{
"foo": TestFile{Content: "foo"},
"subdir/subfile": TestFile{Content: "bar"},
- "x/y/link": TestSymlink{Target: "../../foo"},
+ "x/y/link": TestSymlink{Target: filepath.Clean("../../foo")},
},
want: TestDir{
"foo": TestFile{Content: "foo"},
@@ -248,7 +232,7 @@ func TestTestEnsureFiles(t *testing.T) {
},
"x": TestDir{
"y": TestDir{
- "link": TestSymlink{Target: "../../foo"},
+ "link": TestSymlink{Target: filepath.Clean("../../foo")},
},
},
},
@@ -295,7 +279,6 @@ func TestTestEnsureFiles(t *testing.T) {
},
{
expectFailure: true,
- unixOnly: true,
files: map[string]interface{}{
"foo": TestFile{Content: "foo"},
},
@@ -305,7 +288,6 @@ func TestTestEnsureFiles(t *testing.T) {
},
{
expectFailure: true,
- unixOnly: true,
files: map[string]interface{}{
"foo": TestSymlink{Target: "xxx"},
},
@@ -339,14 +321,7 @@ func TestTestEnsureFiles(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
- if test.unixOnly && runtime.GOOS == "windows" {
- t.Skip("skip on Windows")
- return
- }
-
- tempdir, cleanup := restictest.TempDir(t)
- defer cleanup()
-
+ tempdir := restictest.TempDir(t)
createFilesAt(t, tempdir, test.files)
subtestT := testing.TB(t)
@@ -368,7 +343,6 @@ func TestTestEnsureSnapshot(t *testing.T) {
expectFailure bool
files map[string]interface{}
want TestDir
- unixOnly bool
}{
{
files: map[string]interface{}{
@@ -451,7 +425,6 @@ func TestTestEnsureSnapshot(t *testing.T) {
},
{
expectFailure: true,
- unixOnly: true,
files: map[string]interface{}{
"foo": TestSymlink{Target: filepath.FromSlash("x/y/z")},
},
@@ -476,16 +449,10 @@ func TestTestEnsureSnapshot(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
- if test.unixOnly && runtime.GOOS == "windows" {
- t.Skip("skip on Windows")
- return
- }
-
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tempdir, cleanup := restictest.TempDir(t)
- defer cleanup()
+ tempdir := restictest.TempDir(t)
targetDir := filepath.Join(tempdir, "target")
err := fs.Mkdir(targetDir, 0700)
@@ -498,8 +465,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
back := restictest.Chdir(t, tempdir)
defer back()
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
arch := New(repo, fs.Local{}, Options{})
opts := SnapshotOptions{
diff --git a/internal/archiver/tree_saver.go b/internal/archiver/tree_saver.go
index 5aab09b94..d25781b03 100644
--- a/internal/archiver/tree_saver.go
+++ b/internal/archiver/tree_saver.go
@@ -2,6 +2,7 @@ package archiver
import (
"context"
+ "errors"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
@@ -10,7 +11,7 @@ import (
// TreeSaver concurrently saves incoming trees to the repo.
type TreeSaver struct {
- saveTree func(context.Context, *restic.TreeJSONBuilder) (restic.ID, ItemStats, error)
+ saveBlob func(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse))
errFn ErrorFunc
ch chan<- saveTreeJob
@@ -18,12 +19,12 @@ type TreeSaver struct {
// NewTreeSaver returns a new tree saver. A worker pool with treeWorkers is
// started, it is stopped when ctx is cancelled.
-func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveTree func(context.Context, *restic.TreeJSONBuilder) (restic.ID, ItemStats, error), errFn ErrorFunc) *TreeSaver {
+func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob func(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)), errFn ErrorFunc) *TreeSaver {
ch := make(chan saveTreeJob)
s := &TreeSaver{
ch: ch,
- saveTree: saveTree,
+ saveBlob: saveBlob,
errFn: errFn,
}
@@ -79,6 +80,7 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I
job.nodes = nil
builder := restic.NewTreeJSONBuilder()
+ var lastNode *restic.Node
for i, fn := range nodes {
// fn is a copy, so clear the original value explicitly
@@ -105,19 +107,41 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I
debug.Log("insert %v", fnr.node.Name)
err := builder.AddNode(fnr.node)
+ if err != nil && errors.Is(err, restic.ErrTreeNotOrdered) && lastNode != nil && fnr.node.Equals(*lastNode) {
+ // ignore error if an _identical_ node already exists, but nevertheless issue a warning
+ _ = s.errFn(fnr.target, err)
+ err = nil
+ }
if err != nil {
return nil, stats, err
}
+ lastNode = fnr.node
}
- id, treeStats, err := s.saveTree(ctx, builder)
- stats.Add(treeStats)
+ buf, err := builder.Finalize()
if err != nil {
return nil, stats, err
}
- node.Subtree = &id
- return node, stats, nil
+ b := &Buffer{Data: buf}
+ ch := make(chan SaveBlobResponse, 1)
+ s.saveBlob(ctx, restic.TreeBlob, b, func(res SaveBlobResponse) {
+ ch <- res
+ })
+
+ select {
+ case sbr := <-ch:
+ if !sbr.known {
+ stats.TreeBlobs++
+ stats.TreeSize += uint64(sbr.length)
+ stats.TreeSizeInRepo += uint64(sbr.sizeInRepo)
+ }
+
+ node.Subtree = &sbr.id
+ return node, stats, nil
+ case <-ctx.Done():
+ return nil, stats, ctx.Err()
+ }
}
func (s *TreeSaver) worker(ctx context.Context, jobs <-chan saveTreeJob) error {
diff --git a/internal/archiver/tree_saver_test.go b/internal/archiver/tree_saver_test.go
index 36e585ae1..7cc53346c 100644
--- a/internal/archiver/tree_saver_test.go
+++ b/internal/archiver/tree_saver_test.go
@@ -4,29 +4,44 @@ import (
"context"
"fmt"
"runtime"
- "sync/atomic"
"testing"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
+ "github.com/restic/restic/internal/test"
"golang.org/x/sync/errgroup"
)
-func TestTreeSaver(t *testing.T) {
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
+func treeSaveHelper(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)) {
+ cb(SaveBlobResponse{
+ id: restic.NewRandomID(),
+ known: false,
+ length: len(buf.Data),
+ sizeInRepo: len(buf.Data),
+ })
+}
+func setupTreeSaver() (context.Context, context.CancelFunc, *TreeSaver, func() error) {
+ ctx, cancel := context.WithCancel(context.Background())
wg, ctx := errgroup.WithContext(ctx)
- saveFn := func(context.Context, *restic.TreeJSONBuilder) (restic.ID, ItemStats, error) {
- return restic.NewRandomID(), ItemStats{TreeBlobs: 1, TreeSize: 123}, nil
+ errFn := func(snPath string, err error) error {
+ return err
}
- errFn := func(snPath string, err error) error {
- return nil
+ b := NewTreeSaver(ctx, wg, uint(runtime.NumCPU()), treeSaveHelper, errFn)
+
+ shutdown := func() error {
+ b.TriggerShutdown()
+ return wg.Wait()
}
- b := NewTreeSaver(ctx, wg, uint(runtime.NumCPU()), saveFn, errFn)
+ return ctx, cancel, b, shutdown
+}
+
+func TestTreeSaver(t *testing.T) {
+ ctx, cancel, b, shutdown := setupTreeSaver()
+ defer cancel()
var results []FutureNode
@@ -35,7 +50,7 @@ func TestTreeSaver(t *testing.T) {
Name: fmt.Sprintf("file-%d", i),
}
- fb := b.Save(ctx, "/", node.Name, node, nil, nil)
+ fb := b.Save(ctx, join("/", node.Name), node.Name, node, nil, nil)
results = append(results, fb)
}
@@ -43,9 +58,7 @@ func TestTreeSaver(t *testing.T) {
tree.take(ctx)
}
- b.TriggerShutdown()
-
- err := wg.Wait()
+ err := shutdown()
if err != nil {
t.Fatal(err)
}
@@ -54,7 +67,7 @@ func TestTreeSaver(t *testing.T) {
func TestTreeSaverError(t *testing.T) {
var tests = []struct {
trees int
- failAt int32
+ failAt int
}{
{1, 1},
{20, 2},
@@ -67,36 +80,27 @@ func TestTreeSaverError(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
- ctx, cancel := context.WithCancel(context.Background())
+ ctx, cancel, b, shutdown := setupTreeSaver()
defer cancel()
- wg, ctx := errgroup.WithContext(ctx)
-
- var num int32
- saveFn := func(context.Context, *restic.TreeJSONBuilder) (restic.ID, ItemStats, error) {
- val := atomic.AddInt32(&num, 1)
- if val == test.failAt {
- t.Logf("sending error for request %v\n", test.failAt)
- return restic.ID{}, ItemStats{}, errTest
- }
- return restic.NewRandomID(), ItemStats{TreeBlobs: 1, TreeSize: 123}, nil
- }
-
- errFn := func(snPath string, err error) error {
- t.Logf("ignoring error %v\n", err)
- return nil
- }
-
- b := NewTreeSaver(ctx, wg, uint(runtime.NumCPU()), saveFn, errFn)
-
var results []FutureNode
for i := 0; i < test.trees; i++ {
node := &restic.Node{
Name: fmt.Sprintf("file-%d", i),
}
+ nodes := []FutureNode{
+ newFutureNodeWithResult(futureNodeResult{node: &restic.Node{
+ Name: fmt.Sprintf("child-%d", i),
+ }}),
+ }
+ if (i + 1) == test.failAt {
+ nodes = append(nodes, newFutureNodeWithResult(futureNodeResult{
+ err: errTest,
+ }))
+ }
- fb := b.Save(ctx, "/", node.Name, node, nil, nil)
+ fb := b.Save(ctx, join("/", node.Name), node.Name, node, nodes, nil)
results = append(results, fb)
}
@@ -104,16 +108,51 @@ func TestTreeSaverError(t *testing.T) {
tree.take(ctx)
}
- b.TriggerShutdown()
-
- err := wg.Wait()
+ err := shutdown()
if err == nil {
t.Errorf("expected error not found")
}
-
if err != errTest {
t.Fatalf("unexpected error found: %v", err)
}
})
}
}
+
+func TestTreeSaverDuplicates(t *testing.T) {
+ for _, identicalNodes := range []bool{true, false} {
+ t.Run("", func(t *testing.T) {
+ ctx, cancel, b, shutdown := setupTreeSaver()
+ defer cancel()
+
+ node := &restic.Node{
+ Name: "file",
+ }
+ nodes := []FutureNode{
+ newFutureNodeWithResult(futureNodeResult{node: &restic.Node{
+ Name: "child",
+ }}),
+ }
+ if identicalNodes {
+ nodes = append(nodes, newFutureNodeWithResult(futureNodeResult{node: &restic.Node{
+ Name: "child",
+ }}))
+ } else {
+ nodes = append(nodes, newFutureNodeWithResult(futureNodeResult{node: &restic.Node{
+ Name: "child",
+ Size: 42,
+ }}))
+ }
+
+ fb := b.Save(ctx, join("/", node.Name), node.Name, node, nodes, nil)
+ fb.take(ctx)
+
+ err := shutdown()
+ if identicalNodes {
+ test.Assert(t, err == nil, "unexpected error found: %v", err)
+ } else {
+ test.Assert(t, err != nil, "expected error not found")
+ }
+ })
+ }
+}
diff --git a/internal/archiver/tree_test.go b/internal/archiver/tree_test.go
index 488f74b8c..7852a4c2e 100644
--- a/internal/archiver/tree_test.go
+++ b/internal/archiver/tree_test.go
@@ -439,9 +439,7 @@ func TestTree(t *testing.T) {
t.Skip("skip test on unix")
}
- tempdir, cleanup := restictest.TempDir(t)
- defer cleanup()
-
+ tempdir := restictest.TempDir(t)
TestCreateFiles(t, tempdir, test.src)
back := restictest.Chdir(t, tempdir)
diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go
index 7f3539f07..02433795b 100644
--- a/internal/backend/azure/azure.go
+++ b/internal/backend/azure/azure.go
@@ -1,6 +1,7 @@
package azure
import (
+ "bytes"
"context"
"crypto/md5"
"encoding/base64"
@@ -8,31 +9,38 @@ import (
"hash"
"io"
"net/http"
- "os"
"path"
"strings"
"github.com/restic/restic/internal/backend"
+ "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
- "github.com/Azure/azure-sdk-for-go/storage"
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore"
+ "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming"
+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+ "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
+ azContainer "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
"github.com/cenkalti/backoff/v4"
)
// Backend stores data on an azure endpoint.
type Backend struct {
- accountName string
- container *storage.Container
+ cfg Config
+ container *azContainer.Client
connections uint
sem sema.Semaphore
prefix string
listMaxItems int
- backend.Layout
+ layout.Layout
}
+const saveLargeSize = 256 * 1024 * 1024
const defaultListMaxItems = 5000
// make sure that *Backend implements backend.Backend
@@ -40,29 +48,47 @@ var _ restic.Backend = &Backend{}
func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
debug.Log("open, config %#v", cfg)
- var client storage.Client
+ var client *azContainer.Client
var err error
+
+ url := fmt.Sprintf("https://%s.blob.core.windows.net/%s", cfg.AccountName, cfg.Container)
+ opts := &azContainer.ClientOptions{
+ ClientOptions: azcore.ClientOptions{
+ Transport: http.DefaultClient,
+ },
+ }
+
if cfg.AccountKey.String() != "" {
// We have an account key value, find the BlobServiceClient
// from with a BasicClient
debug.Log(" - using account key")
- client, err = storage.NewBasicClient(cfg.AccountName, cfg.AccountKey.Unwrap())
+ cred, err := azblob.NewSharedKeyCredential(cfg.AccountName, cfg.AccountKey.Unwrap())
if err != nil {
- return nil, errors.Wrap(err, "NewBasicClient")
+ return nil, errors.Wrap(err, "NewSharedKeyCredential")
+ }
+
+ client, err = azContainer.NewClientWithSharedKeyCredential(url, cred, opts)
+
+ if err != nil {
+ return nil, errors.Wrap(err, "NewClientWithSharedKeyCredential")
}
} else if cfg.AccountSAS.String() != "" {
// Get the client using the SAS Token as authentication, this
// is longer winded than above because the SDK wants a URL for the Account
// if your using a SAS token, and not just the account name
// we (as per the SDK ) assume the default Azure portal.
- url := fmt.Sprintf("https://%s.blob.core.windows.net/", cfg.AccountName)
+ // https://github.com/Azure/azure-storage-blob-go/issues/130
debug.Log(" - using sas token")
sas := cfg.AccountSAS.Unwrap()
+
// strip query sign prefix
if sas[0] == '?' {
sas = sas[1:]
}
- client, err = storage.NewAccountSASClientFromEndpointToken(url, sas)
+
+ urlWithSAS := fmt.Sprintf("%s?%s", url, sas)
+
+ client, err = azContainer.NewClientWithNoCredential(urlWithSAS, opts)
if err != nil {
return nil, errors.Wrap(err, "NewAccountSASClientFromEndpointToken")
}
@@ -70,22 +96,17 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
return nil, errors.New("no azure authentication information found")
}
- client.HTTPClient = &http.Client{Transport: rt}
-
- service := client.GetBlobService()
-
sem, err := sema.New(cfg.Connections)
if err != nil {
return nil, err
}
be := &Backend{
- container: service.GetContainerReference(cfg.Container),
- accountName: cfg.AccountName,
+ container: client,
+ cfg: cfg,
connections: cfg.Connections,
sem: sem,
- prefix: cfg.Prefix,
- Layout: &backend.DefaultLayout{
+ Layout: &layout.DefaultLayout{
Path: cfg.Prefix,
Join: path.Join,
},
@@ -96,26 +117,29 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
}
// Open opens the Azure backend at specified container.
-func Open(cfg Config, rt http.RoundTripper) (*Backend, error) {
+func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
return open(cfg, rt)
}
// Create opens the Azure backend at specified container and creates the container if
// it does not exist yet.
-func Create(cfg Config, rt http.RoundTripper) (*Backend, error) {
+func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) {
be, err := open(cfg, rt)
if err != nil {
return nil, errors.Wrap(err, "open")
}
- options := storage.CreateContainerOptions{
- Access: storage.ContainerAccessTypePrivate,
- }
+ _, err = be.container.GetProperties(ctx, &azContainer.GetPropertiesOptions{})
- _, err = be.container.CreateIfNotExists(&options)
- if err != nil {
- return nil, errors.Wrap(err, "container.CreateIfNotExists")
+ if err != nil && bloberror.HasCode(err, bloberror.ContainerNotFound) {
+ _, err = be.container.Create(ctx, &azContainer.CreateOptions{})
+
+ if err != nil {
+ return nil, errors.Wrap(err, "container.Create")
+ }
+ } else if err != nil {
+ return be, err
}
return be, nil
@@ -129,7 +153,7 @@ func (be *Backend) SetListMaxItems(i int) {
// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
debug.Log("IsNotExist(%T, %#v)", err, err)
- return os.IsNotExist(err)
+ return bloberror.HasCode(err, bloberror.BlobNotFound)
}
// Join combines path components with slashes.
@@ -143,7 +167,7 @@ func (be *Backend) Connections() uint {
// Location returns this backend's location (the container name).
func (be *Backend) Location() string {
- return be.Join(be.container.Name, be.prefix)
+ return be.Join(be.cfg.AccountName, be.cfg.Prefix)
}
// Hasher may return a hash function for calculating a content hash for the backend
@@ -161,16 +185,6 @@ func (be *Backend) Path() string {
return be.prefix
}
-type azureAdapter struct {
- restic.RewindReader
-}
-
-func (azureAdapter) Close() error { return nil }
-
-func (a azureAdapter) Len() int {
- return int(a.Length())
-}
-
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
if err := h.Valid(); err != nil {
@@ -183,41 +197,53 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
be.sem.GetToken()
- debug.Log("InsertObject(%v, %v)", be.container.Name, objName)
+ debug.Log("InsertObject(%v, %v)", be.cfg.AccountName, objName)
var err error
- if rd.Length() < 256*1024*1024 {
- // wrap the reader so that net/http client cannot close the reader
- // CreateBlockBlobFromReader reads length from `Len()``
- dataReader := azureAdapter{rd}
-
+ if rd.Length() < saveLargeSize {
// if it's smaller than 256miB, then just create the file directly from the reader
- ref := be.container.GetBlobReference(objName)
- ref.Properties.ContentMD5 = base64.StdEncoding.EncodeToString(rd.Hash())
- err = ref.CreateBlockBlobFromReader(dataReader, nil)
+ err = be.saveSmall(ctx, objName, rd)
} else {
// otherwise use the more complicated method
err = be.saveLarge(ctx, objName, rd)
-
}
be.sem.ReleaseToken()
debug.Log("%v, err %#v", objName, err)
- return errors.Wrap(err, "CreateBlockBlobFromReader")
+ return err
}
-func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.RewindReader) error {
- // create the file on the server
- file := be.container.GetBlobReference(objName)
- err := file.CreateBlockBlob(nil)
+func (be *Backend) saveSmall(ctx context.Context, objName string, rd restic.RewindReader) error {
+ blockBlobClient := be.container.NewBlockBlobClient(objName)
+
+ // upload it as a new "block", use the base64 hash for the ID
+ id := base64.StdEncoding.EncodeToString(rd.Hash())
+
+ buf := make([]byte, rd.Length())
+ _, err := io.ReadFull(rd, buf)
+ if err != nil {
+ return errors.Wrap(err, "ReadFull")
+ }
+
+ reader := bytes.NewReader(buf)
+ _, err = blockBlobClient.StageBlock(ctx, id, streaming.NopCloser(reader), &blockblob.StageBlockOptions{
+ TransactionalContentMD5: rd.Hash(),
+ })
if err != nil {
- return errors.Wrap(err, "CreateBlockBlob")
+ return errors.Wrap(err, "StageBlock")
}
- // read the data, in 100 MiB chunks
+ blocks := []string{id}
+ _, err = blockBlobClient.CommitBlockList(ctx, blocks, &blockblob.CommitBlockListOptions{})
+ return errors.Wrap(err, "CommitBlockList")
+}
+
+func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.RewindReader) error {
+ blockBlobClient := be.container.NewBlockBlobClient(objName)
+
buf := make([]byte, 100*1024*1024)
- var blocks []storage.Block
+ blocks := []string{}
uploadedBytes := 0
for {
@@ -225,6 +251,7 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
if err == io.ErrUnexpectedEOF {
err = nil
}
+
if err == io.EOF {
// end of file reached, no bytes have been read at all
break
@@ -240,16 +267,18 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
// upload it as a new "block", use the base64 hash for the ID
h := md5.Sum(buf)
id := base64.StdEncoding.EncodeToString(h[:])
- debug.Log("PutBlock %v with %d bytes", id, len(buf))
- err = file.PutBlock(id, buf, &storage.PutBlockOptions{ContentMD5: id})
+
+ reader := bytes.NewReader(buf)
+ debug.Log("StageBlock %v with %d bytes", id, len(buf))
+ _, err = blockBlobClient.StageBlock(ctx, id, streaming.NopCloser(reader), &blockblob.StageBlockOptions{
+ TransactionalContentMD5: h[:],
+ })
+
if err != nil {
- return errors.Wrap(err, "PutBlock")
+ return errors.Wrap(err, "StageBlock")
}
- blocks = append(blocks, storage.Block{
- ID: id,
- Status: "Uncommitted",
- })
+ blocks = append(blocks, id)
}
// sanity check
@@ -257,10 +286,10 @@ func (be *Backend) saveLarge(ctx context.Context, objName string, rd restic.Rewi
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", uploadedBytes, rd.Length())
}
+ _, err := blockBlobClient.CommitBlockList(ctx, blocks, &blockblob.CommitBlockListOptions{})
+
debug.Log("uploaded %d parts: %v", len(blocks), blocks)
- err = file.PutBlockList(blocks, nil)
- debug.Log("PutBlockList returned %v", err)
- return errors.Wrap(err, "PutBlockList")
+ return errors.Wrap(err, "CommitBlockList")
}
// Load runs fn with a reader that yields the contents of the file at h at the
@@ -284,26 +313,22 @@ func (be *Backend) openReader(ctx context.Context, h restic.Handle, length int,
}
objName := be.Filename(h)
- blob := be.container.GetBlobReference(objName)
-
- start := uint64(offset)
- var end uint64
-
- if length > 0 {
- end = uint64(offset + int64(length) - 1)
- } else {
- end = 0
- }
+ blockBlobClient := be.container.NewBlobClient(objName)
be.sem.GetToken()
+ resp, err := blockBlobClient.DownloadStream(ctx, &blob.DownloadStreamOptions{
+ Range: azblob.HTTPRange{
+ Offset: offset,
+ Count: int64(length),
+ },
+ })
- rd, err := blob.GetRange(&storage.GetBlobRangeOptions{Range: &storage.BlobRange{Start: start, End: end}})
if err != nil {
be.sem.ReleaseToken()
return nil, err
}
- return be.sem.ReleaseTokenOnClose(rd, nil), err
+ return be.sem.ReleaseTokenOnClose(resp.Body, nil), err
}
// Stat returns information about a blob.
@@ -311,10 +336,10 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo,
debug.Log("%v", h)
objName := be.Filename(h)
- blob := be.container.GetBlobReference(objName)
+ blobClient := be.container.NewBlobClient(objName)
be.sem.GetToken()
- err := blob.GetProperties(nil)
+ props, err := blobClient.GetProperties(ctx, nil)
be.sem.ReleaseToken()
if err != nil {
@@ -323,35 +348,27 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo,
}
fi := restic.FileInfo{
- Size: int64(blob.Properties.ContentLength),
+ Size: *props.ContentLength,
Name: h.Name,
}
return fi, nil
}
-// Test returns true if a blob of the given type and name exists in the backend.
-func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
- objName := be.Filename(h)
-
- be.sem.GetToken()
- found, err := be.container.GetBlobReference(objName).Exists()
- be.sem.ReleaseToken()
-
- if err != nil {
- return false, err
- }
- return found, nil
-}
-
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
objName := be.Filename(h)
+ blob := be.container.NewBlobClient(objName)
be.sem.GetToken()
- _, err := be.container.GetBlobReference(objName).DeleteIfExists(nil)
+ _, err := blob.Delete(ctx, &azblob.DeleteBlobOptions{})
be.sem.ReleaseToken()
debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
+
+ if be.IsNotExist(err) {
+ return nil
+ }
+
return errors.Wrap(err, "client.RemoveObject")
}
@@ -367,31 +384,34 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
prefix += "/"
}
- params := storage.ListBlobsParameters{
- MaxResults: uint(be.listMaxItems),
- Prefix: prefix,
+ max := int32(be.listMaxItems)
+
+ opts := &azContainer.ListBlobsFlatOptions{
+ MaxResults: &max,
+ Prefix: &prefix,
}
+ lister := be.container.NewListBlobsFlatPager(opts)
- for {
+ for lister.More() {
be.sem.GetToken()
- obj, err := be.container.ListBlobs(params)
+ resp, err := lister.NextPage(ctx)
be.sem.ReleaseToken()
if err != nil {
return err
}
- debug.Log("got %v objects", len(obj.Blobs))
+ debug.Log("got %v objects", len(resp.Segment.BlobItems))
- for _, item := range obj.Blobs {
- m := strings.TrimPrefix(item.Name, prefix)
+ for _, item := range resp.Segment.BlobItems {
+ m := strings.TrimPrefix(*item.Name, prefix)
if m == "" {
continue
}
fi := restic.FileInfo{
Name: path.Base(m),
- Size: item.Properties.ContentLength,
+ Size: *item.Properties.ContentLength,
}
if ctx.Err() != nil {
@@ -408,11 +428,6 @@ func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.F
}
}
-
- if obj.NextMarker == "" {
- break
- }
- params.Marker = obj.NextMarker
}
return ctx.Err()
diff --git a/internal/backend/azure/azure_test.go b/internal/backend/azure/azure_test.go
index f1e58dea3..ada6ec2ca 100644
--- a/internal/backend/azure/azure_test.go
+++ b/internal/backend/azure/azure_test.go
@@ -46,17 +46,18 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(azure.Config)
- be, err := azure.Create(cfg, tr)
+ ctx := context.TODO()
+ be, err := azure.Create(ctx, cfg, tr)
if err != nil {
return nil, err
}
- exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
- if err != nil {
+ _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+ if err != nil && !be.IsNotExist(err) {
return nil, err
}
- if exists {
+ if err == nil {
return nil, errors.New("config already exists")
}
@@ -66,15 +67,15 @@ func newAzureTestSuite(t testing.TB) *test.Suite {
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(azure.Config)
-
- return azure.Open(cfg, tr)
+ ctx := context.TODO()
+ return azure.Open(ctx, cfg, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(azure.Config)
-
- be, err := azure.Open(cfg, tr)
+ ctx := context.TODO()
+ be, err := azure.Open(ctx, cfg, tr)
if err != nil {
return err
}
@@ -155,7 +156,7 @@ func TestUploadLargeFile(t *testing.T) {
t.Fatal(err)
}
- be, err := azure.Create(cfg, tr)
+ be, err := azure.Create(ctx, cfg, tr)
if err != nil {
t.Fatal(err)
}
diff --git a/internal/backend/azure/config.go b/internal/backend/azure/config.go
index cc5169e3e..55b26d4f1 100644
--- a/internal/backend/azure/config.go
+++ b/internal/backend/azure/config.go
@@ -43,14 +43,13 @@ func ParseConfig(s string) (interface{}, error) {
// use the first entry of the path as the bucket name and the
// remainder as prefix
- data := strings.SplitN(s, ":", 2)
- if len(data) < 2 {
+ container, prefix, colon := strings.Cut(s, ":")
+ if !colon {
return nil, errors.New("azure: invalid format: bucket name or path not found")
}
- container, path := data[0], path.Clean(data[1])
- path = strings.TrimPrefix(path, "/")
+ prefix = strings.TrimPrefix(path.Clean(prefix), "/")
cfg := NewConfig()
cfg.Container = container
- cfg.Prefix = path
+ cfg.Prefix = prefix
return cfg, nil
}
diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go
index 150d396d5..40dbbf893 100644
--- a/internal/backend/b2/b2.go
+++ b/internal/backend/b2/b2.go
@@ -6,8 +6,11 @@ import (
"io"
"net/http"
"path"
+ "sync"
+ "time"
"github.com/restic/restic/internal/backend"
+ "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
@@ -15,6 +18,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/kurin/blazer/b2"
+ "github.com/kurin/blazer/base"
)
// b2Backend is a backend which stores its data on Backblaze B2.
@@ -23,20 +27,49 @@ type b2Backend struct {
bucket *b2.Bucket
cfg Config
listMaxItems int
- backend.Layout
+ layout.Layout
sem sema.Semaphore
+
+ canDelete bool
}
-const defaultListMaxItems = 1000
+// Billing happens in 1000 item granlarity, but we are more interested in reducing the number of network round trips
+const defaultListMaxItems = 10 * 1000
// ensure statically that *b2Backend implements restic.Backend.
var _ restic.Backend = &b2Backend{}
+type sniffingRoundTripper struct {
+ sync.Mutex
+ lastErr error
+ http.RoundTripper
+}
+
+func (s *sniffingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+ res, err := s.RoundTripper.RoundTrip(req)
+ if err != nil {
+ s.Lock()
+ s.lastErr = err
+ s.Unlock()
+ }
+ return res, err
+}
+
func newClient(ctx context.Context, cfg Config, rt http.RoundTripper) (*b2.Client, error) {
- opts := []b2.ClientOption{b2.Transport(rt)}
+ sniffer := &sniffingRoundTripper{RoundTripper: rt}
+ opts := []b2.ClientOption{b2.Transport(sniffer)}
+ // if the connection B2 fails, this can cause the client to hang
+ // cancel the connection after a minute to at least provide some feedback to the user
+ ctx, cancel := context.WithTimeout(ctx, time.Minute)
+ defer cancel()
c, err := b2.NewClient(ctx, cfg.AccountID, cfg.Key.Unwrap(), opts...)
- if err != nil {
+ if err == context.DeadlineExceeded {
+ if sniffer.lastErr != nil {
+ return nil, sniffer.lastErr
+ }
+ return nil, errors.New("connection to B2 failed")
+ } else if err != nil {
return nil, errors.Wrap(err, "b2.NewClient")
}
return c, nil
@@ -68,12 +101,13 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend
client: client,
bucket: bucket,
cfg: cfg,
- Layout: &backend.DefaultLayout{
+ Layout: &layout.DefaultLayout{
Join: path.Join,
Path: cfg.Prefix,
},
listMaxItems: defaultListMaxItems,
sem: sem,
+ canDelete: true,
}
return be, nil
@@ -109,7 +143,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe
client: client,
bucket: bucket,
cfg: cfg,
- Layout: &backend.DefaultLayout{
+ Layout: &layout.DefaultLayout{
Join: path.Join,
Path: cfg.Prefix,
},
@@ -117,12 +151,12 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backe
sem: sem,
}
- present, err := be.Test(ctx, restic.Handle{Type: restic.ConfigFile})
- if err != nil {
+ _, err = be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
+ if err != nil && !be.IsNotExist(err) {
return nil, err
}
- if present {
+ if err == nil {
return nil, errors.New("config already exists")
}
@@ -155,7 +189,14 @@ func (be *b2Backend) HasAtomicReplace() bool {
// IsNotExist returns true if the error is caused by a non-existing file.
func (be *b2Backend) IsNotExist(err error) bool {
- return b2.IsNotExist(errors.Cause(err))
+ // blazer/b2 does not export its error types and values,
+ // so we can't use errors.{As,Is}.
+ for ; err != nil; err = errors.Unwrap(err) {
+ if b2.IsNotExist(err) {
+ return true
+ }
+ }
+ return false
}
// Load runs fn with a reader that yields the contents of the file at h at the
@@ -250,23 +291,6 @@ func (be *b2Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileI
return restic.FileInfo{Size: info.Size, Name: h.Name}, nil
}
-// Test returns true if a blob of the given type and name exists in the backend.
-func (be *b2Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
- debug.Log("Test %v", h)
-
- be.sem.GetToken()
- defer be.sem.ReleaseToken()
-
- found := false
- name := be.Filename(h)
- obj := be.bucket.Object(name)
- info, err := obj.Attrs(ctx)
- if err == nil && info != nil && info.Status == b2.Uploaded {
- found = true
- }
- return found, nil
-}
-
// Remove removes the blob with the given name and type.
func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
debug.Log("Remove %v", h)
@@ -274,14 +298,38 @@ func (be *b2Backend) Remove(ctx context.Context, h restic.Handle) error {
be.sem.GetToken()
defer be.sem.ReleaseToken()
- obj := be.bucket.Object(be.Filename(h))
- err := obj.Delete(ctx)
- // consider a file as removed if b2 informs us that it does not exist
- if b2.IsNotExist(err) {
- return nil
+ // the retry backend will also repeat the remove method up to 10 times
+ for i := 0; i < 3; i++ {
+ obj := be.bucket.Object(be.Filename(h))
+
+ var err error
+ if be.canDelete {
+ err = obj.Delete(ctx)
+ if err == nil {
+ // keep deleting until we are sure that no leftover file versions exist
+ continue
+ }
+
+ code, _ := base.Code(err)
+ if code == 401 { // unauthorized
+ // fallback to hide if not allowed to delete files
+ be.canDelete = false
+ debug.Log("Removing %v failed, falling back to b2_hide_file.", h)
+ continue
+ }
+ } else {
+ // hide adds a new file version hiding all older ones, thus retries are not necessary
+ err = obj.Hide(ctx)
+ }
+
+ // consider a file as removed if b2 informs us that it does not exist
+ if b2.IsNotExist(err) {
+ return nil
+ }
+ return errors.Wrap(err, "Delete")
}
- return errors.Wrap(err, "Delete")
+ return errors.New("failed to delete all file versions")
}
type semLocker struct {
@@ -349,7 +397,7 @@ func (be *b2Backend) Delete(ctx context.Context) error {
}
}
err := be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
- if err != nil && b2.IsNotExist(errors.Cause(err)) {
+ if err != nil && be.IsNotExist(err) {
err = nil
}
diff --git a/internal/backend/b2/config.go b/internal/backend/b2/config.go
index 98e8e1445..ba5141834 100644
--- a/internal/backend/b2/config.go
+++ b/internal/backend/b2/config.go
@@ -37,7 +37,7 @@ var bucketName = regexp.MustCompile("^[a-zA-Z0-9-]+$")
// https://help.backblaze.com/hc/en-us/articles/217666908-What-you-need-to-know-about-B2-Bucket-names
func checkBucketName(name string) error {
if name == "" {
- return errors.New("bucket name is empty")
+ return errors.New("bucket name not found")
}
if len(name) < 6 {
@@ -64,30 +64,18 @@ func ParseConfig(s string) (interface{}, error) {
}
s = s[3:]
- data := strings.SplitN(s, ":", 2)
- if len(data) == 0 || len(data[0]) == 0 {
- return nil, errors.New("bucket name not found")
- }
-
- cfg := NewConfig()
- cfg.Bucket = data[0]
-
- if err := checkBucketName(cfg.Bucket); err != nil {
+ bucket, prefix, _ := strings.Cut(s, ":")
+ if err := checkBucketName(bucket); err != nil {
return nil, err
}
- if len(data) == 2 {
- p := data[1]
- if len(p) > 0 {
- p = path.Clean(p)
- }
-
- if len(p) > 0 && path.IsAbs(p) {
- p = p[1:]
- }
-
- cfg.Prefix = p
+ if len(prefix) > 0 {
+ prefix = strings.TrimPrefix(path.Clean(prefix), "/")
}
+ cfg := NewConfig()
+ cfg.Bucket = bucket
+ cfg.Prefix = prefix
+
return cfg, nil
}
diff --git a/internal/backend/backend_error.go b/internal/backend/backend_error.go
deleted file mode 100644
index 77a931858..000000000
--- a/internal/backend/backend_error.go
+++ /dev/null
@@ -1,84 +0,0 @@
-package backend
-
-import (
- "context"
- "io"
- "io/ioutil"
- "math/rand"
- "sync"
-
- "github.com/restic/restic/internal/errors"
- "github.com/restic/restic/internal/restic"
-)
-
-// ErrorBackend is used to induce errors into various function calls and test
-// the retry functions.
-type ErrorBackend struct {
- FailSave float32
- FailSaveRead float32
- FailLoad float32
- FailStat float32
- restic.Backend
-
- r *rand.Rand
- m sync.Mutex
-}
-
-// statically ensure that ErrorBackend implements restic.Backend.
-var _ restic.Backend = &ErrorBackend{}
-
-// NewErrorBackend wraps be with a backend that returns errors according to
-// given probabilities.
-func NewErrorBackend(be restic.Backend, seed int64) *ErrorBackend {
- return &ErrorBackend{
- Backend: be,
- r: rand.New(rand.NewSource(seed)),
- }
-}
-
-func (be *ErrorBackend) fail(p float32) bool {
- be.m.Lock()
- v := be.r.Float32()
- be.m.Unlock()
-
- return v < p
-}
-
-// Save stores the data in the backend under the given handle.
-func (be *ErrorBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
- if be.fail(be.FailSave) {
- return errors.Errorf("Save(%v) random error induced", h)
- }
-
- if be.fail(be.FailSaveRead) {
- _, err := io.CopyN(ioutil.Discard, rd, be.r.Int63n(1000))
- if err != nil {
- return err
- }
-
- return errors.Errorf("Save(%v) random error with partial read induced", h)
- }
-
- return be.Backend.Save(ctx, h, rd)
-}
-
-// Load returns a reader that yields the contents of the file at h at the
-// given offset. If length is larger than zero, only a portion of the file
-// is returned. rd must be closed after use. If an error is returned, the
-// ReadCloser must be nil.
-func (be *ErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
- if be.fail(be.FailLoad) {
- return errors.Errorf("Load(%v, %v, %v) random error induced", h, length, offset)
- }
-
- return be.Backend.Load(ctx, h, length, offset, consumer)
-}
-
-// Stat returns information about the File identified by h.
-func (be *ErrorBackend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
- if be.fail(be.FailLoad) {
- return restic.FileInfo{}, errors.Errorf("Stat(%v) random error induced", h)
- }
-
- return be.Stat(ctx, h)
-}
diff --git a/internal/backend/dryrun/dry_backend.go b/internal/backend/dryrun/dry_backend.go
index 31012df43..37569c320 100644
--- a/internal/backend/dryrun/dry_backend.go
+++ b/internal/backend/dryrun/dry_backend.go
@@ -86,7 +86,3 @@ func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
return be.b.Stat(ctx, h)
}
-
-func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
- return be.b.Test(ctx, h)
-}
diff --git a/internal/backend/dryrun/dry_backend_test.go b/internal/backend/dryrun/dry_backend_test.go
index 1b512ad20..6b8f74e0f 100644
--- a/internal/backend/dryrun/dry_backend_test.go
+++ b/internal/backend/dryrun/dry_backend_test.go
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"io"
- "io/ioutil"
"sort"
"strings"
"testing"
@@ -42,12 +41,9 @@ func TestDry(t *testing.T) {
{d, "stat", "a", "", "not found"},
{d, "list", "", "", ""},
{d, "save", "", "", "invalid"},
- {d, "test", "a", "", ""},
{m, "save", "a", "baz", ""}, // save a directly to the mem backend
{d, "save", "b", "foob", ""}, // b is not saved
{d, "save", "b", "xxx", ""}, // no error as b is not saved
- {d, "test", "a", "1", ""},
- {d, "test", "b", "", ""},
{d, "stat", "", "", "invalid"},
{d, "stat", "a", "a 3", ""},
{d, "load", "a", "baz", ""},
@@ -66,17 +62,11 @@ func TestDry(t *testing.T) {
for i, step := range steps {
var err error
- var boolRes bool
handle := restic.Handle{Type: restic.PackFile, Name: step.fname}
switch step.op {
case "save":
err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content), step.be.Hasher()))
- case "test":
- boolRes, err = step.be.Test(ctx, handle)
- if boolRes != (step.content != "") {
- t.Errorf("%d. Test(%q) = %v, want %v", i, step.fname, boolRes, step.content != "")
- }
case "list":
fileList := []string{}
err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
@@ -109,7 +99,7 @@ func TestDry(t *testing.T) {
case "load":
data := ""
err = step.be.Load(ctx, handle, 100, 0, func(rd io.Reader) error {
- buf, err := ioutil.ReadAll(rd)
+ buf, err := io.ReadAll(rd)
data = string(buf)
return err
})
diff --git a/internal/backend/errdot_119.go b/internal/backend/errdot_119.go
new file mode 100644
index 000000000..3676a099d
--- /dev/null
+++ b/internal/backend/errdot_119.go
@@ -0,0 +1,20 @@
+//go:build go1.19
+// +build go1.19
+
+// This file provides a function to check whether an error from cmd.Start() is
+// exec.ErrDot which was introduced in Go 1.19.
+// This function is needed so that we can perform this check only for Go 1.19 and
+// up, whereas for older versions we use a dummy/stub in the file errdot_old.go.
+// Once the minimum Go version restic supports is 1.19, remove this file and
+// replace any calls to it with the corresponding code as per below.
+
+package backend
+
+import (
+ "errors"
+ "os/exec"
+)
+
+func IsErrDot(err error) bool {
+ return errors.Is(err, exec.ErrDot)
+}
diff --git a/internal/backend/errdot_old.go b/internal/backend/errdot_old.go
new file mode 100644
index 000000000..92a58ad25
--- /dev/null
+++ b/internal/backend/errdot_old.go
@@ -0,0 +1,13 @@
+//go:build !go1.19
+// +build !go1.19
+
+// This file provides a stub for IsErrDot() for Go versions below 1.19.
+// See the corresponding file errdot_119.go for more information.
+// Once the minimum Go version restic supports is 1.19, remove this file
+// and perform the actions listed in errdot_119.go.
+
+package backend
+
+func IsErrDot(err error) bool {
+ return false
+}
diff --git a/internal/backend/gs/config.go b/internal/backend/gs/config.go
index bd152d775..33aec4c99 100644
--- a/internal/backend/gs/config.go
+++ b/internal/backend/gs/config.go
@@ -42,17 +42,15 @@ func ParseConfig(s string) (interface{}, error) {
// use the first entry of the path as the bucket name and the
// remainder as prefix
- data := strings.SplitN(s, ":", 2)
- if len(data) < 2 {
+ bucket, prefix, colon := strings.Cut(s, ":")
+ if !colon {
return nil, errors.New("gs: invalid format: bucket name or path not found")
}
- bucket, path := data[0], path.Clean(data[1])
-
- path = strings.TrimPrefix(path, "/")
+ prefix = strings.TrimPrefix(path.Clean(prefix), "/")
cfg := NewConfig()
cfg.Bucket = bucket
- cfg.Prefix = path
+ cfg.Prefix = prefix
return cfg, nil
}
diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go
index 3c68b55e7..77cbcda97 100644
--- a/internal/backend/gs/gs.go
+++ b/internal/backend/gs/gs.go
@@ -14,6 +14,7 @@ import (
"cloud.google.com/go/storage"
"github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
+ "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
@@ -41,7 +42,7 @@ type Backend struct {
bucket *storage.BucketHandle
prefix string
listMaxItems int
- backend.Layout
+ layout.Layout
}
// Ensure that *Backend implements restic.Backend.
@@ -111,7 +112,7 @@ func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
bucketName: cfg.Bucket,
bucket: gcsClient.Bucket(cfg.Bucket),
prefix: cfg.Prefix,
- Layout: &backend.DefaultLayout{
+ Layout: &layout.DefaultLayout{
Path: cfg.Prefix,
Join: path.Join,
},
@@ -169,18 +170,7 @@ func (be *Backend) SetListMaxItems(i int) {
// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
debug.Log("IsNotExist(%T, %#v)", err, err)
-
- if os.IsNotExist(err) {
- return true
- }
-
- if er, ok := err.(*googleapi.Error); ok {
- if er.Code == 404 {
- return true
- }
- }
-
- return false
+ return errors.Is(err, storage.ErrObjectNotExist)
}
// Join combines path components with slashes.
@@ -333,22 +323,6 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
return restic.FileInfo{Size: attr.Size, Name: h.Name}, nil
}
-// Test returns true if a blob of the given type and name exists in the backend.
-func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
- found := false
- objName := be.Filename(h)
-
- be.sem.GetToken()
- _, err := be.bucket.Object(objName).Attrs(ctx)
- be.sem.ReleaseToken()
-
- if err == nil {
- found = true
- }
- // If error, then not found
- return found, nil
-}
-
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
objName := be.Filename(h)
diff --git a/internal/backend/gs/gs_test.go b/internal/backend/gs/gs_test.go
index d7bf1422c..77f8986f1 100644
--- a/internal/backend/gs/gs_test.go
+++ b/internal/backend/gs/gs_test.go
@@ -47,12 +47,12 @@ func newGSTestSuite(t testing.TB) *test.Suite {
return nil, err
}
- exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
- if err != nil {
+ _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+ if err != nil && !be.IsNotExist(err) {
return nil, err
}
- if exists {
+ if err == nil {
return nil, errors.New("config already exists")
}
diff --git a/internal/backend/http_transport.go b/internal/backend/http_transport.go
index 2ff56c887..9ee1c91f1 100644
--- a/internal/backend/http_transport.go
+++ b/internal/backend/http_transport.go
@@ -4,9 +4,9 @@ import (
"crypto/tls"
"crypto/x509"
"encoding/pem"
- "io/ioutil"
"net"
"net/http"
+ "os"
"strings"
"time"
@@ -30,7 +30,7 @@ type TransportOptions struct {
// readPEMCertKey reads a file and returns the PEM encoded certificate and key
// blocks.
func readPEMCertKey(filename string) (certs []byte, key []byte, err error) {
- data, err := ioutil.ReadFile(filename)
+ data, err := os.ReadFile(filename)
if err != nil {
return nil, nil, errors.Wrap(err, "ReadFile")
}
@@ -105,7 +105,7 @@ func Transport(opts TransportOptions) (http.RoundTripper, error) {
if filename == "" {
return nil, errors.Errorf("empty filename for root certificate supplied")
}
- b, err := ioutil.ReadFile(filename)
+ b, err := os.ReadFile(filename)
if err != nil {
return nil, errors.Errorf("unable to read root certificate: %v", err)
}
diff --git a/internal/backend/layout.go b/internal/backend/layout/layout.go
similarity index 98%
rename from internal/backend/layout.go
rename to internal/backend/layout/layout.go
index ebd54e4af..14fb8dcdc 100644
--- a/internal/backend/layout.go
+++ b/internal/backend/layout/layout.go
@@ -1,4 +1,4 @@
-package backend
+package layout
import (
"context"
@@ -71,7 +71,7 @@ var backendFilename = regexp.MustCompile(fmt.Sprintf("^[a-fA-F0-9]{%d}$", backen
func hasBackendFile(ctx context.Context, fs Filesystem, dir string) (bool, error) {
entries, err := fs.ReadDir(ctx, dir)
- if err != nil && fs.IsNotExist(errors.Cause(err)) {
+ if err != nil && fs.IsNotExist(err) {
return false, nil
}
diff --git a/internal/backend/layout_default.go b/internal/backend/layout/layout_default.go
similarity index 99%
rename from internal/backend/layout_default.go
rename to internal/backend/layout/layout_default.go
index 3bc3087ed..17c250e8f 100644
--- a/internal/backend/layout_default.go
+++ b/internal/backend/layout/layout_default.go
@@ -1,4 +1,4 @@
-package backend
+package layout
import (
"encoding/hex"
diff --git a/internal/backend/layout_rest.go b/internal/backend/layout/layout_rest.go
similarity index 98%
rename from internal/backend/layout_rest.go
rename to internal/backend/layout/layout_rest.go
index 1d65828a8..2aa869995 100644
--- a/internal/backend/layout_rest.go
+++ b/internal/backend/layout/layout_rest.go
@@ -1,4 +1,4 @@
-package backend
+package layout
import "github.com/restic/restic/internal/restic"
diff --git a/internal/backend/layout_s3legacy.go b/internal/backend/layout/layout_s3legacy.go
similarity index 99%
rename from internal/backend/layout_s3legacy.go
rename to internal/backend/layout/layout_s3legacy.go
index f83355860..ac88e77ad 100644
--- a/internal/backend/layout_s3legacy.go
+++ b/internal/backend/layout/layout_s3legacy.go
@@ -1,4 +1,4 @@
-package backend
+package layout
import "github.com/restic/restic/internal/restic"
diff --git a/internal/backend/layout_test.go b/internal/backend/layout/layout_test.go
similarity index 93%
rename from internal/backend/layout_test.go
rename to internal/backend/layout/layout_test.go
index d319a0b2d..fc9c6e214 100644
--- a/internal/backend/layout_test.go
+++ b/internal/backend/layout/layout_test.go
@@ -1,4 +1,4 @@
-package backend
+package layout
import (
"context"
@@ -14,8 +14,7 @@ import (
)
func TestDefaultLayout(t *testing.T) {
- tempdir, cleanup := rtest.TempDir(t)
- defer cleanup()
+ tempdir := rtest.TempDir(t)
var tests = []struct {
path string
@@ -141,8 +140,7 @@ func TestDefaultLayout(t *testing.T) {
}
func TestRESTLayout(t *testing.T) {
- path, cleanup := rtest.TempDir(t)
- defer cleanup()
+ path := rtest.TempDir(t)
var tests = []struct {
restic.Handle
@@ -287,8 +285,7 @@ func TestRESTLayoutURLs(t *testing.T) {
}
func TestS3LegacyLayout(t *testing.T) {
- path, cleanup := rtest.TempDir(t)
- defer cleanup()
+ path := rtest.TempDir(t)
var tests = []struct {
restic.Handle
@@ -355,22 +352,21 @@ func TestS3LegacyLayout(t *testing.T) {
}
func TestDetectLayout(t *testing.T) {
- path, cleanup := rtest.TempDir(t)
- defer cleanup()
+ path := rtest.TempDir(t)
var tests = []struct {
filename string
want string
}{
- {"repo-layout-default.tar.gz", "*backend.DefaultLayout"},
- {"repo-layout-s3legacy.tar.gz", "*backend.S3LegacyLayout"},
+ {"repo-layout-default.tar.gz", "*layout.DefaultLayout"},
+ {"repo-layout-s3legacy.tar.gz", "*layout.S3LegacyLayout"},
}
var fs = &LocalFilesystem{}
for _, test := range tests {
for _, fs := range []Filesystem{fs, nil} {
t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) {
- rtest.SetupTarTestFixture(t, path, filepath.Join("testdata", test.filename))
+ rtest.SetupTarTestFixture(t, path, filepath.Join("../testdata", test.filename))
layout, err := DetectLayout(context.TODO(), fs, filepath.Join(path, "repo"))
if err != nil {
@@ -393,20 +389,19 @@ func TestDetectLayout(t *testing.T) {
}
func TestParseLayout(t *testing.T) {
- path, cleanup := rtest.TempDir(t)
- defer cleanup()
+ path := rtest.TempDir(t)
var tests = []struct {
layoutName string
defaultLayoutName string
want string
}{
- {"default", "", "*backend.DefaultLayout"},
- {"s3legacy", "", "*backend.S3LegacyLayout"},
- {"", "", "*backend.DefaultLayout"},
+ {"default", "", "*layout.DefaultLayout"},
+ {"s3legacy", "", "*layout.S3LegacyLayout"},
+ {"", "", "*layout.DefaultLayout"},
}
- rtest.SetupTarTestFixture(t, path, filepath.Join("testdata", "repo-layout-default.tar.gz"))
+ rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", "repo-layout-default.tar.gz"))
for _, test := range tests {
t.Run(test.layoutName, func(t *testing.T) {
@@ -433,8 +428,7 @@ func TestParseLayout(t *testing.T) {
}
func TestParseLayoutInvalid(t *testing.T) {
- path, cleanup := rtest.TempDir(t)
- defer cleanup()
+ path := rtest.TempDir(t)
var invalidNames = []string{
"foo", "bar", "local",
diff --git a/internal/backend/local/layout_test.go b/internal/backend/local/layout_test.go
index 9da702877..a4fccd2cb 100644
--- a/internal/backend/local/layout_test.go
+++ b/internal/backend/local/layout_test.go
@@ -10,8 +10,7 @@ import (
)
func TestLayout(t *testing.T) {
- path, cleanup := rtest.TempDir(t)
- defer cleanup()
+ path := rtest.TempDir(t)
var tests = []struct {
filename string
diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go
index bb644c949..1716e0f07 100644
--- a/internal/backend/local/local.go
+++ b/internal/backend/local/local.go
@@ -4,12 +4,12 @@ import (
"context"
"hash"
"io"
- "io/ioutil"
"os"
"path/filepath"
"syscall"
"github.com/restic/restic/internal/backend"
+ "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
@@ -23,7 +23,7 @@ import (
type Local struct {
Config
sem sema.Semaphore
- backend.Layout
+ layout.Layout
backend.Modes
}
@@ -33,7 +33,7 @@ var _ restic.Backend = &Local{}
const defaultLayout = "default"
func open(ctx context.Context, cfg Config) (*Local, error) {
- l, err := backend.ParseLayout(ctx, &backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
+ l, err := layout.ParseLayout(ctx, &layout.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
@@ -176,7 +176,7 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade
// Ignore error if filesystem does not support fsync.
err = f.Sync()
- syncNotSup := errors.Is(err, syscall.ENOTSUP)
+ syncNotSup := err != nil && (errors.Is(err, syscall.ENOTSUP) || isMacENOTTY(err))
if err != nil && !syncNotSup {
return errors.WithStack(err)
}
@@ -208,7 +208,7 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReade
return nil
}
-var tempFile = ioutil.TempFile // Overridden by test.
+var tempFile = os.CreateTemp // Overridden by test.
// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
@@ -269,24 +269,6 @@ func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, err
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}
-// Test returns true if a blob of the given type and name exists in the backend.
-func (b *Local) Test(ctx context.Context, h restic.Handle) (bool, error) {
- debug.Log("Test %v", h)
-
- b.sem.GetToken()
- defer b.sem.ReleaseToken()
-
- _, err := fs.Stat(b.Filename(h))
- if err != nil {
- if b.IsNotExist(err) {
- return false, nil
- }
- return false, errors.WithStack(err)
- }
-
- return true, nil
-}
-
// Remove removes the blob with the given name and type.
func (b *Local) Remove(ctx context.Context, h restic.Handle) error {
debug.Log("Remove %v", h)
diff --git a/internal/backend/local/local_internal_test.go b/internal/backend/local/local_internal_test.go
index 8de3d3c2f..1e80e72ed 100644
--- a/internal/backend/local/local_internal_test.go
+++ b/internal/backend/local/local_internal_test.go
@@ -24,8 +24,7 @@ func TestNoSpacePermanent(t *testing.T) {
return nil, fmt.Errorf("not creating tempfile, %w", syscall.ENOSPC)
}
- dir, cleanup := rtest.TempDir(t)
- defer cleanup()
+ dir := rtest.TempDir(t)
be, err := Open(context.Background(), Config{Path: dir, Connections: 2})
rtest.OK(t, err)
diff --git a/internal/backend/local/local_test.go b/internal/backend/local/local_test.go
index 75c3b8ed7..495f220a0 100644
--- a/internal/backend/local/local_test.go
+++ b/internal/backend/local/local_test.go
@@ -2,7 +2,6 @@ package local_test
import (
"context"
- "io/ioutil"
"os"
"path/filepath"
"testing"
@@ -17,7 +16,7 @@ func newTestSuite(t testing.TB) *test.Suite {
return &test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
- dir, err := ioutil.TempDir(rtest.TestTempDir, "restic-test-local-")
+ dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-local-")
if err != nil {
t.Fatal(err)
}
@@ -121,8 +120,7 @@ func removeAll(t testing.TB, dir string) {
}
func TestOpenNotExistingDirectory(t *testing.T) {
- dir, cleanup := rtest.TempDir(t)
- defer cleanup()
+ dir := rtest.TempDir(t)
// local.Open must not create any files dirs in the repo
openclose(t, filepath.Join(dir, "repo"))
diff --git a/internal/backend/local/local_unix.go b/internal/backend/local/local_unix.go
index 3dde753a8..e3256ed7a 100644
--- a/internal/backend/local/local_unix.go
+++ b/internal/backend/local/local_unix.go
@@ -6,6 +6,7 @@ package local
import (
"errors"
"os"
+ "runtime"
"syscall"
"github.com/restic/restic/internal/fs"
@@ -19,7 +20,9 @@ func fsyncDir(dir string) error {
}
err = d.Sync()
- if errors.Is(err, syscall.ENOTSUP) || errors.Is(err, syscall.ENOENT) || errors.Is(err, syscall.EINVAL) {
+ if err != nil &&
+ (errors.Is(err, syscall.ENOTSUP) || errors.Is(err, syscall.ENOENT) ||
+ errors.Is(err, syscall.EINVAL) || isMacENOTTY(err)) {
err = nil
}
@@ -31,6 +34,15 @@ func fsyncDir(dir string) error {
return err
}
+// The ExFAT driver on some versions of macOS can return ENOTTY,
+// "inappropriate ioctl for device", for fsync.
+//
+// https://github.com/restic/restic/issues/4016
+// https://github.com/realm/realm-core/issues/5789
+func isMacENOTTY(err error) bool {
+ return runtime.GOOS == "darwin" && errors.Is(err, syscall.ENOTTY)
+}
+
// set file to readonly
func setFileReadonly(f string, mode os.FileMode) error {
return fs.Chmod(f, mode&^0222)
diff --git a/internal/backend/local/local_windows.go b/internal/backend/local/local_windows.go
index 72ced630c..d69b9eec8 100644
--- a/internal/backend/local/local_windows.go
+++ b/internal/backend/local/local_windows.go
@@ -7,6 +7,9 @@ import (
// Can't explicitly flush directory changes on Windows.
func fsyncDir(dir string) error { return nil }
+// Windows is not macOS.
+func isMacENOTTY(err error) bool { return false }
+
// We don't modify read-only on windows,
// since it will make us unable to delete the file,
// and this isn't common practice on this platform.
diff --git a/internal/backend/location/location.go b/internal/backend/location/location.go
index daef51d40..a732233cc 100644
--- a/internal/backend/location/location.go
+++ b/internal/backend/location/location.go
@@ -127,6 +127,6 @@ func StripPassword(s string) string {
}
func extractScheme(s string) string {
- data := strings.SplitN(s, ":", 2)
- return data[0]
+ scheme, _, _ := strings.Cut(s, ":")
+ return scheme
}
diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go
index 7e8ae5356..0c46dcd6e 100644
--- a/internal/backend/mem/mem_backend.go
+++ b/internal/backend/mem/mem_backend.go
@@ -3,13 +3,12 @@ package mem
import (
"bytes"
"context"
- "crypto/md5"
"encoding/base64"
"hash"
"io"
- "io/ioutil"
"sync"
+ "github.com/cespare/xxhash/v2"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug"
@@ -53,23 +52,6 @@ func New() *MemoryBackend {
return be
}
-// Test returns whether a file exists.
-func (be *MemoryBackend) Test(ctx context.Context, h restic.Handle) (bool, error) {
- be.sem.GetToken()
- defer be.sem.ReleaseToken()
-
- be.m.Lock()
- defer be.m.Unlock()
-
- debug.Log("Test %v", h)
-
- if _, ok := be.data[h]; ok {
- return true, ctx.Err()
- }
-
- return false, ctx.Err()
-}
-
// IsNotExist returns true if the file does not exist.
func (be *MemoryBackend) IsNotExist(err error) bool {
return errors.Is(err, errNotFound)
@@ -96,7 +78,7 @@ func (be *MemoryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Re
return errors.New("file already exists")
}
- buf, err := ioutil.ReadAll(rd)
+ buf, err := io.ReadAll(rd)
if err != nil {
return err
}
@@ -168,7 +150,7 @@ func (be *MemoryBackend) openReader(ctx context.Context, h restic.Handle, length
buf = buf[:length]
}
- return be.sem.ReleaseTokenOnClose(ioutil.NopCloser(bytes.NewReader(buf)), nil), ctx.Err()
+ return be.sem.ReleaseTokenOnClose(io.NopCloser(bytes.NewReader(buf)), nil), ctx.Err()
}
// Stat returns information about a file in the backend.
@@ -266,7 +248,7 @@ func (be *MemoryBackend) Location() string {
// Hasher may return a hash function for calculating a content hash for the backend
func (be *MemoryBackend) Hasher() hash.Hash {
- return md5.New()
+ return xxhash.New()
}
// HasAtomicReplace returns whether Save() can atomically replace files
diff --git a/internal/backend/mem/mem_backend_test.go b/internal/backend/mem/mem_backend_test.go
index 15e66ac83..819c6a2b6 100644
--- a/internal/backend/mem/mem_backend_test.go
+++ b/internal/backend/mem/mem_backend_test.go
@@ -26,12 +26,12 @@ func newTestSuite() *test.Suite {
Create: func(cfg interface{}) (restic.Backend, error) {
c := cfg.(*memConfig)
if c.be != nil {
- ok, err := c.be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
- if err != nil {
+ _, err := c.be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+ if err != nil && !c.be.IsNotExist(err) {
return nil, err
}
- if ok {
+ if err == nil {
return nil, errors.New("config already exists")
}
}
diff --git a/internal/backend/mock/backend.go b/internal/backend/mock/backend.go
index 655499b15..875e55e71 100644
--- a/internal/backend/mock/backend.go
+++ b/internal/backend/mock/backend.go
@@ -18,7 +18,6 @@ type Backend struct {
StatFn func(ctx context.Context, h restic.Handle) (restic.FileInfo, error)
ListFn func(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error
RemoveFn func(ctx context.Context, h restic.Handle) error
- TestFn func(ctx context.Context, h restic.Handle) (bool, error)
DeleteFn func(ctx context.Context) error
ConnectionsFn func() uint
LocationFn func() string
@@ -143,15 +142,6 @@ func (m *Backend) Remove(ctx context.Context, h restic.Handle) error {
return m.RemoveFn(ctx, h)
}
-// Test for the existence of a specific item.
-func (m *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
- if m.TestFn == nil {
- return false, errors.New("not implemented")
- }
-
- return m.TestFn(ctx, h)
-}
-
// Delete all data.
func (m *Backend) Delete(ctx context.Context) error {
if m.DeleteFn == nil {
diff --git a/internal/backend/paths.go b/internal/backend/paths.go
index eaa1a433a..7e511be9c 100644
--- a/internal/backend/paths.go
+++ b/internal/backend/paths.go
@@ -2,25 +2,6 @@ package backend
import "os"
-// Paths contains the default paths for file-based backends (e.g. local).
-var Paths = struct {
- Data string
- Snapshots string
- Index string
- Locks string
- Keys string
- Temp string
- Config string
-}{
- "data",
- "snapshots",
- "index",
- "locks",
- "keys",
- "tmp",
- "config",
-}
-
type Modes struct {
Dir os.FileMode
File os.FileMode
diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go
index cccc52384..085c89945 100644
--- a/internal/backend/rclone/backend.go
+++ b/internal/backend/rclone/backend.go
@@ -13,6 +13,7 @@ import (
"os"
"os/exec"
"sync"
+ "syscall"
"time"
"github.com/cenkalti/backoff/v4"
@@ -21,7 +22,6 @@ import (
"github.com/restic/restic/internal/backend/rest"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
- "golang.org/x/net/context/ctxhttp"
"golang.org/x/net/http2"
)
@@ -37,29 +37,32 @@ type Backend struct {
}
// run starts command with args and initializes the StdioConn.
-func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, func() error, error) {
+func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, chan struct{}, func() error, error) {
cmd := exec.Command(command, args...)
p, err := cmd.StderrPipe()
if err != nil {
- return nil, nil, nil, err
+ return nil, nil, nil, nil, err
}
var wg sync.WaitGroup
+ waitCh := make(chan struct{})
// start goroutine to add a prefix to all messages printed by to stderr by rclone
wg.Add(1)
go func() {
defer wg.Done()
+ defer close(waitCh)
sc := bufio.NewScanner(p)
for sc.Scan() {
fmt.Fprintf(os.Stderr, "rclone: %v\n", sc.Text())
}
+ debug.Log("command has exited, closing waitCh")
}()
r, stdin, err := os.Pipe()
if err != nil {
- return nil, nil, nil, err
+ return nil, nil, nil, nil, err
}
stdout, w, err := os.Pipe()
@@ -67,7 +70,7 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, func() er
// close first pipe and ignore subsequent errors
_ = r.Close()
_ = stdin.Close()
- return nil, nil, nil, err
+ return nil, nil, nil, nil, err
}
cmd.Stdin = r
@@ -85,7 +88,10 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, func() er
err = errW
}
if err != nil {
- return nil, nil, nil, err
+ if backend.IsErrDot(err) {
+ return nil, nil, nil, nil, errors.Errorf("cannot implicitly run relative executable %v found in current directory, use -o rclone.program=./<program> to override", cmd.Path)
+ }
+ return nil, nil, nil, nil, err
}
c := &StdioConn{
@@ -94,7 +100,7 @@ func run(command string, args ...string) (*StdioConn, *sync.WaitGroup, func() er
cmd: cmd,
}
- return c, &wg, bg, nil
+ return c, &wg, waitCh, bg, nil
}
// wrappedConn adds bandwidth limiting capabilities to the StdioConn by
@@ -158,7 +164,7 @@ func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) {
arg0, args := args[0], args[1:]
debug.Log("running command: %v %v", arg0, args)
- stdioConn, wg, bg, err := run(arg0, args...)
+ stdioConn, wg, waitCh, bg, err := run(arg0, args...)
if err != nil {
return nil, err
}
@@ -183,7 +189,6 @@ func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) {
}
cmd := stdioConn.cmd
- waitCh := make(chan struct{})
be := &Backend{
tr: tr,
cmd: cmd,
@@ -192,36 +197,25 @@ func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) {
wg: wg,
}
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
wg.Add(1)
go func() {
defer wg.Done()
- debug.Log("waiting for error result")
+ <-waitCh
+ cancel()
+
+ // according to the documentation of StdErrPipe, Wait() must only be called after the former has completed
err := cmd.Wait()
debug.Log("Wait returned %v", err)
be.waitResult = err
// close our side of the pipes to rclone, ignore errors
_ = stdioConn.CloseAll()
- close(waitCh)
- }()
-
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
-
- wg.Add(1)
- go func() {
- defer wg.Done()
- debug.Log("monitoring command to cancel first HTTP request context")
- select {
- case <-ctx.Done():
- debug.Log("context has been cancelled, returning")
- case <-be.waitCh:
- debug.Log("command has exited, cancelling context")
- cancel()
- }
}()
// send an HTTP request to the base URL, see if the server is there
- client := &http.Client{
+ client := http.Client{
Transport: debug.RoundTripper(tr),
Timeout: cfg.Timeout,
}
@@ -236,12 +230,20 @@ func newBackend(cfg Config, lim limiter.Limiter) (*Backend, error) {
}
req.Header.Set("Accept", rest.ContentTypeV2)
- res, err := ctxhttp.Do(ctx, client, req)
+ res, err := client.Do(req)
if err != nil {
// ignore subsequent errors
_ = bg()
_ = cmd.Process.Kill()
- return nil, errors.Errorf("error talking HTTP to rclone: %v", err)
+
+ // wait for rclone to exit
+ wg.Wait()
+ // try to return the program exit code if communication with rclone has failed
+ if be.waitResult != nil && (errors.Is(err, context.Canceled) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, syscall.EPIPE) || errors.Is(err, os.ErrClosed)) {
+ err = be.waitResult
+ }
+
+ return nil, fmt.Errorf("error talking HTTP to rclone: %w", err)
}
debug.Log("HTTP status %q returned, moving instance to background", res.Status)
diff --git a/internal/backend/rclone/backend_test.go b/internal/backend/rclone/backend_test.go
index 9708c6af2..12fed6274 100644
--- a/internal/backend/rclone/backend_test.go
+++ b/internal/backend/rclone/backend_test.go
@@ -13,7 +13,7 @@ import (
)
func newTestSuite(t testing.TB) *test.Suite {
- dir, cleanup := rtest.TempDir(t)
+ dir := rtest.TempDir(t)
return &test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
@@ -43,13 +43,6 @@ func newTestSuite(t testing.TB) *test.Suite {
cfg := config.(rclone.Config)
return rclone.Open(cfg, nil)
},
-
- // CleanupFn removes data created during the tests.
- Cleanup: func(config interface{}) error {
- t.Logf("cleanup dir %v", dir)
- cleanup()
- return nil
- },
}
}
diff --git a/internal/backend/rclone/internal_test.go b/internal/backend/rclone/internal_test.go
index fe9a63d30..bfec2b98c 100644
--- a/internal/backend/rclone/internal_test.go
+++ b/internal/backend/rclone/internal_test.go
@@ -12,9 +12,7 @@ import (
// restic should detect rclone exiting.
func TestRcloneExit(t *testing.T) {
- dir, cleanup := rtest.TempDir(t)
- defer cleanup()
-
+ dir := rtest.TempDir(t)
cfg := NewConfig()
cfg.Remote = dir
be, err := Open(cfg, nil)
@@ -41,3 +39,16 @@ func TestRcloneExit(t *testing.T) {
rtest.Assert(t, err != nil, "expected an error")
}
}
+
+// restic should detect rclone startup failures
+func TestRcloneFailedStart(t *testing.T) {
+ cfg := NewConfig()
+ // exits with exit code 1
+ cfg.Program = "false"
+ _, err := Open(cfg, nil)
+ var e *exec.ExitError
+ if !errors.As(err, &e) {
+ // unexpected error
+ rtest.OK(t, err)
+ }
+}
diff --git a/internal/backend/rest/config.go b/internal/backend/rest/config.go
index 51ff3b27c..388153fce 100644
--- a/internal/backend/rest/config.go
+++ b/internal/backend/rest/config.go
@@ -34,9 +34,8 @@ func ParseConfig(s string) (interface{}, error) {
s = prepareURL(s)
u, err := url.Parse(s)
-
if err != nil {
- return nil, errors.Wrap(err, "url.Parse")
+ return nil, errors.WithStack(err)
}
cfg := NewConfig()
diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go
index cd41bc0ce..f4c2897b9 100644
--- a/internal/backend/rest/rest.go
+++ b/internal/backend/rest/rest.go
@@ -6,17 +6,12 @@ import (
"fmt"
"hash"
"io"
- "io/ioutil"
"net/http"
- "net/textproto"
"net/url"
"path"
- "strconv"
"strings"
- "golang.org/x/net/context/ctxhttp"
-
- "github.com/restic/restic/internal/backend"
+ "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
@@ -33,8 +28,8 @@ type Backend struct {
url *url.URL
connections uint
sem sema.Semaphore
- client *http.Client
- backend.Layout
+ client http.Client
+ layout.Layout
}
// the REST API protocol version is decided by HTTP request headers, these are the constants.
@@ -45,8 +40,6 @@ const (
// Open opens the REST backend with the given config.
func Open(cfg Config, rt http.RoundTripper) (*Backend, error) {
- client := &http.Client{Transport: rt}
-
sem, err := sema.New(cfg.Connections)
if err != nil {
return nil, err
@@ -60,8 +53,8 @@ func Open(cfg Config, rt http.RoundTripper) (*Backend, error) {
be := &Backend{
url: cfg.URL,
- client: client,
- Layout: &backend.RESTLayout{URL: url, Join: path.Join},
+ client: http.Client{Transport: rt},
+ Layout: &layout.RESTLayout{URL: url, Join: path.Join},
connections: cfg.Connections,
sem: sem,
}
@@ -95,7 +88,7 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er
return nil, errors.Fatalf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode)
}
- _, err = io.Copy(ioutil.Discard, resp.Body)
+ _, err = io.Copy(io.Discard, resp.Body)
if err != nil {
return nil, err
}
@@ -138,9 +131,10 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
defer cancel()
// make sure that client.Post() cannot close the reader by wrapping it
- req, err := http.NewRequest(http.MethodPost, b.Filename(h), ioutil.NopCloser(rd))
+ req, err := http.NewRequestWithContext(ctx,
+ http.MethodPost, b.Filename(h), io.NopCloser(rd))
if err != nil {
- return errors.Wrap(err, "NewRequest")
+ return errors.WithStack(err)
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Accept", ContentTypeV2)
@@ -150,17 +144,17 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
req.ContentLength = rd.Length()
b.sem.GetToken()
- resp, err := ctxhttp.Do(ctx, b.client, req)
+ resp, err := b.client.Do(req)
b.sem.ReleaseToken()
var cerr error
if resp != nil {
- _, _ = io.Copy(ioutil.Discard, resp.Body)
+ _, _ = io.Copy(io.Discard, resp.Body)
cerr = resp.Body.Close()
}
if err != nil {
- return errors.Wrap(err, "client.Post")
+ return errors.WithStack(err)
}
if resp.StatusCode != 200 {
@@ -217,44 +211,6 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
return err
}
-// checkContentLength returns an error if the server returned a value in the
-// Content-Length header in an HTTP2 connection, but closed the connection
-// before any data was sent.
-//
-// This is a workaround for https://github.com/golang/go/issues/46071
-//
-// See also https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10
-func checkContentLength(resp *http.Response) error {
- // the following code is based on
- // https://github.com/golang/go/blob/b7a85e0003cedb1b48a1fd3ae5b746ec6330102e/src/net/http/h2_bundle.go#L8646
-
- if resp.ContentLength != 0 {
- return nil
- }
-
- if resp.ProtoMajor != 2 && resp.ProtoMinor != 0 {
- return nil
- }
-
- if len(resp.Header[textproto.CanonicalMIMEHeaderKey("Content-Length")]) != 1 {
- return nil
- }
-
- // make sure that if the server returned a content length and we can
- // parse it, it is really zero, otherwise return an error
- contentLength := resp.Header.Get("Content-Length")
- cl, err := strconv.ParseUint(contentLength, 10, 63)
- if err != nil {
- return fmt.Errorf("unable to parse Content-Length %q: %w", contentLength, err)
- }
-
- if cl != 0 {
- return errors.Errorf("unexpected EOF: got 0 instead of %v bytes", cl)
- }
-
- return nil
-}
-
func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Load %v, length %v, offset %v", h, length, offset)
if err := h.Valid(); err != nil {
@@ -269,9 +225,9 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
return nil, errors.Errorf("invalid length %d", length)
}
- req, err := http.NewRequest("GET", b.Filename(h), nil)
+ req, err := http.NewRequestWithContext(ctx, "GET", b.Filename(h), nil)
if err != nil {
- return nil, errors.Wrap(err, "http.NewRequest")
+ return nil, errors.WithStack(err)
}
byteRange := fmt.Sprintf("bytes=%d-", offset)
@@ -283,12 +239,12 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
debug.Log("Load(%v) send range %v", h, byteRange)
b.sem.GetToken()
- resp, err := ctxhttp.Do(ctx, b.client, req)
+ resp, err := b.client.Do(req)
b.sem.ReleaseToken()
if err != nil {
if resp != nil {
- _, _ = io.Copy(ioutil.Discard, resp.Body)
+ _, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}
return nil, errors.Wrap(err, "client.Do")
@@ -304,14 +260,6 @@ func (b *Backend) openReader(ctx context.Context, h restic.Handle, length int, o
return nil, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status)
}
- // workaround https://github.com/golang/go/issues/46071
- // see also https://forum.restic.net/t/http2-stream-closed-connection-reset-context-canceled/3743/10
- err = checkContentLength(resp)
- if err != nil {
- _ = resp.Body.Close()
- return nil, err
- }
-
return resp.Body, nil
}
@@ -321,20 +269,20 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e
return restic.FileInfo{}, backoff.Permanent(err)
}
- req, err := http.NewRequest(http.MethodHead, b.Filename(h), nil)
+ req, err := http.NewRequestWithContext(ctx, http.MethodHead, b.Filename(h), nil)
if err != nil {
- return restic.FileInfo{}, errors.Wrap(err, "NewRequest")
+ return restic.FileInfo{}, errors.WithStack(err)
}
req.Header.Set("Accept", ContentTypeV2)
b.sem.GetToken()
- resp, err := ctxhttp.Do(ctx, b.client, req)
+ resp, err := b.client.Do(req)
b.sem.ReleaseToken()
if err != nil {
- return restic.FileInfo{}, errors.Wrap(err, "client.Head")
+ return restic.FileInfo{}, errors.WithStack(err)
}
- _, _ = io.Copy(ioutil.Discard, resp.Body)
+ _, _ = io.Copy(io.Discard, resp.Body)
if err = resp.Body.Close(); err != nil {
return restic.FileInfo{}, errors.Wrap(err, "Close")
}
@@ -360,30 +308,20 @@ func (b *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, e
return bi, nil
}
-// Test returns true if a blob of the given type and name exists in the backend.
-func (b *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
- _, err := b.Stat(ctx, h)
- if err != nil {
- return false, nil
- }
-
- return true, nil
-}
-
// Remove removes the blob with the given name and type.
func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
if err := h.Valid(); err != nil {
return backoff.Permanent(err)
}
- req, err := http.NewRequest("DELETE", b.Filename(h), nil)
+ req, err := http.NewRequestWithContext(ctx, "DELETE", b.Filename(h), nil)
if err != nil {
- return errors.Wrap(err, "http.NewRequest")
+ return errors.WithStack(err)
}
req.Header.Set("Accept", ContentTypeV2)
b.sem.GetToken()
- resp, err := ctxhttp.Do(ctx, b.client, req)
+ resp, err := b.client.Do(req)
b.sem.ReleaseToken()
if err != nil {
@@ -399,7 +337,7 @@ func (b *Backend) Remove(ctx context.Context, h restic.Handle) error {
return errors.Errorf("blob not removed, server response: %v (%v)", resp.Status, resp.StatusCode)
}
- _, err = io.Copy(ioutil.Discard, resp.Body)
+ _, err = io.Copy(io.Discard, resp.Body)
if err != nil {
return errors.Wrap(err, "Copy")
}
@@ -415,14 +353,14 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi
url += "/"
}
- req, err := http.NewRequest(http.MethodGet, url, nil)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
- return errors.Wrap(err, "NewRequest")
+ return errors.WithStack(err)
}
req.Header.Set("Accept", ContentTypeV2)
b.sem.GetToken()
- resp, err := ctxhttp.Do(ctx, b.client, req)
+ resp, err := b.client.Do(req)
b.sem.ReleaseToken()
if err != nil {
diff --git a/internal/backend/rest/rest_int_go114_test.go b/internal/backend/rest/rest_int_go114_test.go
deleted file mode 100644
index 7c040bf51..000000000
--- a/internal/backend/rest/rest_int_go114_test.go
+++ /dev/null
@@ -1,72 +0,0 @@
-//go:build go1.14 && !go1.18
-// +build go1.14,!go1.18
-
-// missing eof error is fixed in golang >= 1.17.3 or >= 1.16.10
-// remove the workaround from rest.go when the minimum golang version
-// supported by restic reaches 1.18.
-
-package rest_test
-
-import (
- "context"
- "io"
- "io/ioutil"
- "net/http"
- "net/http/httptest"
- "net/url"
- "testing"
-
- "github.com/restic/restic/internal/backend/rest"
- "github.com/restic/restic/internal/restic"
-)
-
-func TestZeroLengthRead(t *testing.T) {
- // Test workaround for https://github.com/golang/go/issues/46071. Can be removed once this is fixed in Go
- // and the minimum golang version supported by restic includes the fix.
- numRequests := 0
- srv := httptest.NewUnstartedServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
- numRequests++
- t.Logf("req %v %v", req.Method, req.URL.Path)
- if req.Method == "GET" {
- res.Header().Set("Content-Length", "42")
- // Now the handler fails for some reason and is unable to send data
- return
- }
-
- t.Errorf("unhandled request %v %v", req.Method, req.URL.Path)
- }))
- srv.EnableHTTP2 = true
- srv.StartTLS()
- defer srv.Close()
-
- srvURL, err := url.Parse(srv.URL)
- if err != nil {
- t.Fatal(err)
- }
-
- cfg := rest.Config{
- Connections: 5,
- URL: srvURL,
- }
- be, err := rest.Open(cfg, srv.Client().Transport)
- if err != nil {
- t.Fatal(err)
- }
- defer func() {
- err = be.Close()
- if err != nil {
- t.Fatal(err)
- }
- }()
-
- err = be.Load(context.TODO(), restic.Handle{Type: restic.ConfigFile}, 0, 0, func(rd io.Reader) error {
- _, err := ioutil.ReadAll(rd)
- if err == nil {
- t.Fatal("ReadAll should have returned an 'Unexpected EOF' error")
- }
- return nil
- })
- if err == nil {
- t.Fatal("Got no unexpected EOF error")
- }
-}
diff --git a/internal/backend/rest/rest_test.go b/internal/backend/rest/rest_test.go
index 45aeabb99..a473e4440 100644
--- a/internal/backend/rest/rest_test.go
+++ b/internal/backend/rest/rest_test.go
@@ -112,9 +112,7 @@ func TestBackendREST(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- dir, cleanup := rtest.TempDir(t)
- defer cleanup()
-
+ dir := rtest.TempDir(t)
serverURL, cleanup := runRESTServer(ctx, t, dir)
defer cleanup()
@@ -144,9 +142,7 @@ func BenchmarkBackendREST(t *testing.B) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- dir, cleanup := rtest.TempDir(t)
- defer cleanup()
-
+ dir := rtest.TempDir(t)
serverURL, cleanup := runRESTServer(ctx, t, dir)
defer cleanup()
diff --git a/internal/backend/backend_retry.go b/internal/backend/retry/backend_retry.go
similarity index 54%
rename from internal/backend/backend_retry.go
rename to internal/backend/retry/backend_retry.go
index 5d7a58b64..b5f2706f4 100644
--- a/internal/backend/backend_retry.go
+++ b/internal/backend/retry/backend_retry.go
@@ -1,4 +1,4 @@
-package backend
+package retry
import (
"context"
@@ -11,28 +11,53 @@ import (
"github.com/restic/restic/internal/restic"
)
-// RetryBackend retries operations on the backend in case of an error with a
+// Backend retries operations on the backend in case of an error with a
// backoff.
-type RetryBackend struct {
+type Backend struct {
restic.Backend
MaxTries int
Report func(string, error, time.Duration)
+ Success func(string, int)
}
// statically ensure that RetryBackend implements restic.Backend.
-var _ restic.Backend = &RetryBackend{}
+var _ restic.Backend = &Backend{}
-// NewRetryBackend wraps be with a backend that retries operations after a
+// New wraps be with a backend that retries operations after a
// backoff. report is called with a description and the error, if one occurred.
-func NewRetryBackend(be restic.Backend, maxTries int, report func(string, error, time.Duration)) *RetryBackend {
- return &RetryBackend{
+// success is called with the number of retries before a successful operation
+// (it is not called if it succeeded on the first try)
+func New(be restic.Backend, maxTries int, report func(string, error, time.Duration), success func(string, int)) *Backend {
+ return &Backend{
Backend: be,
MaxTries: maxTries,
Report: report,
+ Success: success,
}
}
-func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) error {
+// retryNotifyErrorWithSuccess is an extension of backoff.RetryNotify with notification of success after an error.
+// success is NOT notified on the first run of operation (only after an error).
+func retryNotifyErrorWithSuccess(operation backoff.Operation, b backoff.BackOff, notify backoff.Notify, success func(retries int)) error {
+ if success == nil {
+ return backoff.RetryNotify(operation, b, notify)
+ }
+ retries := 0
+ operationWrapper := func() error {
+ err := operation()
+ if err != nil {
+ retries++
+ } else if retries > 0 {
+ success(retries)
+ }
+ return err
+ }
+ return backoff.RetryNotify(operationWrapper, b, notify)
+}
+
+var fastRetries = false
+
+func (be *Backend) retry(ctx context.Context, msg string, f func() error) error {
// Don't do anything when called with an already cancelled context. There would be
// no retries in that case either, so be consistent and abort always.
// This enforces a strict contract for backend methods: Using a cancelled context
@@ -43,20 +68,31 @@ func (be *RetryBackend) retry(ctx context.Context, msg string, f func() error) e
return ctx.Err()
}
- err := backoff.RetryNotify(f,
- backoff.WithContext(backoff.WithMaxRetries(backoff.NewExponentialBackOff(), uint64(be.MaxTries)), ctx),
+ bo := backoff.NewExponentialBackOff()
+ if fastRetries {
+ // speed up integration tests
+ bo.InitialInterval = 1 * time.Millisecond
+ }
+
+ err := retryNotifyErrorWithSuccess(f,
+ backoff.WithContext(backoff.WithMaxRetries(bo, uint64(be.MaxTries)), ctx),
func(err error, d time.Duration) {
if be.Report != nil {
be.Report(msg, err, d)
}
},
+ func(retries int) {
+ if be.Success != nil {
+ be.Success(msg, retries)
+ }
+ },
)
return err
}
// Save stores the data in the backend under the given handle.
-func (be *RetryBackend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
return be.retry(ctx, fmt.Sprintf("Save(%v)", h), func() error {
err := rd.Rewind()
if err != nil {
@@ -68,10 +104,16 @@ func (be *RetryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rew
return nil
}
- debug.Log("Save(%v) failed with error, removing file: %v", h, err)
- rerr := be.Backend.Remove(ctx, h)
- if rerr != nil {
- debug.Log("Remove(%v) returned error: %v", h, err)
+ if be.Backend.HasAtomicReplace() {
+ debug.Log("Save(%v) failed with error: %v", h, err)
+ // there is no need to remove files from backends which can atomically replace files
+ // in fact if something goes wrong at the backend side the delete operation might delete the wrong instance of the file
+ } else {
+ debug.Log("Save(%v) failed with error, removing file: %v", h, err)
+ rerr := be.Backend.Remove(ctx, h)
+ if rerr != nil {
+ debug.Log("Remove(%v) returned error: %v", h, err)
+ }
}
// return original error
@@ -83,7 +125,7 @@ func (be *RetryBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rew
// given offset. If length is larger than zero, only a portion of the file
// is returned. rd must be closed after use. If an error is returned, the
// ReadCloser must be nil.
-func (be *RetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
+func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (err error) {
return be.retry(ctx, fmt.Sprintf("Load(%v, %v, %v)", h, length, offset),
func() error {
return be.Backend.Load(ctx, h, length, offset, consumer)
@@ -91,40 +133,33 @@ func (be *RetryBackend) Load(ctx context.Context, h restic.Handle, length int, o
}
// Stat returns information about the File identified by h.
-func (be *RetryBackend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) {
+func (be *Backend) Stat(ctx context.Context, h restic.Handle) (fi restic.FileInfo, err error) {
err = be.retry(ctx, fmt.Sprintf("Stat(%v)", h),
func() error {
var innerError error
fi, innerError = be.Backend.Stat(ctx, h)
+ if be.Backend.IsNotExist(innerError) {
+ // do not retry if file is not found, as stat is usually used to check whether a file exists
+ return backoff.Permanent(innerError)
+ }
return innerError
})
return fi, err
}
// Remove removes a File with type t and name.
-func (be *RetryBackend) Remove(ctx context.Context, h restic.Handle) (err error) {
+func (be *Backend) Remove(ctx context.Context, h restic.Handle) (err error) {
return be.retry(ctx, fmt.Sprintf("Remove(%v)", h), func() error {
return be.Backend.Remove(ctx, h)
})
}
-// Test a boolean value whether a File with the name and type exists.
-func (be *RetryBackend) Test(ctx context.Context, h restic.Handle) (exists bool, err error) {
- err = be.retry(ctx, fmt.Sprintf("Test(%v)", h), func() error {
- var innerError error
- exists, innerError = be.Backend.Test(ctx, h)
-
- return innerError
- })
- return exists, err
-}
-
// List runs fn for each file in the backend which has the type t. When an
// error is returned by the underlying backend, the request is retried. When fn
// returns an error, the operation is aborted and the error is returned to the
// caller.
-func (be *RetryBackend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
+func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
// create a new context that we can cancel when fn returns an error, so
// that listing is aborted
listCtx, cancel := context.WithCancel(ctx)
diff --git a/internal/backend/backend_retry_test.go b/internal/backend/retry/backend_retry_test.go
similarity index 66%
rename from internal/backend/backend_retry_test.go
rename to internal/backend/retry/backend_retry_test.go
index e8f4d7315..9f2f39589 100644
--- a/internal/backend/backend_retry_test.go
+++ b/internal/backend/retry/backend_retry_test.go
@@ -1,12 +1,13 @@
-package backend
+package retry
import (
"bytes"
"context"
"io"
- "io/ioutil"
"testing"
+ "time"
+ "github.com/cenkalti/backoff/v4"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
@@ -20,7 +21,7 @@ func TestBackendSaveRetry(t *testing.T) {
SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
if errcount == 0 {
errcount++
- _, err := io.CopyN(ioutil.Discard, rd, 120)
+ _, err := io.CopyN(io.Discard, rd, 120)
if err != nil {
return err
}
@@ -33,7 +34,8 @@ func TestBackendSaveRetry(t *testing.T) {
},
}
- retryBackend := NewRetryBackend(be, 10, nil)
+ TestFastRetries(t)
+ retryBackend := New(be, 10, nil, nil)
data := test.Random(23, 5*1024*1024+11241)
err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher()))
@@ -50,6 +52,37 @@ func TestBackendSaveRetry(t *testing.T) {
}
}
+func TestBackendSaveRetryAtomic(t *testing.T) {
+ errcount := 0
+ calledRemove := false
+ be := &mock.Backend{
+ SaveFn: func(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+ if errcount == 0 {
+ errcount++
+ return errors.New("injected error")
+ }
+ return nil
+ },
+ RemoveFn: func(ctx context.Context, h restic.Handle) error {
+ calledRemove = true
+ return nil
+ },
+ HasAtomicReplaceFn: func() bool { return true },
+ }
+
+ TestFastRetries(t)
+ retryBackend := New(be, 10, nil, nil)
+
+ data := test.Random(23, 5*1024*1024+11241)
+ err := retryBackend.Save(context.TODO(), restic.Handle{}, restic.NewByteReader(data, be.Hasher()))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if calledRemove {
+ t.Fatal("remove must not be called")
+ }
+}
+
func TestBackendListRetry(t *testing.T) {
const (
ID1 = "id1"
@@ -71,7 +104,8 @@ func TestBackendListRetry(t *testing.T) {
},
}
- retryBackend := NewRetryBackend(be, 10, nil)
+ TestFastRetries(t)
+ retryBackend := New(be, 10, nil, nil)
var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
@@ -100,7 +134,8 @@ func TestBackendListRetryErrorFn(t *testing.T) {
},
}
- retryBackend := NewRetryBackend(be, 10, nil)
+ TestFastRetries(t)
+ retryBackend := New(be, 10, nil, nil)
var ErrTest = errors.New("test error")
@@ -155,8 +190,9 @@ func TestBackendListRetryErrorBackend(t *testing.T) {
},
}
+ TestFastRetries(t)
const maxRetries = 2
- retryBackend := NewRetryBackend(be, maxRetries, nil)
+ retryBackend := New(be, maxRetries, nil, nil)
var listed []string
err := retryBackend.List(context.TODO(), restic.PackFile, func(fi restic.FileInfo) error {
@@ -225,11 +261,12 @@ func TestBackendLoadRetry(t *testing.T) {
return failingReader{data: data, limit: limit}, nil
}
- retryBackend := NewRetryBackend(be, 10, nil)
+ TestFastRetries(t)
+ retryBackend := New(be, 10, nil, nil)
var buf []byte
err := retryBackend.Load(context.TODO(), restic.Handle{}, 0, 0, func(rd io.Reader) (err error) {
- buf, err = ioutil.ReadAll(rd)
+ buf, err = io.ReadAll(rd)
return err
})
test.OK(t, err)
@@ -237,6 +274,32 @@ func TestBackendLoadRetry(t *testing.T) {
test.Equals(t, 2, attempt)
}
+func TestBackendStatNotExists(t *testing.T) {
+ // stat should not retry if the error matches IsNotExist
+ notFound := errors.New("not found")
+ attempt := 0
+
+ be := mock.NewBackend()
+ be.StatFn = func(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
+ attempt++
+ if attempt > 1 {
+ t.Fail()
+ return restic.FileInfo{}, errors.New("must not retry")
+ }
+ return restic.FileInfo{}, notFound
+ }
+ be.IsNotExistFn = func(err error) bool {
+ return errors.Is(err, notFound)
+ }
+
+ TestFastRetries(t)
+ retryBackend := New(be, 10, nil, nil)
+
+ _, err := retryBackend.Stat(context.TODO(), restic.Handle{})
+ test.Assert(t, be.IsNotExistFn(err), "unexpected error %v", err)
+ test.Equals(t, 1, attempt)
+}
+
func assertIsCanceled(t *testing.T, err error) {
test.Assert(t, err == context.Canceled, "got unexpected err %v", err)
}
@@ -244,16 +307,15 @@ func assertIsCanceled(t *testing.T, err error) {
func TestBackendCanceledContext(t *testing.T) {
// unimplemented mock backend functions return an error by default
// check that we received the expected context canceled error instead
- retryBackend := NewRetryBackend(mock.NewBackend(), 2, nil)
+ TestFastRetries(t)
+ retryBackend := New(mock.NewBackend(), 2, nil, nil)
h := restic.Handle{Type: restic.PackFile, Name: restic.NewRandomID().String()}
// create an already canceled context
ctx, cancel := context.WithCancel(context.Background())
cancel()
- _, err := retryBackend.Test(ctx, h)
- assertIsCanceled(t, err)
- _, err = retryBackend.Stat(ctx, h)
+ _, err := retryBackend.Stat(ctx, h)
assertIsCanceled(t, err)
err = retryBackend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
@@ -271,3 +333,56 @@ func TestBackendCanceledContext(t *testing.T) {
// don't test "Delete" as it is not used by normal code
}
+
+func TestNotifyWithSuccessIsNotCalled(t *testing.T) {
+ operation := func() error {
+ return nil
+ }
+
+ notify := func(error, time.Duration) {
+ t.Fatal("Notify should not have been called")
+ }
+
+ success := func(retries int) {
+ t.Fatal("Success should not have been called")
+ }
+
+ err := retryNotifyErrorWithSuccess(operation, &backoff.ZeroBackOff{}, notify, success)
+ if err != nil {
+ t.Fatal("retry should not have returned an error")
+ }
+}
+
+func TestNotifyWithSuccessIsCalled(t *testing.T) {
+ operationCalled := 0
+ operation := func() error {
+ operationCalled++
+ if operationCalled <= 2 {
+ return errors.New("expected error in test")
+ }
+ return nil
+ }
+
+ notifyCalled := 0
+ notify := func(error, time.Duration) {
+ notifyCalled++
+ }
+
+ successCalled := 0
+ success := func(retries int) {
+ successCalled++
+ }
+
+ err := retryNotifyErrorWithSuccess(operation, &backoff.ZeroBackOff{}, notify, success)
+ if err != nil {
+ t.Fatal("retry should not have returned an error")
+ }
+
+ if notifyCalled != 2 {
+ t.Fatalf("Notify should have been called 2 times, but was called %d times instead", notifyCalled)
+ }
+
+ if successCalled != 1 {
+ t.Fatalf("Success should have been called only once, but was called %d times instead", successCalled)
+ }
+}
diff --git a/internal/backend/retry/testing.go b/internal/backend/retry/testing.go
new file mode 100644
index 000000000..797573b03
--- /dev/null
+++ b/internal/backend/retry/testing.go
@@ -0,0 +1,8 @@
+package retry
+
+import "testing"
+
+// TestFastRetries reduces the initial retry delay to 1 millisecond
+func TestFastRetries(t testing.TB) {
+ fastRetries = true
+}
diff --git a/internal/backend/s3/config.go b/internal/backend/s3/config.go
index 9e83f4004..9050e20f4 100644
--- a/internal/backend/s3/config.go
+++ b/internal/backend/s3/config.go
@@ -52,15 +52,15 @@ func ParseConfig(s string) (interface{}, error) {
// bucket name and prefix
url, err := url.Parse(s[3:])
if err != nil {
- return nil, errors.Wrap(err, "url.Parse")
+ return nil, errors.WithStack(err)
}
if url.Path == "" {
return nil, errors.New("s3: bucket name not found")
}
- path := strings.SplitN(url.Path[1:], "/", 2)
- return createConfig(url.Host, path, url.Scheme == "http")
+ bucket, path, _ := strings.Cut(url.Path[1:], "/")
+ return createConfig(url.Host, bucket, path, url.Scheme == "http")
case strings.HasPrefix(s, "s3://"):
s = s[5:]
case strings.HasPrefix(s, "s3:"):
@@ -70,24 +70,24 @@ func ParseConfig(s string) (interface{}, error) {
}
// use the first entry of the path as the endpoint and the
// remainder as bucket name and prefix
- path := strings.SplitN(s, "/", 3)
- return createConfig(path[0], path[1:], false)
+ endpoint, rest, _ := strings.Cut(s, "/")
+ bucket, prefix, _ := strings.Cut(rest, "/")
+ return createConfig(endpoint, bucket, prefix, false)
}
-func createConfig(endpoint string, p []string, useHTTP bool) (interface{}, error) {
- if len(p) < 1 {
+func createConfig(endpoint, bucket, prefix string, useHTTP bool) (interface{}, error) {
+ if endpoint == "" {
return nil, errors.New("s3: invalid format, host/region or bucket name not found")
}
- var prefix string
- if len(p) > 1 && p[1] != "" {
- prefix = path.Clean(p[1])
+ if prefix != "" {
+ prefix = path.Clean(prefix)
}
cfg := NewConfig()
cfg.Endpoint = endpoint
cfg.UseHTTP = useHTTP
- cfg.Bucket = p[0]
+ cfg.Bucket = bucket
cfg.Prefix = prefix
return cfg, nil
}
diff --git a/internal/backend/s3/config_test.go b/internal/backend/s3/config_test.go
index 77a31fda3..821fbc244 100644
--- a/internal/backend/s3/config_test.go
+++ b/internal/backend/s3/config_test.go
@@ -1,6 +1,9 @@
package s3
-import "testing"
+import (
+ "strings"
+ "testing"
+)
var configTests = []struct {
s string
@@ -111,3 +114,14 @@ func TestParseConfig(t *testing.T) {
}
}
}
+
+func TestParseError(t *testing.T) {
+ const prefix = "s3: invalid format,"
+
+ for _, s := range []string{"", "/", "//", "/bucket/prefix"} {
+ _, err := ParseConfig("s3://" + s)
+ if err == nil || !strings.HasPrefix(err.Error(), prefix) {
+ t.Errorf("expected %q, got %q", prefix, err)
+ }
+ }
+}
diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go
index 0b3816c06..ad652a206 100644
--- a/internal/backend/s3/s3.go
+++ b/internal/backend/s3/s3.go
@@ -5,7 +5,6 @@ import (
"fmt"
"hash"
"io"
- "io/ioutil"
"net/http"
"os"
"path"
@@ -13,6 +12,7 @@ import (
"time"
"github.com/restic/restic/internal/backend"
+ "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
@@ -28,7 +28,7 @@ type Backend struct {
client *minio.Client
sem sema.Semaphore
cfg Config
- backend.Layout
+ layout.Layout
}
// make sure that *Backend implements backend.Backend
@@ -113,7 +113,7 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro
cfg: cfg,
}
- l, err := backend.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix)
+ l, err := layout.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix)
if err != nil {
return nil, err
}
@@ -164,15 +164,12 @@ func isAccessDenied(err error) bool {
debug.Log("isAccessDenied(%T, %#v)", err, err)
var e minio.ErrorResponse
- return errors.As(err, &e) && e.Code == "Access Denied"
+ return errors.As(err, &e) && e.Code == "AccessDenied"
}
// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
debug.Log("IsNotExist(%T, %#v)", err, err)
- if errors.Is(err, os.ErrNotExist) {
- return true
- }
var e minio.ErrorResponse
return errors.As(err, &e) && e.Code == "NoSuchKey"
@@ -295,7 +292,7 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRe
opts.PartSize = 200 * 1024 * 1024
debug.Log("PutObject(%v, %v, %v)", be.cfg.Bucket, objName, rd.Length())
- info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, ioutil.NopCloser(rd), int64(rd.Length()), opts)
+ info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, io.NopCloser(rd), int64(rd.Length()), opts)
debug.Log("%v -> %v bytes, err %#v: %v", objName, info.Size, err, err)
@@ -392,23 +389,6 @@ func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
return restic.FileInfo{Size: fi.Size, Name: h.Name}, nil
}
-// Test returns true if a blob of the given type and name exists in the backend.
-func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
- found := false
- objName := be.Filename(h)
-
- be.sem.GetToken()
- _, err := be.client.StatObject(ctx, be.cfg.Bucket, objName, minio.StatObjectOptions{})
- be.sem.ReleaseToken()
-
- if err == nil {
- found = true
- }
-
- // If error, then not found
- return found, nil
-}
-
// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
objName := be.Filename(h)
@@ -514,7 +494,7 @@ func (be *Backend) Delete(ctx context.Context) error {
func (be *Backend) Close() error { return nil }
// Rename moves a file based on the new layout l.
-func (be *Backend) Rename(ctx context.Context, h restic.Handle, l backend.Layout) error {
+func (be *Backend) Rename(ctx context.Context, h restic.Handle, l layout.Layout) error {
debug.Log("Rename %v to %v", h, l)
oldname := be.Filename(h)
newname := l.Filename(h)
diff --git a/internal/backend/s3/s3_test.go b/internal/backend/s3/s3_test.go
index 46d96fcbd..c024251a9 100644
--- a/internal/backend/s3/s3_test.go
+++ b/internal/backend/s3/s3_test.go
@@ -101,9 +101,8 @@ func newRandomCredentials(t testing.TB) (key, secret string) {
type MinioTestConfig struct {
s3.Config
- tempdir string
- removeTempdir func()
- stopServer func()
+ tempdir string
+ stopServer func()
}
func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be restic.Backend, err error) {
@@ -132,7 +131,7 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
NewConfig: func() (interface{}, error) {
cfg := MinioTestConfig{}
- cfg.tempdir, cfg.removeTempdir = rtest.TempDir(t)
+ cfg.tempdir = rtest.TempDir(t)
key, secret := newRandomCredentials(t)
cfg.stopServer = runMinio(ctx, t, cfg.tempdir, key, secret)
@@ -155,12 +154,12 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
return nil, err
}
- exists, err := be.Test(ctx, restic.Handle{Type: restic.ConfigFile})
- if err != nil {
+ _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+ if err != nil && !be.IsNotExist(err) {
return nil, err
}
- if exists {
+ if err == nil {
return nil, errors.New("config already exists")
}
@@ -179,9 +178,6 @@ func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
if cfg.stopServer != nil {
cfg.stopServer()
}
- if cfg.removeTempdir != nil {
- cfg.removeTempdir()
- }
return nil
},
}
@@ -254,12 +250,12 @@ func newS3TestSuite(t testing.TB) *test.Suite {
return nil, err
}
- exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
- if err != nil {
+ _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+ if err != nil && !be.IsNotExist(err) {
return nil, err
}
- if exists {
+ if err == nil {
return nil, errors.New("config already exists")
}
diff --git a/internal/backend/sftp/config.go b/internal/backend/sftp/config.go
index ec38ee467..76d6d145d 100644
--- a/internal/backend/sftp/config.go
+++ b/internal/backend/sftp/config.go
@@ -42,7 +42,7 @@ func ParseConfig(s string) (interface{}, error) {
// parse the "sftp://user@host/path" url format
url, err := url.Parse(s)
if err != nil {
- return nil, errors.Wrap(err, "url.Parse")
+ return nil, errors.WithStack(err)
}
if url.User != nil {
user = url.User.Username()
@@ -60,14 +60,13 @@ func ParseConfig(s string) (interface{}, error) {
// "user@host:path" in s
s = s[5:]
// split user@host and path at the colon
- data := strings.SplitN(s, ":", 2)
- if len(data) < 2 {
+ var colon bool
+ host, dir, colon = strings.Cut(s, ":")
+ if !colon {
return nil, errors.New("sftp: invalid format, hostname or path not found")
}
- host = data[0]
- dir = data[1]
// split user and host at the "@"
- data = strings.SplitN(host, "@", 3)
+ data := strings.SplitN(host, "@", 3)
if len(data) == 3 {
user = data[0] + "@" + data[1]
host = data[2]
diff --git a/internal/backend/sftp/layout_test.go b/internal/backend/sftp/layout_test.go
index 3b654b1bb..fc8d80928 100644
--- a/internal/backend/sftp/layout_test.go
+++ b/internal/backend/sftp/layout_test.go
@@ -16,8 +16,7 @@ func TestLayout(t *testing.T) {
t.Skip("sftp server binary not available")
}
- path, cleanup := rtest.TempDir(t)
- defer cleanup()
+ path := rtest.TempDir(t)
var tests = []struct {
filename string
diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go
index 0aa2700a6..514dd58da 100644
--- a/internal/backend/sftp/sftp.go
+++ b/internal/backend/sftp/sftp.go
@@ -14,6 +14,7 @@ import (
"time"
"github.com/restic/restic/internal/backend"
+ "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
@@ -35,7 +36,7 @@ type SFTP struct {
posixRename bool
sem sema.Semaphore
- backend.Layout
+ layout.Layout
Config
backend.Modes
}
@@ -44,7 +45,12 @@ var _ restic.Backend = &SFTP{}
const defaultLayout = "default"
-func startClient(program string, args ...string) (*SFTP, error) {
+func startClient(cfg Config) (*SFTP, error) {
+ program, args, err := buildSSHCommand(cfg)
+ if err != nil {
+ return nil, err
+ }
+
debug.Log("start client %v %v", program, args)
// Connect to a remote host and request the sftp subsystem via the 'ssh'
// command. This assumes that passwordless login is correctly configured.
@@ -75,7 +81,10 @@ func startClient(program string, args ...string) (*SFTP, error) {
bg, err := backend.StartForeground(cmd)
if err != nil {
- return nil, errors.Wrap(err, "cmd.Start")
+ if backend.IsErrDot(err) {
+ return nil, errors.Errorf("cannot implicitly run relative executable %v found in current directory, use -o sftp.command=./<command> to override", cmd.Path)
+ }
+ return nil, err
}
// wait in a different goroutine
@@ -121,30 +130,29 @@ func (r *SFTP) clientError() error {
func Open(ctx context.Context, cfg Config) (*SFTP, error) {
debug.Log("open backend with config %#v", cfg)
- sem, err := sema.New(cfg.Connections)
+ sftp, err := startClient(cfg)
if err != nil {
+ debug.Log("unable to start program: %v", err)
return nil, err
}
- cmd, args, err := buildSSHCommand(cfg)
- if err != nil {
- return nil, err
- }
+ return open(ctx, sftp, cfg)
+}
- sftp, err := startClient(cmd, args...)
+func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) {
+ sem, err := sema.New(cfg.Connections)
if err != nil {
- debug.Log("unable to start program: %v", err)
return nil, err
}
- sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
+ sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
debug.Log("layout: %v\n", sftp.Layout)
- fi, err := sftp.c.Stat(Join(cfg.Path, backend.Paths.Config))
+ fi, err := sftp.c.Stat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile}))
m := backend.DeriveModesFromFileInfo(fi, err)
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
@@ -230,18 +238,13 @@ func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
// Create creates an sftp backend as described by the config by running "ssh"
// with the appropriate arguments (or cfg.Command, if set).
func Create(ctx context.Context, cfg Config) (*SFTP, error) {
- cmd, args, err := buildSSHCommand(cfg)
- if err != nil {
- return nil, err
- }
-
- sftp, err := startClient(cmd, args...)
+ sftp, err := startClient(cfg)
if err != nil {
debug.Log("unable to start program: %v", err)
return nil, err
}
- sftp.Layout, err = backend.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
+ sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path)
if err != nil {
return nil, err
}
@@ -249,7 +252,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
sftp.Modes = backend.DefaultModes
// test if config file already exists
- _, err = sftp.c.Lstat(Join(cfg.Path, backend.Paths.Config))
+ _, err = sftp.c.Lstat(sftp.Layout.Filename(restic.Handle{Type: restic.ConfigFile}))
if err == nil {
return nil, errors.New("config file already exists")
}
@@ -259,13 +262,8 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
return nil, err
}
- err = sftp.Close()
- if err != nil {
- return nil, errors.Wrap(err, "Close")
- }
-
- // open backend
- return Open(ctx, cfg)
+ // repurpose existing connection
+ return open(ctx, sftp, cfg)
}
func (r *SFTP) Connections() uint {
@@ -356,14 +354,13 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader
debug.Log("sftp: failed to remove broken file %v: %v",
f.Name(), rmErr)
}
-
- err = r.checkNoSpace(dirname, rd.Length(), err)
}()
// save data, make sure to use the optimized sftp upload method
wbytes, err := f.ReadFrom(rd)
if err != nil {
_ = f.Close()
+ err = r.checkNoSpace(dirname, rd.Length(), err)
return errors.Wrap(err, "Write")
}
@@ -405,7 +402,7 @@ func (r *SFTP) checkNoSpace(dir string, size int64, origErr error) error {
debug.Log("sftp: StatVFS returned %v", err)
return origErr
}
- if fsinfo.Favail == 0 || fsinfo.FreeSpace() < uint64(size) {
+ if fsinfo.Favail == 0 || fsinfo.Frsize*fsinfo.Bavail < uint64(size) {
err := errors.New("sftp: no space left on device")
return backoff.Permanent(err)
}
@@ -497,28 +494,6 @@ func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, erro
return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}
-// Test returns true if a blob of the given type and name exists in the backend.
-func (r *SFTP) Test(ctx context.Context, h restic.Handle) (bool, error) {
- debug.Log("Test(%v)", h)
- if err := r.clientError(); err != nil {
- return false, err
- }
-
- r.sem.GetToken()
- defer r.sem.ReleaseToken()
-
- _, err := r.c.Lstat(r.Filename(h))
- if errors.Is(err, os.ErrNotExist) {
- return false, nil
- }
-
- if err != nil {
- return false, errors.Wrap(err, "Lstat")
- }
-
- return true, nil
-}
-
// Remove removes the content stored at name.
func (r *SFTP) Remove(ctx context.Context, h restic.Handle) error {
debug.Log("Remove(%v)", h)
diff --git a/internal/backend/sftp/sftp_test.go b/internal/backend/sftp/sftp_test.go
index e682b343a..0dbcd291c 100644
--- a/internal/backend/sftp/sftp_test.go
+++ b/internal/backend/sftp/sftp_test.go
@@ -3,7 +3,6 @@ package sftp_test
import (
"context"
"fmt"
- "io/ioutil"
"os"
"path/filepath"
"strings"
@@ -34,7 +33,7 @@ func newTestSuite(t testing.TB) *test.Suite {
return &test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
- dir, err := ioutil.TempDir(rtest.TestTempDir, "restic-test-sftp-")
+ dir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-sftp-")
if err != nil {
t.Fatal(err)
}
diff --git a/internal/backend/swift/config.go b/internal/backend/swift/config.go
index d2751dd1a..ced256752 100644
--- a/internal/backend/swift/config.go
+++ b/internal/backend/swift/config.go
@@ -51,17 +51,13 @@ func NewConfig() Config {
// ParseConfig parses the string s and extract swift's container name and prefix.
func ParseConfig(s string) (interface{}, error) {
- data := strings.SplitN(s, ":", 3)
- if len(data) != 3 {
+ if !strings.HasPrefix(s, "swift:") {
return nil, errors.New("invalid URL, expected: swift:container-name:/[prefix]")
}
+ s = strings.TrimPrefix(s, "swift:")
- scheme, container, prefix := data[0], data[1], data[2]
- if scheme != "swift" {
- return nil, errors.Errorf("unexpected prefix: %s", data[0])
- }
-
- if len(prefix) == 0 {
+ container, prefix, _ := strings.Cut(s, ":")
+ if prefix == "" {
return nil, errors.Errorf("prefix is empty")
}
diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go
index a739b2c6b..764c7bb62 100644
--- a/internal/backend/swift/swift.go
+++ b/internal/backend/swift/swift.go
@@ -14,6 +14,7 @@ import (
"time"
"github.com/restic/restic/internal/backend"
+ "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/sema"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
@@ -30,7 +31,7 @@ type beSwift struct {
sem sema.Semaphore
container string // Container name
prefix string // Prefix of object names in the container
- backend.Layout
+ layout.Layout
}
// ensure statically that *beSwift implements restic.Backend.
@@ -74,7 +75,7 @@ func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (restic.Backend
sem: sem,
container: cfg.Container,
prefix: cfg.Prefix,
- Layout: &backend.DefaultLayout{
+ Layout: &layout.DefaultLayout{
Path: cfg.Prefix,
Join: path.Join,
},
@@ -225,25 +226,6 @@ func (be *beSwift) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInf
return restic.FileInfo{Size: obj.Bytes, Name: h.Name}, nil
}
-// Test returns true if a blob of the given type and name exists in the backend.
-func (be *beSwift) Test(ctx context.Context, h restic.Handle) (bool, error) {
- objName := be.Filename(h)
-
- be.sem.GetToken()
- defer be.sem.ReleaseToken()
-
- switch _, _, err := be.conn.Object(ctx, be.container, objName); err {
- case nil:
- return true, nil
-
- case swift.ObjectNotFound:
- return false, nil
-
- default:
- return false, errors.Wrap(err, "conn.Object")
- }
-}
-
// Remove removes the blob with the given name and type.
func (be *beSwift) Remove(ctx context.Context, h restic.Handle) error {
objName := be.Filename(h)
diff --git a/internal/backend/swift/swift_test.go b/internal/backend/swift/swift_test.go
index 459ab76a5..0912e4f7e 100644
--- a/internal/backend/swift/swift_test.go
+++ b/internal/backend/swift/swift_test.go
@@ -66,12 +66,12 @@ func newSwiftTestSuite(t testing.TB) *test.Suite {
return nil, err
}
- exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
- if err != nil {
+ _, err = be.Stat(context.TODO(), restic.Handle{Type: restic.ConfigFile})
+ if err != nil && !be.IsNotExist(err) {
return nil, err
}
- if exists {
+ if err == nil {
return nil, errors.New("config already exists")
}
diff --git a/internal/backend/test/suite.go b/internal/backend/test/suite.go
index 342ac38e4..45c6d96bd 100644
--- a/internal/backend/test/suite.go
+++ b/internal/backend/test/suite.go
@@ -60,8 +60,10 @@ func (s *Suite) RunTests(t *testing.T) {
return
}
- if err = s.Cleanup(s.Config); err != nil {
- t.Fatal(err)
+ if s.Cleanup != nil {
+ if err = s.Cleanup(s.Config); err != nil {
+ t.Fatal(err)
+ }
}
}
diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go
index f05366f6d..b98af59c3 100644
--- a/internal/backend/test/tests.go
+++ b/internal/backend/test/tests.go
@@ -5,7 +5,6 @@ import (
"context"
"fmt"
"io"
- "io/ioutil"
"math/rand"
"os"
"reflect"
@@ -28,6 +27,15 @@ func seedRand(t testing.TB) {
t.Logf("rand initialized with seed %d", seed)
}
+func beTest(ctx context.Context, be restic.Backend, h restic.Handle) (bool, error) {
+ _, err := be.Stat(ctx, h)
+ if err != nil && be.IsNotExist(err) {
+ return false, nil
+ }
+
+ return err == nil, err
+}
+
// TestCreateWithConfig tests that creating a backend in a location which already
// has a config file fails.
func (s *Suite) TestCreateWithConfig(t *testing.T) {
@@ -36,7 +44,7 @@ func (s *Suite) TestCreateWithConfig(t *testing.T) {
// remove a config if present
cfgHandle := restic.Handle{Type: restic.ConfigFile}
- cfgPresent, err := b.Test(context.TODO(), cfgHandle)
+ cfgPresent, err := beTest(context.TODO(), b, cfgHandle)
if err != nil {
t.Fatalf("unable to test for config: %+v", err)
}
@@ -84,6 +92,7 @@ func (s *Suite) TestConfig(t *testing.T) {
if err == nil {
t.Fatalf("did not get expected error for non-existing config")
}
+ test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize error from LoadAll(): %v", err)
err = b.Save(context.TODO(), restic.Handle{Type: restic.ConfigFile}, restic.NewByteReader([]byte(testString), b.Hasher()))
if err != nil {
@@ -123,11 +132,13 @@ func (s *Suite) TestLoad(t *testing.T) {
if err == nil {
t.Fatalf("Load() did not return an error for invalid handle")
}
+ test.Assert(t, !b.IsNotExist(err), "IsNotExist() should not accept an invalid handle error: %v", err)
err = testLoad(b, restic.Handle{Type: restic.PackFile, Name: "foobar"}, 0, 0)
if err == nil {
t.Fatalf("Load() did not return an error for non-existing blob")
}
+ test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize non-existing blob: %v", err)
length := rand.Intn(1<<24) + 2000
@@ -148,7 +159,7 @@ func (s *Suite) TestLoad(t *testing.T) {
}
err = b.Load(context.TODO(), handle, 0, 0, func(rd io.Reader) error {
- _, err := io.Copy(ioutil.Discard, rd)
+ _, err := io.Copy(io.Discard, rd)
if err != nil {
t.Fatal(err)
}
@@ -189,7 +200,7 @@ func (s *Suite) TestLoad(t *testing.T) {
var buf []byte
err := b.Load(context.TODO(), handle, getlen, int64(o), func(rd io.Reader) (ierr error) {
- buf, ierr = ioutil.ReadAll(rd)
+ buf, ierr = io.ReadAll(rd)
return ierr
})
if err != nil {
@@ -522,7 +533,7 @@ func (s *Suite) TestSave(t *testing.T) {
}
// test saving from a tempfile
- tmpfile, err := ioutil.TempFile("", "restic-backend-save-test-")
+ tmpfile, err := os.CreateTemp("", "restic-backend-save-test-")
if err != nil {
t.Fatal(err)
}
@@ -643,7 +654,7 @@ func (s *Suite) TestSaveWrongHash(t *testing.T) {
// test that upload with hash mismatch fails
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
err := b.Save(context.TODO(), h, &wrongByteReader{ByteReader: *restic.NewByteReader(data, b.Hasher())})
- exists, err2 := b.Test(context.TODO(), h)
+ exists, err2 := beTest(context.TODO(), b, h)
if err2 != nil {
t.Fatal(err2)
}
@@ -678,7 +689,7 @@ func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) res
// testLoad loads a blob (but discards its contents).
func testLoad(b restic.Backend, h restic.Handle, length int, offset int64) error {
return b.Load(context.TODO(), h, 0, 0, func(rd io.Reader) (ierr error) {
- _, ierr = io.Copy(ioutil.Discard, rd)
+ _, ierr = io.Copy(io.Discard, rd)
return ierr
})
}
@@ -703,7 +714,7 @@ func (s *Suite) delayedRemove(t testing.TB, be restic.Backend, handles ...restic
var found bool
var err error
for time.Since(start) <= s.WaitForDelayedRemoval {
- found, err = be.Test(context.TODO(), h)
+ found, err = beTest(context.TODO(), be, h)
if s.ErrorHandler != nil {
err = s.ErrorHandler(t, be, err)
}
@@ -754,6 +765,8 @@ func (s *Suite) TestBackend(t *testing.T) {
b := s.open(t)
defer s.close(t, b)
+ test.Assert(t, !b.IsNotExist(nil), "IsNotExist() recognized nil error")
+
for _, tpe := range []restic.FileType{
restic.PackFile, restic.KeyFile, restic.LockFile,
restic.SnapshotFile, restic.IndexFile,
@@ -765,20 +778,22 @@ func (s *Suite) TestBackend(t *testing.T) {
// test if blob is already in repository
h := restic.Handle{Type: tpe, Name: id.String()}
- ret, err := b.Test(context.TODO(), h)
+ ret, err := beTest(context.TODO(), b, h)
test.OK(t, err)
test.Assert(t, !ret, "blob was found to exist before creating")
// try to stat a not existing blob
_, err = b.Stat(context.TODO(), h)
test.Assert(t, err != nil, "blob data could be extracted before creation")
+ test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Stat() error: %v", err)
// try to read not existing blob
err = testLoad(b, h, 0, 0)
test.Assert(t, err != nil, "blob could be read before creation")
+ test.Assert(t, b.IsNotExist(err), "IsNotExist() did not recognize Load() error: %v", err)
// try to get string out, should fail
- ret, err = b.Test(context.TODO(), h)
+ ret, err = beTest(context.TODO(), b, h)
test.OK(t, err)
test.Assert(t, !ret, "id %q was found (but should not have)", ts.id)
}
@@ -819,7 +834,7 @@ func (s *Suite) TestBackend(t *testing.T) {
test.OK(t, err)
// test that the blob is gone
- ok, err := b.Test(context.TODO(), h)
+ ok, err := beTest(context.TODO(), b, h)
test.OK(t, err)
test.Assert(t, !ok, "removed blob still present")
@@ -855,9 +870,9 @@ func (s *Suite) TestBackend(t *testing.T) {
h := restic.Handle{Type: tpe, Name: id.String()}
- found, err := b.Test(context.TODO(), h)
+ found, err := beTest(context.TODO(), b, h)
test.OK(t, err)
- test.Assert(t, found, fmt.Sprintf("id %q not found", id))
+ test.Assert(t, found, fmt.Sprintf("id %v/%q not found", tpe, id))
handles = append(handles, h)
}
diff --git a/internal/backend/test/tests_test.go b/internal/backend/test/tests_test.go
deleted file mode 100644
index 8e52e3c59..000000000
--- a/internal/backend/test/tests_test.go
+++ /dev/null
@@ -1,66 +0,0 @@
-package test_test
-
-import (
- "context"
- "testing"
-
- "github.com/restic/restic/internal/errors"
- "github.com/restic/restic/internal/restic"
-
- "github.com/restic/restic/internal/backend/mem"
- "github.com/restic/restic/internal/backend/test"
-)
-
-type memConfig struct {
- be restic.Backend
-}
-
-func newTestSuite(t testing.TB) *test.Suite {
- return &test.Suite{
- // NewConfig returns a config for a new temporary backend that will be used in tests.
- NewConfig: func() (interface{}, error) {
- return &memConfig{}, nil
- },
-
- // CreateFn is a function that creates a temporary repository for the tests.
- Create: func(cfg interface{}) (restic.Backend, error) {
- c := cfg.(*memConfig)
- if c.be != nil {
- ok, err := c.be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
- if err != nil {
- return nil, err
- }
-
- if ok {
- return nil, errors.New("config already exists")
- }
- }
-
- c.be = mem.New()
- return c.be, nil
- },
-
- // OpenFn is a function that opens a previously created temporary repository.
- Open: func(cfg interface{}) (restic.Backend, error) {
- c := cfg.(*memConfig)
- if c.be == nil {
- c.be = mem.New()
- }
- return c.be, nil
- },
-
- // CleanupFn removes data created during the tests.
- Cleanup: func(cfg interface{}) error {
- // no cleanup needed
- return nil
- },
- }
-}
-
-func TestSuiteBackendMem(t *testing.T) {
- newTestSuite(t).RunTests(t)
-}
-
-func BenchmarkSuiteBackendMem(b *testing.B) {
- newTestSuite(b).RunBenchmarks(b)
-}
diff --git a/internal/backend/utils.go b/internal/backend/utils.go
index be1c2a9e0..d2ac44670 100644
--- a/internal/backend/utils.go
+++ b/internal/backend/utils.go
@@ -6,6 +6,8 @@ import (
"fmt"
"io"
+ "github.com/restic/restic/internal/debug"
+ "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
)
@@ -13,6 +15,7 @@ import (
// buffer, which is truncated. If the buffer is not large enough or nil, a new
// one is allocated.
func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle) ([]byte, error) {
+ retriedInvalidData := false
err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this is idempotent, in case an error occurs this function may be called multiple times!
wr := bytes.NewBuffer(buf[:0])
@@ -21,6 +24,18 @@ func LoadAll(ctx context.Context, buf []byte, be restic.Backend, h restic.Handle
return cerr
}
buf = wr.Bytes()
+
+ // retry loading damaged data only once. If a file fails to download correctly
+ // the second time, then it is likely corrupted at the backend. Return the data
+ // to the caller in that case to let it decide what to do with the data.
+ if !retriedInvalidData && h.Type != restic.ConfigFile {
+ id, err := restic.ParseID(h.Name)
+ if err == nil && !restic.Hash(buf).Equal(id) {
+ debug.Log("retry loading broken blob %v", h)
+ retriedInvalidData = true
+ return errors.Errorf("loadAll(%v): invalid data returned", h)
+ }
+ }
return nil
})
diff --git a/internal/backend/utils_test.go b/internal/backend/utils_test.go
index 2e77fa9bd..16995afd3 100644
--- a/internal/backend/utils_test.go
+++ b/internal/backend/utils_test.go
@@ -56,6 +56,47 @@ func save(t testing.TB, be restic.Backend, buf []byte) restic.Handle {
return h
}
+type quickRetryBackend struct {
+ restic.Backend
+}
+
+func (be *quickRetryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+ err := be.Backend.Load(ctx, h, length, offset, fn)
+ if err != nil {
+ // retry
+ err = be.Backend.Load(ctx, h, length, offset, fn)
+ }
+ return err
+}
+
+func TestLoadAllBroken(t *testing.T) {
+ b := mock.NewBackend()
+
+ data := rtest.Random(23, rand.Intn(MiB)+500*KiB)
+ id := restic.Hash(data)
+ // damage buffer
+ data[0] ^= 0xff
+
+ b.OpenReaderFn = func(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
+ return io.NopCloser(bytes.NewReader(data)), nil
+ }
+
+ // must fail on first try
+ _, err := backend.LoadAll(context.TODO(), nil, b, restic.Handle{Type: restic.PackFile, Name: id.String()})
+ if err == nil {
+ t.Fatalf("missing expected error")
+ }
+
+ // must return the broken data after a retry
+ be := &quickRetryBackend{Backend: b}
+ buf, err := backend.LoadAll(context.TODO(), nil, be, restic.Handle{Type: restic.PackFile, Name: id.String()})
+ rtest.OK(t, err)
+
+ if !bytes.Equal(buf, data) {
+ t.Fatalf("wrong data returned")
+ }
+}
+
func TestLoadAllAppend(t *testing.T) {
b := mem.New()
diff --git a/internal/bloblru/cache.go b/internal/bloblru/cache.go
index dfd9b2fd1..302ecc769 100644
--- a/internal/bloblru/cache.go
+++ b/internal/bloblru/cache.go
@@ -6,7 +6,7 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
- "github.com/hashicorp/golang-lru/simplelru"
+ "github.com/hashicorp/golang-lru/v2/simplelru"
)
// Crude estimate of the overhead per blob: a SHA-256, a linked list node
@@ -17,7 +17,7 @@ const overhead = len(restic.ID{}) + 64
// It is safe for concurrent access.
type Cache struct {
mu sync.Mutex
- c *simplelru.LRU
+ c *simplelru.LRU[restic.ID, []byte]
free, size int // Current and max capacity, in bytes.
}
@@ -33,7 +33,7 @@ func New(size int) *Cache {
// The actual maximum will be smaller than size/overhead, because we
// evict entries (RemoveOldest in add) to maintain our size bound.
maxEntries := size / overhead
- lru, err := simplelru.NewLRU(maxEntries, c.evict)
+ lru, err := simplelru.NewLRU[restic.ID, []byte](maxEntries, c.evict)
if err != nil {
panic(err) // Can only be maxEntries <= 0.
}
@@ -55,24 +55,21 @@ func (c *Cache) Add(id restic.ID, blob []byte) (old []byte) {
c.mu.Lock()
defer c.mu.Unlock()
- var key interface{} = id
-
- if c.c.Contains(key) { // Doesn't update the recency list.
+ if c.c.Contains(id) { // Doesn't update the recency list.
return
}
// This loop takes at most min(maxEntries, maxchunksize/overhead)
// iterations.
for size > c.free {
- _, val, _ := c.c.RemoveOldest()
- b := val.([]byte)
+ _, b, _ := c.c.RemoveOldest()
if cap(b) > cap(old) {
// We can only return one buffer, so pick the largest.
old = b
}
}
- c.c.Add(key, blob)
+ c.c.Add(id, blob)
c.free -= size
return old
@@ -80,17 +77,15 @@ func (c *Cache) Add(id restic.ID, blob []byte) (old []byte) {
func (c *Cache) Get(id restic.ID) ([]byte, bool) {
c.mu.Lock()
- value, ok := c.c.Get(id)
+ blob, ok := c.c.Get(id)
c.mu.Unlock()
debug.Log("bloblru.Cache: get %v, hit %v", id, ok)
- blob, ok := value.([]byte)
return blob, ok
}
-func (c *Cache) evict(key, value interface{}) {
- blob := value.([]byte)
+func (c *Cache) evict(key restic.ID, blob []byte) {
debug.Log("bloblru.Cache: evict %v, %d bytes", key, cap(blob))
c.free += cap(blob) + overhead
}
diff --git a/internal/bloblru/cache_test.go b/internal/bloblru/cache_test.go
index 34280e35c..aa6f4465c 100644
--- a/internal/bloblru/cache_test.go
+++ b/internal/bloblru/cache_test.go
@@ -1,6 +1,7 @@
package bloblru
import (
+ "math/rand"
"testing"
"github.com/restic/restic/internal/restic"
@@ -50,3 +51,29 @@ func TestCache(t *testing.T) {
rtest.Equals(t, cacheSize, c.size)
rtest.Equals(t, cacheSize, c.free)
}
+
+func BenchmarkAdd(b *testing.B) {
+ const (
+ MiB = 1 << 20
+ nblobs = 64
+ )
+
+ c := New(64 * MiB)
+
+ buf := make([]byte, 8*MiB)
+ ids := make([]restic.ID, nblobs)
+ sizes := make([]int, nblobs)
+
+ r := rand.New(rand.NewSource(100))
+ for i := range ids {
+ r.Read(ids[i][:])
+ sizes[i] = r.Intn(8 * MiB)
+ }
+
+ b.ResetTimer()
+ b.ReportAllocs()
+
+ for i := 0; i < b.N; i++ {
+ c.Add(ids[i%nblobs], buf[:sizes[i%nblobs]])
+ }
+}
diff --git a/internal/cache/backend.go b/internal/cache/backend.go
index 4078ed2fa..a707f8243 100644
--- a/internal/cache/backend.go
+++ b/internal/cache/backend.go
@@ -131,21 +131,19 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
return nil
}
-// loadFromCacheOrDelegate will try to load the file from the cache, and fall
-// back to the backend if that fails.
-func (b *Backend) loadFromCacheOrDelegate(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
+// loadFromCache will try to load the file from the cache.
+func (b *Backend) loadFromCache(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) (bool, error) {
rd, err := b.Cache.load(h, length, offset)
if err != nil {
- debug.Log("error caching %v: %v, falling back to backend", h, err)
- return b.Backend.Load(ctx, h, length, offset, consumer)
+ return false, err
}
err = consumer(rd)
if err != nil {
_ = rd.Close() // ignore secondary errors
- return err
+ return true, err
}
- return rd.Close()
+ return true, rd.Close()
}
// Load loads a file from the cache or the backend.
@@ -160,19 +158,17 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
debug.Log("downloading %v finished", h)
}
- if b.Cache.Has(h) {
- debug.Log("Load(%v, %v, %v) from cache", h, length, offset)
- rd, err := b.Cache.load(h, length, offset)
+ // try loading from cache without checking that the handle is actually cached
+ inCache, err := b.loadFromCache(ctx, h, length, offset, consumer)
+ if inCache {
if err == nil {
- err = consumer(rd)
- if err != nil {
- _ = rd.Close() // ignore secondary errors
- return err
- }
- return rd.Close()
+ return nil
}
- debug.Log("error loading %v from cache: %v", h, err)
+
+ // drop from cache and retry once
+ _ = b.Cache.remove(h)
}
+ debug.Log("error loading %v from cache: %v", h, err)
// if we don't automatically cache this file type, fall back to the backend
if !autoCacheTypes(h) {
@@ -181,9 +177,12 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
}
debug.Log("auto-store %v in the cache", h)
- err := b.cacheFile(ctx, h)
+ err = b.cacheFile(ctx, h)
if err == nil {
- return b.loadFromCacheOrDelegate(ctx, h, length, offset, consumer)
+ inCache, err = b.loadFromCache(ctx, h, length, offset, consumer)
+ if inCache {
+ return err
+ }
}
debug.Log("error caching %v: %v, falling back to backend", h, err)
diff --git a/internal/cache/backend_test.go b/internal/cache/backend_test.go
index 79b838eb2..3ddab5952 100644
--- a/internal/cache/backend_test.go
+++ b/internal/cache/backend_test.go
@@ -48,7 +48,6 @@ func remove(t testing.TB, be restic.Backend, h restic.Handle) {
func randomData(n int) (restic.Handle, []byte) {
data := test.Random(rand.Int(), n)
id := restic.Hash(data)
- copy(id[:], data)
h := restic.Handle{
Type: restic.IndexFile,
Name: id.String(),
@@ -58,10 +57,7 @@ func randomData(n int) (restic.Handle, []byte) {
func TestBackend(t *testing.T) {
be := mem.New()
-
- c, cleanup := TestNewCache(t)
- defer cleanup()
-
+ c := TestNewCache(t)
wbe := c.Wrap(be)
h, data := randomData(5234142)
@@ -129,10 +125,7 @@ func (be loadErrorBackend) Load(ctx context.Context, h restic.Handle, length int
func TestErrorBackend(t *testing.T) {
be := mem.New()
-
- c, cleanup := TestNewCache(t)
- defer cleanup()
-
+ c := TestNewCache(t)
h, data := randomData(5234142)
// save directly in backend
@@ -172,3 +165,38 @@ func TestErrorBackend(t *testing.T) {
wg.Wait()
}
+
+func TestBackendRemoveBroken(t *testing.T) {
+ be := mem.New()
+ c := TestNewCache(t)
+
+ h, data := randomData(5234142)
+ // save directly in backend
+ save(t, be, h, data)
+
+ // prime cache with broken copy
+ broken := append([]byte{}, data...)
+ broken[0] ^= 0xff
+ err := c.Save(h, bytes.NewReader(broken))
+ test.OK(t, err)
+
+ // loadall retries if broken data was returned
+ buf, err := backend.LoadAll(context.TODO(), nil, c.Wrap(be), h)
+ test.OK(t, err)
+
+ if !bytes.Equal(buf, data) {
+ t.Fatalf("wrong data returned")
+ }
+
+ // check that the cache now contains the correct data
+ rd, err := c.load(h, 0, 0)
+ defer func() {
+ _ = rd.Close()
+ }()
+ test.OK(t, err)
+ cached, err := io.ReadAll(rd)
+ test.OK(t, err)
+ if !bytes.Equal(cached, data) {
+ t.Fatalf("wrong data cache")
+ }
+}
diff --git a/internal/cache/cache.go b/internal/cache/cache.go
index a43b2bbf2..075f7f6a1 100644
--- a/internal/cache/cache.go
+++ b/internal/cache/cache.go
@@ -2,7 +2,6 @@ package cache
import (
"fmt"
- "io/ioutil"
"os"
"path/filepath"
"regexp"
@@ -26,7 +25,7 @@ const dirMode = 0700
const fileMode = 0644
func readVersion(dir string) (v uint, err error) {
- buf, err := ioutil.ReadFile(filepath.Join(dir, "version"))
+ buf, err := os.ReadFile(filepath.Join(dir, "version"))
if errors.Is(err, os.ErrNotExist) {
return 0, nil
}
@@ -130,7 +129,7 @@ func New(id string, basedir string) (c *Cache, err error) {
}
if v < cacheVersion {
- err = ioutil.WriteFile(filepath.Join(cachedir, "version"), []byte(fmt.Sprintf("%d", cacheVersion)), fileMode)
+ err = os.WriteFile(filepath.Join(cachedir, "version"), []byte(fmt.Sprintf("%d", cacheVersion)), fileMode)
if err != nil {
return nil, errors.WithStack(err)
}
diff --git a/internal/cache/file.go b/internal/cache/file.go
index 8ed4be77e..c315be19f 100644
--- a/internal/cache/file.go
+++ b/internal/cache/file.go
@@ -2,12 +2,12 @@ package cache
import (
"io"
- "io/ioutil"
"os"
"path/filepath"
"runtime"
"github.com/pkg/errors"
+ "github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
@@ -27,23 +27,15 @@ func (c *Cache) canBeCached(t restic.FileType) bool {
return false
}
- if _, ok := cacheLayoutPaths[t]; !ok {
- return false
- }
-
- return true
-}
-
-type readCloser struct {
- io.Reader
- io.Closer
+ _, ok := cacheLayoutPaths[t]
+ return ok
}
// Load returns a reader that yields the contents of the file with the
// given handle. rd must be closed after use. If an error is returned, the
// ReadCloser is nil.
func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
- debug.Log("Load from cache: %v", h)
+ debug.Log("Load(%v, %v, %v) from cache", h, length, offset)
if !c.canBeCached(h.Type) {
return nil, errors.New("cannot be cached")
}
@@ -59,13 +51,14 @@ func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser,
return nil, errors.WithStack(err)
}
- if fi.Size() <= int64(crypto.CiphertextLength(0)) {
+ size := fi.Size()
+ if size <= int64(crypto.CiphertextLength(0)) {
_ = f.Close()
_ = c.remove(h)
return nil, errors.Errorf("cached file %v is truncated, removing", h)
}
- if fi.Size() < offset+int64(length) {
+ if size < offset+int64(length) {
_ = f.Close()
_ = c.remove(h)
return nil, errors.Errorf("cached file %v is too small, removing", h)
@@ -78,12 +71,10 @@ func (c *Cache) load(h restic.Handle, length int, offset int64) (io.ReadCloser,
}
}
- rd := readCloser{Reader: f, Closer: f}
- if length > 0 {
- rd.Reader = io.LimitReader(f, int64(length))
+ if length <= 0 {
+ return f, nil
}
-
- return rd, nil
+ return backend.LimitReadCloser(f, int64(length)), nil
}
// Save saves a file in the cache.
@@ -105,7 +96,7 @@ func (c *Cache) Save(h restic.Handle, rd io.Reader) error {
// First save to a temporary location. This allows multiple concurrent
// restics to use a single cache dir.
- f, err := ioutil.TempFile(dir, "tmp-")
+ f, err := os.CreateTemp(dir, "tmp-")
if err != nil {
return err
}
diff --git a/internal/cache/file_test.go b/internal/cache/file_test.go
index 388d7d4f7..111a2430f 100644
--- a/internal/cache/file_test.go
+++ b/internal/cache/file_test.go
@@ -3,9 +3,10 @@ package cache
import (
"bytes"
"fmt"
- "io/ioutil"
+ "io"
"math/rand"
"os"
+ "runtime"
"testing"
"time"
@@ -54,7 +55,7 @@ func load(t testing.TB, c *Cache, h restic.Handle) []byte {
t.Fatalf("load() returned nil reader")
}
- buf, err := ioutil.ReadAll(rd)
+ buf, err := io.ReadAll(rd)
if err != nil {
t.Fatal(err)
}
@@ -86,8 +87,7 @@ func TestFiles(t *testing.T) {
t.Logf("seed is %v", seed)
rand.Seed(seed)
- c, cleanup := TestNewCache(t)
- defer cleanup()
+ c := TestNewCache(t)
var tests = []restic.FileType{
restic.SnapshotFile,
@@ -96,7 +96,7 @@ func TestFiles(t *testing.T) {
}
for _, tpe := range tests {
- t.Run(fmt.Sprintf("%v", tpe), func(t *testing.T) {
+ t.Run(tpe.String(), func(t *testing.T) {
ids := generateRandomFiles(t, tpe, c)
id := randomID(ids)
@@ -139,8 +139,7 @@ func TestFileLoad(t *testing.T) {
t.Logf("seed is %v", seed)
rand.Seed(seed)
- c, cleanup := TestNewCache(t)
- defer cleanup()
+ c := TestNewCache(t)
// save about 5 MiB of data in the cache
data := test.Random(rand.Int(), 5234142)
@@ -173,7 +172,7 @@ func TestFileLoad(t *testing.T) {
t.Fatal(err)
}
- buf, err := ioutil.ReadAll(rd)
+ buf, err := io.ReadAll(rd)
if err != nil {
t.Fatal(err)
}
@@ -204,13 +203,26 @@ func TestFileLoad(t *testing.T) {
}
// Simulate multiple processes writing to a cache, using goroutines.
+//
+// The possibility of sharing a cache between multiple concurrent restic
+// processes isn't guaranteed in the docs and doesn't always work on Windows, hence the
+// check on GOOS. Cache sharing is considered a "nice to have" on POSIX, for now.
+//
+// The cache first creates a temporary file and then renames it to its final name.
+// On Windows renaming internally creates a file handle with a shareMode which
+// includes FILE_SHARE_DELETE. The Go runtime opens files without FILE_SHARE_DELETE,
+// thus Open(fn) will fail until the file handle used for renaming was closed.
+// See https://devblogs.microsoft.com/oldnewthing/20211022-00/?p=105822
+// for hints on how to fix this properly.
func TestFileSaveConcurrent(t *testing.T) {
- const nproc = 40
+ if runtime.GOOS == "windows" {
+ t.Skip("may not work due to FILE_SHARE_DELETE issue")
+ }
- c, cleanup := TestNewCache(t)
- defer cleanup()
+ const nproc = 40
var (
+ c = TestNewCache(t)
data = test.Random(1, 10000)
g errgroup.Group
id restic.ID
@@ -242,7 +254,7 @@ func TestFileSaveConcurrent(t *testing.T) {
}
defer func() { _ = f.Close() }()
- read, err := ioutil.ReadAll(f)
+ read, err := io.ReadAll(f)
if err == nil && !bytes.Equal(read, data) {
err = errors.New("mismatch between Save and Load")
}
diff --git a/internal/cache/testing.go b/internal/cache/testing.go
index b3156374d..d4ce8c2ff 100644
--- a/internal/cache/testing.go
+++ b/internal/cache/testing.go
@@ -9,12 +9,12 @@ import (
// TestNewCache returns a cache in a temporary directory which is removed when
// cleanup is called.
-func TestNewCache(t testing.TB) (*Cache, func()) {
- dir, cleanup := test.TempDir(t)
+func TestNewCache(t testing.TB) *Cache {
+ dir := test.TempDir(t)
t.Logf("created new cache at %v", dir)
cache, err := New(restic.NewRandomID().String(), dir)
if err != nil {
t.Fatal(err)
}
- return cache, cleanup
+ return cache
}
diff --git a/internal/checker/checker.go b/internal/checker/checker.go
index 0e4310c95..b2512969e 100644
--- a/internal/checker/checker.go
+++ b/internal/checker/checker.go
@@ -6,7 +6,6 @@ import (
"context"
"fmt"
"io"
- "io/ioutil"
"runtime"
"sort"
"sync"
@@ -18,6 +17,7 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/hashing"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@@ -38,7 +38,7 @@ type Checker struct {
}
trackUnused bool
- masterIndex *repository.MasterIndex
+ masterIndex *index.MasterIndex
snapshots restic.Lister
repo restic.Repository
@@ -48,7 +48,7 @@ type Checker struct {
func New(repo restic.Repository, trackUnused bool) *Checker {
c := &Checker{
packs: make(map[restic.ID]int64),
- masterIndex: repository.NewMasterIndex(),
+ masterIndex: index.NewMasterIndex(),
repo: repo,
trackUnused: trackUnused,
}
@@ -59,11 +59,7 @@ func New(repo restic.Repository, trackUnused bool) *Checker {
}
// ErrLegacyLayout is returned when the repository uses the S3 legacy layout.
-type ErrLegacyLayout struct{}
-
-func (e *ErrLegacyLayout) Error() string {
- return "repository uses S3 legacy layout"
-}
+var ErrLegacyLayout = errors.New("repository uses S3 legacy layout")
// ErrDuplicatePacks is returned when a pack is found in more than one index.
type ErrDuplicatePacks struct {
@@ -100,12 +96,28 @@ func (c *Checker) LoadSnapshots(ctx context.Context) error {
return err
}
+func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID]restic.BlobType {
+ packs := make(map[restic.ID]restic.BlobType)
+ idx.Each(ctx, func(pb restic.PackedBlob) {
+ tpe, exists := packs[pb.PackID]
+ if exists {
+ if pb.Type != tpe {
+ tpe = restic.InvalidBlob
+ }
+ } else {
+ tpe = pb.Type
+ }
+ packs[pb.PackID] = tpe
+ })
+ return packs
+}
+
// LoadIndex loads all index files.
func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
debug.Log("Start")
packToIndex := make(map[restic.ID]restic.IDSet)
- err := repository.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error {
+ err := index.ForAllIndexes(ctx, c.repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
debug.Log("process index %v, err %v", id, err)
if oldFormat {
@@ -124,14 +136,14 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
debug.Log("process blobs")
cnt := 0
- for blob := range index.Each(ctx) {
+ index.Each(ctx, func(blob restic.PackedBlob) {
cnt++
if _, ok := packToIndex[blob.PackID]; !ok {
packToIndex[blob.PackID] = restic.NewIDSet()
}
packToIndex[blob.PackID].Insert(id)
- }
+ })
debug.Log("%d blobs processed", cnt)
return nil
@@ -149,6 +161,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
// compute pack size using index entries
c.packs = pack.Size(ctx, c.masterIndex, false)
+ packTypes := computePackTypes(ctx, c.masterIndex)
debug.Log("checking for duplicate packs")
for packID := range c.packs {
@@ -159,7 +172,7 @@ func (c *Checker) LoadIndex(ctx context.Context) (hints []error, errs []error) {
Indexes: packToIndex[packID],
})
}
- if c.masterIndex.IsMixedPack(packID) {
+ if packTypes[packID] == restic.InvalidBlob {
hints = append(hints, &ErrMixedPack{
PackID: packID,
})
@@ -214,7 +227,7 @@ func (c *Checker) Packs(ctx context.Context, errChan chan<- error) {
defer close(errChan)
if isS3Legacy(c.repo.Backend()) {
- errChan <- &ErrLegacyLayout{}
+ errChan <- ErrLegacyLayout
}
debug.Log("checking for %d packs", len(c.packs))
@@ -272,7 +285,7 @@ type Error struct {
Err error
}
-func (e Error) Error() string {
+func (e *Error) Error() string {
if !e.TreeID.IsNull() {
return "tree " + e.TreeID.String() + ": " + e.Err.Error()
}
@@ -386,12 +399,12 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
switch node.Type {
case "file":
if node.Content == nil {
- errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q has nil blob list", node.Name)})
+ errs = append(errs, &Error{TreeID: id, Err: errors.Errorf("file %q has nil blob list", node.Name)})
}
for b, blobID := range node.Content {
if blobID.IsNull() {
- errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q blob %d has null ID", node.Name, b)})
+ errs = append(errs, &Error{TreeID: id, Err: errors.Errorf("file %q blob %d has null ID", node.Name, b)})
continue
}
// Note that we do not use the blob size. The "obvious" check
@@ -402,7 +415,7 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
_, found := c.repo.LookupBlobSize(blobID, restic.DataBlob)
if !found {
debug.Log("tree %v references blob %v which isn't contained in index", id, blobID)
- errs = append(errs, Error{TreeID: id, Err: errors.Errorf("file %q blob %v not found in index", node.Name, blobID)})
+ errs = append(errs, &Error{TreeID: id, Err: errors.Errorf("file %q blob %v not found in index", node.Name, blobID)})
}
}
@@ -422,12 +435,12 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
case "dir":
if node.Subtree == nil {
- errs = append(errs, Error{TreeID: id, Err: errors.Errorf("dir node %q has no subtree", node.Name)})
+ errs = append(errs, &Error{TreeID: id, Err: errors.Errorf("dir node %q has no subtree", node.Name)})
continue
}
if node.Subtree.IsNull() {
- errs = append(errs, Error{TreeID: id, Err: errors.Errorf("dir node %q subtree id is null", node.Name)})
+ errs = append(errs, &Error{TreeID: id, Err: errors.Errorf("dir node %q subtree id is null", node.Name)})
continue
}
@@ -435,11 +448,11 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
// nothing to check
default:
- errs = append(errs, Error{TreeID: id, Err: errors.Errorf("node %q with invalid type %q", node.Name, node.Type)})
+ errs = append(errs, &Error{TreeID: id, Err: errors.Errorf("node %q with invalid type %q", node.Name, node.Type)})
}
if node.Name == "" {
- errs = append(errs, Error{TreeID: id, Err: errors.New("node with empty name")})
+ errs = append(errs, &Error{TreeID: id, Err: errors.New("node with empty name")})
}
}
@@ -458,13 +471,13 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
- for blob := range c.repo.Index().Each(ctx) {
+ c.repo.Index().Each(ctx, func(blob restic.PackedBlob) {
h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
if !c.blobRefs.M.Has(h) {
debug.Log("blob %v not referenced", h)
blobs = append(blobs, h)
}
- }
+ })
return blobs
}
@@ -538,7 +551,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r
}
// read remainder, which should be the pack header
- hdrBuf, err = ioutil.ReadAll(bufRd)
+ hdrBuf, err = io.ReadAll(bufRd)
if err != nil {
return err
}
diff --git a/internal/checker/checker_test.go b/internal/checker/checker_test.go
index b3a736152..775484652 100644
--- a/internal/checker/checker_test.go
+++ b/internal/checker/checker_test.go
@@ -3,7 +3,6 @@ package checker_test
import (
"context"
"io"
- "io/ioutil"
"math/rand"
"os"
"path/filepath"
@@ -208,7 +207,7 @@ func TestModifiedIndex(t *testing.T) {
Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
}
- tmpfile, err := ioutil.TempFile("", "restic-test-mod-index-")
+ tmpfile, err := os.CreateTemp("", "restic-test-mod-index-")
if err != nil {
t.Fatal(err)
}
@@ -341,9 +340,7 @@ func induceError(data []byte) {
}
func TestCheckerModifiedData(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
-
+ repo := repository.TestRepository(t)
sn := archiver.TestSnapshot(t, repo, ".", nil)
t.Logf("archived as %v", sn.ID().Str())
@@ -458,8 +455,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
damagedNode := &restic.Node{
Name: "damaged",
@@ -592,11 +588,7 @@ func benchmarkSnapshotScaling(t *testing.B, newSnapshots int) {
chkr, repo, cleanup := loadBenchRepository(t)
defer cleanup()
- snID, err := restic.FindSnapshot(context.TODO(), repo.Backend(), "51d249d2")
- if err != nil {
- t.Fatal(err)
- }
-
+ snID := restic.TestParseID("51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02")
sn2, err := restic.LoadSnapshot(context.TODO(), repo, snID)
if err != nil {
t.Fatal(err)
diff --git a/internal/debug/round_tripper.go b/internal/debug/round_tripper.go
index 6795d43d0..9dced95c6 100644
--- a/internal/debug/round_tripper.go
+++ b/internal/debug/round_tripper.go
@@ -3,7 +3,6 @@ package debug
import (
"fmt"
"io"
- "io/ioutil"
"net/http"
"net/http/httputil"
"os"
@@ -30,7 +29,7 @@ func (rd *eofDetectReader) Read(p []byte) (n int, err error) {
func (rd *eofDetectReader) Close() error {
if !rd.eofSeen {
- buf, err := ioutil.ReadAll(rd)
+ buf, err := io.ReadAll(rd)
msg := fmt.Sprintf("body not drained, %d bytes not read", len(buf))
if err != nil {
msg += fmt.Sprintf(", error: %v", err)
diff --git a/internal/dump/common_test.go b/internal/dump/common_test.go
index 7892a4fa9..3ee9112af 100644
--- a/internal/dump/common_test.go
+++ b/internal/dump/common_test.go
@@ -12,18 +12,13 @@ import (
rtest "github.com/restic/restic/internal/test"
)
-func prepareTempdirRepoSrc(t testing.TB, src archiver.TestDir) (tempdir string, repo restic.Repository, cleanup func()) {
- tempdir, removeTempdir := rtest.TempDir(t)
- repo, removeRepository := repository.TestRepository(t)
+func prepareTempdirRepoSrc(t testing.TB, src archiver.TestDir) (string, restic.Repository) {
+ tempdir := rtest.TempDir(t)
+ repo := repository.TestRepository(t)
archiver.TestCreateFiles(t, tempdir, src)
- cleanup = func() {
- removeRepository()
- removeTempdir()
- }
-
- return tempdir, repo, cleanup
+ return tempdir, repo
}
type CheckDump func(t *testing.T, testDir string, testDump *bytes.Buffer) error
@@ -77,9 +72,7 @@ func WriteTest(t *testing.T, format string, cd CheckDump) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- tmpdir, repo, cleanup := prepareTempdirRepoSrc(t, tt.args)
- defer cleanup()
-
+ tmpdir, repo := prepareTempdirRepoSrc(t, tt.args)
arch := archiver.New(repo, fs.Track{FS: fs.Local{}}, archiver.Options{})
back := rtest.Chdir(t, tmpdir)
diff --git a/internal/dump/tar.go b/internal/dump/tar.go
index 65b68ee5b..6e87aabe5 100644
--- a/internal/dump/tar.go
+++ b/internal/dump/tar.go
@@ -38,6 +38,15 @@ const (
cISVTX = 0o1000 // Save text (sticky bit)
)
+// in a 32-bit build of restic:
+// substitute a uid or gid of -1 (which was converted to 2^32 - 1) with 0
+func tarIdentifier(id uint32) int {
+ if int(id) == -1 {
+ return 0
+ }
+ return int(id)
+}
+
func (d *Dumper) dumpNodeTar(ctx context.Context, node *restic.Node, w *tar.Writer) error {
relPath, err := filepath.Rel("/", node.Path)
if err != nil {
@@ -48,8 +57,8 @@ func (d *Dumper) dumpNodeTar(ctx context.Context, node *restic.Node, w *tar.Writ
Name: filepath.ToSlash(relPath),
Size: int64(node.Size),
Mode: int64(node.Mode.Perm()), // cIS* constants are added later
- Uid: int(node.UID),
- Gid: int(node.GID),
+ Uid: tarIdentifier(node.UID),
+ Gid: tarIdentifier(node.GID),
Uname: node.User,
Gname: node.Group,
ModTime: node.ModTime,
diff --git a/internal/dump/tar_test.go b/internal/dump/tar_test.go
index 9f094ae44..0f2cb27a8 100644
--- a/internal/dump/tar_test.go
+++ b/internal/dump/tar_test.go
@@ -5,7 +5,6 @@ import (
"bytes"
"fmt"
"io"
- "io/ioutil"
"os"
"path/filepath"
"strings"
@@ -91,7 +90,7 @@ func checkTar(t *testing.T, testDir string, srcTar *bytes.Buffer) error {
if match.Size() != hdr.Size {
return fmt.Errorf("size does not match got %v want %v", hdr.Size, match.Size())
}
- contentsFile, err := ioutil.ReadFile(matchPath)
+ contentsFile, err := os.ReadFile(matchPath)
if err != nil {
t.Fatal(err)
}
diff --git a/internal/dump/zip_test.go b/internal/dump/zip_test.go
index 4d0cb3a51..0c304d3da 100644
--- a/internal/dump/zip_test.go
+++ b/internal/dump/zip_test.go
@@ -4,7 +4,6 @@ import (
"archive/zip"
"bytes"
"fmt"
- "io/ioutil"
"os"
"path/filepath"
"strings"
@@ -107,7 +106,7 @@ func checkZip(t *testing.T, testDir string, srcZip *bytes.Buffer) error {
if uint64(match.Size()) != f.UncompressedSize64 {
return fmt.Errorf("size does not match got %v want %v", f.UncompressedSize64, match.Size())
}
- contentsFile, err := ioutil.ReadFile(matchPath)
+ contentsFile, err := os.ReadFile(matchPath)
if err != nil {
t.Fatal(err)
}
diff --git a/internal/errors/errors.go b/internal/errors/errors.go
index 021da72c5..0327ea0da 100644
--- a/internal/errors/errors.go
+++ b/internal/errors/errors.go
@@ -1,9 +1,8 @@
package errors
import (
- "net/url"
+ stderrors "errors"
- "github.com/cenkalti/backoff/v4"
"github.com/pkg/errors"
)
@@ -23,35 +22,12 @@ var Wrap = errors.Wrap
// nil, Wrapf returns nil.
var Wrapf = errors.Wrapf
-// WithMessage annotates err with a new message. If err is nil, WithMessage
-// returns nil.
-var WithMessage = errors.WithMessage
-
var WithStack = errors.WithStack
-// Cause returns the cause of an error. It will also unwrap certain errors,
-// e.g. *url.Error returned by the net/http client.
-func Cause(err error) error {
- type Causer interface {
- Cause() error
- }
-
- for {
- switch e := err.(type) {
- case Causer: // github.com/pkg/errors
- err = e.Cause()
- case *backoff.PermanentError:
- err = e.Err
- case *url.Error:
- err = e.Err
- default:
- return err
- }
- }
-}
-
// Go 1.13-style error handling.
-func As(err error, tgt interface{}) bool { return errors.As(err, tgt) }
+func As(err error, tgt interface{}) bool { return stderrors.As(err, tgt) }
+
+func Is(x, y error) bool { return stderrors.Is(x, y) }
-func Is(x, y error) bool { return errors.Is(x, y) }
+func Unwrap(err error) error { return stderrors.Unwrap(err) }
diff --git a/internal/errors/fatal.go b/internal/errors/fatal.go
index 5fb615cf1..9370a68d7 100644
--- a/internal/errors/fatal.go
+++ b/internal/errors/fatal.go
@@ -1,6 +1,9 @@
package errors
-import "fmt"
+import (
+ "errors"
+ "fmt"
+)
// fatalError is an error that should be printed to the user, then the program
// should exit with an error code.
@@ -10,31 +13,19 @@ func (e fatalError) Error() string {
return string(e)
}
-func (e fatalError) Fatal() bool {
- return true
-}
-
-// Fataler is an error which should be printed to the user directly.
-// Afterwards, the program should exit with an error.
-type Fataler interface {
- Fatal() bool
-}
-
// IsFatal returns true if err is a fatal message that should be printed to the
// user. Then, the program should exit.
func IsFatal(err error) bool {
- // unwrap "Wrap" method
- err = Cause(err)
- e, ok := err.(Fataler)
- return ok && e.Fatal()
+ var fatal fatalError
+ return errors.As(err, &fatal)
}
-// Fatal returns a wrapped error which implements the Fataler interface.
+// Fatal returns an error that is marked fatal.
func Fatal(s string) error {
return Wrap(fatalError(s), "Fatal")
}
-// Fatalf returns an error which implements the Fataler interface.
+// Fatalf returns an error that is marked fatal.
func Fatalf(s string, data ...interface{}) error {
return Wrap(fatalError(fmt.Sprintf(s, data...)), "Fatal")
}
diff --git a/internal/filter/filter.go b/internal/filter/filter.go
index cfacb8cc5..473f1f4cb 100644
--- a/internal/filter/filter.go
+++ b/internal/filter/filter.go
@@ -220,10 +220,18 @@ func match(pattern Pattern, strs []string) (matched bool, err error) {
return false, nil
}
+type InvalidPatternError struct {
+ InvalidPatterns []string
+}
+
+func (e *InvalidPatternError) Error() string {
+ return "invalid pattern(s) provided:\n" + strings.Join(e.InvalidPatterns, "\n")
+}
+
// ValidatePatterns validates a slice of patterns.
// Returns true if all patterns are valid - false otherwise, along with the invalid patterns.
-func ValidatePatterns(patterns []string) (allValid bool, invalidPatterns []string) {
- invalidPatterns = make([]string, 0)
+func ValidatePatterns(patterns []string) error {
+ invalidPatterns := make([]string, 0)
for _, Pattern := range ParsePatterns(patterns) {
// Validate all pattern parts
@@ -238,7 +246,10 @@ func ValidatePatterns(patterns []string) (allValid bool, invalidPatterns []strin
}
}
- return len(invalidPatterns) == 0, invalidPatterns
+ if len(invalidPatterns) > 0 {
+ return &InvalidPatternError{InvalidPatterns: invalidPatterns}
+ }
+ return nil
}
// ParsePatterns prepares a list of patterns for use with List.
diff --git a/internal/filter/filter_patterns_test.go b/internal/filter/filter_patterns_test.go
index 215471500..5971a4e1e 100644
--- a/internal/filter/filter_patterns_test.go
+++ b/internal/filter/filter_patterns_test.go
@@ -1,14 +1,6 @@
-//go:build go1.16
-// +build go1.16
-
-// Before Go 1.16 filepath.Match returned early on a failed match,
-// and thus did not report any later syntax error in the pattern.
-// https://go.dev/doc/go1.16#path/filepath
-
package filter_test
import (
- "strings"
"testing"
"github.com/restic/restic/internal/filter"
@@ -18,11 +10,15 @@ import (
func TestValidPatterns(t *testing.T) {
// Test invalid patterns are detected and returned
t.Run("detect-invalid-patterns", func(t *testing.T) {
- allValid, invalidPatterns := filter.ValidatePatterns([]string{"*.foo", "*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"})
+ err := filter.ValidatePatterns([]string{"*.foo", "*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"})
- rtest.Assert(t, allValid == false, "Expected invalid patterns to be detected")
+ rtest.Assert(t, err != nil, "Expected invalid patterns to be detected")
- rtest.Equals(t, invalidPatterns, []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"})
+ if ip, ok := err.(*filter.InvalidPatternError); ok {
+ rtest.Equals(t, ip.InvalidPatterns, []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"})
+ } else {
+ t.Errorf("wrong error type %v", err)
+ }
})
// Test all patterns defined in matchTests are valid
@@ -33,10 +29,10 @@ func TestValidPatterns(t *testing.T) {
}
t.Run("validate-patterns", func(t *testing.T) {
- allValid, invalidPatterns := filter.ValidatePatterns(patterns)
+ err := filter.ValidatePatterns(patterns)
- if !allValid {
- t.Errorf("Found invalid pattern(s):\n%s", strings.Join(invalidPatterns, "\n"))
+ if err != nil {
+ t.Error(err)
}
})
@@ -48,10 +44,10 @@ func TestValidPatterns(t *testing.T) {
}
t.Run("validate-child-patterns", func(t *testing.T) {
- allValid, invalidPatterns := filter.ValidatePatterns(childPatterns)
+ err := filter.ValidatePatterns(childPatterns)
- if !allValid {
- t.Errorf("Found invalid child pattern(s):\n%s", strings.Join(invalidPatterns, "\n"))
+ if err != nil {
+ t.Error(err)
}
})
}
diff --git a/internal/fs/file.go b/internal/fs/file.go
index e8e9080d7..f35901c06 100644
--- a/internal/fs/file.go
+++ b/internal/fs/file.go
@@ -51,7 +51,7 @@ func Rename(oldpath, newpath string) error {
// Symlink creates newname as a symbolic link to oldname.
// If there is an error, it will be of type *LinkError.
func Symlink(oldname, newname string) error {
- return os.Symlink(fixpath(oldname), fixpath(newname))
+ return os.Symlink(oldname, fixpath(newname))
}
// Link creates newname as a hard link to oldname.
diff --git a/internal/fs/file_unix.go b/internal/fs/file_unix.go
index 3edc60be6..65f10c844 100644
--- a/internal/fs/file_unix.go
+++ b/internal/fs/file_unix.go
@@ -4,7 +4,6 @@
package fs
import (
- "io/ioutil"
"os"
"syscall"
)
@@ -18,7 +17,7 @@ func fixpath(name string) string {
// TempFile creates a temporary file which has already been deleted (on
// supported platforms)
func TempFile(dir, prefix string) (f *os.File, err error) {
- f, err = ioutil.TempFile(dir, prefix)
+ f, err = os.CreateTemp(dir, prefix)
if err != nil {
return nil, err
}
diff --git a/internal/fs/file_windows.go b/internal/fs/file_windows.go
index 56effdc18..d19a744e1 100644
--- a/internal/fs/file_windows.go
+++ b/internal/fs/file_windows.go
@@ -36,7 +36,7 @@ func fixpath(name string) string {
// TempFile creates a temporary file which is marked as delete-on-close
func TempFile(dir, prefix string) (f *os.File, err error) {
- // slightly modified implementation of ioutil.TempFile(dir, prefix) to allow us to add
+ // slightly modified implementation of os.CreateTemp(dir, prefix) to allow us to add
// the FILE_ATTRIBUTE_TEMPORARY | FILE_FLAG_DELETE_ON_CLOSE flags.
// These provide two large benefits:
// FILE_ATTRIBUTE_TEMPORARY tells Windows to keep the file in memory only if possible
diff --git a/internal/fs/fs_reader.go b/internal/fs/fs_reader.go
index dac4aac9e..1551ad919 100644
--- a/internal/fs/fs_reader.go
+++ b/internal/fs/fs_reader.go
@@ -1,6 +1,7 @@
package fs
import (
+ "fmt"
"io"
"os"
"path"
@@ -47,7 +48,7 @@ func (fs *Reader) Open(name string) (f File, err error) {
})
if f == nil {
- return nil, syscall.EIO
+ return nil, pathError("open", name, syscall.EIO)
}
return f, nil
@@ -58,7 +59,7 @@ func (fs *Reader) Open(name string) (f File, err error) {
return f, nil
}
- return nil, syscall.ENOENT
+ return nil, pathError("open", name, syscall.ENOENT)
}
func (fs *Reader) fi() os.FileInfo {
@@ -74,10 +75,11 @@ func (fs *Reader) fi() os.FileInfo {
// or Create instead. It opens the named file with specified flag
// (O_RDONLY etc.) and perm, (0666 etc.) if applicable. If successful,
// methods on the returned File can be used for I/O.
-// If there is an error, it will be of type *PathError.
+// If there is an error, it will be of type *os.PathError.
func (fs *Reader) OpenFile(name string, flag int, perm os.FileMode) (f File, err error) {
if flag & ^(O_RDONLY|O_NOFOLLOW) != 0 {
- return nil, errors.Errorf("invalid combination of flags 0x%x", flag)
+ return nil, pathError("open", name,
+ fmt.Errorf("invalid combination of flags 0x%x", flag))
}
fs.open.Do(func() {
@@ -85,14 +87,14 @@ func (fs *Reader) OpenFile(name string, flag int, perm os.FileMode) (f File, err
})
if f == nil {
- return nil, syscall.EIO
+ return nil, pathError("open", name, syscall.EIO)
}
return f, nil
}
// Stat returns a FileInfo describing the named file. If there is an error, it
-// will be of type *PathError.
+// will be of type *os.PathError.
func (fs *Reader) Stat(name string) (os.FileInfo, error) {
return fs.Lstat(name)
}
@@ -100,7 +102,7 @@ func (fs *Reader) Stat(name string) (os.FileInfo, error) {
// Lstat returns the FileInfo structure describing the named file.
// If the file is a symbolic link, the returned FileInfo
// describes the symbolic link. Lstat makes no attempt to follow the link.
-// If there is an error, it will be of type *PathError.
+// If there is an error, it will be of type *os.PathError.
func (fs *Reader) Lstat(name string) (os.FileInfo, error) {
getDirInfo := func(name string) os.FileInfo {
fi := fakeFileInfo{
@@ -130,7 +132,7 @@ func (fs *Reader) Lstat(name string) (os.FileInfo, error) {
dir = fs.Dir(dir)
}
- return nil, os.ErrNotExist
+ return nil, pathError("lstat", name, os.ErrNotExist)
}
// Join joins any number of path elements into a single path, adding a
@@ -207,11 +209,7 @@ func (r *readerFile) Read(p []byte) (int, error) {
// return an error if we did not read any data
if err == io.EOF && !r.AllowEmptyFile && !r.bytesRead {
- return n, &os.PathError{
- Path: r.fakeFile.name,
- Op: "read",
- Err: ErrFileEmpty,
- }
+ return n, pathError("read", r.fakeFile.name, ErrFileEmpty)
}
return n, err
@@ -239,19 +237,19 @@ func (f fakeFile) Fd() uintptr {
}
func (f fakeFile) Readdirnames(n int) ([]string, error) {
- return nil, os.ErrInvalid
+ return nil, pathError("readdirnames", f.name, os.ErrInvalid)
}
func (f fakeFile) Readdir(n int) ([]os.FileInfo, error) {
- return nil, os.ErrInvalid
+ return nil, pathError("readdir", f.name, os.ErrInvalid)
}
func (f fakeFile) Seek(int64, int) (int64, error) {
- return 0, os.ErrInvalid
+ return 0, pathError("seek", f.name, os.ErrInvalid)
}
func (f fakeFile) Read(p []byte) (int, error) {
- return 0, os.ErrInvalid
+ return 0, pathError("read", f.name, os.ErrInvalid)
}
func (f fakeFile) Close() error {
@@ -274,7 +272,7 @@ type fakeDir struct {
func (d fakeDir) Readdirnames(n int) ([]string, error) {
if n > 0 {
- return nil, errors.New("not implemented")
+ return nil, pathError("readdirnames", d.name, errors.New("not implemented"))
}
names := make([]string, 0, len(d.entries))
for _, entry := range d.entries {
@@ -286,7 +284,7 @@ func (d fakeDir) Readdirnames(n int) ([]string, error) {
func (d fakeDir) Readdir(n int) ([]os.FileInfo, error) {
if n > 0 {
- return nil, errors.New("not implemented")
+ return nil, pathError("readdir", d.name, errors.New("not implemented"))
}
return d.entries, nil
}
@@ -322,3 +320,7 @@ func (fi fakeFileInfo) IsDir() bool {
func (fi fakeFileInfo) Sys() interface{} {
return nil
}
+
+func pathError(op, name string, err error) *os.PathError {
+ return &os.PathError{Op: op, Path: name, Err: err}
+}
diff --git a/internal/fs/fs_reader_test.go b/internal/fs/fs_reader_test.go
index 173aec10d..d3ef5608a 100644
--- a/internal/fs/fs_reader_test.go
+++ b/internal/fs/fs_reader_test.go
@@ -2,7 +2,8 @@ package fs
import (
"bytes"
- "io/ioutil"
+ "errors"
+ "io"
"os"
"path"
"sort"
@@ -20,7 +21,7 @@ func verifyFileContentOpen(t testing.TB, fs FS, filename string, want []byte) {
t.Fatal(err)
}
- buf, err := ioutil.ReadAll(f)
+ buf, err := io.ReadAll(f)
if err != nil {
t.Fatal(err)
}
@@ -41,7 +42,7 @@ func verifyFileContentOpenFile(t testing.TB, fs FS, filename string, want []byte
t.Fatal(err)
}
- buf, err := ioutil.ReadAll(f)
+ buf, err := io.ReadAll(f)
if err != nil {
t.Fatal(err)
}
@@ -288,7 +289,7 @@ func TestFSReader(t *testing.T) {
name: "dir/Lstat-error-not-exist",
f: func(t *testing.T, fs FS) {
_, err := fs.Lstat("other")
- if err != os.ErrNotExist {
+ if !errors.Is(err, os.ErrNotExist) {
t.Fatal(err)
}
},
@@ -320,7 +321,7 @@ func TestFSReader(t *testing.T) {
for _, test := range tests {
fs := &Reader{
Name: filename,
- ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+ ReadCloser: io.NopCloser(bytes.NewReader(data)),
Mode: 0644,
Size: int64(len(data)),
@@ -355,7 +356,7 @@ func TestFSReaderDir(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
fs := &Reader{
Name: test.filename,
- ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),
+ ReadCloser: io.NopCloser(bytes.NewReader(data)),
Mode: 0644,
Size: int64(len(data)),
@@ -410,7 +411,7 @@ func TestFSReaderMinFileSize(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
fs := &Reader{
Name: "testfile",
- ReadCloser: ioutil.NopCloser(strings.NewReader(test.data)),
+ ReadCloser: io.NopCloser(strings.NewReader(test.data)),
Mode: 0644,
ModTime: time.Now(),
AllowEmptyFile: test.allowEmpty,
@@ -421,7 +422,7 @@ func TestFSReaderMinFileSize(t *testing.T) {
t.Fatal(err)
}
- buf, err := ioutil.ReadAll(f)
+ buf, err := io.ReadAll(f)
if test.readMustErr {
if err == nil {
t.Fatal("expected error not found, got nil")
diff --git a/internal/fs/setflags_linux_test.go b/internal/fs/setflags_linux_test.go
index 089b7c615..b561a1009 100644
--- a/internal/fs/setflags_linux_test.go
+++ b/internal/fs/setflags_linux_test.go
@@ -2,7 +2,6 @@ package fs
import (
"io"
- "io/ioutil"
"os"
"testing"
"time"
@@ -13,7 +12,7 @@ import (
)
func TestNoatime(t *testing.T) {
- f, err := ioutil.TempFile("", "restic-test-noatime")
+ f, err := os.CreateTemp("", "restic-test-noatime")
if err != nil {
t.Fatal(err)
}
diff --git a/internal/fs/stat_bsd.go b/internal/fs/stat_bsd.go
index 33a7879f4..11e075b50 100644
--- a/internal/fs/stat_bsd.go
+++ b/internal/fs/stat_bsd.go
@@ -4,7 +4,6 @@
package fs
import (
- "fmt"
"os"
"syscall"
"time"
@@ -12,10 +11,7 @@ import (
// extendedStat extracts info into an ExtendedFileInfo for unix based operating systems.
func extendedStat(fi os.FileInfo) ExtendedFileInfo {
- s, ok := fi.Sys().(*syscall.Stat_t)
- if !ok {
- panic(fmt.Sprintf("conversion to syscall.Stat_t failed, type is %T", fi.Sys()))
- }
+ s := fi.Sys().(*syscall.Stat_t)
extFI := ExtendedFileInfo{
FileInfo: fi,
diff --git a/internal/fs/stat_test.go b/internal/fs/stat_test.go
index 43e514047..a5ec77c7a 100644
--- a/internal/fs/stat_test.go
+++ b/internal/fs/stat_test.go
@@ -1,7 +1,7 @@
package fs
import (
- "io/ioutil"
+ "os"
"path/filepath"
"testing"
@@ -9,11 +9,9 @@ import (
)
func TestExtendedStat(t *testing.T) {
- tempdir, cleanup := restictest.TempDir(t)
- defer cleanup()
-
+ tempdir := restictest.TempDir(t)
filename := filepath.Join(tempdir, "file")
- err := ioutil.WriteFile(filename, []byte("foobar"), 0640)
+ err := os.WriteFile(filename, []byte("foobar"), 0640)
if err != nil {
t.Fatal(err)
}
diff --git a/internal/fs/stat_unix.go b/internal/fs/stat_unix.go
index bf0d5ceca..c55571031 100644
--- a/internal/fs/stat_unix.go
+++ b/internal/fs/stat_unix.go
@@ -4,7 +4,6 @@
package fs
import (
- "fmt"
"os"
"syscall"
"time"
@@ -12,10 +11,7 @@ import (
// extendedStat extracts info into an ExtendedFileInfo for unix based operating systems.
func extendedStat(fi os.FileInfo) ExtendedFileInfo {
- s, ok := fi.Sys().(*syscall.Stat_t)
- if !ok {
- panic(fmt.Sprintf("conversion to syscall.Stat_t failed, type is %T", fi.Sys()))
- }
+ s := fi.Sys().(*syscall.Stat_t)
extFI := ExtendedFileInfo{
FileInfo: fi,
diff --git a/internal/fuse/dir.go b/internal/fuse/dir.go
index dcacaa96a..3984f15af 100644
--- a/internal/fuse/dir.go
+++ b/internal/fuse/dir.go
@@ -5,12 +5,13 @@ package fuse
import (
"context"
+ "errors"
"os"
"path/filepath"
"sync"
- "bazil.org/fuse"
- "bazil.org/fuse/fs"
+ "github.com/anacrolix/fuse"
+ "github.com/anacrolix/fuse/fs"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
@@ -33,7 +34,7 @@ func cleanupNodeName(name string) string {
return filepath.Base(name)
}
-func newDir(ctx context.Context, root *Root, inode, parentInode uint64, node *restic.Node) (*dir, error) {
+func newDir(root *Root, inode, parentInode uint64, node *restic.Node) (*dir, error) {
debug.Log("new dir for %v (%v)", node.Name, node.Subtree)
return &dir{
@@ -44,6 +45,16 @@ func newDir(ctx context.Context, root *Root, inode, parentInode uint64, node *re
}, nil
}
+// returing a wrapped context.Canceled error will instead result in returing
+// an input / output error to the user. Thus unwrap the error to match the
+// expectations of bazil/fuse
+func unwrapCtxCanceled(err error) error {
+ if errors.Is(err, context.Canceled) {
+ return context.Canceled
+ }
+ return err
+}
+
// replaceSpecialNodes replaces nodes with name "." and "/" by their contents.
// Otherwise, the node is returned.
func replaceSpecialNodes(ctx context.Context, repo restic.Repository, node *restic.Node) ([]*restic.Node, error) {
@@ -57,13 +68,13 @@ func replaceSpecialNodes(ctx context.Context, repo restic.Repository, node *rest
tree, err := restic.LoadTree(ctx, repo, *node.Subtree)
if err != nil {
- return nil, err
+ return nil, unwrapCtxCanceled(err)
}
return tree.Nodes, nil
}
-func newDirFromSnapshot(ctx context.Context, root *Root, inode uint64, snapshot *restic.Snapshot) (*dir, error) {
+func newDirFromSnapshot(root *Root, inode uint64, snapshot *restic.Snapshot) (*dir, error) {
debug.Log("new dir for snapshot %v (%v)", snapshot.ID(), snapshot.Tree)
return &dir{
root: root,
@@ -91,7 +102,7 @@ func (d *dir) open(ctx context.Context) error {
tree, err := restic.LoadTree(ctx, d.root.repo, *d.node.Subtree)
if err != nil {
debug.Log(" error loading tree %v: %v", d.node.Subtree, err)
- return err
+ return unwrapCtxCanceled(err)
}
items := make(map[string]*restic.Node)
for _, n := range tree.Nodes {
@@ -171,7 +182,7 @@ func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
}
ret = append(ret, fuse.Dirent{
- Inode: fs.GenerateDynamicInode(d.inode, name),
+ Inode: inodeFromNode(d.inode, node),
Type: typ,
Name: name,
})
@@ -193,15 +204,16 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
debug.Log(" Lookup(%v) -> not found", name)
return nil, fuse.ENOENT
}
+ inode := inodeFromNode(d.inode, node)
switch node.Type {
case "dir":
- return newDir(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), d.inode, node)
+ return newDir(d.root, inode, d.inode, node)
case "file":
- return newFile(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), node)
+ return newFile(d.root, inode, node)
case "symlink":
- return newLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), node)
+ return newLink(d.root, inode, node)
case "dev", "chardev", "fifo", "socket":
- return newOther(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), node)
+ return newOther(d.root, inode, node)
default:
debug.Log(" node %v has unknown type %v", name, node.Type)
return nil, fuse.ENOENT
diff --git a/internal/fuse/file.go b/internal/fuse/file.go
index 571d5a865..28ff5d450 100644
--- a/internal/fuse/file.go
+++ b/internal/fuse/file.go
@@ -11,8 +11,8 @@ import (
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/restic"
- "bazil.org/fuse"
- "bazil.org/fuse/fs"
+ "github.com/anacrolix/fuse"
+ "github.com/anacrolix/fuse/fs"
)
// The default block size to report in stat
@@ -36,7 +36,7 @@ type openFile struct {
cumsize []uint64
}
-func newFile(ctx context.Context, root *Root, inode uint64, node *restic.Node) (fusefile *file, err error) {
+func newFile(root *Root, inode uint64, node *restic.Node) (fusefile *file, err error) {
debug.Log("create new file for %v with %d blobs", node.Name, len(node.Content))
return &file{
inode: inode,
@@ -105,7 +105,7 @@ func (f *openFile) getBlobAt(ctx context.Context, i int) (blob []byte, err error
blob, err = f.root.repo.LoadBlob(ctx, restic.DataBlob, f.node.Content[i], nil)
if err != nil {
debug.Log("LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
- return nil, err
+ return nil, unwrapCtxCanceled(err)
}
f.root.blobCache.Add(f.node.Content[i], blob)
diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go
index df24f77af..e71bf6fee 100644
--- a/internal/fuse/fuse_test.go
+++ b/internal/fuse/fuse_test.go
@@ -15,8 +15,8 @@ import (
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
- "bazil.org/fuse"
- "bazil.org/fuse/fs"
+ "github.com/anacrolix/fuse"
+ "github.com/anacrolix/fuse/fs"
rtest "github.com/restic/restic/internal/test"
)
@@ -65,8 +65,7 @@ func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree {
}
func TestFuseFile(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -118,8 +117,8 @@ func TestFuseFile(t *testing.T) {
}
root := &Root{repo: repo, blobCache: bloblru.New(blobCacheSize)}
- inode := fs.GenerateDynamicInode(1, "foo")
- f, err := newFile(context.TODO(), root, inode, node)
+ inode := inodeFromNode(1, node)
+ f, err := newFile(root, inode, node)
rtest.OK(t, err)
of, err := f.Open(context.TODO(), nil, nil)
rtest.OK(t, err)
@@ -148,8 +147,7 @@ func TestFuseFile(t *testing.T) {
}
func TestFuseDir(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
root := &Root{repo: repo, blobCache: bloblru.New(blobCacheSize)}
@@ -161,9 +159,9 @@ func TestFuseDir(t *testing.T) {
ChangeTime: time.Unix(1606773732, 0),
ModTime: time.Unix(1606773733, 0),
}
- parentInode := fs.GenerateDynamicInode(0, "parent")
- inode := fs.GenerateDynamicInode(1, "foo")
- d, err := newDir(context.TODO(), root, inode, parentInode, node)
+ parentInode := inodeFromName(0, "parent")
+ inode := inodeFromName(1, "foo")
+ d, err := newDir(root, inode, parentInode, node)
rtest.OK(t, err)
// don't open the directory as that would require setting up a proper tree blob
@@ -180,9 +178,7 @@ func TestFuseDir(t *testing.T) {
// Test top-level directories for their UID and GID.
func TestTopUIDGID(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
-
+ repo := repository.TestRepository(t)
restic.TestCreateSnapshot(t, repo, time.Unix(1460289341, 207401672), 0, 0)
testTopUIDGID(t, Config{}, repo, uint32(os.Getuid()), uint32(os.Getgid()))
@@ -219,3 +215,40 @@ func testTopUIDGID(t *testing.T, cfg Config, repo restic.Repository, uid, gid ui
rtest.Equals(t, uint32(0), attr.Uid)
rtest.Equals(t, uint32(0), attr.Gid)
}
+
+func TestInodeFromNode(t *testing.T) {
+ node := &restic.Node{Name: "foo.txt", Type: "chardev", Links: 2}
+ ino1 := inodeFromNode(1, node)
+ ino2 := inodeFromNode(2, node)
+ rtest.Assert(t, ino1 == ino2, "inodes %d, %d of hard links differ", ino1, ino2)
+
+ node.Links = 1
+ ino1 = inodeFromNode(1, node)
+ ino2 = inodeFromNode(2, node)
+ rtest.Assert(t, ino1 != ino2, "same inode %d but different parent", ino1)
+}
+
+var sink uint64
+
+func BenchmarkInode(b *testing.B) {
+ for _, sub := range []struct {
+ name string
+ node restic.Node
+ }{
+ {
+ name: "no_hard_links",
+ node: restic.Node{Name: "a somewhat long-ish filename.svg.bz2", Type: "fifo"},
+ },
+ {
+ name: "hard_link",
+ node: restic.Node{Name: "some other filename", Type: "file", Links: 2},
+ },
+ } {
+ b.Run(sub.name, func(b *testing.B) {
+ b.ReportAllocs()
+ for i := 0; i < b.N; i++ {
+ sink = inodeFromNode(1, &sub.node)
+ }
+ })
+ }
+}
diff --git a/internal/fuse/inode.go b/internal/fuse/inode.go
new file mode 100644
index 000000000..de975b167
--- /dev/null
+++ b/internal/fuse/inode.go
@@ -0,0 +1,44 @@
+//go:build darwin || freebsd || linux
+// +build darwin freebsd linux
+
+package fuse
+
+import (
+ "encoding/binary"
+
+ "github.com/cespare/xxhash/v2"
+ "github.com/restic/restic/internal/restic"
+)
+
+// inodeFromName generates an inode number for a file in a meta dir.
+func inodeFromName(parent uint64, name string) uint64 {
+ inode := parent ^ xxhash.Sum64String(cleanupNodeName(name))
+
+ // Inode 0 is invalid and 1 is the root. Remap those.
+ if inode < 2 {
+ inode += 2
+ }
+ return inode
+}
+
+// inodeFromNode generates an inode number for a file within a snapshot.
+func inodeFromNode(parent uint64, node *restic.Node) (inode uint64) {
+ if node.Links > 1 && node.Type != "dir" {
+ // If node has hard links, give them all the same inode,
+ // irrespective of the parent.
+ var buf [16]byte
+ binary.LittleEndian.PutUint64(buf[:8], node.DeviceID)
+ binary.LittleEndian.PutUint64(buf[8:], node.Inode)
+ inode = xxhash.Sum64(buf[:])
+ } else {
+ // Else, use the name and the parent inode.
+ // node.{DeviceID,Inode} may not even be reliable.
+ inode = parent ^ xxhash.Sum64String(cleanupNodeName(node.Name))
+ }
+
+ // Inode 0 is invalid and 1 is the root. Remap those.
+ if inode < 2 {
+ inode += 2
+ }
+ return inode
+}
diff --git a/internal/fuse/link.go b/internal/fuse/link.go
index 6953ecfb3..f910aadc4 100644
--- a/internal/fuse/link.go
+++ b/internal/fuse/link.go
@@ -6,8 +6,8 @@ package fuse
import (
"context"
- "bazil.org/fuse"
- "bazil.org/fuse/fs"
+ "github.com/anacrolix/fuse"
+ "github.com/anacrolix/fuse/fs"
"github.com/restic/restic/internal/restic"
)
@@ -20,7 +20,7 @@ type link struct {
inode uint64
}
-func newLink(ctx context.Context, root *Root, inode uint64, node *restic.Node) (*link, error) {
+func newLink(root *Root, inode uint64, node *restic.Node) (*link, error) {
return &link{root: root, inode: inode, node: node}, nil
}
diff --git a/internal/fuse/other.go b/internal/fuse/other.go
index f7745172b..1a78403a7 100644
--- a/internal/fuse/other.go
+++ b/internal/fuse/other.go
@@ -6,7 +6,7 @@ package fuse
import (
"context"
- "bazil.org/fuse"
+ "github.com/anacrolix/fuse"
"github.com/restic/restic/internal/restic"
)
@@ -16,7 +16,7 @@ type other struct {
inode uint64
}
-func newOther(ctx context.Context, root *Root, inode uint64, node *restic.Node) (*other, error) {
+func newOther(root *Root, inode uint64, node *restic.Node) (*other, error) {
return &other{root: root, inode: inode, node: node}, nil
}
diff --git a/internal/fuse/root.go b/internal/fuse/root.go
index 63ab96d6d..fc8841964 100644
--- a/internal/fuse/root.go
+++ b/internal/fuse/root.go
@@ -10,7 +10,7 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
- "bazil.org/fuse/fs"
+ "github.com/anacrolix/fuse/fs"
)
// Config holds settings for the fuse mount.
diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go
index 34bcccc4d..977d0ab17 100644
--- a/internal/fuse/snapshots_dir.go
+++ b/internal/fuse/snapshots_dir.go
@@ -10,8 +10,8 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
- "bazil.org/fuse"
- "bazil.org/fuse/fs"
+ "github.com/anacrolix/fuse"
+ "github.com/anacrolix/fuse/fs"
)
// SnapshotsDir is a actual fuse directory generated from SnapshotsDirStructure
@@ -58,7 +58,7 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
// update snapshots
meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
if err != nil {
- return nil, err
+ return nil, unwrapCtxCanceled(err)
} else if meta == nil {
return nil, fuse.ENOENT
}
@@ -78,7 +78,7 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
for name, entry := range meta.names {
d := fuse.Dirent{
- Inode: fs.GenerateDynamicInode(d.inode, name),
+ Inode: inodeFromName(d.inode, name),
Name: name,
Type: fuse.DT_Dir,
}
@@ -97,19 +97,20 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error)
meta, err := d.dirStruct.UpdatePrefix(ctx, d.prefix)
if err != nil {
- return nil, err
+ return nil, unwrapCtxCanceled(err)
} else if meta == nil {
return nil, fuse.ENOENT
}
entry := meta.names[name]
if entry != nil {
+ inode := inodeFromName(d.inode, name)
if entry.linkTarget != "" {
- return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), entry.linkTarget, entry.snapshot)
+ return newSnapshotLink(d.root, inode, entry.linkTarget, entry.snapshot)
} else if entry.snapshot != nil {
- return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), entry.snapshot)
+ return newDirFromSnapshot(d.root, inode, entry.snapshot)
} else {
- return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.inode, name), d.inode, d.dirStruct, d.prefix+"/"+name), nil
+ return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil
}
}
@@ -127,7 +128,7 @@ type snapshotLink struct {
var _ = fs.NodeReadlinker(&snapshotLink{})
// newSnapshotLink
-func newSnapshotLink(ctx context.Context, root *Root, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
+func newSnapshotLink(root *Root, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) {
return &snapshotLink{root: root, inode: inode, target: target, snapshot: snapshot}, nil
}
diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go
index d0eb080c7..f8e66d076 100644
--- a/internal/fuse/snapshots_dirstruct.go
+++ b/internal/fuse/snapshots_dirstruct.go
@@ -4,6 +4,7 @@
package fuse
import (
+ "bytes"
"context"
"fmt"
"path"
@@ -14,6 +15,8 @@ import (
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
+
+ "github.com/minio/sha256-simd"
)
type MetaDirData struct {
@@ -39,7 +42,7 @@ type SnapshotsDirStructure struct {
// that way we don't need path processing special cases when using the entries tree
entries map[string]*MetaDirData
- snCount int
+ hash [sha256.Size]byte // Hash at last check.
lastCheck time.Time
}
@@ -49,7 +52,6 @@ func NewSnapshotsDirStructure(root *Root, pathTemplates []string, timeTemplate s
root: root,
pathTemplates: pathTemplates,
timeTemplate: timeTemplate,
- snCount: -1,
}
}
@@ -292,14 +294,36 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
return nil
}
- snapshots, err := restic.FindFilteredSnapshots(ctx, d.root.repo.Backend(), d.root.repo, d.root.cfg.Hosts, d.root.cfg.Tags, d.root.cfg.Paths)
+ var snapshots restic.Snapshots
+ err := restic.FindFilteredSnapshots(ctx, d.root.repo.Backend(), d.root.repo, d.root.cfg.Hosts, d.root.cfg.Tags, d.root.cfg.Paths, nil, func(id string, sn *restic.Snapshot, err error) error {
+ if sn != nil {
+ snapshots = append(snapshots, sn)
+ }
+ return nil
+ })
if err != nil {
return err
}
- // sort snapshots ascending by time (default order is descending)
- sort.Sort(sort.Reverse(snapshots))
- if d.snCount == len(snapshots) {
+ // Sort snapshots ascending by time, using the id to break ties.
+ // This needs to be done before hashing.
+ sort.Slice(snapshots, func(i, j int) bool {
+ si, sj := snapshots[i], snapshots[j]
+ if si.Time.Equal(sj.Time) {
+ return bytes.Compare(si.ID()[:], sj.ID()[:]) < 0
+ }
+ return si.Time.Before(sj.Time)
+ })
+
+ // We update the snapshots when the hash of their id's changes.
+ h := sha256.New()
+ for _, sn := range snapshots {
+ h.Write(sn.ID()[:])
+ }
+ var hash [sha256.Size]byte
+ h.Sum(hash[:0])
+
+ if d.hash == hash {
d.lastCheck = time.Now()
return nil
}
@@ -310,7 +334,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error {
}
d.lastCheck = time.Now()
- d.snCount = len(snapshots)
+ d.hash = hash
d.makeDirs(snapshots)
return nil
}
diff --git a/internal/hashing/reader_test.go b/internal/hashing/reader_test.go
index d17f264de..ec2151fe6 100644
--- a/internal/hashing/reader_test.go
+++ b/internal/hashing/reader_test.go
@@ -5,7 +5,6 @@ import (
"crypto/rand"
"crypto/sha256"
"io"
- "io/ioutil"
"testing"
)
@@ -22,7 +21,7 @@ func TestReader(t *testing.T) {
expectedHash := sha256.Sum256(data)
rd := NewReader(bytes.NewReader(data), sha256.New())
- n, err := io.Copy(ioutil.Discard, rd)
+ n, err := io.Copy(io.Discard, rd)
if err != nil {
t.Fatal(err)
}
@@ -54,7 +53,7 @@ func BenchmarkReader(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
rd := NewReader(bytes.NewReader(buf), sha256.New())
- n, err := io.Copy(ioutil.Discard, rd)
+ n, err := io.Copy(io.Discard, rd)
if err != nil {
b.Fatal(err)
}
diff --git a/internal/hashing/writer_test.go b/internal/hashing/writer_test.go
index 46999f20f..d1a5495e6 100644
--- a/internal/hashing/writer_test.go
+++ b/internal/hashing/writer_test.go
@@ -5,7 +5,6 @@ import (
"crypto/rand"
"crypto/sha256"
"io"
- "io/ioutil"
"testing"
)
@@ -21,7 +20,7 @@ func TestWriter(t *testing.T) {
expectedHash := sha256.Sum256(data)
- wr := NewWriter(ioutil.Discard, sha256.New())
+ wr := NewWriter(io.Discard, sha256.New())
n, err := io.Copy(wr, bytes.NewReader(data))
if err != nil {
@@ -54,7 +53,7 @@ func BenchmarkWriter(b *testing.B) {
b.SetBytes(int64(len(buf)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
- wr := NewWriter(ioutil.Discard, sha256.New())
+ wr := NewWriter(io.Discard, sha256.New())
n, err := io.Copy(wr, bytes.NewReader(buf))
if err != nil {
b.Fatal(err)
diff --git a/internal/repository/index.go b/internal/index/index.go
similarity index 86%
rename from internal/repository/index.go
rename to internal/index/index.go
index 28863436b..ecd481594 100644
--- a/internal/repository/index.go
+++ b/internal/index/index.go
@@ -1,4 +1,4 @@
-package repository
+package index
import (
"context"
@@ -43,10 +43,9 @@ import (
// Index holds lookup tables for id -> pack.
type Index struct {
- m sync.Mutex
- byType [restic.NumBlobTypes]indexMap
- packs restic.IDs
- mixedPacks restic.IDSet
+ m sync.Mutex
+ byType [restic.NumBlobTypes]indexMap
+ packs restic.IDs
final bool // set to true for all indexes read from the backend ("finalized")
ids restic.IDs // set to the IDs of the contained finalized indexes
@@ -57,8 +56,7 @@ type Index struct {
// NewIndex returns a new index.
func NewIndex() *Index {
return &Index{
- mixedPacks: restic.NewIDSet(),
- created: time.Now(),
+ created: time.Now(),
}
}
@@ -217,34 +215,22 @@ func (idx *Index) AddToSupersedes(ids ...restic.ID) error {
return nil
}
-// Each returns a channel that yields all blobs known to the index. When the
-// context is cancelled, the background goroutine terminates. This blocks any
+// Each passes all blobs known to the index to the callback fn. This blocks any
// modification of the index.
-func (idx *Index) Each(ctx context.Context) <-chan restic.PackedBlob {
+func (idx *Index) Each(ctx context.Context, fn func(restic.PackedBlob)) {
idx.m.Lock()
+ defer idx.m.Unlock()
- ch := make(chan restic.PackedBlob)
-
- go func() {
- defer idx.m.Unlock()
- defer func() {
- close(ch)
- }()
-
- for typ := range idx.byType {
- m := &idx.byType[typ]
- m.foreach(func(e *indexEntry) bool {
- select {
- case <-ctx.Done():
- return false
- case ch <- idx.toPackedBlob(e, restic.BlobType(typ)):
- return true
- }
- })
- }
- }()
-
- return ch
+ for typ := range idx.byType {
+ m := &idx.byType[typ]
+ m.foreach(func(e *indexEntry) bool {
+ if ctx.Err() != nil {
+ return false
+ }
+ fn(idx.toPackedBlob(e, restic.BlobType(typ)))
+ return true
+ })
+ }
}
type EachByPackResult struct {
@@ -266,21 +252,18 @@ func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-
go func() {
defer idx.m.Unlock()
- defer func() {
- close(ch)
- }()
+ defer close(ch)
- byPack := make(map[restic.ID][][]*indexEntry)
+ byPack := make(map[restic.ID][restic.NumBlobTypes][]*indexEntry)
for typ := range idx.byType {
m := &idx.byType[typ]
m.foreach(func(e *indexEntry) bool {
packID := idx.packs[e.packIndex]
if !idx.final || !packBlacklist.Has(packID) {
- if _, ok := byPack[packID]; !ok {
- byPack[packID] = make([][]*indexEntry, restic.NumBlobTypes)
- }
- byPack[packID][typ] = append(byPack[packID][typ], e)
+ v := byPack[packID]
+ v[typ] = append(v[typ], e)
+ byPack[packID] = v
}
return true
})
@@ -294,6 +277,8 @@ func (idx *Index) EachByPack(ctx context.Context, packBlacklist restic.IDSet) <-
result.Blobs = append(result.Blobs, idx.toPackedBlob(e, restic.BlobType(typ)).Blob)
}
}
+ // allow GC once entry is no longer necessary
+ delete(byPack, packID)
select {
case <-ctx.Done():
return
@@ -332,9 +317,9 @@ type blobJSON struct {
}
// generatePackList returns a list of packs.
-func (idx *Index) generatePackList() ([]*packJSON, error) {
- list := []*packJSON{}
- packs := make(map[restic.ID]*packJSON)
+func (idx *Index) generatePackList() ([]packJSON, error) {
+ list := make([]packJSON, 0, len(idx.packs))
+ packs := make(map[restic.ID]int, len(list)) // Maps to index in list.
for typ := range idx.byType {
m := &idx.byType[typ]
@@ -344,18 +329,13 @@ func (idx *Index) generatePackList() ([]*packJSON, error) {
panic("null pack id")
}
- debug.Log("handle blob %v", e.id)
-
- // see if pack is already in map
- p, ok := packs[packID]
+ i, ok := packs[packID]
if !ok {
- // else create new pack
- p = &packJSON{ID: packID}
-
- // and append it to the list and map
- list = append(list, p)
- packs[p.ID] = p
+ i = len(list)
+ list = append(list, packJSON{ID: packID})
+ packs[packID] = i
}
+ p := &list[i]
// add blob
p.Blobs = append(p.Blobs, blobJSON{
@@ -370,14 +350,12 @@ func (idx *Index) generatePackList() ([]*packJSON, error) {
})
}
- debug.Log("done")
-
return list, nil
}
type jsonIndex struct {
- Supersedes restic.IDs `json:"supersedes,omitempty"`
- Packs []*packJSON `json:"packs"`
+ Supersedes restic.IDs `json:"supersedes,omitempty"`
+ Packs []packJSON `json:"packs"`
}
// Encode writes the JSON serialization of the index to the writer w.
@@ -472,11 +450,6 @@ func (idx *Index) Dump(w io.Writer) error {
return nil
}
-// MixedPacks returns an IDSet that contain packs which have mixed blobs.
-func (idx *Index) MixedPacks() restic.IDSet {
- return idx.mixedPacks
-}
-
// merge() merges indexes, i.e. idx.merge(idx2) merges the contents of idx2 into idx.
// During merging exact duplicates are removed; idx2 is not changed by this method.
func (idx *Index) merge(idx2 *Index) error {
@@ -519,7 +492,6 @@ func (idx *Index) merge(idx2 *Index) error {
})
}
- idx.mixedPacks.Merge(idx2.mixedPacks)
idx.ids = append(idx.ids, idx2.ids...)
idx.supersedes = append(idx.supersedes, idx2.supersedes...)
@@ -553,7 +525,6 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, oldFormat bool, err erro
idx = NewIndex()
for _, pack := range idxJSON.Packs {
- var data, tree bool
packID := idx.addToPacks(pack.ID)
for _, blob := range pack.Blobs {
@@ -565,17 +536,6 @@ func DecodeIndex(buf []byte, id restic.ID) (idx *Index, oldFormat bool, err erro
Length: blob.Length,
UncompressedLength: blob.UncompressedLength,
})
-
- switch blob.Type {
- case restic.DataBlob:
- data = true
- case restic.TreeBlob:
- tree = true
- }
- }
-
- if data && tree {
- idx.mixedPacks.Insert(pack.ID)
}
}
idx.supersedes = idxJSON.Supersedes
@@ -599,7 +559,6 @@ func decodeOldIndex(buf []byte) (idx *Index, err error) {
idx = NewIndex()
for _, pack := range list {
- var data, tree bool
packID := idx.addToPacks(pack.ID)
for _, blob := range pack.Blobs {
@@ -611,17 +570,6 @@ func decodeOldIndex(buf []byte) (idx *Index, err error) {
Length: blob.Length,
// no compressed length in the old index format
})
-
- switch blob.Type {
- case restic.DataBlob:
- data = true
- case restic.TreeBlob:
- tree = true
- }
- }
-
- if data && tree {
- idx.mixedPacks.Insert(pack.ID)
}
}
idx.final = true
diff --git a/internal/index/index_parallel.go b/internal/index/index_parallel.go
new file mode 100644
index 000000000..a76b08a4e
--- /dev/null
+++ b/internal/index/index_parallel.go
@@ -0,0 +1,36 @@
+package index
+
+import (
+ "context"
+ "runtime"
+ "sync"
+
+ "github.com/restic/restic/internal/restic"
+)
+
+// ForAllIndexes loads all index files in parallel and calls the given callback.
+// It is guaranteed that the function is not run concurrently. If the callback
+// returns an error, this function is cancelled and also returns that error.
+func ForAllIndexes(ctx context.Context, repo restic.Repository,
+ fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error {
+
+ // decoding an index can take quite some time such that this can be both CPU- or IO-bound
+ // as the whole index is kept in memory anyways, a few workers too much don't matter
+ workerCount := repo.Connections() + uint(runtime.GOMAXPROCS(0))
+
+ var m sync.Mutex
+ return restic.ParallelList(ctx, repo.Backend(), restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error {
+ var err error
+ var idx *Index
+ oldFormat := false
+
+ buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
+ if err == nil {
+ idx, oldFormat, err = DecodeIndex(buf, id)
+ }
+
+ m.Lock()
+ defer m.Unlock()
+ return fn(id, idx, oldFormat, err)
+ })
+}
diff --git a/internal/repository/index_parallel_test.go b/internal/index/index_parallel_test.go
similarity index 69%
rename from internal/repository/index_parallel_test.go
rename to internal/index/index_parallel_test.go
index 0202be05c..760374510 100644
--- a/internal/repository/index_parallel_test.go
+++ b/internal/index/index_parallel_test.go
@@ -1,15 +1,19 @@
-package repository_test
+package index_test
import (
"context"
+ "path/filepath"
"testing"
"github.com/restic/restic/internal/errors"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
+var repoFixture = filepath.Join("..", "repository", "testdata", "test-repo.tar.gz")
+
func TestRepositoryForAllIndexes(t *testing.T) {
repodir, cleanup := rtest.Env(t, repoFixture)
defer cleanup()
@@ -25,7 +29,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// check that all expected indexes are loaded without errors
indexIDs := restic.NewIDSet()
var indexErr error
- rtest.OK(t, repository.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error {
+ rtest.OK(t, index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
if err != nil {
indexErr = err
}
@@ -38,7 +42,7 @@ func TestRepositoryForAllIndexes(t *testing.T) {
// must failed with the returned error
iterErr := errors.New("error to pass upwards")
- err := repository.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *repository.Index, oldFormat bool, err error) error {
+ err := index.ForAllIndexes(context.TODO(), repo, func(id restic.ID, index *index.Index, oldFormat bool, err error) error {
return iterErr
})
diff --git a/internal/repository/index_test.go b/internal/index/index_test.go
similarity index 92%
rename from internal/repository/index_test.go
rename to internal/index/index_test.go
index 6bc7afca2..4f0dbd2a0 100644
--- a/internal/repository/index_test.go
+++ b/internal/index/index_test.go
@@ -1,13 +1,14 @@
-package repository_test
+package index_test
import (
"bytes"
"context"
+ "fmt"
"math/rand"
"sync"
"testing"
- "github.com/restic/restic/internal/repository"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@@ -15,7 +16,7 @@ import (
func TestIndexSerialize(t *testing.T) {
tests := []restic.PackedBlob{}
- idx := repository.NewIndex()
+ idx := index.NewIndex()
// create 50 packs with 20 blobs each
for i := 0; i < 50; i++ {
@@ -51,7 +52,7 @@ func TestIndexSerialize(t *testing.T) {
rtest.OK(t, err)
idx2ID := restic.NewRandomID()
- idx2, oldFormat, err := repository.DecodeIndex(wr.Bytes(), idx2ID)
+ idx2, oldFormat, err := index.DecodeIndex(wr.Bytes(), idx2ID)
rtest.OK(t, err)
rtest.Assert(t, idx2 != nil,
"nil returned for decoded index")
@@ -121,7 +122,7 @@ func TestIndexSerialize(t *testing.T) {
rtest.OK(t, err)
rtest.Equals(t, restic.IDs{id}, ids)
- idx3, oldFormat, err := repository.DecodeIndex(wr3.Bytes(), id)
+ idx3, oldFormat, err := index.DecodeIndex(wr3.Bytes(), id)
rtest.OK(t, err)
rtest.Assert(t, idx3 != nil,
"nil returned for decoded index")
@@ -143,7 +144,7 @@ func TestIndexSerialize(t *testing.T) {
}
func TestIndexSize(t *testing.T) {
- idx := repository.NewIndex()
+ idx := index.NewIndex()
packs := 200
blobCount := 100
@@ -309,7 +310,7 @@ func TestIndexUnserialize(t *testing.T) {
} {
oldIdx := restic.IDs{restic.TestParseID("ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452")}
- idx, oldFormat, err := repository.DecodeIndex(task.idxBytes, restic.NewRandomID())
+ idx, oldFormat, err := index.DecodeIndex(task.idxBytes, restic.NewRandomID())
rtest.OK(t, err)
rtest.Assert(t, !oldFormat, "new index format recognized as old format")
@@ -354,12 +355,12 @@ func TestIndexUnserialize(t *testing.T) {
}
}
-func listPack(idx *repository.Index, id restic.ID) (pbs []restic.PackedBlob) {
- for pb := range idx.Each(context.TODO()) {
+func listPack(idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) {
+ idx.Each(context.TODO(), func(pb restic.PackedBlob) {
if pb.PackID.Equal(id) {
pbs = append(pbs, pb)
}
- }
+ })
return pbs
}
@@ -386,7 +387,7 @@ func BenchmarkDecodeIndex(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
- _, _, err := repository.DecodeIndex(benchmarkIndexJSON, id)
+ _, _, err := index.DecodeIndex(benchmarkIndexJSON, id)
rtest.OK(b, err)
}
}
@@ -399,14 +400,34 @@ func BenchmarkDecodeIndexParallel(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
- _, _, err := repository.DecodeIndex(benchmarkIndexJSON, id)
+ _, _, err := index.DecodeIndex(benchmarkIndexJSON, id)
rtest.OK(b, err)
}
})
}
+func BenchmarkEncodeIndex(b *testing.B) {
+ for _, n := range []int{100, 1000, 10000} {
+ idx, _ := createRandomIndex(rand.New(rand.NewSource(0)), n)
+
+ b.Run(fmt.Sprint(n), func(b *testing.B) {
+ buf := new(bytes.Buffer)
+ err := idx.Encode(buf)
+ rtest.OK(b, err)
+
+ b.ResetTimer()
+ b.ReportAllocs()
+
+ for i := 0; i < b.N; i++ {
+ buf.Reset()
+ _ = idx.Encode(buf)
+ }
+ })
+ }
+}
+
func TestIndexUnserializeOld(t *testing.T) {
- idx, oldFormat, err := repository.DecodeIndex(docOldExample, restic.NewRandomID())
+ idx, oldFormat, err := index.DecodeIndex(docOldExample, restic.NewRandomID())
rtest.OK(t, err)
rtest.Assert(t, oldFormat, "old index format recognized as new format")
@@ -427,7 +448,7 @@ func TestIndexUnserializeOld(t *testing.T) {
}
func TestIndexPacks(t *testing.T) {
- idx := repository.NewIndex()
+ idx := index.NewIndex()
packs := restic.NewIDSet()
for i := 0; i < 20; i++ {
@@ -456,8 +477,8 @@ func NewRandomTestID(rng *rand.Rand) restic.ID {
return id
}
-func createRandomIndex(rng *rand.Rand, packfiles int) (idx *repository.Index, lookupBh restic.BlobHandle) {
- idx = repository.NewIndex()
+func createRandomIndex(rng *rand.Rand, packfiles int) (idx *index.Index, lookupBh restic.BlobHandle) {
+ idx = index.NewIndex()
// create index with given number of pack files
for i := 0; i < packfiles; i++ {
@@ -536,7 +557,7 @@ func BenchmarkIndexAllocParallel(b *testing.B) {
func TestIndexHas(t *testing.T) {
tests := []restic.PackedBlob{}
- idx := repository.NewIndex()
+ idx := index.NewIndex()
// create 50 packs with 20 blobs each
for i := 0; i < 50; i++ {
@@ -576,7 +597,7 @@ func TestIndexHas(t *testing.T) {
}
func TestMixedEachByPack(t *testing.T) {
- idx := repository.NewIndex()
+ idx := index.NewIndex()
expected := make(map[restic.ID]int)
// create 50 packs with 2 blobs each
@@ -615,7 +636,7 @@ func TestMixedEachByPack(t *testing.T) {
}
func TestEachByPackIgnoes(t *testing.T) {
- idx := repository.NewIndex()
+ idx := index.NewIndex()
ignores := restic.NewIDSet()
expected := make(map[restic.ID]int)
diff --git a/internal/repository/indexmap.go b/internal/index/indexmap.go
similarity index 99%
rename from internal/repository/indexmap.go
rename to internal/index/indexmap.go
index 99c3fd331..ef3539d48 100644
--- a/internal/repository/indexmap.go
+++ b/internal/index/indexmap.go
@@ -1,4 +1,4 @@
-package repository
+package index
import (
"hash/maphash"
diff --git a/internal/repository/indexmap_test.go b/internal/index/indexmap_test.go
similarity index 99%
rename from internal/repository/indexmap_test.go
rename to internal/index/indexmap_test.go
index 6699b3601..391131ca0 100644
--- a/internal/repository/indexmap_test.go
+++ b/internal/index/indexmap_test.go
@@ -1,4 +1,4 @@
-package repository
+package index
import (
"math/rand"
diff --git a/internal/repository/master_index.go b/internal/index/master_index.go
similarity index 92%
rename from internal/repository/master_index.go
rename to internal/index/master_index.go
index 955080e82..ca7c16135 100644
--- a/internal/repository/master_index.go
+++ b/internal/index/master_index.go
@@ -1,4 +1,4 @@
-package repository
+package index
import (
"bytes"
@@ -31,7 +31,7 @@ func NewMasterIndex() *MasterIndex {
return &MasterIndex{idx: idx, pendingBlobs: restic.NewBlobSet()}
}
-func (mi *MasterIndex) markCompressed() {
+func (mi *MasterIndex) MarkCompressed() {
mi.compress = true
}
@@ -65,7 +65,7 @@ func (mi *MasterIndex) LookupSize(bh restic.BlobHandle) (uint, bool) {
// Before doing so it checks if this blob is already known.
// Returns true if adding was successful and false if the blob
// was already known
-func (mi *MasterIndex) addPending(bh restic.BlobHandle) bool {
+func (mi *MasterIndex) AddPending(bh restic.BlobHandle) bool {
mi.idxMutex.Lock()
defer mi.idxMutex.Unlock()
@@ -106,18 +106,6 @@ func (mi *MasterIndex) Has(bh restic.BlobHandle) bool {
return false
}
-func (mi *MasterIndex) IsMixedPack(packID restic.ID) bool {
- mi.idxMutex.RLock()
- defer mi.idxMutex.RUnlock()
-
- for _, idx := range mi.idx {
- if idx.MixedPacks().Has(packID) {
- return true
- }
- }
- return false
-}
-
// IDs returns the IDs of all indexes contained in the index.
func (mi *MasterIndex) IDs() restic.IDSet {
mi.idxMutex.RLock()
@@ -150,7 +138,7 @@ func (mi *MasterIndex) Packs(packBlacklist restic.IDSet) restic.IDSet {
packs := restic.NewIDSet()
for _, idx := range mi.idx {
idxPacks := idx.Packs()
- if idx.final {
+ if idx.final && len(packBlacklist) > 0 {
idxPacks = idxPacks.Sub(packBlacklist)
}
packs.Merge(idxPacks)
@@ -234,30 +222,15 @@ func (mi *MasterIndex) finalizeFullIndexes() []*Index {
return list
}
-// Each returns a channel that yields all blobs known to the index. When the
-// context is cancelled, the background goroutine terminates. This blocks any
-// modification of the index.
-func (mi *MasterIndex) Each(ctx context.Context) <-chan restic.PackedBlob {
+// Each runs fn on all blobs known to the index. When the context is cancelled,
+// the index iteration return immediately. This blocks any modification of the index.
+func (mi *MasterIndex) Each(ctx context.Context, fn func(restic.PackedBlob)) {
mi.idxMutex.RLock()
+ defer mi.idxMutex.RUnlock()
- ch := make(chan restic.PackedBlob)
-
- go func() {
- defer mi.idxMutex.RUnlock()
- defer close(ch)
-
- for _, idx := range mi.idx {
- for pb := range idx.Each(ctx) {
- select {
- case <-ctx.Done():
- return
- case ch <- pb:
- }
- }
- }
- }()
-
- return ch
+ for _, idx := range mi.idx {
+ idx.Each(ctx, fn)
+ }
}
// MergeFinalIndexes merges all final indexes together.
@@ -450,14 +423,16 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan
if len(packBlob) == 0 {
continue
}
- for pb := range mi.Each(ctx) {
+ mi.Each(ctx, func(pb restic.PackedBlob) {
if packs.Has(pb.PackID) && pb.PackID[0]&0xf == i {
packBlob[pb.PackID] = append(packBlob[pb.PackID], pb.Blob)
}
- }
+ })
// pass on packs
for packID, pbs := range packBlob {
+ // allow GC
+ packBlob[packID] = nil
select {
case out <- restic.PackBlobs{PackID: packID, Blobs: pbs}:
case <-ctx.Done():
diff --git a/internal/repository/master_index_test.go b/internal/index/master_index_test.go
similarity index 88%
rename from internal/repository/master_index_test.go
rename to internal/index/master_index_test.go
index 2430c83dc..e97339564 100644
--- a/internal/repository/master_index_test.go
+++ b/internal/index/master_index_test.go
@@ -1,4 +1,4 @@
-package repository_test
+package index_test
import (
"context"
@@ -9,6 +9,7 @@ import (
"github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/crypto"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
@@ -57,15 +58,15 @@ func TestMasterIndex(t *testing.T) {
},
}
- idx1 := repository.NewIndex()
+ idx1 := index.NewIndex()
idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
idx1.StorePack(blob12a.PackID, []restic.Blob{blob12a.Blob})
- idx2 := repository.NewIndex()
+ idx2 := index.NewIndex()
idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
idx2.StorePack(blob12b.PackID, []restic.Blob{blob12b.Blob})
- mIdx := repository.NewMasterIndex()
+ mIdx := index.NewMasterIndex()
mIdx.Insert(idx1)
mIdx.Insert(idx2)
@@ -148,24 +149,24 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
},
}
- idx1 := repository.NewIndex()
+ idx1 := index.NewIndex()
idx1.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
- idx2 := repository.NewIndex()
+ idx2 := index.NewIndex()
idx2.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
- mIdx := repository.NewMasterIndex()
+ mIdx := index.NewMasterIndex()
mIdx.Insert(idx1)
mIdx.Insert(idx2)
- finalIndexes, idxCount := repository.TestMergeIndex(t, mIdx)
- rtest.Equals(t, []*repository.Index{idx1, idx2}, finalIndexes)
+ finalIndexes, idxCount := index.TestMergeIndex(t, mIdx)
+ rtest.Equals(t, []*index.Index{idx1, idx2}, finalIndexes)
rtest.Equals(t, 1, idxCount)
blobCount := 0
- for range mIdx.Each(context.TODO()) {
+ mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
blobCount++
- }
+ })
rtest.Equals(t, 2, blobCount)
blobs := mIdx.Lookup(bhInIdx1)
@@ -178,13 +179,13 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
// merge another index containing identical blobs
- idx3 := repository.NewIndex()
+ idx3 := index.NewIndex()
idx3.StorePack(blob1.PackID, []restic.Blob{blob1.Blob})
idx3.StorePack(blob2.PackID, []restic.Blob{blob2.Blob})
mIdx.Insert(idx3)
- finalIndexes, idxCount = repository.TestMergeIndex(t, mIdx)
- rtest.Equals(t, []*repository.Index{idx3}, finalIndexes)
+ finalIndexes, idxCount = index.TestMergeIndex(t, mIdx)
+ rtest.Equals(t, []*index.Index{idx3}, finalIndexes)
rtest.Equals(t, 1, idxCount)
// Index should have same entries as before!
@@ -195,14 +196,14 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
blobCount = 0
- for range mIdx.Each(context.TODO()) {
+ mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
blobCount++
- }
+ })
rtest.Equals(t, 2, blobCount)
}
-func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*repository.MasterIndex, restic.BlobHandle) {
- mIdx := repository.NewMasterIndex()
+func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*index.MasterIndex, restic.BlobHandle) {
+ mIdx := index.NewMasterIndex()
for i := 0; i < num-1; i++ {
idx, _ := createRandomIndex(rng, size)
mIdx.Insert(idx)
@@ -210,7 +211,7 @@ func createRandomMasterIndex(t testing.TB, rng *rand.Rand, num, size int) (*repo
idx1, lookupBh := createRandomIndex(rng, size)
mIdx.Insert(idx1)
- repository.TestMergeIndex(t, mIdx)
+ index.TestMergeIndex(t, mIdx)
return mIdx, lookupBh
}
@@ -308,19 +309,32 @@ func BenchmarkMasterIndexLookupBlobSize(b *testing.B) {
}
}
+func BenchmarkMasterIndexEach(b *testing.B) {
+ rng := rand.New(rand.NewSource(0))
+ mIdx, _ := createRandomMasterIndex(b, rand.New(rng), 5, 200000)
+
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ entries := 0
+ mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
+ entries++
+ })
+ }
+}
+
var (
snapshotTime = time.Unix(1470492820, 207401672)
depth = 3
)
-func createFilledRepo(t testing.TB, snapshots int, dup float32, version uint) (restic.Repository, func()) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
+func createFilledRepo(t testing.TB, snapshots int, dup float32, version uint) restic.Repository {
+ repo := repository.TestRepositoryWithVersion(t, version)
for i := 0; i < 3; i++ {
restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth, dup)
}
-
- return repo, cleanup
+ return repo
}
func TestIndexSave(t *testing.T) {
@@ -328,8 +342,7 @@ func TestIndexSave(t *testing.T) {
}
func testIndexSave(t *testing.T, version uint) {
- repo, cleanup := createFilledRepo(t, 3, 0, version)
- defer cleanup()
+ repo := createFilledRepo(t, 3, 0, version)
err := repo.LoadIndex(context.TODO())
if err != nil {
diff --git a/internal/index/testing.go b/internal/index/testing.go
new file mode 100644
index 000000000..7c05ac651
--- /dev/null
+++ b/internal/index/testing.go
@@ -0,0 +1,18 @@
+package index
+
+import (
+ "testing"
+
+ "github.com/restic/restic/internal/restic"
+ "github.com/restic/restic/internal/test"
+)
+
+func TestMergeIndex(t testing.TB, mi *MasterIndex) ([]*Index, int) {
+ finalIndexes := mi.finalizeNotFinalIndexes()
+ for _, idx := range finalIndexes {
+ test.OK(t, idx.SetID(restic.NewRandomID()))
+ }
+
+ test.OK(t, mi.MergeFinalIndexes())
+ return finalIndexes, len(mi.idx)
+}
diff --git a/internal/migrations/interface.go b/internal/migrations/interface.go
index 99100bce3..63682b46b 100644
--- a/internal/migrations/interface.go
+++ b/internal/migrations/interface.go
@@ -8,8 +8,8 @@ import (
// Migration implements a data migration.
type Migration interface {
- // Check returns true if the migration can be applied to a repo.
- Check(context.Context, restic.Repository) (bool, error)
+ // Check returns true if the migration can be applied to a repo. If the option is not applicable it can return a specific reason.
+ Check(context.Context, restic.Repository) (bool, string, error)
RepoCheck() bool
diff --git a/internal/migrations/s3_layout.go b/internal/migrations/s3_layout.go
index afb14b848..d42b94bf8 100644
--- a/internal/migrations/s3_layout.go
+++ b/internal/migrations/s3_layout.go
@@ -6,7 +6,7 @@ import (
"os"
"path"
- "github.com/restic/restic/internal/backend"
+ "github.com/restic/restic/internal/backend/layout"
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/cache"
"github.com/restic/restic/internal/debug"
@@ -38,19 +38,19 @@ func toS3Backend(repo restic.Repository) *s3.Backend {
}
// Check tests whether the migration can be applied.
-func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, error) {
+func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, string, error) {
be := toS3Backend(repo)
if be == nil {
debug.Log("backend is not s3")
- return false, nil
+ return false, "backend is not s3", nil
}
if be.Layout.Name() != "s3legacy" {
debug.Log("layout is not s3legacy")
- return false, nil
+ return false, "not using the legacy s3 layout", nil
}
- return true, nil
+ return true, "", nil
}
func (m *S3Layout) RepoCheck() bool {
@@ -74,7 +74,7 @@ func retry(max int, fail func(err error), f func() error) error {
// maxErrors for retrying renames on s3.
const maxErrors = 20
-func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layout, t restic.FileType) error {
+func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layout, t restic.FileType) error {
printErr := func(err error) {
fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err)
}
@@ -97,12 +97,12 @@ func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error {
return errors.New("backend is not s3")
}
- oldLayout := &backend.S3LegacyLayout{
+ oldLayout := &layout.S3LegacyLayout{
Path: be.Path(),
Join: path.Join,
}
- newLayout := &backend.DefaultLayout{
+ newLayout := &layout.DefaultLayout{
Path: be.Path(),
Join: path.Join,
}
diff --git a/internal/migrations/upgrade_repo_v2.go b/internal/migrations/upgrade_repo_v2.go
index b29fbcdcc..43a6cd91c 100644
--- a/internal/migrations/upgrade_repo_v2.go
+++ b/internal/migrations/upgrade_repo_v2.go
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"io"
- "io/ioutil"
"os"
"path/filepath"
@@ -45,9 +44,13 @@ func (*UpgradeRepoV2) Desc() string {
return "upgrade a repository to version 2"
}
-func (*UpgradeRepoV2) Check(ctx context.Context, repo restic.Repository) (bool, error) {
+func (*UpgradeRepoV2) Check(ctx context.Context, repo restic.Repository) (bool, string, error) {
isV1 := repo.Config().Version == 1
- return isV1, nil
+ reason := ""
+ if !isV1 {
+ reason = fmt.Sprintf("repository is already upgraded to version %v", repo.Config().Version)
+ }
+ return isV1, reason, nil
}
func (*UpgradeRepoV2) RepoCheck() bool {
@@ -77,7 +80,7 @@ func (*UpgradeRepoV2) upgrade(ctx context.Context, repo restic.Repository) error
}
func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error {
- tempdir, err := ioutil.TempDir("", "restic-migrate-upgrade-repo-v2-")
+ tempdir, err := os.MkdirTemp("", "restic-migrate-upgrade-repo-v2-")
if err != nil {
return fmt.Errorf("create temp dir failed: %w", err)
}
@@ -87,7 +90,7 @@ func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error
// read raw config file and save it to a temp dir, just in case
var rawConfigFile []byte
err = repo.Backend().Load(ctx, h, 0, 0, func(rd io.Reader) (err error) {
- rawConfigFile, err = ioutil.ReadAll(rd)
+ rawConfigFile, err = io.ReadAll(rd)
return err
})
if err != nil {
@@ -95,7 +98,7 @@ func (m *UpgradeRepoV2) Apply(ctx context.Context, repo restic.Repository) error
}
backupFileName := filepath.Join(tempdir, "config")
- err = ioutil.WriteFile(backupFileName, rawConfigFile, 0600)
+ err = os.WriteFile(backupFileName, rawConfigFile, 0600)
if err != nil {
return fmt.Errorf("write config file backup to %v failed: %w", tempdir, err)
}
diff --git a/internal/migrations/upgrade_repo_v2_test.go b/internal/migrations/upgrade_repo_v2_test.go
index 0d86d265c..96fc7788e 100644
--- a/internal/migrations/upgrade_repo_v2_test.go
+++ b/internal/migrations/upgrade_repo_v2_test.go
@@ -14,16 +14,14 @@ import (
)
func TestUpgradeRepoV2(t *testing.T) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, 1)
- defer cleanup()
-
+ repo := repository.TestRepositoryWithVersion(t, 1)
if repo.Config().Version != 1 {
t.Fatal("test repo has wrong version")
}
m := &UpgradeRepoV2{}
- ok, err := m.Check(context.Background(), repo)
+ ok, _, err := m.Check(context.Background(), repo)
if err != nil {
t.Fatal(err)
}
@@ -63,8 +61,7 @@ func (be *failBackend) Save(ctx context.Context, h restic.Handle, rd restic.Rewi
}
func TestUpgradeRepoV2Failure(t *testing.T) {
- be, cleanup := repository.TestBackend(t)
- defer cleanup()
+ be := repository.TestBackend(t)
// wrap backend so that it fails upgrading the config after the initial write
be = &failBackend{
@@ -72,16 +69,14 @@ func TestUpgradeRepoV2Failure(t *testing.T) {
Backend: be,
}
- repo, cleanup := repository.TestRepositoryWithBackend(t, be, 1)
- defer cleanup()
-
+ repo := repository.TestRepositoryWithBackend(t, be, 1)
if repo.Config().Version != 1 {
t.Fatal("test repo has wrong version")
}
m := &UpgradeRepoV2{}
- ok, err := m.Check(context.Background(), repo)
+ ok, _, err := m.Check(context.Background(), repo)
if err != nil {
t.Fatal(err)
}
diff --git a/internal/options/options.go b/internal/options/options.go
index 43d9b63da..7490ac430 100644
--- a/internal/options/options.go
+++ b/internal/options/options.go
@@ -92,14 +92,10 @@ func (h helpList) Swap(i, j int) {
// splitKeyValue splits at the first equals (=) sign.
func splitKeyValue(s string) (key string, value string) {
- data := strings.SplitN(s, "=", 2)
- key = strings.ToLower(strings.TrimSpace(data[0]))
- if len(data) == 1 {
- // no equals sign is treated as the empty value
- return key, ""
- }
-
- return key, strings.TrimSpace(data[1])
+ key, value, _ = strings.Cut(s, "=")
+ key = strings.ToLower(strings.TrimSpace(key))
+ value = strings.TrimSpace(value)
+ return key, value
}
// Parse takes a slice of key=value pairs and returns an Options type.
diff --git a/internal/pack/pack.go b/internal/pack/pack.go
index 11be41697..34ad9d071 100644
--- a/internal/pack/pack.go
+++ b/internal/pack/pack.go
@@ -370,7 +370,7 @@ func CalculateHeaderSize(blobs []restic.Blob) int {
func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) map[restic.ID]int64 {
packSize := make(map[restic.ID]int64)
- for blob := range mi.Each(ctx) {
+ mi.Each(ctx, func(blob restic.PackedBlob) {
size, ok := packSize[blob.PackID]
if !ok {
size = headerSize
@@ -379,7 +379,7 @@ func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) map[restic.I
size += int64(blob.Length)
}
packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob))
- }
+ })
return packSize
}
diff --git a/internal/repository/fuzz_test.go b/internal/repository/fuzz_test.go
index 7a98477b6..b4036288c 100644
--- a/internal/repository/fuzz_test.go
+++ b/internal/repository/fuzz_test.go
@@ -1,6 +1,3 @@
-//go:build go1.18
-// +build go1.18
-
package repository
import (
@@ -22,7 +19,7 @@ func FuzzSaveLoadBlob(f *testing.F) {
}
id := restic.Hash(blob)
- repo, _ := TestRepositoryWithBackend(t, mem.New(), 2)
+ repo := TestRepositoryWithBackend(t, mem.New(), 2)
var wg errgroup.Group
repo.StartPackUploader(context.TODO(), &wg)
diff --git a/internal/repository/index_parallel.go b/internal/repository/index_parallel.go
deleted file mode 100644
index dcf33113e..000000000
--- a/internal/repository/index_parallel.go
+++ /dev/null
@@ -1,79 +0,0 @@
-package repository
-
-import (
- "context"
- "runtime"
- "sync"
-
- "github.com/restic/restic/internal/debug"
- "github.com/restic/restic/internal/restic"
- "golang.org/x/sync/errgroup"
-)
-
-// ForAllIndexes loads all index files in parallel and calls the given callback.
-// It is guaranteed that the function is not run concurrently. If the callback
-// returns an error, this function is cancelled and also returns that error.
-func ForAllIndexes(ctx context.Context, repo restic.Repository,
- fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error {
-
- debug.Log("Start")
-
- type FileInfo struct {
- restic.ID
- Size int64
- }
-
- var m sync.Mutex
-
- // track spawned goroutines using wg, create a new context which is
- // cancelled as soon as an error occurs.
- wg, ctx := errgroup.WithContext(ctx)
-
- ch := make(chan FileInfo)
- // send list of index files through ch, which is closed afterwards
- wg.Go(func() error {
- defer close(ch)
- return repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
- select {
- case <-ctx.Done():
- return ctx.Err()
- case ch <- FileInfo{id, size}:
- }
- return nil
- })
- })
-
- // a worker receives an index ID from ch, loads the index, and sends it to indexCh
- worker := func() error {
- var buf []byte
- for fi := range ch {
- debug.Log("worker got file %v", fi.ID.Str())
- var err error
- var idx *Index
- oldFormat := false
-
- buf, err = repo.LoadUnpacked(ctx, restic.IndexFile, fi.ID, buf[:0])
- if err == nil {
- idx, oldFormat, err = DecodeIndex(buf, fi.ID)
- }
-
- m.Lock()
- err = fn(fi.ID, idx, oldFormat, err)
- m.Unlock()
- if err != nil {
- return err
- }
- }
- return nil
- }
-
- // decoding an index can take quite some time such that this can be both CPU- or IO-bound
- // as the whole index is kept in memory anyways, a few workers too much don't matter
- workerCount := int(repo.Connections()) + runtime.GOMAXPROCS(0)
- // run workers on ch
- for i := 0; i < workerCount; i++ {
- wg.Go(worker)
- }
-
- return wg.Wait()
-}
diff --git a/internal/repository/key.go b/internal/repository/key.go
index 4ce59a1f5..b3e13ade4 100644
--- a/internal/repository/key.go
+++ b/internal/repository/key.go
@@ -40,7 +40,7 @@ type Key struct {
user *crypto.Key
master *crypto.Key
- name string
+ id restic.ID
}
// Params tracks the parameters used for the KDF. If not set, it will be
@@ -62,10 +62,10 @@ func createMasterKey(ctx context.Context, s *Repository, password string) (*Key,
}
// OpenKey tries do decrypt the key specified by name with the given password.
-func OpenKey(ctx context.Context, s *Repository, name string, password string) (*Key, error) {
- k, err := LoadKey(ctx, s, name)
+func OpenKey(ctx context.Context, s *Repository, id restic.ID, password string) (*Key, error) {
+ k, err := LoadKey(ctx, s, id)
if err != nil {
- debug.Log("LoadKey(%v) returned error %v", name, err)
+ debug.Log("LoadKey(%v) returned error %v", id.String(), err)
return nil, err
}
@@ -99,7 +99,7 @@ func OpenKey(ctx context.Context, s *Repository, name string, password string) (
debug.Log("Unmarshal() returned error %v", err)
return nil, errors.Wrap(err, "Unmarshal")
}
- k.name = name
+ k.id = id
if !k.Valid() {
return nil, errors.New("Invalid key for repository")
@@ -136,22 +136,16 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
defer cancel()
// try at most maxKeys keys in repo
- err = s.Backend().List(listCtx, restic.KeyFile, func(fi restic.FileInfo) error {
+ err = s.List(listCtx, restic.KeyFile, func(id restic.ID, size int64) error {
checked++
if maxKeys > 0 && checked > maxKeys {
return ErrMaxKeysReached
}
- _, err := restic.ParseID(fi.Name)
+ debug.Log("trying key %q", id.String())
+ key, err := OpenKey(ctx, s, id, password)
if err != nil {
- debug.Log("rejecting key with invalid name: %v", fi.Name)
- return nil
- }
-
- debug.Log("trying key %q", fi.Name)
- key, err := OpenKey(ctx, s, fi.Name, password)
- if err != nil {
- debug.Log("key %v returned error %v", fi.Name, err)
+ debug.Log("key %v returned error %v", id.String(), err)
// ErrUnauthenticated means the password is wrong, try the next key
if errors.Is(err, crypto.ErrUnauthenticated) {
@@ -161,7 +155,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
return err
}
- debug.Log("successfully opened key %v", fi.Name)
+ debug.Log("successfully opened key %v", id.String())
k = key
cancel()
return nil
@@ -183,8 +177,8 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
}
// LoadKey loads a key from the backend.
-func LoadKey(ctx context.Context, s *Repository, name string) (k *Key, err error) {
- h := restic.Handle{Type: restic.KeyFile, Name: name}
+func LoadKey(ctx context.Context, s *Repository, id restic.ID) (k *Key, err error) {
+ h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
data, err := backend.LoadAll(ctx, nil, s.be, h)
if err != nil {
return nil, err
@@ -274,10 +268,11 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str
return nil, errors.Wrap(err, "Marshal")
}
+ id := restic.Hash(buf)
// store in repository and return
h := restic.Handle{
Type: restic.KeyFile,
- Name: restic.Hash(buf).String(),
+ Name: id.String(),
}
err = s.be.Save(ctx, h, restic.NewByteReader(buf, s.be.Hasher()))
@@ -285,7 +280,7 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str
return nil, err
}
- newkey.name = h.Name
+ newkey.id = id
return newkey, nil
}
@@ -297,9 +292,9 @@ func (k *Key) String() string {
return fmt.Sprintf("<Key of %s@%s, created on %s>", k.Username, k.Hostname, k.Created)
}
-// Name returns an identifier for the key.
-func (k Key) Name() string {
- return k.name
+// ID returns an identifier for the key.
+func (k Key) ID() restic.ID {
+ return k.id
}
// Valid tests whether the mac and encryption keys are valid (i.e. not zero)
diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go
index e83bf8769..4422e3418 100644
--- a/internal/repository/packer_manager.go
+++ b/internal/repository/packer_manager.go
@@ -4,7 +4,6 @@ import (
"bufio"
"context"
"io"
- "io/ioutil"
"os"
"runtime"
"sync"
@@ -111,7 +110,7 @@ func (r *packerManager) newPacker() (packer *Packer, err error) {
debug.Log("create new pack")
tmpfile, err := fs.TempFile("", "restic-temp-pack-")
if err != nil {
- return nil, errors.Wrap(err, "fs.TempFile")
+ return nil, errors.WithStack(err)
}
bufWr := bufio.NewWriter(tmpfile)
@@ -151,14 +150,13 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe
}
hr := hashing.NewReader(rd, sha256.New())
- _, err = io.Copy(ioutil.Discard, hr)
+ _, err = io.Copy(io.Discard, hr)
if err != nil {
return err
}
id := restic.IDFromHash(hr.Sum(nil))
- h := restic.Handle{Type: restic.PackFile, Name: id.String(),
- ContainedBlobType: t}
+ h := restic.Handle{Type: restic.PackFile, Name: id.String(), ContainedBlobType: t}
var beHash []byte
if beHr != nil {
beHash = beHr.Sum(nil)
@@ -185,7 +183,7 @@ func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packe
if runtime.GOOS != "windows" {
err = fs.RemoveIfExists(p.tmpfile.Name())
if err != nil {
- return errors.Wrap(err, "Remove")
+ return errors.WithStack(err)
}
}
diff --git a/internal/repository/repack.go b/internal/repository/repack.go
index bf6b65c8f..6adff69f4 100644
--- a/internal/repository/repack.go
+++ b/internal/repository/repack.go
@@ -12,6 +12,12 @@ import (
"golang.org/x/sync/errgroup"
)
+type repackBlobSet interface {
+ Has(bh restic.BlobHandle) bool
+ Delete(bh restic.BlobHandle)
+ Len() int
+}
+
// Repack takes a list of packs together with a list of blobs contained in
// these packs. Each pack is loaded and the blobs listed in keepBlobs is saved
// into a new pack. Returned is the list of obsolete packs which can then
@@ -19,10 +25,10 @@ import (
//
// The map keepBlobs is modified by Repack, it is used to keep track of which
// blobs have been processed.
-func Repack(ctx context.Context, repo restic.Repository, dstRepo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet, p *progress.Counter) (obsoletePacks restic.IDSet, err error) {
- debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))
+func Repack(ctx context.Context, repo restic.Repository, dstRepo restic.Repository, packs restic.IDSet, keepBlobs repackBlobSet, p *progress.Counter) (obsoletePacks restic.IDSet, err error) {
+ debug.Log("repacking %d packs while keeping %d blobs", len(packs), keepBlobs.Len())
- if repo == dstRepo && dstRepo.Backend().Connections() < 2 {
+ if repo == dstRepo && dstRepo.Connections() < 2 {
return nil, errors.Fatal("repack step requires a backend connection limit of at least two")
}
@@ -41,7 +47,7 @@ func Repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito
return obsoletePacks, nil
}
-func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet, p *progress.Counter) (obsoletePacks restic.IDSet, err error) {
+func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Repository, packs restic.IDSet, keepBlobs repackBlobSet, p *progress.Counter) (obsoletePacks restic.IDSet, err error) {
wg, wgCtx := errgroup.WithContext(ctx)
var keepMutex sync.Mutex
@@ -114,6 +120,10 @@ func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito
// as packs are streamed the concurrency is limited by IO
// reduce by one to ensure that uploading is always possible
repackWorkerCount := int(repo.Connections() - 1)
+ if repo != dstRepo {
+ // no need to share the upload and download connections for different repositories
+ repackWorkerCount = int(repo.Connections())
+ }
for i := 0; i < repackWorkerCount; i++ {
wg.Go(worker)
}
diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go
index f8cefc00b..bb31eba77 100644
--- a/internal/repository/repack_test.go
+++ b/internal/repository/repack_test.go
@@ -6,6 +6,7 @@ import (
"testing"
"time"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
@@ -170,7 +171,7 @@ func flush(t *testing.T, repo restic.Repository) {
}
func rebuildIndex(t *testing.T, repo restic.Repository) {
- err := repo.SetIndex(repository.NewMasterIndex())
+ err := repo.SetIndex(index.NewMasterIndex())
if err != nil {
t.Fatal(err)
}
@@ -207,7 +208,7 @@ func rebuildIndex(t *testing.T, repo restic.Repository) {
}
func reloadIndex(t *testing.T, repo restic.Repository) {
- err := repo.SetIndex(repository.NewMasterIndex())
+ err := repo.SetIndex(index.NewMasterIndex())
if err != nil {
t.Fatal(err)
}
@@ -222,8 +223,7 @@ func TestRepack(t *testing.T) {
}
func testRepack(t *testing.T, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
+ repo := repository.TestRepositoryWithVersion(t, version)
seed := time.Now().UnixNano()
rand.Seed(seed)
@@ -292,11 +292,21 @@ func TestRepackCopy(t *testing.T) {
repository.TestAllVersions(t, testRepackCopy)
}
+type oneConnectionRepo struct {
+ restic.Repository
+}
+
+func (r oneConnectionRepo) Connections() uint {
+ return 1
+}
+
func testRepackCopy(t *testing.T, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
- dstRepo, dstCleanup := repository.TestRepositoryWithVersion(t, version)
- defer dstCleanup()
+ repo := repository.TestRepositoryWithVersion(t, version)
+ dstRepo := repository.TestRepositoryWithVersion(t, version)
+
+ // test with minimal possible connection count
+ repoWrapped := &oneConnectionRepo{repo}
+ dstRepoWrapped := &oneConnectionRepo{dstRepo}
seed := time.Now().UnixNano()
rand.Seed(seed)
@@ -308,7 +318,7 @@ func testRepackCopy(t *testing.T, version uint) {
_, keepBlobs := selectBlobs(t, repo, 0.2)
copyPacks := findPacksForBlobs(t, repo, keepBlobs)
- _, err := repository.Repack(context.TODO(), repo, dstRepo, copyPacks, keepBlobs, nil)
+ _, err := repository.Repack(context.TODO(), repoWrapped, dstRepoWrapped, copyPacks, keepBlobs, nil)
if err != nil {
t.Fatal(err)
}
@@ -336,8 +346,7 @@ func TestRepackWrongBlob(t *testing.T) {
}
func testRepackWrongBlob(t *testing.T, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
+ repo := repository.TestRepositoryWithVersion(t, version)
seed := time.Now().UnixNano()
rand.Seed(seed)
@@ -362,8 +371,7 @@ func TestRepackBlobFallback(t *testing.T) {
}
func testRepackBlobFallback(t *testing.T, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
+ repo := repository.TestRepositoryWithVersion(t, version)
seed := time.Now().UnixNano()
rand.Seed(seed)
diff --git a/internal/repository/repository.go b/internal/repository/repository.go
index 625ad9b16..df8a6fb68 100644
--- a/internal/repository/repository.go
+++ b/internal/repository/repository.go
@@ -19,6 +19,7 @@ import (
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/pack"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui/progress"
@@ -34,12 +35,12 @@ const MaxPackSize = 128 * 1024 * 1024
// Repository is used to access a repository in a backend.
type Repository struct {
- be restic.Backend
- cfg restic.Config
- key *crypto.Key
- keyName string
- idx *MasterIndex
- Cache *cache.Cache
+ be restic.Backend
+ cfg restic.Config
+ key *crypto.Key
+ keyID restic.ID
+ idx *index.MasterIndex
+ Cache *cache.Cache
opts Options
@@ -66,9 +67,10 @@ type CompressionMode uint
// Constants for the different compression levels.
const (
- CompressionAuto CompressionMode = 0
- CompressionOff CompressionMode = 1
- CompressionMax CompressionMode = 2
+ CompressionAuto CompressionMode = 0
+ CompressionOff CompressionMode = 1
+ CompressionMax CompressionMode = 2
+ CompressionInvalid CompressionMode = 3
)
// Set implements the method needed for pflag command flag parsing.
@@ -81,6 +83,7 @@ func (c *CompressionMode) Set(s string) error {
case "max":
*c = CompressionMax
default:
+ *c = CompressionInvalid
return fmt.Errorf("invalid compression mode %q, must be one of (auto|off|max)", s)
}
@@ -106,6 +109,10 @@ func (c *CompressionMode) Type() string {
// New returns a new repository with backend be.
func New(be restic.Backend, opts Options) (*Repository, error) {
+ if opts.Compression == CompressionInvalid {
+ return nil, errors.Fatalf("invalid compression mode")
+ }
+
if opts.PackSize == 0 {
opts.PackSize = DefaultPackSize
}
@@ -118,7 +125,7 @@ func New(be restic.Backend, opts Options) (*Repository, error) {
repo := &Repository{
be: be,
opts: opts,
- idx: NewMasterIndex(),
+ idx: index.NewMasterIndex(),
}
return repo, nil
@@ -134,7 +141,7 @@ func (r *Repository) DisableAutoIndexUpdate() {
func (r *Repository) setConfig(cfg restic.Config) {
r.cfg = cfg
if r.cfg.Version >= 2 {
- r.idx.markCompressed()
+ r.idx.MarkCompressed()
}
}
@@ -163,12 +170,6 @@ func (r *Repository) SetDryRun() {
r.be = dryrun.New(r.be)
}
-// PrefixLength returns the number of bytes required so that all prefixes of
-// all IDs of type t are unique.
-func (r *Repository) PrefixLength(ctx context.Context, t restic.FileType) (int, error) {
- return restic.PrefixLength(ctx, r.be, t)
-}
-
// LoadUnpacked loads and decrypts the file with the given type and ID, using
// the supplied buffer (which must be empty). If the buffer is nil, a new
// buffer will be allocated and returned.
@@ -183,7 +184,11 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
id = restic.ID{}
}
+ ctx, cancel := context.WithCancel(ctx)
+
h := restic.Handle{Type: t, Name: id.String()}
+ retriedInvalidData := false
+ var dataErr error
err := r.be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this call is idempotent, in case an error occurs
wr := bytes.NewBuffer(buf[:0])
@@ -192,17 +197,30 @@ func (r *Repository) LoadUnpacked(ctx context.Context, t restic.FileType, id res
return cerr
}
buf = wr.Bytes()
+
+ if t != restic.ConfigFile && !restic.Hash(buf).Equal(id) {
+ debug.Log("retry loading broken blob %v", h)
+ if !retriedInvalidData {
+ retriedInvalidData = true
+ } else {
+ // with a canceled context there is not guarantee which error will
+ // be returned by `be.Load`.
+ dataErr = fmt.Errorf("load(%v): %w", h, restic.ErrInvalidData)
+ cancel()
+ }
+ return restic.ErrInvalidData
+
+ }
return nil
})
+ if dataErr != nil {
+ return nil, dataErr
+ }
if err != nil {
return nil, err
}
- if t != restic.ConfigFile && !restic.Hash(buf).Equal(id) {
- return nil, errors.Errorf("load %v: invalid data returned", h)
- }
-
nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():]
plaintext, err := r.key.Open(ciphertext[:0], nonce, ciphertext, nil)
if err != nil {
@@ -268,12 +286,7 @@ func (r *Repository) LoadBlob(ctx context.Context, t restic.BlobType, id restic.
}
// load blob from pack
- bt := t
- if r.idx.IsMixedPack(blob.PackID) {
- bt = restic.InvalidBlob
- }
- h := restic.Handle{Type: restic.PackFile,
- Name: blob.PackID.String(), ContainedBlobType: bt}
+ h := restic.Handle{Type: restic.PackFile, Name: blob.PackID.String(), ContainedBlobType: t}
switch {
case cap(buf) < int(blob.Length):
@@ -565,8 +578,8 @@ func (r *Repository) Index() restic.MasterIndex {
// SetIndex instructs the repository to use the given index.
func (r *Repository) SetIndex(i restic.MasterIndex) error {
- r.idx = i.(*MasterIndex)
- return r.PrepareCache()
+ r.idx = i.(*index.MasterIndex)
+ return r.prepareCache()
}
// LoadIndex loads all index files from the backend in parallel and stores them
@@ -574,7 +587,7 @@ func (r *Repository) SetIndex(i restic.MasterIndex) error {
func (r *Repository) LoadIndex(ctx context.Context) error {
debug.Log("Loading index")
- err := ForAllIndexes(ctx, r, func(id restic.ID, idx *Index, oldFormat bool, err error) error {
+ err := index.ForAllIndexes(ctx, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
if err != nil {
return err
}
@@ -595,15 +608,20 @@ func (r *Repository) LoadIndex(ctx context.Context) error {
// sanity check
ctx, cancel := context.WithCancel(ctx)
defer cancel()
- for blob := range r.idx.Each(ctx) {
+
+ invalidIndex := false
+ r.idx.Each(ctx, func(blob restic.PackedBlob) {
if blob.IsCompressed() {
- return errors.Fatal("index uses feature not supported by repository version 1")
+ invalidIndex = true
}
+ })
+ if invalidIndex {
+ return errors.Fatal("index uses feature not supported by repository version 1")
}
}
// remove index files from the cache which have been removed in the repo
- return r.PrepareCache()
+ return r.prepareCache()
}
// CreateIndexFromPacks creates a new index by reading all given pack files (with sizes).
@@ -669,9 +687,9 @@ func (r *Repository) CreateIndexFromPacks(ctx context.Context, packsize map[rest
return invalid, nil
}
-// PrepareCache initializes the local cache. indexIDs is the list of IDs of
+// prepareCache initializes the local cache. indexIDs is the list of IDs of
// index files still present in the repo.
-func (r *Repository) PrepareCache() error {
+func (r *Repository) prepareCache() error {
if r.Cache == nil {
return nil
}
@@ -705,10 +723,10 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int
}
r.key = key.master
- r.keyName = key.Name()
+ r.keyID = key.ID()
cfg, err := restic.LoadConfig(ctx, r)
if err == crypto.ErrUnauthenticated {
- return errors.Fatalf("config or key %v is damaged: %v", key.Name(), err)
+ return errors.Fatalf("config or key %v is damaged: %v", key.ID(), err)
} else if err != nil {
return errors.Fatalf("config cannot be loaded: %v", err)
}
@@ -728,11 +746,11 @@ func (r *Repository) Init(ctx context.Context, version uint, password string, ch
return fmt.Errorf("repository version %v too low", version)
}
- has, err := r.be.Test(ctx, restic.Handle{Type: restic.ConfigFile})
- if err != nil {
+ _, err := r.be.Stat(ctx, restic.Handle{Type: restic.ConfigFile})
+ if err != nil && !r.be.IsNotExist(err) {
return err
}
- if has {
+ if err == nil {
return errors.New("repository master key and config already initialized")
}
@@ -756,7 +774,7 @@ func (r *Repository) init(ctx context.Context, password string, cfg restic.Confi
}
r.key = key.master
- r.keyName = key.Name()
+ r.keyID = key.ID()
r.setConfig(cfg)
return restic.SaveConfig(ctx, r, cfg)
}
@@ -766,9 +784,9 @@ func (r *Repository) Key() *crypto.Key {
return r.key
}
-// KeyName returns the name of the current key in the backend.
-func (r *Repository) KeyName() string {
- return r.keyName
+// KeyID returns the id of the current key in the backend.
+func (r *Repository) KeyID() restic.ID {
+ return r.keyID
}
// List runs fn for all files of type t in the repo.
@@ -813,13 +831,20 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte
// compute plaintext hash if not already set
if id.IsNull() {
- newID = restic.Hash(buf)
+ // Special case the hash calculation for all zero chunks. This is especially
+ // useful for sparse files containing large all zero regions. For these we can
+ // process chunks as fast as we can read the from disk.
+ if len(buf) == chunker.MinSize && restic.ZeroPrefixLen(buf) == chunker.MinSize {
+ newID = ZeroChunk()
+ } else {
+ newID = restic.Hash(buf)
+ }
} else {
newID = id
}
// first try to add to pending blobs; if not successful, this blob is already known
- known = !r.idx.addPending(restic.BlobHandle{ID: newID, Type: t})
+ known = !r.idx.AddPending(restic.BlobHandle{ID: newID, Type: t})
// only save when needed or explicitly told
if !known || storeDuplicate {
@@ -967,3 +992,14 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key,
})
return errors.Wrap(err, "StreamPack")
}
+
+var zeroChunkOnce sync.Once
+var zeroChunkID restic.ID
+
+// ZeroChunk computes and returns (cached) the ID of an all-zero chunk with size chunker.MinSize
+func ZeroChunk() restic.ID {
+ zeroChunkOnce.Do(func() {
+ zeroChunkID = restic.Hash(make([]byte, chunker.MinSize))
+ })
+ return zeroChunkID
+}
diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go
index b5b0ff92d..6c04f1f95 100644
--- a/internal/repository/repository_test.go
+++ b/internal/repository/repository_test.go
@@ -16,7 +16,9 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/klauspost/compress/zstd"
+ "github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/crypto"
+ "github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
@@ -33,8 +35,7 @@ func TestSave(t *testing.T) {
}
func testSave(t *testing.T, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
+ repo := repository.TestRepositoryWithVersion(t, version)
for _, size := range testSizes {
data := make([]byte, size)
@@ -75,8 +76,7 @@ func TestSaveFrom(t *testing.T) {
}
func testSaveFrom(t *testing.T, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
+ repo := repository.TestRepositoryWithVersion(t, version)
for _, size := range testSizes {
data := make([]byte, size)
@@ -115,9 +115,7 @@ func BenchmarkSaveAndEncrypt(t *testing.B) {
}
func benchmarkSaveAndEncrypt(t *testing.B, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
-
+ repo := repository.TestRepositoryWithVersion(t, version)
size := 4 << 20 // 4MiB
data := make([]byte, size)
@@ -125,6 +123,8 @@ func benchmarkSaveAndEncrypt(t *testing.B, version uint) {
rtest.OK(t, err)
id := restic.ID(sha256.Sum256(data))
+ var wg errgroup.Group
+ repo.StartPackUploader(context.Background(), &wg)
t.ReportAllocs()
t.ResetTimer()
@@ -141,9 +141,7 @@ func TestLoadBlob(t *testing.T) {
}
func testLoadBlob(t *testing.T, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
-
+ repo := repository.TestRepositoryWithVersion(t, version)
length := 1000000
buf := crypto.NewBlobBuffer(length)
_, err := io.ReadFull(rnd, buf)
@@ -177,9 +175,7 @@ func BenchmarkLoadBlob(b *testing.B) {
}
func benchmarkLoadBlob(b *testing.B, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(b, version)
- defer cleanup()
-
+ repo := repository.TestRepositoryWithVersion(b, version)
length := 1000000
buf := crypto.NewBlobBuffer(length)
_, err := io.ReadFull(rnd, buf)
@@ -220,9 +216,7 @@ func BenchmarkLoadUnpacked(b *testing.B) {
}
func benchmarkLoadUnpacked(b *testing.B, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(b, version)
- defer cleanup()
-
+ repo := repository.TestRepositoryWithVersion(b, version)
length := 1000000
buf := crypto.NewBlobBuffer(length)
_, err := io.ReadFull(rnd, buf)
@@ -266,19 +260,74 @@ func TestRepositoryLoadIndex(t *testing.T) {
}
// loadIndex loads the index id from backend and returns it.
-func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*repository.Index, error) {
+func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*index.Index, error) {
buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id, nil)
if err != nil {
return nil, err
}
- idx, oldFormat, err := repository.DecodeIndex(buf, id)
+ idx, oldFormat, err := index.DecodeIndex(buf, id)
if oldFormat {
fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str())
}
return idx, err
}
+func TestRepositoryLoadUnpackedBroken(t *testing.T) {
+ repodir, cleanup := rtest.Env(t, repoFixture)
+ defer cleanup()
+
+ data := rtest.Random(23, 12345)
+ id := restic.Hash(data)
+ h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
+ // damage buffer
+ data[0] ^= 0xff
+
+ repo := repository.TestOpenLocal(t, repodir)
+ // store broken file
+ err := repo.Backend().Save(context.TODO(), h, restic.NewByteReader(data, nil))
+ rtest.OK(t, err)
+
+ // without a retry backend this will just return an error that the file is broken
+ _, err = repo.LoadUnpacked(context.TODO(), restic.IndexFile, id, nil)
+ if err == nil {
+ t.Fatal("missing expected error")
+ }
+ rtest.Assert(t, strings.Contains(err.Error(), "invalid data returned"), "unexpected error: %v", err)
+}
+
+type damageOnceBackend struct {
+ restic.Backend
+}
+
+func (be *damageOnceBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+ // don't break the config file as we can't retry it
+ if h.Type == restic.ConfigFile {
+ return be.Backend.Load(ctx, h, length, offset, fn)
+ }
+ // return broken data on the first try
+ err := be.Backend.Load(ctx, h, length+1, offset, fn)
+ if err != nil {
+ // retry
+ err = be.Backend.Load(ctx, h, length, offset, fn)
+ }
+ return err
+}
+
+func TestRepositoryLoadUnpackedRetryBroken(t *testing.T) {
+ repodir, cleanup := rtest.Env(t, repoFixture)
+ defer cleanup()
+
+ be, err := local.Open(context.TODO(), local.Config{Path: repodir, Connections: 2})
+ rtest.OK(t, err)
+ repo, err := repository.New(&damageOnceBackend{Backend: be}, repository.Options{})
+ rtest.OK(t, err)
+ err = repo.SearchKey(context.TODO(), test.TestPassword, 10, "")
+ rtest.OK(t, err)
+
+ rtest.OK(t, repo.LoadIndex(context.TODO()))
+}
+
func BenchmarkLoadIndex(b *testing.B) {
repository.BenchmarkAllVersions(b, benchmarkLoadIndex)
}
@@ -286,10 +335,8 @@ func BenchmarkLoadIndex(b *testing.B) {
func benchmarkLoadIndex(b *testing.B, version uint) {
repository.TestUseLowSecurityKDFParameters(b)
- repo, cleanup := repository.TestRepositoryWithVersion(b, version)
- defer cleanup()
-
- idx := repository.NewIndex()
+ repo := repository.TestRepositoryWithVersion(b, version)
+ idx := index.NewIndex()
for i := 0; i < 5000; i++ {
idx.StorePack(restic.NewRandomID(), []restic.Blob{
@@ -301,7 +348,7 @@ func benchmarkLoadIndex(b *testing.B, version uint) {
})
}
- id, err := repository.SaveIndex(context.TODO(), repo, idx)
+ id, err := index.SaveIndex(context.TODO(), repo, idx)
rtest.OK(b, err)
b.Logf("index saved as %v", id.Str())
@@ -339,12 +386,9 @@ func TestRepositoryIncrementalIndex(t *testing.T) {
}
func testRepositoryIncrementalIndex(t *testing.T, version uint) {
- r, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
+ repo := repository.TestRepositoryWithVersion(t, version).(*repository.Repository)
- repo := r.(*repository.Repository)
-
- repository.IndexFull = func(*repository.Index, bool) bool { return true }
+ index.IndexFull = func(*index.Index, bool) bool { return true }
// add a few rounds of packs
for j := 0; j < 5; j++ {
@@ -362,13 +406,13 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) {
idx, err := loadIndex(context.TODO(), repo, id)
rtest.OK(t, err)
- for pb := range idx.Each(context.TODO()) {
+ idx.Each(context.TODO(), func(pb restic.PackedBlob) {
if _, ok := packEntries[pb.PackID]; !ok {
packEntries[pb.PackID] = make(map[restic.ID]struct{})
}
packEntries[pb.PackID][id] = struct{}{}
- }
+ })
return nil
})
if err != nil {
@@ -622,3 +666,11 @@ func testStreamPack(t *testing.T, version uint) {
}
})
}
+
+func TestInvalidCompression(t *testing.T) {
+ var comp repository.CompressionMode
+ err := comp.Set("nope")
+ rtest.Assert(t, err != nil, "missing error")
+ _, err = repository.New(nil, repository.Options{Compression: comp})
+ rtest.Assert(t, err != nil, "missing error")
+}
diff --git a/internal/repository/testing.go b/internal/repository/testing.go
index 380a47d04..879650336 100644
--- a/internal/repository/testing.go
+++ b/internal/repository/testing.go
@@ -8,6 +8,7 @@ import (
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/backend/mem"
+ "github.com/restic/restic/internal/backend/retry"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
@@ -33,8 +34,8 @@ func TestUseLowSecurityKDFParameters(t logger) {
}
// TestBackend returns a fully configured in-memory backend.
-func TestBackend(t testing.TB) (be restic.Backend, cleanup func()) {
- return mem.New(), func() {}
+func TestBackend(t testing.TB) restic.Backend {
+ return mem.New()
}
const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173)
@@ -42,14 +43,13 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173)
// TestRepositoryWithBackend returns a repository initialized with a test
// password. If be is nil, an in-memory backend is used. A constant polynomial
// is used for the chunker and low-security test parameters.
-func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) (r restic.Repository, cleanup func()) {
+func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) restic.Repository {
t.Helper()
TestUseLowSecurityKDFParameters(t)
restic.TestDisableCheckPolynomial(t)
- var beCleanup func()
if be == nil {
- be, beCleanup = TestBackend(t)
+ be = TestBackend(t)
}
repo, err := New(be, Options{})
@@ -63,23 +63,19 @@ func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) (r
t.Fatalf("TestRepository(): initialize repo failed: %v", err)
}
- return repo, func() {
- if beCleanup != nil {
- beCleanup()
- }
- }
+ return repo
}
// TestRepository returns a repository initialized with a test password on an
// in-memory backend. When the environment variable RESTIC_TEST_REPO is set to
// a non-existing directory, a local backend is created there and this is used
// instead. The directory is not removed, but left there for inspection.
-func TestRepository(t testing.TB) (r restic.Repository, cleanup func()) {
+func TestRepository(t testing.TB) restic.Repository {
t.Helper()
return TestRepositoryWithVersion(t, 0)
}
-func TestRepositoryWithVersion(t testing.TB, version uint) (r restic.Repository, cleanup func()) {
+func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository {
t.Helper()
dir := os.Getenv("RESTIC_TEST_REPO")
if dir != "" {
@@ -102,11 +98,14 @@ func TestRepositoryWithVersion(t testing.TB, version uint) (r restic.Repository,
// TestOpenLocal opens a local repository.
func TestOpenLocal(t testing.TB, dir string) (r restic.Repository) {
+ var be restic.Backend
be, err := local.Open(context.TODO(), local.Config{Path: dir, Connections: 2})
if err != nil {
t.Fatal(err)
}
+ be = retry.New(be, 3, nil, nil)
+
repo, err := New(be, Options{})
if err != nil {
t.Fatal(err)
@@ -138,13 +137,3 @@ func BenchmarkAllVersions(b *testing.B, bench VersionedBenchmark) {
})
}
}
-
-func TestMergeIndex(t testing.TB, mi *MasterIndex) ([]*Index, int) {
- finalIndexes := mi.finalizeNotFinalIndexes()
- for _, idx := range finalIndexes {
- test.OK(t, idx.SetID(restic.NewRandomID()))
- }
-
- test.OK(t, mi.MergeFinalIndexes())
- return finalIndexes, len(mi.idx)
-}
diff --git a/internal/restic/backend.go b/internal/restic/backend.go
index 6ec10e685..bc139fc8b 100644
--- a/internal/restic/backend.go
+++ b/internal/restic/backend.go
@@ -27,9 +27,6 @@ type Backend interface {
// HasAtomicReplace returns whether Save() can atomically replace files
HasAtomicReplace() bool
- // Test a boolean value whether a File with the name and type exists.
- Test(ctx context.Context, h Handle) (bool, error)
-
// Remove removes a File described by h.
Remove(ctx context.Context, h Handle) error
@@ -64,6 +61,9 @@ type Backend interface {
// IsNotExist returns true if the error was caused by a non-existing file
// in the backend.
+ //
+ // The argument may be a wrapped error. The implementation is responsible
+ // for unwrapping it.
IsNotExist(err error) bool
// Delete removes all data in the backend.
diff --git a/internal/restic/backend_find.go b/internal/restic/backend_find.go
index b85cc9199..7c78b3355 100644
--- a/internal/restic/backend_find.go
+++ b/internal/restic/backend_find.go
@@ -3,6 +3,8 @@ package restic
import (
"context"
"fmt"
+
+ "github.com/restic/restic/internal/debug"
)
// A MultipleIDMatchesError is returned by Find() when multiple IDs with a
@@ -24,16 +26,23 @@ func (e *NoIDByPrefixError) Error() string {
// Find loads the list of all files of type t and searches for names which
// start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned.
// If more than one is found, nil and ErrMultipleIDMatches is returned.
-func Find(ctx context.Context, be Lister, t FileType, prefix string) (string, error) {
- match := ""
+func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) {
+ match := ID{}
ctx, cancel := context.WithCancel(ctx)
defer cancel()
err := be.List(ctx, t, func(fi FileInfo) error {
+ // ignore filename which are not an id
+ id, err := ParseID(fi.Name)
+ if err != nil {
+ debug.Log("unable to parse %v as an ID", fi.Name)
+ return nil
+ }
+
if len(fi.Name) >= len(prefix) && prefix == fi.Name[:len(prefix)] {
- if match == "" {
- match = fi.Name
+ if match.IsNull() {
+ match = id
} else {
return &MultipleIDMatchesError{prefix}
}
@@ -43,51 +52,12 @@ func Find(ctx context.Context, be Lister, t FileType, prefix string) (string, er
})
if err != nil {
- return "", err
+ return ID{}, err
}
- if match != "" {
+ if !match.IsNull() {
return match, nil
}
- return "", &NoIDByPrefixError{prefix}
-}
-
-const minPrefixLength = 8
-
-// PrefixLength returns the number of bytes required so that all prefixes of
-// all names of type t are unique.
-func PrefixLength(ctx context.Context, be Lister, t FileType) (int, error) {
- // load all IDs of the given type
- list := make([]string, 0, 100)
-
- ctx, cancel := context.WithCancel(ctx)
- defer cancel()
-
- err := be.List(ctx, t, func(fi FileInfo) error {
- list = append(list, fi.Name)
- return nil
- })
-
- if err != nil {
- return 0, err
- }
-
- // select prefixes of length l, test if the last one is the same as the current one
- var id ID
-outer:
- for l := minPrefixLength; l < len(id); l++ {
- var last string
-
- for _, name := range list {
- if last == name[:l] {
- continue outer
- }
- last = name[:l]
- }
-
- return l, nil
- }
-
- return len(id), nil
+ return ID{}, &NoIDByPrefixError{prefix}
}
diff --git a/internal/restic/backend_find_test.go b/internal/restic/backend_find_test.go
index 52eb38ef3..cbd5e7f48 100644
--- a/internal/restic/backend_find_test.go
+++ b/internal/restic/backend_find_test.go
@@ -43,7 +43,7 @@ func TestFind(t *testing.T) {
if err != nil {
t.Error(err)
}
- expectedMatch := "20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"
+ expectedMatch := TestParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff")
if f != expectedMatch {
t.Errorf("Wrong match returned want %s, got %s", expectedMatch, f)
}
@@ -52,7 +52,7 @@ func TestFind(t *testing.T) {
if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), "NotAPrefix") {
t.Error("Expected no snapshots to be found.")
}
- if f != "" {
+ if !f.IsNull() {
t.Errorf("Find should not return a match on error.")
}
@@ -62,7 +62,7 @@ func TestFind(t *testing.T) {
if _, ok := err.(*NoIDByPrefixError); !ok || !strings.Contains(err.Error(), extraLengthID) {
t.Errorf("Wrong error %v for no snapshots matched", err)
}
- if f != "" {
+ if !f.IsNull() {
t.Errorf("Find should not return a match on error.")
}
@@ -71,48 +71,7 @@ func TestFind(t *testing.T) {
if _, ok := err.(*MultipleIDMatchesError); !ok {
t.Errorf("Wrong error %v for multiple snapshots", err)
}
- if f != "" {
+ if !f.IsNull() {
t.Errorf("Find should not return a match on error.")
}
}
-
-func TestPrefixLength(t *testing.T) {
- list := samples
-
- m := mockBackend{}
- m.list = func(ctx context.Context, t FileType, fn func(FileInfo) error) error {
- for _, id := range list {
- err := fn(FileInfo{Name: id.String()})
- if err != nil {
- return err
- }
- }
- return nil
- }
-
- l, err := PrefixLength(context.TODO(), m, SnapshotFile)
- if err != nil {
- t.Error(err)
- }
- if l != 19 {
- t.Errorf("wrong prefix length returned, want %d, got %d", 19, l)
- }
-
- list = samples[:3]
- l, err = PrefixLength(context.TODO(), m, SnapshotFile)
- if err != nil {
- t.Error(err)
- }
- if l != 19 {
- t.Errorf("wrong prefix length returned, want %d, got %d", 19, l)
- }
-
- list = samples[3:]
- l, err = PrefixLength(context.TODO(), m, SnapshotFile)
- if err != nil {
- t.Error(err)
- }
- if l != 8 {
- t.Errorf("wrong prefix length returned, want %d, got %d", 8, l)
- }
-}
diff --git a/internal/restic/blob.go b/internal/restic/blob.go
index 4ac149adb..260f40fde 100644
--- a/internal/restic/blob.go
+++ b/internal/restic/blob.go
@@ -114,11 +114,7 @@ func (h BlobHandles) Less(i, j int) bool {
continue
}
- if b < h[j].ID[k] {
- return true
- }
-
- return false
+ return b < h[j].ID[k]
}
return h[i].Type < h[j].Type
@@ -133,5 +129,5 @@ func (h BlobHandles) String() string {
for _, e := range h {
elements = append(elements, e.String())
}
- return fmt.Sprintf("%v", elements)
+ return fmt.Sprint(elements)
}
diff --git a/internal/restic/blob_set.go b/internal/restic/blob_set.go
index 07e88fed0..acacd57d4 100644
--- a/internal/restic/blob_set.go
+++ b/internal/restic/blob_set.go
@@ -31,6 +31,10 @@ func (s BlobSet) Delete(h BlobHandle) {
delete(s, h)
}
+func (s BlobSet) Len() int {
+ return len(s)
+}
+
// Equals returns true iff s equals other.
func (s BlobSet) Equals(other BlobSet) bool {
if len(s) != len(other) {
diff --git a/internal/restic/counted_blob_set.go b/internal/restic/counted_blob_set.go
new file mode 100644
index 000000000..f965d3129
--- /dev/null
+++ b/internal/restic/counted_blob_set.go
@@ -0,0 +1,68 @@
+package restic
+
+import "sort"
+
+// CountedBlobSet is a set of blobs. For each blob it also stores a uint8 value
+// which can be used to track some information. The CountedBlobSet does not use
+// that value in any way. New entries are created with value 0.
+type CountedBlobSet map[BlobHandle]uint8
+
+// NewCountedBlobSet returns a new CountedBlobSet, populated with ids.
+func NewCountedBlobSet(handles ...BlobHandle) CountedBlobSet {
+ m := make(CountedBlobSet)
+ for _, h := range handles {
+ m[h] = 0
+ }
+
+ return m
+}
+
+// Has returns true iff id is contained in the set.
+func (s CountedBlobSet) Has(h BlobHandle) bool {
+ _, ok := s[h]
+ return ok
+}
+
+// Insert adds id to the set.
+func (s CountedBlobSet) Insert(h BlobHandle) {
+ s[h] = 0
+}
+
+// Delete removes id from the set.
+func (s CountedBlobSet) Delete(h BlobHandle) {
+ delete(s, h)
+}
+
+func (s CountedBlobSet) Len() int {
+ return len(s)
+}
+
+// List returns a sorted slice of all BlobHandle in the set.
+func (s CountedBlobSet) List() BlobHandles {
+ list := make(BlobHandles, 0, len(s))
+ for h := range s {
+ list = append(list, h)
+ }
+
+ sort.Sort(list)
+
+ return list
+}
+
+func (s CountedBlobSet) String() string {
+ str := s.List().String()
+ if len(str) < 2 {
+ return "{}"
+ }
+
+ return "{" + str[1:len(str)-1] + "}"
+}
+
+// Copy returns a copy of the CountedBlobSet.
+func (s CountedBlobSet) Copy() CountedBlobSet {
+ cp := make(CountedBlobSet, len(s))
+ for k, v := range s {
+ cp[k] = v
+ }
+ return cp
+}
diff --git a/internal/restic/counted_blob_set_test.go b/internal/restic/counted_blob_set_test.go
new file mode 100644
index 000000000..681751e91
--- /dev/null
+++ b/internal/restic/counted_blob_set_test.go
@@ -0,0 +1,45 @@
+package restic_test
+
+import (
+ "testing"
+
+ "github.com/restic/restic/internal/restic"
+ "github.com/restic/restic/internal/test"
+)
+
+func TestCountedBlobSet(t *testing.T) {
+ bs := restic.NewCountedBlobSet()
+ test.Equals(t, bs.Len(), 0)
+ test.Equals(t, bs.List(), restic.BlobHandles{})
+
+ bh := restic.NewRandomBlobHandle()
+ // check non existant
+ test.Equals(t, bs.Has(bh), false)
+
+ // test insert
+ bs.Insert(bh)
+ test.Equals(t, bs.Has(bh), true)
+ test.Equals(t, bs.Len(), 1)
+ test.Equals(t, bs.List(), restic.BlobHandles{bh})
+
+ // test remove
+ bs.Delete(bh)
+ test.Equals(t, bs.Len(), 0)
+ test.Equals(t, bs.Has(bh), false)
+ test.Equals(t, bs.List(), restic.BlobHandles{})
+
+ bs = restic.NewCountedBlobSet(bh)
+ test.Equals(t, bs.Len(), 1)
+ test.Equals(t, bs.List(), restic.BlobHandles{bh})
+
+ s := bs.String()
+ test.Assert(t, len(s) > 10, "invalid string: %v", s)
+}
+
+func TestCountedBlobSetCopy(t *testing.T) {
+ bs := restic.NewCountedBlobSet(restic.NewRandomBlobHandle(), restic.NewRandomBlobHandle(), restic.NewRandomBlobHandle())
+ test.Equals(t, bs.Len(), 3)
+ cp := bs.Copy()
+ test.Equals(t, cp.Len(), 3)
+ test.Equals(t, bs.List(), cp.List())
+}
diff --git a/internal/restic/file.go b/internal/restic/file.go
index d058a71c0..0e9f046ae 100644
--- a/internal/restic/file.go
+++ b/internal/restic/file.go
@@ -7,18 +7,38 @@ import (
)
// FileType is the type of a file in the backend.
-type FileType string
+type FileType uint8
// These are the different data types a backend can store.
const (
- PackFile FileType = "data" // use data, as packs are stored under /data in repo
- KeyFile FileType = "key"
- LockFile FileType = "lock"
- SnapshotFile FileType = "snapshot"
- IndexFile FileType = "index"
- ConfigFile FileType = "config"
+ PackFile FileType = 1 + iota
+ KeyFile
+ LockFile
+ SnapshotFile
+ IndexFile
+ ConfigFile
)
+func (t FileType) String() string {
+ s := "invalid"
+ switch t {
+ case PackFile:
+ // Spelled "data" instead of "pack" for historical reasons.
+ s = "data"
+ case KeyFile:
+ s = "key"
+ case LockFile:
+ s = "lock"
+ case SnapshotFile:
+ s = "snapshot"
+ case IndexFile:
+ s = "index"
+ case ConfigFile:
+ s = "config"
+ }
+ return s
+}
+
// Handle is used to store and access data in a backend.
type Handle struct {
Type FileType
@@ -36,10 +56,6 @@ func (h Handle) String() string {
// Valid returns an error if h is not valid.
func (h Handle) Valid() error {
- if h.Type == "" {
- return errors.New("type is empty")
- }
-
switch h.Type {
case PackFile:
case KeyFile:
@@ -48,7 +64,7 @@ func (h Handle) Valid() error {
case IndexFile:
case ConfigFile:
default:
- return errors.Errorf("invalid Type %q", h.Type)
+ return errors.Errorf("invalid Type %d", h.Type)
}
if h.Type == ConfigFile {
diff --git a/internal/restic/file_test.go b/internal/restic/file_test.go
index 76f00baac..cc54c2924 100644
--- a/internal/restic/file_test.go
+++ b/internal/restic/file_test.go
@@ -1,20 +1,28 @@
package restic
-import "testing"
+import (
+ "testing"
-var handleTests = []struct {
- h Handle
- valid bool
-}{
- {Handle{Name: "foo"}, false},
- {Handle{Type: "foobar"}, false},
- {Handle{Type: ConfigFile, Name: ""}, true},
- {Handle{Type: PackFile, Name: ""}, false},
- {Handle{Type: "", Name: "x"}, false},
- {Handle{Type: LockFile, Name: "010203040506"}, true},
+ rtest "github.com/restic/restic/internal/test"
+)
+
+func TestHandleString(t *testing.T) {
+ rtest.Equals(t, "<data/foobar>", Handle{Type: PackFile, Name: "foobar"}.String())
+ rtest.Equals(t, "<lock/1>", Handle{Type: LockFile, Name: "1"}.String())
}
func TestHandleValid(t *testing.T) {
+ var handleTests = []struct {
+ h Handle
+ valid bool
+ }{
+ {Handle{Name: "foo"}, false},
+ {Handle{Type: 0}, false},
+ {Handle{Type: ConfigFile, Name: ""}, true},
+ {Handle{Type: PackFile, Name: ""}, false},
+ {Handle{Type: LockFile, Name: "010203040506"}, true},
+ }
+
for i, test := range handleTests {
err := test.h.Valid()
if err != nil && test.valid {
diff --git a/internal/restic/find.go b/internal/restic/find.go
index 6544f2b3d..08670a49f 100644
--- a/internal/restic/find.go
+++ b/internal/restic/find.go
@@ -15,9 +15,14 @@ type Loader interface {
Connections() uint
}
+type findBlobSet interface {
+ Has(bh BlobHandle) bool
+ Insert(bh BlobHandle)
+}
+
// FindUsedBlobs traverses the tree ID and adds all seen blobs (trees and data
// blobs) to the set blobs. Already seen tree blobs will not be visited again.
-func FindUsedBlobs(ctx context.Context, repo Loader, treeIDs IDs, blobs BlobSet, p *progress.Counter) error {
+func FindUsedBlobs(ctx context.Context, repo Loader, treeIDs IDs, blobs findBlobSet, p *progress.Counter) error {
var lock sync.Mutex
wg, ctx := errgroup.WithContext(ctx)
diff --git a/internal/restic/find_test.go b/internal/restic/find_test.go
index b415501dc..f5e288b9d 100644
--- a/internal/restic/find_test.go
+++ b/internal/restic/find_test.go
@@ -84,8 +84,7 @@ const (
var findTestTime = time.Unix(1469960361, 23)
func TestFindUsedBlobs(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
var snapshots []*restic.Snapshot
for i := 0; i < findTestSnapshots; i++ {
@@ -94,7 +93,7 @@ func TestFindUsedBlobs(t *testing.T) {
snapshots = append(snapshots, sn)
}
- p := progress.New(time.Second, findTestSnapshots, func(value uint64, total uint64, runtime time.Duration, final bool) {})
+ p := progress.NewCounter(time.Second, findTestSnapshots, func(value uint64, total uint64, runtime time.Duration, final bool) {})
defer p.Done()
for i, sn := range snapshots {
@@ -110,7 +109,8 @@ func TestFindUsedBlobs(t *testing.T) {
continue
}
- test.Equals(t, p.Get(), uint64(i+1))
+ v, _ := p.Get()
+ test.Equals(t, v, uint64(i+1))
goldenFilename := filepath.Join("testdata", fmt.Sprintf("used_blobs_snapshot%d", i))
want := loadIDSet(t, goldenFilename)
@@ -127,8 +127,7 @@ func TestFindUsedBlobs(t *testing.T) {
}
func TestMultiFindUsedBlobs(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
var snapshotTrees restic.IDs
for i := 0; i < findTestSnapshots; i++ {
@@ -143,7 +142,7 @@ func TestMultiFindUsedBlobs(t *testing.T) {
want.Merge(loadIDSet(t, goldenFilename))
}
- p := progress.New(time.Second, findTestSnapshots, func(value uint64, total uint64, runtime time.Duration, final bool) {})
+ p := progress.NewCounter(time.Second, findTestSnapshots, func(value uint64, total uint64, runtime time.Duration, final bool) {})
defer p.Done()
// run twice to check progress bar handling of duplicate tree roots
@@ -151,7 +150,8 @@ func TestMultiFindUsedBlobs(t *testing.T) {
for i := 1; i < 3; i++ {
err := restic.FindUsedBlobs(context.TODO(), repo, snapshotTrees, usedBlobs, p)
test.OK(t, err)
- test.Equals(t, p.Get(), uint64(i*len(snapshotTrees)))
+ v, _ := p.Get()
+ test.Equals(t, v, uint64(i*len(snapshotTrees)))
if !want.Equals(usedBlobs) {
t.Errorf("wrong list of blobs returned:\n missing blobs: %v\n extra blobs: %v",
@@ -175,8 +175,7 @@ func (r ForbiddenRepo) Connections() uint {
}
func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
snapshot := restic.TestCreateSnapshot(t, repo, findTestTime, findTestDepth, 0)
t.Logf("snapshot %v saved, tree %v", snapshot.ID().Str(), snapshot.Tree.Str())
@@ -194,8 +193,7 @@ func TestFindUsedBlobsSkipsSeenBlobs(t *testing.T) {
}
func BenchmarkFindUsedBlobs(b *testing.B) {
- repo, cleanup := repository.TestRepository(b)
- defer cleanup()
+ repo := repository.TestRepository(b)
sn := restic.TestCreateSnapshot(b, repo, findTestTime, findTestDepth, 0)
diff --git a/internal/restic/id.go b/internal/restic/id.go
index 6d85ed68a..e71c6d71b 100644
--- a/internal/restic/id.go
+++ b/internal/restic/id.go
@@ -6,8 +6,6 @@ import (
"fmt"
"io"
- "github.com/restic/restic/internal/errors"
-
"github.com/minio/sha256-simd"
)
@@ -24,14 +22,13 @@ type ID [idSize]byte
// ParseID converts the given string to an ID.
func ParseID(s string) (ID, error) {
- b, err := hex.DecodeString(s)
-
- if err != nil {
- return ID{}, errors.Wrap(err, "hex.DecodeString")
+ if len(s) != hex.EncodedLen(idSize) {
+ return ID{}, fmt.Errorf("invalid length for ID: %q", s)
}
- if len(b) != idSize {
- return ID{}, errors.New("invalid length for hash")
+ b, err := hex.DecodeString(s)
+ if err != nil {
+ return ID{}, fmt.Errorf("invalid ID: %s", err)
}
id := ID{}
@@ -82,19 +79,6 @@ func (id ID) Equal(other ID) bool {
return id == other
}
-// EqualString compares this ID to another one, given as a string.
-func (id ID) EqualString(other string) (bool, error) {
- s, err := hex.DecodeString(other)
- if err != nil {
- return false, errors.Wrap(err, "hex.DecodeString")
- }
-
- id2 := ID{}
- copy(id2[:], s)
-
- return id == id2, nil
-}
-
// MarshalJSON returns the JSON encoding of id.
func (id ID) MarshalJSON() ([]byte, error) {
buf := make([]byte, 2+hex.EncodedLen(len(id)))
@@ -109,34 +93,21 @@ func (id ID) MarshalJSON() ([]byte, error) {
// UnmarshalJSON parses the JSON-encoded data and stores the result in id.
func (id *ID) UnmarshalJSON(b []byte) error {
// check string length
- if len(b) < 2 {
- return fmt.Errorf("invalid ID: %q", b)
- }
-
- if len(b)%2 != 0 {
- return fmt.Errorf("invalid ID length: %q", b)
+ if len(b) != len(`""`)+hex.EncodedLen(idSize) {
+ return fmt.Errorf("invalid length for ID: %q", b)
}
- // check string delimiters
- if b[0] != '"' && b[0] != '\'' {
+ if b[0] != '"' {
return fmt.Errorf("invalid start of string: %q", b[0])
}
- last := len(b) - 1
- if b[0] != b[last] {
- return fmt.Errorf("starting string delimiter (%q) does not match end (%q)", b[0], b[last])
- }
-
- // strip JSON string delimiters
- b = b[1:last]
-
- if len(b) != 2*len(id) {
- return fmt.Errorf("invalid length for ID")
- }
+ // Strip JSON string delimiters. The json.Unmarshaler contract says we get
+ // a valid JSON value, so we don't need to check that b[len(b)-1] == '"'.
+ b = b[1 : len(b)-1]
_, err := hex.Decode(id[:], b)
if err != nil {
- return errors.Wrap(err, "hex.Decode")
+ return fmt.Errorf("invalid ID: %s", err)
}
return nil
diff --git a/internal/restic/id_test.go b/internal/restic/id_test.go
index ff1dc54e0..9a9fddcda 100644
--- a/internal/restic/id_test.go
+++ b/internal/restic/id_test.go
@@ -30,14 +30,6 @@ func TestID(t *testing.T) {
t.Errorf("ID.Equal() does not work as expected")
}
- ret, err := id.EqualString(test.id)
- if err != nil {
- t.Error(err)
- }
- if !ret {
- t.Error("ID.EqualString() returned wrong value")
- }
-
// test json marshalling
buf, err := id.MarshalJSON()
if err != nil {
diff --git a/internal/restic/ids.go b/internal/restic/ids.go
index cc5ad18da..de91dbb4c 100644
--- a/internal/restic/ids.go
+++ b/internal/restic/ids.go
@@ -2,10 +2,10 @@ package restic
import (
"encoding/hex"
- "fmt"
+ "strings"
)
-// IDs is an ordered list of IDs that implements sort.Interface.
+// IDs is a slice of IDs that implements sort.Interface and fmt.Stringer.
type IDs []ID
func (ids IDs) Len() int {
@@ -13,57 +13,28 @@ func (ids IDs) Len() int {
}
func (ids IDs) Less(i, j int) bool {
- if len(ids[i]) < len(ids[j]) {
- return true
- }
-
- for k, b := range ids[i] {
- if b == ids[j][k] {
- continue
- }
-
- if b < ids[j][k] {
- return true
- }
-
- return false
- }
-
- return false
+ return string(ids[i][:]) < string(ids[j][:])
}
func (ids IDs) Swap(i, j int) {
ids[i], ids[j] = ids[j], ids[i]
}
-// Uniq returns list without duplicate IDs. The returned list retains the order
-// of the original list so that the order of the first occurrence of each ID
-// stays the same.
-func (ids IDs) Uniq() (list IDs) {
- seen := NewIDSet()
+func (ids IDs) String() string {
+ var sb strings.Builder
+ sb.Grow(1 + (1+2*shortStr)*len(ids))
+
+ buf := make([]byte, 2*shortStr)
- for _, id := range ids {
- if seen.Has(id) {
- continue
+ sb.WriteByte('[')
+ for i, id := range ids {
+ if i > 0 {
+ sb.WriteByte(' ')
}
-
- list = append(list, id)
- seen.Insert(id)
+ hex.Encode(buf, id[:shortStr])
+ sb.Write(buf)
}
+ sb.WriteByte(']')
- return list
-}
-
-type shortID ID
-
-func (id shortID) String() string {
- return hex.EncodeToString(id[:shortStr])
-}
-
-func (ids IDs) String() string {
- elements := make([]shortID, 0, len(ids))
- for _, id := range ids {
- elements = append(elements, shortID(id))
- }
- return fmt.Sprintf("%v", elements)
+ return sb.String()
}
diff --git a/internal/restic/ids_test.go b/internal/restic/ids_test.go
index 9ce02607b..8f1e9d5a9 100644
--- a/internal/restic/ids_test.go
+++ b/internal/restic/ids_test.go
@@ -1,55 +1,17 @@
package restic
import (
- "reflect"
"testing"
-)
-var uniqTests = []struct {
- before, after IDs
-}{
- {
- IDs{
- TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
- TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
- TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
- },
- IDs{
- TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
- TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
- },
- },
- {
- IDs{
- TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
- TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
- TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
- },
- IDs{
- TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
- TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
- },
- },
- {
- IDs{
- TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
- TestParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"),
- TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
- TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
- },
- IDs{
- TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
- TestParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"),
- TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
- },
- },
-}
+ rtest "github.com/restic/restic/internal/test"
+)
-func TestUniqIDs(t *testing.T) {
- for i, test := range uniqTests {
- uniq := test.before.Uniq()
- if !reflect.DeepEqual(uniq, test.after) {
- t.Errorf("uniqIDs() test %v failed\n wanted: %v\n got: %v", i, test.after, uniq)
- }
+func TestIDsString(t *testing.T) {
+ ids := IDs{
+ TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
+ TestParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"),
+ TestParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"),
}
+
+ rtest.Equals(t, "[7bb086db 1285b303 7bb086db]", ids.String())
}
diff --git a/internal/restic/idset.go b/internal/restic/idset.go
index c31ca7747..1b12a6398 100644
--- a/internal/restic/idset.go
+++ b/internal/restic/idset.go
@@ -31,7 +31,7 @@ func (s IDSet) Delete(id ID) {
delete(s, id)
}
-// List returns a slice of all IDs in the set.
+// List returns a sorted slice of all IDs in the set.
func (s IDSet) List() IDs {
list := make(IDs, 0, len(s))
for id := range s {
@@ -103,9 +103,5 @@ func (s IDSet) Sub(other IDSet) (result IDSet) {
func (s IDSet) String() string {
str := s.List().String()
- if len(str) < 2 {
- return "{}"
- }
-
return "{" + str[1:len(str)-1] + "}"
}
diff --git a/internal/restic/idset_test.go b/internal/restic/idset_test.go
index 5525eab79..734b31237 100644
--- a/internal/restic/idset_test.go
+++ b/internal/restic/idset_test.go
@@ -2,6 +2,8 @@ package restic
import (
"testing"
+
+ rtest "github.com/restic/restic/internal/test"
)
var idsetTests = []struct {
@@ -22,6 +24,8 @@ var idsetTests = []struct {
func TestIDSet(t *testing.T) {
set := NewIDSet()
+ rtest.Equals(t, "{}", set.String())
+
for i, test := range idsetTests {
seen := set.Has(test.id)
if seen != test.seen {
@@ -29,4 +33,6 @@ func TestIDSet(t *testing.T) {
}
set.Insert(test.id)
}
+
+ rtest.Equals(t, "{1285b303 7bb086db f658198b}", set.String())
}
diff --git a/internal/restic/lock.go b/internal/restic/lock.go
index 3f233483f..d500c019a 100644
--- a/internal/restic/lock.go
+++ b/internal/restic/lock.go
@@ -7,12 +7,12 @@ import (
"os/signal"
"os/user"
"sync"
+ "sync/atomic"
"syscall"
"testing"
"time"
"github.com/restic/restic/internal/errors"
- "golang.org/x/sync/errgroup"
"github.com/restic/restic/internal/debug"
)
@@ -26,6 +26,7 @@ import (
// A lock must be refreshed regularly to not be considered stale, this must be
// triggered by regularly calling Refresh.
type Lock struct {
+ lock sync.Mutex
Time time.Time `json:"time"`
Exclusive bool `json:"exclusive"`
Hostname string `json:"hostname"`
@@ -59,6 +60,27 @@ func IsAlreadyLocked(err error) bool {
return errors.As(err, &e)
}
+// invalidLockError is returned when NewLock or NewExclusiveLock fail due
+// to an invalid lock.
+type invalidLockError struct {
+ err error
+}
+
+func (e *invalidLockError) Error() string {
+ return fmt.Sprintf("invalid lock file: %v", e.err)
+}
+
+func (e *invalidLockError) Unwrap() error {
+ return e.err
+}
+
+// IsInvalidLock returns true iff err indicates that locking failed due to
+// an invalid lock.
+func IsInvalidLock(err error) bool {
+ var e *invalidLockError
+ return errors.As(err, &e)
+}
+
// NewLock returns a new, non-exclusive lock for the repository. If an
// exclusive lock is already held by another process, it returns an error
// that satisfies IsAlreadyLocked.
@@ -126,7 +148,7 @@ func (l *Lock) fillUserInfo() error {
}
l.Username = usr.Username
- l.UID, l.GID, err = uidGidInt(*usr)
+ l.UID, l.GID, err = uidGidInt(usr)
return err
}
@@ -137,23 +159,40 @@ func (l *Lock) fillUserInfo() error {
// non-exclusive lock is to be created, an error is only returned when an
// exclusive lock is found.
func (l *Lock) checkForOtherLocks(ctx context.Context) error {
- return ForAllLocks(ctx, l.repo, l.lockID, func(id ID, lock *Lock, err error) error {
- if err != nil {
- // ignore locks that cannot be loaded
- debug.Log("ignore lock %v: %v", id, err)
- return nil
- }
+ var err error
+ // retry locking a few times
+ for i := 0; i < 3; i++ {
+ err = ForAllLocks(ctx, l.repo, l.lockID, func(id ID, lock *Lock, err error) error {
+ if err != nil {
+ // if we cannot load a lock then it is unclear whether it can be ignored
+ // it could either be invalid or just unreadable due to network/permission problems
+ debug.Log("ignore lock %v: %v", id, err)
+ return err
+ }
- if l.Exclusive {
- return &alreadyLockedError{otherLock: lock}
- }
+ if l.Exclusive {
+ return &alreadyLockedError{otherLock: lock}
+ }
- if !l.Exclusive && lock.Exclusive {
- return &alreadyLockedError{otherLock: lock}
- }
+ if !l.Exclusive && lock.Exclusive {
+ return &alreadyLockedError{otherLock: lock}
+ }
- return nil
- })
+ return nil
+ })
+ // no lock detected
+ if err == nil {
+ return nil
+ }
+ // lock conflicts are permanent
+ if _, ok := err.(*alreadyLockedError); ok {
+ return err
+ }
+ }
+ if errors.Is(err, ErrInvalidData) {
+ return &invalidLockError{err}
+ }
+ return err
}
// createLock acquires the lock by creating a file in the repository.
@@ -175,14 +214,16 @@ func (l *Lock) Unlock() error {
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: l.lockID.String()})
}
-var staleTimeout = 30 * time.Minute
+var StaleLockTimeout = 30 * time.Minute
// Stale returns true if the lock is stale. A lock is stale if the timestamp is
// older than 30 minutes or if it was created on the current machine and the
// process isn't alive any more.
func (l *Lock) Stale() bool {
- debug.Log("testing if lock %v for process %d is stale", l, l.PID)
- if time.Since(l.Time) > staleTimeout {
+ l.lock.Lock()
+ defer l.lock.Unlock()
+ debug.Log("testing if lock %v for process %d is stale", l.lockID, l.PID)
+ if time.Since(l.Time) > StaleLockTimeout {
debug.Log("lock is stale, timestamp is too old: %v\n", l.Time)
return true
}
@@ -215,12 +256,17 @@ func (l *Lock) Stale() bool {
// timestamp. Afterwards the old lock is removed.
func (l *Lock) Refresh(ctx context.Context) error {
debug.Log("refreshing lock %v", l.lockID)
+ l.lock.Lock()
l.Time = time.Now()
+ l.lock.Unlock()
id, err := l.createLock(ctx)
if err != nil {
return err
}
+ l.lock.Lock()
+ defer l.lock.Unlock()
+
debug.Log("new lock ID %v", id)
oldLockID := l.lockID
l.lockID = &id
@@ -228,7 +274,10 @@ func (l *Lock) Refresh(ctx context.Context) error {
return l.repo.Backend().Remove(context.TODO(), Handle{Type: LockFile, Name: oldLockID.String()})
}
-func (l Lock) String() string {
+func (l *Lock) String() string {
+ l.lock.Lock()
+ defer l.lock.Unlock()
+
text := fmt.Sprintf("PID %d on %s by %s (UID %d, GID %d)\nlock was created at %s (%s ago)\nstorage ID %v",
l.PID, l.Hostname, l.Username, l.UID, l.GID,
l.Time.Format("2006-01-02 15:04:05"), time.Since(l.Time),
@@ -264,8 +313,9 @@ func LoadLock(ctx context.Context, repo Repository, id ID) (*Lock, error) {
}
// RemoveStaleLocks deletes all locks detected as stale from the repository.
-func RemoveStaleLocks(ctx context.Context, repo Repository) error {
- return ForAllLocks(ctx, repo, nil, func(id ID, lock *Lock, err error) error {
+func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) {
+ var processed uint
+ err := ForAllLocks(ctx, repo, nil, func(id ID, lock *Lock, err error) error {
if err != nil {
// ignore locks that cannot be loaded
debug.Log("ignore lock %v: %v", id, err)
@@ -273,18 +323,29 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) error {
}
if lock.Stale() {
- return repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()})
+ err = repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()})
+ if err == nil {
+ processed++
+ }
+ return err
}
return nil
})
+ return processed, err
}
// RemoveAllLocks removes all locks forcefully.
-func RemoveAllLocks(ctx context.Context, repo Repository) error {
- return repo.List(ctx, LockFile, func(id ID, size int64) error {
- return repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()})
+func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) {
+ var processed uint32
+ err := ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
+ err := repo.Backend().Remove(ctx, Handle{Type: LockFile, Name: id.String()})
+ if err == nil {
+ atomic.AddUint32(&processed, 1)
+ }
+ return err
})
+ return uint(processed), err
}
// ForAllLocks reads all locks in parallel and calls the given callback.
@@ -294,50 +355,21 @@ func RemoveAllLocks(ctx context.Context, repo Repository) error {
func ForAllLocks(ctx context.Context, repo Repository, excludeID *ID, fn func(ID, *Lock, error) error) error {
var m sync.Mutex
- // track spawned goroutines using wg, create a new context which is
- // cancelled as soon as an error occurs.
- wg, ctx := errgroup.WithContext(ctx)
-
- ch := make(chan ID)
-
- // send list of lock files through ch, which is closed afterwards
- wg.Go(func() error {
- defer close(ch)
- return repo.List(ctx, LockFile, func(id ID, size int64) error {
- if excludeID != nil && id.Equal(*excludeID) {
- return nil
- }
-
- select {
- case <-ctx.Done():
- return nil
- case ch <- id:
- }
+ // For locks decoding is nearly for free, thus just assume were only limited by IO
+ return ParallelList(ctx, repo.Backend(), LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error {
+ if excludeID != nil && id.Equal(*excludeID) {
return nil
- })
- })
-
- // a worker receives an snapshot ID from ch, loads the snapshot
- // and runs fn with id, the snapshot and the error
- worker := func() error {
- for id := range ch {
- debug.Log("load lock %v", id)
- lock, err := LoadLock(ctx, repo, id)
-
- m.Lock()
- err = fn(id, lock, err)
- m.Unlock()
- if err != nil {
- return err
- }
}
- return nil
- }
-
- // For locks decoding is nearly for free, thus just assume were only limited by IO
- for i := 0; i < int(repo.Connections()); i++ {
- wg.Go(worker)
- }
+ if size == 0 {
+ // Ignore empty lock files as some backends do not guarantee atomic uploads.
+ // These may leave empty files behind if an upload was interrupted between
+ // creating the file and writing its data.
+ return nil
+ }
+ lock, err := LoadLock(ctx, repo, id)
- return wg.Wait()
+ m.Lock()
+ defer m.Unlock()
+ return fn(id, lock, err)
+ })
}
diff --git a/internal/restic/lock_test.go b/internal/restic/lock_test.go
index b92eace70..2d14499bd 100644
--- a/internal/restic/lock_test.go
+++ b/internal/restic/lock_test.go
@@ -2,18 +2,20 @@ package restic_test
import (
"context"
+ "fmt"
+ "io"
"os"
"testing"
"time"
+ "github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func TestLock(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
lock, err := restic.NewLock(context.TODO(), repo)
rtest.OK(t, err)
@@ -22,8 +24,7 @@ func TestLock(t *testing.T) {
}
func TestDoubleUnlock(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
lock, err := restic.NewLock(context.TODO(), repo)
rtest.OK(t, err)
@@ -36,8 +37,7 @@ func TestDoubleUnlock(t *testing.T) {
}
func TestMultipleLock(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
lock1, err := restic.NewLock(context.TODO(), repo)
rtest.OK(t, err)
@@ -49,9 +49,32 @@ func TestMultipleLock(t *testing.T) {
rtest.OK(t, lock2.Unlock())
}
+type failLockLoadingBackend struct {
+ restic.Backend
+}
+
+func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
+ if h.Type == restic.LockFile {
+ return fmt.Errorf("error loading lock")
+ }
+ return be.Backend.Load(ctx, h, length, offset, fn)
+}
+
+func TestMultipleLockFailure(t *testing.T) {
+ be := &failLockLoadingBackend{Backend: mem.New()}
+ repo := repository.TestRepositoryWithBackend(t, be, 0)
+
+ lock1, err := restic.NewLock(context.TODO(), repo)
+ rtest.OK(t, err)
+
+ _, err = restic.NewLock(context.TODO(), repo)
+ rtest.Assert(t, err != nil, "unreadable lock file did not result in an error")
+
+ rtest.OK(t, lock1.Unlock())
+}
+
func TestLockExclusive(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
elock, err := restic.NewExclusiveLock(context.TODO(), repo)
rtest.OK(t, err)
@@ -59,8 +82,7 @@ func TestLockExclusive(t *testing.T) {
}
func TestLockOnExclusiveLockedRepo(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
elock, err := restic.NewExclusiveLock(context.TODO(), repo)
rtest.OK(t, err)
@@ -76,8 +98,7 @@ func TestLockOnExclusiveLockedRepo(t *testing.T) {
}
func TestExclusiveLockOnLockedRepo(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
elock, err := restic.NewLock(context.TODO(), repo)
rtest.OK(t, err)
@@ -165,15 +186,15 @@ func TestLockStale(t *testing.T) {
func lockExists(repo restic.Repository, t testing.TB, id restic.ID) bool {
h := restic.Handle{Type: restic.LockFile, Name: id.String()}
- exists, err := repo.Backend().Test(context.TODO(), h)
- rtest.OK(t, err)
-
- return exists
+ _, err := repo.Backend().Stat(context.TODO(), h)
+ if err != nil && !repo.Backend().IsNotExist(err) {
+ t.Fatal(err)
+ }
+ return err == nil
}
func TestLockWithStaleLock(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid())
rtest.OK(t, err)
@@ -184,7 +205,8 @@ func TestLockWithStaleLock(t *testing.T) {
id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500000)
rtest.OK(t, err)
- rtest.OK(t, restic.RemoveStaleLocks(context.TODO(), repo))
+ processed, err := restic.RemoveStaleLocks(context.TODO(), repo)
+ rtest.OK(t, err)
rtest.Assert(t, lockExists(repo, t, id1) == false,
"stale lock still exists after RemoveStaleLocks was called")
@@ -192,13 +214,15 @@ func TestLockWithStaleLock(t *testing.T) {
"non-stale lock was removed by RemoveStaleLocks")
rtest.Assert(t, lockExists(repo, t, id3) == false,
"stale lock still exists after RemoveStaleLocks was called")
+ rtest.Assert(t, processed == 2,
+ "number of locks removed does not match: expected %d, got %d",
+ 2, processed)
rtest.OK(t, removeLock(repo, id2))
}
func TestRemoveAllLocks(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid())
rtest.OK(t, err)
@@ -209,7 +233,8 @@ func TestRemoveAllLocks(t *testing.T) {
id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500000)
rtest.OK(t, err)
- rtest.OK(t, restic.RemoveAllLocks(context.TODO(), repo))
+ processed, err := restic.RemoveAllLocks(context.TODO(), repo)
+ rtest.OK(t, err)
rtest.Assert(t, lockExists(repo, t, id1) == false,
"lock still exists after RemoveAllLocks was called")
@@ -217,11 +242,13 @@ func TestRemoveAllLocks(t *testing.T) {
"lock still exists after RemoveAllLocks was called")
rtest.Assert(t, lockExists(repo, t, id3) == false,
"lock still exists after RemoveAllLocks was called")
+ rtest.Assert(t, processed == 3,
+ "number of locks removed does not match: expected %d, got %d",
+ 3, processed)
}
func TestLockRefresh(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
lock, err := restic.NewLock(context.TODO(), repo)
rtest.OK(t, err)
diff --git a/internal/restic/lock_unix.go b/internal/restic/lock_unix.go
index dbf23fc6c..3f426ae32 100644
--- a/internal/restic/lock_unix.go
+++ b/internal/restic/lock_unix.go
@@ -9,31 +9,27 @@ import (
"strconv"
"syscall"
- "github.com/restic/restic/internal/errors"
-
"github.com/restic/restic/internal/debug"
+ "github.com/restic/restic/internal/errors"
)
// uidGidInt returns uid, gid of the user as a number.
-func uidGidInt(u user.User) (uid, gid uint32, err error) {
- var ui, gi int64
- ui, err = strconv.ParseInt(u.Uid, 10, 32)
+func uidGidInt(u *user.User) (uid, gid uint32, err error) {
+ ui, err := strconv.ParseUint(u.Uid, 10, 32)
if err != nil {
- return uid, gid, errors.Wrap(err, "ParseInt")
+ return 0, 0, errors.Errorf("invalid UID %q", u.Uid)
}
- gi, err = strconv.ParseInt(u.Gid, 10, 32)
+ gi, err := strconv.ParseUint(u.Gid, 10, 32)
if err != nil {
- return uid, gid, errors.Wrap(err, "ParseInt")
+ return 0, 0, errors.Errorf("invalid GID %q", u.Gid)
}
- uid = uint32(ui)
- gid = uint32(gi)
- return
+ return uint32(ui), uint32(gi), nil
}
// checkProcess will check if the process retaining the lock
// exists and responds to SIGHUP signal.
// Returns true if the process exists and responds.
-func (l Lock) processExists() bool {
+func (l *Lock) processExists() bool {
proc, err := os.FindProcess(l.PID)
if err != nil {
debug.Log("error searching for process %d: %v\n", l.PID, err)
diff --git a/internal/restic/lock_windows.go b/internal/restic/lock_windows.go
index 5697b6efb..ee24e3bca 100644
--- a/internal/restic/lock_windows.go
+++ b/internal/restic/lock_windows.go
@@ -8,7 +8,7 @@ import (
)
// uidGidInt always returns 0 on Windows, since uid isn't numbers
-func uidGidInt(u user.User) (uid, gid uint32, err error) {
+func uidGidInt(u *user.User) (uid, gid uint32, err error) {
return 0, 0, nil
}
diff --git a/internal/restic/node.go b/internal/restic/node.go
index 54b67c672..a1aff18ac 100644
--- a/internal/restic/node.go
+++ b/internal/restic/node.go
@@ -14,7 +14,6 @@ import (
"github.com/restic/restic/internal/errors"
"bytes"
- "runtime"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/fs"
@@ -192,14 +191,14 @@ func (node Node) restoreMetadata(path string) error {
debug.Log("not running as root, ignoring lchown permission error for %v: %v",
path, err)
} else {
- firsterr = errors.Wrap(err, "Lchown")
+ firsterr = errors.WithStack(err)
}
}
if node.Type != "symlink" {
if err := fs.Chmod(path, node.Mode); err != nil {
if firsterr != nil {
- firsterr = errors.Wrap(err, "Chmod")
+ firsterr = errors.WithStack(err)
}
}
}
@@ -251,7 +250,7 @@ func (node Node) RestoreTimestamps(path string) error {
func (node Node) createDirAt(path string) error {
err := fs.Mkdir(path, node.Mode)
if err != nil && !os.IsExist(err) {
- return errors.Wrap(err, "Mkdir")
+ return errors.WithStack(err)
}
return nil
@@ -260,7 +259,7 @@ func (node Node) createDirAt(path string) error {
func (node Node) createFileAt(ctx context.Context, path string, repo Repository) error {
f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
- return errors.Wrap(err, "OpenFile")
+ return errors.WithStack(err)
}
err = node.writeNodeContent(ctx, repo, f)
@@ -271,7 +270,7 @@ func (node Node) createFileAt(ctx context.Context, path string, repo Repository)
}
if closeErr != nil {
- return errors.Wrap(closeErr, "Close")
+ return errors.WithStack(closeErr)
}
return nil
@@ -287,7 +286,7 @@ func (node Node) writeNodeContent(ctx context.Context, repo Repository, f *os.Fi
_, err = f.Write(buf)
if err != nil {
- return errors.Wrap(err, "Write")
+ return errors.WithStack(err)
}
}
@@ -295,15 +294,15 @@ func (node Node) writeNodeContent(ctx context.Context, repo Repository, f *os.Fi
}
func (node Node) createSymlinkAt(path string) error {
- // Windows does not allow non-admins to create soft links.
- if runtime.GOOS == "windows" {
- return nil
- }
- err := fs.Symlink(node.LinkTarget, path)
- if err != nil {
+
+ if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return errors.Wrap(err, "Symlink")
}
+ if err := fs.Symlink(node.LinkTarget, path); err != nil {
+ return errors.WithStack(err)
+ }
+
return nil
}
@@ -592,7 +591,7 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error {
node.LinkTarget, err = fs.Readlink(path)
node.Links = uint64(stat.nlink())
if err != nil {
- return errors.Wrap(err, "Readlink")
+ return errors.WithStack(err)
}
case "dev":
node.Device = uint64(stat.rdev())
diff --git a/internal/restic/node_linux.go b/internal/restic/node_linux.go
index 2eb80db90..85a363830 100644
--- a/internal/restic/node_linux.go
+++ b/internal/restic/node_linux.go
@@ -13,7 +13,7 @@ import (
func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error {
dir, err := fs.Open(filepath.Dir(path))
if err != nil {
- return errors.Wrap(err, "Open")
+ return errors.WithStack(err)
}
times := []unix.Timespec{
diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go
index 8139ee57b..60342e9a4 100644
--- a/internal/restic/node_test.go
+++ b/internal/restic/node_test.go
@@ -2,7 +2,6 @@ package restic_test
import (
"context"
- "io/ioutil"
"os"
"path/filepath"
"runtime"
@@ -14,7 +13,7 @@ import (
)
func BenchmarkNodeFillUser(t *testing.B) {
- tempfile, err := ioutil.TempFile("", "restic-test-temp-")
+ tempfile, err := os.CreateTemp("", "restic-test-temp-")
if err != nil {
t.Fatal(err)
}
@@ -38,7 +37,7 @@ func BenchmarkNodeFillUser(t *testing.B) {
}
func BenchmarkNodeFromFileInfo(t *testing.B) {
- tempfile, err := ioutil.TempFile("", "restic-test-temp-")
+ tempfile, err := os.CreateTemp("", "restic-test-temp-")
if err != nil {
t.Fatal(err)
}
@@ -167,7 +166,7 @@ var nodeTests = []restic.Node{
}
func TestNodeRestoreAt(t *testing.T) {
- tempdir, err := ioutil.TempDir(rtest.TestTempDir, "restic-test-")
+ tempdir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-")
rtest.OK(t, err)
defer func() {
@@ -183,9 +182,6 @@ func TestNodeRestoreAt(t *testing.T) {
rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil))
rtest.OK(t, test.RestoreMetadata(nodePath))
- if test.Type == "symlink" && runtime.GOOS == "windows" {
- continue
- }
if test.Type == "dir" {
rtest.OK(t, test.RestoreTimestamps(nodePath))
}
diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go
index 04a4fe62b..fc6439b40 100644
--- a/internal/restic/node_windows.go
+++ b/internal/restic/node_windows.go
@@ -17,7 +17,21 @@ func lchown(path string, uid int, gid int) (err error) {
}
func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error {
- return nil
+ // tweaked version of UtimesNano from go/src/syscall/syscall_windows.go
+ pathp, e := syscall.UTF16PtrFromString(path)
+ if e != nil {
+ return e
+ }
+ h, e := syscall.CreateFile(pathp,
+ syscall.FILE_WRITE_ATTRIBUTES, syscall.FILE_SHARE_WRITE, nil, syscall.OPEN_EXISTING,
+ syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OPEN_REPARSE_POINT, 0)
+ if e != nil {
+ return e
+ }
+ defer syscall.Close(h)
+ a := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[0]))
+ w := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[1]))
+ return syscall.SetFileTime(h, nil, &a, &w)
}
// Getxattr retrieves extended attribute data associated with path.
diff --git a/internal/restic/parallel.go b/internal/restic/parallel.go
new file mode 100644
index 000000000..df160f018
--- /dev/null
+++ b/internal/restic/parallel.go
@@ -0,0 +1,59 @@
+package restic
+
+import (
+ "context"
+
+ "github.com/restic/restic/internal/debug"
+ "golang.org/x/sync/errgroup"
+)
+
+func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, fn func(context.Context, ID, int64) error) error {
+
+ type FileIDInfo struct {
+ ID
+ Size int64
+ }
+
+ // track spawned goroutines using wg, create a new context which is
+ // cancelled as soon as an error occurs.
+ wg, ctx := errgroup.WithContext(ctx)
+
+ ch := make(chan FileIDInfo)
+ // send list of index files through ch, which is closed afterwards
+ wg.Go(func() error {
+ defer close(ch)
+ return r.List(ctx, t, func(fi FileInfo) error {
+ id, err := ParseID(fi.Name)
+ if err != nil {
+ debug.Log("unable to parse %v as an ID", fi.Name)
+ return nil
+ }
+
+ select {
+ case <-ctx.Done():
+ return nil
+ case ch <- FileIDInfo{id, fi.Size}:
+ }
+ return nil
+ })
+ })
+
+ // a worker receives an index ID from ch, loads the index, and sends it to indexCh
+ worker := func() error {
+ for fi := range ch {
+ debug.Log("worker got file %v", fi.ID.Str())
+ err := fn(ctx, fi.ID, fi.Size)
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+ }
+
+ // run workers on ch
+ for i := uint(0); i < parallelism; i++ {
+ wg.Go(worker)
+ }
+
+ return wg.Wait()
+}
diff --git a/internal/restic/repository.go b/internal/restic/repository.go
index 36f5a73bf..e01d204e6 100644
--- a/internal/restic/repository.go
+++ b/internal/restic/repository.go
@@ -4,10 +4,14 @@ import (
"context"
"github.com/restic/restic/internal/crypto"
+ "github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/ui/progress"
"golang.org/x/sync/errgroup"
)
+// ErrInvalidData is used to report that a file is corrupted
+var ErrInvalidData = errors.New("invalid data returned")
+
// Repository stores data in a backend. It provides high-level functions and
// transparently encrypts/decrypts data.
type Repository interface {
@@ -83,10 +87,9 @@ type MasterIndex interface {
Has(BlobHandle) bool
Lookup(BlobHandle) []PackedBlob
- // Each returns a channel that yields all blobs known to the index. When
- // the context is cancelled, the background goroutine terminates. This
- // blocks any modification of the index.
- Each(ctx context.Context) <-chan PackedBlob
+ // Each runs fn on all blobs known to the index. When the context is cancelled,
+ // the index iteration return immediately. This blocks any modification of the index.
+ Each(ctx context.Context, fn func(PackedBlob))
ListPacks(ctx context.Context, packs IDSet) <-chan PackBlobs
Save(ctx context.Context, repo SaverUnpacked, packBlacklist IDSet, extraObsolete IDs, p *progress.Counter) (obsolete IDSet, err error)
diff --git a/internal/restic/rewind_reader_test.go b/internal/restic/rewind_reader_test.go
index b553cdfb0..8ec79ddcd 100644
--- a/internal/restic/rewind_reader_test.go
+++ b/internal/restic/rewind_reader_test.go
@@ -5,7 +5,6 @@ import (
"crypto/md5"
"hash"
"io"
- "io/ioutil"
"math/rand"
"os"
"path/filepath"
@@ -28,11 +27,9 @@ func TestByteReader(t *testing.T) {
func TestFileReader(t *testing.T) {
buf := []byte("foobar")
- d, cleanup := test.TempDir(t)
- defer cleanup()
-
+ d := test.TempDir(t)
filename := filepath.Join(d, "file-reader-test")
- err := ioutil.WriteFile(filename, buf, 0600)
+ err := os.WriteFile(filename, buf, 0600)
if err != nil {
t.Fatal(err)
}
diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go
index 10c4f218e..58d863526 100644
--- a/internal/restic/snapshot.go
+++ b/internal/restic/snapshot.go
@@ -8,8 +8,6 @@ import (
"sync"
"time"
- "golang.org/x/sync/errgroup"
-
"github.com/restic/restic/internal/debug"
)
@@ -82,58 +80,17 @@ func SaveSnapshot(ctx context.Context, repo SaverUnpacked, sn *Snapshot) (ID, er
func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excludeIDs IDSet, fn func(ID, *Snapshot, error) error) error {
var m sync.Mutex
- // track spawned goroutines using wg, create a new context which is
- // cancelled as soon as an error occurs.
- wg, ctx := errgroup.WithContext(ctx)
-
- ch := make(chan ID)
-
- // send list of snapshot files through ch, which is closed afterwards
- wg.Go(func() error {
- defer close(ch)
- return be.List(ctx, SnapshotFile, func(fi FileInfo) error {
- id, err := ParseID(fi.Name)
- if err != nil {
- debug.Log("unable to parse %v as an ID", fi.Name)
- return nil
- }
-
- if excludeIDs.Has(id) {
- return nil
- }
-
- select {
- case <-ctx.Done():
- return nil
- case ch <- id:
- }
+ // For most snapshots decoding is nearly for free, thus just assume were only limited by IO
+ return ParallelList(ctx, be, SnapshotFile, loader.Connections(), func(ctx context.Context, id ID, size int64) error {
+ if excludeIDs.Has(id) {
return nil
- })
- })
-
- // a worker receives an snapshot ID from ch, loads the snapshot
- // and runs fn with id, the snapshot and the error
- worker := func() error {
- for id := range ch {
- debug.Log("load snapshot %v", id)
- sn, err := LoadSnapshot(ctx, loader, id)
-
- m.Lock()
- err = fn(id, sn, err)
- m.Unlock()
- if err != nil {
- return err
- }
}
- return nil
- }
- // For most snapshots decoding is nearly for free, thus just assume were only limited by IO
- for i := 0; i < int(loader.Connections()); i++ {
- wg.Go(worker)
- }
-
- return wg.Wait()
+ sn, err := LoadSnapshot(ctx, loader, id)
+ m.Lock()
+ defer m.Unlock()
+ return fn(id, sn, err)
+ })
}
func (sn Snapshot) String() string {
@@ -154,7 +111,7 @@ func (sn *Snapshot) fillUserInfo() error {
sn.Username = usr.Username
// set userid and groupid
- sn.UID, sn.GID, err = uidGidInt(*usr)
+ sn.UID, sn.GID, err = uidGidInt(usr)
return err
}
@@ -237,19 +194,14 @@ func (sn *Snapshot) HasTagList(l []TagList) bool {
return false
}
-func (sn *Snapshot) hasPath(path string) bool {
- for _, snPath := range sn.Paths {
- if path == snPath {
- return true
- }
- }
- return false
-}
-
// HasPaths returns true if the snapshot has all of the paths.
func (sn *Snapshot) HasPaths(paths []string) bool {
+ m := make(map[string]struct{}, len(sn.Paths))
+ for _, snPath := range sn.Paths {
+ m[snPath] = struct{}{}
+ }
for _, path := range paths {
- if !sn.hasPath(path) {
+ if _, ok := m[path]; !ok {
return false
}
}
diff --git a/internal/restic/snapshot_find.go b/internal/restic/snapshot_find.go
index 49f9d62bf..4f8231a7f 100644
--- a/internal/restic/snapshot_find.go
+++ b/internal/restic/snapshot_find.go
@@ -3,7 +3,6 @@ package restic
import (
"context"
"fmt"
- "os"
"path/filepath"
"time"
@@ -13,27 +12,23 @@ import (
// ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found.
var ErrNoSnapshotFound = errors.New("no snapshot found")
-// FindLatestSnapshot finds latest snapshot with optional target/directory, tags, hostname, and timestamp filters.
-func FindLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, targets []string,
- tagLists []TagList, hostnames []string, timeStampLimit *time.Time) (ID, error) {
+// findLatestSnapshot finds latest snapshot with optional target/directory, tags, hostname, and timestamp filters.
+func findLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string,
+ tags []TagList, paths []string, timeStampLimit *time.Time) (*Snapshot, error) {
var err error
- absTargets := make([]string, 0, len(targets))
- for _, target := range targets {
+ absTargets := make([]string, 0, len(paths))
+ for _, target := range paths {
if !filepath.IsAbs(target) {
target, err = filepath.Abs(target)
if err != nil {
- return ID{}, errors.Wrap(err, "Abs")
+ return nil, errors.Wrap(err, "Abs")
}
}
absTargets = append(absTargets, filepath.Clean(target))
}
- var (
- latest time.Time
- latestID ID
- found bool
- )
+ var latest *Snapshot
err = ForAllSnapshots(ctx, be, loader, nil, func(id ID, snapshot *Snapshot, err error) error {
if err != nil {
@@ -44,15 +39,15 @@ func FindLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, t
return nil
}
- if snapshot.Time.Before(latest) {
+ if latest != nil && snapshot.Time.Before(latest.Time) {
return nil
}
- if !snapshot.HasHostname(hostnames) {
+ if !snapshot.HasHostname(hosts) {
return nil
}
- if !snapshot.HasTagList(tagLists) {
+ if !snapshot.HasTagList(tags) {
return nil
}
@@ -60,58 +55,107 @@ func FindLatestSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, t
return nil
}
- latest = snapshot.Time
- latestID = id
- found = true
+ latest = snapshot
return nil
})
if err != nil {
- return ID{}, err
+ return nil, err
}
- if !found {
- return ID{}, ErrNoSnapshotFound
+ if latest == nil {
+ return nil, ErrNoSnapshotFound
}
- return latestID, nil
+ return latest, nil
}
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
// the string as closely as possible.
-func FindSnapshot(ctx context.Context, be Lister, s string) (ID, error) {
-
- // find snapshot id with prefix
- name, err := Find(ctx, be, SnapshotFile, s)
+func FindSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, s string) (*Snapshot, error) {
+ // no need to list snapshots if `s` is already a full id
+ id, err := ParseID(s)
if err != nil {
- return ID{}, err
+ // find snapshot id with prefix
+ id, err = Find(ctx, be, SnapshotFile, s)
+ if err != nil {
+ return nil, err
+ }
}
+ return LoadSnapshot(ctx, loader, id)
+}
- return ParseID(name)
+// FindFilteredSnapshot returns either the latests from a filtered list of all snapshots or a snapshot specified by `snapshotID`.
+func FindFilteredSnapshot(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, tags []TagList, paths []string, timeStampLimit *time.Time, snapshotID string) (*Snapshot, error) {
+ if snapshotID == "latest" {
+ sn, err := findLatestSnapshot(ctx, be, loader, hosts, tags, paths, timeStampLimit)
+ if err == ErrNoSnapshotFound {
+ err = fmt.Errorf("snapshot filter (Paths:%v Tags:%v Hosts:%v): %w", paths, tags, hosts, err)
+ }
+ return sn, err
+ }
+ return FindSnapshot(ctx, be, loader, snapshotID)
}
-// FindFilteredSnapshots yields Snapshots filtered from the list of all
-// snapshots.
-func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, tags []TagList, paths []string) (Snapshots, error) {
- results := make(Snapshots, 0, 20)
+type SnapshotFindCb func(string, *Snapshot, error) error
+
+// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
+func FindFilteredSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, hosts []string, tags []TagList, paths []string, snapshotIDs []string, fn SnapshotFindCb) error {
+ if len(snapshotIDs) != 0 {
+ var err error
+ usedFilter := false
+
+ ids := NewIDSet()
+ // Process all snapshot IDs given as arguments.
+ for _, s := range snapshotIDs {
+ var sn *Snapshot
+ if s == "latest" {
+ if usedFilter {
+ continue
+ }
+
+ usedFilter = true
+
+ sn, err = findLatestSnapshot(ctx, be, loader, hosts, tags, paths, nil)
+ if err == ErrNoSnapshotFound {
+ err = errors.Errorf("no snapshot matched given filter (Paths:%v Tags:%v Hosts:%v)", paths, tags, hosts)
+ }
+ if sn != nil {
+ ids.Insert(*sn.ID())
+ }
+ } else {
+ sn, err = FindSnapshot(ctx, be, loader, s)
+ if err == nil {
+ if ids.Has(*sn.ID()) {
+ continue
+ } else {
+ ids.Insert(*sn.ID())
+ s = sn.ID().String()
+ }
+ }
+ }
+ err = fn(s, sn, err)
+ if err != nil {
+ return err
+ }
+ }
+
+ // Give the user some indication their filters are not used.
+ if !usedFilter && (len(hosts) != 0 || len(tags) != 0 || len(paths) != 0) {
+ return fn("filters", nil, errors.Errorf("explicit snapshot ids are given"))
+ }
+ return nil
+ }
- err := ForAllSnapshots(ctx, be, loader, nil, func(id ID, sn *Snapshot, err error) error {
+ return ForAllSnapshots(ctx, be, loader, nil, func(id ID, sn *Snapshot, err error) error {
if err != nil {
- fmt.Fprintf(os.Stderr, "could not load snapshot %v: %v\n", id.Str(), err)
- return nil
+ return fn(id.String(), sn, err)
}
if !sn.HasHostname(hosts) || !sn.HasTagList(tags) || !sn.HasPaths(paths) {
return nil
}
- results = append(results, sn)
- return nil
+ return fn(id.String(), sn, err)
})
-
- if err != nil {
- return nil, err
- }
-
- return results, nil
}
diff --git a/internal/restic/snapshot_find_test.go b/internal/restic/snapshot_find_test.go
index 534eb456d..3c587dde1 100644
--- a/internal/restic/snapshot_find_test.go
+++ b/internal/restic/snapshot_find_test.go
@@ -9,39 +9,35 @@ import (
)
func TestFindLatestSnapshot(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
-
+ repo := repository.TestRepository(t)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1, 0)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0)
latestSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0)
- id, err := restic.FindLatestSnapshot(context.TODO(), repo.Backend(), repo, []string{}, []restic.TagList{}, []string{"foo"}, nil)
+ sn, err := restic.FindFilteredSnapshot(context.TODO(), repo.Backend(), repo, []string{"foo"}, []restic.TagList{}, []string{}, nil, "latest")
if err != nil {
t.Fatalf("FindLatestSnapshot returned error: %v", err)
}
- if id != *latestSnapshot.ID() {
- t.Errorf("FindLatestSnapshot returned wrong snapshot ID: %v", id)
+ if *sn.ID() != *latestSnapshot.ID() {
+ t.Errorf("FindLatestSnapshot returned wrong snapshot ID: %v", *sn.ID())
}
}
func TestFindLatestSnapshotWithMaxTimestamp(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
-
+ repo := repository.TestRepository(t)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2015-05-05 05:05:05"), 1, 0)
desiredSnapshot := restic.TestCreateSnapshot(t, repo, parseTimeUTC("2017-07-07 07:07:07"), 1, 0)
restic.TestCreateSnapshot(t, repo, parseTimeUTC("2019-09-09 09:09:09"), 1, 0)
maxTimestamp := parseTimeUTC("2018-08-08 08:08:08")
- id, err := restic.FindLatestSnapshot(context.TODO(), repo.Backend(), repo, []string{}, []restic.TagList{}, []string{"foo"}, &maxTimestamp)
+ sn, err := restic.FindFilteredSnapshot(context.TODO(), repo.Backend(), repo, []string{"foo"}, []restic.TagList{}, []string{}, &maxTimestamp, "latest")
if err != nil {
t.Fatalf("FindLatestSnapshot returned error: %v", err)
}
- if id != *desiredSnapshot.ID() {
- t.Errorf("FindLatestSnapshot returned wrong snapshot ID: %v", id)
+ if *sn.ID() != *desiredSnapshot.ID() {
+ t.Errorf("FindLatestSnapshot returned wrong snapshot ID: %v", *sn.ID())
}
}
diff --git a/internal/restic/snapshot_policy_test.go b/internal/restic/snapshot_policy_test.go
index cb35a946d..918ea4ec7 100644
--- a/internal/restic/snapshot_policy_test.go
+++ b/internal/restic/snapshot_policy_test.go
@@ -3,7 +3,7 @@ package restic_test
import (
"encoding/json"
"fmt"
- "io/ioutil"
+ "os"
"path/filepath"
"testing"
"time"
@@ -61,7 +61,7 @@ type ApplyPolicyResult struct {
}
func loadGoldenFile(t testing.TB, filename string) (res ApplyPolicyResult) {
- buf, err := ioutil.ReadFile(filename)
+ buf, err := os.ReadFile(filename)
if err != nil {
t.Fatalf("error loading golden file %v: %v", filename, err)
}
@@ -85,7 +85,7 @@ func saveGoldenFile(t testing.TB, filename string, keep restic.Snapshots, reason
t.Fatalf("error marshaling result: %v", err)
}
- if err = ioutil.WriteFile(filename, buf, 0644); err != nil {
+ if err = os.WriteFile(filename, buf, 0644); err != nil {
t.Fatalf("unable to update golden file: %v", err)
}
}
diff --git a/internal/restic/snapshot_test.go b/internal/restic/snapshot_test.go
index 96325debf..b32c771d4 100644
--- a/internal/restic/snapshot_test.go
+++ b/internal/restic/snapshot_test.go
@@ -32,8 +32,7 @@ func TestLoadJSONUnpacked(t *testing.T) {
}
func testLoadJSONUnpacked(t *testing.T, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
+ repo := repository.TestRepositoryWithVersion(t, version)
// archive a snapshot
sn := restic.Snapshot{}
diff --git a/internal/restic/tag_list.go b/internal/restic/tag_list.go
index 4d57cb14b..b293b14f7 100644
--- a/internal/restic/tag_list.go
+++ b/internal/restic/tag_list.go
@@ -37,7 +37,7 @@ func (TagList) Type() string {
type TagLists []TagList
func (l TagLists) String() string {
- return fmt.Sprintf("%v", []TagList(l))
+ return fmt.Sprint([]TagList(l))
}
// Flatten returns the list of all tags provided in TagLists
diff --git a/internal/restic/testing_test.go b/internal/restic/testing_test.go
index 7ee7461a5..2af5c607e 100644
--- a/internal/restic/testing_test.go
+++ b/internal/restic/testing_test.go
@@ -37,9 +37,7 @@ func loadAllSnapshots(ctx context.Context, repo restic.Repository, excludeIDs re
}
func TestCreateSnapshot(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
-
+ repo := repository.TestRepository(t)
for i := 0; i < testCreateSnapshots; i++ {
restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth, 0)
}
@@ -70,8 +68,7 @@ func TestCreateSnapshot(t *testing.T) {
}
func BenchmarkTestCreateSnapshot(t *testing.B) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
t.ResetTimer()
diff --git a/internal/restic/tree.go b/internal/restic/tree.go
index d1264074c..373b36746 100644
--- a/internal/restic/tree.go
+++ b/internal/restic/tree.go
@@ -145,6 +145,8 @@ func SaveTree(ctx context.Context, r BlobSaver, t *Tree) (ID, error) {
return id, err
}
+var ErrTreeNotOrdered = errors.New("nodes are not ordered or duplicate")
+
type TreeJSONBuilder struct {
buf bytes.Buffer
lastName string
@@ -158,7 +160,7 @@ func NewTreeJSONBuilder() *TreeJSONBuilder {
func (builder *TreeJSONBuilder) AddNode(node *Node) error {
if node.Name <= builder.lastName {
- return errors.Errorf("nodes are not ordered got %q, last %q", node.Name, builder.lastName)
+ return fmt.Errorf("node %q, last%q: %w", node.Name, builder.lastName, ErrTreeNotOrdered)
}
if builder.lastName != "" {
_ = builder.buf.WriteByte(',')
@@ -182,14 +184,3 @@ func (builder *TreeJSONBuilder) Finalize() ([]byte, error) {
builder.buf = bytes.Buffer{}
return buf, nil
}
-
-func TreeToBuilder(t *Tree) (*TreeJSONBuilder, error) {
- builder := NewTreeJSONBuilder()
- for _, node := range t.Nodes {
- err := builder.AddNode(node)
- if err != nil {
- return nil, err
- }
- }
- return builder, nil
-}
diff --git a/internal/restic/tree_test.go b/internal/restic/tree_test.go
index 811f0c6c6..fb25ca373 100644
--- a/internal/restic/tree_test.go
+++ b/internal/restic/tree_test.go
@@ -3,7 +3,7 @@ package restic_test
import (
"context"
"encoding/json"
- "io/ioutil"
+ "errors"
"os"
"path/filepath"
"strconv"
@@ -26,7 +26,7 @@ var testFiles = []struct {
}
func createTempDir(t *testing.T) string {
- tempdir, err := ioutil.TempDir(rtest.TestTempDir, "restic-test-")
+ tempdir, err := os.MkdirTemp(rtest.TestTempDir, "restic-test-")
rtest.OK(t, err)
for _, test := range testFiles {
@@ -97,8 +97,7 @@ func TestNodeComparison(t *testing.T) {
}
func TestEmptyLoadTree(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
var wg errgroup.Group
repo.StartPackUploader(context.TODO(), &wg)
@@ -136,6 +135,7 @@ func TestTreeEqualSerialization(t *testing.T) {
rtest.Assert(t, tree.Insert(node) != nil, "no error on duplicate node")
rtest.Assert(t, builder.AddNode(node) != nil, "no error on duplicate node")
+ rtest.Assert(t, errors.Is(builder.AddNode(node), restic.ErrTreeNotOrdered), "wrong error returned")
}
treeBytes, err := json.Marshal(tree)
@@ -176,14 +176,12 @@ func TestLoadTree(t *testing.T) {
}
func testLoadTree(t *testing.T, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
-
if rtest.BenchArchiveDirectory == "" {
t.Skip("benchdir not set, skipping")
}
// archive a few files
+ repo := repository.TestRepositoryWithVersion(t, version)
sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil)
rtest.OK(t, repo.Flush(context.Background()))
@@ -196,14 +194,12 @@ func BenchmarkLoadTree(t *testing.B) {
}
func benchmarkLoadTree(t *testing.B, version uint) {
- repo, cleanup := repository.TestRepositoryWithVersion(t, version)
- defer cleanup()
-
if rtest.BenchArchiveDirectory == "" {
t.Skip("benchdir not set, skipping")
}
// archive a few files
+ repo := repository.TestRepositoryWithVersion(t, version)
sn := archiver.TestSnapshot(t, repo, rtest.BenchArchiveDirectory, nil)
rtest.OK(t, repo.Flush(context.Background()))
diff --git a/internal/restic/zeroprefix.go b/internal/restic/zeroprefix.go
new file mode 100644
index 000000000..b25e7ab27
--- /dev/null
+++ b/internal/restic/zeroprefix.go
@@ -0,0 +1,21 @@
+package restic
+
+import "bytes"
+
+// ZeroPrefixLen returns the length of the longest all-zero prefix of p.
+func ZeroPrefixLen(p []byte) (n int) {
+ // First skip 1kB-sized blocks, for speed.
+ var zeros [1024]byte
+
+ for len(p) >= len(zeros) && bytes.Equal(p[:len(zeros)], zeros[:]) {
+ p = p[len(zeros):]
+ n += len(zeros)
+ }
+
+ for len(p) > 0 && p[0] == 0 {
+ p = p[1:]
+ n++
+ }
+
+ return n
+}
diff --git a/internal/restic/zeroprefix_test.go b/internal/restic/zeroprefix_test.go
new file mode 100644
index 000000000..a21806851
--- /dev/null
+++ b/internal/restic/zeroprefix_test.go
@@ -0,0 +1,52 @@
+package restic_test
+
+import (
+ "math/rand"
+ "testing"
+
+ "github.com/restic/restic/internal/restic"
+ "github.com/restic/restic/internal/test"
+)
+
+func TestZeroPrefixLen(t *testing.T) {
+ var buf [2048]byte
+
+ // test zero prefixes of various lengths
+ for i := len(buf) - 1; i >= 0; i-- {
+ buf[i] = 42
+ skipped := restic.ZeroPrefixLen(buf[:])
+ test.Equals(t, i, skipped)
+ }
+ // test buffers of various sizes
+ for i := 0; i < len(buf); i++ {
+ skipped := restic.ZeroPrefixLen(buf[i:])
+ test.Equals(t, 0, skipped)
+ }
+}
+
+func BenchmarkZeroPrefixLen(b *testing.B) {
+ var (
+ buf [4<<20 + 37]byte
+ r = rand.New(rand.NewSource(0x618732))
+ sumSkipped int64
+ )
+
+ b.ReportAllocs()
+ b.SetBytes(int64(len(buf)))
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ j := r.Intn(len(buf))
+ buf[j] = 0xff
+
+ skipped := restic.ZeroPrefixLen(buf[:])
+ sumSkipped += int64(skipped)
+
+ buf[j] = 0
+ }
+
+ // The closer this is to .5, the better. If it's far off, give the
+ // benchmark more time to run with -benchtime.
+ b.Logf("average number of zeros skipped: %.3f",
+ float64(sumSkipped)/(float64(b.N*len(buf))))
+}
diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go
index 362d821d2..2deef1cd2 100644
--- a/internal/restorer/filerestorer.go
+++ b/internal/restorer/filerestorer.go
@@ -27,6 +27,7 @@ const (
type fileInfo struct {
lock sync.Mutex
inProgress bool
+ sparse bool
size int64
location string // file on local filesystem relative to restorer basedir
blobs interface{} // blobs of the file
@@ -51,6 +52,8 @@ type fileRestorer struct {
workerCount int
filesWriter *filesWriter
+ zeroChunk restic.ID
+ sparse bool
dst string
files []*fileInfo
@@ -61,7 +64,8 @@ func newFileRestorer(dst string,
packLoader repository.BackendLoadFn,
key *crypto.Key,
idx func(restic.BlobHandle) []restic.PackedBlob,
- connections uint) *fileRestorer {
+ connections uint,
+ sparse bool) *fileRestorer {
// as packs are streamed the concurrency is limited by IO
workerCount := int(connections)
@@ -71,6 +75,8 @@ func newFileRestorer(dst string,
idx: idx,
packLoader: packLoader,
filesWriter: newFilesWriter(workerCount),
+ zeroChunk: repository.ZeroChunk(),
+ sparse: sparse,
workerCount: workerCount,
dst: dst,
Error: restorerAbortOnAllErrors,
@@ -133,7 +139,16 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error {
packOrder = append(packOrder, packID)
}
pack.files[file] = struct{}{}
+ if blob.ID.Equal(r.zeroChunk) {
+ file.sparse = r.sparse
+ }
})
+ if len(fileBlobs) == 1 {
+ // no need to preallocate files with a single block, thus we can always consider them to be sparse
+ // in addition, a short chunk will never match r.zeroChunk which would prevent sparseness for short files
+ file.sparse = r.sparse
+ }
+
if err != nil {
// repository index is messed up, can't do anything
return err
@@ -253,7 +268,7 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error {
file.inProgress = true
createSize = file.size
}
- return r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize)
+ return r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse)
}
err := sanitizeError(file, writeToFile())
if err != nil {
diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go
index fa781f8c8..b39afa249 100644
--- a/internal/restorer/filerestorer_test.go
+++ b/internal/restorer/filerestorer_test.go
@@ -5,7 +5,7 @@ import (
"context"
"fmt"
"io"
- "io/ioutil"
+ "os"
"testing"
"github.com/restic/restic/internal/crypto"
@@ -147,10 +147,10 @@ func newTestRepo(content []TestFile) *TestRepo {
return repo
}
-func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files map[string]bool) {
+func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files map[string]bool, sparse bool) {
repo := newTestRepo(content)
- r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2)
+ r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, sparse)
if files == nil {
r.files = repo.files
@@ -171,7 +171,7 @@ func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files ma
func verifyRestore(t *testing.T, r *fileRestorer, repo *TestRepo) {
for _, file := range r.files {
target := r.targetPath(file.location)
- data, err := ioutil.ReadFile(target)
+ data, err := os.ReadFile(target)
if err != nil {
t.Errorf("unable to read file %v: %v", file.location, err)
continue
@@ -185,69 +185,70 @@ func verifyRestore(t *testing.T, r *fileRestorer, repo *TestRepo) {
}
func TestFileRestorerBasic(t *testing.T) {
- tempdir, cleanup := rtest.TempDir(t)
- defer cleanup()
-
- restoreAndVerify(t, tempdir, []TestFile{
- {
- name: "file1",
- blobs: []TestBlob{
- {"data1-1", "pack1-1"},
- {"data1-2", "pack1-2"},
+ tempdir := rtest.TempDir(t)
+
+ for _, sparse := range []bool{false, true} {
+ restoreAndVerify(t, tempdir, []TestFile{
+ {
+ name: "file1",
+ blobs: []TestBlob{
+ {"data1-1", "pack1-1"},
+ {"data1-2", "pack1-2"},
+ },
},
- },
- {
- name: "file2",
- blobs: []TestBlob{
- {"data2-1", "pack2-1"},
- {"data2-2", "pack2-2"},
+ {
+ name: "file2",
+ blobs: []TestBlob{
+ {"data2-1", "pack2-1"},
+ {"data2-2", "pack2-2"},
+ },
},
- },
- {
- name: "file3",
- blobs: []TestBlob{
- // same blob multiple times
- {"data3-1", "pack3-1"},
- {"data3-1", "pack3-1"},
+ {
+ name: "file3",
+ blobs: []TestBlob{
+ // same blob multiple times
+ {"data3-1", "pack3-1"},
+ {"data3-1", "pack3-1"},
+ },
},
- },
- }, nil)
+ }, nil, sparse)
+ }
}
func TestFileRestorerPackSkip(t *testing.T) {
- tempdir, cleanup := rtest.TempDir(t)
- defer cleanup()
+ tempdir := rtest.TempDir(t)
files := make(map[string]bool)
files["file2"] = true
- restoreAndVerify(t, tempdir, []TestFile{
- {
- name: "file1",
- blobs: []TestBlob{
- {"data1-1", "pack1"},
- {"data1-2", "pack1"},
- {"data1-3", "pack1"},
- {"data1-4", "pack1"},
- {"data1-5", "pack1"},
- {"data1-6", "pack1"},
+ for _, sparse := range []bool{false, true} {
+ restoreAndVerify(t, tempdir, []TestFile{
+ {
+ name: "file1",
+ blobs: []TestBlob{
+ {"data1-1", "pack1"},
+ {"data1-2", "pack1"},
+ {"data1-3", "pack1"},
+ {"data1-4", "pack1"},
+ {"data1-5", "pack1"},
+ {"data1-6", "pack1"},
+ },
},
- },
- {
- name: "file2",
- blobs: []TestBlob{
- // file is contained in pack1 but need pack parts to be skipped
- {"data1-2", "pack1"},
- {"data1-4", "pack1"},
- {"data1-6", "pack1"},
+ {
+ name: "file2",
+ blobs: []TestBlob{
+ // file is contained in pack1 but need pack parts to be skipped
+ {"data1-2", "pack1"},
+ {"data1-4", "pack1"},
+ {"data1-6", "pack1"},
+ },
},
- },
- }, files)
+ }, files, sparse)
+ }
}
func TestErrorRestoreFiles(t *testing.T) {
- tempdir, cleanup := rtest.TempDir(t)
- defer cleanup()
+ tempdir := rtest.TempDir(t)
content := []TestFile{
{
name: "file1",
@@ -264,7 +265,7 @@ func TestErrorRestoreFiles(t *testing.T) {
return loadError
}
- r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2)
+ r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false)
r.files = repo.files
err := r.restoreFiles(context.TODO())
@@ -278,8 +279,7 @@ func TestDownloadError(t *testing.T) {
}
func testPartialDownloadError(t *testing.T, part int) {
- tempdir, cleanup := rtest.TempDir(t)
- defer cleanup()
+ tempdir := rtest.TempDir(t)
content := []TestFile{
{
name: "file1",
@@ -304,7 +304,7 @@ func testPartialDownloadError(t *testing.T, part int) {
return loader(ctx, h, length, offset, fn)
}
- r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2)
+ r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false)
r.files = repo.files
r.Error = func(s string, e error) error {
// ignore errors as in the `restore` command
diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go
index 8b7ee4353..0a26101f4 100644
--- a/internal/restorer/fileswriter.go
+++ b/internal/restorer/fileswriter.go
@@ -19,30 +19,34 @@ type filesWriter struct {
type filesWriterBucket struct {
lock sync.Mutex
- files map[string]*os.File
- users map[string]int
+ files map[string]*partialFile
+}
+
+type partialFile struct {
+ *os.File
+ users int // Reference count.
+ sparse bool
}
func newFilesWriter(count int) *filesWriter {
buckets := make([]filesWriterBucket, count)
for b := 0; b < count; b++ {
- buckets[b].files = make(map[string]*os.File)
- buckets[b].users = make(map[string]int)
+ buckets[b].files = make(map[string]*partialFile)
}
return &filesWriter{
buckets: buckets,
}
}
-func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64) error {
+func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, createSize int64, sparse bool) error {
bucket := &w.buckets[uint(xxhash.Sum64String(path))%uint(len(w.buckets))]
- acquireWriter := func() (*os.File, error) {
+ acquireWriter := func() (*partialFile, error) {
bucket.lock.Lock()
defer bucket.lock.Unlock()
if wr, ok := bucket.files[path]; ok {
- bucket.users[path]++
+ bucket.files[path].users++
return wr, nil
}
@@ -53,39 +57,45 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create
flags = os.O_WRONLY
}
- wr, err := os.OpenFile(path, flags, 0600)
+ f, err := os.OpenFile(path, flags, 0600)
if err != nil {
return nil, err
}
+ wr := &partialFile{File: f, users: 1, sparse: sparse}
bucket.files[path] = wr
- bucket.users[path] = 1
if createSize >= 0 {
- err := preallocateFile(wr, createSize)
- if err != nil {
- // Just log the preallocate error but don't let it cause the restore process to fail.
- // Preallocate might return an error if the filesystem (implementation) does not
- // support preallocation or our parameters combination to the preallocate call
- // This should yield a syscall.ENOTSUP error, but some other errors might also
- // show up.
- debug.Log("Failed to preallocate %v with size %v: %v", path, createSize, err)
+ if sparse {
+ err = truncateSparse(f, createSize)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ err := preallocateFile(wr.File, createSize)
+ if err != nil {
+ // Just log the preallocate error but don't let it cause the restore process to fail.
+ // Preallocate might return an error if the filesystem (implementation) does not
+ // support preallocation or our parameters combination to the preallocate call
+ // This should yield a syscall.ENOTSUP error, but some other errors might also
+ // show up.
+ debug.Log("Failed to preallocate %v with size %v: %v", path, createSize, err)
+ }
}
}
return wr, nil
}
- releaseWriter := func(wr *os.File) error {
+ releaseWriter := func(wr *partialFile) error {
bucket.lock.Lock()
defer bucket.lock.Unlock()
- if bucket.users[path] == 1 {
+ if bucket.files[path].users == 1 {
delete(bucket.files, path)
- delete(bucket.users, path)
return wr.Close()
}
- bucket.users[path]--
+ bucket.files[path].users--
return nil
}
diff --git a/internal/restorer/fileswriter_test.go b/internal/restorer/fileswriter_test.go
index a6b7e011b..7beb9a2dc 100644
--- a/internal/restorer/fileswriter_test.go
+++ b/internal/restorer/fileswriter_test.go
@@ -1,42 +1,36 @@
package restorer
import (
- "io/ioutil"
+ "os"
"testing"
rtest "github.com/restic/restic/internal/test"
)
func TestFilesWriterBasic(t *testing.T) {
- dir, cleanup := rtest.TempDir(t)
- defer cleanup()
-
+ dir := rtest.TempDir(t)
w := newFilesWriter(1)
f1 := dir + "/f1"
f2 := dir + "/f2"
- rtest.OK(t, w.writeToFile(f1, []byte{1}, 0, 2))
+ rtest.OK(t, w.writeToFile(f1, []byte{1}, 0, 2, false))
rtest.Equals(t, 0, len(w.buckets[0].files))
- rtest.Equals(t, 0, len(w.buckets[0].users))
- rtest.OK(t, w.writeToFile(f2, []byte{2}, 0, 2))
+ rtest.OK(t, w.writeToFile(f2, []byte{2}, 0, 2, false))
rtest.Equals(t, 0, len(w.buckets[0].files))
- rtest.Equals(t, 0, len(w.buckets[0].users))
- rtest.OK(t, w.writeToFile(f1, []byte{1}, 1, -1))
+ rtest.OK(t, w.writeToFile(f1, []byte{1}, 1, -1, false))
rtest.Equals(t, 0, len(w.buckets[0].files))
- rtest.Equals(t, 0, len(w.buckets[0].users))
- rtest.OK(t, w.writeToFile(f2, []byte{2}, 1, -1))
+ rtest.OK(t, w.writeToFile(f2, []byte{2}, 1, -1, false))
rtest.Equals(t, 0, len(w.buckets[0].files))
- rtest.Equals(t, 0, len(w.buckets[0].users))
- buf, err := ioutil.ReadFile(f1)
+ buf, err := os.ReadFile(f1)
rtest.OK(t, err)
rtest.Equals(t, []byte{1, 1}, buf)
- buf, err = ioutil.ReadFile(f2)
+ buf, err = os.ReadFile(f2)
rtest.OK(t, err)
rtest.Equals(t, []byte{2, 2}, buf)
}
diff --git a/internal/restorer/preallocate_test.go b/internal/restorer/preallocate_test.go
index 158b8a74c..0cc2b3f5d 100644
--- a/internal/restorer/preallocate_test.go
+++ b/internal/restorer/preallocate_test.go
@@ -14,8 +14,7 @@ import (
func TestPreallocate(t *testing.T) {
for _, i := range []int64{0, 1, 4096, 1024 * 1024} {
t.Run(strconv.FormatInt(i, 10), func(t *testing.T) {
- dirpath, cleanup := test.TempDir(t)
- defer cleanup()
+ dirpath := test.TempDir(t)
flags := os.O_CREATE | os.O_TRUNC | os.O_WRONLY
wr, err := os.OpenFile(path.Join(dirpath, "test"), flags, 0600)
diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go
index 829e5aedc..4dfe3c3a8 100644
--- a/internal/restorer/restorer.go
+++ b/internal/restorer/restorer.go
@@ -16,8 +16,9 @@ import (
// Restorer is used to restore a snapshot to a directory.
type Restorer struct {
- repo restic.Repository
- sn *restic.Snapshot
+ repo restic.Repository
+ sn *restic.Snapshot
+ sparse bool
Error func(location string, err error) error
SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool)
@@ -26,21 +27,16 @@ type Restorer struct {
var restorerAbortOnAllErrors = func(location string, err error) error { return err }
// NewRestorer creates a restorer preloaded with the content from the snapshot id.
-func NewRestorer(ctx context.Context, repo restic.Repository, id restic.ID) (*Restorer, error) {
+func NewRestorer(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, sparse bool) *Restorer {
r := &Restorer{
repo: repo,
+ sparse: sparse,
Error: restorerAbortOnAllErrors,
SelectFilter: func(string, string, *restic.Node) (bool, bool) { return true, true },
+ sn: sn,
}
- var err error
-
- r.sn, err = restic.LoadSnapshot(ctx, repo, id)
- if err != nil {
- return nil, err
- }
-
- return r, nil
+ return r
}
type treeVisitor struct {
@@ -188,7 +184,7 @@ func (res *Restorer) restoreHardlinkAt(node *restic.Node, target, path, location
}
err := fs.Link(target, path)
if err != nil {
- return errors.Wrap(err, "CreateHardlink")
+ return errors.WithStack(err)
}
// TODO investigate if hardlinks have separate metadata on any supported system
return res.restoreNodeMetadataTo(node, path, location)
@@ -219,7 +215,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error {
}
idx := NewHardlinkIndex()
- filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, res.repo.Connections())
+ filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, res.repo.Connections(), res.sparse)
filerestorer.Error = res.Error
debug.Log("first pass for %q", dst)
diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go
index 2eea1a6fd..d6cd0c80a 100644
--- a/internal/restorer/restorer_test.go
+++ b/internal/restorer/restorer_test.go
@@ -3,7 +3,8 @@ package restorer
import (
"bytes"
"context"
- "io/ioutil"
+ "io"
+ "math"
"os"
"path/filepath"
"runtime"
@@ -11,6 +12,7 @@ import (
"testing"
"time"
+ "github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@@ -319,19 +321,13 @@ func TestRestorer(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
- _, id := saveSnapshot(t, repo, test.Snapshot)
+ repo := repository.TestRepository(t)
+ sn, id := saveSnapshot(t, repo, test.Snapshot)
t.Logf("snapshot saved as %v", id.Str())
- res, err := NewRestorer(context.TODO(), repo, id)
- if err != nil {
- t.Fatal(err)
- }
-
- tempdir, cleanup := rtest.TempDir(t)
- defer cleanup()
+ res := NewRestorer(context.TODO(), repo, sn, false)
+ tempdir := rtest.TempDir(t)
// make sure we're creating a new subdir of the tempdir
tempdir = filepath.Join(tempdir, "target")
@@ -364,7 +360,7 @@ func TestRestorer(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- err = res.RestoreTo(ctx, tempdir)
+ err := res.RestoreTo(ctx, tempdir)
if err != nil {
t.Fatal(err)
}
@@ -402,7 +398,7 @@ func TestRestorer(t *testing.T) {
}
for filename, content := range test.Files {
- data, err := ioutil.ReadFile(filepath.Join(tempdir, filepath.FromSlash(filename)))
+ data, err := os.ReadFile(filepath.Join(tempdir, filepath.FromSlash(filename)))
if err != nil {
t.Errorf("unable to read file %v: %v", filename, err)
continue
@@ -441,21 +437,15 @@ func TestRestorerRelative(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
- _, id := saveSnapshot(t, repo, test.Snapshot)
+ sn, id := saveSnapshot(t, repo, test.Snapshot)
t.Logf("snapshot saved as %v", id.Str())
- res, err := NewRestorer(context.TODO(), repo, id)
- if err != nil {
- t.Fatal(err)
- }
+ res := NewRestorer(context.TODO(), repo, sn, false)
- tempdir, cleanup := rtest.TempDir(t)
- defer cleanup()
-
- cleanup = rtest.Chdir(t, tempdir)
+ tempdir := rtest.TempDir(t)
+ cleanup := rtest.Chdir(t, tempdir)
defer cleanup()
errors := make(map[string]string)
@@ -468,7 +458,7 @@ func TestRestorerRelative(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- err = res.RestoreTo(ctx, "restore")
+ err := res.RestoreTo(ctx, "restore")
if err != nil {
t.Fatal(err)
}
@@ -481,7 +471,7 @@ func TestRestorerRelative(t *testing.T) {
}
for filename, content := range test.Files {
- data, err := ioutil.ReadFile(filepath.Join(tempdir, "restore", filepath.FromSlash(filename)))
+ data, err := os.ReadFile(filepath.Join(tempdir, "restore", filepath.FromSlash(filename)))
if err != nil {
t.Errorf("unable to read file %v: %v", filename, err)
continue
@@ -678,27 +668,21 @@ func TestRestorerTraverseTree(t *testing.T) {
for _, test := range tests {
t.Run("", func(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
- sn, id := saveSnapshot(t, repo, test.Snapshot)
+ repo := repository.TestRepository(t)
+ sn, _ := saveSnapshot(t, repo, test.Snapshot)
- res, err := NewRestorer(context.TODO(), repo, id)
- if err != nil {
- t.Fatal(err)
- }
+ res := NewRestorer(context.TODO(), repo, sn, false)
res.SelectFilter = test.Select
- tempdir, cleanup := rtest.TempDir(t)
- defer cleanup()
-
+ tempdir := rtest.TempDir(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// make sure we're creating a new subdir of the tempdir
target := filepath.Join(tempdir, "target")
- _, err = res.traverseTree(ctx, target, string(filepath.Separator), *sn.Tree, test.Visitor(t))
+ _, err := res.traverseTree(ctx, target, string(filepath.Separator), *sn.Tree, test.Visitor(t))
if err != nil {
t.Fatal(err)
}
@@ -730,10 +714,9 @@ func checkConsistentInfo(t testing.TB, file string, fi os.FileInfo, modtime time
func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) {
timeForTest := time.Date(2019, time.January, 9, 1, 46, 40, 0, time.UTC)
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
- _, id := saveSnapshot(t, repo, Snapshot{
+ sn, _ := saveSnapshot(t, repo, Snapshot{
Nodes: map[string]Node{
"dir": Dir{
Mode: normalizeFileMode(0750 | os.ModeDir),
@@ -764,8 +747,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) {
},
})
- res, err := NewRestorer(context.TODO(), repo, id)
- rtest.OK(t, err)
+ res := NewRestorer(context.TODO(), repo, sn, false)
res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
switch filepath.ToSlash(item) {
@@ -784,13 +766,11 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) {
return selectedForRestore, childMayBeSelected
}
- tempdir, cleanup := rtest.TempDir(t)
- defer cleanup()
-
+ tempdir := rtest.TempDir(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- err = res.RestoreTo(ctx, tempdir)
+ err := res.RestoreTo(ctx, tempdir)
rtest.OK(t, err)
var testPatterns = []struct {
@@ -819,22 +799,17 @@ func TestVerifyCancel(t *testing.T) {
},
}
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
+ sn, _ := saveSnapshot(t, repo, snapshot)
- _, id := saveSnapshot(t, repo, snapshot)
-
- res, err := NewRestorer(context.TODO(), repo, id)
- rtest.OK(t, err)
-
- tempdir, cleanup := rtest.TempDir(t)
- defer cleanup()
+ res := NewRestorer(context.TODO(), repo, sn, false)
+ tempdir := rtest.TempDir(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
rtest.OK(t, res.RestoreTo(ctx, tempdir))
- err = ioutil.WriteFile(filepath.Join(tempdir, "foo"), []byte("bar"), 0644)
+ err := os.WriteFile(filepath.Join(tempdir, "foo"), []byte("bar"), 0644)
rtest.OK(t, err)
var errs []error
@@ -849,3 +824,54 @@ func TestVerifyCancel(t *testing.T) {
rtest.Equals(t, 1, len(errs))
rtest.Assert(t, strings.Contains(errs[0].Error(), "Invalid file size for"), "wrong error %q", errs[0].Error())
}
+
+func TestRestorerSparseFiles(t *testing.T) {
+ repo := repository.TestRepository(t)
+
+ var zeros [1<<20 + 13]byte
+
+ target := &fs.Reader{
+ Mode: 0600,
+ Name: "/zeros",
+ ReadCloser: io.NopCloser(bytes.NewReader(zeros[:])),
+ }
+ sc := archiver.NewScanner(target)
+ err := sc.Scan(context.TODO(), []string{"/zeros"})
+ rtest.OK(t, err)
+
+ arch := archiver.New(repo, target, archiver.Options{})
+ sn, _, err := arch.Snapshot(context.Background(), []string{"/zeros"},
+ archiver.SnapshotOptions{})
+ rtest.OK(t, err)
+
+ res := NewRestorer(context.TODO(), repo, sn, true)
+
+ tempdir := rtest.TempDir(t)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ err = res.RestoreTo(ctx, tempdir)
+ rtest.OK(t, err)
+
+ filename := filepath.Join(tempdir, "zeros")
+ content, err := os.ReadFile(filename)
+ rtest.OK(t, err)
+
+ rtest.Equals(t, len(zeros[:]), len(content))
+ rtest.Equals(t, zeros[:], content)
+
+ blocks := getBlockCount(t, filename)
+ if blocks < 0 {
+ return
+ }
+
+ // st.Blocks is the size in 512-byte blocks.
+ denseBlocks := math.Ceil(float64(len(zeros)) / 512)
+ sparsity := 1 - float64(blocks)/denseBlocks
+
+ // This should report 100% sparse. We don't assert that,
+ // as the behavior of sparse writes depends on the underlying
+ // file system as well as the OS.
+ t.Logf("wrote %d zeros as %d blocks, %.1f%% sparse",
+ len(zeros), blocks, 100*sparsity)
+}
diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go
index 13e318c98..dc327a9c9 100644
--- a/internal/restorer/restorer_unix_test.go
+++ b/internal/restorer/restorer_unix_test.go
@@ -16,10 +16,9 @@ import (
)
func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) {
- repo, cleanup := repository.TestRepository(t)
- defer cleanup()
+ repo := repository.TestRepository(t)
- _, id := saveSnapshot(t, repo, Snapshot{
+ sn, _ := saveSnapshot(t, repo, Snapshot{
Nodes: map[string]Node{
"dirtest": Dir{
Nodes: map[string]Node{
@@ -30,20 +29,17 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) {
},
})
- res, err := NewRestorer(context.TODO(), repo, id)
- rtest.OK(t, err)
+ res := NewRestorer(context.TODO(), repo, sn, false)
res.SelectFilter = func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
return true, true
}
- tempdir, cleanup := rtest.TempDir(t)
- defer cleanup()
-
+ tempdir := rtest.TempDir(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- err = res.RestoreTo(ctx, tempdir)
+ err := res.RestoreTo(ctx, tempdir)
rtest.OK(t, err)
f1, err := os.Stat(filepath.Join(tempdir, "dirtest/file1"))
@@ -60,3 +56,13 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) {
rtest.Equals(t, s1.Ino, s2.Ino)
}
}
+
+func getBlockCount(t *testing.T, filename string) int64 {
+ fi, err := os.Stat(filename)
+ rtest.OK(t, err)
+ st := fi.Sys().(*syscall.Stat_t)
+ if st == nil {
+ return -1
+ }
+ return st.Blocks
+}
diff --git a/internal/restorer/restorer_windows_test.go b/internal/restorer/restorer_windows_test.go
new file mode 100644
index 000000000..3ec4b1f11
--- /dev/null
+++ b/internal/restorer/restorer_windows_test.go
@@ -0,0 +1,35 @@
+//go:build windows
+// +build windows
+
+package restorer
+
+import (
+ "math"
+ "syscall"
+ "testing"
+ "unsafe"
+
+ rtest "github.com/restic/restic/internal/test"
+ "golang.org/x/sys/windows"
+)
+
+func getBlockCount(t *testing.T, filename string) int64 {
+ libkernel32 := windows.NewLazySystemDLL("kernel32.dll")
+ err := libkernel32.Load()
+ rtest.OK(t, err)
+ proc := libkernel32.NewProc("GetCompressedFileSizeW")
+ err = proc.Find()
+ rtest.OK(t, err)
+
+ namePtr, err := syscall.UTF16PtrFromString(filename)
+ rtest.OK(t, err)
+
+ result, _, _ := proc.Call(uintptr(unsafe.Pointer(namePtr)), 0)
+
+ const invalidFileSize = uintptr(4294967295)
+ if result == invalidFileSize {
+ return -1
+ }
+
+ return int64(math.Ceil(float64(result) / 512))
+}
diff --git a/internal/restorer/sparsewrite.go b/internal/restorer/sparsewrite.go
new file mode 100644
index 000000000..2c1f234de
--- /dev/null
+++ b/internal/restorer/sparsewrite.go
@@ -0,0 +1,37 @@
+//go:build !windows
+// +build !windows
+
+package restorer
+
+import (
+ "github.com/restic/restic/internal/restic"
+)
+
+// WriteAt writes p to f.File at offset. It tries to do a sparse write
+// and updates f.size.
+func (f *partialFile) WriteAt(p []byte, offset int64) (n int, err error) {
+ if !f.sparse {
+ return f.File.WriteAt(p, offset)
+ }
+
+ n = len(p)
+
+ // Skip the longest all-zero prefix of p.
+ // If it's long enough, we can punch a hole in the file.
+ skipped := restic.ZeroPrefixLen(p)
+ p = p[skipped:]
+ offset += int64(skipped)
+
+ switch {
+ case len(p) == 0:
+ // All zeros, file already big enough. A previous WriteAt or
+ // Truncate will have produced the zeros in f.File.
+
+ default:
+ var n2 int
+ n2, err = f.File.WriteAt(p, offset)
+ n = skipped + n2
+ }
+
+ return n, err
+}
diff --git a/internal/restorer/truncate_other.go b/internal/restorer/truncate_other.go
new file mode 100644
index 000000000..ed7ab04c5
--- /dev/null
+++ b/internal/restorer/truncate_other.go
@@ -0,0 +1,10 @@
+//go:build !windows
+// +build !windows
+
+package restorer
+
+import "os"
+
+func truncateSparse(f *os.File, size int64) error {
+ return f.Truncate(size)
+}
diff --git a/internal/restorer/truncate_windows.go b/internal/restorer/truncate_windows.go
new file mode 100644
index 000000000..831a117d1
--- /dev/null
+++ b/internal/restorer/truncate_windows.go
@@ -0,0 +1,19 @@
+package restorer
+
+import (
+ "os"
+
+ "github.com/restic/restic/internal/debug"
+ "golang.org/x/sys/windows"
+)
+
+func truncateSparse(f *os.File, size int64) error {
+ // try setting the sparse file attribute, but ignore the error if it fails
+ var t uint32
+ err := windows.DeviceIoControl(windows.Handle(f.Fd()), windows.FSCTL_SET_SPARSE, nil, 0, nil, 0, &t, nil)
+ if err != nil {
+ debug.Log("failed to set sparse attribute for %v: %v", f.Name(), err)
+ }
+
+ return f.Truncate(size)
+}
diff --git a/internal/selfupdate/download.go b/internal/selfupdate/download.go
index b12afd61c..2c7441e3e 100644
--- a/internal/selfupdate/download.go
+++ b/internal/selfupdate/download.go
@@ -10,7 +10,6 @@ import (
"encoding/hex"
"fmt"
"io"
- "io/ioutil"
"os"
"path/filepath"
"runtime"
@@ -69,7 +68,7 @@ func extractToFile(buf []byte, filename, target string, printf func(string, ...i
// Write everything to a temp file
dir := filepath.Dir(target)
- new, err := ioutil.TempFile(dir, "restic")
+ new, err := os.CreateTemp(dir, "restic")
if err != nil {
return err
}
diff --git a/internal/selfupdate/download_test.go b/internal/selfupdate/download_test.go
new file mode 100644
index 000000000..00160eef2
--- /dev/null
+++ b/internal/selfupdate/download_test.go
@@ -0,0 +1,44 @@
+package selfupdate
+
+import (
+ "archive/zip"
+ "bytes"
+ "os"
+ "path/filepath"
+ "testing"
+
+ rtest "github.com/restic/restic/internal/test"
+)
+
+func TestExtractToFileZip(t *testing.T) {
+ printf := func(string, ...interface{}) {}
+ dir := t.TempDir()
+
+ ext := "zip"
+ data := []byte("Hello World!")
+
+ // create dummy archive
+ var archive bytes.Buffer
+ zw := zip.NewWriter(&archive)
+ w, err := zw.CreateHeader(&zip.FileHeader{
+ Name: "example.exe",
+ UncompressedSize64: uint64(len(data)),
+ })
+ rtest.OK(t, err)
+ _, err = w.Write(data[:])
+ rtest.OK(t, err)
+ rtest.OK(t, zw.Close())
+
+ // run twice to test creating a new file and overwriting
+ for i := 0; i < 2; i++ {
+ outfn := filepath.Join(dir, ext+"-out")
+ rtest.OK(t, extractToFile(archive.Bytes(), "src."+ext, outfn, printf))
+
+ outdata, err := os.ReadFile(outfn)
+ rtest.OK(t, err)
+ rtest.Assert(t, bytes.Equal(data[:], outdata), "%v contains wrong data", outfn)
+
+ // overwrite to test the file is properly overwritten
+ rtest.OK(t, os.WriteFile(outfn, []byte{1, 2, 3}, 0))
+ }
+}
diff --git a/internal/selfupdate/download_windows.go b/internal/selfupdate/download_windows.go
index 4f2797385..50480eab6 100644
--- a/internal/selfupdate/download_windows.go
+++ b/internal/selfupdate/download_windows.go
@@ -7,11 +7,18 @@ import (
"fmt"
"os"
"path/filepath"
+
+ "github.com/restic/restic/internal/errors"
)
// Rename (rather than remove) the running version. The running binary will be locked
// on Windows and cannot be removed while still executing.
func removeResticBinary(dir, target string) error {
+ // nothing to do if the target does not exist
+ if _, err := os.Stat(target); errors.Is(err, os.ErrNotExist) {
+ return nil
+ }
+
backup := filepath.Join(dir, filepath.Base(target)+".bak")
if _, err := os.Stat(backup); err == nil {
_ = os.Remove(backup)
diff --git a/internal/selfupdate/github.go b/internal/selfupdate/github.go
index 637f6765e..cdee8c74d 100644
--- a/internal/selfupdate/github.go
+++ b/internal/selfupdate/github.go
@@ -4,13 +4,12 @@ import (
"context"
"encoding/json"
"fmt"
- "io/ioutil"
+ "io"
"net/http"
"strings"
"time"
"github.com/pkg/errors"
- "golang.org/x/net/context/ctxhttp"
)
// Release collects data about a single release on GitHub.
@@ -53,7 +52,7 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro
defer cancel()
url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", owner, repo)
- req, err := http.NewRequest(http.MethodGet, url, nil)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return Release{}, err
}
@@ -61,7 +60,7 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro
// pin API version 3
req.Header.Set("Accept", "application/vnd.github.v3+json")
- res, err := ctxhttp.Do(ctx, http.DefaultClient, req)
+ res, err := http.DefaultClient.Do(req)
if err != nil {
return Release{}, err
}
@@ -81,7 +80,7 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro
return Release{}, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
}
- buf, err := ioutil.ReadAll(res.Body)
+ buf, err := io.ReadAll(res.Body)
if err != nil {
_ = res.Body.Close()
return Release{}, err
@@ -112,7 +111,7 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro
}
func getGithubData(ctx context.Context, url string) ([]byte, error) {
- req, err := http.NewRequest(http.MethodGet, url, nil)
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
@@ -120,7 +119,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
// request binary data
req.Header.Set("Accept", "application/octet-stream")
- res, err := ctxhttp.Do(ctx, http.DefaultClient, req)
+ res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
@@ -129,7 +128,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
}
- buf, err := ioutil.ReadAll(res.Body)
+ buf, err := io.ReadAll(res.Body)
if err != nil {
_ = res.Body.Close()
return nil, err
diff --git a/internal/test/helpers.go b/internal/test/helpers.go
index 9ace2df5e..93178ae10 100644
--- a/internal/test/helpers.go
+++ b/internal/test/helpers.go
@@ -5,7 +5,6 @@ import (
"compress/gzip"
"fmt"
"io"
- "io/ioutil"
"os"
"os/exec"
"path/filepath"
@@ -131,7 +130,7 @@ func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) {
// Env creates a test environment and extracts the repository fixture.
// Returned is the repo path and a cleanup function.
func Env(t testing.TB, repoFixture string) (repodir string, cleanup func()) {
- tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-env-")
+ tempdir, err := os.MkdirTemp(TestTempDir, "restic-test-env-")
OK(t, err)
fd, err := os.Open(repoFixture)
@@ -192,22 +191,23 @@ func RemoveAll(t testing.TB, path string) {
OK(t, err)
}
-// TempDir returns a temporary directory that is removed when cleanup is
-// called, except if TestCleanupTempDirs is set to false.
-func TempDir(t testing.TB) (path string, cleanup func()) {
- tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-")
+// TempDir returns a temporary directory that is removed by t.Cleanup,
+// except if TestCleanupTempDirs is set to false.
+func TempDir(t testing.TB) string {
+ tempdir, err := os.MkdirTemp(TestTempDir, "restic-test-")
if err != nil {
t.Fatal(err)
}
- return tempdir, func() {
+ t.Cleanup(func() {
if !TestCleanupTempDirs {
t.Logf("leaving temporary directory %v used for test", tempdir)
return
}
RemoveAll(t, tempdir)
- }
+ })
+ return tempdir
}
// Chdir changes the current directory to dest.
diff --git a/internal/textfile/read.go b/internal/textfile/read.go
index 3129ba8fe..5b5367b26 100644
--- a/internal/textfile/read.go
+++ b/internal/textfile/read.go
@@ -4,7 +4,7 @@ package textfile
import (
"bytes"
- "io/ioutil"
+ "os"
"golang.org/x/text/encoding/unicode"
)
@@ -34,7 +34,7 @@ func Decode(data []byte) ([]byte, error) {
// Read returns the contents of the file, converted to UTF-8, stripped of any BOM.
func Read(filename string) ([]byte, error) {
- data, err := ioutil.ReadFile(filename)
+ data, err := os.ReadFile(filename)
if err != nil {
return nil, err
}
diff --git a/internal/textfile/read_test.go b/internal/textfile/read_test.go
index d4de03513..575de2f65 100644
--- a/internal/textfile/read_test.go
+++ b/internal/textfile/read_test.go
@@ -3,7 +3,6 @@ package textfile
import (
"bytes"
"encoding/hex"
- "io/ioutil"
"os"
"testing"
)
@@ -13,7 +12,7 @@ import (
func writeTempfile(t testing.TB, data []byte) (string, func()) {
t.Helper()
- f, err := ioutil.TempFile("", "restic-test-textfile-read-")
+ f, err := os.CreateTemp("", "restic-test-textfile-read-")
if err != nil {
t.Fatal(err)
}
diff --git a/internal/ui/backup/json.go b/internal/ui/backup/json.go
index 1cbd0c197..85076b3bb 100644
--- a/internal/ui/backup/json.go
+++ b/internal/ui/backup/json.go
@@ -15,7 +15,6 @@ import (
// JSONProgress reports progress for the `backup` command in JSON.
type JSONProgress struct {
*ui.Message
- *ui.StdioWrapper
term *termstatus.Terminal
v uint
@@ -27,10 +26,9 @@ var _ ProgressPrinter = &JSONProgress{}
// NewJSONProgress returns a new backup progress reporter.
func NewJSONProgress(term *termstatus.Terminal, verbosity uint) *JSONProgress {
return &JSONProgress{
- Message: ui.NewMessage(term, verbosity),
- StdioWrapper: ui.NewStdioWrapper(term),
- term: term,
- v: verbosity,
+ Message: ui.NewMessage(term, verbosity),
+ term: term,
+ v: verbosity,
}
}
@@ -191,7 +189,7 @@ func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *Su
TotalFilesProcessed: summary.Files.New + summary.Files.Changed + summary.Files.Unchanged,
TotalBytesProcessed: summary.ProcessedBytes,
TotalDuration: time.Since(start).Seconds(),
- SnapshotID: snapshotID.Str(),
+ SnapshotID: snapshotID.String(),
DryRun: dryRun,
})
}
diff --git a/internal/ui/backup/progress.go b/internal/ui/backup/progress.go
index a4b641fe9..720a2a58f 100644
--- a/internal/ui/backup/progress.go
+++ b/internal/ui/backup/progress.go
@@ -1,16 +1,16 @@
package backup
import (
- "context"
- "io"
"sync"
"time"
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/restic"
- "github.com/restic/restic/internal/ui/signals"
+ "github.com/restic/restic/internal/ui/progress"
)
+// A ProgressPrinter can print various progress messages.
+// It must be safe to call its methods from concurrent goroutines.
type ProgressPrinter interface {
Update(total, processed Counter, errors uint, currentFiles map[string]struct{}, start time.Time, secs uint64)
Error(item string, err error) error
@@ -20,39 +20,15 @@ type ProgressPrinter interface {
Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool)
Reset()
- // ui.StdioWrapper
- Stdout() io.WriteCloser
- Stderr() io.WriteCloser
-
- E(msg string, args ...interface{})
P(msg string, args ...interface{})
V(msg string, args ...interface{})
- VV(msg string, args ...interface{})
}
type Counter struct {
Files, Dirs, Bytes uint64
}
-type fileWorkerMessage struct {
- filename string
- done bool
-}
-
-type ProgressReporter interface {
- CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration)
- StartFile(filename string)
- CompleteBlob(filename string, bytes uint64)
- ScannerError(item string, err error) error
- ReportTotal(item string, s archiver.ScanStats)
- SetMinUpdatePause(d time.Duration)
- Run(ctx context.Context) error
- Error(item string, err error) error
- Finish(snapshotID restic.ID)
-}
-
type Summary struct {
- sync.Mutex
Files, Dirs struct {
New uint
Changed uint
@@ -64,149 +40,85 @@ type Summary struct {
// Progress reports progress for the `backup` command.
type Progress struct {
- MinUpdatePause time.Duration
+ progress.Updater
+ mu sync.Mutex
start time.Time
- dry bool
- totalBytes uint64
+ scanStarted, scanFinished bool
- totalCh chan Counter
- processedCh chan Counter
- errCh chan struct{}
- workerCh chan fileWorkerMessage
- closed chan struct{}
+ currentFiles map[string]struct{}
+ processed, total Counter
+ errors uint
- summary *Summary
+ summary Summary
printer ProgressPrinter
}
-func NewProgress(printer ProgressPrinter) *Progress {
- return &Progress{
- // limit to 60fps by default
- MinUpdatePause: time.Second / 60,
- start: time.Now(),
-
- totalCh: make(chan Counter),
- processedCh: make(chan Counter),
- errCh: make(chan struct{}),
- workerCh: make(chan fileWorkerMessage),
- closed: make(chan struct{}),
-
- summary: &Summary{},
-
- printer: printer,
+func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress {
+ p := &Progress{
+ start: time.Now(),
+ currentFiles: make(map[string]struct{}),
+ printer: printer,
}
-}
-
-// Run regularly updates the status lines. It should be called in a separate
-// goroutine.
-func (p *Progress) Run(ctx context.Context) error {
- var (
- lastUpdate time.Time
- total, processed Counter
- errors uint
- started bool
- currentFiles = make(map[string]struct{})
- secondsRemaining uint64
- )
-
- t := time.NewTicker(time.Second)
- signalsCh := signals.GetProgressChannel()
- defer t.Stop()
- defer close(p.closed)
- // Reset status when finished
- defer p.printer.Reset()
-
- for {
- forceUpdate := false
- select {
- case <-ctx.Done():
- return nil
- case t, ok := <-p.totalCh:
- if ok {
- total = t
- started = true
- } else {
- // scan has finished
- p.totalCh = nil
- p.totalBytes = total.Bytes
- }
- case s := <-p.processedCh:
- processed.Files += s.Files
- processed.Dirs += s.Dirs
- processed.Bytes += s.Bytes
- started = true
- case <-p.errCh:
- errors++
- started = true
- case m := <-p.workerCh:
- if m.done {
- delete(currentFiles, m.filename)
- } else {
- currentFiles[m.filename] = struct{}{}
- }
- case <-t.C:
- if !started {
- continue
+ p.Updater = *progress.NewUpdater(interval, func(runtime time.Duration, final bool) {
+ if final {
+ p.printer.Reset()
+ } else {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ if !p.scanStarted {
+ return
}
- if p.totalCh == nil {
- secs := float64(time.Since(p.start) / time.Second)
- todo := float64(total.Bytes - processed.Bytes)
- secondsRemaining = uint64(secs / float64(processed.Bytes) * todo)
+ var secondsRemaining uint64
+ if p.scanFinished {
+ secs := float64(runtime / time.Second)
+ todo := float64(p.total.Bytes - p.processed.Bytes)
+ secondsRemaining = uint64(secs / float64(p.processed.Bytes) * todo)
}
- case <-signalsCh:
- forceUpdate = true
- }
- // limit update frequency
- if !forceUpdate && (time.Since(lastUpdate) < p.MinUpdatePause || p.MinUpdatePause == 0) {
- continue
+ p.printer.Update(p.total, p.processed, p.errors, p.currentFiles, p.start, secondsRemaining)
}
- lastUpdate = time.Now()
-
- p.printer.Update(total, processed, errors, currentFiles, p.start, secondsRemaining)
- }
-}
-
-// ScannerError is the error callback function for the scanner, it prints the
-// error in verbose mode and returns nil.
-func (p *Progress) ScannerError(item string, err error) error {
- return p.printer.ScannerError(item, err)
+ })
+ return p
}
// Error is the error callback function for the archiver, it prints the error and returns nil.
func (p *Progress) Error(item string, err error) error {
- cbErr := p.printer.Error(item, err)
+ p.mu.Lock()
+ p.errors++
+ p.scanStarted = true
+ p.mu.Unlock()
- select {
- case p.errCh <- struct{}{}:
- case <-p.closed:
- }
- return cbErr
+ return p.printer.Error(item, err)
}
// StartFile is called when a file is being processed by a worker.
func (p *Progress) StartFile(filename string) {
- select {
- case p.workerCh <- fileWorkerMessage{filename: filename}:
- case <-p.closed:
- }
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ p.currentFiles[filename] = struct{}{}
+}
+
+func (p *Progress) addProcessed(c Counter) {
+ p.processed.Files += c.Files
+ p.processed.Dirs += c.Dirs
+ p.processed.Bytes += c.Bytes
+ p.scanStarted = true
}
// CompleteBlob is called for all saved blobs for files.
-func (p *Progress) CompleteBlob(filename string, bytes uint64) {
- select {
- case p.processedCh <- Counter{Bytes: bytes}:
- case <-p.closed:
- }
+func (p *Progress) CompleteBlob(bytes uint64) {
+ p.mu.Lock()
+ p.addProcessed(Counter{Bytes: bytes})
+ p.mu.Unlock()
}
// CompleteItem is the status callback function for the archiver when a
// file/dir has been saved successfully.
func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) {
- p.summary.Lock()
+ p.mu.Lock()
p.summary.ItemStats.Add(s)
// for the last item "/", current is nil
@@ -214,111 +126,87 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a
p.summary.ProcessedBytes += current.Size
}
- p.summary.Unlock()
+ p.mu.Unlock()
if current == nil {
// error occurred, tell the status display to remove the line
- select {
- case p.workerCh <- fileWorkerMessage{filename: item, done: true}:
- case <-p.closed:
- }
+ p.mu.Lock()
+ delete(p.currentFiles, item)
+ p.mu.Unlock()
return
}
switch current.Type {
- case "file":
- select {
- case p.processedCh <- Counter{Files: 1}:
- case <-p.closed:
- }
- select {
- case p.workerCh <- fileWorkerMessage{filename: item, done: true}:
- case <-p.closed:
- }
case "dir":
- select {
- case p.processedCh <- Counter{Dirs: 1}:
- case <-p.closed:
- }
- }
+ p.mu.Lock()
+ p.addProcessed(Counter{Dirs: 1})
+ p.mu.Unlock()
- if current.Type == "dir" {
- if previous == nil {
+ switch {
+ case previous == nil:
p.printer.CompleteItem("dir new", item, previous, current, s, d)
- p.summary.Lock()
+ p.mu.Lock()
p.summary.Dirs.New++
- p.summary.Unlock()
- return
- }
+ p.mu.Unlock()
- if previous.Equals(*current) {
+ case previous.Equals(*current):
p.printer.CompleteItem("dir unchanged", item, previous, current, s, d)
- p.summary.Lock()
+ p.mu.Lock()
p.summary.Dirs.Unchanged++
- p.summary.Unlock()
- } else {
+ p.mu.Unlock()
+
+ default:
p.printer.CompleteItem("dir modified", item, previous, current, s, d)
- p.summary.Lock()
+ p.mu.Lock()
p.summary.Dirs.Changed++
- p.summary.Unlock()
+ p.mu.Unlock()
}
- } else if current.Type == "file" {
- select {
- case p.workerCh <- fileWorkerMessage{done: true, filename: item}:
- case <-p.closed:
- }
+ case "file":
+ p.mu.Lock()
+ p.addProcessed(Counter{Files: 1})
+ delete(p.currentFiles, item)
+ p.mu.Unlock()
- if previous == nil {
+ switch {
+ case previous == nil:
p.printer.CompleteItem("file new", item, previous, current, s, d)
- p.summary.Lock()
+ p.mu.Lock()
p.summary.Files.New++
- p.summary.Unlock()
- return
- }
+ p.mu.Unlock()
- if previous.Equals(*current) {
+ case previous.Equals(*current):
p.printer.CompleteItem("file unchanged", item, previous, current, s, d)
- p.summary.Lock()
+ p.mu.Lock()
p.summary.Files.Unchanged++
- p.summary.Unlock()
- } else {
+ p.mu.Unlock()
+
+ default:
p.printer.CompleteItem("file modified", item, previous, current, s, d)
- p.summary.Lock()
+ p.mu.Lock()
p.summary.Files.Changed++
- p.summary.Unlock()
+ p.mu.Unlock()
}
}
}
// ReportTotal sets the total stats up to now
func (p *Progress) ReportTotal(item string, s archiver.ScanStats) {
- select {
- case p.totalCh <- Counter{Files: uint64(s.Files), Dirs: uint64(s.Dirs), Bytes: s.Bytes}:
- case <-p.closed:
- }
+ p.mu.Lock()
+ defer p.mu.Unlock()
+
+ p.total = Counter{Files: uint64(s.Files), Dirs: uint64(s.Dirs), Bytes: s.Bytes}
+ p.scanStarted = true
if item == "" {
+ p.scanFinished = true
p.printer.ReportTotal(item, p.start, s)
- close(p.totalCh)
- return
}
}
// Finish prints the finishing messages.
-func (p *Progress) Finish(snapshotID restic.ID) {
+func (p *Progress) Finish(snapshotID restic.ID, dryrun bool) {
// wait for the status update goroutine to shut down
- <-p.closed
- p.printer.Finish(snapshotID, p.start, p.summary, p.dry)
-}
-
-// SetMinUpdatePause sets b.MinUpdatePause. It satisfies the
-// ArchiveProgressReporter interface.
-func (p *Progress) SetMinUpdatePause(d time.Duration) {
- p.MinUpdatePause = d
-}
-
-// SetDryRun marks the backup as a "dry run".
-func (p *Progress) SetDryRun() {
- p.dry = true
+ p.Updater.Done()
+ p.printer.Finish(snapshotID, p.start, &p.summary, dryrun)
}
diff --git a/internal/ui/backup/progress_test.go b/internal/ui/backup/progress_test.go
new file mode 100644
index 000000000..a7282c7da
--- /dev/null
+++ b/internal/ui/backup/progress_test.go
@@ -0,0 +1,78 @@
+package backup
+
+import (
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/restic/restic/internal/archiver"
+ "github.com/restic/restic/internal/restic"
+)
+
+type mockPrinter struct {
+ sync.Mutex
+ dirUnchanged, fileNew bool
+ id restic.ID
+}
+
+func (p *mockPrinter) Update(total, processed Counter, errors uint, currentFiles map[string]struct{}, start time.Time, secs uint64) {
+}
+func (p *mockPrinter) Error(item string, err error) error { return err }
+func (p *mockPrinter) ScannerError(item string, err error) error { return err }
+
+func (p *mockPrinter) CompleteItem(messageType string, item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) {
+ p.Lock()
+ defer p.Unlock()
+
+ switch messageType {
+ case "dir unchanged":
+ p.dirUnchanged = true
+ case "file new":
+ p.fileNew = true
+ }
+}
+
+func (p *mockPrinter) ReportTotal(_ string, _ time.Time, _ archiver.ScanStats) {}
+func (p *mockPrinter) Finish(id restic.ID, _ time.Time, summary *Summary, dryRun bool) {
+ p.Lock()
+ defer p.Unlock()
+
+ _ = *summary // Should not be nil.
+ p.id = id
+}
+
+func (p *mockPrinter) Reset() {}
+
+func (p *mockPrinter) P(msg string, args ...interface{}) {}
+func (p *mockPrinter) V(msg string, args ...interface{}) {}
+
+func TestProgress(t *testing.T) {
+ t.Parallel()
+
+ prnt := &mockPrinter{}
+ prog := NewProgress(prnt, time.Millisecond)
+
+ prog.StartFile("foo")
+ prog.CompleteBlob(1024)
+
+ // "dir unchanged"
+ node := restic.Node{Type: "dir"}
+ prog.CompleteItem("foo", &node, &node, archiver.ItemStats{}, 0)
+ // "file new"
+ node.Type = "file"
+ prog.CompleteItem("foo", nil, &node, archiver.ItemStats{}, 0)
+
+ time.Sleep(10 * time.Millisecond)
+ id := restic.NewRandomID()
+ prog.Finish(id, false)
+
+ if !prnt.dirUnchanged {
+ t.Error(`"dir unchanged" event not seen`)
+ }
+ if !prnt.fileNew {
+ t.Error(`"file new" event not seen`)
+ }
+ if prnt.id != id {
+ t.Errorf("id not stored (has %v)", prnt.id)
+ }
+}
diff --git a/internal/ui/backup/text.go b/internal/ui/backup/text.go
index 03013bec1..0c5f897dd 100644
--- a/internal/ui/backup/text.go
+++ b/internal/ui/backup/text.go
@@ -14,7 +14,6 @@ import (
// TextProgress reports progress for the `backup` command.
type TextProgress struct {
*ui.Message
- *ui.StdioWrapper
term *termstatus.Terminal
}
@@ -25,9 +24,8 @@ var _ ProgressPrinter = &TextProgress{}
// NewTextProgress returns a new backup progress reporter.
func NewTextProgress(term *termstatus.Terminal, verbosity uint) *TextProgress {
return &TextProgress{
- Message: ui.NewMessage(term, verbosity),
- StdioWrapper: ui.NewStdioWrapper(term),
- term: term,
+ Message: ui.NewMessage(term, verbosity),
+ term: term,
}
}
@@ -37,26 +35,26 @@ func (b *TextProgress) Update(total, processed Counter, errors uint, currentFile
if total.Files == 0 && total.Dirs == 0 {
// no total count available yet
status = fmt.Sprintf("[%s] %v files, %s, %d errors",
- formatDuration(time.Since(start)),
- processed.Files, formatBytes(processed.Bytes), errors,
+ ui.FormatDuration(time.Since(start)),
+ processed.Files, ui.FormatBytes(processed.Bytes), errors,
)
} else {
var eta, percent string
if secs > 0 && processed.Bytes < total.Bytes {
- eta = fmt.Sprintf(" ETA %s", formatSeconds(secs))
- percent = formatPercent(processed.Bytes, total.Bytes)
+ eta = fmt.Sprintf(" ETA %s", ui.FormatSeconds(secs))
+ percent = ui.FormatPercent(processed.Bytes, total.Bytes)
percent += " "
}
// include totals
status = fmt.Sprintf("[%s] %s%v files %s, total %v files %v, %d errors%s",
- formatDuration(time.Since(start)),
+ ui.FormatDuration(time.Since(start)),
percent,
processed.Files,
- formatBytes(processed.Bytes),
+ ui.FormatBytes(processed.Bytes),
total.Files,
- formatBytes(total.Bytes),
+ ui.FormatBytes(total.Bytes),
errors,
eta,
)
@@ -85,69 +83,28 @@ func (b *TextProgress) Error(item string, err error) error {
return nil
}
-func formatPercent(numerator uint64, denominator uint64) string {
- if denominator == 0 {
- return ""
- }
-
- percent := 100.0 * float64(numerator) / float64(denominator)
-
- if percent > 100 {
- percent = 100
- }
-
- return fmt.Sprintf("%3.2f%%", percent)
-}
-
-func formatSeconds(sec uint64) string {
- hours := sec / 3600
- sec -= hours * 3600
- min := sec / 60
- sec -= min * 60
- if hours > 0 {
- return fmt.Sprintf("%d:%02d:%02d", hours, min, sec)
- }
-
- return fmt.Sprintf("%d:%02d", min, sec)
-}
-
-func formatDuration(d time.Duration) string {
- sec := uint64(d / time.Second)
- return formatSeconds(sec)
-}
-
-func formatBytes(c uint64) string {
- b := float64(c)
- switch {
- case c > 1<<40:
- return fmt.Sprintf("%.3f TiB", b/(1<<40))
- case c > 1<<30:
- return fmt.Sprintf("%.3f GiB", b/(1<<30))
- case c > 1<<20:
- return fmt.Sprintf("%.3f MiB", b/(1<<20))
- case c > 1<<10:
- return fmt.Sprintf("%.3f KiB", b/(1<<10))
- default:
- return fmt.Sprintf("%d B", c)
- }
-}
-
// CompleteItem is the status callback function for the archiver when a
// file/dir has been saved successfully.
func (b *TextProgress) CompleteItem(messageType, item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) {
switch messageType {
case "dir new":
- b.VV("new %v, saved in %.3fs (%v added, %v stored, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo), formatBytes(s.TreeSizeInRepo))
+ b.VV("new %v, saved in %.3fs (%v added, %v stored, %v metadata)",
+ item, d.Seconds(), ui.FormatBytes(s.DataSize),
+ ui.FormatBytes(s.DataSizeInRepo), ui.FormatBytes(s.TreeSizeInRepo))
case "dir unchanged":
b.VV("unchanged %v", item)
case "dir modified":
- b.VV("modified %v, saved in %.3fs (%v added, %v stored, %v metadata)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo), formatBytes(s.TreeSizeInRepo))
+ b.VV("modified %v, saved in %.3fs (%v added, %v stored, %v metadata)",
+ item, d.Seconds(), ui.FormatBytes(s.DataSize),
+ ui.FormatBytes(s.DataSizeInRepo), ui.FormatBytes(s.TreeSizeInRepo))
case "file new":
- b.VV("new %v, saved in %.3fs (%v added)", item, d.Seconds(), formatBytes(s.DataSize))
+ b.VV("new %v, saved in %.3fs (%v added)", item,
+ d.Seconds(), ui.FormatBytes(s.DataSize))
case "file unchanged":
b.VV("unchanged %v", item)
case "file modified":
- b.VV("modified %v, saved in %.3fs (%v added, %v stored)", item, d.Seconds(), formatBytes(s.DataSize), formatBytes(s.DataSizeInRepo))
+ b.VV("modified %v, saved in %.3fs (%v added, %v stored)", item,
+ d.Seconds(), ui.FormatBytes(s.DataSize), ui.FormatBytes(s.DataSizeInRepo))
}
}
@@ -155,7 +112,7 @@ func (b *TextProgress) CompleteItem(messageType, item string, previous, current
func (b *TextProgress) ReportTotal(item string, start time.Time, s archiver.ScanStats) {
b.V("scan finished in %.3fs: %v files, %s",
time.Since(start).Seconds(),
- s.Files, formatBytes(s.Bytes),
+ s.Files, ui.FormatBytes(s.Bytes),
)
}
@@ -177,11 +134,13 @@ func (b *TextProgress) Finish(snapshotID restic.ID, start time.Time, summary *Su
if dryRun {
verb = "Would add"
}
- b.P("%s to the repository: %-5s (%-5s stored)\n", verb, formatBytes(summary.ItemStats.DataSize+summary.ItemStats.TreeSize), formatBytes(summary.ItemStats.DataSizeInRepo+summary.ItemStats.TreeSizeInRepo))
+ b.P("%s to the repository: %-5s (%-5s stored)\n", verb,
+ ui.FormatBytes(summary.ItemStats.DataSize+summary.ItemStats.TreeSize),
+ ui.FormatBytes(summary.ItemStats.DataSizeInRepo+summary.ItemStats.TreeSizeInRepo))
b.P("\n")
b.P("processed %v files, %v in %s",
summary.Files.New+summary.Files.Changed+summary.Files.Unchanged,
- formatBytes(summary.ProcessedBytes),
- formatDuration(time.Since(start)),
+ ui.FormatBytes(summary.ProcessedBytes),
+ ui.FormatDuration(time.Since(start)),
)
}
diff --git a/internal/ui/format.go b/internal/ui/format.go
new file mode 100644
index 000000000..13d02f9e3
--- /dev/null
+++ b/internal/ui/format.go
@@ -0,0 +1,55 @@
+package ui
+
+import (
+ "fmt"
+ "time"
+)
+
+func FormatBytes(c uint64) string {
+ b := float64(c)
+ switch {
+ case c >= 1<<40:
+ return fmt.Sprintf("%.3f TiB", b/(1<<40))
+ case c >= 1<<30:
+ return fmt.Sprintf("%.3f GiB", b/(1<<30))
+ case c >= 1<<20:
+ return fmt.Sprintf("%.3f MiB", b/(1<<20))
+ case c >= 1<<10:
+ return fmt.Sprintf("%.3f KiB", b/(1<<10))
+ default:
+ return fmt.Sprintf("%d B", c)
+ }
+}
+
+// FormatPercent formats numerator/denominator as a percentage.
+func FormatPercent(numerator uint64, denominator uint64) string {
+ if denominator == 0 {
+ return ""
+ }
+
+ percent := 100.0 * float64(numerator) / float64(denominator)
+ if percent > 100 {
+ percent = 100
+ }
+
+ return fmt.Sprintf("%3.2f%%", percent)
+}
+
+// FormatDuration formats d as FormatSeconds would.
+func FormatDuration(d time.Duration) string {
+ sec := uint64(d / time.Second)
+ return FormatSeconds(sec)
+}
+
+// FormatSeconds formats sec as MM:SS, or HH:MM:SS if sec seconds
+// is at least an hour.
+func FormatSeconds(sec uint64) string {
+ hours := sec / 3600
+ sec -= hours * 3600
+ min := sec / 60
+ sec -= min * 60
+ if hours > 0 {
+ return fmt.Sprintf("%d:%02d:%02d", hours, min, sec)
+ }
+ return fmt.Sprintf("%d:%02d", min, sec)
+}
diff --git a/internal/ui/format_test.go b/internal/ui/format_test.go
new file mode 100644
index 000000000..b6a1c13d1
--- /dev/null
+++ b/internal/ui/format_test.go
@@ -0,0 +1,38 @@
+package ui
+
+import "testing"
+
+func TestFormatBytes(t *testing.T) {
+ for _, c := range []struct {
+ size uint64
+ want string
+ }{
+ {0, "0 B"},
+ {1023, "1023 B"},
+ {1024, "1.000 KiB"},
+ {5<<20 + 1<<19, "5.500 MiB"},
+ {1 << 30, "1.000 GiB"},
+ {2 << 30, "2.000 GiB"},
+ {1<<40 - 1<<36, "960.000 GiB"},
+ {1 << 40, "1.000 TiB"},
+ } {
+ if got := FormatBytes(c.size); got != c.want {
+ t.Errorf("want %q, got %q", c.want, got)
+ }
+ }
+}
+
+func TestFormatPercent(t *testing.T) {
+ for _, c := range []struct {
+ num, denom uint64
+ want string
+ }{
+ {0, 5, "0.00%"},
+ {3, 7, "42.86%"},
+ {99, 99, "100.00%"},
+ } {
+ if got := FormatPercent(c.num, c.denom); got != c.want {
+ t.Errorf("want %q, got %q", c.want, got)
+ }
+ }
+}
diff --git a/internal/ui/progress/counter.go b/internal/ui/progress/counter.go
index d2f75c9bf..c1275d2f2 100644
--- a/internal/ui/progress/counter.go
+++ b/internal/ui/progress/counter.go
@@ -3,9 +3,6 @@ package progress
import (
"sync"
"time"
-
- "github.com/restic/restic/internal/debug"
- "github.com/restic/restic/internal/ui/signals"
)
// A Func is a callback for a Counter.
@@ -19,32 +16,22 @@ type Func func(value uint64, total uint64, runtime time.Duration, final bool)
//
// The Func is also called when SIGUSR1 (or SIGINFO, on BSD) is received.
type Counter struct {
- report Func
- start time.Time
- stopped chan struct{} // Closed by run.
- stop chan struct{} // Close to stop run.
- tick *time.Ticker
+ Updater
valueMutex sync.Mutex
value uint64
max uint64
}
-// New starts a new Counter.
-func New(interval time.Duration, total uint64, report Func) *Counter {
+// NewCounter starts a new Counter.
+func NewCounter(interval time.Duration, total uint64, report Func) *Counter {
c := &Counter{
- report: report,
- start: time.Now(),
- stopped: make(chan struct{}),
- stop: make(chan struct{}),
- max: total,
- }
-
- if interval > 0 {
- c.tick = time.NewTicker(interval)
+ max: total,
}
-
- go c.run()
+ c.Updater = *NewUpdater(interval, func(runtime time.Duration, final bool) {
+ v, max := c.Get()
+ report(v, max, runtime, final)
+ })
return c
}
@@ -69,60 +56,18 @@ func (c *Counter) SetMax(max uint64) {
c.valueMutex.Unlock()
}
-// Done tells a Counter to stop and waits for it to report its final value.
-func (c *Counter) Done() {
- if c == nil {
- return
- }
- if c.tick != nil {
- c.tick.Stop()
- }
- close(c.stop)
- <-c.stopped // Wait for last progress report.
- *c = Counter{} // Prevent reuse.
-}
-
-// Get the current Counter value. This method is concurrency-safe.
-func (c *Counter) Get() uint64 {
+// Get returns the current value and the maximum of c.
+// This method is concurrency-safe.
+func (c *Counter) Get() (v, max uint64) {
c.valueMutex.Lock()
- v := c.value
+ v, max = c.value, c.max
c.valueMutex.Unlock()
- return v
+ return v, max
}
-func (c *Counter) getMax() uint64 {
- c.valueMutex.Lock()
- max := c.max
- c.valueMutex.Unlock()
-
- return max
-}
-
-func (c *Counter) run() {
- defer close(c.stopped)
- defer func() {
- // Must be a func so that time.Since isn't called at defer time.
- c.report(c.Get(), c.getMax(), time.Since(c.start), true)
- }()
-
- var tick <-chan time.Time
- if c.tick != nil {
- tick = c.tick.C
- }
- signalsCh := signals.GetProgressChannel()
- for {
- var now time.Time
-
- select {
- case now = <-tick:
- case sig := <-signalsCh:
- debug.Log("Signal received: %v\n", sig)
- now = time.Now()
- case <-c.stop:
- return
- }
-
- c.report(c.Get(), c.getMax(), now.Sub(c.start), false)
+func (c *Counter) Done() {
+ if c != nil {
+ c.Updater.Done()
}
}
diff --git a/internal/ui/progress/counter_test.go b/internal/ui/progress/counter_test.go
index 85695d209..49c694e06 100644
--- a/internal/ui/progress/counter_test.go
+++ b/internal/ui/progress/counter_test.go
@@ -35,7 +35,7 @@ func TestCounter(t *testing.T) {
lastTotal = total
ncalls++
}
- c := progress.New(10*time.Millisecond, startTotal, report)
+ c := progress.NewCounter(10*time.Millisecond, startTotal, report)
done := make(chan struct{})
go func() {
@@ -63,24 +63,6 @@ func TestCounterNil(t *testing.T) {
// Shouldn't panic.
var c *progress.Counter
c.Add(1)
+ c.SetMax(42)
c.Done()
}
-
-func TestCounterNoTick(t *testing.T) {
- finalSeen := false
- otherSeen := false
-
- report := func(value, total uint64, d time.Duration, final bool) {
- if final {
- finalSeen = true
- } else {
- otherSeen = true
- }
- }
- c := progress.New(0, 1, report)
- time.Sleep(time.Millisecond)
- c.Done()
-
- test.Assert(t, finalSeen, "final call did not happen")
- test.Assert(t, !otherSeen, "unexpected status update")
-}
diff --git a/internal/ui/progress/updater.go b/internal/ui/progress/updater.go
new file mode 100644
index 000000000..7fb6c8836
--- /dev/null
+++ b/internal/ui/progress/updater.go
@@ -0,0 +1,84 @@
+package progress
+
+import (
+ "time"
+
+ "github.com/restic/restic/internal/debug"
+ "github.com/restic/restic/internal/ui/signals"
+)
+
+// An UpdateFunc is a callback for a (progress) Updater.
+//
+// The final argument is true if Updater.Done has been called,
+// which means that the current call will be the last.
+type UpdateFunc func(runtime time.Duration, final bool)
+
+// An Updater controls a goroutine that periodically calls an UpdateFunc.
+//
+// The UpdateFunc is also called when SIGUSR1 (or SIGINFO, on BSD) is received.
+type Updater struct {
+ report UpdateFunc
+ start time.Time
+ stopped chan struct{} // Closed by run.
+ stop chan struct{} // Close to stop run.
+ tick *time.Ticker
+}
+
+// NewUpdater starts a new Updater.
+func NewUpdater(interval time.Duration, report UpdateFunc) *Updater {
+ c := &Updater{
+ report: report,
+ start: time.Now(),
+ stopped: make(chan struct{}),
+ stop: make(chan struct{}),
+ }
+
+ if interval > 0 {
+ c.tick = time.NewTicker(interval)
+ }
+
+ go c.run()
+ return c
+}
+
+// Done tells an Updater to stop and waits for it to report its final value.
+// Later calls do nothing.
+func (c *Updater) Done() {
+ if c == nil || c.stop == nil {
+ return
+ }
+ if c.tick != nil {
+ c.tick.Stop()
+ }
+ close(c.stop)
+ <-c.stopped // Wait for last progress report.
+ c.stop = nil
+}
+
+func (c *Updater) run() {
+ defer close(c.stopped)
+ defer func() {
+ // Must be a func so that time.Since isn't called at defer time.
+ c.report(time.Since(c.start), true)
+ }()
+
+ var tick <-chan time.Time
+ if c.tick != nil {
+ tick = c.tick.C
+ }
+ signalsCh := signals.GetProgressChannel()
+ for {
+ var now time.Time
+
+ select {
+ case now = <-tick:
+ case sig := <-signalsCh:
+ debug.Log("Signal received: %v\n", sig)
+ now = time.Now()
+ case <-c.stop:
+ return
+ }
+
+ c.report(now.Sub(c.start), false)
+ }
+}
diff --git a/internal/ui/progress/updater_test.go b/internal/ui/progress/updater_test.go
new file mode 100644
index 000000000..5b5207dd5
--- /dev/null
+++ b/internal/ui/progress/updater_test.go
@@ -0,0 +1,52 @@
+package progress_test
+
+import (
+ "testing"
+ "time"
+
+ "github.com/restic/restic/internal/test"
+ "github.com/restic/restic/internal/ui/progress"
+)
+
+func TestUpdater(t *testing.T) {
+ finalSeen := false
+ var ncalls int
+
+ report := func(d time.Duration, final bool) {
+ if final {
+ finalSeen = true
+ }
+ ncalls++
+ }
+ c := progress.NewUpdater(10*time.Millisecond, report)
+ time.Sleep(100 * time.Millisecond)
+ c.Done()
+
+ test.Assert(t, finalSeen, "final call did not happen")
+ test.Assert(t, ncalls > 0, "no progress was reported")
+}
+
+func TestUpdaterStopTwice(t *testing.T) {
+ c := progress.NewUpdater(0, func(runtime time.Duration, final bool) {})
+ c.Done()
+ c.Done()
+}
+
+func TestUpdaterNoTick(t *testing.T) {
+ finalSeen := false
+ otherSeen := false
+
+ report := func(d time.Duration, final bool) {
+ if final {
+ finalSeen = true
+ } else {
+ otherSeen = true
+ }
+ }
+ c := progress.NewUpdater(0, report)
+ time.Sleep(time.Millisecond)
+ c.Done()
+
+ test.Assert(t, finalSeen, "final call did not happen")
+ test.Assert(t, !otherSeen, "unexpected status update")
+}
diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go
index 88c4f898e..fdc7e14f6 100644
--- a/internal/ui/termstatus/status.go
+++ b/internal/ui/termstatus/status.go
@@ -25,6 +25,7 @@ type Terminal struct {
msg chan message
status chan status
canUpdateStatus bool
+ lastStatusLen int
// will be closed when the goroutine which runs Run() terminates, so it'll
// yield a default value immediately
@@ -154,6 +155,18 @@ func (t *Terminal) run(ctx context.Context) {
}
func (t *Terminal) writeStatus(status []string) {
+ statusLen := len(status)
+ status = append([]string{}, status...)
+ for i := len(status); i < t.lastStatusLen; i++ {
+ // clear no longer used status lines
+ status = append(status, "")
+ if i > 0 {
+ // all lines except the last one must have a line break
+ status[i-1] = status[i-1] + "\n"
+ }
+ }
+ t.lastStatusLen = statusLen
+
for _, line := range status {
t.clearCurrentLine(t.wr, t.fd)
diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go
new file mode 100644
index 000000000..6f063831e
--- /dev/null
+++ b/internal/walker/rewriter.go
@@ -0,0 +1,91 @@
+package walker
+
+import (
+ "context"
+ "fmt"
+ "path"
+
+ "github.com/restic/restic/internal/debug"
+ "github.com/restic/restic/internal/restic"
+)
+
+// SelectByNameFunc returns true for all items that should be included (files and
+// dirs). If false is returned, files are ignored and dirs are not even walked.
+type SelectByNameFunc func(item string) bool
+
+type TreeFilterVisitor struct {
+ SelectByName SelectByNameFunc
+ PrintExclude func(string)
+}
+
+type BlobLoadSaver interface {
+ restic.BlobSaver
+ restic.BlobLoader
+}
+
+func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID restic.ID, visitor *TreeFilterVisitor) (newNodeID restic.ID, err error) {
+ curTree, err := restic.LoadTree(ctx, repo, nodeID)
+ if err != nil {
+ return restic.ID{}, err
+ }
+
+ // check that we can properly encode this tree without losing information
+ // The alternative of using json/Decoder.DisallowUnknownFields() doesn't work as we use
+ // a custom UnmarshalJSON to decode trees, see also https://github.com/golang/go/issues/41144
+ testID, err := restic.SaveTree(ctx, repo, curTree)
+ if err != nil {
+ return restic.ID{}, err
+ }
+ if nodeID != testID {
+ return restic.ID{}, fmt.Errorf("cannot encode tree at %q without loosing information", nodepath)
+ }
+
+ debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str())
+
+ changed := false
+ tb := restic.NewTreeJSONBuilder()
+ for _, node := range curTree.Nodes {
+ path := path.Join(nodepath, node.Name)
+ if !visitor.SelectByName(path) {
+ if visitor.PrintExclude != nil {
+ visitor.PrintExclude(path)
+ }
+ changed = true
+ continue
+ }
+
+ if node.Subtree == nil {
+ err = tb.AddNode(node)
+ if err != nil {
+ return restic.ID{}, err
+ }
+ continue
+ }
+ newID, err := FilterTree(ctx, repo, path, *node.Subtree, visitor)
+ if err != nil {
+ return restic.ID{}, err
+ }
+ if !node.Subtree.Equal(newID) {
+ changed = true
+ }
+ node.Subtree = &newID
+ err = tb.AddNode(node)
+ if err != nil {
+ return restic.ID{}, err
+ }
+ }
+
+ if changed {
+ tree, err := tb.Finalize()
+ if err != nil {
+ return restic.ID{}, err
+ }
+
+ // Save new tree
+ newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false)
+ debug.Log("filterTree: save new tree for %s as %v\n", nodepath, newTreeID)
+ return newTreeID, err
+ }
+
+ return nodeID, nil
+}
diff --git a/internal/walker/rewriter_test.go b/internal/walker/rewriter_test.go
new file mode 100644
index 000000000..3dcf0ac9e
--- /dev/null
+++ b/internal/walker/rewriter_test.go
@@ -0,0 +1,222 @@
+package walker
+
+import (
+ "context"
+ "fmt"
+ "testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/pkg/errors"
+ "github.com/restic/restic/internal/restic"
+)
+
+// WritableTreeMap also support saving
+type WritableTreeMap struct {
+ TreeMap
+}
+
+func (t WritableTreeMap) SaveBlob(ctx context.Context, tpe restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, size int, err error) {
+ if tpe != restic.TreeBlob {
+ return restic.ID{}, false, 0, errors.New("can only save trees")
+ }
+
+ if id.IsNull() {
+ id = restic.Hash(buf)
+ }
+ _, ok := t.TreeMap[id]
+ if ok {
+ return id, false, 0, nil
+ }
+
+ t.TreeMap[id] = append([]byte{}, buf...)
+ return id, true, len(buf), nil
+}
+
+func (t WritableTreeMap) Dump() {
+ for k, v := range t.TreeMap {
+ fmt.Printf("%v: %v", k, string(v))
+ }
+}
+
+type checkRewriteFunc func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB))
+
+// checkRewriteItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'.
+func checkRewriteItemOrder(want []string) checkRewriteFunc {
+ pos := 0
+ return func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) {
+ vis := TreeFilterVisitor{
+ SelectByName: func(path string) bool {
+ if pos >= len(want) {
+ t.Errorf("additional unexpected path found: %v", path)
+ return false
+ }
+
+ if path != want[pos] {
+ t.Errorf("wrong path found, want %q, got %q", want[pos], path)
+ }
+ pos++
+ return true
+ },
+ }
+
+ final = func(t testing.TB) {
+ if pos != len(want) {
+ t.Errorf("not enough items returned, want %d, got %d", len(want), pos)
+ }
+ }
+
+ return vis, final
+ }
+}
+
+// checkRewriteSkips excludes nodes if path is in skipFor, it checks that all excluded entries are printed.
+func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteFunc {
+ var pos int
+ printed := make(map[string]struct{})
+
+ return func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) {
+ vis := TreeFilterVisitor{
+ SelectByName: func(path string) bool {
+ if pos >= len(want) {
+ t.Errorf("additional unexpected path found: %v", path)
+ return false
+ }
+
+ if path != want[pos] {
+ t.Errorf("wrong path found, want %q, got %q", want[pos], path)
+ }
+ pos++
+
+ _, ok := skipFor[path]
+ return !ok
+ },
+ PrintExclude: func(s string) {
+ if _, ok := printed[s]; ok {
+ t.Errorf("path was already printed %v", s)
+ }
+ printed[s] = struct{}{}
+ },
+ }
+
+ final = func(t testing.TB) {
+ if !cmp.Equal(skipFor, printed) {
+ t.Errorf("unexpected paths skipped: %s", cmp.Diff(skipFor, printed))
+ }
+ if pos != len(want) {
+ t.Errorf("not enough items returned, want %d, got %d", len(want), pos)
+ }
+ }
+
+ return vis, final
+ }
+}
+
+func TestRewriter(t *testing.T) {
+ var tests = []struct {
+ tree TestTree
+ newTree TestTree
+ check checkRewriteFunc
+ }{
+ { // don't change
+ tree: TestTree{
+ "foo": TestFile{},
+ "subdir": TestTree{
+ "subfile": TestFile{},
+ },
+ },
+ check: checkRewriteItemOrder([]string{
+ "/foo",
+ "/subdir",
+ "/subdir/subfile",
+ }),
+ },
+ { // exclude file
+ tree: TestTree{
+ "foo": TestFile{},
+ "subdir": TestTree{
+ "subfile": TestFile{},
+ },
+ },
+ newTree: TestTree{
+ "foo": TestFile{},
+ "subdir": TestTree{},
+ },
+ check: checkRewriteSkips(
+ map[string]struct{}{
+ "/subdir/subfile": {},
+ },
+ []string{
+ "/foo",
+ "/subdir",
+ "/subdir/subfile",
+ },
+ ),
+ },
+ { // exclude dir
+ tree: TestTree{
+ "foo": TestFile{},
+ "subdir": TestTree{
+ "subfile": TestFile{},
+ },
+ },
+ newTree: TestTree{
+ "foo": TestFile{},
+ },
+ check: checkRewriteSkips(
+ map[string]struct{}{
+ "/subdir": {},
+ },
+ []string{
+ "/foo",
+ "/subdir",
+ },
+ ),
+ },
+ }
+
+ for _, test := range tests {
+ t.Run("", func(t *testing.T) {
+ repo, root := BuildTreeMap(test.tree)
+ if test.newTree == nil {
+ test.newTree = test.tree
+ }
+ expRepo, expRoot := BuildTreeMap(test.newTree)
+ modrepo := WritableTreeMap{repo}
+
+ ctx, cancel := context.WithCancel(context.TODO())
+ defer cancel()
+
+ vis, last := test.check(t)
+ newRoot, err := FilterTree(ctx, modrepo, "/", root, &vis)
+ if err != nil {
+ t.Error(err)
+ }
+ last(t)
+
+ // verifying against the expected tree root also implicitly checks the structural integrity
+ if newRoot != expRoot {
+ t.Error("hash mismatch")
+ fmt.Println("Got")
+ modrepo.Dump()
+ fmt.Println("Expected")
+ WritableTreeMap{expRepo}.Dump()
+ }
+ })
+ }
+}
+
+func TestRewriterFailOnUnknownFields(t *testing.T) {
+ tm := WritableTreeMap{TreeMap{}}
+ node := []byte(`{"nodes":[{"name":"subfile","type":"file","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","uid":0,"gid":0,"content":null,"unknown_field":42}]}`)
+ id := restic.Hash(node)
+ tm.TreeMap[id] = node
+
+ ctx, cancel := context.WithCancel(context.TODO())
+ defer cancel()
+ // use nil visitor to crash if the tree loading works unexpectedly
+ _, err := FilterTree(ctx, tm, "/", id, nil)
+
+ if err == nil {
+ t.Error("missing error on unknown field")
+ }
+}
diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go
index 90ca7c2b8..6c4fd3436 100644
--- a/internal/walker/walker_test.go
+++ b/internal/walker/walker_test.go
@@ -2,8 +2,8 @@ package walker
import (
"context"
- "encoding/json"
"fmt"
+ "sort"
"testing"
"github.com/pkg/errors"
@@ -23,12 +23,18 @@ func BuildTreeMap(tree TestTree) (m TreeMap, root restic.ID) {
}
func buildTreeMap(tree TestTree, m TreeMap) restic.ID {
- res := restic.NewTree(0)
+ tb := restic.NewTreeJSONBuilder()
+ var names []string
+ for name := range tree {
+ names = append(names, name)
+ }
+ sort.Strings(names)
- for name, item := range tree {
+ for _, name := range names {
+ item := tree[name]
switch elem := item.(type) {
case TestFile:
- err := res.Insert(&restic.Node{
+ err := tb.AddNode(&restic.Node{
Name: name,
Type: "file",
})
@@ -37,7 +43,7 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID {
}
case TestTree:
id := buildTreeMap(elem, m)
- err := res.Insert(&restic.Node{
+ err := tb.AddNode(&restic.Node{
Name: name,
Subtree: &id,
Type: "dir",
@@ -50,7 +56,7 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID {
}
}
- buf, err := json.Marshal(res)
+ buf, err := tb.Finalize()
if err != nil {
panic(err)
}
@@ -58,14 +64,14 @@ func buildTreeMap(tree TestTree, m TreeMap) restic.ID {
id := restic.Hash(buf)
if _, ok := m[id]; !ok {
- m[id] = res
+ m[id] = buf
}
return id
}
// TreeMap returns the trees from the map on LoadTree.
-type TreeMap map[restic.ID]*restic.Tree
+type TreeMap map[restic.ID][]byte
func (t TreeMap) LoadBlob(ctx context.Context, tpe restic.BlobType, id restic.ID, buf []byte) ([]byte, error) {
if tpe != restic.TreeBlob {
@@ -75,14 +81,7 @@ func (t TreeMap) LoadBlob(ctx context.Context, tpe restic.BlobType, id restic.ID
if !ok {
return nil, errors.New("tree not found")
}
-
- tbuf, err := json.Marshal(tree)
- if err != nil {
- panic(err)
- }
- tbuf = append(tbuf, '\n')
-
- return tbuf, nil
+ return tree, nil
}
func (t TreeMap) Connections() uint {
@@ -256,10 +255,10 @@ func TestWalker(t *testing.T) {
"/subdir/subfile",
}),
checkParentTreeOrder([]string{
- "2593e9dba52232c043d68c40d0f9c236b4448e37224941298ea6e223ca1e3a1b", // tree /
- "2593e9dba52232c043d68c40d0f9c236b4448e37224941298ea6e223ca1e3a1b", // tree /
- "2593e9dba52232c043d68c40d0f9c236b4448e37224941298ea6e223ca1e3a1b", // tree /
- "a7f5be55bdd94db9df706a428e0726a4044720c9c94b9ebeb81000debe032087", // tree /subdir
+ "a760536a8fd64dd63f8dd95d85d788d71fd1bee6828619350daf6959dcb499a0", // tree /
+ "a760536a8fd64dd63f8dd95d85d788d71fd1bee6828619350daf6959dcb499a0", // tree /
+ "a760536a8fd64dd63f8dd95d85d788d71fd1bee6828619350daf6959dcb499a0", // tree /
+ "670046b44353a89b7cd6ef84c78422232438f10eb225c29c07989ae05283d797", // tree /subdir
}),
checkSkipFor(
map[string]struct{}{
@@ -307,14 +306,14 @@ func TestWalker(t *testing.T) {
"/subdir2/subsubdir2/subsubfile3",
}),
checkParentTreeOrder([]string{
- "31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree /
- "31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree /
- "31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree /
- "af838dc7a83d353f0273c33d93fcdba3220d4517576f09694a971dd23b8e94dc", // tree /subdir1
- "31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree /
- "fb749ba6ae01a3814bed9b59d74af8d7593d3074a681d4112c4983d461089e5b", // tree /subdir2
- "fb749ba6ae01a3814bed9b59d74af8d7593d3074a681d4112c4983d461089e5b", // tree /subdir2
- "eb8dd587a9c5e6be87b69d2c5264a19622f75bf6704927aaebaee78d0992531d", // tree /subdir2/subsubdir2
+ "7a0e59b986cc83167d9fbeeefc54e4629770124c5825d391f7ee0598667fcdf1", // tree /
+ "7a0e59b986cc83167d9fbeeefc54e4629770124c5825d391f7ee0598667fcdf1", // tree /
+ "7a0e59b986cc83167d9fbeeefc54e4629770124c5825d391f7ee0598667fcdf1", // tree /
+ "22c9feefa0b9fabc7ec5383c90cfe84ba714babbe4d2968fcb78f0ec7612e82f", // tree /subdir1
+ "7a0e59b986cc83167d9fbeeefc54e4629770124c5825d391f7ee0598667fcdf1", // tree /
+ "9bfe4aab3ac0ad7a81909355d7221801441fb20f7ed06c0142196b3f10358493", // tree /subdir2
+ "9bfe4aab3ac0ad7a81909355d7221801441fb20f7ed06c0142196b3f10358493", // tree /subdir2
+ "6b962fef064ef9beecc27dfcd6e0f2e7beeebc9c1f1f4f477d4af59fc45f411d", // tree /subdir2/subsubdir2
}),
checkSkipFor(
map[string]struct{}{
@@ -391,21 +390,21 @@ func TestWalker(t *testing.T) {
"/zzz other",
}),
checkParentTreeOrder([]string{
- "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
- "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
- "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
- "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir1
- "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir1
- "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir1
- "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
- "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir2
- "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir2
- "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir2
- "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
- "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir3
- "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir3
- "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir3
- "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree /
+ "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree /
+ "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree /
+ "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree /
+ "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir1
+ "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir1
+ "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir1
+ "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree /
+ "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir2
+ "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir2
+ "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir2
+ "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree /
+ "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir3
+ "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir3
+ "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir3
+ "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree /
}),
checkIgnore(
map[string]struct{}{