aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFélix Sipma <felix+debian@gueux.org>2017-04-25 18:58:58 +0200
committerFélix Sipma <felix+debian@gueux.org>2017-04-25 18:58:58 +0200
commit16c6fc9dbf3d0882f8d16d8870cd0c983a6fafcb (patch)
treee5833207acf5f451c0a0def196baa9b94ca21044
parent183ce67b7520d0e56e4a78365addb67d2385d843 (diff)
parenta7e2efaabd25960faf9666640ec826f3bb9b2773 (diff)
Updated version 0.5.0 from 'upstream/0.5.0'
with Debian dir da377b1713dc40027fb2d3968e7e314d6468a476
-rw-r--r--.travis.yml11
-rw-r--r--CONTRIBUTING.md2
-rw-r--r--Dockerfile4
-rw-r--r--README.md24
-rw-r--r--VERSION2
-rw-r--r--Vagrantfile2
-rw-r--r--appveyor.yml4
-rw-r--r--build.go6
-rwxr-xr-xbuild_release_binaries.sh4
-rw-r--r--doc/Design.md38
-rw-r--r--doc/Manual.md150
-rw-r--r--src/cmds/restic/cmd_backup.go88
-rw-r--r--src/cmds/restic/cmd_cat.go2
-rw-r--r--src/cmds/restic/cmd_check.go11
-rw-r--r--src/cmds/restic/cmd_dump.go4
-rw-r--r--src/cmds/restic/cmd_find.go128
-rw-r--r--src/cmds/restic/cmd_forget.go203
-rw-r--r--src/cmds/restic/cmd_key.go24
-rw-r--r--src/cmds/restic/cmd_list.go6
-rw-r--r--src/cmds/restic/cmd_ls.go78
-rw-r--r--src/cmds/restic/cmd_mount.go36
-rw-r--r--src/cmds/restic/cmd_prune.go71
-rw-r--r--src/cmds/restic/cmd_rebuild_index.go54
-rw-r--r--src/cmds/restic/cmd_restore.go6
-rw-r--r--src/cmds/restic/cmd_snapshots.go104
-rw-r--r--src/cmds/restic/cmd_tag.go142
-rw-r--r--src/cmds/restic/cmd_unlock.go2
-rw-r--r--src/cmds/restic/cmd_version.go2
-rw-r--r--src/cmds/restic/find.go78
-rw-r--r--src/cmds/restic/format.go24
-rw-r--r--src/cmds/restic/global.go22
-rw-r--r--src/cmds/restic/integration_helpers_test.go4
-rw-r--r--src/cmds/restic/integration_helpers_unix_test.go34
-rw-r--r--src/cmds/restic/integration_helpers_windows_test.go22
-rw-r--r--src/cmds/restic/integration_test.go239
-rw-r--r--src/cmds/restic/testdata/test.hl.tar.gzbin0 -> 198 bytes
-rw-r--r--src/restic/archiver/archive_reader.go22
-rw-r--r--src/restic/archiver/archive_reader_test.go92
-rw-r--r--src/restic/archiver/archiver.go48
-rw-r--r--src/restic/archiver/archiver_test.go24
-rw-r--r--src/restic/archiver/testing.go2
-rw-r--r--src/restic/backend/rest/rest.go2
-rw-r--r--src/restic/backend/s3/s3.go49
-rw-r--r--src/restic/backend/sftp/sftp.go4
-rw-r--r--src/restic/backend/test/tests.go4
-rw-r--r--src/restic/blob.go5
-rw-r--r--src/restic/checker/checker.go94
-rw-r--r--src/restic/checker/checker_test.go69
-rw-r--r--src/restic/errors/wrap.go4
-rw-r--r--src/restic/fs/doc.go2
-rw-r--r--src/restic/fs/file.go6
-rw-r--r--src/restic/fuse/dir.go34
-rw-r--r--src/restic/fuse/file.go21
-rw-r--r--src/restic/fuse/link.go3
-rw-r--r--src/restic/fuse/snapshot.go17
-rw-r--r--src/restic/hardlinks_index.go57
-rw-r--r--src/restic/hardlinks_index_test.go35
-rw-r--r--src/restic/id.go2
-rw-r--r--src/restic/list/list.go2
-rw-r--r--src/restic/lock.go2
-rw-r--r--src/restic/node.go184
-rw-r--r--src/restic/node_openbsd.go16
-rw-r--r--src/restic/node_test.go4
-rw-r--r--src/restic/node_windows.go17
-rw-r--r--src/restic/node_xattr.go39
-rw-r--r--src/restic/repository/index.go2
-rw-r--r--src/restic/repository/index_rebuild.go66
-rw-r--r--src/restic/repository/repack.go11
-rw-r--r--src/restic/repository/repack_test.go23
-rw-r--r--src/restic/repository/repository.go60
-rw-r--r--src/restic/restorer.go19
-rw-r--r--src/restic/snapshot.go98
-rw-r--r--src/restic/snapshot_test.go2
-rw-r--r--src/restic/testing.go2
-rw-r--r--src/restic/tree_test.go2
-rw-r--r--src/restic/walk/walk_test.go2
-rw-r--r--vendor/manifest6
77 files changed, 2085 insertions, 699 deletions
diff --git a/.travis.yml b/.travis.yml
index a70d48d..5704231 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,8 +2,8 @@ language: go
sudo: false
go:
- - 1.6.4
- - 1.7.4
+ - 1.7.5
+ - 1.8
- tip
os:
@@ -17,14 +17,14 @@ env:
matrix:
exclude:
- os: osx
- go: 1.6.4
+ go: 1.7.5
- os: osx
go: tip
- os: linux
- go: 1.7.4
+ go: 1.8
include:
- os: linux
- go: 1.7.4
+ go: 1.8
sudo: true
env:
RESTIC_TEST_FUSE=1
@@ -48,7 +48,6 @@ install:
- export GOBIN="$GOPATH/bin"
- export PATH="$PATH:$GOBIN"
- go env
- - ulimit -n 2048
script:
- go run run_integration_tests.go
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 5529ea7..5ffdc40 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -77,7 +77,7 @@ Just clone the repository, `cd` to it and run `gb build` to build the binary:
[...]
$ bin/restic version
restic compiled manually
- compiled at unknown time with go1.6
+ compiled at unknown time with go1.7
The following commands can be used to run all the tests:
diff --git a/Dockerfile b/Dockerfile
index 3fea9ab..0f0a837 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,11 +14,11 @@
# docker run --rm -v $PWD:/home/travis/restic restic/test gb test -v ./backend
#
# build the image for an older version of Go:
-# docker build --build-arg GOVERSION=1.3.3 -t restic/test:go1.3.3 .
+# docker build --build-arg GOVERSION=1.6.4 -t restic/test:go1.6.4 .
FROM ubuntu:14.04
-ARG GOVERSION=1.7
+ARG GOVERSION=1.7.5
ARG GOARCH=amd64
# install dependencies
diff --git a/README.md b/README.md
index 8645935..6981158 100644
--- a/README.md
+++ b/README.md
@@ -19,10 +19,22 @@ The latest documentation can be viewed online at
a menu that allows switching to the documentation and user manual for the
latest released version.
+News
+====
+
+You can follow the restic project on Twitter
+[@resticbackup](https://twitter.com/resticbackup) or by subscribing to the
+[development blog](https://restic.github.io/blog/).
+
+Install restic
+==============
+
+You can download the latest pre-compiled binary from the [restic release page](https://github.com/restic/restic/releases/latest).
+
Build restic
============
-Install Go/Golang (at least version 1.6), then run `go run build.go`,
+Install Go/Golang (at least version 1.7), then run `go run build.go`,
afterwards you'll find the binary in the current directory:
$ go run build.go
@@ -32,6 +44,16 @@ afterwards you'll find the binary in the current directory:
restic [OPTIONS] <command>
[...]
+You can easily cross-compile restic for all supported platforms, just supply
+the target OS and platform via the command-line options like this (for Windows
+and FreeBSD respectively):
+
+ $ go run build.go --goos windows --goarch amd64
+
+ $ go run build.go --goos freebsd --goarch 386
+
+The resulting binary is statically linked and does not require any libraries.
+
More documentation can be found in the [user manual](doc/Manual.md).
At the moment, the only tested compiler for restic is the official Go compiler.
diff --git a/VERSION b/VERSION
index 1d0ba9e..8f0916f 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.4.0
+0.5.0
diff --git a/Vagrantfile b/Vagrantfile
index a26aa6b..ee872b3 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -1,7 +1,7 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
-GO_VERSION = '1.6'
+GO_VERSION = '1.7'
def packages_freebsd
return <<-EOF
diff --git a/appveyor.yml b/appveyor.yml
index 49bbbaf..435f4ef 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -17,8 +17,8 @@ init:
install:
- rmdir c:\go /s /q
- - appveyor DownloadFile https://storage.googleapis.com/golang/go1.7.4.windows-amd64.msi
- - msiexec /i go1.7.4.windows-amd64.msi /q
+ - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.windows-amd64.msi
+ - msiexec /i go1.8.windows-amd64.msi /q
- go version
- go env
- appveyor DownloadFile http://sourceforge.netcologne.de/project/gnuwin32/tar/1.13-1/tar-1.13-1-bin.zip -FileName tar.zip
diff --git a/build.go b/build.go
index ec55660..8a041df 100644
--- a/build.go
+++ b/build.go
@@ -291,6 +291,12 @@ func (cs Constants) LDFlags() string {
}
func main() {
+ ver := runtime.Version()
+ if strings.HasPrefix(ver, "go1") && ver < "go1.7" {
+ fmt.Fprintf(os.Stderr, "Go version %s detected, restic requires at least Go 1.7\n", ver)
+ os.Exit(1)
+ }
+
buildTags := []string{}
skipNext := false
diff --git a/build_release_binaries.sh b/build_release_binaries.sh
index dced596..df86752 100755
--- a/build_release_binaries.sh
+++ b/build_release_binaries.sh
@@ -36,7 +36,7 @@ for R in \
echo $filename
- go run ../build.go --goos $OS --goarch $ARCH --output ${filename}
+ go run build.go --goos $OS --goarch $ARCH --output ${filename}
if [[ "$OS" == "windows" ]]; then
zip ${filename%.exe}.zip ${filename}
rm ${filename}
@@ -53,7 +53,7 @@ mv restic-$VERSION.tar.gz ${dir}
echo "creating checksums"
pushd ${dir}
-sha256sum restic_*.{zip,bz2} > SHA256SUMS
+sha256sum restic_*.{zip,bz2} restic-$VERSION.tar.gz > SHA256SUMS
gpg --armor --detach-sign SHA256SUMS
popd
diff --git a/doc/Design.md b/doc/Design.md
index 117554d..52a228a 100644
--- a/doc/Design.md
+++ b/doc/Design.md
@@ -285,7 +285,7 @@ This way, the password can be changed without having to re-encrypt all data.
Snapshots
---------
-A snapshots represents a directory with all files and sub-directories at a
+A snapshot represents a directory with all files and sub-directories at a
given point in time. For each backup that is made, a new snapshot is created. A
snapshot is a JSON document that is stored in an encrypted file below the
directory `snapshots` in the repository. The filename is the storage ID. This
@@ -295,7 +295,7 @@ The command `restic cat snapshot` can be used as follows to decrypt and
pretty-print the contents of a snapshot file:
```console
-$ restic -r /tmp/restic-repo cat snapshot 22a5af1b
+$ restic -r /tmp/restic-repo cat snapshot 251c2e58
enter password for repository:
{
"time": "2015-01-02T18:10:50.895208559+01:00",
@@ -304,12 +304,42 @@ enter password for repository:
"hostname": "kasimir",
"username": "fd0",
"uid": 1000,
- "gid": 100
+ "gid": 100,
+ "tags": [
+ "NL"
+ ]
}
```
Here it can be seen that this snapshot represents the contents of the directory
-`/tmp/testdata`. The most important field is `tree`.
+`/tmp/testdata`. The most important field is `tree`. When the meta data (e.g.
+the tags) of a snapshot change, the snapshot needs to be re-encrypted and saved.
+This will change the storage ID, so in order to relate these seemingly
+different snapshots, a field `original` is introduced which contains the ID of
+the original snapshot, e.g. after adding the tag `DE` to the snapshot above it
+becomes:
+
+```console
+$ restic -r /tmp/restic-repo cat snapshot 22a5af1b
+enter password for repository:
+{
+ "time": "2015-01-02T18:10:50.895208559+01:00",
+ "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf",
+ "dir": "/tmp/testdata",
+ "hostname": "kasimir",
+ "username": "fd0",
+ "uid": 1000,
+ "gid": 100,
+ "tags": [
+ "NL",
+ "DE"
+ ],
+ "original": "251c2e5841355f743f9d4ffd3260bee765acee40a6229857e32b60446991b837"
+}
+```
+
+Once introduced, the `original` field is not modified when the snapshot's meta
+data is changed again.
All content within a restic repository is referenced according to its SHA-256
hash. Before saving, each file is split into variable sized Blobs of data. The
diff --git a/doc/Manual.md b/doc/Manual.md
index 0219a6a..6036400 100644
--- a/doc/Manual.md
+++ b/doc/Manual.md
@@ -1,7 +1,13 @@
-Thanks for using restic. This document will give you an overview of the basic
+nhanks for using restic. This document will give you an overview of the basic
functionality provided by restic.
-# Building/installing restic
+# Installing restic
+
+## from pre-compiled binary
+
+You can download the latest pre-compiled binary from the [restic release page](https://github.com/restic/restic/releases/latest).
+
+## Mac OS X
If you are using Mac OS X, you can install restic using the
[homebrew](http://brew.sh/) packet manager:
@@ -11,25 +17,19 @@ $ brew tap restic/restic
$ brew install restic
```
+## archlinux
+
On archlinux, there is a package called `restic-git` which can be installed from AUR, e.g. with `pacaur`:
```console
$ pacaur -S restic-git
```
-At debian stable you can install 'go' directly from the repositories (as root):
-
-```console
-$ apt-get install golang-go
-```
+# Building restic
-after installation of 'go' go straight forward to 'git clone [...]'
-
-If you are using Linux, BSD or Windows, the only way to install restic on your
-system right now is to compile it from source. restic is written in the Go
-programming language and you need at least Go version 1.6. Building restic may
-also work with older versions of Go, but that's not supported. See the [Getting
-started](https://golang.org/doc/install) guide of the Go project for
+restic is written in the Go programming language and you need at least Go version 1.7.
+Building restic may also work with older versions of Go, but that's not supported.
+See the [Getting started](https://golang.org/doc/install) guide of the Go project for
instructions how to install Go.
In order to build restic from source, execute the following steps:
@@ -43,9 +43,21 @@ $ cd restic
$ go run build.go
```
+You can easily cross-compile restic for all supported platforms, just supply
+the target OS and platform via the command-line options like this (for Windows
+and FreeBSD respectively):
+
+ $ go run build.go --goos windows --goarch amd64
+
+ $ go run build.go --goos freebsd --goarch 386
+
+The resulting binary is statically linked and does not require any libraries.
+
At the moment, the only tested compiler for restic is the official Go compiler.
Building restic with gccgo may work, but is not supported.
+# Usage help
+
Usage help is available:
```console
@@ -71,10 +83,12 @@ Available Commands:
rebuild-index build a new index file
restore extract the data from a snapshot
snapshots list all snapshots
+ tag modifies tags on snapshots
unlock remove locks other processes created
version Print version information
Flags:
+ --json set output mode to JSON for commands that support it
--no-lock do not lock the repo, this allows some operations on read-only repos
-p, --password-file string read the repository password from a file
-q, --quiet do not output comprehensive progress report
@@ -108,6 +122,7 @@ Flags:
--tag tag add a tag for the new snapshot (can be specified multiple times)
Global Flags:
+ --json set output mode to JSON for commands that support it
--no-lock do not lock the repo, this allows some operations on read-only repos
-p, --password-file string read the repository password from a file
-q, --quiet do not output comprehensive progress report
@@ -139,6 +154,8 @@ Please note that knowledge of your password is required to access the repository
Losing your password means that your data is irrecoverably lost.
```
+Other backends like sftp and s3 are [described in a later section](#create-an-sftp-repository) of this document.
+
Remembering your password is important! If you lose it, you won't be able to
access data stored in the repository.
@@ -212,6 +229,15 @@ snapshot 31f7bd63 saved
In fact several hosts may use the same repository to backup directories and
files leading to a greater de-duplication.
+Please be aware that when you backup different directories (or the directories
+to be saved have a variable name component like a time/date), restic always
+needs to read all files and only afterwards can compute which parts of the
+files need to be saved. When you backup the same directory again (maybe with
+new or changed files) restic will find the old snapshot in the repo and by
+default only reads those files that are new or have been modified since the
+last snapshot. This is decided based on the modify date of the file in the
+file system.
+
You can exclude folders and files by specifying exclude-patterns.
Either specify them with multiple `--exclude`'s or one `--exclude-file`
@@ -226,7 +252,7 @@ $ restic -r /tmp/backup backup ~/work --exclude=*.c --exclude-file=exclude
Patterns use [`filepath.Glob`](https://golang.org/pkg/path/filepath/#Glob) internally,
see [`filepath.Match`](https://golang.org/pkg/path/filepath/#Match) for syntax.
-Additionally `**` exludes arbitrary subdirectories.
+Additionally `**` excludes arbitrary subdirectories.
Environment-variables in exclude-files are expanded with [`os.ExpandEnv`](https://golang.org/pkg/os/#ExpandEnv).
By specifying the option `--one-file-system` you can instruct restic to only
@@ -352,7 +378,6 @@ enter password for repository:
restoring <Snapshot of [/home/art] at 2015-05-08 21:45:17.884408621 +0200 CEST> to /tmp/restore-work
```
-
# Manage repository keys
The `key` command allows you to set multiple access keys or passwords per
@@ -380,6 +405,45 @@ enter password for repository:
*eb78040b username kasimir 2015-08-12 13:29:57
```
+# Manage tags
+
+Managing tags on snapshots is done with the `tag` command. The existing set of
+tags can be replaced completely, tags can be added to removed. The result is
+directly visible in the `snapshots` command.
+
+Let's say we want to tag snapshot `590c8fc8` with the tags `NL` and `CH` and
+remove all other tags that may be present, the following command does that:
+
+```console
+$ restic -r /tmp/backup tag --set NL,CH 590c8fc8
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+```
+
+Note the snapshot ID has changed, so between each change we need to look up
+the new ID of the snapshot. But there is an even better way, the `tag` command
+accepts `--tag` for a filter, so we can filter snapshots based on the tag we
+just added.
+
+So we can add and remove tags incrementally like this:
+
+```console
+$ restic -r /tmp/backup tag --tag NL --remove CH
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --add UK
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --remove NL
+Create exclusive lock for repository
+Modified tags on 1 snapshots
+
+$ restic -r /tmp/backup tag --tag NL --add SOMETHING
+No snapshots were modified
+```
+
# Check integrity and consistency
Imagine your repository is saved on a server that has a faulty hard drive, or
@@ -425,6 +489,11 @@ Don't forget to umount after quitting!
Mounting repositories via FUSE is not possible on Windows and OpenBSD.
+Restic supports storage and preservation of hard links. However, since hard links
+exist in the scope of a filesystem by definition, restoring hard links from a fuse
+mount should be done by a program that preserves hard links. A program that does so
+is rsync, used with the option --hard-links.
+
# Create an SFTP repository
In order to backup data via SFTP, you must first set up a server with SSH and
@@ -509,8 +578,8 @@ only available via HTTP, you can specify the URL to the server like this:
### Pre-Requisites
-* Download and Install [Minio Server](https://minio.io/download/).
-* You can also refer to [https://docs.minio.io](https://docs.minio.io) for step by step guidance on installation and getting started on Minio CLient and Minio Server.
+* Download and Install [Minio Server](https://minio.io/downloads/#minio-server).
+* You can also refer to [https://docs.minio.io](https://docs.minio.io) for step by step guidance on installation and getting started on Minio Client and Minio Server.
You must first setup the following environment variables with the credentials of your running Minio Server.
@@ -537,7 +606,8 @@ be done either manually (by specifying a snapshot ID to remove) or by using a
policy that describes which snapshots to forget. For all remove operations, two
commands need to be called in sequence: `forget` to remove a snapshot and
`prune` to actually remove the data that was referenced by the snapshot from
-the repository.
+the repository. This can be automated with the `--prune` option of the `forget`
+command, which runs `prune` automatically if snapshots have been removed.
## Remove a single snapshot
@@ -602,6 +672,41 @@ done
Afterwards the repository is smaller.
+You can automate this two-step process by using the `--prune` switch to
+`forget`:
+
+```console
+$ restic forget --keep-last 1 --prune
+snapshots for host mopped, directories /home/user/work:
+
+keep 1 snapshots:
+ID Date Host Tags Directory
+----------------------------------------------------------------------
+4bba301e 2017-02-21 10:49:18 mopped /home/user/work
+
+remove 1 snapshots:
+ID Date Host Tags Directory
+----------------------------------------------------------------------
+8c02b94b 2017-02-21 10:48:33 mopped /home/user/work
+
+1 snapshots have been removed, running prune
+counting files in repo
+building new index for repo
+[0:00] 100.00% 37 / 37 packs
+repository contains 37 packs (5521 blobs) with 151.012 MiB bytes
+processed 5521 blobs: 0 duplicate blobs, 0B duplicate
+load all snapshots
+find data that is still in use for 1 snapshots
+[0:00] 100.00% 1 / 1 snapshots
+found 5323 of 5521 data blobs still in use, removing 198 blobs
+will delete 0 packs and rewrite 27 packs, this frees 22.106 MiB
+creating new index
+[0:00] 100.00% 30 / 30 packs
+saved new index as b49f3e68
+done
+```
+
+
## Removing snapshots according to a policy
Removing snapshots manually is tedious and error-prone, therefore restic allows
@@ -733,3 +838,10 @@ enter password for repository:
"gid": 20
}
```
+
+# Scripting restic
+
+Restic supports the output of some commands in JSON format. The JSON flag ```--json``` is currently supported only by ```restic snapshots```.
+
+```console
+$ restic -r /tmp/backup snapshots --json```
diff --git a/src/cmds/restic/cmd_backup.go b/src/cmds/restic/cmd_backup.go
index c8e3933..d98e7e5 100644
--- a/src/cmds/restic/cmd_backup.go
+++ b/src/cmds/restic/cmd_backup.go
@@ -3,14 +3,13 @@ package main
import (
"bufio"
"fmt"
+ "io"
"os"
"path/filepath"
"restic"
"strings"
"time"
- "golang.org/x/crypto/ssh/terminal"
-
"github.com/spf13/cobra"
"restic/archiver"
@@ -28,6 +27,10 @@ The "backup" command creates a new snapshot and saves the files and directories
given as the arguments.
`,
RunE: func(cmd *cobra.Command, args []string) error {
+ if backupOptions.Stdin && backupOptions.FilesFrom == "-" {
+ return errors.Fatal("cannot use both `--stdin` and `--files-from -`")
+ }
+
if backupOptions.Stdin {
return readBackupFromStdin(backupOptions, globalOptions, args)
}
@@ -46,6 +49,7 @@ type BackupOptions struct {
Stdin bool
StdinFilename string
Tags []string
+ Hostname string
FilesFrom string
}
@@ -54,15 +58,22 @@ var backupOptions BackupOptions
func init() {
cmdRoot.AddCommand(cmdBackup)
+ hostname, err := os.Hostname()
+ if err != nil {
+ debug.Log("os.Hostname() returned err: %v", err)
+ hostname = ""
+ }
+
f := cmdBackup.Flags()
f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
- f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories. Overrides the "parent" flag`)
- f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", []string{}, "exclude a `pattern` (can be specified multiple times)")
+ f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
+ f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
f.StringVar(&backupOptions.ExcludeFile, "exclude-file", "", "read exclude patterns from a file")
- f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "Exclude other file systems")
+ f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems")
f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
- f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "", "file name to use when reading from stdin")
- f.StringSliceVar(&backupOptions.Tags, "tag", []string{}, "add a `tag` for the new snapshot (can be specified multiple times)")
+ f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin")
+ f.StringSliceVar(&backupOptions.Tags, "tag", nil, "add a `tag` for the new snapshot (can be specified multiple times)")
+ f.StringVar(&backupOptions.Hostname, "hostname", hostname, "set the `hostname` for the snapshot manually")
f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
}
@@ -123,8 +134,7 @@ func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress
s.Errors)
status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))
- w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
- if err == nil {
+ if w := stdoutTerminalWidth(); w > 0 {
maxlen := w - len(status2) - 1
if maxlen < 4 {
@@ -168,8 +178,7 @@ func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
formatBytes(s.Bytes),
formatBytes(bps))
- w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
- if err == nil {
+ if w := stdoutTerminalWidth(); w > 0 {
maxlen := w - len(status1)
if maxlen < 4 {
@@ -232,7 +241,15 @@ func gatherDevices(items []string) (deviceMap map[uint64]struct{}, err error) {
func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
if len(args) != 0 {
- return errors.Fatalf("when reading from stdin, no additional files can be specified")
+ return errors.Fatal("when reading from stdin, no additional files can be specified")
+ }
+
+ if opts.StdinFilename == "" {
+ return errors.Fatal("filename for backup from stdin must not be empty")
+ }
+
+ if gopts.password == "" && gopts.PasswordFile == "" {
+ return errors.Fatal("unable to read password from stdin when data is to be read from stdin, use --password-file or $RESTIC_PASSWORD")
}
repo, err := OpenRepository(gopts)
@@ -251,7 +268,13 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
return err
}
- _, id, err := archiver.ArchiveReader(repo, newArchiveStdinProgress(gopts), os.Stdin, opts.StdinFilename, opts.Tags)
+ r := &archiver.Reader{
+ Repository: repo,
+ Tags: opts.Tags,
+ Hostname: opts.Hostname,
+ }
+
+ _, id, err := r.Archive(opts.StdinFilename, os.Stdin, newArchiveStdinProgress(gopts))
if err != nil {
return err
}
@@ -262,23 +285,32 @@ func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string)
// readFromFile will read all lines from the given filename and write them to a
// string array, if filename is empty readFromFile returns and empty string
-// array
+// array. If filename is a dash (-), readFromFile will read the lines from
+// the standard input.
func readLinesFromFile(filename string) ([]string, error) {
if filename == "" {
return nil, nil
}
- file, err := os.Open(filename)
- if err != nil {
- return nil, err
+ var r io.Reader = os.Stdin
+ if filename != "-" {
+ f, err := os.Open(filename)
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ r = f
}
- defer file.Close()
var lines []string
- scanner := bufio.NewScanner(file)
+ scanner := bufio.NewScanner(r)
for scanner.Scan() {
- lines = append(lines, scanner.Text())
+ line := scanner.Text()
+ if line == "" {
+ continue
+ }
+ lines = append(lines, line)
}
if err := scanner.Err(); err != nil {
@@ -289,6 +321,10 @@ func readLinesFromFile(filename string) ([]string, error) {
}
func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
+ if opts.FilesFrom == "-" && gopts.password == "" && gopts.PasswordFile == "" {
+ return errors.Fatal("no password; either use `--password-file` option or put the password into the RESTIC_PASSWORD environment variable")
+ }
+
fromfile, err := readLinesFromFile(opts.FilesFrom)
if err != nil {
return err
@@ -299,7 +335,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
// same time
args = append(args, fromfile...)
if len(args) == 0 {
- return errors.Fatalf("wrong number of parameters")
+ return errors.Fatal("wrong number of parameters")
}
target := make([]string, 0, len(args))
@@ -355,13 +391,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
// Find last snapshot to set it as parent, if not already set
if !opts.Force && parentSnapshotID == nil {
- hostname, err := os.Hostname()
- if err != nil {
- debug.Log("os.Hostname() returned err: %v", err)
- hostname = ""
- }
-
- id, err := restic.FindLatestSnapshot(repo, target, hostname)
+ id, err := restic.FindLatestSnapshot(repo, target, opts.Tags, opts.Hostname)
if err == nil {
parentSnapshotID = &id
} else if err != restic.ErrNoSnapshotFound {
@@ -437,7 +467,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
}
- _, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, parentSnapshotID)
+ _, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, opts.Hostname, parentSnapshotID)
if err != nil {
return err
}
diff --git a/src/cmds/restic/cmd_cat.go b/src/cmds/restic/cmd_cat.go
index d9e723a..ee5798d 100644
--- a/src/cmds/restic/cmd_cat.go
+++ b/src/cmds/restic/cmd_cat.go
@@ -30,7 +30,7 @@ func init() {
func runCat(gopts GlobalOptions, args []string) error {
if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) {
- return errors.Fatalf("type or ID not specified")
+ return errors.Fatal("type or ID not specified")
}
repo, err := OpenRepository(gopts)
diff --git a/src/cmds/restic/cmd_check.go b/src/cmds/restic/cmd_check.go
index 093bbe1..2f0064f 100644
--- a/src/cmds/restic/cmd_check.go
+++ b/src/cmds/restic/cmd_check.go
@@ -7,8 +7,6 @@ import (
"github.com/spf13/cobra"
- "golang.org/x/crypto/ssh/terminal"
-
"restic"
"restic/checker"
"restic/errors"
@@ -26,7 +24,7 @@ finds. It can also be used to read all data and therefore simulate a restore.
},
}
-// CheckOptions bundle all options for the 'check' command.
+// CheckOptions bundles all options for the 'check' command.
type CheckOptions struct {
ReadData bool
CheckUnused bool
@@ -38,8 +36,8 @@ func init() {
cmdRoot.AddCommand(cmdCheck)
f := cmdCheck.Flags()
- f.BoolVar(&checkOptions.ReadData, "read-data", false, "Read all data blobs")
- f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "Find unused blobs")
+ f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
+ f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
}
func newReadProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
@@ -55,8 +53,7 @@ func newReadProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
formatPercent(s.Blobs, todo.Blobs),
s.Blobs, todo.Blobs)
- w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
- if err == nil {
+ if w := stdoutTerminalWidth(); w > 0 {
if len(status) > w {
max := w - len(status) - 4
status = status[:max] + "... "
diff --git a/src/cmds/restic/cmd_dump.go b/src/cmds/restic/cmd_dump.go
index 1a93bb4..350e4d7 100644
--- a/src/cmds/restic/cmd_dump.go
+++ b/src/cmds/restic/cmd_dump.go
@@ -22,7 +22,7 @@ var cmdDump = &cobra.Command{
Use: "dump [indexes|snapshots|trees|all|packs]",
Short: "dump data structures",
Long: `
-The "dump" command dumps data structures from a repository as JSON objects. It
+The "dump" command dumps data structures from the repository as JSON objects. It
is used for debugging purposes only.`,
RunE: func(cmd *cobra.Command, args []string) error {
return runDump(globalOptions, args)
@@ -168,7 +168,7 @@ func dumpIndexes(repo restic.Repository) error {
func runDump(gopts GlobalOptions, args []string) error {
if len(args) != 1 {
- return errors.Fatalf("type not specified")
+ return errors.Fatal("type not specified")
}
repo, err := OpenRepository(gopts)
diff --git a/src/cmds/restic/cmd_find.go b/src/cmds/restic/cmd_find.go
index 484ae71..23c3948 100644
--- a/src/cmds/restic/cmd_find.go
+++ b/src/cmds/restic/cmd_find.go
@@ -1,7 +1,9 @@
package main
import (
+ "context"
"path/filepath"
+ "strings"
"time"
"github.com/spf13/cobra"
@@ -23,11 +25,16 @@ repo. `,
},
}
-// FindOptions bundle all options for the find command.
+// FindOptions bundles all options for the find command.
type FindOptions struct {
- Oldest string
- Newest string
- Snapshot string
+ Oldest string
+ Newest string
+ Snapshots []string
+ CaseInsensitive bool
+ ListLong bool
+ Host string
+ Paths []string
+ Tags []string
}
var findOptions FindOptions
@@ -36,19 +43,21 @@ func init() {
cmdRoot.AddCommand(cmdFind)
f := cmdFind.Flags()
- f.StringVarP(&findOptions.Oldest, "oldest", "o", "", "Oldest modification date/time")
- f.StringVarP(&findOptions.Newest, "newest", "n", "", "Newest modification date/time")
- f.StringVarP(&findOptions.Snapshot, "snapshot", "s", "", "Snapshot ID to search in")
+ f.StringVarP(&findOptions.Oldest, "oldest", "o", "", "oldest modification date/time")
+ f.StringVarP(&findOptions.Newest, "newest", "n", "", "newest modification date/time")
+ f.StringSliceVarP(&findOptions.Snapshots, "snapshot", "s", nil, "snapshot `id` to search in (can be given multiple times)")
+ f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
+ f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+
+ f.StringVarP(&findOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+ f.StringSliceVar(&findOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
+ f.StringSliceVar(&findOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
}
type findPattern struct {
oldest, newest time.Time
pattern string
-}
-
-type findResult struct {
- node *restic.Node
- path string
+ ignoreCase bool
}
var timeFormats = []string{
@@ -75,20 +84,25 @@ func parseTime(str string) (time.Time, error) {
return time.Time{}, errors.Fatalf("unable to parse time: %q", str)
}
-func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path string) ([]findResult, error) {
+func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, prefix string, snapshotID *string) error {
debug.Log("checking tree %v\n", id)
+
tree, err := repo.LoadTree(id)
if err != nil {
- return nil, err
+ return err
}
- results := []findResult{}
for _, node := range tree.Nodes {
debug.Log(" testing entry %q\n", node.Name)
- m, err := filepath.Match(pat.pattern, node.Name)
+ name := node.Name
+ if pat.ignoreCase {
+ name = strings.ToLower(name)
+ }
+
+ m, err := filepath.Match(pat.pattern, name)
if err != nil {
- return nil, err
+ return err
}
if m {
@@ -103,69 +117,55 @@ func findInTree(repo *repository.Repository, pat findPattern, id restic.ID, path
continue
}
- results = append(results, findResult{node: node, path: path})
+ if snapshotID != nil {
+ Verbosef("Found matching entries in snapshot %s\n", *snapshotID)
+ snapshotID = nil
+ }
+ Printf(formatNode(prefix, node, findOptions.ListLong) + "\n")
} else {
debug.Log(" pattern does not match\n")
}
if node.Type == "dir" {
- subdirResults, err := findInTree(repo, pat, *node.Subtree, filepath.Join(path, node.Name))
- if err != nil {
- return nil, err
+ if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), snapshotID); err != nil {
+ return err
}
-
- results = append(results, subdirResults...)
}
}
- return results, nil
+ return nil
}
-func findInSnapshot(repo *repository.Repository, pat findPattern, id restic.ID) error {
- debug.Log("searching in snapshot %s\n for entries within [%s %s]", id.Str(), pat.oldest, pat.newest)
-
- sn, err := restic.LoadSnapshot(repo, id)
- if err != nil {
- return err
- }
+func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern) error {
+ debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest)
- results, err := findInTree(repo, pat, *sn.Tree, "")
- if err != nil {
+ snapshotID := sn.ID().Str()
+ if err := findInTree(repo, pat, *sn.Tree, string(filepath.Separator), &snapshotID); err != nil {
return err
}
-
- if len(results) == 0 {
- return nil
- }
- Verbosef("found %d matching entries in snapshot %s\n", len(results), id)
- for _, res := range results {
- res.node.Name = filepath.Join(res.path, res.node.Name)
- Printf(" %s\n", res.node)
- }
-
return nil
}
func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
if len(args) != 1 {
- return errors.Fatalf("wrong number of arguments")
+ return errors.Fatal("wrong number of arguments")
}
- var (
- err error
- pat findPattern
- )
+ var err error
+ pat := findPattern{pattern: args[0]}
+ if opts.CaseInsensitive {
+ pat.pattern = strings.ToLower(pat.pattern)
+ pat.ignoreCase = true
+ }
if opts.Oldest != "" {
- pat.oldest, err = parseTime(opts.Oldest)
- if err != nil {
+ if pat.oldest, err = parseTime(opts.Oldest); err != nil {
return err
}
}
if opts.Newest != "" {
- pat.newest, err = parseTime(opts.Newest)
- if err != nil {
+ if pat.newest, err = parseTime(opts.Newest); err != nil {
return err
}
}
@@ -183,28 +183,14 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error {
}
}
- err = repo.LoadIndex()
- if err != nil {
+ if err = repo.LoadIndex(); err != nil {
return err
}
- pat.pattern = args[0]
-
- if opts.Snapshot != "" {
- snapshotID, err := restic.FindSnapshot(repo, opts.Snapshot)
- if err != nil {
- return errors.Fatalf("invalid id %q: %v", args[1], err)
- }
-
- return findInSnapshot(repo, pat, snapshotID)
- }
-
- done := make(chan struct{})
- defer close(done)
- for snapshotID := range repo.List(restic.SnapshotFile, done) {
- err := findInSnapshot(repo, pat, snapshotID)
-
- if err != nil {
+ ctx, cancel := context.WithCancel(gopts.ctx)
+ defer cancel()
+ for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) {
+ if err = findInSnapshot(repo, sn, pat); err != nil {
return err
}
}
diff --git a/src/cmds/restic/cmd_forget.go b/src/cmds/restic/cmd_forget.go
index df84f2a..bc372fc 100644
--- a/src/cmds/restic/cmd_forget.go
+++ b/src/cmds/restic/cmd_forget.go
@@ -1,10 +1,10 @@
package main
import (
- "encoding/hex"
- "fmt"
- "io"
+ "context"
+ "encoding/json"
"restic"
+ "sort"
"strings"
"github.com/spf13/cobra"
@@ -25,19 +25,21 @@ data after 'forget' was run successfully, see the 'prune' command. `,
// ForgetOptions collects all options for the forget command.
type ForgetOptions struct {
- Last int
- Hourly int
- Daily int
- Weekly int
- Monthly int
- Yearly int
-
+ Last int
+ Hourly int
+ Daily int
+ Weekly int
+ Monthly int
+ Yearly int
KeepTags []string
- Hostname string
- Tags []string
+ Host string
+ Tags []string
+ Paths []string
- DryRun bool
+ GroupByTags bool
+ DryRun bool
+ Prune bool
}
var forgetOptions ForgetOptions
@@ -53,51 +55,17 @@ func init() {
f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots")
f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots")
- f.StringSliceVar(&forgetOptions.KeepTags, "keep-tag", []string{}, "always keep snapshots with this `tag` (can be specified multiple times)")
- f.StringVar(&forgetOptions.Hostname, "hostname", "", "only forget snapshots for the given hostname")
- f.StringSliceVar(&forgetOptions.Tags, "tag", []string{}, "only forget snapshots with the `tag` (can be specified multiple times)")
+ f.StringSliceVar(&forgetOptions.KeepTags, "keep-tag", []string{}, "keep snapshots with this `tag` (can be specified multiple times)")
+ f.BoolVarP(&forgetOptions.GroupByTags, "group-by-tags", "G", false, "Group by host,paths,tags instead of just host,paths")
+ // Sadly the commonly used shortcut `H` is already used.
+ f.StringVar(&forgetOptions.Host, "host", "", "only consider snapshots with the given `host`")
+ // Deprecated since 2017-03-07.
+ f.StringVar(&forgetOptions.Host, "hostname", "", "only consider snapshots with the given `hostname` (deprecated)")
+ f.StringSliceVar(&forgetOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
+ f.StringSliceVar(&forgetOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` (can be specified multiple times)")
f.BoolVarP(&forgetOptions.DryRun, "dry-run", "n", false, "do not delete anything, just print what would be done")
-}
-
-func printSnapshots(w io.Writer, snapshots restic.Snapshots) {
- tab := NewTable()
- tab.Header = fmt.Sprintf("%-8s %-19s %-10s %-10s %s", "ID", "Date", "Host", "Tags", "Directory")
- tab.RowFormat = "%-8s %-19s %-10s %-10s %s"
-
- for _, sn := range snapshots {
- if len(sn.Paths) == 0 {
- continue
- }
-
- firstTag := ""
- if len(sn.Tags) > 0 {
- firstTag = sn.Tags[0]
- }
-
- tab.Rows = append(tab.Rows, []interface{}{sn.ID().Str(), sn.Time.Format(TimeFormat), sn.Hostname, firstTag, sn.Paths[0]})
-
- rows := len(sn.Paths)
- if len(sn.Tags) > rows {
- rows = len(sn.Tags)
- }
-
- for i := 1; i < rows; i++ {
- path := ""
- if len(sn.Paths) > i {
- path = sn.Paths[i]
- }
-
- tag := ""
- if len(sn.Tags) > i {
- tag = sn.Tags[i]
- }
-
- tab.Rows = append(tab.Rows, []interface{}{"", "", "", tag, path})
- }
- }
-
- tab.Write(w)
+ f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
}
func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
@@ -112,38 +80,45 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
return err
}
- // parse arguments as hex strings
- var ids []string
- for _, s := range args {
- _, err := hex.DecodeString(s)
- if err != nil {
- Warnf("argument %q is not a snapshot ID, ignoring\n", s)
- continue
- }
-
- ids = append(ids, s)
+ // group by hostname and dirs
+ type key struct {
+ Hostname string
+ Paths []string
+ Tags []string
}
-
- // process all snapshot IDs given as arguments
- for _, s := range ids {
- id, err := restic.FindSnapshot(repo, s)
- if err != nil {
- Warnf("cound not find a snapshot for ID %q, ignoring\n", s)
- continue
- }
-
- if !opts.DryRun {
- h := restic.Handle{Type: restic.SnapshotFile, Name: id.String()}
- err = repo.Backend().Remove(h)
+ snapshotGroups := make(map[string]restic.Snapshots)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+ if len(args) > 0 {
+ // When explicit snapshots args are given, remove them immediately.
+ if !opts.DryRun {
+ h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+ if err = repo.Backend().Remove(h); err != nil {
+ return err
+ }
+ Verbosef("removed snapshot %v\n", sn.ID().Str())
+ } else {
+ Verbosef("would have removed snapshot %v\n", sn.ID().Str())
+ }
+ } else {
+ var tags []string
+ if opts.GroupByTags {
+ tags = sn.Tags
+ sort.StringSlice(tags).Sort()
+ }
+ sort.StringSlice(sn.Paths).Sort()
+ k, err := json.Marshal(key{Hostname: sn.Hostname, Tags: tags, Paths: sn.Paths})
if err != nil {
return err
}
-
- Verbosef("removed snapshot %v\n", id.Str())
- } else {
- Verbosef("would remove snapshot %v\n", id.Str())
+ snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
}
}
+ if len(args) > 0 {
+ return nil
+ }
policy := restic.ExpirePolicy{
Last: opts.Last,
@@ -156,49 +131,36 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
}
if policy.Empty() {
+ Verbosef("no policy was specified, no snapshots will be removed\n")
return nil
}
- // then, load all remaining snapshots
- snapshots, err := restic.LoadAllSnapshots(repo)
- if err != nil {
- return err
- }
-
- // group by hostname and dirs
- type key struct {
- Hostname string
- Dirs string
- }
-
- snapshotGroups := make(map[key]restic.Snapshots)
-
- for _, sn := range snapshots {
- if opts.Hostname != "" && sn.Hostname != opts.Hostname {
- continue
+ removeSnapshots := 0
+ for k, snapshotGroup := range snapshotGroups {
+ var key key
+ if json.Unmarshal([]byte(k), &key) != nil {
+ return err
}
-
- if !sn.HasTags(opts.Tags) {
- continue
+ if opts.GroupByTags {
+ Printf("snapshots for host %v, tags [%v], paths: [%v]:\n\n", key.Hostname, strings.Join(key.Tags, ", "), strings.Join(key.Paths, ", "))
+ } else {
+ Printf("snapshots for host %v, paths: [%v]:\n\n", key.Hostname, strings.Join(key.Paths, ", "))
}
-
- k := key{Hostname: sn.Hostname, Dirs: strings.Join(sn.Paths, ":")}
- list := snapshotGroups[k]
- list = append(list, sn)
- snapshotGroups[k] = list
- }
-
- for key, snapshotGroup := range snapshotGroups {
- Printf("snapshots for host %v, directories %v:\n\n", key.Hostname, key.Dirs)
keep, remove := restic.ApplyPolicy(snapshotGroup, policy)
- Printf("keep %d snapshots:\n", len(keep))
- printSnapshots(globalOptions.stdout, keep)
- Printf("\n")
+ if len(keep) != 0 {
+ Printf("keep %d snapshots:\n", len(keep))
+ PrintSnapshots(globalOptions.stdout, keep)
+ Printf("\n")
+ }
+
+ if len(remove) != 0 {
+ Printf("remove %d snapshots:\n", len(remove))
+ PrintSnapshots(globalOptions.stdout, remove)
+ Printf("\n")
+ }
- Printf("remove %d snapshots:\n", len(remove))
- printSnapshots(globalOptions.stdout, remove)
- Printf("\n")
+ removeSnapshots += len(remove)
if !opts.DryRun {
for _, sn := range remove {
@@ -211,5 +173,12 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error {
}
}
+ if removeSnapshots > 0 && opts.Prune {
+ Printf("%d snapshots have been removed, running prune\n", removeSnapshots)
+ if !opts.DryRun {
+ return pruneRepository(gopts, repo)
+ }
+ }
+
return nil
}
diff --git a/src/cmds/restic/cmd_key.go b/src/cmds/restic/cmd_key.go
index 946585a..052dd5b 100644
--- a/src/cmds/restic/cmd_key.go
+++ b/src/cmds/restic/cmd_key.go
@@ -1,20 +1,20 @@
package main
import (
+ "context"
"fmt"
"restic"
-
- "github.com/spf13/cobra"
-
"restic/errors"
"restic/repository"
+
+ "github.com/spf13/cobra"
)
var cmdKey = &cobra.Command{
Use: "key [list|add|rm|passwd] [ID]",
Short: "manage keys (passwords)",
Long: `
-The "key" command manages keys (passwords) for accessing a repository.
+The "key" command manages keys (passwords) for accessing the repository.
`,
RunE: func(cmd *cobra.Command, args []string) error {
return runKey(globalOptions, args)
@@ -25,15 +25,12 @@ func init() {
cmdRoot.AddCommand(cmdKey)
}
-func listKeys(s *repository.Repository) error {
+func listKeys(ctx context.Context, s *repository.Repository) error {
tab := NewTable()
tab.Header = fmt.Sprintf(" %-10s %-10s %-10s %s", "ID", "User", "Host", "Created")
tab.RowFormat = "%s%-10s %-10s %-10s %s"
- done := make(chan struct{})
- defer close(done)
-
- for id := range s.List(restic.KeyFile, done) {
+ for id := range s.List(restic.KeyFile, ctx.Done()) {
k, err := repository.LoadKey(s, id.String())
if err != nil {
Warnf("LoadKey() failed: %v\n", err)
@@ -120,10 +117,13 @@ func changePassword(gopts GlobalOptions, repo *repository.Repository) error {
}
func runKey(gopts GlobalOptions, args []string) error {
- if len(args) < 1 || (args[0] == "rm" && len(args) != 2) {
- return errors.Fatalf("wrong number of arguments")
+ if len(args) < 1 || (args[0] == "rm" && len(args) != 2) || (args[0] != "rm" && len(args) != 1) {
+ return errors.Fatal("wrong number of arguments")
}
+ ctx, cancel := context.WithCancel(gopts.ctx)
+ defer cancel()
+
repo, err := OpenRepository(gopts)
if err != nil {
return err
@@ -137,7 +137,7 @@ func runKey(gopts GlobalOptions, args []string) error {
return err
}
- return listKeys(repo)
+ return listKeys(ctx, repo)
case "add":
lock, err := lockRepo(repo)
defer unlockRepo(lock)
diff --git a/src/cmds/restic/cmd_list.go b/src/cmds/restic/cmd_list.go
index 561943a..105f70a 100644
--- a/src/cmds/restic/cmd_list.go
+++ b/src/cmds/restic/cmd_list.go
@@ -11,9 +11,9 @@ import (
var cmdList = &cobra.Command{
Use: "list [blobs|packs|index|snapshots|keys|locks]",
- Short: "list items in the repository",
+ Short: "list objects in the repository",
Long: `
-
+The "list" command allows listing objects in the repository based on type.
`,
RunE: func(cmd *cobra.Command, args []string) error {
return runList(globalOptions, args)
@@ -26,7 +26,7 @@ func init() {
func runList(opts GlobalOptions, args []string) error {
if len(args) != 1 {
- return errors.Fatalf("type not specified")
+ return errors.Fatal("type not specified")
}
repo, err := OpenRepository(opts)
diff --git a/src/cmds/restic/cmd_ls.go b/src/cmds/restic/cmd_ls.go
index bee9d1e..7d613b4 100644
--- a/src/cmds/restic/cmd_ls.go
+++ b/src/cmds/restic/cmd_ls.go
@@ -1,8 +1,7 @@
package main
import (
- "fmt"
- "os"
+ "context"
"path/filepath"
"github.com/spf13/cobra"
@@ -13,7 +12,7 @@ import (
)
var cmdLs = &cobra.Command{
- Use: "ls [flags] snapshot-ID",
+ Use: "ls [flags] [snapshot-ID ...]",
Short: "list files in a snapshot",
Long: `
The "ls" command allows listing files and directories in a snapshot.
@@ -21,7 +20,7 @@ The "ls" command allows listing files and directories in a snapshot.
The special snapshot-ID "latest" can be used to list files and directories of the latest snapshot in the repository.
`,
RunE: func(cmd *cobra.Command, args []string) error {
- return runLs(globalOptions, args)
+ return runLs(lsOptions, globalOptions, args)
},
}
@@ -29,6 +28,7 @@ The special snapshot-ID "latest" can be used to list files and directories of th
type LsOptions struct {
ListLong bool
Host string
+ Tags []string
Paths []string
}
@@ -40,42 +40,22 @@ func init() {
flags := cmdLs.Flags()
flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
- flags.StringVarP(&lsOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
- flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
+ flags.StringVarP(&lsOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+ flags.StringSliceVar(&lsOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot ID is given")
+ flags.StringSliceVar(&lsOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
}
-func printNode(prefix string, n *restic.Node) string {
- if !lsOptions.ListLong {
- return filepath.Join(prefix, n.Name)
- }
-
- switch n.Type {
- case "file":
- return fmt.Sprintf("%s %5d %5d %6d %s %s",
- n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
- case "dir":
- return fmt.Sprintf("%s %5d %5d %6d %s %s",
- n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
- case "symlink":
- return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
- n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
- default:
- return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
- }
-}
-
-func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
- tree, err := repo.LoadTree(id)
+func printTree(repo *repository.Repository, id *restic.ID, prefix string) error {
+ tree, err := repo.LoadTree(*id)
if err != nil {
return err
}
for _, entry := range tree.Nodes {
- Printf(printNode(prefix, entry) + "\n")
+ Printf(formatNode(prefix, entry, lsOptions.ListLong) + "\n")
if entry.Type == "dir" && entry.Subtree != nil {
- err = printTree(filepath.Join(prefix, entry.Name), repo, *entry.Subtree)
- if err != nil {
+ if err = printTree(repo, entry.Subtree, filepath.Join(prefix, entry.Name)); err != nil {
return err
}
}
@@ -84,9 +64,9 @@ func printTree(prefix string, repo *repository.Repository, id restic.ID) error {
return nil
}
-func runLs(gopts GlobalOptions, args []string) error {
- if len(args) < 1 || len(args) > 2 {
- return errors.Fatalf("no snapshot ID given")
+func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
+ if len(args) == 0 && opts.Host == "" && len(opts.Tags) == 0 && len(opts.Paths) == 0 {
+ return errors.Fatal("Invalid arguments, either give one or more snapshot IDs or set filters.")
}
repo, err := OpenRepository(gopts)
@@ -94,32 +74,18 @@ func runLs(gopts GlobalOptions, args []string) error {
return err
}
- err = repo.LoadIndex()
- if err != nil {
+ if err = repo.LoadIndex(); err != nil {
return err
}
- snapshotIDString := args[0]
- var id restic.ID
+ ctx, cancel := context.WithCancel(gopts.ctx)
+ defer cancel()
+ for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+ Verbosef("snapshot %s of %v at %s):\n", sn.ID().Str(), sn.Paths, sn.Time)
- if snapshotIDString == "latest" {
- id, err = restic.FindLatestSnapshot(repo, lsOptions.Paths, lsOptions.Host)
- if err != nil {
- Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, lsOptions.Paths, lsOptions.Host)
- }
- } else {
- id, err = restic.FindSnapshot(repo, snapshotIDString)
- if err != nil {
- Exitf(1, "invalid id %q: %v", snapshotIDString, err)
+ if err = printTree(repo, sn.Tree, string(filepath.Separator)); err != nil {
+ return err
}
}
-
- sn, err := restic.LoadSnapshot(repo, id)
- if err != nil {
- return err
- }
-
- Verbosef("snapshot of %v at %s:\n", sn.Paths, sn.Time)
-
- return printTree("", repo, *sn.Tree)
+ return nil
}
diff --git a/src/cmds/restic/cmd_mount.go b/src/cmds/restic/cmd_mount.go
index e339e5e..20ce5de 100644
--- a/src/cmds/restic/cmd_mount.go
+++ b/src/cmds/restic/cmd_mount.go
@@ -32,7 +32,12 @@ read-only mount.
// MountOptions collects all options for the mount command.
type MountOptions struct {
- OwnerRoot bool
+ OwnerRoot bool
+ AllowRoot bool
+ AllowOther bool
+ Host string
+ Tags []string
+ Paths []string
}
var mountOptions MountOptions
@@ -40,7 +45,14 @@ var mountOptions MountOptions
func init() {
cmdRoot.AddCommand(cmdMount)
- cmdMount.Flags().BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
+ mountFlags := cmdMount.Flags()
+ mountFlags.BoolVar(&mountOptions.OwnerRoot, "owner-root", false, "use 'root' as the owner of files and dirs")
+ mountFlags.BoolVar(&mountOptions.AllowRoot, "allow-root", false, "allow root user to access the data in the mounted directory")
+ mountFlags.BoolVar(&mountOptions.AllowOther, "allow-other", false, "allow other users to access the data in the mounted directory")
+
+ mountFlags.StringVarP(&mountOptions.Host, "host", "H", "", `only consider snapshots for this host`)
+ mountFlags.StringSliceVar(&mountOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`")
+ mountFlags.StringSliceVar(&mountOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`")
}
func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
@@ -64,11 +76,21 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
return err
}
}
- c, err := systemFuse.Mount(
- mountpoint,
+
+ mountOptions := []systemFuse.MountOption{
systemFuse.ReadOnly(),
systemFuse.FSName("restic"),
- )
+ }
+
+ if opts.AllowRoot {
+ mountOptions = append(mountOptions, systemFuse.AllowRoot())
+ }
+
+ if opts.AllowOther {
+ mountOptions = append(mountOptions, systemFuse.AllowOther())
+ }
+
+ c, err := systemFuse.Mount(mountpoint, mountOptions...)
if err != nil {
return err
}
@@ -77,7 +99,7 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error {
Printf("Don't forget to umount after quitting!\n")
root := fs.Tree{}
- root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot))
+ root.Add("snapshots", fuse.NewSnapshotsDir(repo, opts.OwnerRoot, opts.Paths, opts.Tags, opts.Host))
debug.Log("serving mount at %v", mountpoint)
err = fs.Serve(c, &root)
@@ -95,7 +117,7 @@ func umount(mountpoint string) error {
func runMount(opts MountOptions, gopts GlobalOptions, args []string) error {
if len(args) == 0 {
- return errors.Fatalf("wrong number of parameters")
+ return errors.Fatal("wrong number of parameters")
}
mountpoint := args[0]
diff --git a/src/cmds/restic/cmd_prune.go b/src/cmds/restic/cmd_prune.go
index f43c99f..03d14d3 100644
--- a/src/cmds/restic/cmd_prune.go
+++ b/src/cmds/restic/cmd_prune.go
@@ -1,8 +1,8 @@
package main
import (
+ "context"
"fmt"
- "os"
"restic"
"restic/debug"
"restic/errors"
@@ -11,8 +11,6 @@ import (
"time"
"github.com/spf13/cobra"
-
- "golang.org/x/crypto/ssh/terminal"
)
var cmdPrune = &cobra.Command{
@@ -45,8 +43,7 @@ func newProgressMax(show bool, max uint64, description string) *restic.Progress
formatPercent(s.Blobs, max),
s.Blobs, max, description)
- w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
- if err == nil {
+ if w := stdoutTerminalWidth(); w > 0 {
if len(status) > w {
max := w - len(status) - 4
status = status[:max] + "... "
@@ -75,13 +72,17 @@ func runPrune(gopts GlobalOptions) error {
return err
}
- err = repo.LoadIndex()
+ return pruneRepository(gopts, repo)
+}
+
+func pruneRepository(gopts GlobalOptions, repo restic.Repository) error {
+ err := repo.LoadIndex()
if err != nil {
return err
}
- done := make(chan struct{})
- defer close(done)
+ ctx, cancel := context.WithCancel(gopts.ctx)
+ defer cancel()
var stats struct {
blobs int
@@ -91,7 +92,7 @@ func runPrune(gopts GlobalOptions) error {
}
Verbosef("counting files in repo\n")
- for _ = range repo.List(restic.DataFile, done) {
+ for _ = range repo.List(restic.DataFile, ctx.Done()) {
stats.packs++
}
@@ -180,7 +181,7 @@ func runPrune(gopts GlobalOptions) error {
}
}
- removeBytes := 0
+ removeBytes := duplicateBytes
// find packs that are unneeded
removePacks := restic.NewIDSet()
@@ -213,47 +214,33 @@ func runPrune(gopts GlobalOptions) error {
Verbosef("will delete %d packs and rewrite %d packs, this frees %s\n",
len(removePacks), len(rewritePacks), formatBytes(uint64(removeBytes)))
- err = repository.Repack(repo, rewritePacks, usedBlobs)
- if err != nil {
- return err
- }
-
- for packID := range removePacks {
- h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
- err = repo.Backend().Remove(h)
+ if len(rewritePacks) != 0 {
+ bar = newProgressMax(!gopts.Quiet, uint64(len(rewritePacks)), "packs rewritten")
+ bar.Start()
+ err = repository.Repack(repo, rewritePacks, usedBlobs, bar)
if err != nil {
- Warnf("unable to remove file %v from the repository\n", packID.Str())
+ return err
}
+ bar.Done()
}
- Verbosef("creating new index\n")
-
- stats.packs = 0
- for _ = range repo.List(restic.DataFile, done) {
- stats.packs++
- }
- bar = newProgressMax(!gopts.Quiet, uint64(stats.packs), "packs")
- idx, err = index.New(repo, bar)
- if err != nil {
- return err
- }
-
- var supersedes restic.IDs
- for idxID := range repo.List(restic.IndexFile, done) {
- h := restic.Handle{Type: restic.IndexFile, Name: idxID.String()}
- err := repo.Backend().Remove(h)
- if err != nil {
- fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", idxID.Str(), err)
+ if len(removePacks) != 0 {
+ bar = newProgressMax(!gopts.Quiet, uint64(len(removePacks)), "packs deleted")
+ bar.Start()
+ for packID := range removePacks {
+ h := restic.Handle{Type: restic.DataFile, Name: packID.String()}
+ err = repo.Backend().Remove(h)
+ if err != nil {
+ Warnf("unable to remove file %v from the repository\n", packID.Str())
+ }
+ bar.Report(restic.Stat{Blobs: 1})
}
-
- supersedes = append(supersedes, idxID)
+ bar.Done()
}
- id, err := idx.Save(repo, supersedes)
- if err != nil {
+ if err = rebuildIndex(ctx, repo); err != nil {
return err
}
- Verbosef("saved new index as %v\n", id.Str())
Verbosef("done\n")
return nil
diff --git a/src/cmds/restic/cmd_rebuild_index.go b/src/cmds/restic/cmd_rebuild_index.go
index 2dfac08..e392b80 100644
--- a/src/cmds/restic/cmd_rebuild_index.go
+++ b/src/cmds/restic/cmd_rebuild_index.go
@@ -1,7 +1,9 @@
package main
import (
- "restic/repository"
+ "context"
+ "restic"
+ "restic/index"
"github.com/spf13/cobra"
)
@@ -10,8 +12,8 @@ var cmdRebuildIndex = &cobra.Command{
Use: "rebuild-index [flags]",
Short: "build a new index file",
Long: `
-The "rebuild-index" command creates a new index by combining the index files
-into a new one.
+The "rebuild-index" command creates a new index based on the pack files in the
+repository.
`,
RunE: func(cmd *cobra.Command, args []string) error {
return runRebuildIndex(globalOptions)
@@ -34,5 +36,49 @@ func runRebuildIndex(gopts GlobalOptions) error {
return err
}
- return repository.RebuildIndex(repo)
+ ctx, cancel := context.WithCancel(gopts.ctx)
+ defer cancel()
+ return rebuildIndex(ctx, repo)
+}
+
+func rebuildIndex(ctx context.Context, repo restic.Repository) error {
+ Verbosef("counting files in repo\n")
+
+ var packs uint64
+ for _ = range repo.List(restic.DataFile, ctx.Done()) {
+ packs++
+ }
+
+ bar := newProgressMax(!globalOptions.Quiet, packs, "packs")
+ idx, err := index.New(repo, bar)
+ if err != nil {
+ return err
+ }
+
+ Verbosef("finding old index files\n")
+
+ var supersedes restic.IDs
+ for id := range repo.List(restic.IndexFile, ctx.Done()) {
+ supersedes = append(supersedes, id)
+ }
+
+ id, err := idx.Save(repo, supersedes)
+ if err != nil {
+ return err
+ }
+
+ Verbosef("saved new index as %v\n", id.Str())
+
+ Verbosef("remove %d old index files\n", len(supersedes))
+
+ for _, id := range supersedes {
+ if err := repo.Backend().Remove(restic.Handle{
+ Type: restic.IndexFile,
+ Name: id.String(),
+ }); err != nil {
+ Warnf("error removing old index %v: %v\n", id.Str(), err)
+ }
+ }
+
+ return nil
}
diff --git a/src/cmds/restic/cmd_restore.go b/src/cmds/restic/cmd_restore.go
index 9ac25bc..6a9ec95 100644
--- a/src/cmds/restic/cmd_restore.go
+++ b/src/cmds/restic/cmd_restore.go
@@ -31,6 +31,7 @@ type RestoreOptions struct {
Target string
Host string
Paths []string
+ Tags []string
}
var restoreOptions RestoreOptions
@@ -44,12 +45,13 @@ func init() {
flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
flags.StringVarP(&restoreOptions.Host, "host", "H", "", `only consider snapshots for this host when the snapshot ID is "latest"`)
+ flags.StringSliceVar(&restoreOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` for snapshot ID \"latest\"")
flags.StringSliceVar(&restoreOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path` for snapshot ID \"latest\"")
}
func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
if len(args) != 1 {
- return errors.Fatalf("no snapshot ID specified")
+ return errors.Fatal("no snapshot ID specified")
}
if opts.Target == "" {
@@ -85,7 +87,7 @@ func runRestore(opts RestoreOptions, gopts GlobalOptions, args []string) error {
var id restic.ID
if snapshotIDString == "latest" {
- id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Host)
+ id, err = restic.FindLatestSnapshot(repo, opts.Paths, opts.Tags, opts.Host)
if err != nil {
Exitf(1, "latest snapshot for criteria not found: %v Paths:%v Host:%v", err, opts.Paths, opts.Host)
}
diff --git a/src/cmds/restic/cmd_snapshots.go b/src/cmds/restic/cmd_snapshots.go
index 74a7c12..7a3fa98 100644
--- a/src/cmds/restic/cmd_snapshots.go
+++ b/src/cmds/restic/cmd_snapshots.go
@@ -1,9 +1,10 @@
package main
import (
+ "context"
+ "encoding/json"
"fmt"
- "os"
- "restic/errors"
+ "io"
"sort"
"github.com/spf13/cobra"
@@ -12,19 +13,20 @@ import (
)
var cmdSnapshots = &cobra.Command{
- Use: "snapshots",
+ Use: "snapshots [snapshotID ...]",
Short: "list all snapshots",
Long: `
-The "snapshots" command lists all snapshots stored in a repository.
+The "snapshots" command lists all snapshots stored in the repository.
`,
RunE: func(cmd *cobra.Command, args []string) error {
return runSnapshots(snapshotOptions, globalOptions, args)
},
}
-// SnapshotOptions bundle all options for the snapshots command.
+// SnapshotOptions bundles all options for the snapshots command.
type SnapshotOptions struct {
Host string
+ Tags []string
Paths []string
}
@@ -34,15 +36,12 @@ func init() {
cmdRoot.AddCommand(cmdSnapshots)
f := cmdSnapshots.Flags()
- f.StringVar(&snapshotOptions.Host, "host", "", "only print snapshots for this host")
- f.StringSliceVar(&snapshotOptions.Paths, "path", []string{}, "only print snapshots for this `path` (can be specified multiple times)")
+ f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
+ f.StringSliceVar(&snapshotOptions.Tags, "tag", nil, "only consider snapshots which include this `tag` (can be specified multiple times)")
+ f.StringSliceVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
}
func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
- if len(args) != 0 {
- return errors.Fatalf("wrong number of arguments")
- }
-
repo, err := OpenRepository(gopts)
if err != nil {
return err
@@ -56,37 +55,47 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
}
}
- tab := NewTable()
- tab.Header = fmt.Sprintf("%-8s %-19s %-10s %-10s %-3s %s", "ID", "Date", "Host", "Tags", "", "Directory")
- tab.RowFormat = "%-8s %-19s %-10s %-10s %-3s %s"
+ ctx, cancel := context.WithCancel(gopts.ctx)
+ defer cancel()
- done := make(chan struct{})
- defer close(done)
+ var list restic.Snapshots
+ for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+ list = append(list, sn)
+ }
+ sort.Sort(sort.Reverse(list))
- list := []*restic.Snapshot{}
- for id := range repo.List(restic.SnapshotFile, done) {
- sn, err := restic.LoadSnapshot(repo, id)
+ if gopts.JSON {
+ err := printSnapshotsJSON(gopts.stdout, list)
if err != nil {
- fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)
- continue
+ Warnf("error printing snapshot: %v\n", err)
}
+ return nil
+ }
+ PrintSnapshots(gopts.stdout, list)
- if restic.SamePaths(sn.Paths, opts.Paths) && (opts.Host == "" || opts.Host == sn.Hostname) {
- pos := sort.Search(len(list), func(i int) bool {
- return list[i].Time.After(sn.Time)
- })
-
- if pos < len(list) {
- list = append(list, nil)
- copy(list[pos+1:], list[pos:])
- list[pos] = sn
- } else {
- list = append(list, sn)
+ return nil
+}
+
+// PrintSnapshots prints a text table of the snapshots in list to stdout.
+func PrintSnapshots(stdout io.Writer, list restic.Snapshots) {
+
+ // Determine the max widths for host and tag.
+ maxHost, maxTag := 10, 6
+ for _, sn := range list {
+ if len(sn.Hostname) > maxHost {
+ maxHost = len(sn.Hostname)
+ }
+ for _, tag := range sn.Tags {
+ if len(tag) > maxTag {
+ maxTag = len(tag)
}
}
-
}
+ tab := NewTable()
+ tab.Header = fmt.Sprintf("%-8s %-19s %-*s %-*s %-3s %s", "ID", "Date", -maxHost, "Host", -maxTag, "Tags", "", "Directory")
+ tab.RowFormat = fmt.Sprintf("%%-8s %%-19s %%%ds %%%ds %%-3s %%s", -maxHost, -maxTag)
+
for _, sn := range list {
if len(sn.Paths) == 0 {
continue
@@ -98,6 +107,9 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
}
rows := len(sn.Paths)
+ if rows < len(sn.Tags) {
+ rows = len(sn.Tags)
+ }
treeElement := " "
if rows != 1 {
@@ -130,7 +142,29 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro
}
}
- tab.Write(os.Stdout)
+ tab.Write(stdout)
+}
- return nil
+// Snapshot helps to print Snaphots as JSON with their ID included.
+type Snapshot struct {
+ *restic.Snapshot
+
+ ID *restic.ID `json:"id"`
+}
+
+// printSnapshotsJSON writes the JSON representation of list to stdout.
+func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error {
+
+ var snapshots []Snapshot
+
+ for _, sn := range list {
+
+ k := Snapshot{
+ Snapshot: sn,
+ ID: sn.ID(),
+ }
+ snapshots = append(snapshots, k)
+ }
+
+ return json.NewEncoder(stdout).Encode(snapshots)
}
diff --git a/src/cmds/restic/cmd_tag.go b/src/cmds/restic/cmd_tag.go
new file mode 100644
index 0000000..17ed819
--- /dev/null
+++ b/src/cmds/restic/cmd_tag.go
@@ -0,0 +1,142 @@
+package main
+
+import (
+ "context"
+
+ "github.com/spf13/cobra"
+
+ "restic"
+ "restic/debug"
+ "restic/errors"
+ "restic/repository"
+)
+
+var cmdTag = &cobra.Command{
+ Use: "tag [flags] [snapshot-ID ...]",
+ Short: "modifies tags on snapshots",
+ Long: `
+The "tag" command allows you to modify tags on exiting snapshots.
+
+You can either set/replace the entire set of tags on a snapshot, or
+add tags to/remove tags from the existing set.
+
+When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified.
+`,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return runTag(tagOptions, globalOptions, args)
+ },
+}
+
+// TagOptions bundles all options for the 'tag' command.
+type TagOptions struct {
+ Host string
+ Paths []string
+ Tags []string
+ SetTags []string
+ AddTags []string
+ RemoveTags []string
+}
+
+var tagOptions TagOptions
+
+func init() {
+ cmdRoot.AddCommand(cmdTag)
+
+ tagFlags := cmdTag.Flags()
+ tagFlags.StringSliceVar(&tagOptions.SetTags, "set", nil, "`tag` which will replace the existing tags (can be given multiple times)")
+ tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)")
+ tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)")
+
+ tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", "only consider snapshots for this `host`, when no snapshot ID is given")
+ tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given")
+ tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+}
+
+func changeTags(repo *repository.Repository, sn *restic.Snapshot, setTags, addTags, removeTags []string) (bool, error) {
+ var changed bool
+
+ if len(setTags) != 0 {
+ // Setting the tag to an empty string really means no tags.
+ if len(setTags) == 1 && setTags[0] == "" {
+ setTags = nil
+ }
+ sn.Tags = setTags
+ changed = true
+ } else {
+ changed = sn.AddTags(addTags)
+ if sn.RemoveTags(removeTags) {
+ changed = true
+ }
+ }
+
+ if changed {
+ // Retain the original snapshot id over all tag changes.
+ if sn.Original == nil {
+ sn.Original = sn.ID()
+ }
+
+ // Save the new snapshot.
+ id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
+ if err != nil {
+ return false, err
+ }
+
+ debug.Log("new snapshot saved as %v", id.Str())
+
+ if err = repo.Flush(); err != nil {
+ return false, err
+ }
+
+ // Remove the old snapshot.
+ h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+ if err = repo.Backend().Remove(h); err != nil {
+ return false, err
+ }
+
+ debug.Log("old snapshot %v removed", sn.ID())
+ }
+ return changed, nil
+}
+
+func runTag(opts TagOptions, gopts GlobalOptions, args []string) error {
+ if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 {
+ return errors.Fatal("nothing to do!")
+ }
+ if len(opts.SetTags) != 0 && (len(opts.AddTags) != 0 || len(opts.RemoveTags) != 0) {
+ return errors.Fatal("--set and --add/--remove cannot be given at the same time")
+ }
+
+ repo, err := OpenRepository(gopts)
+ if err != nil {
+ return err
+ }
+
+ if !gopts.NoLock {
+ Verbosef("Create exclusive lock for repository\n")
+ lock, err := lockRepoExclusive(repo)
+ defer unlockRepo(lock)
+ if err != nil {
+ return err
+ }
+ }
+
+ changeCnt := 0
+ ctx, cancel := context.WithCancel(gopts.ctx)
+ defer cancel()
+ for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
+ changed, err := changeTags(repo, sn, opts.SetTags, opts.AddTags, opts.RemoveTags)
+ if err != nil {
+ Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", sn.ID(), err)
+ continue
+ }
+ if changed {
+ changeCnt++
+ }
+ }
+ if changeCnt == 0 {
+ Verbosef("No snapshots were modified\n")
+ } else {
+ Verbosef("Modified tags on %v snapshots\n", changeCnt)
+ }
+ return nil
+}
diff --git a/src/cmds/restic/cmd_unlock.go b/src/cmds/restic/cmd_unlock.go
index 38004ea..6601909 100644
--- a/src/cmds/restic/cmd_unlock.go
+++ b/src/cmds/restic/cmd_unlock.go
@@ -27,7 +27,7 @@ var unlockOptions UnlockOptions
func init() {
cmdRoot.AddCommand(unlockCmd)
- unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "Remove all locks, even non-stale ones")
+ unlockCmd.Flags().BoolVar(&unlockOptions.RemoveAll, "remove-all", false, "remove all locks, even non-stale ones")
}
func runUnlock(opts UnlockOptions, gopts GlobalOptions) error {
diff --git a/src/cmds/restic/cmd_version.go b/src/cmds/restic/cmd_version.go
index 9290af0..d13ec06 100644
--- a/src/cmds/restic/cmd_version.go
+++ b/src/cmds/restic/cmd_version.go
@@ -9,7 +9,7 @@ import (
var versionCmd = &cobra.Command{
Use: "version",
- Short: "Print version information",
+ Short: "print version information",
Long: `
The "version" command prints detailed information about the build environment
and the version of this software.
diff --git a/src/cmds/restic/find.go b/src/cmds/restic/find.go
new file mode 100644
index 0000000..fa9d716
--- /dev/null
+++ b/src/cmds/restic/find.go
@@ -0,0 +1,78 @@
+package main
+
+import (
+ "context"
+
+ "restic"
+ "restic/repository"
+)
+
+// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
+func FindFilteredSnapshots(ctx context.Context, repo *repository.Repository, host string, tags []string, paths []string, snapshotIDs []string) <-chan *restic.Snapshot {
+ out := make(chan *restic.Snapshot)
+ go func() {
+ defer close(out)
+ if len(snapshotIDs) != 0 {
+ var (
+ id restic.ID
+ usedFilter bool
+ err error
+ )
+ ids := make(restic.IDs, 0, len(snapshotIDs))
+ // Process all snapshot IDs given as arguments.
+ for _, s := range snapshotIDs {
+ if s == "latest" {
+ id, err = restic.FindLatestSnapshot(repo, paths, tags, host)
+ if err != nil {
+ Warnf("Ignoring %q, no snapshot matched given filter (Paths:%v Tags:%v Host:%v)\n", s, paths, tags, host)
+ usedFilter = true
+ continue
+ }
+ } else {
+ id, err = restic.FindSnapshot(repo, s)
+ if err != nil {
+ Warnf("Ignoring %q, it is not a snapshot id\n", s)
+ continue
+ }
+ }
+ ids = append(ids, id)
+ }
+
+ // Give the user some indication their filters are not used.
+ if !usedFilter && (host != "" || len(tags) != 0 || len(paths) != 0) {
+ Warnf("Ignoring filters as there are explicit snapshot ids given\n")
+ }
+
+ for _, id := range ids.Uniq() {
+ sn, err := restic.LoadSnapshot(repo, id)
+ if err != nil {
+ Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
+ continue
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case out <- sn:
+ }
+ }
+ return
+ }
+
+ for id := range repo.List(restic.SnapshotFile, ctx.Done()) {
+ sn, err := restic.LoadSnapshot(repo, id)
+ if err != nil {
+ Warnf("Ignoring %q, could not load snapshot: %v\n", id, err)
+ continue
+ }
+ if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !sn.HasPaths(paths) {
+ continue
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case out <- sn:
+ }
+ }
+ }()
+ return out
+}
diff --git a/src/cmds/restic/format.go b/src/cmds/restic/format.go
index 68fa29f..16c3746 100644
--- a/src/cmds/restic/format.go
+++ b/src/cmds/restic/format.go
@@ -2,7 +2,11 @@ package main
import (
"fmt"
+ "os"
+ "path/filepath"
"time"
+
+ "restic"
)
func formatBytes(c uint64) string {
@@ -58,3 +62,23 @@ func formatDuration(d time.Duration) string {
sec := uint64(d / time.Second)
return formatSeconds(sec)
}
+
+func formatNode(prefix string, n *restic.Node, long bool) string {
+ if !long {
+ return filepath.Join(prefix, n.Name)
+ }
+
+ switch n.Type {
+ case "file":
+ return fmt.Sprintf("%s %5d %5d %6d %s %s",
+ n.Mode, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
+ case "dir":
+ return fmt.Sprintf("%s %5d %5d %6d %s %s",
+ n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name))
+ case "symlink":
+ return fmt.Sprintf("%s %5d %5d %6d %s %s -> %s",
+ n.Mode|os.ModeSymlink, n.UID, n.GID, n.Size, n.ModTime.Format(TimeFormat), filepath.Join(prefix, n.Name), n.LinkTarget)
+ default:
+ return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
+ }
+}
diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go
index 8757130..4acd790 100644
--- a/src/cmds/restic/global.go
+++ b/src/cmds/restic/global.go
@@ -1,6 +1,7 @@
package main
import (
+ "context"
"fmt"
"io"
"io/ioutil"
@@ -31,7 +32,9 @@ type GlobalOptions struct {
PasswordFile string
Quiet bool
NoLock bool
+ JSON bool
+ ctx context.Context
password string
stdout io.Writer
stderr io.Writer
@@ -48,11 +51,19 @@ func init() {
globalOptions.password = pw
}
+ var cancel context.CancelFunc
+ globalOptions.ctx, cancel = context.WithCancel(context.Background())
+ AddCleanupHandler(func() error {
+ cancel()
+ return nil
+ })
+
f := cmdRoot.PersistentFlags()
f.StringVarP(&globalOptions.Repo, "repo", "r", os.Getenv("RESTIC_REPOSITORY"), "repository to backup to or restore from (default: $RESTIC_REPOSITORY)")
f.StringVarP(&globalOptions.PasswordFile, "password-file", "p", "", "read the repository password from a file")
f.BoolVarP(&globalOptions.Quiet, "quiet", "q", false, "do not output comprehensive progress report")
f.BoolVar(&globalOptions.NoLock, "no-lock", false, "do not lock the repo, this allows some operations on read-only repos")
+ f.BoolVarP(&globalOptions.JSON, "json", "", false, "set output mode to JSON for commands that support it")
restoreTerminal()
}
@@ -80,6 +91,14 @@ func stdoutIsTerminal() bool {
return terminal.IsTerminal(int(os.Stdout.Fd()))
}
+func stdoutTerminalWidth() int {
+ w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+ if err != nil {
+ return 0
+ }
+ return w
+}
+
// restoreTerminal installs a cleanup handler that restores the previous
// terminal state on exit.
func restoreTerminal() {
@@ -108,8 +127,7 @@ func restoreTerminal() {
// current windows cmd shell.
func ClearLine() string {
if runtime.GOOS == "windows" {
- w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
- if err == nil {
+ if w := stdoutTerminalWidth(); w > 0 {
return strings.Repeat(" ", w-1) + "\r"
}
return ""
diff --git a/src/cmds/restic/integration_helpers_test.go b/src/cmds/restic/integration_helpers_test.go
index 72fb09f..ad6acc8 100644
--- a/src/cmds/restic/integration_helpers_test.go
+++ b/src/cmds/restic/integration_helpers_test.go
@@ -1,6 +1,7 @@
package main
import (
+ "context"
"fmt"
"io/ioutil"
"os"
@@ -15,6 +16,7 @@ import (
type dirEntry struct {
path string
fi os.FileInfo
+ link uint64
}
func walkDir(dir string) <-chan *dirEntry {
@@ -36,6 +38,7 @@ func walkDir(dir string) <-chan *dirEntry {
ch <- &dirEntry{
path: name,
fi: info,
+ link: nlink(info),
}
return nil
@@ -192,6 +195,7 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions))
gopts := GlobalOptions{
Repo: env.repo,
Quiet: true,
+ ctx: context.Background(),
password: TestPassword,
stdout: os.Stdout,
stderr: os.Stderr,
diff --git a/src/cmds/restic/integration_helpers_unix_test.go b/src/cmds/restic/integration_helpers_unix_test.go
index a182898..01a0fd5 100644
--- a/src/cmds/restic/integration_helpers_unix_test.go
+++ b/src/cmds/restic/integration_helpers_unix_test.go
@@ -4,7 +4,9 @@ package main
import (
"fmt"
+ "io/ioutil"
"os"
+ "path/filepath"
"syscall"
)
@@ -37,5 +39,37 @@ func (e *dirEntry) equals(other *dirEntry) bool {
return false
}
+ if stat.Nlink != stat2.Nlink {
+ fmt.Fprintf(os.Stderr, "%v: Number of links do not match (%v != %v)\n", e.path, stat.Nlink, stat2.Nlink)
+ return false
+ }
+
return true
}
+
+func nlink(info os.FileInfo) uint64 {
+ stat, _ := info.Sys().(*syscall.Stat_t)
+ return uint64(stat.Nlink)
+}
+
+func inode(info os.FileInfo) uint64 {
+ stat, _ := info.Sys().(*syscall.Stat_t)
+ return uint64(stat.Ino)
+}
+
+func createFileSetPerHardlink(dir string) map[uint64][]string {
+ var stat syscall.Stat_t
+ linkTests := make(map[uint64][]string)
+ files, err := ioutil.ReadDir(dir)
+ if err != nil {
+ return nil
+ }
+ for _, f := range files {
+
+ if err := syscall.Stat(filepath.Join(dir, f.Name()), &stat); err != nil {
+ return nil
+ }
+ linkTests[uint64(stat.Ino)] = append(linkTests[uint64(stat.Ino)], f.Name())
+ }
+ return linkTests
+}
diff --git a/src/cmds/restic/integration_helpers_windows_test.go b/src/cmds/restic/integration_helpers_windows_test.go
index d67e9ca..9e3fbac 100644
--- a/src/cmds/restic/integration_helpers_windows_test.go
+++ b/src/cmds/restic/integration_helpers_windows_test.go
@@ -4,6 +4,7 @@ package main
import (
"fmt"
+ "io/ioutil"
"os"
)
@@ -25,3 +26,24 @@ func (e *dirEntry) equals(other *dirEntry) bool {
return true
}
+
+func nlink(info os.FileInfo) uint64 {
+ return 1
+}
+
+func inode(info os.FileInfo) uint64 {
+ return uint64(0)
+}
+
+func createFileSetPerHardlink(dir string) map[uint64][]string {
+ linkTests := make(map[uint64][]string)
+ files, err := ioutil.ReadDir(dir)
+ if err != nil {
+ return nil
+ }
+ for i, f := range files {
+ linkTests[uint64(i)] = append(linkTests[uint64(i)], f.Name())
+ i++
+ }
+ return linkTests
+}
diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go
index 4263677..0adf495 100644
--- a/src/cmds/restic/integration_test.go
+++ b/src/cmds/restic/integration_test.go
@@ -4,6 +4,7 @@ import (
"bufio"
"bytes"
"crypto/rand"
+ "encoding/json"
"fmt"
"io"
"io/ioutil"
@@ -141,7 +142,9 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
globalOptions.Quiet = quiet
}()
- OK(t, runLs(gopts, []string{snapshotID}))
+ opts := LsOptions{}
+
+ OK(t, runLs(opts, gopts, []string{snapshotID}))
return strings.Split(string(buf.Bytes()), "\n")
}
@@ -160,6 +163,32 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string {
return strings.Split(string(buf.Bytes()), "\n")
}
+func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snapmap map[restic.ID]Snapshot) {
+ buf := bytes.NewBuffer(nil)
+ globalOptions.stdout = buf
+ globalOptions.JSON = true
+ defer func() {
+ globalOptions.stdout = os.Stdout
+ globalOptions.JSON = gopts.JSON
+ }()
+
+ opts := SnapshotOptions{}
+
+ OK(t, runSnapshots(opts, globalOptions, []string{}))
+
+ snapshots := []Snapshot{}
+ OK(t, json.Unmarshal(buf.Bytes(), &snapshots))
+
+ snapmap = make(map[restic.ID]Snapshot, len(snapshots))
+ for _, sn := range snapshots {
+ snapmap[*sn.ID] = sn
+ if newest == nil || sn.Time.After(newest.Time) {
+ newest = &sn
+ }
+ }
+ return
+}
+
func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
opts := ForgetOptions{}
OK(t, runForget(opts, gopts, args))
@@ -516,23 +545,23 @@ func TestBackupExclude(t *testing.T) {
testRunBackup(t, []string{datadir}, opts, gopts)
snapshots, snapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
files := testRunLs(t, gopts, snapshotID)
- Assert(t, includes(files, filepath.Join("testdata", "foo.tar.gz")),
+ Assert(t, includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
"expected file %q in first snapshot, but it's not included", "foo.tar.gz")
opts.Excludes = []string{"*.tar.gz"}
testRunBackup(t, []string{datadir}, opts, gopts)
snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
files = testRunLs(t, gopts, snapshotID)
- Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+ Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
opts.Excludes = []string{"*.tar.gz", "private/secret"}
testRunBackup(t, []string{datadir}, opts, gopts)
snapshots, snapshotID = lastSnapshot(snapshots, loadSnapshotMap(t, gopts))
files = testRunLs(t, gopts, snapshotID)
- Assert(t, !includes(files, filepath.Join("testdata", "foo.tar.gz")),
+ Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "foo.tar.gz")),
"expected file %q not in first snapshot, but it's included", "foo.tar.gz")
- Assert(t, !includes(files, filepath.Join("testdata", "private", "secret", "passwords.txt")),
+ Assert(t, !includes(files, filepath.Join(string(filepath.Separator), "testdata", "private", "secret", "passwords.txt")),
"expected file %q not in first snapshot, but it's included", "passwords.txt")
})
}
@@ -602,6 +631,105 @@ func TestIncrementalBackup(t *testing.T) {
})
}
+func TestBackupTags(t *testing.T) {
+ withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+ datafile := filepath.Join("testdata", "backup-data.tar.gz")
+ testRunInit(t, gopts)
+ SetupTarTestFixture(t, env.testdata, datafile)
+
+ opts := BackupOptions{}
+
+ testRunBackup(t, []string{env.testdata}, opts, gopts)
+ testRunCheck(t, gopts)
+ newest, _ := testRunSnapshots(t, gopts)
+ Assert(t, newest != nil, "expected a new backup, got nil")
+ Assert(t, len(newest.Tags) == 0,
+ "expected no tags, got %v", newest.Tags)
+
+ opts.Tags = []string{"NL"}
+ testRunBackup(t, []string{env.testdata}, opts, gopts)
+ testRunCheck(t, gopts)
+ newest, _ = testRunSnapshots(t, gopts)
+ Assert(t, newest != nil, "expected a new backup, got nil")
+ Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+ "expected one NL tag, got %v", newest.Tags)
+ })
+}
+
+func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
+ OK(t, runTag(opts, gopts, []string{}))
+}
+
+func TestTag(t *testing.T) {
+ withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+ datafile := filepath.Join("testdata", "backup-data.tar.gz")
+ testRunInit(t, gopts)
+ SetupTarTestFixture(t, env.testdata, datafile)
+
+ testRunBackup(t, []string{env.testdata}, BackupOptions{}, gopts)
+ testRunCheck(t, gopts)
+ newest, _ := testRunSnapshots(t, gopts)
+ Assert(t, newest != nil, "expected a new backup, got nil")
+ Assert(t, len(newest.Tags) == 0,
+ "expected no tags, got %v", newest.Tags)
+ Assert(t, newest.Original == nil,
+ "expected original ID to be nil, got %v", newest.Original)
+ originalID := *newest.ID
+
+ testRunTag(t, TagOptions{SetTags: []string{"NL"}}, gopts)
+ testRunCheck(t, gopts)
+ newest, _ = testRunSnapshots(t, gopts)
+ Assert(t, newest != nil, "expected a new backup, got nil")
+ Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL",
+ "set failed, expected one NL tag, got %v", newest.Tags)
+ Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+ Assert(t, *newest.Original == originalID,
+ "expected original ID to be set to the first snapshot id")
+
+ testRunTag(t, TagOptions{AddTags: []string{"CH"}}, gopts)
+ testRunCheck(t, gopts)
+ newest, _ = testRunSnapshots(t, gopts)
+ Assert(t, newest != nil, "expected a new backup, got nil")
+ Assert(t, len(newest.Tags) == 2 && newest.Tags[0] == "NL" && newest.Tags[1] == "CH",
+ "add failed, expected CH,NL tags, got %v", newest.Tags)
+ Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+ Assert(t, *newest.Original == originalID,
+ "expected original ID to be set to the first snapshot id")
+
+ testRunTag(t, TagOptions{RemoveTags: []string{"NL"}}, gopts)
+ testRunCheck(t, gopts)
+ newest, _ = testRunSnapshots(t, gopts)
+ Assert(t, newest != nil, "expected a new backup, got nil")
+ Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "CH",
+ "remove failed, expected one CH tag, got %v", newest.Tags)
+ Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+ Assert(t, *newest.Original == originalID,
+ "expected original ID to be set to the first snapshot id")
+
+ testRunTag(t, TagOptions{AddTags: []string{"US", "RU"}}, gopts)
+ testRunTag(t, TagOptions{RemoveTags: []string{"CH", "US", "RU"}}, gopts)
+ testRunCheck(t, gopts)
+ newest, _ = testRunSnapshots(t, gopts)
+ Assert(t, newest != nil, "expected a new backup, got nil")
+ Assert(t, len(newest.Tags) == 0,
+ "expected no tags, got %v", newest.Tags)
+ Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+ Assert(t, *newest.Original == originalID,
+ "expected original ID to be set to the first snapshot id")
+
+ // Check special case of removing all tags.
+ testRunTag(t, TagOptions{SetTags: []string{""}}, gopts)
+ testRunCheck(t, gopts)
+ newest, _ = testRunSnapshots(t, gopts)
+ Assert(t, newest != nil, "expected a new backup, got nil")
+ Assert(t, len(newest.Tags) == 0,
+ "expected no tags, got %v", newest.Tags)
+ Assert(t, newest.Original != nil, "expected original snapshot id, got nil")
+ Assert(t, *newest.Original == originalID,
+ "expected original ID to be set to the first snapshot id")
+ })
+}
+
func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
buf := bytes.NewBuffer(nil)
@@ -836,7 +964,7 @@ func TestRestoreWithPermissionFailure(t *testing.T) {
testRunRestore(t, gopts, filepath.Join(env.base, "restore"), snapshots[0])
- // make sure that all files have been restored, regardeless of any
+ // make sure that all files have been restored, regardless of any
// permission errors
files := testRunLs(t, gopts, snapshots[0].String())
for _, filename := range files {
@@ -935,7 +1063,7 @@ func TestRebuildIndex(t *testing.T) {
}
if !strings.Contains(out, "restic rebuild-index") {
- t.Fatalf("did not find hint for rebuild-index comman")
+ t.Fatalf("did not find hint for rebuild-index command")
}
testRunRebuildIndex(t, gopts)
@@ -1011,3 +1139,100 @@ func TestPrune(t *testing.T) {
testRunCheck(t, gopts)
})
}
+
+func TestHardLink(t *testing.T) {
+ // this test assumes a test set with a single directory containing hard linked files
+ withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
+ datafile := filepath.Join("testdata", "test.hl.tar.gz")
+ fd, err := os.Open(datafile)
+ if os.IsNotExist(errors.Cause(err)) {
+ t.Skipf("unable to find data file %q, skipping", datafile)
+ return
+ }
+ OK(t, err)
+ OK(t, fd.Close())
+
+ testRunInit(t, gopts)
+
+ SetupTarTestFixture(t, env.testdata, datafile)
+
+ linkTests := createFileSetPerHardlink(env.testdata)
+
+ opts := BackupOptions{}
+
+ // first backup
+ testRunBackup(t, []string{env.testdata}, opts, gopts)
+ snapshotIDs := testRunList(t, "snapshots", gopts)
+ Assert(t, len(snapshotIDs) == 1,
+ "expected one snapshot, got %v", snapshotIDs)
+
+ testRunCheck(t, gopts)
+
+ // restore all backups and compare
+ for i, snapshotID := range snapshotIDs {
+ restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
+ t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
+ testRunRestore(t, gopts, restoredir, snapshotIDs[0])
+ Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")),
+ "directories are not equal")
+
+ linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
+ Assert(t, linksEqual(linkTests, linkResults),
+ "links are not equal")
+ }
+
+ testRunCheck(t, gopts)
+ })
+}
+
+func linksEqual(source, dest map[uint64][]string) bool {
+ for _, vs := range source {
+ found := false
+ for kd, vd := range dest {
+ if linkEqual(vs, vd) {
+ delete(dest, kd)
+ found = true
+ break
+ }
+ }
+ if !found {
+ return false
+ }
+ }
+
+ if len(dest) != 0 {
+ return false
+ }
+
+ return true
+}
+
+func linkEqual(source, dest []string) bool {
+ // equal if sliced are equal without considering order
+ if source == nil && dest == nil {
+ return true
+ }
+
+ if source == nil || dest == nil {
+ return false
+ }
+
+ if len(source) != len(dest) {
+ return false
+ }
+
+ for i := range source {
+ found := false
+ for j := range dest {
+ if source[i] == dest[j] {
+ found = true
+ break
+ }
+ }
+ if !found {
+ return false
+ }
+ }
+
+ return true
+}
diff --git a/src/cmds/restic/testdata/test.hl.tar.gz b/src/cmds/restic/testdata/test.hl.tar.gz
new file mode 100644
index 0000000..3025781
--- /dev/null
+++ b/src/cmds/restic/testdata/test.hl.tar.gz
Binary files differ
diff --git a/src/restic/archiver/archive_reader.go b/src/restic/archiver/archive_reader.go
index 08a7fb0..6ed72ab 100644
--- a/src/restic/archiver/archive_reader.go
+++ b/src/restic/archiver/archive_reader.go
@@ -11,11 +11,22 @@ import (
"github.com/restic/chunker"
)
-// ArchiveReader reads from the reader and archives the data. Returned is the
-// resulting snapshot and its ID.
-func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, name string, tags []string) (*restic.Snapshot, restic.ID, error) {
+// Reader allows saving a stream of data to the repository.
+type Reader struct {
+ restic.Repository
+
+ Tags []string
+ Hostname string
+}
+
+// Archive reads data from the reader and saves it to the repo.
+func (r *Reader) Archive(name string, rd io.Reader, p *restic.Progress) (*restic.Snapshot, restic.ID, error) {
+ if name == "" {
+ return nil, restic.ID{}, errors.New("no filename given")
+ }
+
debug.Log("start archiving %s", name)
- sn, err := restic.NewSnapshot([]string{name}, tags)
+ sn, err := restic.NewSnapshot([]string{name}, r.Tags, r.Hostname)
if err != nil {
return nil, restic.ID{}, err
}
@@ -23,9 +34,10 @@ func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, nam
p.Start()
defer p.Done()
+ repo := r.Repository
chnker := chunker.New(rd, repo.Config().ChunkerPolynomial)
- var ids restic.IDs
+ ids := restic.IDs{}
var fileSize uint64
for {
diff --git a/src/restic/archiver/archive_reader_test.go b/src/restic/archiver/archive_reader_test.go
index c24a0be..a8ab186 100644
--- a/src/restic/archiver/archive_reader_test.go
+++ b/src/restic/archiver/archive_reader_test.go
@@ -2,9 +2,11 @@ package archiver
import (
"bytes"
+ "errors"
"io"
"math/rand"
"restic"
+ "restic/checker"
"restic/repository"
"testing"
)
@@ -77,7 +79,13 @@ func TestArchiveReader(t *testing.T) {
f := fakeFile(t, seed, size)
- sn, id, err := ArchiveReader(repo, nil, f, "fakefile", []string{"test"})
+ r := &Reader{
+ Repository: repo,
+ Hostname: "localhost",
+ Tags: []string{"test"},
+ }
+
+ sn, id, err := r.Archive("fakefile", f, nil)
if err != nil {
t.Fatalf("ArchiveReader() returned error %v", err)
}
@@ -89,6 +97,80 @@ func TestArchiveReader(t *testing.T) {
t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str())
checkSavedFile(t, repo, *sn.Tree, "fakefile", fakeFile(t, seed, size))
+
+ checker.TestCheckRepo(t, repo)
+}
+
+func TestArchiveReaderNull(t *testing.T) {
+ repo, cleanup := repository.TestRepository(t)
+ defer cleanup()
+
+ r := &Reader{
+ Repository: repo,
+ Hostname: "localhost",
+ Tags: []string{"test"},
+ }
+
+ sn, id, err := r.Archive("fakefile", bytes.NewReader(nil), nil)
+ if err != nil {
+ t.Fatalf("ArchiveReader() returned error %v", err)
+ }
+
+ if id.IsNull() {
+ t.Fatalf("ArchiveReader() returned null ID")
+ }
+
+ t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str())
+
+ checker.TestCheckRepo(t, repo)
+}
+
+type errReader string
+
+func (e errReader) Read([]byte) (int, error) {
+ return 0, errors.New(string(e))
+}
+
+func countSnapshots(t testing.TB, repo restic.Repository) int {
+ done := make(chan struct{})
+ defer close(done)
+
+ snapshots := 0
+ for range repo.List(restic.SnapshotFile, done) {
+ snapshots++
+ }
+ return snapshots
+}
+
+func TestArchiveReaderError(t *testing.T) {
+ repo, cleanup := repository.TestRepository(t)
+ defer cleanup()
+
+ r := &Reader{
+ Repository: repo,
+ Hostname: "localhost",
+ Tags: []string{"test"},
+ }
+
+ sn, id, err := r.Archive("fakefile", errReader("error returned by reading stdin"), nil)
+ if err == nil {
+ t.Errorf("expected error not returned")
+ }
+
+ if sn != nil {
+ t.Errorf("Snapshot should be nil, but isn't")
+ }
+
+ if !id.IsNull() {
+ t.Errorf("id should be null, but %v returned", id.Str())
+ }
+
+ n := countSnapshots(t, repo)
+ if n > 0 {
+ t.Errorf("expected zero snapshots, but got %d", n)
+ }
+
+ checker.TestCheckRepo(t, repo)
}
func BenchmarkArchiveReader(t *testing.B) {
@@ -103,11 +185,17 @@ func BenchmarkArchiveReader(t *testing.B) {
t.Fatal(err)
}
+ r := &Reader{
+ Repository: repo,
+ Hostname: "localhost",
+ Tags: []string{"test"},
+ }
+
t.SetBytes(size)
t.ResetTimer()
for i := 0; i < t.N; i++ {
- _, _, err := ArchiveReader(repo, nil, bytes.NewReader(buf), "fakefile", []string{"test"})
+ _, _, err := r.Archive("fakefile", bytes.NewReader(buf), nil)
if err != nil {
t.Fatal(err)
}
diff --git a/src/restic/archiver/archiver.go b/src/restic/archiver/archiver.go
index b579524..e90cfd8 100644
--- a/src/restic/archiver/archiver.go
+++ b/src/restic/archiver/archiver.go
@@ -142,7 +142,7 @@ func (arch *Archiver) reloadFileIfChanged(node *restic.Node, file fs.File) (*res
node, err = restic.NodeFromFileInfo(node.Path, fi)
if err != nil {
debug.Log("restic.NodeFromFileInfo returned error for %v: %v", node.Path, err)
- return nil, err
+ arch.Warn(node.Path, fi, err)
}
return node, nil
@@ -275,11 +275,8 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-
node, err := restic.NodeFromFileInfo(e.Fullpath(), e.Info())
if err != nil {
- // TODO: integrate error reporting
debug.Log("restic.NodeFromFileInfo returned error for %v: %v", node.Path, err)
- e.Result() <- nil
- p.Report(restic.Stat{Errors: 1})
- continue
+ arch.Warn(e.Fullpath(), e.Info(), err)
}
// try to use old node, if present
@@ -307,11 +304,11 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *restic.Progress, done <-
// otherwise read file normally
if node.Type == "file" && len(node.Content) == 0 {
- debug.Log(" read and save %v, content: %v", e.Path(), node.Content)
+ debug.Log(" read and save %v", e.Path())
node, err = arch.SaveFile(p, node)
if err != nil {
- // TODO: integrate error reporting
fmt.Fprintf(os.Stderr, "error for %v: %v\n", node.Path, err)
+ arch.Warn(e.Path(), nil, err)
// ignore this file
e.Result() <- nil
p.Report(restic.Stat{Errors: 1})
@@ -371,25 +368,28 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *restic.Progress, done <-c
// else insert node
node := res.(*restic.Node)
- tree.Insert(node)
if node.Type == "dir" {
debug.Log("got tree node for %s: %v", node.Path, node.Subtree)
+ if node.Subtree == nil {
+ debug.Log("subtree is nil for node %v", node.Path)
+ continue
+ }
+
if node.Subtree.IsNull() {
panic("invalid null subtree restic.ID")
}
}
+ tree.Insert(node)
}
node := &restic.Node{}
if dir.Path() != "" && dir.Info() != nil {
- n, err := restic.NodeFromFileInfo(dir.Path(), dir.Info())
+ n, err := restic.NodeFromFileInfo(dir.Fullpath(), dir.Info())
if err != nil {
- n.Error = err.Error()
- dir.Result() <- n
- continue
+ arch.Warn(dir.Path(), dir.Info(), err)
}
node = n
}
@@ -634,7 +634,7 @@ func (p baseNameSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
// Snapshot creates a snapshot of the given paths. If parentrestic.ID is set, this is
// used to compare the files to the ones archived at the time this snapshot was
// taken.
-func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentID *restic.ID) (*restic.Snapshot, restic.ID, error) {
+func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, hostname string, parentID *restic.ID) (*restic.Snapshot, restic.ID, error) {
paths = unique(paths)
sort.Sort(baseNameSlice(paths))
@@ -650,7 +650,7 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
defer p.Done()
// create new snapshot
- sn, err := restic.NewSnapshot(paths, tags)
+ sn, err := restic.NewSnapshot(paths, tags, hostname)
if err != nil {
return nil, restic.ID{}, err
}
@@ -734,6 +734,21 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
return nil, restic.ID{}, err
}
+ // receive the top-level tree
+ root := (<-resCh).(*restic.Node)
+ debug.Log("root node received: %v", root.Subtree.Str())
+ sn.Tree = root.Subtree
+
+ // load top-level tree again to see if it is empty
+ toptree, err := arch.repo.LoadTree(*root.Subtree)
+ if err != nil {
+ return nil, restic.ID{}, err
+ }
+
+ if len(toptree.Nodes) == 0 {
+ return nil, restic.ID{}, errors.Fatal("no files/dirs saved, refusing to create empty snapshot")
+ }
+
// save index
err = arch.repo.SaveIndex()
if err != nil {
@@ -743,11 +758,6 @@ func (arch *Archiver) Snapshot(p *restic.Progress, paths, tags []string, parentI
debug.Log("saved indexes")
- // receive the top-level tree
- root := (<-resCh).(*restic.Node)
- debug.Log("root node received: %v", root.Subtree.Str())
- sn.Tree = root.Subtree
-
// save snapshot
id, err := arch.repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
if err != nil {
diff --git a/src/restic/archiver/archiver_test.go b/src/restic/archiver/archiver_test.go
index 5c47b19..72e3b3a 100644
--- a/src/restic/archiver/archiver_test.go
+++ b/src/restic/archiver/archiver_test.go
@@ -104,7 +104,7 @@ func archiveDirectory(b testing.TB) {
arch := archiver.New(repo)
- _, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil, nil)
+ _, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil, "localhost", nil)
OK(b, err)
b.Logf("snapshot archived as %v", id)
@@ -220,7 +220,7 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) {
errChannels := [](<-chan error){}
- // interweaved processing of subsequent chunks
+ // interwoven processing of subsequent chunks
maxParallel := 2*duplication - 1
barrier := make(chan struct{}, maxParallel)
@@ -294,3 +294,23 @@ func assertNoUnreferencedPacks(t *testing.T, chkr *checker.Checker) {
OK(t, err)
}
}
+
+func TestArchiveEmptySnapshot(t *testing.T) {
+ repo, cleanup := repository.TestRepository(t)
+ defer cleanup()
+
+ arch := archiver.New(repo)
+
+ sn, id, err := arch.Snapshot(nil, []string{"file-does-not-exist-123123213123", "file2-does-not-exist-too-123123123"}, nil, "localhost", nil)
+ if err == nil {
+ t.Errorf("expected error for empty snapshot, got nil")
+ }
+
+ if !id.IsNull() {
+ t.Errorf("expected null ID for empty snapshot, got %v", id.Str())
+ }
+
+ if sn != nil {
+ t.Errorf("expected null snapshot for empty snapshot, got %v", sn)
+ }
+}
diff --git a/src/restic/archiver/testing.go b/src/restic/archiver/testing.go
index fef61f6..aad8ea1 100644
--- a/src/restic/archiver/testing.go
+++ b/src/restic/archiver/testing.go
@@ -8,7 +8,7 @@ import (
// TestSnapshot creates a new snapshot of path.
func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *restic.ID) *restic.Snapshot {
arch := New(repo)
- sn, _, err := arch.Snapshot(nil, []string{path}, []string{"test"}, parent)
+ sn, _, err := arch.Snapshot(nil, []string{path}, []string{"test"}, "localhost", parent)
if err != nil {
t.Fatal(err)
}
diff --git a/src/restic/backend/rest/rest.go b/src/restic/backend/rest/rest.go
index 7121de9..2c13387 100644
--- a/src/restic/backend/rest/rest.go
+++ b/src/restic/backend/rest/rest.go
@@ -17,7 +17,7 @@ import (
"restic/backend"
)
-const connLimit = 10
+const connLimit = 40
// make sure the rest backend implements restic.Backend
var _ restic.Backend = &restBackend{}
diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go
index 18f84ae..c7b92c8 100644
--- a/src/restic/backend/s3/s3.go
+++ b/src/restic/backend/s3/s3.go
@@ -3,6 +3,7 @@ package s3
import (
"bytes"
"io"
+ "net/http"
"path"
"restic"
"strings"
@@ -15,7 +16,7 @@ import (
"restic/debug"
)
-const connLimit = 10
+const connLimit = 40
// s3 is a backend which stores the data on an S3 endpoint.
type s3 struct {
@@ -36,6 +37,10 @@ func Open(cfg Config) (restic.Backend, error) {
}
be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix}
+
+ tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
+ client.SetCustomTransport(tr)
+
be.createConnections()
found, err := client.BucketExists(cfg.Bucket)
@@ -104,6 +109,18 @@ func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
return errors.Wrap(err, "client.PutObject")
}
+// wrapReader wraps an io.ReadCloser to run an additional function on Close.
+type wrapReader struct {
+ io.ReadCloser
+ f func()
+}
+
+func (wr wrapReader) Close() error {
+ err := wr.ReadCloser.Close()
+ wr.f()
+ return err
+}
+
// Load returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
@@ -125,29 +142,49 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er
objName := be.s3path(h)
+ // get token for connection
<-be.connChan
- defer func() {
- be.connChan <- struct{}{}
- }()
obj, err := be.client.GetObject(be.bucketname, objName)
if err != nil {
debug.Log(" err %v", err)
+
+ // return token
+ be.connChan <- struct{}{}
+
return nil, errors.Wrap(err, "client.GetObject")
}
// if we're going to read the whole object, just pass it on.
if length == 0 {
debug.Log("Load %v: pass on object", h)
+
_, err = obj.Seek(offset, 0)
if err != nil {
_ = obj.Close()
+
+ // return token
+ be.connChan <- struct{}{}
+
return nil, errors.Wrap(err, "obj.Seek")
}
- return obj, nil
+ rd := wrapReader{
+ ReadCloser: obj,
+ f: func() {
+ debug.Log("Close()")
+ // return token
+ be.connChan <- struct{}{}
+ },
+ }
+ return rd, nil
}
+ defer func() {
+ // return token
+ be.connChan <- struct{}{}
+ }()
+
// otherwise use a buffer with ReadAt
info, err := obj.Stat()
if err != nil {
@@ -157,7 +194,7 @@ func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, er
if offset > info.Size {
_ = obj.Close()
- return nil, errors.Errorf("offset larger than file size")
+ return nil, errors.New("offset larger than file size")
}
l := int64(length)
diff --git a/src/restic/backend/sftp/sftp.go b/src/restic/backend/sftp/sftp.go
index 65e6f7f..8894cdc 100644
--- a/src/restic/backend/sftp/sftp.go
+++ b/src/restic/backend/sftp/sftp.go
@@ -153,7 +153,7 @@ func buildSSHCommand(cfg Config) []string {
}
// OpenWithConfig opens an sftp backend as described by the config by running
-// "ssh" with the appropiate arguments.
+// "ssh" with the appropriate arguments.
func OpenWithConfig(cfg Config) (*SFTP, error) {
debug.Log("open with config %v", cfg)
return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
@@ -193,7 +193,7 @@ func Create(dir string, program string, args ...string) (*SFTP, error) {
}
// CreateWithConfig creates an sftp backend as described by the config by running
-// "ssh" with the appropiate arguments.
+// "ssh" with the appropriate arguments.
func CreateWithConfig(cfg Config) (*SFTP, error) {
debug.Log("config %v", cfg)
return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
diff --git a/src/restic/backend/test/tests.go b/src/restic/backend/test/tests.go
index a6105f9..5c11d2e 100644
--- a/src/restic/backend/test/tests.go
+++ b/src/restic/backend/test/tests.go
@@ -245,21 +245,25 @@ func TestLoad(t testing.TB) {
buf, err := ioutil.ReadAll(rd)
if err != nil {
t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %v", l, o, err)
+ rd.Close()
continue
}
if l <= len(d) && len(buf) != l {
t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
+ rd.Close()
continue
}
if l > len(d) && len(buf) != len(d) {
t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
+ rd.Close()
continue
}
if !bytes.Equal(buf, d) {
t.Errorf("Load(%d, %d) returned wrong bytes", l, o)
+ rd.Close()
continue
}
diff --git a/src/restic/blob.go b/src/restic/blob.go
index 670fd2f..3588712 100644
--- a/src/restic/blob.go
+++ b/src/restic/blob.go
@@ -14,6 +14,11 @@ type Blob struct {
Offset uint
}
+func (b Blob) String() string {
+ return fmt.Sprintf("<Blob (%v) %v, offset %v, length %v>",
+ b.Type, b.ID.Str(), b.Offset, b.Length)
+}
+
// PackedBlob is a blob stored within a file.
type PackedBlob struct {
Blob
diff --git a/src/restic/checker/checker.go b/src/restic/checker/checker.go
index 7b37f7d..1fd7388 100644
--- a/src/restic/checker/checker.go
+++ b/src/restic/checker/checker.go
@@ -1,14 +1,17 @@
package checker
import (
- "bytes"
+ "crypto/sha256"
"fmt"
+ "io"
+ "io/ioutil"
+ "os"
"sync"
"restic/errors"
+ "restic/hashing"
"restic"
- "restic/backend"
"restic/crypto"
"restic/debug"
"restic/pack"
@@ -77,6 +80,7 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
debug.Log("Start")
type indexRes struct {
Index *repository.Index
+ err error
ID string
}
@@ -92,39 +96,40 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
idx, err = repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeOldIndex)
}
- if err != nil {
- return err
- }
+ err = errors.Wrapf(err, "error loading index %v", id.Str())
select {
- case indexCh <- indexRes{Index: idx, ID: id.String()}:
+ case indexCh <- indexRes{Index: idx, ID: id.String(), err: err}:
case <-done:
}
return nil
}
- var perr error
go func() {
defer close(indexCh)
debug.Log("start loading indexes in parallel")
- perr = repository.FilesInParallel(c.repo.Backend(), restic.IndexFile, defaultParallelism,
+ err := repository.FilesInParallel(c.repo.Backend(), restic.IndexFile, defaultParallelism,
repository.ParallelWorkFuncParseID(worker))
- debug.Log("loading indexes finished, error: %v", perr)
+ debug.Log("loading indexes finished, error: %v", err)
+ if err != nil {
+ panic(err)
+ }
}()
done := make(chan struct{})
defer close(done)
- if perr != nil {
- errs = append(errs, perr)
- return hints, errs
- }
-
packToIndex := make(map[restic.ID]restic.IDSet)
for res := range indexCh {
- debug.Log("process index %v", res.ID)
+ debug.Log("process index %v, err %v", res.ID, res.err)
+
+ if res.err != nil {
+ errs = append(errs, res.err)
+ continue
+ }
+
idxID, err := restic.ParseID(res.ID)
if err != nil {
errs = append(errs, errors.Errorf("unable to parse as index ID: %v", res.ID))
@@ -151,8 +156,6 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
debug.Log("%d blobs processed", cnt)
}
- debug.Log("done, error %v", perr)
-
debug.Log("checking for duplicate packs")
for packID := range c.packs {
debug.Log(" check pack %v: contained in %d indexes", packID.Str(), len(packToIndex[packID]))
@@ -659,36 +662,77 @@ func (c *Checker) CountPacks() uint64 {
func checkPack(r restic.Repository, id restic.ID) error {
debug.Log("checking pack %v", id.Str())
h := restic.Handle{Type: restic.DataFile, Name: id.String()}
- buf, err := backend.LoadAll(r.Backend(), h)
+
+ rd, err := r.Backend().Load(h, 0, 0)
+ if err != nil {
+ return err
+ }
+
+ packfile, err := ioutil.TempFile("", "restic-temp-check-")
+ if err != nil {
+ return errors.Wrap(err, "TempFile")
+ }
+
+ defer func() {
+ packfile.Close()
+ os.Remove(packfile.Name())
+ }()
+
+ hrd := hashing.NewReader(rd, sha256.New())
+ size, err := io.Copy(packfile, hrd)
if err != nil {
+ return errors.Wrap(err, "Copy")
+ }
+
+ if err = rd.Close(); err != nil {
return err
}
- hash := restic.Hash(buf)
+ hash := restic.IDFromHash(hrd.Sum(nil))
+ debug.Log("hash for pack %v is %v", id.Str(), hash.Str())
+
if !hash.Equal(id) {
debug.Log("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
}
- blobs, err := pack.List(r.Key(), bytes.NewReader(buf), int64(len(buf)))
+ blobs, err := pack.List(r.Key(), packfile, size)
if err != nil {
return err
}
var errs []error
+ var buf []byte
for i, blob := range blobs {
- debug.Log(" check blob %d: %v", i, blob.ID.Str())
+ debug.Log(" check blob %d: %v", i, blob)
+
+ buf = buf[:cap(buf)]
+ if uint(len(buf)) < blob.Length {
+ buf = make([]byte, blob.Length)
+ }
+ buf = buf[:blob.Length]
+
+ _, err := packfile.Seek(int64(blob.Offset), 0)
+ if err != nil {
+ return errors.Errorf("Seek(%v): %v", blob.Offset, err)
+ }
+
+ _, err = io.ReadFull(packfile, buf)
+ if err != nil {
+ debug.Log(" error loading blob %v: %v", blob.ID.Str(), err)
+ errs = append(errs, errors.Errorf("blob %v: %v", i, err))
+ continue
+ }
- plainBuf := make([]byte, blob.Length)
- n, err := crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length])
+ n, err := crypto.Decrypt(r.Key(), buf, buf)
if err != nil {
debug.Log(" error decrypting blob %v: %v", blob.ID.Str(), err)
errs = append(errs, errors.Errorf("blob %v: %v", i, err))
continue
}
- plainBuf = plainBuf[:n]
+ buf = buf[:n]
- hash := restic.Hash(plainBuf)
+ hash := restic.Hash(buf)
if !hash.Equal(blob.ID) {
debug.Log(" Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())
errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()))
diff --git a/src/restic/checker/checker_test.go b/src/restic/checker/checker_test.go
index d41f34b..65e7641 100644
--- a/src/restic/checker/checker_test.go
+++ b/src/restic/checker/checker_test.go
@@ -179,6 +179,48 @@ func TestUnreferencedBlobs(t *testing.T) {
test.Equals(t, unusedBlobsBySnapshot, blobs)
}
+func TestModifiedIndex(t *testing.T) {
+ repodir, cleanup := test.Env(t, checkerTestData)
+ defer cleanup()
+
+ repo := repository.TestOpenLocal(t, repodir)
+
+ done := make(chan struct{})
+ defer close(done)
+
+ h := restic.Handle{
+ Type: restic.IndexFile,
+ Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
+ }
+ f, err := repo.Backend().Load(h, 0, 0)
+ test.OK(t, err)
+
+ // save the index again with a modified name so that the hash doesn't match
+ // the content any more
+ h2 := restic.Handle{
+ Type: restic.IndexFile,
+ Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
+ }
+ err = repo.Backend().Save(h2, f)
+ test.OK(t, err)
+
+ test.OK(t, f.Close())
+
+ chkr := checker.New(repo)
+ hints, errs := chkr.LoadIndex()
+ if len(errs) == 0 {
+ t.Fatalf("expected errors not found")
+ }
+
+ for _, err := range errs {
+ t.Logf("found expected error %v", err)
+ }
+
+ if len(hints) > 0 {
+ t.Errorf("expected no hints, got %v: %v", len(hints), hints)
+ }
+}
+
var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
func TestDuplicatePacksInIndex(t *testing.T) {
@@ -261,7 +303,7 @@ func TestCheckerModifiedData(t *testing.T) {
defer cleanup()
arch := archiver.New(repo)
- _, id, err := arch.Snapshot(nil, []string{"."}, nil, nil)
+ _, id, err := arch.Snapshot(nil, []string{"."}, nil, "localhost", nil)
test.OK(t, err)
t.Logf("archived as %v", id.Str())
@@ -299,3 +341,28 @@ func TestCheckerModifiedData(t *testing.T) {
t.Fatal("no error found, checker is broken")
}
}
+
+func BenchmarkChecker(t *testing.B) {
+ repodir, cleanup := test.Env(t, checkerTestData)
+ defer cleanup()
+
+ repo := repository.TestOpenLocal(t, repodir)
+
+ chkr := checker.New(repo)
+ hints, errs := chkr.LoadIndex()
+ if len(errs) > 0 {
+ t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
+ }
+
+ if len(hints) > 0 {
+ t.Errorf("expected no hints, got %v: %v", len(hints), hints)
+ }
+
+ t.ResetTimer()
+
+ for i := 0; i < t.N; i++ {
+ test.OKs(t, checkPacks(chkr))
+ test.OKs(t, checkStruct(chkr))
+ test.OKs(t, checkData(chkr))
+ }
+}
diff --git a/src/restic/errors/wrap.go b/src/restic/errors/wrap.go
index 5906bd6..99d6e88 100644
--- a/src/restic/errors/wrap.go
+++ b/src/restic/errors/wrap.go
@@ -18,3 +18,7 @@ var Errorf = errors.Errorf
// Wrap wraps an error retrieved from outside of restic. Wrapped so that this
// package does not appear in the stack trace.
var Wrap = errors.Wrap
+
+// Wrapf returns an error annotating err with the format specifier. If err is
+// nil, Wrapf returns nil.
+var Wrapf = errors.Wrapf
diff --git a/src/restic/fs/doc.go b/src/restic/fs/doc.go
index 29bfa66..0607238 100644
--- a/src/restic/fs/doc.go
+++ b/src/restic/fs/doc.go
@@ -1,3 +1,3 @@
-// Package fs implements an OS independend abstraction of a file system
+// Package fs implements an OS independent abstraction of a file system
// suitable for backup purposes.
package fs
diff --git a/src/restic/fs/file.go b/src/restic/fs/file.go
index b7e81a3..2ba5d13 100644
--- a/src/restic/fs/file.go
+++ b/src/restic/fs/file.go
@@ -102,6 +102,12 @@ func Symlink(oldname, newname string) error {
return os.Symlink(fixpath(oldname), fixpath(newname))
}
+// Link creates newname as a hard link to oldname.
+// If there is an error, it will be of type *LinkError.
+func Link(oldname, newname string) error {
+ return os.Link(fixpath(oldname), fixpath(newname))
+}
+
// Stat returns a FileInfo structure describing the named file.
// If there is an error, it will be of type *PathError.
func Stat(name string) (os.FileInfo, error) {
diff --git a/src/restic/fuse/dir.go b/src/restic/fuse/dir.go
index e5f68a9..7968a2e 100644
--- a/src/restic/fuse/dir.go
+++ b/src/restic/fuse/dir.go
@@ -114,9 +114,25 @@ func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
a.Atime = d.node.AccessTime
a.Ctime = d.node.ChangeTime
a.Mtime = d.node.ModTime
+
+ a.Nlink = d.calcNumberOfLinks()
+
return nil
}
+func (d *dir) calcNumberOfLinks() uint32 {
+ // a directory d has 2 hardlinks + the number
+ // of directories contained by d
+ var count uint32
+ count = 2
+ for _, node := range d.items {
+ if node.Type == "dir" {
+ count++
+ }
+ }
+ return count
+}
+
func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
debug.Log("called")
ret := make([]fuse.Dirent, 0, len(d.items))
@@ -161,3 +177,21 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
return nil, fuse.ENOENT
}
}
+
+func (d *dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
+ debug.Log("Listxattr(%v, %v)", d.node.Name, req.Size)
+ for _, attr := range d.node.ExtendedAttributes {
+ resp.Append(attr.Name)
+ }
+ return nil
+}
+
+func (d *dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
+ debug.Log("Getxattr(%v, %v, %v)", d.node.Name, req.Name, req.Size)
+ attrval := d.node.GetExtendedAttribute(req.Name)
+ if attrval != nil {
+ resp.Xattr = attrval
+ return nil
+ }
+ return fuse.ErrNoXattr
+}
diff --git a/src/restic/fuse/file.go b/src/restic/fuse/file.go
index 6bfa2e4..dfc8aa3 100644
--- a/src/restic/fuse/file.go
+++ b/src/restic/fuse/file.go
@@ -74,6 +74,7 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
a.Size = f.node.Size
a.Blocks = (f.node.Size / blockSize) + 1
a.BlockSize = blockSize
+ a.Nlink = uint32(f.node.Links)
if !f.ownerIsRoot {
a.Uid = f.node.UID
@@ -82,7 +83,9 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
a.Atime = f.node.AccessTime
a.Ctime = f.node.ChangeTime
a.Mtime = f.node.ModTime
+
return nil
+
}
func (f *file) getBlobAt(i int) (blob []byte, err error) {
@@ -161,3 +164,21 @@ func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
}
return nil
}
+
+func (f *file) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
+ debug.Log("Listxattr(%v, %v)", f.node.Name, req.Size)
+ for _, attr := range f.node.ExtendedAttributes {
+ resp.Append(attr.Name)
+ }
+ return nil
+}
+
+func (f *file) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
+ debug.Log("Getxattr(%v, %v, %v)", f.node.Name, req.Name, req.Size)
+ attrval := f.node.GetExtendedAttribute(req.Name)
+ if attrval != nil {
+ resp.Xattr = attrval
+ return nil
+ }
+ return fuse.ErrNoXattr
+}
diff --git a/src/restic/fuse/link.go b/src/restic/fuse/link.go
index 43fb350..e230acb 100644
--- a/src/restic/fuse/link.go
+++ b/src/restic/fuse/link.go
@@ -38,5 +38,8 @@ func (l *link) Attr(ctx context.Context, a *fuse.Attr) error {
a.Atime = l.node.AccessTime
a.Ctime = l.node.ChangeTime
a.Mtime = l.node.ModTime
+
+ a.Nlink = uint32(l.node.Links)
+
return nil
}
diff --git a/src/restic/fuse/snapshot.go b/src/restic/fuse/snapshot.go
index 1e1092d..2a65439 100644
--- a/src/restic/fuse/snapshot.go
+++ b/src/restic/fuse/snapshot.go
@@ -32,6 +32,9 @@ var _ = fs.NodeStringLookuper(&SnapshotsDir{})
type SnapshotsDir struct {
repo restic.Repository
ownerIsRoot bool
+ paths []string
+ tags []string
+ host string
// knownSnapshots maps snapshot timestamp to the snapshot
sync.RWMutex
@@ -40,12 +43,15 @@ type SnapshotsDir struct {
}
// NewSnapshotsDir returns a new dir object for the snapshots.
-func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool) *SnapshotsDir {
+func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, tags []string, host string) *SnapshotsDir {
debug.Log("fuse mount initiated")
return &SnapshotsDir{
repo: repo,
- knownSnapshots: make(map[string]SnapshotWithId),
ownerIsRoot: ownerIsRoot,
+ paths: paths,
+ tags: tags,
+ host: host,
+ knownSnapshots: make(map[string]SnapshotWithId),
processed: restic.NewIDSet(),
}
}
@@ -79,6 +85,13 @@ func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
return err
}
+ // Filter snapshots we don't care for.
+ if (sn.host != "" && sn.host != snapshot.Hostname) ||
+ !snapshot.HasTags(sn.tags) ||
+ !snapshot.HasPaths(sn.paths) {
+ continue
+ }
+
timestamp := snapshot.Time.Format(time.RFC3339)
for i := 1; ; i++ {
if _, ok := sn.knownSnapshots[timestamp]; !ok {
diff --git a/src/restic/hardlinks_index.go b/src/restic/hardlinks_index.go
new file mode 100644
index 0000000..0874f32
--- /dev/null
+++ b/src/restic/hardlinks_index.go
@@ -0,0 +1,57 @@
+package restic
+
+import (
+ "sync"
+)
+
+// HardlinkKey is a composed key for finding inodes on a specific device.
+type HardlinkKey struct {
+ Inode, Device uint64
+}
+
+// HardlinkIndex contains a list of inodes, devices these inodes are one, and associated file names.
+type HardlinkIndex struct {
+ m sync.Mutex
+ Index map[HardlinkKey]string
+}
+
+// NewHardlinkIndex create a new index for hard links
+func NewHardlinkIndex() *HardlinkIndex {
+ return &HardlinkIndex{
+ Index: make(map[HardlinkKey]string),
+ }
+}
+
+// Has checks wether the link already exist in the index.
+func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool {
+ idx.m.Lock()
+ defer idx.m.Unlock()
+ _, ok := idx.Index[HardlinkKey{inode, device}]
+
+ return ok
+}
+
+// Add adds a link to the index.
+func (idx *HardlinkIndex) Add(inode uint64, device uint64, name string) {
+ idx.m.Lock()
+ defer idx.m.Unlock()
+ _, ok := idx.Index[HardlinkKey{inode, device}]
+
+ if !ok {
+ idx.Index[HardlinkKey{inode, device}] = name
+ }
+}
+
+// GetFilename obtains the filename from the index.
+func (idx *HardlinkIndex) GetFilename(inode uint64, device uint64) string {
+ idx.m.Lock()
+ defer idx.m.Unlock()
+ return idx.Index[HardlinkKey{inode, device}]
+}
+
+// Remove removes a link from the index.
+func (idx *HardlinkIndex) Remove(inode uint64, device uint64) {
+ idx.m.Lock()
+ defer idx.m.Unlock()
+ delete(idx.Index, HardlinkKey{inode, device})
+}
diff --git a/src/restic/hardlinks_index_test.go b/src/restic/hardlinks_index_test.go
new file mode 100644
index 0000000..c0a6756
--- /dev/null
+++ b/src/restic/hardlinks_index_test.go
@@ -0,0 +1,35 @@
+package restic_test
+
+import (
+ "testing"
+
+ "restic"
+ . "restic/test"
+)
+
+// TestHardLinks contains various tests for HardlinkIndex.
+func TestHardLinks(t *testing.T) {
+
+ idx := restic.NewHardlinkIndex()
+
+ idx.Add(1, 2, "inode1-file1-on-device2")
+ idx.Add(2, 3, "inode2-file2-on-device3")
+
+ var sresult string
+ sresult = idx.GetFilename(1, 2)
+ Equals(t, sresult, "inode1-file1-on-device2")
+
+ sresult = idx.GetFilename(2, 3)
+ Equals(t, sresult, "inode2-file2-on-device3")
+
+ var bresult bool
+ bresult = idx.Has(1, 2)
+ Equals(t, bresult, true)
+
+ bresult = idx.Has(1, 3)
+ Equals(t, bresult, false)
+
+ idx.Remove(1, 2)
+ bresult = idx.Has(1, 2)
+ Equals(t, bresult, false)
+}
diff --git a/src/restic/id.go b/src/restic/id.go
index c64508a..25bf951 100644
--- a/src/restic/id.go
+++ b/src/restic/id.go
@@ -43,7 +43,7 @@ func (id ID) String() string {
return hex.EncodeToString(id[:])
}
-// NewRandomID retuns a randomly generated ID. When reading from rand fails,
+// NewRandomID returns a randomly generated ID. When reading from rand fails,
// the function panics.
func NewRandomID() ID {
id := ID{}
diff --git a/src/restic/list/list.go b/src/restic/list/list.go
index 18bfb60..70b3c7d 100644
--- a/src/restic/list/list.go
+++ b/src/restic/list/list.go
@@ -25,7 +25,7 @@ func (l Result) PackID() restic.ID {
return l.packID
}
-// Size ruturns the size of the pack.
+// Size returns the size of the pack.
func (l Result) Size() int64 {
return l.size
}
diff --git a/src/restic/lock.go b/src/restic/lock.go
index 8fec753..97f2d65 100644
--- a/src/restic/lock.go
+++ b/src/restic/lock.go
@@ -203,7 +203,7 @@ func (l *Lock) Stale() bool {
hn, err := os.Hostname()
if err != nil {
- debug.Log("unable to find current hostnanme: %v", err)
+ debug.Log("unable to find current hostname: %v", err)
// since we cannot find the current hostname, assume that the lock is
// not stale.
return false
diff --git a/src/restic/node.go b/src/restic/node.go
index bf41f42..9f416a2 100644
--- a/src/restic/node.go
+++ b/src/restic/node.go
@@ -12,31 +12,38 @@ import (
"restic/errors"
- "runtime"
-
+ "bytes"
"restic/debug"
"restic/fs"
+ "runtime"
)
+// ExtendedAttribute is a tuple storing the xattr name and value.
+type ExtendedAttribute struct {
+ Name string `json:"name"`
+ Value []byte `json:"value"`
+}
+
// Node is a file, directory or other item in a backup.
type Node struct {
- Name string `json:"name"`
- Type string `json:"type"`
- Mode os.FileMode `json:"mode,omitempty"`
- ModTime time.Time `json:"mtime,omitempty"`
- AccessTime time.Time `json:"atime,omitempty"`
- ChangeTime time.Time `json:"ctime,omitempty"`
- UID uint32 `json:"uid"`
- GID uint32 `json:"gid"`
- User string `json:"user,omitempty"`
- Group string `json:"group,omitempty"`
- Inode uint64 `json:"inode,omitempty"`
- Size uint64 `json:"size,omitempty"`
- Links uint64 `json:"links,omitempty"`
- LinkTarget string `json:"linktarget,omitempty"`
- Device uint64 `json:"device,omitempty"`
- Content IDs `json:"content"`
- Subtree *ID `json:"subtree,omitempty"`
+ Name string `json:"name"`
+ Type string `json:"type"`
+ Mode os.FileMode `json:"mode,omitempty"`
+ ModTime time.Time `json:"mtime,omitempty"`
+ AccessTime time.Time `json:"atime,omitempty"`
+ ChangeTime time.Time `json:"ctime,omitempty"`
+ UID uint32 `json:"uid"`
+ GID uint32 `json:"gid"`
+ User string `json:"user,omitempty"`
+ Group string `json:"group,omitempty"`
+ Inode uint64 `json:"inode,omitempty"`
+ Size uint64 `json:"size,omitempty"`
+ Links uint64 `json:"links,omitempty"`
+ LinkTarget string `json:"linktarget,omitempty"`
+ ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"`
+ Device uint64 `json:"device,omitempty"`
+ Content IDs `json:"content"`
+ Subtree *ID `json:"subtree,omitempty"`
Error string `json:"error,omitempty"`
@@ -56,7 +63,8 @@ func (node Node) String() string {
return fmt.Sprintf("<Node(%s) %s>", node.Type, node.Name)
}
-// NodeFromFileInfo returns a new node from the given path and FileInfo.
+// NodeFromFileInfo returns a new node from the given path and FileInfo. It
+// returns the first error that is encountered, together with a node.
func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky
node := &Node{
@@ -96,8 +104,18 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string {
return ""
}
+// GetExtendedAttribute gets the extended attribute.
+func (node Node) GetExtendedAttribute(a string) []byte {
+ for _, attr := range node.ExtendedAttributes {
+ if attr.Name == a {
+ return attr.Value
+ }
+ }
+ return nil
+}
+
// CreateAt creates the node at the given path and restores all the meta data.
-func (node *Node) CreateAt(path string, repo Repository) error {
+func (node *Node) CreateAt(path string, repo Repository, idx *HardlinkIndex) error {
debug.Log("create node %v at %v", node.Name, path)
switch node.Type {
@@ -106,7 +124,7 @@ func (node *Node) CreateAt(path string, repo Repository) error {
return err
}
case "file":
- if err := node.createFileAt(path, repo); err != nil {
+ if err := node.createFileAt(path, repo, idx); err != nil {
return err
}
case "symlink":
@@ -162,6 +180,22 @@ func (node Node) restoreMetadata(path string) error {
}
}
+ err = node.restoreExtendedAttributes(path)
+ if err != nil {
+ debug.Log("error restoring extended attributes for %v: %v", path, err)
+ return err
+ }
+
+ return nil
+}
+
+func (node Node) restoreExtendedAttributes(path string) error {
+ for _, attr := range node.ExtendedAttributes {
+ err := Setxattr(path, attr.Name, attr.Value)
+ if err != nil {
+ return err
+ }
+ }
return nil
}
@@ -191,7 +225,15 @@ func (node Node) createDirAt(path string) error {
return nil
}
-func (node Node) createFileAt(path string, repo Repository) error {
+func (node Node) createFileAt(path string, repo Repository, idx *HardlinkIndex) error {
+ if node.Links > 1 && idx.Has(node.Inode, node.Device) {
+ err := fs.Link(idx.GetFilename(node.Inode, node.Device), path)
+ if err != nil {
+ return errors.Wrap(err, "CreateHardlink")
+ }
+ return nil
+ }
+
f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
defer f.Close()
@@ -207,7 +249,7 @@ func (node Node) createFileAt(path string, repo Repository) error {
}
buf = buf[:cap(buf)]
- if uint(len(buf)) < size {
+ if len(buf) < CiphertextLength(int(size)) {
buf = NewBlobBuffer(int(size))
}
@@ -223,6 +265,8 @@ func (node Node) createFileAt(path string, repo Repository) error {
}
}
+ idx.Add(node.Inode, node.Device, path)
+
return nil
}
@@ -340,6 +384,9 @@ func (node Node) Equals(other Node) bool {
if !node.sameContent(other) {
return false
}
+ if !node.sameExtendedAttributes(other) {
+ return false
+ }
if node.Subtree != nil {
if other.Subtree == nil {
return false
@@ -378,6 +425,51 @@ func (node Node) sameContent(other Node) bool {
return false
}
}
+ return true
+}
+
+func (node Node) sameExtendedAttributes(other Node) bool {
+ if len(node.ExtendedAttributes) != len(other.ExtendedAttributes) {
+ return false
+ }
+
+ // build a set of all attributes that node has
+ type mapvalue struct {
+ value []byte
+ present bool
+ }
+ attributes := make(map[string]mapvalue)
+ for _, attr := range node.ExtendedAttributes {
+ attributes[attr.Name] = mapvalue{value: attr.Value}
+ }
+
+ for _, attr := range other.ExtendedAttributes {
+ v, ok := attributes[attr.Name]
+ if !ok {
+ // extended attribute is not set for node
+ debug.Log("other node has attribute %v, which is not present in node", attr.Name)
+ return false
+
+ }
+
+ if !bytes.Equal(v.value, attr.Value) {
+ // attribute has different value
+ debug.Log("attribute %v has different value", attr.Name)
+ return false
+ }
+
+ // remember that this attribute is present in other.
+ v.present = true
+ attributes[attr.Name] = v
+ }
+
+ // check for attributes that are not present in other
+ for name, v := range attributes {
+ if !v.present {
+ debug.Log("attribute %v not present in other node", name)
+ return false
+ }
+ }
return true
}
@@ -485,18 +577,56 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error {
case "dir":
case "symlink":
node.LinkTarget, err = fs.Readlink(path)
- err = errors.Wrap(err, "Readlink")
+ node.Links = uint64(stat.nlink())
+ if err != nil {
+ return errors.Wrap(err, "Readlink")
+ }
case "dev":
node.Device = uint64(stat.rdev())
+ node.Links = uint64(stat.nlink())
case "chardev":
node.Device = uint64(stat.rdev())
+ node.Links = uint64(stat.nlink())
case "fifo":
case "socket":
default:
- err = errors.Errorf("invalid node type %q", node.Type)
+ return errors.Errorf("invalid node type %q", node.Type)
}
- return err
+ if err = node.fillExtendedAttributes(path); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (node *Node) fillExtendedAttributes(path string) error {
+ if node.Type == "symlink" {
+ return nil
+ }
+
+ xattrs, err := Listxattr(path)
+ debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err)
+ if err != nil {
+ return err
+ }
+
+ node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs))
+ for _, attr := range xattrs {
+ attrVal, err := Getxattr(path, attr)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path)
+ continue
+ }
+ attr := ExtendedAttribute{
+ Name: attr,
+ Value: attrVal,
+ }
+
+ node.ExtendedAttributes = append(node.ExtendedAttributes, attr)
+ }
+
+ return nil
}
type statT interface {
diff --git a/src/restic/node_openbsd.go b/src/restic/node_openbsd.go
index 4c77798..8ca4f95 100644
--- a/src/restic/node_openbsd.go
+++ b/src/restic/node_openbsd.go
@@ -9,3 +9,19 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe
func (s statUnix) atim() syscall.Timespec { return s.Atim }
func (s statUnix) mtim() syscall.Timespec { return s.Mtim }
func (s statUnix) ctim() syscall.Timespec { return s.Ctim }
+
+// Getxattr retrieves extended attribute data associated with path.
+func Getxattr(path, name string) ([]byte, error) {
+ return nil, nil
+}
+
+// Listxattr retrieves a list of names of extended attributes associated with the
+// given path in the file system.
+func Listxattr(path string) ([]string, error) {
+ return nil, nil
+}
+
+// Setxattr associates name and data together as an attribute of path.
+func Setxattr(path, name string, data []byte) error {
+ return nil
+}
diff --git a/src/restic/node_test.go b/src/restic/node_test.go
index e219d89..1641415 100644
--- a/src/restic/node_test.go
+++ b/src/restic/node_test.go
@@ -176,9 +176,11 @@ func TestNodeRestoreAt(t *testing.T) {
}
}()
+ idx := restic.NewHardlinkIndex()
+
for _, test := range nodeTests {
nodePath := filepath.Join(tempdir, test.Name)
- OK(t, test.CreateAt(nodePath, nil))
+ OK(t, test.CreateAt(nodePath, nil, idx))
if test.Type == "symlink" && runtime.GOOS == "windows" {
continue
diff --git a/src/restic/node_windows.go b/src/restic/node_windows.go
index 050de8f..b75f148 100644
--- a/src/restic/node_windows.go
+++ b/src/restic/node_windows.go
@@ -22,8 +22,25 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe
return nil
}
+// Getxattr retrieves extended attribute data associated with path.
+func Getxattr(path, name string) ([]byte, error) {
+ return nil, nil
+}
+
+// Listxattr retrieves a list of names of extended attributes associated with the
+// given path in the file system.
+func Listxattr(path string) ([]string, error) {
+ return nil, nil
+}
+
+// Setxattr associates name and data together as an attribute of path.
+func Setxattr(path, name string, data []byte) error {
+ return nil
+}
+
type statWin syscall.Win32FileAttributeData
+//ToStatT call the Windows system call Win32FileAttributeData.
func toStatT(i interface{}) (statT, bool) {
if i == nil {
return nil, false
diff --git a/src/restic/node_xattr.go b/src/restic/node_xattr.go
new file mode 100644
index 0000000..a61c44a
--- /dev/null
+++ b/src/restic/node_xattr.go
@@ -0,0 +1,39 @@
+// +build !openbsd
+// +build !windows
+
+package restic
+
+import (
+ "restic/errors"
+ "syscall"
+
+ "github.com/pkg/xattr"
+)
+
+// Getxattr retrieves extended attribute data associated with path.
+func Getxattr(path, name string) ([]byte, error) {
+ b, e := xattr.Getxattr(path, name)
+ if err, ok := e.(*xattr.XAttrError); ok && err.Err == syscall.ENOTSUP {
+ return nil, nil
+ }
+ return b, errors.Wrap(e, "Getxattr")
+}
+
+// Listxattr retrieves a list of names of extended attributes associated with the
+// given path in the file system.
+func Listxattr(path string) ([]string, error) {
+ s, e := xattr.Listxattr(path)
+ if err, ok := e.(*xattr.XAttrError); ok && err.Err == syscall.ENOTSUP {
+ return nil, nil
+ }
+ return s, errors.Wrap(e, "Listxattr")
+}
+
+// Setxattr associates name and data together as an attribute of path.
+func Setxattr(path, name string, data []byte) error {
+ e := xattr.Setxattr(path, name, data)
+ if err, ok := e.(*xattr.XAttrError); ok && err.Err == syscall.ENOTSUP {
+ return nil
+ }
+ return errors.Wrap(e, "Setxattr")
+}
diff --git a/src/restic/repository/index.go b/src/restic/repository/index.go
index 4257c7d..0db6832 100644
--- a/src/restic/repository/index.go
+++ b/src/restic/repository/index.go
@@ -392,7 +392,7 @@ func (idx *Index) SetID(id restic.ID) error {
defer idx.m.Unlock()
if !idx.final {
- return errors.New("indexs is not final")
+ return errors.New("index is not final")
}
if !idx.id.IsNull() {
diff --git a/src/restic/repository/index_rebuild.go b/src/restic/repository/index_rebuild.go
deleted file mode 100644
index fcfc302..0000000
--- a/src/restic/repository/index_rebuild.go
+++ /dev/null
@@ -1,66 +0,0 @@
-package repository
-
-import (
- "fmt"
- "os"
- "restic"
- "restic/debug"
- "restic/list"
- "restic/worker"
-)
-
-// RebuildIndex lists all packs in the repo, writes a new index and removes all
-// old indexes. This operation should only be done with an exclusive lock in
-// place.
-func RebuildIndex(repo restic.Repository) error {
- debug.Log("start rebuilding index")
-
- done := make(chan struct{})
- defer close(done)
-
- ch := make(chan worker.Job)
- go list.AllPacks(repo, ch, done)
-
- idx := NewIndex()
- for job := range ch {
- id := job.Data.(restic.ID)
-
- if job.Error != nil {
- fmt.Fprintf(os.Stderr, "error for pack %v: %v\n", id, job.Error)
- continue
- }
-
- res := job.Result.(list.Result)
-
- for _, entry := range res.Entries() {
- pb := restic.PackedBlob{
- Blob: entry,
- PackID: res.PackID(),
- }
- idx.Store(pb)
- }
- }
-
- oldIndexes := restic.NewIDSet()
- for id := range repo.List(restic.IndexFile, done) {
- idx.AddToSupersedes(id)
- oldIndexes.Insert(id)
- }
-
- id, err := SaveIndex(repo, idx)
- if err != nil {
- debug.Log("error saving index: %v", err)
- return err
- }
- debug.Log("new index saved as %v", id.Str())
-
- for indexID := range oldIndexes {
- h := restic.Handle{Type: restic.IndexFile, Name: indexID.String()}
- err := repo.Backend().Remove(h)
- if err != nil {
- fmt.Fprintf(os.Stderr, "unable to remove index %v: %v\n", indexID.Str(), err)
- }
- }
-
- return nil
-}
diff --git a/src/restic/repository/repack.go b/src/restic/repository/repack.go
index 59bc70b..7cd1c5f 100644
--- a/src/restic/repository/repack.go
+++ b/src/restic/repository/repack.go
@@ -18,7 +18,7 @@ import (
// these packs. Each pack is loaded and the blobs listed in keepBlobs is saved
// into a new pack. Afterwards, the packs are removed. This operation requires
// an exclusive lock on the repo.
-func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) {
+func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet, p *restic.Progress) (err error) {
debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))
for packID := range packs {
@@ -35,14 +35,16 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
return err
}
- defer beRd.Close()
-
hrd := hashing.NewReader(beRd, sha256.New())
packLength, err := io.Copy(tempfile, hrd)
if err != nil {
return errors.Wrap(err, "Copy")
}
+ if err = beRd.Close(); err != nil {
+ return errors.Wrap(err, "Close")
+ }
+
hash := restic.IDFromHash(hrd.Sum(nil))
debug.Log("pack %v loaded (%d bytes), hash %v", packID.Str(), packLength, hash.Str())
@@ -116,6 +118,9 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
if err = os.Remove(tempfile.Name()); err != nil {
return errors.Wrap(err, "Remove")
}
+ if p != nil {
+ p.Report(restic.Stat{Blobs: 1})
+ }
}
if err := repo.Flush(); err != nil {
diff --git a/src/restic/repository/repack_test.go b/src/restic/repository/repack_test.go
index 6d910c9..622b3ba 100644
--- a/src/restic/repository/repack_test.go
+++ b/src/restic/repository/repack_test.go
@@ -4,6 +4,7 @@ import (
"io"
"math/rand"
"restic"
+ "restic/index"
"restic/repository"
"testing"
)
@@ -131,7 +132,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe
}
func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs restic.BlobSet) {
- err := repository.Repack(repo, packs, blobs)
+ err := repository.Repack(repo, packs, blobs, nil)
if err != nil {
t.Fatal(err)
}
@@ -144,8 +145,24 @@ func saveIndex(t *testing.T, repo restic.Repository) {
}
func rebuildIndex(t *testing.T, repo restic.Repository) {
- if err := repository.RebuildIndex(repo); err != nil {
- t.Fatalf("error rebuilding index: %v", err)
+ idx, err := index.New(repo, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for id := range repo.List(restic.IndexFile, nil) {
+ err = repo.Backend().Remove(restic.Handle{
+ Type: restic.IndexFile,
+ Name: id.String(),
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ _, err = idx.Save(repo, nil)
+ if err != nil {
+ t.Fatal(err)
}
}
diff --git a/src/restic/repository/repository.go b/src/restic/repository/repository.go
index 45046da..4b93db7 100644
--- a/src/restic/repository/repository.go
+++ b/src/restic/repository/repository.go
@@ -56,12 +56,12 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
h := restic.Handle{Type: t, Name: id.String()}
buf, err := backend.LoadAll(r.be, h)
if err != nil {
- debug.Log("error loading %v: %v", id.Str(), err)
+ debug.Log("error loading %v: %v", h, err)
return nil, err
}
if t != restic.ConfigFile && !restic.Hash(buf).Equal(id) {
- return nil, errors.New("invalid data returned")
+ return nil, errors.Errorf("load %v: invalid data returned", h)
}
// decrypt
@@ -442,50 +442,22 @@ func (r *Repository) KeyName() string {
return r.keyName
}
-func (r *Repository) list(t restic.FileType, done <-chan struct{}, out chan<- restic.ID) {
- defer close(out)
- in := r.be.List(t, done)
-
- var (
- // disable sending on the outCh until we received a job
- outCh chan<- restic.ID
- // enable receiving from in
- inCh = in
- id restic.ID
- err error
- )
-
- for {
- select {
- case <-done:
- return
- case strID, ok := <-inCh:
- if !ok {
- // input channel closed, we're done
- return
- }
- id, err = restic.ParseID(strID)
- if err != nil {
- // ignore invalid IDs
- continue
- }
-
- inCh = nil
- outCh = out
- case outCh <- id:
- outCh = nil
- inCh = in
- }
- }
-}
-
// List returns a channel that yields all IDs of type t in the backend.
func (r *Repository) List(t restic.FileType, done <-chan struct{}) <-chan restic.ID {
- outCh := make(chan restic.ID)
-
- go r.list(t, done, outCh)
-
- return outCh
+ out := make(chan restic.ID)
+ go func() {
+ defer close(out)
+ for strID := range r.be.List(t, done) {
+ if id, err := restic.ParseID(strID); err == nil {
+ select {
+ case out <- id:
+ case <-done:
+ return
+ }
+ }
+ }
+ }()
+ return out
}
// ListPack returns the list of blobs saved in the pack id and the length of
diff --git a/src/restic/restorer.go b/src/restic/restorer.go
index 5397c8d..56916f3 100644
--- a/src/restic/restorer.go
+++ b/src/restic/restorer.go
@@ -38,7 +38,7 @@ func NewRestorer(repo Repository, id ID) (*Restorer, error) {
return r, nil
}
-func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
+func (res *Restorer) restoreTo(dst string, dir string, treeID ID, idx *HardlinkIndex) error {
tree, err := res.repo.LoadTree(treeID)
if err != nil {
return res.Error(dir, nil, err)
@@ -50,7 +50,7 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
debug.Log("SelectForRestore returned %v", selectedForRestore)
if selectedForRestore {
- err := res.restoreNodeTo(node, dir, dst)
+ err := res.restoreNodeTo(node, dir, dst, idx)
if err != nil {
return err
}
@@ -62,7 +62,7 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
}
subp := filepath.Join(dir, node.Name)
- err = res.restoreTo(dst, subp, *node.Subtree)
+ err = res.restoreTo(dst, subp, *node.Subtree, idx)
if err != nil {
err = res.Error(subp, node, err)
if err != nil {
@@ -83,11 +83,11 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
return nil
}
-func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
+func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string, idx *HardlinkIndex) error {
debug.Log("node %v, dir %v, dst %v", node.Name, dir, dst)
dstPath := filepath.Join(dst, dir, node.Name)
- err := node.CreateAt(dstPath, res.repo)
+ err := node.CreateAt(dstPath, res.repo, idx)
if err != nil {
debug.Log("node.CreateAt(%s) error %v", dstPath, err)
}
@@ -99,7 +99,7 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
// Create parent directories and retry
err = fs.MkdirAll(filepath.Dir(dstPath), 0700)
if err == nil || os.IsExist(errors.Cause(err)) {
- err = node.CreateAt(dstPath, res.repo)
+ err = node.CreateAt(dstPath, res.repo, idx)
}
}
@@ -116,10 +116,11 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
return nil
}
-// RestoreTo creates the directories and files in the snapshot below dir.
+// RestoreTo creates the directories and files in the snapshot below dst.
// Before an item is created, res.Filter is called.
-func (res *Restorer) RestoreTo(dir string) error {
- return res.restoreTo(dir, "", *res.sn.Tree)
+func (res *Restorer) RestoreTo(dst string) error {
+ idx := NewHardlinkIndex()
+ return res.restoreTo(dst, string(filepath.Separator), *res.sn.Tree, idx)
}
// Snapshot returns the snapshot this restorer is configured to use.
diff --git a/src/restic/snapshot.go b/src/restic/snapshot.go
index 2320fd9..343dfe4 100644
--- a/src/restic/snapshot.go
+++ b/src/restic/snapshot.go
@@ -2,7 +2,6 @@ package restic
import (
"fmt"
- "os"
"os/user"
"path/filepath"
"time"
@@ -22,13 +21,14 @@ type Snapshot struct {
GID uint32 `json:"gid,omitempty"`
Excludes []string `json:"excludes,omitempty"`
Tags []string `json:"tags,omitempty"`
+ Original *ID `json:"original,omitempty"`
id *ID // plaintext ID, used during restore
}
// NewSnapshot returns an initialized snapshot struct for the current user and
// time.
-func NewSnapshot(paths []string, tags []string) (*Snapshot, error) {
+func NewSnapshot(paths []string, tags []string, hostname string) (*Snapshot, error) {
for i, path := range paths {
if p, err := filepath.Abs(path); err != nil {
paths[i] = p
@@ -36,17 +36,13 @@ func NewSnapshot(paths []string, tags []string) (*Snapshot, error) {
}
sn := &Snapshot{
- Paths: paths,
- Time: time.Now(),
- Tags: tags,
+ Paths: paths,
+ Time: time.Now(),
+ Tags: tags,
+ Hostname: hostname,
}
- hn, err := os.Hostname()
- if err == nil {
- sn.Hostname = hn
- }
-
- err = sn.fillUserInfo()
+ err := sn.fillUserInfo()
if err != nil {
return nil, err
}
@@ -78,8 +74,7 @@ func LoadAllSnapshots(repo Repository) (snapshots []*Snapshot, err error) {
snapshots = append(snapshots, sn)
}
-
- return snapshots, nil
+ return
}
func (sn Snapshot) String() string {
@@ -87,7 +82,7 @@ func (sn Snapshot) String() string {
sn.id.Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname)
}
-// ID retuns the snapshot's ID.
+// ID returns the snapshot's ID.
func (sn Snapshot) ID() *ID {
return sn.id
}
@@ -104,7 +99,42 @@ func (sn *Snapshot) fillUserInfo() error {
return err
}
-// HasTags returns true if the snapshot has all the tags.
+// AddTags adds the given tags to the snapshots tags, preventing duplicates.
+// It returns true if any changes were made.
+func (sn *Snapshot) AddTags(addTags []string) (changed bool) {
+nextTag:
+ for _, add := range addTags {
+ for _, tag := range sn.Tags {
+ if tag == add {
+ continue nextTag
+ }
+ }
+ sn.Tags = append(sn.Tags, add)
+ changed = true
+ }
+ return
+}
+
+// RemoveTags removes the given tags from the snapshots tags and
+// returns true if any changes were made.
+func (sn *Snapshot) RemoveTags(removeTags []string) (changed bool) {
+ for _, remove := range removeTags {
+ for i, tag := range sn.Tags {
+ if tag == remove {
+ // https://github.com/golang/go/wiki/SliceTricks
+ sn.Tags[i] = sn.Tags[len(sn.Tags)-1]
+ sn.Tags[len(sn.Tags)-1] = ""
+ sn.Tags = sn.Tags[:len(sn.Tags)-1]
+
+ changed = true
+ break
+ }
+ }
+ }
+ return
+}
+
+// HasTags returns true if the snapshot has at least all of tags.
func (sn *Snapshot) HasTags(tags []string) bool {
nextTag:
for _, tag := range tags {
@@ -120,33 +150,35 @@ nextTag:
return true
}
-// SamePaths compares the Snapshot's paths and provided paths are exactly the same
-func SamePaths(expected, actual []string) bool {
- if len(expected) == 0 || len(actual) == 0 {
- return true
- }
-
- for i := range expected {
- found := false
- for j := range actual {
- if expected[i] == actual[j] {
- found = true
- break
+// HasPaths returns true if the snapshot has at least all of paths.
+func (sn *Snapshot) HasPaths(paths []string) bool {
+nextPath:
+ for _, path := range paths {
+ for _, snPath := range sn.Paths {
+ if path == snPath {
+ continue nextPath
}
}
- if !found {
- return false
- }
+
+ return false
}
return true
}
+// SamePaths returns true if the snapshot matches the entire paths set
+func (sn *Snapshot) SamePaths(paths []string) bool {
+ if len(sn.Paths) != len(paths) {
+ return false
+ }
+ return sn.HasPaths(paths)
+}
+
// ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found.
var ErrNoSnapshotFound = errors.New("no snapshot found")
-// FindLatestSnapshot finds latest snapshot with optional target/directory and hostname filters.
-func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID, error) {
+// FindLatestSnapshot finds latest snapshot with optional target/directory, tags and hostname filters.
+func FindLatestSnapshot(repo Repository, targets []string, tags []string, hostname string) (ID, error) {
var (
latest time.Time
latestID ID
@@ -158,7 +190,7 @@ func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID,
if err != nil {
return ID{}, errors.Errorf("Error listing snapshot: %v", err)
}
- if snapshot.Time.After(latest) && SamePaths(snapshot.Paths, targets) && (hostname == "" || hostname == snapshot.Hostname) {
+ if snapshot.Time.After(latest) && (hostname == "" || hostname == snapshot.Hostname) && snapshot.HasTags(tags) && snapshot.HasPaths(targets) {
latest = snapshot.Time
latestID = snapshotID
found = true
diff --git a/src/restic/snapshot_test.go b/src/restic/snapshot_test.go
index 2457e8d..76e95b1 100644
--- a/src/restic/snapshot_test.go
+++ b/src/restic/snapshot_test.go
@@ -10,6 +10,6 @@ import (
func TestNewSnapshot(t *testing.T) {
paths := []string{"/home/foobar"}
- _, err := restic.NewSnapshot(paths, nil)
+ _, err := restic.NewSnapshot(paths, nil, "foo")
OK(t, err)
}
diff --git a/src/restic/testing.go b/src/restic/testing.go
index 719ff33..9df54e7 100644
--- a/src/restic/testing.go
+++ b/src/restic/testing.go
@@ -163,7 +163,7 @@ func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int,
t.Logf("create fake snapshot at %s with seed %d", at, seed)
fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05"))
- snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"})
+ snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo")
if err != nil {
t.Fatal(err)
}
diff --git a/src/restic/tree_test.go b/src/restic/tree_test.go
index 1d23e92..967ac18 100644
--- a/src/restic/tree_test.go
+++ b/src/restic/tree_test.go
@@ -82,7 +82,7 @@ func TestNodeComparison(t *testing.T) {
fi, err := os.Lstat("tree_test.go")
OK(t, err)
- node, err := restic.NodeFromFileInfo("foo", fi)
+ node, err := restic.NodeFromFileInfo("tree_test.go", fi)
OK(t, err)
n2 := *node
diff --git a/src/restic/walk/walk_test.go b/src/restic/walk/walk_test.go
index afaebbb..057e512 100644
--- a/src/restic/walk/walk_test.go
+++ b/src/restic/walk/walk_test.go
@@ -24,7 +24,7 @@ func TestWalkTree(t *testing.T) {
// archive a few files
arch := archiver.New(repo)
- sn, _, err := arch.Snapshot(nil, dirs, nil, nil)
+ sn, _, err := arch.Snapshot(nil, dirs, nil, "localhost", nil)
OK(t, err)
// flush repo, write all packs
diff --git a/vendor/manifest b/vendor/manifest
index 1d128b7..ec75a1f 100644
--- a/vendor/manifest
+++ b/vendor/manifest
@@ -50,6 +50,12 @@
"branch": "master"
},
{
+ "importpath": "github.com/pkg/xattr",
+ "repository": "https://github.com/pkg/xattr",
+ "revision": "b867675798fa7708a444945602b452ca493f2272",
+ "branch": "master"
+ },
+ {
"importpath": "github.com/restic/chunker",
"repository": "https://github.com/restic/chunker",
"revision": "49e9b5212b022a1ab373faf981ed4f2fc807502a",