Codebase list golang-github-vbatts-go-mtree / 1cf0881
New upstream version 0.4.2 Dmitry Smirnov 5 years ago
106 changed file(s) with 7871 addition(s) and 0 deletion(s). Raw diff Collapse all Expand all
+6
-0
.gitignore less more
0 *~
1 .cli.test
2 .lint
3 .test
4 .vet
5 gomtree
+21
-0
.travis.yml less more
0 language: go
1 go:
2 - 1.x
3 - 1.8.x
4 - 1.7.x
5 - 1.6.3
6
7 sudo: false
8
9 before_install:
10 - git config --global url."https://".insteadOf git://
11 - go get -u github.com/golang/lint/golint
12 - go get -u github.com/Masterminds/glide
13 - mkdir -p $GOPATH/src/github.com/vbatts && ln -sf $(pwd) $GOPATH/src/github.com/vbatts/go-mtree
14
15 install: true
16
17 script:
18 - make validation
19 - make validation.tags
20 - make build.arches
+66
-0
.vscode/tasks.json less more
0 // Available variables which can be used inside of strings.
1 // ${workspaceRoot}: the root folder of the team
2 // ${file}: the current opened file
3 // ${fileBasename}: the current opened file's basename
4 // ${fileDirname}: the current opened file's dirname
5 // ${fileExtname}: the current opened file's extension
6 // ${cwd}: the current working directory of the spawned process
7
8 {
9 // See https://go.microsoft.com/fwlink/?LinkId=733558
10 // for the documentation about the tasks.json format
11 "version": "2.0.0",
12 "tasks": [
13 {
14 "taskName": "build",
15 "type": "shell",
16 "command": "time go build .",
17 "problemMatcher": [
18 "$go"
19 ],
20 "group": {
21 "kind": "build",
22 "isDefault": true
23 },
24 "presentation": {
25 "echo": true,
26 "reveal": "silent",
27 "focus": true,
28 "panel": "shared"
29 }
30 },
31 {
32 "taskName": "build.arches",
33 "type": "shell",
34 "command": "make build.arches",
35 "problemMatcher": [
36 "$go"
37 ],
38 "group": "build",
39 "presentation": {
40 "echo": true,
41 "reveal": "always",
42 "focus": true,
43 "panel": "shared"
44 }
45 },
46 {
47 "taskName": "test",
48 "command": "time go test -v .",
49 "type": "shell",
50 "group": {
51 "kind": "test",
52 "isDefault": true
53 },
54 "problemMatcher": [
55 "$go"
56 ],
57 "presentation": {
58 "echo": true,
59 "reveal": "always",
60 "focus": true,
61 "panel": "shared"
62 }
63 }
64 ]
65 }⏎
+28
-0
LICENSE less more
0 Copyright (c) 2016 Vincent Batts, Raleigh, NC, USA
1
2 All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions are met:
6
7 1. Redistributions of source code must retain the above copyright notice, this
8 list of conditions and the following disclaimer.
9
10 2. Redistributions in binary form must reproduce the above copyright notice,
11 this list of conditions and the following disclaimer in the documentation
12 and/or other materials provided with the distribution.
13
14 3. Neither the name of the copyright holder nor the names of its contributors
15 may be used to endorse or promote products derived from this software without
16 specific prior written permission.
17
18 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22 FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24 SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25 CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26 OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+81
-0
Makefile less more
0
1 BUILD := gomtree
2 BUILDPATH := github.com/vbatts/go-mtree/cmd/gomtree
3 CWD := $(shell pwd)
4 SOURCE_FILES := $(shell find . -type f -name "*.go")
5 CLEAN_FILES := *~
6 TAGS :=
7 ARCHES := linux,386 linux,amd64 linux,arm linux,arm64 openbsd,amd64 windows,amd64 darwin,amd64
8
9 default: build validation
10
11 .PHONY: validation
12 validation: .test .lint .vet .cli.test
13
14 .PHONY: validation.tags
15 validation.tags: .test.tags .vet.tags .cli.test
16
17 .PHONY: test
18 test: .test
19
20 CLEAN_FILES += .test .test.tags
21
22 .test: $(SOURCE_FILES)
23 go test -v $$(glide novendor) && touch $@
24
25 .test.tags: $(SOURCE_FILES)
26 set -e ; for tag in $(TAGS) ; do go test -tags $$tag -v $$(glide novendor) ; done && touch $@
27
28 .PHONY: lint
29 lint: .lint
30
31 CLEAN_FILES += .lint
32
33 .lint: $(SOURCE_FILES)
34 set -e ; for dir in $$(glide novendor) ; do golint -set_exit_status $$dir ; done && touch $@
35
36 .PHONY: vet
37 vet: .vet .vet.tags
38
39 CLEAN_FILES += .vet .vet.tags
40
41 .vet: $(SOURCE_FILES)
42 go vet $$(glide novendor) && touch $@
43
44 .vet.tags: $(SOURCE_FILES)
45 set -e ; for tag in $(TAGS) ; do go vet -tags $$tag -v $$(glide novendor) ; done && touch $@
46
47 .PHONY: cli.test
48 cli.test: .cli.test
49
50 CLEAN_FILES += .cli.test .cli.test.tags
51
52 .cli.test: $(BUILD) $(wildcard ./test/cli/*.sh)
53 @go run ./test/cli.go ./test/cli/*.sh && touch $@
54
55 .cli.test.tags: $(BUILD) $(wildcard ./test/cli/*.sh)
56 @set -e ; for tag in $(TAGS) ; do go run -tags $$tag ./test/cli.go ./test/cli/*.sh ; done && touch $@
57
58 .PHONY: build
59 build: $(BUILD)
60
61 $(BUILD): $(SOURCE_FILES)
62 go build -o $(BUILD) $(BUILDPATH)
63
64 ./bin:
65 mkdir -p $@
66
67 CLEAN_FILES += bin
68
69 build.arches: ./bin
70 @set -e ;\
71 for pair in $(ARCHES); do \
72 p=$$(echo $$pair | cut -d , -f 1);\
73 a=$$(echo $$pair | cut -d , -f 2);\
74 echo "Building $$p/$$a ...";\
75 GOOS=$$p GOARCH=$$a go build -o ./bin/gomtree.$$p.$$a $(BUILDPATH) ;\
76 done
77
78 clean:
79 rm -rf $(BUILD) $(CLEAN_FILES)
80
+213
-0
README.md less more
0 # go-mtree
1
2 [![Go Report Card](https://goreportcard.com/badge/github.com/vbatts/go-mtree)](https://goreportcard.com/report/github.com/vbatts/go-mtree)
3
4 `mtree` is a filesystem hierarchy validation tooling and format.
5 This is a library and simple cli tool for [mtree(8)][mtree(8)] support.
6
7 While the traditional `mtree` cli utility is primarily on BSDs (FreeBSD,
8 openBSD, etc), even broader support for the `mtree` specification format is
9 provided with libarchive ([libarchive-formats(5)][libarchive-formats(5)]).
10
11 There is also an [mtree port for Linux][archiecobbs/mtree-port] though it is
12 not widely packaged for Linux distributions.
13
14
15 ## Format
16
17 The format of hierarchy specification is consistent with the `# mtree v2.0`
18 format. Both the BSD `mtree` and libarchive ought to be interoperable with it
19 with only one definite caveat. On Linux, extended attributes (`xattr`) on
20 files are often a critical aspect of the file, holding ACLs, capabilities, etc.
21 While FreeBSD filesystem do support `extattr`, this feature has not made its
22 way into their `mtree`.
23
24 This implementation of mtree supports a few non-upstream "keyword"s, such as:
25 `xattr` and `tar_time`. If you include these keywords, the FreeBSD `mtree`
26 will fail, as they are unknown keywords to that implementation.
27
28 To have `go-mtree` produce specifications that will be
29 strictly compatible with the BSD `mtree`, use the `-bsd-keywords` flag when
30 creating a manifest. This will make sure that only the keywords supported by
31 BSD `mtree` are used in the program.
32
33
34 ### Typical form
35
36 With the standard keywords, plus say `sha256digest`, the hierarchy
37 specification looks like:
38
39 ```mtree
40 # .
41 /set type=file nlink=1 mode=0664 uid=1000 gid=100
42 . size=4096 type=dir mode=0755 nlink=6 time=1459370393.273231538
43 LICENSE size=1502 mode=0644 time=1458851690.0 sha256digest=ef4e53d83096be56dc38dbf9bc8ba9e3068bec1ec37c179033d1e8f99a1c2a95
44 README.md size=2820 mode=0644 time=1459370256.316148361 sha256digest=d9b955134d99f84b17c0a711ce507515cc93cd7080a9dcd50400e3d993d876ac
45
46 [...]
47 ```
48
49 See the directory presently in, and the files present. Along with each
50 path, is provided the keywords and the unique values for each path. Any common
51 keyword and values are established in the `/set` command.
52
53
54 ### Extended attributes form
55
56 ```mtree
57 # .
58 /set type=file nlink=1 mode=0664 uid=1000 gid=1000
59 . size=4096 type=dir mode=0775 nlink=6 time=1459370191.11179595 xattr.security.selinux=dW5jb25maW5lZF91Om9iamVjdF9yOnVzZXJfaG9tZV90OnMwAA==
60 LICENSE size=1502 time=1458851690.583562292 xattr.security.selinux=dW5jb25maW5lZF91Om9iamVjdF9yOnVzZXJfaG9tZV90OnMwAA==
61 README.md size=2366 mode=0644 time=1459369604.0 xattr.security.selinux=dW5jb25maW5lZF91Om9iamVjdF9yOnVzZXJfaG9tZV90OnMwAA==
62
63 [...]
64 ```
65
66 See the keyword prefixed with `xattr.` followed by the extended attribute's
67 namespace and keyword. This setup is consistent for use with Linux extended
68 attributes as well as FreeBSD extended attributes.
69
70 Since extended attributes are an unordered hashmap, this approach allows for
71 checking each `<namespace>.<key>` individually.
72
73 The value is the [base64 encoded][base64] of the value of the particular
74 extended attribute. Since the values themselves could be raw bytes, this
75 approach avoids issues with encoding.
76
77 ### Tar form
78
79 ```mtree
80 # .
81 /set type=file mode=0664 uid=1000 gid=1000
82 . type=dir mode=0775 tar_time=1468430408.000000000
83
84 # samedir
85 samedir type=dir mode=0775 tar_time=1468000972.000000000
86 file2 size=0 tar_time=1467999782.000000000
87 file1 size=0 tar_time=1467999781.000000000
88
89 [...]
90 ```
91
92 While `go-mtree` serves mainly as a library for upstream `mtree` support,
93 `go-mtree` is also compatible with [tar archives][tar] (which is not an upstream feature).
94 This means that we can now create and validate a manifest by specifying a tar file.
95 More interestingly, this also means that we can create a manifest from an archive, and then
96 validate this manifest against a filesystem hierarchy that's on disk, and vice versa.
97
98 Notice that for the output of creating a validation manifest from a tar file, the default behavior
99 for evaluating a notion of time is to use the `tar_time` keyword. In the
100 "filesystem hierarchy" format of mtree, `time` is being evaluated with
101 nanosecond precision. However, GNU tar truncates a file's modification time
102 to 1-second precision. That is, if a file's full modification time is
103 123456789.123456789, the "tar time" equivalent would be 123456789.000000000.
104 This way, if you validate a manifest created using a tar file against an
105 actual root directory, there will be no complaints from `go-mtree` so long as the
106 1-second precision time of a file in the root directory is the same.
107
108
109 ## Usage
110
111 To use the Go programming language library, see [the docs][godoc].
112
113 To use the command line tool, first [build it](#Building), then the following.
114
115
116 ### Create a manifest
117
118 This will also include the sha512 digest of the files.
119
120 ```bash
121 gomtree -c -K sha512digest -p . > /tmp/root.mtree
122 ```
123
124 With a tar file:
125
126 ```bash
127 gomtree -c -K sha512digest -T sometarfile.tar > /tmp/tar.mtree
128 ```
129
130 ### Validate a manifest
131
132 ```bash
133 gomtree -p . -f /tmp/root.mtree
134 ```
135
136 With a tar file:
137
138 ```bash
139 gomtree -T sometarfile.tar -f /tmp/root.mtree
140 ```
141
142 ### See the supported keywords
143
144 ```bash
145 gomtree -list-keywords
146 Available keywords:
147 uname
148 sha1
149 sha1digest
150 sha256digest
151 xattrs (not upstream)
152 link (default)
153 nlink (default)
154 md5digest
155 rmd160digest
156 mode (default)
157 cksum
158 md5
159 rmd160
160 type (default)
161 time (default)
162 uid (default)
163 gid (default)
164 sha256
165 sha384
166 sha512
167 xattr (not upstream)
168 tar_time (not upstream)
169 size (default)
170 ripemd160digest
171 sha384digest
172 sha512digest
173 ```
174
175
176 ## Building
177
178 Either:
179
180 ```bash
181 go get github.com/vbatts/go-mtree/cmd/gomtree
182 ```
183
184 or
185
186 ```bash
187 git clone git://github.com/vbatts/go-mtree.git $GOPATH/src/github.com/vbatts/go-mtree
188 cd $GOPATH/src/github.com/vbatts/go-mtree
189 go build ./cmd/gomtree
190 ```
191
192 ## Testing
193
194 On Linux:
195 ```bash
196 cd $GOPATH/src/github.com/vbatts/go-mtree
197 make
198 ```
199
200 On FreeBSD:
201 ```bash
202 cd $GOPATH/src/github.com/vbatts/go-mtree
203 gmake
204 ```
205
206
207 [mtree(8)]: https://www.freebsd.org/cgi/man.cgi?mtree(8)
208 [libarchive-formats(5)]: https://www.freebsd.org/cgi/man.cgi?query=libarchive-formats&sektion=5&n=1
209 [archiecobbs/mtree-port]: https://github.com/archiecobbs/mtree-port
210 [godoc]: https://godoc.org/github.com/vbatts/go-mtree
211 [tar]: http://man7.org/linux/man-pages/man1/tar.1.html
212 [base64]: https://tools.ietf.org/html/rfc4648
+30
-0
check.go less more
0 package mtree
1
2 // Check a root directory path against the DirectoryHierarchy, regarding only
3 // the available keywords from the list and each entry in the hierarchy.
4 // If keywords is nil, the check all present in the DirectoryHierarchy
5 //
6 // This is equivalent to creating a new DirectoryHierarchy with Walk(root, nil,
7 // keywords, fs) and then doing a Compare(dh, newDh, keywords).
8 func Check(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval) ([]InodeDelta, error) {
9 if keywords == nil {
10 keywords = dh.UsedKeywords()
11 }
12
13 newDh, err := Walk(root, nil, keywords, fs)
14 if err != nil {
15 return nil, err
16 }
17
18 return Compare(dh, newDh, keywords)
19 }
20
21 // TarCheck is the tar equivalent of checking a file hierarchy spec against a
22 // tar stream to determine if files have been changed. This is precisely
23 // equivalent to Compare(dh, tarDH, keywords).
24 func TarCheck(tarDH, dh *DirectoryHierarchy, keywords []Keyword) ([]InodeDelta, error) {
25 if keywords == nil {
26 return Compare(dh, tarDH, dh.UsedKeywords())
27 }
28 return Compare(dh, tarDH, keywords)
29 }
+320
-0
check_test.go less more
0 package mtree
1
2 import (
3 "bytes"
4 "io/ioutil"
5 "os"
6 "path/filepath"
7 "testing"
8 "time"
9 )
10
11 // simple walk of current directory, and imediately check it.
12 // may not be parallelizable.
13 func TestCheck(t *testing.T) {
14 dh, err := Walk(".", nil, append(DefaultKeywords, []Keyword{"sha1", "xattr"}...), nil)
15 if err != nil {
16 t.Fatal(err)
17 }
18
19 res, err := Check(".", dh, nil, nil)
20 if err != nil {
21 t.Fatal(err)
22 }
23
24 if len(res) > 0 {
25 t.Errorf("%#v", res)
26 }
27 }
28
29 // make a directory, walk it, check it, modify the timestamp and ensure it fails.
30 // only check again for size and sha1, and ignore time, and ensure it passes
31 func TestCheckKeywords(t *testing.T) {
32 content := []byte("I know half of you half as well as I ought to")
33 dir, err := ioutil.TempDir("", "test-check-keywords")
34 if err != nil {
35 t.Fatal(err)
36 }
37 defer os.RemoveAll(dir) // clean up
38
39 tmpfn := filepath.Join(dir, "tmpfile")
40 if err := ioutil.WriteFile(tmpfn, content, 0666); err != nil {
41 t.Fatal(err)
42 }
43
44 // Walk this tempdir
45 dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
46 if err != nil {
47 t.Fatal(err)
48 }
49
50 // Check for sanity. This ought to pass.
51 res, err := Check(dir, dh, nil, nil)
52 if err != nil {
53 t.Fatal(err)
54 }
55 if len(res) > 0 {
56 t.Errorf("%#v", res)
57 }
58
59 // Touch a file, so the mtime changes.
60 newtime := time.Date(2006, time.February, 1, 3, 4, 5, 0, time.UTC)
61 if err := os.Chtimes(tmpfn, newtime, newtime); err != nil {
62 t.Fatal(err)
63 }
64
65 // Check again. This ought to fail.
66 res, err = Check(dir, dh, nil, nil)
67 if err != nil {
68 t.Fatal(err)
69 }
70 if len(res) != 1 {
71 t.Fatal("expected to get 1 delta on changed mtimes, but did not")
72 }
73 if res[0].Type() != Modified {
74 t.Errorf("expected to get modified delta on changed mtimes, but did not")
75 }
76
77 // Check again, but only sha1 and mode. This ought to pass.
78 res, err = Check(dir, dh, []Keyword{"sha1", "mode"}, nil)
79 if err != nil {
80 t.Fatal(err)
81 }
82 if len(res) > 0 {
83 t.Errorf("%#v", res)
84 }
85 }
86
87 func ExampleCheck() {
88 dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
89 if err != nil {
90 // handle error ...
91 }
92
93 res, err := Check(".", dh, nil, nil)
94 if err != nil {
95 // handle error ...
96 }
97 if len(res) > 0 {
98 // handle failed validity ...
99 }
100 }
101
102 // Tests default action for evaluating a symlink, which is just to compare the
103 // link itself, not to follow it
104 func TestDefaultBrokenLink(t *testing.T) {
105 dh, err := Walk("./testdata/dirwithbrokenlink", nil, append(DefaultKeywords, "sha1"), nil)
106 if err != nil {
107 t.Fatal(err)
108 }
109 res, err := Check("./testdata/dirwithbrokenlink", dh, nil, nil)
110 if err != nil {
111 t.Fatal(err)
112 }
113 if len(res) > 0 {
114 for _, delta := range res {
115 t.Error(delta)
116 }
117 }
118 }
119
120 // https://github.com/vbatts/go-mtree/issues/8
121 func TestTimeComparison(t *testing.T) {
122 dir, err := ioutil.TempDir("", "test-time.")
123 if err != nil {
124 t.Fatal(err)
125 }
126 defer os.RemoveAll(dir)
127
128 // This is the format of time from FreeBSD
129 spec := `
130 /set type=file time=5.000000000
131 . type=dir
132 file time=5.000000000
133 ..
134 `
135
136 fh, err := os.Create(filepath.Join(dir, "file"))
137 if err != nil {
138 t.Fatal(err)
139 }
140 // This is what mode we're checking for. Round integer of epoch seconds
141 epoch := time.Unix(5, 0)
142 if err := os.Chtimes(fh.Name(), epoch, epoch); err != nil {
143 t.Fatal(err)
144 }
145 if err := os.Chtimes(dir, epoch, epoch); err != nil {
146 t.Fatal(err)
147 }
148 if err := fh.Close(); err != nil {
149 t.Error(err)
150 }
151
152 dh, err := ParseSpec(bytes.NewBufferString(spec))
153 if err != nil {
154 t.Fatal(err)
155 }
156
157 res, err := Check(dir, dh, nil, nil)
158 if err != nil {
159 t.Error(err)
160 }
161 if len(res) > 0 {
162 t.Fatal(res)
163 }
164 }
165
166 func TestTarTime(t *testing.T) {
167 dir, err := ioutil.TempDir("", "test-tar-time.")
168 if err != nil {
169 t.Fatal(err)
170 }
171 defer os.RemoveAll(dir)
172
173 // This is the format of time from FreeBSD
174 spec := `
175 /set type=file time=5.454353132
176 . type=dir time=5.123456789
177 file time=5.911134111
178 ..
179 `
180
181 fh, err := os.Create(filepath.Join(dir, "file"))
182 if err != nil {
183 t.Fatal(err)
184 }
185 // This is what mode we're checking for. Round integer of epoch seconds
186 epoch := time.Unix(5, 0)
187 if err := os.Chtimes(fh.Name(), epoch, epoch); err != nil {
188 t.Fatal(err)
189 }
190 if err := os.Chtimes(dir, epoch, epoch); err != nil {
191 t.Fatal(err)
192 }
193 if err := fh.Close(); err != nil {
194 t.Error(err)
195 }
196
197 dh, err := ParseSpec(bytes.NewBufferString(spec))
198 if err != nil {
199 t.Fatal(err)
200 }
201
202 keywords := dh.UsedKeywords()
203
204 // make sure "time" keyword works
205 _, err = Check(dir, dh, keywords, nil)
206 if err != nil {
207 t.Error(err)
208 }
209
210 // make sure tar_time wins
211 res, err := Check(dir, dh, append(keywords, "tar_time"), nil)
212 if err != nil {
213 t.Error(err)
214 }
215 if len(res) > 0 {
216 t.Fatal(res)
217 }
218 }
219
220 func TestIgnoreComments(t *testing.T) {
221 dir, err := ioutil.TempDir("", "test-comments.")
222 if err != nil {
223 t.Fatal(err)
224 }
225 defer os.RemoveAll(dir)
226
227 // This is the format of time from FreeBSD
228 spec := `
229 /set type=file time=5.000000000
230 . type=dir
231 file1 time=5.000000000
232 ..
233 `
234
235 fh, err := os.Create(filepath.Join(dir, "file1"))
236 if err != nil {
237 t.Fatal(err)
238 }
239 // This is what mode we're checking for. Round integer of epoch seconds
240 epoch := time.Unix(5, 0)
241 if err := os.Chtimes(fh.Name(), epoch, epoch); err != nil {
242 t.Fatal(err)
243 }
244 if err := os.Chtimes(dir, epoch, epoch); err != nil {
245 t.Fatal(err)
246 }
247 if err := fh.Close(); err != nil {
248 t.Error(err)
249 }
250
251 dh, err := ParseSpec(bytes.NewBufferString(spec))
252 if err != nil {
253 t.Fatal(err)
254 }
255
256 res, err := Check(dir, dh, nil, nil)
257 if err != nil {
258 t.Error(err)
259 }
260
261 if len(res) > 0 {
262 t.Fatal(res)
263 }
264
265 // now change the spec to a comment that looks like an actual Entry but has
266 // whitespace in front of it
267 spec = `
268 /set type=file time=5.000000000
269 . type=dir
270 file1 time=5.000000000
271 #file2 time=5.000000000
272 ..
273 `
274 dh, err = ParseSpec(bytes.NewBufferString(spec))
275
276 res, err = Check(dir, dh, nil, nil)
277 if err != nil {
278 t.Error(err)
279 }
280
281 if len(res) > 0 {
282 t.Fatal(res)
283 }
284 }
285
286 func TestCheckNeedsEncoding(t *testing.T) {
287 dir, err := ioutil.TempDir("", "test-needs-encoding")
288 if err != nil {
289 t.Fatal(err)
290 }
291 defer os.RemoveAll(dir)
292
293 fh, err := os.Create(filepath.Join(dir, "file[ "))
294 if err != nil {
295 t.Fatal(err)
296 }
297 if err := fh.Close(); err != nil {
298 t.Error(err)
299 }
300 fh, err = os.Create(filepath.Join(dir, " , should work"))
301 if err != nil {
302 t.Fatal(err)
303 }
304 if err := fh.Close(); err != nil {
305 t.Error(err)
306 }
307
308 dh, err := Walk(dir, nil, DefaultKeywords, nil)
309 if err != nil {
310 t.Fatal(err)
311 }
312 res, err := Check(dir, dh, nil, nil)
313 if err != nil {
314 t.Fatal(err)
315 }
316 if len(res) > 0 {
317 t.Fatal(res)
318 }
319 }
+49
-0
cksum.go less more
0 package mtree
1
2 import (
3 "bufio"
4 "io"
5 )
6
7 const posixPolynomial uint32 = 0x04C11DB7
8
9 // cksum is an implementation of the POSIX CRC algorithm
10 func cksum(r io.Reader) (uint32, int, error) {
11 in := bufio.NewReader(r)
12 count := 0
13 var sum uint32
14 f := func(b byte) {
15 for i := 7; i >= 0; i-- {
16 msb := sum & (1 << 31)
17 sum = sum << 1
18 if msb != 0 {
19 sum = sum ^ posixPolynomial
20 }
21 }
22 sum ^= uint32(b)
23 }
24
25 for done := false; !done; {
26 switch b, err := in.ReadByte(); err {
27 case io.EOF:
28 done = true
29 case nil:
30 f(b)
31 count++
32 default:
33 return ^sum, count, err
34 }
35 }
36 for m := count; ; {
37 f(byte(m) & 0xff)
38 m = m >> 8
39 if m == 0 {
40 break
41 }
42 }
43 f(0)
44 f(0)
45 f(0)
46 f(0)
47 return ^sum, count, nil
48 }
+31
-0
cksum_test.go less more
0 package mtree
1
2 import (
3 "os"
4 "testing"
5 )
6
7 var (
8 checkFile = "./testdata/source.mtree"
9 checkSum uint32 = 1048442895
10 checkSize = 9110
11 )
12
13 // testing that the cksum function matches that of cksum(1) utility (silly POSIX crc32)
14 func TestCksum(t *testing.T) {
15 fh, err := os.Open(checkFile)
16 if err != nil {
17 t.Fatal(err)
18 }
19 defer fh.Close()
20 sum, i, err := cksum(fh)
21 if err != nil {
22 t.Fatal(err)
23 }
24 if i != checkSize {
25 t.Errorf("%q: expected size %d, got %d", checkFile, checkSize, i)
26 }
27 if sum != checkSum {
28 t.Errorf("%q: expected sum %d, got %d", checkFile, checkSum, sum)
29 }
30 }
+471
-0
cmd/gomtree/main.go less more
0 package main
1
2 import (
3 "bytes"
4 "encoding/json"
5 "flag"
6 "fmt"
7 "io"
8 "io/ioutil"
9 "os"
10 "strings"
11
12 "github.com/Sirupsen/logrus"
13 "github.com/vbatts/go-mtree"
14 )
15
16 var (
17 // Flags common with mtree(8)
18 flCreate = flag.Bool("c", false, "create a directory hierarchy spec")
19 flFile = flag.String("f", "", "directory hierarchy spec to validate")
20 flPath = flag.String("p", "", "root path that the hierarchy spec is relative to")
21 flAddKeywords = flag.String("K", "", "Add the specified (delimited by comma or space) keywords to the current set of keywords")
22 flUseKeywords = flag.String("k", "", "Use the specified (delimited by comma or space) keywords as the current set of keywords")
23 flDirectoryOnly = flag.Bool("d", false, "Ignore everything except directory type files")
24 flUpdateAttributes = flag.Bool("u", false, "Modify the owner, group, permissions and xattrs of files, symbolic links and devices, to match the provided specification. This is not compatible with '-T'.")
25
26 // Flags unique to gomtree
27 flListKeywords = flag.Bool("list-keywords", false, "List the keywords available")
28 flResultFormat = flag.String("result-format", "bsd", "output the validation results using the given format (bsd, json, path)")
29 flTar = flag.String("T", "", "use tar archive to create or validate a directory hierarchy spec (\"-\" indicates stdin)")
30 flBsdKeywords = flag.Bool("bsd-keywords", false, "only operate on keywords that are supported by upstream mtree(8)")
31 flListUsedKeywords = flag.Bool("list-used", false, "list all the keywords found in a validation manifest")
32 flDebug = flag.Bool("debug", false, "output debug info to STDERR")
33 flVersion = flag.Bool("version", false, "display the version of this tool")
34 )
35
36 func main() {
37 // so that defers cleanly exec
38 if err := app(); err != nil {
39 logrus.Fatal(err)
40 }
41 }
42
43 func app() error {
44 flag.Parse()
45
46 if *flDebug {
47 os.Setenv("DEBUG", "1")
48 logrus.SetLevel(logrus.DebugLevel)
49 }
50
51 if *flVersion {
52 fmt.Printf("%s :: %s\n", mtree.AppName, mtree.Version)
53 return nil
54 }
55
56 // -list-keywords
57 if *flListKeywords {
58 fmt.Println("Available keywords:")
59 for k := range mtree.KeywordFuncs {
60 fmt.Print(" ")
61 fmt.Print(k)
62 if mtree.Keyword(k).Default() {
63 fmt.Print(" (default)")
64 }
65 if !mtree.Keyword(k).Bsd() {
66 fmt.Print(" (not upstream)")
67 }
68 fmt.Print("\n")
69 }
70 return nil
71 }
72
73 // --result-format
74 formatFunc, ok := formats[*flResultFormat]
75 if !ok {
76 return fmt.Errorf("invalid output format: %s", *flResultFormat)
77 }
78
79 var (
80 err error
81 tmpKeywords []mtree.Keyword
82 currentKeywords []mtree.Keyword
83 )
84
85 // -k <keywords>
86 if *flUseKeywords != "" {
87 tmpKeywords = splitKeywordsArg(*flUseKeywords)
88 if !mtree.InKeywordSlice("type", tmpKeywords) {
89 tmpKeywords = append([]mtree.Keyword{"type"}, tmpKeywords...)
90 }
91 } else {
92 if *flTar != "" {
93 tmpKeywords = mtree.DefaultTarKeywords[:]
94 } else {
95 tmpKeywords = mtree.DefaultKeywords[:]
96 }
97 }
98
99 // -K <keywords>
100 if *flAddKeywords != "" {
101 for _, kw := range splitKeywordsArg(*flAddKeywords) {
102 if !mtree.InKeywordSlice(kw, tmpKeywords) {
103 tmpKeywords = append(tmpKeywords, kw)
104 }
105 }
106 }
107
108 // -bsd-keywords
109 if *flBsdKeywords {
110 for _, k := range tmpKeywords {
111 if mtree.Keyword(k).Bsd() {
112 currentKeywords = append(currentKeywords, k)
113 } else {
114 fmt.Fprintf(os.Stderr, "INFO: ignoring %q as it is not an upstream keyword\n", k)
115 }
116 }
117 } else {
118 currentKeywords = tmpKeywords
119 }
120
121 // Check mutual exclusivity of keywords.
122 // TODO(cyphar): Abstract this inside keywords.go.
123 if mtree.InKeywordSlice("tar_time", currentKeywords) && mtree.InKeywordSlice("time", currentKeywords) {
124 return fmt.Errorf("tar_time and time are mutually exclusive keywords")
125 }
126
127 // If we're doing a comparison, we always are comparing between a spec and
128 // state DH. If specDh is nil, we are generating a new one.
129 var (
130 specDh *mtree.DirectoryHierarchy
131 stateDh *mtree.DirectoryHierarchy
132 specKeywords []mtree.Keyword
133 )
134
135 // -f <file>
136 if *flFile != "" && !*flCreate {
137 // load the hierarchy, if we're not creating a new spec
138 fh, err := os.Open(*flFile)
139 if err != nil {
140 return err
141 }
142 specDh, err = mtree.ParseSpec(fh)
143 fh.Close()
144 if err != nil {
145 return err
146 }
147
148 // We can't check against more fields than in the specKeywords list, so
149 // currentKeywords can only have a subset of specKeywords.
150 specKeywords = specDh.UsedKeywords()
151 }
152
153 // -list-used
154 if *flListUsedKeywords {
155 if specDh == nil {
156 return fmt.Errorf("no specification provided. please provide a validation manifest")
157 }
158
159 if *flResultFormat == "json" {
160 // if they're asking for json, give it to them
161 data := map[string][]mtree.Keyword{*flFile: specKeywords}
162 buf, err := json.MarshalIndent(data, "", " ")
163 if err != nil {
164 return err
165 }
166 fmt.Println(string(buf))
167 } else {
168 fmt.Printf("Keywords used in [%s]:\n", *flFile)
169 for _, kw := range specKeywords {
170 fmt.Printf(" %s", kw)
171 if _, ok := mtree.KeywordFuncs[kw]; !ok {
172 fmt.Print(" (unsupported)")
173 }
174 fmt.Printf("\n")
175 }
176 }
177 return nil
178 }
179
180 if specKeywords != nil {
181 // If we didn't actually change the set of keywords, we can just use specKeywords.
182 if *flUseKeywords == "" && *flAddKeywords == "" {
183 currentKeywords = specKeywords
184 }
185
186 for _, keyword := range currentKeywords {
187 // As always, time is a special case.
188 // TODO: Fix that.
189 if (keyword == "time" && mtree.InKeywordSlice("tar_time", specKeywords)) || (keyword == "tar_time" && mtree.InKeywordSlice("time", specKeywords)) {
190 continue
191 }
192 }
193 }
194
195 // -p and -T are mutually exclusive
196 if *flPath != "" && *flTar != "" {
197 return fmt.Errorf("options -T and -p are mutually exclusive")
198 }
199
200 // -p <path>
201 var rootPath = "."
202 if *flPath != "" {
203 rootPath = *flPath
204 }
205
206 excludes := []mtree.ExcludeFunc{}
207 // -d
208 if *flDirectoryOnly {
209 excludes = append(excludes, mtree.ExcludeNonDirectories)
210 }
211
212 // -u
213 // Failing early here. Processing is done below.
214 if *flUpdateAttributes && *flTar != "" {
215 return fmt.Errorf("ERROR: -u can not be used with -T")
216 }
217
218 // -T <tar file>
219 if *flTar != "" {
220 var input io.Reader
221 if *flTar == "-" {
222 input = os.Stdin
223 } else {
224 fh, err := os.Open(*flTar)
225 if err != nil {
226 return err
227 }
228 defer fh.Close()
229 input = fh
230 }
231 ts := mtree.NewTarStreamer(input, excludes, currentKeywords)
232
233 if _, err := io.Copy(ioutil.Discard, ts); err != nil && err != io.EOF {
234 return err
235 }
236 if err := ts.Close(); err != nil {
237 return err
238 }
239 var err error
240 stateDh, err = ts.Hierarchy()
241 if err != nil {
242 return err
243 }
244 } else {
245 // with a root directory
246 stateDh, err = mtree.Walk(rootPath, excludes, currentKeywords, nil)
247 if err != nil {
248 return err
249 }
250 }
251
252 // -u
253 if *flUpdateAttributes && stateDh != nil {
254 // -u
255 // this comes before the next case, intentionally.
256 result, err := mtree.Update(rootPath, specDh, mtree.DefaultUpdateKeywords, nil)
257 if err != nil {
258 return err
259 }
260 if result != nil && len(result) > 0 {
261 fmt.Printf("%#v\n", result)
262 }
263
264 var res []mtree.InodeDelta
265 // only check the keywords that we just updated
266 res, err = mtree.Check(rootPath, specDh, mtree.DefaultUpdateKeywords, nil)
267 if err != nil {
268 return err
269 }
270 if res != nil {
271 out := formatFunc(res)
272 if _, err := os.Stdout.Write([]byte(out)); err != nil {
273 return err
274 }
275
276 // TODO: This should be a flag. Allowing files to be added and
277 // removed and still returning "it's all good" is simply
278 // unsafe IMO.
279 for _, diff := range res {
280 if diff.Type() == mtree.Modified {
281 return fmt.Errorf("mainfest validation failed")
282 }
283 }
284 }
285
286 return nil
287 }
288
289 // -c
290 if *flCreate {
291 fh := os.Stdout
292 if *flFile != "" {
293 fh, err = os.Create(*flFile)
294 if err != nil {
295 return err
296 }
297 }
298
299 // output stateDh
300 stateDh.WriteTo(fh)
301 return nil
302 }
303
304 // no spec manifest has been provided yet, so look for it on stdin
305 if specDh == nil {
306 // load the hierarchy
307 specDh, err = mtree.ParseSpec(os.Stdin)
308 if err != nil {
309 return err
310 }
311
312 // We can't check against more fields than in the specKeywords list, so
313 // currentKeywords can only have a subset of specKeywords.
314 specKeywords = specDh.UsedKeywords()
315 }
316
317 // This is a validation.
318 if specDh != nil && stateDh != nil {
319 var res []mtree.InodeDelta
320 res, err = mtree.Compare(specDh, stateDh, currentKeywords)
321 if err != nil {
322 return err
323 }
324 if res != nil {
325 if isTarSpec(specDh) || *flTar != "" {
326 res = filterMissingKeywords(res)
327 }
328
329 out := formatFunc(res)
330 if _, err := os.Stdout.Write([]byte(out)); err != nil {
331 return err
332 }
333
334 // TODO: This should be a flag. Allowing files to be added and
335 // removed and still returning "it's all good" is simply
336 // unsafe IMO.
337 for _, diff := range res {
338 if diff.Type() == mtree.Modified {
339 return fmt.Errorf("mainfest validation failed")
340 }
341 }
342 }
343 } else {
344 return fmt.Errorf("neither validating or creating a manifest. Please provide additional arguments")
345 }
346 return nil
347 }
348
349 var formats = map[string]func([]mtree.InodeDelta) string{
350 // Outputs the errors in the BSD format.
351 "bsd": func(d []mtree.InodeDelta) string {
352 var buffer bytes.Buffer
353 for _, delta := range d {
354 fmt.Fprintln(&buffer, delta)
355 }
356 return buffer.String()
357 },
358
359 // Outputs the full result struct in JSON.
360 "json": func(d []mtree.InodeDelta) string {
361 var buffer bytes.Buffer
362 if err := json.NewEncoder(&buffer).Encode(d); err != nil {
363 panic(err)
364 }
365 return buffer.String()
366 },
367
368 // Outputs only the paths which failed to validate.
369 "path": func(d []mtree.InodeDelta) string {
370 var buffer bytes.Buffer
371 for _, delta := range d {
372 if delta.Type() == mtree.Modified {
373 fmt.Fprintln(&buffer, delta.Path())
374 }
375 }
376 return buffer.String()
377 },
378 }
379
380 // isDirEntry returns wheter an mtree.Entry describes a directory.
381 func isDirEntry(e mtree.Entry) bool {
382 for _, kw := range e.Keywords {
383 kv := mtree.KeyVal(kw)
384 if kv.Keyword() == "type" {
385 return kv.Value() == "dir"
386 }
387 }
388 // Shouldn't be reached.
389 return false
390 }
391
392 // filterMissingKeywords is a fairly annoying hack to get around the fact that
393 // tar archive manifest generation has certain unsolveable problems regarding
394 // certain keywords. For example, the size=... keyword cannot be implemented
395 // for directories in a tar archive (which causes Missing errors for that
396 // keyword).
397 //
398 // This function just removes all instances of Missing errors for keywords.
399 // This makes certain assumptions about the type of issues tar archives have.
400 // Only call this on tar archive manifest comparisons.
401 func filterMissingKeywords(diffs []mtree.InodeDelta) []mtree.InodeDelta {
402 newDiffs := []mtree.InodeDelta{}
403 loop:
404 for _, diff := range diffs {
405 if diff.Type() == mtree.Modified {
406 // We only apply this filtering to directories.
407 // NOTE: This will probably break if someone drops the size keyword.
408 if isDirEntry(*diff.Old()) || isDirEntry(*diff.New()) {
409 // If this applies to '.' then we just filter everything
410 // (meaning we remove this entry). This is because note all tar
411 // archives include a '.' entry. Which makes checking this not
412 // practical.
413 if diff.Path() == "." {
414 continue
415 }
416
417 // Only filter out the size keyword.
418 // NOTE: This currently takes advantage of the fact the
419 // diff.Diff() returns the actual slice to diff.keys.
420 keys := diff.Diff()
421 for idx, k := range keys {
422 // Delete the key if it's "size". Unfortunately in Go you
423 // can't delete from a slice without reassigning it. So we
424 // just overwrite it with the last value.
425 if k.Name() == "size" {
426 if len(keys) < 2 {
427 continue loop
428 }
429 keys[idx] = keys[len(keys)-1]
430 }
431 }
432 }
433 }
434
435 // If we got here, append to the new set.
436 newDiffs = append(newDiffs, diff)
437 }
438 return newDiffs
439 }
440
441 // isTarSpec returns whether the spec provided came from the tar generator.
442 // This takes advantage of an unsolveable problem in tar generation.
443 func isTarSpec(spec *mtree.DirectoryHierarchy) bool {
444 // Find a directory and check whether it's missing size=...
445 // NOTE: This will definitely break if someone drops the size=... keyword.
446 for _, e := range spec.Entries {
447 if !isDirEntry(e) {
448 continue
449 }
450
451 for _, kw := range e.Keywords {
452 kv := mtree.KeyVal(kw)
453 if kv.Keyword() == "size" {
454 return false
455 }
456 }
457 return true
458 }
459
460 // Should never be reached.
461 return false
462 }
463
464 func splitKeywordsArg(str string) []mtree.Keyword {
465 keywords := []mtree.Keyword{}
466 for _, kw := range strings.Fields(strings.Replace(str, ",", " ", -1)) {
467 keywords = append(keywords, mtree.KeywordSynonym(kw))
468 }
469 return keywords
470 }
+438
-0
compare.go less more
0 package mtree
1
2 import (
3 "encoding/json"
4 "fmt"
5 "strconv"
6 )
7
8 // XXX: Do we need a Difference interface to make it so people can do var x
9 // Difference = <something>? The main problem is that keys and inodes need to
10 // have different interfaces, so it's just a pain.
11
12 // DifferenceType represents the type of a discrepancy encountered for
13 // an object. This is also used to represent discrepancies between keys
14 // for objects.
15 type DifferenceType string
16
17 const (
18 // Missing represents a discrepancy where the object is present in
19 // the @old manifest but is not present in the @new manifest.
20 Missing DifferenceType = "missing"
21
22 // Extra represents a discrepancy where the object is not present in
23 // the @old manifest but is present in the @new manifest.
24 Extra DifferenceType = "extra"
25
26 // Modified represents a discrepancy where the object is present in
27 // both the @old and @new manifests, but one or more of the keys
28 // have different values (or have not been set in one of the
29 // manifests).
30 Modified DifferenceType = "modified"
31
32 // ErrorDifference represents an attempted update to the values of
33 // a keyword that failed
34 ErrorDifference DifferenceType = "errored"
35 )
36
37 // These functions return *type from the parameter. It's just shorthand, to
38 // ensure that we don't accidentally expose pointers to the caller that are
39 // internal data.
40 func ePtr(e Entry) *Entry { return &e }
41 func sPtr(s string) *string { return &s }
42
43 // InodeDelta Represents a discrepancy in a filesystem object between two
44 // DirectoryHierarchy manifests. Discrepancies are caused by entries only
45 // present in one manifest [Missing, Extra], keys only present in one of the
46 // manifests [Modified] or a difference between the keys of the same object in
47 // both manifests [Modified].
48 type InodeDelta struct {
49 diff DifferenceType
50 path string
51 new Entry
52 old Entry
53 keys []KeyDelta
54 }
55
56 // Type returns the type of discrepancy encountered when comparing this inode
57 // between the two DirectoryHierarchy manifests.
58 func (i InodeDelta) Type() DifferenceType {
59 return i.diff
60 }
61
62 // Path returns the path to the inode (relative to the root of the
63 // DirectoryHierarchy manifests).
64 func (i InodeDelta) Path() string {
65 return i.path
66 }
67
68 // Diff returns the set of key discrepancies between the two manifests for the
69 // specific inode. If the DifferenceType of the inode is not Modified, then
70 // Diff returns nil.
71 func (i InodeDelta) Diff() []KeyDelta {
72 return i.keys
73 }
74
75 // Old returns the value of the inode Entry in the "old" DirectoryHierarchy (as
76 // determined by the ordering of parameters to Compare).
77 func (i InodeDelta) Old() *Entry {
78 if i.diff == Modified || i.diff == Missing {
79 return ePtr(i.old)
80 }
81 return nil
82 }
83
84 // New returns the value of the inode Entry in the "new" DirectoryHierarchy (as
85 // determined by the ordering of parameters to Compare).
86 func (i InodeDelta) New() *Entry {
87 if i.diff == Modified || i.diff == Extra {
88 return ePtr(i.new)
89 }
90 return nil
91 }
92
93 // MarshalJSON creates a JSON-encoded version of InodeDelta.
94 func (i InodeDelta) MarshalJSON() ([]byte, error) {
95 return json.Marshal(struct {
96 Type DifferenceType `json:"type"`
97 Path string `json:"path"`
98 Keys []KeyDelta `json:"keys"`
99 }{
100 Type: i.diff,
101 Path: i.path,
102 Keys: i.keys,
103 })
104 }
105
106 // String returns a "pretty" formatting for InodeDelta.
107 func (i InodeDelta) String() string {
108 switch i.diff {
109 case Modified:
110 // Output the first failure.
111 f := i.keys[0]
112 return fmt.Sprintf("%q: keyword %q: expected %s; got %s", i.path, f.name, f.old, f.new)
113 case Extra:
114 return fmt.Sprintf("%q: unexpected path", i.path)
115 case Missing:
116 return fmt.Sprintf("%q: missing path", i.path)
117 default:
118 panic("programming error")
119 }
120 }
121
122 // KeyDelta Represents a discrepancy in a key for a particular filesystem
123 // object between two DirectoryHierarchy manifests. Discrepancies are caused by
124 // keys only present in one manifest [Missing, Extra] or a difference between
125 // the keys of the same object in both manifests [Modified]. A set of these is
126 // returned with InodeDelta.Diff().
127 type KeyDelta struct {
128 diff DifferenceType
129 name Keyword
130 old string
131 new string
132 err error // used for update delta results
133 }
134
135 // Type returns the type of discrepancy encountered when comparing this key
136 // between the two DirectoryHierarchy manifests' relevant inode entry.
137 func (k KeyDelta) Type() DifferenceType {
138 return k.diff
139 }
140
141 // Name returns the name (the key) of the KeyDeltaVal entry in the
142 // DirectoryHierarchy.
143 func (k KeyDelta) Name() Keyword {
144 return k.name
145 }
146
147 // Old returns the value of the KeyDeltaVal entry in the "old" DirectoryHierarchy
148 // (as determined by the ordering of parameters to Compare). Returns nil if
149 // there was no entry in the "old" DirectoryHierarchy.
150 func (k KeyDelta) Old() *string {
151 if k.diff == Modified || k.diff == Missing {
152 return sPtr(k.old)
153 }
154 return nil
155 }
156
157 // New returns the value of the KeyDeltaVal entry in the "new" DirectoryHierarchy
158 // (as determined by the ordering of parameters to Compare). Returns nil if
159 // there was no entry in the "old" DirectoryHierarchy.
160 func (k KeyDelta) New() *string {
161 if k.diff == Modified || k.diff == Extra {
162 return sPtr(k.old)
163 }
164 return nil
165 }
166
167 // MarshalJSON creates a JSON-encoded version of KeyDelta.
168 func (k KeyDelta) MarshalJSON() ([]byte, error) {
169 return json.Marshal(struct {
170 Type DifferenceType `json:"type"`
171 Name Keyword `json:"name"`
172 Old string `json:"old"`
173 New string `json:"new"`
174 }{
175 Type: k.diff,
176 Name: k.name,
177 Old: k.old,
178 New: k.new,
179 })
180 }
181
182 // Like Compare, but for single inode entries only. Used to compute the
183 // cached version of inode.keys.
184 func compareEntry(oldEntry, newEntry Entry) ([]KeyDelta, error) {
185 // Represents the new and old states for an entry's keys.
186 type stateT struct {
187 Old *KeyVal
188 New *KeyVal
189 }
190
191 diffs := map[Keyword]*stateT{}
192 oldKeys := oldEntry.AllKeys()
193 newKeys := newEntry.AllKeys()
194
195 // Fill the map with the old keys first.
196 for _, kv := range oldKeys {
197 key := kv.Keyword()
198 // only add this diff if the new keys has this keyword
199 if key != "tar_time" && key != "time" && key.Prefix() != "xattr" && len(HasKeyword(newKeys, key)) == 0 {
200 continue
201 }
202
203 // Cannot take &kv because it's the iterator.
204 copy := new(KeyVal)
205 *copy = kv
206
207 _, ok := diffs[key]
208 if !ok {
209 diffs[key] = new(stateT)
210 }
211 diffs[key].Old = copy
212 }
213
214 // Then fill the new keys.
215 for _, kv := range newKeys {
216 key := kv.Keyword()
217 // only add this diff if the old keys has this keyword
218 if key != "tar_time" && key != "time" && key.Prefix() != "xattr" && len(HasKeyword(oldKeys, key)) == 0 {
219 continue
220 }
221
222 // Cannot take &kv because it's the iterator.
223 copy := new(KeyVal)
224 *copy = kv
225
226 _, ok := diffs[key]
227 if !ok {
228 diffs[key] = new(stateT)
229 }
230 diffs[key].New = copy
231 }
232
233 // We need a full list of the keys so we can deal with different keyvalue
234 // orderings.
235 var kws []Keyword
236 for kw := range diffs {
237 kws = append(kws, kw)
238 }
239
240 // If both tar_time and time were specified in the set of keys, we have to
241 // mess with the diffs. This is an unfortunate side-effect of tar archives.
242 // TODO(cyphar): This really should be abstracted inside keywords.go
243 if InKeywordSlice("tar_time", kws) && InKeywordSlice("time", kws) {
244 // Delete "time".
245 timeStateT := diffs["time"]
246 delete(diffs, "time")
247
248 // Make a new tar_time.
249 if diffs["tar_time"].Old == nil {
250 time, err := strconv.ParseFloat(timeStateT.Old.Value(), 64)
251 if err != nil {
252 return nil, fmt.Errorf("failed to parse old time: %s", err)
253 }
254
255 newTime := new(KeyVal)
256 *newTime = KeyVal(fmt.Sprintf("tar_time=%d.000000000", int64(time)))
257
258 diffs["tar_time"].Old = newTime
259 } else if diffs["tar_time"].New == nil {
260 time, err := strconv.ParseFloat(timeStateT.New.Value(), 64)
261 if err != nil {
262 return nil, fmt.Errorf("failed to parse new time: %s", err)
263 }
264
265 newTime := new(KeyVal)
266 *newTime = KeyVal(fmt.Sprintf("tar_time=%d.000000000", int64(time)))
267
268 diffs["tar_time"].New = newTime
269 } else {
270 return nil, fmt.Errorf("time and tar_time set in the same manifest")
271 }
272 }
273
274 // Are there any differences?
275 var results []KeyDelta
276 for name, diff := range diffs {
277 // Invalid
278 if diff.Old == nil && diff.New == nil {
279 return nil, fmt.Errorf("invalid state: both old and new are nil: key=%s", name)
280 }
281
282 switch {
283 // Missing
284 case diff.New == nil:
285 results = append(results, KeyDelta{
286 diff: Missing,
287 name: name,
288 old: diff.Old.Value(),
289 })
290
291 // Extra
292 case diff.Old == nil:
293 results = append(results, KeyDelta{
294 diff: Extra,
295 name: name,
296 new: diff.New.Value(),
297 })
298
299 // Modified
300 default:
301 if !diff.Old.Equal(*diff.New) {
302 results = append(results, KeyDelta{
303 diff: Modified,
304 name: name,
305 old: diff.Old.Value(),
306 new: diff.New.Value(),
307 })
308 }
309 }
310 }
311
312 return results, nil
313 }
314
315 // Compare compares two directory hierarchy manifests, and returns the
316 // list of discrepancies between the two. All of the entries in the
317 // manifest are considered, with differences being generated for
318 // RelativeType and FullType entries. Differences in structure (such as
319 // the way /set and /unset are written) are not considered to be
320 // discrepancies. The list of differences are all filesystem objects.
321 //
322 // keys controls which keys will be compared, but if keys is nil then all
323 // possible keys will be compared between the two manifests (allowing for
324 // missing entries and the like). A missing or extra key is treated as a
325 // Modified type.
326 //
327 // NB: The order of the parameters matters (old, new) because Extra and
328 // Missing are considered as different discrepancy types.
329 func Compare(oldDh, newDh *DirectoryHierarchy, keys []Keyword) ([]InodeDelta, error) {
330 // Represents the new and old states for an entry.
331 type stateT struct {
332 Old *Entry
333 New *Entry
334 }
335
336 // To deal with different orderings of the entries, use a path-keyed
337 // map to make sure we don't start comparing unrelated entries.
338 diffs := map[string]*stateT{}
339
340 // First, iterate over the old hierarchy.
341 for _, e := range oldDh.Entries {
342 if e.Type == RelativeType || e.Type == FullType {
343 path, err := e.Path()
344 if err != nil {
345 return nil, err
346 }
347
348 // Cannot take &kv because it's the iterator.
349 copy := new(Entry)
350 *copy = e
351
352 _, ok := diffs[path]
353 if !ok {
354 diffs[path] = &stateT{}
355 }
356 diffs[path].Old = copy
357 }
358 }
359
360 // Then, iterate over the new hierarchy.
361 for _, e := range newDh.Entries {
362 if e.Type == RelativeType || e.Type == FullType {
363 path, err := e.Path()
364 if err != nil {
365 return nil, err
366 }
367
368 // Cannot take &kv because it's the iterator.
369 copy := new(Entry)
370 *copy = e
371
372 _, ok := diffs[path]
373 if !ok {
374 diffs[path] = &stateT{}
375 }
376 diffs[path].New = copy
377 }
378 }
379
380 // Now we compute the diff.
381 var results []InodeDelta
382 for path, diff := range diffs {
383 // Invalid
384 if diff.Old == nil && diff.New == nil {
385 return nil, fmt.Errorf("invalid state: both old and new are nil: path=%s", path)
386 }
387
388 switch {
389 // Missing
390 case diff.New == nil:
391 results = append(results, InodeDelta{
392 diff: Missing,
393 path: path,
394 old: *diff.Old,
395 })
396
397 // Extra
398 case diff.Old == nil:
399 results = append(results, InodeDelta{
400 diff: Extra,
401 path: path,
402 new: *diff.New,
403 })
404
405 // Modified
406 default:
407 changed, err := compareEntry(*diff.Old, *diff.New)
408 if err != nil {
409 return nil, fmt.Errorf("comparison failed %s: %s", path, err)
410 }
411
412 // Now remove "changed" entries that don't match the keys.
413 if keys != nil {
414 var filterChanged []KeyDelta
415 for _, keyDiff := range changed {
416 if InKeywordSlice(keyDiff.name.Prefix(), keys) {
417 filterChanged = append(filterChanged, keyDiff)
418 }
419 }
420 changed = filterChanged
421 }
422
423 // Check if there were any actual changes.
424 if len(changed) > 0 {
425 results = append(results, InodeDelta{
426 diff: Modified,
427 path: path,
428 old: *diff.Old,
429 new: *diff.New,
430 keys: changed,
431 })
432 }
433 }
434 }
435
436 return results, nil
437 }
+459
-0
compare_test.go less more
0 package mtree
1
2 import (
3 "archive/tar"
4 "bytes"
5 "encoding/json"
6 "io"
7 "io/ioutil"
8 "os"
9 "path/filepath"
10 "testing"
11 "time"
12 )
13
14 // simple walk of current directory, and imediately check it.
15 // may not be parallelizable.
16 func TestCompare(t *testing.T) {
17 old, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
18 if err != nil {
19 t.Fatal(err)
20 }
21
22 new, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
23 if err != nil {
24 t.Fatal(err)
25 }
26
27 diffs, err := Compare(old, new, nil)
28 if err != nil {
29 t.Fatal(err)
30 }
31
32 if len(diffs) > 0 {
33 t.Errorf("%#v", diffs)
34 }
35 }
36
37 func TestCompareModified(t *testing.T) {
38 dir, err := ioutil.TempDir("", "test-compare-modified")
39 if err != nil {
40 t.Fatal(err)
41 }
42 defer os.RemoveAll(dir)
43
44 // Create a bunch of objects.
45 tmpfile := filepath.Join(dir, "tmpfile")
46 if err := ioutil.WriteFile(tmpfile, []byte("some content here"), 0666); err != nil {
47 t.Fatal(err)
48 }
49
50 tmpdir := filepath.Join(dir, "testdir")
51 if err := os.Mkdir(tmpdir, 0755); err != nil {
52 t.Fatal(err)
53 }
54
55 tmpsubfile := filepath.Join(tmpdir, "anotherfile")
56 if err := ioutil.WriteFile(tmpsubfile, []byte("some different content"), 0666); err != nil {
57 t.Fatal(err)
58 }
59
60 // Walk the current state.
61 old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
62 if err != nil {
63 t.Fatal(err)
64 }
65
66 // Overwrite the content in one of the files.
67 if err := ioutil.WriteFile(tmpsubfile, []byte("modified content"), 0666); err != nil {
68 t.Fatal(err)
69 }
70
71 // Walk the new state.
72 new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
73 if err != nil {
74 t.Fatal(err)
75 }
76
77 // Compare.
78 diffs, err := Compare(old, new, nil)
79 if err != nil {
80 t.Fatal(err)
81 }
82
83 // 1 object
84 if len(diffs) != 1 {
85 t.Errorf("expected the diff length to be 1, got %d", len(diffs))
86 for i, diff := range diffs {
87 t.Logf("diff[%d] = %#v", i, diff)
88 }
89 }
90
91 // These cannot fail.
92 tmpfile, _ = filepath.Rel(dir, tmpfile)
93 tmpdir, _ = filepath.Rel(dir, tmpdir)
94 tmpsubfile, _ = filepath.Rel(dir, tmpsubfile)
95
96 for _, diff := range diffs {
97 switch diff.Path() {
98 case tmpsubfile:
99 if diff.Type() != Modified {
100 t.Errorf("unexpected diff type for %s: %s", diff.Path(), diff.Type())
101 }
102
103 if diff.Diff() == nil {
104 t.Errorf("expect to not get nil for .Diff()")
105 }
106
107 old := diff.Old()
108 new := diff.New()
109 if old == nil || new == nil {
110 t.Errorf("expected to get (!nil, !nil) for (.Old, .New), got (%#v, %#v)", old, new)
111 }
112 default:
113 t.Errorf("unexpected diff found: %#v", diff)
114 }
115 }
116 }
117
118 func TestCompareMissing(t *testing.T) {
119 dir, err := ioutil.TempDir("", "test-compare-missing")
120 if err != nil {
121 t.Fatal(err)
122 }
123 defer os.RemoveAll(dir)
124
125 // Create a bunch of objects.
126 tmpfile := filepath.Join(dir, "tmpfile")
127 if err := ioutil.WriteFile(tmpfile, []byte("some content here"), 0666); err != nil {
128 t.Fatal(err)
129 }
130
131 tmpdir := filepath.Join(dir, "testdir")
132 if err := os.Mkdir(tmpdir, 0755); err != nil {
133 t.Fatal(err)
134 }
135
136 tmpsubfile := filepath.Join(tmpdir, "anotherfile")
137 if err := ioutil.WriteFile(tmpsubfile, []byte("some different content"), 0666); err != nil {
138 t.Fatal(err)
139 }
140
141 // Walk the current state.
142 old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
143 if err != nil {
144 t.Fatal(err)
145 }
146
147 // Delete the objects.
148 if err := os.RemoveAll(tmpfile); err != nil {
149 t.Fatal(err)
150 }
151
152 if err := os.RemoveAll(tmpsubfile); err != nil {
153 t.Fatal(err)
154 }
155
156 if err := os.RemoveAll(tmpdir); err != nil {
157 t.Fatal(err)
158 }
159
160 // Walk the new state.
161 new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
162 if err != nil {
163 t.Fatal(err)
164 }
165
166 // Compare.
167 diffs, err := Compare(old, new, nil)
168 if err != nil {
169 t.Fatal(err)
170 }
171
172 // 3 objects + the changes to '.'
173 if len(diffs) != 4 {
174 t.Errorf("expected the diff length to be 4, got %d", len(diffs))
175 for i, diff := range diffs {
176 t.Logf("diff[%d] = %#v", i, diff)
177 }
178 }
179
180 // These cannot fail.
181 tmpfile, _ = filepath.Rel(dir, tmpfile)
182 tmpdir, _ = filepath.Rel(dir, tmpdir)
183 tmpsubfile, _ = filepath.Rel(dir, tmpsubfile)
184
185 for _, diff := range diffs {
186 switch diff.Path() {
187 case ".":
188 // ignore these changes
189 case tmpfile, tmpdir, tmpsubfile:
190 if diff.Type() != Missing {
191 t.Errorf("unexpected diff type for %s: %s", diff.Path(), diff.Type())
192 }
193
194 if diff.Diff() != nil {
195 t.Errorf("expect to get nil for .Diff(), got %#v", diff.Diff())
196 }
197
198 old := diff.Old()
199 new := diff.New()
200 if old == nil || new != nil {
201 t.Errorf("expected to get (!nil, nil) for (.Old, .New), got (%#v, %#v)", old, new)
202 }
203 default:
204 t.Errorf("unexpected diff found: %#v", diff)
205 }
206 }
207 }
208
209 func TestCompareExtra(t *testing.T) {
210 dir, err := ioutil.TempDir("", "test-compare-extra")
211 if err != nil {
212 t.Fatal(err)
213 }
214 defer os.RemoveAll(dir)
215
216 // Walk the current state.
217 old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
218 if err != nil {
219 t.Fatal(err)
220 }
221
222 // Create a bunch of objects.
223 tmpfile := filepath.Join(dir, "tmpfile")
224 if err := ioutil.WriteFile(tmpfile, []byte("some content here"), 0666); err != nil {
225 t.Fatal(err)
226 }
227
228 tmpdir := filepath.Join(dir, "testdir")
229 if err := os.Mkdir(tmpdir, 0755); err != nil {
230 t.Fatal(err)
231 }
232
233 tmpsubfile := filepath.Join(tmpdir, "anotherfile")
234 if err := ioutil.WriteFile(tmpsubfile, []byte("some different content"), 0666); err != nil {
235 t.Fatal(err)
236 }
237
238 // Walk the new state.
239 new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
240 if err != nil {
241 t.Fatal(err)
242 }
243
244 // Compare.
245 diffs, err := Compare(old, new, nil)
246 if err != nil {
247 t.Fatal(err)
248 }
249
250 // 3 objects + the changes to '.'
251 if len(diffs) != 4 {
252 t.Errorf("expected the diff length to be 4, got %d", len(diffs))
253 for i, diff := range diffs {
254 t.Logf("diff[%d] = %#v", i, diff)
255 }
256 }
257
258 // These cannot fail.
259 tmpfile, _ = filepath.Rel(dir, tmpfile)
260 tmpdir, _ = filepath.Rel(dir, tmpdir)
261 tmpsubfile, _ = filepath.Rel(dir, tmpsubfile)
262
263 for _, diff := range diffs {
264 switch diff.Path() {
265 case ".":
266 // ignore these changes
267 case tmpfile, tmpdir, tmpsubfile:
268 if diff.Type() != Extra {
269 t.Errorf("unexpected diff type for %s: %s", diff.Path(), diff.Type())
270 }
271
272 if diff.Diff() != nil {
273 t.Errorf("expect to get nil for .Diff(), got %#v", diff.Diff())
274 }
275
276 old := diff.Old()
277 new := diff.New()
278 if old != nil || new == nil {
279 t.Errorf("expected to get (!nil, nil) for (.Old, .New), got (%#v, %#v)", old, new)
280 }
281 default:
282 t.Errorf("unexpected diff found: %#v", diff)
283 }
284 }
285 }
286
287 func TestCompareKeys(t *testing.T) {
288 dir, err := ioutil.TempDir("", "test-compare-keys")
289 if err != nil {
290 t.Fatal(err)
291 }
292 defer os.RemoveAll(dir)
293
294 // Create a bunch of objects.
295 tmpfile := filepath.Join(dir, "tmpfile")
296 if err := ioutil.WriteFile(tmpfile, []byte("some content here"), 0666); err != nil {
297 t.Fatal(err)
298 }
299
300 tmpdir := filepath.Join(dir, "testdir")
301 if err := os.Mkdir(tmpdir, 0755); err != nil {
302 t.Fatal(err)
303 }
304
305 tmpsubfile := filepath.Join(tmpdir, "anotherfile")
306 if err := ioutil.WriteFile(tmpsubfile, []byte("aaa"), 0666); err != nil {
307 t.Fatal(err)
308 }
309
310 // Walk the current state.
311 old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
312 if err != nil {
313 t.Fatal(err)
314 }
315
316 // Overwrite the content in one of the files, but without changing the size.
317 if err := ioutil.WriteFile(tmpsubfile, []byte("bbb"), 0666); err != nil {
318 t.Fatal(err)
319 }
320
321 // Walk the new state.
322 new, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
323 if err != nil {
324 t.Fatal(err)
325 }
326
327 // Compare.
328 diffs, err := Compare(old, new, []Keyword{"size"})
329 if err != nil {
330 t.Fatal(err)
331 }
332
333 // 0 objects
334 if len(diffs) != 0 {
335 t.Errorf("expected the diff length to be 0, got %d", len(diffs))
336 for i, diff := range diffs {
337 t.Logf("diff[%d] = %#v", i, diff)
338 }
339 }
340 }
341
342 func TestTarCompare(t *testing.T) {
343 dir, err := ioutil.TempDir("", "test-compare-tar")
344 if err != nil {
345 t.Fatal(err)
346 }
347 defer os.RemoveAll(dir)
348
349 // Create a bunch of objects.
350 tmpfile := filepath.Join(dir, "tmpfile")
351 if err := ioutil.WriteFile(tmpfile, []byte("some content"), 0644); err != nil {
352 t.Fatal(err)
353 }
354
355 tmpdir := filepath.Join(dir, "testdir")
356 if err := os.Mkdir(tmpdir, 0755); err != nil {
357 t.Fatal(err)
358 }
359
360 tmpsubfile := filepath.Join(tmpdir, "anotherfile")
361 if err := ioutil.WriteFile(tmpsubfile, []byte("aaa"), 0644); err != nil {
362 t.Fatal(err)
363 }
364
365 // Create a tar-like archive.
366 compareFiles := []fakeFile{
367 {"./", "", 0700, tar.TypeDir, 100, 0, nil},
368 {"tmpfile", "some content", 0644, tar.TypeReg, 100, 0, nil},
369 {"testdir/", "", 0755, tar.TypeDir, 100, 0, nil},
370 {"testdir/anotherfile", "aaa", 0644, tar.TypeReg, 100, 0, nil},
371 }
372
373 for _, file := range compareFiles {
374 path := filepath.Join(dir, file.Name)
375
376 // Change the time to something known with nanosec != 0.
377 chtime := time.Unix(file.Sec, 987654321)
378 if err := os.Chtimes(path, chtime, chtime); err != nil {
379 t.Fatal(err)
380 }
381 }
382
383 // Walk the current state.
384 old, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
385 if err != nil {
386 t.Fatal(err)
387 }
388
389 ts, err := makeTarStream(compareFiles)
390 if err != nil {
391 t.Fatal(err)
392 }
393
394 str := NewTarStreamer(bytes.NewBuffer(ts), nil, append(DefaultTarKeywords, "sha1"))
395 if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
396 t.Fatal(err)
397 }
398 if err = str.Close(); err != nil {
399 t.Fatal(err)
400 }
401
402 new, err := str.Hierarchy()
403 if err != nil {
404 t.Fatal(err)
405 }
406
407 // Compare.
408 diffs, err := Compare(old, new, append(DefaultTarKeywords, "sha1"))
409 if err != nil {
410 t.Fatal(err)
411 }
412
413 // 0 objects
414 if len(diffs) != 0 {
415 actualFailure := false
416 for i, delta := range diffs {
417 // XXX: Tar generation is slightly broken, so we need to ignore some bugs.
418 if delta.Path() == "." && delta.Type() == Modified {
419 // FIXME: This is a known bug.
420 t.Logf("'.' is different in the tar -- this is a bug in the tar generation")
421
422 // The tar generation bug means that '.' is missing a bunch of keys.
423 allMissing := true
424 for _, keyDelta := range delta.Diff() {
425 if keyDelta.Type() != Missing {
426 allMissing = false
427 }
428 }
429 if !allMissing {
430 t.Errorf("'.' has changed in a way not consistent with known bugs")
431 }
432
433 continue
434 }
435
436 // XXX: Another bug.
437 keys := delta.Diff()
438 if len(keys) == 1 && keys[0].Name() == "size" && keys[0].Type() == Missing {
439 // FIXME: Also a known bug with tar generation dropping size=.
440 t.Logf("'%s' is missing a size= keyword -- a bug in tar generation", delta.Path())
441
442 continue
443 }
444
445 actualFailure = true
446 buf, err := json.MarshalIndent(delta, "", " ")
447 if err == nil {
448 t.Logf("FAILURE: diff[%d] = %s", i, string(buf))
449 } else {
450 t.Logf("FAILURE: diff[%d] = %#v", i, delta)
451 }
452 }
453
454 if actualFailure {
455 t.Errorf("expected the diff length to be 0, got %d", len(diffs))
456 }
457 }
458 }
+10
-0
creator.go less more
0 package mtree
1
2 // dhCreator is used in when building a DirectoryHierarchy
3 type dhCreator struct {
4 DH *DirectoryHierarchy
5 fs FsEval
6 curSet *Entry
7 curDir *Entry
8 curEnt *Entry
9 }
+152
-0
entry.go less more
0 package mtree
1
2 import (
3 "fmt"
4 "path/filepath"
5 "strings"
6
7 "github.com/vbatts/go-mtree/pkg/govis"
8 )
9
10 type byPos []Entry
11
12 func (bp byPos) Len() int { return len(bp) }
13 func (bp byPos) Less(i, j int) bool { return bp[i].Pos < bp[j].Pos }
14 func (bp byPos) Swap(i, j int) { bp[i], bp[j] = bp[j], bp[i] }
15
16 // Entry is each component of content in the mtree spec file
17 type Entry struct {
18 Parent *Entry // up
19 Children []*Entry // down
20 Prev, Next *Entry // left, right
21 Set *Entry // current `/set` for additional keywords
22 Pos int // order in the spec
23 Raw string // file or directory name
24 Name string // file or directory name
25 Keywords []KeyVal // TODO(vbatts) maybe a keyword typed set of values?
26 Type EntryType
27 }
28
29 // Descend searches thru an Entry's children to find the Entry associated with
30 // `filename`. Directories are stored at the end of an Entry's children so do a
31 // traverse backwards. If you descend to a "."
32 func (e Entry) Descend(filename string) *Entry {
33 if filename == "." || filename == "" {
34 return &e
35 }
36 numChildren := len(e.Children)
37 for i := range e.Children {
38 c := e.Children[numChildren-1-i]
39 if c.Name == filename {
40 return c
41 }
42 }
43 return nil
44 }
45
46 // Find is a wrapper around Descend that takes in a whole string path and tries
47 // to find that Entry
48 func (e Entry) Find(filepath string) *Entry {
49 resultnode := &e
50 for _, path := range strings.Split(filepath, "/") {
51 encoded, err := govis.Vis(path, DefaultVisFlags)
52 if err != nil {
53 return nil
54 }
55 resultnode = resultnode.Descend(encoded)
56 if resultnode == nil {
57 return nil
58 }
59 }
60 return resultnode
61 }
62
63 // Ascend gets the parent of an Entry. Serves mainly to maintain readability
64 // when traversing up and down an Entry tree
65 func (e Entry) Ascend() *Entry {
66 return e.Parent
67 }
68
69 // Path provides the full path of the file, despite RelativeType or FullType. It
70 // will be in Unvis'd form.
71 func (e Entry) Path() (string, error) {
72 decodedName, err := govis.Unvis(e.Name, DefaultVisFlags)
73 if err != nil {
74 return "", err
75 }
76 if e.Parent == nil || e.Type == FullType {
77 return filepath.Clean(decodedName), nil
78 }
79 parentName, err := e.Parent.Path()
80 if err != nil {
81 return "", err
82 }
83 return filepath.Clean(filepath.Join(parentName, decodedName)), nil
84 }
85
86 // String joins a file with its associated keywords. The file name will be the
87 // Vis'd encoded version so that it can be parsed appropriately when Check'd.
88 func (e Entry) String() string {
89 if e.Raw != "" {
90 return e.Raw
91 }
92 if e.Type == BlankType {
93 return ""
94 }
95 if e.Type == DotDotType {
96 return e.Name
97 }
98 if e.Type == SpecialType || e.Type == FullType || inKeyValSlice("type=dir", e.Keywords) {
99 return fmt.Sprintf("%s %s", e.Name, strings.Join(KeyValToString(e.Keywords), " "))
100 }
101 return fmt.Sprintf(" %s %s", e.Name, strings.Join(KeyValToString(e.Keywords), " "))
102 }
103
104 // AllKeys returns the full set of KeyVal for the given entry, based on the
105 // /set keys as well as the entry-local keys. Entry-local keys always take
106 // precedence.
107 func (e Entry) AllKeys() []KeyVal {
108 if e.Set != nil {
109 return MergeKeyValSet(e.Set.Keywords, e.Keywords)
110 }
111 return e.Keywords
112 }
113
114 // IsDir checks the type= value for this entry on whether it is a directory
115 func (e Entry) IsDir() bool {
116 for _, kv := range e.AllKeys() {
117 if kv.Keyword().Prefix() == "type" {
118 return kv.Value() == "dir"
119 }
120 }
121 return false
122 }
123
124 // EntryType are the formats of lines in an mtree spec file
125 type EntryType int
126
127 // The types of lines to be found in an mtree spec file
128 const (
129 SignatureType EntryType = iota // first line of the file, like `#mtree v2.0`
130 BlankType // blank lines are ignored
131 CommentType // Lines beginning with `#` are ignored
132 SpecialType // line that has `/` prefix issue a "special" command (currently only /set and /unset)
133 RelativeType // if the first white-space delimited word does not have a '/' in it. Options/keywords are applied.
134 DotDotType // .. - A relative path step. keywords/options are ignored
135 FullType // if the first word on the line has a `/` after the first character, it interpretted as a file pathname with options
136 )
137
138 // String returns the name of the EntryType
139 func (et EntryType) String() string {
140 return typeNames[et]
141 }
142
143 var typeNames = map[EntryType]string{
144 SignatureType: "SignatureType",
145 BlankType: "BlankType",
146 CommentType: "CommentType",
147 SpecialType: "SpecialType",
148 RelativeType: "RelativeType",
149 DotDotType: "DotDotType",
150 FullType: "FullType",
151 }
+54
-0
fseval.go less more
0 package mtree
1
2 import "os"
3
4 // FsEval is a mock-friendly method of specifying to go-mtree how to carry out
5 // filesystem operations such as opening files and the like. The semantics of
6 // all of these wrappers MUST be identical to the semantics described here.
7 type FsEval interface {
8 // Open must have the same semantics as os.Open.
9 Open(path string) (*os.File, error)
10
11 // Lstat must have the same semantics as os.Lstat.
12 Lstat(path string) (os.FileInfo, error)
13
14 // Readdir must have the same semantics as calling os.Open on the given
15 // path and then returning the result of (*os.File).Readdir(-1).
16 Readdir(path string) ([]os.FileInfo, error)
17
18 // KeywordFunc must return a wrapper around the provided function (in other
19 // words, the returned function must refer to the same keyword).
20 KeywordFunc(fn KeywordFunc) KeywordFunc
21 }
22
23 // DefaultFsEval is the default implementation of FsEval (and is the default
24 // used if a nil interface is passed to any mtree function). It does not modify
25 // or wrap any of the methods (they all just call out to os.*).
26 type DefaultFsEval struct{}
27
28 // Open must have the same semantics as os.Open.
29 func (fs DefaultFsEval) Open(path string) (*os.File, error) {
30 return os.Open(path)
31 }
32
33 // Lstat must have the same semantics as os.Lstat.
34 func (fs DefaultFsEval) Lstat(path string) (os.FileInfo, error) {
35 return os.Lstat(path)
36 }
37
38 // Readdir must have the same semantics as calling os.Open on the given
39 // path and then returning the result of (*os.File).Readdir(-1).
40 func (fs DefaultFsEval) Readdir(path string) ([]os.FileInfo, error) {
41 fh, err := os.Open(path)
42 if err != nil {
43 return nil, err
44 }
45 defer fh.Close()
46 return fh.Readdir(-1)
47 }
48
49 // KeywordFunc must return a wrapper around the provided function (in other
50 // words, the returned function must refer to the same keyword).
51 func (fs DefaultFsEval) KeywordFunc(fn KeywordFunc) KeywordFunc {
52 return fn
53 }
+166
-0
fseval_test.go less more
0 package mtree
1
2 import (
3 "encoding/json"
4 "io/ioutil"
5 "os"
6 "path/filepath"
7 "testing"
8 "time"
9 )
10
11 var mockTime = time.Unix(1337888823, 0)
12
13 // Here be some dodgy testing. In particular, we have to mess around with some
14 // of the FsEval functions. In particular, we change all of the FileInfos to a
15 // different value.
16
17 type mockFileInfo struct {
18 os.FileInfo
19 }
20
21 func (fi mockFileInfo) Mode() os.FileMode {
22 return os.FileMode(fi.FileInfo.Mode() | 0777)
23 }
24
25 func (fi mockFileInfo) ModTime() time.Time {
26 return mockTime
27 }
28
29 type MockFsEval struct {
30 open, lstat, readdir, keywordFunc int
31 }
32
33 // Open must have the same semantics as os.Open.
34 func (fs *MockFsEval) Open(path string) (*os.File, error) {
35 fs.open++
36 return os.Open(path)
37 }
38
39 // Lstat must have the same semantics as os.Lstat.
40 func (fs *MockFsEval) Lstat(path string) (os.FileInfo, error) {
41 fs.lstat++
42 fi, err := os.Lstat(path)
43 return mockFileInfo{fi}, err
44 }
45
46 // Readdir must have the same semantics as calling os.Open on the given
47 // path and then returning the result of (*os.File).Readdir(-1).
48 func (fs *MockFsEval) Readdir(path string) ([]os.FileInfo, error) {
49 fs.readdir++
50 fh, err := os.Open(path)
51 if err != nil {
52 return nil, err
53 }
54 defer fh.Close()
55
56 fis, err := fh.Readdir(-1)
57 if err != nil {
58 return nil, err
59 }
60 for idx := range fis {
61 fis[idx] = mockFileInfo{fis[idx]}
62 }
63 return fis, nil
64 }
65
66 // KeywordFunc must return a wrapper around the provided function (in other
67 // words, the returned function must refer to the same keyword).
68 func (fs *MockFsEval) KeywordFunc(fn KeywordFunc) KeywordFunc {
69 fs.keywordFunc++
70 return fn
71 }
72
73 func TestCheckFsEval(t *testing.T) {
74 dir, err := ioutil.TempDir("", "test-check-fs-eval")
75 if err != nil {
76 t.Fatal(err)
77 }
78 defer os.RemoveAll(dir) // clean up
79
80 content := []byte("If you hide your ignorance, no one will hit you and you'll never learn.")
81 tmpfn := filepath.Join(dir, "tmpfile")
82 if err := ioutil.WriteFile(tmpfn, content, 0451); err != nil {
83 t.Fatal(err)
84 }
85
86 // Walk this tempdir
87 mock := &MockFsEval{}
88 dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), mock)
89 if err != nil {
90 t.Fatal(err)
91 }
92 // Make sure that mock functions have been called.
93 if mock.open == 0 {
94 t.Errorf("mock.Open not called")
95 }
96 if mock.lstat == 0 {
97 t.Errorf("mock.Lstat not called")
98 }
99 if mock.readdir == 0 {
100 t.Errorf("mock.Readdir not called")
101 }
102 if mock.keywordFunc == 0 {
103 t.Errorf("mock.KeywordFunc not called")
104 }
105
106 // Check for sanity. This ought to pass.
107 mock = &MockFsEval{}
108 res, err := Check(dir, dh, nil, mock)
109 if err != nil {
110 t.Fatal(err)
111 }
112 if len(res) > 0 {
113 t.Errorf("%#v", res)
114 }
115 // Make sure that mock functions have been called.
116 if mock.open == 0 {
117 t.Errorf("mock.Open not called")
118 }
119 if mock.lstat == 0 {
120 t.Errorf("mock.Lstat not called")
121 }
122 if mock.readdir == 0 {
123 t.Errorf("mock.Readdir not called")
124 }
125 if mock.keywordFunc == 0 {
126 t.Errorf("mock.KeywordFunc not called")
127 }
128
129 // This should FAIL.
130 res, err = Check(dir, dh, nil, nil)
131 if err != nil {
132 t.Fatal(err)
133 }
134 if len(res) == 0 {
135 t.Errorf("expected Check to fail")
136 }
137
138 // Modify the metadata so you can get the right output.
139 if err := os.Chmod(tmpfn, 0777); err != nil {
140 t.Fatal(err)
141 }
142 if err := os.Chtimes(tmpfn, mockTime, mockTime); err != nil {
143 t.Fatal(err)
144 }
145 if err := os.Chmod(dir, 0777); err != nil {
146 t.Fatal(err)
147 }
148 if err := os.Chtimes(dir, mockTime, mockTime); err != nil {
149 t.Fatal(err)
150 }
151
152 // It should now succeed.
153 res, err = Check(dir, dh, nil, nil)
154 if err != nil {
155 t.Fatal(err)
156 }
157 if len(res) > 0 {
158 buf, err := json.MarshalIndent(res, "", " ")
159 if err != nil {
160 t.Errorf("%#v", res)
161 } else {
162 t.Errorf("%s", buf)
163 }
164 }
165 }
+15
-0
glide.lock less more
0 hash: 72477bc1ec8c50d432740cbf3940cae19387d596c02f1046c7e989458462e536
1 updated: 2017-10-20T11:31:42.730083725+02:00
2 imports:
3 - name: github.com/Sirupsen/logrus
4 version: f006c2ac4710855cf0f916dd6b77acf6b048dc6e
5 - name: golang.org/x/crypto
6 version: 1351f936d976c60a0a48d728281922cf63eafb8d
7 subpackages:
8 - ripemd160
9 - ssh/terminal
10 - name: golang.org/x/sys
11 version: 8dbc5d05d6edcc104950cc299a1ce6641235bc86
12 subpackages:
13 - unix
14 testImports: []
+14
-0
glide.yaml less more
0 package: github.com/vbatts/go-mtree
1 description: File systems verification utility and library, in likeness of mtree(8)
2 homepage: https://github.com/vbatts/go-mtree
3 license: BSD-3-Clause
4 import:
5 - package: golang.org/x/crypto
6 subpackages:
7 - ripemd160
8 - package: github.com/Sirupsen/logrus
9 version: ^1.0.0
10 - package: golang.org/x/sys
11 version: 8dbc5d05d6edcc104950cc299a1ce6641235bc86
12 subpackages:
13 - unix
+48
-0
hierarchy.go less more
0 package mtree
1
2 import (
3 "io"
4 "sort"
5 )
6
7 // DirectoryHierarchy is the mapped structure for an mtree directory hierarchy
8 // spec
9 type DirectoryHierarchy struct {
10 Entries []Entry
11 }
12
13 // WriteTo simplifies the output of the resulting hierarchy spec
14 func (dh DirectoryHierarchy) WriteTo(w io.Writer) (n int64, err error) {
15 sort.Sort(byPos(dh.Entries))
16 var sum int64
17 for _, e := range dh.Entries {
18 str := e.String()
19 i, err := io.WriteString(w, str+"\n")
20 if err != nil {
21 return sum, err
22 }
23 sum += int64(i)
24 }
25 return sum, nil
26 }
27
28 // UsedKeywords collects and returns all the keywords used in a
29 // a DirectoryHierarchy
30 func (dh DirectoryHierarchy) UsedKeywords() []Keyword {
31 usedkeywords := []Keyword{}
32 for _, e := range dh.Entries {
33 switch e.Type {
34 case FullType, RelativeType, SpecialType:
35 if e.Type != SpecialType || e.Name == "/set" {
36 kvs := e.Keywords
37 for _, kv := range kvs {
38 kw := KeyVal(kv).Keyword().Prefix()
39 if !InKeywordSlice(kw, usedkeywords) {
40 usedkeywords = append(usedkeywords, KeywordSynonym(string(kw)))
41 }
42 }
43 }
44 }
45 }
46 return usedkeywords
47 }
+51
-0
hierarchy_test.go less more
0 package mtree
1
2 import (
3 "strings"
4 "testing"
5 )
6
7 var checklist = []struct {
8 blob string
9 set []Keyword
10 }{
11 {blob: `
12 # machine: bananaboat
13 # tree: .git
14 # date: Wed Nov 16 14:54:17 2016
15
16 # .
17 /set type=file nlink=1 mode=0664 uid=1000 gid=100
18 . size=4096 type=dir mode=0755 nlink=8 time=1479326055.423853146
19 .COMMIT_EDITMSG.un~ size=1006 mode=0644 time=1479325423.450468662 sha1digest=dead0face
20 .TAG_EDITMSG.un~ size=1069 mode=0600 time=1471362316.801317529 sha256digest=dead0face
21 `, set: []Keyword{"size", "mode", "time", "sha256digest"}},
22 {blob: `
23 # user: cyphar
24 # machine: ryuk
25 # tree: xattr
26 # date: Fri Sep 29 21:00:41 2017
27 # keywords: size,type,uid,gid,mode,link,nlink,time,xattr
28
29 # .
30 /set type=file nlink=1 mode=0664 uid=1000 gid=100 xattr.user.kira=SSdsbCB0YWtlIGEgcG90YXRvIGNoaXAuLi4gYW5kIGVhdCBpdCE=
31 . size=8 type=dir mode=0755 time=1506666472.255992830
32 file size=0 mode=0644 time=1506666472.255992830 xattr.user.something=dGVzdA==
33 ..
34 `, set: []Keyword{"size", "type", "uid", "gid", "mode", "nlink", "time", "xattr"}},
35 }
36
37 func TestUsedKeywords(t *testing.T) {
38 for i, item := range checklist {
39 dh, err := ParseSpec(strings.NewReader(item.blob))
40 if err != nil {
41 t.Error(err)
42 }
43 used := dh.UsedKeywords()
44 for _, k := range item.set {
45 if !InKeywordSlice(k, used) {
46 t.Errorf("%d: expected to find %q in %q", i, k, used)
47 }
48 }
49 }
50 }
+172
-0
keywordfunc.go less more
0 package mtree
1
2 import (
3 "archive/tar"
4 "crypto/md5"
5 "crypto/sha1"
6 "crypto/sha256"
7 "crypto/sha512"
8 "fmt"
9 "hash"
10 "io"
11 "os"
12
13 "github.com/vbatts/go-mtree/pkg/govis"
14 "golang.org/x/crypto/ripemd160"
15 )
16
17 // KeywordFunc is the type of a function called on each file to be included in
18 // a DirectoryHierarchy, that will produce the string output of the keyword to
19 // be included for the file entry. Otherwise, empty string.
20 // io.Reader `r` is to the file stream for the file payload. While this
21 // function takes an io.Reader, the caller needs to reset it to the beginning
22 // for each new KeywordFunc
23 type KeywordFunc func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error)
24
25 var (
26 // KeywordFuncs is the map of all keywords (and the functions to produce them)
27 KeywordFuncs = map[Keyword]KeywordFunc{
28 "size": sizeKeywordFunc, // The size, in bytes, of the file
29 "type": typeKeywordFunc, // The type of the file
30 "time": timeKeywordFunc, // The last modification time of the file
31 "link": linkKeywordFunc, // The target of the symbolic link when type=link
32 "uid": uidKeywordFunc, // The file owner as a numeric value
33 "gid": gidKeywordFunc, // The file group as a numeric value
34 "nlink": nlinkKeywordFunc, // The number of hard links the file is expected to have
35 "uname": unameKeywordFunc, // The file owner as a symbolic name
36 "gname": gnameKeywordFunc, // The file group as a symbolic name
37 "mode": modeKeywordFunc, // The current file's permissions as a numeric (octal) or symbolic value
38 "cksum": cksumKeywordFunc, // The checksum of the file using the default algorithm specified by the cksum(1) utility
39 "md5": hasherKeywordFunc("md5digest", md5.New), // The MD5 message digest of the file
40 "md5digest": hasherKeywordFunc("md5digest", md5.New), // A synonym for `md5`
41 "rmd160": hasherKeywordFunc("ripemd160digest", ripemd160.New), // The RIPEMD160 message digest of the file
42 "rmd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160`
43 "ripemd160digest": hasherKeywordFunc("ripemd160digest", ripemd160.New), // A synonym for `rmd160`
44 "sha1": hasherKeywordFunc("sha1digest", sha1.New), // The SHA1 message digest of the file
45 "sha1digest": hasherKeywordFunc("sha1digest", sha1.New), // A synonym for `sha1`
46 "sha256": hasherKeywordFunc("sha256digest", sha256.New), // The SHA256 message digest of the file
47 "sha256digest": hasherKeywordFunc("sha256digest", sha256.New), // A synonym for `sha256`
48 "sha384": hasherKeywordFunc("sha384digest", sha512.New384), // The SHA384 message digest of the file
49 "sha384digest": hasherKeywordFunc("sha384digest", sha512.New384), // A synonym for `sha384`
50 "sha512": hasherKeywordFunc("sha512digest", sha512.New), // The SHA512 message digest of the file
51 "sha512digest": hasherKeywordFunc("sha512digest", sha512.New), // A synonym for `sha512`
52 "sha512256": hasherKeywordFunc("sha512digest", sha512.New512_256), // The SHA512/256 message digest of the file
53 "sha512256digest": hasherKeywordFunc("sha512digest", sha512.New512_256), // A synonym for `sha512256`
54
55 "flags": flagsKeywordFunc, // NOTE: this is a noop, but here to support the presence of the "flags" keyword.
56
57 // This is not an upstreamed keyword, but used to vary from "time", as tar
58 // archives do not store nanosecond precision. So comparing on "time" will
59 // be only seconds level accurate.
60 "tar_time": tartimeKeywordFunc, // The last modification time of the file, from a tar archive mtime
61
62 // This is not an upstreamed keyword, but a needed attribute for file validation.
63 // The pattern for this keyword key is prefixed by "xattr." followed by the extended attribute "namespace.key".
64 // The keyword value is the SHA1 digest of the extended attribute's value.
65 // In this way, the order of the keys does not matter, and the contents of the value is not revealed.
66 "xattr": xattrKeywordFunc,
67 "xattrs": xattrKeywordFunc,
68 }
69 )
70 var (
71 modeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
72 permissions := info.Mode().Perm()
73 if os.ModeSetuid&info.Mode() > 0 {
74 permissions |= (1 << 11)
75 }
76 if os.ModeSetgid&info.Mode() > 0 {
77 permissions |= (1 << 10)
78 }
79 if os.ModeSticky&info.Mode() > 0 {
80 permissions |= (1 << 9)
81 }
82 return []KeyVal{KeyVal(fmt.Sprintf("mode=%#o", permissions))}, nil
83 }
84 sizeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
85 if sys, ok := info.Sys().(*tar.Header); ok {
86 if sys.Typeflag == tar.TypeSymlink {
87 return []KeyVal{KeyVal(fmt.Sprintf("size=%d", len(sys.Linkname)))}, nil
88 }
89 }
90 return []KeyVal{KeyVal(fmt.Sprintf("size=%d", info.Size()))}, nil
91 }
92 cksumKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
93 if !info.Mode().IsRegular() {
94 return nil, nil
95 }
96 sum, _, err := cksum(r)
97 if err != nil {
98 return nil, err
99 }
100 return []KeyVal{KeyVal(fmt.Sprintf("cksum=%d", sum))}, nil
101 }
102 hasherKeywordFunc = func(name string, newHash func() hash.Hash) KeywordFunc {
103 return func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
104 if !info.Mode().IsRegular() {
105 return nil, nil
106 }
107 h := newHash()
108 if _, err := io.Copy(h, r); err != nil {
109 return nil, err
110 }
111 return []KeyVal{KeyVal(fmt.Sprintf("%s=%x", KeywordSynonym(name), h.Sum(nil)))}, nil
112 }
113 }
114 tartimeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
115 return []KeyVal{KeyVal(fmt.Sprintf("tar_time=%d.%9.9d", info.ModTime().Unix(), 0))}, nil
116 }
117 timeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
118 tSec := info.ModTime().Unix()
119 tNano := info.ModTime().Nanosecond()
120 return []KeyVal{KeyVal(fmt.Sprintf("time=%d.%9.9d", tSec, tNano))}, nil
121 }
122 linkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
123 if sys, ok := info.Sys().(*tar.Header); ok {
124 if sys.Linkname != "" {
125 linkname, err := govis.Vis(sys.Linkname, DefaultVisFlags)
126 if err != nil {
127 return nil, nil
128 }
129 return []KeyVal{KeyVal(fmt.Sprintf("link=%s", linkname))}, nil
130 }
131 return nil, nil
132 }
133
134 if info.Mode()&os.ModeSymlink != 0 {
135 str, err := os.Readlink(path)
136 if err != nil {
137 return nil, nil
138 }
139 linkname, err := govis.Vis(str, DefaultVisFlags)
140 if err != nil {
141 return nil, nil
142 }
143 return []KeyVal{KeyVal(fmt.Sprintf("link=%s", linkname))}, nil
144 }
145 return nil, nil
146 }
147 typeKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
148 if info.Mode().IsDir() {
149 return []KeyVal{"type=dir"}, nil
150 }
151 if info.Mode().IsRegular() {
152 return []KeyVal{"type=file"}, nil
153 }
154 if info.Mode()&os.ModeSocket != 0 {
155 return []KeyVal{"type=socket"}, nil
156 }
157 if info.Mode()&os.ModeSymlink != 0 {
158 return []KeyVal{"type=link"}, nil
159 }
160 if info.Mode()&os.ModeNamedPipe != 0 {
161 return []KeyVal{"type=fifo"}, nil
162 }
163 if info.Mode()&os.ModeDevice != 0 {
164 if info.Mode()&os.ModeCharDevice != 0 {
165 return []KeyVal{"type=char"}, nil
166 }
167 return []KeyVal{"type=block"}, nil
168 }
169 return nil, nil
170 }
171 )
+69
-0
keywordfuncs_bsd.go less more
0 // +build darwin freebsd netbsd openbsd
1
2 package mtree
3
4 import (
5 "archive/tar"
6 "fmt"
7 "io"
8 "os"
9 "os/user"
10 "syscall"
11 )
12
13 var (
14 flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
15 // ideally this will pull in from here https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
16 return nil, nil
17 }
18
19 unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
20 if hdr, ok := info.Sys().(*tar.Header); ok {
21 return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", hdr.Uname))}, nil
22 }
23
24 stat := info.Sys().(*syscall.Stat_t)
25 u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid))
26 if err != nil {
27 return nil, err
28 }
29 return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", u.Username))}, nil
30 }
31 gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
32 if hdr, ok := info.Sys().(*tar.Header); ok {
33 return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", hdr.Gname))}, nil
34 }
35
36 stat := info.Sys().(*syscall.Stat_t)
37 g, err := lookupGroupID(fmt.Sprintf("%d", stat.Gid))
38 if err != nil {
39 return nil, err
40 }
41 return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", g.Name))}, nil
42 }
43 uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
44 if hdr, ok := info.Sys().(*tar.Header); ok {
45 return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", hdr.Uid))}, nil
46 }
47 stat := info.Sys().(*syscall.Stat_t)
48 return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", stat.Uid))}, nil
49 }
50 gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
51 if hdr, ok := info.Sys().(*tar.Header); ok {
52 return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", hdr.Gid))}, nil
53 }
54 if stat, ok := info.Sys().(*syscall.Stat_t); ok {
55 return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", stat.Gid))}, nil
56 }
57 return nil, nil
58 }
59 nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
60 if stat, ok := info.Sys().(*syscall.Stat_t); ok {
61 return []KeyVal{KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink))}, nil
62 }
63 return nil, nil
64 }
65 xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
66 return nil, nil
67 }
68 )
+107
-0
keywordfuncs_linux.go less more
0 // +build linux
1
2 package mtree
3
4 import (
5 "archive/tar"
6 "encoding/base64"
7 "fmt"
8 "io"
9 "os"
10 "os/user"
11 "syscall"
12
13 "github.com/vbatts/go-mtree/pkg/govis"
14 "github.com/vbatts/go-mtree/xattr"
15 )
16
17 var (
18 // this is bsd specific https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
19 flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
20 return nil, nil
21 }
22
23 unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
24 if hdr, ok := info.Sys().(*tar.Header); ok {
25 return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", hdr.Uname))}, nil
26 }
27
28 stat := info.Sys().(*syscall.Stat_t)
29 u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid))
30 if err != nil {
31 return nil, nil
32 }
33 return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", u.Username))}, nil
34 }
35 gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
36 if hdr, ok := info.Sys().(*tar.Header); ok {
37 return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", hdr.Gname))}, nil
38 }
39
40 stat := info.Sys().(*syscall.Stat_t)
41 g, err := lookupGroupID(fmt.Sprintf("%d", stat.Gid))
42 if err != nil {
43 return nil, nil
44 }
45 return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", g.Name))}, nil
46 }
47 uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
48 if hdr, ok := info.Sys().(*tar.Header); ok {
49 return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", hdr.Uid))}, nil
50 }
51 stat := info.Sys().(*syscall.Stat_t)
52 return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", stat.Uid))}, nil
53 }
54 gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
55 if hdr, ok := info.Sys().(*tar.Header); ok {
56 return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", hdr.Gid))}, nil
57 }
58 if stat, ok := info.Sys().(*syscall.Stat_t); ok {
59 return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", stat.Gid))}, nil
60 }
61 return nil, nil
62 }
63 nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
64 if stat, ok := info.Sys().(*syscall.Stat_t); ok {
65 return []KeyVal{KeyVal(fmt.Sprintf("nlink=%d", stat.Nlink))}, nil
66 }
67 return nil, nil
68 }
69 xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
70 if hdr, ok := info.Sys().(*tar.Header); ok {
71 if len(hdr.Xattrs) == 0 {
72 return nil, nil
73 }
74 klist := []KeyVal{}
75 for k, v := range hdr.Xattrs {
76 encKey, err := govis.Vis(k, DefaultVisFlags)
77 if err != nil {
78 return nil, nil
79 }
80 klist = append(klist, KeyVal(fmt.Sprintf("xattr.%s=%s", encKey, base64.StdEncoding.EncodeToString([]byte(v)))))
81 }
82 return klist, nil
83 }
84 if !info.Mode().IsRegular() && !info.Mode().IsDir() {
85 return nil, nil
86 }
87
88 xlist, err := xattr.List(path)
89 if err != nil {
90 return nil, nil
91 }
92 klist := make([]KeyVal, len(xlist))
93 for i := range xlist {
94 data, err := xattr.Get(path, xlist[i])
95 if err != nil {
96 return nil, nil
97 }
98 encKey, err := govis.Vis(xlist[i], DefaultVisFlags)
99 if err != nil {
100 return nil, nil
101 }
102 klist[i] = KeyVal(fmt.Sprintf("xattr.%s=%s", encKey, base64.StdEncoding.EncodeToString(data)))
103 }
104 return klist, nil
105 }
106 )
+47
-0
keywordfuncs_unsupported.go less more
0 // +build !linux,!darwin,!freebsd,!netbsd,!openbsd
1
2 package mtree
3
4 import (
5 "archive/tar"
6 "fmt"
7 "io"
8 "os"
9 )
10
11 var (
12 // this is bsd specific https://www.freebsd.org/cgi/man.cgi?query=chflags&sektion=2
13 flagsKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
14 return nil, nil
15 }
16 unameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
17 if hdr, ok := info.Sys().(*tar.Header); ok {
18 return []KeyVal{KeyVal(fmt.Sprintf("uname=%s", hdr.Uname))}, nil
19 }
20 return nil, nil
21 }
22 gnameKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
23 if hdr, ok := info.Sys().(*tar.Header); ok {
24 return []KeyVal{KeyVal(fmt.Sprintf("gname=%s", hdr.Gname))}, nil
25 }
26 return nil, nil
27 }
28 uidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
29 if hdr, ok := info.Sys().(*tar.Header); ok {
30 return []KeyVal{KeyVal(fmt.Sprintf("uid=%d", hdr.Uid))}, nil
31 }
32 return nil, nil
33 }
34 gidKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
35 if hdr, ok := info.Sys().(*tar.Header); ok {
36 return []KeyVal{KeyVal(fmt.Sprintf("gid=%d", hdr.Gid))}, nil
37 }
38 return nil, nil
39 }
40 nlinkKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
41 return nil, nil
42 }
43 xattrKeywordFunc = func(path string, info os.FileInfo, r io.Reader) ([]KeyVal, error) {
44 return nil, nil
45 }
46 )
+327
-0
keywords.go less more
0 package mtree
1
2 import (
3 "fmt"
4 "strings"
5
6 "github.com/vbatts/go-mtree/pkg/govis"
7 )
8
9 // DefaultVisFlags is the set of Vis flags used when encoding filenames and
10 // other similar entries.
11 const DefaultVisFlags govis.VisFlag = govis.VisWhite | govis.VisOctal | govis.VisGlob
12
13 // Keyword is the string name of a keyword, with some convenience functions for
14 // determining whether it is a default or bsd standard keyword.
15 // It first portion before the "="
16 type Keyword string
17
18 // Prefix is the portion of the keyword before a first "." (if present).
19 //
20 // Primarly for the xattr use-case, where the keyword `xattr.security.selinux` would have a Suffix of `security.selinux`.
21 func (k Keyword) Prefix() Keyword {
22 if strings.Contains(string(k), ".") {
23 return Keyword(strings.SplitN(string(k), ".", 2)[0])
24 }
25 return k
26 }
27
28 // Suffix is the portion of the keyword after a first ".".
29 // This is an option feature.
30 //
31 // Primarly for the xattr use-case, where the keyword `xattr.security.selinux` would have a Suffix of `security.selinux`.
32 func (k Keyword) Suffix() string {
33 if strings.Contains(string(k), ".") {
34 return strings.SplitN(string(k), ".", 2)[1]
35 }
36 return string(k)
37 }
38
39 // Default returns whether this keyword is in the default set of keywords
40 func (k Keyword) Default() bool {
41 return InKeywordSlice(k, DefaultKeywords)
42 }
43
44 // Bsd returns whether this keyword is in the upstream FreeBSD mtree(8)
45 func (k Keyword) Bsd() bool {
46 return InKeywordSlice(k, BsdKeywords)
47 }
48
49 // Synonym returns the canonical name for this keyword. This is provides the
50 // same functionality as KeywordSynonym()
51 func (k Keyword) Synonym() Keyword {
52 return KeywordSynonym(string(k))
53 }
54
55 // InKeywordSlice checks for the presence of `a` in `list`
56 func InKeywordSlice(a Keyword, list []Keyword) bool {
57 for _, b := range list {
58 if b == a {
59 return true
60 }
61 }
62 return false
63 }
64 func inKeyValSlice(a KeyVal, list []KeyVal) bool {
65 for _, b := range list {
66 if b == a {
67 return true
68 }
69 }
70 return false
71 }
72
73 // ToKeywords makes a list of Keyword from a list of string
74 func ToKeywords(list []string) []Keyword {
75 ret := make([]Keyword, len(list))
76 for i := range list {
77 ret[i] = Keyword(list[i])
78 }
79 return ret
80 }
81
82 // FromKeywords makes a list of string from a list of Keyword
83 func FromKeywords(list []Keyword) []string {
84 ret := make([]string, len(list))
85 for i := range list {
86 ret[i] = string(list[i])
87 }
88 return ret
89 }
90
91 // KeyValToString constructs a list of string from the list of KeyVal
92 func KeyValToString(list []KeyVal) []string {
93 ret := make([]string, len(list))
94 for i := range list {
95 ret[i] = string(list[i])
96 }
97 return ret
98 }
99
100 // StringToKeyVals constructs a list of KeyVal from the list of strings, like "keyword=value"
101 func StringToKeyVals(list []string) []KeyVal {
102 ret := make([]KeyVal, len(list))
103 for i := range list {
104 ret[i] = KeyVal(list[i])
105 }
106 return ret
107 }
108
109 // KeyVal is a "keyword=value"
110 type KeyVal string
111
112 // Keyword is the mapping to the available keywords
113 func (kv KeyVal) Keyword() Keyword {
114 if !strings.Contains(string(kv), "=") {
115 return Keyword("")
116 }
117 return Keyword(strings.SplitN(strings.TrimSpace(string(kv)), "=", 2)[0])
118 }
119
120 // Value is the data/value portion of "keyword=value"
121 func (kv KeyVal) Value() string {
122 if !strings.Contains(string(kv), "=") {
123 return ""
124 }
125 return strings.SplitN(strings.TrimSpace(string(kv)), "=", 2)[1]
126 }
127
128 // NewValue returns a new KeyVal with the newval
129 func (kv KeyVal) NewValue(newval string) KeyVal {
130 return KeyVal(fmt.Sprintf("%s=%s", kv.Keyword(), newval))
131 }
132
133 // Equal returns whether two KeyVal are equivalent. This takes
134 // care of certain odd cases such as tar_mtime, and should be used over
135 // using == comparisons directly unless you really know what you're
136 // doing.
137 func (kv KeyVal) Equal(b KeyVal) bool {
138 // TODO: Implement handling of tar_mtime.
139 return kv.Keyword() == b.Keyword() && kv.Value() == b.Value()
140 }
141
142 func keywordPrefixes(kvset []Keyword) []Keyword {
143 kvs := []Keyword{}
144 for _, kv := range kvset {
145 kvs = append(kvs, kv.Prefix())
146 }
147 return kvs
148 }
149
150 // keyvalSelector takes an array of KeyVal ("keyword=value") and filters out
151 // that only the set of keywords
152 func keyvalSelector(keyval []KeyVal, keyset []Keyword) []KeyVal {
153 retList := []KeyVal{}
154 for _, kv := range keyval {
155 if InKeywordSlice(kv.Keyword().Prefix(), keywordPrefixes(keyset)) {
156 retList = append(retList, kv)
157 }
158 }
159 return retList
160 }
161
162 func keyValDifference(this, that []KeyVal) []KeyVal {
163 if len(this) == 0 {
164 return that
165 }
166 diff := []KeyVal{}
167 for _, kv := range this {
168 if !inKeyValSlice(kv, that) {
169 diff = append(diff, kv)
170 }
171 }
172 return diff
173 }
174 func keyValCopy(set []KeyVal) []KeyVal {
175 ret := make([]KeyVal, len(set))
176 for i := range set {
177 ret[i] = set[i]
178 }
179 return ret
180 }
181
182 // Has the "keyword" present in the list of KeyVal, and returns the
183 // corresponding KeyVal, else an empty string.
184 func Has(keyvals []KeyVal, keyword string) []KeyVal {
185 return HasKeyword(keyvals, Keyword(keyword))
186 }
187
188 // HasKeyword the "keyword" present in the list of KeyVal, and returns the
189 // corresponding KeyVal, else an empty string.
190 // This match is done on the Prefix of the keyword only.
191 func HasKeyword(keyvals []KeyVal, keyword Keyword) []KeyVal {
192 kvs := []KeyVal{}
193 for i := range keyvals {
194 if keyvals[i].Keyword().Prefix() == keyword.Prefix() {
195 kvs = append(kvs, keyvals[i])
196 }
197 }
198 return kvs
199 }
200
201 // MergeSet takes the current setKeyVals, and then applies the entryKeyVals
202 // such that the entry's values win. The union is returned.
203 func MergeSet(setKeyVals, entryKeyVals []string) []KeyVal {
204 retList := StringToKeyVals(setKeyVals)
205 eKVs := StringToKeyVals(entryKeyVals)
206 return MergeKeyValSet(retList, eKVs)
207 }
208
209 // MergeKeyValSet does a merge of the two sets of KeyVal, and the KeyVal of
210 // entryKeyVals win when there is a duplicate Keyword.
211 func MergeKeyValSet(setKeyVals, entryKeyVals []KeyVal) []KeyVal {
212 retList := keyValCopy(setKeyVals)
213 seenKeywords := []Keyword{}
214 for i := range retList {
215 word := retList[i].Keyword()
216 for _, kv := range HasKeyword(entryKeyVals, word) {
217 // match on the keyword prefix and suffix here
218 if kv.Keyword() == word {
219 retList[i] = kv
220 }
221 }
222 seenKeywords = append(seenKeywords, word)
223 }
224 for i := range entryKeyVals {
225 if !InKeywordSlice(entryKeyVals[i].Keyword(), seenKeywords) {
226 retList = append(retList, entryKeyVals[i])
227 }
228 }
229 return retList
230 }
231
232 var (
233 // DefaultKeywords has the several default keyword producers (uid, gid,
234 // mode, nlink, type, size, mtime)
235 DefaultKeywords = []Keyword{
236 "size",
237 "type",
238 "uid",
239 "gid",
240 "mode",
241 "link",
242 "nlink",
243 "time",
244 }
245
246 // DefaultTarKeywords has keywords that should be used when creating a manifest from
247 // an archive. Currently, evaluating the # of hardlinks has not been implemented yet
248 DefaultTarKeywords = []Keyword{
249 "size",
250 "type",
251 "uid",
252 "gid",
253 "mode",
254 "link",
255 "tar_time",
256 }
257
258 // BsdKeywords is the set of keywords that is only in the upstream FreeBSD mtree
259 BsdKeywords = []Keyword{
260 "cksum",
261 "flags", // this one is really mostly BSD specific ...
262 "ignore",
263 "gid",
264 "gname",
265 "link",
266 "md5",
267 "md5digest",
268 "mode",
269 "nlink",
270 "nochange",
271 "optional",
272 "ripemd160digest",
273 "rmd160",
274 "rmd160digest",
275 "sha1",
276 "sha1digest",
277 "sha256",
278 "sha256digest",
279 "sha384",
280 "sha384digest",
281 "sha512",
282 "sha512digest",
283 "size",
284 "tags",
285 "time",
286 "type",
287 "uid",
288 "uname",
289 }
290
291 // SetKeywords is the default set of keywords calculated for a `/set` SpecialType
292 SetKeywords = []Keyword{
293 "uid",
294 "gid",
295 }
296 )
297
298 // KeywordSynonym returns the canonical name for keywords that have synonyms,
299 // and just returns the name provided if there is no synonym. In this way it
300 // ought to be safe to wrap any keyword name.
301 func KeywordSynonym(name string) Keyword {
302 var retname string
303 switch name {
304 case "md5":
305 retname = "md5digest"
306 case "rmd160":
307 retname = "ripemd160digest"
308 case "rmd160digest":
309 retname = "ripemd160digest"
310 case "sha1":
311 retname = "sha1digest"
312 case "sha256":
313 retname = "sha256digest"
314 case "sha384":
315 retname = "sha384digest"
316 case "sha512":
317 retname = "sha512digest"
318 case "sha512256":
319 retname = "sha512256digest"
320 case "xattrs":
321 retname = "xattr"
322 default:
323 retname = name
324 }
325 return Keyword(retname)
326 }
+84
-0
keywords_linux_test.go less more
0 // +build linux
1
2 package mtree
3
4 import (
5 "fmt"
6 "io/ioutil"
7 "os"
8 "path/filepath"
9 "testing"
10
11 "github.com/vbatts/go-mtree/xattr"
12 )
13
14 func TestXattr(t *testing.T) {
15 testDir, present := os.LookupEnv("MTREE_TESTDIR")
16 if present == false {
17 // a bit dirty to create/destory a directory in cwd,
18 // but often /tmp is mounted tmpfs and doesn't support
19 // xattrs
20 testDir = "."
21 }
22 dir, err := ioutil.TempDir(testDir, "test.xattrs.")
23 if err != nil {
24 t.Fatal(err)
25 }
26 defer os.RemoveAll(dir)
27 fh, err := os.Create(filepath.Join(dir, "file"))
28 if err != nil {
29 t.Fatal(err)
30 }
31 fh.WriteString("howdy")
32 fh.Sync()
33 if _, err := fh.Seek(0, 0); err != nil {
34 t.Fatal(err)
35 }
36
37 if err := os.Symlink("./no/such/path", filepath.Join(dir, "symlink")); err != nil {
38 t.Fatal(err)
39 }
40
41 if err := xattr.Set(dir, "user.test", []byte("directory")); err != nil {
42 t.Skip(fmt.Sprintf("skipping: %q does not support xattrs", dir))
43 }
44 if err := xattr.Set(filepath.Join(dir, "file"), "user.test", []byte("regular file")); err != nil {
45 t.Fatal(err)
46 }
47
48 dirstat, err := os.Lstat(dir)
49 if err != nil {
50 t.Fatal(err)
51 }
52 // Check the directory
53 kvs, err := xattrKeywordFunc(dir, dirstat, nil)
54 if err != nil {
55 t.Error(err)
56 }
57 if len(kvs) == 0 {
58 t.Errorf("expected a keyval; got none")
59 }
60
61 filestat, err := fh.Stat()
62 if err != nil {
63 t.Fatal(err)
64 }
65 // Check the regular file
66 kvs, err = xattrKeywordFunc(filepath.Join(dir, "file"), filestat, fh)
67 if err != nil {
68 t.Error(err)
69 }
70 if len(kvs) == 0 {
71 t.Errorf("expected a keyval; got none")
72 }
73
74 linkstat, err := os.Lstat(filepath.Join(dir, "symlink"))
75 if err != nil {
76 t.Fatal(err)
77 }
78 // Check a broken symlink
79 _, err = xattrKeywordFunc(filepath.Join(dir, "symlink"), linkstat, nil)
80 if err != nil {
81 t.Error(err)
82 }
83 }
+173
-0
keywords_test.go less more
0 package mtree
1
2 import (
3 "fmt"
4 "os"
5 "testing"
6 "time"
7 )
8
9 func TestKeyValRoundtrip(t *testing.T) {
10 kv := KeyVal("xattr.security.selinux=dW5jb25maW5lZF91Om9iamVjdF9yOnVzZXJfaG9tZV90OnMwAA==")
11 expected := "xattr.security.selinux"
12 got := string(kv.Keyword())
13 if got != expected {
14 t.Errorf("expected %q; got %q", expected, got)
15 }
16
17 expected = "xattr"
18 got = string(kv.Keyword().Prefix())
19 if got != expected {
20 t.Errorf("expected %q; got %q", expected, got)
21 }
22
23 expected = "security.selinux"
24 got = kv.Keyword().Suffix()
25 if got != expected {
26 t.Errorf("expected %q; got %q", expected, got)
27 }
28
29 expected = "dW5jb25maW5lZF91Om9iamVjdF9yOnVzZXJfaG9tZV90OnMwAA=="
30 got = kv.Value()
31 if got != expected {
32 t.Errorf("expected %q; got %q", expected, got)
33 }
34
35 expected = "xattr.security.selinux=farts"
36 got = string(kv.NewValue("farts"))
37 if got != expected {
38 t.Errorf("expected %q; got %q", expected, got)
39 }
40
41 expected = "xattr.security.selinux=farts"
42 kv1 := KeyVal(got)
43 kv2 := kv.NewValue("farts")
44 if !kv2.Equal(kv1) {
45 t.Errorf("expected equality of %q and %q", kv1, kv2)
46 }
47
48 }
49
50 type fakeFileInfo struct {
51 mtime time.Time
52 }
53
54 func (ffi fakeFileInfo) Name() string {
55 // noop
56 return ""
57 }
58
59 func (ffi fakeFileInfo) Size() int64 {
60 // noop
61 return -1
62 }
63
64 func (ffi fakeFileInfo) Mode() os.FileMode {
65 // noop
66 return 0
67 }
68
69 func (ffi fakeFileInfo) ModTime() time.Time {
70 return ffi.mtime
71 }
72
73 func (ffi fakeFileInfo) IsDir() bool {
74 return ffi.Mode().IsDir()
75 }
76
77 func (ffi fakeFileInfo) Sys() interface{} {
78 // noop
79 return nil
80 }
81
82 func TestKeywordsTimeNano(t *testing.T) {
83 // We have to make sure that timeKeywordFunc always returns the correct
84 // formatting with regards to the nanotime.
85
86 for _, test := range []struct {
87 sec, nsec int64
88 }{
89 {1234, 123456789},
90 {5555, 987654321},
91 {1337, 100000000},
92 {8888, 999999999},
93 {144123582122, 1},
94 {857125628319, 0},
95 } {
96 mtime := time.Unix(test.sec, test.nsec)
97 expected := KeyVal(fmt.Sprintf("time=%d.%9.9d", test.sec, test.nsec))
98 got, err := timeKeywordFunc("", fakeFileInfo{
99 mtime: mtime,
100 }, nil)
101 if err != nil {
102 t.Errorf("unexpected error while parsing '%q': %q", mtime, err)
103 }
104 if len(got) != 1 {
105 t.Errorf("expected 1 KeyVal, but got %d", len(got))
106 }
107 if expected != got[0] {
108 t.Errorf("keyword didn't match, expected '%s' got '%s'", expected, got[0])
109 }
110 }
111 }
112
113 func TestKeywordsTimeTar(t *testing.T) {
114 // tartimeKeywordFunc always has nsec = 0.
115
116 for _, test := range []struct {
117 sec, nsec int64
118 }{
119 {1234, 123456789},
120 {5555, 987654321},
121 {1337, 100000000},
122 {8888, 999999999},
123 {144123582122, 1},
124 {857125628319, 0},
125 } {
126 mtime := time.Unix(test.sec, test.nsec)
127 expected := KeyVal(fmt.Sprintf("tar_time=%d.%9.9d", test.sec, 0))
128 got, err := tartimeKeywordFunc("", fakeFileInfo{
129 mtime: mtime,
130 }, nil)
131 if err != nil {
132 t.Errorf("unexpected error while parsing '%q': %q", mtime, err)
133 }
134 if len(got) != 1 {
135 t.Errorf("expected 1 KeyVal, but got %d", len(got))
136 }
137 if expected != got[0] {
138 t.Errorf("keyword didn't match, expected '%s' got '%s'", expected, got[0])
139 }
140 }
141 }
142
143 func TestKeywordSynonym(t *testing.T) {
144 checklist := []struct {
145 give string
146 expect Keyword
147 }{
148 {give: "time", expect: "time"},
149 {give: "md5", expect: "md5digest"},
150 {give: "md5digest", expect: "md5digest"},
151 {give: "rmd160", expect: "ripemd160digest"},
152 {give: "rmd160digest", expect: "ripemd160digest"},
153 {give: "ripemd160digest", expect: "ripemd160digest"},
154 {give: "sha1", expect: "sha1digest"},
155 {give: "sha1digest", expect: "sha1digest"},
156 {give: "sha256", expect: "sha256digest"},
157 {give: "sha256digest", expect: "sha256digest"},
158 {give: "sha384", expect: "sha384digest"},
159 {give: "sha384digest", expect: "sha384digest"},
160 {give: "sha512", expect: "sha512digest"},
161 {give: "sha512digest", expect: "sha512digest"},
162 {give: "xattr", expect: "xattr"},
163 {give: "xattrs", expect: "xattr"},
164 }
165
166 for i, check := range checklist {
167 got := KeywordSynonym(check.give)
168 if got != check.expect {
169 t.Errorf("%d: expected %q; got %q", i, check.expect, got)
170 }
171 }
172 }
+22
-0
lchtimes_unix.go less more
0 // +build darwin dragonfly freebsd openbsd linux netbsd solaris
1
2 package mtree
3
4 import (
5 "os"
6 "time"
7
8 "golang.org/x/sys/unix"
9 )
10
11 func lchtimes(name string, atime time.Time, mtime time.Time) error {
12 utimes := []unix.Timespec{
13 unix.NsecToTimespec(atime.UnixNano()),
14 unix.NsecToTimespec(mtime.UnixNano()),
15 }
16 if e := unix.UtimesNanoAt(unix.AT_FDCWD, name, utimes, unix.AT_SYMLINK_NOFOLLOW); e != nil {
17 return &os.PathError{Op: "chtimes", Path: name, Err: e}
18 }
19 return nil
20
21 }
+11
-0
lchtimes_unsupported.go less more
0 // +build windows
1
2 package mtree
3
4 import (
5 "time"
6 )
7
8 func lchtimes(name string, atime time.Time, mtime time.Time) error {
9 return nil
10 }
+9
-0
lookup_new.go less more
0 // +build go1.7
1
2 package mtree
3
4 import (
5 "os/user"
6 )
7
8 var lookupGroupID = user.LookupGroupId
+102
-0
lookup_old.go less more
0 // +build !go1.7
1
2 package mtree
3
4 import (
5 "bufio"
6 "bytes"
7 "io"
8 "os"
9 "strconv"
10 "strings"
11 )
12
13 const groupFile = "/etc/group"
14
15 var colon = []byte{':'}
16
17 // Group represents a grouping of users.
18 //
19 // On POSIX systems Gid contains a decimal number representing the group ID.
20 type Group struct {
21 Gid string // group ID
22 Name string // group name
23 }
24
25 func lookupGroupID(id string) (*Group, error) {
26 f, err := os.Open(groupFile)
27 if err != nil {
28 return nil, err
29 }
30 defer f.Close()
31 return findGroupID(id, f)
32 }
33
34 func findGroupID(id string, r io.Reader) (*Group, error) {
35 if v, err := readColonFile(r, matchGroupIndexValue(id, 2)); err != nil {
36 return nil, err
37 } else if v != nil {
38 return v.(*Group), nil
39 }
40 return nil, UnknownGroupIDError(id)
41 }
42
43 // lineFunc returns a value, an error, or (nil, nil) to skip the row.
44 type lineFunc func(line []byte) (v interface{}, err error)
45
46 // readColonFile parses r as an /etc/group or /etc/passwd style file, running
47 // fn for each row. readColonFile returns a value, an error, or (nil, nil) if
48 // the end of the file is reached without a match.
49 func readColonFile(r io.Reader, fn lineFunc) (v interface{}, err error) {
50 bs := bufio.NewScanner(r)
51 for bs.Scan() {
52 line := bs.Bytes()
53 // There's no spec for /etc/passwd or /etc/group, but we try to follow
54 // the same rules as the glibc parser, which allows comments and blank
55 // space at the beginning of a line.
56 line = bytes.TrimSpace(line)
57 if len(line) == 0 || line[0] == '#' {
58 continue
59 }
60 v, err = fn(line)
61 if v != nil || err != nil {
62 return
63 }
64 }
65 return nil, bs.Err()
66 }
67
68 func matchGroupIndexValue(value string, idx int) lineFunc {
69 var leadColon string
70 if idx > 0 {
71 leadColon = ":"
72 }
73 substr := []byte(leadColon + value + ":")
74 return func(line []byte) (v interface{}, err error) {
75 if !bytes.Contains(line, substr) || bytes.Count(line, colon) < 3 {
76 return
77 }
78 // wheel:*:0:root
79 parts := strings.SplitN(string(line), ":", 4)
80 if len(parts) < 4 || parts[0] == "" || parts[idx] != value ||
81 // If the file contains +foo and you search for "foo", glibc
82 // returns an "invalid argument" error. Similarly, if you search
83 // for a gid for a row where the group name starts with "+" or "-",
84 // glibc fails to find the record.
85 parts[0][0] == '+' || parts[0][0] == '-' {
86 return
87 }
88 if _, err := strconv.Atoi(parts[2]); err != nil {
89 return nil, nil
90 }
91 return &Group{Name: parts[0], Gid: parts[2]}, nil
92 }
93 }
94
95 // UnknownGroupIDError is returned by LookupGroupId when
96 // a group cannot be found.
97 type UnknownGroupIDError string
98
99 func (e UnknownGroupIDError) Error() string {
100 return "group: unknown groupid " + string(e)
101 }
+68
-0
mtree_test.go less more
0 package mtree
1
2 import (
3 "io/ioutil"
4 "os"
5 "testing"
6 )
7
8 var (
9 testFiles = []string{"testdata/source.mtree"}
10 numEntries = map[EntryType]int{
11 FullType: 0,
12 RelativeType: 45,
13 CommentType: 37,
14 SpecialType: 7,
15 DotDotType: 17,
16 BlankType: 34,
17 }
18 expectedLength = int64(7887)
19 )
20
21 func TestParser(t *testing.T) {
22 for _, file := range testFiles {
23 func() {
24 fh, err := os.Open(file)
25 if err != nil {
26 t.Error(err)
27 return
28 }
29 defer fh.Close()
30
31 dh, err := ParseSpec(fh)
32 if err != nil {
33 t.Error(err)
34 }
35 gotNums := countTypes(dh)
36 for typ, num := range numEntries {
37 if gNum, ok := gotNums[typ]; ok {
38 if num != gNum {
39 t.Errorf("for type %s: expected %d, got %d", typ, num, gNum)
40 }
41 }
42 }
43
44 i, err := dh.WriteTo(ioutil.Discard)
45 if err != nil {
46 t.Error(err)
47 }
48 if i != expectedLength {
49 t.Errorf("expected to write %d, but wrote %d", expectedLength, i)
50 }
51
52 }()
53 }
54 }
55
56 func countTypes(dh *DirectoryHierarchy) map[EntryType]int {
57 nT := map[EntryType]int{}
58 for i := range dh.Entries {
59 typ := dh.Entries[i].Type
60 if _, ok := nT[typ]; !ok {
61 nT[typ] = 1
62 } else {
63 nT[typ]++
64 }
65 }
66 return nT
67 }
+105
-0
parse.go less more
0 package mtree
1
2 import (
3 "bufio"
4 "io"
5 "path/filepath"
6 "strings"
7 )
8
9 // ParseSpec reads a stream of an mtree specification, and returns the DirectoryHierarchy
10 func ParseSpec(r io.Reader) (*DirectoryHierarchy, error) {
11 s := bufio.NewScanner(r)
12 i := int(0)
13 creator := dhCreator{
14 DH: &DirectoryHierarchy{},
15 }
16 for s.Scan() {
17 str := s.Text()
18 trimmedStr := strings.TrimLeftFunc(str, func(c rune) bool {
19 return c == ' ' || c == '\t'
20 })
21 e := Entry{Pos: i}
22 switch {
23 case strings.HasPrefix(trimmedStr, "#"):
24 e.Raw = str
25 if strings.HasPrefix(trimmedStr, "#mtree") {
26 e.Type = SignatureType
27 } else {
28 e.Type = CommentType
29 // from here, the comment could be "# key: value" metadata
30 // or a relative path hint
31 }
32 case str == "":
33 e.Type = BlankType
34 // nothing else to do here
35 case strings.HasPrefix(str, "/"):
36 e.Type = SpecialType
37 // collapse any escaped newlines
38 for {
39 if strings.HasSuffix(str, `\`) {
40 str = str[:len(str)-1]
41 s.Scan()
42 str += s.Text()
43 } else {
44 break
45 }
46 }
47 // parse the options
48 f := strings.Fields(str)
49 e.Name = f[0]
50 e.Keywords = StringToKeyVals(f[1:])
51 if e.Name == "/set" {
52 creator.curSet = &e
53 } else if e.Name == "/unset" {
54 creator.curSet = nil
55 }
56 case len(strings.Fields(str)) > 0 && strings.Fields(str)[0] == "..":
57 e.Type = DotDotType
58 e.Raw = str
59 if creator.curDir != nil {
60 creator.curDir = creator.curDir.Parent
61 }
62 // nothing else to do here
63 case len(strings.Fields(str)) > 0:
64 // collapse any escaped newlines
65 for {
66 if strings.HasSuffix(str, `\`) {
67 str = str[:len(str)-1]
68 s.Scan()
69 str += s.Text()
70 } else {
71 break
72 }
73 }
74 // parse the options
75 f := strings.Fields(str)
76 e.Name = filepath.Clean(f[0])
77 if strings.Contains(e.Name, "/") {
78 e.Type = FullType
79 } else {
80 e.Type = RelativeType
81 }
82 e.Keywords = StringToKeyVals(f[1:])
83 // TODO: gather keywords if using tar stream
84 e.Parent = creator.curDir
85 for i := range e.Keywords {
86 kv := KeyVal(e.Keywords[i])
87 if kv.Keyword() == "type" {
88 if kv.Value() == "dir" {
89 creator.curDir = &e
90 } else {
91 creator.curEnt = &e
92 }
93 }
94 }
95 e.Set = creator.curSet
96 default:
97 // TODO(vbatts) log a warning?
98 continue
99 }
100 creator.DH.Entries = append(creator.DH.Entries, e)
101 i++
102 }
103 return creator.DH, s.Err()
104 }
+202
-0
pkg/govis/COPYING less more
0
1 Apache License
2 Version 2.0, January 2004
3 http://www.apache.org/licenses/
4
5 TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6
7 1. Definitions.
8
9 "License" shall mean the terms and conditions for use, reproduction,
10 and distribution as defined by Sections 1 through 9 of this document.
11
12 "Licensor" shall mean the copyright owner or entity authorized by
13 the copyright owner that is granting the License.
14
15 "Legal Entity" shall mean the union of the acting entity and all
16 other entities that control, are controlled by, or are under common
17 control with that entity. For the purposes of this definition,
18 "control" means (i) the power, direct or indirect, to cause the
19 direction or management of such entity, whether by contract or
20 otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 outstanding shares, or (iii) beneficial ownership of such entity.
22
23 "You" (or "Your") shall mean an individual or Legal Entity
24 exercising permissions granted by this License.
25
26 "Source" form shall mean the preferred form for making modifications,
27 including but not limited to software source code, documentation
28 source, and configuration files.
29
30 "Object" form shall mean any form resulting from mechanical
31 transformation or translation of a Source form, including but
32 not limited to compiled object code, generated documentation,
33 and conversions to other media types.
34
35 "Work" shall mean the work of authorship, whether in Source or
36 Object form, made available under the License, as indicated by a
37 copyright notice that is included in or attached to the work
38 (an example is provided in the Appendix below).
39
40 "Derivative Works" shall mean any work, whether in Source or Object
41 form, that is based on (or derived from) the Work and for which the
42 editorial revisions, annotations, elaborations, or other modifications
43 represent, as a whole, an original work of authorship. For the purposes
44 of this License, Derivative Works shall not include works that remain
45 separable from, or merely link (or bind by name) to the interfaces of,
46 the Work and Derivative Works thereof.
47
48 "Contribution" shall mean any work of authorship, including
49 the original version of the Work and any modifications or additions
50 to that Work or Derivative Works thereof, that is intentionally
51 submitted to Licensor for inclusion in the Work by the copyright owner
52 or by an individual or Legal Entity authorized to submit on behalf of
53 the copyright owner. For the purposes of this definition, "submitted"
54 means any form of electronic, verbal, or written communication sent
55 to the Licensor or its representatives, including but not limited to
56 communication on electronic mailing lists, source code control systems,
57 and issue tracking systems that are managed by, or on behalf of, the
58 Licensor for the purpose of discussing and improving the Work, but
59 excluding communication that is conspicuously marked or otherwise
60 designated in writing by the copyright owner as "Not a Contribution."
61
62 "Contributor" shall mean Licensor and any individual or Legal Entity
63 on behalf of whom a Contribution has been received by Licensor and
64 subsequently incorporated within the Work.
65
66 2. Grant of Copyright License. Subject to the terms and conditions of
67 this License, each Contributor hereby grants to You a perpetual,
68 worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 copyright license to reproduce, prepare Derivative Works of,
70 publicly display, publicly perform, sublicense, and distribute the
71 Work and such Derivative Works in Source or Object form.
72
73 3. Grant of Patent License. Subject to the terms and conditions of
74 this License, each Contributor hereby grants to You a perpetual,
75 worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 (except as stated in this section) patent license to make, have made,
77 use, offer to sell, sell, import, and otherwise transfer the Work,
78 where such license applies only to those patent claims licensable
79 by such Contributor that are necessarily infringed by their
80 Contribution(s) alone or by combination of their Contribution(s)
81 with the Work to which such Contribution(s) was submitted. If You
82 institute patent litigation against any entity (including a
83 cross-claim or counterclaim in a lawsuit) alleging that the Work
84 or a Contribution incorporated within the Work constitutes direct
85 or contributory patent infringement, then any patent licenses
86 granted to You under this License for that Work shall terminate
87 as of the date such litigation is filed.
88
89 4. Redistribution. You may reproduce and distribute copies of the
90 Work or Derivative Works thereof in any medium, with or without
91 modifications, and in Source or Object form, provided that You
92 meet the following conditions:
93
94 (a) You must give any other recipients of the Work or
95 Derivative Works a copy of this License; and
96
97 (b) You must cause any modified files to carry prominent notices
98 stating that You changed the files; and
99
100 (c) You must retain, in the Source form of any Derivative Works
101 that You distribute, all copyright, patent, trademark, and
102 attribution notices from the Source form of the Work,
103 excluding those notices that do not pertain to any part of
104 the Derivative Works; and
105
106 (d) If the Work includes a "NOTICE" text file as part of its
107 distribution, then any Derivative Works that You distribute must
108 include a readable copy of the attribution notices contained
109 within such NOTICE file, excluding those notices that do not
110 pertain to any part of the Derivative Works, in at least one
111 of the following places: within a NOTICE text file distributed
112 as part of the Derivative Works; within the Source form or
113 documentation, if provided along with the Derivative Works; or,
114 within a display generated by the Derivative Works, if and
115 wherever such third-party notices normally appear. The contents
116 of the NOTICE file are for informational purposes only and
117 do not modify the License. You may add Your own attribution
118 notices within Derivative Works that You distribute, alongside
119 or as an addendum to the NOTICE text from the Work, provided
120 that such additional attribution notices cannot be construed
121 as modifying the License.
122
123 You may add Your own copyright statement to Your modifications and
124 may provide additional or different license terms and conditions
125 for use, reproduction, or distribution of Your modifications, or
126 for any such Derivative Works as a whole, provided Your use,
127 reproduction, and distribution of the Work otherwise complies with
128 the conditions stated in this License.
129
130 5. Submission of Contributions. Unless You explicitly state otherwise,
131 any Contribution intentionally submitted for inclusion in the Work
132 by You to the Licensor shall be under the terms and conditions of
133 this License, without any additional terms or conditions.
134 Notwithstanding the above, nothing herein shall supersede or modify
135 the terms of any separate license agreement you may have executed
136 with Licensor regarding such Contributions.
137
138 6. Trademarks. This License does not grant permission to use the trade
139 names, trademarks, service marks, or product names of the Licensor,
140 except as required for reasonable and customary use in describing the
141 origin of the Work and reproducing the content of the NOTICE file.
142
143 7. Disclaimer of Warranty. Unless required by applicable law or
144 agreed to in writing, Licensor provides the Work (and each
145 Contributor provides its Contributions) on an "AS IS" BASIS,
146 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 implied, including, without limitation, any warranties or conditions
148 of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 PARTICULAR PURPOSE. You are solely responsible for determining the
150 appropriateness of using or redistributing the Work and assume any
151 risks associated with Your exercise of permissions under this License.
152
153 8. Limitation of Liability. In no event and under no legal theory,
154 whether in tort (including negligence), contract, or otherwise,
155 unless required by applicable law (such as deliberate and grossly
156 negligent acts) or agreed to in writing, shall any Contributor be
157 liable to You for damages, including any direct, indirect, special,
158 incidental, or consequential damages of any character arising as a
159 result of this License or out of the use or inability to use the
160 Work (including but not limited to damages for loss of goodwill,
161 work stoppage, computer failure or malfunction, or any and all
162 other commercial damages or losses), even if such Contributor
163 has been advised of the possibility of such damages.
164
165 9. Accepting Warranty or Additional Liability. While redistributing
166 the Work or Derivative Works thereof, You may choose to offer,
167 and charge a fee for, acceptance of support, warranty, indemnity,
168 or other liability obligations and/or rights consistent with this
169 License. However, in accepting such obligations, You may act only
170 on Your own behalf and on Your sole responsibility, not on behalf
171 of any other Contributor, and only if You agree to indemnify,
172 defend, and hold each Contributor harmless for any liability
173 incurred by, or claims asserted against, such Contributor by reason
174 of your accepting any such warranty or additional liability.
175
176 END OF TERMS AND CONDITIONS
177
178 APPENDIX: How to apply the Apache License to your work.
179
180 To apply the Apache License to your work, attach the following
181 boilerplate notice, with the fields enclosed by brackets "[]"
182 replaced with your own identifying information. (Don't include
183 the brackets!) The text should be enclosed in the appropriate
184 comment syntax for the file format. We also recommend that a
185 file or class name and description of purpose be included on the
186 same "printed page" as the copyright notice for easier
187 identification within third-party archives.
188
189 Copyright [yyyy] [name of copyright owner]
190
191 Licensed under the Apache License, Version 2.0 (the "License");
192 you may not use this file except in compliance with the License.
193 You may obtain a copy of the License at
194
195 http://www.apache.org/licenses/LICENSE-2.0
196
197 Unless required by applicable law or agreed to in writing, software
198 distributed under the License is distributed on an "AS IS" BASIS,
199 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 See the License for the specific language governing permissions and
201 limitations under the License.
+27
-0
pkg/govis/README.md less more
0 ## `govis` ##
1
2 `govis` is a BSD-compatible `vis(3)` and `unvis(3)` encoding implementation
3 that is unicode aware and written in Go. None of this code comes from the
4 original BSD code, nor does it come from `go-mtree`'s port of said code.
5 Because 80s BSD code is not very nice to read.
6
7 ### License ###
8
9 `govis` is licensed under the Apache 2.0 license.
10
11 ```
12 govis: unicode aware vis(3) encoding implementation
13 Copyright (C) 2017 SUSE LLC.
14
15 Licensed under the Apache License, Version 2.0 (the "License");
16 you may not use this file except in compliance with the License.
17 You may obtain a copy of the License at
18
19 http://www.apache.org/licenses/LICENSE-2.0
20
21 Unless required by applicable law or agreed to in writing, software
22 distributed under the License is distributed on an "AS IS" BASIS,
23 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
24 See the License for the specific language governing permissions and
25 limitations under the License.
26 ```
+39
-0
pkg/govis/govis.go less more
0 /*
1 * govis: unicode aware vis(3) encoding implementation
2 * Copyright (C) 2017 SUSE LLC.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 package govis
18
19 // VisFlag manipulates how the characters are encoded/decoded
20 type VisFlag uint
21
22 // vis() has a variety of flags when deciding what encodings to use. While
23 // mtree only uses one set of flags, implementing them all is necessary in
24 // order to have compatibility with BSD's vis() and unvis() commands.
25 const (
26 VisOctal VisFlag = (1 << iota) // VIS_OCTAL: Use octal \ddd format.
27 VisCStyle // VIS_CSTYLE: Use \[nrft0..] where appropriate.
28 VisSpace // VIS_SP: Also encode space.
29 VisTab // VIS_TAB: Also encode tab.
30 VisNewline // VIS_NL: Also encode newline.
31 VisSafe // VIS_SAFE: Encode unsafe characters.
32 VisNoSlash // VIS_NOSLASH: Inhibit printing '\'.
33 VisHTTPStyle // VIS_HTTPSTYLE: HTTP-style escape %xx.
34 VisGlob // VIS_GLOB: Encode glob(3) magics.
35 visMask VisFlag = (1 << iota) - 1 // Mask of all flags.
36
37 VisWhite VisFlag = (VisSpace | VisTab | VisNewline)
38 )
+194
-0
pkg/govis/govis_test.go less more
0 /*
1 * govis: unicode aware vis(3) encoding implementation
2 * Copyright (C) 2017 SUSE LLC.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 package govis
18
19 import (
20 "bytes"
21 "crypto/rand"
22 "testing"
23 )
24
25 const DefaultVisFlags = VisWhite | VisOctal | VisGlob
26
27 func TestRandomVisUnvis(t *testing.T) {
28 // Randomly generate N strings.
29 const N = 100
30
31 for i := 0; i < N; i++ {
32 testBytes := make([]byte, 256)
33 if n, err := rand.Read(testBytes); n != cap(testBytes) || err != nil {
34 t.Fatalf("could not read enough bytes: err=%v n=%d", err, n)
35 }
36 test := string(testBytes)
37
38 for flag := VisFlag(0); flag <= visMask; flag++ {
39 // VisNoSlash is frankly just a dumb flag, and it is impossible for us
40 // to actually preserve things in a round-trip.
41 if flag&VisNoSlash == VisNoSlash {
42 continue
43 }
44
45 enc, err := Vis(test, flag)
46 if err != nil {
47 t.Errorf("unexpected error doing vis(%q, %b): %s", test, flag, err)
48 continue
49 }
50 dec, err := Unvis(enc, flag)
51 if err != nil {
52 t.Errorf("unexpected error doing unvis(%q, %b): %s", enc, flag, err)
53 continue
54 }
55 if dec != test {
56 t.Errorf("roundtrip failed: unvis(vis(%q, %b) = %q, %b) = %q", test, flag, enc, flag, dec)
57 }
58 }
59 }
60 }
61
62 func TestRandomVisVisUnvisUnvis(t *testing.T) {
63 // Randomly generate N strings.
64 const N = 100
65
66 for i := 0; i < N; i++ {
67 testBytes := make([]byte, 256)
68 if n, err := rand.Read(testBytes); n != cap(testBytes) || err != nil {
69 t.Fatalf("could not read enough bytes: err=%v n=%d", err, n)
70 }
71 test := string(testBytes)
72
73 for flag := VisFlag(0); flag <= visMask; flag++ {
74 // VisNoSlash is frankly just a dumb flag, and it is impossible for us
75 // to actually preserve things in a round-trip.
76 if flag&VisNoSlash == VisNoSlash {
77 continue
78 }
79
80 enc, err := Vis(test, flag)
81 if err != nil {
82 t.Errorf("unexpected error doing vis(%q, %b): %s", test, flag, err)
83 continue
84 }
85 enc2, err := Vis(enc, flag)
86 if err != nil {
87 t.Errorf("unexpected error doing vis(%q, %b): %s", enc, flag, err)
88 continue
89 }
90 dec, err := Unvis(enc2, flag)
91 if err != nil {
92 t.Errorf("unexpected error doing unvis(%q, %b): %s", enc2, flag, err)
93 continue
94 }
95 dec2, err := Unvis(dec, flag)
96 if err != nil {
97 t.Errorf("unexpected error doing unvis(%q, %b): %s", dec, flag, err)
98 continue
99 }
100 if dec2 != test {
101 t.Errorf("roundtrip failed: unvis(unvis(vis(vis(%q) = %q) = %q) = %q, %b) = %q", test, enc, enc2, dec, flag, dec2)
102 }
103 }
104 }
105 }
106
107 func TestVisUnvis(t *testing.T) {
108 for flag := VisFlag(0); flag <= visMask; flag++ {
109 // VisNoSlash is frankly just a dumb flag, and it is impossible for us
110 // to actually preserve things in a round-trip.
111 if flag&VisNoSlash == VisNoSlash {
112 continue
113 }
114
115 // Round-trip testing.
116 for _, test := range []string{
117 "",
118 "hello world",
119 "THIS\\IS_A_TEST1234",
120 "this.is.a.normal_string",
121 "AC_Ra\u00edz_Certic\u00e1mara_S.A..pem",
122 "NetLock_Arany_=Class_Gold=_F\u0151tan\u00fas\u00edtv\u00e1ny.pem",
123 "T\u00dcB\u0130TAK_UEKAE_K\u00f6k_Sertifika_Hizmet_Sa\u011flay\u0131c\u0131s\u0131_-_S\u00fcr\u00fcm_3.pem",
124 "hello world [ this string needs=enco ding! ]",
125 "even \n more encoding necessary\a\a ",
126 "\024 <-- some more weird characters --> \u4f60\u597d\uff0c\u4e16\u754c",
127 "\\xff\\n double encoding is also great fun \\x",
128 "AC_Ra\\M-C\\M--z_Certic\\M-C\\M-!mara_S.A..pem",
129 "z^i3i$\u00d3\u008anqgh5/t\u00e5<86>\u00b2kzla\\e^lv\u00df\u0093nv\u00df\u00aea|3}\u00d8\u0088\u00d6\u0084",
130 `z^i3i$\M-C\M^S\M-B\M^Jnqgh5/t\M-C\M-%<86>\M-B\M-2kzla\\e^lv\M-C\M^_\M-B\M^Snv\M-C\M^_\M-B\M-.a|3}\M-C\M^X\M-B\M^H\M-C\M^V\M-B\M^D`,
131 "@?e1xs+.R_Kjo]7s8pgRP:*nXCE4{!c",
132 "62_\u00c6\u00c62\u00ae\u00b7m\u00db\u00c3r^\u00bfp\u00c6u'q\u00fbc2\u00f0u\u00b8\u00dd\u00e8v\u00ff\u00b0\u00dc\u00c2\u00f53\u00db-k\u00f2sd4\\p\u00da\u00a6\u00d3\u00eea<\u00e6s{\u00a0p\u00f0\u00ffj\u00e0\u00e8\u00b8\u00b8\u00bc\u00fcb",
133 `62_\M-C\M^F\M-C\M^F2\M-B\M-.\M-B\M-7m\M-C\M^[\M-C\M^Cr^\M-B\M-?p\M-C\M^Fu'q\M-C\M-;c2\M-C\M-0u\M-B\M-8\M-C\M^]\M-C\M-(v\M-C\M-?\M-B\M-0\M-C\M^\\M-C\M^B\M-C\M-53\M-C\M^[-k\M-C\M-2sd4\\p\M-C\M^Z\M-B\M-&\M-C\M^S\M-C\M-.a<\M-C\M-&s{\M-B\240p\M-C\M-0\M-C\M-?j\M-C\240\M-C\M-(\M-B\M-8\M-B\M-8\M-B\M-<\M-C\M-<b`,
134 "\u9003\"9v1)T798|o;fly jnKX\u0489Be=",
135 `\M-i\M^@\M^C"9v1)T798|o;fly jnKX\M-R\M^IBe=`,
136 "'3Ze\u050e|\u02del\u069du-Rpct4+Z5b={@_{b",
137 `'3Ze\M-T\M^N|\M-K\M^^l\M-Z\M^]u-Rpct4+Z5b={@_{b`,
138 "1\u00c6\u00abTcz+Vda?)k1%\\\"P;`po`h",
139 `1%C3%86%C2%ABTcz+Vda%3F)k1%25%5C%22P%3B%60po%60h`,
140 } {
141 enc, err := Vis(test, flag)
142 if err != nil {
143 t.Errorf("unexpected error doing vis(%q, %b): %s", test, flag, err)
144 continue
145 }
146 dec, err := Unvis(enc, flag)
147 if err != nil {
148 t.Errorf("unexpected error doing unvis(%q, %b): %s", enc, flag, err)
149 continue
150 }
151 if dec != test {
152 t.Errorf("roundtrip failed: unvis(vis(%q, %b) = %q, %b) = %q", test, flag, enc, flag, dec)
153 }
154 }
155 }
156 }
157
158 func TestByteStrings(t *testing.T) {
159 // It's important to make sure that we don't mess around with the layout of
160 // bytes when doing a round-trip. Otherwise we risk outputting visually
161 // identical but bit-stream non-identical strings (causing much confusion
162 // when trying to access such files).
163
164 for _, test := range [][]byte{
165 []byte("This is a man in business suit levitating: \U0001f574"),
166 {0x7f, 0x17, 0x01, 0x33},
167 // TODO: Test arbitrary byte streams like the one below. Currently this
168 // fails because Vis() is messing around with it (converting it
169 // to a rune and spacing it out).
170 //{'\xef', '\xae', 'h', '\077', 'k'},
171 } {
172 testString := string(test)
173 enc, err := Vis(testString, DefaultVisFlags)
174 if err != nil {
175 t.Errorf("unexpected error doing vis(%q): %s", test, err)
176 continue
177 }
178 dec, err := Unvis(enc, DefaultVisFlags)
179 if err != nil {
180 t.Errorf("unexpected error doing unvis(%q): %s", enc, err)
181 continue
182 }
183 decBytes := []byte(dec)
184
185 if dec != testString {
186 t.Errorf("roundtrip failed [string comparison]: unvis(vis(%q) = %q) = %q", test, enc, dec)
187 }
188 if !bytes.Equal(decBytes, test) {
189 t.Errorf("roundtrip failed [byte comparison]: unvis(vis(%q) = %q) = %q", test, enc, dec)
190 }
191 }
192
193 }
+294
-0
pkg/govis/unvis.go less more
0 /*
1 * govis: unicode aware vis(3) encoding implementation
2 * Copyright (C) 2017 SUSE LLC.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 package govis
18
19 import (
20 "fmt"
21 "strconv"
22 "unicode"
23 )
24
25 // unvisParser stores the current state of the token parser.
26 type unvisParser struct {
27 tokens []rune
28 idx int
29 flag VisFlag
30 }
31
32 // Next moves the index to the next character.
33 func (p *unvisParser) Next() {
34 p.idx++
35 }
36
37 // Peek gets the current token.
38 func (p *unvisParser) Peek() (rune, error) {
39 if p.idx >= len(p.tokens) {
40 return unicode.ReplacementChar, fmt.Errorf("tried to read past end of token list")
41 }
42 return p.tokens[p.idx], nil
43 }
44
45 // End returns whether all of the tokens have been consumed.
46 func (p *unvisParser) End() bool {
47 return p.idx >= len(p.tokens)
48 }
49
50 func newParser(input string, flag VisFlag) *unvisParser {
51 return &unvisParser{
52 tokens: []rune(input),
53 idx: 0,
54 flag: flag,
55 }
56 }
57
58 // While a recursive descent parser is overkill for parsing simple escape
59 // codes, this is IMO much easier to read than the ugly 80s coroutine code used
60 // by the original unvis(3) parser. Here's the EBNF for an unvis sequence:
61 //
62 // <input> ::= (<rune>)*
63 // <rune> ::= ("\" <escape-sequence>) | ("%" <escape-hex>) | <plain-rune>
64 // <plain-rune> ::= any rune
65 // <escape-sequence> ::= ("x" <escape-hex>) | ("M" <escape-meta>) | ("^" <escape-ctrl) | <escape-cstyle> | <escape-octal>
66 // <escape-meta> ::= ("-" <escape-meta1>) | ("^" <escape-ctrl>)
67 // <escape-meta1> ::= any rune
68 // <escape-ctrl> ::= "?" | any rune
69 // <escape-cstyle> ::= "\" | "n" | "r" | "b" | "a" | "v" | "t" | "f"
70 // <escape-hex> ::= [0-9a-f] [0-9a-f]
71 // <escape-octal> ::= [0-7] ([0-7] ([0-7])?)?
72
73 func unvisPlainRune(p *unvisParser) ([]byte, error) {
74 ch, err := p.Peek()
75 if err != nil {
76 return nil, fmt.Errorf("plain rune: %c", ch)
77 }
78 p.Next()
79
80 // XXX: Maybe we should not be converting to runes and then back to strings
81 // here. Are we sure that the byte-for-byte representation is the
82 // same? If the bytes change, then using these strings for paths will
83 // break...
84
85 str := string(ch)
86 return []byte(str), nil
87 }
88
89 func unvisEscapeCStyle(p *unvisParser) ([]byte, error) {
90 ch, err := p.Peek()
91 if err != nil {
92 return nil, fmt.Errorf("escape hex: %s", err)
93 }
94
95 output := ""
96 switch ch {
97 case 'n':
98 output = "\n"
99 case 'r':
100 output = "\r"
101 case 'b':
102 output = "\b"
103 case 'a':
104 output = "\x07"
105 case 'v':
106 output = "\v"
107 case 't':
108 output = "\t"
109 case 'f':
110 output = "\f"
111 case 's':
112 output = " "
113 case 'E':
114 output = "\x1b"
115 case '\n':
116 // Hidden newline.
117 case '$':
118 // Hidden marker.
119 default:
120 // XXX: We should probably allow falling through and return "\" here...
121 return nil, fmt.Errorf("escape cstyle: unknown escape character: %q", ch)
122 }
123
124 p.Next()
125 return []byte(output), nil
126 }
127
128 func unvisEscapeDigits(p *unvisParser, base int, force bool) ([]byte, error) {
129 var code int
130
131 for i := int(0xFF); i > 0; i /= base {
132 ch, err := p.Peek()
133 if err != nil {
134 if !force && i != 0xFF {
135 break
136 }
137 return nil, fmt.Errorf("escape base %d: %s", base, err)
138 }
139
140 digit, err := strconv.ParseInt(string(ch), base, 8)
141 if err != nil {
142 if !force && i != 0xFF {
143 break
144 }
145 return nil, fmt.Errorf("escape base %d: could not parse digit: %s", base, err)
146 }
147
148 code = (code * base) + int(digit)
149 p.Next()
150 }
151
152 if code > unicode.MaxLatin1 {
153 return nil, fmt.Errorf("escape base %d: code %q outside latin-1 encoding", base, code)
154 }
155
156 char := byte(code & 0xFF)
157 return []byte{char}, nil
158 }
159
160 func unvisEscapeCtrl(p *unvisParser, mask byte) ([]byte, error) {
161 ch, err := p.Peek()
162 if err != nil {
163 return nil, fmt.Errorf("escape ctrl: %s", err)
164 }
165 if ch > unicode.MaxLatin1 {
166 return nil, fmt.Errorf("escape ctrl: code %q outside latin-1 encoding", ch)
167 }
168
169 char := byte(ch) & 0x1f
170 if ch == '?' {
171 char = 0x7f
172 }
173
174 p.Next()
175 return []byte{mask | char}, nil
176 }
177
178 func unvisEscapeMeta(p *unvisParser) ([]byte, error) {
179 ch, err := p.Peek()
180 if err != nil {
181 return nil, fmt.Errorf("escape meta: %s", err)
182 }
183
184 mask := byte(0x80)
185
186 switch ch {
187 case '^':
188 // The same as "\^..." except we apply a mask.
189 p.Next()
190 return unvisEscapeCtrl(p, mask)
191
192 case '-':
193 p.Next()
194
195 ch, err := p.Peek()
196 if err != nil {
197 return nil, fmt.Errorf("escape meta1: %s", err)
198 }
199 if ch > unicode.MaxLatin1 {
200 return nil, fmt.Errorf("escape meta1: code %q outside latin-1 encoding", ch)
201 }
202
203 // Add mask to character.
204 p.Next()
205 return []byte{mask | byte(ch)}, nil
206 }
207
208 return nil, fmt.Errorf("escape meta: unknown escape char: %s", err)
209 }
210
211 func unvisEscapeSequence(p *unvisParser) ([]byte, error) {
212 ch, err := p.Peek()
213 if err != nil {
214 return nil, fmt.Errorf("escape sequence: %s", err)
215 }
216
217 switch ch {
218 case '\\':
219 p.Next()
220 return []byte("\\"), nil
221
222 case '0', '1', '2', '3', '4', '5', '6', '7':
223 return unvisEscapeDigits(p, 8, false)
224
225 case 'x':
226 p.Next()
227 return unvisEscapeDigits(p, 16, true)
228
229 case '^':
230 p.Next()
231 return unvisEscapeCtrl(p, 0x00)
232
233 case 'M':
234 p.Next()
235 return unvisEscapeMeta(p)
236
237 default:
238 return unvisEscapeCStyle(p)
239 }
240 }
241
242 func unvisRune(p *unvisParser) ([]byte, error) {
243 ch, err := p.Peek()
244 if err != nil {
245 return nil, fmt.Errorf("rune: %s", err)
246 }
247
248 switch ch {
249 case '\\':
250 p.Next()
251 return unvisEscapeSequence(p)
252
253 case '%':
254 // % HEX HEX only applies to HTTPStyle encodings.
255 if p.flag&VisHTTPStyle == VisHTTPStyle {
256 p.Next()
257 return unvisEscapeDigits(p, 16, true)
258 }
259 fallthrough
260
261 default:
262 return unvisPlainRune(p)
263 }
264 }
265
266 func unvis(p *unvisParser) (string, error) {
267 var output []byte
268 for !p.End() {
269 ch, err := unvisRune(p)
270 if err != nil {
271 return "", fmt.Errorf("input: %s", err)
272 }
273 output = append(output, ch...)
274 }
275 return string(output), nil
276 }
277
278 // Unvis takes a string formatted with the given Vis flags (though only the
279 // VisHTTPStyle flag is checked) and output the un-encoded version of the
280 // encoded string. An error is returned if any escape sequences in the input
281 // string were invalid.
282 func Unvis(input string, flag VisFlag) (string, error) {
283 // TODO: Check all of the VisFlag bits.
284 p := newParser(input, flag)
285 output, err := unvis(p)
286 if err != nil {
287 return "", fmt.Errorf("unvis: %s", err)
288 }
289 if !p.End() {
290 return "", fmt.Errorf("unvis: trailing characters at end of input")
291 }
292 return output, nil
293 }
+166
-0
pkg/govis/unvis_test.go less more
0 /*
1 * govis: unicode aware vis(3) encoding implementation
2 * Copyright (C) 2017 SUSE LLC.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 package govis
18
19 import (
20 "testing"
21 )
22
23 func TestUnvisError(t *testing.T) {
24 for _, test := range []string{
25 // Octal escape codes allow you to specify invalid byte values.
26 "\\777",
27 "\\420\\322\\455",
28 "\\652\\233",
29 } {
30 got, err := Unvis(test, DefaultVisFlags)
31 if err == nil {
32 t.Errorf("expected unvis(%q) to give an error, got %q", test, got)
33 }
34 }
35 }
36
37 func TestUnvisCStyleEscape(t *testing.T) {
38 for _, test := range []struct {
39 input string
40 expected string
41 }{
42 {"", ""},
43 {"\\n\\v\\t\\s", "\n\v\t "},
44 {"\\\\n\\tt", "\\n\tt"},
45 {"\\b", "\b"},
46 {"\\r\\b\\n", "\r\b\n"},
47 {"\\a\\a\\b", "\x07\x07\b"},
48 {"\\f\\s\\E", "\f \x1b"},
49 // Hidden markers. They actually aren't generated by vis(3) but for
50 // some reason, they're supported...
51 {"test\\\ning", "testing"},
52 {"test\\$\\$ing", "testing"},
53 } {
54 got, err := Unvis(test.input, DefaultVisFlags)
55 if err != nil {
56 t.Errorf("unexpected error doing unvis(%q): %q", test.input, err)
57 continue
58 }
59 if got != test.expected {
60 t.Errorf("expected unvis(%q) = %q, got %q", test.input, test.expected, got)
61 }
62 }
63 }
64
65 func TestUnvisMetaEscape(t *testing.T) {
66 for _, test := range []struct {
67 input string
68 expected string
69 }{
70 {"", ""},
71 {"\\M^ ?\\^ ", "\x80?\x00"},
72 {"\\M- ?\\^?", "\xa0?\x7f"},
73 {"\\M-x butterfly\\M^?", "\xf8 butterfly\xff"},
74 {"\\M^X steady-hand \\^& needle", "\x98 steady-hand \x06 needle"},
75 // TODO: Add some more of these tests, but I need to have some
76 // secondary source to verify these outputs properly.
77 } {
78 got, err := Unvis(test.input, DefaultVisFlags)
79 if err != nil {
80 t.Errorf("unexpected error doing unvis(%q): %q", test.input, err)
81 continue
82 }
83 if got != test.expected {
84 t.Errorf("expected unvis(%q) = %q, got %q", test.input, test.expected, got)
85 }
86 }
87 }
88
89 func TestUnvisOctalEscape(t *testing.T) {
90 for _, test := range []struct {
91 input string
92 expected string
93 }{
94 {"", ""},
95 {"\\1", "\001"},
96 {"\\01\\02\\3", "\001\002\003"},
97 {"\\001\\023\\32", "\001\023\032"},
98 {"this is a test\\0k1\\133", "this is a test\000k1\133"},
99 {"\\170YET\\01another test\\1\\\\82", "\170YET\001another test\001\\82"},
100 {"\\177MORE tests\\09a", "\177MORE tests\x009a"},
101 {"\\\\710more\\1215testing", "\\710more\1215testing"},
102 // Make sure that decoding unicode works properly, when it's been encoded as single bytes.
103 {"\\360\\237\\225\\264", "\U0001f574"},
104 {"T\\303\\234B\\304\\260TAK_UEKAE_K\\303\\266k_Sertifika_Hizmet_Sa\\304\\237lay\\304\\261c\\304\\261s\\304\\261_-_S\\303\\274r\\303\\274m_3.pem", "TÜBİTAK_UEKAE_Kök_Sertifika_Hizmet_Sağlayıcısı_-_Sürüm_3.pem"},
105 // Some invalid characters...
106 {"\\377\\2\\225\\264", "\xff\x02\x95\xb4"},
107 } {
108 got, err := Unvis(test.input, DefaultVisFlags)
109 if err != nil {
110 t.Errorf("unexpected error doing unvis(%q): %q", test.input, err)
111 continue
112 }
113 if got != test.expected {
114 t.Errorf("expected unvis(%q) = %q, got %q", test.input, test.expected, got)
115 }
116 }
117 }
118
119 func TestUnvisHexEscape(t *testing.T) {
120 for _, test := range []struct {
121 input string
122 expected string
123 }{
124 {"", ""},
125 {"\\x01", "\x01"},
126 {"\\x01\\x02\\x7a", "\x01\x02\x7a"},
127 {"this is a test\\x13\\x52\\x6f", "this is a test\x13\x52\x6f"},
128 {"\\x170YET\\x01a\\x22nother test\\x11", "\x170YET\x01a\x22nother test\x11"},
129 {"\\\\x007more\\\\x215testing", "\\x007more\\x215testing"},
130 // Make sure that decoding unicode works properly, when it's been encoded as single bytes.
131 {"\\xf0\\x9f\\x95\\xb4", "\U0001f574"},
132 {"T\\xc3\\x9cB\\xc4\\xb0TAK_UEKAE_K\\xc3\\xb6k_Sertifika_Hizmet_Sa\\xc4\\x9flay\\xc4\\xb1c\\xc4\\xb1s\\xc4\\xb1_-_S\\xc3\\xbcr\\xc3\\xbcm_3.pem", "TÜBİTAK_UEKAE_Kök_Sertifika_Hizmet_Sağlayıcısı_-_Sürüm_3.pem"},
133 // Some invalid characters...
134 {"\\xff\\x02\\x95\\xb4", "\xff\x02\x95\xb4"},
135 } {
136 got, err := Unvis(test.input, DefaultVisFlags)
137 if err != nil {
138 t.Errorf("unexpected error doing unvis(%q): %q", test.input, err)
139 continue
140 }
141 if got != test.expected {
142 t.Errorf("expected unvis(%q) = %q, got %q", test.input, test.expected, got)
143 }
144 }
145 }
146
147 func TestUnvisUnicode(t *testing.T) {
148 // Ensure that unicode strings are not messed up by Unvis.
149 for _, test := range []string{
150 "",
151 "this.is.a.normal_string",
152 "AC_Raíz_Certicámara_S.A..pem",
153 "NetLock_Arany_=Class_Gold=_Főtanúsítvány.pem",
154 "TÜBİTAK_UEKAE_Kök_Sertifika_Hizmet_Sağlayıcısı_-_Sürüm_3.pem",
155 } {
156 got, err := Unvis(test, DefaultVisFlags)
157 if err != nil {
158 t.Errorf("unexpected error doing unvis(%q): %s", test, err)
159 continue
160 }
161 if got != test {
162 t.Errorf("expected %q to be unchanged, got %q", test, got)
163 }
164 }
165 }
+177
-0
pkg/govis/vis.go less more
0 /*
1 * govis: unicode aware vis(3) encoding implementation
2 * Copyright (C) 2017 SUSE LLC.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 package govis
18
19 import (
20 "fmt"
21 "unicode"
22 )
23
24 func isunsafe(ch rune) bool {
25 return ch == '\b' || ch == '\007' || ch == '\r'
26 }
27
28 func isglob(ch rune) bool {
29 return ch == '*' || ch == '?' || ch == '[' || ch == '#'
30 }
31
32 // ishttp is defined by RFC 1808.
33 func ishttp(ch rune) bool {
34 // RFC1808 does not really consider characters outside of ASCII, so just to
35 // be safe always treat characters outside the ASCII character set as "not
36 // HTTP".
37 if ch > unicode.MaxASCII {
38 return false
39 }
40
41 return unicode.IsDigit(ch) || unicode.IsLetter(ch) ||
42 // Safe characters.
43 ch == '$' || ch == '-' || ch == '_' || ch == '.' || ch == '+' ||
44 // Extra characters.
45 ch == '!' || ch == '*' || ch == '\'' || ch == '(' ||
46 ch == ')' || ch == ','
47 }
48
49 func isgraph(ch rune) bool {
50 return unicode.IsGraphic(ch) && !unicode.IsSpace(ch) && ch <= unicode.MaxASCII
51 }
52
53 // vis converts a single *byte* into its encoding. While Go supports the
54 // concept of runes (and thus native utf-8 parsing), in order to make sure that
55 // the bit-stream will be completely maintained through an Unvis(Vis(...))
56 // round-trip. The downside is that Vis() will never output unicode -- but on
57 // the plus side this is actually a benefit on the encoding side (it will
58 // always work with the simple unvis(3) implementation). It also means that we
59 // don't have to worry about different multi-byte encodings.
60 func vis(b byte, flag VisFlag) (string, error) {
61 // Treat the single-byte character as a rune.
62 ch := rune(b)
63
64 // XXX: This is quite a horrible thing to support.
65 if flag&VisHTTPStyle == VisHTTPStyle {
66 if !ishttp(ch) {
67 return "%" + fmt.Sprintf("%.2X", ch), nil
68 }
69 }
70
71 // Figure out if the character doesn't need to be encoded. Effectively, we
72 // encode most "normal" (graphical) characters as themselves unless we have
73 // been specifically asked not to. Note though that we *ALWAYS* encode
74 // everything outside ASCII.
75 // TODO: Switch this to much more logical code.
76
77 if ch > unicode.MaxASCII {
78 /* ... */
79 } else if flag&VisGlob == VisGlob && isglob(ch) {
80 /* ... */
81 } else if isgraph(ch) ||
82 (flag&VisSpace != VisSpace && ch == ' ') ||
83 (flag&VisTab != VisTab && ch == '\t') ||
84 (flag&VisNewline != VisNewline && ch == '\n') ||
85 (flag&VisSafe != 0 && isunsafe(ch)) {
86
87 encoded := string(ch)
88 if ch == '\\' && flag&VisNoSlash == 0 {
89 encoded += "\\"
90 }
91 return encoded, nil
92 }
93
94 // Try to use C-style escapes first.
95 if flag&VisCStyle == VisCStyle {
96 switch ch {
97 case ' ':
98 return "\\s", nil
99 case '\n':
100 return "\\n", nil
101 case '\r':
102 return "\\r", nil
103 case '\b':
104 return "\\b", nil
105 case '\a':
106 return "\\a", nil
107 case '\v':
108 return "\\v", nil
109 case '\t':
110 return "\\t", nil
111 case '\f':
112 return "\\f", nil
113 case '\x00':
114 // Output octal just to be safe.
115 return "\\000", nil
116 }
117 }
118
119 // For graphical characters we generate octal output (and also if it's
120 // being forced by the caller's flags). Also spaces should always be
121 // encoded as octal.
122 if flag&VisOctal == VisOctal || isgraph(ch) || ch&0x7f == ' ' {
123 // Always output three-character octal just to be safe.
124 return fmt.Sprintf("\\%.3o", ch), nil
125 }
126
127 // Now we have to output meta or ctrl escapes. As far as I can tell, this
128 // is not actually defined by any standard -- so this logic is basically
129 // copied from the original vis(3) implementation. Hopefully nobody
130 // actually relies on this (octal and hex are better).
131
132 encoded := ""
133 if flag&VisNoSlash == 0 {
134 encoded += "\\"
135 }
136
137 // Meta characters have 0x80 set, but are otherwise identical to control
138 // characters.
139 if b&0x80 != 0 {
140 b &= 0x7f
141 encoded += "M"
142 }
143
144 if unicode.IsControl(rune(b)) {
145 encoded += "^"
146 if b == 0x7f {
147 encoded += "?"
148 } else {
149 encoded += fmt.Sprintf("%c", b+'@')
150 }
151 } else {
152 encoded += fmt.Sprintf("-%c", b)
153 }
154
155 return encoded, nil
156 }
157
158 // Vis encodes the provided string to a BSD-compatible encoding using BSD's
159 // vis() flags. However, it will correctly handle multi-byte encoding (which is
160 // not done properly by BSD's vis implementation).
161 func Vis(src string, flag VisFlag) (string, error) {
162 if flag&visMask != flag {
163 return "", fmt.Errorf("vis: flag %q contains unknown or unsupported flags", flag)
164 }
165
166 output := ""
167 for _, ch := range []byte(src) {
168 encodedCh, err := vis(ch, flag)
169 if err != nil {
170 return "", err
171 }
172 output += encodedCh
173 }
174
175 return output, nil
176 }
+127
-0
pkg/govis/vis_test.go less more
0 /*
1 * govis: unicode aware vis(3) encoding implementation
2 * Copyright (C) 2017 SUSE LLC.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 package govis
18
19 import (
20 "testing"
21 )
22
23 func TestVisUnchanged(t *testing.T) {
24 for _, test := range []struct {
25 input string
26 flag VisFlag
27 }{
28 {"", DefaultVisFlags},
29 {"helloworld", DefaultVisFlags},
30 {"THIS_IS_A_TEST1234", DefaultVisFlags},
31 {"SomeEncodingsAreCool", DefaultVisFlags},
32 {"spaces are totally safe", DefaultVisFlags &^ VisSpace},
33 {"tabs\tare\talso\tsafe!!", DefaultVisFlags &^ VisTab},
34 {"just\a\atrustme\r\b\b!!", DefaultVisFlags | VisSafe},
35 } {
36 enc, err := Vis(test.input, test.flag)
37 if err != nil {
38 t.Errorf("unexpected error with %q: %s", test, err)
39 }
40 if enc != test.input {
41 t.Errorf("expected encoding of %q (flag=%q) to be unchanged, got %q", test.input, test.flag, enc)
42 }
43 }
44 }
45
46 func TestVisFlags(t *testing.T) {
47 for _, test := range []struct {
48 input string
49 output string
50 flag VisFlag
51 }{
52 // Default
53 {"AC_Ra\u00edz_Certic\u00e1mara_S.A..pem", "AC_Ra\\M-C\\M--z_Certic\\M-C\\M-!mara_S.A..pem", 0},
54 {"z^i3i$\u00d3\u008anqgh5/t\u00e5<86>\u00b2kzla\\e^lv\u00df\u0093nv\u00df\u00aea|3}\u00d8\u0088\u00d6\u0084", `z^i3i$\M-C\M^S\M-B\M^Jnqgh5/t\M-C\M-%<86>\M-B\M-2kzla\\e^lv\M-C\M^_\M-B\M^Snv\M-C\M^_\M-B\M-.a|3}\M-C\M^X\M-B\M^H\M-C\M^V\M-B\M^D`, 0},
55 {"@?e1xs+.R_Kjo]7s8pgRP:*nXCE4{!c", "@?e1xs+.R_Kjo]7s8pgRP:*nXCE4{!c", 0},
56 {"62_\u00c6\u00c62\u00ae\u00b7m\u00db\u00c3r^\u00bfp\u00c6u'q\u00fbc2\u00f0u\u00b8\u00dd\u00e8v\u00ff\u00b0\u00dc\u00c2\u00f53\u00db-k\u00f2sd4\\p\u00da\u00a6\u00d3\u00eea<\u00e6s{\u00a0p\u00f0\u00ffj\u00e0\u00e8\u00b8\u00b8\u00bc\u00fcb", `62_\M-C\M^F\M-C\M^F2\M-B\M-.\M-B\M-7m\M-C\M^[\M-C\M^Cr^\M-B\M-?p\M-C\M^Fu'q\M-C\M-;c2\M-C\M-0u\M-B\M-8\M-C\M^]\M-C\M-(v\M-C\M-?\M-B\M-0\M-C\M^\\M-C\M^B\M-C\M-53\M-C\M^[-k\M-C\M-2sd4\\p\M-C\M^Z\M-B\M-&\M-C\M^S\M-C\M-.a<\M-C\M-&s{\M-B\240p\M-C\M-0\M-C\M-?j\M-C\240\M-C\M-(\M-B\M-8\M-B\M-8\M-B\M-<\M-C\M-<b`, 0},
57 {"\u9003\"9v1)T798|o;fly jnKX\u0489Be=", `\M-i\M^@\M^C"9v1)T798|o;fly jnKX\M-R\M^IBe=`, 0},
58 // VisOctal
59 {"", "", VisOctal},
60 {"\022", "\\022", VisOctal},
61 {"\n \t", "\\012\\040\t", VisNewline | VisSpace | VisOctal},
62 {"\x12\f\a\n\v\b \U00012312", "\\022\\014\\007\n\\013\\010 \\360\\222\\214\\222", VisOctal},
63 {"AC_Ra\u00edz_Certic\u00e1mara_S.A..pem", "AC_Ra\\303\\255z_Certic\\303\\241mara_S.A..pem", VisOctal},
64 {"z^i3i$\u00d3\u008anqgh5/t\u00e5<86>\u00b2kzla\\e^lv\u00df\u0093nv\u00df\u00aea|3}\u00d8\u0088\u00d6\u0084", `z^i3i$\303\223\302\212nqgh5/t\303\245<86>\302\262kzla\\e^lv\303\237\302\223nv\303\237\302\256a|3}\303\230\302\210\303\226\302\204`, VisOctal},
65 {"62_\u00c6\u00c62\u00ae\u00b7m\u00db\u00c3r^\u00bfp\u00c6u'q\u00fbc2\u00f0u\u00b8\u00dd\u00e8v\u00ff\u00b0\u00dc\u00c2\u00f53\u00db-k\u00f2sd4\\p\u00da\u00a6\u00d3\u00eea<\u00e6s{\u00a0p\u00f0\u00ffj\u00e0\u00e8\u00b8\u00b8\u00bc\u00fcb", `62_\303\206\303\2062\302\256\302\267m\303\233\303\203r^\302\277p\303\206u'q\303\273c2\303\260u\302\270\303\235\303\250v\303\277\302\260\303\234\303\202\303\2653\303\233-k\303\262sd4\\p\303\232\302\246\303\223\303\256a<\303\246s{\302\240p\303\260\303\277j\303\240\303\250\302\270\302\270\302\274\303\274b`, VisOctal},
66 {"\u9003\"9v1)T798|o;fly jnKX\u0489Be=", `\351\200\203"9v1)T798|o;fly jnKX\322\211Be=`, VisOctal},
67 // VisCStyle
68 {"\x00 \f \a \n\v\b \r \t\r", "\\000 \\f \\a \n\\v\\b \\r \t\\r", VisCStyle},
69 {"\t \n\v\b", "\\t \n\\v\\b", VisTab | VisCStyle},
70 {"\n\v\t ", "\n\\v\t\\s\\s\\s", VisSpace | VisCStyle},
71 {"\n \n ", "\\n \\n ", VisNewline | VisCStyle},
72 {"z^i3i$\u00d3\u008anqgh5/t\u00e5<86>\u00b2kzla\\e^lv\u00df\u0093nv\u00df\u00aea|3}\u00d8\u0088\u00d6\u0084", `z^i3i$\M-C\M^S\M-B\M^Jnqgh5/t\M-C\M-%<86>\M-B\M-2kzla\\e^lv\M-C\M^_\M-B\M^Snv\M-C\M^_\M-B\M-.a|3}\M-C\M^X\M-B\M^H\M-C\M^V\M-B\M^D`, VisCStyle},
73 {"62_\u00c6\u00c62\u00ae\u00b7m\u00db\u00c3r^\u00bfp\u00c6u'q\u00fbc2\u00f0u\u00b8\u00dd\u00e8v\u00ff\u00b0\u00dc\u00c2\u00f53\u00db-k\u00f2sd4\\p\u00da\u00a6\u00d3\u00eea<\u00e6s{\u00a0p\u00f0\u00ffj\u00e0\u00e8\u00b8\u00b8\u00bc\u00fcb", `62_\M-C\M^F\M-C\M^F2\M-B\M-.\M-B\M-7m\M-C\M^[\M-C\M^Cr^\M-B\M-?p\M-C\M^Fu'q\M-C\M-;c2\M-C\M-0u\M-B\M-8\M-C\M^]\M-C\M-(v\M-C\M-?\M-B\M-0\M-C\M^\\M-C\M^B\M-C\M-53\M-C\M^[-k\M-C\M-2sd4\\p\M-C\M^Z\M-B\M-&\M-C\M^S\M-C\M-.a<\M-C\M-&s{\M-B\240p\M-C\M-0\M-C\M-?j\M-C\240\M-C\M-(\M-B\M-8\M-B\M-8\M-B\M-<\M-C\M-<b`, VisCStyle},
74 {"\u9003\"9v1)T798|o;fly jnKX\u0489Be=", `\M-i\M^@\M^C"9v1)T798|o;fly\sjnKX\M-R\M^IBe=`, VisCStyle | VisSpace},
75 // VisSpace
76 {" ", `\040\040`, VisSpace},
77 {"\t \t", "\t\\040\t", VisSpace},
78 {"\\040 plenty of characters here ", `\\040\040\040\040plenty\040of\040characters\040here\040\040\040`, VisSpace},
79 {"Js9L\u00cd\u00b2o?4824y'$|P}FIr%mW /KL9$]~", `Js9L\M-C\M^M\M-B\M-2o?4824y'$|P}FIr%mW\040/KL9$]~`, VisWhite},
80 {"1\u00c6\u00abTcz+Vda?)k1%\\\"P;`po`h", `1\M-C\M^F\M-B\M-+Tcz+Vda?)k1%\\"P;` + "`po`" + `h`, VisWhite},
81 {"\u9003\"9v1)T798|o;fly jnKX\u0489Be=", `\M-i\M^@\M^C"9v1)T798|o;fly\040jnKX\M-R\M^IBe=`, VisSpace},
82 // VisTab
83 {"\t \v", "\\^I \\^K", VisTab},
84 {"\t \v", "\\011 \\013", VisTab | VisOctal},
85 // VisNewline
86 {"\t\n \v\r\n", "\t\\^J \\^K\\^M\\^J", VisNewline},
87 {"\t\n \v\r\n", "\t\\012 \\013\\015\\012", VisNewline | VisOctal},
88 // VisSafe
89 // VisHTTPStyle
90 {"\x12\f\a\n\v\b \U00012312", `%12%0C%07%0A%0B%08%20%20%F0%92%8C%92`, VisHTTPStyle},
91 {"1\u00c6\u00abTcz+Vda?)k1%\\\"P;`po`h", `1%C3%86%C2%ABTcz+Vda%3F)k1%25%5C%22P%3B%60po%60h`, VisHTTPStyle},
92 {"62_\u00c6\u00c62\u00ae\u00b7m\u00db\u00c3r^\u00bfp\u00c6u'q\u00fbc2\u00f0u\u00b8\u00dd\u00e8v\u00ff\u00b0\u00dc\u00c2\u00f53\u00db-k\u00f2sd4\\p\u00da\u00a6\u00d3\u00eea<\u00e6s{\u00a0p\u00f0\u00ffj\u00e0\u00e8\u00b8\u00b8\u00bc\u00fcb", `62_%C3%86%C3%862%C2%AE%C2%B7m%C3%9B%C3%83r%5E%C2%BFp%C3%86u'q%C3%BBc2%C3%B0u%C2%B8%C3%9D%C3%A8v%C3%BF%C2%B0%C3%9C%C3%82%C3%B53%C3%9B-k%C3%B2sd4%5Cp%C3%9A%C2%A6%C3%93%C3%AEa%3C%C3%A6s%7B%C2%A0p%C3%B0%C3%BFj%C3%A0%C3%A8%C2%B8%C2%B8%C2%BC%C3%BCb`, VisHTTPStyle},
93 {"'3Ze\u050e|\u02del\u069du-Rpct4+Z5b={@_{b", `'3Ze%D4%8E%7C%CB%9El%DA%9Du-Rpct4+Z5b%3D%7B%40_%7Bb`, VisHTTPStyle},
94 // VisGlob
95 {"cat /proc/**/status | grep '[pid]' ;; # cool code here", `cat /proc/\052\052/status | grep '\133pid]' ;; \043 cool code here`, VisGlob},
96 {"@?e1xs+.R_Kjo]7s8pgRP:*nXCE4{!c", `@\077e1xs+.R_Kjo]7s8pgRP:\052nXCE4{!c`, VisGlob},
97 {"62_\u00c6\u00c62\u00ae\u00b7m\u00db\u00c3r^\u00bfp\u00c6u'q\u00fbc2\u00f0u\u00b8\u00dd\u00e8v\u00ff\u00b0\u00dc\u00c2\u00f53\u00db-k\u00f2sd4\\p\u00da\u00a6\u00d3\u00eea<\u00e6s{\u00a0p\u00f0\u00ffj\u00e0\u00e8\u00b8\u00b8\u00bc\u00fcb", `62_\M-C\M^F\M-C\M^F2\M-B\M-.\M-B\M-7m\M-C\M^[\M-C\M^Cr^\M-B\M-?p\M-C\M^Fu'q\M-C\M-;c2\M-C\M-0u\M-B\M-8\M-C\M^]\M-C\M-(v\M-C\M-?\M-B\M-0\M-C\M^\\M-C\M^B\M-C\M-53\M-C\M^[-k\M-C\M-2sd4\\p\M-C\M^Z\M-B\M-&\M-C\M^S\M-C\M-.a<\M-C\M-&s{\M-B\240p\M-C\M-0\M-C\M-?j\M-C\240\M-C\M-(\M-B\M-8\M-B\M-8\M-B\M-<\M-C\M-<b`, VisGlob},
98 {"62_\u00c6\u00c62\u00ae\u00b7m\u00db\u00c3r^\u00bfp\u00c6u'q\u00fbc2\u00f0u\u00b8\u00dd\u00e8v\u00ff\u00b0\u00dc\u00c2\u00f53\u00db-k\u00f2sd4\\p\u00da\u00a6\u00d3\u00eea<\u00e6s{\u00a0p\u00f0\u00ffj\u00e0\u00e8\u00b8\u00b8\u00bc\u00fcb", `62_\303\206\303\2062\302\256\302\267m\303\233\303\203r^\302\277p\303\206u'q\303\273c2\303\260u\302\270\303\235\303\250v\303\277\302\260\303\234\303\202\303\2653\303\233-k\303\262sd4\\p\303\232\302\246\303\223\303\256a<\303\246s{\302\240p\303\260\303\277j\303\240\303\250\302\270\302\270\302\274\303\274b`, VisGlob | VisOctal},
99 {"'3Ze\u050e|\u02del\u069du-Rpct4+Z5b={@_{b", `'3Ze\M-T\M^N|\M-K\M^^l\M-Z\M^]u-Rpct4+Z5b={@_{b`, VisGlob},
100 {"'3Ze\u050e|\u02del\u069du-Rpct4+Z5b={@_{b", `'3Ze\324\216|\313\236l\332\235u-Rpct4+Z5b={@_{b`, VisGlob | VisOctal},
101 } {
102 enc, err := Vis(test.input, test.flag)
103 if err != nil {
104 t.Errorf("unexpected error with %q: %s", test, err)
105 }
106 if enc != test.output {
107 t.Errorf("expected vis(%q, flag=%b) = %q, got %q", test.input, test.flag, test.output, enc)
108 }
109 }
110 }
111
112 func TestVisChanged(t *testing.T) {
113 for _, test := range []string{
114 "hello world",
115 "THIS\\IS_A_TEST1234",
116 "AC_Ra\u00edz_Certic\u00e1mara_S.A..pem",
117 } {
118 enc, err := Vis(test, DefaultVisFlags)
119 if err != nil {
120 t.Errorf("unexpected error with %q: %s", test, err)
121 }
122 if enc == test {
123 t.Errorf("expected encoding of %q to be changed", test)
124 }
125 }
126 }
+11
-0
releases.md less more
0 # How to do releases:
1
2 * Create a changeset with an update to `version.go`
3 - this commit will be tagged
4 - add another commit putting it back with '-dev' appended
5 * gpg sign the commit with an incremented version, like 'vX.Y.Z'
6 * Push the tag
7 * Create a "release" from the tag on github
8 - include the binaries from `make build.arches`
9 - write about notable changes, and their contributors
10 - PRs merged for the release
+18
-0
stat_unix.go less more
0 // +build !windows
1
2 package mtree
3
4 import (
5 "os"
6 "syscall"
7 )
8
9 func statIsUID(stat os.FileInfo, uid int) bool {
10 statT := stat.Sys().(*syscall.Stat_t)
11 return statT.Uid == uint32(uid)
12 }
13
14 func statIsGID(stat os.FileInfo, gid int) bool {
15 statT := stat.Sys().(*syscall.Stat_t)
16 return statT.Gid == uint32(gid)
17 }
+12
-0
stat_windows.go less more
0 // +build windows
1
2 package mtree
3
4 import "os"
5
6 func statIsUID(stat os.FileInfo, uid int) bool {
7 return false
8 }
9 func statIsGID(stat os.FileInfo, uid int) bool {
10 return false
11 }
+461
-0
tar.go less more
0 package mtree
1
2 import (
3 "archive/tar"
4 "fmt"
5 "io"
6 "io/ioutil"
7 "os"
8 "path/filepath"
9 "strings"
10
11 "github.com/Sirupsen/logrus"
12 "github.com/vbatts/go-mtree/pkg/govis"
13 )
14
15 // Streamer creates a file hierarchy out of a tar stream
16 type Streamer interface {
17 io.ReadCloser
18 Hierarchy() (*DirectoryHierarchy, error)
19 }
20
21 var tarDefaultSetKeywords = []KeyVal{
22 "type=file",
23 "flags=none",
24 "mode=0664",
25 }
26
27 // NewTarStreamer streams a tar archive and creates a file hierarchy based off
28 // of the tar metadata headers
29 func NewTarStreamer(r io.Reader, excludes []ExcludeFunc, keywords []Keyword) Streamer {
30 pR, pW := io.Pipe()
31 ts := &tarStream{
32 pipeReader: pR,
33 pipeWriter: pW,
34 creator: dhCreator{DH: &DirectoryHierarchy{}},
35 teeReader: io.TeeReader(r, pW),
36 tarReader: tar.NewReader(pR),
37 keywords: keywords,
38 hardlinks: map[string][]string{},
39 excludes: excludes,
40 }
41
42 go ts.readHeaders()
43 return ts
44 }
45
46 type tarStream struct {
47 root *Entry
48 hardlinks map[string][]string
49 creator dhCreator
50 pipeReader *io.PipeReader
51 pipeWriter *io.PipeWriter
52 teeReader io.Reader
53 tarReader *tar.Reader
54 keywords []Keyword
55 excludes []ExcludeFunc
56 err error
57 }
58
59 func (ts *tarStream) readHeaders() {
60 // remove "time" keyword
61 notimekws := []Keyword{}
62 for _, kw := range ts.keywords {
63 if !InKeywordSlice(kw, notimekws) {
64 if kw == "time" {
65 if !InKeywordSlice("tar_time", ts.keywords) {
66 notimekws = append(notimekws, "tar_time")
67 }
68 } else {
69 notimekws = append(notimekws, kw)
70 }
71 }
72 }
73 ts.keywords = notimekws
74 // We have to start with the directory we're in, and anything beyond these
75 // items is determined at the time a tar is extracted.
76 ts.root = &Entry{
77 Name: ".",
78 Type: RelativeType,
79 Prev: &Entry{
80 Raw: "# .",
81 Type: CommentType,
82 },
83 Set: nil,
84 Keywords: []KeyVal{"type=dir"},
85 }
86 // insert signature and metadata comments first (user, machine, tree, date)
87 for _, e := range signatureEntries("<user specified tar archive>") {
88 e.Pos = len(ts.creator.DH.Entries)
89 ts.creator.DH.Entries = append(ts.creator.DH.Entries, e)
90 }
91 // insert keyword metadata next
92 for _, e := range keywordEntries(ts.keywords) {
93 e.Pos = len(ts.creator.DH.Entries)
94 ts.creator.DH.Entries = append(ts.creator.DH.Entries, e)
95 }
96 hdrloop:
97 for {
98 hdr, err := ts.tarReader.Next()
99 if err != nil {
100 ts.pipeReader.CloseWithError(err)
101 return
102 }
103
104 for _, ex := range ts.excludes {
105 if ex(hdr.Name, hdr.FileInfo()) {
106 continue hdrloop
107 }
108 }
109
110 // Because the content of the file may need to be read by several
111 // KeywordFuncs, it needs to be an io.Seeker as well. So, just reading from
112 // ts.tarReader is not enough.
113 tmpFile, err := ioutil.TempFile("", "ts.payload.")
114 if err != nil {
115 ts.pipeReader.CloseWithError(err)
116 return
117 }
118 // for good measure
119 if err := tmpFile.Chmod(0600); err != nil {
120 tmpFile.Close()
121 os.Remove(tmpFile.Name())
122 ts.pipeReader.CloseWithError(err)
123 return
124 }
125 if _, err := io.Copy(tmpFile, ts.tarReader); err != nil {
126 tmpFile.Close()
127 os.Remove(tmpFile.Name())
128 ts.pipeReader.CloseWithError(err)
129 return
130 }
131 // Alright, it's either file or directory
132 encodedName, err := govis.Vis(filepath.Base(hdr.Name), DefaultVisFlags)
133 if err != nil {
134 tmpFile.Close()
135 os.Remove(tmpFile.Name())
136 ts.pipeReader.CloseWithError(err)
137 return
138 }
139 e := Entry{
140 Name: encodedName,
141 Type: RelativeType,
142 }
143
144 // Keep track of which files are hardlinks so we can resolve them later
145 if hdr.Typeflag == tar.TypeLink {
146 keyFunc := KeywordFuncs["link"]
147 kvs, err := keyFunc(hdr.Name, hdr.FileInfo(), nil)
148 if err != nil {
149 logrus.Warn(err)
150 break // XXX is breaking an okay thing to do here?
151 }
152 linkname, err := govis.Unvis(KeyVal(kvs[0]).Value(), DefaultVisFlags)
153 if err != nil {
154 logrus.Warn(err)
155 break // XXX is breaking an okay thing to do here?
156 }
157 if _, ok := ts.hardlinks[linkname]; !ok {
158 ts.hardlinks[linkname] = []string{hdr.Name}
159 } else {
160 ts.hardlinks[linkname] = append(ts.hardlinks[linkname], hdr.Name)
161 }
162 }
163
164 // now collect keywords on the file
165 for _, keyword := range ts.keywords {
166 if keyFunc, ok := KeywordFuncs[keyword.Prefix()]; ok {
167 // We can't extract directories on to disk, so "size" keyword
168 // is irrelevant for now
169 if hdr.FileInfo().IsDir() && keyword == "size" {
170 continue
171 }
172 kvs, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
173 if err != nil {
174 ts.setErr(err)
175 }
176 // for good measure, check that we actually get a value for a keyword
177 if len(kvs) > 0 && kvs[0] != "" {
178 e.Keywords = append(e.Keywords, kvs[0])
179 }
180
181 // don't forget to reset the reader
182 if _, err := tmpFile.Seek(0, 0); err != nil {
183 tmpFile.Close()
184 os.Remove(tmpFile.Name())
185 ts.pipeReader.CloseWithError(err)
186 return
187 }
188 }
189 }
190 // collect meta-set keywords for a directory so that we can build the
191 // actual sets in `flatten`
192 if hdr.FileInfo().IsDir() {
193 s := Entry{
194 Name: "meta-set",
195 Type: SpecialType,
196 }
197 for _, setKW := range SetKeywords {
198 if keyFunc, ok := KeywordFuncs[setKW.Prefix()]; ok {
199 kvs, err := keyFunc(hdr.Name, hdr.FileInfo(), tmpFile)
200 if err != nil {
201 ts.setErr(err)
202 }
203 for _, kv := range kvs {
204 if kv != "" {
205 s.Keywords = append(s.Keywords, kv)
206 }
207 }
208 if _, err := tmpFile.Seek(0, 0); err != nil {
209 tmpFile.Close()
210 os.Remove(tmpFile.Name())
211 ts.pipeReader.CloseWithError(err)
212 }
213 }
214 }
215 e.Set = &s
216 }
217 err = populateTree(ts.root, &e, hdr)
218 if err != nil {
219 ts.setErr(err)
220 }
221 tmpFile.Close()
222 os.Remove(tmpFile.Name())
223 }
224 }
225
226 // populateTree creates a pseudo file tree hierarchy using an Entry's Parent and
227 // Children fields. When examining the Entry e to insert in the tree, we
228 // determine if the path to that Entry exists yet. If it does, insert it in the
229 // appropriate position in the tree. If not, create a path up until the Entry's
230 // directory that it is contained in. Then, insert the Entry.
231 // root: the "." Entry
232 // e: the Entry we are looking to insert
233 // hdr: the tar header struct associated with e
234 func populateTree(root, e *Entry, hdr *tar.Header) error {
235 if root == nil || e == nil {
236 return fmt.Errorf("cannot populate or insert nil Entry's")
237 } else if root.Prev == nil {
238 return fmt.Errorf("root needs to be an Entry associated with a directory")
239 }
240 isDir := hdr.FileInfo().IsDir()
241 wd := filepath.Clean(hdr.Name)
242 if !isDir {
243 // directory up until the actual file
244 wd = filepath.Dir(wd)
245 if wd == "." {
246 root.Children = append([]*Entry{e}, root.Children...)
247 e.Parent = root
248 return nil
249 }
250 }
251 dirNames := strings.Split(wd, "/")
252 parent := root
253 for _, name := range dirNames[:] {
254 encoded, err := govis.Vis(name, DefaultVisFlags)
255 if err != nil {
256 return err
257 }
258 if node := parent.Descend(encoded); node == nil {
259 // Entry for directory doesn't exist in tree relative to root.
260 // We don't know if this directory is an actual tar header (because a
261 // user could have just specified a path to a deep file), so we must
262 // specify this placeholder directory as a "type=dir", and Set=nil.
263 newEntry := Entry{
264 Name: encoded,
265 Type: RelativeType,
266 Parent: parent,
267 Keywords: []KeyVal{"type=dir"}, // temp data
268 Set: nil, // temp data
269 }
270 pathname, err := newEntry.Path()
271 if err != nil {
272 return err
273 }
274 newEntry.Prev = &Entry{
275 Type: CommentType,
276 Raw: "# " + pathname,
277 }
278 parent.Children = append(parent.Children, &newEntry)
279 parent = &newEntry
280 } else {
281 // Entry for directory exists in tree, just keep going
282 parent = node
283 }
284 }
285 if !isDir {
286 parent.Children = append([]*Entry{e}, parent.Children...)
287 e.Parent = parent
288 } else {
289 // fill in the actual data from e
290 parent.Keywords = e.Keywords
291 parent.Set = e.Set
292 }
293 return nil
294 }
295
296 // After constructing a pseudo file hierarchy tree, we want to "flatten" this
297 // tree by putting the Entries into a slice with appropriate positioning.
298 // root: the "head" of the sub-tree to flatten
299 // creator: a dhCreator that helps with the '/set' keyword
300 // keywords: keywords specified by the user that should be evaluated
301 func flatten(root *Entry, creator *dhCreator, keywords []Keyword) {
302 if root == nil || creator == nil {
303 return
304 }
305 if root.Prev != nil {
306 // root.Prev != nil implies root is a directory
307 creator.DH.Entries = append(creator.DH.Entries,
308 Entry{
309 Type: BlankType,
310 Pos: len(creator.DH.Entries),
311 })
312 root.Prev.Pos = len(creator.DH.Entries)
313 creator.DH.Entries = append(creator.DH.Entries, *root.Prev)
314
315 if root.Set != nil {
316 // Check if we need a new set
317 consolidatedKeys := keyvalSelector(append(tarDefaultSetKeywords, root.Set.Keywords...), keywords)
318 if creator.curSet == nil {
319 creator.curSet = &Entry{
320 Type: SpecialType,
321 Name: "/set",
322 Keywords: consolidatedKeys,
323 Pos: len(creator.DH.Entries),
324 }
325 creator.DH.Entries = append(creator.DH.Entries, *creator.curSet)
326 } else {
327 needNewSet := false
328 for _, k := range root.Set.Keywords {
329 if !inKeyValSlice(k, creator.curSet.Keywords) {
330 needNewSet = true
331 break
332 }
333 }
334 if needNewSet {
335 creator.curSet = &Entry{
336 Name: "/set",
337 Type: SpecialType,
338 Pos: len(creator.DH.Entries),
339 Keywords: consolidatedKeys,
340 }
341 creator.DH.Entries = append(creator.DH.Entries, *creator.curSet)
342 }
343 }
344 } else if creator.curSet != nil {
345 // Getting into here implies that the Entry's set has not and
346 // was not supposed to be evaluated, thus, we need to reset curSet
347 creator.DH.Entries = append(creator.DH.Entries, Entry{
348 Name: "/unset",
349 Type: SpecialType,
350 Pos: len(creator.DH.Entries),
351 })
352 creator.curSet = nil
353 }
354 }
355 root.Set = creator.curSet
356 if creator.curSet != nil {
357 root.Keywords = keyValDifference(root.Keywords, creator.curSet.Keywords)
358 }
359 root.Pos = len(creator.DH.Entries)
360 creator.DH.Entries = append(creator.DH.Entries, *root)
361 for _, c := range root.Children {
362 flatten(c, creator, keywords)
363 }
364 if root.Prev != nil {
365 // Show a comment when stepping out
366 root.Prev.Pos = len(creator.DH.Entries)
367 creator.DH.Entries = append(creator.DH.Entries, *root.Prev)
368 dotEntry := Entry{
369 Type: DotDotType,
370 Name: "..",
371 Pos: len(creator.DH.Entries),
372 }
373 creator.DH.Entries = append(creator.DH.Entries, dotEntry)
374 }
375 return
376 }
377
378 // resolveHardlinks goes through an Entry tree, and finds the Entry's associated
379 // with hardlinks and fills them in with the actual data from the base file.
380 func resolveHardlinks(root *Entry, hardlinks map[string][]string, countlinks bool) {
381 originals := make(map[string]*Entry)
382 for base, links := range hardlinks {
383 var basefile *Entry
384 if seen, ok := originals[base]; !ok {
385 basefile = root.Find(base)
386 if basefile == nil {
387 logrus.Printf("%s does not exist in this tree\n", base)
388 continue
389 }
390 originals[base] = basefile
391 } else {
392 basefile = seen
393 }
394 for _, link := range links {
395 linkfile := root.Find(link)
396 if linkfile == nil {
397 logrus.Printf("%s does not exist in this tree\n", link)
398 continue
399 }
400 linkfile.Keywords = basefile.Keywords
401 if countlinks {
402 linkfile.Keywords = append(linkfile.Keywords, KeyVal(fmt.Sprintf("nlink=%d", len(links)+1)))
403 }
404 }
405 if countlinks {
406 basefile.Keywords = append(basefile.Keywords, KeyVal(fmt.Sprintf("nlink=%d", len(links)+1)))
407 }
408 }
409 }
410
411 // filter takes in a pointer to an Entry, and returns a slice of Entry's that
412 // satisfy the predicate p
413 func filter(root *Entry, p func(*Entry) bool) []Entry {
414 if root != nil {
415 var validEntrys []Entry
416 if len(root.Children) > 0 || root.Prev != nil {
417 for _, c := range root.Children {
418 // filter the sub-directory
419 if c.Prev != nil {
420 validEntrys = append(validEntrys, filter(c, p)...)
421 }
422 if p(c) {
423 if c.Prev == nil {
424 validEntrys = append([]Entry{*c}, validEntrys...)
425 } else {
426 validEntrys = append(validEntrys, *c)
427 }
428 }
429 }
430 return validEntrys
431 }
432 }
433 return nil
434 }
435
436 func (ts *tarStream) setErr(err error) {
437 ts.err = err
438 }
439
440 func (ts *tarStream) Read(p []byte) (n int, err error) {
441 return ts.teeReader.Read(p)
442 }
443
444 func (ts *tarStream) Close() error {
445 return ts.pipeReader.Close()
446 }
447
448 // Hierarchy returns the DirectoryHierarchy of the archive. It flattens the
449 // Entry tree before returning the DirectoryHierarchy
450 func (ts *tarStream) Hierarchy() (*DirectoryHierarchy, error) {
451 if ts.err != nil && ts.err != io.EOF {
452 return nil, ts.err
453 }
454 if ts.root == nil {
455 return nil, fmt.Errorf("root Entry not found, nothing to flatten")
456 }
457 resolveHardlinks(ts.root, ts.hardlinks, InKeywordSlice(Keyword("nlink"), ts.keywords))
458 flatten(ts.root, &ts.creator, ts.keywords)
459 return ts.creator.DH, nil
460 }
+407
-0
tar_test.go less more
0 package mtree
1
2 import (
3 "archive/tar"
4 "bytes"
5 "io"
6 "io/ioutil"
7 "os"
8 "path/filepath"
9 "syscall"
10 "testing"
11 "time"
12 )
13
14 func ExampleStreamer() {
15 fh, err := os.Open("./testdata/test.tar")
16 if err != nil {
17 // handle error ...
18 }
19 str := NewTarStreamer(fh, nil, nil)
20 if err := extractTar("/tmp/dir", str); err != nil {
21 // handle error ...
22 }
23
24 dh, err := str.Hierarchy()
25 if err != nil {
26 // handle error ...
27 }
28
29 res, err := Check("/tmp/dir/", dh, nil, nil)
30 if err != nil {
31 // handle error ...
32 }
33 if len(res) > 0 {
34 // handle validation issue ...
35 }
36 }
37 func extractTar(root string, tr io.Reader) error {
38 return nil
39 }
40
41 func TestTar(t *testing.T) {
42 /*
43 data, err := makeTarStream()
44 if err != nil {
45 t.Fatal(err)
46 }
47 buf := bytes.NewBuffer(data)
48 str := NewTarStreamer(buf, append(DefaultKeywords, "sha1"))
49 */
50 /*
51 // open empty folder and check size.
52 fh, err := os.Open("./testdata/empty")
53 if err != nil {
54 t.Fatal(err)
55 }
56 log.Println(fh.Stat())
57 fh.Close() */
58 fh, err := os.Open("./testdata/test.tar")
59 if err != nil {
60 t.Fatal(err)
61 }
62 str := NewTarStreamer(fh, nil, append(DefaultKeywords, "sha1"))
63
64 if _, err := io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
65 t.Fatal(err)
66 }
67 if err := str.Close(); err != nil {
68 t.Fatal(err)
69 }
70 defer fh.Close()
71
72 // get DirectoryHierarcy struct from walking the tar archive
73 tdh, err := str.Hierarchy()
74 if err != nil {
75 t.Fatal(err)
76 }
77 if tdh == nil {
78 t.Fatal("expected a DirectoryHierarchy struct, but got nil")
79 }
80
81 testDir, present := os.LookupEnv("MTREE_TESTDIR")
82 if present == false {
83 testDir = "."
84 }
85 testPath := filepath.Join(testDir, "test.mtree")
86 fh, err = os.Create(testPath)
87 if err != nil {
88 t.Fatal(err)
89 }
90 defer os.Remove(testPath)
91
92 // put output of tar walk into test.mtree
93 _, err = tdh.WriteTo(fh)
94 if err != nil {
95 t.Fatal(err)
96 }
97 fh.Close()
98
99 // now simulate gomtree -T testdata/test.tar -f testdata/test.mtree
100 fh, err = os.Open(testPath)
101 if err != nil {
102 t.Fatal(err)
103 }
104 defer fh.Close()
105
106 dh, err := ParseSpec(fh)
107 if err != nil {
108 t.Fatal(err)
109 }
110
111 res, err := TarCheck(tdh, dh, append(DefaultKeywords, "sha1"))
112 if err != nil {
113 t.Fatal(err)
114 }
115
116 // print any failures, and then call t.Fatal once all failures/extra/missing
117 // are outputted
118 if len(res) > 0 {
119 for _, delta := range res {
120 t.Error(delta)
121 }
122 t.Fatal("unexpected errors")
123 }
124 }
125
126 // This test checks how gomtree handles archives that were created
127 // with multiple directories, i.e, archives created with something like:
128 // `tar -cvf some.tar dir1 dir2 dir3 dir4/dir5 dir6` ... etc.
129 // The testdata of collection.tar resemble such an archive. the `collection` folder
130 // is the contents of `collection.tar` extracted
131 func TestArchiveCreation(t *testing.T) {
132 fh, err := os.Open("./testdata/collection.tar")
133 if err != nil {
134 t.Fatal(err)
135 }
136 str := NewTarStreamer(fh, nil, []Keyword{"sha1"})
137
138 if _, err := io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
139 t.Fatal(err)
140 }
141 if err := str.Close(); err != nil {
142 t.Fatal(err)
143 }
144 defer fh.Close()
145
146 // get DirectoryHierarcy struct from walking the tar archive
147 tdh, err := str.Hierarchy()
148 if err != nil {
149 t.Fatal(err)
150 }
151
152 // Test the tar manifest against the actual directory
153 res, err := Check("./testdata/collection", tdh, []Keyword{"sha1"}, nil)
154 if err != nil {
155 t.Fatal(err)
156 }
157
158 if len(res) > 0 {
159 for _, delta := range res {
160 t.Error(delta)
161 }
162 t.Fatal("unexpected errors")
163 }
164
165 // Test the tar manifest against itself
166 res, err = TarCheck(tdh, tdh, []Keyword{"sha1"})
167 if err != nil {
168 t.Fatal(err)
169 }
170 if len(res) > 0 {
171 for _, delta := range res {
172 t.Error(delta)
173 }
174 t.Fatal("unexpected errors")
175 }
176
177 // Validate the directory manifest against the archive
178 dh, err := Walk("./testdata/collection", nil, []Keyword{"sha1"}, nil)
179 if err != nil {
180 t.Fatal(err)
181 }
182 res, err = TarCheck(tdh, dh, []Keyword{"sha1"})
183 if err != nil {
184 t.Fatal(err)
185 }
186 if len(res) > 0 {
187 for _, delta := range res {
188 t.Error(delta)
189 }
190 t.Fatal("unexpected errors")
191 }
192 }
193
194 // Now test a tar file that was created with just the path to a file. In this
195 // test case, the traversal and creation of "placeholder" directories are
196 // evaluated. Also, The fact that this archive contains a single entry, yet the
197 // entry is associated with a file that has parent directories, means that the
198 // "." directory should be the lowest sub-directory under which `file` is contained.
199 func TestTreeTraversal(t *testing.T) {
200 fh, err := os.Open("./testdata/traversal.tar")
201 if err != nil {
202 t.Fatal(err)
203 }
204 str := NewTarStreamer(fh, nil, DefaultTarKeywords)
205
206 if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
207 t.Fatal(err)
208 }
209 if err = str.Close(); err != nil {
210 t.Fatal(err)
211 }
212
213 fh.Close()
214 tdh, err := str.Hierarchy()
215
216 if err != nil {
217 t.Fatal(err)
218 }
219
220 res, err := TarCheck(tdh, tdh, []Keyword{"sha1"})
221 if err != nil {
222 t.Fatal(err)
223 }
224 if len(res) > 0 {
225 for _, delta := range res {
226 t.Error(delta)
227 }
228 t.Fatal("unexpected errors")
229 }
230
231 // top-level "." directory will contain contents of traversal.tar
232 res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}, nil)
233 if err != nil {
234 t.Fatal(err)
235 }
236 if len(res) > 0 {
237 var failed bool
238 for _, delta := range res {
239 // We only care about missing or modified files.
240 // The original test was written using the old check code.
241 if delta.Type() != Extra {
242 failed = true
243 t.Error(delta)
244 }
245 }
246 if failed {
247 t.Fatal("unexpected errors")
248 }
249 }
250
251 // Now test an archive that requires placeholder directories, i.e, there are
252 // no headers in the archive that are associated with the actual directory name
253 fh, err = os.Open("./testdata/singlefile.tar")
254 if err != nil {
255 t.Fatal(err)
256 }
257 str = NewTarStreamer(fh, nil, DefaultTarKeywords)
258 if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
259 t.Fatal(err)
260 }
261 if err = str.Close(); err != nil {
262 t.Fatal(err)
263 }
264 tdh, err = str.Hierarchy()
265 if err != nil {
266 t.Fatal(err)
267 }
268
269 // Implied top-level "." directory will contain the contents of singlefile.tar
270 res, err = Check("./testdata/.", tdh, []Keyword{"sha1"}, nil)
271 if err != nil {
272 t.Fatal(err)
273 }
274 if len(res) > 0 {
275 var failed bool
276 for _, delta := range res {
277 // We only care about missing or modified files.
278 // The original test was written using the old check code.
279 if delta.Type() != Extra {
280 failed = true
281 t.Error(delta)
282 }
283 }
284 if failed {
285 t.Fatal("unexpected errors")
286 }
287 }
288 }
289
290 func TestHardlinks(t *testing.T) {
291 fh, err := os.Open("./testdata/hardlinks.tar")
292 if err != nil {
293 t.Fatal(err)
294 }
295 str := NewTarStreamer(fh, nil, append(DefaultTarKeywords, "nlink"))
296
297 if _, err = io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
298 t.Fatal(err)
299 }
300 if err = str.Close(); err != nil {
301 t.Fatal(err)
302 }
303
304 fh.Close()
305 tdh, err := str.Hierarchy()
306
307 if err != nil {
308 t.Fatal(err)
309 }
310 foundnlink := false
311 for _, e := range tdh.Entries {
312 if e.Type == RelativeType {
313 for _, kv := range e.Keywords {
314 if KeyVal(kv).Keyword() == "nlink" {
315 foundnlink = true
316 if KeyVal(kv).Value() != "3" {
317 t.Errorf("expected to have 3 hardlinks for %s", e.Name)
318 }
319 }
320 }
321 }
322 }
323 if !foundnlink {
324 t.Errorf("nlink expected to be evaluated")
325 }
326 }
327
328 type fakeFile struct {
329 Name, Body string
330 Mode int64
331 Type byte
332 Sec, Nsec int64
333 Xattrs map[string]string
334 }
335
336 // minimal tar archive that mimics what is in ./testdata/test.tar
337 var minimalFiles = []fakeFile{
338 {"x/", "", 0755, '5', 0, 0, nil},
339 {"x/files", "howdy\n", 0644, '0', 0, 0, nil},
340 }
341
342 func makeTarStream(ff []fakeFile) ([]byte, error) {
343 buf := new(bytes.Buffer)
344
345 // Create a new tar archive.
346 tw := tar.NewWriter(buf)
347
348 // Add some files to the archive.
349 for _, file := range ff {
350 hdr := &tar.Header{
351 Name: file.Name,
352 Uid: syscall.Getuid(),
353 Gid: syscall.Getgid(),
354 Mode: file.Mode,
355 Typeflag: file.Type,
356 Size: int64(len(file.Body)),
357 ModTime: time.Unix(file.Sec, file.Nsec),
358 AccessTime: time.Unix(file.Sec, file.Nsec),
359 ChangeTime: time.Unix(file.Sec, file.Nsec),
360 Xattrs: file.Xattrs,
361 }
362 if err := tw.WriteHeader(hdr); err != nil {
363 return nil, err
364 }
365 if len(file.Body) > 0 {
366 if _, err := tw.Write([]byte(file.Body)); err != nil {
367 return nil, err
368 }
369 }
370 }
371 // Make sure to check the error on Close.
372 if err := tw.Close(); err != nil {
373 return nil, err
374 }
375 return buf.Bytes(), nil
376 }
377
378 func TestArchiveExcludeNonDirectory(t *testing.T) {
379 fh, err := os.Open("./testdata/collection.tar")
380 if err != nil {
381 t.Fatal(err)
382 }
383 str := NewTarStreamer(fh, []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"})
384
385 if _, err := io.Copy(ioutil.Discard, str); err != nil && err != io.EOF {
386 t.Fatal(err)
387 }
388 if err := str.Close(); err != nil {
389 t.Fatal(err)
390 }
391 fh.Close()
392 // get DirectoryHierarcy struct from walking the tar archive
393 tdh, err := str.Hierarchy()
394 if err != nil {
395 t.Fatal(err)
396 }
397 for i := range tdh.Entries {
398 for _, keyval := range tdh.Entries[i].AllKeys() {
399 if tdh.Entries[i].Type == FullType || tdh.Entries[i].Type == RelativeType {
400 if keyval.Keyword() == "type" && keyval.Value() != "dir" {
401 t.Errorf("expected only directories, but %q is a %q", tdh.Entries[i].Name, keyval.Value())
402 }
403 }
404 }
405 }
406 }
+44
-0
test/cli/0001.sh less more
0 #!/bin/bash
1 set -e
2
3 name=$(basename $0)
4 root="$(dirname $(dirname $(dirname $0)))"
5 gomtree=$(readlink -f ${root}/gomtree)
6 t=$(mktemp -t -d go-mtree.XXXXXX)
7
8 echo "[${name}] Running in ${t}"
9 # This test is for basic running check of manifest, and check against tar and file system
10 #
11
12 pushd ${root}
13
14 git archive --format=tar HEAD^{tree} . > ${t}/${name}.tar
15
16 prev_umask=$(umask)
17 umask 0 # this is so the tar command can set the mode's properly
18 mkdir -p ${t}/extract
19 tar -C ${t}/extract/ -xf ${t}/${name}.tar
20 umask ${prev_umask}
21
22 # create manifest from tar
23 ${gomtree} -K sha256digest -c -T ${t}/${name}.tar > ${t}/${name}.mtree
24
25 # check tar-manifest against the tar
26 ${gomtree} -f ${t}/${name}.mtree -T ${t}/${name}.tar
27
28 # check tar-manifest against the filesystem
29 # git archive makes the uid/gid as 0, so don't check them for this test
30 ${gomtree} -k size,sha256digest,mode,type -f ${t}/${name}.mtree -p ${t}/extract/
31
32 # create a manifest from filesystem
33 ${gomtree} -K sha256digest -c -p ${t}/extract/ > ${t}/${name}.mtree
34
35 # check filesystem-manifest against the filesystem
36 ${gomtree} -f ${t}/${name}.mtree -p ${t}/extract/
37
38 # check filesystem-manifest against the tar
39 # git archive makes the uid/gid as 0, so don't check them for this test
40 ${gomtree} -k size,sha256digest,mode,type -f ${t}/${name}.mtree -T ${t}/${name}.tar
41
42 popd
43 rm -rf ${t}
+24
-0
test/cli/0002.sh less more
0 #!/bin/bash
1 set -e
2
3 name=$(basename $0)
4 root="$(dirname $(dirname $(dirname $0)))"
5 gomtree=$(readlink -f ${root}/gomtree)
6 t=$(mktemp -t -d go-mtree.XXXXXX)
7
8 echo "[${name}] Running in ${t}"
9 # This test is for basic running check of manifest, and check against tar and file system
10 #
11
12 pushd ${root}
13
14 # Create a symlink with spaces in the entries.
15 mkdir ${t}/root
16 ln -s "this is a dummy symlink" ${t}/root/link
17
18 # Create manifest and check it against the same symlink.
19 ${gomtree} -K link,sha256digest -c -p ${t}/root > ${t}/root.mtree
20 ${gomtree} -K link,sha256digest -f ${t}/root.mtree -p ${t}/root
21
22 popd
23 rm -rf ${t}
+33
-0
test/cli/0003.sh less more
0 #!/bin/bash
1 set -e
2
3 name=$(basename $0)
4 root="$(dirname $(dirname $(dirname $0)))"
5 gomtree=$(readlink -f ${root}/gomtree)
6 t=$(mktemp -t -d go-mtree.XXXXXX)
7
8 setfattr -n user.has.xattrs -v "true" "${t}" || exit 0
9
10 echo "[${name}] Running in ${t}"
11
12 mkdir "${t}/dir"
13 touch "${t}/dir/file"
14
15 setfattr -n user.mtree.testing -v "apples and=bananas" "${t}/dir/file"
16 $gomtree -c -k "sha256digest,xattrs" -p ${t}/dir > ${t}/${name}.mtree
17
18 setfattr -n user.mtree.testing -v "bananas and lemons" "${t}/dir/file"
19 ! $gomtree -p ${t}/dir -f ${t}/${name}.mtree
20
21 setfattr -x user.mtree.testing "${t}/dir/file"
22 ! $gomtree -p ${t}/dir -f ${t}/${name}.mtree
23
24 setfattr -n user.mtree.testing -v "apples and=bananas" "${t}/dir/file"
25 setfattr -n user.mtree.another -v "another a=b" "${t}/dir/file"
26 ! $gomtree -p ${t}/dir -f ${t}/${name}.mtree
27
28 setfattr -n user.mtree.testing -v "apples and=bananas" "${t}/dir/file"
29 setfattr -x user.mtree.another "${t}/dir/file"
30 $gomtree -p ${t}/dir -f ${t}/${name}.mtree
31
32 rm -fr ${t}
+24
-0
test/cli/0004.sh less more
0 #!/bin/bash
1 set -e
2 #set -x
3
4 name=$(basename $0)
5 root="$(dirname $(dirname $(dirname $0)))"
6 gomtree=$(readlink -f ${root}/gomtree)
7 t=$(mktemp -t -d go-mtree.XXXXXX)
8
9 echo "[${name}] Running in ${t}"
10
11 pushd ${root}
12 git archive --format=tar HEAD^{tree} . > ${t}/${name}.tar
13 mkdir -p ${t}/extract
14 tar -C ${t}/extract/ -xf ${t}/${name}.tar
15
16 ## This is a checking that keyword synonyms are respected
17 ${gomtree} -k sha1digest -c -p ${t}/extract/ > ${t}/${name}.mtree
18 ${gomtree} -k sha1 -f ${t}/${name}.mtree -p ${t}/extract/
19 ${gomtree} -k sha1 -c -p ${t}/extract/ > ${t}/${name}.mtree
20 ${gomtree} -k sha1digest -f ${t}/${name}.mtree -p ${t}/extract/
21
22 popd
23 rm -rf ${t}
+19
-0
test/cli/0005.sh less more
0 #!/bin/bash
1 set -e
2
3 name=$(basename $0)
4 root="$(dirname $(dirname $(dirname $0)))"
5 gomtree=$(readlink -f ${root}/gomtree)
6 t=$(mktemp -t -d go-mtree.XXXXXX)
7
8 echo "[${name}] Running in ${t}"
9
10 pushd ${root}
11 mkdir -p ${t}/extract
12 git archive --format=tar HEAD^{tree} . | tar -C ${t}/extract/ -x
13
14 ${gomtree} -k sha1digest -c -p ${t}/extract/ > ${t}/${name}.mtree
15 ${gomtree} -f ${t}/${name}.mtree -k md5digest -p ${t}/extract/
16
17 popd
18 rm -rf ${t}
+33
-0
test/cli/0006.sh less more
0 #!/bin/bash
1 set -e
2
3 name=$(basename $0)
4 root="$(dirname $(dirname $(dirname $0)))"
5 gomtree=$(readlink -f ${root}/gomtree)
6 t=$(mktemp -t -d go-mtree.XXXXXX)
7
8 echo "[${name}] Running in ${t}"
9 # This test is for basic running check of manifest, and check against tar and file system
10 #
11
12 pushd ${root}
13
14 git archive --format=tar HEAD^{tree} . > ${t}/${name}.tar
15
16 prev_umask=$(umask)
17 umask 0 # this is so the tar command can set the mode's properly
18 mkdir -p ${t}/extract
19 tar -C ${t}/extract/ -xf ${t}/${name}.tar
20 umask ${prev_umask}
21
22 # create manifest from tar, ignoring non directories
23 ${gomtree} -d -c -k type -T ${t}/${name}.tar > ${t}/${name}.mtree
24
25 # check tar-manifest against the tar
26 ${gomtree} -d -f ${t}/${name}.mtree -T ${t}/${name}.tar
27
28 # check filesystem-manifest against the filesystem
29 ${gomtree} -f ${t}/${name}.mtree -p ${t}/extract/
30
31 popd
32 rm -rf ${t}
+18
-0
test/cli/0007.sh less more
0 #!/bin/bash
1 set -e
2
3 name=$(basename $0)
4 root="$(dirname $(dirname $(dirname $0)))"
5 gomtree=$(readlink -f ${root}/gomtree)
6 left=$(mktemp -t -d go-mtree.XXXXXX)
7 right=$(mktemp -t -d go-mtree.XXXXXX)
8
9 echo "[${name}] Running in ${left} and ${right}"
10
11 touch ${left}/one
12 touch ${left}/two
13 cp -a ${left}/one ${right}/
14
15 $gomtree -K "sha256digest" -p ${left} -c > /tmp/left.mtree
16 $gomtree -k "sha256digest" -p ${right} -f /tmp/left.mtree
17 rm -fr ${left} ${right}
+22
-0
test/cli/0008.sh less more
0 #!/bin/bash
1 set -e
2
3 name=$(basename $0)
4 root="$(dirname $(dirname $(dirname $0)))"
5 gomtree=$(readlink -f ${root}/gomtree)
6 t=$(mktemp -d /tmp/go-mtree.XXXXXX)
7
8 echo "[${name}] Running in ${t}"
9
10 pushd ${root}
11 mkdir -p ${t}/extract
12 git archive --format=tar HEAD^{tree} . | tar -C ${t}/extract/ -x
13
14 ${gomtree} -K sha256digest -c -p ${t}/extract/ > ${t}/${name}.mtree
15
16 ## This is a use-case for checking a directory, but by reading the manifest from stdin
17 ## since the `-f` flag is not provided.
18 cat ${t}/${name}.mtree | ${gomtree} -p ${t}/extract/
19
20 popd
21 rm -rf ${t}
+33
-0
test/cli/0009.sh less more
0 #!/bin/bash
1 set -e
2
3 name=$(basename $0)
4 root="$(dirname $(dirname $(dirname $0)))"
5 gomtree=$(readlink -f ${root}/gomtree)
6 t=$(mktemp -t -d go-mtree.XXXXXX)
7
8 echo "[${name}] Running in ${t}"
9
10 pushd ${root}
11
12 # Create some unicode files.
13 mkdir ${t}/root
14 echo "some data" > "${t}/root/$(printf 'this file has \u042a some unicode !!')"
15 echo "more data" > "${t}/root/$(printf 'even more \x07 unicode \ua4ff characters \udead\ubeef\ucafe')"
16 mkdir -p "${t}/root/$(printf '\024 <-- some more weird characters --> \u4f60\u597d\uff0c\u4e16\u754c')"
17 ln -s "$(printf '62_\u00c6\u00c62\u00ae\u00b7m\u00db\u00c3r^\u00bfp\u00c6u"q\u00fbc2\u00f0u\u00b8\u00dd\u00e8v\u00ff\u00b0\u00dc\u00c2\u00f53\u00db')" "${t}/root/$(printf 'k\u00f2sd4\\p\u00da\u00a6\u00d3\u00eea<\u00e6s{\u00a0p\u00f0\u00ffj\u00e0\u00e8\u00b8\u00b8\u00bc\u00fcb')"
18 printf 'some lovely data 62_\u00c6\u00c62\u00ae\u00b7m\u00db\u00c3r' > "${t}/root/$(printf 'T\u00dcB\u0130TAK_UEKAE_K\u00f6k_Sertifika_Hizmet_Sa\u011flay\u0131c\u0131s\u0131_-_S\u00fcr\u00fcm_3.pem')"
19
20 # Create manifest and check it against the same root.
21 ${gomtree} -k uid,gid,size,type,link,nlink,sha256digest -c -p ${t}/root > ${t}/root.mtree
22 ${gomtree} -k uid,gid,size,type,link,nlink,sha256digest -f ${t}/root.mtree -p ${t}/root
23
24 # Modify it and make sure that it successfully figures out what changed.
25 echo "othe data" > "${t}/root/$(printf 'this file has \u042a some unicode !!')"
26 ! ${gomtree} -k uid,gid,size,type,link,nlink,sha256digest -f ${t}/root.mtree -p ${t}/root
27
28 echo "some data" > "${t}/root/$(printf 'this file has \u042a some unicode !!')"
29 ${gomtree} -k uid,gid,size,type,link,nlink,sha256digest -f ${t}/root.mtree -p ${t}/root
30
31 popd
32 rm -rf ${t}
+30
-0
test/cli.go less more
0 package main
1
2 import (
3 "flag"
4 "fmt"
5 "os"
6 "os/exec"
7 )
8
9 func main() {
10 flag.Parse()
11
12 failed := 0
13 for _, arg := range flag.Args() {
14 cmd := exec.Command("bash", arg)
15 if os.Getenv("TMPDIR") != "" {
16 cmd.Env = append(cmd.Env, "TMPDIR="+os.Getenv("TMPDIR"))
17 }
18 cmd.Stderr = os.Stderr
19 cmd.Stdout = os.Stdout
20 if err := cmd.Run(); err != nil {
21 failed++
22 fmt.Fprintf(os.Stderr, "FAILED: %s\n", arg)
23 }
24 }
25 if failed > 0 {
26 fmt.Fprintf(os.Stderr, "%d FAILED tests\n", failed)
27 os.Exit(1)
28 }
29 }
+0
-0
testdata/collection/dir1/file1 less more
(New empty file)
+0
-0
testdata/collection/dir2/file2 less more
(New empty file)
+0
-0
testdata/collection/dir3/file3 less more
(New empty file)
+0
-0
testdata/collection/dir4/file4 less more
(New empty file)
+0
-0
testdata/collection/dir5/dir6/dir7/lonelyfile less more
(New empty file)
+1
-0
testdata/collection/file1 less more
0 im
+1
-0
testdata/collection/file2 less more
0 hello
+1
-0
testdata/collection/file3 less more
0 programming
testdata/collection.tar less more
Binary diff not shown
+0
-0
testdata/dirwithbrokenlink/ file with spaces less more
(New empty file)
+0
-0
testdata/dirwithbrokenlink/deepdir/dir3/dir4/deepfile less more
(New empty file)
+1
-0
testdata/dirwithbrokenlink/deepdir/dir3/dir4/dir5/dir6/deeperfile less more
0 some spacious text
+1
-0
testdata/dirwithbrokenlink/deepdir/dir3/dir4/dir5/dir7/deeperfile less more
0 some nice text
+0
-0
testdata/dirwithbrokenlink/deepdir/dir3/file less more
(New empty file)
+0
-0
testdata/dirwithbrokenlink/deepdir/dir3/file6 less more
(New empty file)
+0
-0
testdata/dirwithbrokenlink/deepdir/dir3/file7 less more
(New empty file)
+0
-0
testdata/dirwithbrokenlink/deepdir/this_is_a_file less more
(New empty file)
+0
-0
testdata/dirwithbrokenlink/dir1/.hiddenfile less more
(New empty file)
+1
-0
testdata/dirwithbrokenlink/dir1/badlink less more
0 badfile⏎
+1
-0
testdata/dirwithbrokenlink/dir1/dir2/.hidden/goodlink less more
0 ../../goodfile⏎
+1
-0
testdata/dirwithbrokenlink/dir1/goodfile less more
0 link text
+0
-0
testdata/dirwithbrokenlink/file1 less more
(New empty file)
+0
-0
testdata/dirwithbrokenlink/file2 less more
(New empty file)
testdata/hardlinks.tar less more
Binary diff not shown
+0
-0
testdata/singlefile/dir2/dir3/file less more
(New empty file)
testdata/singlefile.tar less more
Binary diff not shown
+196
-0
testdata/source.mtree less more
0 # user: vbatts
1 # machine: valse.usersys.redhat.com
2 # tree: /home/vbatts/src/vb/go-mtree
3 # date: Thu Mar 10 16:15:36 2016
4
5 # .
6 /set type=file uid=1000 gid=1000 mode=0664 nlink=1 flags=none
7 . type=dir mode=0775 nlink=5 size=4096 \
8 time=1457644509.817998120
9 .mtree.go.swp \
10 mode=0644 size=12288 time=1457644483.833957552 \
11 sha512digest=7efd38cc957b53e3d5abd25e9df9bde715a5e18dc18abfa8f0c0eaa9bca0fda32faad4972bd4d9b555270fa3b3b92e6a1e97ad1e3bfff14fc933284004f7e8d3
12 .mtree_test.go.swp \
13 mode=0644 size=12288 time=1457644509.813998114 \
14 sha512digest=2aaeb1e5f03a563de86a4c1f722bde40312bbecbe4504d67fb441240c049e6ad11294ef834114a05e620a390ee9f42c1c77b43bf1a4f2d47f128226183968a33
15 mtree.go size=1719 time=1457644447.020900074 \
16 sha512digest=2dee9f50ba167cd90d4f6e8494d4735c6b064c279ea4d26ed2c47552a5c0b82b0bdcadb214e0ca919a1768ce0e321e8d1bb71eacb9c5b4bb5fd4342967055a04
17 mtree.go~ size=1719 time=1457644447.012900062 \
18 sha512digest=2dee9f50ba167cd90d4f6e8494d4735c6b064c279ea4d26ed2c47552a5c0b82b0bdcadb214e0ca919a1768ce0e321e8d1bb71eacb9c5b4bb5fd4342967055a04
19 mtree_test.go \
20 size=581 time=1457644509.817998120 \
21 sha512digest=1b3ab2de2759df6e7d424d05ffd1d45101fd10ae86328abad1657e68044add69d83d6b3271225070f4f4154a7c0c995016077f8c7969890e068cd0a54b23f4b6
22 mtree_test.go~ \
23 size=581 time=1457644509.812998113 \
24 sha512digest=1b3ab2de2759df6e7d424d05ffd1d45101fd10ae86328abad1657e68044add69d83d6b3271225070f4f4154a7c0c995016077f8c7969890e068cd0a54b23f4b6
25
26 # ./.git
27 /set type=file uid=1000 gid=1000 mode=0775 nlink=1 flags=none
28 .git type=dir nlink=7 size=4096 time=1457644470.212936285
29 HEAD mode=0664 size=23 time=1454678171.602940889 \
30 sha512digest=76b199f1bffdc6d21547895431b0f85f3857df400225cbd1ae55e7ded09786a8a44e1cb21f1feff4951cf57c893043a0be696dcb0e4f5758a4d6081a833eb6cf
31 config mode=0664 size=92 time=1454678171.607940896 \
32 sha512digest=961231503987d2154696364e8e4d7b3aaf2d2abe5da55367b615a5d9e1bcda30824c30880bfdb8e9959444040d91cd3ec6fa225e8a36a93cfc182bf263d6e02a
33 description mode=0664 size=73 time=1454678171.602940889 \
34 sha512digest=f7e152f75b124c3e1c5d12f00427729d9eec4e2c1bf70d7606440a6600d003367eb178331e75ab976a61496e79c2c822020849d28ffd170946397b934611b471
35 index mode=0664 size=272 time=1457644470.212936285 \
36 sha512digest=fcecb444e3748cccfc498a63939f489f6456f1cf9e0e8ace21b7cdf8eecfb0e0a3db9bf6d449db2608752e5c0dbae786b4facb46f359555871b7e48d80b9a765
37
38 # ./.git/branches
39 branches type=dir nlink=2 size=4096 time=1454678171.596940880
40 # ./.git/branches
41 ..
42
43
44 # ./.git/hooks
45 hooks type=dir nlink=2 size=4096 time=1454678171.600940886
46 applypatch-msg.sample \
47 size=478 time=1454678171.597940882 \
48 sha512digest=536cce804d84e25813993efdd240537b52d00ce9cdcecf1982f85096d56a521290104c825c00b370b2752201952a9616a3f4e28c5d27a5b4e4842101a2ff9bee
49 commit-msg.sample \
50 size=896 time=1454678171.598940883 \
51 sha512digest=d6bb7fa747f4625adf1877f546565cbe812ca7dd4168f7e9068e6732555d8737eba549546cf5946649e3f38de82d173aaf9c160a4c9f9445655258b4c5f955eb
52 post-update.sample \
53 size=189 time=1454678171.598940883 \
54 sha512digest=473ad124642571656276bf83b9ff63ab1804d3c23a5bdae52391c6f70a894849ac60c10c9d31deff3938922ce83b68b1e60c11592bbf7ea503f4acd39968cefa
55 pre-applypatch.sample \
56 size=424 time=1454678171.598940883 \
57 sha512digest=cb78aa7e9b9c146e5db65d86dd83f04e2b6942a06fab50c704a0fd900683f3b6ad1164e74afe2f267f6da91cdff0b9ab07713e12cefc6f8d741b5df194f4fda6
58 pre-commit.sample \
59 size=1642 time=1454678171.599940885 \
60 sha512digest=4a7ac2ed2ffe50d2a997c3477b5855c3d19423c295a82f7d660be3418217a183b0ce25f94ae2e350f4d93eda3b75a4bd62cf27f0685d8a837919bcc9e7e0962a
61 pre-push.sample \
62 size=1348 time=1454678171.599940885 \
63 sha512digest=37cd20a090494acb16c2255fe93dadeb409f2553afddc409dad5b5e99c9f5612a28eba397c0f3ae771e17edbe0a6c2a3af311154285e6d9c43e4cc4f06ba5b8d
64 pre-rebase.sample \
65 size=4951 time=1454678171.600940886 \
66 sha512digest=de9bdf513f383b3215e467c025ca851a0f2661b91811bfd441486b678e4d45e450ce9185a7c3ac2988bd6154d4983b3e29427623117e73623f2af43cf4b746da
67 prepare-commit-msg.sample \
68 size=1239 time=1454678171.600940886 \
69 sha512digest=fbf15f12136a413dd590e57392e6122714d67456beed50d9ef285e40076a138a7d44291b5642fe5c518dc56c640436caf6225d1a966e06c042b30e6a7d08dcef
70 update.sample \
71 size=3611 time=1454678171.601940888 \
72 sha512digest=f8e20df0e6c970fa809d5c9494cb9d5e32894fe20f23f6730df84f5fa065e52c55cca98d0db1ed17df53c606629c787ae56b6af6faca527f62c817f8cb148c1a
73 # ./.git/hooks
74 ..
75
76
77 # ./.git/info
78 /set type=file uid=1000 gid=1000 mode=0664 nlink=1 flags=none
79 info type=dir mode=0775 nlink=2 size=4096 \
80 time=1454678171.601940888
81 exclude size=240 time=1454678171.601940888 \
82 sha512digest=9828c6ecdf91bf117416e17f4ee9caee2e1e37b6fb00b9ff04035ace17a3089b9d0a25c6baa1046c0e1c62d3da88838e8fca74ea82973d6b975905fde58f3072
83 # ./.git/info
84 ..
85
86
87 # ./.git/objects
88 /set type=file uid=1000 gid=1000 mode=0775 nlink=1 flags=none
89 objects type=dir nlink=7 size=4096 time=1457644470.212936285
90
91 # ./.git/objects/35
92 /set type=file uid=1000 gid=1000 mode=0444 nlink=1 flags=none
93 35 type=dir mode=0775 nlink=2 size=4096 \
94 time=1457644470.212936285
95 51bcca10021bb56f13dd57a84b97cdf7fda0cf \
96 size=344 time=1457644470.212936285 \
97 sha512digest=cc9cbb88b1c060f78a06d65decb59f2e4f7cafe25fc962d549b1826b91f2b27f22d13ec104acdea1dc30fa22c533fa88da9df2785a782dacbf93264741099043
98 # ./.git/objects/35
99 ..
100
101
102 # ./.git/objects/45
103 45 type=dir mode=0775 nlink=2 size=4096 \
104 time=1457644470.212936285
105 15ff8d078d5291b75e99cbaa3e6c5e83ffaa2e \
106 size=363 time=1457644470.212936285 \
107 sha512digest=b4cb2c25df4a6d5a158b4d945d1c1bd26a17b286c472bb9ad5b43624ee09c01b24c817875ac7611992f2b45eea0287cb0d8e4ae257ed054d1e33ae6f481728c8
108 # ./.git/objects/45
109 ..
110
111
112 # ./.git/objects/88
113 88 type=dir mode=0775 nlink=2 size=4096 \
114 time=1454678174.768945501
115 2ba9732d762797f0f338de29bf79a8c0b7a679 \
116 size=30 time=1454678174.768945501 \
117 sha512digest=cc95f98c49e284de74626cc5da73ff54388a65fd796f532ae070c09635784ae2dac04537e1d2b1c73fae7be8ad96c96fab3ddddd943a7abc7b3ee8741fe56de5
118 # ./.git/objects/88
119 ..
120
121
122 # ./.git/objects/info
123 info type=dir mode=0775 nlink=2 size=4096 \
124 time=1454678171.609940899
125 # ./.git/objects/info
126 ..
127
128
129 # ./.git/objects/pack
130 pack type=dir mode=0775 nlink=2 size=4096 \
131 time=1454678171.609940899
132 # ./.git/objects/pack
133 ..
134
135 # ./.git/objects
136 ..
137
138
139 # ./.git/refs
140 /set type=file uid=1000 gid=1000 mode=0775 nlink=1 flags=none
141 refs type=dir nlink=4 size=4096 time=1454678171.595940879
142
143 # ./.git/refs/heads
144 heads type=dir nlink=2 size=4096 time=1454678171.595940879
145 # ./.git/refs/heads
146 ..
147
148
149 # ./.git/refs/tags
150 tags type=dir nlink=2 size=4096 time=1454678171.595940879
151 # ./.git/refs/tags
152 ..
153
154 # ./.git/refs
155 ..
156
157 # ./.git
158 ..
159
160
161 # ./cmd
162 cmd type=dir nlink=3 size=4096 time=1457644245.753596701
163
164 # ./cmd/gomtree
165 /set type=file uid=1000 gid=1000 mode=0664 nlink=1 flags=none
166 gomtree type=dir mode=0775 nlink=2 size=4096 \
167 time=1457644313.182697118
168 .main.go.swp \
169 mode=0644 size=12288 time=1457644334.048728193 \
170 sha512digest=ac1a05940c2033962a7ba6878a25aeeac84df3d41548b15c2b3ae283c8090012f3646821906b0edc4b716cf3a220536c862ea370ce235bffc7b7520f77df8903
171 main.go size=552 time=1457644313.182697118 \
172 sha512digest=519009a7756506e7aac81734b9701ac33aa0c22b1866abdc1fc1550e6c4e7164fa794d60f524116e7cfcdc9456f8637f504359749690f8199008263925350dad
173 main.go~ size=552 time=1457644313.171697102 \
174 sha512digest=519009a7756506e7aac81734b9701ac33aa0c22b1866abdc1fc1550e6c4e7164fa794d60f524116e7cfcdc9456f8637f504359749690f8199008263925350dad
175 # ./cmd/gomtree
176 ..
177
178 # ./cmd
179 ..
180
181
182 # ./testdata
183 testdata type=dir mode=0775 nlink=2 size=4096 \
184 time=1457644536.529039825
185 Downloads.mtree \
186 size=31406 time=1457644391.210813319 \
187 sha512digest=a31c61a8907078a82c1ad542ded518e10e862dcf9f561970a6174652d716c4e1ea745410c6a69d7382b13607e85426a190268955d4ecb657990c3fa456e39b46
188 source.mtree \
189 size=8192 time=1457644536.536039836 \
190 sha512digest=9748642c0ef6784611d155030067ccc5cdad5d51b4dc61b103da08377bd75aa3132756dcdb81c850e6555de5e845f26e22996b8bf1f76d39e71128585fbe6d17
191 # ./testdata
192 ..
193
194 ..
195
testdata/test.tar less more
Binary diff not shown
+0
-0
testdata/traversal/dir2/dir3/actualdir1/file1 less more
(New empty file)
+0
-0
testdata/traversal/dir2/dir3/actualdir1/file2 less more
(New empty file)
+1
-0
testdata/traversal/dir2/dir3/actualdir2/a less more
0 hello
+1
-0
testdata/traversal/dir2/dir3/actualdir2/b less more
0 I'm
+1
-0
testdata/traversal/dir2/dir3/actualdir2/c less more
0 programming
+0
-0
testdata/traversal/dir2/dir3/file less more
(New empty file)
+0
-0
testdata/traversal/dir2/dir3/file3 less more
(New empty file)
testdata/traversal.tar less more
Binary diff not shown
+154
-0
update.go less more
0 package mtree
1
2 import (
3 "container/heap"
4 "os"
5 "sort"
6
7 "github.com/Sirupsen/logrus"
8 )
9
10 // DefaultUpdateKeywords is the default set of keywords that can take updates to the files on disk
11 var DefaultUpdateKeywords = []Keyword{
12 "uid",
13 "gid",
14 "mode",
15 "xattr",
16 "link",
17 "time",
18 }
19
20 // Update attempts to set the attributes of root directory path, given the values of `keywords` in dh DirectoryHierarchy.
21 func Update(root string, dh *DirectoryHierarchy, keywords []Keyword, fs FsEval) ([]InodeDelta, error) {
22 creator := dhCreator{DH: dh}
23 curDir, err := os.Getwd()
24 if err == nil {
25 defer os.Chdir(curDir)
26 }
27
28 if err := os.Chdir(root); err != nil {
29 return nil, err
30 }
31 sort.Sort(byPos(creator.DH.Entries))
32
33 // This is for deferring the update of mtimes of directories, to unwind them
34 // in a most specific path first
35 h := &pathUpdateHeap{}
36 heap.Init(h)
37
38 results := []InodeDelta{}
39 for i, e := range creator.DH.Entries {
40 switch e.Type {
41 case SpecialType:
42 if e.Name == "/set" {
43 creator.curSet = &creator.DH.Entries[i]
44 } else if e.Name == "/unset" {
45 creator.curSet = nil
46 }
47 logrus.Debugf("%#v", e)
48 continue
49 case RelativeType, FullType:
50 e.Set = creator.curSet
51 pathname, err := e.Path()
52 if err != nil {
53 return nil, err
54 }
55
56 // filter the keywords to update on the file, from the keywords available for this entry:
57 var kvToUpdate []KeyVal
58 kvToUpdate = keyvalSelector(e.AllKeys(), keywords)
59 logrus.Debugf("kvToUpdate(%q): %#v", pathname, kvToUpdate)
60
61 for _, kv := range kvToUpdate {
62 if !InKeywordSlice(kv.Keyword().Prefix(), keywordPrefixes(keywords)) {
63 continue
64 }
65 logrus.Debugf("finding function for %q (%q)", kv.Keyword(), kv.Keyword().Prefix())
66 ukFunc, ok := UpdateKeywordFuncs[kv.Keyword().Prefix()]
67 if !ok {
68 logrus.Debugf("no UpdateKeywordFunc for %s; skipping", kv.Keyword())
69 continue
70 }
71
72 // TODO check for the type=dir of the entry as well
73 if kv.Keyword().Prefix() == "time" && e.IsDir() {
74 heap.Push(h, pathUpdate{
75 Path: pathname,
76 E: e,
77 KV: kv,
78 Func: ukFunc,
79 })
80
81 continue
82 }
83
84 if _, err := ukFunc(pathname, kv); err != nil {
85 results = append(results, InodeDelta{
86 diff: ErrorDifference,
87 path: pathname,
88 old: e,
89 keys: []KeyDelta{
90 {
91 diff: ErrorDifference,
92 name: kv.Keyword(),
93 err: err,
94 },
95 }})
96 }
97 // XXX really would be great to have a Check() or Compare() right here,
98 // to compare each entry as it is encountered, rather than just running
99 // Check() on this path after the whole update is finished.
100 }
101 }
102 }
103
104 for h.Len() > 0 {
105 pu := heap.Pop(h).(pathUpdate)
106 if _, err := pu.Func(pu.Path, pu.KV); err != nil {
107 results = append(results, InodeDelta{
108 diff: ErrorDifference,
109 path: pu.Path,
110 old: pu.E,
111 keys: []KeyDelta{
112 {
113 diff: ErrorDifference,
114 name: pu.KV.Keyword(),
115 err: err,
116 },
117 }})
118 }
119 }
120 return results, nil
121 }
122
123 type pathUpdateHeap []pathUpdate
124
125 func (h pathUpdateHeap) Len() int { return len(h) }
126 func (h pathUpdateHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
127
128 // This may end up looking backwards, but for container/heap, Less evaluates
129 // the negative priority. So when popping members of the array, it will be
130 // sorted by least. For this use-case, we want the most-qualified-name popped
131 // first (the longest path name), such that "." is the last entry popped.
132 func (h pathUpdateHeap) Less(i, j int) bool {
133 return len(h[i].Path) > len(h[j].Path)
134 }
135
136 func (h *pathUpdateHeap) Push(x interface{}) {
137 *h = append(*h, x.(pathUpdate))
138 }
139
140 func (h *pathUpdateHeap) Pop() interface{} {
141 old := *h
142 n := len(old)
143 x := old[n-1]
144 *h = old[0 : n-1]
145 return x
146 }
147
148 type pathUpdate struct {
149 Path string
150 E Entry
151 KV KeyVal
152 Func UpdateKeywordFunc
153 }
+94
-0
update_linux_test.go less more
0 package mtree
1
2 import (
3 "encoding/json"
4 "fmt"
5 "io/ioutil"
6 "os"
7 "path/filepath"
8 "testing"
9
10 "github.com/vbatts/go-mtree/xattr"
11 )
12
13 func init() {
14 //logrus.SetLevel(logrus.DebugLevel)
15 }
16
17 func TestXattrUpdate(t *testing.T) {
18 content := []byte("I know half of you half as well as I ought to")
19 // a bit dirty to create/destory a directory in cwd, but often /tmp is
20 // mounted tmpfs and doesn't support xattrs
21 dir, err := ioutil.TempDir(".", "test.xattr.restore.")
22 if err != nil {
23 t.Fatal(err)
24 }
25 defer os.RemoveAll(dir) // clean up
26
27 tmpfn := filepath.Join(dir, "tmpfile")
28 if err := ioutil.WriteFile(tmpfn, content, 0666); err != nil {
29 t.Fatal(err)
30 }
31
32 if err := xattr.Set(dir, "user.test", []byte("directory")); err != nil {
33 t.Skip(fmt.Sprintf("skipping: %q does not support xattrs", dir))
34 }
35 if err := xattr.Set(tmpfn, "user.test", []byte("regular file")); err != nil {
36 t.Fatal(err)
37 }
38
39 // Walk this tempdir
40 dh, err := Walk(dir, nil, append(DefaultKeywords, []Keyword{"xattr", "sha1"}...), nil)
41 if err != nil {
42 t.Fatal(err)
43 }
44
45 // Now check that we're sane
46 res, err := Check(dir, dh, nil, nil)
47 if err != nil {
48 t.Fatal(err)
49 }
50 if len(res) != 0 {
51 t.Errorf("expecting no failures, but got %q", res)
52 }
53
54 if err := xattr.Set(tmpfn, "user.test", []byte("let it fly")); err != nil {
55 t.Fatal(err)
56 }
57
58 // Now check that we fail the check
59 res, err = Check(dir, dh, nil, nil)
60 if err != nil {
61 t.Fatal(err)
62 }
63 if len(res) == 0 {
64 t.Error("expected failures (like xattrs), but got none")
65 }
66
67 // restore the xattrs to original
68 res, err = Update(dir, dh, append(DefaultUpdateKeywords, "xattr"), nil)
69 if err != nil {
70 t.Error(err)
71 }
72 if len(res) != 0 {
73 t.Errorf("expecting no failures, but got %q", res)
74 }
75
76 // Now check that we're sane again
77 res, err = Check(dir, dh, nil, nil)
78 if err != nil {
79 t.Fatal(err)
80 }
81 if len(res) != 0 {
82 // pretty this shit up
83 buf, err := json.MarshalIndent(res, "", " ")
84 if err != nil {
85 t.Errorf("expecting no failures, but got %q", res)
86 } else {
87 t.Errorf("expecting no failures, but got %s", string(buf))
88 }
89 }
90
91 // TODO make a test for xattr here. Likely in the user space for privileges. Even still this may be prone to error for some tmpfs don't act right with xattrs. :-\
92 // I'd hate to have to t.Skip() a test rather than fail alltogether.
93 }
+136
-0
update_test.go less more
0 // +build go1.7
1
2 package mtree
3
4 import (
5 "container/heap"
6 "encoding/json"
7 "io/ioutil"
8 "os"
9 "os/user"
10 "path/filepath"
11 "strconv"
12 "testing"
13 "time"
14
15 "github.com/Sirupsen/logrus"
16 )
17
18 func init() {
19 logrus.SetLevel(logrus.DebugLevel)
20 }
21
22 func TestUpdate(t *testing.T) {
23 content := []byte("I know half of you half as well as I ought to")
24 dir, err := ioutil.TempDir("", "test-check-keywords")
25 if err != nil {
26 t.Fatal(err)
27 }
28 defer os.RemoveAll(dir) // clean up
29
30 tmpfn := filepath.Join(dir, "tmpfile")
31 if err := ioutil.WriteFile(tmpfn, content, 0666); err != nil {
32 t.Fatal(err)
33 }
34
35 // Walk this tempdir
36 dh, err := Walk(dir, nil, append(DefaultKeywords, "sha1"), nil)
37 if err != nil {
38 t.Fatal(err)
39 }
40
41 // Touch a file, so the mtime changes.
42 now := time.Now()
43 if err := os.Chtimes(tmpfn, now, now); err != nil {
44 t.Fatal(err)
45 }
46 if err := os.Chmod(tmpfn, os.FileMode(0600)); err != nil {
47 t.Fatal(err)
48 }
49
50 // Changing user is a little tough, but the group can be changed by a limited user to any group that the user is a member of. So just choose one that is not the current main group.
51 u, err := user.Current()
52 if err != nil {
53 t.Fatal(err)
54 }
55 ugroups, err := u.GroupIds()
56 if err != nil {
57 t.Fatal(err)
58 }
59 for _, ugroup := range ugroups {
60 if ugroup == u.Gid {
61 continue
62 }
63 gid, err := strconv.Atoi(ugroup)
64 if err != nil {
65 t.Fatal(ugroup)
66 }
67 if err := os.Lchown(tmpfn, -1, gid); err != nil {
68 t.Fatal(err)
69 }
70 }
71
72 // Check for sanity. This ought to have failures
73 res, err := Check(dir, dh, nil, nil)
74 if err != nil {
75 t.Fatal(err)
76 }
77 if len(res) == 0 {
78 t.Error("expected failures (like mtimes), but got none")
79 }
80 //dh.WriteTo(os.Stdout)
81
82 res, err = Update(dir, dh, DefaultUpdateKeywords, nil)
83 if err != nil {
84 t.Error(err)
85 }
86 if len(res) > 0 {
87 // pretty this shit up
88 buf, err := json.MarshalIndent(res, "", " ")
89 if err != nil {
90 t.Errorf("%#v", res)
91 }
92 t.Error(string(buf))
93 }
94
95 // Now check that we're sane again
96 res, err = Check(dir, dh, nil, nil)
97 if err != nil {
98 t.Fatal(err)
99 }
100 // should have no failures now
101 if len(res) > 0 {
102 // pretty this shit up
103 buf, err := json.MarshalIndent(res, "", " ")
104 if err != nil {
105 t.Errorf("%#v", res)
106 } else {
107 t.Error(string(buf))
108 }
109 }
110
111 }
112
113 func TestPathUpdateHeap(t *testing.T) {
114 h := &pathUpdateHeap{
115 pathUpdate{Path: "not/the/longest"},
116 pathUpdate{Path: "almost/the/longest"},
117 pathUpdate{Path: "."},
118 pathUpdate{Path: "short"},
119 }
120 heap.Init(h)
121 v := "this/is/one/is/def/the/longest"
122 heap.Push(h, pathUpdate{Path: v})
123
124 longest := len(v)
125 var p string
126 for h.Len() > 0 {
127 p = heap.Pop(h).(pathUpdate).Path
128 if len(p) > longest {
129 t.Errorf("expected next path to be shorter, but it was not %q is longer than %d", p, longest)
130 }
131 }
132 if p != "." {
133 t.Errorf("expected \".\" to be the last, but got %q", p)
134 }
135 }
+201
-0
updatefuncs.go less more
0 package mtree
1
2 import (
3 "fmt"
4 "os"
5 "strconv"
6 "strings"
7 "time"
8
9 "github.com/Sirupsen/logrus"
10 "github.com/vbatts/go-mtree/pkg/govis"
11 )
12
13 // UpdateKeywordFunc is the signature for a function that will restore a file's
14 // attributes. Where path is relative path to the file, and value to be
15 // restored to.
16 type UpdateKeywordFunc func(path string, kv KeyVal) (os.FileInfo, error)
17
18 // UpdateKeywordFuncs is the registered list of functions to update file attributes.
19 // Keyed by the keyword as it would show up in the manifest
20 var UpdateKeywordFuncs = map[Keyword]UpdateKeywordFunc{
21 "mode": modeUpdateKeywordFunc,
22 "time": timeUpdateKeywordFunc,
23 "tar_time": tartimeUpdateKeywordFunc,
24 "uid": uidUpdateKeywordFunc,
25 "gid": gidUpdateKeywordFunc,
26 "xattr": xattrUpdateKeywordFunc,
27 "link": linkUpdateKeywordFunc,
28 }
29
30 func uidUpdateKeywordFunc(path string, kv KeyVal) (os.FileInfo, error) {
31 uid, err := strconv.Atoi(kv.Value())
32 if err != nil {
33 return nil, err
34 }
35
36 stat, err := os.Lstat(path)
37 if err != nil {
38 return nil, err
39 }
40 if statIsUID(stat, uid) {
41 return stat, nil
42 }
43
44 if err := os.Lchown(path, uid, -1); err != nil {
45 return nil, err
46 }
47 return os.Lstat(path)
48 }
49
50 func gidUpdateKeywordFunc(path string, kv KeyVal) (os.FileInfo, error) {
51 gid, err := strconv.Atoi(kv.Value())
52 if err != nil {
53 return nil, err
54 }
55
56 stat, err := os.Lstat(path)
57 if err != nil {
58 return nil, err
59 }
60 if statIsGID(stat, gid) {
61 return stat, nil
62 }
63
64 if err := os.Lchown(path, -1, gid); err != nil {
65 return nil, err
66 }
67 return os.Lstat(path)
68 }
69
70 func modeUpdateKeywordFunc(path string, kv KeyVal) (os.FileInfo, error) {
71 info, err := os.Lstat(path)
72 if err != nil {
73 return nil, err
74 }
75
76 // don't set mode on symlinks, as it passes through to the backing file
77 if info.Mode()&os.ModeSymlink != 0 {
78 return info, nil
79 }
80 vmode, err := strconv.ParseInt(kv.Value(), 8, 32)
81 if err != nil {
82 return nil, err
83 }
84
85 stat, err := os.Lstat(path)
86 if err != nil {
87 return nil, err
88 }
89 if stat.Mode() == os.FileMode(vmode) {
90 return stat, nil
91 }
92
93 logrus.Debugf("path: %q, kv.Value(): %q, vmode: %o", path, kv.Value(), vmode)
94 if err := os.Chmod(path, os.FileMode(vmode)); err != nil {
95 return nil, err
96 }
97 return os.Lstat(path)
98 }
99
100 // since tar_time will only be second level precision, then when restoring the
101 // filepath from a tar_time, then compare the seconds first and only Chtimes if
102 // the seconds value is different.
103 func tartimeUpdateKeywordFunc(path string, kv KeyVal) (os.FileInfo, error) {
104 info, err := os.Lstat(path)
105 if err != nil {
106 return nil, err
107 }
108
109 v := strings.SplitN(kv.Value(), ".", 2)
110 if len(v) != 2 {
111 return nil, fmt.Errorf("expected a number like 1469104727.000000000")
112 }
113 sec, err := strconv.ParseInt(v[0], 10, 64)
114 if err != nil {
115 return nil, fmt.Errorf("expected seconds, but got %q", v[0])
116 }
117
118 // if the seconds are the same, don't do anything, because the file might
119 // have nanosecond value, and if using tar_time it would zero it out.
120 if info.ModTime().Unix() == sec {
121 return info, nil
122 }
123
124 vtime := time.Unix(sec, 0)
125
126 // if times are same then don't modify anything
127 // comparing Unix, since it does not include Nano seconds
128 if info.ModTime().Unix() == vtime.Unix() {
129 return info, nil
130 }
131
132 // symlinks are strange and most of the time passes through to the backing file
133 if info.Mode()&os.ModeSymlink != 0 {
134 if err := lchtimes(path, vtime, vtime); err != nil {
135 return nil, err
136 }
137 } else if err := os.Chtimes(path, vtime, vtime); err != nil {
138 return nil, err
139 }
140 return os.Lstat(path)
141 }
142
143 // this is nano second precision
144 func timeUpdateKeywordFunc(path string, kv KeyVal) (os.FileInfo, error) {
145 info, err := os.Lstat(path)
146 if err != nil {
147 return nil, err
148 }
149
150 v := strings.SplitN(kv.Value(), ".", 2)
151 if len(v) != 2 {
152 return nil, fmt.Errorf("expected a number like 1469104727.871937272")
153 }
154 nsec, err := strconv.ParseInt(v[0]+v[1], 10, 64)
155 if err != nil {
156 return nil, fmt.Errorf("expected nano seconds, but got %q", v[0]+v[1])
157 }
158 logrus.Debugf("arg: %q; nsec: %d", v[0]+v[1], nsec)
159
160 vtime := time.Unix(0, nsec)
161
162 // if times are same then don't modify anything
163 if info.ModTime().Equal(vtime) {
164 return info, nil
165 }
166
167 // symlinks are strange and most of the time passes through to the backing file
168 if info.Mode()&os.ModeSymlink != 0 {
169 if err := lchtimes(path, vtime, vtime); err != nil {
170 return nil, err
171 }
172 } else if err := os.Chtimes(path, vtime, vtime); err != nil {
173 return nil, err
174 }
175 return os.Lstat(path)
176 }
177
178 func linkUpdateKeywordFunc(path string, kv KeyVal) (os.FileInfo, error) {
179 linkname, err := govis.Unvis(kv.Value(), DefaultVisFlags)
180 if err != nil {
181 return nil, err
182 }
183 got, err := os.Readlink(path)
184 if err != nil {
185 return nil, err
186 }
187 if got == linkname {
188 return os.Lstat(path)
189 }
190
191 logrus.Debugf("linkUpdateKeywordFunc: removing %q to link to %q", path, linkname)
192 if err := os.Remove(path); err != nil {
193 return nil, err
194 }
195 if err := os.Symlink(linkname, path); err != nil {
196 return nil, err
197 }
198
199 return os.Lstat(path)
200 }
+21
-0
updatefuncs_linux.go less more
0 // +build linux
1
2 package mtree
3
4 import (
5 "encoding/base64"
6 "os"
7
8 "github.com/vbatts/go-mtree/xattr"
9 )
10
11 func xattrUpdateKeywordFunc(path string, kv KeyVal) (os.FileInfo, error) {
12 buf, err := base64.StdEncoding.DecodeString(kv.Value())
13 if err != nil {
14 return nil, err
15 }
16 if err := xattr.Set(path, kv.Keyword().Suffix(), buf); err != nil {
17 return nil, err
18 }
19 return os.Lstat(path)
20 }
+11
-0
updatefuncs_unsupported.go less more
0 // +build !linux
1
2 package mtree
3
4 import (
5 "os"
6 )
7
8 func xattrUpdateKeywordFunc(path string, kv KeyVal) (os.FileInfo, error) {
9 return os.Lstat(path)
10 }
+23
-0
version.go less more
0 package mtree
1
2 import "fmt"
3
4 const (
5 // AppName is the name ... of this library/application
6 AppName = "gomtree"
7 )
8
9 const (
10 // VersionMajor is for an API incompatible changes
11 VersionMajor = 0
12 // VersionMinor is for functionality in a backwards-compatible manner
13 VersionMinor = 4
14 // VersionPatch is for backwards-compatible bug fixes
15 VersionPatch = 2
16
17 // VersionDev indicates development branch. Releases will be empty string.
18 VersionDev = ""
19 )
20
21 // Version is the specification version that the package types support.
22 var Version = fmt.Sprintf("%d.%d.%d%s", VersionMajor, VersionMinor, VersionPatch, VersionDev)
+385
-0
walk.go less more
0 package mtree
1
2 import (
3 "fmt"
4 "io"
5 "os"
6 "os/user"
7 "path/filepath"
8 "sort"
9 "strings"
10 "time"
11
12 "github.com/vbatts/go-mtree/pkg/govis"
13 )
14
15 // ExcludeFunc is the type of function called on each path walked to determine
16 // whether to be excluded from the assembled DirectoryHierarchy. If the func
17 // returns true, then the path is not included in the spec.
18 type ExcludeFunc func(path string, info os.FileInfo) bool
19
20 // ExcludeNonDirectories is an ExcludeFunc for excluding all paths that are not directories
21 var ExcludeNonDirectories = func(path string, info os.FileInfo) bool {
22 return !info.IsDir()
23 }
24
25 var defaultSetKeyVals = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=0664"}
26
27 // Walk from root directory and assemble the DirectoryHierarchy
28 // * `excludes` provided are used to skip paths
29 // * `keywords` are the set to collect from the walked paths. The recommended default list is DefaultKeywords.
30 // * `fsEval` is the interface to use in evaluating files. If `nil`, then DefaultFsEval is used.
31 func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval) (*DirectoryHierarchy, error) {
32 if fsEval == nil {
33 fsEval = DefaultFsEval{}
34 }
35 creator := dhCreator{DH: &DirectoryHierarchy{}, fs: fsEval}
36 // insert signature and metadata comments first (user, machine, tree, date)
37 for _, e := range signatureEntries(root) {
38 e.Pos = len(creator.DH.Entries)
39 creator.DH.Entries = append(creator.DH.Entries, e)
40 }
41 // insert keyword metadata next
42 for _, e := range keywordEntries(keywords) {
43 e.Pos = len(creator.DH.Entries)
44 creator.DH.Entries = append(creator.DH.Entries, e)
45 }
46 // walk the directory and add entries
47 err := startWalk(&creator, root, func(path string, info os.FileInfo, err error) error {
48 if err != nil {
49 return err
50 }
51 for _, ex := range excludes {
52 if ex(path, info) {
53 return nil
54 }
55 }
56
57 entryPathName := filepath.Base(path)
58 if info.IsDir() {
59 creator.DH.Entries = append(creator.DH.Entries, Entry{
60 Type: BlankType,
61 Pos: len(creator.DH.Entries),
62 })
63
64 // Insert a comment of the full path of the directory's name
65 if creator.curDir != nil {
66 dirname, err := creator.curDir.Path()
67 if err != nil {
68 return err
69 }
70 creator.DH.Entries = append(creator.DH.Entries, Entry{
71 Pos: len(creator.DH.Entries),
72 Raw: "# " + filepath.Join(dirname, entryPathName),
73 Type: CommentType,
74 })
75 } else {
76 entryPathName = "."
77 creator.DH.Entries = append(creator.DH.Entries, Entry{
78 Pos: len(creator.DH.Entries),
79 Raw: "# .",
80 Type: CommentType,
81 })
82 }
83
84 // set the initial /set keywords
85 if creator.curSet == nil {
86 e := Entry{
87 Name: "/set",
88 Type: SpecialType,
89 Pos: len(creator.DH.Entries),
90 Keywords: keyvalSelector(defaultSetKeyVals, keywords),
91 }
92 for _, keyword := range SetKeywords {
93 err := func() error {
94 var r io.Reader
95 if info.Mode().IsRegular() {
96 fh, err := creator.fs.Open(path)
97 if err != nil {
98 return err
99 }
100 defer fh.Close()
101 r = fh
102 }
103 keyFunc, ok := KeywordFuncs[keyword.Prefix()]
104 if !ok {
105 return fmt.Errorf("Unknown keyword %q for file %q", keyword.Prefix(), path)
106 }
107 kvs, err := creator.fs.KeywordFunc(keyFunc)(path, info, r)
108 if err != nil {
109 return err
110 }
111 for _, kv := range kvs {
112 if kv != "" {
113 e.Keywords = append(e.Keywords, kv)
114 }
115 }
116 return nil
117 }()
118 if err != nil {
119 return err
120 }
121 }
122 creator.curSet = &e
123 creator.DH.Entries = append(creator.DH.Entries, e)
124 } else if creator.curSet != nil {
125 // check the attributes of the /set keywords and re-set if changed
126 klist := []KeyVal{}
127 for _, keyword := range SetKeywords {
128 err := func() error {
129 var r io.Reader
130 if info.Mode().IsRegular() {
131 fh, err := creator.fs.Open(path)
132 if err != nil {
133 return err
134 }
135 defer fh.Close()
136 r = fh
137 }
138 keyFunc, ok := KeywordFuncs[keyword.Prefix()]
139 if !ok {
140 return fmt.Errorf("Unknown keyword %q for file %q", keyword.Prefix(), path)
141 }
142 kvs, err := creator.fs.KeywordFunc(keyFunc)(path, info, r)
143 if err != nil {
144 return err
145 }
146 for _, kv := range kvs {
147 if kv != "" {
148 klist = append(klist, kv)
149 }
150 }
151 return nil
152 }()
153 if err != nil {
154 return err
155 }
156 }
157
158 needNewSet := false
159 for _, k := range klist {
160 if !inKeyValSlice(k, creator.curSet.Keywords) {
161 needNewSet = true
162 }
163 }
164 if needNewSet {
165 e := Entry{
166 Name: "/set",
167 Type: SpecialType,
168 Pos: len(creator.DH.Entries),
169 Keywords: keyvalSelector(append(defaultSetKeyVals, klist...), keywords),
170 }
171 creator.curSet = &e
172 creator.DH.Entries = append(creator.DH.Entries, e)
173 }
174 }
175 }
176 encodedEntryName, err := govis.Vis(entryPathName, DefaultVisFlags)
177 if err != nil {
178 return err
179 }
180 e := Entry{
181 Name: encodedEntryName,
182 Pos: len(creator.DH.Entries),
183 Type: RelativeType,
184 Set: creator.curSet,
185 Parent: creator.curDir,
186 }
187 for _, keyword := range keywords {
188 err := func() error {
189 var r io.Reader
190 if info.Mode().IsRegular() {
191 fh, err := creator.fs.Open(path)
192 if err != nil {
193 return err
194 }
195 defer fh.Close()
196 r = fh
197 }
198 keyFunc, ok := KeywordFuncs[keyword.Prefix()]
199 if !ok {
200 return fmt.Errorf("Unknown keyword %q for file %q", keyword.Prefix(), path)
201 }
202 kvs, err := creator.fs.KeywordFunc(keyFunc)(path, info, r)
203 if err != nil {
204 return err
205 }
206 for _, kv := range kvs {
207 if kv != "" && !inKeyValSlice(kv, creator.curSet.Keywords) {
208 e.Keywords = append(e.Keywords, kv)
209 }
210 }
211 return nil
212 }()
213 if err != nil {
214 return err
215 }
216 }
217 if info.IsDir() {
218 if creator.curDir != nil {
219 creator.curDir.Next = &e
220 }
221 e.Prev = creator.curDir
222 creator.curDir = &e
223 } else {
224 if creator.curEnt != nil {
225 creator.curEnt.Next = &e
226 }
227 e.Prev = creator.curEnt
228 creator.curEnt = &e
229 }
230 creator.DH.Entries = append(creator.DH.Entries, e)
231 return nil
232 })
233 return creator.DH, err
234 }
235
236 // startWalk walks the file tree rooted at root, calling walkFn for each file or
237 // directory in the tree, including root. All errors that arise visiting files
238 // and directories are filtered by walkFn. The files are walked in lexical
239 // order, which makes the output deterministic but means that for very
240 // large directories Walk can be inefficient.
241 // Walk does not follow symbolic links.
242 func startWalk(c *dhCreator, root string, walkFn filepath.WalkFunc) error {
243 info, err := c.fs.Lstat(root)
244 if err != nil {
245 return walkFn(root, nil, err)
246 }
247 return walk(c, root, info, walkFn)
248 }
249
250 // walk recursively descends path, calling w.
251 func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc) error {
252 err := walkFn(path, info, nil)
253 if err != nil {
254 if info.IsDir() && err == filepath.SkipDir {
255 return nil
256 }
257 return err
258 }
259
260 if !info.IsDir() {
261 return nil
262 }
263
264 names, err := readOrderedDirNames(c, path)
265 if err != nil {
266 return walkFn(path, info, err)
267 }
268
269 for _, name := range names {
270 filename := filepath.Join(path, name)
271 fileInfo, err := c.fs.Lstat(filename)
272 if err != nil {
273 if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir {
274 return err
275 }
276 } else {
277 err = walk(c, filename, fileInfo, walkFn)
278 if err != nil {
279 if !fileInfo.IsDir() || err != filepath.SkipDir {
280 return err
281 }
282 }
283 }
284 }
285 c.DH.Entries = append(c.DH.Entries, Entry{
286 Name: "..",
287 Type: DotDotType,
288 Pos: len(c.DH.Entries),
289 })
290 if c.curDir != nil {
291 c.curDir = c.curDir.Parent
292 }
293 return nil
294 }
295
296 // readOrderedDirNames reads the directory and returns a sorted list of all
297 // entries with non-directories first, followed by directories.
298 func readOrderedDirNames(c *dhCreator, dirname string) ([]string, error) {
299 infos, err := c.fs.Readdir(dirname)
300 if err != nil {
301 return nil, err
302 }
303
304 names := []string{}
305 dirnames := []string{}
306 for _, info := range infos {
307 if info.IsDir() {
308 dirnames = append(dirnames, info.Name())
309 continue
310 }
311 names = append(names, info.Name())
312 }
313 sort.Strings(names)
314 sort.Strings(dirnames)
315 return append(names, dirnames...), nil
316 }
317
318 // signatureEntries is a simple helper function that returns a slice of Entry's
319 // that describe the metadata signature about the host. Items like date, user,
320 // machine, and tree (which is specified by argument `root`), are considered.
321 // These Entry's construct comments in the mtree specification, so if there is
322 // an error trying to obtain a particular metadata, we simply don't construct
323 // the Entry.
324 func signatureEntries(root string) []Entry {
325 var sigEntries []Entry
326 user, err := user.Current()
327 if err == nil {
328 userEntry := Entry{
329 Type: CommentType,
330 Raw: fmt.Sprintf("#%16s%s", "user: ", user.Username),
331 }
332 sigEntries = append(sigEntries, userEntry)
333 }
334
335 hostname, err := os.Hostname()
336 if err == nil {
337 hostEntry := Entry{
338 Type: CommentType,
339 Raw: fmt.Sprintf("#%16s%s", "machine: ", hostname),
340 }
341 sigEntries = append(sigEntries, hostEntry)
342 }
343
344 if tree := filepath.Clean(root); tree == "." || tree == ".." {
345 root, err := os.Getwd()
346 if err == nil {
347 // use parent directory of current directory
348 if tree == ".." {
349 root = filepath.Dir(root)
350 }
351 treeEntry := Entry{
352 Type: CommentType,
353 Raw: fmt.Sprintf("#%16s%s", "tree: ", filepath.Clean(root)),
354 }
355 sigEntries = append(sigEntries, treeEntry)
356 }
357 } else {
358 treeEntry := Entry{
359 Type: CommentType,
360 Raw: fmt.Sprintf("#%16s%s", "tree: ", filepath.Clean(root)),
361 }
362 sigEntries = append(sigEntries, treeEntry)
363 }
364
365 dateEntry := Entry{
366 Type: CommentType,
367 Raw: fmt.Sprintf("#%16s%s", "date: ", time.Now().Format("Mon Jan 2 15:04:05 2006")),
368 }
369 sigEntries = append(sigEntries, dateEntry)
370
371 return sigEntries
372 }
373
374 // keywordEntries returns a slice of entries including a comment of the
375 // keywords requested when generating this manifest.
376 func keywordEntries(keywords []Keyword) []Entry {
377 // Convert all of the keywords to zero-value keyvals.
378 return []Entry{
379 {
380 Type: CommentType,
381 Raw: fmt.Sprintf("#%16s%s", "keywords: ", strings.Join(FromKeywords(keywords), ",")),
382 },
383 }
384 }
+54
-0
walk_test.go less more
0 package mtree
1
2 import (
3 "io/ioutil"
4 "os"
5 "testing"
6 )
7
8 func TestWalk(t *testing.T) {
9 dh, err := Walk(".", nil, append(DefaultKeywords, "sha1"), nil)
10 if err != nil {
11 t.Fatal(err)
12 }
13 numEntries = countTypes(dh)
14
15 fh, err := ioutil.TempFile("", "walk.")
16 if err != nil {
17 t.Fatal(err)
18 }
19 defer os.Remove(fh.Name())
20 defer fh.Close()
21
22 if _, err = dh.WriteTo(fh); err != nil {
23 t.Fatal(err)
24 }
25 if _, err := fh.Seek(0, 0); err != nil {
26 t.Fatal(err)
27 }
28 if dh, err = ParseSpec(fh); err != nil {
29 t.Fatal(err)
30 }
31 for k, v := range countTypes(dh) {
32 if numEntries[k] != v {
33 t.Errorf("for type %s: expected %d, got %d", k, numEntries[k], v)
34 }
35 }
36 }
37
38 func TestWalkDirectory(t *testing.T) {
39 dh, err := Walk(".", []ExcludeFunc{ExcludeNonDirectories}, []Keyword{"type"}, nil)
40 if err != nil {
41 t.Fatal(err)
42 }
43
44 for i := range dh.Entries {
45 for _, keyval := range dh.Entries[i].AllKeys() {
46 if dh.Entries[i].Type == FullType || dh.Entries[i].Type == RelativeType {
47 if keyval.Keyword() == "type" && keyval.Value() != "dir" {
48 t.Errorf("expected only directories, but %q is a %q", dh.Entries[i].Name, keyval.Value())
49 }
50 }
51 }
52 }
53 }
+42
-0
xattr/xattr.go less more
0 // +build linux
1
2 package xattr
3
4 import (
5 "strings"
6 "syscall"
7 )
8
9 // Get returns the extended attributes (xattr) on file `path`, for the given `name`.
10 func Get(path, name string) ([]byte, error) {
11 dest := make([]byte, 1024)
12 i, err := syscall.Getxattr(path, name, dest)
13 if err != nil {
14 return nil, err
15 }
16 return dest[:i], nil
17 }
18
19 // Set sets the extended attributes (xattr) on file `path`, for the given `name` and `value`
20 func Set(path, name string, value []byte) error {
21 return syscall.Setxattr(path, name, value, 0)
22 }
23
24 // List returns a list of all the extended attributes (xattr) for file `path`
25 func List(path string) ([]string, error) {
26 dest := make([]byte, 1024)
27 i, err := syscall.Listxattr(path, dest)
28 if err != nil {
29 return nil, err
30 }
31
32 // If the returned list is empty, return nil instead of []string{""}
33 str := string(dest[:i])
34 if str == "" {
35 return nil, nil
36 }
37
38 return strings.Split(strings.TrimRight(str, nilByte), nilByte), nil
39 }
40
41 const nilByte = "\x00"
+44
-0
xattr/xattr_test.go less more
0 // +build linux
1
2 package xattr
3
4 import (
5 "bytes"
6 "io/ioutil"
7 "os"
8 "testing"
9 )
10
11 func TestXattr(t *testing.T) {
12 testDir, present := os.LookupEnv("MTREE_TESTDIR")
13 if present == false {
14 testDir = "."
15 }
16 fh, err := ioutil.TempFile(testDir, "xattr.")
17 if err != nil {
18 t.Fatal(err)
19 }
20 defer os.Remove(fh.Name())
21 if err := fh.Close(); err != nil {
22 t.Fatal(err)
23 }
24
25 expected := []byte("1234")
26 if err := Set(fh.Name(), "user.testing", expected); err != nil {
27 t.Fatal(fh.Name(), err)
28 }
29 l, err := List(fh.Name())
30 if err != nil {
31 t.Error(fh.Name(), err)
32 }
33 if !(len(l) > 0) {
34 t.Errorf("%q: expected a list of at least 1; got %d", fh.Name(), len(l))
35 }
36 got, err := Get(fh.Name(), "user.testing")
37 if err != nil {
38 t.Fatal(fh.Name(), err)
39 }
40 if !bytes.Equal(got, expected) {
41 t.Errorf("%q: expected %q; got %q", fh.Name(), expected, got)
42 }
43 }
+21
-0
xattr/xattr_unsupported.go less more
0 // +build !linux
1
2 package xattr
3
4 // Get would return the extended attributes, but this unsupported feature
5 // returns nil, nil
6 func Get(path, name string) ([]byte, error) {
7 return nil, nil
8 }
9
10 // Set would set the extended attributes, but this unsupported feature returns
11 // nil
12 func Set(path, name string, value []byte) error {
13 return nil
14 }
15
16 // List would return the keys of extended attributes, but this unsupported
17 // feature returns nil, nil
18 func List(path string) ([]string, error) {
19 return nil, nil
20 }
+44
-0
xattr/xattr_unsupported_test.go less more
0 // +build !linux
1
2 package xattr
3
4 import (
5 "bytes"
6 "io/ioutil"
7 "os"
8 "testing"
9 )
10
11 func TestXattrUnsupported(t *testing.T) {
12 fh, err := ioutil.TempFile(".", "xattr.")
13 if err != nil {
14 t.Fatal(err)
15 }
16 defer os.Remove(fh.Name())
17 if err := fh.Close(); err != nil {
18 t.Fatal(err)
19 }
20
21 // because xattrs are "not supported" on this platform, they're like a black
22 // box.
23 write := []byte("1234")
24 expected := []byte("")
25
26 if err := Set(fh.Name(), "user.testing", write); err != nil {
27 t.Fatal(fh.Name(), err)
28 }
29 l, err := List(fh.Name())
30 if err != nil {
31 t.Error(fh.Name(), err)
32 }
33 if len(l) > 0 {
34 t.Errorf("%q: expected a list of at least 0; got %d", fh.Name(), len(l))
35 }
36 got, err := Get(fh.Name(), "user.testing")
37 if err != nil {
38 t.Fatal(fh.Name(), err)
39 }
40 if !bytes.Equal(got, expected) {
41 t.Errorf("%q: expected %q; got %q", fh.Name(), expected, got)
42 }
43 }