Codebase list golang-github-saracen-walker / 43e87f1
Merge pull request #2 from saracen/error-callback Added WithErrorCallback option Arran Walker authored 4 years ago GitHub committed 4 years ago
4 changed file(s) with 90 addition(s) and 21 deletion(s). Raw diff Collapse all Expand all
+17
-3
README.md less more
44 `walker` is a faster, parallel version, of `filepath.Walk`.
55
66 ```
7 walker.Walk("/tmp", func(pathname string, fi os.FileInfo) error {
7 // walk function called for every path found
8 walkFn := func(pathname string, fi os.FileInfo) error {
89 fmt.Printf("%s: %d bytes\n", pathname, fi.Size())
10 return nil
11 }
12
13 // error function called for every error encountered
14 errorCallbackOption := walker.WithErrorCallback(func(pathname string, err error) error {
15 // ignore permissione errors
16 if os.IsPermission(err) {
17 return nil
18 }
19 // halt traversal on any other error
20 return err
921 })
22
23 walker.Walk("/tmp", walkFn, errorCallbackOption)
1024 ```
1125
1226 ## Benchmarks
1327
1428 - Standard library (`filepath.Walk`) is `FilepathWalk`.
1529 - This library is `WalkerWalk`
16 - `FastwalkWalk` is [https://github.com/golang/tools/tree/master/internal/fastwalk](fastwalk).
17 - `GodirwalkWalk` is [https://github.com/karrick/godirwalk](godirwalk).
30 - `FastwalkWalk` is [fastwalk](https://github.com/golang/tools/tree/master/internal/fastwalk).
31 - `GodirwalkWalk` is [godirwalk](https://github.com/karrick/godirwalk).
1832
1933 This library and `filepath.Walk` both perform `os.Lstat` calls and provide a full `os.FileInfo` structure to the callback. `BenchmarkFastwalkWalkLstat` and `BenchmarkGodirwalkWalkLstat` include this stat call for better comparison with `BenchmarkFilepathWalk` and `BenchmarkWalkerWalk`.
2034
+20
-7
walker.go less more
1010 )
1111
1212 // Walk wraps WalkWithContext using the background context.
13 func Walk(root string, walkFn func(pathname string, fi os.FileInfo) error) error {
14 return WalkWithContext(context.Background(), root, walkFn)
13 func Walk(root string, walkFn func(pathname string, fi os.FileInfo) error, opts ...WalkerOption) error {
14 return WalkWithContext(context.Background(), root, walkFn, opts...)
1515 }
1616
1717 // WalkWithContext walks the file tree rooted at root, calling walkFn for each
2121 //
2222 // Multiple goroutines stat the filesystem concurrently. The provided
2323 // walkFn must be safe for concurrent use.
24 func WalkWithContext(ctx context.Context, root string, walkFn func(pathname string, fi os.FileInfo) error) error {
24 func WalkWithContext(ctx context.Context, root string, walkFn func(pathname string, fi os.FileInfo) error, opts ...WalkerOption) error {
2525 wg, ctx := errgroup.WithContext(ctx)
2626
2727 fi, err := os.Lstat(root)
4646 w.limit = 4
4747 }
4848
49 for _, o := range opts {
50 err := o(&w.options)
51 if err != nil {
52 return err
53 }
54 }
55
4956 w.wg.Go(func() error {
5057 return w.gowalk(root)
5158 })
5966 ctx context.Context
6067 wg *errgroup.Group
6168 fn func(pathname string, fi os.FileInfo) error
69 options walkerOptions
6270 }
6371
6472 func (w *walker) walk(dirname string, fi os.FileInfo) error {
100108 }
101109
102110 // if we've reached our limit, continue with this goroutine
103 return w.readdir(pathname)
111 err = w.readdir(pathname)
112 if err != nil && w.options.errorCallback != nil {
113 err = w.options.errorCallback(pathname, err)
114 }
115 return err
104116 }
105117
106118 func (w *walker) gowalk(pathname string) error {
107 if err := w.readdir(pathname); err != nil {
108 return err
119 err := w.readdir(pathname)
120 if err != nil && w.options.errorCallback != nil {
121 err = w.options.errorCallback(pathname, err)
109122 }
110123
111124 atomic.AddUint32(&w.counter, ^uint32(0))
112 return nil
125 return err
113126 }
+18
-0
walker_option.go less more
0 package walker
1
2 // WalkerOption is an option to configure Walk() behaviour.
3 type WalkerOption func(*walkerOptions) error
4
5 type walkerOptions struct {
6 errorCallback func(pathname string, err error) error
7 }
8
9 // WithErrorCallback sets a callback to be used for error handling. Any error
10 // returned will halt the Walk function and return the error. If the callback
11 // returns nil Walk will continue.
12 func WithErrorCallback(callback func(pathname string, err error) error) WalkerOption {
13 return func(o *walkerOptions) error {
14 o.errorCallback = callback
15 return nil
16 }
17 }
+35
-11
walker_test.go less more
2626
2727 for path, mode := range files {
2828 path = filepath.Join(dir, path)
29 if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
29 err := os.MkdirAll(filepath.Dir(path), 0777)
30 if err != nil {
3031 t.Fatal(err)
3132 }
3233
3334 switch {
3435 case mode&os.ModeSymlink != 0 && mode&os.ModeDir != 0:
35 if err := os.Symlink(filepath.Dir(path), path); err != nil {
36 t.Fatal(err)
37 }
36 err = os.Symlink(filepath.Dir(path), path)
3837
3938 case mode&os.ModeSymlink != 0:
40 if err := os.Symlink("foo/foo.go", path); err != nil {
41 t.Fatal(err)
42 }
39 err = os.Symlink("foo/foo.go", path)
40
41 case mode&os.ModeDir != 0:
42 err = os.Mkdir(path, mode)
4343
4444 default:
45 if err := ioutil.WriteFile(path, []byte(path), mode); err != nil {
46 t.Fatal(err)
47 }
45 err = ioutil.WriteFile(path, []byte(path), mode)
46 }
47 if err != nil {
48 t.Fatal(err)
4849 }
4950 }
5051
5253 err = filepath.Walk(dir, func(pathname string, fi os.FileInfo, err error) error {
5354 if strings.Contains(pathname, "skip") {
5455 return filepath.SkipDir
56 }
57
58 if filepath.Base(pathname) == "perm-error" {
59 if err == nil {
60 t.Errorf("expected permission error for path %v", pathname)
61 }
62 } else {
63 if err != nil {
64 t.Errorf("unexpected error for path %v", pathname)
65 }
5566 }
5667
5768 filepathResults[pathname] = fi
7384 l.Unlock()
7485
7586 return nil
76 })
87 }, walker.WithErrorCallback(func(pathname string, err error) error {
88 if filepath.Base(pathname) == "perm-error" {
89 if err == nil {
90 t.Errorf("expected permission error for path %v", pathname)
91 }
92 } else {
93 if err != nil {
94 t.Errorf("unexpected error for path %v", pathname)
95 }
96 }
97 return nil
98 }))
99
77100 if err != nil {
78101 t.Fatal(err)
79102 }
102125 "skip/file": 0700,
103126 "bar/symlink": os.ModeDir | os.ModeSymlink | 0777,
104127 "bar/symlink.go": os.ModeSymlink | 0777,
128 "perm-error": os.ModeDir | 0000,
105129 })
106130 }
107131