Codebase list golang-github-boltdb-bolt / upstream/1.2.1
Imported Upstream version 1.2.1 aviau 7 years ago
7 changed file(s) with 203 addition(s) and 75 deletion(s). Raw diff Collapse all Expand all
+9
-3
README.md less more
0 Bolt [![Build Status](https://drone.io/github.com/boltdb/bolt/status.png)](https://drone.io/github.com/boltdb/bolt/latest) [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.0-green.svg)
0 Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.0-green.svg)
11 ====
22
33 Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas]
426426 })
427427 ```
428428
429 Note that, while RFC3339 is sortable, the Golang implementation of RFC3339Nano does not use a fixed number of digits after the decimal point and is therefore not sortable.
430
429431
430432 #### ForEach()
431433
436438 db.View(func(tx *bolt.Tx) error {
437439 // Assume bucket exists and has keys
438440 b := tx.Bucket([]byte("MyBucket"))
439
441
440442 b.ForEach(func(k, v []byte) error {
441443 fmt.Printf("key=%s, value=%s\n", k, v)
442444 return nil
616618 {
617619 NSURL* URL= [NSURL fileURLWithPath: filePathString];
618620 assert([[NSFileManager defaultManager] fileExistsAtPath: [URL path]]);
619
621
620622 NSError *error = nil;
621623 BOOL success = [URL setResourceValue: [NSNumber numberWithBool: YES]
622624 forKey: NSURLIsExcludedFromBackupKey error: &error];
839841 * [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service.
840842 * [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners.
841843 * [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores.
844 * [Storm](https://github.com/asdine/storm) - A simple ORM around BoltDB.
845 * [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB.
846 * [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings.
847 * [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend.
842848
843849 If you are using Bolt in a project please send a pull request to add it to the list.
+40
-0
bucket_test.go less more
7373 if tx.Bucket([]byte("widgets")).Get([]byte("foo")) != nil {
7474 t.Fatal("expected nil value")
7575 }
76 return nil
77 }); err != nil {
78 t.Fatal(err)
79 }
80 }
81
82 // Ensure that a slice returned from a bucket has a capacity equal to its length.
83 // This also allows slices to be appended to since it will require a realloc by Go.
84 //
85 // https://github.com/boltdb/bolt/issues/544
86 func TestBucket_Get_Capacity(t *testing.T) {
87 db := MustOpenDB()
88 defer db.MustClose()
89
90 // Write key to a bucket.
91 if err := db.Update(func(tx *bolt.Tx) error {
92 b, err := tx.CreateBucket([]byte("bucket"))
93 if err != nil {
94 return err
95 }
96 return b.Put([]byte("key"), []byte("val"))
97 }); err != nil {
98 t.Fatal(err)
99 }
100
101 // Retrieve value and attempt to append to it.
102 if err := db.Update(func(tx *bolt.Tx) error {
103 k, v := tx.Bucket([]byte("bucket")).Cursor().First()
104
105 // Verify capacity.
106 if len(k) != cap(k) {
107 t.Fatalf("unexpected key slice capacity: %d", cap(k))
108 } else if len(v) != cap(v) {
109 t.Fatalf("unexpected value slice capacity: %d", cap(v))
110 }
111
112 // Ensure slice can be appended to without a segfault.
113 k = append(k, []byte("123")...)
114 v = append(v, []byte("123")...)
115
76116 return nil
77117 }); err != nil {
78118 t.Fatal(err)
+63
-20
db.go less more
3535 DefaultAllocSize = 16 * 1024 * 1024
3636 )
3737
38 // default page size for db is set to the OS page size.
39 var defaultPageSize = os.Getpagesize()
40
3841 // DB represents a collection of buckets persisted to a file on disk.
3942 // All data access is performed through transactions which can be obtained through the DB.
4043 // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called.
9396 path string
9497 file *os.File
9598 lockfile *os.File // windows only
96 dataref []byte // mmap'ed readonly, write throws SEGV
99 dataref []byte // mmap'ed readonly, write throws SEGV
97100 data *[maxMapSize]byte
98101 datasz int
99102 filesz int // current on disk file size
106109 freelist *freelist
107110 stats Stats
108111
112 pagePool sync.Pool
113
109114 batchMu sync.Mutex
110115 batch *batch
111116
199204 if _, err := db.file.ReadAt(buf[:], 0); err == nil {
200205 m := db.pageInBuffer(buf[:], 0).meta()
201206 if err := m.validate(); err != nil {
202 return nil, err
207 // If we can't read the page size, we can assume it's the same
208 // as the OS -- since that's how the page size was chosen in the
209 // first place.
210 //
211 // If the first page is invalid and this OS uses a different
212 // page size than what the database was created with then we
213 // are out of luck and cannot access the database.
214 db.pageSize = os.Getpagesize()
215 } else {
216 db.pageSize = int(m.pageSize)
203217 }
204 db.pageSize = int(m.pageSize)
205 }
218 }
219 }
220
221 // Initialize page pool.
222 db.pagePool = sync.Pool{
223 New: func() interface{} {
224 return make([]byte, db.pageSize)
225 },
206226 }
207227
208228 // Memory map the data file.
261281 db.meta0 = db.page(0).meta()
262282 db.meta1 = db.page(1).meta()
263283
264 // Validate the meta pages.
265 if err := db.meta0.validate(); err != nil {
266 return err
267 }
268 if err := db.meta1.validate(); err != nil {
269 return err
284 // Validate the meta pages. We only return an error if both meta pages fail
285 // validation, since meta0 failing validation means that it wasn't saved
286 // properly -- but we can recover using meta1. And vice-versa.
287 err0 := db.meta0.validate()
288 err1 := db.meta1.validate()
289 if err0 != nil && err1 != nil {
290 return err0
270291 }
271292
272293 return nil
338359 m.root = bucket{root: 3}
339360 m.pgid = 4
340361 m.txid = txid(i)
362 m.checksum = m.sum64()
341363 }
342364
343365 // Write an empty freelist at page 3.
382404 if !db.opened {
383405 return nil
384406 }
385
407
386408 db.opened = false
387409
388410 db.freelist = nil
389 db.path = ""
390411
391412 // Clear ops.
392413 db.ops.writeAt = nil
413434 db.file = nil
414435 }
415436
437 db.path = ""
416438 return nil
417439 }
418440
777799
778800 // meta retrieves the current meta page reference.
779801 func (db *DB) meta() *meta {
780 if db.meta0.txid > db.meta1.txid {
781 return db.meta0
782 }
783 return db.meta1
802 // We have to return the meta with the highest txid which doesn't fail
803 // validation. Otherwise, we can cause errors when in fact the database is
804 // in a consistent state. metaA is the one with the higher txid.
805 metaA := db.meta0
806 metaB := db.meta1
807 if db.meta1.txid > db.meta0.txid {
808 metaA = db.meta1
809 metaB = db.meta0
810 }
811
812 // Use higher meta page if valid. Otherwise fallback to previous, if valid.
813 if err := metaA.validate(); err == nil {
814 return metaA
815 } else if err := metaB.validate(); err == nil {
816 return metaB
817 }
818
819 // This should never be reached, because both meta1 and meta0 were validated
820 // on mmap() and we do fsync() on every write.
821 panic("bolt.DB.meta(): invalid meta pages")
784822 }
785823
786824 // allocate returns a contiguous block of memory starting at a given page.
787825 func (db *DB) allocate(count int) (*page, error) {
788826 // Allocate a temporary buffer for the page.
789 buf := make([]byte, count*db.pageSize)
827 var buf []byte
828 if count == 1 {
829 buf = db.pagePool.Get().([]byte)
830 } else {
831 buf = make([]byte, count*db.pageSize)
832 }
790833 p := (*page)(unsafe.Pointer(&buf[0]))
791834 p.overflow = uint32(count - 1)
792835
936979
937980 // validate checks the marker bytes and version of the meta page to ensure it matches this binary.
938981 func (m *meta) validate() error {
939 if m.checksum != 0 && m.checksum != m.sum64() {
940 return ErrChecksum
941 } else if m.magic != magic {
982 if m.magic != magic {
942983 return ErrInvalid
943984 } else if m.version != version {
944985 return ErrVersionMismatch
986 } else if m.checksum != 0 && m.checksum != m.sum64() {
987 return ErrChecksum
945988 }
946989 return nil
947990 }
+69
-47
db_test.go less more
4444 _ uint32
4545 _ [16]byte
4646 _ uint64
47 _ uint64
47 pgid uint64
4848 _ uint64
4949 checksum uint64
5050 }
8484 }
8585 }
8686
87 // Ensure that opening a file with wrong checksum returns ErrChecksum.
88 func TestOpen_ErrChecksum(t *testing.T) {
89 buf := make([]byte, pageSize)
90 meta := (*meta)(unsafe.Pointer(&buf[0]))
91 meta.magic = magic
92 meta.version = version
93 meta.checksum = 123
94
95 path := tempfile()
96 f, err := os.Create(path)
97 if err != nil {
98 t.Fatal(err)
99 }
100 if _, err := f.WriteAt(buf, pageHeaderSize); err != nil {
101 t.Fatal(err)
102 }
103 if err := f.Close(); err != nil {
104 t.Fatal(err)
105 }
106 defer os.Remove(path)
107
108 if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrChecksum {
109 t.Fatalf("unexpected error: %s", err)
110 }
111 }
112
11387 // Ensure that opening a file that is not a Bolt database returns ErrInvalid.
11488 func TestOpen_ErrInvalid(t *testing.T) {
11589 path := tempfile()
131105 }
132106 }
133107
134 // Ensure that opening a file created with a different version of Bolt returns
135 // ErrVersionMismatch.
108 // Ensure that opening a file with two invalid versions returns ErrVersionMismatch.
136109 func TestOpen_ErrVersionMismatch(t *testing.T) {
137 buf := make([]byte, pageSize)
138 meta := (*meta)(unsafe.Pointer(&buf[0]))
139 meta.magic = magic
140 meta.version = version + 100
141
142 path := tempfile()
143 f, err := os.Create(path)
144 if err != nil {
145 t.Fatal(err)
146 }
147 if _, err := f.WriteAt(buf, pageHeaderSize); err != nil {
148 t.Fatal(err)
149 }
150 if err := f.Close(); err != nil {
151 t.Fatal(err)
152 }
153 defer os.Remove(path)
154
110 if pageSize != os.Getpagesize() {
111 t.Skip("page size mismatch")
112 }
113
114 // Create empty database.
115 db := MustOpenDB()
116 path := db.Path()
117 defer db.MustClose()
118
119 // Close database.
120 if err := db.DB.Close(); err != nil {
121 t.Fatal(err)
122 }
123
124 // Read data file.
125 buf, err := ioutil.ReadFile(path)
126 if err != nil {
127 t.Fatal(err)
128 }
129
130 // Rewrite meta pages.
131 meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize]))
132 meta0.version++
133 meta1 := (*meta)(unsafe.Pointer(&buf[pageSize+pageHeaderSize]))
134 meta1.version++
135 if err := ioutil.WriteFile(path, buf, 0666); err != nil {
136 t.Fatal(err)
137 }
138
139 // Reopen data file.
155140 if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrVersionMismatch {
141 t.Fatalf("unexpected error: %s", err)
142 }
143 }
144
145 // Ensure that opening a file with two invalid checksums returns ErrChecksum.
146 func TestOpen_ErrChecksum(t *testing.T) {
147 if pageSize != os.Getpagesize() {
148 t.Skip("page size mismatch")
149 }
150
151 // Create empty database.
152 db := MustOpenDB()
153 path := db.Path()
154 defer db.MustClose()
155
156 // Close database.
157 if err := db.DB.Close(); err != nil {
158 t.Fatal(err)
159 }
160
161 // Read data file.
162 buf, err := ioutil.ReadFile(path)
163 if err != nil {
164 t.Fatal(err)
165 }
166
167 // Rewrite meta pages.
168 meta0 := (*meta)(unsafe.Pointer(&buf[pageHeaderSize]))
169 meta0.pgid++
170 meta1 := (*meta)(unsafe.Pointer(&buf[pageSize+pageHeaderSize]))
171 meta1.pgid++
172 if err := ioutil.WriteFile(path, buf, 0666); err != nil {
173 t.Fatal(err)
174 }
175
176 // Reopen data file.
177 if _, err := bolt.Open(path, 0666, nil); err != bolt.ErrChecksum {
156178 t.Fatalf("unexpected error: %s", err)
157179 }
158180 }
+2
-1
errors.go less more
1111 // already open.
1212 ErrDatabaseOpen = errors.New("database already open")
1313
14 // ErrInvalid is returned when a data file is not a Bolt-formatted database.
14 // ErrInvalid is returned when both meta pages on a database are invalid.
15 // This typically occurs when a file is not a bolt database.
1516 ErrInvalid = errors.New("invalid database")
1617
1718 // ErrVersionMismatch is returned when the data file was created with a
+2
-2
page.go less more
110110 // key returns a byte slice of the node key.
111111 func (n *leafPageElement) key() []byte {
112112 buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
113 return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize]
113 return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos]))[:n.ksize:n.ksize]
114114 }
115115
116116 // value returns a byte slice of the node value.
117117 func (n *leafPageElement) value() []byte {
118118 buf := (*[maxAllocSize]byte)(unsafe.Pointer(n))
119 return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize]
119 return (*[maxAllocSize]byte)(unsafe.Pointer(&buf[n.pos+n.ksize]))[:n.vsize:n.vsize]
120120 }
121121
122122 // PageInfo represents human readable information about a page.
+18
-2
tx.go less more
472472 for _, p := range tx.pages {
473473 pages = append(pages, p)
474474 }
475 // Clear out page cache early.
476 tx.pages = make(map[pgid]*page)
475477 sort.Sort(pages)
476478
477479 // Write pages to disk in order.
516518 }
517519 }
518520
519 // Clear out page cache.
520 tx.pages = make(map[pgid]*page)
521 // Put small pages back to page pool.
522 for _, p := range pages {
523 // Ignore page sizes over 1 page.
524 // These are allocated using make() instead of the page pool.
525 if int(p.overflow) != 0 {
526 continue
527 }
528
529 buf := (*[maxAllocSize]byte)(unsafe.Pointer(p))[:tx.db.pageSize]
530
531 // See https://go.googlesource.com/go/+/f03c9202c43e0abb130669852082117ca50aa9b1
532 for i := range buf {
533 buf[i] = 0
534 }
535 tx.db.pagePool.Put(buf)
536 }
521537
522538 return nil
523539 }