Commit 5c01042659135f82cacc204bed2476d18c43aa75 - golang-github-nebulouslabs-bolt

New upstream snapshot. Debian Janitor 2 years ago

13 changed file(s) with 98 addition(s) and 926 deletion(s). Raw diff Collapse all Expand all

-0

.gitignore less more

	0	*.prof
	1	*.test
	2	*.swp
	3	/bin/

-915

README.md less more

0		Bolt [![Coverage Status](https://coveralls.io/repos/boltdb/bolt/badge.svg?branch=master)](https://coveralls.io/r/boltdb/bolt?branch=master) [![GoDoc](https://godoc.org/github.com/boltdb/bolt?status.svg)](https://godoc.org/github.com/boltdb/bolt) ![Version](https://img.shields.io/badge/version-1.2.1-green.svg)
1		====
2
3		Bolt is a pure Go key/value store inspired by [Howard Chu's][hyc_symas]
4		[LMDB project][lmdb]. The goal of the project is to provide a simple,
5		fast, and reliable database for projects that don't require a full database
6		server such as Postgres or MySQL.
7
8		Since Bolt is meant to be used as such a low-level piece of functionality,
9		simplicity is key. The API will be small and only focus on getting values
10		and setting values. That's it.
11
12		[hyc_symas]: https://twitter.com/hyc_symas
13		[lmdb]: http://symas.com/mdb/
14
15		## Project Status
16
17		Bolt is stable, the API is fixed, and the file format is fixed. Full unit
18		test coverage and randomized black box testing are used to ensure database
19		consistency and thread safety. Bolt is currently used in high-load production
20		environments serving databases as large as 1TB. Many companies such as
21		Shopify and Heroku use Bolt-backed services every day.
22
23		## Table of Contents
24
25		- [Getting Started](#getting-started)
26		- [Installing](#installing)
27		- [Opening a database](#opening-a-database)
28		- [Transactions](#transactions)
29		- [Read-write transactions](#read-write-transactions)
30		- [Read-only transactions](#read-only-transactions)
31		- [Batch read-write transactions](#batch-read-write-transactions)
32		- [Managing transactions manually](#managing-transactions-manually)
33		- [Using buckets](#using-buckets)
34		- [Using key/value pairs](#using-keyvalue-pairs)
35		- [Autoincrementing integer for the bucket](#autoincrementing-integer-for-the-bucket)
36		- [Iterating over keys](#iterating-over-keys)
37		- [Prefix scans](#prefix-scans)
38		- [Range scans](#range-scans)
39		- [ForEach()](#foreach)
40		- [Nested buckets](#nested-buckets)
41		- [Database backups](#database-backups)
42		- [Statistics](#statistics)
43		- [Read-Only Mode](#read-only-mode)
44		- [Mobile Use (iOS/Android)](#mobile-use-iosandroid)
45		- [Resources](#resources)
46		- [Comparison with other databases](#comparison-with-other-databases)
47		- [Postgres, MySQL, & other relational databases](#postgres-mysql--other-relational-databases)
48		- [LevelDB, RocksDB](#leveldb-rocksdb)
49		- [LMDB](#lmdb)
50		- [Caveats & Limitations](#caveats--limitations)
51		- [Reading the Source](#reading-the-source)
52		- [Other Projects Using Bolt](#other-projects-using-bolt)
53
54		## Getting Started
55
56		### Installing
57
58		To start using Bolt, install Go and run `go get`:
59
60		```sh
61		$ go get github.com/boltdb/bolt/...
62		```
63
64		This will retrieve the library and install the `bolt` command line utility into
65		your `$GOBIN` path.
66
67
68		### Opening a database
69
70		The top-level object in Bolt is a `DB`. It is represented as a single file on
71		your disk and represents a consistent snapshot of your data.
72
73		To open your database, simply use the `bolt.Open()` function:
74
75		```go
76		package main
77
78		import (
79		"log"
80
81		"github.com/boltdb/bolt"
82		)
83
84		func main() {
85		// Open the my.db data file in your current directory.
86		// It will be created if it doesn't exist.
87		db, err := bolt.Open("my.db", 0600, nil)
88		if err != nil {
89		log.Fatal(err)
90		}
91		defer db.Close()
92
93		...
94		}
95		```
96
97		Please note that Bolt obtains a file lock on the data file so multiple processes
98		cannot open the same database at the same time. Opening an already open Bolt
99		database will cause it to hang until the other process closes it. To prevent
100		an indefinite wait you can pass a timeout option to the `Open()` function:
101
102		```go
103		db, err := bolt.Open("my.db", 0600, &bolt.Options{Timeout: 1 * time.Second})
104		```
105
106
107		### Transactions
108
109		Bolt allows only one read-write transaction at a time but allows as many
110		read-only transactions as you want at a time. Each transaction has a consistent
111		view of the data as it existed when the transaction started.
112
113		Individual transactions and all objects created from them (e.g. buckets, keys)
114		are not thread safe. To work with data in multiple goroutines you must start
115		a transaction for each one or use locking to ensure only one goroutine accesses
116		a transaction at a time. Creating transaction from the `DB` is thread safe.
117
118		Read-only transactions and read-write transactions should not depend on one
119		another and generally shouldn't be opened simultaneously in the same goroutine.
120		This can cause a deadlock as the read-write transaction needs to periodically
121		re-map the data file but it cannot do so while a read-only transaction is open.
122
123
124		#### Read-write transactions
125
126		To start a read-write transaction, you can use the `DB.Update()` function:
127
128		```go
129		err := db.Update(func(tx *bolt.Tx) error {
130		...
131		return nil
132		})
133		```
134
135		Inside the closure, you have a consistent view of the database. You commit the
136		transaction by returning `nil` at the end. You can also rollback the transaction
137		at any point by returning an error. All database operations are allowed inside
138		a read-write transaction.
139
140		Always check the return error as it will report any disk failures that can cause
141		your transaction to not complete. If you return an error within your closure
142		it will be passed through.
143
144
145		#### Read-only transactions
146
147		To start a read-only transaction, you can use the `DB.View()` function:
148
149		```go
150		err := db.View(func(tx *bolt.Tx) error {
151		...
152		return nil
153		})
154		```
155
156		You also get a consistent view of the database within this closure, however,
157		no mutating operations are allowed within a read-only transaction. You can only
158		retrieve buckets, retrieve values, and copy the database within a read-only
159		transaction.
160
161
162		#### Batch read-write transactions
163
164		Each `DB.Update()` waits for disk to commit the writes. This overhead
165		can be minimized by combining multiple updates with the `DB.Batch()`
166		function:
167
168		```go
169		err := db.Batch(func(tx *bolt.Tx) error {
170		...
171		return nil
172		})
173		```
174
175		Concurrent Batch calls are opportunistically combined into larger
176		transactions. Batch is only useful when there are multiple goroutines
177		calling it.
178
179		The trade-off is that `Batch` can call the given
180		function multiple times, if parts of the transaction fail. The
181		function must be idempotent and side effects must take effect only
182		after a successful return from `DB.Batch()`.
183
184		For example: don't display messages from inside the function, instead
185		set variables in the enclosing scope:
186
187		```go
188		var id uint64
189		err := db.Batch(func(tx *bolt.Tx) error {
190		// Find last key in bucket, decode as bigendian uint64, increment
191		// by one, encode back to []byte, and add new key.
192		...
193		id = newValue
194		return nil
195		})
196		if err != nil {
197		return ...
198		}
199		fmt.Println("Allocated ID %d", id)
200		```
201
202
203		#### Managing transactions manually
204
205		The `DB.View()` and `DB.Update()` functions are wrappers around the `DB.Begin()`
206		function. These helper functions will start the transaction, execute a function,
207		and then safely close your transaction if an error is returned. This is the
208		recommended way to use Bolt transactions.
209
210		However, sometimes you may want to manually start and end your transactions.
211		You can use the `DB.Begin()` function directly but please be sure to close
212		the transaction.
213
214		```go
215		// Start a writable transaction.
216		tx, err := db.Begin(true)
217		if err != nil {
218		return err
219		}
220		defer tx.Rollback()
221
222		// Use the transaction...
223		_, err := tx.CreateBucket([]byte("MyBucket"))
224		if err != nil {
225		return err
226		}
227
228		// Commit the transaction and check for error.
229		if err := tx.Commit(); err != nil {
230		return err
231		}
232		```
233
234		The first argument to `DB.Begin()` is a boolean stating if the transaction
235		should be writable.
236
237
238		### Using buckets
239
240		Buckets are collections of key/value pairs within the database. All keys in a
241		bucket must be unique. You can create a bucket using the `DB.CreateBucket()`
242		function:
243
244		```go
245		db.Update(func(tx *bolt.Tx) error {
246		b, err := tx.CreateBucket([]byte("MyBucket"))
247		if err != nil {
248		return fmt.Errorf("create bucket: %s", err)
249		}
250		return nil
251		})
252		```
253
254		You can also create a bucket only if it doesn't exist by using the
255		`Tx.CreateBucketIfNotExists()` function. It's a common pattern to call this
256		function for all your top-level buckets after you open your database so you can
257		guarantee that they exist for future transactions.
258
259		To delete a bucket, simply call the `Tx.DeleteBucket()` function.
260
261
262		### Using key/value pairs
263
264		To save a key/value pair to a bucket, use the `Bucket.Put()` function:
265
266		```go
267		db.Update(func(tx *bolt.Tx) error {
268		b := tx.Bucket([]byte("MyBucket"))
269		err := b.Put([]byte("answer"), []byte("42"))
270		return err
271		})
272		```
273
274		This will set the value of the `"answer"` key to `"42"` in the `MyBucket`
275		bucket. To retrieve this value, we can use the `Bucket.Get()` function:
276
277		```go
278		db.View(func(tx *bolt.Tx) error {
279		b := tx.Bucket([]byte("MyBucket"))
280		v := b.Get([]byte("answer"))
281		fmt.Printf("The answer is: %s\n", v)
282		return nil
283		})
284		```
285
286		The `Get()` function does not return an error because its operation is
287		guaranteed to work (unless there is some kind of system failure). If the key
288		exists then it will return its byte slice value. If it doesn't exist then it
289		will return `nil`. It's important to note that you can have a zero-length value
290		set to a key which is different than the key not existing.
291
292		Use the `Bucket.Delete()` function to delete a key from the bucket.
293
294		Please note that values returned from `Get()` are only valid while the
295		transaction is open. If you need to use a value outside of the transaction
296		then you must use `copy()` to copy it to another byte slice.
297
298
299		### Autoincrementing integer for the bucket
300		By using the `NextSequence()` function, you can let Bolt determine a sequence
301		which can be used as the unique identifier for your key/value pairs. See the
302		example below.
303
304		```go
305		// CreateUser saves u to the store. The new user ID is set on u once the data is persisted.
306		func (s Store) CreateUser(u User) error {
307		return s.db.Update(func(tx *bolt.Tx) error {
308		// Retrieve the users bucket.
309		// This should be created when the DB is first opened.
310		b := tx.Bucket([]byte("users"))
311
312		// Generate ID for the user.
313		// This returns an error only if the Tx is closed or not writeable.
314		// That can't happen in an Update() call so I ignore the error check.
315		id, _ := b.NextSequence()
316		u.ID = int(id)
317
318		// Marshal user data into bytes.
319		buf, err := json.Marshal(u)
320		if err != nil {
321		return err
322		}
323
324		// Persist bytes to users bucket.
325		return b.Put(itob(u.ID), buf)
326		})
327		}
328
329		// itob returns an 8-byte big endian representation of v.
330		func itob(v int) []byte {
331		b := make([]byte, 8)
332		binary.BigEndian.PutUint64(b, uint64(v))
333		return b
334		}
335
336		type User struct {
337		ID int
338		...
339		}
340		```
341
342		### Iterating over keys
343
344		Bolt stores its keys in byte-sorted order within a bucket. This makes sequential
345		iteration over these keys extremely fast. To iterate over keys we'll use a
346		`Cursor`:
347
348		```go
349		db.View(func(tx *bolt.Tx) error {
350		// Assume bucket exists and has keys
351		b := tx.Bucket([]byte("MyBucket"))
352
353		c := b.Cursor()
354
355		for k, v := c.First(); k != nil; k, v = c.Next() {
356		fmt.Printf("key=%s, value=%s\n", k, v)
357		}
358
359		return nil
360		})
361		```
362
363		The cursor allows you to move to a specific point in the list of keys and move
364		forward or backward through the keys one at a time.
365
366		The following functions are available on the cursor:
367
368		```
369		First() Move to the first key.
370		Last() Move to the last key.
371		Seek() Move to a specific key.
372		Next() Move to the next key.
373		Prev() Move to the previous key.
374		```
375
376		Each of those functions has a return signature of `(key []byte, value []byte)`.
377		When you have iterated to the end of the cursor then `Next()` will return a
378		`nil` key. You must seek to a position using `First()`, `Last()`, or `Seek()`
379		before calling `Next()` or `Prev()`. If you do not seek to a position then
380		these functions will return a `nil` key.
381
382		During iteration, if the key is non-`nil` but the value is `nil`, that means
383		the key refers to a bucket rather than a value. Use `Bucket.Bucket()` to
384		access the sub-bucket.
385
386
387		#### Prefix scans
388
389		To iterate over a key prefix, you can combine `Seek()` and `bytes.HasPrefix()`:
390
391		```go
392		db.View(func(tx *bolt.Tx) error {
393		// Assume bucket exists and has keys
394		c := tx.Bucket([]byte("MyBucket")).Cursor()
395
396		prefix := []byte("1234")
397		for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() {
398		fmt.Printf("key=%s, value=%s\n", k, v)
399		}
400
401		return nil
402		})
403		```
404
405		#### Range scans
406
407		Another common use case is scanning over a range such as a time range. If you
408		use a sortable time encoding such as RFC3339 then you can query a specific
409		date range like this:
410
411		```go
412		db.View(func(tx *bolt.Tx) error {
413		// Assume our events bucket exists and has RFC3339 encoded time keys.
414		c := tx.Bucket([]byte("Events")).Cursor()
415
416		// Our time range spans the 90's decade.
417		min := []byte("1990-01-01T00:00:00Z")
418		max := []byte("2000-01-01T00:00:00Z")
419
420		// Iterate over the 90's.
421		for k, v := c.Seek(min); k != nil && bytes.Compare(k, max) <= 0; k, v = c.Next() {
422		fmt.Printf("%s: %s\n", k, v)
423		}
424
425		return nil
426		})
427		```
428
429		Note that, while RFC3339 is sortable, the Golang implementation of RFC3339Nano does not use a fixed number of digits after the decimal point and is therefore not sortable.
430
431
432		#### ForEach()
433
434		You can also use the function `ForEach()` if you know you'll be iterating over
435		all the keys in a bucket:
436
437		```go
438		db.View(func(tx *bolt.Tx) error {
439		// Assume bucket exists and has keys
440		b := tx.Bucket([]byte("MyBucket"))
441
442		b.ForEach(func(k, v []byte) error {
443		fmt.Printf("key=%s, value=%s\n", k, v)
444		return nil
445		})
446		return nil
447		})
448		```
449
450		Please note that keys and values in `ForEach()` are only valid while
451		the transaction is open. If you need to use a key or value outside of
452		the transaction, you must use `copy()` to copy it to another byte
453		slice.
454
455		### Nested buckets
456
457		You can also store a bucket in a key to create nested buckets. The API is the
458		same as the bucket management API on the `DB` object:
459
460		```go
461		func (Bucket) CreateBucket(key []byte) (Bucket, error)
462		func (Bucket) CreateBucketIfNotExists(key []byte) (Bucket, error)
463		func (*Bucket) DeleteBucket(key []byte) error
464		```
465
466		Say you had a multi-tenant application where the root level bucket was the account bucket. Inside of this bucket was a sequence of accounts which themselves are buckets. And inside the sequence bucket you could have many buckets pertaining to the Account itself (Users, Notes, etc) isolating the information into logical groupings.
467
468		```go
469
470		// createUser creates a new user in the given account.
471		func createUser(accountID int, u *User) error {
472		// Start the transaction.
473		tx, err := db.Begin(true)
474		if err != nil {
475		return err
476		}
477		defer tx.Rollback()
478
479		// Retrieve the root bucket for the account.
480		// Assume this has already been created when the account was set up.
481		root := tx.Bucket([]byte(strconv.FormatUint(accountID, 10)))
482
483		// Setup the users bucket.
484		bkt, err := root.CreateBucketIfNotExists([]byte("USERS"))
485		if err != nil {
486		return err
487		}
488
489		// Generate an ID for the new user.
490		userID, err := bkt.NextSequence()
491		if err != nil {
492		return err
493		}
494		u.ID = userID
495
496		// Marshal and save the encoded user.
497		if buf, err := json.Marshal(u); err != nil {
498		return err
499		} else if err := bkt.Put([]byte(strconv.FormatUint(u.ID, 10)), buf); err != nil {
500		return err
501		}
502
503		// Commit the transaction.
504		if err := tx.Commit(); err != nil {
505		return err
506		}
507
508		return nil
509		}
510
511		```
512
513
514
515
516		### Database backups
517
518		Bolt is a single file so it's easy to backup. You can use the `Tx.WriteTo()`
519		function to write a consistent view of the database to a writer. If you call
520		this from a read-only transaction, it will perform a hot backup and not block
521		your other database reads and writes.
522
523		By default, it will use a regular file handle which will utilize the operating
524		system's page cache. See the [`Tx`](https://godoc.org/github.com/boltdb/bolt#Tx)
525		documentation for information about optimizing for larger-than-RAM datasets.
526
527		One common use case is to backup over HTTP so you can use tools like `cURL` to
528		do database backups:
529
530		```go
531		func BackupHandleFunc(w http.ResponseWriter, req *http.Request) {
532		err := db.View(func(tx *bolt.Tx) error {
533		w.Header().Set("Content-Type", "application/octet-stream")
534		w.Header().Set("Content-Disposition", `attachment; filename="my.db"`)
535		w.Header().Set("Content-Length", strconv.Itoa(int(tx.Size())))
536		_, err := tx.WriteTo(w)
537		return err
538		})
539		if err != nil {
540		http.Error(w, err.Error(), http.StatusInternalServerError)
541		}
542		}
543		```
544
545		Then you can backup using this command:
546
547		```sh
548		$ curl http://localhost/backup > my.db
549		```
550
551		Or you can open your browser to `http://localhost/backup` and it will download
552		automatically.
553
554		If you want to backup to another file you can use the `Tx.CopyFile()` helper
555		function.
556
557
558		### Statistics
559
560		The database keeps a running count of many of the internal operations it
561		performs so you can better understand what's going on. By grabbing a snapshot
562		of these stats at two points in time we can see what operations were performed
563		in that time range.
564
565		For example, we could start a goroutine to log stats every 10 seconds:
566
567		```go
568		go func() {
569		// Grab the initial stats.
570		prev := db.Stats()
571
572		for {
573		// Wait for 10s.
574		time.Sleep(10 * time.Second)
575
576		// Grab the current stats and diff them.
577		stats := db.Stats()
578		diff := stats.Sub(&prev)
579
580		// Encode stats to JSON and print to STDERR.
581		json.NewEncoder(os.Stderr).Encode(diff)
582
583		// Save stats for the next loop.
584		prev = stats
585		}
586		}()
587		```
588
589		It's also useful to pipe these stats to a service such as statsd for monitoring
590		or to provide an HTTP endpoint that will perform a fixed-length sample.
591
592
593		### Read-Only Mode
594
595		Sometimes it is useful to create a shared, read-only Bolt database. To this,
596		set the `Options.ReadOnly` flag when opening your database. Read-only mode
597		uses a shared lock to allow multiple processes to read from the database but
598		it will block any processes from opening the database in read-write mode.
599
600		```go
601		db, err := bolt.Open("my.db", 0666, &bolt.Options{ReadOnly: true})
602		if err != nil {
603		log.Fatal(err)
604		}
605		```
606
607		### Mobile Use (iOS/Android)
608
609		Bolt is able to run on mobile devices by leveraging the binding feature of the
610		[gomobile](https://github.com/golang/mobile) tool. Create a struct that will
611		contain your database logic and a reference to a `*bolt.DB` with a initializing
612		constructor that takes in a filepath where the database file will be stored.
613		Neither Android nor iOS require extra permissions or cleanup from using this method.
614
615		```go
616		func NewBoltDB(filepath string) *BoltDB {
617		db, err := bolt.Open(filepath+"/demo.db", 0600, nil)
618		if err != nil {
619		log.Fatal(err)
620		}
621
622		return &BoltDB{db}
623		}
624
625		type BoltDB struct {
626		db *bolt.DB
627		...
628		}
629
630		func (b *BoltDB) Path() string {
631		return b.db.Path()
632		}
633
634		func (b *BoltDB) Close() {
635		b.db.Close()
636		}
637		```
638
639		Database logic should be defined as methods on this wrapper struct.
640
641		To initialize this struct from the native language (both platforms now sync
642		their local storage to the cloud. These snippets disable that functionality for the
643		database file):
644
645		#### Android
646
647		```java
648		String path;
649		if (android.os.Build.VERSION.SDK_INT >=android.os.Build.VERSION_CODES.LOLLIPOP){
650		path = getNoBackupFilesDir().getAbsolutePath();
651		} else{
652		path = getFilesDir().getAbsolutePath();
653		}
654		Boltmobiledemo.BoltDB boltDB = Boltmobiledemo.NewBoltDB(path)
655		```
656
657		#### iOS
658
659		```objc
660		- (void)demo {
661		NSString* path = [NSSearchPathForDirectoriesInDomains(NSLibraryDirectory,
662		NSUserDomainMask,
663		YES) objectAtIndex:0];
664		GoBoltmobiledemoBoltDB * demo = GoBoltmobiledemoNewBoltDB(path);
665		[self addSkipBackupAttributeToItemAtPath:demo.path];
666		//Some DB Logic would go here
667		[demo close];
668		}
669
670		- (BOOL)addSkipBackupAttributeToItemAtPath:(NSString *) filePathString
671		{
672		NSURL* URL= [NSURL fileURLWithPath: filePathString];
673		assert([[NSFileManager defaultManager] fileExistsAtPath: [URL path]]);
674
675		NSError *error = nil;
676		BOOL success = [URL setResourceValue: [NSNumber numberWithBool: YES]
677		forKey: NSURLIsExcludedFromBackupKey error: &error];
678		if(!success){
679		NSLog(@"Error excluding %@ from backup %@", [URL lastPathComponent], error);
680		}
681		return success;
682		}
683
684		```
685
686		## Resources
687
688		For more information on getting started with Bolt, check out the following articles:
689
690		* [Intro to BoltDB: Painless Performant Persistence](http://npf.io/2014/07/intro-to-boltdb-painless-performant-persistence/) by [Nate Finch](https://github.com/natefinch).
691		* [Bolt -- an embedded key/value database for Go](https://www.progville.com/go/bolt-embedded-db-golang/) by Progville
692
693
694		## Comparison with other databases
695
696		### Postgres, MySQL, & other relational databases
697
698		Relational databases structure data into rows and are only accessible through
699		the use of SQL. This approach provides flexibility in how you store and query
700		your data but also incurs overhead in parsing and planning SQL statements. Bolt
701		accesses all data by a byte slice key. This makes Bolt fast to read and write
702		data by key but provides no built-in support for joining values together.
703
704		Most relational databases (with the exception of SQLite) are standalone servers
705		that run separately from your application. This gives your systems
706		flexibility to connect multiple application servers to a single database
707		server but also adds overhead in serializing and transporting data over the
708		network. Bolt runs as a library included in your application so all data access
709		has to go through your application's process. This brings data closer to your
710		application but limits multi-process access to the data.
711
712
713		### LevelDB, RocksDB
714
715		LevelDB and its derivatives (RocksDB, HyperLevelDB) are similar to Bolt in that
716		they are libraries bundled into the application, however, their underlying
717		structure is a log-structured merge-tree (LSM tree). An LSM tree optimizes
718		random writes by using a write ahead log and multi-tiered, sorted files called
719		SSTables. Bolt uses a B+tree internally and only a single file. Both approaches
720		have trade-offs.
721
722		If you require a high random write throughput (>10,000 w/sec) or you need to use
723		spinning disks then LevelDB could be a good choice. If your application is
724		read-heavy or does a lot of range scans then Bolt could be a good choice.
725
726		One other important consideration is that LevelDB does not have transactions.
727		It supports batch writing of key/values pairs and it supports read snapshots
728		but it will not give you the ability to do a compare-and-swap operation safely.
729		Bolt supports fully serializable ACID transactions.
730
731
732		### LMDB
733
734		Bolt was originally a port of LMDB so it is architecturally similar. Both use
735		a B+tree, have ACID semantics with fully serializable transactions, and support
736		lock-free MVCC using a single writer and multiple readers.
737
738		The two projects have somewhat diverged. LMDB heavily focuses on raw performance
739		while Bolt has focused on simplicity and ease of use. For example, LMDB allows
740		several unsafe actions such as direct writes for the sake of performance. Bolt
741		opts to disallow actions which can leave the database in a corrupted state. The
742		only exception to this in Bolt is `DB.NoSync`.
743
744		There are also a few differences in API. LMDB requires a maximum mmap size when
745		opening an `mdb_env` whereas Bolt will handle incremental mmap resizing
746		automatically. LMDB overloads the getter and setter functions with multiple
747		flags whereas Bolt splits these specialized cases into their own functions.
748
749
750		## Caveats & Limitations
751
752		It's important to pick the right tool for the job and Bolt is no exception.
753		Here are a few things to note when evaluating and using Bolt:
754
755		* Bolt is good for read intensive workloads. Sequential write performance is
756		also fast but random writes can be slow. You can use `DB.Batch()` or add a
757		write-ahead log to help mitigate this issue.
758
759		* Bolt uses a B+tree internally so there can be a lot of random page access.
760		SSDs provide a significant performance boost over spinning disks.
761
762		* Try to avoid long running read transactions. Bolt uses copy-on-write so
763		old pages cannot be reclaimed while an old transaction is using them.
764
765		* Byte slices returned from Bolt are only valid during a transaction. Once the
766		transaction has been committed or rolled back then the memory they point to
767		can be reused by a new page or can be unmapped from virtual memory and you'll
768		see an `unexpected fault address` panic when accessing it.
769
770		* Bolt uses an exclusive write lock on the database file so it cannot be
771		shared by multiple processes.
772
773		* Be careful when using `Bucket.FillPercent`. Setting a high fill percent for
774		buckets that have random inserts will cause your database to have very poor
775		page utilization.
776
777		* Use larger buckets in general. Smaller buckets causes poor page utilization
778		once they become larger than the page size (typically 4KB).
779
780		* Bulk loading a lot of random writes into a new bucket can be slow as the
781		page will not split until the transaction is committed. Randomly inserting
782		more than 100,000 key/value pairs into a single new bucket in a single
783		transaction is not advised.
784
785		* Bolt uses a memory-mapped file so the underlying operating system handles the
786		caching of the data. Typically, the OS will cache as much of the file as it
787		can in memory and will release memory as needed to other processes. This means
788		that Bolt can show very high memory usage when working with large databases.
789		However, this is expected and the OS will release memory as needed. Bolt can
790		handle databases much larger than the available physical RAM, provided its
791		memory-map fits in the process virtual address space. It may be problematic
792		on 32-bits systems.
793
794		* The data structures in the Bolt database are memory mapped so the data file
795		will be endian specific. This means that you cannot copy a Bolt file from a
796		little endian machine to a big endian machine and have it work. For most
797		users this is not a concern since most modern CPUs are little endian.
798
799		* Because of the way pages are laid out on disk, Bolt cannot truncate data files
800		and return free pages back to the disk. Instead, Bolt maintains a free list
801		of unused pages within its data file. These free pages can be reused by later
802		transactions. This works well for many use cases as databases generally tend
803		to grow. However, it's important to note that deleting large chunks of data
804		will not allow you to reclaim that space on disk.
805
806		For more information on page allocation, [see this comment][page-allocation].
807
808		[page-allocation]: https://github.com/boltdb/bolt/issues/308#issuecomment-74811638
809
810
811		## Reading the Source
812
813		Bolt is a relatively small code base (<3KLOC) for an embedded, serializable,
814		transactional key/value database so it can be a good starting point for people
815		interested in how databases work.
816
817		The best places to start are the main entry points into Bolt:
818
819		- `Open()` - Initializes the reference to the database. It's responsible for
820		creating the database if it doesn't exist, obtaining an exclusive lock on the
821		file, reading the meta pages, & memory-mapping the file.
822
823		- `DB.Begin()` - Starts a read-only or read-write transaction depending on the
824		value of the `writable` argument. This requires briefly obtaining the "meta"
825		lock to keep track of open transactions. Only one read-write transaction can
826		exist at a time so the "rwlock" is acquired during the life of a read-write
827		transaction.
828
829		- `Bucket.Put()` - Writes a key/value pair into a bucket. After validating the
830		arguments, a cursor is used to traverse the B+tree to the page and position
831		where they key & value will be written. Once the position is found, the bucket
832		materializes the underlying page and the page's parent pages into memory as
833		"nodes". These nodes are where mutations occur during read-write transactions.
834		These changes get flushed to disk during commit.
835
836		- `Bucket.Get()` - Retrieves a key/value pair from a bucket. This uses a cursor
837		to move to the page & position of a key/value pair. During a read-only
838		transaction, the key and value data is returned as a direct reference to the
839		underlying mmap file so there's no allocation overhead. For read-write
840		transactions, this data may reference the mmap file or one of the in-memory
841		node values.
842
843		- `Cursor` - This object is simply for traversing the B+tree of on-disk pages
844		or in-memory nodes. It can seek to a specific key, move to the first or last
845		value, or it can move forward or backward. The cursor handles the movement up
846		and down the B+tree transparently to the end user.
847
848		- `Tx.Commit()` - Converts the in-memory dirty nodes and the list of free pages
849		into pages to be written to disk. Writing to disk then occurs in two phases.
850		First, the dirty pages are written to disk and an `fsync()` occurs. Second, a
851		new meta page with an incremented transaction ID is written and another
852		`fsync()` occurs. This two phase write ensures that partially written data
853		pages are ignored in the event of a crash since the meta page pointing to them
854		is never written. Partially written meta pages are invalidated because they
855		are written with a checksum.
856
857		If you have additional notes that could be helpful for others, please submit
858		them via pull request.
859
860
861		## Other Projects Using Bolt
862
863		Below is a list of public, open source projects that use Bolt:
864
865		* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files.
866		* [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard.
867		* [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside.
868		* [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb.
869		* [Skybox Analytics](https://github.com/skybox/skybox) - A standalone funnel analysis tool for web analytics.
870		* [Scuttlebutt](https://github.com/benbjohnson/scuttlebutt) - Uses Bolt to store and process all Twitter mentions of GitHub projects.
871		* [Wiki](https://github.com/peterhellberg/wiki) - A tiny wiki using Goji, BoltDB and Blackfriday.
872		* [ChainStore](https://github.com/pressly/chainstore) - Simple key-value interface to a variety of storage engines organized as a chain of operations.
873		* [MetricBase](https://github.com/msiebuhr/MetricBase) - Single-binary version of Graphite.
874		* [Gitchain](https://github.com/gitchain/gitchain) - Decentralized, peer-to-peer Git repositories aka "Git meets Bitcoin".
875		* [event-shuttle](https://github.com/sclasen/event-shuttle) - A Unix system service to collect and reliably deliver messages to Kafka.
876		* [ipxed](https://github.com/kelseyhightower/ipxed) - Web interface and api for ipxed.
877		* [BoltStore](https://github.com/yosssi/boltstore) - Session store using Bolt.
878		* [photosite/session](https://godoc.org/bitbucket.org/kardianos/photosite/session) - Sessions for a photo viewing site.
879		* [LedisDB](https://github.com/siddontang/ledisdb) - A high performance NoSQL, using Bolt as optional storage.
880		* [ipLocator](https://github.com/AndreasBriese/ipLocator) - A fast ip-geo-location-server using bolt with bloom filters.
881		* [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend.
882		* [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend.
883		* [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server.
884		* [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read.
885		* [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics.
886		* [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data.
887		* [Prometheus Annotation Server](https://github.com/oliver006/prom_annotation_server) - Annotation server for PromDash & Prometheus service monitoring system.
888		* [Consul](https://github.com/hashicorp/consul) - Consul is service discovery and configuration made easy. Distributed, highly available, and datacenter-aware.
889		* [Kala](https://github.com/ajvb/kala) - Kala is a modern job scheduler optimized to run on a single node. It is persistent, JSON over HTTP API, ISO 8601 duration notation, and dependent jobs.
890		* [drive](https://github.com/odeke-em/drive) - drive is an unofficial Google Drive command line client for \*NIX operating systems.
891		* [stow](https://github.com/djherbis/stow) - a persistence manager for objects
892		backed by boltdb.
893		* [buckets](https://github.com/joyrexus/buckets) - a bolt wrapper streamlining
894		simple tx and key scans.
895		* [mbuckets](https://github.com/abhigupta912/mbuckets) - A Bolt wrapper that allows easy operations on multi level (nested) buckets.
896		* [Request Baskets](https://github.com/darklynx/request-baskets) - A web service to collect arbitrary HTTP requests and inspect them via REST API or simple web UI, similar to [RequestBin](http://requestb.in/) service
897		* [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service.
898		* [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners.
899		* [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores.
900		* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB.
901		* [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB.
902		* [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings.
903		* [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend.
904		* [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files.
905		* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter.
906		* [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development.
907		* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains
908		* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal.
909		* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet.
910		* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency.
911		* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies
912		* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB
913
914		If you are using Bolt in a project please send a pull request to add it to the list.
	0	This repository has moved to [GitLab](https://gitlab.com/NebulousLabs/bolt).

-1

bucket_test.go less more

12	12	"testing"
13	13	"testing/quick"
14	14
15		"github.com/boltdb/bolt"
	15	"github.com/NebulousLabs/bolt"
16	16	)
17	17
18	18	// Ensure that a bucket that gets a non-existent key returns nil.

-1

cmd/bolt/main.go less more

18	18	"unicode/utf8"
19	19	"unsafe"
20	20
21		"github.com/boltdb/bolt"
	21	"github.com/NebulousLabs/bolt"
22	22	)
23	23
24	24	var (

-2

cmd/bolt/main_test.go less more

11	11	"strconv"
12	12	"testing"
13	13
14		"github.com/boltdb/bolt"
15		"github.com/boltdb/bolt/cmd/bolt"
	14	"github.com/NebulousLabs/bolt"
	15	"github.com/NebulousLabs/bolt/cmd/bolt"
16	16	)
17	17
18	18	// Ensure the "info" command can print information about a database.

-1

cursor_test.go less more

10	10	"testing"
11	11	"testing/quick"
12	12
13		"github.com/boltdb/bolt"
	13	"github.com/NebulousLabs/bolt"
14	14	)
15	15
16	16	// Ensure that a cursor can return a reference to the bucket that created it.

-1

db.go less more

274	274
275	275	// Memory-map the data file as a byte slice.
276	276	if err := mmap(db, size); err != nil {
277		return err
	277	// mmap failed; the system may have run out of space. Fallback to
	278	// mapping the bare minimum needed for the current db size.
	279	if err2 := mmap(db, db.datasz); err2 != nil {
	280	panic(fmt.Sprintf("failed to revert db size after failed mmap: %v", err2))
	281	}
	282	return MmapError(err.Error())
278	283	}
279	284
280	285	// Save references to the meta pages.

-1

db_test.go less more

18	18	"time"
19	19	"unsafe"
20	20
21		"github.com/boltdb/bolt"
	21	"github.com/NebulousLabs/bolt"
22	22	)
23	23
24	24	var statsFlag = flag.Bool("stats", false, "show performance stats")

-2

debian/changelog less more

0		golang-github-nebulouslabs-bolt (1.0+git20170131.0.ca9f208-2) UNRELEASED; urgency=medium
	0	golang-github-nebulouslabs-bolt (1.0+git20181203.0.0366476-1) UNRELEASED; urgency=medium
1	1
2	2	[ Paul Tagliamonte ]
3	3	* Remove Built-Using from arch:all -dev package

13	13	* Bump debhelper dependency to >= 9, since that's what is used in
14	14	debian/compat.
15	15	* Change priority extra to priority optional.
	16	* New upstream snapshot.
16	17
17		-- Bjorn Dolk <debian@bjorndolk.com> Tue, 08 Aug 2017 23:10:03 -0400
	18	-- Bjorn Dolk <debian@bjorndolk.com> Sat, 04 Sep 2021 16:44:08 -0000
18	19
19	20	golang-github-nebulouslabs-bolt (1.0+git20170131.0.ca9f208-1) unstable; urgency=medium
20	21

-0

errors.go less more

68	68	// non-bucket key on an existing bucket key.
69	69	ErrIncompatibleValue = errors.New("incompatible value")
70	70	)
	71
	72	// MmapError represents an error resulting from a failed mmap call. Typically,
	73	// this error means that no further database writes will be possible. The most
	74	// common cause is insufficient disk space.
	75	type MmapError string
	76
	77	func (e MmapError) Error() string { return string(e) }

-1

simulation_test.go less more

6	6	"sync"
7	7	"testing"
8	8
9		"github.com/boltdb/bolt"
	9	"github.com/NebulousLabs/bolt"
10	10	)
11	11
12	12	func TestSimulate_1op_1p(t *testing.T) { testSimulate(t, 1, 1) }

+12

-0

tx.go less more

361	361	return err
362	362	}
363	363	return f.Close()
	364	}
	365
	366	// FlushDBPages unmaps and re-mmaps the db file, flushing all of the DB's
	367	// pages from resident memory. It returns an error if called from a read-only
	368	// transaction.
	369	func (tx *Tx) FlushDBPages() error {
	370	if tx.db == nil {
	371	return ErrTxClosed
	372	} else if !tx.writable {
	373	return ErrTxNotWritable
	374	}
	375	return tx.db.mmap(tx.db.datasz)
364	376	}
365	377
366	378	// Check performs several consistency checks on the database for this transaction.

+58

-1

tx_test.go less more

7	7	"os"
8	8	"testing"
9	9
10		"github.com/boltdb/bolt"
	10	"github.com/NebulousLabs/bolt"
11	11	)
12	12
13	13	// Ensure that committing a closed transaction returns an error.

526	526	}
527	527	}
528	528
	529	// Ensure that the database can flush pages during normal operation without
	530	// crashing.
	531	func TestTx_FlushDBPages(t *testing.T) {
	532	db := MustOpenDB()
	533	defer db.MustClose()
	534
	535	if err := db.Update(func(tx *bolt.Tx) error {
	536	b, err := tx.CreateBucket([]byte("widgets"))
	537	if err != nil {
	538	t.Fatal(err)
	539	}
	540	if err := b.Put([]byte("foo"), []byte("bar")); err != nil {
	541	t.Fatal(err)
	542	}
	543	if err := b.Put([]byte("baz"), []byte("bat")); err != nil {
	544	t.Fatal(err)
	545	}
	546	return nil
	547	}); err != nil {
	548	t.Fatal(err)
	549	}
	550
	551	if err := db.Update(func(tx *bolt.Tx) error {
	552	return tx.FlushDBPages()
	553	}); err != nil {
	554	t.Fatal(err)
	555	}
	556
	557	// Flush concurrently with a read and write
	558	errs := make(chan error)
	559	go func() {
	560	errs <- db.Update(func(tx *bolt.Tx) error {
	561	return tx.FlushDBPages()
	562	})
	563	}()
	564	go func() {
	565	errs <- db.Update(func(tx *bolt.Tx) error {
	566	return tx.Bucket([]byte("widgets")).Put([]byte("foo"), []byte("quux"))
	567	})
	568	}()
	569	go func() {
	570	errs <- db.View(func(tx *bolt.Tx) error {
	571	_ = tx.Bucket([]byte("widgets")).Get([]byte("foo"))
	572	return nil
	573	})
	574	}()
	575	for i := 0; i < 3; i++ {
	576	if err := <-errs; err != nil {
	577	t.Fatal(err)
	578	}
	579	}
	580
	581	if err := db.Close(); err != nil {
	582	t.Fatal(err)
	583	}
	584	}
	585
529	586	type failWriterError struct{}
530	587
531	588	func (failWriterError) Error() string {