Skip to content

Commit d8ec4f9

Browse files
benbjohnsonbenbjohnson
committed
Merge pull request boltdb#128 from benbjohnson/import-export
Add import/export to CLI.
2 parents e9c8f14 + e6b5fdc commit d8ec4f9

File tree

11 files changed

+310
-24
lines changed

11 files changed

+310
-24
lines changed

‎Makefile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ test: fmt errcheck
4242
@go test -v -cover -test.run=$(TEST)
4343
@echo ""
4444
@echo ""
45+
@echo "=== CLI ==="
46+
@go test -v -test.run=$(TEST) ./cmd/bolt
47+
@echo ""
48+
@echo ""
4549
@echo "=== RACE DETECTOR ==="
4650
@go test -v -race -test.run=Parallel
4751

‎cmd/bolt/buckets_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,15 @@ import (
1111
// Ensure that a list of buckets can be retrieved.
1212
func TestBuckets(t *testing.T) {
1313
SetTestMode(true)
14-
open(func(db *bolt.DB) {
14+
open(func(db *bolt.DB, path string) {
1515
db.Update(func(tx *bolt.Tx) error {
1616
tx.CreateBucket("woojits")
1717
tx.CreateBucket("widgets")
1818
tx.CreateBucket("whatchits")
1919
return nil
2020
})
21-
output := run("buckets", db.Path())
21+
db.Close()
22+
output := run("buckets", path)
2223
assert.Equal(t, "whatchits\nwidgets\nwoojits", output)
2324
})
2425
}

‎cmd/bolt/export.go

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"os"
7+
8+
"github.com/boltdb/bolt"
9+
)
10+
11+
// Export exports the entire database as a JSON document.
12+
func Export(path string) {
13+
if _, err := os.Stat(path); os.IsNotExist(err) {
14+
fatal(err)
15+
return
16+
}
17+
18+
// Open the database.
19+
db, err := bolt.Open(path, 0600)
20+
if err != nil {
21+
fatal(err)
22+
return
23+
}
24+
defer db.Close()
25+
26+
db.View(func(tx *bolt.Tx) error {
27+
// Loop over every bucket and export it as a raw message.
28+
var root []*rawMessage
29+
for _, b := range tx.Buckets() {
30+
message, err := exportBucket(b)
31+
if err != nil {
32+
fatal(err)
33+
}
34+
root = append(root, message)
35+
}
36+
37+
// Encode all buckets into JSON.
38+
output, err := json.Marshal(root)
39+
if err != nil {
40+
fatal("encode: ", err)
41+
}
42+
print(string(output))
43+
return nil
44+
})
45+
}
46+
47+
func exportBucket(b *bolt.Bucket) (*rawMessage, error) {
48+
// Encode individual key/value pairs into raw messages.
49+
var children = make([]*rawMessage, 0)
50+
err := b.ForEach(func(k, v []byte) error {
51+
var err error
52+
53+
var child = &rawMessage{Key: k}
54+
if child.Value, err = json.Marshal(v); err != nil {
55+
return fmt.Errorf("value: %s", err)
56+
}
57+
58+
children = append(children, child)
59+
return nil
60+
})
61+
if err != nil {
62+
return nil, err
63+
}
64+
65+
// Encode bucket into a raw message.
66+
var root = rawMessage{Type: "bucket"}
67+
root.Key = []byte(b.Name())
68+
if root.Value, err = json.Marshal(children); err != nil {
69+
return nil, fmt.Errorf("children: %s", err)
70+
}
71+
72+
return &root, nil
73+
}

‎cmd/bolt/export_test.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package main_test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/boltdb/bolt"
7+
. "github.com/boltdb/bolt/cmd/bolt"
8+
"github.com/stretchr/testify/assert"
9+
)
10+
11+
// Ensure that a database can be exported.
12+
func TestExport(t *testing.T) {
13+
SetTestMode(true)
14+
open(func(db *bolt.DB, path string) {
15+
db.Update(func(tx *bolt.Tx) error {
16+
tx.CreateBucket("widgets")
17+
b := tx.Bucket("widgets")
18+
b.Put([]byte("foo"), []byte("0000"))
19+
b.Put([]byte("bar"), []byte(""))
20+
21+
tx.CreateBucket("woojits")
22+
b = tx.Bucket("woojits")
23+
b.Put([]byte("baz"), []byte("XXXX"))
24+
return nil
25+
})
26+
db.Close()
27+
output := run("export", path)
28+
assert.Equal(t, `[{"type":"bucket","key":"d2lkZ2V0cw==","value":[{"key":"YmFy","value":""},{"key":"Zm9v","value":"MDAwMA=="}]},{"type":"bucket","key":"d29vaml0cw==","value":[{"key":"YmF6","value":"WFhYWA=="}]}]`, output)
29+
})
30+
}
31+
32+
// Ensure that an error is reported if the database is not found.
33+
func TestExport_NotFound(t *testing.T) {
34+
SetTestMode(true)
35+
output := run("export", "no/such/db")
36+
assert.Equal(t, "stat no/such/db: no such file or directory", output)
37+
}

‎cmd/bolt/get_test.go

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,14 @@ import (
1111
// Ensure that a value can be retrieved from the CLI.
1212
func TestGet(t *testing.T) {
1313
SetTestMode(true)
14-
open(func(db *bolt.DB) {
14+
open(func(db *bolt.DB, path string) {
1515
db.Update(func(tx *bolt.Tx) error {
1616
tx.CreateBucket("widgets")
1717
tx.Bucket("widgets").Put([]byte("foo"), []byte("bar"))
1818
return nil
1919
})
20-
output := run("get", db.Path(), "widgets", "foo")
20+
db.Close()
21+
output := run("get", path, "widgets", "foo")
2122
assert.Equal(t, "bar", output)
2223
})
2324
}
@@ -32,20 +33,22 @@ func TestGetDBNotFound(t *testing.T) {
3233
// Ensure that an error is reported if the bucket is not found.
3334
func TestGetBucketNotFound(t *testing.T) {
3435
SetTestMode(true)
35-
open(func(db *bolt.DB) {
36-
output := run("get", db.Path(), "widgets", "foo")
36+
open(func(db *bolt.DB, path string) {
37+
db.Close()
38+
output := run("get", path, "widgets", "foo")
3739
assert.Equal(t, "bucket not found: widgets", output)
3840
})
3941
}
4042

4143
// Ensure that an error is reported if the key is not found.
4244
func TestGetKeyNotFound(t *testing.T) {
4345
SetTestMode(true)
44-
open(func(db *bolt.DB) {
46+
open(func(db *bolt.DB, path string) {
4547
db.Update(func(tx *bolt.Tx) error {
4648
return tx.CreateBucket("widgets")
4749
})
48-
output := run("get", db.Path(), "widgets", "foo")
50+
db.Close()
51+
output := run("get", path, "widgets", "foo")
4952
assert.Equal(t, "key not found: foo", output)
5053
})
5154
}

‎cmd/bolt/import.go

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"os"
7+
8+
"github.com/boltdb/bolt"
9+
)
10+
11+
// Import converts an exported database dump into a new database.
12+
func Import(path string, input string) {
13+
f, err := os.Open(input)
14+
if err != nil {
15+
fatal(err)
16+
return
17+
}
18+
defer f.Close()
19+
20+
// Read in entire dump.
21+
var root []*rawMessage
22+
if err := json.NewDecoder(f).Decode(&root); err != nil {
23+
fatal(err)
24+
}
25+
26+
// Open the database.
27+
db, err := bolt.Open(path, 0600)
28+
if err != nil {
29+
fatal(err)
30+
return
31+
}
32+
defer db.Close()
33+
34+
// Insert entire dump into database.
35+
err = db.Update(func(tx *bolt.Tx) error {
36+
// Loop over every message and create a bucket.
37+
for _, message := range root {
38+
// Validate that root messages are buckets.
39+
if message.Type != "bucket" {
40+
return fmt.Errorf("invalid root type: %q", message.Type)
41+
}
42+
43+
// Create the bucket if it doesn't exist.
44+
if err := tx.CreateBucketIfNotExists(string(message.Key)); err != nil {
45+
return fmt.Errorf("create bucket: %s", err)
46+
}
47+
48+
// Decode child messages.
49+
var children []*rawMessage
50+
if err := json.Unmarshal(message.Value, &children); err != nil {
51+
return fmt.Errorf("decode children: %s", err)
52+
}
53+
54+
// Import all the values into the bucket.
55+
b := tx.Bucket(string(message.Key))
56+
if err := importBucket(b, children); err != nil {
57+
return fmt.Errorf("import bucket: %s", err)
58+
}
59+
}
60+
return nil
61+
})
62+
if err != nil {
63+
fatal("update: ", err)
64+
}
65+
}
66+
67+
func importBucket(b *bolt.Bucket, children []*rawMessage) error {
68+
// Decode each message into a key/value pair.
69+
for _, child := range children {
70+
// Decode the base64 value.
71+
var value []byte
72+
if err := json.Unmarshal(child.Value, &value); err != nil {
73+
return fmt.Errorf("decode value: %s", err)
74+
}
75+
76+
// Insert key/value into bucket.
77+
if err := b.Put(child.Key, value); err != nil {
78+
return fmt.Errorf("put: %s", err)
79+
}
80+
}
81+
return nil
82+
}

‎cmd/bolt/import_test.go

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package main_test
2+
3+
import (
4+
"io/ioutil"
5+
"testing"
6+
7+
"github.com/boltdb/bolt"
8+
. "github.com/boltdb/bolt/cmd/bolt"
9+
"github.com/stretchr/testify/assert"
10+
)
11+
12+
// Ensure that a database can be imported.
13+
func TestImport(t *testing.T) {
14+
SetTestMode(true)
15+
16+
// Write input file.
17+
input := tempfile()
18+
assert.NoError(t, ioutil.WriteFile(input, []byte(`[{"type":"bucket","key":"d2lkZ2V0cw==","value":[{"key":"YmFy","value":""},{"key":"Zm9v","value":"MDAwMA=="}]},{"type":"bucket","key":"d29vaml0cw==","value":[{"key":"YmF6","value":"WFhYWA=="}]}]`), 0600))
19+
20+
// Import database.
21+
path := tempfile()
22+
output := run("import", path, "--input", input)
23+
assert.Equal(t, ``, output)
24+
25+
// Open database and verify contents.
26+
db, err := bolt.Open(path, 0600)
27+
assert.NoError(t, err)
28+
db.View(func(tx *bolt.Tx) error {
29+
b := tx.Bucket("widgets")
30+
if assert.NotNil(t, b) {
31+
assert.Equal(t, []byte("0000"), b.Get([]byte("foo")))
32+
assert.Equal(t, []byte(""), b.Get([]byte("bar")))
33+
}
34+
35+
b = tx.Bucket("woojits")
36+
if assert.NotNil(t, b) {
37+
assert.Equal(t, []byte("XXXX"), b.Get([]byte("baz")))
38+
}
39+
40+
return nil
41+
})
42+
db.Close()
43+
}
44+
45+
// Ensure that an error is reported if the database is not found.
46+
func TestImport_NotFound(t *testing.T) {
47+
SetTestMode(true)
48+
output := run("import", "path/to/db", "--input", "no/such/file")
49+
assert.Equal(t, "open no/such/file: no such file or directory", output)
50+
}

‎cmd/bolt/keys_test.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,16 @@ import (
1111
// Ensure that a list of keys can be retrieved for a given bucket.
1212
func TestKeys(t *testing.T) {
1313
SetTestMode(true)
14-
open(func(db *bolt.DB) {
14+
open(func(db *bolt.DB, path string) {
1515
db.Update(func(tx *bolt.Tx) error {
1616
tx.CreateBucket("widgets")
1717
tx.Bucket("widgets").Put([]byte("0002"), []byte(""))
1818
tx.Bucket("widgets").Put([]byte("0001"), []byte(""))
1919
tx.Bucket("widgets").Put([]byte("0003"), []byte(""))
2020
return nil
2121
})
22-
output := run("keys", db.Path(), "widgets")
22+
db.Close()
23+
output := run("keys", path, "widgets")
2324
assert.Equal(t, "0001\n0002\n0003", output)
2425
})
2526
}
@@ -34,8 +35,9 @@ func TestKeysDBNotFound(t *testing.T) {
3435
// Ensure that an error is reported if the bucket is not found.
3536
func TestKeysBucketNotFound(t *testing.T) {
3637
SetTestMode(true)
37-
open(func(db *bolt.DB) {
38-
output := run("keys", db.Path(), "widgets")
38+
open(func(db *bolt.DB, path string) {
39+
db.Close()
40+
output := run("keys", path, "widgets")
3941
assert.Equal(t, "bucket not found: widgets", output)
4042
})
4143
}

‎cmd/bolt/main.go

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package main
22

33
import (
44
"bytes"
5+
"encoding/json"
56
"fmt"
67
"log"
78
"os"
@@ -55,6 +56,24 @@ func NewApp() *cli.App {
5556
Buckets(path)
5657
},
5758
},
59+
{
60+
Name: "import",
61+
Usage: "Imports from a JSON dump into a database",
62+
Flags: []cli.Flag{
63+
&cli.StringFlag{Name: "input"},
64+
},
65+
Action: func(c *cli.Context) {
66+
Import(c.Args().Get(0), c.String("input"))
67+
},
68+
},
69+
{
70+
Name: "export",
71+
Usage: "Exports a database to JSON",
72+
Action: func(c *cli.Context) {
73+
path := c.Args().Get(0)
74+
Export(path)
75+
},
76+
},
5877
{
5978
Name: "pages",
6079
Usage: "Dumps page information for a database",
@@ -144,3 +163,10 @@ func SetTestMode(value bool) {
144163
logger = log.New(os.Stderr, "", 0)
145164
}
146165
}
166+
167+
// rawMessage represents a JSON element in the import/export document.
168+
type rawMessage struct {
169+
Type string `json:"type,omitempty"`
170+
Key []byte `json:"key"`
171+
Value json.RawMessage `json:"value"`
172+
}

0 commit comments

Comments
 (0)