dockage

README

dockage

This is an embedded document/json store based on badger key-value store.

word

• Document id must be database-wide unique. There is no notion of grouping documents (tables, collections, buckets, etc).
• id and rev are mandatory fields that must be present inside document json.

put document

First, make sure to close the db when it is no longer needed:

db := createDB()
defer db.Close()

Database operations are safe to be done concurrently. In fact it is a characteristic of badger that writes will get a performance boost when done concurrently.

Define a struct that has id and rev json tags. These two fields are not optional.

type post struct {
    ID   string    `json:"id"`
    Rev  string    `json:"rev"`
    By   string    `json:"by,omitempty"`
    Text string    `json:"text,omitempty"`
    At   time.Time `json:"at,omitempty"`
    Tags []string  `json:"tags,omitempty"`
}

Now to put a post document inside the database:

p := &post{
    ID:   "POST:001",
    By:   "Frodo Baggins",
    Text: "Awesome blog post!",
    At:   time.Now(),
    Tags: []string{"golang", "nosql"},
}

db.Put(p)

get document

Documents can be read from database, using their id.

var result []post
db.Get(&result, "POST:001")

delete document

Deleting documents is also done using their id.

db.Delete("POST:001")

Do not set the field with rev tag. It is used for optimistic concurrency and is filled automatically by Put() method.

query posts by day

Queries can be performed using Views, which are predefined queries. Queries must be defined right after opening the database.

This View emits the day of posts:

db.AddView(NewView("time-day",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*post)
		if !ok {
			return
		}
		em.Emit([]byte(c.At.Format("2006-01-02")), nil)
		return
	}))

View functions will be executed after put operations, inside the same write transaction, right before commit. So the Views are also consistent. An Emitter, the id of the document and the document itself is passed to View function. time-day is a View that only indexes posts and ignores other type of documents.

Now we can query this view:

db.Query(Q{View: "time-day", Start: []byte("2018-05-20"), End: []byte("2018-05-30")})

Queries return the document id (view key) and other stuff generated by the view. For querying all documents, leave View field empty.

query posts by tag

Nice thing about views is that it is possible to index and query nested structs.

Let's define a view that allows us to query posts based on their tags:

db.AddView(NewView("tags",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*post)
		if !ok {
			return
		}
		for _, v := range c.Tags {
			em.Emit([]byte(v), nil)
		}
		return
    }))

Here Emit() method is called multiple times, for each tag.

To query and find posts that are tagged with golang:

db.Query(Q{View: "tags", Start: []byte("golang"), Prefix: []byte("golang")})

Prefix is set because we do not need tags greater than golang - like gozoo!

Documentation

Overview

Package dockage is an embedded document (json) database.

Index

• Variables
• type DB
  • func Open(opt Options) (resdb *DB, reserr error)
  • func (db *DB) AddView(v View)
  • func (db *DB) Close() error
  • func (db *DB) Delete(ids ...string) (reserr error)
  • func (db *DB) DeleteView(v string) (reserr error)
  • func (db *DB) Get(docs interface{}, firstID string, restID ...string) (reserr error)
  • func (db *DB) Put(docs ...interface{}) (reserr error)
  • func (db *DB) Query(params Q) (reslist []Res, rescount int, reserr error)
• type Emitter
• type KV
• type Options
• type Q
• type Res
• type View
  • func NewView(name string, viewFn ViewFn) (resview View)
• type ViewFn

Examples

• DB (Cas)
• DB (Doc1)
• DB.Delete
• DB.Get
• DB.Put
• View
• View (ByTime)
• View (Count)
• View (End)
• View (EndAll)
• View (Limit)
• View (Skip)
• View (TimestampInt64)
• View (ViewVal)

Variables

var (
	ErrNoID      = errors.New("no id field in doc json")
	ErrNoRev     = errors.New("no rev field in doc json")
	ErrInvalidID = fmt.Errorf("id must not contain these characters: %s %s %s %s %s",
		viewsp,
		keysp,
		syssp,
		viewk2x,
		viewx2k)
	ErrNoMatchRev = errors.New("rev field in doc json not matching")
)

errors

Types

type DB

type DB struct {
	// contains filtered or unexported fields
}

DB represents a database instance.

Example (Cas)

// CAS is performed using a mandatory rev field inside json document.

db := createDB()
defer db.Close()

docID := "CMNT::001"

cmnt := &comment{
	ID:   docID,
	By:   "Frodo Baggins",
	Text: "Hi!",
	At:   time.Now(),
	Tags: []string{"tech", "golang"},
}

fmt.Println(db.Put(cmnt))

var res []comment
err := db.Get(&res, docID)
fmt.Println(err)
*cmnt = res[0]

fmt.Println(cmnt.ID)
fmt.Println(cmnt.By)
fmt.Println(cmnt.Text)
fmt.Println(cmnt.Tags)
fmt.Println(cmnt.Rev)

rev := cmnt.Rev
cmnt.Rev = "dummy"
fmt.Println("error:", db.Put(cmnt))

cmnt.Rev = rev
cmnt.Text = "Back again!"
fmt.Println(db.Put(cmnt))

res = nil
err = db.Get(&res, docID)
fmt.Println(err)
*cmnt = res[0]

fmt.Println(cmnt.ID)
fmt.Println(cmnt.By)
fmt.Println(cmnt.Text)
fmt.Println(cmnt.Tags)
fmt.Println(cmnt.Rev)

Output:

<nil>
<nil>
CMNT::001
Frodo Baggins
Hi!
[tech golang]
0000000000000000
error: rev field in doc json not matching
<nil>
<nil>
CMNT::001
Frodo Baggins
Back again!
[tech golang]
0000000000000001

Example (Doc1)

db := createDB()
defer db.Close()

type post struct {
	ID   string    `json:"id"`
	Rev  string    `json:"rev"`
	By   string    `json:"by,omitempty"`
	Text string    `json:"text,omitempty"`
	At   time.Time `json:"at,omitempty"`
	Tags []string  `json:"tags,omitempty"`
}

db.AddView(NewView("tags",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*post)
		if !ok {
			return
		}
		for _, v := range c.Tags {
			em.Emit([]byte(v), nil)
		}
		return
	}))
db.AddView(NewView("time-day",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*post)
		if !ok {
			return
		}
		em.Emit([]byte(c.At.Format("2006-01-02")), nil)
		return
	}))

p := &post{
	ID:   "POST:001",
	By:   "Frodo Baggins",
	Text: "Awesome blog post!",
	At:   time.Now(),
	Tags: []string{"golang", "nosql"},
}

db.Put(p)

var result []post
db.Get(&result, "POST:001")

db.Delete("POST:001")

db.Query(Q{View: "time-day", Start: []byte("2018-05-20"), End: []byte("2018-05-30")})

db.Query(Q{View: "tags", Start: []byte("golang"), Prefix: []byte("golang")})

fmt.Println(p.ID)

Output:

POST:001

func Open

func Open(opt Options) (resdb *DB, reserr error)

Open opens the database with provided options.

func (*DB) AddView

func (db *DB) AddView(v View)

AddView adds a view. All views must be added right after Open(...). It is not safe to call this method concurrently.

func (*DB) Close

func (db *DB) Close() error

Close closes the database.

func (*DB) Delete

func (db *DB) Delete(ids ...string) (reserr error)

Delete a list of documents based on their ids. All documents will be deleted from database in one write transaction.

Example

db := createDB()
defer db.Close()

cmnt := &comment{
	ID:   "CMNT::001",
	By:   "Frodo Baggins",
	Text: "Hi!",
	At:   time.Now(),
	Tags: []string{"tech", "golang"},
}

fmt.Println(db.Put(cmnt))

fmt.Println(db.Delete("CMNT::001"))

var res []comment
err := db.Get(&res, "CMNT::001")
fmt.Println(err, res)

Output:

<nil>
<nil>
Key not found []

func (*DB) DeleteView

func (db *DB) DeleteView(v string) (reserr error)

DeleteView deletes the data of a view.

func (*DB) Get

func (db *DB) Get(docs interface{}, firstID string, restID ...string) (reserr error)

Get a list of documents based on their ids. Param docs is pointer to slice of struct. All documents will be read from database in one read transaction.

Example

db := createDB()
defer db.Close()

var list []interface{}
for i := 1; i <= 3; i++ {
	cmnt := comment{
		ID:   fmt.Sprintf("CMNT::%03d", i),
		By:   "Frodo Baggins",
		Text: "Hi!",
		At:   time.Now(),
		Tags: []string{"tech", "golang"},
	}
	list = append(list, cmnt)
}
fmt.Println(db.Put(list...))

var res []comment
err := db.Get(&res,
	"CMNT::001",
	"CMNT::002",
	"CMNT::003")

fmt.Println(err)

for _, v := range res {
	fmt.Printf("%s %s %s\n", v.ID, v.Text, v.By)
}

Output:

<nil>
<nil>
CMNT::001 Hi! Frodo Baggins
CMNT::002 Hi! Frodo Baggins
CMNT::003 Hi! Frodo Baggins

func (*DB) Put

func (db *DB) Put(docs ...interface{}) (reserr error)

Put a list of documents inside database, in a single transaction. Document must have a json field named "id" and a json field named "rev". All documents passed by docs parameter will be inserted into the database in one transaction. Also all views will be computer in the same transaction.

Example

db := createDB()
defer db.Close()

cmnt := comment{
	ID:   "CMNT::001",
	By:   "Frodo Baggins",
	Text: "Hi!",
	At:   time.Now(),
	Tags: []string{"tech", "golang"},
}

fmt.Println(db.Put(cmnt))

var res []comment
err := db.Get(&res, "CMNT::001")
fmt.Println(err)

cmnt = res[0]

fmt.Println(cmnt.ID)
fmt.Println(cmnt.By)
fmt.Println(cmnt.Text)
fmt.Println(cmnt.Tags)

Output:

<nil>
<nil>
CMNT::001
Frodo Baggins
Hi!
[tech golang]

func (*DB) Query

func (db *DB) Query(params Q) (reslist []Res, rescount int, reserr error)

Query queries a view using provided parameters. If no View is provided, it searches all ids using parameters. Number of results is always limited - default 100 documents. If total count for a query is needed by setting params.Count to true, no documents will be returned - because it might be a costly action. All documents will be read from database in one read transaction.

type Emitter

type Emitter interface {
	Emit(viewKey, viewValue []byte)
}

Emitter .

type KV

type KV struct {
	Key, Val []byte
}

KV tuple.

type Options

type Options struct {
	// 1. Mandatory flags
	// -------------------
	// Directory to store the data in. Should exist and be writable.
	Dir string
	// Directory to store the value log in. Can be the same as Dir. Should
	// exist and be writable.
	ValueDir string
}

Options are params for creating DB object.

type Q

type Q struct {
	View               string
	Start, End, Prefix []byte
	Skip, Limit        int
	Count              bool
}

Q query parameters

type Res

type Res struct {
	KV
	Index []byte
}

Res represents the result of a Query(...) call. Key is the document key, Index is the calculated index and Val is the calculated value by the view.

type View

type View struct {
	// contains filtered or unexported fields
}

View is a calculated, persistent index.

Example

db := createDB()
defer db.Close()

// em Emitter allows to emit our view index into the view, in this case
// the tags of a comment. By emitting tags one by one, it is possible
// to query comments based on their tags.

db.AddView(NewView("tags",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*comment)
		if !ok {
			return
		}
		for _, v := range c.Tags {
			em.Emit([]byte(v), nil)
		}
	}))

var list []interface{}
for i := 1; i <= 3; i++ {
	cmnt := &comment{
		ID:   fmt.Sprintf("CMNT::%03d", i),
		By:   "Frodo Baggins",
		Text: "Hi!",
		At:   time.Now(),
		Tags: []string{"tech", "golang"},
	}
	list = append(list, cmnt)
}
fmt.Println(db.Put(list...))

res, _, err := db.Query(Q{View: "tags", Start: []byte("tech")})
fmt.Println(err)

for _, v := range res {
	fmt.Printf("%s %s %s\n", v.Key, v.Val, v.Index)
}

Output:

<nil>
<nil>
CMNT::001  tech
CMNT::002  tech
CMNT::003  tech

Example (ByTime)

db := createDB()
defer db.Close()

db.AddView(NewView("by_time",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*comment)
		if !ok {
			return
		}
		t := c.At
		if t.IsZero() {
			return
		}
		em.Emit([]byte(t.Format("2006-01-02")), nil)
		return
	}))

at := time.Date(2018, 1, 1, 12, 0, 0, 0, time.Local)
var list []interface{}
for i := 1; i <= 3; i++ {
	cmnt := &comment{
		ID:   fmt.Sprintf("CMNT::%03d", i),
		By:   "Frodo Baggins",
		Text: "Hi!",
		At:   at,
		Tags: []string{"tech", "golang"},
	}
	list = append(list, cmnt)
	at = at.Add(time.Hour * 24)
}
fmt.Println(db.Put(list...))

start := []byte("2018-01-01")
prefix := []byte("2018-01")
res, _, err := db.Query(Q{View: "by_time", Start: start, Prefix: prefix})
fmt.Println(err)

for _, v := range res {
	fmt.Printf("%s %s %s\n", v.Key, v.Val, v.Index)
}

Output:

<nil>
<nil>
CMNT::001  2018-01-01
CMNT::002  2018-01-02
CMNT::003  2018-01-03

Example (Count)

db := createDB()
defer db.Close()

db.AddView(NewView("tags",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(comment)
		if !ok {
			return
		}
		for _, v := range c.Tags {
			em.Emit([]byte(v), nil)
		}
		return
	}))

var list []interface{}
for i := 1; i <= 3; i++ {
	cmnt := comment{
		ID:   fmt.Sprintf("CMNT::%03d", i),
		By:   "Frodo Baggins",
		Text: "Hi!",
		At:   time.Now(),
		Tags: []string{"tech", "golang"},
	}
	list = append(list, cmnt)
}
fmt.Println(db.Put(list...))

res, cnt, err := db.Query(Q{View: "tags", Start: []byte("tech"), Count: true})
fmt.Println(err)
fmt.Println(len(res))
fmt.Println(cnt)

Output:

<nil>
<nil>
0
3

Example (End)

db := createDB()
defer db.Close()

db.AddView(NewView("by_time",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*comment)
		if !ok {
			return
		}
		t := c.At
		if t.IsZero() {
			return
		}
		em.Emit([]byte(t.Format("2006-01-02")), nil)
		return
	}))

at := time.Date(2018, 1, 1, 12, 0, 0, 0, time.Local)
var list []interface{}
for i := 1; i <= 3; i++ {
	cmnt := &comment{
		ID:   fmt.Sprintf("CMNT::%03d", i),
		By:   "Frodo Baggins",
		Text: "Hi!",
		At:   at,
		Tags: []string{"tech", "golang"},
	}
	list = append(list, cmnt)
	at = at.Add(time.Hour * 24)
}
fmt.Println(db.Put(list...))

start := []byte("2018-01-01")
prefix := []byte("2018-01")
end := []byte("2018-01-03") // exclusive
res, _, err := db.Query(Q{View: "by_time", Start: start, Prefix: prefix, End: end})
fmt.Println(err)

for _, v := range res {
	fmt.Printf("%s %s %s\n", v.Key, v.Val, v.Index)
}

Output:

<nil>
<nil>
CMNT::001  2018-01-01
CMNT::002  2018-01-02

Example (EndAll)

db := createDB()
defer db.Close()

db.AddView(NewView("by_time",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*comment)
		if !ok {
			return
		}
		t := c.At
		if t.IsZero() {
			return
		}
		em.Emit([]byte(t.Format("2006-01-02")), []byte(c.By))
		return
	}))

at := time.Date(2018, 1, 1, 12, 0, 0, 0, time.Local)
var list []interface{}
for i := 1; i <= 3; i++ {
	cmnt := &comment{
		ID:   fmt.Sprintf("CMNT::%03d", i),
		By:   "Frodo Baggins",
		Text: "Hi!",
		At:   at,
		Tags: []string{"tech", "golang"},
	}
	list = append(list, cmnt)
	at = at.Add(time.Hour * 24)
}
fmt.Println(db.Put(list...))

start := []byte("2018-01-01")
prefix := []byte("2018-01")
end := []byte("2018-01\uffff")
res, _, err := db.Query(Q{View: "by_time", Start: start, Prefix: prefix, End: end})
fmt.Println(err)

for _, v := range res {
	fmt.Printf("%s %s %s\n", v.Key, v.Val, v.Index)
}

Output:

<nil>
<nil>
CMNT::001 Frodo Baggins 2018-01-01
CMNT::002 Frodo Baggins 2018-01-02
CMNT::003 Frodo Baggins 2018-01-03

Example (Limit)

db := createDB()
defer db.Close()

db.AddView(NewView("by_time",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*comment)
		if !ok {
			return
		}
		t := c.At
		if t.IsZero() {
			return
		}
		em.Emit([]byte(t.Format("2006-01-02")), nil)
		return
	}))

at := time.Date(2018, 1, 1, 12, 0, 0, 0, time.Local)
var list []interface{}
for i := 1; i <= 3; i++ {
	cmnt := &comment{
		ID:   fmt.Sprintf("CMNT::%03d", i),
		By:   "Frodo Baggins",
		Text: "Hi!",
		At:   at,
		Tags: []string{"tech", "golang"},
	}
	list = append(list, cmnt)
	at = at.Add(time.Hour * 24)
}
fmt.Println(db.Put(list...))

start := []byte("2018-01-01")
prefix := []byte("2018-01")
res, _, err := db.Query(Q{View: "by_time", Start: start, Prefix: prefix, Limit: 1})
fmt.Println(err)

for _, v := range res {
	fmt.Printf("%s %s %s\n", v.Key, v.Val, v.Index)
}

Output:

<nil>
<nil>
CMNT::001  2018-01-01

Example (Skip)

db := createDB()
defer db.Close()

db.AddView(NewView("by_time",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*comment)
		if !ok {
			return
		}
		t := c.At
		if t.IsZero() {
			return
		}
		em.Emit([]byte(t.Format("2006-01-02")), []byte(c.By))
		return
	}))

at := time.Date(2018, 1, 1, 12, 0, 0, 0, time.Local)
var list []interface{}
for i := 1; i <= 3; i++ {
	cmnt := &comment{
		ID:   fmt.Sprintf("CMNT::%03d", i),
		By:   "Frodo Baggins",
		Text: "Hi!",
		At:   at,
		Tags: []string{"tech", "golang"},
	}
	list = append(list, cmnt)
	at = at.Add(time.Hour * 24)
}
fmt.Println(db.Put(list...))

start := []byte("2018-01-01")
prefix := []byte("2018-01")
end := []byte("2018-01\uffff")
res, _, err := db.Query(Q{View: "by_time", Start: start, Prefix: prefix, End: end, Skip: 1})
fmt.Println(err)

for _, v := range res {
	fmt.Printf("%s %s %s\n", v.Key, v.Val, v.Index)
}

Output:

<nil>
<nil>
CMNT::002 Frodo Baggins 2018-01-02
CMNT::003 Frodo Baggins 2018-01-03

Example (TimestampInt64)

db := createDB()
defer db.Close()

type comment struct {
	ID   string   `json:"id"`
	Rev  string   `json:"rev"`
	By   string   `json:"by,omitempty"`
	Text string   `json:"text,omitempty"`
	At   int64    `json:"at,omitempty"`
	Tags []string `json:"tags,omitempty"`
}

db.AddView(NewView("by_time",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*comment)
		if !ok {
			return
		}
		t := c.At
		ts := make([]byte, 8)
		binary.BigEndian.PutUint64(ts, uint64(t))
		em.Emit(ts, nil)
		return
	}))

at := time.Date(2018, 1, 1, 12, 0, 0, 0, time.Local)
startTS := at.Unix()
first := startTS
var list []interface{}
for i := 1; i <= 3; i++ {
	cmnt := &comment{
		ID:   fmt.Sprintf("CMNT::%03d", i),
		By:   "Frodo Baggins",
		Text: "Hi!",
		At:   startTS,
		Tags: []string{"tech", "golang"},
	}
	list = append(list, cmnt)
	at = at.Add(time.Hour * 24)
	startTS = at.Unix()
}
fmt.Println(db.Put(list...))

start := make([]byte, 8)
binary.BigEndian.PutUint64(start, uint64(first))
res, _, err := db.Query(Q{View: "by_time", Start: start})
fmt.Println(err)

for _, v := range res {
	fmt.Printf("%s %s %x\n", v.Key, v.Val, v.Index)
}

Output:

<nil>
<nil>
CMNT::001  000000005a49f188
CMNT::002  000000005a4b4308
CMNT::003  000000005a4c9488

Example (ViewVal)

db := createDB()
defer db.Close()

db.AddView(NewView("by_time",
	func(em Emitter, id string, doc interface{}) {
		c, ok := doc.(*comment)
		if !ok {
			return
		}
		t := c.At
		if t.IsZero() {
			return
		}
		em.Emit([]byte(t.Format("2006-01-02")), []byte(c.By))
		return
	}))

at := time.Date(2018, 1, 1, 12, 0, 0, 0, time.Local)
var list []interface{}
for i := 1; i <= 3; i++ {
	cmnt := &comment{
		ID:   fmt.Sprintf("CMNT::%03d", i),
		By:   "Frodo Baggins",
		Text: "Hi!",
		At:   at,
		Tags: []string{"tech", "golang"},
	}
	list = append(list, cmnt)
	at = at.Add(time.Hour * 24)
}
fmt.Println(db.Put(list...))

start := []byte("2018-01-01")
prefix := []byte("2018-01")
res, _, err := db.Query(Q{View: "by_time", Start: start, Prefix: prefix})
fmt.Println(err)

for _, v := range res {
	fmt.Printf("%s %s %s\n", v.Key, v.Val, v.Index)
}

Output:

<nil>
<nil>
CMNT::001 Frodo Baggins 2018-01-01
CMNT::002 Frodo Baggins 2018-01-02
CMNT::003 Frodo Baggins 2018-01-03

func NewView

func NewView(name string, viewFn ViewFn) (resview View)

NewView creates a new View. Function viewFn must have no side effects.

type ViewFn

type ViewFn func(emitter Emitter, id string, doc interface{})

ViewFn function that emits view keys (and json docs as view values).