storage

Documentation

Index

• Constants
• Variables
• func PutAll(ctx context.Context, store KeyValueStore, items ...ListItem) (err error)
• type BlobInfo
• type BlobReader
• type BlobRef
  • func (ref *BlobRef) IsValid() bool
• type BlobWriter
• type Blobs
• type FormatVersion
• type Items
  • func CloneItems(items Items) Items
  • func ListV2(ctx context.Context, store KeyValueStore, opts ListOptions) (result Items, more bool, err error)
  • func ReverseItems(items Items) Items
  • func SelectPrefixed(items Items, prefix []byte) Items
  • func SortAndCollapse(items Items, prefix []byte) Items
  • func (items Items) GetKeys() Keys
  • func (items Items) Len() int
  • func (items Items) Less(i, k int) bool
  • func (items Items) Swap(i, k int)
• type IterateOptions
• type Iterator
• type IteratorFunc
  • func (next IteratorFunc) Next(ctx context.Context, item *ListItem) bool
• type Key
  • func AfterPrefix(key Key) Key
  • func CloneKey(key Key) Key
  • func NextKey(key Key) Key
  • func (key Key) Equal(b Key) bool
  • func (key Key) IsZero() bool
  • func (key Key) Less(b Key) bool
  • func (key Key) MarshalBinary() ([]byte, error)
  • func (key Key) String() string
• type KeyValueStore
• type Keys
  • func ListKeys(ctx context.Context, store KeyValueStore, first Key, limit int) (_ Keys, err error)
  • func ReverseKeys(keys Keys) Keys
  • func (keys Keys) ByteSlices() [][]byte
  • func (keys Keys) Strings() []string
• type ListItem
  • func CloneItem(item ListItem) ListItem
  • func (item ListItem) Less(b ListItem) bool
• type ListOptions
• type StaticIterator
  • func (it *StaticIterator) Next(ctx context.Context, item *ListItem) bool
• type Value
  • func CloneValue(value Value) Value
  • func (value Value) IsZero() bool
  • func (value Value) MarshalBinary() ([]byte, error)
• type Values

Constants

const Delimiter = '/'

Delimiter separates nested paths in storage

const LookupLimit = 1000

LookupLimit is enforced by storage implementations

Variables

var ErrEmptyKey = errs.Class("empty key")

ErrEmptyKey is returned when an empty key is used in Put or in CompareAndSwap

var ErrEmptyQueue = errs.Class("empty queue")

ErrEmptyQueue is returned when attempting to Dequeue from an empty queue

var ErrInvalidBlobRef = errs.Class("invalid blob ref")

ErrInvalidBlobRef is returned when an blob reference is invalid

var ErrKeyNotFound = errs.Class("key not found")

ErrKeyNotFound used when something doesn't exist

var ErrLimitExceeded = errors.New("limit exceeded")

ErrLimitExceeded is returned when request limit is exceeded

var ErrValueChanged = errs.Class("value changed")

ErrValueChanged is returned when the current value of the key does not match the oldValue in CompareAndSwap

Functions

func PutAll

func PutAll(ctx context.Context, store KeyValueStore, items ...ListItem) (err error)

PutAll adds multiple values to the store

Types

type BlobInfo

type BlobInfo interface {
	// BlobRef returns the relevant BlobRef for the blob
	BlobRef() BlobRef
	// StorageFormatVersion indicates the storage format version used to store the piece
	StorageFormatVersion() FormatVersion
	// FullPath gives the full path to the on-disk blob file
	FullPath(ctx context.Context) (string, error)
	// Stat does a stat on the on-disk blob file
	Stat(ctx context.Context) (os.FileInfo, error)
}

BlobInfo allows lazy inspection of a blob and its underlying file during iteration with WalkNamespace-type methods

type BlobReader

type BlobReader interface {
	io.Reader
	io.ReaderAt
	io.Seeker
	io.Closer
	// Size returns the size of the blob
	Size() (int64, error)
	// StorageFormatVersion returns the storage format version associated with the blob.
	StorageFormatVersion() FormatVersion
}

BlobReader is an interface that groups Read, ReadAt, Seek and Close.

type BlobRef

type BlobRef struct {
	Namespace []byte
	Key       []byte
}

BlobRef is a reference to a blob

func (*BlobRef) IsValid

func (ref *BlobRef) IsValid() bool

IsValid returns whether both namespace and key are specified

type BlobWriter

type BlobWriter interface {
	io.Writer
	io.Seeker
	// Cancel discards the blob.
	Cancel(context.Context) error
	// Commit ensures that the blob is readable by others.
	Commit(context.Context) error
	// Size returns the size of the blob
	Size() (int64, error)
	// StorageFormatVersion returns the storage format version associated with the blob.
	StorageFormatVersion() FormatVersion
}

BlobWriter defines the interface that must be satisfied for a general blob storage provider. BlobWriter instances are returned by the Create() method on Blobs instances.

type Blobs

type Blobs interface {
	// Create creates a new blob that can be written
	// optionally takes a size argument for performance improvements, -1 is unknown size
	Create(ctx context.Context, ref BlobRef, size int64) (BlobWriter, error)
	// Open opens a reader with the specified namespace and key
	Open(ctx context.Context, ref BlobRef) (BlobReader, error)
	// OpenWithStorageFormat opens a reader for the already-located blob, avoiding the potential
	// need to check multiple storage formats to find the blob.
	OpenWithStorageFormat(ctx context.Context, ref BlobRef, formatVer FormatVersion) (BlobReader, error)
	// Delete deletes the blob with the namespace and key
	Delete(ctx context.Context, ref BlobRef) error
	// Stat looks up disk metadata on the blob file
	Stat(ctx context.Context, ref BlobRef) (BlobInfo, error)
	// StatWithStorageFormat looks up disk metadata for the blob file with the given storage format
	// version. This avoids the potential need to check multiple storage formats for the blob
	// when the format is already known.
	StatWithStorageFormat(ctx context.Context, ref BlobRef, formatVer FormatVersion) (BlobInfo, error)
	// FreeSpace return how much free space left for writing
	FreeSpace() (int64, error)
	// SpaceUsed adds up how much is used in all namespaces
	SpaceUsed(ctx context.Context) (int64, error)
	// SpaceUsedInNamespace adds up how much is used in the given namespace
	SpaceUsedInNamespace(ctx context.Context, namespace []byte) (int64, error)
	// ListNamespaces finds all namespaces in which keys might currently be stored.
	ListNamespaces(ctx context.Context) ([][]byte, error)
	// WalkNamespace executes walkFunc for each locally stored blob, stored with
	// storage format V1 or greater, in the given namespace. If walkFunc returns a non-nil
	// error, WalkNamespace will stop iterating and return the error immediately. The ctx
	// parameter is intended to allow canceling iteration early.
	WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(BlobInfo) error) error
}

Blobs is a blob storage interface

type FormatVersion

type FormatVersion int

FormatVersion represents differing storage format version values. Different Blobs implementors might interpret different FormatVersion values differently, but they share a type so that there can be a common StorageFormatVersion() call on the interface.

Changes in FormatVersion might affect how a Blobs or BlobReader or BlobWriter instance works, or they might only be relevant to some higher layer. A FormatVersion must be specified when writing a new blob, and the blob storage interface must store that value with the blob somehow, so that the same FormatVersion is returned later when reading that stored blob.

type Items

type Items []ListItem

Items keeps all ListItem

func CloneItems

func CloneItems(items Items) Items

CloneItems creates a deep copy of items

func ListV2

func ListV2(ctx context.Context, store KeyValueStore, opts ListOptions) (result Items, more bool, err error)

ListV2 lists all keys corresponding to ListOptions limit is capped to LookupLimit

more indicates if the result was truncated. If false then the result []ListItem includes all requested keys. If true then the caller must call List again to get more results by setting `StartAfter` or `EndBefore` appropriately.

func ReverseItems

func ReverseItems(items Items) Items

ReverseItems reverses items in the list items will be reused and modified TODO: remove this

func SelectPrefixed

func SelectPrefixed(items Items, prefix []byte) Items

SelectPrefixed keeps only items that have prefix items will be reused and modified TODO: remove this

func SortAndCollapse

func SortAndCollapse(items Items, prefix []byte) Items

SortAndCollapse sorts items and combines elements based on Delimiter items will be reused and modified TODO: remove this

func (Items) GetKeys

func (items Items) GetKeys() Keys

GetKeys gets all the Keys in []ListItem and converts them to Keys

func (Items) Len

func (items Items) Len() int

Len is the number of elements in the collection.

func (Items) Less

func (items Items) Less(i, k int) bool

Less reports whether the element with index i should sort before the element with index j.

func (Items) Swap

func (items Items) Swap(i, k int)

Swap swaps the elements with indexes i and j.

type IterateOptions

type IterateOptions struct {
	// Prefix ensure
	Prefix Key
	// First will be the first item iterator returns or the next item (previous when reverse)
	First Key
	// Recurse, do not collapse items based on Delimiter
	Recurse bool
	// Reverse iterates in reverse order
	Reverse bool
}

IterateOptions contains options for iterator

type Iterator

type Iterator interface {
	// Next prepares the next list item.
	// It returns true on success, or false if there is no next result row or an error happened while preparing it.
	Next(ctx context.Context, item *ListItem) bool
}

Iterator iterates over a sequence of ListItems

type IteratorFunc

type IteratorFunc func(ctx context.Context, item *ListItem) bool

IteratorFunc implements basic iterator

func (IteratorFunc) Next

func (next IteratorFunc) Next(ctx context.Context, item *ListItem) bool

Next returns the next item

type Key

type Key []byte

Key is the type for the keys in a `KeyValueStore`

func AfterPrefix

func AfterPrefix(key Key) Key

AfterPrefix returns the key after prefix

func CloneKey

func CloneKey(key Key) Key

CloneKey creates a copy of key

func NextKey

func NextKey(key Key) Key

NextKey returns the successive key

func (Key) Equal

func (key Key) Equal(b Key) bool

Equal returns whether key and b are equal

func (Key) IsZero

func (key Key) IsZero() bool

IsZero returns true if the key struct is it's zero value

func (Key) Less

func (key Key) Less(b Key) bool

Less returns whether key should be sorted before b

func (Key) MarshalBinary

func (key Key) MarshalBinary() ([]byte, error)

MarshalBinary implements the encoding.BinaryMarshaler interface for the Key type

func (Key) String

func (key Key) String() string

String implements the Stringer interface

type KeyValueStore

type KeyValueStore interface {
	// Put adds a value to store
	Put(context.Context, Key, Value) error
	// Get gets a value to store
	Get(context.Context, Key) (Value, error)
	// GetAll gets all values from the store
	GetAll(context.Context, Keys) (Values, error)
	// Delete deletes key and the value
	Delete(context.Context, Key) error
	// List lists all keys starting from start and upto limit items
	List(ctx context.Context, start Key, limit int) (Keys, error)
	// Iterate iterates over items based on opts
	Iterate(ctx context.Context, opts IterateOptions, fn func(context.Context, Iterator) error) error
	// CompareAndSwap atomically compares and swaps oldValue with newValue
	CompareAndSwap(ctx context.Context, key Key, oldValue, newValue Value) error
	// Close closes the store
	Close() error
}

KeyValueStore describes key/value stores like redis and boltdb

type Keys

type Keys []Key

Keys is the type for a slice of keys in a `KeyValueStore`

func ListKeys

func ListKeys(ctx context.Context, store KeyValueStore, first Key, limit int) (_ Keys, err error)

ListKeys returns keys starting from first and upto limit limit is capped to LookupLimit

func ReverseKeys

func ReverseKeys(keys Keys) Keys

ReverseKeys reverses the list of keys

func (Keys) ByteSlices

func (keys Keys) ByteSlices() [][]byte

ByteSlices converts a `Keys` struct to a slice of byte-slices (i.e. `[][]byte`)

func (Keys) Strings

func (keys Keys) Strings() []string

Strings returns everything as strings

type ListItem

type ListItem struct {
	Key      Key
	Value    Value
	IsPrefix bool
}

ListItem returns Key, Value, IsPrefix

func CloneItem

func CloneItem(item ListItem) ListItem

CloneItem creates a deep copy of item

func (ListItem) Less

func (item ListItem) Less(b ListItem) bool

Less returns whether item should be sorted before b

type ListOptions

type ListOptions struct {
	Prefix       Key
	StartAfter   Key // StartAfter is relative to Prefix
	EndBefore    Key // EndBefore is relative to Prefix
	Recursive    bool
	IncludeValue bool
	Limit        int
}

ListOptions are items that are optional for the LIST method

type StaticIterator

type StaticIterator struct {
	Items Items
	Index int
}

StaticIterator implements an iterator over list of items

func (*StaticIterator) Next

func (it *StaticIterator) Next(ctx context.Context, item *ListItem) bool

Next returns the next item from the iterator

type Value

type Value []byte

Value is the type for the values in a `ValueValueStore`

func CloneValue

func CloneValue(value Value) Value

CloneValue creates a copy of value

func (Value) IsZero

func (value Value) IsZero() bool

IsZero returns true if the value struct is it's zero value

func (Value) MarshalBinary

func (value Value) MarshalBinary() ([]byte, error)

MarshalBinary implements the encoding.BinaryMarshaler interface for the Value type

type Values

type Values []Value

Values is the type for a slice of Values in a `KeyValueStore`