dyc

package module
v1.1.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 28, 2021 License: MIT Imports: 14 Imported by: 1

README

Go Report Card GoDoc license

dyc

dyc is a golang package which provides a dynamodb client and query builder. It utilizes the go AWS SDK to provide a more convenient way to interact with dynamodb.

Features
  • Utilize current dynamodb query language
  • Easily reference columns that conflict with dynamodb reserved words by wrapping column in a single quote
  • Build queries with ease
  • Input substitution via ?
  • Parallel Scan support
  • Copy table support
  • In Support
  • Basic Conjunctions support
Examples
Client setup
// setup aws session
awsSession := session.Must(session.NewSession(aws.NewConfig()))
// setup aws dynamodb sdk client
db := dynamodb.New(awsSession)
// setup the dyc client
cli := dyc.NewClient(db)
Query

Iterator

// error ignored for demonstration purposes only
it, err := li.Builder().Table("MyTable").
             WhereKey(`PK = ?`, "PartitionKey").
             Index("SomeIndex").
             // Where is equivalent to filter expression
             Where(`'Some'.'Nested'.'Field' = ? AND 'another' = ?`, "cool", true).
             ToQueryIterator(ctx)

var results []map[string]dynamodb.AttributeValue
for it.Next() {
  // error ignored for demonstration purposes only
  output, _ := it.QueryValue(), it.Err()
  results = append(results, output.Items...)
}

Iterate

err := cli.Builder().Table("MyTable").
  WhereKey(`PK = ?`, "PartitionKey").
  Index("SomeIndex").
  // Where is equivalent to filter expression
  Where(`'Some'.'Nested'.'Field' = ? AND 'another' = ?`, "cool", true).
  QueryIterate(ctx, func(output *dynamodb.QueryOutput) error {
    // get results
    return nil
  })

Query All

type Row struct {
	PK     string
	SK     string
	Some map[string]interface
}

var rows []Row
// results will be an array of dynamodb attribute maps
results, err := cli.Builder().Table("MyTable").
  // WhereKey is equivalent to query expression
  WhereKey(`PK = ?`, "PartitionKey").
  // Result Unmarshals the raw results into the custom type
  Result(&rows).
  Index("SomeIndex").
  Where(`'Some'.'Nested'.'Field' = ? AND 'another' = ?`, "cool", true).
  QueryAll(ctx context.Context)

Query Single

type Row struct {
	PK     string
	SK     string
	Some map[string]interface
}

var row Row
//result will be a dynamodb attribute map
result, err := cli.Builder().Table("MyTable").
  WhereKey(`PK = ?`, "PartitionKey").
  // Result Unmarshals the raw results into the custom type
  Result(&row).
  Index("SomeIndex").
  Where(`'Some'.'Nested'.'Field' = ? AND 'another' = ?`, "cool", true).
  QuerySingle(ctx context.Context)

Delete By Query

primaryKeyExtractor := dyc.FieldsExtractor("PK", "SK")
err := cli.Builder().Table("MyTable").
  WhereKey("PK = ? AND SK BETWEEN ? AND ?", "key", 1, 1337)
  Where(`'Some'.'Nested'.'Field' = ? AND 'another' = ?`, "cool", true).
  QueryDelete(ctx, primaryKeyExtractor)
  • deletes all records matching the query

Insert Item

type Row struct {
	PK     string
	SK     string
	StrMap map[string]string
}
row := Row{PK: "yo", SK: "lo", StrMap: map[string]string{"who": "goes there"}}
result, err := cli.Builder().Table("MyTable").
  PutItem(context.Background(), row)

Insert Item Conditionally

type Row struct {
	PK     string
	SK     string
	StrMap map[string]string
}
row := Row{PK: "yo", SK: "lo", StrMap: map[string]string{"who": "goes there"}}
result, err := cli.Builder().Table("MyTable").
  Condition("attribute_not_exists(PK)").
  PutItem(context.Background(), row)

Update Item

result, err := cli.Builder().Table("MyTable").
  Key("PK", "yo", "SK", "lo").
  Update(`REMOVE 'StrMap'.'who' SET 'StrMap'.'goes' = ?`, "who there").
  UpdateItem(context.Background())
Scan

Iterator

// error ignored for demonstration purposes only
it, err := li.Builder().Table("MyTable").
             WhereKey(`PK = ?`, "PartitionKey").
             Index("SomeIndex").
             // Where is equivalent to filter expression
             Where(`'Some'.'Nested'.'Field' = ? AND 'another' = ?`, "cool", true).
             ToScanIterator(ctx)

var results []map[string]dynamodb.AttributeValue
for it.Next() {
  // error ignored for demonstration purposes only
  output, _ := it.ScanValue(), it.Err()
  results = append(results, output.Items...)
}

Iterate

err := cli.Builder().Table("MyTable").
  Where(`'Some'.'Nested'.'Field' = ? AND 'another' = ?`, "cool", true).
  ScanIterate(ctx, func(output *dynamodb.ScanOutput) error {
    // get results
    return nil
  })

In

err := cli.Builder().Table("MyTable").
  IN(`'Some'.'Nested'.'Field'`, "value1", "value2").
  ScanIterate(ctx, func(output *dynamodb.ScanOutput) error {
    // get results
    return nil
  })

InSlice

err := cli.Builder().Table("MyTable").
  IN(`'Some'.'Nested'.'Field'`, []string{"value1", "value2"}).
  ScanIterate(ctx, func(output *dynamodb.ScanOutput) error {
    // get results
    return nil
  })

Conjunctions Example

results, err := cli.Builder().
  Table("MyTable").
  Select(`PK`, "DAT", "SOME.NESTED.FIELD").
  WhereKey("PK = ?", "some key").
  Where(`'SOMEVALUE'.here = ?`, true).
  OrWhere(`'SOMEOTHER'.'VALUE' = ?`, false).
  OrIN("field1", 23, 432, 200, 23).
  InSlice("field2", []string{"one", "two", "three"})
  Condition("attribute_not_exists(YO.'id')").
  OrCondition("attribute_exists(YOLO)").
  IN(`'Some'.'Nested'.'Field'`, []string{"value1", "value2"}).
  QueryAll(context.TODO())

Scan Delete

err := cli.Builder().Table("MyTable").
  Where(`'Some'.'Nested'.'Field' = ? AND 'another' = ?`, "cool", true).
  ScanDelete(ctx, dyc.FieldsExtractor("PK", "SK"))
  • deletes all records matching the scan
  • PK and SK are the partition key and sort key needed to delete the matching records
Copy table example
totalWorkers := 40
err := cli.CopyTable(ctx, "destinationTable", "sourceTable", totalWorkers, nil)

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	// ErrClientNotSet occurs if you try to make a client call from a builder without setting the client
	ErrClientNotSet = errors.New("client not set")
	// ErrBadKeyType occurs if you try to provide a non string key name
	ErrBadKeyType = errors.New("strings are the only type of keys supported")
	// ErrKeyRequired occurs if key is required for the given operation
	ErrKeyRequired = errors.New("key not set")
	// ErrBadKeyParams occurs if k/v params don't line up
	ErrBadKeyParams = errors.New("expected an even amount of additionalKVs parameters")
	// ErrUnsupportedType occurs if you are trying to add an unsupported type as an input for a query
	ErrUnsupportedType = errors.New("unsupported type")
	// ErrQueryMisMatch occurs if the number of ? don't line up with the given inputs for a query
	ErrQueryMisMatch = errors.New("inputs don't match query")
	// ErrNotSlice occurs if a non slice type is provided as a value for any of the IN builder query functions
	ErrNotSlice = errors.New("provided value is not a slice")
	// ErrNotPointer occurs if a non pointer type is provided to the Result method of the builder type
	ErrNotPointer = errors.New("provided result type is not a slice")
)

Functions

func Int 

func Int(num int) *dynamodb.AttributeValue

Int converts an integer into a integer type

func IntList 

func IntList(arr ...int) *dynamodb.AttributeValue

IntList converts an array of integers to an integer list type

func PKSKExtractor 

func PKSKExtractor(attrs map[string]*dynamodb.AttributeValue) map[string]*dynamodb.AttributeValue

PKSKExtractor extracts the fields PK and SK

func String 

func String(str string) *dynamodb.AttributeValue

String converts a string to a string type

func StringList 

func StringList(arr ...string) *dynamodb.AttributeValue

StringList converts an array of strings to a string list type

func StringSet 

func StringSet(arr ...string) *dynamodb.AttributeValue

StringSet converts an array of strings to a string set type

Types

type Builder 

type Builder struct {
	// contains filtered or unexported fields
}

Builder allows you to build dynamo queries in a more convenient fashion

func NewBuilder 

func NewBuilder() *Builder

NewBuilder creates a new builder

func (*Builder) Builder added in v1.1.1 

func (s *Builder) Builder() *Builder

Builder returns a new builder instance. If client or table was set it will also be set on new instance

func (*Builder) Client 

func (s *Builder) Client(client *Client) *Builder

Client sets client that will be used for client operations based on built object

func (*Builder) Condition 

func (s *Builder) Condition(query string, vals ...interface{}) *Builder

Condition allows you do make a condition expression. e.g Condition("'MyKey' = ?", "yourKey") note: calling this multiple times combines conditions with an AND

func (*Builder) ConsistentRead 

func (s *Builder) ConsistentRead(consistent bool) *Builder

ConsistentRead sets the consistent read flag

func (*Builder) DeleteItem 

func (s *Builder) DeleteItem(ctx context.Context) (*dynamodb.DeleteItemOutput, error)

DeleteItem deletes a single item utilizing data set via Table, Keys and Condition method calls

func (*Builder) GetItem 

func (s *Builder) GetItem(ctx context.Context) (*dynamodb.GetItemOutput, error)

GetItem builds and runs a query using info in key and table

func (*Builder) IN 

func (s *Builder) IN(column string, vals ...interface{}) *Builder

IN lets you build an IN filter expression e.g IN(`COL_NAME_HERE`, 1,2,3,4,5) note: if you have an existing filter expression this will be prefixed with an AND

func (*Builder) INSlice 

func (s *Builder) INSlice(column string, val interface{}) *Builder

INSlice lets you build an IN filter expression from a slice e.g INSlice(`COL_NAME_HERE`, []int{1,2,3,4,5}) note: if you have an existing filter expression this will be prefixed with an AND

func (*Builder) Index 

func (s *Builder) Index(index string) *Builder

Index sets the index to use

func (*Builder) Key 

func (s *Builder) Key(keyName string, value interface{}, additionalKVs ...interface{}) *Builder

Key allows you to set the key for a given item e.g Key("PK", "hello", "SK", "there")

func (*Builder) Limit 

func (s *Builder) Limit(limit int) *Builder

Limit sets the limit for results

func (*Builder) OrCondition 

func (s *Builder) OrCondition(query string, vals ...interface{}) *Builder

OrCondition allows you do make an OR if you have multiple conditions e.g Condition("'MyKey' = ?", "yourKey") note: calling this multiple times combines conditions with an OR

func (*Builder) OrIN 

func (s *Builder) OrIN(column string, vals ...interface{}) *Builder

OrIN lets you build an IN filter expression e.g IN(`COL_NAME_HERE`, 1,2,3,4,5) note: if you have an existing filter expression this will be prefixed with an OR

func (*Builder) OrINSlice 

func (s *Builder) OrINSlice(column string, val interface{}) *Builder

OrINSlice lets you build an IN filter expression from a slice e.g INSlice(`COL_NAME_HERE`, []int{1,2,3,4,5}) note: if you have an existing filter expression this will be prefixed with an OR

func (*Builder) OrWhere 

func (s *Builder) OrWhere(query string, vals ...interface{}) *Builder

OrWhere is equivalent to a filter expression with an OR e.g Where("'Hey' = ? AND 'Test'.'Nested'" = ?, "yo", true).OrWhere("'Foo' = ?", "bar")

func (*Builder) ParallelScanIterate 

func (s *Builder) ParallelScanIterate(ctx context.Context, workers int, fn func(output *dynamodb.ScanOutput) error, unsafe bool) error

ParallelScanIterate allows you to do a parallel scan in dynamo based on the built object. the fn parameter will be called as often as needed to retrieve all results

func (*Builder) PutItem added in v1.1.0 

func (s *Builder) PutItem(ctx context.Context, data interface{}) (*dynamodb.PutItemOutput, error)

PutItem inserts the provided data and marshal maps it using the aws sdk

func (*Builder) QueryAll 

func (s *Builder) QueryAll(ctx context.Context) ([]map[string]*dynamodb.AttributeValue, error)

QueryAll returns an all results matching the built query

func (*Builder) QueryDelete 

func (s *Builder) QueryDelete(ctx context.Context, keyFn KeyExtractor) error

QueryDelete deletes all records matching the query. note: you must provide a function that will select the relevant Key fields needed for deletion

func (*Builder) QueryIterate 

func (s *Builder) QueryIterate(ctx context.Context, fn func(output *dynamodb.QueryOutput) error) error

QueryIterate allows you to query dynamo based on the built object. the fn parameter will be called as often as needed to retrieve all results

func (*Builder) QuerySingle 

func (s *Builder) QuerySingle(ctx context.Context) (map[string]*dynamodb.AttributeValue, error)

QuerySingle returns a single result matching the built query

func (*Builder) Result 

func (s *Builder) Result(val interface{}) *Builder

Result decodes the final result into the provided variable. Provided variable must be a pointer

func (*Builder) ScanDelete 

func (s *Builder) ScanDelete(ctx context.Context, keyFn KeyExtractor) error

ScanDelete deletes all records matching the scan. note: you must provide a function that will select the relevant Key fields needed for deletion

func (*Builder) ScanIterate 

func (s *Builder) ScanIterate(ctx context.Context, fn func(output *dynamodb.ScanOutput) error) error

ScanIterate allows you to query dynamo based on the built object. the fn parameter will be called as often as needed to retrieve all results

func (*Builder) SelectFields 

func (s *Builder) SelectFields(fields ...string) *Builder

SelectFields allows you to select which fields to pull back from dynamo

func (*Builder) Sort 

func (s *Builder) Sort(ascending bool) *Builder

Sort sets sort as either ascending or descending

func (*Builder) Table 

func (s *Builder) Table(tbl string) *Builder

Table sets the table name

func (*Builder) ToDelete 

func (s *Builder) ToDelete() (dynamodb.DeleteItemInput, error)

ToDelete produces a dynamodb.DeleteItemInput value based on configured builder

func (*Builder) ToGet 

func (s *Builder) ToGet() (dynamodb.GetItemInput, error)

ToGet produces a dynamodb.GetItemInput value based on configured builder

func (*Builder) ToPut added in v1.1.0 

func (s *Builder) ToPut(item interface{}) (dynamodb.PutItemInput, error)

ToPut produces a dynamodb.PutItemInput value based on configured builder

func (*Builder) ToQuery 

func (s *Builder) ToQuery() (dynamodb.QueryInput, error)

ToQuery produces a dynamodb.QueryInput value based on configured builder

func (*Builder) ToScan 

func (s *Builder) ToScan() (dynamodb.ScanInput, error)

ToScan produces a dynamodb.ScanInput value based on configured builder

func (*Builder) ToUpdate 

func (s *Builder) ToUpdate() (dynamodb.UpdateItemInput, error)

ToUpdate produces a dynamodb.UpdateItemInput value based on configured builder

func (*Builder) Update 

func (s *Builder) Update(query string, vals ...interface{}) *Builder

Update is equivalent to an update expression e.g Update("SET 'Hey' = ?, 'Test'.'Nested'" = ?, "yo", true)

func (*Builder) UpdateItem 

func (s *Builder) UpdateItem(ctx context.Context) (*dynamodb.UpdateItemOutput, error)

UpdateItem builds and runs an update query

func (*Builder) Where 

func (s *Builder) Where(query string, vals ...interface{}) *Builder

Where is equivalent to a filter expression e.g Where("'Hey' = ? AND 'Test'.'Nested'" = ?, "yo", true) note: calling this multiple times combines conditions with an AND

func (*Builder) WhereKey 

func (s *Builder) WhereKey(query string, vals ...interface{}) *Builder

WhereKey allows you do make a key expression e.g WhereKey("'MyKey' = ?", "yourKey") note: calling this multiple times combines conditions with an AND

type Client 

type Client struct {
	*dynamodb.DynamoDB
}

Client is a wrapper around the dynamodb SDK that provides useful behavior such as iteration, processing unprocessed items and more

func NewClient 

func NewClient(db *dynamodb.DynamoDB) *Client

NewClient creates a new dyc client

func (*Client) BatchGetIterator 

func (c *Client) BatchGetIterator(ctx context.Context, input *dynamodb.BatchGetItemInput, fn func(output *dynamodb.GetItemOutput) error) error

BatchGetIterator retrieves all items from the batch get input

func (*Client) BatchPut 

func (c *Client) BatchPut(ctx context.Context, tableName string, items ...interface{}) (int, error)

BatchPut allows you to put a batch of items to a table items will me converted to a marshal map

func (*Client) BatchWriter 

func (c *Client) BatchWriter(ctx context.Context, tableName string, requests ...*dynamodb.WriteRequest) (int, error)

BatchWriter batch writes an array of write requests to a table

func (*Client) Builder 

func (c *Client) Builder() *Builder

Builder produces a builder configured with the current client

func (*Client) ChunkWriteRequests 

func (c *Client) ChunkWriteRequests(requests []*dynamodb.WriteRequest) [][]*dynamodb.WriteRequest

ChunkWriteRequests chunks write requests into batches of 25 (the current maximum size in AWS)

func (*Client) CopyTable 

func (c *Client) CopyTable(parentCtx context.Context, dst string, src string, workers int, onError func(err error, cancelFunc context.CancelFunc)) error

CopyTable copies all data in source to the existing destination table using

func (*Client) ExtractFields 

func (c *Client) ExtractFields(data map[string]*dynamodb.AttributeValue, fields ...string) map[string]*dynamodb.AttributeValue

ExtractFields extracts fields from a map of dynamo attribute values

func (*Client) ParallelScanIterator 

func (c *Client) ParallelScanIterator(ctx context.Context, input *dynamodb.ScanInput, workers int, fn func(output *dynamodb.ScanOutput) error, noLock bool) error

ParallelScanIterator is a thread safe method that performs a parallel scan on dynamodb utilizing the configured amount of workers

func (*Client) QueryDeleter 

func (c *Client) QueryDeleter(ctx context.Context, table string, input *dynamodb.QueryInput, keyFn KeyExtractor) error

QueryDeleter deletes all records that match the query

func (*Client) QueryIterator 

func (c *Client) QueryIterator(ctx context.Context, input *dynamodb.QueryInput, fn func(output *dynamodb.QueryOutput) error) error

QueryIterator iterates all results of a query

func (*Client) ScanCount 

func (c *Client) ScanCount(ctx context.Context, input *dynamodb.ScanInput) (int64, error)

ScanCount counts all records matching the scan query

func (*Client) ScanDeleter 

func (c *Client) ScanDeleter(ctx context.Context, table string, input *dynamodb.ScanInput, keyFn KeyExtractor) error

ScanDeleter deletes all records that match the scan query

func (*Client) ScanIterator 

func (c *Client) ScanIterator(ctx context.Context, input *dynamodb.ScanInput, fn func(output *dynamodb.ScanOutput) error) error

ScanIterator iterates all results of a scan

func (*Client) ToBatchGetItemInput 

func (c *Client) ToBatchGetItemInput(tableName string, req []map[string]*dynamodb.AttributeValue) *dynamodb.BatchGetItemInput

ToBatchGetItemInput converts an array of mapped dynamo attributes to a batch get item input

type Iterator 

type Iterator struct {
	// contains filtered or unexported fields
}

Iterator provides result iteration behavior

func NewIteratorFromQuery 

func NewIteratorFromQuery(ctx context.Context, cli IteratorClient, input *dynamodb.QueryInput) *Iterator

NewIteratorFromQuery creates a new iterator from a query

func NewIteratorFromScan 

func NewIteratorFromScan(ctx context.Context, cli IteratorClient, input *dynamodb.ScanInput) *Iterator

NewIteratorFromScan creates a new iterator from a scan

func (*Iterator) Err 

func (i *Iterator) Err() error

Err returns the last err

func (*Iterator) Next 

func (i *Iterator) Next() bool

Next returns true if iteration can continue and false otherwise

func (*Iterator) QueryValue 

func (i *Iterator) QueryValue() *dynamodb.QueryOutput

QueryValue returns the current query output

func (*Iterator) ScanValue 

func (i *Iterator) ScanValue() *dynamodb.ScanOutput

ScanValue returns the current scan output

func (*Iterator) Value 

func (i *Iterator) Value() interface{}

Value returns the current value

type IteratorClient 

type IteratorClient interface {
	QueryRequest(input *dynamodb.QueryInput) (req *request.Request, output *dynamodb.QueryOutput)
	ScanRequest(input *dynamodb.ScanInput) (req *request.Request, output *dynamodb.ScanOutput)
}

IteratorClient is an interface for all methods utilized by iterator

type KeyExtractor 

type KeyExtractor func(map[string]*dynamodb.AttributeValue) map[string]*dynamodb.AttributeValue

KeyExtractor is a type primarily used to get necessary fields needed to delete a record

func FieldsExtractor 

func FieldsExtractor(fields ...string) KeyExtractor

FieldsExtractor extracts the provided fields

Source Files

View all Source files

Directories

Path Synopsis
internal
testing/dynamotest

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.