s3

README

ethdb/s3

Integration testing

To run integration tests, specify the integration tag during tests and pass in the appropriate settings for your S3-compatible bucket:

$ go test -tags integration . -endpoint nyc3.digitaloceanspaces.com -bucket gochain-test -access-key-id 00000000000000000000 -secret-access-key 0000000/00000000000000000000000000000000000

Files on the bucket will be automatically cleaned up after a successful test run.

Documentation

Index

• Constants
• func ConfigureDB(db *ethdb.DB, config ethdb.Config) error
• func SegmentKey(table, name string) string
• type Client
  • func NewClient() *Client
  • func (c *Client) FGetObject(ctx context.Context, key, path string) (err error)
  • func (c *Client) FPutObject(ctx context.Context, key, path string) (n int64, err error)
  • func (c *Client) ListObjectKeys(prefix string) ([]string, error)
  • func (c *Client) Open() (err error)
  • func (c *Client) PutObject(ctx context.Context, key string, value []byte) (n int64, err error)
  • func (c *Client) RemoveObject(ctx context.Context, key string) error
• type Segment
  • func NewSegment(client *Client, table, name, path string) *Segment
  • func (s *Segment) Close() error
  • func (s *Segment) Get(key []byte) ([]byte, error)
  • func (s *Segment) Has(key []byte) (bool, error)
  • func (s *Segment) Iterator() ethdb.SegmentIterator
  • func (s *Segment) Name() string
  • func (s *Segment) Path() string
  • func (s *Segment) Purge() error
• type SegmentCompactor
  • func NewSegmentCompactor(client *Client) *SegmentCompactor
  • func (c *SegmentCompactor) CompactSegment(ctx context.Context, table string, s *ethdb.LDBSegment) (ethdb.Segment, error)
  • func (c *SegmentCompactor) UncompactSegment(ctx context.Context, table string, s ethdb.Segment) (*ethdb.LDBSegment, error)
• type SegmentIterator
  • func (itr *SegmentIterator) Close() error
• type SegmentOpener
  • func NewSegmentOpener(client *Client) *SegmentOpener
  • func (o *SegmentOpener) ListSegmentNames(path, table string) ([]string, error)
  • func (o *SegmentOpener) OpenSegment(table, name, path string) (ethdb.Segment, error)

Constants

const (
	// FGetObjectInterval represents the time between attempts to successfully
	// fetch objects from the S3 store.
	FGetObjectInterval = 2 * time.Second
)

Functions

func ConfigureDB

func ConfigureDB(db *ethdb.DB, config ethdb.Config) error

ConfigureDB updates db to archive to S3 if S3 configuration enabled.

func SegmentKey

func SegmentKey(table, name string) string

SegmentKey returns the key used for the segment on S3.

Types

type Client

type Client struct {

	// Connection information for S3-compatible bucket.
	// Must be set before calling Open().
	Endpoint string
	Bucket   string

	// Authentication for S3-compatible bucket.
	// Must be set before calling Open().
	AccessKeyID     string
	SecretAccessKey string
	// contains filtered or unexported fields
}

Client represents a client to an S3 compatible bucket.

func NewClient

func NewClient() *Client

NewClient returns a new instance of Client.

func (*Client) FGetObject

func (c *Client) FGetObject(ctx context.Context, key, path string) (err error)

FGetObject fetches the object at key and atomically writes it to path. Attempts multiple times until a successful fetch has been acheived.

func (*Client) FPutObject

func (c *Client) FPutObject(ctx context.Context, key, path string) (n int64, err error)

FPutObject writes an object to key from a file at path.

func (*Client) ListObjectKeys

func (c *Client) ListObjectKeys(prefix string) ([]string, error)

ListObjectKeys returns a list of all object keys with a given prefix.

func (*Client) Open

func (c *Client) Open() (err error)

func (*Client) PutObject

func (c *Client) PutObject(ctx context.Context, key string, value []byte) (n int64, err error)

PutObject writes an object to a key.

func (*Client) RemoveObject

func (c *Client) RemoveObject(ctx context.Context, key string) error

RemoveObject removes an object by key.

type Segment

type Segment struct {
	// contains filtered or unexported fields
}

Segment represents an ethdb.FileSegment stored in S3.

func NewSegment

func NewSegment(client *Client, table, name, path string) *Segment

NewSegment returns a new instance of Segment.

func (*Segment) Close

func (s *Segment) Close() error

Close closes the underlying file segment.

func (*Segment) Get

func (s *Segment) Get(key []byte) ([]byte, error)

Get returns the value of the given key.

func (*Segment) Has

func (s *Segment) Has(key []byte) (bool, error)

Has returns true if the key exists.

func (*Segment) Iterator

func (s *Segment) Iterator() ethdb.SegmentIterator

Iterator returns an iterator for the segment.

func (*Segment) Name

func (s *Segment) Name() string

Name returns the name of the segment.

func (*Segment) Path

func (s *Segment) Path() string

Path returns the local path of the segment.

func (*Segment) Purge

func (s *Segment) Purge() error

Purge closes the underlying file segment and removes the on-disk file.

type SegmentCompactor

type SegmentCompactor struct {
	Client *Client
}

SegmentCompactor wraps ethdb.FileSegmentCompactor and uploads to S3 after compaction.

func NewSegmentCompactor

func NewSegmentCompactor(client *Client) *SegmentCompactor

NewSegmentCompactor returns a new instance of SegmentCompactor.

func (*SegmentCompactor) CompactSegment

func (c *SegmentCompactor) CompactSegment(ctx context.Context, table string, s *ethdb.LDBSegment) (ethdb.Segment, error)

CompactSegment compacts s into a FileSegement and uploads it to S3.

func (*SegmentCompactor) UncompactSegment

func (c *SegmentCompactor) UncompactSegment(ctx context.Context, table string, s ethdb.Segment) (*ethdb.LDBSegment, error)

UncompactSegment uncompacts s into an LDBSegement.

type SegmentIterator

type SegmentIterator struct {
	ethdb.SegmentIterator
	// contains filtered or unexported fields
}

SegmentIterator represents a wrapper around ethdb.SegmentIterator. Releases read lock on close.

func (*SegmentIterator) Close

func (itr *SegmentIterator) Close() error

Close releases iterator resources and releases the read lock on the segment.

type SegmentOpener

type SegmentOpener struct {
	Client *Client
}

SegmentOpener opens segments as a s3.Segments.

func NewSegmentOpener

func NewSegmentOpener(client *Client) *SegmentOpener

NewSegmentOpener returns a new instance of SegmentOpener.

func (*SegmentOpener) ListSegmentNames

func (o *SegmentOpener) ListSegmentNames(path, table string) ([]string, error)

ListSegmentNames returns a list of segment names for a table.

func (*SegmentOpener) OpenSegment

func (o *SegmentOpener) OpenSegment(table, name, path string) (ethdb.Segment, error)

OpenSegment returns creates and opens a reference to a remote immutable segment.