bucket

Documentation

Index

• Constants
• type Bucket
  • func NewBucket(name string) (Bucket, error)
• type BucketIterator
• type GCS
  • func (g *GCS) GetName() string
  • func (g *GCS) GetObject(key string) (io.ReadCloser, error)
  • func (g *GCS) GetObjectIfMatch(key string, match string) (io.ReadCloser, error)
  • func (g *GCS) GetOwner() string
  • func (g *GCS) GetPipeline() string
  • func (g *GCS) GetSession() *storage.Client
  • func (g *GCS) Key(t time.Time) string
  • func (g *GCS) KeyPrefix(t time.Time) string
  • func (g *GCS) ListObjects(path string, latest string, helper BucketIterator) error
  • func (g *GCS) NewSession() error
  • func (g *GCS) Path() string
  • func (g *GCS) SetFlags()
  • func (g *GCS) SetName(o string)
  • func (g *GCS) String() string
  • func (g *GCS) Upload(b *bytes.Buffer) error
  • func (g *GCS) UploadCallback(b *bytes.Buffer, intf interface{}, cb UploadCallback) error
  • func (g *GCS) UploadKey(key string, b *bytes.Buffer) error
  • func (g *GCS) UploadKeyCallback(key string, b *bytes.Buffer, intf interface{}, cb UploadCallback) error
• type PipelineObject
  • func NewPipelineObject(key string, etag string) *PipelineObject
  • func (oc PipelineObject) MatchesBucket(bc Bucket) bool
  • func (oc PipelineObject) MatchesStream(sc *stream.StreamConfig) bool
• type S3
  • func (s *S3) GetCredentials() *credentials.Credentials
  • func (s *S3) GetName() string
  • func (s *S3) GetObject(key string) (io.ReadCloser, error)
  • func (s *S3) GetObjectIfMatch(key string, match string) (io.ReadCloser, error)
  • func (s *S3) GetOwner() string
  • func (s *S3) GetPipeline() string
  • func (s *S3) GetSession() *session.Session
  • func (s *S3) Key(t time.Time) string
  • func (s *S3) KeyPrefix(t time.Time) string
  • func (s *S3) ListObjects(path string, latest string, iterator BucketIterator) error
  • func (s *S3) NewSession() error
  • func (s *S3) Path() string
  • func (s *S3) SetFlags()
  • func (s *S3) SetName(o string)
  • func (s *S3) String() string
  • func (s *S3) Upload(b *bytes.Buffer) error
  • func (s *S3) UploadCallback(b *bytes.Buffer, intf interface{}, cb UploadCallback) error
  • func (s *S3) UploadKey(key string, b *bytes.Buffer) error
  • func (s *S3) UploadKeyCallback(key string, b *bytes.Buffer, intf interface{}, cb UploadCallback) error
• type UploadCallback

Constants

const DefaultStorageClass = "STANDARD"

const GoogleStorageClass = "STANDARD"

Types

type Bucket

type Bucket interface {
	GetName() string
	SetName(string)
	GetOwner() string
	GetPipeline() string

	NewSession() error
	GetObject(string) (io.ReadCloser, error)
	GetObjectIfMatch(string, string) (io.ReadCloser, error)

	Path() string
	KeyPrefix(time.Time) string
	Upload(*bytes.Buffer) error
	ListObjects(string, string, BucketIterator) error

	SetFlags()
}

func NewBucket

func NewBucket(name string) (Bucket, error)

NewBucket takes s3://foo or gs://bar and returns either S3{} or GCS{}, respectively.

Note: prefix (s3:// or gs://) is stripped.

The default is S3{}

NewBucket returns error if illegal characters are present.

type BucketIterator

type BucketIterator func(*PipelineObject) bool

type GCS

type GCS struct {
	Pipeline string `default:"pipeline.unset" json:"pipeline"`
	Owner    string `default:"data" json:"owner"`
	Bucket   string `default:"atsu.unset" json:"bucket"`
	Suffix   string `default:"json.gz" json:"suffix"`
	Name     string `json:"name"`

	Objects int `json:"objects"`
	Bytes   int `json:"bytes"`
	// contains filtered or unexported fields
}

uprovides GCS-related configuration

func (*GCS) GetName

func (g *GCS) GetName() string

func (*GCS) GetObject

func (g *GCS) GetObject(key string) (io.ReadCloser, error)

func (*GCS) GetObjectIfMatch

func (g *GCS) GetObjectIfMatch(key string, match string) (io.ReadCloser, error)

func (*GCS) GetOwner

func (g *GCS) GetOwner() string

func (*GCS) GetPipeline

func (g *GCS) GetPipeline() string

func (*GCS) GetSession

func (g *GCS) GetSession() *storage.Client

func (*GCS) Key

func (g *GCS) Key(t time.Time) string

Key returns the full GCS key

func (*GCS) KeyPrefix

func (g *GCS) KeyPrefix(t time.Time) string

KeyPrefix generates the GCS prefix, including time-based prefix

func (*GCS) ListObjects

func (g *GCS) ListObjects(path string, latest string, helper BucketIterator) error

func (*GCS) NewSession

func (g *GCS) NewSession() error

NewSession creates a new Google session; requires GOOGLE_APPLICATION_CREDENTIALS

func (*GCS) Path

func (g *GCS) Path() string

Path generates the GCS prefix, excluding time-based prefix

func (*GCS) SetFlags

func (g *GCS) SetFlags()

SetFlags to install various command-line flag(s)

func (*GCS) SetName

func (g *GCS) SetName(o string)

func (*GCS) String

func (g *GCS) String() string

String returns JSON representation

func (*GCS) Upload

func (g *GCS) Upload(b *bytes.Buffer) error

Upload uploads the provided buffer via GCS

func (*GCS) UploadCallback

func (g *GCS) UploadCallback(b *bytes.Buffer, intf interface{}, cb UploadCallback) error

func (*GCS) UploadKey

func (g *GCS) UploadKey(key string, b *bytes.Buffer) error

func (*GCS) UploadKeyCallback

func (g *GCS) UploadKeyCallback(key string, b *bytes.Buffer, intf interface{}, cb UploadCallback) error

type PipelineObject

type PipelineObject struct {
	Owner    string
	Pipeline string
	Name     string
	Date     time.Time
	File     string

	Key  string
	Etag string
}

PipelineObject represents an individual object stored in a Bucket

func NewPipelineObject

func NewPipelineObject(key string, etag string) *PipelineObject

NewPipelineObject parses an S3 key and returns an PipelineObject

func (PipelineObject) MatchesBucket

func (oc PipelineObject) MatchesBucket(bc Bucket) bool

Ensure this object belongs to us

func (PipelineObject) MatchesStream

func (oc PipelineObject) MatchesStream(sc *stream.StreamConfig) bool

Deprecated: Use MatchesBucket instead

type S3

type S3 struct {
	Pipeline string `default:"pipeline.unset" json:"pipeline"`
	Owner    string `default:"unset" json:"owner"`
	Bucket   string `default:"atsu.unset" json:"bucket"`
	Suffix   string `default:"json.gz" json:"suffix"`
	Name     string `json:"name"`

	Objects int `json:"objects"`
	Bytes   int `json:"bytes"`
	// contains filtered or unexported fields
}

S3 provides S3-related configuration

func (*S3) GetCredentials

func (s *S3) GetCredentials() *credentials.Credentials

func (*S3) GetName

func (s *S3) GetName() string

func (*S3) GetObject

func (s *S3) GetObject(key string) (io.ReadCloser, error)

func (*S3) GetObjectIfMatch

func (s *S3) GetObjectIfMatch(key string, match string) (io.ReadCloser, error)

func (*S3) GetOwner

func (s *S3) GetOwner() string

func (*S3) GetPipeline

func (s *S3) GetPipeline() string

func (*S3) GetSession

func (s *S3) GetSession() *session.Session

func (*S3) Key

func (s *S3) Key(t time.Time) string

Key returns the full S3 key

func (*S3) KeyPrefix

func (s *S3) KeyPrefix(t time.Time) string

KeyPrefix generates the S3 prefix, including time-based prefix

func (*S3) ListObjects

func (s *S3) ListObjects(path string, latest string, iterator BucketIterator) error

func (*S3) NewSession

func (s *S3) NewSession() error

NewSession creates a new AWS session using "atsu.io" credentials

func (*S3) Path

func (s *S3) Path() string

Path generates the S3 prefix, excluding time-based prefix

func (*S3) SetFlags

func (s *S3) SetFlags()

SetFlags to install various command-line flag(s)

func (*S3) SetName

func (s *S3) SetName(o string)

func (*S3) String

func (s *S3) String() string

String returns JSON representation

func (*S3) Upload

func (s *S3) Upload(b *bytes.Buffer) error

Upload uploads the provided buffer via S3

func (*S3) UploadCallback

func (s *S3) UploadCallback(b *bytes.Buffer, intf interface{}, cb UploadCallback) error

func (*S3) UploadKey

func (s *S3) UploadKey(key string, b *bytes.Buffer) error

func (*S3) UploadKeyCallback

func (s *S3) UploadKeyCallback(key string, b *bytes.Buffer, intf interface{}, cb UploadCallback) error

type UploadCallback

type UploadCallback func(string, *bytes.Buffer, interface{}, error) error