gcs

Documentation

Overview

Package gcs implements common object storage abstractions against Google Cloud Storage.

Index

• Constants
• Variables
• func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error)
• type Bucket
  • func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, ...) (*Bucket, error)
  • func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, ...) (*Bucket, error)
  • func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)
  • func (b *Bucket) Close() error
  • func (b *Bucket) Delete(ctx context.Context, name string) error
  • func (b *Bucket) Exists(ctx context.Context, name string) (bool, error)
  • func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error)
  • func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)
  • func (b *Bucket) Handle() *storage.BucketHandle
  • func (b *Bucket) IsAccessDeniedErr(err error) bool
  • func (b *Bucket) IsObjNotFoundErr(err error) bool
  • func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, ...) error
  • func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, ...) error
  • func (b *Bucket) Name() string
  • func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType
  • func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error
• type Config

Constants

const DirDelim = "/"

DirDelim is the delimiter used to model a directory structure in an object store bucket.

Variables

var DefaultConfig = Config{
	HTTPConfig: exthttp.DefaultHTTPConfig,
}

Functions

func NewTestBucket

func NewTestBucket(t testing.TB, project string) (objstore.Bucket, func(), error)

NewTestBucket creates test bkt client that before returning creates temporary bucket. In a close function it empties and deletes the bucket.

Types

type Bucket

type Bucket struct {
	// contains filtered or unexported fields
}

Bucket implements the store.Bucket and shipper.Bucket interfaces against GCS.

func NewBucket

func NewBucket(ctx context.Context, logger log.Logger, conf []byte, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error)

NewBucket returns a new Bucket against the given bucket handle.

func NewBucketWithConfig

func NewBucketWithConfig(ctx context.Context, logger log.Logger, gc Config, component string, wrapRoundtripper func(http.RoundTripper) http.RoundTripper) (*Bucket, error)

NewBucketWithConfig returns a new Bucket with gcs Config struct.

func (*Bucket) Attributes

func (b *Bucket) Attributes(ctx context.Context, name string) (objstore.ObjectAttributes, error)

Attributes returns information about the specified object.

func (*Bucket) Close

func (b *Bucket) Close() error

func (*Bucket) Delete

func (b *Bucket) Delete(ctx context.Context, name string) error

Delete removes the object with the given name.

func (*Bucket) Exists

func (b *Bucket) Exists(ctx context.Context, name string) (bool, error)

Exists checks if the given object exists.

func (*Bucket) Get

func (b *Bucket) Get(ctx context.Context, name string) (io.ReadCloser, error)

Get returns a reader for the given object name.

func (*Bucket) GetRange

func (b *Bucket) GetRange(ctx context.Context, name string, off, length int64) (io.ReadCloser, error)

GetRange returns a new range reader for the given object name and range.

func (*Bucket) Handle

func (b *Bucket) Handle() *storage.BucketHandle

Handle returns the underlying GCS bucket handle. Used for testing purposes (we return handle, so it is not instrumented).

func (*Bucket) IsAccessDeniedErr

func (b *Bucket) IsAccessDeniedErr(err error) bool

IsAccessDeniedErr returns true if access to object is denied.

func (*Bucket) IsObjNotFoundErr

func (b *Bucket) IsObjNotFoundErr(err error) bool

IsObjNotFoundErr returns true if error means that object is not found. Relevant to Get operations.

func (*Bucket) Iter

func (b *Bucket) Iter(ctx context.Context, dir string, f func(string) error, opts ...objstore.IterOption) error

Iter calls f for each entry in the given directory. The argument to f is the full object name including the prefix of the inspected directory.

func (*Bucket) IterWithAttributes

func (b *Bucket) IterWithAttributes(ctx context.Context, dir string, f func(attrs objstore.IterObjectAttributes) error, options ...objstore.IterOption) error

func (*Bucket) Name

func (b *Bucket) Name() string

Name returns the bucket name for gcs.

func (*Bucket) SupportedIterOptions

func (b *Bucket) SupportedIterOptions() []objstore.IterOptionType

func (*Bucket) Upload

func (b *Bucket) Upload(ctx context.Context, name string, r io.Reader) error

Upload writes the file specified in src to remote GCS location specified as target.

type Config

type Config struct {
	Bucket         string `yaml:"bucket"`
	ServiceAccount string `yaml:"service_account"`
	UseGRPC        bool   `yaml:"use_grpc"`
	// GRPCConnPoolSize controls the size of the gRPC connection pool and should only be used
	// when direct path is not enabled.
	// See https://pkg.go.dev/cloud.google.com/go/storage#hdr-Experimental_gRPC_API for more details
	// on how to enable direct path.
	GRPCConnPoolSize int                `yaml:"grpc_conn_pool_size"`
	HTTPConfig       exthttp.HTTPConfig `yaml:"http_config"`

	// ChunkSizeBytes controls the maximum number of bytes of the object that the
	// Writer will attempt to send to the server in a single request
	// Used as storage.Writer.ChunkSize of https://pkg.go.dev/google.golang.org/cloud/storage#Writer
	ChunkSizeBytes int `yaml:"chunk_size_bytes"`

	// MaxRetries controls the number of retries for idempotent operations.
	// Overrides the default gcs storage client behavior if this value is greater than 0.
	// Set this to 1 to disable retries.
	MaxRetries int `yaml:"max_retries"`
	// contains filtered or unexported fields
}

Config stores the configuration for gcs bucket.