gcp

Documentation

Index

• Variables
• func HashPrefix(input string) string
• func NewBigtableObjectClient(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (client.Client, error)
• func NewStorageClientColumnKey(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (index.Client, error)
• func NewStorageClientV1(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (index.Client, error)
• func NewTableClient(ctx context.Context, cfg Config) (index.TableClient, error)
• type ClientFactory
• type Config
  • func (cfg *Config) RegisterFlags(f *flag.FlagSet)
  • func (cfg *Config) Validate() error
• type GCSConfig
  • func (cfg *GCSConfig) RegisterFlags(f *flag.FlagSet)
  • func (cfg *GCSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)
• type GCSObjectClient
  • func NewGCSObjectClient(ctx context.Context, cfg GCSConfig, hedgingCfg hedging.Config) (*GCSObjectClient, error)
  • func (s *GCSObjectClient) DeleteObject(ctx context.Context, objectKey string) error
  • func (s *GCSObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error)
  • func (s *GCSObjectClient) GetObjectRange(ctx context.Context, objectKey string, offset, length int64) (io.ReadCloser, error)
  • func (s *GCSObjectClient) IsObjectNotFoundErr(err error) bool
  • func (s *GCSObjectClient) IsRetryableErr(err error) bool
  • func (s *GCSObjectClient) IsStorageThrottledErr(err error) bool
  • func (s *GCSObjectClient) IsStorageTimeoutErr(err error) bool
  • func (s *GCSObjectClient) List(ctx context.Context, prefix, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error)
  • func (s *GCSObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error)
  • func (s *GCSObjectClient) PutObject(ctx context.Context, objectKey string, object io.Reader) error
  • func (s *GCSObjectClient) Stop()

Variables

var Fixtures = func() []testutils.Fixture {
	fixtures := []testutils.Fixture{}
	for _, gcsObjectClient := range []bool{true, false} {
		for _, columnKeyClient := range []bool{true, false} {
			for _, hashPrefix := range []bool{true, false} {
				fixtures = append(fixtures, &fixture{
					name:            fmt.Sprintf("bigtable-columnkey:%v-gcsObjectClient:%v-hashPrefix:%v", columnKeyClient, gcsObjectClient, hashPrefix),
					columnKeyClient: columnKeyClient,
					gcsObjectClient: gcsObjectClient,
					hashPrefix:      hashPrefix,
				})
			}
		}
	}
	return fixtures
}()

Fixtures for unit testing GCP storage.

Functions

func HashPrefix

func HashPrefix(input string) string

HashPrefix calculates a 64bit hash of the input string and hex-encodes the result, taking care to zero pad etc.

func NewBigtableObjectClient

func NewBigtableObjectClient(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (client.Client, error)

NewBigtableObjectClient makes a new chunk.Client that stores chunks in Bigtable.

func NewStorageClientColumnKey

func NewStorageClientColumnKey(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (index.Client, error)

NewStorageClientColumnKey returns a new v2 StorageClient.

func NewStorageClientV1

func NewStorageClientV1(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (index.Client, error)

NewStorageClientV1 returns a new v1 StorageClient.

func NewTableClient

func NewTableClient(ctx context.Context, cfg Config) (index.TableClient, error)

NewTableClient returns a new TableClient.

Types

type ClientFactory

type ClientFactory func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error)

type Config

type Config struct {
	Project  string `yaml:"project"`
	Instance string `yaml:"instance"`

	GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"`

	ColumnKey      bool `yaml:"-"`
	DistributeKeys bool `yaml:"-"`

	TableCacheEnabled    bool          `yaml:"table_cache_enabled"`
	TableCacheExpiration time.Duration `yaml:"table_cache_expiration"`
}

Config for a StorageClient

func (*Config) RegisterFlags

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

func (*Config) Validate

func (cfg *Config) Validate() error

type GCSConfig

type GCSConfig struct {
	BucketName       string         `yaml:"bucket_name"`
	ServiceAccount   flagext.Secret `yaml:"service_account"`
	ChunkBufferSize  int            `yaml:"chunk_buffer_size"`
	RequestTimeout   time.Duration  `yaml:"request_timeout"`
	EnableOpenCensus bool           `yaml:"enable_opencensus"`
	EnableHTTP2      bool           `yaml:"enable_http2"`

	// TODO(dannyk): remove this and disable GCS client retries; move a layer higher instead.
	EnableRetries bool `yaml:"enable_retries"`

	Insecure bool `yaml:"-"`
}

GCSConfig is config for the GCS Chunk Client.

func (*GCSConfig) RegisterFlags

func (cfg *GCSConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers flags.

func (*GCSConfig) RegisterFlagsWithPrefix

func (cfg *GCSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

RegisterFlagsWithPrefix registers flags with prefix.

type GCSObjectClient

type GCSObjectClient struct {
	// contains filtered or unexported fields
}

func NewGCSObjectClient

func NewGCSObjectClient(ctx context.Context, cfg GCSConfig, hedgingCfg hedging.Config) (*GCSObjectClient, error)

NewGCSObjectClient makes a new chunk.Client that writes chunks to GCS.

func (*GCSObjectClient) DeleteObject

func (s *GCSObjectClient) DeleteObject(ctx context.Context, objectKey string) error

DeleteObject deletes the specified object key from the configured GCS bucket.

func (*GCSObjectClient) GetObject

func (s *GCSObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error)

GetObject returns a reader and the size for the specified object key from the configured GCS bucket.

func (*GCSObjectClient) GetObjectRange

func (s *GCSObjectClient) GetObjectRange(ctx context.Context, objectKey string, offset, length int64) (io.ReadCloser, error)

GetObject returns a reader and the size for the specified object key from the configured GCS bucket.

func (*GCSObjectClient) IsObjectNotFoundErr

func (s *GCSObjectClient) IsObjectNotFoundErr(err error) bool

IsObjectNotFoundErr returns true if error means that object is not found. Relevant to GetObject and DeleteObject operations.

func (*GCSObjectClient) IsRetryableErr

func (s *GCSObjectClient) IsRetryableErr(err error) bool

IsRetryableErr returns true if the request failed due to some retryable server-side scenario

func (*GCSObjectClient) IsStorageThrottledErr

func (s *GCSObjectClient) IsStorageThrottledErr(err error) bool

IsStorageThrottledErr returns true if error means that object cannot be retrieved right now due to throttling.

func (*GCSObjectClient) IsStorageTimeoutErr

func (s *GCSObjectClient) IsStorageTimeoutErr(err error) bool

IsStorageTimeoutErr returns true if error means that object cannot be retrieved right now due to server-side timeouts.

func (*GCSObjectClient) List

func (s *GCSObjectClient) List(ctx context.Context, prefix, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error)

List implements chunk.ObjectClient.

func (*GCSObjectClient) ObjectExists

func (s *GCSObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error)

func (*GCSObjectClient) PutObject

func (s *GCSObjectClient) PutObject(ctx context.Context, objectKey string, object io.Reader) error

PutObject puts the specified bytes into the configured GCS bucket at the provided key

func (*GCSObjectClient) Stop

func (s *GCSObjectClient) Stop()