gcp

Documentation

Index

• Variables
• func HashPrefix(input string) string
• func IsRetryableErr(err error) bool
• func IsStorageThrottledErr(err error) bool
• func IsStorageTimeoutErr(err error) bool
• func NewBigtableObjectClient(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (client.Client, error)
• func NewGCSThanosObjectClient(ctx context.Context, cfg bucket.Config, component string, logger log.Logger, ...) (client.ObjectClient, error)
• func NewStorageClientColumnKey(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (index.Client, error)
• func NewStorageClientV1(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (index.Client, error)
• func NewTableClient(ctx context.Context, cfg Config) (index.TableClient, error)
• type ClientFactory
• type Config
  • func (cfg *Config) RegisterFlags(f *flag.FlagSet)
  • func (cfg *Config) Validate() error
• type GCSConfig
  • func (cfg *GCSConfig) RegisterFlags(f *flag.FlagSet)
  • func (cfg *GCSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)
• type GCSObjectClient
  • func NewGCSObjectClient(ctx context.Context, cfg GCSConfig, hedgingCfg hedging.Config) (*GCSObjectClient, error)
  • func (s *GCSObjectClient) DeleteObject(ctx context.Context, objectKey string) error
  • func (s *GCSObjectClient) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error)
  • func (s *GCSObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error)
  • func (s *GCSObjectClient) GetObjectRange(ctx context.Context, objectKey string, offset, length int64) (io.ReadCloser, error)
  • func (s *GCSObjectClient) IsObjectNotFoundErr(err error) bool
  • func (s *GCSObjectClient) IsRetryableErr(err error) bool
  • func (s *GCSObjectClient) List(ctx context.Context, prefix, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error)
  • func (s *GCSObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error)
  • func (s *GCSObjectClient) PutObject(ctx context.Context, objectKey string, object io.Reader) error
  • func (s *GCSObjectClient) Stop()

Variables

var Fixtures = func() []testutils.Fixture {
	fixtures := []testutils.Fixture{}
	for _, gcsObjectClient := range []bool{true, false} {
		for _, columnKeyClient := range []bool{true, false} {
			for _, hashPrefix := range []bool{true, false} {
				fixtures = append(fixtures, &fixture{
					name:            fmt.Sprintf("bigtable-columnkey:%v-gcsObjectClient:%v-hashPrefix:%v", columnKeyClient, gcsObjectClient, hashPrefix),
					columnKeyClient: columnKeyClient,
					gcsObjectClient: gcsObjectClient,
					hashPrefix:      hashPrefix,
				})
			}
		}
	}
	return fixtures
}()

Fixtures for unit testing GCP storage.

Functions

func HashPrefix

func HashPrefix(input string) string

HashPrefix calculates a 64bit hash of the input string and hex-encodes the result, taking care to zero pad etc.

func IsRetryableErr

func IsRetryableErr(err error) bool

IsRetryableErr returns true if the request failed due to some retryable server-side scenario

func IsStorageThrottledErr

func IsStorageThrottledErr(err error) bool

IsStorageThrottledErr returns true if error means that object cannot be retrieved right now due to throttling.

func IsStorageTimeoutErr

func IsStorageTimeoutErr(err error) bool

IsStorageTimeoutErr returns true if error means that object cannot be retrieved right now due to server-side timeouts.

func NewBigtableObjectClient

func NewBigtableObjectClient(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (client.Client, error)

NewBigtableObjectClient makes a new chunk.Client that stores chunks in Bigtable.

func NewGCSThanosObjectClient

func NewGCSThanosObjectClient(ctx context.Context, cfg bucket.Config, component string, logger log.Logger, hedgingCfg hedging.Config) (client.ObjectClient, error)

func NewStorageClientColumnKey

func NewStorageClientColumnKey(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (index.Client, error)

NewStorageClientColumnKey returns a new v2 StorageClient.

func NewStorageClientV1

func NewStorageClientV1(ctx context.Context, cfg Config, schemaCfg config.SchemaConfig) (index.Client, error)

NewStorageClientV1 returns a new v1 StorageClient.

func NewTableClient

func NewTableClient(ctx context.Context, cfg Config) (index.TableClient, error)

NewTableClient returns a new TableClient.

Types

type ClientFactory

type ClientFactory func(ctx context.Context, opts ...option.ClientOption) (*storage.Client, error)

type Config

type Config struct {
	Project  string `yaml:"project"`
	Instance string `yaml:"instance"`

	GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"`

	ColumnKey      bool `yaml:"-"`
	DistributeKeys bool `yaml:"-"`

	TableCacheEnabled    bool          `yaml:"table_cache_enabled"`
	TableCacheExpiration time.Duration `yaml:"table_cache_expiration"`
}

Config for a StorageClient

func (*Config) RegisterFlags

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

func (*Config) Validate

func (cfg *Config) Validate() error

type GCSConfig

type GCSConfig struct {
	BucketName       string         `yaml:"bucket_name"`
	ServiceAccount   flagext.Secret `yaml:"service_account"`
	ChunkBufferSize  int            `yaml:"chunk_buffer_size"`
	RequestTimeout   time.Duration  `yaml:"request_timeout"`
	EnableOpenCensus bool           `yaml:"enable_opencensus"`
	EnableHTTP2      bool           `yaml:"enable_http2"`

	// TODO(dannyk): remove this and disable GCS client retries; move a layer higher instead.
	EnableRetries bool `yaml:"enable_retries"`

	Insecure bool `yaml:"-"`
}

GCSConfig is config for the GCS Chunk Client.

func (*GCSConfig) RegisterFlags

func (cfg *GCSConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers flags.

func (*GCSConfig) RegisterFlagsWithPrefix

func (cfg *GCSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

RegisterFlagsWithPrefix registers flags with prefix.

type GCSObjectClient

type GCSObjectClient struct {
	// contains filtered or unexported fields
}

func NewGCSObjectClient

func NewGCSObjectClient(ctx context.Context, cfg GCSConfig, hedgingCfg hedging.Config) (*GCSObjectClient, error)

NewGCSObjectClient makes a new chunk.Client that writes chunks to GCS.

func (*GCSObjectClient) DeleteObject

func (s *GCSObjectClient) DeleteObject(ctx context.Context, objectKey string) error

DeleteObject deletes the specified object key from the configured GCS bucket.

func (*GCSObjectClient) GetAttributes

func (s *GCSObjectClient) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error)

func (*GCSObjectClient) GetObject

func (s *GCSObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error)

GetObject returns a reader and the size for the specified object key from the configured GCS bucket.

func (*GCSObjectClient) GetObjectRange

func (s *GCSObjectClient) GetObjectRange(ctx context.Context, objectKey string, offset, length int64) (io.ReadCloser, error)

GetObject returns a reader and the size for the specified object key from the configured GCS bucket.

func (*GCSObjectClient) IsObjectNotFoundErr

func (s *GCSObjectClient) IsObjectNotFoundErr(err error) bool

IsObjectNotFoundErr returns true if error means that object is not found. Relevant to GetObject and DeleteObject operations.

func (*GCSObjectClient) IsRetryableErr

func (s *GCSObjectClient) IsRetryableErr(err error) bool

IsRetryableErr returns true if the request failed due to some retryable server-side scenario

func (*GCSObjectClient) List

func (s *GCSObjectClient) List(ctx context.Context, prefix, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error)

List implements chunk.ObjectClient.

func (*GCSObjectClient) ObjectExists

func (s *GCSObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error)

func (*GCSObjectClient) PutObject

func (s *GCSObjectClient) PutObject(ctx context.Context, objectKey string, object io.Reader) error

PutObject puts the specified bytes into the configured GCS bucket at the provided key

func (*GCSObjectClient) Stop

func (s *GCSObjectClient) Stop()