aws

Documentation

Index

• Constants
• Variables
• func NewDynamoDBChunkClient(cfg DynamoDBConfig, schemaCfg config.SchemaConfig, reg prometheus.Registerer) (chunkclient.Client, error)
• func NewDynamoDBIndexClient(cfg DynamoDBConfig, schemaCfg config.SchemaConfig, reg prometheus.Registerer) (index.Client, error)
• func NewDynamoDBIndexReader(cfg DynamoDBConfig, schemaCfg config.SchemaConfig, reg prometheus.Registerer, ...) (index.Reader, error)
• func NewDynamoDBTableClient(cfg DynamoDBConfig, reg prometheus.Registerer) (index.TableClient, error)
• type DynamoDBConfig
  • func (cfg *DynamoDBConfig) RegisterFlags(f *flag.FlagSet)
• type HTTPConfig
• type InjectRequestMiddleware
• type MetricsAutoScalingConfig
  • func (cfg *MetricsAutoScalingConfig) RegisterFlags(f *flag.FlagSet)
• type S3Config
  • func (cfg *S3Config) RegisterFlags(f *flag.FlagSet)
  • func (cfg *S3Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)
  • func (cfg *S3Config) Validate() error
• type S3ObjectClient
  • func NewS3ObjectClient(cfg S3Config, hedgingCfg hedging.Config) (*S3ObjectClient, error)
  • func (a *S3ObjectClient) DeleteObject(ctx context.Context, objectKey string) error
  • func (a *S3ObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error)
  • func (a *S3ObjectClient) IsObjectNotFoundErr(err error) bool
  • func (a *S3ObjectClient) List(ctx context.Context, prefix, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error)
  • func (a *S3ObjectClient) PutObject(ctx context.Context, objectKey string, object io.ReadSeeker) error
  • func (a *S3ObjectClient) Stop()
• type SSEParsedConfig
  • func NewSSEParsedConfig(cfg bucket_s3.SSEConfig) (*SSEParsedConfig, error)
• type StorageConfig
  • func (cfg *StorageConfig) RegisterFlags(f *flag.FlagSet)
  • func (cfg *StorageConfig) Validate() error

Constants

const (
	SignatureVersionV4 = "v4"
	SignatureVersionV2 = "v2"
)

Variables

var Fixtures = []testutils.Fixture{
	fixture{
		// contains filtered or unexported fields
	},
	dynamoDBFixture(0, 10, 20),
	dynamoDBFixture(0, 0, 20),
	dynamoDBFixture(2, 10, 20),
}

Fixtures for testing the various configuration of AWS storage.

Functions

func NewDynamoDBChunkClient

func NewDynamoDBChunkClient(cfg DynamoDBConfig, schemaCfg config.SchemaConfig, reg prometheus.Registerer) (chunkclient.Client, error)

NewDynamoDBChunkClient makes a new DynamoDB-backed chunk.Client.

func NewDynamoDBIndexClient

func NewDynamoDBIndexClient(cfg DynamoDBConfig, schemaCfg config.SchemaConfig, reg prometheus.Registerer) (index.Client, error)

NewDynamoDBIndexClient makes a new DynamoDB-backed IndexClient.

func NewDynamoDBIndexReader

func NewDynamoDBIndexReader(cfg DynamoDBConfig, schemaCfg config.SchemaConfig, reg prometheus.Registerer, l gklog.Logger, rowsRead prometheus.Counter) (index.Reader, error)

NewDynamoDBIndexReader returns an object that can scan an entire index table

func NewDynamoDBTableClient

func NewDynamoDBTableClient(cfg DynamoDBConfig, reg prometheus.Registerer) (index.TableClient, error)

NewDynamoDBTableClient makes a new DynamoTableClient.

Types

type DynamoDBConfig

type DynamoDBConfig struct {
	DynamoDB               flagext.URLValue         `yaml:"dynamodb_url"`
	APILimit               float64                  `yaml:"api_limit"`
	ThrottleLimit          float64                  `yaml:"throttle_limit"`
	Metrics                MetricsAutoScalingConfig `yaml:"metrics"`
	ChunkGangSize          int                      `yaml:"chunk_gang_size"`
	ChunkGetMaxParallelism int                      `yaml:"chunk_get_max_parallelism"`
	BackoffConfig          backoff.Config           `yaml:"backoff_config"`
}

DynamoDBConfig specifies config for a DynamoDB database.

func (*DynamoDBConfig) RegisterFlags

func (cfg *DynamoDBConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

type HTTPConfig

type HTTPConfig struct {
	IdleConnTimeout       time.Duration `yaml:"idle_conn_timeout"`
	ResponseHeaderTimeout time.Duration `yaml:"response_header_timeout"`
	InsecureSkipVerify    bool          `yaml:"insecure_skip_verify"`
	CAFile                string        `yaml:"ca_file"`
}

HTTPConfig stores the http.Transport configuration

type InjectRequestMiddleware

type InjectRequestMiddleware func(next http.RoundTripper) http.RoundTripper

InjectRequestMiddleware gives users of this client the ability to make arbitrary changes to outgoing requests.

type MetricsAutoScalingConfig

type MetricsAutoScalingConfig struct {
	URL              string  `yaml:"url"`                   // URL to contact Prometheus store on
	TargetQueueLen   int64   `yaml:"target_queue_length"`   // Queue length above which we will scale up capacity
	ScaleUpFactor    float64 `yaml:"scale_up_factor"`       // Scale up capacity by this multiple
	MinThrottling    float64 `yaml:"ignore_throttle_below"` // Ignore throttling below this level
	QueueLengthQuery string  `yaml:"queue_length_query"`    // Promql query to fetch ingester queue length
	ThrottleQuery    string  `yaml:"write_throttle_query"`  // Promql query to fetch throttle rate per table
	UsageQuery       string  `yaml:"write_usage_query"`     // Promql query to fetch write capacity usage per table
	ReadUsageQuery   string  `yaml:"read_usage_query"`      // Promql query to fetch read usage per table
	ReadErrorQuery   string  `yaml:"read_error_query"`      // Promql query to fetch read errors per table
}

MetricsAutoScalingConfig holds parameters to configure how it works

func (*MetricsAutoScalingConfig) RegisterFlags

func (cfg *MetricsAutoScalingConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

type S3Config

type S3Config struct {
	S3               flagext.URLValue
	S3ForcePathStyle bool

	BucketNames      string
	Endpoint         string              `yaml:"endpoint"`
	Region           string              `yaml:"region"`
	AccessKeyID      string              `yaml:"access_key_id"`
	SecretAccessKey  flagext.Secret      `yaml:"secret_access_key"`
	Insecure         bool                `yaml:"insecure"`
	SSEEncryption    bool                `yaml:"sse_encryption"`
	HTTPConfig       HTTPConfig          `yaml:"http_config"`
	SignatureVersion string              `yaml:"signature_version"`
	SSEConfig        bucket_s3.SSEConfig `yaml:"sse"`
	BackoffConfig    backoff.Config      `yaml:"backoff_config"`

	Inject InjectRequestMiddleware `yaml:"-"`
}

S3Config specifies config for storing chunks on AWS S3.

func (*S3Config) RegisterFlags

func (cfg *S3Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

func (*S3Config) RegisterFlagsWithPrefix

func (cfg *S3Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet with a specified prefix

func (*S3Config) Validate

func (cfg *S3Config) Validate() error

Validate config and returns error on failure

type S3ObjectClient

type S3ObjectClient struct {
	S3 s3iface.S3API
	// contains filtered or unexported fields
}

func NewS3ObjectClient

func NewS3ObjectClient(cfg S3Config, hedgingCfg hedging.Config) (*S3ObjectClient, error)

NewS3ObjectClient makes a new S3-backed ObjectClient.

func (*S3ObjectClient) DeleteObject

func (a *S3ObjectClient) DeleteObject(ctx context.Context, objectKey string) error

DeleteObject deletes the specified objectKey from the appropriate S3 bucket

func (*S3ObjectClient) GetObject

func (a *S3ObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error)

GetObject returns a reader and the size for the specified object key from the configured S3 bucket.

func (*S3ObjectClient) IsObjectNotFoundErr

func (a *S3ObjectClient) IsObjectNotFoundErr(err error) bool

IsObjectNotFoundErr returns true if error means that object is not found. Relevant to GetObject and DeleteObject operations.

func (*S3ObjectClient) List

func (a *S3ObjectClient) List(ctx context.Context, prefix, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error)

List implements chunk.ObjectClient.

func (*S3ObjectClient) PutObject

func (a *S3ObjectClient) PutObject(ctx context.Context, objectKey string, object io.ReadSeeker) error

PutObject into the store

func (*S3ObjectClient) Stop

func (a *S3ObjectClient) Stop()

Stop fulfills the chunk.ObjectClient interface

type SSEParsedConfig

type SSEParsedConfig struct {
	ServerSideEncryption string
	KMSKeyID             *string
	KMSEncryptionContext *string
}

SSEParsedConfig configures server side encryption (SSE) struct used internally to configure AWS S3

func NewSSEParsedConfig

func NewSSEParsedConfig(cfg bucket_s3.SSEConfig) (*SSEParsedConfig, error)

NewSSEParsedConfig creates a struct to configure server side encryption (SSE)

type StorageConfig

type StorageConfig struct {
	DynamoDBConfig `yaml:"dynamodb"`
	S3Config       `yaml:",inline"`
}

StorageConfig specifies config for storing data on AWS.

func (*StorageConfig) RegisterFlags

func (cfg *StorageConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

func (*StorageConfig) Validate

func (cfg *StorageConfig) Validate() error

Validate config and returns error on failure