aws

package
v3.0.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 9, 2024 License: AGPL-3.0 Imports: 60 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	SignatureVersionV4 = "v4"
)

Variables

View Source 
var Fixtures = []testutils.Fixture{
	fixture{
		// contains filtered or unexported fields
	},
	dynamoDBFixture(0, 10, 20),
	dynamoDBFixture(0, 0, 20),
	dynamoDBFixture(2, 10, 20),
}

Fixtures for testing the various configuration of AWS storage.

Functions

func NewDynamoDBChunkClient 

func NewDynamoDBChunkClient(cfg DynamoDBConfig, schemaCfg config.SchemaConfig, reg prometheus.Registerer) (chunkclient.Client, error)

NewDynamoDBChunkClient makes a new DynamoDB-backed chunk.Client.

func NewDynamoDBIndexClient 

func NewDynamoDBIndexClient(cfg DynamoDBConfig, schemaCfg config.SchemaConfig, reg prometheus.Registerer) (index.Client, error)

NewDynamoDBIndexClient makes a new DynamoDB-backed IndexClient.

func NewDynamoDBIndexReader 

func NewDynamoDBIndexReader(cfg DynamoDBConfig, schemaCfg config.SchemaConfig, reg prometheus.Registerer, l gklog.Logger, rowsRead prometheus.Counter) (index.Reader, error)

NewDynamoDBIndexReader returns an object that can scan an entire index table

func NewDynamoDBTableClient 

func NewDynamoDBTableClient(cfg DynamoDBConfig, reg prometheus.Registerer) (index.TableClient, error)

NewDynamoDBTableClient makes a new DynamoTableClient.

Types

type DynamoDBConfig 

type DynamoDBConfig struct {
	DynamoDB               flagext.URLValue         `yaml:"dynamodb_url"`
	APILimit               float64                  `yaml:"api_limit"`
	ThrottleLimit          float64                  `yaml:"throttle_limit"`
	Metrics                MetricsAutoScalingConfig `yaml:"metrics"`
	ChunkGangSize          int                      `yaml:"chunk_gang_size"`
	ChunkGetMaxParallelism int                      `yaml:"chunk_get_max_parallelism"`
	BackoffConfig          backoff.Config           `yaml:"backoff_config"`
	KMSKeyID               string                   `yaml:"kms_key_id"`
}

DynamoDBConfig specifies config for a DynamoDB database.

func (*DynamoDBConfig) RegisterFlags 

func (cfg *DynamoDBConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

type HTTPConfig 

type HTTPConfig struct {
	Timeout               time.Duration `yaml:"timeout"`
	IdleConnTimeout       time.Duration `yaml:"idle_conn_timeout"`
	ResponseHeaderTimeout time.Duration `yaml:"response_header_timeout"`
	InsecureSkipVerify    bool          `yaml:"insecure_skip_verify"`
	CAFile                string        `yaml:"ca_file"`
}

HTTPConfig stores the http.Transport configuration

type InjectRequestMiddleware 

type InjectRequestMiddleware func(next http.RoundTripper) http.RoundTripper

InjectRequestMiddleware gives users of this client the ability to make arbitrary changes to outgoing requests.

type MetricsAutoScalingConfig 

type MetricsAutoScalingConfig struct {
	URL              string  `yaml:"url"`                   // URL to contact Prometheus store on
	TargetQueueLen   int64   `yaml:"target_queue_length"`   // Queue length above which we will scale up capacity
	ScaleUpFactor    float64 `yaml:"scale_up_factor"`       // Scale up capacity by this multiple
	MinThrottling    float64 `yaml:"ignore_throttle_below"` // Ignore throttling below this level
	QueueLengthQuery string  `yaml:"queue_length_query"`    // Promql query to fetch ingester queue length
	ThrottleQuery    string  `yaml:"write_throttle_query"`  // Promql query to fetch throttle rate per table
	UsageQuery       string  `yaml:"write_usage_query"`     // Promql query to fetch write capacity usage per table
	ReadUsageQuery   string  `yaml:"read_usage_query"`      // Promql query to fetch read usage per table
	ReadErrorQuery   string  `yaml:"read_error_query"`      // Promql query to fetch read errors per table
}

MetricsAutoScalingConfig holds parameters to configure how it works

func (*MetricsAutoScalingConfig) RegisterFlags 

func (cfg *MetricsAutoScalingConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

type S3Config 

type S3Config struct {
	S3               flagext.URLValue
	S3ForcePathStyle bool

	BucketNames      string
	Endpoint         string              `yaml:"endpoint"`
	Region           string              `yaml:"region"`
	AccessKeyID      string              `yaml:"access_key_id"`
	SecretAccessKey  flagext.Secret      `yaml:"secret_access_key"`
	SessionToken     flagext.Secret      `yaml:"session_token"`
	Insecure         bool                `yaml:"insecure"`
	HTTPConfig       HTTPConfig          `yaml:"http_config"`
	SignatureVersion string              `yaml:"signature_version"`
	StorageClass     string              `yaml:"storage_class"`
	SSEConfig        bucket_s3.SSEConfig `yaml:"sse"`
	BackoffConfig    backoff.Config      `yaml:"backoff_config" doc:"description=Configures back off when S3 get Object."`

	Inject InjectRequestMiddleware `yaml:"-"`
}

S3Config specifies config for storing chunks on AWS S3.

func (*S3Config) RegisterFlags 

func (cfg *S3Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

func (*S3Config) RegisterFlagsWithPrefix 

func (cfg *S3Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet with a specified prefix

func (*S3Config) Validate 

func (cfg *S3Config) Validate() error

Validate config and returns error on failure

type S3ObjectClient 

type S3ObjectClient struct {
	S3 s3iface.S3API
	// contains filtered or unexported fields
}

func NewS3ObjectClient 

func NewS3ObjectClient(cfg S3Config, hedgingCfg hedging.Config) (*S3ObjectClient, error)

NewS3ObjectClient makes a new S3-backed ObjectClient.

func (*S3ObjectClient) DeleteObject 

func (a *S3ObjectClient) DeleteObject(ctx context.Context, objectKey string) error

DeleteObject deletes the specified objectKey from the appropriate S3 bucket

func (*S3ObjectClient) GetObject 

func (a *S3ObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error)

GetObject returns a reader and the size for the specified object key from the configured S3 bucket.

func (*S3ObjectClient) IsObjectNotFoundErr 

func (a *S3ObjectClient) IsObjectNotFoundErr(err error) bool

IsObjectNotFoundErr returns true if error means that object is not found. Relevant to GetObject and DeleteObject operations.

func (*S3ObjectClient) IsRetryableErr 

func (a *S3ObjectClient) IsRetryableErr(error) bool

TODO(dannyk): implement for client

func (*S3ObjectClient) List 

func (a *S3ObjectClient) List(ctx context.Context, prefix, delimiter string) ([]client.StorageObject, []client.StorageCommonPrefix, error)

List implements chunk.ObjectClient.

func (*S3ObjectClient) ObjectExists 

func (a *S3ObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error)

func (*S3ObjectClient) PutObject 

func (a *S3ObjectClient) PutObject(ctx context.Context, objectKey string, object io.ReadSeeker) error

PutObject into the store

func (*S3ObjectClient) Stop 

func (a *S3ObjectClient) Stop()

Stop fulfills the chunk.ObjectClient interface

type SSEParsedConfig 

type SSEParsedConfig struct {
	ServerSideEncryption string
	KMSKeyID             *string
	KMSEncryptionContext *string
}

SSEParsedConfig configures server side encryption (SSE) struct used internally to configure AWS S3

func NewSSEParsedConfig 

func NewSSEParsedConfig(cfg bucket_s3.SSEConfig) (*SSEParsedConfig, error)

NewSSEParsedConfig creates a struct to configure server side encryption (SSE)

type StorageConfig 

type StorageConfig struct {
	DynamoDBConfig `yaml:"dynamodb" doc:"description=Deprecated: Configures storing indexes in DynamoDB."`
	S3Config       `yaml:",inline"`
}

StorageConfig specifies config for storing data on AWS.

func (*StorageConfig) RegisterFlags 

func (cfg *StorageConfig) RegisterFlags(f *flag.FlagSet)

RegisterFlags adds the flags required to config this to the given FlagSet

func (*StorageConfig) Validate 

func (cfg *StorageConfig) Validate() error

Validate config and returns error on failure

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.