client

Documentation

Index

• Variables
• type ChunkClientMetrics
  • func NewChunkClientMetrics(reg prometheus.Registerer) ChunkClientMetrics
• type Client
  • func NewClient(store ObjectClient, encoder KeyEncoder, schema config.SchemaConfig) Client
  • func NewClientWithMaxParallel(store ObjectClient, encoder KeyEncoder, maxParallel int, ...) Client
• type KeyEncoder
• type MetricsChunkClient
  • func NewMetricsChunkClient(client Client, metrics ChunkClientMetrics) MetricsChunkClient
  • func (c MetricsChunkClient) DeleteChunk(ctx context.Context, userID, chunkID string) error
  • func (c MetricsChunkClient) GetChunks(ctx context.Context, chunks []chunk.Chunk) ([]chunk.Chunk, error)
  • func (c MetricsChunkClient) IsChunkNotFoundErr(err error) bool
  • func (c MetricsChunkClient) IsRetryableErr(err error) bool
  • func (c MetricsChunkClient) PutChunks(ctx context.Context, chunks []chunk.Chunk) error
  • func (c MetricsChunkClient) Stop()
• type ObjectAndIndexClient
• type ObjectAttributes
• type ObjectClient
  • func NewPrefixedObjectClient(downstreamClient ObjectClient, prefix string) ObjectClient
• type PrefixedObjectClient
  • func (p PrefixedObjectClient) DeleteObject(ctx context.Context, objectKey string) error
  • func (p PrefixedObjectClient) GetAttributes(ctx context.Context, objectKey string) (ObjectAttributes, error)
  • func (p PrefixedObjectClient) GetDownstream() ObjectClient
  • func (p PrefixedObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error)
  • func (p PrefixedObjectClient) GetObjectRange(ctx context.Context, objectKey string, offset, length int64) (io.ReadCloser, error)
  • func (p PrefixedObjectClient) GetPrefix() string
  • func (p PrefixedObjectClient) IsObjectNotFoundErr(err error) bool
  • func (p PrefixedObjectClient) IsRetryableErr(err error) bool
  • func (p PrefixedObjectClient) List(ctx context.Context, prefix, delimiter string) ([]StorageObject, []StorageCommonPrefix, error)
  • func (p PrefixedObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error)
  • func (p PrefixedObjectClient) PutObject(ctx context.Context, objectKey string, object io.Reader) error
  • func (p PrefixedObjectClient) Stop()
• type StorageCommonPrefix
• type StorageObject

Variables

var (
	// ErrMethodNotImplemented when any of the storage clients do not implement a method
	ErrMethodNotImplemented = errors.New("method is not implemented")
	// ErrStorageObjectNotFound when object storage does not have requested object
	ErrStorageObjectNotFound = errors.New("object not found in storage")
)

var FSEncoder = func(schema config.SchemaConfig, chk chunk.Chunk) string {

	key := schema.ExternalKey(chk.ChunkRef)
	if schema.VersionForChunk(chk.ChunkRef) > 11 {
		split := strings.LastIndexByte(key, '/')
		encodedTail := base64Encoder(key[split+1:])
		return strings.Join([]string{key[:split], encodedTail}, "/")

	}
	return base64Encoder(key)
}

Types

type ChunkClientMetrics

type ChunkClientMetrics struct {
	// contains filtered or unexported fields
}

func NewChunkClientMetrics

func NewChunkClientMetrics(reg prometheus.Registerer) ChunkClientMetrics

type Client

type Client interface {
	Stop()
	PutChunks(ctx context.Context, chunks []chunk.Chunk) error
	GetChunks(ctx context.Context, chunks []chunk.Chunk) ([]chunk.Chunk, error)
	DeleteChunk(ctx context.Context, userID, chunkID string) error
	IsChunkNotFoundErr(err error) bool
	IsRetryableErr(err error) bool
}

Client is for storing and retrieving chunks.

func NewClient

func NewClient(store ObjectClient, encoder KeyEncoder, schema config.SchemaConfig) Client

NewClient wraps the provided ObjectClient with a chunk.Client implementation

func NewClientWithMaxParallel

func NewClientWithMaxParallel(store ObjectClient, encoder KeyEncoder, maxParallel int, schema config.SchemaConfig) Client

type KeyEncoder

type KeyEncoder func(schema config.SchemaConfig, chk chunk.Chunk) string

KeyEncoder is used to encode chunk keys before writing/retrieving chunks from the underlying ObjectClient Schema/Chunk are passed as arguments to allow this to improve over revisions

type MetricsChunkClient

type MetricsChunkClient struct {
	Client Client
	// contains filtered or unexported fields
}

takes a chunk client and exposes metrics for its operations.

func NewMetricsChunkClient

func NewMetricsChunkClient(client Client, metrics ChunkClientMetrics) MetricsChunkClient

func (MetricsChunkClient) DeleteChunk

func (c MetricsChunkClient) DeleteChunk(ctx context.Context, userID, chunkID string) error

func (MetricsChunkClient) GetChunks

func (c MetricsChunkClient) GetChunks(ctx context.Context, chunks []chunk.Chunk) ([]chunk.Chunk, error)

func (MetricsChunkClient) IsChunkNotFoundErr

func (c MetricsChunkClient) IsChunkNotFoundErr(err error) bool

func (MetricsChunkClient) IsRetryableErr

func (c MetricsChunkClient) IsRetryableErr(err error) bool

func (MetricsChunkClient) PutChunks

func (c MetricsChunkClient) PutChunks(ctx context.Context, chunks []chunk.Chunk) error

func (MetricsChunkClient) Stop

func (c MetricsChunkClient) Stop()

type ObjectAndIndexClient

type ObjectAndIndexClient interface {
	PutChunksAndIndex(ctx context.Context, chunks []chunk.Chunk, index index.WriteBatch) error
}

ObjectAndIndexClient allows optimisations where the same client handles both Only used by DynamoDB (dynamodbIndexReader and dynamoDBStorageClient)

type ObjectAttributes

type ObjectAttributes struct {
	Size int64
}

type ObjectClient

type ObjectClient interface {
	ObjectExists(ctx context.Context, objectKey string) (bool, error)
	GetAttributes(ctx context.Context, objectKey string) (ObjectAttributes, error)

	PutObject(ctx context.Context, objectKey string, object io.Reader) error
	// NOTE: The consumer of GetObject should always call the Close method when it is done reading which otherwise could cause a resource leak.
	GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error)
	GetObjectRange(ctx context.Context, objectKey string, off, length int64) (io.ReadCloser, error)

	// List objects with given prefix.
	//
	// If delimiter is empty, all objects are returned, even if they are in nested in "subdirectories".
	// If delimiter is not empty, it is used to compute common prefixes ("subdirectories"),
	// and objects containing delimiter in the name will not be returned in the result.
	//
	// For example, if the prefix is "notes/" and the delimiter is a slash (/) as in "notes/summer/july", the common prefix is "notes/summer/".
	// Common prefixes will always end with passed delimiter.
	//
	// Keys of returned storage objects have given prefix.
	List(ctx context.Context, prefix string, delimiter string) ([]StorageObject, []StorageCommonPrefix, error)
	DeleteObject(ctx context.Context, objectKey string) error
	IsObjectNotFoundErr(err error) bool
	IsRetryableErr(err error) bool
	Stop()
}

ObjectClient is used to store arbitrary data in Object Store (S3/GCS/Azure/...)

func NewPrefixedObjectClient

func NewPrefixedObjectClient(downstreamClient ObjectClient, prefix string) ObjectClient

type PrefixedObjectClient

type PrefixedObjectClient struct {
	// contains filtered or unexported fields
}

func (PrefixedObjectClient) DeleteObject

func (p PrefixedObjectClient) DeleteObject(ctx context.Context, objectKey string) error

func (PrefixedObjectClient) GetAttributes

func (p PrefixedObjectClient) GetAttributes(ctx context.Context, objectKey string) (ObjectAttributes, error)

func (PrefixedObjectClient) GetDownstream

func (p PrefixedObjectClient) GetDownstream() ObjectClient

func (PrefixedObjectClient) GetObject

func (p PrefixedObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error)

func (PrefixedObjectClient) GetObjectRange

func (p PrefixedObjectClient) GetObjectRange(ctx context.Context, objectKey string, offset, length int64) (io.ReadCloser, error)

func (PrefixedObjectClient) GetPrefix

func (p PrefixedObjectClient) GetPrefix() string

func (PrefixedObjectClient) IsObjectNotFoundErr

func (p PrefixedObjectClient) IsObjectNotFoundErr(err error) bool

func (PrefixedObjectClient) IsRetryableErr

func (p PrefixedObjectClient) IsRetryableErr(err error) bool

func (PrefixedObjectClient) List

func (p PrefixedObjectClient) List(ctx context.Context, prefix, delimiter string) ([]StorageObject, []StorageCommonPrefix, error)

func (PrefixedObjectClient) ObjectExists

func (p PrefixedObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error)

func (PrefixedObjectClient) PutObject

func (p PrefixedObjectClient) PutObject(ctx context.Context, objectKey string, object io.Reader) error

func (PrefixedObjectClient) Stop

func (p PrefixedObjectClient) Stop()

type StorageCommonPrefix

type StorageCommonPrefix string

StorageCommonPrefix represents a common prefix aka a synthetic directory in Object Store. It is guaranteed to always end with delimiter passed to List method.

type StorageObject

type StorageObject struct {
	Key        string
	ModifiedAt time.Time
}

StorageObject represents an object being stored in an Object Store