bucketindex

Documentation

Index

• Constants
• Variables
• func BlockDeletionMarkFilepath(blockID ulid.ULID) string
• func BucketWithGlobalMarkers(b objstore.Bucket) objstore.Bucket
• func IsBlockDeletionMarkFilename(name string) (ulid.ULID, bool)
• type Block
  • func BlockFromThanosMeta(meta metadata.Meta) *Block
  • func (m *Block) GetUploadedAt() time.Time
  • func (m *Block) String() string
  • func (m *Block) ThanosMeta(userID string) metadata.Meta
• type BlockDeletionMark
  • func BlockDeletionMarkFromThanosMarker(mark *metadata.DeletionMark) *BlockDeletionMark
• type Blocks
  • func (s Blocks) GetULIDs() []ulid.ULID
  • func (s Blocks) String() string
• type Index
  • func ReadIndex(ctx context.Context, bkt objstore.Bucket, userID string, logger log.Logger) (*Index, error)
• type Writer
  • func NewWriter(bkt objstore.Bucket, userID string, logger log.Logger) *Writer
  • func (w *Writer) GenerateIndex(ctx context.Context, old *Index) (*Index, error)
  • func (w *Writer) WriteIndex(ctx context.Context, old *Index) (*Index, error)

Constants

const (
	IndexFilename           = "bucket-index.json"
	IndexCompressedFilename = IndexFilename + ".gz"
	IndexVersion1           = 1

	SegmentsFormatUnknown = ""

	// SegmentsFormat1Based6Digits defined segments numbered with 6 digits numbers in a sequence starting from number 1
	// eg. (000001, 000002, 000003).
	SegmentsFormat1Based6Digits = "1b6d"
)

const (
	MarkersPathname = "markers"
)

Variables

var (
	ErrIndexNotFound  = errors.New("bucket index not found")
	ErrIndexCorrupted = errors.New("bucket index corrupted")
)

var (
	ErrBlockMetaNotFound          = block.ErrorSyncMetaNotFound
	ErrBlockMetaCorrupted         = block.ErrorSyncMetaCorrupted
	ErrBlockDeletionMarkNotFound  = errors.New("block deletion mark not found")
	ErrBlockDeletionMarkCorrupted = errors.New("block deletion mark corrupted")
)

Functions

func BlockDeletionMarkFilepath

func BlockDeletionMarkFilepath(blockID ulid.ULID) string

BlockDeletionMarkFilepath returns the path, relative to the tenant's bucket location, of a block deletion mark in the bucket markers location.

func BucketWithGlobalMarkers

func BucketWithGlobalMarkers(b objstore.Bucket) objstore.Bucket

BucketWithGlobalMarkers wraps the input bucket into a bucket which also keeps track of markers in the global markers location.

func IsBlockDeletionMarkFilename

func IsBlockDeletionMarkFilename(name string) (ulid.ULID, bool)

IsBlockDeletionMarkFilename returns whether the input filename matches the expected pattern of block deletion markers stored in the markers location.

Types

type Block

type Block struct {
	// Block ID.
	ID ulid.ULID `json:"block_id"`

	// MinTime and MaxTime specify the time range all samples in the block are in (millis precision).
	MinTime int64 `json:"min_time"`
	MaxTime int64 `json:"max_time"`

	// SegmentsFormat and SegmentsNum stores the format and number of chunks segments
	// in the block, if they match a known pattern. We don't store the full segments
	// files list in order to keep the index small. SegmentsFormat is empty if segments
	// are unknown or don't match a known format.
	SegmentsFormat string `json:"segments_format,omitempty"`
	SegmentsNum    int    `json:"segments_num,omitempty"`

	// UploadedAt is a unix timestamp (seconds precision) of when the block has been completed to be uploaded
	// to the storage.
	UploadedAt int64 `json:"uploaded_at"`
}

Block holds the information about a block in the index.

func BlockFromThanosMeta

func BlockFromThanosMeta(meta metadata.Meta) *Block

func (*Block) GetUploadedAt

func (m *Block) GetUploadedAt() time.Time

func (*Block) String

func (m *Block) String() string

func (*Block) ThanosMeta

func (m *Block) ThanosMeta(userID string) metadata.Meta

ThanosMeta returns a block meta based on the known information in the index. The returned meta doesn't include all original meta.json data but only a subset of it.

type BlockDeletionMark

type BlockDeletionMark struct {
	// Block ID.
	ID ulid.ULID `json:"block_id"`

	// DeletionTime is a unix timestamp (seconds precision) of when the block was marked to be deleted.
	DeletionTime int64 `json:"deletion_time"`
}

BlockDeletionMark holds the information about a block's deletion mark in the index.

func BlockDeletionMarkFromThanosMarker

func BlockDeletionMarkFromThanosMarker(mark *metadata.DeletionMark) *BlockDeletionMark

type Blocks

type Blocks []*Block

Blocks holds a set of blocks in the index. No ordering guaranteed.

func (Blocks) GetULIDs

func (s Blocks) GetULIDs() []ulid.ULID

func (Blocks) String

func (s Blocks) String() string

type Index

type Index struct {
	// Version of the index format.
	Version int `json:"version"`

	// List of complete blocks (partial blocks are excluded from the index).
	Blocks []*Block `json:"blocks"`

	// List of block deletion marks.
	BlockDeletionMarks []*BlockDeletionMark `json:"block_deletion_marks"`

	// UpdatedAt is a unix timestamp (seconds precision) of when the index has been updated
	// (written in the storage) the last time.
	UpdatedAt int64 `json:"updated_at"`
}

Index contains all known blocks and markers of a tenant.

func ReadIndex

func ReadIndex(ctx context.Context, bkt objstore.Bucket, userID string, logger log.Logger) (*Index, error)

ReadIndex reads, parses and returns a bucket index from the bucket.

type Writer

type Writer struct {
	// contains filtered or unexported fields
}

Writer is responsible to generate and write a bucket index.

func NewWriter

func NewWriter(bkt objstore.Bucket, userID string, logger log.Logger) *Writer

func (*Writer) GenerateIndex

func (w *Writer) GenerateIndex(ctx context.Context, old *Index) (*Index, error)

GenerateIndex generates the bucket index and returns it, without storing it to the storage. If the old index is not passed in input, then the bucket index will be generated from scratch.

func (*Writer) WriteIndex

func (w *Writer) WriteIndex(ctx context.Context, old *Index) (*Index, error)

WriteIndex generates the bucket index and writes it to the storage. If the old index is not passed in input, then the bucket index will be generated from scratch.