bucketindex

package
v1.18.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 15, 2024 License: Apache-2.0 Imports: 30 Imported by: 1

Documentation

Index

Constants

View Source 
const (
	IndexFilename           = "bucket-index.json"
	IndexCompressedFilename = IndexFilename + ".gz"
	IndexVersion1           = 1

	SegmentsFormatUnknown = ""

	// SegmentsFormat1Based6Digits defined segments numbered with 6 digits numbers in a sequence starting from number 1
	// eg. (000001, 000002, 000003).
	SegmentsFormat1Based6Digits = "1b6d"
)
View Source 
const (
	// SyncStatusFile is the known json filename for representing the most recent bucket index sync.
	SyncStatusFile = "bucket-index-sync-status.json"
	// SyncStatusFileVersion is the current supported version of bucket-index-sync-status.json file.
	SyncStatusFileVersion = 1
)
View Source 
const (
	MarkersPathname = "markers"
)

Variables

View Source 
var (
	ErrIndexNotFound  = errors.New("bucket index not found")
	ErrIndexCorrupted = errors.New("bucket index corrupted")

	UnknownStatus = Status{
		Version:            SyncStatusFileVersion,
		Status:             Unknown,
		NonQueryableReason: Unknown,
	}
)
View Source 
var (
	ErrBlockMetaNotFound          = block.ErrorSyncMetaNotFound
	ErrBlockMetaCorrupted         = block.ErrorSyncMetaCorrupted
	ErrBlockDeletionMarkNotFound  = errors.New("block deletion mark not found")
	ErrBlockDeletionMarkCorrupted = errors.New("block deletion mark corrupted")
)
View Source 
var (
	MarkersMap = map[string]func(ulid.ULID) string{
		metadata.DeletionMarkFilename:  BlockDeletionMarkFilepath,
		metadata.NoCompactMarkFilename: NoCompactMarkFilenameMarkFilepath,
	}
)

Functions

func BlockDeletionMarkFilepath 

func BlockDeletionMarkFilepath(blockID ulid.ULID) string

BlockDeletionMarkFilepath returns the path, relative to the tenant's bucket location, of a block deletion mark in the bucket markers location.

func BucketWithGlobalMarkers 

func BucketWithGlobalMarkers(b objstore.InstrumentedBucket) objstore.InstrumentedBucket

BucketWithGlobalMarkers wraps the input bucket into a bucket which also keeps track of markers in the global markers location.

func DeleteIndex added in v1.7.0 

func DeleteIndex(ctx context.Context, bkt objstore.Bucket, userID string, cfgProvider bucket.TenantConfigProvider) error

DeleteIndex deletes the bucket index from the storage. No error is returned if the index does not exist.

func DeleteIndexSyncStatus added in v1.16.0 

func DeleteIndexSyncStatus(ctx context.Context, bkt objstore.Bucket, userID string) error

DeleteIndexSyncStatus deletes the bucket index sync status file from the storage. No error is returned if the file does not exist.

func IsBlockDeletionMarkFilename 

func IsBlockDeletionMarkFilename(name string) (ulid.ULID, bool)

IsBlockDeletionMarkFilename returns whether the input filename matches the expected pattern of block deletion markers stored in the markers location.

func IsBlockNoCompactMarkFilename added in v1.13.0 

func IsBlockNoCompactMarkFilename(name string) (ulid.ULID, bool)

IsBlockNoCompactMarkFilename returns whether the input filename matches the expected pattern of block no compact markers stored in the markers location.

func MigrateBlockDeletionMarksToGlobalLocation added in v1.7.0 

func MigrateBlockDeletionMarksToGlobalLocation(ctx context.Context, bkt objstore.Bucket, userID string, cfgProvider bucket.TenantConfigProvider) error

MigrateBlockDeletionMarksToGlobalLocation list all tenant's blocks and, for each of them, look for a deletion mark in the block location. Found deletion marks are copied to the global markers location. The migration continues on error and returns once all blocks have been checked.

func NoCompactMarkFilenameMarkFilepath added in v1.13.0 

func NoCompactMarkFilenameMarkFilepath(blockID ulid.ULID) string

NoCompactMarkFilenameMarkFilepath returns the path, relative to the tenant's bucket location, of a block no compact mark in the bucket markers location.

func WriteIndex added in v1.7.0 

func WriteIndex(ctx context.Context, bkt objstore.Bucket, userID string, cfgProvider bucket.TenantConfigProvider, idx *Index) error

WriteIndex uploads the provided index to the storage.

func WriteSyncStatus added in v1.16.0 

func WriteSyncStatus(ctx context.Context, bkt objstore.Bucket, userID string, ss Status, logger log.Logger)

WriteSyncStatus upload the sync status file with the corresponding SyncStatus This file is not encrypted using the CMK configuration

Types

type Block 

type Block struct {
	// Block ID.
	ID ulid.ULID `json:"block_id"`

	// MinTime and MaxTime specify the time range all samples in the block are in (millis precision).
	MinTime int64 `json:"min_time"`
	MaxTime int64 `json:"max_time"`

	// SegmentsFormat and SegmentsNum stores the format and number of chunks segments
	// in the block, if they match a known pattern. We don't store the full segments
	// files list in order to keep the index small. SegmentsFormat is empty if segments
	// are unknown or don't match a known format.
	SegmentsFormat string `json:"segments_format,omitempty"`
	SegmentsNum    int    `json:"segments_num,omitempty"`

	// Max size in bytes of series and chunk in the block.
	SeriesMaxSize int64 `json:"series_max_size,omitempty"`
	ChunkMaxSize  int64 `json:"chunk_max_size,omitempty"`

	// UploadedAt is a unix timestamp (seconds precision) of when the block has been completed to be uploaded
	// to the storage.
	UploadedAt int64 `json:"uploaded_at"`
}

Block holds the information about a block in the index.

func BlockFromThanosMeta 

func BlockFromThanosMeta(meta metadata.Meta) *Block

func (*Block) GetUploadedAt 

func (m *Block) GetUploadedAt() time.Time

func (*Block) String 

func (m *Block) String() string

func (*Block) ThanosMeta 

func (m *Block) ThanosMeta(userID string) *metadata.Meta

ThanosMeta returns a block meta based on the known information in the index. The returned meta doesn't include all original meta.json data but only a subset of it.

func (*Block) Within added in v1.7.0 

func (m *Block) Within(minT, maxT int64) bool

Within returns whether the block contains samples within the provided range. Input minT and maxT are both inclusive.

type BlockDeletionMark 

type BlockDeletionMark struct {
	// Block ID.
	ID ulid.ULID `json:"block_id"`

	// DeletionTime is a unix timestamp (seconds precision) of when the block was marked to be deleted.
	DeletionTime int64 `json:"deletion_time"`
}

BlockDeletionMark holds the information about a block's deletion mark in the index.

func BlockDeletionMarkFromThanosMarker 

func BlockDeletionMarkFromThanosMarker(mark *metadata.DeletionMark) *BlockDeletionMark

func (*BlockDeletionMark) GetDeletionTime added in v1.7.0 

func (m *BlockDeletionMark) GetDeletionTime() time.Time

func (*BlockDeletionMark) ThanosDeletionMark added in v1.7.0 

func (m *BlockDeletionMark) ThanosDeletionMark() *metadata.DeletionMark

ThanosDeletionMark returns the Thanos deletion mark.

type BlockDeletionMarks added in v1.7.0 

type BlockDeletionMarks []*BlockDeletionMark

BlockDeletionMarks holds a set of block deletion marks in the index. No ordering guaranteed.

func (BlockDeletionMarks) Clone added in v1.7.0 

func (s BlockDeletionMarks) Clone() BlockDeletionMarks

func (BlockDeletionMarks) GetULIDs added in v1.7.0 

func (s BlockDeletionMarks) GetULIDs() []ulid.ULID

type BlockLister added in v1.17.0 

type BlockLister struct {
	// contains filtered or unexported fields
}

func NewBlockLister added in v1.17.0 

func NewBlockLister(logger log.Logger, bkt objstore.Bucket, userID string, cfgProvider bucket.TenantConfigProvider) *BlockLister

func (*BlockLister) GetActiveAndPartialBlockIDs added in v1.17.0 

func (f *BlockLister) GetActiveAndPartialBlockIDs(ctx context.Context, ch chan<- ulid.ULID) (partialBlocks map[ulid.ULID]bool, err error)

type Blocks 

type Blocks []*Block

Blocks holds a set of blocks in the index. No ordering guaranteed.

func (Blocks) GetULIDs 

func (s Blocks) GetULIDs() []ulid.ULID

func (Blocks) String 

func (s Blocks) String() string

type Index 

type Index struct {
	// Version of the index format.
	Version int `json:"version"`

	// List of complete blocks (partial blocks are excluded from the index).
	Blocks Blocks `json:"blocks"`

	// List of block deletion marks.
	BlockDeletionMarks BlockDeletionMarks `json:"block_deletion_marks"`

	// UpdatedAt is a unix timestamp (seconds precision) of when the index has been updated
	// (written in the storage) the last time.
	UpdatedAt int64 `json:"updated_at"`
}

Index contains all known blocks and markers of a tenant.

func ReadIndex 

func ReadIndex(ctx context.Context, bkt objstore.Bucket, userID string, cfgProvider bucket.TenantConfigProvider, logger log.Logger) (*Index, error)

ReadIndex reads, parses and returns a bucket index from the bucket.

func (*Index) GetUpdatedAt added in v1.7.0 

func (idx *Index) GetUpdatedAt() time.Time

func (*Index) RemoveBlock added in v1.7.0 

func (idx *Index) RemoveBlock(id ulid.ULID)

RemoveBlock removes block and its deletion mark (if any) from index.

type Loader added in v1.7.0 

type Loader struct {
	services.Service
	// contains filtered or unexported fields
}

Loader is responsible to lazy load bucket indexes and, once loaded for the first time, keep them updated in background. Loaded indexes are automatically offloaded once the idle timeout expires.

func NewLoader added in v1.7.0 

func NewLoader(cfg LoaderConfig, bucketClient objstore.Bucket, cfgProvider bucket.TenantConfigProvider, logger log.Logger, reg prometheus.Registerer) *Loader

NewLoader makes a new Loader.

func (*Loader) GetIndex added in v1.7.0 

func (l *Loader) GetIndex(ctx context.Context, userID string) (*Index, Status, error)

GetIndex returns the bucket index for the given user. It returns the in-memory cached index if available, or load it from the bucket otherwise.

type LoaderConfig added in v1.7.0 

type LoaderConfig struct {
	CheckInterval         time.Duration
	UpdateOnStaleInterval time.Duration
	UpdateOnErrorInterval time.Duration
	IdleTimeout           time.Duration
}

type Status added in v1.16.0 

type Status struct {
	// SyncTime is a unix timestamp of when the bucket index was synced
	SyncTime int64 `json:"sync_ime"`
	// Version of the file.
	Version int `json:"version"`
	// Last Sync status
	Status SyncStatus `json:"status"`
	// Should not allow query until this time
	NonQueryableUntil int64 `json:"non_queryable_until"`
	// Should not allow query until this time
	NonQueryableReason SyncStatus `json:"non_queryable_reason"`
}

func ReadSyncStatus added in v1.16.0 

func ReadSyncStatus(ctx context.Context, b objstore.Bucket, userID string, logger log.Logger) (Status, error)

ReadSyncStatus retrieves the SyncStatus from the sync status file If the file is not found, it returns `Unknown`

func (*Status) GetNonQueryableUntil added in v1.16.0 

func (s *Status) GetNonQueryableUntil() time.Time

type SyncStatus added in v1.16.0 

type SyncStatus string

SyncStatus is an enum for the possibles sync status. 

const (
	Ok                      SyncStatus = "Ok"
	GenericError            SyncStatus = "GenericError"
	CustomerManagedKeyError SyncStatus = "CustomerManagedKeyError"
	Unknown                 SyncStatus = "Unknown"
)

Possible SyncStatus.

type Updater added in v1.7.0 

type Updater struct {
	// contains filtered or unexported fields
}

Updater is responsible to generate an update in-memory bucket index.

func NewUpdater added in v1.7.0 

func NewUpdater(bkt objstore.Bucket, userID string, cfgProvider bucket.TenantConfigProvider, logger log.Logger) *Updater

func (*Updater) UpdateIndex added in v1.7.0 

func (w *Updater) UpdateIndex(ctx context.Context, old *Index) (*Index, map[ulid.ULID]error, int64, error)

UpdateIndex generates the bucket index and returns it, without storing it to the storage. If the old index is not passed in input, then the bucket index will be generated from scratch.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.