metastore

Documentation

Index

• type CompactionCommandHandler
  • func NewCompactionCommandHandler(logger log.Logger, index IndexReplacer, compactor compaction.Compactor, ...) *CompactionCommandHandler
  • func (h *CompactionCommandHandler) GetCompactionPlanUpdate(tx *bbolt.Tx, cmd *raft.Log, req *raft_log.GetCompactionPlanUpdateRequest) (*raft_log.GetCompactionPlanUpdateResponse, error)
  • func (h *CompactionCommandHandler) UpdateCompactionPlan(tx *bbolt.Tx, cmd *raft.Log, req *raft_log.UpdateCompactionPlanRequest) (*raft_log.UpdateCompactionPlanResponse, error)
• type CompactionService
  • func NewCompactionService(logger log.Logger, raft Raft) *CompactionService
  • func (svc *CompactionService) PollCompactionJobs(_ context.Context, req *metastorev1.PollCompactionJobsRequest) (*metastorev1.PollCompactionJobsResponse, error)
• type Config
  • func (cfg *Config) RegisterFlags(f *flag.FlagSet)
  • func (cfg *Config) Validate() error
• type Index
• type IndexBlockFinder
• type IndexCommandHandler
  • func NewIndexCommandHandler(logger log.Logger, index Index, tombstones Tombstones, ...) *IndexCommandHandler
  • func (m *IndexCommandHandler) AddBlock(tx *bbolt.Tx, cmd *raft.Log, req *metastorev1.AddBlockRequest) (*metastorev1.AddBlockResponse, error)
• type IndexQuerier
• type IndexReplacer
• type IndexService
  • func NewIndexService(logger log.Logger, raft Raft, state State, index IndexBlockFinder, ...) *IndexService
  • func (svc *IndexService) AddBlock(ctx context.Context, req *metastorev1.AddBlockRequest) (resp *metastorev1.AddBlockResponse, err error)
  • func (svc *IndexService) AddRecoveredBlock(ctx context.Context, req *metastorev1.AddBlockRequest) (*metastorev1.AddBlockResponse, error)
  • func (svc *IndexService) GetBlockMetadata(ctx context.Context, req *metastorev1.GetBlockMetadataRequest) (resp *metastorev1.GetBlockMetadataResponse, err error)
• type MetadataQueryService
  • func NewMetadataQueryService(logger log.Logger, state State, index IndexQuerier) *MetadataQueryService
  • func (svc *MetadataQueryService) QueryMetadata(ctx context.Context, req *metastorev1.QueryMetadataRequest) (resp *metastorev1.QueryMetadataResponse, err error)
• type Metastore
  • func New(config Config, logger log.Logger, reg prometheus.Registerer, ...) (*Metastore, error)
  • func (m *Metastore) CheckReady(ctx context.Context) error
  • func (m *Metastore) Register(server *grpc.Server)
  • func (m *Metastore) Service() services.Service
• type PlacementStats
• type Raft
• type State
• type TenantIndex
• type TenantService
  • func NewTenantService(logger log.Logger, state State, index TenantIndex) *TenantService
  • func (svc *TenantService) DeleteTenant(context.Context, *metastorev1.DeleteTenantRequest) (*metastorev1.DeleteTenantResponse, error)
  • func (svc *TenantService) GetTenant(ctx context.Context, req *metastorev1.GetTenantRequest) (resp *metastorev1.GetTenantResponse, err error)
• type TombstoneDeleter
• type Tombstones

Types

type CompactionCommandHandler

type CompactionCommandHandler struct {
	// contains filtered or unexported fields
}

func NewCompactionCommandHandler

func NewCompactionCommandHandler(
	logger log.Logger,
	index IndexReplacer,
	compactor compaction.Compactor,
	planner compaction.Planner,
	scheduler compaction.Scheduler,
	tombstones TombstoneDeleter,
) *CompactionCommandHandler

func (*CompactionCommandHandler) GetCompactionPlanUpdate

func (h *CompactionCommandHandler) GetCompactionPlanUpdate(
	tx *bbolt.Tx, cmd *raft.Log, req *raft_log.GetCompactionPlanUpdateRequest,
) (*raft_log.GetCompactionPlanUpdateResponse, error)

func (*CompactionCommandHandler) UpdateCompactionPlan

func (h *CompactionCommandHandler) UpdateCompactionPlan(
	tx *bbolt.Tx, cmd *raft.Log, req *raft_log.UpdateCompactionPlanRequest,
) (*raft_log.UpdateCompactionPlanResponse, error)

type CompactionService

type CompactionService struct {
	metastorev1.CompactionServiceServer
	// contains filtered or unexported fields
}

func NewCompactionService

func NewCompactionService(
	logger log.Logger,
	raft Raft,
) *CompactionService

func (*CompactionService) PollCompactionJobs

func (svc *CompactionService) PollCompactionJobs(
	_ context.Context,
	req *metastorev1.PollCompactionJobsRequest,
) (*metastorev1.PollCompactionJobsResponse, error)

type Config

type Config struct {
	Address          string             `yaml:"address"`
	GRPCClientConfig grpcclient.Config  `yaml:"grpc_client_config" doc:"description=Configures the gRPC client used to communicate with the metastore."`
	DataDir          string             `yaml:"data_dir"`
	MinReadyDuration time.Duration      `yaml:"min_ready_duration" category:"advanced"`
	Raft             raft.Config        `yaml:"raft"`
	Index            index.Config       `yaml:",inline" category:"advanced"`
	DLQRecovery      dlq.RecoveryConfig `yaml:",inline" category:"advanced"`
	Compactor        compactor.Config   `yaml:",inline" category:"advanced"`
	Scheduler        scheduler.Config   `yaml:",inline" category:"advanced"`
}

func (*Config) RegisterFlags

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

func (*Config) Validate

func (cfg *Config) Validate() error

type Index

type Index interface {
	InsertBlock(*bbolt.Tx, *metastorev1.BlockMeta) error
}

type IndexBlockFinder

type IndexBlockFinder interface {
	FindBlocks(*bbolt.Tx, *metastorev1.BlockList) ([]*metastorev1.BlockMeta, error)
}

type IndexCommandHandler

type IndexCommandHandler struct {
	// contains filtered or unexported fields
}

func NewIndexCommandHandler

func NewIndexCommandHandler(
	logger log.Logger,
	index Index,
	tombstones Tombstones,
	compactor compaction.Compactor,
) *IndexCommandHandler

func (*IndexCommandHandler) AddBlock

func (m *IndexCommandHandler) AddBlock(tx *bbolt.Tx, cmd *raft.Log, req *metastorev1.AddBlockRequest) (*metastorev1.AddBlockResponse, error)

type IndexQuerier

type IndexQuerier interface {
	QueryMetadata(*bbolt.Tx, index.MetadataQuery) iter.Iterator[*metastorev1.BlockMeta]
}

type IndexReplacer

type IndexReplacer interface {
	ReplaceBlocks(*bbolt.Tx, *metastorev1.CompactedBlocks) error
}

type IndexService

type IndexService struct {
	metastorev1.IndexServiceServer
	// contains filtered or unexported fields
}

func NewIndexService

func NewIndexService(
	logger log.Logger,
	raft Raft,
	state State,
	index IndexBlockFinder,
	stats PlacementStats,
) *IndexService

func (*IndexService) AddBlock

func (svc *IndexService) AddBlock(
	ctx context.Context,
	req *metastorev1.AddBlockRequest,
) (resp *metastorev1.AddBlockResponse, err error)

func (*IndexService) AddRecoveredBlock

func (svc *IndexService) AddRecoveredBlock(
	ctx context.Context,
	req *metastorev1.AddBlockRequest,
) (*metastorev1.AddBlockResponse, error)

func (*IndexService) GetBlockMetadata

func (svc *IndexService) GetBlockMetadata(
	ctx context.Context,
	req *metastorev1.GetBlockMetadataRequest,
) (resp *metastorev1.GetBlockMetadataResponse, err error)

type MetadataQueryService

type MetadataQueryService struct {
	metastorev1.MetadataQueryServiceServer
	// contains filtered or unexported fields
}

func NewMetadataQueryService

func NewMetadataQueryService(
	logger log.Logger,
	state State,
	index IndexQuerier,
) *MetadataQueryService

func (*MetadataQueryService) QueryMetadata

func (svc *MetadataQueryService) QueryMetadata(
	ctx context.Context,
	req *metastorev1.QueryMetadataRequest,
) (resp *metastorev1.QueryMetadataResponse, err error)

type Metastore

type Metastore struct {
	// contains filtered or unexported fields
}

func New

func New(
	config Config,
	logger log.Logger,
	reg prometheus.Registerer,
	healthService health.Service,
	client raftnodepb.RaftNodeServiceClient,
	bucket objstore.Bucket,
	placementMgr *placement.Manager,
) (*Metastore, error)

func (*Metastore) CheckReady

func (m *Metastore) CheckReady(ctx context.Context) error

CheckReady verifies if the metastore is ready to serve requests by ensuring the node is up-to-date with the leader's commit index.

func (*Metastore) Register

func (m *Metastore) Register(server *grpc.Server)

func (*Metastore) Service

func (m *Metastore) Service() services.Service

type PlacementStats

type PlacementStats interface {
	RecordStats(iter.Iterator[placement.Sample])
}

type Raft

type Raft interface {
	Propose(fsm.RaftLogEntryType, proto.Message) (proto.Message, error)
}

Raft represents a Raft consensus protocol interface. Any modifications to the state should be proposed through the Raft interface.

type State

type State interface {
	ConsistentRead(context.Context, func(*bbolt.Tx, raftnode.ReadIndex)) error
}

State represents a consistent read-only view of the metastore. The write interface is provided through the FSM raft command handlers.

type TenantIndex

type TenantIndex interface {
	GetTenantStats(tenant string) *metastorev1.TenantStats
}

type TenantService

type TenantService struct {
	metastorev1.TenantServiceServer
	// contains filtered or unexported fields
}

func NewTenantService

func NewTenantService(
	logger log.Logger,
	state State,
	index TenantIndex,
) *TenantService

func (*TenantService) DeleteTenant

func (svc *TenantService) DeleteTenant(
	context.Context,
	*metastorev1.DeleteTenantRequest,
) (*metastorev1.DeleteTenantResponse, error)

func (*TenantService) GetTenant

func (svc *TenantService) GetTenant(
	ctx context.Context,
	req *metastorev1.GetTenantRequest,
) (resp *metastorev1.GetTenantResponse, err error)

type TombstoneDeleter

type TombstoneDeleter interface {
	DeleteTombstones(*bbolt.Tx, *raft.Log, ...*metastorev1.Tombstones) error
	AddTombstones(*bbolt.Tx, *raft.Log, *metastorev1.Tombstones) error
}

type Tombstones

type Tombstones interface {
	Exists(tenant string, shard uint32, block string) bool
}