shipper

Documentation

Index

• func NewBoltDBShipperTableClient(objectClient client.ObjectClient, storageKeyPrefix string) index.TableClient
• func NewShipper(cfg Config, storageClient client.ObjectClient, limits downloads.Limits, ...) (index.Client, error)
• type Config
  • func (cfg *Config) RegisterFlags(f *flag.FlagSet)
  • func (cfg *Config) Validate() error
• type GatewayClient
  • func NewGatewayClient(cfg IndexGatewayClientConfig, r prometheus.Registerer, logger log.Logger) (*GatewayClient, error)
  • func (s *GatewayClient) BatchWrite(ctx context.Context, batch index.WriteBatch) error
  • func (s *GatewayClient) GetChunkRef(ctx context.Context, in *indexgatewaypb.GetChunkRefRequest, ...) (*indexgatewaypb.GetChunkRefResponse, error)
  • func (s *GatewayClient) LabelNamesForMetricName(ctx context.Context, in *indexgatewaypb.LabelNamesForMetricNameRequest, ...) (*indexgatewaypb.LabelResponse, error)
  • func (s *GatewayClient) LabelValuesForMetricName(ctx context.Context, in *indexgatewaypb.LabelValuesForMetricNameRequest, ...) (*indexgatewaypb.LabelResponse, error)
  • func (s *GatewayClient) NewWriteBatch() index.WriteBatch
  • func (s *GatewayClient) QueryPages(ctx context.Context, queries []index.Query, callback index.QueryPagesCallback) error
  • func (s *GatewayClient) Stop()
• type IndexGatewayClientConfig
  • func (i *IndexGatewayClientConfig) RegisterFlags(f *flag.FlagSet)
  • func (i *IndexGatewayClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)
• type IndexGatewayGRPCPool
  • func NewIndexGatewayGRPCPool(address string, opts []grpc.DialOption) (*IndexGatewayGRPCPool, error)
• type Mode
  • func (m Mode) String() string
• type Shipper
  • func (s *Shipper) BatchWrite(ctx context.Context, batch index.WriteBatch) error
  • func (s *Shipper) NewWriteBatch() index.WriteBatch
  • func (s *Shipper) QueryPages(ctx context.Context, queries []index.Query, callback index.QueryPagesCallback) error
  • func (s *Shipper) Stop()

Functions

func NewBoltDBShipperTableClient

func NewBoltDBShipperTableClient(objectClient client.ObjectClient, storageKeyPrefix string) index.TableClient

func NewShipper

func NewShipper(cfg Config, storageClient client.ObjectClient, limits downloads.Limits, registerer prometheus.Registerer) (index.Client, error)

NewShipper creates a shipper for syncing local objects with a store

Types

type Config

type Config struct {
	ActiveIndexDirectory     string                   `yaml:"active_index_directory"`
	SharedStoreType          string                   `yaml:"shared_store"`
	SharedStoreKeyPrefix     string                   `yaml:"shared_store_key_prefix"`
	CacheLocation            string                   `yaml:"cache_location"`
	CacheTTL                 time.Duration            `yaml:"cache_ttl"`
	ResyncInterval           time.Duration            `yaml:"resync_interval"`
	QueryReadyNumDays        int                      `yaml:"query_ready_num_days"`
	IndexGatewayClientConfig IndexGatewayClientConfig `yaml:"index_gateway_client"`
	BuildPerTenantIndex      bool                     `yaml:"build_per_tenant_index"`
	IngesterName             string                   `yaml:"-"`
	Mode                     Mode                     `yaml:"-"`
	IngesterDBRetainPeriod   time.Duration            `yaml:"-"`
}

func (*Config) RegisterFlags

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers flags.

func (*Config) Validate

func (cfg *Config) Validate() error

type GatewayClient

type GatewayClient struct {
	// contains filtered or unexported fields
}

func NewGatewayClient

func NewGatewayClient(cfg IndexGatewayClientConfig, r prometheus.Registerer, logger log.Logger) (*GatewayClient, error)

NewGatewayClient instantiates a new client used to communicate with an Index Gateway instance.

If it is configured to be in ring mode, a pool of GRPC connections to all Index Gateway instances is created. Otherwise, it creates a single GRPC connection to an Index Gateway instance running in simple mode.

func (*GatewayClient) BatchWrite

func (s *GatewayClient) BatchWrite(ctx context.Context, batch index.WriteBatch) error

func (*GatewayClient) GetChunkRef

func (s *GatewayClient) GetChunkRef(ctx context.Context, in *indexgatewaypb.GetChunkRefRequest, opts ...grpc.CallOption) (*indexgatewaypb.GetChunkRefResponse, error)

func (*GatewayClient) LabelNamesForMetricName

func (s *GatewayClient) LabelNamesForMetricName(ctx context.Context, in *indexgatewaypb.LabelNamesForMetricNameRequest, opts ...grpc.CallOption) (*indexgatewaypb.LabelResponse, error)

func (*GatewayClient) LabelValuesForMetricName

func (s *GatewayClient) LabelValuesForMetricName(ctx context.Context, in *indexgatewaypb.LabelValuesForMetricNameRequest, opts ...grpc.CallOption) (*indexgatewaypb.LabelResponse, error)

func (*GatewayClient) NewWriteBatch

func (s *GatewayClient) NewWriteBatch() index.WriteBatch

func (*GatewayClient) QueryPages

func (s *GatewayClient) QueryPages(ctx context.Context, queries []index.Query, callback index.QueryPagesCallback) error

func (*GatewayClient) Stop

func (s *GatewayClient) Stop()

Stop stops the execution of this gateway client.

If it is in simple mode, the single GRPC connection is closed. Otherwise, nothing happens.

type IndexGatewayClientConfig

type IndexGatewayClientConfig struct {
	// Mode sets in which mode the client will operate. It is actually defined at the
	// index_gateway YAML section and reused here.
	Mode indexgateway.Mode `yaml:"-"`

	// PoolConfig defines the behavior of the gRPC connection pool used to communicate
	// with the Index Gateway.
	//
	// Only relevant for the ring mode.
	// It is defined at the distributors YAML section and reused here.
	PoolConfig clientpool.PoolConfig `yaml:"-"`

	// Ring is the Index Gateway ring used to find the appropriate Index Gateway instance
	// this client should talk to.
	//
	// Only relevant for the ring mode.
	Ring ring.ReadRing `yaml:"-"`

	// GRPCClientConfig configures the gRPC connection between the Index Gateway client and the server.
	//
	// Used by both, ring and simple mode.
	GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config"`

	// Address of the Index Gateway instance responsible for retaining the index for all tenants.
	//
	// Only relevant for the simple mode.
	Address string `yaml:"server_address,omitempty"`

	// Forcefully disable the use of the index gateway client for the storage.
	// This is mainly useful for the index-gateway component which should always use the storage.
	Disabled bool `yaml:"-"`
}

IndexGatewayClientConfig configures the Index Gateway client used to communicate with the Index Gateway server.

func (*IndexGatewayClientConfig) RegisterFlags

func (i *IndexGatewayClientConfig) RegisterFlags(f *flag.FlagSet)

func (*IndexGatewayClientConfig) RegisterFlagsWithPrefix

func (i *IndexGatewayClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)

RegisterFlagsWithPrefix register client-specific flags with the given prefix.

Flags that are used by both, client and server, are defined in the indexgateway package.

type IndexGatewayGRPCPool

type IndexGatewayGRPCPool struct {
	grpc_health_v1.HealthClient
	indexgatewaypb.IndexGatewayClient
	io.Closer
}

IndexGatewayGRPCPool represents a pool of gRPC connections to different index gateway instances.

Only used when Index Gateway is configured to run in ring mode.

func NewIndexGatewayGRPCPool

func NewIndexGatewayGRPCPool(address string, opts []grpc.DialOption) (*IndexGatewayGRPCPool, error)

NewIndexGatewayGRPCPool instantiates a new pool of IndexGateway GRPC connections.

Internally, it also instantiates a protobuf index gateway client and a health client.

type Mode

type Mode int

const (
	// ModeReadWrite is to allow both read and write
	ModeReadWrite Mode = iota
	// ModeReadOnly is to allow only read operations
	ModeReadOnly
	// ModeWriteOnly is to allow only write operations
	ModeWriteOnly

	// FilesystemObjectStoreType holds the periodic config type for the filesystem store
	FilesystemObjectStoreType = "filesystem"

	// UploadInterval defines interval for when we check if there are new index files to upload.
	UploadInterval = 1 * time.Minute
)

func (Mode) String

func (m Mode) String() string

type Shipper

type Shipper struct {
	// contains filtered or unexported fields
}

func (*Shipper) BatchWrite

func (s *Shipper) BatchWrite(ctx context.Context, batch index.WriteBatch) error

func (*Shipper) NewWriteBatch

func (s *Shipper) NewWriteBatch() index.WriteBatch

func (*Shipper) QueryPages

func (s *Shipper) QueryPages(ctx context.Context, queries []index.Query, callback index.QueryPagesCallback) error

func (*Shipper) Stop

func (s *Shipper) Stop()