ingester

Documentation

Index

• Constants
• func NewConsumerFactory(metastoreClient MetadataStore, storage ObjectStorage, ...) partition.ConsumerFactory
• type Config
  • func (cfg *Config) RegisterFlags(f *flag.FlagSet)
  • func (cfg *Config) Validate() error
• type Ingester
  • func New(cfg Config, consumerFactory partition.ConsumerFactory, logger log.Logger, ...) (*Ingester, error)
  • func (i *Ingester) CheckReady(ctx context.Context) error
  • func (i *Ingester) Flush()
  • func (i *Ingester) PreparePartitionDownscaleHandler(w http.ResponseWriter, r *http.Request)
  • func (i *Ingester) ServeHTTP(w http.ResponseWriter, r *http.Request)
  • func (i *Ingester) TransferOut(_ context.Context) error
  • func (*Ingester) Watch(*grpc_health_v1.HealthCheckRequest, grpc_health_v1.Health_WatchServer) error
• type Interface
• type MetadataStore
• type ObjectStorage
• type Wrapper

Constants

const (
	RingName          = "kafka-ingester"
	PartitionRingName = "kafka-partition"
)

Functions

func NewConsumerFactory

func NewConsumerFactory(
	metastoreClient MetadataStore,
	storage ObjectStorage,
	flushInterval time.Duration,
	maxFlushSize int64,
	logger log.Logger,
	reg prometheus.Registerer,
) partition.ConsumerFactory

NewConsumerFactory creates and initializes a new consumer instance

Types

type Config

type Config struct {
	Enabled             bool                  `yaml:"enabled" doc:"description=Whether the kafka ingester is enabled."`
	LifecyclerConfig    ring.LifecyclerConfig `` /* 145-byte string literal not displayed */
	ShutdownMarkerPath  string                `yaml:"shutdown_marker_path"`
	FlushInterval       time.Duration         `` /* 167-byte string literal not displayed */
	FlushSize           int64                 `` /* 155-byte string literal not displayed */
	PartitionRingConfig partitionring.Config  `yaml:"partition_ring" category:"experimental"`
	KafkaConfig         kafka.Config          `yaml:"-"`
}

Config for an ingester.

func (*Config) RegisterFlags

func (cfg *Config) RegisterFlags(f *flag.FlagSet)

RegisterFlags registers the flags.

func (*Config) Validate

func (cfg *Config) Validate() error

type Ingester

type Ingester struct {
	services.Service
	// contains filtered or unexported fields
}

Ingester builds chunks for incoming log streams.

func New

func New(cfg Config,
	consumerFactory partition.ConsumerFactory,
	logger log.Logger,
	metricsNamespace string,
	registerer prometheus.Registerer,
) (*Ingester, error)

New makes a new Ingester.

func (*Ingester) CheckReady

func (i *Ingester) CheckReady(ctx context.Context) error

ReadinessHandler is used to indicate to k8s when the ingesters are ready for the addition removal of another ingester. Returns 204 when the ingester is ready, 500 otherwise.

func (*Ingester) Flush

func (i *Ingester) Flush()

Flush implements ring.FlushTransferer Flush triggers a flush of all the chunks and closes the flush queues. Called from the Lifecycler as part of the ingester shutdown.

func (*Ingester) PreparePartitionDownscaleHandler

func (i *Ingester) PreparePartitionDownscaleHandler(w http.ResponseWriter, r *http.Request)

func (*Ingester) ServeHTTP

func (i *Ingester) ServeHTTP(w http.ResponseWriter, r *http.Request)

ServeHTTP implements the pattern ring status page.

func (*Ingester) TransferOut

func (i *Ingester) TransferOut(_ context.Context) error

func (*Ingester) Watch

func (*Ingester) Watch(*grpc_health_v1.HealthCheckRequest, grpc_health_v1.Health_WatchServer) error

Watch implements grpc_health_v1.HealthCheck.

type Interface

type Interface interface {
	services.Service
	http.Handler
	CheckReady(ctx context.Context) error
	FlushHandler(w http.ResponseWriter, _ *http.Request)
}

Interface is an interface for the Ingester

type MetadataStore

type MetadataStore interface {
	AddBlock(ctx context.Context, in *metastorepb.AddBlockRequest, opts ...grpc.CallOption) (*metastorepb.AddBlockResponse, error)
}

MetadataStore defines an interface for metadata storage operations

type ObjectStorage

type ObjectStorage interface {
	PutObject(ctx context.Context, objectKey string, object io.Reader) error
}

ObjectStorage defines an interface for object storage operations

type Wrapper

type Wrapper interface {
	Wrap(wrapped Interface) Interface
}