ingestor

Documentation

Index

• type Service
  • func NewService(opts ServiceOpts) (*Service, error)
  • func (s *Service) Close() error
  • func (s *Service) DisableWrites() error
  • func (s *Service) HandleLogs(w http.ResponseWriter, r *http.Request)
  • func (s *Service) HandleReceive(w http.ResponseWriter, r *http.Request)
  • func (s *Service) HandleTransfer(w http.ResponseWriter, r *http.Request)
  • func (s *Service) Open(ctx context.Context) error
  • func (s *Service) UploadSegments() error
• type ServiceOpts

Types

type Service

type Service struct {
	// contains filtered or unexported fields
}

func NewService

func NewService(opts ServiceOpts) (*Service, error)

func (*Service) Close

func (s *Service) Close() error

func (*Service) DisableWrites

func (s *Service) DisableWrites() error

func (*Service) HandleLogs

func (s *Service) HandleLogs(w http.ResponseWriter, r *http.Request)

HandleLogs handles OTLP logs requests and writes them to the store.

func (*Service) HandleReceive

func (s *Service) HandleReceive(w http.ResponseWriter, r *http.Request)

HandleReceive handles the prometheus remote write requests and writes them to the store.

func (*Service) HandleTransfer

func (s *Service) HandleTransfer(w http.ResponseWriter, r *http.Request)

HandleTransfer handles the transfer WAL segments from other nodes in the cluster.

func (*Service) Open

func (s *Service) Open(ctx context.Context) error

func (*Service) UploadSegments

func (s *Service) UploadSegments() error

type ServiceOpts

type ServiceOpts struct {
	StorageDir     string
	Uploader       adx.Uploader
	MaxSegmentSize int64
	MaxSegmentAge  time.Duration

	// StaticColumns is a slice of column=value elements where each element will be added all rows.
	StaticColumns []string

	// LiftedColumns is a slice of label names where each element will be added as a column if the label exists.
	LiftedColumns []string

	// DropLabels is a map of metric names regexes to label name regexes.  When both match, the label will be dropped.
	DropLabels map[*regexp.Regexp]*regexp.Regexp

	// DropMetrics is a slice of regexes that drops metrics when the metric name matches.  The metric name format
	// should match the Prometheus naming style before the metric is translated to a Kusto table name.
	DropMetrics []*regexp.Regexp

	K8sCli kubernetes.Interface

	// MetricsKustoCli is the Kusto client connected to the metrics kusto cluster.
	MetricsKustoCli []metrics.StatementExecutor

	// LogsKustoCli is the Kusto client connected to the logs kusto cluster.
	LogsKustoCli []metrics.StatementExecutor

	// InsecureSkipVerify disables TLS certificate verification.
	InsecureSkipVerify bool

	// Namespace is the namespace used for peer discovery.
	Namespace string

	// Hostname is the hostname of the current node.
	Hostname string

	// Region is a location identifier
	Region string

	// DisablePeerTransfer disables peer discovery and prevents transfers of small segments to an owner.
	// Each instance of ingestor will upload received segments directly.
	DisablePeerTransfer bool

	// MaxTransferSize is the minimum size of a segment that will be transferred to another node.  If a segment
	// exceeds this size, it will be uploaded directly by the current node.
	MaxTransferSize int64

	// MaxTransferAge is the maximum age of a segment that will be transferred to another node.  If a segment
	// exceeds this age, it will be uploaded directly by the current node.
	MaxTransferAge time.Duration

	// MaxSegmentCount is the maximum number of segments files allowed on disk before signaling back-pressure.
	MaxSegmentCount int64

	// MaxDiskUsage is the maximum disk usage allowed before signaling back-pressure.
	MaxDiskUsage int64

	// AllowedDatabases is the distinct set of database names that are allowed to be written to.
	AllowedDatabase []string

	// MetricsDatabase is the name of the metrics database.
	MetricsDatabases []string

	// LogsDatabases is a slice of log database names.
	LogsDatabases []string

	// PartitionSize is the max size of the group of nodes forming a partition.  A partition is a set of nodes where
	// keys are distributed.
	PartitionSize int

	// FunctionStore is the store of functions to sync with the Kusto cluster.
	FunctionStore adx.FunctionStore
}