config

package
v0.0.0-...-551a65d Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 11, 2021 License: Apache-2.0 Imports: 16 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ChanSize          = 100
	NumberOfPipelines = 4
)

Pipeline constraints

View Source 
const (
	TCPType           = "tcp"
	UDPType           = "udp"
	FileType          = "file"
	DockerType        = "docker"
	JournaldType      = "journald"
	WindowsEventType  = "windows_event"
	SnmpTrapsType     = "snmp_traps"
	StringChannelType = "string_channel"

	// UTF16BE for UTF-16 Big endian encoding
	UTF16BE string = "utf-16-be"
	// UTF16LE for UTF-16 Little Endian encoding
	UTF16LE string = "utf-16-le"
)

Logs source types

View Source 
const (
	ForceBeginning = iota
	ForceEnd
	Beginning
	End
)

Tailing Modes

View Source 
const (
	ExcludeAtMatch = "exclude_at_match"
	IncludeAtMatch = "include_at_match"
	MaskSequences  = "mask_sequences"
	MultiLine      = "multi_line"
)

Processing rule types

View Source 
const ContainerCollectAll = "container_collect_all"

ContainerCollectAll is the name of the docker integration that collect logs from all containers

View Source 
const (
	// DateFormat is the default date format.
	DateFormat = "2006-01-02T15:04:05.000000000Z"
)
View Source 
const SnmpTraps = "snmp_traps"

SnmpTraps is the name of the integration that collects logs from SNMP traps received by the Agent

Variables

This section is empty.

Functions

func AggregationTimeout added in v0.9.0 

func AggregationTimeout() time.Duration

AggregationTimeout is used when performing aggregation operations

func CompileProcessingRules 

func CompileProcessingRules(rules []*ProcessingRule) error

CompileProcessingRules compiles all processing rule regular expressions.

func ContainsWildcard 

func ContainsWildcard(path string) bool

ContainsWildcard returns true if the path contains any wildcard character

func ExpectedTagsDuration added in v0.9.0 

func ExpectedTagsDuration() time.Duration

ExpectedTagsDuration returns a duration of the time expected tags will be submitted for.

func IsExpectedTagsSet added in v0.9.0 

func IsExpectedTagsSet() bool

IsExpectedTagsSet returns boolean showing if expected tags feature is enabled.

func TaggerWarmupDuration 

func TaggerWarmupDuration() time.Duration

TaggerWarmupDuration is used to configure the tag providers

func ValidateProcessingRules 

func ValidateProcessingRules(rules []*ProcessingRule) error

ValidateProcessingRules validates the rules and raises an error if one is misconfigured. Each processing rule must have: - a valid name - a valid type - a valid pattern that compiles

Types

type ChannelMessage added in v0.9.0 

type ChannelMessage struct {
	Content []byte
	// Optional. Must be UTC. If not provided, time.Now().UTC() will be used
	// Used in the Serverless Agent
	Timestamp time.Time
	// Optional.
	// Used in the Serverless Agent
	Lambda *Lambda
}

ChannelMessage represents a log line sent to datadog, with its metadata

func NewChannelMessageFromLambda added in v0.9.0 

func NewChannelMessageFromLambda(content []byte, utcTime time.Time, ARN, reqID, functionName string) *ChannelMessage

NewChannelMessageFromLambda construts a message with content and with the given timestamp and Lambda metadata

type CountInfo added in v0.9.0 

type CountInfo struct {
	// contains filtered or unexported fields
}

CountInfo records a simple count

func NewCountInfo added in v0.9.0 

func NewCountInfo(key string) *CountInfo

NewCountInfo creates a new CountInfo instance

func (*CountInfo) Add added in v0.9.0 

func (c *CountInfo) Add(v int32)

Add a new value to the count

func (*CountInfo) Info added in v0.9.0 

func (c *CountInfo) Info() []string

Info returns the info

func (*CountInfo) InfoKey added in v0.9.0 

func (c *CountInfo) InfoKey() string

InfoKey returns the key

type Endpoint 

type Endpoint struct {
	APIKey                  string `mapstructure:"api_key" json:"api_key"`
	Host                    string
	Port                    int
	UseSSL                  bool
	UseCompression          bool `mapstructure:"use_compression" json:"use_compression"`
	CompressionLevel        int  `mapstructure:"compression_level" json:"compression_level"`
	ProxyAddress            string
	ConnectionResetInterval time.Duration

	BackoffFactor    float64
	BackoffBase      float64
	BackoffMax       float64
	RecoveryInterval int
	RecoveryReset    bool
}

Endpoint holds all the organization and network parameters to send logs to Datadog.

type Endpoints 

type Endpoints struct {
	Main                   Endpoint
	Additionals            []Endpoint
	UseProto               bool
	UseHTTP                bool
	BatchWait              time.Duration
	BatchMaxConcurrentSend int
	BatchMaxSize           int
	BatchMaxContentSize    int
}

Endpoints holds the main endpoint and additional ones to dualship logs.

func BuildEndpoints 

func BuildEndpoints(httpConnectivity HTTPConnectivity) (*Endpoints, error)

BuildEndpoints returns the endpoints to send logs.

func BuildEndpointsWithConfig added in v0.9.0 

func BuildEndpointsWithConfig(logsConfig *LogsConfigKeys, endpointPrefix string, httpConnectivity HTTPConnectivity) (*Endpoints, error)

BuildEndpointsWithConfig returns the endpoints to send logs.

func BuildHTTPEndpoints 

func BuildHTTPEndpoints() (*Endpoints, error)

BuildHTTPEndpoints returns the HTTP endpoints to send logs to.

func BuildHTTPEndpointsWithConfig added in v0.9.0 

func BuildHTTPEndpointsWithConfig(logsConfig *LogsConfigKeys, endpointPrefix string) (*Endpoints, error)

BuildHTTPEndpointsWithConfig uses two arguments that instructs it how to access configuration parameters, then returns the HTTP endpoints to send logs to. This function is able to default to the 'classic' BuildHTTPEndpoints() w ldHTTPEndpointsWithConfigdefault variables logsConfigDefaultKeys and httpEndpointPrefix

func BuildServerlessEndpoints added in v0.9.0 

func BuildServerlessEndpoints() (*Endpoints, error)

BuildServerlessEndpoints returns the endpoints to send logs for the Serverless agent.

func NewEndpoints 

func NewEndpoints(main Endpoint, additionals []Endpoint, useProto bool, useHTTP bool) *Endpoints

NewEndpoints returns a new endpoints composite with default batching settings

func NewEndpointsWithBatchSettings added in v0.9.0 

func NewEndpointsWithBatchSettings(main Endpoint, additionals []Endpoint, useProto bool, useHTTP bool, batchWait time.Duration, batchMaxConcurrentSend int, batchMaxSize int, batchMaxContentSize int) *Endpoints

NewEndpointsWithBatchSettings returns a new endpoints composite with non-default batching settings specified

type HTTPConnectivity 

type HTTPConnectivity bool

HTTPConnectivity is the status of the HTTP connectivity 

var (
	// HTTPConnectivitySuccess is the status for successful HTTP connectivity
	HTTPConnectivitySuccess HTTPConnectivity = true
	// HTTPConnectivityFailure is the status for failed HTTP connectivity
	HTTPConnectivityFailure HTTPConnectivity = false
)

type InfoProvider added in v0.9.0 

type InfoProvider interface {
	InfoKey() string
	Info() []string
}

InfoProvider is a general interface to provide info about a log source. It is used in the agent status page. The expected usage is for a piece of code that wants to surface something on the status page register an info provider with the source with a unique key/name. This file contains useful base implementations, but InfoProvider can be extended/implemented for more complex data.

When implementing InfoProvider - be aware of the 2 ways it is used by the status page:

  1. when a single message is returned, the statuspage will display a single line: InfoKey(): Info()[0]
  1. when multiple messages are returned, the status page will display an indented list: InfoKey(): Info()[0] Info()[1] Info()[n]

InfoKey only needs to be unique per source, and should be human readable.

type Lambda added in v0.9.0 

type Lambda struct {
	ARN          string
	RequestID    string
	FunctionName string
}

Lambda is a struct storing information about the Lambda function and function execution.

type LogSource 

type LogSource struct {
	// Put expvar Int first because it's modified with sync/atomic, so it needs to
	// be 64-bit aligned on 32-bit systems. See https://golang.org/pkg/sync/atomic/#pkg-note-BUG
	BytesRead expvar.Int

	Name   string
	Config *LogsConfig
	Status *LogStatus

	Messages *Messages

	// In the case that the source is overridden, keep a reference to the parent for bubbling up information about the child
	ParentSource *LogSource
	// LatencyStats tracks internal stats on the time spent by messages from this source in a processing pipeline, i.e.
	// the duration between when a message is decoded by the tailer/listener/decoder and when the message is handled by a sender
	LatencyStats *util.StatsTracker
	// contains filtered or unexported fields
}

LogSource holds a reference to an integration name and a log configuration, and allows to track errors and successful operations on it. Both name and configuration are static for now and determined at creation time. Changing the status is designed to be thread safe.

func ContainerCollectAllSource added in v0.9.0 

func ContainerCollectAllSource() *LogSource

ContainerCollectAllSource returns a source to collect all logs from all containers.

func NewLogSource 

func NewLogSource(name string, config *LogsConfig) *LogSource

NewLogSource creates a new log source.

func SNMPTrapsSource added in v0.9.0 

func SNMPTrapsSource() *LogSource

SNMPTrapsSource returs a source to forward SNMP traps as logs.

func (*LogSource) AddInput 

func (s *LogSource) AddInput(input string)

AddInput registers an input as being handled by this source.

func (*LogSource) GetInfo added in v0.9.0 

func (s *LogSource) GetInfo(key string) InfoProvider

GetInfo gets an InfoProvider instance by the key

func (*LogSource) GetInfoStatus added in v0.9.0 

func (s *LogSource) GetInfoStatus() map[string][]string

GetInfoStatus returns a primitive representation of the info for the status page

func (*LogSource) GetInputs 

func (s *LogSource) GetInputs() []string

GetInputs returns the inputs handled by this source.

func (*LogSource) GetSourceType 

func (s *LogSource) GetSourceType() SourceType

GetSourceType returns the sourceType used by this source

func (*LogSource) RegisterInfo added in v0.9.0 

func (s *LogSource) RegisterInfo(i InfoProvider)

RegisterInfo registers some info to display on the status page

func (*LogSource) RemoveInput 

func (s *LogSource) RemoveInput(input string)

RemoveInput removes an input from this source.

func (*LogSource) SetSourceType 

func (s *LogSource) SetSourceType(sourceType SourceType)

SetSourceType sets a format that give information on how the source lines should be parsed

type LogSources 

type LogSources struct {
	// contains filtered or unexported fields
}

LogSources stores a list of log sources.

func CreateSources 

func CreateSources(sourcesArray []*LogSource) *LogSources

CreateSources creates sources

func NewLogSources 

func NewLogSources() *LogSources

NewLogSources creates a new log sources.

func (*LogSources) AddSource 

func (s *LogSources) AddSource(source *LogSource)

AddSource adds a new source.

func (*LogSources) GetAddedForType 

func (s *LogSources) GetAddedForType(sourceType string) chan *LogSource

GetAddedForType returns the new added sources matching the provided type.

func (*LogSources) GetRemovedForType 

func (s *LogSources) GetRemovedForType(sourceType string) chan *LogSource

GetRemovedForType returns the new removed sources matching the provided type.

func (*LogSources) GetSources 

func (s *LogSources) GetSources() []*LogSource

GetSources returns all the sources currently held.

func (*LogSources) RemoveSource 

func (s *LogSources) RemoveSource(source *LogSource)

RemoveSource removes a source.

type LogStatus 

type LogStatus struct {
	// contains filtered or unexported fields
}

LogStatus tracks errors and success.

func NewLogStatus 

func NewLogStatus() *LogStatus

NewLogStatus creates a new log status.

func (*LogStatus) Error 

func (s *LogStatus) Error(err error)

Error records the given error and invalidates the source.

func (*LogStatus) GetError 

func (s *LogStatus) GetError() string

GetError returns the error.

func (*LogStatus) IsError 

func (s *LogStatus) IsError() bool

IsError returns whether the current status is an error.

func (*LogStatus) IsPending 

func (s *LogStatus) IsPending() bool

IsPending returns whether the current status is not yet determined.

func (*LogStatus) IsSuccess 

func (s *LogStatus) IsSuccess() bool

IsSuccess returns whether the current status is a success.

func (*LogStatus) Success 

func (s *LogStatus) Success()

Success sets the status to success.

type LogsConfig 

type LogsConfig struct {
	Type string

	Port int    // Network
	Path string // File, Journald

	Encoding     string   `mapstructure:"encoding" json:"encoding"`             // File
	ExcludePaths []string `mapstructure:"exclude_paths" json:"exclude_paths"`   // File
	TailingMode  string   `mapstructure:"start_position" json:"start_position"` // File

	IncludeUnits  []string `mapstructure:"include_units" json:"include_units"`   // Journald
	ExcludeUnits  []string `mapstructure:"exclude_units" json:"exclude_units"`   // Journald
	ContainerMode bool     `mapstructure:"container_mode" json:"container_mode"` // Journald

	Image string // Docker
	Label string // Docker
	// Name contains the container name
	Name string // Docker
	// Identifier contains the container ID
	Identifier string // Docker

	ChannelPath string `mapstructure:"channel_path" json:"channel_path"` // Windows Event
	Query       string // Windows Event

	// used as input only by the Channel tailer.
	// could have been unidirectional but the tailer could not close it in this case.
	Channel chan *ChannelMessage

	Service         string
	Source          string
	SourceCategory  string
	Tags            []string
	ProcessingRules []*ProcessingRule `mapstructure:"log_processing_rules" json:"log_processing_rules"`
}

LogsConfig represents a log source config, which can be for instance a file to tail or a port to listen to.

func ParseJSON 

func ParseJSON(data []byte) ([]*LogsConfig, error)

ParseJSON parses the data formatted in JSON returns an error if the parsing failed.

func ParseYAML 

func ParseYAML(data []byte) ([]*LogsConfig, error)

ParseYAML parses the data formatted in YAML, returns an error if the parsing failed.

func (*LogsConfig) Validate 

func (c *LogsConfig) Validate() error

Validate returns an error if the config is misconfigured

type LogsConfigKeys added in v0.9.0 

type LogsConfigKeys struct {
	// contains filtered or unexported fields
}

LogsConfigKeys stores logs configuration keys stored in YAML configuration files

func NewLogsConfigKeys added in v0.9.0 

func NewLogsConfigKeys(configPrefix string, config coreConfig.Config) *LogsConfigKeys

NewLogsConfigKeys returns a new logs configuration keys set

type MappedInfo added in v0.9.0 

type MappedInfo struct {
	// contains filtered or unexported fields
}

MappedInfo collects multiple info messages with a unique key

func NewMappedInfo added in v0.9.0 

func NewMappedInfo(key string) *MappedInfo

NewMappedInfo creates a new MappedInfo instance

func (*MappedInfo) Info added in v0.9.0 

func (m *MappedInfo) Info() []string

Info returns the info

func (*MappedInfo) InfoKey added in v0.9.0 

func (m *MappedInfo) InfoKey() string

InfoKey returns the key

func (*MappedInfo) RemoveMessage added in v0.9.0 

func (m *MappedInfo) RemoveMessage(key string)

RemoveMessage removes a message with a unique key

func (*MappedInfo) SetMessage added in v0.9.0 

func (m *MappedInfo) SetMessage(key string, message string)

SetMessage sets a message with a unique key

type Messages 

type Messages struct {
	// contains filtered or unexported fields
}

Messages holds messages and warning that can be displayed in the status Warnings are display at the top of the log section in the status and messages are displayed in the log source that generated the message

func NewMessages 

func NewMessages() *Messages

NewMessages initialize Messages with the default values

func (*Messages) AddMessage 

func (m *Messages) AddMessage(key string, message string)

AddMessage create a message

func (*Messages) GetMessages 

func (m *Messages) GetMessages() []string

GetMessages returns all the messages

func (*Messages) RemoveMessage 

func (m *Messages) RemoveMessage(key string)

RemoveMessage removes a message

type ProcessingRule 

type ProcessingRule struct {
	Type               string
	Name               string
	ReplacePlaceholder string `mapstructure:"replace_placeholder" json:"replace_placeholder"`
	Pattern            string
	// TODO: should be moved out
	Regex       *regexp.Regexp
	Placeholder []byte
}

ProcessingRule defines an exclusion or a masking rule to be applied on log lines

func GlobalProcessingRules 

func GlobalProcessingRules() ([]*ProcessingRule, error)

GlobalProcessingRules returns the global processing rules to apply to all logs.

type SourceType 

type SourceType string

SourceType used for log line parsing logic. TODO: remove this logic. 

const (
	// DockerSourceType docker source type
	DockerSourceType SourceType = "docker"
	// KubernetesSourceType kubernetes source type
	KubernetesSourceType SourceType = "kubernetes"
)

type TailingMode 

type TailingMode uint8

TailingMode type

func TailingModeFromString 

func TailingModeFromString(mode string) (TailingMode, bool)

TailingModeFromString parses a string and returns a corresponding tailing mode, default to End if not found

func (TailingMode) String 

func (mode TailingMode) String() string

TailingModeToString returns seelog string representation for a specified tailing mode. Returns "" for invalid tailing mode.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.