insertutils

Documentation

Index

• Variables
• func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error)
• func ParseUnixTimestamp(s string) (int64, error)
• type BenchmarkLogMessageProcessor
  • func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _ []logstorage.Field)
  • func (blp *BenchmarkLogMessageProcessor) MustClose()
• type CommonParams
  • func GetCommonParams(r *http.Request) (*CommonParams, error)
  • func GetCommonParamsForSyslog(tenantID logstorage.TenantID) *CommonParams
  • func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor
• type LogMessageProcessor
• type TestLogMessageProcessor
  • func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field)
  • func (tlp *TestLogMessageProcessor) MustClose()
  • func (tlp *TestLogMessageProcessor) Verify(rowsExpected int, timestampsExpected []int64, resultExpected string) error

Variables

var (
	// MaxLineSizeBytes is the maximum length of a single line for /insert/* handlers
	MaxLineSizeBytes = flagutil.NewBytes("insert.maxLineSizeBytes", 256*1024, "The maximum size of a single line, which can be read by /insert/* handlers")

	// MaxFieldsPerLine is the maximum number of fields per line for /insert/* handlers
	MaxFieldsPerLine = flag.Int("insert.maxFieldsPerLine", 1000, "The maximum number of log fields per line, which can be read by /insert/* handlers")
)

Functions

func ExtractTimestampRFC3339NanoFromFields

func ExtractTimestampRFC3339NanoFromFields(timeField string, fields []logstorage.Field) (int64, error)

ExtractTimestampRFC3339NanoFromFields extracts RFC3339 timestamp in nanoseconds from the field with the name timeField at fields.

The value for the timeField is set to empty string after returning from the function, so it could be ignored during data ingestion.

The current timestamp is returned if fields do not contain a field with timeField name or if the timeField value is empty.

func ParseUnixTimestamp

func ParseUnixTimestamp(s string) (int64, error)

ParseUnixTimestamp parses s as unix timestamp in either seconds or milliseconds and returns the parsed timestamp in nanoseconds.

Types

type BenchmarkLogMessageProcessor

type BenchmarkLogMessageProcessor struct{}

BenchmarkLogMessageProcessor implements LogMessageProcessor for benchmarks.

func (*BenchmarkLogMessageProcessor) AddRow

func (blp *BenchmarkLogMessageProcessor) AddRow(_ int64, _ []logstorage.Field)

AddRow implements LogMessageProcessor interface.

func (*BenchmarkLogMessageProcessor) MustClose

func (blp *BenchmarkLogMessageProcessor) MustClose()

MustClose implements LogMessageProcessor interface.

type CommonParams

type CommonParams struct {
	TenantID     logstorage.TenantID
	TimeField    string
	MsgField     string
	StreamFields []string
	IgnoreFields []string

	Debug           bool
	DebugRequestURI string
	DebugRemoteAddr string
}

CommonParams contains common HTTP parameters used by log ingestion APIs.

See https://docs.victoriametrics.com/victorialogs/data-ingestion/#http-parameters

func GetCommonParams

func GetCommonParams(r *http.Request) (*CommonParams, error)

GetCommonParams returns CommonParams from r.

func GetCommonParamsForSyslog

func GetCommonParamsForSyslog(tenantID logstorage.TenantID) *CommonParams

GetCommonParamsForSyslog returns common params needed for parsing syslog messages and storing them to the given tenantID.

func (*CommonParams) NewLogMessageProcessor

func (cp *CommonParams) NewLogMessageProcessor() LogMessageProcessor

NewLogMessageProcessor returns new LogMessageProcessor for the given cp.

MustClose() must be called on the returned LogMessageProcessor when it is no longer needed.

type LogMessageProcessor

type LogMessageProcessor interface {
	// AddRow must add row to the LogMessageProcessor with the given timestamp and the given fields.
	//
	// The LogMessageProcessor implementation cannot hold references to fields, since the caller can re-use them.
	AddRow(timestamp int64, fields []logstorage.Field)

	// MustClose() must flush all the remaining fields and free up resources occupied by LogMessageProcessor.
	MustClose()
}

LogMessageProcessor is an interface for log message processors.

type TestLogMessageProcessor

type TestLogMessageProcessor struct {
	// contains filtered or unexported fields
}

TestLogMessageProcessor implements LogMessageProcessor for testing.

func (*TestLogMessageProcessor) AddRow

func (tlp *TestLogMessageProcessor) AddRow(timestamp int64, fields []logstorage.Field)

AddRow adds row with the given timestamp and fields to tlp

func (*TestLogMessageProcessor) MustClose

func (tlp *TestLogMessageProcessor) MustClose()

MustClose closes tlp.

func (*TestLogMessageProcessor) Verify

func (tlp *TestLogMessageProcessor) Verify(rowsExpected int, timestampsExpected []int64, resultExpected string) error

Verify verifies the number of rows, timestamps and results after AddRow calls.