logs

Documentation

Index

• Constants
• Variables
• func AnalyzeBackendLogLines(logLines []state.LogLine) (logLinesOut []state.LogLine, samples []state.PostgresQuerySample)
• func AnalyzeLogLines(logLinesIn []state.LogLine) (logLinesOut []state.LogLine, samples []state.PostgresQuerySample)
• func ParseAndAnalyzeBuffer(logStream LineReader, linesNewerThan time.Time, server *state.Server) ([]state.LogLine, []state.PostgresQuerySample)
• func PrintDebugInfo(logLines []state.LogLine, samples []state.PostgresQuerySample)
• func PrintDebugLogLines(logLines []state.LogLine, ...)
• func ReplaceSecrets(logLines []state.LogLine, filterLogSecret []state.LogSecretKind)
• func SyncLogParser(server *state.Server, settings []state.PostgresSetting)
• func ValidateLogCollectionConfig(server *state.Server, settings []state.PostgresSetting) (bool, bool, bool, string)
• type LineReader
• type LogParser
  • func NewLogParser(prefix string, tz *time.Location, isSyslog bool) *LogParser
  • func (lp *LogParser) GetOccurredAt(timePart string) time.Time
  • func (lp *LogParser) GetPrefixAndContent(line string) (prefix string, content string, ok bool)
  • func (lp *LogParser) Matches(prefix string, tz *time.Location, isSyslog bool) bool
  • func (lp *LogParser) ParseLine(line string) (logLine state.LogLine, ok bool)
  • func (lp *LogParser) ValidatePrefix() error
• type MaybeHerokuLogReader
  • func NewMaybeHerokuLogReader(r io.Reader) *MaybeHerokuLogReader
  • func (lr *MaybeHerokuLogReader) ReadString(delim byte) (string, error)
• type PrefixEscape

Constants

const LogPrefixAmazonRds string = "%t:%r:%u@%d:[%p]:"

const LogPrefixAzure string = "%t-%c-"

const LogPrefixCustom1 string = "%m [%p][%v] : [%l-1] %q[app=%a] "

const LogPrefixCustom10 string = "%m [%p]: [%l-1] db=%d,user=%u "

const LogPrefixCustom11 string = "pid=%p,user=%u,db=%d,app=%a,client=%h "

const LogPrefixCustom12 string = "user=%u,db=%d,app=%a,client=%h "

const LogPrefixCustom13 string = "%p-%s-%c-%l-%h-%u-%d-%m "

const LogPrefixCustom14 string = "%m [%p][%b][%v][%x] %q[user=%u,db=%d,app=%a] "

const LogPrefixCustom15 string = "%m [%p] %q%u@%d "

const LogPrefixCustom16 string = "%t [%p] %q%u@%d %h "

const LogPrefixCustom2 string = "%t [%p-%l] %q%u@%d "

const LogPrefixCustom3 string = "%m [%p] %q[user=%u,db=%d,app=%a] "

const LogPrefixCustom4 string = "%m [%p] %q[user=%u,db=%d,app=%a,host=%h] "

const LogPrefixCustom5 string = "%t [%p]: [%l-1] user=%u,db=%d - PG-%e "

const LogPrefixCustom6 string = "%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h "

const LogPrefixCustom7 string = "%t [%p]: [%l-1] [trx_id=%x] user=%u,db=%d "

const LogPrefixCustom8 string = "[%p]: [%l-1] db=%d,user=%u "

const LogPrefixCustom9 string = "%m %r %u %a [%c] [%p] "

const LogPrefixEmpty string = ""

const LogPrefixHeroku1 string = " sql_error_code = %e "

const LogPrefixHeroku2 string = `` /* 186-byte string literal not displayed */

const LogPrefixHerokuHobbyTier string = " database = %d connection_source = %r sql_error_code = %e "

Used only to recognize the Heroku hobby tier log_line_prefix to give a warning (logs are not supported on hobby tier) and avoid errors during prefix check; logs with this prefix are never actually received

const LogPrefixRecommended = LogPrefixCustom3

const LogPrefixSimple string = "%m [%p] "

const MinSupportedLogMinDurationStatement = 10

Variables

var AppInsideBracketsRegexp = `(\[unknown\]|[^,\]]*)` // %a

var DbRegexp = `(\S*)` // %d

var EscapeMatchers = map[rune]PrefixEscape{

	'a': {
		Regexp: `.+?`,
		ApplyValue: func(value string, logLine *state.LogLine, parser *LogParser) {
			if value == "[unknown]" {
				return
			}
			logLine.Application = value
		},
		Optional: true,
	},

	'u': {
		Regexp: `.+?`,
		ApplyValue: func(value string, logLine *state.LogLine, parser *LogParser) {
			if value == "[unknown]" {
				return
			}
			logLine.Username = value
		},
		Optional: true,
	},

	'd': {
		Regexp: `.+?`,
		ApplyValue: func(value string, logLine *state.LogLine, parser *LogParser) {
			if value == "[unknown]" {
				return
			}
			logLine.Database = value
		},
		Optional: true,
	},

	'r': {
		Regexp:   `[a-zA-Z0-9:.-]+\(\d{1,5}\)|\[local\]`,
		Optional: true,
	},

	'h': {
		Regexp:   `[a-zA-Z0-9:.-]+|\[local\]`,
		Optional: true,
	},

	'b': {
		Regexp: `[a-z ]+`,
	},

	'p': {
		Regexp: `\d+`,
		ApplyValue: func(value string, logLine *state.LogLine, parser *LogParser) {
			intVal, _ := strconv.ParseInt(value, 10, 32)
			logLine.BackendPid = int32(intVal)
		},
	},

	'P': {
		Regexp:   `\d+`,
		Optional: true,
	},

	't': {
		Regexp: `\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} (?:[A-Z]{1,4}|[+-]\d+)`,
		ApplyValue: func(value string, logLine *state.LogLine, parser *LogParser) {
			logLine.OccurredAt = parser.GetOccurredAt(value)
		},
	},

	'm': {
		Regexp: `\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} (?:[A-Z]{1,4}|[+-]\d+)`,
		ApplyValue: func(value string, logLine *state.LogLine, parser *LogParser) {
			logLine.OccurredAt = parser.GetOccurredAt(value)
		},
	},

	'n': {
		Regexp: `\d+\.\d+`,
		ApplyValue: func(value string, logLine *state.LogLine, parser *LogParser) {
			tsparts := strings.SplitN(value, ".", 2)
			seconds, _ := strconv.ParseInt(tsparts[0], 10, 64)
			millis, _ := strconv.ParseInt(tsparts[1], 10, 64)
			logLine.OccurredAt = time.Unix(seconds, millis*1_000_000)
		},
	},

	'i': {
		Regexp:   `[A-Z_ ]+`,
		Optional: true,
	},

	'e': {
		Regexp: `[0-9A-Z]{5}`,
	},

	'c': {
		Regexp:   `[0-9a-f]{1,8}\.[0-9a-f]{1,8}`,
		Optional: true,
	},

	'l': {
		Regexp: `\d+`,
		ApplyValue: func(value string, logLine *state.LogLine, parser *LogParser) {
			intVal, _ := strconv.ParseInt(value, 10, 32)
			logLine.LogLineNumber = int32(intVal)
		},
	},

	's': {
		Regexp: `\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} (?:[A-Z]{1,4}|[+-]\d+)`,
	},

	'v': {
		Regexp:   `\d+\/\d+`,
		Optional: true,
	},

	'x': {
		Regexp:   `\d+`,
		Optional: true,
	},

	'Q': {
		Regexp: `-?\d+`,
	},
}

This is a map of the various log_line_prefix format strings; see https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-LOG-LINE-PREFIX not included: %q and %%, which are easier to handle by special-casing

var HerokuPostgresDebugRegexp = regexp.MustCompile(`^(\w+ \d+ \d+:\d+:\d+ \w+ app\[postgres\] \w+ )?\[(\w+)\] \[\d+-\d+\] (.+)`)

var LevelAndContentRegexp = `(\w+):\s+(.*\n?)$`

var LogPrefixNoTimestampUserDatabaseAppRegexp = regexp.MustCompile(`(?s)^\[user=` + UserRegexp + `,db=` + DbRegexp + `,app=` + AppInsideBracketsRegexp + `\] ` + LevelAndContentRegexp)

var PidRegexp = `(\d+)` // %p

var RsyslogHostnameRegxp = `(\S+)`

var RsyslogLevelAndContentRegexp = `(?:(\w+):\s+)?(.*\n?)$`

var RsyslogProcessNameRegexp = `(\w+)`

var RsyslogRegexp = regexp.MustCompile(`^` + RsyslogTimeRegexp + ` ` + RsyslogHostnameRegxp + ` ` + RsyslogProcessNameRegexp + `\[` + PidRegexp + `\]: ` + SyslogSequenceAndSplitRegexp + ` ` + RsyslogLevelAndContentRegexp)

var RsyslogTimeRegexp = `(\w+\s+\d+ \d{2}:\d{2}:\d{2})`

var SyslogSequenceAndSplitRegexp = `(\[[\d-]+\])?`

var UserRegexp = `(\S*)` // %u

Functions

func AnalyzeBackendLogLines

func AnalyzeBackendLogLines(logLines []state.LogLine) (logLinesOut []state.LogLine, samples []state.PostgresQuerySample)

func AnalyzeLogLines

func AnalyzeLogLines(logLinesIn []state.LogLine) (logLinesOut []state.LogLine, samples []state.PostgresQuerySample)

func ParseAndAnalyzeBuffer

func ParseAndAnalyzeBuffer(logStream LineReader, linesNewerThan time.Time, server *state.Server) ([]state.LogLine, []state.PostgresQuerySample)

func PrintDebugInfo

func PrintDebugInfo(logLines []state.LogLine, samples []state.PostgresQuerySample)

func PrintDebugLogLines

func PrintDebugLogLines(logLines []state.LogLine, classifications map[pganalyze_collector.LogLineInformation_LogClassification]bool)

func ReplaceSecrets

func ReplaceSecrets(logLines []state.LogLine, filterLogSecret []state.LogSecretKind)

func SyncLogParser

func SyncLogParser(server *state.Server, settings []state.PostgresSetting)

func ValidateLogCollectionConfig

func ValidateLogCollectionConfig(server *state.Server, settings []state.PostgresSetting) (bool, bool, bool, string)

Types

type LineReader

type LineReader interface {
	ReadString(delim byte) (string, error)
}

type LogParser

type LogParser struct {
	// contains filtered or unexported fields
}

func NewLogParser

func NewLogParser(prefix string, tz *time.Location, isSyslog bool) *LogParser

func (*LogParser) GetOccurredAt

func (lp *LogParser) GetOccurredAt(timePart string) time.Time

func (*LogParser) GetPrefixAndContent

func (lp *LogParser) GetPrefixAndContent(line string) (prefix string, content string, ok bool)

func (*LogParser) Matches

func (lp *LogParser) Matches(prefix string, tz *time.Location, isSyslog bool) bool

func (*LogParser) ParseLine

func (lp *LogParser) ParseLine(line string) (logLine state.LogLine, ok bool)

func (*LogParser) ValidatePrefix

func (lp *LogParser) ValidatePrefix() error

type MaybeHerokuLogReader

type MaybeHerokuLogReader struct {
	LineReader
}

func NewMaybeHerokuLogReader

func NewMaybeHerokuLogReader(r io.Reader) *MaybeHerokuLogReader

func (*MaybeHerokuLogReader) ReadString

func (lr *MaybeHerokuLogReader) ReadString(delim byte) (string, error)

type PrefixEscape

type PrefixEscape struct {
	Regexp     string
	ApplyValue func(value string, logLine *state.LogLine, parser *LogParser)
	// Indicates a value may not always be present for this escape (e.g., when logging from a non-backend process)
	Optional bool
}