logging

README

hellofresh/logging-go

Generic logging configuration library written in Go

This is generic logging configuration library that we at HelloFresh use in our projects to write applications logs to different log collecting solutions.

Key Features

• Uses logrus as logging library
• Allows applying logging configuration from config file or environment variables, uses viper under the hood
• The following hooks/writers are available:
  • stderr
  • stdout
  • discard aka /dev/null
  • logstash
  • syslog
  • graylog

Installation

go get -u github.com/hellofresh/logging-go

Usage

Standalone logging configuring

YAML config file example

level: info
format: logstash
formatSettings:
  type: MyService
  ts: RFC3339Nano
writer: stderr
hooks:
- format: logstash
  settings: {network: udp, host: logstash.mycompany.io, port: 8911, type: MyService, ts: RFC3339Nano}
- format: syslog
  settings: {network: udp, host: localhost, port: 514, tag: MyService, facility: LOG_LOCAL0, severity: LOG_INFO}
- format: graylog
  settings: {host: graylog.mycompany.io, port: 9000}
- format: stackdriver
  settings: {service: myservice, version: v1}

Environment variable config example

export LOG_LEVEL="info"
export LOG_FORMAT="logstash"
export LOG_FORMAT_SETTINGS="type=MyService,ts:RFC3339Nano"
export LOG_WRITER="stderr"
export LOG_HOOKS='[{"format":"logstash", "settings":{"type":"MyService","ts":"RFC3339Nano", "network": "udp","host":"logstash.mycompany.io","port": "8911"}},{"format":"syslog","settings":{"network": "udp", "host":"localhost", "port": "514", "tag": "MyService", "facility": "LOG_LOCAL0", "severity": "LOG_INFO"}},{"format":"graylog","settings":{"host":"graylog.mycompany.io","port":"9000"}},{"format":"stackdriver", "settings":{"service":"myservice","version":"v1"}}]'

Loading and applying configuration

package main

import (
        "github.com/hellofresh/logging-go"
        log "github.com/sirupsen/logrus"
        "github.com/spf13/viper"
)

func main() {
      logging.InitDefaults(viper.GetViper(), "")
      logConfig, err := logging.Load(viper.GetViper(), "/path/to/config.yml")
      if nil != err {
            panic(err)
      }

      err = logConfig.Apply()
      if nil != err {
            panic(err)
      }
      defer logConfig.Flush()

      log.Info("Logger successfully initialised!")
}

Logging as part of the global config

YAML config file example

foo: rule
bar: 34
log:
    level: info
    format: json
    output: stderr
    hooks:
    - format: logstash
      settings: {network: udp, host: logstash.mycompany.io, port: 8911, type: MyService, ts: RFC3339Nano}
    - format: syslog
      settings: {network: udp, host: localhost, port: 514, tag: MyService, facility: LOG_LOCAL0, severity: LOG_INFO}
    - format: graylog
      settings: {host: graylog.mycompany.io, port: 9000}
    - format: stackdriver
      settings: {service: myservice, version: v1}

Environment variable config example

export APP_FOO="rule"
export APP_BAR="34"
export LOG_LEVEL="info"
export LOG_FORMAT="json"
export LOG_WRITER="stderr"
export LOG_HOOKS='[{"format":"logstash", "settings":{"type":"MyService","ts":"RFC3339Nano", "network": "udp","host":"logstash.mycompany.io","port": "8911"}},{"format":"syslog","settings":{"network": "udp", "host":"localhost", "port": "514", "tag": "MyService", "facility": "LOG_LOCAL0", "severity": "LOG_INFO"}},{"format":"graylog","settings":{"host":"graylog.mycompany.io","port":"9000"}},{"format":"stackdriver", "settings":{"service":"myservice","version":"v1"}}]'

Loading and applying configuration

package main

import (
        "github.com/hellofresh/logging-go"
        log "github.com/sirupsen/logrus"
        "github.com/spf13/viper"
)

func init() {
        viper.SetDefault("foo", "foo")
        viper.SetDefault("bar", 42)
        logging.InitDefaults(viper.GetViper(), "log")
}

type AppConfig struct {
        Foo string `envconfig:"APP_FOO"`
        Bar int    `envconfig:"APP_BAR"`

        Log logging.LogConfig
}

func LoadAppConfig() (*AppConfig, error) {
        var instance AppConfig
        
        ...
        
        return &instance, nil
}

func main() {
        appConfig, err := LoadAppConfig()
        if nil != err {
                panic(err)
        }

        err = appConfig.Log.Apply()
        if nil != err {
                panic(err)
        }
        defer appConfig.Log.Flush()

        log.Info("Application successfully initialised!")
}

Track requests metrics with middleware

package main

import (
        "net/http"
        
        "github.com/go-chi/chi"
        "github.com/hellofresh/logging-go/middleware"
)

func main() {
        appConfig, err := LoadAppConfig()
        if nil != err {
                panic(err)
        }

        err = appConfig.Log.Apply()
        if nil != err {
                panic(err)
        }
        defer appConfig.Log.Flush()

        r := chi.NewRouter()
        r.Use(middleware.New())

        r.GET("/", func(c *gin.Context) {
                c.JSON(http.StatusOK, "I'm producing logs!")
        })

        http.ListenAndServe(":8080", r)
}

Configuration

Base logger

• level (env LOG_LEVEL, default: info): panic, fatal, error, warn (warning), info, debug (see logrus.ParseLevel())
• format (env LOG_FORMAT, default: json):
  • text - plain text
  • json - all fields encoded into JSON string
  • logstash - same as json but includes additional logstash fields (e.g. @version) and format settings (see bellow)
• formatSettings (env LOG_FORMAT_SETTINGS):
  • type (used only for logstash format) - any valid string field that will be added to all log entries
  • ts (used only for logstash format) - timestamp field format, the following values are available: RFC3339, RFC3339Nano (become time.RFC3339 and time.RFC3339Nano time package constants)
• writer (env LOG_WRITER, default: stderr): stderr, stdout, discard
• hooks (env LOG_HOOKS) - each hook has te following fields: format and settings. Currently te following formats are available:

logstash

Uses github.com/bshuster-repo/logrus-logstash-hook implementation

Setting	Required	Description
host	YES	Logstash host name or IP address
port	YES	Logstash host port
network	YES	udp or tcp
type	no	same as formatSettings.type
ts	no	same as formatSettings.type

syslog

Not supported on Windows. Uses logstash implementation of log/syslog

Setting	Required	Description
host	YES	Syslog host name or IP address
port	YES	Syslog host port
network	YES	udp or tcp
severity	YES	severity part of syslog priority (LOG_INFO, LOG_NOTICE, etc.)
facility	YES	facility part of syslog priority (LOG_LOCAL0, LOG_CRON, etc.)
tag	no	any valid string that will be sent to syslog as tag

graylog

Uses github.com/gemnasium/logrus-graylog-hook implementation

Setting	Required	Description
host	YES	Graylog host name or IP address
port	YES	Graylog host port
async	no	send log messages to Graylog in synchronous or asynchronous mode, string value must be parsable to bool

Stackdriver

Stackdriver formatter for Google Cloud Container Engine(GKE/Kubernetes).

Uses github.com/TV4/logrus-stackdriver-formatter implementation

Setting	Required	Description
service	no	Optional Service Name referring to this logger
version	no	Optional Service version

Contributing

To start contributing, please check CONTRIBUTING.

Documentation

• hellofresh/logging-go Docs: https://godoc.org/github.com/hellofresh/logging-go
• Go lang: https://golang.org/

Documentation

Index

• Constants
• Variables
• func InitDefaults(v *viper.Viper, prefix string)
• type LogConfig
  • func Load(v *viper.Viper, configPath string) (LogConfig, error)
  • func LoadConfigFromEnv(v *viper.Viper) (LogConfig, error)
  • func (c LogConfig) Apply() error
  • func (c LogConfig) Flush()
• type LogFormat
• type LogHook
• type LogHooks
  • func (lh *LogHooks) UnmarshalText(text []byte) error
• type LogWriter

Constants

const (
	// Text is plain text log format
	Text LogFormat = "text"
	// JSON is json log format
	JSON LogFormat = "json"
	// Logstash is json log format with some additional fields required for logstash
	Logstash LogFormat = "logstash"
	// Stackdriver is Google cloud FluentD Stackdriver format
	Stackdriver LogFormat = "stackdriver"

	// StdErr is os stderr log writer
	StdErr LogWriter = "stderr"
	// StdOut is os stdout log writer
	StdOut LogWriter = "stdout"
	// Discard is the quite mode for log writer aka /dev/null
	Discard LogWriter = "discard"

	// HookLogstash is logstash hook format
	HookLogstash = "logstash"
	// HookSyslog is syslog hook format
	HookSyslog = "syslog"
	// HookGraylog is graylog hook format
	HookGraylog = "graylog"
)

Variables

var (
	// ErrUnknownLogHookFormat is the error returned when trying to initialise hook of unknown format
	ErrUnknownLogHookFormat = errors.New("Failed to init log hooks: unknown hook found")
	// ErrMissingLogHookSetting is the error returned when trying to initialise hook with required settings missing
	ErrMissingLogHookSetting = errors.New("Failed to init log hooks: missing required hook setting")
	// ErrFailedToConfigureLogHook is the error returned when hook configuring failed for some reasons
	ErrFailedToConfigureLogHook = errors.New("Failed to init log hooks: failed to configure hook")
)

Functions

func InitDefaults

func InitDefaults(v *viper.Viper, prefix string)

InitDefaults initialises default logger settings

Types

type LogConfig

type LogConfig struct {
	Level          string            `envconfig:"LOG_LEVEL"`
	Format         LogFormat         `envconfig:"LOG_FORMAT"`
	FormatSettings map[string]string `envconfig:"LOG_FORMAT_SETTINGS"`
	Writer         LogWriter         `envconfig:"LOG_WRITER"`
	Hooks          LogHooks          `envconfig:"LOG_HOOKS"`
	// contains filtered or unexported fields
}

LogConfig is the struct that stores all the logging configuration and routines for applying configurations to logger

func Load

func Load(v *viper.Viper, configPath string) (LogConfig, error)

Load loads config values from file, fallback to load from environment variables if file is not found or failed to read

func LoadConfigFromEnv

func LoadConfigFromEnv(v *viper.Viper) (LogConfig, error)

LoadConfigFromEnv loads config values from environment variables

func (LogConfig) Apply

func (c LogConfig) Apply() error

Apply configures logger and all enabled hooks

func (LogConfig) Flush

func (c LogConfig) Flush()

Flush waits for all buffering loggers to finish flushing buffers

type LogFormat

type LogFormat string

LogFormat type for enumerating available log formats

type LogHook

type LogHook struct {
	Format   string
	Settings map[string]string
}

LogHook is a struct holding settings for each enabled hook

type LogHooks

type LogHooks []LogHook

LogHooks is collection of enabled hooks

func (*LogHooks) UnmarshalText

func (lh *LogHooks) UnmarshalText(text []byte) error

UnmarshalText is an implementation of encoding.TextUnmarshaler for LogHooks type

type LogWriter

type LogWriter string

LogWriter for enumerating available log writers