streamer

README

Gitlab Log Streamer

Gitlab Log Streamer is a tool designed to overcome the limitations of Gitlab's audit_log.json and auth_log.json.

By default, Gitlab writes its audit events to the audit_log.json file, which limits their usefulness as they stay in your GItLab server filesystem.

This project parses the log files, stores the events in a SQLite database, and allows forwarding of new log entries using syslog format (RFC5424) or IBM QRadar's proprietary LEEF. It also supports defining an HTTP endpoint for POST requests with the event, enabling triggers and actions similar to Gitlab System hooks.

Table of Contents

• Gitlab Log Streamer
  • Table of Contents
  • Installation
  • Usage

Installation

Just head to https://github.com/juanfont/gitlab-log-streamer/releases and grab the latest version.

Then place it in your PATH (e.g., /usr/local/bin)

Usage

You need to create a file named config.yaml in the same directory as the binary or in /etc/gitlab-log-streamer:

---
db_path: "streamer.sqlite"
gitlab_hostname: "gitlab.font.eu"

sources:
  audit_log_path: "/var/log/gitlab/gitlab-rails/audit_json.log"
  auth_log_path: "/var/log/gitlab/gitlab-rails/auth_json.log"


destinations:
  http:
    url: "http://localhost:8080"
    headers:
      Authorization: "Bearer 1234567890"
      Content-Type: "application/json"
  syslog:
    server_addr: "localhost:1489"
    protocol: "udp"

    # Optional. If true, the syslog message will be in LEEF format. Otherwise, syslog in RFC5424 format.
    use_leef: false

And then just execute:

gitlab-log-streamer watch

Documentation

Index

• Constants
• type Add
• type As
• type AuditEvent
• type AuthEvent
• type AuthorClass
• type Change
• type Config
• type GitLabLogStreamer
  • func NewGitLabLogStreamer(config Config) (*GitLabLogStreamer, error)
  • func (s *GitLabLogStreamer) Watch() error
• type MergeAccessLevelElement
• type Severity
• type With

Constants

const (
	GitlabVersionManifestPath = "/opt/gitlab/version-manifest.txt"
	PreloadEventsPeriodDays   = 30

	CleanAuthLogEventsPeriod = 60 * time.Hour * 24 // time to keep the auth events in the DB
)

const (
	AuditEventLoginWithTwoFactor = "two-factor"
	AuditEventLoginWithU2F       = "two-factor-via-u2f-device"
	AuditEventLoginWithWebAuthn  = "two-factor-via-webauthn-device"
	AuditEventLoginStandard      = "standard"
)

Types

type Add

type Add string

const (
	CiGroupVariable Add = "ci_group_variable"
	Email           Add = "email"
	Group           Add = "group"
	Project         Add = "project"
	User            Add = "user"
	UserAccess      Add = "user_access"
)

type As

type As string

const (
	Developer As = "Developer"
	Guest     As = "Guest"
	Owner     As = "Owner"
)

type AuditEvent

type AuditEvent struct {
	ID            uint64 `gorm:"primary_key" json:"-"`
	CorrelationID string `gorm:"type:varchar(64)" json:"correlation_id"`

	Severity      string      `json:"severity"`
	Time          time.Time   `json:"time"`
	AuthorID      int64       `json:"author_id"`
	AuthorName    string      `json:"author_name"`
	EntityID      int64       `json:"entity_id"`
	EntityType    AuthorClass `json:"entity_type"`
	IPAddress     string      `json:"ip_address"`
	With          *With       `json:"with,omitempty"`
	TargetID      *int64      `json:"target_id"`
	TargetType    AuthorClass `json:"target_type"`
	TargetDetails *string     `json:"target_details"`
	EntityPath    string      `json:"entity_path"`
	Remove        *Add        `json:"remove,omitempty"`
	Add           *Add        `json:"add,omitempty"`
	// Details       *Details    `json:"details,omitempty"`
	// PushAccessLevels          []MergeAccessLevelElement `json:"push_access_levels,omitempty"`
	// MergeAccessLevels         []MergeAccessLevelElement `json:"merge_access_levels,omitempty"`
	AllowForcePush            *bool        `json:"allow_force_push,omitempty"`
	CodeOwnerApprovalRequired *bool        `json:"code_owner_approval_required,omitempty"`
	AuthorClass               *AuthorClass `json:"author_class,omitempty"`
	CustomMessage             *string      `json:"custom_message,omitempty"`
	As                        *As          `json:"as,omitempty"`
	MemberID                  *int64       `json:"member_id,omitempty"`
	Change                    *Change      `json:"change,omitempty"`
	From                      *string      `json:"from,omitempty"`
	To                        *string      `json:"to,omitempty"`
	Action                    *string      `json:"action,omitempty"`
	ExpiryFrom                *string      `json:"expiry_from"`
	ExpiryTo                  *string      `json:"expiry_to"`

	MetaCallerID        string `json:"meta.caller_id"`
	MetaRemoteIP        string `json:"meta.remote_ip"`
	MetaFeatureCategory string `json:"meta.feature_category"`
	MetaClientID        string `json:"meta.client_id"`
	MetaUser            string `json:"meta.user,omitempty"`
	MetaUserID          int    `json:"meta.user_id,omitempty"`

	OriginalData datatypes.JSON
}

type AuthEvent

type AuthEvent struct {
	ID            uint64    `gorm:"primary_key" json:"-"`
	CorrelationID string    `gorm:"type:varchar(64)" json:"correlation_id"`
	Severity      string    `json:"severity"`
	Time          time.Time `json:"time"`

	Message              *string `json:"message,omitempty"`
	Env                  *string `json:"env,omitempty"`
	RemoteIP             *string `json:"remote_ip,omitempty"`
	RequestMethod        *string `json:"request_method,omitempty"`
	Path                 *string `json:"path,omitempty"`
	MetaCallerID         *string `json:"meta.caller_id,omitempty"`
	MetaRemoteIP         *string `json:"meta.remote_ip,omitempty"`
	MetaFeatureCategory  *string `json:"meta.feature_category,omitempty"`
	MetaClientID         *string `json:"meta.client_id,omitempty"`
	ScopeType            *string `json:"scope_type,omitempty"`
	RequestedProjectPath *string `json:"requested_project_path,omitempty"`
	// RequestedActions     []string `json:"requested_actions,omitempty"`
	// AuthorizedActions    []string `json:"authorized_actions,omitempty"`
	HTTPUser            *string `json:"http_user,omitempty"`
	AuthService         *string `json:"auth_service,omitempty"`
	AuthResultType      *string `json:"auth_result.type"`
	AuthResultActorType *string `json:"auth_result.actor_type"`
	PayloadType         *string `json:"payload_type,omitempty"`

	OriginalData datatypes.JSON
}

type AuthorClass

type AuthorClass string

const (
	AuthorClassCiGroupVariable AuthorClass = "Ci::GroupVariable"
	AuthorClassEmail           AuthorClass = "Email"
	AuthorClassGroup           AuthorClass = "Group"
	AuthorClassProject         AuthorClass = "Project"
	AuthorClassUser            AuthorClass = "User"
	CiRunner                   AuthorClass = "Ci::Runner"
	PersonalAccessToken        AuthorClass = "PersonalAccessToken"
	ProtectedBranch            AuthorClass = "ProtectedBranch"
)

type Change

type Change string

const (
	AccessLevel   Change = "access_level"
	AllowedToPush Change = "allowed to push"
	EmailAddress  Change = "email address"
	Name          Change = "name"
)

type Config

type Config struct {
	AuditLogForwardingEndpoint string
	AuthLogForwardingEndpoint  string
	GitlabHostname             string
	AuditLogPath               string
	AuthLogPath                string
	DBpath                     string

	SyslogServerAddr string
	SyslogProtocol   string
	UseLEEF          bool // Use QRadar propietary LEEF format
}

type GitLabLogStreamer

type GitLabLogStreamer struct {
	// contains filtered or unexported fields
}

func NewGitLabLogStreamer

func NewGitLabLogStreamer(config Config) (*GitLabLogStreamer, error)

func (*GitLabLogStreamer) Watch

func (s *GitLabLogStreamer) Watch() error

type MergeAccessLevelElement

type MergeAccessLevelElement string

const (
	Maintainers MergeAccessLevelElement = "Maintainers"
)

type Severity

type Severity string

type With

type With string

const (
	Saml                       With = "saml"
	TwoFactor                  With = "two-factor"
	TwoFactorViaWebauthnDevice With = "two-factor-via-webauthn-device"
)