prepalert

README

prepalert

Toil reduction tool to prepare before responding to Mackerel alerts

preplert consists of two parts: a webhook server that receives Mackerel webhooks and sends the payload to Amazon SQS, and a worker that queries various data based on the webhooks and pastes information for alert response as a GraphAnnotation.

Install

Binary packages

Releases

Usage with AWS Lambda (serverless)

prepalert works with AWS Lambda and Amazon SQS.

Lambda Function requires a webhook and a worker

sequenceDiagram
  autonumber
  Mackerel->>+webhook lambda function : POST /
  webhook lambda function ->>+Amazon SQS: SendMessage
  Amazon SQS-->- webhook lambda function: 200 Ok
  webhook lambda function-->- Mackerel: 200 Ok
  Amazon SQS ->>+ worker lambda function: trigger by AWS Lambda
  worker lambda function ->>+ Data Source: query
  Data Source -->- worker lambda function: query results
  worker lambda function  ->>+ Mackerel: Create Graph Annotation
  Mackerel-->- worker lambda function : 200 Ok
  worker lambda function ->>-  Amazon SQS: Success Delete message

The most small configuration file is as follows:

prepalert {
    required_version = ">=v0.2.0"
    sqs_queue_name   = "prepalert"
    service          = "prod"
}

rule "any_alert" {
    alert {
        any = true
    }

    infomation = <<EOF
How do you respond to alerts?
Describe information about your alert response here.
(This area can use Go's template notation.)
EOF
}

Let's solidify the Lambda package with the following configuration (runtime provided.al2)

lambda.zip
├── bootstrap    # build binary
└── config.yaml  # configuration file

A related document is https://docs.aws.amazon.com/lambda/latest/dg/runtimes-custom.html

LICENSE

MIT License

Copyright (c) 2022 IKEDA Masashi

Documentation

Index

• type Alert
• type App
  • func New(apikey string, cfg *hclconfig.Config) (*App, error)
  • func (app *App) CheckBasicAuth(r *http.Request) bool
  • func (app *App) EnableBasicAuth() bool
  • func (app *App) HandleSQS(ctx context.Context, event *events.SQSEvent) (*grat.BatchItemFailureResponse, error)
  • func (app *App) ProcessRule(ctx context.Context, rule *Rule, body *WebhookBody) error
  • func (app *App) Run(ctx context.Context, opts RunOptions) error
  • func (app *App) ServeHTTP(w http.ResponseWriter, r *http.Request)
  • func (app *App) WithQueryRunningContext(ctx context.Context, reqID uint64, message *events.SQSMessage) context.Context
• type Host
• type QueryData
• type RenderInfomationData
• type Role
• type Rule
  • func NewRule(client *mackerel.Client, cfg *hclconfig.RuleBlock) (*Rule, error)
  • func (rule *Rule) BuildInfomation(ctx context.Context, body *WebhookBody) (string, error)
  • func (rule *Rule) Match(body *WebhookBody) bool
  • func (rule *Rule) RenderInfomation(ctx context.Context, data *RenderInfomationData) (string, error)
• type RunOptions
• type Service
• type WebhookBody

Types

type Alert

type Alert struct {
	OpenedAt          int64   `json:"openedAt"`
	ClosedAt          int64   `json:"closedAt"`
	CreatedAt         int64   `json:"createdAt"`
	CriticalThreshold float64 `json:"criticalThreshold"`
	Duration          int64   `json:"duration"`
	IsOpen            bool    `json:"isOpen"`
	MetricLabel       string  `json:"metricLabel"`
	MetricValue       float64 `json:"metricValue"`
	MonitorName       string  `json:"monitorName"`
	MonitorOperator   string  `json:"monitorOperator"`
	Status            string  `json:"status"`
	Trigger           string  `json:"trigger"`
	ID                string  `json:"id"`
	URL               string  `json:"url"`
	WarningThreshold  float64 `json:"warningThreshold"`
}

type App

type App struct {
	// contains filtered or unexported fields
}

func New

func New(apikey string, cfg *hclconfig.Config) (*App, error)

func (*App) CheckBasicAuth

func (app *App) CheckBasicAuth(r *http.Request) bool

func (*App) EnableBasicAuth

func (app *App) EnableBasicAuth() bool

func (*App) HandleSQS

func (app *App) HandleSQS(ctx context.Context, event *events.SQSEvent) (*grat.BatchItemFailureResponse, error)

func (*App) ProcessRule

func (app *App) ProcessRule(ctx context.Context, rule *Rule, body *WebhookBody) error

func (*App) Run

func (app *App) Run(ctx context.Context, opts RunOptions) error

func (*App) ServeHTTP

func (app *App) ServeHTTP(w http.ResponseWriter, r *http.Request)

func (*App) WithQueryRunningContext

func (app *App) WithQueryRunningContext(ctx context.Context, reqID uint64, message *events.SQSMessage) context.Context

type Host

type Host struct {
	ID        string  `json:"id"`
	Name      string  `json:"name"`
	URL       string  `json:"url"`
	Type      string  `json:"type"`
	Status    string  `json:"status"`
	Memo      string  `json:"memo"`
	IsRetired bool    `json:"isRetired"`
	Roles     []*Role `json:"roles"`
}

type QueryData

type QueryData struct {
	*WebhookBody
	Params interface{}
}

type RenderInfomationData

type RenderInfomationData struct {
	*WebhookBody
	QueryResults map[string]*queryrunner.QueryResult
	Params       interface{}
}

type Role

type Role struct {
	Fullname    string `json:"fullname"`
	ServiceName string `json:"serviceName"`
	ServiceURL  string `json:"serviceUrl"`
	RoleName    string `json:"roleName"`
	RoleURL     string `json:"roleUrl"`
}

type Rule

type Rule struct {
	// contains filtered or unexported fields
}

func NewRule

func NewRule(client *mackerel.Client, cfg *hclconfig.RuleBlock) (*Rule, error)

func (*Rule) BuildInfomation

func (rule *Rule) BuildInfomation(ctx context.Context, body *WebhookBody) (string, error)

func (*Rule) Match

func (rule *Rule) Match(body *WebhookBody) bool

func (*Rule) RenderInfomation

func (rule *Rule) RenderInfomation(ctx context.Context, data *RenderInfomationData) (string, error)

type RunOptions

type RunOptions struct {
	Mode      string
	Address   string
	Prefix    string
	BatchSize int
}

type Service

type Service struct {
	ID    string  `json:"id"`
	Memo  string  `json:"memo"`
	Name  string  `json:"name"`
	OrgID string  `json:"orgId"`
	Roles []*Role `json:"roles"`
}

type WebhookBody

type WebhookBody struct {
	OrgName  string   `json:"orgName"`
	Event    string   `json:"event"`
	ImageURL string   `json:"imageUrl"`
	Memo     string   `json:"memo"`
	Host     *Host    `json:"host,omitempty"`
	Service  *Service `json:"service,omitempty"`
	Alert    *Alert   `json:"alert"`
}