clickhousemetrics

package module
v0.2.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 29, 2023 License: MIT Imports: 16 Imported by: 0

README

go-clickhouse-metrics

More documentation can be found here.

Install

go get git.debtfair.ru/external/go-clickhouse-metrics

Configure

Mandatory Name Type Description
Yes Exchange string Name of ClickHouse exchange in RabbitMQ.
Yes RoutingKey string Routing key between exchange and queue in RabbitMQ.
Yes BatchSize int Maximum number of metrics in one RabbitMQ message.
Yes MaxSenders int Maximum number of parallel senders.
Yes ExportPeriod time.Duration Period between metrics exporting.
Yes StopTimeout time.Duration Timeout for last metrics export after graceful stop.
Yes IdleTimeout time.Duration IDLE timeout for RabbitMQ channel.
No ExportEnabled bool This setting allows you to disable export. May be helpful in development environment.

A basic example

package main

import (
	"context"
	"errors"
	"log"
	"sync"
	"time"

	rabbit "git.debtfair.ru/external/amqp-reconnect"
	clickhousemetrics "git.debtfair.ru/external/go-clickhouse-metrics"
	"github.com/google/uuid"
	"github.com/rabbitmq/amqp091-go"
)

func main() {
	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
	defer cancel()

	conn, err := rabbit.NewConnection(ctx, &rabbit.Config{
		Host:       "localhost",
		Port:       5672,
		User:       "guest",
		Password:   "guest",
		MaxRetries: 1,
		Heartbeat:  1 * time.Second,
		AmqpCfg: amqp091.Config{
			ChannelMax: 0,
			FrameSize:  0,
		},
	})
	if err != nil {
		log.Default().Fatal(err)
	}

	collector := clickhousemetrics.New(
		clickhousemetrics.Deps{
			Rabbit: conn,
		},
		clickhousemetrics.Config{
			BatchSize:     25,
			MaxSenders:    72,
			Exchange:      "clickhouse",
			RoutingKey:    "metrics",
			IdleTimeout:   30 * time.Second,
			ExportPeriod:  15 * time.Second,
			StopTimeout:   10 * time.Second,
			ExportEnabled: true,
		},
	)

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		if err := collector.Collect(ctx); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
			log.Default().Fatal(err)
		}
	}()

	t := time.Now()
	modelId := uuid.New().String()
	for i := 0; i < 72; i++ {
		wg.Add(1)
		go func(index int) {
			defer wg.Done()

			for j := 0; j < 1000000; j++ {
				if err := collector.Add("metric_name", "value", "model", modelId, 1, 0, 1, 1, 14.3, 10.1, false, &t, &t, []string{}); err != nil {
					break
				}
			}
		}(i)
	}

	<-ctx.Done()
	collector.Stop()

	wg.Wait()
}

Output RabbitMQ message:

{"uuid":"ae8e645a-1115-459a-97b9-3517b1139b32","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"d44768f9-bf7c-4922-85f8-92244b998706","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"d9784a89-af12-48ee-a865-99c6b8d03c14","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"f7e143d1-19a7-4382-87b8-63cb4f341e8d","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"bc9f7404-5ede-4a3e-a60c-7010dc5ee12f","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"b06f054b-b72a-4c29-9f90-104d3d34d95e","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"21376579-ed73-477b-8675-b028aae7500b","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"99fe6eea-7171-4c08-9088-e65be5fae01b","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"51394e63-2b64-42c3-b4d5-296b35047cc9","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"604f9ad4-1372-4f3f-b55c-64024567fe0c","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"96a3002f-7e5d-479f-8052-cf9ee7dcbfd1","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"b46a5763-1846-4c18-8d6e-109cc1ae4e02","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"a0b10be0-1c96-4b55-9d46-02c02b66341c","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"5ea8c1f4-bc4f-499d-af49-07ebe6d39913","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"2ec57f7c-eed4-4116-90a7-db1bb3a1148f","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"5cc95df2-7628-4cde-b2cd-624f7a541c12","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"22f7f1b7-306c-479f-8988-62804531ca2d","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"be381fc9-3108-40f3-be85-a20d58d88b41","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"d50c650b-f351-45eb-9b59-ab77f13d436a","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"3ff05054-7b02-4d92-9ef9-397c4d7e052e","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"6eeb87ce-57ac-499b-ad77-4c3ba45d28f2","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"f0126bb7-7db3-40d8-949e-aa2329523eb4","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"62f53f5b-0ddc-4148-85bd-a6c07757f92e","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"596f9988-9973-4626-bdb8-bb7bc47f9216","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}
{"uuid":"3798b114-b1a5-4485-a7d1-3c7951901d64","name":"metric_name","value_string":"value","model_type":"model","model_id":"2db7f373-fdb2-49dc-885f-8602788a3aca","created_at":"2023-07-06 13:29:41","value_datetime":"2023-07-06 13:29:41","created_date":"2023-07-06","value":1,"user_id":0,"company_id":1,"domain_id":1,"value_float":14.3,"duration":10.1,"expirable":false,"labels":[]}

Documentation

Index

Constants

View Source 
const (
	DateLayout     = "2006-01-02"
	DateTimeLayout = "2006-01-02 15:04:05"
)

Variables

View Source 
var (
	ErrCollectorAlreadyStopped = errors.New("collector already stopped")
)

Functions

This section is empty.

Types

type Collector 

type Collector interface {
	// Collects all metrics and sends them into ClickHouse queue in RabbitMQ.
	Collect(ctx context.Context) error
	// Stops all collectors and sends remained metrics.
	Stop() error

	// Writes new metric into channel.
	Add(metric Metric) error
}

func New 

func New(deps Deps, config Config) Collector

Returns a new metrics Collector.

type Config 

type Config struct {
	BatchSize     int           // max size of batch with metrics
	MaxSenders    int           // max number of concurrent senders
	Exchange      string        // ClickHouse exchange
	RoutingKey    string        // routing key between exchange and queue in RabbitMQ
	ExportEnabled bool          // is export enabled? if false, metrics will not be sent to RabbitMQ
	IdleTimeout   time.Duration // IDLE timeout for sender`s RabbitMQ channel
	ExportPeriod  time.Duration // period between metrics export
	StopTimeout   time.Duration // timeout for graceful stop
}

type Deps 

type Deps struct {
	Rabbit *rabbit.Connection // TCP connection for RabbitMQ
}

type Metric added in v0.2.0 

type Metric struct {
	Name                 string     // metric name
	ValueString          string     // string part of value
	ModelType            string     // type of measurable entity
	ModelID              string     // id of measurable entity
	ModelKind            string     // kind of measurable entity
	SenderDomain         string     // from domain
	ReceiverDomain       string     // to domain
	ErrorEnhancedCode    string     // error_enhanced_code
	ErrorKind            string     // kind of error
	Source               string     // metric source
	IP                   string     // associated ip
	UserAgent            string     // full user agent
	UserAgentName        string     // name from ua
	UserAgentNameVersion string     // version of name from ua
	UserAgentOS          string     // os from ua
	UserAgentOSVersion   string     // version of os from ua
	UserAgentDevice      string     // device from ua
	UserAgentDeviceType  string     // type of device from ua
	Country              string     // country code
	Region               string     // region name
	RegionCode           string     // region iso code
	City                 string     // city name
	Value                uint64     // integer part of value
	UserID               uint64     // who performed a measurable action
	CompanyID            uint64     // metric owner
	DomainID             uint64     // metric owner
	MailingListID        uint64     // newsletter id
	TemplateID           uint64     // html template id
	ChangedLinks         uint64     // number of changed links
	ErrorCode            uint32     // smtp error code
	LinkNumber           uint32     // number of link in mail
	ValueFloat           float64    // float part of value
	Duration             float64    // for durable metrics (in milliseconds)
	Expirable            bool       // is metric expirable? (auto-deleted after TTL)
	WithAttachments      bool       // any attachments?
	WithPixel            bool       // hidden pixel?
	WithUnsubscribe      bool       // unsubscribe block?
	CreatedAt            *time.Time // timestamp of metric creation
	ValueDateTime        *time.Time // timestamp part of metric
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.