carbon-clickhouse

README

carbon-clickhouse

Graphite metrics receiver with ClickHouse as storage

Production status

Last releases are stable and ready for production use

Build

# build binary
git clone https://github.com/lomik/carbon-clickhouse.git
cd carbon-clickhouse
make

ClickHouse configuration

1. Add graphite_rollup section to config.xml. Sample here. You can use carbon-schema-to-clickhouse for generate rollup xml from graphite storage-schemas.conf.

2. Create tables

CREATE TABLE graphite ( 
  Path String,  
  Value Float64,  
  Time UInt32,  
  Date Date,  
  Timestamp UInt32
) ENGINE = GraphiteMergeTree(Date, (Path, Time), 8192, 'graphite_rollup');
 
-- optional table for faster metric search
CREATE TABLE graphite_tree (
  Date Date,
  Level UInt32,
  Path String,
  Deleted UInt8,
  Version UInt32
) ENGINE = ReplacingMergeTree(Date, (Level, Path), 8192, Version);

GraphiteMergeTree documentation

You can create Replicated tables. See ClickHouse documentation

Configuration

$ carbon-clickhouse -help
Usage of carbon-clickhouse:
  -check-config=false: Check config and exit
  -config="": Filename of config
  -config-print-default=false: Print default config
  -version=false: Print version

[common]
# Prefix for store all internal carbon-clickhouse graphs. Supported macroses: {host}
metric-prefix = "carbon.agents.{host}"
# Endpoint for store internal carbon metrics. Valid values: "" or "local", "tcp://host:port", "udp://host:port"
metric-endpoint = "local"
# Interval of storing internal metrics. Like CARBON_METRIC_INTERVAL
metric-interval = "1m0s"
# GOMAXPROCS
max-cpu = 1

[logging]
# "stderr", "stdout" can be used as file name
file = "/var/log/carbon-clickhouse/carbon-clickhouse.log"
# Logging error level. Valid values: "debug", "info", "warn" "error"
level = "info"

[data]
# Folder for buffering received data
path = "/data/carbon-clickhouse/"
# Rotate (and upload) file interval.
# Minimize chunk-interval for minimize lag between point receive and store
chunk-interval = "1s"
# Auto-increase chunk interval if the number of unprocessed files is grown
# Sample, set chunk interval to 10 if unhandled files count >= 5 and set to 60s if unhandled files count >= 20:
# chunk-auto-interval = "5:10s,20:60s"
chunk-auto-interval = ""

[upload.graphite]
type = "points"
table = "graphite"
threads = 1
url = "http://localhost:8123/"
timeout = "1m0s"

[upload.graphite_tree]
type = "tree"
table = "graphite_tree"
date = "2016-11-01"
threads = 1
url = "http://localhost:8123/"
timeout = "1m0s"
cache-ttl = "12h0m0s"

# # You can define additional upload destinations of any supported type:
# # - points
# # - tree
# # - series (is described below)
# # - tagged (is described below)
# # - points-reverse (same scheme as points, but path 'a1.b2.c3' stored as 'c3.b2.a1')
# # - series-reverse (same scheme as series, but path 'a1.b2.c3' stored as 'c3.b2.a1')

# # Extra table with daily series list
#
# # CREATE TABLE graphite_series (
# #   Date Date,
# #   Level UInt32,
# #   Path String,
# #   Deleted UInt8,
# #   Version UInt32
# # ) ENGINE = ReplacingMergeTree(Date, (Level, Path, Date), 8192, Version);
# [upload.graphite_series]
# type = "series"
# table = "graphite_series"
# threads = 1
# url = "http://localhost:8123/"
# timeout = "1m0s"
# cache-ttl = "12h0m0s"

# # Extra table which can be used as index for tagged series
#
# # CREATE TABLE graphite_tagged (
# #   Date Date,
# #   Tag1 String,
# #   Path String,
# #   Tags Array(String),
# #   Version UInt32,
# #   Deleted UInt8
# # ) ENGINE = ReplacingMergeTree(Date, (Tag1, Path, Date), 8192, Version);
# [upload.graphite_tagged]
# type = "tagged"
# table = "graphite_tagged"
# threads = 1
# url = "http://localhost:8123/"
# timeout = "1m0s"
# cache-ttl = "12h0m0s"

[udp]
listen = ":2003"
enabled = true

[tcp]
listen = ":2003"
enabled = true

[pickle]
listen = ":2004"
enabled = true

# https://github.com/lomik/carbon-clickhouse/blob/master/grpc/carbon.proto
[grpc]
listen = ":2005"
enabled = false

[pprof]
listen = "localhost:7007"
enabled = false