clickhouse

package module
v2.0.0-alpha.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 19, 2022 License: MIT Imports: 26 Imported by: 558

README

ClickHouse

run-tests Go Report Card

Golang SQL database driver for Yandex ClickHouse. Supported by Kinescope.

Key features

  • Uses native ClickHouse TCP client-server protocol
  • Compatibility with database/sql (slower than native interface!)
  • Marshal rows into structs (ScanStruct, Select)
  • Connection pool
  • Failover and load balancing
  • Bulk write support (for database/sql use begin->prepare->(in loop exec)->commit)
  • Named and numeric placeholders support
  • LZ4 compression support
  • External data

Support for the ClickHouse protocol advanced features using Context:

  • Query ID
  • Quota Key
  • Settings
  • OpenTelemetry
  • Execution events:
    • Logs
    • Progress
    • Profile events

database/sql interface

DSN

  • hosts - comma-separated list of single address hosts for load-balancing and failover
  • username/password - auth credentials
  • database - select the current default database
  • dial_timeout - a duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix such as "300ms", "1s". Valid time units are "ms", "s", "m".
  • connection_open_strategy - random/in_order (default random).
    • round-robin - choose a round-robin server from the set
    • in_order - first live server is chosen in specified order
  • debug - enable debug output (boolean value)
  • compress - enable lz4 compression (boolean value)

SSL/TLS parameters:

  • secure - establish secure connection (default is false)
  • skip_verify - skip certificate verification (default is false)

Example:

clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms

TODO

  • Bigint types
  • ZSTD
  • Geo

Benchmark

V1 (READ) V2 (READ) std V2 (READ) native
1.218s 924.390ms 675.721ms
V1 (WRITE) V2 (WRITE) std V2 (WRITE) native V2 (WRITE) by column
1.899s 1.177s 699.203ms 661.973ms

Install

go get -u github.com/ClickHouse/clickhouse-go/v2

Examples

native interface
package main

import (
	"context"
	"fmt"
	"log"
	"time"

	"github.com/ClickHouse/clickhouse-go/v2"
)

func example() error {
	conn, err := clickhouse.Open(&clickhouse.Options{
		Addr: []string{"127.0.0.1:9000"},
		Auth: clickhouse.Auth{
			Database: "default",
			Username: "default",
			Password: "",
		},
		//Debug:           true,
		DialTimeout:     time.Second,
		MaxOpenConns:    10,
		MaxIdleConns:    5,
		ConnMaxLifetime: time.Hour,
		Compression: &clickhouse.Compression{
			Method: clickhouse.CompressionLZ4,
		},
	})
	if err != nil {
		return err
	}
	ctx := clickhouse.Context(context.Background(), clickhouse.WithSettings(clickhouse.Settings{
		"max_block_size": 10,
	}), clickhouse.WithProgress(func(p *clickhouse.Progress) {
		fmt.Println("progress: ", p)
	}))
	if err := conn.Ping(ctx); err != nil {
		if exception, ok := err.(*clickhouse.Exception); ok {
			fmt.Printf("Catch exception [%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
		}
		return err
	}
	if err := conn.Exec(ctx, `DROP TABLE IF EXISTS example`); err != nil {
		return err
	}
	err = conn.Exec(ctx, `
		CREATE TABLE IF NOT EXISTS example (
			Col1 UInt8,
			Col2 String,
			Col3 DateTime
		) engine=Memory
	`)
	if err != nil {
		return err
	}
	batch, err := conn.PrepareBatch(ctx, "INSERT INTO example (Col1, Col2, Col3)")
	if err != nil {
		return err
	}
	for i := 0; i < 10; i++ {
		if err := batch.Append(uint8(i), fmt.Sprintf("value_%d", i), time.Now()); err != nil {
			return err
		}
	}
	if err := batch.Send(); err != nil {
		return err
	}

	rows, err := conn.Query(ctx, "SELECT Col1, Col2, Col3 FROM example WHERE Col1 >= $1 AND Col2 <> $2", 0, "xxx")
	if err != nil {
		return err
	}
	for rows.Next() {
		var (
			col1 uint8
			col2 string
			col3 time.Time
		)
		if err := rows.Scan(&col1, &col2, &col3); err != nil {
			return err
		}
		fmt.Printf("row: col1=%d, col2=%s, col3=%s\n", col1, col2, col3)
	}
	rows.Close()
	return rows.Err()
}

func main() {
	if err := example(); err != nil {
		log.Fatal(err)
	}
}
std database/sql interface
package main

import (
	"context"
	"database/sql"
	"fmt"
	"log"
	"time"

	"github.com/ClickHouse/clickhouse-go/v2"
)

func example() error {
	conn, err := sql.Open("clickhouse", "clickhouse://127.0.0.1:9000?dial_timeout=1s&compress=true")
	if err != nil {
		return err
	}
	conn.SetMaxIdleConns(5)
	conn.SetMaxOpenConns(10)
	conn.SetConnMaxLifetime(time.Hour)
	ctx := clickhouse.Context(context.Background(), clickhouse.WithSettings(clickhouse.Settings{
		"max_block_size": 10,
	}), clickhouse.WithProgress(func(p *clickhouse.Progress) {
		fmt.Println("progress: ", p)
	}))
	if err := conn.PingContext(ctx); err != nil {
		if exception, ok := err.(*clickhouse.Exception); ok {
			fmt.Printf("Catch exception [%d] %s \n%s\n", exception.Code, exception.Message, exception.StackTrace)
		}
		return err
	}
	if _, err := conn.ExecContext(ctx, `DROP TABLE IF EXISTS example`); err != nil {
		return err
	}
	_, err = conn.ExecContext(ctx, `
		CREATE TABLE IF NOT EXISTS example (
			Col1 UInt8,
			Col2 String,
			Col3 DateTime
		) engine=Memory
	`)
	if err != nil {
		return err
	}
	scope, err := conn.Begin()
	if err != nil {
		return err
	}
	{
		batch, err := scope.PrepareContext(ctx, "INSERT INTO example (Col1, Col2, Col3)")
		if err != nil {
			return err
		}
		for i := 0; i < 10; i++ {
			if _, err := batch.Exec(uint8(i), fmt.Sprintf("value_%d", i), time.Now()); err != nil {
				return err
			}
		}
	}
	if err := scope.Commit(); err != nil {
		return err
	}
	rows, err := conn.QueryContext(ctx, "SELECT Col1, Col2, Col3 FROM example WHERE Col1 >= $1 AND Col2 <> $2", 0, "xxx")
	if err != nil {
		return err
	}
	for rows.Next() {
		var (
			col1 uint8
			col2 string
			col3 time.Time
		)
		if err := rows.Scan(&col1, &col2, &col3); err != nil {
			return err
		}
		fmt.Printf("row: col1=%d, col2=%s, col3=%s\n", col1, col2, col3)
	}
	rows.Close()
	return rows.Err()
}

func main() {
	if err := example(); err != nil {
		log.Fatal(err)
	}
}

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	CompressionLZ4 compress.Method = compress.LZ4
)
View Source 
var ErrUnsupportedServerRevision = errors.New("unsupported server revision")

Functions

func Context 

func Context(parent context.Context, options ...QueryOption) context.Context

func Named 

func Named(name string, value interface{}) driver.NamedValue

func Open 

func Open(opt *Options) (driver.Conn, error)

Types

type AcquireConnTimeout 

type AcquireConnTimeout struct {
}

func (*AcquireConnTimeout) Error 

func (e *AcquireConnTimeout) Error() string

type Auth 

type Auth struct {
	Database string
	Username string
	Password string
}

type BatchAlreadySent 

type BatchAlreadySent struct{}

func (*BatchAlreadySent) Error 

func (e *BatchAlreadySent) Error() string

type BindMixedNamedAndNumericParams 

type BindMixedNamedAndNumericParams struct{}

func (*BindMixedNamedAndNumericParams) Error 

func (e *BindMixedNamedAndNumericParams) Error() string

type Compression 

type Compression struct {
	Method compress.Method
}

type Conn 

type Conn = driver.Conn

type ConnOpenStrategy 

type ConnOpenStrategy uint8
const (
	ConnOpenInOrder ConnOpenStrategy = iota
	ConnOpenRoundRobin
)

type Date 

type Date time.Time

type DateTime 

type DateTime time.Time

type Exception 

type Exception = proto.Exception

type InvalidColumnIndex 

type InvalidColumnIndex struct {
	// contains filtered or unexported fields
}

func (*InvalidColumnIndex) Error 

func (e *InvalidColumnIndex) Error() string

type Log 

type Log struct {
	Time      time.Time
	TimeMicro uint32
	Hostname  string
	QueryID   string
	ThreadID  uint64
	Priority  int8
	Source    string
	Text      string
}

type Options 

type Options struct {
	TLS              *tls.Config
	Addr             []string
	Auth             Auth
	Debug            bool
	Settings         Settings
	DialTimeout      time.Duration
	Compression      *Compression
	MaxOpenConns     int
	MaxIdleConns     int
	ConnMaxLifetime  time.Duration
	ConnOpenStrategy ConnOpenStrategy
}

type ProfileEvent 

type ProfileEvent struct {
	Hostname    string
	CurrentTime time.Time
	ThreadID    uint64
	Type        string
	Name        string
	Value       int64
}

type Progress 

type Progress = proto.Progress

type QueryOption 

type QueryOption func(*QueryOptions) error

func WithExternalTable 

func WithExternalTable(t ...*external.Table) QueryOption

func WithLogs 

func WithLogs(fn func(*Log)) QueryOption

func WithProfileEvents 

func WithProfileEvents(fn func([]ProfileEvent)) QueryOption

func WithProgress 

func WithProgress(fn func(*Progress)) QueryOption

func WithQueryID 

func WithQueryID(queryID string) QueryOption

func WithQuotaKey 

func WithQuotaKey(quotaKey string) QueryOption

func WithSettings 

func WithSettings(settings Settings) QueryOption

func WithSpan 

func WithSpan(span trace.SpanContext) QueryOption

type QueryOptions 

type QueryOptions struct {
	// contains filtered or unexported fields
}

type ScanStructErr 

type ScanStructErr struct {
	// contains filtered or unexported fields
}

func (*ScanStructErr) Error 

func (e *ScanStructErr) Error() string

type ServerVersion 

type ServerVersion = proto.ServerHandshake

type Settings 

type Settings map[string]interface{}

type UnexpectedArguments 

type UnexpectedArguments = proto.UnexpectedArguments

type UnexpectedPacket 

type UnexpectedPacket struct {
	// contains filtered or unexported fields
}

func (*UnexpectedPacket) Error 

func (e *UnexpectedPacket) Error() string

type UnexpectedScanDestination 

type UnexpectedScanDestination struct {
	// contains filtered or unexported fields
}

func (*UnexpectedScanDestination) Error 

func (e *UnexpectedScanDestination) Error() string

Source Files

View all Source files

Directories

Path Synopsis
benchmark
v1/read
v1/write
v2/read
v2/read-native
v2/write
v2/write-native
v2/write-native-columnar
examples
native
std
lib
binary
cityhash102
* COPY from https://github.com/zentures/cityhash/ NOTE: The code is modified to be compatible with CityHash128 used in ClickHouse
 * COPY from https://github.com/zentures/cityhash/ NOTE: The code is modified to be compatible with CityHash128 used in ClickHouse
column
column/codegen
compress
driver
io
proto
timezone
issues
std
stress

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.