rkclickhouse

package module
v1.2.21 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 7, 2023 License: Apache-2.0 Imports: 19 Imported by: 1

README

rk-db/clickhouse

Init gorm from YAML config.

This belongs to rk-boot family. We suggest use this lib from rk-boot.

Supported bootstrap

Bootstrap Description
YAML based Start gorm from YAML
Code based Start gorm from code

Supported Instances

All instances could be configured via YAML or Code.

User can enable anyone of those as needed! No mandatory binding!

Instance Description
gorm.DB Compatible with original gorm
Logger Implementation of gorm wrapped by uber-go/zap logger
AutoCreation Automatically create DB if missing in ClickHouse

Installation

  • rk-boot: Bootstrapper base
  • rk-gin: Bootstrapper for gin-gonic/gin Web Framework for API
  • rk-db/clickhouse: Bootstrapper for gorm of clickhouse
go get github.com/rookie-ninja/rk-boot/v2
go get github.com/rookie-ninja/rk-gin/v2
go get github.com/rookie-ninja/rk-db/clickhouse

Quick Start

In the bellow example, we will run ClickHouse locally and implement API of Create/List/Get/Update/Delete for User model with Gin.

  • GET /v1/user, List users
  • GET /v1/user/:id, Get user
  • PUT /v1/user, Create user
  • POST /v1/user/:id, Update user
  • DELETE /v1/user/:id, Delete user

Please refer example at example.

1.Create boot.yaml

boot.yaml

  • Create web server with Gin framework at port 8080
  • Create ClickHouse entry which connects ClickHouse at localhost:9000
---
gin:
  - name: user-service
    port: 8080
    enabled: true
clickhouse:
  - name: user-db                          # Required
    enabled: true                          # Required
    domain: "*"                            # Optional
    addr: "localhost:9000"                 # Optional, default: localhost:9000
    user: default                          # Optional, default: default
    pass: ""                               # Optional, default: ""
#    logger:
#      entry: ""
#      level: info
#      encoding: json
#      outputPaths: [ "stdout", "log/db.log" ]
#      slowThresholdMs: 5000
#      ignoreRecordNotFoundError: false
    database:
      - name: user                         # Required
        autoCreate: true                   # Optional, default: false
#        dryRun: false                     # Optional, default: false
#        params: []                        # Optional, default: []
2.Create main.go

In the main() function, we implement two things.

  • Add User{} as auto migrate option which will create table in DB automatically if missing.
  • Register APIs into Gin router.
// Copyright (c) 2021 rookie-ninja
//
// Use of this source code is governed by an Apache-style
// license that can be found in the LICENSE file.
package main

import (
	"context"
	"github.com/gin-gonic/gin"
	"github.com/rookie-ninja/rk-boot/v2"
	"github.com/rookie-ninja/rk-db/clickhouse"
	"github.com/rookie-ninja/rk-gin/v2/boot"
	"github.com/rs/xid"
	"gorm.io/gorm"
	"net/http"
	"time"
)

var userDb *gorm.DB

func main() {
	boot := rkboot.NewBoot()

	boot.Bootstrap(context.TODO())

	// Auto migrate database and init global userDb variable
	clickHouseEntry := rkclickhouse.GetClickHouseEntry("user-db")
	userDb = clickHouseEntry.GetDB("user")
	if !userDb.DryRun {
		userDb.AutoMigrate(&User{})
	}

	// Register APIs
	ginEntry := rkgin.GetGinEntry("user-service")
	ginEntry.Router.GET("/v1/user", ListUsers)
	ginEntry.Router.GET("/v1/user/:id", GetUser)
	ginEntry.Router.PUT("/v1/user", CreateUser)
	ginEntry.Router.POST("/v1/user/:id", UpdateUser)
	ginEntry.Router.DELETE("/v1/user/:id", DeleteUser)

	boot.WaitForShutdownSig(context.TODO())
}

// *************************************
// *************** Model ***************
// *************************************

type Base struct {
	CreatedAt time.Time `yaml:"-" json:"-"`
	UpdatedAt time.Time `yaml:"-" json:"-"`
}

type User struct {
	Base
	Id   string `yaml:"id" json:"id"`
	Name string `yaml:"name" json:"name"`
}

func ListUsers(ctx *gin.Context) {
	userList := make([]*User, 0)
	res := userDb.Find(&userList)

	if res.Error != nil {
		ctx.JSON(http.StatusInternalServerError, res.Error)
		return
	}
	ctx.JSON(http.StatusOK, userList)
}

func GetUser(ctx *gin.Context) {
	uid := ctx.Param("id")
	user := &User{}
	res := userDb.Find(user, "id = ?", uid)

	if res.Error != nil {
		ctx.JSON(http.StatusInternalServerError, res.Error)
		return
	}
	ctx.JSON(http.StatusOK, user)
}

func CreateUser(ctx *gin.Context) {
	user := &User{
		Id:   xid.New().String(),
		Name: ctx.Query("name"),
	}

	res := userDb.Create(user)

	if res.Error != nil {
		ctx.JSON(http.StatusInternalServerError, res.Error)
		return
	}
	ctx.JSON(http.StatusOK, user)
}

func UpdateUser(ctx *gin.Context) {
	uid := ctx.Param("id")
	user := &User{
		Id:   uid,
		Name: ctx.Query("name"),
	}

	res := userDb.Where("id = ?", uid).Updates(user)

	if res.Error != nil {
		ctx.JSON(http.StatusInternalServerError, res.Error)
		return
	}

	ctx.JSON(http.StatusOK, user)
}

func DeleteUser(ctx *gin.Context) {
	uid := ctx.Param("id")

	res := userDb.Delete(&User{}, "id = ?", uid)

	if res.Error != nil {
		ctx.JSON(http.StatusInternalServerError, res.Error)
		return
	}

	ctx.String(http.StatusOK, "success")
}
3.Start server
$ go run main.go

2022-01-07T03:11:18.538+0800    INFO    boot/gin_entry.go:913   Bootstrap ginEntry      {"eventId": "181b17a7-591f-419a-95cc-2cda7efc61f2", "entryName": "user-service"}
------------------------------------------------------------------------
endTime=2022-01-07T03:11:18.53883+08:00
startTime=2022-01-07T03:11:18.538741+08:00
elapsedNano=88391
timezone=CST
ids={"eventId":"181b17a7-591f-419a-95cc-2cda7efc61f2"}
app={"appName":"rk","appVersion":"","entryName":"user-service","entryType":"GinEntry"}
env={"arch":"amd64","az":"*","domain":"*","hostname":"lark.local","localIP":"10.8.0.6","os":"darwin","realm":"*","region":"*"}
payloads={"ginPort":8080}
error={}
counters={}
pairs={}
timing={}
remoteAddr=localhost
operation=Bootstrap
resCode=OK
eventStatus=Ended
EOE
2022-01-07T03:11:18.538+0800    INFO    Bootstrap ClickHouse entry      {"entryName": "user-db", "clickHouseUser": "default", "clickHouseAddr": "localhost:9000"}
2022-01-07T03:11:18.538+0800    INFO    creating database user if not exists
2022-01-07T03:11:18.556+0800    INFO    creating successs or database user exists
2022-01-07T03:11:18.556+0800    INFO    connecting to database user
2022-01-07T03:11:18.567+0800    INFO    connecting to database user success
4.Validation
4.1 Create user

Create a user with name of rk-dev.

$ curl -X PUT "localhost:8080/v1/user?name=rk-dev"
{"id":"c7bjufjd0cvqfaenpqjg","name":"rk-dev"}
4.1 Update user

Update user name to rk-dev-updated.

$ curl -X POST "localhost:8080/v1/user/c7bjufjd0cvqfaenpqjg?name=rk-dev-updated"
{"id":"c7bjufjd0cvqfaenpqjg","name":"rk-dev-updated"}
4.1 List users

List users.

$ curl -X GET localhost:8080/v1/user
[{"id":"c7bjufjd0cvqfaenpqjg","name":"rk-dev-updated"}]
4.1 Get user

Get user with id=c7bjtobd0cvqfaenpqj0.

$ curl -X GET localhost:8080/v1/user/c7bjufjd0cvqfaenpqjg
{"id":"c7bjufjd0cvqfaenpqjg","name":"rk-dev-updated"}
4.1 Delete user
$ curl -X DELETE localhost:8080/v1/user/c7bjufjd0cvqfaenpqjg
success

image

YAML Options

User can start multiple gorm instances at the same time. Please make sure use different names.

name Required description type default value
clickhouse.name Required The name of entry string ClickHouse
clickhouse.enabled Required Enable entry or not bool false
clickhouse.domain Optional See locale description bellow string ""
clickhouse.description Optional Description of echo entry. string ""
clickhouse.user Optional ClickHouse username string root
clickhouse.pass Optional ClickHouse password string pass
clickhouse.addr Optional ClickHouse remote address string localhost:9000
clickhouse.database.name Required Name of database string ""
clickhouse.database.autoCreate Optional Create DB if missing bool false
clickhouse.database.dryRun Optional Run gorm.DB with dry run mode bool false
clickhouse.database.params Optional Connection params []string [""]
clickhouse.database.plugins.prom.enabled Optional Enable prometheus plugin bool false
clickhouse.logger.entry Optional Reference of zap logger entry name string ""
clickhouse.logger.level Optional Logging level, [info, warn, error, silent] string warn
clickhouse.logger.encoding Optional log encoding, [console, json] string console
clickhouse.logger.outputPaths Optional log output paths []string ["stdout"]
clickhouse.logger.slowThresholdMs Optional Slow SQL threshold int 5000
clickhouse.logger.ignoreRecordNotFoundError Optional As name described bool false
Usage of domain
RK use <domain> to distinguish different environment.
Variable of <locale> could be composed as form of <domain>
- domain: Stands for different environment, like dev, test, prod and so on, users can define it by themselves.
          Environment variable: DOMAIN
          Eg: prod
          Wildcard: supported

How it works?
Firstly, get environment variable named as  DOMAIN.
Secondly, compare every element in locale variable and environment variable.
If variables in locale represented as wildcard(*), we will ignore comparison step.

Example:
# let's assuming we are going to define DB address which is different based on environment.
# Then, user can distinguish DB address based on locale.
# We recommend to include locale with wildcard.
---
DB:
  - name: redis-default
    domain: "*"
    addr: "192.0.0.1:6379"
  - name: redis-in-test
    domain: "test"
    addr: "192.0.0.1:6379"
  - name: redis-in-prod
    domain: "prod"
    addr: "176.0.0.1:6379"

Documentation

Overview

Package rkclickhouse is an implementation of rkentry.Entry which could be used gorm.DB instance.

Index

Constants

View Source 
const ClickHouseEntryType = "ClickHouseEntry"

Variables

This section is empty.

Functions

func RegisterClickHouseEntryYAML added in v1.0.0 

func RegisterClickHouseEntryYAML(raw []byte) map[string]rkentry.Entry

RegisterClickHouseEntryYAML register ClickHouseEntry based on config file into rkentry.GlobalAppCtx

Types

type BootConfig 

type BootConfig struct {
	ClickHouse []*BootConfigE `yaml:"clickhouse" json:"clickhouse"`
}

BootConfig ClickHouse entry boot config which reflects to YAML config

type BootConfigE added in v1.1.0 

type BootConfigE struct {
	Enabled     bool   `yaml:"enabled" json:"enabled"`
	Name        string `yaml:"name" json:"name"`
	Description string `yaml:"description" json:"description"`
	Domain      string `yaml:"domain" json:"domain"`
	User        string `yaml:"user" json:"user"`
	Pass        string `yaml:"pass" json:"pass"`
	Addr        string `yaml:"addr" json:"addr"`
	Database    []struct {
		Name       string   `yaml:"name" json:"name"`
		Params     []string `yaml:"params" json:"params"`
		DryRun     bool     `yaml:"dryRun" json:"dryRun"`
		AutoCreate bool     `yaml:"autoCreate" json:"autoCreate"`
		Plugins    struct {
			Prom plugins.PromConfig `yaml:"prom"`
		} `yaml:"plugins" json:"plugins"`
	} `yaml:"database" json:"database"`
	Logger struct {
		Entry                     string   `json:"entry" yaml:"entry"`
		Level                     string   `json:"level" yaml:"level"`
		Encoding                  string   `json:"encoding" yaml:"encoding"`
		OutputPaths               []string `json:"outputPaths" yaml:"outputPaths"`
		SlowThresholdMs           int      `json:"slowThresholdMs" yaml:"slowThresholdMs"`
		IgnoreRecordNotFoundError bool     `json:"ignoreRecordNotFoundError" yaml:"ignoreRecordNotFoundError"`
	} `json:"logger" yaml:"logger"`
}

type ClickHouseEntry 

type ClickHouseEntry struct {
	User string `yaml:"-" json:"-"`

	Addr string `yaml:"-" json:"-"`

	GormDbMap     map[string]*gorm.DB     `yaml:"-" json:"-"`
	GormConfigMap map[string]*gorm.Config `yaml:"-" json:"-"`
	// contains filtered or unexported fields
}

ClickHouseEntry will init gorm.DB or SqlMock with provided arguments

func GetClickHouseEntry 

func GetClickHouseEntry(name string) *ClickHouseEntry

GetClickHouseEntry returns ClickHouseEntry instance

func RegisterClickHouseEntry 

func RegisterClickHouseEntry(opts ...Option) *ClickHouseEntry

RegisterClickHouseEntry will register Entry into GlobalAppCtx

func (*ClickHouseEntry) Bootstrap 

func (entry *ClickHouseEntry) Bootstrap(ctx context.Context)

Bootstrap ClickHouseEntry

func (*ClickHouseEntry) GetDB 

func (entry *ClickHouseEntry) GetDB(name string) *gorm.DB

GetDB returns gorm.DB

func (*ClickHouseEntry) GetDescription 

func (entry *ClickHouseEntry) GetDescription() string

GetDescription returns entry description

func (*ClickHouseEntry) GetName 

func (entry *ClickHouseEntry) GetName() string

GetName returns entry name

func (*ClickHouseEntry) GetType 

func (entry *ClickHouseEntry) GetType() string

GetType returns entry type

func (*ClickHouseEntry) Interrupt 

func (entry *ClickHouseEntry) Interrupt(ctx context.Context)

Interrupt ClickHouseEntry

func (*ClickHouseEntry) IsHealthy 

func (entry *ClickHouseEntry) IsHealthy() bool

IsHealthy checks healthy status remote provider

func (*ClickHouseEntry) RegisterPromMetrics added in v1.2.7 

func (entry *ClickHouseEntry) RegisterPromMetrics(registry *prometheus.Registry) error

func (*ClickHouseEntry) String 

func (entry *ClickHouseEntry) String() string

String returns json marshalled string

type Logger 

type Logger struct {
	SlowThreshold             time.Duration
	IgnoreRecordNotFoundError bool
	LogLevel                  gormLogger.LogLevel
	// contains filtered or unexported fields
}

func (*Logger) Error added in v1.2.5 

func (l *Logger) Error(ctx context.Context, msg string, data ...interface{})

func (*Logger) Info added in v1.2.5 

func (l *Logger) Info(ctx context.Context, msg string, data ...interface{})

func (*Logger) LogMode added in v1.2.5 

func (l *Logger) LogMode(level gormLogger.LogLevel) gormLogger.Interface

func (*Logger) Trace added in v1.2.5 

func (l *Logger) Trace(ctx context.Context, begin time.Time, fc func() (sql string, rowsAffected int64), err error)

func (*Logger) Warn added in v1.2.5 

func (l *Logger) Warn(ctx context.Context, msg string, data ...interface{})

type Option 

type Option func(*ClickHouseEntry)

func WithAddr 

func WithAddr(addr string) Option

WithAddr provide address

func WithDatabase 

func WithDatabase(name string, dryRun, autoCreate bool, params ...string) Option

WithDatabase provide database

func WithDescription 

func WithDescription(description string) Option

WithDescription provide name.

func WithLogger added in v1.2.6 

func WithLogger(logger *Logger) Option

WithLogger provide Logger

func WithName 

func WithName(name string) Option

WithName provide name.

func WithPass 

func WithPass(pass string) Option

WithPass provide password

func WithPlugin added in v1.2.7 

func WithPlugin(name string, plugin gorm.Plugin) Option

func WithUser 

func WithUser(user string) Option

WithUser provide user

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.