sharding

README

Gorm Sharding

Gorm Sharding plugin using SQL parser and replace for splits large tables into smaller ones, redirects Query into sharding tables. Give you a high performance database access.

Gorm Sharding 是一个高性能的数据库分表中间件。

它基于 Conn 层做 SQL 拦截、AST 解析、分表路由、自增主键填充，带来的额外开销极小。对开发者友好、透明，使用上与普通 SQL、Gorm 查询无差别，只需要额外注意一下分表键条件。

Features

• Non-intrusive design. Load the plugin, specify the config, and all done.
• Lighting-fast. No network based middlewares, as fast as Go.
• Multiple database support. PostgreSQL tested, MySQL and SQLite is coming.
• Allows you custom the Primary Key generator (Longkey, Sequence, Snowflake ...).

Sharding process

This graph show up how Gorm Sharding works.

Install

go get -u gorm.io/sharding

Usage

Open a db session.

dsn := "postgres://localhost:5432/sharding-db?sslmode=disable"
db, err := gorm.Open(postgres.New(postgres.Config{DSN: dsn}))

Config the sharding middleware, register the tables which you want to shard. See Godoc for config details.

db.Use(sharding.Register(sharding.Config{
    ShardingKey: "user_id",
    ShardingAlgorithm: func(value interface{}) (suffix string, err error) {
        if user_id, ok := value.(int64); ok {
            return fmt.Sprintf("_%02d", user_id%64), nil
        }
        return "", errors.New("invalid user_id")
    },
    PrimaryKeyGenerate: func(tableIdx int64) int64 {
        // use LongKey for generate a sequence primary key with table index
        return longkey.Next(tableIdx)
    }
}, "orders").Register(sharding.Config{
    ShardingKey: "user_id",
    ShardingAlgorithm: func(value interface{}) (suffix string, err error) {
        if user_id, ok := value.(int64); ok {
            return fmt.Sprintf("_%02d", user_id%256), nil
        }
        return "", errors.New("invalid user_id")
    },
    PrimaryKeyGenerate: func(tableIdx int64) int64 {
        return snowflake_node.Generate().Int64()
    }
    // This case for show up give notifications, audit_logs table use same sharding rule.
}, Notification{}, AuditLog{}))

Use the db session as usual. Just note that the query should have the Sharding Key when operate sharding tables.

// Gorm create example, this will insert to orders_02
db.Create(&Order{UserID: 2})
// sql: INSERT INTO orders_2 ...

// Show have use Raw SQL to insert, this will insert into orders_03
db.Exec("INSERT INTO orders(user_id) VALUES(?)", int64(3))

// This will throw ErrMissingShardingKey error, because there not have sharding key presented.
db.Create(&Order{Amount: 10, ProductID: 100})
fmt.Println(err)

// Find, this will redirect query to orders_02
var orders []Order
db.Model(&Order{}).Where("user_id", int64(2)).Find(&orders)
fmt.Printf("%#v\n", orders)

// Raw SQL also supported
db.Raw("SELECT * FROM orders WHERE user_id = ?", int64(3)).Scan(&orders)
fmt.Printf("%#v\n", orders)

// This will throw ErrMissingShardingKey error, because WHERE conditions not included sharding key
err = db.Model(&Order{}).Where("product_id", "1").Find(&orders).Error
fmt.Println(err)

// Update and Delete are similar to create and query
db.Exec("UPDATE orders SET product_id = ? WHERE user_id = ?", 2, int64(3))
err = db.Exec("DELETE FROM orders WHERE product_id = 3").Error
fmt.Println(err) // ErrMissingShardingKey

The full example is here.

Primary Key

When you sharding tables, you need consider how the primary key generate.

Recommend options:

• LongKey
• Database sequence by manully
• Snowflake

License

MIT license.

Original fork from Longbridge.

Documentation

Index

• Variables
• type Config
• type ConnPool
  • func (pool *ConnPool) BeginTx(ctx context.Context, opt *sql.TxOptions) (gorm.ConnPool, error)
  • func (pool *ConnPool) Commit() error
  • func (pool ConnPool) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)
  • func (pool *ConnPool) Ping() error
  • func (pool ConnPool) PrepareContext(ctx context.Context, query string) (*sql.Stmt, error)
  • func (pool ConnPool) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)
  • func (pool ConnPool) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row
  • func (pool *ConnPool) Rollback() error
  • func (pool *ConnPool) String() string
• type Sharding
  • func Register(config Config, tables ...interface{}) *Sharding
  • func (s *Sharding) Initialize(db *gorm.DB) error
  • func (s *Sharding) LastQuery() string
  • func (s *Sharding) Name() string
  • func (s *Sharding) Register(config Config, tables ...interface{}) *Sharding

Variables

var (
	ErrMissingShardingKey = errors.New("sharding key or id required, and use operator =")
	ErrInvalidID          = errors.New("invalid id format")
)

Types

type Config

type Config struct {
	// When DoubleWrite enabled, data will double write to both main table and sharding table.
	DoubleWrite bool

	// ShardingKey specifies the table column you want to used for sharding the table rows.
	// For example, for a product order table, you may want to split the rows by `user_id`.
	ShardingKey string

	// ShardingAlgorithm specifies a function to generate the sharding
	// table's suffix by the column value.
	// For example, this function implements a mod sharding algorithm.
	//
	// 	func(value interface{}) (suffix string, err error) {
	//		if uid, ok := value.(int64);ok {
	//			return fmt.Sprintf("_%02d", user_id % 64), nil
	//		}
	//		return "", errors.New("invalid user_id")
	// 	}
	ShardingAlgorithm func(columnValue interface{}) (suffix string, err error)

	// ShardingAlgorithmByPrimaryKey specifies a function to generate the sharding
	// table's suffix by the primary key. Used when no sharding key specified.
	// For example, this function use the LongKey library to generate the suffix.
	//
	// 	func(id int64) (suffix string) {
	//		return fmt.Sprintf("_%02d", longkey.TableIdx(id))
	//	}
	ShardingAlgorithmByPrimaryKey func(id int64) (suffix string)

	// PrimaryKeyGenerate specifies a function to generate the primary key.
	// Used only when insert and the record does not contains an id field.
	// We recommend you use the
	// [LongKey](https://github.com/longbridgeapp/longkey) component,
	// it is a distributed primary key generator.
	// When use auto-increment like generator, the tableIdx argument could ignored.
	//
	// For example, this function use the LongKey library to generate the primary key.
	//
	// 	func(tableIdx int64) int64 {
	//		return longkey.Next(tableIdx)
	//	}
	PrimaryKeyGenerate func(tableIdx int64) int64
}

Config specifies the configuration for sharding.

type ConnPool

type ConnPool struct {
	gorm.ConnPool
	// contains filtered or unexported fields
}

ConnPool Implement a ConnPool for replace db.Statement.ConnPool in Gorm

func (*ConnPool) BeginTx

func (pool *ConnPool) BeginTx(ctx context.Context, opt *sql.TxOptions) (gorm.ConnPool, error)

BeginTx Implement ConnPoolBeginner.BeginTx

func (*ConnPool) Commit

func (pool *ConnPool) Commit() error

Implement TxCommitter.Commit

func (ConnPool) ExecContext

func (pool ConnPool) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error)

func (*ConnPool) Ping

func (pool *ConnPool) Ping() error

func (ConnPool) PrepareContext

func (pool ConnPool) PrepareContext(ctx context.Context, query string) (*sql.Stmt, error)

func (ConnPool) QueryContext

func (pool ConnPool) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error)

https://github.com/go-gorm/gorm/blob/v1.21.11/callbacks/query.go#L18

func (ConnPool) QueryRowContext

func (pool ConnPool) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row

func (*ConnPool) Rollback

func (pool *ConnPool) Rollback() error

Implement TxCommitter.Rollback

func (*ConnPool) String

func (pool *ConnPool) String() string

type Sharding

type Sharding struct {
	*gorm.DB
	ConnPool *ConnPool
	// contains filtered or unexported fields
}

func Register

func Register(config Config, tables ...interface{}) *Sharding

func (*Sharding) Initialize

func (s *Sharding) Initialize(db *gorm.DB) error

Initialize implement for Gorm plugin interface

func (*Sharding) LastQuery

func (s *Sharding) LastQuery() string

LastQuery get last SQL query

func (*Sharding) Name

func (s *Sharding) Name() string

Name plugin name for Gorm plugin interface

func (*Sharding) Register

func (s *Sharding) Register(config Config, tables ...interface{}) *Sharding