golayeredcache

package module
v0.0.0-...-41bdb7a Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 10, 2024 License: MIT Imports: 11 Imported by: 0

README

简体中文|English

Golang 多级缓存器 Go-Layered-Cache

基于redis和本机内存的支持分布式环境的多级缓存框架

特性:

  • 支持分布式环境下多级缓存同步
  • 支持redis布隆过滤器数据和本地内存布隆过滤器双向同步
  • 支持redis布隆过滤器bf.scandump数据导入本地内存布隆过滤器
  • 支持redis布谷鸟过滤器数据和本地内存布谷鸟过滤器双向同步
  • 支持redis布谷鸟过滤器cf.scandump数据导入本地内存布谷鸟过滤器
  • 支持redis set、get缓存同步
  • 基于golang实现了redisbloom模块的布隆过滤器和布谷鸟过滤器

开始

安装

支持 Go module 并通过以下方式引入你的代码中

import "github.com/begonia-org/go-layered-cache"

随后执行go [build|run|test] 将会自动安装必须的依赖。

另外，你也可以通过下面的方式安装 go-layered-cache 包:

$ go get -u github.com/begonia-org/go-layered-cache
示例

首先，你需要导入 go-layered-cache 包以使用 go-layered-cache，下面是一个最简单的示例:

package main

import (
	"context"
	"log"
	"time"

	"github.com/allegro/bigcache/v3"
	glc "github.com/begonia-org/go-layered-cache"
	"github.com/begonia-org/go-layered-cache/gobloom"
	"github.com/begonia-org/go-layered-cache/gocuckoo"
	"github.com/begonia-org/go-layered-cache/source"
	"github.com/redis/go-redis/v9"
	"github.com/sirupsen/logrus"
)

type Cache struct {
	KV     glc.LayeredKeyValueCache
	bloom  glc.LayeredFilter
	cuckoo glc.LayeredCuckooFilter
}

func NewCache() *Cache {
	ctx := context.Background()
	kvWatcher := source.NewWatchOptions([]interface{}{"test:kv:channel"})
	rdb := redis.NewClient(&redis.Options{
		Addr: "localhost:6379",
		DB:   0,
	})
	KvOptions := glc.LayeredBuildOptions{
		RDB:       rdb,
		Strategy:  glc.LocalThenSource,
		Watcher:   kvWatcher,
		Channel:   "test:kv:channel",
		Log:       logrus.New(),
		KeyPrefix: "cache:test:kv",
	}
	kv, err := glc.NewKeyValueCache(ctx, KvOptions, bigcache.DefaultConfig(10*time.Minute))
	if err != nil {
		panic(err)

	}
	bloomWatcher := source.NewWatchOptions([]interface{}{"test:bloom:channel"})
	bloomOptions := &glc.LayeredBuildOptions{
		RDB:       rdb,
		Strategy:  glc.LocalThenSource,
		Watcher:   bloomWatcher,
		Channel:   "test:bloom:channel",
		Log:       logrus.New(),
		KeyPrefix: "cache:test:bloom",
	}

	cuckooWatcher := source.NewWatchOptions([]interface{}{"test:cuckoo:channel"})
	cuckooOptions := &glc.LayeredBuildOptions{
		RDB:       rdb,
		Strategy:  glc.LocalThenSource,
		Watcher:   cuckooWatcher,
		Channel:   "test:bloom:channel",
		Log:       logrus.New(),
		KeyPrefix: "cache:test:bloom",
	}
	return &Cache{
		KV:     kv,
		bloom:  glc.NewLayeredBloom(bloomOptions, gobloom.DefaultBuildBloomOptions),
		cuckoo: glc.NewLayeredCuckoo(cuckooOptions, gocuckoo.DefaultBuildCuckooOptions),
	}
}
func (c *Cache) watcher(ctx context.Context) {
	errKV := c.KV.Watch(ctx)
	go func(ctx context.Context) {
		for {
			select {
			case <-ctx.Done():
				return
			case err := <-errKV:
				log.Println("kv error:", err)
			}
		}
	}(ctx)
	errBloom := c.bloom.Watch(ctx)
	go func(ctx context.Context) {
		for {
			select {
			case <-ctx.Done():
				return
			case err := <-errBloom:
				log.Println("bloom error:", err)
			}
		}
	}(ctx)
	errCuckoo := c.cuckoo.Watch(ctx)
	go func(ctx context.Context) {
		for {
			select {
			case <-ctx.Done():
				return
			case err := <-errCuckoo:
				log.Println("cuckoo error:", err)
			}
		}

	}(ctx)

}
func main() {
	cache := NewCache()
	ctx := context.Background()
	cache.watcher(ctx)
	err := cache.KV.Set(ctx, "key", []byte("value"))
	if err != nil {
		panic(err)
	}
	value, err := cache.KV.Get(ctx, "key")
	if err != nil {
		panic(err)
	}
	log.Println("kv value:", string(value))

	err = cache.bloom.Add(ctx, "bloom", []byte("value"))
	if err != nil {
		panic(err)
	}
	ret, err := cache.bloom.Check(ctx, "bloom", []byte("value"))
	if err != nil {
		panic(err)
	}

	log.Println("bloom value:", ret)

	err = cache.cuckoo.Add(ctx, "cuckoo", []byte("value"))
	if err != nil {
		panic(err)
	}
	ret, err = cache.cuckoo.Check(ctx, "cuckoo", []byte("value"))
	if err != nil {
		panic(err)
	}
	log.Println("cuckoo value:", ret)
	time.Sleep(10 * time.Second)
}

使用 Go 命令来运行这个demo:

$ go run main.go

贡献

非常期待和欢迎您提交PR为go-layered-cache做出贡献

Documentation

Overview

Package golayeredcache implements a Golang multi-level caching framework based on Redis and local memory for distributed scenarios.

See https://github.com/begonia-org/go-layered-cache for more information about go-layered-cache.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func DecodeFromBytes 

func DecodeFromBytes(src []byte, dst interface{}) error

func EncodeToBytes 

func EncodeToBytes(src interface{}) ([]byte, error)

Types

type BaseLayeredCacheImpl 

type BaseLayeredCacheImpl struct {
	// contains filtered or unexported fields
}

func (*BaseLayeredCacheImpl) Del 

func (lc *BaseLayeredCacheImpl) Del(ctx context.Context, key interface{}, args ...interface{}) error

func (*BaseLayeredCacheImpl) DelOnLocal 

func (lc *BaseLayeredCacheImpl) DelOnLocal(ctx context.Context, key interface{}, args ...interface{}) error

func (*BaseLayeredCacheImpl) Dump 

func (lc *BaseLayeredCacheImpl) Dump(ctx context.Context, key interface{}, args ...interface{}) <-chan interface{}

func (*BaseLayeredCacheImpl) Get 

func (lc *BaseLayeredCacheImpl) Get(ctx context.Context, key interface{}, args ...interface{}) ([]interface{}, error)

func (*BaseLayeredCacheImpl) GetFromLocal 

func (lc *BaseLayeredCacheImpl) GetFromLocal(ctx context.Context, key interface{}, args ...interface{}) ([]interface{}, error)

func (*BaseLayeredCacheImpl) Load 

func (lc *BaseLayeredCacheImpl) Load(ctx context.Context, key interface{}, args ...interface{}) error

func (*BaseLayeredCacheImpl) Scan 

func (lc *BaseLayeredCacheImpl) Scan(ctx context.Context, pattern string, onScan source.OnScan) <-chan error

func (*BaseLayeredCacheImpl) Set 

func (lc *BaseLayeredCacheImpl) Set(ctx context.Context, key interface{}, args ...interface{}) error

func (*BaseLayeredCacheImpl) SetStrategy 

func (lc *BaseLayeredCacheImpl) SetStrategy(strategy CacheReadStrategy)

func (*BaseLayeredCacheImpl) SetToLocal 

func (lc *BaseLayeredCacheImpl) SetToLocal(ctx context.Context, key interface{}, args ...interface{}) error

func (*BaseLayeredCacheImpl) UnWatch 

func (lc *BaseLayeredCacheImpl) UnWatch() error

func (*BaseLayeredCacheImpl) Watch 

func (lc *BaseLayeredCacheImpl) Watch(ctx context.Context, onMessage source.OnMessage) <-chan error

type Cache 

type Cache struct {
	KV     LayeredKeyValueCache
	Bloom  LayeredFilter
	Cuckoo LayeredCuckooFilter
}

type CacheBuildOptions func (cache *Cache) error

func New 

func New(options *CacheOptions) (*Cache, error)

func WithKvCache

type CacheOptions 

type CacheOptions struct {
	RDB                *redis.Client
	KVChannel          string
	BigCacheMaxEntries int
	BloomChannel       string
	CuckooChannel      string
	Log                *logrus.Logger
	KvKeyPrefix        string
	BloomKeyPrefix     string
	CuckooKeyPrefix    string
	Strategy           CacheReadStrategy
}

type CacheReadStrategy 

type CacheReadStrategy int
const (
	// Read from the local cache only
	LocalOnly CacheReadStrategy = iota
	// Read from the local cache first, then from the source cache
	LocalThenSource
)

type LayeredBloomFilter 

type LayeredBloomFilter struct {
	*BaseLayeredCacheImpl
	// contains filtered or unexported fields
}

func (*LayeredBloomFilter) Add 

func (lc *LayeredBloomFilter) Add(ctx context.Context, key string, value []byte) error

func (*LayeredBloomFilter) AddLocalFilter 

func (lc *LayeredBloomFilter) AddLocalFilter(key string, filter local.Filter) error

func (*LayeredBloomFilter) Check 

func (lc *LayeredBloomFilter) Check(ctx context.Context, key string, value []byte) (bool, error)

func (*LayeredBloomFilter) LoadDump 

func (lb *LayeredBloomFilter) LoadDump(ctx context.Context) error

func (*LayeredBloomFilter) OnMessage 

func (lb *LayeredBloomFilter) OnMessage(ctx context.Context, from string, message interface{}) error

func (*LayeredBloomFilter) UnWatch 

func (lc *LayeredBloomFilter) UnWatch() error

func (*LayeredBloomFilter) Watch 

func (lc *LayeredBloomFilter) Watch(ctx context.Context) <-chan error

type LayeredBuildOptions 

type LayeredBuildOptions struct {
	// RDB is the redis client as the source cache
	RDB *redis.Client
	// Watcher is the watch options for the source cache
	Watcher *source.WatchOptions
	// Log is the logger
	Log *logrus.Logger
	// Channel is the channel for the source cache
	// It is used to receive the message from the source cache
	// It is also used to publish the message to the source cache
	// In the case of the source cache is redis, it is the redis xstream channel
	Channel interface{}
	// KeyPrefix is the key prefix for cache,like "cache:test:bloom"
	KeyPrefix string

	// Strategy is the read strategy
	// It is used to determine the read strategy when the cache is read.
	// For Read from the local cache only, it is LocalOnly.
	// For Read from the local cache first, then from the source cache when the local cache is none, it is LocalThenSource
	Strategy CacheReadStrategy
}

LayeredBuildOptions is the options for building the layered cache

type LayeredCache 

type LayeredCache interface {
	LayeredLocalCache

	// Get key value
	Get(ctx context.Context, key interface{}, args ...interface{}) ([]interface{}, error)
	Set(ctx context.Context, key interface{}, args ...interface{}) error

	Watch(ctx context.Context) <-chan error
	UnWatch() error
	// OnMessage is a callback function for the message from the source cache
	OnMessage(ctx context.Context, from string, message interface{}) error
	LoadDump(ctx context.Context) error
}

LayeredCache is the interface for the layered cache

type LayeredCuckooFilter 

type LayeredCuckooFilter interface {
	LayeredFilter
	Del(ctx context.Context, key string, value []byte) error
}

func NewLayeredCuckoo 

func NewLayeredCuckoo(options *LayeredBuildOptions, buildCuckooOptions gocuckoo.CuckooBuildOptions, extOptions ...LayeredFilterOptions) LayeredCuckooFilter

NewLayeredCuckoo creates a new LayeredCuckooFilter. It is only for cuckoo filter. The cuckoo filter is a layered cache, which means it has a local cuckoo filter cache and a redis cuckoo filter cache.

Parameters:

  • options: LayeredBuildOptions is a struct that contains the options for building the layered cache.
  • buildCuckooOptions: gocuckoo.CuckooBuildOptions is the configuration for the local cuckoo filter.

Returns:

  • LayeredCuckooFilter: the new LayeredCuckooFilter

type LayeredCuckooFilterImpl 

type LayeredCuckooFilterImpl struct {
	*BaseLayeredCacheImpl
	// contains filtered or unexported fields
}

func (*LayeredCuckooFilterImpl) Add 

func (lc *LayeredCuckooFilterImpl) Add(ctx context.Context, key string, value []byte) error

func (*LayeredCuckooFilterImpl) AddLocalFilter 

func (lc *LayeredCuckooFilterImpl) AddLocalFilter(key string, filter local.Filter) error

func (*LayeredCuckooFilterImpl) Check 

func (lc *LayeredCuckooFilterImpl) Check(ctx context.Context, key string, value []byte) (bool, error)

func (*LayeredCuckooFilterImpl) Del 

func (lc *LayeredCuckooFilterImpl) Del(ctx context.Context, key string, value []byte) error

func (*LayeredCuckooFilterImpl) LoadDump 

func (lb *LayeredCuckooFilterImpl) LoadDump(ctx context.Context) error

func (*LayeredCuckooFilterImpl) OnMessage 

func (lb *LayeredCuckooFilterImpl) OnMessage(ctx context.Context, from string, message interface{}) error

func (*LayeredCuckooFilterImpl) UnWatch 

func (lc *LayeredCuckooFilterImpl) UnWatch() error

func (*LayeredCuckooFilterImpl) Watch 

func (lc *LayeredCuckooFilterImpl) Watch(ctx context.Context) <-chan error

type LayeredCuckooFilterOptions 

type LayeredCuckooFilterOptions struct {
	RDB                       *redis.Client
	Watcher                   *source.WatchOptions
	Log                       *logrus.Logger
	Entries                   uint64
	Errors                    float64
	Channel                   interface{}
	KeyPrefix                 string
	Strategy                  CacheReadStrategy
	DefaultBuildCuckooOptions *gocuckoo.CuckooBuildOptions
}

LayeredBloomFilterOptions is the options for building the layered bloom filter

type LayeredFilter 

type LayeredFilter interface {
	Watcher
	Loader
	Check(ctx context.Context, key string, value []byte) (bool, error)
	Add(ctx context.Context, key string, value []byte) error
	AddLocalFilter(key string, filter local.Filter) error
}

func NewLayeredBloom 

func NewLayeredBloom(options *LayeredBuildOptions, bloomOptions gobloom.BloomBuildOptions, extOptions ...LayeredFilterOptions) LayeredFilter

NewLayeredBloom creates a new LayeredBloomFilter. It is only for bloom filter. The bloom filter is a layered cache, which means it has a local bloom filter cache and a redis bloom filter cache.

Parameters:

  • options: LayeredBuildOptions is a struct that contains the options for building the layered cache.
  • bloomOptions: gobloom.BloomBuildOptions is the configuration for the local bloom filter.

Returns:

  • LayeredBloomFilter: the new LayeredBloomFilter

type LayeredFilterOptions 

type LayeredFilterOptions func(layered LayeredFilter) error

func WithInitLocalFilterOptions 

func WithInitLocalFilterOptions(filters map[string]local.Filter) LayeredFilterOptions

type LayeredKeyValueCache 

type LayeredKeyValueCache interface {
	// LayeredCache
	LayeredLocalCache
	Watcher
	Loader
	Get(ctx context.Context, key string) ([]byte, error)
	Set(ctx context.Context, key string, value []byte, exp time.Duration) error
	Del(ctx context.Context, key string) error
}

func NewKeyValueCache 

func NewKeyValueCache(ctx context.Context, options LayeredBuildOptions, maxEntries int) (LayeredKeyValueCache, error)

NewKeyValueCache creates a new LayeredKeyValueCache. It is only for key-value cache. The cache is a layered cache, which means it has a local cache and a source cache. The local cache is a bigcache, and the source cache is a redis cache.

Parameters:

  • ctx: context.Context
  • options: LayeredBuildOptions is a struct that contains the options for building the cache.
  • cacheConfig: bigcache.Config is the configuration for the bigcache.
  • maxEntries: int is the max entries for the bigcache.

Returns:

  • LayeredKeyValueCache: the new LayeredKeyValueCache
  • error: if any error occurs

type LayeredKeyValueCacheImpl 

type LayeredKeyValueCacheImpl struct {
	*BaseLayeredCacheImpl
	// contains filtered or unexported fields
}

func (*LayeredKeyValueCacheImpl) Del 

func (lc *LayeredKeyValueCacheImpl) Del(ctx context.Context, key string) error

func (*LayeredKeyValueCacheImpl) Get 

func (lc *LayeredKeyValueCacheImpl) Get(ctx context.Context, key string) ([]byte, error)

func (*LayeredKeyValueCacheImpl) LoadDump 

func (lb *LayeredKeyValueCacheImpl) LoadDump(ctx context.Context) error

func (*LayeredKeyValueCacheImpl) OnMessage 

func (lb *LayeredKeyValueCacheImpl) OnMessage(ctx context.Context, from string, message interface{}) error

func (*LayeredKeyValueCacheImpl) Set 

func (lc *LayeredKeyValueCacheImpl) Set(ctx context.Context, key string, value []byte, exp time.Duration) error

func (*LayeredKeyValueCacheImpl) UnWatch 

func (lc *LayeredKeyValueCacheImpl) UnWatch() error

func (*LayeredKeyValueCacheImpl) Watch 

func (lc *LayeredKeyValueCacheImpl) Watch(ctx context.Context) <-chan error

type LayeredLocalCache 

type LayeredLocalCache interface {
	GetFromLocal(ctx context.Context, key interface{}, args ...interface{}) ([]interface{}, error)
	SetToLocal(ctx context.Context, key interface{}, args ...interface{}) error
}

LayeredLocalCache is the interface for the local cache

type Loader 

type Loader interface {
	LoadDump(ctx context.Context) error
}

type TypeEnum 

type TypeEnum int
const (
	// TypeNone is the default type
	TypeInt TypeEnum = iota
	TypeUint
	TypeInt8
	TypeUint8
	TypeInt16
	TypeUint16
	TypeInt32
	TypeUint32
	TypeInt64
	TypeUint64
	TypeFloat32
	TypeFloat64
	TypeComplex64
	TypeComplex128
	TypeNumber

	TypeString
	TypeBytes
	TypeBool
	TypeStruct
	TypeJson
)

type Watcher 

type Watcher interface {
	Watch(ctx context.Context) <-chan error
	UnWatch() error
}

type WithDelOperator 

type WithDelOperator interface {
	Del(ctx context.Context, key interface{}, args ...interface{}) error
}

type WithLayeredCuckooOptions 

type WithLayeredCuckooOptions func(layered LayeredCuckooFilter) error

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.