storage

README

storage

存储引擎，第一个版本依据海量小文件存储, 这是一个库，可以直接嵌入到程序中使用。目前版本不建议在生产中使用，也自带了一个很简易的服务端，方便压测性能

运行服务端

./storage server -d ./my-store -s 64GB

运行压测

./storage benchmark -p -b "hello world" -d 20s -s 127.0.0.1:8080
  Completed           99870 requests [2022-10-30 23:45:53.946218]
  Completed          197209 requests [2022-10-30 23:45:55.946235]
  Completed          297909 requests [2022-10-30 23:45:57.946233]
  Completed          399386 requests [2022-10-30 23:45:59.946219]
  Completed          501585 requests [2022-10-30 23:46:01.946443]
  Completed          588044 requests [2022-10-30 23:46:03.946283]
  Completed          687964 requests [2022-10-30 23:46:05.946247]
  Completed          786152 requests [2022-10-30 23:46:07.946261]
  Completed          883646 requests [2022-10-30 23:46:09.946316]
  Completed          979513 requests [2022-10-30 23:46:11.946272]
  Finished           979523 requests


Status Codes:            979523:200  [count:code]
Concurrency Level:      10
Time taken for tests:   20.000225667s
Complete requests:      979523
Failed requests:        0
Total Read Data:        170860963 bytes
Total Read body         50379634 bytes
Total Write Body        10774753 bytes
Requests per second:    48975.59739119318 [#/sec] (mean)
Time per request:       0.02041833184825675 [ms] (mean)
Time per request:       0.20418331848256754 [ms] (mean, across all concurrent requests)
Transfer rate:          8342.726325083295 [Kbytes/sec] received
Percentage of the requests served within a certain time (ms)
  50%    173.375µs
  66%    206.5µs
  75%    233.833µs
  80%    258.541µs
  90%    341.625µs
  95%    406.792µs
  98%    533.459µs
  99%    664.417µs
 100%    61.231584ms

Documentation

Index

• Constants
• Variables
• type Data
• type Group
  • func (g *Group) Close() (err error)
  • func (g *Group) Delete(key string) (err error)
  • func (g *Group) Get(key string) (element Data, ok bool, err error)
  • func (g *Group) Put(data []byte) (index string, err error)
• type IdxVersion0
  • func (*IdxVersion0) Descriptor() ([]byte, []int)
  • func (x *IdxVersion0) GetCrc32() uint32
  • func (x *IdxVersion0) GetKey() int64
  • func (x *IdxVersion0) GetOffset() int64
  • func (x *IdxVersion0) GetSize() int32
  • func (x *IdxVersion0) GetTimeout() int64
  • func (*IdxVersion0) ProtoMessage()
  • func (x *IdxVersion0) ProtoReflect() protoreflect.Message
  • func (x *IdxVersion0) Reset()
  • func (x *IdxVersion0) String() string
• type Index
• type IndexInMemory
  • func (i *IndexInMemory) Close() (err error)
  • func (i *IndexInMemory) Delete(key int64) error
  • func (i *IndexInMemory) Get(key int64) (element Data, ok bool, err error)
  • func (i *IndexInMemory) GetSeq() (key int64)
  • func (i *IndexInMemory) Put(key int64, data []byte) (err error)
• type Size
• type Storage
  • func Open(name string, max Size) (s Storage, err error)
• type Storager

Constants

const (
	KB Size = 1024
	MB      = KB * 1024
	GB      = MB * 1024
	TB      = 1024 * GB
)

Variables

var (
	ErrIllegalKey = errors.New("Illegal key")
	ErrDirName    = errors.New("dir name is empty")
)

var (
	ErrFull = errors.New("The space is full")
)

var File_storage_proto protoreflect.FileDescriptor

Types

type Data

type Data struct {
	Index
	Data []byte
}

type Group

type Group struct {
	// contains filtered or unexported fields
}

一组里面有多个引擎，一个引擎只存储32GB

func (*Group) Close

func (g *Group) Close() (err error)

关闭所有索引

func (*Group) Delete

func (g *Group) Delete(key string) (err error)

func (*Group) Get

func (g *Group) Get(key string) (element Data, ok bool, err error)

func (*Group) Put

func (g *Group) Put(data []byte) (index string, err error)

type IdxVersion0

type IdxVersion0 struct {
	Key     int64  `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"`         //返回给客户端的值
	Size    int32  `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`       //大小
	Offset  int64  `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"`   //偏移量
	Timeout int64  `protobuf:"varint,4,opt,name=timeout,proto3" json:"timeout,omitempty"` //超时时间
	Crc32   uint32 `protobuf:"varint,5,opt,name=crc32,proto3" json:"crc32,omitempty"`     //crc32校验和
	// contains filtered or unexported fields
}

索引的数据结构 完整的索引，还有payload(放在第一位)和crc32(第在最后一位)

func (*IdxVersion0) Descriptor

func (*IdxVersion0) Descriptor() ([]byte, []int)

Deprecated: Use IdxVersion0.ProtoReflect.Descriptor instead.

func (*IdxVersion0) GetCrc32

func (x *IdxVersion0) GetCrc32() uint32

func (*IdxVersion0) GetKey

func (x *IdxVersion0) GetKey() int64

func (*IdxVersion0) GetOffset

func (x *IdxVersion0) GetOffset() int64

func (*IdxVersion0) GetSize

func (x *IdxVersion0) GetSize() int32

func (*IdxVersion0) GetTimeout

func (x *IdxVersion0) GetTimeout() int64

func (*IdxVersion0) ProtoMessage

func (*IdxVersion0) ProtoMessage()

func (*IdxVersion0) ProtoReflect

func (x *IdxVersion0) ProtoReflect() protoreflect.Message

func (*IdxVersion0) Reset

func (x *IdxVersion0) Reset()

func (*IdxVersion0) String

func (x *IdxVersion0) String() string

type Index

type Index struct {
	Key     int32  `protobuf:"varint,1,opt,name=key,proto3" json:"key,omitempty"`         //返回给客户端的值
	Size    int32  `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"`       //大小
	Offset  int64  `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"`   //偏移量
	Timeout int64  `protobuf:"varint,4,opt,name=timeout,proto3" json:"timeout,omitempty"` //超时时间
	Crc32   uint32 `protobuf:"varint,5,opt,name=crc32,proto3" json:"crc32,omitempty"`     //crc32校验和
}

type IndexInMemory

type IndexInMemory struct {
	// contains filtered or unexported fields
}

一个内存索引管理32GB文件

func (*IndexInMemory) Close

func (i *IndexInMemory) Close() (err error)

close

func (*IndexInMemory) Delete

func (i *IndexInMemory) Delete(key int64) error

删除

func (*IndexInMemory) Get

func (i *IndexInMemory) Get(key int64) (element Data, ok bool, err error)

获取

func (*IndexInMemory) GetSeq

func (i *IndexInMemory) GetSeq() (key int64)

func (*IndexInMemory) Put

func (i *IndexInMemory) Put(key int64, data []byte) (err error)

保存

type Size

type Size int64

type Storage

type Storage struct {
	*Group
}

func Open

func Open(name string, max Size) (s Storage, err error)

type Storager

type Storager interface {
	Put(key int64, data []byte) (err error)
	Get(key int64) (element Data, ok bool, err error)
	GetSeq() int64
	Delete(key int64) error
	Close() error
}