filekv

Documentation

Index

• Variables
• type FileDB
  • func Open(options Options) (*FileDB, error)
  • func (fdb *FileDB) Close()
  • func (f *FileDB) Merge(items ...interface{}) (uint, error)
  • func (f *FileDB) MergeFile(filename string) (uint, error)
  • func (f *FileDB) MergeReader(reader io.Reader) (uint, error)
  • func (fdb *FileDB) Process() error
  • func (fdb *FileDB) Reset() error
  • func (fdb *FileDB) Scan(handler func([]byte, []byte) error) error
  • func (fdb *FileDB) Set(k, v []byte) error
  • func (fdb *FileDB) Size() int64
• type Options
• type Stats
• type Strategy

Variables

var (
	ErrItemExists   = errors.New("item already exist")
	ErrItemFiltered = errors.New("item filtered")
)

var (
	BufferSize = 50 * 1024 * 1024 // 50Mb
	Separator  = ";;;"
	NewLine    = "\n"
	FpRatio    = 0.0001
	MaxItems   = uint(250000)
)

Types

type FileDB

type FileDB struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

FileDB - represents a file db implementation

func Open

func Open(options Options) (*FileDB, error)

Open a new file based db

func (*FileDB) Close

func (fdb *FileDB) Close()

Close ...

func (*FileDB) Merge

func (f *FileDB) Merge(items ...interface{}) (uint, error)

func (*FileDB) MergeFile

func (f *FileDB) MergeFile(filename string) (uint, error)

func (*FileDB) MergeReader

func (f *FileDB) MergeReader(reader io.Reader) (uint, error)

func (*FileDB) Process

func (fdb *FileDB) Process() error

Process added files/slices/elements

func (*FileDB) Reset

func (fdb *FileDB) Reset() error

Reset the db

func (*FileDB) Scan

func (fdb *FileDB) Scan(handler func([]byte, []byte) error) error

Scan - iterate over the whole store using the handler function

func (*FileDB) Set

func (fdb *FileDB) Set(k, v []byte) error

func (*FileDB) Size

func (fdb *FileDB) Size() int64

Size - returns the size of the database in bytes

type Options

type Options struct {
	Path           string
	Compress       bool
	MaxItems       uint
	Cleanup        bool
	SkipEmpty      bool
	FilterCallback func(k, v []byte) bool
	Dedupe         Strategy
}

var DefaultOptions Options = Options{
	Compress:  false,
	Cleanup:   true,
	Dedupe:    MemoryLRU,
	SkipEmpty: true,
}

type Stats

type Stats struct {
	NumberOfFilteredItems uint
	NumberOfAddedItems    uint
	NumberOfDupedItems    uint
	NumberOfItems         uint
}

type Strategy

type Strategy uint8

const (
	None Strategy = iota
	// Uses go standard map without eviction - grows linearly with the items number
	MemoryMap
	// MemoryLRU keeps in memory the last x items and filter with a look-back probabilistic window with fixed size
	MemoryLRU
	// MemoryFilter uses bitset in-memory filters to remove duplicates
	MemoryFilter
	// Use full disk kv store to remove all duplicates - it should have low heap memory footprint but lots of I/O interactions
	DiskFilter
)