file

Documentation

Index

• Constants
• func CheckStoreSize(directory, filePrefix string, maxFiles int, maxFileSize int64) (err error)
• func FindFileIndexes(directory, filePrefix string) (minIndex, maxIndex uint64, maxDataSize int64, totalFiles int, err error)
• func IncrementWriteFile(directory, filePrefix string, maxFiles int) (index, data string, err error)
• func MegaByteToByte(n int64) int64
• func PruneDirectory(directory, filePrefix, readIndexFile string) (err error)
• func ReadData(data map[uint64]*[]byte, indexes ...*Index) (err error)
• func ReadIndexes(directory, filePrefix string, indices map[uint64]*Index) (indexRead, indexWrite uint64, err error)
• func Size(filepath string) (size int64, err error)
• func Sizes(directory, filePrefix, fileSuffix string) (int64, error)
• func WriteIndex(indexes ...*Index) (err error)
• type Bytes
  • func WriteData(dataFile string, maxFileSize int64, dataIn []Bytes) (starts, finishes []int64, dataOut []Bytes, err error)
• type Files
  • func FindFiles(directory, filePrefix string) (files Files, dataSizeWrite int64, err error)
• type Index
  • func ReadIndex(indexFile string) (indexes []*Index, indexRead, indexWrite uint64, err error)
  • func (i *Index) MarshalBinary() (data []byte, err error)
  • func (i *Index) UnmarshalBinary(data []byte) (err error)

Constants

const (
	ErrLessBytes       string = "not enough bytes to decode"
	ErrBadBytesWritten string = "bytes written not equal to input slice"
	ErrMaxFilesCreated string = "max files created, store is full"
	ErrMaxStoreSize    string = "max store size reached, store is full"
)

error constants

const (
	SizeIndex          int    = 25
	FileIndexf         string = "%s_%03d.idx"
	FileDataf          string = "%s_%03d.dat"
	FileLockf          string = "%s_lock.lock"
	ScanFilef          string = "%s_%s%s"
	FileExtensionIndex string = ".idx"
	FileExtensionData  string = ".dat"
	RecoverPanicf      string = "recovered from panic: %v"
)

Functions

func CheckStoreSize

func CheckStoreSize(directory, filePrefix string, maxFiles int, maxFileSize int64) (err error)

CheckStoreSize can be used to determine if the store is full

func FindFileIndexes

func FindFileIndexes(directory, filePrefix string) (minIndex, maxIndex uint64, maxDataSize int64, totalFiles int, err error)

func IncrementWriteFile

func IncrementWriteFile(directory, filePrefix string, maxFiles int) (index, data string, err error)

incrementWriteFile can be used to increment the index and data files associated with a store it will pull the current index and generate new index/data filepaths

func MegaByteToByte

func MegaByteToByte(n int64) int64

func PruneDirectory

func PruneDirectory(directory, filePrefix, readIndexFile string) (err error)

pruneDirectory can be used to remove un-used files less than the provided index or all files with the given file prefix if set to -1

func ReadData

func ReadData(data map[uint64]*[]byte, indexes ...*Index) (err error)

readData can be used to read one or more indexes

func ReadIndexes

func ReadIndexes(directory, filePrefix string, indices map[uint64]*Index) (indexRead, indexWrite uint64, err error)

ReadIndexes will take a directory, get all the index files from it and load them into an indices map while ignoring any deleted indexes

func Size

func Size(filepath string) (size int64, err error)

func Sizes

func Sizes(directory, filePrefix, fileSuffix string) (int64, error)

func WriteIndex

func WriteIndex(indexes ...*Index) (err error)

writeIndex can be used to write multiple indexes

Types

type Bytes

type Bytes []byte

Bytes is used to simplify some internal operations

func WriteData

func WriteData(dataFile string, maxFileSize int64, dataIn []Bytes) (starts, finishes []int64, dataOut []Bytes, err error)

writeData can be used to write one or more slices of data to disk

type Files

type Files struct {
	LockFile   string
	IndexWrite string
	DataRead   string
	DataWrite  string
}

Files

func FindFiles

func FindFiles(directory, filePrefix string) (files Files, dataSizeWrite int64, err error)

findFiles will attempt to find the min and max indexes present in the files that aren't deleted and provide the appropriate read/write index and data files

type Index

type Index struct {
	DataFile  string //the data file associated with the index
	IndexFile string //the index file associated with the index
	MinIndex  uint64 //minimum index for the file
	Index     uint64 //the index/key
	Start     int64  //the file start offset
	Finish    int64  //the file finish offset
	Deleted   bool   //whether or not the index is deleted
}

Index is a type used to describe a fixed-size index of a single data entry of a store, although all fields are exported, only Index, Start, Finish and Deleted are actually stored on disk, DataFile, IndexFile, and MinIndex are populated when the index file is read in as a whole

func ReadIndex

func ReadIndex(indexFile string) (indexes []*Index, indexRead, indexWrite uint64, err error)

func (*Index) MarshalBinary

func (i *Index) MarshalBinary() (data []byte, err error)

MarshalBinary will convert the provided index into a byte slice

func (*Index) UnmarshalBinary

func (i *Index) UnmarshalBinary(data []byte) (err error)

UnmarshalBinary will decode a slice of bytes into an index