file

Documentation

Overview

Document collection data file.

Common data file features.

A hash table of uint64 key-value pairs.

Index

• Constants
• func ConsecutiveTwenty0s(buf gommap.MMap) bool
• type ColFile
  • func OpenCol(name string) (*ColFile, error)
  • func (col *ColFile) Delete(id uint64)
  • func (col *ColFile) ForAll(fun func(id uint64, doc []byte) bool)
  • func (col *ColFile) Insert(data []byte) (id uint64, err error)
  • func (col *ColFile) Read(id uint64) []byte
  • func (col *ColFile) Update(id uint64, data []byte) (uint64, error)
• type File
  • func Open(name string, growth uint64) (file *File, err error)
  • func (file *File) CheckSize(more uint64) bool
  • func (file *File) CheckSizeAndEnsure(more uint64)
  • func (file *File) Close() (err error)
  • func (file *File) Flush() error
• type HashTable
  • func OpenHash(name string, hashBits, perBucket uint64) (ht *HashTable, err error)
  • func (ht *HashTable) Get(key, limit uint64, filter func(uint64, uint64) bool) (keys, vals []uint64)
  • func (ht *HashTable) GetAll(limit uint64) (keys, vals []uint64)
  • func (ht *HashTable) Put(key, val uint64)
  • func (ht *HashTable) Remove(key, val uint64)

Constants

const (
	COL_FILE_GROWTH      = uint64(134217728) // Grows by 128MB
	DOC_MAX_ROOM         = 33554432          // Maximum size of a single document (initial size is halved)
	DOC_HEADER           = 1 + 10            // Document header: validity (byte), document room (uint64)
	DOC_VALID            = byte(1)           // Document valid flag
	DOC_INVALID          = byte(0)           // Document invalid flag
	COL_FILE_REGION_SIZE = 1024 * 64         // Granulairty of locks

	// Document padding made of spaces (pre-compiled, 2048 bytes)
	PADDING = "                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                "
	LEN_PADDING = uint64(len(PADDING))
)

const (
	HASH_TABLE_GROWTH      = uint64(134217728)   // Grows every 128MB
	ENTRY_VALID            = byte(1)             // Entry valid flag
	ENTRY_INVALID          = byte(0)             // Entry invalid flag
	ENTRY_SIZE             = uint64(1 + 10 + 10) // Entry header: validity (byte), hash key (uint64) and value (uint64)
	BUCKET_HEADER_SIZE     = uint64(10)          // Bucket header: next bucket in chain (uint64)
	HASH_TABLE_REGION_SIZE = 1024 * 16384        // Granularity of locks
)

const FILE_GROWTH_INCREMENTAL = uint64(16777216)

Functions

func ConsecutiveTwenty0s

func ConsecutiveTwenty0s(buf gommap.MMap) bool

Return true if the buffer begins with ten consecutive 0s.

Types

type ColFile

type ColFile struct {
	File *File
	// contains filtered or unexported fields
}

func OpenCol

func OpenCol(name string) (*ColFile, error)

Open a collection file.

func (*ColFile) Delete

func (col *ColFile) Delete(id uint64)

Delete a document.

func (*ColFile) ForAll

func (col *ColFile) ForAll(fun func(id uint64, doc []byte) bool)

Scan the entire data file, look for documents and invoke the function on each.

func (*ColFile) Insert

func (col *ColFile) Insert(data []byte) (id uint64, err error)

Insert a document, return its ID.

func (*ColFile) Read

func (col *ColFile) Read(id uint64) []byte

Retrieve document data given its ID.

func (*ColFile) Update

func (col *ColFile) Update(id uint64, data []byte) (uint64, error)

Update a document, return its new ID.

type File

type File struct {
	Name                   string   // File path and name
	Fh                     *os.File // File handle (in operating system)
	UsedSize, Size, Growth uint64
	Buf                    gommap.MMap // Mapped file buffer
}

func Open

func Open(name string, growth uint64) (file *File, err error)

Open the file, or create it if non-existing.

func (*File) CheckSize

func (file *File) CheckSize(more uint64) bool

Return true only if the file has enough room for more data.

func (*File) CheckSizeAndEnsure

func (file *File) CheckSizeAndEnsure(more uint64)

Ensure that the file has enough room for more data. Grow the file if necessary.

func (*File) Close

func (file *File) Close() (err error)

Close the file.

func (*File) Flush

func (file *File) Flush() error

Synchronize file buffer with underlying storage device.

type HashTable

type HashTable struct {
	File                            *File
	BucketSize, HashBits, PerBucket uint64 // Hash table configuration - size of bucket, number of bits
	NumBuckets, InitialBuckets      uint64 // Total number of buckets, and number of "head" buckets
	// contains filtered or unexported fields
}

func OpenHash

func OpenHash(name string, hashBits, perBucket uint64) (ht *HashTable, err error)

Open a hash table file.

func (*HashTable) Get

func (ht *HashTable) Get(key, limit uint64, filter func(uint64, uint64) bool) (keys, vals []uint64)

Get key-value pairs.

func (*HashTable) GetAll

func (ht *HashTable) GetAll(limit uint64) (keys, vals []uint64)

Return all entries in the hash table.

func (*HashTable) Put

func (ht *HashTable) Put(key, val uint64)

Put a new key-value pair.

func (*HashTable) Remove

func (ht *HashTable) Remove(key, val uint64)

Remove specific key-value pair.