chunkfile

Documentation

Overview

Document collection file.

A static hash table made of uint64 key-value pairs.

Index

• Constants
• type ColFile
  • func OpenCol(name string) (*ColFile, error)
  • func (col *ColFile) Delete(id uint64)
  • func (col *ColFile) ForAll(fun func(id uint64, doc []byte) bool)
  • func (col *ColFile) Insert(data []byte) (id uint64, err error)
  • func (col *ColFile) Read(id uint64) []byte
  • func (col *ColFile) Update(id uint64, data []byte) (newID uint64, err error)
• type HashTable
  • func OpenHash(name string, path []string) (ht *HashTable, err error)
  • func (ht *HashTable) Clear()
  • func (ht *HashTable) Get(key, limit uint64) (keys, vals []uint64)
  • func (ht *HashTable) GetAll(limit uint64) (keys, vals []uint64)
  • func (ht *HashTable) HashKey(key uint64) uint64
  • func (ht *HashTable) NextBucket(bucket uint64) uint64
  • func (ht *HashTable) Put(key, val uint64)
  • func (ht *HashTable) Remove(key, val uint64)

Constants

const (
	COL_FILE_SIZE   = uint64(512 * 1024 * 1) // Size of collection data file
	DOC_MAX_ROOM    = uint64(512 * 1024 * 1) // Max single document size
	DOC_HEADER_SIZE = 1 + 10                 // Size of document header - validity (byte), document room (uint64)
	DOC_VALID       = byte(1)                // Document valid flag
	DOC_INVALID     = byte(0)                // Document invalid flag

	// Pre-compiled document padding (2048 spaces)
	PADDING = "                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                " +
		"                                                                                                                                "
	LEN_PADDING = uint64(len(PADDING))
)

const (
	HT_FILE_SIZE       = uint64(1024 * 1024 * 4) // Size of a hash table, it may grow by this size
	ENTRY_VALID        = byte(1)                 // Entry valid flag
	ENTRY_INVALID      = byte(0)                 // Entry invalid flag
	ENTRY_SIZE         = uint64(1 + 10 + 10)     // Size of entry header - validity (byte), hash key (uint64) and value (uint64)
	BUCKET_HEADER_SIZE = uint64(10)              // Size of bucket header - next bucket in chain (uint64)
	// Hash table configuration
	PER_BUCKET  = uint64(15)
	BUCKET_SIZE = uint64(PER_BUCKET*ENTRY_SIZE + BUCKET_HEADER_SIZE)
	HASH_BITS   = uint64(14)
	// INITIAL_BUCKETS = 2 to the power of HASH_BITS
	INITIAL_BUCKETS = uint64(16384)
)

Types

type ColFile

type ColFile struct {
	File *commonfile.File
}

func OpenCol

func OpenCol(name string) (*ColFile, error)

Open a collection file.

func (*ColFile) Delete

func (col *ColFile) Delete(id uint64)

Delete a document.

func (*ColFile) ForAll

func (col *ColFile) ForAll(fun func(id uint64, doc []byte) bool)

Scan the entire data file, look for documents and invoke the function on each.

func (*ColFile) Insert

func (col *ColFile) Insert(data []byte) (id uint64, err error)

Insert a document, return its ID.

func (*ColFile) Read

func (col *ColFile) Read(id uint64) []byte

Retrieve document data given its ID.

func (*ColFile) Update

func (col *ColFile) Update(id uint64, data []byte) (newID uint64, err error)

Update a document, return its new ID.

type HashTable

type HashTable struct {
	Path       []string
	File       *commonfile.File
	Mutex      *sync.RWMutex
	NumBuckets uint64 // Total number of buckets
}

func OpenHash

func OpenHash(name string, path []string) (ht *HashTable, err error)

Open a hash table file.

func (*HashTable) Clear

func (ht *HashTable) Clear()

Clear all index entries, return to the initial size as well.

func (*HashTable) Get

func (ht *HashTable) Get(key, limit uint64) (keys, vals []uint64)

Get key-value pairs.

func (*HashTable) GetAll

func (ht *HashTable) GetAll(limit uint64) (keys, vals []uint64)

Return all entries in the hash table.

func (*HashTable) HashKey

func (ht *HashTable) HashKey(key uint64) uint64

Return a hash key to be used by hash table by masking non-key bits.

func (*HashTable) NextBucket

func (ht *HashTable) NextBucket(bucket uint64) uint64

Return the number (not address) of next chained bucket, 0 if there is not any.

func (*HashTable) Put

func (ht *HashTable) Put(key, val uint64)

Put a new key-value pair.

func (*HashTable) Remove

func (ht *HashTable) Remove(key, val uint64)

Remove specific key-value pair.