index

Documentation

Index

• Variables
• type Index
  • func (i *Index) GobDecode(data []byte) error
  • func (i *Index) GobEncode() ([]byte, error)
• type IndexManager
  • func NewIndexManager(walPath, indexPath, queueDirPath string, useCommit bool) (*IndexManager, error)
  • func (im *IndexManager) Add(host string, id string, position uint64, size uint64) (commit uint64, err error)
  • func (im *IndexManager) AwaitWALCommitted(commit uint64)
  • func (im *IndexManager) Close() error
  • func (im *IndexManager) GetHosts() []string
  • func (im *IndexManager) GetStats() string
  • func (im *IndexManager) IsEmpty() bool
  • func (im *IndexManager) IsWALCommitted(commit uint64) bool
  • func (im *IndexManager) Pop(host string) (commit uint64, id string, position uint64, size uint64, err error)
  • func (im *IndexManager) RecoverFromCrash() error
  • func (im *IndexManager) WALCommit() uint64
• type Operation
• type WALEntry

Variables

var (
	// ErrHostNotFound is returned when the host is not found in the index
	ErrHostNotFound = errors.New("host not found")

	// ErrHostEmpty is returned when the host is empty
	ErrHostEmpty = errors.New("host cannot be empty")

	// ErrIDEmpty is returned when the given ID is empty
	ErrIDEmpty = errors.New("id cannot be empty")

	// ErrNoWALEntriesReplayed is returned when no WAL entries were replayed
	ErrNoWALEntriesReplayed = errors.New("no WAL entries replayed")
)

Types

type Index

type Index struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func (*Index) GobDecode

func (i *Index) GobDecode(data []byte) error

func (*Index) GobEncode

func (i *Index) GobEncode() ([]byte, error)

type IndexManager

type IndexManager struct {
	sync.Mutex

	WalWait time.Duration // interval **between** between after-sync and next sync
	// contains filtered or unexported fields
}

func NewIndexManager

func NewIndexManager(walPath, indexPath, queueDirPath string, useCommit bool) (*IndexManager, error)

NewIndexManager creates a new IndexManager instance and loads the index from the index file.

func (*IndexManager) Add

func (im *IndexManager) Add(host string, id string, position uint64, size uint64) (commit uint64, err error)

func (*IndexManager) AwaitWALCommitted

func (im *IndexManager) AwaitWALCommitted(commit uint64)

AwaitWALCommitted blocks until the given commit ID is committed to disk by Syncer. DO NOT call this function with im.Lock() held, it will deadlock.

func (*IndexManager) Close

func (im *IndexManager) Close() error

Close closes the index manager and performs a final dump of the index to disk.

func (*IndexManager) GetHosts

func (im *IndexManager) GetHosts() []string

GetHosts returns a list of all hosts in the index

func (*IndexManager) GetStats

func (im *IndexManager) GetStats() string

func (*IndexManager) IsEmpty

func (im *IndexManager) IsEmpty() bool

func (*IndexManager) IsWALCommitted

func (im *IndexManager) IsWALCommitted(commit uint64) bool

func (*IndexManager) Pop

func (im *IndexManager) Pop(host string) (commit uint64, id string, position uint64, size uint64, err error)

Pop removes the oldest blob from the specified host's queue and returns its ID, position, and size. Pop is responsible for synchronizing the pop of the blob from the in-memory index and writing to the WAL. First it starts a goroutine that waits for the to-be-popped blob infos through blobChan, then writes to the WAL and if successful informs index.pop() through WALSuccessChan to either continue as normal or return an error.

func (*IndexManager) RecoverFromCrash

func (im *IndexManager) RecoverFromCrash() error

func (*IndexManager) WALCommit

func (im *IndexManager) WALCommit() uint64

increments the WAL commit counter and returns the new commit ID.

type Operation

type Operation int

const (
	OpAdd Operation = iota
	OpPop
)

type WALEntry

type WALEntry struct {
	Op       Operation
	Host     string
	BlobID   string
	Position uint64
	Size     uint64
}