block

package
v0.3.0-pre9 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: May 31, 2019 License: Apache-2.0 Imports: 37 Imported by: 0

Documentation

Overview

Package block implements repository support content-addressable storage blocks.

Index

Constants

View Source
const DefaultEncryption = "SALSA20"

DefaultEncryption is the name of the default encryption algorithm.

View Source
const DefaultHash = "BLAKE2B-256-128"

DefaultHash is the name of the default hash algorithm.

View Source
const PackBlockPrefix = "p"

PackBlockPrefix is the prefix for all pack storage blocks.

Variables

This section is empty.

Functions

func CreateHashAndEncryptor

func CreateHashAndEncryptor(f FormattingOptions) (HashFunc, Encryptor, error)

func RegisterEncryption

func RegisterEncryption(name string, newEncryptor EncryptorFactory)

RegisterEncryption registers new encryption algorithm.

func RegisterHash

func RegisterHash(name string, newHashFunc HashFuncFactory)

RegisterHash registers a hash function with a given name.

func SupportedEncryptionAlgorithms

func SupportedEncryptionAlgorithms() []string

func SupportedHashAlgorithms

func SupportedHashAlgorithms() []string

func UsingBlockCache

func UsingBlockCache(ctx context.Context, enabled bool) context.Context

UsingBlockCache returns a derived context that causes block manager to use cache.

func UsingListCache

func UsingListCache(ctx context.Context, enabled bool) context.Context

UsingListCache returns a derived context that causes block manager to use cache.

Types

type CachingOptions

type CachingOptions struct {
	CacheDirectory          string `json:"cacheDirectory,omitempty"`
	MaxCacheSizeBytes       int64  `json:"maxCacheSize,omitempty"`
	MaxListCacheDurationSec int    `json:"maxListCacheDuration,omitempty"`
	IgnoreListCache         bool   `json:"-"`
	HMACSecret              []byte `json:"-"`
}

CachingOptions specifies configuration of local cache.

type CompactOptions

type CompactOptions struct {
	MinSmallBlocks       int
	MaxSmallBlocks       int
	AllBlocks            bool
	SkipDeletedOlderThan time.Duration
}

CompactOptions provides options for compaction

type Encryptor

type Encryptor interface {
	// Encrypt returns encrypted bytes corresponding to the given plaintext. Must not clobber the input slice.
	Encrypt(plainText []byte, blockID []byte) ([]byte, error)

	// Decrypt returns unencrypted bytes corresponding to the given ciphertext. Must not clobber the input slice.
	Decrypt(cipherText []byte, blockID []byte) ([]byte, error)
}

Encryptor performs encryption and decryption of blocks of data.

type EncryptorFactory

type EncryptorFactory func(o FormattingOptions) (Encryptor, error)

EncryptorFactory creates new Encryptor for given FormattingOptions

type Format

type Format struct {
	Version    byte   // format version number must be 0x01
	KeySize    byte   // size of each key in bytes
	EntrySize  uint16 // size of each entry in bytes, big-endian
	EntryCount uint32 // number of sorted (key,value) entries that follow

	Entries []struct {
		Key   []byte // key bytes (KeySize)
		Entry entry
	}

	ExtraData []byte // extra data
}

Format describes a format of a single pack index. The actual structure is not used, it's purely for documentation purposes. The struct is byte-aligned.

type FormattingOptions

type FormattingOptions struct {
	Version     int    `json:"version,omitempty"`     // version number, must be "1"
	Hash        string `json:"hash,omitempty"`        // identifier of the hash algorithm used
	Encryption  string `json:"encryption,omitempty"`  // identifier of the encryption algorithm used
	HMACSecret  []byte `json:"secret,omitempty"`      // HMAC secret used to generate encryption keys
	MasterKey   []byte `json:"masterKey,omitempty"`   // master encryption key (SIV-mode encryption only)
	MaxPackSize int    `json:"maxPackSize,omitempty"` // maximum size of a pack object
}

FormattingOptions describes the rules for formatting blocks in repository.

type HashFunc

type HashFunc func(data []byte) []byte

HashFunc computes hash of block of data using a cryptographic hash function, possibly with HMAC and/or truncation.

type HashFuncFactory

type HashFuncFactory func(o FormattingOptions) (HashFunc, error)

HashFuncFactory returns a hash function for given formatting options.

type IndexInfo

type IndexInfo struct {
	FileName  string
	Length    int64
	Timestamp time.Time
}

IndexInfo is an information about a single index block managed by Manager.

type Info

type Info struct {
	BlockID          string `json:"blockID"`
	Length           uint32 `json:"length"`
	TimestampSeconds int64  `json:"time"`
	PackFile         string `json:"packFile,omitempty"`
	PackOffset       uint32 `json:"packOffset,omitempty"`
	Deleted          bool   `json:"deleted"`
	Payload          []byte `json:"payload"` // set for payloads stored inline
	FormatVersion    byte   `json:"formatVersion"`
}

Info is an information about a single block managed by Manager.

func (Info) Timestamp

func (i Info) Timestamp() time.Time

Timestamp returns the time when a block was created or deleted.

type Manager

type Manager struct {
	Format FormattingOptions
	// contains filtered or unexported fields
}

Manager manages storage blocks at a low level with encryption, deduplication and packaging.

func NewManager

func NewManager(ctx context.Context, st storage.Storage, f FormattingOptions, caching CachingOptions, repositoryFormatBytes []byte) (*Manager, error)

NewManager creates new block manager with given packing options and a formatter.

func (*Manager) BlockInfo

func (bm *Manager) BlockInfo(ctx context.Context, blockID string) (Info, error)

BlockInfo returns information about a single block.

func (*Manager) Close

func (bm *Manager) Close()

Close closes the block manager.

func (*Manager) CompactIndexes

func (bm *Manager) CompactIndexes(ctx context.Context, opt CompactOptions) error

CompactIndexes performs compaction of index blocks ensuring that # of small blocks is between minSmallBlockCount and maxSmallBlockCount

func (*Manager) DeleteBlock

func (bm *Manager) DeleteBlock(blockID string) error

DeleteBlock marks the given blockID as deleted.

NOTE: To avoid race conditions only blocks that cannot be possibly re-created should ever be deleted. That means that contents of such blocks should include some element of randomness or a contemporaneous timestamp that will never reappear.

func (*Manager) DisableIndexFlush

func (bm *Manager) DisableIndexFlush()

DisableIndexFlush increments the counter preventing automatic index flushes.

func (*Manager) EnableIndexFlush

func (bm *Manager) EnableIndexFlush()

EnableIndexFlush decrements the counter preventing automatic index flushes. The flushes will be reenabled when the index drops to zero.

func (*Manager) FindUnreferencedStorageFiles

func (bm *Manager) FindUnreferencedStorageFiles(ctx context.Context) ([]storage.BlockMetadata, error)

FindUnreferencedStorageFiles returns the list of unreferenced storage blocks.

func (*Manager) Flush

func (bm *Manager) Flush(ctx context.Context) error

Flush completes writing any pending packs and writes pack indexes to the underlyign storage.

func (*Manager) GetBlock

func (bm *Manager) GetBlock(ctx context.Context, blockID string) ([]byte, error)

GetBlock gets the contents of a given block. If the block is not found returns blob.ErrBlockNotFound.

func (*Manager) IndexBlocks

func (bm *Manager) IndexBlocks(ctx context.Context) ([]IndexInfo, error)

IndexBlocks returns the list of active index blocks.

func (*Manager) ListBlockInfos

func (bm *Manager) ListBlockInfos(prefix string, includeDeleted bool) ([]Info, error)

ListBlockInfos returns the metadata about blocks with a given prefix and kind.

func (*Manager) ListBlocks

func (bm *Manager) ListBlocks(prefix string) ([]string, error)

ListBlocks returns IDs of blocks matching given prefix.

func (*Manager) RecoverIndexFromPackFile

func (bm *Manager) RecoverIndexFromPackFile(ctx context.Context, packFile string, packFileLength int64, commit bool) ([]Info, error)

RecoverIndexFromPackFile attempts to recover index block entries from a given pack file. Pack file length may be provided (if known) to reduce the number of bytes that are read from the storage.

func (*Manager) Refresh

func (bm *Manager) Refresh(ctx context.Context) (bool, error)

Refresh reloads the committed block indexes.

func (*Manager) ResetStats

func (bm *Manager) ResetStats()

ResetStats resets statistics to zero values.

func (*Manager) RewriteBlock

func (bm *Manager) RewriteBlock(ctx context.Context, blockID string) error

RewriteBlock causes reads and re-writes a given block using the most recent format.

func (*Manager) Stats

func (bm *Manager) Stats() Stats

Stats returns statistics about block manager operations.

func (*Manager) WriteBlock

func (bm *Manager) WriteBlock(ctx context.Context, data []byte, prefix string) (string, error)

WriteBlock saves a given block of data to a pack group with a provided name and returns a blockID that's based on the contents of data written.

type Stats

type Stats struct {
	// Keep int64 fields first to ensure they get aligned to at least 64-bit boundaries
	// which is required for atomic access on ARM and x86-32.
	ReadBytes      int64 `json:"readBytes,omitempty"`
	WrittenBytes   int64 `json:"writtenBytes,omitempty"`
	DecryptedBytes int64 `json:"decryptedBytes,omitempty"`
	EncryptedBytes int64 `json:"encryptedBytes,omitempty"`
	HashedBytes    int64 `json:"hashedBytes,omitempty"`

	ReadBlocks    int32 `json:"readBlocks,omitempty"`
	WrittenBlocks int32 `json:"writtenBlocks,omitempty"`
	CheckedBlocks int32 `json:"checkedBlocks,omitempty"`
	HashedBlocks  int32 `json:"hashedBlocks,omitempty"`
	InvalidBlocks int32 `json:"invalidBlocks,omitempty"`
	PresentBlocks int32 `json:"presentBlocks,omitempty"`
	ValidBlocks   int32 `json:"validBlocks,omitempty"`
}

Stats exposes statistics about block operation.

func (*Stats) Reset

func (s *Stats) Reset()

Reset clears all repository statistics.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL