stodiskaccess

Documentation

Overview

diskaccess ties together DB metadata read/write in addition to writing to disk

Index

• Variables
• type BlobMeta
• type Controller
  • func New(metadataStore MetadataStore) *Controller
  • func (d *Controller) BestVolumeId(volumeIds []int) (int, error)
  • func (d *Controller) Fetch(ref stotypes.BlobRef, encryptionKeys []stotypes.KeyEnvelope, volumeId int) (io.ReadCloser, error)
  • func (d *Controller) Initialize(ctx context.Context, volumeUuid string, driver blobstore.Driver) error
  • func (d *Controller) IsMounted(volumeId int) bool
  • func (d *Controller) Mount(ctx context.Context, volumeId int, expectedVolumeUuid string, ...) error
  • func (d *Controller) Mountable(ctx context.Context, volumeId int, expectedVolumeUuid string, ...) error
  • func (d *Controller) Replicate(ctx context.Context, fromVolumeId int, toVolumeId int, ref stotypes.BlobRef) error
  • func (d *Controller) Scrub(ref stotypes.BlobRef, volumeId int) (int64, error)
  • func (d *Controller) WriteBlob(volumeId int, collId string, ref stotypes.BlobRef, content io.Reader, ...) error
  • func (d *Controller) WriteBlobNoVerify(volumeId int, collId string, ref stotypes.BlobRef, content io.Reader, ...) error
• type MetadataStore

Variables

var (
	ErrVolumeDescriptorNotFound = errors.New("volume descriptor not found")
)

Types

type BlobMeta

type BlobMeta struct {
	Ref             stotypes.BlobRef
	RealSize        int32
	SizeOnDisk      int32 // after optional compression
	IsCompressed    bool
	EncryptionKeyId string
	EncryptionKey   []byte // this is set when read from QueryBlobMetadata(), but not when given to WriteBlobCreated()
	ExpectedCrc32   []byte
}

type Controller

type Controller struct {
	// contains filtered or unexported fields
}

func New

func New(metadataStore MetadataStore) *Controller

func (*Controller) BestVolumeId

func (d *Controller) BestVolumeId(volumeIds []int) (int, error)

currently looks for the first ID mounted on this node, but in the future could use richer heuristics: - is the HDD currently spinning - best latency & bandwidth

func (*Controller) Fetch

func (d *Controller) Fetch(ref stotypes.BlobRef, encryptionKeys []stotypes.KeyEnvelope, volumeId int) (io.ReadCloser, error)

does decrypt(optional_decompress(blobOnDisk)) verifies blob integrity for you!

func (*Controller) Initialize

func (d *Controller) Initialize(ctx context.Context, volumeUuid string, driver blobstore.Driver) error

initializes volume for Varasto use by writing a volume descriptor (see verifyOnDiskVolumeUuid() for rationale)

func (*Controller) IsMounted

func (d *Controller) IsMounted(volumeId int) bool

func (*Controller) Mount

func (d *Controller) Mount(ctx context.Context, volumeId int, expectedVolumeUuid string, driver blobstore.Driver) error

call only during server boot (this is not threadsafe)

func (*Controller) Mountable

func (d *Controller) Mountable(ctx context.Context, volumeId int, expectedVolumeUuid string, driver blobstore.Driver) error

mount command currently wants to check if volume would be mountable without actually mounting it

func (*Controller) Replicate

func (d *Controller) Replicate(ctx context.Context, fromVolumeId int, toVolumeId int, ref stotypes.BlobRef) error

in theory we wouldn't need to do this since we could do a Fetch()-followed by Store(), but we can optimize by just transferring the raw on-disk format

func (*Controller) Scrub

func (d *Controller) Scrub(ref stotypes.BlobRef, volumeId int) (int64, error)

runs a scrub for a blob in a given volume to detect errors https://en.wikipedia.org/wiki/Data_scrubbing we could actually just do a Fetch() but that would require access to the encryption keys. this way we can verify on-disk integrity without encryption keys.

func (*Controller) WriteBlob

func (d *Controller) WriteBlob(
	volumeId int,
	collId string,
	ref stotypes.BlobRef,
	content io.Reader,
	maybeCompressible bool,
) error

func (*Controller) WriteBlobNoVerify

func (d *Controller) WriteBlobNoVerify(
	volumeId int,
	collId string,
	ref stotypes.BlobRef,
	content io.Reader,
	maybeCompressible bool,
) error

type MetadataStore

type MetadataStore interface {
	// returns os.ErrNotExist if ref does not exist
	QueryBlobMetadata(ref stotypes.BlobRef, encryptionKeys []stotypes.KeyEnvelope) (*BlobMeta, error)
	QueryBlobCrc32(ref stotypes.BlobRef) ([]byte, error)
	QueryBlobExists(ref stotypes.BlobRef) (bool, error)
	QueryCollectionEncryptionKeyForNewBlobs(collId string) (string, []byte, error)
	WriteBlobCreated(meta *BlobMeta, volumeId int) error
	WriteBlobReplicated(ref stotypes.BlobRef, volumeId int) error
}