committed

package
v0.47.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jul 28, 2021 License: Apache-2.0 Imports: 15 Imported by: 0

Documentation

Index

Constants

View Source
const (
	MetadataTypeKey        = "type"
	MetadataRangesType     = "ranges"
	MetadataMetarangesType = "metaranges"
)
View Source
const MaxValueComponentBytes = 640 << 16

MaxValueComponentBytes is the longest size allowed for the data length of a graveler value (or its identity, but that is controlled by code here, so less likely). It (only) protects the process from unbounded serialization. "640 KB should be enough for anyone" - even at a few 10s of KiBs you may be better served with some other format or implementation.

Variables

View Source
var (
	ErrUnsortedKeys = errors.New("keys should be written in ascending order")
	ErrNilValue     = errors.New("record value should not be nil")
)
View Source
var ErrBadValueBytes = errors.New("bad bytes format for graveler.Value")

ErrBadValueBytes is an error that is probably returned when unmarshalling bytes that are supposed to encode a Value.

View Source
var (
	ErrMultipleWaitCalls = errors.New("wait has already been called")
)
View Source
var ErrNeedBatchClosers = errors.New("need at least 1 batch uploaded")
View Source
var ErrNoRange = errors.New("diff is not currently in a range")

ErrNoRange occurs when calling nextRange while not in a range, could happen when the diff is currently comparing keys in two different ranges

View Source
var (
	ErrNotFound = errors.New("not found")
)
View Source
var ErrTooLong = errors.New("too long")

ErrTooLong is an error that is returned when trying to marshal too long a key or value. This should never normally happen in graveler files generated by graveler.

View Source
var ErrUnsupportedRangeDiffType = errors.New("range diff type unsupported - supports only added and removed")
View Source
var File_committed_proto protoreflect.FileDescriptor

Functions

func Apply

func Apply(ctx context.Context, writer MetaRangeWriter, source Iterator, diffs Iterator, opts *ApplyOptions) (graveler.DiffSummary, error)

func MarshalRange

func MarshalRange(r Range) ([]byte, error)

func MarshalValue

func MarshalValue(v *graveler.Value) ([]byte, error)

MarshalValue returns bytes that uniquely unmarshal into a Value equal to v.

func MustMarshalValue

func MustMarshalValue(v *graveler.Value) []byte

MustMarshalValue an MarshalValue that will panic on error

func MustUnmarshalValue

func MustUnmarshalValue(b []byte) *graveler.Value

MustUnmarshalValue an UnmarshalValue that will panic on error

func NewCompareIterator

func NewCompareIterator(ctx context.Context, diffDestToSource DiffIterator, base Iterator) *compareIterator

NewCompareIterator accepts an iterator describing a diff from the merge destination to the source. It returns a DiffIterator with the changes to perform on the destination branch, in order to merge the source into it, relative to base as the merge base. When reaching a conflict, the returned Diff will be of type graveler.DiffTypeConflict.

func NewCompareValueIterator added in v0.40.0

func NewCompareValueIterator(ctx context.Context, diffDestToSource DiffIterator, base Iterator) *compareValueIterator

NewCompareValueIterator Wraps CompareIterator in order to return only values

func NewDiffValueIterator added in v0.40.0

func NewDiffValueIterator(ctx context.Context, left Iterator, right Iterator) graveler.DiffIterator

func NewMergeIterator

func NewMergeIterator(ctx context.Context, diffDestToSource DiffIterator, base Iterator) *mergeIterator

NewMergeIterator accepts an iterator describing a diff from the merge destination to the source. It returns an Iterator with the changes to perform on the destination branch, in order to merge the source into it, relative to base as the merge base. When reaching a conflict, the iterator will enter an error state with the graveler.ErrConflictFound error.

func NewValueIterator

func NewValueIterator(it Iterator) graveler.ValueIterator

func UnmarshalIdentity

func UnmarshalIdentity(b []byte) ([]byte, error)

UnmarshalIdentity returns *only* the Identity field encoded by b. It does not even examine any bytes beyond the prefix of b holding Identity.

func UnmarshalValue

func UnmarshalValue(b []byte) (*graveler.Value, error)

Types

type ApplyOptions

type ApplyOptions struct {
	// Set to allow commits that change nothing (otherwise ErrNoChanges)
	AllowEmpty bool
}

type BatchCloser

type BatchCloser struct {
	// contains filtered or unexported fields
}

func NewBatchCloser

func NewBatchCloser(numClosers int) *BatchCloser

NewBatchCloser returns a new BatchCloser

func (*BatchCloser) CloseWriterAsync

func (bc *BatchCloser) CloseWriterAsync(w ResultCloser) error

CloseWriterAsync adds RangeWriter instance for the BatchCloser to handle. Any writes executed to the writer after this call are not guaranteed to succeed. If Wait() has already been called, returns an error.

func (*BatchCloser) Wait

func (bc *BatchCloser) Wait() ([]WriteResult, error)

Wait returns when all Writers finished. Returns a nil results slice and an error if *any* RangeWriter failed to close and upload.

type BatchWriterCloser

type BatchWriterCloser interface {
	// CloseWriterAsync adds MetaRangeWriter instance for the BatchWriterCloser to handle.
	// Any writes executed to the writer after this call are not guaranteed to succeed.
	// If Wait() has already been called, returns an error.
	CloseWriterAsync(ResultCloser) error

	// Wait returns when all Writers finished.
	// Any failure to close a single MetaRangeWriter will return with a nil results slice and an error.
	Wait() ([]WriteResult, error)
}
	// blocks until all writers finished or any writer failed
	res, err := bwc.Wait()
	// handle err, results, etc..
}

type DiffIterator added in v0.40.0

type DiffIterator interface {
	// Next moves to look at the next value in the current Range, or a header for the next Range if the current Range is over and a next range exists.
	Next() bool
	// NextRange skips the current range
	// If the next Range is a "headerless" range it will return the first value, otherwise will return the header
	// calling NextRange from a "headerless" should result with ErrNoRange
	NextRange() bool
	// Value returns a nil ValueRecord and a Range before starting a Range, a Value and that Range when inside a Range, or a value with no range when inside a headerless Range
	Value() (*graveler.Diff, *RangeDiff)
	SeekGE(id graveler.Key)
	Err() error
	Close()
}

DiffIterator iterates over all Range headers and values of a Diff, allowing seeking by entire ranges. DiffIterator might contain ranges without headers for example:

left [min].R1.[max] [min].R3.[max] [min]...............R5..............[max]

------------------------------------------------------------------------------------------------

right [min].R2.[max] [min.....R4....max] [min].R6.[max] [min].R7.[max]

R1 - will return as diff with header R2 - will return as diff with header R3 and R4 - could not return a header because we must enter the ranges in order to get some header values (such as count) R5 and R6 - same as R3 and R4 R7 - in case R5 has no values in the R7 range, R7 would return as a diff with header

func NewDiffIterator

func NewDiffIterator(ctx context.Context, left Iterator, right Iterator) DiffIterator

func NewDiffIteratorWrapper added in v0.40.0

func NewDiffIteratorWrapper(iter graveler.DiffIterator) DiffIterator

type DiffIteratorWrapper added in v0.40.0

type DiffIteratorWrapper struct {
	graveler.DiffIterator
	// contains filtered or unexported fields
}

func (*DiffIteratorWrapper) Err added in v0.40.0

func (d *DiffIteratorWrapper) Err() error

func (*DiffIteratorWrapper) NextRange added in v0.40.0

func (d *DiffIteratorWrapper) NextRange() bool

func (*DiffIteratorWrapper) Value added in v0.40.0

func (d *DiffIteratorWrapper) Value() (*graveler.Diff, *RangeDiff)

type GeneralMetaRangeWriter

type GeneralMetaRangeWriter struct {
	// contains filtered or unexported fields
}

func NewGeneralMetaRangeWriter

func NewGeneralMetaRangeWriter(ctx context.Context, rangeManager, metaRangeManager RangeManager, params *Params, namespace Namespace, md graveler.Metadata) *GeneralMetaRangeWriter

func (*GeneralMetaRangeWriter) Abort

func (w *GeneralMetaRangeWriter) Abort() error

func (*GeneralMetaRangeWriter) Close

func (*GeneralMetaRangeWriter) WriteRange

func (w *GeneralMetaRangeWriter) WriteRange(rng Range) error

func (*GeneralMetaRangeWriter) WriteRecord

func (w *GeneralMetaRangeWriter) WriteRecord(record graveler.ValueRecord) error

WriteRecord writes a record to the current range, decides if should close range

type ID

type ID string

ID is an identifier for a Range

type Iterator

type Iterator interface {
	// Next moves to look at the next value in the current Range, or a header for the next
	// Range if the current Range is over.
	Next() bool
	// NextRange() skips the current Range and continues at the header for the next Range.
	NextRange() bool
	// Value returns a nil ValueRecord and a Range before starting a Range, or a Value and
	// that Range when inside a Range.
	Value() (*graveler.ValueRecord, *Range)
	SeekGE(id graveler.Key)
	Err() error
	Close()
}

Iterator iterates over all Range headers and values of a MetaRange, allowing seeking by entire ranges.

func NewEmptyIterator

func NewEmptyIterator() Iterator

func NewIterator

func NewIterator(ctx context.Context, manager RangeManager, namespace Namespace, rangesIt ValueIterator) Iterator

func NewIteratorWrapper added in v0.40.0

func NewIteratorWrapper(iter graveler.ValueIterator) Iterator

type IteratorWrapper added in v0.40.0

type IteratorWrapper struct {
	graveler.ValueIterator
	// contains filtered or unexported fields
}

func (*IteratorWrapper) Err added in v0.40.0

func (i *IteratorWrapper) Err() error

func (*IteratorWrapper) NextRange added in v0.40.0

func (i *IteratorWrapper) NextRange() bool

func (*IteratorWrapper) Value added in v0.40.0

func (i *IteratorWrapper) Value() (*graveler.ValueRecord, *Range)

type Key

type Key []byte

Key and Value types for to be stored in any Range of the MetaRange

func (Key) Copy

func (k Key) Copy() Key

type MetaRangeManager

type MetaRangeManager interface {
	Exists(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (bool, error)

	// GetValue returns the matching in-range graveler.ValueRecord for key in the
	// MetaRange with id.
	GetValue(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID, key graveler.Key) (*graveler.ValueRecord, error)

	// NewRangeWriter returns a writer that is used for creating new MetaRanges
	NewWriter(ctx context.Context, ns graveler.StorageNamespace, metadata graveler.Metadata) MetaRangeWriter

	// NewMetaRangeIterator returns an Iterator over the MetaRange with id.
	NewMetaRangeIterator(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (Iterator, error)

	// GetMetaRangeURI returns a URI with an object representing metarange ID.  It may
	// return a URI that does not resolve (rather than an error) if ID does not exist.
	GetMetaRangeURI(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (string, error)

	// GetRangeURI returns a URI with an object representing metarange ID.  It may
	// return a URI that does not resolve (rather than an error) if ID does not exist.
	GetRangeURI(ctx context.Context, ns graveler.StorageNamespace, rangeID graveler.RangeID) (string, error)
}

MetaRangeManager is an abstraction for a repository of MetaRanges that exposes operations on them

func NewMetaRangeManager

func NewMetaRangeManager(params Params, metaManager, rangeManager RangeManager) (MetaRangeManager, error)

type MetaRangeWriter

type MetaRangeWriter interface {
	// WriteRecord adds a record to the MetaRange. The key must be greater than any other key that was written
	// (in other words - values must be entered sorted by key order).
	// If the most recent insertion was using WriteRange, the key must be greater than any key in the added ranges.
	WriteRecord(graveler.ValueRecord) error

	// AddRange adds a complete range to the MetaRange at the current insertion point.
	// Added Range must not contain keys smaller than last previously written value.
	WriteRange(Range) error

	// Close finalizes the MetaRange creation. It's invalid to add records after calling this method.
	// During MetaRange writing, ranges are closed asynchronously and copied by tierFS
	// while writing continues. Close waits until closing and copying all ranges.
	Close() (*graveler.MetaRangeID, error)

	Abort() error
}

MetaRangeWriter is an abstraction for creating new MetaRanges

type Namespace

type Namespace string

Namespace is namespace for ID ranges

type Params

type Params struct {
	// MinRangeSizeBytes is the smallest size for splitting a range partition as a result
	// of adding a record.  Smaller ranges are still possible due to re-using an existing
	MinRangeSizeBytes uint64
	// MaxRangeSizeBytes is the largest size of a range partition.  In practice the range
	// is split only after an additional record.
	MaxRangeSizeBytes uint64
	// RangeSizeEntriesRaggedness allows raggedness in splitting range partitions.  It is
	// the expected number of records after MinRangeSizeBytes at which to split the range
	// -- ranges are split at the first key with hash divisible by this raggedness.
	RangeSizeEntriesRaggedness float64
	// MaxUploaders is the maximal number of uploaders to use in a single metarange writer.
	MaxUploaders int
}

type Range

type Range struct {
	ID            ID
	MinKey        Key
	MaxKey        Key
	EstimatedSize uint64 // EstimatedSize estimated Range size in bytes
	Count         int64
	Tombstone     bool
}

func UnmarshalRange

func UnmarshalRange(b []byte) (Range, error)

func (Range) Copy added in v0.40.0

func (r Range) Copy() *Range

type RangeData

type RangeData struct {

	// First key of range, must be >= key of range in SSTable.  If missing, == key of range in SSTable.
	MinKey        []byte `protobuf:"bytes,1,opt,name=min_key,json=minKey,proto3" json:"min_key,omitempty"`
	MaxKey        []byte `protobuf:"bytes,2,opt,name=max_key,json=maxKey,proto3" json:"max_key,omitempty"`
	EstimatedSize uint64 `protobuf:"varint,3,opt,name=estimated_size,json=estimatedSize,proto3" json:"estimated_size,omitempty"`
	Count         int64  `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"`
	// contains filtered or unexported fields
}

func (*RangeData) Descriptor deprecated

func (*RangeData) Descriptor() ([]byte, []int)

Deprecated: Use RangeData.ProtoReflect.Descriptor instead.

func (*RangeData) GetCount

func (x *RangeData) GetCount() int64

func (*RangeData) GetEstimatedSize

func (x *RangeData) GetEstimatedSize() uint64

func (*RangeData) GetMaxKey

func (x *RangeData) GetMaxKey() []byte

func (*RangeData) GetMinKey

func (x *RangeData) GetMinKey() []byte

func (*RangeData) ProtoMessage

func (*RangeData) ProtoMessage()

func (*RangeData) ProtoReflect

func (x *RangeData) ProtoReflect() protoreflect.Message

func (*RangeData) Reset

func (x *RangeData) Reset()

func (*RangeData) String

func (x *RangeData) String() string

type RangeDiff added in v0.40.0

type RangeDiff struct {
	Type  graveler.DiffType
	Range *Range
}

RangeDiff represents a change in Range

func (RangeDiff) Copy added in v0.40.0

func (r RangeDiff) Copy() *RangeDiff

type RangeManager

type RangeManager interface {
	// Exists returns true if id references a Range.
	Exists(ctx context.Context, ns Namespace, id ID) (bool, error)

	// GetValue returns the value matching key in the Range referenced by id.  If id not
	// found, it return (nil, ErrNotFound).
	GetValue(ctx context.Context, ns Namespace, id ID, key Key) (*Record, error)

	// GetValueGE returns the first value keyed at or after key in the Range referenced by
	// id.  If all values are keyed before key, it returns (nil, ErrNotFound).
	GetValueGE(ctx context.Context, ns Namespace, id ID, key Key) (*Record, error)

	// NewRangeIterator returns an iterator over values in the Range with ID.
	NewRangeIterator(ctx context.Context, ns Namespace, pid ID) (ValueIterator, error)

	// GetWriter returns a new Range writer instance
	GetWriter(ctx context.Context, ns Namespace, metadata graveler.Metadata) (RangeWriter, error)

	// GetURI returns a URI from which to read the contents of id.  If id does not exist
	// it may return a URI that resolves nowhere rather than an error.
	GetURI(ctx context.Context, ns Namespace, id ID) (string, error)
}

type RangeWriter

type RangeWriter interface {
	// WriteRecord appends the given record to the Range.
	WriteRecord(record Record) error

	// AddMetadata associates metadata value (which will be stringified when the writer is
	// Closed and added to the resulting range ID) with key.
	SetMetadata(key, value string)

	// GetApproximateSize returns an estimate of the current written size of the Range.
	GetApproximateSize() uint64

	// Close flushes all records to the disk and returns the WriteResult.
	Close() (*WriteResult, error)

	// Abort terminates the non-closed file and removes all traces.
	Abort() error
}

RangeWriter is an abstraction for writing Ranges. Written records must be sorted by key.

type Record

type Record struct {
	Key   Key
	Value Value
}

type ResultCloser

type ResultCloser interface {
	Close() (*WriteResult, error)
}

type UnmarshalIterator

type UnmarshalIterator struct {
	// contains filtered or unexported fields
}

UnmarshalIterator wrap value iterator and unmarshal each value

func NewUnmarshalIterator

func NewUnmarshalIterator(it ValueIterator) *UnmarshalIterator

func (*UnmarshalIterator) Close

func (r *UnmarshalIterator) Close()

func (*UnmarshalIterator) Err

func (r *UnmarshalIterator) Err() error

func (*UnmarshalIterator) Next

func (r *UnmarshalIterator) Next() bool

func (*UnmarshalIterator) SeekGE

func (r *UnmarshalIterator) SeekGE(id graveler.Key)

func (*UnmarshalIterator) Value

type Value

type Value []byte

type ValueIterator

type ValueIterator interface {
	Next() bool
	SeekGE(id Key)
	Value() *Record
	Err() error
	Close()
}

type WriteResult

type WriteResult struct {
	// ID is the identifier for the written Range.
	// Calculated by an hash function to all keys and values' identity.
	RangeID ID

	// First is the first key in the Range.
	First Key

	// Last is the last key in the Range.
	Last Key

	// Count is the number of records in the Range.
	Count int

	// EstimatedRangeSizeBytes is Approximate size of each Range
	EstimatedRangeSizeBytes uint64
}

WriteResult is the result of a completed write of a Range

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL