committed

package
v0.33.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 17, 2021 License: Apache-2.0 Imports: 15 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	MetadataTypeKey        = "type"
	MetadataRangesType     = "ranges"
	MetadataMetarangesType = "metaranges"
)

Variables

View Source 
var (
	ErrUnsortedKeys = errors.New("keys should be written in ascending order")
	ErrNilValue     = errors.New("record value should not be nil")
)
View Source 
var ErrBadValueBytes = errors.New("bad bytes format for graveler.Value")

ErrBadValueBytes is an error that is probably returned when unmarshalling bytes that are supposed to encode a Value.

View Source 
var (
	ErrMultipleWaitCalls = errors.New("wait has already been called")
)
View Source 
var ErrNeedBatchClosers = errors.New("need at least 1 batch uploaded")
View Source 
var (
	ErrNotFound = errors.New("not found")
)
View Source 
var File_committed_proto protoreflect.FileDescriptor

Functions

func Apply 

func Apply(ctx context.Context, writer MetaRangeWriter, source Iterator, diffs graveler.ValueIterator, opts *ApplyOptions) (graveler.DiffSummary, error)

func MarshalRange 

func MarshalRange(r Range) ([]byte, error)

func MarshalValue 

func MarshalValue(v *graveler.Value) ([]byte, error)

MarshalValue returns bytes that uniquely unmarshal into a Value equal to v.

func MustMarshalValue 

func MustMarshalValue(v *graveler.Value) []byte

MustMarshalValue an MarshalValue that will panic on error

func MustUnmarshalValue 

func MustUnmarshalValue(b []byte) *graveler.Value

MustUnmarshalValue an UnmarshalValue that will panic on error

func NewCommittedManager 

func NewCommittedManager(m MetaRangeManager) graveler.CommittedManager

func NewCompareIterator 

func NewCompareIterator(ctx context.Context, diffDestToSource graveler.DiffIterator, base Iterator) *compareIterator

NewCompareIterator accepts an iterator describing a diff from the merge destination to the source. It returns a graveler.DiffIterator with the changes to perform on the destination branch, in order to merge the source into it, relative to base as the merge base. When reaching a conflict, the returned Diff will be of type graveler.DiffTypeConflict.

func NewDiffIterator 

func NewDiffIterator(ctx context.Context, left Iterator, right Iterator) graveler.DiffIterator

func NewMergeIterator 

func NewMergeIterator(ctx context.Context, diffDestToSource graveler.DiffIterator, base Iterator) *compareValueIterator

NewMergeIterator accepts an iterator describing a diff from the merge destination to the source. It returns a graveler.ValueIterator with the changes to perform on the destination branch, in order to merge the source into it, relative to base as the merge base. When reaching a conflict, the iterator will enter an error state with the graveler.ErrConflictFound error.

func NewValueIterator 

func NewValueIterator(it Iterator) graveler.ValueIterator

func UnmarshalIdentity 

func UnmarshalIdentity(b []byte) ([]byte, error)

UnmarshalIdentity returns *only* the Identity field encoded by b. It does not even examine any bytes beyond the prefix of b holding Identity.

func UnmarshalValue 

func UnmarshalValue(b []byte) (*graveler.Value, error)

Types

type ApplyOptions 

type ApplyOptions struct {
	// Set to allow commits that change nothing (otherwise ErrNoChanges)
	AllowEmpty bool
}

type BatchCloser 

type BatchCloser struct {
	// contains filtered or unexported fields
}

func NewBatchCloser 

func NewBatchCloser(numClosers int) *BatchCloser

NewBatchCloser returns a new BatchCloser

func (*BatchCloser) CloseWriterAsync 

func (bc *BatchCloser) CloseWriterAsync(w ResultCloser) error

CloseWriterAsync adds RangeWriter instance for the BatchCloser to handle. Any writes executed to the writer after this call are not guaranteed to succeed. If Wait() has already been called, returns an error.

func (*BatchCloser) Wait 

func (bc *BatchCloser) Wait() ([]WriteResult, error)

Wait returns when all Writers finished. Returns a nil results slice and an error if *any* RangeWriter failed to close and upload.

type BatchWriterCloser 

type BatchWriterCloser interface {
	// CloseWriterAsync adds MetaRangeWriter instance for the BatchWriterCloser to handle.
	// Any writes executed to the writer after this call are not guaranteed to succeed.
	// If Wait() has already been called, returns an error.
	CloseWriterAsync(ResultCloser) error

	// Wait returns when all Writers finished.
	// Any failure to close a single MetaRangeWriter will return with a nil results slice and an error.
	Wait() ([]WriteResult, error)
}
	// blocks until all writers finished or any writer failed
	res, err := bwc.Wait()
	// handle err, results, etc..
}

type GeneralMetaRangeWriter 

type GeneralMetaRangeWriter struct {
	// contains filtered or unexported fields
}

func NewGeneralMetaRangeWriter 

func NewGeneralMetaRangeWriter(ctx context.Context, rangeManager, metaRangeManager RangeManager, params *Params, namespace Namespace, md graveler.Metadata) *GeneralMetaRangeWriter

func (*GeneralMetaRangeWriter) Abort 

func (w *GeneralMetaRangeWriter) Abort() error

func (*GeneralMetaRangeWriter) Close 

func (w *GeneralMetaRangeWriter) Close() (*graveler.MetaRangeID, error)

func (*GeneralMetaRangeWriter) WriteRange 

func (w *GeneralMetaRangeWriter) WriteRange(rng Range) error

func (*GeneralMetaRangeWriter) WriteRecord 

func (w *GeneralMetaRangeWriter) WriteRecord(record graveler.ValueRecord) error

WriteRecord writes a record to the current range, decides if should close range

type ID 

type ID string

ID is an identifier for a Range

type Iterator 

type Iterator interface {
	// Next moves to look at the next value in the current Range, or a header for the next
	// Range if the current Range is over.
	Next() bool
	// NextRange() skips the current Range and continues at the header for the next Range.
	NextRange() bool
	// Value returns a nil ValueRecord and a Range before starting a Range, or a Value and
	// that Range when inside a Range.
	Value() (*graveler.ValueRecord, *Range)
	SeekGE(id graveler.Key)
	Err() error
	Close()
}

Iterator iterates over all Range headers and values of a MetaRange, allowing seeking by entire ranges.

func NewEmptyIterator 

func NewEmptyIterator() Iterator

func NewIterator 

func NewIterator(ctx context.Context, manager RangeManager, namespace Namespace, rangesIt ValueIterator) Iterator

type Key 

type Key []byte

Key and Value types for to be stored in any Range of the MetaRange

func (Key) Copy 

func (k Key) Copy() Key

type MetaRangeManager 

type MetaRangeManager interface {
	Exists(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (bool, error)

	// GetValue returns the matching in-range graveler.ValueRecord for key in the
	// MetaRange with id.
	GetValue(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID, key graveler.Key) (*graveler.ValueRecord, error)

	// NewRangeWriter returns a writer that is used for creating new MetaRanges
	NewWriter(ctx context.Context, ns graveler.StorageNamespace, metadata graveler.Metadata) MetaRangeWriter

	// NewMetaRangeIterator returns an Iterator over the MetaRange with id.
	NewMetaRangeIterator(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (Iterator, error)

	// GetMetaRangeURI returns a URI with an object representing metarange ID.  It may
	// return a URI that does not resolve (rather than an error) if ID does not exist.
	GetMetaRangeURI(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (string, error)

	// GetRangeURI returns a URI with an object representing metarange ID.  It may
	// return a URI that does not resolve (rather than an error) if ID does not exist.
	GetRangeURI(ctx context.Context, ns graveler.StorageNamespace, rangeID graveler.RangeID) (string, error)
}

MetaRangeManager is an abstraction for a repository of MetaRanges that exposes operations on them

func NewMetaRangeManager 

func NewMetaRangeManager(params Params, metaManager, rangeManager RangeManager) (MetaRangeManager, error)

type MetaRangeWriter 

type MetaRangeWriter interface {
	// WriteRecord adds a record to the MetaRange. The key must be greater than any other key that was written
	// (in other words - values must be entered sorted by key order).
	// If the most recent insertion was using WriteRange, the key must be greater than any key in the added ranges.
	WriteRecord(graveler.ValueRecord) error

	// AddRange adds a complete range to the MetaRange at the current insertion point.
	// Added Range must not contain keys smaller than last previously written value.
	WriteRange(Range) error

	// Close finalizes the MetaRange creation. It's invalid to add records after calling this method.
	// During MetaRange writing, ranges are closed asynchronously and copied by tierFS
	// while writing continues. Close waits until closing and copying all ranges.
	Close() (*graveler.MetaRangeID, error)

	Abort() error
}

MetaRangeWriter is an abstraction for creating new MetaRanges

type Namespace 

type Namespace string

Namespace is namespace for ID ranges

type Params 

type Params struct {
	// MinRangeSizeBytes is the smallest size for splitting a range partition as a result
	// of adding a record.  Smaller ranges are still possible due to re-using an existing
	MinRangeSizeBytes uint64
	// MaxRangeSizeBytes is the largest size of a range partition.  In practice the range
	// is split only after an additional record.
	MaxRangeSizeBytes uint64
	// RangeSizeEntriesRaggedness allows raggedness in splitting range partitions.  It is
	// the expected number of records after MinRangeSizeBytes at which to split the range
	// -- ranges are split at the first key with hash divisible by this raggedness.
	RangeSizeEntriesRaggedness float64
	// MaxUploaders is the maximal number of uploaders to use in a single metarange writer.
	MaxUploaders int
}

type Range 

type Range struct {
	ID            ID
	MinKey        Key
	MaxKey        Key
	EstimatedSize uint64 // EstimatedSize estimated Range size in bytes
	Count         int64
}

func UnmarshalRange 

func UnmarshalRange(b []byte) (Range, error)

type RangeData 

type RangeData struct {

	// First key of range, must be >= key of range in SSTable.  If missing, == key of range in SSTable.
	MinKey        []byte `protobuf:"bytes,1,opt,name=min_key,json=minKey,proto3" json:"min_key,omitempty"`
	MaxKey        []byte `protobuf:"bytes,2,opt,name=max_key,json=maxKey,proto3" json:"max_key,omitempty"`
	EstimatedSize uint64 `protobuf:"varint,3,opt,name=estimated_size,json=estimatedSize,proto3" json:"estimated_size,omitempty"`
	Count         int64  `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"`
	// contains filtered or unexported fields
}

func (*RangeData) Descriptor deprecated 

func (*RangeData) Descriptor() ([]byte, []int)

Deprecated: Use RangeData.ProtoReflect.Descriptor instead.

func (*RangeData) GetCount 

func (x *RangeData) GetCount() int64

func (*RangeData) GetEstimatedSize 

func (x *RangeData) GetEstimatedSize() uint64

func (*RangeData) GetMaxKey 

func (x *RangeData) GetMaxKey() []byte

func (*RangeData) GetMinKey 

func (x *RangeData) GetMinKey() []byte

func (*RangeData) ProtoMessage 

func (*RangeData) ProtoMessage()

func (*RangeData) ProtoReflect 

func (x *RangeData) ProtoReflect() protoreflect.Message

func (*RangeData) Reset 

func (x *RangeData) Reset()

func (*RangeData) String 

func (x *RangeData) String() string

type RangeManager 

type RangeManager interface {
	// Exists returns true if id references a Range.
	Exists(ctx context.Context, ns Namespace, id ID) (bool, error)

	// GetValue returns the value matching key in the Range referenced by id.  If id not
	// found, it return (nil, ErrNotFound).
	GetValue(ctx context.Context, ns Namespace, id ID, key Key) (*Record, error)

	// GetValueGE returns the first value keyed at or after key in the Range referenced by
	// id.  If all values are keyed before key, it returns (nil, ErrNotFound).
	GetValueGE(ctx context.Context, ns Namespace, id ID, key Key) (*Record, error)

	// NewRangeIterator returns an iterator over values in the Range with ID.
	NewRangeIterator(ctx context.Context, ns Namespace, pid ID) (ValueIterator, error)

	// GetWriter returns a new Range writer instance
	GetWriter(ctx context.Context, ns Namespace, metadata graveler.Metadata) (RangeWriter, error)

	// GetURI returns a URI from which to read the contents of id.  If id does not exist
	// it may return a URI that resolves nowhere rather than an error.
	GetURI(ctx context.Context, ns Namespace, id ID) (string, error)
}

type RangeWriter 

type RangeWriter interface {
	// WriteRecord appends the given record to the Range.
	WriteRecord(record Record) error

	// AddMetadata associates metadata value (which will be stringified when the writer is
	// Closed and added to the resulting range ID) with key.
	SetMetadata(key, value string)

	// GetApproximateSize returns an estimate of the current written size of the Range.
	GetApproximateSize() uint64

	// Close flushes all records to the disk and returns the WriteResult.
	Close() (*WriteResult, error)

	// Abort terminates the non-closed file and removes all traces.
	Abort() error
}

RangeWriter is an abstraction for writing Ranges. Written records must be sorted by key.

type Record 

type Record struct {
	Key   Key
	Value Value
}

type ResultCloser 

type ResultCloser interface {
	Close() (*WriteResult, error)
}

type UnmarshalIterator 

type UnmarshalIterator struct {
	// contains filtered or unexported fields
}

UnmarshalIterator wrap value iterator and unmarshal each value

func NewUnmarshalIterator 

func NewUnmarshalIterator(it ValueIterator) *UnmarshalIterator

func (*UnmarshalIterator) Close 

func (r *UnmarshalIterator) Close()

func (*UnmarshalIterator) Err 

func (r *UnmarshalIterator) Err() error

func (*UnmarshalIterator) Next 

func (r *UnmarshalIterator) Next() bool

func (*UnmarshalIterator) SeekGE 

func (r *UnmarshalIterator) SeekGE(id graveler.Key)

func (*UnmarshalIterator) Value 

func (r *UnmarshalIterator) Value() *graveler.ValueRecord

type Value 

type Value []byte

type ValueIterator 

type ValueIterator interface {
	Next() bool
	SeekGE(id Key)
	Value() *Record
	Err() error
	Close()
}

type WriteResult 

type WriteResult struct {
	// ID is the identifier for the written Range.
	// Calculated by an hash function to all keys and values' identity.
	RangeID ID

	// First is the first key in the Range.
	First Key

	// Last is the last key in the Range.
	Last Key

	// Count is the number of records in the Range.
	Count int

	// EstimatedRangeSizeBytes is Approximate size of each Range
	EstimatedRangeSizeBytes uint64
}

WriteResult is the result of a completed write of a Range

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.