committed

package
v1.34.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 16, 2024 License: Apache-2.0 Imports: 14 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	MetadataTypeKey        = "type"
	MetadataRangesType     = "ranges"
	MetadataMetarangesType = "metaranges"
)
View Source 
const MaxValueComponentBytes = 640 << 16

MaxValueComponentBytes is the longest size allowed for the data length of a graveler value (or its identity, but that is controlled by code here, so less likely). It (only) protects the process from unbounded serialization. "640 KB should be enough for anyone" - even at a few 10s of KiBs you may be better served with some other format or implementation.

Variables

View Source 
var (
	ErrUnsortedKeys = errors.New("keys should be written in ascending order")
	ErrNilValue     = errors.New("record value should not be nil")
)
View Source 
var ErrBadValueBytes = errors.New("bad bytes format for graveler.Value")

ErrBadValueBytes is an error that is probably returned when unmarshalling bytes that are supposed to encode a Value.

View Source 
var ErrMultipleWaitCalls = errors.New("wait has already been called")
View Source 
var ErrNeedBatchClosers = errors.New("need at least 1 batch uploaded")
View Source 
var ErrNoRange = errors.New("diff is not currently in a range")

ErrNoRange occurs when calling nextRange while not in a range, could happen when the diff is currently comparing keys in two different ranges

View Source 
var ErrNotFound = graveler.ErrNotFound
View Source 
var ErrTooLong = errors.New("too long")

ErrTooLong is an error that is returned when trying to marshal too long a key or value. This should never normally happen in graveler files generated by graveler.

View Source 
var ErrUnsupportedRangeDiffType = errors.New("range diff type unsupported - supports only added and removed")
View Source 
var File_graveler_committed_committed_proto protoreflect.FileDescriptor

Functions

func Commit added in v0.58.0 

func Commit(ctx context.Context, writer MetaRangeWriter, base Iterator, changes graveler.ValueIterator, opts *CommitOptions) (graveler.DiffSummary, error)

func MarshalRange 

func MarshalRange(r Range) ([]byte, error)

func MarshalValue 

func MarshalValue(v *graveler.Value) ([]byte, error)

MarshalValue returns bytes that uniquely unmarshal into a Value equal to v.

func Merge added in v0.58.1 

func Merge(ctx context.Context, writer MetaRangeWriter, base Iterator, source Iterator, destination Iterator, strategy graveler.MergeStrategy) error

func MustMarshalValue 

func MustMarshalValue(v *graveler.Value) []byte

MustMarshalValue a MarshalValue that will panic on error

func NewCommittedManager 

func NewCommittedManager(m MetaRangeManager, r RangeManager, p Params) graveler.CommittedManager

func NewCompareIterator 

func NewCompareIterator(ctx context.Context, diffDestToSource DiffIterator, base Iterator) *compareIterator

NewCompareIterator accepts an iterator describing a diff from the merge destination to the source. It returns a DiffIterator with the changes to perform on the destination branch, in order to merge the source into it, relative to base as the merge base. When reaching a conflict, the returned Diff will be of type graveler.DiffTypeConflict.

func NewCompareValueIterator added in v0.40.0 

func NewCompareValueIterator(ctx context.Context, diffDestToSource DiffIterator, base Iterator) *compareValueIterator

NewCompareValueIterator Wraps CompareIterator in order to return only values

func NewDiffValueIterator added in v0.40.0 

func NewDiffValueIterator(ctx context.Context, left Iterator, right Iterator) graveler.DiffIterator

func NewValueIterator 

func NewValueIterator(it Iterator) graveler.ValueIterator

func UnmarshalValue 

func UnmarshalValue(b []byte) (*graveler.Value, error)

Types

type BatchCloser 

type BatchCloser struct {
	// contains filtered or unexported fields
}

func NewBatchCloser 

func NewBatchCloser(numClosers int) *BatchCloser

NewBatchCloser returns a new BatchCloser

func (*BatchCloser) CloseWriterAsync 

func (bc *BatchCloser) CloseWriterAsync(w ResultCloser) error

CloseWriterAsync adds RangeWriter instance for the BatchCloser to handle. Any writes executed to the writer after this call are not guaranteed to succeed. If Wait() has already been called, returns an error.

func (*BatchCloser) Wait 

func (bc *BatchCloser) Wait() ([]WriteResult, error)

Wait returns when all Writers finished. Returns a nil results slice and an error if *any* RangeWriter failed to close and upload.

type BatchWriterCloser 

type BatchWriterCloser interface {
	// CloseWriterAsync adds MetaRangeWriter instance for the BatchWriterCloser to handle.
	// Any writes executed to the writer after this call are not guaranteed to succeed.
	// If Wait() has already been called, returns an error.
	CloseWriterAsync(ResultCloser) error

	// Wait returns when all Writers finished.
	// Any failure to close a single MetaRangeWriter will return with a nil results slice and an error.
	Wait() ([]WriteResult, error)
}

BatchWriterCloser collects Range writers and handles the asynchronous flushing and closing of the writers. Example usage: 

func batch(manager RangeManager, bwc BatchWriterCloser) {
		w1, _ := manager.GetWriter()
		_ = w1.WriteRecord(graveler.ValueRecord{Key: "foo1", Value: &graveler.Value{Address: "bar1"}})
		_ = w1.WriteRecord(graveler.ValueRecord{Key: "foo2", Value: &graveler.Value{Address: "bar2"}})
		_ = bwc.CloseWriterAsync(w1)

		w2, _ := manager.GetWriter()
		_ = w2.WriteRecord(graveler.ValueRecord{Key: "goo1", Value: &graveler.Value{Address: "baz1"}})
		_ = bwc.CloseWriterAsync(w2)

		// blocks until all writers finished or any writer failed
		res, err := bwc.Wait()
		// handle err, results, etc..
	}

type CommitOptions added in v0.58.0 

type CommitOptions struct {
	// Set to allow commits that change nothing (otherwise ErrNoChanges)
	AllowEmpty bool
}

type DiffIterator added in v0.40.0 

type DiffIterator interface {
	// Next moves to look at the next value in the current Range, or a header for the next Range if the current Range is over and a next range exists.
	Next() bool
	// NextRange skips the current range
	// If the next Range is a "headerless" range it will return the first value, otherwise will return the header
	// calling NextRange from a "headerless" should result with ErrNoRange
	NextRange() bool
	// Value returns a nil ValueRecord and a Range before starting a Range, a Value and that Range when inside a Range, or a value with no range when inside a headerless Range
	Value() (*graveler.Diff, *RangeDiff)
	SeekGE(id graveler.Key)
	Err() error
	Close()
}

DiffIterator iterates over all Range headers and values of a Diff, allowing seeking by entire ranges. DiffIterator might contain ranges without headers for example:

left [min].R1.[max] [min].R3.[max] [min]...............R5..............[max] 

------------------------------------------------------------------------------------------------

right [min].R2.[max] [min.....R4....max] [min].R6.[max] [min].R7.[max]

R1 - will return as diff with header R2 - will return as diff with header R3 and R4 - could not return a header because we must enter the ranges in order to get some header values (such as count) R5 and R6 - same as R3 and R4 R7 - in case R5 has no values in the R7 range, R7 would return as a diff with header

func NewDiffIterator 

func NewDiffIterator(ctx context.Context, left Iterator, right Iterator) DiffIterator

func NewDiffIteratorWrapper added in v0.40.0 

func NewDiffIteratorWrapper(iter graveler.DiffIterator) DiffIterator

type DiffIteratorWrapper added in v0.40.0 

type DiffIteratorWrapper struct {
	graveler.DiffIterator
	// contains filtered or unexported fields
}

func (*DiffIteratorWrapper) Close added in v0.70.1 

func (d *DiffIteratorWrapper) Close()

func (*DiffIteratorWrapper) Err added in v0.40.0 

func (d *DiffIteratorWrapper) Err() error

func (*DiffIteratorWrapper) NextRange added in v0.40.0 

func (d *DiffIteratorWrapper) NextRange() bool

func (*DiffIteratorWrapper) Value added in v0.40.0 

func (d *DiffIteratorWrapper) Value() (*graveler.Diff, *RangeDiff)

type GeneralMetaRangeWriter 

type GeneralMetaRangeWriter struct {
	// contains filtered or unexported fields
}

func NewGeneralMetaRangeWriter 

func NewGeneralMetaRangeWriter(ctx context.Context, rangeManager, metaRangeManager RangeManager, params *Params, namespace Namespace, md graveler.Metadata) *GeneralMetaRangeWriter

func (*GeneralMetaRangeWriter) Abort 

func (w *GeneralMetaRangeWriter) Abort() error

func (*GeneralMetaRangeWriter) Close 

func (w *GeneralMetaRangeWriter) Close(ctx context.Context) (*graveler.MetaRangeID, error)

func (*GeneralMetaRangeWriter) WriteRange 

func (w *GeneralMetaRangeWriter) WriteRange(rng Range) error

func (*GeneralMetaRangeWriter) WriteRecord 

func (w *GeneralMetaRangeWriter) WriteRecord(record graveler.ValueRecord) error

WriteRecord writes a record to the current range, decides if should close range

type ID 

type ID string

ID is an identifier for a Range

type ImportIterator added in v0.105.0 

type ImportIterator interface {
	IsCurrentRangeBoundedByPrefix() bool
	IsCurrentPrefixIncludedInRange() bool
}

type Iterator 

type Iterator interface {
	// Next moves to look at the next value in the current Range, or a header for the next
	// Range if the current Range is over.
	Next() bool
	// NextRange skips the current Range and continues at the header for the next Range.
	NextRange() bool
	// Value returns a nil ValueRecord and a Range before starting a Range, or a Value and
	// that Range when inside a Range.
	Value() (*graveler.ValueRecord, *Range)
	SeekGE(id graveler.Key)
	Err() error
	Close()
}

Iterator iterates over all Range headers and values of a MetaRange, allowing seeking by entire ranges.

func NewEmptyIterator 

func NewEmptyIterator() Iterator

func NewIterator 

func NewIterator(ctx context.Context, manager RangeManager, namespace Namespace, rangesIt ValueIterator) Iterator

type Key 

type Key []byte

Key and Value types for to be stored in any Range of the MetaRange

func (Key) Copy 

func (k Key) Copy() Key

type MetaRangeManager 

type MetaRangeManager interface {
	Exists(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (bool, error)

	// GetValue returns the matching in-range graveler.ValueRecord for key in the
	// MetaRange with id.
	GetValue(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID, key graveler.Key) (*graveler.ValueRecord, error)

	// NewWriter returns a writer that is used for creating new MetaRanges
	NewWriter(ctx context.Context, ns graveler.StorageNamespace, metadata graveler.Metadata) MetaRangeWriter

	// NewMetaRangeIterator returns an Iterator over the MetaRange with id.
	NewMetaRangeIterator(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (Iterator, error)

	// GetMetaRangeURI returns a URI with an object representing metarange ID.  It may
	// return a URI that does not resolve (rather than an error) if ID does not exist.
	GetMetaRangeURI(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (string, error)

	// GetRangeURI returns a URI with an object representing range ID.  It may
	// return a URI that does not resolve (rather than an error) if ID does not exist.
	GetRangeURI(ctx context.Context, ns graveler.StorageNamespace, rangeID graveler.RangeID) (string, error)

	// GetRangeByKey returns the Range that contains key in the MetaRange with id.
	GetRangeByKey(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID, key graveler.Key) (*Range, error)
}

MetaRangeManager is an abstraction for a repository of MetaRanges that exposes operations on them

func NewMetaRangeManager 

func NewMetaRangeManager(params Params, metaManager, rangeManager RangeManager) (MetaRangeManager, error)

type MetaRangeWriter 

type MetaRangeWriter interface {
	// WriteRecord adds a record to the MetaRange. The key must be greater than any other key that was written
	// (in other words - values must be entered sorted by key order).
	// If the most recent insertion was using WriteRange, the key must be greater than any key in the added ranges.
	WriteRecord(graveler.ValueRecord) error

	// WriteRange adds a complete range to the MetaRange at the current insertion point.
	// Added Range must not contain keys smaller than last previously written value.
	WriteRange(Range) error

	// Close finalizes the MetaRange creation. It's invalid to add records after calling this method.
	// During MetaRange writing, ranges are closed asynchronously and copied by tierFS
	// while writing continues. Close waits until closing and copying all ranges.
	Close(context.Context) (*graveler.MetaRangeID, error)

	Abort() error
}

MetaRangeWriter is an abstraction for creating new MetaRanges

type Namespace 

type Namespace string

Namespace is namespace for ID ranges

type Params 

type Params struct {
	// MinRangeSizeBytes is the smallest size for splitting a range partition as a result
	// of adding a record.  Smaller ranges are still possible due to re-using an existing
	MinRangeSizeBytes uint64
	// MaxRangeSizeBytes is the largest size of a range partition.  In practice the range
	// is split only after an additional record.
	MaxRangeSizeBytes uint64
	// RangeSizeEntriesRaggedness allows raggedness in splitting range partitions.  It is
	// the expected number of records after MinRangeSizeBytes at which to split the range
	// -- ranges are split at the first key with hash divisible by this raggedness.
	RangeSizeEntriesRaggedness float64
	// MaxUploaders is the maximal number of uploaders to use in a single metarange writer.
	MaxUploaders int
}

type Range 

type Range struct {
	ID            ID
	MinKey        Key
	MaxKey        Key
	EstimatedSize uint64 // EstimatedSize estimated Range size in bytes
	Count         int64
	Tombstone     bool
}

Range represents a range of sorted Keys

func UnmarshalRange 

func UnmarshalRange(b []byte) (Range, error)

func (Range) BeforeRange added in v0.105.0 

func (r Range) BeforeRange(o *Range) bool

func (Range) Copy added in v0.40.0 

func (r Range) Copy() *Range

func (Range) EqualBounds added in v0.105.0 

func (r Range) EqualBounds(o *Range) bool

type RangeData 

type RangeData struct {

	// First key of range, must be >= key of range in SSTable.  If missing, == key of range in SSTable.
	MinKey        []byte `protobuf:"bytes,1,opt,name=min_key,json=minKey,proto3" json:"min_key,omitempty"`
	MaxKey        []byte `protobuf:"bytes,2,opt,name=max_key,json=maxKey,proto3" json:"max_key,omitempty"`
	EstimatedSize uint64 `protobuf:"varint,3,opt,name=estimated_size,json=estimatedSize,proto3" json:"estimated_size,omitempty"`
	Count         int64  `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"`
	// contains filtered or unexported fields
}

func (*RangeData) Descriptor deprecated 

func (*RangeData) Descriptor() ([]byte, []int)

Deprecated: Use RangeData.ProtoReflect.Descriptor instead.

func (*RangeData) GetCount 

func (x *RangeData) GetCount() int64

func (*RangeData) GetEstimatedSize 

func (x *RangeData) GetEstimatedSize() uint64

func (*RangeData) GetMaxKey 

func (x *RangeData) GetMaxKey() []byte

func (*RangeData) GetMinKey 

func (x *RangeData) GetMinKey() []byte

func (*RangeData) ProtoMessage 

func (*RangeData) ProtoMessage()

func (*RangeData) ProtoReflect 

func (x *RangeData) ProtoReflect() protoreflect.Message

func (*RangeData) Reset 

func (x *RangeData) Reset()

func (*RangeData) String 

func (x *RangeData) String() string

type RangeDiff added in v0.40.0 

type RangeDiff struct {
	Type         graveler.DiffType
	Range        *Range
	LeftIdentity ID // the Identity of the value on the left side of the diff (populated on DiffTypeChanged )
}

RangeDiff represents a change in Range

func (RangeDiff) Copy added in v0.40.0 

func (r RangeDiff) Copy() *RangeDiff

type RangeManager 

type RangeManager interface {
	// Exists returns true if id references a Range.
	Exists(ctx context.Context, ns Namespace, id ID) (bool, error)

	// GetValue returns the value matching key in the Range referenced by id. If id not
	// found, it return (nil, ErrNotFound).
	GetValue(ctx context.Context, ns Namespace, id ID, key Key) (*Record, error)

	// GetValueGE returns the first value keyed at or after key in the Range referenced by
	// id.  If all values are keyed before key, it returns (nil, ErrNotFound).
	GetValueGE(ctx context.Context, ns Namespace, id ID, key Key) (*Record, error)

	// NewRangeIterator returns an iterator over values in the Range with ID.
	NewRangeIterator(ctx context.Context, ns Namespace, pid ID) (ValueIterator, error)

	// GetWriter returns a new Range writer instance
	GetWriter(ctx context.Context, ns Namespace, metadata graveler.Metadata) (RangeWriter, error)

	// GetURI returns a URI from which to read the contents of id.  If id does not exist
	// it may return a URI that resolves nowhere rather than an error.
	GetURI(ctx context.Context, ns Namespace, id ID) (string, error)
}

type RangeWriter 

type RangeWriter interface {
	// WriteRecord appends the given record to the Range.
	WriteRecord(record Record) error

	// SetMetadata associates metadata value (which will be stringify when the writer is
	// closed and added to the resulting range ID) with key.
	SetMetadata(key, value string)

	// GetApproximateSize returns an estimate of the current written size of the Range.
	GetApproximateSize() uint64

	// Close flushes all records to the disk and returns the WriteResult.
	Close() (*WriteResult, error)

	// Abort terminates the non-closed file and removes all traces.
	Abort() error

	// ShouldBreakAtKey returns true if should break range after the given key
	ShouldBreakAtKey(key graveler.Key, params *Params) bool
}

RangeWriter is an abstraction for writing Ranges. Written records must be sorted by key.

type Record 

type Record struct {
	Key   Key
	Value Value
}

type ResultCloser 

type ResultCloser interface {
	Close() (*WriteResult, error)
}

type SkipPrefixIterator added in v0.105.0 

type SkipPrefixIterator struct {
	// contains filtered or unexported fields
}

func NewSkipPrefixIterator added in v0.105.0 

func NewSkipPrefixIterator(prefixes []graveler.Prefix, rangeIterator Iterator) *SkipPrefixIterator

func (*SkipPrefixIterator) Close added in v0.105.0 

func (ipi *SkipPrefixIterator) Close()

func (*SkipPrefixIterator) Err added in v0.105.0 

func (ipi *SkipPrefixIterator) Err() error

func (*SkipPrefixIterator) IsCurrentPrefixIncludedInRange added in v0.105.0 

func (ipi *SkipPrefixIterator) IsCurrentPrefixIncludedInRange() bool

IsCurrentPrefixIncludedInRange returns true if the examined prefix is either a prefix of the range's min or max key, or if the prefix is between the range's min and max keys.

func (*SkipPrefixIterator) IsCurrentRangeBoundedByPrefix added in v0.105.0 

func (ipi *SkipPrefixIterator) IsCurrentRangeBoundedByPrefix() bool

IsCurrentRangeBoundedByPrefix returns true if both the range's max and min keys have the current prefix.

func (*SkipPrefixIterator) Next added in v0.105.0 

func (ipi *SkipPrefixIterator) Next() bool

func (*SkipPrefixIterator) NextRange added in v0.105.0 

func (ipi *SkipPrefixIterator) NextRange() bool

func (*SkipPrefixIterator) SeekGE added in v0.105.0 

func (ipi *SkipPrefixIterator) SeekGE(id graveler.Key)

func (*SkipPrefixIterator) Value added in v0.105.0 

func (ipi *SkipPrefixIterator) Value() (*graveler.ValueRecord, *Range)

type UnmarshalIterator 

type UnmarshalIterator struct {
	// contains filtered or unexported fields
}

UnmarshalIterator wrap value iterator and unmarshal each value

func NewUnmarshalIterator 

func NewUnmarshalIterator(it ValueIterator) *UnmarshalIterator

func (*UnmarshalIterator) Close 

func (r *UnmarshalIterator) Close()

func (*UnmarshalIterator) Err 

func (r *UnmarshalIterator) Err() error

func (*UnmarshalIterator) Next 

func (r *UnmarshalIterator) Next() bool

func (*UnmarshalIterator) SeekGE 

func (r *UnmarshalIterator) SeekGE(id graveler.Key)

func (*UnmarshalIterator) Value 

func (r *UnmarshalIterator) Value() *graveler.ValueRecord

type Value 

type Value []byte

type ValueIterator 

type ValueIterator interface {
	Next() bool
	SeekGE(id Key)
	Value() *Record
	Err() error
	Close()
}

type WriteResult 

type WriteResult struct {
	// ID is the identifier for the written Range.
	// Calculated by an hash function to all keys and values' identity.
	RangeID ID

	// First is the first key in the Range.
	First Key

	// Last is the last key in the Range.
	Last Key

	// Count is the number of records in the Range.
	Count int

	// EstimatedRangeSizeBytes is Approximate size of each Range
	EstimatedRangeSizeBytes uint64
}

WriteResult is the result of a completed write of a Range

Source Files

View all Source files

Directories

Path Synopsis
Package mock is a generated GoMock package.
 Package mock is a generated GoMock package.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.