Documentation
¶
Index ¶
- Constants
- Variables
- func Commit(ctx context.Context, writer MetaRangeWriter, base Iterator, ...) (graveler.DiffSummary, error)
- func MarshalRange(r Range) ([]byte, error)
- func MarshalValue(v *graveler.Value) ([]byte, error)
- func Merge(ctx context.Context, writer MetaRangeWriter, base Iterator, source Iterator, ...) error
- func MustMarshalValue(v *graveler.Value) []byte
- func MustUnmarshalValue(b []byte) *graveler.Value
- func NewCommittedManager(m MetaRangeManager) graveler.CommittedManager
- func NewCompareIterator(ctx context.Context, diffDestToSource DiffIterator, base Iterator) *compareIterator
- func NewCompareValueIterator(ctx context.Context, diffDestToSource DiffIterator, base Iterator) *compareValueIterator
- func NewDiffValueIterator(ctx context.Context, left Iterator, right Iterator) graveler.DiffIterator
- func NewValueIterator(it Iterator) graveler.ValueIterator
- func UnmarshalIdentity(b []byte) ([]byte, error)
- func UnmarshalValue(b []byte) (*graveler.Value, error)
- type BatchCloser
- type BatchWriterCloser
- type CommitOptions
- type DiffIterator
- type DiffIteratorWrapper
- type GeneralMetaRangeWriter
- type ID
- type Iterator
- type IteratorWrapper
- type Key
- type MetaRangeManager
- type MetaRangeWriter
- type Namespace
- type Params
- type Range
- type RangeData
- func (*RangeData) Descriptor() ([]byte, []int)deprecated
- func (x *RangeData) GetCount() int64
- func (x *RangeData) GetEstimatedSize() uint64
- func (x *RangeData) GetMaxKey() []byte
- func (x *RangeData) GetMinKey() []byte
- func (*RangeData) ProtoMessage()
- func (x *RangeData) ProtoReflect() protoreflect.Message
- func (x *RangeData) Reset()
- func (x *RangeData) String() string
- type RangeDiff
- type RangeManager
- type RangeWriter
- type Record
- type ResultCloser
- type UnmarshalIterator
- type Value
- type ValueIterator
- type WriteResult
Constants ¶
const ( MetadataTypeKey = "type" MetadataRangesType = "ranges" MetadataMetarangesType = "metaranges" )
const MaxValueComponentBytes = 640 << 16
MaxValueComponentBytes is the longest size allowed for the data length of a graveler value (or its identity, but that is controlled by code here, so less likely). It (only) protects the process from unbounded serialization. "640 KB should be enough for anyone" - even at a few 10s of KiBs you may be better served with some other format or implementation.
Variables ¶
var ( ErrUnsortedKeys = errors.New("keys should be written in ascending order") ErrNilValue = errors.New("record value should not be nil") )
var ErrBadValueBytes = errors.New("bad bytes format for graveler.Value")
ErrBadValueBytes is an error that is probably returned when unmarshalling bytes that are supposed to encode a Value.
var (
ErrMultipleWaitCalls = errors.New("wait has already been called")
)
var ErrNeedBatchClosers = errors.New("need at least 1 batch uploaded")
var ErrNoRange = errors.New("diff is not currently in a range")
ErrNoRange occurs when calling nextRange while not in a range, could happen when the diff is currently comparing keys in two different ranges
var (
ErrNotFound = errors.New("not found")
)
var ErrTooLong = errors.New("too long")
ErrTooLong is an error that is returned when trying to marshal too long a key or value. This should never normally happen in graveler files generated by graveler.
var ErrUnsupportedRangeDiffType = errors.New("range diff type unsupported - supports only added and removed")
var File_committed_proto protoreflect.FileDescriptor
Functions ¶
func Commit ¶ added in v0.58.0
func Commit(ctx context.Context, writer MetaRangeWriter, base Iterator, changes graveler.ValueIterator, opts *CommitOptions) (graveler.DiffSummary, error)
func MarshalRange ¶
func MarshalValue ¶
MarshalValue returns bytes that uniquely unmarshal into a Value equal to v.
func MustMarshalValue ¶
MustMarshalValue an MarshalValue that will panic on error
func MustUnmarshalValue ¶
MustUnmarshalValue an UnmarshalValue that will panic on error
func NewCommittedManager ¶
func NewCommittedManager(m MetaRangeManager) graveler.CommittedManager
func NewCompareIterator ¶
func NewCompareIterator(ctx context.Context, diffDestToSource DiffIterator, base Iterator) *compareIterator
NewCompareIterator accepts an iterator describing a diff from the merge destination to the source. It returns a DiffIterator with the changes to perform on the destination branch, in order to merge the source into it, relative to base as the merge base. When reaching a conflict, the returned Diff will be of type graveler.DiffTypeConflict.
func NewCompareValueIterator ¶ added in v0.40.0
func NewCompareValueIterator(ctx context.Context, diffDestToSource DiffIterator, base Iterator) *compareValueIterator
NewCompareValueIterator Wraps CompareIterator in order to return only values
func NewDiffValueIterator ¶ added in v0.40.0
func NewValueIterator ¶
func NewValueIterator(it Iterator) graveler.ValueIterator
func UnmarshalIdentity ¶
UnmarshalIdentity returns *only* the Identity field encoded by b. It does not even examine any bytes beyond the prefix of b holding Identity.
Types ¶
type BatchCloser ¶
type BatchCloser struct {
// contains filtered or unexported fields
}
func NewBatchCloser ¶
func NewBatchCloser(numClosers int) *BatchCloser
NewBatchCloser returns a new BatchCloser
func (*BatchCloser) CloseWriterAsync ¶
func (bc *BatchCloser) CloseWriterAsync(w ResultCloser) error
CloseWriterAsync adds RangeWriter instance for the BatchCloser to handle. Any writes executed to the writer after this call are not guaranteed to succeed. If Wait() has already been called, returns an error.
func (*BatchCloser) Wait ¶
func (bc *BatchCloser) Wait() ([]WriteResult, error)
Wait returns when all Writers finished. Returns a nil results slice and an error if *any* RangeWriter failed to close and upload.
type BatchWriterCloser ¶
type BatchWriterCloser interface { // CloseWriterAsync adds MetaRangeWriter instance for the BatchWriterCloser to handle. // Any writes executed to the writer after this call are not guaranteed to succeed. // If Wait() has already been called, returns an error. CloseWriterAsync(ResultCloser) error // Wait returns when all Writers finished. // Any failure to close a single MetaRangeWriter will return with a nil results slice and an error. Wait() ([]WriteResult, error) }
// blocks until all writers finished or any writer failed res, err := bwc.Wait() // handle err, results, etc.. }
type CommitOptions ¶ added in v0.58.0
type CommitOptions struct { // Set to allow commits that change nothing (otherwise ErrNoChanges) AllowEmpty bool }
type DiffIterator ¶ added in v0.40.0
type DiffIterator interface { // Next moves to look at the next value in the current Range, or a header for the next Range if the current Range is over and a next range exists. Next() bool // NextRange skips the current range // If the next Range is a "headerless" range it will return the first value, otherwise will return the header // calling NextRange from a "headerless" should result with ErrNoRange NextRange() bool // Value returns a nil ValueRecord and a Range before starting a Range, a Value and that Range when inside a Range, or a value with no range when inside a headerless Range Value() (*graveler.Diff, *RangeDiff) SeekGE(id graveler.Key) Err() error Close() }
DiffIterator iterates over all Range headers and values of a Diff, allowing seeking by entire ranges. DiffIterator might contain ranges without headers for example:
left [min].R1.[max] [min].R3.[max] [min]...............R5..............[max]
------------------------------------------------------------------------------------------------
right [min].R2.[max] [min.....R4....max] [min].R6.[max] [min].R7.[max]
R1 - will return as diff with header R2 - will return as diff with header R3 and R4 - could not return a header because we must enter the ranges in order to get some header values (such as count) R5 and R6 - same as R3 and R4 R7 - in case R5 has no values in the R7 range, R7 would return as a diff with header
func NewDiffIterator ¶
func NewDiffIterator(ctx context.Context, left Iterator, right Iterator) DiffIterator
func NewDiffIteratorWrapper ¶ added in v0.40.0
func NewDiffIteratorWrapper(iter graveler.DiffIterator) DiffIterator
type DiffIteratorWrapper ¶ added in v0.40.0
type DiffIteratorWrapper struct { graveler.DiffIterator // contains filtered or unexported fields }
func (*DiffIteratorWrapper) Err ¶ added in v0.40.0
func (d *DiffIteratorWrapper) Err() error
func (*DiffIteratorWrapper) NextRange ¶ added in v0.40.0
func (d *DiffIteratorWrapper) NextRange() bool
type GeneralMetaRangeWriter ¶
type GeneralMetaRangeWriter struct {
// contains filtered or unexported fields
}
func NewGeneralMetaRangeWriter ¶
func NewGeneralMetaRangeWriter(ctx context.Context, rangeManager, metaRangeManager RangeManager, params *Params, namespace Namespace, md graveler.Metadata) *GeneralMetaRangeWriter
func (*GeneralMetaRangeWriter) Abort ¶
func (w *GeneralMetaRangeWriter) Abort() error
func (*GeneralMetaRangeWriter) Close ¶
func (w *GeneralMetaRangeWriter) Close() (*graveler.MetaRangeID, error)
func (*GeneralMetaRangeWriter) WriteRange ¶
func (w *GeneralMetaRangeWriter) WriteRange(rng Range) error
func (*GeneralMetaRangeWriter) WriteRecord ¶
func (w *GeneralMetaRangeWriter) WriteRecord(record graveler.ValueRecord) error
WriteRecord writes a record to the current range, decides if should close range
type Iterator ¶
type Iterator interface { // Next moves to look at the next value in the current Range, or a header for the next // Range if the current Range is over. Next() bool // NextRange() skips the current Range and continues at the header for the next Range. NextRange() bool // Value returns a nil ValueRecord and a Range before starting a Range, or a Value and // that Range when inside a Range. Value() (*graveler.ValueRecord, *Range) SeekGE(id graveler.Key) Err() error Close() }
Iterator iterates over all Range headers and values of a MetaRange, allowing seeking by entire ranges.
func NewEmptyIterator ¶
func NewEmptyIterator() Iterator
func NewIterator ¶
func NewIterator(ctx context.Context, manager RangeManager, namespace Namespace, rangesIt ValueIterator) Iterator
func NewIteratorWrapper ¶ added in v0.40.0
func NewIteratorWrapper(iter graveler.ValueIterator) Iterator
type IteratorWrapper ¶ added in v0.40.0
type IteratorWrapper struct { graveler.ValueIterator // contains filtered or unexported fields }
func (*IteratorWrapper) Err ¶ added in v0.40.0
func (i *IteratorWrapper) Err() error
func (*IteratorWrapper) NextRange ¶ added in v0.40.0
func (i *IteratorWrapper) NextRange() bool
func (*IteratorWrapper) Value ¶ added in v0.40.0
func (i *IteratorWrapper) Value() (*graveler.ValueRecord, *Range)
type MetaRangeManager ¶
type MetaRangeManager interface { Exists(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID) (bool, error) // GetValue returns the matching in-range graveler.ValueRecord for key in the // MetaRange with id. GetValue(ctx context.Context, ns graveler.StorageNamespace, id graveler.MetaRangeID, key graveler.Key) (*graveler.ValueRecord, error) // NewRangeWriter returns a writer that is used for creating new MetaRanges NewWriter(ctx context.Context, ns graveler.StorageNamespace, metadata graveler.Metadata) MetaRangeWriter // NewMetaRangeIterator returns an Iterator over the MetaRange with id. NewMetaRangeIterator(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (Iterator, error) // GetMetaRangeURI returns a URI with an object representing metarange ID. It may // return a URI that does not resolve (rather than an error) if ID does not exist. GetMetaRangeURI(ctx context.Context, ns graveler.StorageNamespace, metaRangeID graveler.MetaRangeID) (string, error) // GetRangeURI returns a URI with an object representing metarange ID. It may // return a URI that does not resolve (rather than an error) if ID does not exist. GetRangeURI(ctx context.Context, ns graveler.StorageNamespace, rangeID graveler.RangeID) (string, error) }
MetaRangeManager is an abstraction for a repository of MetaRanges that exposes operations on them
func NewMetaRangeManager ¶
func NewMetaRangeManager(params Params, metaManager, rangeManager RangeManager) (MetaRangeManager, error)
type MetaRangeWriter ¶
type MetaRangeWriter interface { // WriteRecord adds a record to the MetaRange. The key must be greater than any other key that was written // (in other words - values must be entered sorted by key order). // If the most recent insertion was using WriteRange, the key must be greater than any key in the added ranges. WriteRecord(graveler.ValueRecord) error // AddRange adds a complete range to the MetaRange at the current insertion point. // Added Range must not contain keys smaller than last previously written value. WriteRange(Range) error // Close finalizes the MetaRange creation. It's invalid to add records after calling this method. // During MetaRange writing, ranges are closed asynchronously and copied by tierFS // while writing continues. Close waits until closing and copying all ranges. Close() (*graveler.MetaRangeID, error) Abort() error }
MetaRangeWriter is an abstraction for creating new MetaRanges
type Params ¶
type Params struct { // MinRangeSizeBytes is the smallest size for splitting a range partition as a result // of adding a record. Smaller ranges are still possible due to re-using an existing MinRangeSizeBytes uint64 // MaxRangeSizeBytes is the largest size of a range partition. In practice the range // is split only after an additional record. MaxRangeSizeBytes uint64 // RangeSizeEntriesRaggedness allows raggedness in splitting range partitions. It is // the expected number of records after MinRangeSizeBytes at which to split the range // -- ranges are split at the first key with hash divisible by this raggedness. RangeSizeEntriesRaggedness float64 // MaxUploaders is the maximal number of uploaders to use in a single metarange writer. MaxUploaders int }
type Range ¶
type Range struct { ID ID MinKey Key MaxKey Key EstimatedSize uint64 // EstimatedSize estimated Range size in bytes Count int64 Tombstone bool }
func UnmarshalRange ¶
type RangeData ¶
type RangeData struct { // First key of range, must be >= key of range in SSTable. If missing, == key of range in SSTable. MinKey []byte `protobuf:"bytes,1,opt,name=min_key,json=minKey,proto3" json:"min_key,omitempty"` MaxKey []byte `protobuf:"bytes,2,opt,name=max_key,json=maxKey,proto3" json:"max_key,omitempty"` EstimatedSize uint64 `protobuf:"varint,3,opt,name=estimated_size,json=estimatedSize,proto3" json:"estimated_size,omitempty"` Count int64 `protobuf:"varint,4,opt,name=count,proto3" json:"count,omitempty"` // contains filtered or unexported fields }
func (*RangeData) Descriptor
deprecated
func (*RangeData) GetEstimatedSize ¶
func (*RangeData) ProtoMessage ¶
func (*RangeData) ProtoMessage()
func (*RangeData) ProtoReflect ¶
func (x *RangeData) ProtoReflect() protoreflect.Message
type RangeDiff ¶ added in v0.40.0
type RangeDiff struct { Type graveler.DiffType Range *Range LeftIdentity ID // the Identity of the value on the left side of the diff (populated on DiffTypeChanged ) }
RangeDiff represents a change in Range
type RangeManager ¶
type RangeManager interface { // Exists returns true if id references a Range. Exists(ctx context.Context, ns Namespace, id ID) (bool, error) // GetValue returns the value matching key in the Range referenced by id. If id not // found, it return (nil, ErrNotFound). GetValue(ctx context.Context, ns Namespace, id ID, key Key) (*Record, error) // GetValueGE returns the first value keyed at or after key in the Range referenced by // id. If all values are keyed before key, it returns (nil, ErrNotFound). GetValueGE(ctx context.Context, ns Namespace, id ID, key Key) (*Record, error) // NewRangeIterator returns an iterator over values in the Range with ID. NewRangeIterator(ctx context.Context, ns Namespace, pid ID) (ValueIterator, error) // GetWriter returns a new Range writer instance GetWriter(ctx context.Context, ns Namespace, metadata graveler.Metadata) (RangeWriter, error) // GetURI returns a URI from which to read the contents of id. If id does not exist // it may return a URI that resolves nowhere rather than an error. GetURI(ctx context.Context, ns Namespace, id ID) (string, error) }
type RangeWriter ¶
type RangeWriter interface { // WriteRecord appends the given record to the Range. WriteRecord(record Record) error // SetMetadata associates metadata value (which will be stringified when the writer is // Closed and added to the resulting range ID) with key. SetMetadata(key, value string) // GetApproximateSize returns an estimate of the current written size of the Range. GetApproximateSize() uint64 // Close flushes all records to the disk and returns the WriteResult. Close() (*WriteResult, error) // Abort terminates the non-closed file and removes all traces. Abort() error }
RangeWriter is an abstraction for writing Ranges. Written records must be sorted by key.
type ResultCloser ¶
type ResultCloser interface {
Close() (*WriteResult, error)
}
type UnmarshalIterator ¶
type UnmarshalIterator struct {
// contains filtered or unexported fields
}
UnmarshalIterator wrap value iterator and unmarshal each value
func NewUnmarshalIterator ¶
func NewUnmarshalIterator(it ValueIterator) *UnmarshalIterator
func (*UnmarshalIterator) Close ¶
func (r *UnmarshalIterator) Close()
func (*UnmarshalIterator) Err ¶
func (r *UnmarshalIterator) Err() error
func (*UnmarshalIterator) Next ¶
func (r *UnmarshalIterator) Next() bool
func (*UnmarshalIterator) SeekGE ¶
func (r *UnmarshalIterator) SeekGE(id graveler.Key)
func (*UnmarshalIterator) Value ¶
func (r *UnmarshalIterator) Value() *graveler.ValueRecord
type ValueIterator ¶
type WriteResult ¶
type WriteResult struct { // ID is the identifier for the written Range. // Calculated by an hash function to all keys and values' identity. RangeID ID // First is the first key in the Range. First Key // Last is the last key in the Range. Last Key // Count is the number of records in the Range. Count int // EstimatedRangeSizeBytes is Approximate size of each Range EstimatedRangeSizeBytes uint64 }
WriteResult is the result of a completed write of a Range
Source Files
¶
- batch.go
- batch_write_closer.go
- commit.go
- committed.pb.go
- compare_iterator.go
- diff.go
- diff_values.go
- internal_value.go
- iterator.go
- manager.go
- merge.go
- merge_value_iterator.go
- meta_range.go
- meta_range_manager.go
- meta_range_writer.go
- range.go
- range_manager.go
- unmarshal_iterator.go
- value.go
- value_iterator.go
- wrappers.go