graveler

package
v0.43.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 24, 2021 License: Apache-2.0 Imports: 21 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	EntityTypeKey    = "entity"
	EntityTypeCommit = "commit"
	EntityTypeBranch = "branch"
	EntityTypeTag    = "tag"

	EntitySchemaKey    = "schema_name"
	EntitySchemaCommit = "io.treeverse.lakefs.graveler.CommitData"
	EntitySchemaBranch = "io.treeverse.lakefs.graveler.BranchData"
	EntitySchemaTag    = "io.treeverse.lakefs.graveler.TagData"

	EntitySchemaDefinitionKey = "schema_definition"
)
View Source 
const FirstCommitMsg = "Repository created"

FirstCommitMsg is the message of the first (zero) commit of a lakeFS repository

Variables

View Source 
var (
	// Base error for "user-visible" errors, which should not be wrapped with internal
	// debug info.
	ErrUserVisible = errors.New("")

	// TODO(ariels): Wrap with ErrUserVisible once db is gone.
	ErrNotFound               = wrapError(db.ErrNotFound, "not found")
	ErrNotUnique              = errors.New("not unique")
	ErrPreconditionFailed     = errors.New("precondition failed")
	ErrInvalidValue           = errors.New("invalid value")
	ErrInvalidMergeBase       = fmt.Errorf("only 2 commits allowed in FindMergeBase: %w", ErrInvalidValue)
	ErrNoMergeBase            = errors.New("no merge base")
	ErrInvalidRef             = fmt.Errorf("ref: %w", ErrInvalidValue)
	ErrInvalidCommitID        = fmt.Errorf("commit id: %w", ErrInvalidValue)
	ErrCommitNotFound         = fmt.Errorf("commit %w", ErrNotFound)
	ErrCreateBranchNoCommit   = fmt.Errorf("can't create a branch without commit")
	ErrRepositoryNotFound     = fmt.Errorf("repository %w", ErrNotFound)
	ErrBranchNotFound         = fmt.Errorf("branch %w", ErrNotFound)
	ErrTagNotFound            = fmt.Errorf("tag %w", ErrNotFound)
	ErrRefAmbiguous           = fmt.Errorf("reference is ambiguous: %w", ErrNotFound)
	ErrNoChanges              = wrapError(ErrUserVisible, "no changes")
	ErrConflictFound          = errors.New("conflict found")
	ErrCommitNotHeadBranch    = errors.New("commit is not head of branch")
	ErrBranchExists           = fmt.Errorf("branch already exists: %w", ErrNotUnique)
	ErrTagAlreadyExists       = fmt.Errorf("tag already exists: %w", ErrNotUnique)
	ErrDirtyBranch            = errors.New("can't apply meta-range on dirty branch")
	ErrMetaRangeNotFound      = errors.New("metarange not found")
	ErrLockNotAcquired        = errors.New("lock not acquired")
	ErrRevertMergeNoParent    = errors.New("must specify 1-based parent number for reverting merge commit")
	ErrAddCommitNoParent      = errors.New("added commit must have a parent")
	ErrMultipleParents        = errors.New("cannot have more than a single parent")
	ErrRevertParentOutOfRange = errors.New("given commit does not have the given parent number")
)
View Source 
var File_graveler_proto protoreflect.FileDescriptor

Functions

func NewRunID 

func NewRunID() string

func UpperBoundForPrefix 

func UpperBoundForPrefix(prefix []byte) []byte

UpperBoundForPrefix returns, given a prefix `p`, a slice 'q' such that a byte slice `s` starts with `p` if and only if p <= s < q. Namely, it returns an exclusive upper bound for the set of all byte arrays that start with this prefix. It returns nil if there is no such byte slice because all bytes of `p` are maximal.

Types

type Branch 

type Branch struct {
	CommitID     CommitID
	StagingToken StagingToken
}

Branch is a pointer to a commit

type BranchData 

type BranchData struct {
	Id       string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	CommitId string `protobuf:"bytes,2,opt,name=commit_id,json=commitId,proto3" json:"commit_id,omitempty"`
	// contains filtered or unexported fields
}

func (*BranchData) Descriptor deprecated 

func (*BranchData) Descriptor() ([]byte, []int)

Deprecated: Use BranchData.ProtoReflect.Descriptor instead.

func (*BranchData) GetCommitId 

func (x *BranchData) GetCommitId() string

func (*BranchData) GetId 

func (x *BranchData) GetId() string

func (*BranchData) ProtoMessage 

func (*BranchData) ProtoMessage()

func (*BranchData) ProtoReflect 

func (x *BranchData) ProtoReflect() protoreflect.Message

func (*BranchData) Reset 

func (x *BranchData) Reset()

func (*BranchData) String 

func (x *BranchData) String() string

type BranchID 

type BranchID string

BranchID is an identifier for a branch

func (BranchID) Ref 

func (id BranchID) Ref() Ref

func (BranchID) String 

func (id BranchID) String() string

type BranchIterator 

type BranchIterator interface {
	Next() bool
	SeekGE(id BranchID)
	Value() *BranchRecord
	Err() error
	Close()
}

type BranchLocker 

type BranchLocker interface {
	Writer(ctx context.Context, repositoryID RepositoryID, branchID BranchID, lockedFn BranchLockerFunc) (interface{}, error)
	MetadataUpdater(ctx context.Context, repositoryID RepositoryID, branchID BranchID, lockeFn BranchLockerFunc) (interface{}, error)
}

type BranchLockerFunc 

type BranchLockerFunc func() (interface{}, error)

BranchLockerFunc callback function when branch is locked for operation (ex: writer or metadata updater)

type BranchRecord 

type BranchRecord struct {
	BranchID BranchID
	*Branch
}

BranchRecord holds BranchID with the associated Branch data

type CombinedIterator 

type CombinedIterator struct {
	// contains filtered or unexported fields
}

CombinedIterator iterates over two listing iterators, in case of duplication (in values or in errors) returns value in iterA

func NewCombinedIterator 

func NewCombinedIterator(iterA, iterB ValueIterator) *CombinedIterator

func (*CombinedIterator) Close 

func (c *CombinedIterator) Close()

func (*CombinedIterator) Err 

func (c *CombinedIterator) Err() error

func (*CombinedIterator) Next 

func (c *CombinedIterator) Next() bool

func (*CombinedIterator) SeekGE 

func (c *CombinedIterator) SeekGE(id Key)

func (*CombinedIterator) Value 

func (c *CombinedIterator) Value() *ValueRecord

type Commit 

type Commit struct {
	Version      CommitVersion `db:"version"`
	Committer    string        `db:"committer"`
	Message      string        `db:"message"`
	MetaRangeID  MetaRangeID   `db:"meta_range_id"`
	CreationDate time.Time     `db:"creation_date"`
	Parents      CommitParents `db:"parents"`
	Metadata     Metadata      `db:"metadata"`
	Generation   int           `db:"generation"`
}

Commit represents commit metadata (author, time, MetaRangeID)

func NewCommit added in v0.40.0 

func NewCommit() Commit

func (Commit) Identity 

func (c Commit) Identity() []byte

type CommitData 

type CommitData struct {
	Id           string                 `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	Committer    string                 `protobuf:"bytes,2,opt,name=committer,proto3" json:"committer,omitempty"`
	Message      string                 `protobuf:"bytes,3,opt,name=message,proto3" json:"message,omitempty"`
	CreationDate *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=creation_date,json=creationDate,proto3" json:"creation_date,omitempty"`
	MetaRangeId  string                 `protobuf:"bytes,5,opt,name=meta_range_id,json=metaRangeId,proto3" json:"meta_range_id,omitempty"`
	Metadata     map[string]string      `` /* 157-byte string literal not displayed */
	Parents      []string               `protobuf:"bytes,7,rep,name=parents,proto3" json:"parents,omitempty"`
	Version      int32                  `protobuf:"varint,8,opt,name=version,proto3" json:"version,omitempty"`
	Generation   int32                  `protobuf:"varint,9,opt,name=generation,proto3" json:"generation,omitempty"`
	// contains filtered or unexported fields
}

func (*CommitData) Descriptor deprecated 

func (*CommitData) Descriptor() ([]byte, []int)

Deprecated: Use CommitData.ProtoReflect.Descriptor instead.

func (*CommitData) GetCommitter 

func (x *CommitData) GetCommitter() string

func (*CommitData) GetCreationDate 

func (x *CommitData) GetCreationDate() *timestamppb.Timestamp

func (*CommitData) GetGeneration added in v0.41.1 

func (x *CommitData) GetGeneration() int32

func (*CommitData) GetId 

func (x *CommitData) GetId() string

func (*CommitData) GetMessage 

func (x *CommitData) GetMessage() string

func (*CommitData) GetMetaRangeId 

func (x *CommitData) GetMetaRangeId() string

func (*CommitData) GetMetadata 

func (x *CommitData) GetMetadata() map[string]string

func (*CommitData) GetParents 

func (x *CommitData) GetParents() []string

func (*CommitData) GetVersion added in v0.40.0 

func (x *CommitData) GetVersion() int32

func (*CommitData) ProtoMessage 

func (*CommitData) ProtoMessage()

func (*CommitData) ProtoReflect 

func (x *CommitData) ProtoReflect() protoreflect.Message

func (*CommitData) Reset 

func (x *CommitData) Reset()

func (*CommitData) String 

func (x *CommitData) String() string

type CommitID 

type CommitID string

CommitID is a content addressable hash representing a Commit object

func (CommitID) Ref 

func (id CommitID) Ref() Ref

func (CommitID) String 

func (id CommitID) String() string

type CommitIDAndSummary 

type CommitIDAndSummary struct {
	ID      CommitID
	Summary DiffSummary
}

type CommitIterator 

type CommitIterator interface {
	Next() bool
	SeekGE(id CommitID)
	Value() *CommitRecord
	Err() error
	Close()
}

type CommitParams 

type CommitParams struct {
	Committer string
	Message   string
	Metadata  Metadata
}

type CommitParents 

type CommitParents []CommitID

func (CommitParents) AsStringSlice 

func (cp CommitParents) AsStringSlice() []string

func (CommitParents) Contains 

func (cp CommitParents) Contains(commitID CommitID) bool

func (CommitParents) Identity 

func (cp CommitParents) Identity() []byte

type CommitRecord 

type CommitRecord struct {
	CommitID CommitID `db:"id"`
	*Commit
}

CommitRecord holds CommitID with the associated Commit data

type CommitVersion added in v0.40.0 

type CommitVersion int

CommitVersion used to track changes in Commit schema. Each version is change that a constant describes. 

const (
	CommitVersionInitial CommitVersion = iota
	CommitVersionParentSwitch

	CurrentCommitVersion = CommitVersionParentSwitch
)

type CommittedManager 

type CommittedManager interface {
	// Get returns the provided key, if exists, from the provided MetaRangeID
	Get(ctx context.Context, ns StorageNamespace, rangeID MetaRangeID, key Key) (*Value, error)

	// Exists returns true if a MetaRange matching ID exists in namespace ns.
	Exists(ctx context.Context, ns StorageNamespace, id MetaRangeID) (bool, error)

	// WriteMetaRange flushes the iterator to a new MetaRange and returns the created ID.
	WriteMetaRange(ctx context.Context, ns StorageNamespace, it ValueIterator, metadata Metadata) (*MetaRangeID, error)

	// List takes a given tree and returns an ValueIterator
	List(ctx context.Context, ns StorageNamespace, rangeID MetaRangeID) (ValueIterator, error)

	// Diff receives two metaRanges and returns a DiffIterator describing all differences between them.
	// This is similar to a two-dot diff in git (left..right)
	Diff(ctx context.Context, ns StorageNamespace, left, right MetaRangeID) (DiffIterator, error)

	// Compare returns the difference between 'source' and 'destination', relative to a merge base 'base'.
	// This is similar to a three-dot diff in git.
	Compare(ctx context.Context, ns StorageNamespace, destination, source, base MetaRangeID) (DiffIterator, error)

	// Merge applies changes from 'source' to 'destination', relative to a merge base 'base' and
	// returns the ID of the new metarange and a summary of diffs.  This is similar to a
	// git merge operation. The resulting tree is expected to be immediately addressable.
	Merge(ctx context.Context, ns StorageNamespace, destination, source, base MetaRangeID) (MetaRangeID, DiffSummary, error)

	// Apply is the act of taking an existing metaRange (snapshot) and applying a set of changes to it.
	// A change is either an entity to write/overwrite, or a tombstone to mark a deletion
	// it returns a new MetaRangeID that is expected to be immediately addressable
	Apply(ctx context.Context, ns StorageNamespace, rangeID MetaRangeID, iterator ValueIterator) (MetaRangeID, DiffSummary, error)

	// GetMetaRange returns information where metarangeID is stored.
	GetMetaRange(ctx context.Context, ns StorageNamespace, metaRangeID MetaRangeID) (MetaRangeInfo, error)
	// GetRange returns information where rangeID is stored.
	GetRange(ctx context.Context, ns StorageNamespace, rangeID RangeID) (RangeInfo, error)
}

CommittedManager reads and applies committed snapshots it is responsible for de-duping them, persisting them and providing basic diff, merge and list capabilities

type Diff 

type Diff struct {
	Type         DiffType
	Key          Key
	Value        *Value
	LeftIdentity []byte // the Identity of the value on the left side of the diff
}

Diff represents a change in value based on key

func (*Diff) Copy 

func (d *Diff) Copy() *Diff

type DiffIterator 

type DiffIterator interface {
	Next() bool
	SeekGE(id Key)
	Value() *Diff
	Err() error
	Close()
}

func NewUncommittedDiffIterator 

func NewUncommittedDiffIterator(ctx context.Context, committedList ValueIterator, uncommittedList ValueIterator, sn StorageNamespace, metaRangeID MetaRangeID) DiffIterator

NewUncommittedDiffIterator lists uncommitted changes as a diff. If `metaRangeID` is empty then there is no commit and it returns all objects as added

type DiffSummary 

type DiffSummary struct {
	Count map[DiffType]int
}

type DiffType 

type DiffType uint8

DiffType represents the type of the change 

const (
	DiffTypeAdded DiffType = iota
	DiffTypeRemoved
	DiffTypeChanged
	DiffTypeConflict
)

type Dumper 

type Dumper interface {
	// DumpCommits iterates through all commits and dumps them in Graveler format
	DumpCommits(ctx context.Context, repositoryID RepositoryID) (*MetaRangeID, error)

	// DumpBranches iterates through all branches and dumps them in Graveler format
	DumpBranches(ctx context.Context, repositoryID RepositoryID) (*MetaRangeID, error)

	// DumpTags iterates through all tags and dumps them in Graveler format
	DumpTags(ctx context.Context, repositoryID RepositoryID) (*MetaRangeID, error)
}

type EventType 

type EventType string
const (
	EventTypePreCommit  EventType = "pre-commit"
	EventTypePostCommit EventType = "post-commit"
	EventTypePreMerge   EventType = "pre-merge"
	EventTypePostMerge  EventType = "post-merge"
)

type GarbageCollectionManager added in v0.43.0 

type GarbageCollectionManager interface {
	GetRules(ctx context.Context, storageNamespace StorageNamespace) (*GarbageCollectionRules, error)
	SaveRules(ctx context.Context, storageNamespace StorageNamespace, rules *GarbageCollectionRules) error

	SaveGarbageCollectionCommits(ctx context.Context, storageNamespace StorageNamespace, repositoryID RepositoryID, rules *GarbageCollectionRules, previouslyExpiredCommits []CommitID) (string, error)
	GetRunExpiredCommits(ctx context.Context, storageNamespace StorageNamespace, runID string) ([]CommitID, error)
}

type GarbageCollectionRules added in v0.43.0 

type GarbageCollectionRules struct {
	DefaultRetentionDays int32            `protobuf:"varint,1,opt,name=default_retention_days,json=defaultRetentionDays,proto3" json:"default_retention_days,omitempty"`
	BranchRetentionDays  map[string]int32 `` /* 209-byte string literal not displayed */
	// contains filtered or unexported fields
}

func (*GarbageCollectionRules) Descriptor deprecated added in v0.43.0 

func (*GarbageCollectionRules) Descriptor() ([]byte, []int)

Deprecated: Use GarbageCollectionRules.ProtoReflect.Descriptor instead.

func (*GarbageCollectionRules) GetBranchRetentionDays added in v0.43.0 

func (x *GarbageCollectionRules) GetBranchRetentionDays() map[string]int32

func (*GarbageCollectionRules) GetDefaultRetentionDays added in v0.43.0 

func (x *GarbageCollectionRules) GetDefaultRetentionDays() int32

func (*GarbageCollectionRules) ProtoMessage added in v0.43.0 

func (*GarbageCollectionRules) ProtoMessage()

func (*GarbageCollectionRules) ProtoReflect added in v0.43.0 

func (x *GarbageCollectionRules) ProtoReflect() protoreflect.Message

func (*GarbageCollectionRules) Reset added in v0.43.0 

func (x *GarbageCollectionRules) Reset()

func (*GarbageCollectionRules) String added in v0.43.0 

func (x *GarbageCollectionRules) String() string

type Graveler 

type Graveler struct {
	CommittedManager CommittedManager
	StagingManager   StagingManager
	RefManager       RefManager
	// contains filtered or unexported fields
}

func NewGraveler 

func NewGraveler(branchLocker BranchLocker, committedManager CommittedManager, stagingManager StagingManager, refManager RefManager, gcManager GarbageCollectionManager) *Graveler

func (*Graveler) AddCommit 

func (g *Graveler) AddCommit(ctx context.Context, repositoryID RepositoryID, commit Commit) (CommitID, error)

func (*Graveler) AddCommitToBranchHead 

func (g *Graveler) AddCommitToBranchHead(ctx context.Context, repositoryID RepositoryID, branchID BranchID, commit Commit) (CommitID, error)

func (*Graveler) Commit 

func (g *Graveler) Commit(ctx context.Context, repositoryID RepositoryID, branchID BranchID, params CommitParams) (CommitID, error)

func (*Graveler) Compare 

func (g *Graveler) Compare(ctx context.Context, repositoryID RepositoryID, from, to Ref) (DiffIterator, error)

func (*Graveler) CreateBareRepository 

func (g *Graveler) CreateBareRepository(ctx context.Context, repositoryID RepositoryID, storageNamespace StorageNamespace, defaultBranchID BranchID) (*Repository, error)

func (*Graveler) CreateBranch 

func (g *Graveler) CreateBranch(ctx context.Context, repositoryID RepositoryID, branchID BranchID, ref Ref) (*Branch, error)

func (*Graveler) CreateRepository 

func (g *Graveler) CreateRepository(ctx context.Context, repositoryID RepositoryID, storageNamespace StorageNamespace, branchID BranchID) (*Repository, error)

func (*Graveler) CreateTag 

func (g *Graveler) CreateTag(ctx context.Context, repositoryID RepositoryID, tagID TagID, commitID CommitID) error

func (*Graveler) Delete 

func (g *Graveler) Delete(ctx context.Context, repositoryID RepositoryID, branchID BranchID, key Key) error

func (*Graveler) DeleteBranch 

func (g *Graveler) DeleteBranch(ctx context.Context, repositoryID RepositoryID, branchID BranchID) error

func (*Graveler) DeleteRepository 

func (g *Graveler) DeleteRepository(ctx context.Context, repositoryID RepositoryID) error

func (*Graveler) DeleteTag 

func (g *Graveler) DeleteTag(ctx context.Context, repositoryID RepositoryID, tagID TagID) error

func (*Graveler) Dereference 

func (g *Graveler) Dereference(ctx context.Context, repositoryID RepositoryID, ref Ref) (CommitID, error)

func (*Graveler) Diff 

func (g *Graveler) Diff(ctx context.Context, repositoryID RepositoryID, left, right Ref) (DiffIterator, error)

func (*Graveler) DiffUncommitted 

func (g *Graveler) DiffUncommitted(ctx context.Context, repositoryID RepositoryID, branchID BranchID) (DiffIterator, error)

func (*Graveler) DumpBranches 

func (g *Graveler) DumpBranches(ctx context.Context, repositoryID RepositoryID) (*MetaRangeID, error)

func (*Graveler) DumpCommits 

func (g *Graveler) DumpCommits(ctx context.Context, repositoryID RepositoryID) (*MetaRangeID, error)

func (*Graveler) DumpTags 

func (g *Graveler) DumpTags(ctx context.Context, repositoryID RepositoryID) (*MetaRangeID, error)

func (*Graveler) Get 

func (g *Graveler) Get(ctx context.Context, repositoryID RepositoryID, ref Ref, key Key) (*Value, error)

func (*Graveler) GetBranch 

func (g *Graveler) GetBranch(ctx context.Context, repositoryID RepositoryID, branchID BranchID) (*Branch, error)

func (*Graveler) GetCommit 

func (g *Graveler) GetCommit(ctx context.Context, repositoryID RepositoryID, commitID CommitID) (*Commit, error)

func (*Graveler) GetGarbageCollectionRules added in v0.43.0 

func (g *Graveler) GetGarbageCollectionRules(ctx context.Context, repositoryID RepositoryID) (*GarbageCollectionRules, error)

func (*Graveler) GetMetaRange 

func (g *Graveler) GetMetaRange(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) (MetaRangeInfo, error)

func (*Graveler) GetRange 

func (g *Graveler) GetRange(ctx context.Context, repositoryID RepositoryID, rangeID RangeID) (RangeInfo, error)

func (*Graveler) GetRepository 

func (g *Graveler) GetRepository(ctx context.Context, repositoryID RepositoryID) (*Repository, error)

func (*Graveler) GetStagingToken added in v0.40.0 

func (g *Graveler) GetStagingToken(ctx context.Context, repositoryID RepositoryID, branchID BranchID) (*StagingToken, error)

func (*Graveler) GetTag 

func (g *Graveler) GetTag(ctx context.Context, repositoryID RepositoryID, tagID TagID) (*CommitID, error)

func (*Graveler) List 

func (g *Graveler) List(ctx context.Context, repositoryID RepositoryID, ref Ref) (ValueIterator, error)

func (*Graveler) ListBranches 

func (g *Graveler) ListBranches(ctx context.Context, repositoryID RepositoryID) (BranchIterator, error)

func (*Graveler) ListRepositories 

func (g *Graveler) ListRepositories(ctx context.Context) (RepositoryIterator, error)

func (*Graveler) ListTags 

func (g *Graveler) ListTags(ctx context.Context, repositoryID RepositoryID) (TagIterator, error)

func (*Graveler) LoadBranches 

func (g *Graveler) LoadBranches(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error

func (*Graveler) LoadCommits 

func (g *Graveler) LoadCommits(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error

func (*Graveler) LoadTags 

func (g *Graveler) LoadTags(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error

func (*Graveler) Log 

func (g *Graveler) Log(ctx context.Context, repositoryID RepositoryID, commitID CommitID) (CommitIterator, error)

func (*Graveler) Merge 

func (g *Graveler) Merge(ctx context.Context, repositoryID RepositoryID, destination BranchID, source Ref, commitParams CommitParams) (CommitID, DiffSummary, error)

func (*Graveler) Reset 

func (g *Graveler) Reset(ctx context.Context, repositoryID RepositoryID, branchID BranchID) error

func (*Graveler) ResetKey 

func (g *Graveler) ResetKey(ctx context.Context, repositoryID RepositoryID, branchID BranchID, key Key) error

func (*Graveler) ResetPrefix 

func (g *Graveler) ResetPrefix(ctx context.Context, repositoryID RepositoryID, branchID BranchID, key Key) error

func (*Graveler) Revert 

func (g *Graveler) Revert(ctx context.Context, repositoryID RepositoryID, branchID BranchID, ref Ref, parentNumber int, commitParams CommitParams) (CommitID, DiffSummary, error)

Revert creates a reverse patch to the commit given as 'ref', and applies it as a new commit on the given branch. This is implemented by merging the parent of 'ref' into the branch, with 'ref' as the merge base. Example: consider the following tree: C1 -> C2 -> C3, with the branch pointing at C3. To revert C2, we merge C1 into the branch, with C2 as the merge base. That is, try to apply the diff from C2 to C1 on the tip of the branch. If the commit is a merge commit, 'parentNumber' is the parent number (1-based) relative to which the revert is done.

func (*Graveler) SaveGarbageCollectionCommits added in v0.43.0 

func (g *Graveler) SaveGarbageCollectionCommits(ctx context.Context, repositoryID RepositoryID, previousRunID string) (runID string, err error)

func (*Graveler) Set 

func (g *Graveler) Set(ctx context.Context, repositoryID RepositoryID, branchID BranchID, key Key, value Value, writeConditions ...WriteConditionOption) error

func (*Graveler) SetGarbageCollectionRules added in v0.43.0 

func (g *Graveler) SetGarbageCollectionRules(ctx context.Context, repositoryID RepositoryID, rules *GarbageCollectionRules) error

func (*Graveler) SetHooksHandler 

func (g *Graveler) SetHooksHandler(handler HooksHandler)

func (*Graveler) UpdateBranch 

func (g *Graveler) UpdateBranch(ctx context.Context, repositoryID RepositoryID, branchID BranchID, ref Ref) (*Branch, error)

func (*Graveler) WriteMetaRange 

func (g *Graveler) WriteMetaRange(ctx context.Context, repositoryID RepositoryID, it ValueIterator) (*MetaRangeID, error)

type HookAbortError 

type HookAbortError struct {
	EventType EventType
	RunID     string
	Err       error
}

HookAbortError abort by hook error, holds the event type with the run id to trace back the run

func (*HookAbortError) Error 

func (e *HookAbortError) Error() string

func (*HookAbortError) Unwrap 

func (e *HookAbortError) Unwrap() error

type HookRecord 

type HookRecord struct {
	RunID            string
	EventType        EventType
	RepositoryID     RepositoryID
	StorageNamespace StorageNamespace
	BranchID         BranchID
	SourceRef        Ref
	Commit           Commit
	CommitID         CommitID
	PreRunID         string
}

type HooksHandler 

type HooksHandler interface {
	PreCommitHook(ctx context.Context, record HookRecord) error
	PostCommitHook(ctx context.Context, record HookRecord) error
	PreMergeHook(ctx context.Context, record HookRecord) error
	PostMergeHook(ctx context.Context, record HookRecord) error
}

type HooksNoOp 

type HooksNoOp struct{}

func (*HooksNoOp) PostCommitHook 

func (h *HooksNoOp) PostCommitHook(context.Context, HookRecord) error

func (*HooksNoOp) PostMergeHook 

func (h *HooksNoOp) PostMergeHook(context.Context, HookRecord) error

func (*HooksNoOp) PreCommitHook 

func (h *HooksNoOp) PreCommitHook(context.Context, HookRecord) error

func (*HooksNoOp) PreMergeHook 

func (h *HooksNoOp) PreMergeHook(context.Context, HookRecord) error

type Key 

type Key []byte

Key represents a logical path for an value

func (Key) Copy 

func (id Key) Copy() Key

func (Key) String 

func (id Key) String() string

type KeyValueStore 

type KeyValueStore interface {
	// Get returns value from repository / reference by key, nil value is a valid value for tombstone
	// returns error if value does not exist
	Get(ctx context.Context, repositoryID RepositoryID, ref Ref, key Key) (*Value, error)

	// Set stores value on repository / branch by key. nil value is a valid value for tombstone
	Set(ctx context.Context, repositoryID RepositoryID, branchID BranchID, key Key, value Value, writeConditions ...WriteConditionOption) error

	// Delete value from repository / branch branch by key
	Delete(ctx context.Context, repositoryID RepositoryID, branchID BranchID, key Key) error

	// List lists values on repository / ref
	List(ctx context.Context, repositoryID RepositoryID, ref Ref) (ValueIterator, error)
}

type Loader 

type Loader interface {
	// LoadCommits iterates through all commits in Graveler format and loads them into repositoryID
	LoadCommits(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error

	// LoadBranches iterates through all branches in Graveler format and loads them into repositoryID
	LoadBranches(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error

	// LoadTags iterates through all tags in Graveler format and loads them into repositoryID
	LoadTags(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) error
}

type MetaRangeID 

type MetaRangeID string

MetaRangeID represents a snapshot of the MetaRange, referenced by a commit

type MetaRangeInfo 

type MetaRangeInfo struct {
	// URI of metarange file.
	Address string
}

type Metadata 

type Metadata map[string]string

Metadata key/value strings to holds metadata information on value and commit

type Plumbing 

type Plumbing interface {
	// GetMetaRange returns information where metarangeID is stored.
	GetMetaRange(ctx context.Context, repositoryID RepositoryID, metaRangeID MetaRangeID) (MetaRangeInfo, error)
	// GetRange returns information where rangeID is stored.
	GetRange(ctx context.Context, repositoryID RepositoryID, rangeID RangeID) (RangeInfo, error)
}

Plumbing includes commands for fiddling more directly with graveler implementation internals.

type RangeID 

type RangeID string

RangeID represents a part of a MetaRange, useful only for plumbing.

type RangeInfo 

type RangeInfo struct {
	// URI of range file.
	Address string
}

type Ref 

type Ref string

Ref could be a commit ID, a branch name, a Tag

func (Ref) String 

func (id Ref) String() string

type RefManager 

type RefManager interface {
	// GetRepository returns the Repository metadata object for the given RepositoryID
	GetRepository(ctx context.Context, repositoryID RepositoryID) (*Repository, error)

	// CreateRepository stores a new Repository under RepositoryID with the given Branch as default branch
	CreateRepository(ctx context.Context, repositoryID RepositoryID, repository Repository, token StagingToken) error

	// CreateBareRepository stores a new repository under RepositoryID without creating an initial commit and branch
	CreateBareRepository(ctx context.Context, repositoryID RepositoryID, repository Repository) error

	// ListRepositories lists repositories
	ListRepositories(ctx context.Context) (RepositoryIterator, error)

	// DeleteRepository deletes the repository
	DeleteRepository(ctx context.Context, repositoryID RepositoryID) error

	// RevParse returns the Reference matching the given Ref
	RevParse(ctx context.Context, repositoryID RepositoryID, ref Ref) (Reference, error)

	// GetBranch returns the Branch metadata object for the given BranchID
	GetBranch(ctx context.Context, repositoryID RepositoryID, branchID BranchID) (*Branch, error)

	// SetBranch points the given BranchID at the given Branch metadata
	SetBranch(ctx context.Context, repositoryID RepositoryID, branchID BranchID, branch Branch) error

	// DeleteBranch deletes the branch
	DeleteBranch(ctx context.Context, repositoryID RepositoryID, branchID BranchID) error

	// ListBranches lists branches
	ListBranches(ctx context.Context, repositoryID RepositoryID) (BranchIterator, error)

	// GetTag returns the Tag metadata object for the given TagID
	GetTag(ctx context.Context, repositoryID RepositoryID, tagID TagID) (*CommitID, error)

	// CreateTag create a given tag pointing to a commit
	CreateTag(ctx context.Context, repositoryID RepositoryID, tagID TagID, commitID CommitID) error

	// DeleteTag deletes the tag
	DeleteTag(ctx context.Context, repositoryID RepositoryID, tagID TagID) error

	// ListTags lists tags
	ListTags(ctx context.Context, repositoryID RepositoryID) (TagIterator, error)

	// GetCommit returns the Commit metadata object for the given CommitID.
	GetCommit(ctx context.Context, repositoryID RepositoryID, commitID CommitID) (*Commit, error)

	// AddCommit stores the Commit object, returning its ID
	AddCommit(ctx context.Context, repositoryID RepositoryID, commit Commit) (CommitID, error)

	// FindMergeBase returns the merge-base for the given CommitIDs
	// see: https://git-scm.com/docs/git-merge-base
	// and internally: https://github.com/treeverse/lakeFS/blob/09954804baeb36ada74fa17d8fdc13a38552394e/index/dag/commits.go
	FindMergeBase(ctx context.Context, repositoryID RepositoryID, commitIDs ...CommitID) (*Commit, error)

	// Log returns an iterator starting at commit ID up to repository root
	Log(ctx context.Context, repositoryID RepositoryID, commitID CommitID) (CommitIterator, error)

	// ListCommits returns an iterator over all known commits, ordered by their commit ID
	ListCommits(ctx context.Context, repositoryID RepositoryID) (CommitIterator, error)

	// FillGenerations computes and updates the generation field for all commits in a repository.
	// It should be used for restoring commits from a commit-dump which was performed before the field was introduced.
	FillGenerations(ctx context.Context, repositoryID RepositoryID) error
}

RefManager handles references: branches, commits, probably tags in the future it also handles the structure of the commit graph and its traversal (notably, merge-base and log)

type Reference 

type Reference interface {
	Type() ReferenceType
	Branch() Branch
	CommitID() CommitID
}

type ReferenceType 

type ReferenceType uint8

ReferenceType represents the type of the reference 

const (
	ReferenceTypeCommit ReferenceType = iota
	ReferenceTypeTag
	ReferenceTypeBranch
)

type Repository 

type Repository struct {
	StorageNamespace StorageNamespace `db:"storage_namespace"`
	CreationDate     time.Time        `db:"creation_date"`
	DefaultBranchID  BranchID         `db:"default_branch"`
}

Repository represents repository metadata

type RepositoryID 

type RepositoryID string

RepositoryID is an identifier for a repo

func (RepositoryID) String 

func (id RepositoryID) String() string

type RepositoryIterator 

type RepositoryIterator interface {
	Next() bool
	SeekGE(id RepositoryID)
	Value() *RepositoryRecord
	Err() error
	Close()
}

type RepositoryRecord 

type RepositoryRecord struct {
	RepositoryID RepositoryID `db:"id"`
	*Repository
}

type StagingManager 

type StagingManager interface {
	// Get returns the value for the provided staging token and key
	// Returns ErrNotFound if no value found on key.
	Get(ctx context.Context, st StagingToken, key Key) (*Value, error)

	// Set writes a (possibly nil) value under the given staging token and key.
	Set(ctx context.Context, st StagingToken, key Key, value *Value, overwrite bool) error

	// List returns a ValueIterator for the given staging token
	List(ctx context.Context, st StagingToken) (ValueIterator, error)

	// DropKey clears a value by staging token and key
	DropKey(ctx context.Context, st StagingToken, key Key) error

	// Drop clears the given staging area
	Drop(ctx context.Context, st StagingToken) error

	// DropByPrefix drops all keys starting with the given prefix, from the given staging area
	DropByPrefix(ctx context.Context, st StagingToken, prefix Key) error
}

StagingManager manages entries in a staging area, denoted by a staging token

type StagingToken 

type StagingToken string

StagingToken represents a namespace for writes to apply as uncommitted

type StorageNamespace 

type StorageNamespace string

StorageNamespace is the URI to the storage location

func (StorageNamespace) String 

func (ns StorageNamespace) String() string

type TagData 

type TagData struct {
	Id       string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
	CommitId string `protobuf:"bytes,2,opt,name=commit_id,json=commitId,proto3" json:"commit_id,omitempty"`
	// contains filtered or unexported fields
}

func (*TagData) Descriptor deprecated 

func (*TagData) Descriptor() ([]byte, []int)

Deprecated: Use TagData.ProtoReflect.Descriptor instead.

func (*TagData) GetCommitId 

func (x *TagData) GetCommitId() string

func (*TagData) GetId 

func (x *TagData) GetId() string

func (*TagData) ProtoMessage 

func (*TagData) ProtoMessage()

func (*TagData) ProtoReflect 

func (x *TagData) ProtoReflect() protoreflect.Message

func (*TagData) Reset 

func (x *TagData) Reset()

func (*TagData) String 

func (x *TagData) String() string

type TagID 

type TagID string

TagID represents a named tag pointing at a commit

func (TagID) String 

func (id TagID) String() string

type TagIterator 

type TagIterator interface {
	Next() bool
	SeekGE(id TagID)
	Value() *TagRecord
	Err() error
	Close()
}

type TagRecord 

type TagRecord struct {
	TagID    TagID
	CommitID CommitID
}

TagRecord holds TagID with the associated Tag data

type Value 

type Value struct {
	Identity []byte `db:"identity"`
	Data     []byte `db:"data"`
}

Value represents metadata or a given object (modified date, physical address, etc)

type ValueIterator 

type ValueIterator interface {
	Next() bool
	SeekGE(id Key)
	Value() *ValueRecord
	Err() error
	Close()
}

type ValueRecord 

type ValueRecord struct {
	Key Key `db:"key"`
	*Value
}

ValueRecord holds Key with the associated Value information

func (*ValueRecord) IsTombstone 

func (v *ValueRecord) IsTombstone() bool

type VersionController 

type VersionController interface {
	// GetRepository returns the Repository metadata object for the given RepositoryID
	GetRepository(ctx context.Context, repositoryID RepositoryID) (*Repository, error)

	// CreateRepository stores a new Repository under RepositoryID with the given Branch as default branch
	CreateRepository(ctx context.Context, repositoryID RepositoryID, storageNamespace StorageNamespace, branchID BranchID) (*Repository, error)

	// CreateBareRepository stores a new Repository under RepositoryID with no initial branch or commit
	CreateBareRepository(ctx context.Context, repositoryID RepositoryID, storageNamespace StorageNamespace, defaultBranchID BranchID) (*Repository, error)

	// ListRepositories returns iterator to scan repositories
	ListRepositories(ctx context.Context) (RepositoryIterator, error)

	// DeleteRepository deletes the repository
	DeleteRepository(ctx context.Context, repositoryID RepositoryID) error

	// CreateBranch creates branch on repository pointing to ref
	CreateBranch(ctx context.Context, repositoryID RepositoryID, branchID BranchID, ref Ref) (*Branch, error)

	// UpdateBranch updates branch on repository pointing to ref
	UpdateBranch(ctx context.Context, repositoryID RepositoryID, branchID BranchID, ref Ref) (*Branch, error)

	// GetBranch gets branch information by branch / repository id
	GetBranch(ctx context.Context, repositoryID RepositoryID, branchID BranchID) (*Branch, error)

	// GetTag gets tag's commit id
	GetTag(ctx context.Context, repositoryID RepositoryID, tagID TagID) (*CommitID, error)

	// CreateTag creates tag on a repository pointing to a commit id
	CreateTag(ctx context.Context, repositoryID RepositoryID, tagID TagID, commitID CommitID) error

	// DeleteTag remove tag from a repository
	DeleteTag(ctx context.Context, repositoryID RepositoryID, tagID TagID) error

	// ListTags lists tags on a repository
	ListTags(ctx context.Context, repositoryID RepositoryID) (TagIterator, error)

	// Log returns an iterator starting at commit ID up to repository root
	Log(ctx context.Context, repositoryID RepositoryID, commitID CommitID) (CommitIterator, error)

	// ListBranches lists branches on repositories
	ListBranches(ctx context.Context, repositoryID RepositoryID) (BranchIterator, error)

	// DeleteBranch deletes branch from repository
	DeleteBranch(ctx context.Context, repositoryID RepositoryID, branchID BranchID) error

	// Commit the staged data and returns a commit ID that references that change
	//   ErrNothingToCommit in case there is no data in stage
	Commit(ctx context.Context, repositoryID RepositoryID, branchID BranchID, commitParams CommitParams) (CommitID, error)

	// WriteMetaRange accepts a ValueIterator and writes the entire iterator to a new MetaRange
	// and returns the result ID.
	WriteMetaRange(ctx context.Context, repositoryID RepositoryID, it ValueIterator) (*MetaRangeID, error)

	// AddCommitToBranchHead creates a commit in the branch from the given pre-existing tree.
	// Returns ErrMetaRangeNotFound if the referenced metaRangeID doesn't exist.
	// Returns ErrCommitNotHeadBranch if the branch is no longer referencing to the parentCommit
	AddCommitToBranchHead(ctx context.Context, repositoryID RepositoryID, branchID BranchID, commit Commit) (CommitID, error)

	// AddCommit creates a dangling (no referencing branch) commit in the repo from the pre-existing commit.
	// Returns ErrMetaRangeNotFound if the referenced metaRangeID doesn't exist.
	AddCommit(ctx context.Context, repositoryID RepositoryID, commit Commit) (CommitID, error)

	// GetCommit returns the Commit metadata object for the given CommitID
	GetCommit(ctx context.Context, repositoryID RepositoryID, commitID CommitID) (*Commit, error)

	// Dereference returns the commit ID based on 'ref' reference
	Dereference(ctx context.Context, repositoryID RepositoryID, ref Ref) (CommitID, error)

	// Reset throws all staged data on the repository / branch
	Reset(ctx context.Context, repositoryID RepositoryID, branchID BranchID) error

	// ResetKey throws all staged data under the specified key on the repository / branch
	ResetKey(ctx context.Context, repositoryID RepositoryID, branchID BranchID, key Key) error

	// ResetPrefix throws all staged data starting with the given prefix on the repository / branch
	ResetPrefix(ctx context.Context, repositoryID RepositoryID, branchID BranchID, key Key) error

	// Revert creates a reverse patch to the commit given as 'ref', and applies it as a new commit on the given branch.
	Revert(ctx context.Context, repositoryID RepositoryID, branchID BranchID, ref Ref, parentNumber int, commitParams CommitParams) (CommitID, DiffSummary, error)

	// Merge merges 'source' into 'destination' and returns the commit id for the created merge commit, and a summary of results.
	Merge(ctx context.Context, repositoryID RepositoryID, destination BranchID, source Ref, commitParams CommitParams) (CommitID, DiffSummary, error)

	// DiffUncommitted returns iterator to scan the changes made on the branch
	DiffUncommitted(ctx context.Context, repositoryID RepositoryID, branchID BranchID) (DiffIterator, error)

	// Diff returns the changes between 'left' and 'right' ref.
	// This is similar to a two-dot (left..right) diff in git.
	Diff(ctx context.Context, repositoryID RepositoryID, left, right Ref) (DiffIterator, error)

	// Compare returns the difference between the commit where 'to' was last synced into 'from', and the most recent commit of `from`.
	// This is similar to a three-dot (from...to) diff in git.
	Compare(ctx context.Context, repositoryID RepositoryID, from, to Ref) (DiffIterator, error)

	// SetHooksHandler set handler for all graveler hooks
	SetHooksHandler(handler HooksHandler)

	// GetStagingToken returns the token identifying current staging for branchID of
	// repositoryID.
	GetStagingToken(ctx context.Context, repositoryID RepositoryID, branchID BranchID) (*StagingToken, error)

	GetGarbageCollectionRules(ctx context.Context, repositoryID RepositoryID) (*GarbageCollectionRules, error)

	SetGarbageCollectionRules(ctx context.Context, repositoryID RepositoryID, rules *GarbageCollectionRules) error

	// SaveGarbageCollectionCommits saves the sets of active and expired commits, according to the branch rules for garbage collection.
	// Returns a run id which can later be used to retrieve the set of commits.
	// If a previousRunID is specified, commits that were already expired and their ancestors will not be considered as expired/active.
	// Note: Ancestors of previously expired commits may still be considered if they can be reached from a non-expired commit.
	SaveGarbageCollectionCommits(ctx context.Context, repositoryID RepositoryID, previousRunID string) (runID string, err error)
}

type WriteCondition added in v0.40.3 

type WriteCondition struct {
	IfAbsent bool
}

type WriteConditionOption added in v0.40.3 

type WriteConditionOption func(condition *WriteCondition)

func IfAbsent added in v0.40.3 

func IfAbsent(v bool) WriteConditionOption

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.