catalog

package
v0.33.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 17, 2021 License: Apache-2.0 Imports: 34 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	RangeFSName     = "range"
	MetaRangeFSName = "meta-range"
)
View Source 
const (
	ListRepositoriesLimitMax = 1000
	ListBranchesLimitMax     = 1000
	ListTagsLimitMax         = 1000
	DiffLimitMax             = 1000
	ListEntriesLimitMax      = 10000
)
View Source 
const (
	DBEntryFieldChecksum        = "checksum"
	DBEntryFieldPhysicalAddress = "physical_address"
)
View Source 
const (
	DefaultPathDelimiter = "/"
)
View Source 
const (
	MaxPathLength = 1024
)

Variables

View Source 
var (
	ErrNotFound                 = db.ErrNotFound
	ErrInvalidMetadataSrcFormat = errors.New("invalid metadata src format")
	ErrExpired                  = errors.New("expired from storage")
	ErrFeatureNotSupported      = errors.New("feature not supported")
	ErrBranchNotFound           = fmt.Errorf("branch %w", ErrNotFound)
	ErrRepositoryNotFound       = fmt.Errorf("repository %w", ErrNotFound)
	ErrInvalidValue             = errors.New("invalid value")
	ErrNoDifferenceWasFound     = errors.New("no difference was found")
	ErrConflictFound            = errors.New("conflict found")
)
View Source 
var (
	ErrInvalidType   = errors.New("invalid type")
	ErrRequiredValue = errors.New("required value")
)
View Source 
var ErrUnknownDiffType = errors.New("unknown graveler difference type")
View Source 
var File_catalog_proto protoreflect.FileDescriptor
View Source 
var ValidatePathOptional = MakeValidateOptional(ValidatePath)
View Source 
var ValidateTagIDOptional = MakeValidateOptional(ValidateTagID)

Functions

func EntryToValue 

func EntryToValue(entry *Entry) (*graveler.Value, error)

func MustEntryToValue 

func MustEntryToValue(entry *Entry) *graveler.Value

func NewEntryToValueIterator 

func NewEntryToValueIterator(it EntryIterator) *entryValueIterator

func NewValueToEntryIterator 

func NewValueToEntryIterator(it graveler.ValueIterator) *valueEntryIterator

func Validate 

func Validate(args []ValidateArg) error

func ValidateBranchID 

func ValidateBranchID(v interface{}) error

func ValidateCommitID 

func ValidateCommitID(v interface{}) error

func ValidateNonNegativeInt 

func ValidateNonNegativeInt(v interface{}) error

func ValidatePath 

func ValidatePath(v interface{}) error

func ValidateRef 

func ValidateRef(v interface{}) error

func ValidateRepositoryID 

func ValidateRepositoryID(v interface{}) error

func ValidateRequiredString 

func ValidateRequiredString(v interface{}) error

func ValidateStorageNamespace 

func ValidateStorageNamespace(v interface{}) error

func ValidateTagID 

func ValidateTagID(v interface{}) error

Types

type ActionsOutputWriter 

type ActionsOutputWriter struct {
	// contains filtered or unexported fields
}

func NewActionsOutputWriter 

func NewActionsOutputWriter(blockAdapter block.Adapter) *ActionsOutputWriter

func (*ActionsOutputWriter) OutputWrite 

func (o *ActionsOutputWriter) OutputWrite(ctx context.Context, storageNamespace, name string, reader io.Reader, size int64) error

type ActionsSource 

type ActionsSource struct {
	// contains filtered or unexported fields
}

func NewActionsSource 

func NewActionsSource(catalog *Catalog) *ActionsSource

func (*ActionsSource) List 

func (s *ActionsSource) List(ctx context.Context, record graveler.HookRecord) ([]string, error)

func (*ActionsSource) Load 

func (s *ActionsSource) Load(ctx context.Context, record graveler.HookRecord, name string) ([]byte, error)

type Branch 

type Branch struct {
	Name      string `db:"name"`
	Reference string
}

type Catalog 

type Catalog struct {
	BlockAdapter block.Adapter
	Store        Store
	// contains filtered or unexported fields
}

func New 

func New(ctx context.Context, cfg Config) (*Catalog, error)

func (*Catalog) BranchExists 

func (c *Catalog) BranchExists(ctx context.Context, repository string, branch string) (bool, error)

func (*Catalog) Close 

func (c *Catalog) Close() error

func (*Catalog) Commit 

func (c *Catalog) Commit(ctx context.Context, repository string, branch string, message string, committer string, metadata Metadata) (*CommitLog, error)

func (*Catalog) Compare 

func (c *Catalog) Compare(ctx context.Context, repository, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)

func (*Catalog) CreateBareRepository 

func (c *Catalog) CreateBareRepository(ctx context.Context, repository string, storageNamespace string, defaultBranchID string) (*Repository, error)

CreateBareRepository creates a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with no initial branch or commit

func (*Catalog) CreateBranch 

func (c *Catalog) CreateBranch(ctx context.Context, repository string, branch string, sourceBranch string) (*CommitLog, error)

func (*Catalog) CreateEntries 

func (c *Catalog) CreateEntries(ctx context.Context, repository string, branch string, entries []DBEntry) error

func (*Catalog) CreateEntry 

func (c *Catalog) CreateEntry(ctx context.Context, repository string, branch string, entry DBEntry) error

func (*Catalog) CreateRepository 

func (c *Catalog) CreateRepository(ctx context.Context, repository string, storageNamespace string, branch string) (*Repository, error)

CreateRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with default branch name 'branch'

func (*Catalog) CreateTag 

func (c *Catalog) CreateTag(ctx context.Context, repository string, tagID string, ref string) (string, error)

func (*Catalog) DeleteBranch 

func (c *Catalog) DeleteBranch(ctx context.Context, repository string, branch string) error

func (*Catalog) DeleteEntry 

func (c *Catalog) DeleteEntry(ctx context.Context, repository string, branch string, path string) error

func (*Catalog) DeleteRepository 

func (c *Catalog) DeleteRepository(ctx context.Context, repository string) error

DeleteRepository delete a repository

func (*Catalog) DeleteTag 

func (c *Catalog) DeleteTag(ctx context.Context, repository string, tagID string) error

func (*Catalog) Diff 

func (c *Catalog) Diff(ctx context.Context, repository string, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)

func (*Catalog) DiffUncommitted 

func (c *Catalog) DiffUncommitted(ctx context.Context, repository string, branch string, limit int, after string) (Differences, bool, error)

func (*Catalog) DumpBranches 

func (c *Catalog) DumpBranches(ctx context.Context, repositoryID string) (string, error)

func (*Catalog) DumpCommits 

func (c *Catalog) DumpCommits(ctx context.Context, repositoryID string) (string, error)

func (*Catalog) DumpTags 

func (c *Catalog) DumpTags(ctx context.Context, repositoryID string) (string, error)

func (*Catalog) GetBranchReference 

func (c *Catalog) GetBranchReference(ctx context.Context, repository string, branch string) (string, error)

func (*Catalog) GetCommit 

func (c *Catalog) GetCommit(ctx context.Context, repository string, reference string) (*CommitLog, error)

func (*Catalog) GetEntry 

func (c *Catalog) GetEntry(ctx context.Context, repository string, reference string, path string, _ GetEntryParams) (*DBEntry, error)

GetEntry returns the current entry for path in repository branch reference. Returns the entry with ExpiredError if it has expired from underlying storage.

func (*Catalog) GetMetaRange 

func (c *Catalog) GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeInfo, error)

func (*Catalog) GetRange 

func (c *Catalog) GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeInfo, error)

func (*Catalog) GetRepository 

func (c *Catalog) GetRepository(ctx context.Context, repository string) (*Repository, error)

GetRepository get repository information

func (*Catalog) GetTag 

func (c *Catalog) GetTag(ctx context.Context, repository string, tagID string) (string, error)

func (*Catalog) ListBranches 

func (c *Catalog) ListBranches(ctx context.Context, repository string, prefix string, limit int, after string) ([]*Branch, bool, error)

func (*Catalog) ListCommits 

func (c *Catalog) ListCommits(ctx context.Context, repository string, branch string, fromReference string, limit int) ([]*CommitLog, bool, error)

func (*Catalog) ListEntries 

func (c *Catalog) ListEntries(ctx context.Context, repository string, reference string, prefix string, after string, delimiter string, limit int) ([]*DBEntry, bool, error)

func (*Catalog) ListRepositories 

func (c *Catalog) ListRepositories(ctx context.Context, limit int, after string) ([]*Repository, bool, error)

ListRepositories list repositories information, the bool returned is true when more repositories can be listed. In this case pass the last repository name as 'after' on the next call to ListRepositories

func (*Catalog) ListTags 

func (c *Catalog) ListTags(ctx context.Context, repository string, limit int, after string) ([]*Tag, bool, error)

func (*Catalog) LoadBranches 

func (c *Catalog) LoadBranches(ctx context.Context, repositoryID, branchesMetaRangeID string) error

func (*Catalog) LoadCommits 

func (c *Catalog) LoadCommits(ctx context.Context, repositoryID, commitsMetaRangeID string) error

func (*Catalog) LoadTags 

func (c *Catalog) LoadTags(ctx context.Context, repositoryID, tagsMetaRangeID string) error

func (*Catalog) Merge 

func (c *Catalog) Merge(ctx context.Context, repository string, destinationBranch string, sourceRef string, committer string, message string, metadata Metadata) (*MergeResult, error)

func (*Catalog) ResetBranch 

func (c *Catalog) ResetBranch(ctx context.Context, repository string, branch string) error

func (*Catalog) ResetEntries 

func (c *Catalog) ResetEntries(ctx context.Context, repository string, branch string, prefix string) error

func (*Catalog) ResetEntry 

func (c *Catalog) ResetEntry(ctx context.Context, repository string, branch string, path string) error

func (*Catalog) Revert 

func (c *Catalog) Revert(ctx context.Context, repository string, branch string, params RevertParams) error

func (*Catalog) RollbackCommit 

func (c *Catalog) RollbackCommit(_ context.Context, _ string, _ string, _ string) error

func (*Catalog) SetHooksHandler 

func (c *Catalog) SetHooksHandler(hooks graveler.HooksHandler)

type CommitLog 

type CommitLog struct {
	Reference    string
	Committer    string    `db:"committer"`
	Message      string    `db:"message"`
	CreationDate time.Time `db:"creation_date"`
	Metadata     Metadata  `db:"metadata"`
	MetaRangeID  string    `db:"meta_range_id"`
	Parents      []string
}

type Config 

type Config struct {
	Config *config.Config
	DB     db.Database
	LockDB db.Database
}

type DBEntry 

type DBEntry struct {
	CommonLevel     bool
	Path            string    `db:"path"`
	PhysicalAddress string    `db:"physical_address"`
	CreationDate    time.Time `db:"creation_date"`
	Size            int64     `db:"size"`
	Checksum        string    `db:"checksum"`
	Metadata        Metadata  `db:"metadata"`
	Expired         bool      `db:"is_expired"`
}

type DiffParams 

type DiffParams struct {
	Limit            int
	After            string
	AdditionalFields []string // db fields names that will be load in additional to Path on Difference's Entry
}

type DiffResultRecord 

type DiffResultRecord struct {
	TargetEntryNotInDirectBranch bool // the entry is reflected via lineage, NOT in the branch itself
	Difference
	EntryCtid *string // CTID of the modified/added entry. Do not use outside of catalog diff-by-iterators. https://github.com/treeverse/lakeFS/issues/831
}

type Difference 

type Difference struct {
	DBEntry                // Partially filled. Path is always set.
	Type    DifferenceType `db:"diff_type"`
}

func (Difference) String 

func (d Difference) String() string

type DifferenceType 

type DifferenceType int
const (
	DifferenceTypeAdded DifferenceType = iota
	DifferenceTypeRemoved
	DifferenceTypeChanged
	DifferenceTypeConflict
	DifferenceTypeNone
)

type Differences 

type Differences []Difference

func (Differences) Equal 

func (d Differences) Equal(other Differences) bool

type Entry 

type Entry struct {
	Address      string                 `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
	LastModified *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=last_modified,json=lastModified,proto3" json:"last_modified,omitempty"`
	Size         int64                  `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
	ETag         string                 `protobuf:"bytes,4,opt,name=e_tag,json=eTag,proto3" json:"e_tag,omitempty"`
	Metadata     map[string]string      `` /* 157-byte string literal not displayed */
	// contains filtered or unexported fields
}

func EntryFromCatalogEntry 

func EntryFromCatalogEntry(entry DBEntry) *Entry

func ValueToEntry 

func ValueToEntry(value *graveler.Value) (*Entry, error)

func (*Entry) Descriptor deprecated 

func (*Entry) Descriptor() ([]byte, []int)

Deprecated: Use Entry.ProtoReflect.Descriptor instead.

func (*Entry) GetAddress 

func (x *Entry) GetAddress() string

func (*Entry) GetETag 

func (x *Entry) GetETag() string

func (*Entry) GetLastModified 

func (x *Entry) GetLastModified() *timestamppb.Timestamp

func (*Entry) GetMetadata 

func (x *Entry) GetMetadata() map[string]string

func (*Entry) GetSize 

func (x *Entry) GetSize() int64

func (*Entry) ProtoMessage 

func (*Entry) ProtoMessage()

func (*Entry) ProtoReflect 

func (x *Entry) ProtoReflect() protoreflect.Message

func (*Entry) Reset 

func (x *Entry) Reset()

func (*Entry) String 

func (x *Entry) String() string

type EntryDiff 

type EntryDiff struct {
	Type  graveler.DiffType
	Path  Path
	Entry *Entry
}

type EntryDiffIterator 

type EntryDiffIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryDiff
	Err() error
	Close()
}

func NewEntryDiffIterator 

func NewEntryDiffIterator(it graveler.DiffIterator) EntryDiffIterator

type EntryIterator 

type EntryIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryRecord
	Err() error
	Close()
}

func NewPrefixIterator 

func NewPrefixIterator(it EntryIterator, prefix Path) EntryIterator

type EntryListing 

type EntryListing struct {
	CommonPrefix bool
	Path
	*Entry
}

type EntryListingIterator 

type EntryListingIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryListing
	Err() error
	Close()
}

func NewEntryListingIterator 

func NewEntryListingIterator(it EntryIterator, prefix Path, delimiter Path) EntryListingIterator

type EntryRecord 

type EntryRecord struct {
	Path Path
	*Entry
}

type ExpireResult 

type ExpireResult struct {
	Repository        string
	Branch            string
	PhysicalAddress   string
	InternalReference string
}

type ExpiryRows 

type ExpiryRows interface {
	Close()
	Next() bool
	Err() error
	// Read returns the current from ExpiryRows, or an error on failure.  Call it only after
	// successfully calling Next.
	Read() (*ExpireResult, error)
}

ExpiryRows is a database iterator over ExpiryResults. Use Next to advance from row to row.

type GetEntryParams 

type GetEntryParams struct {
	// For entries to expired objects the Expired bit is set.  If true, GetEntry returns
	// successfully for expired entries, otherwise it returns the entry with ErrExpired.
	ReturnExpired bool
}

GetEntryParams configures what entries GetEntry returns.

type Interface 

type Interface interface {
	// CreateRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with default branch name 'branch'
	CreateRepository(ctx context.Context, repository string, storageNamespace string, branch string) (*Repository, error)

	// CreateBareRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with no initial branch or commit
	// defaultBranchID will point to a non-existent branch on creation, it is up to the caller to eventually create it.
	CreateBareRepository(ctx context.Context, repository string, storageNamespace string, defaultBranchID string) (*Repository, error)

	// GetRepository get repository information
	GetRepository(ctx context.Context, repository string) (*Repository, error)

	// DeleteRepository delete a repository
	DeleteRepository(ctx context.Context, repository string) error

	// ListRepositories list repositories information, the bool returned is true when more repositories can be listed.
	// In this case pass the last repository name as 'after' on the next call to ListRepositories
	ListRepositories(ctx context.Context, limit int, after string) ([]*Repository, bool, error)

	CreateBranch(ctx context.Context, repository, branch string, sourceRef string) (*CommitLog, error)
	DeleteBranch(ctx context.Context, repository, branch string) error
	ListBranches(ctx context.Context, repository string, prefix string, limit int, after string) ([]*Branch, bool, error)
	BranchExists(ctx context.Context, repository string, branch string) (bool, error)
	GetBranchReference(ctx context.Context, repository, branch string) (string, error)
	ResetBranch(ctx context.Context, repository, branch string) error

	CreateTag(ctx context.Context, repository, tagID string, ref string) (string, error)
	DeleteTag(ctx context.Context, repository, tagID string) error
	ListTags(ctx context.Context, repository string, limit int, after string) ([]*Tag, bool, error)
	GetTag(ctx context.Context, repository, tagID string) (string, error)

	// GetEntry returns the current entry for path in repository branch reference.  Returns
	// the entry with ExpiredError if it has expired from underlying storage.
	GetEntry(ctx context.Context, repository, reference string, path string, params GetEntryParams) (*DBEntry, error)
	CreateEntry(ctx context.Context, repository, branch string, entry DBEntry) error
	CreateEntries(ctx context.Context, repository, branch string, entries []DBEntry) error
	DeleteEntry(ctx context.Context, repository, branch string, path string) error
	ListEntries(ctx context.Context, repository, reference string, prefix, after string, delimiter string, limit int) ([]*DBEntry, bool, error)
	ResetEntry(ctx context.Context, repository, branch string, path string) error
	ResetEntries(ctx context.Context, repository, branch string, prefix string) error

	Commit(ctx context.Context, repository, branch string, message string, committer string, metadata Metadata) (*CommitLog, error)
	GetCommit(ctx context.Context, repository, reference string) (*CommitLog, error)
	ListCommits(ctx context.Context, repository, branch string, fromReference string, limit int) ([]*CommitLog, bool, error)

	// RollbackCommit sets the branch to point at the given commit, losing all later commits.
	RollbackCommit(ctx context.Context, repository, branch string, reference string) error
	// Revert creates a reverse patch to the given commit, and applies it as a new commit on the given branch.
	Revert(ctx context.Context, repository, branch string, params RevertParams) error

	Diff(ctx context.Context, repository, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)
	Compare(ctx context.Context, repository, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)
	DiffUncommitted(ctx context.Context, repository, branch string, limit int, after string) (Differences, bool, error)

	Merge(ctx context.Context, repository, destinationBranch, sourceRef, committer, message string, metadata Metadata) (*MergeResult, error)

	// dump/load metadata
	DumpCommits(ctx context.Context, repositoryID string) (string, error)
	DumpBranches(ctx context.Context, repositoryID string) (string, error)
	DumpTags(ctx context.Context, repositoryID string) (string, error)
	LoadCommits(ctx context.Context, repositoryID, commitsMetaRangeID string) error
	LoadBranches(ctx context.Context, repositoryID, branchesMetaRangeID string) error
	LoadTags(ctx context.Context, repositoryID, tagsMetaRangeID string) error

	// forward metadata for thick clients
	GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeInfo, error)
	GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeInfo, error)

	io.Closer
}

type MergeResult 

type MergeResult struct {
	Summary   map[DifferenceType]int
	Reference string
}

type Metadata 

type Metadata map[string]string

func (*Metadata) Scan 

func (j *Metadata) Scan(src interface{}) error

func (Metadata) Value 

func (j Metadata) Value() (driver.Value, error)

type Path 

type Path string

func (Path) String 

func (id Path) String() string

type Repository 

type Repository struct {
	Name             string    `db:"name"`
	StorageNamespace string    `db:"storage_namespace"`
	DefaultBranch    string    `db:"default_branch"`
	CreationDate     time.Time `db:"creation_date"`
}

type RevertParams 

type RevertParams struct {
	Reference    string // the commit to revert
	ParentNumber int    // if reverting a merge commit, the change will be reversed relative to this parent number (1-based).
	Committer    string
}

type Store 

type Store interface {
	graveler.KeyValueStore
	graveler.VersionController
	graveler.Dumper
	graveler.Loader
	graveler.Plumbing
}

type Tag 

type Tag struct {
	ID       string
	CommitID string
}

type ValidateArg 

type ValidateArg struct {
	Name  string
	Value interface{}
	Fn    ValidateFunc
}

type ValidateFunc 

type ValidateFunc func(v interface{}) error

func MakeValidateOptional 

func MakeValidateOptional(fn ValidateFunc) ValidateFunc

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.