catalog

package
v0.111.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 3, 2023 License: Apache-2.0 Imports: 51 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DefaultActionsCacheSize   = 100
	DefaultActionsCacheExpiry = 5 * time.Second
	DefaultActionsCacheJitter = DefaultActionsCacheExpiry / 2
)
View Source 
const (
	RangeFSName     = "range"
	MetaRangeFSName = "meta-range"
)
View Source 
const (
	ListRepositoriesLimitMax = 1000
	ListBranchesLimitMax     = 1000
	ListTagsLimitMax         = 1000
	DiffLimitMax             = 1000
	ListEntriesLimitMax      = 10000
)
View Source 
const (
	ImportPathTypePrefix = "common_prefix"
	ImportPathTypeObject = "object"
)
View Source 
const (
	DefaultContentType = "application/octet-stream"
)
View Source 
const (
	DefaultPathDelimiter = "/"
)
View Source 
const ImportCanceled = "Canceled"
View Source 
const (
	MaxPathLength = 1024
)
View Source 
const (
	NumberOfParentsOfNonMergeCommit = 1
)

Variables

View Source 
var (
	Entry_AddressType_name = map[int32]string{
		0: "BY_PREFIX_DEPRECATED",
		1: "RELATIVE",
		2: "FULL",
	}
	Entry_AddressType_value = map[string]int32{
		"BY_PREFIX_DEPRECATED": 0,
		"RELATIVE":             1,
		"FULL":                 2,
	}
)

Enum value maps for Entry_AddressType.

View Source 
var (
	ErrUnknownDiffType          = errors.New("unknown graveler difference type")
	ErrPathRequiredValue        = fmt.Errorf("missing path: %w", graveler.ErrRequiredValue)
	ErrInvalidMetadataSrcFormat = errors.New("invalid metadata src format")
	ErrExpired                  = errors.New("expired from storage")
	// ErrItClosed is used to determine the reason for the end of the walk
	ErrItClosed = errors.New("iterator closed")

	ErrFeatureNotSupported = errors.New("feature not supported")
)

Define errors we raise from this package - do not convert underlying errors, optionally wrap if needed to consolidate

View Source 
var ErrImportClosed = errors.New("import closed")
View Source 
var File_catalog_proto protoreflect.FileDescriptor
View Source 
var ValidatePathOptional = validator.MakeValidateOptional(ValidatePath)

Functions

func ContentTypeOrDefault added in v0.53.0 

func ContentTypeOrDefault(ct string) string

func EntryToValue 

func EntryToValue(entry *Entry) (*graveler.Value, error)

func GetStartPos added in v0.42.0 

func GetStartPos(prefix, after, delimiter string) string

GetStartPos returns a key that SeekGE will transform to a place start iterating on all elements in 

the keys that start with 'prefix' after 'after' and taking 'delimiter' into account

func MustEntryToValue 

func MustEntryToValue(entry *Entry) *graveler.Value

func NewEntryToValueIterator 

func NewEntryToValueIterator(it EntryIterator) *entryValueIterator

func NewValueToEntryIterator 

func NewValueToEntryIterator(it graveler.ValueIterator) *valueEntryIterator

func NewWalkEntryIterator added in v0.65.0 

func NewWalkEntryIterator(ctx context.Context, walker *store.WalkerWrapper, sourceType ImportPathType, destination, after, continuationToken string) (*walkEntryIterator, error)

func ValidatePath 

func ValidatePath(v interface{}) error

Types

type ActionsCacheConfig added in v0.84.0 

type ActionsCacheConfig struct {
	Size   int
	Expiry time.Duration
	Jitter time.Duration
}

type ActionsOutputWriter 

type ActionsOutputWriter struct {
	// contains filtered or unexported fields
}

func NewActionsOutputWriter 

func NewActionsOutputWriter(blockAdapter block.Adapter) *ActionsOutputWriter

func (*ActionsOutputWriter) OutputWrite 

func (o *ActionsOutputWriter) OutputWrite(ctx context.Context, storageNamespace, name string, reader io.Reader, size int64) error

type ActionsSource 

type ActionsSource struct {
	// contains filtered or unexported fields
}

func NewActionsSource 

func NewActionsSource(catalog *Catalog) *ActionsSource

func (*ActionsSource) List 

func (s *ActionsSource) List(ctx context.Context, record graveler.HookRecord) ([]string, error)

func (*ActionsSource) Load 

func (s *ActionsSource) Load(ctx context.Context, record graveler.HookRecord, name string) ([]byte, error)

type AddressType added in v0.40.0 

type AddressType int32

AddressType is the type of an entry address 

const (
	// Deprecated: indicates that the address might be relative or full.
	// Used only for backward compatibility and should not be used for creating entries.
	AddressTypeByPrefixDeprecated AddressType = 0

	// AddressTypeRelative indicates that the address is relative to the storage namespace.
	// For example: "foo/bar"
	AddressTypeRelative AddressType = 1

	// AddressTypeFull indicates that the address is the full address of the object in the object store.
	// For example: "s3://bucket/foo/bar"
	AddressTypeFull AddressType = 2
)

func (AddressType) ToIdentifierType added in v0.40.0 

func (at AddressType) ToIdentifierType() block.IdentifierType

type Branch 

type Branch struct {
	Name      string
	Reference string
}

type Catalog 

type Catalog struct {
	BlockAdapter block.Adapter
	Store        Store

	PathProvider      *upload.PathPartitionProvider
	BackgroundLimiter ratelimit.Limiter
	KVStore           kv.Store
	KVStoreLimited    kv.Store

	UGCPrepareMaxFileSize int64
	UGCPrepareInterval    time.Duration
	// contains filtered or unexported fields
}

func New 

func New(ctx context.Context, cfg Config) (*Catalog, error)

func (*Catalog) BranchExists 

func (c *Catalog) BranchExists(ctx context.Context, repositoryID string, branch string) (bool, error)

func (*Catalog) CancelImport added in v0.102.0 

func (c *Catalog) CancelImport(ctx context.Context, repositoryID, importID string) error

func (*Catalog) CherryPick added in v0.97.0 

func (c *Catalog) CherryPick(ctx context.Context, repositoryID string, branch string, params CherryPickParams) (*CommitLog, error)

func (*Catalog) Close 

func (c *Catalog) Close() error

func (*Catalog) Commit 

func (c *Catalog) Commit(ctx context.Context, repositoryID, branch, message, committer string, metadata Metadata, date *int64, sourceMetarange *string) (*CommitLog, error)

func (*Catalog) Compare 

func (c *Catalog) Compare(ctx context.Context, repositoryID, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)

func (*Catalog) CopyEntry added in v0.90.1 

func (c *Catalog) CopyEntry(ctx context.Context, srcRepository, srcRef, srcPath, destRepository, destBranch, destPath string) (*DBEntry, error)

CopyEntry copy entry information by using the block adapter to make a copy of the data to a new physical address.

func (*Catalog) CreateBareRepository 

func (c *Catalog) CreateBareRepository(ctx context.Context, repository string, storageNamespace string, defaultBranchID string) (*Repository, error)

CreateBareRepository creates a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with no initial branch or commit

func (*Catalog) CreateBranch 

func (c *Catalog) CreateBranch(ctx context.Context, repositoryID string, branch string, sourceBranch string) (*CommitLog, error)

func (*Catalog) CreateEntry 

func (c *Catalog) CreateEntry(ctx context.Context, repositoryID string, branch string, entry DBEntry, opts ...graveler.SetOptionsFunc) error

func (*Catalog) CreateRepository 

func (c *Catalog) CreateRepository(ctx context.Context, repository string, storageNamespace string, branch string) (*Repository, error)

CreateRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with default branch name 'branch'

func (*Catalog) CreateTag 

func (c *Catalog) CreateTag(ctx context.Context, repositoryID string, tagID string, ref string) (string, error)

func (*Catalog) DeleteBranch 

func (c *Catalog) DeleteBranch(ctx context.Context, repositoryID string, branch string) error

func (*Catalog) DeleteEntries added in v0.82.0 

func (c *Catalog) DeleteEntries(ctx context.Context, repositoryID string, branch string, paths []string) error

func (*Catalog) DeleteEntry 

func (c *Catalog) DeleteEntry(ctx context.Context, repositoryID string, branch string, path string) error

func (*Catalog) DeleteExpiredImports added in v0.102.0 

func (c *Catalog) DeleteExpiredImports(ctx context.Context)

func (*Catalog) DeleteExpiredLinkAddresses added in v0.90.0 

func (c *Catalog) DeleteExpiredLinkAddresses(ctx context.Context)

func (*Catalog) DeleteRepository 

func (c *Catalog) DeleteRepository(ctx context.Context, repository string) error

DeleteRepository delete a repository

func (*Catalog) DeleteTag 

func (c *Catalog) DeleteTag(ctx context.Context, repositoryID string, tagID string) error

func (*Catalog) Diff 

func (c *Catalog) Diff(ctx context.Context, repositoryID string, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)

func (*Catalog) DiffUncommitted 

func (c *Catalog) DiffUncommitted(ctx context.Context, repositoryID, branch, prefix, delimiter string, limit int, after string) (Differences, bool, error)

func (*Catalog) DumpBranches 

func (c *Catalog) DumpBranches(ctx context.Context, repositoryID string) (string, error)

func (*Catalog) DumpCommits 

func (c *Catalog) DumpCommits(ctx context.Context, repositoryID string) (string, error)

func (*Catalog) DumpTags 

func (c *Catalog) DumpTags(ctx context.Context, repositoryID string) (string, error)

func (*Catalog) FindMergeBase added in v0.94.1 

func (c *Catalog) FindMergeBase(ctx context.Context, repositoryID string, destinationRef string, sourceRef string) (string, string, string, error)

func (*Catalog) GetBranchProtectionRules added in v0.52.0 

func (c *Catalog) GetBranchProtectionRules(ctx context.Context, repositoryID string) (*graveler.BranchProtectionRules, *string, error)

func (*Catalog) GetBranchReference 

func (c *Catalog) GetBranchReference(ctx context.Context, repositoryID string, branch string) (string, error)

func (*Catalog) GetCommit 

func (c *Catalog) GetCommit(ctx context.Context, repositoryID string, reference string) (*CommitLog, error)

func (*Catalog) GetEntry 

func (c *Catalog) GetEntry(ctx context.Context, repositoryID string, reference string, path string, params GetEntryParams) (*DBEntry, error)

GetEntry returns the current entry for a path in repository branch reference. Returns the entry with ExpiredError if it has expired from underlying storage.

func (*Catalog) GetGarbageCollectionRules added in v0.43.0 

func (c *Catalog) GetGarbageCollectionRules(ctx context.Context, repositoryID string) (*graveler.GarbageCollectionRules, error)

func (*Catalog) GetImportStatus added in v0.102.0 

func (c *Catalog) GetImportStatus(ctx context.Context, repositoryID, importID string) (*graveler.ImportStatus, error)

func (*Catalog) GetMetaRange 

func (c *Catalog) GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeAddress, error)

func (*Catalog) GetRange 

func (c *Catalog) GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeAddress, error)

func (*Catalog) GetRepository 

func (c *Catalog) GetRepository(ctx context.Context, repository string) (*Repository, error)

GetRepository get repository information

func (*Catalog) GetRepositoryMetadata added in v0.104.0 

func (c *Catalog) GetRepositoryMetadata(ctx context.Context, repository string) (graveler.RepositoryMetadata, error)

func (*Catalog) GetStagingToken added in v0.40.0 

func (c *Catalog) GetStagingToken(ctx context.Context, repositoryID string, branch string) (*string, error)

func (*Catalog) GetTag 

func (c *Catalog) GetTag(ctx context.Context, repositoryID string, tagID string) (string, error)

func (*Catalog) Import added in v0.102.0 

func (c *Catalog) Import(ctx context.Context, repositoryID, branchID string, params ImportRequest) (string, error)

func (*Catalog) ListBranches 

func (c *Catalog) ListBranches(ctx context.Context, repositoryID string, prefix string, limit int, after string) ([]*Branch, bool, error)

func (*Catalog) ListCommits 

func (c *Catalog) ListCommits(ctx context.Context, repositoryID string, branch string, params LogParams) ([]*CommitLog, bool, error)

func (*Catalog) ListEntries 

func (c *Catalog) ListEntries(ctx context.Context, repositoryID string, reference string, prefix string, after string, delimiter string, limit int) ([]*DBEntry, bool, error)

func (*Catalog) ListRepositories 

func (c *Catalog) ListRepositories(ctx context.Context, limit int, prefix, after string) ([]*Repository, bool, error)

ListRepositories list repository information, the bool returned is true when more repositories can be listed. In this case, pass the last repository name as 'after' on the next call to ListRepositories

func (*Catalog) ListTags 

func (c *Catalog) ListTags(ctx context.Context, repositoryID string, prefix string, limit int, after string) ([]*Tag, bool, error)

func (*Catalog) LoadBranches 

func (c *Catalog) LoadBranches(ctx context.Context, repositoryID, branchesMetaRangeID string) error

func (*Catalog) LoadCommits 

func (c *Catalog) LoadCommits(ctx context.Context, repositoryID, commitsMetaRangeID string) error

func (*Catalog) LoadTags 

func (c *Catalog) LoadTags(ctx context.Context, repositoryID, tagsMetaRangeID string) error

func (*Catalog) Merge 

func (c *Catalog) Merge(ctx context.Context, repositoryID string, destinationBranch string, sourceRef string, committer string, message string, metadata Metadata, strategy string) (string, error)

func (*Catalog) PrepareExpiredCommits added in v0.43.0 

func (c *Catalog) PrepareExpiredCommits(ctx context.Context, repositoryID string) (*graveler.GarbageCollectionRunMetadata, error)

func (*Catalog) PrepareGCUncommitted added in v0.85.0 

func (c *Catalog) PrepareGCUncommitted(ctx context.Context, repositoryID string, mark *GCUncommittedMark) (*PrepareGCUncommittedInfo, error)

func (*Catalog) ResetBranch 

func (c *Catalog) ResetBranch(ctx context.Context, repositoryID string, branch string) error

func (*Catalog) ResetEntries 

func (c *Catalog) ResetEntries(ctx context.Context, repositoryID string, branch string, prefix string) error

func (*Catalog) ResetEntry 

func (c *Catalog) ResetEntry(ctx context.Context, repositoryID string, branch string, path string) error

func (*Catalog) Revert 

func (c *Catalog) Revert(ctx context.Context, repositoryID string, branch string, params RevertParams) error

func (*Catalog) SetBranchProtectionRules added in v0.111.0 

func (c *Catalog) SetBranchProtectionRules(ctx context.Context, repositoryID string, rules *graveler.BranchProtectionRules, lastKnownChecksum *string) error

func (*Catalog) SetGarbageCollectionRules added in v0.43.0 

func (c *Catalog) SetGarbageCollectionRules(ctx context.Context, repositoryID string, rules *graveler.GarbageCollectionRules) error

func (*Catalog) SetHooksHandler 

func (c *Catalog) SetHooksHandler(hooks graveler.HooksHandler)

func (*Catalog) SetLinkAddress added in v0.90.0 

func (c *Catalog) SetLinkAddress(ctx context.Context, repository, token string) error

func (*Catalog) VerifyLinkAddress added in v0.90.0 

func (c *Catalog) VerifyLinkAddress(ctx context.Context, repository, token string) error

func (*Catalog) WriteMetaRange added in v0.65.0 

func (c *Catalog) WriteMetaRange(ctx context.Context, repositoryID string, ranges []*graveler.RangeInfo) (*graveler.MetaRangeInfo, error)

func (*Catalog) WriteRange added in v0.65.0 

func (c *Catalog) WriteRange(ctx context.Context, repositoryID string, params WriteRangeRequest) (*graveler.RangeInfo, *Mark, error)

type CherryPickParams added in v0.97.0 

type CherryPickParams struct {
	Reference    string // the commit to pick
	ParentNumber *int   // if a merge commit was picked, the change will be applied relative to this parent number (1-based).
	Committer    string
}

type CommitLog 

type CommitLog struct {
	Reference    string
	Committer    string
	Message      string
	CreationDate time.Time
	Metadata     Metadata
	MetaRangeID  string
	Parents      []string
}

func CommitRecordToLog added in v0.102.0 

func CommitRecordToLog(val *graveler.CommitRecord) *CommitLog

type Config 

type Config struct {
	Config                *config.Config
	KVStore               kv.Store
	WalkerFactory         WalkerFactory
	SettingsManagerOption settings.ManagerOption
	PathProvider          *upload.PathPartitionProvider
}

type DBEntry 

type DBEntry struct {
	CommonLevel     bool
	Path            string
	PhysicalAddress string
	CreationDate    time.Time
	Size            int64
	Checksum        string
	Metadata        Metadata
	Expired         bool
	AddressType     AddressType
	ContentType     string
}

type DBEntryBuilder added in v0.53.0 

type DBEntryBuilder struct {
	// contains filtered or unexported fields
}

DBEntryBuilder DBEntry builder

func NewDBEntryBuilder added in v0.53.0 

func NewDBEntryBuilder() *DBEntryBuilder

func (*DBEntryBuilder) AddressType added in v0.53.0 

func (b *DBEntryBuilder) AddressType(addressType AddressType) *DBEntryBuilder

func (*DBEntryBuilder) Build added in v0.53.0 

func (b *DBEntryBuilder) Build() DBEntry

func (*DBEntryBuilder) Checksum added in v0.53.0 

func (b *DBEntryBuilder) Checksum(checksum string) *DBEntryBuilder

func (*DBEntryBuilder) CommonLevel added in v0.53.0 

func (b *DBEntryBuilder) CommonLevel(commonLevel bool) *DBEntryBuilder

func (*DBEntryBuilder) ContentType added in v0.53.0 

func (b *DBEntryBuilder) ContentType(contentType string) *DBEntryBuilder

func (*DBEntryBuilder) CreationDate added in v0.53.0 

func (b *DBEntryBuilder) CreationDate(creationDate time.Time) *DBEntryBuilder

func (*DBEntryBuilder) Expired added in v0.53.0 

func (b *DBEntryBuilder) Expired(expired bool) *DBEntryBuilder

func (*DBEntryBuilder) Metadata added in v0.53.0 

func (b *DBEntryBuilder) Metadata(metadata Metadata) *DBEntryBuilder

func (*DBEntryBuilder) Path added in v0.53.0 

func (b *DBEntryBuilder) Path(path string) *DBEntryBuilder

func (*DBEntryBuilder) PhysicalAddress added in v0.53.0 

func (b *DBEntryBuilder) PhysicalAddress(physicalAddress string) *DBEntryBuilder

func (*DBEntryBuilder) RelativeAddress added in v0.53.0 

func (b *DBEntryBuilder) RelativeAddress(relative bool) *DBEntryBuilder

func (*DBEntryBuilder) Size added in v0.53.0 

func (b *DBEntryBuilder) Size(size int64) *DBEntryBuilder

type DiffParams 

type DiffParams struct {
	Limit            int
	After            string
	Prefix           string
	Delimiter        string
	AdditionalFields []string // db fields names that will be load in additional to Path on Difference's Entry
}

type DiffResultRecord 

type DiffResultRecord struct {
	TargetEntryNotInDirectBranch bool // the entry is reflected via lineage, NOT in the branch itself
	Difference
}

type Difference 

type Difference struct {
	DBEntry                // Partially filled. Path is always set.
	Type    DifferenceType `db:"diff_type"`
}

func (Difference) String 

func (d Difference) String() string

type DifferenceType 

type DifferenceType int
const (
	DifferenceTypeAdded DifferenceType = iota
	DifferenceTypeRemoved
	DifferenceTypeChanged
	DifferenceTypePrefixChanged
	DifferenceTypeConflict
	DifferenceTypeNone
)

type Differences 

type Differences []Difference

func (Differences) Equal 

func (d Differences) Equal(other Differences) bool

type Entry 

type Entry struct {
	Address      string                 `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
	LastModified *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=last_modified,json=lastModified,proto3" json:"last_modified,omitempty"`
	Size         int64                  `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
	ETag         string                 `protobuf:"bytes,4,opt,name=e_tag,json=eTag,proto3" json:"e_tag,omitempty"`
	Metadata     map[string]string      `` /* 157-byte string literal not displayed */
	AddressType  Entry_AddressType      `` /* 126-byte string literal not displayed */
	ContentType  string                 `protobuf:"bytes,7,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"`
	// contains filtered or unexported fields
}

func ValueToEntry 

func ValueToEntry(value *graveler.Value) (*Entry, error)

func (*Entry) Descriptor deprecated 

func (*Entry) Descriptor() ([]byte, []int)

Deprecated: Use Entry.ProtoReflect.Descriptor instead.

func (*Entry) GetAddress 

func (x *Entry) GetAddress() string

func (*Entry) GetAddressType added in v0.40.0 

func (x *Entry) GetAddressType() Entry_AddressType

func (*Entry) GetContentType added in v0.53.0 

func (x *Entry) GetContentType() string

func (*Entry) GetETag 

func (x *Entry) GetETag() string

func (*Entry) GetLastModified 

func (x *Entry) GetLastModified() *timestamppb.Timestamp

func (*Entry) GetMetadata 

func (x *Entry) GetMetadata() map[string]string

func (*Entry) GetSize 

func (x *Entry) GetSize() int64

func (*Entry) ProtoMessage 

func (*Entry) ProtoMessage()

func (*Entry) ProtoReflect 

func (x *Entry) ProtoReflect() protoreflect.Message

func (*Entry) Reset 

func (x *Entry) Reset()

func (*Entry) String 

func (x *Entry) String() string

type EntryDiff 

type EntryDiff struct {
	Type  graveler.DiffType
	Path  Path
	Entry *Entry
}

type EntryDiffIterator 

type EntryDiffIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryDiff
	Err() error
	Close()
}

func NewEntryDiffIterator 

func NewEntryDiffIterator(it graveler.DiffIterator) EntryDiffIterator

type EntryIterator 

type EntryIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryRecord
	Err() error
	Close()
}

func NewPrefixIterator 

func NewPrefixIterator(it EntryIterator, prefix Path) EntryIterator

type EntryListing 

type EntryListing struct {
	CommonPrefix bool
	Path
	*Entry
}

type EntryListingIterator 

type EntryListingIterator interface {
	Next() bool
	SeekGE(id Path)
	Value() *EntryListing
	Err() error
	Close()
}

func NewEntryListingIterator 

func NewEntryListingIterator(it EntryIterator, prefix Path, delimiter Path) EntryListingIterator

type EntryRecord 

type EntryRecord struct {
	Path Path
	*Entry
}

type EntryWithMarker added in v0.65.0 

type EntryWithMarker struct {
	EntryRecord
	Mark
}

type Entry_AddressType added in v0.40.0 

type Entry_AddressType int32
const (
	// Deprecated.
	// Unknown address type (should only exist for old commits)
	// is resolved (to Relative or Full) by the prefix of the address.
	Entry_BY_PREFIX_DEPRECATED Entry_AddressType = 0
	Entry_RELATIVE             Entry_AddressType = 1
	Entry_FULL                 Entry_AddressType = 2
)

func (Entry_AddressType) Descriptor added in v0.40.0 

func (Entry_AddressType) Descriptor() protoreflect.EnumDescriptor

func (Entry_AddressType) Enum added in v0.40.0 

func (x Entry_AddressType) Enum() *Entry_AddressType

func (Entry_AddressType) EnumDescriptor deprecated added in v0.40.0 

func (Entry_AddressType) EnumDescriptor() ([]byte, []int)

Deprecated: Use Entry_AddressType.Descriptor instead.

func (Entry_AddressType) Number added in v0.40.0 

func (x Entry_AddressType) Number() protoreflect.EnumNumber

func (Entry_AddressType) String added in v0.40.0 

func (x Entry_AddressType) String() string

func (Entry_AddressType) Type added in v0.40.0 

func (Entry_AddressType) Type() protoreflect.EnumType

type ExpireResult 

type ExpireResult struct {
	Repository        string
	Branch            string
	PhysicalAddress   string
	InternalReference string
}

type ExpiryRows 

type ExpiryRows interface {
	Close()
	Next() bool
	Err() error
	// Read returns the current from ExpiryRows, or an error on failure.  Call it only after
	// successfully calling Next.
	Read() (*ExpireResult, error)
}

ExpiryRows is a database iterator over ExpiryResults. Use Next to advance from row to row.

type GCUncommittedMark added in v0.85.0 

type GCUncommittedMark struct {
	BranchID graveler.BranchID `json:"branch"`
	Path     Path              `json:"path"`
	RunID    string            `json:"run_id"`
	Key      string            `json:"key"`
}

GCUncommittedMark Marks the *next* item to be scanned by the paginated call to PrepareGCUncommitted

type GetEntryParams 

type GetEntryParams struct {
	// StageOnly when true will return entry found on stage without checking committed data
	StageOnly bool
}

GetEntryParams configures what entries GetEntry returns.

type Import added in v0.102.0 

type Import struct {
	// contains filtered or unexported fields
}

func NewImport added in v0.102.0 

func NewImport(ctx context.Context, cancel context.CancelFunc, logger logging.Logger, kvStore kv.Store, repository *graveler.RepositoryRecord, importID string) (*Import, error)

func (*Import) Close added in v0.102.0 

func (i *Import) Close()

func (*Import) Closed added in v0.102.0 

func (i *Import) Closed() bool

func (*Import) Ingest added in v0.102.0 

func (i *Import) Ingest(it *walkEntryIterator) error

func (*Import) NewItr added in v0.102.0 

func (i *Import) NewItr() (*importIterator, error)

func (*Import) SetError added in v0.102.0 

func (i *Import) SetError(err error)

func (*Import) SetStatus added in v0.102.0 

func (i *Import) SetStatus(status graveler.ImportStatus)

func (*Import) Status added in v0.102.0 

func (i *Import) Status() graveler.ImportStatus

type ImportCommit added in v0.102.0 

type ImportCommit struct {
	CommitMessage string
	Committer     string
	Metadata      Metadata
}

type ImportPath added in v0.102.0 

type ImportPath struct {
	Path        string
	Destination string
	Type        ImportPathType
}

type ImportPathType added in v0.102.0 

type ImportPathType string

func GetImportPathType added in v0.102.0 

func GetImportPathType(t string) (ImportPathType, error)

type ImportRequest added in v0.102.0 

type ImportRequest struct {
	Paths  []ImportPath
	Commit ImportCommit
}

type Interface 

type Interface interface {
	// CreateRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with default branch name 'branch'
	CreateRepository(ctx context.Context, repository string, storageNamespace string, branch string) (*Repository, error)

	// CreateBareRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with no initial branch or commit
	// defaultBranchID will point to a non-existent branch on creation, it is up to the caller to eventually create it.
	CreateBareRepository(ctx context.Context, repository string, storageNamespace string, defaultBranchID string) (*Repository, error)

	// GetRepository get repository information
	GetRepository(ctx context.Context, repository string) (*Repository, error)

	// DeleteRepository delete a repository
	DeleteRepository(ctx context.Context, repository string) error

	// GetRepositoryMetadata get repository metadata
	GetRepositoryMetadata(ctx context.Context, repository string) (graveler.RepositoryMetadata, error)

	// ListRepositories list repository information, the bool returned is true when more repositories can be listed.
	// In this case pass the last repository name as 'after' on the next call to ListRepositories
	ListRepositories(ctx context.Context, limit int, prefix, after string) ([]*Repository, bool, error)

	GetStagingToken(ctx context.Context, repository string, branch string) (*string, error)

	CreateBranch(ctx context.Context, repository, branch string, sourceRef string) (*CommitLog, error)
	DeleteBranch(ctx context.Context, repository, branch string) error
	ListBranches(ctx context.Context, repository string, prefix string, limit int, after string) ([]*Branch, bool, error)
	BranchExists(ctx context.Context, repository string, branch string) (bool, error)
	GetBranchReference(ctx context.Context, repository, branch string) (string, error)
	ResetBranch(ctx context.Context, repository, branch string) error

	CreateTag(ctx context.Context, repository, tagID string, ref string) (string, error)
	DeleteTag(ctx context.Context, repository, tagID string) error
	ListTags(ctx context.Context, repository string, prefix string, limit int, after string) ([]*Tag, bool, error)
	GetTag(ctx context.Context, repository, tagID string) (string, error)

	// GetEntry returns the current entry for path in repository branch reference.  Returns
	// the entry with ExpiredError if it has expired from underlying storage.
	GetEntry(ctx context.Context, repository, reference string, path string, params GetEntryParams) (*DBEntry, error)
	CreateEntry(ctx context.Context, repository, branch string, entry DBEntry, opts ...graveler.SetOptionsFunc) error
	DeleteEntry(ctx context.Context, repository, branch string, path string) error
	DeleteEntries(ctx context.Context, repository, branch string, paths []string) error
	ListEntries(ctx context.Context, repository, reference string, prefix, after string, delimiter string, limit int) ([]*DBEntry, bool, error)
	ResetEntry(ctx context.Context, repository, branch string, path string) error
	ResetEntries(ctx context.Context, repository, branch string, prefix string) error
	CopyEntry(ctx context.Context, srcRepository, srcRef, srcPath, destRepository, destBranch, destPath string) (*DBEntry, error)

	Commit(ctx context.Context, repository, branch, message, committer string, metadata Metadata, date *int64, sourceMetarange *string) (*CommitLog, error)
	GetCommit(ctx context.Context, repository, reference string) (*CommitLog, error)
	ListCommits(ctx context.Context, repository, branch string, params LogParams) ([]*CommitLog, bool, error)

	// Revert creates a reverse patch to the given commit, and applies it as a new commit on the given branch.
	Revert(ctx context.Context, repository, branch string, params RevertParams) error

	// CherryPick creates a patch to the given commit, and applies it as a new commit on the given branch.
	CherryPick(ctx context.Context, repository, branch string, params CherryPickParams) (*CommitLog, error)

	Diff(ctx context.Context, repository, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)
	Compare(ctx context.Context, repository, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)
	DiffUncommitted(ctx context.Context, repository, branch, prefix, delimiter string, limit int, after string) (Differences, bool, error)

	Merge(ctx context.Context, repository, destinationBranch, sourceRef, committer, message string, metadata Metadata, strategy string) (string, error)
	FindMergeBase(ctx context.Context, repositoryID string, destinationRef string, sourceRef string) (string, string, string, error)

	// dump/load metadata
	DumpCommits(ctx context.Context, repositoryID string) (string, error)
	DumpBranches(ctx context.Context, repositoryID string) (string, error)
	DumpTags(ctx context.Context, repositoryID string) (string, error)
	LoadCommits(ctx context.Context, repositoryID, commitsMetaRangeID string) error
	LoadBranches(ctx context.Context, repositoryID, branchesMetaRangeID string) error
	LoadTags(ctx context.Context, repositoryID, tagsMetaRangeID string) error

	// forward metadata for thick clients
	GetMetaRange(ctx context.Context, repositoryID, metaRangeID string) (graveler.MetaRangeAddress, error)
	GetRange(ctx context.Context, repositoryID, rangeID string) (graveler.RangeAddress, error)
	Import(ctx context.Context, repositoryID, branchID string, params ImportRequest) (string, error)
	GetImportStatus(ctx context.Context, repositoryID, importID string) (*graveler.ImportStatus, error)
	CancelImport(ctx context.Context, repositoryID, importID string) error
	WriteRange(ctx context.Context, repositoryID string, params WriteRangeRequest) (*graveler.RangeInfo, *Mark, error)
	WriteMetaRange(ctx context.Context, repositoryID string, ranges []*graveler.RangeInfo) (*graveler.MetaRangeInfo, error)

	GetGarbageCollectionRules(ctx context.Context, repositoryID string) (*graveler.GarbageCollectionRules, error)
	SetGarbageCollectionRules(ctx context.Context, repositoryID string, rules *graveler.GarbageCollectionRules) error
	PrepareExpiredCommits(ctx context.Context, repositoryID string) (*graveler.GarbageCollectionRunMetadata, error)
	// PrepareGCUncommitted Creates parquet files listing of all uncommitted objects in the given repositoryID and saves them under the GC runID in the object store
	// Since this operation might take a very long time, we save 20MB files at a time and return a mark of the next item to read, which can be provided to a consecutive call
	// Consecutive calls must be made using the returned run ID, upon completion mark will return nil
	PrepareGCUncommitted(ctx context.Context, repositoryID string, mark *GCUncommittedMark) (*PrepareGCUncommittedInfo, error)

	// GetBranchProtectionRules returns the branch protection rules for the given repository.
	// The returned checksum represents the current state of the rules, and can be passed to SetBranchProtectionRules for conditional updates.
	GetBranchProtectionRules(ctx context.Context, repositoryID string) (*graveler.BranchProtectionRules, *string, error)
	// SetBranchProtectionRules sets the branch protection rules for the given repository.
	// If lastKnownChecksum doesn't match the current state, the update will fail with ErrPreconditionFailed.
	// If lastKnownChecksum is nil, the update is performed only if no rules exist.
	// If lastKnownChecksum is equal to BranchProtectionSkipValidationChecksum, the update is always performed.
	SetBranchProtectionRules(ctx context.Context, repositoryID string, rules *graveler.BranchProtectionRules, lastKnownChecksum *string) error

	// SetLinkAddress to validate single use limited in time of a given physical address
	SetLinkAddress(ctx context.Context, repository, token string) error
	VerifyLinkAddress(ctx context.Context, repository, token string) error
	DeleteExpiredLinkAddresses(ctx context.Context)

	io.Closer
}

type LogParams added in v0.54.0 

type LogParams struct {
	PathList      []PathRecord
	FromReference string
	Amount        int
	Limit         bool
	FirstParent   bool
}

type Mark added in v0.65.0 

type Mark struct {
	block.Mark
	StagingToken string
}

Mark stands for pagination information when listing objects from the blockstore. It is used for server-client communication on the status of range ingestion.

type Metadata 

type Metadata map[string]string

func (*Metadata) Scan 

func (j *Metadata) Scan(src interface{}) error

func (Metadata) Value 

func (j Metadata) Value() (driver.Value, error)

type Path 

type Path string

func (Path) String 

func (id Path) String() string

type PathRecord added in v0.54.0 

type PathRecord struct {
	Path     Path
	IsPrefix bool
}

type PrepareGCUncommittedInfo added in v0.85.0 

type PrepareGCUncommittedInfo struct {
	RunID    string `json:"run_id"`
	Location string `json:"location"`
	Filename string `json:"filename"`
	Mark     *GCUncommittedMark
}

type Repository 

type Repository struct {
	Name             string
	StorageNamespace string
	DefaultBranch    string
	CreationDate     time.Time
}

type RevertParams 

type RevertParams struct {
	Reference    string // the commit to revert
	ParentNumber int    // if reverting a merge commit, the change will be reversed relative to this parent number (1-based).
	Committer    string
}

type Store 

type Store interface {
	graveler.KeyValueStore
	graveler.VersionController
	graveler.Dumper
	graveler.Loader
	graveler.Plumbing
}

type Tag 

type Tag struct {
	ID       string
	CommitID string
}

type UncommittedIterator added in v0.85.0 

type UncommittedIterator struct {
	// contains filtered or unexported fields
}

func NewUncommittedIterator added in v0.85.0 

func NewUncommittedIterator(ctx context.Context, store Store, repository *graveler.RepositoryRecord) (*UncommittedIterator, error)

func (*UncommittedIterator) Close added in v0.85.0 

func (u *UncommittedIterator) Close()

func (*UncommittedIterator) Err added in v0.85.0 

func (u *UncommittedIterator) Err() error

func (*UncommittedIterator) Next added in v0.85.0 

func (u *UncommittedIterator) Next() bool

Next returns the next entry - if entryItr is still valid - gets the next value from it otherwise call u.next

func (*UncommittedIterator) SeekGE added in v0.85.0 

func (u *UncommittedIterator) SeekGE(branchID graveler.BranchID, id Path)

func (*UncommittedIterator) Value added in v0.85.0 

func (u *UncommittedIterator) Value() *UncommittedRecord

type UncommittedParquetObject added in v0.85.0 

type UncommittedParquetObject struct {
	PhysicalAddress string `parquet:"name=physical_address, type=BYTE_ARRAY, convertedtype=UTF8, encoding=PLAIN_DICTIONARY"`
	CreationDate    int64  `parquet:"name=creation_date, type=INT64, convertedtype=INT_64"`
}

type UncommittedRecord added in v0.85.0 

type UncommittedRecord struct {
	*EntryRecord
	// contains filtered or unexported fields
}

type UncommittedWriter added in v0.85.0 

type UncommittedWriter struct {
	// contains filtered or unexported fields
}

UncommittedWriter wraps io.Writer and tracks the total size of writes done on this writer Used to get the current file size written without expensive calls to Flush and Stat

func NewUncommittedWriter added in v0.85.0 

func NewUncommittedWriter(writer io.Writer) *UncommittedWriter

func (*UncommittedWriter) Size added in v0.85.0 

func (w *UncommittedWriter) Size() int64

func (*UncommittedWriter) Write added in v0.85.0 

func (w *UncommittedWriter) Write(p []byte) (n int, err error)

type WalkerFactory added in v0.65.0 

type WalkerFactory interface {
	GetWalker(ctx context.Context, opts store.WalkerOptions) (*store.WalkerWrapper, error)
}

WalkerFactory provides an abstraction for creating Walker

type WriteRangeRequest added in v0.98.0 

type WriteRangeRequest struct {
	SourceURI         string
	Prepend           string
	After             string
	StagingToken      string
	ContinuationToken string
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.