catalog

package
v0.21.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 8, 2020 License: Apache-2.0 Imports: 10 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DefaultCommitter        = ""
	DefaultBranchName       = "master"
	DefaultImportBranchName = "import-from-inventory"
	DefaultPathDelimiter    = "/"
)
View Source 
const (
	ExportStatusInProgress = CatalogBranchExportStatus("in-progress")
	ExportStatusSuccess    = CatalogBranchExportStatus("exported-successfully")
	ExportStatusFailed     = CatalogBranchExportStatus("export-failed")
	ExportStatusRepaired   = CatalogBranchExportStatus("export-repaired")
	ExportStatusUnknown    = CatalogBranchExportStatus("[unknown]")
)
View Source 
const (
	DBEntryFieldChecksum        = "checksum"
	DBEntryFieldPhysicalAddress = "physical_address"
)

Variables

View Source 
var (
	ErrInvalidReference            = errors.New("invalid reference")
	ErrInvalidMetadataSrcFormat    = errors.New("invalid metadata src format")
	ErrExpired                     = errors.New("expired from storage")
	ErrByteSliceTypeAssertion      = errors.New("type assertion to []byte failed")
	ErrFeatureNotSupported         = errors.New("feature not supported")
	ErrBranchNotFound              = fmt.Errorf("branch %w", db.ErrNotFound)
	ErrCommitNotFound              = fmt.Errorf("commit %w", db.ErrNotFound)
	ErrRepositoryNotFound          = fmt.Errorf("repository %w", db.ErrNotFound)
	ErrEntryNotFound               = fmt.Errorf("entry %w", db.ErrNotFound)
	ErrUnexpected                  = errors.New("unexpected error")
	ErrReadEntryTimeout            = errors.New("read entry timeout")
	ErrInvalidValue                = errors.New("invalid value")
	ErrNonDirectNotSupported       = errors.New("non direct diff not supported")
	ErrSameBranchMergeNotSupported = errors.New("same branch merge not supported")
	ErrLineageCorrupted            = errors.New("lineage corrupted")
	ErrOperationNotPermitted       = errors.New("operation not permitted")
	ErrNothingToCommit             = errors.New("nothing to commit")
	ErrInvalidLockValue            = errors.New("invalid lock value")
	ErrNoDifferenceWasFound        = errors.New("no difference was found")
	ErrConflictFound               = errors.New("conflict found")
	ErrUnsupportedRelation         = errors.New("unsupported relation")
	ErrUnsupportedDelimiter        = errors.New("unsupported delimiter")
	ErrBadTypeConversion           = errors.New("bad type")
	ErrExportFailed                = errors.New("export failed")
	ErrRollbackWithActiveBranch    = fmt.Errorf("%w: rollback with active branch", ErrFeatureNotSupported)
)

Functions

This section is empty.

Types

type Branch 

type Branch struct {
	Repository string `db:"repository"`
	Name       string `db:"name"`
}

type CatalogBranchExportStatus added in v0.16.0 

type CatalogBranchExportStatus string

func (*CatalogBranchExportStatus) Scan added in v0.16.0 

func (dst *CatalogBranchExportStatus) Scan(src interface{}) error

nolint: stylecheck

func (CatalogBranchExportStatus) Value added in v0.16.0 

func (src CatalogBranchExportStatus) Value() (driver.Value, error)

type Cataloger 

type Cataloger interface {
	// CreateRepository create a new repository pointing to 'storageNamespace' (ex: s3://bucket1/repo) with default branch name 'branch'
	CreateRepository(ctx context.Context, repository string, storageNamespace string, branch string) (*Repository, error)

	// GetRepository get repository information
	GetRepository(ctx context.Context, repository string) (*Repository, error)

	// DeleteRepository delete a repository
	DeleteRepository(ctx context.Context, repository string) error

	// ListRepositories list repositories information, the bool returned is true when more repositories can be listed.
	// In this case pass the last repository name as 'after' on the next call to ListRepositories
	ListRepositories(ctx context.Context, limit int, after string) ([]*Repository, bool, error)

	CreateBranch(ctx context.Context, repository, branch string, sourceBranch string) (*CommitLog, error)
	DeleteBranch(ctx context.Context, repository, branch string) error
	ListBranches(ctx context.Context, repository string, prefix string, limit int, after string) ([]*Branch, bool, error)
	BranchExists(ctx context.Context, repository string, branch string) (bool, error)
	GetBranchReference(ctx context.Context, repository, branch string) (string, error)
	ResetBranch(ctx context.Context, repository, branch string) error

	// GetEntry returns the current entry for path in repository branch reference.  Returns
	// the entry with ExpiredError if it has expired from underlying storage.
	GetEntry(ctx context.Context, repository, reference string, path string, params GetEntryParams) (*Entry, error)
	CreateEntry(ctx context.Context, repository, branch string, entry Entry, params CreateEntryParams) error
	CreateEntries(ctx context.Context, repository, branch string, entries []Entry) error
	DeleteEntry(ctx context.Context, repository, branch string, path string) error
	ListEntries(ctx context.Context, repository, reference string, prefix, after string, delimiter string, limit int) ([]*Entry, bool, error)
	ResetEntry(ctx context.Context, repository, branch string, path string) error
	ResetEntries(ctx context.Context, repository, branch string, prefix string) error

	// QueryEntriesToExpire returns ExpiryRows iterating over all objects to expire on
	// repositoryName according to policy.
	QueryEntriesToExpire(ctx context.Context, repositoryName string, policy *Policy) (ExpiryRows, error)
	// MarkEntriesExpired marks all entries identified by expire as expired.  It is a batch operation.
	MarkEntriesExpired(ctx context.Context, repositoryName string, expireResults []*ExpireResult) error
	// MarkObjectsForDeletion marks objects in catalog_object_dedup as "deleting" if all
	// their entries are expired, and returns the new total number of objects marked (or an
	// error).  These objects are not yet safe to delete: there could be a race between
	// marking objects as expired deduping newly-uploaded objects.  See
	// DeleteOrUnmarkObjectsForDeletion for that actual deletion.
	MarkObjectsForDeletion(ctx context.Context, repositoryName string) (int64, error)
	// DeleteOrUnmarkObjectsForDeletion scans objects in catalog_object_dedup for objects
	// marked "deleting" and returns an iterator over physical addresses of those objects
	// all of whose referring entries are still expired.  If called after MarkEntriesExpired
	// and MarkObjectsForDeletion this is safe, because no further entries can refer to
	// expired objects.  It also removes the "deleting" mark from those objects that have an
	// entry _not_ marked as expiring and therefore were not on the returned rows.
	DeleteOrUnmarkObjectsForDeletion(ctx context.Context, repositoryName string) (StringIterator, error)

	DedupReportChannel() chan *DedupReport

	Commit(ctx context.Context, repository, branch string, message string, committer string, metadata Metadata) (*CommitLog, error)
	GetCommit(ctx context.Context, repository, reference string) (*CommitLog, error)
	ListCommits(ctx context.Context, repository, branch string, fromReference string, limit int) ([]*CommitLog, bool, error)
	RollbackCommit(ctx context.Context, repository, branch string, reference string) error

	Diff(ctx context.Context, repository, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)
	DiffUncommitted(ctx context.Context, repository, branch string, limit int, after string) (Differences, bool, error)

	Merge(ctx context.Context, repository, leftBranch, rightBranch, committer, message string, metadata Metadata) (*MergeResult, error)

	Hooks() *CatalogerHooks

	GetExportConfigurationForBranch(repository string, branch string) (ExportConfiguration, error)
	GetExportConfigurations() ([]ExportConfigurationForBranch, error)
	PutExportConfiguration(repository string, branch string, conf *ExportConfiguration) error

	ExportStateSet(repo, branch string, cb ExportStateCallback) error
	// GetExportState returns the current Export state params
	GetExportState(repo string, branch string) (ExportState, error)

	io.Closer
}

type CatalogerHooks added in v0.15.0 

type CatalogerHooks struct {
	// PostCommit hooks are called at the end of a commit.
	PostCommit []PostCommitFunc

	// PostMerge hooks are called at the end of a merge.
	PostMerge []PostMergeFunc
}

CatalogerHooks describes the hooks available for some operations on the catalog. Hooks are called after the transaction ends; if they return an error they do not affect commit/merge.

func (*CatalogerHooks) AddPostCommit added in v0.15.0 

func (h *CatalogerHooks) AddPostCommit(f PostCommitFunc) *CatalogerHooks

func (*CatalogerHooks) AddPostMerge added in v0.15.0 

func (h *CatalogerHooks) AddPostMerge(f PostMergeFunc) *CatalogerHooks

type CommitLog 

type CommitLog struct {
	Reference    string
	Committer    string    `db:"committer"`
	Message      string    `db:"message"`
	CreationDate time.Time `db:"creation_date"`
	Metadata     Metadata  `db:"metadata"`
	Parents      []string
}

type CreateEntryParams 

type CreateEntryParams struct {
	Dedup DedupParams
}

type DedupParams 

type DedupParams struct {
	ID               string
	StorageNamespace string
}

type DedupReport 

type DedupReport struct {
	Repository         string
	StorageNamespace   string
	DedupID            string
	Entry              *Entry
	NewPhysicalAddress string
	Timestamp          time.Time
}

type DiffParams added in v0.15.0 

type DiffParams struct {
	Limit            int
	After            string
	AdditionalFields []string // db fields names that will be load in additional to Path on Difference's Entry
}

type DiffResultRecord added in v0.16.0 

type DiffResultRecord struct {
	TargetEntryNotInDirectBranch bool // the entry is reflected via lineage, NOT in the branch itself
	Difference
	EntryCtid *string // CTID of the modified/added entry. Do not use outside of catalog diff-by-iterators. https://github.com/treeverse/lakeFS/issues/831
}

type Difference 

type Difference struct {
	Entry                // Partially filled. Path is always set.
	Type  DifferenceType `db:"diff_type"`
}

func (Difference) String 

func (d Difference) String() string

type DifferenceType 

type DifferenceType int
const (
	DifferenceTypeAdded DifferenceType = iota
	DifferenceTypeRemoved
	DifferenceTypeChanged
	DifferenceTypeConflict
	DifferenceTypeNone
)

type Differences 

type Differences []Difference

func (Differences) Equal 

func (d Differences) Equal(other Differences) bool

type Entry 

type Entry struct {
	CommonLevel     bool
	Path            string    `db:"path"`
	PhysicalAddress string    `db:"physical_address"`
	CreationDate    time.Time `db:"creation_date"`
	Size            int64     `db:"size"`
	Checksum        string    `db:"checksum"`
	Metadata        Metadata  `db:"metadata"`
	Expired         bool      `db:"is_expired"`
}

type Expiration 

type Expiration struct {
	All         *TimePeriodHours `json:",omitempty"`
	Uncommitted *TimePeriodHours `json:",omitempty"`
	Noncurrent  *TimePeriodHours `json:",omitempty"`
}

type ExpireResult 

type ExpireResult struct {
	Repository        string
	Branch            string
	PhysicalAddress   string
	InternalReference string
}

type ExpiryRows 

type ExpiryRows interface {
	Close()
	Next() bool
	Err() error
	// Read returns the current from ExpiryRows, or an error on failure.  Call it only after
	// successfully calling Next.
	Read() (*ExpireResult, error)
}

ExpiryRows is a database iterator over ExpiryResults. Use Next to advance from row to row.

type ExportConfiguration added in v0.15.0 

type ExportConfiguration struct {
	Path                   string         `db:"export_path" json:"export_path"`
	StatusPath             string         `db:"export_status_path" json:"export_status_path"`
	LastKeysInPrefixRegexp pq.StringArray `db:"last_keys_in_prefix_regexp" json:"last_keys_in_prefix_regexp"`
	IsContinuous           bool           `db:"continuous" json:"is_continuous"`
}

ExportConfiguration describes the export configuration of a branch, as passed on wire, used internally, and stored in DB.

type ExportConfigurationForBranch added in v0.15.0 

type ExportConfigurationForBranch struct {
	Repository string `db:"repository"`
	Branch     string `db:"branch"`

	Path                   string         `db:"export_path"`
	StatusPath             string         `db:"export_status_path"`
	LastKeysInPrefixRegexp pq.StringArray `db:"last_keys_in_prefix_regexp"`
	IsContinuous           bool           `db:"continuous"`
}

ExportConfigurationForBranch describes how to export BranchID. It is stored in the database. Unfortunately golang sql doesn't know about embedded structs, so you get a useless copy of ExportConfiguration embedded here.

type ExportState added in v0.17.0 

type ExportState struct {
	CurrentRef   string                    `db:"current_ref"`
	State        CatalogBranchExportStatus `db:"state"`
	ErrorMessage *string                   `db:"error_message"`
}

ExportStatus describes the current export status of a branch, as passed on wire, used internally, and stored in DB.

type ExportStateCallback added in v0.17.0 

type ExportStateCallback func(oldRef string, state CatalogBranchExportStatus) (newRef string, newState CatalogBranchExportStatus, newMessage *string, err error)

ExportStateCallback returns the new ref, state and message regarding the old ref and state

type GetEntryParams 

type GetEntryParams struct {
	// For entries to expired objects the Expired bit is set.  If true, GetEntry returns
	// successfully for expired entries, otherwise it returns the entry with ErrExpired.
	ReturnExpired bool
}

GetEntryParams configures what entries GetEntry returns.

type MergeResult 

type MergeResult struct {
	Summary   map[DifferenceType]int
	Reference string
}

type Metadata 

type Metadata map[string]string

func (*Metadata) Scan 

func (j *Metadata) Scan(src interface{}) error

func (Metadata) Value 

func (j Metadata) Value() (driver.Value, error)

type Policy 

type Policy struct {
	Rules       Rules
	Description string
}

type PolicyWithCreationTime 

type PolicyWithCreationTime struct {
	Policy
	CreatedAt time.Time `db:"created_at"`
}

type PostCommitFunc added in v0.17.0 

type PostCommitFunc func(ctx context.Context, repo, branch string, commitLog CommitLog) error

type PostMergeFunc added in v0.17.0 

type PostMergeFunc func(ctx context.Context, repo, branch string, mergeResult MergeResult) error

type Repository 

type Repository struct {
	Name             string    `db:"name"`
	StorageNamespace string    `db:"storage_namespace"`
	DefaultBranch    string    `db:"default_branch"`
	CreationDate     time.Time `db:"creation_date"`
}

type Rule 

type Rule struct {
	Enabled      bool
	FilterPrefix string `json:",omitempty"`
	Expiration   Expiration
}

type Rules 

type Rules []Rule

func (*Rules) Scan 

func (a *Rules) Scan(value interface{}) error

type RulesHolder 

type RulesHolder struct {
	Rules Rules
}

RulesHolder is a dummy struct for helping pg serialization: it has poor support for passing an array-valued parameter.

func (*RulesHolder) Value 

func (a *RulesHolder) Value() (driver.Value, error)

type StringIterator added in v0.17.0 

type StringIterator interface {
	Next() bool
	Err() error
	Read() (string, error)
	Close()
}

type TimePeriodHours 

type TimePeriodHours int

Avoid rounding by keeping whole hours (not Durations)

Source Files

View all Source files

Directories

Path Synopsis
params

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.