catalog

package
v0.15.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 26, 2020 License: Apache-2.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CatalogerCommitter      = ""
	DefaultBranchName       = "master"
	DefaultImportBranchName = "import-from-inventory"
	DefaultPathDelimiter    = "/"

	MaxReadQueue = 10
)
View Source 
const (
	ListEntriesMaxLimit        = 1000
	ListEntriesBranchBatchSize = 32
)
View Source 
const (
	DBEntryFieldChecksum        = "checksum"
	DBEntryFieldPhysicalAddress = "physical_address"
)
View Source 
const (
	CommittedID                   CommitID = -1
	UncommittedID                 CommitID = 0
	MaxCommitID                   CommitID = 1_000_000_000_000_000_000
	MinCommitUncommittedIndicator          = MaxCommitID
	TombstoneCommitID             CommitID = 0

	CommittedSuffix = ":HEAD"
	CommitPrefix    = "~"

	InternalObjectRefSeparator = "$"
	InternalObjectRefFormat    = "int:pbm:%s"
	InternalObjectRefParts     = 3
)
View Source 
const (
	DirectoryTerminationValue = utf8.MaxRune
	DirectoryTermination      = string(rune(DirectoryTerminationValue))
)
View Source 
const DBScannerDefaultBufferSize = 1024
View Source 
const (
	DiffMaxLimit = 1000
)
View Source 
const ListBranchesMaxLimit = 10000
View Source 
const ListCommitsMaxLimit = 10000
View Source 
const ListRepositoriesMaxLimit = 10000

Variables

View Source 
var (
	ErrFeatureNotSupported           = errors.New("feature not supported")
	ErrOperationNotPermitted         = errors.New("operation not permitted")
	ErrInvalidLockValue              = errors.New("invalid lock value")
	ErrNothingToCommit               = errors.New("nothing to commit")
	ErrNoDifferenceWasFound          = errors.New("no difference was found")
	ErrConflictFound                 = errors.New("conflict found")
	ErrUnsupportedRelation           = errors.New("unsupported relation")
	ErrUnsupportedDelimiter          = errors.New("unsupported delimiter")
	ErrInvalidReference              = errors.New("invalid reference")
	ErrBranchNotFound                = fmt.Errorf("branch %w", db.ErrNotFound)
	ErrCommitNotFound                = fmt.Errorf("commit %w", db.ErrNotFound)
	ErrRepositoryNotFound            = fmt.Errorf("repository %w", db.ErrNotFound)
	ErrMultipartUploadNotFound       = fmt.Errorf("multipart upload %w", db.ErrNotFound)
	ErrEntryNotFound                 = fmt.Errorf("entry %w", db.ErrNotFound)
	ErrByteSliceTypeAssertion        = errors.New("type assertion to []byte failed")
	ErrInvalidMetadataSrcFormat      = errors.New("invalid metadata src format")
	ErrUnexpected                    = errors.New("unexpected error")
	ErrReadEntryTimeout              = errors.New("read entry timeout")
	ErrMissingDiffResultsIDInContext = errors.New("missing diff results id in context")
	ErrInvalidValue                  = errors.New("invalid value")
)
View Source 
var ErrExpired = errors.New("expired from storage")

Functions

func IsNonEmptyString 

func IsNonEmptyString(s string) bool

func IsValidBranchName 

func IsValidBranchName(branch string) bool

func IsValidReference 

func IsValidReference(reference string) bool

func IsValidRepositoryName 

func IsValidRepositoryName(repository string) bool

func MakeReference 

func MakeReference(branch string, commitID CommitID) string

func Validate 

func Validate(validators ValidateFields) error

Types

type Branch 

type Branch struct {
	Repository string `db:"repository"`
	Name       string `db:"name"`
}

type BranchCataloger 

type BranchCataloger interface {
	CreateBranch(ctx context.Context, repository, branch string, sourceBranch string) (*CommitLog, error)
	DeleteBranch(ctx context.Context, repository, branch string) error
	ListBranches(ctx context.Context, repository string, prefix string, limit int, after string) ([]*Branch, bool, error)
	BranchExists(ctx context.Context, repository string, branch string) (bool, error)
	GetBranchReference(ctx context.Context, repository, branch string) (string, error)
	ResetBranch(ctx context.Context, repository, branch string) error
}

type Cache 

type Cache interface {
	Repository(repository string, setFn GetRepositoryFn) (*Repository, error)
	RepositoryID(repository string, setFn GetRepositoryIDFn) (int, error)
	BranchID(repository string, branch string, setFn GetBranchIDFn) (int64, error)
}

type CacheConfig 

type CacheConfig struct {
	Enabled bool
	Size    int
	Expiry  time.Duration
	Jitter  time.Duration
}

type Cataloger 

type Cataloger interface {
	RepositoryCataloger
	BranchCataloger
	EntryCataloger
	Committer
	MultipartUpdateCataloger
	Differ
	Merger
	Hookser
	ExportConfigurator
	io.Closer
}

func NewCataloger 

func NewCataloger(db db.Database, options ...CatalogerOption) Cataloger

type CatalogerHooks added in v0.15.0 

type CatalogerHooks struct {
	// PostCommit hooks are called at the end of a commit.
	PostCommit []func(ctx context.Context, tx db.Tx, commitLog *CommitLog) error

	// PostMerge hooks are called at the end of a merge.
	PostMerge []func(ctx context.Context, tx db.Tx, mergeResult *MergeResult) error
}

CatalogerHooks describes the hooks available for some operations on the catalog. Hooks are called in a current transaction context; if they return an error the transaction is rolled back. Because these transactions are current, the hook can see the effect the operation only on the passed transaction.

func (*CatalogerHooks) AddPostCommit added in v0.15.0 

func (h *CatalogerHooks) AddPostCommit(f func(context.Context, db.Tx, *CommitLog) error) *CatalogerHooks

func (*CatalogerHooks) AddPostMerge added in v0.15.0 

func (h *CatalogerHooks) AddPostMerge(f func(context.Context, db.Tx, *MergeResult) error) *CatalogerHooks

type CatalogerOption 

type CatalogerOption func(*cataloger)

func WithCacheEnabled added in v0.10.0 

func WithCacheEnabled(b bool) CatalogerOption

func WithDedupReportChannel 

func WithDedupReportChannel(b bool) CatalogerOption

func WithParams added in v0.10.0 

func WithParams(p params.Catalog) CatalogerOption

type CommitID 

type CommitID int64

type CommitLog 

type CommitLog struct {
	Reference    string
	Committer    string    `db:"committer"`
	Message      string    `db:"message"`
	CreationDate time.Time `db:"creation_date"`
	Metadata     Metadata  `db:"metadata"`
	Parents      []string
}

type Committer 

type Committer interface {
	Commit(ctx context.Context, repository, branch string, message string, committer string, metadata Metadata) (*CommitLog, error)
	GetCommit(ctx context.Context, repository, reference string) (*CommitLog, error)
	ListCommits(ctx context.Context, repository, branch string, fromReference string, limit int) ([]*CommitLog, bool, error)
	RollbackCommit(ctx context.Context, repository, reference string) error
}

type CreateEntryParams 

type CreateEntryParams struct {
	Dedup DedupParams
}

type DBBranchScanner added in v0.15.0 

type DBBranchScanner struct {
	// contains filtered or unexported fields
}

func NewDBBranchScanner added in v0.15.0 

func NewDBBranchScanner(tx db.Tx, branchID int64, commitID CommitID, opts *DBScannerOptions) *DBBranchScanner

func (*DBBranchScanner) Err added in v0.15.0 

func (s *DBBranchScanner) Err() error

func (*DBBranchScanner) Next added in v0.15.0 

func (s *DBBranchScanner) Next() bool

func (*DBBranchScanner) Value added in v0.15.0 

func (s *DBBranchScanner) Value() *DBScannerEntry

type DBLineageScanner added in v0.15.0 

type DBLineageScanner struct {
	// contains filtered or unexported fields
}

func NewDBLineageScanner added in v0.15.0 

func NewDBLineageScanner(tx db.Tx, branchID int64, commitID CommitID, opts *DBScannerOptions) *DBLineageScanner

func (*DBLineageScanner) Err added in v0.15.0 

func (s *DBLineageScanner) Err() error

func (*DBLineageScanner) Next added in v0.15.0 

func (s *DBLineageScanner) Next() bool

func (*DBLineageScanner) ReadLineage added in v0.15.0 

func (s *DBLineageScanner) ReadLineage() ([]lineageCommit, error)

func (*DBLineageScanner) Value added in v0.15.0 

func (s *DBLineageScanner) Value() *DBScannerEntry

type DBScanner added in v0.15.0 

type DBScanner interface {
	Next() bool
	Value() *DBScannerEntry
	Err() error
}

type DBScannerEntry added in v0.15.0 

type DBScannerEntry struct {
	BranchID int64  `db:"branch_id"`
	RowCtid  string `db:"ctid"`
	MinMaxCommit
	Entry
}

func ScanDBEntryUntil added in v0.15.0 

func ScanDBEntryUntil(s DBScanner, ent *DBScannerEntry, p string) (*DBScannerEntry, error)

type DBScannerOptions added in v0.15.0 

type DBScannerOptions struct {
	BufferSize       int
	After            string
	AdditionalFields []string
	AdditionalWhere  sq.Sqlizer
}

type DedupParams 

type DedupParams struct {
	ID               string
	StorageNamespace string
}

type DedupReport 

type DedupReport struct {
	Repository         string
	StorageNamespace   string
	DedupID            string
	Entry              *Entry
	NewPhysicalAddress string
	Timestamp          time.Time
}

type DiffParams added in v0.15.0 

type DiffParams struct {
	Limit            int
	After            string
	AdditionalFields []string // db fields names that will be load in additional to Path on Difference's Entry
}

type Differ 

type Differ interface {
	Diff(ctx context.Context, repository, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)
	DiffUncommitted(ctx context.Context, repository, branch string, limit int, after string) (Differences, bool, error)
}

type Difference 

type Difference struct {
	Entry                // Partially filled.
	Type  DifferenceType `db:"diff_type"`
}

func (Difference) String 

func (d Difference) String() string

type DifferenceType 

type DifferenceType int
const (
	DifferenceTypeAdded DifferenceType = iota
	DifferenceTypeRemoved
	DifferenceTypeChanged
	DifferenceTypeConflict
	DifferenceTypeNone
)

type Differences 

type Differences []Difference

func (Differences) CountByType 

func (d Differences) CountByType() map[DifferenceType]int

func (Differences) Equal 

func (d Differences) Equal(other Differences) bool

type DummyCache 

type DummyCache struct{}

func (*DummyCache) BranchID 

func (c *DummyCache) BranchID(repository string, branch string, setFn GetBranchIDFn) (int64, error)

func (*DummyCache) Repository 

func (c *DummyCache) Repository(repository string, setFn GetRepositoryFn) (*Repository, error)

func (*DummyCache) RepositoryID 

func (c *DummyCache) RepositoryID(repository string, setFn GetRepositoryIDFn) (int, error)

type Entry 

type Entry struct {
	CommonLevel     bool
	Path            string    `db:"path"`
	PhysicalAddress string    `db:"physical_address"`
	CreationDate    time.Time `db:"creation_date"`
	Size            int64     `db:"size"`
	Checksum        string    `db:"checksum"`
	Metadata        Metadata  `db:"metadata"`
	Expired         bool      `db:"is_expired"`
}

type EntryCataloger 

type EntryCataloger interface {
	// GetEntry returns the current entry for path in repository branch reference.  Returns
	// the entry with ExpiredError if it has expired from underlying storage.
	GetEntry(ctx context.Context, repository, reference string, path string, params GetEntryParams) (*Entry, error)
	CreateEntry(ctx context.Context, repository, branch string, entry Entry, params CreateEntryParams) error
	CreateEntries(ctx context.Context, repository, branch string, entries []Entry) error
	DeleteEntry(ctx context.Context, repository, branch string, path string) error
	ListEntries(ctx context.Context, repository, reference string, prefix, after string, delimiter string, limit int) ([]*Entry, bool, error)
	ResetEntry(ctx context.Context, repository, branch string, path string) error
	ResetEntries(ctx context.Context, repository, branch string, prefix string) error

	// QueryEntriesToExpire returns ExpiryRows iterating over all objects to expire on
	// repositoryName according to policy.
	QueryEntriesToExpire(ctx context.Context, repositoryName string, policy *Policy) (ExpiryRows, error)
	// MarkEntriesExpired marks all entries identified by expire as expired.  It is a batch operation.
	MarkEntriesExpired(ctx context.Context, repositoryName string, expireResults []*ExpireResult) error
	// MarkObjectsForDeletion marks objects in catalog_object_dedup as "deleting" if all
	// their entries are expired, and returns the new total number of objects marked (or an
	// error).  These objects are not yet safe to delete: there could be a race between
	// marking objects as expired deduping newly-uploaded objects.  See
	// DeleteOrUnmarkObjectsForDeletion for that actual deletion.
	MarkObjectsForDeletion(ctx context.Context, repositoryName string) (int64, error)
	// DeleteOrUnmarkObjectsForDeletion scans objects in catalog_object_dedup for objects
	// marked "deleting" and returns an iterator over physical addresses of those objects
	// all of whose referring entries are still expired.  If called after MarkEntriesExpired
	// and MarkObjectsForDeletion this is safe, because no further entries can refer to
	// expired objects.  It also removes the "deleting" mark from those objects that have an
	// entry _not_ marked as expiring and therefore were not on the returned rows.
	DeleteOrUnmarkObjectsForDeletion(ctx context.Context, repositoryName string) (StringRows, error)

	DedupReportChannel() chan *DedupReport
}

type Expiration 

type Expiration struct {
	All         *TimePeriodHours `json:",omitempty"`
	Uncommitted *TimePeriodHours `json:",omitempty"`
	Noncurrent  *TimePeriodHours `json:",omitempty"`
}

type ExpireResult 

type ExpireResult struct {
	Repository        string
	Branch            string
	PhysicalAddress   string
	InternalReference string
}

type ExpiryRows 

type ExpiryRows interface {
	io.Closer
	Next() bool
	Err() error
	// Read returns the current from ExpiryRows, or an error on failure.  Call it only after
	// successfully calling Next.
	Read() (*ExpireResult, error)
}

ExpiryRows is a database iterator over ExpiryResults. Use Next to advance from row to row.

type ExportConfiguration added in v0.15.0 

type ExportConfiguration struct {
	Path                   string         `db:"export_path"`
	StatusPath             string         `db:"export_status_path"`
	LastKeysInPrefixRegexp pq.StringArray `db:"last_keys_in_prefix_regexp"`
}

ExportConfiguration describes the export configuration of a branch, as passed on wire, used internally, and stored in DB.

type ExportConfigurationForBranch added in v0.15.0 

type ExportConfigurationForBranch struct {
	Repository string `db:"repository"`
	Branch     string `db:"branch"`

	Path                   string         `db:"export_path"`
	StatusPath             string         `db:"export_status_path"`
	LastKeysInPrefixRegexp pq.StringArray `db:"last_keys_in_prefix_regexp"`
}

ExportConfigurationForBranch describes how to export BranchID. It is stored in the database. Unfortunately golang sql doesn't know about embedded structs, so you get a useless copy of ExportConfiguration embedded here.

type ExportConfigurator added in v0.15.0 

type ExportConfigurator interface {
	GetExportConfigurationForBranch(repository string, branch string) (ExportConfiguration, error)
	GetExportConfigurations() ([]ExportConfigurationForBranch, error)
	PutExportConfiguration(repository string, branch string, conf *ExportConfiguration) error
}

type GetBranchIDFn 

type GetBranchIDFn func(repository string, branch string) (int64, error)

type GetEntryParams 

type GetEntryParams struct {
	// For entries to expired objects the Expired bit is set.  If true, GetEntry returns
	// successfully for expired entries, otherwise it returns the entry with ErrExpired.
	ReturnExpired bool
}

GetEntryParams configures what entries GetEntry returns.

type GetRepositoryFn 

type GetRepositoryFn func(repository string) (*Repository, error)

type GetRepositoryIDFn 

type GetRepositoryIDFn func(repository string) (int, error)

type Hookser added in v0.15.0 

type Hookser interface {
	Hooks() *CatalogerHooks
}

type InternalObjectRef 

type InternalObjectRef struct {
	BranchID  int64
	MinCommit CommitID
	Path      string
}

InternalObjectRef provides information that uniquely identifies an object between transactions. It might be invalidated by some database changes.

func ParseInternalObjectRef 

func ParseInternalObjectRef(refString string) (InternalObjectRef, error)

func (*InternalObjectRef) String 

func (sor *InternalObjectRef) String() string

type LRUCache 

type LRUCache struct {
	// contains filtered or unexported fields
}

func NewLRUCache 

func NewLRUCache(size int, expiry, jitter time.Duration) *LRUCache

func (*LRUCache) BranchID 

func (c *LRUCache) BranchID(repository string, branch string, setFn GetBranchIDFn) (int64, error)

func (*LRUCache) Repository 

func (c *LRUCache) Repository(repository string, setFn GetRepositoryFn) (*Repository, error)

func (*LRUCache) RepositoryID 

func (c *LRUCache) RepositoryID(repository string, setFn GetRepositoryIDFn) (int, error)

type LockType 

type LockType int
const (
	LockTypeNone LockType = iota
	LockTypeShare
	LockTypeUpdate
)

type MergeResult 

type MergeResult struct {
	Summary   map[DifferenceType]int
	Reference string
}

type Merger 

type Merger interface {
	Merge(ctx context.Context, repository, leftBranch, rightBranch, committer, message string, metadata Metadata) (*MergeResult, error)
}

type Metadata 

type Metadata map[string]string

func (*Metadata) Scan 

func (j *Metadata) Scan(src interface{}) error

func (Metadata) Value 

func (j Metadata) Value() (driver.Value, error)

type MinMaxCommit added in v0.13.0 

type MinMaxCommit struct {
	MinCommit CommitID `db:"min_commit"`
	MaxCommit CommitID `db:"max_commit"`
}

func (MinMaxCommit) ChangedAfterCommit added in v0.13.0 

func (m MinMaxCommit) ChangedAfterCommit(commitID CommitID) bool

func (MinMaxCommit) IsCommitted added in v0.13.0 

func (m MinMaxCommit) IsCommitted() bool

func (MinMaxCommit) IsDeleted added in v0.13.0 

func (m MinMaxCommit) IsDeleted() bool

func (MinMaxCommit) IsTombstone added in v0.13.0 

func (m MinMaxCommit) IsTombstone() bool

type MultipartUpdateCataloger 

type MultipartUpdateCataloger interface {
	CreateMultipartUpload(ctx context.Context, repository, uploadID, path, physicalAddress string, creationTime time.Time) error
	GetMultipartUpload(ctx context.Context, repository, uploadID string) (*MultipartUpload, error)
	DeleteMultipartUpload(ctx context.Context, repository, uploadID string) error
}

type MultipartUpload 

type MultipartUpload struct {
	Repository      string    `db:"repository"`
	UploadID        string    `db:"upload_id"`
	Path            string    `db:"path"`
	CreationDate    time.Time `db:"creation_date"`
	PhysicalAddress string    `db:"physical_address"`
}

type Policy 

type Policy struct {
	Rules       Rules
	Description string
}

type PolicyWithCreationTime 

type PolicyWithCreationTime struct {
	Policy
	CreatedAt time.Time `db:"created_at"`
}

type Ref 

type Ref struct {
	Branch   string
	CommitID CommitID
}

func ParseRef 

func ParseRef(ref string) (*Ref, error)

func (Ref) String 

func (r Ref) String() string

type RelationType 

type RelationType string
const (
	RelationTypeNone       RelationType = "none"
	RelationTypeFromParent RelationType = "from_parent"
	RelationTypeFromChild  RelationType = "from_child"
	RelationTypeNotDirect  RelationType = "non_direct"
	RelationTypeSame       RelationType = "same"
)

type Repository 

type Repository struct {
	Name             string    `db:"name"`
	StorageNamespace string    `db:"storage_namespace"`
	DefaultBranch    string    `db:"default_branch"`
	CreationDate     time.Time `db:"creation_date"`
}

type RepositoryCataloger 

type RepositoryCataloger interface {
	CreateRepository(ctx context.Context, repository string, storageNamespace string, branch string) (*Repository, error)
	GetRepository(ctx context.Context, repository string) (*Repository, error)
	DeleteRepository(ctx context.Context, repository string) error
	ListRepositories(ctx context.Context, limit int, after string) ([]*Repository, bool, error)
}

type Rule 

type Rule struct {
	Enabled      bool
	FilterPrefix string `json:",omitempty"`
	Expiration   Expiration
}

type Rules 

type Rules []Rule

func (*Rules) Scan 

func (a *Rules) Scan(value interface{}) error

type RulesHolder 

type RulesHolder struct {
	Rules Rules
}

RulesHolder is a dummy struct for helping pg serialization: it has poor support for passing an array-valued parameter.

func (*RulesHolder) Value 

func (a *RulesHolder) Value() (driver.Value, error)

type StringRows added in v0.9.0 

type StringRows struct {
	// contains filtered or unexported fields
}

func (*StringRows) Close added in v0.9.0 

func (s *StringRows) Close() error

func (*StringRows) Err added in v0.9.0 

func (s *StringRows) Err() error

func (*StringRows) Next added in v0.9.0 

func (s *StringRows) Next() bool

func (*StringRows) Read added in v0.9.0 

func (s *StringRows) Read() (string, error)

type TimePeriodHours 

type TimePeriodHours int

Avoid rounding by keeping whole hours (not Durations)

type ValidateField 

type ValidateField struct {
	Name    string
	IsValid ValidateFunc
}

type ValidateFields 

type ValidateFields []ValidateField

type ValidateFunc 

type ValidateFunc func() bool

func ValidateBranchName 

func ValidateBranchName(branch string) ValidateFunc

func ValidateCommitMessage 

func ValidateCommitMessage(message string) ValidateFunc

func ValidateCommitter 

func ValidateCommitter(name string) ValidateFunc

func ValidateOptionalString 

func ValidateOptionalString(s string, validator func(string) bool) ValidateFunc

func ValidatePath 

func ValidatePath(name string) ValidateFunc

func ValidatePhysicalAddress 

func ValidatePhysicalAddress(addr string) ValidateFunc

func ValidateReference 

func ValidateReference(reference string) ValidateFunc

func ValidateRepositoryName 

func ValidateRepositoryName(repository string) ValidateFunc

func ValidateStorageNamespace 

func ValidateStorageNamespace(storageNamespace string) ValidateFunc

func ValidateUploadID 

func ValidateUploadID(uploadID string) ValidateFunc

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.