catalog

package
v0.15.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Oct 26, 2020 License: Apache-2.0 Imports: 25 Imported by: 0

Documentation

Index

Constants

View Source
const (
	CatalogerCommitter      = ""
	DefaultBranchName       = "master"
	DefaultImportBranchName = "import-from-inventory"
	DefaultPathDelimiter    = "/"

	MaxReadQueue = 10
)
View Source
const (
	ListEntriesMaxLimit        = 1000
	ListEntriesBranchBatchSize = 32
)
View Source
const (
	DBEntryFieldChecksum        = "checksum"
	DBEntryFieldPhysicalAddress = "physical_address"
)
View Source
const (
	CommittedID                   CommitID = -1
	UncommittedID                 CommitID = 0
	MaxCommitID                   CommitID = 1_000_000_000_000_000_000
	MinCommitUncommittedIndicator          = MaxCommitID
	TombstoneCommitID             CommitID = 0

	CommittedSuffix = ":HEAD"
	CommitPrefix    = "~"

	InternalObjectRefSeparator = "$"
	InternalObjectRefFormat    = "int:pbm:%s"
	InternalObjectRefParts     = 3
)
View Source
const (
	DirectoryTerminationValue = utf8.MaxRune
	DirectoryTermination      = string(rune(DirectoryTerminationValue))
)
View Source
const DBScannerDefaultBufferSize = 1024
View Source
const (
	DiffMaxLimit = 1000
)
View Source
const ListBranchesMaxLimit = 10000
View Source
const ListCommitsMaxLimit = 10000
View Source
const ListRepositoriesMaxLimit = 10000

Variables

View Source
var (
	ErrFeatureNotSupported           = errors.New("feature not supported")
	ErrOperationNotPermitted         = errors.New("operation not permitted")
	ErrInvalidLockValue              = errors.New("invalid lock value")
	ErrNothingToCommit               = errors.New("nothing to commit")
	ErrNoDifferenceWasFound          = errors.New("no difference was found")
	ErrConflictFound                 = errors.New("conflict found")
	ErrUnsupportedRelation           = errors.New("unsupported relation")
	ErrUnsupportedDelimiter          = errors.New("unsupported delimiter")
	ErrInvalidReference              = errors.New("invalid reference")
	ErrBranchNotFound                = fmt.Errorf("branch %w", db.ErrNotFound)
	ErrCommitNotFound                = fmt.Errorf("commit %w", db.ErrNotFound)
	ErrRepositoryNotFound            = fmt.Errorf("repository %w", db.ErrNotFound)
	ErrMultipartUploadNotFound       = fmt.Errorf("multipart upload %w", db.ErrNotFound)
	ErrEntryNotFound                 = fmt.Errorf("entry %w", db.ErrNotFound)
	ErrByteSliceTypeAssertion        = errors.New("type assertion to []byte failed")
	ErrInvalidMetadataSrcFormat      = errors.New("invalid metadata src format")
	ErrUnexpected                    = errors.New("unexpected error")
	ErrReadEntryTimeout              = errors.New("read entry timeout")
	ErrMissingDiffResultsIDInContext = errors.New("missing diff results id in context")
	ErrInvalidValue                  = errors.New("invalid value")
)
View Source
var ErrExpired = errors.New("expired from storage")

Functions

func IsNonEmptyString

func IsNonEmptyString(s string) bool

func IsValidBranchName

func IsValidBranchName(branch string) bool

func IsValidReference

func IsValidReference(reference string) bool

func IsValidRepositoryName

func IsValidRepositoryName(repository string) bool

func MakeReference

func MakeReference(branch string, commitID CommitID) string

func Validate

func Validate(validators ValidateFields) error

Types

type Branch

type Branch struct {
	Repository string `db:"repository"`
	Name       string `db:"name"`
}

type BranchCataloger

type BranchCataloger interface {
	CreateBranch(ctx context.Context, repository, branch string, sourceBranch string) (*CommitLog, error)
	DeleteBranch(ctx context.Context, repository, branch string) error
	ListBranches(ctx context.Context, repository string, prefix string, limit int, after string) ([]*Branch, bool, error)
	BranchExists(ctx context.Context, repository string, branch string) (bool, error)
	GetBranchReference(ctx context.Context, repository, branch string) (string, error)
	ResetBranch(ctx context.Context, repository, branch string) error
}

type Cache

type Cache interface {
	Repository(repository string, setFn GetRepositoryFn) (*Repository, error)
	RepositoryID(repository string, setFn GetRepositoryIDFn) (int, error)
	BranchID(repository string, branch string, setFn GetBranchIDFn) (int64, error)
}

type CacheConfig

type CacheConfig struct {
	Enabled bool
	Size    int
	Expiry  time.Duration
	Jitter  time.Duration
}

type CatalogerHooks added in v0.15.0

type CatalogerHooks struct {
	// PostCommit hooks are called at the end of a commit.
	PostCommit []func(ctx context.Context, tx db.Tx, commitLog *CommitLog) error

	// PostMerge hooks are called at the end of a merge.
	PostMerge []func(ctx context.Context, tx db.Tx, mergeResult *MergeResult) error
}

CatalogerHooks describes the hooks available for some operations on the catalog. Hooks are called in a current transaction context; if they return an error the transaction is rolled back. Because these transactions are current, the hook can see the effect the operation only on the passed transaction.

func (*CatalogerHooks) AddPostCommit added in v0.15.0

func (h *CatalogerHooks) AddPostCommit(f func(context.Context, db.Tx, *CommitLog) error) *CatalogerHooks

func (*CatalogerHooks) AddPostMerge added in v0.15.0

func (h *CatalogerHooks) AddPostMerge(f func(context.Context, db.Tx, *MergeResult) error) *CatalogerHooks

type CatalogerOption

type CatalogerOption func(*cataloger)

func WithCacheEnabled added in v0.10.0

func WithCacheEnabled(b bool) CatalogerOption

func WithDedupReportChannel

func WithDedupReportChannel(b bool) CatalogerOption

func WithParams added in v0.10.0

func WithParams(p params.Catalog) CatalogerOption

type CommitID

type CommitID int64

type CommitLog

type CommitLog struct {
	Reference    string
	Committer    string    `db:"committer"`
	Message      string    `db:"message"`
	CreationDate time.Time `db:"creation_date"`
	Metadata     Metadata  `db:"metadata"`
	Parents      []string
}

type Committer

type Committer interface {
	Commit(ctx context.Context, repository, branch string, message string, committer string, metadata Metadata) (*CommitLog, error)
	GetCommit(ctx context.Context, repository, reference string) (*CommitLog, error)
	ListCommits(ctx context.Context, repository, branch string, fromReference string, limit int) ([]*CommitLog, bool, error)
	RollbackCommit(ctx context.Context, repository, reference string) error
}

type CreateEntryParams

type CreateEntryParams struct {
	Dedup DedupParams
}

type DBBranchScanner added in v0.15.0

type DBBranchScanner struct {
	// contains filtered or unexported fields
}

func NewDBBranchScanner added in v0.15.0

func NewDBBranchScanner(tx db.Tx, branchID int64, commitID CommitID, opts *DBScannerOptions) *DBBranchScanner

func (*DBBranchScanner) Err added in v0.15.0

func (s *DBBranchScanner) Err() error

func (*DBBranchScanner) Next added in v0.15.0

func (s *DBBranchScanner) Next() bool

func (*DBBranchScanner) Value added in v0.15.0

func (s *DBBranchScanner) Value() *DBScannerEntry

type DBLineageScanner added in v0.15.0

type DBLineageScanner struct {
	// contains filtered or unexported fields
}

func NewDBLineageScanner added in v0.15.0

func NewDBLineageScanner(tx db.Tx, branchID int64, commitID CommitID, opts *DBScannerOptions) *DBLineageScanner

func (*DBLineageScanner) Err added in v0.15.0

func (s *DBLineageScanner) Err() error

func (*DBLineageScanner) Next added in v0.15.0

func (s *DBLineageScanner) Next() bool

func (*DBLineageScanner) ReadLineage added in v0.15.0

func (s *DBLineageScanner) ReadLineage() ([]lineageCommit, error)

func (*DBLineageScanner) Value added in v0.15.0

func (s *DBLineageScanner) Value() *DBScannerEntry

type DBScanner added in v0.15.0

type DBScanner interface {
	Next() bool
	Value() *DBScannerEntry
	Err() error
}

type DBScannerEntry added in v0.15.0

type DBScannerEntry struct {
	BranchID int64  `db:"branch_id"`
	RowCtid  string `db:"ctid"`
	MinMaxCommit
	Entry
}

func ScanDBEntryUntil added in v0.15.0

func ScanDBEntryUntil(s DBScanner, ent *DBScannerEntry, p string) (*DBScannerEntry, error)

type DBScannerOptions added in v0.15.0

type DBScannerOptions struct {
	BufferSize       int
	After            string
	AdditionalFields []string
	AdditionalWhere  sq.Sqlizer
}

type DedupParams

type DedupParams struct {
	ID               string
	StorageNamespace string
}

type DedupReport

type DedupReport struct {
	Repository         string
	StorageNamespace   string
	DedupID            string
	Entry              *Entry
	NewPhysicalAddress string
	Timestamp          time.Time
}

type DiffParams added in v0.15.0

type DiffParams struct {
	Limit            int
	After            string
	AdditionalFields []string // db fields names that will be load in additional to Path on Difference's Entry
}

type Differ

type Differ interface {
	Diff(ctx context.Context, repository, leftReference string, rightReference string, params DiffParams) (Differences, bool, error)
	DiffUncommitted(ctx context.Context, repository, branch string, limit int, after string) (Differences, bool, error)
}

type Difference

type Difference struct {
	Entry                // Partially filled.
	Type  DifferenceType `db:"diff_type"`
}

func (Difference) String

func (d Difference) String() string

type DifferenceType

type DifferenceType int
const (
	DifferenceTypeAdded DifferenceType = iota
	DifferenceTypeRemoved
	DifferenceTypeChanged
	DifferenceTypeConflict
	DifferenceTypeNone
)

type Differences

type Differences []Difference

func (Differences) CountByType

func (d Differences) CountByType() map[DifferenceType]int

func (Differences) Equal

func (d Differences) Equal(other Differences) bool

type DummyCache

type DummyCache struct{}

func (*DummyCache) BranchID

func (c *DummyCache) BranchID(repository string, branch string, setFn GetBranchIDFn) (int64, error)

func (*DummyCache) Repository

func (c *DummyCache) Repository(repository string, setFn GetRepositoryFn) (*Repository, error)

func (*DummyCache) RepositoryID

func (c *DummyCache) RepositoryID(repository string, setFn GetRepositoryIDFn) (int, error)

type Entry

type Entry struct {
	CommonLevel     bool
	Path            string    `db:"path"`
	PhysicalAddress string    `db:"physical_address"`
	CreationDate    time.Time `db:"creation_date"`
	Size            int64     `db:"size"`
	Checksum        string    `db:"checksum"`
	Metadata        Metadata  `db:"metadata"`
	Expired         bool      `db:"is_expired"`
}

type EntryCataloger

type EntryCataloger interface {
	// GetEntry returns the current entry for path in repository branch reference.  Returns
	// the entry with ExpiredError if it has expired from underlying storage.
	GetEntry(ctx context.Context, repository, reference string, path string, params GetEntryParams) (*Entry, error)
	CreateEntry(ctx context.Context, repository, branch string, entry Entry, params CreateEntryParams) error
	CreateEntries(ctx context.Context, repository, branch string, entries []Entry) error
	DeleteEntry(ctx context.Context, repository, branch string, path string) error
	ListEntries(ctx context.Context, repository, reference string, prefix, after string, delimiter string, limit int) ([]*Entry, bool, error)
	ResetEntry(ctx context.Context, repository, branch string, path string) error
	ResetEntries(ctx context.Context, repository, branch string, prefix string) error

	// QueryEntriesToExpire returns ExpiryRows iterating over all objects to expire on
	// repositoryName according to policy.
	QueryEntriesToExpire(ctx context.Context, repositoryName string, policy *Policy) (ExpiryRows, error)
	// MarkEntriesExpired marks all entries identified by expire as expired.  It is a batch operation.
	MarkEntriesExpired(ctx context.Context, repositoryName string, expireResults []*ExpireResult) error
	// MarkObjectsForDeletion marks objects in catalog_object_dedup as "deleting" if all
	// their entries are expired, and returns the new total number of objects marked (or an
	// error).  These objects are not yet safe to delete: there could be a race between
	// marking objects as expired deduping newly-uploaded objects.  See
	// DeleteOrUnmarkObjectsForDeletion for that actual deletion.
	MarkObjectsForDeletion(ctx context.Context, repositoryName string) (int64, error)
	// DeleteOrUnmarkObjectsForDeletion scans objects in catalog_object_dedup for objects
	// marked "deleting" and returns an iterator over physical addresses of those objects
	// all of whose referring entries are still expired.  If called after MarkEntriesExpired
	// and MarkObjectsForDeletion this is safe, because no further entries can refer to
	// expired objects.  It also removes the "deleting" mark from those objects that have an
	// entry _not_ marked as expiring and therefore were not on the returned rows.
	DeleteOrUnmarkObjectsForDeletion(ctx context.Context, repositoryName string) (StringRows, error)

	DedupReportChannel() chan *DedupReport
}

type Expiration

type Expiration struct {
	All         *TimePeriodHours `json:",omitempty"`
	Uncommitted *TimePeriodHours `json:",omitempty"`
	Noncurrent  *TimePeriodHours `json:",omitempty"`
}

type ExpireResult

type ExpireResult struct {
	Repository        string
	Branch            string
	PhysicalAddress   string
	InternalReference string
}

type ExpiryRows

type ExpiryRows interface {
	io.Closer
	Next() bool
	Err() error
	// Read returns the current from ExpiryRows, or an error on failure.  Call it only after
	// successfully calling Next.
	Read() (*ExpireResult, error)
}

ExpiryRows is a database iterator over ExpiryResults. Use Next to advance from row to row.

type ExportConfiguration added in v0.15.0

type ExportConfiguration struct {
	Path                   string         `db:"export_path"`
	StatusPath             string         `db:"export_status_path"`
	LastKeysInPrefixRegexp pq.StringArray `db:"last_keys_in_prefix_regexp"`
}

ExportConfiguration describes the export configuration of a branch, as passed on wire, used internally, and stored in DB.

type ExportConfigurationForBranch added in v0.15.0

type ExportConfigurationForBranch struct {
	Repository string `db:"repository"`
	Branch     string `db:"branch"`

	Path                   string         `db:"export_path"`
	StatusPath             string         `db:"export_status_path"`
	LastKeysInPrefixRegexp pq.StringArray `db:"last_keys_in_prefix_regexp"`
}

ExportConfigurationForBranch describes how to export BranchID. It is stored in the database. Unfortunately golang sql doesn't know about embedded structs, so you get a useless copy of ExportConfiguration embedded here.

type ExportConfigurator added in v0.15.0

type ExportConfigurator interface {
	GetExportConfigurationForBranch(repository string, branch string) (ExportConfiguration, error)
	GetExportConfigurations() ([]ExportConfigurationForBranch, error)
	PutExportConfiguration(repository string, branch string, conf *ExportConfiguration) error
}

type GetBranchIDFn

type GetBranchIDFn func(repository string, branch string) (int64, error)

type GetEntryParams

type GetEntryParams struct {
	// For entries to expired objects the Expired bit is set.  If true, GetEntry returns
	// successfully for expired entries, otherwise it returns the entry with ErrExpired.
	ReturnExpired bool
}

GetEntryParams configures what entries GetEntry returns.

type GetRepositoryFn

type GetRepositoryFn func(repository string) (*Repository, error)

type GetRepositoryIDFn

type GetRepositoryIDFn func(repository string) (int, error)

type Hookser added in v0.15.0

type Hookser interface {
	Hooks() *CatalogerHooks
}

type InternalObjectRef

type InternalObjectRef struct {
	BranchID  int64
	MinCommit CommitID
	Path      string
}

InternalObjectRef provides information that uniquely identifies an object between transactions. It might be invalidated by some database changes.

func ParseInternalObjectRef

func ParseInternalObjectRef(refString string) (InternalObjectRef, error)

func (*InternalObjectRef) String

func (sor *InternalObjectRef) String() string

type LRUCache

type LRUCache struct {
	// contains filtered or unexported fields
}

func NewLRUCache

func NewLRUCache(size int, expiry, jitter time.Duration) *LRUCache

func (*LRUCache) BranchID

func (c *LRUCache) BranchID(repository string, branch string, setFn GetBranchIDFn) (int64, error)

func (*LRUCache) Repository

func (c *LRUCache) Repository(repository string, setFn GetRepositoryFn) (*Repository, error)

func (*LRUCache) RepositoryID

func (c *LRUCache) RepositoryID(repository string, setFn GetRepositoryIDFn) (int, error)

type LockType

type LockType int
const (
	LockTypeNone LockType = iota
	LockTypeShare
	LockTypeUpdate
)

type MergeResult

type MergeResult struct {
	Summary   map[DifferenceType]int
	Reference string
}

type Merger

type Merger interface {
	Merge(ctx context.Context, repository, leftBranch, rightBranch, committer, message string, metadata Metadata) (*MergeResult, error)
}

type Metadata

type Metadata map[string]string

func (*Metadata) Scan

func (j *Metadata) Scan(src interface{}) error

func (Metadata) Value

func (j Metadata) Value() (driver.Value, error)

type MinMaxCommit added in v0.13.0

type MinMaxCommit struct {
	MinCommit CommitID `db:"min_commit"`
	MaxCommit CommitID `db:"max_commit"`
}

func (MinMaxCommit) ChangedAfterCommit added in v0.13.0

func (m MinMaxCommit) ChangedAfterCommit(commitID CommitID) bool

func (MinMaxCommit) IsCommitted added in v0.13.0

func (m MinMaxCommit) IsCommitted() bool

func (MinMaxCommit) IsDeleted added in v0.13.0

func (m MinMaxCommit) IsDeleted() bool

func (MinMaxCommit) IsTombstone added in v0.13.0

func (m MinMaxCommit) IsTombstone() bool

type MultipartUpdateCataloger

type MultipartUpdateCataloger interface {
	CreateMultipartUpload(ctx context.Context, repository, uploadID, path, physicalAddress string, creationTime time.Time) error
	GetMultipartUpload(ctx context.Context, repository, uploadID string) (*MultipartUpload, error)
	DeleteMultipartUpload(ctx context.Context, repository, uploadID string) error
}

type MultipartUpload

type MultipartUpload struct {
	Repository      string    `db:"repository"`
	UploadID        string    `db:"upload_id"`
	Path            string    `db:"path"`
	CreationDate    time.Time `db:"creation_date"`
	PhysicalAddress string    `db:"physical_address"`
}

type Policy

type Policy struct {
	Rules       Rules
	Description string
}

type PolicyWithCreationTime

type PolicyWithCreationTime struct {
	Policy
	CreatedAt time.Time `db:"created_at"`
}

type Ref

type Ref struct {
	Branch   string
	CommitID CommitID
}

func ParseRef

func ParseRef(ref string) (*Ref, error)

func (Ref) String

func (r Ref) String() string

type RelationType

type RelationType string
const (
	RelationTypeNone       RelationType = "none"
	RelationTypeFromParent RelationType = "from_parent"
	RelationTypeFromChild  RelationType = "from_child"
	RelationTypeNotDirect  RelationType = "non_direct"
	RelationTypeSame       RelationType = "same"
)

type Repository

type Repository struct {
	Name             string    `db:"name"`
	StorageNamespace string    `db:"storage_namespace"`
	DefaultBranch    string    `db:"default_branch"`
	CreationDate     time.Time `db:"creation_date"`
}

type RepositoryCataloger

type RepositoryCataloger interface {
	CreateRepository(ctx context.Context, repository string, storageNamespace string, branch string) (*Repository, error)
	GetRepository(ctx context.Context, repository string) (*Repository, error)
	DeleteRepository(ctx context.Context, repository string) error
	ListRepositories(ctx context.Context, limit int, after string) ([]*Repository, bool, error)
}

type Rule

type Rule struct {
	Enabled      bool
	FilterPrefix string `json:",omitempty"`
	Expiration   Expiration
}

type Rules

type Rules []Rule

func (*Rules) Scan

func (a *Rules) Scan(value interface{}) error

type RulesHolder

type RulesHolder struct {
	Rules Rules
}

RulesHolder is a dummy struct for helping pg serialization: it has poor support for passing an array-valued parameter.

func (*RulesHolder) Value

func (a *RulesHolder) Value() (driver.Value, error)

type StringRows added in v0.9.0

type StringRows struct {
	// contains filtered or unexported fields
}

func (*StringRows) Close added in v0.9.0

func (s *StringRows) Close() error

func (*StringRows) Err added in v0.9.0

func (s *StringRows) Err() error

func (*StringRows) Next added in v0.9.0

func (s *StringRows) Next() bool

func (*StringRows) Read added in v0.9.0

func (s *StringRows) Read() (string, error)

type TimePeriodHours

type TimePeriodHours int

Avoid rounding by keeping whole hours (not Durations)

type ValidateField

type ValidateField struct {
	Name    string
	IsValid ValidateFunc
}

type ValidateFields

type ValidateFields []ValidateField

type ValidateFunc

type ValidateFunc func() bool

func ValidateBranchName

func ValidateBranchName(branch string) ValidateFunc

func ValidateCommitMessage

func ValidateCommitMessage(message string) ValidateFunc

func ValidateCommitter

func ValidateCommitter(name string) ValidateFunc

func ValidateOptionalString

func ValidateOptionalString(s string, validator func(string) bool) ValidateFunc

func ValidatePath

func ValidatePath(name string) ValidateFunc

func ValidatePhysicalAddress

func ValidatePhysicalAddress(addr string) ValidateFunc

func ValidateReference

func ValidateReference(reference string) ValidateFunc

func ValidateRepositoryName

func ValidateRepositoryName(repository string) ValidateFunc

func ValidateStorageNamespace

func ValidateStorageNamespace(storageNamespace string) ValidateFunc

func ValidateUploadID

func ValidateUploadID(uploadID string) ValidateFunc

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL