catalog

package
v0.13.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 4, 2020 License: Apache-2.0 Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	CatalogerCommitter = ""

	DefaultPathDelimiter = "/"

	MaxReadQueue = 10
)
View Source 
const (
	ListEntriesMaxLimit        = 1000
	ListEntriesBranchBatchSize = 32
)
View Source 
const (
	CommittedID                   CommitID = -1
	UncommittedID                 CommitID = 0
	MaxCommitID                   CommitID = 1_000_000_000_000_000_000
	MinCommitUncommittedIndicator          = MaxCommitID
	TombstoneCommitID             CommitID = 0

	CommittedSuffix = ":HEAD"
	CommitPrefix    = "~"

	InternalObjectRefSeparator = "$"
	InternalObjectRefFormat    = "int:pbm:%s"
	InternalObjectRefParts     = 3
)
View Source 
const (
	DirectoryTerminationValue = utf8.MaxRune
	DirectoryTermination      = string(rune(DirectoryTerminationValue))
)
View Source 
const (
	DBReaderMaxLimit = 1000
)
View Source 
const (
	DiffMaxLimit = 1000
)
View Source 
const ListBranchesMaxLimit = 10000
View Source 
const ListCommitsMaxLimit = 10000
View Source 
const ListRepositoriesMaxLimit = 10000

Variables

View Source 
var (
	ErrFeatureNotSupported      = errors.New("feature not supported")
	ErrOperationNotPermitted    = errors.New("operation not permitted")
	ErrInvalidLockValue         = errors.New("invalid lock value")
	ErrNothingToCommit          = errors.New("nothing to commit")
	ErrNoDifferenceWasFound     = errors.New("no difference was found")
	ErrConflictFound            = errors.New("conflict found")
	ErrUnsupportedRelation      = errors.New("unsupported relation")
	ErrUnsupportedDelimiter     = errors.New("unsupported delimiter")
	ErrInvalidReference         = errors.New("invalid reference")
	ErrBranchNotFound           = fmt.Errorf("branch %w", db.ErrNotFound)
	ErrCommitNotFound           = fmt.Errorf("commit %w", db.ErrNotFound)
	ErrRepositoryNotFound       = fmt.Errorf("repository %w", db.ErrNotFound)
	ErrMultipartUploadNotFound  = fmt.Errorf("multipart upload %w", db.ErrNotFound)
	ErrEntryNotFound            = fmt.Errorf("entry %w", db.ErrNotFound)
	ErrByteSliceTypeAssertion   = errors.New("type assertion to []byte failed")
	ErrInvalidMetadataSrcFormat = errors.New("invalid metadata src format")
	ErrUnexpected               = errors.New("unexpected error")
	ErrReadEntryTimeout         = errors.New("read entry timeout")
)
View Source 
var ErrExpired = errors.New("expired from storage")
View Source 
var (
	ErrInvalidValue = errors.New("invalid value")
)
View Source 
var ErrMissingDiffResultsIDInContext = errors.New("missing diff results id in context")

Functions

func IsNonEmptyString 

func IsNonEmptyString(s string) bool

func IsValidBranchName 

func IsValidBranchName(branch string) bool

func IsValidReference 

func IsValidReference(reference string) bool

func IsValidRepositoryName 

func IsValidRepositoryName(repository string) bool

func MakeReference 

func MakeReference(branch string, commitID CommitID) string

func Validate 

func Validate(validators ValidateFields) error

Types

type Branch 

type Branch struct {
	Repository string `db:"repository"`
	Name       string `db:"name"`
}

type BranchCataloger 

type BranchCataloger interface {
	CreateBranch(ctx context.Context, repository, branch string, sourceBranch string) (*CommitLog, error)
	DeleteBranch(ctx context.Context, repository, branch string) error
	ListBranches(ctx context.Context, repository string, prefix string, limit int, after string) ([]*Branch, bool, error)
	BranchExists(ctx context.Context, repository string, branch string) (bool, error)
	GetBranchReference(ctx context.Context, repository, branch string) (string, error)
	ResetBranch(ctx context.Context, repository, branch string) error
}

type Cache 

type Cache interface {
	Repository(repository string, setFn GetRepositoryFn) (*Repository, error)
	RepositoryID(repository string, setFn GetRepositoryIDFn) (int, error)
	BranchID(repository string, branch string, setFn GetBranchIDFn) (int64, error)
}

type CacheConfig 

type CacheConfig struct {
	Enabled bool
	Size    int
	Expiry  time.Duration
	Jitter  time.Duration
}

type Cataloger 

type Cataloger interface {
	RepositoryCataloger
	BranchCataloger
	EntryCataloger
	Committer
	MultipartUpdateCataloger
	Differ
	Merger
	io.Closer
}

func NewCataloger 

func NewCataloger(db db.Database, options ...CatalogerOption) Cataloger

type CatalogerOption 

type CatalogerOption func(*cataloger)

func WithCacheEnabled added in v0.10.0 

func WithCacheEnabled(b bool) CatalogerOption

func WithDedupReportChannel 

func WithDedupReportChannel(b bool) CatalogerOption

func WithParams added in v0.10.0 

func WithParams(p params.Catalog) CatalogerOption

type CommitID 

type CommitID int64

type CommitLog 

type CommitLog struct {
	Reference    string
	Committer    string    `db:"committer"`
	Message      string    `db:"message"`
	CreationDate time.Time `db:"creation_date"`
	Metadata     Metadata  `db:"metadata"`
	Parents      []string
}

type Committer 

type Committer interface {
	Commit(ctx context.Context, repository, branch string, message string, committer string, metadata Metadata) (*CommitLog, error)
	GetCommit(ctx context.Context, repository, reference string) (*CommitLog, error)
	ListCommits(ctx context.Context, repository, branch string, fromReference string, limit int) ([]*CommitLog, bool, error)
	RollbackCommit(ctx context.Context, repository, reference string) error
}

type CreateEntryParams 

type CreateEntryParams struct {
	Dedup DedupParams
}

type DBBranchReader added in v0.12.0 

type DBBranchReader struct {
	EOF bool
	// contains filtered or unexported fields
}

func NewDBBranchReader added in v0.12.0 

func NewDBBranchReader(tx db.Tx, branchID int64, commitID CommitID, bufSize int, after string) *DBBranchReader

func (*DBBranchReader) Next added in v0.12.0 

func (r *DBBranchReader) Next() (*DBReaderEntry, error)

type DBLineageReader added in v0.12.0 

type DBLineageReader struct {
	EOF bool
	// contains filtered or unexported fields
}

func NewDBLineageReader added in v0.12.0 

func NewDBLineageReader(tx db.Tx, branchID int64, commitID CommitID, bufSize int, limit int, after string) (*DBLineageReader, error)

func (*DBLineageReader) Next added in v0.12.0 

func (r *DBLineageReader) Next() (*DBReaderEntry, error)

type DBReaderEntry added in v0.12.0 

type DBReaderEntry struct {
	BranchID int64  `db:"branch_id"`
	Path     string `db:"path"`
	MinMaxCommit
	RowCtid string `db:"ctid"`
}

type DedupParams 

type DedupParams struct {
	ID               string
	StorageNamespace string
}

type DedupReport 

type DedupReport struct {
	Repository         string
	StorageNamespace   string
	DedupID            string
	Entry              *Entry
	NewPhysicalAddress string
	Timestamp          time.Time
}

type Differ 

type Differ interface {
	Diff(ctx context.Context, repository, leftBranch string, rightBranch string, limit int, after string) (Differences, bool, error)
	DiffUncommitted(ctx context.Context, repository, branch string, limit int, after string) (Differences, bool, error)
}

type Difference 

type Difference struct {
	Type DifferenceType `db:"diff_type"`
	Path string         `db:"path"`
}

func (Difference) String 

func (d Difference) String() string

type DifferenceType 

type DifferenceType int
const (
	DifferenceTypeAdded DifferenceType = iota
	DifferenceTypeRemoved
	DifferenceTypeChanged
	DifferenceTypeConflict
)

type Differences 

type Differences []Difference

func (Differences) CountByType 

func (d Differences) CountByType() map[DifferenceType]int

func (Differences) Equal 

func (d Differences) Equal(other Differences) bool

type DummyCache 

type DummyCache struct{}

func (*DummyCache) BranchID 

func (c *DummyCache) BranchID(repository string, branch string, setFn GetBranchIDFn) (int64, error)

func (*DummyCache) Repository 

func (c *DummyCache) Repository(repository string, setFn GetRepositoryFn) (*Repository, error)

func (*DummyCache) RepositoryID 

func (c *DummyCache) RepositoryID(repository string, setFn GetRepositoryIDFn) (int, error)

type Entry 

type Entry struct {
	CommonLevel     bool
	Path            string    `db:"path"`
	PhysicalAddress string    `db:"physical_address"`
	CreationDate    time.Time `db:"creation_date"`
	Size            int64     `db:"size"`
	Checksum        string    `db:"checksum"`
	Metadata        Metadata  `db:"metadata"`
	Expired         bool      `db:"is_expired"`
}

type EntryCataloger 

type EntryCataloger interface {
	// GetEntry returns the current entry for path in repository branch reference.  Returns
	// the entry with ExpiredError if it has expired from underlying storage.
	GetEntry(ctx context.Context, repository, reference string, path string, params GetEntryParams) (*Entry, error)
	CreateEntry(ctx context.Context, repository, branch string, entry Entry, params CreateEntryParams) error
	CreateEntries(ctx context.Context, repository, branch string, entries []Entry) error
	DeleteEntry(ctx context.Context, repository, branch string, path string) error
	ListEntries(ctx context.Context, repository, reference string, prefix, after string, delimiter string, limit int) ([]*Entry, bool, error)
	ResetEntry(ctx context.Context, repository, branch string, path string) error
	ResetEntries(ctx context.Context, repository, branch string, prefix string) error

	// QueryEntriesToExpire returns ExpiryRows iterating over all objects to expire on
	// repositoryName according to policy.
	QueryEntriesToExpire(ctx context.Context, repositoryName string, policy *Policy) (ExpiryRows, error)
	// MarkEntriesExpired marks all entries identified by expire as expired.  It is a batch operation.
	MarkEntriesExpired(ctx context.Context, repositoryName string, expireResults []*ExpireResult) error
	// MarkObjectsForDeletion marks objects in catalog_object_dedup as "deleting" if all
	// their entries are expired, and returns the new total number of objects marked (or an
	// error).  These objects are not yet safe to delete: there could be a race between
	// marking objects as expired deduping newly-uploaded objects.  See
	// DeleteOrUnmarkObjectsForDeletion for that actual deletion.
	MarkObjectsForDeletion(ctx context.Context, repositoryName string) (int64, error)
	// DeleteOrUnmarkObjectsForDeletion scans objects in catalog_object_dedup for objects
	// marked "deleting" and returns an iterator over physical addresses of those objects
	// all of whose referring entries are still expired.  If called after MarkEntriesExpired
	// and MarkObjectsForDeletion this is safe, because no further entries can refer to
	// expired objects.  It also removes the "deleting" mark from those objects that have an
	// entry _not_ marked as expiring and therefore were not on the returned rows.
	DeleteOrUnmarkObjectsForDeletion(ctx context.Context, repositoryName string) (StringRows, error)

	DedupReportChannel() chan *DedupReport
}

type Expiration 

type Expiration struct {
	All         *TimePeriodHours `json:",omitempty"`
	Uncommitted *TimePeriodHours `json:",omitempty"`
	Noncurrent  *TimePeriodHours `json:",omitempty"`
}

type ExpireResult 

type ExpireResult struct {
	Repository        string
	Branch            string
	PhysicalAddress   string
	InternalReference string
}

type ExpiryRows 

type ExpiryRows interface {
	io.Closer
	Next() bool
	Err() error
	// Read returns the current from ExpiryRows, or an error on failure.  Call it only after
	// successfully calling Next.
	Read() (*ExpireResult, error)
}

ExpiryRows is a database iterator over ExpiryResults. Use Next to advance from row to row.

type GetBranchIDFn 

type GetBranchIDFn func(repository string, branch string) (int64, error)

type GetEntryParams 

type GetEntryParams struct {
	// For entries to expired objects the Expired bit is set.  If true, GetEntry returns
	// successfully for expired entries, otherwise it returns the entry with ErrExpired.
	ReturnExpired bool
}

GetEntryParams configures what entries GetEntry returns.

type GetRepositoryFn 

type GetRepositoryFn func(repository string) (*Repository, error)

type GetRepositoryIDFn 

type GetRepositoryIDFn func(repository string) (int, error)

type InternalObjectRef 

type InternalObjectRef struct {
	BranchID  int64
	MinCommit CommitID
	Path      string
}

InternalObjectRef provides information that uniquely identifies an object between transactions. It might be invalidated by some database changes.

func ParseInternalObjectRef 

func ParseInternalObjectRef(refString string) (InternalObjectRef, error)

func (*InternalObjectRef) String 

func (sor *InternalObjectRef) String() string

type LRUCache 

type LRUCache struct {
	// contains filtered or unexported fields
}

func NewLRUCache 

func NewLRUCache(size int, expiry, jitter time.Duration) *LRUCache

func (*LRUCache) BranchID 

func (c *LRUCache) BranchID(repository string, branch string, setFn GetBranchIDFn) (int64, error)

func (*LRUCache) Repository 

func (c *LRUCache) Repository(repository string, setFn GetRepositoryFn) (*Repository, error)

func (*LRUCache) RepositoryID 

func (c *LRUCache) RepositoryID(repository string, setFn GetRepositoryIDFn) (int, error)

type LockType 

type LockType int
const (
	LockTypeNone LockType = iota
	LockTypeShare
	LockTypeUpdate
)

type MergeResult 

type MergeResult struct {
	Summary   map[DifferenceType]int
	Reference string
}

type Merger 

type Merger interface {
	Merge(ctx context.Context, repository, leftBranch, rightBranch, committer, message string, metadata Metadata) (*MergeResult, error)
}

type Metadata 

type Metadata map[string]string

func (*Metadata) Scan 

func (j *Metadata) Scan(src interface{}) error

func (Metadata) Value 

func (j Metadata) Value() (driver.Value, error)

type MinMaxCommit added in v0.13.0 

type MinMaxCommit struct {
	MinCommit CommitID `db:"min_commit"`
	MaxCommit CommitID `db:"max_commit"`
}

func (MinMaxCommit) ChangedAfterCommit added in v0.13.0 

func (m MinMaxCommit) ChangedAfterCommit(commitID CommitID) bool

func (MinMaxCommit) IsCommitted added in v0.13.0 

func (m MinMaxCommit) IsCommitted() bool

func (MinMaxCommit) IsDeleted added in v0.13.0 

func (m MinMaxCommit) IsDeleted() bool

func (MinMaxCommit) IsTombstone added in v0.13.0 

func (m MinMaxCommit) IsTombstone() bool

type MultipartUpdateCataloger 

type MultipartUpdateCataloger interface {
	CreateMultipartUpload(ctx context.Context, repository, uploadID, path, physicalAddress string, creationTime time.Time) error
	GetMultipartUpload(ctx context.Context, repository, uploadID string) (*MultipartUpload, error)
	DeleteMultipartUpload(ctx context.Context, repository, uploadID string) error
}

type MultipartUpload 

type MultipartUpload struct {
	Repository      string    `db:"repository"`
	UploadID        string    `db:"upload_id"`
	Path            string    `db:"path"`
	CreationDate    time.Time `db:"creation_date"`
	PhysicalAddress string    `db:"physical_address"`
}

type Policy 

type Policy struct {
	Rules       Rules
	Description string
}

type PolicyWithCreationTime 

type PolicyWithCreationTime struct {
	Policy
	CreatedAt time.Time `db:"created_at"`
}

type Ref 

type Ref struct {
	Branch   string
	CommitID CommitID
}

func ParseRef 

func ParseRef(ref string) (*Ref, error)

func (Ref) String 

func (r Ref) String() string

type RelationType 

type RelationType string
const (
	RelationTypeNone       RelationType = "none"
	RelationTypeFromParent RelationType = "from_parent"
	RelationTypeFromChild  RelationType = "from_child"
	RelationTypeNotDirect  RelationType = "non_direct"
)

type Repository 

type Repository struct {
	Name             string    `db:"name"`
	StorageNamespace string    `db:"storage_namespace"`
	DefaultBranch    string    `db:"default_branch"`
	CreationDate     time.Time `db:"creation_date"`
}

type RepositoryCataloger 

type RepositoryCataloger interface {
	CreateRepository(ctx context.Context, repository string, storageNamespace string, branch string) error
	GetRepository(ctx context.Context, repository string) (*Repository, error)
	DeleteRepository(ctx context.Context, repository string) error
	ListRepositories(ctx context.Context, limit int, after string) ([]*Repository, bool, error)
}

type Rule 

type Rule struct {
	Enabled      bool
	FilterPrefix string `json:",omitempty"`
	Expiration   Expiration
}

type Rules 

type Rules []Rule

func (*Rules) Scan 

func (a *Rules) Scan(value interface{}) error

type RulesHolder 

type RulesHolder struct {
	Rules Rules
}

RulesHolder is a dummy struct for helping pg serialization: it has poor support for passing an array-valued parameter.

func (*RulesHolder) Value 

func (a *RulesHolder) Value() (driver.Value, error)

type StringRows added in v0.9.0 

type StringRows struct {
	// contains filtered or unexported fields
}

func (*StringRows) Close added in v0.9.0 

func (s *StringRows) Close() error

func (*StringRows) Err added in v0.9.0 

func (s *StringRows) Err() error

func (*StringRows) Next added in v0.9.0 

func (s *StringRows) Next() bool

func (*StringRows) Read added in v0.9.0 

func (s *StringRows) Read() (string, error)

type TimePeriodHours 

type TimePeriodHours int

Avoid rounding by keeping whole hours (not Durations)

type ValidateField 

type ValidateField struct {
	Name    string
	IsValid ValidateFunc
}

type ValidateFields 

type ValidateFields []ValidateField

type ValidateFunc 

type ValidateFunc func() bool

func ValidateBranchName 

func ValidateBranchName(branch string) ValidateFunc

func ValidateCommitMessage 

func ValidateCommitMessage(message string) ValidateFunc

func ValidateCommitter 

func ValidateCommitter(name string) ValidateFunc

func ValidateOptionalString 

func ValidateOptionalString(s string, validator func(string) bool) ValidateFunc

func ValidatePath 

func ValidatePath(name string) ValidateFunc

func ValidatePhysicalAddress 

func ValidatePhysicalAddress(addr string) ValidateFunc

func ValidateReference 

func ValidateReference(reference string) ValidateFunc

func ValidateRepositoryName 

func ValidateRepositoryName(repository string) ValidateFunc

func ValidateStorageNamespace 

func ValidateStorageNamespace(storageNamespace string) ValidateFunc

func ValidateUploadID 

func ValidateUploadID(uploadID string) ValidateFunc

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.