metadata

Documentation

Overview

Package metadata stores all labels and object specific metadata by namespace. This package also contains the main garbage collection logic for cleaning up resources consistently and atomically. Resources used by backends will be tracked in the metadata store to be exposed to consumers of this package.

The layout where a "/" delineates a bucket is described in the following section. Please try to follow this as closely as possible when adding functionality. We can bolster this with helpers and more structure if that becomes an issue.

Generically, we try to do the following:

<version>/<namespace>/<object>/<key> -> <field>

version: Currently, this is "v1". Additions can be made to v1 in a backwards compatible way. If the layout changes, a new version must be made, along with a migration.

namespace: the namespace to which this object belongs.

object: defines which object set is stored in the bucket. There are two special objects, "labels" and "indexes". The "labels" bucket stores the labels for the parent namespace. The "indexes" object is reserved for indexing objects, if we require in the future.

key: object-specific key identifying the storage bucket for the objects contents.

Below is the current database schema. This should be updated each time the structure is changed in addition to adding a migration and incrementing the database version. Note that `╘══*...*` refers to maps with arbitrary keys.

├──version : <varint>                        - Latest version, see migrations
└──v1                                        - Schema version bucket
   ╘══*namespace*
      ├──labels
      │  ╘══*key* : <string>                 - Label value
      ├──image
      │  ╘══*image name*
      │     ├──createdat : <binary time>     - Created at
      │     ├──updatedat : <binary time>     - Updated at
      │     ├──target
      │     │  ├──digest : <digest>          - Descriptor digest
      │     │  ├──mediatype : <string>       - Descriptor media type
      │     │  └──size : <varint>            - Descriptor size
      │     └──labels
      │        ╘══*key* : <string>           - Label value
      ├──containers
      │  ╘══*container id*
      │     ├──createdat : <binary time>     - Created at
      │     ├──updatedat : <binary time>     - Updated at
      │     ├──spec : <binary>               - Proto marshaled spec
      │     ├──image : <string>              - Image name
      │     ├──snapshotter : <string>        - Snapshotter name
      │     ├──snapshotKey : <string>        - Snapshot key
      │     ├──runtime
      │     │  ├──name : <string>            - Runtime name
      │     │  ├──extensions
      │     │  │  ╘══*name* : <binary>       - Proto marshaled extension
      │     │  └──options : <binary>         - Proto marshaled options
      │     └──labels
      │        ╘══*key* : <string>           - Label value
      ├──snapshots
      │  ╘══*snapshotter*
      │     ╘══*snapshot key*
      │        ├──name : <string>            - Snapshot name in backend
      │        ├──createdat : <binary time>  - Created at
      │        ├──updatedat : <binary time>  - Updated at
      │        ├──parent : <string>          - Parent snapshot name
      │        ├──children
      │        │  ╘══*snapshot key* : <nil>  - Child snapshot reference
      │        └──labels
      │           ╘══*key* : <string>        - Label value
      ├──content
      │  ├──blob
      │  │  ╘══*blob digest*
      │  │     ├──createdat : <binary time>  - Created at
      │  │     ├──updatedat : <binary time>  - Updated at
      │  │     ├──size : <varint>            - Blob size
      │  │     └──labels
      │  │        ╘══*key* : <string>        - Label value
      │  └──ingests
      │     ╘══*ingest reference*
      │        ├──ref : <string>             - Ingest reference in backend
      │        ├──expireat : <binary time>   - Time to expire ingest
      │        └──expected : <digest>        - Expected commit digest
      └──leases
         ╘══*lease id*
            ├──createdat : <binary time>     - Created at
            ├──labels
            │  ╘══*key* : <string>           - Label value
            ├──snapshots
            │  ╘══*snapshotter*
            │     ╘══*snapshot key* : <nil>  - Snapshot reference
            ├──content
            │  ╘══*blob digest* : <nil>      - Content blob reference
            └──ingests
               ╘══*ingest reference* : <nil> - Content ingest reference

Index

• Constants
• func NewContainerStore(tx *bolt.Tx) containers.Store
• func NewImageStore(db *DB) images.Store
• func NewNamespaceStore(tx *bolt.Tx) namespaces.Store
• func WithPolicyIsolated(o *dbOptions)
• func WithTransactionContext(ctx context.Context, tx *bolt.Tx) context.Context
• type DB
  • func NewDB(db *bolt.DB, cs content.Store, ss map[string]snapshots.Snapshotter, ...) *DB
  • func (m *DB) ContentStore() content.Store
  • func (m *DB) GarbageCollect(ctx context.Context) (gc.Stats, error)
  • func (m *DB) Init(ctx context.Context) error
  • func (m *DB) RegisterMutationCallback(fn func(bool))
  • func (m *DB) Snapshotter(name string) snapshots.Snapshotter
  • func (m *DB) Snapshotters() map[string]snapshots.Snapshotter
  • func (m *DB) Update(fn func(*bolt.Tx) error) error
  • func (m *DB) View(fn func(*bolt.Tx) error) error
• type DBOpt
• type GCStats
  • func (s GCStats) Elapsed() time.Duration
• type LeaseManager
  • func NewLeaseManager(tx *bolt.Tx) *LeaseManager
  • func (lm *LeaseManager) AddResource(ctx context.Context, lease leases.Lease, r leases.Resource) error
  • func (lm *LeaseManager) Create(ctx context.Context, opts ...leases.Opt) (leases.Lease, error)
  • func (lm *LeaseManager) Delete(ctx context.Context, lease leases.Lease, _ ...leases.DeleteOpt) error
  • func (lm *LeaseManager) DeleteResource(ctx context.Context, lease leases.Lease, r leases.Resource) error
  • func (lm *LeaseManager) List(ctx context.Context, fs ...string) ([]leases.Lease, error)
  • func (lm *LeaseManager) ListResources(ctx context.Context, lease leases.Lease) ([]leases.Resource, error)

Constants

const (
	// ResourceUnknown specifies an unknown resource
	ResourceUnknown gc.ResourceType = iota
	// ResourceContent specifies a content resource
	ResourceContent
	// ResourceSnapshot specifies a snapshot resource
	ResourceSnapshot
	// ResourceContainer specifies a container resource
	ResourceContainer
	// ResourceTask specifies a task resource
	ResourceTask
	// ResourceLease specifies a lease
	ResourceLease
	// ResourceIngest specifies a content ingest
	ResourceIngest
)

Functions

func NewContainerStore

func NewContainerStore(tx *bolt.Tx) containers.Store

NewContainerStore returns a Store backed by an underlying bolt DB

func NewImageStore

func NewImageStore(db *DB) images.Store

NewImageStore returns a store backed by a bolt DB

func NewNamespaceStore

func NewNamespaceStore(tx *bolt.Tx) namespaces.Store

NewNamespaceStore returns a store backed by a bolt DB

func WithPolicyIsolated

func WithPolicyIsolated(o *dbOptions)

WithPolicyIsolated isolates contents between namespaces

func WithTransactionContext

func WithTransactionContext(ctx context.Context, tx *bolt.Tx) context.Context

WithTransactionContext returns a new context holding the provided bolt transaction. Functions which require a bolt transaction will first check to see if a transaction is already created on the context before creating their own.

Types

type DB

type DB struct {
	// contains filtered or unexported fields
}

DB represents a metadata database backed by a bolt database. The database is fully namespaced and stores image, container, namespace, snapshot, and content data while proxying data shared across namespaces to backend datastores for content and snapshots.

func NewDB

func NewDB(db *bolt.DB, cs content.Store, ss map[string]snapshots.Snapshotter, opts ...DBOpt) *DB

NewDB creates a new metadata database using the provided bolt database, content store, and snapshotters.

func (*DB) ContentStore

func (m *DB) ContentStore() content.Store

ContentStore returns a namespaced content store proxied to a content store.

func (*DB) GarbageCollect

func (m *DB) GarbageCollect(ctx context.Context) (gc.Stats, error)

GarbageCollect starts garbage collection

func (*DB) Init

func (m *DB) Init(ctx context.Context) error

Init ensures the database is at the correct version and performs any needed migrations.

func (*DB) RegisterMutationCallback

func (m *DB) RegisterMutationCallback(fn func(bool))

RegisterMutationCallback registers a function to be called after a metadata mutations has been performed.

The callback function is an argument for whether a deletion has occurred since the last garbage collection.

func (*DB) Snapshotter

func (m *DB) Snapshotter(name string) snapshots.Snapshotter

Snapshotter returns a namespaced content store for the requested snapshotter name proxied to a snapshotter.

func (*DB) Snapshotters

func (m *DB) Snapshotters() map[string]snapshots.Snapshotter

Snapshotters returns all available snapshotters.

func (*DB) Update

func (m *DB) Update(fn func(*bolt.Tx) error) error

Update runs a writable transaction on the metadata store.

func (*DB) View

func (m *DB) View(fn func(*bolt.Tx) error) error

View runs a readonly transaction on the metadata store.

type DBOpt

type DBOpt func(*dbOptions)

DBOpt configures how we set up the DB

type GCStats

type GCStats struct {
	MetaD     time.Duration
	ContentD  time.Duration
	SnapshotD map[string]time.Duration
}

GCStats holds the duration for the different phases of the garbage collector

func (GCStats) Elapsed

func (s GCStats) Elapsed() time.Duration

Elapsed returns the duration which elapsed during a collection

type LeaseManager

type LeaseManager struct {
	// contains filtered or unexported fields
}

LeaseManager manages the create/delete lifecycle of leases and also returns existing leases

func NewLeaseManager

func NewLeaseManager(tx *bolt.Tx) *LeaseManager

NewLeaseManager creates a new lease manager for managing leases using the provided database transaction.

func (*LeaseManager) AddResource

func (lm *LeaseManager) AddResource(ctx context.Context, lease leases.Lease, r leases.Resource) error

AddResource references the resource by the provided lease.

func (*LeaseManager) Create

func (lm *LeaseManager) Create(ctx context.Context, opts ...leases.Opt) (leases.Lease, error)

Create creates a new lease using the provided lease

func (*LeaseManager) Delete

func (lm *LeaseManager) Delete(ctx context.Context, lease leases.Lease, _ ...leases.DeleteOpt) error

Delete deletes the lease with the provided lease ID

func (*LeaseManager) DeleteResource

func (lm *LeaseManager) DeleteResource(ctx context.Context, lease leases.Lease, r leases.Resource) error

DeleteResource dereferences the resource by the provided lease.

func (*LeaseManager) List

func (lm *LeaseManager) List(ctx context.Context, fs ...string) ([]leases.Lease, error)

List lists all active leases

func (*LeaseManager) ListResources

func (lm *LeaseManager) ListResources(ctx context.Context, lease leases.Lease) ([]leases.Resource, error)

ListResources lists all the resources referenced by the lease.