baggageclaim

README

baggage claim

a volume manager for garden containers

by atmx

reporting issues and requesting features

please report all issues and feature requests in concourse/concourse

about

baggageclaim allows you to create and layer volumes on a remote server. This is particularly useful when used with bind mounts or the RootFS when using Garden. It allows directories to be made which can be populated before having copy-on-write layers layered on top. e.g. to provide caching.

Documentation

Overview

Package baggageclaim is the interface for communicating with a BaggageClaim volume server.

BaggageClaim is an auxilary service that can be collocated with various container servers (Garden, Docker, etc.) to let them share directories. BaggageClaim provides a number of benefits over regular bind mounts:

By bringing everything into a the same Volume model we can compose different technologies together. For example, a Docker image is a stack of layered volumes which can have a Concourse build cache layered on top of them.

Volumes can be Copy-on-Write (COW) copies of other volumes. This lets us download a Docker image once and then let it be used by untrusted jobs without fear that they'll mutate it in some unexpected way. This same COW strategy can be applied to any volume that BaggageClaim supports.

BaggageClaim volumes go through a three stage lifecycle of being born, existing, and then dying. This state model is required as creating large amounts of data can potentially take a long time to materialize. You are only able to interact with volumes that are in the middle state.

It's the responsibility of the API consumer to delete child volumes before parent volumes.

The standard way to construct a client is:

import "github.com/concourse/baggageclaim/client"

bcClient := client.New("http://baggageclaim.example.com:7788")
bcClient.CreateVolume(...)

Index

• Constants
• Variables
• type COWStrategy
  • func (strategy COWStrategy) Encode() *json.RawMessage
• type Client
• type EmptyStrategy
  • func (EmptyStrategy) Encode() *json.RawMessage
• type Encoding
• type ImportStrategy
  • func (strategy ImportStrategy) Encode() *json.RawMessage
• type PrivilegedRequest
• type PropertyRequest
• type Strategy
• type Volume
• type VolumeFuture
• type VolumeFutureResponse
• type VolumeProperties
• type VolumeRequest
• type VolumeResponse
• type VolumeSpec
• type Volumes
  • func (v Volumes) Handles() []string

Constants

const (
	ListVolumes    = "ListVolumes"
	GetVolume      = "GetVolume"
	CreateVolume   = "CreateVolume"
	DestroyVolume  = "DestroyVolume"
	DestroyVolumes = "DestroyVolumes"

	CreateVolumeAsync       = "CreateVolumeAsync"
	CreateVolumeAsyncCancel = "CreateVolumeAsyncCancel"
	CreateVolumeAsyncCheck  = "CreateVolumeAsyncCheck"

	SetProperty   = "SetProperty"
	GetPrivileged = "GetPrivileged"
	SetPrivileged = "SetPrivileged"
	StreamIn      = "StreamIn"
	StreamOut     = "StreamOut"
	StreamP2pOut  = "StreamP2pOut"

	GetP2pUrl = "GetP2pUrl"
)

Variables

var ErrFileNotFound = errors.New("file not found")

var ErrVolumeNotFound = errors.New("volume not found")

var Routes = rata.Routes{
	{Path: "/volumes", Method: "GET", Name: ListVolumes},
	{Path: "/volumes", Method: "POST", Name: CreateVolume},

	{Path: "/volumes-async", Method: "POST", Name: CreateVolumeAsync},
	{Path: "/volumes-async/:handle", Method: "GET", Name: CreateVolumeAsyncCheck},
	{Path: "/volumes-async/:handle", Method: "DELETE", Name: CreateVolumeAsyncCancel},

	{Path: "/volumes/:handle", Method: "GET", Name: GetVolume},
	{Path: "/volumes/:handle/properties/:property", Method: "PUT", Name: SetProperty},
	{Path: "/volumes/:handle/privileged", Method: "GET", Name: GetPrivileged},
	{Path: "/volumes/:handle/privileged", Method: "PUT", Name: SetPrivileged},
	{Path: "/volumes/:handle/stream-in", Method: "PUT", Name: StreamIn},
	{Path: "/volumes/:handle/stream-out", Method: "PUT", Name: StreamOut},
	{Path: "/volumes/:handle/stream-p2p-out", Method: "PUT", Name: StreamP2pOut},
	{Path: "/volumes/destroy", Method: "DELETE", Name: DestroyVolumes},
	{Path: "/volumes/:handle", Method: "DELETE", Name: DestroyVolume},

	{Path: "/p2p-url", Method: "GET", Name: GetP2pUrl},
}

Types

type COWStrategy

type COWStrategy struct {
	// The parent volume that we should base the new volume on.
	Parent Volume
}

COWStrategy creates a Copy-On-Write layer of another Volume.

func (COWStrategy) Encode

func (strategy COWStrategy) Encode() *json.RawMessage

type Client

type Client interface {
	// CreateVolume will create a volume on the remote server. By passing in a
	// VolumeSpec with a different strategy you can choose the type of volume
	// that you want to create.
	//
	// You are required to pass in a logger to the call to retain context across
	// the library boundary.
	//
	// CreateVolume returns the volume that was created or an error as to why it
	// could not be created.
	CreateVolume(lager.Logger, string, VolumeSpec) (Volume, error)

	// ListVolumes lists the volumes that are present on the server. A
	// VolumeProperties object can be passed in to filter the volumes that are in
	// the response.
	//
	// You are required to pass in a logger to the call to retain context across
	// the library boundary.
	//
	// ListVolumes returns the volumes that were found or an error as to why they
	// could not be listed.
	ListVolumes(lager.Logger, VolumeProperties) (Volumes, error)

	// LookupVolume finds a volume that is present on the server. It takes a
	// string that corresponds to the Handle of the Volume.
	//
	// You are required to pass in a logger to the call to retain context across
	// the library boundary.
	//
	// LookupVolume returns a bool if the volume is found with the matching volume
	// or an error as to why the volume could not be found.
	LookupVolume(lager.Logger, string) (Volume, bool, error)

	// DestroyVolumes deletes the list of volumes that is present on the server. It takes
	// a string of volumes
	//
	// You are required to pass in a logger to the call to retain context across
	// the library boundary.
	//
	// DestroyVolumes returns an error if any of the volume deletion fails. It does not
	// return an error if volumes were not found on the server.
	// DestroyVolumes returns an error as to why one or more volumes could not be deleted.
	DestroyVolumes(lager.Logger, []string) error

	// DestroyVolume deletes the volume with the provided handle that is present on the server.
	//
	// You are required to pass in a logger to the call to retain context across
	// the library boundary.
	//
	// DestroyVolume returns an error if the volume deletion fails. It does not
	// return an error if the volume was not found on the server.
	DestroyVolume(lager.Logger, string) error
}

Client represents a client connection to a BaggageClaim server.

type EmptyStrategy

type EmptyStrategy struct{}

EmptyStrategy created a new empty volume.

func (EmptyStrategy) Encode

func (EmptyStrategy) Encode() *json.RawMessage

type Encoding

type Encoding string

const GzipEncoding Encoding = "gzip"

const ZstdEncoding Encoding = "zstd"

type ImportStrategy

type ImportStrategy struct {
	// The location on the host to import. If the path is a directory, its
	// contents will be copied in. If the path is a file, it is assumed to be a
	// .tar.gz file, and its contents will be unpacked in to the volume.
	Path string

	// Follow symlinks and import them as files instead of links.
	FollowSymlinks bool
}

ImportStrategy creates a volume by copying a directory from the host.

func (ImportStrategy) Encode

func (strategy ImportStrategy) Encode() *json.RawMessage

type PrivilegedRequest

type PrivilegedRequest struct {
	Value bool `json:"value"`
}

type PropertyRequest

type PropertyRequest struct {
	Value string `json:"value"`
}

type Strategy

type Strategy interface {
	Encode() *json.RawMessage
}

type Volume

type Volume interface {
	// Handle returns a per-server unique identifier for the volume. The URL of
	// the server and a handle is enough to universally identify a volume.
	Handle() string

	// Path returns the filesystem path to the volume on the server. This can be
	// supplied to other systems in order to let them use the volume.
	Path() string

	// SetProperty sets a property on the Volume. Properties can be used to
	// filter the results in the ListVolumes call above.
	SetProperty(key string, value string) error

	// SetPrivileged namespaces or un-namespaces the UID/GID ownership of the
	// volume's contents.
	SetPrivileged(bool) error

	// GetPrivileged returns a bool indicating if the volume is privileged.
	GetPrivileged() (bool, error)

	// StreamIn calls BaggageClaim API endpoint in order to initialize tarStream
	// to stream the contents of the Reader into this volume at the specified path.
	StreamIn(ctx context.Context, path string, encoding Encoding, tarStream io.Reader) error

	StreamOut(ctx context.Context, path string, encoding Encoding) (io.ReadCloser, error)

	// Properties returns the currently set properties for a Volume. An error is
	// returned if these could not be retrieved.
	Properties() (VolumeProperties, error)

	// Destroy removes the volume and its contents. Note that it does not
	// safeguard against child volumes being present.
	Destroy() error

	// GetStreamInP2pUrl returns a modified StreamIn URL for this volume. The
	// returned URL contains a hostname that is reachable by other baggageclaim
	// servers on the same network. The URL can be passed to another
	// baggageclaim server to stream the contents of its source volume into
	// this target volume.
	GetStreamInP2pUrl(ctx context.Context, path string) (string, error)

	// StreamP2pOut streams the contents of this volume directly to another
	// baggageclaim server on the same network.
	StreamP2pOut(ctx context.Context, path string, streamInURL string, encoding Encoding) error
}

Volume represents a volume in the BaggageClaim system.

type VolumeFuture

type VolumeFuture interface {
	// Wait will wait until the future has been provided with a value, which is
	// either the volume that was created or an error as to why it could not be
	// created.
	Wait() (Volume, error)

	// Destroy removes the future from the remote server. This can be used to
	// either stop waiting for a value, or remove the value from the remote
	// server after it is no longer needed.
	Destroy() error
}

type VolumeFutureResponse

type VolumeFutureResponse struct {
	Handle string `json:"handle"`
}

type VolumeProperties

type VolumeProperties map[string]string

VolumeProperties represents the properties for a particular volume.

type VolumeRequest

type VolumeRequest struct {
	Handle     string           `json:"handle"`
	Strategy   *json.RawMessage `json:"strategy"`
	Properties VolumeProperties `json:"properties"`
	Privileged bool             `json:"privileged,omitempty"`
}

type VolumeResponse

type VolumeResponse struct {
	Handle     string           `json:"handle"`
	Path       string           `json:"path"`
	Properties VolumeProperties `json:"properties"`
}

type VolumeSpec

type VolumeSpec struct {
	// Strategy is the information that the server requires to materialise the
	// volume. There are examples of these in this package.
	Strategy Strategy

	// Properties is the set of initial properties that the Volume should have.
	Properties VolumeProperties

	// Privileged is used to determine whether or not we need to perform a UID
	// translation of the files in the volume so that they can be read by a
	// non-privileged user.
	Privileged bool
}

VolumeSpec is a specification representing the kind of volume that you'd like from the server.

type Volumes

type Volumes []Volume

Volumes represents a list of Volume object.

func (Volumes) Handles

func (v Volumes) Handles() []string