objectstore

Documentation

Index

• Constants
• func BackendTypeOrDefault(objectStoreType objectstore.BackendType) objectstore.BackendType
• func DefaultBackendType() objectstore.BackendType
• type ClaimExtender
• type Claimer
• type FileObjectStoreConfig
• type HashFileSystemAccessor
• type MetadataService
• type ObjectStoreWorkerFunc
• type Option
  • func WithAllowDraining(allowDraining bool) Option
  • func WithClaimer(claimer Claimer) Option
  • func WithClock(clock clock.Clock) Option
  • func WithLogger(logger logger.Logger) Option
  • func WithMetadataService(metadataService MetadataService) Option
  • func WithRootBucket(rootBucket string) Option
  • func WithRootDir(rootDir string) Option
  • func WithS3Client(client objectstore.Client) Option
• type S3ObjectStoreConfig
• type TrackedObjectStore
  • func NewFileObjectStore(cfg FileObjectStoreConfig) (TrackedObjectStore, error)
  • func NewS3ObjectStore(cfg S3ObjectStoreConfig) (TrackedObjectStore, error)
  • func ObjectStoreFactory(ctx context.Context, backendType objectstore.BackendType, namespace string, ...) (TrackedObjectStore, error)

Constants

const (
	// ErrFileLocked is returned when a file is locked.
	ErrFileLocked = errors.ConstError("file locked")
)

Functions

func BackendTypeOrDefault

func BackendTypeOrDefault(objectStoreType objectstore.BackendType) objectstore.BackendType

BackendTypeOrDefault returns the default backend type for the given object store type or falls back to the default backend type.

func DefaultBackendType

func DefaultBackendType() objectstore.BackendType

DefaultBackendType returns the default backend type for the given object store type or falls back to the default backend type.

Types

type ClaimExtender

type ClaimExtender interface {
	// Extend extends the lock for the given hash.
	Extend(ctx context.Context) error

	// Duration returns the duration of the lock.
	Duration() time.Duration
}

ClaimExtender is the interface that is used to extend a lock.

type Claimer

type Claimer interface {
	// Claim locks the blob with the given hash.
	Claim(ctx context.Context, hash string) (ClaimExtender, error)
	// Release releases the blob with the given hash.
	Release(ctx context.Context, hash string) error
}

Claimer is the interface that is used to claim an exclusive lock for a particular object store blob. The lock is used to prevent concurrent access to the same blob for put and remove operations.

type FileObjectStoreConfig

type FileObjectStoreConfig struct {
	// RootDir is the root directory for the file object store.
	RootDir string
	// Namespace is the namespace for the file object store (typically the
	// model UUID)
	Namespace string
	// MetadataService is the metadata service for translating paths to
	// hashes.
	MetadataService objectstore.ObjectStoreMetadata
	// Claimer is the claimer for the file object store.
	Claimer Claimer

	Logger logger.Logger
	Clock  clock.Clock
}

FileObjectStoreConfig is the configuration for the file object store.

type HashFileSystemAccessor

type HashFileSystemAccessor interface {
	// HashExists checks if the file exists in the file backed object store.
	// Returns a NotFound error if the file doesn't exist.
	HashExists(ctx context.Context, hash string) error

	// GetByHash returns an io.ReadCloser for the file at the given hash.
	GetByHash(ctx context.Context, hash string) (io.ReadCloser, int64, error)

	// DeleteByHash deletes the file at the given hash.
	DeleteByHash(ctx context.Context, hash string) error
}

HashFileSystemAccessor is the interface for reading and deleting files from the file system. The file system accessor is used for draining files from the file backed object store to the s3 object store. It should at no point be used for writing files to the file system.

type MetadataService

type MetadataService interface {
	ObjectStore() objectstore.ObjectStoreMetadata
}

MetadataService is the interface that is used to get a object store.

type ObjectStoreWorkerFunc

type ObjectStoreWorkerFunc func(context.Context, objectstore.BackendType, string, ...Option) (TrackedObjectStore, error)

ObjectStoreWorkerFunc is the function signature for creating a new object store worker.

type Option

type Option func(*options)

Option is the function signature for the options to create a new object store.

func WithAllowDraining

func WithAllowDraining(allowDraining bool) Option

WithAllowDraining is the option to set the allow draining to use.

func WithClaimer

func WithClaimer(claimer Claimer) Option

WithClaimer is the option to set the claimer to use.

func WithClock

func WithClock(clock clock.Clock) Option

WithClock is the option to set the clock to use.

func WithLogger

func WithLogger(logger logger.Logger) Option

WithLogger is the option to set the logger to use.

func WithMetadataService

func WithMetadataService(metadataService MetadataService) Option

WithMetadataService is the option to set the metadata service to use.

func WithRootBucket

func WithRootBucket(rootBucket string) Option

WithRootBucket is the option to set the root bucket to use.

func WithRootDir

func WithRootDir(rootDir string) Option

WithRootDir is the option to set the root directory to use.

func WithS3Client

func WithS3Client(client objectstore.Client) Option

WithS3Client is the option to set the s3 client to use.

type S3ObjectStoreConfig

type S3ObjectStoreConfig struct {
	// RootDir is the root directory for the object store. This is the location
	// where the tmp directory will be created.
	// This is different than /tmp because the /tmp directory might be
	// mounted on a different file system.
	RootDir string
	// RootBucket is the name of the root bucket.
	RootBucket string
	// Namespace is the namespace for the object store (typically the
	// model UUID).
	Namespace string
	// Client is the object store client (s3 client).
	Client objectstore.Client
	// MetadataService is the metadata service for translating paths to
	// hashes.
	MetadataService objectstore.ObjectStoreMetadata
	// Claimer is the claimer for locking files.
	Claimer Claimer
	// HashFileSystemAccessor is used for draining files from the file backed
	// object store to the s3 object store.
	HashFileSystemAccessor HashFileSystemAccessor
	// AllowDraining is a flag to allow draining files from the file backed
	// object store to the s3 object store.
	AllowDraining bool

	Logger logger.Logger
	Clock  clock.Clock
}

S3ObjectStoreConfig is the configuration for the s3 object store.

type TrackedObjectStore

type TrackedObjectStore interface {
	worker.Worker
	objectstore.ObjectStore
}

TrackedObjectStore is a ObjectStore that is also a worker, to ensure the lifecycle of the objectStore is managed.

func NewFileObjectStore

func NewFileObjectStore(cfg FileObjectStoreConfig) (TrackedObjectStore, error)

NewFileObjectStore returns a new object store worker based on the file storage.

func NewS3ObjectStore

func NewS3ObjectStore(cfg S3ObjectStoreConfig) (TrackedObjectStore, error)

NewS3ObjectStore returns a new object store worker based on the s3 backing storage.

func ObjectStoreFactory

func ObjectStoreFactory(ctx context.Context, backendType objectstore.BackendType, namespace string, options ...Option) (TrackedObjectStore, error)

ObjectStoreFactory is the function to create a new object store based on the backend type.