bucket

Documentation

Index

• Constants
• func GetDummyIdent(executorID model.ExecutorID) internal.ResourceIdent
• func GetDummyJobID(executorID model.ExecutorID) model.JobID
• func GetDummyResPath(filename string) string
• func GetDummyResourceKey(executorID model.ExecutorID) resModel.ResourceKey
• func GetDummyResourceName() string
• func GetExternalStorageFromURI(ctx context.Context, uri string, opts *brStorage.BackendOptions) (brStorage.ExternalStorage, error)
• func GetS3OptionsForUT() (*brStorage.S3BackendOptions, error)
• func NewResourceController(config *resModel.Config) *resourceController
• func PreCheckConfig(config *resModel.Config) error
• type Creator
• type CreatorImpl
  • func NewCreator(config *model.Config) *CreatorImpl
• type FileManager
  • func NewFileManager(executorID resModel.ExecutorID, creator Creator) *FileManager
  • func NewFileManagerForUT(tempDir string, executorID resModel.ExecutorID) (*FileManager, *mockBucketCreator)
  • func NewFileManagerForUTFromSharedStorageFactory(executorID model.ExecutorID, creator *mockBucketCreator) *FileManager
  • func NewFileManagerWithConfig(executorID resModel.ExecutorID, config *resModel.Config) *FileManager
  • func (m *FileManager) CleanOrRecreatePersistedResource(ctx context.Context, ident internal.ResourceIdent) (internal.ResourceDescriptor, error)
  • func (m *FileManager) CreateResource(ctx context.Context, ident internal.ResourceIdent) (internal.ResourceDescriptor, error)
  • func (m *FileManager) GetPersistedResource(ctx context.Context, ident internal.ResourceIdent) (internal.ResourceDescriptor, error)
  • func (m *FileManager) RemoveResource(ctx context.Context, ident internal.ResourceIdent) error
  • func (m *FileManager) RemoveTemporaryFiles(ctx context.Context, scope internal.ResourceScope) error
  • func (m *FileManager) SetPersisted(ctx context.Context, ident internal.ResourceIdent) error

Constants

const (
	// DummyWorkerID is a dummy worker ID used for the s3 storage.
	DummyWorkerID = "keep-alive-worker"
	// DummyResourceID is a dummy resource ID used for the s3 storage.
	DummyResourceID = "/s3/dummy"
)

const (

	// UtBucketName is the bucket name for UT
	UtBucketName = "engine-ut"
	// MockExecutorID is the executor ID for mock executor
	MockExecutorID = "executor-1"
)

Functions

func GetDummyIdent

func GetDummyIdent(executorID model.ExecutorID) internal.ResourceIdent

GetDummyIdent returns a dummy resource ident for testing.

func GetDummyJobID

func GetDummyJobID(executorID model.ExecutorID) model.JobID

GetDummyJobID returns a dummy job ID for s3 storage.

func GetDummyResPath

func GetDummyResPath(filename string) string

GetDummyResPath returns a file path located in dummy resource for s3 storage.

func GetDummyResourceKey

func GetDummyResourceKey(executorID model.ExecutorID) resModel.ResourceKey

GetDummyResourceKey returns a dummy resource key for s3 storage.

func GetDummyResourceName

func GetDummyResourceName() string

GetDummyResourceName returns a dummy resource name for s3 storage.

func GetExternalStorageFromURI

func GetExternalStorageFromURI(
	ctx context.Context, uri string, opts *brStorage.BackendOptions,
) (brStorage.ExternalStorage, error)

GetExternalStorageFromURI creates a new brStorage.ExternalStorage from a uri.

func GetS3OptionsForUT

func GetS3OptionsForUT() (*brStorage.S3BackendOptions, error)

GetS3OptionsForUT returns the s3 options for unit test.

func NewResourceController

func NewResourceController(config *resModel.Config) *resourceController

NewResourceController creates a new bucket resourceController.

func PreCheckConfig

func PreCheckConfig(config *resModel.Config) error

PreCheckConfig does a preflight check on the executor's storage configurations.

Types

type Creator

type Creator interface {
	// contains filtered or unexported methods
}

Creator represents a creator used to create brStorage.ExternalStorage. Implementing mock or stub Creator will make unit testing easier.

type CreatorImpl

type CreatorImpl struct {
	// Bucket represents a name of an s3 bucket.
	Bucket string
	// Prefix is an optional prefix in the S3 file path.
	// It can be useful when a shared bucket is used for testing purposes.
	Prefix string
	// Options provide necessary information such as endpoints and access key
	// for creating an s3 client.
	Options *brStorage.BackendOptions
	// ResourceType is the bucket type of this creator
	ResourceType model.ResourceType
}

CreatorImpl implements Creator. It is exported for testing purposes.

func NewCreator

func NewCreator(
	config *model.Config,
) *CreatorImpl

NewCreator creates a new Creator

type FileManager

type FileManager struct {
	// contains filtered or unexported fields
}

FileManager manages resource files stored on s3.

func NewFileManager

func NewFileManager(
	executorID resModel.ExecutorID,
	creator Creator,
) *FileManager

NewFileManager creates a new bucket FileManager.

func NewFileManagerForUT

func NewFileManagerForUT(tempDir string, executorID resModel.ExecutorID) (*FileManager, *mockBucketCreator)

NewFileManagerForUT returns a file manager for UT.

func NewFileManagerForUTFromSharedStorageFactory

func NewFileManagerForUTFromSharedStorageFactory(
	executorID model.ExecutorID, creator *mockBucketCreator,
) *FileManager

NewFileManagerForUTFromSharedStorageFactory returns a file manager for UT.

func NewFileManagerWithConfig

func NewFileManagerWithConfig(
	executorID resModel.ExecutorID, config *resModel.Config,
) *FileManager

NewFileManagerWithConfig returns a new bucket FileManager. Note that the lifetime of the returned object should span the whole lifetime of the executor.

func (*FileManager) CleanOrRecreatePersistedResource

func (m *FileManager) CleanOrRecreatePersistedResource(
	ctx context.Context, ident internal.ResourceIdent,
) (internal.ResourceDescriptor, error)

CleanOrRecreatePersistedResource cleans the bucket directory or recreates placeholder file of the given resource. Note that CleanOrRecreatePersistedResource will work on any executor for any persisted resource.

func (*FileManager) CreateResource

func (m *FileManager) CreateResource(
	ctx context.Context, ident internal.ResourceIdent,
) (internal.ResourceDescriptor, error)

CreateResource creates a new resource on s3. It can only be used to create resources that belong to the current executor.

func (*FileManager) GetPersistedResource

func (m *FileManager) GetPersistedResource(
	ctx context.Context, ident internal.ResourceIdent,
) (internal.ResourceDescriptor, error)

GetPersistedResource returns the descriptor of a resource that has already been marked as persisted. Note that GetPersistedResource will work on any executor for any persisted resource.

func (*FileManager) RemoveResource

func (m *FileManager) RemoveResource(
	ctx context.Context, ident internal.ResourceIdent,
) error

RemoveResource removes a resource from s3. It can be called on any executor node.

func (*FileManager) RemoveTemporaryFiles

func (m *FileManager) RemoveTemporaryFiles(
	ctx context.Context, scope internal.ResourceScope,
) error

RemoveTemporaryFiles removes all temporary resources (those that are not persisted). It can only be used to clean up resources created by the local executor.

func (*FileManager) SetPersisted

func (m *FileManager) SetPersisted(
	ctx context.Context, ident internal.ResourceIdent,
) error

SetPersisted marks a resource as persisted. It can only be called on the creator of the resource.