objcli

package
v3.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 4, 2024 License: Apache-2.0 Imports: 22 Imported by: 0

Documentation

Overview

Package objcli exposes a unified 'Client' interface for accessing/managing objects stored in the cloud.

Index

Constants

View Source 
const (
	// NoUploadID is a readability definition which should be used by client implementations for cloud providers which
	// do not utilize uploads ids.
	NoUploadID = ""

	// NoPartNumber is a readability definition which should be used by client implementations for cloud providers that
	// do not need numbers to order parts for multipart uploads.
	NoPartNumber = 0
)

Variables

View Source 
var (
	// ErrIncludeAndExcludeAreMutuallyExclusive is returned if the user attempts to supply both the include and exclude
	// arguments to 'IterateObjects' at once.
	ErrIncludeAndExcludeAreMutuallyExclusive = errors.New("include/exclude are mutually exclusive")

	// ErrExpectedNoUploadID is returned if the user has provided an upload id for a client which doesn't generate or
	// require upload ids.
	ErrExpectedNoUploadID = errors.New("received an unexpected upload id, cloud provider doesn't required upload ids")
)

Functions

func ShouldIgnore 

func ShouldIgnore(query string, include, exclude []*regexp.Regexp) bool

ShouldIgnore uses the given regular expressions to determine if we should skip listing the provided file.

func TestDownloadJSON 

func TestDownloadJSON(t *testing.T, client Client, key string, data any)

TestDownloadJSON downloads the given object, unmarshaling it into the provided interface.

func TestDownloadRAW 

func TestDownloadRAW(t *testing.T, client Client, key string) []byte

TestDownloadRAW downloads the object as raw data.

func TestListObjects 

func TestListObjects(t *testing.T, client Client, prefix string) []*objval.ObjectAttrs

TestListObjects returns the attributes of all the existing objects.

func TestRequireKeyExists 

func TestRequireKeyExists(t *testing.T, client Client, key string)

TestRequireKeyExists asserts that the given key exists.

func TestRequireKeyNotFound 

func TestRequireKeyNotFound(t *testing.T, client Client, key string)

TestRequireKeyNotFound asserts that the given key does not exist.

func TestUploadJSON 

func TestUploadJSON(t *testing.T, client Client, key string, body any)

TestUploadJSON uploads the given data as JSON.

func TestUploadRAW 

func TestUploadRAW(t *testing.T, client Client, key string, body []byte)

TestUploadRAW uploads the given raw data.

Types

type AbortMultipartUploadOptions 

type AbortMultipartUploadOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// UploadID is the id of the upload being operated on.
	UploadID string

	// Key is the key (path) of the object/blob being operated on.
	Key string
}

AbortMultipartUploadOptions encapsulates the options available when using the 'AbortMultipartUpload' function.

type AppendToObjectOptions 

type AppendToObjectOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Key is the key (path) of the object/blob being operated on.
	Key string

	// Body is the data that will be appended.
	Body io.ReadSeeker
}

AppendToObjectOptions encapsulates the options available when using the 'AppendToObject' function.

type Client 

type Client interface {
	// Provider returns the cloud provider this client is interfacing with.
	//
	// NOTE: This may be used to change high level behavior which may be cloud provider specific.
	Provider() objval.Provider

	// GetObject retrieves an object form the cloud, an optional byte range argument may be supplied which causes only
	// the requested byte range to be returned.
	//
	// NOTE: The returned objects body must be closed to avoid resource leaks.
	GetObject(ctx context.Context, opts GetObjectOptions) (*objval.Object, error)

	// GetObjectAttrs returns general metadata about the object with the given key.
	GetObjectAttrs(ctx context.Context, opts GetObjectAttrsOptions) (*objval.ObjectAttrs, error)

	// PutObject creates an object in the cloud with the given key/options.
	PutObject(ctx context.Context, opts PutObjectOptions) error

	// CopyObject copies an object from one location to another, this may be within the same bucket.
	//
	// NOTE: Each cloud provider has limitations on the max size for copied objects therefore using this function
	// directly is not recommend; see 'objutil.CopyObject' which handles these nuances.
	CopyObject(ctx context.Context, opts CopyObjectOptions) error

	// AppendToObject appends the provided data to the object with the given key, this is a binary concatenation.
	//
	// NOTE: If the given object does not already exist, it will be created.
	AppendToObject(ctx context.Context, opts AppendToObjectOptions) error

	// DeleteObjects deletes all the objects with the given keys ignoring any errors for keys which are not found.
	//
	// NOTE: Depending on the underlying client and support from its SDK, this function may batch operations into pages.
	DeleteObjects(ctx context.Context, opts DeleteObjectsOptions) error

	// DeleteDirectory deletes all the objects which have the given prefix.
	//
	// NOTE: Depending on the underlying client and support from its SDK, this function may batch operations into pages.
	DeleteDirectory(ctx context.Context, opts DeleteDirectoryOptions) error

	// IterateObjects iterates through the objects a bucket running the provided iteration function for each object
	// which matches the given filtering parameters.
	IterateObjects(ctx context.Context, opts IterateObjectsOptions) error

	// CreateMultipartUpload creates a new multipart upload for the given key.
	//
	// NOTE: Not all clients directly support multipart uploads, the interface exposed should be used as if they do. The
	// underlying client will handle any nuances.
	CreateMultipartUpload(ctx context.Context, opts CreateMultipartUploadOptions) (string, error)

	// ListParts returns the list of parts staged or uploaded for the given upload id/key pair.
	//
	// NOTE: The returned parts will not have their part number populated as this is not stored by all cloud providers.
	ListParts(ctx context.Context, opts ListPartsOptions) ([]objval.Part, error)

	// UploadPart creates/uploads a new part for the multipart upload with the given id.
	UploadPart(ctx context.Context, opts UploadPartOptions) (objval.Part, error)

	// UploadPartCopy creates a new part for the multipart upload using an existing object (or part of an existing
	// object).
	UploadPartCopy(ctx context.Context, opts UploadPartCopyOptions) (objval.Part, error)

	// CompleteMultipartUpload completes the multipart upload with the given id, the given parts should be provided in
	// the order that they should be constructed.
	CompleteMultipartUpload(ctx context.Context, opts CompleteMultipartUploadOptions) error

	// AbortMultipartUpload aborts the multipart upload with the given id whilst cleaning up any abandoned parts.
	AbortMultipartUpload(ctx context.Context, opts AbortMultipartUploadOptions) error

	// Close the underlying client/SDK where applicable; use of the client, or the underlying SDK after a call to Close
	// has undefined behavior. This is required to stop memory leaks in GCP.
	Close() error
}

Client is a unified interface for accessing/managing objects stored in the cloud.

type CompleteMultipartUploadOptions 

type CompleteMultipartUploadOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// UploadID is the id of the upload being operated on.
	UploadID string

	// Key is the key (path) of the object/blob being operated on.
	Key string

	// Parts is an ordered list of parts that should be constructed into the completed object.
	Parts []objval.Part
}

CompleteMultipartUploadOptions encapsulates the options available when using the 'CompleteMultipartUpload' function.

type CopyObjectOptions 

type CopyObjectOptions struct {
	// DestinationBucket is the bucket the will be copied into.
	DestinationBucket string

	// DestinationKey is the key for the copied object.
	DestinationKey string

	// SourceBucket is the bucket containing the object being copied.
	SourceBucket string

	// SourceKey is the key of the object being copied.
	SourceKey string
}

CopyObjectOptions encapsulates the options available when using the 'CopyObject' function.

type CreateMultipartUploadOptions 

type CreateMultipartUploadOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Key is the key (path) of the object/blob being operated on.
	Key string
}

CreateMultipartUploadOptions encapsulates the options available when using the 'CreateMultipartUpload' function.

type DeleteDirectoryOptions 

type DeleteDirectoryOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Prefix is the prefix that will be operated on.
	Prefix string
}

DeleteDirectoryOptions encapsulates the options available when using the 'DeleteDirectory' function.

type DeleteObjectsOptions 

type DeleteObjectsOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Keys are the keys that will be deleted.
	Keys []string
}

DeleteObjectsOptions encapsulates the options available when using the 'DeleteObjects' function.

type GetObjectAttrsOptions 

type GetObjectAttrsOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Key is the key (path) of the object/blob being operated on.
	Key string
}

GetObjectAttrsOptions encapsulates the options available when using the 'GetObjectAttrs' function.

type GetObjectOptions 

type GetObjectOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Key is the key (path) of the object/blob being operated on.
	Key string

	// ByteRange allows specifying a start/end offset to be operated on.
	ByteRange *objval.ByteRange
}

GetObjectOptions encapsulates the options available when using the 'GetObject' function.

type IterateFunc 

type IterateFunc func(attrs *objval.ObjectAttrs) error

IterateFunc is the function used when iterating over objects, this function will be called once for each object whose key matches the provided filtering.

type IterateObjectsOptions 

type IterateObjectsOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Prefix is the prefix that will listed.
	Prefix string

	// Delimiter use to group keys e.g. '/' causes listing to only occur within a "directory".
	Delimiter string

	// Include objects where the keys match any of the given regular expressions.
	Include []*regexp.Regexp

	// Exclude objects where the keys match any of the given regular expressions.
	Exclude []*regexp.Regexp

	// Func is executed for each object listed.
	Func IterateFunc
}

IterateObjectsOptions encapsulates the options available when using the 'IterateObjects' function.

type ListPartsOptions 

type ListPartsOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// UploadID - string
	UploadID string

	// Key is the key (path) of the object/blob being operated on.
	Key string
}

ListPartsOptions encapsulates the options available when using the 'ListParts' function.

type MockClient 

type MockClient struct {
	mock.Mock
}

MockClient is an autogenerated mock type for the Client type

func NewMockClient 

func NewMockClient(t interface {
	mock.TestingT
	Cleanup(func())
},
) *MockClient

NewMockClient creates a new instance of MockClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. The first argument is typically a *testing.T value.

func (*MockClient) AbortMultipartUpload 

func (_m *MockClient) AbortMultipartUpload(ctx context.Context, opts AbortMultipartUploadOptions) error

AbortMultipartUpload provides a mock function with given fields: ctx, opts

func (*MockClient) AppendToObject 

func (_m *MockClient) AppendToObject(ctx context.Context, opts AppendToObjectOptions) error

AppendToObject provides a mock function with given fields: ctx, opts

func (*MockClient) Close 

func (_m *MockClient) Close() error

Close provides a mock function with given fields:

func (*MockClient) CompleteMultipartUpload 

func (_m *MockClient) CompleteMultipartUpload(ctx context.Context, opts CompleteMultipartUploadOptions) error

CompleteMultipartUpload provides a mock function with given fields: ctx, opts

func (*MockClient) CopyObject 

func (_m *MockClient) CopyObject(ctx context.Context, opts CopyObjectOptions) error

CopyObject provides a mock function with given fields: ctx, opts

func (*MockClient) CreateMultipartUpload 

func (_m *MockClient) CreateMultipartUpload(ctx context.Context, opts CreateMultipartUploadOptions) (string, error)

CreateMultipartUpload provides a mock function with given fields: ctx, opts

func (*MockClient) DeleteDirectory 

func (_m *MockClient) DeleteDirectory(ctx context.Context, opts DeleteDirectoryOptions) error

DeleteDirectory provides a mock function with given fields: ctx, opts

func (*MockClient) DeleteObjects 

func (_m *MockClient) DeleteObjects(ctx context.Context, opts DeleteObjectsOptions) error

DeleteObjects provides a mock function with given fields: ctx, opts

func (*MockClient) GetObject 

func (_m *MockClient) GetObject(ctx context.Context, opts GetObjectOptions) (*objval.Object, error)

GetObject provides a mock function with given fields: ctx, opts

func (*MockClient) GetObjectAttrs 

func (_m *MockClient) GetObjectAttrs(ctx context.Context, opts GetObjectAttrsOptions) (*objval.ObjectAttrs, error)

GetObjectAttrs provides a mock function with given fields: ctx, opts

func (*MockClient) IterateObjects 

func (_m *MockClient) IterateObjects(ctx context.Context, opts IterateObjectsOptions) error

IterateObjects provides a mock function with given fields: ctx, opts

func (*MockClient) ListParts 

func (_m *MockClient) ListParts(ctx context.Context, opts ListPartsOptions) ([]objval.Part, error)

ListParts provides a mock function with given fields: ctx, opts

func (*MockClient) Provider 

func (_m *MockClient) Provider() objval.Provider

Provider provides a mock function with given fields:

func (*MockClient) PutObject 

func (_m *MockClient) PutObject(ctx context.Context, opts PutObjectOptions) error

PutObject provides a mock function with given fields: ctx, opts

func (*MockClient) UploadPart 

func (_m *MockClient) UploadPart(ctx context.Context, opts UploadPartOptions) (objval.Part, error)

UploadPart provides a mock function with given fields: ctx, opts

func (*MockClient) UploadPartCopy 

func (_m *MockClient) UploadPartCopy(ctx context.Context, opts UploadPartCopyOptions) (objval.Part, error)

UploadPartCopy provides a mock function with given fields: ctx, opts

type PutObjectOptions 

type PutObjectOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// Key is the key (path) of the object/blob being operated on.
	Key string

	// Body is the data that will be uploaded.
	//
	// NOTE: Required to be a 'ReadSeeker' to support checksum calculation/validation.
	Body io.ReadSeeker
}

PutObjectOptions encapsulates the options available when using the 'PutObject' function.

type RateLimitedClient 

type RateLimitedClient struct {
	// contains filtered or unexported fields
}

RateLimitedClient implements objcli.Client interface mostly by deferring to the underlying Client, but where the methods which involve uploading/downloading objects, the rate limiter is used to control the rate of data transfer.

The rate-limited methods are:

- GetObject - PutObject - AppendToObject - UploadPart

func NewRateLimitedClient 

func NewRateLimitedClient(c Client, rl *rate.Limiter) *RateLimitedClient

NewRateLimitedClient returns a RateLimitedClient.

func (*RateLimitedClient) AbortMultipartUpload 

func (r *RateLimitedClient) AbortMultipartUpload(ctx context.Context, opts AbortMultipartUploadOptions) error

func (*RateLimitedClient) AppendToObject 

func (r *RateLimitedClient) AppendToObject(ctx context.Context, opts AppendToObjectOptions) error

func (*RateLimitedClient) CompleteMultipartUpload 

func (r *RateLimitedClient) CompleteMultipartUpload(ctx context.Context, opts CompleteMultipartUploadOptions) error

func (*RateLimitedClient) CreateMultipartUpload 

func (r *RateLimitedClient) CreateMultipartUpload(
	ctx context.Context,
	opts CreateMultipartUploadOptions,
) (string, error)

func (*RateLimitedClient) DeleteDirectory 

func (r *RateLimitedClient) DeleteDirectory(ctx context.Context, opts DeleteDirectoryOptions) error

func (*RateLimitedClient) DeleteObjects 

func (r *RateLimitedClient) DeleteObjects(ctx context.Context, opts DeleteObjectsOptions) error

func (*RateLimitedClient) GetObject 

func (r *RateLimitedClient) GetObject(ctx context.Context, opts GetObjectOptions) (*objval.Object, error)

func (*RateLimitedClient) GetObjectAttrs 

func (r *RateLimitedClient) GetObjectAttrs(
	ctx context.Context,
	opts GetObjectAttrsOptions,
) (*objval.ObjectAttrs, error)

func (*RateLimitedClient) IterateObjects 

func (r *RateLimitedClient) IterateObjects(ctx context.Context, opts IterateObjectsOptions) error

func (*RateLimitedClient) ListParts 

func (r *RateLimitedClient) ListParts(ctx context.Context, opts ListPartsOptions) ([]objval.Part, error)

func (*RateLimitedClient) Provider 

func (r *RateLimitedClient) Provider() objval.Provider

func (*RateLimitedClient) PutObject 

func (r *RateLimitedClient) PutObject(ctx context.Context, opts PutObjectOptions) error

func (*RateLimitedClient) UploadPart 

func (r *RateLimitedClient) UploadPart(ctx context.Context, opts UploadPartOptions) (objval.Part, error)

func (*RateLimitedClient) UploadPartCopy 

func (r *RateLimitedClient) UploadPartCopy(ctx context.Context, opts UploadPartCopyOptions) (objval.Part, error)

type TestClient 

type TestClient struct {

	// Buckets is the in memory state maintained by the client. Internally, access is guarded by a mutex, however, it's
	// not safe/recommended to access this attribute whilst a test is running; it should only be used to inspect state
	// (to perform assertions) once testing is complete.
	Buckets objval.TestBuckets
	// contains filtered or unexported fields
}

TestClient implementation of the 'Client' interface which stores state in memory, and can be used to avoid having to manually mock a client during unit testing.

func NewTestClient 

func NewTestClient(t *testing.T, provider objval.Provider) *TestClient

NewTestClient returns a new test client, which has no buckets/objects.

func (*TestClient) AbortMultipartUpload 

func (t *TestClient) AbortMultipartUpload(_ context.Context, opts AbortMultipartUploadOptions) error

func (*TestClient) AppendToObject 

func (t *TestClient) AppendToObject(_ context.Context, opts AppendToObjectOptions) error

func (*TestClient) Close 

func (t *TestClient) Close() error

func (*TestClient) CompleteMultipartUpload 

func (t *TestClient) CompleteMultipartUpload(_ context.Context, opts CompleteMultipartUploadOptions) error

func (*TestClient) CopyObject 

func (t *TestClient) CopyObject(_ context.Context, opts CopyObjectOptions) error

func (*TestClient) CreateMultipartUpload 

func (t *TestClient) CreateMultipartUpload(_ context.Context, _ CreateMultipartUploadOptions) (string, error)

func (*TestClient) DeleteDirectory 

func (t *TestClient) DeleteDirectory(_ context.Context, opts DeleteDirectoryOptions) error

func (*TestClient) DeleteObjects 

func (t *TestClient) DeleteObjects(_ context.Context, opts DeleteObjectsOptions) error

func (*TestClient) GetObject 

func (t *TestClient) GetObject(_ context.Context, opts GetObjectOptions) (*objval.Object, error)

func (*TestClient) GetObjectAttrs 

func (t *TestClient) GetObjectAttrs(_ context.Context, opts GetObjectAttrsOptions) (*objval.ObjectAttrs, error)

func (*TestClient) IterateObjects 

func (t *TestClient) IterateObjects(_ context.Context, opts IterateObjectsOptions) error

func (*TestClient) ListParts 

func (t *TestClient) ListParts(ctx context.Context, opts ListPartsOptions) ([]objval.Part, error)

func (*TestClient) Provider 

func (t *TestClient) Provider() objval.Provider

func (*TestClient) PutObject 

func (t *TestClient) PutObject(_ context.Context, opts PutObjectOptions) error

func (*TestClient) UploadPart 

func (t *TestClient) UploadPart(_ context.Context, opts UploadPartOptions) (objval.Part, error)

func (*TestClient) UploadPartCopy 

func (t *TestClient) UploadPartCopy(_ context.Context, opts UploadPartCopyOptions) (objval.Part, error)

type UploadPartCopyOptions 

type UploadPartCopyOptions struct {
	// DestinationBucket is the bucket the will be copied into.
	DestinationBucket string

	// UploadID is the id of the upload being operated on.
	UploadID string

	// DestinationKey is the key for the copied object.
	DestinationKey string

	// SourceBucket is the bucket containing the object being copied.
	SourceBucket string

	// SourceKey is the key of the object being copied.
	SourceKey string

	// Number is the number that will be assigned to the part.
	//
	// NOTE: Should be between 1-10,000 and is used for the ordering of parts upon completion.
	Number int

	// ByteRange allows specifying a start/end offset to be operated on.
	//
	// NOTE: Not supported by all cloud providers.
	ByteRange *objval.ByteRange
}

UploadPartCopyOptions encapsulates the options available when using the 'UploadPartCopy' function.

type UploadPartOptions 

type UploadPartOptions struct {
	// Bucket is the bucket being operated on.
	Bucket string

	// UploadID - string
	UploadID string

	// Key is the key (path) of the object/blob being operated on.
	Key string

	// Number is the number that will be assigned to the part.
	//
	// NOTE: Should be between 1-10,000 and is used for the ordering of parts upon completion.
	Number int

	// Body is the data that will be uploaded.
	Body io.ReadSeeker
}

UploadPartOptions encapsulates the options available when using the 'UploadPart' function.

Source Files

View all Source files

Directories

Path Synopsis
Package objaws provides an implementation of 'objstore.Client' for use with AWS S3.
 Package objaws provides an implementation of 'objstore.Client' for use with AWS S3.
Package objazure provides an implementation of 'objstore.Client' for use with Azure blob storage.
 Package objazure provides an implementation of 'objstore.Client' for use with Azure blob storage.
Package objgcp provides an implementation of 'objstore.Client' for use with GCS.
 Package objgcp provides an implementation of 'objstore.Client' for use with GCS.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.