Documentation ¶
Overview ¶
Package objcli exposes a unified 'Client' interface for accessing/managing objects stored in the cloud.
Index ¶
- Constants
- Variables
- func ShouldIgnore(query string, include, exclude []*regexp.Regexp) bool
- func TestDownloadJSON(t *testing.T, client Client, key string, data any)
- func TestDownloadRAW(t *testing.T, client Client, key string) []byte
- func TestListObjects(t *testing.T, client Client, prefix string) []*objval.ObjectAttrs
- func TestRequireKeyExists(t *testing.T, client Client, key string)
- func TestRequireKeyNotFound(t *testing.T, client Client, key string)
- func TestUploadJSON(t *testing.T, client Client, key string, body any)
- func TestUploadRAW(t *testing.T, client Client, key string, body []byte)
- type Client
- type IterateFunc
- type MockClient
- func (_m *MockClient) AbortMultipartUpload(ctx context.Context, bucket, id, key string) error
- func (_m *MockClient) AppendToObject(ctx context.Context, bucket, key string, data io.ReadSeeker) error
- func (_m *MockClient) CompleteMultipartUpload(ctx context.Context, bucket, id, key string, parts ...objval.Part) error
- func (_m *MockClient) CreateMultipartUpload(ctx context.Context, bucket, key string) (string, error)
- func (_m *MockClient) DeleteDirectory(ctx context.Context, bucket, prefix string) error
- func (_m *MockClient) DeleteObjects(ctx context.Context, bucket string, keys ...string) error
- func (_m *MockClient) GetObject(ctx context.Context, bucket, key string, br *objval.ByteRange) (*objval.Object, error)
- func (_m *MockClient) GetObjectAttrs(ctx context.Context, bucket, key string) (*objval.ObjectAttrs, error)
- func (_m *MockClient) IterateObjects(ctx context.Context, bucket, prefix, delimiter string, ...) error
- func (_m *MockClient) ListParts(ctx context.Context, bucket, id, key string) ([]objval.Part, error)
- func (_m *MockClient) Provider() objval.Provider
- func (_m *MockClient) PutObject(ctx context.Context, bucket, key string, body io.ReadSeeker) error
- func (_m *MockClient) UploadPart(ctx context.Context, bucket, id, key string, number int, body io.ReadSeeker) (objval.Part, error)
- func (_m *MockClient) UploadPartCopy(ctx context.Context, bucket, id, dst, src string, number int, ...) (objval.Part, error)
- type RateLimitedClient
- func (r *RateLimitedClient) AbortMultipartUpload(ctx context.Context, bucket, id, key string) error
- func (r *RateLimitedClient) AppendToObject(ctx context.Context, bucket, key string, data io.ReadSeeker) error
- func (r *RateLimitedClient) CompleteMultipartUpload(ctx context.Context, bucket, id, key string, parts ...objval.Part) error
- func (r *RateLimitedClient) CreateMultipartUpload(ctx context.Context, bucket, key string) (string, error)
- func (r *RateLimitedClient) DeleteDirectory(ctx context.Context, bucket, prefix string) error
- func (r *RateLimitedClient) DeleteObjects(ctx context.Context, bucket string, keys ...string) error
- func (r *RateLimitedClient) GetObject(ctx context.Context, bucket, key string, br *objval.ByteRange) (*objval.Object, error)
- func (r *RateLimitedClient) GetObjectAttrs(ctx context.Context, bucket, key string) (*objval.ObjectAttrs, error)
- func (r *RateLimitedClient) IterateObjects(ctx context.Context, bucket, prefix, delimiter string, ...) error
- func (r *RateLimitedClient) ListParts(ctx context.Context, bucket, id, key string) ([]objval.Part, error)
- func (r *RateLimitedClient) Provider() objval.Provider
- func (r *RateLimitedClient) PutObject(ctx context.Context, bucket, key string, body io.ReadSeeker) error
- func (r *RateLimitedClient) UploadPart(ctx context.Context, bucket, id, key string, number int, body io.ReadSeeker) (objval.Part, error)
- func (r *RateLimitedClient) UploadPartCopy(ctx context.Context, bucket, id, dst, src string, number int, ...) (objval.Part, error)
- type TestClient
- func (t *TestClient) AbortMultipartUpload(_ context.Context, bucket, id, key string) error
- func (t *TestClient) AppendToObject(_ context.Context, bucket, key string, data io.ReadSeeker) error
- func (t *TestClient) CompleteMultipartUpload(_ context.Context, bucket, id, key string, parts ...objval.Part) error
- func (t *TestClient) CreateMultipartUpload(_ context.Context, _, _ string) (string, error)
- func (t *TestClient) DeleteDirectory(_ context.Context, bucket, prefix string) error
- func (t *TestClient) DeleteObjects(_ context.Context, bucket string, keys ...string) error
- func (t *TestClient) GetObject(_ context.Context, bucket, key string, br *objval.ByteRange) (*objval.Object, error)
- func (t *TestClient) GetObjectAttrs(_ context.Context, bucket, key string) (*objval.ObjectAttrs, error)
- func (t *TestClient) IterateObjects(_ context.Context, bucket, prefix, delimiter string, ...) error
- func (t *TestClient) ListParts(ctx context.Context, bucket, id, key string) ([]objval.Part, error)
- func (t *TestClient) Provider() objval.Provider
- func (t *TestClient) PutObject(_ context.Context, bucket, key string, body io.ReadSeeker) error
- func (t *TestClient) UploadPart(_ context.Context, bucket, id, key string, number int, body io.ReadSeeker) (objval.Part, error)
- func (t *TestClient) UploadPartCopy(_ context.Context, bucket, id, dst, src string, number int, ...) (objval.Part, error)
Constants ¶
const ( // NoUploadID is a readability definition which should be used by client implementations for cloud providers which // do not utilize uploads ids. NoUploadID = "" // NoPartNumber is a readability definition which should be used by client implementations for cloud providers that // do not need numbers to order parts for multipart uploads. NoPartNumber = 0 )
Variables ¶
var ( // ErrIncludeAndExcludeAreMutuallyExclusive is returned if the user attempts to supply both the include and exclude // arguments to 'IterateObjects' at once. ErrIncludeAndExcludeAreMutuallyExclusive = errors.New("include/exclude are mutually exclusive") // ErrExpectedNoUploadID is returned if the user has provided an upload id for a client which doesn't generate or // require upload ids. ErrExpectedNoUploadID = errors.New("received an unexpected upload id, cloud provider doesn't required upload ids") )
Functions ¶
func ShouldIgnore ¶
ShouldIgnore uses the given regular expressions to determine if we should skip listing the provided file.
func TestDownloadJSON ¶
TestDownloadJSON downloads the given object, unmarshaling it into the provided interface.
func TestDownloadRAW ¶
TestDownloadRAW downloads the object as raw data.
func TestListObjects ¶
TestListObjects returns the attributes of all the existing objects.
func TestRequireKeyExists ¶
TestRequireKeyExists asserts that the given key exists.
func TestRequireKeyNotFound ¶
TestRequireKeyNotFound asserts that the given key does not exist.
func TestUploadJSON ¶
TestUploadJSON uploads the given data as JSON.
Types ¶
type Client ¶
type Client interface { // Provider returns the cloud provider this client is interfacing with. // // NOTE: This may be used to change high level behavior which may be cloud provider specific. Provider() objval.Provider // GetObject retrieves an object form the cloud, an optional byte range argument may be supplied which causes only // the requested byte range to be returned. // // NOTE: The returned objects body must be closed to avoid resource leaks. GetObject(ctx context.Context, bucket, key string, br *objval.ByteRange) (*objval.Object, error) // GetObjectAttrs returns general metadata about the object with the given key. GetObjectAttrs(ctx context.Context, bucket, key string) (*objval.ObjectAttrs, error) // PutObject creates an object in the cloud with the given key/options. // // NOTE: The body is required to be a 'ReadSeeker' to support checksum calculation/validation. PutObject(ctx context.Context, bucket, key string, body io.ReadSeeker) error // AppendToObject appends the provided data to the object with the given key, this is a binary concatenation. // // NOTE: If the given object does not already exist, it will be created. AppendToObject(ctx context.Context, bucket, key string, data io.ReadSeeker) error // DeleteObjects deletes all the objects with the given keys ignoring any errors for keys which are not found. // // NOTE: Depending on the underlying client and support from its SDK, this function may batch operations into pages. DeleteObjects(ctx context.Context, bucket string, keys ...string) error // DeleteDirectory deletes all the objects which have the given prefix. // // NOTE: Depending on the underlying client and support from its SDK, this function may batch operations into pages. DeleteDirectory(ctx context.Context, bucket, prefix string) error // IterateObjects iterates through the objects a bucket running the provided iteration function for each object // which matches the given filtering parameters. IterateObjects( ctx context.Context, bucket, prefix, delimiter string, include, exclude []*regexp.Regexp, fn IterateFunc, ) error // CreateMultipartUpload creates a new multipart upload for the given key. // // NOTE: Not all clients directly support multipart uploads, the interface exposed should be used as if they do. The // underlying client will handle any nuances. CreateMultipartUpload(ctx context.Context, bucket, key string) (string, error) // ListParts returns the list of parts staged or uploaded for the given upload id/key pair. // // NOTE: The returned parts will not have their part number populated as this is not stored by all cloud providers. ListParts(ctx context.Context, bucket, id, key string) ([]objval.Part, error) // UploadPart creates/uploads a new part for the multipart upload with the given id. // // NOTE: The part 'number' should be between 1-10,000 and is used for the ordering of parts upon completion. UploadPart(ctx context.Context, bucket, id, key string, number int, body io.ReadSeeker) (objval.Part, error) // UploadPartCopy creates a new part for the multipart upload using an existing object (or part of an existing // object). // // NOTE: Not all cloud providers support providing a byte range. UploadPartCopy( ctx context.Context, bucket, id, dst, src string, number int, br *objval.ByteRange, ) (objval.Part, error) // CompleteMultipartUpload completes the multipart upload with the given id, the given parts should be provided in // the order that they should be constructed. CompleteMultipartUpload(ctx context.Context, bucket, id, key string, parts ...objval.Part) error // AbortMultipartUpload aborts the multipart upload with the given id whilst cleaning up any abandoned parts. AbortMultipartUpload(ctx context.Context, bucket, id, key string) error }
Client is a unified interface for accessing/managing objects stored in the cloud.
type IterateFunc ¶
type IterateFunc func(attrs *objval.ObjectAttrs) error
IterateFunc is the function used when iterating over objects, this function will be called once for each object whose key matches the provided filtering.
type MockClient ¶
MockClient is an autogenerated mock type for the Client type
func NewMockClient ¶
func NewMockClient(t mockConstructorTestingTNewMockClient) *MockClient
NewMockClient creates a new instance of MockClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
func (*MockClient) AbortMultipartUpload ¶
func (_m *MockClient) AbortMultipartUpload(ctx context.Context, bucket, id, key string) error
AbortMultipartUpload provides a mock function with given fields: ctx, bucket, id, key
func (*MockClient) AppendToObject ¶
func (_m *MockClient) AppendToObject(ctx context.Context, bucket, key string, data io.ReadSeeker) error
AppendToObject provides a mock function with given fields: ctx, bucket, key, data
func (*MockClient) CompleteMultipartUpload ¶
func (_m *MockClient) CompleteMultipartUpload(ctx context.Context, bucket, id, key string, parts ...objval.Part) error
CompleteMultipartUpload provides a mock function with given fields: ctx, bucket, id, key, parts
func (*MockClient) CreateMultipartUpload ¶
func (_m *MockClient) CreateMultipartUpload(ctx context.Context, bucket, key string) (string, error)
CreateMultipartUpload provides a mock function with given fields: ctx, bucket, key
func (*MockClient) DeleteDirectory ¶
func (_m *MockClient) DeleteDirectory(ctx context.Context, bucket, prefix string) error
DeleteDirectory provides a mock function with given fields: ctx, bucket, prefix
func (*MockClient) DeleteObjects ¶
DeleteObjects provides a mock function with given fields: ctx, bucket, keys
func (*MockClient) GetObject ¶
func (_m *MockClient) GetObject(ctx context.Context, bucket, key string, br *objval.ByteRange) (*objval.Object, error)
GetObject provides a mock function with given fields: ctx, bucket, key, br
func (*MockClient) GetObjectAttrs ¶
func (_m *MockClient) GetObjectAttrs(ctx context.Context, bucket, key string) (*objval.ObjectAttrs, error)
GetObjectAttrs provides a mock function with given fields: ctx, bucket, key
func (*MockClient) IterateObjects ¶
func (_m *MockClient) IterateObjects(ctx context.Context, bucket, prefix, delimiter string, include, exclude []*regexp.Regexp, fn IterateFunc) error
IterateObjects provides a mock function with given fields: ctx, bucket, prefix, delimiter, include, exclude, fn
func (*MockClient) ListParts ¶
ListParts provides a mock function with given fields: ctx, bucket, id, key
func (*MockClient) Provider ¶
func (_m *MockClient) Provider() objval.Provider
Provider provides a mock function with given fields:
func (*MockClient) PutObject ¶
func (_m *MockClient) PutObject(ctx context.Context, bucket, key string, body io.ReadSeeker) error
PutObject provides a mock function with given fields: ctx, bucket, key, body
func (*MockClient) UploadPart ¶
func (_m *MockClient) UploadPart(ctx context.Context, bucket, id, key string, number int, body io.ReadSeeker) (objval.Part, error)
UploadPart provides a mock function with given fields: ctx, bucket, id, key, number, body
type RateLimitedClient ¶
type RateLimitedClient struct {
// contains filtered or unexported fields
}
RateLimitedClient implements objcli.Client interface mostly by deferring to the underlying Client, but where the methods which involve uploading/downloading objects, the rate limiter is used to control the rate of data transfer.
The rate-limited methods are:
* GetObject
* PutObject
* AppendToObject
* UploadPart
func NewRateLimitedClient ¶
func NewRateLimitedClient(c Client, rl *rate.Limiter) *RateLimitedClient
NewRateLimitedClient returns a RateLimitedClient.
func (*RateLimitedClient) AbortMultipartUpload ¶
func (r *RateLimitedClient) AbortMultipartUpload(ctx context.Context, bucket, id, key string) error
func (*RateLimitedClient) AppendToObject ¶
func (r *RateLimitedClient) AppendToObject(ctx context.Context, bucket, key string, data io.ReadSeeker) error
func (*RateLimitedClient) CompleteMultipartUpload ¶
func (*RateLimitedClient) CreateMultipartUpload ¶
func (*RateLimitedClient) DeleteDirectory ¶
func (r *RateLimitedClient) DeleteDirectory(ctx context.Context, bucket, prefix string) error
func (*RateLimitedClient) DeleteObjects ¶
func (*RateLimitedClient) GetObjectAttrs ¶
func (r *RateLimitedClient) GetObjectAttrs(ctx context.Context, bucket, key string) (*objval.ObjectAttrs, error)
func (*RateLimitedClient) IterateObjects ¶
func (r *RateLimitedClient) IterateObjects(ctx context.Context, bucket, prefix, delimiter string, include, exclude []*regexp.Regexp, fn IterateFunc, ) error
func (*RateLimitedClient) Provider ¶
func (r *RateLimitedClient) Provider() objval.Provider
func (*RateLimitedClient) PutObject ¶
func (r *RateLimitedClient) PutObject(ctx context.Context, bucket, key string, body io.ReadSeeker) error
func (*RateLimitedClient) UploadPart ¶
func (r *RateLimitedClient) UploadPart(ctx context.Context, bucket, id, key string, number int, body io.ReadSeeker, ) (objval.Part, error)
type TestClient ¶
type TestClient struct { // Buckets is the in memory state maintained by the client. Internally, access is guarded by a mutex, however, it's // not safe/recommended to access this attribute whilst a test is running; it should only be used to inspect state // (to perform assertions) once testing is complete. Buckets objval.TestBuckets // contains filtered or unexported fields }
TestClient implementation of the 'Client' interface which stores state in memory, and can be used to avoid having to manually mock a client during unit testing.
func NewTestClient ¶
func NewTestClient(t *testing.T, provider objval.Provider) *TestClient
NewTestClient returns a new test client, which has no buckets/objects.
func (*TestClient) AbortMultipartUpload ¶
func (t *TestClient) AbortMultipartUpload(_ context.Context, bucket, id, key string) error
func (*TestClient) AppendToObject ¶
func (t *TestClient) AppendToObject(_ context.Context, bucket, key string, data io.ReadSeeker) error
func (*TestClient) CompleteMultipartUpload ¶
func (*TestClient) CreateMultipartUpload ¶
func (*TestClient) DeleteDirectory ¶
func (t *TestClient) DeleteDirectory(_ context.Context, bucket, prefix string) error
func (*TestClient) DeleteObjects ¶
func (*TestClient) GetObjectAttrs ¶
func (t *TestClient) GetObjectAttrs(_ context.Context, bucket, key string) (*objval.ObjectAttrs, error)
func (*TestClient) IterateObjects ¶
func (t *TestClient) IterateObjects(_ context.Context, bucket, prefix, delimiter string, include, exclude []*regexp.Regexp, fn IterateFunc, ) error
func (*TestClient) Provider ¶
func (t *TestClient) Provider() objval.Provider
func (*TestClient) PutObject ¶
func (t *TestClient) PutObject(_ context.Context, bucket, key string, body io.ReadSeeker) error
func (*TestClient) UploadPart ¶
func (t *TestClient) UploadPart( _ context.Context, bucket, id, key string, number int, body io.ReadSeeker, ) (objval.Part, error)
Source Files ¶
Directories ¶
Path | Synopsis |
---|---|
Package objaws provides an implementation of 'objstore.Client' for use with AWS S3.
|
Package objaws provides an implementation of 'objstore.Client' for use with AWS S3. |
Package objazure provides an implementation of 'objstore.Client' for use with Azure blob storage.
|
Package objazure provides an implementation of 'objstore.Client' for use with Azure blob storage. |
Package objgcp provides an implementation of 'objstore.Client' for use with GCS.
|
Package objgcp provides an implementation of 'objstore.Client' for use with GCS. |