storageutil

Documentation

Index

• Constants
• func CRC32C(contents []byte) *uint32
• func ConvertMinObjectAndExtendedObjectAttributesToObject(m *gcs.MinObject, e *gcs.ExtendedObjectAttributes) *gcs.Object
• func ConvertMinObjectToObject(m *gcs.MinObject) *gcs.Object
• func ConvertObjToExtendedObjectAttributes(o *gcs.Object) *gcs.ExtendedObjectAttributes
• func ConvertObjToMinObject(o *gcs.Object) *gcs.MinObject
• func CreateEmptyObjects(ctx context.Context, bucket gcs.Bucket, names []string) (err error)
• func CreateHttpClient(storageClientConfig *StorageClientConfig) (httpClient *http.Client, err error)
• func CreateObject(ctx context.Context, bucket gcs.Bucket, name string, contents []byte) (*gcs.Object, error)
• func CreateObjects(ctx context.Context, bucket gcs.Bucket, input map[string][]byte) (err error)
• func DeleteAllObjects(ctx context.Context, bucket gcs.Bucket) error
• func ListAll(ctx context.Context, bucket gcs.Bucket, req *gcs.ListObjectsRequest) (objects []*gcs.Object, runs []string, err error)
• func ListPrefix(ctx context.Context, bucket gcs.Bucket, prefix string, ...) (err error)
• func MD5(contents []byte) *[md5.Size]byte
• func ObjectAttrsToBucketObject(attrs *storage.ObjectAttrs) *gcs.Object
• func ReadObject(ctx context.Context, bucket gcs.Bucket, name string) (contents []byte, err error)
• func SetAttrsInWriter(wc *storage.Writer, req *gcs.CreateObjectRequest) *storage.Writer
• func ShouldRetry(err error) (b bool)
• type StorageClientConfig
  • func GetDefaultStorageClientConfig() (clientConfig StorageClientConfig)

Constants

const CustomEndpoint = "https://localhost:9000"

const CustomTokenUrl = "http://custom-token-url"

const DummyKeyFile = "test/test_creds.json"

Functions

func CRC32C

func CRC32C(contents []byte) *uint32

Return a value appropriate for placing in CreateObjectRequest.CRC32C for the given object contents.

func ConvertMinObjectAndExtendedObjectAttributesToObject

func ConvertMinObjectAndExtendedObjectAttributesToObject(m *gcs.MinObject,
	e *gcs.ExtendedObjectAttributes) *gcs.Object

func ConvertMinObjectToObject

func ConvertMinObjectToObject(m *gcs.MinObject) *gcs.Object

func ConvertObjToExtendedObjectAttributes

func ConvertObjToExtendedObjectAttributes(o *gcs.Object) *gcs.ExtendedObjectAttributes

func ConvertObjToMinObject

func ConvertObjToMinObject(o *gcs.Object) *gcs.MinObject

func CreateEmptyObjects

func CreateEmptyObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	names []string) (err error)

Create empty objects with default attributes for all of the supplied names.

func CreateHttpClient

func CreateHttpClient(storageClientConfig *StorageClientConfig) (httpClient *http.Client, err error)

func CreateObject

func CreateObject(
	ctx context.Context,
	bucket gcs.Bucket,
	name string,
	contents []byte) (*gcs.Object, error)

Create an object with the supplied contents in the given bucket with the given name.

func CreateObjects

func CreateObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	input map[string][]byte) (err error)

Create multiple objects with some parallelism, with contents according to the supplied map from name to contents.

func DeleteAllObjects

func DeleteAllObjects(
	ctx context.Context,
	bucket gcs.Bucket) error

Delete all objects from the supplied bucket. Results are undefined if the bucket is being concurrently updated.

func ListAll

func ListAll(
	ctx context.Context,
	bucket gcs.Bucket,
	req *gcs.ListObjectsRequest) (
	objects []*gcs.Object,
	runs []string,
	err error)

Repeatedly call bucket.ListObjects until there is nothing further to list, returning all objects and collapsed runs encountered.

May modify *req.

func ListPrefix

func ListPrefix(
	ctx context.Context,
	bucket gcs.Bucket,
	prefix string,
	objects chan<- *gcs.Object) (err error)

List objects in the supplied bucket whose name starts with the given prefix. Write them into the supplied channel in an undefined order.

func MD5

func MD5(contents []byte) *[md5.Size]byte

Return a value appropriate for placing in CreateObjectRequest.MD5 for the given object contents.

func ObjectAttrsToBucketObject

func ObjectAttrsToBucketObject(attrs *storage.ObjectAttrs) *gcs.Object

func ReadObject

func ReadObject(
	ctx context.Context,
	bucket gcs.Bucket,
	name string) (contents []byte, err error)

Read the contents of the latest generation of the object with the supplied name.

func SetAttrsInWriter

func SetAttrsInWriter(wc *storage.Writer, req *gcs.CreateObjectRequest) *storage.Writer

SetAttrsInWriter - for setting object-attributes filed in storage.Writer object. These attributes will be assigned to the newly created or old object.

func ShouldRetry

func ShouldRetry(err error) (b bool)

Types

type StorageClientConfig

type StorageClientConfig struct {
	ClientProtocol             mountpkg.ClientProtocol
	MaxConnsPerHost            int
	MaxIdleConnsPerHost        int
	HttpClientTimeout          time.Duration
	MaxRetrySleep              time.Duration
	RetryMultiplier            float64
	UserAgent                  string
	CustomEndpoint             *url.URL
	KeyFile                    string
	TokenUrl                   string
	ReuseTokenFromUrl          bool
	ExperimentalEnableJsonRead bool
}

func GetDefaultStorageClientConfig

func GetDefaultStorageClientConfig() (clientConfig StorageClientConfig)

GetDefaultStorageClientConfig is only for test, making the default endpoint non-nil, so that we can create dummy tokenSource while unit test.