gcs

Documentation

Index

• func DeleteGCSObject(ctx context.Context, gcsClient *storage.Client, inputURL string) error
• func DownloadGCSObject(ctx context.Context, gcsClient *storage.Client, ...) error
• func DownloadGCSObjectFromURL(ctx context.Context, gcsClient *storage.Client, ...) error
• func DownloadGCSObjectString(ctx context.Context, gcsClient *storage.Client, inputURL string) (string, error)
• func GCSObjectExists(ctx context.Context, gcsClient *storage.Client, gcsBucket, objectName string) (bool, error)
• func GCSObjectExistsFromURL(ctx context.Context, gcsClient *storage.Client, inputURL string) (bool, error)
• func ListGCSBucket(ctx context.Context, gcsClient *storage.Client, gcsBucket, prefix string) ([]string, error)
• func UploadGCSObject(ctx context.Context, gcsClient *storage.Client, ...) error
• func UploadGCSObjectString(ctx context.Context, gcsClient *storage.Client, ...) error
• type GCSBucket
  • func GCSBucketFromPath(client *storage.Client, gcsPath string) (*GCSBucket, error)
  • func NewGCSBucket(client *storage.Client, bucketName, prefix string) *GCSBucket
  • func (bucket *GCSBucket) DeleteDir(ctx context.Context, dirName string) error
  • func (bucket *GCSBucket) DeleteObject(ctx context.Context, objectName string) error
  • func (bucket *GCSBucket) DownloadDir(ctx context.Context, srcDir string, dstDir string) error
  • func (bucket *GCSBucket) DownloadDirParallel(ctx context.Context, srcDir string, dstDir string, workers int) error
  • func (bucket *GCSBucket) DownloadObject(ctx context.Context, objectName string) ([]byte, error)
  • func (bucket *GCSBucket) DownloadObjectToFile(ctx context.Context, objectName string, localPath string) error
  • func (bucket *GCSBucket) DownloadObjects(ctx context.Context, downloads []ObjectDownload) error
  • func (bucket *GCSBucket) DownloadObjectsParallel(ctx context.Context, downloads []ObjectDownload, workers int) error
  • func (bucket *GCSBucket) DownloadTextObject(ctx context.Context, objectName string) (string, error)
  • func (bucket *GCSBucket) Exists(ctx context.Context, objectName string) (bool, error)
  • func (bucket *GCSBucket) ListDescendants(ctx context.Context, dirName string) ([]string, error)
  • func (bucket *GCSBucket) ListDir(ctx context.Context, dirName string) ([]string, error)
  • func (bucket *GCSBucket) Path(objectOrDirName string) string
  • func (bucket *GCSBucket) QueryObjectNames(ctx context.Context, query *storage.Query) ([]string, error)
  • func (bucket *GCSBucket) URI(objectName string) string
  • func (bucket *GCSBucket) UploadDir(ctx context.Context, srcDir string, dstDir string) error
  • func (bucket *GCSBucket) UploadObject(ctx context.Context, reader io.Reader, objectName string) error
  • func (bucket *GCSBucket) UploadObjectFromFile(ctx context.Context, localPath string, objectName string) error
• type ObjectDownload
  • func NewObjectDownload(objectPath, localPath string) ObjectDownload

Functions

func DeleteGCSObject

func DeleteGCSObject(ctx context.Context,
	gcsClient *storage.Client, inputURL string) error

DeleteGCSObject deletes an object at the input URL

func DownloadGCSObject

func DownloadGCSObject(ctx context.Context, gcsClient *storage.Client, gcsBucket, objectName, destinationPath string) error

DownloadGCSObject downloads the object at bucket, objectName and saves it at destinationPath

func DownloadGCSObjectFromURL

func DownloadGCSObjectFromURL(ctx context.Context, gcsClient *storage.Client, inputURL, destinationPath string) error

DownloadGCSObjectFromURL downloads the object at inputURL and saves it at destinationPath

func DownloadGCSObjectString

func DownloadGCSObjectString(ctx context.Context,
	gcsClient *storage.Client, inputURL string) (string, error)

DownloadGCSObjectString downloads the object at inputURL and saves the contents of the object file to a string

func GCSObjectExists

func GCSObjectExists(ctx context.Context, gcsClient *storage.Client, gcsBucket, objectName string) (bool, error)

GCSObjectExists checks if an object with objectName exists at gcsBucket objectName.

func GCSObjectExistsFromURL

func GCSObjectExistsFromURL(ctx context.Context, gcsClient *storage.Client, inputURL string) (bool, error)

GCSObjectExistsFromURL checks if an object exists at inputURL

func ListGCSBucket

func ListGCSBucket(ctx context.Context, gcsClient *storage.Client, gcsBucket, prefix string) ([]string, error)

ListGCSBucket lists all the objectNames in gcsBucket with prefix.

func UploadGCSObject

func UploadGCSObject(ctx context.Context, gcsClient *storage.Client, inputPath, destinationURL string) error

UploadGCSObject uploads an object at inputPath to destination URL

func UploadGCSObjectString

func UploadGCSObjectString(ctx context.Context, gcsClient *storage.Client, inputStr, destinationURL string) error

UploadGCSObjectString uploads an input string as a file to destination URL

Types

type GCSBucket

type GCSBucket struct {
	// contains filtered or unexported fields
}

GCSBucket represents a bucket which can be used for repeated GCS access. If a prefix is provided, all GCSBucket methods will be performed relative to that prefix.

func GCSBucketFromPath

func GCSBucketFromPath(client *storage.Client, gcsPath string) (*GCSBucket, error)

GCSBucketFromPath returns a GCSBucket for a client with the bucket name and prefix extracted from a provided GCS path.

Returns an error if the GCS path parsing fails.

func NewGCSBucket

func NewGCSBucket(client *storage.Client, bucketName, prefix string) *GCSBucket

NewGCSBucket returns a GCSBucket for a client with the given bucket name and prefix.

func (*GCSBucket) DeleteDir

func (bucket *GCSBucket) DeleteDir(ctx context.Context, dirName string) error

DeleteDir deletes all objects in a bucket which are descendents of the provided directory.

func (*GCSBucket) DeleteObject

func (bucket *GCSBucket) DeleteObject(ctx context.Context, objectName string) error

DeleteObject deletes an object from the bucket.

func (*GCSBucket) DownloadDir

func (bucket *GCSBucket) DownloadDir(ctx context.Context, srcDir string, dstDir string) error

DownloadDir downloads all objects from a bucket which are descendents of the provided source directory and saves them to the local destination directory, preserving the directory structure from the bucket.

func (*GCSBucket) DownloadDirParallel

func (bucket *GCSBucket) DownloadDirParallel(ctx context.Context, srcDir string, dstDir string, workers int) error

DownloadDirParallel downloads all objects from a bucket which are descendents of the provided source directory and saves them to the local destination directory, preserving the directory structure from the bucket.

See DownloadObjectsParallel for more info on `workers` and possible errors.

func (*GCSBucket) DownloadObject

func (bucket *GCSBucket) DownloadObject(ctx context.Context, objectName string) ([]byte, error)

DownloadObject downloads an object from the bucket and returns its contents as a byte slice.

func (*GCSBucket) DownloadObjectToFile

func (bucket *GCSBucket) DownloadObjectToFile(ctx context.Context, objectName string, localPath string) error

DownloadObjectToFile downloads an object from the bucket to a local file. Creates the parent directories of the local file if they don't already exist.

func (*GCSBucket) DownloadObjects

func (bucket *GCSBucket) DownloadObjects(ctx context.Context, downloads []ObjectDownload) error

DownloadObjects downloads a list of objects from a bucket.

This will not stop downloading objects if some downloads fail. After all downloads have finished or failed, an error will be returned which summarizes the set of failed downloads.

func (*GCSBucket) DownloadObjectsParallel

func (bucket *GCSBucket) DownloadObjectsParallel(ctx context.Context, downloads []ObjectDownload, workers int) error

DownloadObjectsParallel downloads a list of objects from a bucket in parallel.

If `workers` is negative or 0, this will spawn one worker per object which will be downloaded. Otherwise, `workers` will be the maximum number of workers which will be downloading at one time.

This will not stop downloading objects if some downloads fail. After all downloads have finished or failed, an error will be returned which summarizes the set of failed downloads.

func (*GCSBucket) DownloadTextObject

func (bucket *GCSBucket) DownloadTextObject(ctx context.Context, objectName string) (string, error)

DownloadTextObject downloads an object from the bucket and returns its contents as a string.

func (*GCSBucket) Exists

func (bucket *GCSBucket) Exists(ctx context.Context, objectName string) (bool, error)

Exists returns whether or not there is an object with the given name in the bucket.

func (*GCSBucket) ListDescendants

func (bucket *GCSBucket) ListDescendants(ctx context.Context, dirName string) ([]string, error)

ListDescendants returns the list of all objects in a bucket which have a path beginning with the provided directory name. The object names will be relative to the bucket prefix.

func (*GCSBucket) ListDir

func (bucket *GCSBucket) ListDir(ctx context.Context, dirName string) ([]string, error)

ListDir returns the list of all paths in a bucket which are one directory level below the provided path. The paths will be relative to the bucket prefix.

func (*GCSBucket) Path

func (bucket *GCSBucket) Path(objectOrDirName string) string

Path returns the path of an object or directory relative to the bucket's prefix. For instance, if bucket.prefix is "abc/def", then bucket.Path("ghi/jkl.txt") will return "abc/def/ghi/jkl.txt".

func (*GCSBucket) QueryObjectNames

func (bucket *GCSBucket) QueryObjectNames(ctx context.Context, query *storage.Query) ([]string, error)

QueryObjectNames returns the names of all objects in a bucket matching a query. If a delimiter is provided, the output may include synthetic "directory" entries, which match the query but are not real objects.

func (*GCSBucket) URI

func (bucket *GCSBucket) URI(objectName string) string

URI returns the GCS URI of an object relative to the bucket's prefix. For instance, if bucket.name is "mybucket" and bucket.prefix is 'abc', then bucket.URI('def.txt') will return gs://mybucket/abc/def.txt.

func (*GCSBucket) UploadDir

func (bucket *GCSBucket) UploadDir(ctx context.Context, srcDir string, dstDir string) error

UploadDir uploads the contents of a local source directory to a destination directory in the bucket, preserving the directory structure.

func (*GCSBucket) UploadObject

func (bucket *GCSBucket) UploadObject(ctx context.Context, reader io.Reader, objectName string) error

UploadObject uploads the contents of a reader to an object in the bucket.

func (*GCSBucket) UploadObjectFromFile

func (bucket *GCSBucket) UploadObjectFromFile(ctx context.Context, localPath string, objectName string) error

UploadObjectFromFile uploads the contents of a local file to an object in the bucket.

type ObjectDownload

type ObjectDownload struct {
	// contains filtered or unexported fields
}

An ObjectDownload specifies the remote object path for a download and the local path to which an object should be downloaded.

func NewObjectDownload

func NewObjectDownload(objectPath, localPath string) ObjectDownload

NewObjectDownload returns a new ObjectDownload for the provided remote object path and local path.