gcsx

Documentation

Overview

Package gcsx contains utilities for working with Google Cloud Storage (GCS).

Example

package main

import (
	"context"
	"time"

	"cloud.google.com/go/storage"
	"github.com/cd-paliv/beam-fork/sdks/v3/go/pkg/beam/util/gcsx"
)

func main() {
	ctx := context.Background()
	c, err := gcsx.NewClient(ctx, storage.ScopeReadOnly)
	if err != nil {
		// do something
	}

	buckets, object, err := gcsx.ParseObject("gs://some-bucket/some-object")
	if err != nil {
		// do something
	}

	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	bytes, err := gcsx.ReadObject(ctx, c, buckets, object)
	if err != nil {
		// do something
	}

	_ = bytes
}

Index

• func BucketExists(ctx context.Context, client *storage.Client, bucket string) (bool, error)
• func CreateBucket(ctx context.Context, client *storage.Client, project, bucket string) error
• func Join(object string, elms ...string) string
• func MakeObject(bucket, path string) string
• func NewClient(ctx context.Context, scope string) (*storage.Client, error)
• func NewUnauthenticatedClient(ctx context.Context) (*storage.Client, error)
• func ParseObject(object string) (bucket, path string, err error)
• func ReadObject(ctx context.Context, client *storage.Client, bucket, object string) ([]byte, error)
• func Upload(ctx context.Context, client *storage.Client, project, bucket, object string, ...) (string, error)
• func WriteObject(ctx context.Context, client *storage.Client, bucket, object string, ...) error

Examples

• Package

Functions

func BucketExists

func BucketExists(ctx context.Context, client *storage.Client, bucket string) (bool, error)

BucketExists returns true iff the given bucket exists.

func CreateBucket

func CreateBucket(ctx context.Context, client *storage.Client, project, bucket string) error

CreateBucket creates a bucket in GCS.

func Join

func Join(object string, elms ...string) string

Join joins a GCS path with an element. Preserves the gs:// prefix.

func MakeObject

func MakeObject(bucket, path string) string

MakeObject creates a object location from bucket and path. For example, MakeObject("foo", "bar/baz") returns "gs://foo/bar/baz". The bucket must be non-empty.

func NewClient

func NewClient(ctx context.Context, scope string) (*storage.Client, error)

NewClient creates a new GCS client with default application credentials, and supplied OAuth scope. The OAuth scopes are defined in https://pkg.go.dev/cloud.google.com/go/storage#pkg-constants. Sets the user agent to Beam.

func NewUnauthenticatedClient

func NewUnauthenticatedClient(ctx context.Context) (*storage.Client, error)

NewUnauthenticatedClient creates a new GCS client without authentication. Sets the user agent to Beam.

func ParseObject

func ParseObject(object string) (bucket, path string, err error)

ParseObject deconstructs a GCS object name into (bucket, name).

func ReadObject

func ReadObject(ctx context.Context, client *storage.Client, bucket, object string) ([]byte, error)

ReadObject reads the content of the given object in full.

func Upload

func Upload(ctx context.Context, client *storage.Client, project, bucket, object string, r io.Reader) (string, error)

Upload writes the given content to GCS. If the specified bucket does not exist, it is created first. Returns the full path of the object.

func WriteObject

func WriteObject(ctx context.Context, client *storage.Client, bucket, object string, r io.Reader) error

WriteObject writes the given content to the specified object. If the object already exist, it is overwritten.