storage

package
v0.0.0-...-a74b21b Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 7, 2024 License: Apache-2.0 Imports: 26 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func ConcatGCSPath 

func ConcatGCSPath(pathElements ...string) string

ConcatGCSPath concatenates multiple elements of GCS path into a GCS path.

func GetBucketNameFromGCSPath 

func GetBucketNameFromGCSPath(p string) (string, error)

GetBucketNameFromGCSPath splits GCS path to get bucket name

func GetGCSObjectPathElements 

func GetGCSObjectPathElements(p string) (string, string, error)

GetGCSObjectPathElements returns bucket name, object path within the bucket for a valid object path. Error is returned otherwise.

func SplitGCSPath 

func SplitGCSPath(p string) (string, string, error)

SplitGCSPath splits GCS path into bucket and object path portions

Types

type BucketIterator 

type BucketIterator struct {
	// contains filtered or unexported fields
}

BucketIterator is a wrapper around storage.BucketIterator. Implements BucketIteratorInterface.

func (*BucketIterator) Next 

func (bi *BucketIterator) Next() (*storage.BucketAttrs, error)

Next returns the next result. Its second return value is iterator.Done if there are no more results. Once Next returns iterator.Done, all subsequent calls will return iterator.Done.

type BucketIteratorCreator 

type BucketIteratorCreator struct {
}

BucketIteratorCreator is responsible for creating GCS bucket iterator

func (*BucketIteratorCreator) CreateBucketIterator 

func (bic *BucketIteratorCreator) CreateBucketIterator(ctx context.Context,
	storageClient domain.StorageClientInterface, projectID string) domain.BucketIteratorInterface

CreateBucketIterator creates GCS bucket iterator

type BufferedWriter 

type BufferedWriter struct {
	sync.Mutex
	sync.WaitGroup
	// contains filtered or unexported fields
}

BufferedWriter is responsible for multipart component upload while using a local buffer.

func NewBufferedWriter 

func NewBufferedWriter(ctx context.Context, size, workers int64, client gcsClient, oauth, prefix, bkt, obj, errLogPrefix string) *BufferedWriter

NewBufferedWriter creates a BufferedWriter

func (*BufferedWriter) Close 

func (b *BufferedWriter) Close() error

Close composes the objects and close buffered writer.

func (*BufferedWriter) Write 

func (b *BufferedWriter) Write(d []byte) (int, error)

Write writes the passed in bytes to buffer.

type Client 

type Client struct {
	StorageClient *storage.Client
	Logger        logging.Logger
	Ctx           context.Context
	Oic           domain.ObjectIteratorCreatorInterface
	Soc           domain.StorageObjectCreatorInterface
}

Client implements domain.StorageClientInterface. It implements main Storage functions used by image import features.

func NewStorageClient 

func NewStorageClient(ctx context.Context,
	logger logging.Logger, storageOptions ...option.ClientOption) (*Client, error)

NewStorageClient creates a Client

func (*Client) Buckets 

func (sc *Client) Buckets(projectID string) *storage.BucketIterator

Buckets returns a bucket iterator for all buckets within a project

func (*Client) Close 

func (sc *Client) Close() error

Close closes the Client.

Close need not be called at program exit.

func (*Client) CreateBucket 

func (sc *Client) CreateBucket(
	bucketName string, project string, attrs *storage.BucketAttrs) error

CreateBucket creates a GCS bucket

func (*Client) DeleteGcsPath 

func (sc *Client) DeleteGcsPath(gcsPath string) error

DeleteGcsPath deletes a GCS path, including files

func (*Client) DeleteObject 

func (sc *Client) DeleteObject(gcsPath string) error

DeleteObject deletes the object with the path `gcsPath`.

func (*Client) FindGcsFile 

func (sc *Client) FindGcsFile(gcsDirectoryPath string, fileExtension string) (*storage.ObjectHandle, error)

FindGcsFile finds a file in a GCS directory path for given file extension. File extension can be a file name as well. The lookup is done recursively.

func (*Client) FindGcsFileDepthLimited 

func (sc *Client) FindGcsFileDepthLimited(gcsDirectoryPath string, fileExtension string, lookupDepth int) (*storage.ObjectHandle, error)

FindGcsFileDepthLimited finds a file in a GCS directory path for given file extension up to lookupDepth deep. If lookup should be only for files directly in gcsDirectoryPath, lookupDepth should be set as 0. For recursive lookup with no limitations on depth, lookupDepth should be -1 File extension can be a file name as well.

func (*Client) GetBucket 

func (sc *Client) GetBucket(bucket string) *storage.BucketHandle

GetBucket returns a BucketHandle, which provides operations on the named bucket.

func (*Client) GetBucketAttrs 

func (sc *Client) GetBucketAttrs(bucket string) (*storage.BucketAttrs, error)

GetBucketAttrs returns bucket attributes for given bucket

func (*Client) GetGcsFileContent 

func (sc *Client) GetGcsFileContent(gcsObject *storage.ObjectHandle) ([]byte, error)

GetGcsFileContent returns content of a GCS object as byte array

func (*Client) GetObject 

func (sc *Client) GetObject(bucket string, objectPath string) domain.StorageObject

GetObject returns storage object for the given bucket and path

func (*Client) GetObjectAttrs 

func (sc *Client) GetObjectAttrs(bucket string, objectPath string) (*storage.ObjectAttrs, error)

GetObjectAttrs returns storage object attributes

func (*Client) GetObjects 

func (sc *Client) GetObjects(bucket string, objectPath string) domain.ObjectIteratorInterface

GetObjects returns object iterator for given bucket and path

func (*Client) UpdateBucket 

func (sc *Client) UpdateBucket(
	bucketName string, attrs storage.BucketAttrsToUpdate) error

UpdateBucket updates a GCS bucket

func (*Client) WriteToGCS 

func (sc *Client) WriteToGCS(
	destinationBucketName string, destinationObjectPath string, reader io.Reader) error

WriteToGCS writes content from a reader to destination bucket and path

type HTTPClient 

type HTTPClient struct {
	// contains filtered or unexported fields
}

HTTPClient implements domain.HTTPClientInterface which abstracts HTTP functionality used by image import features.

func (*HTTPClient) Get 

func (hc *HTTPClient) Get(url string) (resp *http.Response, err error)

Get executes HTTP GET request for given URL

type ObjectIterator 

type ObjectIterator struct {
	// contains filtered or unexported fields
}

ObjectIterator is a wrapper around storage.ObjectIterator. Implements ObjectIteratorInterface.

func (*ObjectIterator) Next 

func (bi *ObjectIterator) Next() (*storage.ObjectAttrs, error)

Next returns the next result. Its second return value is iterator.Done if there are no more results. Once Next returns iterator.Done, all subsequent calls will return iterator.Done.

type ObjectIteratorCreator 

type ObjectIteratorCreator struct {
	// contains filtered or unexported fields
}

ObjectIteratorCreator is responsible for creating GCS Object iterator

func (*ObjectIteratorCreator) CreateObjectIterator 

func (bic *ObjectIteratorCreator) CreateObjectIterator(
	bucket string, objectPath string) domain.ObjectIteratorInterface

CreateObjectIterator creates GCS Object iterator

type ResourceLocationRetriever 

type ResourceLocationRetriever struct {
	Mgce              domain.MetadataGCEInterface
	ComputeGCEService daisyCompute.Client
}

ResourceLocationRetriever is responsible for retrieving GCE zone to run import in

func NewResourceLocationRetriever 

func NewResourceLocationRetriever(aMgce domain.MetadataGCEInterface, cs daisyCompute.Client) *ResourceLocationRetriever

NewResourceLocationRetriever creates a ResourceLocationRetriever

func (*ResourceLocationRetriever) GetLargestStorageLocation 

func (rlr *ResourceLocationRetriever) GetLargestStorageLocation(storageLocation string) string

GetLargestStorageLocation returns the largest storage location that includes provided argument. If argument is a multi-region, the argument is returned. If argument is a region within a multi-region, the multi-region is returned. If argument is a region not within a multi-region, argument is returned.

func (*ResourceLocationRetriever) GetZone 

func (rlr *ResourceLocationRetriever) GetZone(storageLocation string, project string) (string, error)

GetZone retrieves GCE zone to run import in based on imported source file location and available zones in the project. If storageRegion is provided and valid, first zone within that region will be used. If no storageRegion is provided, GCE Zone from the running process will be used.

type ScratchBucketCreator 

type ScratchBucketCreator struct {
	StorageClient         domain.StorageClientInterface
	Ctx                   context.Context
	BucketIteratorCreator domain.BucketIteratorCreatorInterface
}

ScratchBucketCreator is responsible for creating Daisy scratch bucketets

func NewScratchBucketCreator 

func NewScratchBucketCreator(ctx context.Context, storageClient domain.StorageClientInterface) *ScratchBucketCreator

NewScratchBucketCreator creates a ScratchBucketCreator

func (*ScratchBucketCreator) CreateScratchBucket 

func (c *ScratchBucketCreator) CreateScratchBucket(sourceFileFlag string, project string,
	fallbackZone string, enableUniformBucketLevelAccess bool) (string, string, error)

CreateScratchBucket creates scratch bucket in the same region as sourceFileFlag. If failed to determine region by source file, we will try to determine region by fallbackZone. Returns (bucket_name, region, error)

func (*ScratchBucketCreator) IsBucketInProject 

func (c *ScratchBucketCreator) IsBucketInProject(project string, bucketName string) bool

IsBucketInProject checks if bucket belongs to a project

type TarGcsExtractor 

type TarGcsExtractor struct {
	// contains filtered or unexported fields
}

TarGcsExtractor is responsible for extracting TAR archives from GCS to GCS

func NewTarGcsExtractor 

func NewTarGcsExtractor(ctx context.Context, sc domain.StorageClientInterface, logger logging.Logger) *TarGcsExtractor

NewTarGcsExtractor creates new TarGcsExtractor

func (*TarGcsExtractor) ExtractTarToGcs 

func (tge *TarGcsExtractor) ExtractTarToGcs(tarGcsPath string, destinationGcsPath string) error

ExtractTarToGcs extracts a tar file in GCS back into GCS directory

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.