gcs

Documentation

Index

• func ConfigPath() string
• func GetFileModificationTime(attrs *storage.ObjectAttrs) time.Time
• func ParseFileModificationTimeMetadata(attrs *storage.ObjectAttrs) time.Time
• type GCS
  • func (g *GCS) AttemptLock(bucket, object string, ttl time.Duration) error
  • func (g *GCS) AttemptUnLock(bucket, object string) error
  • func (g *GCS) Attributes(bucket, prefix string) (*system.Attrs, error)
  • func (g *GCS) BatchAttributes(bucket, prefix string, recursive bool) ([]*system.Attrs, error)
  • func (g *GCS) Cat(bucket, prefix string) ([]byte, error)
  • func (g *GCS) Copy(srcBucket, srcPrefix, dstBucket, dstPrefix string) error
  • func (g *GCS) Delete(bucket, prefix string) error
  • func (g *GCS) DeleteObject(bucket, prefix string) error
  • func (g *GCS) DiskUsage(bucket, prefix string, recursive bool) ([]system.DiskUsage, error)
  • func (g *GCS) DoAttemptLock(bucket, object string, ttl time.Duration) (int64, error)
  • func (g *GCS) DoAttemptUnlock(bucket, object string, generation int64) error
  • func (g *GCS) Download(bucket, prefix, dstFile string, forceChecksum bool, ctx system.RunContext) error
  • func (g *GCS) GCSAttrs(bucket, prefix string) (*storage.ObjectAttrs, error)
  • func (g *GCS) GetObjectReader(bucket, prefix string) (io.ReadCloser, error)
  • func (g *GCS) GetObjectWriter(bucket, prefix string) (io.WriteCloser, error)
  • func (g *GCS) Init(_ ...string) error
  • func (g *GCS) IsDirectory(bucket, prefix string) (bool, error)
  • func (g *GCS) IsObject(bucket, prefix string) (bool, error)
  • func (g *GCS) List(bucket, prefix string, recursive bool) ([]*system.FileObject, error)
  • func (g *GCS) Move(srcBucket, srcPrefix, dstBucket, dstPrefix string) error
  • func (g *GCS) MustEqualCRC32C(flag bool, localPath, bucket, object string) error
  • func (g *GCS) Scheme() string
  • func (g *GCS) Upload(srcFile, bucket, object string, ctx system.RunContext) error

Functions

func ConfigPath

func ConfigPath() string

ConfigPath gets gcp config path from env

func GetFileModificationTime

func GetFileModificationTime(attrs *storage.ObjectAttrs) time.Time

GetFileModificationTime get file modification time

func ParseFileModificationTimeMetadata

func ParseFileModificationTimeMetadata(attrs *storage.ObjectAttrs) time.Time

ParseFileModificationTimeMetadata parsed reserved modification time from metadata

Types

type GCS

type GCS struct {
	// contains filtered or unexported fields
}

func (*GCS) AttemptLock

func (g *GCS) AttemptLock(bucket, object string, ttl time.Duration) error

AttemptLock attempts to write a remote lock file

func (*GCS) AttemptUnLock

func (g *GCS) AttemptUnLock(bucket, object string) error

AttemptUnLock attempts to release a remote lock file

func (*GCS) Attributes

func (g *GCS) Attributes(bucket, prefix string) (*system.Attrs, error)

GetObjectAttributes gets the attributes of an object

func (*GCS) BatchAttributes

func (g *GCS) BatchAttributes(bucket, prefix string, recursive bool) ([]*system.Attrs, error)

GetObjectsAttributes gets the attributes of all the objects under a prefix

func (*GCS) Cat

func (g *GCS) Cat(bucket, prefix string) ([]byte, error)

OutputObject outputs an object

func (*GCS) Copy

func (g *GCS) Copy(srcBucket, srcPrefix, dstBucket, dstPrefix string) error

CopyObject copies an object

func (*GCS) Delete

func (g *GCS) Delete(bucket, prefix string) error

DeleteObject deletes an object

func (*GCS) DeleteObject

func (g *GCS) DeleteObject(bucket, prefix string) error

func (*GCS) DiskUsage

func (g *GCS) DiskUsage(bucket, prefix string, recursive bool) ([]system.DiskUsage, error)

GetDiskUsageObjects gets disk usage of objects under a prefix

func (*GCS) DoAttemptLock

func (g *GCS) DoAttemptLock(bucket, object string, ttl time.Duration) (int64, error)

DoAttemptLock returns generation and potential error

func (*GCS) DoAttemptUnlock

func (g *GCS) DoAttemptUnlock(bucket, object string, generation int64) error

DoAttemptUnlock takes generation as input and returns potential error

func (*GCS) Download

func (g *GCS) Download(
	bucket, prefix, dstFile string,
	forceChecksum bool,
	ctx system.RunContext,
) error

DownloadObjectWithWorkerPool downloads a specific byte range of an object to a file.

func (*GCS) GCSAttrs

func (g *GCS) GCSAttrs(bucket, prefix string) (*storage.ObjectAttrs, error)

func (*GCS) GetObjectReader

func (g *GCS) GetObjectReader(bucket, prefix string) (io.ReadCloser, error)

func (*GCS) GetObjectWriter

func (g *GCS) GetObjectWriter(bucket, prefix string) (io.WriteCloser, error)

func (*GCS) Init

func (g *GCS) Init(_ ...string) error

storageClient gets or creates a gcp storage client

func (*GCS) IsDirectory

func (g *GCS) IsDirectory(bucket, prefix string) (bool, error)

IsDirectory checks if is a directory case 1: gs://abc/def -> gs://abc/def/ : true case 2: gs://abc/de -> gs://abc/def/ : false case 3: gs://abc/def/ -> gs://abc/def/ : true case 4: gs://abc/def -> gs://abc/def : false

func (*GCS) IsObject

func (g *GCS) IsObject(bucket, prefix string) (bool, error)

IsObject checks if is an object case 1: gs://abc/def -> gs://abc/def/ : false case 2: gs://abc/de -> gs://abc/def/ : false case 3: gs://abc/def/ -> gs://abc/def/ : false case 4: gs://abc/def -> gs://abc/def : true

func (*GCS) List

func (g *GCS) List(bucket, prefix string, recursive bool) ([]*system.FileObject, error)

List objects under a prefix

func (*GCS) Move

func (g *GCS) Move(srcBucket, srcPrefix, dstBucket, dstPrefix string) error

MoveObject moves an object

func (*GCS) MustEqualCRC32C

func (g *GCS) MustEqualCRC32C(flag bool, localPath, bucket, object string) error

MustEqualCRC32C compare CRC32C values if flag is set - compare a local file with an object from gcp - exit process if values are different

func (*GCS) Scheme

func (g *GCS) Scheme() string

func (*GCS) Upload

func (g *GCS) Upload(srcFile, bucket, object string, ctx system.RunContext) error

UploadObject uploads an object from a file