block

Documentation

Index

• Constants
• Variables
• func DefaultExample(scheme string) string
• func DefaultValidationRegex(scheme string) string
• func JoinPathParts(parts []string) string
• type Adapter
• type CompleteMultiPartUploadResponse
• type CreateMultiPartUploadOpts
• type CreateMultiPartUploadResponse
• type HashingReader
  • func NewHashingReader(body io.Reader, hashTypes ...int) *HashingReader
  • func (s *HashingReader) Read(p []byte) (int, error)
• type IdentifierType
• type Inventory
• type InventoryGenerator
• type InventoryIterator
• type InventoryObject
• type MultipartPart
• type MultipartUploadCompletion
• type ObjectPointer
• type Path
  • func NewPath(str, entryType string) *Path
  • func (p *Path) BaseName() string
  • func (p *Path) Equals(other *Path) bool
  • func (p *Path) IsRoot() bool
  • func (p *Path) ParentPath() string
  • func (p *Path) Split() []string
  • func (p *Path) String() string
• type PreSignMode
• type Properties
• type PutOpts
• type QualifiedKey
  • func ResolveNamespace(defaultNamespace, key string, identifierType IdentifierType) (QualifiedKey, error)
  • func (qk QualifiedKey) Format() string
• type QualifiedPrefix
  • func ResolveNamespacePrefix(defaultNamespace, prefix string) (QualifiedPrefix, error)
• type StorageNamespaceInfo
  • func DefaultStorageNamespaceInfo(scheme string) StorageNamespaceInfo
• type StorageType
  • func GetStorageType(namespaceURL *url.URL) (StorageType, error)
  • func (s StorageType) BlockstoreType() string
  • func (s StorageType) Scheme() string
• type UploadPartResponse
• type WalkFunc
• type WalkOpts

Constants

const (
	BlockstoreTypeS3        = "s3"
	BlockstoreTypeGS        = "gs"
	BlockstoreTypeAzure     = "azure"
	BlockstoreTypeLocal     = "local"
	BlockstoreTypeMem       = "mem"
	BlockstoreTypeTransient = "transient"
)

const (
	HashFunctionMD5 = iota
	HashFunctionSHA256
)

const (
	StorageTypeMem = iota
	StorageTypeLocal
	StorageTypeS3
	StorageTypeGS
	StorageTypeAzure
)

const (
	Separator = "/"

	EntryTypeTree   = "tree"
	EntryTypeObject = "object"
)

const DefaultPreSignExpiryDuration = time.Minute * 15

DefaultPreSignExpiryDuration is the amount of time pre-signed requests are valid for.

Variables

var (
	ErrDataNotFound          = errors.New("not found")
	ErrOperationNotSupported = errors.New("operation not supported")
)

var ErrInvalidNamespace = errors.New("invalid namespace")

var RootPath = NewPath("", EntryTypeTree)

Functions

func DefaultExample

func DefaultExample(scheme string) string

func DefaultValidationRegex

func DefaultValidationRegex(scheme string) string

func JoinPathParts

func JoinPathParts(parts []string) string

Types

type Adapter

type Adapter interface {
	InventoryGenerator
	Put(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, opts PutOpts) error
	Get(ctx context.Context, obj ObjectPointer, expectedSize int64) (io.ReadCloser, error)
	GetPreSignedURL(ctx context.Context, obj ObjectPointer, mode PreSignMode) (string, error)
	Walk(ctx context.Context, walkOpt WalkOpts, walkFn WalkFunc) error
	Exists(ctx context.Context, obj ObjectPointer) (bool, error)
	GetRange(ctx context.Context, obj ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)
	GetProperties(ctx context.Context, obj ObjectPointer) (Properties, error)
	Remove(ctx context.Context, obj ObjectPointer) error
	Copy(ctx context.Context, sourceObj, destinationObj ObjectPointer) error
	CreateMultiPartUpload(ctx context.Context, obj ObjectPointer, r *http.Request, opts CreateMultiPartUploadOpts) (*CreateMultiPartUploadResponse, error)
	UploadPart(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, uploadID string, partNumber int) (*UploadPartResponse, error)
	UploadCopyPart(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, partNumber int) (*UploadPartResponse, error)
	UploadCopyPartRange(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, partNumber int, startPosition, endPosition int64) (*UploadPartResponse, error)
	AbortMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string) error
	CompleteMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string, multipartList *MultipartUploadCompletion) (*CompleteMultiPartUploadResponse, error)
	BlockstoreType() string
	GetStorageNamespaceInfo() StorageNamespaceInfo
	RuntimeStats() map[string]string
}

type CompleteMultiPartUploadResponse

type CompleteMultiPartUploadResponse struct {
	ETag             string
	ContentLength    int64
	ServerSideHeader http.Header
}

CompleteMultiPartUploadResponse complete multipart etag, content length and additional headers (implementation specific) currently it targets s3. The ETag is a hex string value of the content checksum

type CreateMultiPartUploadOpts

type CreateMultiPartUploadOpts struct {
	StorageClass *string // S3 storage class
}

CreateMultiPartUploadOpts contains optional arguments for CreateMultiPartUpload. These should be analogous to options on some underlying storage layer. Missing arguments are mapped to the default if a storage layer implements the option.

If the same CreateMultiPartUpload command is implemented multiple times with the same contents but different option values, the first supplied option value is retained.

type CreateMultiPartUploadResponse

type CreateMultiPartUploadResponse struct {
	UploadID         string
	ServerSideHeader http.Header
}

CreateMultiPartUploadResponse multipart upload ID and additional headers (implementation specific) currently it targets s3 capabilities to enable encryption properties

type HashingReader

type HashingReader struct {
	Md5    hash.Hash
	Sha256 hash.Hash

	CopiedSize int64
	// contains filtered or unexported fields
}

func NewHashingReader

func NewHashingReader(body io.Reader, hashTypes ...int) *HashingReader

func (*HashingReader) Read

func (s *HashingReader) Read(p []byte) (int, error)

type IdentifierType

type IdentifierType int32

IdentifierType is the type the ObjectPointer Identifier

const (
	// Deprecated: indicates that the identifier might be relative or full.
	IdentifierTypeUnknownDeprecated IdentifierType = 0

	// IdentifierTypeRelative indicates that the address is relative to the storage namespace.
	// For example: "/foo/bar"
	IdentifierTypeRelative IdentifierType = 1

	// IdentifierTypeFull indicates that the address is the full address of the object in the object store.
	// For example: "s3://bucket/foo/bar"
	IdentifierTypeFull IdentifierType = 2
)

type Inventory

type Inventory interface {
	Iterator() InventoryIterator
	SourceName() string
	InventoryURL() string
}

Inventory represents a snapshot of the storage space

type InventoryGenerator

type InventoryGenerator interface {
	GenerateInventory(ctx context.Context, logger logging.Logger, inventoryURL string, shouldSort bool, prefixes []string) (Inventory, error)
}

type InventoryIterator

type InventoryIterator interface {
	cmdutils.ProgressReporter
	Next() bool
	Err() error
	Get() *InventoryObject
}

type InventoryObject

type InventoryObject struct {
	Bucket          string
	Key             string
	Size            int64
	LastModified    *time.Time
	Checksum        string
	PhysicalAddress string
}

type MultipartPart

type MultipartPart struct {
	ETag       string
	PartNumber int
}

MultipartPart single multipart information

type MultipartUploadCompletion

type MultipartUploadCompletion struct {
	Part []MultipartPart
}

MultipartUploadCompletion parts described as part of complete multipart upload. Each part holds the part number and ETag received while calling part upload. NOTE that S3 implementation and our S3 gateway accept and returns ETag value surrounded with double-quotes ("), while the adapter implementations supply the raw value of the etag (without double quotes) and let the gateway manage the s3 protocol specifications.

type ObjectPointer

type ObjectPointer struct {
	StorageNamespace string
	Identifier       string

	// Indicates whether the Identifier is relative to the StorageNamespace,
	// full address to an object, or unknown.
	IdentifierType IdentifierType
}

ObjectPointer is a unique identifier of an object in the object store: the store is a 1:1 mapping between pointers and objects.

type Path

type Path struct {
	// contains filtered or unexported fields
}

func NewPath

func NewPath(str, entryType string) *Path

func (*Path) BaseName

func (p *Path) BaseName() string

func (*Path) Equals

func (p *Path) Equals(other *Path) bool

func (*Path) IsRoot

func (p *Path) IsRoot() bool

func (*Path) ParentPath

func (p *Path) ParentPath() string

func (*Path) Split

func (p *Path) Split() []string

func (*Path) String

func (p *Path) String() string

type PreSignMode

type PreSignMode int32

PreSignMode is the mode to use when generating a pre-signed URL (read/write)

const (
	PreSignModeRead PreSignMode = iota
	PreSignModeWrite
)

type Properties

type Properties struct {
	StorageClass *string
}

Properties of an object stored on the underlying block store. Refer to the actual underlying Adapter for which properties are actually reported.

type PutOpts

type PutOpts struct {
	StorageClass *string // S3 storage class
}

PutOpts contains optional arguments for Put. These should be analogous to options on some underlying storage layer. Missing arguments are mapped to the default if a storage layer implements the option.

If the same Put command is implemented multiple times with the same contents but different option values, the first supplied option value is retained.

type QualifiedKey

type QualifiedKey struct {
	StorageType      StorageType
	StorageNamespace string
	Key              string
}

func ResolveNamespace

func ResolveNamespace(defaultNamespace, key string, identifierType IdentifierType) (QualifiedKey, error)

func (QualifiedKey) Format

func (qk QualifiedKey) Format() string

type QualifiedPrefix

type QualifiedPrefix struct {
	StorageType      StorageType
	StorageNamespace string
	Prefix           string
}

func ResolveNamespacePrefix

func ResolveNamespacePrefix(defaultNamespace, prefix string) (QualifiedPrefix, error)

type StorageNamespaceInfo

type StorageNamespaceInfo struct {
	ValidityRegex  string // regex pattern that could be used to validate the namespace
	Example        string // example of a valid namespace
	PreSignSupport bool
}

func DefaultStorageNamespaceInfo

func DefaultStorageNamespaceInfo(scheme string) StorageNamespaceInfo

type StorageType

type StorageType int

func GetStorageType

func GetStorageType(namespaceURL *url.URL) (StorageType, error)

func (StorageType) BlockstoreType

func (s StorageType) BlockstoreType() string

func (StorageType) Scheme

func (s StorageType) Scheme() string

type UploadPartResponse

type UploadPartResponse struct {
	ETag             string
	ServerSideHeader http.Header
}

UploadPartResponse upload part ETag and additional headers (implementation specific) currently it targets s3 capabilities to enable encryption properties The ETag is a hex string value of the content checksum

type WalkFunc

type WalkFunc func(id string) error

WalkFunc is called for each object visited by the Walk. The id argument contains the argument to Walk as a prefix; that is, if Walk is called with "test/data/", which is a prefix containing the object "test/data/a", the walk function will be called with argument "test/data/a". If there was a problem walking to the object, the incoming error will describe the problem and the function can decide how to handle that error. If an error is returned, processing stops.

type WalkOpts

type WalkOpts struct {
	StorageNamespace string
	Prefix           string
}

WalkOpts is a unique identifier of a prefix in the object store.