block

package
v1.36.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 30, 2024 License: Apache-2.0 Imports: 14 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	BlockstoreTypeS3        = "s3"
	BlockstoreTypeGS        = "gs"
	BlockstoreTypeAzure     = "azure"
	BlockstoreTypeLocal     = "local"
	BlockstoreTypeMem       = "mem"
	BlockstoreTypeTransient = "transient"
)
View Source 
const (
	StorageTypeMem = iota
	StorageTypeLocal
	StorageTypeS3
	StorageTypeGS
	StorageTypeAzure
)
View Source 
const (
	Separator = "/"

	EntryTypeTree   = "tree"
	EntryTypeObject = "object"
)
View Source 
const DefaultPreSignExpiryDuration = 15 * time.Minute

DefaultPreSignExpiryDuration is the amount of time pre-signed requests are valid for.

Variables

View Source 
var (
	ErrDataNotFound          = errors.New("not found")
	ErrOperationNotSupported = errors.New("operation not supported")
	ErrAsyncCopyFailed       = errors.New("async copy failed")
	ErrBadIndex              = errors.New("bad index")
	ErrForbidden             = errors.New("forbidden")
	ErrInvalidAddress        = errors.New("invalid address")
	ErrInvalidNamespace      = errors.New("invalid namespace")
)
View Source 
var RootPath = NewPath("", EntryTypeTree)

Functions

func DefaultExample 

func DefaultExample(scheme string) string

func DefaultValidationRegex 

func DefaultValidationRegex(scheme string) string

func JoinPathParts 

func JoinPathParts(parts []string) string

func ValidateInterRegionStorage added in v1.26.0 

func ValidateInterRegionStorage(ctx context.Context, adapter Adapter, storageNamespace string) error

func ValidateStorageType added in v0.97.0 

func ValidateStorageType(uri *url.URL, expectedStorage StorageType) error

Types

type Adapter 

type Adapter interface {
	Put(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, opts PutOpts) error
	Get(ctx context.Context, obj ObjectPointer) (io.ReadCloser, error)

	// GetWalker is never called on the server side.
	// TODO(itaiad200): Remove it from this interface.
	GetWalker(uri *url.URL) (Walker, error)

	// GetPreSignedURL returns a pre-signed URL for accessing obj with mode, and the
	// expiry time for this URL.  The expiry time IsZero() if reporting
	// expiry is not supported.  The expiry time will be sooner than
	// Config.*.PreSignedExpiry if an auth token is about to expire.
	GetPreSignedURL(ctx context.Context, obj ObjectPointer, mode PreSignMode) (string, time.Time, error)
	GetPresignUploadPartURL(ctx context.Context, obj ObjectPointer, uploadID string, partNumber int) (string, error)

	Exists(ctx context.Context, obj ObjectPointer) (bool, error)
	GetRange(ctx context.Context, obj ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)
	GetProperties(ctx context.Context, obj ObjectPointer) (Properties, error)
	Remove(ctx context.Context, obj ObjectPointer) error
	Copy(ctx context.Context, sourceObj, destinationObj ObjectPointer) error

	CreateMultiPartUpload(ctx context.Context, obj ObjectPointer, r *http.Request, opts CreateMultiPartUploadOpts) (*CreateMultiPartUploadResponse, error)
	UploadPart(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, uploadID string, partNumber int) (*UploadPartResponse, error)
	ListParts(ctx context.Context, obj ObjectPointer, uploadID string, opts ListPartsOpts) (*ListPartsResponse, error)
	UploadCopyPart(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, partNumber int) (*UploadPartResponse, error)
	UploadCopyPartRange(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, partNumber int, startPosition, endPosition int64) (*UploadPartResponse, error)
	AbortMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string) error
	CompleteMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string, multipartList *MultipartUploadCompletion) (*CompleteMultiPartUploadResponse, error)

	BlockstoreType() string
	BlockstoreMetadata(ctx context.Context) (*BlockstoreMetadata, error)
	GetStorageNamespaceInfo() StorageNamespaceInfo
	ResolveNamespace(storageNamespace, key string, identifierType IdentifierType) (QualifiedKey, error)

	GetRegion(ctx context.Context, storageNamespace string) (string, error)

	RuntimeStats() map[string]string
}

Adapter abstract Storage Adapter for persistence of version controlled data. The methods generally map to S3 API methods - Generally some type of Object Storage - Can also be block storage or even in-memory

func NewMetricsAdapter added in v1.28.1 

func NewMetricsAdapter(adapter Adapter) Adapter

type BlockstoreMetadata added in v1.26.0 

type BlockstoreMetadata struct {
	Region *string
}

type CommonQualifiedKey added in v0.97.0 

type CommonQualifiedKey struct {
	StorageType      StorageType
	StorageNamespace string
	Key              string
}

func DefaultResolveNamespace added in v0.97.0 

func DefaultResolveNamespace(defaultNamespace, key string, identifierType IdentifierType) (CommonQualifiedKey, error)

func (CommonQualifiedKey) Format added in v0.97.0 

func (qk CommonQualifiedKey) Format() string

func (CommonQualifiedKey) GetKey added in v0.97.0 

func (qk CommonQualifiedKey) GetKey() string

func (CommonQualifiedKey) GetStorageNamespace added in v0.97.0 

func (qk CommonQualifiedKey) GetStorageNamespace() string

func (CommonQualifiedKey) GetStorageType added in v0.97.0 

func (qk CommonQualifiedKey) GetStorageType() StorageType

type CompleteMultiPartUploadResponse added in v0.54.0 

type CompleteMultiPartUploadResponse struct {
	ETag             string
	ContentLength    int64
	ServerSideHeader http.Header
}

CompleteMultiPartUploadResponse complete multipart etag, content length and additional headers (implementation specific) currently it targets s3. The ETag is a hex string value of the content checksum

type CreateMultiPartUploadOpts 

type CreateMultiPartUploadOpts struct {
	StorageClass *string // S3 storage class
}

CreateMultiPartUploadOpts contains optional arguments for CreateMultiPartUpload. These should be analogous to options on some underlying storage layer. Missing arguments are mapped to the default if a storage layer implements the option.

If the same CreateMultiPartUpload command is implemented multiple times with the same contents but different option values, the first supplied option value is retained.

type CreateMultiPartUploadResponse added in v0.54.0 

type CreateMultiPartUploadResponse struct {
	UploadID         string
	ServerSideHeader http.Header
}

CreateMultiPartUploadResponse multipart upload ID and additional headers (implementation specific) currently it targets s3 capabilities to enable encryption properties

type HashFunction added in v1.3.1 

type HashFunction int
const (
	HashFunctionMD5 HashFunction = iota
	HashFunctionSHA256
)

type HashingReader 

type HashingReader struct {
	Md5    hash.Hash
	Sha256 hash.Hash

	CopiedSize int64
	// contains filtered or unexported fields
}

func NewHashingReader 

func NewHashingReader(body io.Reader, hashTypes ...HashFunction) *HashingReader

func (*HashingReader) Read 

func (s *HashingReader) Read(p []byte) (int, error)

type IdentifierType added in v0.40.0 

type IdentifierType int32

IdentifierType is the type the ObjectPointer Identifier 

const (
	// Deprecated: indicates that the identifier might be relative or full.
	IdentifierTypeUnknownDeprecated IdentifierType = 0

	// IdentifierTypeRelative indicates that the address is relative to the storage namespace.
	// For example: "/foo/bar"
	IdentifierTypeRelative IdentifierType = 1

	// IdentifierTypeFull indicates that the address is the full address of the object in the object store.
	// For example: "s3://bucket/foo/bar"
	IdentifierTypeFull IdentifierType = 2
)

type ListPartsOpts added in v1.19.0 

type ListPartsOpts struct {
	MaxParts         *int32
	PartNumberMarker *string
}

ListPartsOpts contains optional arguments for the ListParts request.

type ListPartsResponse added in v1.19.0 

type ListPartsResponse struct {
	Parts                []MultipartPart
	NextPartNumberMarker *string
	IsTruncated          bool
}

type Mark added in v0.97.0 

type Mark struct {
	ContinuationToken string
	LastKey           string
	HasMore           bool
}

type MetricsAdapter added in v1.28.1 

type MetricsAdapter struct {
	// contains filtered or unexported fields
}

func (*MetricsAdapter) AbortMultiPartUpload added in v1.28.1 

func (m *MetricsAdapter) AbortMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string) error

func (*MetricsAdapter) BlockstoreMetadata added in v1.28.1 

func (m *MetricsAdapter) BlockstoreMetadata(ctx context.Context) (*BlockstoreMetadata, error)

func (*MetricsAdapter) BlockstoreType added in v1.28.1 

func (m *MetricsAdapter) BlockstoreType() string

func (*MetricsAdapter) CompleteMultiPartUpload added in v1.28.1 

func (m *MetricsAdapter) CompleteMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string, multipartList *MultipartUploadCompletion) (*CompleteMultiPartUploadResponse, error)

func (*MetricsAdapter) Copy added in v1.28.1 

func (m *MetricsAdapter) Copy(ctx context.Context, sourceObj, destinationObj ObjectPointer) error

func (*MetricsAdapter) CreateMultiPartUpload added in v1.28.1 

func (m *MetricsAdapter) CreateMultiPartUpload(ctx context.Context, obj ObjectPointer, r *http.Request, opts CreateMultiPartUploadOpts) (*CreateMultiPartUploadResponse, error)

func (*MetricsAdapter) Exists added in v1.28.1 

func (m *MetricsAdapter) Exists(ctx context.Context, obj ObjectPointer) (bool, error)

func (*MetricsAdapter) Get added in v1.28.1 

func (m *MetricsAdapter) Get(ctx context.Context, obj ObjectPointer) (io.ReadCloser, error)

func (*MetricsAdapter) GetPreSignedURL added in v1.28.1 

func (m *MetricsAdapter) GetPreSignedURL(ctx context.Context, obj ObjectPointer, mode PreSignMode) (string, time.Time, error)

func (*MetricsAdapter) GetPresignUploadPartURL added in v1.28.1 

func (m *MetricsAdapter) GetPresignUploadPartURL(ctx context.Context, obj ObjectPointer, uploadID string, partNumber int) (string, error)

func (*MetricsAdapter) GetProperties added in v1.28.1 

func (m *MetricsAdapter) GetProperties(ctx context.Context, obj ObjectPointer) (Properties, error)

func (*MetricsAdapter) GetRange added in v1.28.1 

func (m *MetricsAdapter) GetRange(ctx context.Context, obj ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)

func (*MetricsAdapter) GetRegion added in v1.28.1 

func (m *MetricsAdapter) GetRegion(ctx context.Context, storageNamespace string) (string, error)

func (*MetricsAdapter) GetStorageNamespaceInfo added in v1.28.1 

func (m *MetricsAdapter) GetStorageNamespaceInfo() StorageNamespaceInfo

func (*MetricsAdapter) GetWalker added in v1.28.1 

func (m *MetricsAdapter) GetWalker(uri *url.URL) (Walker, error)

func (*MetricsAdapter) InnerAdapter added in v1.28.1 

func (m *MetricsAdapter) InnerAdapter() Adapter

func (*MetricsAdapter) ListParts added in v1.28.1 

func (m *MetricsAdapter) ListParts(ctx context.Context, obj ObjectPointer, uploadID string, opts ListPartsOpts) (*ListPartsResponse, error)

func (*MetricsAdapter) Put added in v1.28.1 

func (m *MetricsAdapter) Put(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, opts PutOpts) error

func (*MetricsAdapter) Remove added in v1.28.1 

func (m *MetricsAdapter) Remove(ctx context.Context, obj ObjectPointer) error

func (*MetricsAdapter) ResolveNamespace added in v1.28.1 

func (m *MetricsAdapter) ResolveNamespace(storageNamespace, key string, identifierType IdentifierType) (QualifiedKey, error)

func (*MetricsAdapter) RuntimeStats added in v1.28.1 

func (m *MetricsAdapter) RuntimeStats() map[string]string

func (*MetricsAdapter) UploadCopyPart added in v1.28.1 

func (m *MetricsAdapter) UploadCopyPart(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, partNumber int) (*UploadPartResponse, error)

func (*MetricsAdapter) UploadCopyPartRange added in v1.28.1 

func (m *MetricsAdapter) UploadCopyPartRange(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, partNumber int, startPosition, endPosition int64) (*UploadPartResponse, error)

func (*MetricsAdapter) UploadPart added in v1.28.1 

func (m *MetricsAdapter) UploadPart(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, uploadID string, partNumber int) (*UploadPartResponse, error)

type MultipartPart added in v0.55.0 

type MultipartPart struct {
	ETag         string
	PartNumber   int
	LastModified time.Time
	Size         int64
}

MultipartPart single multipart information

type MultipartUploadCompletion 

type MultipartUploadCompletion struct {
	Part []MultipartPart
}

MultipartUploadCompletion parts described as part of complete multipart upload. Each part holds the part number and ETag received while calling part upload. NOTE that S3 implementation and our S3 gateway accept and returns ETag value surrounded with double-quotes ("), while the adapter implementations supply the raw value of the etag (without double quotes) and let the gateway manage the s3 protocol specifications.

type ObjectPointer 

type ObjectPointer struct {
	StorageNamespace string
	Identifier       string

	// Indicates whether the Identifier is relative to the StorageNamespace,
	// full address to an object, or unknown.
	IdentifierType IdentifierType
}

ObjectPointer is a unique identifier of an object in the object store: the store is a 1:1 mapping between pointers and objects.

type ObjectStoreEntry added in v0.97.0 

type ObjectStoreEntry struct {
	// FullKey represents the fully qualified path in the object store namespace for the given entry
	FullKey string `json:"full_key,omitempty"`
	// RelativeKey represents a path relative to prefix (or directory). If none specified, will be identical to FullKey
	RelativeKey string `json:"relative_key,omitempty"`
	// Address is a full URI for the entry, including the storage namespace (i.e. s3://bucket/path/to/key)
	Address string `json:"address,omitempty"`
	// ETag represents a hash of the entry's content. Generally as hex encoded MD5,
	// but depends on the underlying object store
	ETag string `json:"etag,omitempty"`
	// Mtime is the last-modified datetime of the entry
	Mtime time.Time `json:"mtime,omitempty"`
	// Size in bytes
	Size int64 `json:"size"`
}

func (ObjectStoreEntry) String added in v0.97.0 

func (e ObjectStoreEntry) String() string

type Path 

type Path struct {
	// contains filtered or unexported fields
}

func NewPath 

func NewPath(str, entryType string) *Path

func (*Path) BaseName 

func (p *Path) BaseName() string

func (*Path) Equals 

func (p *Path) Equals(other *Path) bool

func (*Path) IsRoot 

func (p *Path) IsRoot() bool

func (*Path) ParentPath 

func (p *Path) ParentPath() string

func (*Path) Split 

func (p *Path) Split() []string

func (*Path) String 

func (p *Path) String() string

type PreSignMode added in v0.90.0 

type PreSignMode int32

PreSignMode is the mode to use when generating a pre-signed URL (read/write) 

const (
	PreSignModeRead PreSignMode = iota
	PreSignModeWrite
)

type Properties 

type Properties struct {
	StorageClass *string
}

Properties of an object stored on the underlying block store. Refer to the actual underlying Adapter for which properties are actually reported.

type PutOpts 

type PutOpts struct {
	StorageClass *string // S3 storage class
}

PutOpts contains optional arguments for Put. These should be analogous to options on some underlying storage layer. Missing arguments are mapped to the default if a storage layer implements the option.

If the same Put command is implemented multiple times with the same contents but different option values, the first supplied option value is retained.

type QualifiedKey 

type QualifiedKey interface {
	Format() string
	GetStorageType() StorageType
	GetStorageNamespace() string
	GetKey() string
}

type StorageNamespaceInfo 

type StorageNamespaceInfo struct {
	ValidityRegex           string // regex pattern that could be used to validate the namespace
	Example                 string // example of a valid namespace
	DefaultNamespacePrefix  string // when a repo is created from the UI, suggest a default storage namespace under this prefix
	PreSignSupport          bool
	PreSignSupportUI        bool
	PreSignSupportMultipart bool
	ImportSupport           bool
	ImportValidityRegex     string
}

func DefaultStorageNamespaceInfo 

func DefaultStorageNamespaceInfo(scheme string) StorageNamespaceInfo

type StorageType 

type StorageType int

func GetStorageType 

func GetStorageType(namespaceURL *url.URL) (StorageType, error)

func (StorageType) BlockstoreType added in v0.45.1 

func (s StorageType) BlockstoreType() string

func (StorageType) Scheme added in v0.45.1 

func (s StorageType) Scheme() string

type UploadPartResponse added in v0.54.0 

type UploadPartResponse struct {
	ETag             string
	ServerSideHeader http.Header
}

UploadPartResponse upload part ETag and additional headers (implementation specific) currently it targets s3 capabilities to enable encryption properties The ETag is a hex string value of the content checksum

type WalkOptions added in v0.97.0 

type WalkOptions struct {
	// After - all walked items must be greater than After
	After string

	// ContinuationToken is passed to the client for efficient listing. Value is Opaque to the caller.
	ContinuationToken string
}

type WalkOpts 

type WalkOpts struct {
	StorageNamespace string
	Prefix           string
}

WalkOpts is a unique identifier of a prefix in the object store.

type Walker added in v0.97.0 

type Walker interface {
	Walk(ctx context.Context, storageURI *url.URL, op WalkOptions, walkFn func(e ObjectStoreEntry) error) error
	Marker() Mark
	GetSkippedEntries() []ObjectStoreEntry
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.