Documentation ¶
Index ¶
- Constants
- Variables
- func DefaultExample(scheme string) string
- func DefaultValidationRegex(scheme string) string
- func JoinPathParts(parts []string) string
- func ValidateInterRegionStorage(ctx context.Context, adapter Adapter, storageNamespace string) error
- func ValidateStorageType(uri *url.URL, expectedStorage StorageType) error
- type Adapter
- type BlockstoreMetadata
- type CommonQualifiedKey
- type CompleteMultiPartUploadResponse
- type CreateMultiPartUploadOpts
- type CreateMultiPartUploadResponse
- type HashFunction
- type HashingReader
- type IdentifierType
- type ListPartsOpts
- type ListPartsResponse
- type Mark
- type MetricsAdapter
- func (m *MetricsAdapter) AbortMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string) error
- func (m *MetricsAdapter) BlockstoreMetadata(ctx context.Context) (*BlockstoreMetadata, error)
- func (m *MetricsAdapter) BlockstoreType() string
- func (m *MetricsAdapter) CompleteMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string, ...) (*CompleteMultiPartUploadResponse, error)
- func (m *MetricsAdapter) Copy(ctx context.Context, sourceObj, destinationObj ObjectPointer) error
- func (m *MetricsAdapter) CreateMultiPartUpload(ctx context.Context, obj ObjectPointer, r *http.Request, ...) (*CreateMultiPartUploadResponse, error)
- func (m *MetricsAdapter) Exists(ctx context.Context, obj ObjectPointer) (bool, error)
- func (m *MetricsAdapter) Get(ctx context.Context, obj ObjectPointer) (io.ReadCloser, error)
- func (m *MetricsAdapter) GetPreSignedURL(ctx context.Context, obj ObjectPointer, mode PreSignMode) (string, time.Time, error)
- func (m *MetricsAdapter) GetPresignUploadPartURL(ctx context.Context, obj ObjectPointer, uploadID string, partNumber int) (string, error)
- func (m *MetricsAdapter) GetProperties(ctx context.Context, obj ObjectPointer) (Properties, error)
- func (m *MetricsAdapter) GetRange(ctx context.Context, obj ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)
- func (m *MetricsAdapter) GetRegion(ctx context.Context, storageNamespace string) (string, error)
- func (m *MetricsAdapter) GetStorageNamespaceInfo() StorageNamespaceInfo
- func (m *MetricsAdapter) GetWalker(uri *url.URL) (Walker, error)
- func (m *MetricsAdapter) InnerAdapter() Adapter
- func (m *MetricsAdapter) ListParts(ctx context.Context, obj ObjectPointer, uploadID string, opts ListPartsOpts) (*ListPartsResponse, error)
- func (m *MetricsAdapter) Put(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, ...) (*PutResponse, error)
- func (m *MetricsAdapter) Remove(ctx context.Context, obj ObjectPointer) error
- func (m *MetricsAdapter) ResolveNamespace(storageNamespace, key string, identifierType IdentifierType) (QualifiedKey, error)
- func (m *MetricsAdapter) RuntimeStats() map[string]string
- func (m *MetricsAdapter) UploadCopyPart(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, ...) (*UploadPartResponse, error)
- func (m *MetricsAdapter) UploadCopyPartRange(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, ...) (*UploadPartResponse, error)
- func (m *MetricsAdapter) UploadPart(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, ...) (*UploadPartResponse, error)
- type MultipartPart
- type MultipartUploadCompletion
- type ObjectPointer
- type ObjectStoreEntry
- type Path
- type PreSignMode
- type Properties
- type PutOpts
- type PutResponse
- type QualifiedKey
- type StorageNamespaceInfo
- type StorageType
- type UploadPartResponse
- type WalkOptions
- type WalkOpts
- type Walker
Constants ¶
const ( BlockstoreTypeS3 = "s3" BlockstoreTypeGS = "gs" BlockstoreTypeAzure = "azure" BlockstoreTypeLocal = "local" BlockstoreTypeMem = "mem" BlockstoreTypeTransient = "transient" )
const ( StorageTypeMem = iota StorageTypeLocal StorageTypeS3 StorageTypeGS StorageTypeAzure )
const ( Separator = "/" EntryTypeTree = "tree" EntryTypeObject = "object" )
const DefaultPreSignExpiryDuration = 15 * time.Minute
DefaultPreSignExpiryDuration is the amount of time pre-signed requests are valid for.
Variables ¶
var ( ErrDataNotFound = errors.New("not found") ErrOperationNotSupported = errors.New("operation not supported") ErrAsyncCopyFailed = errors.New("async copy failed") ErrBadIndex = errors.New("bad index") ErrForbidden = errors.New("forbidden") ErrInvalidAddress = errors.New("invalid address") ErrInvalidNamespace = errors.New("invalid namespace") )
var RootPath = NewPath("", EntryTypeTree)
Functions ¶
func DefaultExample ¶
func DefaultValidationRegex ¶
func JoinPathParts ¶
func ValidateInterRegionStorage ¶ added in v1.26.0
func ValidateStorageType ¶ added in v0.97.0
func ValidateStorageType(uri *url.URL, expectedStorage StorageType) error
Types ¶
type Adapter ¶
type Adapter interface { Put(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, opts PutOpts) (*PutResponse, error) Get(ctx context.Context, obj ObjectPointer) (io.ReadCloser, error) // GetWalker is never called on the server side. // TODO(itaiad200): Remove it from this interface. GetWalker(uri *url.URL) (Walker, error) // GetPreSignedURL returns a pre-signed URL for accessing obj with mode, and the // expiry time for this URL. The expiry time IsZero() if reporting // expiry is not supported. The expiry time will be sooner than // Config.*.PreSignedExpiry if an auth token is about to expire. GetPreSignedURL(ctx context.Context, obj ObjectPointer, mode PreSignMode) (string, time.Time, error) GetPresignUploadPartURL(ctx context.Context, obj ObjectPointer, uploadID string, partNumber int) (string, error) Exists(ctx context.Context, obj ObjectPointer) (bool, error) GetRange(ctx context.Context, obj ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error) GetProperties(ctx context.Context, obj ObjectPointer) (Properties, error) Remove(ctx context.Context, obj ObjectPointer) error Copy(ctx context.Context, sourceObj, destinationObj ObjectPointer) error CreateMultiPartUpload(ctx context.Context, obj ObjectPointer, r *http.Request, opts CreateMultiPartUploadOpts) (*CreateMultiPartUploadResponse, error) UploadPart(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, uploadID string, partNumber int) (*UploadPartResponse, error) ListParts(ctx context.Context, obj ObjectPointer, uploadID string, opts ListPartsOpts) (*ListPartsResponse, error) UploadCopyPart(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, partNumber int) (*UploadPartResponse, error) UploadCopyPartRange(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, partNumber int, startPosition, endPosition int64) (*UploadPartResponse, error) AbortMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string) error CompleteMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string, multipartList *MultipartUploadCompletion) (*CompleteMultiPartUploadResponse, error) BlockstoreType() string BlockstoreMetadata(ctx context.Context) (*BlockstoreMetadata, error) GetStorageNamespaceInfo() StorageNamespaceInfo ResolveNamespace(storageNamespace, key string, identifierType IdentifierType) (QualifiedKey, error) GetRegion(ctx context.Context, storageNamespace string) (string, error) RuntimeStats() map[string]string }
Adapter abstract Storage Adapter for persistence of version controlled data. The methods generally map to S3 API methods - Generally some type of Object Storage - Can also be block storage or even in-memory
func NewMetricsAdapter ¶ added in v1.28.1
type BlockstoreMetadata ¶ added in v1.26.0
type BlockstoreMetadata struct {
Region *string
}
type CommonQualifiedKey ¶ added in v0.97.0
type CommonQualifiedKey struct { StorageType StorageType StorageNamespace string Key string }
func DefaultResolveNamespace ¶ added in v0.97.0
func DefaultResolveNamespace(defaultNamespace, key string, identifierType IdentifierType) (CommonQualifiedKey, error)
func (CommonQualifiedKey) Format ¶ added in v0.97.0
func (qk CommonQualifiedKey) Format() string
func (CommonQualifiedKey) GetKey ¶ added in v0.97.0
func (qk CommonQualifiedKey) GetKey() string
func (CommonQualifiedKey) GetStorageNamespace ¶ added in v0.97.0
func (qk CommonQualifiedKey) GetStorageNamespace() string
func (CommonQualifiedKey) GetStorageType ¶ added in v0.97.0
func (qk CommonQualifiedKey) GetStorageType() StorageType
type CompleteMultiPartUploadResponse ¶ added in v0.54.0
type CompleteMultiPartUploadResponse struct { // ETag is a hex string value of the content checksum ETag string // MTime, if non-nil, is the creation time of the resulting object. Typically the // object store returns it on a Last-Modified header from some operations. MTime *time.Time ContentLength int64 ServerSideHeader http.Header }
CompleteMultiPartUploadResponse complete multipart etag, content length and additional headers (implementation specific).
type CreateMultiPartUploadOpts ¶
type CreateMultiPartUploadOpts struct {
StorageClass *string // S3 storage class
}
CreateMultiPartUploadOpts contains optional arguments for CreateMultiPartUpload. These should be analogous to options on some underlying storage layer. Missing arguments are mapped to the default if a storage layer implements the option.
If the same CreateMultiPartUpload command is implemented multiple times with the same contents but different option values, the first supplied option value is retained.
type CreateMultiPartUploadResponse ¶ added in v0.54.0
CreateMultiPartUploadResponse multipart upload ID and additional headers (implementation specific) currently it targets s3 capabilities to enable encryption properties
type HashFunction ¶ added in v1.3.1
type HashFunction int
const ( HashFunctionMD5 HashFunction = iota HashFunctionSHA256 )
type HashingReader ¶
type HashingReader struct { Md5 hash.Hash Sha256 hash.Hash CopiedSize int64 // contains filtered or unexported fields }
func NewHashingReader ¶
func NewHashingReader(body io.Reader, hashTypes ...HashFunction) *HashingReader
type IdentifierType ¶ added in v0.40.0
type IdentifierType int32
IdentifierType is the type the ObjectPointer Identifier
const ( // Deprecated: indicates that the identifier might be relative or full. IdentifierTypeUnknownDeprecated IdentifierType = 0 // IdentifierTypeRelative indicates that the address is relative to the storage namespace. // For example: "/foo/bar" IdentifierTypeRelative IdentifierType = 1 // IdentifierTypeFull indicates that the address is the full address of the object in the object store. // For example: "s3://bucket/foo/bar" IdentifierTypeFull IdentifierType = 2 )
type ListPartsOpts ¶ added in v1.19.0
ListPartsOpts contains optional arguments for the ListParts request.
type ListPartsResponse ¶ added in v1.19.0
type ListPartsResponse struct { Parts []MultipartPart NextPartNumberMarker *string IsTruncated bool }
type MetricsAdapter ¶ added in v1.28.1
type MetricsAdapter struct {
// contains filtered or unexported fields
}
func (*MetricsAdapter) AbortMultiPartUpload ¶ added in v1.28.1
func (m *MetricsAdapter) AbortMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string) error
func (*MetricsAdapter) BlockstoreMetadata ¶ added in v1.28.1
func (m *MetricsAdapter) BlockstoreMetadata(ctx context.Context) (*BlockstoreMetadata, error)
func (*MetricsAdapter) BlockstoreType ¶ added in v1.28.1
func (m *MetricsAdapter) BlockstoreType() string
func (*MetricsAdapter) CompleteMultiPartUpload ¶ added in v1.28.1
func (m *MetricsAdapter) CompleteMultiPartUpload(ctx context.Context, obj ObjectPointer, uploadID string, multipartList *MultipartUploadCompletion) (*CompleteMultiPartUploadResponse, error)
func (*MetricsAdapter) Copy ¶ added in v1.28.1
func (m *MetricsAdapter) Copy(ctx context.Context, sourceObj, destinationObj ObjectPointer) error
func (*MetricsAdapter) CreateMultiPartUpload ¶ added in v1.28.1
func (m *MetricsAdapter) CreateMultiPartUpload(ctx context.Context, obj ObjectPointer, r *http.Request, opts CreateMultiPartUploadOpts) (*CreateMultiPartUploadResponse, error)
func (*MetricsAdapter) Exists ¶ added in v1.28.1
func (m *MetricsAdapter) Exists(ctx context.Context, obj ObjectPointer) (bool, error)
func (*MetricsAdapter) Get ¶ added in v1.28.1
func (m *MetricsAdapter) Get(ctx context.Context, obj ObjectPointer) (io.ReadCloser, error)
func (*MetricsAdapter) GetPreSignedURL ¶ added in v1.28.1
func (m *MetricsAdapter) GetPreSignedURL(ctx context.Context, obj ObjectPointer, mode PreSignMode) (string, time.Time, error)
func (*MetricsAdapter) GetPresignUploadPartURL ¶ added in v1.28.1
func (m *MetricsAdapter) GetPresignUploadPartURL(ctx context.Context, obj ObjectPointer, uploadID string, partNumber int) (string, error)
func (*MetricsAdapter) GetProperties ¶ added in v1.28.1
func (m *MetricsAdapter) GetProperties(ctx context.Context, obj ObjectPointer) (Properties, error)
func (*MetricsAdapter) GetRange ¶ added in v1.28.1
func (m *MetricsAdapter) GetRange(ctx context.Context, obj ObjectPointer, startPosition int64, endPosition int64) (io.ReadCloser, error)
func (*MetricsAdapter) GetStorageNamespaceInfo ¶ added in v1.28.1
func (m *MetricsAdapter) GetStorageNamespaceInfo() StorageNamespaceInfo
func (*MetricsAdapter) GetWalker ¶ added in v1.28.1
func (m *MetricsAdapter) GetWalker(uri *url.URL) (Walker, error)
func (*MetricsAdapter) InnerAdapter ¶ added in v1.28.1
func (m *MetricsAdapter) InnerAdapter() Adapter
func (*MetricsAdapter) ListParts ¶ added in v1.28.1
func (m *MetricsAdapter) ListParts(ctx context.Context, obj ObjectPointer, uploadID string, opts ListPartsOpts) (*ListPartsResponse, error)
func (*MetricsAdapter) Put ¶ added in v1.28.1
func (m *MetricsAdapter) Put(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, opts PutOpts) (*PutResponse, error)
func (*MetricsAdapter) Remove ¶ added in v1.28.1
func (m *MetricsAdapter) Remove(ctx context.Context, obj ObjectPointer) error
func (*MetricsAdapter) ResolveNamespace ¶ added in v1.28.1
func (m *MetricsAdapter) ResolveNamespace(storageNamespace, key string, identifierType IdentifierType) (QualifiedKey, error)
func (*MetricsAdapter) RuntimeStats ¶ added in v1.28.1
func (m *MetricsAdapter) RuntimeStats() map[string]string
func (*MetricsAdapter) UploadCopyPart ¶ added in v1.28.1
func (m *MetricsAdapter) UploadCopyPart(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, partNumber int) (*UploadPartResponse, error)
func (*MetricsAdapter) UploadCopyPartRange ¶ added in v1.28.1
func (m *MetricsAdapter) UploadCopyPartRange(ctx context.Context, sourceObj, destinationObj ObjectPointer, uploadID string, partNumber int, startPosition, endPosition int64) (*UploadPartResponse, error)
func (*MetricsAdapter) UploadPart ¶ added in v1.28.1
func (m *MetricsAdapter) UploadPart(ctx context.Context, obj ObjectPointer, sizeBytes int64, reader io.Reader, uploadID string, partNumber int) (*UploadPartResponse, error)
type MultipartPart ¶ added in v0.55.0
MultipartPart single multipart information
type MultipartUploadCompletion ¶
type MultipartUploadCompletion struct {
Part []MultipartPart
}
MultipartUploadCompletion parts described as part of complete multipart upload. Each part holds the part number and ETag received while calling part upload. NOTE that S3 implementation and our S3 gateway accept and returns ETag value surrounded with double-quotes ("), while the adapter implementations supply the raw value of the etag (without double quotes) and let the gateway manage the s3 protocol specifications.
type ObjectPointer ¶
type ObjectPointer struct { StorageNamespace string Identifier string // Indicates whether the Identifier is relative to the StorageNamespace, // full address to an object, or unknown. IdentifierType IdentifierType }
ObjectPointer is a unique identifier of an object in the object store: the store is a 1:1 mapping between pointers and objects.
type ObjectStoreEntry ¶ added in v0.97.0
type ObjectStoreEntry struct { // FullKey represents the fully qualified path in the object store namespace for the given entry FullKey string `json:"full_key,omitempty"` // RelativeKey represents a path relative to prefix (or directory). If none specified, will be identical to FullKey RelativeKey string `json:"relative_key,omitempty"` // Address is a full URI for the entry, including the storage namespace (i.e. s3://bucket/path/to/key) Address string `json:"address,omitempty"` // ETag represents a hash of the entry's content. Generally as hex encoded MD5, // but depends on the underlying object store ETag string `json:"etag,omitempty"` // Mtime is the last-modified datetime of the entry Mtime time.Time `json:"mtime,omitempty"` // Size in bytes Size int64 `json:"size"` }
func (ObjectStoreEntry) String ¶ added in v0.97.0
func (e ObjectStoreEntry) String() string
type Path ¶
type Path struct {
// contains filtered or unexported fields
}
func (*Path) ParentPath ¶
type PreSignMode ¶ added in v0.90.0
type PreSignMode int32
PreSignMode is the mode to use when generating a pre-signed URL (read/write)
const ( PreSignModeRead PreSignMode = iota PreSignModeWrite )
type Properties ¶
type Properties struct {
StorageClass *string
}
Properties of an object stored on the underlying block store. Refer to the actual underlying Adapter for which properties are actually reported.
type PutOpts ¶
type PutOpts struct {
StorageClass *string // S3 storage class
}
PutOpts contains optional arguments for Put. These should be analogous to options on some underlying storage layer. Missing arguments are mapped to the default if a storage layer implements the option.
If the same Put command is implemented multiple times with the same contents but different option values, the first supplied option value is retained.
type PutResponse ¶ added in v1.41.0
func (*PutResponse) GetMtime ¶ added in v1.41.0
func (r *PutResponse) GetMtime() time.Time
type QualifiedKey ¶
type QualifiedKey interface { Format() string GetStorageType() StorageType GetStorageNamespace() string GetKey() string }
type StorageNamespaceInfo ¶
type StorageNamespaceInfo struct { ValidityRegex string // regex pattern that could be used to validate the namespace Example string // example of a valid namespace DefaultNamespacePrefix string // when a repo is created from the UI, suggest a default storage namespace under this prefix PreSignSupport bool PreSignSupportUI bool PreSignSupportMultipart bool ImportSupport bool ImportValidityRegex string }
func DefaultStorageNamespaceInfo ¶
func DefaultStorageNamespaceInfo(scheme string) StorageNamespaceInfo
type StorageType ¶
type StorageType int
func GetStorageType ¶
func GetStorageType(namespaceURL *url.URL) (StorageType, error)
func (StorageType) BlockstoreType ¶ added in v0.45.1
func (s StorageType) BlockstoreType() string
func (StorageType) Scheme ¶ added in v0.45.1
func (s StorageType) Scheme() string
type UploadPartResponse ¶ added in v0.54.0
UploadPartResponse upload part ETag and additional headers (implementation specific) currently it targets s3 capabilities to enable encryption properties The ETag is a hex string value of the content checksum
type WalkOptions ¶ added in v0.97.0
type Walker ¶ added in v0.97.0
type Walker interface { Walk(ctx context.Context, storageURI *url.URL, op WalkOptions, walkFn func(e ObjectStoreEntry) error) error Marker() Mark GetSkippedEntries() []ObjectStoreEntry }