object

package
v0.1.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 18, 2024 License: Apache-2.0 Imports: 121 Imported by: 15

Documentation

Index

Constants

View Source 
const (
	// AsyncWriteBack writes the object asynchronously to the backend.
	AsyncWriteBack = iota

	// WriteBack writes the object synchronously to the backend.
	WriteBack

	// Ephemeral only writes the object to the dfdaemon.
	// It is only provided for creating temporary objects between peers,
	// and users are not allowed to use this mode.
	Ephemeral
)
View Source 
const (
	// HeaderDragonflyObjectMetaLastModifiedTime is used for last modified time of object storage.
	HeaderDragonflyObjectMetaLastModifiedTime = "X-Dragonfly-Object-Meta-Last-Modified-Time"

	// HeaderDragonflyObjectMetaStorageClass is used for storage class of object storage.
	HeaderDragonflyObjectMetaStorageClass = "X-Dragonfly-Object-Meta-Storage-Class"

	// HeaderDragonflyObjectOperation is used for object storage operation.
	HeaderDragonflyObjectOperation = "X-Dragonfly-Object-Operation"
)
View Source 
const (
	// Upper limit of maxGetObjectMetadatas.
	MaxGetObjectMetadatasLimit = 1000

	// DefaultMaxReplicas is the default value of maxReplicas.
	DefaultMaxReplicas = 0

	// Upper limit of maxReplicas.
	MaxReplicasLimit = 100
)
View Source 
const (
	// FilterOSS is the filter of oss url for generating task id.
	FilterOSS = "Expires&Signature"

	// FilterS3 is the filter of s3 url for generating task id.
	FilterS3 = "X-Amz-Algorithm&X-Amz-Credential&X-Amz-Date&X-Amz-Expires&X-Amz-SignedHeaders&X-Amz-Signature"

	// FilterOBS is the filter of obs url for generating task id.
	FilterOBS = "X-Amz-Algorithm&X-Amz-Credential&X-Amz-Date&X-Obs-Date&X-Amz-Expires&X-Amz-SignedHeaders&X-Amz-Signature"
)
View Source 
const (
	AES256GCM_RSA = "aes256gcm-rsa"
	CHACHA20_RSA  = "chacha20-rsa"
)
View Source 
const (
	// CopyOperation is the operation of copying object.
	CopyOperation = "copy"
)
View Source 
const DefaultStorageClass = "STANDARD"

Variables

View Source 
var HEADER_NAMES = []string{"Content-MD5", "Content-Type", "Date"}
View Source 
var OVHCompileRegexp = `^s3\.(\w*)(\.\w*)?\.cloud\.ovh\.net$`
View Source 
var PutInplace bool
View Source 
var TryCFR bool // try copy_file_range
View Source 
var UserAgent = "JuiceFS"

Functions

func ExportRsaPrivateKeyToPem 

func ExportRsaPrivateKeyToPem(key *rsa.PrivateKey, passphrase string) string

func GetHttpClient 

func GetHttpClient() *http.Client

func IsErrReplicating 

func IsErrReplicating(err error) bool

func IsFileSystem 

func IsFileSystem(object ObjectStorage) bool

func ListAll 

func ListAll(store ObjectStorage, prefix, marker string, followLink bool) (<-chan Object, error)

ListAll on all the keys that starts at marker from object storage.

func ListAllWithDelimiter 

func ListAllWithDelimiter(store ObjectStorage, prefix, start, end string, followLink bool) (<-chan Object, error)

func MarshalObject 

func MarshalObject(o Object) map[string]interface{}

func ParseRsaPrivateKeyFromPath 

func ParseRsaPrivateKeyFromPath(path, passphrase string) (*rsa.PrivateKey, error)

func ParseRsaPrivateKeyFromPem 

func ParseRsaPrivateKeyFromPem(enc []byte, passphrase []byte) (*rsa.PrivateKey, error)

func Register 

func Register(name string, register Creator)

func Shutdown 

func Shutdown(o ObjectStorage)

Types

type AttrGetter 

type AttrGetter func(attrs *ResponseAttrs)

func WithRequestID 

func WithRequestID(id *string) AttrGetter

func WithStorageClass 

func WithStorageClass(sc *string) AttrGetter

type COS 

type COS struct {
	// contains filtered or unexported fields
}

func (*COS) AbortUpload 

func (c *COS) AbortUpload(key string, uploadID string)

func (*COS) CompleteUpload 

func (c *COS) CompleteUpload(key string, uploadID string, parts []*Part) error

func (*COS) Copy 

func (c *COS) Copy(dst, src string) error

func (*COS) Create 

func (c *COS) Create() error

func (*COS) CreateMultipartUpload 

func (c *COS) CreateMultipartUpload(key string) (*MultipartUpload, error)

func (*COS) Delete 

func (c *COS) Delete(key string, getters ...AttrGetter) error

func (*COS) Get 

func (c *COS) Get(key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)

func (*COS) Head 

func (c *COS) Head(key string) (Object, error)

func (*COS) Limits 

func (c *COS) Limits() Limits

func (*COS) List 

func (c *COS) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error)

func (*COS) ListAll 

func (c *COS) ListAll(prefix, marker string, followLink bool) (<-chan Object, error)

func (*COS) ListUploads 

func (c *COS) ListUploads(marker string) ([]*PendingPart, string, error)

func (*COS) Put 

func (c *COS) Put(key string, in io.Reader, getters ...AttrGetter) error

func (*COS) SetStorageClass 

func (c *COS) SetStorageClass(sc string) error

func (*COS) String 

func (c *COS) String() string

func (*COS) UploadPart 

func (c *COS) UploadPart(key string, uploadID string, num int, body []byte) (*Part, error)

func (*COS) UploadPartCopy 

func (c *COS) UploadPartCopy(key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error)

type Creator 

type Creator func(bucket, accessKey, secretKey, token string) (ObjectStorage, error)

type DataItem 

type DataItem struct {
	FileName   string
	Size       int64
	ModifyTime int
}

type DefaultObjectStorage 

type DefaultObjectStorage struct{}

func (DefaultObjectStorage) AbortUpload 

func (s DefaultObjectStorage) AbortUpload(key string, uploadID string)

func (DefaultObjectStorage) CompleteUpload 

func (s DefaultObjectStorage) CompleteUpload(key string, uploadID string, parts []*Part) error

func (DefaultObjectStorage) Copy 

func (s DefaultObjectStorage) Copy(dst, src string) error

func (DefaultObjectStorage) Create 

func (s DefaultObjectStorage) Create() error

func (DefaultObjectStorage) CreateMultipartUpload 

func (s DefaultObjectStorage) CreateMultipartUpload(key string) (*MultipartUpload, error)

func (DefaultObjectStorage) Head 

func (s DefaultObjectStorage) Head(key string) (Object, error)

func (DefaultObjectStorage) Limits 

func (s DefaultObjectStorage) Limits() Limits

func (DefaultObjectStorage) List 

func (s DefaultObjectStorage) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error)

func (DefaultObjectStorage) ListAll 

func (s DefaultObjectStorage) ListAll(prefix, marker string, followLink bool) (<-chan Object, error)

func (DefaultObjectStorage) ListUploads 

func (s DefaultObjectStorage) ListUploads(marker string) ([]*PendingPart, string, error)

func (DefaultObjectStorage) UploadPart 

func (s DefaultObjectStorage) UploadPart(key string, uploadID string, num int, body []byte) (*Part, error)

func (DefaultObjectStorage) UploadPartCopy 

func (s DefaultObjectStorage) UploadPartCopy(key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error)

type Encryptor 

type Encryptor interface {
	Encrypt(plaintext []byte) ([]byte, error)
	Decrypt(ciphertext []byte) ([]byte, error)
}

func NewDataEncryptor 

func NewDataEncryptor(keyEncryptor Encryptor, algo string) (Encryptor, error)

func NewRSAEncryptor 

func NewRSAEncryptor(privKey *rsa.PrivateKey) Encryptor

type File 

type File interface {
	Object
	Owner() string
	Group() string
	Mode() os.FileMode
}

type FileSystem 

type FileSystem interface {
	MtimeChanger
	Chmod(path string, mode os.FileMode) error
	Chown(path string, owner, group string) error
}

type Limits 

type Limits struct {
	IsSupportMultipartUpload bool
	IsSupportUploadPartCopy  bool
	MinPartSize              int
	MaxPartSize              int64
	MaxPartCount             int
}

type MtimeChanger 

type MtimeChanger interface {
	Chtimes(path string, mtime time.Time) error
}

type MultipartUpload 

type MultipartUpload struct {
	MinPartSize int
	MaxCount    int
	UploadID    string
}

type Object 

type Object interface {
	Key() string
	Size() int64
	Mtime() time.Time
	IsDir() bool
	IsSymlink() bool
	StorageClass() string
}

func UnmarshalObject 

func UnmarshalObject(m map[string]interface{}) Object

type ObjectMetadata 

type ObjectMetadata struct {
	// Key is object key.
	Key string

	// ContentDisposition is Content-Disposition header.
	ContentDisposition string

	// ContentEncoding is Content-Encoding header.
	ContentEncoding string

	// ContentLanguage is Content-Language header.
	ContentLanguage string

	// ContentLanguage is Content-Length header.
	ContentLength int64

	// ContentType is Content-Type header.
	ContentType string

	// ETag is ETag header.
	ETag string

	// Digest is object digest.
	Digest string

	// LastModifiedTime is last modified time.
	LastModifiedTime time.Time

	// StorageClass is object storage class.
	StorageClass string
}

ObjectMetadata is the object metadata.

type ObjectMetadatas 

type ObjectMetadatas struct {
	// CommonPrefixes are similar prefixes in object storage.
	CommonPrefixes []string `json:"CommonPrefixes"`

	// Metadatas are object metadata.
	Metadatas []*ObjectMetadata `json:"Metadatas"`
}

ObjectMetadatas is the object metadata list.

type ObjectStorage 

type ObjectStorage interface {
	// Description of the object storage.
	String() string
	// Limits of the object storage.
	Limits() Limits
	// Create the bucket if not existed.
	Create() error
	// Get the data for the given object specified by key.
	Get(key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)
	// Put data read from a reader to an object specified by key.
	Put(key string, in io.Reader, getters ...AttrGetter) error
	// Copy an object from src to dst.
	Copy(dst, src string) error
	// Delete a object.
	Delete(key string, getters ...AttrGetter) error

	// Head returns some information about the object or an error if not found.
	Head(key string) (Object, error)
	// List returns a list of objects.
	List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error)
	// ListAll returns all the objects as an channel.
	ListAll(prefix, marker string, followLink bool) (<-chan Object, error)

	// CreateMultipartUpload starts to upload a large object part by part.
	CreateMultipartUpload(key string) (*MultipartUpload, error)
	// UploadPart upload a part of an object.
	UploadPart(key string, uploadID string, num int, body []byte) (*Part, error)
	// UploadPartCopy Uploads a part by copying data from an existing object as data source.
	UploadPartCopy(key string, uploadID string, num int, srcKey string, off, size int64) (*Part, error)
	// AbortUpload abort a multipart upload.
	AbortUpload(key string, uploadID string)
	// CompleteUpload finish an multipart upload.
	CompleteUpload(key string, uploadID string, parts []*Part) error
	// ListUploads lists existing multipart uploads.
	ListUploads(marker string) ([]*PendingPart, string, error)
}

ObjectStorage is the interface for object storage. all of these API should be idempotent.

func CreateStorage 

func CreateStorage(name, endpoint, accessKey, secretKey, token string) (ObjectStorage, error)

func NewEncrypted 

func NewEncrypted(o ObjectStorage, enc Encryptor) ObjectStorage

NewEncrypted returns a encrypted object storage

func NewSharded 

func NewSharded(name, endpoint, ak, sk, token string, shards int) (ObjectStorage, error)

func WithPrefix 

func WithPrefix(os ObjectStorage, prefix string) ObjectStorage

WithPrefix return an object storage that add a prefix to keys.

type ObjectStorageMetadata 

type ObjectStorageMetadata struct {
	// Name is object storage name of type, it can be s3, oss or obs.
	Name string

	// Region is storage region.
	Region string

	// Endpoint is datacenter endpoint.
	Endpoint string
}

ObjectStorageMetadata is the object storage metadata.

type Part 

type Part struct {
	Num  int
	Size int
	ETag string
}

type PendingPart 

type PendingPart struct {
	Key      string
	UploadID string
	Created  time.Time
}

type ResponseAttrs 

type ResponseAttrs struct {
	// contains filtered or unexported fields
}

A generic way to get attributes from different object storage clients

func (*ResponseAttrs) SetRequestID 

func (r *ResponseAttrs) SetRequestID(id string) *ResponseAttrs

func (*ResponseAttrs) SetStorageClass 

func (r *ResponseAttrs) SetStorageClass(sc string) *ResponseAttrs

type RestfulStorage 

type RestfulStorage struct {
	DefaultObjectStorage
	// contains filtered or unexported fields
}

func (*RestfulStorage) Copy 

func (s *RestfulStorage) Copy(dst, src string) error

func (*RestfulStorage) Delete 

func (s *RestfulStorage) Delete(key string, getters ...AttrGetter) error

func (*RestfulStorage) Get 

func (s *RestfulStorage) Get(key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error)

func (*RestfulStorage) Head 

func (s *RestfulStorage) Head(key string) (Object, error)

func (*RestfulStorage) List 

func (s *RestfulStorage) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error)

func (*RestfulStorage) Put 

func (u *RestfulStorage) Put(key string, body io.Reader, getters ...AttrGetter) error

func (*RestfulStorage) String 

func (s *RestfulStorage) String() string

type SectionReaderCloser 

type SectionReaderCloser struct {
	*io.SectionReader
	io.Closer
}

type Shutdownable 

type Shutdownable interface {
	Shutdown()
}

type SupportStorageClass 

type SupportStorageClass interface {
	SetStorageClass(sc string) error
}
type SupportSymlink interface {
	// Symlink create a symbolic link
	Symlink(oldName, newName string) error
	// Readlink read a symbolic link
	Readlink(name string) (string, error)
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.