needle

package
v0.0.0-...-0b345dd Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 6, 2022 License: Apache-2.0 Imports: 24 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	NeedleChecksumSize = 4
	PairNamePrefix     = "Seaweed-"
)
View Source 
const (
	FlagIsCompressed        = 0x01
	FlagHasName             = 0x02
	FlagHasMime             = 0x04
	FlagHasLastModifiedDate = 0x08
	FlagHasTtl              = 0x10
	FlagHasPairs            = 0x20
	FlagIsChunkManifest     = 0x80
	LastModifiedBytesLength = 5
	TtlBytesLength          = 2
)
View Source 
const (
	// stored unit types
	Empty byte = iota
	Minute
	Hour
	Day
	Week
	Month
	Year
)
View Source 
const (
	Version1       = Version(1)
	Version2       = Version(2)
	Version3       = Version(3)
	CurrentVersion = Version3
)

Variables

View Source 
var EMPTY_TTL = &TTL{}
View Source 
var ErrorSizeMismatch = errors.New("size mismatch")

Functions

func GetActualSize 

func GetActualSize(size Size, version Version) int64

func NeedleBodyLength 

func NeedleBodyLength(needleSize Size, version Version) int64

func PaddingLength 

func PaddingLength(needleSize Size, version Version) Size

func ParseNeedleIdCookie 

func ParseNeedleIdCookie(key_hash_string string) (NeedleId, Cookie, error)

func ReadNeedleBlob 

func ReadNeedleBlob(r backend.BackendStorageFile, offset int64, size Size, version Version) (dataSlice []byte, err error)

func SecondsToTTL 

func SecondsToTTL(seconds int32) string

func WriteNeedleBlob 

func WriteNeedleBlob(w backend.BackendStorageFile, dataSlice []byte, size Size, appendAtNs uint64, version Version) (offset uint64, err error)

Types

type AsyncRequest 

type AsyncRequest struct {
	N              *Needle
	IsWriteRequest bool
	ActualSize     int64
	// contains filtered or unexported fields
}

func NewAsyncRequest 

func NewAsyncRequest(n *Needle, isWriteRequest bool) *AsyncRequest

func (*AsyncRequest) Complete 

func (r *AsyncRequest) Complete(offset uint64, size uint64, isUnchanged bool, err error)

func (*AsyncRequest) IsSucceed 

func (r *AsyncRequest) IsSucceed() bool

func (*AsyncRequest) Submit 

func (r *AsyncRequest) Submit()

func (*AsyncRequest) UpdateResult 

func (r *AsyncRequest) UpdateResult(offset uint64, size uint64, isUnchanged bool, err error)

func (*AsyncRequest) WaitComplete 

func (r *AsyncRequest) WaitComplete() (uint64, uint64, bool, error)

type CRC 

type CRC uint32

func NewCRC 

func NewCRC(b []byte) CRC

func (CRC) Update 

func (c CRC) Update(b []byte) CRC

func (CRC) Value 

func (c CRC) Value() uint32

type CRCwriter 

type CRCwriter struct {
	// contains filtered or unexported fields
}

func NewCRCwriter 

func NewCRCwriter(w io.Writer) *CRCwriter

func (*CRCwriter) Sum 

func (c *CRCwriter) Sum() uint32

func (*CRCwriter) Write 

func (c *CRCwriter) Write(p []byte) (n int, err error)

type FileId 

type FileId struct {
	VolumeId VolumeId
	Key      NeedleId
	Cookie   Cookie
}

func NewFileId 

func NewFileId(VolumeId VolumeId, key uint64, cookie uint32) *FileId

func NewFileIdFromNeedle 

func NewFileIdFromNeedle(VolumeId VolumeId, n *Needle) *FileId

func ParseFileIdFromString 

func ParseFileIdFromString(fid string) (*FileId, error)

Deserialize the file id

func (*FileId) GetCookie 

func (n *FileId) GetCookie() Cookie

func (*FileId) GetNeedleId 

func (n *FileId) GetNeedleId() NeedleId

func (*FileId) GetNeedleIdCookie 

func (n *FileId) GetNeedleIdCookie() string

func (*FileId) GetVolumeId 

func (n *FileId) GetVolumeId() VolumeId

func (*FileId) String 

func (n *FileId) String() string

type Needle 

type Needle struct {
	Cookie Cookie   `comment:"random number to mitigate brute force lookups"`
	Id     NeedleId `comment:"needle id"`
	Size   Size     `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`

	DataSize     uint32 `comment:"Data size"` //version2
	Data         []byte `comment:"The actual file data"`
	Flags        byte   `comment:"boolean flags"` //version2
	NameSize     uint8  //version2
	Name         []byte `comment:"maximum 255 characters"` //version2
	MimeSize     uint8  //version2
	Mime         []byte `comment:"maximum 255 characters"` //version2
	PairsSize    uint16 //version2
	Pairs        []byte `comment:"additional name value pairs, json format, maximum 64kB"`
	LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
	Ttl          *TTL

	Checksum   CRC    `comment:"CRC32 to check integrity"`
	AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
	Padding    []byte `comment:"Aligned to 8 bytes"`
}

* A Needle means a uploaded and stored file. * Needle file size is limited to 4GB for now.

func CreateNeedleFromRequest 

func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool, sizeLimit int64, bytesBuffer *bytes.Buffer) (n *Needle, originalSize int, contentMd5 string, e error)

func ReadNeedleHeader 

func ReadNeedleHeader(r backend.BackendStorageFile, version Version, offset int64) (n *Needle, bytes []byte, bodyLength int64, err error)

func (*Needle) Append 

func (n *Needle) Append(w backend.BackendStorageFile, version Version) (offset uint64, size Size, actualSize int64, err error)

func (*Needle) DiskSize 

func (n *Needle) DiskSize(version Version) int64

func (*Needle) Etag 

func (n *Needle) Etag() string

func (*Needle) HasLastModifiedDate 

func (n *Needle) HasLastModifiedDate() bool

func (*Needle) HasMime 

func (n *Needle) HasMime() bool

func (*Needle) HasName 

func (n *Needle) HasName() bool

func (*Needle) HasPairs 

func (n *Needle) HasPairs() bool

func (*Needle) HasTtl 

func (n *Needle) HasTtl() bool

func (*Needle) IsChunkedManifest 

func (n *Needle) IsChunkedManifest() bool

func (*Needle) IsCompressed 

func (n *Needle) IsCompressed() bool

func (*Needle) LastModifiedString 

func (n *Needle) LastModifiedString() string

func (*Needle) ParseNeedleHeader 

func (n *Needle) ParseNeedleHeader(bytes []byte)

func (*Needle) ParsePath 

func (n *Needle) ParsePath(fid string) (err error)

func (*Needle) ReadBytes 

func (n *Needle) ReadBytes(bytes []byte, offset int64, size Size, version Version) (err error)

ReadBytes hydrates the needle from the bytes buffer, with only n.Id is set.

func (*Needle) ReadData 

func (n *Needle) ReadData(r backend.BackendStorageFile, offset int64, size Size, version Version) (err error)

ReadData hydrates the needle from the file, with only n.Id is set.

func (*Needle) ReadNeedleBody 

func (n *Needle) ReadNeedleBody(r backend.BackendStorageFile, version Version, offset int64, bodyLength int64) (bytes []byte, err error)

n should be a needle already read the header the input stream will read until next file entry

func (*Needle) ReadNeedleBodyBytes 

func (n *Needle) ReadNeedleBodyBytes(needleBody []byte, version Version) (err error)

func (*Needle) SetHasLastModifiedDate 

func (n *Needle) SetHasLastModifiedDate()

func (*Needle) SetHasMime 

func (n *Needle) SetHasMime()

func (*Needle) SetHasName 

func (n *Needle) SetHasName()

func (*Needle) SetHasPairs 

func (n *Needle) SetHasPairs()

func (*Needle) SetHasTtl 

func (n *Needle) SetHasTtl()

func (*Needle) SetIsChunkManifest 

func (n *Needle) SetIsChunkManifest()

func (*Needle) SetIsCompressed 

func (n *Needle) SetIsCompressed()

func (*Needle) String 

func (n *Needle) String() (str string)

type ParsedUpload 

type ParsedUpload struct {
	FileName string
	Data     []byte

	MimeType  string
	PairMap   map[string]string
	IsGzipped bool
	// IsZstd           bool
	OriginalDataSize int
	ModifiedTime     uint64
	Ttl              *TTL
	IsChunkedFile    bool
	UncompressedData []byte
	ContentMd5       string
	// contains filtered or unexported fields
}

func ParseUpload 

func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (pu *ParsedUpload, e error)

type TTL 

type TTL struct {
	Count byte
	Unit  byte
}

func LoadTTLFromBytes 

func LoadTTLFromBytes(input []byte) (t *TTL)

read stored bytes to a ttl

func LoadTTLFromUint32 

func LoadTTLFromUint32(ttl uint32) (t *TTL)

read stored bytes to a ttl

func ReadTTL 

func ReadTTL(ttlString string) (*TTL, error)

translate a readable ttl to internal ttl Supports format example: 3m: 3 minutes 4h: 4 hours 5d: 5 days 6w: 6 weeks 7M: 7 months 8y: 8 years

func (TTL) Minutes 

func (t TTL) Minutes() uint32

func (*TTL) String 

func (t *TTL) String() string

func (*TTL) ToBytes 

func (t *TTL) ToBytes(output []byte)

save stored bytes to an output with 2 bytes

func (*TTL) ToUint32 

func (t *TTL) ToUint32() (output uint32)

type Version 

type Version uint8

type VolumeId 

type VolumeId uint32

func NewVolumeId 

func NewVolumeId(vid string) (VolumeId, error)

func (VolumeId) Next 

func (vid VolumeId) Next() VolumeId

func (VolumeId) String 

func (vid VolumeId) String() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.