needle

package
v0.0.0-...-e27f3f3 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 2, 2019 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

View Source
const (
	HeaderSize            = 16 //should never change this
	PaddingSize           = 8
	ChecksumSize          = 4
	MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8
	TombstoneFileSize     = math.MaxUint32
	PairNamePrefix        = "Seaweed-"
	IndexSize             = 16

	FlagGzip                = 0x01
	FlagHasName             = 0x02
	FlagHasMime             = 0x04
	FlagHasLastModifiedDate = 0x08
	FlagHasTtl              = 0x10
	FlagHasPairs            = 0x20
	FlagIsChunkManifest     = 0x80
	LastModifiedBytesLength = 5
	TtlBytesLength          = 2
)
View Source
const (
	RowsToRead = 1024
)

Variables

View Source
var (
	EnableBytesCache = true
)

Functions

func IdxFileEntry

func IdxFileEntry(bytes []byte) (key uint64, offset uint32, size uint32)

func ParseKeyHash

func ParseKeyHash(key_hash_string string) (uint64, uint32, error)

func ParseUpload

func ParseUpload(r *http.Request) (
	fileName string, data []byte,
	mimeType string, pairMap map[string]string,
	isGzipped bool, originalDataSize int,
	modifiedTime uint64,
	TL *ttl.TTL,
	e error)

func WalkIndexFile

func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) error

walks through the index file, calls fn function with each key, offset, size stops with the error returned by the fn function

Types

type Block

type Block struct {
	Bytes []byte
	// contains filtered or unexported fields
}

type CompactMap

type CompactMap struct {
	// contains filtered or unexported fields
}

This map assumes mostly inserting increasing keys

func NewCompactMap

func NewCompactMap() *CompactMap

func (*CompactMap) Delete

func (cm *CompactMap) Delete(key Key) uint32

func (*CompactMap) Get

func (cm *CompactMap) Get(key Key) (*NeedleValue, bool)

func (*CompactMap) Set

func (cm *CompactMap) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32)

func (*CompactMap) Visit

func (cm *CompactMap) Visit(visit func(NeedleValue) error) error

Visit visits all entries or stop if any error when visiting

type CompactSection

type CompactSection struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

func NewCompactSection

func NewCompactSection(start Key) *CompactSection

func (*CompactSection) Delete

func (cs *CompactSection) Delete(key Key) uint32

return old entry size

func (*CompactSection) Get

func (cs *CompactSection) Get(key Key) (*NeedleValue, bool)

func (*CompactSection) Set

func (cs *CompactSection) Set(key Key, offset, size uint32) (oldOffset, oldSize uint32)

return old entry size

type Key

type Key uint64

type Mapper

type Mapper interface {
	Put(key uint64, offset uint32, size uint32) error
	Get(key uint64) (element *NeedleValue, ok bool)
	Delete(key uint64, offset uint32) error
	Close()
	Destroy() error
	ContentSize() uint64
	DeletedSize() uint64
	FileCount() int
	DeletedCount() int
	MaxFileKey() uint64
	IndexFileSize() uint64
	IndexFileContent() ([]byte, error)
	IndexFileName() string
}

type Needle

type Needle struct {
	Cookie       uint32 `comment:"random number to mitigate brute force lookups"`
	Id           uint64 `comment:"needle id"`
	Size         uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
	DataSize     uint32 `comment:"Data size"` //version2
	Data         []byte `comment:"The actual file data"`
	Flags        byte   `comment:"boolean flags"` //version2
	NameSize     uint8  //version2
	Name         []byte `comment:"maximum 256 characters"` //version2
	MimeSize     uint8  //version2
	Mime         []byte `comment:"maximum 256 characters"` //version2
	PairsSize    uint16 //version2
	Pairs        []byte `comment:"additional name value pairs, json format, maximum 64kB"`
	LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
	Ttl          *ttl.TTL
	Checksum     crc.CRC `comment:"CRC32 to check integrity"`
	Padding      []byte  `comment:"Aligned to 8 bytes"`
	// contains filtered or unexported fields
}

* A Needle means a uploaded and stored file. * Needle file size is limited to 4GB for now.

func NewNeedle

func NewNeedle(r *http.Request) (n *Needle, originalSize int, e error)

func (*Needle) Append

func (n *Needle) Append(w io.Writer, vers version.Version) (size uint32, actualSize int64, err error)

func (*Needle) Etag

func (n *Needle) Etag() string

func (*Needle) HasLastModifiedDate

func (n *Needle) HasLastModifiedDate() bool

func (*Needle) HasMime

func (n *Needle) HasMime() bool

func (*Needle) HasName

func (n *Needle) HasName() bool

func (*Needle) HasPairs

func (n *Needle) HasPairs() bool

func (*Needle) HasTtl

func (n *Needle) HasTtl() bool

func (*Needle) IsGzipped

func (n *Needle) IsGzipped() bool

func (*Needle) ParseHeader

func (n *Needle) ParseHeader(byte []byte)

func (*Needle) ParsePath

func (n *Needle) ParsePath(fid string) (err error)

func (*Needle) ReadData

func (n *Needle) ReadData(r *os.File, offset int64, size uint32, vers version.Version) (err error)

func (*Needle) ReleaseMemory

func (n *Needle) ReleaseMemory()

func (*Needle) SetGzipped

func (n *Needle) SetGzipped()

func (*Needle) SetHasLastModifiedDate

func (n *Needle) SetHasLastModifiedDate()

func (*Needle) SetHasMime

func (n *Needle) SetHasMime()

func (*Needle) SetHasName

func (n *Needle) SetHasName()

func (*Needle) SetHasPairs

func (n *Needle) SetHasPairs()

func (*Needle) SetHasTtl

func (n *Needle) SetHasTtl()

type NeedleMap

type NeedleMap struct {
	// contains filtered or unexported fields
}

func LoadCompactNeedleMap

func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error)

func NewCompactNeedleMap

func NewCompactNeedleMap(file *os.File) *NeedleMap

func (*NeedleMap) Close

func (nm *NeedleMap) Close()

func (*NeedleMap) Delete

func (nm *NeedleMap) Delete(key uint64, offset uint32) error

func (*NeedleMap) Destroy

func (nm *NeedleMap) Destroy() error

func (*NeedleMap) Get

func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool)

func (*NeedleMap) IndexFileContent

func (nm *NeedleMap) IndexFileContent() ([]byte, error)

func (*NeedleMap) IndexFileName

func (nm *NeedleMap) IndexFileName() string

func (*NeedleMap) IndexFileSize

func (nm *NeedleMap) IndexFileSize() uint64

func (*NeedleMap) Put

func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) error

type NeedleValue

type NeedleValue struct {
	Key    Key
	Offset uint32 `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
	Size   uint32 `comment:"Size of the data portion"`
}

type NeedleValueMap

type NeedleValueMap interface {
	Set(key Key, offset, size uint32) (oldOffset, oldSize uint32)
	Delete(key Key) uint32
	Get(key Key) (*NeedleValue, bool)
	Visit(visit func(NeedleValue) error) error
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL