storage

package
v0.0.0-...-b78b3a4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 16, 2019 License: GPL-3.0 Imports: 32 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ErrInit = iota
	ErrNotFound
	ErrIO
	ErrUnauthorized
	ErrInvalidValue
	ErrDataOverflow
	ErrNothingToReturn
	ErrCorruptData
	ErrInvalidSignature
	ErrNotSynced
	ErrPeriodDepth
	ErrCnt
)
View Source 
const (
	DataChunk = 0
	TreeChunk = 1
)
View Source 
const (
	BMTHash     = "BMT"
	SHA3Hash    = "SHA3" //
	DefaultHash = BMTHash
)
View Source 
const (
	ChunkProcessors = 8
)
View Source 
const KeyLength = 32
View Source 
const MaxPO = 16

Variables

View Source 
var (
	ErrChunkNotFound    = errors.New("chunk not found")
	ErrFetching         = errors.New("chunk still fetching")
	ErrChunkInvalid     = errors.New("invalid chunk")
	ErrChunkForward     = errors.New("cannot forward")
	ErrChunkUnavailable = errors.New("chunk unavailable")
	ErrChunkTimeout     = errors.New("timeout")
)
View Source 
var ZeroAddr = Address(common.Hash{}.Bytes())

Functions

func BytesToU64 

func BytesToU64(data []byte) uint64

func IsZeroAddr 

func IsZeroAddr(addr Address) bool

func NewHasherStore 

func NewHasherStore(chunkStore ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore

func Proximity 

func Proximity(one, other []byte) (ret int)

func PutChunks 

func PutChunks(store *LocalStore, chunks ...*Chunk)

func U64ToBytes 

func U64ToBytes(val uint64) []byte

Types

type Address 

type Address []byte

func PyramidAppend 

func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error)

func PyramidSplit 

func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error)

func TreeSplit 

func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) (k Address, wait func(context.Context) error, err error)

func (Address) Hex 

func (a Address) Hex() string

func (Address) Log 

func (a Address) Log() string

func (Address) MarshalJSON 

func (a Address) MarshalJSON() (out []byte, err error)

func (Address) Size 

func (a Address) Size() uint

func (Address) String 

func (a Address) String() string

func (*Address) UnmarshalJSON 

func (a *Address) UnmarshalJSON(value []byte) error

type AddressCollection 

type AddressCollection []Address

func NewAddressCollection 

func NewAddressCollection(l int) AddressCollection

func (AddressCollection) Len 

func (c AddressCollection) Len() int

func (AddressCollection) Less 

func (c AddressCollection) Less(i, j int) bool

func (AddressCollection) Swap 

func (c AddressCollection) Swap(i, j int)

type Chunk 

type Chunk struct {
	Addr  Address //
	SData []byte  //
	Size  int64   //
	C    chan bool //
	ReqC chan bool //
	// contains filtered or unexported fields
}

func GenerateRandomChunk 

func GenerateRandomChunk(dataSize int64) *Chunk

func GenerateRandomChunks 

func GenerateRandomChunks(dataSize int64, count int) (chunks []*Chunk)

func NewChunk 

func NewChunk(addr Address, reqC chan bool) *Chunk

func (*Chunk) GetErrored 

func (c *Chunk) GetErrored() error

func (*Chunk) SetErrored 

func (c *Chunk) SetErrored(err error)

func (*Chunk) String 

func (c *Chunk) String() string

func (*Chunk) WaitToStore 

func (c *Chunk) WaitToStore() error

type ChunkData 

type ChunkData []byte

func (ChunkData) Data 

func (c ChunkData) Data() []byte

func (ChunkData) Size 

func (c ChunkData) Size() int64

type ChunkStore 

type ChunkStore interface {
	Put(context.Context, *Chunk) //
	Get(context.Context, Address) (*Chunk, error)
	Close()
}

type ChunkValidator 

type ChunkValidator interface {
	Validate(addr Address, data []byte) bool
}

type ChunkerParams 

type ChunkerParams struct {
	// contains filtered or unexported fields
}

type ContentAddressValidator 

type ContentAddressValidator struct {
	Hasher SwarmHasher
}

func NewContentAddressValidator 

func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator

func (*ContentAddressValidator) Validate 

func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool

type DBAPI 

type DBAPI struct {
	// contains filtered or unexported fields
}

func NewDBAPI 

func NewDBAPI(loc *LocalStore) *DBAPI

func (*DBAPI) CurrentBucketStorageIndex 

func (d *DBAPI) CurrentBucketStorageIndex(po uint8) uint64

func (*DBAPI) Get 

func (d *DBAPI) Get(ctx context.Context, addr Address) (*Chunk, error)

func (*DBAPI) GetOrCreateRequest 

func (d *DBAPI) GetOrCreateRequest(ctx context.Context, addr Address) (*Chunk, bool)

func (*DBAPI) Iterator 

func (d *DBAPI) Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error

func (*DBAPI) Put 

func (d *DBAPI) Put(ctx context.Context, chunk *Chunk)

type FileStore 

type FileStore struct {
	ChunkStore
	// contains filtered or unexported fields
}

func NewFileStore 

func NewFileStore(store ChunkStore, params *FileStoreParams) *FileStore

func NewLocalFileStore 

func NewLocalFileStore(datadir string, basekey []byte) (*FileStore, error)

func (*FileStore) HashSize 

func (f *FileStore) HashSize() int

func (*FileStore) Retrieve 

func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChunkReader, isEncrypted bool)

func (*FileStore) Store 

func (f *FileStore) Store(ctx context.Context, data io.Reader, size int64, toEncrypt bool) (addr Address, wait func(context.Context) error, err error)

type FileStoreParams 

type FileStoreParams struct {
	Hash string
}

func NewFileStoreParams 

func NewFileStoreParams() *FileStoreParams

type Getter 

type Getter interface {
	Get(context.Context, Reference) (ChunkData, error)
}

type HashWithLength 

type HashWithLength struct {
	hash.Hash
}

func (*HashWithLength) ResetWithLength 

func (h *HashWithLength) ResetWithLength(length []byte)

type Hasher 

type Hasher func() hash.Hash

type JoinerParams 

type JoinerParams struct {
	ChunkerParams
	// contains filtered or unexported fields
}

type LDBDatabase 

type LDBDatabase struct {
	// contains filtered or unexported fields
}

func NewLDBDatabase 

func NewLDBDatabase(file string) (*LDBDatabase, error)

func (*LDBDatabase) Close 

func (db *LDBDatabase) Close()

func (*LDBDatabase) Delete 

func (db *LDBDatabase) Delete(key []byte) error

func (*LDBDatabase) Get 

func (db *LDBDatabase) Get(key []byte) ([]byte, error)

func (*LDBDatabase) LastKnownTD 

func (db *LDBDatabase) LastKnownTD() []byte

func (*LDBDatabase) NewIterator 

func (db *LDBDatabase) NewIterator() iterator.Iterator

func (*LDBDatabase) Put 

func (db *LDBDatabase) Put(key []byte, value []byte)

func (*LDBDatabase) Write 

func (db *LDBDatabase) Write(batch *leveldb.Batch) error

type LDBStore 

type LDBStore struct {
	// contains filtered or unexported fields
}

func NewLDBStore 

func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error)

func NewMockDbStore 

func NewMockDbStore(params *LDBStoreParams, mockStore *mock.NodeStore) (s *LDBStore, err error)

func (*LDBStore) Cleanup 

func (s *LDBStore) Cleanup()

func (*LDBStore) Close 

func (s *LDBStore) Close()

func (*LDBStore) CurrentBucketStorageIndex 

func (s *LDBStore) CurrentBucketStorageIndex(po uint8) uint64

func (*LDBStore) CurrentStorageIndex 

func (s *LDBStore) CurrentStorageIndex() uint64

func (*LDBStore) Export 

func (s *LDBStore) Export(out io.Writer) (int64, error)

func (*LDBStore) Get 

func (s *LDBStore) Get(ctx context.Context, addr Address) (chunk *Chunk, err error)

func (*LDBStore) Import 

func (s *LDBStore) Import(in io.Reader) (int64, error)

func (*LDBStore) Put 

func (s *LDBStore) Put(ctx context.Context, chunk *Chunk)

func (*LDBStore) ReIndex 

func (s *LDBStore) ReIndex()

func (*LDBStore) Size 

func (s *LDBStore) Size() uint64

func (*LDBStore) SyncIterator 

func (s *LDBStore) SyncIterator(since uint64, until uint64, po uint8, f func(Address, uint64) bool) error

type LDBStoreParams 

type LDBStoreParams struct {
	*StoreParams
	Path string
	Po   func(Address) uint8
}

func NewLDBStoreParams 

func NewLDBStoreParams(storeparams *StoreParams, path string) *LDBStoreParams

type LazyChunkReader 

type LazyChunkReader struct {
	Ctx context.Context
	// contains filtered or unexported fields
}

func TreeJoin 

func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader

func (*LazyChunkReader) Context 

func (r *LazyChunkReader) Context() context.Context

func (*LazyChunkReader) Read 

func (r *LazyChunkReader) Read(b []byte) (read int, err error)

func (*LazyChunkReader) ReadAt 

func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error)

func (*LazyChunkReader) Seek 

func (r *LazyChunkReader) Seek(offset int64, whence int) (int64, error)

func (*LazyChunkReader) Size 

func (r *LazyChunkReader) Size(ctx context.Context, quitC chan bool) (n int64, err error)

type LazySectionReader 

type LazySectionReader interface {
	Context() context.Context
	Size(context.Context, chan bool) (int64, error)
	io.Seeker
	io.Reader
	io.ReaderAt
}

type LazyTestSectionReader 

type LazyTestSectionReader struct {
	*io.SectionReader
}

func (*LazyTestSectionReader) Context 

func (r *LazyTestSectionReader) Context() context.Context

func (*LazyTestSectionReader) Size 

func (r *LazyTestSectionReader) Size(context.Context, chan bool) (int64, error)

type LocalStore 

type LocalStore struct {
	Validators []ChunkValidator

	DbStore *LDBStore
	// contains filtered or unexported fields
}

func NewLocalStore 

func NewLocalStore(params *LocalStoreParams, mockStore *mock.NodeStore) (*LocalStore, error)

func NewTestLocalStoreForAddr 

func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error)

func (*LocalStore) Close 

func (ls *LocalStore) Close()

func (*LocalStore) Get 

func (ls *LocalStore) Get(ctx context.Context, addr Address) (chunk *Chunk, err error)

func (*LocalStore) GetOrCreateRequest 

func (ls *LocalStore) GetOrCreateRequest(ctx context.Context, addr Address) (chunk *Chunk, created bool)

func (*LocalStore) Put 

func (ls *LocalStore) Put(ctx context.Context, chunk *Chunk)

func (*LocalStore) RequestsCacheLen 

func (ls *LocalStore) RequestsCacheLen() int

type LocalStoreParams 

type LocalStoreParams struct {
	*StoreParams
	ChunkDbPath string
	Validators  []ChunkValidator `toml:"-"`
}

func NewDefaultLocalStoreParams 

func NewDefaultLocalStoreParams() *LocalStoreParams

func (*LocalStoreParams) Init 

func (p *LocalStoreParams) Init(path string)

type MapChunkStore 

type MapChunkStore struct {
	// contains filtered or unexported fields
}

func NewMapChunkStore 

func NewMapChunkStore() *MapChunkStore

func (*MapChunkStore) Close 

func (m *MapChunkStore) Close()

func (*MapChunkStore) Get 

func (m *MapChunkStore) Get(ctx context.Context, addr Address) (*Chunk, error)

func (*MapChunkStore) Put 

func (m *MapChunkStore) Put(ctx context.Context, chunk *Chunk)

type MemStore 

type MemStore struct {
	// contains filtered or unexported fields
}

func NewMemStore 

func NewMemStore(params *StoreParams, _ *LDBStore) (m *MemStore)

func (*MemStore) Close 

func (s *MemStore) Close()

func (*MemStore) Get 

func (m *MemStore) Get(ctx context.Context, addr Address) (*Chunk, error)

func (*MemStore) Put 

func (m *MemStore) Put(ctx context.Context, c *Chunk)

type NetStore 

type NetStore struct {
	// contains filtered or unexported fields
}

func NewNetStore 

func NewNetStore(localStore *LocalStore, retrieve func(ctx context.Context, chunk *Chunk) error) *NetStore

func (*NetStore) Close 

func (ns *NetStore) Close()

func (*NetStore) Get 

func (ns *NetStore) Get(ctx context.Context, addr Address) (chunk *Chunk, err error)

func (*NetStore) GetWithTimeout 

func (ns *NetStore) GetWithTimeout(ctx context.Context, addr Address, timeout time.Duration) (chunk *Chunk, err error)

func (*NetStore) Put 

func (ns *NetStore) Put(ctx context.Context, chunk *Chunk)

type Peer 

type Peer interface{}

type Putter 

type Putter interface {
	Put(context.Context, ChunkData) (Reference, error)
	//
	RefSize() int64
	//
	Close()
	//
	Wait(context.Context) error
}

type PyramidChunker 

type PyramidChunker struct {
	// contains filtered or unexported fields
}

func NewPyramidSplitter 

func NewPyramidSplitter(params *PyramidSplitterParams) (pc *PyramidChunker)

func (*PyramidChunker) Append 

func (pc *PyramidChunker) Append(ctx context.Context) (k Address, wait func(context.Context) error, err error)

func (*PyramidChunker) Join 

func (pc *PyramidChunker) Join(addr Address, getter Getter, depth int) LazySectionReader

func (*PyramidChunker) Split 

func (pc *PyramidChunker) Split(ctx context.Context) (k Address, wait func(context.Context) error, err error)

type PyramidSplitterParams 

type PyramidSplitterParams struct {
	SplitterParams
	// contains filtered or unexported fields
}

func NewPyramidSplitterParams 

func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, getter Getter, chunkSize int64) *PyramidSplitterParams

type Reference 

type Reference []byte

type SplitterParams 

type SplitterParams struct {
	ChunkerParams
	// contains filtered or unexported fields
}

type StoreParams 

type StoreParams struct {
	Hash                       SwarmHasher `toml:"-"`
	DbCapacity                 uint64
	CacheCapacity              uint
	ChunkRequestsCacheCapacity uint
	BaseKey                    []byte
}

func NewDefaultStoreParams 

func NewDefaultStoreParams() *StoreParams

func NewStoreParams 

func NewStoreParams(ldbCap uint64, cacheCap uint, requestsCap uint, hash SwarmHasher, basekey []byte) *StoreParams

type SwarmHash 

type SwarmHash interface {
	hash.Hash
	ResetWithLength([]byte)
}

type SwarmHasher 

type SwarmHasher func() SwarmHash

func MakeHashFunc 

func MakeHashFunc(hash string) SwarmHasher

type TreeChunker 

type TreeChunker struct {
	// contains filtered or unexported fields
}

func NewTreeJoiner 

func NewTreeJoiner(params *JoinerParams) *TreeChunker

func NewTreeSplitter 

func NewTreeSplitter(params *TreeSplitterParams) *TreeChunker

func (*TreeChunker) Append 

func (tc *TreeChunker) Append() (Address, func(), error)

func (*TreeChunker) Join 

func (tc *TreeChunker) Join(ctx context.Context) *LazyChunkReader

func (*TreeChunker) Split 

func (tc *TreeChunker) Split(ctx context.Context) (k Address, wait func(context.Context) error, err error)

type TreeEntry 

type TreeEntry struct {
	// contains filtered or unexported fields
}

func NewTreeEntry 

func NewTreeEntry(pyramid *PyramidChunker) *TreeEntry

type TreeSplitterParams 

type TreeSplitterParams struct {
	SplitterParams
	// contains filtered or unexported fields
}

Source Files

View all Source files

Directories

Path Synopsis
db
mem
rpc
test

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.