Documentation ¶
Overview ¶
Copyright 2016 The go-ethereum Authors This file is part of the go-ethereum library.
The go-ethereum library is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
The go-ethereum library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
Index ¶
- Constants
- Variables
- func BytesToU64(data []byte) uint64
- func GenerateRandomData(l int) (r io.Reader, slice []byte)
- func IsZeroAddr(addr Address) bool
- func NewChunk(addr Address, data []byte) *chunk
- func NewHasherStore(store ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore
- func Proximity(one, other []byte) (ret int)
- func U64ToBytes(val uint64) []byte
- type Address
- func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, ...) (Address, func(context.Context) error, error)
- func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error)
- func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) (k Address, wait func(context.Context) error, err error)
- type AddressCollection
- type Chunk
- type ChunkData
- type ChunkStore
- type ChunkValidator
- type ChunkerParams
- type ContentAddressValidator
- type FakeChunkStore
- type FileStore
- type FileStoreParams
- type Getter
- type HashWithLength
- type Hasher
- type JoinerParams
- type LDBDatabase
- func (db *LDBDatabase) Close()
- func (db *LDBDatabase) Delete(key []byte) error
- func (db *LDBDatabase) Get(key []byte) ([]byte, error)
- func (db *LDBDatabase) LastKnownTD() []byte
- func (db *LDBDatabase) NewIterator() iterator.Iterator
- func (db *LDBDatabase) Put(key []byte, value []byte) error
- func (db *LDBDatabase) Write(batch *leveldb.Batch) error
- type LDBStore
- func (s *LDBStore) BinIndex(po uint8) uint64
- func (s *LDBStore) Cleanup(f func(*chunk) bool)
- func (s *LDBStore) Close()
- func (s *LDBStore) CurrentStorageIndex() uint64
- func (s *LDBStore) Delete(addr Address)
- func (s *LDBStore) Export(out io.Writer) (int64, error)
- func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error)
- func (s *LDBStore) GetSchema() (string, error)
- func (s *LDBStore) Import(in io.Reader) (int64, error)
- func (s *LDBStore) Put(ctx context.Context, chunk Chunk) error
- func (s *LDBStore) PutSchema(schema string) error
- func (s *LDBStore) ReIndex()
- func (s *LDBStore) Size() uint64
- func (s *LDBStore) SyncIterator(since uint64, until uint64, po uint8, f func(Address, uint64) bool) error
- type LDBStoreParams
- type LazyChunkReader
- func (r *LazyChunkReader) Context() context.Context
- func (r *LazyChunkReader) Read(b []byte) (read int, err error)
- func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error)
- func (r *LazyChunkReader) Seek(offset int64, whence int) (int64, error)
- func (r *LazyChunkReader) Size(ctx context.Context, quitC chan bool) (n int64, err error)
- type LazySectionReader
- type LazyTestSectionReader
- type LocalStore
- func (ls *LocalStore) BinIndex(po uint8) uint64
- func (ls *LocalStore) Close()
- func (ls *LocalStore) FetchFunc(ctx context.Context, addr Address) func(context.Context) error
- func (ls *LocalStore) Get(ctx context.Context, addr Address) (chunk Chunk, err error)
- func (ls *LocalStore) Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error
- func (ls *LocalStore) Migrate() error
- func (ls *LocalStore) Put(ctx context.Context, chunk Chunk) error
- type LocalStoreParams
- type MemStore
- type NetFetcher
- type NetStore
- func (n *NetStore) BinIndex(po uint8) uint64
- func (n *NetStore) Close()
- func (n *NetStore) FetchFunc(ctx context.Context, ref Address) func(context.Context) error
- func (n *NetStore) Get(rctx context.Context, ref Address) (Chunk, error)
- func (n *NetStore) Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error
- func (n *NetStore) Put(ctx context.Context, ch Chunk) error
- func (n *NetStore) RequestsCacheLen() int
- type NewNetFetcherFunc
- type Peer
- type Putter
- type PyramidChunker
- func (pc *PyramidChunker) Append(ctx context.Context) (k Address, wait func(context.Context) error, err error)
- func (pc *PyramidChunker) Join(addr Address, getter Getter, depth int) LazySectionReader
- func (pc *PyramidChunker) Split(ctx context.Context) (k Address, wait func(context.Context) error, err error)
- type PyramidSplitterParams
- type Reference
- type SplitterParams
- type StoreParams
- type SwarmHash
- type SwarmHasher
- type SyncChunkStore
- type TreeChunker
- type TreeEntry
- type TreeSplitterParams
Constants ¶
const ( ErrInit = iota ErrNotFound ErrIO ErrInvalidValue ErrDataOverflow ErrNothingToReturn ErrCorruptData ErrInvalidSignature ErrNotSynced ErrPeriodDepth ErrCnt )
const ( DataChunk = 0 TreeChunk = 1 )
const ( BMTHash = "BMT" SHA3Hash = "SHA3" // http://golang.org/pkg/hash/#Hash DefaultHash = BMTHash )
const AddressLength = 32
const (
ChunkProcessors = 8
)
const CurrentDbSchema = DbSchemaPurity
const DbSchemaPurity = "purity"
"purity" is the first formal schema of LevelDB we release together with Swarm 0.3.5
const MaxPO = 16
Variables ¶
var ( ErrChunkNotFound = errors.New("chunk not found") ErrFetching = errors.New("chunk still fetching") ErrChunkInvalid = errors.New("invalid chunk") ErrChunkForward = errors.New("cannot forward") ErrChunkTimeout = errors.New("timeout") )
var (
ErrDBClosed = errors.New("LDBStore closed")
)
var ZeroAddr = Address(common.Hash{}.Bytes())
Functions ¶
func BytesToU64 ¶
func GenerateRandomData ¶ added in v0.3.6
func IsZeroAddr ¶ added in v0.3.3
func NewHasherStore ¶ added in v0.3.3
func NewHasherStore(store ChunkStore, hashFunc SwarmHasher, toEncrypt bool) *hasherStore
NewHasherStore creates a hasherStore object, which implements Putter and Getter interfaces. With the HasherStore you can put and get chunk data (which is just []byte) into a ChunkStore and the hasherStore will take core of encryption/decryption of data if necessary
func U64ToBytes ¶
Types ¶
type Address ¶ added in v0.3.3
type Address []byte
func PyramidAppend ¶ added in v0.3.3
func PyramidSplit ¶ added in v0.3.3
func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error)
When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Address), the root hash of the entire content will fill this once processing finishes. New chunks to store are store using the putter which the caller provides.
func TreeSplit ¶ added in v0.3.3
func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) (k Address, wait func(context.Context) error, err error)
When splitting, data is given as a SectionReader, and the key is a hashSize long byte slice (Key), the root hash of the entire content will fill this once processing finishes. New chunks to store are store using the putter which the caller provides.
func (Address) MarshalJSON ¶ added in v0.3.3
func (*Address) UnmarshalJSON ¶ added in v0.3.3
type AddressCollection ¶ added in v0.3.3
type AddressCollection []Address
func NewAddressCollection ¶ added in v0.3.3
func NewAddressCollection(l int) AddressCollection
func (AddressCollection) Len ¶ added in v0.3.3
func (c AddressCollection) Len() int
func (AddressCollection) Less ¶ added in v0.3.3
func (c AddressCollection) Less(i, j int) bool
func (AddressCollection) Swap ¶ added in v0.3.3
func (c AddressCollection) Swap(i, j int)
type Chunk ¶
type Chunk interface { Address() Address Payload() []byte SpanBytes() []byte Span() int64 Data() []byte }
Chunk interface implemented by context.Contexts and data chunks
func GenerateRandomChunk ¶ added in v0.3.3
func GenerateRandomChunks ¶ added in v0.3.3
type ChunkStore ¶
type ChunkValidator ¶ added in v0.3.3
type ChunkerParams ¶
type ChunkerParams struct {
// contains filtered or unexported fields
}
type ContentAddressValidator ¶ added in v0.3.3
type ContentAddressValidator struct {
Hasher SwarmHasher
}
Provides method for validation of content address in chunks Holds the corresponding hasher to create the address
func NewContentAddressValidator ¶ added in v0.3.3
func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator
Constructor
type FakeChunkStore ¶ added in v0.3.6
type FakeChunkStore struct { }
FakeChunkStore doesn't store anything, just implements the ChunkStore interface It can be used to inject into a hasherStore if you don't want to actually store data just do the hashing
func (*FakeChunkStore) Close ¶ added in v0.3.6
func (f *FakeChunkStore) Close()
Close doesn't store anything it is just here to implement ChunkStore
type FileStore ¶ added in v0.3.3
type FileStore struct { ChunkStore // contains filtered or unexported fields }
func NewFileStore ¶ added in v0.3.3
func NewFileStore(store ChunkStore, params *FileStoreParams) *FileStore
func NewLocalFileStore ¶ added in v0.3.3
for testing locally
func (*FileStore) Retrieve ¶ added in v0.3.3
func (f *FileStore) Retrieve(ctx context.Context, addr Address) (reader *LazyChunkReader, isEncrypted bool)
Public API. Main entry point for document retrieval directly. Used by the FS-aware API and httpaccess Chunk retrieval blocks on netStore requests with a timeout so reader will report error if retrieval of chunks within requested range time out. It returns a reader with the chunk data and whether the content was encrypted
type FileStoreParams ¶ added in v0.3.3
type FileStoreParams struct {
Hash string
}
func NewFileStoreParams ¶ added in v0.3.3
func NewFileStoreParams() *FileStoreParams
type HashWithLength ¶
func (*HashWithLength) ResetWithLength ¶
func (h *HashWithLength) ResetWithLength(length []byte)
type JoinerParams ¶ added in v0.3.3
type JoinerParams struct { ChunkerParams // contains filtered or unexported fields }
type LDBDatabase ¶
type LDBDatabase struct {
// contains filtered or unexported fields
}
func NewLDBDatabase ¶
func NewLDBDatabase(file string) (*LDBDatabase, error)
func (*LDBDatabase) Close ¶
func (db *LDBDatabase) Close()
func (*LDBDatabase) Delete ¶
func (db *LDBDatabase) Delete(key []byte) error
func (*LDBDatabase) LastKnownTD ¶
func (db *LDBDatabase) LastKnownTD() []byte
func (*LDBDatabase) NewIterator ¶
func (db *LDBDatabase) NewIterator() iterator.Iterator
type LDBStore ¶ added in v0.3.3
type LDBStore struct {
// contains filtered or unexported fields
}
func NewLDBStore ¶ added in v0.3.3
func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error)
TODO: Instead of passing the distance function, just pass the address from which distances are calculated to avoid the appearance of a pluggable distance metric and opportunities of bugs associated with providing a function different from the one that is actually used.
func NewMockDbStore ¶ added in v0.3.3
func NewMockDbStore(params *LDBStoreParams, mockStore *mock.NodeStore) (s *LDBStore, err error)
NewMockDbStore creates a new instance of DbStore with mockStore set to a provided value. If mockStore argument is nil, this function behaves exactly as NewDbStore.
func (*LDBStore) Cleanup ¶ added in v0.3.3
Cleanup iterates over the database and deletes chunks if they pass the `f` condition
func (*LDBStore) CurrentStorageIndex ¶ added in v0.3.3
func (*LDBStore) Export ¶ added in v0.3.3
Export writes all chunks from the store to a tar archive, returning the number of chunks written.
func (*LDBStore) GetSchema ¶ added in v0.3.7
GetSchema is returning the current named schema of the datastore as read from LevelDB
type LDBStoreParams ¶ added in v0.3.3
type LDBStoreParams struct { *StoreParams Path string Po func(Address) uint8 }
func NewLDBStoreParams ¶ added in v0.3.3
func NewLDBStoreParams(storeparams *StoreParams, path string) *LDBStoreParams
NewLDBStoreParams constructs LDBStoreParams with the specified values.
type LazyChunkReader ¶
type LazyChunkReader struct {
// contains filtered or unexported fields
}
LazyChunkReader implements LazySectionReader
func TreeJoin ¶ added in v0.3.3
Join reconstructs original content based on a root key. When joining, the caller gets returned a Lazy SectionReader, which is seekable and implements on-demand fetching of chunks as and where it is read. New chunks to retrieve are coming from the getter, which the caller provides. If an error is encountered during joining, it appears as a reader error. The SectionReader. As a result, partial reads from a document are possible even if other parts are corrupt or lost. The chunks are not meant to be validated by the chunker when joining. This is because it is left to the DPA to decide which sources are trusted.
func (*LazyChunkReader) Context ¶ added in v0.3.3
func (r *LazyChunkReader) Context() context.Context
func (*LazyChunkReader) Read ¶
func (r *LazyChunkReader) Read(b []byte) (read int, err error)
Read keeps a cursor so cannot be called simulateously, see ReadAt
func (*LazyChunkReader) ReadAt ¶
func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error)
read at can be called numerous times concurrent reads are allowed Size() needs to be called synchronously on the LazyChunkReader first
type LazySectionReader ¶
type LazySectionReader interface { Context() context.Context Size(context.Context, chan bool) (int64, error) io.Seeker io.Reader io.ReaderAt }
Size, Seek, Read, ReadAt
type LazyTestSectionReader ¶
type LazyTestSectionReader struct {
*io.SectionReader
}
func (*LazyTestSectionReader) Context ¶ added in v0.3.3
func (r *LazyTestSectionReader) Context() context.Context
type LocalStore ¶
type LocalStore struct { Validators []ChunkValidator DbStore *LDBStore // contains filtered or unexported fields }
LocalStore is a combination of inmemory db over a disk persisted db implements a Get/Put with fallback (caching) logic using any 2 ChunkStores
func NewLocalStore ¶
func NewLocalStore(params *LocalStoreParams, mockStore *mock.NodeStore) (*LocalStore, error)
This constructor uses MemStore and DbStore as components
func NewTestLocalStoreForAddr ¶ added in v0.3.3
func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error)
func (*LocalStore) BinIndex ¶ added in v0.3.6
func (ls *LocalStore) BinIndex(po uint8) uint64
func (*LocalStore) Get ¶
Get(chunk *Chunk) looks up a chunk in the local stores This method is blocking until the chunk is retrieved so additional timeout may be needed to wrap this call if ChunkStores are remote and can have long latency
func (*LocalStore) Migrate ¶ added in v0.3.7
func (ls *LocalStore) Migrate() error
Migrate checks the datastore schema vs the runtime schema, and runs migrations if they don't match
func (*LocalStore) Put ¶
func (ls *LocalStore) Put(ctx context.Context, chunk Chunk) error
Put is responsible for doing validation and storage of the chunk by using configured ChunkValidators, MemStore and LDBStore. If the chunk is not valid, its GetErrored function will return ErrChunkInvalid. This method will check if the chunk is already in the MemStore and it will return it if it is. If there is an error from the MemStore.Get, it will be returned by calling GetErrored on the chunk. This method is responsible for closing Chunk.ReqC channel when the chunk is stored in memstore. After the LDBStore.Put, it is ensured that the MemStore contains the chunk with the same data, but nil ReqC channel.
type LocalStoreParams ¶ added in v0.3.3
type LocalStoreParams struct { *StoreParams ChunkDbPath string Validators []ChunkValidator `toml:"-"` }
func NewDefaultLocalStoreParams ¶ added in v0.3.3
func NewDefaultLocalStoreParams() *LocalStoreParams
func (*LocalStoreParams) Init ¶ added in v0.3.3
func (p *LocalStoreParams) Init(path string)
this can only finally be set after all config options (file, cmd line, env vars) have been evaluated
type MemStore ¶
type MemStore struct {
// contains filtered or unexported fields
}
func NewMemStore ¶
func NewMemStore(params *StoreParams, _ *LDBStore) (m *MemStore)
NewMemStore is instantiating a MemStore cache keeping all frequently requested chunks in the `cache` LRU cache.
type NetFetcher ¶ added in v0.3.6
type NetStore ¶
type NetStore struct { NewNetFetcherFunc NewNetFetcherFunc // contains filtered or unexported fields }
NetStore is an extension of local storage it implements the ChunkStore interface on request it initiates remote cloud retrieval using a fetcher fetchers are unique to a chunk and are stored in fetchers LRU memory cache fetchFuncFactory is a factory object to create a fetch function for a specific chunk address
func NewNetStore ¶
func NewNetStore(store SyncChunkStore, nnf NewNetFetcherFunc) (*NetStore, error)
NewNetStore creates a new NetStore object using the given local store. newFetchFunc is a constructor function that can create a fetch function for a specific chunk address.
func (*NetStore) FetchFunc ¶ added in v0.3.6
FetchFunc returns nil if the store contains the given address. Otherwise it returns a wait function, which returns after the chunk is available or the context is done
func (*NetStore) Get ¶
Get retrieves the chunk from the NetStore DPA synchronously. It calls NetStore.get, and if the chunk is not in local Storage it calls fetch with the request, which blocks until the chunk arrived or context is done
func (*NetStore) Put ¶
Put stores a chunk in localstore, and delivers to all requestor peers using the fetcher stored in the fetchers cache
func (*NetStore) RequestsCacheLen ¶ added in v0.3.6
RequestsCacheLen returns the current number of outgoing requests stored in the cache
type NewNetFetcherFunc ¶ added in v0.3.6
type Peer ¶
type Peer interface{}
Peer is the recorded as Source on the chunk should probably not be here? but network should wrap chunk object
type Putter ¶ added in v0.3.3
type Putter interface { Put(context.Context, ChunkData) (Reference, error) // RefSize returns the length of the Reference created by this Putter RefSize() int64 // Close is to indicate that no more chunk data will be Put on this Putter Close() // Wait returns if all data has been store and the Close() was called. Wait(context.Context) error }
Putter is responsible to store data and create a reference for it
type PyramidChunker ¶
type PyramidChunker struct {
// contains filtered or unexported fields
}
func NewPyramidSplitter ¶ added in v0.3.3
func NewPyramidSplitter(params *PyramidSplitterParams) (pc *PyramidChunker)
func (*PyramidChunker) Join ¶
func (pc *PyramidChunker) Join(addr Address, getter Getter, depth int) LazySectionReader
type PyramidSplitterParams ¶ added in v0.3.3
type PyramidSplitterParams struct { SplitterParams // contains filtered or unexported fields }
func NewPyramidSplitterParams ¶ added in v0.3.3
type SplitterParams ¶ added in v0.3.3
type SplitterParams struct { ChunkerParams // contains filtered or unexported fields }
type StoreParams ¶
type StoreParams struct { Hash SwarmHasher `toml:"-"` DbCapacity uint64 CacheCapacity uint BaseKey []byte }
func NewDefaultStoreParams ¶ added in v0.3.2
func NewDefaultStoreParams() *StoreParams
func NewStoreParams ¶
func NewStoreParams(ldbCap uint64, cacheCap uint, hash SwarmHasher, basekey []byte) *StoreParams
type SwarmHasher ¶
type SwarmHasher func() SwarmHash
func MakeHashFunc ¶
func MakeHashFunc(hash string) SwarmHasher
type SyncChunkStore ¶ added in v0.3.6
type SyncChunkStore interface { ChunkStore BinIndex(po uint8) uint64 Iterator(from uint64, to uint64, po uint8, f func(Address, uint64) bool) error FetchFunc(ctx context.Context, ref Address) func(context.Context) error }
SyncChunkStore is a ChunkStore which supports syncing
type TreeChunker ¶
type TreeChunker struct {
// contains filtered or unexported fields
}
func NewTreeJoiner ¶ added in v0.3.3
func NewTreeJoiner(params *JoinerParams) *TreeChunker
func NewTreeSplitter ¶ added in v0.3.3
func NewTreeSplitter(params *TreeSplitterParams) *TreeChunker
func (*TreeChunker) Append ¶
func (tc *TreeChunker) Append() (Address, func(), error)
func (*TreeChunker) Join ¶
func (tc *TreeChunker) Join(ctx context.Context) *LazyChunkReader
type TreeEntry ¶
type TreeEntry struct {
// contains filtered or unexported fields
}
Entry to create a tree node
func NewTreeEntry ¶
func NewTreeEntry(pyramid *PyramidChunker) *TreeEntry
type TreeSplitterParams ¶ added in v0.3.3
type TreeSplitterParams struct { SplitterParams // contains filtered or unexported fields }
Source Files ¶
Directories ¶
Path | Synopsis |
---|---|
Package feeds defines Swarm Feeds.
|
Package feeds defines Swarm Feeds. |
lookup
Package lookup defines feed lookup algorithms and provides tools to place updates so they can be found
|
Package lookup defines feed lookup algorithms and provides tools to place updates so they can be found |
Package mock defines types that are used by different implementations of mock storages.
|
Package mock defines types that are used by different implementations of mock storages. |
db
Package db implements a mock store that keeps all chunk data in LevelDB database.
|
Package db implements a mock store that keeps all chunk data in LevelDB database. |
mem
Package mem implements a mock store that keeps all chunk data in memory.
|
Package mem implements a mock store that keeps all chunk data in memory. |
rpc
Package rpc implements an RPC client that connect to a centralized mock store.
|
Package rpc implements an RPC client that connect to a centralized mock store. |
test
Package test provides functions that are used for testing GlobalStorer implementations.
|
Package test provides functions that are used for testing GlobalStorer implementations. |