retriever

package
v0.8.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 9, 2023 License: Apache-2.0, MIT Imports: 48 Imported by: 3

Documentation

Index

Constants

View Source 
const BufferWindow = 5 * time.Millisecond
View Source 
const DefaultConcurrency = 6

Variables

View Source 
var (
	ErrRetrieverNotStarted         = errors.New("retriever not started")
	ErrDealProposalFailed          = errors.New("deal proposal failed")
	ErrNoCandidates                = errors.New("no candidates")
	ErrUnexpectedRetrieval         = errors.New("unexpected active retrieval")
	ErrHitRetrievalLimit           = errors.New("hit retrieval limit")
	ErrProposalCreationFailed      = errors.New("proposal creation failed")
	ErrRetrievalRegistrationFailed = errors.New("retrieval registration failed")
	ErrRetrievalFailed             = errors.New("retrieval failed")
	ErrAllRetrievalsFailed         = errors.New("all retrievals failed")
	ErrConnectFailed               = errors.New("unable to connect to provider")
	ErrAllQueriesFailed            = errors.New("all queries failed")
	ErrRetrievalTimedOut           = errors.New("retrieval timed out")
	ErrRetrievalAlreadyRunning     = errors.New("retrieval already running for CID")
)

Functions

func NewProtocolSplitter added in v0.4.0 

func NewProtocolSplitter(protocols []multicodec.Code) types.CandidateSplitter[multicodec.Code]

func RetrievalProposalForAsk 

func RetrievalProposalForAsk(ask *retrievaltypes.QueryResponse, c cid.Cid, selector ipld.Node) (*retrievaltypes.DealProposal, error)

Types

type AssignableCandidateFinder added in v0.4.0 

type AssignableCandidateFinder struct {
	// contains filtered or unexported fields
}

AssignableCandidateFinder finds and filters candidates for a given retrieval

func NewAssignableCandidateFinder added in v0.4.0 

func NewAssignableCandidateFinder(candidateFinder CandidateFinder, filterIndexerCandidate FilterIndexerCandidate) AssignableCandidateFinder

func NewAssignableCandidateFinderWithClock added in v0.6.0 

func NewAssignableCandidateFinderWithClock(candidateFinder CandidateFinder, filterIndexerCandidate FilterIndexerCandidate, clock clock.Clock) AssignableCandidateFinder

func (AssignableCandidateFinder) FindCandidates added in v0.4.0 

func (acf AssignableCandidateFinder) FindCandidates(ctx context.Context, request types.RetrievalRequest, eventsCallback func(types.RetrievalEvent), onCandidates func([]types.RetrievalCandidate)) error

type BitswapConfig added in v0.4.0 

type BitswapConfig struct {
	BlockTimeout time.Duration
	TempDir      string
	Concurrency  int
}

BitswapConfig contains configurable parameters for bitswap fetching

type BitswapRetriever added in v0.4.0 

type BitswapRetriever struct {
	// contains filtered or unexported fields
}

BitswapRetriever uses bitswap to retrieve data BitswapRetriever retieves using a combination of a go-bitswap client specially configured per retrieval, underneath a blockservice and a go-fetcher Fetcher. Selectors are used to travers the dag to make sure the CARs for bitswap match graphsync Note: this is a tradeoff over go-merkledag for traversal, cause selector execution is slow. But the solution is to improve selector execution, not introduce unpredictable encoding.

func NewBitswapRetrieverFromDeps added in v0.4.0 

func NewBitswapRetrieverFromDeps(bsrv blockservice.BlockService, routing IndexerRouting, inProgressCids InProgressCids, bstore MultiBlockstore, cfg BitswapConfig, clock clock.Clock, awaitReceivedCandidates chan<- struct{}) *BitswapRetriever

NewBitswapRetrieverFromDeps is primarily for testing, constructing behavior from direct dependencies

func NewBitswapRetrieverFromHost added in v0.4.0 

func NewBitswapRetrieverFromHost(ctx context.Context, host host.Host, cfg BitswapConfig) *BitswapRetriever

NewBitswapRetrieverFromHost constructs a new bitswap retriever for the given libp2p host

func (*BitswapRetriever) Retrieve added in v0.4.0 

func (br *BitswapRetriever) Retrieve(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.CandidateRetrieval

Retrieve initializes a new bitswap session

type CandidateCallback 

type CandidateCallback func(types.RetrievalCandidate) error

type CandidateErrorCallback 

type CandidateErrorCallback func(types.RetrievalCandidate, error)

type CandidateFinder 

type CandidateFinder interface {
	FindCandidates(context.Context, cid.Cid) ([]types.RetrievalCandidate, error)
	FindCandidatesAsync(context.Context, cid.Cid, func(types.RetrievalCandidate)) error
}

type CounterCallback 

type CounterCallback func(int) error

type DirectCandidateFinder added in v0.4.6 

type DirectCandidateFinder struct {
	// contains filtered or unexported fields
}

DirectCandidateFinder finds candidate protocols from a fixed set of peers

func NewDirectCandidateFinder added in v0.4.6 

func NewDirectCandidateFinder(h host.Host, providers []peer.AddrInfo) *DirectCandidateFinder

NewDirectCandidateFinder returns a new DirectCandidateFinder for the given providers

func (*DirectCandidateFinder) FindCandidates added in v0.4.6 

func (d *DirectCandidateFinder) FindCandidates(ctx context.Context, c cid.Cid) ([]types.RetrievalCandidate, error)

func (*DirectCandidateFinder) FindCandidatesAsync added in v0.4.6 

func (d *DirectCandidateFinder) FindCandidatesAsync(ctx context.Context, c cid.Cid, cb func(types.RetrievalCandidate)) error

FindCandidatesAsync finds supported protocols for each peer TODO: Cache the results?

type FilterIndexerCandidate added in v0.6.6 

type FilterIndexerCandidate func(types.RetrievalCandidate) (bool, types.RetrievalCandidate)

type GetStorageProviderTimeout 

type GetStorageProviderTimeout func(peer peer.ID) time.Duration

type GraphSyncRetriever added in v0.4.0 

type GraphSyncRetriever struct {
	GetStorageProviderTimeout GetStorageProviderTimeout
	Client                    RetrievalClient
	Clock                     clock.Clock
	QueueInitialPause         time.Duration
}

func NewGraphsyncRetriever added in v0.7.0 

func NewGraphsyncRetriever(getStorageProviderTimeout GetStorageProviderTimeout, client RetrievalClient) *GraphSyncRetriever

func (*GraphSyncRetriever) Retrieve added in v0.4.0 

func (cfg *GraphSyncRetriever) Retrieve(
	ctx context.Context,
	retrievalRequest types.RetrievalRequest,
	eventsCallback func(types.RetrievalEvent),
) types.CandidateRetrieval

RetrieveFromCandidates performs a retrieval for a given CID by querying the indexer, then attempting to query all candidates and attempting to perform a full retrieval from the best and fastest storage provider as the queries are received.

type InProgressCids added in v0.4.0 

type InProgressCids interface {
	Inc(cid.Cid, types.RetrievalID)
	Dec(cid.Cid, types.RetrievalID)
}

type IndexerRouting added in v0.4.0 

type IndexerRouting interface {
	AddProviders(types.RetrievalID, []types.RetrievalCandidate)
	RemoveProviders(types.RetrievalID)
}

IndexerRouting are the required methods to track indexer routing

type MultiBlockstore added in v0.4.0 

type MultiBlockstore interface {
	AddLinkSystem(id types.RetrievalID, lsys *linking.LinkSystem) error
	RemoveLinkSystem(id types.RetrievalID)
}

MultiBlockstore are the require methods to track linksystems

type ProtocolSplitter added in v0.4.0 

type ProtocolSplitter struct {
	// contains filtered or unexported fields
}

func (*ProtocolSplitter) SplitRetrievalRequest added in v0.6.0 

func (ps *ProtocolSplitter) SplitRetrievalRequest(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.RetrievalSplitter[multicodec.Code]

type RetrievalClient 

type RetrievalClient interface {
	Connect(ctx context.Context, peerAddr peer.AddrInfo) error
	RetrieveFromPeer(
		ctx context.Context,
		linkSystem ipld.LinkSystem,
		peerID peer.ID,
		proposal *retrievaltypes.DealProposal,
		selector ipld.Node,
		maxLinks uint64,
		eventsCallback datatransfer.Subscriber,
		gracefulShutdownRequested <-chan struct{},
	) (*types.RetrievalStats, error)
}

type Retriever 

type Retriever struct {
	// contains filtered or unexported fields
}

func NewRetriever 

func NewRetriever(
	ctx context.Context,
	session Session,
	candidateFinder CandidateFinder,
	protocolRetrievers map[multicodec.Code]types.CandidateRetriever,
) (*Retriever, error)

func NewRetrieverWithClock added in v0.7.0 

func NewRetrieverWithClock(
	ctx context.Context,
	session Session,
	candidateFinder CandidateFinder,
	protocolRetrievers map[multicodec.Code]types.CandidateRetriever,
	clock clock.Clock,
) (*Retriever, error)

func (*Retriever) RegisterSubscriber added in v0.3.0 

func (retriever *Retriever) RegisterSubscriber(subscriber types.RetrievalEventSubscriber) func()

RegisterSubscriber registers a subscriber to receive all events fired during the process of making a retrieval, including the process of querying available storage providers to find compatible ones to attempt retrieval from.

func (*Retriever) Retrieve 

func (retriever *Retriever) Retrieve(
	ctx context.Context,
	request types.RetrievalRequest,
	eventsCB func(types.RetrievalEvent),
) (*types.RetrievalStats, error)

Retrieve attempts to retrieve the given CID using the configured CandidateFinder to find storage providers that should have the CID.

func (*Retriever) Start added in v0.3.0 

func (retriever *Retriever) Start()

Start will start the retriever events system

func (*Retriever) Stop added in v0.3.0 

func (retriever *Retriever) Stop() chan struct{}

Stop will stop the retriever events system and return a channel that will be closed when shutdown has completed

type Session added in v0.6.8 

type Session interface {
	RegisterRetrieval(retrievalId types.RetrievalID, cid cid.Cid, selector datamodel.Node) bool
	EndRetrieval(retrievalId types.RetrievalID) error
	AddToRetrieval(retrievalId types.RetrievalID, storageProviderIds []peer.ID) error
	RecordFailure(storageProviderId peer.ID, retrievalId types.RetrievalID) error
	FilterIndexerCandidate(candidate types.RetrievalCandidate) (bool, types.RetrievalCandidate)
}

type TimeCounter added in v0.6.4 

type TimeCounter struct {
	// contains filtered or unexported fields
}

timeCounter is used to generate a monotonically increasing sequence. It starts at the current time, then increments on each call to next.

func NewTimeCounter added in v0.6.4 

func NewTimeCounter() *TimeCounter

func (*TimeCounter) Next added in v0.6.4 

func (tc *TimeCounter) Next() uint64

Source Files

View all Source files

Directories

Path Synopsis
Package combinators contains tools to put various retrieval components together to make full retrievers
 Package combinators contains tools to put various retrieval components together to make full retrievers
Package coordinators contains retrievers that coordinate multiple child retrievers
 Package coordinators contains retrievers that coordinate multiple child retrievers
Package prioritywaitqueue implements a blocking queue for prioritised coordination of goroutine execution.
 Package prioritywaitqueue implements a blocking queue for prioritised coordination of goroutine execution.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.