retriever

package
v0.6.5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 10, 2023 License: Apache-2.0, MIT Imports: 45 Imported by: 3

Documentation

Index

Constants

View Source 
const BufferWindow = 5 * time.Millisecond

Variables

View Source 
var (
	ErrRetrieverNotStarted         = errors.New("retriever not started")
	ErrDealProposalFailed          = errors.New("deal proposal failed")
	ErrNoCandidates                = errors.New("no candidates")
	ErrUnexpectedRetrieval         = errors.New("unexpected active retrieval")
	ErrHitRetrievalLimit           = errors.New("hit retrieval limit")
	ErrProposalCreationFailed      = errors.New("proposal creation failed")
	ErrRetrievalRegistrationFailed = errors.New("retrieval registration failed")
	ErrRetrievalFailed             = errors.New("retrieval failed")
	ErrAllRetrievalsFailed         = errors.New("all retrievals failed")
	ErrQueryFailed                 = errors.New("query failed")
	ErrAllQueriesFailed            = errors.New("all queries failed")
	ErrRetrievalTimedOut           = errors.New("retrieval timed out")
	ErrRetrievalAlreadyRunning     = errors.New("retrieval already running for CID")
)

Functions

func NewProtocolSplitter added in v0.4.0 

func NewProtocolSplitter(protocols []multicodec.Code) types.CandidateSplitter[multicodec.Code]

func RetrievalProposalForAsk 

func RetrievalProposalForAsk(ask *retrievaltypes.QueryResponse, c cid.Cid, optionalSelector ipld.Node) (*retrievaltypes.DealProposal, error)

Types

type AssignableCandidateFinder added in v0.4.0 

type AssignableCandidateFinder struct {
	// contains filtered or unexported fields
}

AssignableCandidateFinder finds and filters candidates for a given retrieval

func NewAssignableCandidateFinder added in v0.4.0 

func NewAssignableCandidateFinder(candidateFinder CandidateFinder, isAcceptableStorageProvider IsAcceptableStorageProvider) AssignableCandidateFinder

func NewAssignableCandidateFinderWithClock added in v0.6.0 

func NewAssignableCandidateFinderWithClock(candidateFinder CandidateFinder, isAcceptableStorageProvider IsAcceptableStorageProvider, clock clock.Clock) AssignableCandidateFinder

func (AssignableCandidateFinder) FindCandidates added in v0.4.0 

func (acf AssignableCandidateFinder) FindCandidates(ctx context.Context, request types.RetrievalRequest, eventsCallback func(types.RetrievalEvent), onCandidates func([]types.RetrievalCandidate)) error

type BitswapConfig added in v0.4.0 

type BitswapConfig struct {
	BlockTimeout time.Duration
}

BitswapConfig contains configurable parameters for bitswap fetching

type BitswapRetriever added in v0.4.0 

type BitswapRetriever struct {
	// contains filtered or unexported fields
}

BitswapRetriever uses bitswap to retrieve data BitswapRetriever retieves using a combination of a go-bitswap client specially configured per retrieval, underneath a blockservice and a go-fetcher Fetcher. Selectors are used to travers the dag to make sure the CARs for bitswap match graphsync Note: this is a tradeoff over go-merkledag for traversal, cause selector execution is slow. But the solution is to improve selector execution, not introduce unpredictable encoding.

func NewBitswapRetrieverFromDeps added in v0.4.0 

func NewBitswapRetrieverFromDeps(bsrv blockservice.BlockService, routing IndexerRouting, inProgressCids InProgressCids, bstore MultiBlockstore, cfg BitswapConfig, clock clock.Clock, awaitReceivedCandidates chan<- struct{}) *BitswapRetriever

NewBitswapRetrieverFromDeps is primarily for testing, constructing behavior from direct dependencies

func NewBitswapRetrieverFromHost added in v0.4.0 

func NewBitswapRetrieverFromHost(ctx context.Context, host host.Host, cfg BitswapConfig) *BitswapRetriever

NewBitswapRetrieverFromHost constructs a new bitswap retriever for the given libp2p host

func (*BitswapRetriever) Retrieve added in v0.4.0 

func (br *BitswapRetriever) Retrieve(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.CandidateRetrieval

Retrieve initializes a new bitswap session

type CandidateCallback 

type CandidateCallback func(types.RetrievalCandidate) error

type CandidateErrorCallback 

type CandidateErrorCallback func(types.RetrievalCandidate, error)

type CandidateFinder 

type CandidateFinder interface {
	FindCandidates(context.Context, cid.Cid) ([]types.RetrievalCandidate, error)
	FindCandidatesAsync(context.Context, cid.Cid) (<-chan types.FindCandidatesResult, error)
}

type CounterCallback 

type CounterCallback func(int) error

type DirectCandidateFinder added in v0.4.6 

type DirectCandidateFinder struct {
	// contains filtered or unexported fields
}

DirectCandidateFinder finds candidate protocols from a fixed set of peers

func NewDirectCandidateFinder added in v0.4.6 

func NewDirectCandidateFinder(h host.Host, providers []peer.AddrInfo) *DirectCandidateFinder

NewDirectCandidateFinder returns a new DirectCandidateFinder for the given providers

func (*DirectCandidateFinder) FindCandidates added in v0.4.6 

func (d *DirectCandidateFinder) FindCandidates(ctx context.Context, c cid.Cid) ([]types.RetrievalCandidate, error)

func (*DirectCandidateFinder) FindCandidatesAsync added in v0.4.6 

func (d *DirectCandidateFinder) FindCandidatesAsync(ctx context.Context, c cid.Cid) (<-chan types.FindCandidatesResult, error)

FindCandidatesAsync finds supported protocols for each peer TODO: Cache the results?

type GetStorageProviderTimeout 

type GetStorageProviderTimeout func(peer peer.ID) time.Duration

type GraphSyncRetriever added in v0.4.0 

type GraphSyncRetriever struct {
	GetStorageProviderTimeout GetStorageProviderTimeout
	IsAcceptableQueryResponse IsAcceptableQueryResponse
	Client                    RetrievalClient
}

func (*GraphSyncRetriever) Retrieve added in v0.4.0 

func (cfg *GraphSyncRetriever) Retrieve(
	ctx context.Context,
	retrievalRequest types.RetrievalRequest,
	eventsCallback func(types.RetrievalEvent),
) types.CandidateRetrieval

RetrieveFromCandidates performs a retrieval for a given CID by querying the indexer, then attempting to query all candidates and attempting to perform a full retrieval from the best and fastest storage provider as the queries are received.

type InProgressCids added in v0.4.0 

type InProgressCids interface {
	Inc(cid.Cid, types.RetrievalID)
	Dec(cid.Cid, types.RetrievalID)
}

type IndexerRouting added in v0.4.0 

type IndexerRouting interface {
	AddProviders(types.RetrievalID, []types.RetrievalCandidate)
	RemoveProviders(types.RetrievalID)
}

IndexerRouting are the required methods to track indexer routing

type IsAcceptableQueryResponse 

type IsAcceptableQueryResponse func(peer peer.ID, req types.RetrievalRequest, queryResponse *retrievaltypes.QueryResponse) bool

type IsAcceptableStorageProvider 

type IsAcceptableStorageProvider func(peer peer.ID) bool

type MinerConfig 

type MinerConfig struct {
	RetrievalTimeout        time.Duration
	MaxConcurrentRetrievals uint
}

type MultiBlockstore added in v0.4.0 

type MultiBlockstore interface {
	AddLinkSystem(id types.RetrievalID, lsys *linking.LinkSystem) error
	RemoveLinkSystem(id types.RetrievalID)
}

MultiBlockstore are the require methods to track linksystems

type ProtocolSplitter added in v0.4.0 

type ProtocolSplitter struct {
	// contains filtered or unexported fields
}

func (*ProtocolSplitter) SplitRetrievalRequest added in v0.6.0 

func (ps *ProtocolSplitter) SplitRetrievalRequest(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.RetrievalSplitter[multicodec.Code]

type RetrievalClient 

type RetrievalClient interface {
	RetrievalQueryToPeer(
		ctx context.Context,
		minerPeer peer.AddrInfo,
		cid cid.Cid,
		onConnected func(),
	) (*retrievaltypes.QueryResponse, error)

	RetrieveFromPeer(
		ctx context.Context,
		linkSystem ipld.LinkSystem,
		peerID peer.ID,
		minerWallet address.Address,
		proposal *retrievaltypes.DealProposal,
		selector ipld.Node,
		eventsCallback datatransfer.Subscriber,
		gracefulShutdownRequested <-chan struct{},
	) (*types.RetrievalStats, error)
}

type Retriever 

type Retriever struct {
	// contains filtered or unexported fields
}

func NewRetriever 

func NewRetriever(
	ctx context.Context,
	config RetrieverConfig,
	client RetrievalClient,
	candidateFinder CandidateFinder,
	bitswapRetriever types.CandidateRetriever,
) (*Retriever, error)

func (*Retriever) RegisterSubscriber added in v0.3.0 

func (retriever *Retriever) RegisterSubscriber(subscriber types.RetrievalEventSubscriber) func()

RegisterSubscriber registers a subscriber to receive all events fired during the process of making a retrieval, including the process of querying available storage providers to find compatible ones to attempt retrieval from.

func (*Retriever) Retrieve 

func (retriever *Retriever) Retrieve(
	ctx context.Context,
	request types.RetrievalRequest,
	eventsCB func(types.RetrievalEvent),
) (*types.RetrievalStats, error)

Retrieve attempts to retrieve the given CID using the configured CandidateFinder to find storage providers that should have the CID.

func (*Retriever) Start added in v0.3.0 

func (retriever *Retriever) Start()

Start will start the retriever events system

func (*Retriever) Stop added in v0.3.0 

func (retriever *Retriever) Stop() chan struct{}

Stop will stop the retriever events system and return a channel that will be closed when shutdown has completed

type RetrieverConfig 

type RetrieverConfig struct {
	MinerBlacklist     map[peer.ID]bool
	MinerWhitelist     map[peer.ID]bool
	DefaultMinerConfig MinerConfig
	MinerConfigs       map[peer.ID]MinerConfig
	PaidRetrievals     bool
	DisableGraphsync   bool
}

All config values should be safe to leave uninitialized

type TimeCounter added in v0.6.4 

type TimeCounter struct {
	// contains filtered or unexported fields
}

timeCounter is used to generate a monotonically increasing sequence. It starts at the current time, then increments on each call to next.

func NewTimeCounter added in v0.6.4 

func NewTimeCounter() *TimeCounter

func (*TimeCounter) Next added in v0.6.4 

func (tc *TimeCounter) Next() uint64

Source Files

View all Source files

Directories

Path Synopsis
Package combinators contains tools to put various retrieval components together to make full retrievers
 Package combinators contains tools to put various retrieval components together to make full retrievers
Package coordinators contains retrievers that coordinate multiple child retrievers
 Package coordinators contains retrievers that coordinate multiple child retrievers

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.