Documentation ¶
Index ¶
- Constants
- Variables
- func NewGraphsyncRetriever(session Session, client GraphsyncClient) types.CandidateRetriever
- func NewGraphsyncRetrieverWithConfig(session Session, client GraphsyncClient, clock clock.Clock, ...) types.CandidateRetriever
- func NewHttpRetriever(session Session, client *http.Client) types.CandidateRetriever
- func NewHttpRetrieverWithDeps(session Session, client *http.Client, clock clock.Clock, ...) types.CandidateRetriever
- func NewProtocolSplitter(protocols []multicodec.Code) types.CandidateSplitter[multicodec.Code]
- func RetrievalProposalForAsk(ask *retrievaltypes.QueryResponse, c cid.Cid, selector ipld.Node) (*retrievaltypes.DealProposal, error)
- type AssignableCandidateFinder
- type BitswapConfig
- type BitswapRetriever
- type CandidateFinder
- type DirectCandidateFinder
- type ErrHttpRequestFailure
- type FilterIndexerCandidate
- type GetStorageProviderTimeout
- type GraphsyncClient
- type InProgressCids
- type IndexerRouting
- type MultiBlockstore
- type ProtocolGraphsync
- func (pg ProtocolGraphsync) Code() multicodec.Code
- func (pg *ProtocolGraphsync) Connect(ctx context.Context, retrieval *retrieval, candidate types.RetrievalCandidate) (time.Duration, error)
- func (pg ProtocolGraphsync) GetMergedMetadata(cid cid.Cid, currentMetadata, newMetadata metadata.Protocol) metadata.Protocol
- func (pg *ProtocolGraphsync) Retrieve(ctx context.Context, retrieval *retrieval, shared *retrievalShared, ...) (*types.RetrievalStats, error)
- type ProtocolHttp
- func (ph ProtocolHttp) Code() multicodec.Code
- func (ph *ProtocolHttp) Connect(ctx context.Context, retrieval *retrieval, candidate types.RetrievalCandidate) (time.Duration, error)
- func (ph ProtocolHttp) GetMergedMetadata(cid cid.Cid, currentMetadata, newMetadata metadata.Protocol) metadata.Protocol
- func (ph *ProtocolHttp) Retrieve(ctx context.Context, retrieval *retrieval, shared *retrievalShared, ...) (*types.RetrievalStats, error)
- type ProtocolSplitter
- type Retriever
- func (retriever *Retriever) RegisterSubscriber(subscriber types.RetrievalEventSubscriber) func()
- func (retriever *Retriever) Retrieve(ctx context.Context, request types.RetrievalRequest, ...) (*types.RetrievalStats, error)
- func (retriever *Retriever) Start()
- func (retriever *Retriever) Stop() chan struct{}
- type Session
- type TimeCounter
- type TransportProtocol
Constants ¶
const BufferWindow = 5 * time.Millisecond
const GraphsyncDefaultInitialWait = 2 * time.Millisecond
Connect() may be a near-noop for already-connect libp2p connections, so this allows parallel goroutines of already-connected peers to queue and have the scoring logic to select one to start.
const HttpDefaultInitialWait time.Duration = 2 * time.Millisecond
Connect() is currently a noop, so this simply allows parallel goroutines to queue and the scoring logic to select one to start.
Variables ¶
var ( ErrHttpSelectorRequest = errors.New("HTTP retrieval for an explicit selector request") ErrNoHttpForPeer = errors.New("no HTTP url for peer") ErrBadPathForRequest = errors.New("bad path for request") )
var ( ErrRetrieverNotStarted = errors.New("retriever not started") ErrDealProposalFailed = errors.New("deal proposal failed") ErrNoCandidates = errors.New("no candidates") ErrUnexpectedRetrieval = errors.New("unexpected active retrieval") ErrHitRetrievalLimit = errors.New("hit retrieval limit") ErrProposalCreationFailed = errors.New("proposal creation failed") ErrRetrievalRegistrationFailed = errors.New("retrieval registration failed") ErrRetrievalFailed = errors.New("retrieval failed") ErrAllRetrievalsFailed = errors.New("all retrievals failed") ErrConnectFailed = errors.New("unable to connect to provider") ErrAllQueriesFailed = errors.New("all queries failed") ErrRetrievalTimedOut = errors.New("retrieval timed out") ErrRetrievalAlreadyRunning = errors.New("retrieval already running for CID") )
Functions ¶
func NewGraphsyncRetriever ¶ added in v0.7.0
func NewGraphsyncRetriever(session Session, client GraphsyncClient) types.CandidateRetriever
NewGraphsyncRetriever makes a new CandidateRetriever for Graphsync retrievals (transport-graphsync-filecoinv1).
func NewGraphsyncRetrieverWithConfig ¶ added in v0.10.0
func NewGraphsyncRetrieverWithConfig( session Session, client GraphsyncClient, clock clock.Clock, initialPause time.Duration, noDirtyClose bool, ) types.CandidateRetriever
func NewHttpRetriever ¶ added in v0.10.0
func NewHttpRetriever(session Session, client *http.Client) types.CandidateRetriever
NewHttpRetriever makes a new CandidateRetriever for verified CAR HTTP retrievals (transport-ipfs-gateway-http).
func NewHttpRetrieverWithDeps ¶ added in v0.10.0
func NewProtocolSplitter ¶ added in v0.4.0
func NewProtocolSplitter(protocols []multicodec.Code) types.CandidateSplitter[multicodec.Code]
func RetrievalProposalForAsk ¶
func RetrievalProposalForAsk(ask *retrievaltypes.QueryResponse, c cid.Cid, selector ipld.Node) (*retrievaltypes.DealProposal, error)
Types ¶
type AssignableCandidateFinder ¶ added in v0.4.0
type AssignableCandidateFinder struct {
// contains filtered or unexported fields
}
AssignableCandidateFinder finds and filters candidates for a given retrieval
func NewAssignableCandidateFinder ¶ added in v0.4.0
func NewAssignableCandidateFinder(candidateFinder CandidateFinder, filterIndexerCandidate FilterIndexerCandidate) AssignableCandidateFinder
func NewAssignableCandidateFinderWithClock ¶ added in v0.6.0
func NewAssignableCandidateFinderWithClock(candidateFinder CandidateFinder, filterIndexerCandidate FilterIndexerCandidate, clock clock.Clock) AssignableCandidateFinder
func (AssignableCandidateFinder) FindCandidates ¶ added in v0.4.0
func (acf AssignableCandidateFinder) FindCandidates(ctx context.Context, request types.RetrievalRequest, eventsCallback func(types.RetrievalEvent), onCandidates func([]types.RetrievalCandidate)) error
type BitswapConfig ¶ added in v0.4.0
type BitswapConfig struct { BlockTimeout time.Duration Concurrency int ConcurrencyPerRetrieval int }
BitswapConfig contains configurable parameters for bitswap fetching
type BitswapRetriever ¶ added in v0.4.0
type BitswapRetriever struct {
// contains filtered or unexported fields
}
BitswapRetriever uses bitswap to retrieve data BitswapRetriever retieves using a combination of a go-bitswap client specially configured per retrieval, underneath a blockservice and a go-fetcher Fetcher. Selectors are used to travers the dag to make sure the CARs for bitswap match graphsync Note: this is a tradeoff over go-merkledag for traversal, cause selector execution is slow. But the solution is to improve selector execution, not introduce unpredictable encoding.
func NewBitswapRetrieverFromDeps ¶ added in v0.4.0
func NewBitswapRetrieverFromDeps( ctx context.Context, bsrv blockservice.BlockService, routing IndexerRouting, inProgressCids InProgressCids, bstore MultiBlockstore, cfg BitswapConfig, clock clock.Clock, awaitReceivedCandidates chan<- struct{}, ) *BitswapRetriever
NewBitswapRetrieverFromDeps is primarily for testing, constructing behavior from direct dependencies
func NewBitswapRetrieverFromHost ¶ added in v0.4.0
func NewBitswapRetrieverFromHost( ctx context.Context, host host.Host, cfg BitswapConfig, ) *BitswapRetriever
NewBitswapRetrieverFromHost constructs a new bitswap retriever for the given libp2p host
func (*BitswapRetriever) Retrieve ¶ added in v0.4.0
func (br *BitswapRetriever) Retrieve( ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent), ) types.CandidateRetrieval
Retrieve initializes a new bitswap session
type CandidateFinder ¶
type CandidateFinder interface { FindCandidates(context.Context, cid.Cid) ([]types.RetrievalCandidate, error) FindCandidatesAsync(context.Context, cid.Cid, func(types.RetrievalCandidate)) error }
type DirectCandidateFinder ¶ added in v0.4.6
type DirectCandidateFinder struct {
// contains filtered or unexported fields
}
DirectCandidateFinder finds candidate protocols from a fixed set of peers
func NewDirectCandidateFinder ¶ added in v0.4.6
func NewDirectCandidateFinder(h host.Host, providers []peer.AddrInfo) *DirectCandidateFinder
NewDirectCandidateFinder returns a new DirectCandidateFinder for the given providers
func (*DirectCandidateFinder) FindCandidates ¶ added in v0.4.6
func (d *DirectCandidateFinder) FindCandidates(ctx context.Context, c cid.Cid) ([]types.RetrievalCandidate, error)
func (*DirectCandidateFinder) FindCandidatesAsync ¶ added in v0.4.6
func (d *DirectCandidateFinder) FindCandidatesAsync(ctx context.Context, c cid.Cid, cb func(types.RetrievalCandidate)) error
FindCandidatesAsync finds supported protocols for each peer TODO: Cache the results?
type ErrHttpRequestFailure ¶ added in v0.11.0
type ErrHttpRequestFailure struct {
Code int
}
func (ErrHttpRequestFailure) Error ¶ added in v0.11.0
func (e ErrHttpRequestFailure) Error() string
type FilterIndexerCandidate ¶ added in v0.6.6
type FilterIndexerCandidate func(types.RetrievalCandidate) (bool, types.RetrievalCandidate)
type GraphsyncClient ¶ added in v0.10.0
type GraphsyncClient interface { Connect(ctx context.Context, peerAddr peer.AddrInfo) error RetrieveFromPeer( ctx context.Context, linkSystem ipld.LinkSystem, peerID peer.ID, proposal *retrievaltypes.DealProposal, selector ipld.Node, maxLinks uint64, eventsCallback datatransfer.Subscriber, gracefulShutdownRequested <-chan struct{}, ) (*types.RetrievalStats, error) }
type InProgressCids ¶ added in v0.4.0
type InProgressCids interface { Inc(cid.Cid, types.RetrievalID) Dec(cid.Cid, types.RetrievalID) }
type IndexerRouting ¶ added in v0.4.0
type IndexerRouting interface { AddProviders(types.RetrievalID, []types.RetrievalCandidate) RemoveProviders(types.RetrievalID) }
IndexerRouting are the required methods to track indexer routing
type MultiBlockstore ¶ added in v0.4.0
type MultiBlockstore interface { AddLinkSystem(id types.RetrievalID, lsys *linking.LinkSystem) error RemoveLinkSystem(id types.RetrievalID) }
MultiBlockstore are the require methods to track linksystems
type ProtocolGraphsync ¶ added in v0.10.0
type ProtocolGraphsync struct { Client GraphsyncClient Clock clock.Clock }
func (ProtocolGraphsync) Code ¶ added in v0.10.0
func (pg ProtocolGraphsync) Code() multicodec.Code
func (*ProtocolGraphsync) Connect ¶ added in v0.10.0
func (pg *ProtocolGraphsync) Connect(ctx context.Context, retrieval *retrieval, candidate types.RetrievalCandidate) (time.Duration, error)
func (ProtocolGraphsync) GetMergedMetadata ¶ added in v0.10.0
func (pg ProtocolGraphsync) GetMergedMetadata(cid cid.Cid, currentMetadata, newMetadata metadata.Protocol) metadata.Protocol
func (*ProtocolGraphsync) Retrieve ¶ added in v0.10.0
func (pg *ProtocolGraphsync) Retrieve( ctx context.Context, retrieval *retrieval, shared *retrievalShared, timeout time.Duration, candidate types.RetrievalCandidate, ) (*types.RetrievalStats, error)
type ProtocolHttp ¶ added in v0.10.0
func (ProtocolHttp) Code ¶ added in v0.10.0
func (ph ProtocolHttp) Code() multicodec.Code
func (*ProtocolHttp) Connect ¶ added in v0.10.0
func (ph *ProtocolHttp) Connect(ctx context.Context, retrieval *retrieval, candidate types.RetrievalCandidate) (time.Duration, error)
func (ProtocolHttp) GetMergedMetadata ¶ added in v0.10.0
func (ph ProtocolHttp) GetMergedMetadata(cid cid.Cid, currentMetadata, newMetadata metadata.Protocol) metadata.Protocol
func (*ProtocolHttp) Retrieve ¶ added in v0.10.0
func (ph *ProtocolHttp) Retrieve( ctx context.Context, retrieval *retrieval, shared *retrievalShared, timeout time.Duration, candidate types.RetrievalCandidate, ) (*types.RetrievalStats, error)
type ProtocolSplitter ¶ added in v0.4.0
type ProtocolSplitter struct {
// contains filtered or unexported fields
}
func (*ProtocolSplitter) SplitRetrievalRequest ¶ added in v0.6.0
func (ps *ProtocolSplitter) SplitRetrievalRequest(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.RetrievalSplitter[multicodec.Code]
type Retriever ¶
type Retriever struct {
// contains filtered or unexported fields
}
func NewRetriever ¶
func NewRetriever( ctx context.Context, session Session, candidateFinder CandidateFinder, protocolRetrievers map[multicodec.Code]types.CandidateRetriever, ) (*Retriever, error)
func NewRetrieverWithClock ¶ added in v0.7.0
func NewRetrieverWithClock( ctx context.Context, session Session, candidateFinder CandidateFinder, protocolRetrievers map[multicodec.Code]types.CandidateRetriever, clock clock.Clock, ) (*Retriever, error)
func (*Retriever) RegisterSubscriber ¶ added in v0.3.0
func (retriever *Retriever) RegisterSubscriber(subscriber types.RetrievalEventSubscriber) func()
RegisterSubscriber registers a subscriber to receive all events fired during the process of making a retrieval, including the process of querying available storage providers to find compatible ones to attempt retrieval from.
func (*Retriever) Retrieve ¶
func (retriever *Retriever) Retrieve( ctx context.Context, request types.RetrievalRequest, eventsCB func(types.RetrievalEvent), ) (*types.RetrievalStats, error)
Retrieve attempts to retrieve the given CID using the configured CandidateFinder to find storage providers that should have the CID.
type Session ¶ added in v0.6.8
type Session interface { GetStorageProviderTimeout(storageProviderId peer.ID) time.Duration FilterIndexerCandidate(candidate types.RetrievalCandidate) (bool, types.RetrievalCandidate) RegisterRetrieval(retrievalId types.RetrievalID, cid cid.Cid, selector datamodel.Node) bool AddToRetrieval(retrievalId types.RetrievalID, storageProviderIds []peer.ID) error EndRetrieval(retrievalId types.RetrievalID) error RecordConnectTime(storageProviderId peer.ID, connectTime time.Duration) RecordFirstByteTime(storageProviderId peer.ID, firstByteTime time.Duration) RecordFailure(retrievalId types.RetrievalID, storageProviderId peer.ID) error RecordSuccess(storageProviderId peer.ID, bandwidthBytesPerSecond uint64) ChooseNextProvider(peers []peer.ID, metadata []metadata.Protocol) int }
type TimeCounter ¶ added in v0.6.4
type TimeCounter struct {
// contains filtered or unexported fields
}
timeCounter is used to generate a monotonically increasing sequence. It starts at the current time, then increments on each call to next.
func NewTimeCounter ¶ added in v0.6.4
func NewTimeCounter() *TimeCounter
func (*TimeCounter) Next ¶ added in v0.6.4
func (tc *TimeCounter) Next() uint64
type TransportProtocol ¶ added in v0.10.0
type TransportProtocol interface { Code() multicodec.Code GetMergedMetadata(cid cid.Cid, currentMetadata, newMetadata metadata.Protocol) metadata.Protocol Connect(ctx context.Context, retrieval *retrieval, candidate types.RetrievalCandidate) (time.Duration, error) Retrieve( ctx context.Context, retrieval *retrieval, shared *retrievalShared, timeout time.Duration, candidate types.RetrievalCandidate, ) (*types.RetrievalStats, error) }
TransportProtocol implements the protocol-specific portions of a parallel- peer retriever. It is responsible for communicating with individual peers and also bears responsibility for some of the peer-selection logic.
Source Files ¶
Directories ¶
Path | Synopsis |
---|---|
Package combinators contains tools to put various retrieval components together to make full retrievers
|
Package combinators contains tools to put various retrieval components together to make full retrievers |
Package coordinators contains retrievers that coordinate multiple child retrievers
|
Package coordinators contains retrievers that coordinate multiple child retrievers |
Package prioritywaitqueue implements a blocking queue for prioritised coordination of goroutine execution.
|
Package prioritywaitqueue implements a blocking queue for prioritised coordination of goroutine execution. |