retriever

Jan 27, 2024

Changes in this version

+ type DirectCandidateSource struct

+ func NewDirectCandidateSource(providers []types.Provider, opts ...Option) *DirectCandidateSource

+ func (d *DirectCandidateSource) FindCandidates(ctx context.Context, c cid.Cid, cb func(types.RetrievalCandidate)) error

+ type Option func(*DirectCandidateSource)

+ func WithLibp2pCandidateDiscovery(h host.Host) Option

Sep 11, 2023

Changes in this version

type BitswapConfig

+ ConcurrencyPerRetrieval int

Jun 5, 2023

Changes in this version

type ProtocolHttp

+ Clock clock.Clock

type Session

+ ChooseNextProvider func(peers []peer.ID, metadata []metadata.Protocol) int

+ RecordConnectTime func(storageProviderId peer.ID, connectTime time.Duration)

+ RecordFirstByteTime func(storageProviderId peer.ID, firstByteTime time.Duration)

+ RecordSuccess func(storageProviderId peer.ID, bandwidthBytesPerSecond uint64)

May 25, 2023

Changes in this version

+ const DefaultUserAgent

+ type ErrHttpRequestFailure struct

+ Code int

+ func (e ErrHttpRequestFailure) Error() string

type ProtocolGraphsync

+ Clock clock.Clock

type Session

+ CompareStorageProviders func(protocol multicodec.Code, a, b peer.ID, mda, mdb metadata.Protocol) bool

+ GetStorageProviderTimeout func(storageProviderId peer.ID) time.Duration

+ RegisterConnectTime func(storageProviderId peer.ID, connectTime time.Duration)

May 15, 2023

Changes in this version

+ const GraphsyncDefaultInitialWait

+ const HttpDefaultInitialWait

+ var ErrBadPathForRequest = errors.New("bad path for request")

+ var ErrHttpSelectorRequest = errors.New("HTTP retrieval for an explicit selector request")

+ var ErrNoHttpForPeer = errors.New("no HTTP url for peer")

+ func NewGraphsyncRetrieverWithConfig(getStorageProviderTimeout GetStorageProviderTimeout, client GraphsyncClient, ...) types.CandidateRetriever

+ func NewHttpRetriever(getStorageProviderTimeout GetStorageProviderTimeout, client *http.Client) types.CandidateRetriever

+ func NewHttpRetrieverWithDeps(getStorageProviderTimeout GetStorageProviderTimeout, client *http.Client, ...) types.CandidateRetriever

+ type ComparableCandidate struct

+ Duration time.Duration

+ PeerID peer.ID

+ type GraphsyncClient interface

+ Connect func(ctx context.Context, peerAddr peer.AddrInfo) error

+ RetrieveFromPeer func(ctx context.Context, linkSystem ipld.LinkSystem, peerID peer.ID, ...) (*types.RetrievalStats, error)

+ type ProtocolGraphsync struct

+ Client GraphsyncClient

+ func (pg *ProtocolGraphsync) Connect(ctx context.Context, retrieval *retrieval, candidate types.RetrievalCandidate) error

+ func (pg *ProtocolGraphsync) Retrieve(ctx context.Context, retrieval *retrieval, session *retrievalSession, ...) (*types.RetrievalStats, error)

+ func (pg ProtocolGraphsync) Code() multicodec.Code

+ func (pg ProtocolGraphsync) CompareCandidates(a, b ComparableCandidate, mda, mdb metadata.Protocol) bool

+ func (pg ProtocolGraphsync) GetMergedMetadata(cid cid.Cid, currentMetadata, newMetadata metadata.Protocol) metadata.Protocol

+ type ProtocolHttp struct

+ Client *http.Client

+ func (ph *ProtocolHttp) Connect(ctx context.Context, retrieval *retrieval, candidate types.RetrievalCandidate) error

+ func (ph *ProtocolHttp) Retrieve(ctx context.Context, retrieval *retrieval, session *retrievalSession, ...) (*types.RetrievalStats, error)

+ func (ph ProtocolHttp) Code() multicodec.Code

+ func (ph ProtocolHttp) CompareCandidates(a, b ComparableCandidate, mda, mdb metadata.Protocol) bool

+ func (ph ProtocolHttp) GetMergedMetadata(cid cid.Cid, currentMetadata, newMetadata metadata.Protocol) metadata.Protocol

+ type TransportProtocol interface

+ Code func() multicodec.Code

+ CompareCandidates func(a, b ComparableCandidate, mda, mdb metadata.Protocol) bool

+ Connect func(ctx context.Context, retrieval *retrieval, candidate types.RetrievalCandidate) error

+ GetMergedMetadata func(cid cid.Cid, currentMetadata, newMetadata metadata.Protocol) metadata.Protocol

+ Retrieve func(ctx context.Context, retrieval *retrieval, session *retrievalSession, ...) (*types.RetrievalStats, error)

Apr 7, 2023

Changes in this version

type GraphSyncRetriever

+ QueueInitialPause time.Duration

Mar 31, 2023

Changes in this version

+ const DefaultConcurrency

type BitswapConfig

+ Concurrency int

+ TempDir string

type GraphSyncRetriever

+ TempDir string

Mar 27, 2023

Changes in this version

+ var ErrConnectFailed = errors.New("unable to connect to provider")

type GraphSyncRetriever

+ Clock clock.Clock

+ func NewGraphsyncRetriever(getStorageProviderTimeout GetStorageProviderTimeout, client RetrievalClient) *GraphSyncRetriever

+ func NewGraphsyncRetrieverWithClock(getStorageProviderTimeout GetStorageProviderTimeout, client RetrievalClient, ...) *GraphSyncRetriever

type RetrievalClient

+ Connect func(ctx context.Context, peerAddr peer.AddrInfo) error

type Retriever

+ func NewRetrieverWithClock(ctx context.Context, session Session, candidateFinder CandidateFinder, ...) (*Retriever, error)

Mar 20, 2023

Changes in this version

+ type Session interface

+ AddToRetrieval func(retrievalId types.RetrievalID, storageProviderIds []peer.ID) error

+ EndRetrieval func(retrievalId types.RetrievalID) error

+ FilterIndexerCandidate func(candidate types.RetrievalCandidate) (bool, types.RetrievalCandidate)

+ RecordFailure func(storageProviderId peer.ID, retrievalId types.RetrievalID) error

+ RegisterRetrieval func(retrievalId types.RetrievalID, cid cid.Cid, selector datamodel.Node) bool

Mar 15, 2023

Changes in this version

+ type FilterIndexerCandidate func(types.RetrievalCandidate) (bool, types.RetrievalCandidate)

Mar 9, 2023

Changes in this version

+ type TimeCounter struct

+ func NewTimeCounter() *TimeCounter

+ func (tc *TimeCounter) Next() uint64

Mar 3, 2023

Changes in this version

+ const BufferWindow

type AssignableCandidateFinder

+ func NewAssignableCandidateFinderWithClock(candidateFinder CandidateFinder, ...) AssignableCandidateFinder

type ProtocolSplitter

+ func (ps *ProtocolSplitter) SplitRetrievalRequest(ctx context.Context, request types.RetrievalRequest, ...) types.RetrievalSplitter[multicodec.Code]

Mar 1, 2023

Changes in this version

type RetrieverConfig

+ DisableGraphsync bool

Feb 27, 2023

Changes in this version

+ type DirectCandidateFinder struct

+ func NewDirectCandidateFinder(h host.Host, providers []peer.AddrInfo) *DirectCandidateFinder

+ func (d *DirectCandidateFinder) FindCandidates(ctx context.Context, c cid.Cid) ([]types.RetrievalCandidate, error)

+ func (d *DirectCandidateFinder) FindCandidatesAsync(ctx context.Context, c cid.Cid) (<-chan types.FindCandidatesResult, error)

Feb 16, 2023

Changes in this version

+ type AssignableCandidateFinder struct

+ func NewAssignableCandidateFinder(candidateFinder CandidateFinder, ...) AssignableCandidateFinder

+ func (rcf AssignableCandidateFinder) FindCandidates(ctx context.Context, request types.RetrievalRequest, ...) ([]types.RetrievalCandidate, error)

+ type BitswapConfig struct

+ BlockTimeout time.Duration

+ type BitswapRetriever struct

+ func NewBitswapRetrieverFromDeps(bsrv blockservice.BlockService, routing IndexerRouting, ...) *BitswapRetriever

+ func NewBitswapRetrieverFromHost(ctx context.Context, host host.Host, cfg BitswapConfig) *BitswapRetriever

+ func (br *BitswapRetriever) Retrieve(ctx context.Context, request types.RetrievalRequest, ...) types.CandidateRetrieval

type CandidateFinder

+ FindCandidatesAsync func(context.Context, cid.Cid) (<-chan types.FindCandidatesResult, error)

+ type GraphSyncRetriever struct

+ Client RetrievalClient

+ GetStorageProviderTimeout GetStorageProviderTimeout

+ IsAcceptableQueryResponse IsAcceptableQueryResponse

+ func (cfg *GraphSyncRetriever) Retrieve(ctx context.Context, retrievalRequest types.RetrievalRequest, ...) types.CandidateRetrieval

+ type InProgressCids interface

+ Dec func(cid.Cid, types.RetrievalID)

+ Inc func(cid.Cid, types.RetrievalID)

+ type IndexerRouting interface

+ AddProviders func(types.RetrievalID, []types.RetrievalCandidate)

+ RemoveProviders func(types.RetrievalID)

+ type MultiBlockstore interface

+ AddLinkSystem func(id types.RetrievalID, lsys *linking.LinkSystem) error

+ RemoveLinkSystem func(id types.RetrievalID)

+ type ProtocolSplitter struct

+ func NewProtocolSplitter(protocols []multicodec.Code) *ProtocolSplitter

+ func (ps *ProtocolSplitter) SplitRetrieval(ctx context.Context, request types.RetrievalRequest, ...) types.RetrievalSplitter

Feb 3, 2023

Changes in this version

+ var ErrDealProposalFailed = errors.New("deal proposal failed")

+ var ErrRetrieverNotStarted = errors.New("retriever not started")

+ type Executor struct

+ Client RetrievalClient

+ GetStorageProviderTimeout GetStorageProviderTimeout

+ IsAcceptableQueryResponse IsAcceptableQueryResponse

+ IsAcceptableStorageProvider IsAcceptableStorageProvider

+ func (cfg *Executor) RetrieveFromCandidates(ctx context.Context, retrievalRequest types.RetrievalRequest, ...) (*types.RetrievalStats, error)

+ type RetrievalCandidateFinder struct

+ func NewRetrievalCandidateFinder(candidateFinder CandidateFinder, ...) *RetrievalCandidateFinder

+ func (rcf *RetrievalCandidateFinder) FindCandidates(ctx context.Context, request types.RetrievalRequest, ...) ([]types.RetrievalCandidate, error)

type RetrievalClient

+ RetrieveFromPeer func(ctx context.Context, linkSystem ipld.LinkSystem, peerID peer.ID, ...) (*types.RetrievalStats, error)

type Retriever

+ func (retriever *Retriever) RegisterSubscriber(subscriber types.RetrievalEventSubscriber) func()

+ func (retriever *Retriever) Start()

+ func (retriever *Retriever) Stop() chan struct{}

Jan 13, 2023

Changes in this version

+ var ErrAllQueriesFailed = errors.New("all queries failed")

+ var ErrAllRetrievalsFailed = errors.New("all retrievals failed")

+ var ErrHitRetrievalLimit = errors.New("hit retrieval limit")

+ var ErrNoCandidates = errors.New("no candidates")

+ var ErrProposalCreationFailed = errors.New("proposal creation failed")

+ var ErrQueryFailed = errors.New("query failed")

+ var ErrRetrievalFailed = errors.New("retrieval failed")

+ var ErrRetrievalRegistrationFailed = errors.New("retrieval registration failed")

+ var ErrRetrievalTimedOut = errors.New("retrieval timed out")

+ var ErrUnexpectedRetrieval = errors.New("unexpected active retrieval")

+ func RetrievalProposalForAsk(ask *retrievalmarket.QueryResponse, c cid.Cid, optionalSelector ipld.Node) (*retrievalmarket.DealProposal, error)

+ type ActiveRetrievalsManager struct

+ func NewActiveRetrievalsManager() *ActiveRetrievalsManager

+ func (arm *ActiveRetrievalsManager) GetActiveRetrievalCountFor(storageProviderId peer.ID) uint

+ func (arm *ActiveRetrievalsManager) GetStatusFor(retrievalCid cid.Cid, phase eventpublisher.Phase) (uuid.UUID, cid.Cid, time.Time, bool)

+ func (arm *ActiveRetrievalsManager) New(retrievalCid cid.Cid, queryCandidateCount int, retrievalCandidateCount int) (uuid.UUID, error)

+ func (arm *ActiveRetrievalsManager) QueryCandidatedFinished(retrievalCid cid.Cid)

+ func (arm *ActiveRetrievalsManager) RetrievalCandidatedFinished(retrievalCid cid.Cid, success bool)

+ func (arm *ActiveRetrievalsManager) SetRetrievalCandidate(retrievalCid, rootCid cid.Cid, storageProviderId peer.ID, maxConcurrent uint) error

+ func (arm *ActiveRetrievalsManager) SetRetrievalCandidateCount(retrievalCid cid.Cid, candidateCount int)

+ type BlockConfirmer func(c cid.Cid) (bool, error)

+ type CandidateCallback func(RetrievalCandidate) error

+ type CandidateErrorCallback func(RetrievalCandidate, error)

+ type CandidateFinder interface

+ FindCandidates func(context.Context, cid.Cid) ([]RetrievalCandidate, error)

+ type CounterCallback func(int) error

+ type ErrRetrievalAlreadyRunning struct

+ func (e ErrRetrievalAlreadyRunning) Error() string

+ type EventManager struct

+ func NewEventManager(ctx context.Context) *EventManager

+ func (em *EventManager) FireQueryFailure(retrievalId uuid.UUID, requestedCid cid.Cid, phaseStartTime time.Time, ...)

+ func (em *EventManager) FireQueryProgress(retrievalId uuid.UUID, requestedCid cid.Cid, phaseStartTime time.Time, ...)

+ func (em *EventManager) FireQuerySuccess(retrievalId uuid.UUID, requestedCid cid.Cid, phaseStartTime time.Time, ...)

+ func (em *EventManager) FireRetrievalFailure(retrievalId uuid.UUID, requestedCid cid.Cid, phaseStartTime time.Time, ...)

+ func (em *EventManager) FireRetrievalProgress(retrievalId uuid.UUID, requestedCid cid.Cid, phaseStartTime time.Time, ...)

+ func (em *EventManager) FireRetrievalSuccess(retrievalId uuid.UUID, requestedCid cid.Cid, phaseStartTime time.Time, ...)

+ func (em *EventManager) RegisterListener(listener RetrievalEventListener) func()

+ type GetStorageProviderTimeout func(peer peer.ID) time.Duration

+ type Instrumentation interface

+ OnErrorQueryingRetrievalCandidate func(candidate RetrievalCandidate, err error)

+ OnErrorRetrievingFromCandidate func(candidate RetrievalCandidate, err error)

+ OnFilteredRetrievalQueryForCandidate func(candidate RetrievalCandidate, queryResponse *retrievalmarket.QueryResponse)

+ OnRetrievalCandidatesFiltered func(filteredCount int) error

+ OnRetrievalCandidatesFound func(foundCount int) error

+ OnRetrievalQueryForCandidate func(candidate RetrievalCandidate, queryResponse *retrievalmarket.QueryResponse)

+ OnRetrievingFromCandidate func(candidate RetrievalCandidate)

+ type IsAcceptableQueryResponse func(*retrievalmarket.QueryResponse) bool

+ type IsAcceptableStorageProvider func(peer peer.ID) bool

+ type MinerConfig struct

+ MaxConcurrentRetrievals uint

+ RetrievalTimeout time.Duration

+ type RetrievalCandidate struct

+ MinerPeer peer.AddrInfo

+ RootCid cid.Cid

+ type RetrievalClient interface

+ RetrievalQueryToPeer func(ctx context.Context, minerPeer peer.AddrInfo, pcid cid.Cid) (*retrievalmarket.QueryResponse, error)

+ RetrieveContentFromPeerAsync func(ctx context.Context, peerID peer.ID, minerWallet address.Address, ...) (<-chan RetrievalResult, <-chan uint64, func())

+ SubscribeToRetrievalEvents func(subscriber eventpublisher.RetrievalSubscriber)

+ type RetrievalConfig struct

+ GetStorageProviderTimeout GetStorageProviderTimeout

+ Instrumentation Instrumentation

+ IsAcceptableQueryResponse IsAcceptableQueryResponse

+ IsAcceptableStorageProvider IsAcceptableStorageProvider

+ type RetrievalEventListener interface

+ QueryFailure func(retrievalId uuid.UUID, phaseStartTime, eventTime time.Time, ...)

+ QueryProgress func(retrievalId uuid.UUID, phaseStartTime, eventTime time.Time, ...)

+ QuerySuccess func(retrievalId uuid.UUID, phaseStartTime, eventTime time.Time, ...)

+ RetrievalFailure func(retrievalId uuid.UUID, phaseStartTime, eventTime time.Time, ...)

+ RetrievalProgress func(retrievalId uuid.UUID, phaseStartTime, eventTime time.Time, ...)

+ RetrievalSuccess func(retrievalId uuid.UUID, phaseStartTime, eventTime time.Time, ...)

+ type RetrievalResult struct

+ Err error

+ type RetrievalStats struct

+ AskPrice abi.TokenAmount

+ AverageSpeed uint64

+ Duration time.Duration

+ NumPayments int

+ RootCid cid.Cid

+ Size uint64

+ StorageProviderId peer.ID

+ TotalPayment abi.TokenAmount

+ func RetrieveFromCandidates(ctx context.Context, cfg *RetrievalConfig, candidateFinder CandidateFinder, ...) (*RetrievalStats, error)

+ type RetrievalSubscriber interface

+ OnRetrievalEvent func(eventpublisher.RetrievalEvent)

+ type Retriever struct

+ func NewRetriever(ctx context.Context, config RetrieverConfig, client RetrievalClient, ...) (*Retriever, error)

+ func (retriever *Retriever) OnRetrievalEvent(event eventpublisher.RetrievalEvent)

+ func (retriever *Retriever) RegisterListener(listener RetrievalEventListener) func()

+ func (retriever *Retriever) Retrieve(ctx context.Context, cid cid.Cid) (*RetrievalStats, error)

+ type RetrieverConfig struct

+ DefaultMinerConfig MinerConfig

+ MinerBlacklist map[peer.ID]bool

+ MinerConfigs map[peer.ID]MinerConfig

+ MinerWhitelist map[peer.ID]bool

+ PaidRetrievals bool

+ func (cfg *RetrieverConfig) GetMinerConfig(peer peer.ID) MinerConfig