retriever

Documentation

Index

• Variables
• func RetrievalProposalForAsk(ask *retrievalmarket.QueryResponse, c cid.Cid, optionalSelector ipld.Node) (*retrievalmarket.DealProposal, error)
• type AssignableCandidateFinder
  • func NewAssignableCandidateFinder(candidateFinder CandidateFinder, ...) AssignableCandidateFinder
  • func (rcf AssignableCandidateFinder) FindCandidates(ctx context.Context, request types.RetrievalRequest, ...) ([]types.RetrievalCandidate, error)
• type BitswapConfig
• type BitswapRetriever
  • func NewBitswapRetrieverFromDeps(bsrv blockservice.BlockService, routing IndexerRouting, ...) *BitswapRetriever
  • func NewBitswapRetrieverFromHost(ctx context.Context, host host.Host, cfg BitswapConfig) *BitswapRetriever
  • func (br *BitswapRetriever) Retrieve(ctx context.Context, request types.RetrievalRequest, ...) types.CandidateRetrieval
• type CandidateCallback
• type CandidateErrorCallback
• type CandidateFinder
• type CounterCallback
• type GetStorageProviderTimeout
• type GraphSyncRetriever
  • func (cfg *GraphSyncRetriever) Retrieve(ctx context.Context, retrievalRequest types.RetrievalRequest, ...) types.CandidateRetrieval
• type InProgressCids
• type IndexerRouting
• type IsAcceptableQueryResponse
• type IsAcceptableStorageProvider
• type MinerConfig
• type MultiBlockstore
• type ProtocolSplitter
  • func NewProtocolSplitter(protocols []multicodec.Code) *ProtocolSplitter
  • func (ps *ProtocolSplitter) SplitRetrieval(ctx context.Context, request types.RetrievalRequest, ...) types.RetrievalSplitter
• type RetrievalClient
• type Retriever
  • func NewRetriever(ctx context.Context, config RetrieverConfig, client RetrievalClient, ...) (*Retriever, error)
  • func (retriever *Retriever) RegisterSubscriber(subscriber types.RetrievalEventSubscriber) func()
  • func (retriever *Retriever) Retrieve(ctx context.Context, request types.RetrievalRequest, ...) (*types.RetrievalStats, error)
  • func (retriever *Retriever) Start()
  • func (retriever *Retriever) Stop() chan struct{}
• type RetrieverConfig

Variables

var (
	ErrRetrieverNotStarted         = errors.New("retriever not started")
	ErrDealProposalFailed          = errors.New("deal proposal failed")
	ErrNoCandidates                = errors.New("no candidates")
	ErrUnexpectedRetrieval         = errors.New("unexpected active retrieval")
	ErrHitRetrievalLimit           = errors.New("hit retrieval limit")
	ErrProposalCreationFailed      = errors.New("proposal creation failed")
	ErrRetrievalRegistrationFailed = errors.New("retrieval registration failed")
	ErrRetrievalFailed             = errors.New("retrieval failed")
	ErrAllRetrievalsFailed         = errors.New("all retrievals failed")
	ErrQueryFailed                 = errors.New("query failed")
	ErrAllQueriesFailed            = errors.New("all queries failed")
	ErrRetrievalTimedOut           = errors.New("retrieval timed out")
	ErrRetrievalAlreadyRunning     = errors.New("retrieval already running for CID")
)

Functions

func RetrievalProposalForAsk

func RetrievalProposalForAsk(ask *retrievalmarket.QueryResponse, c cid.Cid, optionalSelector ipld.Node) (*retrievalmarket.DealProposal, error)

Types

type AssignableCandidateFinder

type AssignableCandidateFinder struct {
	// contains filtered or unexported fields
}

AssignableCandidateFinder finds and filters candidates for a given retrieval

func NewAssignableCandidateFinder

func NewAssignableCandidateFinder(candidateFinder CandidateFinder, isAcceptableStorageProvider IsAcceptableStorageProvider) AssignableCandidateFinder

func (AssignableCandidateFinder) FindCandidates

func (rcf AssignableCandidateFinder) FindCandidates(ctx context.Context, request types.RetrievalRequest, eventsCallback func(types.RetrievalEvent)) ([]types.RetrievalCandidate, error)

type BitswapConfig

type BitswapConfig struct {
	BlockTimeout time.Duration
}

BitswapConfig contains configurable parameters for bitswap fetching

type BitswapRetriever

type BitswapRetriever struct {
	// contains filtered or unexported fields
}

BitswapRetriever uses bitswap to retrieve data BitswapRetriever retieves using a combination of a go-bitswap client specially configured per retrieval, underneath a blockservice and a go-fetcher Fetcher. Selectors are used to travers the dag to make sure the CARs for bitswap match graphsync Note: this is a tradeoff over go-merkledag for traversal, cause selector execution is slow. But the solution is to improve selector execution, not introduce unpredictable encoding.

func NewBitswapRetrieverFromDeps

func NewBitswapRetrieverFromDeps(bsrv blockservice.BlockService, routing IndexerRouting, inProgressCids InProgressCids, bstore MultiBlockstore, cfg BitswapConfig, clock clock.Clock) *BitswapRetriever

NewBitswapRetrieverFromDeps is primarily for testing, constructing behavior from direct dependencies

func NewBitswapRetrieverFromHost

func NewBitswapRetrieverFromHost(ctx context.Context, host host.Host, cfg BitswapConfig) *BitswapRetriever

NewBitswapRetrieverFromHost constructs a new bitswap retriever for the given libp2p host

func (*BitswapRetriever) Retrieve

func (br *BitswapRetriever) Retrieve(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.CandidateRetrieval

Retrieve initializes a new bitswap session

type CandidateCallback

type CandidateCallback func(types.RetrievalCandidate) error

type CandidateErrorCallback

type CandidateErrorCallback func(types.RetrievalCandidate, error)

type CandidateFinder

type CandidateFinder interface {
	FindCandidates(context.Context, cid.Cid) ([]types.RetrievalCandidate, error)
	FindCandidatesAsync(context.Context, cid.Cid) (<-chan types.FindCandidatesResult, error)
}

type CounterCallback

type CounterCallback func(int) error

type GetStorageProviderTimeout

type GetStorageProviderTimeout func(peer peer.ID) time.Duration

type GraphSyncRetriever

type GraphSyncRetriever struct {
	GetStorageProviderTimeout GetStorageProviderTimeout
	IsAcceptableQueryResponse IsAcceptableQueryResponse
	Client                    RetrievalClient
	// contains filtered or unexported fields
}

func (*GraphSyncRetriever) Retrieve

func (cfg *GraphSyncRetriever) Retrieve(
	ctx context.Context,
	retrievalRequest types.RetrievalRequest,
	eventsCallback func(types.RetrievalEvent),
) types.CandidateRetrieval

RetrieveFromCandidates performs a retrieval for a given CID by querying the indexer, then attempting to query all candidates and attempting to perform a full retrieval from the best and fastest storage provider as the queries are received.

type InProgressCids

type InProgressCids interface {
	Inc(cid.Cid, types.RetrievalID)
	Dec(cid.Cid, types.RetrievalID)
}

type IndexerRouting

type IndexerRouting interface {
	AddProviders(types.RetrievalID, []types.RetrievalCandidate)
	RemoveProviders(types.RetrievalID)
}

IndexerRouting are the required methods to track indexer routing

type IsAcceptableQueryResponse

type IsAcceptableQueryResponse func(peer peer.ID, req types.RetrievalRequest, queryResponse *retrievalmarket.QueryResponse) bool

type IsAcceptableStorageProvider

type IsAcceptableStorageProvider func(peer peer.ID) bool

type MinerConfig

type MinerConfig struct {
	RetrievalTimeout        time.Duration
	MaxConcurrentRetrievals uint
}

type MultiBlockstore

type MultiBlockstore interface {
	AddLinkSystem(id types.RetrievalID, lsys *linking.LinkSystem) error
	RemoveLinkSystem(id types.RetrievalID)
}

MultiBlockstore are the require methods to track linksystems

type ProtocolSplitter

type ProtocolSplitter struct {
	// contains filtered or unexported fields
}

func NewProtocolSplitter

func NewProtocolSplitter(protocols []multicodec.Code) *ProtocolSplitter

func (*ProtocolSplitter) SplitRetrieval

func (ps *ProtocolSplitter) SplitRetrieval(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.RetrievalSplitter

type RetrievalClient

type RetrievalClient interface {
	RetrievalQueryToPeer(
		ctx context.Context,
		minerPeer peer.AddrInfo,
		cid cid.Cid,
		onConnected func(),
	) (*retrievalmarket.QueryResponse, error)

	RetrieveFromPeer(
		ctx context.Context,
		linkSystem ipld.LinkSystem,
		peerID peer.ID,
		minerWallet address.Address,
		proposal *retrievalmarket.DealProposal,
		eventsCallback datatransfer.Subscriber,
		gracefulShutdownRequested <-chan struct{},
	) (*types.RetrievalStats, error)
}

type Retriever

type Retriever struct {
	// contains filtered or unexported fields
}

func NewRetriever

func NewRetriever(
	ctx context.Context,
	config RetrieverConfig,
	client RetrievalClient,
	candidateFinder CandidateFinder,
	bitswapRetriever types.CandidateRetriever,
) (*Retriever, error)

func (*Retriever) RegisterSubscriber

func (retriever *Retriever) RegisterSubscriber(subscriber types.RetrievalEventSubscriber) func()

RegisterSubscriber registers a subscriber to receive all events fired during the process of making a retrieval, including the process of querying available storage providers to find compatible ones to attempt retrieval from.

func (*Retriever) Retrieve

func (retriever *Retriever) Retrieve(
	ctx context.Context,
	request types.RetrievalRequest,
	eventsCB func(types.RetrievalEvent),
) (*types.RetrievalStats, error)

Retrieve attempts to retrieve the given CID using the configured CandidateFinder to find storage providers that should have the CID.

func (*Retriever) Start

func (retriever *Retriever) Start()

Start will start the retriever events system

func (*Retriever) Stop

func (retriever *Retriever) Stop() chan struct{}

Stop will stop the retriever events system and return a channel that will be closed when shutdown has completed

type RetrieverConfig

type RetrieverConfig struct {
	MinerBlacklist     map[peer.ID]bool
	MinerWhitelist     map[peer.ID]bool
	DefaultMinerConfig MinerConfig
	MinerConfigs       map[peer.ID]MinerConfig
	PaidRetrievals     bool
}

All config values should be safe to leave uninitialized