retriever

package
v0.4.0 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Feb 16, 2023 License: Apache-2.0, MIT Imports: 42 Imported by: 3

Documentation

Index

Constants

This section is empty.

Variables

View Source
var (
	ErrRetrieverNotStarted         = errors.New("retriever not started")
	ErrDealProposalFailed          = errors.New("deal proposal failed")
	ErrNoCandidates                = errors.New("no candidates")
	ErrUnexpectedRetrieval         = errors.New("unexpected active retrieval")
	ErrHitRetrievalLimit           = errors.New("hit retrieval limit")
	ErrProposalCreationFailed      = errors.New("proposal creation failed")
	ErrRetrievalRegistrationFailed = errors.New("retrieval registration failed")
	ErrRetrievalFailed             = errors.New("retrieval failed")
	ErrAllRetrievalsFailed         = errors.New("all retrievals failed")
	ErrQueryFailed                 = errors.New("query failed")
	ErrAllQueriesFailed            = errors.New("all queries failed")
	ErrRetrievalTimedOut           = errors.New("retrieval timed out")
	ErrRetrievalAlreadyRunning     = errors.New("retrieval already running for CID")
)

Functions

func RetrievalProposalForAsk

func RetrievalProposalForAsk(ask *retrievalmarket.QueryResponse, c cid.Cid, optionalSelector ipld.Node) (*retrievalmarket.DealProposal, error)

Types

type AssignableCandidateFinder added in v0.4.0

type AssignableCandidateFinder struct {
	// contains filtered or unexported fields
}

AssignableCandidateFinder finds and filters candidates for a given retrieval

func NewAssignableCandidateFinder added in v0.4.0

func NewAssignableCandidateFinder(candidateFinder CandidateFinder, isAcceptableStorageProvider IsAcceptableStorageProvider) AssignableCandidateFinder

func (AssignableCandidateFinder) FindCandidates added in v0.4.0

func (rcf AssignableCandidateFinder) FindCandidates(ctx context.Context, request types.RetrievalRequest, eventsCallback func(types.RetrievalEvent)) ([]types.RetrievalCandidate, error)

type BitswapConfig added in v0.4.0

type BitswapConfig struct {
	BlockTimeout time.Duration
}

BitswapConfig contains configurable parameters for bitswap fetching

type BitswapRetriever added in v0.4.0

type BitswapRetriever struct {
	// contains filtered or unexported fields
}

BitswapRetriever uses bitswap to retrieve data BitswapRetriever retieves using a combination of a go-bitswap client specially configured per retrieval, underneath a blockservice and a go-fetcher Fetcher. Selectors are used to travers the dag to make sure the CARs for bitswap match graphsync Note: this is a tradeoff over go-merkledag for traversal, cause selector execution is slow. But the solution is to improve selector execution, not introduce unpredictable encoding.

func NewBitswapRetrieverFromDeps added in v0.4.0

func NewBitswapRetrieverFromDeps(bsrv blockservice.BlockService, routing IndexerRouting, inProgressCids InProgressCids, bstore MultiBlockstore, cfg BitswapConfig, clock clock.Clock) *BitswapRetriever

NewBitswapRetrieverFromDeps is primarily for testing, constructing behavior from direct dependencies

func NewBitswapRetrieverFromHost added in v0.4.0

func NewBitswapRetrieverFromHost(ctx context.Context, host host.Host, cfg BitswapConfig) *BitswapRetriever

NewBitswapRetrieverFromHost constructs a new bitswap retriever for the given libp2p host

func (*BitswapRetriever) Retrieve added in v0.4.0

Retrieve initializes a new bitswap session

type CandidateCallback

type CandidateCallback func(types.RetrievalCandidate) error

type CandidateErrorCallback

type CandidateErrorCallback func(types.RetrievalCandidate, error)

type CandidateFinder

type CandidateFinder interface {
	FindCandidates(context.Context, cid.Cid) ([]types.RetrievalCandidate, error)
	FindCandidatesAsync(context.Context, cid.Cid) (<-chan types.FindCandidatesResult, error)
}

type CounterCallback

type CounterCallback func(int) error

type GetStorageProviderTimeout

type GetStorageProviderTimeout func(peer peer.ID) time.Duration

type GraphSyncRetriever added in v0.4.0

type GraphSyncRetriever struct {
	GetStorageProviderTimeout GetStorageProviderTimeout
	IsAcceptableQueryResponse IsAcceptableQueryResponse
	Client                    RetrievalClient
	// contains filtered or unexported fields
}

func (*GraphSyncRetriever) Retrieve added in v0.4.0

func (cfg *GraphSyncRetriever) Retrieve(
	ctx context.Context,
	retrievalRequest types.RetrievalRequest,
	eventsCallback func(types.RetrievalEvent),
) types.CandidateRetrieval

RetrieveFromCandidates performs a retrieval for a given CID by querying the indexer, then attempting to query all candidates and attempting to perform a full retrieval from the best and fastest storage provider as the queries are received.

type InProgressCids added in v0.4.0

type InProgressCids interface {
	Inc(cid.Cid, types.RetrievalID)
	Dec(cid.Cid, types.RetrievalID)
}

type IndexerRouting added in v0.4.0

type IndexerRouting interface {
	AddProviders(types.RetrievalID, []types.RetrievalCandidate)
	RemoveProviders(types.RetrievalID)
}

IndexerRouting are the required methods to track indexer routing

type IsAcceptableQueryResponse

type IsAcceptableQueryResponse func(peer peer.ID, req types.RetrievalRequest, queryResponse *retrievalmarket.QueryResponse) bool

type IsAcceptableStorageProvider

type IsAcceptableStorageProvider func(peer peer.ID) bool

type MinerConfig

type MinerConfig struct {
	RetrievalTimeout        time.Duration
	MaxConcurrentRetrievals uint
}

type MultiBlockstore added in v0.4.0

type MultiBlockstore interface {
	AddLinkSystem(id types.RetrievalID, lsys *linking.LinkSystem) error
	RemoveLinkSystem(id types.RetrievalID)
}

MultiBlockstore are the require methods to track linksystems

type ProtocolSplitter added in v0.4.0

type ProtocolSplitter struct {
	// contains filtered or unexported fields
}

func NewProtocolSplitter added in v0.4.0

func NewProtocolSplitter(protocols []multicodec.Code) *ProtocolSplitter

func (*ProtocolSplitter) SplitRetrieval added in v0.4.0

func (ps *ProtocolSplitter) SplitRetrieval(ctx context.Context, request types.RetrievalRequest, events func(types.RetrievalEvent)) types.RetrievalSplitter

type RetrievalClient

type RetrievalClient interface {
	RetrievalQueryToPeer(
		ctx context.Context,
		minerPeer peer.AddrInfo,
		cid cid.Cid,
		onConnected func(),
	) (*retrievalmarket.QueryResponse, error)

	RetrieveFromPeer(
		ctx context.Context,
		linkSystem ipld.LinkSystem,
		peerID peer.ID,
		minerWallet address.Address,
		proposal *retrievalmarket.DealProposal,
		eventsCallback datatransfer.Subscriber,
		gracefulShutdownRequested <-chan struct{},
	) (*types.RetrievalStats, error)
}

type Retriever

type Retriever struct {
	// contains filtered or unexported fields
}

func NewRetriever

func NewRetriever(
	ctx context.Context,
	config RetrieverConfig,
	client RetrievalClient,
	candidateFinder CandidateFinder,
	bitswapRetriever types.CandidateRetriever,
) (*Retriever, error)

func (*Retriever) RegisterSubscriber added in v0.3.0

func (retriever *Retriever) RegisterSubscriber(subscriber types.RetrievalEventSubscriber) func()

RegisterSubscriber registers a subscriber to receive all events fired during the process of making a retrieval, including the process of querying available storage providers to find compatible ones to attempt retrieval from.

func (*Retriever) Retrieve

func (retriever *Retriever) Retrieve(
	ctx context.Context,
	request types.RetrievalRequest,
	eventsCB func(types.RetrievalEvent),
) (*types.RetrievalStats, error)

Retrieve attempts to retrieve the given CID using the configured CandidateFinder to find storage providers that should have the CID.

func (*Retriever) Start added in v0.3.0

func (retriever *Retriever) Start()

Start will start the retriever events system

func (*Retriever) Stop added in v0.3.0

func (retriever *Retriever) Stop() chan struct{}

Stop will stop the retriever events system and return a channel that will be closed when shutdown has completed

type RetrieverConfig

type RetrieverConfig struct {
	MinerBlacklist     map[peer.ID]bool
	MinerWhitelist     map[peer.ID]bool
	DefaultMinerConfig MinerConfig
	MinerConfigs       map[peer.ID]MinerConfig
	PaidRetrievals     bool
}

All config values should be safe to leave uninitialized

Directories

Path Synopsis
Package combinators contains tools to put various retrieval components together to make full retrievers
Package combinators contains tools to put various retrieval components together to make full retrievers
Package coordinators contains retrievers that coordinate multiple child retrievers
Package coordinators contains retrievers that coordinate multiple child retrievers

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL