README ¶
Caboose
A remote-car blockstore.
Provides a blockstore interface over a dynamic set of remote car providers.
Usage
go run ./cmd/caboose bafkreiaqv66m5nd6mwgkk7h5lwqnjzj54s4f7knmnrjhb7ylzqfg2vdo54 out.car
License
Documentation ¶
Index ¶
- Constants
- Variables
- type Caboose
- func (c *Caboose) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error)
- func (c *Caboose) Close()
- func (c *Caboose) DeleteBlock(context.Context, cid.Cid) error
- func (c *Caboose) Fetch(ctx context.Context, path string, cb DataCallback) error
- func (c *Caboose) Get(ctx context.Context, it cid.Cid) (blocks.Block, error)
- func (c *Caboose) GetPoolPerf() map[string]*tieredhashing.NodePerf
- func (c *Caboose) GetSize(ctx context.Context, it cid.Cid) (int, error)
- func (c *Caboose) Has(ctx context.Context, it cid.Cid) (bool, error)
- func (c *Caboose) HashOnRead(enabled bool)
- func (c *Caboose) Put(context.Context, blocks.Block) error
- func (c *Caboose) PutMany(context.Context, []blocks.Block) error
- type Config
- type DataCallback
- type ErrCoolDown
- type ErrInvalidResponse
- type ErrPartialResponse
- type ErrSaturnTooManyRequests
- type TrackingReader
Constants ¶
const (
BackendOverrideKey = "CABOOSE_BACKEND_OVERRIDE"
)
const DefaultComplianceCidPeriod = int64(100)
This represents, on average, how many requests caboose makes before requesting a compliance cid. Example: a period of 100 implies Caboose will on average make a compliance CID request once every 100 requests.
const DefaultLoggingInterval = 5 * time.Second
const DefaultOrchestratorEndpoint = "https://orchestrator.strn.pl/nodes?maxNodes=200"
const DefaultPoolRefreshInterval = 5 * time.Minute
const DefaultSaturnBlockRequestTimeout = 19 * time.Second
const DefaultSaturnCarRequestTimeout = 30 * time.Minute
const DefaultSaturnMirrorRequestTimeout = 30 * time.Second
const DefaultSaturnOrchestratorRequestTimeout = 30 * time.Second
const (
SaturnEnvKey = "STRN_ENV_TAG"
)
Variables ¶
var CabooseMetrics = prometheus.NewRegistry()
var ErrContentProviderNotFound error = errors.New("saturn failed to find content providers")
var ErrNoBackend error = errors.New("no available saturn backend")
var ErrNotImplemented error = errors.New("not implemented")
var ErrSaturnTimeout error = errors.New("saturn backend timed out")
Functions ¶
This section is empty.
Types ¶
type Caboose ¶
type Caboose struct {
// contains filtered or unexported fields
}
func NewCaboose ¶
NewCaboose sets up a caboose fetcher. Note: Caboose is NOT a persistent blockstore and does NOT have an in-memory cache. Every request will result in a remote network request.
func (*Caboose) AllKeysChan ¶
func (*Caboose) Fetch ¶
Fetch allows fetching car archives by a path of the form `/ipfs/<cid>[/path/to/file]`
func (*Caboose) GetPoolPerf ¶
func (c *Caboose) GetPoolPerf() map[string]*tieredhashing.NodePerf
for testing only
func (*Caboose) HashOnRead ¶
HashOnRead specifies if every read block should be rehashed to make sure it matches its CID.
type Config ¶
type Config struct { // OrchestratorEndpoint is the URL of the Saturn orchestrator. OrchestratorEndpoint *url.URL // OrchestratorClient is the HTTP client to use when communicating with the Saturn orchestrator. OrchestratorClient *http.Client // OrchestratorOverride replaces calls to the orchestrator with a fixed response. OrchestratorOverride []tieredhashing.NodeInfo // LoggingEndpoint is the URL of the logging endpoint where we submit logs pertaining to our Saturn retrieval requests. LoggingEndpoint url.URL // LoggingClient is the HTTP client to use when communicating with the logging endpoint. LoggingClient *http.Client // LoggingInterval is the interval at which we submit logs to the logging endpoint. LoggingInterval time.Duration // SaturnClient is the HTTP client to use when retrieving content from the Saturn network. SaturnClient *http.Client ExtraHeaders *http.Header // DoValidation is used to determine if we should validate the blocks recieved from the Saturn network. DoValidation bool // If set, AffinityKey is used instead of the block CID as the key on the // Saturn node pool to determine which Saturn node to retrieve the block from. // NOTE: If gateway.ContentPathKey is present in request context, // it will be used as AffinityKey automatically. AffinityKey string // PoolRefresh is the interval at which we refresh the pool of Saturn nodes. PoolRefresh time.Duration // MirrorFraction is what fraction of requests will be mirrored to another random node in order to track metrics / determine the current best nodes. MirrorFraction float64 // MaxRetrievalAttempts determines the number of times we will attempt to retrieve a block from the Saturn network before failing. MaxRetrievalAttempts int // MaxFetchFailuresBeforeCoolDown is the maximum number of retrieval failures across the pool for a url before we auto-reject subsequent // fetches of that url. MaxFetchFailuresBeforeCoolDown int // FetchKeyCoolDownDuration is duration of time a key will stay in the cool down cache // before we start making retrieval attempts for it. FetchKeyCoolDownDuration time.Duration // SaturnNodeCoolOff is the cool off duration for a saturn node once we determine that we shouldn't be sending requests to it for a while. SaturnNodeCoolOff time.Duration TieredHashingOpts []tieredhashing.Option ComplianceCidPeriod int64 }
type DataCallback ¶
DataCallback allows for extensible validation of path-retrieved data.
type ErrCoolDown ¶
type ErrCoolDown struct { Cid cid.Cid Path string // contains filtered or unexported fields }
func (*ErrCoolDown) Error ¶
func (e *ErrCoolDown) Error() string
func (*ErrCoolDown) RetryAfter ¶
func (e *ErrCoolDown) RetryAfter() time.Duration
type ErrInvalidResponse ¶ added in v0.0.3
type ErrInvalidResponse struct {
Message string
}
ErrInvalidResponse can be returned from a DataCallback to indicate that the data provided for the requested resource was explicitly 'incorrect' - that blocks not in the requested dag, or non-car-conforming data was returned.
func (ErrInvalidResponse) Error ¶ added in v0.0.5
func (e ErrInvalidResponse) Error() string
type ErrPartialResponse ¶
type ErrPartialResponse struct { StillNeed []string // contains filtered or unexported fields }
ErrPartialResponse can be returned from a DataCallback to indicate that some of the requested resource was successfully fetched, and that instead of retrying the full resource, that there are one or more more specific resources that should be fetched (via StillNeed) to complete the request.
func (ErrPartialResponse) Error ¶
func (epr ErrPartialResponse) Error() string
type ErrSaturnTooManyRequests ¶
type ErrSaturnTooManyRequests struct { Node string // contains filtered or unexported fields }
func (*ErrSaturnTooManyRequests) Error ¶
func (e *ErrSaturnTooManyRequests) Error() string
func (*ErrSaturnTooManyRequests) RetryAfter ¶
func (e *ErrSaturnTooManyRequests) RetryAfter() time.Duration