checker

Documentation

Index

• Variables
• type Config
• type Observer
  • func NewObserver(logger *zap.Logger, repairQueue queue.RepairQueue, overlay *overlay.Service, ...) *Observer
  • func (observer *Observer) Finish(ctx context.Context) (err error)
  • func (observer *Observer) Fork(ctx context.Context) (_ rangedloop.Partial, err error)
  • func (observer *Observer) Join(ctx context.Context, partial rangedloop.Partial) (err error)
  • func (observer *Observer) RefreshReliabilityCache(ctx context.Context) error
  • func (observer *Observer) Start(ctx context.Context, startTime time.Time) (err error)
  • func (observer *Observer) TestingCompareInjuredSegmentIDs(ctx context.Context, streamIDs []uuid.UUID) error
• type ReliabilityCache
  • func NewReliabilityCache(overlay *overlay.Service, staleness time.Duration, ...) *ReliabilityCache
  • func (cache *ReliabilityCache) LastUpdate() time.Time
  • func (cache *ReliabilityCache) MissingPieces(ctx context.Context, created time.Time, pieces metabase.Pieces) (_ metabase.Pieces, err error)
  • func (cache *ReliabilityCache) NumNodes(ctx context.Context) (numNodes int, err error)
  • func (cache *ReliabilityCache) OutOfPlacementPieces(ctx context.Context, created time.Time, pieces metabase.Pieces, ...) (_ metabase.Pieces, err error)
  • func (cache *ReliabilityCache) PiecesNodesLastNetsInOrder(ctx context.Context, created time.Time, pieces metabase.Pieces) (lastNets []string, err error)
  • func (cache *ReliabilityCache) Refresh(ctx context.Context) (err error)
• type RepairOverride
  • func (ro *RepairOverride) Set(s string) error
  • func (ro *RepairOverride) String() string
  • func (RepairOverride) Type() string
• type RepairOverrides
  • func (ros *RepairOverrides) GetMap() RepairOverridesMap
  • func (ros *RepairOverrides) Set(s string) error
  • func (ros *RepairOverrides) String() string
  • func (RepairOverrides) Type() string
• type RepairOverridesMap
  • func (rom *RepairOverridesMap) GetOverrideValue(rs storj.RedundancyScheme) int32
  • func (rom *RepairOverridesMap) GetOverrideValuePB(rs *pb.RedundancyScheme) int32

Variables

var (
	// Error is a standard error class for this package.
	Error = errs.Class("repair checker")
)

Types

type Config

type Config struct {
	Interval time.Duration `` /* 129-byte string literal not displayed */

	ReliabilityCacheStaleness time.Duration   `help:"how stale reliable node cache can be" releaseDefault:"5m" devDefault:"5m" testDefault:"1m"`
	RepairOverrides           RepairOverrides `` /* 186-byte string literal not displayed */
	// Node failure rate is an estimation based on a 6 hour checker run interval (4 checker iterations per day), a network of about 9200 nodes, and about 2 nodes churning per day.
	// This results in `2/9200/4 = 0.00005435` being the probability of any single node going down in the interval of one checker iteration.
	NodeFailureRate            float64  `help:"the probability of a single node going down within the next checker iteration" default:"0.00005435" `
	RepairQueueInsertBatchSize int      `help:"Number of damaged segments to buffer in-memory before flushing to the repair queue" default:"100" `
	RepairExcludedCountryCodes []string `help:"list of country codes to treat node from this country as offline " default:"" hidden:"true"`
	DoDeclumping               bool     `help:"Treat pieces on the same network as in need of repair" default:"true"`
	DoPlacementCheck           bool     `help:"Treat pieces out of segment placement as in need of repair" default:"true"`
}

Config contains configurable values for checker.

type Observer

type Observer struct {
	TotalStats aggregateStats
	// contains filtered or unexported fields
}

Observer implements the ranged loop Observer interface.

architecture: Observer

func NewObserver

func NewObserver(logger *zap.Logger, repairQueue queue.RepairQueue, overlay *overlay.Service, placementRules overlay.PlacementRules, config Config) *Observer

NewObserver creates new checker observer instance.

func (*Observer) Finish

func (observer *Observer) Finish(ctx context.Context) (err error)

Finish is called after all segments are processed by all observers.

func (*Observer) Fork

func (observer *Observer) Fork(ctx context.Context) (_ rangedloop.Partial, err error)

Fork creates a Partial to process a chunk of all the segments.

func (*Observer) Join

func (observer *Observer) Join(ctx context.Context, partial rangedloop.Partial) (err error)

Join is called after the chunk for Partial is done. This gives the opportunity to merge the output like in a reduce step.

func (*Observer) RefreshReliabilityCache

func (observer *Observer) RefreshReliabilityCache(ctx context.Context) error

RefreshReliabilityCache forces refreshing node online status cache.

func (*Observer) Start

func (observer *Observer) Start(ctx context.Context, startTime time.Time) (err error)

Start starts parallel segments loop.

func (*Observer) TestingCompareInjuredSegmentIDs

func (observer *Observer) TestingCompareInjuredSegmentIDs(ctx context.Context, streamIDs []uuid.UUID) error

TestingCompareInjuredSegmentIDs compares stream id of injured segment.

type ReliabilityCache

type ReliabilityCache struct {
	// contains filtered or unexported fields
}

ReliabilityCache caches the reliable nodes for the specified staleness duration and updates automatically from overlay.

architecture: Service

func NewReliabilityCache

func NewReliabilityCache(overlay *overlay.Service, staleness time.Duration, placementRules overlay.PlacementRules, excludedCountries []string) *ReliabilityCache

NewReliabilityCache creates a new reliability checking cache.

func (*ReliabilityCache) LastUpdate

func (cache *ReliabilityCache) LastUpdate() time.Time

LastUpdate returns when the cache was last updated, or the zero value (time.Time{}) if it has never yet been updated. LastUpdate() does not trigger an update itself.

func (*ReliabilityCache) MissingPieces

func (cache *ReliabilityCache) MissingPieces(ctx context.Context, created time.Time, pieces metabase.Pieces) (_ metabase.Pieces, err error)

MissingPieces returns piece indices that are unreliable with the given staleness period.

func (*ReliabilityCache) NumNodes

func (cache *ReliabilityCache) NumNodes(ctx context.Context) (numNodes int, err error)

NumNodes returns the number of online active nodes (as determined by the reliability cache). This number is not guaranteed to be consistent with either the nodes database or the reliability cache after returning; it is just a best-effort count and should be treated as an estimate.

func (*ReliabilityCache) OutOfPlacementPieces

func (cache *ReliabilityCache) OutOfPlacementPieces(ctx context.Context, created time.Time, pieces metabase.Pieces, placement storj.PlacementConstraint) (_ metabase.Pieces, err error)

OutOfPlacementPieces checks which pieces are out of segment placement. Piece placement is defined by node location which is storing it.

func (*ReliabilityCache) PiecesNodesLastNetsInOrder

func (cache *ReliabilityCache) PiecesNodesLastNetsInOrder(ctx context.Context, created time.Time, pieces metabase.Pieces) (lastNets []string, err error)

PiecesNodesLastNetsInOrder returns the /24 subnet for each piece storage node, in order. If a requested node is not in the database or it's unreliable, an empty string will be returned corresponding to that node's last_net.

func (*ReliabilityCache) Refresh

func (cache *ReliabilityCache) Refresh(ctx context.Context) (err error)

Refresh refreshes the cache.

type RepairOverride

type RepairOverride struct {
	Min      int
	Success  int
	Total    int
	Override int32
}

RepairOverride is a configuration struct that contains an override repair value for a given RS k/o/n (min/success/total).

Can be used as a flag.

func (*RepairOverride) Set

func (ro *RepairOverride) Set(s string) error

Set sets the value from a string in the format k/o/n-override (min/optimal/total-repairOverride).

func (*RepairOverride) String

func (ro *RepairOverride) String() string

String is required for pflag.Value.

func (RepairOverride) Type

func (RepairOverride) Type() string

Type implements pflag.Value.

type RepairOverrides

type RepairOverrides struct {
	List []RepairOverride
}

RepairOverrides is a configuration struct that contains a list of override repair values for various given RS combinations of k/o/n (min/success/total).

Can be used as a flag.

func (*RepairOverrides) GetMap

func (ros *RepairOverrides) GetMap() RepairOverridesMap

GetMap creates a RepairOverridesMap from the config.

func (*RepairOverrides) Set

func (ros *RepairOverrides) Set(s string) error

Set sets the value from a string in the format "k/o/n-override,k/o/n-override,...".

func (*RepairOverrides) String

func (ros *RepairOverrides) String() string

String is required for pflag.Value. It is a comma separated list of RepairOverride configs.

func (RepairOverrides) Type

func (RepairOverrides) Type() string

Type implements pflag.Value.

type RepairOverridesMap

type RepairOverridesMap struct {
	// contains filtered or unexported fields
}

RepairOverridesMap is derived from the RepairOverrides config, and is used for quickly retrieving repair override values.

func (*RepairOverridesMap) GetOverrideValue

func (rom *RepairOverridesMap) GetOverrideValue(rs storj.RedundancyScheme) int32

GetOverrideValue returns the override value for an RS scheme if it exists, or 0 otherwise.

func (*RepairOverridesMap) GetOverrideValuePB

func (rom *RepairOverridesMap) GetOverrideValuePB(rs *pb.RedundancyScheme) int32

GetOverrideValuePB returns the override value for a pb RS scheme if it exists, or 0 otherwise.