overlay

package
v0.14.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 24, 2019 License: AGPL-3.0 Imports: 10 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var ErrBucketNotFound = errs.New("bucket not found")

ErrBucketNotFound is returned if a bucket is unable to be found in the routing table

View Source 
var ErrEmptyNode = errs.New("empty node ID")

ErrEmptyNode is returned when the nodeID is empty

View Source 
var ErrNodeDisqualified = errs.Class("node is disqualified")

ErrNodeDisqualified is returned if a nodes is disqualified

View Source 
var ErrNodeNotFound = errs.Class("node not found")

ErrNodeNotFound is returned if a node does not exist in database

View Source 
var ErrNodeOffline = errs.Class("node is offline")

ErrNodeOffline is returned if a nodes is offline

View Source 
var ErrNotEnoughNodes = errs.Class("not enough nodes")

ErrNotEnoughNodes is when selecting nodes failed with the given parameters

View Source 
var (

	// Error represents an overlay error
	Error = errs.Class("overlay error")
)
View Source 
var OverlayError = errs.Class("overlay error")

OverlayError creates class of errors for stack traces

Functions

func GetNetwork added in v0.14.0 

func GetNetwork(ctx context.Context, target string) (network string, err error)

GetNetwork resolves the target address and determines its IP /24 Subnet

Types

type Cache 

type Cache struct {
	// contains filtered or unexported fields
}

Cache is used to store and handle node information

func NewCache 

func NewCache(log *zap.Logger, db DB, preferences NodeSelectionConfig) *Cache

NewCache returns a new Cache

func (*Cache) Close 

func (cache *Cache) Close() error

Close closes resources

func (*Cache) ConnFailure 

func (cache *Cache) ConnFailure(ctx context.Context, node *pb.Node, failureError error)

ConnFailure implements the Transport Observer `ConnFailure` function

func (*Cache) ConnSuccess 

func (cache *Cache) ConnSuccess(ctx context.Context, node *pb.Node)

ConnSuccess implements the Transport Observer `ConnSuccess` function

func (*Cache) FindStorageNodes 

func (cache *Cache) FindStorageNodes(ctx context.Context, req FindStorageNodesRequest) (_ []*pb.Node, err error)

FindStorageNodes searches the overlay network for nodes that meet the provided requirements

func (*Cache) FindStorageNodesWithPreferences 

func (cache *Cache) FindStorageNodesWithPreferences(ctx context.Context, req FindStorageNodesRequest, preferences *NodeSelectionConfig) (nodes []*pb.Node, err error)

FindStorageNodesWithPreferences searches the overlay network for nodes that meet the provided criteria

func (*Cache) Get 

func (cache *Cache) Get(ctx context.Context, nodeID storj.NodeID) (_ *NodeDossier, err error)

Get looks up the provided nodeID from the overlay cache

func (*Cache) GetMissingPieces added in v0.11.4 

func (cache *Cache) GetMissingPieces(ctx context.Context, pieces []*pb.RemotePiece) (missingPieces []int32, err error)

GetMissingPieces returns the list of offline nodes

func (*Cache) Inspect 

func (cache *Cache) Inspect(ctx context.Context) (_ storage.Keys, err error)

Inspect lists limited number of items in the cache

func (*Cache) IsOnline added in v0.10.0 

func (cache *Cache) IsOnline(node *NodeDossier) bool

IsOnline checks if a node is 'online' based on the collected statistics.

func (*Cache) IsVetted added in v0.13.0 

func (cache *Cache) IsVetted(ctx context.Context, nodeID storj.NodeID) (reputable bool, err error)

IsVetted returns whether or not the node reaches reputable thresholds

func (*Cache) KnownOffline added in v0.14.0 

func (cache *Cache) KnownOffline(ctx context.Context, nodeIds storj.NodeIDList) (offlineNodes storj.NodeIDList, err error)

KnownOffline filters a set of nodes to offline nodes

func (*Cache) KnownUnreliableOrOffline added in v0.11.0 

func (cache *Cache) KnownUnreliableOrOffline(ctx context.Context, nodeIds storj.NodeIDList) (badNodes storj.NodeIDList, err error)

KnownUnreliableOrOffline filters a set of nodes to unhealth or offlines node, independent of new.

func (*Cache) Paginate 

func (cache *Cache) Paginate(ctx context.Context, offset int64, limit int) (_ []*NodeDossier, _ bool, err error)

Paginate returns a list of `limit` nodes starting from `start` offset.

func (*Cache) Put 

func (cache *Cache) Put(ctx context.Context, nodeID storj.NodeID, value pb.Node) (err error)

Put adds a node id and proto definition into the overlay cache

func (*Cache) UpdateNodeInfo added in v0.9.0 

func (cache *Cache) UpdateNodeInfo(ctx context.Context, node storj.NodeID, nodeInfo *pb.InfoResponse) (stats *NodeDossier, err error)

UpdateNodeInfo updates node dossier with info requested from the node itself like node type, email, wallet, capacity, and version.

func (*Cache) UpdateStats 

func (cache *Cache) UpdateStats(ctx context.Context, request *UpdateRequest) (stats *NodeStats, err error)

UpdateStats all parts of single storagenode's stats.

func (*Cache) UpdateUptime 

func (cache *Cache) UpdateUptime(ctx context.Context, nodeID storj.NodeID, isUp bool) (stats *NodeStats, err error)

UpdateUptime updates a single storagenode's uptime stats.

type Config 

type Config struct {
	Node NodeSelectionConfig
}

Config is a configuration struct for everything you need to start the Overlay cache responsibility.

type DB 

type DB interface {
	// SelectStorageNodes looks up nodes based on criteria
	SelectStorageNodes(ctx context.Context, count int, criteria *NodeCriteria) ([]*pb.Node, error)
	// SelectNewStorageNodes looks up nodes based on new node criteria
	SelectNewStorageNodes(ctx context.Context, count int, criteria *NodeCriteria) ([]*pb.Node, error)

	// Get looks up the node by nodeID
	Get(ctx context.Context, nodeID storj.NodeID) (*NodeDossier, error)
	// KnownOffline filters a set of nodes to offline nodes
	KnownOffline(context.Context, *NodeCriteria, storj.NodeIDList) (storj.NodeIDList, error)
	// KnownUnreliableOrOffline filters a set of nodes to unhealth or offlines node, independent of new
	KnownUnreliableOrOffline(context.Context, *NodeCriteria, storj.NodeIDList) (storj.NodeIDList, error)
	// Paginate will page through the database nodes
	Paginate(ctx context.Context, offset int64, limit int) ([]*NodeDossier, bool, error)
	// IsVetted returns whether or not the node reaches reputable thresholds
	IsVetted(ctx context.Context, id storj.NodeID, criteria *NodeCriteria) (bool, error)
	// Update updates node address
	UpdateAddress(ctx context.Context, value *pb.Node, defaults NodeSelectionConfig) error
	// UpdateStats all parts of single storagenode's stats.
	UpdateStats(ctx context.Context, request *UpdateRequest) (stats *NodeStats, err error)
	// UpdateNodeInfo updates node dossier with info requested from the node itself like node type, email, wallet, capacity, and version.
	UpdateNodeInfo(ctx context.Context, node storj.NodeID, nodeInfo *pb.InfoResponse) (stats *NodeDossier, err error)
	// UpdateUptime updates a single storagenode's uptime stats.
	UpdateUptime(ctx context.Context, nodeID storj.NodeID, isUp bool, lambda, weight, uptimeDQ float64) (stats *NodeStats, err error)
}

DB implements the database for overlay.Cache

type FindStorageNodesRequest 

type FindStorageNodesRequest struct {
	MinimumRequiredNodes int
	RequestedCount       int
	FreeBandwidth        int64
	FreeDisk             int64
	ExcludedNodes        []storj.NodeID
	MinimumVersion       string // semver or empty
}

FindStorageNodesRequest defines easy request parameters.

type Inspector 

type Inspector struct {
	// contains filtered or unexported fields
}

Inspector is a gRPC service for inspecting overlay cache internals

func NewInspector 

func NewInspector(cache *Cache) *Inspector

NewInspector creates an Inspector

func (*Inspector) CountNodes 

func (srv *Inspector) CountNodes(ctx context.Context, req *pb.CountNodesRequest) (_ *pb.CountNodesResponse, err error)

CountNodes returns the number of nodes in the cache

func (*Inspector) DumpNodes 

func (srv *Inspector) DumpNodes(ctx context.Context, req *pb.DumpNodesRequest) (_ *pb.DumpNodesResponse, err error)

DumpNodes returns all of the nodes in the overlay cachea

type NodeCriteria 

type NodeCriteria struct {
	FreeBandwidth  int64
	FreeDisk       int64
	AuditCount     int64
	UptimeCount    int64
	ExcludedNodes  []storj.NodeID
	ExcludedIPs    []string
	MinimumVersion string // semver or empty
	OnlineWindow   time.Duration
	DistinctIP     bool
}

NodeCriteria are the requirements for selecting nodes

type NodeDossier added in v0.9.0 

type NodeDossier struct {
	pb.Node
	Type         pb.NodeType
	Operator     pb.NodeOperator
	Capacity     pb.NodeCapacity
	Reputation   NodeStats
	Version      pb.NodeVersion
	Contained    bool
	Disqualified *time.Time
}

NodeDossier is the complete info that the satellite tracks for a storage node

type NodeSelectionConfig 

type NodeSelectionConfig struct {
	UptimeCount       int64         `` /* 127-byte string literal not displayed */
	AuditCount        int64         `help:"the number of times a node has been audited to not be considered a New Node" releaseDefault:"500" devDefault:"0"`
	NewNodePercentage float64       `help:"the percentage of new nodes allowed per request" default:"0.05"` // TODO: fix, this is not percentage, it's ratio
	MinimumVersion    string        `help:"the minimum node software version for node selection queries" default:""`
	OnlineWindow      time.Duration `help:"the amount of time without seeing a node before its considered offline" default:"1h"`
	DistinctIP        bool          `help:"require distinct IPs when choosing nodes for upload" releaseDefault:"true" devDefault:"false"`

	AuditReputationRepairWeight  float64 `help:"weight to apply to audit reputation for total repair reputation calculation" default:"1.0"`
	AuditReputationUplinkWeight  float64 `help:"weight to apply to audit reputation for total uplink reputation calculation" default:"1.0"`
	AuditReputationAlpha0        float64 `help:"the initial shape 'alpha' used to calculate audit SNs reputation" default:"1.0"`
	AuditReputationBeta0         float64 `help:"the initial shape 'beta' value used to calculate audit SNs reputation" default:"0.0"`
	AuditReputationLambda        float64 `help:"the forgetting factor used to calculate the audit SNs reputation" default:"0.95"`
	AuditReputationWeight        float64 `help:"the normalization weight used to calculate the audit SNs reputation" default:"1.0"`
	AuditReputationDQ            float64 `help:"the reputation cut-off for disqualifying SNs based on audit history" default:"0.6"`
	UptimeReputationRepairWeight float64 `help:"weight to apply to uptime reputation for total repair reputation calculation" default:"1.0"`
	UptimeReputationUplinkWeight float64 `help:"weight to apply to uptime reputation for total uplink reputation calculation" default:"1.0"`
	UptimeReputationAlpha0       float64 `help:"the initial shape 'alpha' used to calculate uptime SNs reputation" default:"2.0"`
	UptimeReputationBeta0        float64 `help:"the initial shape 'beta' value used to calculate uptime SNs reputation" default:"0.0"`
	UptimeReputationLambda       float64 `help:"the forgetting factor used to calculate the uptime SNs reputation" default:"0.99"`
	UptimeReputationWeight       float64 `help:"the normalization weight used to calculate the uptime SNs reputation" default:"1.0"`
	UptimeReputationDQ           float64 `help:"the reputation cut-off for disqualifying SNs based on uptime history" default:"0.4"`
}

NodeSelectionConfig is a configuration struct to determine the minimum values for nodes to select

type NodeStats 

type NodeStats struct {
	Latency90             int64
	AuditSuccessCount     int64
	AuditCount            int64
	UptimeSuccessCount    int64
	UptimeCount           int64
	LastContactSuccess    time.Time
	LastContactFailure    time.Time
	AuditReputationAlpha  float64
	UptimeReputationAlpha float64
	AuditReputationBeta   float64
	UptimeReputationBeta  float64
	Disqualified          *time.Time
}

NodeStats contains statistics about a node.

type UpdateRequest 

type UpdateRequest struct {
	NodeID       storj.NodeID
	AuditSuccess bool
	IsUp         bool
	// n.b. these are set values from the satellite.
	// They are part of the UpdateRequest struct in order to be
	// more easily accessible in satellite/satellitedb/overlaycache.go.
	AuditLambda  float64
	AuditWeight  float64
	AuditDQ      float64
	UptimeLambda float64
	UptimeWeight float64
	UptimeDQ     float64
}

UpdateRequest is used to update a node status.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.