overlay

package
v0.19.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 29, 2019 License: AGPL-3.0 Imports: 12 Imported by: 2

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var ErrBucketNotFound = errs.New("bucket not found")

ErrBucketNotFound is returned if a bucket is unable to be found in the routing table

View Source 
var ErrEmptyNode = errs.New("empty node ID")

ErrEmptyNode is returned when the nodeID is empty

View Source 
var ErrNodeDisqualified = errs.Class("node is disqualified")

ErrNodeDisqualified is returned if a nodes is disqualified

View Source 
var ErrNodeNotFound = errs.Class("node not found")

ErrNodeNotFound is returned if a node does not exist in database

View Source 
var ErrNodeOffline = errs.Class("node is offline")

ErrNodeOffline is returned if a nodes is offline

View Source 
var ErrNotEnoughNodes = errs.Class("not enough nodes")

ErrNotEnoughNodes is when selecting nodes failed with the given parameters

View Source 
var (

	// Error represents an overlay error
	Error = errs.Class("overlay error")
)

Functions

func GetNetwork 

func GetNetwork(ctx context.Context, target string) (network string, err error)

GetNetwork resolves the target address and determines its IP /24 Subnet

Types

type CombinedCache added in v0.17.0 

type CombinedCache struct {
	DB
	// contains filtered or unexported fields
}

CombinedCache is a simple caching mechanism for overlaycache updates. It provdes methods to help reduce calls to UpdateAddress and UpdateTime, but can be extended for other calls in the future.

func NewCombinedCache added in v0.17.0 

func NewCombinedCache(db DB) *CombinedCache

NewCombinedCache instantiates a new CombinedCache

func (*CombinedCache) UpdateAddress added in v0.17.0 

func (c *CombinedCache) UpdateAddress(ctx context.Context, info *pb.Node, defaults NodeSelectionConfig) (err error)

UpdateAddress overrides the underlying db.UpdateAddress and provides a simple caching layer to reduce calls to the underlying db. The cache is guaranteed to match the values held in the database; however this code does not guarantee that concurrent UpdateAddress calls will be handled in any particular order.

type Config 

type Config struct {
	Node                 NodeSelectionConfig
	UpdateStatsBatchSize int `help:"number of update requests to process per transaction" default:"100"`
}

Config is a configuration for overlay service.

type DB 

type DB interface {
	// SelectStorageNodes looks up nodes based on criteria
	SelectStorageNodes(ctx context.Context, count int, criteria *NodeCriteria) ([]*pb.Node, error)
	// SelectNewStorageNodes looks up nodes based on new node criteria
	SelectNewStorageNodes(ctx context.Context, count int, criteria *NodeCriteria) ([]*pb.Node, error)

	// Get looks up the node by nodeID
	Get(ctx context.Context, nodeID storj.NodeID) (*NodeDossier, error)
	// KnownOffline filters a set of nodes to offline nodes
	KnownOffline(context.Context, *NodeCriteria, storj.NodeIDList) (storj.NodeIDList, error)
	// KnownUnreliableOrOffline filters a set of nodes to unhealth or offlines node, independent of new
	KnownUnreliableOrOffline(context.Context, *NodeCriteria, storj.NodeIDList) (storj.NodeIDList, error)
	// Reliable returns all nodes that are reliable
	Reliable(context.Context, *NodeCriteria) (storj.NodeIDList, error)
	// Paginate will page through the database nodes
	Paginate(ctx context.Context, offset int64, limit int) ([]*NodeDossier, bool, error)
	// PaginateQualified will page through the qualified nodes
	PaginateQualified(ctx context.Context, offset int64, limit int) ([]*pb.Node, bool, error)
	// IsVetted returns whether or not the node reaches reputable thresholds
	IsVetted(ctx context.Context, id storj.NodeID, criteria *NodeCriteria) (bool, error)
	// Update updates node address
	UpdateAddress(ctx context.Context, value *pb.Node, defaults NodeSelectionConfig) error
	// BatchUpdateStats updates multiple storagenode's stats in one transaction
	BatchUpdateStats(ctx context.Context, updateRequests []*UpdateRequest, batchSize int) (failed storj.NodeIDList, err error)
	// UpdateStats all parts of single storagenode's stats.
	UpdateStats(ctx context.Context, request *UpdateRequest) (stats *NodeStats, err error)
	// UpdateNodeInfo updates node dossier with info requested from the node itself like node type, email, wallet, capacity, and version.
	UpdateNodeInfo(ctx context.Context, node storj.NodeID, nodeInfo *pb.InfoResponse) (stats *NodeDossier, err error)
	// UpdateUptime updates a single storagenode's uptime stats.
	UpdateUptime(ctx context.Context, nodeID storj.NodeID, isUp bool, lambda, weight, uptimeDQ float64) (stats *NodeStats, err error)

	// AllPieceCounts returns a map of node IDs to piece counts from the db.
	AllPieceCounts(ctx context.Context) (pieceCounts map[storj.NodeID]int, err error)
	// UpdatePieceCounts sets the piece count field for the given node IDs.
	UpdatePieceCounts(ctx context.Context, pieceCounts map[storj.NodeID]int) (err error)
}

DB implements the database for overlay.Service

type FindStorageNodesRequest 

type FindStorageNodesRequest struct {
	MinimumRequiredNodes int
	RequestedCount       int
	FreeBandwidth        int64
	FreeDisk             int64
	ExcludedNodes        []storj.NodeID
	MinimumVersion       string // semver or empty
}

FindStorageNodesRequest defines easy request parameters.

type Inspector 

type Inspector struct {
	// contains filtered or unexported fields
}

Inspector is a gRPC service for inspecting overlay internals

func NewInspector 

func NewInspector(service *Service) *Inspector

NewInspector creates an Inspector

func (*Inspector) CountNodes 

func (srv *Inspector) CountNodes(ctx context.Context, req *pb.CountNodesRequest) (_ *pb.CountNodesResponse, err error)

CountNodes returns the number of nodes in the overlay.

func (*Inspector) DumpNodes 

func (srv *Inspector) DumpNodes(ctx context.Context, req *pb.DumpNodesRequest) (_ *pb.DumpNodesResponse, err error)

DumpNodes returns all of the nodes in the overlay.

type KeyLock added in v0.17.0 

type KeyLock struct {
	// contains filtered or unexported fields
}

KeyLock provides per-key RW locking. Locking is key-specific, meaning lock contention only exists for locking calls using the same key parameter. As with all locks, do not call Unlock() or RUnlock() before a corresponding Lock() or RLock() call.

Note on memory usage: internally KeyLock lazily and atomically creates a separate sync.RWMutex for each key. To maintain synchronization guarantees, these interal mutexes are not freed until the entire KeyLock instance is freed.

func NewKeyLock added in v0.17.0 

func NewKeyLock() *KeyLock

NewKeyLock create a new KeyLock

func (*KeyLock) Lock added in v0.17.0 

func (l *KeyLock) Lock(nodeID storj.NodeID) UnlockFunc

Lock the provided key. Returns the unlock function.

func (*KeyLock) RLock added in v0.17.0 

func (l *KeyLock) RLock(nodeID storj.NodeID) UnlockFunc

RLock the provided key. Returns the unlock function.

type NodeCriteria 

type NodeCriteria struct {
	FreeBandwidth  int64
	FreeDisk       int64
	AuditCount     int64
	UptimeCount    int64
	ExcludedNodes  []storj.NodeID
	ExcludedIPs    []string
	MinimumVersion string // semver or empty
	OnlineWindow   time.Duration
	DistinctIP     bool
}

NodeCriteria are the requirements for selecting nodes

type NodeDossier 

type NodeDossier struct {
	pb.Node
	Type         pb.NodeType
	Operator     pb.NodeOperator
	Capacity     pb.NodeCapacity
	Reputation   NodeStats
	Version      pb.NodeVersion
	Contained    bool
	Disqualified *time.Time
	PieceCount   int64
}

NodeDossier is the complete info that the satellite tracks for a storage node

type NodeSelectionConfig 

type NodeSelectionConfig struct {
	UptimeCount       int64         `` /* 127-byte string literal not displayed */
	AuditCount        int64         `help:"the number of times a node has been audited to not be considered a New Node" releaseDefault:"100" devDefault:"0"`
	NewNodePercentage float64       `help:"the percentage of new nodes allowed per request" default:"0.05"` // TODO: fix, this is not percentage, it's ratio
	MinimumVersion    string        `help:"the minimum node software version for node selection queries" default:""`
	OnlineWindow      time.Duration `help:"the amount of time without seeing a node before its considered offline" default:"1h"`
	DistinctIP        bool          `help:"require distinct IPs when choosing nodes for upload" releaseDefault:"true" devDefault:"false"`

	AuditReputationRepairWeight  float64 `help:"weight to apply to audit reputation for total repair reputation calculation" default:"1.0"`
	AuditReputationUplinkWeight  float64 `help:"weight to apply to audit reputation for total uplink reputation calculation" default:"1.0"`
	AuditReputationAlpha0        float64 `help:"the initial shape 'alpha' used to calculate audit SNs reputation" default:"1.0"`
	AuditReputationBeta0         float64 `help:"the initial shape 'beta' value used to calculate audit SNs reputation" default:"0.0"`
	AuditReputationLambda        float64 `help:"the forgetting factor used to calculate the audit SNs reputation" default:"0.95"`
	AuditReputationWeight        float64 `help:"the normalization weight used to calculate the audit SNs reputation" default:"1.0"`
	AuditReputationDQ            float64 `help:"the reputation cut-off for disqualifying SNs based on audit history" default:"0.6"`
	UptimeReputationRepairWeight float64 `help:"weight to apply to uptime reputation for total repair reputation calculation" default:"1.0"`
	UptimeReputationUplinkWeight float64 `help:"weight to apply to uptime reputation for total uplink reputation calculation" default:"1.0"`
	UptimeReputationAlpha0       float64 `help:"the initial shape 'alpha' used to calculate uptime SNs reputation" default:"2.0"`
	UptimeReputationBeta0        float64 `help:"the initial shape 'beta' value used to calculate uptime SNs reputation" default:"0.0"`
	UptimeReputationLambda       float64 `help:"the forgetting factor used to calculate the uptime SNs reputation" default:"0.99"`
	UptimeReputationWeight       float64 `help:"the normalization weight used to calculate the uptime SNs reputation" default:"1.0"`
	UptimeReputationDQ           float64 `help:"the reputation cut-off for disqualifying SNs based on uptime history" default:"0"`
}

NodeSelectionConfig is a configuration struct to determine the minimum values for nodes to select

type NodeStats 

type NodeStats struct {
	Latency90             int64
	AuditSuccessCount     int64
	AuditCount            int64
	UptimeSuccessCount    int64
	UptimeCount           int64
	LastContactSuccess    time.Time
	LastContactFailure    time.Time
	AuditReputationAlpha  float64
	UptimeReputationAlpha float64
	AuditReputationBeta   float64
	UptimeReputationBeta  float64
	Disqualified          *time.Time
}

NodeStats contains statistics about a node.

type PeerIdentities added in v0.19.0 

type PeerIdentities interface {
	// Set adds a peer identity entry for a node
	Set(context.Context, storj.NodeID, *identity.PeerIdentity) error
	// Get gets peer identity
	Get(context.Context, storj.NodeID) (*identity.PeerIdentity, error)
	// BatchGet gets all nodes peer identities in a transaction
	BatchGet(context.Context, storj.NodeIDList) ([]*identity.PeerIdentity, error)
}

PeerIdentities stores storagenode peer identities

type Service added in v0.17.0 

type Service struct {
	// contains filtered or unexported fields
}

Service is used to store and handle node information

func NewService added in v0.17.0 

func NewService(log *zap.Logger, db DB, config Config) *Service

NewService returns a new Service

func (*Service) BatchUpdateStats added in v0.17.0 

func (service *Service) BatchUpdateStats(ctx context.Context, requests []*UpdateRequest) (failed storj.NodeIDList, err error)

BatchUpdateStats updates multiple storagenode's stats in one transaction

func (*Service) Close added in v0.17.0 

func (service *Service) Close() error

Close closes resources

func (*Service) ConnFailure added in v0.17.0 

func (service *Service) ConnFailure(ctx context.Context, node *pb.Node, failureError error)

ConnFailure implements the Transport Observer `ConnFailure` function

func (*Service) ConnSuccess added in v0.17.0 

func (service *Service) ConnSuccess(ctx context.Context, node *pb.Node)

ConnSuccess implements the Transport Observer `ConnSuccess` function

func (*Service) FindStorageNodes added in v0.17.0 

func (service *Service) FindStorageNodes(ctx context.Context, req FindStorageNodesRequest) (_ []*pb.Node, err error)

FindStorageNodes searches the overlay network for nodes that meet the provided requirements

func (*Service) FindStorageNodesWithPreferences added in v0.17.0 

func (service *Service) FindStorageNodesWithPreferences(ctx context.Context, req FindStorageNodesRequest, preferences *NodeSelectionConfig) (nodes []*pb.Node, err error)

FindStorageNodesWithPreferences searches the overlay network for nodes that meet the provided criteria

func (*Service) Get added in v0.17.0 

func (service *Service) Get(ctx context.Context, nodeID storj.NodeID) (_ *NodeDossier, err error)

Get looks up the provided nodeID from the overlay.

func (*Service) GetMissingPieces added in v0.17.0 

func (service *Service) GetMissingPieces(ctx context.Context, pieces []*pb.RemotePiece) (missingPieces []int32, err error)

GetMissingPieces returns the list of offline nodes

func (*Service) Inspect added in v0.17.0 

func (service *Service) Inspect(ctx context.Context) (_ storage.Keys, err error)

Inspect lists limited number of items in the cache

func (*Service) IsOnline added in v0.17.0 

func (service *Service) IsOnline(node *NodeDossier) bool

IsOnline checks if a node is 'online' based on the collected statistics.

func (*Service) IsVetted added in v0.17.0 

func (service *Service) IsVetted(ctx context.Context, nodeID storj.NodeID) (reputable bool, err error)

IsVetted returns whether or not the node reaches reputable thresholds

func (*Service) KnownOffline added in v0.17.0 

func (service *Service) KnownOffline(ctx context.Context, nodeIds storj.NodeIDList) (offlineNodes storj.NodeIDList, err error)

KnownOffline filters a set of nodes to offline nodes

func (*Service) KnownUnreliableOrOffline added in v0.17.0 

func (service *Service) KnownUnreliableOrOffline(ctx context.Context, nodeIds storj.NodeIDList) (badNodes storj.NodeIDList, err error)

KnownUnreliableOrOffline filters a set of nodes to unhealth or offlines node, independent of new.

func (*Service) Paginate added in v0.17.0 

func (service *Service) Paginate(ctx context.Context, offset int64, limit int) (_ []*NodeDossier, _ bool, err error)

Paginate returns a list of `limit` nodes starting from `start` offset.

func (*Service) PaginateQualified added in v0.17.0 

func (service *Service) PaginateQualified(ctx context.Context, offset int64, limit int) (_ []*pb.Node, _ bool, err error)

PaginateQualified returns a list of `limit` qualified nodes starting from `start` offset.

func (*Service) Put added in v0.17.0 

func (service *Service) Put(ctx context.Context, nodeID storj.NodeID, value pb.Node) (err error)

Put adds a node id and proto definition into the overlay.

func (*Service) Reliable added in v0.17.0 

func (service *Service) Reliable(ctx context.Context) (nodes storj.NodeIDList, err error)

Reliable filters a set of nodes that are reliable, independent of new.

func (*Service) UpdateNodeInfo added in v0.17.0 

func (service *Service) UpdateNodeInfo(ctx context.Context, node storj.NodeID, nodeInfo *pb.InfoResponse) (stats *NodeDossier, err error)

UpdateNodeInfo updates node dossier with info requested from the node itself like node type, email, wallet, capacity, and version.

func (*Service) UpdateStats added in v0.17.0 

func (service *Service) UpdateStats(ctx context.Context, request *UpdateRequest) (stats *NodeStats, err error)

UpdateStats all parts of single storagenode's stats.

func (*Service) UpdateUptime added in v0.17.0 

func (service *Service) UpdateUptime(ctx context.Context, nodeID storj.NodeID, isUp bool) (stats *NodeStats, err error)

UpdateUptime updates a single storagenode's uptime stats.

type UnlockFunc added in v0.17.0 

type UnlockFunc func()

UnlockFunc is the function to unlock the associated successful lock

type UpdateRequest 

type UpdateRequest struct {
	NodeID       storj.NodeID
	AuditSuccess bool
	IsUp         bool
	// n.b. these are set values from the satellite.
	// They are part of the UpdateRequest struct in order to be
	// more easily accessible in satellite/satellitedb/overlaycache.go.
	AuditLambda  float64
	AuditWeight  float64
	AuditDQ      float64
	UptimeLambda float64
	UptimeWeight float64
	UptimeDQ     float64
}

UpdateRequest is used to update a node status.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.