overlay

package
v0.26.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 20, 2019 License: AGPL-3.0 Imports: 12 Imported by: 2

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var ErrEmptyNode = errs.New("empty node ID")

ErrEmptyNode is returned when the nodeID is empty

View Source 
var ErrNodeDisqualified = errs.Class("node is disqualified")

ErrNodeDisqualified is returned if a nodes is disqualified

View Source 
var ErrNodeNotFound = errs.Class("node not found")

ErrNodeNotFound is returned if a node does not exist in database

View Source 
var ErrNodeOffline = errs.Class("node is offline")

ErrNodeOffline is returned if a nodes is offline

View Source 
var ErrNotEnoughNodes = errs.Class("not enough nodes")

ErrNotEnoughNodes is when selecting nodes failed with the given parameters

View Source 
var (

	// Error represents an overlay error
	Error = errs.Class("overlay error")
)

Functions

func GetNetwork 

func GetNetwork(ctx context.Context, target string) (network string, err error)

GetNetwork resolves the target address and determines its IP /24 Subnet

Types

type CombinedCache added in v0.17.0 

type CombinedCache struct {
	DB
	// contains filtered or unexported fields
}

CombinedCache is a simple caching mechanism for overlaycache updates. It provdes methods to help reduce calls to UpdateAddress and UpdateTime, but can be extended for other calls in the future.

architecture: Service

func NewCombinedCache added in v0.17.0 

func NewCombinedCache(db DB) *CombinedCache

NewCombinedCache instantiates a new CombinedCache

func (*CombinedCache) UpdateAddress added in v0.17.0 

func (c *CombinedCache) UpdateAddress(ctx context.Context, info *pb.Node, defaults NodeSelectionConfig) (err error)

UpdateAddress overrides the underlying db.UpdateAddress and provides a simple caching layer to reduce calls to the underlying db. The cache is guaranteed to match the values held in the database; however this code does not guarantee that concurrent UpdateAddress calls will be handled in any particular order.

type Config 

type Config struct {
	Node                 NodeSelectionConfig
	UpdateStatsBatchSize int `help:"number of update requests to process per transaction" default:"100"`
}

Config is a configuration for overlay service.

type DB 

type DB interface {
	// SelectStorageNodes looks up nodes based on criteria
	SelectStorageNodes(ctx context.Context, count int, criteria *NodeCriteria) ([]*pb.Node, error)
	// SelectNewStorageNodes looks up nodes based on new node criteria
	SelectNewStorageNodes(ctx context.Context, count int, criteria *NodeCriteria) ([]*pb.Node, error)

	// Get looks up the node by nodeID
	Get(ctx context.Context, nodeID storj.NodeID) (*NodeDossier, error)
	// KnownOffline filters a set of nodes to offline nodes
	KnownOffline(context.Context, *NodeCriteria, storj.NodeIDList) (storj.NodeIDList, error)
	// KnownUnreliableOrOffline filters a set of nodes to unhealth or offlines node, independent of new
	KnownUnreliableOrOffline(context.Context, *NodeCriteria, storj.NodeIDList) (storj.NodeIDList, error)
	// Reliable returns all nodes that are reliable
	Reliable(context.Context, *NodeCriteria) (storj.NodeIDList, error)
	// Paginate will page through the database nodes
	Paginate(ctx context.Context, offset int64, limit int) ([]*NodeDossier, bool, error)
	// PaginateQualified will page through the qualified nodes
	PaginateQualified(ctx context.Context, offset int64, limit int) ([]*pb.Node, bool, error)
	// Update updates node address
	UpdateAddress(ctx context.Context, value *pb.Node, defaults NodeSelectionConfig) error
	// BatchUpdateStats updates multiple storagenode's stats in one transaction
	BatchUpdateStats(ctx context.Context, updateRequests []*UpdateRequest, batchSize int) (failed storj.NodeIDList, err error)
	// UpdateStats all parts of single storagenode's stats.
	UpdateStats(ctx context.Context, request *UpdateRequest) (stats *NodeStats, err error)
	// UpdateNodeInfo updates node dossier with info requested from the node itself like node type, email, wallet, capacity, and version.
	UpdateNodeInfo(ctx context.Context, node storj.NodeID, nodeInfo *pb.InfoResponse) (stats *NodeDossier, err error)
	// UpdateUptime updates a single storagenode's uptime stats.
	UpdateUptime(ctx context.Context, nodeID storj.NodeID, isUp bool, lambda, weight, uptimeDQ float64) (stats *NodeStats, err error)
	// UpdateCheckIn updates a single storagenode's check-in stats.
	UpdateCheckIn(ctx context.Context, node NodeCheckInInfo, timestamp time.Time, config NodeSelectionConfig) (err error)

	// AllPieceCounts returns a map of node IDs to piece counts from the db.
	AllPieceCounts(ctx context.Context) (pieceCounts map[storj.NodeID]int, err error)
	// UpdatePieceCounts sets the piece count field for the given node IDs.
	UpdatePieceCounts(ctx context.Context, pieceCounts map[storj.NodeID]int) (err error)

	// UpdateExitStatus is used to update a node's graceful exit status.
	UpdateExitStatus(ctx context.Context, request *ExitStatusRequest) (_ *NodeDossier, err error)
	// GetExitingNodes returns nodes who have initiated a graceful exit, but have not completed it.
	GetExitingNodes(ctx context.Context) (exitingNodes []*ExitStatus, err error)
	// GetGracefulExitCompletedByTimeFrame returns nodes who have completed graceful exit within a time window (time window is around graceful exit completion).
	GetGracefulExitCompletedByTimeFrame(ctx context.Context, begin, end time.Time) (exitedNodes storj.NodeIDList, err error)
	// GetGracefulExitIncompleteByTimeFrame returns nodes who have initiated, but not completed graceful exit within a time window (time window is around graceful exit initiation).
	GetGracefulExitIncompleteByTimeFrame(ctx context.Context, begin, end time.Time) (exitingNodes storj.NodeIDList, err error)
	// GetExitStatus returns a node's graceful exit status.
	GetExitStatus(ctx context.Context, nodeID storj.NodeID) (exitStatus *ExitStatus, err error)

	// GetNodeIPs returns a list of IP addresses associated with given node IDs.
	GetNodeIPs(ctx context.Context, nodeIDs []storj.NodeID) (nodeIPs []string, err error)
}

DB implements the database for overlay.Service

architecture: Database

type ExitStatus added in v0.24.0 

type ExitStatus struct {
	NodeID              storj.NodeID
	ExitInitiatedAt     *time.Time
	ExitLoopCompletedAt *time.Time
	ExitFinishedAt      *time.Time
	ExitSuccess         bool
}

ExitStatus is used for reading graceful exit status.

type ExitStatusRequest added in v0.22.0 

type ExitStatusRequest struct {
	NodeID              storj.NodeID
	ExitInitiatedAt     time.Time
	ExitLoopCompletedAt time.Time
	ExitFinishedAt      time.Time
	ExitSuccess         bool
}

ExitStatusRequest is used to update a node's graceful exit status.

type FindStorageNodesRequest 

type FindStorageNodesRequest struct {
	MinimumRequiredNodes int
	RequestedCount       int
	FreeBandwidth        int64
	FreeDisk             int64
	ExcludedNodes        []storj.NodeID
	MinimumVersion       string // semver or empty
}

FindStorageNodesRequest defines easy request parameters.

type Inspector 

type Inspector struct {
	// contains filtered or unexported fields
}

Inspector is a gRPC service for inspecting overlay internals

architecture: Endpoint

func NewInspector 

func NewInspector(service *Service) *Inspector

NewInspector creates an Inspector

func (*Inspector) CountNodes 

func (srv *Inspector) CountNodes(ctx context.Context, req *pb.CountNodesRequest) (_ *pb.CountNodesResponse, err error)

CountNodes returns the number of nodes in the overlay.

func (*Inspector) DumpNodes 

func (srv *Inspector) DumpNodes(ctx context.Context, req *pb.DumpNodesRequest) (_ *pb.DumpNodesResponse, err error)

DumpNodes returns all of the nodes in the overlay.

type KeyLock added in v0.17.0 

type KeyLock struct {
	// contains filtered or unexported fields
}

KeyLock provides per-key RW locking. Locking is key-specific, meaning lock contention only exists for locking calls using the same key parameter. As with all locks, do not call Unlock() or RUnlock() before a corresponding Lock() or RLock() call.

Note on memory usage: internally KeyLock lazily and atomically creates a separate sync.RWMutex for each key. To maintain synchronization guarantees, these interal mutexes are not freed until the entire KeyLock instance is freed.

func NewKeyLock added in v0.17.0 

func NewKeyLock() *KeyLock

NewKeyLock create a new KeyLock

func (*KeyLock) Lock added in v0.17.0 

func (l *KeyLock) Lock(nodeID storj.NodeID) UnlockFunc

Lock the provided key. Returns the unlock function.

func (*KeyLock) RLock added in v0.17.0 

func (l *KeyLock) RLock(nodeID storj.NodeID) UnlockFunc

RLock the provided key. Returns the unlock function.

type NodeCheckInInfo added in v0.21.3 

type NodeCheckInInfo struct {
	NodeID   storj.NodeID
	Address  *pb.NodeAddress
	LastIP   string
	IsUp     bool
	Operator *pb.NodeOperator
	Capacity *pb.NodeCapacity
	Version  *pb.NodeVersion
}

NodeCheckInInfo contains all the info that will be updated when a node checkins

type NodeCriteria 

type NodeCriteria struct {
	FreeBandwidth  int64
	FreeDisk       int64
	AuditCount     int64
	UptimeCount    int64
	ExcludedNodes  []storj.NodeID
	ExcludedIPs    []string
	MinimumVersion string // semver or empty
	OnlineWindow   time.Duration
	DistinctIP     bool
}

NodeCriteria are the requirements for selecting nodes

type NodeDossier 

type NodeDossier struct {
	pb.Node
	Type         pb.NodeType
	Operator     pb.NodeOperator
	Capacity     pb.NodeCapacity
	Reputation   NodeStats
	Version      pb.NodeVersion
	Contained    bool
	Disqualified *time.Time
	PieceCount   int64
	ExitStatus   ExitStatus
	CreatedAt    time.Time
}

NodeDossier is the complete info that the satellite tracks for a storage node

type NodeSelectionConfig 

type NodeSelectionConfig struct {
	UptimeCount       int64         `` /* 127-byte string literal not displayed */
	AuditCount        int64         `help:"the number of times a node has been audited to not be considered a New Node" releaseDefault:"100" devDefault:"0"`
	NewNodePercentage float64       `help:"the percentage of new nodes allowed per request" default:"0.05"` // TODO: fix, this is not percentage, it's ratio
	MinimumVersion    string        `help:"the minimum node software version for node selection queries" default:""`
	OnlineWindow      time.Duration `help:"the amount of time without seeing a node before its considered offline" default:"4h"`
	DistinctIP        bool          `help:"require distinct IPs when choosing nodes for upload" releaseDefault:"true" devDefault:"false"`

	AuditReputationRepairWeight  float64 `help:"weight to apply to audit reputation for total repair reputation calculation" default:"1.0"`
	AuditReputationUplinkWeight  float64 `help:"weight to apply to audit reputation for total uplink reputation calculation" default:"1.0"`
	AuditReputationAlpha0        float64 `help:"the initial shape 'alpha' used to calculate audit SNs reputation" default:"1.0"`
	AuditReputationBeta0         float64 `help:"the initial shape 'beta' value used to calculate audit SNs reputation" default:"0.0"`
	AuditReputationLambda        float64 `help:"the forgetting factor used to calculate the audit SNs reputation" default:"0.95"`
	AuditReputationWeight        float64 `help:"the normalization weight used to calculate the audit SNs reputation" default:"1.0"`
	AuditReputationDQ            float64 `help:"the reputation cut-off for disqualifying SNs based on audit history" default:"0.6"`
	UptimeReputationRepairWeight float64 `help:"weight to apply to uptime reputation for total repair reputation calculation" default:"1.0"`
	UptimeReputationUplinkWeight float64 `help:"weight to apply to uptime reputation for total uplink reputation calculation" default:"1.0"`
	UptimeReputationAlpha0       float64 `help:"the initial shape 'alpha' used to calculate uptime SNs reputation" default:"2.0"`
	UptimeReputationBeta0        float64 `help:"the initial shape 'beta' value used to calculate uptime SNs reputation" default:"0.0"`
	UptimeReputationLambda       float64 `help:"the forgetting factor used to calculate the uptime SNs reputation" default:"0.99"`
	UptimeReputationWeight       float64 `help:"the normalization weight used to calculate the uptime SNs reputation" default:"1.0"`
	UptimeReputationDQ           float64 `help:"the reputation cut-off for disqualifying SNs based on uptime history" default:"0"`
}

NodeSelectionConfig is a configuration struct to determine the minimum values for nodes to select

type NodeStats 

type NodeStats struct {
	Latency90             int64
	AuditSuccessCount     int64
	AuditCount            int64
	UptimeSuccessCount    int64
	UptimeCount           int64
	LastContactSuccess    time.Time
	LastContactFailure    time.Time
	AuditReputationAlpha  float64
	UptimeReputationAlpha float64
	AuditReputationBeta   float64
	UptimeReputationBeta  float64
	Disqualified          *time.Time
}

NodeStats contains statistics about a node.

type PeerIdentities added in v0.19.0 

type PeerIdentities interface {
	// Set adds a peer identity entry for a node
	Set(context.Context, storj.NodeID, *identity.PeerIdentity) error
	// Get gets peer identity
	Get(context.Context, storj.NodeID) (*identity.PeerIdentity, error)
	// BatchGet gets all nodes peer identities in a transaction
	BatchGet(context.Context, storj.NodeIDList) ([]*identity.PeerIdentity, error)
}

PeerIdentities stores storagenode peer identities

architecture: Database

type Service added in v0.17.0 

type Service struct {
	// contains filtered or unexported fields
}

Service is used to store and handle node information

architecture: Service

func NewService added in v0.17.0 

func NewService(log *zap.Logger, db DB, config Config) *Service

NewService returns a new Service

func (*Service) BatchUpdateStats added in v0.17.0 

func (service *Service) BatchUpdateStats(ctx context.Context, requests []*UpdateRequest) (failed storj.NodeIDList, err error)

BatchUpdateStats updates multiple storagenode's stats in one transaction

func (*Service) Close added in v0.17.0 

func (service *Service) Close() error

Close closes resources

func (*Service) FindStorageNodes added in v0.17.0 

func (service *Service) FindStorageNodes(ctx context.Context, req FindStorageNodesRequest) (_ []*pb.Node, err error)

FindStorageNodes searches the overlay network for nodes that meet the provided requirements

func (*Service) FindStorageNodesWithPreferences added in v0.17.0 

func (service *Service) FindStorageNodesWithPreferences(ctx context.Context, req FindStorageNodesRequest, preferences *NodeSelectionConfig) (nodes []*pb.Node, err error)

FindStorageNodesWithPreferences searches the overlay network for nodes that meet the provided criteria

func (*Service) Get added in v0.17.0 

func (service *Service) Get(ctx context.Context, nodeID storj.NodeID) (_ *NodeDossier, err error)

Get looks up the provided nodeID from the overlay.

func (*Service) GetMissingPieces added in v0.17.0 

func (service *Service) GetMissingPieces(ctx context.Context, pieces []*pb.RemotePiece) (missingPieces []int32, err error)

GetMissingPieces returns the list of offline nodes

func (*Service) Inspect added in v0.17.0 

func (service *Service) Inspect(ctx context.Context) (_ storage.Keys, err error)

Inspect lists limited number of items in the cache

func (*Service) IsOnline added in v0.17.0 

func (service *Service) IsOnline(node *NodeDossier) bool

IsOnline checks if a node is 'online' based on the collected statistics.

func (*Service) KnownOffline added in v0.17.0 

func (service *Service) KnownOffline(ctx context.Context, nodeIds storj.NodeIDList) (offlineNodes storj.NodeIDList, err error)

KnownOffline filters a set of nodes to offline nodes

func (*Service) KnownUnreliableOrOffline added in v0.17.0 

func (service *Service) KnownUnreliableOrOffline(ctx context.Context, nodeIds storj.NodeIDList) (badNodes storj.NodeIDList, err error)

KnownUnreliableOrOffline filters a set of nodes to unhealth or offlines node, independent of new.

func (*Service) Paginate added in v0.17.0 

func (service *Service) Paginate(ctx context.Context, offset int64, limit int) (_ []*NodeDossier, _ bool, err error)

Paginate returns a list of `limit` nodes starting from `start` offset.

func (*Service) PaginateQualified added in v0.17.0 

func (service *Service) PaginateQualified(ctx context.Context, offset int64, limit int) (_ []*pb.Node, _ bool, err error)

PaginateQualified returns a list of `limit` qualified nodes starting from `start` offset.

func (*Service) Put added in v0.17.0 

func (service *Service) Put(ctx context.Context, nodeID storj.NodeID, value pb.Node) (err error)

Put adds a node id and proto definition into the overlay.

func (*Service) Reliable added in v0.17.0 

func (service *Service) Reliable(ctx context.Context) (nodes storj.NodeIDList, err error)

Reliable filters a set of nodes that are reliable, independent of new.

func (*Service) UpdateCheckIn added in v0.21.3 

func (service *Service) UpdateCheckIn(ctx context.Context, node NodeCheckInInfo, timestamp time.Time) (err error)

UpdateCheckIn updates a single storagenode's check-in info.

func (*Service) UpdateNodeInfo added in v0.17.0 

func (service *Service) UpdateNodeInfo(ctx context.Context, node storj.NodeID, nodeInfo *pb.InfoResponse) (stats *NodeDossier, err error)

UpdateNodeInfo updates node dossier with info requested from the node itself like node type, email, wallet, capacity, and version.

func (*Service) UpdateStats added in v0.17.0 

func (service *Service) UpdateStats(ctx context.Context, request *UpdateRequest) (stats *NodeStats, err error)

UpdateStats all parts of single storagenode's stats.

func (*Service) UpdateUptime added in v0.17.0 

func (service *Service) UpdateUptime(ctx context.Context, nodeID storj.NodeID, isUp bool) (stats *NodeStats, err error)

UpdateUptime updates a single storagenode's uptime stats.

type UnlockFunc added in v0.17.0 

type UnlockFunc func()

UnlockFunc is the function to unlock the associated successful lock

type UpdateRequest 

type UpdateRequest struct {
	NodeID       storj.NodeID
	AuditSuccess bool
	IsUp         bool
	// n.b. these are set values from the satellite.
	// They are part of the UpdateRequest struct in order to be
	// more easily accessible in satellite/satellitedb/overlaycache.go.
	AuditLambda  float64
	AuditWeight  float64
	AuditDQ      float64
	UptimeLambda float64
	UptimeWeight float64
	UptimeDQ     float64
}

UpdateRequest is used to update a node status.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.