overlay

package
v1.93.0-rc Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 22, 2023 License: AGPL-3.0 Imports: 23 Imported by: 2

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var ErrEmptyNode = errs.New("empty node ID")

ErrEmptyNode is returned when the nodeID is empty.

View Source 
var ErrLowDifficulty = errs.Class("node id difficulty too low")

ErrLowDifficulty is when the node id's difficulty is too low.

View Source 
var ErrNodeDisqualified = errs.Class("node is disqualified")

ErrNodeDisqualified is returned if a nodes is disqualified.

View Source 
var ErrNodeFinishedGE = errs.Class("node finished graceful exit")

ErrNodeFinishedGE is returned if a node has finished graceful exit.

View Source 
var ErrNodeNotFound = errs.Class("node not found")

ErrNodeNotFound is returned if a node does not exist in database.

View Source 
var ErrNodeOffline = errs.Class("node is offline")

ErrNodeOffline is returned if a nodes is offline.

View Source 
var ErrNotEnoughNodes = errs.Class("not enough nodes")

ErrNotEnoughNodes is when selecting nodes failed with the given parameters.

View Source 
var ErrPlacement = errs.Class("placement")

ErrPlacement is used for placement definition related parsing errors.

View Source 
var (

	// Error represents an overlay error.
	Error = errs.Class("overlay")
)

Functions

func MaskOffLastNet added in v1.75.2 

func MaskOffLastNet(config NodeSelectionConfig, addr net.IP, port string) (string, error)

MaskOffLastNet truncates the target address to the configured CIDR ipv6Cidr or ipv6Cidr prefix, if DistinctIP is enabled in the config. Otherwise, it returns the joined IP and port.

func ResolveIPAndNetwork added in v0.35.2 

func ResolveIPAndNetwork(ctx context.Context, target string, config NodeSelectionConfig, lastNetFunc LastNetFunc) (ip net.IP, port, network string, err error)

ResolveIPAndNetwork resolves the target address and determines its IP and appropriate last_net, as indicated.

Types

type AsOfSystemTimeConfig added in v1.20.1 

type AsOfSystemTimeConfig struct {
	Enabled         bool          `help:"enables the use of the AS OF SYSTEM TIME feature in CRDB" default:"true"`
	DefaultInterval time.Duration `help:"default duration for AS OF SYSTEM TIME" devDefault:"-1ms" releaseDefault:"-10s" testDefault:"-1µs"`
}

AsOfSystemTimeConfig is a configuration struct to enable 'AS OF SYSTEM TIME' for CRDB queries.

func (*AsOfSystemTimeConfig) Interval added in v1.57.8 

func (aost *AsOfSystemTimeConfig) Interval() time.Duration

Interval returns the configured interval respecting Enabled property.

type Config 

type Config struct {
	Node                            NodeSelectionConfig
	NodeSelectionCache              UploadSelectionCacheConfig
	GeoIP                           GeoIPConfig
	UpdateStatsBatchSize            int           `help:"number of update requests to process per transaction" default:"100"`
	NodeCheckInWaitPeriod           time.Duration `` /* 152-byte string literal not displayed */
	NodeSoftwareUpdateEmailCooldown time.Duration `help:"the amount of time to wait between sending Node Software Update emails" default:"168h"`
	RepairExcludedCountryCodes      []string      `help:"list of country codes to exclude nodes from target repair selection" default:"" testDefault:"FR,BE"`
	SendNodeEmails                  bool          `help:"whether to send emails to nodes" default:"false"`
	MinimumNewNodeIDDifficulty      int           `` /* 126-byte string literal not displayed */
	AsOfSystemTime                  time.Duration `help:"default AS OF SYSTEM TIME for service" default:"-10s" testDefault:"0"`
}

Config is a configuration for overlay service.

type ConfigurablePlacementRule added in v1.84.1 

type ConfigurablePlacementRule struct {
	PlacementRules string
}

ConfigurablePlacementRule is a string configuration includes all placement rules in the form of id1:def1,id2:def2...

func (ConfigurablePlacementRule) Parse added in v1.89.2 

func (c ConfigurablePlacementRule) Parse() (*PlacementDefinitions, error)

Parse creates the PlacementDefinitions from the string rules.

func (*ConfigurablePlacementRule) Set added in v1.84.1 

func (c *ConfigurablePlacementRule) Set(s string) error

Set implements pflag.Value.

func (*ConfigurablePlacementRule) String added in v1.84.1 

func (c *ConfigurablePlacementRule) String() string

String implements pflag.Value.

func (*ConfigurablePlacementRule) Type added in v1.84.1 

func (c *ConfigurablePlacementRule) Type() string

Type implements pflag.Value.

type DB 

type DB interface {
	// GetOnlineNodesForAuditRepair returns a map of nodes for the supplied nodeIDs.
	// The return value contains necessary information to create orders as well as nodes'
	// current reputation status.
	GetOnlineNodesForAuditRepair(ctx context.Context, nodeIDs []storj.NodeID, onlineWindow time.Duration) (map[storj.NodeID]*NodeReputation, error)
	// SelectStorageNodes looks up nodes based on criteria
	SelectStorageNodes(ctx context.Context, totalNeededNodes, newNodeCount int, criteria *NodeCriteria) ([]*nodeselection.SelectedNode, error)
	// SelectAllStorageNodesUpload returns all nodes that qualify to store data, organized as reputable nodes and new nodes
	SelectAllStorageNodesUpload(ctx context.Context, selectionCfg NodeSelectionConfig) (reputable, new []*nodeselection.SelectedNode, err error)
	// SelectAllStorageNodesDownload returns a nodes that are ready for downloading
	SelectAllStorageNodesDownload(ctx context.Context, onlineWindow time.Duration, asOf AsOfSystemTimeConfig) ([]*nodeselection.SelectedNode, error)

	// Get looks up the node by nodeID
	Get(ctx context.Context, nodeID storj.NodeID) (*NodeDossier, error)
	// GetNodes gets records for all specified nodes as of the given system interval. The
	// onlineWindow is used to determine whether each node is marked as Online. The results are
	// returned in a slice of the same length as the input nodeIDs, and each index of the returned
	// list corresponds to the same index in nodeIDs. If a node is not known, or is disqualified
	// or exited, the corresponding returned SelectedNode will have a zero value.
	GetNodes(ctx context.Context, nodeIDs storj.NodeIDList, onlineWindow, asOfSystemInterval time.Duration) (_ []nodeselection.SelectedNode, err error)
	// GetParticipatingNodes returns all known participating nodes (this includes all known nodes
	// excluding nodes that have been disqualified or gracefully exited).
	GetParticipatingNodes(ctx context.Context, onlineWindow, asOfSystemInterval time.Duration) (_ []nodeselection.SelectedNode, err error)
	// UpdateReputation updates the DB columns for all reputation fields in ReputationStatus.
	UpdateReputation(ctx context.Context, id storj.NodeID, request ReputationUpdate) error
	// UpdateNodeInfo updates node dossier with info requested from the node itself like node type, email, wallet, capacity, and version.
	UpdateNodeInfo(ctx context.Context, node storj.NodeID, nodeInfo *InfoResponse) (stats *NodeDossier, err error)
	// UpdateCheckIn updates a single storagenode's check-in stats.
	UpdateCheckIn(ctx context.Context, node NodeCheckInInfo, timestamp time.Time, config NodeSelectionConfig) (err error)
	// SetNodeContained updates the contained field for the node record.
	SetNodeContained(ctx context.Context, node storj.NodeID, contained bool) (err error)
	// SetAllContainedNodes updates the contained field for all nodes, as necessary.
	SetAllContainedNodes(ctx context.Context, containedNodes []storj.NodeID) (err error)

	// ActiveNodesPieceCounts returns a map of node IDs to piece counts from the db.
	// Returns only pieces for nodes that are not disqualified.
	ActiveNodesPieceCounts(ctx context.Context) (pieceCounts map[storj.NodeID]int64, err error)
	// UpdatePieceCounts sets the piece count field for the given node IDs.
	UpdatePieceCounts(ctx context.Context, pieceCounts map[storj.NodeID]int64) (err error)

	// UpdateExitStatus is used to update a node's graceful exit status.
	UpdateExitStatus(ctx context.Context, request *ExitStatusRequest) (_ *NodeDossier, err error)
	// GetExitingNodes returns nodes who have initiated a graceful exit, but have not completed it.
	GetExitingNodes(ctx context.Context) (exitingNodes []*ExitStatus, err error)
	// GetGracefulExitCompletedByTimeFrame returns nodes who have completed graceful exit within a time window (time window is around graceful exit completion).
	GetGracefulExitCompletedByTimeFrame(ctx context.Context, begin, end time.Time) (exitedNodes storj.NodeIDList, err error)
	// GetGracefulExitIncompleteByTimeFrame returns nodes who have initiated, but not completed graceful exit within a time window (time window is around graceful exit initiation).
	GetGracefulExitIncompleteByTimeFrame(ctx context.Context, begin, end time.Time) (exitingNodes storj.NodeIDList, err error)
	// GetExitStatus returns a node's graceful exit status.
	GetExitStatus(ctx context.Context, nodeID storj.NodeID) (exitStatus *ExitStatus, err error)

	// GetNodesNetwork returns the last_net subnet for each storage node, order is not guaranteed.
	GetNodesNetwork(ctx context.Context, nodeIDs []storj.NodeID) (nodeNets []string, err error)
	// GetNodesNetworkInOrder returns the last_net subnet for each storage node in order of the requested nodeIDs.
	GetNodesNetworkInOrder(ctx context.Context, nodeIDs []storj.NodeID) (nodeNets []string, err error)

	// DisqualifyNode disqualifies a storage node.
	DisqualifyNode(ctx context.Context, nodeID storj.NodeID, disqualifiedAt time.Time, reason DisqualificationReason) (email string, err error)

	// GetOfflineNodesForEmail gets offline nodes in need of an email.
	GetOfflineNodesForEmail(ctx context.Context, offlineWindow time.Duration, cutoff time.Duration, cooldown time.Duration, limit int) (nodes map[storj.NodeID]string, err error)
	// UpdateLastOfflineEmail updates last_offline_email for a list of nodes.
	UpdateLastOfflineEmail(ctx context.Context, nodeIDs storj.NodeIDList, timestamp time.Time) (err error)

	// DQNodesLastSeenBefore disqualifies a limited number of nodes where last_contact_success < cutoff except those already disqualified
	// or gracefully exited or where last_contact_success = '0001-01-01 00:00:00+00'.
	DQNodesLastSeenBefore(ctx context.Context, cutoff time.Time, limit int) (nodeEmails map[storj.NodeID]string, count int, err error)

	// TestSuspendNodeUnknownAudit suspends a storage node for unknown audits.
	TestSuspendNodeUnknownAudit(ctx context.Context, nodeID storj.NodeID, suspendedAt time.Time) (err error)
	// TestUnsuspendNodeUnknownAudit unsuspends a storage node for unknown audits.
	TestUnsuspendNodeUnknownAudit(ctx context.Context, nodeID storj.NodeID) (err error)

	// TestVetNode directly sets a node's vetted_at timestamp to make testing easier
	TestVetNode(ctx context.Context, nodeID storj.NodeID) (vettedTime *time.Time, err error)
	// TestUnvetNode directly sets a node's vetted_at timestamp to null to make testing easier
	TestUnvetNode(ctx context.Context, nodeID storj.NodeID) (err error)
	// TestSuspendNodeOffline directly sets a node's offline_suspended timestamp to make testing easier
	TestSuspendNodeOffline(ctx context.Context, nodeID storj.NodeID, suspendedAt time.Time) (err error)
	// TestNodeCountryCode sets node country code.
	TestNodeCountryCode(ctx context.Context, nodeID storj.NodeID, countryCode string) (err error)
	// TestUpdateCheckInDirectUpdate tries to update a node info directly. Returns true if it succeeded, false if there were no node with the provided (used for testing).
	TestUpdateCheckInDirectUpdate(ctx context.Context, node NodeCheckInInfo, timestamp time.Time, semVer version.SemVer, walletFeatures string) (updated bool, err error)
	// OneTimeFixLastNets updates the last_net values for all node records to be equal to their
	// last_ip_port values.
	OneTimeFixLastNets(ctx context.Context) error

	// IterateAllContactedNodes will call cb on all known nodes (used in restore trash contexts).
	IterateAllContactedNodes(context.Context, func(context.Context, *nodeselection.SelectedNode) error) error
	// IterateAllNodeDossiers will call cb on all known nodes (used for invoice generation).
	IterateAllNodeDossiers(context.Context, func(context.Context, *NodeDossier) error) error

	// UpdateNodeTags insert (or refresh) node tags.
	UpdateNodeTags(ctx context.Context, tags nodeselection.NodeTags) error

	// GetNodeTags returns all nodes for a specific node.
	GetNodeTags(ctx context.Context, id storj.NodeID) (nodeselection.NodeTags, error)
}

DB implements the database for overlay.Service.

architecture: Database

type DisqualificationReason added in v1.54.1 

type DisqualificationReason int

DisqualificationReason is disqualification reason enum type. 

const (
	// DisqualificationReasonUnknown denotes undetermined disqualification reason.
	DisqualificationReasonUnknown DisqualificationReason = 0
	// DisqualificationReasonAuditFailure denotes disqualification due to audit score falling below threshold.
	DisqualificationReasonAuditFailure DisqualificationReason = 1
	// DisqualificationReasonSuspension denotes disqualification due to unknown audit failure after grace period for unknown audits
	// has elapsed.
	DisqualificationReasonSuspension DisqualificationReason = 2
	// DisqualificationReasonNodeOffline denotes disqualification due to node's online score falling below threshold after tracking
	// period has elapsed.
	DisqualificationReasonNodeOffline DisqualificationReason = 3
)

type DownloadSelectionCache added in v1.22.1 

type DownloadSelectionCache struct {
	// contains filtered or unexported fields
}

DownloadSelectionCache keeps a list of all the storage nodes that are qualified to download data from. The cache will sync with the nodes table in the database and get refreshed once the staleness time has past.

func NewDownloadSelectionCache added in v1.22.1 

func NewDownloadSelectionCache(log *zap.Logger, db DownloadSelectionDB, placementRules PlacementRules, config DownloadSelectionCacheConfig) (*DownloadSelectionCache, error)

NewDownloadSelectionCache creates a new cache that keeps a list of all the storage nodes that are qualified to download data from.

func (*DownloadSelectionCache) GetNodeIPsFromPlacement added in v1.81.2 

func (cache *DownloadSelectionCache) GetNodeIPsFromPlacement(ctx context.Context, nodes []storj.NodeID, placement storj.PlacementConstraint) (_ map[storj.NodeID]string, err error)

GetNodeIPsFromPlacement gets the last node ip:port from the cache, refreshing when needed. Results are filtered out by placement.

func (*DownloadSelectionCache) GetNodes added in v1.60.1 

func (cache *DownloadSelectionCache) GetNodes(ctx context.Context, nodes []storj.NodeID) (_ map[storj.NodeID]*nodeselection.SelectedNode, err error)

GetNodes gets nodes by ID from the cache, and refreshes the cache if it is stale.

func (*DownloadSelectionCache) Refresh added in v1.22.1 

func (cache *DownloadSelectionCache) Refresh(ctx context.Context) (err error)

Refresh populates the cache with all of the reputableNodes and newNode nodes This method is useful for tests.

func (*DownloadSelectionCache) Run added in v1.60.1 

func (cache *DownloadSelectionCache) Run(ctx context.Context) (err error)

Run runs the background task for cache.

func (*DownloadSelectionCache) Size added in v1.22.1 

func (cache *DownloadSelectionCache) Size(ctx context.Context) (int, error)

Size returns how many nodes are in the cache.

type DownloadSelectionCacheConfig added in v1.22.1 

type DownloadSelectionCacheConfig struct {
	Staleness      time.Duration
	OnlineWindow   time.Duration
	AsOfSystemTime AsOfSystemTimeConfig
}

DownloadSelectionCacheConfig contains configuration for the selection cache.

type DownloadSelectionCacheState added in v1.22.1 

type DownloadSelectionCacheState struct {
	// contains filtered or unexported fields
}

DownloadSelectionCacheState contains state of download selection cache.

func NewDownloadSelectionCacheState added in v1.22.1 

func NewDownloadSelectionCacheState(nodes []*nodeselection.SelectedNode) *DownloadSelectionCacheState

NewDownloadSelectionCacheState creates a new state from the nodes.

func (*DownloadSelectionCacheState) FilteredIPs added in v1.84.1 

func (state *DownloadSelectionCacheState) FilteredIPs(nodes []storj.NodeID, filter nodeselection.NodeFilter) map[storj.NodeID]string

FilteredIPs returns node ip:port for nodes that are in state. Results are filtered out..

func (*DownloadSelectionCacheState) IPs added in v1.22.1 

func (state *DownloadSelectionCacheState) IPs(nodes []storj.NodeID) map[storj.NodeID]string

IPs returns node ip:port for nodes that are in state.

func (*DownloadSelectionCacheState) Nodes added in v1.60.1 

func (state *DownloadSelectionCacheState) Nodes(nodes []storj.NodeID) map[storj.NodeID]*nodeselection.SelectedNode

Nodes returns node ip:port for nodes that are in state.

func (*DownloadSelectionCacheState) Size added in v1.22.1 

func (state *DownloadSelectionCacheState) Size() int

Size returns how many nodes are in the state.

type DownloadSelectionDB added in v1.22.1 

type DownloadSelectionDB interface {
	// SelectAllStorageNodesDownload returns nodes that are ready for downloading
	SelectAllStorageNodesDownload(ctx context.Context, onlineWindow time.Duration, asOf AsOfSystemTimeConfig) ([]*nodeselection.SelectedNode, error)
}

DownloadSelectionDB implements the database for download selection cache.

architecture: Database

type ExitStatus added in v0.24.0 

type ExitStatus struct {
	NodeID              storj.NodeID
	ExitInitiatedAt     *time.Time
	ExitLoopCompletedAt *time.Time
	ExitFinishedAt      *time.Time
	ExitSuccess         bool
}

ExitStatus is used for reading graceful exit status.

type ExitStatusRequest added in v0.22.0 

type ExitStatusRequest struct {
	NodeID              storj.NodeID
	ExitInitiatedAt     time.Time
	ExitLoopCompletedAt time.Time
	ExitFinishedAt      time.Time
	ExitSuccess         bool
}

ExitStatusRequest is used to update a node's graceful exit status.

type FindStorageNodesRequest 

type FindStorageNodesRequest struct {
	RequestedCount     int
	ExcludedIDs        []storj.NodeID
	MinimumVersion     string        // semver or empty
	AsOfSystemInterval time.Duration // only used for CRDB queries
	Placement          storj.PlacementConstraint
}

FindStorageNodesRequest defines easy request parameters.

type GeoIPConfig added in v1.44.1 

type GeoIPConfig struct {
	DB            string   `help:"the location of the maxmind database containing geoip country information"`
	MockCountries []string `help:"a mock list of countries the satellite will attribute to nodes (useful for testing)"`
}

GeoIPConfig is a configuration struct that helps configure the GeoIP lookup features on the satellite.

type InfoResponse added in v1.7.1 

type InfoResponse struct {
	Operator *pb.NodeOperator
	Capacity *pb.NodeCapacity
	Version  *pb.NodeVersion
}

InfoResponse contains node dossier info requested from the storage node.

type LastNetFunc added in v1.75.2 

type LastNetFunc func(config NodeSelectionConfig, ip net.IP, port string) (string, error)

LastNetFunc is the type of a function that will be used to derive a network from an ip and port.

type NodeCheckInInfo added in v0.21.3 

type NodeCheckInInfo struct {
	NodeID                  storj.NodeID
	Address                 *pb.NodeAddress
	LastNet                 string
	LastIPPort              string
	IsUp                    bool
	Operator                *pb.NodeOperator
	Capacity                *pb.NodeCapacity
	Version                 *pb.NodeVersion
	CountryCode             location.CountryCode
	SoftwareUpdateEmailSent bool
	VersionBelowMin         bool
}

NodeCheckInInfo contains all the info that will be updated when a node checkins.

type NodeCriteria 

type NodeCriteria struct {
	FreeDisk           int64
	ExcludedIDs        []storj.NodeID
	ExcludedNetworks   []string // the /24 subnet IPv4 or /64 subnet IPv6 for nodes
	MinimumVersion     string   // semver or empty
	OnlineWindow       time.Duration
	AsOfSystemInterval time.Duration // only used for CRDB queries
}

NodeCriteria are the requirements for selecting nodes.

type NodeDossier 

type NodeDossier struct {
	pb.Node
	Operator                pb.NodeOperator
	Capacity                pb.NodeCapacity
	Reputation              NodeStats
	Version                 pb.NodeVersion
	Contained               bool
	Disqualified            *time.Time
	DisqualificationReason  *DisqualificationReason
	UnknownAuditSuspended   *time.Time
	OfflineSuspended        *time.Time
	OfflineUnderReview      *time.Time
	PieceCount              int64
	ExitStatus              ExitStatus
	CreatedAt               time.Time
	LastNet                 string
	LastIPPort              string
	LastOfflineEmail        *time.Time
	LastSoftwareUpdateEmail *time.Time
	CountryCode             location.CountryCode
}

NodeDossier is the complete info that the satellite tracks for a storage node.

type NodeLastContact added in v0.29.0 

type NodeLastContact struct {
	URL                storj.NodeURL
	LastIPPort         string
	LastContactSuccess time.Time
	LastContactFailure time.Time
}

NodeLastContact contains the ID, address, and timestamp.

type NodeReputation added in v1.47.3 

type NodeReputation struct {
	ID         storj.NodeID
	Address    *pb.NodeAddress
	LastNet    string
	LastIPPort string
	Reputation ReputationStatus
}

NodeReputation is used as a result for creating orders limits for audits.

type NodeSelectionConfig 

type NodeSelectionConfig struct {
	NewNodeFraction   float64       `help:"the fraction of new nodes allowed per request" releaseDefault:"0.05" devDefault:"1"`
	MinimumVersion    string        `help:"the minimum node software version for node selection queries" default:""`
	OnlineWindow      time.Duration `help:"the amount of time without seeing a node before its considered offline" default:"4h" testDefault:"1m"`
	DistinctIP        bool          `help:"require distinct IPs when choosing nodes for upload" releaseDefault:"true" devDefault:"false"`
	NetworkPrefixIPv4 int           `help:"the prefix to use in determining 'network' for IPv4 addresses" default:"24" hidden:"true"`
	NetworkPrefixIPv6 int           `help:"the prefix to use in determining 'network' for IPv6 addresses" default:"64" hidden:"true"`
	MinimumDiskSpace  memory.Size   `help:"how much disk space a node at minimum must have to be selected for upload" default:"500.00MB" testDefault:"100.00MB"`

	AsOfSystemTime AsOfSystemTimeConfig

	UploadExcludedCountryCodes []string `help:"list of country codes to exclude from node selection for uploads" default:"" testDefault:"FR,BE"`
}

NodeSelectionConfig is a configuration struct to determine the minimum values for nodes to select.

type NodeStats 

type NodeStats struct {
	Latency90          int64
	LastContactSuccess time.Time
	LastContactFailure time.Time
	OfflineUnderReview *time.Time
	Status             ReputationStatus
}

NodeStats contains statistics about a node.

type PeerIdentities added in v0.19.0 

type PeerIdentities interface {
	// Set adds a peer identity entry for a node
	Set(context.Context, storj.NodeID, *identity.PeerIdentity) error
	// Get gets peer identity
	Get(context.Context, storj.NodeID) (*identity.PeerIdentity, error)
	// BatchGet gets all nodes peer identities in a transaction
	BatchGet(context.Context, storj.NodeIDList) ([]*identity.PeerIdentity, error)
}

PeerIdentities stores storagenode peer identities.

architecture: Database

type PlacementDefinitions added in v1.89.2 

type PlacementDefinitions struct {
	// contains filtered or unexported fields
}

PlacementDefinitions can include the placement definitions for each known identifier.

func NewPlacementDefinitions added in v1.89.2 

func NewPlacementDefinitions() *PlacementDefinitions

NewPlacementDefinitions creates a fully initialized NewPlacementDefinitions.

func (*PlacementDefinitions) AddLegacyStaticRules added in v1.89.2 

func (d *PlacementDefinitions) AddLegacyStaticRules()

AddLegacyStaticRules initializes all the placement rules defined earlier in static golang code.

func (*PlacementDefinitions) AddPlacementFromString added in v1.89.2 

func (d *PlacementDefinitions) AddPlacementFromString(definitions string) error

AddPlacementFromString parses placement definition form string representations from id:definition;id:definition;...

func (*PlacementDefinitions) AddPlacementRule added in v1.89.2 

func (d *PlacementDefinitions) AddPlacementRule(id storj.PlacementConstraint, filter nodeselection.NodeFilter)

AddPlacementRule registers a new placement.

func (*PlacementDefinitions) CreateFilters added in v1.89.2 

func (d *PlacementDefinitions) CreateFilters(constraint storj.PlacementConstraint) (filter nodeselection.NodeFilter)

CreateFilters implements PlacementCondition.

func (*PlacementDefinitions) SupportedPlacements added in v1.90.1 

func (d *PlacementDefinitions) SupportedPlacements() (res []storj.PlacementConstraint)

SupportedPlacements returns all the IDs, which have associated placement rules.

type PlacementRules added in v1.84.1 

type PlacementRules func(constraint storj.PlacementConstraint) (filter nodeselection.NodeFilter)

PlacementRules can crate filter based on the placement identifier.

type ReputationStatus added in v1.34.1 

type ReputationStatus struct {
	Email                  string
	Disqualified           *time.Time
	DisqualificationReason *DisqualificationReason
	UnknownAuditSuspended  *time.Time
	OfflineSuspended       *time.Time
	VettedAt               *time.Time
}

ReputationStatus indicates current reputation status for a node.

type ReputationUpdate added in v1.54.1 

type ReputationUpdate struct {
	Disqualified           *time.Time
	DisqualificationReason DisqualificationReason
	UnknownAuditSuspended  *time.Time
	OfflineSuspended       *time.Time
	VettedAt               *time.Time
}

ReputationUpdate contains reputation update data for a node.

type Service added in v0.17.0 

type Service struct {
	GeoIP                  geoip.IPToCountry
	UploadSelectionCache   *UploadSelectionCache
	DownloadSelectionCache *DownloadSelectionCache
	LastNetFunc            LastNetFunc
	// contains filtered or unexported fields
}

Service is used to store and handle node information.

architecture: Service

func NewService added in v0.17.0 

func NewService(log *zap.Logger, db DB, nodeEvents nodeevents.DB, placementRules PlacementRules, satelliteAddr, satelliteName string, config Config) (*Service, error)

NewService returns a new Service.

func (*Service) CachedGetOnlineNodesForGet added in v1.60.1 

func (service *Service) CachedGetOnlineNodesForGet(ctx context.Context, nodeIDs []storj.NodeID) (_ map[storj.NodeID]*nodeselection.SelectedNode, err error)

CachedGetOnlineNodesForGet returns a map of nodes from the download selection cache from the suppliedIDs.

func (*Service) Close added in v0.17.0 

func (service *Service) Close() error

Close closes resources.

func (*Service) DQNodesLastSeenBefore added in v1.67.1 

func (service *Service) DQNodesLastSeenBefore(ctx context.Context, cutoff time.Time, limit int) (count int, err error)

DQNodesLastSeenBefore disqualifies nodes who have not been contacted since the cutoff time.

func (*Service) DisqualifyNode added in v0.29.0 

func (service *Service) DisqualifyNode(ctx context.Context, nodeID storj.NodeID, reason DisqualificationReason) (err error)

DisqualifyNode disqualifies a storage node.

func (*Service) FindStorageNodesForGracefulExit added in v1.4.1 

func (service *Service) FindStorageNodesForGracefulExit(ctx context.Context, req FindStorageNodesRequest) (_ []*nodeselection.SelectedNode, err error)

FindStorageNodesForGracefulExit searches the overlay network for nodes that meet the provided requirements for graceful-exit requests.

func (*Service) FindStorageNodesForUpload added in v1.4.1 

func (service *Service) FindStorageNodesForUpload(ctx context.Context, req FindStorageNodesRequest) (_ []*nodeselection.SelectedNode, err error)

FindStorageNodesForUpload searches the for nodes in the cache that meet the provided requirements for upload.

func (*Service) Get added in v0.17.0 

func (service *Service) Get(ctx context.Context, nodeID storj.NodeID) (_ *NodeDossier, err error)

Get looks up the provided nodeID from the overlay.

func (*Service) GetLocationFromPlacement added in v1.87.1 

func (service *Service) GetLocationFromPlacement(placement storj.PlacementConstraint) string

GetLocationFromPlacement returns the value for `nodeselection.Location` that placement is currently annotated with.

func (*Service) GetNodeIPsFromPlacement added in v1.81.2 

func (service *Service) GetNodeIPsFromPlacement(ctx context.Context, nodeIDs []storj.NodeID, placement storj.PlacementConstraint) (_ map[storj.NodeID]string, err error)

GetNodeIPsFromPlacement returns a map of node ip:port for the supplied nodeIDs. Results are filtered out by placement.

func (*Service) GetNodeTags added in v1.83.2 

func (service *Service) GetNodeTags(ctx context.Context, id storj.NodeID) (nodeselection.NodeTags, error)

GetNodeTags returns the node tags of a node.

func (*Service) GetNodes added in v1.88.2 

func (service *Service) GetNodes(ctx context.Context, nodeIDs storj.NodeIDList) (records []nodeselection.SelectedNode, err error)

GetNodes gets records for all specified nodes. The configured OnlineWindow is used to determine whether each node is marked as Online. The results are returned in a slice of the same length as the input nodeIDs, and each index of the returned list corresponds to the same index in nodeIDs. If a node is not known, or is disqualified or exited, the corresponding returned SelectedNode will have a zero value.

func (*Service) GetOnlineNodesForAuditRepair added in v1.47.3 

func (service *Service) GetOnlineNodesForAuditRepair(ctx context.Context, nodeIDs []storj.NodeID) (_ map[storj.NodeID]*NodeReputation, err error)

GetOnlineNodesForAuditRepair returns a map of nodes for the supplied nodeIDs.

func (*Service) GetParticipatingNodes added in v1.88.2 

func (service *Service) GetParticipatingNodes(ctx context.Context) (records []nodeselection.SelectedNode, err error)

GetParticipatingNodes returns all known participating nodes (this includes all known nodes excluding nodes that have been disqualified or gracefully exited).

func (*Service) InsertOfflineNodeEvents added in v1.68.1 

func (service *Service) InsertOfflineNodeEvents(ctx context.Context, cooldown time.Duration, cutoff time.Duration, limit int) (count int, err error)

InsertOfflineNodeEvents inserts offline events into node events.

func (*Service) IsOnline added in v0.17.0 

func (service *Service) IsOnline(node *NodeDossier) bool

IsOnline checks if a node is 'online' based on the collected statistics.

func (*Service) ResolveIPAndNetwork added in v1.75.2 

func (service *Service) ResolveIPAndNetwork(ctx context.Context, target string) (ip net.IP, port, network string, err error)

ResolveIPAndNetwork resolves the target address and determines its IP and appropriate subnet IPv4 or subnet IPv6.

func (*Service) Run added in v1.60.1 

func (service *Service) Run(ctx context.Context) error

Run runs the background processes needed for caches.

func (*Service) SelectAllStorageNodesDownload added in v1.65.1 

func (service *Service) SelectAllStorageNodesDownload(ctx context.Context, onlineWindow time.Duration, asOf AsOfSystemTimeConfig) (_ []*nodeselection.SelectedNode, err error)

SelectAllStorageNodesDownload returns a nodes that are ready for downloading.

func (*Service) SetNodeContained added in v1.69.1 

func (service *Service) SetNodeContained(ctx context.Context, node storj.NodeID, contained bool) (err error)

SetNodeContained updates the contained field for the node record. If `contained` is true, the contained field in the record is set to the current database time, if it is not already set. If `contained` is false, the contained field in the record is set to NULL. All other fields are left alone.

func (*Service) TestNodeCountryCode added in v1.50.1 

func (service *Service) TestNodeCountryCode(ctx context.Context, nodeID storj.NodeID, countryCode string) (err error)

TestNodeCountryCode directly sets a node's vetted_at timestamp to null to make testing easier.

func (*Service) TestUnvetNode added in v1.13.1 

func (service *Service) TestUnvetNode(ctx context.Context, nodeID storj.NodeID) (err error)

TestUnvetNode directly sets a node's vetted_at timestamp to null to make testing easier.

func (*Service) TestVetNode added in v1.13.1 

func (service *Service) TestVetNode(ctx context.Context, nodeID storj.NodeID) (vettedTime *time.Time, err error)

TestVetNode directly sets a node's vetted_at timestamp to make testing easier.

func (*Service) UpdateCheckIn added in v0.21.3 

func (service *Service) UpdateCheckIn(ctx context.Context, node NodeCheckInInfo, timestamp time.Time) (err error)

UpdateCheckIn updates a single storagenode's check-in info if needed.

The check-in info is updated in the database if: 

(1) there is no previous entry and the node is allowed (id difficulty, etc);
(2) it has been too long since the last known entry; or
(3) the node hostname, IP address, port, wallet, sw version, or disk capacity
has changed.

Note that there can be a race between acquiring the previous entry and performing the update, so if two updates happen at about the same time it is not defined which one will end up in the database.

func (*Service) UpdateNodeInfo added in v0.17.0 

func (service *Service) UpdateNodeInfo(ctx context.Context, node storj.NodeID, nodeInfo *InfoResponse) (stats *NodeDossier, err error)

UpdateNodeInfo updates node dossier with info requested from the node itself like node type, email, wallet, capacity, and version.

func (*Service) UpdateNodeTags added in v1.83.2 

func (service *Service) UpdateNodeTags(ctx context.Context, tags []nodeselection.NodeTag) error

UpdateNodeTags persists all new and old node tags.

func (*Service) UpdateReputation added in v1.34.1 

func (service *Service) UpdateReputation(ctx context.Context, id storj.NodeID, email string, request ReputationUpdate, reputationChanges []nodeevents.Type) (err error)

UpdateReputation updates the DB columns for any of the reputation fields.

type UploadSelectionCache added in v1.22.1 

type UploadSelectionCache struct {
	// contains filtered or unexported fields
}

UploadSelectionCache keeps a list of all the storage nodes that are qualified to store data We organize the nodes by if they are reputable or a new node on the network. The cache will sync with the nodes table in the database and get refreshed once the staleness time has past.

func NewUploadSelectionCache added in v1.22.1 

func NewUploadSelectionCache(log *zap.Logger, db UploadSelectionDB, staleness time.Duration, config NodeSelectionConfig, defaultFilter nodeselection.NodeFilters, placementRules PlacementRules) (*UploadSelectionCache, error)

NewUploadSelectionCache creates a new cache that keeps a list of all the storage nodes that are qualified to store data.

func (*UploadSelectionCache) GetNodes added in v1.22.1 

func (cache *UploadSelectionCache) GetNodes(ctx context.Context, req FindStorageNodesRequest) (_ []*nodeselection.SelectedNode, err error)

GetNodes selects nodes from the cache that will be used to upload a file. Every node selected will be from a distinct network. If the cache hasn't been refreshed recently it will do so first.

func (*UploadSelectionCache) Refresh added in v1.22.1 

func (cache *UploadSelectionCache) Refresh(ctx context.Context) (err error)

Refresh populates the cache with all of the reputableNodes and newNode nodes This method is useful for tests.

func (*UploadSelectionCache) Run added in v1.60.1 

func (cache *UploadSelectionCache) Run(ctx context.Context) (err error)

Run runs the background task for cache.

type UploadSelectionCacheConfig added in v1.22.1 

type UploadSelectionCacheConfig struct {
	Disabled  bool          `help:"disable node cache" default:"false" deprecated:"true"`
	Staleness time.Duration `help:"how stale the node selection cache can be" releaseDefault:"3m" devDefault:"5m" testDefault:"3m"`
}

UploadSelectionCacheConfig is a configuration for upload selection cache.

type UploadSelectionDB added in v1.22.1 

type UploadSelectionDB interface {
	// SelectAllStorageNodesUpload returns all nodes that qualify to store data, organized as reputable nodes and new nodes
	SelectAllStorageNodesUpload(ctx context.Context, selectionCfg NodeSelectionConfig) (reputable, new []*nodeselection.SelectedNode, err error)
}

UploadSelectionDB implements the database for upload selection cache.

architecture: Database

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.