Documentation ¶
Index ¶
- Constants
- Variables
- type BucketStorageTally
- type BucketTally
- type BucketUsage
- type BucketUsageCursor
- type BucketUsagePage
- type BucketUsageRollup
- type CSVRow
- type Cache
- type ProjectAccounting
- type ProjectLimitCache
- func (c *ProjectLimitCache) Get(ctx context.Context, projectID uuid.UUID) (ProjectLimits, error)
- func (c *ProjectLimitCache) GetProjectBandwidthLimit(ctx context.Context, projectID uuid.UUID) (_ memory.Size, err error)
- func (c *ProjectLimitCache) GetProjectLimits(ctx context.Context, projectID uuid.UUID) (_ ProjectLimits, err error)
- func (c *ProjectLimitCache) GetProjectStorageLimit(ctx context.Context, projectID uuid.UUID) (_ memory.Size, err error)
- type ProjectLimitConfig
- type ProjectLimitDB
- type ProjectLimits
- type ProjectUsage
- type Rollup
- type RollupStats
- type Service
- func (usage *Service) AddProjectStorageUsage(ctx context.Context, projectID uuid.UUID, spaceUsed int64) (err error)
- func (usage *Service) ExceedsBandwidthUsage(ctx context.Context, projectID uuid.UUID) (_ bool, limit memory.Size, err error)
- func (usage *Service) ExceedsStorageUsage(ctx context.Context, projectID uuid.UUID) (_ bool, limit memory.Size, err error)
- func (usage *Service) GetProjectBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int) (_ int64, err error)
- func (usage *Service) GetProjectBandwidthLimit(ctx context.Context, projectID uuid.UUID) (_ memory.Size, err error)
- func (usage *Service) GetProjectBandwidthTotals(ctx context.Context, projectID uuid.UUID) (_ int64, err error)
- func (usage *Service) GetProjectBandwidthUsage(ctx context.Context, projectID uuid.UUID) (currentUsed int64, err error)
- func (usage *Service) GetProjectStorageLimit(ctx context.Context, projectID uuid.UUID) (_ memory.Size, err error)
- func (usage *Service) GetProjectStorageTotals(ctx context.Context, projectID uuid.UUID) (total int64, err error)
- func (usage *Service) SetNow(now func() time.Time)
- func (usage *Service) UpdateProjectBandwidthUsage(ctx context.Context, projectID uuid.UUID, increment int64) (err error)
- func (usage *Service) UpdateProjectLimits(ctx context.Context, projectID uuid.UUID, limit memory.Size) (err error)
- type StorageNodePeriodUsage
- type StorageNodeUsage
- type StoragenodeAccounting
- type StoragenodeBandwidthRollup
- type StoragenodeStorageTally
Constants ¶
const ( // LastAtRestTally represents the accounting timestamp for the at-rest data calculation. LastAtRestTally = "LastAtRestTally" // LastBandwidthTally represents the accounting timestamp for the bandwidth allocation query. LastBandwidthTally = "LastBandwidthTally" // LastRollup represents the accounting timestamp for rollup calculations. LastRollup = "LastRollup" )
Constants for accounting_raw, accounting_rollup, and accounting_timestamps.
Variables ¶
var ( // ErrInvalidArgument is returned when a function argument has an invalid // business domain value. ErrInvalidArgument = errs.Class("invalid argument") // ErrSystemOrNetError is returned when the used storage backend returns an // internal system or network error. ErrSystemOrNetError = errs.Class("accounting backend") // ErrKeyNotFound is returned when the key is not found in the cache. ErrKeyNotFound = errs.Class("key not found") // ErrUnexpectedValue is returned when an unexpected value according the // business domain is in the cache. ErrUnexpectedValue = errs.Class("unexpected value") )
var ( // ErrProjectLimitType error for project limit type. ErrProjectLimitType = errs.Class("project limit type") // ErrGetProjectLimit error for getting project limits from database. ErrGetProjectLimit = errs.Class("get project limits") // ErrGetProjectLimitCache error for getting project limits from cache. ErrGetProjectLimitCache = errs.Class("get project limits from cache") )
var ( // ErrProjectUsage general error for project usage. ErrProjectUsage = errs.Class("project usage") )
Functions ¶
This section is empty.
Types ¶
type BucketStorageTally ¶
type BucketStorageTally struct { BucketName string ProjectID uuid.UUID IntervalStart time.Time ObjectCount int64 TotalSegmentCount int64 InlineSegmentCount int64 RemoteSegmentCount int64 TotalBytes int64 InlineBytes int64 RemoteBytes int64 MetadataSize int64 }
BucketStorageTally holds data about a bucket tally.
func (*BucketStorageTally) Bytes ¶ added in v1.34.1
func (s *BucketStorageTally) Bytes() int64
Bytes returns total bytes.
type BucketTally ¶
type BucketTally struct { metabase.BucketLocation ObjectCount int64 TotalSegments int64 InlineSegments int64 RemoteSegments int64 TotalBytes int64 InlineBytes int64 RemoteBytes int64 MetadataSize int64 }
BucketTally contains information about aggregate data stored in a bucket.
func (*BucketTally) Combine ¶
func (s *BucketTally) Combine(o *BucketTally)
Combine aggregates all the tallies.
func (*BucketTally) Segments ¶
func (s *BucketTally) Segments() int64
Segments returns total number of segments.
type BucketUsage ¶
type BucketUsage struct { ProjectID uuid.UUID BucketName string Storage float64 Egress float64 ObjectCount int64 Since time.Time Before time.Time }
BucketUsage consist of total bucket usage for period.
type BucketUsageCursor ¶ added in v0.26.0
BucketUsageCursor holds info for bucket usage cursor pagination.
type BucketUsagePage ¶ added in v0.26.0
type BucketUsagePage struct { BucketUsages []BucketUsage Search string Limit uint Offset uint64 PageCount uint CurrentPage uint TotalCount uint64 }
BucketUsagePage represents bucket usage page result.
type BucketUsageRollup ¶ added in v0.26.0
type BucketUsageRollup struct { ProjectID uuid.UUID BucketName []byte TotalStoredData float64 TotalSegments float64 ObjectCount float64 MetadataSize float64 RepairEgress float64 GetEgress float64 AuditEgress float64 Since time.Time Before time.Time }
BucketUsageRollup is total bucket usage info for certain period.
type CSVRow ¶
type CSVRow struct { NodeID storj.NodeID NodeCreationDate time.Time AtRestTotal float64 GetRepairTotal int64 PutRepairTotal int64 GetAuditTotal int64 PutTotal int64 GetTotal int64 Wallet string Disqualified *time.Time }
CSVRow represents data from QueryPaymentInfo without exposing dbx.
type Cache ¶ added in v0.24.0
type Cache interface { // GetProjectStorageUsage returns the project's storage usage. GetProjectStorageUsage(ctx context.Context, projectID uuid.UUID) (totalUsed int64, err error) // GetProjectBandwidthUsage returns the project's bandwidth usage. GetProjectBandwidthUsage(ctx context.Context, projectID uuid.UUID, now time.Time) (currentUsed int64, err error) // UpdateProjectBandthUsage updates the project's bandwidth usage increasing // it. The projectID is inserted to the increment when it doesn't exists, // hence this method will never return ErrKeyNotFound error's class. UpdateProjectBandwidthUsage(ctx context.Context, projectID uuid.UUID, increment int64, ttl time.Duration, now time.Time) error // AddProjectStorageUsage adds to the projects storage usage the spacedUsed. // The projectID is inserted to the spaceUsed when it doesn't exists, hence // this method will never return ErrKeyNotFound. AddProjectStorageUsage(ctx context.Context, projectID uuid.UUID, spaceUsed int64) error // GetAllProjectTotals return the total projects' storage used space. GetAllProjectTotals(ctx context.Context) (map[uuid.UUID]int64, error) // Close the client, releasing any open resources. Once it's called any other // method must be called. Close() error }
Cache stores live information about project storage which has not yet been synced to ProjectAccounting.
All the implementations must follow the convention of returning errors of one of the classes defined in this package.
All the methods return:
ErrInvalidArgument: an implementation may return if some parameter contain a value which isn't accepted, nonetheless, not all the implementations impose the same constraints on them.
ErrSystemOrNetError: any method will return this if there is an error with the underlining system or the network.
ErrKeyNotFound: returned when a key is not found.
ErrUnexpectedValue: returned when a key or value stored in the underlying system isn't of the expected format or type according the business domain.
architecture: Database
type ProjectAccounting ¶
type ProjectAccounting interface { // SaveTallies saves the latest project info SaveTallies(ctx context.Context, intervalStart time.Time, bucketTallies map[metabase.BucketLocation]*BucketTally) error // GetTallies retrieves all tallies GetTallies(ctx context.Context) ([]BucketTally, error) // CreateStorageTally creates a record for BucketStorageTally in the accounting DB table CreateStorageTally(ctx context.Context, tally BucketStorageTally) error // GetAllocatedBandwidthTotal returns the sum of GET bandwidth usage allocated for a projectID in the past time frame GetAllocatedBandwidthTotal(ctx context.Context, projectID uuid.UUID, from time.Time) (int64, error) // GetProjectBandwidth returns project allocated bandwidth for the specified year, month and day. GetProjectBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int, asOfSystemInterval time.Duration) (int64, error) // GetProjectDailyBandwidth returns bandwidth (allocated and settled) for the specified day. GetProjectDailyBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int) (int64, int64, error) // DeleteProjectBandwidthBefore deletes project bandwidth rollups before the given time DeleteProjectBandwidthBefore(ctx context.Context, before time.Time) error // UpdateProjectUsageLimit updates project usage limit. UpdateProjectUsageLimit(ctx context.Context, projectID uuid.UUID, limit memory.Size) error // UpdateProjectBandwidthLimit updates project bandwidth limit. UpdateProjectBandwidthLimit(ctx context.Context, projectID uuid.UUID, limit memory.Size) error // GetProjectStorageLimit returns project storage usage limit. GetProjectStorageLimit(ctx context.Context, projectID uuid.UUID) (*int64, error) // GetProjectBandwidthLimit returns project bandwidth usage limit. GetProjectBandwidthLimit(ctx context.Context, projectID uuid.UUID) (*int64, error) // GetProjectLimits returns current project limit for both storage and bandwidth. GetProjectLimits(ctx context.Context, projectID uuid.UUID) (ProjectLimits, error) // GetProjectTotal returns project usage summary for specified period of time. GetProjectTotal(ctx context.Context, projectID uuid.UUID, since, before time.Time) (*ProjectUsage, error) // GetBucketUsageRollups returns usage rollup per each bucket for specified period of time. GetBucketUsageRollups(ctx context.Context, projectID uuid.UUID, since, before time.Time) ([]BucketUsageRollup, error) // GetBucketTotals returns per bucket usage summary for specified period of time. GetBucketTotals(ctx context.Context, projectID uuid.UUID, cursor BucketUsageCursor, since, before time.Time) (*BucketUsagePage, error) // ArchiveRollupsBefore archives rollups older than a given time and returns number of bucket bandwidth rollups archived. ArchiveRollupsBefore(ctx context.Context, before time.Time, batchSize int) (numArchivedBucketBW int, err error) // GetRollupsSince retrieves all archived bandwidth rollup records since a given time. A hard limit batch size is used for results. GetRollupsSince(ctx context.Context, since time.Time) ([]orders.BucketBandwidthRollup, error) // GetArchivedRollupsSince retrieves all archived bandwidth rollup records since a given time. A hard limit batch size is used for results. GetArchivedRollupsSince(ctx context.Context, since time.Time) ([]orders.BucketBandwidthRollup, error) }
ProjectAccounting stores information about bandwidth and storage usage for projects.
architecture: Database
type ProjectLimitCache ¶ added in v1.14.1
type ProjectLimitCache struct {
// contains filtered or unexported fields
}
ProjectLimitCache stores the values for both storage usage limit and bandwidth limit for each project ID if they differ from the default limits.
func NewProjectLimitCache ¶ added in v1.14.1
func NewProjectLimitCache(db ProjectLimitDB, defaultMaxUsage, defaultMaxBandwidth memory.Size, config ProjectLimitConfig) *ProjectLimitCache
NewProjectLimitCache creates a new project limit cache to store the project limits for each project ID.
func (*ProjectLimitCache) Get ¶ added in v1.14.1
func (c *ProjectLimitCache) Get(ctx context.Context, projectID uuid.UUID) (ProjectLimits, error)
Get returns the storage usage limit for a project ID.
func (*ProjectLimitCache) GetProjectBandwidthLimit ¶ added in v1.14.1
func (c *ProjectLimitCache) GetProjectBandwidthLimit(ctx context.Context, projectID uuid.UUID) (_ memory.Size, err error)
GetProjectBandwidthLimit return the bandwidth usage limit for a project ID.
func (*ProjectLimitCache) GetProjectLimits ¶ added in v1.14.1
func (c *ProjectLimitCache) GetProjectLimits(ctx context.Context, projectID uuid.UUID) (_ ProjectLimits, err error)
GetProjectLimits returns current project limit for both storage and bandwidth.
func (*ProjectLimitCache) GetProjectStorageLimit ¶ added in v1.14.1
func (c *ProjectLimitCache) GetProjectStorageLimit(ctx context.Context, projectID uuid.UUID) (_ memory.Size, err error)
GetProjectStorageLimit returns the storage usage limit for a project ID.
type ProjectLimitConfig ¶ added in v1.14.1
type ProjectLimitConfig struct { CacheCapacity int `help:"number of projects to cache." releaseDefault:"10000" devDefault:"100"` CacheExpiration time.Duration `help:"how long to cache the project limits." releaseDefault:"10m" devDefault:"30s"` }
ProjectLimitConfig is a configuration struct for project limit.
type ProjectLimitDB ¶ added in v1.14.1
type ProjectLimitDB interface { // GetProjectLimits returns current project limit for both storage and bandwidth. GetProjectLimits(ctx context.Context, projectID uuid.UUID) (ProjectLimits, error) }
ProjectLimitDB stores information about projects limits for storage and bandwidth limits.
architecture: Database
type ProjectLimits ¶ added in v1.14.1
ProjectLimits contains the storage and bandwidth limits.
type ProjectUsage ¶
type ProjectUsage struct { Storage float64 `json:"storage"` Egress int64 `json:"egress"` ObjectCount float64 `json:"objectCount"` Since time.Time `json:"since"` Before time.Time `json:"before"` }
ProjectUsage consist of period total storage, egress and objects count per hour for certain Project in bytes.
type Rollup ¶
type Rollup struct { ID int64 NodeID storj.NodeID StartTime time.Time PutTotal int64 GetTotal int64 GetAuditTotal int64 GetRepairTotal int64 PutRepairTotal int64 AtRestTotal float64 }
Rollup mirrors dbx.AccountingRollup, allowing us to use that struct without leaking dbx.
type RollupStats ¶
RollupStats is a convenience alias.
type Service ¶ added in v0.26.0
type Service struct {
// contains filtered or unexported fields
}
Service is handling project usage related logic.
architecture: Service
func NewService ¶ added in v0.26.0
func NewService(projectAccountingDB ProjectAccounting, liveAccounting Cache, limitCache *ProjectLimitCache, bandwidthCacheTTL, asOfSystemInterval time.Duration) *Service
NewService created new instance of project usage service.
func (*Service) AddProjectStorageUsage ¶ added in v0.26.0
func (usage *Service) AddProjectStorageUsage(ctx context.Context, projectID uuid.UUID, spaceUsed int64) (err error)
AddProjectStorageUsage lets the live accounting know that the given project has just added spaceUsed bytes of storage (from the user's perspective; i.e. segment size).
It can return one of the following errors returned by storj.io/storj/satellite/accounting.Cache.AddProjectStorageUsage, wrapped by ErrProjectUsage.
func (*Service) ExceedsBandwidthUsage ¶ added in v0.26.0
func (usage *Service) ExceedsBandwidthUsage(ctx context.Context, projectID uuid.UUID) (_ bool, limit memory.Size, err error)
ExceedsBandwidthUsage returns true if the bandwidth usage limits have been exceeded for a project in the past month (30 days). The usage limit is (e.g 25GB) multiplied by the redundancy expansion factor, so that the uplinks have a raw limit.
Among others,it can return one of the following errors returned by storj.io/storj/satellite/accounting.Cache except the ErrKeyNotFound, wrapped by ErrProjectUsage.
func (*Service) ExceedsStorageUsage ¶ added in v0.26.0
func (usage *Service) ExceedsStorageUsage(ctx context.Context, projectID uuid.UUID) (_ bool, limit memory.Size, err error)
ExceedsStorageUsage returns true if the storage usage for a project is currently over that project's limit.
func (*Service) GetProjectBandwidth ¶ added in v1.31.1
func (usage *Service) GetProjectBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int) (_ int64, err error)
GetProjectBandwidth returns project allocated bandwidth for the specified year, month and day.
func (*Service) GetProjectBandwidthLimit ¶ added in v0.28.0
func (usage *Service) GetProjectBandwidthLimit(ctx context.Context, projectID uuid.UUID) (_ memory.Size, err error)
GetProjectBandwidthLimit returns current project bandwidth limit.
func (*Service) GetProjectBandwidthTotals ¶ added in v0.28.0
func (usage *Service) GetProjectBandwidthTotals(ctx context.Context, projectID uuid.UUID) (_ int64, err error)
GetProjectBandwidthTotals returns total amount of allocated bandwidth used for past 30 days.
func (*Service) GetProjectBandwidthUsage ¶ added in v1.7.1
func (usage *Service) GetProjectBandwidthUsage(ctx context.Context, projectID uuid.UUID) (currentUsed int64, err error)
GetProjectBandwidthUsage get the current bandwidth usage from cache.
It can return one of the following errors returned by storj.io/storj/satellite/accounting.Cache.GetProjectBandwidthUsage, wrapped by ErrProjectUsage.
func (*Service) GetProjectStorageLimit ¶ added in v0.28.0
func (usage *Service) GetProjectStorageLimit(ctx context.Context, projectID uuid.UUID) (_ memory.Size, err error)
GetProjectStorageLimit returns current project storage limit.
func (*Service) GetProjectStorageTotals ¶ added in v0.28.0
func (usage *Service) GetProjectStorageTotals(ctx context.Context, projectID uuid.UUID) (total int64, err error)
GetProjectStorageTotals returns total amount of storage used by project.
It can return one of the following errors returned by storj.io/storj/satellite/accounting.Cache.GetProjectStorageUsage except the ErrKeyNotFound, wrapped by ErrProjectUsage.
func (*Service) SetNow ¶ added in v1.4.3
SetNow allows tests to have the Service act as if the current time is whatever they want.
func (*Service) UpdateProjectBandwidthUsage ¶ added in v1.7.1
func (usage *Service) UpdateProjectBandwidthUsage(ctx context.Context, projectID uuid.UUID, increment int64) (err error)
UpdateProjectBandwidthUsage increments the bandwidth cache key for a specific project.
It can return one of the following errors returned by storj.io/storj/satellite/accounting.Cache.UpdatProjectBandwidthUsage, wrapped by ErrProjectUsage.
type StorageNodePeriodUsage ¶ added in v1.1.1
type StorageNodePeriodUsage struct { NodeID storj.NodeID AtRestTotal float64 GetTotal int64 PutTotal int64 GetRepairTotal int64 PutRepairTotal int64 GetAuditTotal int64 }
StorageNodePeriodUsage represents a statement for a node for a compensation period.
type StorageNodeUsage ¶ added in v0.18.0
StorageNodeUsage is node at rest space usage over a period of time.
type StoragenodeAccounting ¶
type StoragenodeAccounting interface { // SaveTallies records tallies of data at rest SaveTallies(ctx context.Context, latestTally time.Time, nodeData map[storj.NodeID]float64) error // GetTallies retrieves all tallies GetTallies(ctx context.Context) ([]*StoragenodeStorageTally, error) // GetTalliesSince retrieves all tallies since latestRollup GetTalliesSince(ctx context.Context, latestRollup time.Time) ([]*StoragenodeStorageTally, error) // GetBandwidthSince retrieves all bandwidth rollup entires since latestRollup GetBandwidthSince(ctx context.Context, latestRollup time.Time, cb func(context.Context, *StoragenodeBandwidthRollup) error) error // SaveRollup records tally and bandwidth rollup aggregations to the database SaveRollup(ctx context.Context, latestTally time.Time, stats RollupStats) error // LastTimestamp records and returns the latest last tallied time. LastTimestamp(ctx context.Context, timestampType string) (time.Time, error) // QueryPaymentInfo queries Nodes and Accounting_Rollup on nodeID QueryPaymentInfo(ctx context.Context, start time.Time, end time.Time) ([]*CSVRow, error) // QueryStorageNodePeriodUsage returns accounting statements for nodes for a given compensation period QueryStorageNodePeriodUsage(ctx context.Context, period compensation.Period) ([]StorageNodePeriodUsage, error) // QueryStorageNodeUsage returns slice of StorageNodeUsage for given period QueryStorageNodeUsage(ctx context.Context, nodeID storj.NodeID, start time.Time, end time.Time) ([]StorageNodeUsage, error) // DeleteTalliesBefore deletes all tallies prior to some time DeleteTalliesBefore(ctx context.Context, latestRollup time.Time) error // ArchiveRollupsBefore archives rollups older than a given time and returns num storagenode and bucket bandwidth rollups archived. ArchiveRollupsBefore(ctx context.Context, before time.Time, batchSize int) (numArchivedNodeBW int, err error) // GetRollupsSince retrieves all archived bandwidth rollup records since a given time. A hard limit batch size is used for results. GetRollupsSince(ctx context.Context, since time.Time) ([]StoragenodeBandwidthRollup, error) // GetArchivedRollupsSince retrieves all archived bandwidth rollup records since a given time. A hard limit batch size is used for results. GetArchivedRollupsSince(ctx context.Context, since time.Time) ([]StoragenodeBandwidthRollup, error) }
StoragenodeAccounting stores information about bandwidth and storage usage for storage nodes.
architecture: Database
Source Files ¶
Directories ¶
Path | Synopsis |
---|---|
Package live provides live accounting functionality.
|
Package live provides live accounting functionality. |