accounting

Documentation

Index

• Constants
• Variables
• type BucketBandwidthRollup
• type BucketRollup
• type BucketRollupCursor
• type BucketStorageTally
• type BucketTally
  • func (s *BucketTally) AddSegment(pointer *pb.Pointer, last bool)
  • func (s *BucketTally) Combine(o *BucketTally)
  • func (s *BucketTally) Report(prefix string)
• type BucketUsage
• type CSVRow
• type Order
• type ProjectAccounting
• type ProjectUsage
  • func NewProjectUsage(projectAccountingDB ProjectAccounting, liveAccounting live.Service, ...) *ProjectUsage
  • func (usage *ProjectUsage) AddProjectStorageUsage(ctx context.Context, projectID uuid.UUID, ...) (err error)
  • func (usage *ProjectUsage) ExceedsBandwidthUsage(ctx context.Context, projectID uuid.UUID, bucketID []byte) (_ bool, limit memory.Size, err error)
  • func (usage *ProjectUsage) ExceedsStorageUsage(ctx context.Context, projectID uuid.UUID) (_ bool, limit memory.Size, err error)
• type Rollup
• type RollupStats
• type StoragenodeAccounting
• type StoragenodeBandwidthRollup
• type StoragenodeStorageTally

Constants

const (
	// LastAtRestTally represents the accounting timestamp for the at-rest data calculation
	LastAtRestTally = "LastAtRestTally"
	// LastBandwidthTally represents the accounting timestamp for the bandwidth allocation query
	LastBandwidthTally = "LastBandwidthTally"
	// LastRollup represents the accounting timestamp for rollup calculations
	LastRollup = "LastRollup"
)

Constants for accounting_raw, accounting_rollup, and accounting_timestamps

const (
	// AverageDaysInMonth is how many days in a month
	AverageDaysInMonth = 30
	// ExpansionFactor is the expansion for redundancy, based on the default
	// redundancy scheme for the uplink.
	ExpansionFactor = 3
)

Variables

var (
	// ErrProjectUsage general error for project usage
	ErrProjectUsage = errs.Class("project usage error")
)

Types

type BucketBandwidthRollup

type BucketBandwidthRollup struct {
	BucketName string
	ProjectID  uuid.UUID

	IntervalStart   time.Time
	IntervalSeconds uint
	Action          uint

	Inline    uint64
	Allocated uint64
	Settled   uint64
}

BucketBandwidthRollup contains data about bandwidth rollup

type BucketRollup

type BucketRollup struct {
	ID       uuid.UUID
	BucketID uuid.UUID

	RollupEndTime time.Time

	RemoteStoredData uint64
	InlineStoredData uint64
	RemoteSegments   uint
	InlineSegments   uint
	Objects          uint
	MetadataSize     uint64

	RepairEgress uint64
	GetEgress    uint64
	AuditEgress  uint64
}

BucketRollup holds usage rollup info

type BucketRollupCursor

type BucketRollupCursor struct {
	BucketID uuid.UUID
	Before   time.Time
	After    time.Time

	Order Order

	PageSize int
	Next     *BucketRollupCursor
}

BucketRollupCursor encapsulates cursor based page

type BucketStorageTally

type BucketStorageTally struct {
	BucketName    string
	ProjectID     uuid.UUID
	IntervalStart time.Time

	InlineSegmentCount int64
	RemoteSegmentCount int64

	ObjectCount int64

	InlineBytes  int64
	RemoteBytes  int64
	MetadataSize int64
}

BucketStorageTally holds data about a bucket tally

type BucketTally

type BucketTally struct {
	BucketName []byte
	ProjectID  []byte

	Segments        int64
	InlineSegments  int64
	RemoteSegments  int64
	UnknownSegments int64

	Files       int64
	InlineFiles int64
	RemoteFiles int64

	Bytes       int64
	InlineBytes int64
	RemoteBytes int64

	MetadataSize int64
}

BucketTally contains information about aggregate data stored in a bucket

func (*BucketTally) AddSegment

func (s *BucketTally) AddSegment(pointer *pb.Pointer, last bool)

AddSegment groups all the data based the passed pointer

func (*BucketTally) Combine

func (s *BucketTally) Combine(o *BucketTally)

Combine aggregates all the tallies

func (*BucketTally) Report

func (s *BucketTally) Report(prefix string)

Report reports the stats thru monkit

type BucketUsage

type BucketUsage interface {
	Get(ctx context.Context, id uuid.UUID) (*BucketRollup, error)
	GetPaged(ctx context.Context, cursor *BucketRollupCursor) ([]BucketRollup, error)
	Create(ctx context.Context, rollup BucketRollup) (*BucketRollup, error)
	Delete(ctx context.Context, id uuid.UUID) error
}

BucketUsage is bucket usage rollup repository

type CSVRow

type CSVRow struct {
	NodeID           storj.NodeID
	NodeCreationDate time.Time
	AtRestTotal      float64
	GetRepairTotal   int64
	PutRepairTotal   int64
	GetAuditTotal    int64
	PutTotal         int64
	GetTotal         int64
	Wallet           string
	Disqualified     *time.Time
}

CSVRow represents data from QueryPaymentInfo without exposing dbx

type Order

type Order string

Order is sorting order can be asc or desc

const (
	// Asc ascending sort order
	Asc Order = "asc"
	// Desc descending sort order
	Desc Order = "desc"
)

type ProjectAccounting

type ProjectAccounting interface {
	// SaveTallies saves the latest project info
	SaveTallies(ctx context.Context, intervalStart time.Time, bucketTallies map[string]*BucketTally) ([]BucketTally, error)
	// CreateStorageTally creates a record for BucketStorageTally in the accounting DB table
	CreateStorageTally(ctx context.Context, tally BucketStorageTally) error
	// GetAllocatedBandwidthTotal returns the sum of GET bandwidth usage allocated for a projectID in the past time frame
	GetAllocatedBandwidthTotal(ctx context.Context, bucketID []byte, from time.Time) (int64, error)
	// GetStorageTotals returns the current inline and remote storage usage for a projectID
	GetStorageTotals(ctx context.Context, projectID uuid.UUID) (int64, int64, error)
	// GetProjectUsageLimits returns project usage limit
	GetProjectUsageLimits(ctx context.Context, projectID uuid.UUID) (memory.Size, error)
}

ProjectAccounting stores information about bandwidth and storage usage for projects

type ProjectUsage

type ProjectUsage struct {
	// contains filtered or unexported fields
}

ProjectUsage defines project usage

func NewProjectUsage

func NewProjectUsage(projectAccountingDB ProjectAccounting, liveAccounting live.Service, maxAlphaUsage memory.Size) *ProjectUsage

NewProjectUsage created new instance of project usage service

func (*ProjectUsage) AddProjectStorageUsage

func (usage *ProjectUsage) AddProjectStorageUsage(ctx context.Context, projectID uuid.UUID, inlineSpaceUsed, remoteSpaceUsed int64) (err error)

AddProjectStorageUsage lets the live accounting know that the given project has just added inlineSpaceUsed bytes of inline space usage and remoteSpaceUsed bytes of remote space usage.

func (*ProjectUsage) ExceedsBandwidthUsage

func (usage *ProjectUsage) ExceedsBandwidthUsage(ctx context.Context, projectID uuid.UUID, bucketID []byte) (_ bool, limit memory.Size, err error)

ExceedsBandwidthUsage returns true if the bandwidth usage limits have been exceeded for a project in the past month (30 days). The usage limit is (e.g 25GB) multiplied by the redundancy expansion factor, so that the uplinks have a raw limit. Ref: https://storjlabs.atlassian.net/browse/V3-1274

func (*ProjectUsage) ExceedsStorageUsage

func (usage *ProjectUsage) ExceedsStorageUsage(ctx context.Context, projectID uuid.UUID) (_ bool, limit memory.Size, err error)

ExceedsStorageUsage returns true if the storage usage limits have been exceeded for a project in the past month (30 days). The usage limit is (e.g. 25GB) multiplied by the redundancy expansion factor, so that the uplinks have a raw limit. Ref: https://storjlabs.atlassian.net/browse/V3-1274

type Rollup

type Rollup struct {
	ID             int64
	NodeID         storj.NodeID
	StartTime      time.Time
	PutTotal       int64
	GetTotal       int64
	GetAuditTotal  int64
	GetRepairTotal int64
	PutRepairTotal int64
	AtRestTotal    float64
}

Rollup mirrors dbx.AccountingRollup, allowing us to use that struct without leaking dbx

type RollupStats

type RollupStats map[time.Time]map[storj.NodeID]*Rollup

RollupStats is a convenience alias

type StoragenodeAccounting

type StoragenodeAccounting interface {
	// SaveTallies records tallies of data at rest
	SaveTallies(ctx context.Context, latestTally time.Time, nodeData map[storj.NodeID]float64) error
	// GetTallies retrieves all tallies
	GetTallies(ctx context.Context) ([]*StoragenodeStorageTally, error)
	// GetTalliesSince retrieves all tallies since latestRollup
	GetTalliesSince(ctx context.Context, latestRollup time.Time) ([]*StoragenodeStorageTally, error)
	// GetBandwidthSince retrieves all bandwidth rollup entires since latestRollup
	GetBandwidthSince(ctx context.Context, latestRollup time.Time) ([]*StoragenodeBandwidthRollup, error)
	// SaveRollup records tally and bandwidth rollup aggregations to the database
	SaveRollup(ctx context.Context, latestTally time.Time, stats RollupStats) error
	// LastTimestamp records and returns the latest last tallied time.
	LastTimestamp(ctx context.Context, timestampType string) (time.Time, error)
	// QueryPaymentInfo queries Nodes and Accounting_Rollup on nodeID
	QueryPaymentInfo(ctx context.Context, start time.Time, end time.Time) ([]*CSVRow, error)
	// DeleteTalliesBefore deletes all tallies prior to some time
	DeleteTalliesBefore(ctx context.Context, latestRollup time.Time) error
}

StoragenodeAccounting stores information about bandwidth and storage usage for storage nodes

type StoragenodeBandwidthRollup

type StoragenodeBandwidthRollup struct {
	NodeID        storj.NodeID
	IntervalStart time.Time
	Action        uint
	Settled       uint64
}

StoragenodeBandwidthRollup mirrors dbx.StoragenodeBandwidthRollup, allowing us to use the struct without leaking dbx

type StoragenodeStorageTally

type StoragenodeStorageTally struct {
	ID              int64
	NodeID          storj.NodeID
	IntervalEndTime time.Time
	DataTotal       float64
}

StoragenodeStorageTally mirrors dbx.StoragenodeStorageTally, allowing us to use that struct without leaking dbx