accounting

Documentation

Index

• Constants
• func ExceedsAlphaUsage(bandwidthGetTotal, inlineTotal, remoteTotal int64, maxAlphaUsageGB memory.Size) (bool, string)
• type BucketBandwidthRollup
• type BucketRollup
• type BucketRollupCursor
• type BucketStorageTally
• type BucketTally
  • func (s *BucketTally) AddSegment(pointer *pb.Pointer, last bool)
  • func (s *BucketTally) Combine(o *BucketTally)
  • func (s *BucketTally) Report(prefix string)
• type BucketUsage
• type CSVRow
• type Order
• type ProjectAccounting
• type Rollup
• type RollupStats
• type StoragenodeAccounting
• type StoragenodeBandwidthRollup
• type StoragenodeStorageTally

Constants

const (
	// AtRest is the data_type representing at-rest data calculated from metainfo
	BandwidthPut       = int(pb.BandwidthAction_PUT)
	BandwidthGet       = int(pb.BandwidthAction_GET)
	BandwidthGetAudit  = int(pb.BandwidthAction_GET_AUDIT)
	BandwidthGetRepair = int(pb.BandwidthAction_GET_REPAIR)
	BandwidthPutRepair = int(pb.BandwidthAction_PUT_REPAIR)
	AtRest             = int(pb.BandwidthAction_PUT_REPAIR + 1)
	// LastAtRestTally represents the accounting timestamp for the at-rest data calculation
	LastAtRestTally = "LastAtRestTally"
	// LastBandwidthTally represents the accounting timestamp for the bandwidth allocation query
	LastBandwidthTally = "LastBandwidthTally"
	// LastRollup represents the accounting timestamp for rollup calculations
	LastRollup = "LastRollup"
)

Constants for accounting_raw, accounting_rollup, and accounting_timestamps

const (
	// AverageDaysInMonth is how many days in a month
	AverageDaysInMonth = 30
	// ExpansionFactor is the expansion for redundancy, based on the default
	// redundancy scheme for the uplink.
	ExpansionFactor = 3
)

Functions

func ExceedsAlphaUsage

func ExceedsAlphaUsage(bandwidthGetTotal, inlineTotal, remoteTotal int64, maxAlphaUsageGB memory.Size) (bool, string)

ExceedsAlphaUsage returns true if the storage or bandwidth usage limits have been exceeded for a project in the past month (30 days). The usage limit is 25GB multiplied by the redundancy expansion factor, so that the uplinks have a raw limit of 25GB. TODO(jg): remove this code once we no longer need usage limiting for alpha release Ref: https://storjlabs.atlassian.net/browse/V3-1274

Types

type BucketBandwidthRollup

type BucketBandwidthRollup struct {
	BucketName string
	ProjectID  uuid.UUID

	IntervalStart   time.Time
	IntervalSeconds uint
	Action          uint

	Inline    uint64
	Allocated uint64
	Settled   uint64
}

BucketBandwidthRollup contains data about bandwidth rollup

type BucketRollup

type BucketRollup struct {
	ID       uuid.UUID
	BucketID uuid.UUID

	RollupEndTime time.Time

	RemoteStoredData uint64
	InlineStoredData uint64
	RemoteSegments   uint
	InlineSegments   uint
	Objects          uint
	MetadataSize     uint64

	RepairEgress uint64
	GetEgress    uint64
	AuditEgress  uint64
}

BucketRollup holds usage rollup info

type BucketRollupCursor

type BucketRollupCursor struct {
	BucketID uuid.UUID
	Before   time.Time
	After    time.Time

	Order Order

	PageSize int
	Next     *BucketRollupCursor
}

BucketRollupCursor encapsulates cursor based page

type BucketStorageTally

type BucketStorageTally struct {
	BucketName    string
	ProjectID     uuid.UUID
	IntervalStart time.Time

	InlineSegmentCount int64
	RemoteSegmentCount int64

	ObjectCount int64

	InlineBytes  int64
	RemoteBytes  int64
	MetadataSize int64
}

BucketStorageTally holds data about a bucket tally

type BucketTally

type BucketTally struct {
	BucketName []byte
	ProjectID  []byte

	Segments        int64
	InlineSegments  int64
	RemoteSegments  int64
	UnknownSegments int64

	Files       int64
	InlineFiles int64
	RemoteFiles int64

	Bytes       int64
	InlineBytes int64
	RemoteBytes int64

	MetadataSize int64
}

BucketTally contains information about aggregate data stored in a bucket

func (*BucketTally) AddSegment

func (s *BucketTally) AddSegment(pointer *pb.Pointer, last bool)

AddSegment groups all the data based the passed pointer

func (*BucketTally) Combine

func (s *BucketTally) Combine(o *BucketTally)

Combine aggregates all the tallies

func (*BucketTally) Report

func (s *BucketTally) Report(prefix string)

Report reports the stats thru monkit

type BucketUsage

type BucketUsage interface {
	Get(ctx context.Context, id uuid.UUID) (*BucketRollup, error)
	GetPaged(ctx context.Context, cursor *BucketRollupCursor) ([]BucketRollup, error)
	Create(ctx context.Context, rollup BucketRollup) (*BucketRollup, error)
	Delete(ctx context.Context, id uuid.UUID) error
}

BucketUsage is bucket usage rollup repository

type CSVRow

type CSVRow struct {
	NodeID            storj.NodeID
	NodeCreationDate  time.Time
	AuditSuccessRatio float64
	AtRestTotal       float64
	GetRepairTotal    int64
	PutRepairTotal    int64
	GetAuditTotal     int64
	PutTotal          int64
	GetTotal          int64
	Wallet            string
}

CSVRow represents data from QueryPaymentInfo without exposing dbx

type Order

type Order string

Order is sorting order can be asc or desc

const (
	// Asc ascending sort order
	Asc Order = "asc"
	// Desc descending sort order
	Desc Order = "desc"
)

type ProjectAccounting

type ProjectAccounting interface {
	// SaveTallies saves the latest project info
	SaveTallies(ctx context.Context, intervalStart time.Time, bucketTallies map[string]*BucketTally) ([]BucketTally, error)
	// CreateStorageTally creates a record for BucketStorageTally in the accounting DB table
	CreateStorageTally(ctx context.Context, tally BucketStorageTally) error
	// GetAllocatedBandwidthTotal returns the sum of GET bandwidth usage allocated for a projectID in the past time frame
	GetAllocatedBandwidthTotal(ctx context.Context, bucketID []byte, from time.Time) (int64, error)
	// GetStorageTotals returns the current inline and remote storage usage for a projectID
	GetStorageTotals(ctx context.Context, projectID uuid.UUID) (int64, int64, error)
}

ProjectAccounting stores information about bandwidth and storage usage for projects

type Rollup

type Rollup struct {
	ID             int64
	NodeID         storj.NodeID
	StartTime      time.Time
	PutTotal       int64
	GetTotal       int64
	GetAuditTotal  int64
	GetRepairTotal int64
	PutRepairTotal int64
	AtRestTotal    float64
}

Rollup mirrors dbx.AccountingRollup, allowing us to use that struct without leaking dbx

type RollupStats

type RollupStats map[time.Time]map[storj.NodeID]*Rollup

RollupStats is a convenience alias

type StoragenodeAccounting

type StoragenodeAccounting interface {
	// SaveTallies records tallies of data at rest
	SaveTallies(ctx context.Context, latestTally time.Time, nodeData map[storj.NodeID]float64) error
	// GetTallies retrieves all tallies
	GetTallies(ctx context.Context) ([]*StoragenodeStorageTally, error)
	// GetTalliesSince retrieves all tallies since latestRollup
	GetTalliesSince(ctx context.Context, latestRollup time.Time) ([]*StoragenodeStorageTally, error)
	// GetBandwidthSince retrieves all bandwidth rollup entires since latestRollup
	GetBandwidthSince(ctx context.Context, latestRollup time.Time) ([]*StoragenodeBandwidthRollup, error)
	// SaveRollup records tally and bandwidth rollup aggregations to the database
	SaveRollup(ctx context.Context, latestTally time.Time, stats RollupStats) error
	// LastTimestamp records and returns the latest last tallied time.
	LastTimestamp(ctx context.Context, timestampType string) (time.Time, error)
	// QueryPaymentInfo queries Nodes and Accounting_Rollup on nodeID
	QueryPaymentInfo(ctx context.Context, start time.Time, end time.Time) ([]*CSVRow, error)
	// DeleteTalliesBefore deletes all tallies prior to some time
	DeleteTalliesBefore(ctx context.Context, latestRollup time.Time) error
}

StoragenodeAccounting stores information about bandwidth and storage usage for storage nodes

type StoragenodeBandwidthRollup

type StoragenodeBandwidthRollup struct {
	NodeID        storj.NodeID
	IntervalStart time.Time
	Action        uint
	Settled       uint64
}

StoragenodeBandwidthRollup mirrors dbx.StoragenodeBandwidthRollup, allowing us to use the struct without leaking dbx

type StoragenodeStorageTally

type StoragenodeStorageTally struct {
	ID              int64
	NodeID          storj.NodeID
	IntervalEndTime time.Time
	DataTotal       float64
}

StoragenodeStorageTally mirrors dbx.StoragenodeStorageTally, allowing us to use that struct without leaking dbx