accounting

Documentation

Index

• Constants
• func ExceedsAlphaUsage(bandwidthGetTotal, inlineTotal, remoteTotal int64, maxAlphaUsageGB memory.Size) (bool, string)
• type BucketBandwidthRollup
• type BucketRollup
• type BucketRollupCursor
• type BucketStorageTally
• type BucketTally
  • func (s *BucketTally) AddSegment(pointer *pb.Pointer, last bool)
  • func (s *BucketTally) Combine(o *BucketTally)
  • func (s *BucketTally) Report(prefix string)
• type BucketUsage
• type CSVRow
• type DB
• type Order
• type Raw
• type Rollup
• type RollupStats

Constants

const (
	// AtRest is the data_type representing at-rest data calculated from pointerdb
	BandwidthPut       = int(pb.BandwidthAction_PUT)
	BandwidthGet       = int(pb.BandwidthAction_GET)
	BandwidthGetAudit  = int(pb.BandwidthAction_GET_AUDIT)
	BandwidthGetRepair = int(pb.BandwidthAction_GET_REPAIR)
	BandwidthPutRepair = int(pb.BandwidthAction_PUT_REPAIR)
	AtRest             = int(pb.BandwidthAction_PUT_REPAIR + 1)
	// LastAtRestTally represents the accounting timestamp for the at-rest data calculation
	LastAtRestTally = "LastAtRestTally"
	// LastBandwidthTally represents the accounting timestamp for the bandwidth allocation query
	LastBandwidthTally = "LastBandwidthTally"
	// LastRollup represents the accounting timestamp for rollup calculations
	LastRollup = "LastRollup"
)

Constants for accounting_raw, accounting_rollup, and accounting_timestamps

const (
	// AverageDaysInMonth is how many days in a month
	AverageDaysInMonth = 30
)

Functions

func ExceedsAlphaUsage

func ExceedsAlphaUsage(bandwidthGetTotal, inlineTotal, remoteTotal int64, maxAlphaUsageGB memory.Size) (bool, string)

ExceedsAlphaUsage returns true if more than 25GB of storage is currently in use or if 25GB of bandwidth or has been used in the past month (30 days) TODO(jg): remove this code once we no longer need usage limiting for alpha release Ref: https://storjlabs.atlassian.net/browse/V3-1274

Types

type BucketBandwidthRollup

type BucketBandwidthRollup struct {
	BucketName string
	ProjectID  uuid.UUID

	IntervalStart   time.Time
	IntervalSeconds uint
	Action          uint

	Inline    uint64
	Allocated uint64
	Settled   uint64
}

BucketBandwidthRollup contains data about bandwidth rollup

type BucketRollup

type BucketRollup struct {
	ID       uuid.UUID
	BucketID uuid.UUID

	RollupEndTime time.Time

	RemoteStoredData uint64
	InlineStoredData uint64
	RemoteSegments   uint
	InlineSegments   uint
	Objects          uint
	MetadataSize     uint64

	RepairEgress uint64
	GetEgress    uint64
	AuditEgress  uint64
}

BucketRollup holds usage rollup info

type BucketRollupCursor

type BucketRollupCursor struct {
	BucketID uuid.UUID
	Before   time.Time
	After    time.Time

	Order Order

	PageSize int
	Next     *BucketRollupCursor
}

BucketRollupCursor encapsulates cursor based page

type BucketStorageTally

type BucketStorageTally struct {
	BucketName    string
	ProjectID     uuid.UUID
	IntervalStart time.Time

	InlineSegmentCount int64
	RemoteSegmentCount int64

	ObjectCount int64

	InlineBytes  int64
	RemoteBytes  int64
	MetadataSize int64
}

BucketStorageTally holds data about a bucket tally

type BucketTally

type BucketTally struct {
	Segments        int64
	InlineSegments  int64
	RemoteSegments  int64
	UnknownSegments int64

	Files       int64
	InlineFiles int64
	RemoteFiles int64

	Bytes       int64
	InlineBytes int64
	RemoteBytes int64

	MetadataSize int64
}

BucketTally contains information about aggregate data stored in a bucket

func (*BucketTally) AddSegment

func (s *BucketTally) AddSegment(pointer *pb.Pointer, last bool)

AddSegment groups all the data based the passed pointer

func (*BucketTally) Combine

func (s *BucketTally) Combine(o *BucketTally)

Combine aggregates all the tallies

func (*BucketTally) Report

func (s *BucketTally) Report(prefix string)

Report reports the stats thru monkit

type BucketUsage

type BucketUsage interface {
	Get(ctx context.Context, id uuid.UUID) (*BucketRollup, error)
	GetPaged(ctx context.Context, cursor *BucketRollupCursor) ([]BucketRollup, error)
	Create(ctx context.Context, rollup BucketRollup) (*BucketRollup, error)
	Delete(ctx context.Context, id uuid.UUID) error
}

BucketUsage is bucket usage rollup repository

type CSVRow

type CSVRow struct {
	NodeID            storj.NodeID
	NodeCreationDate  time.Time
	AuditSuccessRatio float64
	AtRestTotal       float64
	GetRepairTotal    int64
	PutRepairTotal    int64
	GetAuditTotal     int64
	PutTotal          int64
	GetTotal          int64
	Wallet            string
}

CSVRow represents data from QueryPaymentInfo without exposing dbx

type DB

type DB interface {
	// LastTimestamp records the latest last tallied time.
	LastTimestamp(ctx context.Context, timestampType string) (time.Time, error)
	// SaveBWRaw records raw sums of agreement values to the database and updates the LastTimestamp.
	SaveBWRaw(ctx context.Context, tallyEnd time.Time, created time.Time, bwTotals map[storj.NodeID][]int64) error
	// SaveAtRestRaw records raw tallies of at-rest-data.
	SaveAtRestRaw(ctx context.Context, latestTally time.Time, created time.Time, nodeData map[storj.NodeID]float64) error
	// GetRaw retrieves all raw tallies
	GetRaw(ctx context.Context) ([]*Raw, error)
	// GetRawSince r retrieves all raw tallies sinces
	GetRawSince(ctx context.Context, latestRollup time.Time) ([]*Raw, error)
	// SaveRollup records raw tallies of at rest data to the database
	SaveRollup(ctx context.Context, latestTally time.Time, stats RollupStats) error
	// SaveBucketTallies saves the latest bucket info
	SaveBucketTallies(ctx context.Context, intervalStart time.Time, bucketTallies map[string]*BucketTally) error
	// QueryPaymentInfo queries Overlay, Accounting Rollup on nodeID
	QueryPaymentInfo(ctx context.Context, start time.Time, end time.Time) ([]*CSVRow, error)
	// DeleteRawBefore deletes all raw tallies prior to some time
	DeleteRawBefore(ctx context.Context, latestRollup time.Time) error
	// CreateBucketStorageTally creates a record for BucketStorageTally in the accounting DB table
	CreateBucketStorageTally(ctx context.Context, tally BucketStorageTally) error
	// ProjectBandwidthTotal returns the sum of GET bandwidth usage for a projectID in the past time frame
	ProjectBandwidthTotal(ctx context.Context, bucketID []byte, from time.Time) (int64, error)
	// ProjectStorageTotals returns the current inline and remote storage usage for a projectID
	ProjectStorageTotals(ctx context.Context, projectID uuid.UUID) (int64, int64, error)
}

DB stores information about bandwidth and storage usage

type Order

type Order string

Order is sorting order can be asc or desc

const (
	// Asc ascending sort order
	Asc Order = "asc"
	// Desc descending sort order
	Desc Order = "desc"
)

type Raw

type Raw struct {
	ID              int64
	NodeID          storj.NodeID
	IntervalEndTime time.Time
	DataTotal       float64
	DataType        int
	CreatedAt       time.Time
}

Raw mirrors dbx.AccountingRaw, allowing us to use that struct without leaking dbx

type Rollup

type Rollup struct {
	ID             int64
	NodeID         storj.NodeID
	StartTime      time.Time
	PutTotal       int64
	GetTotal       int64
	GetAuditTotal  int64
	GetRepairTotal int64
	PutRepairTotal int64
	AtRestTotal    float64
}

Rollup mirrors dbx.AccountingRollup, allowing us to use that struct without leaking dbx

type RollupStats

type RollupStats map[time.Time]map[storj.NodeID]*Rollup

RollupStats is a convenience alias