accounting

Documentation

Index

• Constants
• func ExceedsAlphaUsage(bandwidthGetTotal, inlineTotal, remoteTotal int64, maxAlphaUsageGB memory.Size) (bool, string)
• type BucketBandwidthRollup
• type BucketRollup
• type BucketRollupCursor
• type BucketStorageTally
• type BucketTally
  • func (s *BucketTally) AddSegment(pointer *pb.Pointer, last bool)
  • func (s *BucketTally) Combine(o *BucketTally)
  • func (s *BucketTally) Report(prefix string)
• type BucketUsage
• type CSVRow
• type DB
• type Order
• type Raw
• type Rollup
• type RollupStats
• type StoragenodeBandwidthRollup

Constants

const (
	// AtRest is the data_type representing at-rest data calculated from pointerdb
	BandwidthPut       = int(pb.BandwidthAction_PUT)
	BandwidthGet       = int(pb.BandwidthAction_GET)
	BandwidthGetAudit  = int(pb.BandwidthAction_GET_AUDIT)
	BandwidthGetRepair = int(pb.BandwidthAction_GET_REPAIR)
	BandwidthPutRepair = int(pb.BandwidthAction_PUT_REPAIR)
	AtRest             = int(pb.BandwidthAction_PUT_REPAIR + 1)
	// LastAtRestTally represents the accounting timestamp for the at-rest data calculation
	LastAtRestTally = "LastAtRestTally"
	// LastBandwidthTally represents the accounting timestamp for the bandwidth allocation query
	LastBandwidthTally = "LastBandwidthTally"
	// LastRollup represents the accounting timestamp for rollup calculations
	LastRollup = "LastRollup"
)

Constants for accounting_raw, accounting_rollup, and accounting_timestamps

const (
	// AverageDaysInMonth is how many days in a month
	AverageDaysInMonth = 30
	// ExpansionFactor is the expansion for redundancy, based on the default
	// redundancy scheme for the uplink.
	ExpansionFactor = 3
)

Functions

func ExceedsAlphaUsage

func ExceedsAlphaUsage(bandwidthGetTotal, inlineTotal, remoteTotal int64, maxAlphaUsageGB memory.Size) (bool, string)

ExceedsAlphaUsage returns true if the storage or bandwidth usage limits have been exceeded for a project in the past month (30 days). The usage limit is 25GB multiplied by the redundancy expansion factor, so that the uplinks have a raw limit of 25GB. TODO(jg): remove this code once we no longer need usage limiting for alpha release Ref: https://storjlabs.atlassian.net/browse/V3-1274

Types

type BucketBandwidthRollup

type BucketBandwidthRollup struct {
	BucketName string
	ProjectID  uuid.UUID

	IntervalStart   time.Time
	IntervalSeconds uint
	Action          uint

	Inline    uint64
	Allocated uint64
	Settled   uint64
}

BucketBandwidthRollup contains data about bandwidth rollup

type BucketRollup

type BucketRollup struct {
	ID       uuid.UUID
	BucketID uuid.UUID

	RollupEndTime time.Time

	RemoteStoredData uint64
	InlineStoredData uint64
	RemoteSegments   uint
	InlineSegments   uint
	Objects          uint
	MetadataSize     uint64

	RepairEgress uint64
	GetEgress    uint64
	AuditEgress  uint64
}

BucketRollup holds usage rollup info

type BucketRollupCursor

type BucketRollupCursor struct {
	BucketID uuid.UUID
	Before   time.Time
	After    time.Time

	Order Order

	PageSize int
	Next     *BucketRollupCursor
}

BucketRollupCursor encapsulates cursor based page

type BucketStorageTally

type BucketStorageTally struct {
	BucketName    string
	ProjectID     uuid.UUID
	IntervalStart time.Time

	InlineSegmentCount int64
	RemoteSegmentCount int64

	ObjectCount int64

	InlineBytes  int64
	RemoteBytes  int64
	MetadataSize int64
}

BucketStorageTally holds data about a bucket tally

type BucketTally

type BucketTally struct {
	BucketName []byte
	ProjectID  []byte

	Segments        int64
	InlineSegments  int64
	RemoteSegments  int64
	UnknownSegments int64

	Files       int64
	InlineFiles int64
	RemoteFiles int64

	Bytes       int64
	InlineBytes int64
	RemoteBytes int64

	MetadataSize int64
}

BucketTally contains information about aggregate data stored in a bucket

func (*BucketTally) AddSegment

func (s *BucketTally) AddSegment(pointer *pb.Pointer, last bool)

AddSegment groups all the data based the passed pointer

func (*BucketTally) Combine

func (s *BucketTally) Combine(o *BucketTally)

Combine aggregates all the tallies

func (*BucketTally) Report

func (s *BucketTally) Report(prefix string)

Report reports the stats thru monkit

type BucketUsage

type BucketUsage interface {
	Get(ctx context.Context, id uuid.UUID) (*BucketRollup, error)
	GetPaged(ctx context.Context, cursor *BucketRollupCursor) ([]BucketRollup, error)
	Create(ctx context.Context, rollup BucketRollup) (*BucketRollup, error)
	Delete(ctx context.Context, id uuid.UUID) error
}

BucketUsage is bucket usage rollup repository

type CSVRow

type CSVRow struct {
	NodeID            storj.NodeID
	NodeCreationDate  time.Time
	AuditSuccessRatio float64
	AtRestTotal       float64
	GetRepairTotal    int64
	PutRepairTotal    int64
	GetAuditTotal     int64
	PutTotal          int64
	GetTotal          int64
	Wallet            string
}

CSVRow represents data from QueryPaymentInfo without exposing dbx

type DB

type DB interface {
	// LastTimestamp records the latest last tallied time.
	LastTimestamp(ctx context.Context, timestampType string) (time.Time, error)
	// SaveAtRestRaw records raw tallies of at-rest-data.
	SaveAtRestRaw(ctx context.Context, latestTally time.Time, created time.Time, nodeData map[storj.NodeID]float64) error
	// GetRaw retrieves all raw tallies
	GetRaw(ctx context.Context) ([]*Raw, error)
	// GetRawSince retrieves all raw tallies since latestRollup
	GetRawSince(ctx context.Context, latestRollup time.Time) ([]*Raw, error)
	// GetStoragenodeBandwidthSince retrieves all storagenode_bandwidth_rollup entires since latestRollup
	GetStoragenodeBandwidthSince(ctx context.Context, latestRollup time.Time) ([]*StoragenodeBandwidthRollup, error)
	// SaveRollup records raw tallies of at rest data to the database
	SaveRollup(ctx context.Context, latestTally time.Time, stats RollupStats) error
	// SaveBucketTallies saves the latest bucket info
	SaveBucketTallies(ctx context.Context, intervalStart time.Time, bucketTallies map[string]*BucketTally) ([]BucketTally, error)
	// QueryPaymentInfo queries Overlay, Accounting Rollup on nodeID
	QueryPaymentInfo(ctx context.Context, start time.Time, end time.Time) ([]*CSVRow, error)
	// DeleteRawBefore deletes all raw tallies prior to some time
	DeleteRawBefore(ctx context.Context, latestRollup time.Time) error
	// CreateBucketStorageTally creates a record for BucketStorageTally in the accounting DB table
	CreateBucketStorageTally(ctx context.Context, tally BucketStorageTally) error
	// ProjectAllocatedBandwidthTotal returns the sum of GET bandwidth usage allocated for a projectID in the past time frame
	ProjectAllocatedBandwidthTotal(ctx context.Context, bucketID []byte, from time.Time) (int64, error)
	// ProjectStorageTotals returns the current inline and remote storage usage for a projectID
	ProjectStorageTotals(ctx context.Context, projectID uuid.UUID) (int64, int64, error)
}

DB stores information about bandwidth and storage usage

type Order

type Order string

Order is sorting order can be asc or desc

const (
	// Asc ascending sort order
	Asc Order = "asc"
	// Desc descending sort order
	Desc Order = "desc"
)

type Raw

type Raw struct {
	ID              int64
	NodeID          storj.NodeID
	IntervalEndTime time.Time
	DataTotal       float64
	DataType        int
	CreatedAt       time.Time
}

Raw mirrors dbx.AccountingRaw, allowing us to use that struct without leaking dbx

type Rollup

type Rollup struct {
	ID             int64
	NodeID         storj.NodeID
	StartTime      time.Time
	PutTotal       int64
	GetTotal       int64
	GetAuditTotal  int64
	GetRepairTotal int64
	PutRepairTotal int64
	AtRestTotal    float64
}

Rollup mirrors dbx.AccountingRollup, allowing us to use that struct without leaking dbx

type RollupStats

type RollupStats map[time.Time]map[storj.NodeID]*Rollup

RollupStats is a convenience alias

type StoragenodeBandwidthRollup

type StoragenodeBandwidthRollup struct {
	NodeID        storj.NodeID
	IntervalStart time.Time
	Action        uint
	Settled       uint64
}

StoragenodeBandwidthRollup mirrors dbx.StoragenodeBandwidthRollup, allowing us to use the struct without leaking dbx