satellitedb

package
v1.120.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 14, 2025 License: AGPL-3.0 Imports: 78 Imported by: 6

Documentation

Index

Constants

View Source 
const RepairQueueSelectLimit = 1000

RepairQueueSelectLimit defines how many items can be selected at the same time.

View Source 
const (
	// VerifyRetryInterval defines a limit on how frequently we retry
	// verification audits. At least this long should elapse between
	// attempts.
	VerifyRetryInterval = 4 * time.Hour
)

Variables

View Source 
var (
	// ErrMigrate is for tracking migration errors.
	ErrMigrate = errs.Class("migrate")
	// ErrMigrateMinVersion is for migration min version errors.
	ErrMigrateMinVersion = errs.Class("migrate min version")
)
View Source 
var (
	// ErrDifferentStorageNodes is returned when ProcessOrders gets orders from different storage nodes.
	ErrDifferentStorageNodes = errs.Class("different storage nodes")
	// ErrBucketFromSerial is returned when there is an error trying to get the bucket name from the serial number.
	ErrBucketFromSerial = errs.Class("bucket from serial number")
	// ErrUpdateBucketBandwidthSettle is returned when there is an error updating bucket bandwidth.
	ErrUpdateBucketBandwidthSettle = errs.Class("update bucket bandwidth settle")
	// ErrProcessOrderWithWindowTx is returned when there is an error with the ProcessOrders transaction.
	ErrProcessOrderWithWindowTx = errs.Class("process order with window transaction")
	// ErrGetStoragenodeBandwidthInWindow is returned when there is an error getting all storage node bandwidth for a window.
	ErrGetStoragenodeBandwidthInWindow = errs.Class("get storagenode bandwidth in window")
	// ErrCreateStoragenodeBandwidth is returned when there is an error updating storage node bandwidth.
	ErrCreateStoragenodeBandwidth = errs.Class("create storagenode bandwidth")
)
View Source 
var (
	// ErrVetting is the error class for the following test methods.
	ErrVetting = errs.Class("vetting")
)
View Source 
var Error = errs.Class("satellitedb")

Error is the default satellitedb errs class.

View Source 
var SpannerExtraStatements = (func() []string {
	x := &satelliteDB{}
	migration := x.testMigrationSpanner()

	step := migration.Steps[0]
	sql := step.Action.(migrate.SQL)[0]

	sql = strings.TrimPrefix(sql, "-- AUTOGENERATED BY storj.io/dbx\n")
	sql = strings.TrimPrefix(sql, "-- DO NOT EDIT\n")
	sql = strings.Trim(sql, "\n ;")

	statements := spannerutil.MustSplitSQLStatements(sql)

	statements = append(statements,
		`CREATE TABLE IF NOT EXISTS `+migration.Table+` (version INT64, commited_at STRING(MAX)) PRIMARY KEY (version)`)
	statements = append(statements,
		`INSERT INTO `+migration.Table+`(version, commited_at) VALUES (`+strconv.Itoa(step.Version)+`, '')`)

	for i, statement := range statements {
		statements[i] = strings.TrimSpace(statement)
	}

	return statements
})()

SpannerExtraStatements contains the list of queries needed to setup spanner satellite database.

TODO: make this a nicer implementation.

Functions

func GetTestApiKey added in v1.120.1 

func GetTestApiKey(satelliteId string) (string, error)

GetTestApiKey can calculate an access grant for the predefined test users/project.

func MigrateSatelliteDB added in v1.120.1 

func MigrateSatelliteDB(ctx context.Context, log *zap.Logger, db satellite.DB, migrationType string) (err error)

MigrateSatelliteDB migrates satellite database.

func Module added in v1.120.1 

func Module(ball *mud.Ball)

Module is a mud module.

func Open added in v1.17.1 

func Open(ctx context.Context, log *zap.Logger, databaseURL string, opts Options) (rv satellite.DB, err error)

Open creates instance of satellite.DB.

func OpenDBWithMigration added in v1.120.1 

func OpenDBWithMigration(ctx context.Context, logger *zap.Logger, cfg DatabaseOptions) (satellite.DB, error)

OpenDBWithMigration is a wrapper for opening database and do optional migration.

func SettledAmountsMatch added in v1.8.1 

func SettledAmountsMatch(rows []*dbx.StoragenodeBandwidthRollup, orderActionAmounts map[int32]int64) bool

SettledAmountsMatch checks if database rows match the orders. If the settled amount for each action are not the same then false is returned.

func SortBandwidthRollupKeys added in v1.106.1 

func SortBandwidthRollupKeys(bandwidthRollupKeys []BandwidthRollupKey)

SortBandwidthRollupKeys sorts bandwidth rollups.

Types

type BandwidthRollupKey added in v1.106.1 

type BandwidthRollupKey struct {
	BucketName    string
	ProjectID     uuid.UUID
	IntervalStart int64
	Action        pb.PieceAction
}

BandwidthRollupKey is used to collect data for a query.

type BucketBandwidthRollup added in v1.111.4 

type BucketBandwidthRollup struct {
	BucketName      []byte
	ProjectID       uuid.UUID
	IntervalStart   time.Time
	IntervalSeconds int64
	Action          int64
	Inline          int64
	Allocated       int64
	Settled         int64
}

BucketBandwidthRollup is a type to encapsulate the values to insert into a record for the bucket_bandwidth_rollups table.

type DatabaseOptions added in v1.120.1 

type DatabaseOptions struct {
	URL          string `help:"satellite database connection string" releaseDefault:"postgres://" devDefault:"postgres://"`
	APIKeysCache struct {
		Expiration time.Duration `help:"satellite database api key expiration" default:"60s"`
		Capacity   int           `help:"satellite database api key lru capacity" default:"10000"`
	}
	RevocationsCache struct {
		Expiration time.Duration `help:"macaroon revocation cache expiration" default:"5m"`
		Capacity   int           `help:"macaroon revocation cache capacity" default:"10000"`
	}
	MigrationUnsafe string `` /* 305-byte string literal not displayed */
}

DatabaseOptions are the configurations for satellitedb.

type Options added in v0.29.4 

type Options struct {
	ApplicationName      string
	APIKeysLRUOptions    lrucache.Options
	RevocationLRUOptions lrucache.Options

	// How many storage node rollups to save/read in one batch.
	SaveRollupBatchSize int
	ReadRollupBatchSize int
}

Options includes options for how a satelliteDB runs.

type ProjectAccounting added in v0.12.0 

type ProjectAccounting struct {
	// contains filtered or unexported fields
}

ProjectAccounting implements the accounting/db ProjectAccounting interface.

func (*ProjectAccounting) ArchiveRollupsBefore added in v1.22.1 

func (db *ProjectAccounting) ArchiveRollupsBefore(ctx context.Context, before time.Time, batchSize int) (archivedCount int, err error)

ArchiveRollupsBefore archives rollups older than a given time.

func (*ProjectAccounting) CreateStorageTally added in v0.12.0 

func (db *ProjectAccounting) CreateStorageTally(ctx context.Context, tally accounting.BucketStorageTally) (err error)

CreateStorageTally creates a record in the bucket_storage_tallies accounting table.

func (*ProjectAccounting) DeleteProjectBandwidthBefore added in v1.31.1 

func (db *ProjectAccounting) DeleteProjectBandwidthBefore(ctx context.Context, before time.Time) (err error)

DeleteProjectBandwidthBefore deletes project bandwidth rollups before the given time.

func (*ProjectAccounting) DeleteTalliesBefore added in v1.118.4 

func (db *ProjectAccounting) DeleteTalliesBefore(ctx context.Context, before time.Time) (_ int64, err error)

DeleteTalliesBefore deletes tallies with an interval start before the given time.

Spanner implementation returns an estimated count of the number of rows deleted. The actual number of affected rows may be greater than the estimate.

func (*ProjectAccounting) GetArchivedRollupsSince added in v1.22.1 

func (db *ProjectAccounting) GetArchivedRollupsSince(ctx context.Context, since time.Time) (bwRollups []orders.BucketBandwidthRollup, err error)

GetArchivedRollupsSince retrieves all archived rollup records since a given time.

func (*ProjectAccounting) GetBucketTotals added in v0.26.0 

func (db *ProjectAccounting) GetBucketTotals(ctx context.Context, projectID uuid.UUID, cursor accounting.BucketUsageCursor, before time.Time) (_ *accounting.BucketUsagePage, err error)

GetBucketTotals retrieves bucket usage totals for period of time.

func (*ProjectAccounting) GetBucketUsageRollups added in v0.26.0 

func (db *ProjectAccounting) GetBucketUsageRollups(ctx context.Context, projectID uuid.UUID, since, before time.Time) (_ []accounting.BucketUsageRollup, err error)

GetBucketUsageRollups retrieves summed usage rollups for every bucket of particular project for a given period.

func (*ProjectAccounting) GetNonEmptyTallyBucketsInRange added in v1.81.2 

func (db *ProjectAccounting) GetNonEmptyTallyBucketsInRange(ctx context.Context, from, to metabase.BucketLocation, asOfSystemInterval time.Duration) (result []metabase.BucketLocation, err error)

GetNonEmptyTallyBucketsInRange returns a list of bucket locations within the given range whose most recent tally does not represent empty usage.

func (*ProjectAccounting) GetProjectBandwidth added in v1.31.1 

func (db *ProjectAccounting) GetProjectBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int, asOfSystemInterval time.Duration) (_ int64, err error)

GetProjectBandwidth returns the used bandwidth (settled or allocated) for the specified year, month and day.

func (*ProjectAccounting) GetProjectBandwidthLimit added in v0.27.0 

func (db *ProjectAccounting) GetProjectBandwidthLimit(ctx context.Context, projectID uuid.UUID) (_ *int64, err error)

GetProjectBandwidthLimit returns project bandwidth usage limit.

func (*ProjectAccounting) GetProjectDailyBandwidth added in v1.31.1 

func (db *ProjectAccounting) GetProjectDailyBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int) (allocated int64, settled, dead int64, err error)

GetProjectDailyBandwidth returns project bandwidth (allocated and settled) for the specified day.

func (*ProjectAccounting) GetProjectDailyUsageByDateRange added in v1.49.3 

func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context, projectID uuid.UUID, from, to time.Time, crdbInterval time.Duration) (_ *accounting.ProjectDailyUsage, err error)

GetProjectDailyUsageByDateRange returns project daily allocated, settled bandwidth and storage usage by specific date range.

func (*ProjectAccounting) GetProjectLimits added in v1.14.1 

func (db *ProjectAccounting) GetProjectLimits(ctx context.Context, projectID uuid.UUID) (_ accounting.ProjectLimits, err error)

GetProjectLimits returns all project limits including user specified usage and bandwidth limits.

func (*ProjectAccounting) GetProjectObjectsSegments added in v1.45.2 

func (db *ProjectAccounting) GetProjectObjectsSegments(ctx context.Context, projectID uuid.UUID) (objectsSegments accounting.ProjectObjectsSegments, err error)

GetProjectObjectsSegments returns project objects and segments number.

func (*ProjectAccounting) GetProjectSegmentLimit added in v1.45.2 

func (db *ProjectAccounting) GetProjectSegmentLimit(ctx context.Context, projectID uuid.UUID) (_ *int64, err error)

GetProjectSegmentLimit returns project segment limit.

func (*ProjectAccounting) GetProjectSettledBandwidth added in v1.83.2 

func (db *ProjectAccounting) GetProjectSettledBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, asOfSystemInterval time.Duration) (_ int64, err error)

GetProjectSettledBandwidth returns the used settled bandwidth for the specified year and month.

func (*ProjectAccounting) GetProjectSettledBandwidthTotal added in v1.71.1 

func (db *ProjectAccounting) GetProjectSettledBandwidthTotal(ctx context.Context, projectID uuid.UUID, from time.Time) (_ int64, err error)

GetProjectSettledBandwidthTotal returns the sum of GET bandwidth usage settled for a projectID in the past time frame.

func (*ProjectAccounting) GetProjectStorageLimit added in v0.27.0 

func (db *ProjectAccounting) GetProjectStorageLimit(ctx context.Context, projectID uuid.UUID) (_ *int64, err error)

GetProjectStorageLimit returns project storage usage limit.

func (*ProjectAccounting) GetProjectTotal added in v0.26.0 

func (db *ProjectAccounting) GetProjectTotal(ctx context.Context, projectID uuid.UUID, since, before time.Time) (_ *accounting.ProjectUsage, err error)

GetProjectTotal retrieves project usage for a given period.

func (*ProjectAccounting) GetProjectTotalByPartner added in v1.74.1 

func (db *ProjectAccounting) GetProjectTotalByPartner(ctx context.Context, projectID uuid.UUID, partnerNames []string, since, before time.Time) (usages map[string]accounting.ProjectUsage, err error)

GetProjectTotalByPartner retrieves project usage for a given period categorized by partner name. Unpartnered usage or usage for a partner not present in partnerNames is mapped to the empty string.

func (*ProjectAccounting) GetRollupsSince added in v1.22.1 

func (db *ProjectAccounting) GetRollupsSince(ctx context.Context, since time.Time) (bwRollups []orders.BucketBandwidthRollup, err error)

GetRollupsSince retrieves all archived rollup records since a given time.

func (*ProjectAccounting) GetSingleBucketTotals added in v1.115.1 

func (db *ProjectAccounting) GetSingleBucketTotals(ctx context.Context, projectID uuid.UUID, bucketName string, before time.Time) (usage *accounting.BucketUsage, err error)

GetSingleBucketTotals retrieves single bucket usage totals for period of time.

func (*ProjectAccounting) GetSingleBucketUsageRollup added in v1.51.1 

func (db *ProjectAccounting) GetSingleBucketUsageRollup(ctx context.Context, projectID uuid.UUID, bucket string, since, before time.Time) (_ *accounting.BucketUsageRollup, err error)

GetSingleBucketUsageRollup retrieves usage rollup for a single bucket of particular project for a given period.

func (*ProjectAccounting) GetTallies added in v0.21.0 

func (db *ProjectAccounting) GetTallies(ctx context.Context) (tallies []accounting.BucketTally, err error)

GetTallies retrieves all tallies ordered by interval start (descending).

func (*ProjectAccounting) SaveTallies added in v0.12.0 

func (db *ProjectAccounting) SaveTallies(ctx context.Context, intervalStart time.Time, bucketTallies map[metabase.BucketLocation]*accounting.BucketTally) (err error)

SaveTallies saves the latest bucket info.

func (*ProjectAccounting) UpdateProjectBandwidthLimit added in v1.4.3 

func (db *ProjectAccounting) UpdateProjectBandwidthLimit(ctx context.Context, projectID uuid.UUID, limit memory.Size) (err error)

UpdateProjectBandwidthLimit updates project bandwidth limit.

func (*ProjectAccounting) UpdateProjectSegmentLimit added in v1.45.2 

func (db *ProjectAccounting) UpdateProjectSegmentLimit(ctx context.Context, projectID uuid.UUID, limit int64) (err error)

UpdateProjectSegmentLimit updates project segment limit.

func (*ProjectAccounting) UpdateProjectUsageLimit added in v0.27.0 

func (db *ProjectAccounting) UpdateProjectUsageLimit(ctx context.Context, projectID uuid.UUID, limit memory.Size) (err error)

UpdateProjectUsageLimit updates project usage limit.

type ProjectBandwidthDailyRollup added in v1.111.4 

type ProjectBandwidthDailyRollup struct {
	ProjectID       uuid.UUID
	IntervalStart   civil.Date
	EgressAllocated int64
	EgressSettled   int64
	EgressDead      int64
}

ProjectBandwidthDailyRollup is a type to encapsulate the values to insert into a record for the project_bandwidth_daily_rollups table.

type StoragenodeAccounting added in v0.12.0 

type StoragenodeAccounting struct {
	// contains filtered or unexported fields
}

StoragenodeAccounting implements the accounting/db StoragenodeAccounting interface.

func (*StoragenodeAccounting) ArchiveRollupsBefore added in v1.22.1 

func (db *StoragenodeAccounting) ArchiveRollupsBefore(ctx context.Context, before time.Time, batchSize int) (nodeRollupsDeleted int, err error)

ArchiveRollupsBefore archives rollups older than a given time.

func (*StoragenodeAccounting) DeleteTalliesBefore added in v0.12.0 

func (db *StoragenodeAccounting) DeleteTalliesBefore(ctx context.Context, latestRollup time.Time, batchSize int) (err error)

DeleteTalliesBefore deletes all raw tallies prior to some time.

func (*StoragenodeAccounting) GetArchivedRollupsSince added in v1.22.1 

func (db *StoragenodeAccounting) GetArchivedRollupsSince(ctx context.Context, since time.Time) (bwRollups []accounting.StoragenodeBandwidthRollup, err error)

GetArchivedRollupsSince retrieves all archived bandwidth rollup records since a given time.

func (*StoragenodeAccounting) GetBandwidthSince added in v0.12.0 

func (db *StoragenodeAccounting) GetBandwidthSince(ctx context.Context, latestRollup time.Time,
	cb func(context.Context, *accounting.StoragenodeBandwidthRollup) error) (err error)

GetBandwidthSince retrieves all storagenode_bandwidth_rollup entires since latestRollup.

func (*StoragenodeAccounting) GetRollupsSince added in v1.22.1 

func (db *StoragenodeAccounting) GetRollupsSince(ctx context.Context, since time.Time) (bwRollups []accounting.StoragenodeBandwidthRollup, err error)

GetRollupsSince retrieves all archived bandwidth rollup records since a given time.

func (*StoragenodeAccounting) GetTallies added in v0.12.0 

func (db *StoragenodeAccounting) GetTallies(ctx context.Context) (_ []*accounting.StoragenodeStorageTally, err error)

GetTallies retrieves all raw tallies.

func (*StoragenodeAccounting) GetTalliesSince added in v0.12.0 

func (db *StoragenodeAccounting) GetTalliesSince(ctx context.Context, latestRollup time.Time) (_ []*accounting.StoragenodeStorageTally, err error)

GetTalliesSince retrieves all raw tallies since latestRollup.

func (*StoragenodeAccounting) LastTimestamp added in v0.12.0 

func (db *StoragenodeAccounting) LastTimestamp(ctx context.Context, timestampType string) (_ time.Time, err error)

LastTimestamp records the greatest last tallied time.

func (*StoragenodeAccounting) QueryPaymentInfo added in v0.12.0 

func (db *StoragenodeAccounting) QueryPaymentInfo(ctx context.Context, start time.Time, end time.Time) (_ []accounting.NodePaymentInfo, err error)

QueryPaymentInfo queries Overlay, Accounting Rollup on nodeID.

func (*StoragenodeAccounting) QueryStorageNodePeriodUsage added in v1.1.1 

func (db *StoragenodeAccounting) QueryStorageNodePeriodUsage(ctx context.Context, period compensation.Period) (_ []accounting.StorageNodePeriodUsage, err error)

QueryStorageNodePeriodUsage returns usage invoices for nodes for a compensation period.

func (*StoragenodeAccounting) QueryStorageNodeUsage added in v0.18.0 

func (db *StoragenodeAccounting) QueryStorageNodeUsage(ctx context.Context, nodeID storj.NodeID, start time.Time, end time.Time) (_ []accounting.StorageNodeUsage, err error)

QueryStorageNodeUsage returns slice of StorageNodeUsage for given period.

func (*StoragenodeAccounting) SaveRollup added in v0.12.0 

func (db *StoragenodeAccounting) SaveRollup(ctx context.Context, latestRollup time.Time, stats accounting.RollupStats) (err error)

SaveRollup records raw tallies of at rest data to the database.

func (*StoragenodeAccounting) SaveTallies added in v0.12.0 

func (db *StoragenodeAccounting) SaveTallies(ctx context.Context, latestTally time.Time, nodeIDs []storj.NodeID, totals []float64) (err error)

SaveTallies records raw tallies of at rest data to the database.

type StripeSignupInfo added in v1.108.1 

type StripeSignupInfo struct {
	ID              uuid.UUID
	Email           string
	SignupPromoCode string
}

StripeSignupInfo is used in generateStripeCustomer information.

Source Files

View all Source files

Directories

Path Synopsis
dbx
gen

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.