metainfo

package
v1.25.5-rc-multipart Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 30, 2021 License: AGPL-3.0 Imports: 47 Imported by: 1

Documentation

Index

Constants

View Source 
const (
	// BoltPointerBucket is the string representing the bucket used for `PointerEntries` in BoltDB.
	BoltPointerBucket = "pointers"
)

Variables

View Source 
var (
	// LoopError is a standard error class for this component.
	LoopError = errs.Class("metainfo loop error")
	// LoopClosedError is a loop closed error.
	LoopClosedError = LoopError.New("loop closed")
)
View Source 
var (

	// Error general metainfo error.
	Error = errs.Class("metainfo error")
	// ErrNodeAlreadyExists pointer already has a piece for a node err.
	ErrNodeAlreadyExists = errs.Class("metainfo error: node already exists")
)
View Source 
var (
	// ErrBucketNotEmpty is returned when bucket is required to be empty for an operation.
	ErrBucketNotEmpty = errs.Class("bucket not empty")
)

Functions

func CreatePath 

func CreatePath(ctx context.Context, projectID uuid.UUID, segmentIndex uint32, bucket, path []byte) (_ metabase.SegmentLocation, err error)

CreatePath creates a segment key.

func EncodeSegmentID added in v1.17.1 

func EncodeSegmentID(ctx context.Context, segmentID *internalpb.SegmentID) (_ []byte, err error)

EncodeSegmentID encodes segment ID into bytes for signing.

func EncodeStreamID added in v1.17.1 

func EncodeStreamID(ctx context.Context, streamID *internalpb.StreamID) (_ []byte, err error)

EncodeStreamID encodes stream ID into bytes for signing.

func SignSegmentID added in v1.17.1 

func SignSegmentID(ctx context.Context, signer signing.Signer, unsigned *internalpb.SegmentID) (_ *internalpb.SegmentID, err error)

SignSegmentID signs the segment ID using the specified signer. Signer is a satellite.

func SignStreamID added in v1.17.1 

func SignStreamID(ctx context.Context, signer signing.Signer, unsigned *internalpb.StreamID) (_ *internalpb.StreamID, err error)

SignStreamID signs the stream ID using the specified signer. Signer is a satellite.

func VerifySegmentID added in v1.17.1 

func VerifySegmentID(ctx context.Context, satellite signing.Signee, signed *internalpb.SegmentID) (err error)

VerifySegmentID verifies that the signature inside segment ID belongs to the satellite.

func VerifyStreamID added in v1.17.1 

func VerifyStreamID(ctx context.Context, satellite signing.Signer, signed *internalpb.StreamID) (err error)

VerifyStreamID verifies that the signature inside stream ID belongs to the satellite.

Types

type APIKeys 

type APIKeys interface {
	GetByHead(ctx context.Context, head []byte) (*console.APIKeyInfo, error)
}

APIKeys is api keys store methods used by endpoint.

architecture: Database

type BucketsDB added in v0.15.0 

type BucketsDB interface {
	// Create creates a new bucket
	CreateBucket(ctx context.Context, bucket storj.Bucket) (_ storj.Bucket, err error)
	// Get returns an existing bucket
	GetBucket(ctx context.Context, bucketName []byte, projectID uuid.UUID) (bucket storj.Bucket, err error)
	// GetBucketID returns an existing bucket id.
	GetBucketID(ctx context.Context, bucket metabase.BucketLocation) (id uuid.UUID, err error)
	// UpdateBucket updates an existing bucket
	UpdateBucket(ctx context.Context, bucket storj.Bucket) (_ storj.Bucket, err error)
	// Delete deletes a bucket
	DeleteBucket(ctx context.Context, bucketName []byte, projectID uuid.UUID) (err error)
	// List returns all buckets for a project
	ListBuckets(ctx context.Context, projectID uuid.UUID, listOpts storj.BucketListOptions, allowedBuckets macaroon.AllowedBuckets) (bucketList storj.BucketList, err error)
	// CountBuckets returns the number of buckets a project currently has
	CountBuckets(ctx context.Context, projectID uuid.UUID) (int, error)
}

BucketsDB is the interface for the database to interact with buckets.

architecture: Database

type Config added in v0.11.0 

type Config struct {
	DatabaseURL          string               `help:"the database connection string to use" default:"postgres://"`
	MinRemoteSegmentSize memory.Size          `default:"1240" help:"minimum remote segment size"`
	MaxInlineSegmentSize memory.Size          `default:"4KiB" help:"maximum inline segment size"`
	MaxSegmentSize       memory.Size          `default:"64MiB" help:"maximum segment size"`
	MaxMetadataSize      memory.Size          `default:"2KiB" help:"maximum segment metadata size"`
	MaxCommitInterval    time.Duration        `default:"48h" help:"maximum time allowed to pass between creating and committing a segment"`
	Overlay              bool                 `default:"true" help:"toggle flag if overlay is enabled"`
	RS                   RSConfig             `` /* 132-byte string literal not displayed */
	Loop                 LoopConfig           `help:"loop configuration"`
	RateLimiter          RateLimiterConfig    `help:"rate limiter configuration"`
	ProjectLimits        ProjectLimitConfig   `help:"project limit configuration"`
	PieceDeletion        piecedeletion.Config `help:"piece deletion configuration"`
}

Config is a configuration struct that is everything you need to start a metainfo.

type Endpoint 

type Endpoint struct {
	// contains filtered or unexported fields
}

Endpoint metainfo endpoint.

architecture: Endpoint

func NewEndpoint 

func NewEndpoint(log *zap.Logger, metainfo *Service, deletePieces *piecedeletion.Service,
	orders *orders.Service, cache *overlay.Service, attributions attribution.DB,
	partners *rewards.PartnersService, peerIdentities overlay.PeerIdentities,
	apiKeys APIKeys, projectUsage *accounting.Service, projects console.Projects,
	satellite signing.Signer, revocations revocation.DB, config Config) (*Endpoint, error)

NewEndpoint creates new metainfo endpoint instance.

func (*Endpoint) Batch added in v0.17.0 

func (endpoint *Endpoint) Batch(ctx context.Context, req *pb.BatchRequest) (resp *pb.BatchResponse, err error)

Batch handle requests sent in batch.

func (*Endpoint) BeginDeleteObject added in v0.16.0 

func (endpoint *Endpoint) BeginDeleteObject(ctx context.Context, req *pb.ObjectBeginDeleteRequest) (resp *pb.ObjectBeginDeleteResponse, err error)

BeginDeleteObject begins object deletion process.

func (*Endpoint) BeginDeleteSegment added in v0.16.0 

func (endpoint *Endpoint) BeginDeleteSegment(ctx context.Context, req *pb.SegmentBeginDeleteRequest) (resp *pb.SegmentBeginDeleteResponse, err error)

BeginDeleteSegment begins segment deletion process.

func (*Endpoint) BeginObject added in v0.16.0 

func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRequest) (resp *pb.ObjectBeginResponse, err error)

BeginObject begins object.

func (*Endpoint) BeginSegment added in v0.16.0 

func (endpoint *Endpoint) BeginSegment(ctx context.Context, req *pb.SegmentBeginRequest) (resp *pb.SegmentBeginResponse, err error)

BeginSegment begins segment uploading.

func (*Endpoint) Close 

func (endpoint *Endpoint) Close() error

Close closes resources.

func (*Endpoint) CommitObject added in v0.16.0 

func (endpoint *Endpoint) CommitObject(ctx context.Context, req *pb.ObjectCommitRequest) (resp *pb.ObjectCommitResponse, err error)

CommitObject commits an object when all its segments have already been committed.

func (*Endpoint) CommitSegment 

func (endpoint *Endpoint) CommitSegment(ctx context.Context, req *pb.SegmentCommitRequest) (resp *pb.SegmentCommitResponse, err error)

CommitSegment commits segment after uploading.

func (*Endpoint) CountBuckets added in v1.9.1 

func (endpoint *Endpoint) CountBuckets(ctx context.Context, projectID uuid.UUID) (count int, err error)

CountBuckets returns the number of buckets a project currently has. TODO: add this to the uplink client side.

func (*Endpoint) CreateBucket added in v0.15.0 

func (endpoint *Endpoint) CreateBucket(ctx context.Context, req *pb.BucketCreateRequest) (resp *pb.BucketCreateResponse, err error)

CreateBucket creates a new bucket.

func (*Endpoint) DeleteBucket added in v0.15.0 

func (endpoint *Endpoint) DeleteBucket(ctx context.Context, req *pb.BucketDeleteRequest) (resp *pb.BucketDeleteResponse, err error)

DeleteBucket deletes a bucket.

func (*Endpoint) DeleteCommittedObject added in v1.26.2 

func (endpoint *Endpoint) DeleteCommittedObject(
	ctx context.Context, projectID uuid.UUID, bucket string, object metabase.ObjectKey,
) (deletedObjects []*pb.Object, err error)

DeleteCommittedObject deletes all the pieces of the storage nodes that belongs to the specified object.

NOTE: this method is exported for being able to individually test it without having import cycles.

func (*Endpoint) DeleteObjectAnyStatus added in v1.26.2 

func (endpoint *Endpoint) DeleteObjectAnyStatus(ctx context.Context, location metabase.ObjectLocation,
) (deletedObjects []*pb.Object, err error)

DeleteObjectAnyStatus deletes all the pieces of the storage nodes that belongs to the specified object.

NOTE: this method is exported for being able to individually test it without having import cycles.

func (*Endpoint) DeletePendingObject added in v1.26.2 

func (endpoint *Endpoint) DeletePendingObject(ctx context.Context, projectID uuid.UUID, bucket string, objectKey metabase.ObjectKey, version int32, streamID uuid.UUID) (deletedObjects []*pb.Object, err error)

DeletePendingObject deletes all the pieces of the storage nodes that belongs to the specified pending object.

NOTE: this method is exported for being able to individually test it without having import cycles.

func (*Endpoint) DownloadSegment 

func (endpoint *Endpoint) DownloadSegment(ctx context.Context, req *pb.SegmentDownloadRequest) (resp *pb.SegmentDownloadResponse, err error)

DownloadSegment returns data necessary to download segment.

func (*Endpoint) FinishDeleteObject added in v0.16.0 

func (endpoint *Endpoint) FinishDeleteObject(ctx context.Context, req *pb.ObjectFinishDeleteRequest) (resp *pb.ObjectFinishDeleteResponse, err error)

FinishDeleteObject finishes object deletion.

func (*Endpoint) FinishDeleteSegment added in v0.16.0 

func (endpoint *Endpoint) FinishDeleteSegment(ctx context.Context, req *pb.SegmentFinishDeleteRequest) (resp *pb.SegmentFinishDeleteResponse, err error)

FinishDeleteSegment finishes segment deletion process.

func (*Endpoint) GetBucket added in v0.15.0 

func (endpoint *Endpoint) GetBucket(ctx context.Context, req *pb.BucketGetRequest) (resp *pb.BucketGetResponse, err error)

GetBucket returns a bucket.

func (*Endpoint) GetObject added in v0.16.0 

func (endpoint *Endpoint) GetObject(ctx context.Context, req *pb.ObjectGetRequest) (resp *pb.ObjectGetResponse, err error)

GetObject gets single object metadata.

func (*Endpoint) GetObjectIPs added in v1.11.1 

func (endpoint *Endpoint) GetObjectIPs(ctx context.Context, req *pb.ObjectGetIPsRequest) (resp *pb.ObjectGetIPsResponse, err error)

GetObjectIPs returns the IP addresses of the nodes holding the pieces for the provided object. This is useful for knowing the locations of the pieces.

func (*Endpoint) GetPendingObjects added in v1.21.1 

func (endpoint *Endpoint) GetPendingObjects(ctx context.Context, req *pb.GetPendingObjectsRequest) (resp *pb.GetPendingObjectsResponse, err error)

GetPendingObjects get pending objects according to specific parameters.

func (*Endpoint) ListBuckets added in v0.15.0 

func (endpoint *Endpoint) ListBuckets(ctx context.Context, req *pb.BucketListRequest) (resp *pb.BucketListResponse, err error)

ListBuckets returns buckets in a project where the bucket name matches the request cursor.

func (*Endpoint) ListObjects added in v0.16.0 

func (endpoint *Endpoint) ListObjects(ctx context.Context, req *pb.ObjectListRequest) (resp *pb.ObjectListResponse, err error)

ListObjects list objects according to specific parameters.

func (*Endpoint) ListPendingObjectStreams added in v1.22.1 

func (endpoint *Endpoint) ListPendingObjectStreams(ctx context.Context, req *pb.ObjectListPendingStreamsRequest) (resp *pb.ObjectListPendingStreamsResponse, err error)

ListPendingObjectStreams list pending objects according to specific parameters.

func (*Endpoint) ListSegments 

func (endpoint *Endpoint) ListSegments(ctx context.Context, req *pb.SegmentListRequest) (resp *pb.SegmentListResponse, err error)

ListSegments list object segments.

func (*Endpoint) MakeInlineSegment added in v0.16.0 

func (endpoint *Endpoint) MakeInlineSegment(ctx context.Context, req *pb.SegmentMakeInlineRequest) (resp *pb.SegmentMakeInlineResponse, err error)

MakeInlineSegment makes inline segment on satellite.

func (*Endpoint) ProjectInfo added in v0.14.0 

func (endpoint *Endpoint) ProjectInfo(ctx context.Context, req *pb.ProjectInfoRequest) (_ *pb.ProjectInfoResponse, err error)

ProjectInfo returns allowed ProjectInfo for the provided API key.

func (*Endpoint) ResolvePartnerID added in v1.2.1 

func (endpoint *Endpoint) ResolvePartnerID(ctx context.Context, header *pb.RequestHeader) (uuid.UUID, error)

ResolvePartnerID returns partnerIDBytes as parsed or UUID corresponding to header.UserAgent. returns empty uuid when neither is defined.

func (*Endpoint) RevokeAPIKey added in v1.7.1 

func (endpoint *Endpoint) RevokeAPIKey(ctx context.Context, req *pb.RevokeAPIKeyRequest) (resp *pb.RevokeAPIKeyResponse, err error)

RevokeAPIKey handles requests to revoke an api key.

type Loop added in v0.16.0 

type Loop struct {
	// contains filtered or unexported fields
}

Loop is a metainfo loop service.

architecture: Service

func NewLoop added in v0.16.0 

func NewLoop(config LoopConfig, metabaseDB MetabaseDB) *Loop

NewLoop creates a new metainfo loop service.

func (*Loop) Close added in v0.22.0 

func (loop *Loop) Close() (err error)

Close closes the looping services.

func (*Loop) Join added in v0.16.0 

func (loop *Loop) Join(ctx context.Context, observers ...Observer) (err error)

Join will join the looper for one full cycle until completion and then returns. On ctx cancel the observer will return without completely finishing. Only on full complete iteration it will return nil. Safe to be called concurrently.

func (*Loop) Run added in v0.16.0 

func (loop *Loop) Run(ctx context.Context) (err error)

Run starts the looping service. It can only be called once, otherwise a panic will occur.

func (*Loop) RunOnce 

func (loop *Loop) RunOnce(ctx context.Context) (err error)

RunOnce goes through metainfo one time and sends information to observers.

It is not safe to call this concurrently with Run.

func (*Loop) Wait added in v0.16.0 

func (loop *Loop) Wait()

Wait waits for run to be finished. Safe to be called concurrently.

type LoopConfig added in v0.16.0 

type LoopConfig struct {
	CoalesceDuration time.Duration `help:"how long to wait for new observers before starting iteration" releaseDefault:"5s" devDefault:"5s"`
	RateLimit        float64       `help:"rate limit (default is 0 which is unlimited segments per second)" default:"0"`
	ListLimit        int           `help:"how many items to query in a batch" default:"2500"`
}

LoopConfig contains configurable values for the metainfo loop.

type MetabaseDB added in v1.26.2 

type MetabaseDB interface {
	io.Closer

	// MigrateToLatest migrates to latest schema version.
	MigrateToLatest(ctx context.Context) error
	// CheckVersion checks the database is the correct version
	CheckVersion(ctx context.Context) error

	// DeleteObjectAnyStatusAllVersions deletes all object versions.
	DeleteObjectAnyStatusAllVersions(ctx context.Context, opts metabase.DeleteObjectAnyStatusAllVersions) (result metabase.DeleteObjectResult, err error)
	// DeleteObjectsAllVersions deletes all versions of multiple objects from the same bucket.
	DeleteObjectsAllVersions(ctx context.Context, opts metabase.DeleteObjectsAllVersions) (result metabase.DeleteObjectResult, err error)
	// DeletePendingObject deletes a pending object.
	DeletePendingObject(ctx context.Context, opts metabase.DeletePendingObject) (result metabase.DeleteObjectResult, err error)
	// DeleteBucketObjects deletes all objects in the specified bucket.
	DeleteBucketObjects(ctx context.Context, opts metabase.DeleteBucketObjects) (deletedObjectCount int64, err error)
	// DeleteExpiredObjects deletes all objects that expired before expiredBefore.
	DeleteExpiredObjects(ctx context.Context, opts metabase.DeleteExpiredObjects) error
	// DeleteObjectLatestVersion deletes latest object version.
	DeleteObjectLatestVersion(ctx context.Context, opts metabase.DeleteObjectLatestVersion) (result metabase.DeleteObjectResult, err error)
	// BeginObjectExactVersion adds a pending object to the database, with specific version.
	BeginObjectExactVersion(ctx context.Context, opts metabase.BeginObjectExactVersion) (committed metabase.Object, err error)
	// CommitObject adds a pending object to the database.
	CommitObject(ctx context.Context, opts metabase.CommitObject) (object metabase.Object, err error)
	// BeginSegment verifies whether a new segment upload can be started.
	BeginSegment(ctx context.Context, opts metabase.BeginSegment) (err error)
	// CommitSegment commits segment to the database.
	CommitSegment(ctx context.Context, opts metabase.CommitSegment) (err error)
	// CommitInlineSegment commits inline segment to the database.
	CommitInlineSegment(ctx context.Context, opts metabase.CommitInlineSegment) (err error)
	// GetObjectLatestVersion returns object information for latest version.
	GetObjectLatestVersion(ctx context.Context, opts metabase.GetObjectLatestVersion) (_ metabase.Object, err error)
	// GetSegmentByLocation returns a information about segment on the specified location.
	GetSegmentByLocation(ctx context.Context, opts metabase.GetSegmentByLocation) (segment metabase.Segment, err error)
	// GetSegmentByPosition returns a information about segment which covers specified offset.
	GetSegmentByPosition(ctx context.Context, opts metabase.GetSegmentByPosition) (segment metabase.Segment, err error)
	// GetLatestObjectLastSegment returns an object last segment information.
	GetLatestObjectLastSegment(ctx context.Context, opts metabase.GetLatestObjectLastSegment) (segment metabase.Segment, err error)
	// ListSegments lists specified stream segments.
	ListSegments(ctx context.Context, opts metabase.ListSegments) (result metabase.ListSegmentsResult, err error)
	// IterateObjectsAllVersions iterates through all versions of all objects.
	IterateObjectsAllVersions(ctx context.Context, opts metabase.IterateObjects, fn func(context.Context, metabase.ObjectsIterator) error) (err error)
	// IterateObjectsAllVersionsWithStatus iterates through all versions of all objects with specified status.
	IterateObjectsAllVersionsWithStatus(ctx context.Context, opts metabase.IterateObjectsWithStatus, fn func(context.Context, metabase.ObjectsIterator) error) (err error)
	// IteratePendingObjectsByKey iterates through all StreamID for a given ObjectKey.
	IteratePendingObjectsByKey(ctx context.Context, opts metabase.IteratePendingObjectsByKey, fn func(context.Context, metabase.ObjectsIterator) error) (err error)
	// IterateLoopObjects iterates through all objects in metabase for metainfo loop purpose.
	IterateLoopObjects(ctx context.Context, opts metabase.IterateLoopObjects, fn func(context.Context, metabase.LoopObjectsIterator) error) (err error)
	// IterateLoopStreams iterates through all streams passed in as arguments.
	IterateLoopStreams(ctx context.Context, opts metabase.IterateLoopStreams, handleStream func(ctx context.Context, streamID uuid.UUID, next metabase.SegmentIterator) error) (err error)
	// BucketEmpty returns true if bucket does not contain objects (pending or committed).
	// This method doesn't check bucket existence.
	BucketEmpty(ctx context.Context, opts metabase.BucketEmpty) (empty bool, err error)
	// UpdateSegmentPieces updates pieces for specified segment. If provided old pieces won't match current database state update will fail.
	UpdateSegmentPieces(ctx context.Context, opts metabase.UpdateSegmentPieces) (err error)
	// EnsureNodeAliases ensures that the supplied node ID-s have a alias.
	// It's safe to concurrently try and create node ID-s for the same NodeID.
	EnsureNodeAliases(ctx context.Context, opts metabase.EnsureNodeAliases) (err error)
	// ListNodeAliases lists all node alias mappings.
	ListNodeAliases(ctx context.Context) (_ []metabase.NodeAliasEntry, err error)

	// TestingAllCommittedObjects gets all committed objects from bucket. Use only for testing purposes.
	TestingAllCommittedObjects(ctx context.Context, projectID uuid.UUID, bucketName string) (objects []metabase.ObjectEntry, err error)
	// TestingAllPendingObjects gets all pending objects from bucket. Use only for testing purposes.
	TestingAllPendingObjects(ctx context.Context, projectID uuid.UUID, bucketName string) (objects []metabase.ObjectEntry, err error)
	// TestingAllObjectSegments gets all segments for given object. Use only for testing purposes.
	TestingAllObjectSegments(ctx context.Context, objectLocation metabase.ObjectLocation) (segments []metabase.Segment, err error)
	// TestingAllObjects gets all objects. Use only for testing purposes.
	TestingAllObjects(ctx context.Context) (segments []metabase.Object, err error)
	// TestingAllSegments gets all segments. Use only for testing purposes.
	TestingAllSegments(ctx context.Context) (segments []metabase.Segment, err error)

	// InternalImplementation returns *metabase.DB.
	// TODO: remove.
	InternalImplementation() interface{}
}

MetabaseDB stores objects and segments.

func OpenMetabase added in v1.26.2 

func OpenMetabase(ctx context.Context, log *zap.Logger, dbURLString string) (db MetabaseDB, err error)

OpenMetabase returns database for storing objects and segments.

type NullObserver added in v0.33.3 

type NullObserver struct{}

NullObserver is an observer that does nothing. This is useful for joining and ensuring the metainfo loop runs once before you use a real observer.

func (NullObserver) InlineSegment added in v0.33.3 

func (NullObserver) InlineSegment(context.Context, *Segment) error

InlineSegment implements the Observer interface.

func (NullObserver) Object added in v0.33.3 

func (NullObserver) Object(context.Context, *Object) error

Object implements the Observer interface.

func (NullObserver) RemoteSegment added in v0.33.3 

func (NullObserver) RemoteSegment(context.Context, *Segment) error

RemoteSegment implements the Observer interface.

type Object added in v1.16.1 

type Object metabase.LoopObjectEntry

Object is the object info passed to Observer by metainfo loop.

func (*Object) Expired added in v1.16.1 

func (object *Object) Expired(now time.Time) bool

Expired checks if object expired relative to now.

type Observer added in v0.16.0 

type Observer interface {
	Object(context.Context, *Object) error
	RemoteSegment(context.Context, *Segment) error
	InlineSegment(context.Context, *Segment) error
}

Observer is an interface defining an observer that can subscribe to the metainfo loop.

architecture: Observer

type PointerDB added in v0.21.0 

type PointerDB interface {
	// MigrateToLatest migrates to latest schema version.
	MigrateToLatest(ctx context.Context) error

	storage.KeyValueStore
}

PointerDB stores pointers.

architecture: Database

func OpenStore added in v1.17.1 

func OpenStore(ctx context.Context, logger *zap.Logger, dbURLString string, app string) (db PointerDB, err error)

OpenStore returns database for storing pointer data.

type ProjectLimitConfig added in v1.9.1 

type ProjectLimitConfig struct {
	MaxBuckets          int         `help:"max bucket count for a project." default:"100"`
	DefaultMaxUsage     memory.Size `help:"the default storage usage limit" default:"500.00GB"`
	DefaultMaxBandwidth memory.Size `help:"the default bandwidth usage limit" default:"500.00GB"`
}

ProjectLimitConfig is a configuration struct for default project limits.

type RSConfig added in v0.14.0 

type RSConfig struct {
	ErasureShareSize memory.Size
	Min              int
	Repair           int
	Success          int
	Total            int
}

RSConfig is a configuration struct that keeps details about default redundancy strategy information.

Can be used as a flag.

func (*RSConfig) Set added in v1.17.1 

func (rs *RSConfig) Set(s string) error

Set sets the value from a string in the format k/m/o/n-size (min/repair/optimal/total-erasuresharesize).

func (*RSConfig) String added in v1.17.1 

func (rs *RSConfig) String() string

String is required for pflag.Value.

func (RSConfig) Type added in v1.17.1 

func (RSConfig) Type() string

Type implements pflag.Value.

type RateLimiterConfig added in v0.31.0 

type RateLimiterConfig struct {
	Enabled         bool          `help:"whether rate limiting is enabled." releaseDefault:"true" devDefault:"true"`
	Rate            float64       `help:"request rate per project per second." releaseDefault:"1000" devDefault:"100"`
	CacheCapacity   int           `help:"number of projects to cache." releaseDefault:"10000" devDefault:"10"`
	CacheExpiration time.Duration `help:"how long to cache the projects limiter." releaseDefault:"10m" devDefault:"10s"`
}

RateLimiterConfig is a configuration struct for endpoint rate limiting.

type Segment added in v1.16.1 

type Segment struct {
	Location       metabase.SegmentLocation // tally, repair, graceful exit, audit
	CreationDate   time.Time                // repair
	ExpirationDate time.Time                // tally, repair
	LastRepaired   time.Time                // repair

	metabase.LoopSegmentEntry
}

Segment is the segment info passed to Observer by metainfo loop.

func (*Segment) Expired added in v1.16.1 

func (segment *Segment) Expired(now time.Time) bool

Expired checks if segment is expired relative to now.

type Service added in v0.11.0 

type Service struct {
	// contains filtered or unexported fields
}

Service provides the metainfo service dependencies.

architecture: Service

func NewService added in v0.11.0 

func NewService(logger *zap.Logger, db PointerDB, bucketsDB BucketsDB, metabaseDB MetabaseDB) *Service

NewService creates new metainfo service.

func (*Service) CountBuckets added in v1.9.1 

func (s *Service) CountBuckets(ctx context.Context, projectID uuid.UUID) (count int, err error)

CountBuckets returns the number of buckets a project currently has.

func (*Service) CreateBucket added in v0.15.0 

func (s *Service) CreateBucket(ctx context.Context, bucket storj.Bucket) (_ storj.Bucket, err error)

CreateBucket creates a new bucket in the buckets db.

func (*Service) Delete added in v0.11.0 

func (s *Service) Delete(ctx context.Context, key metabase.SegmentKey, oldPointerBytes []byte) (err error)

Delete deletes a pointer bytes when it matches oldPointerBytes, otherwise it'll fail.

func (*Service) DeleteBucket added in v0.15.0 

func (s *Service) DeleteBucket(ctx context.Context, bucketName []byte, projectID uuid.UUID) (err error)

DeleteBucket deletes a bucket from the bucekts db.

func (*Service) Get added in v0.11.0 

func (s *Service) Get(ctx context.Context, key metabase.SegmentKey) (_ *pb.Pointer, err error)

Get gets decoded pointer from DB.

func (*Service) GetBucket added in v0.15.0 

func (s *Service) GetBucket(ctx context.Context, bucketName []byte, projectID uuid.UUID) (_ storj.Bucket, err error)

GetBucket returns an existing bucket in the buckets db.

func (*Service) GetItems added in v1.9.1 

func (s *Service) GetItems(ctx context.Context, keys []metabase.SegmentKey) (_ []*pb.Pointer, err error)

GetItems gets decoded pointers from DB. The return value is in the same order as the argument paths.

func (*Service) GetWithBytes added in v0.25.1 

func (s *Service) GetWithBytes(ctx context.Context, key metabase.SegmentKey) (pointerBytes []byte, pointer *pb.Pointer, err error)

GetWithBytes gets the protocol buffers encoded and decoded pointer from the DB.

func (*Service) IsBucketEmpty added in v0.34.1 

func (s *Service) IsBucketEmpty(ctx context.Context, projectID uuid.UUID, bucketName []byte) (bool, error)

IsBucketEmpty returns whether bucket is empty.

func (*Service) List added in v0.11.0 

func (s *Service) List(ctx context.Context, prefix metabase.SegmentKey, startAfter string, recursive bool, limit int32,
	metaFlags uint32) (items []*pb.ListResponse_Item, more bool, err error)

List returns all Path keys in the pointers bucket.

func (*Service) ListBuckets added in v0.15.0 

func (s *Service) ListBuckets(ctx context.Context, projectID uuid.UUID, listOpts storj.BucketListOptions, allowedBuckets macaroon.AllowedBuckets) (bucketList storj.BucketList, err error)

ListBuckets returns a list of buckets for a project.

func (*Service) Put added in v0.11.0 

func (s *Service) Put(ctx context.Context, key metabase.SegmentKey, pointer *pb.Pointer) (err error)

Put puts pointer to db under specific path.

func (*Service) UnsynchronizedDelete added in v0.25.1 

func (s *Service) UnsynchronizedDelete(ctx context.Context, key metabase.SegmentKey) (err error)

UnsynchronizedDelete deletes item from db without verifying whether the pointer has changed in the database.

func (*Service) UnsynchronizedGetDel added in v1.9.1 

func (s *Service) UnsynchronizedGetDel(ctx context.Context, keys []metabase.SegmentKey) ([]metabase.SegmentKey, []*pb.Pointer, error)

UnsynchronizedGetDel deletes items from db without verifying whether the pointers have changed in the database, and it returns deleted items.

func (*Service) UnsynchronizedPut added in v0.33.3 

func (s *Service) UnsynchronizedPut(ctx context.Context, key metabase.SegmentKey, pointer *pb.Pointer) (err error)

UnsynchronizedPut puts pointer to db under specific path without verifying for existing pointer under the same path.

func (*Service) UpdateBucket added in v0.16.0 

func (s *Service) UpdateBucket(ctx context.Context, bucket storj.Bucket) (_ storj.Bucket, err error)

UpdateBucket returns an updated bucket in the buckets db.

func (*Service) UpdatePieces added in v0.16.0 

func (s *Service) UpdatePieces(ctx context.Context, key metabase.SegmentKey, ref *pb.Pointer, toAdd, toRemove []*pb.RemotePiece) (pointer *pb.Pointer, err error)

UpdatePieces calls UpdatePiecesCheckDuplicates with checkDuplicates equal to false.

func (*Service) UpdatePiecesCheckDuplicates added in v0.26.0 

func (s *Service) UpdatePiecesCheckDuplicates(ctx context.Context, key metabase.SegmentKey, ref *pb.Pointer, toAdd, toRemove []*pb.RemotePiece, checkDuplicates bool) (pointer *pb.Pointer, err error)

UpdatePiecesCheckDuplicates atomically adds toAdd pieces and removes toRemove pieces from the pointer under path. ref is the pointer that caller received via Get prior to calling this method.

It will first check if the pointer has been deleted or replaced. Then if checkDuplicates is true it will return an error if the nodes to be added are already in the pointer. Then it will remove the toRemove pieces and then it will add the toAdd pieces. Replacing the node ID and the hash of a piece can be done by adding the piece to both toAdd and toRemove.

Source Files

View all Source files

Directories

Path Synopsis
Package expireddeletion contains the functions needed to run expired segment deletion The expireddeletion.expiredDeleter implements the metainfo loop Observer interface allowing us to subscribe to the loop to get information for every segment in the metainfo database.
 Package expireddeletion contains the functions needed to run expired segment deletion The expireddeletion.expiredDeleter implements the metainfo loop Observer interface allowing us to subscribe to the loop to get information for every segment in the metainfo database.
Package metabase implements storing objects and segements.
 Package metabase implements storing objects and segements.
Package piecedeletion implements service for deleting pieces that combines concurrent requests.
 Package piecedeletion implements service for deleting pieces that combines concurrent requests.
Package pointerverification implements verification of pointers.
 Package pointerverification implements verification of pointers.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.