Documentation ¶
Index ¶
- Constants
- Variables
- func CreatePath(ctx context.Context, projectID uuid.UUID, segmentIndex int64, ...) (_ storj.Path, err error)
- type APIKeys
- type BucketsDB
- type Config
- type Containment
- type Endpoint
- func (endpoint *Endpoint) Batch(ctx context.Context, req *pb.BatchRequest) (resp *pb.BatchResponse, err error)
- func (endpoint *Endpoint) BeginDeleteObject(ctx context.Context, req *pb.ObjectBeginDeleteRequest) (resp *pb.ObjectBeginDeleteResponse, err error)
- func (endpoint *Endpoint) BeginDeleteSegment(ctx context.Context, req *pb.SegmentBeginDeleteRequest) (resp *pb.SegmentBeginDeleteResponse, err error)
- func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRequest) (resp *pb.ObjectBeginResponse, err error)
- func (endpoint *Endpoint) BeginSegment(ctx context.Context, req *pb.SegmentBeginRequest) (resp *pb.SegmentBeginResponse, err error)
- func (endpoint *Endpoint) Close() error
- func (endpoint *Endpoint) CommitObject(ctx context.Context, req *pb.ObjectCommitRequest) (resp *pb.ObjectCommitResponse, err error)
- func (endpoint *Endpoint) CommitSegment(ctx context.Context, req *pb.SegmentCommitRequest) (resp *pb.SegmentCommitResponse, err error)
- func (endpoint *Endpoint) CommitSegmentOld(ctx context.Context, req *pb.SegmentCommitRequestOld) (resp *pb.SegmentCommitResponseOld, err error)
- func (endpoint *Endpoint) CreateBucket(ctx context.Context, req *pb.BucketCreateRequest) (resp *pb.BucketCreateResponse, err error)
- func (endpoint *Endpoint) CreateSegmentOld(ctx context.Context, req *pb.SegmentWriteRequestOld) (resp *pb.SegmentWriteResponseOld, err error)
- func (endpoint *Endpoint) DeleteBucket(ctx context.Context, req *pb.BucketDeleteRequest) (resp *pb.BucketDeleteResponse, err error)
- func (endpoint *Endpoint) DeleteSegmentOld(ctx context.Context, req *pb.SegmentDeleteRequestOld) (resp *pb.SegmentDeleteResponseOld, err error)
- func (endpoint *Endpoint) DownloadSegment(ctx context.Context, req *pb.SegmentDownloadRequest) (resp *pb.SegmentDownloadResponse, err error)
- func (endpoint *Endpoint) DownloadSegmentOld(ctx context.Context, req *pb.SegmentDownloadRequestOld) (resp *pb.SegmentDownloadResponseOld, err error)
- func (endpoint *Endpoint) FinishDeleteObject(ctx context.Context, req *pb.ObjectFinishDeleteRequest) (resp *pb.ObjectFinishDeleteResponse, err error)
- func (endpoint *Endpoint) FinishDeleteSegment(ctx context.Context, req *pb.SegmentFinishDeleteRequest) (resp *pb.SegmentFinishDeleteResponse, err error)
- func (endpoint *Endpoint) GetBucket(ctx context.Context, req *pb.BucketGetRequest) (resp *pb.BucketGetResponse, err error)
- func (endpoint *Endpoint) GetObject(ctx context.Context, req *pb.ObjectGetRequest) (resp *pb.ObjectGetResponse, err error)
- func (endpoint *Endpoint) ListBuckets(ctx context.Context, req *pb.BucketListRequest) (resp *pb.BucketListResponse, err error)
- func (endpoint *Endpoint) ListObjects(ctx context.Context, req *pb.ObjectListRequest) (resp *pb.ObjectListResponse, err error)
- func (endpoint *Endpoint) ListSegments(ctx context.Context, req *pb.SegmentListRequest) (resp *pb.SegmentListResponse, err error)
- func (endpoint *Endpoint) ListSegmentsOld(ctx context.Context, req *pb.ListSegmentsRequestOld) (resp *pb.ListSegmentsResponseOld, err error)
- func (endpoint *Endpoint) MakeInlineSegment(ctx context.Context, req *pb.SegmentMakeInlineRequest) (resp *pb.SegmentMakeInlineResponse, err error)
- func (endpoint *Endpoint) ProjectInfo(ctx context.Context, req *pb.ProjectInfoRequest) (_ *pb.ProjectInfoResponse, err error)
- func (endpoint *Endpoint) SegmentInfoOld(ctx context.Context, req *pb.SegmentInfoRequestOld) (resp *pb.SegmentInfoResponseOld, err error)
- func (endpoint *Endpoint) SetAttributionOld(ctx context.Context, req *pb.SetAttributionRequestOld) (_ *pb.SetAttributionResponseOld, err error)
- func (endpoint *Endpoint) SetBucketAttribution(ctx context.Context, req *pb.BucketSetAttributionRequest) (resp *pb.BucketSetAttributionResponse, err error)
- type Loop
- type LoopConfig
- type Observer
- type PointerDB
- type RSConfig
- type Revocations
- type ScopedPath
- type Service
- func (s *Service) CreateBucket(ctx context.Context, bucket storj.Bucket) (_ storj.Bucket, err error)
- func (s *Service) Delete(ctx context.Context, path string) (err error)
- func (s *Service) DeleteBucket(ctx context.Context, bucketName []byte, projectID uuid.UUID) (err error)
- func (s *Service) Get(ctx context.Context, path string) (pointer *pb.Pointer, err error)
- func (s *Service) GetBucket(ctx context.Context, bucketName []byte, projectID uuid.UUID) (_ storj.Bucket, err error)
- func (s *Service) Iterate(ctx context.Context, prefix string, first string, recurse bool, reverse bool, ...) (err error)
- func (s *Service) List(ctx context.Context, prefix string, startAfter string, endBefore string, ...) (items []*pb.ListResponse_Item, more bool, err error)
- func (s *Service) ListBuckets(ctx context.Context, projectID uuid.UUID, listOpts storj.BucketListOptions, ...) (bucketList storj.BucketList, err error)
- func (s *Service) Put(ctx context.Context, path string, pointer *pb.Pointer) (err error)
- func (s *Service) UpdateBucket(ctx context.Context, bucket storj.Bucket) (_ storj.Bucket, err error)
- func (s *Service) UpdatePieces(ctx context.Context, path string, ref *pb.Pointer, ...) (pointer *pb.Pointer, err error)
- type TTLItem
Constants ¶
const (
// BoltPointerBucket is the string representing the bucket used for `PointerEntries` in BoltDB
BoltPointerBucket = "pointers"
)
Variables ¶
var ( // LoopError is a standard error class for this component. LoopError = errs.Class("metainfo loop error") // LoopClosedError is a loop closed error LoopClosedError = LoopError.New("loop closed") )
var ( // Error general metainfo error Error = errs.Class("metainfo error") )
Functions ¶
Types ¶
type BucketsDB ¶ added in v0.15.0
type BucketsDB interface { // Create creates a new bucket CreateBucket(ctx context.Context, bucket storj.Bucket) (_ storj.Bucket, err error) // Get returns an existing bucket GetBucket(ctx context.Context, bucketName []byte, projectID uuid.UUID) (bucket storj.Bucket, err error) // UpdateBucket updates an existing bucket UpdateBucket(ctx context.Context, bucket storj.Bucket) (_ storj.Bucket, err error) // Delete deletes a bucket DeleteBucket(ctx context.Context, bucketName []byte, projectID uuid.UUID) (err error) // List returns all buckets for a project ListBuckets(ctx context.Context, projectID uuid.UUID, listOpts storj.BucketListOptions, allowedBuckets macaroon.AllowedBuckets) (bucketList storj.BucketList, err error) }
BucketsDB is the interface for the database to interact with buckets
architecture: Database
type Config ¶ added in v0.11.0
type Config struct { DatabaseURL string `help:"the database connection string to use" releaseDefault:"postgres://" devDefault:"bolt://$CONFDIR/pointerdb.db"` MinRemoteSegmentSize memory.Size `default:"1240" help:"minimum remote segment size"` MaxInlineSegmentSize memory.Size `default:"8000" help:"maximum inline segment size"` Overlay bool `default:"true" help:"toggle flag if overlay is enabled"` RS RSConfig `help:"redundancy scheme configuration"` Loop LoopConfig `help:"metainfo loop configuration"` }
Config is a configuration struct that is everything you need to start a metainfo
type Containment ¶ added in v0.12.0
Containment is a copy/paste of containment interface to avoid import cycle error
architecture: Database
type Endpoint ¶
type Endpoint struct {
// contains filtered or unexported fields
}
Endpoint metainfo endpoint
architecture: Endpoint
func NewEndpoint ¶
func NewEndpoint(log *zap.Logger, metainfo *Service, orders *orders.Service, cache *overlay.Service, partnerinfo attribution.DB, containment Containment, apiKeys APIKeys, projectUsage *accounting.ProjectUsage, rsConfig RSConfig, satellite signing.Signer) *Endpoint
NewEndpoint creates new metainfo endpoint instance
func (*Endpoint) Batch ¶ added in v0.17.0
func (endpoint *Endpoint) Batch(ctx context.Context, req *pb.BatchRequest) (resp *pb.BatchResponse, err error)
Batch handle requests sent in batch
func (*Endpoint) BeginDeleteObject ¶ added in v0.16.0
func (endpoint *Endpoint) BeginDeleteObject(ctx context.Context, req *pb.ObjectBeginDeleteRequest) (resp *pb.ObjectBeginDeleteResponse, err error)
BeginDeleteObject begins object deletion process
func (*Endpoint) BeginDeleteSegment ¶ added in v0.16.0
func (endpoint *Endpoint) BeginDeleteSegment(ctx context.Context, req *pb.SegmentBeginDeleteRequest) (resp *pb.SegmentBeginDeleteResponse, err error)
BeginDeleteSegment begins segment deletion process
func (*Endpoint) BeginObject ¶ added in v0.16.0
func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRequest) (resp *pb.ObjectBeginResponse, err error)
BeginObject begins object
func (*Endpoint) BeginSegment ¶ added in v0.16.0
func (endpoint *Endpoint) BeginSegment(ctx context.Context, req *pb.SegmentBeginRequest) (resp *pb.SegmentBeginResponse, err error)
BeginSegment begins segment uploading
func (*Endpoint) CommitObject ¶ added in v0.16.0
func (endpoint *Endpoint) CommitObject(ctx context.Context, req *pb.ObjectCommitRequest) (resp *pb.ObjectCommitResponse, err error)
CommitObject commits object when all segments are also committed
func (*Endpoint) CommitSegment ¶
func (endpoint *Endpoint) CommitSegment(ctx context.Context, req *pb.SegmentCommitRequest) (resp *pb.SegmentCommitResponse, err error)
CommitSegment commits segment after uploading
func (*Endpoint) CommitSegmentOld ¶ added in v0.15.0
func (endpoint *Endpoint) CommitSegmentOld(ctx context.Context, req *pb.SegmentCommitRequestOld) (resp *pb.SegmentCommitResponseOld, err error)
CommitSegmentOld commits segment metadata
func (*Endpoint) CreateBucket ¶ added in v0.15.0
func (endpoint *Endpoint) CreateBucket(ctx context.Context, req *pb.BucketCreateRequest) (resp *pb.BucketCreateResponse, err error)
CreateBucket creates a new bucket
func (*Endpoint) CreateSegmentOld ¶ added in v0.15.0
func (endpoint *Endpoint) CreateSegmentOld(ctx context.Context, req *pb.SegmentWriteRequestOld) (resp *pb.SegmentWriteResponseOld, err error)
CreateSegmentOld will generate requested number of OrderLimit with coresponding node addresses for them
func (*Endpoint) DeleteBucket ¶ added in v0.15.0
func (endpoint *Endpoint) DeleteBucket(ctx context.Context, req *pb.BucketDeleteRequest) (resp *pb.BucketDeleteResponse, err error)
DeleteBucket deletes a bucket
func (*Endpoint) DeleteSegmentOld ¶ added in v0.15.0
func (endpoint *Endpoint) DeleteSegmentOld(ctx context.Context, req *pb.SegmentDeleteRequestOld) (resp *pb.SegmentDeleteResponseOld, err error)
DeleteSegmentOld deletes segment metadata from satellite and returns OrderLimit array to remove them from storage node
func (*Endpoint) DownloadSegment ¶
func (endpoint *Endpoint) DownloadSegment(ctx context.Context, req *pb.SegmentDownloadRequest) (resp *pb.SegmentDownloadResponse, err error)
DownloadSegment returns data necessary to download segment
func (*Endpoint) DownloadSegmentOld ¶ added in v0.15.0
func (endpoint *Endpoint) DownloadSegmentOld(ctx context.Context, req *pb.SegmentDownloadRequestOld) (resp *pb.SegmentDownloadResponseOld, err error)
DownloadSegmentOld gets Pointer incase of INLINE data or list of OrderLimit necessary to download remote data
func (*Endpoint) FinishDeleteObject ¶ added in v0.16.0
func (endpoint *Endpoint) FinishDeleteObject(ctx context.Context, req *pb.ObjectFinishDeleteRequest) (resp *pb.ObjectFinishDeleteResponse, err error)
FinishDeleteObject finishes object deletion
func (*Endpoint) FinishDeleteSegment ¶ added in v0.16.0
func (endpoint *Endpoint) FinishDeleteSegment(ctx context.Context, req *pb.SegmentFinishDeleteRequest) (resp *pb.SegmentFinishDeleteResponse, err error)
FinishDeleteSegment finishes segment deletion process
func (*Endpoint) GetBucket ¶ added in v0.15.0
func (endpoint *Endpoint) GetBucket(ctx context.Context, req *pb.BucketGetRequest) (resp *pb.BucketGetResponse, err error)
GetBucket returns a bucket
func (*Endpoint) GetObject ¶ added in v0.16.0
func (endpoint *Endpoint) GetObject(ctx context.Context, req *pb.ObjectGetRequest) (resp *pb.ObjectGetResponse, err error)
GetObject gets single object
func (*Endpoint) ListBuckets ¶ added in v0.15.0
func (endpoint *Endpoint) ListBuckets(ctx context.Context, req *pb.BucketListRequest) (resp *pb.BucketListResponse, err error)
ListBuckets returns buckets in a project where the bucket name matches the request cursor
func (*Endpoint) ListObjects ¶ added in v0.16.0
func (endpoint *Endpoint) ListObjects(ctx context.Context, req *pb.ObjectListRequest) (resp *pb.ObjectListResponse, err error)
ListObjects list objects according to specific parameters
func (*Endpoint) ListSegments ¶
func (endpoint *Endpoint) ListSegments(ctx context.Context, req *pb.SegmentListRequest) (resp *pb.SegmentListResponse, err error)
ListSegments list object segments
func (*Endpoint) ListSegmentsOld ¶ added in v0.15.0
func (endpoint *Endpoint) ListSegmentsOld(ctx context.Context, req *pb.ListSegmentsRequestOld) (resp *pb.ListSegmentsResponseOld, err error)
ListSegmentsOld returns all Path keys in the Pointers bucket
func (*Endpoint) MakeInlineSegment ¶ added in v0.16.0
func (endpoint *Endpoint) MakeInlineSegment(ctx context.Context, req *pb.SegmentMakeInlineRequest) (resp *pb.SegmentMakeInlineResponse, err error)
MakeInlineSegment makes inline segment on satellite
func (*Endpoint) ProjectInfo ¶ added in v0.14.0
func (endpoint *Endpoint) ProjectInfo(ctx context.Context, req *pb.ProjectInfoRequest) (_ *pb.ProjectInfoResponse, err error)
ProjectInfo returns allowed ProjectInfo for the provided API key
func (*Endpoint) SegmentInfoOld ¶ added in v0.15.0
func (endpoint *Endpoint) SegmentInfoOld(ctx context.Context, req *pb.SegmentInfoRequestOld) (resp *pb.SegmentInfoResponseOld, err error)
SegmentInfoOld returns segment metadata info
func (*Endpoint) SetAttributionOld ¶ added in v0.15.0
func (endpoint *Endpoint) SetAttributionOld(ctx context.Context, req *pb.SetAttributionRequestOld) (_ *pb.SetAttributionResponseOld, err error)
SetAttributionOld tries to add attribution to the bucket.
func (*Endpoint) SetBucketAttribution ¶ added in v0.15.0
func (endpoint *Endpoint) SetBucketAttribution(ctx context.Context, req *pb.BucketSetAttributionRequest) (resp *pb.BucketSetAttributionResponse, err error)
SetBucketAttribution sets the bucket attribution.
type Loop ¶ added in v0.16.0
type Loop struct {
// contains filtered or unexported fields
}
Loop is a metainfo loop service.
architecture: Service
func NewLoop ¶ added in v0.16.0
func NewLoop(config LoopConfig, metainfo *Service) *Loop
NewLoop creates a new metainfo loop service.
func (*Loop) Join ¶ added in v0.16.0
Join will join the looper for one full cycle until completion and then returns. On ctx cancel the observer will return without completely finishing. Only on full complete iteration it will return nil. Safe to be called concurrently.
type LoopConfig ¶ added in v0.16.0
type LoopConfig struct {
CoalesceDuration time.Duration `help:"how long to wait for new observers before starting iteration" releaseDefault:"5s" devDefault:"5s"`
}
LoopConfig contains configurable values for the metainfo loop.
type Observer ¶ added in v0.16.0
type Observer interface { Object(context.Context, ScopedPath, *pb.Pointer) error RemoteSegment(context.Context, ScopedPath, *pb.Pointer) error InlineSegment(context.Context, ScopedPath, *pb.Pointer) error }
Observer is an interface defining an observer that can subscribe to the metainfo loop.
architecture: Observer
type PointerDB ¶ added in v0.21.0
type PointerDB interface { storage.KeyValueStore }
PointerDB stores pointers.
architecture: Database
type RSConfig ¶ added in v0.14.0
type RSConfig struct { MaxSegmentSize memory.Size `help:"maximum segment size" default:"64MiB"` MaxBufferMem memory.Size `help:"maximum buffer memory to be allocated for read buffers" default:"4MiB"` MinThreshold int `help:"the minimum pieces required to recover a segment. k." releaseDefault:"29" devDefault:"4"` RepairThreshold int `help:"the minimum safe pieces before a repair is triggered. m." releaseDefault:"35" devDefault:"6"` SuccessThreshold int `help:"the desired total pieces for a segment. o." releaseDefault:"80" devDefault:"8"` MaxThreshold int `help:"the largest amount of pieces to encode to. n." releaseDefault:"130" devDefault:"10"` Validate bool `help:"validate redundancy scheme configuration" default:"true"` }
RSConfig is a configuration struct that keeps details about default redundancy strategy information
type Revocations ¶ added in v0.12.0
type Revocations interface {
GetByProjectID(ctx context.Context, projectID uuid.UUID) ([][]byte, error)
}
Revocations is the revocations store methods used by the endpoint
architecture: Database
type ScopedPath ¶ added in v0.21.0
type ScopedPath struct { ProjectID uuid.UUID ProjectIDString string BucketName string // Raw is the same path as pointerDB is using. Raw storj.Path }
ScopedPath contains full expanded information about the path
type Service ¶ added in v0.11.0
type Service struct { DB PointerDB // contains filtered or unexported fields }
Service structure
architecture: Service
func NewService ¶ added in v0.11.0
NewService creates new metainfo service
func (*Service) CreateBucket ¶ added in v0.15.0
func (s *Service) CreateBucket(ctx context.Context, bucket storj.Bucket) (_ storj.Bucket, err error)
CreateBucket creates a new bucket in the buckets db
func (*Service) DeleteBucket ¶ added in v0.15.0
func (s *Service) DeleteBucket(ctx context.Context, bucketName []byte, projectID uuid.UUID) (err error)
DeleteBucket deletes a bucket from the bucekts db
func (*Service) GetBucket ¶ added in v0.15.0
func (s *Service) GetBucket(ctx context.Context, bucketName []byte, projectID uuid.UUID) (_ storj.Bucket, err error)
GetBucket returns an existing bucket in the buckets db
func (*Service) Iterate ¶ added in v0.11.0
func (s *Service) Iterate(ctx context.Context, prefix string, first string, recurse bool, reverse bool, f func(context.Context, storage.Iterator) error) (err error)
Iterate iterates over items in db
func (*Service) List ¶ added in v0.11.0
func (s *Service) List(ctx context.Context, prefix string, startAfter string, endBefore string, recursive bool, limit int32, metaFlags uint32) (items []*pb.ListResponse_Item, more bool, err error)
List returns all Path keys in the pointers bucket
func (*Service) ListBuckets ¶ added in v0.15.0
func (s *Service) ListBuckets(ctx context.Context, projectID uuid.UUID, listOpts storj.BucketListOptions, allowedBuckets macaroon.AllowedBuckets) (bucketList storj.BucketList, err error)
ListBuckets returns a list of buckets for a project
func (*Service) UpdateBucket ¶ added in v0.16.0
func (s *Service) UpdateBucket(ctx context.Context, bucket storj.Bucket) (_ storj.Bucket, err error)
UpdateBucket returns an updated bucket in the buckets db
func (*Service) UpdatePieces ¶ added in v0.16.0
func (s *Service) UpdatePieces(ctx context.Context, path string, ref *pb.Pointer, toAdd, toRemove []*pb.RemotePiece) (pointer *pb.Pointer, err error)
UpdatePieces atomically adds toAdd pieces and removes toRemove pieces from the pointer under path. ref is the pointer that caller received via Get prior to calling this method.
It will first check if the pointer has been deleted or replaced. Then it will remove the toRemove pieces and then it will add the toAdd pieces. Replacing the node ID and the hash of a piece can be done by adding the piece to both toAdd and toRemove.